@shushed/helpers 0.0.197 → 0.0.198-v2-20251104135028

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (234) hide show
  1. package/dist/cjs/dist-dereferenced/asset.js +4 -0
  2. package/dist/cjs/dist-dereferenced/category.js +4 -0
  3. package/dist/cjs/dist-dereferenced/country.js +4 -0
  4. package/dist/cjs/dist-dereferenced/currency.js +4 -0
  5. package/dist/cjs/dist-dereferenced/customer-segment.js +4 -0
  6. package/dist/cjs/dist-dereferenced/development-colour.js +4 -0
  7. package/dist/cjs/dist-dereferenced/index.js +69 -0
  8. package/dist/cjs/dist-dereferenced/marketing-preferences.js +4 -0
  9. package/dist/cjs/dist-dereferenced/messages/ean-change.js +4 -0
  10. package/dist/cjs/dist-dereferenced/messages/index.js +50 -0
  11. package/dist/cjs/dist-dereferenced/messages/order/delivered.js +4 -0
  12. package/dist/cjs/dist-dereferenced/messages/order/index.js +18 -0
  13. package/dist/cjs/dist-dereferenced/messages/order/new.js +4 -0
  14. package/dist/cjs/dist-dereferenced/messages/order/processed.js +4 -0
  15. package/dist/cjs/dist-dereferenced/messages/order/return-initiated.js +4 -0
  16. package/dist/cjs/dist-dereferenced/messages/order/returned.js +4 -0
  17. package/dist/cjs/dist-dereferenced/messages/order/shipped.js +4 -0
  18. package/dist/cjs/dist-dereferenced/messages/price-change.js +4 -0
  19. package/dist/cjs/dist-dereferenced/messages/product-category.js +4 -0
  20. package/dist/cjs/dist-dereferenced/messages/product-draft.js +4 -0
  21. package/dist/cjs/dist-dereferenced/messages/product.js +4 -0
  22. package/dist/cjs/dist-dereferenced/money.js +4 -0
  23. package/dist/cjs/dist-dereferenced/order/address.js +4 -0
  24. package/dist/cjs/dist-dereferenced/order/customer.js +4 -0
  25. package/dist/cjs/dist-dereferenced/order/index.js +51 -0
  26. package/dist/cjs/dist-dereferenced/order/item.js +4 -0
  27. package/dist/cjs/dist-dereferenced/order/orderMain.js +4 -0
  28. package/dist/cjs/dist-dereferenced/order/payment.js +4 -0
  29. package/dist/cjs/dist-dereferenced/order/shipment/index.js +45 -0
  30. package/dist/cjs/dist-dereferenced/order/shipment/item/index.js +11 -0
  31. package/dist/cjs/dist-dereferenced/order/shipment/item/itemMain.js +4 -0
  32. package/dist/cjs/dist-dereferenced/order/shipment/item/returned.js +4 -0
  33. package/dist/cjs/dist-dereferenced/order/shipment/pos/index.js +13 -0
  34. package/dist/cjs/dist-dereferenced/order/shipment/pos/outbound.js +4 -0
  35. package/dist/cjs/dist-dereferenced/order/shipment/pos/posMain.js +4 -0
  36. package/dist/cjs/dist-dereferenced/order/shipment/pos/return.js +4 -0
  37. package/dist/cjs/dist-dereferenced/order/shipment/shipmentMain.js +4 -0
  38. package/dist/cjs/dist-dereferenced/order/shipment/shipped/index.js +13 -0
  39. package/dist/cjs/dist-dereferenced/order/shipment/shipped/outbound.js +4 -0
  40. package/dist/cjs/dist-dereferenced/order/shipment/shipped/return.js +4 -0
  41. package/dist/cjs/dist-dereferenced/order/shipment/shipped/shippedMain.js +4 -0
  42. package/dist/cjs/dist-dereferenced/price.js +4 -0
  43. package/dist/cjs/dist-dereferenced/product-category.js +4 -0
  44. package/dist/cjs/dist-dereferenced/product-draft.js +4 -0
  45. package/dist/cjs/dist-dereferenced/product.js +4 -0
  46. package/dist/cjs/dist-dereferenced/stock.js +4 -0
  47. package/dist/cjs/dist-dereferenced/total.js +4 -0
  48. package/dist/cjs/dist-types/asset.js +2 -0
  49. package/dist/cjs/dist-types/category.js +2 -0
  50. package/dist/cjs/dist-types/country.js +2 -0
  51. package/dist/cjs/dist-types/currency.js +2 -0
  52. package/dist/cjs/dist-types/customer-segment.js +2 -0
  53. package/dist/cjs/dist-types/development-colour.js +2 -0
  54. package/dist/cjs/dist-types/index.js +38 -0
  55. package/dist/cjs/dist-types/marketing-preferences.js +2 -0
  56. package/dist/cjs/dist-types/messages/ean-change.js +2 -0
  57. package/dist/cjs/dist-types/messages/index.js +37 -0
  58. package/dist/cjs/dist-types/messages/order/delivered.js +2 -0
  59. package/dist/cjs/dist-types/messages/order/index.js +2 -0
  60. package/dist/cjs/dist-types/messages/order/new.js +2 -0
  61. package/dist/cjs/dist-types/messages/order/processed.js +2 -0
  62. package/dist/cjs/dist-types/messages/order/return-initiated.js +2 -0
  63. package/dist/cjs/dist-types/messages/order/returned.js +2 -0
  64. package/dist/cjs/dist-types/messages/order/shipped.js +2 -0
  65. package/dist/cjs/dist-types/messages/price-change.js +2 -0
  66. package/dist/cjs/dist-types/messages/product-category.js +2 -0
  67. package/dist/cjs/dist-types/messages/product-draft.js +2 -0
  68. package/dist/cjs/dist-types/messages/product.js +2 -0
  69. package/dist/cjs/dist-types/money.js +2 -0
  70. package/dist/cjs/dist-types/order/address.js +2 -0
  71. package/dist/cjs/dist-types/order/customer.js +2 -0
  72. package/dist/cjs/dist-types/order/index.js +37 -0
  73. package/dist/cjs/dist-types/order/item.js +2 -0
  74. package/dist/cjs/dist-types/order/orderMain.js +2 -0
  75. package/dist/cjs/dist-types/order/payment.js +2 -0
  76. package/dist/cjs/dist-types/order/shipment/index.js +39 -0
  77. package/dist/cjs/dist-types/order/shipment/item/index.js +2 -0
  78. package/dist/cjs/dist-types/order/shipment/item/itemMain.js +2 -0
  79. package/dist/cjs/dist-types/order/shipment/item/returned.js +2 -0
  80. package/dist/cjs/dist-types/order/shipment/pos/index.js +2 -0
  81. package/dist/cjs/dist-types/order/shipment/pos/outbound.js +2 -0
  82. package/dist/cjs/dist-types/order/shipment/pos/posMain.js +2 -0
  83. package/dist/cjs/dist-types/order/shipment/pos/return.js +2 -0
  84. package/dist/cjs/dist-types/order/shipment/shipmentMain.js +2 -0
  85. package/dist/cjs/dist-types/order/shipment/shipped/index.js +2 -0
  86. package/dist/cjs/dist-types/order/shipment/shipped/outbound.js +2 -0
  87. package/dist/cjs/dist-types/order/shipment/shipped/return.js +2 -0
  88. package/dist/cjs/dist-types/order/shipment/shipped/shippedMain.js +2 -0
  89. package/dist/cjs/dist-types/price.js +2 -0
  90. package/dist/cjs/dist-types/product-category.js +2 -0
  91. package/dist/cjs/dist-types/product-draft.js +2 -0
  92. package/dist/cjs/dist-types/product.js +2 -0
  93. package/dist/cjs/dist-types/stock.js +2 -0
  94. package/dist/cjs/dist-types/total.js +2 -0
  95. package/dist/cjs/index.js +39 -0
  96. package/dist/cjs/src-public/airtable.js +530 -0
  97. package/dist/cjs/src-public/bigquery.js +59 -0
  98. package/dist/cjs/src-public/cloudtasks.js +207 -0
  99. package/dist/cjs/src-public/dato.js +260 -0
  100. package/dist/cjs/src-public/env.js +902 -0
  101. package/dist/cjs/src-public/getEventTime.js +28 -0
  102. package/dist/cjs/src-public/index.js +50 -0
  103. package/dist/cjs/src-public/ipValidation.js +167 -0
  104. package/dist/cjs/src-public/jwks.js +108 -0
  105. package/dist/cjs/src-public/pubsub.js +446 -0
  106. package/dist/cjs/src-public/rateLimit.js +140 -0
  107. package/dist/cjs/src-public/redisClient.js +44 -0
  108. package/dist/cjs/src-public/runtime.js +119 -0
  109. package/dist/cjs/src-public/sanitize.js +25 -0
  110. package/dist/cjs/src-public/scheduler.js +247 -0
  111. package/dist/cjs/src-public/secret.js +16 -0
  112. package/dist/cjs/src-public/setHeaders.js +16 -0
  113. package/dist/cjs/src-public/types.js +2 -0
  114. package/dist/cjs/src-public/utils.js +286 -0
  115. package/dist/cjs/src-public/validate.js +40 -0
  116. package/dist/package.json +6 -3
  117. package/dist/types/dist-dereferenced/asset.d.ts +34 -0
  118. package/dist/types/dist-dereferenced/category.d.ts +58 -0
  119. package/dist/types/dist-dereferenced/country.d.ts +8 -0
  120. package/dist/types/dist-dereferenced/currency.d.ts +8 -0
  121. package/dist/types/dist-dereferenced/customer-segment.d.ts +26 -0
  122. package/dist/types/dist-dereferenced/development-colour.d.ts +97 -0
  123. package/dist/types/dist-dereferenced/index.d.ts +16 -0
  124. package/dist/types/dist-dereferenced/marketing-preferences.d.ts +31 -0
  125. package/dist/types/dist-dereferenced/messages/ean-change.d.ts +17 -0
  126. package/dist/types/dist-dereferenced/messages/index.d.ts +6 -0
  127. package/dist/types/dist-dereferenced/messages/order/delivered.d.ts +3414 -0
  128. package/dist/types/dist-dereferenced/messages/order/index.d.ts +6 -0
  129. package/dist/types/dist-dereferenced/messages/order/new.d.ts +3414 -0
  130. package/dist/types/dist-dereferenced/messages/order/processed.d.ts +3408 -0
  131. package/dist/types/dist-dereferenced/messages/order/return-initiated.d.ts +3414 -0
  132. package/dist/types/dist-dereferenced/messages/order/returned.d.ts +3413 -0
  133. package/dist/types/dist-dereferenced/messages/order/shipped.d.ts +3412 -0
  134. package/dist/types/dist-dereferenced/messages/price-change.d.ts +58 -0
  135. package/dist/types/dist-dereferenced/messages/product-category.d.ts +33 -0
  136. package/dist/types/dist-dereferenced/messages/product-draft.d.ts +589 -0
  137. package/dist/types/dist-dereferenced/messages/product.d.ts +628 -0
  138. package/dist/types/dist-dereferenced/money.d.ts +41 -0
  139. package/dist/types/dist-dereferenced/order/address.d.ts +98 -0
  140. package/dist/types/dist-dereferenced/order/customer.d.ts +309 -0
  141. package/dist/types/dist-dereferenced/order/index.d.ts +3404 -0
  142. package/dist/types/dist-dereferenced/order/item.d.ts +336 -0
  143. package/dist/types/dist-dereferenced/order/orderMain.d.ts +3399 -0
  144. package/dist/types/dist-dereferenced/order/payment.d.ts +94 -0
  145. package/dist/types/dist-dereferenced/order/shipment/index.d.ts +253 -0
  146. package/dist/types/dist-dereferenced/order/shipment/item/index.d.ts +19 -0
  147. package/dist/types/dist-dereferenced/order/shipment/item/itemMain.d.ts +18 -0
  148. package/dist/types/dist-dereferenced/order/shipment/item/returned.d.ts +34 -0
  149. package/dist/types/dist-dereferenced/order/shipment/pos/index.d.ts +282 -0
  150. package/dist/types/dist-dereferenced/order/shipment/pos/outbound.d.ts +320 -0
  151. package/dist/types/dist-dereferenced/order/shipment/pos/posMain.d.ts +280 -0
  152. package/dist/types/dist-dereferenced/order/shipment/pos/return.d.ts +331 -0
  153. package/dist/types/dist-dereferenced/order/shipment/shipmentMain.d.ts +250 -0
  154. package/dist/types/dist-dereferenced/order/shipment/shipped/index.d.ts +288 -0
  155. package/dist/types/dist-dereferenced/order/shipment/shipped/outbound.d.ts +286 -0
  156. package/dist/types/dist-dereferenced/order/shipment/shipped/return.d.ts +287 -0
  157. package/dist/types/dist-dereferenced/order/shipment/shipped/shippedMain.d.ts +286 -0
  158. package/dist/types/dist-dereferenced/price.d.ts +49 -0
  159. package/dist/types/dist-dereferenced/product-category.d.ts +24 -0
  160. package/dist/types/dist-dereferenced/product-draft.d.ts +580 -0
  161. package/dist/types/dist-dereferenced/product.d.ts +619 -0
  162. package/dist/types/dist-dereferenced/stock.d.ts +74 -0
  163. package/dist/types/dist-dereferenced/total.d.ts +172 -0
  164. package/dist/types/dist-types/asset.d.ts +9 -0
  165. package/dist/types/dist-types/category.d.ts +20 -0
  166. package/dist/types/dist-types/country.d.ts +2 -0
  167. package/dist/types/dist-types/currency.d.ts +2 -0
  168. package/dist/types/dist-types/customer-segment.d.ts +6 -0
  169. package/dist/types/dist-types/development-colour.d.ts +24 -0
  170. package/dist/types/dist-types/index.d.ts +16 -0
  171. package/dist/types/dist-types/marketing-preferences.d.ts +9 -0
  172. package/dist/types/dist-types/messages/ean-change.d.ts +5 -0
  173. package/dist/types/dist-types/messages/index.d.ts +6 -0
  174. package/dist/types/dist-types/messages/order/delivered.d.ts +292 -0
  175. package/dist/types/dist-types/messages/order/index.d.ts +6 -0
  176. package/dist/types/dist-types/messages/order/new.d.ts +292 -0
  177. package/dist/types/dist-types/messages/order/processed.d.ts +292 -0
  178. package/dist/types/dist-types/messages/order/return-initiated.d.ts +292 -0
  179. package/dist/types/dist-types/messages/order/returned.d.ts +293 -0
  180. package/dist/types/dist-types/messages/order/shipped.d.ts +293 -0
  181. package/dist/types/dist-types/messages/price-change.d.ts +15 -0
  182. package/dist/types/dist-types/messages/product-category.d.ts +7 -0
  183. package/dist/types/dist-types/messages/product-draft.d.ts +136 -0
  184. package/dist/types/dist-types/messages/product.d.ts +144 -0
  185. package/dist/types/dist-types/money.d.ts +11 -0
  186. package/dist/types/dist-types/order/address.d.ts +27 -0
  187. package/dist/types/dist-types/order/customer.d.ts +81 -0
  188. package/dist/types/dist-types/order/index.d.ts +8 -0
  189. package/dist/types/dist-types/order/item.d.ts +85 -0
  190. package/dist/types/dist-types/order/orderMain.d.ts +289 -0
  191. package/dist/types/dist-types/order/payment.d.ts +26 -0
  192. package/dist/types/dist-types/order/shipment/index.d.ts +6 -0
  193. package/dist/types/dist-types/order/shipment/item/index.d.ts +4 -0
  194. package/dist/types/dist-types/order/shipment/item/itemMain.d.ts +6 -0
  195. package/dist/types/dist-types/order/shipment/item/returned.d.ts +11 -0
  196. package/dist/types/dist-types/order/shipment/pos/index.d.ts +5 -0
  197. package/dist/types/dist-types/order/shipment/pos/outbound.d.ts +76 -0
  198. package/dist/types/dist-types/order/shipment/pos/posMain.d.ts +71 -0
  199. package/dist/types/dist-types/order/shipment/pos/return.d.ts +75 -0
  200. package/dist/types/dist-types/order/shipment/shipmentMain.d.ts +62 -0
  201. package/dist/types/dist-types/order/shipment/shipped/index.d.ts +5 -0
  202. package/dist/types/dist-types/order/shipment/shipped/outbound.d.ts +68 -0
  203. package/dist/types/dist-types/order/shipment/shipped/return.d.ts +68 -0
  204. package/dist/types/dist-types/order/shipment/shipped/shippedMain.d.ts +69 -0
  205. package/dist/types/dist-types/price.d.ts +14 -0
  206. package/dist/types/dist-types/product-category.d.ts +6 -0
  207. package/dist/types/dist-types/product-draft.d.ts +135 -0
  208. package/dist/types/dist-types/product.d.ts +143 -0
  209. package/dist/types/dist-types/stock.d.ts +19 -0
  210. package/dist/types/dist-types/total.d.ts +44 -0
  211. package/dist/types/index.d.ts +3 -0
  212. package/dist/types/src-public/airtable.d.ts +211 -0
  213. package/dist/types/src-public/bigquery.d.ts +17 -0
  214. package/dist/types/src-public/cloudtasks.d.ts +74 -0
  215. package/dist/types/src-public/dato.d.ts +55 -0
  216. package/dist/types/src-public/env.d.ts +144 -0
  217. package/dist/types/src-public/getEventTime.d.ts +1 -0
  218. package/dist/types/src-public/index.d.ts +16 -0
  219. package/dist/types/src-public/ipValidation.d.ts +15 -0
  220. package/dist/types/src-public/jwks.d.ts +15 -0
  221. package/dist/types/src-public/pubsub.d.ts +93 -0
  222. package/dist/types/src-public/rateLimit.d.ts +21 -0
  223. package/dist/types/src-public/redisClient.d.ts +6 -0
  224. package/dist/types/src-public/runtime.d.ts +46 -0
  225. package/dist/types/src-public/sanitize.d.ts +4 -0
  226. package/dist/types/src-public/scheduler.d.ts +71 -0
  227. package/dist/types/src-public/secret.d.ts +6 -0
  228. package/dist/types/src-public/setHeaders.d.ts +13 -0
  229. package/dist/types/src-public/types.d.ts +263 -0
  230. package/dist/types/src-public/utils.d.ts +51 -0
  231. package/dist/types/src-public/validate.d.ts +9 -0
  232. package/package.json +7 -4
  233. package/dist/index.d.ts +0 -34842
  234. package/dist/index.js +0 -110556
@@ -0,0 +1,446 @@
1
+ "use strict";
2
+ var __importDefault = (this && this.__importDefault) || function (mod) {
3
+ return (mod && mod.__esModule) ? mod : { "default": mod };
4
+ };
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ const co_body_1 = __importDefault(require("co-body"));
7
+ const lodash_chunk_1 = __importDefault(require("lodash.chunk"));
8
+ const lodash_isequal_1 = __importDefault(require("lodash.isequal"));
9
+ const lodash_omit_1 = __importDefault(require("lodash.omit"));
10
+ const runtime_1 = __importDefault(require("./runtime"));
11
+ const utils_1 = require("./utils");
12
+ function isPubSubMessage(req) {
13
+ return req && typeof req.ack === 'function' && typeof req.data !== 'undefined' && typeof req.ackId !== 'undefined';
14
+ }
15
+ const pubSubPublishTopics = global.HUSH_PUBSUB_PUBLISH_TOPICS || {};
16
+ global.HUSH_PUBSUB_PUBLISH_TOPICS = pubSubPublishTopics;
17
+ function isSubscriptionOptionsPull(opts) {
18
+ return 'endpointUrl' in opts && opts.endpointUrl === null;
19
+ }
20
+ function isSubscriptionOptionsPush(opts) {
21
+ return 'endpointUrl' in opts && !!opts.endpointUrl && (opts.endpointUrl.startsWith('https://') || opts.endpointUrl.startsWith('http://'));
22
+ }
23
+ class PubSubHelper extends runtime_1.default {
24
+ serviceAccountEmail;
25
+ pubSub;
26
+ constructor(opts, pubsubOrSettings, pubSub) {
27
+ super(opts);
28
+ if (!pubSub) {
29
+ this.pubSub = pubsubOrSettings;
30
+ this.serviceAccountEmail = opts.serviceAccountEmail;
31
+ }
32
+ else {
33
+ this.pubSub = pubSub;
34
+ this.serviceAccountEmail = pubsubOrSettings.serviceAccountEmail;
35
+ }
36
+ }
37
+ static async decodeMessageFromRequest(req) {
38
+ let rawMessage;
39
+ let rawBody;
40
+ if (!isPubSubMessage(req)) {
41
+ rawMessage = await co_body_1.default.json(req);
42
+ rawBody = Buffer.from(rawMessage.message?.data, "base64").toString('utf-8');
43
+ }
44
+ else {
45
+ rawMessage = {
46
+ message: req
47
+ };
48
+ rawBody = req.data.toString('utf-8');
49
+ }
50
+ let messageBody;
51
+ try {
52
+ messageBody = JSON.parse(rawBody);
53
+ }
54
+ catch (err) {
55
+ throw new Error('Could not parse the message body. Original body: ' + rawBody + ' . Error: ' + err.message);
56
+ }
57
+ const message = {
58
+ messageId: rawMessage.message?.messageId || undefined,
59
+ sourceSystem: rawMessage.message?.attributes?.source_system || 'unknown',
60
+ targetSystem: rawMessage.message?.attributes?.target_system || '*',
61
+ processStartedAt: (0, utils_1.parseDateOrDefault)(rawMessage.message?.attributes?.process_started_at),
62
+ recordModifiedAt: (0, utils_1.parseDateOrDefault)((0, utils_1.getEventTime)(messageBody, rawMessage.attributes?.record_modified_at)),
63
+ originalPublishTime: (0, utils_1.parseDateOrDefault)(rawMessage.message?.attributes?.original_publish_time, (0, utils_1.parseDateOrDefault)(rawMessage.message?.publish_time)),
64
+ buildshipId: rawMessage.message.attributes?.buildship_id || 'unknown',
65
+ publishTime: (0, utils_1.parseDateOrDefault)(rawMessage.message?.publish_time),
66
+ body: messageBody,
67
+ bodyTxt: rawBody,
68
+ extraAttributes: (0, lodash_omit_1.default)(rawMessage.message?.attributes || {}, ['source_system', 'target_system', 'process_started_at', 'record_modified_at', 'original_publish_time', 'buildship_id', 'publish_time', 'bigquery']),
69
+ subscriptionName: rawMessage.subscription?.split('/').pop(),
70
+ };
71
+ return message;
72
+ }
73
+ async publish(options, payload) {
74
+ if (!pubSubPublishTopics[options.topicName]) {
75
+ pubSubPublishTopics[options.topicName] = this.pubSub.topic(options.topicName, {
76
+ batching: {
77
+ maxMessages: 100,
78
+ maxMilliseconds: 50
79
+ },
80
+ });
81
+ }
82
+ const topic = pubSubPublishTopics[options.topicName];
83
+ let normPayloads = [];
84
+ if (!Array.isArray(payload)) {
85
+ normPayloads = [payload];
86
+ }
87
+ else {
88
+ normPayloads = payload;
89
+ }
90
+ const batches = normPayloads.length > 5 ? (0, lodash_chunk_1.default)(normPayloads, 10) : [normPayloads];
91
+ const attributes = {
92
+ ...(0, lodash_omit_1.default)(options.extraAttributes || {}, ['source_system', 'target_system', 'buildship_id', 'bigquery']),
93
+ source_system: options.sourceSystem,
94
+ target_system: options.targetSystem,
95
+ buildship_id: this.systemEnvName + '-' + this.workflowId + '-' + this.triggerId,
96
+ };
97
+ const messageIds = [];
98
+ for (let i = 0; i < batches.length; i++) {
99
+ const batch = batches[i];
100
+ const results = await Promise.allSettled(batch.map((x) => topic.publishMessage({
101
+ data: Buffer.from(JSON.stringify(x)),
102
+ attributes: options.attributesGenerator ? Object.assign({}, attributes, options.attributesGenerator(x)) : attributes
103
+ })));
104
+ for (const result of results) {
105
+ if (result.status === 'fulfilled') {
106
+ messageIds.push(result.value);
107
+ }
108
+ else {
109
+ messageIds.push(null);
110
+ }
111
+ }
112
+ if (i !== batches.length - 1) {
113
+ await new Promise(resolve => setTimeout(resolve, 30));
114
+ }
115
+ }
116
+ return messageIds;
117
+ }
118
+ async createOrUpdate(opts) {
119
+ const [topics] = await this.pubSub.getTopics();
120
+ const topicName = opts.topicName;
121
+ const mainSubscriptionName = opts.subscriptionName || opts.topicName;
122
+ const mainDlqTopicName = `${mainSubscriptionName}.dlq`;
123
+ const tableDlqTopicName = `${mainSubscriptionName}.bq.dlq`;
124
+ const mainDlqSubscriptionName = `${mainSubscriptionName}.dlq`;
125
+ const tableSubscriptionName = `${mainSubscriptionName}.bq`;
126
+ const tableDlqSubscriptionName = `${topicName}.bq.dlq`;
127
+ let mainTopic = topics.find(x => this.getNameFromFullyQualifiedName(x.name) === opts.topicName);
128
+ let mainDlqTopic = topics.find(x => this.getNameFromFullyQualifiedName(x.name) === mainDlqTopicName);
129
+ let tableDlqTopic = topics.find(x => this.getNameFromFullyQualifiedName(x.name) === tableDlqTopicName);
130
+ let mainTopicSubscriptions = null;
131
+ let mainDlqTopicSubscriptions = null;
132
+ let tableDlqTopicSubscriptions = null;
133
+ if (mainTopic) {
134
+ [mainTopicSubscriptions] = await mainTopic.getSubscriptions();
135
+ }
136
+ if (mainDlqTopic) {
137
+ [mainDlqTopicSubscriptions] = await mainDlqTopic.getSubscriptions();
138
+ }
139
+ if (tableDlqTopic) {
140
+ [tableDlqTopicSubscriptions] = await tableDlqTopic.getSubscriptions();
141
+ }
142
+ let mainSubscription = mainTopicSubscriptions?.find(x => this.getNameFromFullyQualifiedName(x.name) === mainSubscriptionName) || null;
143
+ let mainDlqSubscription = mainDlqTopicSubscriptions?.find(x => this.getNameFromFullyQualifiedName(x.name) === mainDlqSubscriptionName) || null;
144
+ let tableSubscription = mainTopicSubscriptions?.find(x => this.getNameFromFullyQualifiedName(x.name) === tableSubscriptionName) || null;
145
+ let tableDlqSubscription = tableDlqTopicSubscriptions?.find(x => this.getNameFromFullyQualifiedName(x.name) === tableDlqSubscriptionName) || null;
146
+ const topic = await this.ensureTopicExists(opts.topicName);
147
+ if (opts.table || opts.pull || opts.push) {
148
+ mainTopic = await this.ensureTopicExists(opts.topicName);
149
+ }
150
+ if (!opts.table) {
151
+ if (tableSubscription) {
152
+ await tableSubscription.delete();
153
+ tableSubscription = null;
154
+ }
155
+ if (tableDlqSubscription) {
156
+ await tableDlqSubscription.delete();
157
+ tableDlqSubscription = null;
158
+ }
159
+ }
160
+ else {
161
+ if (opts.createDeadLetterQueues) {
162
+ tableDlqTopic = await this.ensureTopicExists(tableDlqTopicName);
163
+ tableDlqSubscription = await this.ensureSubscribtionExists(tableDlqSubscriptionName, {
164
+ topic: tableDlqTopic,
165
+ endpointUrl: null,
166
+ dlqTopic: null,
167
+ });
168
+ }
169
+ tableSubscription = await this.ensureSubscribtionExists(tableSubscriptionName, {
170
+ topic: topic,
171
+ tableId: opts.table.tableId,
172
+ ackDeadline: opts.table.ackDeadline,
173
+ retryLimit: opts.table.retryLimit,
174
+ retryMinDelay: opts.table.retryMinDelay,
175
+ retryMaxDelay: opts.table.retryMaxDelay,
176
+ dlqTopic: tableDlqTopic || null
177
+ });
178
+ }
179
+ const pullPushSubscription = opts.pull || opts.push;
180
+ if (!pullPushSubscription) {
181
+ if (mainSubscription) {
182
+ await mainSubscription.delete();
183
+ mainSubscription = null;
184
+ }
185
+ if (mainDlqSubscription) {
186
+ await mainDlqSubscription.delete();
187
+ mainDlqSubscription = null;
188
+ }
189
+ }
190
+ else {
191
+ if (opts.createDeadLetterQueues) {
192
+ mainDlqTopic = await this.ensureTopicExists(mainDlqTopicName);
193
+ mainDlqSubscription = await this.ensureSubscribtionExists(mainDlqSubscriptionName, {
194
+ topic: mainDlqTopic,
195
+ dlqTopic: null,
196
+ endpointUrl: null,
197
+ });
198
+ }
199
+ mainSubscription = await this.ensureSubscribtionExists(mainSubscriptionName, {
200
+ topic,
201
+ dlqTopic: mainDlqTopic || null,
202
+ exactlyOnceDelivery: opts.push ? false : true,
203
+ subscriptionFilter: opts.subscriptionFilter,
204
+ endpointUrl: opts.push ? this.runtimeUrl + `/executeWorkflow/${this.workflowId}/${this.triggerId}?topic_name=${opts.topicName}` : null,
205
+ retryMinDelay: pullPushSubscription.retryMinDelay,
206
+ retryMaxDelay: pullPushSubscription.retryMaxDelay,
207
+ retryLimit: pullPushSubscription.retryLimit,
208
+ ackDeadline: Math.max(pullPushSubscription.ackDeadline, opts.push ? 10 : 60)
209
+ });
210
+ }
211
+ if (!opts.createDeadLetterQueues) {
212
+ if (mainDlqSubscription) {
213
+ await mainDlqSubscription.delete();
214
+ mainDlqSubscription = null;
215
+ }
216
+ if (tableDlqSubscription) {
217
+ await tableDlqSubscription.delete();
218
+ tableDlqSubscription = null;
219
+ }
220
+ if (tableDlqTopic) {
221
+ await tableDlqTopic.delete();
222
+ }
223
+ if (mainDlqTopic)
224
+ await mainDlqTopic.delete();
225
+ }
226
+ return {
227
+ mainTopic,
228
+ mainDlqTopic,
229
+ tableDlqTopic,
230
+ mainSubscription,
231
+ mainDlqSubscription,
232
+ tableDlqSubscription,
233
+ tableSubscription,
234
+ };
235
+ }
236
+ async findTopic(topicName) {
237
+ const [topics] = await this.pubSub.getTopics();
238
+ return topics.find(topic => this.getNameFromFullyQualifiedName(topic.name) === topicName);
239
+ }
240
+ async ensureTopicExists(topicName) {
241
+ let topic = await this.findTopic(topicName);
242
+ let topicMetadata;
243
+ try {
244
+ [topicMetadata] = await topic?.getMetadata() || [null];
245
+ }
246
+ catch (err) {
247
+ throw new Error('Invariant: While getting topic metadata - ' + err.message);
248
+ }
249
+ const metadata = {
250
+ labels: {
251
+ 'bs-envname': this.systemEnvName.toLowerCase(),
252
+ 'env': this.envName.toLowerCase()
253
+ }
254
+ };
255
+ if (topic && !process.env.PUBSUB_EMULATOR_HOST) {
256
+ const simMetadata = Object.assign({}, topicMetadata?.labels || {}, metadata.labels);
257
+ if (!(0, lodash_isequal_1.default)(topicMetadata?.labels, simMetadata)) {
258
+ simMetadata['last-modified-at'] = new Date().getTime().toString();
259
+ try {
260
+ await topic.setMetadata({
261
+ labels: simMetadata
262
+ });
263
+ }
264
+ catch (err) {
265
+ this.logging.error('ensureTopicExists received error while setting metadata', err.message, err.stack);
266
+ throw err;
267
+ }
268
+ }
269
+ }
270
+ else if (!topic) {
271
+ const payload = {
272
+ name: topicName,
273
+ };
274
+ if (!process.env.PUBSUB_EMULATOR_HOST) {
275
+ payload.labels = metadata.labels;
276
+ }
277
+ await this.pubSub.createTopic(payload);
278
+ topic = await this.findTopic(topicName);
279
+ }
280
+ if (!topic) {
281
+ throw new Error(`Invariant: While creating topic - topic is not found`);
282
+ }
283
+ return topic;
284
+ }
285
+ getNameFromFullyQualifiedName(fullyQName) {
286
+ const name = fullyQName.split('/');
287
+ return name[name.length - 1];
288
+ }
289
+ async updateSubscription(subscriptionName, subscription, subscriptionMetaData, inputOpts) {
290
+ const defaultOptions = {
291
+ retryLimit: 5,
292
+ retryMinDelay: 10,
293
+ retryMaxDelay: 360,
294
+ ackDeadline: 90,
295
+ exactlyOnceDelivery: false
296
+ };
297
+ const opts = Object.assign({}, defaultOptions, inputOpts);
298
+ let nextMetadata = null;
299
+ if (isSubscriptionOptionsPush(opts)) {
300
+ if (opts.endpointUrl !== subscriptionMetaData?.pushConfig?.pushEndpoint
301
+ || this.serviceAccountEmail !== subscriptionMetaData?.pushConfig.oidcToken?.serviceAccountEmail
302
+ || process.env.GCLOUD_PROJECT + "/" + this.triggerId !== subscriptionMetaData.pushConfig.oidcToken.audience) {
303
+ if (subscriptionMetaData?.pushConfig?.pushEndpoint && subscription) {
304
+ try {
305
+ await subscription.modifyPushConfig({
306
+ pushEndpoint: opts.endpointUrl,
307
+ oidcToken: {
308
+ serviceAccountEmail: this.serviceAccountEmail,
309
+ audience: process.env.GCLOUD_PROJECT + "/" + this.triggerId,
310
+ },
311
+ });
312
+ }
313
+ catch (err) {
314
+ this.logging.error('Invariant: While updating push subscription', err.message, err.stack);
315
+ throw new Error('Invariant: While updating push subscription - ' + err.message);
316
+ }
317
+ }
318
+ else if (opts.endpointUrl) {
319
+ nextMetadata = nextMetadata || {};
320
+ nextMetadata.pushConfig = {
321
+ pushEndpoint: opts.endpointUrl,
322
+ oidcToken: {
323
+ serviceAccountEmail: this.serviceAccountEmail,
324
+ audience: process.env.GCLOUD_PROJECT + "/" + this.triggerId,
325
+ },
326
+ };
327
+ }
328
+ }
329
+ }
330
+ else {
331
+ if (subscriptionMetaData?.pushConfig?.pushEndpoint) {
332
+ nextMetadata = nextMetadata || {};
333
+ nextMetadata.pushConfig = null;
334
+ }
335
+ }
336
+ if (opts.dlqTopic) {
337
+ if (opts.dlqTopic.name !== subscriptionMetaData?.deadLetterPolicy?.deadLetterTopic
338
+ || opts.retryLimit !== subscriptionMetaData?.deadLetterPolicy.maxDeliveryAttempts) {
339
+ nextMetadata = nextMetadata || {};
340
+ nextMetadata.deadLetterPolicy = {
341
+ deadLetterTopic: opts.dlqTopic.name,
342
+ maxDeliveryAttempts: opts.retryLimit
343
+ };
344
+ }
345
+ }
346
+ else if (subscriptionMetaData?.deadLetterPolicy?.deadLetterTopic) {
347
+ nextMetadata = nextMetadata || {};
348
+ nextMetadata.deadLetterPolicy = null;
349
+ }
350
+ if (opts.exactlyOnceDelivery !== subscriptionMetaData?.enableExactlyOnceDelivery) {
351
+ if (typeof opts.exactlyOnceDelivery === 'boolean' && isSubscriptionOptionsPull(opts)) {
352
+ nextMetadata = nextMetadata || {};
353
+ nextMetadata.enableExactlyOnceDelivery = opts.exactlyOnceDelivery;
354
+ }
355
+ else {
356
+ nextMetadata = nextMetadata || {};
357
+ nextMetadata.enableExactlyOnceDelivery = null;
358
+ }
359
+ }
360
+ if ((opts.subscriptionFilter && subscriptionMetaData?.filter) && opts.subscriptionFilter !== subscriptionMetaData?.filter) {
361
+ nextMetadata = nextMetadata || {};
362
+ nextMetadata.filter = opts.subscriptionFilter;
363
+ }
364
+ if (opts.retryMinDelay != subscriptionMetaData?.retryPolicy?.minimumBackoff?.seconds
365
+ || opts.retryMaxDelay != subscriptionMetaData?.retryPolicy?.maximumBackoff?.seconds) {
366
+ nextMetadata = nextMetadata || {};
367
+ nextMetadata.retryPolicy = {
368
+ minimumBackoff: { seconds: opts.retryMinDelay },
369
+ maximumBackoff: { seconds: opts.retryMaxDelay }
370
+ };
371
+ }
372
+ const proposedLabels = {
373
+ 'env': this.envName.toLowerCase(),
374
+ 'bs-triggerid': this.triggerId.toLowerCase(),
375
+ 'bs-workflowid': this.workflowId.toLowerCase(),
376
+ 'bs-envname': this.systemEnvName.toLowerCase(),
377
+ 'bs-is-deadletter': subscriptionName.indexOf('-dlq') > -1 ? 'true' : 'false',
378
+ 'bs-is-push-enabled': isSubscriptionOptionsPush(opts) && opts.endpointUrl ? 'true' : 'false',
379
+ 'bs-bigquery-tableid': !(isSubscriptionOptionsPush(opts) || isSubscriptionOptionsPull(opts)) && opts.tableId ? opts.tableId.replace(/\./g, '-').toLowerCase() : '',
380
+ 'bs-is-bigquery': !(isSubscriptionOptionsPush(opts) || isSubscriptionOptionsPull(opts)) && opts.tableId ? 'true' : 'false',
381
+ };
382
+ const simLabels = Object.assign({}, subscriptionMetaData?.labels || {}, proposedLabels);
383
+ if (!(0, lodash_isequal_1.default)(simLabels, subscriptionMetaData?.labels)) {
384
+ nextMetadata = nextMetadata || {};
385
+ nextMetadata.labels = simLabels;
386
+ }
387
+ if (opts.ackDeadline !== subscriptionMetaData?.ackDeadlineSeconds) {
388
+ nextMetadata = nextMetadata || {};
389
+ nextMetadata.ackDeadlineSeconds = opts.ackDeadline;
390
+ }
391
+ if (!isSubscriptionOptionsPull(opts) && !isSubscriptionOptionsPush(opts)) {
392
+ if (opts.tableId !== subscriptionMetaData?.bigqueryConfig?.table || this.serviceAccountEmail !== subscriptionMetaData?.bigqueryConfig?.serviceAccountEmail) {
393
+ if (opts.tableId) {
394
+ nextMetadata = nextMetadata || {};
395
+ nextMetadata.bigqueryConfig = {
396
+ table: opts.tableId,
397
+ writeMetadata: true,
398
+ dropUnknownFields: true,
399
+ serviceAccountEmail: this.serviceAccountEmail,
400
+ useTableSchema: false,
401
+ };
402
+ }
403
+ else {
404
+ nextMetadata = nextMetadata || {};
405
+ nextMetadata.bigqueryConfig = null;
406
+ }
407
+ }
408
+ }
409
+ else if (subscriptionMetaData?.bigqueryConfig?.table) {
410
+ nextMetadata = nextMetadata || {};
411
+ nextMetadata.bigqueryConfig = null;
412
+ }
413
+ if (subscription && nextMetadata) {
414
+ try {
415
+ await subscription.setMetadata(nextMetadata);
416
+ }
417
+ catch (err) {
418
+ this.logging.error('updateSubscription received error while setting metadata', err.message, err.stack);
419
+ throw new Error('Invariant: While updating subscription - ' + err.message);
420
+ }
421
+ }
422
+ return nextMetadata;
423
+ }
424
+ async ensureSubscribtionExists(subscriptionName, opts) {
425
+ let [subscriptions] = await opts.topic.getSubscriptions();
426
+ let subscription = subscriptions.find(subscription => this.getNameFromFullyQualifiedName(subscription.name) === subscriptionName) || null;
427
+ let subscriptionMetaData;
428
+ try {
429
+ [subscriptionMetaData] = (await subscription?.getMetadata()) || [null];
430
+ }
431
+ catch (err) {
432
+ throw new Error('Invariant: While getting subscription metadata - ' + err.message);
433
+ }
434
+ const createPayload = await this.updateSubscription(subscriptionName, subscription, subscriptionMetaData, opts);
435
+ if (createPayload && !subscription) {
436
+ await opts.topic.createSubscription(subscriptionName, createPayload);
437
+ [subscriptions] = await opts.topic.getSubscriptions();
438
+ subscription = subscriptions.find(subscription => this.getNameFromFullyQualifiedName(subscription.name) === subscriptionName) || null;
439
+ }
440
+ if (!subscription) {
441
+ throw new Error(`Invariant: While creating subscription - subscription is not found`);
442
+ }
443
+ return subscription;
444
+ }
445
+ }
446
+ exports.default = PubSubHelper;
@@ -0,0 +1,140 @@
1
+ "use strict";
2
+ var __importDefault = (this && this.__importDefault) || function (mod) {
3
+ return (mod && mod.__esModule) ? mod : { "default": mod };
4
+ };
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ exports.RedisConnectionError = void 0;
7
+ class RedisConnectionError extends Error {
8
+ constructor(message) {
9
+ super(message);
10
+ this.name = 'RedisConnectionError';
11
+ }
12
+ }
13
+ exports.RedisConnectionError = RedisConnectionError;
14
+ const runtime_1 = __importDefault(require("./runtime"));
15
+ function normalizeId(id) {
16
+ return encodeURIComponent(id);
17
+ }
18
+ class RateLimit extends runtime_1.default {
19
+ redisInstance;
20
+ constructor(opts, redisInstance) {
21
+ super(opts);
22
+ this.redisInstance = redisInstance;
23
+ }
24
+ async acquireLock(id, maxConcurrentExecutions, expirationInSeconds = 60 * 5) {
25
+ const key = `ratelimit:${normalizeId(id)}`;
26
+ const now = Date.now();
27
+ const lockInfo = {
28
+ id,
29
+ workflowId: this.workflowId,
30
+ triggerId: this.triggerId,
31
+ url: this.runtimeUrl,
32
+ envName: this.envName,
33
+ creation_time: now,
34
+ expiration_time: now + expirationInSeconds * 1000,
35
+ };
36
+ try {
37
+ const timeout = 300;
38
+ const redis = this.redisInstance;
39
+ if (!redis.isOpen) {
40
+ try {
41
+ await Promise.race([
42
+ redis.connect(),
43
+ new Promise((_, reject) => setTimeout(() => reject(new RedisConnectionError('Redis connect timeout')), timeout)),
44
+ ]);
45
+ }
46
+ catch (err) {
47
+ if (err instanceof RedisConnectionError)
48
+ throw err;
49
+ throw new RedisConnectionError(err.message);
50
+ }
51
+ }
52
+ await Promise.race([
53
+ redis.zRemRangeByScore(key, '-inf', now),
54
+ new Promise((_, reject) => setTimeout(() => reject(new Error('Redis timeout')), timeout)),
55
+ ]);
56
+ const count = await Promise.race([
57
+ redis.zCard(key),
58
+ new Promise((_, reject) => setTimeout(() => reject(new Error('Redis timeout')), timeout)),
59
+ ]);
60
+ if (count >= maxConcurrentExecutions) {
61
+ return false;
62
+ }
63
+ await Promise.race([
64
+ redis.zAdd(key, { score: lockInfo.expiration_time, value: JSON.stringify(lockInfo) }),
65
+ new Promise((_, reject) => setTimeout(() => reject(new Error('Redis timeout')), timeout)),
66
+ ]);
67
+ await Promise.race([
68
+ redis.expire(key, expirationInSeconds),
69
+ new Promise((_, reject) => setTimeout(() => reject(new Error('Redis timeout')), timeout)),
70
+ ]);
71
+ return true;
72
+ }
73
+ catch (err) {
74
+ if (err instanceof RedisConnectionError)
75
+ throw err;
76
+ return false;
77
+ }
78
+ }
79
+ async releaseLock(id) {
80
+ const key = `ratelimit:${normalizeId(id)}`;
81
+ const redis = this.redisInstance;
82
+ const timeout = 300;
83
+ if (!redis.isOpen) {
84
+ try {
85
+ await Promise.race([
86
+ redis.connect(),
87
+ new Promise((_, reject) => setTimeout(() => reject(new RedisConnectionError('Redis connect timeout')), timeout)),
88
+ ]);
89
+ }
90
+ catch (err) {
91
+ if (err instanceof RedisConnectionError)
92
+ throw err;
93
+ throw new RedisConnectionError(err.message);
94
+ }
95
+ }
96
+ const locks = await redis.zRange(key, 0, -1);
97
+ const filtered = locks.filter((lockStr) => {
98
+ try {
99
+ const lock = JSON.parse(lockStr);
100
+ return lock.workflowId === this.workflowId && lock.triggerId === this.triggerId && lock.envName === this.envName && lock.url === this.runtimeUrl;
101
+ }
102
+ catch {
103
+ return false;
104
+ }
105
+ });
106
+ for (const lockStr of filtered) {
107
+ await redis.zRem(key, lockStr);
108
+ }
109
+ }
110
+ async viewLocks(id) {
111
+ const key = `ratelimit:${normalizeId(id)}`;
112
+ const redis = this.redisInstance;
113
+ const timeout = 300;
114
+ if (!redis.isOpen) {
115
+ try {
116
+ await Promise.race([
117
+ redis.connect(),
118
+ new Promise((_, reject) => setTimeout(() => reject(new RedisConnectionError('Redis connect timeout')), timeout)),
119
+ ]);
120
+ }
121
+ catch (err) {
122
+ if (err instanceof RedisConnectionError)
123
+ throw err;
124
+ throw new RedisConnectionError(err.message);
125
+ }
126
+ }
127
+ const now = Date.now();
128
+ await redis.zRemRangeByScore(key, '-inf', now);
129
+ const locks = await redis.zRange(key, 0, -1);
130
+ return locks.reduce((acc, lockStr) => {
131
+ try {
132
+ acc.push(JSON.parse(lockStr));
133
+ }
134
+ catch {
135
+ }
136
+ return acc;
137
+ }, []);
138
+ }
139
+ }
140
+ exports.default = RateLimit;
@@ -0,0 +1,44 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.default = getRedisClient;
4
+ exports.getConnectedRedisClient = getConnectedRedisClient;
5
+ const redis_1 = require("redis");
6
+ global.REDIS_CLIENTS = global.REDIS_CLIENTS || {};
7
+ function getRedisClient(sysEnv) {
8
+ const jsonEnv = JSON.parse(process.env[`${sysEnv}ProjectEnv`] || '{}');
9
+ const redisEndpoint = jsonEnv['redis_host'];
10
+ let REDIS_CLIENT = global.REDIS_CLIENTS;
11
+ if (!REDIS_CLIENTS[redisEndpoint]) {
12
+ if (!redisEndpoint && process.env.K_REVISION) {
13
+ throw new Error('Invariant: Redis endpoint is not set in the Environment Variables. Please set the redis_host with a value of IP:PORT i.e. 10.67.130.27:6379');
14
+ }
15
+ else if (!redisEndpoint) {
16
+ return null;
17
+ }
18
+ REDIS_CLIENT = (0, redis_1.createClient)({
19
+ url: `redis://${redisEndpoint}`
20
+ });
21
+ global.REDIS_CLIENTS[redisEndpoint] = REDIS_CLIENT;
22
+ }
23
+ return global.REDIS_CLIENTS[redisEndpoint];
24
+ }
25
+ function getConnectedRedisClient(url) {
26
+ const redisClient = getRedisClient(url);
27
+ if (!redisClient) {
28
+ return Promise.resolve(null);
29
+ }
30
+ let race = Promise.resolve(redisClient);
31
+ if (!redisClient.isOpen) {
32
+ race = Promise.race([
33
+ redisClient.connect().then(() => redisClient).catch((err) => {
34
+ console.log('Hush - Redis connect error', err.message);
35
+ return null;
36
+ }),
37
+ new Promise((resolve) => setTimeout(() => {
38
+ console.log('Hush - Redis connect timeout');
39
+ resolve(null);
40
+ }, 100)),
41
+ ]);
42
+ }
43
+ return race;
44
+ }