@shushed/helpers 0.0.199 → 0.0.200-runtime-url-fix-20251125155626
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/dist-dereferenced/asset.js +4 -0
- package/dist/cjs/dist-dereferenced/category.js +4 -0
- package/dist/cjs/dist-dereferenced/country.js +4 -0
- package/dist/cjs/dist-dereferenced/currency.js +4 -0
- package/dist/cjs/dist-dereferenced/customer-segment.js +4 -0
- package/dist/cjs/dist-dereferenced/development-colour.js +4 -0
- package/dist/cjs/dist-dereferenced/index.js +69 -0
- package/dist/cjs/dist-dereferenced/marketing-preferences.js +4 -0
- package/dist/cjs/dist-dereferenced/messages/ean-change.js +4 -0
- package/dist/cjs/dist-dereferenced/messages/index.js +50 -0
- package/dist/cjs/dist-dereferenced/messages/order/delivered.js +4 -0
- package/dist/cjs/dist-dereferenced/messages/order/index.js +18 -0
- package/dist/cjs/dist-dereferenced/messages/order/new.js +4 -0
- package/dist/cjs/dist-dereferenced/messages/order/processed.js +4 -0
- package/dist/cjs/dist-dereferenced/messages/order/return-initiated.js +4 -0
- package/dist/cjs/dist-dereferenced/messages/order/returned.js +4 -0
- package/dist/cjs/dist-dereferenced/messages/order/shipped.js +4 -0
- package/dist/cjs/dist-dereferenced/messages/price-change.js +4 -0
- package/dist/cjs/dist-dereferenced/messages/product-category.js +4 -0
- package/dist/cjs/dist-dereferenced/messages/product-draft.js +4 -0
- package/dist/cjs/dist-dereferenced/messages/product.js +4 -0
- package/dist/cjs/dist-dereferenced/money.js +4 -0
- package/dist/cjs/dist-dereferenced/order/address.js +4 -0
- package/dist/cjs/dist-dereferenced/order/customer.js +4 -0
- package/dist/cjs/dist-dereferenced/order/index.js +51 -0
- package/dist/cjs/dist-dereferenced/order/item.js +4 -0
- package/dist/cjs/dist-dereferenced/order/orderMain.js +4 -0
- package/dist/cjs/dist-dereferenced/order/payment.js +4 -0
- package/dist/cjs/dist-dereferenced/order/shipment/index.js +45 -0
- package/dist/cjs/dist-dereferenced/order/shipment/item/index.js +11 -0
- package/dist/cjs/dist-dereferenced/order/shipment/item/itemMain.js +4 -0
- package/dist/cjs/dist-dereferenced/order/shipment/item/returned.js +4 -0
- package/dist/cjs/dist-dereferenced/order/shipment/pos/index.js +13 -0
- package/dist/cjs/dist-dereferenced/order/shipment/pos/outbound.js +4 -0
- package/dist/cjs/dist-dereferenced/order/shipment/pos/posMain.js +4 -0
- package/dist/cjs/dist-dereferenced/order/shipment/pos/return.js +4 -0
- package/dist/cjs/dist-dereferenced/order/shipment/shipmentMain.js +4 -0
- package/dist/cjs/dist-dereferenced/order/shipment/shipped/index.js +13 -0
- package/dist/cjs/dist-dereferenced/order/shipment/shipped/outbound.js +4 -0
- package/dist/cjs/dist-dereferenced/order/shipment/shipped/return.js +4 -0
- package/dist/cjs/dist-dereferenced/order/shipment/shipped/shippedMain.js +4 -0
- package/dist/cjs/dist-dereferenced/price.js +4 -0
- package/dist/cjs/dist-dereferenced/product-category.js +4 -0
- package/dist/cjs/dist-dereferenced/product-draft.js +4 -0
- package/dist/cjs/dist-dereferenced/product.js +4 -0
- package/dist/cjs/dist-dereferenced/stock.js +4 -0
- package/dist/cjs/dist-dereferenced/total.js +4 -0
- package/dist/cjs/dist-types/asset.js +2 -0
- package/dist/cjs/dist-types/category.js +2 -0
- package/dist/cjs/dist-types/country.js +2 -0
- package/dist/cjs/dist-types/currency.js +2 -0
- package/dist/cjs/dist-types/customer-segment.js +2 -0
- package/dist/cjs/dist-types/development-colour.js +2 -0
- package/dist/cjs/dist-types/index.js +38 -0
- package/dist/cjs/dist-types/marketing-preferences.js +2 -0
- package/dist/cjs/dist-types/messages/ean-change.js +2 -0
- package/dist/cjs/dist-types/messages/index.js +37 -0
- package/dist/cjs/dist-types/messages/order/delivered.js +2 -0
- package/dist/cjs/dist-types/messages/order/index.js +2 -0
- package/dist/cjs/dist-types/messages/order/new.js +2 -0
- package/dist/cjs/dist-types/messages/order/processed.js +2 -0
- package/dist/cjs/dist-types/messages/order/return-initiated.js +2 -0
- package/dist/cjs/dist-types/messages/order/returned.js +2 -0
- package/dist/cjs/dist-types/messages/order/shipped.js +2 -0
- package/dist/cjs/dist-types/messages/price-change.js +2 -0
- package/dist/cjs/dist-types/messages/product-category.js +2 -0
- package/dist/cjs/dist-types/messages/product-draft.js +2 -0
- package/dist/cjs/dist-types/messages/product.js +2 -0
- package/dist/cjs/dist-types/money.js +2 -0
- package/dist/cjs/dist-types/order/address.js +2 -0
- package/dist/cjs/dist-types/order/customer.js +2 -0
- package/dist/cjs/dist-types/order/index.js +37 -0
- package/dist/cjs/dist-types/order/item.js +2 -0
- package/dist/cjs/dist-types/order/orderMain.js +2 -0
- package/dist/cjs/dist-types/order/payment.js +2 -0
- package/dist/cjs/dist-types/order/shipment/index.js +39 -0
- package/dist/cjs/dist-types/order/shipment/item/index.js +2 -0
- package/dist/cjs/dist-types/order/shipment/item/itemMain.js +2 -0
- package/dist/cjs/dist-types/order/shipment/item/returned.js +2 -0
- package/dist/cjs/dist-types/order/shipment/pos/index.js +2 -0
- package/dist/cjs/dist-types/order/shipment/pos/outbound.js +2 -0
- package/dist/cjs/dist-types/order/shipment/pos/posMain.js +2 -0
- package/dist/cjs/dist-types/order/shipment/pos/return.js +2 -0
- package/dist/cjs/dist-types/order/shipment/shipmentMain.js +2 -0
- package/dist/cjs/dist-types/order/shipment/shipped/index.js +2 -0
- package/dist/cjs/dist-types/order/shipment/shipped/outbound.js +2 -0
- package/dist/cjs/dist-types/order/shipment/shipped/return.js +2 -0
- package/dist/cjs/dist-types/order/shipment/shipped/shippedMain.js +2 -0
- package/dist/cjs/dist-types/price.js +2 -0
- package/dist/cjs/dist-types/product-category.js +2 -0
- package/dist/cjs/dist-types/product-draft.js +2 -0
- package/dist/cjs/dist-types/product.js +2 -0
- package/dist/cjs/dist-types/stock.js +2 -0
- package/dist/cjs/dist-types/total.js +2 -0
- package/dist/cjs/index.js +39 -0
- package/dist/cjs/src-public/airtable.js +562 -0
- package/dist/cjs/src-public/bigquery.js +59 -0
- package/dist/cjs/src-public/cloudtasks.js +207 -0
- package/dist/cjs/src-public/dato.js +637 -0
- package/dist/cjs/src-public/env.js +897 -0
- package/dist/cjs/src-public/getEventTime.js +28 -0
- package/dist/cjs/src-public/index.js +50 -0
- package/dist/cjs/src-public/ipValidation.js +167 -0
- package/dist/cjs/src-public/jwks.js +108 -0
- package/dist/cjs/src-public/pubsub.js +423 -0
- package/dist/cjs/src-public/rateLimit.js +140 -0
- package/dist/cjs/src-public/redisClient.js +44 -0
- package/dist/cjs/src-public/runtime.js +141 -0
- package/dist/cjs/src-public/sanitize.js +25 -0
- package/dist/cjs/src-public/scheduler.js +247 -0
- package/dist/cjs/src-public/secret.js +16 -0
- package/dist/cjs/src-public/setHeaders.js +16 -0
- package/dist/cjs/src-public/types.js +2 -0
- package/dist/cjs/src-public/utils.js +336 -0
- package/dist/cjs/src-public/validate.js +40 -0
- package/dist/package.json +6 -3
- package/dist/types/dist-dereferenced/asset.d.ts +51 -0
- package/dist/types/dist-dereferenced/category.d.ts +58 -0
- package/dist/types/dist-dereferenced/country.d.ts +8 -0
- package/dist/types/dist-dereferenced/currency.d.ts +8 -0
- package/dist/types/dist-dereferenced/customer-segment.d.ts +26 -0
- package/dist/types/dist-dereferenced/development-colour.d.ts +114 -0
- package/dist/types/dist-dereferenced/index.d.ts +16 -0
- package/dist/types/dist-dereferenced/marketing-preferences.d.ts +31 -0
- package/dist/types/dist-dereferenced/messages/ean-change.d.ts +17 -0
- package/dist/types/dist-dereferenced/messages/index.d.ts +6 -0
- package/dist/types/dist-dereferenced/messages/order/delivered.d.ts +3432 -0
- package/dist/types/dist-dereferenced/messages/order/index.d.ts +6 -0
- package/dist/types/dist-dereferenced/messages/order/new.d.ts +3432 -0
- package/dist/types/dist-dereferenced/messages/order/processed.d.ts +3426 -0
- package/dist/types/dist-dereferenced/messages/order/return-initiated.d.ts +3432 -0
- package/dist/types/dist-dereferenced/messages/order/returned.d.ts +3431 -0
- package/dist/types/dist-dereferenced/messages/order/shipped.d.ts +3430 -0
- package/dist/types/dist-dereferenced/messages/price-change.d.ts +58 -0
- package/dist/types/dist-dereferenced/messages/product-category.d.ts +33 -0
- package/dist/types/dist-dereferenced/messages/product-draft.d.ts +589 -0
- package/dist/types/dist-dereferenced/messages/product.d.ts +628 -0
- package/dist/types/dist-dereferenced/money.d.ts +41 -0
- package/dist/types/dist-dereferenced/order/address.d.ts +98 -0
- package/dist/types/dist-dereferenced/order/customer.d.ts +309 -0
- package/dist/types/dist-dereferenced/order/index.d.ts +3422 -0
- package/dist/types/dist-dereferenced/order/item.d.ts +336 -0
- package/dist/types/dist-dereferenced/order/orderMain.d.ts +3417 -0
- package/dist/types/dist-dereferenced/order/payment.d.ts +110 -0
- package/dist/types/dist-dereferenced/order/shipment/index.d.ts +253 -0
- package/dist/types/dist-dereferenced/order/shipment/item/index.d.ts +19 -0
- package/dist/types/dist-dereferenced/order/shipment/item/itemMain.d.ts +18 -0
- package/dist/types/dist-dereferenced/order/shipment/item/returned.d.ts +34 -0
- package/dist/types/dist-dereferenced/order/shipment/pos/index.d.ts +282 -0
- package/dist/types/dist-dereferenced/order/shipment/pos/outbound.d.ts +319 -0
- package/dist/types/dist-dereferenced/order/shipment/pos/posMain.d.ts +280 -0
- package/dist/types/dist-dereferenced/order/shipment/pos/return.d.ts +330 -0
- package/dist/types/dist-dereferenced/order/shipment/shipmentMain.d.ts +250 -0
- package/dist/types/dist-dereferenced/order/shipment/shipped/index.d.ts +288 -0
- package/dist/types/dist-dereferenced/order/shipment/shipped/outbound.d.ts +286 -0
- package/dist/types/dist-dereferenced/order/shipment/shipped/return.d.ts +287 -0
- package/dist/types/dist-dereferenced/order/shipment/shipped/shippedMain.d.ts +286 -0
- package/dist/types/dist-dereferenced/price.d.ts +49 -0
- package/dist/types/dist-dereferenced/product-category.d.ts +24 -0
- package/dist/types/dist-dereferenced/product-draft.d.ts +580 -0
- package/dist/types/dist-dereferenced/product.d.ts +619 -0
- package/dist/types/dist-dereferenced/stock.d.ts +74 -0
- package/dist/types/dist-dereferenced/total.d.ts +172 -0
- package/dist/types/dist-types/asset.d.ts +16 -0
- package/dist/types/dist-types/category.d.ts +20 -0
- package/dist/types/dist-types/country.d.ts +2 -0
- package/dist/types/dist-types/currency.d.ts +2 -0
- package/dist/types/dist-types/customer-segment.d.ts +6 -0
- package/dist/types/dist-types/development-colour.d.ts +31 -0
- package/dist/types/dist-types/index.d.ts +16 -0
- package/dist/types/dist-types/marketing-preferences.d.ts +9 -0
- package/dist/types/dist-types/messages/ean-change.d.ts +5 -0
- package/dist/types/dist-types/messages/index.d.ts +6 -0
- package/dist/types/dist-types/messages/order/delivered.d.ts +298 -0
- package/dist/types/dist-types/messages/order/index.d.ts +6 -0
- package/dist/types/dist-types/messages/order/new.d.ts +298 -0
- package/dist/types/dist-types/messages/order/processed.d.ts +298 -0
- package/dist/types/dist-types/messages/order/return-initiated.d.ts +298 -0
- package/dist/types/dist-types/messages/order/returned.d.ts +299 -0
- package/dist/types/dist-types/messages/order/shipped.d.ts +299 -0
- package/dist/types/dist-types/messages/price-change.d.ts +15 -0
- package/dist/types/dist-types/messages/product-category.d.ts +7 -0
- package/dist/types/dist-types/messages/product-draft.d.ts +136 -0
- package/dist/types/dist-types/messages/product.d.ts +144 -0
- package/dist/types/dist-types/money.d.ts +11 -0
- package/dist/types/dist-types/order/address.d.ts +27 -0
- package/dist/types/dist-types/order/customer.d.ts +81 -0
- package/dist/types/dist-types/order/index.d.ts +8 -0
- package/dist/types/dist-types/order/item.d.ts +85 -0
- package/dist/types/dist-types/order/orderMain.d.ts +295 -0
- package/dist/types/dist-types/order/payment.d.ts +30 -0
- package/dist/types/dist-types/order/shipment/index.d.ts +6 -0
- package/dist/types/dist-types/order/shipment/item/index.d.ts +4 -0
- package/dist/types/dist-types/order/shipment/item/itemMain.d.ts +6 -0
- package/dist/types/dist-types/order/shipment/item/returned.d.ts +11 -0
- package/dist/types/dist-types/order/shipment/pos/index.d.ts +5 -0
- package/dist/types/dist-types/order/shipment/pos/outbound.d.ts +76 -0
- package/dist/types/dist-types/order/shipment/pos/posMain.d.ts +71 -0
- package/dist/types/dist-types/order/shipment/pos/return.d.ts +75 -0
- package/dist/types/dist-types/order/shipment/shipmentMain.d.ts +62 -0
- package/dist/types/dist-types/order/shipment/shipped/index.d.ts +5 -0
- package/dist/types/dist-types/order/shipment/shipped/outbound.d.ts +68 -0
- package/dist/types/dist-types/order/shipment/shipped/return.d.ts +68 -0
- package/dist/types/dist-types/order/shipment/shipped/shippedMain.d.ts +69 -0
- package/dist/types/dist-types/price.d.ts +14 -0
- package/dist/types/dist-types/product-category.d.ts +6 -0
- package/dist/types/dist-types/product-draft.d.ts +135 -0
- package/dist/types/dist-types/product.d.ts +143 -0
- package/dist/types/dist-types/stock.d.ts +19 -0
- package/dist/types/dist-types/total.d.ts +44 -0
- package/dist/types/index.d.ts +3 -0
- package/dist/types/src-public/airtable.d.ts +220 -0
- package/dist/types/src-public/bigquery.d.ts +17 -0
- package/dist/types/src-public/cloudtasks.d.ts +74 -0
- package/dist/types/src-public/dato.d.ts +88 -0
- package/dist/types/src-public/env.d.ts +144 -0
- package/dist/types/src-public/getEventTime.d.ts +1 -0
- package/dist/types/src-public/index.d.ts +16 -0
- package/dist/types/src-public/ipValidation.d.ts +15 -0
- package/dist/types/src-public/jwks.d.ts +15 -0
- package/dist/types/src-public/pubsub.d.ts +95 -0
- package/dist/types/src-public/rateLimit.d.ts +21 -0
- package/dist/types/src-public/redisClient.d.ts +6 -0
- package/dist/types/src-public/runtime.d.ts +57 -0
- package/dist/types/src-public/sanitize.d.ts +4 -0
- package/dist/types/src-public/scheduler.d.ts +71 -0
- package/dist/types/src-public/secret.d.ts +6 -0
- package/dist/types/src-public/setHeaders.d.ts +13 -0
- package/dist/types/src-public/types.d.ts +264 -0
- package/dist/types/src-public/utils.d.ts +67 -0
- package/dist/types/src-public/validate.d.ts +9 -0
- package/package.json +7 -4
- package/dist/index.d.ts +0 -34970
- package/dist/index.js +0 -112969
|
@@ -0,0 +1,423 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.isSubscriptionBigQuery = isSubscriptionBigQuery;
|
|
7
|
+
const co_body_1 = __importDefault(require("co-body"));
|
|
8
|
+
const lodash_chunk_1 = __importDefault(require("lodash.chunk"));
|
|
9
|
+
const lodash_isequal_1 = __importDefault(require("lodash.isequal"));
|
|
10
|
+
const lodash_omit_1 = __importDefault(require("lodash.omit"));
|
|
11
|
+
const runtime_1 = __importDefault(require("./runtime"));
|
|
12
|
+
const utils_1 = require("./utils");
|
|
13
|
+
function isPubSubMessage(req) {
|
|
14
|
+
return req && typeof req.ack === 'function' && typeof req.data !== 'undefined' && typeof req.ackId !== 'undefined';
|
|
15
|
+
}
|
|
16
|
+
const pubSubPublishTopics = global.HUSH_PUBSUB_PUBLISH_TOPICS || {};
|
|
17
|
+
global.HUSH_PUBSUB_PUBLISH_TOPICS = pubSubPublishTopics;
|
|
18
|
+
function isSubscriptionBigQuery(opts) {
|
|
19
|
+
return 'tableId' in opts && opts.tableId !== null;
|
|
20
|
+
}
|
|
21
|
+
class PubSubHelper extends runtime_1.default {
|
|
22
|
+
serviceAccountEmail;
|
|
23
|
+
pubSub;
|
|
24
|
+
constructor(opts, pubsubOrSettings, pubSub) {
|
|
25
|
+
super(opts);
|
|
26
|
+
if (!pubSub) {
|
|
27
|
+
this.pubSub = pubsubOrSettings;
|
|
28
|
+
this.serviceAccountEmail = opts.serviceAccountEmail;
|
|
29
|
+
}
|
|
30
|
+
else {
|
|
31
|
+
this.pubSub = pubSub;
|
|
32
|
+
this.serviceAccountEmail = pubsubOrSettings.serviceAccountEmail;
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
static async decodeMessageFromRequest(req) {
|
|
36
|
+
let rawMessage;
|
|
37
|
+
let rawBody;
|
|
38
|
+
if (!isPubSubMessage(req)) {
|
|
39
|
+
rawMessage = await co_body_1.default.json(req);
|
|
40
|
+
rawBody = Buffer.from(rawMessage.message?.data, "base64").toString('utf-8');
|
|
41
|
+
}
|
|
42
|
+
else {
|
|
43
|
+
rawMessage = {
|
|
44
|
+
message: req
|
|
45
|
+
};
|
|
46
|
+
rawBody = req.data.toString('utf-8');
|
|
47
|
+
}
|
|
48
|
+
let messageBody;
|
|
49
|
+
try {
|
|
50
|
+
messageBody = JSON.parse(rawBody);
|
|
51
|
+
}
|
|
52
|
+
catch (err) {
|
|
53
|
+
throw new Error('Could not parse the message body. Original body: ' + rawBody + ' . Error: ' + err.message);
|
|
54
|
+
}
|
|
55
|
+
const message = {
|
|
56
|
+
messageId: rawMessage.message?.messageId || undefined,
|
|
57
|
+
sourceSystem: rawMessage.message?.attributes?.source_system || 'unknown',
|
|
58
|
+
targetSystem: rawMessage.message?.attributes?.target_system || '*',
|
|
59
|
+
processStartedAt: (0, utils_1.parseDateOrDefault)(rawMessage.message?.attributes?.process_started_at),
|
|
60
|
+
recordModifiedAt: (0, utils_1.parseDateOrDefault)((0, utils_1.getEventTime)(messageBody, rawMessage.attributes?.record_modified_at)),
|
|
61
|
+
originalPublishTime: (0, utils_1.parseDateOrDefault)(rawMessage.message?.attributes?.original_publish_time, (0, utils_1.parseDateOrDefault)(rawMessage.message?.publish_time)),
|
|
62
|
+
buildshipId: rawMessage.message.attributes?.buildship_id || 'unknown',
|
|
63
|
+
publishTime: (0, utils_1.parseDateOrDefault)(rawMessage.message?.publish_time),
|
|
64
|
+
body: messageBody,
|
|
65
|
+
bodyTxt: rawBody,
|
|
66
|
+
extraAttributes: (0, lodash_omit_1.default)(rawMessage.message?.attributes || {}, ['source_system', 'target_system', 'process_started_at', 'record_modified_at', 'original_publish_time', 'buildship_id', 'publish_time', 'bigquery']),
|
|
67
|
+
subscriptionName: rawMessage.subscription?.split('/').pop(),
|
|
68
|
+
};
|
|
69
|
+
return message;
|
|
70
|
+
}
|
|
71
|
+
async publish(options, payloads) {
|
|
72
|
+
if (!pubSubPublishTopics[options.topicName]) {
|
|
73
|
+
pubSubPublishTopics[options.topicName] = this.pubSub.topic(options.topicName, {
|
|
74
|
+
batching: {
|
|
75
|
+
maxMessages: 10,
|
|
76
|
+
maxMilliseconds: 15
|
|
77
|
+
},
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
const topic = pubSubPublishTopics[options.topicName];
|
|
81
|
+
const batches = payloads.length > 5 ? (0, lodash_chunk_1.default)(payloads, 10) : [payloads];
|
|
82
|
+
const attributes = {
|
|
83
|
+
...options.extraAttributes,
|
|
84
|
+
source_system: options.sourceSystem,
|
|
85
|
+
target_system: options.targetSystem,
|
|
86
|
+
buildship_id: this.systemEnvName + '-' + this.workflowId + '-' + this.triggerId,
|
|
87
|
+
};
|
|
88
|
+
const messageIds = [];
|
|
89
|
+
for (let i = 0; i < batches.length; i++) {
|
|
90
|
+
const batch = batches[i];
|
|
91
|
+
const results = await Promise.allSettled(batch.map((x) => topic.publishMessage({
|
|
92
|
+
data: Buffer.from(JSON.stringify(x)),
|
|
93
|
+
attributes: Object.fromEntries(Object.entries(options.attributesGenerator ? Object.assign({}, attributes, options.attributesGenerator(x)) : attributes).filter(([_, v]) => v !== undefined && v !== null).map(([k, v]) => [k, `${v}`]))
|
|
94
|
+
})));
|
|
95
|
+
for (const result of results) {
|
|
96
|
+
if (result.status === 'fulfilled') {
|
|
97
|
+
messageIds.push(result.value);
|
|
98
|
+
}
|
|
99
|
+
else {
|
|
100
|
+
messageIds.push(result.reason instanceof Error ? result.reason : new Error(result.reason));
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
if (i !== batches.length - 1) {
|
|
104
|
+
await new Promise(resolve => setTimeout(resolve, 30));
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
return messageIds;
|
|
108
|
+
}
|
|
109
|
+
async createOrUpdate(opts) {
|
|
110
|
+
const [topics] = await this.pubSub.getTopics();
|
|
111
|
+
let topicName = opts.topicName;
|
|
112
|
+
if (topicName.indexOf('rn.') !== 0 && /^\d/.test(topicName)) {
|
|
113
|
+
topicName = `rn.${topicName}`;
|
|
114
|
+
}
|
|
115
|
+
const mainSubscriptionName = opts.subscriptionName || topicName;
|
|
116
|
+
const tableSubscriptionName = `${mainSubscriptionName}.bq`;
|
|
117
|
+
const mainDlqTopicName = `${mainSubscriptionName}.dlq`;
|
|
118
|
+
const mainDlqSubscriptionName = `${mainSubscriptionName}.dlq`;
|
|
119
|
+
const tableDlqTopicName = `${mainSubscriptionName}.bq.dlq`;
|
|
120
|
+
const tableDlqSubscriptionName = `${mainSubscriptionName}.bq.dlq`;
|
|
121
|
+
const mainTopic = topics.find(x => this.getNameFromFullyQualifiedName(x.name) === topicName);
|
|
122
|
+
let mainDlqTopic = topics.find(x => this.getNameFromFullyQualifiedName(x.name) === mainDlqTopicName);
|
|
123
|
+
let tableDlqTopic = topics.find(x => this.getNameFromFullyQualifiedName(x.name) === tableDlqTopicName);
|
|
124
|
+
let mainTopicSubscriptions = null;
|
|
125
|
+
let mainDlqTopicSubscriptions = null;
|
|
126
|
+
let tableDlqTopicSubscriptions = null;
|
|
127
|
+
if (mainTopic) {
|
|
128
|
+
[mainTopicSubscriptions] = await mainTopic.getSubscriptions();
|
|
129
|
+
}
|
|
130
|
+
if (mainDlqTopic) {
|
|
131
|
+
[mainDlqTopicSubscriptions] = await mainDlqTopic.getSubscriptions();
|
|
132
|
+
}
|
|
133
|
+
if (tableDlqTopic) {
|
|
134
|
+
[tableDlqTopicSubscriptions] = await tableDlqTopic.getSubscriptions();
|
|
135
|
+
}
|
|
136
|
+
let mainSubscription = mainTopicSubscriptions?.find(x => this.getNameFromFullyQualifiedName(x.name) === mainSubscriptionName) || null;
|
|
137
|
+
let mainDlqSubscription = mainDlqTopicSubscriptions?.find(x => this.getNameFromFullyQualifiedName(x.name) === mainDlqSubscriptionName) || null;
|
|
138
|
+
let tableSubscription = mainTopicSubscriptions?.find(x => this.getNameFromFullyQualifiedName(x.name) === tableSubscriptionName) || null;
|
|
139
|
+
let tableDlqSubscription = tableDlqTopicSubscriptions?.find(x => this.getNameFromFullyQualifiedName(x.name) === tableDlqSubscriptionName) || null;
|
|
140
|
+
const topic = await this.ensureTopicExists(opts.topicName);
|
|
141
|
+
if (!opts.table) {
|
|
142
|
+
if (tableSubscription) {
|
|
143
|
+
await tableSubscription.delete();
|
|
144
|
+
tableSubscription = null;
|
|
145
|
+
}
|
|
146
|
+
if (tableDlqSubscription) {
|
|
147
|
+
await tableDlqSubscription.delete();
|
|
148
|
+
tableDlqSubscription = null;
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
else {
|
|
152
|
+
if (opts.createDeadLetterQueues) {
|
|
153
|
+
tableDlqTopic = await this.ensureTopicExists(tableDlqTopicName);
|
|
154
|
+
tableDlqSubscription = await this.ensureSubscribtionExists(tableDlqSubscriptionName, {
|
|
155
|
+
topic: tableDlqTopic,
|
|
156
|
+
dlqTopic: null,
|
|
157
|
+
});
|
|
158
|
+
}
|
|
159
|
+
tableSubscription = await this.ensureSubscribtionExists(tableSubscriptionName, {
|
|
160
|
+
topic: topic,
|
|
161
|
+
tableId: opts.table.tableId,
|
|
162
|
+
ackDeadline: opts.table.ackDeadline,
|
|
163
|
+
retryLimit: opts.table.retryLimit,
|
|
164
|
+
retryMinDelay: opts.table.retryMinDelay,
|
|
165
|
+
retryMaxDelay: opts.table.retryMaxDelay,
|
|
166
|
+
dlqTopic: tableDlqTopic || null
|
|
167
|
+
});
|
|
168
|
+
}
|
|
169
|
+
const pullPushSubscription = opts.pull;
|
|
170
|
+
if (!pullPushSubscription) {
|
|
171
|
+
if (mainSubscription) {
|
|
172
|
+
await mainSubscription.delete();
|
|
173
|
+
mainSubscription = null;
|
|
174
|
+
}
|
|
175
|
+
if (mainDlqSubscription) {
|
|
176
|
+
await mainDlqSubscription.delete();
|
|
177
|
+
mainDlqSubscription = null;
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
else {
|
|
181
|
+
if (opts.createDeadLetterQueues) {
|
|
182
|
+
mainDlqTopic = await this.ensureTopicExists(mainDlqTopicName);
|
|
183
|
+
mainDlqSubscription = await this.ensureSubscribtionExists(mainDlqSubscriptionName, {
|
|
184
|
+
topic: mainDlqTopic,
|
|
185
|
+
dlqTopic: null,
|
|
186
|
+
});
|
|
187
|
+
}
|
|
188
|
+
mainSubscription = await this.ensureSubscribtionExists(mainSubscriptionName, {
|
|
189
|
+
topic,
|
|
190
|
+
dlqTopic: mainDlqTopic || null,
|
|
191
|
+
exactlyOnceDelivery: true,
|
|
192
|
+
subscriptionFilter: opts.subscriptionFilter,
|
|
193
|
+
retryMinDelay: pullPushSubscription.retryMinDelay,
|
|
194
|
+
retryMaxDelay: pullPushSubscription.retryMaxDelay,
|
|
195
|
+
retryLimit: pullPushSubscription.retryLimit,
|
|
196
|
+
ackDeadline: pullPushSubscription.ackDeadline
|
|
197
|
+
});
|
|
198
|
+
}
|
|
199
|
+
if (!opts.createDeadLetterQueues) {
|
|
200
|
+
if (mainDlqSubscription) {
|
|
201
|
+
await mainDlqSubscription.delete();
|
|
202
|
+
mainDlqSubscription = null;
|
|
203
|
+
}
|
|
204
|
+
if (tableDlqSubscription) {
|
|
205
|
+
await tableDlqSubscription.delete();
|
|
206
|
+
tableDlqSubscription = null;
|
|
207
|
+
}
|
|
208
|
+
if (tableDlqTopic) {
|
|
209
|
+
await tableDlqTopic.delete();
|
|
210
|
+
}
|
|
211
|
+
if (mainDlqTopic)
|
|
212
|
+
await mainDlqTopic.delete();
|
|
213
|
+
}
|
|
214
|
+
return {
|
|
215
|
+
mainTopic: topic,
|
|
216
|
+
mainDlqTopic,
|
|
217
|
+
tableDlqTopic,
|
|
218
|
+
mainSubscription,
|
|
219
|
+
mainDlqSubscription,
|
|
220
|
+
tableDlqSubscription,
|
|
221
|
+
tableSubscription,
|
|
222
|
+
};
|
|
223
|
+
}
|
|
224
|
+
async findTopic(topicName) {
|
|
225
|
+
const [topics] = await this.pubSub.getTopics();
|
|
226
|
+
return topics.find(topic => this.getNameFromFullyQualifiedName(topic.name) === topicName);
|
|
227
|
+
}
|
|
228
|
+
async ensureTopicExists(topicName) {
|
|
229
|
+
let topic = await this.findTopic(topicName);
|
|
230
|
+
let topicMetadata;
|
|
231
|
+
try {
|
|
232
|
+
[topicMetadata] = await topic?.getMetadata() || [null];
|
|
233
|
+
}
|
|
234
|
+
catch (err) {
|
|
235
|
+
throw new Error('Invariant: While getting topic metadata - ' + err.message);
|
|
236
|
+
}
|
|
237
|
+
const metadata = {
|
|
238
|
+
labels: {
|
|
239
|
+
'bs-envname': this.systemEnvName.toLowerCase(),
|
|
240
|
+
'env': this.envName.toLowerCase(),
|
|
241
|
+
'bs-is-deadletter': topicName.indexOf('.dlq') > -1 ? '1' : '0',
|
|
242
|
+
'respectful-nudge': '1'
|
|
243
|
+
}
|
|
244
|
+
};
|
|
245
|
+
if (topic && !process.env.PUBSUB_EMULATOR_HOST) {
|
|
246
|
+
const simMetadata = Object.assign({}, topicMetadata?.labels || {}, metadata.labels);
|
|
247
|
+
if (!(0, lodash_isequal_1.default)(topicMetadata?.labels, simMetadata)) {
|
|
248
|
+
simMetadata['last-modified-at'] = new Date().getTime().toString();
|
|
249
|
+
try {
|
|
250
|
+
await topic.setMetadata({
|
|
251
|
+
labels: simMetadata
|
|
252
|
+
});
|
|
253
|
+
}
|
|
254
|
+
catch (err) {
|
|
255
|
+
this.logging.error('ensureTopicExists received error while setting metadata', err.message, err.stack);
|
|
256
|
+
throw err;
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
else if (!topic) {
|
|
261
|
+
const payload = {
|
|
262
|
+
name: topicName,
|
|
263
|
+
};
|
|
264
|
+
if (!process.env.PUBSUB_EMULATOR_HOST) {
|
|
265
|
+
payload.labels = metadata.labels;
|
|
266
|
+
}
|
|
267
|
+
await this.pubSub.createTopic(payload);
|
|
268
|
+
topic = await this.findTopic(topicName);
|
|
269
|
+
}
|
|
270
|
+
if (!topic) {
|
|
271
|
+
throw new Error(`Invariant: While creating topic - topic is not found`);
|
|
272
|
+
}
|
|
273
|
+
return topic;
|
|
274
|
+
}
|
|
275
|
+
getNameFromFullyQualifiedName(fullyQName) {
|
|
276
|
+
const name = fullyQName.split('/');
|
|
277
|
+
return name[name.length - 1];
|
|
278
|
+
}
|
|
279
|
+
async updateSubscription(subscriptionName, subscription, subscriptionMetaData, inputOpts) {
|
|
280
|
+
const defaultOptions = {
|
|
281
|
+
retryLimit: 5,
|
|
282
|
+
retryMinDelay: 10,
|
|
283
|
+
retryMaxDelay: 360,
|
|
284
|
+
ackDeadline: 90,
|
|
285
|
+
exactlyOnceDelivery: false
|
|
286
|
+
};
|
|
287
|
+
const opts = Object.assign({}, defaultOptions, inputOpts);
|
|
288
|
+
let nextMetadata = null;
|
|
289
|
+
if (!isSubscriptionBigQuery(opts)) {
|
|
290
|
+
if (subscriptionMetaData?.pushConfig?.pushEndpoint) {
|
|
291
|
+
nextMetadata = nextMetadata || {};
|
|
292
|
+
nextMetadata.pushConfig = null;
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
if (opts.dlqTopic) {
|
|
296
|
+
if (opts.dlqTopic.name !== subscriptionMetaData?.deadLetterPolicy?.deadLetterTopic
|
|
297
|
+
|| opts.retryLimit !== subscriptionMetaData?.deadLetterPolicy?.maxDeliveryAttempts) {
|
|
298
|
+
nextMetadata = nextMetadata || {};
|
|
299
|
+
nextMetadata.deadLetterPolicy = {
|
|
300
|
+
deadLetterTopic: opts.dlqTopic.name,
|
|
301
|
+
maxDeliveryAttempts: opts.retryLimit
|
|
302
|
+
};
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
else if (subscriptionMetaData?.deadLetterPolicy?.deadLetterTopic) {
|
|
306
|
+
nextMetadata = nextMetadata || {};
|
|
307
|
+
nextMetadata.deadLetterPolicy = null;
|
|
308
|
+
}
|
|
309
|
+
if (opts.exactlyOnceDelivery !== subscriptionMetaData?.enableExactlyOnceDelivery) {
|
|
310
|
+
if (typeof opts.exactlyOnceDelivery === 'boolean' && !isSubscriptionBigQuery(opts)) {
|
|
311
|
+
nextMetadata = nextMetadata || {};
|
|
312
|
+
nextMetadata.enableExactlyOnceDelivery = opts.exactlyOnceDelivery;
|
|
313
|
+
}
|
|
314
|
+
else {
|
|
315
|
+
nextMetadata = nextMetadata || {};
|
|
316
|
+
nextMetadata.enableExactlyOnceDelivery = null;
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
if (!isSubscriptionBigQuery(opts)) {
|
|
320
|
+
if ((opts.subscriptionFilter && subscriptionMetaData?.filter) && opts.subscriptionFilter !== subscriptionMetaData?.filter) {
|
|
321
|
+
nextMetadata = nextMetadata || {};
|
|
322
|
+
nextMetadata.filter = opts.subscriptionFilter || null;
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
if (opts.retryMinDelay != subscriptionMetaData?.retryPolicy?.minimumBackoff?.seconds
|
|
326
|
+
|| opts.retryMaxDelay != subscriptionMetaData?.retryPolicy?.maximumBackoff?.seconds) {
|
|
327
|
+
nextMetadata = nextMetadata || {};
|
|
328
|
+
nextMetadata.retryPolicy = {
|
|
329
|
+
minimumBackoff: { seconds: opts.retryMinDelay },
|
|
330
|
+
maximumBackoff: { seconds: opts.retryMaxDelay }
|
|
331
|
+
};
|
|
332
|
+
}
|
|
333
|
+
const proposedLabels = {
|
|
334
|
+
'env': this.envName.toLowerCase(),
|
|
335
|
+
'bs-workflowid': this.workflowId.toLowerCase(),
|
|
336
|
+
'bs-envname': this.systemEnvName.toLowerCase(),
|
|
337
|
+
'bs-is-deadletter': subscriptionName.indexOf('.dlq') > -1 ? '1' : '0',
|
|
338
|
+
'bs-bigquery-tableid': isSubscriptionBigQuery(opts) && opts.tableId ? opts.tableId.replace(/\./g, '-').toLowerCase() : '',
|
|
339
|
+
'bs-is-bigquery': isSubscriptionBigQuery(opts) && opts.tableId ? '1' : '0',
|
|
340
|
+
'respectful-nudge': '1'
|
|
341
|
+
};
|
|
342
|
+
const simLabels = Object.assign({}, subscriptionMetaData?.labels || {}, proposedLabels);
|
|
343
|
+
if (!(0, lodash_isequal_1.default)(simLabels, subscriptionMetaData?.labels)) {
|
|
344
|
+
nextMetadata = nextMetadata || {};
|
|
345
|
+
nextMetadata.labels = simLabels;
|
|
346
|
+
}
|
|
347
|
+
if (opts.ackDeadline !== subscriptionMetaData?.ackDeadlineSeconds) {
|
|
348
|
+
nextMetadata = nextMetadata || {};
|
|
349
|
+
nextMetadata.ackDeadlineSeconds = opts.ackDeadline;
|
|
350
|
+
}
|
|
351
|
+
if (isSubscriptionBigQuery(opts)) {
|
|
352
|
+
if (opts.tableId !== subscriptionMetaData?.bigqueryConfig?.table || this.serviceAccountEmail !== subscriptionMetaData?.bigqueryConfig?.serviceAccountEmail) {
|
|
353
|
+
if (opts.tableId) {
|
|
354
|
+
nextMetadata = nextMetadata || {};
|
|
355
|
+
nextMetadata.bigqueryConfig = {
|
|
356
|
+
table: opts.tableId,
|
|
357
|
+
writeMetadata: true,
|
|
358
|
+
dropUnknownFields: true,
|
|
359
|
+
serviceAccountEmail: this.serviceAccountEmail,
|
|
360
|
+
useTableSchema: false,
|
|
361
|
+
};
|
|
362
|
+
}
|
|
363
|
+
else {
|
|
364
|
+
nextMetadata = nextMetadata || {};
|
|
365
|
+
nextMetadata.bigqueryConfig = null;
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
else if (subscriptionMetaData?.bigqueryConfig?.table) {
|
|
370
|
+
nextMetadata = nextMetadata || {};
|
|
371
|
+
nextMetadata.bigqueryConfig = null;
|
|
372
|
+
}
|
|
373
|
+
this.logging.log('ensureSubscribtionExists updating subscription ' + subscriptionName, nextMetadata);
|
|
374
|
+
if (subscription && nextMetadata) {
|
|
375
|
+
try {
|
|
376
|
+
if (isSubscriptionBigQuery(opts) && opts.tableId) {
|
|
377
|
+
nextMetadata.bigqueryConfig = {
|
|
378
|
+
table: opts.tableId,
|
|
379
|
+
writeMetadata: true,
|
|
380
|
+
dropUnknownFields: true,
|
|
381
|
+
serviceAccountEmail: this.serviceAccountEmail,
|
|
382
|
+
useTableSchema: false,
|
|
383
|
+
};
|
|
384
|
+
}
|
|
385
|
+
await subscription.setMetadata(nextMetadata);
|
|
386
|
+
}
|
|
387
|
+
catch (err) {
|
|
388
|
+
this.logging.error('updateSubscription received error while setting metadata', err.message, err.stack);
|
|
389
|
+
throw new Error('Invariant: While updating subscription - ' + subscriptionName + ' - ' + err.message);
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
return nextMetadata;
|
|
393
|
+
}
|
|
394
|
+
async ensureSubscribtionExists(subscriptionName, opts) {
|
|
395
|
+
let [subscriptions] = await opts.topic.getSubscriptions();
|
|
396
|
+
let subscription = subscriptions.find((subscription) => this.getNameFromFullyQualifiedName(subscription.name) === subscriptionName) || null;
|
|
397
|
+
let subscriptionMetaData;
|
|
398
|
+
try {
|
|
399
|
+
[subscriptionMetaData] = (await subscription?.getMetadata()) || [null];
|
|
400
|
+
}
|
|
401
|
+
catch (err) {
|
|
402
|
+
throw new Error('Invariant: While getting subscription metadata - ' + err.message);
|
|
403
|
+
}
|
|
404
|
+
const createPayload = await this.updateSubscription(subscriptionName, subscription, subscriptionMetaData, opts);
|
|
405
|
+
if (createPayload && !subscription) {
|
|
406
|
+
this.logging.log('ensureSubscribtionExists creating subscription ' + subscriptionName, createPayload);
|
|
407
|
+
try {
|
|
408
|
+
await opts.topic.createSubscription(subscriptionName, createPayload);
|
|
409
|
+
}
|
|
410
|
+
catch (err) {
|
|
411
|
+
this.logging.error('ensureSubscribtionExists received error while creating subscription ' + subscriptionName, err.message, err.stack);
|
|
412
|
+
throw new Error('Invariant: While creating subscription - ' + err.message);
|
|
413
|
+
}
|
|
414
|
+
[subscriptions] = await opts.topic.getSubscriptions();
|
|
415
|
+
subscription = subscriptions.find(subscription => this.getNameFromFullyQualifiedName(subscription.name) === subscriptionName) || null;
|
|
416
|
+
}
|
|
417
|
+
if (!subscription) {
|
|
418
|
+
throw new Error(`Invariant: While creating subscription - subscription is not found`);
|
|
419
|
+
}
|
|
420
|
+
return subscription;
|
|
421
|
+
}
|
|
422
|
+
}
|
|
423
|
+
exports.default = PubSubHelper;
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.RedisConnectionError = void 0;
|
|
7
|
+
class RedisConnectionError extends Error {
|
|
8
|
+
constructor(message) {
|
|
9
|
+
super(message);
|
|
10
|
+
this.name = 'RedisConnectionError';
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
exports.RedisConnectionError = RedisConnectionError;
|
|
14
|
+
const runtime_1 = __importDefault(require("./runtime"));
|
|
15
|
+
function normalizeId(id) {
|
|
16
|
+
return encodeURIComponent(id);
|
|
17
|
+
}
|
|
18
|
+
class RateLimit extends runtime_1.default {
|
|
19
|
+
redisInstance;
|
|
20
|
+
constructor(opts, redisInstance) {
|
|
21
|
+
super(opts);
|
|
22
|
+
this.redisInstance = redisInstance;
|
|
23
|
+
}
|
|
24
|
+
async acquireLock(id, maxConcurrentExecutions, expirationInSeconds = 60 * 5) {
|
|
25
|
+
const key = `ratelimit:${normalizeId(id)}`;
|
|
26
|
+
const now = Date.now();
|
|
27
|
+
const lockInfo = {
|
|
28
|
+
id,
|
|
29
|
+
workflowId: this.workflowId,
|
|
30
|
+
triggerId: this.triggerId,
|
|
31
|
+
url: this.runtimeUrl,
|
|
32
|
+
envName: this.envName,
|
|
33
|
+
creation_time: now,
|
|
34
|
+
expiration_time: now + expirationInSeconds * 1000,
|
|
35
|
+
};
|
|
36
|
+
try {
|
|
37
|
+
const timeout = 300;
|
|
38
|
+
const redis = this.redisInstance;
|
|
39
|
+
if (!redis.isOpen) {
|
|
40
|
+
try {
|
|
41
|
+
await Promise.race([
|
|
42
|
+
redis.connect(),
|
|
43
|
+
new Promise((_, reject) => setTimeout(() => reject(new RedisConnectionError('Redis connect timeout')), timeout)),
|
|
44
|
+
]);
|
|
45
|
+
}
|
|
46
|
+
catch (err) {
|
|
47
|
+
if (err instanceof RedisConnectionError)
|
|
48
|
+
throw err;
|
|
49
|
+
throw new RedisConnectionError(err.message);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
await Promise.race([
|
|
53
|
+
redis.zRemRangeByScore(key, '-inf', now),
|
|
54
|
+
new Promise((_, reject) => setTimeout(() => reject(new Error('Redis timeout')), timeout)),
|
|
55
|
+
]);
|
|
56
|
+
const count = await Promise.race([
|
|
57
|
+
redis.zCard(key),
|
|
58
|
+
new Promise((_, reject) => setTimeout(() => reject(new Error('Redis timeout')), timeout)),
|
|
59
|
+
]);
|
|
60
|
+
if (count >= maxConcurrentExecutions) {
|
|
61
|
+
return false;
|
|
62
|
+
}
|
|
63
|
+
await Promise.race([
|
|
64
|
+
redis.zAdd(key, { score: lockInfo.expiration_time, value: JSON.stringify(lockInfo) }),
|
|
65
|
+
new Promise((_, reject) => setTimeout(() => reject(new Error('Redis timeout')), timeout)),
|
|
66
|
+
]);
|
|
67
|
+
await Promise.race([
|
|
68
|
+
redis.expire(key, expirationInSeconds),
|
|
69
|
+
new Promise((_, reject) => setTimeout(() => reject(new Error('Redis timeout')), timeout)),
|
|
70
|
+
]);
|
|
71
|
+
return true;
|
|
72
|
+
}
|
|
73
|
+
catch (err) {
|
|
74
|
+
if (err instanceof RedisConnectionError)
|
|
75
|
+
throw err;
|
|
76
|
+
return false;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
async releaseLock(id) {
|
|
80
|
+
const key = `ratelimit:${normalizeId(id)}`;
|
|
81
|
+
const redis = this.redisInstance;
|
|
82
|
+
const timeout = 300;
|
|
83
|
+
if (!redis.isOpen) {
|
|
84
|
+
try {
|
|
85
|
+
await Promise.race([
|
|
86
|
+
redis.connect(),
|
|
87
|
+
new Promise((_, reject) => setTimeout(() => reject(new RedisConnectionError('Redis connect timeout')), timeout)),
|
|
88
|
+
]);
|
|
89
|
+
}
|
|
90
|
+
catch (err) {
|
|
91
|
+
if (err instanceof RedisConnectionError)
|
|
92
|
+
throw err;
|
|
93
|
+
throw new RedisConnectionError(err.message);
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
const locks = await redis.zRange(key, 0, -1);
|
|
97
|
+
const filtered = locks.filter((lockStr) => {
|
|
98
|
+
try {
|
|
99
|
+
const lock = JSON.parse(lockStr);
|
|
100
|
+
return lock.workflowId === this.workflowId && lock.triggerId === this.triggerId && lock.envName === this.envName && lock.url === this.runtimeUrl;
|
|
101
|
+
}
|
|
102
|
+
catch {
|
|
103
|
+
return false;
|
|
104
|
+
}
|
|
105
|
+
});
|
|
106
|
+
for (const lockStr of filtered) {
|
|
107
|
+
await redis.zRem(key, lockStr);
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
async viewLocks(id) {
|
|
111
|
+
const key = `ratelimit:${normalizeId(id)}`;
|
|
112
|
+
const redis = this.redisInstance;
|
|
113
|
+
const timeout = 300;
|
|
114
|
+
if (!redis.isOpen) {
|
|
115
|
+
try {
|
|
116
|
+
await Promise.race([
|
|
117
|
+
redis.connect(),
|
|
118
|
+
new Promise((_, reject) => setTimeout(() => reject(new RedisConnectionError('Redis connect timeout')), timeout)),
|
|
119
|
+
]);
|
|
120
|
+
}
|
|
121
|
+
catch (err) {
|
|
122
|
+
if (err instanceof RedisConnectionError)
|
|
123
|
+
throw err;
|
|
124
|
+
throw new RedisConnectionError(err.message);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
const now = Date.now();
|
|
128
|
+
await redis.zRemRangeByScore(key, '-inf', now);
|
|
129
|
+
const locks = await redis.zRange(key, 0, -1);
|
|
130
|
+
return locks.reduce((acc, lockStr) => {
|
|
131
|
+
try {
|
|
132
|
+
acc.push(JSON.parse(lockStr));
|
|
133
|
+
}
|
|
134
|
+
catch {
|
|
135
|
+
}
|
|
136
|
+
return acc;
|
|
137
|
+
}, []);
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
exports.default = RateLimit;
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.default = getRedisClient;
|
|
4
|
+
exports.getConnectedRedisClient = getConnectedRedisClient;
|
|
5
|
+
const redis_1 = require("redis");
|
|
6
|
+
global.REDIS_CLIENTS = global.REDIS_CLIENTS || {};
|
|
7
|
+
function getRedisClient(sysEnv) {
|
|
8
|
+
const jsonEnv = JSON.parse(process.env[`${sysEnv}ProjectEnv`] || '{}');
|
|
9
|
+
const redisEndpoint = jsonEnv['redis_host'];
|
|
10
|
+
let REDIS_CLIENT = global.REDIS_CLIENTS;
|
|
11
|
+
if (!REDIS_CLIENTS[redisEndpoint]) {
|
|
12
|
+
if (!redisEndpoint && process.env.K_REVISION) {
|
|
13
|
+
throw new Error('Invariant: Redis endpoint is not set in the Environment Variables. Please set the redis_host with a value of IP:PORT i.e. 10.67.130.27:6379');
|
|
14
|
+
}
|
|
15
|
+
else if (!redisEndpoint) {
|
|
16
|
+
return null;
|
|
17
|
+
}
|
|
18
|
+
REDIS_CLIENT = (0, redis_1.createClient)({
|
|
19
|
+
url: `redis://${redisEndpoint}`
|
|
20
|
+
});
|
|
21
|
+
global.REDIS_CLIENTS[redisEndpoint] = REDIS_CLIENT;
|
|
22
|
+
}
|
|
23
|
+
return global.REDIS_CLIENTS[redisEndpoint];
|
|
24
|
+
}
|
|
25
|
+
function getConnectedRedisClient(url) {
|
|
26
|
+
const redisClient = getRedisClient(url);
|
|
27
|
+
if (!redisClient) {
|
|
28
|
+
return Promise.resolve(null);
|
|
29
|
+
}
|
|
30
|
+
let race = Promise.resolve(redisClient);
|
|
31
|
+
if (!redisClient.isOpen) {
|
|
32
|
+
race = Promise.race([
|
|
33
|
+
redisClient.connect().then(() => redisClient).catch((err) => {
|
|
34
|
+
console.log('Hush - Redis connect error', err.message);
|
|
35
|
+
return null;
|
|
36
|
+
}),
|
|
37
|
+
new Promise((resolve) => setTimeout(() => {
|
|
38
|
+
console.log('Hush - Redis connect timeout');
|
|
39
|
+
resolve(null);
|
|
40
|
+
}, 100)),
|
|
41
|
+
]);
|
|
42
|
+
}
|
|
43
|
+
return race;
|
|
44
|
+
}
|