@shushed/helpers 0.0.198-v2-20251104214132 → 0.0.198-v2-20251105075102
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -3,6 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
3
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.isSubscriptionBigQuery = isSubscriptionBigQuery;
|
|
6
7
|
const co_body_1 = __importDefault(require("co-body"));
|
|
7
8
|
const lodash_chunk_1 = __importDefault(require("lodash.chunk"));
|
|
8
9
|
const lodash_isequal_1 = __importDefault(require("lodash.isequal"));
|
|
@@ -14,11 +15,8 @@ function isPubSubMessage(req) {
|
|
|
14
15
|
}
|
|
15
16
|
const pubSubPublishTopics = global.HUSH_PUBSUB_PUBLISH_TOPICS || {};
|
|
16
17
|
global.HUSH_PUBSUB_PUBLISH_TOPICS = pubSubPublishTopics;
|
|
17
|
-
function
|
|
18
|
-
return '
|
|
19
|
-
}
|
|
20
|
-
function isSubscriptionOptionsPush(opts) {
|
|
21
|
-
return 'endpointUrl' in opts && !!opts.endpointUrl && (opts.endpointUrl.startsWith('https://') || opts.endpointUrl.startsWith('http://'));
|
|
18
|
+
function isSubscriptionBigQuery(opts) {
|
|
19
|
+
return 'tableId' in opts && opts.tableId !== null;
|
|
22
20
|
}
|
|
23
21
|
class PubSubHelper extends runtime_1.default {
|
|
24
22
|
serviceAccountEmail;
|
|
@@ -144,7 +142,7 @@ class PubSubHelper extends runtime_1.default {
|
|
|
144
142
|
let tableSubscription = mainTopicSubscriptions?.find(x => this.getNameFromFullyQualifiedName(x.name) === tableSubscriptionName) || null;
|
|
145
143
|
let tableDlqSubscription = tableDlqTopicSubscriptions?.find(x => this.getNameFromFullyQualifiedName(x.name) === tableDlqSubscriptionName) || null;
|
|
146
144
|
const topic = await this.ensureTopicExists(opts.topicName);
|
|
147
|
-
if (opts.table || opts.pull
|
|
145
|
+
if (opts.table || opts.pull) {
|
|
148
146
|
mainTopic = await this.ensureTopicExists(opts.topicName);
|
|
149
147
|
}
|
|
150
148
|
if (!opts.table) {
|
|
@@ -162,7 +160,6 @@ class PubSubHelper extends runtime_1.default {
|
|
|
162
160
|
tableDlqTopic = await this.ensureTopicExists(tableDlqTopicName);
|
|
163
161
|
tableDlqSubscription = await this.ensureSubscribtionExists(tableDlqSubscriptionName, {
|
|
164
162
|
topic: tableDlqTopic,
|
|
165
|
-
endpointUrl: null,
|
|
166
163
|
dlqTopic: null,
|
|
167
164
|
});
|
|
168
165
|
}
|
|
@@ -176,7 +173,7 @@ class PubSubHelper extends runtime_1.default {
|
|
|
176
173
|
dlqTopic: tableDlqTopic || null
|
|
177
174
|
});
|
|
178
175
|
}
|
|
179
|
-
const pullPushSubscription = opts.pull
|
|
176
|
+
const pullPushSubscription = opts.pull;
|
|
180
177
|
if (!pullPushSubscription) {
|
|
181
178
|
if (mainSubscription) {
|
|
182
179
|
await mainSubscription.delete();
|
|
@@ -193,19 +190,17 @@ class PubSubHelper extends runtime_1.default {
|
|
|
193
190
|
mainDlqSubscription = await this.ensureSubscribtionExists(mainDlqSubscriptionName, {
|
|
194
191
|
topic: mainDlqTopic,
|
|
195
192
|
dlqTopic: null,
|
|
196
|
-
endpointUrl: null,
|
|
197
193
|
});
|
|
198
194
|
}
|
|
199
195
|
mainSubscription = await this.ensureSubscribtionExists(mainSubscriptionName, {
|
|
200
196
|
topic,
|
|
201
197
|
dlqTopic: mainDlqTopic || null,
|
|
202
|
-
exactlyOnceDelivery:
|
|
198
|
+
exactlyOnceDelivery: true,
|
|
203
199
|
subscriptionFilter: opts.subscriptionFilter,
|
|
204
|
-
endpointUrl: opts.push ? this.runtimeUrl + `/executeWorkflow/${this.workflowId}/${this.triggerId}?topic_name=${opts.topicName}` : null,
|
|
205
200
|
retryMinDelay: pullPushSubscription.retryMinDelay,
|
|
206
201
|
retryMaxDelay: pullPushSubscription.retryMaxDelay,
|
|
207
202
|
retryLimit: pullPushSubscription.retryLimit,
|
|
208
|
-
ackDeadline:
|
|
203
|
+
ackDeadline: pullPushSubscription.ackDeadline
|
|
209
204
|
});
|
|
210
205
|
}
|
|
211
206
|
if (!opts.createDeadLetterQueues) {
|
|
@@ -296,46 +291,13 @@ class PubSubHelper extends runtime_1.default {
|
|
|
296
291
|
};
|
|
297
292
|
const opts = Object.assign({}, defaultOptions, inputOpts);
|
|
298
293
|
let nextMetadata = null;
|
|
299
|
-
if (
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|| process.env.GCLOUD_PROJECT + "/" + this.triggerId !== subscriptionMetaData.pushConfig.oidcToken.audience) {
|
|
303
|
-
if (subscriptionMetaData?.pushConfig?.pushEndpoint && subscription) {
|
|
304
|
-
try {
|
|
305
|
-
await subscription.modifyPushConfig({
|
|
306
|
-
pushEndpoint: opts.endpointUrl,
|
|
307
|
-
oidcToken: {
|
|
308
|
-
serviceAccountEmail: this.serviceAccountEmail,
|
|
309
|
-
audience: process.env.GCLOUD_PROJECT + "/" + this.triggerId,
|
|
310
|
-
},
|
|
311
|
-
});
|
|
312
|
-
}
|
|
313
|
-
catch (err) {
|
|
314
|
-
this.logging.error('Invariant: While updating push subscription', err.message, err.stack);
|
|
315
|
-
throw new Error('Invariant: While updating push subscription - ' + err.message);
|
|
316
|
-
}
|
|
317
|
-
}
|
|
318
|
-
else if (opts.endpointUrl) {
|
|
319
|
-
nextMetadata = nextMetadata || {};
|
|
320
|
-
nextMetadata.pushConfig = {
|
|
321
|
-
pushEndpoint: opts.endpointUrl,
|
|
322
|
-
oidcToken: {
|
|
323
|
-
serviceAccountEmail: this.serviceAccountEmail,
|
|
324
|
-
audience: process.env.GCLOUD_PROJECT + "/" + this.triggerId,
|
|
325
|
-
},
|
|
326
|
-
};
|
|
327
|
-
}
|
|
328
|
-
}
|
|
329
|
-
}
|
|
330
|
-
else {
|
|
331
|
-
if (subscriptionMetaData?.pushConfig?.pushEndpoint) {
|
|
332
|
-
nextMetadata = nextMetadata || {};
|
|
333
|
-
nextMetadata.pushConfig = null;
|
|
334
|
-
}
|
|
294
|
+
if (subscriptionMetaData?.pushConfig?.pushEndpoint) {
|
|
295
|
+
nextMetadata = nextMetadata || {};
|
|
296
|
+
nextMetadata.pushConfig = null;
|
|
335
297
|
}
|
|
336
298
|
if (opts.dlqTopic) {
|
|
337
299
|
if (opts.dlqTopic.name !== subscriptionMetaData?.deadLetterPolicy?.deadLetterTopic
|
|
338
|
-
|| opts.retryLimit !== subscriptionMetaData?.deadLetterPolicy
|
|
300
|
+
|| opts.retryLimit !== subscriptionMetaData?.deadLetterPolicy?.maxDeliveryAttempts) {
|
|
339
301
|
nextMetadata = nextMetadata || {};
|
|
340
302
|
nextMetadata.deadLetterPolicy = {
|
|
341
303
|
deadLetterTopic: opts.dlqTopic.name,
|
|
@@ -348,7 +310,7 @@ class PubSubHelper extends runtime_1.default {
|
|
|
348
310
|
nextMetadata.deadLetterPolicy = null;
|
|
349
311
|
}
|
|
350
312
|
if (opts.exactlyOnceDelivery !== subscriptionMetaData?.enableExactlyOnceDelivery) {
|
|
351
|
-
if (typeof opts.exactlyOnceDelivery === 'boolean' &&
|
|
313
|
+
if (typeof opts.exactlyOnceDelivery === 'boolean' && !isSubscriptionBigQuery(opts)) {
|
|
352
314
|
nextMetadata = nextMetadata || {};
|
|
353
315
|
nextMetadata.enableExactlyOnceDelivery = opts.exactlyOnceDelivery;
|
|
354
316
|
}
|
|
@@ -359,7 +321,7 @@ class PubSubHelper extends runtime_1.default {
|
|
|
359
321
|
}
|
|
360
322
|
if ((opts.subscriptionFilter && subscriptionMetaData?.filter) && opts.subscriptionFilter !== subscriptionMetaData?.filter) {
|
|
361
323
|
nextMetadata = nextMetadata || {};
|
|
362
|
-
nextMetadata.filter = opts.subscriptionFilter;
|
|
324
|
+
nextMetadata.filter = opts.subscriptionFilter || null;
|
|
363
325
|
}
|
|
364
326
|
if (opts.retryMinDelay != subscriptionMetaData?.retryPolicy?.minimumBackoff?.seconds
|
|
365
327
|
|| opts.retryMaxDelay != subscriptionMetaData?.retryPolicy?.maximumBackoff?.seconds) {
|
|
@@ -371,13 +333,11 @@ class PubSubHelper extends runtime_1.default {
|
|
|
371
333
|
}
|
|
372
334
|
const proposedLabels = {
|
|
373
335
|
'env': this.envName.toLowerCase(),
|
|
374
|
-
'bs-triggerid': this.triggerId.toLowerCase(),
|
|
375
336
|
'bs-workflowid': this.workflowId.toLowerCase(),
|
|
376
337
|
'bs-envname': this.systemEnvName.toLowerCase(),
|
|
377
338
|
'bs-is-deadletter': subscriptionName.indexOf('-dlq') > -1 ? 'true' : 'false',
|
|
378
|
-
'bs-
|
|
379
|
-
'bs-bigquery
|
|
380
|
-
'bs-is-bigquery': !(isSubscriptionOptionsPush(opts) || isSubscriptionOptionsPull(opts)) && opts.tableId ? 'true' : 'false',
|
|
339
|
+
'bs-bigquery-tableid': isSubscriptionBigQuery(opts) && opts.tableId ? opts.tableId.replace(/\./g, '-').toLowerCase() : '',
|
|
340
|
+
'bs-is-bigquery': isSubscriptionBigQuery(opts) && opts.tableId ? 'true' : 'false',
|
|
381
341
|
};
|
|
382
342
|
const simLabels = Object.assign({}, subscriptionMetaData?.labels || {}, proposedLabels);
|
|
383
343
|
if (!(0, lodash_isequal_1.default)(simLabels, subscriptionMetaData?.labels)) {
|
|
@@ -388,7 +348,7 @@ class PubSubHelper extends runtime_1.default {
|
|
|
388
348
|
nextMetadata = nextMetadata || {};
|
|
389
349
|
nextMetadata.ackDeadlineSeconds = opts.ackDeadline;
|
|
390
350
|
}
|
|
391
|
-
if (
|
|
351
|
+
if (isSubscriptionBigQuery(opts)) {
|
|
392
352
|
if (opts.tableId !== subscriptionMetaData?.bigqueryConfig?.table || this.serviceAccountEmail !== subscriptionMetaData?.bigqueryConfig?.serviceAccountEmail) {
|
|
393
353
|
if (opts.tableId) {
|
|
394
354
|
nextMetadata = nextMetadata || {};
|
|
@@ -423,7 +383,7 @@ class PubSubHelper extends runtime_1.default {
|
|
|
423
383
|
}
|
|
424
384
|
async ensureSubscribtionExists(subscriptionName, opts) {
|
|
425
385
|
let [subscriptions] = await opts.topic.getSubscriptions();
|
|
426
|
-
let subscription = subscriptions.find(subscription => this.getNameFromFullyQualifiedName(subscription.name) === subscriptionName) || null;
|
|
386
|
+
let subscription = subscriptions.find((subscription) => this.getNameFromFullyQualifiedName(subscription.name) === subscriptionName) || null;
|
|
427
387
|
let subscriptionMetaData;
|
|
428
388
|
try {
|
|
429
389
|
[subscriptionMetaData] = (await subscription?.getMetadata()) || [null];
|
|
@@ -433,7 +393,13 @@ class PubSubHelper extends runtime_1.default {
|
|
|
433
393
|
}
|
|
434
394
|
const createPayload = await this.updateSubscription(subscriptionName, subscription, subscriptionMetaData, opts);
|
|
435
395
|
if (createPayload && !subscription) {
|
|
436
|
-
|
|
396
|
+
try {
|
|
397
|
+
await opts.topic.createSubscription(subscriptionName, createPayload);
|
|
398
|
+
}
|
|
399
|
+
catch (err) {
|
|
400
|
+
this.logging.error('ensureSubscribtionExists received error while creating subscription ' + subscriptionName, err.message, err.stack);
|
|
401
|
+
throw new Error('Invariant: While creating subscription - ' + err.message);
|
|
402
|
+
}
|
|
437
403
|
[subscriptions] = await opts.topic.getSubscriptions();
|
|
438
404
|
subscription = subscriptions.find(subscription => this.getNameFromFullyQualifiedName(subscription.name) === subscriptionName) || null;
|
|
439
405
|
}
|
|
@@ -112,6 +112,9 @@ class Runtime {
|
|
|
112
112
|
if (options.env?.runtimeEnv) {
|
|
113
113
|
return options.env.runtimeEnv;
|
|
114
114
|
}
|
|
115
|
+
if (options.runtimeEnv) {
|
|
116
|
+
return options.runtimeEnv;
|
|
117
|
+
}
|
|
115
118
|
let systemEnvName = null;
|
|
116
119
|
if (options.env?.triggerEnv) {
|
|
117
120
|
systemEnvName = options.env.triggerEnv.systemEnvName;
|
|
@@ -122,7 +125,7 @@ class Runtime {
|
|
|
122
125
|
systemEnvName = hostId.split('-')[0];
|
|
123
126
|
}
|
|
124
127
|
}
|
|
125
|
-
return systemEnvName ||
|
|
128
|
+
return systemEnvName || 'test-env';
|
|
126
129
|
}
|
|
127
130
|
}
|
|
128
131
|
exports.default = Runtime;
|
|
@@ -4,6 +4,7 @@ import Runtime, { Opts as RuntimeOpts } from './runtime';
|
|
|
4
4
|
declare global {
|
|
5
5
|
var HUSH_PUBSUB_PUBLISH_TOPICS: Record<string, Topic>;
|
|
6
6
|
}
|
|
7
|
+
export declare function isSubscriptionBigQuery(opts: SubscriptionOptionsBigQuery | SubscriptionOptionsPull): opts is SubscriptionOptionsBigQuery;
|
|
7
8
|
export type Message = {
|
|
8
9
|
buildshipId: string;
|
|
9
10
|
subscriptionName?: string;
|
|
@@ -30,6 +31,26 @@ type MessageBody = {
|
|
|
30
31
|
} & {
|
|
31
32
|
[key: string]: any;
|
|
32
33
|
};
|
|
34
|
+
type SubscriptionOptionsPull = {
|
|
35
|
+
topic: Topic;
|
|
36
|
+
exactlyOnceDelivery?: boolean;
|
|
37
|
+
retryMinDelay?: number;
|
|
38
|
+
retryMaxDelay?: number;
|
|
39
|
+
retryLimit?: number;
|
|
40
|
+
dlqTopic: Topic | null;
|
|
41
|
+
ackDeadline?: number;
|
|
42
|
+
subscriptionFilter?: string | null;
|
|
43
|
+
};
|
|
44
|
+
type SubscriptionOptionsBigQuery = {
|
|
45
|
+
topic: Topic;
|
|
46
|
+
tableId: string;
|
|
47
|
+
dlqTopic: Topic | null;
|
|
48
|
+
retryMinDelay?: number;
|
|
49
|
+
retryMaxDelay?: number;
|
|
50
|
+
retryLimit?: number;
|
|
51
|
+
ackDeadline?: number;
|
|
52
|
+
subscriptionFilter?: string | null;
|
|
53
|
+
};
|
|
33
54
|
declare class PubSubHelper extends Runtime {
|
|
34
55
|
private serviceAccountEmail;
|
|
35
56
|
private pubSub;
|
|
@@ -62,12 +83,6 @@ declare class PubSubHelper extends Runtime {
|
|
|
62
83
|
retryMinDelay: number;
|
|
63
84
|
retryMaxDelay: number;
|
|
64
85
|
};
|
|
65
|
-
push?: null | {
|
|
66
|
-
retryLimit: number;
|
|
67
|
-
ackDeadline: number;
|
|
68
|
-
retryMinDelay: number;
|
|
69
|
-
retryMaxDelay: number;
|
|
70
|
-
};
|
|
71
86
|
table?: null | {
|
|
72
87
|
tableId: string;
|
|
73
88
|
retryLimit: number;
|