@dudousxd/nestjs-durable-transport-sqs 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +128 -36
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +36 -3
- package/dist/index.d.ts +36 -3
- package/dist/index.js +129 -37
- package/dist/index.js.map +1 -1
- package/package.json +13 -3
package/dist/index.cjs
CHANGED
|
@@ -28,6 +28,7 @@ module.exports = __toCommonJS(index_exports);
|
|
|
28
28
|
// src/sqs-transport.ts
|
|
29
29
|
var import_client_sqs = require("@aws-sdk/client-sqs");
|
|
30
30
|
var import_nestjs_durable_core = require("@dudousxd/nestjs-durable-core");
|
|
31
|
+
var RESULT_VISIBILITY_SEC = 30;
|
|
31
32
|
var SqsTransport = class {
|
|
32
33
|
static {
|
|
33
34
|
__name(this, "SqsTransport");
|
|
@@ -42,8 +43,11 @@ var SqsTransport = class {
|
|
|
42
43
|
visibilityTimeoutSec;
|
|
43
44
|
taskQueueUrlResolver;
|
|
44
45
|
resultsQueueUrlOption;
|
|
46
|
+
onError;
|
|
45
47
|
handlers = /* @__PURE__ */ new Map();
|
|
46
48
|
urlCache = /* @__PURE__ */ new Map();
|
|
49
|
+
/** `taskQueueUrl` resolutions per group (the promise, so concurrent dispatches share one call). */
|
|
50
|
+
groupUrlCache = /* @__PURE__ */ new Map();
|
|
47
51
|
running = true;
|
|
48
52
|
abort = new AbortController();
|
|
49
53
|
taskLoop;
|
|
@@ -59,6 +63,7 @@ var SqsTransport = class {
|
|
|
59
63
|
this.visibilityTimeoutSec = options.visibilityTimeoutSec ?? 60;
|
|
60
64
|
this.taskQueueUrlResolver = options.taskQueueUrl;
|
|
61
65
|
this.resultsQueueUrlOption = options.resultsQueueUrl;
|
|
66
|
+
this.onError = options.onError ?? ((err) => console.error("[SqsTransport]", err));
|
|
62
67
|
}
|
|
63
68
|
// ── queue resolution ───────────────────────────────────────────────────────────────────────
|
|
64
69
|
async resolveByName(name) {
|
|
@@ -84,8 +89,17 @@ var SqsTransport = class {
|
|
|
84
89
|
return url;
|
|
85
90
|
}
|
|
86
91
|
async tasksUrl(group) {
|
|
87
|
-
|
|
88
|
-
return this.resolveByName(`${this.prefix}-tasks-${group}`);
|
|
92
|
+
const resolver = this.taskQueueUrlResolver;
|
|
93
|
+
if (!resolver) return this.resolveByName(`${this.prefix}-tasks-${group}`);
|
|
94
|
+
let url = this.groupUrlCache.get(group);
|
|
95
|
+
if (!url) {
|
|
96
|
+
url = Promise.resolve().then(() => resolver(group)).catch((err) => {
|
|
97
|
+
this.groupUrlCache.delete(group);
|
|
98
|
+
throw err;
|
|
99
|
+
});
|
|
100
|
+
this.groupUrlCache.set(group, url);
|
|
101
|
+
}
|
|
102
|
+
return url;
|
|
89
103
|
}
|
|
90
104
|
async resultsUrl() {
|
|
91
105
|
const opt = this.resultsQueueUrlOption;
|
|
@@ -120,45 +134,123 @@ var SqsTransport = class {
|
|
|
120
134
|
onHeartbeat(_handler) {
|
|
121
135
|
}
|
|
122
136
|
// ── polling loops ──────────────────────────────────────────────────────────────────────────
|
|
123
|
-
|
|
124
|
-
|
|
137
|
+
/**
|
|
138
|
+
* Receive a batch and hand each message to `each`, one at a time, never letting a failure escape:
|
|
139
|
+
* a throw is reported through `onError` and its message is left on the queue (SQS redelivers it
|
|
140
|
+
* after the visibility timeout; a redrive policy DLQs one that keeps failing). Every message of the
|
|
141
|
+
* batch is kept invisible until it is handled (see {@link holdVisible}), so the ones waiting behind a
|
|
142
|
+
* slow handler are never redelivered to another consumer meanwhile.
|
|
143
|
+
*/
|
|
144
|
+
async pollLoop(queueUrl, visibilitySec, each) {
|
|
125
145
|
while (this.running) {
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
146
|
+
let QueueUrl;
|
|
147
|
+
try {
|
|
148
|
+
QueueUrl = await queueUrl();
|
|
149
|
+
} catch (err) {
|
|
150
|
+
if (!this.running) return;
|
|
151
|
+
this.onError(err);
|
|
152
|
+
await this.backoff();
|
|
153
|
+
continue;
|
|
154
|
+
}
|
|
155
|
+
const messages = await this.receive(QueueUrl, visibilitySec);
|
|
156
|
+
const pending = new Set(messages);
|
|
157
|
+
const stopHolding = this.holdVisible(QueueUrl, pending, visibilitySec);
|
|
158
|
+
try {
|
|
159
|
+
for (const msg of messages) {
|
|
160
|
+
if (!this.running) break;
|
|
161
|
+
try {
|
|
162
|
+
if (this.shouldSkip(msg)) {
|
|
163
|
+
pending.delete(msg);
|
|
164
|
+
await this.release(QueueUrl, msg.ReceiptHandle);
|
|
165
|
+
continue;
|
|
141
166
|
}
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
167
|
+
await each(QueueUrl, msg);
|
|
168
|
+
} catch (err) {
|
|
169
|
+
this.onError(err);
|
|
170
|
+
} finally {
|
|
171
|
+
pending.delete(msg);
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
} finally {
|
|
175
|
+
stopHolding();
|
|
145
176
|
}
|
|
146
177
|
}
|
|
147
178
|
}
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
179
|
+
/**
|
|
180
|
+
* Keep `messages` invisible while they wait or run: every third of the visibility timeout, extend
|
|
181
|
+
* each one still in the set back to the full timeout. Returns the stop function.
|
|
182
|
+
*/
|
|
183
|
+
holdVisible(QueueUrl, messages, visibilitySec) {
|
|
184
|
+
if (messages.size === 0) return () => {
|
|
185
|
+
};
|
|
186
|
+
const intervalMs = Math.max(200, Math.floor(visibilitySec * 1e3 / 3));
|
|
187
|
+
let round;
|
|
188
|
+
const timer = setInterval(() => {
|
|
189
|
+
if (round) return;
|
|
190
|
+
round = Promise.all([
|
|
191
|
+
...messages
|
|
192
|
+
].map((msg) => msg.ReceiptHandle ? this.client.send(new import_client_sqs.ChangeMessageVisibilityCommand({
|
|
193
|
+
QueueUrl,
|
|
194
|
+
ReceiptHandle: msg.ReceiptHandle,
|
|
195
|
+
VisibilityTimeout: visibilitySec
|
|
196
|
+
})).then(() => void 0).catch((err) => {
|
|
197
|
+
if (this.running) this.onError(err);
|
|
198
|
+
}) : void 0)).then(() => {
|
|
199
|
+
round = void 0;
|
|
200
|
+
});
|
|
201
|
+
}, intervalMs);
|
|
202
|
+
timer.unref?.();
|
|
203
|
+
return () => clearInterval(timer);
|
|
204
|
+
}
|
|
205
|
+
parse(msg) {
|
|
206
|
+
try {
|
|
207
|
+
return JSON.parse(msg.Body ?? "");
|
|
208
|
+
} catch {
|
|
209
|
+
throw new Error(`[SqsTransport] message ${msg.MessageId ?? "?"} has a non-JSON body; leaving it for the queue's redrive policy`);
|
|
160
210
|
}
|
|
161
211
|
}
|
|
212
|
+
pollTasks(group) {
|
|
213
|
+
return this.pollLoop(() => this.tasksUrl(group), this.visibilityTimeoutSec, async (QueueUrl, msg) => {
|
|
214
|
+
const task = this.parse(msg);
|
|
215
|
+
const result = await (0, import_nestjs_durable_core.runStepHandler)(task, this.handlers.get(task.name));
|
|
216
|
+
await this.client.send(new import_client_sqs.SendMessageCommand({
|
|
217
|
+
QueueUrl: await this.resultsUrl(),
|
|
218
|
+
MessageBody: JSON.stringify(result),
|
|
219
|
+
MessageAttributes: this.marker ? {
|
|
220
|
+
[this.marker]: {
|
|
221
|
+
DataType: "String",
|
|
222
|
+
StringValue: "1"
|
|
223
|
+
}
|
|
224
|
+
} : void 0
|
|
225
|
+
}));
|
|
226
|
+
await this.del(QueueUrl, msg.ReceiptHandle);
|
|
227
|
+
});
|
|
228
|
+
}
|
|
229
|
+
pollResults(handler) {
|
|
230
|
+
return this.pollLoop(() => this.resultsUrl(), RESULT_VISIBILITY_SEC, async (QueueUrl, msg) => {
|
|
231
|
+
await handler(this.parse(msg));
|
|
232
|
+
await this.del(QueueUrl, msg.ReceiptHandle);
|
|
233
|
+
});
|
|
234
|
+
}
|
|
235
|
+
/** Wait 1s, or until `close()` aborts. The abort listener is removed when the timer fires, so
|
|
236
|
+
* sustained failures never accumulate listeners on the shared signal. */
|
|
237
|
+
backoff() {
|
|
238
|
+
return new Promise((resolve) => {
|
|
239
|
+
const signal = this.abort.signal;
|
|
240
|
+
if (signal.aborted) return resolve();
|
|
241
|
+
const onAbort = /* @__PURE__ */ __name(() => {
|
|
242
|
+
clearTimeout(timer);
|
|
243
|
+
resolve();
|
|
244
|
+
}, "onAbort");
|
|
245
|
+
const timer = setTimeout(() => {
|
|
246
|
+
signal.removeEventListener("abort", onAbort);
|
|
247
|
+
resolve();
|
|
248
|
+
}, 1e3);
|
|
249
|
+
signal.addEventListener("abort", onAbort, {
|
|
250
|
+
once: true
|
|
251
|
+
});
|
|
252
|
+
});
|
|
253
|
+
}
|
|
162
254
|
/** One long-poll receive. Swallows the abort thrown on `close()`; logs and backs off otherwise. */
|
|
163
255
|
async receive(QueueUrl, visibility) {
|
|
164
256
|
try {
|
|
@@ -176,8 +268,8 @@ var SqsTransport = class {
|
|
|
176
268
|
return out.Messages ?? [];
|
|
177
269
|
} catch (err) {
|
|
178
270
|
if (!this.running) return [];
|
|
179
|
-
|
|
180
|
-
await
|
|
271
|
+
this.onError(err);
|
|
272
|
+
await this.backoff();
|
|
181
273
|
return [];
|
|
182
274
|
}
|
|
183
275
|
}
|
package/dist/index.cjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/index.ts","../src/sqs-transport.ts"],"sourcesContent":["export * from './sqs-transport';\n","import {\n ChangeMessageVisibilityCommand,\n CreateQueueCommand,\n DeleteMessageCommand,\n GetQueueUrlCommand,\n ReceiveMessageCommand,\n SendMessageCommand,\n SQSClient,\n type SQSClientConfig,\n} from '@aws-sdk/client-sqs';\nimport {\n type Heartbeat,\n type RemoteTask,\n type StepHandler,\n type StepResult,\n type Transport,\n runStepHandler,\n} from '@dudousxd/nestjs-durable-core';\n\n/** Resolve a group's tasks queue URL. Sync or async; resolved once and cached. */\nexport type QueueUrlResolver = (group: string) => string | Promise<string>;\n\nexport interface SqsTransportOptions {\n /**\n * Reuse an existing SQS client (recommended — share the app's). When omitted, one is built\n * from `clientConfig` and destroyed on `close()`.\n */\n client?: SQSClient;\n /** Used only when `client` is omitted — e.g. `{ region, endpoint, credentials }` for ElasticMQ. */\n clientConfig?: SQSClientConfig;\n\n /** The worker group this instance serves. Required to register `handle()` consumers. */\n group?: string;\n\n /**\n * Map a group → an **existing** tasks queue URL. This is the seam for reusing queues you already\n * have (e.g. `extraction`, `ingestion`) instead of letting the transport mint its own. When\n * omitted, the transport resolves `${prefix}-tasks-${group}` by name (and creates it if\n * `autoCreate`). Return value is awaited and cached.\n */\n taskQueueUrl?: QueueUrlResolver;\n /**\n * The results queue URL (worker → engine). A literal URL or a resolver. When omitted, the\n * transport resolves `${prefix}-results` by name (and creates it if `autoCreate`).\n */\n resultsQueueUrl?: string | (() => string | Promise<string>);\n\n /** Prefix for the fallback `${prefix}-tasks-*` / `${prefix}-results` names. Default `durable`. */\n prefix?: string;\n /**\n * Create the fallback-named queues on first use if they don't exist (handy for ElasticMQ / dev).\n * Ignored for queues you supply via `taskQueueUrl` / `resultsQueueUrl` — those must already exist.\n */\n autoCreate?: boolean;\n\n /**\n * When set, durable messages carry this SQS message attribute and pollers **skip** messages\n * that lack it (releasing them for a co-resident legacy consumer). Enables sharing one physical\n * queue with a non-durable consumer — but only if that consumer likewise ignores tagged messages.\n * Leave unset for the clean case: a queue dedicated to durable traffic.\n */\n marker?: string;\n\n /** SQS receive long-poll seconds (0–20). Default 20. */\n waitTimeSeconds?: number;\n /** Visibility timeout for an in-flight task, in seconds. Must exceed the step's runtime. Default 60. */\n visibilityTimeoutSec?: number;\n}\n\n/**\n * An SQS-backed `Transport` — dispatch remote steps over Amazon SQS (or ElasticMQ). Unlike the\n * BullMQ transport, it does **not** mint its own queue names: pass `taskQueueUrl` / `resultsQueueUrl`\n * to ride the queues you already operate. The wire payload is the documented `RemoteTask` /\n * `StepResult` JSON, so non-Node workers on the same queues interoperate.\n *\n * Run one instance engine-side (calls `onResult`, dispatches tasks) and one per worker process\n * (calls `handle()` for its group). SQS is at-least-once: a task whose process crashes before the\n * result is sent reappears after the visibility timeout (durable retry); a duplicate `StepResult`\n * is harmless because the engine resolves a step's `pending` exactly once by `stepId`.\n */\nexport class SqsTransport implements Transport {\n private readonly client: SQSClient;\n private readonly ownsClient: boolean;\n private readonly group?: string | undefined;\n private readonly prefix: string;\n private readonly autoCreate: boolean;\n private readonly marker?: string | undefined;\n private readonly waitTimeSeconds: number;\n private readonly visibilityTimeoutSec: number;\n private readonly taskQueueUrlResolver?: QueueUrlResolver | undefined;\n private readonly resultsQueueUrlOption?: string | (() => string | Promise<string>) | undefined;\n\n private readonly handlers = new Map<string, StepHandler>();\n private readonly urlCache = new Map<string, string>();\n private running = true;\n private readonly abort = new AbortController();\n private taskLoop?: Promise<void>;\n private resultLoop?: Promise<void>;\n\n constructor(options: SqsTransportOptions) {\n this.client = options.client ?? new SQSClient(options.clientConfig ?? {});\n this.ownsClient = !options.client;\n this.group = options.group;\n this.prefix = options.prefix ?? 'durable';\n this.autoCreate = options.autoCreate ?? false;\n this.marker = options.marker;\n this.waitTimeSeconds = options.waitTimeSeconds ?? 20;\n this.visibilityTimeoutSec = options.visibilityTimeoutSec ?? 60;\n this.taskQueueUrlResolver = options.taskQueueUrl;\n this.resultsQueueUrlOption = options.resultsQueueUrl;\n }\n\n // ── queue resolution ───────────────────────────────────────────────────────────────────────\n\n private async resolveByName(name: string): Promise<string> {\n const cached = this.urlCache.get(name);\n if (cached) return cached;\n let url: string;\n try {\n const out = await this.client.send(new GetQueueUrlCommand({ QueueName: name }));\n url = out.QueueUrl as string;\n } catch (err) {\n if (this.autoCreate) {\n const created = await this.client.send(new CreateQueueCommand({ QueueName: name }));\n url = created.QueueUrl as string;\n } else {\n throw err;\n }\n }\n this.urlCache.set(name, url);\n return url;\n }\n\n private async tasksUrl(group: string): Promise<string> {\n if (this.taskQueueUrlResolver) return this.taskQueueUrlResolver(group);\n return this.resolveByName(`${this.prefix}-tasks-${group}`);\n }\n\n private async resultsUrl(): Promise<string> {\n const opt = this.resultsQueueUrlOption;\n if (typeof opt === 'string') return opt;\n if (typeof opt === 'function') return opt();\n return this.resolveByName(`${this.prefix}-results`);\n }\n\n // ── engine → worker ────────────────────────────────────────────────────────────────────────\n\n async dispatch(task: RemoteTask): Promise<void> {\n const QueueUrl = await this.tasksUrl(task.group);\n await this.client.send(\n new SendMessageCommand({\n QueueUrl,\n MessageBody: JSON.stringify(task),\n MessageAttributes: this.marker\n ? { [this.marker]: { DataType: 'String', StringValue: '1' } }\n : undefined,\n }),\n );\n }\n\n // ── worker → engine ────────────────────────────────────────────────────────────────────────\n\n /** Register a step handler (worker side). Starts the group's task consumer on first call. */\n handle(name: string, fn: StepHandler): void {\n if (!this.group) throw new Error('SqsTransport needs a `group` to register handlers');\n this.handlers.set(name, fn);\n if (!this.taskLoop) this.taskLoop = this.pollTasks(this.group);\n }\n\n onResult(handler: (result: StepResult) => Promise<void>): void {\n if (!this.resultLoop) this.resultLoop = this.pollResults(handler);\n }\n\n onHeartbeat(_handler: (beat: Heartbeat) => Promise<void>): void {\n // Not modelled over SQS; the queue's visibility-timeout redelivery is the liveness mechanism.\n }\n\n // ── polling loops ──────────────────────────────────────────────────────────────────────────\n\n private async pollTasks(group: string): Promise<void> {\n const QueueUrl = await this.tasksUrl(group);\n while (this.running) {\n const messages = await this.receive(QueueUrl, this.visibilityTimeoutSec);\n for (const msg of messages) {\n if (this.shouldSkip(msg)) {\n // Not ours (shared queue): release immediately so the legacy consumer can take it.\n await this.release(QueueUrl, msg.ReceiptHandle);\n continue;\n }\n const task = JSON.parse(msg.Body ?? '{}') as RemoteTask;\n const result = await runStepHandler(task, this.handlers.get(task.name));\n await this.client.send(\n new SendMessageCommand({\n QueueUrl: await this.resultsUrl(),\n MessageBody: JSON.stringify(result),\n MessageAttributes: this.marker\n ? { [this.marker]: { DataType: 'String', StringValue: '1' } }\n : undefined,\n }),\n );\n await this.del(QueueUrl, msg.ReceiptHandle);\n }\n }\n }\n\n private async pollResults(handler: (result: StepResult) => Promise<void>): Promise<void> {\n const QueueUrl = await this.resultsUrl();\n while (this.running) {\n const messages = await this.receive(QueueUrl, 30);\n for (const msg of messages) {\n if (this.shouldSkip(msg)) {\n await this.release(QueueUrl, msg.ReceiptHandle);\n continue;\n }\n await handler(JSON.parse(msg.Body ?? '{}') as StepResult);\n await this.del(QueueUrl, msg.ReceiptHandle);\n }\n }\n }\n\n /** One long-poll receive. Swallows the abort thrown on `close()`; logs and backs off otherwise. */\n private async receive(QueueUrl: string, visibility: number) {\n try {\n const out = await this.client.send(\n new ReceiveMessageCommand({\n QueueUrl,\n MaxNumberOfMessages: 10,\n WaitTimeSeconds: this.waitTimeSeconds,\n VisibilityTimeout: visibility,\n MessageAttributeNames: this.marker ? ['All'] : undefined,\n }),\n { abortSignal: this.abort.signal },\n );\n return out.Messages ?? [];\n } catch (err) {\n if (!this.running) return [];\n console.error('[SqsTransport] receive failed', err);\n await new Promise((r) => setTimeout(r, 1000));\n return [];\n }\n }\n\n private shouldSkip(msg: { MessageAttributes?: Record<string, unknown> | undefined }): boolean {\n return !!this.marker && !msg.MessageAttributes?.[this.marker];\n }\n\n private async del(QueueUrl: string, ReceiptHandle?: string): Promise<void> {\n if (!ReceiptHandle) return;\n await this.client.send(new DeleteMessageCommand({ QueueUrl, ReceiptHandle }));\n }\n\n private async release(QueueUrl: string, ReceiptHandle?: string): Promise<void> {\n if (!ReceiptHandle) return;\n await this.client\n .send(new ChangeMessageVisibilityCommand({ QueueUrl, ReceiptHandle, VisibilityTimeout: 0 }))\n .catch(() => {});\n }\n\n /** Stop the pollers and (if we created it) destroy the SQS client so the process can exit. */\n async close(): Promise<void> {\n this.running = false;\n this.abort.abort();\n await Promise.allSettled([this.taskLoop, this.resultLoop]);\n if (this.ownsClient) this.client.destroy();\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAAA;;;;;;;ACAA,wBASO;AACP,iCAOO;AA+DA,IAAMA,eAAN,MAAMA;EAhFb,OAgFaA;;;EACMC;EACAC;EACAC;EACAC;EACAC;EACAC;EACAC;EACAC;EACAC;EACAC;EAEAC,WAAW,oBAAIC,IAAAA;EACfC,WAAW,oBAAID,IAAAA;EACxBE,UAAU;EACDC,QAAQ,IAAIC,gBAAAA;EACrBC;EACAC;EAER,YAAYC,SAA8B;AACxC,SAAKlB,SAASkB,QAAQlB,UAAU,IAAImB,4BAAUD,QAAQE,gBAAgB,CAAC,CAAA;AACvE,SAAKnB,aAAa,CAACiB,QAAQlB;AAC3B,SAAKE,QAAQgB,QAAQhB;AACrB,SAAKC,SAASe,QAAQf,UAAU;AAChC,SAAKC,aAAac,QAAQd,cAAc;AACxC,SAAKC,SAASa,QAAQb;AACtB,SAAKC,kBAAkBY,QAAQZ,mBAAmB;AAClD,SAAKC,uBAAuBW,QAAQX,wBAAwB;AAC5D,SAAKC,uBAAuBU,QAAQG;AACpC,SAAKZ,wBAAwBS,QAAQI;EACvC;;EAIA,MAAcC,cAAcC,MAA+B;AACzD,UAAMC,SAAS,KAAKb,SAASc,IAAIF,IAAAA;AACjC,QAAIC,OAAQ,QAAOA;AACnB,QAAIE;AACJ,QAAI;AACF,YAAMC,MAAM,MAAM,KAAK5B,OAAO6B,KAAK,IAAIC,qCAAmB;QAAEC,WAAWP;MAAK,CAAA,CAAA;AAC5EG,YAAMC,IAAII;IACZ,SAASC,KAAK;AACZ,UAAI,KAAK7B,YAAY;AACnB,cAAM8B,UAAU,MAAM,KAAKlC,OAAO6B,KAAK,IAAIM,qCAAmB;UAAEJ,WAAWP;QAAK,CAAA,CAAA;AAChFG,cAAMO,QAAQF;MAChB,OAAO;AACL,cAAMC;MACR;IACF;AACA,SAAKrB,SAASwB,IAAIZ,MAAMG,GAAAA;AACxB,WAAOA;EACT;EAEA,MAAcU,SAASnC,OAAgC;AACrD,QAAI,KAAKM,qBAAsB,QAAO,KAAKA,qBAAqBN,KAAAA;AAChE,WAAO,KAAKqB,cAAc,GAAG,KAAKpB,MAAM,UAAUD,KAAAA,EAAO;EAC3D;EAEA,MAAcoC,aAA8B;AAC1C,UAAMC,MAAM,KAAK9B;AACjB,QAAI,OAAO8B,QAAQ,SAAU,QAAOA;AACpC,QAAI,OAAOA,QAAQ,WAAY,QAAOA,IAAAA;AACtC,WAAO,KAAKhB,cAAc,GAAG,KAAKpB,MAAM,UAAU;EACpD;;EAIA,MAAMqC,SAASC,MAAiC;AAC9C,UAAMT,WAAW,MAAM,KAAKK,SAASI,KAAKvC,KAAK;AAC/C,UAAM,KAAKF,OAAO6B,KAChB,IAAIa,qCAAmB;MACrBV;MACAW,aAAaC,KAAKC,UAAUJ,IAAAA;MAC5BK,mBAAmB,KAAKzC,SACpB;QAAE,CAAC,KAAKA,MAAM,GAAG;UAAE0C,UAAU;UAAUC,aAAa;QAAI;MAAE,IAC1DC;IACN,CAAA,CAAA;EAEJ;;;EAKAC,OAAO1B,MAAc2B,IAAuB;AAC1C,QAAI,CAAC,KAAKjD,MAAO,OAAM,IAAIkD,MAAM,mDAAA;AACjC,SAAK1C,SAAS0B,IAAIZ,MAAM2B,EAAAA;AACxB,QAAI,CAAC,KAAKnC,SAAU,MAAKA,WAAW,KAAKqC,UAAU,KAAKnD,KAAK;EAC/D;EAEAoD,SAASC,SAAsD;AAC7D,QAAI,CAAC,KAAKtC,WAAY,MAAKA,aAAa,KAAKuC,YAAYD,OAAAA;EAC3D;EAEAE,YAAYC,UAAoD;EAEhE;;EAIA,MAAcL,UAAUnD,OAA8B;AACpD,UAAM8B,WAAW,MAAM,KAAKK,SAASnC,KAAAA;AACrC,WAAO,KAAKW,SAAS;AACnB,YAAM8C,WAAW,MAAM,KAAKC,QAAQ5B,UAAU,KAAKzB,oBAAoB;AACvE,iBAAWsD,OAAOF,UAAU;AAC1B,YAAI,KAAKG,WAAWD,GAAAA,GAAM;AAExB,gBAAM,KAAKE,QAAQ/B,UAAU6B,IAAIG,aAAa;AAC9C;QACF;AACA,cAAMvB,OAAOG,KAAKqB,MAAMJ,IAAIK,QAAQ,IAAA;AACpC,cAAMC,SAAS,UAAMC,2CAAe3B,MAAM,KAAK/B,SAASgB,IAAIe,KAAKjB,IAAI,CAAA;AACrE,cAAM,KAAKxB,OAAO6B,KAChB,IAAIa,qCAAmB;UACrBV,UAAU,MAAM,KAAKM,WAAU;UAC/BK,aAAaC,KAAKC,UAAUsB,MAAAA;UAC5BrB,mBAAmB,KAAKzC,SACpB;YAAE,CAAC,KAAKA,MAAM,GAAG;cAAE0C,UAAU;cAAUC,aAAa;YAAI;UAAE,IAC1DC;QACN,CAAA,CAAA;AAEF,cAAM,KAAKoB,IAAIrC,UAAU6B,IAAIG,aAAa;MAC5C;IACF;EACF;EAEA,MAAcR,YAAYD,SAA+D;AACvF,UAAMvB,WAAW,MAAM,KAAKM,WAAU;AACtC,WAAO,KAAKzB,SAAS;AACnB,YAAM8C,WAAW,MAAM,KAAKC,QAAQ5B,UAAU,EAAA;AAC9C,iBAAW6B,OAAOF,UAAU;AAC1B,YAAI,KAAKG,WAAWD,GAAAA,GAAM;AACxB,gBAAM,KAAKE,QAAQ/B,UAAU6B,IAAIG,aAAa;AAC9C;QACF;AACA,cAAMT,QAAQX,KAAKqB,MAAMJ,IAAIK,QAAQ,IAAA,CAAA;AACrC,cAAM,KAAKG,IAAIrC,UAAU6B,IAAIG,aAAa;MAC5C;IACF;EACF;;EAGA,MAAcJ,QAAQ5B,UAAkBsC,YAAoB;AAC1D,QAAI;AACF,YAAM1C,MAAM,MAAM,KAAK5B,OAAO6B,KAC5B,IAAI0C,wCAAsB;QACxBvC;QACAwC,qBAAqB;QACrBC,iBAAiB,KAAKnE;QACtBoE,mBAAmBJ;QACnBK,uBAAuB,KAAKtE,SAAS;UAAC;YAAS4C;MACjD,CAAA,GACA;QAAE2B,aAAa,KAAK9D,MAAM+D;MAAO,CAAA;AAEnC,aAAOjD,IAAIkD,YAAY,CAAA;IACzB,SAAS7C,KAAK;AACZ,UAAI,CAAC,KAAKpB,QAAS,QAAO,CAAA;AAC1BkE,cAAQC,MAAM,iCAAiC/C,GAAAA;AAC/C,YAAM,IAAIgD,QAAQ,CAACC,MAAMC,WAAWD,GAAG,GAAA,CAAA;AACvC,aAAO,CAAA;IACT;EACF;EAEQpB,WAAWD,KAA2E;AAC5F,WAAO,CAAC,CAAC,KAAKxD,UAAU,CAACwD,IAAIf,oBAAoB,KAAKzC,MAAM;EAC9D;EAEA,MAAcgE,IAAIrC,UAAkBgC,eAAuC;AACzE,QAAI,CAACA,cAAe;AACpB,UAAM,KAAKhE,OAAO6B,KAAK,IAAIuD,uCAAqB;MAAEpD;MAAUgC;IAAc,CAAA,CAAA;EAC5E;EAEA,MAAcD,QAAQ/B,UAAkBgC,eAAuC;AAC7E,QAAI,CAACA,cAAe;AACpB,UAAM,KAAKhE,OACR6B,KAAK,IAAIwD,iDAA+B;MAAErD;MAAUgC;MAAeU,mBAAmB;IAAE,CAAA,CAAA,EACxFY,MAAM,MAAA;IAAO,CAAA;EAClB;;EAGA,MAAMC,QAAuB;AAC3B,SAAK1E,UAAU;AACf,SAAKC,MAAMA,MAAK;AAChB,UAAMmE,QAAQO,WAAW;MAAC,KAAKxE;MAAU,KAAKC;KAAW;AACzD,QAAI,KAAKhB,WAAY,MAAKD,OAAOyF,QAAO;EAC1C;AACF;","names":["SqsTransport","client","ownsClient","group","prefix","autoCreate","marker","waitTimeSeconds","visibilityTimeoutSec","taskQueueUrlResolver","resultsQueueUrlOption","handlers","Map","urlCache","running","abort","AbortController","taskLoop","resultLoop","options","SQSClient","clientConfig","taskQueueUrl","resultsQueueUrl","resolveByName","name","cached","get","url","out","send","GetQueueUrlCommand","QueueName","QueueUrl","err","created","CreateQueueCommand","set","tasksUrl","resultsUrl","opt","dispatch","task","SendMessageCommand","MessageBody","JSON","stringify","MessageAttributes","DataType","StringValue","undefined","handle","fn","Error","pollTasks","onResult","handler","pollResults","onHeartbeat","_handler","messages","receive","msg","shouldSkip","release","ReceiptHandle","parse","Body","result","runStepHandler","del","visibility","ReceiveMessageCommand","MaxNumberOfMessages","WaitTimeSeconds","VisibilityTimeout","MessageAttributeNames","abortSignal","signal","Messages","console","error","Promise","r","setTimeout","DeleteMessageCommand","ChangeMessageVisibilityCommand","catch","close","allSettled","destroy"]}
|
|
1
|
+
{"version":3,"sources":["../src/index.ts","../src/sqs-transport.ts"],"sourcesContent":["export * from './sqs-transport';\n","import {\n ChangeMessageVisibilityCommand,\n CreateQueueCommand,\n DeleteMessageCommand,\n GetQueueUrlCommand,\n ReceiveMessageCommand,\n SQSClient,\n type SQSClientConfig,\n SendMessageCommand,\n} from '@aws-sdk/client-sqs';\nimport {\n type Heartbeat,\n type RemoteTask,\n type StepHandler,\n type StepResult,\n type Transport,\n runStepHandler,\n} from '@dudousxd/nestjs-durable-core';\n\n/** Resolve a group's tasks queue URL. Sync or async; resolved once per group and cached (a rejected\n * resolution is not cached — the next call asks again). */\nexport type QueueUrlResolver = (group: string) => string | Promise<string>;\n\nexport interface SqsTransportOptions {\n /**\n * Reuse an existing SQS client (recommended — share the app's). When omitted, one is built\n * from `clientConfig` and destroyed on `close()`.\n */\n client?: SQSClient;\n /** Used only when `client` is omitted — e.g. `{ region, endpoint, credentials }` for ElasticMQ. */\n clientConfig?: SQSClientConfig;\n\n /** The worker group this instance serves. Required to register `handle()` consumers. */\n group?: string;\n\n /**\n * Map a group → an **existing** tasks queue URL. This is the seam for reusing queues you already\n * have (e.g. `extraction`, `ingestion`) instead of letting the transport mint its own. When\n * omitted, the transport resolves `${prefix}-tasks-${group}` by name (and creates it if\n * `autoCreate`). Return value is awaited and cached per group.\n */\n taskQueueUrl?: QueueUrlResolver;\n /**\n * The results queue URL (worker → engine). A literal URL or a resolver. When omitted, the\n * transport resolves `${prefix}-results` by name (and creates it if `autoCreate`).\n */\n resultsQueueUrl?: string | (() => string | Promise<string>);\n\n /** Prefix for the fallback `${prefix}-tasks-*` / `${prefix}-results` names. Default `durable`. */\n prefix?: string;\n /**\n * Create the fallback-named queues on first use if they don't exist (handy for ElasticMQ / dev).\n * Ignored for queues you supply via `taskQueueUrl` / `resultsQueueUrl` — those must already exist.\n */\n autoCreate?: boolean;\n\n /**\n * When set, durable messages carry this SQS message attribute and pollers **skip** messages\n * that lack it (releasing them for a co-resident legacy consumer). Enables sharing one physical\n * queue with a non-durable consumer — but only if that consumer likewise ignores tagged messages.\n * Leave unset for the clean case: a queue dedicated to durable traffic.\n */\n marker?: string;\n\n /** SQS receive long-poll seconds (0–20). Default 20. */\n waitTimeSeconds?: number;\n /**\n * Visibility timeout for a received task, in seconds. Default 60. Every message this instance has\n * received — the one running and the ones of the same batch waiting behind it — is kept invisible by\n * re-extending it (`ChangeMessageVisibility`) every third of this, so a step may run longer; only a\n * crashed instance's messages reappear, after at most this long.\n */\n visibilityTimeoutSec?: number;\n /**\n * Where a transport failure is reported: a failing receive, a result handler that throws, a result\n * send that fails, a message whose body isn't JSON. The message involved is NOT deleted: SQS\n * redelivers it after its visibility timeout, and a queue redrive policy (`maxReceiveCount` → DLQ)\n * moves one that keeps failing aside. Default `console.error`.\n */\n onError?: (err: unknown) => void;\n}\n\n/** A received message, as the poll loops use it. */\ninterface Received {\n Body?: string | undefined;\n ReceiptHandle?: string | undefined;\n MessageId?: string | undefined;\n MessageAttributes?: Record<string, unknown> | undefined;\n}\n\n/** The results queue's visibility timeout (seconds). */\nconst RESULT_VISIBILITY_SEC = 30;\n\n/**\n * An SQS-backed `Transport` — dispatch remote steps over Amazon SQS (or ElasticMQ). Unlike the\n * BullMQ transport, it does **not** mint its own queue names: pass `taskQueueUrl` / `resultsQueueUrl`\n * to ride the queues you already operate. The wire payload is the documented `RemoteTask` /\n * `StepResult` JSON, so non-Node workers on the same queues interoperate.\n *\n * Run one instance engine-side (calls `onResult`, dispatches tasks) and one per worker process\n * (calls `handle()` for its group). SQS is at-least-once: a task whose process crashes before the\n * result is sent reappears after the visibility timeout (durable retry); a duplicate `StepResult`\n * is harmless because the engine resolves a step's `pending` exactly once by `stepId`.\n */\nexport class SqsTransport implements Transport {\n private readonly client: SQSClient;\n private readonly ownsClient: boolean;\n private readonly group?: string | undefined;\n private readonly prefix: string;\n private readonly autoCreate: boolean;\n private readonly marker?: string | undefined;\n private readonly waitTimeSeconds: number;\n private readonly visibilityTimeoutSec: number;\n private readonly taskQueueUrlResolver?: QueueUrlResolver | undefined;\n private readonly resultsQueueUrlOption?: string | (() => string | Promise<string>) | undefined;\n private readonly onError: (err: unknown) => void;\n\n private readonly handlers = new Map<string, StepHandler>();\n private readonly urlCache = new Map<string, string>();\n /** `taskQueueUrl` resolutions per group (the promise, so concurrent dispatches share one call). */\n private readonly groupUrlCache = new Map<string, Promise<string>>();\n private running = true;\n private readonly abort = new AbortController();\n private taskLoop?: Promise<void>;\n private resultLoop?: Promise<void>;\n\n constructor(options: SqsTransportOptions) {\n this.client = options.client ?? new SQSClient(options.clientConfig ?? {});\n this.ownsClient = !options.client;\n this.group = options.group;\n this.prefix = options.prefix ?? 'durable';\n this.autoCreate = options.autoCreate ?? false;\n this.marker = options.marker;\n this.waitTimeSeconds = options.waitTimeSeconds ?? 20;\n this.visibilityTimeoutSec = options.visibilityTimeoutSec ?? 60;\n this.taskQueueUrlResolver = options.taskQueueUrl;\n this.resultsQueueUrlOption = options.resultsQueueUrl;\n this.onError = options.onError ?? ((err) => console.error('[SqsTransport]', err));\n }\n\n // ── queue resolution ───────────────────────────────────────────────────────────────────────\n\n private async resolveByName(name: string): Promise<string> {\n const cached = this.urlCache.get(name);\n if (cached) return cached;\n let url: string;\n try {\n const out = await this.client.send(new GetQueueUrlCommand({ QueueName: name }));\n url = out.QueueUrl as string;\n } catch (err) {\n if (this.autoCreate) {\n const created = await this.client.send(new CreateQueueCommand({ QueueName: name }));\n url = created.QueueUrl as string;\n } else {\n throw err;\n }\n }\n this.urlCache.set(name, url);\n return url;\n }\n\n private async tasksUrl(group: string): Promise<string> {\n const resolver = this.taskQueueUrlResolver;\n if (!resolver) return this.resolveByName(`${this.prefix}-tasks-${group}`);\n let url = this.groupUrlCache.get(group);\n if (!url) {\n url = Promise.resolve()\n .then(() => resolver(group))\n .catch((err) => {\n this.groupUrlCache.delete(group);\n throw err;\n });\n this.groupUrlCache.set(group, url);\n }\n return url;\n }\n\n private async resultsUrl(): Promise<string> {\n const opt = this.resultsQueueUrlOption;\n if (typeof opt === 'string') return opt;\n if (typeof opt === 'function') return opt();\n return this.resolveByName(`${this.prefix}-results`);\n }\n\n // ── engine → worker ────────────────────────────────────────────────────────────────────────\n\n async dispatch(task: RemoteTask): Promise<void> {\n const QueueUrl = await this.tasksUrl(task.group);\n await this.client.send(\n new SendMessageCommand({\n QueueUrl,\n MessageBody: JSON.stringify(task),\n MessageAttributes: this.marker\n ? { [this.marker]: { DataType: 'String', StringValue: '1' } }\n : undefined,\n }),\n );\n }\n\n // ── worker → engine ────────────────────────────────────────────────────────────────────────\n\n /** Register a step handler (worker side). Starts the group's task consumer on first call. */\n handle(name: string, fn: StepHandler): void {\n if (!this.group) throw new Error('SqsTransport needs a `group` to register handlers');\n this.handlers.set(name, fn);\n if (!this.taskLoop) this.taskLoop = this.pollTasks(this.group);\n }\n\n onResult(handler: (result: StepResult) => Promise<void>): void {\n if (!this.resultLoop) this.resultLoop = this.pollResults(handler);\n }\n\n onHeartbeat(_handler: (beat: Heartbeat) => Promise<void>): void {\n // Not modelled over SQS; the queue's visibility-timeout redelivery is the liveness mechanism.\n }\n\n // ── polling loops ──────────────────────────────────────────────────────────────────────────\n\n /**\n * Receive a batch and hand each message to `each`, one at a time, never letting a failure escape:\n * a throw is reported through `onError` and its message is left on the queue (SQS redelivers it\n * after the visibility timeout; a redrive policy DLQs one that keeps failing). Every message of the\n * batch is kept invisible until it is handled (see {@link holdVisible}), so the ones waiting behind a\n * slow handler are never redelivered to another consumer meanwhile.\n */\n private async pollLoop(\n queueUrl: () => Promise<string>,\n visibilitySec: number,\n each: (QueueUrl: string, msg: Received) => Promise<void>,\n ): Promise<void> {\n while (this.running) {\n let QueueUrl: string;\n try {\n QueueUrl = await queueUrl();\n } catch (err) {\n if (!this.running) return;\n this.onError(err);\n await this.backoff();\n continue;\n }\n const messages = await this.receive(QueueUrl, visibilitySec);\n const pending = new Set<Received>(messages);\n const stopHolding = this.holdVisible(QueueUrl, pending, visibilitySec);\n try {\n for (const msg of messages) {\n if (!this.running) break; // close(): what is left reappears after its visibility timeout\n try {\n if (this.shouldSkip(msg)) {\n // Not ours (shared queue): release immediately so the legacy consumer can take it.\n pending.delete(msg);\n await this.release(QueueUrl, msg.ReceiptHandle);\n continue;\n }\n await each(QueueUrl, msg);\n } catch (err) {\n this.onError(err);\n } finally {\n pending.delete(msg);\n }\n }\n } finally {\n stopHolding();\n }\n }\n }\n\n /**\n * Keep `messages` invisible while they wait or run: every third of the visibility timeout, extend\n * each one still in the set back to the full timeout. Returns the stop function.\n */\n private holdVisible(\n QueueUrl: string,\n messages: Set<Received>,\n visibilitySec: number,\n ): () => void {\n if (messages.size === 0) return () => {};\n const intervalMs = Math.max(200, Math.floor((visibilitySec * 1000) / 3));\n let round: Promise<void> | undefined;\n const timer = setInterval(() => {\n if (round) return;\n round = Promise.all(\n [...messages].map((msg) =>\n msg.ReceiptHandle\n ? this.client\n .send(\n new ChangeMessageVisibilityCommand({\n QueueUrl,\n ReceiptHandle: msg.ReceiptHandle,\n VisibilityTimeout: visibilitySec,\n }),\n )\n .then(() => undefined)\n .catch((err) => {\n if (this.running) this.onError(err);\n })\n : undefined,\n ),\n ).then(() => {\n round = undefined;\n });\n }, intervalMs);\n timer.unref?.();\n return () => clearInterval(timer);\n }\n\n private parse<T>(msg: Received): T {\n try {\n return JSON.parse(msg.Body ?? '') as T;\n } catch {\n throw new Error(\n `[SqsTransport] message ${msg.MessageId ?? '?'} has a non-JSON body; leaving it for the queue's redrive policy`,\n );\n }\n }\n\n private pollTasks(group: string): Promise<void> {\n return this.pollLoop(\n () => this.tasksUrl(group),\n this.visibilityTimeoutSec,\n async (QueueUrl, msg) => {\n const task = this.parse<RemoteTask>(msg);\n const result = await runStepHandler(task, this.handlers.get(task.name));\n await this.client.send(\n new SendMessageCommand({\n QueueUrl: await this.resultsUrl(),\n MessageBody: JSON.stringify(result),\n MessageAttributes: this.marker\n ? { [this.marker]: { DataType: 'String', StringValue: '1' } }\n : undefined,\n }),\n );\n await this.del(QueueUrl, msg.ReceiptHandle);\n },\n );\n }\n\n private pollResults(handler: (result: StepResult) => Promise<void>): Promise<void> {\n return this.pollLoop(\n () => this.resultsUrl(),\n RESULT_VISIBILITY_SEC,\n async (QueueUrl, msg) => {\n await handler(this.parse<StepResult>(msg));\n await this.del(QueueUrl, msg.ReceiptHandle);\n },\n );\n }\n\n /** Wait 1s, or until `close()` aborts. The abort listener is removed when the timer fires, so\n * sustained failures never accumulate listeners on the shared signal. */\n private backoff(): Promise<void> {\n return new Promise((resolve) => {\n const signal = this.abort.signal;\n if (signal.aborted) return resolve();\n const onAbort = () => {\n clearTimeout(timer);\n resolve();\n };\n const timer = setTimeout(() => {\n signal.removeEventListener('abort', onAbort);\n resolve();\n }, 1000);\n signal.addEventListener('abort', onAbort, { once: true });\n });\n }\n\n /** One long-poll receive. Swallows the abort thrown on `close()`; logs and backs off otherwise. */\n private async receive(QueueUrl: string, visibility: number): Promise<Received[]> {\n try {\n const out = await this.client.send(\n new ReceiveMessageCommand({\n QueueUrl,\n MaxNumberOfMessages: 10,\n WaitTimeSeconds: this.waitTimeSeconds,\n VisibilityTimeout: visibility,\n MessageAttributeNames: this.marker ? ['All'] : undefined,\n }),\n { abortSignal: this.abort.signal },\n );\n return out.Messages ?? [];\n } catch (err) {\n if (!this.running) return [];\n this.onError(err);\n await this.backoff();\n return [];\n }\n }\n\n private shouldSkip(msg: { MessageAttributes?: Record<string, unknown> | undefined }): boolean {\n return !!this.marker && !msg.MessageAttributes?.[this.marker];\n }\n\n private async del(QueueUrl: string, ReceiptHandle?: string): Promise<void> {\n if (!ReceiptHandle) return;\n await this.client.send(new DeleteMessageCommand({ QueueUrl, ReceiptHandle }));\n }\n\n private async release(QueueUrl: string, ReceiptHandle?: string): Promise<void> {\n if (!ReceiptHandle) return;\n await this.client\n .send(new ChangeMessageVisibilityCommand({ QueueUrl, ReceiptHandle, VisibilityTimeout: 0 }))\n .catch(() => {});\n }\n\n /** Stop the pollers and (if we created it) destroy the SQS client so the process can exit. */\n async close(): Promise<void> {\n this.running = false;\n this.abort.abort();\n await Promise.allSettled([this.taskLoop, this.resultLoop]);\n if (this.ownsClient) this.client.destroy();\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAAA;;;;;;;ACAA,wBASO;AACP,iCAOO;AA0EP,IAAMA,wBAAwB;AAavB,IAAMC,eAAN,MAAMA;EAxGb,OAwGaA;;;EACMC;EACAC;EACAC;EACAC;EACAC;EACAC;EACAC;EACAC;EACAC;EACAC;EACAC;EAEAC,WAAW,oBAAIC,IAAAA;EACfC,WAAW,oBAAID,IAAAA;;EAEfE,gBAAgB,oBAAIF,IAAAA;EAC7BG,UAAU;EACDC,QAAQ,IAAIC,gBAAAA;EACrBC;EACAC;EAER,YAAYC,SAA8B;AACxC,SAAKpB,SAASoB,QAAQpB,UAAU,IAAIqB,4BAAUD,QAAQE,gBAAgB,CAAC,CAAA;AACvE,SAAKrB,aAAa,CAACmB,QAAQpB;AAC3B,SAAKE,QAAQkB,QAAQlB;AACrB,SAAKC,SAASiB,QAAQjB,UAAU;AAChC,SAAKC,aAAagB,QAAQhB,cAAc;AACxC,SAAKC,SAASe,QAAQf;AACtB,SAAKC,kBAAkBc,QAAQd,mBAAmB;AAClD,SAAKC,uBAAuBa,QAAQb,wBAAwB;AAC5D,SAAKC,uBAAuBY,QAAQG;AACpC,SAAKd,wBAAwBW,QAAQI;AACrC,SAAKd,UAAUU,QAAQV,YAAY,CAACe,QAAQC,QAAQC,MAAM,kBAAkBF,GAAAA;EAC9E;;EAIA,MAAcG,cAAcC,MAA+B;AACzD,UAAMC,SAAS,KAAKjB,SAASkB,IAAIF,IAAAA;AACjC,QAAIC,OAAQ,QAAOA;AACnB,QAAIE;AACJ,QAAI;AACF,YAAMC,MAAM,MAAM,KAAKjC,OAAOkC,KAAK,IAAIC,qCAAmB;QAAEC,WAAWP;MAAK,CAAA,CAAA;AAC5EG,YAAMC,IAAII;IACZ,SAASZ,KAAK;AACZ,UAAI,KAAKrB,YAAY;AACnB,cAAMkC,UAAU,MAAM,KAAKtC,OAAOkC,KAAK,IAAIK,qCAAmB;UAAEH,WAAWP;QAAK,CAAA,CAAA;AAChFG,cAAMM,QAAQD;MAChB,OAAO;AACL,cAAMZ;MACR;IACF;AACA,SAAKZ,SAAS2B,IAAIX,MAAMG,GAAAA;AACxB,WAAOA;EACT;EAEA,MAAcS,SAASvC,OAAgC;AACrD,UAAMwC,WAAW,KAAKlC;AACtB,QAAI,CAACkC,SAAU,QAAO,KAAKd,cAAc,GAAG,KAAKzB,MAAM,UAAUD,KAAAA,EAAO;AACxE,QAAI8B,MAAM,KAAKlB,cAAciB,IAAI7B,KAAAA;AACjC,QAAI,CAAC8B,KAAK;AACRA,YAAMW,QAAQC,QAAO,EAClBC,KAAK,MAAMH,SAASxC,KAAAA,CAAAA,EACpB4C,MAAM,CAACrB,QAAAA;AACN,aAAKX,cAAciC,OAAO7C,KAAAA;AAC1B,cAAMuB;MACR,CAAA;AACF,WAAKX,cAAc0B,IAAItC,OAAO8B,GAAAA;IAChC;AACA,WAAOA;EACT;EAEA,MAAcgB,aAA8B;AAC1C,UAAMC,MAAM,KAAKxC;AACjB,QAAI,OAAOwC,QAAQ,SAAU,QAAOA;AACpC,QAAI,OAAOA,QAAQ,WAAY,QAAOA,IAAAA;AACtC,WAAO,KAAKrB,cAAc,GAAG,KAAKzB,MAAM,UAAU;EACpD;;EAIA,MAAM+C,SAASC,MAAiC;AAC9C,UAAMd,WAAW,MAAM,KAAKI,SAASU,KAAKjD,KAAK;AAC/C,UAAM,KAAKF,OAAOkC,KAChB,IAAIkB,qCAAmB;MACrBf;MACAgB,aAAaC,KAAKC,UAAUJ,IAAAA;MAC5BK,mBAAmB,KAAKnD,SACpB;QAAE,CAAC,KAAKA,MAAM,GAAG;UAAEoD,UAAU;UAAUC,aAAa;QAAI;MAAE,IAC1DC;IACN,CAAA,CAAA;EAEJ;;;EAKAC,OAAO/B,MAAcgC,IAAuB;AAC1C,QAAI,CAAC,KAAK3D,MAAO,OAAM,IAAI4D,MAAM,mDAAA;AACjC,SAAKnD,SAAS6B,IAAIX,MAAMgC,EAAAA;AACxB,QAAI,CAAC,KAAK3C,SAAU,MAAKA,WAAW,KAAK6C,UAAU,KAAK7D,KAAK;EAC/D;EAEA8D,SAASC,SAAsD;AAC7D,QAAI,CAAC,KAAK9C,WAAY,MAAKA,aAAa,KAAK+C,YAAYD,OAAAA;EAC3D;EAEAE,YAAYC,UAAoD;EAEhE;;;;;;;;;EAWA,MAAcC,SACZC,UACAC,eACAC,MACe;AACf,WAAO,KAAKzD,SAAS;AACnB,UAAIsB;AACJ,UAAI;AACFA,mBAAW,MAAMiC,SAAAA;MACnB,SAAS7C,KAAK;AACZ,YAAI,CAAC,KAAKV,QAAS;AACnB,aAAKL,QAAQe,GAAAA;AACb,cAAM,KAAKgD,QAAO;AAClB;MACF;AACA,YAAMC,WAAW,MAAM,KAAKC,QAAQtC,UAAUkC,aAAAA;AAC9C,YAAMK,UAAU,IAAIC,IAAcH,QAAAA;AAClC,YAAMI,cAAc,KAAKC,YAAY1C,UAAUuC,SAASL,aAAAA;AACxD,UAAI;AACF,mBAAWS,OAAON,UAAU;AAC1B,cAAI,CAAC,KAAK3D,QAAS;AACnB,cAAI;AACF,gBAAI,KAAKkE,WAAWD,GAAAA,GAAM;AAExBJ,sBAAQ7B,OAAOiC,GAAAA;AACf,oBAAM,KAAKE,QAAQ7C,UAAU2C,IAAIG,aAAa;AAC9C;YACF;AACA,kBAAMX,KAAKnC,UAAU2C,GAAAA;UACvB,SAASvD,KAAK;AACZ,iBAAKf,QAAQe,GAAAA;UACf,UAAA;AACEmD,oBAAQ7B,OAAOiC,GAAAA;UACjB;QACF;MACF,UAAA;AACEF,oBAAAA;MACF;IACF;EACF;;;;;EAMQC,YACN1C,UACAqC,UACAH,eACY;AACZ,QAAIG,SAASU,SAAS,EAAG,QAAO,MAAA;IAAO;AACvC,UAAMC,aAAaC,KAAKC,IAAI,KAAKD,KAAKE,MAAOjB,gBAAgB,MAAQ,CAAA,CAAA;AACrE,QAAIkB;AACJ,UAAMC,QAAQC,YAAY,MAAA;AACxB,UAAIF,MAAO;AACXA,cAAQ9C,QAAQiD,IACd;WAAIlB;QAAUmB,IAAI,CAACb,QACjBA,IAAIG,gBACA,KAAKnF,OACFkC,KACC,IAAI4D,iDAA+B;QACjCzD;QACA8C,eAAeH,IAAIG;QACnBY,mBAAmBxB;MACrB,CAAA,CAAA,EAED1B,KAAK,MAAMc,MAAAA,EACXb,MAAM,CAACrB,QAAAA;AACN,YAAI,KAAKV,QAAS,MAAKL,QAAQe,GAAAA;MACjC,CAAA,IACFkC,MAAAA,CAAAA,EAENd,KAAK,MAAA;AACL4C,gBAAQ9B;MACV,CAAA;IACF,GAAG0B,UAAAA;AACHK,UAAMM,QAAK;AACX,WAAO,MAAMC,cAAcP,KAAAA;EAC7B;EAEQQ,MAASlB,KAAkB;AACjC,QAAI;AACF,aAAO1B,KAAK4C,MAAMlB,IAAImB,QAAQ,EAAA;IAChC,QAAQ;AACN,YAAM,IAAIrC,MACR,0BAA0BkB,IAAIoB,aAAa,GAAA,iEAAoE;IAEnH;EACF;EAEQrC,UAAU7D,OAA8B;AAC9C,WAAO,KAAKmE,SACV,MAAM,KAAK5B,SAASvC,KAAAA,GACpB,KAAKK,sBACL,OAAO8B,UAAU2C,QAAAA;AACf,YAAM7B,OAAO,KAAK+C,MAAkBlB,GAAAA;AACpC,YAAMqB,SAAS,UAAMC,2CAAenD,MAAM,KAAKxC,SAASoB,IAAIoB,KAAKtB,IAAI,CAAA;AACrE,YAAM,KAAK7B,OAAOkC,KAChB,IAAIkB,qCAAmB;QACrBf,UAAU,MAAM,KAAKW,WAAU;QAC/BK,aAAaC,KAAKC,UAAU8C,MAAAA;QAC5B7C,mBAAmB,KAAKnD,SACpB;UAAE,CAAC,KAAKA,MAAM,GAAG;YAAEoD,UAAU;YAAUC,aAAa;UAAI;QAAE,IAC1DC;MACN,CAAA,CAAA;AAEF,YAAM,KAAK4C,IAAIlE,UAAU2C,IAAIG,aAAa;IAC5C,CAAA;EAEJ;EAEQjB,YAAYD,SAA+D;AACjF,WAAO,KAAKI,SACV,MAAM,KAAKrB,WAAU,GACrBlD,uBACA,OAAOuC,UAAU2C,QAAAA;AACf,YAAMf,QAAQ,KAAKiC,MAAkBlB,GAAAA,CAAAA;AACrC,YAAM,KAAKuB,IAAIlE,UAAU2C,IAAIG,aAAa;IAC5C,CAAA;EAEJ;;;EAIQV,UAAyB;AAC/B,WAAO,IAAI9B,QAAQ,CAACC,YAAAA;AAClB,YAAM4D,SAAS,KAAKxF,MAAMwF;AAC1B,UAAIA,OAAOC,QAAS,QAAO7D,QAAAA;AAC3B,YAAM8D,UAAU,6BAAA;AACdC,qBAAajB,KAAAA;AACb9C,gBAAAA;MACF,GAHgB;AAIhB,YAAM8C,QAAQkB,WAAW,MAAA;AACvBJ,eAAOK,oBAAoB,SAASH,OAAAA;AACpC9D,gBAAAA;MACF,GAAG,GAAA;AACH4D,aAAOM,iBAAiB,SAASJ,SAAS;QAAEK,MAAM;MAAK,CAAA;IACzD,CAAA;EACF;;EAGA,MAAcpC,QAAQtC,UAAkB2E,YAAyC;AAC/E,QAAI;AACF,YAAM/E,MAAM,MAAM,KAAKjC,OAAOkC,KAC5B,IAAI+E,wCAAsB;QACxB5E;QACA6E,qBAAqB;QACrBC,iBAAiB,KAAK7G;QACtByF,mBAAmBiB;QACnBI,uBAAuB,KAAK/G,SAAS;UAAC;YAASsD;MACjD,CAAA,GACA;QAAE0D,aAAa,KAAKrG,MAAMwF;MAAO,CAAA;AAEnC,aAAOvE,IAAIqF,YAAY,CAAA;IACzB,SAAS7F,KAAK;AACZ,UAAI,CAAC,KAAKV,QAAS,QAAO,CAAA;AAC1B,WAAKL,QAAQe,GAAAA;AACb,YAAM,KAAKgD,QAAO;AAClB,aAAO,CAAA;IACT;EACF;EAEQQ,WAAWD,KAA2E;AAC5F,WAAO,CAAC,CAAC,KAAK3E,UAAU,CAAC2E,IAAIxB,oBAAoB,KAAKnD,MAAM;EAC9D;EAEA,MAAckG,IAAIlE,UAAkB8C,eAAuC;AACzE,QAAI,CAACA,cAAe;AACpB,UAAM,KAAKnF,OAAOkC,KAAK,IAAIqF,uCAAqB;MAAElF;MAAU8C;IAAc,CAAA,CAAA;EAC5E;EAEA,MAAcD,QAAQ7C,UAAkB8C,eAAuC;AAC7E,QAAI,CAACA,cAAe;AACpB,UAAM,KAAKnF,OACRkC,KAAK,IAAI4D,iDAA+B;MAAEzD;MAAU8C;MAAeY,mBAAmB;IAAE,CAAA,CAAA,EACxFjD,MAAM,MAAA;IAAO,CAAA;EAClB;;EAGA,MAAM0E,QAAuB;AAC3B,SAAKzG,UAAU;AACf,SAAKC,MAAMA,MAAK;AAChB,UAAM2B,QAAQ8E,WAAW;MAAC,KAAKvG;MAAU,KAAKC;KAAW;AACzD,QAAI,KAAKlB,WAAY,MAAKD,OAAO0H,QAAO;EAC1C;AACF;","names":["RESULT_VISIBILITY_SEC","SqsTransport","client","ownsClient","group","prefix","autoCreate","marker","waitTimeSeconds","visibilityTimeoutSec","taskQueueUrlResolver","resultsQueueUrlOption","onError","handlers","Map","urlCache","groupUrlCache","running","abort","AbortController","taskLoop","resultLoop","options","SQSClient","clientConfig","taskQueueUrl","resultsQueueUrl","err","console","error","resolveByName","name","cached","get","url","out","send","GetQueueUrlCommand","QueueName","QueueUrl","created","CreateQueueCommand","set","tasksUrl","resolver","Promise","resolve","then","catch","delete","resultsUrl","opt","dispatch","task","SendMessageCommand","MessageBody","JSON","stringify","MessageAttributes","DataType","StringValue","undefined","handle","fn","Error","pollTasks","onResult","handler","pollResults","onHeartbeat","_handler","pollLoop","queueUrl","visibilitySec","each","backoff","messages","receive","pending","Set","stopHolding","holdVisible","msg","shouldSkip","release","ReceiptHandle","size","intervalMs","Math","max","floor","round","timer","setInterval","all","map","ChangeMessageVisibilityCommand","VisibilityTimeout","unref","clearInterval","parse","Body","MessageId","result","runStepHandler","del","signal","aborted","onAbort","clearTimeout","setTimeout","removeEventListener","addEventListener","once","visibility","ReceiveMessageCommand","MaxNumberOfMessages","WaitTimeSeconds","MessageAttributeNames","abortSignal","Messages","DeleteMessageCommand","close","allSettled","destroy"]}
|
package/dist/index.d.cts
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { SQSClient, SQSClientConfig } from '@aws-sdk/client-sqs';
|
|
2
2
|
import { Transport, RemoteTask, StepHandler, StepResult, Heartbeat } from '@dudousxd/nestjs-durable-core';
|
|
3
3
|
|
|
4
|
-
/** Resolve a group's tasks queue URL. Sync or async; resolved once and cached
|
|
4
|
+
/** Resolve a group's tasks queue URL. Sync or async; resolved once per group and cached (a rejected
|
|
5
|
+
* resolution is not cached — the next call asks again). */
|
|
5
6
|
type QueueUrlResolver = (group: string) => string | Promise<string>;
|
|
6
7
|
interface SqsTransportOptions {
|
|
7
8
|
/**
|
|
@@ -17,7 +18,7 @@ interface SqsTransportOptions {
|
|
|
17
18
|
* Map a group → an **existing** tasks queue URL. This is the seam for reusing queues you already
|
|
18
19
|
* have (e.g. `extraction`, `ingestion`) instead of letting the transport mint its own. When
|
|
19
20
|
* omitted, the transport resolves `${prefix}-tasks-${group}` by name (and creates it if
|
|
20
|
-
* `autoCreate`). Return value is awaited and cached.
|
|
21
|
+
* `autoCreate`). Return value is awaited and cached per group.
|
|
21
22
|
*/
|
|
22
23
|
taskQueueUrl?: QueueUrlResolver;
|
|
23
24
|
/**
|
|
@@ -41,8 +42,20 @@ interface SqsTransportOptions {
|
|
|
41
42
|
marker?: string;
|
|
42
43
|
/** SQS receive long-poll seconds (0–20). Default 20. */
|
|
43
44
|
waitTimeSeconds?: number;
|
|
44
|
-
/**
|
|
45
|
+
/**
|
|
46
|
+
* Visibility timeout for a received task, in seconds. Default 60. Every message this instance has
|
|
47
|
+
* received — the one running and the ones of the same batch waiting behind it — is kept invisible by
|
|
48
|
+
* re-extending it (`ChangeMessageVisibility`) every third of this, so a step may run longer; only a
|
|
49
|
+
* crashed instance's messages reappear, after at most this long.
|
|
50
|
+
*/
|
|
45
51
|
visibilityTimeoutSec?: number;
|
|
52
|
+
/**
|
|
53
|
+
* Where a transport failure is reported: a failing receive, a result handler that throws, a result
|
|
54
|
+
* send that fails, a message whose body isn't JSON. The message involved is NOT deleted: SQS
|
|
55
|
+
* redelivers it after its visibility timeout, and a queue redrive policy (`maxReceiveCount` → DLQ)
|
|
56
|
+
* moves one that keeps failing aside. Default `console.error`.
|
|
57
|
+
*/
|
|
58
|
+
onError?: (err: unknown) => void;
|
|
46
59
|
}
|
|
47
60
|
/**
|
|
48
61
|
* An SQS-backed `Transport` — dispatch remote steps over Amazon SQS (or ElasticMQ). Unlike the
|
|
@@ -66,8 +79,11 @@ declare class SqsTransport implements Transport {
|
|
|
66
79
|
private readonly visibilityTimeoutSec;
|
|
67
80
|
private readonly taskQueueUrlResolver?;
|
|
68
81
|
private readonly resultsQueueUrlOption?;
|
|
82
|
+
private readonly onError;
|
|
69
83
|
private readonly handlers;
|
|
70
84
|
private readonly urlCache;
|
|
85
|
+
/** `taskQueueUrl` resolutions per group (the promise, so concurrent dispatches share one call). */
|
|
86
|
+
private readonly groupUrlCache;
|
|
71
87
|
private running;
|
|
72
88
|
private readonly abort;
|
|
73
89
|
private taskLoop?;
|
|
@@ -81,8 +97,25 @@ declare class SqsTransport implements Transport {
|
|
|
81
97
|
handle(name: string, fn: StepHandler): void;
|
|
82
98
|
onResult(handler: (result: StepResult) => Promise<void>): void;
|
|
83
99
|
onHeartbeat(_handler: (beat: Heartbeat) => Promise<void>): void;
|
|
100
|
+
/**
|
|
101
|
+
* Receive a batch and hand each message to `each`, one at a time, never letting a failure escape:
|
|
102
|
+
* a throw is reported through `onError` and its message is left on the queue (SQS redelivers it
|
|
103
|
+
* after the visibility timeout; a redrive policy DLQs one that keeps failing). Every message of the
|
|
104
|
+
* batch is kept invisible until it is handled (see {@link holdVisible}), so the ones waiting behind a
|
|
105
|
+
* slow handler are never redelivered to another consumer meanwhile.
|
|
106
|
+
*/
|
|
107
|
+
private pollLoop;
|
|
108
|
+
/**
|
|
109
|
+
* Keep `messages` invisible while they wait or run: every third of the visibility timeout, extend
|
|
110
|
+
* each one still in the set back to the full timeout. Returns the stop function.
|
|
111
|
+
*/
|
|
112
|
+
private holdVisible;
|
|
113
|
+
private parse;
|
|
84
114
|
private pollTasks;
|
|
85
115
|
private pollResults;
|
|
116
|
+
/** Wait 1s, or until `close()` aborts. The abort listener is removed when the timer fires, so
|
|
117
|
+
* sustained failures never accumulate listeners on the shared signal. */
|
|
118
|
+
private backoff;
|
|
86
119
|
/** One long-poll receive. Swallows the abort thrown on `close()`; logs and backs off otherwise. */
|
|
87
120
|
private receive;
|
|
88
121
|
private shouldSkip;
|
package/dist/index.d.ts
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { SQSClient, SQSClientConfig } from '@aws-sdk/client-sqs';
|
|
2
2
|
import { Transport, RemoteTask, StepHandler, StepResult, Heartbeat } from '@dudousxd/nestjs-durable-core';
|
|
3
3
|
|
|
4
|
-
/** Resolve a group's tasks queue URL. Sync or async; resolved once and cached
|
|
4
|
+
/** Resolve a group's tasks queue URL. Sync or async; resolved once per group and cached (a rejected
|
|
5
|
+
* resolution is not cached — the next call asks again). */
|
|
5
6
|
type QueueUrlResolver = (group: string) => string | Promise<string>;
|
|
6
7
|
interface SqsTransportOptions {
|
|
7
8
|
/**
|
|
@@ -17,7 +18,7 @@ interface SqsTransportOptions {
|
|
|
17
18
|
* Map a group → an **existing** tasks queue URL. This is the seam for reusing queues you already
|
|
18
19
|
* have (e.g. `extraction`, `ingestion`) instead of letting the transport mint its own. When
|
|
19
20
|
* omitted, the transport resolves `${prefix}-tasks-${group}` by name (and creates it if
|
|
20
|
-
* `autoCreate`). Return value is awaited and cached.
|
|
21
|
+
* `autoCreate`). Return value is awaited and cached per group.
|
|
21
22
|
*/
|
|
22
23
|
taskQueueUrl?: QueueUrlResolver;
|
|
23
24
|
/**
|
|
@@ -41,8 +42,20 @@ interface SqsTransportOptions {
|
|
|
41
42
|
marker?: string;
|
|
42
43
|
/** SQS receive long-poll seconds (0–20). Default 20. */
|
|
43
44
|
waitTimeSeconds?: number;
|
|
44
|
-
/**
|
|
45
|
+
/**
|
|
46
|
+
* Visibility timeout for a received task, in seconds. Default 60. Every message this instance has
|
|
47
|
+
* received — the one running and the ones of the same batch waiting behind it — is kept invisible by
|
|
48
|
+
* re-extending it (`ChangeMessageVisibility`) every third of this, so a step may run longer; only a
|
|
49
|
+
* crashed instance's messages reappear, after at most this long.
|
|
50
|
+
*/
|
|
45
51
|
visibilityTimeoutSec?: number;
|
|
52
|
+
/**
|
|
53
|
+
* Where a transport failure is reported: a failing receive, a result handler that throws, a result
|
|
54
|
+
* send that fails, a message whose body isn't JSON. The message involved is NOT deleted: SQS
|
|
55
|
+
* redelivers it after its visibility timeout, and a queue redrive policy (`maxReceiveCount` → DLQ)
|
|
56
|
+
* moves one that keeps failing aside. Default `console.error`.
|
|
57
|
+
*/
|
|
58
|
+
onError?: (err: unknown) => void;
|
|
46
59
|
}
|
|
47
60
|
/**
|
|
48
61
|
* An SQS-backed `Transport` — dispatch remote steps over Amazon SQS (or ElasticMQ). Unlike the
|
|
@@ -66,8 +79,11 @@ declare class SqsTransport implements Transport {
|
|
|
66
79
|
private readonly visibilityTimeoutSec;
|
|
67
80
|
private readonly taskQueueUrlResolver?;
|
|
68
81
|
private readonly resultsQueueUrlOption?;
|
|
82
|
+
private readonly onError;
|
|
69
83
|
private readonly handlers;
|
|
70
84
|
private readonly urlCache;
|
|
85
|
+
/** `taskQueueUrl` resolutions per group (the promise, so concurrent dispatches share one call). */
|
|
86
|
+
private readonly groupUrlCache;
|
|
71
87
|
private running;
|
|
72
88
|
private readonly abort;
|
|
73
89
|
private taskLoop?;
|
|
@@ -81,8 +97,25 @@ declare class SqsTransport implements Transport {
|
|
|
81
97
|
handle(name: string, fn: StepHandler): void;
|
|
82
98
|
onResult(handler: (result: StepResult) => Promise<void>): void;
|
|
83
99
|
onHeartbeat(_handler: (beat: Heartbeat) => Promise<void>): void;
|
|
100
|
+
/**
|
|
101
|
+
* Receive a batch and hand each message to `each`, one at a time, never letting a failure escape:
|
|
102
|
+
* a throw is reported through `onError` and its message is left on the queue (SQS redelivers it
|
|
103
|
+
* after the visibility timeout; a redrive policy DLQs one that keeps failing). Every message of the
|
|
104
|
+
* batch is kept invisible until it is handled (see {@link holdVisible}), so the ones waiting behind a
|
|
105
|
+
* slow handler are never redelivered to another consumer meanwhile.
|
|
106
|
+
*/
|
|
107
|
+
private pollLoop;
|
|
108
|
+
/**
|
|
109
|
+
* Keep `messages` invisible while they wait or run: every third of the visibility timeout, extend
|
|
110
|
+
* each one still in the set back to the full timeout. Returns the stop function.
|
|
111
|
+
*/
|
|
112
|
+
private holdVisible;
|
|
113
|
+
private parse;
|
|
84
114
|
private pollTasks;
|
|
85
115
|
private pollResults;
|
|
116
|
+
/** Wait 1s, or until `close()` aborts. The abort listener is removed when the timer fires, so
|
|
117
|
+
* sustained failures never accumulate listeners on the shared signal. */
|
|
118
|
+
private backoff;
|
|
86
119
|
/** One long-poll receive. Swallows the abort thrown on `close()`; logs and backs off otherwise. */
|
|
87
120
|
private receive;
|
|
88
121
|
private shouldSkip;
|
package/dist/index.js
CHANGED
|
@@ -2,8 +2,9 @@ var __defProp = Object.defineProperty;
|
|
|
2
2
|
var __name = (target, value) => __defProp(target, "name", { value, configurable: true });
|
|
3
3
|
|
|
4
4
|
// src/sqs-transport.ts
|
|
5
|
-
import { ChangeMessageVisibilityCommand, CreateQueueCommand, DeleteMessageCommand, GetQueueUrlCommand, ReceiveMessageCommand,
|
|
5
|
+
import { ChangeMessageVisibilityCommand, CreateQueueCommand, DeleteMessageCommand, GetQueueUrlCommand, ReceiveMessageCommand, SQSClient, SendMessageCommand } from "@aws-sdk/client-sqs";
|
|
6
6
|
import { runStepHandler } from "@dudousxd/nestjs-durable-core";
|
|
7
|
+
var RESULT_VISIBILITY_SEC = 30;
|
|
7
8
|
var SqsTransport = class {
|
|
8
9
|
static {
|
|
9
10
|
__name(this, "SqsTransport");
|
|
@@ -18,8 +19,11 @@ var SqsTransport = class {
|
|
|
18
19
|
visibilityTimeoutSec;
|
|
19
20
|
taskQueueUrlResolver;
|
|
20
21
|
resultsQueueUrlOption;
|
|
22
|
+
onError;
|
|
21
23
|
handlers = /* @__PURE__ */ new Map();
|
|
22
24
|
urlCache = /* @__PURE__ */ new Map();
|
|
25
|
+
/** `taskQueueUrl` resolutions per group (the promise, so concurrent dispatches share one call). */
|
|
26
|
+
groupUrlCache = /* @__PURE__ */ new Map();
|
|
23
27
|
running = true;
|
|
24
28
|
abort = new AbortController();
|
|
25
29
|
taskLoop;
|
|
@@ -35,6 +39,7 @@ var SqsTransport = class {
|
|
|
35
39
|
this.visibilityTimeoutSec = options.visibilityTimeoutSec ?? 60;
|
|
36
40
|
this.taskQueueUrlResolver = options.taskQueueUrl;
|
|
37
41
|
this.resultsQueueUrlOption = options.resultsQueueUrl;
|
|
42
|
+
this.onError = options.onError ?? ((err) => console.error("[SqsTransport]", err));
|
|
38
43
|
}
|
|
39
44
|
// ── queue resolution ───────────────────────────────────────────────────────────────────────
|
|
40
45
|
async resolveByName(name) {
|
|
@@ -60,8 +65,17 @@ var SqsTransport = class {
|
|
|
60
65
|
return url;
|
|
61
66
|
}
|
|
62
67
|
async tasksUrl(group) {
|
|
63
|
-
|
|
64
|
-
return this.resolveByName(`${this.prefix}-tasks-${group}`);
|
|
68
|
+
const resolver = this.taskQueueUrlResolver;
|
|
69
|
+
if (!resolver) return this.resolveByName(`${this.prefix}-tasks-${group}`);
|
|
70
|
+
let url = this.groupUrlCache.get(group);
|
|
71
|
+
if (!url) {
|
|
72
|
+
url = Promise.resolve().then(() => resolver(group)).catch((err) => {
|
|
73
|
+
this.groupUrlCache.delete(group);
|
|
74
|
+
throw err;
|
|
75
|
+
});
|
|
76
|
+
this.groupUrlCache.set(group, url);
|
|
77
|
+
}
|
|
78
|
+
return url;
|
|
65
79
|
}
|
|
66
80
|
async resultsUrl() {
|
|
67
81
|
const opt = this.resultsQueueUrlOption;
|
|
@@ -96,45 +110,123 @@ var SqsTransport = class {
|
|
|
96
110
|
onHeartbeat(_handler) {
|
|
97
111
|
}
|
|
98
112
|
// ── polling loops ──────────────────────────────────────────────────────────────────────────
|
|
99
|
-
|
|
100
|
-
|
|
113
|
+
/**
|
|
114
|
+
* Receive a batch and hand each message to `each`, one at a time, never letting a failure escape:
|
|
115
|
+
* a throw is reported through `onError` and its message is left on the queue (SQS redelivers it
|
|
116
|
+
* after the visibility timeout; a redrive policy DLQs one that keeps failing). Every message of the
|
|
117
|
+
* batch is kept invisible until it is handled (see {@link holdVisible}), so the ones waiting behind a
|
|
118
|
+
* slow handler are never redelivered to another consumer meanwhile.
|
|
119
|
+
*/
|
|
120
|
+
async pollLoop(queueUrl, visibilitySec, each) {
|
|
101
121
|
while (this.running) {
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
122
|
+
let QueueUrl;
|
|
123
|
+
try {
|
|
124
|
+
QueueUrl = await queueUrl();
|
|
125
|
+
} catch (err) {
|
|
126
|
+
if (!this.running) return;
|
|
127
|
+
this.onError(err);
|
|
128
|
+
await this.backoff();
|
|
129
|
+
continue;
|
|
130
|
+
}
|
|
131
|
+
const messages = await this.receive(QueueUrl, visibilitySec);
|
|
132
|
+
const pending = new Set(messages);
|
|
133
|
+
const stopHolding = this.holdVisible(QueueUrl, pending, visibilitySec);
|
|
134
|
+
try {
|
|
135
|
+
for (const msg of messages) {
|
|
136
|
+
if (!this.running) break;
|
|
137
|
+
try {
|
|
138
|
+
if (this.shouldSkip(msg)) {
|
|
139
|
+
pending.delete(msg);
|
|
140
|
+
await this.release(QueueUrl, msg.ReceiptHandle);
|
|
141
|
+
continue;
|
|
117
142
|
}
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
143
|
+
await each(QueueUrl, msg);
|
|
144
|
+
} catch (err) {
|
|
145
|
+
this.onError(err);
|
|
146
|
+
} finally {
|
|
147
|
+
pending.delete(msg);
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
} finally {
|
|
151
|
+
stopHolding();
|
|
121
152
|
}
|
|
122
153
|
}
|
|
123
154
|
}
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
155
|
+
/**
|
|
156
|
+
* Keep `messages` invisible while they wait or run: every third of the visibility timeout, extend
|
|
157
|
+
* each one still in the set back to the full timeout. Returns the stop function.
|
|
158
|
+
*/
|
|
159
|
+
holdVisible(QueueUrl, messages, visibilitySec) {
|
|
160
|
+
if (messages.size === 0) return () => {
|
|
161
|
+
};
|
|
162
|
+
const intervalMs = Math.max(200, Math.floor(visibilitySec * 1e3 / 3));
|
|
163
|
+
let round;
|
|
164
|
+
const timer = setInterval(() => {
|
|
165
|
+
if (round) return;
|
|
166
|
+
round = Promise.all([
|
|
167
|
+
...messages
|
|
168
|
+
].map((msg) => msg.ReceiptHandle ? this.client.send(new ChangeMessageVisibilityCommand({
|
|
169
|
+
QueueUrl,
|
|
170
|
+
ReceiptHandle: msg.ReceiptHandle,
|
|
171
|
+
VisibilityTimeout: visibilitySec
|
|
172
|
+
})).then(() => void 0).catch((err) => {
|
|
173
|
+
if (this.running) this.onError(err);
|
|
174
|
+
}) : void 0)).then(() => {
|
|
175
|
+
round = void 0;
|
|
176
|
+
});
|
|
177
|
+
}, intervalMs);
|
|
178
|
+
timer.unref?.();
|
|
179
|
+
return () => clearInterval(timer);
|
|
180
|
+
}
|
|
181
|
+
parse(msg) {
|
|
182
|
+
try {
|
|
183
|
+
return JSON.parse(msg.Body ?? "");
|
|
184
|
+
} catch {
|
|
185
|
+
throw new Error(`[SqsTransport] message ${msg.MessageId ?? "?"} has a non-JSON body; leaving it for the queue's redrive policy`);
|
|
136
186
|
}
|
|
137
187
|
}
|
|
188
|
+
pollTasks(group) {
|
|
189
|
+
return this.pollLoop(() => this.tasksUrl(group), this.visibilityTimeoutSec, async (QueueUrl, msg) => {
|
|
190
|
+
const task = this.parse(msg);
|
|
191
|
+
const result = await runStepHandler(task, this.handlers.get(task.name));
|
|
192
|
+
await this.client.send(new SendMessageCommand({
|
|
193
|
+
QueueUrl: await this.resultsUrl(),
|
|
194
|
+
MessageBody: JSON.stringify(result),
|
|
195
|
+
MessageAttributes: this.marker ? {
|
|
196
|
+
[this.marker]: {
|
|
197
|
+
DataType: "String",
|
|
198
|
+
StringValue: "1"
|
|
199
|
+
}
|
|
200
|
+
} : void 0
|
|
201
|
+
}));
|
|
202
|
+
await this.del(QueueUrl, msg.ReceiptHandle);
|
|
203
|
+
});
|
|
204
|
+
}
|
|
205
|
+
pollResults(handler) {
|
|
206
|
+
return this.pollLoop(() => this.resultsUrl(), RESULT_VISIBILITY_SEC, async (QueueUrl, msg) => {
|
|
207
|
+
await handler(this.parse(msg));
|
|
208
|
+
await this.del(QueueUrl, msg.ReceiptHandle);
|
|
209
|
+
});
|
|
210
|
+
}
|
|
211
|
+
/** Wait 1s, or until `close()` aborts. The abort listener is removed when the timer fires, so
|
|
212
|
+
* sustained failures never accumulate listeners on the shared signal. */
|
|
213
|
+
backoff() {
|
|
214
|
+
return new Promise((resolve) => {
|
|
215
|
+
const signal = this.abort.signal;
|
|
216
|
+
if (signal.aborted) return resolve();
|
|
217
|
+
const onAbort = /* @__PURE__ */ __name(() => {
|
|
218
|
+
clearTimeout(timer);
|
|
219
|
+
resolve();
|
|
220
|
+
}, "onAbort");
|
|
221
|
+
const timer = setTimeout(() => {
|
|
222
|
+
signal.removeEventListener("abort", onAbort);
|
|
223
|
+
resolve();
|
|
224
|
+
}, 1e3);
|
|
225
|
+
signal.addEventListener("abort", onAbort, {
|
|
226
|
+
once: true
|
|
227
|
+
});
|
|
228
|
+
});
|
|
229
|
+
}
|
|
138
230
|
/** One long-poll receive. Swallows the abort thrown on `close()`; logs and backs off otherwise. */
|
|
139
231
|
async receive(QueueUrl, visibility) {
|
|
140
232
|
try {
|
|
@@ -152,8 +244,8 @@ var SqsTransport = class {
|
|
|
152
244
|
return out.Messages ?? [];
|
|
153
245
|
} catch (err) {
|
|
154
246
|
if (!this.running) return [];
|
|
155
|
-
|
|
156
|
-
await
|
|
247
|
+
this.onError(err);
|
|
248
|
+
await this.backoff();
|
|
157
249
|
return [];
|
|
158
250
|
}
|
|
159
251
|
}
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/sqs-transport.ts"],"sourcesContent":["import {\n ChangeMessageVisibilityCommand,\n CreateQueueCommand,\n DeleteMessageCommand,\n GetQueueUrlCommand,\n ReceiveMessageCommand,\n SendMessageCommand,\n SQSClient,\n type SQSClientConfig,\n} from '@aws-sdk/client-sqs';\nimport {\n type Heartbeat,\n type RemoteTask,\n type StepHandler,\n type StepResult,\n type Transport,\n runStepHandler,\n} from '@dudousxd/nestjs-durable-core';\n\n/** Resolve a group's tasks queue URL. Sync or async; resolved once and cached. */\nexport type QueueUrlResolver = (group: string) => string | Promise<string>;\n\nexport interface SqsTransportOptions {\n /**\n * Reuse an existing SQS client (recommended — share the app's). When omitted, one is built\n * from `clientConfig` and destroyed on `close()`.\n */\n client?: SQSClient;\n /** Used only when `client` is omitted — e.g. `{ region, endpoint, credentials }` for ElasticMQ. */\n clientConfig?: SQSClientConfig;\n\n /** The worker group this instance serves. Required to register `handle()` consumers. */\n group?: string;\n\n /**\n * Map a group → an **existing** tasks queue URL. This is the seam for reusing queues you already\n * have (e.g. `extraction`, `ingestion`) instead of letting the transport mint its own. When\n * omitted, the transport resolves `${prefix}-tasks-${group}` by name (and creates it if\n * `autoCreate`). Return value is awaited and cached.\n */\n taskQueueUrl?: QueueUrlResolver;\n /**\n * The results queue URL (worker → engine). A literal URL or a resolver. When omitted, the\n * transport resolves `${prefix}-results` by name (and creates it if `autoCreate`).\n */\n resultsQueueUrl?: string | (() => string | Promise<string>);\n\n /** Prefix for the fallback `${prefix}-tasks-*` / `${prefix}-results` names. Default `durable`. */\n prefix?: string;\n /**\n * Create the fallback-named queues on first use if they don't exist (handy for ElasticMQ / dev).\n * Ignored for queues you supply via `taskQueueUrl` / `resultsQueueUrl` — those must already exist.\n */\n autoCreate?: boolean;\n\n /**\n * When set, durable messages carry this SQS message attribute and pollers **skip** messages\n * that lack it (releasing them for a co-resident legacy consumer). Enables sharing one physical\n * queue with a non-durable consumer — but only if that consumer likewise ignores tagged messages.\n * Leave unset for the clean case: a queue dedicated to durable traffic.\n */\n marker?: string;\n\n /** SQS receive long-poll seconds (0–20). Default 20. */\n waitTimeSeconds?: number;\n /** Visibility timeout for an in-flight task, in seconds. Must exceed the step's runtime. Default 60. */\n visibilityTimeoutSec?: number;\n}\n\n/**\n * An SQS-backed `Transport` — dispatch remote steps over Amazon SQS (or ElasticMQ). Unlike the\n * BullMQ transport, it does **not** mint its own queue names: pass `taskQueueUrl` / `resultsQueueUrl`\n * to ride the queues you already operate. The wire payload is the documented `RemoteTask` /\n * `StepResult` JSON, so non-Node workers on the same queues interoperate.\n *\n * Run one instance engine-side (calls `onResult`, dispatches tasks) and one per worker process\n * (calls `handle()` for its group). SQS is at-least-once: a task whose process crashes before the\n * result is sent reappears after the visibility timeout (durable retry); a duplicate `StepResult`\n * is harmless because the engine resolves a step's `pending` exactly once by `stepId`.\n */\nexport class SqsTransport implements Transport {\n private readonly client: SQSClient;\n private readonly ownsClient: boolean;\n private readonly group?: string | undefined;\n private readonly prefix: string;\n private readonly autoCreate: boolean;\n private readonly marker?: string | undefined;\n private readonly waitTimeSeconds: number;\n private readonly visibilityTimeoutSec: number;\n private readonly taskQueueUrlResolver?: QueueUrlResolver | undefined;\n private readonly resultsQueueUrlOption?: string | (() => string | Promise<string>) | undefined;\n\n private readonly handlers = new Map<string, StepHandler>();\n private readonly urlCache = new Map<string, string>();\n private running = true;\n private readonly abort = new AbortController();\n private taskLoop?: Promise<void>;\n private resultLoop?: Promise<void>;\n\n constructor(options: SqsTransportOptions) {\n this.client = options.client ?? new SQSClient(options.clientConfig ?? {});\n this.ownsClient = !options.client;\n this.group = options.group;\n this.prefix = options.prefix ?? 'durable';\n this.autoCreate = options.autoCreate ?? false;\n this.marker = options.marker;\n this.waitTimeSeconds = options.waitTimeSeconds ?? 20;\n this.visibilityTimeoutSec = options.visibilityTimeoutSec ?? 60;\n this.taskQueueUrlResolver = options.taskQueueUrl;\n this.resultsQueueUrlOption = options.resultsQueueUrl;\n }\n\n // ── queue resolution ───────────────────────────────────────────────────────────────────────\n\n private async resolveByName(name: string): Promise<string> {\n const cached = this.urlCache.get(name);\n if (cached) return cached;\n let url: string;\n try {\n const out = await this.client.send(new GetQueueUrlCommand({ QueueName: name }));\n url = out.QueueUrl as string;\n } catch (err) {\n if (this.autoCreate) {\n const created = await this.client.send(new CreateQueueCommand({ QueueName: name }));\n url = created.QueueUrl as string;\n } else {\n throw err;\n }\n }\n this.urlCache.set(name, url);\n return url;\n }\n\n private async tasksUrl(group: string): Promise<string> {\n if (this.taskQueueUrlResolver) return this.taskQueueUrlResolver(group);\n return this.resolveByName(`${this.prefix}-tasks-${group}`);\n }\n\n private async resultsUrl(): Promise<string> {\n const opt = this.resultsQueueUrlOption;\n if (typeof opt === 'string') return opt;\n if (typeof opt === 'function') return opt();\n return this.resolveByName(`${this.prefix}-results`);\n }\n\n // ── engine → worker ────────────────────────────────────────────────────────────────────────\n\n async dispatch(task: RemoteTask): Promise<void> {\n const QueueUrl = await this.tasksUrl(task.group);\n await this.client.send(\n new SendMessageCommand({\n QueueUrl,\n MessageBody: JSON.stringify(task),\n MessageAttributes: this.marker\n ? { [this.marker]: { DataType: 'String', StringValue: '1' } }\n : undefined,\n }),\n );\n }\n\n // ── worker → engine ────────────────────────────────────────────────────────────────────────\n\n /** Register a step handler (worker side). Starts the group's task consumer on first call. */\n handle(name: string, fn: StepHandler): void {\n if (!this.group) throw new Error('SqsTransport needs a `group` to register handlers');\n this.handlers.set(name, fn);\n if (!this.taskLoop) this.taskLoop = this.pollTasks(this.group);\n }\n\n onResult(handler: (result: StepResult) => Promise<void>): void {\n if (!this.resultLoop) this.resultLoop = this.pollResults(handler);\n }\n\n onHeartbeat(_handler: (beat: Heartbeat) => Promise<void>): void {\n // Not modelled over SQS; the queue's visibility-timeout redelivery is the liveness mechanism.\n }\n\n // ── polling loops ──────────────────────────────────────────────────────────────────────────\n\n private async pollTasks(group: string): Promise<void> {\n const QueueUrl = await this.tasksUrl(group);\n while (this.running) {\n const messages = await this.receive(QueueUrl, this.visibilityTimeoutSec);\n for (const msg of messages) {\n if (this.shouldSkip(msg)) {\n // Not ours (shared queue): release immediately so the legacy consumer can take it.\n await this.release(QueueUrl, msg.ReceiptHandle);\n continue;\n }\n const task = JSON.parse(msg.Body ?? '{}') as RemoteTask;\n const result = await runStepHandler(task, this.handlers.get(task.name));\n await this.client.send(\n new SendMessageCommand({\n QueueUrl: await this.resultsUrl(),\n MessageBody: JSON.stringify(result),\n MessageAttributes: this.marker\n ? { [this.marker]: { DataType: 'String', StringValue: '1' } }\n : undefined,\n }),\n );\n await this.del(QueueUrl, msg.ReceiptHandle);\n }\n }\n }\n\n private async pollResults(handler: (result: StepResult) => Promise<void>): Promise<void> {\n const QueueUrl = await this.resultsUrl();\n while (this.running) {\n const messages = await this.receive(QueueUrl, 30);\n for (const msg of messages) {\n if (this.shouldSkip(msg)) {\n await this.release(QueueUrl, msg.ReceiptHandle);\n continue;\n }\n await handler(JSON.parse(msg.Body ?? '{}') as StepResult);\n await this.del(QueueUrl, msg.ReceiptHandle);\n }\n }\n }\n\n /** One long-poll receive. Swallows the abort thrown on `close()`; logs and backs off otherwise. */\n private async receive(QueueUrl: string, visibility: number) {\n try {\n const out = await this.client.send(\n new ReceiveMessageCommand({\n QueueUrl,\n MaxNumberOfMessages: 10,\n WaitTimeSeconds: this.waitTimeSeconds,\n VisibilityTimeout: visibility,\n MessageAttributeNames: this.marker ? ['All'] : undefined,\n }),\n { abortSignal: this.abort.signal },\n );\n return out.Messages ?? [];\n } catch (err) {\n if (!this.running) return [];\n console.error('[SqsTransport] receive failed', err);\n await new Promise((r) => setTimeout(r, 1000));\n return [];\n }\n }\n\n private shouldSkip(msg: { MessageAttributes?: Record<string, unknown> | undefined }): boolean {\n return !!this.marker && !msg.MessageAttributes?.[this.marker];\n }\n\n private async del(QueueUrl: string, ReceiptHandle?: string): Promise<void> {\n if (!ReceiptHandle) return;\n await this.client.send(new DeleteMessageCommand({ QueueUrl, ReceiptHandle }));\n }\n\n private async release(QueueUrl: string, ReceiptHandle?: string): Promise<void> {\n if (!ReceiptHandle) return;\n await this.client\n .send(new ChangeMessageVisibilityCommand({ QueueUrl, ReceiptHandle, VisibilityTimeout: 0 }))\n .catch(() => {});\n }\n\n /** Stop the pollers and (if we created it) destroy the SQS client so the process can exit. */\n async close(): Promise<void> {\n this.running = false;\n this.abort.abort();\n await Promise.allSettled([this.taskLoop, this.resultLoop]);\n if (this.ownsClient) this.client.destroy();\n }\n}\n"],"mappings":";;;;AAAA,SACEA,gCACAC,oBACAC,sBACAC,oBACAC,uBACAC,oBACAC,iBAEK;AACP,SAMEC,sBACK;AA+DA,IAAMC,eAAN,MAAMA;EAhFb,OAgFaA;;;EACMC;EACAC;EACAC;EACAC;EACAC;EACAC;EACAC;EACAC;EACAC;EACAC;EAEAC,WAAW,oBAAIC,IAAAA;EACfC,WAAW,oBAAID,IAAAA;EACxBE,UAAU;EACDC,QAAQ,IAAIC,gBAAAA;EACrBC;EACAC;EAER,YAAYC,SAA8B;AACxC,SAAKlB,SAASkB,QAAQlB,UAAU,IAAImB,UAAUD,QAAQE,gBAAgB,CAAC,CAAA;AACvE,SAAKnB,aAAa,CAACiB,QAAQlB;AAC3B,SAAKE,QAAQgB,QAAQhB;AACrB,SAAKC,SAASe,QAAQf,UAAU;AAChC,SAAKC,aAAac,QAAQd,cAAc;AACxC,SAAKC,SAASa,QAAQb;AACtB,SAAKC,kBAAkBY,QAAQZ,mBAAmB;AAClD,SAAKC,uBAAuBW,QAAQX,wBAAwB;AAC5D,SAAKC,uBAAuBU,QAAQG;AACpC,SAAKZ,wBAAwBS,QAAQI;EACvC;;EAIA,MAAcC,cAAcC,MAA+B;AACzD,UAAMC,SAAS,KAAKb,SAASc,IAAIF,IAAAA;AACjC,QAAIC,OAAQ,QAAOA;AACnB,QAAIE;AACJ,QAAI;AACF,YAAMC,MAAM,MAAM,KAAK5B,OAAO6B,KAAK,IAAIC,mBAAmB;QAAEC,WAAWP;MAAK,CAAA,CAAA;AAC5EG,YAAMC,IAAII;IACZ,SAASC,KAAK;AACZ,UAAI,KAAK7B,YAAY;AACnB,cAAM8B,UAAU,MAAM,KAAKlC,OAAO6B,KAAK,IAAIM,mBAAmB;UAAEJ,WAAWP;QAAK,CAAA,CAAA;AAChFG,cAAMO,QAAQF;MAChB,OAAO;AACL,cAAMC;MACR;IACF;AACA,SAAKrB,SAASwB,IAAIZ,MAAMG,GAAAA;AACxB,WAAOA;EACT;EAEA,MAAcU,SAASnC,OAAgC;AACrD,QAAI,KAAKM,qBAAsB,QAAO,KAAKA,qBAAqBN,KAAAA;AAChE,WAAO,KAAKqB,cAAc,GAAG,KAAKpB,MAAM,UAAUD,KAAAA,EAAO;EAC3D;EAEA,MAAcoC,aAA8B;AAC1C,UAAMC,MAAM,KAAK9B;AACjB,QAAI,OAAO8B,QAAQ,SAAU,QAAOA;AACpC,QAAI,OAAOA,QAAQ,WAAY,QAAOA,IAAAA;AACtC,WAAO,KAAKhB,cAAc,GAAG,KAAKpB,MAAM,UAAU;EACpD;;EAIA,MAAMqC,SAASC,MAAiC;AAC9C,UAAMT,WAAW,MAAM,KAAKK,SAASI,KAAKvC,KAAK;AAC/C,UAAM,KAAKF,OAAO6B,KAChB,IAAIa,mBAAmB;MACrBV;MACAW,aAAaC,KAAKC,UAAUJ,IAAAA;MAC5BK,mBAAmB,KAAKzC,SACpB;QAAE,CAAC,KAAKA,MAAM,GAAG;UAAE0C,UAAU;UAAUC,aAAa;QAAI;MAAE,IAC1DC;IACN,CAAA,CAAA;EAEJ;;;EAKAC,OAAO1B,MAAc2B,IAAuB;AAC1C,QAAI,CAAC,KAAKjD,MAAO,OAAM,IAAIkD,MAAM,mDAAA;AACjC,SAAK1C,SAAS0B,IAAIZ,MAAM2B,EAAAA;AACxB,QAAI,CAAC,KAAKnC,SAAU,MAAKA,WAAW,KAAKqC,UAAU,KAAKnD,KAAK;EAC/D;EAEAoD,SAASC,SAAsD;AAC7D,QAAI,CAAC,KAAKtC,WAAY,MAAKA,aAAa,KAAKuC,YAAYD,OAAAA;EAC3D;EAEAE,YAAYC,UAAoD;EAEhE;;EAIA,MAAcL,UAAUnD,OAA8B;AACpD,UAAM8B,WAAW,MAAM,KAAKK,SAASnC,KAAAA;AACrC,WAAO,KAAKW,SAAS;AACnB,YAAM8C,WAAW,MAAM,KAAKC,QAAQ5B,UAAU,KAAKzB,oBAAoB;AACvE,iBAAWsD,OAAOF,UAAU;AAC1B,YAAI,KAAKG,WAAWD,GAAAA,GAAM;AAExB,gBAAM,KAAKE,QAAQ/B,UAAU6B,IAAIG,aAAa;AAC9C;QACF;AACA,cAAMvB,OAAOG,KAAKqB,MAAMJ,IAAIK,QAAQ,IAAA;AACpC,cAAMC,SAAS,MAAMC,eAAe3B,MAAM,KAAK/B,SAASgB,IAAIe,KAAKjB,IAAI,CAAA;AACrE,cAAM,KAAKxB,OAAO6B,KAChB,IAAIa,mBAAmB;UACrBV,UAAU,MAAM,KAAKM,WAAU;UAC/BK,aAAaC,KAAKC,UAAUsB,MAAAA;UAC5BrB,mBAAmB,KAAKzC,SACpB;YAAE,CAAC,KAAKA,MAAM,GAAG;cAAE0C,UAAU;cAAUC,aAAa;YAAI;UAAE,IAC1DC;QACN,CAAA,CAAA;AAEF,cAAM,KAAKoB,IAAIrC,UAAU6B,IAAIG,aAAa;MAC5C;IACF;EACF;EAEA,MAAcR,YAAYD,SAA+D;AACvF,UAAMvB,WAAW,MAAM,KAAKM,WAAU;AACtC,WAAO,KAAKzB,SAAS;AACnB,YAAM8C,WAAW,MAAM,KAAKC,QAAQ5B,UAAU,EAAA;AAC9C,iBAAW6B,OAAOF,UAAU;AAC1B,YAAI,KAAKG,WAAWD,GAAAA,GAAM;AACxB,gBAAM,KAAKE,QAAQ/B,UAAU6B,IAAIG,aAAa;AAC9C;QACF;AACA,cAAMT,QAAQX,KAAKqB,MAAMJ,IAAIK,QAAQ,IAAA,CAAA;AACrC,cAAM,KAAKG,IAAIrC,UAAU6B,IAAIG,aAAa;MAC5C;IACF;EACF;;EAGA,MAAcJ,QAAQ5B,UAAkBsC,YAAoB;AAC1D,QAAI;AACF,YAAM1C,MAAM,MAAM,KAAK5B,OAAO6B,KAC5B,IAAI0C,sBAAsB;QACxBvC;QACAwC,qBAAqB;QACrBC,iBAAiB,KAAKnE;QACtBoE,mBAAmBJ;QACnBK,uBAAuB,KAAKtE,SAAS;UAAC;YAAS4C;MACjD,CAAA,GACA;QAAE2B,aAAa,KAAK9D,MAAM+D;MAAO,CAAA;AAEnC,aAAOjD,IAAIkD,YAAY,CAAA;IACzB,SAAS7C,KAAK;AACZ,UAAI,CAAC,KAAKpB,QAAS,QAAO,CAAA;AAC1BkE,cAAQC,MAAM,iCAAiC/C,GAAAA;AAC/C,YAAM,IAAIgD,QAAQ,CAACC,MAAMC,WAAWD,GAAG,GAAA,CAAA;AACvC,aAAO,CAAA;IACT;EACF;EAEQpB,WAAWD,KAA2E;AAC5F,WAAO,CAAC,CAAC,KAAKxD,UAAU,CAACwD,IAAIf,oBAAoB,KAAKzC,MAAM;EAC9D;EAEA,MAAcgE,IAAIrC,UAAkBgC,eAAuC;AACzE,QAAI,CAACA,cAAe;AACpB,UAAM,KAAKhE,OAAO6B,KAAK,IAAIuD,qBAAqB;MAAEpD;MAAUgC;IAAc,CAAA,CAAA;EAC5E;EAEA,MAAcD,QAAQ/B,UAAkBgC,eAAuC;AAC7E,QAAI,CAACA,cAAe;AACpB,UAAM,KAAKhE,OACR6B,KAAK,IAAIwD,+BAA+B;MAAErD;MAAUgC;MAAeU,mBAAmB;IAAE,CAAA,CAAA,EACxFY,MAAM,MAAA;IAAO,CAAA;EAClB;;EAGA,MAAMC,QAAuB;AAC3B,SAAK1E,UAAU;AACf,SAAKC,MAAMA,MAAK;AAChB,UAAMmE,QAAQO,WAAW;MAAC,KAAKxE;MAAU,KAAKC;KAAW;AACzD,QAAI,KAAKhB,WAAY,MAAKD,OAAOyF,QAAO;EAC1C;AACF;","names":["ChangeMessageVisibilityCommand","CreateQueueCommand","DeleteMessageCommand","GetQueueUrlCommand","ReceiveMessageCommand","SendMessageCommand","SQSClient","runStepHandler","SqsTransport","client","ownsClient","group","prefix","autoCreate","marker","waitTimeSeconds","visibilityTimeoutSec","taskQueueUrlResolver","resultsQueueUrlOption","handlers","Map","urlCache","running","abort","AbortController","taskLoop","resultLoop","options","SQSClient","clientConfig","taskQueueUrl","resultsQueueUrl","resolveByName","name","cached","get","url","out","send","GetQueueUrlCommand","QueueName","QueueUrl","err","created","CreateQueueCommand","set","tasksUrl","resultsUrl","opt","dispatch","task","SendMessageCommand","MessageBody","JSON","stringify","MessageAttributes","DataType","StringValue","undefined","handle","fn","Error","pollTasks","onResult","handler","pollResults","onHeartbeat","_handler","messages","receive","msg","shouldSkip","release","ReceiptHandle","parse","Body","result","runStepHandler","del","visibility","ReceiveMessageCommand","MaxNumberOfMessages","WaitTimeSeconds","VisibilityTimeout","MessageAttributeNames","abortSignal","signal","Messages","console","error","Promise","r","setTimeout","DeleteMessageCommand","ChangeMessageVisibilityCommand","catch","close","allSettled","destroy"]}
|
|
1
|
+
{"version":3,"sources":["../src/sqs-transport.ts"],"sourcesContent":["import {\n ChangeMessageVisibilityCommand,\n CreateQueueCommand,\n DeleteMessageCommand,\n GetQueueUrlCommand,\n ReceiveMessageCommand,\n SQSClient,\n type SQSClientConfig,\n SendMessageCommand,\n} from '@aws-sdk/client-sqs';\nimport {\n type Heartbeat,\n type RemoteTask,\n type StepHandler,\n type StepResult,\n type Transport,\n runStepHandler,\n} from '@dudousxd/nestjs-durable-core';\n\n/** Resolve a group's tasks queue URL. Sync or async; resolved once per group and cached (a rejected\n * resolution is not cached — the next call asks again). */\nexport type QueueUrlResolver = (group: string) => string | Promise<string>;\n\nexport interface SqsTransportOptions {\n /**\n * Reuse an existing SQS client (recommended — share the app's). When omitted, one is built\n * from `clientConfig` and destroyed on `close()`.\n */\n client?: SQSClient;\n /** Used only when `client` is omitted — e.g. `{ region, endpoint, credentials }` for ElasticMQ. */\n clientConfig?: SQSClientConfig;\n\n /** The worker group this instance serves. Required to register `handle()` consumers. */\n group?: string;\n\n /**\n * Map a group → an **existing** tasks queue URL. This is the seam for reusing queues you already\n * have (e.g. `extraction`, `ingestion`) instead of letting the transport mint its own. When\n * omitted, the transport resolves `${prefix}-tasks-${group}` by name (and creates it if\n * `autoCreate`). Return value is awaited and cached per group.\n */\n taskQueueUrl?: QueueUrlResolver;\n /**\n * The results queue URL (worker → engine). A literal URL or a resolver. When omitted, the\n * transport resolves `${prefix}-results` by name (and creates it if `autoCreate`).\n */\n resultsQueueUrl?: string | (() => string | Promise<string>);\n\n /** Prefix for the fallback `${prefix}-tasks-*` / `${prefix}-results` names. Default `durable`. */\n prefix?: string;\n /**\n * Create the fallback-named queues on first use if they don't exist (handy for ElasticMQ / dev).\n * Ignored for queues you supply via `taskQueueUrl` / `resultsQueueUrl` — those must already exist.\n */\n autoCreate?: boolean;\n\n /**\n * When set, durable messages carry this SQS message attribute and pollers **skip** messages\n * that lack it (releasing them for a co-resident legacy consumer). Enables sharing one physical\n * queue with a non-durable consumer — but only if that consumer likewise ignores tagged messages.\n * Leave unset for the clean case: a queue dedicated to durable traffic.\n */\n marker?: string;\n\n /** SQS receive long-poll seconds (0–20). Default 20. */\n waitTimeSeconds?: number;\n /**\n * Visibility timeout for a received task, in seconds. Default 60. Every message this instance has\n * received — the one running and the ones of the same batch waiting behind it — is kept invisible by\n * re-extending it (`ChangeMessageVisibility`) every third of this, so a step may run longer; only a\n * crashed instance's messages reappear, after at most this long.\n */\n visibilityTimeoutSec?: number;\n /**\n * Where a transport failure is reported: a failing receive, a result handler that throws, a result\n * send that fails, a message whose body isn't JSON. The message involved is NOT deleted: SQS\n * redelivers it after its visibility timeout, and a queue redrive policy (`maxReceiveCount` → DLQ)\n * moves one that keeps failing aside. Default `console.error`.\n */\n onError?: (err: unknown) => void;\n}\n\n/** A received message, as the poll loops use it. */\ninterface Received {\n Body?: string | undefined;\n ReceiptHandle?: string | undefined;\n MessageId?: string | undefined;\n MessageAttributes?: Record<string, unknown> | undefined;\n}\n\n/** The results queue's visibility timeout (seconds). */\nconst RESULT_VISIBILITY_SEC = 30;\n\n/**\n * An SQS-backed `Transport` — dispatch remote steps over Amazon SQS (or ElasticMQ). Unlike the\n * BullMQ transport, it does **not** mint its own queue names: pass `taskQueueUrl` / `resultsQueueUrl`\n * to ride the queues you already operate. The wire payload is the documented `RemoteTask` /\n * `StepResult` JSON, so non-Node workers on the same queues interoperate.\n *\n * Run one instance engine-side (calls `onResult`, dispatches tasks) and one per worker process\n * (calls `handle()` for its group). SQS is at-least-once: a task whose process crashes before the\n * result is sent reappears after the visibility timeout (durable retry); a duplicate `StepResult`\n * is harmless because the engine resolves a step's `pending` exactly once by `stepId`.\n */\nexport class SqsTransport implements Transport {\n private readonly client: SQSClient;\n private readonly ownsClient: boolean;\n private readonly group?: string | undefined;\n private readonly prefix: string;\n private readonly autoCreate: boolean;\n private readonly marker?: string | undefined;\n private readonly waitTimeSeconds: number;\n private readonly visibilityTimeoutSec: number;\n private readonly taskQueueUrlResolver?: QueueUrlResolver | undefined;\n private readonly resultsQueueUrlOption?: string | (() => string | Promise<string>) | undefined;\n private readonly onError: (err: unknown) => void;\n\n private readonly handlers = new Map<string, StepHandler>();\n private readonly urlCache = new Map<string, string>();\n /** `taskQueueUrl` resolutions per group (the promise, so concurrent dispatches share one call). */\n private readonly groupUrlCache = new Map<string, Promise<string>>();\n private running = true;\n private readonly abort = new AbortController();\n private taskLoop?: Promise<void>;\n private resultLoop?: Promise<void>;\n\n constructor(options: SqsTransportOptions) {\n this.client = options.client ?? new SQSClient(options.clientConfig ?? {});\n this.ownsClient = !options.client;\n this.group = options.group;\n this.prefix = options.prefix ?? 'durable';\n this.autoCreate = options.autoCreate ?? false;\n this.marker = options.marker;\n this.waitTimeSeconds = options.waitTimeSeconds ?? 20;\n this.visibilityTimeoutSec = options.visibilityTimeoutSec ?? 60;\n this.taskQueueUrlResolver = options.taskQueueUrl;\n this.resultsQueueUrlOption = options.resultsQueueUrl;\n this.onError = options.onError ?? ((err) => console.error('[SqsTransport]', err));\n }\n\n // ── queue resolution ───────────────────────────────────────────────────────────────────────\n\n private async resolveByName(name: string): Promise<string> {\n const cached = this.urlCache.get(name);\n if (cached) return cached;\n let url: string;\n try {\n const out = await this.client.send(new GetQueueUrlCommand({ QueueName: name }));\n url = out.QueueUrl as string;\n } catch (err) {\n if (this.autoCreate) {\n const created = await this.client.send(new CreateQueueCommand({ QueueName: name }));\n url = created.QueueUrl as string;\n } else {\n throw err;\n }\n }\n this.urlCache.set(name, url);\n return url;\n }\n\n private async tasksUrl(group: string): Promise<string> {\n const resolver = this.taskQueueUrlResolver;\n if (!resolver) return this.resolveByName(`${this.prefix}-tasks-${group}`);\n let url = this.groupUrlCache.get(group);\n if (!url) {\n url = Promise.resolve()\n .then(() => resolver(group))\n .catch((err) => {\n this.groupUrlCache.delete(group);\n throw err;\n });\n this.groupUrlCache.set(group, url);\n }\n return url;\n }\n\n private async resultsUrl(): Promise<string> {\n const opt = this.resultsQueueUrlOption;\n if (typeof opt === 'string') return opt;\n if (typeof opt === 'function') return opt();\n return this.resolveByName(`${this.prefix}-results`);\n }\n\n // ── engine → worker ────────────────────────────────────────────────────────────────────────\n\n async dispatch(task: RemoteTask): Promise<void> {\n const QueueUrl = await this.tasksUrl(task.group);\n await this.client.send(\n new SendMessageCommand({\n QueueUrl,\n MessageBody: JSON.stringify(task),\n MessageAttributes: this.marker\n ? { [this.marker]: { DataType: 'String', StringValue: '1' } }\n : undefined,\n }),\n );\n }\n\n // ── worker → engine ────────────────────────────────────────────────────────────────────────\n\n /** Register a step handler (worker side). Starts the group's task consumer on first call. */\n handle(name: string, fn: StepHandler): void {\n if (!this.group) throw new Error('SqsTransport needs a `group` to register handlers');\n this.handlers.set(name, fn);\n if (!this.taskLoop) this.taskLoop = this.pollTasks(this.group);\n }\n\n onResult(handler: (result: StepResult) => Promise<void>): void {\n if (!this.resultLoop) this.resultLoop = this.pollResults(handler);\n }\n\n onHeartbeat(_handler: (beat: Heartbeat) => Promise<void>): void {\n // Not modelled over SQS; the queue's visibility-timeout redelivery is the liveness mechanism.\n }\n\n // ── polling loops ──────────────────────────────────────────────────────────────────────────\n\n /**\n * Receive a batch and hand each message to `each`, one at a time, never letting a failure escape:\n * a throw is reported through `onError` and its message is left on the queue (SQS redelivers it\n * after the visibility timeout; a redrive policy DLQs one that keeps failing). Every message of the\n * batch is kept invisible until it is handled (see {@link holdVisible}), so the ones waiting behind a\n * slow handler are never redelivered to another consumer meanwhile.\n */\n private async pollLoop(\n queueUrl: () => Promise<string>,\n visibilitySec: number,\n each: (QueueUrl: string, msg: Received) => Promise<void>,\n ): Promise<void> {\n while (this.running) {\n let QueueUrl: string;\n try {\n QueueUrl = await queueUrl();\n } catch (err) {\n if (!this.running) return;\n this.onError(err);\n await this.backoff();\n continue;\n }\n const messages = await this.receive(QueueUrl, visibilitySec);\n const pending = new Set<Received>(messages);\n const stopHolding = this.holdVisible(QueueUrl, pending, visibilitySec);\n try {\n for (const msg of messages) {\n if (!this.running) break; // close(): what is left reappears after its visibility timeout\n try {\n if (this.shouldSkip(msg)) {\n // Not ours (shared queue): release immediately so the legacy consumer can take it.\n pending.delete(msg);\n await this.release(QueueUrl, msg.ReceiptHandle);\n continue;\n }\n await each(QueueUrl, msg);\n } catch (err) {\n this.onError(err);\n } finally {\n pending.delete(msg);\n }\n }\n } finally {\n stopHolding();\n }\n }\n }\n\n /**\n * Keep `messages` invisible while they wait or run: every third of the visibility timeout, extend\n * each one still in the set back to the full timeout. Returns the stop function.\n */\n private holdVisible(\n QueueUrl: string,\n messages: Set<Received>,\n visibilitySec: number,\n ): () => void {\n if (messages.size === 0) return () => {};\n const intervalMs = Math.max(200, Math.floor((visibilitySec * 1000) / 3));\n let round: Promise<void> | undefined;\n const timer = setInterval(() => {\n if (round) return;\n round = Promise.all(\n [...messages].map((msg) =>\n msg.ReceiptHandle\n ? this.client\n .send(\n new ChangeMessageVisibilityCommand({\n QueueUrl,\n ReceiptHandle: msg.ReceiptHandle,\n VisibilityTimeout: visibilitySec,\n }),\n )\n .then(() => undefined)\n .catch((err) => {\n if (this.running) this.onError(err);\n })\n : undefined,\n ),\n ).then(() => {\n round = undefined;\n });\n }, intervalMs);\n timer.unref?.();\n return () => clearInterval(timer);\n }\n\n private parse<T>(msg: Received): T {\n try {\n return JSON.parse(msg.Body ?? '') as T;\n } catch {\n throw new Error(\n `[SqsTransport] message ${msg.MessageId ?? '?'} has a non-JSON body; leaving it for the queue's redrive policy`,\n );\n }\n }\n\n private pollTasks(group: string): Promise<void> {\n return this.pollLoop(\n () => this.tasksUrl(group),\n this.visibilityTimeoutSec,\n async (QueueUrl, msg) => {\n const task = this.parse<RemoteTask>(msg);\n const result = await runStepHandler(task, this.handlers.get(task.name));\n await this.client.send(\n new SendMessageCommand({\n QueueUrl: await this.resultsUrl(),\n MessageBody: JSON.stringify(result),\n MessageAttributes: this.marker\n ? { [this.marker]: { DataType: 'String', StringValue: '1' } }\n : undefined,\n }),\n );\n await this.del(QueueUrl, msg.ReceiptHandle);\n },\n );\n }\n\n private pollResults(handler: (result: StepResult) => Promise<void>): Promise<void> {\n return this.pollLoop(\n () => this.resultsUrl(),\n RESULT_VISIBILITY_SEC,\n async (QueueUrl, msg) => {\n await handler(this.parse<StepResult>(msg));\n await this.del(QueueUrl, msg.ReceiptHandle);\n },\n );\n }\n\n /** Wait 1s, or until `close()` aborts. The abort listener is removed when the timer fires, so\n * sustained failures never accumulate listeners on the shared signal. */\n private backoff(): Promise<void> {\n return new Promise((resolve) => {\n const signal = this.abort.signal;\n if (signal.aborted) return resolve();\n const onAbort = () => {\n clearTimeout(timer);\n resolve();\n };\n const timer = setTimeout(() => {\n signal.removeEventListener('abort', onAbort);\n resolve();\n }, 1000);\n signal.addEventListener('abort', onAbort, { once: true });\n });\n }\n\n /** One long-poll receive. Swallows the abort thrown on `close()`; logs and backs off otherwise. */\n private async receive(QueueUrl: string, visibility: number): Promise<Received[]> {\n try {\n const out = await this.client.send(\n new ReceiveMessageCommand({\n QueueUrl,\n MaxNumberOfMessages: 10,\n WaitTimeSeconds: this.waitTimeSeconds,\n VisibilityTimeout: visibility,\n MessageAttributeNames: this.marker ? ['All'] : undefined,\n }),\n { abortSignal: this.abort.signal },\n );\n return out.Messages ?? [];\n } catch (err) {\n if (!this.running) return [];\n this.onError(err);\n await this.backoff();\n return [];\n }\n }\n\n private shouldSkip(msg: { MessageAttributes?: Record<string, unknown> | undefined }): boolean {\n return !!this.marker && !msg.MessageAttributes?.[this.marker];\n }\n\n private async del(QueueUrl: string, ReceiptHandle?: string): Promise<void> {\n if (!ReceiptHandle) return;\n await this.client.send(new DeleteMessageCommand({ QueueUrl, ReceiptHandle }));\n }\n\n private async release(QueueUrl: string, ReceiptHandle?: string): Promise<void> {\n if (!ReceiptHandle) return;\n await this.client\n .send(new ChangeMessageVisibilityCommand({ QueueUrl, ReceiptHandle, VisibilityTimeout: 0 }))\n .catch(() => {});\n }\n\n /** Stop the pollers and (if we created it) destroy the SQS client so the process can exit. */\n async close(): Promise<void> {\n this.running = false;\n this.abort.abort();\n await Promise.allSettled([this.taskLoop, this.resultLoop]);\n if (this.ownsClient) this.client.destroy();\n }\n}\n"],"mappings":";;;;AAAA,SACEA,gCACAC,oBACAC,sBACAC,oBACAC,uBACAC,WAEAC,0BACK;AACP,SAMEC,sBACK;AA0EP,IAAMC,wBAAwB;AAavB,IAAMC,eAAN,MAAMA;EAxGb,OAwGaA;;;EACMC;EACAC;EACAC;EACAC;EACAC;EACAC;EACAC;EACAC;EACAC;EACAC;EACAC;EAEAC,WAAW,oBAAIC,IAAAA;EACfC,WAAW,oBAAID,IAAAA;;EAEfE,gBAAgB,oBAAIF,IAAAA;EAC7BG,UAAU;EACDC,QAAQ,IAAIC,gBAAAA;EACrBC;EACAC;EAER,YAAYC,SAA8B;AACxC,SAAKpB,SAASoB,QAAQpB,UAAU,IAAIqB,UAAUD,QAAQE,gBAAgB,CAAC,CAAA;AACvE,SAAKrB,aAAa,CAACmB,QAAQpB;AAC3B,SAAKE,QAAQkB,QAAQlB;AACrB,SAAKC,SAASiB,QAAQjB,UAAU;AAChC,SAAKC,aAAagB,QAAQhB,cAAc;AACxC,SAAKC,SAASe,QAAQf;AACtB,SAAKC,kBAAkBc,QAAQd,mBAAmB;AAClD,SAAKC,uBAAuBa,QAAQb,wBAAwB;AAC5D,SAAKC,uBAAuBY,QAAQG;AACpC,SAAKd,wBAAwBW,QAAQI;AACrC,SAAKd,UAAUU,QAAQV,YAAY,CAACe,QAAQC,QAAQC,MAAM,kBAAkBF,GAAAA;EAC9E;;EAIA,MAAcG,cAAcC,MAA+B;AACzD,UAAMC,SAAS,KAAKjB,SAASkB,IAAIF,IAAAA;AACjC,QAAIC,OAAQ,QAAOA;AACnB,QAAIE;AACJ,QAAI;AACF,YAAMC,MAAM,MAAM,KAAKjC,OAAOkC,KAAK,IAAIC,mBAAmB;QAAEC,WAAWP;MAAK,CAAA,CAAA;AAC5EG,YAAMC,IAAII;IACZ,SAASZ,KAAK;AACZ,UAAI,KAAKrB,YAAY;AACnB,cAAMkC,UAAU,MAAM,KAAKtC,OAAOkC,KAAK,IAAIK,mBAAmB;UAAEH,WAAWP;QAAK,CAAA,CAAA;AAChFG,cAAMM,QAAQD;MAChB,OAAO;AACL,cAAMZ;MACR;IACF;AACA,SAAKZ,SAAS2B,IAAIX,MAAMG,GAAAA;AACxB,WAAOA;EACT;EAEA,MAAcS,SAASvC,OAAgC;AACrD,UAAMwC,WAAW,KAAKlC;AACtB,QAAI,CAACkC,SAAU,QAAO,KAAKd,cAAc,GAAG,KAAKzB,MAAM,UAAUD,KAAAA,EAAO;AACxE,QAAI8B,MAAM,KAAKlB,cAAciB,IAAI7B,KAAAA;AACjC,QAAI,CAAC8B,KAAK;AACRA,YAAMW,QAAQC,QAAO,EAClBC,KAAK,MAAMH,SAASxC,KAAAA,CAAAA,EACpB4C,MAAM,CAACrB,QAAAA;AACN,aAAKX,cAAciC,OAAO7C,KAAAA;AAC1B,cAAMuB;MACR,CAAA;AACF,WAAKX,cAAc0B,IAAItC,OAAO8B,GAAAA;IAChC;AACA,WAAOA;EACT;EAEA,MAAcgB,aAA8B;AAC1C,UAAMC,MAAM,KAAKxC;AACjB,QAAI,OAAOwC,QAAQ,SAAU,QAAOA;AACpC,QAAI,OAAOA,QAAQ,WAAY,QAAOA,IAAAA;AACtC,WAAO,KAAKrB,cAAc,GAAG,KAAKzB,MAAM,UAAU;EACpD;;EAIA,MAAM+C,SAASC,MAAiC;AAC9C,UAAMd,WAAW,MAAM,KAAKI,SAASU,KAAKjD,KAAK;AAC/C,UAAM,KAAKF,OAAOkC,KAChB,IAAIkB,mBAAmB;MACrBf;MACAgB,aAAaC,KAAKC,UAAUJ,IAAAA;MAC5BK,mBAAmB,KAAKnD,SACpB;QAAE,CAAC,KAAKA,MAAM,GAAG;UAAEoD,UAAU;UAAUC,aAAa;QAAI;MAAE,IAC1DC;IACN,CAAA,CAAA;EAEJ;;;EAKAC,OAAO/B,MAAcgC,IAAuB;AAC1C,QAAI,CAAC,KAAK3D,MAAO,OAAM,IAAI4D,MAAM,mDAAA;AACjC,SAAKnD,SAAS6B,IAAIX,MAAMgC,EAAAA;AACxB,QAAI,CAAC,KAAK3C,SAAU,MAAKA,WAAW,KAAK6C,UAAU,KAAK7D,KAAK;EAC/D;EAEA8D,SAASC,SAAsD;AAC7D,QAAI,CAAC,KAAK9C,WAAY,MAAKA,aAAa,KAAK+C,YAAYD,OAAAA;EAC3D;EAEAE,YAAYC,UAAoD;EAEhE;;;;;;;;;EAWA,MAAcC,SACZC,UACAC,eACAC,MACe;AACf,WAAO,KAAKzD,SAAS;AACnB,UAAIsB;AACJ,UAAI;AACFA,mBAAW,MAAMiC,SAAAA;MACnB,SAAS7C,KAAK;AACZ,YAAI,CAAC,KAAKV,QAAS;AACnB,aAAKL,QAAQe,GAAAA;AACb,cAAM,KAAKgD,QAAO;AAClB;MACF;AACA,YAAMC,WAAW,MAAM,KAAKC,QAAQtC,UAAUkC,aAAAA;AAC9C,YAAMK,UAAU,IAAIC,IAAcH,QAAAA;AAClC,YAAMI,cAAc,KAAKC,YAAY1C,UAAUuC,SAASL,aAAAA;AACxD,UAAI;AACF,mBAAWS,OAAON,UAAU;AAC1B,cAAI,CAAC,KAAK3D,QAAS;AACnB,cAAI;AACF,gBAAI,KAAKkE,WAAWD,GAAAA,GAAM;AAExBJ,sBAAQ7B,OAAOiC,GAAAA;AACf,oBAAM,KAAKE,QAAQ7C,UAAU2C,IAAIG,aAAa;AAC9C;YACF;AACA,kBAAMX,KAAKnC,UAAU2C,GAAAA;UACvB,SAASvD,KAAK;AACZ,iBAAKf,QAAQe,GAAAA;UACf,UAAA;AACEmD,oBAAQ7B,OAAOiC,GAAAA;UACjB;QACF;MACF,UAAA;AACEF,oBAAAA;MACF;IACF;EACF;;;;;EAMQC,YACN1C,UACAqC,UACAH,eACY;AACZ,QAAIG,SAASU,SAAS,EAAG,QAAO,MAAA;IAAO;AACvC,UAAMC,aAAaC,KAAKC,IAAI,KAAKD,KAAKE,MAAOjB,gBAAgB,MAAQ,CAAA,CAAA;AACrE,QAAIkB;AACJ,UAAMC,QAAQC,YAAY,MAAA;AACxB,UAAIF,MAAO;AACXA,cAAQ9C,QAAQiD,IACd;WAAIlB;QAAUmB,IAAI,CAACb,QACjBA,IAAIG,gBACA,KAAKnF,OACFkC,KACC,IAAI4D,+BAA+B;QACjCzD;QACA8C,eAAeH,IAAIG;QACnBY,mBAAmBxB;MACrB,CAAA,CAAA,EAED1B,KAAK,MAAMc,MAAAA,EACXb,MAAM,CAACrB,QAAAA;AACN,YAAI,KAAKV,QAAS,MAAKL,QAAQe,GAAAA;MACjC,CAAA,IACFkC,MAAAA,CAAAA,EAENd,KAAK,MAAA;AACL4C,gBAAQ9B;MACV,CAAA;IACF,GAAG0B,UAAAA;AACHK,UAAMM,QAAK;AACX,WAAO,MAAMC,cAAcP,KAAAA;EAC7B;EAEQQ,MAASlB,KAAkB;AACjC,QAAI;AACF,aAAO1B,KAAK4C,MAAMlB,IAAImB,QAAQ,EAAA;IAChC,QAAQ;AACN,YAAM,IAAIrC,MACR,0BAA0BkB,IAAIoB,aAAa,GAAA,iEAAoE;IAEnH;EACF;EAEQrC,UAAU7D,OAA8B;AAC9C,WAAO,KAAKmE,SACV,MAAM,KAAK5B,SAASvC,KAAAA,GACpB,KAAKK,sBACL,OAAO8B,UAAU2C,QAAAA;AACf,YAAM7B,OAAO,KAAK+C,MAAkBlB,GAAAA;AACpC,YAAMqB,SAAS,MAAMC,eAAenD,MAAM,KAAKxC,SAASoB,IAAIoB,KAAKtB,IAAI,CAAA;AACrE,YAAM,KAAK7B,OAAOkC,KAChB,IAAIkB,mBAAmB;QACrBf,UAAU,MAAM,KAAKW,WAAU;QAC/BK,aAAaC,KAAKC,UAAU8C,MAAAA;QAC5B7C,mBAAmB,KAAKnD,SACpB;UAAE,CAAC,KAAKA,MAAM,GAAG;YAAEoD,UAAU;YAAUC,aAAa;UAAI;QAAE,IAC1DC;MACN,CAAA,CAAA;AAEF,YAAM,KAAK4C,IAAIlE,UAAU2C,IAAIG,aAAa;IAC5C,CAAA;EAEJ;EAEQjB,YAAYD,SAA+D;AACjF,WAAO,KAAKI,SACV,MAAM,KAAKrB,WAAU,GACrBlD,uBACA,OAAOuC,UAAU2C,QAAAA;AACf,YAAMf,QAAQ,KAAKiC,MAAkBlB,GAAAA,CAAAA;AACrC,YAAM,KAAKuB,IAAIlE,UAAU2C,IAAIG,aAAa;IAC5C,CAAA;EAEJ;;;EAIQV,UAAyB;AAC/B,WAAO,IAAI9B,QAAQ,CAACC,YAAAA;AAClB,YAAM4D,SAAS,KAAKxF,MAAMwF;AAC1B,UAAIA,OAAOC,QAAS,QAAO7D,QAAAA;AAC3B,YAAM8D,UAAU,6BAAA;AACdC,qBAAajB,KAAAA;AACb9C,gBAAAA;MACF,GAHgB;AAIhB,YAAM8C,QAAQkB,WAAW,MAAA;AACvBJ,eAAOK,oBAAoB,SAASH,OAAAA;AACpC9D,gBAAAA;MACF,GAAG,GAAA;AACH4D,aAAOM,iBAAiB,SAASJ,SAAS;QAAEK,MAAM;MAAK,CAAA;IACzD,CAAA;EACF;;EAGA,MAAcpC,QAAQtC,UAAkB2E,YAAyC;AAC/E,QAAI;AACF,YAAM/E,MAAM,MAAM,KAAKjC,OAAOkC,KAC5B,IAAI+E,sBAAsB;QACxB5E;QACA6E,qBAAqB;QACrBC,iBAAiB,KAAK7G;QACtByF,mBAAmBiB;QACnBI,uBAAuB,KAAK/G,SAAS;UAAC;YAASsD;MACjD,CAAA,GACA;QAAE0D,aAAa,KAAKrG,MAAMwF;MAAO,CAAA;AAEnC,aAAOvE,IAAIqF,YAAY,CAAA;IACzB,SAAS7F,KAAK;AACZ,UAAI,CAAC,KAAKV,QAAS,QAAO,CAAA;AAC1B,WAAKL,QAAQe,GAAAA;AACb,YAAM,KAAKgD,QAAO;AAClB,aAAO,CAAA;IACT;EACF;EAEQQ,WAAWD,KAA2E;AAC5F,WAAO,CAAC,CAAC,KAAK3E,UAAU,CAAC2E,IAAIxB,oBAAoB,KAAKnD,MAAM;EAC9D;EAEA,MAAckG,IAAIlE,UAAkB8C,eAAuC;AACzE,QAAI,CAACA,cAAe;AACpB,UAAM,KAAKnF,OAAOkC,KAAK,IAAIqF,qBAAqB;MAAElF;MAAU8C;IAAc,CAAA,CAAA;EAC5E;EAEA,MAAcD,QAAQ7C,UAAkB8C,eAAuC;AAC7E,QAAI,CAACA,cAAe;AACpB,UAAM,KAAKnF,OACRkC,KAAK,IAAI4D,+BAA+B;MAAEzD;MAAU8C;MAAeY,mBAAmB;IAAE,CAAA,CAAA,EACxFjD,MAAM,MAAA;IAAO,CAAA;EAClB;;EAGA,MAAM0E,QAAuB;AAC3B,SAAKzG,UAAU;AACf,SAAKC,MAAMA,MAAK;AAChB,UAAM2B,QAAQ8E,WAAW;MAAC,KAAKvG;MAAU,KAAKC;KAAW;AACzD,QAAI,KAAKlB,WAAY,MAAKD,OAAO0H,QAAO;EAC1C;AACF;","names":["ChangeMessageVisibilityCommand","CreateQueueCommand","DeleteMessageCommand","GetQueueUrlCommand","ReceiveMessageCommand","SQSClient","SendMessageCommand","runStepHandler","RESULT_VISIBILITY_SEC","SqsTransport","client","ownsClient","group","prefix","autoCreate","marker","waitTimeSeconds","visibilityTimeoutSec","taskQueueUrlResolver","resultsQueueUrlOption","onError","handlers","Map","urlCache","groupUrlCache","running","abort","AbortController","taskLoop","resultLoop","options","SQSClient","clientConfig","taskQueueUrl","resultsQueueUrl","err","console","error","resolveByName","name","cached","get","url","out","send","GetQueueUrlCommand","QueueName","QueueUrl","created","CreateQueueCommand","set","tasksUrl","resolver","Promise","resolve","then","catch","delete","resultsUrl","opt","dispatch","task","SendMessageCommand","MessageBody","JSON","stringify","MessageAttributes","DataType","StringValue","undefined","handle","fn","Error","pollTasks","onResult","handler","pollResults","onHeartbeat","_handler","pollLoop","queueUrl","visibilitySec","each","backoff","messages","receive","pending","Set","stopHolding","holdVisible","msg","shouldSkip","release","ReceiptHandle","size","intervalMs","Math","max","floor","round","timer","setInterval","all","map","ChangeMessageVisibilityCommand","VisibilityTimeout","unref","clearInterval","parse","Body","MessageId","result","runStepHandler","del","signal","aborted","onAbort","clearTimeout","setTimeout","removeEventListener","addEventListener","once","visibility","ReceiveMessageCommand","MaxNumberOfMessages","WaitTimeSeconds","MessageAttributeNames","abortSignal","Messages","DeleteMessageCommand","close","allSettled","destroy"]}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@dudousxd/nestjs-durable-transport-sqs",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.1",
|
|
4
4
|
"description": "Amazon SQS Transport for nestjs-durable — dispatch remote steps over existing SQS queues (ElasticMQ-compatible)",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "Davide Carvalho",
|
|
@@ -9,6 +9,16 @@
|
|
|
9
9
|
"url": "https://github.com/DavideCarvalho/nestjs-durable.git",
|
|
10
10
|
"directory": "packages/transport-sqs"
|
|
11
11
|
},
|
|
12
|
+
"homepage": "https://davidecarvalho.github.io/aviary/docs/durable",
|
|
13
|
+
"bugs": {
|
|
14
|
+
"url": "https://github.com/DavideCarvalho/nestjs-durable/issues"
|
|
15
|
+
},
|
|
16
|
+
"keywords": [
|
|
17
|
+
"nestjs",
|
|
18
|
+
"durable",
|
|
19
|
+
"transport",
|
|
20
|
+
"sqs"
|
|
21
|
+
],
|
|
12
22
|
"type": "module",
|
|
13
23
|
"main": "./dist/index.cjs",
|
|
14
24
|
"types": "./dist/index.d.ts",
|
|
@@ -20,12 +30,12 @@
|
|
|
20
30
|
"@dudousxd/nestjs-durable-core": ">=0.1.0 <1.0.0"
|
|
21
31
|
},
|
|
22
32
|
"devDependencies": {
|
|
23
|
-
"@aws-sdk/client-sqs": "3.
|
|
33
|
+
"@aws-sdk/client-sqs": "3.1130.0",
|
|
24
34
|
"testcontainers": "10.28.0",
|
|
25
35
|
"typescript": "5.9.3",
|
|
26
36
|
"zod": "3.25.76",
|
|
27
37
|
"tsup": "8.3.5",
|
|
28
|
-
"@dudousxd/nestjs-durable-core": "^0.
|
|
38
|
+
"@dudousxd/nestjs-durable-core": "^0.80.0"
|
|
29
39
|
},
|
|
30
40
|
"module": "./dist/index.js",
|
|
31
41
|
"exports": {
|