@remit/sqs-client 0.0.15 → 0.0.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/attempt-budget.d.ts +14 -0
- package/{src/attempt-budget.ts → dist/attempt-budget.js} +8 -10
- package/{src/heartbeat.ts → dist/heartbeat.d.ts} +2 -28
- package/dist/heartbeat.js +35 -0
- package/dist/index.d.ts +5 -0
- package/{src/index.ts → dist/index.js} +7 -16
- package/dist/poller.d.ts +54 -0
- package/dist/poller.js +180 -0
- package/{src/producer.ts → dist/producer.d.ts} +23 -39
- package/dist/producer.js +105 -0
- package/package.json +52 -46
- package/src/attempt-budget.test.ts +0 -58
- package/src/heartbeat.test.ts +0 -102
- package/src/index.test.ts +0 -39
- package/src/poller.test.ts +0 -301
- package/src/poller.ts +0 -283
- package/src/producer.test.ts +0 -150
- package/tsconfig.json +0 -8
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Reads a handler's redelivery budget from the environment. CDK derives every
|
|
3
|
+
* `*_MAX_ATTEMPTS` var from the corresponding queue's own `MAX_RECEIVE_COUNT`
|
|
4
|
+
* (`infra/stacks/dev/stacks/remit-queue-stack.ts`, `deploy/vps/queues.json`)
|
|
5
|
+
* so a worker's idea of "last attempt" cannot drift from the redrive policy
|
|
6
|
+
* that actually dead-letters the record (issue #1270). SQS's
|
|
7
|
+
* `ApproximateReceiveCount` is the source of truth for how many times a record
|
|
8
|
+
* has been delivered; once it reaches this budget the current invocation is
|
|
9
|
+
* the last attempt, and the handler resolves exhaustion itself instead of
|
|
10
|
+
* letting the record dead-letter undiagnosed. The fallback keeps an
|
|
11
|
+
* environment that never injects the var — local dev, unit tests — behaving
|
|
12
|
+
* like production.
|
|
13
|
+
*/
|
|
14
|
+
export declare const attemptBudget: (envName: string, fallback: number, processEnv?: NodeJS.ProcessEnv) => number;
|
|
@@ -11,14 +11,12 @@
|
|
|
11
11
|
* environment that never injects the var — local dev, unit tests — behaving
|
|
12
12
|
* like production.
|
|
13
13
|
*/
|
|
14
|
-
export const attemptBudget = (
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
if (!Number.isFinite(parsed) || parsed <= 0) return fallback;
|
|
23
|
-
return parsed;
|
|
14
|
+
export const attemptBudget = (envName, fallback, processEnv = process.env) => {
|
|
15
|
+
const raw = processEnv[envName];
|
|
16
|
+
if (!raw)
|
|
17
|
+
return fallback;
|
|
18
|
+
const parsed = Number.parseInt(raw, 10);
|
|
19
|
+
if (!Number.isFinite(parsed) || parsed <= 0)
|
|
20
|
+
return fallback;
|
|
21
|
+
return parsed;
|
|
24
22
|
};
|
|
@@ -1,6 +1,3 @@
|
|
|
1
|
-
import { readdir, rm, writeFile } from "node:fs/promises";
|
|
2
|
-
import { basename, dirname, join } from "node:path";
|
|
3
|
-
|
|
4
1
|
/**
|
|
5
2
|
* Poll-loop liveness as a file (docs/design/standalone-observability.md, D1).
|
|
6
3
|
*
|
|
@@ -25,9 +22,6 @@ import { basename, dirname, join } from "node:path";
|
|
|
25
22
|
* each cycle is healthy here.
|
|
26
23
|
*/
|
|
27
24
|
export type Heartbeat = () => Promise<void>;
|
|
28
|
-
|
|
29
|
-
const noHeartbeat: Heartbeat = () => Promise.resolve();
|
|
30
|
-
|
|
31
25
|
/**
|
|
32
26
|
* `WORKER_HEARTBEAT_PREFIX` is set per worker service by the self-host compose
|
|
33
27
|
* stack, which owns both the path and the volume it lands on; each loop appends
|
|
@@ -38,31 +32,11 @@ const noHeartbeat: Heartbeat = () => Promise.resolve();
|
|
|
38
32
|
* stat-able: an empty rewrite is not guaranteed to advance mtime when the file
|
|
39
33
|
* is already zero-length, and mtime is the whole signal.
|
|
40
34
|
*/
|
|
41
|
-
export const createHeartbeat
|
|
42
|
-
name: string,
|
|
43
|
-
prefix: string | undefined = process.env.WORKER_HEARTBEAT_PREFIX,
|
|
44
|
-
): Heartbeat => {
|
|
45
|
-
if (!prefix) return noHeartbeat;
|
|
46
|
-
const filePath = `${prefix}.${name}`;
|
|
47
|
-
return () => writeFile(filePath, `${new Date().toISOString()}\n`);
|
|
48
|
-
};
|
|
49
|
-
|
|
35
|
+
export declare const createHeartbeat: (name: string, prefix?: string | undefined) => Heartbeat;
|
|
50
36
|
/**
|
|
51
37
|
* Drops the files this service left on the volume, before its loops start
|
|
52
38
|
* writing again. The healthcheck fails on the oldest file it finds, so a queue
|
|
53
39
|
* a later version stops polling would otherwise leave a file nobody rewrites,
|
|
54
40
|
* and a worker that can never report healthy again.
|
|
55
41
|
*/
|
|
56
|
-
export const clearHeartbeats
|
|
57
|
-
prefix: string | undefined = process.env.WORKER_HEARTBEAT_PREFIX,
|
|
58
|
-
): Promise<void> => {
|
|
59
|
-
if (!prefix) return;
|
|
60
|
-
const directory = dirname(prefix);
|
|
61
|
-
const owned = `${basename(prefix)}.`;
|
|
62
|
-
const names = await readdir(directory);
|
|
63
|
-
await Promise.all(
|
|
64
|
-
names
|
|
65
|
-
.filter((name) => name.startsWith(owned))
|
|
66
|
-
.map((name) => rm(join(directory, name))),
|
|
67
|
-
);
|
|
68
|
-
};
|
|
42
|
+
export declare const clearHeartbeats: (prefix?: string | undefined) => Promise<void>;
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { readdir, rm, writeFile } from "node:fs/promises";
|
|
2
|
+
import { basename, dirname, join } from "node:path";
|
|
3
|
+
const noHeartbeat = () => Promise.resolve();
|
|
4
|
+
/**
|
|
5
|
+
* `WORKER_HEARTBEAT_PREFIX` is set per worker service by the self-host compose
|
|
6
|
+
* stack, which owns both the path and the volume it lands on; each loop appends
|
|
7
|
+
* its own queue name. Unset — the Lambda deployment, where the poller does not
|
|
8
|
+
* run at all, and any local process — writes nothing.
|
|
9
|
+
*
|
|
10
|
+
* The written value is the timestamp itself, so the file is readable as well as
|
|
11
|
+
* stat-able: an empty rewrite is not guaranteed to advance mtime when the file
|
|
12
|
+
* is already zero-length, and mtime is the whole signal.
|
|
13
|
+
*/
|
|
14
|
+
export const createHeartbeat = (name, prefix = process.env.WORKER_HEARTBEAT_PREFIX) => {
|
|
15
|
+
if (!prefix)
|
|
16
|
+
return noHeartbeat;
|
|
17
|
+
const filePath = `${prefix}.${name}`;
|
|
18
|
+
return () => writeFile(filePath, `${new Date().toISOString()}\n`);
|
|
19
|
+
};
|
|
20
|
+
/**
|
|
21
|
+
* Drops the files this service left on the volume, before its loops start
|
|
22
|
+
* writing again. The healthcheck fails on the oldest file it finds, so a queue
|
|
23
|
+
* a later version stops polling would otherwise leave a file nobody rewrites,
|
|
24
|
+
* and a worker that can never report healthy again.
|
|
25
|
+
*/
|
|
26
|
+
export const clearHeartbeats = async (prefix = process.env.WORKER_HEARTBEAT_PREFIX) => {
|
|
27
|
+
if (!prefix)
|
|
28
|
+
return;
|
|
29
|
+
const directory = dirname(prefix);
|
|
30
|
+
const owned = `${basename(prefix)}.`;
|
|
31
|
+
const names = await readdir(directory);
|
|
32
|
+
await Promise.all(names
|
|
33
|
+
.filter((name) => name.startsWith(owned))
|
|
34
|
+
.map((name) => rm(join(directory, name))));
|
|
35
|
+
};
|
package/dist/index.d.ts
ADDED
|
@@ -1,23 +1,14 @@
|
|
|
1
|
-
export interface SqsStaticCredentials {
|
|
2
|
-
accessKeyId: string;
|
|
3
|
-
secretAccessKey: string;
|
|
4
|
-
}
|
|
5
|
-
|
|
6
1
|
// On AWS, SQS shares the process-wide credential chain (AWS_ACCESS_KEY_ID / IAM
|
|
7
2
|
// role). On Scaleway, Messaging & Queuing has its own credential pair, distinct
|
|
8
3
|
// from the Object Storage (S3) keys, so it can't ride the default chain — the app
|
|
9
4
|
// is handed SQS_ACCESS_KEY_ID / SQS_SECRET_ACCESS_KEY and must feed them to the
|
|
10
5
|
// SQS client explicitly. When they are absent (AWS, or local ElasticMQ), returning
|
|
11
6
|
// undefined lets the SDK fall back to its default credential provider chain.
|
|
12
|
-
export function resolveSqsCredentials(
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
return { accessKeyId, secretAccessKey };
|
|
20
|
-
}
|
|
21
|
-
|
|
22
|
-
return undefined;
|
|
7
|
+
export function resolveSqsCredentials(env = process.env) {
|
|
8
|
+
const accessKeyId = env.SQS_ACCESS_KEY_ID;
|
|
9
|
+
const secretAccessKey = env.SQS_SECRET_ACCESS_KEY;
|
|
10
|
+
if (accessKeyId && secretAccessKey) {
|
|
11
|
+
return { accessKeyId, secretAccessKey };
|
|
12
|
+
}
|
|
13
|
+
return undefined;
|
|
23
14
|
}
|
package/dist/poller.d.ts
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import type { SQSHandler } from "aws-lambda";
|
|
2
|
+
import { type Heartbeat } from "./heartbeat.js";
|
|
3
|
+
/**
|
|
4
|
+
* Production queue poller — the non-e2e counterpart of the per-worker
|
|
5
|
+
* `e2e-processor-shim.ts` scripts. It receives from an SQS-compatible
|
|
6
|
+
* endpoint (real SQS or ElasticMQ), wraps each batch in an `SQSEvent`, and
|
|
7
|
+
* invokes the exact production Lambda `handler`, honouring
|
|
8
|
+
* `batchItemFailures` the way the SQS service does: delete what succeeded,
|
|
9
|
+
* leave failures for redelivery.
|
|
10
|
+
*
|
|
11
|
+
* Unlike the e2e shim (one child process per queue, via `node:cluster`),
|
|
12
|
+
* this runs every target queue as a concurrent poll loop inside a single
|
|
13
|
+
* process — the right shape for a container whose only job is "poll these
|
|
14
|
+
* queues, invoke this handler, exit on SIGTERM."
|
|
15
|
+
*/
|
|
16
|
+
export interface QueuePollerLog {
|
|
17
|
+
info: (fields: Record<string, unknown>, message: string) => void;
|
|
18
|
+
error: (fields: Record<string, unknown>, message: string) => void;
|
|
19
|
+
}
|
|
20
|
+
export interface QueuePollerTarget {
|
|
21
|
+
readonly queueUrl: string;
|
|
22
|
+
readonly handler: SQSHandler;
|
|
23
|
+
/** Reported as the Lambda `Context.functionName` the handler observes. */
|
|
24
|
+
readonly functionName: string;
|
|
25
|
+
readonly maxMessages?: number;
|
|
26
|
+
readonly visibilityTimeoutSeconds?: number;
|
|
27
|
+
}
|
|
28
|
+
export interface RunQueuePollerOptions {
|
|
29
|
+
readonly targets: readonly QueuePollerTarget[];
|
|
30
|
+
readonly log: QueuePollerLog;
|
|
31
|
+
readonly signals?: readonly NodeJS.Signals[];
|
|
32
|
+
/** One per target. Defaults to a file under `WORKER_HEARTBEAT_PREFIX`. */
|
|
33
|
+
readonly createHeartbeat?: (name: string) => Heartbeat;
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* What a handler has before the queue redelivers the record underneath it.
|
|
37
|
+
* Exported because a handler that budgets its own wait has to agree with it;
|
|
38
|
+
* `deploy/vps/queues.json` sets the same number on the queue itself, and this
|
|
39
|
+
* is the one that applies wherever a target passes no override.
|
|
40
|
+
*/
|
|
41
|
+
export declare const DEFAULT_VISIBILITY_TIMEOUT_SECONDS = 300;
|
|
42
|
+
/**
|
|
43
|
+
* Runs every target's poll loop until one of `signals` is received (default:
|
|
44
|
+
* SIGINT, SIGTERM). The signal aborts each loop's in-flight
|
|
45
|
+
* long-poll receive so an idle worker returns at once instead of waiting out
|
|
46
|
+
* the current 20s poll; a loop mid-handler finishes that handler first. Rejects
|
|
47
|
+
* (crashes the process) if any loop throws — a stuck poller should exit loudly,
|
|
48
|
+
* not degrade silently.
|
|
49
|
+
*
|
|
50
|
+
* Every loop writes its own heartbeat file (see heartbeat.ts) at the top of its
|
|
51
|
+
* receive attempt, which is how a loop that hangs instead of throwing — the
|
|
52
|
+
* case this cannot make loud by itself — becomes visible.
|
|
53
|
+
*/
|
|
54
|
+
export declare const runQueuePoller: (options: RunQueuePollerOptions) => Promise<void>;
|
package/dist/poller.js
ADDED
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { inspect } from "node:util";
|
|
3
|
+
import { DeleteMessageCommand, ReceiveMessageCommand, } from "@aws-sdk/client-sqs";
|
|
4
|
+
import { clearHeartbeats, createHeartbeat, } from "./heartbeat.js";
|
|
5
|
+
import { createSqsClient } from "./producer.js";
|
|
6
|
+
// One message per receive, not the SQS maximum of 10. Handlers process a batch
|
|
7
|
+
// sequentially, so a batch of ten is ten handler runs between two heartbeats:
|
|
8
|
+
// ten stalled SMTP sends at nodemailer's 300 s socket timeout is 50 minutes of
|
|
9
|
+
// legitimate silence, and a threshold that tolerates it detects nothing. At one
|
|
10
|
+
// message the quiet stretch a healthy loop can produce is one handler run, and
|
|
11
|
+
// the extra receive per message is a millisecond against the local queue
|
|
12
|
+
// sidecar — the same total work either way.
|
|
13
|
+
const DEFAULT_MAX_MESSAGES = 1;
|
|
14
|
+
/**
|
|
15
|
+
* What a handler has before the queue redelivers the record underneath it.
|
|
16
|
+
* Exported because a handler that budgets its own wait has to agree with it;
|
|
17
|
+
* `deploy/vps/queues.json` sets the same number on the queue itself, and this
|
|
18
|
+
* is the one that applies wherever a target passes no override.
|
|
19
|
+
*/
|
|
20
|
+
export const DEFAULT_VISIBILITY_TIMEOUT_SECONDS = 300;
|
|
21
|
+
const pollTarget = async (target, log, isShuttingDown, shutdownSignal, buildHeartbeat) => {
|
|
22
|
+
const queueName = new URL(target.queueUrl).pathname.split("/").pop();
|
|
23
|
+
const heartbeat = buildHeartbeat(queueName ?? target.functionName);
|
|
24
|
+
const sqs = createSqsClient({ queueUrl: target.queueUrl });
|
|
25
|
+
const maxMessages = target.maxMessages ?? DEFAULT_MAX_MESSAGES;
|
|
26
|
+
const visibilityTimeout = target.visibilityTimeoutSeconds ?? DEFAULT_VISIBILITY_TIMEOUT_SECONDS;
|
|
27
|
+
log.info({ queue: queueName, maxMessages }, "poller: started");
|
|
28
|
+
// Always long-poll (SQS max WaitTimeSeconds). An empty long poll is free
|
|
29
|
+
// and returns as soon as a message arrives, so this is never slower than
|
|
30
|
+
// short-polling under load — it just stops burning a ReceiveMessage call
|
|
31
|
+
// (and, against real SQS, its cost) every loop iteration while traffic is
|
|
32
|
+
// continuous.
|
|
33
|
+
const LONG_POLL_WAIT_SECONDS = 20;
|
|
34
|
+
while (!isShuttingDown()) {
|
|
35
|
+
// Top of the receive attempt, before the long poll: this loop's own file,
|
|
36
|
+
// so a loop that stops turning — wedged in a socket read, or in a handler
|
|
37
|
+
// that never returns — goes stale on its own however busy its siblings are.
|
|
38
|
+
//
|
|
39
|
+
// A write that fails must not take the worker down with it. A full disk is
|
|
40
|
+
// the likeliest cause and the moment the loop should stay up, failing
|
|
41
|
+
// records into the dead-letter queues with logs rather than crash-looping
|
|
42
|
+
// on the monitoring. The missed beat is itself the signal: the healthcheck
|
|
43
|
+
// goes stale, which is what this file exists to report.
|
|
44
|
+
await heartbeat().catch((error) => {
|
|
45
|
+
log.error({ queue: queueName, error: inspect(error) }, "poller: heartbeat write failed");
|
|
46
|
+
});
|
|
47
|
+
let response;
|
|
48
|
+
try {
|
|
49
|
+
// Abort the receive on shutdown. Without this, a container stopped
|
|
50
|
+
// while idle waits out the current 20s long poll before the loop
|
|
51
|
+
// checks the flag again, and a self-update then burns the whole
|
|
52
|
+
// stop grace on every idle worker — the bulk of the visible restart
|
|
53
|
+
// downtime. The abort ends the in-flight receive at once; an
|
|
54
|
+
// in-flight handler below is never interrupted, so a message being
|
|
55
|
+
// processed still finishes cleanly. The heartbeat above already wrote
|
|
56
|
+
// this iteration, so the abort exits without a further beat.
|
|
57
|
+
response = await sqs.send(new ReceiveMessageCommand({
|
|
58
|
+
QueueUrl: target.queueUrl,
|
|
59
|
+
MaxNumberOfMessages: maxMessages,
|
|
60
|
+
WaitTimeSeconds: LONG_POLL_WAIT_SECONDS,
|
|
61
|
+
VisibilityTimeout: visibilityTimeout,
|
|
62
|
+
MessageSystemAttributeNames: ["ApproximateReceiveCount"],
|
|
63
|
+
}), { abortSignal: shutdownSignal });
|
|
64
|
+
}
|
|
65
|
+
catch (error) {
|
|
66
|
+
if (shutdownSignal.aborted)
|
|
67
|
+
break;
|
|
68
|
+
throw error;
|
|
69
|
+
}
|
|
70
|
+
if (!response.Messages || response.Messages.length === 0) {
|
|
71
|
+
continue;
|
|
72
|
+
}
|
|
73
|
+
const messages = response.Messages.flatMap((m) => m.Body && m.ReceiptHandle && m.MessageId
|
|
74
|
+
? [
|
|
75
|
+
{
|
|
76
|
+
messageId: m.MessageId,
|
|
77
|
+
receiptHandle: m.ReceiptHandle,
|
|
78
|
+
body: m.Body,
|
|
79
|
+
receiveCount: m.Attributes?.ApproximateReceiveCount ?? "1",
|
|
80
|
+
},
|
|
81
|
+
]
|
|
82
|
+
: []);
|
|
83
|
+
if (messages.length === 0)
|
|
84
|
+
continue;
|
|
85
|
+
// One batch is one handler invocation, so the id is minted per batch: it is
|
|
86
|
+
// the `requestId` every line of that invocation is grouped by (see
|
|
87
|
+
// deploy/vps/README.md, "Logs"). A context built once per target would put
|
|
88
|
+
// the whole queue's lifetime under one id, which correlates nothing.
|
|
89
|
+
//
|
|
90
|
+
// Minted before the poller's own bracketing lines rather than next to the
|
|
91
|
+
// handler call: grouping on the id has to reach the line saying whether the
|
|
92
|
+
// message was deleted or left for redelivery, which is what someone looks
|
|
93
|
+
// for after a failure.
|
|
94
|
+
const lambdaContext = {
|
|
95
|
+
functionName: target.functionName,
|
|
96
|
+
awsRequestId: randomUUID(),
|
|
97
|
+
};
|
|
98
|
+
log.info({
|
|
99
|
+
queue: queueName,
|
|
100
|
+
count: messages.length,
|
|
101
|
+
requestId: lambdaContext.awsRequestId,
|
|
102
|
+
}, "poller: invoking handler");
|
|
103
|
+
const event = {
|
|
104
|
+
Records: messages.map((m) => ({
|
|
105
|
+
messageId: m.messageId,
|
|
106
|
+
receiptHandle: m.receiptHandle,
|
|
107
|
+
body: m.body,
|
|
108
|
+
attributes: {
|
|
109
|
+
ApproximateReceiveCount: m.receiveCount,
|
|
110
|
+
SentTimestamp: "0",
|
|
111
|
+
SenderId: "poller",
|
|
112
|
+
ApproximateFirstReceiveTimestamp: "0",
|
|
113
|
+
},
|
|
114
|
+
messageAttributes: {},
|
|
115
|
+
md5OfBody: "",
|
|
116
|
+
eventSource: "aws:sqs",
|
|
117
|
+
eventSourceARN: target.queueUrl,
|
|
118
|
+
awsRegion: "local",
|
|
119
|
+
})),
|
|
120
|
+
};
|
|
121
|
+
const result = (await target.handler(event, lambdaContext, () => { }));
|
|
122
|
+
const failedIds = new Set((result?.batchItemFailures ?? []).map((f) => f.itemIdentifier));
|
|
123
|
+
const succeeded = messages.filter((m) => !failedIds.has(m.messageId));
|
|
124
|
+
// Batch is capped at maxMessages (one by default, 10 is the SQS ceiling),
|
|
125
|
+
// so a handful of concurrent DeleteMessage calls is safe and avoids paying
|
|
126
|
+
// N sequential round trips between every receive/handler cycle.
|
|
127
|
+
await Promise.all(succeeded.map((message) => sqs.send(new DeleteMessageCommand({
|
|
128
|
+
QueueUrl: target.queueUrl,
|
|
129
|
+
ReceiptHandle: message.receiptHandle,
|
|
130
|
+
}))));
|
|
131
|
+
log.info({
|
|
132
|
+
queue: queueName,
|
|
133
|
+
deleted: succeeded.length,
|
|
134
|
+
leftForRedelivery: failedIds.size,
|
|
135
|
+
requestId: lambdaContext.awsRequestId,
|
|
136
|
+
}, "poller: batch processed");
|
|
137
|
+
}
|
|
138
|
+
log.info({ queue: queueName }, "poller: stopped");
|
|
139
|
+
};
|
|
140
|
+
/**
|
|
141
|
+
* Runs every target's poll loop until one of `signals` is received (default:
|
|
142
|
+
* SIGINT, SIGTERM). The signal aborts each loop's in-flight
|
|
143
|
+
* long-poll receive so an idle worker returns at once instead of waiting out
|
|
144
|
+
* the current 20s poll; a loop mid-handler finishes that handler first. Rejects
|
|
145
|
+
* (crashes the process) if any loop throws — a stuck poller should exit loudly,
|
|
146
|
+
* not degrade silently.
|
|
147
|
+
*
|
|
148
|
+
* Every loop writes its own heartbeat file (see heartbeat.ts) at the top of its
|
|
149
|
+
* receive attempt, which is how a loop that hangs instead of throwing — the
|
|
150
|
+
* case this cannot make loud by itself — becomes visible.
|
|
151
|
+
*/
|
|
152
|
+
export const runQueuePoller = async (options) => {
|
|
153
|
+
const { targets, log } = options;
|
|
154
|
+
const signals = options.signals ?? ["SIGINT", "SIGTERM"];
|
|
155
|
+
const buildHeartbeat = options.createHeartbeat ?? createHeartbeat;
|
|
156
|
+
if (targets.length === 0) {
|
|
157
|
+
throw new Error("runQueuePoller: no targets configured");
|
|
158
|
+
}
|
|
159
|
+
// Files from the previous run of this service, before its loops write again:
|
|
160
|
+
// the healthcheck reads the oldest file it finds, so one left behind by a
|
|
161
|
+
// queue this version no longer polls would fail the check forever.
|
|
162
|
+
await clearHeartbeats().catch((error) => {
|
|
163
|
+
log.error({ error: inspect(error) }, "poller: heartbeat cleanup failed");
|
|
164
|
+
});
|
|
165
|
+
let shuttingDown = false;
|
|
166
|
+
const isShuttingDown = () => shuttingDown;
|
|
167
|
+
// Aborted alongside the flag so a loop currently blocked in its long-poll
|
|
168
|
+
// receive returns at once, rather than after up to WaitTimeSeconds.
|
|
169
|
+
const shutdown = new AbortController();
|
|
170
|
+
const onSignal = (signal) => {
|
|
171
|
+
log.info({ signal }, "poller: shutdown signal received");
|
|
172
|
+
shuttingDown = true;
|
|
173
|
+
shutdown.abort();
|
|
174
|
+
};
|
|
175
|
+
for (const signal of signals) {
|
|
176
|
+
process.on(signal, onSignal);
|
|
177
|
+
}
|
|
178
|
+
log.info({ queues: targets.map((t) => t.queueUrl) }, "poller: starting");
|
|
179
|
+
await Promise.all(targets.map((target) => pollTarget(target, log, isShuttingDown, shutdown.signal, buildHeartbeat)));
|
|
180
|
+
};
|
|
@@ -1,21 +1,18 @@
|
|
|
1
1
|
import { SQSClient } from "@aws-sdk/client-sqs";
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
2
|
+
import type { QueueMessage, QueueProducer } from "@remit/data-ports";
|
|
3
|
+
import { type SqsStaticCredentials } from "./index.js";
|
|
4
|
+
export interface CreateSqsClientOptions {
|
|
5
|
+
readonly queueUrl: string;
|
|
6
|
+
readonly endpoint?: string;
|
|
7
|
+
readonly env?: NodeJS.ProcessEnv;
|
|
8
|
+
/**
|
|
9
|
+
* Credentials to use when the queue URL resolves to a local endpoint. A
|
|
10
|
+
* local queue server ignores them, but the SDK still needs something to
|
|
11
|
+
* sign with and a self-hosted stack has no credential chain to fall back on.
|
|
12
|
+
*/
|
|
13
|
+
readonly localCredentials?: SqsStaticCredentials;
|
|
14
|
+
readonly requestTimeoutMs?: number;
|
|
15
15
|
}
|
|
16
|
-
|
|
17
|
-
const LOOPBACK_HOSTNAMES = new Set(["localhost", "127.0.0.1", "[::1]"]);
|
|
18
|
-
|
|
19
16
|
/**
|
|
20
17
|
* Whether a queue URL addresses a local, SQS-compatible queue server
|
|
21
18
|
* (ElasticMQ, the queue sidecar) rather than real SQS.
|
|
@@ -36,17 +33,7 @@ const LOOPBACK_HOSTNAMES = new Set(["localhost", "127.0.0.1", "[::1]"]);
|
|
|
36
33
|
* here would take a worker down at import instead of surfacing the bad
|
|
37
34
|
* configuration at the send that actually uses it.
|
|
38
35
|
*/
|
|
39
|
-
export const isLocalEndpoint
|
|
40
|
-
let url: URL;
|
|
41
|
-
try {
|
|
42
|
-
url = new URL(queueUrl);
|
|
43
|
-
} catch {
|
|
44
|
-
return false;
|
|
45
|
-
}
|
|
46
|
-
if (url.protocol === "http:") return true;
|
|
47
|
-
return url.protocol === "https:" && LOOPBACK_HOSTNAMES.has(url.hostname);
|
|
48
|
-
};
|
|
49
|
-
|
|
36
|
+
export declare const isLocalEndpoint: (queueUrl: string) => boolean;
|
|
50
37
|
/**
|
|
51
38
|
* Producer-side counterpart of `runQueuePoller`: the one place an
|
|
52
39
|
* SQS-compatible client is constructed for sending. A queue URL that points at
|
|
@@ -55,15 +42,12 @@ export const isLocalEndpoint = (queueUrl: string): boolean => {
|
|
|
55
42
|
* endpoint and ride the default credential chain unless
|
|
56
43
|
* `SQS_ACCESS_KEY_ID`/`SQS_SECRET_ACCESS_KEY` are present.
|
|
57
44
|
*/
|
|
58
|
-
export const
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
(local ? localCredentials : undefined) ?? resolveSqsCredentials(env),
|
|
68
|
-
});
|
|
69
|
-
};
|
|
45
|
+
export declare const createSqsClient: (options: CreateSqsClientOptions) => SQSClient;
|
|
46
|
+
/**
|
|
47
|
+
* The SQS adapter for the `QueueProducer` port. A FIFO queue (`.fifo`) gets
|
|
48
|
+
* the message group, and the deduplication id when the message carries one
|
|
49
|
+
* (otherwise the queue deduplicates by content); a standard queue rejects
|
|
50
|
+
* both, so they are left off. A local queue server (ElasticMQ) does not honour
|
|
51
|
+
* a per-message delay on its FIFO queues, so the delay is dropped there.
|
|
52
|
+
*/
|
|
53
|
+
export declare const createQueueProducer: <T, M extends QueueMessage<T> = QueueMessage<T>>(options: CreateSqsClientOptions) => QueueProducer<T, M>;
|
package/dist/producer.js
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import { SendMessageBatchCommand, SendMessageCommand, SQSClient, } from "@aws-sdk/client-sqs";
|
|
2
|
+
import { AwsQueryProtocol } from "@aws-sdk/core/protocols";
|
|
3
|
+
import { resolveSqsCredentials } from "./index.js";
|
|
4
|
+
const DEFAULT_REQUEST_TIMEOUT_MS = 45_000;
|
|
5
|
+
const LOOPBACK_HOSTNAMES = new Set(["localhost", "127.0.0.1", "[::1]"]);
|
|
6
|
+
/**
|
|
7
|
+
* Whether a queue URL addresses a local, SQS-compatible queue server
|
|
8
|
+
* (ElasticMQ, the queue sidecar) rather than real SQS.
|
|
9
|
+
*
|
|
10
|
+
* Any plain `http://` URL is local: real SQS and Scaleway M&Q are always
|
|
11
|
+
* HTTPS, and a self-hosted stack addresses its queue server by container name
|
|
12
|
+
* (`http://queue:9324/...`), not by `localhost`. Over HTTPS only a loopback
|
|
13
|
+
* host is local, which covers a local server behind TLS.
|
|
14
|
+
*
|
|
15
|
+
* The URL is parsed rather than prefix-matched. Prefix-matching the string
|
|
16
|
+
* gets the interesting cases wrong in both directions: it reads
|
|
17
|
+
* `https://localhost.example.com` and `https://localhost@example.com` as
|
|
18
|
+
* local (the host is neither), and misses `https://user:pass@localhost` and
|
|
19
|
+
* an uppercase `HTTP://` scheme (both of which are).
|
|
20
|
+
*
|
|
21
|
+
* A queue URL that does not parse is treated as remote. It cannot be sent to
|
|
22
|
+
* either way, and several callers build their client at module load — throwing
|
|
23
|
+
* here would take a worker down at import instead of surfacing the bad
|
|
24
|
+
* configuration at the send that actually uses it.
|
|
25
|
+
*/
|
|
26
|
+
export const isLocalEndpoint = (queueUrl) => {
|
|
27
|
+
let url;
|
|
28
|
+
try {
|
|
29
|
+
url = new URL(queueUrl);
|
|
30
|
+
}
|
|
31
|
+
catch {
|
|
32
|
+
return false;
|
|
33
|
+
}
|
|
34
|
+
if (url.protocol === "http:")
|
|
35
|
+
return true;
|
|
36
|
+
return url.protocol === "https:" && LOOPBACK_HOSTNAMES.has(url.hostname);
|
|
37
|
+
};
|
|
38
|
+
/**
|
|
39
|
+
* Producer-side counterpart of `runQueuePoller`: the one place an
|
|
40
|
+
* SQS-compatible client is constructed for sending. A queue URL that points at
|
|
41
|
+
* a local endpoint (ElasticMQ) gets its origin as the SDK endpoint and the
|
|
42
|
+
* query protocol ElasticMQ speaks; real SQS and Scaleway M&Q resolve their own
|
|
43
|
+
* endpoint and ride the default credential chain unless
|
|
44
|
+
* `SQS_ACCESS_KEY_ID`/`SQS_SECRET_ACCESS_KEY` are present.
|
|
45
|
+
*/
|
|
46
|
+
export const createSqsClient = (options) => {
|
|
47
|
+
const { queueUrl, endpoint, env, localCredentials } = options;
|
|
48
|
+
const requestTimeout = options.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS;
|
|
49
|
+
const local = isLocalEndpoint(queueUrl);
|
|
50
|
+
return new SQSClient({
|
|
51
|
+
requestHandler: { requestTimeout, throwOnRequestTimeout: true },
|
|
52
|
+
endpoint: endpoint ?? (local ? new URL(queueUrl).origin : undefined),
|
|
53
|
+
...(local && { protocol: AwsQueryProtocol }),
|
|
54
|
+
credentials: (local ? localCredentials : undefined) ?? resolveSqsCredentials(env),
|
|
55
|
+
});
|
|
56
|
+
};
|
|
57
|
+
const isFifoQueue = (queueUrl) => queueUrl.endsWith(".fifo");
|
|
58
|
+
const isGrouped = (message) => "groupId" in message && typeof message.groupId === "string";
|
|
59
|
+
const isDeduplicated = (message) => isGrouped(message) &&
|
|
60
|
+
"deduplicationId" in message &&
|
|
61
|
+
typeof message.deduplicationId === "string";
|
|
62
|
+
/**
|
|
63
|
+
* The SQS adapter for the `QueueProducer` port. A FIFO queue (`.fifo`) gets
|
|
64
|
+
* the message group, and the deduplication id when the message carries one
|
|
65
|
+
* (otherwise the queue deduplicates by content); a standard queue rejects
|
|
66
|
+
* both, so they are left off. A local queue server (ElasticMQ) does not honour
|
|
67
|
+
* a per-message delay on its FIFO queues, so the delay is dropped there.
|
|
68
|
+
*/
|
|
69
|
+
export const createQueueProducer = (options) => {
|
|
70
|
+
const { queueUrl } = options;
|
|
71
|
+
const client = createSqsClient(options);
|
|
72
|
+
const fifo = isFifoQueue(queueUrl);
|
|
73
|
+
const local = isLocalEndpoint(queueUrl);
|
|
74
|
+
const attributes = (message) => ({
|
|
75
|
+
...(fifo && isGrouped(message) && { MessageGroupId: message.groupId }),
|
|
76
|
+
...(fifo &&
|
|
77
|
+
isDeduplicated(message) && {
|
|
78
|
+
MessageDeduplicationId: message.deduplicationId,
|
|
79
|
+
}),
|
|
80
|
+
...(message.delaySeconds &&
|
|
81
|
+
!local && { DelaySeconds: message.delaySeconds }),
|
|
82
|
+
});
|
|
83
|
+
return {
|
|
84
|
+
send: async (message) => {
|
|
85
|
+
await client.send(new SendMessageCommand({
|
|
86
|
+
QueueUrl: queueUrl,
|
|
87
|
+
MessageBody: JSON.stringify(message.body),
|
|
88
|
+
...attributes(message),
|
|
89
|
+
}));
|
|
90
|
+
},
|
|
91
|
+
sendBatch: async (messages) => {
|
|
92
|
+
const result = await client.send(new SendMessageBatchCommand({
|
|
93
|
+
QueueUrl: queueUrl,
|
|
94
|
+
Entries: messages.map((message, index) => ({
|
|
95
|
+
Id: `${index}`,
|
|
96
|
+
MessageBody: JSON.stringify(message.body),
|
|
97
|
+
...attributes(message),
|
|
98
|
+
})),
|
|
99
|
+
}));
|
|
100
|
+
return {
|
|
101
|
+
refused: new Set((result.Failed ?? []).map((entry) => Number(entry.Id))),
|
|
102
|
+
};
|
|
103
|
+
},
|
|
104
|
+
};
|
|
105
|
+
};
|