@remit/sqs-client 0.0.16 → 0.0.17

package/src/poller.ts

@@ -1,284 +0,0 @@
-import { randomUUID } from "node:crypto";
-import { inspect } from "node:util";
-import {
-	DeleteMessageCommand,
-	ReceiveMessageCommand,
-	type ReceiveMessageCommandOutput,
-} from "@aws-sdk/client-sqs";
-import type {
-	Context,
-	SQSBatchResponse,
-	SQSEvent,
-	SQSHandler,
-} from "aws-lambda";
-import {
-	clearHeartbeats,
-	createHeartbeat,
-	type Heartbeat,
-} from "./heartbeat.js";
-import { createQueueProducer } from "./producer.js";
-
-/**
- * Production queue poller — the non-e2e counterpart of the per-worker
- * `e2e-processor-shim.ts` scripts. It receives from an SQS-compatible
- * endpoint (real SQS or ElasticMQ), wraps each batch in an `SQSEvent`, and
- * invokes the exact production Lambda `handler`, honouring
- * `batchItemFailures` the way the SQS service does: delete what succeeded,
- * leave failures for redelivery.
- *
- * Unlike the e2e shim (one child process per queue, via `node:cluster`),
- * this runs every target queue as a concurrent poll loop inside a single
- * process — the right shape for a container whose only job is "poll these
- * queues, invoke this handler, exit on SIGTERM."
- */
-export interface QueuePollerLog {
-	info: (fields: Record<string, unknown>, message: string) => void;
-	error: (fields: Record<string, unknown>, message: string) => void;
-}
-
-export interface QueuePollerTarget {
-	readonly queueUrl: string;
-	readonly handler: SQSHandler;
-	/** Reported as the Lambda `Context.functionName` the handler observes. */
-	readonly functionName: string;
-	readonly maxMessages?: number;
-	readonly visibilityTimeoutSeconds?: number;
-}
-
-export interface RunQueuePollerOptions {
-	readonly targets: readonly QueuePollerTarget[];
-	readonly log: QueuePollerLog;
-	readonly signals?: readonly NodeJS.Signals[];
-	/** One per target. Defaults to a file under `WORKER_HEARTBEAT_PREFIX`. */
-	readonly createHeartbeat?: (name: string) => Heartbeat;
-}
-
-// One message per receive, not the SQS maximum of 10. Handlers process a batch
-// sequentially, so a batch of ten is ten handler runs between two heartbeats:
-// ten stalled SMTP sends at nodemailer's 300 s socket timeout is 50 minutes of
-// legitimate silence, and a threshold that tolerates it detects nothing. At one
-// message the quiet stretch a healthy loop can produce is one handler run, and
-// the extra receive per message is a millisecond against the local queue
-// sidecar — the same total work either way.
-const DEFAULT_MAX_MESSAGES = 1;
-/**
- * What a handler has before the queue redelivers the record underneath it.
- * Exported because a handler that budgets its own wait has to agree with it;
- * `deploy/vps/queues.json` sets the same number on the queue itself, and this
- * is the one that applies wherever a target passes no override.
- */
-export const DEFAULT_VISIBILITY_TIMEOUT_SECONDS = 300;
-
-const pollTarget = async (
-	target: QueuePollerTarget,
-	log: QueuePollerLog,
-	isShuttingDown: () => boolean,
-	shutdownSignal: AbortSignal,
-	buildHeartbeat: (name: string) => Heartbeat,
-): Promise<void> => {
-	const queueName = new URL(target.queueUrl).pathname.split("/").pop();
-	const heartbeat = buildHeartbeat(queueName ?? target.functionName);
-	const sqs = createQueueProducer({ queueUrl: target.queueUrl });
-	const maxMessages = target.maxMessages ?? DEFAULT_MAX_MESSAGES;
-	const visibilityTimeout =
-		target.visibilityTimeoutSeconds ?? DEFAULT_VISIBILITY_TIMEOUT_SECONDS;
-	log.info({ queue: queueName, maxMessages }, "poller: started");
-
-	// Always long-poll (SQS max WaitTimeSeconds). An empty long poll is free
-	// and returns as soon as a message arrives, so this is never slower than
-	// short-polling under load — it just stops burning a ReceiveMessage call
-	// (and, against real SQS, its cost) every loop iteration while traffic is
-	// continuous.
-	const LONG_POLL_WAIT_SECONDS = 20;
-
-	while (!isShuttingDown()) {
-		// Top of the receive attempt, before the long poll: this loop's own file,
-		// so a loop that stops turning — wedged in a socket read, or in a handler
-		// that never returns — goes stale on its own however busy its siblings are.
-		//
-		// A write that fails must not take the worker down with it. A full disk is
-		// the likeliest cause and the moment the loop should stay up, failing
-		// records into the dead-letter queues with logs rather than crash-looping
-		// on the monitoring. The missed beat is itself the signal: the healthcheck
-		// goes stale, which is what this file exists to report.
-		await heartbeat().catch((error) => {
-			log.error(
-				{ queue: queueName, error: inspect(error) },
-				"poller: heartbeat write failed",
-			);
-		});
-
-		let response: ReceiveMessageCommandOutput;
-		try {
-			// Abort the receive on shutdown. Without this, a container stopped
-			// while idle waits out the current 20s long poll before the loop
-			// checks the flag again, and a self-update then burns the whole
-			// stop grace on every idle worker — the bulk of the visible restart
-			// downtime. The abort ends the in-flight receive at once; an
-			// in-flight handler below is never interrupted, so a message being
-			// processed still finishes cleanly. The heartbeat above already wrote
-			// this iteration, so the abort exits without a further beat.
-			response = await sqs.send(
-				new ReceiveMessageCommand({
-					QueueUrl: target.queueUrl,
-					MaxNumberOfMessages: maxMessages,
-					WaitTimeSeconds: LONG_POLL_WAIT_SECONDS,
-					VisibilityTimeout: visibilityTimeout,
-					MessageSystemAttributeNames: ["ApproximateReceiveCount"],
-				}),
-				{ abortSignal: shutdownSignal },
-			);
-		} catch (error) {
-			if (shutdownSignal.aborted) break;
-			throw error;
-		}
-
-		if (!response.Messages || response.Messages.length === 0) {
-			continue;
-		}
-
-		const messages = response.Messages.flatMap((m) =>
-			m.Body && m.ReceiptHandle && m.MessageId
-				? [
-						{
-							messageId: m.MessageId,
-							receiptHandle: m.ReceiptHandle,
-							body: m.Body,
-							receiveCount: m.Attributes?.ApproximateReceiveCount ?? "1",
-						},
-					]
-				: [],
-		);
-		if (messages.length === 0) continue;
-
-		// One batch is one handler invocation, so the id is minted per batch: it is
-		// the `requestId` every line of that invocation is grouped by (see
-		// deploy/vps/README.md, "Logs"). A context built once per target would put
-		// the whole queue's lifetime under one id, which correlates nothing.
-		//
-		// Minted before the poller's own bracketing lines rather than next to the
-		// handler call: grouping on the id has to reach the line saying whether the
-		// message was deleted or left for redelivery, which is what someone looks
-		// for after a failure.
-		const lambdaContext = {
-			functionName: target.functionName,
-			awsRequestId: randomUUID(),
-		} as Context;
-
-		log.info(
-			{
-				queue: queueName,
-				count: messages.length,
-				requestId: lambdaContext.awsRequestId,
-			},
-			"poller: invoking handler",
-		);
-
-		const event: SQSEvent = {
-			Records: messages.map((m) => ({
-				messageId: m.messageId,
-				receiptHandle: m.receiptHandle,
-				body: m.body,
-				attributes: {
-					ApproximateReceiveCount: m.receiveCount,
-					SentTimestamp: "0",
-					SenderId: "poller",
-					ApproximateFirstReceiveTimestamp: "0",
-				},
-				messageAttributes: {},
-				md5OfBody: "",
-				eventSource: "aws:sqs",
-				eventSourceARN: target.queueUrl,
-				awsRegion: "local",
-			})),
-		};
-
-		const result = (await target.handler(event, lambdaContext, () => {})) as
-			| SQSBatchResponse
-			| undefined;
-
-		const failedIds = new Set(
-			(result?.batchItemFailures ?? []).map((f) => f.itemIdentifier),
-		);
-		const succeeded = messages.filter((m) => !failedIds.has(m.messageId));
-
-		// Batch is capped at maxMessages (one by default, 10 is the SQS ceiling),
-		// so a handful of concurrent DeleteMessage calls is safe and avoids paying
-		// N sequential round trips between every receive/handler cycle.
-		await Promise.all(
-			succeeded.map((message) =>
-				sqs.send(
-					new DeleteMessageCommand({
-						QueueUrl: target.queueUrl,
-						ReceiptHandle: message.receiptHandle,
-					}),
-				),
-			),
-		);
-
-		log.info(
-			{
-				queue: queueName,
-				deleted: succeeded.length,
-				leftForRedelivery: failedIds.size,
-				requestId: lambdaContext.awsRequestId,
-			},
-			"poller: batch processed",
-		);
-	}
-
-	log.info({ queue: queueName }, "poller: stopped");
-};
-
-/**
- * Runs every target's poll loop until one of `signals` is received (default:
- * SIGINT, SIGTERM). The signal aborts each loop's in-flight
- * long-poll receive so an idle worker returns at once instead of waiting out
- * the current 20s poll; a loop mid-handler finishes that handler first. Rejects
- * (crashes the process) if any loop throws — a stuck poller should exit loudly,
- * not degrade silently.
- *
- * Every loop writes its own heartbeat file (see heartbeat.ts) at the top of its
- * receive attempt, which is how a loop that hangs instead of throwing — the
- * case this cannot make loud by itself — becomes visible.
- */
-export const runQueuePoller = async (
-	options: RunQueuePollerOptions,
-): Promise<void> => {
-	const { targets, log } = options;
-	const signals = options.signals ?? ["SIGINT", "SIGTERM"];
-	const buildHeartbeat = options.createHeartbeat ?? createHeartbeat;
-
-	if (targets.length === 0) {
-		throw new Error("runQueuePoller: no targets configured");
-	}
-
-	// Files from the previous run of this service, before its loops write again:
-	// the healthcheck reads the oldest file it finds, so one left behind by a
-	// queue this version no longer polls would fail the check forever.
-	await clearHeartbeats().catch((error) => {
-		log.error({ error: inspect(error) }, "poller: heartbeat cleanup failed");
-	});
-
-	let shuttingDown = false;
-	const isShuttingDown = () => shuttingDown;
-	// Aborted alongside the flag so a loop currently blocked in its long-poll
-	// receive returns at once, rather than after up to WaitTimeSeconds.
-	const shutdown = new AbortController();
-	const onSignal = (signal: NodeJS.Signals) => {
-		log.info({ signal }, "poller: shutdown signal received");
-		shuttingDown = true;
-		shutdown.abort();
-	};
-	for (const signal of signals) {
-		process.on(signal, onSignal);
-	}
-
-	log.info({ queues: targets.map((t) => t.queueUrl) }, "poller: starting");
-
-	await Promise.all(
-		targets.map((target) =>
-			pollTarget(target, log, isShuttingDown, shutdown.signal, buildHeartbeat),
-		),
-	);
-};

package/src/producer.test.ts

@@ -1,179 +0,0 @@
-import assert from "node:assert/strict";
-import { createServer } from "node:http";
-import type { AddressInfo } from "node:net";
-import { describe, it } from "node:test";
-import { ReceiveMessageCommand, type SQSClient } from "@aws-sdk/client-sqs";
-import { AwsQueryProtocol } from "@aws-sdk/core/protocols";
-import { createQueueProducer, isLocalEndpoint } from "./producer.js";
-
-// The queue URL a self-hosted compose stack hands its workers: a plain
-// http:// URL addressing the queue container by name, never "localhost".
-const composeQueueUrl = "http://queue:9324/000000000000/mailboxes";
-const awsQueueUrl =
-	"https://sqs.eu-west-1.amazonaws.com/123456789012/remit-dev-mailboxes.fifo";
-
-const usesQueryProtocol = (client: SQSClient): boolean =>
-	client.config.protocol instanceof AwsQueryProtocol;
-
-const resolvedEndpoint = async (
-	client: SQSClient,
-): Promise<string | undefined> => {
-	const { endpoint } = client.config;
-	if (!endpoint) return undefined;
-	const resolved = await endpoint();
-	return `${resolved.protocol}//${resolved.hostname}:${resolved.port}`;
-};
-
-describe("isLocalEndpoint", () => {
-	it("treats a non-localhost http:// queue URL as local", () => {
-		assert.equal(isLocalEndpoint(composeQueueUrl), true);
-	});
-
-	it("treats http://localhost as local", () => {
-		assert.equal(
-			isLocalEndpoint("http://localhost:9324/000000000000/mailboxes"),
-			true,
-		);
-	});
-
-	it("treats https://localhost as local", () => {
-		assert.equal(
-			isLocalEndpoint("https://localhost:9324/000000000000/mailboxes"),
-			true,
-		);
-	});
-
-	it("treats a real SQS queue URL as remote", () => {
-		assert.equal(isLocalEndpoint(awsQueueUrl), false);
-	});
-
-	// A prefix match on "https://localhost" reads all of these as local. The
-	// host is localhost in none of them.
-	it("treats an https host that merely starts with localhost as remote", () => {
-		assert.equal(isLocalEndpoint("https://localhost.example.com/0/q"), false);
-		assert.equal(
-			isLocalEndpoint("https://localhost-queue.example.com/0/q"),
-			false,
-		);
-	});
-
-	it("treats an https URL with localhost in the userinfo as remote", () => {
-		assert.equal(isLocalEndpoint("https://localhost@example.com/0/q"), false);
-		assert.equal(
-			isLocalEndpoint("https://localhost:pass@example.com/0/q"),
-			false,
-		);
-	});
-
-	// The mirror image: a prefix match misses these, and they are local.
-	it("treats a loopback https URL carrying credentials as local", () => {
-		assert.equal(isLocalEndpoint("https://user:pass@localhost/0/q"), true);
-	});
-
-	it("ignores scheme case", () => {
-		assert.equal(isLocalEndpoint("HTTP://queue:9324/0/q"), true);
-		assert.equal(isLocalEndpoint("HtTpS://localhost:9324/0/q"), true);
-	});
-
-	it("treats loopback addresses over https as local", () => {
-		assert.equal(isLocalEndpoint("https://127.0.0.1:9324/0/q"), true);
-		assert.equal(isLocalEndpoint("https://[::1]:9324/0/q"), true);
-	});
-
-	it("treats any http URL as local regardless of host, port, or credentials", () => {
-		assert.equal(isLocalEndpoint("http://queue/0/q"), true);
-		assert.equal(isLocalEndpoint("http://user:pass@queue:9324/0/q"), true);
-		assert.equal(isLocalEndpoint("http://example.com/0/q"), true);
-	});
-
-	// Must not throw: several callers construct their client at module load, so
-	// a throw here would take the worker down at import.
-	it("treats an unparseable queue URL as remote without throwing", () => {
-		assert.equal(isLocalEndpoint(""), false);
-		assert.equal(isLocalEndpoint("not a url"), false);
-		assert.equal(isLocalEndpoint("queue:9324/0/q"), false);
-		assert.equal(isLocalEndpoint("//queue:9324/0/q"), false);
-	});
-});
-
-describe("createQueueProducer", () => {
-	// Regression (self-host stack): a queue URL of http://queue:9324/... used to
-	// fall through to the AWS JSON protocol, so every send hit the local queue
-	// server with a body it cannot parse and mail never synced.
-	it("selects the query protocol for a non-localhost http:// queue URL", async () => {
-		const client = createQueueProducer({ queueUrl: composeQueueUrl });
-		assert.equal(usesQueryProtocol(client), true);
-		assert.equal(await resolvedEndpoint(client), "http://queue:9324");
-	});
-
-	it("selects the query protocol for a localhost queue URL", () => {
-		const client = createQueueProducer({
-			queueUrl: "http://localhost:9324/000000000000/mailboxes",
-		});
-		assert.equal(usesQueryProtocol(client), true);
-	});
-
-	it("leaves the default protocol and endpoint for a real AWS queue URL", async () => {
-		const client = createQueueProducer({ queueUrl: awsQueueUrl });
-		assert.equal(usesQueryProtocol(client), false);
-		assert.equal(await resolvedEndpoint(client), undefined);
-	});
-
-	it("does not throw when the queue URL is unparseable", async () => {
-		const client = createQueueProducer({ queueUrl: "not a url" });
-		assert.equal(usesQueryProtocol(client), false);
-		assert.equal(await resolvedEndpoint(client), undefined);
-	});
-
-	it("accepts an explicit endpoint override", async () => {
-		const client = createQueueProducer({
-			queueUrl: composeQueueUrl,
-			endpoint: "http://custom-endpoint:1234",
-		});
-		assert.equal(usesQueryProtocol(client), true);
-		assert.equal(await resolvedEndpoint(client), "http://custom-endpoint:1234");
-	});
-
-	it("uses localCredentials only for a local endpoint", async () => {
-		const local = createQueueProducer({
-			queueUrl: composeQueueUrl,
-			localCredentials: { accessKeyId: "local", secretAccessKey: "local" },
-			env: {},
-		});
-		assert.equal((await local.config.credentials()).accessKeyId, "local");
-
-		const remote = createQueueProducer({
-			queueUrl: awsQueueUrl,
-			localCredentials: { accessKeyId: "local", secretAccessKey: "local" },
-			env: { SQS_ACCESS_KEY_ID: "real", SQS_SECRET_ACCESS_KEY: "real" },
-		});
-		assert.equal((await remote.config.credentials()).accessKeyId, "real");
-	});
-});
-
-describe("createQueueProducer request timeout", () => {
-	it("fails a receive the queue server never answers", async () => {
-		const server = createServer(() => {});
-		await new Promise<void>((resolve) => {
-			server.listen(0, "127.0.0.1", resolve);
-		});
-		const { port } = server.address() as AddressInfo;
-		const queueUrl = `http://127.0.0.1:${port}/000000000000/hung`;
-		const client = createQueueProducer({
-			queueUrl,
-			requestTimeoutMs: 100,
-			localCredentials: { accessKeyId: "test", secretAccessKey: "test" },
-		});
-
-		try {
-			await assert.rejects(
-				() => client.send(new ReceiveMessageCommand({ QueueUrl: queueUrl })),
-				{ name: "TimeoutError" },
-			);
-		} finally {
-			client.destroy();
-			server.closeAllConnections();
-			await new Promise<void>((resolve) => server.close(() => resolve()));
-		}
-	});
-});

package/src/producer.ts

@@ -1,74 +0,0 @@
-import { SQSClient } from "@aws-sdk/client-sqs";
-import { AwsQueryProtocol } from "@aws-sdk/core/protocols";
-import { resolveSqsCredentials, type SqsStaticCredentials } from "./index.js";
-
-export interface CreateQueueProducerOptions {
-	readonly queueUrl: string;
-	readonly endpoint?: string;
-	readonly env?: NodeJS.ProcessEnv;
-	/**
-	 * Credentials to use when the queue URL resolves to a local endpoint. A
-	 * local queue server ignores them, but the SDK still needs something to
-	 * sign with and a self-hosted stack has no credential chain to fall back on.
-	 */
-	readonly localCredentials?: SqsStaticCredentials;
-	readonly requestTimeoutMs?: number;
-}
-
-const DEFAULT_REQUEST_TIMEOUT_MS = 45_000;
-
-const LOOPBACK_HOSTNAMES = new Set(["localhost", "127.0.0.1", "[::1]"]);
-
-/**
- * Whether a queue URL addresses a local, SQS-compatible queue server
- * (ElasticMQ, the queue sidecar) rather than real SQS.
- *
- * Any plain `http://` URL is local: real SQS and Scaleway M&Q are always
- * HTTPS, and a self-hosted stack addresses its queue server by container name
- * (`http://queue:9324/...`), not by `localhost`. Over HTTPS only a loopback
- * host is local, which covers a local server behind TLS.
- *
- * The URL is parsed rather than prefix-matched. Prefix-matching the string
- * gets the interesting cases wrong in both directions: it reads
- * `https://localhost.example.com` and `https://localhost@example.com` as
- * local (the host is neither), and misses `https://user:pass@localhost` and
- * an uppercase `HTTP://` scheme (both of which are).
- *
- * A queue URL that does not parse is treated as remote. It cannot be sent to
- * either way, and several callers build their client at module load — throwing
- * here would take a worker down at import instead of surfacing the bad
- * configuration at the send that actually uses it.
- */
-export const isLocalEndpoint = (queueUrl: string): boolean => {
-	let url: URL;
-	try {
-		url = new URL(queueUrl);
-	} catch {
-		return false;
-	}
-	if (url.protocol === "http:") return true;
-	return url.protocol === "https:" && LOOPBACK_HOSTNAMES.has(url.hostname);
-};
-
-/**
- * Producer-side counterpart of `runQueuePoller`: the one place an
- * SQS-compatible client is constructed for sending. A queue URL that points at
- * a local endpoint (ElasticMQ) gets its origin as the SDK endpoint and the
- * query protocol ElasticMQ speaks; real SQS and Scaleway M&Q resolve their own
- * endpoint and ride the default credential chain unless
- * `SQS_ACCESS_KEY_ID`/`SQS_SECRET_ACCESS_KEY` are present.
- */
-export const createQueueProducer = (
-	options: CreateQueueProducerOptions,
-): SQSClient => {
-	const { queueUrl, endpoint, env, localCredentials } = options;
-	const requestTimeout = options.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS;
-	const local = isLocalEndpoint(queueUrl);
-	return new SQSClient({
-		requestHandler: { requestTimeout, throwOnRequestTimeout: true },
-		endpoint: endpoint ?? (local ? new URL(queueUrl).origin : undefined),
-		...(local && { protocol: AwsQueryProtocol }),
-		credentials:
-			(local ? localCredentials : undefined) ?? resolveSqsCredentials(env),
-	});
-};

package/tsconfig.json

@@ -1,8 +0,0 @@
-{
-	"extends": "../../tsconfig.json",
-	"compilerOptions": {
-		"outDir": "dist",
-		"rootDir": "src"
-	},
-	"include": ["src/**/*.ts"]
-}