@remit/backend 0.0.6 → 0.0.8

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/backend",
-  "version": "0.0.6",
+  "version": "0.0.8",
   "description": "Remit Mail Inspector API backend",
   "license": "MIT",
   "author": "",
@@ -41,21 +41,14 @@
     "dev:remote": "node --env-file ../../.env.dev --watch --import tsx dev-server/server.ts"
   },
   "devDependencies": {
-    "@aws-sdk/client-secrets-manager": "*",
-    "@aws-sdk/client-sqs": "*",
     "@aws-sdk/core": "*",
     "@remit/electrodb-entities": "*",
-    "drizzle-orm": "^0.45.2",
     "@remit/search-index-worker": "*",
-    "@types/aws-lambda": "*",
     "@types/express": "^5.0.3",
     "@types/swagger-ui-express": "^4.1.8",
     "electrodb": "*",
     "esbuild": "^0.27.7",
-    "expect-env": "*",
     "express": "^5.1.0",
-    "openapi-backend": "*",
-    "p-map": "*",
     "swagger-ui-express": "^5.0.1",
     "tsx": "*"
   },
@@ -73,7 +66,14 @@
     "@remit/storage-service": "*",
     "@remit/auth-service": "*",
     "@remit/drizzle-service": "*",
-    "@remit/sqs-client": "*"
+    "@remit/sqs-client": "*",
+    "@aws-sdk/client-secrets-manager": "*",
+    "@aws-sdk/client-sqs": "*",
+    "drizzle-orm": "^0.45.2",
+    "expect-env": "*",
+    "openapi-backend": "*",
+    "p-map": "*",
+    "@types/aws-lambda": "*"
   },
   "publishConfig": {
     "access": "public"

package/src/handlers/sync.test.ts

@@ -62,6 +62,28 @@ describe("triggerSyncSafe", () => {
 		assert.equal(error.length, 0);
 	});
 
+	// The guarantee behind issue #37: POST /sync asks for a sync of this
+	// account by name, so the event it enqueues has to carry the mark that
+	// makes the worker's fan-out sync every mailbox rather than skip the fresh
+	// ones. Its callers are the refresh control, pull-to-refresh and the
+	// client's automatic poll — the poll is kept from abusing this by the
+	// interval floor in useStaleAccountSync, not by the server.
+	it("marks the event as an explicit request", async () => {
+		const sent: SendMessageCommand[] = [];
+		const { logger } = createLoggerSpy();
+
+		await triggerSyncSafe("acc-a", "config-1", {
+			sqsClient: okSqsClient(sent),
+			queueUrl: QUEUE_URL,
+			logger,
+		});
+
+		const body = JSON.parse(sent[0]?.input.MessageBody ?? "{}") as {
+			explicitRequest?: boolean;
+		};
+		assert.equal(body.explicitRequest, true);
+	});
+
 	it("does NOT reject when the SQS enqueue fails (POST /sync stays resilient)", async () => {
 		const econnrefused = Object.assign(new Error(""), {
 			name: "AggregateError",

package/src/handlers/sync.ts

@@ -54,6 +54,13 @@ export const triggerSyncSafe = async (
 				sqsClient: deps.sqsClient,
 				queueUrl: deps.queueUrl,
 				accountId,
+				// POST /sync asks for a sync of this account by name — the refresh
+				// control, pull-to-refresh, and the client's automatic poll all land
+				// here — so it syncs every mailbox regardless of how recently one
+				// ran. The poll's cadence is floored client-side at the fan-out
+				// gate's window, so this branch cannot be driven faster than the
+				// gate would allow.
+				explicitRequest: true,
 			});
 			deps.logger.info(
 				{ accountId, eventId },

package/src/service/trigger-sync.test.ts

@@ -2,7 +2,6 @@ import assert from "node:assert/strict";
 import { describe, it } from "node:test";
 import { SendMessageCommand, type SQSClient } from "@aws-sdk/client-sqs";
 import {
-	buildManualSyncDedupId,
 	buildScheduledSyncDedupId,
 	buildSyncMailboxesCommand,
 	triggerAccountSync,
@@ -29,20 +28,62 @@ describe("buildSyncMailboxesCommand", () => {
 		assert.equal(cmd.input.MessageGroupId, "account-abc");
 	});
 
-	it("defaults MessageDeduplicationId to the time-bucketed manual dedup id for FIFO queues", () => {
+	it("defaults MessageDeduplicationId to the event's own id for FIFO queues", () => {
 		const cmd = buildSyncMailboxesCommand({
 			sqsClient: {} as SQSClient,
 			queueUrl: FIFO_QUEUE_URL,
 			accountId: "account-abc",
 		});
 
-		assert.ok(
-			cmd.input.MessageDeduplicationId?.startsWith(
-				"SYNC_MAILBOXES:manual:account-abc:",
-			),
+		assert.equal(cmd.input.MessageDeduplicationId, parseBody(cmd).eventId);
+	});
+
+	// Issue #37: a shared, time-bucketed dedup id let SQS FIFO's 5-minute
+	// window discard the second sync of an account, so mail that arrived after
+	// a sync stayed invisible until the window elapsed. Back-to-back triggers
+	// must each reach the queue.
+	it("gives two triggers for one account distinct dedup ids", () => {
+		const first = buildSyncMailboxesCommand({
+			sqsClient: {} as SQSClient,
+			queueUrl: FIFO_QUEUE_URL,
+			accountId: "account-abc",
+		});
+		const second = buildSyncMailboxesCommand({
+			sqsClient: {} as SQSClient,
+			queueUrl: FIFO_QUEUE_URL,
+			accountId: "account-abc",
+		});
+
+		assert.notEqual(
+			first.input.MessageDeduplicationId,
+			second.input.MessageDeduplicationId,
 		);
 	});
 
+	// The worker's fan-out gate reads this off the event: it is what lets a
+	// sync asked for by name cover every folder while a side-effect trigger
+	// skips the ones that were just enumerated.
+	it("marks an explicitly-requested trigger on the event", () => {
+		const cmd = buildSyncMailboxesCommand({
+			sqsClient: {} as SQSClient,
+			queueUrl: FIFO_QUEUE_URL,
+			accountId: "account-abc",
+			explicitRequest: true,
+		});
+
+		assert.equal(parseBody(cmd).explicitRequest, true);
+	});
+
+	it("leaves a side-effect trigger unmarked", () => {
+		const cmd = buildSyncMailboxesCommand({
+			sqsClient: {} as SQSClient,
+			queueUrl: FIFO_QUEUE_URL,
+			accountId: "account-abc",
+		});
+
+		assert.equal(parseBody(cmd).explicitRequest, undefined);
+	});
+
 	it("does not set FIFO params for standard queues", () => {
 		const cmd = buildSyncMailboxesCommand({
 			sqsClient: {} as SQSClient,
@@ -93,43 +134,6 @@ describe("buildSyncMailboxesCommand", () => {
 	});
 });
 
-describe("buildManualSyncDedupId", () => {
-	const SIXTY_SECONDS_MS = 60 * 1000;
-
-	it("is stable within the same time bucket (dedupes a rapid double-tap)", () => {
-		const bucketStart = 10 * SIXTY_SECONDS_MS;
-
-		const first = buildManualSyncDedupId("account-abc", bucketStart);
-		const second = buildManualSyncDedupId("account-abc", bucketStart + 1_000);
-
-		assert.equal(first, second);
-	});
-
-	it("changes across a bucket boundary (never dedupes two polls a minute apart)", () => {
-		const bucketStart = 10 * SIXTY_SECONDS_MS;
-
-		const thisTrigger = buildManualSyncDedupId("account-abc", bucketStart);
-		const nextTrigger = buildManualSyncDedupId(
-			"account-abc",
-			bucketStart + SIXTY_SECONDS_MS,
-		);
-
-		assert.notEqual(thisTrigger, nextTrigger);
-	});
-
-	it("never collides with the scheduler's dedup namespace", () => {
-		const now = 10 * SIXTY_SECONDS_MS;
-		const manual = buildManualSyncDedupId("account-abc", now);
-		const scheduled = buildScheduledSyncDedupId(
-			"account-abc",
-			now,
-			SIXTY_SECONDS_MS,
-		);
-
-		assert.notEqual(manual, scheduled);
-	});
-});
-
 describe("buildScheduledSyncDedupId", () => {
 	const FIVE_MINUTES_MS = 5 * 60 * 1000;
 

package/src/service/trigger-sync.ts

@@ -6,6 +6,7 @@ interface SyncMailboxesEvent {
 	eventId: string;
 	timestamp: number;
 	accountId: string;
+	explicitRequest?: boolean;
 }
 
 interface TriggerAccountSyncInput {
@@ -13,45 +14,42 @@ interface TriggerAccountSyncInput {
 	queueUrl: string;
 	accountId: string;
 	/**
-	 * Override the FIFO `MessageDeduplicationId`. Defaults to a time-bucketed
-	 * manual-trigger id (`buildManualSyncDedupId`) shared by every manual call
-	 * site (POST /sync, OAuth connect, config load, pull-to-refresh, the
-	 * client's online-poll) so a rapid double-tap collapses into one enqueue,
-	 * while distinct triggers a bucket or more apart always pass through.
+	 * Set by POST /sync, whose callers ask for a sync of one named account: the
+	 * refresh control, pull-to-refresh, and the web client's automatic poll
+	 * (`useStaleAccountSync`) — a timer, not a person. It travels on the event
+	 * and makes the worker's fan-out sync every mailbox even if one just ran.
 	 *
-	 * The scheduled-sync tick (#1247) must NOT share that namespace: pass
-	 * `buildScheduledSyncDedupId()` here instead, so the two paths can never
-	 * suppress each other.
+	 * Everything else triggers a sync as a side effect of doing something else
+	 * (config load, OAuth connect, account create, the scheduled tick), leaves
+	 * this unset, and takes the freshness gate — see `mailboxNeedsSync` in
+	 * imap-worker's sync-mailboxes handler.
+	 *
+	 * The poll cannot use this to outrun the gate: its interval is floored at
+	 * the gate's own window (`MIN_POLL_INTERVAL_MS` in the client hook), so
+	 * however low `mailboxPollIntervalSeconds` is set, a timer never fans out
+	 * more often than the gate would have allowed anyway.
+	 */
+	explicitRequest?: boolean;
+	/**
+	 * Override the FIFO `MessageDeduplicationId`. Defaults to the event's own
+	 * id, so the queue suppresses a re-send of one event and nothing else —
+	 * every manual call site (POST /sync, OAuth connect, config load,
+	 * pull-to-refresh, the client's online-poll) always enqueues. A shared,
+	 * time-bucketed id instead turned SQS's 5-minute window into a rate limiter
+	 * and silently discarded a sync the user asked for whenever one had run
+	 * recently (issue #37). What a trigger costs is bounded in the worker's
+	 * fan-out (`mailboxNeedsSync`), by skipping mailboxes not worth
+	 * re-enumerating — never by dropping the trigger.
+	 *
+	 * The scheduled-sync tick (#1247) passes `buildScheduledSyncDedupId()`,
+	 * which is bucketed by the tick's own cadence: there it collapses a
+	 * re-invocation of a single tick, never two ticks or a manual trigger.
 	 */
 	dedupId?: string;
 }
 
 const isFifoQueue = (queueUrl: string): boolean => queueUrl.endsWith(".fifo");
 
-/**
- * Bucket window for the manual-trigger dedup id. SQS FIFO's own dedup window
- * is 5 minutes; a static per-account id (the pre-#1250 behaviour) collided
- * with the client's 5-minute online poll and swallowed a pull-to-refresh
- * within 5 minutes of any prior trigger. Bucketing to ~60s keeps a rapid
- * double-tap deduped while letting distinct polls a minute or more apart
- * always enqueue.
- */
-const MANUAL_DEDUP_BUCKET_MS = 60_000;
-
-/**
- * Build the manual-trigger dedup id (POST /sync, OAuth connect, config load,
- * pull-to-refresh, client online-poll), bucketed so consecutive triggers a
- * bucket apart never collide — see `dedupId` above.
- */
-export const buildManualSyncDedupId = (
-	accountId: string,
-	now: number,
-	bucketMs: number = MANUAL_DEDUP_BUCKET_MS,
-): string => {
-	const bucket = Math.floor(now / bucketMs);
-	return `SYNC_MAILBOXES:manual:${accountId}:${bucket}`;
-};
-
 /**
  * Build the scheduler's own dedup namespace, bucketed by tick interval so
  * consecutive ticks each get a fresh id (never colliding with each other)
@@ -70,12 +68,13 @@ export const buildScheduledSyncDedupId = (
 export const buildSyncMailboxesCommand = (
 	input: TriggerAccountSyncInput,
 ): SendMessageCommand => {
-	const { queueUrl, accountId, dedupId } = input;
+	const { queueUrl, accountId, dedupId, explicitRequest } = input;
 	const event: SyncMailboxesEvent = {
 		type: "SYNC_MAILBOXES",
 		eventId: randomUUID(),
 		timestamp: Date.now(),
 		accountId,
+		...(explicitRequest && { explicitRequest }),
 	};
 
 	const useFifo = isFifoQueue(queueUrl);
@@ -85,8 +84,7 @@ export const buildSyncMailboxesCommand = (
 		MessageBody: JSON.stringify(event),
 		...(useFifo && {
 			MessageGroupId: accountId,
-			MessageDeduplicationId:
-				dedupId ?? buildManualSyncDedupId(accountId, event.timestamp),
+			MessageDeduplicationId: dedupId ?? event.eventId,
 		}),
 	});
 };