@remit/imap-worker 0.0.90 → 0.0.92

package/src/scheduler/runner.ts

@@ -1,125 +0,0 @@
-#!/usr/bin/env node
-import { setTimeout as delay } from "node:timers/promises";
-import { finishHalfDeletedAccounts } from "@remit/backend/account-deletion";
-import { getClient } from "@remit/backend/client";
-import { createLogger } from "@remit/logger";
-import { installNodeLogger } from "@remit/logger-lambda/node";
-import { backfillSearchText } from "@remit/mailbox-service";
-import { clearHeartbeats, createHeartbeat } from "@remit/sqs-client/heartbeat";
-
-import { createEmitEvent } from "../emit.js";
-import { sqsWorkerQueues, workerQueueUrlsFromEnv } from "../queues.js";
-import { buildAttachmentSweep } from "./attachment-sweep.js";
-import { buildCalendarSubscriptionRefresh } from "./calendar-subscriptions.js";
-import { getOfflineIntervalMs, getTickIntervalMs } from "./config.js";
-import { runSchedulerLoop } from "./loop.js";
-import { runSchedulerTick } from "./run-tick.js";
-import { redriveStrandedPlacements } from "./stranded-placement-redrive.js";
-
-installNodeLogger();
-
-/**
- * The scheduled-sync runner for every deployment that has no EventBridge: the
- * self-host stack's `scheduler` service and the local dev compose stack. A
- * managed deployment fires `runSchedulerTick` off an EventBridge schedule (see
- * handler.ts); this process ticks the same function on a plain loop at the same
- * `MAILBOX_SYNC_TICK_INTERVAL_SECONDS` cadence, so there is one scheduling
- * implementation and one set of knobs.
- *
- * A tick failure crashes the process loudly rather than being swallowed —
- * compose's `restart: unless-stopped` brings it back for the next tick.
- *
- * Liveness is a heartbeat file, the same mechanism as a worker's poll loop (D1
- * of docs/design/standalone-observability.md); loop.ts carries where in the loop
- * it is written and why. Clearing the previous generation's file here is what
- * extends that answer across a restart. A completed round is still not the same
- * as mail arriving: the accounts that came due are enqueued for workers to
- * fetch, and whether that fetch lands is what
- * `remit_account_sync_age_seconds` measures.
- */
-
-const log = createLogger();
-
-const queues = sqsWorkerQueues(workerQueueUrlsFromEnv());
-
-const tickIntervalMs = getTickIntervalMs();
-const offlineIntervalMs = getOfflineIntervalMs();
-
-// A persistent failure (e.g. Postgres not up yet at container boot) throws
-// before the loop ever reaches its own `delay`, so `restart: unless-stopped`
-// would otherwise respawn the process immediately — a tight, log-flooding
-// crash loop (review #1250). This fixed pause before exiting is not retry
-// logic (there is nothing to retry here; the container restart IS the
-// retry) — it only paces how fast that restart can happen.
-const CRASH_BACKOFF_MS = 5_000;
-
-log.info(
-	{ tickIntervalMs, offlineIntervalMs },
-	"Scheduled-sync runner started",
-);
-
-const runLoop = async (): Promise<void> => {
-	// Neither call on the monitoring file may take the scheduler down with it. An
-	// unreadable or full /data/heartbeat is a reason to keep enqueuing mail, not
-	// to stop. A clear that fails leaves the previous generation's file, which
-	// ages out at the same threshold; a write that fails is the missed beat that
-	// is itself the signal. Two messages because they are different problems: one
-	// is a volume this container could not read at boot, the other a write it
-	// could not make this round.
-	const onClearError = (error: unknown): void => {
-		log.error({ error }, "Scheduled-sync heartbeat clear failed");
-	};
-	const onBeatError = (error: unknown): void => {
-		log.error({ error }, "Scheduled-sync heartbeat write failed");
-	};
-
-	await clearHeartbeats().catch(onClearError);
-	const client = await getClient();
-	const { account } = client;
-	await finishHalfDeletedAccounts({
-		account,
-		configImport: client.configImport,
-		queue: client.queues.accountFanout,
-		log,
-	});
-	const searchTextBackfill = backfillSearchText(
-		{ messageService: client.message, storageService: client.storage },
-		log,
-	).then((result) => {
-		log.debug({ ...result }, "Search text backfill finished");
-	});
-	const schedulerLoop = runSchedulerLoop({
-		tick: runSchedulerTick,
-		tickDeps: {
-			accountService: account,
-			queue: queues.mailboxes,
-			log,
-			tickIntervalMs,
-			offlineIntervalMs,
-			sweepAttachments: buildAttachmentSweep(client, log),
-			refreshCalendarSubscriptions: buildCalendarSubscriptionRefresh(
-				client,
-				log,
-			),
-			redriveStrandedPlacements: (stranded, now) =>
-				redriveStrandedPlacements(
-					{ client, emit: createEmitEvent(queues), log },
-					stranded,
-					now,
-				),
-		},
-		heartbeat: createHeartbeat("tick"),
-		onHeartbeatError: onBeatError,
-		tickIntervalMs,
-	});
-	await Promise.all([searchTextBackfill, schedulerLoop]);
-};
-
-runLoop()
-	.catch(async (error) => {
-		log.error({ error }, "Scheduled-sync tick failed");
-		await delay(CRASH_BACKOFF_MS);
-	})
-	.finally(() => {
-		process.exit(1);
-	});

package/src/scheduler/stranded-placement-redrive.ts

@@ -1,265 +0,0 @@
-import type { RemitClient } from "@remit/backend/client";
-import type { AccountItem, MessageItem } from "@remit/data-ports";
-import { trashMailboxAt } from "@remit/data-ports/folder-role";
-import {
-	deriveCopyMessageId,
-	deriveMessageId,
-	isValidMessageId,
-} from "@remit/data-ports/id";
-import { PATHLESS_MUTATION_EVENT_SCHEMA_VERSION } from "@remit/data-ports/mutation-events";
-import { MessageStatus, MessageSyncStatus } from "@remit/domain-enums";
-import type { Logger } from "@remit/logger";
-import { isCursorRebuildNeeded } from "@remit/mailbox-service";
-import type {
-	MessageCopyEventV3,
-	MessageDeleteEventV3,
-	MessageMoveEventV3,
-	PlacementMovePushEvent,
-} from "../events.js";
-
-/** Well past three attempts on a 60 s visibility timeout. */
-export const STRANDED_PLACEMENT_AFTER_MILLIS = 60 * 60 * 1000;
-
-type Unstamped<T> = Omit<T, "eventId" | "timestamp">;
-
-export type StrandedPlacementEvent =
-	| Unstamped<PlacementMovePushEvent>
-	| Unstamped<MessageDeleteEventV3>
-	| Unstamped<MessageMoveEventV3>
-	| Unstamped<MessageCopyEventV3>;
-
-export interface StrandedPlacementRedriveDeps {
-	client: Pick<
-		RemitClient,
-		| "mailbox"
-		| "mailboxSpecialUse"
-		| "message"
-		| "placementMove"
-		| "threadMessage"
-	>;
-	emit: (event: StrandedPlacementEvent) => Promise<void>;
-	log: Logger;
-}
-
-const isStrandedCopy = (row: MessageItem): boolean =>
-	row.status === MessageStatus.moving &&
-	row.uid === 0 &&
-	typeof row.originalUid !== "number";
-
-type CopyRedrive =
-	| { kind: "redrive"; event: Unstamped<MessageCopyEventV3> }
-	| { kind: "unresolvable"; reason: string }
-	| { kind: "wait" };
-
-/**
- * The message a uid-0 copy was made from: its id derives from the Message-ID
- * the copy's listing row carries, and the copy's own id derives from it.
- */
-const copyRedriveFor = async (
-	client: StrandedPlacementRedriveDeps["client"],
-	account: AccountItem,
-	copy: MessageItem,
-): Promise<CopyRedrive> => {
-	const listing = await client.threadMessage.findByMessageId(
-		account.accountConfigId,
-		copy.messageId,
-	);
-	if (!listing) return { kind: "unresolvable", reason: "listing row missing" };
-	const header = listing.messageIdHeader;
-	if (!header || !isValidMessageId(header)) {
-		return { kind: "unresolvable", reason: "listing row has no Message-ID" };
-	}
-	const sourceId = deriveMessageId(account.accountId, header);
-	if (deriveCopyMessageId(sourceId, copy.mailboxId) !== copy.messageId) {
-		return {
-			kind: "unresolvable",
-			reason: "copy was not made from the message its Message-ID names",
-		};
-	}
-	const [source] = await client.message.get([sourceId]);
-	if (!source) return { kind: "unresolvable", reason: "source row missing" };
-	if (source.status === MessageStatus.deleted) {
-		return { kind: "unresolvable", reason: "source row deleted" };
-	}
-	if (source.status !== MessageStatus.active) return { kind: "wait" };
-	return {
-		kind: "redrive",
-		event: {
-			type: "MESSAGE_COPY",
-			schemaVersion: PATHLESS_MUTATION_EVENT_SCHEMA_VERSION,
-			accountId: account.accountId,
-			sourceMessageId: source.messageId,
-			newMessageId: copy.messageId,
-			sourceMailboxId: source.mailboxId,
-			destinationMailboxId: copy.mailboxId,
-			uid: source.uid,
-		},
-	};
-};
-
-/** A copy with no source to copy from has nothing to bind and nothing to hide. */
-const removeUnresolvableCopy = async (
-	deps: StrandedPlacementRedriveDeps,
-	account: AccountItem,
-	row: MessageItem,
-	reason: string,
-	now: number,
-): Promise<boolean> => {
-	const removed = await deps.client.message.deleteIfPlacement(row.messageId, {
-		status: MessageStatus.moving,
-		mailboxId: row.mailboxId,
-		uid: 0,
-	});
-	if (!removed) return false;
-	deps.log.error(
-		{
-			alert: "stranded_copy_unresolvable",
-			accountId: account.accountId,
-			messageId: row.messageId,
-			mailboxId: row.mailboxId,
-			reason,
-			untouchedForMs: now - row.updatedAt,
-		},
-		"A copy stranded at uid 0 has no source to copy from; the copy row was removed",
-	);
-	return true;
-};
-
-const eventFor = async (
-	client: StrandedPlacementRedriveDeps["client"],
-	account: AccountItem,
-	row: MessageItem,
-	trashMailboxId: string | undefined,
-): Promise<StrandedPlacementEvent | undefined> => {
-	const { accountId, accountConfigId } = account;
-	const { messageId } = row;
-
-	if (row.status === MessageStatus.deleting) {
-		return {
-			type: "MESSAGE_DELETE",
-			schemaVersion: PATHLESS_MUTATION_EVENT_SCHEMA_VERSION,
-			accountId,
-			messageId,
-			mailboxId: row.mailboxId,
-			uid: row.uid,
-			operation: "permanent_delete",
-		};
-	}
-	if (
-		typeof row.originalMailboxId !== "string" ||
-		typeof row.originalUid !== "number"
-	) {
-		return undefined;
-	}
-
-	const marker = await client.placementMove.find(messageId);
-	if (marker && marker.destinationMailboxId === row.mailboxId) {
-		return {
-			type: "PLACEMENT_MOVE_PUSH",
-			accountId,
-			accountConfigId,
-			messageId,
-		};
-	}
-	if (row.mailboxId === trashMailboxId) {
-		return {
-			type: "MESSAGE_DELETE",
-			schemaVersion: PATHLESS_MUTATION_EVENT_SCHEMA_VERSION,
-			accountId,
-			messageId,
-			mailboxId: row.originalMailboxId,
-			uid: row.originalUid,
-			operation: "move_to_trash",
-			destinationMailboxId: row.mailboxId,
-		};
-	}
-	return {
-		type: "MESSAGE_MOVE",
-		schemaVersion: PATHLESS_MUTATION_EVENT_SCHEMA_VERSION,
-		accountId,
-		messageId,
-		sourceMailboxId: row.originalMailboxId,
-		destinationMailboxId: row.mailboxId,
-		uid: row.originalUid,
-	};
-};
-
-/** The claim restarts the row's clock; a folder awaiting a cursor rebuild waits until it is normal. */
-export const redriveStrandedPlacements = async (
-	deps: StrandedPlacementRedriveDeps,
-	account: AccountItem,
-	now: number,
-): Promise<number> => {
-	const { client, emit, log } = deps;
-	const { accountId } = account;
-
-	const mailboxes = await client.mailbox.listAllByAccount(accountId);
-	const paused = new Set(
-		mailboxes
-			.filter((mailbox) => isCursorRebuildNeeded(mailbox.cursorState))
-			.map((mailbox) => mailbox.mailboxId),
-	);
-	const stranded = (
-		await client.message.listStrandedPlacements(
-			mailboxes.map((mailbox) => mailbox.mailboxId),
-			now - STRANDED_PLACEMENT_AFTER_MILLIS,
-		)
-	).filter(
-		(row) =>
-			!paused.has(row.mailboxId) &&
-			!(row.originalMailboxId && paused.has(row.originalMailboxId)),
-	);
-	if (stranded.length === 0) return 0;
-
-	const trash = trashMailboxAt(
-		await client.mailboxSpecialUse.resolveTrashRole(accountId),
-		"resolved",
-	);
-	const trashMailboxId = trash.allowed ? trash.mailbox.mailboxId : undefined;
-
-	let redriven = 0;
-	for (const row of stranded) {
-		const copy = isStrandedCopy(row)
-			? await copyRedriveFor(client, account, row)
-			: undefined;
-		if (copy?.kind === "wait") continue;
-		if (copy?.kind === "unresolvable") {
-			if (await removeUnresolvableCopy(deps, account, row, copy.reason, now)) {
-				redriven += 1;
-			}
-			continue;
-		}
-		const event =
-			copy?.kind === "redrive"
-				? copy.event
-				: await eventFor(client, account, row, trashMailboxId);
-		if (!event) continue;
-
-		// A copy's earlier COPY may have landed with its answer lost, so the
-		// re-drive marks it `failed`: the copy handler then asks the destination
-		// before it copies again.
-		const claimed = await client.message.transitionPlacement(
-			row.messageId,
-			{ status: row.status, mailboxId: row.mailboxId, uid: row.uid },
-			{
-				syncStatus: copy ? MessageSyncStatus.failed : MessageSyncStatus.pending,
-			},
-		);
-		if (!claimed) continue;
-
-		await emit(event);
-		redriven += 1;
-		log.warn(
-			{
-				metric: "stranded_placement_redriven",
-				accountId,
-				messageId: row.messageId,
-				event: event.type,
-				syncStatus: row.syncStatus,
-				untouchedForMs: now - row.updatedAt,
-			},
-			"A move or delete had no event left to settle it; re-driven",
-		);
-	}
-	return redriven;
-};