@remit/imap-worker 0.0.83 → 0.0.85

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remit/imap-worker",
-  "version": "0.0.83",
+  "version": "0.0.85",
   "type": "module",
   "main": "dist/index.js",
   "exports": {
@@ -41,6 +41,7 @@
     "@remit/search-index-worker": "*",
     "@remit/sqs-client": "*",
     "@remit/backend": "*",
+    "@remit/calendar-service": "*",
     "@remit/domain-enums": "*",
     "@remit/mail-oauth-service": "*",
     "@remit/secrets-service": "*",

package/src/handlers/imported-folder.test.ts

@@ -0,0 +1,63 @@
+import assert from "node:assert";
+import { describe, it } from "node:test";
+import type { MailboxItem } from "@remit/data-ports";
+import type { CreateMailboxQueueInput } from "@remit/mailbox-service";
+import { createImportedFolder } from "./imported-folder.js";
+
+const inbox = {
+	mailboxId: "mbx-inbox",
+	accountId: "acc-1",
+	fullPath: "INBOX",
+	namespaceType: "personal",
+	namespacePrefix: "INBOX.",
+	hierarchyDelimiter: ".",
+} as MailboxItem;
+
+describe("createImportedFolder (#1103)", () => {
+	it("creates the folder through the pending folder-create path, in the INBOX namespace", async () => {
+		const calls: [CreateMailboxQueueInput, string, boolean | undefined][] = [];
+		const create = createImportedFolder(
+			{ findByPath: async () => inbox },
+			{
+				createMailbox: async (input, accountId, subscribe) => {
+					calls.push([input, accountId, subscribe]);
+					return {
+						outcome: "Created" as const,
+						mailbox: {
+							...inbox,
+							mailboxId: "mbx-new",
+							fullPath: input.fullPath,
+						},
+					};
+				},
+			},
+		);
+
+		const created = await create("acc-1", "INBOX.Facturen");
+
+		assert.equal(
+			created.outcome === "Created" && created.mailbox.mailboxId,
+			"mbx-new",
+		);
+		assert.equal(calls.length, 1);
+		const [input, accountId, subscribe] = calls[0];
+		assert.equal(accountId, "acc-1");
+		assert.equal(subscribe, true);
+		assert.equal(input.fullPath, "INBOX.Facturen");
+		assert.equal(input.namespacePrefix, "INBOX.");
+		assert.equal(input.hierarchyDelimiter, ".");
+	});
+
+	it("refuses to guess a namespace for an account with no INBOX", async () => {
+		const create = createImportedFolder(
+			{ findByPath: async () => null },
+			{
+				createMailbox: async () => {
+					throw new Error("must not be called");
+				},
+			},
+		);
+
+		await assert.rejects(create("acc-1", "Facturen"), /no INBOX/);
+	});
+});

package/src/handlers/imported-folder.ts

@@ -0,0 +1,38 @@
+import type { CreateImportedFolder } from "@remit/config-transfer";
+import type { IMailboxRepository } from "@remit/data-ports";
+import type { MailboxQueueService } from "@remit/mailbox-service";
+
+export const createImportedFolder =
+	(
+		mailbox: Pick<IMailboxRepository, "findByPath">,
+		queue: Pick<MailboxQueueService, "createMailbox">,
+	): CreateImportedFolder =>
+	async (accountId, folderPath) => {
+		const inbox = await mailbox.findByPath(accountId, "INBOX");
+		if (!inbox) {
+			throw new Error(
+				`Account ${accountId} has no INBOX to take a namespace from, so "${folderPath}" cannot be created`,
+			);
+		}
+		return queue.createMailbox(
+			{
+				accountId,
+				namespaceType: inbox.namespaceType,
+				namespacePrefix: inbox.namespacePrefix,
+				hierarchyDelimiter: inbox.hierarchyDelimiter,
+				fullPath: folderPath,
+				uidValidity: 0,
+				uidNext: 1,
+				highestModseq: "0",
+				messageCount: 0,
+				unseenCount: 0,
+				deletedCount: 0,
+				totalSize: 0,
+				lastSyncUid: 0,
+				highWaterMarkUid: 0,
+				lastMessageSyncAt: 0,
+			},
+			accountId,
+			true,
+		);
+	};

package/src/handlers/sync-mailboxes.ts

@@ -3,6 +3,7 @@ import { writeFolderRoleAppointment } from "@remit/backend/folder-role-appointme
 import {
 	bindImportedFolders,
 	type ConfigBinderDeps,
+	disableFiltersMissingFolders,
 } from "@remit/config-transfer";
 import type {
 	AccountItem,
@@ -16,10 +17,12 @@ import type { Logger } from "@remit/logger-lambda";
 import { RefreshTokenError } from "@remit/mail-oauth-service";
 import {
 	createConnectionWithCredentials,
+	MailboxQueueService,
 	MailboxSyncService,
 	MailConnectionError,
 	type MailCredentials,
 } from "@remit/mailbox-service";
+import { env } from "expect-env";
 import pMap from "p-map";
 import {
 	isAccountDeleted,
@@ -30,6 +33,7 @@ import { emitEvent } from "../emit.js";
 import type { SyncMailboxesEvent, SyncMessagesEvent } from "../events.js";
 import { withOAuthLifecycle } from "../with-oauth-lifecycle.js";
 import { buildLifecycleDeps } from "../with-oauth-lifecycle-deps.js";
+import { createImportedFolder } from "./imported-folder.js";
 import { orderMailboxesForSync } from "./mailbox-sync-order.js";
 
 const EVENT_EMIT_CONCURRENCY = 20;
@@ -110,6 +114,14 @@ export const syncMailboxes = async (
 				mailboxId,
 				lastKnownPath,
 			),
+		createFolder: createImportedFolder(
+			mailboxService,
+			new MailboxQueueService({
+				mailboxService,
+				sqsQueueUrl: env.SQS_QUEUE_URL_MAILBOX_MGMT,
+				logger: log,
+			}),
+		),
 	};
 
 	const { accountId } = event;
@@ -233,10 +245,21 @@ const syncMailboxesForAccount = async (
 		account.accountConfigId,
 		accountId,
 	);
-	if (bound.bound > 0 || bound.dropped > 0 || bound.stillPending > 0) {
+	if (Object.values(bound).some((count) => count > 0)) {
 		log.info({ accountId, ...bound }, "Bound imported folder references");
 	}
 
+	const disabled = await disableFiltersMissingFolders(
+		binder.repositories,
+		account.accountConfigId,
+	);
+	if (disabled > 0) {
+		log.warn(
+			{ accountId, disabled },
+			"Disabled filters whose move folder no longer exists",
+		);
+	}
+
 	const allMailboxes = await collectAllMailboxes(accountId, mailboxService);
 
 	// This fan-out is where an account's IMAP work is decided: one SEARCH per

package/src/scheduler/calendar-subscriptions.test.ts

@@ -0,0 +1,159 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import type { CalendarFeedFetcher } from "@remit/calendar-service";
+import { MemoryCalendarStore } from "@remit/calendar-service/memory-store";
+import type { CalendarCollectionItem } from "@remit/data-ports";
+import { CalendarSource } from "@remit/domain-enums";
+import type { Logger } from "@remit/logger-lambda";
+import { refreshDueCalendarSubscriptions } from "./calendar-subscriptions.js";
+
+const INTERVAL_MS = 15 * 60 * 1000;
+const NOW = 1_800_000_000_000;
+
+const FEED = `${[
+	"BEGIN:VCALENDAR",
+	"VERSION:2.0",
+	"PRODID:-//Example//Published Calendar//EN",
+	"BEGIN:VEVENT",
+	"UID:ferry@calendar.example",
+	"DTSTAMP:20260901T000000Z",
+	"DTSTART:20260908T070000Z",
+	"DTEND:20260908T080000Z",
+	"SUMMARY:Ferry crossing",
+	"END:VEVENT",
+	"END:VCALENDAR",
+].join("\r\n")}\r\n`;
+
+const logged = (): { log: Logger; lines: string[] } => {
+	const lines: string[] = [];
+	const capture = (...args: unknown[]) => {
+		lines.push(JSON.stringify(args));
+	};
+	const log = {
+		info: capture,
+		warn: capture,
+		error: capture,
+		debug: capture,
+		fatal: capture,
+		trace: capture,
+		child: () => log,
+	} as unknown as Logger;
+	return { log, lines };
+};
+
+const subscription = async (
+	store: MemoryCalendarStore,
+	urlSegment: string,
+	checkedAt: number,
+): Promise<CalendarCollectionItem> => {
+	const created = await store.transaction((repos) =>
+		repos.calendarCollection.create({
+			accountConfigId: "account-config-1",
+			urlSegment,
+			displayName: urlSegment,
+			source: CalendarSource.Subscribed,
+			subscriptionUrl: `https://calendar.example/${urlSegment}/private-feedfacecafe.ics`,
+			subscriptionEnabled: true,
+		}),
+	);
+	return store.transaction((repos) =>
+		repos.calendarCollection.update(
+			created.accountConfigId,
+			created.calendarId,
+			{ subscriptionCheckedAt: checkedAt },
+		),
+	);
+};
+
+const recording =
+	(asked: string[], answer: (url: string) => Response): CalendarFeedFetcher =>
+	async (url) => {
+		asked.push(url);
+		return answer(url);
+	};
+
+describe("refreshDueCalendarSubscriptions", () => {
+	it("fetches the subscriptions that are due and skips the rest", async () => {
+		const store = new MemoryCalendarStore();
+		const due = await subscription(store, "due", NOW - INTERVAL_MS);
+		await subscription(store, "fresh", NOW - 60_000);
+		const paused = await subscription(store, "paused", 0);
+		await store.transaction((repos) =>
+			repos.calendarCollection.update(
+				paused.accountConfigId,
+				paused.calendarId,
+				{ subscriptionEnabled: false },
+			),
+		);
+		const asked: string[] = [];
+
+		const tally = await refreshDueCalendarSubscriptions(
+			{
+				calendarCollection: {
+					listEnabledSubscriptions: () =>
+						store.transaction((repos) =>
+							repos.calendarCollection.listEnabledSubscriptions(),
+						),
+				},
+				calendarUnitOfWork: store,
+				log: logged().log,
+				fetcher: recording(asked, () => new Response(FEED, { status: 200 })),
+			},
+			NOW,
+			INTERVAL_MS,
+		);
+
+		assert.deepEqual(asked, [due.subscriptionUrl]);
+		assert.deepEqual(tally, { refreshed: 1, failed: 0, notDue: 1 });
+		const objects = await store.transaction((repos) =>
+			repos.calendarObject.listByCalendar(due.calendarId),
+		);
+		assert.deepEqual(
+			objects.map((object) => object.summary),
+			["Ferry crossing"],
+		);
+	});
+
+	it("counts a feed that fails, keeps refreshing the others, and never logs an address", async () => {
+		const store = new MemoryCalendarStore();
+		const broken = await subscription(store, "broken", 0);
+		const working = await subscription(store, "working", 0);
+		const { log, lines } = logged();
+
+		const tally = await refreshDueCalendarSubscriptions(
+			{
+				calendarCollection: {
+					listEnabledSubscriptions: () =>
+						store.transaction((repos) =>
+							repos.calendarCollection.listEnabledSubscriptions(),
+						),
+				},
+				calendarUnitOfWork: store,
+				log,
+				fetcher: recording([], (url) =>
+					url === broken.subscriptionUrl
+						? new Response("gone", { status: 410 })
+						: new Response(FEED, { status: 200 }),
+				),
+			},
+			NOW,
+			INTERVAL_MS,
+		);
+
+		assert.deepEqual(tally, { refreshed: 1, failed: 1, notDue: 0 });
+		const stored = await store.transaction((repos) =>
+			repos.calendarCollection.get(broken.accountConfigId, broken.calendarId),
+		);
+		assert.equal(stored.subscriptionError, "the feed answered HTTP 410");
+		assert.equal(stored.subscriptionCheckedAt, NOW);
+		const refreshed = await store.transaction((repos) =>
+			repos.calendarCollection.get(working.accountConfigId, working.calendarId),
+		);
+		assert.equal(refreshed.subscriptionFetchedAt, NOW);
+		assert.ok(lines.some((line) => line.includes(broken.calendarId)));
+		assert.equal(
+			lines.some((line) => line.includes("private-feedfacecafe")),
+			false,
+		);
+	});
+});

package/src/scheduler/calendar-subscriptions.ts

@@ -0,0 +1,103 @@
+import type { RemitClient } from "@remit/backend/client";
+import {
+	type CalendarFeedFetcher,
+	isSubscriptionDue,
+	refreshCalendarSubscription,
+} from "@remit/calendar-service";
+import type {
+	ICalendarCollectionRepository,
+	ICalendarUnitOfWork,
+} from "@remit/data-ports";
+import type { Logger } from "@remit/logger-lambda";
+import pMap from "p-map";
+
+const REFRESH_CONCURRENCY = 4;
+
+export interface CalendarSubscriptionTally {
+	refreshed: number;
+	failed: number;
+	notDue: number;
+}
+
+export interface CalendarSubscriptionRefreshDeps {
+	calendarCollection: Pick<
+		ICalendarCollectionRepository,
+		"listEnabledSubscriptions"
+	>;
+	calendarUnitOfWork: ICalendarUnitOfWork;
+	log: Logger;
+	fetcher?: CalendarFeedFetcher;
+}
+
+/**
+ * Re-reads every enabled subscription whose last fetch is at least
+ * `intervalMs` old. A disabled subscription is never listed, which is what
+ * stops its refresh while leaving its events in place.
+ *
+ * One feed's failure is recorded on its own collection and counted here; it
+ * never stops the others. The log names the calendar and the reason, never the
+ * URL, which for a Google secret address is the whole credential.
+ */
+export const refreshDueCalendarSubscriptions = async (
+	deps: CalendarSubscriptionRefreshDeps,
+	now: number,
+	intervalMs: number,
+): Promise<CalendarSubscriptionTally> => {
+	const enabled = await deps.calendarCollection.listEnabledSubscriptions();
+	const due = enabled.filter((collection) =>
+		isSubscriptionDue(collection, now, intervalMs),
+	);
+
+	const outcomes = await pMap(
+		due,
+		(collection) =>
+			refreshCalendarSubscription(
+				deps.calendarUnitOfWork,
+				collection,
+				now,
+				deps.fetcher,
+			).then(
+				(refresh) => {
+					if (refresh.ok) return true;
+					deps.log.warn(
+						{ calendarId: collection.calendarId, reason: refresh.reason },
+						"Calendar subscription could not be refreshed",
+					);
+					return false;
+				},
+				(error: unknown) => {
+					deps.log.error(
+						{ calendarId: collection.calendarId, error },
+						"Calendar subscription refresh failed",
+					);
+					return false;
+				},
+			),
+		{ concurrency: REFRESH_CONCURRENCY },
+	);
+
+	return {
+		refreshed: outcomes.filter(Boolean).length,
+		failed: outcomes.filter((ok) => !ok).length,
+		notDue: enabled.length - due.length,
+	};
+};
+
+export const buildCalendarSubscriptionRefresh =
+	(
+		client: Pick<RemitClient, "calendarCollection" | "calendarUnitOfWork">,
+		log: Logger,
+	): ((
+		now: number,
+		intervalMs: number,
+	) => Promise<CalendarSubscriptionTally>) =>
+	(now, intervalMs) =>
+		refreshDueCalendarSubscriptions(
+			{
+				calendarCollection: client.calendarCollection,
+				calendarUnitOfWork: client.calendarUnitOfWork,
+				log,
+			},
+			now,
+			intervalMs,
+		);

package/src/scheduler/handler.ts

@@ -4,6 +4,7 @@ import { createQueueProducer } from "@remit/sqs-client/producer";
 import type { ScheduledHandler } from "aws-lambda";
 import { env } from "expect-env";
 import { buildAttachmentSweep } from "./attachment-sweep.js";
+import { buildCalendarSubscriptionRefresh } from "./calendar-subscriptions.js";
 import { getOfflineIntervalMs, getTickIntervalMs } from "./config.js";
 import { runSchedulerTick } from "./run-tick.js";
 
@@ -45,5 +46,6 @@ export const handler: ScheduledHandler = withTelemetry(async (event) => {
 		// to block storage, so nothing refuses an upload against a draft that is
 		// already gone and this sweep is the only thing that collects it.
 		sweepAttachments: buildAttachmentSweep(client, log),
+		refreshCalendarSubscriptions: buildCalendarSubscriptionRefresh(client, log),
 	});
 });

package/src/scheduler/loop.test.ts

@@ -9,6 +9,8 @@ const RESULT: SchedulerTickResult = {
 	skipped: 0,
 	swept: 0,
 	sweepFailed: 0,
+	subscriptionsRefreshed: 0,
+	subscriptionsFailed: 0,
 };
 
 // Only the loop's own two calls matter here; the tick is a stub, so what it is

package/src/scheduler/run-tick.test.ts

@@ -413,3 +413,66 @@ describe("runSchedulerTick and the account's synced services", () => {
 		assert.equal(sent.length, 0);
 	});
 });
+
+describe("the calendar subscription refresh inside a tick", () => {
+	const oneAccount = () => [
+		{
+			items: [baseAccount({ accountId: "acct_1", lastSyncAt: 0 })],
+			cursor: null,
+		} as unknown as AccountSchedulerPage,
+	];
+
+	it("runs once per tick, against the offline interval", async () => {
+		const { sqsClient } = fakeSqsClient();
+		const calls: [number, number][] = [];
+
+		const result = await runSchedulerTick({
+			accountService: fakeAccountService(oneAccount()),
+			sqsClient,
+			queueUrl: "https://queue.test/mailboxes",
+			log: createNoopLogger(),
+			tickIntervalMs: TICK_INTERVAL_MS,
+			offlineIntervalMs: OFFLINE_INTERVAL_MS,
+			now: NOW,
+			refreshCalendarSubscriptions: async (now, intervalMs) => {
+				calls.push([now, intervalMs]);
+				return { refreshed: 2, failed: 1, notDue: 4 };
+			},
+		});
+
+		assert.deepEqual(calls, [[NOW, OFFLINE_INTERVAL_MS]]);
+		assert.equal(result.subscriptionsRefreshed, 2);
+		assert.equal(result.subscriptionsFailed, 1);
+		assert.equal(result.enqueued, 1);
+	});
+
+	it("keeps the tick alive when the refresh throws", async () => {
+		const { sqsClient, sent } = fakeSqsClient();
+		const { log, calls } = createCapturingLogger();
+
+		const result = await runSchedulerTick({
+			accountService: fakeAccountService(oneAccount()),
+			sqsClient,
+			queueUrl: "https://queue.test/mailboxes",
+			log,
+			tickIntervalMs: TICK_INTERVAL_MS,
+			offlineIntervalMs: OFFLINE_INTERVAL_MS,
+			now: NOW,
+			refreshCalendarSubscriptions: async () => {
+				throw new Error("database is locked");
+			},
+		});
+
+		assert.equal(sent.length, 1);
+		assert.equal(result.subscriptionsRefreshed, 0);
+		assert.ok(
+			calls.some(
+				(entry) =>
+					entry.level === "error" &&
+					entry.args.some((arg) =>
+						String(arg).includes("Calendar subscription refresh failed"),
+					),
+			),
+		);
+	});
+});

package/src/scheduler/run-tick.ts

@@ -13,6 +13,7 @@ import {
 	isMailSyncDisabled,
 	isUnsyncableHost,
 } from "../account-check.js";
+import type { CalendarSubscriptionTally } from "./calendar-subscriptions.js";
 import {
 	SCHEDULER_ENQUEUE_CONCURRENCY,
 	SCHEDULER_PAGE_SIZE,
@@ -43,6 +44,15 @@ export interface RunSchedulerTickDeps {
 	 * both real entry points pass it.
 	 */
 	sweepAttachments?: (account: AccountItem) => Promise<void>;
+	/**
+	 * Re-read the calendar subscriptions whose last fetch is older than
+	 * `offlineIntervalMs` — the same knob that says how stale mail may get.
+	 * Optional only so a test can leave it out; both real entry points pass it.
+	 */
+	refreshCalendarSubscriptions?: (
+		now: number,
+		intervalMs: number,
+	) => Promise<CalendarSubscriptionTally>;
 }
 
 export interface SchedulerTickResult {
@@ -51,6 +61,8 @@ export interface SchedulerTickResult {
 	skipped: number;
 	swept: number;
 	sweepFailed: number;
+	subscriptionsRefreshed: number;
+	subscriptionsFailed: number;
 }
 
 /**
@@ -193,10 +205,43 @@ export const runSchedulerTick = async (
 		cursor = page.cursor ?? undefined;
 	} while (cursor);
 
+	let subscriptionsRefreshed = 0;
+	let subscriptionsFailed = 0;
+	if (deps.refreshCalendarSubscriptions) {
+		// After every page is enqueued, for the same reason the sweep runs after
+		// the enqueue: a slow feed must never cost an account its mail. Contained
+		// like the sweep, so an unreadable store costs a tick's refresh and not
+		// the scheduler.
+		const tally = await deps
+			.refreshCalendarSubscriptions(now, offlineIntervalMs)
+			.catch((error: unknown) => {
+				log.error({ error }, "Calendar subscription refresh failed");
+				return { refreshed: 0, failed: 0, notDue: 0 };
+			});
+		subscriptionsRefreshed = tally.refreshed;
+		subscriptionsFailed = tally.failed;
+	}
+
 	log.info(
-		{ scanned, enqueued, skipped, swept, sweepFailed },
+		{
+			scanned,
+			enqueued,
+			skipped,
+			swept,
+			sweepFailed,
+			subscriptionsRefreshed,
+			subscriptionsFailed,
+		},
 		"Scheduled-sync tick complete",
 	);
 
-	return { scanned, enqueued, skipped, swept, sweepFailed };
+	return {
+		scanned,
+		enqueued,
+		skipped,
+		swept,
+		sweepFailed,
+		subscriptionsRefreshed,
+		subscriptionsFailed,
+	};
 };

package/src/scheduler/runner.ts

@@ -7,6 +7,7 @@ import { createQueueProducer } from "@remit/sqs-client/producer";
 
 import { env } from "expect-env";
 import { buildAttachmentSweep } from "./attachment-sweep.js";
+import { buildCalendarSubscriptionRefresh } from "./calendar-subscriptions.js";
 import { getOfflineIntervalMs, getTickIntervalMs } from "./config.js";
 import { runSchedulerLoop } from "./loop.js";
 import { runSchedulerTick } from "./run-tick.js";
@@ -80,6 +81,10 @@ const runLoop = async (): Promise<void> => {
 			tickIntervalMs,
 			offlineIntervalMs,
 			sweepAttachments: buildAttachmentSweep(client, log),
+			refreshCalendarSubscriptions: buildCalendarSubscriptionRefresh(
+				client,
+				log,
+			),
 		},
 		heartbeat: createHeartbeat("tick"),
 		onHeartbeatError: onBeatError,