@remit/imap-worker 0.0.85 → 0.0.87
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/package.json +10 -2
- package/src/account-check.ts +1 -1
- package/src/body-sync-gate.test.ts +1 -1
- package/src/body-sync-gate.ts +1 -1
- package/src/e2e-processor-shim.ts +6 -21
- package/src/emit.test.ts +32 -4
- package/src/emit.ts +58 -23
- package/src/failure-kind.ts +1 -1
- package/src/handlers/append-sent-message.sqlite.test.ts +176 -0
- package/src/handlers/append-sent-message.ts +3 -24
- package/src/handlers/delete-account-objects.test.ts +1 -1
- package/src/handlers/delete-account-objects.ts +1 -1
- package/src/handlers/empty-trash.sqlite.test.ts +209 -0
- package/src/handlers/empty-trash.ts +3 -24
- package/src/handlers/flag-push.ts +11 -18
- package/src/handlers/mailbox-management.sqlite.test.ts +265 -0
- package/src/handlers/mailbox-management.ts +12 -48
- package/src/handlers/message-copy.sqlite.test.ts +298 -0
- package/src/handlers/message-copy.ts +31 -49
- package/src/handlers/message-delete-terminal.ts +12 -10
- package/src/handlers/message-delete.sqlite.test.ts +291 -0
- package/src/handlers/message-delete.ts +26 -32
- package/src/handlers/message-move-terminal.ts +12 -10
- package/src/handlers/message-move.sqlite.test.ts +271 -0
- package/src/handlers/message-move.test.ts +76 -863
- package/src/handlers/message-move.ts +49 -39
- package/src/handlers/placement-move-push.test.ts +96 -329
- package/src/handlers/placement-move-push.ts +75 -29
- package/src/handlers/placement-push.sqlite.test.ts +298 -0
- package/src/handlers/probe-message-id.sqlite.test.ts +197 -0
- package/src/handlers/probe-message-id.ts +15 -0
- package/src/handlers/sync-mailboxes.test.ts +0 -1
- package/src/handlers/sync-mailboxes.ts +1 -1
- package/src/handlers/sync-message-body.sqlite.test.ts +314 -0
- package/src/handlers/sync-message-body.test.ts +3 -299
- package/src/handlers/sync-message-body.ts +21 -11
- package/src/handlers/sync-messages.sqlite.test.ts +183 -0
- package/src/handlers/sync-messages.ts +35 -34
- package/src/index.ts +5 -3
- package/src/mail-sync-selection.sqlite.test.ts +20 -18
- package/src/pending-sync.sqlite.test.ts +167 -0
- package/src/poller.ts +5 -4
- package/src/processor.test.ts +1 -1
- package/src/processor.ts +1 -1
- package/src/scheduler/attachment-sweep.sqlite.test.ts +74 -0
- package/src/scheduler/attachment-sweep.ts +1 -1
- package/src/scheduler/{calendar-subscriptions.test.ts → calendar-subscriptions.sqlite.test.ts} +43 -55
- package/src/scheduler/calendar-subscriptions.ts +6 -6
- package/src/scheduler/half-deleted-accounts.sqlite.test.ts +203 -0
- package/src/scheduler/handler.ts +12 -1
- package/src/scheduler/loop.test.ts +2 -0
- package/src/scheduler/run-tick.sqlite.test.ts +192 -0
- package/src/scheduler/run-tick.ts +39 -1
- package/src/scheduler/runner.ts +21 -1
- package/src/scheduler/stranded-placement-redrive.ts +265 -0
- package/src/stranded-placement.sqlite.test.ts +795 -0
- package/src/test-helpers/test-world.ts +398 -0
- package/src/thread-join.sqlite.test.ts +115 -0
- package/src/with-oauth-lifecycle-deps.sqlite.test.ts +138 -14
- package/src/with-oauth-lifecycle.ts +1 -1
- package/src/handlers/append-sent-message.test.ts +0 -529
- package/src/handlers/empty-trash.test.ts +0 -728
- package/src/handlers/flag-push-placement-race.test.ts +0 -385
- package/src/handlers/flag-push.test.ts +0 -325
- package/src/handlers/mailbox-management.test.ts +0 -850
- package/src/handlers/message-copy.test.ts +0 -654
- package/src/handlers/message-delete.test.ts +0 -1869
- package/src/handlers/message-move-terminal.test.ts +0 -412
- package/src/handlers/sync-messages.test.ts +0 -567
- package/src/scheduler/attachment-sweep.test.ts +0 -169
- package/src/scheduler/run-tick.test.ts +0 -478
- package/src/with-oauth-lifecycle.test.ts +0 -318
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { afterEach, before, beforeEach, describe, it } from "node:test";
|
|
3
|
+
import { SendMessageCommand, SQSClient } from "@aws-sdk/client-sqs";
|
|
4
|
+
import {
|
|
5
|
+
_resetForTest,
|
|
6
|
+
type RemitClient,
|
|
7
|
+
setClient,
|
|
8
|
+
} from "@remit/backend/client";
|
|
9
|
+
import { MailboxLockRepo } from "@remit/drizzle-service";
|
|
10
|
+
import { createShippedSqliteDb } from "@remit/drizzle-service/test-sqlite";
|
|
11
|
+
import { mockClient } from "aws-sdk-client-mock";
|
|
12
|
+
import type * as Emit from "./emit.js";
|
|
13
|
+
import type { SyncMessagesEvent } from "./events.js";
|
|
14
|
+
import { emitMailboxResync, emitMoveResync } from "./handlers/message-move.js";
|
|
15
|
+
import { emitSyncMessagesEvents } from "./handlers/sync-mailboxes.js";
|
|
16
|
+
|
|
17
|
+
const ACCOUNT = "account-1366";
|
|
18
|
+
const FOLDERS = [
|
|
19
|
+
{ mailboxId: "mbx-inbox", fullPath: "INBOX" },
|
|
20
|
+
{ mailboxId: "mbx-sent", fullPath: "Sent" },
|
|
21
|
+
{ mailboxId: "mbx-archive", fullPath: "Archive" },
|
|
22
|
+
];
|
|
23
|
+
|
|
24
|
+
const sqsMock = mockClient(SQSClient);
|
|
25
|
+
|
|
26
|
+
let emitEvent: typeof Emit.emitEvent;
|
|
27
|
+
let SYNC_MESSAGES_PENDING: typeof Emit.SYNC_MESSAGES_PENDING;
|
|
28
|
+
let locks: MailboxLockRepo;
|
|
29
|
+
let closeStore: () => void;
|
|
30
|
+
|
|
31
|
+
type Queued = { mailboxId: string; eventId: string };
|
|
32
|
+
|
|
33
|
+
const queued = (): Queued[] =>
|
|
34
|
+
sqsMock.commandCalls(SendMessageCommand).map((call) => {
|
|
35
|
+
const body = JSON.parse(call.args[0].input.MessageBody ?? "{}") as Queued;
|
|
36
|
+
return { mailboxId: body.mailboxId, eventId: body.eventId };
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
const inboxSync = (): Omit<SyncMessagesEvent, "eventId" | "timestamp"> => ({
|
|
40
|
+
type: "SYNC_MESSAGES",
|
|
41
|
+
accountId: ACCOUNT,
|
|
42
|
+
mailboxId: "mbx-inbox",
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
const fanOut = (): Promise<void> =>
|
|
46
|
+
emitSyncMessagesEvents(ACCOUNT, FOLDERS, emitEvent);
|
|
47
|
+
|
|
48
|
+
before(async () => {
|
|
49
|
+
const fifo = (name: string) =>
|
|
50
|
+
`https://sqs.eu-west-1.amazonaws.com/0/test-${name}.fifo`;
|
|
51
|
+
process.env.SQS_QUEUE_URL_MAILBOXES = fifo("mailboxes");
|
|
52
|
+
process.env.SQS_QUEUE_URL_MESSAGES = fifo("messages");
|
|
53
|
+
process.env.SQS_QUEUE_URL_FLAGS = fifo("flags");
|
|
54
|
+
({ emitEvent, SYNC_MESSAGES_PENDING } = await import("./emit.js"));
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
beforeEach(() => {
|
|
58
|
+
sqsMock.reset();
|
|
59
|
+
sqsMock.on(SendMessageCommand).resolves({});
|
|
60
|
+
const store = createShippedSqliteDb();
|
|
61
|
+
closeStore = store.close;
|
|
62
|
+
locks = new MailboxLockRepo(store.db);
|
|
63
|
+
setClient({ mailboxLock: locks } as unknown as RemitClient);
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
afterEach(() => {
|
|
67
|
+
_resetForTest();
|
|
68
|
+
closeStore();
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
describe("SYNC_MESSAGES under explicit refreshes (#1366)", () => {
|
|
72
|
+
it("queues one event per folder however many rounds arrive while none has started", async () => {
|
|
73
|
+
for (let round = 0; round < 25; round += 1) await fanOut();
|
|
74
|
+
|
|
75
|
+
assert.deepEqual(
|
|
76
|
+
queued()
|
|
77
|
+
.map((event) => event.mailboxId)
|
|
78
|
+
.sort(),
|
|
79
|
+
["mbx-archive", "mbx-inbox", "mbx-sent"],
|
|
80
|
+
);
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
it("queues a folder again once its pending sync has started", async () => {
|
|
84
|
+
await fanOut();
|
|
85
|
+
const inbox = queued().find((event) => event.mailboxId === "mbx-inbox");
|
|
86
|
+
if (!inbox) throw new Error("INBOX was not queued");
|
|
87
|
+
|
|
88
|
+
await locks.releaseLock(
|
|
89
|
+
ACCOUNT,
|
|
90
|
+
inbox.mailboxId,
|
|
91
|
+
SYNC_MESSAGES_PENDING,
|
|
92
|
+
inbox.eventId,
|
|
93
|
+
);
|
|
94
|
+
await fanOut();
|
|
95
|
+
|
|
96
|
+
assert.equal(queued().length, FOLDERS.length + 1);
|
|
97
|
+
assert.equal(queued().at(-1)?.mailboxId, "mbx-inbox");
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
it("keeps a folder's pending marker when a stale event starts", async () => {
|
|
101
|
+
await fanOut();
|
|
102
|
+
await locks.releaseLock(
|
|
103
|
+
ACCOUNT,
|
|
104
|
+
"mbx-inbox",
|
|
105
|
+
SYNC_MESSAGES_PENDING,
|
|
106
|
+
"an-event-from-before-the-marker",
|
|
107
|
+
);
|
|
108
|
+
await fanOut();
|
|
109
|
+
|
|
110
|
+
assert.equal(queued().length, FOLDERS.length);
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
it("takes the same marker for a move's resync and a delete's resync", async () => {
|
|
114
|
+
await emitMoveResync(emitEvent, {
|
|
115
|
+
accountId: ACCOUNT,
|
|
116
|
+
sourceMailboxId: "mbx-inbox",
|
|
117
|
+
destinationMailboxId: "mbx-sent",
|
|
118
|
+
});
|
|
119
|
+
await emitMailboxResync(emitEvent, {
|
|
120
|
+
accountId: ACCOUNT,
|
|
121
|
+
mailboxId: "mbx-inbox",
|
|
122
|
+
});
|
|
123
|
+
await fanOut();
|
|
124
|
+
|
|
125
|
+
assert.equal(queued().length, FOLDERS.length);
|
|
126
|
+
});
|
|
127
|
+
|
|
128
|
+
it("takes over a marker whose event was lost once it has expired", async () => {
|
|
129
|
+
await locks.tryAcquireLock(
|
|
130
|
+
"mbx-inbox",
|
|
131
|
+
"SYNC_MESSAGES_PENDING",
|
|
132
|
+
ACCOUNT,
|
|
133
|
+
"lost-event",
|
|
134
|
+
-1,
|
|
135
|
+
);
|
|
136
|
+
|
|
137
|
+
await emitEvent(inboxSync());
|
|
138
|
+
|
|
139
|
+
assert.equal(queued().length, 1);
|
|
140
|
+
});
|
|
141
|
+
|
|
142
|
+
it("releases the marker when the send fails, so the next round is not swallowed", async () => {
|
|
143
|
+
sqsMock
|
|
144
|
+
.on(SendMessageCommand)
|
|
145
|
+
.rejectsOnce(new Error("queue unreachable"))
|
|
146
|
+
.resolves({});
|
|
147
|
+
|
|
148
|
+
await assert.rejects(emitEvent(inboxSync()), /queue unreachable/);
|
|
149
|
+
await emitEvent(inboxSync());
|
|
150
|
+
|
|
151
|
+
assert.equal(queued().length, 2);
|
|
152
|
+
});
|
|
153
|
+
|
|
154
|
+
it("fails the operation when the marker store fails", async () => {
|
|
155
|
+
setClient({
|
|
156
|
+
mailboxLock: {
|
|
157
|
+
get: async () => null,
|
|
158
|
+
tryAcquireLock: async () => {
|
|
159
|
+
throw new Error("marker store down");
|
|
160
|
+
},
|
|
161
|
+
},
|
|
162
|
+
} as unknown as RemitClient);
|
|
163
|
+
|
|
164
|
+
await assert.rejects(emitEvent(inboxSync()), /marker store down/);
|
|
165
|
+
assert.equal(sqsMock.commandCalls(SendMessageCommand).length, 0);
|
|
166
|
+
});
|
|
167
|
+
});
|
package/src/poller.ts
CHANGED
|
@@ -1,14 +1,15 @@
|
|
|
1
|
-
import { createLogger
|
|
1
|
+
import { createLogger } from "@remit/logger";
|
|
2
|
+
import { startMetricsServer } from "@remit/logger-lambda/metrics";
|
|
3
|
+
import { installNodeLogger } from "@remit/logger-lambda/node";
|
|
2
4
|
import { runQueuePoller } from "@remit/sqs-client/poller";
|
|
3
5
|
import { env } from "expect-env";
|
|
4
6
|
import { handler } from "./index.js";
|
|
5
7
|
|
|
8
|
+
installNodeLogger();
|
|
9
|
+
|
|
6
10
|
/**
|
|
7
11
|
* Production queue poller — the deployed form of `e2e-processor-shim.ts`.
|
|
8
12
|
* Polls every imap-worker queue and invokes the production Lambda handler.
|
|
9
|
-
* The search-index queue is NOT polled here: it is its own image/deployment
|
|
10
|
-
* (`remit-search-index-worker`), unlike the e2e shim which piggybacks it
|
|
11
|
-
* onto this same process for test convenience.
|
|
12
13
|
*/
|
|
13
14
|
const log = createLogger();
|
|
14
15
|
|
package/src/processor.test.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import assert from "node:assert/strict";
|
|
2
2
|
import { describe, it } from "node:test";
|
|
3
|
-
import type { Logger } from "@remit/logger
|
|
3
|
+
import type { Logger } from "@remit/logger";
|
|
4
4
|
import type { ImapWorkerStopEvent } from "./events.js";
|
|
5
5
|
import { processEvent } from "./processor.js";
|
|
6
6
|
|
package/src/processor.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { Logger } from "@remit/logger
|
|
1
|
+
import type { Logger } from "@remit/logger";
|
|
2
2
|
import type { WorkerEvent } from "./events.js";
|
|
3
3
|
import { handleAppendSentMessage } from "./handlers/append-sent-message.js";
|
|
4
4
|
import { handleDeleteAccountObjects } from "./handlers/delete-account-objects.js";
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { randomUUID } from "node:crypto";
|
|
3
|
+
import { after, before, describe, it } from "node:test";
|
|
4
|
+
import { openStore, type World } from "../test-helpers/test-world.js";
|
|
5
|
+
import { buildAttachmentSweep } from "./attachment-sweep.js";
|
|
6
|
+
|
|
7
|
+
let store: Awaited<ReturnType<typeof openStore>>;
|
|
8
|
+
|
|
9
|
+
before(async () => {
|
|
10
|
+
store = await openStore();
|
|
11
|
+
});
|
|
12
|
+
|
|
13
|
+
after(() => store.close());
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* A draft holding one uploaded file, and one object under the same draft that
|
|
17
|
+
* no attachment row names — what a browser leaves behind when it uploads and
|
|
18
|
+
* the reservation lapses before it is completed.
|
|
19
|
+
*/
|
|
20
|
+
const arrange = async (w: World) => {
|
|
21
|
+
const { outbox, attachmentId } = await w.draft(Buffer.from("q1,q2\n1,2\n"));
|
|
22
|
+
await w.client.storage.storeOutboxAttachment({
|
|
23
|
+
accountConfigId: w.account.accountConfigId,
|
|
24
|
+
accountId: w.account.accountId,
|
|
25
|
+
outboxMessageId: outbox.outboxMessageId,
|
|
26
|
+
outboxAttachmentId: `orphan-${randomUUID()}`,
|
|
27
|
+
content: Buffer.from("left behind"),
|
|
28
|
+
});
|
|
29
|
+
return { outboxMessageId: outbox.outboxMessageId, attachmentId };
|
|
30
|
+
};
|
|
31
|
+
|
|
32
|
+
const storedIds = async (w: World, outboxMessageId: string) =>
|
|
33
|
+
(
|
|
34
|
+
await w.client.storage.listOutboxAttachments(
|
|
35
|
+
w.account.accountConfigId,
|
|
36
|
+
w.account.accountId,
|
|
37
|
+
outboxMessageId,
|
|
38
|
+
)
|
|
39
|
+
).map((item) => item.outboxAttachmentId);
|
|
40
|
+
|
|
41
|
+
describe("the outbox attachment sweep", () => {
|
|
42
|
+
it("removes the objects no attachment row names and keeps the ones that are", async () => {
|
|
43
|
+
const w = await store.world();
|
|
44
|
+
const { outboxMessageId, attachmentId } = await arrange(w);
|
|
45
|
+
|
|
46
|
+
await buildAttachmentSweep(w.client, w.log)(w.account);
|
|
47
|
+
|
|
48
|
+
assert.deepEqual(await storedIds(w, outboxMessageId), [attachmentId]);
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
it("runs for every account on the hosted tick, due a sync or not", async () => {
|
|
52
|
+
const first = await store.world();
|
|
53
|
+
const firstDraft = await arrange(first);
|
|
54
|
+
await first.client.account.update(first.account.accountId, {
|
|
55
|
+
lastSyncAt: Date.now(),
|
|
56
|
+
});
|
|
57
|
+
const second = await store.world();
|
|
58
|
+
const secondDraft = await arrange(second);
|
|
59
|
+
|
|
60
|
+
const { handler } = await import("./handler.js");
|
|
61
|
+
await handler(
|
|
62
|
+
{ time: new Date().toISOString() } as never,
|
|
63
|
+
{} as never,
|
|
64
|
+
() => {},
|
|
65
|
+
);
|
|
66
|
+
|
|
67
|
+
assert.deepEqual(await storedIds(first, firstDraft.outboxMessageId), [
|
|
68
|
+
firstDraft.attachmentId,
|
|
69
|
+
]);
|
|
70
|
+
assert.deepEqual(await storedIds(second, secondDraft.outboxMessageId), [
|
|
71
|
+
secondDraft.attachmentId,
|
|
72
|
+
]);
|
|
73
|
+
});
|
|
74
|
+
});
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { RemitClient } from "@remit/backend/client";
|
|
2
2
|
import type { AccountItem } from "@remit/data-ports";
|
|
3
|
-
import type { Logger } from "@remit/logger
|
|
3
|
+
import type { Logger } from "@remit/logger";
|
|
4
4
|
import { sweepAbandonedOutboxAttachments } from "@remit/storage-service";
|
|
5
5
|
/**
|
|
6
6
|
* Collect attachment bytes the database does not know about. Both scheduler
|
package/src/scheduler/{calendar-subscriptions.test.ts → calendar-subscriptions.sqlite.test.ts}
RENAMED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import assert from "node:assert/strict";
|
|
2
2
|
import { describe, it } from "node:test";
|
|
3
3
|
import type { CalendarFeedFetcher } from "@remit/calendar-service";
|
|
4
|
-
import { MemoryCalendarStore } from "@remit/calendar-service/memory-store";
|
|
5
4
|
import type { CalendarCollectionItem } from "@remit/data-ports";
|
|
6
5
|
import { CalendarSource } from "@remit/domain-enums";
|
|
7
|
-
import type { Logger } from "@remit/logger
|
|
6
|
+
import type { Logger } from "@remit/logger";
|
|
7
|
+
import { openStore } from "../test-helpers/test-world.js";
|
|
8
8
|
import { refreshDueCalendarSubscriptions } from "./calendar-subscriptions.js";
|
|
9
9
|
|
|
10
10
|
const INTERVAL_MS = 15 * 60 * 1000;
|
|
@@ -41,27 +41,25 @@ const logged = (): { log: Logger; lines: string[] } => {
|
|
|
41
41
|
return { log, lines };
|
|
42
42
|
};
|
|
43
43
|
|
|
44
|
+
type Client = Awaited<ReturnType<typeof openStore>>["client"];
|
|
45
|
+
|
|
44
46
|
const subscription = async (
|
|
45
|
-
|
|
47
|
+
client: Client,
|
|
46
48
|
urlSegment: string,
|
|
47
49
|
checkedAt: number,
|
|
48
50
|
): Promise<CalendarCollectionItem> => {
|
|
49
|
-
const created = await
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
created.accountConfigId,
|
|
62
|
-
created.calendarId,
|
|
63
|
-
{ subscriptionCheckedAt: checkedAt },
|
|
64
|
-
),
|
|
51
|
+
const created = await client.calendarCollection.create({
|
|
52
|
+
accountConfigId: "account-config-1",
|
|
53
|
+
urlSegment,
|
|
54
|
+
displayName: urlSegment,
|
|
55
|
+
source: CalendarSource.Subscribed,
|
|
56
|
+
subscriptionUrl: `https://calendar.example/${urlSegment}/private-feedfacecafe.ics`,
|
|
57
|
+
subscriptionEnabled: true,
|
|
58
|
+
});
|
|
59
|
+
return client.calendarCollection.update(
|
|
60
|
+
created.accountConfigId,
|
|
61
|
+
created.calendarId,
|
|
62
|
+
{ subscriptionCheckedAt: checkedAt },
|
|
65
63
|
);
|
|
66
64
|
};
|
|
67
65
|
|
|
@@ -73,29 +71,23 @@ const recording =
|
|
|
73
71
|
};
|
|
74
72
|
|
|
75
73
|
describe("refreshDueCalendarSubscriptions", () => {
|
|
76
|
-
it("fetches the subscriptions that are due and skips the rest", async () => {
|
|
77
|
-
const
|
|
78
|
-
|
|
79
|
-
await subscription(
|
|
80
|
-
|
|
81
|
-
await
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
),
|
|
74
|
+
it("fetches the subscriptions that are due and skips the rest", async (t) => {
|
|
75
|
+
const { client, close } = await openStore();
|
|
76
|
+
t.after(close);
|
|
77
|
+
const due = await subscription(client, "due", NOW - INTERVAL_MS);
|
|
78
|
+
await subscription(client, "fresh", NOW - 60_000);
|
|
79
|
+
const paused = await subscription(client, "paused", 0);
|
|
80
|
+
await client.calendarCollection.update(
|
|
81
|
+
paused.accountConfigId,
|
|
82
|
+
paused.calendarId,
|
|
83
|
+
{ subscriptionEnabled: false },
|
|
87
84
|
);
|
|
88
85
|
const asked: string[] = [];
|
|
89
86
|
|
|
90
87
|
const tally = await refreshDueCalendarSubscriptions(
|
|
91
88
|
{
|
|
92
|
-
calendarCollection:
|
|
93
|
-
|
|
94
|
-
store.transaction((repos) =>
|
|
95
|
-
repos.calendarCollection.listEnabledSubscriptions(),
|
|
96
|
-
),
|
|
97
|
-
},
|
|
98
|
-
calendarUnitOfWork: store,
|
|
89
|
+
calendarCollection: client.calendarCollection,
|
|
90
|
+
calendarStore: client.calendarStore,
|
|
99
91
|
log: logged().log,
|
|
100
92
|
fetcher: recording(asked, () => new Response(FEED, { status: 200 })),
|
|
101
93
|
},
|
|
@@ -105,30 +97,24 @@ describe("refreshDueCalendarSubscriptions", () => {
|
|
|
105
97
|
|
|
106
98
|
assert.deepEqual(asked, [due.subscriptionUrl]);
|
|
107
99
|
assert.deepEqual(tally, { refreshed: 1, failed: 0, notDue: 1 });
|
|
108
|
-
const objects = await
|
|
109
|
-
repos.calendarObject.listByCalendar(due.calendarId),
|
|
110
|
-
);
|
|
100
|
+
const objects = await client.calendarObject.listByCalendar(due.calendarId);
|
|
111
101
|
assert.deepEqual(
|
|
112
102
|
objects.map((object) => object.summary),
|
|
113
103
|
["Ferry crossing"],
|
|
114
104
|
);
|
|
115
105
|
});
|
|
116
106
|
|
|
117
|
-
it("counts a feed that fails, keeps refreshing the others, and never logs an address", async () => {
|
|
118
|
-
const
|
|
119
|
-
|
|
120
|
-
const
|
|
107
|
+
it("counts a feed that fails, keeps refreshing the others, and never logs an address", async (t) => {
|
|
108
|
+
const { client, close } = await openStore();
|
|
109
|
+
t.after(close);
|
|
110
|
+
const broken = await subscription(client, "broken", 0);
|
|
111
|
+
const working = await subscription(client, "working", 0);
|
|
121
112
|
const { log, lines } = logged();
|
|
122
113
|
|
|
123
114
|
const tally = await refreshDueCalendarSubscriptions(
|
|
124
115
|
{
|
|
125
|
-
calendarCollection:
|
|
126
|
-
|
|
127
|
-
store.transaction((repos) =>
|
|
128
|
-
repos.calendarCollection.listEnabledSubscriptions(),
|
|
129
|
-
),
|
|
130
|
-
},
|
|
131
|
-
calendarUnitOfWork: store,
|
|
116
|
+
calendarCollection: client.calendarCollection,
|
|
117
|
+
calendarStore: client.calendarStore,
|
|
132
118
|
log,
|
|
133
119
|
fetcher: recording([], (url) =>
|
|
134
120
|
url === broken.subscriptionUrl
|
|
@@ -141,13 +127,15 @@ describe("refreshDueCalendarSubscriptions", () => {
|
|
|
141
127
|
);
|
|
142
128
|
|
|
143
129
|
assert.deepEqual(tally, { refreshed: 1, failed: 1, notDue: 0 });
|
|
144
|
-
const stored = await
|
|
145
|
-
|
|
130
|
+
const stored = await client.calendarCollection.get(
|
|
131
|
+
broken.accountConfigId,
|
|
132
|
+
broken.calendarId,
|
|
146
133
|
);
|
|
147
134
|
assert.equal(stored.subscriptionError, "the feed answered HTTP 410");
|
|
148
135
|
assert.equal(stored.subscriptionCheckedAt, NOW);
|
|
149
|
-
const refreshed = await
|
|
150
|
-
|
|
136
|
+
const refreshed = await client.calendarCollection.get(
|
|
137
|
+
working.accountConfigId,
|
|
138
|
+
working.calendarId,
|
|
151
139
|
);
|
|
152
140
|
assert.equal(refreshed.subscriptionFetchedAt, NOW);
|
|
153
141
|
assert.ok(lines.some((line) => line.includes(broken.calendarId)));
|
|
@@ -6,9 +6,9 @@ import {
|
|
|
6
6
|
} from "@remit/calendar-service";
|
|
7
7
|
import type {
|
|
8
8
|
ICalendarCollectionRepository,
|
|
9
|
-
|
|
9
|
+
ICalendarStore,
|
|
10
10
|
} from "@remit/data-ports";
|
|
11
|
-
import type { Logger } from "@remit/logger
|
|
11
|
+
import type { Logger } from "@remit/logger";
|
|
12
12
|
import pMap from "p-map";
|
|
13
13
|
|
|
14
14
|
const REFRESH_CONCURRENCY = 4;
|
|
@@ -24,7 +24,7 @@ export interface CalendarSubscriptionRefreshDeps {
|
|
|
24
24
|
ICalendarCollectionRepository,
|
|
25
25
|
"listEnabledSubscriptions"
|
|
26
26
|
>;
|
|
27
|
-
|
|
27
|
+
calendarStore: ICalendarStore;
|
|
28
28
|
log: Logger;
|
|
29
29
|
fetcher?: CalendarFeedFetcher;
|
|
30
30
|
}
|
|
@@ -52,7 +52,7 @@ export const refreshDueCalendarSubscriptions = async (
|
|
|
52
52
|
due,
|
|
53
53
|
(collection) =>
|
|
54
54
|
refreshCalendarSubscription(
|
|
55
|
-
deps.
|
|
55
|
+
deps.calendarStore,
|
|
56
56
|
collection,
|
|
57
57
|
now,
|
|
58
58
|
deps.fetcher,
|
|
@@ -85,7 +85,7 @@ export const refreshDueCalendarSubscriptions = async (
|
|
|
85
85
|
|
|
86
86
|
export const buildCalendarSubscriptionRefresh =
|
|
87
87
|
(
|
|
88
|
-
client: Pick<RemitClient, "calendarCollection" | "
|
|
88
|
+
client: Pick<RemitClient, "calendarCollection" | "calendarStore">,
|
|
89
89
|
log: Logger,
|
|
90
90
|
): ((
|
|
91
91
|
now: number,
|
|
@@ -95,7 +95,7 @@ export const buildCalendarSubscriptionRefresh =
|
|
|
95
95
|
refreshDueCalendarSubscriptions(
|
|
96
96
|
{
|
|
97
97
|
calendarCollection: client.calendarCollection,
|
|
98
|
-
|
|
98
|
+
calendarStore: client.calendarStore,
|
|
99
99
|
log,
|
|
100
100
|
},
|
|
101
101
|
now,
|
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { randomBytes } from "node:crypto";
|
|
3
|
+
import { after, before, beforeEach, describe, it } from "node:test";
|
|
4
|
+
import type { SendMessageCommand, SQSClient } from "@aws-sdk/client-sqs";
|
|
5
|
+
import { finishHalfDeletedAccounts } from "@remit/backend/account-deletion";
|
|
6
|
+
import type { RemitClient } from "@remit/backend/client";
|
|
7
|
+
import { importConfig } from "@remit/config-transfer";
|
|
8
|
+
import type { AccountItem } from "@remit/data-ports";
|
|
9
|
+
import { AccountAuthType, ConnectionState } from "@remit/domain-enums";
|
|
10
|
+
import { noopLogger } from "@remit/logger";
|
|
11
|
+
import { openStore } from "../test-helpers/test-world.js";
|
|
12
|
+
|
|
13
|
+
const FANOUT_URL = "http://queue:9324/000000000000/remit-account-fanout";
|
|
14
|
+
|
|
15
|
+
let store: Awaited<ReturnType<typeof openStore>>;
|
|
16
|
+
let client: RemitClient;
|
|
17
|
+
|
|
18
|
+
before(async () => {
|
|
19
|
+
store = await openStore();
|
|
20
|
+
client = store.client;
|
|
21
|
+
});
|
|
22
|
+
|
|
23
|
+
after(() => store.close());
|
|
24
|
+
|
|
25
|
+
const storedId = (): string => randomBytes(13).toString("hex").slice(0, 25);
|
|
26
|
+
|
|
27
|
+
const configDocument = (accountId: string, email: string) => ({
|
|
28
|
+
kind: "reader.config",
|
|
29
|
+
schemaVersion: 1,
|
|
30
|
+
generator: {
|
|
31
|
+
app: "reader",
|
|
32
|
+
version: "v0.1.0",
|
|
33
|
+
exportedAt: "2026-08-27T09:15:00+02:00",
|
|
34
|
+
},
|
|
35
|
+
provenance: { accountConfigId: storedId(), instance: "reader.example.com" },
|
|
36
|
+
accountConfig: { name: "Imported" },
|
|
37
|
+
accounts: [
|
|
38
|
+
{
|
|
39
|
+
accountId,
|
|
40
|
+
email,
|
|
41
|
+
username: email,
|
|
42
|
+
authType: "password",
|
|
43
|
+
credentials: { required: "password" },
|
|
44
|
+
isActive: true,
|
|
45
|
+
imap: { host: "imap.example.com", port: 993, tls: true, startTls: false },
|
|
46
|
+
smtp: {
|
|
47
|
+
enabled: false,
|
|
48
|
+
host: "",
|
|
49
|
+
port: 587,
|
|
50
|
+
tls: false,
|
|
51
|
+
startTls: false,
|
|
52
|
+
username: "",
|
|
53
|
+
},
|
|
54
|
+
displayName: "",
|
|
55
|
+
muted: null,
|
|
56
|
+
composeLanguages: [],
|
|
57
|
+
signature: { plainText: "", html: "" },
|
|
58
|
+
folderRoles: [],
|
|
59
|
+
folderOverrides: [],
|
|
60
|
+
pinnedFolders: [],
|
|
61
|
+
},
|
|
62
|
+
],
|
|
63
|
+
labels: [],
|
|
64
|
+
filters: [],
|
|
65
|
+
addressFlags: [],
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
const seedAccount = (
|
|
69
|
+
accountConfigId: string,
|
|
70
|
+
name: string,
|
|
71
|
+
extra: { deletedAt?: number } = {},
|
|
72
|
+
): Promise<AccountItem> =>
|
|
73
|
+
client.account.create({
|
|
74
|
+
accountConfigId,
|
|
75
|
+
username: `${name}@example.com`,
|
|
76
|
+
email: `${name}@example.com`,
|
|
77
|
+
authType: AccountAuthType.Password,
|
|
78
|
+
passwordHash: '{"ciphertext":"x"}',
|
|
79
|
+
imapHost: "imap.example.com",
|
|
80
|
+
imapPort: 993,
|
|
81
|
+
imapTls: true,
|
|
82
|
+
imapStartTls: false,
|
|
83
|
+
isActive: true,
|
|
84
|
+
connectionState: ConnectionState.Authenticated,
|
|
85
|
+
...extra,
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
const importAccount = async (email: string): Promise<AccountItem> => {
|
|
89
|
+
const accountConfigId = storedId();
|
|
90
|
+
const accountId = storedId();
|
|
91
|
+
const outcome = await importConfig(
|
|
92
|
+
{
|
|
93
|
+
repositories: client,
|
|
94
|
+
appointFolderRole: async () => {},
|
|
95
|
+
},
|
|
96
|
+
{
|
|
97
|
+
accountConfigId,
|
|
98
|
+
userId: accountConfigId,
|
|
99
|
+
document: configDocument(accountId, email),
|
|
100
|
+
mode: "apply",
|
|
101
|
+
onExisting: "abort",
|
|
102
|
+
},
|
|
103
|
+
);
|
|
104
|
+
assert.equal(outcome.outcome, "report");
|
|
105
|
+
return client.account.get(accountId);
|
|
106
|
+
};
|
|
107
|
+
|
|
108
|
+
const recordingSqs = (sent: SendMessageCommand[], fail = false): SQSClient =>
|
|
109
|
+
({
|
|
110
|
+
send: async (command: SendMessageCommand) => {
|
|
111
|
+
if (fail) throw new Error("queue unreachable");
|
|
112
|
+
sent.push(command);
|
|
113
|
+
return {};
|
|
114
|
+
},
|
|
115
|
+
}) as unknown as SQSClient;
|
|
116
|
+
|
|
117
|
+
const heal = (sqsClient: SQSClient) =>
|
|
118
|
+
finishHalfDeletedAccounts({
|
|
119
|
+
account: client.account,
|
|
120
|
+
configImport: client.configImport,
|
|
121
|
+
sqsClient,
|
|
122
|
+
accountFanoutQueueUrl: FANOUT_URL,
|
|
123
|
+
log: noopLogger,
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
describe("finishing an account deleted before #1384 on scheduler start", () => {
|
|
127
|
+
let halfDeleted: AccountItem;
|
|
128
|
+
let live: AccountItem;
|
|
129
|
+
let deleted: AccountItem;
|
|
130
|
+
let imported: AccountItem;
|
|
131
|
+
|
|
132
|
+
beforeEach(async () => {
|
|
133
|
+
const accountConfigId = storedId();
|
|
134
|
+
live = await seedAccount(accountConfigId, `live-${storedId()}`);
|
|
135
|
+
halfDeleted = await seedAccount(accountConfigId, `gone-${storedId()}`);
|
|
136
|
+
await client.account.update(halfDeleted.accountId, { isActive: false });
|
|
137
|
+
deleted = await seedAccount(accountConfigId, `deleted-${storedId()}`, {
|
|
138
|
+
deletedAt: 1_700_000_000_000,
|
|
139
|
+
});
|
|
140
|
+
await client.account.update(deleted.accountId, { isActive: false });
|
|
141
|
+
imported = await importAccount(`imported-${storedId()}@example.com`);
|
|
142
|
+
});
|
|
143
|
+
|
|
144
|
+
const allAccounts = () =>
|
|
145
|
+
client.account.get([
|
|
146
|
+
halfDeleted.accountId,
|
|
147
|
+
live.accountId,
|
|
148
|
+
deleted.accountId,
|
|
149
|
+
imported.accountId,
|
|
150
|
+
]);
|
|
151
|
+
|
|
152
|
+
it("tombstones it and asks for the same purge a delete asks for", async () => {
|
|
153
|
+
const sent: SendMessageCommand[] = [];
|
|
154
|
+
|
|
155
|
+
await heal(recordingSqs(sent));
|
|
156
|
+
|
|
157
|
+
const purges = sent
|
|
158
|
+
.filter((command) => command.input.QueueUrl === FANOUT_URL)
|
|
159
|
+
.map((command) => JSON.parse(command.input.MessageBody ?? "{}"));
|
|
160
|
+
assert.ok(
|
|
161
|
+
purges.some(
|
|
162
|
+
(body) =>
|
|
163
|
+
body.type === "AccountDataPurge" &&
|
|
164
|
+
body.accountId === halfDeleted.accountId &&
|
|
165
|
+
body.accountConfigId === halfDeleted.accountConfigId,
|
|
166
|
+
),
|
|
167
|
+
);
|
|
168
|
+
const byId = new Map(
|
|
169
|
+
(await allAccounts()).map((account) => [account.accountId, account]),
|
|
170
|
+
);
|
|
171
|
+
assert.ok(byId.get(halfDeleted.accountId)?.deletedAt);
|
|
172
|
+
assert.equal(byId.get(live.accountId)?.deletedAt, undefined);
|
|
173
|
+
assert.equal(byId.get(imported.accountId)?.deletedAt, undefined);
|
|
174
|
+
assert.equal(byId.get(deleted.accountId)?.deletedAt, 1_700_000_000_000);
|
|
175
|
+
assert.ok(
|
|
176
|
+
!purges.some((body) =>
|
|
177
|
+
[live.accountId, imported.accountId, deleted.accountId].includes(
|
|
178
|
+
body.accountId,
|
|
179
|
+
),
|
|
180
|
+
),
|
|
181
|
+
);
|
|
182
|
+
});
|
|
183
|
+
|
|
184
|
+
it("does nothing for it on the next start", async () => {
|
|
185
|
+
await heal(recordingSqs([]));
|
|
186
|
+
const sent: SendMessageCommand[] = [];
|
|
187
|
+
|
|
188
|
+
await heal(recordingSqs(sent));
|
|
189
|
+
|
|
190
|
+
assert.ok(
|
|
191
|
+
!sent.some((command) =>
|
|
192
|
+
(command.input.MessageBody ?? "").includes(halfDeleted.accountId),
|
|
193
|
+
),
|
|
194
|
+
);
|
|
195
|
+
});
|
|
196
|
+
|
|
197
|
+
it("fails the start and leaves it to be found again when the purge cannot be asked for", async () => {
|
|
198
|
+
await assert.rejects(heal(recordingSqs([], true)), /queue unreachable/);
|
|
199
|
+
|
|
200
|
+
const after = await client.account.get(halfDeleted.accountId);
|
|
201
|
+
assert.equal(after.deletedAt, undefined);
|
|
202
|
+
});
|
|
203
|
+
});
|