@remit/imap-worker 0.0.87 → 0.0.89
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +6 -6
- package/src/cli.ts +7 -0
- package/src/credential-less-sync-fence.test.ts +5 -9
- package/src/e2e-processor-shim.ts +2 -2
- package/src/emit.test.ts +24 -59
- package/src/emit.ts +50 -73
- package/src/events.ts +1 -19
- package/src/handlers/delete-account-objects.test.ts +25 -14
- package/src/handlers/delete-account-objects.ts +2 -11
- package/src/handlers/placement-push.sqlite.test.ts +1 -1
- package/src/handlers/probe-message-id.sqlite.test.ts +23 -28
- package/src/handlers/sync-mailboxes.ts +2 -2
- package/src/handlers/sync-message-body.ts +14 -18
- package/src/index.ts +6 -0
- package/src/mail-sync-selection.sqlite.test.ts +26 -23
- package/src/pending-sync.sqlite.test.ts +27 -28
- package/src/processor.ts +0 -8
- package/src/queues.ts +49 -0
- package/src/scheduler/attachment-sweep.sqlite.test.ts +1 -0
- package/src/scheduler/half-deleted-accounts.sqlite.test.ts +20 -27
- package/src/scheduler/handler.ts +8 -6
- package/src/scheduler/run-tick.sqlite.test.ts +15 -16
- package/src/scheduler/run-tick.ts +10 -14
- package/src/scheduler/runner.ts +18 -10
- package/src/stranded-placement.sqlite.test.ts +4 -5
- package/src/test-helpers/test-world.ts +40 -11
- package/src/thread-join.sqlite.test.ts +7 -16
- package/src/with-oauth-lifecycle-deps.sqlite.test.ts +9 -2
- package/src/processor.test.ts +0 -58
|
@@ -22,7 +22,6 @@ import {
|
|
|
22
22
|
MailConnectionError,
|
|
23
23
|
type MailCredentials,
|
|
24
24
|
} from "@remit/mailbox-service";
|
|
25
|
-
import { env } from "expect-env";
|
|
26
25
|
import pMap from "p-map";
|
|
27
26
|
import {
|
|
28
27
|
isAccountDeleted,
|
|
@@ -31,6 +30,7 @@ import {
|
|
|
31
30
|
} from "../account-check.js";
|
|
32
31
|
import { emitEvent } from "../emit.js";
|
|
33
32
|
import type { SyncMailboxesEvent, SyncMessagesEvent } from "../events.js";
|
|
33
|
+
import { workerQueues } from "../queues.js";
|
|
34
34
|
import { withOAuthLifecycle } from "../with-oauth-lifecycle.js";
|
|
35
35
|
import { buildLifecycleDeps } from "../with-oauth-lifecycle-deps.js";
|
|
36
36
|
import { createImportedFolder } from "./imported-folder.js";
|
|
@@ -118,7 +118,7 @@ export const syncMailboxes = async (
|
|
|
118
118
|
mailboxService,
|
|
119
119
|
new MailboxQueueService({
|
|
120
120
|
mailboxService,
|
|
121
|
-
|
|
121
|
+
queue: workerQueues().mailboxManagement,
|
|
122
122
|
logger: log,
|
|
123
123
|
}),
|
|
124
124
|
),
|
|
@@ -142,6 +142,7 @@ export const syncMessageBody = async (
|
|
|
142
142
|
flagQueue: flagQueueService,
|
|
143
143
|
calendarSuggestion: calendarSuggestionService,
|
|
144
144
|
calendarStore,
|
|
145
|
+
queues,
|
|
145
146
|
} = await getClient();
|
|
146
147
|
|
|
147
148
|
const account = await accountService.get(accountId);
|
|
@@ -156,18 +157,15 @@ export const syncMessageBody = async (
|
|
|
156
157
|
// PLACEMENT_MOVE_PUSH rides messageMgmtQueue (issue #1271) — same queue
|
|
157
158
|
// the general MessageMoveService move/delete/copy machinery uses, not a
|
|
158
159
|
// dedicated one.
|
|
159
|
-
const
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
sqsQueueUrl: placementMoveQueueUrl,
|
|
169
|
-
})
|
|
170
|
-
: undefined;
|
|
160
|
+
const placementMoveService = new PlacementMoveService({
|
|
161
|
+
messageService,
|
|
162
|
+
threadMessageService,
|
|
163
|
+
markerService,
|
|
164
|
+
addressService,
|
|
165
|
+
mailboxSpecialUseService,
|
|
166
|
+
mailboxService,
|
|
167
|
+
queue: queues.messageManagement,
|
|
168
|
+
});
|
|
171
169
|
|
|
172
170
|
await withOAuthLifecycle(
|
|
173
171
|
buildLifecycleDeps(secrets, accountService),
|
|
@@ -220,12 +218,10 @@ export const syncMessageBody = async (
|
|
|
220
218
|
// A confident, actionable placement verdict moves mail directly on
|
|
221
219
|
// body-sync. Safety lives in the verdict itself (only confident,
|
|
222
220
|
// INBOX/Junk-only) and the movedByRemit loop guard.
|
|
223
|
-
const placementConfig =
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
}
|
|
228
|
-
: undefined;
|
|
221
|
+
const placementConfig = {
|
|
222
|
+
mailboxSpecialUseService,
|
|
223
|
+
placementMoveService,
|
|
224
|
+
};
|
|
229
225
|
|
|
230
226
|
// A matched filter's actions (label upsert, exclusive move) apply on
|
|
231
227
|
// the same body-sync pass, reusing the placement mover for the move
|
package/src/index.ts
CHANGED
|
@@ -9,8 +9,14 @@ import type { SQSBatchResponse, SQSEvent, SQSHandler } from "aws-lambda";
|
|
|
9
9
|
import type { WorkerEvent } from "./events.js";
|
|
10
10
|
import { imapFailureKind } from "./failure-kind.js";
|
|
11
11
|
import { processEvent } from "./processor.js";
|
|
12
|
+
import {
|
|
13
|
+
setWorkerQueues,
|
|
14
|
+
sqsWorkerQueues,
|
|
15
|
+
workerQueueUrlsFromEnv,
|
|
16
|
+
} from "./queues.js";
|
|
12
17
|
|
|
13
18
|
installNodeLogger();
|
|
19
|
+
setWorkerQueues(sqsWorkerQueues(workerQueueUrlsFromEnv()));
|
|
14
20
|
|
|
15
21
|
const log = createLogger();
|
|
16
22
|
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
import assert from "node:assert/strict";
|
|
2
2
|
import { randomUUID } from "node:crypto";
|
|
3
3
|
import { afterEach, beforeEach, describe, it } from "node:test";
|
|
4
|
-
import type { SendMessageCommand, SQSClient } from "@aws-sdk/client-sqs";
|
|
5
4
|
import {
|
|
6
5
|
_resetForTest,
|
|
7
6
|
type RemitClient,
|
|
8
7
|
setClient,
|
|
9
8
|
} from "@remit/backend/client";
|
|
10
9
|
import type { AccountItem } from "@remit/data-ports";
|
|
10
|
+
import { recordingQueue } from "@remit/data-ports/recording-queue";
|
|
11
11
|
import {
|
|
12
12
|
AccountAuthType,
|
|
13
13
|
AccountService,
|
|
@@ -15,16 +15,17 @@ import {
|
|
|
15
15
|
MailboxCursorState,
|
|
16
16
|
} from "@remit/domain-enums";
|
|
17
17
|
import {
|
|
18
|
-
AccountRepo,
|
|
19
|
-
|
|
18
|
+
type AccountRepo,
|
|
19
|
+
createSqliteRepositories,
|
|
20
|
+
type DrizzleMessageRepository,
|
|
20
21
|
MailboxLockRepo,
|
|
21
|
-
MailboxRepo,
|
|
22
|
-
SearchIndex,
|
|
22
|
+
type MailboxRepo,
|
|
23
23
|
} from "@remit/drizzle-service";
|
|
24
24
|
import { createShippedSqliteDb } from "@remit/drizzle-service/test-sqlite";
|
|
25
25
|
import type { Logger } from "@remit/logger";
|
|
26
26
|
import { syncMailboxes } from "./handlers/sync-mailboxes.js";
|
|
27
27
|
import { syncMessages } from "./handlers/sync-messages.js";
|
|
28
|
+
import { setWorkerQueues } from "./queues.js";
|
|
28
29
|
import { runSchedulerTick } from "./scheduler/run-tick.js";
|
|
29
30
|
|
|
30
31
|
const NOW = 1_700_000_000_000;
|
|
@@ -53,12 +54,14 @@ let close: () => void;
|
|
|
53
54
|
beforeEach(() => {
|
|
54
55
|
const store = createShippedSqliteDb();
|
|
55
56
|
close = store.close;
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
store.db
|
|
60
|
-
|
|
61
|
-
|
|
57
|
+
const repos = createSqliteRepositories({
|
|
58
|
+
app: store.db,
|
|
59
|
+
mail: store.db,
|
|
60
|
+
search: store.db,
|
|
61
|
+
});
|
|
62
|
+
accounts = repos.account;
|
|
63
|
+
mailboxes = repos.mailbox;
|
|
64
|
+
messages = repos.message;
|
|
62
65
|
locks = new MailboxLockRepo(store.db);
|
|
63
66
|
client = {
|
|
64
67
|
account: accounts,
|
|
@@ -67,6 +70,15 @@ beforeEach(() => {
|
|
|
67
70
|
mailboxLock: locks,
|
|
68
71
|
} as unknown as RemitClient;
|
|
69
72
|
setClient(client);
|
|
73
|
+
const queue = recordingQueue();
|
|
74
|
+
setWorkerQueues({
|
|
75
|
+
mailboxes: queue,
|
|
76
|
+
messages: queue,
|
|
77
|
+
flags: queue,
|
|
78
|
+
body: queue,
|
|
79
|
+
mailboxManagement: queue,
|
|
80
|
+
messageManagement: queue,
|
|
81
|
+
});
|
|
70
82
|
});
|
|
71
83
|
|
|
72
84
|
afterEach(() => {
|
|
@@ -128,18 +140,11 @@ const seedSyncedAccount = async (): Promise<{
|
|
|
128
140
|
};
|
|
129
141
|
|
|
130
142
|
const tick = async (): Promise<{ enqueued: number; accountIds: string[] }> => {
|
|
131
|
-
const
|
|
132
|
-
const sqsClient = {
|
|
133
|
-
send: async (command: SendMessageCommand) => {
|
|
134
|
-
sent.push(command);
|
|
135
|
-
return {};
|
|
136
|
-
},
|
|
137
|
-
} as unknown as SQSClient;
|
|
143
|
+
const queue = recordingQueue();
|
|
138
144
|
const result = await runSchedulerTick({
|
|
139
145
|
redriveStrandedPlacements: async () => 0,
|
|
140
146
|
accountService: accounts,
|
|
141
|
-
|
|
142
|
-
queueUrl: "https://queue.test/mailboxes",
|
|
147
|
+
queue,
|
|
143
148
|
log: silentLogger,
|
|
144
149
|
tickIntervalMs: 60_000,
|
|
145
150
|
offlineIntervalMs: 0,
|
|
@@ -147,9 +152,7 @@ const tick = async (): Promise<{ enqueued: number; accountIds: string[] }> => {
|
|
|
147
152
|
});
|
|
148
153
|
return {
|
|
149
154
|
enqueued: result.enqueued,
|
|
150
|
-
accountIds: sent.map(
|
|
151
|
-
(command) => JSON.parse(command.input.MessageBody ?? "{}").accountId,
|
|
152
|
-
),
|
|
155
|
+
accountIds: queue.sent.map((message) => String(message.body.accountId)),
|
|
153
156
|
};
|
|
154
157
|
};
|
|
155
158
|
|
|
@@ -1,18 +1,21 @@
|
|
|
1
1
|
import assert from "node:assert/strict";
|
|
2
|
-
import { afterEach,
|
|
3
|
-
import { SendMessageCommand, SQSClient } from "@aws-sdk/client-sqs";
|
|
2
|
+
import { afterEach, beforeEach, describe, it } from "node:test";
|
|
4
3
|
import {
|
|
5
4
|
_resetForTest,
|
|
6
5
|
type RemitClient,
|
|
7
6
|
setClient,
|
|
8
7
|
} from "@remit/backend/client";
|
|
8
|
+
import {
|
|
9
|
+
type RecordingQueue,
|
|
10
|
+
recordingQueue,
|
|
11
|
+
} from "@remit/data-ports/recording-queue";
|
|
9
12
|
import { MailboxLockRepo } from "@remit/drizzle-service";
|
|
10
13
|
import { createShippedSqliteDb } from "@remit/drizzle-service/test-sqlite";
|
|
11
|
-
import {
|
|
12
|
-
import type * as Emit from "./emit.js";
|
|
14
|
+
import { emitEvent, SYNC_MESSAGES_PENDING } from "./emit.js";
|
|
13
15
|
import type { SyncMessagesEvent } from "./events.js";
|
|
14
16
|
import { emitMailboxResync, emitMoveResync } from "./handlers/message-move.js";
|
|
15
17
|
import { emitSyncMessagesEvents } from "./handlers/sync-mailboxes.js";
|
|
18
|
+
import { setWorkerQueues } from "./queues.js";
|
|
16
19
|
|
|
17
20
|
const ACCOUNT = "account-1366";
|
|
18
21
|
const FOLDERS = [
|
|
@@ -21,20 +24,17 @@ const FOLDERS = [
|
|
|
21
24
|
{ mailboxId: "mbx-archive", fullPath: "Archive" },
|
|
22
25
|
];
|
|
23
26
|
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
let emitEvent: typeof Emit.emitEvent;
|
|
27
|
-
let SYNC_MESSAGES_PENDING: typeof Emit.SYNC_MESSAGES_PENDING;
|
|
27
|
+
let queue: RecordingQueue;
|
|
28
28
|
let locks: MailboxLockRepo;
|
|
29
29
|
let closeStore: () => void;
|
|
30
30
|
|
|
31
31
|
type Queued = { mailboxId: string; eventId: string };
|
|
32
32
|
|
|
33
33
|
const queued = (): Queued[] =>
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
});
|
|
34
|
+
queue.sent.map(({ body }) => ({
|
|
35
|
+
mailboxId: String(body.mailboxId),
|
|
36
|
+
eventId: String(body.eventId),
|
|
37
|
+
}));
|
|
38
38
|
|
|
39
39
|
const inboxSync = (): Omit<SyncMessagesEvent, "eventId" | "timestamp"> => ({
|
|
40
40
|
type: "SYNC_MESSAGES",
|
|
@@ -45,18 +45,16 @@ const inboxSync = (): Omit<SyncMessagesEvent, "eventId" | "timestamp"> => ({
|
|
|
45
45
|
const fanOut = (): Promise<void> =>
|
|
46
46
|
emitSyncMessagesEvents(ACCOUNT, FOLDERS, emitEvent);
|
|
47
47
|
|
|
48
|
-
before(async () => {
|
|
49
|
-
const fifo = (name: string) =>
|
|
50
|
-
`https://sqs.eu-west-1.amazonaws.com/0/test-${name}.fifo`;
|
|
51
|
-
process.env.SQS_QUEUE_URL_MAILBOXES = fifo("mailboxes");
|
|
52
|
-
process.env.SQS_QUEUE_URL_MESSAGES = fifo("messages");
|
|
53
|
-
process.env.SQS_QUEUE_URL_FLAGS = fifo("flags");
|
|
54
|
-
({ emitEvent, SYNC_MESSAGES_PENDING } = await import("./emit.js"));
|
|
55
|
-
});
|
|
56
|
-
|
|
57
48
|
beforeEach(() => {
|
|
58
|
-
|
|
59
|
-
|
|
49
|
+
queue = recordingQueue();
|
|
50
|
+
setWorkerQueues({
|
|
51
|
+
mailboxes: queue,
|
|
52
|
+
messages: queue,
|
|
53
|
+
flags: queue,
|
|
54
|
+
body: queue,
|
|
55
|
+
mailboxManagement: queue,
|
|
56
|
+
messageManagement: queue,
|
|
57
|
+
});
|
|
60
58
|
const store = createShippedSqliteDb();
|
|
61
59
|
closeStore = store.close;
|
|
62
60
|
locks = new MailboxLockRepo(store.db);
|
|
@@ -140,10 +138,11 @@ describe("SYNC_MESSAGES under explicit refreshes (#1366)", () => {
|
|
|
140
138
|
});
|
|
141
139
|
|
|
142
140
|
it("releases the marker when the send fails, so the next round is not swallowed", async () => {
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
141
|
+
let sends = 0;
|
|
142
|
+
queue.answer(async () => {
|
|
143
|
+
sends += 1;
|
|
144
|
+
if (sends === 1) throw new Error("queue unreachable");
|
|
145
|
+
});
|
|
147
146
|
|
|
148
147
|
await assert.rejects(emitEvent(inboxSync()), /queue unreachable/);
|
|
149
148
|
await emitEvent(inboxSync());
|
|
@@ -162,6 +161,6 @@ describe("SYNC_MESSAGES under explicit refreshes (#1366)", () => {
|
|
|
162
161
|
} as unknown as RemitClient);
|
|
163
162
|
|
|
164
163
|
await assert.rejects(emitEvent(inboxSync()), /marker store down/);
|
|
165
|
-
assert.equal(
|
|
164
|
+
assert.equal(queue.sent.length, 0);
|
|
166
165
|
});
|
|
167
166
|
});
|
package/src/processor.ts
CHANGED
|
@@ -54,14 +54,6 @@ export const processEvent = async (
|
|
|
54
54
|
return handleAppendSentMessage(event, log, receiveCount);
|
|
55
55
|
case "DELETE_ACCOUNT_OBJECTS":
|
|
56
56
|
return handleDeleteAccountObjects(event, log);
|
|
57
|
-
case "IMAP_WORKER_STOP":
|
|
58
|
-
// Tombstone fence on the account row already stops processing;
|
|
59
|
-
// this event acks the cascade contract and is a no-op today.
|
|
60
|
-
log.info(
|
|
61
|
-
{ accountConfigId: event.accountConfigId, accountId: event.accountId },
|
|
62
|
-
"Imap worker stop signal received",
|
|
63
|
-
);
|
|
64
|
-
return;
|
|
65
57
|
default:
|
|
66
58
|
throw new Error(`Unknown event type: ${(event as WorkerEvent).type}`);
|
|
67
59
|
}
|
package/src/queues.ts
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import type { FifoQueueProducer, QueueProducer } from "@remit/data-ports";
|
|
2
|
+
import { createQueueProducer } from "@remit/sqs-client/producer";
|
|
3
|
+
import { env } from "expect-env";
|
|
4
|
+
import type { ImapEvent, WorkerEvent } from "./events.js";
|
|
5
|
+
|
|
6
|
+
export interface WorkerQueues {
|
|
7
|
+
mailboxes: FifoQueueProducer<ImapEvent>;
|
|
8
|
+
messages: FifoQueueProducer<ImapEvent>;
|
|
9
|
+
flags: FifoQueueProducer<ImapEvent>;
|
|
10
|
+
body: QueueProducer<ImapEvent>;
|
|
11
|
+
mailboxManagement: QueueProducer<ImapEvent>;
|
|
12
|
+
messageManagement: QueueProducer<WorkerEvent>;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export type WorkerQueueUrls = Record<keyof WorkerQueues, string>;
|
|
16
|
+
|
|
17
|
+
export const workerQueueUrlsFromEnv = (): WorkerQueueUrls => ({
|
|
18
|
+
mailboxes: env.SQS_QUEUE_URL_MAILBOXES,
|
|
19
|
+
messages: env.SQS_QUEUE_URL_MESSAGES,
|
|
20
|
+
flags: env.SQS_QUEUE_URL_FLAGS,
|
|
21
|
+
body: env.SQS_QUEUE_URL_BODY,
|
|
22
|
+
mailboxManagement: env.SQS_QUEUE_URL_MAILBOX_MGMT,
|
|
23
|
+
messageManagement: env.SQS_QUEUE_URL_MESSAGE_MGMT,
|
|
24
|
+
});
|
|
25
|
+
|
|
26
|
+
export const sqsWorkerQueues = (urls: WorkerQueueUrls): WorkerQueues => ({
|
|
27
|
+
mailboxes: createQueueProducer({ queueUrl: urls.mailboxes }),
|
|
28
|
+
messages: createQueueProducer({ queueUrl: urls.messages }),
|
|
29
|
+
flags: createQueueProducer({ queueUrl: urls.flags }),
|
|
30
|
+
body: createQueueProducer({ queueUrl: urls.body }),
|
|
31
|
+
mailboxManagement: createQueueProducer({ queueUrl: urls.mailboxManagement }),
|
|
32
|
+
messageManagement: createQueueProducer({ queueUrl: urls.messageManagement }),
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
let composed: WorkerQueues | null = null;
|
|
36
|
+
|
|
37
|
+
/** Register the worker's queues from its entry, before the first event. */
|
|
38
|
+
export const setWorkerQueues = (queues: WorkerQueues): void => {
|
|
39
|
+
composed = queues;
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
export const workerQueues = (): WorkerQueues => {
|
|
43
|
+
if (!composed) {
|
|
44
|
+
throw new Error(
|
|
45
|
+
"no worker queues registered — register them with setWorkerQueues() from the entry",
|
|
46
|
+
);
|
|
47
|
+
}
|
|
48
|
+
return composed;
|
|
49
|
+
};
|
|
@@ -1,17 +1,19 @@
|
|
|
1
1
|
import assert from "node:assert/strict";
|
|
2
2
|
import { randomBytes } from "node:crypto";
|
|
3
3
|
import { after, before, beforeEach, describe, it } from "node:test";
|
|
4
|
-
import type { SendMessageCommand, SQSClient } from "@aws-sdk/client-sqs";
|
|
5
4
|
import { finishHalfDeletedAccounts } from "@remit/backend/account-deletion";
|
|
6
5
|
import type { RemitClient } from "@remit/backend/client";
|
|
7
6
|
import { importConfig } from "@remit/config-transfer";
|
|
8
7
|
import type { AccountItem } from "@remit/data-ports";
|
|
8
|
+
import {
|
|
9
|
+
failingQueue,
|
|
10
|
+
type RecordingQueue,
|
|
11
|
+
recordingQueue,
|
|
12
|
+
} from "@remit/data-ports/recording-queue";
|
|
9
13
|
import { AccountAuthType, ConnectionState } from "@remit/domain-enums";
|
|
10
14
|
import { noopLogger } from "@remit/logger";
|
|
11
15
|
import { openStore } from "../test-helpers/test-world.js";
|
|
12
16
|
|
|
13
|
-
const FANOUT_URL = "http://queue:9324/000000000000/remit-account-fanout";
|
|
14
|
-
|
|
15
17
|
let store: Awaited<ReturnType<typeof openStore>>;
|
|
16
18
|
let client: RemitClient;
|
|
17
19
|
|
|
@@ -105,21 +107,11 @@ const importAccount = async (email: string): Promise<AccountItem> => {
|
|
|
105
107
|
return client.account.get(accountId);
|
|
106
108
|
};
|
|
107
109
|
|
|
108
|
-
const
|
|
109
|
-
({
|
|
110
|
-
send: async (command: SendMessageCommand) => {
|
|
111
|
-
if (fail) throw new Error("queue unreachable");
|
|
112
|
-
sent.push(command);
|
|
113
|
-
return {};
|
|
114
|
-
},
|
|
115
|
-
}) as unknown as SQSClient;
|
|
116
|
-
|
|
117
|
-
const heal = (sqsClient: SQSClient) =>
|
|
110
|
+
const heal = (queue: RecordingQueue) =>
|
|
118
111
|
finishHalfDeletedAccounts({
|
|
119
112
|
account: client.account,
|
|
120
113
|
configImport: client.configImport,
|
|
121
|
-
|
|
122
|
-
accountFanoutQueueUrl: FANOUT_URL,
|
|
114
|
+
queue,
|
|
123
115
|
log: noopLogger,
|
|
124
116
|
});
|
|
125
117
|
|
|
@@ -150,13 +142,11 @@ describe("finishing an account deleted before #1384 on scheduler start", () => {
|
|
|
150
142
|
]);
|
|
151
143
|
|
|
152
144
|
it("tombstones it and asks for the same purge a delete asks for", async () => {
|
|
153
|
-
const
|
|
145
|
+
const queue = recordingQueue();
|
|
154
146
|
|
|
155
|
-
await heal(
|
|
147
|
+
await heal(queue);
|
|
156
148
|
|
|
157
|
-
const purges = sent
|
|
158
|
-
.filter((command) => command.input.QueueUrl === FANOUT_URL)
|
|
159
|
-
.map((command) => JSON.parse(command.input.MessageBody ?? "{}"));
|
|
149
|
+
const purges = queue.sent.map((message) => message.body);
|
|
160
150
|
assert.ok(
|
|
161
151
|
purges.some(
|
|
162
152
|
(body) =>
|
|
@@ -175,27 +165,30 @@ describe("finishing an account deleted before #1384 on scheduler start", () => {
|
|
|
175
165
|
assert.ok(
|
|
176
166
|
!purges.some((body) =>
|
|
177
167
|
[live.accountId, imported.accountId, deleted.accountId].includes(
|
|
178
|
-
body.accountId,
|
|
168
|
+
String(body.accountId),
|
|
179
169
|
),
|
|
180
170
|
),
|
|
181
171
|
);
|
|
182
172
|
});
|
|
183
173
|
|
|
184
174
|
it("does nothing for it on the next start", async () => {
|
|
185
|
-
await heal(
|
|
186
|
-
const
|
|
175
|
+
await heal(recordingQueue());
|
|
176
|
+
const queue = recordingQueue();
|
|
187
177
|
|
|
188
|
-
await heal(
|
|
178
|
+
await heal(queue);
|
|
189
179
|
|
|
190
180
|
assert.ok(
|
|
191
|
-
!sent.some(
|
|
192
|
-
(
|
|
181
|
+
!queue.sent.some(
|
|
182
|
+
(message) => message.body.accountId === halfDeleted.accountId,
|
|
193
183
|
),
|
|
194
184
|
);
|
|
195
185
|
});
|
|
196
186
|
|
|
197
187
|
it("fails the start and leaves it to be found again when the purge cannot be asked for", async () => {
|
|
198
|
-
await assert.rejects(
|
|
188
|
+
await assert.rejects(
|
|
189
|
+
heal(failingQueue(new Error("queue unreachable"))),
|
|
190
|
+
/queue unreachable/,
|
|
191
|
+
);
|
|
199
192
|
|
|
200
193
|
const after = await client.account.get(halfDeleted.accountId);
|
|
201
194
|
assert.equal(after.deletedAt, undefined);
|
package/src/scheduler/handler.ts
CHANGED
|
@@ -1,10 +1,14 @@
|
|
|
1
1
|
import { getClient } from "@remit/backend/client";
|
|
2
2
|
import { createLogger } from "@remit/logger";
|
|
3
3
|
import { installNodeLogger, withTelemetry } from "@remit/logger-lambda/node";
|
|
4
|
-
import { createQueueProducer } from "@remit/sqs-client/producer";
|
|
5
4
|
import type { ScheduledHandler } from "aws-lambda";
|
|
6
|
-
import { env } from "expect-env";
|
|
7
5
|
import { emitEvent } from "../emit.js";
|
|
6
|
+
import {
|
|
7
|
+
setWorkerQueues,
|
|
8
|
+
sqsWorkerQueues,
|
|
9
|
+
workerQueues,
|
|
10
|
+
workerQueueUrlsFromEnv,
|
|
11
|
+
} from "../queues.js";
|
|
8
12
|
import { buildAttachmentSweep } from "./attachment-sweep.js";
|
|
9
13
|
import { buildCalendarSubscriptionRefresh } from "./calendar-subscriptions.js";
|
|
10
14
|
import { getOfflineIntervalMs, getTickIntervalMs } from "./config.js";
|
|
@@ -15,8 +19,7 @@ installNodeLogger();
|
|
|
15
19
|
|
|
16
20
|
const log = createLogger();
|
|
17
21
|
|
|
18
|
-
|
|
19
|
-
const sqsClient = createQueueProducer({ queueUrl: mailboxesQueueUrl });
|
|
22
|
+
setWorkerQueues(sqsWorkerQueues(workerQueueUrlsFromEnv()));
|
|
20
23
|
|
|
21
24
|
/**
|
|
22
25
|
* EventBridge-scheduled entry point for the periodic mailbox-sync tick
|
|
@@ -41,8 +44,7 @@ export const handler: ScheduledHandler = withTelemetry(async (event) => {
|
|
|
41
44
|
|
|
42
45
|
await runSchedulerTick({
|
|
43
46
|
accountService: client.account,
|
|
44
|
-
|
|
45
|
-
queueUrl: mailboxesQueueUrl,
|
|
47
|
+
queue: workerQueues().mailboxes,
|
|
46
48
|
log,
|
|
47
49
|
tickIntervalMs: getTickIntervalMs(),
|
|
48
50
|
offlineIntervalMs: getOfflineIntervalMs(),
|
|
@@ -1,13 +1,16 @@
|
|
|
1
1
|
import assert from "node:assert/strict";
|
|
2
2
|
import { afterEach, beforeEach, describe, it } from "node:test";
|
|
3
|
-
import type { SendMessageCommand, SQSClient } from "@aws-sdk/client-sqs";
|
|
4
3
|
import type { AccountItem } from "@remit/data-ports";
|
|
4
|
+
import { recordingQueue } from "@remit/data-ports/recording-queue";
|
|
5
5
|
import {
|
|
6
6
|
AccountAuthType,
|
|
7
7
|
AccountService,
|
|
8
8
|
ConnectionState,
|
|
9
9
|
} from "@remit/domain-enums";
|
|
10
|
-
import {
|
|
10
|
+
import {
|
|
11
|
+
type AccountRepo,
|
|
12
|
+
createSqliteRepositories,
|
|
13
|
+
} from "@remit/drizzle-service";
|
|
11
14
|
import { createShippedSqliteDb } from "@remit/drizzle-service/test-sqlite";
|
|
12
15
|
import { capturingLogger, type LogLine } from "../test-helpers/test-world.js";
|
|
13
16
|
import { runSchedulerTick } from "./run-tick.js";
|
|
@@ -22,7 +25,11 @@ let close: () => void;
|
|
|
22
25
|
beforeEach(() => {
|
|
23
26
|
const store = createShippedSqliteDb();
|
|
24
27
|
close = store.close;
|
|
25
|
-
accounts =
|
|
28
|
+
accounts = createSqliteRepositories({
|
|
29
|
+
app: store.db,
|
|
30
|
+
mail: store.db,
|
|
31
|
+
search: store.db,
|
|
32
|
+
}).account;
|
|
26
33
|
});
|
|
27
34
|
|
|
28
35
|
afterEach(() => close());
|
|
@@ -53,18 +60,12 @@ const account = async (
|
|
|
53
60
|
const tick = async (
|
|
54
61
|
options: Partial<Parameters<typeof runSchedulerTick>[0]> = {},
|
|
55
62
|
) => {
|
|
56
|
-
const
|
|
63
|
+
const queue = recordingQueue();
|
|
64
|
+
const { sent } = queue;
|
|
57
65
|
const logs: LogLine[] = [];
|
|
58
66
|
const result = await runSchedulerTick({
|
|
59
67
|
accountService: accounts,
|
|
60
|
-
|
|
61
|
-
send: async (command: SendMessageCommand) => {
|
|
62
|
-
sent.push(command);
|
|
63
|
-
return {};
|
|
64
|
-
},
|
|
65
|
-
} as unknown as SQSClient,
|
|
66
|
-
queueUrl:
|
|
67
|
-
"https://sqs.eu-west-1.amazonaws.com/123/remit-dev-mailboxes.fifo",
|
|
68
|
+
queue,
|
|
68
69
|
log: capturingLogger(logs),
|
|
69
70
|
tickIntervalMs: TICK_INTERVAL_MS,
|
|
70
71
|
offlineIntervalMs: OFFLINE_INTERVAL_MS,
|
|
@@ -72,9 +73,7 @@ const tick = async (
|
|
|
72
73
|
redriveStrandedPlacements: async () => 0,
|
|
73
74
|
...options,
|
|
74
75
|
});
|
|
75
|
-
const enqueued = sent.map(
|
|
76
|
-
(command) => JSON.parse(command.input.MessageBody ?? "{}").accountId,
|
|
77
|
-
);
|
|
76
|
+
const enqueued = sent.map((message) => String(message.body.accountId));
|
|
78
77
|
return { result, sent, enqueued, logs };
|
|
79
78
|
};
|
|
80
79
|
|
|
@@ -122,7 +121,7 @@ describe("a scheduler tick", () => {
|
|
|
122
121
|
|
|
123
122
|
const { sent } = await tick();
|
|
124
123
|
|
|
125
|
-
const dedupId = sent[0]?.
|
|
124
|
+
const dedupId = sent[0]?.deduplicationId;
|
|
126
125
|
assert.ok(
|
|
127
126
|
dedupId?.startsWith(`SYNC_MAILBOXES:scheduled:${live.accountId}:`),
|
|
128
127
|
);
|
|
@@ -1,9 +1,13 @@
|
|
|
1
|
-
import type { SQSClient } from "@aws-sdk/client-sqs";
|
|
2
1
|
import {
|
|
3
2
|
buildScheduledSyncDedupId,
|
|
3
|
+
type SyncMailboxesEvent,
|
|
4
4
|
triggerAccountSync,
|
|
5
5
|
} from "@remit/backend/trigger-sync";
|
|
6
|
-
import type {
|
|
6
|
+
import type {
|
|
7
|
+
AccountItem,
|
|
8
|
+
FifoQueueProducer,
|
|
9
|
+
IAccountRepository,
|
|
10
|
+
} from "@remit/data-ports";
|
|
7
11
|
import type { Logger } from "@remit/logger";
|
|
8
12
|
import { hasStoredCredential } from "@remit/mailbox-service/account-credentials";
|
|
9
13
|
import pMap from "p-map";
|
|
@@ -22,8 +26,7 @@ import { isSyncDue } from "./decide-due.js";
|
|
|
22
26
|
|
|
23
27
|
export interface RunSchedulerTickDeps {
|
|
24
28
|
accountService: Pick<IAccountRepository, "listAllAccountsPage">;
|
|
25
|
-
|
|
26
|
-
queueUrl: string;
|
|
29
|
+
queue: FifoQueueProducer<SyncMailboxesEvent>;
|
|
27
30
|
log: Logger;
|
|
28
31
|
offlineIntervalMs: number;
|
|
29
32
|
/**
|
|
@@ -125,14 +128,8 @@ const isEligible = (account: AccountItem): boolean => {
|
|
|
125
128
|
export const runSchedulerTick = async (
|
|
126
129
|
deps: RunSchedulerTickDeps,
|
|
127
130
|
): Promise<SchedulerTickResult> => {
|
|
128
|
-
const {
|
|
129
|
-
|
|
130
|
-
sqsClient,
|
|
131
|
-
queueUrl,
|
|
132
|
-
log,
|
|
133
|
-
offlineIntervalMs,
|
|
134
|
-
tickIntervalMs,
|
|
135
|
-
} = deps;
|
|
131
|
+
const { accountService, queue, log, offlineIntervalMs, tickIntervalMs } =
|
|
132
|
+
deps;
|
|
136
133
|
const now = deps.now ?? Date.now();
|
|
137
134
|
|
|
138
135
|
let cursor: string | undefined;
|
|
@@ -161,8 +158,7 @@ export const runSchedulerTick = async (
|
|
|
161
158
|
due,
|
|
162
159
|
(account) =>
|
|
163
160
|
triggerAccountSync({
|
|
164
|
-
|
|
165
|
-
queueUrl,
|
|
161
|
+
queue,
|
|
166
162
|
accountId: account.accountId,
|
|
167
163
|
dedupId: buildScheduledSyncDedupId(
|
|
168
164
|
account.accountId,
|