@remit/imap-worker 0.0.82 → 0.0.84
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -1
- package/src/account-check.test.ts +27 -0
- package/src/account-check.ts +13 -0
- package/src/handlers/sync-mailboxes.ts +9 -1
- package/src/handlers/sync-messages.test.ts +23 -0
- package/src/handlers/sync-messages.ts +9 -1
- package/src/mail-sync-selection.sqlite.test.ts +229 -0
- package/src/scheduler/attachment-sweep.test.ts +1 -0
- package/src/scheduler/calendar-subscriptions.test.ts +159 -0
- package/src/scheduler/calendar-subscriptions.ts +103 -0
- package/src/scheduler/handler.ts +2 -0
- package/src/scheduler/loop.test.ts +2 -0
- package/src/scheduler/run-tick.test.ts +112 -0
- package/src/scheduler/run-tick.ts +49 -2
- package/src/scheduler/runner.ts +5 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@remit/imap-worker",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.84",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"exports": {
|
|
@@ -41,6 +41,7 @@
|
|
|
41
41
|
"@remit/search-index-worker": "*",
|
|
42
42
|
"@remit/sqs-client": "*",
|
|
43
43
|
"@remit/backend": "*",
|
|
44
|
+
"@remit/calendar-service": "*",
|
|
44
45
|
"@remit/domain-enums": "*",
|
|
45
46
|
"@remit/mail-oauth-service": "*",
|
|
46
47
|
"@remit/secrets-service": "*",
|
|
@@ -4,6 +4,7 @@ import type { AccountItem } from "@remit/data-ports";
|
|
|
4
4
|
import {
|
|
5
5
|
isAccountDeleted,
|
|
6
6
|
isAccountReauthRequired,
|
|
7
|
+
isMailSyncDisabled,
|
|
7
8
|
isReservedHost,
|
|
8
9
|
isUnsyncableHost,
|
|
9
10
|
} from "./account-check.js";
|
|
@@ -120,3 +121,29 @@ describe("isAccountReauthRequired", () => {
|
|
|
120
121
|
);
|
|
121
122
|
});
|
|
122
123
|
});
|
|
124
|
+
|
|
125
|
+
describe("isMailSyncDisabled", () => {
|
|
126
|
+
it("returns false for a mail-only account", () => {
|
|
127
|
+
assert.equal(
|
|
128
|
+
isMailSyncDisabled(buildAccount({ syncedServices: ["Mail"] }), log),
|
|
129
|
+
false,
|
|
130
|
+
);
|
|
131
|
+
});
|
|
132
|
+
|
|
133
|
+
it("returns false for an account syncing mail and calendar", () => {
|
|
134
|
+
assert.equal(
|
|
135
|
+
isMailSyncDisabled(
|
|
136
|
+
buildAccount({ syncedServices: ["Mail", "Calendar"] }),
|
|
137
|
+
log,
|
|
138
|
+
),
|
|
139
|
+
false,
|
|
140
|
+
);
|
|
141
|
+
});
|
|
142
|
+
|
|
143
|
+
it("returns true for a calendar-only account", () => {
|
|
144
|
+
assert.equal(
|
|
145
|
+
isMailSyncDisabled(buildAccount({ syncedServices: ["Calendar"] }), log),
|
|
146
|
+
true,
|
|
147
|
+
);
|
|
148
|
+
});
|
|
149
|
+
});
|
package/src/account-check.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { AccountItem } from "@remit/data-ports";
|
|
2
|
+
import { AccountService } from "@remit/domain-enums";
|
|
2
3
|
import type { Logger } from "@remit/logger-lambda";
|
|
3
4
|
|
|
4
5
|
/**
|
|
@@ -86,3 +87,15 @@ export const isAccountReauthRequired = (
|
|
|
86
87
|
}
|
|
87
88
|
return false;
|
|
88
89
|
};
|
|
90
|
+
|
|
91
|
+
export const isMailSyncDisabled = (
|
|
92
|
+
account: AccountItem,
|
|
93
|
+
log: Logger,
|
|
94
|
+
): boolean => {
|
|
95
|
+
if (account.syncedServices.includes(AccountService.Mail)) return false;
|
|
96
|
+
log.info(
|
|
97
|
+
{ accountId: account.accountId, syncedServices: account.syncedServices },
|
|
98
|
+
"Skipping account: mail sync is off",
|
|
99
|
+
);
|
|
100
|
+
return true;
|
|
101
|
+
};
|
|
@@ -21,7 +21,11 @@ import {
|
|
|
21
21
|
type MailCredentials,
|
|
22
22
|
} from "@remit/mailbox-service";
|
|
23
23
|
import pMap from "p-map";
|
|
24
|
-
import {
|
|
24
|
+
import {
|
|
25
|
+
isAccountDeleted,
|
|
26
|
+
isMailSyncDisabled,
|
|
27
|
+
isUnsyncableHost,
|
|
28
|
+
} from "../account-check.js";
|
|
25
29
|
import { emitEvent } from "../emit.js";
|
|
26
30
|
import type { SyncMailboxesEvent, SyncMessagesEvent } from "../events.js";
|
|
27
31
|
import { withOAuthLifecycle } from "../with-oauth-lifecycle.js";
|
|
@@ -120,6 +124,10 @@ export const syncMailboxes = async (
|
|
|
120
124
|
return;
|
|
121
125
|
}
|
|
122
126
|
|
|
127
|
+
if (isMailSyncDisabled(account, log)) {
|
|
128
|
+
return;
|
|
129
|
+
}
|
|
130
|
+
|
|
123
131
|
// A reserved/never-resolvable IMAP host (RFC 2606) can never connect, so a
|
|
124
132
|
// sync attempt would retry and dead-letter forever. Skip cleanly — ack the
|
|
125
133
|
// event without connecting or throwing.
|
|
@@ -256,6 +256,7 @@ const liveAccount = {
|
|
|
256
256
|
accountId: "acc-1",
|
|
257
257
|
accountConfigId: "acc-cfg-1",
|
|
258
258
|
imapHost: "localhost",
|
|
259
|
+
syncedServices: ["Mail"],
|
|
259
260
|
username: "user@localhost",
|
|
260
261
|
} as unknown as AccountItem;
|
|
261
262
|
|
|
@@ -542,3 +543,25 @@ describe("syncMessages — a delete that lands mid-round (issue #339)", () => {
|
|
|
542
543
|
assert.equal(harness.accountUpdates[0]?.syncPhase, "error");
|
|
543
544
|
});
|
|
544
545
|
});
|
|
546
|
+
|
|
547
|
+
describe("syncMessages — an account with mail sync off", () => {
|
|
548
|
+
it("acks the event without connecting and without touching the mailbox", async () => {
|
|
549
|
+
const { log } = buildLogger();
|
|
550
|
+
let mailboxReads = 0;
|
|
551
|
+
const harness = buildSyncDeps({
|
|
552
|
+
accountGet: async () =>
|
|
553
|
+
({ ...liveAccount, syncedServices: ["Calendar"] }) as AccountItem,
|
|
554
|
+
mailboxGet: async () => {
|
|
555
|
+
mailboxReads += 1;
|
|
556
|
+
return {};
|
|
557
|
+
},
|
|
558
|
+
});
|
|
559
|
+
|
|
560
|
+
await assert.doesNotReject(
|
|
561
|
+
syncMessages(syncEvent("mbx-1"), log, harness.deps),
|
|
562
|
+
);
|
|
563
|
+
|
|
564
|
+
assert.equal(harness.lifecycleCalls, 0);
|
|
565
|
+
assert.equal(mailboxReads, 0);
|
|
566
|
+
});
|
|
567
|
+
});
|
|
@@ -27,7 +27,11 @@ import {
|
|
|
27
27
|
type SyncedMessage,
|
|
28
28
|
} from "@remit/mailbox-service";
|
|
29
29
|
import pMap from "p-map";
|
|
30
|
-
import {
|
|
30
|
+
import {
|
|
31
|
+
isAccountDeleted,
|
|
32
|
+
isMailSyncDisabled,
|
|
33
|
+
isUnsyncableHost,
|
|
34
|
+
} from "../account-check.js";
|
|
31
35
|
import { emitEvent } from "../emit.js";
|
|
32
36
|
import type {
|
|
33
37
|
FlagPushEvent,
|
|
@@ -126,6 +130,10 @@ export const syncMessages = async (
|
|
|
126
130
|
return;
|
|
127
131
|
}
|
|
128
132
|
|
|
133
|
+
if (isMailSyncDisabled(account, log)) {
|
|
134
|
+
return;
|
|
135
|
+
}
|
|
136
|
+
|
|
129
137
|
// A reserved/never-resolvable IMAP host (RFC 2606) can never connect, so a
|
|
130
138
|
// sync attempt would retry and dead-letter forever. Skip cleanly — ack the
|
|
131
139
|
// event without connecting or throwing.
|
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { randomUUID } from "node:crypto";
|
|
3
|
+
import { afterEach, beforeEach, describe, it } from "node:test";
|
|
4
|
+
import type { SendMessageCommand, SQSClient } from "@aws-sdk/client-sqs";
|
|
5
|
+
import {
|
|
6
|
+
_resetForTest,
|
|
7
|
+
type RemitClient,
|
|
8
|
+
setClient,
|
|
9
|
+
} from "@remit/backend/client";
|
|
10
|
+
import type { AccountItem } from "@remit/data-ports";
|
|
11
|
+
import {
|
|
12
|
+
AccountAuthType,
|
|
13
|
+
AccountService,
|
|
14
|
+
ConnectionState,
|
|
15
|
+
MailboxCursorState,
|
|
16
|
+
} from "@remit/domain-enums";
|
|
17
|
+
import {
|
|
18
|
+
AccountRepo,
|
|
19
|
+
DrizzleMessageRepository,
|
|
20
|
+
MailboxRepo,
|
|
21
|
+
} from "@remit/drizzle-service";
|
|
22
|
+
import { createShippedSqliteDb } from "@remit/drizzle-service/test-sqlite";
|
|
23
|
+
import type { Logger } from "@remit/logger-lambda";
|
|
24
|
+
import { syncMailboxes } from "./handlers/sync-mailboxes.js";
|
|
25
|
+
import {
|
|
26
|
+
type SyncMessagesDeps,
|
|
27
|
+
syncMessages,
|
|
28
|
+
} from "./handlers/sync-messages.js";
|
|
29
|
+
import { runSchedulerTick } from "./scheduler/run-tick.js";
|
|
30
|
+
|
|
31
|
+
const NOW = 1_700_000_000_000;
|
|
32
|
+
|
|
33
|
+
const silentLogger: Logger = (() => {
|
|
34
|
+
const noop = () => {};
|
|
35
|
+
const stub = {
|
|
36
|
+
info: noop,
|
|
37
|
+
warn: noop,
|
|
38
|
+
error: noop,
|
|
39
|
+
debug: noop,
|
|
40
|
+
fatal: noop,
|
|
41
|
+
trace: noop,
|
|
42
|
+
child: () => stub,
|
|
43
|
+
};
|
|
44
|
+
return stub as unknown as Logger;
|
|
45
|
+
})();
|
|
46
|
+
|
|
47
|
+
let accounts: AccountRepo;
|
|
48
|
+
let mailboxes: MailboxRepo;
|
|
49
|
+
let messages: DrizzleMessageRepository;
|
|
50
|
+
let client: RemitClient;
|
|
51
|
+
let close: () => void;
|
|
52
|
+
|
|
53
|
+
beforeEach(() => {
|
|
54
|
+
const store = createShippedSqliteDb();
|
|
55
|
+
close = store.close;
|
|
56
|
+
accounts = new AccountRepo(store.db);
|
|
57
|
+
mailboxes = new MailboxRepo(store.db);
|
|
58
|
+
messages = new DrizzleMessageRepository(store.db as never);
|
|
59
|
+
client = {
|
|
60
|
+
account: accounts,
|
|
61
|
+
mailbox: mailboxes,
|
|
62
|
+
message: messages,
|
|
63
|
+
} as unknown as RemitClient;
|
|
64
|
+
setClient(client);
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
afterEach(() => {
|
|
68
|
+
_resetForTest();
|
|
69
|
+
close();
|
|
70
|
+
});
|
|
71
|
+
|
|
72
|
+
const seedSyncedAccount = async (): Promise<{
|
|
73
|
+
account: AccountItem;
|
|
74
|
+
mailboxId: string;
|
|
75
|
+
messageId: string;
|
|
76
|
+
}> => {
|
|
77
|
+
const account = await accounts.create({
|
|
78
|
+
accountConfigId: "cfg-1181",
|
|
79
|
+
username: "person@example.com",
|
|
80
|
+
email: "person@example.com",
|
|
81
|
+
authType: AccountAuthType.Password,
|
|
82
|
+
passwordHash: '{"ciphertext":"x"}',
|
|
83
|
+
imapHost: "imap.example.com",
|
|
84
|
+
imapPort: 993,
|
|
85
|
+
imapTls: true,
|
|
86
|
+
imapStartTls: false,
|
|
87
|
+
isActive: true,
|
|
88
|
+
connectionState: ConnectionState.Authenticated,
|
|
89
|
+
});
|
|
90
|
+
const created = await mailboxes.create({
|
|
91
|
+
accountId: account.accountId,
|
|
92
|
+
namespacePrefix: "",
|
|
93
|
+
hierarchyDelimiter: "/",
|
|
94
|
+
fullPath: "INBOX",
|
|
95
|
+
uidValidity: 7,
|
|
96
|
+
uidNext: 4,
|
|
97
|
+
highestModseq: "42",
|
|
98
|
+
messageCount: 1,
|
|
99
|
+
unseenCount: 0,
|
|
100
|
+
deletedCount: 0,
|
|
101
|
+
totalSize: 10,
|
|
102
|
+
lastSyncUid: 3,
|
|
103
|
+
highWaterMarkUid: 3,
|
|
104
|
+
lastMessageSyncAt: NOW,
|
|
105
|
+
cursorState: MailboxCursorState.normal,
|
|
106
|
+
} as Parameters<MailboxRepo["create"]>[0]);
|
|
107
|
+
if (created.outcome !== "Created") throw new Error("INBOX path taken");
|
|
108
|
+
const mailbox = created.mailbox;
|
|
109
|
+
const messageId = randomUUID();
|
|
110
|
+
await messages.create({
|
|
111
|
+
messageId,
|
|
112
|
+
mailboxId: mailbox.mailboxId,
|
|
113
|
+
uid: 3,
|
|
114
|
+
sequenceNumber: 1,
|
|
115
|
+
rfc822Size: 10,
|
|
116
|
+
internalDate: NOW,
|
|
117
|
+
envelopeId: randomUUID(),
|
|
118
|
+
rootBodyPartId: randomUUID(),
|
|
119
|
+
status: "active",
|
|
120
|
+
syncStatus: "synced",
|
|
121
|
+
});
|
|
122
|
+
return { account, mailboxId: mailbox.mailboxId, messageId };
|
|
123
|
+
};
|
|
124
|
+
|
|
125
|
+
const tick = async (): Promise<{ enqueued: number; accountIds: string[] }> => {
|
|
126
|
+
const sent: SendMessageCommand[] = [];
|
|
127
|
+
const sqsClient = {
|
|
128
|
+
send: async (command: SendMessageCommand) => {
|
|
129
|
+
sent.push(command);
|
|
130
|
+
return {};
|
|
131
|
+
},
|
|
132
|
+
} as unknown as SQSClient;
|
|
133
|
+
const result = await runSchedulerTick({
|
|
134
|
+
accountService: accounts,
|
|
135
|
+
sqsClient,
|
|
136
|
+
queueUrl: "https://queue.test/mailboxes",
|
|
137
|
+
log: silentLogger,
|
|
138
|
+
tickIntervalMs: 60_000,
|
|
139
|
+
offlineIntervalMs: 0,
|
|
140
|
+
now: NOW,
|
|
141
|
+
});
|
|
142
|
+
return {
|
|
143
|
+
enqueued: result.enqueued,
|
|
144
|
+
accountIds: sent.map(
|
|
145
|
+
(command) => JSON.parse(command.input.MessageBody ?? "{}").accountId,
|
|
146
|
+
),
|
|
147
|
+
};
|
|
148
|
+
};
|
|
149
|
+
|
|
150
|
+
const setServices = (
|
|
151
|
+
accountId: string,
|
|
152
|
+
syncedServices: AccountItem["syncedServices"],
|
|
153
|
+
) => accounts.update(accountId, { syncedServices });
|
|
154
|
+
|
|
155
|
+
describe("turning mail sync off and on again", () => {
|
|
156
|
+
it("stops every mail sync entry point and keeps the stored mail", async () => {
|
|
157
|
+
const { account, mailboxId, messageId } = await seedSyncedAccount();
|
|
158
|
+
const storedMailboxes = await mailboxes.listAllByAccount(account.accountId);
|
|
159
|
+
const storedMessage = await messages.get(messageId);
|
|
160
|
+
|
|
161
|
+
await setServices(account.accountId, [AccountService.Calendar]);
|
|
162
|
+
|
|
163
|
+
assert.deepEqual(await tick(), { enqueued: 0, accountIds: [] });
|
|
164
|
+
|
|
165
|
+
await syncMailboxes(
|
|
166
|
+
{
|
|
167
|
+
type: "SYNC_MAILBOXES",
|
|
168
|
+
eventId: "evt-mailboxes",
|
|
169
|
+
timestamp: NOW,
|
|
170
|
+
accountId: account.accountId,
|
|
171
|
+
},
|
|
172
|
+
silentLogger,
|
|
173
|
+
);
|
|
174
|
+
|
|
175
|
+
let lifecycleCalls = 0;
|
|
176
|
+
const deps = {
|
|
177
|
+
getClient: async () => client,
|
|
178
|
+
buildLifecycleDeps: () => ({}),
|
|
179
|
+
withOAuthLifecycle: async () => {
|
|
180
|
+
lifecycleCalls += 1;
|
|
181
|
+
},
|
|
182
|
+
} as unknown as SyncMessagesDeps;
|
|
183
|
+
await syncMessages(
|
|
184
|
+
{
|
|
185
|
+
type: "SYNC_MESSAGES",
|
|
186
|
+
eventId: "evt-messages",
|
|
187
|
+
timestamp: NOW,
|
|
188
|
+
accountId: account.accountId,
|
|
189
|
+
mailboxId,
|
|
190
|
+
},
|
|
191
|
+
silentLogger,
|
|
192
|
+
deps,
|
|
193
|
+
);
|
|
194
|
+
assert.equal(lifecycleCalls, 0);
|
|
195
|
+
|
|
196
|
+
assert.deepEqual(
|
|
197
|
+
await mailboxes.listAllByAccount(account.accountId),
|
|
198
|
+
storedMailboxes,
|
|
199
|
+
);
|
|
200
|
+
assert.deepEqual(await messages.get(messageId), storedMessage);
|
|
201
|
+
assert.equal(
|
|
202
|
+
(await accounts.get(account.accountId)).connectionState,
|
|
203
|
+
ConnectionState.Authenticated,
|
|
204
|
+
);
|
|
205
|
+
});
|
|
206
|
+
|
|
207
|
+
it("resumes from the stored markers when mail is turned back on", async () => {
|
|
208
|
+
const { account, messageId } = await seedSyncedAccount();
|
|
209
|
+
const storedMailboxes = await mailboxes.listAllByAccount(account.accountId);
|
|
210
|
+
const storedMessage = await messages.get(messageId);
|
|
211
|
+
|
|
212
|
+
await setServices(account.accountId, [AccountService.Calendar]);
|
|
213
|
+
await tick();
|
|
214
|
+
await setServices(account.accountId, [
|
|
215
|
+
AccountService.Mail,
|
|
216
|
+
AccountService.Calendar,
|
|
217
|
+
]);
|
|
218
|
+
|
|
219
|
+
assert.deepEqual(await tick(), {
|
|
220
|
+
enqueued: 1,
|
|
221
|
+
accountIds: [account.accountId],
|
|
222
|
+
});
|
|
223
|
+
assert.deepEqual(
|
|
224
|
+
await mailboxes.listAllByAccount(account.accountId),
|
|
225
|
+
storedMailboxes,
|
|
226
|
+
);
|
|
227
|
+
assert.deepEqual(await messages.get(messageId), storedMessage);
|
|
228
|
+
});
|
|
229
|
+
});
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { describe, it } from "node:test";
|
|
3
|
+
import type { CalendarFeedFetcher } from "@remit/calendar-service";
|
|
4
|
+
import { MemoryCalendarStore } from "@remit/calendar-service/memory-store";
|
|
5
|
+
import type { CalendarCollectionItem } from "@remit/data-ports";
|
|
6
|
+
import { CalendarSource } from "@remit/domain-enums";
|
|
7
|
+
import type { Logger } from "@remit/logger-lambda";
|
|
8
|
+
import { refreshDueCalendarSubscriptions } from "./calendar-subscriptions.js";
|
|
9
|
+
|
|
10
|
+
const INTERVAL_MS = 15 * 60 * 1000;
|
|
11
|
+
const NOW = 1_800_000_000_000;
|
|
12
|
+
|
|
13
|
+
const FEED = `${[
|
|
14
|
+
"BEGIN:VCALENDAR",
|
|
15
|
+
"VERSION:2.0",
|
|
16
|
+
"PRODID:-//Example//Published Calendar//EN",
|
|
17
|
+
"BEGIN:VEVENT",
|
|
18
|
+
"UID:ferry@calendar.example",
|
|
19
|
+
"DTSTAMP:20260901T000000Z",
|
|
20
|
+
"DTSTART:20260908T070000Z",
|
|
21
|
+
"DTEND:20260908T080000Z",
|
|
22
|
+
"SUMMARY:Ferry crossing",
|
|
23
|
+
"END:VEVENT",
|
|
24
|
+
"END:VCALENDAR",
|
|
25
|
+
].join("\r\n")}\r\n`;
|
|
26
|
+
|
|
27
|
+
const logged = (): { log: Logger; lines: string[] } => {
|
|
28
|
+
const lines: string[] = [];
|
|
29
|
+
const capture = (...args: unknown[]) => {
|
|
30
|
+
lines.push(JSON.stringify(args));
|
|
31
|
+
};
|
|
32
|
+
const log = {
|
|
33
|
+
info: capture,
|
|
34
|
+
warn: capture,
|
|
35
|
+
error: capture,
|
|
36
|
+
debug: capture,
|
|
37
|
+
fatal: capture,
|
|
38
|
+
trace: capture,
|
|
39
|
+
child: () => log,
|
|
40
|
+
} as unknown as Logger;
|
|
41
|
+
return { log, lines };
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
const subscription = async (
|
|
45
|
+
store: MemoryCalendarStore,
|
|
46
|
+
urlSegment: string,
|
|
47
|
+
checkedAt: number,
|
|
48
|
+
): Promise<CalendarCollectionItem> => {
|
|
49
|
+
const created = await store.transaction((repos) =>
|
|
50
|
+
repos.calendarCollection.create({
|
|
51
|
+
accountConfigId: "account-config-1",
|
|
52
|
+
urlSegment,
|
|
53
|
+
displayName: urlSegment,
|
|
54
|
+
source: CalendarSource.Subscribed,
|
|
55
|
+
subscriptionUrl: `https://calendar.example/${urlSegment}/private-feedfacecafe.ics`,
|
|
56
|
+
subscriptionEnabled: true,
|
|
57
|
+
}),
|
|
58
|
+
);
|
|
59
|
+
return store.transaction((repos) =>
|
|
60
|
+
repos.calendarCollection.update(
|
|
61
|
+
created.accountConfigId,
|
|
62
|
+
created.calendarId,
|
|
63
|
+
{ subscriptionCheckedAt: checkedAt },
|
|
64
|
+
),
|
|
65
|
+
);
|
|
66
|
+
};
|
|
67
|
+
|
|
68
|
+
const recording =
|
|
69
|
+
(asked: string[], answer: (url: string) => Response): CalendarFeedFetcher =>
|
|
70
|
+
async (url) => {
|
|
71
|
+
asked.push(url);
|
|
72
|
+
return answer(url);
|
|
73
|
+
};
|
|
74
|
+
|
|
75
|
+
describe("refreshDueCalendarSubscriptions", () => {
|
|
76
|
+
it("fetches the subscriptions that are due and skips the rest", async () => {
|
|
77
|
+
const store = new MemoryCalendarStore();
|
|
78
|
+
const due = await subscription(store, "due", NOW - INTERVAL_MS);
|
|
79
|
+
await subscription(store, "fresh", NOW - 60_000);
|
|
80
|
+
const paused = await subscription(store, "paused", 0);
|
|
81
|
+
await store.transaction((repos) =>
|
|
82
|
+
repos.calendarCollection.update(
|
|
83
|
+
paused.accountConfigId,
|
|
84
|
+
paused.calendarId,
|
|
85
|
+
{ subscriptionEnabled: false },
|
|
86
|
+
),
|
|
87
|
+
);
|
|
88
|
+
const asked: string[] = [];
|
|
89
|
+
|
|
90
|
+
const tally = await refreshDueCalendarSubscriptions(
|
|
91
|
+
{
|
|
92
|
+
calendarCollection: {
|
|
93
|
+
listEnabledSubscriptions: () =>
|
|
94
|
+
store.transaction((repos) =>
|
|
95
|
+
repos.calendarCollection.listEnabledSubscriptions(),
|
|
96
|
+
),
|
|
97
|
+
},
|
|
98
|
+
calendarUnitOfWork: store,
|
|
99
|
+
log: logged().log,
|
|
100
|
+
fetcher: recording(asked, () => new Response(FEED, { status: 200 })),
|
|
101
|
+
},
|
|
102
|
+
NOW,
|
|
103
|
+
INTERVAL_MS,
|
|
104
|
+
);
|
|
105
|
+
|
|
106
|
+
assert.deepEqual(asked, [due.subscriptionUrl]);
|
|
107
|
+
assert.deepEqual(tally, { refreshed: 1, failed: 0, notDue: 1 });
|
|
108
|
+
const objects = await store.transaction((repos) =>
|
|
109
|
+
repos.calendarObject.listByCalendar(due.calendarId),
|
|
110
|
+
);
|
|
111
|
+
assert.deepEqual(
|
|
112
|
+
objects.map((object) => object.summary),
|
|
113
|
+
["Ferry crossing"],
|
|
114
|
+
);
|
|
115
|
+
});
|
|
116
|
+
|
|
117
|
+
it("counts a feed that fails, keeps refreshing the others, and never logs an address", async () => {
|
|
118
|
+
const store = new MemoryCalendarStore();
|
|
119
|
+
const broken = await subscription(store, "broken", 0);
|
|
120
|
+
const working = await subscription(store, "working", 0);
|
|
121
|
+
const { log, lines } = logged();
|
|
122
|
+
|
|
123
|
+
const tally = await refreshDueCalendarSubscriptions(
|
|
124
|
+
{
|
|
125
|
+
calendarCollection: {
|
|
126
|
+
listEnabledSubscriptions: () =>
|
|
127
|
+
store.transaction((repos) =>
|
|
128
|
+
repos.calendarCollection.listEnabledSubscriptions(),
|
|
129
|
+
),
|
|
130
|
+
},
|
|
131
|
+
calendarUnitOfWork: store,
|
|
132
|
+
log,
|
|
133
|
+
fetcher: recording([], (url) =>
|
|
134
|
+
url === broken.subscriptionUrl
|
|
135
|
+
? new Response("gone", { status: 410 })
|
|
136
|
+
: new Response(FEED, { status: 200 }),
|
|
137
|
+
),
|
|
138
|
+
},
|
|
139
|
+
NOW,
|
|
140
|
+
INTERVAL_MS,
|
|
141
|
+
);
|
|
142
|
+
|
|
143
|
+
assert.deepEqual(tally, { refreshed: 1, failed: 1, notDue: 0 });
|
|
144
|
+
const stored = await store.transaction((repos) =>
|
|
145
|
+
repos.calendarCollection.get(broken.accountConfigId, broken.calendarId),
|
|
146
|
+
);
|
|
147
|
+
assert.equal(stored.subscriptionError, "the feed answered HTTP 410");
|
|
148
|
+
assert.equal(stored.subscriptionCheckedAt, NOW);
|
|
149
|
+
const refreshed = await store.transaction((repos) =>
|
|
150
|
+
repos.calendarCollection.get(working.accountConfigId, working.calendarId),
|
|
151
|
+
);
|
|
152
|
+
assert.equal(refreshed.subscriptionFetchedAt, NOW);
|
|
153
|
+
assert.ok(lines.some((line) => line.includes(broken.calendarId)));
|
|
154
|
+
assert.equal(
|
|
155
|
+
lines.some((line) => line.includes("private-feedfacecafe")),
|
|
156
|
+
false,
|
|
157
|
+
);
|
|
158
|
+
});
|
|
159
|
+
});
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
import type { RemitClient } from "@remit/backend/client";
|
|
2
|
+
import {
|
|
3
|
+
type CalendarFeedFetcher,
|
|
4
|
+
isSubscriptionDue,
|
|
5
|
+
refreshCalendarSubscription,
|
|
6
|
+
} from "@remit/calendar-service";
|
|
7
|
+
import type {
|
|
8
|
+
ICalendarCollectionRepository,
|
|
9
|
+
ICalendarUnitOfWork,
|
|
10
|
+
} from "@remit/data-ports";
|
|
11
|
+
import type { Logger } from "@remit/logger-lambda";
|
|
12
|
+
import pMap from "p-map";
|
|
13
|
+
|
|
14
|
+
const REFRESH_CONCURRENCY = 4;
|
|
15
|
+
|
|
16
|
+
export interface CalendarSubscriptionTally {
|
|
17
|
+
refreshed: number;
|
|
18
|
+
failed: number;
|
|
19
|
+
notDue: number;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export interface CalendarSubscriptionRefreshDeps {
|
|
23
|
+
calendarCollection: Pick<
|
|
24
|
+
ICalendarCollectionRepository,
|
|
25
|
+
"listEnabledSubscriptions"
|
|
26
|
+
>;
|
|
27
|
+
calendarUnitOfWork: ICalendarUnitOfWork;
|
|
28
|
+
log: Logger;
|
|
29
|
+
fetcher?: CalendarFeedFetcher;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Re-reads every enabled subscription whose last fetch is at least
|
|
34
|
+
* `intervalMs` old. A disabled subscription is never listed, which is what
|
|
35
|
+
* stops its refresh while leaving its events in place.
|
|
36
|
+
*
|
|
37
|
+
* One feed's failure is recorded on its own collection and counted here; it
|
|
38
|
+
* never stops the others. The log names the calendar and the reason, never the
|
|
39
|
+
* URL, which for a Google secret address is the whole credential.
|
|
40
|
+
*/
|
|
41
|
+
export const refreshDueCalendarSubscriptions = async (
|
|
42
|
+
deps: CalendarSubscriptionRefreshDeps,
|
|
43
|
+
now: number,
|
|
44
|
+
intervalMs: number,
|
|
45
|
+
): Promise<CalendarSubscriptionTally> => {
|
|
46
|
+
const enabled = await deps.calendarCollection.listEnabledSubscriptions();
|
|
47
|
+
const due = enabled.filter((collection) =>
|
|
48
|
+
isSubscriptionDue(collection, now, intervalMs),
|
|
49
|
+
);
|
|
50
|
+
|
|
51
|
+
const outcomes = await pMap(
|
|
52
|
+
due,
|
|
53
|
+
(collection) =>
|
|
54
|
+
refreshCalendarSubscription(
|
|
55
|
+
deps.calendarUnitOfWork,
|
|
56
|
+
collection,
|
|
57
|
+
now,
|
|
58
|
+
deps.fetcher,
|
|
59
|
+
).then(
|
|
60
|
+
(refresh) => {
|
|
61
|
+
if (refresh.ok) return true;
|
|
62
|
+
deps.log.warn(
|
|
63
|
+
{ calendarId: collection.calendarId, reason: refresh.reason },
|
|
64
|
+
"Calendar subscription could not be refreshed",
|
|
65
|
+
);
|
|
66
|
+
return false;
|
|
67
|
+
},
|
|
68
|
+
(error: unknown) => {
|
|
69
|
+
deps.log.error(
|
|
70
|
+
{ calendarId: collection.calendarId, error },
|
|
71
|
+
"Calendar subscription refresh failed",
|
|
72
|
+
);
|
|
73
|
+
return false;
|
|
74
|
+
},
|
|
75
|
+
),
|
|
76
|
+
{ concurrency: REFRESH_CONCURRENCY },
|
|
77
|
+
);
|
|
78
|
+
|
|
79
|
+
return {
|
|
80
|
+
refreshed: outcomes.filter(Boolean).length,
|
|
81
|
+
failed: outcomes.filter((ok) => !ok).length,
|
|
82
|
+
notDue: enabled.length - due.length,
|
|
83
|
+
};
|
|
84
|
+
};
|
|
85
|
+
|
|
86
|
+
export const buildCalendarSubscriptionRefresh =
|
|
87
|
+
(
|
|
88
|
+
client: Pick<RemitClient, "calendarCollection" | "calendarUnitOfWork">,
|
|
89
|
+
log: Logger,
|
|
90
|
+
): ((
|
|
91
|
+
now: number,
|
|
92
|
+
intervalMs: number,
|
|
93
|
+
) => Promise<CalendarSubscriptionTally>) =>
|
|
94
|
+
(now, intervalMs) =>
|
|
95
|
+
refreshDueCalendarSubscriptions(
|
|
96
|
+
{
|
|
97
|
+
calendarCollection: client.calendarCollection,
|
|
98
|
+
calendarUnitOfWork: client.calendarUnitOfWork,
|
|
99
|
+
log,
|
|
100
|
+
},
|
|
101
|
+
now,
|
|
102
|
+
intervalMs,
|
|
103
|
+
);
|
package/src/scheduler/handler.ts
CHANGED
|
@@ -4,6 +4,7 @@ import { createQueueProducer } from "@remit/sqs-client/producer";
|
|
|
4
4
|
import type { ScheduledHandler } from "aws-lambda";
|
|
5
5
|
import { env } from "expect-env";
|
|
6
6
|
import { buildAttachmentSweep } from "./attachment-sweep.js";
|
|
7
|
+
import { buildCalendarSubscriptionRefresh } from "./calendar-subscriptions.js";
|
|
7
8
|
import { getOfflineIntervalMs, getTickIntervalMs } from "./config.js";
|
|
8
9
|
import { runSchedulerTick } from "./run-tick.js";
|
|
9
10
|
|
|
@@ -45,5 +46,6 @@ export const handler: ScheduledHandler = withTelemetry(async (event) => {
|
|
|
45
46
|
// to block storage, so nothing refuses an upload against a draft that is
|
|
46
47
|
// already gone and this sweep is the only thing that collects it.
|
|
47
48
|
sweepAttachments: buildAttachmentSweep(client, log),
|
|
49
|
+
refreshCalendarSubscriptions: buildCalendarSubscriptionRefresh(client, log),
|
|
48
50
|
});
|
|
49
51
|
});
|
|
@@ -2,6 +2,7 @@ import assert from "node:assert/strict";
|
|
|
2
2
|
import { describe, it } from "node:test";
|
|
3
3
|
import type { SendMessageCommand, SQSClient } from "@aws-sdk/client-sqs";
|
|
4
4
|
import type { AccountItem, AccountSchedulerPage } from "@remit/data-ports";
|
|
5
|
+
import { AccountService } from "@remit/domain-enums";
|
|
5
6
|
import type { Logger } from "@remit/logger-lambda";
|
|
6
7
|
import { runSchedulerTick } from "./run-tick.js";
|
|
7
8
|
|
|
@@ -62,6 +63,7 @@ const baseAccount = (overrides: Partial<AccountItem>): AccountItem =>
|
|
|
62
63
|
smtpTls: false,
|
|
63
64
|
smtpStartTls: true,
|
|
64
65
|
smtpUsername: "",
|
|
66
|
+
syncedServices: [AccountService.Mail],
|
|
65
67
|
isActive: true,
|
|
66
68
|
connectionState: "authenticated",
|
|
67
69
|
createdAt: NOW - 1_000_000,
|
|
@@ -364,3 +366,113 @@ describe("the attachment sweep inside a tick", () => {
|
|
|
364
366
|
assert.equal(result.sweepFailed, 0);
|
|
365
367
|
});
|
|
366
368
|
});
|
|
369
|
+
|
|
370
|
+
describe("runSchedulerTick and the account's synced services", () => {
|
|
371
|
+
const tickWith = async (syncedServices: AccountItem["syncedServices"]) => {
|
|
372
|
+
const accountService = fakeAccountService([
|
|
373
|
+
{
|
|
374
|
+
items: [baseAccount({ accountId: "acct_1", syncedServices })],
|
|
375
|
+
cursor: null,
|
|
376
|
+
},
|
|
377
|
+
]);
|
|
378
|
+
const { sqsClient, sent } = fakeSqsClient();
|
|
379
|
+
const result = await runSchedulerTick({
|
|
380
|
+
accountService,
|
|
381
|
+
sqsClient,
|
|
382
|
+
queueUrl: "https://queue.test/mailboxes",
|
|
383
|
+
log: createNoopLogger(),
|
|
384
|
+
tickIntervalMs: TICK_INTERVAL_MS,
|
|
385
|
+
offlineIntervalMs: OFFLINE_INTERVAL_MS,
|
|
386
|
+
now: NOW,
|
|
387
|
+
});
|
|
388
|
+
return { result, sent };
|
|
389
|
+
};
|
|
390
|
+
|
|
391
|
+
it("enqueues mail sync for a mail-only account", async () => {
|
|
392
|
+
const { result, sent } = await tickWith([AccountService.Mail]);
|
|
393
|
+
|
|
394
|
+
assert.equal(result.enqueued, 1);
|
|
395
|
+
assert.equal(sent.length, 1);
|
|
396
|
+
});
|
|
397
|
+
|
|
398
|
+
it("enqueues mail sync for an account syncing mail and calendar", async () => {
|
|
399
|
+
const { result, sent } = await tickWith([
|
|
400
|
+
AccountService.Mail,
|
|
401
|
+
AccountService.Calendar,
|
|
402
|
+
]);
|
|
403
|
+
|
|
404
|
+
assert.equal(result.enqueued, 1);
|
|
405
|
+
assert.equal(sent.length, 1);
|
|
406
|
+
});
|
|
407
|
+
|
|
408
|
+
it("skips a calendar-only account", async () => {
|
|
409
|
+
const { result, sent } = await tickWith([AccountService.Calendar]);
|
|
410
|
+
|
|
411
|
+
assert.equal(result.enqueued, 0);
|
|
412
|
+
assert.equal(result.skipped, 1);
|
|
413
|
+
assert.equal(sent.length, 0);
|
|
414
|
+
});
|
|
415
|
+
});
|
|
416
|
+
|
|
417
|
+
describe("the calendar subscription refresh inside a tick", () => {
|
|
418
|
+
const oneAccount = () => [
|
|
419
|
+
{
|
|
420
|
+
items: [baseAccount({ accountId: "acct_1", lastSyncAt: 0 })],
|
|
421
|
+
cursor: null,
|
|
422
|
+
} as unknown as AccountSchedulerPage,
|
|
423
|
+
];
|
|
424
|
+
|
|
425
|
+
it("runs once per tick, against the offline interval", async () => {
|
|
426
|
+
const { sqsClient } = fakeSqsClient();
|
|
427
|
+
const calls: [number, number][] = [];
|
|
428
|
+
|
|
429
|
+
const result = await runSchedulerTick({
|
|
430
|
+
accountService: fakeAccountService(oneAccount()),
|
|
431
|
+
sqsClient,
|
|
432
|
+
queueUrl: "https://queue.test/mailboxes",
|
|
433
|
+
log: createNoopLogger(),
|
|
434
|
+
tickIntervalMs: TICK_INTERVAL_MS,
|
|
435
|
+
offlineIntervalMs: OFFLINE_INTERVAL_MS,
|
|
436
|
+
now: NOW,
|
|
437
|
+
refreshCalendarSubscriptions: async (now, intervalMs) => {
|
|
438
|
+
calls.push([now, intervalMs]);
|
|
439
|
+
return { refreshed: 2, failed: 1, notDue: 4 };
|
|
440
|
+
},
|
|
441
|
+
});
|
|
442
|
+
|
|
443
|
+
assert.deepEqual(calls, [[NOW, OFFLINE_INTERVAL_MS]]);
|
|
444
|
+
assert.equal(result.subscriptionsRefreshed, 2);
|
|
445
|
+
assert.equal(result.subscriptionsFailed, 1);
|
|
446
|
+
assert.equal(result.enqueued, 1);
|
|
447
|
+
});
|
|
448
|
+
|
|
449
|
+
it("keeps the tick alive when the refresh throws", async () => {
|
|
450
|
+
const { sqsClient, sent } = fakeSqsClient();
|
|
451
|
+
const { log, calls } = createCapturingLogger();
|
|
452
|
+
|
|
453
|
+
const result = await runSchedulerTick({
|
|
454
|
+
accountService: fakeAccountService(oneAccount()),
|
|
455
|
+
sqsClient,
|
|
456
|
+
queueUrl: "https://queue.test/mailboxes",
|
|
457
|
+
log,
|
|
458
|
+
tickIntervalMs: TICK_INTERVAL_MS,
|
|
459
|
+
offlineIntervalMs: OFFLINE_INTERVAL_MS,
|
|
460
|
+
now: NOW,
|
|
461
|
+
refreshCalendarSubscriptions: async () => {
|
|
462
|
+
throw new Error("database is locked");
|
|
463
|
+
},
|
|
464
|
+
});
|
|
465
|
+
|
|
466
|
+
assert.equal(sent.length, 1);
|
|
467
|
+
assert.equal(result.subscriptionsRefreshed, 0);
|
|
468
|
+
assert.ok(
|
|
469
|
+
calls.some(
|
|
470
|
+
(entry) =>
|
|
471
|
+
entry.level === "error" &&
|
|
472
|
+
entry.args.some((arg) =>
|
|
473
|
+
String(arg).includes("Calendar subscription refresh failed"),
|
|
474
|
+
),
|
|
475
|
+
),
|
|
476
|
+
);
|
|
477
|
+
});
|
|
478
|
+
});
|
|
@@ -10,8 +10,10 @@ import pMap from "p-map";
|
|
|
10
10
|
import {
|
|
11
11
|
isAccountDeleted,
|
|
12
12
|
isAccountReauthRequired,
|
|
13
|
+
isMailSyncDisabled,
|
|
13
14
|
isUnsyncableHost,
|
|
14
15
|
} from "../account-check.js";
|
|
16
|
+
import type { CalendarSubscriptionTally } from "./calendar-subscriptions.js";
|
|
15
17
|
import {
|
|
16
18
|
SCHEDULER_ENQUEUE_CONCURRENCY,
|
|
17
19
|
SCHEDULER_PAGE_SIZE,
|
|
@@ -42,6 +44,15 @@ export interface RunSchedulerTickDeps {
|
|
|
42
44
|
* both real entry points pass it.
|
|
43
45
|
*/
|
|
44
46
|
sweepAttachments?: (account: AccountItem) => Promise<void>;
|
|
47
|
+
/**
|
|
48
|
+
* Re-read the calendar subscriptions whose last fetch is older than
|
|
49
|
+
* `offlineIntervalMs` — the same knob that says how stale mail may get.
|
|
50
|
+
* Optional only so a test can leave it out; both real entry points pass it.
|
|
51
|
+
*/
|
|
52
|
+
refreshCalendarSubscriptions?: (
|
|
53
|
+
now: number,
|
|
54
|
+
intervalMs: number,
|
|
55
|
+
) => Promise<CalendarSubscriptionTally>;
|
|
45
56
|
}
|
|
46
57
|
|
|
47
58
|
export interface SchedulerTickResult {
|
|
@@ -50,6 +61,8 @@ export interface SchedulerTickResult {
|
|
|
50
61
|
skipped: number;
|
|
51
62
|
swept: number;
|
|
52
63
|
sweepFailed: number;
|
|
64
|
+
subscriptionsRefreshed: number;
|
|
65
|
+
subscriptionsFailed: number;
|
|
53
66
|
}
|
|
54
67
|
|
|
55
68
|
/**
|
|
@@ -78,6 +91,7 @@ const silentLogger: Logger = (() => {
|
|
|
78
91
|
|
|
79
92
|
const isEligible = (account: AccountItem): boolean => {
|
|
80
93
|
if (isAccountDeleted(account, silentLogger)) return false;
|
|
94
|
+
if (isMailSyncDisabled(account, silentLogger)) return false;
|
|
81
95
|
if (isUnsyncableHost(account, silentLogger)) return false;
|
|
82
96
|
if (isAccountReauthRequired(account, silentLogger)) return false;
|
|
83
97
|
// An account storing no credential can only produce a sync that fails at
|
|
@@ -191,10 +205,43 @@ export const runSchedulerTick = async (
|
|
|
191
205
|
cursor = page.cursor ?? undefined;
|
|
192
206
|
} while (cursor);
|
|
193
207
|
|
|
208
|
+
let subscriptionsRefreshed = 0;
|
|
209
|
+
let subscriptionsFailed = 0;
|
|
210
|
+
if (deps.refreshCalendarSubscriptions) {
|
|
211
|
+
// After every page is enqueued, for the same reason the sweep runs after
|
|
212
|
+
// the enqueue: a slow feed must never cost an account its mail. Contained
|
|
213
|
+
// like the sweep, so an unreadable store costs a tick's refresh and not
|
|
214
|
+
// the scheduler.
|
|
215
|
+
const tally = await deps
|
|
216
|
+
.refreshCalendarSubscriptions(now, offlineIntervalMs)
|
|
217
|
+
.catch((error: unknown) => {
|
|
218
|
+
log.error({ error }, "Calendar subscription refresh failed");
|
|
219
|
+
return { refreshed: 0, failed: 0, notDue: 0 };
|
|
220
|
+
});
|
|
221
|
+
subscriptionsRefreshed = tally.refreshed;
|
|
222
|
+
subscriptionsFailed = tally.failed;
|
|
223
|
+
}
|
|
224
|
+
|
|
194
225
|
log.info(
|
|
195
|
-
{
|
|
226
|
+
{
|
|
227
|
+
scanned,
|
|
228
|
+
enqueued,
|
|
229
|
+
skipped,
|
|
230
|
+
swept,
|
|
231
|
+
sweepFailed,
|
|
232
|
+
subscriptionsRefreshed,
|
|
233
|
+
subscriptionsFailed,
|
|
234
|
+
},
|
|
196
235
|
"Scheduled-sync tick complete",
|
|
197
236
|
);
|
|
198
237
|
|
|
199
|
-
return {
|
|
238
|
+
return {
|
|
239
|
+
scanned,
|
|
240
|
+
enqueued,
|
|
241
|
+
skipped,
|
|
242
|
+
swept,
|
|
243
|
+
sweepFailed,
|
|
244
|
+
subscriptionsRefreshed,
|
|
245
|
+
subscriptionsFailed,
|
|
246
|
+
};
|
|
200
247
|
};
|
package/src/scheduler/runner.ts
CHANGED
|
@@ -7,6 +7,7 @@ import { createQueueProducer } from "@remit/sqs-client/producer";
|
|
|
7
7
|
|
|
8
8
|
import { env } from "expect-env";
|
|
9
9
|
import { buildAttachmentSweep } from "./attachment-sweep.js";
|
|
10
|
+
import { buildCalendarSubscriptionRefresh } from "./calendar-subscriptions.js";
|
|
10
11
|
import { getOfflineIntervalMs, getTickIntervalMs } from "./config.js";
|
|
11
12
|
import { runSchedulerLoop } from "./loop.js";
|
|
12
13
|
import { runSchedulerTick } from "./run-tick.js";
|
|
@@ -80,6 +81,10 @@ const runLoop = async (): Promise<void> => {
|
|
|
80
81
|
tickIntervalMs,
|
|
81
82
|
offlineIntervalMs,
|
|
82
83
|
sweepAttachments: buildAttachmentSweep(client, log),
|
|
84
|
+
refreshCalendarSubscriptions: buildCalendarSubscriptionRefresh(
|
|
85
|
+
client,
|
|
86
|
+
log,
|
|
87
|
+
),
|
|
83
88
|
},
|
|
84
89
|
heartbeat: createHeartbeat("tick"),
|
|
85
90
|
onHeartbeatError: onBeatError,
|