@remit/backend 0.0.7 → 0.0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json
CHANGED
|
@@ -62,6 +62,28 @@ describe("triggerSyncSafe", () => {
|
|
|
62
62
|
assert.equal(error.length, 0);
|
|
63
63
|
});
|
|
64
64
|
|
|
65
|
+
// The guarantee behind issue #37: POST /sync asks for a sync of this
|
|
66
|
+
// account by name, so the event it enqueues has to carry the mark that
|
|
67
|
+
// makes the worker's fan-out sync every mailbox rather than skip the fresh
|
|
68
|
+
// ones. Its callers are the refresh control, pull-to-refresh and the
|
|
69
|
+
// client's automatic poll — the poll is kept from abusing this by the
|
|
70
|
+
// interval floor in useStaleAccountSync, not by the server.
|
|
71
|
+
it("marks the event as an explicit request", async () => {
|
|
72
|
+
const sent: SendMessageCommand[] = [];
|
|
73
|
+
const { logger } = createLoggerSpy();
|
|
74
|
+
|
|
75
|
+
await triggerSyncSafe("acc-a", "config-1", {
|
|
76
|
+
sqsClient: okSqsClient(sent),
|
|
77
|
+
queueUrl: QUEUE_URL,
|
|
78
|
+
logger,
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
const body = JSON.parse(sent[0]?.input.MessageBody ?? "{}") as {
|
|
82
|
+
explicitRequest?: boolean;
|
|
83
|
+
};
|
|
84
|
+
assert.equal(body.explicitRequest, true);
|
|
85
|
+
});
|
|
86
|
+
|
|
65
87
|
it("does NOT reject when the SQS enqueue fails (POST /sync stays resilient)", async () => {
|
|
66
88
|
const econnrefused = Object.assign(new Error(""), {
|
|
67
89
|
name: "AggregateError",
|
package/src/handlers/sync.ts
CHANGED
|
@@ -54,6 +54,13 @@ export const triggerSyncSafe = async (
|
|
|
54
54
|
sqsClient: deps.sqsClient,
|
|
55
55
|
queueUrl: deps.queueUrl,
|
|
56
56
|
accountId,
|
|
57
|
+
// POST /sync asks for a sync of this account by name — the refresh
|
|
58
|
+
// control, pull-to-refresh, and the client's automatic poll all land
|
|
59
|
+
// here — so it syncs every mailbox regardless of how recently one
|
|
60
|
+
// ran. The poll's cadence is floored client-side at the fan-out
|
|
61
|
+
// gate's window, so this branch cannot be driven faster than the
|
|
62
|
+
// gate would allow.
|
|
63
|
+
explicitRequest: true,
|
|
57
64
|
});
|
|
58
65
|
deps.logger.info(
|
|
59
66
|
{ accountId, eventId },
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { describe, test } from "node:test";
|
|
3
|
+
import type {
|
|
4
|
+
AccountItem,
|
|
5
|
+
AccountSettingItem,
|
|
6
|
+
MailboxItem,
|
|
7
|
+
} from "@remit/data-ports";
|
|
8
|
+
import {
|
|
9
|
+
buildInboxMailboxMap,
|
|
10
|
+
buildListAllThreadsOptions,
|
|
11
|
+
buildListStarredThreadsOptions,
|
|
12
|
+
type InboxMapClient,
|
|
13
|
+
} from "./unified-threads.js";
|
|
14
|
+
|
|
15
|
+
// #44: Flagged filtered client-side over the newest 50 unified-inbox rows, so a
|
|
16
|
+
// star outside that window — or in any folder that is not INBOX — never showed.
|
|
17
|
+
// The starred scope is every non-muted mailbox, served by its own access
|
|
18
|
+
// pattern.
|
|
19
|
+
|
|
20
|
+
const CONFIG_ID = "cfg-1";
|
|
21
|
+
|
|
22
|
+
const account = (accountId: string): AccountItem =>
|
|
23
|
+
({ accountId }) as unknown as AccountItem;
|
|
24
|
+
|
|
25
|
+
const mailbox = (
|
|
26
|
+
mailboxId: string,
|
|
27
|
+
accountId: string,
|
|
28
|
+
fullPath: string,
|
|
29
|
+
specialUse?: string[],
|
|
30
|
+
): MailboxItem =>
|
|
31
|
+
({ mailboxId, accountId, fullPath, specialUse }) as unknown as MailboxItem;
|
|
32
|
+
|
|
33
|
+
const mutedSetting = (
|
|
34
|
+
name: "AccountMuted" | "MailboxMuted",
|
|
35
|
+
targetId: string,
|
|
36
|
+
): AccountSettingItem =>
|
|
37
|
+
({
|
|
38
|
+
name: `${name}#${targetId}`,
|
|
39
|
+
value: { kind: "MutedFlag", value: { value: true, setAt: 0 } },
|
|
40
|
+
}) as unknown as AccountSettingItem;
|
|
41
|
+
|
|
42
|
+
const buildClient = (
|
|
43
|
+
accounts: AccountItem[],
|
|
44
|
+
mailboxesByAccount: Record<string, MailboxItem[]>,
|
|
45
|
+
settings: AccountSettingItem[] = [],
|
|
46
|
+
): InboxMapClient =>
|
|
47
|
+
({
|
|
48
|
+
account: { listAllByAccountConfig: async () => accounts },
|
|
49
|
+
mailbox: {
|
|
50
|
+
listAllByAccount: async (accountId: string) =>
|
|
51
|
+
mailboxesByAccount[accountId] ?? [],
|
|
52
|
+
},
|
|
53
|
+
accountSetting: { listByAccountConfig: async () => settings },
|
|
54
|
+
}) as unknown as InboxMapClient;
|
|
55
|
+
|
|
56
|
+
describe("buildInboxMailboxMap", () => {
|
|
57
|
+
test("separates the INBOX scope from the wider starred scope", async () => {
|
|
58
|
+
const client = buildClient([account("a1")], {
|
|
59
|
+
a1: [
|
|
60
|
+
mailbox("m-inbox", "a1", "INBOX"),
|
|
61
|
+
mailbox("m-archive", "a1", "Archive", ["Archive"]),
|
|
62
|
+
mailbox("m-sub", "a1", "INBOX/Receipts"),
|
|
63
|
+
],
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
const { inboxMailboxIds, starredMailboxIds, mailboxIdToAccountId } =
|
|
67
|
+
await buildInboxMailboxMap(CONFIG_ID, client);
|
|
68
|
+
|
|
69
|
+
assert.deepEqual([...inboxMailboxIds], ["m-inbox"]);
|
|
70
|
+
assert.deepEqual([...starredMailboxIds].sort(), [
|
|
71
|
+
"m-archive",
|
|
72
|
+
"m-inbox",
|
|
73
|
+
"m-sub",
|
|
74
|
+
]);
|
|
75
|
+
// The map must cover every mailbox, or a starred row from Archive could
|
|
76
|
+
// not resolve its accountId.
|
|
77
|
+
assert.equal(mailboxIdToAccountId.get("m-archive"), "a1");
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
// A star on mail in Spam or Trash is not something the starred view should
|
|
81
|
+
// resurface; Gmail's All Mail is a second copy of everything, so including it
|
|
82
|
+
// renders every starred Gmail message twice.
|
|
83
|
+
test("the starred scope excludes All Mail, Junk and Trash", async () => {
|
|
84
|
+
const client = buildClient([account("a1")], {
|
|
85
|
+
a1: [
|
|
86
|
+
mailbox("m-inbox", "a1", "INBOX"),
|
|
87
|
+
mailbox("m-all", "a1", "[Gmail]/All Mail", ["All"]),
|
|
88
|
+
mailbox("m-junk", "a1", "[Gmail]/Spam", ["Junk"]),
|
|
89
|
+
mailbox("m-trash", "a1", "[Gmail]/Trash", ["Trash"]),
|
|
90
|
+
mailbox("m-sent", "a1", "[Gmail]/Sent Mail", ["Sent"]),
|
|
91
|
+
],
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
const { starredMailboxIds, mailboxIdToAccountId } =
|
|
95
|
+
await buildInboxMailboxMap(CONFIG_ID, client);
|
|
96
|
+
|
|
97
|
+
assert.deepEqual([...starredMailboxIds].sort(), ["m-inbox", "m-sent"]);
|
|
98
|
+
// Excluded mailboxes still resolve an accountId — the map is the wider set.
|
|
99
|
+
assert.equal(mailboxIdToAccountId.get("m-trash"), "a1");
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
test("a mailbox carrying several special uses is excluded on any one of them", async () => {
|
|
103
|
+
const client = buildClient([account("a1")], {
|
|
104
|
+
a1: [mailbox("m-archive-all", "a1", "Archive", ["Archive", "All"])],
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
const { starredMailboxIds } = await buildInboxMailboxMap(CONFIG_ID, client);
|
|
108
|
+
assert.equal(starredMailboxIds.size, 0);
|
|
109
|
+
});
|
|
110
|
+
|
|
111
|
+
test("muted mailboxes are excluded from both scopes", async () => {
|
|
112
|
+
const client = buildClient(
|
|
113
|
+
[account("a1")],
|
|
114
|
+
{
|
|
115
|
+
a1: [
|
|
116
|
+
mailbox("m-inbox", "a1", "INBOX"),
|
|
117
|
+
mailbox("m-muted", "a1", "Archive"),
|
|
118
|
+
],
|
|
119
|
+
},
|
|
120
|
+
[mutedSetting("MailboxMuted", "m-muted")],
|
|
121
|
+
);
|
|
122
|
+
|
|
123
|
+
const { inboxMailboxIds, starredMailboxIds } = await buildInboxMailboxMap(
|
|
124
|
+
CONFIG_ID,
|
|
125
|
+
client,
|
|
126
|
+
);
|
|
127
|
+
|
|
128
|
+
assert.deepEqual([...inboxMailboxIds], ["m-inbox"]);
|
|
129
|
+
assert.deepEqual([...starredMailboxIds], ["m-inbox"]);
|
|
130
|
+
});
|
|
131
|
+
|
|
132
|
+
test("a muted account contributes no mailboxes", async () => {
|
|
133
|
+
const client = buildClient(
|
|
134
|
+
[account("a1")],
|
|
135
|
+
{ a1: [mailbox("m-inbox", "a1", "INBOX")] },
|
|
136
|
+
[mutedSetting("AccountMuted", "a1")],
|
|
137
|
+
);
|
|
138
|
+
|
|
139
|
+
const { starredMailboxIds } = await buildInboxMailboxMap(CONFIG_ID, client);
|
|
140
|
+
assert.equal(starredMailboxIds.size, 0);
|
|
141
|
+
});
|
|
142
|
+
});
|
|
143
|
+
|
|
144
|
+
describe("buildListStarredThreadsOptions", () => {
|
|
145
|
+
test("scopes to every supplied mailbox, not just the inbox", () => {
|
|
146
|
+
const options = buildListStarredThreadsOptions(
|
|
147
|
+
{},
|
|
148
|
+
new Set(["m-inbox", "m-archive"]),
|
|
149
|
+
);
|
|
150
|
+
|
|
151
|
+
assert.deepEqual([...options.mailboxIds].sort(), ["m-archive", "m-inbox"]);
|
|
152
|
+
assert.equal(options.order, "desc");
|
|
153
|
+
assert.equal(options.excludeDeleted, true);
|
|
154
|
+
});
|
|
155
|
+
|
|
156
|
+
test("carries the caller's paging through", () => {
|
|
157
|
+
const options = buildListStarredThreadsOptions(
|
|
158
|
+
{ continuationToken: "tok", order: "asc", limit: 10 },
|
|
159
|
+
new Set(["m-inbox"]),
|
|
160
|
+
);
|
|
161
|
+
|
|
162
|
+
assert.equal(options.continuationToken, "tok");
|
|
163
|
+
assert.equal(options.order, "asc");
|
|
164
|
+
assert.equal(options.limit, 10);
|
|
165
|
+
});
|
|
166
|
+
|
|
167
|
+
test("defaults the page size to the unified default", () => {
|
|
168
|
+
const starred = buildListStarredThreadsOptions({}, new Set(["m-inbox"]));
|
|
169
|
+
const unified = buildListAllThreadsOptions({}, new Set(["m-inbox"]));
|
|
170
|
+
assert.equal(starred.limit, unified.limit);
|
|
171
|
+
});
|
|
172
|
+
});
|
|
@@ -3,6 +3,7 @@ import type {
|
|
|
3
3
|
IAccountSettingRepository,
|
|
4
4
|
MailboxItem,
|
|
5
5
|
} from "@remit/data-ports";
|
|
6
|
+
import { MailboxSpecialUse } from "@remit/domain-enums";
|
|
6
7
|
import type { APIGatewayProxyEvent } from "aws-lambda";
|
|
7
8
|
import type { Context } from "openapi-backend";
|
|
8
9
|
import pMap from "p-map";
|
|
@@ -34,8 +35,34 @@ export interface InboxMapClient {
|
|
|
34
35
|
}
|
|
35
36
|
|
|
36
37
|
/**
|
|
37
|
-
*
|
|
38
|
-
*
|
|
38
|
+
* Special-use folders a star never surfaces from.
|
|
39
|
+
*
|
|
40
|
+
* `Junk` and `Trash` match what Gmail and Fastmail do — a star on mail the user
|
|
41
|
+
* already threw away or that was classed as spam is not something the starred
|
|
42
|
+
* view should resurface. `All` is Gmail's All Mail: a second copy of everything
|
|
43
|
+
* already reachable through its own folder, and because a message's identity
|
|
44
|
+
* includes its mailbox, the copy is a distinct row carrying the same
|
|
45
|
+
* server-side \Flagged. Including it would render every starred Gmail message
|
|
46
|
+
* twice.
|
|
47
|
+
*/
|
|
48
|
+
const STARRED_EXCLUDED_SPECIAL_USE: readonly string[] = [
|
|
49
|
+
MailboxSpecialUse.All,
|
|
50
|
+
MailboxSpecialUse.Junk,
|
|
51
|
+
MailboxSpecialUse.Trash,
|
|
52
|
+
];
|
|
53
|
+
|
|
54
|
+
const isExcludedFromStarred = (mailbox: MailboxItem): boolean =>
|
|
55
|
+
mailbox.specialUse?.some((use) =>
|
|
56
|
+
STARRED_EXCLUDED_SPECIAL_USE.includes(use),
|
|
57
|
+
) === true;
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Build the read scope for the unified listing: a mailboxId→accountId map over
|
|
61
|
+
* every non-muted mailbox of every non-muted account in a given
|
|
62
|
+
* accountConfigId, plus two id sets — `inboxMailboxIds` (top-level INBOX only,
|
|
63
|
+
* the unified inbox scope) and `starredMailboxIds` (every folder a star may
|
|
64
|
+
* surface from, see `STARRED_EXCLUDED_SPECIAL_USE`). The map covers all
|
|
65
|
+
* mailboxes, excluded ones included, so any row still resolves its accountId.
|
|
39
66
|
*
|
|
40
67
|
* INBOX is identified by exact match `fullPath.toUpperCase() === "INBOX"` —
|
|
41
68
|
* MailboxSpecialUse has no Inbox value per RFC 6154. This is the same rule
|
|
@@ -67,6 +94,7 @@ export const buildInboxMailboxMap = async (
|
|
|
67
94
|
): Promise<{
|
|
68
95
|
mailboxIdToAccountId: Map<string, string>;
|
|
69
96
|
inboxMailboxIds: Set<string>;
|
|
97
|
+
starredMailboxIds: Set<string>;
|
|
70
98
|
}> => {
|
|
71
99
|
const [accounts, settings] = await Promise.all([
|
|
72
100
|
client.account.listAllByAccountConfig(accountConfigId),
|
|
@@ -94,21 +122,23 @@ export const buildInboxMailboxMap = async (
|
|
|
94
122
|
|
|
95
123
|
const mailboxIdToAccountId = new Map<string, string>();
|
|
96
124
|
const inboxMailboxIds = new Set<string>();
|
|
125
|
+
const starredMailboxIds = new Set<string>();
|
|
97
126
|
|
|
98
|
-
const
|
|
127
|
+
const activeMailboxes = mailboxLists
|
|
99
128
|
.flat()
|
|
100
|
-
.filter(
|
|
101
|
-
(mailbox) =>
|
|
102
|
-
!isMailboxMuted(mailbox.mailboxId) &&
|
|
103
|
-
mailbox.fullPath.toUpperCase() === "INBOX",
|
|
104
|
-
);
|
|
129
|
+
.filter((mailbox) => !isMailboxMuted(mailbox.mailboxId));
|
|
105
130
|
|
|
106
|
-
for (const mailbox of
|
|
131
|
+
for (const mailbox of activeMailboxes) {
|
|
107
132
|
mailboxIdToAccountId.set(mailbox.mailboxId, mailbox.accountId);
|
|
108
|
-
|
|
133
|
+
if (!isExcludedFromStarred(mailbox)) {
|
|
134
|
+
starredMailboxIds.add(mailbox.mailboxId);
|
|
135
|
+
}
|
|
136
|
+
if (mailbox.fullPath.toUpperCase() === "INBOX") {
|
|
137
|
+
inboxMailboxIds.add(mailbox.mailboxId);
|
|
138
|
+
}
|
|
109
139
|
}
|
|
110
140
|
|
|
111
|
-
return { mailboxIdToAccountId, inboxMailboxIds };
|
|
141
|
+
return { mailboxIdToAccountId, inboxMailboxIds, starredMailboxIds };
|
|
112
142
|
};
|
|
113
143
|
|
|
114
144
|
/**
|
|
@@ -134,6 +164,28 @@ export const buildListAllThreadsOptions = (
|
|
|
134
164
|
excludeDeleted: true,
|
|
135
165
|
});
|
|
136
166
|
|
|
167
|
+
/**
|
|
168
|
+
* Build listByStarred options for the starred (`starred=true`) listing.
|
|
169
|
+
*
|
|
170
|
+
* A star marks the mail, not its placement, so the INBOX narrowing does not
|
|
171
|
+
* apply: the scope is every non-muted mailbox except the ones a star never
|
|
172
|
+
* surfaces from. Same defaults as the unified listing otherwise.
|
|
173
|
+
*/
|
|
174
|
+
export const buildListStarredThreadsOptions = (
|
|
175
|
+
query: {
|
|
176
|
+
continuationToken?: string;
|
|
177
|
+
order?: "asc" | "desc";
|
|
178
|
+
limit?: number;
|
|
179
|
+
},
|
|
180
|
+
starredMailboxIds: Set<string>,
|
|
181
|
+
) => ({
|
|
182
|
+
order: query.order ?? ("desc" as const),
|
|
183
|
+
continuationToken: query.continuationToken,
|
|
184
|
+
limit: query.limit ?? DEFAULT_UNIFIED_THREADS_PAGE_SIZE,
|
|
185
|
+
mailboxIds: starredMailboxIds,
|
|
186
|
+
excludeDeleted: true,
|
|
187
|
+
});
|
|
188
|
+
|
|
137
189
|
/**
|
|
138
190
|
* Attach accountId to each enriched ThreadMessageResponse row using the
|
|
139
191
|
* mailboxId→accountId map built from inbox discovery. Same read-time-attach
|
|
@@ -158,28 +210,40 @@ export const UnifiedThreadOperations: Record<
|
|
|
158
210
|
) => {
|
|
159
211
|
const event = args[0] as APIGatewayProxyEvent;
|
|
160
212
|
const accountConfigId = getAccountConfigIdFromEvent(event);
|
|
161
|
-
const { continuationToken, order, limit } = context.request
|
|
213
|
+
const { continuationToken, order, limit, starred } = context.request
|
|
214
|
+
.query as {
|
|
162
215
|
continuationToken?: string;
|
|
163
216
|
order?: "asc" | "desc";
|
|
164
217
|
limit?: number;
|
|
218
|
+
starred?: boolean | string;
|
|
165
219
|
};
|
|
220
|
+
const starredOnly = starred === true || starred === "true";
|
|
166
221
|
|
|
167
222
|
const client = await getClient();
|
|
168
223
|
|
|
169
|
-
const { mailboxIdToAccountId, inboxMailboxIds } =
|
|
224
|
+
const { mailboxIdToAccountId, inboxMailboxIds, starredMailboxIds } =
|
|
170
225
|
await buildInboxMailboxMap(accountConfigId, client);
|
|
171
226
|
|
|
172
|
-
|
|
227
|
+
const scope = starredOnly ? starredMailboxIds : inboxMailboxIds;
|
|
228
|
+
if (scope.size === 0) {
|
|
173
229
|
return { items: [], continuationToken: undefined };
|
|
174
230
|
}
|
|
175
231
|
|
|
176
|
-
const result =
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
232
|
+
const result = starredOnly
|
|
233
|
+
? await client.threadMessage.listByStarred(
|
|
234
|
+
accountConfigId,
|
|
235
|
+
buildListStarredThreadsOptions(
|
|
236
|
+
{ continuationToken, order, limit },
|
|
237
|
+
starredMailboxIds,
|
|
238
|
+
),
|
|
239
|
+
)
|
|
240
|
+
: await client.threadMessage.listByDate(
|
|
241
|
+
accountConfigId,
|
|
242
|
+
buildListAllThreadsOptions(
|
|
243
|
+
{ continuationToken, order, limit },
|
|
244
|
+
inboxMailboxIds,
|
|
245
|
+
),
|
|
246
|
+
);
|
|
183
247
|
|
|
184
248
|
const enriched = await enrichThreadRows(
|
|
185
249
|
result.items,
|
|
@@ -2,7 +2,6 @@ import assert from "node:assert/strict";
|
|
|
2
2
|
import { describe, it } from "node:test";
|
|
3
3
|
import { SendMessageCommand, type SQSClient } from "@aws-sdk/client-sqs";
|
|
4
4
|
import {
|
|
5
|
-
buildManualSyncDedupId,
|
|
6
5
|
buildScheduledSyncDedupId,
|
|
7
6
|
buildSyncMailboxesCommand,
|
|
8
7
|
triggerAccountSync,
|
|
@@ -29,20 +28,62 @@ describe("buildSyncMailboxesCommand", () => {
|
|
|
29
28
|
assert.equal(cmd.input.MessageGroupId, "account-abc");
|
|
30
29
|
});
|
|
31
30
|
|
|
32
|
-
it("defaults MessageDeduplicationId to the
|
|
31
|
+
it("defaults MessageDeduplicationId to the event's own id for FIFO queues", () => {
|
|
33
32
|
const cmd = buildSyncMailboxesCommand({
|
|
34
33
|
sqsClient: {} as SQSClient,
|
|
35
34
|
queueUrl: FIFO_QUEUE_URL,
|
|
36
35
|
accountId: "account-abc",
|
|
37
36
|
});
|
|
38
37
|
|
|
39
|
-
assert.
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
38
|
+
assert.equal(cmd.input.MessageDeduplicationId, parseBody(cmd).eventId);
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
// Issue #37: a shared, time-bucketed dedup id let SQS FIFO's 5-minute
|
|
42
|
+
// window discard the second sync of an account, so mail that arrived after
|
|
43
|
+
// a sync stayed invisible until the window elapsed. Back-to-back triggers
|
|
44
|
+
// must each reach the queue.
|
|
45
|
+
it("gives two triggers for one account distinct dedup ids", () => {
|
|
46
|
+
const first = buildSyncMailboxesCommand({
|
|
47
|
+
sqsClient: {} as SQSClient,
|
|
48
|
+
queueUrl: FIFO_QUEUE_URL,
|
|
49
|
+
accountId: "account-abc",
|
|
50
|
+
});
|
|
51
|
+
const second = buildSyncMailboxesCommand({
|
|
52
|
+
sqsClient: {} as SQSClient,
|
|
53
|
+
queueUrl: FIFO_QUEUE_URL,
|
|
54
|
+
accountId: "account-abc",
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
assert.notEqual(
|
|
58
|
+
first.input.MessageDeduplicationId,
|
|
59
|
+
second.input.MessageDeduplicationId,
|
|
43
60
|
);
|
|
44
61
|
});
|
|
45
62
|
|
|
63
|
+
// The worker's fan-out gate reads this off the event: it is what lets a
|
|
64
|
+
// sync asked for by name cover every folder while a side-effect trigger
|
|
65
|
+
// skips the ones that were just enumerated.
|
|
66
|
+
it("marks an explicitly-requested trigger on the event", () => {
|
|
67
|
+
const cmd = buildSyncMailboxesCommand({
|
|
68
|
+
sqsClient: {} as SQSClient,
|
|
69
|
+
queueUrl: FIFO_QUEUE_URL,
|
|
70
|
+
accountId: "account-abc",
|
|
71
|
+
explicitRequest: true,
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
assert.equal(parseBody(cmd).explicitRequest, true);
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
it("leaves a side-effect trigger unmarked", () => {
|
|
78
|
+
const cmd = buildSyncMailboxesCommand({
|
|
79
|
+
sqsClient: {} as SQSClient,
|
|
80
|
+
queueUrl: FIFO_QUEUE_URL,
|
|
81
|
+
accountId: "account-abc",
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
assert.equal(parseBody(cmd).explicitRequest, undefined);
|
|
85
|
+
});
|
|
86
|
+
|
|
46
87
|
it("does not set FIFO params for standard queues", () => {
|
|
47
88
|
const cmd = buildSyncMailboxesCommand({
|
|
48
89
|
sqsClient: {} as SQSClient,
|
|
@@ -93,43 +134,6 @@ describe("buildSyncMailboxesCommand", () => {
|
|
|
93
134
|
});
|
|
94
135
|
});
|
|
95
136
|
|
|
96
|
-
describe("buildManualSyncDedupId", () => {
|
|
97
|
-
const SIXTY_SECONDS_MS = 60 * 1000;
|
|
98
|
-
|
|
99
|
-
it("is stable within the same time bucket (dedupes a rapid double-tap)", () => {
|
|
100
|
-
const bucketStart = 10 * SIXTY_SECONDS_MS;
|
|
101
|
-
|
|
102
|
-
const first = buildManualSyncDedupId("account-abc", bucketStart);
|
|
103
|
-
const second = buildManualSyncDedupId("account-abc", bucketStart + 1_000);
|
|
104
|
-
|
|
105
|
-
assert.equal(first, second);
|
|
106
|
-
});
|
|
107
|
-
|
|
108
|
-
it("changes across a bucket boundary (never dedupes two polls a minute apart)", () => {
|
|
109
|
-
const bucketStart = 10 * SIXTY_SECONDS_MS;
|
|
110
|
-
|
|
111
|
-
const thisTrigger = buildManualSyncDedupId("account-abc", bucketStart);
|
|
112
|
-
const nextTrigger = buildManualSyncDedupId(
|
|
113
|
-
"account-abc",
|
|
114
|
-
bucketStart + SIXTY_SECONDS_MS,
|
|
115
|
-
);
|
|
116
|
-
|
|
117
|
-
assert.notEqual(thisTrigger, nextTrigger);
|
|
118
|
-
});
|
|
119
|
-
|
|
120
|
-
it("never collides with the scheduler's dedup namespace", () => {
|
|
121
|
-
const now = 10 * SIXTY_SECONDS_MS;
|
|
122
|
-
const manual = buildManualSyncDedupId("account-abc", now);
|
|
123
|
-
const scheduled = buildScheduledSyncDedupId(
|
|
124
|
-
"account-abc",
|
|
125
|
-
now,
|
|
126
|
-
SIXTY_SECONDS_MS,
|
|
127
|
-
);
|
|
128
|
-
|
|
129
|
-
assert.notEqual(manual, scheduled);
|
|
130
|
-
});
|
|
131
|
-
});
|
|
132
|
-
|
|
133
137
|
describe("buildScheduledSyncDedupId", () => {
|
|
134
138
|
const FIVE_MINUTES_MS = 5 * 60 * 1000;
|
|
135
139
|
|
|
@@ -6,6 +6,7 @@ interface SyncMailboxesEvent {
|
|
|
6
6
|
eventId: string;
|
|
7
7
|
timestamp: number;
|
|
8
8
|
accountId: string;
|
|
9
|
+
explicitRequest?: boolean;
|
|
9
10
|
}
|
|
10
11
|
|
|
11
12
|
interface TriggerAccountSyncInput {
|
|
@@ -13,45 +14,42 @@ interface TriggerAccountSyncInput {
|
|
|
13
14
|
queueUrl: string;
|
|
14
15
|
accountId: string;
|
|
15
16
|
/**
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
* while distinct triggers a bucket or more apart always pass through.
|
|
17
|
+
* Set by POST /sync, whose callers ask for a sync of one named account: the
|
|
18
|
+
* refresh control, pull-to-refresh, and the web client's automatic poll
|
|
19
|
+
* (`useStaleAccountSync`) — a timer, not a person. It travels on the event
|
|
20
|
+
* and makes the worker's fan-out sync every mailbox even if one just ran.
|
|
21
21
|
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
22
|
+
* Everything else triggers a sync as a side effect of doing something else
|
|
23
|
+
* (config load, OAuth connect, account create, the scheduled tick), leaves
|
|
24
|
+
* this unset, and takes the freshness gate — see `mailboxNeedsSync` in
|
|
25
|
+
* imap-worker's sync-mailboxes handler.
|
|
26
|
+
*
|
|
27
|
+
* The poll cannot use this to outrun the gate: its interval is floored at
|
|
28
|
+
* the gate's own window (`MIN_POLL_INTERVAL_MS` in the client hook), so
|
|
29
|
+
* however low `mailboxPollIntervalSeconds` is set, a timer never fans out
|
|
30
|
+
* more often than the gate would have allowed anyway.
|
|
31
|
+
*/
|
|
32
|
+
explicitRequest?: boolean;
|
|
33
|
+
/**
|
|
34
|
+
* Override the FIFO `MessageDeduplicationId`. Defaults to the event's own
|
|
35
|
+
* id, so the queue suppresses a re-send of one event and nothing else —
|
|
36
|
+
* every manual call site (POST /sync, OAuth connect, config load,
|
|
37
|
+
* pull-to-refresh, the client's online-poll) always enqueues. A shared,
|
|
38
|
+
* time-bucketed id instead turned SQS's 5-minute window into a rate limiter
|
|
39
|
+
* and silently discarded a sync the user asked for whenever one had run
|
|
40
|
+
* recently (issue #37). What a trigger costs is bounded in the worker's
|
|
41
|
+
* fan-out (`mailboxNeedsSync`), by skipping mailboxes not worth
|
|
42
|
+
* re-enumerating — never by dropping the trigger.
|
|
43
|
+
*
|
|
44
|
+
* The scheduled-sync tick (#1247) passes `buildScheduledSyncDedupId()`,
|
|
45
|
+
* which is bucketed by the tick's own cadence: there it collapses a
|
|
46
|
+
* re-invocation of a single tick, never two ticks or a manual trigger.
|
|
25
47
|
*/
|
|
26
48
|
dedupId?: string;
|
|
27
49
|
}
|
|
28
50
|
|
|
29
51
|
const isFifoQueue = (queueUrl: string): boolean => queueUrl.endsWith(".fifo");
|
|
30
52
|
|
|
31
|
-
/**
|
|
32
|
-
* Bucket window for the manual-trigger dedup id. SQS FIFO's own dedup window
|
|
33
|
-
* is 5 minutes; a static per-account id (the pre-#1250 behaviour) collided
|
|
34
|
-
* with the client's 5-minute online poll and swallowed a pull-to-refresh
|
|
35
|
-
* within 5 minutes of any prior trigger. Bucketing to ~60s keeps a rapid
|
|
36
|
-
* double-tap deduped while letting distinct polls a minute or more apart
|
|
37
|
-
* always enqueue.
|
|
38
|
-
*/
|
|
39
|
-
const MANUAL_DEDUP_BUCKET_MS = 60_000;
|
|
40
|
-
|
|
41
|
-
/**
|
|
42
|
-
* Build the manual-trigger dedup id (POST /sync, OAuth connect, config load,
|
|
43
|
-
* pull-to-refresh, client online-poll), bucketed so consecutive triggers a
|
|
44
|
-
* bucket apart never collide — see `dedupId` above.
|
|
45
|
-
*/
|
|
46
|
-
export const buildManualSyncDedupId = (
|
|
47
|
-
accountId: string,
|
|
48
|
-
now: number,
|
|
49
|
-
bucketMs: number = MANUAL_DEDUP_BUCKET_MS,
|
|
50
|
-
): string => {
|
|
51
|
-
const bucket = Math.floor(now / bucketMs);
|
|
52
|
-
return `SYNC_MAILBOXES:manual:${accountId}:${bucket}`;
|
|
53
|
-
};
|
|
54
|
-
|
|
55
53
|
/**
|
|
56
54
|
* Build the scheduler's own dedup namespace, bucketed by tick interval so
|
|
57
55
|
* consecutive ticks each get a fresh id (never colliding with each other)
|
|
@@ -70,12 +68,13 @@ export const buildScheduledSyncDedupId = (
|
|
|
70
68
|
export const buildSyncMailboxesCommand = (
|
|
71
69
|
input: TriggerAccountSyncInput,
|
|
72
70
|
): SendMessageCommand => {
|
|
73
|
-
const { queueUrl, accountId, dedupId } = input;
|
|
71
|
+
const { queueUrl, accountId, dedupId, explicitRequest } = input;
|
|
74
72
|
const event: SyncMailboxesEvent = {
|
|
75
73
|
type: "SYNC_MAILBOXES",
|
|
76
74
|
eventId: randomUUID(),
|
|
77
75
|
timestamp: Date.now(),
|
|
78
76
|
accountId,
|
|
77
|
+
...(explicitRequest && { explicitRequest }),
|
|
79
78
|
};
|
|
80
79
|
|
|
81
80
|
const useFifo = isFifoQueue(queueUrl);
|
|
@@ -85,8 +84,7 @@ export const buildSyncMailboxesCommand = (
|
|
|
85
84
|
MessageBody: JSON.stringify(event),
|
|
86
85
|
...(useFifo && {
|
|
87
86
|
MessageGroupId: accountId,
|
|
88
|
-
MessageDeduplicationId:
|
|
89
|
-
dedupId ?? buildManualSyncDedupId(accountId, event.timestamp),
|
|
87
|
+
MessageDeduplicationId: dedupId ?? event.eventId,
|
|
90
88
|
}),
|
|
91
89
|
});
|
|
92
90
|
};
|