@remit/backend 0.0.43 → 0.0.45
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dev-server/log-output.test.ts +173 -0
- package/dev-server/server.ts +22 -11
- package/package.json +1 -1
- package/scripts/backfill-list-id.ts +3 -2
- package/src/derive/enrichThreadRows.ts +12 -18
- package/src/derive/filterThreadCriteria.test.ts +4 -8
- package/src/derive/filterThreadCriteria.ts +11 -20
- package/src/handlers/thread.test.ts +162 -0
- package/src/handlers/thread.ts +3 -1
- package/src/handlers/unified-threads.ts +1 -1
- package/src/index.ts +2 -2
- package/tsconfig.json +12 -1
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { spawn } from "node:child_process";
|
|
3
|
+
import { createServer } from "node:net";
|
|
4
|
+
import { dirname, resolve } from "node:path";
|
|
5
|
+
import { after, before, describe, it } from "node:test";
|
|
6
|
+
import { fileURLToPath } from "node:url";
|
|
7
|
+
|
|
8
|
+
// This file is the backend image's entrypoint, so what it writes IS the
|
|
9
|
+
// container's log stream. deploy/vps/README.md ("Logs") promises one JSON object
|
|
10
|
+
// per line, and a log-shipping pipeline written against that contract breaks on
|
|
11
|
+
// the first line it cannot parse — which is why this runs the real server rather
|
|
12
|
+
// than asserting against a mocked writer.
|
|
13
|
+
//
|
|
14
|
+
// Both streams are held to it. The container log driver merges stdout and
|
|
15
|
+
// stderr into one log and `remit logs` shows both, so a raw line on stderr
|
|
16
|
+
// breaks a Vector parser exactly like a raw line on stdout.
|
|
17
|
+
|
|
18
|
+
const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
|
19
|
+
|
|
20
|
+
const freePort = (): Promise<number> =>
|
|
21
|
+
new Promise((resolveWith, reject) => {
|
|
22
|
+
const probe = createServer();
|
|
23
|
+
probe.on("error", reject);
|
|
24
|
+
probe.listen(0, "127.0.0.1", () => {
|
|
25
|
+
const address = probe.address();
|
|
26
|
+
if (address === null || typeof address === "string") {
|
|
27
|
+
probe.close();
|
|
28
|
+
reject(new Error("could not reserve a port"));
|
|
29
|
+
return;
|
|
30
|
+
}
|
|
31
|
+
probe.close(() => resolveWith(address.port));
|
|
32
|
+
});
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
type Line = Record<string, unknown>;
|
|
36
|
+
type Stream = "stdout" | "stderr";
|
|
37
|
+
|
|
38
|
+
const captured: Record<Stream, string> = { stdout: "", stderr: "" };
|
|
39
|
+
const listeners = new Set<() => void>();
|
|
40
|
+
let port = 0;
|
|
41
|
+
|
|
42
|
+
// The child's output reaches this process asynchronously, so a request that has
|
|
43
|
+
// already been answered is not yet a line here. Wait for the line rather than
|
|
44
|
+
// for a duration, and only once the stream ends on a newline, so no assertion
|
|
45
|
+
// ever runs against half of one.
|
|
46
|
+
const waitForStdout = (contains: string): Promise<void> =>
|
|
47
|
+
new Promise((resolveWith, reject) => {
|
|
48
|
+
const settled = () =>
|
|
49
|
+
captured.stdout.includes(contains) && captured.stdout.endsWith("\n");
|
|
50
|
+
if (settled()) {
|
|
51
|
+
resolveWith();
|
|
52
|
+
return;
|
|
53
|
+
}
|
|
54
|
+
const timer = setTimeout(() => {
|
|
55
|
+
stopWaiting();
|
|
56
|
+
reject(new Error(`stdout never carried ${contains}: ${captured.stdout}`));
|
|
57
|
+
}, 10_000);
|
|
58
|
+
const onData = () => {
|
|
59
|
+
if (!settled()) return;
|
|
60
|
+
stopWaiting();
|
|
61
|
+
resolveWith();
|
|
62
|
+
};
|
|
63
|
+
const stopWaiting = () => {
|
|
64
|
+
clearTimeout(timer);
|
|
65
|
+
listeners.delete(onData);
|
|
66
|
+
};
|
|
67
|
+
listeners.add(onData);
|
|
68
|
+
});
|
|
69
|
+
|
|
70
|
+
const parsedLines = (stream: Stream): Line[] =>
|
|
71
|
+
captured[stream]
|
|
72
|
+
.split("\n")
|
|
73
|
+
.filter((line) => line.trim().length > 0)
|
|
74
|
+
.map((line, index) => {
|
|
75
|
+
try {
|
|
76
|
+
return JSON.parse(line) as Line;
|
|
77
|
+
} catch {
|
|
78
|
+
throw new assert.AssertionError({
|
|
79
|
+
message: `${stream} line ${index + 1} is not JSON, so a log pipeline drops it: ${line}`,
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
describe("the backend entrypoint's log output", () => {
|
|
85
|
+
let server: ReturnType<typeof spawn>;
|
|
86
|
+
|
|
87
|
+
before(async () => {
|
|
88
|
+
port = await freePort();
|
|
89
|
+
|
|
90
|
+
server = spawn(
|
|
91
|
+
process.execPath,
|
|
92
|
+
["--import", "tsx", "dev-server/server.ts"],
|
|
93
|
+
{
|
|
94
|
+
cwd: packageRoot,
|
|
95
|
+
stdio: ["ignore", "pipe", "pipe", "ipc"],
|
|
96
|
+
env: {
|
|
97
|
+
...process.env,
|
|
98
|
+
SERVER_PORT: String(port),
|
|
99
|
+
LOG_LEVEL: "debug",
|
|
100
|
+
REMIT_SERVICE_NAME: "backend",
|
|
101
|
+
// tsx's own loader warnings are an artifact of running the
|
|
102
|
+
// TypeScript source; the image runs a bundle and never emits them.
|
|
103
|
+
NODE_NO_WARNINGS: "1",
|
|
104
|
+
NODE_OPTIONS: "",
|
|
105
|
+
},
|
|
106
|
+
},
|
|
107
|
+
);
|
|
108
|
+
|
|
109
|
+
for (const stream of ["stdout", "stderr"] as const) {
|
|
110
|
+
server[stream]?.setEncoding("utf8");
|
|
111
|
+
server[stream]?.on("data", (chunk: string) => {
|
|
112
|
+
captured[stream] += chunk;
|
|
113
|
+
for (const listener of [...listeners]) listener();
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
await new Promise<void>((resolveWith, reject) => {
|
|
118
|
+
server.on("message", (message) => {
|
|
119
|
+
if (message === "ready") resolveWith();
|
|
120
|
+
});
|
|
121
|
+
// Carry the child's stderr into the failure: a server that cannot boot
|
|
122
|
+
// otherwise reports only an exit code, and the reason is in that buffer.
|
|
123
|
+
server.on("exit", (code) =>
|
|
124
|
+
reject(
|
|
125
|
+
new Error(
|
|
126
|
+
`server exited before listening (code ${code})\n${captured.stderr}`,
|
|
127
|
+
),
|
|
128
|
+
),
|
|
129
|
+
);
|
|
130
|
+
});
|
|
131
|
+
|
|
132
|
+
// A request the router answers without touching a data backend, so the
|
|
133
|
+
// invocation's own lines — the ones withTelemetry writes around the
|
|
134
|
+
// handler — are captured by the time this suite reads them.
|
|
135
|
+
await fetch(`http://127.0.0.1:${port}/no-such-route`);
|
|
136
|
+
await waitForStdout("Request received");
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
after(() => {
|
|
140
|
+
server.kill("SIGKILL");
|
|
141
|
+
});
|
|
142
|
+
|
|
143
|
+
it("is JSON on every line of both streams", () => {
|
|
144
|
+
const lines = [...parsedLines("stdout"), ...parsedLines("stderr")];
|
|
145
|
+
assert.ok(lines.length > 0, "expected the server to have written a line");
|
|
146
|
+
for (const line of lines) {
|
|
147
|
+
assert.equal(typeof line.level, "string");
|
|
148
|
+
assert.equal(typeof line.time, "string");
|
|
149
|
+
assert.equal(line.service, "backend");
|
|
150
|
+
assert.equal(typeof line.msg, "string");
|
|
151
|
+
}
|
|
152
|
+
});
|
|
153
|
+
|
|
154
|
+
it("reports what it is listening on as fields, not a banner", () => {
|
|
155
|
+
const listening = parsedLines("stdout").find(
|
|
156
|
+
(line) => line.msg === "Backend listening",
|
|
157
|
+
);
|
|
158
|
+
assert.ok(listening, "expected a startup line");
|
|
159
|
+
assert.equal(listening.port, port);
|
|
160
|
+
assert.equal(listening.url, `http://localhost:${port}`);
|
|
161
|
+
assert.equal(listening.level, "info");
|
|
162
|
+
});
|
|
163
|
+
|
|
164
|
+
it("correlates the whole invocation under one requestId", () => {
|
|
165
|
+
const lines = parsedLines("stdout").filter(
|
|
166
|
+
(line) => line.msg === "Lambda invocation started" || line.path,
|
|
167
|
+
);
|
|
168
|
+
assert.ok(lines.length >= 2, "expected the invocation to have logged");
|
|
169
|
+
const requestIds = new Set(lines.map((line) => line.requestId));
|
|
170
|
+
assert.equal(requestIds.size, 1);
|
|
171
|
+
assert.equal([...requestIds][0] === undefined, false);
|
|
172
|
+
});
|
|
173
|
+
});
|
package/dev-server/server.ts
CHANGED
|
@@ -288,7 +288,7 @@ app.all(/(.*)/, async (req: Request, res: Response) => {
|
|
|
288
288
|
) {
|
|
289
289
|
const parsed = await safeJsonParse<unknown>(body).catch(() => undefined);
|
|
290
290
|
if (parsed === undefined) {
|
|
291
|
-
|
|
291
|
+
logger.error("Failed to parse JSON body");
|
|
292
292
|
} else if (
|
|
293
293
|
parsed &&
|
|
294
294
|
typeof parsed === "object" &&
|
|
@@ -313,19 +313,30 @@ app.all(/(.*)/, async (req: Request, res: Response) => {
|
|
|
313
313
|
|
|
314
314
|
const port = env.SERVER_PORT;
|
|
315
315
|
|
|
316
|
+
// This file is the backend image's entrypoint, so its startup output is the
|
|
317
|
+
// first thing a log collector reads from the container. It goes through the
|
|
318
|
+
// logger for the same reason every other line does: one JSON object per line is
|
|
319
|
+
// the contract in deploy/vps/README.md, and a banner printed alongside it is a
|
|
320
|
+
// line the pipeline cannot parse.
|
|
321
|
+
//
|
|
322
|
+
// This is also the everyday `npm run dev` server, so the addresses stay whole
|
|
323
|
+
// and clickable — `url`, not a port a developer has to assemble one themselves.
|
|
316
324
|
app.listen(Number(port), "0.0.0.0", () => {
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
325
|
+
// biome-ignore lint/plugin/no-logger-info: the configuration a container came up on is an audit-grade signal
|
|
326
|
+
logger.info(
|
|
327
|
+
{
|
|
328
|
+
port: Number(port),
|
|
329
|
+
url: `http://localhost:${port}`,
|
|
330
|
+
dynamodbPort: env.DYNAMODB_PORT,
|
|
331
|
+
dynamodbTable: env.DYNAMODB_TABLE_NAME,
|
|
332
|
+
nodeEnv: env.NODE_ENV,
|
|
333
|
+
...(isSelfHostBackend
|
|
334
|
+
? {}
|
|
335
|
+
: { apiDocsUrl: `http://localhost:${port}/api-docs` }),
|
|
336
|
+
},
|
|
337
|
+
"Backend listening",
|
|
320
338
|
);
|
|
321
339
|
|
|
322
|
-
console.table({
|
|
323
|
-
SERVER_PORT: port,
|
|
324
|
-
DYNAMODB_PORT: env.DYNAMODB_PORT,
|
|
325
|
-
DYNAMODB_TABLE: env.DYNAMODB_TABLE_NAME,
|
|
326
|
-
NODE_ENV: env.NODE_ENV,
|
|
327
|
-
});
|
|
328
|
-
|
|
329
340
|
process.send?.("ready");
|
|
330
341
|
});
|
|
331
342
|
|
package/package.json
CHANGED
|
@@ -57,12 +57,13 @@ const run = async (): Promise<void> => {
|
|
|
57
57
|
{ checkpointStore: fileCheckpointStore, logger },
|
|
58
58
|
);
|
|
59
59
|
|
|
60
|
-
|
|
60
|
+
// biome-ignore lint/plugin/no-logger-info: a completed full-corpus backfill is an audit-grade signal
|
|
61
|
+
logger.info({ result }, "backfill done");
|
|
61
62
|
};
|
|
62
63
|
|
|
63
64
|
run()
|
|
64
65
|
.then(() => process.exit(0))
|
|
65
66
|
.catch((error: unknown) => {
|
|
66
|
-
|
|
67
|
+
logger.error({ error }, "backfill failed");
|
|
67
68
|
process.exit(1);
|
|
68
69
|
});
|
|
@@ -7,7 +7,7 @@ import type {
|
|
|
7
7
|
ThreadMessageItem,
|
|
8
8
|
} from "@remit/data-ports";
|
|
9
9
|
import { deriveAddressId } from "@remit/data-ports/id";
|
|
10
|
-
import {
|
|
10
|
+
import { SenderTrust, StarColor } from "@remit/domain-enums";
|
|
11
11
|
import { deriveAutoMoved } from "./autoMoved.js";
|
|
12
12
|
import { deriveSenderTrust } from "./senderTrust.js";
|
|
13
13
|
|
|
@@ -50,6 +50,7 @@ const toResponse = (item: ThreadMessageItem): ThreadMessageResponse => ({
|
|
|
50
50
|
hasStars: item.hasStars,
|
|
51
51
|
isDeleted: item.isDeleted,
|
|
52
52
|
snippet: item.snippet,
|
|
53
|
+
category: item.category,
|
|
53
54
|
createdAt: item.createdAt,
|
|
54
55
|
updatedAt: item.updatedAt,
|
|
55
56
|
senderTrust: SenderTrust.Unknown,
|
|
@@ -97,19 +98,20 @@ export const planBatchFetch = (rows: ThreadMessageItem[]): BatchPlan => {
|
|
|
97
98
|
};
|
|
98
99
|
|
|
99
100
|
/**
|
|
100
|
-
* Enrich a page of ThreadMessage rows with `
|
|
101
|
-
*
|
|
102
|
-
*
|
|
103
|
-
*
|
|
101
|
+
* Enrich a page of ThreadMessage rows with `senderTrust` (derived from the From
|
|
102
|
+
* Address's flags map), `authenticity` and `autoMoved` (both projected from the
|
|
103
|
+
* Message row, see `deriveAutoMoved`).
|
|
104
|
+
*
|
|
105
|
+
* `category` is not enriched: it is denormalized onto the ThreadMessage row and
|
|
106
|
+
* carried straight through by `toResponse`, so the value a client renders is the
|
|
107
|
+
* value the category filter matched.
|
|
104
108
|
*
|
|
105
109
|
* Two BatchGetItem calls per page, regardless of page size — see
|
|
106
110
|
* `planBatchFetch` for the dedup contract.
|
|
107
111
|
*
|
|
108
|
-
* Missing rows fall back gracefully: `
|
|
109
|
-
*
|
|
110
|
-
*
|
|
111
|
-
* `senderTrust` defaults to `"unknown"`. `autoMoved` is omitted whenever the
|
|
112
|
-
* move isn't a real, in-effect auto-move (or the Message row is absent).
|
|
112
|
+
* Missing rows fall back gracefully: `senderTrust` defaults to `"unknown"`, and
|
|
113
|
+
* `authenticity` / `autoMoved` are omitted whenever the Message row is absent or
|
|
114
|
+
* the move isn't a real, in-effect auto-move.
|
|
113
115
|
*
|
|
114
116
|
* Not annotated `Promise<ThreadMessageResponse[]>`: `labels` is a new field on
|
|
115
117
|
* it in this same PR, and that package publishes separately from this repo —
|
|
@@ -158,12 +160,6 @@ export const enrichThreadRows = async (
|
|
|
158
160
|
}
|
|
159
161
|
}
|
|
160
162
|
|
|
161
|
-
const categoryByMessageId = new Map(
|
|
162
|
-
messages.map((m) => [
|
|
163
|
-
m.messageId,
|
|
164
|
-
m.category ?? MessageCategory.uncategorized,
|
|
165
|
-
]),
|
|
166
|
-
);
|
|
167
163
|
const authenticityByMessageId = new Map(
|
|
168
164
|
messages.map((m) => [m.messageId, m.authenticity]),
|
|
169
165
|
);
|
|
@@ -176,7 +172,6 @@ export const enrichThreadRows = async (
|
|
|
176
172
|
|
|
177
173
|
return rows.map((row) => {
|
|
178
174
|
const base = toResponse(row);
|
|
179
|
-
const category = categoryByMessageId.get(row.messageId);
|
|
180
175
|
const authenticity = authenticityByMessageId.get(row.messageId);
|
|
181
176
|
const autoMoved = autoMovedByMessageId.get(row.messageId);
|
|
182
177
|
const addressId = plan.addressIdByRow.get(row.threadMessageId);
|
|
@@ -186,7 +181,6 @@ export const enrichThreadRows = async (
|
|
|
186
181
|
const labels = labelsByMessageId.get(row.messageId);
|
|
187
182
|
return {
|
|
188
183
|
...base,
|
|
189
|
-
...(category !== undefined ? { category } : {}),
|
|
190
184
|
...(authenticity !== undefined ? { authenticity } : {}),
|
|
191
185
|
...(autoMoved !== undefined ? { autoMoved } : {}),
|
|
192
186
|
...(labels !== undefined ? { labels } : {}),
|
|
@@ -25,6 +25,7 @@ const row = (
|
|
|
25
25
|
star: StarColor.None,
|
|
26
26
|
isDeleted: false,
|
|
27
27
|
snippet: "",
|
|
28
|
+
category: MessageCategory.uncategorized,
|
|
28
29
|
createdAt: 0,
|
|
29
30
|
updatedAt: 0,
|
|
30
31
|
senderTrust: SenderTrust.Unknown,
|
|
@@ -37,7 +38,7 @@ describe("hasOffRowCriteria", () => {
|
|
|
37
38
|
});
|
|
38
39
|
|
|
39
40
|
it("is false for empty arrays", () => {
|
|
40
|
-
assert.equal(hasOffRowCriteria({ senderTrust: []
|
|
41
|
+
assert.equal(hasOffRowCriteria({ senderTrust: [] }), false);
|
|
41
42
|
});
|
|
42
43
|
|
|
43
44
|
it("is true when any criterion is set", () => {
|
|
@@ -65,17 +66,12 @@ describe("filterByOffRowCriteria", () => {
|
|
|
65
66
|
assert.ok(result.every((r) => r.senderTrust !== SenderTrust.Unknown));
|
|
66
67
|
});
|
|
67
68
|
|
|
68
|
-
it("
|
|
69
|
+
it("leaves category alone — it is a SQL predicate, not an off-row criterion", () => {
|
|
69
70
|
const rows = [
|
|
70
71
|
row({ category: MessageCategory.newsletter }),
|
|
71
72
|
row({ category: MessageCategory.personal }),
|
|
72
|
-
row({ category: undefined }),
|
|
73
73
|
];
|
|
74
|
-
|
|
75
|
-
category: [MessageCategory.newsletter],
|
|
76
|
-
});
|
|
77
|
-
assert.equal(result.length, 1);
|
|
78
|
-
assert.equal(result[0].category, MessageCategory.newsletter);
|
|
74
|
+
assert.equal(filterByOffRowCriteria(rows, {}).length, 2);
|
|
79
75
|
});
|
|
80
76
|
|
|
81
77
|
it("filters by dkimMismatch and never matches rows lacking an authenticity signal", () => {
|
|
@@ -1,31 +1,31 @@
|
|
|
1
1
|
import type {
|
|
2
|
-
MessageCategory,
|
|
3
2
|
SenderTrust,
|
|
4
3
|
ThreadMessageResponse,
|
|
5
4
|
} from "@remit/api-openapi-types";
|
|
6
5
|
|
|
7
6
|
/**
|
|
8
7
|
* Off-row search criteria — fields that live on the underlying Message/Address,
|
|
9
|
-
* not on the ThreadMessage
|
|
10
|
-
*
|
|
11
|
-
*
|
|
8
|
+
* not on the ThreadMessage row. They are resolved by enriching the windowed
|
|
9
|
+
* rows (see enrichThreadRows) and filtering in app code, because no index or
|
|
10
|
+
* FilterExpression can serve them.
|
|
11
|
+
*
|
|
12
|
+
* `senderTrust` derives from AddressItem.flags and `dkimMismatch` from
|
|
13
|
+
* MessageItem.authenticity. `category` is not one of these: it is denormalized
|
|
14
|
+
* onto the ThreadMessage row and filtered in SQL, inside the window.
|
|
12
15
|
*/
|
|
13
16
|
export interface OffRowCriteria {
|
|
14
17
|
senderTrust?: SenderTrust[];
|
|
15
|
-
category?: MessageCategory[];
|
|
16
18
|
dkimMismatch?: boolean;
|
|
17
19
|
}
|
|
18
20
|
|
|
19
21
|
export const hasOffRowCriteria = (criteria: OffRowCriteria): boolean =>
|
|
20
|
-
Boolean(criteria.senderTrust?.length) ||
|
|
21
|
-
Boolean(criteria.category?.length) ||
|
|
22
|
-
criteria.dkimMismatch !== undefined;
|
|
22
|
+
Boolean(criteria.senderTrust?.length) || criteria.dkimMismatch !== undefined;
|
|
23
23
|
|
|
24
24
|
/**
|
|
25
25
|
* Filter enriched rows by the off-row criteria. Each active criterion is an
|
|
26
|
-
* any-of set (AND across criteria, OR within a set). A row with no
|
|
27
|
-
* never matches a
|
|
28
|
-
*
|
|
26
|
+
* any-of set (AND across criteria, OR within a set). A row with no
|
|
27
|
+
* `authenticity` signal never matches a `dkimMismatch` filter (absence means no
|
|
28
|
+
* signal, not a verdict).
|
|
29
29
|
*/
|
|
30
30
|
export const filterByOffRowCriteria = (
|
|
31
31
|
rows: ThreadMessageResponse[],
|
|
@@ -36,19 +36,10 @@ export const filterByOffRowCriteria = (
|
|
|
36
36
|
const trustSet = criteria.senderTrust?.length
|
|
37
37
|
? new Set(criteria.senderTrust)
|
|
38
38
|
: undefined;
|
|
39
|
-
const categorySet = criteria.category?.length
|
|
40
|
-
? new Set(criteria.category)
|
|
41
|
-
: undefined;
|
|
42
39
|
const { dkimMismatch } = criteria;
|
|
43
40
|
|
|
44
41
|
return rows.filter((row) => {
|
|
45
42
|
if (trustSet && !trustSet.has(row.senderTrust)) return false;
|
|
46
|
-
if (
|
|
47
|
-
categorySet &&
|
|
48
|
-
(row.category === undefined || !categorySet.has(row.category))
|
|
49
|
-
) {
|
|
50
|
-
return false;
|
|
51
|
-
}
|
|
52
43
|
if (
|
|
53
44
|
dkimMismatch !== undefined &&
|
|
54
45
|
row.authenticity?.dkimMismatch !== dkimMismatch
|
|
@@ -1,10 +1,19 @@
|
|
|
1
1
|
import assert from "node:assert/strict";
|
|
2
2
|
import { describe, it } from "node:test";
|
|
3
|
+
import type { ThreadMessageItem } from "@remit/data-ports";
|
|
4
|
+
import { MessageCategory, SenderTrust, StarColor } from "@remit/domain-enums";
|
|
3
5
|
import {
|
|
4
6
|
buildListThreadMessagesOptions,
|
|
7
|
+
buildListThreadsOptions,
|
|
8
|
+
buildSearchThreadsOptions,
|
|
5
9
|
dedupeThreadMessages,
|
|
10
|
+
executeThreadSearch,
|
|
11
|
+
type ThreadSearchClient,
|
|
6
12
|
} from "./thread.js";
|
|
7
13
|
|
|
14
|
+
const ACCOUNT = "cfg-1";
|
|
15
|
+
const MAILBOX = "mbx-inbox";
|
|
16
|
+
|
|
8
17
|
type Row = {
|
|
9
18
|
threadMessageId: string;
|
|
10
19
|
messageIdHeader?: string;
|
|
@@ -84,3 +93,156 @@ describe("dedupeThreadMessages", () => {
|
|
|
84
93
|
assert.deepEqual(dedupeThreadMessages([]), []);
|
|
85
94
|
});
|
|
86
95
|
});
|
|
96
|
+
|
|
97
|
+
// #304: `category` is a column on the thread_message row, so it is a SQL
|
|
98
|
+
// predicate the port applies inside its window — not a criterion resolved by
|
|
99
|
+
// enriching whatever the window happened to return. These assertions are on the
|
|
100
|
+
// routing, because that is what a later refactor can silently undo: the SQL
|
|
101
|
+
// clause goes dead and the filter falls back to a window-sized filter with no
|
|
102
|
+
// test failing.
|
|
103
|
+
describe("executeThreadSearch", () => {
|
|
104
|
+
type Category = ThreadMessageItem["category"];
|
|
105
|
+
|
|
106
|
+
const threadRow = (
|
|
107
|
+
threadMessageId: string,
|
|
108
|
+
category: Category,
|
|
109
|
+
): ThreadMessageItem => ({
|
|
110
|
+
threadMessageId,
|
|
111
|
+
threadId: `t-${threadMessageId}`,
|
|
112
|
+
messageId: `m-${threadMessageId}`,
|
|
113
|
+
accountConfigId: ACCOUNT,
|
|
114
|
+
mailboxId: MAILBOX,
|
|
115
|
+
uid: 1,
|
|
116
|
+
referenceOrder: 0,
|
|
117
|
+
internalDate: 0,
|
|
118
|
+
sentDate: 0,
|
|
119
|
+
isRead: false,
|
|
120
|
+
hasAttachment: false,
|
|
121
|
+
star: StarColor.None,
|
|
122
|
+
hasStars: false,
|
|
123
|
+
isDeleted: false,
|
|
124
|
+
category,
|
|
125
|
+
createdAt: 0,
|
|
126
|
+
updatedAt: 0,
|
|
127
|
+
});
|
|
128
|
+
|
|
129
|
+
type RecordedCall = { search: { category?: Category[] } };
|
|
130
|
+
|
|
131
|
+
const fakeClient = (rows: ThreadMessageItem[]) => {
|
|
132
|
+
const windowCalls: RecordedCall[] = [];
|
|
133
|
+
const countCalls: RecordedCall[] = [];
|
|
134
|
+
|
|
135
|
+
// The fake applies the category predicate itself, the way a port does, so
|
|
136
|
+
// a request that never reaches `search` cannot answer correctly by
|
|
137
|
+
// accident.
|
|
138
|
+
const matching = (categories?: Category[]) =>
|
|
139
|
+
categories?.length
|
|
140
|
+
? rows.filter((row) => categories.includes(row.category))
|
|
141
|
+
: rows;
|
|
142
|
+
|
|
143
|
+
const client: ThreadSearchClient = {
|
|
144
|
+
threadMessage: {
|
|
145
|
+
async searchByMailboxWindow(_account, _mailbox, search) {
|
|
146
|
+
windowCalls.push({ search });
|
|
147
|
+
return {
|
|
148
|
+
items: matching(search.category),
|
|
149
|
+
continuationToken: undefined,
|
|
150
|
+
};
|
|
151
|
+
},
|
|
152
|
+
async countByMailbox(_account, _mailbox, search) {
|
|
153
|
+
countCalls.push({ search });
|
|
154
|
+
return matching(search.category).length;
|
|
155
|
+
},
|
|
156
|
+
},
|
|
157
|
+
message: { get: async () => [] },
|
|
158
|
+
address: { getAddress: async () => [] },
|
|
159
|
+
messageLabel: { listByMessageIds: async () => [] },
|
|
160
|
+
label: { listByAccountConfig: async () => [] },
|
|
161
|
+
};
|
|
162
|
+
|
|
163
|
+
return { client, windowCalls, countCalls };
|
|
164
|
+
};
|
|
165
|
+
|
|
166
|
+
const ACCOUNT_ROWS = [
|
|
167
|
+
threadRow("tm-1", MessageCategory.personal),
|
|
168
|
+
threadRow("tm-2", MessageCategory.marketing),
|
|
169
|
+
threadRow("tm-3", MessageCategory.uncategorized),
|
|
170
|
+
];
|
|
171
|
+
|
|
172
|
+
it("passes category to the port's search, not to the off-row filter", async () => {
|
|
173
|
+
const { client, windowCalls } = fakeClient(ACCOUNT_ROWS);
|
|
174
|
+
|
|
175
|
+
const response = await executeThreadSearch(client, ACCOUNT, MAILBOX, {
|
|
176
|
+
category: [MessageCategory.personal],
|
|
177
|
+
});
|
|
178
|
+
|
|
179
|
+
assert.deepEqual(windowCalls.length, 1);
|
|
180
|
+
assert.deepEqual(windowCalls[0].search.category, [
|
|
181
|
+
MessageCategory.personal,
|
|
182
|
+
]);
|
|
183
|
+
assert.deepEqual(
|
|
184
|
+
response.items?.map((item) => item.threadMessageId),
|
|
185
|
+
["tm-1"],
|
|
186
|
+
);
|
|
187
|
+
});
|
|
188
|
+
|
|
189
|
+
it("takes the count-only path for a category-only query", async () => {
|
|
190
|
+
const { client, windowCalls, countCalls } = fakeClient(ACCOUNT_ROWS);
|
|
191
|
+
|
|
192
|
+
const response = await executeThreadSearch(client, ACCOUNT, MAILBOX, {
|
|
193
|
+
category: [MessageCategory.personal, MessageCategory.marketing],
|
|
194
|
+
count: true,
|
|
195
|
+
results: false,
|
|
196
|
+
});
|
|
197
|
+
|
|
198
|
+
assert.equal(windowCalls.length, 0, "no window read in count-only mode");
|
|
199
|
+
assert.equal(countCalls.length, 1);
|
|
200
|
+
assert.deepEqual(countCalls[0].search.category, [
|
|
201
|
+
MessageCategory.personal,
|
|
202
|
+
MessageCategory.marketing,
|
|
203
|
+
]);
|
|
204
|
+
assert.equal(response.count, 2);
|
|
205
|
+
assert.equal(response.items, undefined);
|
|
206
|
+
});
|
|
207
|
+
|
|
208
|
+
// The one request shape that still takes the off-row branch. `category` must
|
|
209
|
+
// reach `search` anyway, so the window is category-filtered before the
|
|
210
|
+
// enrichment the off-row criterion needs.
|
|
211
|
+
it("keeps category in search when an off-row criterion is also set", async () => {
|
|
212
|
+
const { client, windowCalls } = fakeClient(ACCOUNT_ROWS);
|
|
213
|
+
|
|
214
|
+
const response = await executeThreadSearch(client, ACCOUNT, MAILBOX, {
|
|
215
|
+
category: [MessageCategory.personal],
|
|
216
|
+
senderTrust: [SenderTrust.Unknown],
|
|
217
|
+
});
|
|
218
|
+
|
|
219
|
+
assert.equal(windowCalls.length, 1);
|
|
220
|
+
assert.deepEqual(windowCalls[0].search.category, [
|
|
221
|
+
MessageCategory.personal,
|
|
222
|
+
]);
|
|
223
|
+
assert.deepEqual(
|
|
224
|
+
response.items?.map((item) => item.threadMessageId),
|
|
225
|
+
["tm-1"],
|
|
226
|
+
);
|
|
227
|
+
});
|
|
228
|
+
|
|
229
|
+
it("serves category from the row, so it survives an absent message row", async () => {
|
|
230
|
+
const { client } = fakeClient(ACCOUNT_ROWS);
|
|
231
|
+
|
|
232
|
+
const response = await executeThreadSearch(client, ACCOUNT, MAILBOX, {});
|
|
233
|
+
|
|
234
|
+
assert.deepEqual(
|
|
235
|
+
response.items?.map((item) => item.category),
|
|
236
|
+
[
|
|
237
|
+
MessageCategory.personal,
|
|
238
|
+
MessageCategory.marketing,
|
|
239
|
+
MessageCategory.uncategorized,
|
|
240
|
+
],
|
|
241
|
+
);
|
|
242
|
+
});
|
|
243
|
+
|
|
244
|
+
it("projects category, so the DynamoDB port reads it with the row", () => {
|
|
245
|
+
assert.ok(buildSearchThreadsOptions({}).attributes.includes("category"));
|
|
246
|
+
assert.ok(buildListThreadsOptions({}).attributes.includes("category"));
|
|
247
|
+
});
|
|
248
|
+
});
|
package/src/handlers/thread.ts
CHANGED
|
@@ -48,6 +48,7 @@ const THREAD_LIST_ATTRIBUTES: ReadonlyArray<keyof ThreadMessageItem> = [
|
|
|
48
48
|
"hasStars",
|
|
49
49
|
"isDeleted",
|
|
50
50
|
"snippet",
|
|
51
|
+
"category",
|
|
51
52
|
"createdAt",
|
|
52
53
|
"updatedAt",
|
|
53
54
|
];
|
|
@@ -109,6 +110,7 @@ type ThreadSearch = {
|
|
|
109
110
|
unread?: boolean;
|
|
110
111
|
starred?: boolean;
|
|
111
112
|
attachments?: boolean;
|
|
113
|
+
category?: MessageCategory[];
|
|
112
114
|
};
|
|
113
115
|
|
|
114
116
|
/**
|
|
@@ -174,10 +176,10 @@ export const executeThreadSearch = async (
|
|
|
174
176
|
unread: params.unread,
|
|
175
177
|
starred: params.starred,
|
|
176
178
|
attachments: params.attachments,
|
|
179
|
+
category: params.category,
|
|
177
180
|
};
|
|
178
181
|
const offRow = {
|
|
179
182
|
senderTrust: params.senderTrust,
|
|
180
|
-
category: params.category,
|
|
181
183
|
dkimMismatch: params.dkimMismatch,
|
|
182
184
|
};
|
|
183
185
|
const wantCount = params.count === true;
|
|
@@ -325,7 +325,7 @@ export const dedupeByMessageId = <T extends { messageId: string }>(
|
|
|
325
325
|
/**
|
|
326
326
|
* Attach accountId to each enriched ThreadMessageResponse row using the
|
|
327
327
|
* mailboxId→accountId map built from inbox discovery. Same read-time-attach
|
|
328
|
-
* pattern as senderTrust
|
|
328
|
+
* pattern as senderTrust in enrichThreadRows.
|
|
329
329
|
*/
|
|
330
330
|
export const attachAccountIds = (
|
|
331
331
|
rows: Awaited<ReturnType<typeof enrichThreadRows>>,
|
package/src/index.ts
CHANGED
|
@@ -144,11 +144,11 @@ const readOriginHeader = (
|
|
|
144
144
|
// A scope, not `logger.setBindings`: this process serves requests concurrently,
|
|
145
145
|
// and bindings on the shared logger belong to whichever request wrote them last,
|
|
146
146
|
// so a line gets attributed to the wrong request. The scope follows the request
|
|
147
|
-
// through its own async continuations and nothing else.
|
|
147
|
+
// through its own async continuations and nothing else. It nests inside the one
|
|
148
|
+
// `withTelemetry` opens, which is where `requestId` comes from.
|
|
148
149
|
const rawHandler = async (event: APIGatewayProxyEvent, context: Context) =>
|
|
149
150
|
withLogContext(
|
|
150
151
|
{
|
|
151
|
-
requestId: context.awsRequestId,
|
|
152
152
|
path: event.path,
|
|
153
153
|
method: event.httpMethod,
|
|
154
154
|
},
|
package/tsconfig.json
CHANGED
|
@@ -3,5 +3,16 @@
|
|
|
3
3
|
"compilerOptions": {
|
|
4
4
|
"outDir": "dist"
|
|
5
5
|
},
|
|
6
|
-
"include": [
|
|
6
|
+
"include": [
|
|
7
|
+
"src/**/*.ts",
|
|
8
|
+
"dev-server/**/*.ts",
|
|
9
|
+
"scripts/**/*.ts",
|
|
10
|
+
// The backend image's third entrypoint. `migrate.mjs` and
|
|
11
|
+
// `backfill-list-id.mjs` ship inside this image alongside `server.mjs`
|
|
12
|
+
// (npm-scripts/docker-bundle.mjs), so they belong to the same typecheck —
|
|
13
|
+
// living under deploy/ is where it sits in the tree, not a different
|
|
14
|
+
// deliverable. Unchecked, it shipped a container entrypoint with two hard
|
|
15
|
+
// type errors.
|
|
16
|
+
"../../deploy/vps/migrate/**/*.ts"
|
|
17
|
+
]
|
|
7
18
|
}
|