@remit/drizzle-service 0.0.34 → 0.0.36
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +3 -10
- package/src/db.ts +6 -6
- package/src/error.ts +2 -5
- package/src/index.ts +0 -1
- package/src/repair/thread-message-category-contract.ts +2 -4
- package/src/repair/thread-message-category.sqlite.test.ts +1 -2
- package/src/repair/thread-message-category.ts +15 -25
- package/src/repos/cascade-delete.ts +8 -22
- package/src/repos/category-fixture.ts +2 -19
- package/src/repos/filter-anchor.ts +4 -4
- package/src/repos/filter.ts +3 -3
- package/src/repos/i4-account-config.ts +2 -2
- package/src/repos/i4-account-export-request.ts +2 -2
- package/src/repos/i4-account-setting.ts +2 -2
- package/src/repos/i4-account.ts +2 -2
- package/src/repos/i4-mailbox-lock.ts +2 -2
- package/src/repos/i4-mailbox-special-use.ts +2 -2
- package/src/repos/i4-mailbox.ts +2 -2
- package/src/repos/i4-message-flag-push.test.ts +1 -1
- package/src/repos/i4-message-flag-push.ts +3 -3
- package/src/repos/i4-message-placement-move.test.ts +1 -1
- package/src/repos/i4-message-placement-move.ts +4 -3
- package/src/repos/i4-organize-job-request.ts +2 -2
- package/src/repos/i4-outbox-message.ts +2 -2
- package/src/repos/label.ts +2 -2
- package/src/repos/message-flag.ts +2 -2
- package/src/repos/message-label.ts +2 -2
- package/src/repos/message.ts +8 -10
- package/src/repos/quarantine.ts +2 -2
- package/src/repos/test-helpers.ts +11 -213
- package/src/repos/thread-message.test.ts +35 -93
- package/src/repos/thread-message.ts +7 -19
- package/src/repos/thread-search-predicates.ts +13 -41
- package/src/repos/unit-of-work.ts +1 -1
- package/src/schema/i4-account-config.ts +1 -1
- package/src/schema/i4-account-export-request.ts +1 -1
- package/src/schema/i4-account-setting.ts +1 -1
- package/src/schema/i4-address.ts +1 -1
- package/src/schema/i4-mailbox-lock.ts +1 -1
- package/src/schema/i4-mailbox.ts +1 -1
- package/src/schema/i4-message-flag-push.ts +1 -1
- package/src/schema/i4-message-placement-move.ts +1 -1
- package/src/schema/i4-organize-job-request.ts +1 -1
- package/src/schema/i4-outbox-message.ts +1 -1
- package/src/schema/message-data.ts +1 -1
- package/src/schema/outbox.ts +13 -56
- package/src/schema/quarantine.ts +1 -1
- package/src/schema/thread-message.ts +1 -1
- package/src/schema-full-sqlite.ts +9 -7
- package/src/schema.ts +4 -4
- package/src/sqlite-client.ts +4 -5
- package/src/test-db-sqlite.ts +4 -9
- package/src/test-db.ts +11 -66
- package/src/tx.ts +6 -12
- package/src/vps-migrations-drift.sqlite.test.ts +3 -3
- package/drizzle.config.ts +0 -15
- package/src/dialect.ts +0 -13
- package/src/repair/thread-message-category.test.ts +0 -229
- package/src/repos/thread-message-category.test.ts +0 -156
- package/src/schema/active-entities.ts +0 -19
- package/src/schema-full.ts +0 -16
|
@@ -4,11 +4,11 @@ import type {
|
|
|
4
4
|
MessageLabelItem,
|
|
5
5
|
} from "@remit/data-ports";
|
|
6
6
|
import { and, desc, eq, inArray } from "drizzle-orm";
|
|
7
|
-
import type {
|
|
7
|
+
import type { Db } from "../db.js";
|
|
8
8
|
import { deterministicBase36Id } from "../id.js";
|
|
9
9
|
import { messageLabelTable } from "../schema.js";
|
|
10
10
|
|
|
11
|
-
type DB =
|
|
11
|
+
type DB = Db<Record<string, unknown>>;
|
|
12
12
|
|
|
13
13
|
function rowToMessageLabel(
|
|
14
14
|
row: typeof messageLabelTable.$inferSelect,
|
package/src/repos/message.ts
CHANGED
|
@@ -6,7 +6,7 @@ import type {
|
|
|
6
6
|
MessageItem,
|
|
7
7
|
} from "@remit/data-ports";
|
|
8
8
|
import { and, asc, eq, gt, inArray, or } from "drizzle-orm";
|
|
9
|
-
|
|
9
|
+
|
|
10
10
|
import type { Db } from "../db.js";
|
|
11
11
|
import {
|
|
12
12
|
CreateFailedConflictError,
|
|
@@ -99,15 +99,12 @@ function toMessageItem(row: typeof messageTable.$inferSelect): MessageItem {
|
|
|
99
99
|
|
|
100
100
|
/**
|
|
101
101
|
* Emitted into the transactional outbox when a message's rows are deleted, so
|
|
102
|
-
* the
|
|
103
|
-
*
|
|
102
|
+
* the search-index worker relays a search-index REMOVE and the vectors are
|
|
103
|
+
* dropped.
|
|
104
104
|
*/
|
|
105
105
|
export const MESSAGE_REMOVED_EVENT = "message.removed";
|
|
106
106
|
|
|
107
|
-
export type SubtreeDb = Pick<
|
|
108
|
-
NodePgDatabase<Record<string, unknown>>,
|
|
109
|
-
"delete" | "insert"
|
|
110
|
-
>;
|
|
107
|
+
export type SubtreeDb = Pick<Db<Record<string, unknown>>, "delete" | "insert">;
|
|
111
108
|
|
|
112
109
|
/**
|
|
113
110
|
* Delete a message and its whole per-message subtree by message id — the nine
|
|
@@ -201,7 +198,7 @@ export class DrizzleMessageRepository implements IMessageRepository {
|
|
|
201
198
|
};
|
|
202
199
|
|
|
203
200
|
// Faithful to ElectroDB message.create: a duplicate messageId throws
|
|
204
|
-
// CreateFailedConflictError. The plain insert raises a
|
|
201
|
+
// CreateFailedConflictError. The plain insert raises a unique-constraint
|
|
205
202
|
// violation, which rolls back the transaction so NO outbox row is
|
|
206
203
|
// written; we surface it as the domain conflict error.
|
|
207
204
|
try {
|
|
@@ -397,7 +394,8 @@ export class DrizzleMessageRepository implements IMessageRepository {
|
|
|
397
394
|
// A non-empty bodyStorageKey means body-sync just persisted the parsed
|
|
398
395
|
// body, so the message now has embeddable content and its threadMessage
|
|
399
396
|
// exists. Append a search-index event in the same transaction as the write
|
|
400
|
-
// (transactional outbox) — the
|
|
397
|
+
// (transactional outbox) — the search-index worker relays it to SQS and
|
|
398
|
+
// embeds.
|
|
401
399
|
// The outbox is append-only: the worker's content-hash gate makes a
|
|
402
400
|
// redundant pass near-free.
|
|
403
401
|
const bodySynced =
|
|
@@ -540,7 +538,7 @@ export class DrizzleMessageRepository implements IMessageRepository {
|
|
|
540
538
|
// mailbox and its COPYUID). The message's search vectors still carry the
|
|
541
539
|
// old mailbox in their metadata and their body is unchanged, so a normal
|
|
542
540
|
// re-index would skip them on content hash. Enqueue a move re-index event
|
|
543
|
-
// in the same transaction as the update; the
|
|
541
|
+
// in the same transaction as the update; the search-index worker drains it
|
|
544
542
|
// with force, refreshing the stored mailbox metadata.
|
|
545
543
|
const rows = await runInTransaction(this.db, async (tx) => {
|
|
546
544
|
const updated = await tx
|
package/src/repos/quarantine.ts
CHANGED
|
@@ -6,10 +6,10 @@ import type {
|
|
|
6
6
|
} from "@remit/data-ports";
|
|
7
7
|
import { deriveQuarantineId } from "@remit/data-ports/id";
|
|
8
8
|
import { desc, eq } from "drizzle-orm";
|
|
9
|
-
import type {
|
|
9
|
+
import type { Db } from "../db.js";
|
|
10
10
|
import { quarantineTable } from "../schema/quarantine.js";
|
|
11
11
|
|
|
12
|
-
type DB =
|
|
12
|
+
type DB = Db<Record<string, unknown>>;
|
|
13
13
|
|
|
14
14
|
function rowToItem(row: typeof quarantineTable.$inferSelect): QuarantineItem {
|
|
15
15
|
return {
|
|
@@ -1,224 +1,22 @@
|
|
|
1
|
-
import
|
|
2
|
-
import {
|
|
3
|
-
import type { NodePgDatabase } from "drizzle-orm/node-postgres";
|
|
4
|
-
import { drizzle } from "drizzle-orm/node-postgres";
|
|
5
|
-
import EmbeddedPostgres from "embedded-postgres";
|
|
1
|
+
import type Database from "better-sqlite3";
|
|
2
|
+
import type { Db } from "../db.js";
|
|
6
3
|
import {
|
|
7
4
|
type MessageDataSchema,
|
|
8
5
|
messageDataSchema,
|
|
9
6
|
} from "../schema/message-data.js";
|
|
7
|
+
import { createSqliteTestDb } from "../test-db-sqlite.js";
|
|
10
8
|
|
|
11
|
-
export type TestDb =
|
|
12
|
-
|
|
13
|
-
function randomPort(): number {
|
|
14
|
-
return 54500 + Math.floor(Math.random() * 400);
|
|
15
|
-
}
|
|
9
|
+
export type TestDb = Db<MessageDataSchema>;
|
|
16
10
|
|
|
11
|
+
/**
|
|
12
|
+
* The message-data tables on a throwaway in-memory SQLite database — the
|
|
13
|
+
* harness for the repos that only touch that subset.
|
|
14
|
+
*/
|
|
17
15
|
export async function createTestDb(): Promise<{
|
|
18
16
|
db: TestDb;
|
|
17
|
+
sqlite: Database.Database;
|
|
19
18
|
stop: () => Promise<void>;
|
|
20
19
|
}> {
|
|
21
|
-
const
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
const pg = new EmbeddedPostgres({
|
|
25
|
-
databaseDir,
|
|
26
|
-
user: "test",
|
|
27
|
-
password: "test",
|
|
28
|
-
port,
|
|
29
|
-
persistent: false,
|
|
30
|
-
});
|
|
31
|
-
|
|
32
|
-
await pg.initialise();
|
|
33
|
-
await pg.start();
|
|
34
|
-
|
|
35
|
-
const connectionString = `postgresql://test:test@localhost:${port}/postgres`;
|
|
36
|
-
const db = drizzle(connectionString, { schema: messageDataSchema }) as TestDb;
|
|
37
|
-
|
|
38
|
-
await db.execute(sql.raw(DDL));
|
|
39
|
-
|
|
40
|
-
return {
|
|
41
|
-
db,
|
|
42
|
-
stop: async () => {
|
|
43
|
-
const client = (
|
|
44
|
-
db as unknown as { $client: { end: () => Promise<void> } }
|
|
45
|
-
).$client;
|
|
46
|
-
await client.end();
|
|
47
|
-
await pg.stop();
|
|
48
|
-
try {
|
|
49
|
-
rmSync(databaseDir, { recursive: true, force: true });
|
|
50
|
-
} catch {
|
|
51
|
-
// best-effort cleanup
|
|
52
|
-
}
|
|
53
|
-
},
|
|
54
|
-
};
|
|
20
|
+
const { db, sqlite, close } = await createSqliteTestDb(messageDataSchema);
|
|
21
|
+
return { db, sqlite, stop: close };
|
|
55
22
|
}
|
|
56
|
-
|
|
57
|
-
const DDL = `
|
|
58
|
-
CREATE TABLE IF NOT EXISTS envelope (
|
|
59
|
-
envelope_id UUID PRIMARY KEY,
|
|
60
|
-
message_id UUID NOT NULL,
|
|
61
|
-
date_value BIGINT NOT NULL,
|
|
62
|
-
date_raw TEXT NOT NULL,
|
|
63
|
-
subject TEXT,
|
|
64
|
-
message_id_value TEXT,
|
|
65
|
-
created_at BIGINT NOT NULL,
|
|
66
|
-
updated_at BIGINT NOT NULL
|
|
67
|
-
);
|
|
68
|
-
CREATE INDEX IF NOT EXISTS envelope_message_id_idx ON envelope (message_id);
|
|
69
|
-
|
|
70
|
-
CREATE TABLE IF NOT EXISTS message_reference (
|
|
71
|
-
message_reference_id UUID PRIMARY KEY,
|
|
72
|
-
message_id UUID NOT NULL,
|
|
73
|
-
envelope_id UUID NOT NULL,
|
|
74
|
-
message_id_value TEXT NOT NULL,
|
|
75
|
-
reference_type TEXT NOT NULL,
|
|
76
|
-
reference_order INTEGER NOT NULL,
|
|
77
|
-
created_at BIGINT NOT NULL,
|
|
78
|
-
updated_at BIGINT NOT NULL
|
|
79
|
-
);
|
|
80
|
-
CREATE INDEX IF NOT EXISTS message_reference_message_id_idx ON message_reference (message_id);
|
|
81
|
-
|
|
82
|
-
CREATE TABLE IF NOT EXISTS envelope_address (
|
|
83
|
-
envelope_address_id UUID PRIMARY KEY,
|
|
84
|
-
message_id UUID NOT NULL,
|
|
85
|
-
address_id UUID NOT NULL,
|
|
86
|
-
display_name TEXT,
|
|
87
|
-
normalized_email TEXT NOT NULL,
|
|
88
|
-
address_role TEXT NOT NULL,
|
|
89
|
-
address_order INTEGER NOT NULL,
|
|
90
|
-
created_at BIGINT NOT NULL,
|
|
91
|
-
updated_at BIGINT NOT NULL
|
|
92
|
-
);
|
|
93
|
-
CREATE INDEX IF NOT EXISTS envelope_address_message_id_idx ON envelope_address (message_id);
|
|
94
|
-
|
|
95
|
-
CREATE TABLE IF NOT EXISTS body_part (
|
|
96
|
-
body_part_id UUID PRIMARY KEY,
|
|
97
|
-
message_id UUID NOT NULL,
|
|
98
|
-
parent_body_part_id UUID,
|
|
99
|
-
part_path TEXT NOT NULL,
|
|
100
|
-
media_type TEXT NOT NULL,
|
|
101
|
-
media_subtype TEXT NOT NULL,
|
|
102
|
-
content_id TEXT,
|
|
103
|
-
content_description TEXT,
|
|
104
|
-
transfer_encoding TEXT NOT NULL,
|
|
105
|
-
size_octets INTEGER NOT NULL,
|
|
106
|
-
line_count INTEGER,
|
|
107
|
-
md5_hash TEXT,
|
|
108
|
-
disposition TEXT,
|
|
109
|
-
disposition_filename TEXT,
|
|
110
|
-
language TEXT,
|
|
111
|
-
location TEXT,
|
|
112
|
-
is_multipart BOOLEAN NOT NULL,
|
|
113
|
-
multipart_subtype TEXT,
|
|
114
|
-
created_at BIGINT NOT NULL,
|
|
115
|
-
updated_at BIGINT NOT NULL
|
|
116
|
-
);
|
|
117
|
-
CREATE INDEX IF NOT EXISTS body_part_message_id_idx ON body_part (message_id);
|
|
118
|
-
|
|
119
|
-
CREATE TABLE IF NOT EXISTS body_part_parameter (
|
|
120
|
-
body_part_parameter_id UUID PRIMARY KEY,
|
|
121
|
-
message_id UUID NOT NULL,
|
|
122
|
-
body_part_id UUID NOT NULL,
|
|
123
|
-
parameter_name TEXT NOT NULL,
|
|
124
|
-
parameter_value TEXT NOT NULL,
|
|
125
|
-
created_at BIGINT NOT NULL,
|
|
126
|
-
updated_at BIGINT NOT NULL
|
|
127
|
-
);
|
|
128
|
-
CREATE INDEX IF NOT EXISTS body_part_parameter_message_id_idx ON body_part_parameter (message_id);
|
|
129
|
-
|
|
130
|
-
CREATE TABLE IF NOT EXISTS raw_message_storage (
|
|
131
|
-
raw_storage_id UUID PRIMARY KEY,
|
|
132
|
-
message_id UUID NOT NULL,
|
|
133
|
-
storage_type TEXT NOT NULL,
|
|
134
|
-
storage_location TEXT NOT NULL,
|
|
135
|
-
storage_key TEXT NOT NULL,
|
|
136
|
-
size_bytes INTEGER NOT NULL,
|
|
137
|
-
checksum_sha256 TEXT NOT NULL,
|
|
138
|
-
content_encoding TEXT NOT NULL,
|
|
139
|
-
stored_at BIGINT NOT NULL,
|
|
140
|
-
expires_at BIGINT,
|
|
141
|
-
created_at BIGINT NOT NULL,
|
|
142
|
-
updated_at BIGINT NOT NULL
|
|
143
|
-
);
|
|
144
|
-
CREATE INDEX IF NOT EXISTS raw_message_storage_message_id_idx ON raw_message_storage (message_id);
|
|
145
|
-
|
|
146
|
-
CREATE TABLE IF NOT EXISTS body_part_storage (
|
|
147
|
-
body_part_storage_id UUID PRIMARY KEY,
|
|
148
|
-
message_id UUID NOT NULL,
|
|
149
|
-
body_part_id UUID NOT NULL,
|
|
150
|
-
storage_type TEXT NOT NULL,
|
|
151
|
-
storage_location TEXT NOT NULL,
|
|
152
|
-
storage_key TEXT NOT NULL,
|
|
153
|
-
decoded_size_bytes INTEGER NOT NULL,
|
|
154
|
-
checksum_sha256 TEXT NOT NULL,
|
|
155
|
-
content_encoding TEXT NOT NULL,
|
|
156
|
-
is_deduped BOOLEAN NOT NULL,
|
|
157
|
-
dedup_hash TEXT,
|
|
158
|
-
stored_at BIGINT NOT NULL,
|
|
159
|
-
created_at BIGINT NOT NULL,
|
|
160
|
-
updated_at BIGINT NOT NULL
|
|
161
|
-
);
|
|
162
|
-
CREATE INDEX IF NOT EXISTS body_part_storage_message_id_idx ON body_part_storage (message_id);
|
|
163
|
-
|
|
164
|
-
CREATE TABLE IF NOT EXISTS body_part_content (
|
|
165
|
-
body_part_content_id UUID PRIMARY KEY,
|
|
166
|
-
message_id UUID NOT NULL,
|
|
167
|
-
body_part_id UUID NOT NULL,
|
|
168
|
-
content TEXT NOT NULL,
|
|
169
|
-
content_length INTEGER NOT NULL,
|
|
170
|
-
created_at BIGINT NOT NULL,
|
|
171
|
-
updated_at BIGINT NOT NULL
|
|
172
|
-
);
|
|
173
|
-
CREATE INDEX IF NOT EXISTS body_part_content_message_id_idx ON body_part_content (message_id);
|
|
174
|
-
|
|
175
|
-
CREATE TABLE IF NOT EXISTS message (
|
|
176
|
-
message_id UUID PRIMARY KEY,
|
|
177
|
-
mailbox_id UUID NOT NULL,
|
|
178
|
-
uid INTEGER NOT NULL,
|
|
179
|
-
sequence_number INTEGER NOT NULL,
|
|
180
|
-
rfc822_size INTEGER NOT NULL,
|
|
181
|
-
internal_date BIGINT NOT NULL,
|
|
182
|
-
message_id_header TEXT,
|
|
183
|
-
envelope_id UUID NOT NULL,
|
|
184
|
-
root_body_part_id UUID NOT NULL,
|
|
185
|
-
body_storage_key TEXT,
|
|
186
|
-
status TEXT NOT NULL DEFAULT 'active',
|
|
187
|
-
sync_status TEXT NOT NULL DEFAULT 'pending',
|
|
188
|
-
original_mailbox_id UUID,
|
|
189
|
-
original_uid INTEGER,
|
|
190
|
-
category TEXT NOT NULL DEFAULT 'uncategorized',
|
|
191
|
-
authenticity JSONB,
|
|
192
|
-
auth_result JSONB,
|
|
193
|
-
provider_spam JSONB,
|
|
194
|
-
has_list_unsubscribe BOOLEAN NOT NULL DEFAULT false,
|
|
195
|
-
moved_by_remit BOOLEAN NOT NULL DEFAULT false,
|
|
196
|
-
placement_verdict JSONB,
|
|
197
|
-
filter_move JSONB,
|
|
198
|
-
placement_decided_at BIGINT,
|
|
199
|
-
created_at BIGINT NOT NULL,
|
|
200
|
-
updated_at BIGINT NOT NULL
|
|
201
|
-
);
|
|
202
|
-
CREATE INDEX IF NOT EXISTS message_mailbox_id_idx ON message (mailbox_id);
|
|
203
|
-
|
|
204
|
-
CREATE TABLE IF NOT EXISTS message_flag (
|
|
205
|
-
message_flag_id UUID PRIMARY KEY,
|
|
206
|
-
message_id UUID NOT NULL,
|
|
207
|
-
flag_name TEXT NOT NULL,
|
|
208
|
-
set_at BIGINT NOT NULL,
|
|
209
|
-
created_at BIGINT NOT NULL,
|
|
210
|
-
updated_at BIGINT NOT NULL
|
|
211
|
-
);
|
|
212
|
-
CREATE INDEX IF NOT EXISTS message_flag_message_id_idx ON message_flag (message_id);
|
|
213
|
-
|
|
214
|
-
CREATE TABLE IF NOT EXISTS outbox (
|
|
215
|
-
id UUID PRIMARY KEY,
|
|
216
|
-
message_id UUID NOT NULL,
|
|
217
|
-
event TEXT NOT NULL,
|
|
218
|
-
payload JSONB NOT NULL,
|
|
219
|
-
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
|
220
|
-
processed_at BIGINT
|
|
221
|
-
);
|
|
222
|
-
CREATE INDEX IF NOT EXISTS outbox_message_id_idx ON outbox (message_id);
|
|
223
|
-
CREATE INDEX IF NOT EXISTS outbox_unprocessed_idx ON outbox (created_at) WHERE processed_at IS NULL;
|
|
224
|
-
`;
|
|
@@ -1,12 +1,9 @@
|
|
|
1
1
|
import assert from "node:assert";
|
|
2
|
-
import { readFileSync } from "node:fs";
|
|
3
2
|
import { after, before, describe, test } from "node:test";
|
|
4
|
-
import { fileURLToPath } from "node:url";
|
|
5
3
|
import type { CreateThreadMessageInput } from "@remit/data-ports";
|
|
6
|
-
import { sql } from "drizzle-orm";
|
|
7
|
-
import { drizzle } from "drizzle-orm/node-postgres";
|
|
8
4
|
import shortUuid from "short-uuid";
|
|
9
5
|
import { threadMessageTable } from "../schema/thread-message.js";
|
|
6
|
+
import { createSqliteTestDb } from "../test-db-sqlite.js";
|
|
10
7
|
import {
|
|
11
8
|
clampThreadSearchLimit,
|
|
12
9
|
DrizzleThreadMessageRepository,
|
|
@@ -18,50 +15,6 @@ import {
|
|
|
18
15
|
const translator = shortUuid.createTranslator(shortUuid.constants.uuid25Base36);
|
|
19
16
|
const uuid = () => translator.generate();
|
|
20
17
|
|
|
21
|
-
const PG_URL =
|
|
22
|
-
process.env.PG_CONNECTION_URL ??
|
|
23
|
-
"postgresql://remit:remit@localhost:5432/remit_test";
|
|
24
|
-
|
|
25
|
-
// These suites connect to a real Postgres (pg_trgm/unaccent search DDL, which
|
|
26
|
-
// embedded-postgres cannot provide), so they run only in the dedicated pg job
|
|
27
|
-
// that provisions Postgres — mirroring the RUN_INTEG_TESTS gate used across the
|
|
28
|
-
// integration suites. The pure-unit describe below always runs.
|
|
29
|
-
const RUN_INTEG = process.env.RUN_INTEG_TESTS === "1";
|
|
30
|
-
|
|
31
|
-
const DDL = `
|
|
32
|
-
CREATE TABLE IF NOT EXISTS thread_message (
|
|
33
|
-
thread_message_id TEXT PRIMARY KEY,
|
|
34
|
-
account_config_id TEXT NOT NULL,
|
|
35
|
-
thread_id TEXT NOT NULL,
|
|
36
|
-
message_id TEXT NOT NULL,
|
|
37
|
-
mailbox_id TEXT NOT NULL,
|
|
38
|
-
uid INTEGER NOT NULL,
|
|
39
|
-
message_id_header TEXT,
|
|
40
|
-
in_reply_to TEXT,
|
|
41
|
-
reference_order INTEGER NOT NULL DEFAULT 0,
|
|
42
|
-
from_email TEXT,
|
|
43
|
-
from_name TEXT,
|
|
44
|
-
subject TEXT,
|
|
45
|
-
internal_date BIGINT NOT NULL,
|
|
46
|
-
sent_date BIGINT NOT NULL,
|
|
47
|
-
is_read BOOLEAN NOT NULL,
|
|
48
|
-
has_attachment BOOLEAN NOT NULL,
|
|
49
|
-
star TEXT NOT NULL DEFAULT 'none',
|
|
50
|
-
has_stars BOOLEAN NOT NULL,
|
|
51
|
-
is_deleted BOOLEAN NOT NULL,
|
|
52
|
-
snippet TEXT,
|
|
53
|
-
created_at BIGINT NOT NULL,
|
|
54
|
-
updated_at BIGINT NOT NULL
|
|
55
|
-
);
|
|
56
|
-
CREATE INDEX IF NOT EXISTS tm_by_date_idx ON thread_message (account_config_id, sent_date);
|
|
57
|
-
CREATE INDEX IF NOT EXISTS tm_by_mailbox_idx ON thread_message (account_config_id, mailbox_id, sent_date);
|
|
58
|
-
CREATE INDEX IF NOT EXISTS tm_by_attachment_idx ON thread_message (account_config_id, has_attachment, sent_date);
|
|
59
|
-
CREATE INDEX IF NOT EXISTS tm_by_starred_idx ON thread_message (account_config_id, has_stars, sent_date);
|
|
60
|
-
CREATE INDEX IF NOT EXISTS tm_by_mailbox_readstatus_idx ON thread_message (account_config_id, mailbox_id, is_read, sent_date);
|
|
61
|
-
CREATE INDEX IF NOT EXISTS tm_by_thread_idx ON thread_message (thread_id, internal_date);
|
|
62
|
-
CREATE INDEX IF NOT EXISTS tm_by_message_idx ON thread_message (message_id);
|
|
63
|
-
`;
|
|
64
|
-
|
|
65
18
|
function makeInput(
|
|
66
19
|
accountConfigId: string,
|
|
67
20
|
mailboxId: string,
|
|
@@ -85,24 +38,13 @@ function makeInput(
|
|
|
85
38
|
};
|
|
86
39
|
}
|
|
87
40
|
|
|
88
|
-
//
|
|
89
|
-
//
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
)
|
|
95
|
-
"utf8",
|
|
96
|
-
);
|
|
97
|
-
|
|
98
|
-
async function setupDb(): Promise<void> {
|
|
99
|
-
const db = drizzle(PG_URL, { schema: { threadMessage: threadMessageTable } });
|
|
100
|
-
await db.execute(sql.raw(DDL));
|
|
101
|
-
await db.execute(sql.raw(SEARCH_DDL));
|
|
102
|
-
const client = (db as unknown as { $client: { end(): Promise<void> } })
|
|
103
|
-
.$client;
|
|
104
|
-
await client.end();
|
|
105
|
-
}
|
|
41
|
+
// A real better-sqlite3 database with the FTS5 trigram objects the migrator
|
|
42
|
+
// installs, so the search predicates run their shipped path.
|
|
43
|
+
const setupDb = () =>
|
|
44
|
+
createSqliteTestDb(
|
|
45
|
+
{ threadMessage: threadMessageTable },
|
|
46
|
+
{ searchIndex: true },
|
|
47
|
+
);
|
|
106
48
|
|
|
107
49
|
// ─── clampThreadSearchLimit unit tests ───────────────────────────────────────
|
|
108
50
|
|
|
@@ -135,22 +77,22 @@ describe("clampThreadSearchLimit", () => {
|
|
|
135
77
|
// These six scenarios mirror the canonical DynamoDB test suite in
|
|
136
78
|
// packages/remit-electrodb-service/src/models/thread-message.test.ts.
|
|
137
79
|
|
|
138
|
-
describe("DrizzleThreadMessageRepository.searchByMailboxWindow / countByMailbox", {
|
|
139
|
-
skip: !RUN_INTEG,
|
|
140
|
-
}, () => {
|
|
80
|
+
describe("DrizzleThreadMessageRepository.searchByMailboxWindow / countByMailbox", () => {
|
|
141
81
|
let repo: DrizzleThreadMessageRepository;
|
|
82
|
+
let close: () => Promise<void>;
|
|
142
83
|
const cleanup: Array<() => Promise<void>> = [];
|
|
143
84
|
|
|
144
85
|
before(async () => {
|
|
145
|
-
await setupDb();
|
|
146
|
-
|
|
86
|
+
const harness = await setupDb();
|
|
87
|
+
close = harness.close;
|
|
88
|
+
repo = new DrizzleThreadMessageRepository(harness.db);
|
|
147
89
|
});
|
|
148
90
|
|
|
149
91
|
after(async () => {
|
|
150
92
|
for (const fn of cleanup.reverse()) {
|
|
151
93
|
await fn();
|
|
152
94
|
}
|
|
153
|
-
await
|
|
95
|
+
await close();
|
|
154
96
|
});
|
|
155
97
|
|
|
156
98
|
async function seed(
|
|
@@ -248,8 +190,8 @@ describe("DrizzleThreadMessageRepository.searchByMailboxWindow / countByMailbox"
|
|
|
248
190
|
// ── Scenario 4 ────────────────────────────────────────────────────────────
|
|
249
191
|
// Matching runs over the WHOLE mailbox via the trigram index, so an old
|
|
250
192
|
// match well behind the recent rows is still found and paged (the DynamoDB
|
|
251
|
-
// recent-window bound does not apply
|
|
252
|
-
//
|
|
193
|
+
// recent-window bound does not apply here — the improvement over #443 on the
|
|
194
|
+
// DDB path).
|
|
253
195
|
test("an old match behind the recent rows is still found — matching is whole-mailbox, not window-bounded", async () => {
|
|
254
196
|
const acct = uuid();
|
|
255
197
|
const mbx = uuid();
|
|
@@ -387,22 +329,22 @@ describe("DrizzleThreadMessageRepository.searchByMailboxWindow / countByMailbox"
|
|
|
387
329
|
// one query, so matching must span the caller-supplied mailbox scope rather
|
|
388
330
|
// than a single mailbox.
|
|
389
331
|
|
|
390
|
-
describe("DrizzleThreadMessageRepository.searchByDate", {
|
|
391
|
-
skip: !RUN_INTEG,
|
|
392
|
-
}, () => {
|
|
332
|
+
describe("DrizzleThreadMessageRepository.searchByDate", () => {
|
|
393
333
|
let repo: DrizzleThreadMessageRepository;
|
|
334
|
+
let close: () => Promise<void>;
|
|
394
335
|
const cleanup: Array<() => Promise<void>> = [];
|
|
395
336
|
|
|
396
337
|
before(async () => {
|
|
397
|
-
await setupDb();
|
|
398
|
-
|
|
338
|
+
const harness = await setupDb();
|
|
339
|
+
close = harness.close;
|
|
340
|
+
repo = new DrizzleThreadMessageRepository(harness.db);
|
|
399
341
|
});
|
|
400
342
|
|
|
401
343
|
after(async () => {
|
|
402
344
|
for (const fn of cleanup.reverse()) {
|
|
403
345
|
await fn();
|
|
404
346
|
}
|
|
405
|
-
await
|
|
347
|
+
await close();
|
|
406
348
|
});
|
|
407
349
|
|
|
408
350
|
async function seed(
|
|
@@ -659,25 +601,25 @@ describe("DrizzleThreadMessageRepository.searchByDate", {
|
|
|
659
601
|
|
|
660
602
|
// ─── Native text-search semantics ─────────────────────────────────────────────
|
|
661
603
|
// The type-ahead search box lowercases the query before sending it. These tests
|
|
662
|
-
// pin the
|
|
604
|
+
// pin the engine-native behaviour: case- and accent-insensitive substring
|
|
663
605
|
// matching over the whole mailbox, scoped to the account/mailbox.
|
|
664
606
|
|
|
665
|
-
describe("DrizzleThreadMessageRepository — native text search", {
|
|
666
|
-
skip: !RUN_INTEG,
|
|
667
|
-
}, () => {
|
|
607
|
+
describe("DrizzleThreadMessageRepository — native text search", () => {
|
|
668
608
|
let repo: DrizzleThreadMessageRepository;
|
|
609
|
+
let close: () => Promise<void>;
|
|
669
610
|
const cleanup: Array<() => Promise<void>> = [];
|
|
670
611
|
|
|
671
612
|
before(async () => {
|
|
672
|
-
await setupDb();
|
|
673
|
-
|
|
613
|
+
const harness = await setupDb();
|
|
614
|
+
close = harness.close;
|
|
615
|
+
repo = new DrizzleThreadMessageRepository(harness.db);
|
|
674
616
|
});
|
|
675
617
|
|
|
676
618
|
after(async () => {
|
|
677
619
|
for (const fn of cleanup.reverse()) {
|
|
678
620
|
await fn();
|
|
679
621
|
}
|
|
680
|
-
await
|
|
622
|
+
await close();
|
|
681
623
|
});
|
|
682
624
|
|
|
683
625
|
async function seed(
|
|
@@ -834,22 +776,22 @@ describe("DrizzleThreadMessageRepository — native text search", {
|
|
|
834
776
|
|
|
835
777
|
// ─── Smoke tests for the full interface ──────────────────────────────────────
|
|
836
778
|
|
|
837
|
-
describe("DrizzleThreadMessageRepository — core CRUD", {
|
|
838
|
-
skip: !RUN_INTEG,
|
|
839
|
-
}, () => {
|
|
779
|
+
describe("DrizzleThreadMessageRepository — core CRUD", () => {
|
|
840
780
|
let repo: DrizzleThreadMessageRepository;
|
|
781
|
+
let close: () => Promise<void>;
|
|
841
782
|
const cleanup: Array<() => Promise<void>> = [];
|
|
842
783
|
|
|
843
784
|
before(async () => {
|
|
844
|
-
await setupDb();
|
|
845
|
-
|
|
785
|
+
const harness = await setupDb();
|
|
786
|
+
close = harness.close;
|
|
787
|
+
repo = new DrizzleThreadMessageRepository(harness.db);
|
|
846
788
|
});
|
|
847
789
|
|
|
848
790
|
after(async () => {
|
|
849
791
|
for (const fn of cleanup.reverse()) {
|
|
850
792
|
await fn();
|
|
851
793
|
}
|
|
852
|
-
await
|
|
794
|
+
await close();
|
|
853
795
|
});
|
|
854
796
|
|
|
855
797
|
test("create and get round-trip", async () => {
|
|
@@ -19,7 +19,6 @@ import {
|
|
|
19
19
|
type SQL,
|
|
20
20
|
sql,
|
|
21
21
|
} from "drizzle-orm";
|
|
22
|
-
import { drizzle } from "drizzle-orm/node-postgres";
|
|
23
22
|
import shortUuid from "short-uuid";
|
|
24
23
|
import { v5 as uuidv5 } from "uuid";
|
|
25
24
|
import type { Db } from "../db.js";
|
|
@@ -90,7 +89,6 @@ function decodeAccountCursor(token: string): AccountCursor {
|
|
|
90
89
|
|
|
91
90
|
// ─── Schema ──────────────────────────────────────────────────────────────────
|
|
92
91
|
|
|
93
|
-
const SCHEMA = { threadMessage: threadMessageTable };
|
|
94
92
|
type Row = typeof threadMessageTable.$inferSelect;
|
|
95
93
|
|
|
96
94
|
// ─── Row mapping ─────────────────────────────────────────────────────────────
|
|
@@ -128,9 +126,9 @@ function toItem(row: Row): ThreadMessageItem {
|
|
|
128
126
|
|
|
129
127
|
// ─── Search predicates ────────────────────────────────────────────────────────
|
|
130
128
|
|
|
131
|
-
// The accent-/case-insensitive substring predicates are the one
|
|
132
|
-
// seam
|
|
133
|
-
//
|
|
129
|
+
// The accent-/case-insensitive substring predicates are the one engine-specific
|
|
130
|
+
// text-search seam; they live in ./thread-search-predicates.ts (the FTS5 trigram
|
|
131
|
+
// index, with a folded LIKE fallback below three characters, RFC 036 D4).
|
|
134
132
|
|
|
135
133
|
// Translate SearchOptions into SQL conditions: subject/from/query as indexed
|
|
136
134
|
// text predicates, the rest as plain column equalities. A multi-word `query`
|
|
@@ -196,22 +194,12 @@ export class DrizzleThreadMessageRepository
|
|
|
196
194
|
{
|
|
197
195
|
private db: Db<Record<string, unknown>>;
|
|
198
196
|
|
|
199
|
-
constructor(
|
|
200
|
-
this.db =
|
|
201
|
-
typeof connectionOrDb === "string"
|
|
202
|
-
? drizzle(connectionOrDb, { schema: SCHEMA })
|
|
203
|
-
: connectionOrDb;
|
|
197
|
+
constructor(db: Db<Record<string, unknown>>) {
|
|
198
|
+
this.db = db;
|
|
204
199
|
}
|
|
205
200
|
|
|
206
201
|
async close(): Promise<void> {
|
|
207
|
-
// The underlying driver differs by dialect: a pg Pool closes with `end()`,
|
|
208
|
-
// a better-sqlite3 Database with `close()`. Feature-detect so a sqlite
|
|
209
|
-
// handle never hits a missing `end()`.
|
|
210
202
|
const client = (this.db as unknown as { $client?: unknown }).$client;
|
|
211
|
-
if (client && typeof (client as { end?: unknown }).end === "function") {
|
|
212
|
-
await (client as { end: () => Promise<void> }).end();
|
|
213
|
-
return;
|
|
214
|
-
}
|
|
215
203
|
if (client && typeof (client as { close?: unknown }).close === "function") {
|
|
216
204
|
(client as { close: () => void }).close();
|
|
217
205
|
}
|
|
@@ -831,8 +819,8 @@ export class DrizzleThreadMessageRepository
|
|
|
831
819
|
* the boolean filters) runs in SQL over the whole mailbox via the trigram
|
|
832
820
|
* indexes, ordered by sent_date with an id tiebreak. `limit` is a page size
|
|
833
821
|
* over MATCHES (clamped to THREAD_SEARCH_MAX_LIMIT); the DynamoDB name is
|
|
834
|
-
* kept for interface parity, but there is no recent-window read bound —
|
|
835
|
-
*
|
|
822
|
+
* kept for interface parity, but there is no recent-window read bound — the
|
|
823
|
+
* text is indexed, so a match anywhere in the mailbox is reachable.
|
|
836
824
|
*
|
|
837
825
|
* Cursor: the last returned row `(sentDate, threadMessageId)`. A full page
|
|
838
826
|
* yields a cursor so callers can resume.
|