@remit/drizzle-service 0.0.35 → 0.0.37
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +3 -10
- package/src/db.ts +6 -6
- package/src/error.ts +2 -5
- package/src/index.ts +0 -1
- package/src/repair/thread-message-category-contract.ts +2 -4
- package/src/repair/thread-message-category.sqlite.test.ts +1 -2
- package/src/repair/thread-message-category.ts +15 -25
- package/src/repos/cascade-delete.ts +8 -22
- package/src/repos/category-fixture.ts +2 -19
- package/src/repos/filter-anchor.ts +4 -4
- package/src/repos/filter.ts +3 -3
- package/src/repos/i4-account-config.ts +2 -2
- package/src/repos/i4-account-export-request.ts +2 -2
- package/src/repos/i4-account-setting.ts +2 -2
- package/src/repos/i4-account.ts +2 -2
- package/src/repos/i4-mailbox-lock.ts +2 -2
- package/src/repos/i4-mailbox-special-use.ts +2 -2
- package/src/repos/i4-mailbox.ts +2 -2
- package/src/repos/i4-message-flag-push.test.ts +1 -1
- package/src/repos/i4-message-flag-push.ts +3 -3
- package/src/repos/i4-message-placement-move.test.ts +1 -1
- package/src/repos/i4-message-placement-move.ts +4 -3
- package/src/repos/i4-organize-job-request.ts +2 -2
- package/src/repos/i4-outbox-message.ts +2 -2
- package/src/repos/label.ts +2 -2
- package/src/repos/message-flag.ts +2 -2
- package/src/repos/message-label.ts +2 -2
- package/src/repos/message.ts +8 -10
- package/src/repos/quarantine.ts +2 -2
- package/src/repos/test-helpers.ts +11 -213
- package/src/repos/thread-message.sqlite.test.ts +35 -10
- package/src/repos/thread-message.test.ts +45 -105
- package/src/repos/thread-message.ts +11 -25
- package/src/repos/thread-search-predicates.ts +13 -41
- package/src/repos/unit-of-work.ts +1 -1
- package/src/schema/i4-account-config.ts +1 -1
- package/src/schema/i4-account-export-request.ts +1 -1
- package/src/schema/i4-account-setting.ts +1 -1
- package/src/schema/i4-address.ts +1 -1
- package/src/schema/i4-mailbox-lock.ts +1 -1
- package/src/schema/i4-mailbox.ts +1 -1
- package/src/schema/i4-message-flag-push.ts +1 -1
- package/src/schema/i4-message-placement-move.ts +1 -1
- package/src/schema/i4-organize-job-request.ts +1 -1
- package/src/schema/i4-outbox-message.ts +1 -1
- package/src/schema/message-data.ts +1 -1
- package/src/schema/outbox.ts +13 -56
- package/src/schema/quarantine.ts +1 -1
- package/src/schema/thread-message.ts +1 -1
- package/src/schema-full-sqlite.ts +8 -6
- package/src/schema.ts +4 -4
- package/src/sqlite-client.ts +4 -5
- package/src/test-db-sqlite.ts +4 -9
- package/src/test-db.ts +11 -66
- package/src/tx.ts +6 -12
- package/drizzle.config.ts +0 -15
- package/src/dialect.ts +0 -13
- package/src/repair/thread-message-category.test.ts +0 -229
- package/src/repos/thread-message-category.test.ts +0 -156
- package/src/schema/active-entities.ts +0 -19
- package/src/schema-full.ts +0 -16
|
@@ -4,11 +4,11 @@ import type {
|
|
|
4
4
|
MessageLabelItem,
|
|
5
5
|
} from "@remit/data-ports";
|
|
6
6
|
import { and, desc, eq, inArray } from "drizzle-orm";
|
|
7
|
-
import type {
|
|
7
|
+
import type { Db } from "../db.js";
|
|
8
8
|
import { deterministicBase36Id } from "../id.js";
|
|
9
9
|
import { messageLabelTable } from "../schema.js";
|
|
10
10
|
|
|
11
|
-
type DB =
|
|
11
|
+
type DB = Db<Record<string, unknown>>;
|
|
12
12
|
|
|
13
13
|
function rowToMessageLabel(
|
|
14
14
|
row: typeof messageLabelTable.$inferSelect,
|
package/src/repos/message.ts
CHANGED
|
@@ -6,7 +6,7 @@ import type {
|
|
|
6
6
|
MessageItem,
|
|
7
7
|
} from "@remit/data-ports";
|
|
8
8
|
import { and, asc, eq, gt, inArray, or } from "drizzle-orm";
|
|
9
|
-
|
|
9
|
+
|
|
10
10
|
import type { Db } from "../db.js";
|
|
11
11
|
import {
|
|
12
12
|
CreateFailedConflictError,
|
|
@@ -99,15 +99,12 @@ function toMessageItem(row: typeof messageTable.$inferSelect): MessageItem {
|
|
|
99
99
|
|
|
100
100
|
/**
|
|
101
101
|
* Emitted into the transactional outbox when a message's rows are deleted, so
|
|
102
|
-
* the
|
|
103
|
-
*
|
|
102
|
+
* the search-index worker relays a search-index REMOVE and the vectors are
|
|
103
|
+
* dropped.
|
|
104
104
|
*/
|
|
105
105
|
export const MESSAGE_REMOVED_EVENT = "message.removed";
|
|
106
106
|
|
|
107
|
-
export type SubtreeDb = Pick<
|
|
108
|
-
NodePgDatabase<Record<string, unknown>>,
|
|
109
|
-
"delete" | "insert"
|
|
110
|
-
>;
|
|
107
|
+
export type SubtreeDb = Pick<Db<Record<string, unknown>>, "delete" | "insert">;
|
|
111
108
|
|
|
112
109
|
/**
|
|
113
110
|
* Delete a message and its whole per-message subtree by message id — the nine
|
|
@@ -201,7 +198,7 @@ export class DrizzleMessageRepository implements IMessageRepository {
|
|
|
201
198
|
};
|
|
202
199
|
|
|
203
200
|
// Faithful to ElectroDB message.create: a duplicate messageId throws
|
|
204
|
-
// CreateFailedConflictError. The plain insert raises a
|
|
201
|
+
// CreateFailedConflictError. The plain insert raises a unique-constraint
|
|
205
202
|
// violation, which rolls back the transaction so NO outbox row is
|
|
206
203
|
// written; we surface it as the domain conflict error.
|
|
207
204
|
try {
|
|
@@ -397,7 +394,8 @@ export class DrizzleMessageRepository implements IMessageRepository {
|
|
|
397
394
|
// A non-empty bodyStorageKey means body-sync just persisted the parsed
|
|
398
395
|
// body, so the message now has embeddable content and its threadMessage
|
|
399
396
|
// exists. Append a search-index event in the same transaction as the write
|
|
400
|
-
// (transactional outbox) — the
|
|
397
|
+
// (transactional outbox) — the search-index worker relays it to SQS and
|
|
398
|
+
// embeds.
|
|
401
399
|
// The outbox is append-only: the worker's content-hash gate makes a
|
|
402
400
|
// redundant pass near-free.
|
|
403
401
|
const bodySynced =
|
|
@@ -540,7 +538,7 @@ export class DrizzleMessageRepository implements IMessageRepository {
|
|
|
540
538
|
// mailbox and its COPYUID). The message's search vectors still carry the
|
|
541
539
|
// old mailbox in their metadata and their body is unchanged, so a normal
|
|
542
540
|
// re-index would skip them on content hash. Enqueue a move re-index event
|
|
543
|
-
// in the same transaction as the update; the
|
|
541
|
+
// in the same transaction as the update; the search-index worker drains it
|
|
544
542
|
// with force, refreshing the stored mailbox metadata.
|
|
545
543
|
const rows = await runInTransaction(this.db, async (tx) => {
|
|
546
544
|
const updated = await tx
|
package/src/repos/quarantine.ts
CHANGED
|
@@ -6,10 +6,10 @@ import type {
|
|
|
6
6
|
} from "@remit/data-ports";
|
|
7
7
|
import { deriveQuarantineId } from "@remit/data-ports/id";
|
|
8
8
|
import { desc, eq } from "drizzle-orm";
|
|
9
|
-
import type {
|
|
9
|
+
import type { Db } from "../db.js";
|
|
10
10
|
import { quarantineTable } from "../schema/quarantine.js";
|
|
11
11
|
|
|
12
|
-
type DB =
|
|
12
|
+
type DB = Db<Record<string, unknown>>;
|
|
13
13
|
|
|
14
14
|
function rowToItem(row: typeof quarantineTable.$inferSelect): QuarantineItem {
|
|
15
15
|
return {
|
|
@@ -1,224 +1,22 @@
|
|
|
1
|
-
import
|
|
2
|
-
import {
|
|
3
|
-
import type { NodePgDatabase } from "drizzle-orm/node-postgres";
|
|
4
|
-
import { drizzle } from "drizzle-orm/node-postgres";
|
|
5
|
-
import EmbeddedPostgres from "embedded-postgres";
|
|
1
|
+
import type Database from "better-sqlite3";
|
|
2
|
+
import type { Db } from "../db.js";
|
|
6
3
|
import {
|
|
7
4
|
type MessageDataSchema,
|
|
8
5
|
messageDataSchema,
|
|
9
6
|
} from "../schema/message-data.js";
|
|
7
|
+
import { createSqliteTestDb } from "../test-db-sqlite.js";
|
|
10
8
|
|
|
11
|
-
export type TestDb =
|
|
12
|
-
|
|
13
|
-
function randomPort(): number {
|
|
14
|
-
return 54500 + Math.floor(Math.random() * 400);
|
|
15
|
-
}
|
|
9
|
+
export type TestDb = Db<MessageDataSchema>;
|
|
16
10
|
|
|
11
|
+
/**
|
|
12
|
+
* The message-data tables on a throwaway in-memory SQLite database — the
|
|
13
|
+
* harness for the repos that only touch that subset.
|
|
14
|
+
*/
|
|
17
15
|
export async function createTestDb(): Promise<{
|
|
18
16
|
db: TestDb;
|
|
17
|
+
sqlite: Database.Database;
|
|
19
18
|
stop: () => Promise<void>;
|
|
20
19
|
}> {
|
|
21
|
-
const
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
const pg = new EmbeddedPostgres({
|
|
25
|
-
databaseDir,
|
|
26
|
-
user: "test",
|
|
27
|
-
password: "test",
|
|
28
|
-
port,
|
|
29
|
-
persistent: false,
|
|
30
|
-
});
|
|
31
|
-
|
|
32
|
-
await pg.initialise();
|
|
33
|
-
await pg.start();
|
|
34
|
-
|
|
35
|
-
const connectionString = `postgresql://test:test@localhost:${port}/postgres`;
|
|
36
|
-
const db = drizzle(connectionString, { schema: messageDataSchema }) as TestDb;
|
|
37
|
-
|
|
38
|
-
await db.execute(sql.raw(DDL));
|
|
39
|
-
|
|
40
|
-
return {
|
|
41
|
-
db,
|
|
42
|
-
stop: async () => {
|
|
43
|
-
const client = (
|
|
44
|
-
db as unknown as { $client: { end: () => Promise<void> } }
|
|
45
|
-
).$client;
|
|
46
|
-
await client.end();
|
|
47
|
-
await pg.stop();
|
|
48
|
-
try {
|
|
49
|
-
rmSync(databaseDir, { recursive: true, force: true });
|
|
50
|
-
} catch {
|
|
51
|
-
// best-effort cleanup
|
|
52
|
-
}
|
|
53
|
-
},
|
|
54
|
-
};
|
|
20
|
+
const { db, sqlite, close } = await createSqliteTestDb(messageDataSchema);
|
|
21
|
+
return { db, sqlite, stop: close };
|
|
55
22
|
}
|
|
56
|
-
|
|
57
|
-
const DDL = `
|
|
58
|
-
CREATE TABLE IF NOT EXISTS envelope (
|
|
59
|
-
envelope_id UUID PRIMARY KEY,
|
|
60
|
-
message_id UUID NOT NULL,
|
|
61
|
-
date_value BIGINT NOT NULL,
|
|
62
|
-
date_raw TEXT NOT NULL,
|
|
63
|
-
subject TEXT,
|
|
64
|
-
message_id_value TEXT,
|
|
65
|
-
created_at BIGINT NOT NULL,
|
|
66
|
-
updated_at BIGINT NOT NULL
|
|
67
|
-
);
|
|
68
|
-
CREATE INDEX IF NOT EXISTS envelope_message_id_idx ON envelope (message_id);
|
|
69
|
-
|
|
70
|
-
CREATE TABLE IF NOT EXISTS message_reference (
|
|
71
|
-
message_reference_id UUID PRIMARY KEY,
|
|
72
|
-
message_id UUID NOT NULL,
|
|
73
|
-
envelope_id UUID NOT NULL,
|
|
74
|
-
message_id_value TEXT NOT NULL,
|
|
75
|
-
reference_type TEXT NOT NULL,
|
|
76
|
-
reference_order INTEGER NOT NULL,
|
|
77
|
-
created_at BIGINT NOT NULL,
|
|
78
|
-
updated_at BIGINT NOT NULL
|
|
79
|
-
);
|
|
80
|
-
CREATE INDEX IF NOT EXISTS message_reference_message_id_idx ON message_reference (message_id);
|
|
81
|
-
|
|
82
|
-
CREATE TABLE IF NOT EXISTS envelope_address (
|
|
83
|
-
envelope_address_id UUID PRIMARY KEY,
|
|
84
|
-
message_id UUID NOT NULL,
|
|
85
|
-
address_id UUID NOT NULL,
|
|
86
|
-
display_name TEXT,
|
|
87
|
-
normalized_email TEXT NOT NULL,
|
|
88
|
-
address_role TEXT NOT NULL,
|
|
89
|
-
address_order INTEGER NOT NULL,
|
|
90
|
-
created_at BIGINT NOT NULL,
|
|
91
|
-
updated_at BIGINT NOT NULL
|
|
92
|
-
);
|
|
93
|
-
CREATE INDEX IF NOT EXISTS envelope_address_message_id_idx ON envelope_address (message_id);
|
|
94
|
-
|
|
95
|
-
CREATE TABLE IF NOT EXISTS body_part (
|
|
96
|
-
body_part_id UUID PRIMARY KEY,
|
|
97
|
-
message_id UUID NOT NULL,
|
|
98
|
-
parent_body_part_id UUID,
|
|
99
|
-
part_path TEXT NOT NULL,
|
|
100
|
-
media_type TEXT NOT NULL,
|
|
101
|
-
media_subtype TEXT NOT NULL,
|
|
102
|
-
content_id TEXT,
|
|
103
|
-
content_description TEXT,
|
|
104
|
-
transfer_encoding TEXT NOT NULL,
|
|
105
|
-
size_octets INTEGER NOT NULL,
|
|
106
|
-
line_count INTEGER,
|
|
107
|
-
md5_hash TEXT,
|
|
108
|
-
disposition TEXT,
|
|
109
|
-
disposition_filename TEXT,
|
|
110
|
-
language TEXT,
|
|
111
|
-
location TEXT,
|
|
112
|
-
is_multipart BOOLEAN NOT NULL,
|
|
113
|
-
multipart_subtype TEXT,
|
|
114
|
-
created_at BIGINT NOT NULL,
|
|
115
|
-
updated_at BIGINT NOT NULL
|
|
116
|
-
);
|
|
117
|
-
CREATE INDEX IF NOT EXISTS body_part_message_id_idx ON body_part (message_id);
|
|
118
|
-
|
|
119
|
-
CREATE TABLE IF NOT EXISTS body_part_parameter (
|
|
120
|
-
body_part_parameter_id UUID PRIMARY KEY,
|
|
121
|
-
message_id UUID NOT NULL,
|
|
122
|
-
body_part_id UUID NOT NULL,
|
|
123
|
-
parameter_name TEXT NOT NULL,
|
|
124
|
-
parameter_value TEXT NOT NULL,
|
|
125
|
-
created_at BIGINT NOT NULL,
|
|
126
|
-
updated_at BIGINT NOT NULL
|
|
127
|
-
);
|
|
128
|
-
CREATE INDEX IF NOT EXISTS body_part_parameter_message_id_idx ON body_part_parameter (message_id);
|
|
129
|
-
|
|
130
|
-
CREATE TABLE IF NOT EXISTS raw_message_storage (
|
|
131
|
-
raw_storage_id UUID PRIMARY KEY,
|
|
132
|
-
message_id UUID NOT NULL,
|
|
133
|
-
storage_type TEXT NOT NULL,
|
|
134
|
-
storage_location TEXT NOT NULL,
|
|
135
|
-
storage_key TEXT NOT NULL,
|
|
136
|
-
size_bytes INTEGER NOT NULL,
|
|
137
|
-
checksum_sha256 TEXT NOT NULL,
|
|
138
|
-
content_encoding TEXT NOT NULL,
|
|
139
|
-
stored_at BIGINT NOT NULL,
|
|
140
|
-
expires_at BIGINT,
|
|
141
|
-
created_at BIGINT NOT NULL,
|
|
142
|
-
updated_at BIGINT NOT NULL
|
|
143
|
-
);
|
|
144
|
-
CREATE INDEX IF NOT EXISTS raw_message_storage_message_id_idx ON raw_message_storage (message_id);
|
|
145
|
-
|
|
146
|
-
CREATE TABLE IF NOT EXISTS body_part_storage (
|
|
147
|
-
body_part_storage_id UUID PRIMARY KEY,
|
|
148
|
-
message_id UUID NOT NULL,
|
|
149
|
-
body_part_id UUID NOT NULL,
|
|
150
|
-
storage_type TEXT NOT NULL,
|
|
151
|
-
storage_location TEXT NOT NULL,
|
|
152
|
-
storage_key TEXT NOT NULL,
|
|
153
|
-
decoded_size_bytes INTEGER NOT NULL,
|
|
154
|
-
checksum_sha256 TEXT NOT NULL,
|
|
155
|
-
content_encoding TEXT NOT NULL,
|
|
156
|
-
is_deduped BOOLEAN NOT NULL,
|
|
157
|
-
dedup_hash TEXT,
|
|
158
|
-
stored_at BIGINT NOT NULL,
|
|
159
|
-
created_at BIGINT NOT NULL,
|
|
160
|
-
updated_at BIGINT NOT NULL
|
|
161
|
-
);
|
|
162
|
-
CREATE INDEX IF NOT EXISTS body_part_storage_message_id_idx ON body_part_storage (message_id);
|
|
163
|
-
|
|
164
|
-
CREATE TABLE IF NOT EXISTS body_part_content (
|
|
165
|
-
body_part_content_id UUID PRIMARY KEY,
|
|
166
|
-
message_id UUID NOT NULL,
|
|
167
|
-
body_part_id UUID NOT NULL,
|
|
168
|
-
content TEXT NOT NULL,
|
|
169
|
-
content_length INTEGER NOT NULL,
|
|
170
|
-
created_at BIGINT NOT NULL,
|
|
171
|
-
updated_at BIGINT NOT NULL
|
|
172
|
-
);
|
|
173
|
-
CREATE INDEX IF NOT EXISTS body_part_content_message_id_idx ON body_part_content (message_id);
|
|
174
|
-
|
|
175
|
-
CREATE TABLE IF NOT EXISTS message (
|
|
176
|
-
message_id UUID PRIMARY KEY,
|
|
177
|
-
mailbox_id UUID NOT NULL,
|
|
178
|
-
uid INTEGER NOT NULL,
|
|
179
|
-
sequence_number INTEGER NOT NULL,
|
|
180
|
-
rfc822_size INTEGER NOT NULL,
|
|
181
|
-
internal_date BIGINT NOT NULL,
|
|
182
|
-
message_id_header TEXT,
|
|
183
|
-
envelope_id UUID NOT NULL,
|
|
184
|
-
root_body_part_id UUID NOT NULL,
|
|
185
|
-
body_storage_key TEXT,
|
|
186
|
-
status TEXT NOT NULL DEFAULT 'active',
|
|
187
|
-
sync_status TEXT NOT NULL DEFAULT 'pending',
|
|
188
|
-
original_mailbox_id UUID,
|
|
189
|
-
original_uid INTEGER,
|
|
190
|
-
category TEXT NOT NULL DEFAULT 'uncategorized',
|
|
191
|
-
authenticity JSONB,
|
|
192
|
-
auth_result JSONB,
|
|
193
|
-
provider_spam JSONB,
|
|
194
|
-
has_list_unsubscribe BOOLEAN NOT NULL DEFAULT false,
|
|
195
|
-
moved_by_remit BOOLEAN NOT NULL DEFAULT false,
|
|
196
|
-
placement_verdict JSONB,
|
|
197
|
-
filter_move JSONB,
|
|
198
|
-
placement_decided_at BIGINT,
|
|
199
|
-
created_at BIGINT NOT NULL,
|
|
200
|
-
updated_at BIGINT NOT NULL
|
|
201
|
-
);
|
|
202
|
-
CREATE INDEX IF NOT EXISTS message_mailbox_id_idx ON message (mailbox_id);
|
|
203
|
-
|
|
204
|
-
CREATE TABLE IF NOT EXISTS message_flag (
|
|
205
|
-
message_flag_id UUID PRIMARY KEY,
|
|
206
|
-
message_id UUID NOT NULL,
|
|
207
|
-
flag_name TEXT NOT NULL,
|
|
208
|
-
set_at BIGINT NOT NULL,
|
|
209
|
-
created_at BIGINT NOT NULL,
|
|
210
|
-
updated_at BIGINT NOT NULL
|
|
211
|
-
);
|
|
212
|
-
CREATE INDEX IF NOT EXISTS message_flag_message_id_idx ON message_flag (message_id);
|
|
213
|
-
|
|
214
|
-
CREATE TABLE IF NOT EXISTS outbox (
|
|
215
|
-
id UUID PRIMARY KEY,
|
|
216
|
-
message_id UUID NOT NULL,
|
|
217
|
-
event TEXT NOT NULL,
|
|
218
|
-
payload JSONB NOT NULL,
|
|
219
|
-
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
|
220
|
-
processed_at BIGINT
|
|
221
|
-
);
|
|
222
|
-
CREATE INDEX IF NOT EXISTS outbox_message_id_idx ON outbox (message_id);
|
|
223
|
-
CREATE INDEX IF NOT EXISTS outbox_unprocessed_idx ON outbox (created_at) WHERE processed_at IS NULL;
|
|
224
|
-
`;
|
|
@@ -10,7 +10,10 @@ import {
|
|
|
10
10
|
LIVE_TOTALS,
|
|
11
11
|
totalRows,
|
|
12
12
|
} from "./category-fixture.js";
|
|
13
|
-
import {
|
|
13
|
+
import {
|
|
14
|
+
DrizzleThreadMessageRepository,
|
|
15
|
+
THREAD_SEARCH_MAX_LIMIT,
|
|
16
|
+
} from "./thread-message.js";
|
|
14
17
|
|
|
15
18
|
// The thread-message repo on sqlite (RFC 036 D1): CRUD, keyset pagination, and
|
|
16
19
|
// text search — the FTS5 trigram index for terms of three characters or more
|
|
@@ -163,12 +166,9 @@ describe("DrizzleThreadMessageRepository (sqlite)", () => {
|
|
|
163
166
|
});
|
|
164
167
|
|
|
165
168
|
test("countByMailbox counts matches under the same predicate", async () => {
|
|
166
|
-
const n = await repo.countByMailbox(
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
{ subject: "invoice" },
|
|
170
|
-
{ limit: 100 },
|
|
171
|
-
);
|
|
169
|
+
const n = await repo.countByMailbox(ACCOUNT, MAILBOX, {
|
|
170
|
+
subject: "invoice",
|
|
171
|
+
});
|
|
172
172
|
assert.ok(n >= 1);
|
|
173
173
|
});
|
|
174
174
|
|
|
@@ -754,9 +754,6 @@ describe("DrizzleThreadMessageRepository (sqlite)", () => {
|
|
|
754
754
|
assert.equal(seen.length, LIVE_TOTALS.social + LIVE_TOTALS.transactional);
|
|
755
755
|
});
|
|
756
756
|
|
|
757
|
-
// No `limit`: countByMailbox still clamps its answer to the clamped limit,
|
|
758
|
-
// which is what #305 changes. What this asserts is the predicate — the
|
|
759
|
-
// count is over the mailbox rather than over a page of it.
|
|
760
757
|
test("countByMailbox counts the matches in the mailbox, not the page", async () => {
|
|
761
758
|
assert.equal(
|
|
762
759
|
await repo.countByMailbox(
|
|
@@ -769,6 +766,34 @@ describe("DrizzleThreadMessageRepository (sqlite)", () => {
|
|
|
769
766
|
);
|
|
770
767
|
});
|
|
771
768
|
|
|
769
|
+
// Regression for #509. The count was computed in full and then discarded
|
|
770
|
+
// down to THREAD_SEARCH_MAX_LIMIT, so this mailbox's 4,753 personal
|
|
771
|
+
// messages reported as 500 — a number that reads to the user as the whole
|
|
772
|
+
// match. A page size bounds the rows a response carries; it cannot bound
|
|
773
|
+
// how many messages match.
|
|
774
|
+
test("a match far larger than a page reports its real size", async () => {
|
|
775
|
+
assert.ok(
|
|
776
|
+
LIVE_TOTALS.personal > THREAD_SEARCH_MAX_LIMIT,
|
|
777
|
+
"the fixture has to outgrow the page cap for this to mean anything",
|
|
778
|
+
);
|
|
779
|
+
const page = await repo.searchByMailboxWindow(
|
|
780
|
+
CAT_ACCOUNT,
|
|
781
|
+
CAT_MAILBOX,
|
|
782
|
+
{ category: ["personal"] },
|
|
783
|
+
{ order: "desc", excludeDeleted: true },
|
|
784
|
+
);
|
|
785
|
+
assert.equal(page.items.length, THREAD_SEARCH_MAX_LIMIT);
|
|
786
|
+
assert.equal(
|
|
787
|
+
await repo.countByMailbox(
|
|
788
|
+
CAT_ACCOUNT,
|
|
789
|
+
CAT_MAILBOX,
|
|
790
|
+
{ category: ["personal"] },
|
|
791
|
+
{ excludeDeleted: true },
|
|
792
|
+
),
|
|
793
|
+
LIVE_TOTALS.personal,
|
|
794
|
+
);
|
|
795
|
+
});
|
|
796
|
+
|
|
772
797
|
test("the category predicate composes with the other filters", async () => {
|
|
773
798
|
const page = await repo.searchByMailboxWindow(
|
|
774
799
|
CAT_ACCOUNT,
|
|
@@ -1,12 +1,9 @@
|
|
|
1
1
|
import assert from "node:assert";
|
|
2
|
-
import { readFileSync } from "node:fs";
|
|
3
2
|
import { after, before, describe, test } from "node:test";
|
|
4
|
-
import { fileURLToPath } from "node:url";
|
|
5
3
|
import type { CreateThreadMessageInput } from "@remit/data-ports";
|
|
6
|
-
import { sql } from "drizzle-orm";
|
|
7
|
-
import { drizzle } from "drizzle-orm/node-postgres";
|
|
8
4
|
import shortUuid from "short-uuid";
|
|
9
5
|
import { threadMessageTable } from "../schema/thread-message.js";
|
|
6
|
+
import { createSqliteTestDb } from "../test-db-sqlite.js";
|
|
10
7
|
import {
|
|
11
8
|
clampThreadSearchLimit,
|
|
12
9
|
DrizzleThreadMessageRepository,
|
|
@@ -18,50 +15,6 @@ import {
|
|
|
18
15
|
const translator = shortUuid.createTranslator(shortUuid.constants.uuid25Base36);
|
|
19
16
|
const uuid = () => translator.generate();
|
|
20
17
|
|
|
21
|
-
const PG_URL =
|
|
22
|
-
process.env.PG_CONNECTION_URL ??
|
|
23
|
-
"postgresql://remit:remit@localhost:5432/remit_test";
|
|
24
|
-
|
|
25
|
-
// These suites connect to a real Postgres (pg_trgm/unaccent search DDL, which
|
|
26
|
-
// embedded-postgres cannot provide), so they run only in the dedicated pg job
|
|
27
|
-
// that provisions Postgres — mirroring the RUN_INTEG_TESTS gate used across the
|
|
28
|
-
// integration suites. The pure-unit describe below always runs.
|
|
29
|
-
const RUN_INTEG = process.env.RUN_INTEG_TESTS === "1";
|
|
30
|
-
|
|
31
|
-
const DDL = `
|
|
32
|
-
CREATE TABLE IF NOT EXISTS thread_message (
|
|
33
|
-
thread_message_id TEXT PRIMARY KEY,
|
|
34
|
-
account_config_id TEXT NOT NULL,
|
|
35
|
-
thread_id TEXT NOT NULL,
|
|
36
|
-
message_id TEXT NOT NULL,
|
|
37
|
-
mailbox_id TEXT NOT NULL,
|
|
38
|
-
uid INTEGER NOT NULL,
|
|
39
|
-
message_id_header TEXT,
|
|
40
|
-
in_reply_to TEXT,
|
|
41
|
-
reference_order INTEGER NOT NULL DEFAULT 0,
|
|
42
|
-
from_email TEXT,
|
|
43
|
-
from_name TEXT,
|
|
44
|
-
subject TEXT,
|
|
45
|
-
internal_date BIGINT NOT NULL,
|
|
46
|
-
sent_date BIGINT NOT NULL,
|
|
47
|
-
is_read BOOLEAN NOT NULL,
|
|
48
|
-
has_attachment BOOLEAN NOT NULL,
|
|
49
|
-
star TEXT NOT NULL DEFAULT 'none',
|
|
50
|
-
has_stars BOOLEAN NOT NULL,
|
|
51
|
-
is_deleted BOOLEAN NOT NULL,
|
|
52
|
-
snippet TEXT,
|
|
53
|
-
created_at BIGINT NOT NULL,
|
|
54
|
-
updated_at BIGINT NOT NULL
|
|
55
|
-
);
|
|
56
|
-
CREATE INDEX IF NOT EXISTS tm_by_date_idx ON thread_message (account_config_id, sent_date);
|
|
57
|
-
CREATE INDEX IF NOT EXISTS tm_by_mailbox_idx ON thread_message (account_config_id, mailbox_id, sent_date);
|
|
58
|
-
CREATE INDEX IF NOT EXISTS tm_by_attachment_idx ON thread_message (account_config_id, has_attachment, sent_date);
|
|
59
|
-
CREATE INDEX IF NOT EXISTS tm_by_starred_idx ON thread_message (account_config_id, has_stars, sent_date);
|
|
60
|
-
CREATE INDEX IF NOT EXISTS tm_by_mailbox_readstatus_idx ON thread_message (account_config_id, mailbox_id, is_read, sent_date);
|
|
61
|
-
CREATE INDEX IF NOT EXISTS tm_by_thread_idx ON thread_message (thread_id, internal_date);
|
|
62
|
-
CREATE INDEX IF NOT EXISTS tm_by_message_idx ON thread_message (message_id);
|
|
63
|
-
`;
|
|
64
|
-
|
|
65
18
|
function makeInput(
|
|
66
19
|
accountConfigId: string,
|
|
67
20
|
mailboxId: string,
|
|
@@ -85,24 +38,13 @@ function makeInput(
|
|
|
85
38
|
};
|
|
86
39
|
}
|
|
87
40
|
|
|
88
|
-
//
|
|
89
|
-
//
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
)
|
|
95
|
-
"utf8",
|
|
96
|
-
);
|
|
97
|
-
|
|
98
|
-
async function setupDb(): Promise<void> {
|
|
99
|
-
const db = drizzle(PG_URL, { schema: { threadMessage: threadMessageTable } });
|
|
100
|
-
await db.execute(sql.raw(DDL));
|
|
101
|
-
await db.execute(sql.raw(SEARCH_DDL));
|
|
102
|
-
const client = (db as unknown as { $client: { end(): Promise<void> } })
|
|
103
|
-
.$client;
|
|
104
|
-
await client.end();
|
|
105
|
-
}
|
|
41
|
+
// A real better-sqlite3 database with the FTS5 trigram objects the migrator
|
|
42
|
+
// installs, so the search predicates run their shipped path.
|
|
43
|
+
const setupDb = () =>
|
|
44
|
+
createSqliteTestDb(
|
|
45
|
+
{ threadMessage: threadMessageTable },
|
|
46
|
+
{ searchIndex: true },
|
|
47
|
+
);
|
|
106
48
|
|
|
107
49
|
// ─── clampThreadSearchLimit unit tests ───────────────────────────────────────
|
|
108
50
|
|
|
@@ -135,22 +77,22 @@ describe("clampThreadSearchLimit", () => {
|
|
|
135
77
|
// These six scenarios mirror the canonical DynamoDB test suite in
|
|
136
78
|
// packages/remit-electrodb-service/src/models/thread-message.test.ts.
|
|
137
79
|
|
|
138
|
-
describe("DrizzleThreadMessageRepository.searchByMailboxWindow / countByMailbox", {
|
|
139
|
-
skip: !RUN_INTEG,
|
|
140
|
-
}, () => {
|
|
80
|
+
describe("DrizzleThreadMessageRepository.searchByMailboxWindow / countByMailbox", () => {
|
|
141
81
|
let repo: DrizzleThreadMessageRepository;
|
|
82
|
+
let close: () => Promise<void>;
|
|
142
83
|
const cleanup: Array<() => Promise<void>> = [];
|
|
143
84
|
|
|
144
85
|
before(async () => {
|
|
145
|
-
await setupDb();
|
|
146
|
-
|
|
86
|
+
const harness = await setupDb();
|
|
87
|
+
close = harness.close;
|
|
88
|
+
repo = new DrizzleThreadMessageRepository(harness.db);
|
|
147
89
|
});
|
|
148
90
|
|
|
149
91
|
after(async () => {
|
|
150
92
|
for (const fn of cleanup.reverse()) {
|
|
151
93
|
await fn();
|
|
152
94
|
}
|
|
153
|
-
await
|
|
95
|
+
await close();
|
|
154
96
|
});
|
|
155
97
|
|
|
156
98
|
async function seed(
|
|
@@ -248,8 +190,8 @@ describe("DrizzleThreadMessageRepository.searchByMailboxWindow / countByMailbox"
|
|
|
248
190
|
// ── Scenario 4 ────────────────────────────────────────────────────────────
|
|
249
191
|
// Matching runs over the WHOLE mailbox via the trigram index, so an old
|
|
250
192
|
// match well behind the recent rows is still found and paged (the DynamoDB
|
|
251
|
-
// recent-window bound does not apply
|
|
252
|
-
//
|
|
193
|
+
// recent-window bound does not apply here — the improvement over #443 on the
|
|
194
|
+
// DDB path).
|
|
253
195
|
test("an old match behind the recent rows is still found — matching is whole-mailbox, not window-bounded", async () => {
|
|
254
196
|
const acct = uuid();
|
|
255
197
|
const mbx = uuid();
|
|
@@ -299,15 +241,15 @@ describe("DrizzleThreadMessageRepository.searchByMailboxWindow / countByMailbox"
|
|
|
299
241
|
acct,
|
|
300
242
|
mbx,
|
|
301
243
|
{ subject: "match" },
|
|
302
|
-
{ excludeDeleted: true
|
|
244
|
+
{ excludeDeleted: true },
|
|
303
245
|
);
|
|
304
|
-
assert.equal(count,
|
|
246
|
+
assert.equal(count, 4, "count names every match, not one page of them");
|
|
305
247
|
});
|
|
306
248
|
|
|
307
249
|
// ── Scenario 5 ────────────────────────────────────────────────────────────
|
|
308
|
-
// desc default: the first page holds the newest matches; count
|
|
309
|
-
//
|
|
310
|
-
test("desc page holds the newest matches and count
|
|
250
|
+
// desc default: the first page holds the newest matches; the count names the
|
|
251
|
+
// whole match set behind that page.
|
|
252
|
+
test("desc page holds the newest matches and the count names the whole match", async () => {
|
|
311
253
|
const acct = uuid();
|
|
312
254
|
const mbx = uuid();
|
|
313
255
|
const now = Date.now();
|
|
@@ -335,14 +277,13 @@ describe("DrizzleThreadMessageRepository.searchByMailboxWindow / countByMailbox"
|
|
|
335
277
|
acct,
|
|
336
278
|
mbx,
|
|
337
279
|
{ subject: "alpha" },
|
|
338
|
-
{ excludeDeleted: true
|
|
280
|
+
{ excludeDeleted: true },
|
|
339
281
|
);
|
|
340
|
-
assert.equal(count,
|
|
341
|
-
assert.equal(count, 2);
|
|
282
|
+
assert.equal(count, 5, "every alpha row is counted, not the page of two");
|
|
342
283
|
});
|
|
343
284
|
|
|
344
285
|
// ── Scenario 6 ────────────────────────────────────────────────────────────
|
|
345
|
-
// order:asc returns the OLDEST matches first; count
|
|
286
|
+
// order:asc returns the OLDEST matches first; the count is order-independent.
|
|
346
287
|
test("order:asc returns the oldest matches first", async () => {
|
|
347
288
|
const acct = uuid();
|
|
348
289
|
const mbx = uuid();
|
|
@@ -375,10 +316,9 @@ describe("DrizzleThreadMessageRepository.searchByMailboxWindow / countByMailbox"
|
|
|
375
316
|
acct,
|
|
376
317
|
mbx,
|
|
377
318
|
{ subject: "beta" },
|
|
378
|
-
{ excludeDeleted: true,
|
|
319
|
+
{ excludeDeleted: true, order: "asc" },
|
|
379
320
|
);
|
|
380
|
-
assert.equal(count,
|
|
381
|
-
assert.equal(count, 2);
|
|
321
|
+
assert.equal(count, 5, "asc counts the whole match set too");
|
|
382
322
|
});
|
|
383
323
|
});
|
|
384
324
|
|
|
@@ -387,22 +327,22 @@ describe("DrizzleThreadMessageRepository.searchByMailboxWindow / countByMailbox"
|
|
|
387
327
|
// one query, so matching must span the caller-supplied mailbox scope rather
|
|
388
328
|
// than a single mailbox.
|
|
389
329
|
|
|
390
|
-
describe("DrizzleThreadMessageRepository.searchByDate", {
|
|
391
|
-
skip: !RUN_INTEG,
|
|
392
|
-
}, () => {
|
|
330
|
+
describe("DrizzleThreadMessageRepository.searchByDate", () => {
|
|
393
331
|
let repo: DrizzleThreadMessageRepository;
|
|
332
|
+
let close: () => Promise<void>;
|
|
394
333
|
const cleanup: Array<() => Promise<void>> = [];
|
|
395
334
|
|
|
396
335
|
before(async () => {
|
|
397
|
-
await setupDb();
|
|
398
|
-
|
|
336
|
+
const harness = await setupDb();
|
|
337
|
+
close = harness.close;
|
|
338
|
+
repo = new DrizzleThreadMessageRepository(harness.db);
|
|
399
339
|
});
|
|
400
340
|
|
|
401
341
|
after(async () => {
|
|
402
342
|
for (const fn of cleanup.reverse()) {
|
|
403
343
|
await fn();
|
|
404
344
|
}
|
|
405
|
-
await
|
|
345
|
+
await close();
|
|
406
346
|
});
|
|
407
347
|
|
|
408
348
|
async function seed(
|
|
@@ -659,25 +599,25 @@ describe("DrizzleThreadMessageRepository.searchByDate", {
|
|
|
659
599
|
|
|
660
600
|
// ─── Native text-search semantics ─────────────────────────────────────────────
|
|
661
601
|
// The type-ahead search box lowercases the query before sending it. These tests
|
|
662
|
-
// pin the
|
|
602
|
+
// pin the engine-native behaviour: case- and accent-insensitive substring
|
|
663
603
|
// matching over the whole mailbox, scoped to the account/mailbox.
|
|
664
604
|
|
|
665
|
-
describe("DrizzleThreadMessageRepository — native text search", {
|
|
666
|
-
skip: !RUN_INTEG,
|
|
667
|
-
}, () => {
|
|
605
|
+
describe("DrizzleThreadMessageRepository — native text search", () => {
|
|
668
606
|
let repo: DrizzleThreadMessageRepository;
|
|
607
|
+
let close: () => Promise<void>;
|
|
669
608
|
const cleanup: Array<() => Promise<void>> = [];
|
|
670
609
|
|
|
671
610
|
before(async () => {
|
|
672
|
-
await setupDb();
|
|
673
|
-
|
|
611
|
+
const harness = await setupDb();
|
|
612
|
+
close = harness.close;
|
|
613
|
+
repo = new DrizzleThreadMessageRepository(harness.db);
|
|
674
614
|
});
|
|
675
615
|
|
|
676
616
|
after(async () => {
|
|
677
617
|
for (const fn of cleanup.reverse()) {
|
|
678
618
|
await fn();
|
|
679
619
|
}
|
|
680
|
-
await
|
|
620
|
+
await close();
|
|
681
621
|
});
|
|
682
622
|
|
|
683
623
|
async function seed(
|
|
@@ -834,22 +774,22 @@ describe("DrizzleThreadMessageRepository — native text search", {
|
|
|
834
774
|
|
|
835
775
|
// ─── Smoke tests for the full interface ──────────────────────────────────────
|
|
836
776
|
|
|
837
|
-
describe("DrizzleThreadMessageRepository — core CRUD", {
|
|
838
|
-
skip: !RUN_INTEG,
|
|
839
|
-
}, () => {
|
|
777
|
+
describe("DrizzleThreadMessageRepository — core CRUD", () => {
|
|
840
778
|
let repo: DrizzleThreadMessageRepository;
|
|
779
|
+
let close: () => Promise<void>;
|
|
841
780
|
const cleanup: Array<() => Promise<void>> = [];
|
|
842
781
|
|
|
843
782
|
before(async () => {
|
|
844
|
-
await setupDb();
|
|
845
|
-
|
|
783
|
+
const harness = await setupDb();
|
|
784
|
+
close = harness.close;
|
|
785
|
+
repo = new DrizzleThreadMessageRepository(harness.db);
|
|
846
786
|
});
|
|
847
787
|
|
|
848
788
|
after(async () => {
|
|
849
789
|
for (const fn of cleanup.reverse()) {
|
|
850
790
|
await fn();
|
|
851
791
|
}
|
|
852
|
-
await
|
|
792
|
+
await close();
|
|
853
793
|
});
|
|
854
794
|
|
|
855
795
|
test("create and get round-trip", async () => {
|