@remit/drizzle-service 0.0.102 → 0.0.104
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +4 -3
- package/src/batch.sqlite.test.ts +4 -4
- package/src/batch.ts +47 -21
- package/src/databases.ts +24 -6
- package/src/db.ts +20 -5
- package/src/index.ts +12 -0
- package/src/repair/junk-only-address.sqlite.test.ts +7 -2
- package/src/repos/account.conformance.d1.test.ts +20 -0
- package/src/repos/account.conformance.sqlite.test.ts +6 -2
- package/src/repos/calendar-collection.conformance.d1.test.ts +20 -0
- package/src/repos/calendar-collection.ts +2 -2
- package/src/repos/calendar-default.d1.test.ts +4 -0
- package/src/repos/calendar-default.sqlite.test.ts +3 -72
- package/src/repos/calendar-default.suite.ts +68 -0
- package/src/repos/calendar-event-index.conformance.d1.test.ts +20 -0
- package/src/repos/calendar-event-index.ts +2 -2
- package/src/repos/calendar-feed-token.conformance.d1.test.ts +20 -0
- package/src/repos/calendar-feed-token.ts +2 -2
- package/src/repos/calendar-object.conformance.d1.test.ts +20 -0
- package/src/repos/calendar-object.ts +2 -2
- package/src/repos/calendar-put.d1.test.ts +4 -0
- package/src/repos/calendar-put.sqlite.test.ts +3 -205
- package/src/repos/calendar-put.suite.ts +210 -0
- package/src/repos/calendar-store.ts +36 -40
- package/src/repos/calendar-suggestion.ts +2 -2
- package/src/repos/cascade-delete.test.ts +3 -2
- package/src/repos/cascade-delete.ts +12 -8
- package/src/repos/envelope.sqlite.test.ts +3 -2
- package/src/repos/envelope.test.ts +3 -2
- package/src/repos/envelope.ts +14 -9
- package/src/repos/filter.d1.test.ts +4 -0
- package/src/repos/filter.suite.ts +476 -0
- package/src/repos/filter.test.ts +3 -472
- package/src/repos/filter.ts +2 -2
- package/src/repos/i4-account-config.d1.test.ts +4 -0
- package/src/repos/i4-account-config.suite.ts +131 -0
- package/src/repos/i4-account-config.test.ts +3 -123
- package/src/repos/i4-account-config.ts +8 -10
- package/src/repos/i4-account-export-request.d1.test.ts +4 -0
- package/src/repos/i4-account-export-request.suite.ts +44 -0
- package/src/repos/i4-account-export-request.test.ts +3 -40
- package/src/repos/i4-account-export-request.ts +2 -2
- package/src/repos/i4-account-setting.d1.test.ts +4 -0
- package/src/repos/i4-account-setting.suite.ts +100 -0
- package/src/repos/i4-account-setting.test.ts +3 -93
- package/src/repos/i4-account-setting.ts +2 -2
- package/src/repos/i4-account.test.ts +12 -8
- package/src/repos/i4-account.ts +6 -10
- package/src/repos/i4-address-junk-move.sqlite.test.ts +7 -2
- package/src/repos/i4-address.ts +12 -2
- package/src/repos/i4-config-import.ts +2 -2
- package/src/repos/i4-mailbox-lock.ts +2 -2
- package/src/repos/i4-mailbox-special-use.sqlite.test.ts +12 -10
- package/src/repos/i4-mailbox-special-use.ts +29 -40
- package/src/repos/i4-mailbox.sqlite.test.ts +12 -10
- package/src/repos/i4-mailbox.test.ts +15 -14
- package/src/repos/i4-mailbox.ts +13 -7
- package/src/repos/i4-message-flag-push.ts +2 -2
- package/src/repos/i4-message-placement-move.ts +2 -2
- package/src/repos/i4-organize-job-request.ts +2 -2
- package/src/repos/i4-outbox-attachment.test.ts +7 -2
- package/src/repos/i4-outbox-attachment.ts +11 -5
- package/src/repos/i4-outbox-message.ts +2 -2
- package/src/repos/i4-sender-signer-standing.ts +2 -2
- package/src/repos/label.conformance.d1.test.ts +20 -0
- package/src/repos/label.ts +2 -2
- package/src/repos/message-flag.ts +3 -6
- package/src/repos/message-label.ts +2 -2
- package/src/repos/message.sqlite.test.ts +57 -3
- package/src/repos/message.test.ts +6 -5
- package/src/repos/message.ts +81 -10
- package/src/repos/quarantine.ts +2 -2
- package/src/repos/sync-warning.ts +2 -2
- package/src/repos/thread-message-body-muted.sqlite.test.ts +30 -22
- package/src/repos/thread-message-field-terms.sqlite.test.ts +16 -11
- package/src/repos/thread-message.sqlite.test.ts +13 -7
- package/src/repos/thread-message.test.ts +43 -27
- package/src/repos/thread-message.ts +26 -14
- package/src/repos/thread-search-predicates.ts +36 -63
- package/src/schema/message-data.ts +3 -0
- package/src/schema-sets/mail.ts +1 -0
- package/src/search/thread-message-search.sqlite.test.ts +57 -11
- package/src/search/thread-message-search.ts +26 -12
- package/src/sqlite-repositories.ts +48 -32
- package/src/test-d1.ts +40 -0
- package/src/test-db-sqlite.ts +1 -1
- package/src/test-shipped-sqlite-schema.ts +8 -6
- package/src/test-stores.ts +30 -0
|
@@ -6,9 +6,9 @@ import {
|
|
|
6
6
|
} from "../id.js";
|
|
7
7
|
import { messageDataSchema } from "../schema/message-data.js";
|
|
8
8
|
import { threadMessageTable } from "../schema/thread-message.js";
|
|
9
|
-
import {
|
|
9
|
+
import { createSqliteRepositories } from "../sqlite-repositories.js";
|
|
10
10
|
import { createSqliteTestDb, type SqliteTestDb } from "../test-db-sqlite.js";
|
|
11
|
-
import { DrizzleMessageRepository } from "./message.js";
|
|
11
|
+
import type { DrizzleMessageRepository } from "./message.js";
|
|
12
12
|
|
|
13
13
|
const SCHEMA = { ...messageDataSchema, threadMessages: threadMessageTable };
|
|
14
14
|
|
|
@@ -43,7 +43,7 @@ describe("DrizzleMessageRepository (sqlite)", () => {
|
|
|
43
43
|
|
|
44
44
|
before(async () => {
|
|
45
45
|
({ db, close } = await createSqliteTestDb(SCHEMA));
|
|
46
|
-
repo =
|
|
46
|
+
repo = createSqliteRepositories({ app: db, mail: db, search: db }).message;
|
|
47
47
|
});
|
|
48
48
|
|
|
49
49
|
after(async () => {
|
|
@@ -279,4 +279,58 @@ describe("DrizzleMessageRepository (sqlite)", () => {
|
|
|
279
279
|
await repo.delete(MESSAGE_ID);
|
|
280
280
|
await assert.rejects(() => repo.get(MESSAGE_ID));
|
|
281
281
|
});
|
|
282
|
+
|
|
283
|
+
test("listMissingSearchText names stored bodies without search text, after the cursor", async () => {
|
|
284
|
+
const id = (n: number): string => `00000000-0000-0000-3333-00000000000${n}`;
|
|
285
|
+
for (const n of [1, 2, 3, 4]) {
|
|
286
|
+
await repo.create({ ...BASE_INPUT, messageId: id(n), uid: 100 + n });
|
|
287
|
+
}
|
|
288
|
+
for (const n of [1, 2, 4]) {
|
|
289
|
+
await repo.update(id(n), { bodyStorageKey: `file:///bodies/${n}` });
|
|
290
|
+
}
|
|
291
|
+
await repo.setSearchText(id(2), "already indexed");
|
|
292
|
+
|
|
293
|
+
const ours = (rows: Array<{ messageId: string }>): string[] =>
|
|
294
|
+
rows
|
|
295
|
+
.map((row) => row.messageId)
|
|
296
|
+
.filter((messageId) => messageId.includes("-3333-"));
|
|
297
|
+
|
|
298
|
+
assert.deepEqual(ours(await repo.listMissingSearchText("", 100)), [
|
|
299
|
+
id(1),
|
|
300
|
+
id(4),
|
|
301
|
+
]);
|
|
302
|
+
assert.deepEqual(ours(await repo.listMissingSearchText(id(1), 100)), [
|
|
303
|
+
id(4),
|
|
304
|
+
]);
|
|
305
|
+
assert.deepEqual(
|
|
306
|
+
await repo
|
|
307
|
+
.listMissingSearchText(id(1), 100)
|
|
308
|
+
.then((rows) => rows.find((row) => row.messageId === id(4))),
|
|
309
|
+
{
|
|
310
|
+
messageId: id(4),
|
|
311
|
+
bodyStorageKey: "file:///bodies/4",
|
|
312
|
+
},
|
|
313
|
+
);
|
|
314
|
+
});
|
|
315
|
+
|
|
316
|
+
test("copySearchText gives a copy the source's search text", async () => {
|
|
317
|
+
const source = "00000000-0000-0000-4444-000000000001";
|
|
318
|
+
const copy = "00000000-0000-0000-4444-000000000002";
|
|
319
|
+
for (const [messageId, uid] of [
|
|
320
|
+
[source, 201],
|
|
321
|
+
[copy, 202],
|
|
322
|
+
] as const) {
|
|
323
|
+
await repo.create({ ...BASE_INPUT, messageId, uid });
|
|
324
|
+
await repo.update(messageId, { bodyStorageKey: "file:///bodies/shared" });
|
|
325
|
+
}
|
|
326
|
+
await repo.setSearchText(source, "the shared body");
|
|
327
|
+
|
|
328
|
+
await repo.copySearchText(source, copy);
|
|
329
|
+
|
|
330
|
+
const missing = await repo.listMissingSearchText("", 100);
|
|
331
|
+
assert.equal(
|
|
332
|
+
missing.some((row) => row.messageId === copy),
|
|
333
|
+
false,
|
|
334
|
+
);
|
|
335
|
+
});
|
|
282
336
|
});
|
|
@@ -4,9 +4,9 @@ import {
|
|
|
4
4
|
envelopeId as deriveEnvelopeId,
|
|
5
5
|
rootBodyPartId as deriveRootBodyPartId,
|
|
6
6
|
} from "../id.js";
|
|
7
|
-
import {
|
|
8
|
-
import { DrizzleEnvelopeRepository } from "./envelope.js";
|
|
9
|
-
import { DrizzleMessageRepository } from "./message.js";
|
|
7
|
+
import { createSqliteRepositories } from "../sqlite-repositories.js";
|
|
8
|
+
import type { DrizzleEnvelopeRepository } from "./envelope.js";
|
|
9
|
+
import type { DrizzleMessageRepository } from "./message.js";
|
|
10
10
|
import { createTestDb, type TestDb } from "./test-helpers.js";
|
|
11
11
|
|
|
12
12
|
describe("DrizzleMessageRepository", () => {
|
|
@@ -37,8 +37,9 @@ describe("DrizzleMessageRepository", () => {
|
|
|
37
37
|
|
|
38
38
|
before(async () => {
|
|
39
39
|
({ db, stop } = await createTestDb());
|
|
40
|
-
|
|
41
|
-
|
|
40
|
+
const repos = createSqliteRepositories({ app: db, mail: db, search: db });
|
|
41
|
+
messageRepo = repos.message;
|
|
42
|
+
envelopeRepo = repos.envelope;
|
|
42
43
|
});
|
|
43
44
|
|
|
44
45
|
after(async () => {
|
package/src/repos/message.ts
CHANGED
|
@@ -18,8 +18,8 @@ import {
|
|
|
18
18
|
or,
|
|
19
19
|
type SQL,
|
|
20
20
|
} from "drizzle-orm";
|
|
21
|
-
import {
|
|
22
|
-
import type {
|
|
21
|
+
import type { SqliteBatchItem, WriteBatch } from "../batch.js";
|
|
22
|
+
import type { RepositoryDb, SqliteStore } from "../db.js";
|
|
23
23
|
import {
|
|
24
24
|
CreateFailedConflictError,
|
|
25
25
|
isUniqueViolation,
|
|
@@ -37,9 +37,9 @@ import {
|
|
|
37
37
|
bodyStructureRefetchTable,
|
|
38
38
|
envelopeAddressTable,
|
|
39
39
|
envelopeTable,
|
|
40
|
-
type MessageDataSchema,
|
|
41
40
|
messageFlagTable,
|
|
42
41
|
messageReferenceTable,
|
|
42
|
+
messageSearchTextTable,
|
|
43
43
|
messageTable,
|
|
44
44
|
rawMessageStorageTable,
|
|
45
45
|
} from "../schema/message-data.js";
|
|
@@ -56,7 +56,7 @@ import {
|
|
|
56
56
|
toRawMessageStorageItem,
|
|
57
57
|
} from "./mappers.js";
|
|
58
58
|
|
|
59
|
-
type DB =
|
|
59
|
+
type DB = RepositoryDb<Record<string, unknown>>;
|
|
60
60
|
|
|
61
61
|
function toMessageItem(row: typeof messageTable.$inferSelect): MessageItem {
|
|
62
62
|
return {
|
|
@@ -112,7 +112,7 @@ function toMessageItem(row: typeof messageTable.$inferSelect): MessageItem {
|
|
|
112
112
|
};
|
|
113
113
|
}
|
|
114
114
|
|
|
115
|
-
export type SubtreeDb = Pick<
|
|
115
|
+
export type SubtreeDb = Pick<RepositoryDb<Record<string, unknown>>, "delete">;
|
|
116
116
|
|
|
117
117
|
/**
|
|
118
118
|
* The statements that delete a message and its whole per-message subtree by
|
|
@@ -154,6 +154,9 @@ export function deleteMessageSubtree(
|
|
|
154
154
|
.where(
|
|
155
155
|
and(inList(bodyStructureRefetchTable.messageId, messageIds), guard),
|
|
156
156
|
),
|
|
157
|
+
db
|
|
158
|
+
.delete(messageSearchTextTable)
|
|
159
|
+
.where(and(inList(messageSearchTextTable.messageId, messageIds), guard)),
|
|
157
160
|
db
|
|
158
161
|
.delete(envelopeTable)
|
|
159
162
|
.where(and(inList(envelopeTable.messageId, messageIds), guard)),
|
|
@@ -193,10 +196,16 @@ const placementTerms = (expected: PlacementPredicate): SQL[] => {
|
|
|
193
196
|
};
|
|
194
197
|
|
|
195
198
|
export class DrizzleMessageRepository implements IMessageRepository {
|
|
199
|
+
private readonly db: DB;
|
|
200
|
+
private readonly batch: WriteBatch;
|
|
201
|
+
|
|
196
202
|
constructor(
|
|
197
|
-
|
|
203
|
+
store: SqliteStore<Record<string, unknown>>,
|
|
198
204
|
private search: SearchIndex,
|
|
199
|
-
) {
|
|
205
|
+
) {
|
|
206
|
+
this.db = store.db;
|
|
207
|
+
this.batch = store.batch;
|
|
208
|
+
}
|
|
200
209
|
|
|
201
210
|
private rowOf(input: CreateMessageInput) {
|
|
202
211
|
const now = Date.now();
|
|
@@ -439,6 +448,68 @@ export class DrizzleMessageRepository implements IMessageRepository {
|
|
|
439
448
|
return this.get(messageId);
|
|
440
449
|
}
|
|
441
450
|
|
|
451
|
+
async setSearchText(messageId: string, text: string): Promise<void> {
|
|
452
|
+
const now = Date.now();
|
|
453
|
+
await this.db
|
|
454
|
+
.insert(messageSearchTextTable)
|
|
455
|
+
.values({ messageId, text, createdAt: now, updatedAt: now })
|
|
456
|
+
.onConflictDoUpdate({
|
|
457
|
+
target: messageSearchTextTable.messageId,
|
|
458
|
+
set: { text, updatedAt: now },
|
|
459
|
+
});
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
async copySearchText(
|
|
463
|
+
sourceMessageId: string,
|
|
464
|
+
targetMessageId: string,
|
|
465
|
+
): Promise<void> {
|
|
466
|
+
const [source] = await this.db
|
|
467
|
+
.select({ text: messageSearchTextTable.text })
|
|
468
|
+
.from(messageSearchTextTable)
|
|
469
|
+
.where(eq(messageSearchTextTable.messageId, sourceMessageId));
|
|
470
|
+
if (!source) return;
|
|
471
|
+
await this.setSearchText(targetMessageId, source.text);
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
async syncSearch(messageIds: string[]): Promise<void> {
|
|
475
|
+
if (messageIds.length === 0) return;
|
|
476
|
+
const listings = await this.db
|
|
477
|
+
.select({ threadMessageId: threadMessageTable.threadMessageId })
|
|
478
|
+
.from(threadMessageTable)
|
|
479
|
+
.where(inList(threadMessageTable.messageId, messageIds));
|
|
480
|
+
await this.search.sync(listings.map((row) => row.threadMessageId));
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
async listMissingSearchText(
|
|
484
|
+
after: string,
|
|
485
|
+
limit: number,
|
|
486
|
+
): ReturnType<IMessageRepository["listMissingSearchText"]> {
|
|
487
|
+
const rows = await this.db
|
|
488
|
+
.select({
|
|
489
|
+
messageId: messageTable.messageId,
|
|
490
|
+
bodyStorageKey: messageTable.bodyStorageKey,
|
|
491
|
+
})
|
|
492
|
+
.from(messageTable)
|
|
493
|
+
.leftJoin(
|
|
494
|
+
messageSearchTextTable,
|
|
495
|
+
eq(messageSearchTextTable.messageId, messageTable.messageId),
|
|
496
|
+
)
|
|
497
|
+
.where(
|
|
498
|
+
and(
|
|
499
|
+
gt(messageTable.messageId, after),
|
|
500
|
+
isNotNull(messageTable.bodyStorageKey),
|
|
501
|
+
isNull(messageSearchTextTable.messageId),
|
|
502
|
+
),
|
|
503
|
+
)
|
|
504
|
+
.orderBy(asc(messageTable.messageId))
|
|
505
|
+
.limit(limit);
|
|
506
|
+
return rows.flatMap((row) =>
|
|
507
|
+
row.bodyStorageKey
|
|
508
|
+
? [{ messageId: row.messageId, bodyStorageKey: row.bodyStorageKey }]
|
|
509
|
+
: [],
|
|
510
|
+
);
|
|
511
|
+
}
|
|
512
|
+
|
|
442
513
|
async delete(messageId: string): Promise<void> {
|
|
443
514
|
await this.deleteMany([messageId]);
|
|
444
515
|
}
|
|
@@ -462,7 +533,7 @@ export class DrizzleMessageRepository implements IMessageRepository {
|
|
|
462
533
|
.select({ threadMessageId: threadMessageTable.threadMessageId })
|
|
463
534
|
.from(threadMessageTable)
|
|
464
535
|
.where(eq(threadMessageTable.messageId, messageId));
|
|
465
|
-
const results = await
|
|
536
|
+
const results = await this.batch([
|
|
466
537
|
this.db
|
|
467
538
|
.delete(threadMessageTable)
|
|
468
539
|
.where(and(eq(threadMessageTable.messageId, messageId), placed)),
|
|
@@ -475,7 +546,7 @@ export class DrizzleMessageRepository implements IMessageRepository {
|
|
|
475
546
|
|
|
476
547
|
async deleteMany(messageIds: string[]): Promise<void> {
|
|
477
548
|
if (messageIds.length === 0) return;
|
|
478
|
-
await
|
|
549
|
+
await this.batch(deleteMessageSubtree(this.db, messageIds));
|
|
479
550
|
}
|
|
480
551
|
|
|
481
552
|
async listByMailbox(
|
|
@@ -623,7 +694,7 @@ export class DrizzleMessageRepository implements IMessageRepository {
|
|
|
623
694
|
.select({ threadMessageId: threadMessageTable.threadMessageId })
|
|
624
695
|
.from(threadMessageTable)
|
|
625
696
|
.where(eq(threadMessageTable.messageId, messageId));
|
|
626
|
-
const [moved] = await
|
|
697
|
+
const [moved] = await this.batch([
|
|
627
698
|
this.db
|
|
628
699
|
.update(messageTable)
|
|
629
700
|
.set({
|
package/src/repos/quarantine.ts
CHANGED
|
@@ -6,10 +6,10 @@ import type {
|
|
|
6
6
|
} from "@remit/data-ports";
|
|
7
7
|
import { deriveQuarantineId } from "@remit/data-ports/id";
|
|
8
8
|
import { desc, eq } from "drizzle-orm";
|
|
9
|
-
import type {
|
|
9
|
+
import type { RepositoryDb } from "../db.js";
|
|
10
10
|
import { quarantineTable } from "../schema/quarantine.js";
|
|
11
11
|
|
|
12
|
-
type DB =
|
|
12
|
+
type DB = RepositoryDb<Record<string, unknown>>;
|
|
13
13
|
|
|
14
14
|
function rowToItem(row: typeof quarantineTable.$inferSelect): QuarantineItem {
|
|
15
15
|
return {
|
|
@@ -6,11 +6,11 @@ import type {
|
|
|
6
6
|
} from "@remit/data-ports";
|
|
7
7
|
import { deriveSyncWarningId } from "@remit/data-ports/id";
|
|
8
8
|
import { and, count, desc, eq, gte, sql } from "drizzle-orm";
|
|
9
|
-
import type {
|
|
9
|
+
import type { RepositoryDb } from "../db.js";
|
|
10
10
|
import { syncWarningTable } from "../schema/sync-warning.js";
|
|
11
11
|
import { inList } from "./in-list.js";
|
|
12
12
|
|
|
13
|
-
type DB =
|
|
13
|
+
type DB = RepositoryDb<Record<string, unknown>>;
|
|
14
14
|
|
|
15
15
|
export class SyncWarningRepo implements ISyncWarningRepository {
|
|
16
16
|
constructor(private db: DB) {}
|
|
@@ -11,15 +11,11 @@ import assert from "node:assert/strict";
|
|
|
11
11
|
import { after, before, describe, test } from "node:test";
|
|
12
12
|
import type { CreateThreadMessageInput } from "@remit/data-ports";
|
|
13
13
|
import { addressTable } from "../schema/i4-address.js";
|
|
14
|
+
import { messageSearchTextTable } from "../schema/message-data.js";
|
|
14
15
|
import { threadMessageTable } from "../schema/thread-message.js";
|
|
15
|
-
import {
|
|
16
|
+
import { createSqliteRepositories } from "../sqlite-repositories.js";
|
|
16
17
|
import { createSqliteTestDb } from "../test-db-sqlite.js";
|
|
17
|
-
import {
|
|
18
|
-
import { DrizzleThreadMessageRepository } from "./thread-message.js";
|
|
19
|
-
|
|
20
|
-
// The index the mail baseline creates for this predicate, read from the one
|
|
21
|
-
// committed source so the plan assertion below is about the real object.
|
|
22
|
-
const mutedIndexDdl = shippedIndexDdl("address_by_normalized_email");
|
|
18
|
+
import type { DrizzleThreadMessageRepository } from "./thread-message.js";
|
|
23
19
|
|
|
24
20
|
const ACCOUNT = "acct-seam";
|
|
25
21
|
const MAILBOX = "mbx-seam";
|
|
@@ -57,11 +53,18 @@ describe("thread-message body and muted-sender predicates (sqlite)", () => {
|
|
|
57
53
|
|
|
58
54
|
before(async () => {
|
|
59
55
|
({ db, sqlite, close } = await createSqliteTestDb(
|
|
60
|
-
{
|
|
56
|
+
{
|
|
57
|
+
threadMessage: threadMessageTable,
|
|
58
|
+
address: addressTable,
|
|
59
|
+
messageSearchText: messageSearchTextTable,
|
|
60
|
+
},
|
|
61
61
|
{ searchIndex: true },
|
|
62
62
|
));
|
|
63
|
-
|
|
64
|
-
|
|
63
|
+
repo = createSqliteRepositories({
|
|
64
|
+
app: db,
|
|
65
|
+
mail: db,
|
|
66
|
+
search: db,
|
|
67
|
+
}).threadMessage;
|
|
65
68
|
});
|
|
66
69
|
|
|
67
70
|
after(async () => {
|
|
@@ -82,15 +85,20 @@ describe("thread-message body and muted-sender predicates (sqlite)", () => {
|
|
|
82
85
|
}),
|
|
83
86
|
);
|
|
84
87
|
}
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
88
|
+
const parcel = makeInput({
|
|
89
|
+
subject: "unrelated note",
|
|
90
|
+
fromEmail: "noise@example.com",
|
|
91
|
+
fromName: "Noise",
|
|
92
|
+
snippet: "Your parcel was left with the concierge",
|
|
93
|
+
sentDate: BASE_DATE,
|
|
94
|
+
});
|
|
95
|
+
await db.insert(messageSearchTextTable).values({
|
|
96
|
+
messageId: parcel.messageId,
|
|
97
|
+
text: "Your parcel was left with the concierge",
|
|
98
|
+
createdAt: 0,
|
|
99
|
+
updatedAt: 0,
|
|
100
|
+
});
|
|
101
|
+
await repo.create(parcel);
|
|
94
102
|
});
|
|
95
103
|
|
|
96
104
|
test("the term reaches the body text, not just subject and From", async () => {
|
|
@@ -119,16 +127,16 @@ describe("thread-message body and muted-sender predicates (sqlite)", () => {
|
|
|
119
127
|
assert.equal(result.items.length, 1);
|
|
120
128
|
});
|
|
121
129
|
|
|
122
|
-
test("a
|
|
130
|
+
test("a short word prefix reads the body too", async () => {
|
|
123
131
|
const result = await repo.searchByDate(
|
|
124
132
|
ACCOUNT,
|
|
125
|
-
{ query: "
|
|
133
|
+
{ query: "co" },
|
|
126
134
|
{ mailboxIds: SCOPE, limit: 50 },
|
|
127
135
|
);
|
|
128
136
|
|
|
129
137
|
assert.ok(
|
|
130
138
|
result.items.some((item) => item.snippet?.includes("concierge")),
|
|
131
|
-
"the
|
|
139
|
+
"the prefix matched the body preview",
|
|
132
140
|
);
|
|
133
141
|
});
|
|
134
142
|
|
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
import assert from "node:assert/strict";
|
|
2
2
|
import { after, before, describe, test } from "node:test";
|
|
3
3
|
import type { CreateThreadMessageInput } from "@remit/data-ports";
|
|
4
|
+
import { messageSearchTextTable } from "../schema/message-data.js";
|
|
4
5
|
import { threadMessageTable } from "../schema/thread-message.js";
|
|
5
|
-
import {
|
|
6
|
+
import { createSqliteRepositories } from "../sqlite-repositories.js";
|
|
6
7
|
import { createSqliteTestDb } from "../test-db-sqlite.js";
|
|
7
8
|
import {
|
|
8
|
-
DrizzleThreadMessageRepository,
|
|
9
|
+
type DrizzleThreadMessageRepository,
|
|
9
10
|
THREAD_SEARCH_MAX_LIMIT,
|
|
10
11
|
} from "./thread-message.js";
|
|
11
12
|
|
|
@@ -51,14 +52,18 @@ describe("DrizzleThreadMessageRepository.listByFieldTerms (sqlite, #459)", () =>
|
|
|
51
52
|
|
|
52
53
|
before(async () => {
|
|
53
54
|
const created = await createSqliteTestDb(
|
|
54
|
-
{
|
|
55
|
+
{
|
|
56
|
+
threadMessage: threadMessageTable,
|
|
57
|
+
messageSearchText: messageSearchTextTable,
|
|
58
|
+
},
|
|
55
59
|
{ searchIndex: true },
|
|
56
60
|
);
|
|
57
61
|
close = created.close;
|
|
58
|
-
repo =
|
|
59
|
-
created.db,
|
|
60
|
-
|
|
61
|
-
|
|
62
|
+
repo = createSqliteRepositories({
|
|
63
|
+
app: created.db,
|
|
64
|
+
mail: created.db,
|
|
65
|
+
search: created.db,
|
|
66
|
+
}).threadMessage;
|
|
62
67
|
|
|
63
68
|
await repo.create(
|
|
64
69
|
makeInput({
|
|
@@ -159,10 +164,10 @@ describe("DrizzleThreadMessageRepository.listByFieldTerms (sqlite, #459)", () =>
|
|
|
159
164
|
]);
|
|
160
165
|
});
|
|
161
166
|
|
|
162
|
-
//
|
|
163
|
-
//
|
|
164
|
-
//
|
|
165
|
-
//
|
|
167
|
+
// A field term is the folded LIKE, and sqlite's lower() folds ASCII only —
|
|
168
|
+
// `é` never matches a stored `CAFÉ`. Applying it anyway would drop a row
|
|
169
|
+
// the caller's own matcher accepts, which is #459 again for that clause
|
|
170
|
+
// shape, so a non-ASCII term narrows nothing.
|
|
166
171
|
test("drops a short accented term rather than missing the row it should match", async () => {
|
|
167
172
|
const result = await repo.listByFieldTerms(
|
|
168
173
|
ACCOUNT,
|
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import assert from "node:assert/strict";
|
|
2
2
|
import { after, before, describe, test } from "node:test";
|
|
3
3
|
import type { CreateThreadMessageInput } from "@remit/data-ports";
|
|
4
|
+
import { messageSearchTextTable } from "../schema/message-data.js";
|
|
4
5
|
import { threadMessageTable } from "../schema/thread-message.js";
|
|
5
|
-
import {
|
|
6
|
+
import { createSqliteRepositories } from "../sqlite-repositories.js";
|
|
6
7
|
import { createSqliteTestDb } from "../test-db-sqlite.js";
|
|
7
8
|
import {
|
|
8
9
|
BASE_DATE,
|
|
@@ -12,14 +13,14 @@ import {
|
|
|
12
13
|
totalRows,
|
|
13
14
|
} from "./category-fixture.js";
|
|
14
15
|
import {
|
|
15
|
-
DrizzleThreadMessageRepository,
|
|
16
|
+
type DrizzleThreadMessageRepository,
|
|
16
17
|
THREAD_SEARCH_MAX_LIMIT,
|
|
17
18
|
} from "./thread-message.js";
|
|
18
19
|
|
|
19
20
|
// The thread-message repo on sqlite (RFC 036 D1): CRUD, keyset pagination, and
|
|
20
|
-
// text search — the FTS5
|
|
21
|
-
//
|
|
22
|
-
// and case-insensitive
|
|
21
|
+
// text search — the FTS5 unicode61 index (D4), matching words by prefix, and
|
|
22
|
+
// the folded-LIKE scan for terms with no word. The search contract is accent-
|
|
23
|
+
// and case-insensitive word-prefix match over subject and sender.
|
|
23
24
|
|
|
24
25
|
const ACCOUNT = "acct-thread";
|
|
25
26
|
const MAILBOX = "mbx-thread";
|
|
@@ -55,10 +56,15 @@ describe("DrizzleThreadMessageRepository (sqlite)", () => {
|
|
|
55
56
|
({ db, sqlite, close } = await createSqliteTestDb(
|
|
56
57
|
{
|
|
57
58
|
threadMessage: threadMessageTable,
|
|
59
|
+
messageSearchText: messageSearchTextTable,
|
|
58
60
|
},
|
|
59
61
|
{ searchIndex: true },
|
|
60
62
|
));
|
|
61
|
-
repo =
|
|
63
|
+
repo = createSqliteRepositories({
|
|
64
|
+
app: db,
|
|
65
|
+
mail: db,
|
|
66
|
+
search: db,
|
|
67
|
+
}).threadMessage;
|
|
62
68
|
});
|
|
63
69
|
|
|
64
70
|
after(async () => {
|
|
@@ -111,7 +117,7 @@ describe("DrizzleThreadMessageRepository (sqlite)", () => {
|
|
|
111
117
|
assert.ok(byName.items.some((i) => i.fromName === "Alice Kramer"));
|
|
112
118
|
});
|
|
113
119
|
|
|
114
|
-
test("FTS
|
|
120
|
+
test("FTS search folds diacritics both ways", async () => {
|
|
115
121
|
await repo.create(makeInput({ subject: "Réunion budget" }));
|
|
116
122
|
const folded = await repo.searchByMailbox(
|
|
117
123
|
ACCOUNT,
|
|
@@ -1,13 +1,17 @@
|
|
|
1
1
|
import assert from "node:assert";
|
|
2
2
|
import { after, before, describe, test } from "node:test";
|
|
3
|
-
import type {
|
|
3
|
+
import type {
|
|
4
|
+
CreateThreadMessageInput,
|
|
5
|
+
IMessageRepository,
|
|
6
|
+
} from "@remit/data-ports";
|
|
4
7
|
import shortUuid from "short-uuid";
|
|
8
|
+
import { messageSearchTextTable } from "../schema/message-data.js";
|
|
5
9
|
import { threadMessageTable } from "../schema/thread-message.js";
|
|
6
|
-
import {
|
|
10
|
+
import { createSqliteRepositories } from "../sqlite-repositories.js";
|
|
7
11
|
import { createSqliteTestDb } from "../test-db-sqlite.js";
|
|
8
12
|
import {
|
|
9
13
|
clampThreadSearchLimit,
|
|
10
|
-
DrizzleThreadMessageRepository,
|
|
14
|
+
type DrizzleThreadMessageRepository,
|
|
11
15
|
deriveThreadMessageId,
|
|
12
16
|
THREAD_SEARCH_MAX_LIMIT,
|
|
13
17
|
} from "./thread-message.js";
|
|
@@ -40,11 +44,14 @@ function makeInput(
|
|
|
40
44
|
};
|
|
41
45
|
}
|
|
42
46
|
|
|
43
|
-
// A real better-sqlite3 database with the FTS5
|
|
47
|
+
// A real better-sqlite3 database with the FTS5 objects the migrator
|
|
44
48
|
// installs, so the search predicates run their shipped path.
|
|
45
49
|
const setupDb = () =>
|
|
46
50
|
createSqliteTestDb(
|
|
47
|
-
{
|
|
51
|
+
{
|
|
52
|
+
threadMessage: threadMessageTable,
|
|
53
|
+
messageSearchText: messageSearchTextTable,
|
|
54
|
+
},
|
|
48
55
|
{ searchIndex: true },
|
|
49
56
|
);
|
|
50
57
|
|
|
@@ -87,10 +94,11 @@ describe("DrizzleThreadMessageRepository.searchByMailboxWindow / countByMailbox"
|
|
|
87
94
|
before(async () => {
|
|
88
95
|
const harness = await setupDb();
|
|
89
96
|
close = harness.close;
|
|
90
|
-
repo =
|
|
91
|
-
harness.db,
|
|
92
|
-
|
|
93
|
-
|
|
97
|
+
repo = createSqliteRepositories({
|
|
98
|
+
app: harness.db,
|
|
99
|
+
mail: harness.db,
|
|
100
|
+
search: harness.db,
|
|
101
|
+
}).threadMessage;
|
|
94
102
|
});
|
|
95
103
|
|
|
96
104
|
after(async () => {
|
|
@@ -193,7 +201,7 @@ describe("DrizzleThreadMessageRepository.searchByMailboxWindow / countByMailbox"
|
|
|
193
201
|
});
|
|
194
202
|
|
|
195
203
|
// ── Scenario 4 ────────────────────────────────────────────────────────────
|
|
196
|
-
// Matching runs over the WHOLE mailbox via the
|
|
204
|
+
// Matching runs over the WHOLE mailbox via the full-text index, so an old
|
|
197
205
|
// match well behind the recent rows is still found and paged (the DynamoDB
|
|
198
206
|
// recent-window bound does not apply here — the improvement over #443 on the
|
|
199
207
|
// DDB path).
|
|
@@ -385,16 +393,20 @@ describe("DrizzleThreadMessageRepository.searchByMailboxWindow / countByMailbox"
|
|
|
385
393
|
|
|
386
394
|
describe("DrizzleThreadMessageRepository.searchByDate", () => {
|
|
387
395
|
let repo: DrizzleThreadMessageRepository;
|
|
396
|
+
let messages: IMessageRepository;
|
|
388
397
|
let close: () => Promise<void>;
|
|
389
398
|
const cleanup: Array<() => Promise<void>> = [];
|
|
390
399
|
|
|
391
400
|
before(async () => {
|
|
392
401
|
const harness = await setupDb();
|
|
393
402
|
close = harness.close;
|
|
394
|
-
|
|
395
|
-
harness.db,
|
|
396
|
-
|
|
397
|
-
|
|
403
|
+
const repositories = createSqliteRepositories({
|
|
404
|
+
app: harness.db,
|
|
405
|
+
mail: harness.db,
|
|
406
|
+
search: harness.db,
|
|
407
|
+
});
|
|
408
|
+
repo = repositories.threadMessage;
|
|
409
|
+
messages = repositories.message;
|
|
398
410
|
});
|
|
399
411
|
|
|
400
412
|
after(async () => {
|
|
@@ -410,9 +422,11 @@ describe("DrizzleThreadMessageRepository.searchByDate", () => {
|
|
|
410
422
|
rows: Array<Partial<CreateThreadMessageInput>>,
|
|
411
423
|
): Promise<void> {
|
|
412
424
|
for (const overrides of rows) {
|
|
413
|
-
const
|
|
414
|
-
|
|
415
|
-
|
|
425
|
+
const input = makeInput(accountConfigId, mailboxId, overrides);
|
|
426
|
+
if (overrides.snippet) {
|
|
427
|
+
await messages.setSearchText(input.messageId, overrides.snippet);
|
|
428
|
+
}
|
|
429
|
+
const created = await repo.create(input);
|
|
416
430
|
cleanup.push(() => repo.delete(accountConfigId, created.threadMessageId));
|
|
417
431
|
}
|
|
418
432
|
}
|
|
@@ -921,10 +935,11 @@ describe("DrizzleThreadMessageRepository — native text search", () => {
|
|
|
921
935
|
before(async () => {
|
|
922
936
|
const harness = await setupDb();
|
|
923
937
|
close = harness.close;
|
|
924
|
-
repo =
|
|
925
|
-
harness.db,
|
|
926
|
-
|
|
927
|
-
|
|
938
|
+
repo = createSqliteRepositories({
|
|
939
|
+
app: harness.db,
|
|
940
|
+
mail: harness.db,
|
|
941
|
+
search: harness.db,
|
|
942
|
+
}).threadMessage;
|
|
928
943
|
});
|
|
929
944
|
|
|
930
945
|
after(async () => {
|
|
@@ -1050,7 +1065,7 @@ describe("DrizzleThreadMessageRepository — native text search", () => {
|
|
|
1050
1065
|
assert.deepEqual(subjectsOf(res), ["shared token here"]);
|
|
1051
1066
|
});
|
|
1052
1067
|
|
|
1053
|
-
test("LIKE metacharacters in
|
|
1068
|
+
test("LIKE metacharacters in a query with no word match literally", async () => {
|
|
1054
1069
|
const acct = uuid();
|
|
1055
1070
|
const mbx = uuid();
|
|
1056
1071
|
await seed(acct, mbx, [
|
|
@@ -1058,7 +1073,7 @@ describe("DrizzleThreadMessageRepository — native text search", () => {
|
|
|
1058
1073
|
{ subject: "50 percent off" },
|
|
1059
1074
|
]);
|
|
1060
1075
|
|
|
1061
|
-
const res = await repo.searchByMailboxWindow(acct, mbx, { query: "
|
|
1076
|
+
const res = await repo.searchByMailboxWindow(acct, mbx, { query: "%" });
|
|
1062
1077
|
assert.deepEqual(subjectsOf(res), ["50% off today"]);
|
|
1063
1078
|
});
|
|
1064
1079
|
|
|
@@ -1096,10 +1111,11 @@ describe("DrizzleThreadMessageRepository — core CRUD", () => {
|
|
|
1096
1111
|
before(async () => {
|
|
1097
1112
|
const harness = await setupDb();
|
|
1098
1113
|
close = harness.close;
|
|
1099
|
-
repo =
|
|
1100
|
-
harness.db,
|
|
1101
|
-
|
|
1102
|
-
|
|
1114
|
+
repo = createSqliteRepositories({
|
|
1115
|
+
app: harness.db,
|
|
1116
|
+
mail: harness.db,
|
|
1117
|
+
search: harness.db,
|
|
1118
|
+
}).threadMessage;
|
|
1103
1119
|
});
|
|
1104
1120
|
|
|
1105
1121
|
after(async () => {
|