@remit/drizzle-service 0.0.98 → 0.0.100
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/batch.sqlite.test.ts +78 -0
- package/src/batch.ts +45 -0
- package/src/databases.ts +70 -0
- package/src/db.ts +1 -4
- package/src/generated-enum.test.ts +8 -0
- package/src/generated-id.test.ts +19 -0
- package/src/id.test.ts +0 -8
- package/src/id.ts +0 -6
- package/src/index.ts +16 -6
- package/src/repair/address-display-name.sqlite.test.ts +3 -5
- package/src/repair/address-display-name.ts +8 -3
- package/src/repair/junk-only-address.sqlite.test.ts +8 -36
- package/src/repair/stranded-sent-outbox.sqlite.test.ts +2 -9
- package/src/repos/account.conformance.sqlite.test.ts +1 -1
- package/src/repos/calendar-default.sqlite.test.ts +6 -6
- package/src/repos/calendar-put.sqlite.test.ts +40 -39
- package/src/repos/calendar-store.ts +273 -0
- package/src/repos/cascade-delete.sqlite.test.ts +7 -9
- package/src/repos/cascade-delete.test.ts +6 -13
- package/src/repos/cascade-delete.ts +127 -97
- package/src/repos/envelope.test.ts +1 -17
- package/src/repos/envelope.ts +83 -67
- package/src/repos/filter.test.ts +22 -2
- package/src/repos/filter.ts +1 -5
- package/src/repos/i4-account-config.test.ts +2 -2
- package/src/repos/i4-account-config.ts +5 -2
- package/src/repos/i4-account.test.ts +6 -2
- package/src/repos/i4-account.ts +12 -56
- package/src/repos/i4-address-junk-move.sqlite.test.ts +1 -1
- package/src/repos/i4-address-merge-flags.sqlite.test.ts +1 -2
- package/src/repos/i4-address.ts +58 -90
- package/src/repos/i4-mailbox-special-use.sqlite.test.ts +6 -2
- package/src/repos/i4-mailbox-special-use.ts +9 -6
- package/src/repos/i4-mailbox.sqlite.test.ts +12 -14
- package/src/repos/i4-mailbox.test.ts +17 -62
- package/src/repos/i4-mailbox.ts +121 -94
- package/src/repos/i4-organize-job-request.ts +0 -4
- package/src/repos/i4-outbox-attachment.ts +62 -33
- package/src/repos/mappers.ts +0 -16
- package/src/repos/message.sqlite.test.ts +62 -102
- package/src/repos/message.test.ts +6 -167
- package/src/repos/message.ts +227 -218
- package/src/repos/sync-warning.ts +76 -0
- package/src/repos/thread-message-body-muted.sqlite.test.ts +5 -10
- package/src/repos/thread-message-field-terms.sqlite.test.ts +5 -1
- package/src/repos/thread-message.sqlite.test.ts +2 -1
- package/src/repos/thread-message.test.ts +17 -4
- package/src/repos/thread-message.ts +502 -285
- package/src/repos/thread-search-predicates.ts +24 -26
- package/src/schema/message-data.ts +4 -8
- package/src/schema/sync-warning.ts +3 -0
- package/src/schema-sets/app.ts +17 -0
- package/src/schema-sets/mail.ts +32 -0
- package/src/schema-sets/search.ts +5 -0
- package/src/schema-version-accounting.sqlite.test.ts +56 -43
- package/src/schema.ts +3 -4
- package/src/search/thread-message-search.sqlite.test.ts +131 -0
- package/src/search/thread-message-search.ts +271 -0
- package/src/sqlite-client.ts +5 -7
- package/src/sqlite-repositories.ts +75 -0
- package/src/test-db-sqlite.ts +5 -13
- package/src/test-shipped-sqlite-schema.ts +46 -45
- package/src/test-sqlite.ts +35 -11
- package/src/vps-migrations-drift.sqlite.test.ts +81 -15
- package/src/account-granted-scopes-migration.sqlite.test.ts +0 -66
- package/src/account-synced-services-migration.sqlite.test.ts +0 -56
- package/src/mailbox-path-dedupe.sqlite.test.ts +0 -462
- package/src/mailbox-sync-status-backfill.sqlite.test.ts +0 -118
- package/src/outbox-created-cleanup.sqlite.test.ts +0 -83
- package/src/repair/duplicate-mailbox-mail.ts +0 -98
- package/src/repair/thread-message-category-contract.ts +0 -505
- package/src/repair/thread-message-category.sqlite.test.ts +0 -100
- package/src/repair/thread-message-category.ts +0 -415
- package/src/repos/calendar-migrations.sqlite.test.ts +0 -345
- package/src/repos/calendar-unit-of-work.ts +0 -31
- package/src/repos/config-import-atomicity.sqlite.test.ts +0 -255
- package/src/repos/filter-anchor-transaction.test.ts +0 -136
- package/src/repos/filter-anchor-transaction.ts +0 -45
- package/src/repos/filter-anchor.test.ts +0 -131
- package/src/repos/filter-anchor.ts +0 -105
- package/src/repos/i4-mailbox-special-use.structure.test.ts +0 -27
- package/src/repos/message-placement-backfill.sqlite.test.ts +0 -190
- package/src/repos/search-index-shape.test.ts +0 -82
- package/src/repos/search-index-shape.ts +0 -56
- package/src/repos/serialized-writes.sqlite.test.ts +0 -199
- package/src/repos/unit-of-work.sqlite.test.ts +0 -138
- package/src/repos/unit-of-work.test.ts +0 -109
- package/src/repos/unit-of-work.ts +0 -31
- package/src/schema/outbox.ts +0 -46
- package/src/schema-full-sqlite.ts +0 -13
- package/src/tx.sqlite.test.ts +0 -83
- package/src/tx.ts +0 -233
|
@@ -0,0 +1,271 @@
|
|
|
1
|
+
import {
|
|
2
|
+
and,
|
|
3
|
+
asc,
|
|
4
|
+
count,
|
|
5
|
+
desc,
|
|
6
|
+
eq,
|
|
7
|
+
gt,
|
|
8
|
+
inArray,
|
|
9
|
+
lt,
|
|
10
|
+
or,
|
|
11
|
+
type SQL,
|
|
12
|
+
sql,
|
|
13
|
+
} from "drizzle-orm";
|
|
14
|
+
import { integer, sqliteTable, text } from "drizzle-orm/sqlite-core";
|
|
15
|
+
import { runBatch, type SqliteBatchItem } from "../batch.js";
|
|
16
|
+
import type { Db } from "../db.js";
|
|
17
|
+
import { threadMessageTable } from "../schema/thread-message.js";
|
|
18
|
+
|
|
19
|
+
// The FTS5 trigram table the search migration set creates
|
|
20
|
+
// (deploy/vps/migrations-sqlite/search). drizzle-kit cannot express a virtual
|
|
21
|
+
// table, so its DDL is that committed migration and this declaration only lets
|
|
22
|
+
// drizzle build statements against it; no drizzle-kit config reads it.
|
|
23
|
+
export const threadMessageSearchTable = sqliteTable("thread_message_fts", {
|
|
24
|
+
rowid: integer("rowid").notNull(),
|
|
25
|
+
threadMessageId: text("thread_message_id").notNull(),
|
|
26
|
+
accountConfigId: text("account_config_id").notNull(),
|
|
27
|
+
mailboxId: text("mailbox_id").notNull(),
|
|
28
|
+
sentDate: integer("sent_date").notNull(),
|
|
29
|
+
listId: text("list_id").notNull(),
|
|
30
|
+
subject: text("subject").notNull(),
|
|
31
|
+
sender: text("sender").notNull(),
|
|
32
|
+
body: text("body").notNull(),
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
type AnyDb = Db<Record<string, unknown>>;
|
|
36
|
+
|
|
37
|
+
type SearchDocument = typeof threadMessageSearchTable.$inferInsert;
|
|
38
|
+
|
|
39
|
+
// D1 refuses a statement with more than 100 bound parameters. A document is
|
|
40
|
+
// nine parameters, so ten documents per insert stay under it.
|
|
41
|
+
const DOCUMENTS_PER_STATEMENT = 10;
|
|
42
|
+
const DOCUMENTS_PER_REBUILD_PAGE = DOCUMENTS_PER_STATEMENT * 30;
|
|
43
|
+
|
|
44
|
+
// The document's rowid, derived from its thread message id so a write or a
|
|
45
|
+
// removal finds the document without a lookup. 53 bits, the integers a number
|
|
46
|
+
// carries exactly on every driver.
|
|
47
|
+
export const searchRowid = (threadMessageId: string): number => {
|
|
48
|
+
let h1 = 0xdeadbeef;
|
|
49
|
+
let h2 = 0x41c6ce57;
|
|
50
|
+
for (let index = 0; index < threadMessageId.length; index++) {
|
|
51
|
+
const code = threadMessageId.charCodeAt(index);
|
|
52
|
+
h1 = Math.imul(h1 ^ code, 2654435761);
|
|
53
|
+
h2 = Math.imul(h2 ^ code, 1597334677);
|
|
54
|
+
}
|
|
55
|
+
h1 = Math.imul(h1 ^ (h1 >>> 16), 2246822507);
|
|
56
|
+
h1 ^= Math.imul(h2 ^ (h2 >>> 13), 3266489909);
|
|
57
|
+
h2 = Math.imul(h2 ^ (h2 >>> 16), 2246822507);
|
|
58
|
+
h2 ^= Math.imul(h1 ^ (h1 >>> 13), 3266489909);
|
|
59
|
+
return 4294967296 * (2097151 & h2) + (h1 >>> 0);
|
|
60
|
+
};
|
|
61
|
+
|
|
62
|
+
const indexedColumns = {
|
|
63
|
+
threadMessageId: threadMessageTable.threadMessageId,
|
|
64
|
+
accountConfigId: threadMessageTable.accountConfigId,
|
|
65
|
+
mailboxId: threadMessageTable.mailboxId,
|
|
66
|
+
sentDate: threadMessageTable.sentDate,
|
|
67
|
+
listId: threadMessageTable.listId,
|
|
68
|
+
subject: threadMessageTable.subject,
|
|
69
|
+
fromName: threadMessageTable.fromName,
|
|
70
|
+
fromEmail: threadMessageTable.fromEmail,
|
|
71
|
+
snippet: threadMessageTable.snippet,
|
|
72
|
+
};
|
|
73
|
+
|
|
74
|
+
interface IndexedRow {
|
|
75
|
+
threadMessageId: string;
|
|
76
|
+
accountConfigId: string;
|
|
77
|
+
mailboxId: string;
|
|
78
|
+
sentDate: number;
|
|
79
|
+
listId: string | null;
|
|
80
|
+
subject: string | null;
|
|
81
|
+
fromName: string | null;
|
|
82
|
+
fromEmail: string | null;
|
|
83
|
+
snippet: string | null;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
const toDocument = (row: IndexedRow): SearchDocument => ({
|
|
87
|
+
rowid: searchRowid(row.threadMessageId),
|
|
88
|
+
threadMessageId: row.threadMessageId,
|
|
89
|
+
accountConfigId: row.accountConfigId,
|
|
90
|
+
mailboxId: row.mailboxId,
|
|
91
|
+
sentDate: row.sentDate,
|
|
92
|
+
listId: row.listId ?? "",
|
|
93
|
+
subject: row.subject ?? "",
|
|
94
|
+
sender: `${row.fromName ?? ""} ${row.fromEmail ?? ""}`,
|
|
95
|
+
body: row.snippet ?? "",
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
const chunk = <T>(items: readonly T[], size: number): T[][] => {
|
|
99
|
+
const chunks: T[][] = [];
|
|
100
|
+
for (let start = 0; start < items.length; start += size) {
|
|
101
|
+
chunks.push(items.slice(start, start + size));
|
|
102
|
+
}
|
|
103
|
+
return chunks;
|
|
104
|
+
};
|
|
105
|
+
|
|
106
|
+
/** Where a page of candidates resumes: the last one the previous page held. */
|
|
107
|
+
export interface SearchPosition {
|
|
108
|
+
rank: number;
|
|
109
|
+
sentDate: number;
|
|
110
|
+
threadMessageId: string;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
export type SearchCandidate = SearchPosition;
|
|
114
|
+
|
|
115
|
+
export interface CandidateQuery {
|
|
116
|
+
accountConfigId: string;
|
|
117
|
+
/** The mailboxes in scope; every mailbox of the config when absent. */
|
|
118
|
+
mailboxIds?: readonly string[];
|
|
119
|
+
/** The text condition, over the columns of {@link threadMessageSearchTable}. */
|
|
120
|
+
where: SQL;
|
|
121
|
+
/** Ranks first, highest first, when given. */
|
|
122
|
+
rank?: SQL<number>;
|
|
123
|
+
order: "asc" | "desc";
|
|
124
|
+
after: SearchPosition | null;
|
|
125
|
+
limit: number;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* The thread-message search index, kept in its own database.
|
|
130
|
+
*
|
|
131
|
+
* It is written after the mail write it follows has committed, and never in
|
|
132
|
+
* the same batch: a write reads the rows it names back from the mail database
|
|
133
|
+
* and makes the index match them, so repeating one is harmless and a missed one
|
|
134
|
+
* is repaired by the next write to that row or by {@link rebuild}.
|
|
135
|
+
*
|
|
136
|
+
* A search is answered here — matched, scoped to its mailboxes, ordered and
|
|
137
|
+
* cut to a page — and the mail database is then asked for that page's rows
|
|
138
|
+
* only, so no statement carries more ids than a page holds.
|
|
139
|
+
*/
|
|
140
|
+
export class SearchIndex {
|
|
141
|
+
constructor(
|
|
142
|
+
private readonly mail: AnyDb,
|
|
143
|
+
private readonly search: AnyDb,
|
|
144
|
+
) {}
|
|
145
|
+
|
|
146
|
+
/** Make the index match the mail database for these thread messages. */
|
|
147
|
+
async sync(threadMessageIds: readonly string[]): Promise<void> {
|
|
148
|
+
const ids = [...new Set(threadMessageIds)];
|
|
149
|
+
for (const slice of chunk(ids, DOCUMENTS_PER_STATEMENT)) {
|
|
150
|
+
const rows = await this.mail
|
|
151
|
+
.select(indexedColumns)
|
|
152
|
+
.from(threadMessageTable)
|
|
153
|
+
.where(inArray(threadMessageTable.threadMessageId, slice));
|
|
154
|
+
const items: SqliteBatchItem[] = [
|
|
155
|
+
this.search
|
|
156
|
+
.delete(threadMessageSearchTable)
|
|
157
|
+
.where(
|
|
158
|
+
inArray(threadMessageSearchTable.rowid, slice.map(searchRowid)),
|
|
159
|
+
),
|
|
160
|
+
];
|
|
161
|
+
if (rows.length > 0) {
|
|
162
|
+
items.push(
|
|
163
|
+
this.search
|
|
164
|
+
.insert(threadMessageSearchTable)
|
|
165
|
+
.values(rows.map(toDocument)),
|
|
166
|
+
);
|
|
167
|
+
}
|
|
168
|
+
await runBatch(this.search, items);
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
async removeAccount(accountConfigId: string): Promise<void> {
|
|
173
|
+
await this.search
|
|
174
|
+
.delete(threadMessageSearchTable)
|
|
175
|
+
.where(eq(threadMessageSearchTable.accountConfigId, accountConfigId));
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/** One page of matching thread messages, in the query's order. */
|
|
179
|
+
async candidates(query: CandidateQuery): Promise<SearchCandidate[]> {
|
|
180
|
+
const table = threadMessageSearchTable;
|
|
181
|
+
// The rank is computed once, in the inner select, and read by name
|
|
182
|
+
// after it: written out in the keyset and the ordering it would bind
|
|
183
|
+
// every term's parameters four times.
|
|
184
|
+
const matches = this.search
|
|
185
|
+
.select({
|
|
186
|
+
threadMessageId: table.threadMessageId,
|
|
187
|
+
sentDate: table.sentDate,
|
|
188
|
+
rank: (query.rank ?? sql<number>`0`).as("sender_rank"),
|
|
189
|
+
})
|
|
190
|
+
.from(table)
|
|
191
|
+
.where(
|
|
192
|
+
and(
|
|
193
|
+
eq(table.accountConfigId, query.accountConfigId),
|
|
194
|
+
query.mailboxIds
|
|
195
|
+
? sql`${table.mailboxId} in (select value from json_each(${JSON.stringify(query.mailboxIds)}))`
|
|
196
|
+
: undefined,
|
|
197
|
+
query.where,
|
|
198
|
+
),
|
|
199
|
+
)
|
|
200
|
+
.as("matches");
|
|
201
|
+
const { after } = query;
|
|
202
|
+
const dateAfter = after
|
|
203
|
+
? or(
|
|
204
|
+
query.order === "desc"
|
|
205
|
+
? lt(matches.sentDate, after.sentDate)
|
|
206
|
+
: gt(matches.sentDate, after.sentDate),
|
|
207
|
+
and(
|
|
208
|
+
eq(matches.sentDate, after.sentDate),
|
|
209
|
+
gt(matches.threadMessageId, after.threadMessageId),
|
|
210
|
+
),
|
|
211
|
+
)
|
|
212
|
+
: undefined;
|
|
213
|
+
const keyset =
|
|
214
|
+
after && query.rank
|
|
215
|
+
? or(
|
|
216
|
+
lt(matches.rank, after.rank),
|
|
217
|
+
and(eq(matches.rank, after.rank), dateAfter),
|
|
218
|
+
)
|
|
219
|
+
: dateAfter;
|
|
220
|
+
return this.search
|
|
221
|
+
.select()
|
|
222
|
+
.from(matches)
|
|
223
|
+
.where(keyset)
|
|
224
|
+
.orderBy(
|
|
225
|
+
...(query.rank ? [desc(matches.rank)] : []),
|
|
226
|
+
query.order === "desc" ? desc(matches.sentDate) : asc(matches.sentDate),
|
|
227
|
+
asc(matches.threadMessageId),
|
|
228
|
+
)
|
|
229
|
+
.limit(query.limit);
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
/**
|
|
233
|
+
* Whether the index has fallen out of step with the mail database: a
|
|
234
|
+
* different number of documents than thread messages, as a new or deleted
|
|
235
|
+
* search database, or a write that died between the two, leaves it.
|
|
236
|
+
*/
|
|
237
|
+
async needsRebuild(): Promise<boolean> {
|
|
238
|
+
const [[documents], [threads]] = await Promise.all([
|
|
239
|
+
this.search.select({ n: count() }).from(threadMessageSearchTable),
|
|
240
|
+
this.mail.select({ n: count() }).from(threadMessageTable),
|
|
241
|
+
]);
|
|
242
|
+
return documents?.n !== threads?.n;
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
/** Drop every document and index the whole mail database again. */
|
|
246
|
+
async rebuild(): Promise<number> {
|
|
247
|
+
await this.search.delete(threadMessageSearchTable);
|
|
248
|
+
let indexed = 0;
|
|
249
|
+
let after = "";
|
|
250
|
+
for (;;) {
|
|
251
|
+
const rows = await this.mail
|
|
252
|
+
.select(indexedColumns)
|
|
253
|
+
.from(threadMessageTable)
|
|
254
|
+
.where(gt(threadMessageTable.threadMessageId, after))
|
|
255
|
+
.orderBy(asc(threadMessageTable.threadMessageId))
|
|
256
|
+
.limit(DOCUMENTS_PER_REBUILD_PAGE);
|
|
257
|
+
const last = rows.at(-1);
|
|
258
|
+
if (!last) return indexed;
|
|
259
|
+
await runBatch(
|
|
260
|
+
this.search,
|
|
261
|
+
chunk(rows, DOCUMENTS_PER_STATEMENT).map((slice) =>
|
|
262
|
+
this.search
|
|
263
|
+
.insert(threadMessageSearchTable)
|
|
264
|
+
.values(slice.map(toDocument)),
|
|
265
|
+
),
|
|
266
|
+
);
|
|
267
|
+
indexed += rows.length;
|
|
268
|
+
after = last.threadMessageId;
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
}
|
package/src/sqlite-client.ts
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import type { Db } from "./db.js";
|
|
2
|
-
import { serializeSqliteWrites } from "./tx.js";
|
|
3
2
|
|
|
4
3
|
// The one place a SQLite connection is opened for the app (RFC 036 D3). Every
|
|
5
4
|
// writer container (backend, imap-worker, smtp-worker, account-worker) opens the
|
|
@@ -7,10 +6,9 @@ import { serializeSqliteWrites } from "./tx.js";
|
|
|
7
6
|
// readers, a 5 s busy_timeout so a writer waits out another's short transaction
|
|
8
7
|
// instead of failing, synchronous=NORMAL (durable under WAL), foreign keys on.
|
|
9
8
|
//
|
|
10
|
-
//
|
|
11
|
-
//
|
|
12
|
-
//
|
|
13
|
-
// see the wrapper's comment.
|
|
9
|
+
// No write on this handle holds a transaction open across an `await`: a write
|
|
10
|
+
// set runs as one synchronous batch (`batch.ts`), so the shared connection is
|
|
11
|
+
// never inside another caller's transaction when a plain write lands.
|
|
14
12
|
//
|
|
15
13
|
// better-sqlite3 and its drizzle driver are imported dynamically so the whole
|
|
16
14
|
// module stays out of the DynamoDB Lambda bundle (this package is `external`
|
|
@@ -40,10 +38,10 @@ export async function createSqliteDatabase<
|
|
|
40
38
|
sqlite.pragma("synchronous = NORMAL");
|
|
41
39
|
sqlite.pragma("foreign_keys = ON");
|
|
42
40
|
|
|
43
|
-
const
|
|
41
|
+
const db: Db<TSchema> = drizzle(sqlite, { schema });
|
|
44
42
|
|
|
45
43
|
return {
|
|
46
|
-
db
|
|
44
|
+
db,
|
|
47
45
|
close: async () => {
|
|
48
46
|
sqlite.close();
|
|
49
47
|
},
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import type { SqliteDatabases } from "./databases.js";
|
|
2
|
+
import { CalendarCollectionRepo } from "./repos/calendar-collection.js";
|
|
3
|
+
import { CalendarEventIndexRepo } from "./repos/calendar-event-index.js";
|
|
4
|
+
import { CalendarFeedTokenRepo } from "./repos/calendar-feed-token.js";
|
|
5
|
+
import { CalendarObjectRepo } from "./repos/calendar-object.js";
|
|
6
|
+
import { DrizzleCalendarStore } from "./repos/calendar-store.js";
|
|
7
|
+
import { CalendarSuggestionRepo } from "./repos/calendar-suggestion.js";
|
|
8
|
+
import { DrizzleEnvelopeRepository } from "./repos/envelope.js";
|
|
9
|
+
import { FilterRepo } from "./repos/filter.js";
|
|
10
|
+
import { AccountRepo } from "./repos/i4-account.js";
|
|
11
|
+
import { AccountConfigRepo } from "./repos/i4-account-config.js";
|
|
12
|
+
import { AccountExportRequestRepo } from "./repos/i4-account-export-request.js";
|
|
13
|
+
import { AccountSettingRepo } from "./repos/i4-account-setting.js";
|
|
14
|
+
import { AddressRepo } from "./repos/i4-address.js";
|
|
15
|
+
import { ConfigImportRepo } from "./repos/i4-config-import.js";
|
|
16
|
+
import { MailboxRepo } from "./repos/i4-mailbox.js";
|
|
17
|
+
import { MailboxLockRepo } from "./repos/i4-mailbox-lock.js";
|
|
18
|
+
import { MailboxSpecialUseRepo } from "./repos/i4-mailbox-special-use.js";
|
|
19
|
+
import { MessageFlagPushRepo } from "./repos/i4-message-flag-push.js";
|
|
20
|
+
import { MessagePlacementMoveRepo } from "./repos/i4-message-placement-move.js";
|
|
21
|
+
import { OrganizeJobRequestRepo } from "./repos/i4-organize-job-request.js";
|
|
22
|
+
import { OutboxAttachmentRepo } from "./repos/i4-outbox-attachment.js";
|
|
23
|
+
import { OutboxMessageRepo } from "./repos/i4-outbox-message.js";
|
|
24
|
+
import { SenderSignerStandingRepo } from "./repos/i4-sender-signer-standing.js";
|
|
25
|
+
import { LabelRepo } from "./repos/label.js";
|
|
26
|
+
import { DrizzleMessageRepository } from "./repos/message.js";
|
|
27
|
+
import { DrizzleMessageFlagRepository } from "./repos/message-flag.js";
|
|
28
|
+
import { MessageLabelRepo } from "./repos/message-label.js";
|
|
29
|
+
import { QuarantineRepo } from "./repos/quarantine.js";
|
|
30
|
+
import { SyncWarningRepo } from "./repos/sync-warning.js";
|
|
31
|
+
import { DrizzleThreadMessageRepository } from "./repos/thread-message.js";
|
|
32
|
+
import { SearchIndex } from "./search/thread-message-search.js";
|
|
33
|
+
|
|
34
|
+
// Every repo over the database that holds its tables (#1420).
|
|
35
|
+
export const createSqliteRepositories = ({
|
|
36
|
+
app,
|
|
37
|
+
mail,
|
|
38
|
+
search,
|
|
39
|
+
}: SqliteDatabases) => {
|
|
40
|
+
const searchIndex = new SearchIndex(mail, search);
|
|
41
|
+
return {
|
|
42
|
+
accountConfig: new AccountConfigRepo(app, mail),
|
|
43
|
+
account: new AccountRepo(app, mail),
|
|
44
|
+
accountSetting: new AccountSettingRepo(app),
|
|
45
|
+
address: new AddressRepo(mail),
|
|
46
|
+
mailbox: new MailboxRepo(mail, searchIndex),
|
|
47
|
+
mailboxSpecialUse: new MailboxSpecialUseRepo(app, mail),
|
|
48
|
+
mailboxLock: new MailboxLockRepo(mail),
|
|
49
|
+
message: new DrizzleMessageRepository(mail, searchIndex),
|
|
50
|
+
messageFlag: new DrizzleMessageFlagRepository(mail),
|
|
51
|
+
outboxMessage: new OutboxMessageRepo(mail),
|
|
52
|
+
outboxAttachment: new OutboxAttachmentRepo(mail),
|
|
53
|
+
threadMessage: new DrizzleThreadMessageRepository(mail, searchIndex),
|
|
54
|
+
envelope: new DrizzleEnvelopeRepository(mail),
|
|
55
|
+
accountExportRequest: new AccountExportRequestRepo(app),
|
|
56
|
+
configImport: new ConfigImportRepo(app),
|
|
57
|
+
quarantine: new QuarantineRepo(mail),
|
|
58
|
+
syncWarning: new SyncWarningRepo(mail),
|
|
59
|
+
organizeJobRequest: new OrganizeJobRequestRepo(mail),
|
|
60
|
+
placementMove: new MessagePlacementMoveRepo(mail),
|
|
61
|
+
flagPush: new MessageFlagPushRepo(mail),
|
|
62
|
+
filter: new FilterRepo(app),
|
|
63
|
+
label: new LabelRepo(app),
|
|
64
|
+
messageLabel: new MessageLabelRepo(mail),
|
|
65
|
+
senderSignerStanding: new SenderSignerStandingRepo(mail),
|
|
66
|
+
calendarCollection: new CalendarCollectionRepo(app),
|
|
67
|
+
calendarObject: new CalendarObjectRepo(app),
|
|
68
|
+
calendarEventIndex: new CalendarEventIndexRepo(app),
|
|
69
|
+
calendarFeedToken: new CalendarFeedTokenRepo(app),
|
|
70
|
+
calendarSuggestion: new CalendarSuggestionRepo(mail),
|
|
71
|
+
calendarStore: new DrizzleCalendarStore(app, mail),
|
|
72
|
+
};
|
|
73
|
+
};
|
|
74
|
+
|
|
75
|
+
export type SqliteRepositories = ReturnType<typeof createSqliteRepositories>;
|
package/src/test-db-sqlite.ts
CHANGED
|
@@ -1,16 +1,8 @@
|
|
|
1
|
-
import { readFileSync } from "node:fs";
|
|
2
1
|
import Database from "better-sqlite3";
|
|
3
2
|
import { pushSQLiteSchema } from "drizzle-kit/api";
|
|
4
3
|
import { drizzle } from "drizzle-orm/better-sqlite3";
|
|
5
4
|
import type { Db } from "./db.js";
|
|
6
|
-
|
|
7
|
-
// The exact FTS5 objects the migrator installs (RFC 036 D4), read from the one
|
|
8
|
-
// committed source so a test runs the real search path, not a hand-copied twin.
|
|
9
|
-
const searchIndexDdl = (): string =>
|
|
10
|
-
readFileSync(
|
|
11
|
-
new URL("../../../npm-scripts/sqlite-search-index.sql", import.meta.url),
|
|
12
|
-
"utf8",
|
|
13
|
-
);
|
|
5
|
+
import { applyMigrationSet } from "./test-shipped-sqlite-schema.js";
|
|
14
6
|
|
|
15
7
|
// A real better-sqlite3 database (in-memory by default) with the schema pushed
|
|
16
8
|
// from the drizzle table objects — the sqlite `pushSchema` — so a repo runs
|
|
@@ -53,11 +45,11 @@ export async function createSqliteTestDb<
|
|
|
53
45
|
sqlite.exec(statement);
|
|
54
46
|
}
|
|
55
47
|
|
|
56
|
-
// Install the
|
|
57
|
-
// thread-message search predicates run their real trigram path
|
|
58
|
-
//
|
|
48
|
+
// Install the committed search set beside the pushed schema so the
|
|
49
|
+
// thread-message search predicates run their real trigram path. The one
|
|
50
|
+
// handle then plays both the mail and the search database.
|
|
59
51
|
if (options?.searchIndex) {
|
|
60
|
-
sqlite
|
|
52
|
+
applyMigrationSet(sqlite, "search");
|
|
61
53
|
}
|
|
62
54
|
|
|
63
55
|
return {
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { readFileSync } from "node:fs";
|
|
2
2
|
import type Database from "better-sqlite3";
|
|
3
3
|
|
|
4
|
-
// Read the committed SQLite
|
|
4
|
+
// Read the committed SQLite migrations — the DDL a self-host deployment
|
|
5
5
|
// actually runs — so a test can exercise the shipped shape instead of the one
|
|
6
6
|
// `pushSQLiteSchema` derives from the drizzle table objects. The two are
|
|
7
7
|
// generated from the same entities but only the pushed one is regenerated on
|
|
@@ -9,67 +9,68 @@ import type Database from "better-sqlite3";
|
|
|
9
9
|
// (reader#73). Reading the files means a test fails when they drift, and
|
|
10
10
|
// tracks them when they change.
|
|
11
11
|
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
import.meta.url,
|
|
15
|
-
);
|
|
12
|
+
/** The three databases a deployment migrates, one committed set each (#1420). */
|
|
13
|
+
export type MigrationSet = "app" | "mail" | "search";
|
|
16
14
|
|
|
17
|
-
export const
|
|
18
|
-
|
|
15
|
+
export const MIGRATION_SETS: readonly MigrationSet[] = [
|
|
16
|
+
"app",
|
|
17
|
+
"mail",
|
|
18
|
+
"search",
|
|
19
|
+
];
|
|
19
20
|
|
|
20
|
-
|
|
21
|
+
const setDir = (set: MigrationSet): URL =>
|
|
22
|
+
new URL(`../../../deploy/vps/migrations-sqlite/${set}/`, import.meta.url);
|
|
23
|
+
|
|
24
|
+
const migrationSql = (set: MigrationSet, tag: string): string =>
|
|
25
|
+
readFileSync(new URL(`${tag}.sql`, setDir(set)), "utf8");
|
|
26
|
+
|
|
27
|
+
interface MigrationJournalEntry {
|
|
21
28
|
idx: number;
|
|
22
29
|
tag: string;
|
|
23
30
|
}
|
|
24
31
|
|
|
25
|
-
/** Every committed
|
|
26
|
-
|
|
32
|
+
/** Every committed migration of a set, in the order the migrator runs them. */
|
|
33
|
+
const migrationJournal = (set: MigrationSet): MigrationJournalEntry[] =>
|
|
27
34
|
(
|
|
28
35
|
JSON.parse(
|
|
29
|
-
readFileSync(new URL("meta/_journal.json",
|
|
36
|
+
readFileSync(new URL("meta/_journal.json", setDir(set)), "utf8"),
|
|
30
37
|
) as { entries: MigrationJournalEntry[] }
|
|
31
|
-
).entries;
|
|
32
|
-
|
|
33
|
-
/** The `.sql` files present on disk, whatever the journal says about them. */
|
|
34
|
-
export const migrationTagsOnDisk = (): string[] =>
|
|
35
|
-
readdirSync(MIGRATIONS_DIR)
|
|
36
|
-
.filter((name) => name.endsWith(".sql"))
|
|
37
|
-
.map((name) => name.slice(0, -".sql".length))
|
|
38
|
-
.sort();
|
|
39
|
-
|
|
40
|
-
export interface MigrationSnapshot {
|
|
41
|
-
id: string;
|
|
42
|
-
prevId: string;
|
|
43
|
-
}
|
|
38
|
+
).entries.sort((left, right) => left.idx - right.idx);
|
|
44
39
|
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
`meta/${String(idx).padStart(4, "0")}_snapshot.json`,
|
|
50
|
-
MIGRATIONS_DIR,
|
|
51
|
-
),
|
|
52
|
-
"utf8",
|
|
53
|
-
),
|
|
54
|
-
) as MigrationSnapshot;
|
|
55
|
-
|
|
56
|
-
/** The `CREATE TABLE` block for one table, as that migration declares it. */
|
|
57
|
-
export const shippedTableDdl = (tag: string, table: string): string => {
|
|
58
|
-
const match = migrationSql(tag).match(
|
|
40
|
+
/** The `CREATE TABLE` block for one mail table, as the mail baseline declares it. */
|
|
41
|
+
export const shippedTableDdl = (table: string): string => {
|
|
42
|
+
const [baseline] = migrationJournal("mail");
|
|
43
|
+
const match = migrationSql("mail", baseline.tag).match(
|
|
59
44
|
new RegExp(`CREATE TABLE \`${table}\` \\([\\s\\S]*?\\n\\);`),
|
|
60
45
|
);
|
|
61
46
|
if (!match) {
|
|
62
|
-
throw new Error(`${table} DDL not found in migration ${tag}`);
|
|
47
|
+
throw new Error(`${table} DDL not found in migration ${baseline.tag}`);
|
|
63
48
|
}
|
|
64
49
|
return match[0];
|
|
65
50
|
};
|
|
66
51
|
|
|
67
|
-
/**
|
|
68
|
-
export const
|
|
52
|
+
/** The one statement of the mail baseline that creates a named index. */
|
|
53
|
+
export const shippedIndexDdl = (index: string): string => {
|
|
54
|
+
const [baseline] = migrationJournal("mail");
|
|
55
|
+
const statement = migrationSql("mail", baseline.tag)
|
|
56
|
+
.split("--> statement-breakpoint")
|
|
57
|
+
.find((candidate) => candidate.includes(`INDEX \`${index}\``));
|
|
58
|
+
if (!statement) {
|
|
59
|
+
throw new Error(`${index} not found in migration ${baseline.tag}`);
|
|
60
|
+
}
|
|
61
|
+
return statement;
|
|
62
|
+
};
|
|
63
|
+
|
|
64
|
+
/** Run every committed migration of a set against an open database. */
|
|
65
|
+
export const applyMigrationSet = (
|
|
69
66
|
sqlite: Database.Database,
|
|
70
|
-
|
|
67
|
+
set: MigrationSet,
|
|
71
68
|
): void => {
|
|
72
|
-
for (const
|
|
73
|
-
|
|
69
|
+
for (const entry of migrationJournal(set)) {
|
|
70
|
+
for (const statement of migrationSql(set, entry.tag).split(
|
|
71
|
+
"--> statement-breakpoint",
|
|
72
|
+
)) {
|
|
73
|
+
sqlite.exec(statement);
|
|
74
|
+
}
|
|
74
75
|
}
|
|
75
76
|
};
|
package/src/test-sqlite.ts
CHANGED
|
@@ -1,23 +1,25 @@
|
|
|
1
|
+
import { join } from "node:path";
|
|
1
2
|
import Database from "better-sqlite3";
|
|
2
3
|
import { drizzle } from "drizzle-orm/better-sqlite3";
|
|
4
|
+
import { SQLITE_DATABASE_ENV, type SqliteDatabasePaths } from "./databases.js";
|
|
3
5
|
import type { Db } from "./db.js";
|
|
4
6
|
import {
|
|
5
|
-
|
|
6
|
-
|
|
7
|
+
applyMigrationSet,
|
|
8
|
+
MIGRATION_SETS,
|
|
7
9
|
} from "./test-shipped-sqlite-schema.js";
|
|
8
10
|
|
|
9
11
|
/**
|
|
10
|
-
* The store a test runs against: the committed SQLite
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
12
|
+
* The store a test runs against: the committed SQLite migrations, in the order
|
|
13
|
+
* the migrator runs them, over the same engine a self-host deployment boots.
|
|
14
|
+
* Applying the shipped DDL rather than pushing the drizzle table objects is the
|
|
15
|
+
* point — a test then fails when the two drift (reader#73).
|
|
16
|
+
*
|
|
17
|
+
* All three sets land in the one database, which a test then hands to every
|
|
18
|
+
* repo whatever database the repo runs on in a deployment. Their tables do not
|
|
19
|
+
* overlap, so nothing collides.
|
|
14
20
|
*/
|
|
15
21
|
export const applyShippedMigrations = (sqlite: Database.Database): void => {
|
|
16
|
-
for (const
|
|
17
|
-
(left, right) => left.idx - right.idx,
|
|
18
|
-
)) {
|
|
19
|
-
applyMigration(sqlite, entry.tag);
|
|
20
|
-
}
|
|
22
|
+
for (const set of MIGRATION_SETS) applyMigrationSet(sqlite, set);
|
|
21
23
|
};
|
|
22
24
|
|
|
23
25
|
/**
|
|
@@ -37,3 +39,25 @@ export const createShippedSqliteDb = (): {
|
|
|
37
39
|
close: () => sqlite.close(),
|
|
38
40
|
};
|
|
39
41
|
};
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* The three database files a deployment runs on, each migrated with its own
|
|
45
|
+
* set, under `directory` — and the environment pointed at them, so a
|
|
46
|
+
* composition root a test builds opens them exactly as a deployment does.
|
|
47
|
+
*/
|
|
48
|
+
export const createShippedSqliteFiles = (
|
|
49
|
+
directory: string,
|
|
50
|
+
): SqliteDatabasePaths => {
|
|
51
|
+
const paths: SqliteDatabasePaths = {
|
|
52
|
+
app: join(directory, "app.db"),
|
|
53
|
+
mail: join(directory, "mail.db"),
|
|
54
|
+
search: join(directory, "search.db"),
|
|
55
|
+
};
|
|
56
|
+
for (const set of MIGRATION_SETS) {
|
|
57
|
+
const sqlite = new Database(paths[set]);
|
|
58
|
+
applyMigrationSet(sqlite, set);
|
|
59
|
+
sqlite.close();
|
|
60
|
+
process.env[SQLITE_DATABASE_ENV[set]] = paths[set];
|
|
61
|
+
}
|
|
62
|
+
return paths;
|
|
63
|
+
};
|