@remit/drizzle-service 0.0.104 → 0.0.105
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/bound-parameters.test.ts +57 -0
- package/src/bound-parameters.ts +49 -0
- package/src/databases.ts +10 -9
- package/src/db.ts +1 -0
- package/src/index.ts +4 -0
- package/src/repair/junk-only-address.sqlite.test.ts +9 -5
- package/src/repos/account.conformance.sqlite.test.ts +9 -5
- package/src/repos/calendar-event-index.conformance.d1.test.ts +1 -1
- package/src/repos/calendar-event-index.conformance.sqlite.test.ts +3 -1
- package/src/repos/calendar-event-index.ts +36 -21
- package/src/repos/calendar-store.ts +17 -10
- package/src/repos/cascade-delete.test.ts +9 -2
- package/src/repos/cascade-delete.ts +2 -1
- package/src/repos/envelope.d1.test.ts +66 -0
- package/src/repos/envelope.sqlite.test.ts +5 -1
- package/src/repos/envelope.test.ts +5 -1
- package/src/repos/envelope.ts +40 -31
- package/src/repos/i4-account.test.ts +17 -10
- package/src/repos/i4-address-junk-move.sqlite.test.ts +9 -5
- package/src/repos/i4-mailbox-special-use.sqlite.test.ts +9 -5
- package/src/repos/i4-mailbox.sqlite.test.ts +17 -10
- package/src/repos/i4-mailbox.test.ts +17 -10
- package/src/repos/i4-outbox-attachment.test.ts +9 -5
- package/src/repos/message.sqlite.test.ts +5 -1
- package/src/repos/message.test.ts +5 -1
- package/src/repos/sync-warning.ts +35 -22
- package/src/repos/thread-message-body-muted.sqlite.test.ts +9 -5
- package/src/repos/thread-message-field-terms.sqlite.test.ts +9 -5
- package/src/repos/thread-message.sqlite.test.ts +9 -5
- package/src/repos/thread-message.test.ts +33 -20
- package/src/search/thread-message-search.sqlite.test.ts +13 -6
- package/src/search/thread-message-search.ts +14 -17
- package/src/sqlite-repositories.ts +6 -3
- package/src/test-d1.ts +36 -19
- package/src/test-stores.ts +15 -2
package/package.json
CHANGED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { describe, test } from "node:test";
|
|
3
|
+
import { integer, sqliteTable, text } from "drizzle-orm/sqlite-core";
|
|
4
|
+
import { chunkRows, rowsPerStatement } from "./bound-parameters.js";
|
|
5
|
+
|
|
6
|
+
const tenColumns = sqliteTable("ten_columns", {
|
|
7
|
+
c0: text("c0"),
|
|
8
|
+
c1: text("c1"),
|
|
9
|
+
c2: text("c2"),
|
|
10
|
+
c3: text("c3"),
|
|
11
|
+
c4: text("c4"),
|
|
12
|
+
c5: text("c5"),
|
|
13
|
+
c6: text("c6"),
|
|
14
|
+
c7: text("c7"),
|
|
15
|
+
c8: text("c8"),
|
|
16
|
+
c9: integer("c9"),
|
|
17
|
+
});
|
|
18
|
+
|
|
19
|
+
const rows = (count: number): number[] =>
|
|
20
|
+
Array.from({ length: count }, (_, index) => index);
|
|
21
|
+
|
|
22
|
+
describe("chunkRows", () => {
|
|
23
|
+
test("no rows make no statements", () => {
|
|
24
|
+
assert.deepEqual(chunkRows(tenColumns, [], 100), []);
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
test("rows that fill the limit exactly make one statement", () => {
|
|
28
|
+
const chunks = chunkRows(tenColumns, rows(10), 100);
|
|
29
|
+
assert.deepEqual(
|
|
30
|
+
chunks.map((slice) => slice.length),
|
|
31
|
+
[10],
|
|
32
|
+
);
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
test("one row over the limit makes a second statement", () => {
|
|
36
|
+
const chunks = chunkRows(tenColumns, rows(11), 100);
|
|
37
|
+
assert.deepEqual(
|
|
38
|
+
chunks.map((slice) => slice.length),
|
|
39
|
+
[10, 1],
|
|
40
|
+
);
|
|
41
|
+
assert.deepEqual(chunks.flat(), rows(11));
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
test("reserved parameters shrink every statement", () => {
|
|
45
|
+
const chunks = chunkRows(tenColumns, rows(10), 100, 1);
|
|
46
|
+
assert.deepEqual(
|
|
47
|
+
chunks.map((slice) => slice.length),
|
|
48
|
+
[9, 1],
|
|
49
|
+
);
|
|
50
|
+
});
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
describe("rowsPerStatement", () => {
|
|
54
|
+
test("a row wider than the limit is refused", () => {
|
|
55
|
+
assert.throws(() => rowsPerStatement(100, 101), RangeError);
|
|
56
|
+
});
|
|
57
|
+
});
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { getTableColumns, type Table } from "drizzle-orm";
|
|
2
|
+
|
|
3
|
+
export interface SqliteLimits {
|
|
4
|
+
/** The most parameters the host's SQLite binds in one statement: 32766 on better-sqlite3, 100 on D1 and Durable Object storage. */
|
|
5
|
+
maxBoundParameters: number;
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
export const BETTER_SQLITE3_LIMITS: SqliteLimits = {
|
|
9
|
+
maxBoundParameters: 32766,
|
|
10
|
+
};
|
|
11
|
+
|
|
12
|
+
export const chunk = <T>(items: readonly T[], size: number): T[][] => {
|
|
13
|
+
const chunks: T[][] = [];
|
|
14
|
+
for (let start = 0; start < items.length; start += size) {
|
|
15
|
+
chunks.push(items.slice(start, start + size));
|
|
16
|
+
}
|
|
17
|
+
return chunks;
|
|
18
|
+
};
|
|
19
|
+
|
|
20
|
+
export const rowsPerStatement = (
|
|
21
|
+
maxBoundParameters: number,
|
|
22
|
+
parametersPerRow: number,
|
|
23
|
+
reservedParameters = 0,
|
|
24
|
+
): number => {
|
|
25
|
+
const rows = Math.floor(
|
|
26
|
+
(maxBoundParameters - reservedParameters) / parametersPerRow,
|
|
27
|
+
);
|
|
28
|
+
if (rows < 1) {
|
|
29
|
+
throw new RangeError(
|
|
30
|
+
`a row of ${parametersPerRow} bound parameters does not fit a statement limit of ${maxBoundParameters}`,
|
|
31
|
+
);
|
|
32
|
+
}
|
|
33
|
+
return rows;
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
export const chunkRows = <T>(
|
|
37
|
+
table: Table,
|
|
38
|
+
rows: readonly T[],
|
|
39
|
+
maxBoundParameters: number,
|
|
40
|
+
reservedParameters = 0,
|
|
41
|
+
): T[][] =>
|
|
42
|
+
chunk(
|
|
43
|
+
rows,
|
|
44
|
+
rowsPerStatement(
|
|
45
|
+
maxBoundParameters,
|
|
46
|
+
Object.keys(getTableColumns(table)).length,
|
|
47
|
+
reservedParameters,
|
|
48
|
+
),
|
|
49
|
+
);
|
package/src/databases.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { syncBatch } from "./batch.js";
|
|
2
|
+
import type { SqliteLimits } from "./bound-parameters.js";
|
|
2
3
|
import type { Db, SqliteStore } from "./db.js";
|
|
3
4
|
import { messageDataSchema } from "./schema/message-data.js";
|
|
4
5
|
import { createSqliteDatabase } from "./sqlite-client.js";
|
|
@@ -22,16 +23,16 @@ export interface SqliteStores {
|
|
|
22
23
|
|
|
23
24
|
export const syncStore = <TSchema extends Record<string, unknown>>(
|
|
24
25
|
db: Db<TSchema>,
|
|
25
|
-
|
|
26
|
+
{ maxBoundParameters }: SqliteLimits,
|
|
27
|
+
): SqliteStore<TSchema> => ({ db, batch: syncBatch(db), maxBoundParameters });
|
|
26
28
|
|
|
27
|
-
export const syncStores = (
|
|
28
|
-
app,
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
search: syncStore(search),
|
|
29
|
+
export const syncStores = (
|
|
30
|
+
{ app, mail, search }: SqliteDatabases,
|
|
31
|
+
limits: SqliteLimits,
|
|
32
|
+
): SqliteStores => ({
|
|
33
|
+
app: syncStore(app, limits),
|
|
34
|
+
mail: syncStore(mail, limits),
|
|
35
|
+
search: syncStore(search, limits),
|
|
35
36
|
});
|
|
36
37
|
|
|
37
38
|
export interface SqliteDatabasePaths {
|
package/src/db.ts
CHANGED
package/src/index.ts
CHANGED
|
@@ -4,6 +4,7 @@ import { composeFolderRoleAppointmentName } from "@remit/data-ports/folder-role"
|
|
|
4
4
|
import { CanonicalMailboxRole } from "@remit/domain-enums";
|
|
5
5
|
import Database from "better-sqlite3";
|
|
6
6
|
import { drizzle } from "drizzle-orm/better-sqlite3";
|
|
7
|
+
import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
|
|
7
8
|
import { AccountSettingRepo } from "../repos/i4-account-setting.js";
|
|
8
9
|
import type { MailboxSpecialUseRepo } from "../repos/i4-mailbox-special-use.js";
|
|
9
10
|
import { createSqliteRepositories } from "../sqlite-repositories.js";
|
|
@@ -182,11 +183,14 @@ describe("addresses standing only on mail in Junk", () => {
|
|
|
182
183
|
before(() => {
|
|
183
184
|
sqlite = new Database(":memory:");
|
|
184
185
|
const db = drizzle(sqlite);
|
|
185
|
-
specialUse = createSqliteRepositories(
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
186
|
+
specialUse = createSqliteRepositories(
|
|
187
|
+
{
|
|
188
|
+
app: db,
|
|
189
|
+
mail: db,
|
|
190
|
+
search: db,
|
|
191
|
+
},
|
|
192
|
+
BETTER_SQLITE3_LIMITS,
|
|
193
|
+
).mailboxSpecialUse;
|
|
190
194
|
accountSetting = new AccountSettingRepo(db as never);
|
|
191
195
|
applyMigrationSet(sqlite, "app");
|
|
192
196
|
applyMigrationSet(sqlite, "mail");
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { accountRepositoryConformance } from "@remit/data-ports/conformance";
|
|
2
|
+
import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
|
|
2
3
|
import { NotFoundError } from "../error.js";
|
|
3
4
|
import { randomId } from "../id.js";
|
|
4
5
|
import { accountTable } from "../schema.js";
|
|
@@ -13,11 +14,14 @@ accountRepositoryConformance({
|
|
|
13
14
|
accounts: accountTable,
|
|
14
15
|
});
|
|
15
16
|
close = closeDb;
|
|
16
|
-
return createSqliteRepositories(
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
17
|
+
return createSqliteRepositories(
|
|
18
|
+
{
|
|
19
|
+
app: db,
|
|
20
|
+
mail: db,
|
|
21
|
+
search: db,
|
|
22
|
+
},
|
|
23
|
+
BETTER_SQLITE3_LIMITS,
|
|
24
|
+
).account;
|
|
21
25
|
},
|
|
22
26
|
teardown: async () => {
|
|
23
27
|
await close?.();
|
|
@@ -10,7 +10,7 @@ calendarEventIndexRepositoryConformance({
|
|
|
10
10
|
async createRepository() {
|
|
11
11
|
const opened = await openD1TestStores();
|
|
12
12
|
close = opened.close;
|
|
13
|
-
return new CalendarEventIndexRepo(opened.stores.app
|
|
13
|
+
return new CalendarEventIndexRepo(opened.stores.app);
|
|
14
14
|
},
|
|
15
15
|
teardown: async () => {
|
|
16
16
|
await close?.();
|
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import { calendarEventIndexRepositoryConformance } from "@remit/data-ports/conformance";
|
|
2
|
+
import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
|
|
3
|
+
import { syncStore } from "../databases.js";
|
|
2
4
|
import { NotFoundError } from "../error.js";
|
|
3
5
|
import { randomId } from "../id.js";
|
|
4
6
|
import { calendarEventIndexTable } from "../schema.js";
|
|
@@ -13,7 +15,7 @@ calendarEventIndexRepositoryConformance({
|
|
|
13
15
|
calendarEventIndexes: calendarEventIndexTable,
|
|
14
16
|
});
|
|
15
17
|
close = closeDb;
|
|
16
|
-
return new CalendarEventIndexRepo(db
|
|
18
|
+
return new CalendarEventIndexRepo(syncStore(db, BETTER_SQLITE3_LIMITS));
|
|
17
19
|
},
|
|
18
20
|
teardown: async () => {
|
|
19
21
|
await close?.();
|
|
@@ -4,7 +4,9 @@ import type {
|
|
|
4
4
|
ICalendarEventIndexRepository,
|
|
5
5
|
} from "@remit/data-ports";
|
|
6
6
|
import { and, asc, eq, gte, lt } from "drizzle-orm";
|
|
7
|
-
import type {
|
|
7
|
+
import type { SqliteBatchItem, WriteBatch } from "../batch.js";
|
|
8
|
+
import { chunkRows } from "../bound-parameters.js";
|
|
9
|
+
import type { RepositoryDb, SqliteStore } from "../db.js";
|
|
8
10
|
import { calendarEventIndexTable } from "../schema.js";
|
|
9
11
|
|
|
10
12
|
type DB = RepositoryDb<Record<string, unknown>>;
|
|
@@ -28,39 +30,52 @@ function rowToOccurrence(
|
|
|
28
30
|
}
|
|
29
31
|
|
|
30
32
|
export class CalendarEventIndexRepo implements ICalendarEventIndexRepository {
|
|
31
|
-
|
|
33
|
+
private readonly db: DB;
|
|
34
|
+
private readonly batch: WriteBatch;
|
|
35
|
+
private readonly maxBoundParameters: number;
|
|
36
|
+
|
|
37
|
+
constructor(store: SqliteStore<Record<string, unknown>>) {
|
|
38
|
+
this.db = store.db;
|
|
39
|
+
this.batch = store.batch;
|
|
40
|
+
this.maxBoundParameters = store.maxBoundParameters;
|
|
41
|
+
}
|
|
32
42
|
|
|
33
43
|
async replaceForObject(
|
|
34
44
|
calendarId: string,
|
|
35
45
|
calendarObjectId: string,
|
|
36
46
|
occurrences: CalendarOccurrenceInput[],
|
|
37
47
|
): Promise<void> {
|
|
38
|
-
await this.deleteForObject(calendarId, calendarObjectId);
|
|
39
|
-
if (occurrences.length === 0) return;
|
|
40
|
-
|
|
41
48
|
const now = Date.now();
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
49
|
+
const rows = occurrences.map((occurrence) => ({
|
|
50
|
+
calendarId,
|
|
51
|
+
calendarObjectId,
|
|
52
|
+
recurrenceId: occurrence.recurrenceId,
|
|
53
|
+
startAt: occurrence.startAt,
|
|
54
|
+
endAt: occurrence.endAt,
|
|
55
|
+
allDay: occurrence.allDay,
|
|
56
|
+
summary: occurrence.summary,
|
|
57
|
+
status: occurrence.status,
|
|
58
|
+
transparency: occurrence.transparency,
|
|
59
|
+
createdAt: now,
|
|
60
|
+
updatedAt: now,
|
|
61
|
+
}));
|
|
62
|
+
const inserts: SqliteBatchItem[] = chunkRows(
|
|
63
|
+
calendarEventIndexTable,
|
|
64
|
+
rows,
|
|
65
|
+
this.maxBoundParameters,
|
|
66
|
+
).map((slice) => this.db.insert(calendarEventIndexTable).values(slice));
|
|
67
|
+
await this.batch([this.clear(calendarId, calendarObjectId), ...inserts]);
|
|
57
68
|
}
|
|
58
69
|
|
|
59
70
|
async deleteForObject(
|
|
60
71
|
calendarId: string,
|
|
61
72
|
calendarObjectId: string,
|
|
62
73
|
): Promise<void> {
|
|
63
|
-
await this.
|
|
74
|
+
await this.clear(calendarId, calendarObjectId);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
private clear(calendarId: string, calendarObjectId: string) {
|
|
78
|
+
return this.db
|
|
64
79
|
.delete(calendarEventIndexTable)
|
|
65
80
|
.where(
|
|
66
81
|
and(
|
|
@@ -8,6 +8,7 @@ import type {
|
|
|
8
8
|
import { deriveCalendarObjectId } from "@remit/data-ports/id";
|
|
9
9
|
import { and, eq, exists, sql } from "drizzle-orm";
|
|
10
10
|
import type { SqliteBatchItem, WriteBatch } from "../batch.js";
|
|
11
|
+
import { chunkRows } from "../bound-parameters.js";
|
|
11
12
|
import type { RepositoryDb, SqliteStore } from "../db.js";
|
|
12
13
|
import {
|
|
13
14
|
calendarEventIndexTable,
|
|
@@ -36,6 +37,7 @@ export class DrizzleCalendarStore implements ICalendarStore {
|
|
|
36
37
|
|
|
37
38
|
private readonly db: DB;
|
|
38
39
|
private readonly batch: WriteBatch;
|
|
40
|
+
private readonly maxBoundParameters: number;
|
|
39
41
|
|
|
40
42
|
constructor(
|
|
41
43
|
store: SqliteStore<Record<string, unknown>>,
|
|
@@ -43,10 +45,11 @@ export class DrizzleCalendarStore implements ICalendarStore {
|
|
|
43
45
|
) {
|
|
44
46
|
this.db = store.db;
|
|
45
47
|
this.batch = store.batch;
|
|
48
|
+
this.maxBoundParameters = store.maxBoundParameters;
|
|
46
49
|
this.repos = {
|
|
47
50
|
calendarCollection: new CalendarCollectionRepo(store.db),
|
|
48
51
|
calendarObject: new CalendarObjectRepo(store.db),
|
|
49
|
-
calendarEventIndex: new CalendarEventIndexRepo(store
|
|
52
|
+
calendarEventIndex: new CalendarEventIndexRepo(store),
|
|
50
53
|
calendarSuggestion,
|
|
51
54
|
calendarFeedToken: new CalendarFeedTokenRepo(store.db),
|
|
52
55
|
};
|
|
@@ -138,17 +141,21 @@ export class DrizzleCalendarStore implements ICalendarStore {
|
|
|
138
141
|
}),
|
|
139
142
|
this.clearOccurrences(calendarId, calendarObjectId),
|
|
140
143
|
];
|
|
141
|
-
|
|
144
|
+
const occurrences = write.occurrences.map((occurrence) => ({
|
|
145
|
+
...occurrence,
|
|
146
|
+
calendarId,
|
|
147
|
+
calendarObjectId,
|
|
148
|
+
createdAt: now,
|
|
149
|
+
updatedAt: now,
|
|
150
|
+
}));
|
|
142
151
|
return [
|
|
143
152
|
...items,
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
updatedAt: now,
|
|
151
|
-
})),
|
|
153
|
+
...chunkRows(
|
|
154
|
+
calendarEventIndexTable,
|
|
155
|
+
occurrences,
|
|
156
|
+
this.maxBoundParameters,
|
|
157
|
+
).map((slice) =>
|
|
158
|
+
this.db.insert(calendarEventIndexTable).values(slice),
|
|
152
159
|
),
|
|
153
160
|
];
|
|
154
161
|
}
|
|
@@ -2,6 +2,7 @@ import assert from "node:assert";
|
|
|
2
2
|
import { after, before, describe, test } from "node:test";
|
|
3
3
|
import { MailboxCursorState } from "@remit/domain-enums";
|
|
4
4
|
import { eq } from "drizzle-orm";
|
|
5
|
+
import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
|
|
5
6
|
import { syncStores } from "../databases.js";
|
|
6
7
|
import {
|
|
7
8
|
accountSettingTable,
|
|
@@ -300,7 +301,10 @@ describe("runDrizzleCascadeDelete", () => {
|
|
|
300
301
|
|
|
301
302
|
test("deletes every enumerated row across all tables", async () => {
|
|
302
303
|
await runDrizzleCascadeDelete(
|
|
303
|
-
syncStores(
|
|
304
|
+
syncStores(
|
|
305
|
+
{ app: db, mail: db, search: db } as never,
|
|
306
|
+
BETTER_SQLITE3_LIMITS,
|
|
307
|
+
),
|
|
304
308
|
entities,
|
|
305
309
|
log,
|
|
306
310
|
);
|
|
@@ -439,7 +443,10 @@ describe("runDrizzleCascadeDelete", () => {
|
|
|
439
443
|
await assert.rejects(
|
|
440
444
|
() =>
|
|
441
445
|
runDrizzleCascadeDelete(
|
|
442
|
-
syncStores(
|
|
446
|
+
syncStores(
|
|
447
|
+
{ app: db, mail: db, search: db } as never,
|
|
448
|
+
BETTER_SQLITE3_LIMITS,
|
|
449
|
+
),
|
|
443
450
|
[{ entityType: "Nonsense", key: { id: "x" } }],
|
|
444
451
|
log,
|
|
445
452
|
),
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { SqliteBatchItem } from "../batch.js";
|
|
2
|
+
import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
|
|
2
3
|
import {
|
|
3
4
|
openSqliteDatabases,
|
|
4
5
|
type SqliteDatabasePaths,
|
|
@@ -241,6 +242,6 @@ export const createSqliteCascadeDeleter = async (
|
|
|
241
242
|
paths: SqliteDatabasePaths,
|
|
242
243
|
): Promise<CascadeDeleter> => {
|
|
243
244
|
const databases = await openSqliteDatabases(paths);
|
|
244
|
-
const stores = syncStores(databases);
|
|
245
|
+
const stores = syncStores(databases, BETTER_SQLITE3_LIMITS);
|
|
245
246
|
return (entities, log) => runDrizzleCascadeDelete(stores, entities, log);
|
|
246
247
|
};
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { after, before, describe, test } from "node:test";
|
|
3
|
+
import type { BodyPartUpsertInput } from "@remit/data-ports";
|
|
4
|
+
import { ReferenceType } from "@remit/domain-enums";
|
|
5
|
+
import { openD1TestStore } from "../test-d1.js";
|
|
6
|
+
import { DrizzleEnvelopeRepository } from "./envelope.js";
|
|
7
|
+
|
|
8
|
+
const MESSAGE_ID = "00000000-0000-0000-4444-000000000d01";
|
|
9
|
+
|
|
10
|
+
const bodyPart = (index: number): BodyPartUpsertInput => ({
|
|
11
|
+
partPath: `${index + 1}`,
|
|
12
|
+
parentPartPath: null,
|
|
13
|
+
mediaType: "TEXT",
|
|
14
|
+
mediaSubtype: "plain",
|
|
15
|
+
transferEncoding: "7BIT",
|
|
16
|
+
sizeOctets: 128,
|
|
17
|
+
isMultipart: false,
|
|
18
|
+
parameters: [{ parameterName: "charset", parameterValue: "utf-8" }],
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
describe("DrizzleEnvelopeRepository on D1's 100 bound parameter limit", () => {
|
|
22
|
+
let repo: DrizzleEnvelopeRepository;
|
|
23
|
+
let close: () => Promise<void>;
|
|
24
|
+
|
|
25
|
+
before(async () => {
|
|
26
|
+
const opened = await openD1TestStore("mail");
|
|
27
|
+
close = opened.close;
|
|
28
|
+
repo = new DrizzleEnvelopeRepository(opened.store);
|
|
29
|
+
await repo.upsertEnvelope({
|
|
30
|
+
envelopeId: "ignored-derived",
|
|
31
|
+
messageId: MESSAGE_ID,
|
|
32
|
+
dateValue: 1700000000000,
|
|
33
|
+
dateRaw: "Mon, 14 Nov 2023 08:00:00 +0000",
|
|
34
|
+
subject: "Thirty parts",
|
|
35
|
+
});
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
after(async () => {
|
|
39
|
+
await close();
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
test("upsertBodyParts writes thirty parts and their parameters", async () => {
|
|
43
|
+
await repo.upsertBodyParts(
|
|
44
|
+
MESSAGE_ID,
|
|
45
|
+
Array.from({ length: 30 }, (_, index) => bodyPart(index)),
|
|
46
|
+
);
|
|
47
|
+
|
|
48
|
+
const data = await repo.getMessageData(MESSAGE_ID);
|
|
49
|
+
assert.equal(data.bodyPart.length, 30);
|
|
50
|
+
assert.equal(data.bodyPartParameter.length, 30);
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
test("replaceMessageReferences writes thirty references", async () => {
|
|
54
|
+
await repo.replaceMessageReferences(
|
|
55
|
+
MESSAGE_ID,
|
|
56
|
+
Array.from({ length: 30 }, (_, index) => ({
|
|
57
|
+
messageIdValue: `<ref-${index}@example.com>`,
|
|
58
|
+
referenceType: ReferenceType.References,
|
|
59
|
+
referenceOrder: index,
|
|
60
|
+
})),
|
|
61
|
+
);
|
|
62
|
+
|
|
63
|
+
const data = await repo.getMessageData(MESSAGE_ID);
|
|
64
|
+
assert.equal(data.messageReference.length, 30);
|
|
65
|
+
});
|
|
66
|
+
});
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import assert from "node:assert/strict";
|
|
2
2
|
import { after, before, describe, test } from "node:test";
|
|
3
|
+
import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
|
|
3
4
|
import {
|
|
4
5
|
type MessageDataSchema,
|
|
5
6
|
messageDataSchema,
|
|
@@ -20,7 +21,10 @@ describe("DrizzleEnvelopeRepository (sqlite)", () => {
|
|
|
20
21
|
|
|
21
22
|
before(async () => {
|
|
22
23
|
({ db, close } = await createSqliteTestDb(messageDataSchema));
|
|
23
|
-
repo = createSqliteRepositories(
|
|
24
|
+
repo = createSqliteRepositories(
|
|
25
|
+
{ app: db, mail: db, search: db },
|
|
26
|
+
BETTER_SQLITE3_LIMITS,
|
|
27
|
+
).envelope;
|
|
24
28
|
});
|
|
25
29
|
|
|
26
30
|
after(async () => {
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import assert from "node:assert/strict";
|
|
2
2
|
import { after, before, describe, test } from "node:test";
|
|
3
3
|
import type { BodyPartUpsertInput } from "@remit/data-ports";
|
|
4
|
+
import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
|
|
4
5
|
import { envelopeId as deriveEnvelopeId } from "../id.js";
|
|
5
6
|
import { createSqliteRepositories } from "../sqlite-repositories.js";
|
|
6
7
|
import type { DrizzleEnvelopeRepository } from "./envelope.js";
|
|
@@ -19,7 +20,10 @@ describe("DrizzleEnvelopeRepository", () => {
|
|
|
19
20
|
|
|
20
21
|
before(async () => {
|
|
21
22
|
({ db, stop } = await createTestDb());
|
|
22
|
-
repo = createSqliteRepositories(
|
|
23
|
+
repo = createSqliteRepositories(
|
|
24
|
+
{ app: db, mail: db, search: db },
|
|
25
|
+
BETTER_SQLITE3_LIMITS,
|
|
26
|
+
).envelope;
|
|
23
27
|
|
|
24
28
|
await repo.upsertEnvelope({
|
|
25
29
|
envelopeId: ENVELOPE_ID,
|
package/src/repos/envelope.ts
CHANGED
|
@@ -9,7 +9,8 @@ import type {
|
|
|
9
9
|
UpdateEnvelopeInput,
|
|
10
10
|
} from "@remit/data-ports";
|
|
11
11
|
import { eq } from "drizzle-orm";
|
|
12
|
-
import type { WriteBatch } from "../batch.js";
|
|
12
|
+
import type { SqliteBatchItem, WriteBatch } from "../batch.js";
|
|
13
|
+
import { chunkRows } from "../bound-parameters.js";
|
|
13
14
|
import type { RepositoryDb, SqliteStore } from "../db.js";
|
|
14
15
|
import {
|
|
15
16
|
CreateFailedConflictError,
|
|
@@ -57,10 +58,12 @@ function dedupeById<T>(rows: T[], getId: (row: T) => string): T[] {
|
|
|
57
58
|
export class DrizzleEnvelopeRepository implements IEnvelopeRepository {
|
|
58
59
|
private readonly db: DB;
|
|
59
60
|
private readonly batch: WriteBatch;
|
|
61
|
+
private readonly maxBoundParameters: number;
|
|
60
62
|
|
|
61
63
|
constructor(store: SqliteStore<Record<string, unknown>>) {
|
|
62
64
|
this.db = store.db;
|
|
63
65
|
this.batch = store.batch;
|
|
66
|
+
this.maxBoundParameters = store.maxBoundParameters;
|
|
64
67
|
}
|
|
65
68
|
|
|
66
69
|
async getMessageData(messageId: string): Promise<MessageData> {
|
|
@@ -180,22 +183,30 @@ export class DrizzleEnvelopeRepository implements IEnvelopeRepository {
|
|
|
180
183
|
(r) => r.bodyPartParameterId,
|
|
181
184
|
);
|
|
182
185
|
|
|
183
|
-
const
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
186
|
+
const upsertUpdatedAt = 1;
|
|
187
|
+
const insertParts: SqliteBatchItem[] = chunkRows(
|
|
188
|
+
bodyPartTable,
|
|
189
|
+
dedupedBodyPartRows,
|
|
190
|
+
this.maxBoundParameters,
|
|
191
|
+
upsertUpdatedAt,
|
|
192
|
+
).map((slice) =>
|
|
193
|
+
this.db
|
|
194
|
+
.insert(bodyPartTable)
|
|
195
|
+
.values(slice)
|
|
196
|
+
.onConflictDoUpdate({
|
|
197
|
+
target: bodyPartTable.bodyPartId,
|
|
198
|
+
set: { updatedAt: now },
|
|
199
|
+
}),
|
|
200
|
+
);
|
|
201
|
+
const insertParameters: SqliteBatchItem[] = chunkRows(
|
|
202
|
+
bodyPartParameterTable,
|
|
203
|
+
dedupedParameterRows,
|
|
204
|
+
this.maxBoundParameters,
|
|
205
|
+
upsertUpdatedAt,
|
|
206
|
+
).map((slice) =>
|
|
196
207
|
this.db
|
|
197
208
|
.insert(bodyPartParameterTable)
|
|
198
|
-
.values(
|
|
209
|
+
.values(slice)
|
|
199
210
|
.onConflictDoUpdate({
|
|
200
211
|
target: bodyPartParameterTable.bodyPartParameterId,
|
|
201
212
|
set: {
|
|
@@ -203,7 +214,8 @@ export class DrizzleEnvelopeRepository implements IEnvelopeRepository {
|
|
|
203
214
|
updatedAt: now,
|
|
204
215
|
},
|
|
205
216
|
}),
|
|
206
|
-
|
|
217
|
+
);
|
|
218
|
+
await this.batch([...insertParts, ...insertParameters]);
|
|
207
219
|
}
|
|
208
220
|
|
|
209
221
|
async replaceMessageReferences(
|
|
@@ -215,23 +227,20 @@ export class DrizzleEnvelopeRepository implements IEnvelopeRepository {
|
|
|
215
227
|
const clear = this.db
|
|
216
228
|
.delete(messageReferenceTable)
|
|
217
229
|
.where(eq(messageReferenceTable.messageId, messageId));
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
230
|
+
const rows = references.map((reference) => ({
|
|
231
|
+
...reference,
|
|
232
|
+
messageReferenceId: deterministicBase36Id(
|
|
233
|
+
`msgref:${messageId}:${reference.referenceType}:${reference.referenceOrder}`,
|
|
234
|
+
),
|
|
235
|
+
messageId,
|
|
236
|
+
envelopeId,
|
|
237
|
+
createdAt: now,
|
|
238
|
+
updatedAt: now,
|
|
239
|
+
}));
|
|
222
240
|
await this.batch([
|
|
223
241
|
clear,
|
|
224
|
-
this.
|
|
225
|
-
|
|
226
|
-
...reference,
|
|
227
|
-
messageReferenceId: deterministicBase36Id(
|
|
228
|
-
`msgref:${messageId}:${reference.referenceType}:${reference.referenceOrder}`,
|
|
229
|
-
),
|
|
230
|
-
messageId,
|
|
231
|
-
envelopeId,
|
|
232
|
-
createdAt: now,
|
|
233
|
-
updatedAt: now,
|
|
234
|
-
})),
|
|
242
|
+
...chunkRows(messageReferenceTable, rows, this.maxBoundParameters).map(
|
|
243
|
+
(slice) => this.db.insert(messageReferenceTable).values(slice),
|
|
235
244
|
),
|
|
236
245
|
]);
|
|
237
246
|
}
|