@remit/drizzle-service 0.0.104 → 0.0.105

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. package/package.json +1 -1
  2. package/src/bound-parameters.test.ts +57 -0
  3. package/src/bound-parameters.ts +49 -0
  4. package/src/databases.ts +10 -9
  5. package/src/db.ts +1 -0
  6. package/src/index.ts +4 -0
  7. package/src/repair/junk-only-address.sqlite.test.ts +9 -5
  8. package/src/repos/account.conformance.sqlite.test.ts +9 -5
  9. package/src/repos/calendar-event-index.conformance.d1.test.ts +1 -1
  10. package/src/repos/calendar-event-index.conformance.sqlite.test.ts +3 -1
  11. package/src/repos/calendar-event-index.ts +36 -21
  12. package/src/repos/calendar-store.ts +17 -10
  13. package/src/repos/cascade-delete.test.ts +9 -2
  14. package/src/repos/cascade-delete.ts +2 -1
  15. package/src/repos/envelope.d1.test.ts +66 -0
  16. package/src/repos/envelope.sqlite.test.ts +5 -1
  17. package/src/repos/envelope.test.ts +5 -1
  18. package/src/repos/envelope.ts +40 -31
  19. package/src/repos/i4-account.test.ts +17 -10
  20. package/src/repos/i4-address-junk-move.sqlite.test.ts +9 -5
  21. package/src/repos/i4-mailbox-special-use.sqlite.test.ts +9 -5
  22. package/src/repos/i4-mailbox.sqlite.test.ts +17 -10
  23. package/src/repos/i4-mailbox.test.ts +17 -10
  24. package/src/repos/i4-outbox-attachment.test.ts +9 -5
  25. package/src/repos/message.sqlite.test.ts +5 -1
  26. package/src/repos/message.test.ts +5 -1
  27. package/src/repos/sync-warning.ts +35 -22
  28. package/src/repos/thread-message-body-muted.sqlite.test.ts +9 -5
  29. package/src/repos/thread-message-field-terms.sqlite.test.ts +9 -5
  30. package/src/repos/thread-message.sqlite.test.ts +9 -5
  31. package/src/repos/thread-message.test.ts +33 -20
  32. package/src/search/thread-message-search.sqlite.test.ts +13 -6
  33. package/src/search/thread-message-search.ts +14 -17
  34. package/src/sqlite-repositories.ts +6 -3
  35. package/src/test-d1.ts +36 -19
  36. package/src/test-stores.ts +15 -2
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@remit/drizzle-service",
3
- "version": "0.0.104",
3
+ "version": "0.0.105",
4
4
  "description": "Drizzle ORM service over SQLite",
5
5
  "type": "module",
6
6
  "main": "src/index.ts",
@@ -0,0 +1,57 @@
1
+ import assert from "node:assert/strict";
2
+ import { describe, test } from "node:test";
3
+ import { integer, sqliteTable, text } from "drizzle-orm/sqlite-core";
4
+ import { chunkRows, rowsPerStatement } from "./bound-parameters.js";
5
+
6
+ const tenColumns = sqliteTable("ten_columns", {
7
+ c0: text("c0"),
8
+ c1: text("c1"),
9
+ c2: text("c2"),
10
+ c3: text("c3"),
11
+ c4: text("c4"),
12
+ c5: text("c5"),
13
+ c6: text("c6"),
14
+ c7: text("c7"),
15
+ c8: text("c8"),
16
+ c9: integer("c9"),
17
+ });
18
+
19
+ const rows = (count: number): number[] =>
20
+ Array.from({ length: count }, (_, index) => index);
21
+
22
+ describe("chunkRows", () => {
23
+ test("no rows make no statements", () => {
24
+ assert.deepEqual(chunkRows(tenColumns, [], 100), []);
25
+ });
26
+
27
+ test("rows that fill the limit exactly make one statement", () => {
28
+ const chunks = chunkRows(tenColumns, rows(10), 100);
29
+ assert.deepEqual(
30
+ chunks.map((slice) => slice.length),
31
+ [10],
32
+ );
33
+ });
34
+
35
+ test("one row over the limit makes a second statement", () => {
36
+ const chunks = chunkRows(tenColumns, rows(11), 100);
37
+ assert.deepEqual(
38
+ chunks.map((slice) => slice.length),
39
+ [10, 1],
40
+ );
41
+ assert.deepEqual(chunks.flat(), rows(11));
42
+ });
43
+
44
+ test("reserved parameters shrink every statement", () => {
45
+ const chunks = chunkRows(tenColumns, rows(10), 100, 1);
46
+ assert.deepEqual(
47
+ chunks.map((slice) => slice.length),
48
+ [9, 1],
49
+ );
50
+ });
51
+ });
52
+
53
+ describe("rowsPerStatement", () => {
54
+ test("a row wider than the limit is refused", () => {
55
+ assert.throws(() => rowsPerStatement(100, 101), RangeError);
56
+ });
57
+ });
@@ -0,0 +1,49 @@
1
+ import { getTableColumns, type Table } from "drizzle-orm";
2
+
3
+ export interface SqliteLimits {
4
+ /** The most parameters the host's SQLite binds in one statement: 32766 on better-sqlite3, 100 on D1 and Durable Object storage. */
5
+ maxBoundParameters: number;
6
+ }
7
+
8
+ export const BETTER_SQLITE3_LIMITS: SqliteLimits = {
9
+ maxBoundParameters: 32766,
10
+ };
11
+
12
+ export const chunk = <T>(items: readonly T[], size: number): T[][] => {
13
+ const chunks: T[][] = [];
14
+ for (let start = 0; start < items.length; start += size) {
15
+ chunks.push(items.slice(start, start + size));
16
+ }
17
+ return chunks;
18
+ };
19
+
20
+ export const rowsPerStatement = (
21
+ maxBoundParameters: number,
22
+ parametersPerRow: number,
23
+ reservedParameters = 0,
24
+ ): number => {
25
+ const rows = Math.floor(
26
+ (maxBoundParameters - reservedParameters) / parametersPerRow,
27
+ );
28
+ if (rows < 1) {
29
+ throw new RangeError(
30
+ `a row of ${parametersPerRow} bound parameters does not fit a statement limit of ${maxBoundParameters}`,
31
+ );
32
+ }
33
+ return rows;
34
+ };
35
+
36
+ export const chunkRows = <T>(
37
+ table: Table,
38
+ rows: readonly T[],
39
+ maxBoundParameters: number,
40
+ reservedParameters = 0,
41
+ ): T[][] =>
42
+ chunk(
43
+ rows,
44
+ rowsPerStatement(
45
+ maxBoundParameters,
46
+ Object.keys(getTableColumns(table)).length,
47
+ reservedParameters,
48
+ ),
49
+ );
package/src/databases.ts CHANGED
@@ -1,4 +1,5 @@
1
1
  import { syncBatch } from "./batch.js";
2
+ import type { SqliteLimits } from "./bound-parameters.js";
2
3
  import type { Db, SqliteStore } from "./db.js";
3
4
  import { messageDataSchema } from "./schema/message-data.js";
4
5
  import { createSqliteDatabase } from "./sqlite-client.js";
@@ -22,16 +23,16 @@ export interface SqliteStores {
22
23
 
23
24
  export const syncStore = <TSchema extends Record<string, unknown>>(
24
25
  db: Db<TSchema>,
25
- ): SqliteStore<TSchema> => ({ db, batch: syncBatch(db) });
26
+ { maxBoundParameters }: SqliteLimits,
27
+ ): SqliteStore<TSchema> => ({ db, batch: syncBatch(db), maxBoundParameters });
26
28
 
27
- export const syncStores = ({
28
- app,
29
- mail,
30
- search,
31
- }: SqliteDatabases): SqliteStores => ({
32
- app: syncStore(app),
33
- mail: syncStore(mail),
34
- search: syncStore(search),
29
+ export const syncStores = (
30
+ { app, mail, search }: SqliteDatabases,
31
+ limits: SqliteLimits,
32
+ ): SqliteStores => ({
33
+ app: syncStore(app, limits),
34
+ mail: syncStore(mail, limits),
35
+ search: syncStore(search, limits),
35
36
  });
36
37
 
37
38
  export interface SqliteDatabasePaths {
package/src/db.ts CHANGED
@@ -26,4 +26,5 @@ export type RepositoryDb<TSchema extends Record<string, unknown>> = Omit<
26
26
  export interface SqliteStore<TSchema extends Record<string, unknown>> {
27
27
  db: RepositoryDb<TSchema>;
28
28
  batch: WriteBatch;
29
+ maxBoundParameters: number;
29
30
  }
package/src/index.ts CHANGED
@@ -5,6 +5,10 @@ export {
5
5
  syncBatch,
6
6
  type WriteBatch,
7
7
  } from "./batch.js";
8
+ export {
9
+ BETTER_SQLITE3_LIMITS,
10
+ type SqliteLimits,
11
+ } from "./bound-parameters.js";
8
12
  export {
9
13
  hasSqliteDatabaseEnv,
10
14
  openSqliteDatabases,
@@ -4,6 +4,7 @@ import { composeFolderRoleAppointmentName } from "@remit/data-ports/folder-role"
4
4
  import { CanonicalMailboxRole } from "@remit/domain-enums";
5
5
  import Database from "better-sqlite3";
6
6
  import { drizzle } from "drizzle-orm/better-sqlite3";
7
+ import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
7
8
  import { AccountSettingRepo } from "../repos/i4-account-setting.js";
8
9
  import type { MailboxSpecialUseRepo } from "../repos/i4-mailbox-special-use.js";
9
10
  import { createSqliteRepositories } from "../sqlite-repositories.js";
@@ -182,11 +183,14 @@ describe("addresses standing only on mail in Junk", () => {
182
183
  before(() => {
183
184
  sqlite = new Database(":memory:");
184
185
  const db = drizzle(sqlite);
185
- specialUse = createSqliteRepositories({
186
- app: db,
187
- mail: db,
188
- search: db,
189
- }).mailboxSpecialUse;
186
+ specialUse = createSqliteRepositories(
187
+ {
188
+ app: db,
189
+ mail: db,
190
+ search: db,
191
+ },
192
+ BETTER_SQLITE3_LIMITS,
193
+ ).mailboxSpecialUse;
190
194
  accountSetting = new AccountSettingRepo(db as never);
191
195
  applyMigrationSet(sqlite, "app");
192
196
  applyMigrationSet(sqlite, "mail");
@@ -1,4 +1,5 @@
1
1
  import { accountRepositoryConformance } from "@remit/data-ports/conformance";
2
+ import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
2
3
  import { NotFoundError } from "../error.js";
3
4
  import { randomId } from "../id.js";
4
5
  import { accountTable } from "../schema.js";
@@ -13,11 +14,14 @@ accountRepositoryConformance({
13
14
  accounts: accountTable,
14
15
  });
15
16
  close = closeDb;
16
- return createSqliteRepositories({
17
- app: db,
18
- mail: db,
19
- search: db,
20
- }).account;
17
+ return createSqliteRepositories(
18
+ {
19
+ app: db,
20
+ mail: db,
21
+ search: db,
22
+ },
23
+ BETTER_SQLITE3_LIMITS,
24
+ ).account;
21
25
  },
22
26
  teardown: async () => {
23
27
  await close?.();
@@ -10,7 +10,7 @@ calendarEventIndexRepositoryConformance({
10
10
  async createRepository() {
11
11
  const opened = await openD1TestStores();
12
12
  close = opened.close;
13
- return new CalendarEventIndexRepo(opened.stores.app.db);
13
+ return new CalendarEventIndexRepo(opened.stores.app);
14
14
  },
15
15
  teardown: async () => {
16
16
  await close?.();
@@ -1,4 +1,6 @@
1
1
  import { calendarEventIndexRepositoryConformance } from "@remit/data-ports/conformance";
2
+ import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
3
+ import { syncStore } from "../databases.js";
2
4
  import { NotFoundError } from "../error.js";
3
5
  import { randomId } from "../id.js";
4
6
  import { calendarEventIndexTable } from "../schema.js";
@@ -13,7 +15,7 @@ calendarEventIndexRepositoryConformance({
13
15
  calendarEventIndexes: calendarEventIndexTable,
14
16
  });
15
17
  close = closeDb;
16
- return new CalendarEventIndexRepo(db as never);
18
+ return new CalendarEventIndexRepo(syncStore(db, BETTER_SQLITE3_LIMITS));
17
19
  },
18
20
  teardown: async () => {
19
21
  await close?.();
@@ -4,7 +4,9 @@ import type {
4
4
  ICalendarEventIndexRepository,
5
5
  } from "@remit/data-ports";
6
6
  import { and, asc, eq, gte, lt } from "drizzle-orm";
7
- import type { RepositoryDb } from "../db.js";
7
+ import type { SqliteBatchItem, WriteBatch } from "../batch.js";
8
+ import { chunkRows } from "../bound-parameters.js";
9
+ import type { RepositoryDb, SqliteStore } from "../db.js";
8
10
  import { calendarEventIndexTable } from "../schema.js";
9
11
 
10
12
  type DB = RepositoryDb<Record<string, unknown>>;
@@ -28,39 +30,52 @@ function rowToOccurrence(
28
30
  }
29
31
 
30
32
  export class CalendarEventIndexRepo implements ICalendarEventIndexRepository {
31
- constructor(private db: DB) {}
33
+ private readonly db: DB;
34
+ private readonly batch: WriteBatch;
35
+ private readonly maxBoundParameters: number;
36
+
37
+ constructor(store: SqliteStore<Record<string, unknown>>) {
38
+ this.db = store.db;
39
+ this.batch = store.batch;
40
+ this.maxBoundParameters = store.maxBoundParameters;
41
+ }
32
42
 
33
43
  async replaceForObject(
34
44
  calendarId: string,
35
45
  calendarObjectId: string,
36
46
  occurrences: CalendarOccurrenceInput[],
37
47
  ): Promise<void> {
38
- await this.deleteForObject(calendarId, calendarObjectId);
39
- if (occurrences.length === 0) return;
40
-
41
48
  const now = Date.now();
42
- await this.db.insert(calendarEventIndexTable).values(
43
- occurrences.map((occurrence) => ({
44
- calendarId,
45
- calendarObjectId,
46
- recurrenceId: occurrence.recurrenceId,
47
- startAt: occurrence.startAt,
48
- endAt: occurrence.endAt,
49
- allDay: occurrence.allDay,
50
- summary: occurrence.summary,
51
- status: occurrence.status,
52
- transparency: occurrence.transparency,
53
- createdAt: now,
54
- updatedAt: now,
55
- })),
56
- );
49
+ const rows = occurrences.map((occurrence) => ({
50
+ calendarId,
51
+ calendarObjectId,
52
+ recurrenceId: occurrence.recurrenceId,
53
+ startAt: occurrence.startAt,
54
+ endAt: occurrence.endAt,
55
+ allDay: occurrence.allDay,
56
+ summary: occurrence.summary,
57
+ status: occurrence.status,
58
+ transparency: occurrence.transparency,
59
+ createdAt: now,
60
+ updatedAt: now,
61
+ }));
62
+ const inserts: SqliteBatchItem[] = chunkRows(
63
+ calendarEventIndexTable,
64
+ rows,
65
+ this.maxBoundParameters,
66
+ ).map((slice) => this.db.insert(calendarEventIndexTable).values(slice));
67
+ await this.batch([this.clear(calendarId, calendarObjectId), ...inserts]);
57
68
  }
58
69
 
59
70
  async deleteForObject(
60
71
  calendarId: string,
61
72
  calendarObjectId: string,
62
73
  ): Promise<void> {
63
- await this.db
74
+ await this.clear(calendarId, calendarObjectId);
75
+ }
76
+
77
+ private clear(calendarId: string, calendarObjectId: string) {
78
+ return this.db
64
79
  .delete(calendarEventIndexTable)
65
80
  .where(
66
81
  and(
@@ -8,6 +8,7 @@ import type {
8
8
  import { deriveCalendarObjectId } from "@remit/data-ports/id";
9
9
  import { and, eq, exists, sql } from "drizzle-orm";
10
10
  import type { SqliteBatchItem, WriteBatch } from "../batch.js";
11
+ import { chunkRows } from "../bound-parameters.js";
11
12
  import type { RepositoryDb, SqliteStore } from "../db.js";
12
13
  import {
13
14
  calendarEventIndexTable,
@@ -36,6 +37,7 @@ export class DrizzleCalendarStore implements ICalendarStore {
36
37
 
37
38
  private readonly db: DB;
38
39
  private readonly batch: WriteBatch;
40
+ private readonly maxBoundParameters: number;
39
41
 
40
42
  constructor(
41
43
  store: SqliteStore<Record<string, unknown>>,
@@ -43,10 +45,11 @@ export class DrizzleCalendarStore implements ICalendarStore {
43
45
  ) {
44
46
  this.db = store.db;
45
47
  this.batch = store.batch;
48
+ this.maxBoundParameters = store.maxBoundParameters;
46
49
  this.repos = {
47
50
  calendarCollection: new CalendarCollectionRepo(store.db),
48
51
  calendarObject: new CalendarObjectRepo(store.db),
49
- calendarEventIndex: new CalendarEventIndexRepo(store.db),
52
+ calendarEventIndex: new CalendarEventIndexRepo(store),
50
53
  calendarSuggestion,
51
54
  calendarFeedToken: new CalendarFeedTokenRepo(store.db),
52
55
  };
@@ -138,17 +141,21 @@ export class DrizzleCalendarStore implements ICalendarStore {
138
141
  }),
139
142
  this.clearOccurrences(calendarId, calendarObjectId),
140
143
  ];
141
- if (write.occurrences.length === 0) return items;
144
+ const occurrences = write.occurrences.map((occurrence) => ({
145
+ ...occurrence,
146
+ calendarId,
147
+ calendarObjectId,
148
+ createdAt: now,
149
+ updatedAt: now,
150
+ }));
142
151
  return [
143
152
  ...items,
144
- this.db.insert(calendarEventIndexTable).values(
145
- write.occurrences.map((occurrence) => ({
146
- ...occurrence,
147
- calendarId,
148
- calendarObjectId,
149
- createdAt: now,
150
- updatedAt: now,
151
- })),
153
+ ...chunkRows(
154
+ calendarEventIndexTable,
155
+ occurrences,
156
+ this.maxBoundParameters,
157
+ ).map((slice) =>
158
+ this.db.insert(calendarEventIndexTable).values(slice),
152
159
  ),
153
160
  ];
154
161
  }
@@ -2,6 +2,7 @@ import assert from "node:assert";
2
2
  import { after, before, describe, test } from "node:test";
3
3
  import { MailboxCursorState } from "@remit/domain-enums";
4
4
  import { eq } from "drizzle-orm";
5
+ import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
5
6
  import { syncStores } from "../databases.js";
6
7
  import {
7
8
  accountSettingTable,
@@ -300,7 +301,10 @@ describe("runDrizzleCascadeDelete", () => {
300
301
 
301
302
  test("deletes every enumerated row across all tables", async () => {
302
303
  await runDrizzleCascadeDelete(
303
- syncStores({ app: db, mail: db, search: db } as never),
304
+ syncStores(
305
+ { app: db, mail: db, search: db } as never,
306
+ BETTER_SQLITE3_LIMITS,
307
+ ),
304
308
  entities,
305
309
  log,
306
310
  );
@@ -439,7 +443,10 @@ describe("runDrizzleCascadeDelete", () => {
439
443
  await assert.rejects(
440
444
  () =>
441
445
  runDrizzleCascadeDelete(
442
- syncStores({ app: db, mail: db, search: db } as never),
446
+ syncStores(
447
+ { app: db, mail: db, search: db } as never,
448
+ BETTER_SQLITE3_LIMITS,
449
+ ),
443
450
  [{ entityType: "Nonsense", key: { id: "x" } }],
444
451
  log,
445
452
  ),
@@ -1,4 +1,5 @@
1
1
  import type { SqliteBatchItem } from "../batch.js";
2
+ import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
2
3
  import {
3
4
  openSqliteDatabases,
4
5
  type SqliteDatabasePaths,
@@ -241,6 +242,6 @@ export const createSqliteCascadeDeleter = async (
241
242
  paths: SqliteDatabasePaths,
242
243
  ): Promise<CascadeDeleter> => {
243
244
  const databases = await openSqliteDatabases(paths);
244
- const stores = syncStores(databases);
245
+ const stores = syncStores(databases, BETTER_SQLITE3_LIMITS);
245
246
  return (entities, log) => runDrizzleCascadeDelete(stores, entities, log);
246
247
  };
@@ -0,0 +1,66 @@
1
+ import assert from "node:assert/strict";
2
+ import { after, before, describe, test } from "node:test";
3
+ import type { BodyPartUpsertInput } from "@remit/data-ports";
4
+ import { ReferenceType } from "@remit/domain-enums";
5
+ import { openD1TestStore } from "../test-d1.js";
6
+ import { DrizzleEnvelopeRepository } from "./envelope.js";
7
+
8
+ const MESSAGE_ID = "00000000-0000-0000-4444-000000000d01";
9
+
10
+ const bodyPart = (index: number): BodyPartUpsertInput => ({
11
+ partPath: `${index + 1}`,
12
+ parentPartPath: null,
13
+ mediaType: "TEXT",
14
+ mediaSubtype: "plain",
15
+ transferEncoding: "7BIT",
16
+ sizeOctets: 128,
17
+ isMultipart: false,
18
+ parameters: [{ parameterName: "charset", parameterValue: "utf-8" }],
19
+ });
20
+
21
+ describe("DrizzleEnvelopeRepository on D1's 100 bound parameter limit", () => {
22
+ let repo: DrizzleEnvelopeRepository;
23
+ let close: () => Promise<void>;
24
+
25
+ before(async () => {
26
+ const opened = await openD1TestStore("mail");
27
+ close = opened.close;
28
+ repo = new DrizzleEnvelopeRepository(opened.store);
29
+ await repo.upsertEnvelope({
30
+ envelopeId: "ignored-derived",
31
+ messageId: MESSAGE_ID,
32
+ dateValue: 1700000000000,
33
+ dateRaw: "Mon, 14 Nov 2023 08:00:00 +0000",
34
+ subject: "Thirty parts",
35
+ });
36
+ });
37
+
38
+ after(async () => {
39
+ await close();
40
+ });
41
+
42
+ test("upsertBodyParts writes thirty parts and their parameters", async () => {
43
+ await repo.upsertBodyParts(
44
+ MESSAGE_ID,
45
+ Array.from({ length: 30 }, (_, index) => bodyPart(index)),
46
+ );
47
+
48
+ const data = await repo.getMessageData(MESSAGE_ID);
49
+ assert.equal(data.bodyPart.length, 30);
50
+ assert.equal(data.bodyPartParameter.length, 30);
51
+ });
52
+
53
+ test("replaceMessageReferences writes thirty references", async () => {
54
+ await repo.replaceMessageReferences(
55
+ MESSAGE_ID,
56
+ Array.from({ length: 30 }, (_, index) => ({
57
+ messageIdValue: `<ref-${index}@example.com>`,
58
+ referenceType: ReferenceType.References,
59
+ referenceOrder: index,
60
+ })),
61
+ );
62
+
63
+ const data = await repo.getMessageData(MESSAGE_ID);
64
+ assert.equal(data.messageReference.length, 30);
65
+ });
66
+ });
@@ -1,5 +1,6 @@
1
1
  import assert from "node:assert/strict";
2
2
  import { after, before, describe, test } from "node:test";
3
+ import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
3
4
  import {
4
5
  type MessageDataSchema,
5
6
  messageDataSchema,
@@ -20,7 +21,10 @@ describe("DrizzleEnvelopeRepository (sqlite)", () => {
20
21
 
21
22
  before(async () => {
22
23
  ({ db, close } = await createSqliteTestDb(messageDataSchema));
23
- repo = createSqliteRepositories({ app: db, mail: db, search: db }).envelope;
24
+ repo = createSqliteRepositories(
25
+ { app: db, mail: db, search: db },
26
+ BETTER_SQLITE3_LIMITS,
27
+ ).envelope;
24
28
  });
25
29
 
26
30
  after(async () => {
@@ -1,6 +1,7 @@
1
1
  import assert from "node:assert/strict";
2
2
  import { after, before, describe, test } from "node:test";
3
3
  import type { BodyPartUpsertInput } from "@remit/data-ports";
4
+ import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
4
5
  import { envelopeId as deriveEnvelopeId } from "../id.js";
5
6
  import { createSqliteRepositories } from "../sqlite-repositories.js";
6
7
  import type { DrizzleEnvelopeRepository } from "./envelope.js";
@@ -19,7 +20,10 @@ describe("DrizzleEnvelopeRepository", () => {
19
20
 
20
21
  before(async () => {
21
22
  ({ db, stop } = await createTestDb());
22
- repo = createSqliteRepositories({ app: db, mail: db, search: db }).envelope;
23
+ repo = createSqliteRepositories(
24
+ { app: db, mail: db, search: db },
25
+ BETTER_SQLITE3_LIMITS,
26
+ ).envelope;
23
27
 
24
28
  await repo.upsertEnvelope({
25
29
  envelopeId: ENVELOPE_ID,
@@ -9,7 +9,8 @@ import type {
9
9
  UpdateEnvelopeInput,
10
10
  } from "@remit/data-ports";
11
11
  import { eq } from "drizzle-orm";
12
- import type { WriteBatch } from "../batch.js";
12
+ import type { SqliteBatchItem, WriteBatch } from "../batch.js";
13
+ import { chunkRows } from "../bound-parameters.js";
13
14
  import type { RepositoryDb, SqliteStore } from "../db.js";
14
15
  import {
15
16
  CreateFailedConflictError,
@@ -57,10 +58,12 @@ function dedupeById<T>(rows: T[], getId: (row: T) => string): T[] {
57
58
  export class DrizzleEnvelopeRepository implements IEnvelopeRepository {
58
59
  private readonly db: DB;
59
60
  private readonly batch: WriteBatch;
61
+ private readonly maxBoundParameters: number;
60
62
 
61
63
  constructor(store: SqliteStore<Record<string, unknown>>) {
62
64
  this.db = store.db;
63
65
  this.batch = store.batch;
66
+ this.maxBoundParameters = store.maxBoundParameters;
64
67
  }
65
68
 
66
69
  async getMessageData(messageId: string): Promise<MessageData> {
@@ -180,22 +183,30 @@ export class DrizzleEnvelopeRepository implements IEnvelopeRepository {
180
183
  (r) => r.bodyPartParameterId,
181
184
  );
182
185
 
183
- const insertParts = this.db
184
- .insert(bodyPartTable)
185
- .values(dedupedBodyPartRows)
186
- .onConflictDoUpdate({
187
- target: bodyPartTable.bodyPartId,
188
- set: { updatedAt: now },
189
- });
190
- if (dedupedParameterRows.length === 0) {
191
- await this.batch([insertParts]);
192
- return;
193
- }
194
- await this.batch([
195
- insertParts,
186
+ const upsertUpdatedAt = 1;
187
+ const insertParts: SqliteBatchItem[] = chunkRows(
188
+ bodyPartTable,
189
+ dedupedBodyPartRows,
190
+ this.maxBoundParameters,
191
+ upsertUpdatedAt,
192
+ ).map((slice) =>
193
+ this.db
194
+ .insert(bodyPartTable)
195
+ .values(slice)
196
+ .onConflictDoUpdate({
197
+ target: bodyPartTable.bodyPartId,
198
+ set: { updatedAt: now },
199
+ }),
200
+ );
201
+ const insertParameters: SqliteBatchItem[] = chunkRows(
202
+ bodyPartParameterTable,
203
+ dedupedParameterRows,
204
+ this.maxBoundParameters,
205
+ upsertUpdatedAt,
206
+ ).map((slice) =>
196
207
  this.db
197
208
  .insert(bodyPartParameterTable)
198
- .values(dedupedParameterRows)
209
+ .values(slice)
199
210
  .onConflictDoUpdate({
200
211
  target: bodyPartParameterTable.bodyPartParameterId,
201
212
  set: {
@@ -203,7 +214,8 @@ export class DrizzleEnvelopeRepository implements IEnvelopeRepository {
203
214
  updatedAt: now,
204
215
  },
205
216
  }),
206
- ]);
217
+ );
218
+ await this.batch([...insertParts, ...insertParameters]);
207
219
  }
208
220
 
209
221
  async replaceMessageReferences(
@@ -215,23 +227,20 @@ export class DrizzleEnvelopeRepository implements IEnvelopeRepository {
215
227
  const clear = this.db
216
228
  .delete(messageReferenceTable)
217
229
  .where(eq(messageReferenceTable.messageId, messageId));
218
- if (references.length === 0) {
219
- await this.batch([clear]);
220
- return;
221
- }
230
+ const rows = references.map((reference) => ({
231
+ ...reference,
232
+ messageReferenceId: deterministicBase36Id(
233
+ `msgref:${messageId}:${reference.referenceType}:${reference.referenceOrder}`,
234
+ ),
235
+ messageId,
236
+ envelopeId,
237
+ createdAt: now,
238
+ updatedAt: now,
239
+ }));
222
240
  await this.batch([
223
241
  clear,
224
- this.db.insert(messageReferenceTable).values(
225
- references.map((reference) => ({
226
- ...reference,
227
- messageReferenceId: deterministicBase36Id(
228
- `msgref:${messageId}:${reference.referenceType}:${reference.referenceOrder}`,
229
- ),
230
- messageId,
231
- envelopeId,
232
- createdAt: now,
233
- updatedAt: now,
234
- })),
242
+ ...chunkRows(messageReferenceTable, rows, this.maxBoundParameters).map(
243
+ (slice) => this.db.insert(messageReferenceTable).values(slice),
235
244
  ),
236
245
  ]);
237
246
  }