@remit/drizzle-service 0.0.104 → 0.0.106

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/package.json +1 -1
  2. package/src/bound-parameters.test.ts +57 -0
  3. package/src/bound-parameters.ts +49 -0
  4. package/src/databases.ts +10 -9
  5. package/src/db.ts +1 -0
  6. package/src/index.ts +9 -0
  7. package/src/repair/junk-only-address.sqlite.test.ts +9 -5
  8. package/src/repos/account-config-erase.test.ts +127 -0
  9. package/src/repos/account-config-erase.ts +170 -0
  10. package/src/repos/account.conformance.sqlite.test.ts +9 -5
  11. package/src/repos/calendar-event-index.conformance.d1.test.ts +1 -1
  12. package/src/repos/calendar-event-index.conformance.sqlite.test.ts +3 -1
  13. package/src/repos/calendar-event-index.ts +36 -21
  14. package/src/repos/calendar-store.ts +17 -10
  15. package/src/repos/cascade-delete.test.ts +9 -2
  16. package/src/repos/cascade-delete.ts +2 -1
  17. package/src/repos/envelope.d1.test.ts +66 -0
  18. package/src/repos/envelope.sqlite.test.ts +5 -1
  19. package/src/repos/envelope.test.ts +5 -1
  20. package/src/repos/envelope.ts +40 -31
  21. package/src/repos/i4-account.test.ts +17 -10
  22. package/src/repos/i4-address-junk-move.sqlite.test.ts +9 -5
  23. package/src/repos/i4-mailbox-special-use.sqlite.test.ts +9 -5
  24. package/src/repos/i4-mailbox.sqlite.test.ts +17 -10
  25. package/src/repos/i4-mailbox.test.ts +17 -10
  26. package/src/repos/i4-outbox-attachment.test.ts +9 -5
  27. package/src/repos/message.sqlite.test.ts +5 -1
  28. package/src/repos/message.test.ts +5 -1
  29. package/src/repos/sync-warning.ts +35 -22
  30. package/src/repos/thread-message-body-muted.sqlite.test.ts +9 -5
  31. package/src/repos/thread-message-field-terms.sqlite.test.ts +9 -5
  32. package/src/repos/thread-message.sqlite.test.ts +9 -5
  33. package/src/repos/thread-message.test.ts +33 -20
  34. package/src/search/thread-message-search.sqlite.test.ts +13 -6
  35. package/src/search/thread-message-search.ts +14 -17
  36. package/src/sqlite-repositories.ts +6 -3
  37. package/src/test-d1.ts +36 -19
  38. package/src/test-stores.ts +15 -2
@@ -8,6 +8,7 @@ import type {
8
8
  import { deriveCalendarObjectId } from "@remit/data-ports/id";
9
9
  import { and, eq, exists, sql } from "drizzle-orm";
10
10
  import type { SqliteBatchItem, WriteBatch } from "../batch.js";
11
+ import { chunkRows } from "../bound-parameters.js";
11
12
  import type { RepositoryDb, SqliteStore } from "../db.js";
12
13
  import {
13
14
  calendarEventIndexTable,
@@ -36,6 +37,7 @@ export class DrizzleCalendarStore implements ICalendarStore {
36
37
 
37
38
  private readonly db: DB;
38
39
  private readonly batch: WriteBatch;
40
+ private readonly maxBoundParameters: number;
39
41
 
40
42
  constructor(
41
43
  store: SqliteStore<Record<string, unknown>>,
@@ -43,10 +45,11 @@ export class DrizzleCalendarStore implements ICalendarStore {
43
45
  ) {
44
46
  this.db = store.db;
45
47
  this.batch = store.batch;
48
+ this.maxBoundParameters = store.maxBoundParameters;
46
49
  this.repos = {
47
50
  calendarCollection: new CalendarCollectionRepo(store.db),
48
51
  calendarObject: new CalendarObjectRepo(store.db),
49
- calendarEventIndex: new CalendarEventIndexRepo(store.db),
52
+ calendarEventIndex: new CalendarEventIndexRepo(store),
50
53
  calendarSuggestion,
51
54
  calendarFeedToken: new CalendarFeedTokenRepo(store.db),
52
55
  };
@@ -138,17 +141,21 @@ export class DrizzleCalendarStore implements ICalendarStore {
138
141
  }),
139
142
  this.clearOccurrences(calendarId, calendarObjectId),
140
143
  ];
141
- if (write.occurrences.length === 0) return items;
144
+ const occurrences = write.occurrences.map((occurrence) => ({
145
+ ...occurrence,
146
+ calendarId,
147
+ calendarObjectId,
148
+ createdAt: now,
149
+ updatedAt: now,
150
+ }));
142
151
  return [
143
152
  ...items,
144
- this.db.insert(calendarEventIndexTable).values(
145
- write.occurrences.map((occurrence) => ({
146
- ...occurrence,
147
- calendarId,
148
- calendarObjectId,
149
- createdAt: now,
150
- updatedAt: now,
151
- })),
153
+ ...chunkRows(
154
+ calendarEventIndexTable,
155
+ occurrences,
156
+ this.maxBoundParameters,
157
+ ).map((slice) =>
158
+ this.db.insert(calendarEventIndexTable).values(slice),
152
159
  ),
153
160
  ];
154
161
  }
@@ -2,6 +2,7 @@ import assert from "node:assert";
2
2
  import { after, before, describe, test } from "node:test";
3
3
  import { MailboxCursorState } from "@remit/domain-enums";
4
4
  import { eq } from "drizzle-orm";
5
+ import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
5
6
  import { syncStores } from "../databases.js";
6
7
  import {
7
8
  accountSettingTable,
@@ -300,7 +301,10 @@ describe("runDrizzleCascadeDelete", () => {
300
301
 
301
302
  test("deletes every enumerated row across all tables", async () => {
302
303
  await runDrizzleCascadeDelete(
303
- syncStores({ app: db, mail: db, search: db } as never),
304
+ syncStores(
305
+ { app: db, mail: db, search: db } as never,
306
+ BETTER_SQLITE3_LIMITS,
307
+ ),
304
308
  entities,
305
309
  log,
306
310
  );
@@ -439,7 +443,10 @@ describe("runDrizzleCascadeDelete", () => {
439
443
  await assert.rejects(
440
444
  () =>
441
445
  runDrizzleCascadeDelete(
442
- syncStores({ app: db, mail: db, search: db } as never),
446
+ syncStores(
447
+ { app: db, mail: db, search: db } as never,
448
+ BETTER_SQLITE3_LIMITS,
449
+ ),
443
450
  [{ entityType: "Nonsense", key: { id: "x" } }],
444
451
  log,
445
452
  ),
@@ -1,4 +1,5 @@
1
1
  import type { SqliteBatchItem } from "../batch.js";
2
+ import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
2
3
  import {
3
4
  openSqliteDatabases,
4
5
  type SqliteDatabasePaths,
@@ -241,6 +242,6 @@ export const createSqliteCascadeDeleter = async (
241
242
  paths: SqliteDatabasePaths,
242
243
  ): Promise<CascadeDeleter> => {
243
244
  const databases = await openSqliteDatabases(paths);
244
- const stores = syncStores(databases);
245
+ const stores = syncStores(databases, BETTER_SQLITE3_LIMITS);
245
246
  return (entities, log) => runDrizzleCascadeDelete(stores, entities, log);
246
247
  };
@@ -0,0 +1,66 @@
1
+ import assert from "node:assert/strict";
2
+ import { after, before, describe, test } from "node:test";
3
+ import type { BodyPartUpsertInput } from "@remit/data-ports";
4
+ import { ReferenceType } from "@remit/domain-enums";
5
+ import { openD1TestStore } from "../test-d1.js";
6
+ import { DrizzleEnvelopeRepository } from "./envelope.js";
7
+
8
+ const MESSAGE_ID = "00000000-0000-0000-4444-000000000d01";
9
+
10
+ const bodyPart = (index: number): BodyPartUpsertInput => ({
11
+ partPath: `${index + 1}`,
12
+ parentPartPath: null,
13
+ mediaType: "TEXT",
14
+ mediaSubtype: "plain",
15
+ transferEncoding: "7BIT",
16
+ sizeOctets: 128,
17
+ isMultipart: false,
18
+ parameters: [{ parameterName: "charset", parameterValue: "utf-8" }],
19
+ });
20
+
21
+ describe("DrizzleEnvelopeRepository on D1's 100 bound parameter limit", () => {
22
+ let repo: DrizzleEnvelopeRepository;
23
+ let close: () => Promise<void>;
24
+
25
+ before(async () => {
26
+ const opened = await openD1TestStore("mail");
27
+ close = opened.close;
28
+ repo = new DrizzleEnvelopeRepository(opened.store);
29
+ await repo.upsertEnvelope({
30
+ envelopeId: "ignored-derived",
31
+ messageId: MESSAGE_ID,
32
+ dateValue: 1700000000000,
33
+ dateRaw: "Mon, 14 Nov 2023 08:00:00 +0000",
34
+ subject: "Thirty parts",
35
+ });
36
+ });
37
+
38
+ after(async () => {
39
+ await close();
40
+ });
41
+
42
+ test("upsertBodyParts writes thirty parts and their parameters", async () => {
43
+ await repo.upsertBodyParts(
44
+ MESSAGE_ID,
45
+ Array.from({ length: 30 }, (_, index) => bodyPart(index)),
46
+ );
47
+
48
+ const data = await repo.getMessageData(MESSAGE_ID);
49
+ assert.equal(data.bodyPart.length, 30);
50
+ assert.equal(data.bodyPartParameter.length, 30);
51
+ });
52
+
53
+ test("replaceMessageReferences writes thirty references", async () => {
54
+ await repo.replaceMessageReferences(
55
+ MESSAGE_ID,
56
+ Array.from({ length: 30 }, (_, index) => ({
57
+ messageIdValue: `<ref-${index}@example.com>`,
58
+ referenceType: ReferenceType.References,
59
+ referenceOrder: index,
60
+ })),
61
+ );
62
+
63
+ const data = await repo.getMessageData(MESSAGE_ID);
64
+ assert.equal(data.messageReference.length, 30);
65
+ });
66
+ });
@@ -1,5 +1,6 @@
1
1
  import assert from "node:assert/strict";
2
2
  import { after, before, describe, test } from "node:test";
3
+ import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
3
4
  import {
4
5
  type MessageDataSchema,
5
6
  messageDataSchema,
@@ -20,7 +21,10 @@ describe("DrizzleEnvelopeRepository (sqlite)", () => {
20
21
 
21
22
  before(async () => {
22
23
  ({ db, close } = await createSqliteTestDb(messageDataSchema));
23
- repo = createSqliteRepositories({ app: db, mail: db, search: db }).envelope;
24
+ repo = createSqliteRepositories(
25
+ { app: db, mail: db, search: db },
26
+ BETTER_SQLITE3_LIMITS,
27
+ ).envelope;
24
28
  });
25
29
 
26
30
  after(async () => {
@@ -1,6 +1,7 @@
1
1
  import assert from "node:assert/strict";
2
2
  import { after, before, describe, test } from "node:test";
3
3
  import type { BodyPartUpsertInput } from "@remit/data-ports";
4
+ import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
4
5
  import { envelopeId as deriveEnvelopeId } from "../id.js";
5
6
  import { createSqliteRepositories } from "../sqlite-repositories.js";
6
7
  import type { DrizzleEnvelopeRepository } from "./envelope.js";
@@ -19,7 +20,10 @@ describe("DrizzleEnvelopeRepository", () => {
19
20
 
20
21
  before(async () => {
21
22
  ({ db, stop } = await createTestDb());
22
- repo = createSqliteRepositories({ app: db, mail: db, search: db }).envelope;
23
+ repo = createSqliteRepositories(
24
+ { app: db, mail: db, search: db },
25
+ BETTER_SQLITE3_LIMITS,
26
+ ).envelope;
23
27
 
24
28
  await repo.upsertEnvelope({
25
29
  envelopeId: ENVELOPE_ID,
@@ -9,7 +9,8 @@ import type {
9
9
  UpdateEnvelopeInput,
10
10
  } from "@remit/data-ports";
11
11
  import { eq } from "drizzle-orm";
12
- import type { WriteBatch } from "../batch.js";
12
+ import type { SqliteBatchItem, WriteBatch } from "../batch.js";
13
+ import { chunkRows } from "../bound-parameters.js";
13
14
  import type { RepositoryDb, SqliteStore } from "../db.js";
14
15
  import {
15
16
  CreateFailedConflictError,
@@ -57,10 +58,12 @@ function dedupeById<T>(rows: T[], getId: (row: T) => string): T[] {
57
58
  export class DrizzleEnvelopeRepository implements IEnvelopeRepository {
58
59
  private readonly db: DB;
59
60
  private readonly batch: WriteBatch;
61
+ private readonly maxBoundParameters: number;
60
62
 
61
63
  constructor(store: SqliteStore<Record<string, unknown>>) {
62
64
  this.db = store.db;
63
65
  this.batch = store.batch;
66
+ this.maxBoundParameters = store.maxBoundParameters;
64
67
  }
65
68
 
66
69
  async getMessageData(messageId: string): Promise<MessageData> {
@@ -180,22 +183,30 @@ export class DrizzleEnvelopeRepository implements IEnvelopeRepository {
180
183
  (r) => r.bodyPartParameterId,
181
184
  );
182
185
 
183
- const insertParts = this.db
184
- .insert(bodyPartTable)
185
- .values(dedupedBodyPartRows)
186
- .onConflictDoUpdate({
187
- target: bodyPartTable.bodyPartId,
188
- set: { updatedAt: now },
189
- });
190
- if (dedupedParameterRows.length === 0) {
191
- await this.batch([insertParts]);
192
- return;
193
- }
194
- await this.batch([
195
- insertParts,
186
+ const upsertUpdatedAt = 1;
187
+ const insertParts: SqliteBatchItem[] = chunkRows(
188
+ bodyPartTable,
189
+ dedupedBodyPartRows,
190
+ this.maxBoundParameters,
191
+ upsertUpdatedAt,
192
+ ).map((slice) =>
193
+ this.db
194
+ .insert(bodyPartTable)
195
+ .values(slice)
196
+ .onConflictDoUpdate({
197
+ target: bodyPartTable.bodyPartId,
198
+ set: { updatedAt: now },
199
+ }),
200
+ );
201
+ const insertParameters: SqliteBatchItem[] = chunkRows(
202
+ bodyPartParameterTable,
203
+ dedupedParameterRows,
204
+ this.maxBoundParameters,
205
+ upsertUpdatedAt,
206
+ ).map((slice) =>
196
207
  this.db
197
208
  .insert(bodyPartParameterTable)
198
- .values(dedupedParameterRows)
209
+ .values(slice)
199
210
  .onConflictDoUpdate({
200
211
  target: bodyPartParameterTable.bodyPartParameterId,
201
212
  set: {
@@ -203,7 +214,8 @@ export class DrizzleEnvelopeRepository implements IEnvelopeRepository {
203
214
  updatedAt: now,
204
215
  },
205
216
  }),
206
- ]);
217
+ );
218
+ await this.batch([...insertParts, ...insertParameters]);
207
219
  }
208
220
 
209
221
  async replaceMessageReferences(
@@ -215,23 +227,20 @@ export class DrizzleEnvelopeRepository implements IEnvelopeRepository {
215
227
  const clear = this.db
216
228
  .delete(messageReferenceTable)
217
229
  .where(eq(messageReferenceTable.messageId, messageId));
218
- if (references.length === 0) {
219
- await this.batch([clear]);
220
- return;
221
- }
230
+ const rows = references.map((reference) => ({
231
+ ...reference,
232
+ messageReferenceId: deterministicBase36Id(
233
+ `msgref:${messageId}:${reference.referenceType}:${reference.referenceOrder}`,
234
+ ),
235
+ messageId,
236
+ envelopeId,
237
+ createdAt: now,
238
+ updatedAt: now,
239
+ }));
222
240
  await this.batch([
223
241
  clear,
224
- this.db.insert(messageReferenceTable).values(
225
- references.map((reference) => ({
226
- ...reference,
227
- messageReferenceId: deterministicBase36Id(
228
- `msgref:${messageId}:${reference.referenceType}:${reference.referenceOrder}`,
229
- ),
230
- messageId,
231
- envelopeId,
232
- createdAt: now,
233
- updatedAt: now,
234
- })),
242
+ ...chunkRows(messageReferenceTable, rows, this.maxBoundParameters).map(
243
+ (slice) => this.db.insert(messageReferenceTable).values(slice),
235
244
  ),
236
245
  ]);
237
246
  }
@@ -1,5 +1,6 @@
1
1
  import assert from "node:assert";
2
2
  import { after, before, describe, test } from "node:test";
3
+ import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
3
4
  import { createSqliteRepositories } from "../sqlite-repositories.js";
4
5
  import { createTestDb, randomId, type TestDb } from "../test-db.js";
5
6
  import type { AccountRepo } from "./i4-account.js";
@@ -26,11 +27,14 @@ describe("AccountRepo", () => {
26
27
 
27
28
  before(async () => {
28
29
  ({ db, close } = await createTestDb());
29
- repo = createSqliteRepositories({
30
- app: db,
31
- mail: db,
32
- search: db,
33
- }).account;
30
+ repo = createSqliteRepositories(
31
+ {
32
+ app: db,
33
+ mail: db,
34
+ search: db,
35
+ },
36
+ BETTER_SQLITE3_LIMITS,
37
+ ).account;
34
38
  });
35
39
 
36
40
  after(async () => {
@@ -164,11 +168,14 @@ describe("AccountRepo", () => {
164
168
  const accountConfigId = randomId();
165
169
  const account = await repo.create(makeAccountInput(accountConfigId));
166
170
 
167
- const mailboxRepo = createSqliteRepositories({
168
- app: db,
169
- mail: db,
170
- search: db,
171
- }).mailbox;
171
+ const mailboxRepo = createSqliteRepositories(
172
+ {
173
+ app: db,
174
+ mail: db,
175
+ search: db,
176
+ },
177
+ BETTER_SQLITE3_LIMITS,
178
+ ).mailbox;
172
179
  mailboxCreated(
173
180
  await mailboxRepo.create({
174
181
  accountId: account.accountId,
@@ -1,6 +1,7 @@
1
1
  import assert from "node:assert/strict";
2
2
  import { after, before, beforeEach, describe, test } from "node:test";
3
3
  import type Database from "better-sqlite3";
4
+ import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
4
5
  import { createSqliteRepositories } from "../sqlite-repositories.js";
5
6
  import { createTestDb, type TestDb } from "../test-db.js";
6
7
  import { AddressRepo } from "./i4-address.js";
@@ -125,11 +126,14 @@ describe("reconciling one message's addresses at the moment it moves", () => {
125
126
  before(async () => {
126
127
  ({ db, sqlite, close } = await createTestDb());
127
128
  repo = new AddressRepo(db as never);
128
- specialUse = createSqliteRepositories({
129
- app: db,
130
- mail: db,
131
- search: db,
132
- }).mailboxSpecialUse;
129
+ specialUse = createSqliteRepositories(
130
+ {
131
+ app: db,
132
+ mail: db,
133
+ search: db,
134
+ },
135
+ BETTER_SQLITE3_LIMITS,
136
+ ).mailboxSpecialUse;
133
137
  });
134
138
 
135
139
  /**
@@ -6,6 +6,7 @@ import {
6
6
  meetsTrashAssurance,
7
7
  } from "@remit/data-ports/folder-role";
8
8
  import { CanonicalMailboxRole } from "@remit/domain-enums";
9
+ import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
9
10
  import {
10
11
  accountSettingTable,
11
12
  accountTable,
@@ -57,11 +58,14 @@ describe("MailboxSpecialUseRepo role lookups (sqlite)", () => {
57
58
  });
58
59
  close = testDb.close;
59
60
  db = testDb.db;
60
- const repos = createSqliteRepositories({
61
- app: testDb.db,
62
- mail: testDb.db,
63
- search: testDb.db,
64
- });
61
+ const repos = createSqliteRepositories(
62
+ {
63
+ app: testDb.db,
64
+ mail: testDb.db,
65
+ search: testDb.db,
66
+ },
67
+ BETTER_SQLITE3_LIMITS,
68
+ );
65
69
  repo = repos.mailboxSpecialUse;
66
70
  mailboxes = repos.mailbox;
67
71
  accountSettings = repos.accountSetting;
@@ -4,6 +4,7 @@ import { after, before, describe, test } from "node:test";
4
4
  import { MailboxSyncStatus } from "@remit/domain-enums";
5
5
  import Database from "better-sqlite3";
6
6
  import { drizzle } from "drizzle-orm/better-sqlite3";
7
+ import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
7
8
  import { mailboxTable } from "../schema.js";
8
9
  import { createSqliteRepositories } from "../sqlite-repositories.js";
9
10
  import { createSqliteTestDb } from "../test-db-sqlite.js";
@@ -41,11 +42,14 @@ describe("MailboxRepo (sqlite)", () => {
41
42
  before(async () => {
42
43
  const testDb = await createSqliteTestDb({ mailbox: mailboxTable });
43
44
  close = testDb.close;
44
- repo = createSqliteRepositories({
45
- app: testDb.db,
46
- mail: testDb.db,
47
- search: testDb.db,
48
- }).mailbox;
45
+ repo = createSqliteRepositories(
46
+ {
47
+ app: testDb.db,
48
+ mail: testDb.db,
49
+ search: testDb.db,
50
+ },
51
+ BETTER_SQLITE3_LIMITS,
52
+ ).mailbox;
49
53
  });
50
54
 
51
55
  after(async () => {
@@ -211,11 +215,14 @@ describe("MailboxRepo (sqlite, shipped migrations)", () => {
211
215
  const sqlite = new Database(":memory:");
212
216
  applyMigrationSet(sqlite, "mail");
213
217
  const db = drizzle(sqlite, { schema: { mailbox: mailboxTable } });
214
- repo = createSqliteRepositories({
215
- app: db,
216
- mail: db,
217
- search: db,
218
- }).mailbox;
218
+ repo = createSqliteRepositories(
219
+ {
220
+ app: db,
221
+ mail: db,
222
+ search: db,
223
+ },
224
+ BETTER_SQLITE3_LIMITS,
225
+ ).mailbox;
219
226
  close = async () => {
220
227
  sqlite.close();
221
228
  };
@@ -3,6 +3,7 @@ import { after, before, describe, test } from "node:test";
3
3
  import { MailboxSyncStatus } from "@remit/domain-enums";
4
4
  import { eq } from "drizzle-orm";
5
5
  import { syncBatch } from "../batch.js";
6
+ import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
6
7
  import {
7
8
  envelopeId as deriveEnvelopeId,
8
9
  rootBodyPartId as deriveRootBodyPartId,
@@ -54,11 +55,14 @@ describe("MailboxRepo", () => {
54
55
 
55
56
  before(async () => {
56
57
  ({ db, close } = await createTestDb());
57
- repo = createSqliteRepositories({
58
- app: db,
59
- mail: db,
60
- search: db,
61
- }).mailbox;
58
+ repo = createSqliteRepositories(
59
+ {
60
+ app: db,
61
+ mail: db,
62
+ search: db,
63
+ },
64
+ BETTER_SQLITE3_LIMITS,
65
+ ).mailbox;
62
66
  });
63
67
 
64
68
  after(async () => {
@@ -543,11 +547,14 @@ describe("MailboxRepo", () => {
543
547
  describe("deleteMailboxWithMail — the folder's mail goes with it (D8)", () => {
544
548
  const seedMessage = async (mailboxId: string, uid: number) => {
545
549
  const messageId = randomId();
546
- await createSqliteRepositories({
547
- app: db,
548
- mail: db,
549
- search: db,
550
- }).message.create({
550
+ await createSqliteRepositories(
551
+ {
552
+ app: db,
553
+ mail: db,
554
+ search: db,
555
+ },
556
+ BETTER_SQLITE3_LIMITS,
557
+ ).message.create({
551
558
  messageId,
552
559
  mailboxId,
553
560
  uid,
@@ -1,5 +1,6 @@
1
1
  import assert from "node:assert";
2
2
  import { after, before, describe, test } from "node:test";
3
+ import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
3
4
  import { createSqliteRepositories } from "../sqlite-repositories.js";
4
5
  import { createTestDb, randomId } from "../test-db.js";
5
6
  import type { OutboxAttachmentRepo } from "./i4-outbox-attachment.js";
@@ -28,11 +29,14 @@ const input = (
28
29
 
29
30
  before(async () => {
30
31
  harness = await createTestDb();
31
- repo = createSqliteRepositories({
32
- app: harness.db,
33
- mail: harness.db,
34
- search: harness.db,
35
- }).outboxAttachment;
32
+ repo = createSqliteRepositories(
33
+ {
34
+ app: harness.db,
35
+ mail: harness.db,
36
+ search: harness.db,
37
+ },
38
+ BETTER_SQLITE3_LIMITS,
39
+ ).outboxAttachment;
36
40
  });
37
41
 
38
42
  after(async () => {
@@ -1,5 +1,6 @@
1
1
  import assert from "node:assert/strict";
2
2
  import { after, before, describe, test } from "node:test";
3
+ import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
3
4
  import {
4
5
  envelopeId as deriveEnvelopeId,
5
6
  rootBodyPartId as deriveRootBodyPartId,
@@ -43,7 +44,10 @@ describe("DrizzleMessageRepository (sqlite)", () => {
43
44
 
44
45
  before(async () => {
45
46
  ({ db, close } = await createSqliteTestDb(SCHEMA));
46
- repo = createSqliteRepositories({ app: db, mail: db, search: db }).message;
47
+ repo = createSqliteRepositories(
48
+ { app: db, mail: db, search: db },
49
+ BETTER_SQLITE3_LIMITS,
50
+ ).message;
47
51
  });
48
52
 
49
53
  after(async () => {
@@ -1,5 +1,6 @@
1
1
  import assert from "node:assert/strict";
2
2
  import { after, before, describe, test } from "node:test";
3
+ import { BETTER_SQLITE3_LIMITS } from "../bound-parameters.js";
3
4
  import {
4
5
  envelopeId as deriveEnvelopeId,
5
6
  rootBodyPartId as deriveRootBodyPartId,
@@ -37,7 +38,10 @@ describe("DrizzleMessageRepository", () => {
37
38
 
38
39
  before(async () => {
39
40
  ({ db, stop } = await createTestDb());
40
- const repos = createSqliteRepositories({ app: db, mail: db, search: db });
41
+ const repos = createSqliteRepositories(
42
+ { app: db, mail: db, search: db },
43
+ BETTER_SQLITE3_LIMITS,
44
+ );
41
45
  messageRepo = repos.message;
42
46
  envelopeRepo = repos.envelope;
43
47
  });
@@ -6,14 +6,24 @@ import type {
6
6
  } from "@remit/data-ports";
7
7
  import { deriveSyncWarningId } from "@remit/data-ports/id";
8
8
  import { and, count, desc, eq, gte, sql } from "drizzle-orm";
9
- import type { RepositoryDb } from "../db.js";
9
+ import type { WriteBatch } from "../batch.js";
10
+ import { chunkRows } from "../bound-parameters.js";
11
+ import type { RepositoryDb, SqliteStore } from "../db.js";
10
12
  import { syncWarningTable } from "../schema/sync-warning.js";
11
13
  import { inList } from "./in-list.js";
12
14
 
13
15
  type DB = RepositoryDb<Record<string, unknown>>;
14
16
 
15
17
  export class SyncWarningRepo implements ISyncWarningRepository {
16
- constructor(private db: DB) {}
18
+ private readonly db: DB;
19
+ private readonly batch: WriteBatch;
20
+ private readonly maxBoundParameters: number;
21
+
22
+ constructor(store: SqliteStore<Record<string, unknown>>) {
23
+ this.db = store.db;
24
+ this.batch = store.batch;
25
+ this.maxBoundParameters = store.maxBoundParameters;
26
+ }
17
27
 
18
28
  listByMailboxId = async (mailboxId: string): Promise<SyncWarningItem[]> =>
19
29
  this.db
@@ -46,26 +56,29 @@ export class SyncWarningRepo implements ISyncWarningRepository {
46
56
  recordFailures = async (keys: SyncWarningKey[]): Promise<void> => {
47
57
  if (keys.length === 0) return;
48
58
  const now = Date.now();
49
- await this.db
50
- .insert(syncWarningTable)
51
- .values(
52
- keys.map((key) => ({
53
- syncWarningId: deriveSyncWarningId(
54
- key.accountId,
55
- key.mailboxId,
56
- key.uidValidity,
57
- key.uid,
58
- key.stage,
59
- ),
60
- ...key,
61
- attempts: 1,
62
- createdAt: now,
63
- })),
64
- )
65
- .onConflictDoUpdate({
66
- target: syncWarningTable.syncWarningId,
67
- set: { attempts: sql`${syncWarningTable.attempts} + 1` },
68
- });
59
+ const rows = keys.map((key) => ({
60
+ syncWarningId: deriveSyncWarningId(
61
+ key.accountId,
62
+ key.mailboxId,
63
+ key.uidValidity,
64
+ key.uid,
65
+ key.stage,
66
+ ),
67
+ ...key,
68
+ attempts: 1,
69
+ createdAt: now,
70
+ }));
71
+ await this.batch(
72
+ chunkRows(syncWarningTable, rows, this.maxBoundParameters).map((slice) =>
73
+ this.db
74
+ .insert(syncWarningTable)
75
+ .values(slice)
76
+ .onConflictDoUpdate({
77
+ target: syncWarningTable.syncWarningId,
78
+ set: { attempts: sql`${syncWarningTable.attempts} + 1` },
79
+ }),
80
+ ),
81
+ );
69
82
  };
70
83
 
71
84
  deleteMany = async (syncWarningIds: string[]): Promise<void> => {