@remit/drizzle-service 0.0.99 → 0.0.101
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/batch.sqlite.test.ts +78 -0
- package/src/batch.ts +46 -0
- package/src/databases.ts +70 -0
- package/src/db.ts +10 -9
- package/src/generated-enum.test.ts +8 -0
- package/src/generated-id.test.ts +19 -0
- package/src/id.test.ts +0 -8
- package/src/id.ts +0 -6
- package/src/index.ts +16 -6
- package/src/repair/address-display-name.sqlite.test.ts +3 -5
- package/src/repair/address-display-name.ts +8 -3
- package/src/repair/junk-only-address.sqlite.test.ts +8 -36
- package/src/repair/stranded-sent-outbox.sqlite.test.ts +2 -9
- package/src/repos/account.conformance.sqlite.test.ts +1 -1
- package/src/repos/calendar-default.sqlite.test.ts +6 -6
- package/src/repos/calendar-put.sqlite.test.ts +40 -39
- package/src/repos/calendar-store.ts +273 -0
- package/src/repos/cascade-delete.sqlite.test.ts +7 -9
- package/src/repos/cascade-delete.test.ts +6 -13
- package/src/repos/cascade-delete.ts +127 -97
- package/src/repos/envelope.test.ts +1 -17
- package/src/repos/envelope.ts +83 -67
- package/src/repos/filter.test.ts +22 -2
- package/src/repos/filter.ts +1 -5
- package/src/repos/i4-account-config.test.ts +2 -2
- package/src/repos/i4-account-config.ts +5 -2
- package/src/repos/i4-account.test.ts +6 -2
- package/src/repos/i4-account.ts +12 -56
- package/src/repos/i4-address-junk-move.sqlite.test.ts +1 -1
- package/src/repos/i4-address-merge-flags.sqlite.test.ts +1 -2
- package/src/repos/i4-address.ts +58 -90
- package/src/repos/i4-mailbox-special-use.sqlite.test.ts +6 -2
- package/src/repos/i4-mailbox-special-use.ts +9 -6
- package/src/repos/i4-mailbox.sqlite.test.ts +12 -14
- package/src/repos/i4-mailbox.test.ts +17 -62
- package/src/repos/i4-mailbox.ts +121 -94
- package/src/repos/i4-organize-job-request.ts +0 -4
- package/src/repos/i4-outbox-attachment.ts +62 -33
- package/src/repos/mappers.ts +0 -16
- package/src/repos/message.sqlite.test.ts +32 -106
- package/src/repos/message.test.ts +6 -167
- package/src/repos/message.ts +216 -228
- package/src/repos/sync-warning.ts +76 -0
- package/src/repos/thread-message-body-muted.sqlite.test.ts +5 -10
- package/src/repos/thread-message-field-terms.sqlite.test.ts +5 -1
- package/src/repos/thread-message.sqlite.test.ts +2 -1
- package/src/repos/thread-message.test.ts +17 -4
- package/src/repos/thread-message.ts +502 -285
- package/src/repos/thread-search-predicates.ts +24 -26
- package/src/schema/message-data.ts +4 -8
- package/src/schema/sync-warning.ts +3 -0
- package/src/schema-sets/app.ts +17 -0
- package/src/schema-sets/mail.ts +32 -0
- package/src/schema-sets/search.ts +5 -0
- package/src/schema-version-accounting.sqlite.test.ts +56 -43
- package/src/schema.ts +3 -4
- package/src/search/thread-message-search.sqlite.test.ts +131 -0
- package/src/search/thread-message-search.ts +271 -0
- package/src/sqlite-client.ts +5 -7
- package/src/sqlite-repositories.ts +75 -0
- package/src/test-db-sqlite.ts +5 -13
- package/src/test-shipped-sqlite-schema.ts +46 -45
- package/src/test-sqlite.ts +35 -11
- package/src/vps-migrations-drift.sqlite.test.ts +81 -15
- package/src/account-granted-scopes-migration.sqlite.test.ts +0 -66
- package/src/account-synced-services-migration.sqlite.test.ts +0 -56
- package/src/mailbox-path-dedupe.sqlite.test.ts +0 -462
- package/src/mailbox-sync-status-backfill.sqlite.test.ts +0 -118
- package/src/outbox-created-cleanup.sqlite.test.ts +0 -83
- package/src/repair/duplicate-mailbox-mail.ts +0 -98
- package/src/repair/thread-message-category-contract.ts +0 -505
- package/src/repair/thread-message-category.sqlite.test.ts +0 -100
- package/src/repair/thread-message-category.ts +0 -415
- package/src/repos/calendar-migrations.sqlite.test.ts +0 -345
- package/src/repos/calendar-unit-of-work.ts +0 -31
- package/src/repos/config-import-atomicity.sqlite.test.ts +0 -255
- package/src/repos/filter-anchor-transaction.test.ts +0 -136
- package/src/repos/filter-anchor-transaction.ts +0 -45
- package/src/repos/filter-anchor.test.ts +0 -131
- package/src/repos/filter-anchor.ts +0 -105
- package/src/repos/i4-mailbox-special-use.structure.test.ts +0 -27
- package/src/repos/message-placement-backfill.sqlite.test.ts +0 -190
- package/src/repos/search-index-shape.test.ts +0 -82
- package/src/repos/search-index-shape.ts +0 -56
- package/src/repos/serialized-writes.sqlite.test.ts +0 -199
- package/src/repos/unit-of-work.sqlite.test.ts +0 -138
- package/src/repos/unit-of-work.test.ts +0 -109
- package/src/repos/unit-of-work.ts +0 -31
- package/src/schema/outbox.ts +0 -46
- package/src/schema-full-sqlite.ts +0 -13
- package/src/tx.sqlite.test.ts +0 -83
- package/src/tx.ts +0 -233
|
@@ -5,9 +5,10 @@ import { MailboxSyncStatus } from "@remit/domain-enums";
|
|
|
5
5
|
import Database from "better-sqlite3";
|
|
6
6
|
import { drizzle } from "drizzle-orm/better-sqlite3";
|
|
7
7
|
import { mailboxTable } from "../schema.js";
|
|
8
|
+
import { SearchIndex } from "../search/thread-message-search.js";
|
|
8
9
|
import { createSqliteTestDb } from "../test-db-sqlite.js";
|
|
9
10
|
import {
|
|
10
|
-
|
|
11
|
+
applyMigrationSet,
|
|
11
12
|
shippedTableDdl,
|
|
12
13
|
} from "../test-shipped-sqlite-schema.js";
|
|
13
14
|
import { MailboxRepo } from "./i4-mailbox.js";
|
|
@@ -40,7 +41,10 @@ describe("MailboxRepo (sqlite)", () => {
|
|
|
40
41
|
before(async () => {
|
|
41
42
|
const testDb = await createSqliteTestDb({ mailbox: mailboxTable });
|
|
42
43
|
close = testDb.close;
|
|
43
|
-
repo = new MailboxRepo(
|
|
44
|
+
repo = new MailboxRepo(
|
|
45
|
+
testDb.db as never,
|
|
46
|
+
new SearchIndex(testDb.db as never, testDb.db as never),
|
|
47
|
+
);
|
|
44
48
|
});
|
|
45
49
|
|
|
46
50
|
after(async () => {
|
|
@@ -204,15 +208,12 @@ describe("MailboxRepo (sqlite, shipped migrations)", () => {
|
|
|
204
208
|
|
|
205
209
|
before(() => {
|
|
206
210
|
const sqlite = new Database(":memory:");
|
|
207
|
-
sqlite
|
|
208
|
-
applyMigration(sqlite, "0002_highest_modseq_text");
|
|
209
|
-
applyMigration(sqlite, "0026_mailbox_sync_status_backfill");
|
|
210
|
-
applyMigration(sqlite, "0027_mailbox_sync_status_total");
|
|
211
|
-
applyMigration(sqlite, "0028_mailbox_pending_path");
|
|
212
|
-
applyMigration(sqlite, "0031_mailbox_path_unique");
|
|
213
|
-
applyMigration(sqlite, "0033_mailbox_sync_failure_reason");
|
|
211
|
+
applyMigrationSet(sqlite, "mail");
|
|
214
212
|
const db = drizzle(sqlite, { schema: { mailbox: mailboxTable } });
|
|
215
|
-
repo = new MailboxRepo(
|
|
213
|
+
repo = new MailboxRepo(
|
|
214
|
+
db as never,
|
|
215
|
+
new SearchIndex(db as never, db as never),
|
|
216
|
+
);
|
|
216
217
|
close = async () => {
|
|
217
218
|
sqlite.close();
|
|
218
219
|
};
|
|
@@ -223,10 +224,7 @@ describe("MailboxRepo (sqlite, shipped migrations)", () => {
|
|
|
223
224
|
});
|
|
224
225
|
|
|
225
226
|
test("declares highest_modseq as text", () => {
|
|
226
|
-
assert.match(
|
|
227
|
-
shippedTableDdl("0002_highest_modseq_text", "__new_mailbox"),
|
|
228
|
-
/`highest_modseq` text NOT NULL/,
|
|
229
|
-
);
|
|
227
|
+
assert.match(shippedTableDdl("mailbox"), /`highest_modseq` text NOT NULL/);
|
|
230
228
|
});
|
|
231
229
|
|
|
232
230
|
test("reads a plain-digit cursor back as a string", async () => {
|
|
@@ -2,6 +2,7 @@ import assert from "node:assert";
|
|
|
2
2
|
import { after, before, describe, test } from "node:test";
|
|
3
3
|
import { MailboxSyncStatus } from "@remit/domain-enums";
|
|
4
4
|
import { eq } from "drizzle-orm";
|
|
5
|
+
import { runBatch } from "../batch.js";
|
|
5
6
|
import {
|
|
6
7
|
envelopeId as deriveEnvelopeId,
|
|
7
8
|
rootBodyPartId as deriveRootBodyPartId,
|
|
@@ -11,11 +12,10 @@ import {
|
|
|
11
12
|
mailboxSpecialUseTable,
|
|
12
13
|
mailboxTable,
|
|
13
14
|
messageTable,
|
|
14
|
-
outboxTable,
|
|
15
15
|
threadMessageTable,
|
|
16
16
|
} from "../schema.js";
|
|
17
|
+
import { SearchIndex } from "../search/thread-message-search.js";
|
|
17
18
|
import { createTestDb, randomId, type TestDb } from "../test-db.js";
|
|
18
|
-
import { runInTransaction } from "../tx.js";
|
|
19
19
|
import { MailboxRepo } from "./i4-mailbox.js";
|
|
20
20
|
import { DrizzleMessageRepository, deleteMessageSubtree } from "./message.js";
|
|
21
21
|
import { mailboxCreated } from "./test-helpers.js";
|
|
@@ -55,7 +55,10 @@ describe("MailboxRepo", () => {
|
|
|
55
55
|
|
|
56
56
|
before(async () => {
|
|
57
57
|
({ db, close } = await createTestDb());
|
|
58
|
-
repo = new MailboxRepo(
|
|
58
|
+
repo = new MailboxRepo(
|
|
59
|
+
db as never,
|
|
60
|
+
new SearchIndex(db as never, db as never),
|
|
61
|
+
);
|
|
59
62
|
});
|
|
60
63
|
|
|
61
64
|
after(async () => {
|
|
@@ -476,9 +479,7 @@ describe("MailboxRepo", () => {
|
|
|
476
479
|
{
|
|
477
480
|
from: [MailboxSyncStatus.synced, MailboxSyncStatus.failed],
|
|
478
481
|
to: MailboxSyncStatus.pending,
|
|
479
|
-
|
|
480
|
-
pendingPath: row.fullPath.replace("Work", "Archive"),
|
|
481
|
-
}),
|
|
482
|
+
newPath: "Archive",
|
|
482
483
|
},
|
|
483
484
|
);
|
|
484
485
|
|
|
@@ -511,7 +512,7 @@ describe("MailboxRepo", () => {
|
|
|
511
512
|
{
|
|
512
513
|
from: [MailboxSyncStatus.synced],
|
|
513
514
|
to: MailboxSyncStatus.pending,
|
|
514
|
-
|
|
515
|
+
newPath: "Moved/Work",
|
|
515
516
|
},
|
|
516
517
|
);
|
|
517
518
|
|
|
@@ -527,46 +528,12 @@ describe("MailboxRepo", () => {
|
|
|
527
528
|
);
|
|
528
529
|
});
|
|
529
530
|
|
|
530
|
-
test("the from-state predicate rides each UPDATE, not a prior read", async () => {
|
|
531
|
-
// A read-then-check-then-write version passes the case above and still
|
|
532
|
-
// misses a row that leaves an accepted state after the read. The
|
|
533
|
-
// predicate is on the UPDATE, so the affected-row count catches it.
|
|
534
|
-
const accountId = randomId();
|
|
535
|
-
const { parent, child } = await seedSubtree(accountId);
|
|
536
|
-
|
|
537
|
-
const refused = await repo.transitionSubtree(
|
|
538
|
-
accountId,
|
|
539
|
-
parent.mailboxId,
|
|
540
|
-
{
|
|
541
|
-
from: [MailboxSyncStatus.synced],
|
|
542
|
-
to: MailboxSyncStatus.pending,
|
|
543
|
-
rowSet: (row) => {
|
|
544
|
-
if (row.mailboxId === parent.mailboxId) {
|
|
545
|
-
// Not a real concurrent writer — the shape of one. The row
|
|
546
|
-
// leaves `synced` after the subtree was resolved.
|
|
547
|
-
db.update(mailboxTable)
|
|
548
|
-
.set({ syncStatus: MailboxSyncStatus.deleting })
|
|
549
|
-
.where(eq(mailboxTable.mailboxId, child.mailboxId))
|
|
550
|
-
.run();
|
|
551
|
-
}
|
|
552
|
-
return { pendingPath: row.fullPath };
|
|
553
|
-
},
|
|
554
|
-
},
|
|
555
|
-
);
|
|
556
|
-
|
|
557
|
-
assert.equal(refused, null);
|
|
558
|
-
assert.equal(
|
|
559
|
-
(await repo.get(accountId, parent.mailboxId)).syncStatus,
|
|
560
|
-
MailboxSyncStatus.synced,
|
|
561
|
-
);
|
|
562
|
-
});
|
|
563
|
-
|
|
564
531
|
test("an absent folder is a null", async () => {
|
|
565
532
|
assert.equal(
|
|
566
533
|
await repo.transitionSubtree(randomId(), "no-such-mailbox", {
|
|
567
534
|
from: EVERY_STATE,
|
|
568
535
|
to: MailboxSyncStatus.pending,
|
|
569
|
-
|
|
536
|
+
newPath: "Anywhere",
|
|
570
537
|
}),
|
|
571
538
|
null,
|
|
572
539
|
);
|
|
@@ -576,7 +543,10 @@ describe("MailboxRepo", () => {
|
|
|
576
543
|
describe("deleteMailboxWithMail — the folder's mail goes with it (D8)", () => {
|
|
577
544
|
const seedMessage = async (mailboxId: string, uid: number) => {
|
|
578
545
|
const messageId = randomId();
|
|
579
|
-
await new DrizzleMessageRepository(
|
|
546
|
+
await new DrizzleMessageRepository(
|
|
547
|
+
db as never,
|
|
548
|
+
new SearchIndex(db as never, db as never),
|
|
549
|
+
).create({
|
|
580
550
|
messageId,
|
|
581
551
|
mailboxId,
|
|
582
552
|
uid,
|
|
@@ -606,9 +576,6 @@ describe("MailboxRepo", () => {
|
|
|
606
576
|
return messageId;
|
|
607
577
|
};
|
|
608
578
|
|
|
609
|
-
const outboxEventsFor = async (messageId: string) =>
|
|
610
|
-
db.select().from(outboxTable).where(eq(outboxTable.messageId, messageId));
|
|
611
|
-
|
|
612
579
|
test("removes the folder, its mail and its own child rows, and nothing else's", async () => {
|
|
613
580
|
const accountId = randomId();
|
|
614
581
|
const doomed = mailboxCreated(
|
|
@@ -617,8 +584,8 @@ describe("MailboxRepo", () => {
|
|
|
617
584
|
const spared = mailboxCreated(
|
|
618
585
|
await repo.create(makeMailboxInput(accountId, "Keep")),
|
|
619
586
|
);
|
|
620
|
-
|
|
621
|
-
|
|
587
|
+
await seedMessage(doomed.mailboxId, 1);
|
|
588
|
+
await seedMessage(spared.mailboxId, 1);
|
|
622
589
|
await db.insert(mailboxSpecialUseTable).values({
|
|
623
590
|
mailboxSpecialUseId: randomId(),
|
|
624
591
|
mailboxId: doomed.mailboxId,
|
|
@@ -653,24 +620,11 @@ describe("MailboxRepo", () => {
|
|
|
653
620
|
[],
|
|
654
621
|
);
|
|
655
622
|
|
|
656
|
-
// The search index is cleared by the outbox row, not by the delete.
|
|
657
|
-
const removals = await outboxEventsFor(doomedMessage);
|
|
658
|
-
assert.deepEqual(
|
|
659
|
-
removals.map((row) => row.event),
|
|
660
|
-
["message.removed"],
|
|
661
|
-
);
|
|
662
|
-
|
|
663
623
|
const survivors = await db
|
|
664
624
|
.select()
|
|
665
625
|
.from(messageTable)
|
|
666
626
|
.where(eq(messageTable.mailboxId, spared.mailboxId));
|
|
667
627
|
assert.equal(survivors.length, 1);
|
|
668
|
-
assert.equal(
|
|
669
|
-
(await outboxEventsFor(sparedMessage)).some(
|
|
670
|
-
(row) => row.event === "message.removed",
|
|
671
|
-
),
|
|
672
|
-
false,
|
|
673
|
-
);
|
|
674
628
|
assert.equal(
|
|
675
629
|
(await repo.get(accountId, spared.mailboxId)).mailboxId,
|
|
676
630
|
spared.mailboxId,
|
|
@@ -692,6 +646,7 @@ describe("MailboxRepo", () => {
|
|
|
692
646
|
accountConfigId: accountId,
|
|
693
647
|
name: "Invoices → Bound",
|
|
694
648
|
scope: "Standing",
|
|
649
|
+
literalClauses: [{ field: "Subject", value: "invoice" }],
|
|
695
650
|
ruleChangedAt: 0,
|
|
696
651
|
actionChangedAt: 0,
|
|
697
652
|
actionMailboxId: mailbox.mailboxId,
|
|
@@ -719,7 +674,7 @@ describe("MailboxRepo", () => {
|
|
|
719
674
|
|
|
720
675
|
// The shape an interruption leaves: one message's rows already gone,
|
|
721
676
|
// the mailbox row still there and still `deleting`.
|
|
722
|
-
await
|
|
677
|
+
await runBatch(db, deleteMessageSubtree(db, [first]));
|
|
723
678
|
await repo.transition(accountId, mailbox.mailboxId, {
|
|
724
679
|
from: EVERY_STATE,
|
|
725
680
|
to: MailboxSyncStatus.deleting,
|
package/src/repos/i4-mailbox.ts
CHANGED
|
@@ -12,8 +12,22 @@ import type {
|
|
|
12
12
|
UpdateMailboxInput,
|
|
13
13
|
} from "@remit/data-ports";
|
|
14
14
|
import { MailboxCursorState } from "@remit/domain-enums";
|
|
15
|
-
import {
|
|
15
|
+
import {
|
|
16
|
+
and,
|
|
17
|
+
asc,
|
|
18
|
+
eq,
|
|
19
|
+
exists,
|
|
20
|
+
gt,
|
|
21
|
+
inArray,
|
|
22
|
+
isNull,
|
|
23
|
+
not,
|
|
24
|
+
notInArray,
|
|
25
|
+
or,
|
|
26
|
+
type SQL,
|
|
27
|
+
sql,
|
|
28
|
+
} from "drizzle-orm";
|
|
16
29
|
import shortUuid from "short-uuid";
|
|
30
|
+
import { runBatch } from "../batch.js";
|
|
17
31
|
import type { Db } from "../db.js";
|
|
18
32
|
import { NotFoundError } from "../error.js";
|
|
19
33
|
import { decodeToken, resultList } from "../pagination.js";
|
|
@@ -27,8 +41,9 @@ import { mailboxLockTable } from "../schema/i4-mailbox-lock.js";
|
|
|
27
41
|
import { messageFlagPushTable } from "../schema/i4-message-flag-push.js";
|
|
28
42
|
import { messagePlacementMoveTable } from "../schema/i4-message-placement-move.js";
|
|
29
43
|
import { messageTable } from "../schema/message-data.js";
|
|
44
|
+
import { syncWarningTable } from "../schema/sync-warning.js";
|
|
30
45
|
import { threadMessageTable } from "../schema/thread-message.js";
|
|
31
|
-
import {
|
|
46
|
+
import type { SearchIndex } from "../search/thread-message-search.js";
|
|
32
47
|
import { deleteMessageSubtree } from "./message.js";
|
|
33
48
|
|
|
34
49
|
const base36Translator = shortUuid.createTranslator(
|
|
@@ -39,20 +54,24 @@ const generateMailboxId = () => base36Translator.fromUUID(randomUUID());
|
|
|
39
54
|
type DB = Db<Record<string, unknown>>;
|
|
40
55
|
|
|
41
56
|
/**
|
|
42
|
-
* Message subtrees removed per
|
|
57
|
+
* Message subtrees removed per batch by {@link MailboxRepo.deleteMailboxWithMail},
|
|
43
58
|
* matching `SUBTREE_BATCH_SIZE` in the account purge. On SQLite each batch holds
|
|
44
|
-
* the
|
|
45
|
-
*
|
|
59
|
+
* the file's write lock, so the bound is what keeps a large folder's delete from
|
|
60
|
+
* parking every other writer behind it (D8).
|
|
46
61
|
*/
|
|
47
62
|
const MAIL_DELETE_BATCH_SIZE = 100;
|
|
48
63
|
|
|
49
|
-
/**
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
64
|
+
/**
|
|
65
|
+
* The folders under `path`. A flat namespace nests nothing: with no delimiter
|
|
66
|
+
* the prefix is the folder's own path, which would match every sibling
|
|
67
|
+
* starting with it. Lengths are SQL's, so a path is measured the way `substr`
|
|
68
|
+
* cuts it, in characters rather than UTF-16 units.
|
|
69
|
+
*/
|
|
70
|
+
const descendantOf = (path: string, delimiter: string): SQL => {
|
|
71
|
+
if (delimiter.length === 0) return sql`0`;
|
|
72
|
+
const prefix = `${path}${delimiter}`;
|
|
73
|
+
return sql`substr(${mailboxTable.fullPath}, 1, length(${prefix})) = ${prefix}`;
|
|
74
|
+
};
|
|
56
75
|
|
|
57
76
|
/** The WHERE terms of a folder-state transition (folder-rename-and-delete.md D3). */
|
|
58
77
|
const stateTerms = (expected: MailboxStatePredicate): SQL[] => {
|
|
@@ -129,7 +148,10 @@ export function rowToMailbox(
|
|
|
129
148
|
}
|
|
130
149
|
|
|
131
150
|
export class MailboxRepo implements IMailboxRepository {
|
|
132
|
-
constructor(
|
|
151
|
+
constructor(
|
|
152
|
+
private db: DB,
|
|
153
|
+
private search: SearchIndex,
|
|
154
|
+
) {}
|
|
133
155
|
|
|
134
156
|
async create(input: CreateMailboxInput): Promise<CreateMailboxResult> {
|
|
135
157
|
const now = Date.now();
|
|
@@ -291,55 +313,55 @@ export class MailboxRepo implements IMailboxRepository {
|
|
|
291
313
|
mailboxId: string,
|
|
292
314
|
intent: MailboxSubtreeTransitionIntent,
|
|
293
315
|
): Promise<MailboxItem[] | null> {
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
root,
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
root.hierarchyDelimiter,
|
|
310
|
-
)),
|
|
311
|
-
];
|
|
312
|
-
|
|
313
|
-
const written: MailboxItem[] = [];
|
|
314
|
-
for (const row of subtree) {
|
|
315
|
-
// The from-state predicate rides each UPDATE rather than a read
|
|
316
|
-
// taken before them (D3). Read-then-check-then-write is safe on
|
|
317
|
-
// SQLite only because `runInTransaction` serializes top-level
|
|
318
|
-
// writes; under Postgres READ COMMITTED a single-row transition
|
|
319
|
-
// committing in between is missed entirely.
|
|
320
|
-
const [updated] = await tx
|
|
321
|
-
.update(mailboxTable)
|
|
322
|
-
.set(transitionSet(intent.to, intent.rowSet(row)))
|
|
323
|
-
.where(
|
|
324
|
-
and(
|
|
325
|
-
eq(mailboxTable.accountId, accountId),
|
|
326
|
-
eq(mailboxTable.mailboxId, row.mailboxId),
|
|
327
|
-
inArray(mailboxTable.syncStatus, [...intent.from]),
|
|
328
|
-
),
|
|
329
|
-
)
|
|
330
|
-
.returning();
|
|
331
|
-
if (updated) written.push(rowToMailbox(updated));
|
|
332
|
-
}
|
|
316
|
+
const root = await this.get(accountId, mailboxId).catch(
|
|
317
|
+
(error: unknown) => {
|
|
318
|
+
if (error instanceof NotFoundError) return null;
|
|
319
|
+
throw error;
|
|
320
|
+
},
|
|
321
|
+
);
|
|
322
|
+
if (!root) return null;
|
|
323
|
+
|
|
324
|
+
const inSubtree = and(
|
|
325
|
+
eq(mailboxTable.accountId, accountId),
|
|
326
|
+
or(
|
|
327
|
+
eq(mailboxTable.mailboxId, root.mailboxId),
|
|
328
|
+
descendantOf(root.fullPath, root.hierarchyDelimiter),
|
|
329
|
+
),
|
|
330
|
+
);
|
|
333
331
|
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
332
|
+
// One statement, so a subtree is never half-renamed: every row takes its
|
|
333
|
+
// rebased target together, and only while none of them is outside a
|
|
334
|
+
// from-state. Lengths are SQL's, counted the way `substr` counts.
|
|
335
|
+
const written = await this.db
|
|
336
|
+
.update(mailboxTable)
|
|
337
|
+
.set({
|
|
338
|
+
...transitionSet(intent.to, undefined),
|
|
339
|
+
pendingPath: sql`${intent.newPath} || substr(${mailboxTable.fullPath}, length(${root.fullPath}) + 1)`,
|
|
340
|
+
})
|
|
341
|
+
.where(
|
|
342
|
+
and(
|
|
343
|
+
inSubtree,
|
|
344
|
+
not(
|
|
345
|
+
exists(
|
|
346
|
+
this.db
|
|
347
|
+
.select({ mailboxId: mailboxTable.mailboxId })
|
|
348
|
+
.from(mailboxTable)
|
|
349
|
+
.where(
|
|
350
|
+
and(
|
|
351
|
+
inSubtree,
|
|
352
|
+
notInArray(mailboxTable.syncStatus, [...intent.from]),
|
|
353
|
+
),
|
|
354
|
+
),
|
|
355
|
+
),
|
|
356
|
+
),
|
|
357
|
+
),
|
|
358
|
+
)
|
|
359
|
+
.returning();
|
|
360
|
+
if (written.length === 0) return null;
|
|
361
|
+
return [
|
|
362
|
+
...written.filter((row) => row.mailboxId === root.mailboxId),
|
|
363
|
+
...written.filter((row) => row.mailboxId !== root.mailboxId),
|
|
364
|
+
].map(rowToMailbox);
|
|
343
365
|
}
|
|
344
366
|
|
|
345
367
|
async resolveAccountId(mailboxId: string): Promise<string | null> {
|
|
@@ -351,26 +373,29 @@ export class MailboxRepo implements IMailboxRepository {
|
|
|
351
373
|
}
|
|
352
374
|
|
|
353
375
|
async delete(accountId: string, mailboxId: string): Promise<void> {
|
|
354
|
-
await this.
|
|
355
|
-
.delete(mailboxTable)
|
|
356
|
-
.where(
|
|
357
|
-
and(
|
|
358
|
-
eq(mailboxTable.accountId, accountId),
|
|
359
|
-
eq(mailboxTable.mailboxId, mailboxId),
|
|
360
|
-
),
|
|
361
|
-
);
|
|
376
|
+
await this.deleteMany(accountId, [mailboxId]);
|
|
362
377
|
}
|
|
363
378
|
|
|
364
379
|
async deleteMany(accountId: string, mailboxIds: string[]): Promise<void> {
|
|
365
380
|
if (mailboxIds.length === 0) return;
|
|
366
|
-
await this.db
|
|
367
|
-
.
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
381
|
+
await runBatch(this.db, [
|
|
382
|
+
this.db
|
|
383
|
+
.delete(syncWarningTable)
|
|
384
|
+
.where(
|
|
385
|
+
and(
|
|
386
|
+
eq(syncWarningTable.accountId, accountId),
|
|
387
|
+
inArray(syncWarningTable.mailboxId, mailboxIds),
|
|
388
|
+
),
|
|
372
389
|
),
|
|
373
|
-
|
|
390
|
+
this.db
|
|
391
|
+
.delete(mailboxTable)
|
|
392
|
+
.where(
|
|
393
|
+
and(
|
|
394
|
+
eq(mailboxTable.accountId, accountId),
|
|
395
|
+
inArray(mailboxTable.mailboxId, mailboxIds),
|
|
396
|
+
),
|
|
397
|
+
),
|
|
398
|
+
]);
|
|
374
399
|
}
|
|
375
400
|
|
|
376
401
|
async listByAccount(
|
|
@@ -465,17 +490,16 @@ export class MailboxRepo implements IMailboxRepository {
|
|
|
465
490
|
pathPrefix: string,
|
|
466
491
|
delimiter = "/",
|
|
467
492
|
): Promise<MailboxItem[]> {
|
|
468
|
-
// A flat namespace nests nothing: with no delimiter the prefix is the
|
|
469
|
-
// folder’s own path, which would match every sibling starting with it.
|
|
470
|
-
if (delimiter.length === 0) return [];
|
|
471
493
|
const rows = await this.db
|
|
472
494
|
.select()
|
|
473
495
|
.from(mailboxTable)
|
|
474
|
-
.where(
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
496
|
+
.where(
|
|
497
|
+
and(
|
|
498
|
+
eq(mailboxTable.accountId, accountId),
|
|
499
|
+
descendantOf(pathPrefix, delimiter),
|
|
500
|
+
),
|
|
501
|
+
);
|
|
502
|
+
return rows.map(rowToMailbox);
|
|
479
503
|
}
|
|
480
504
|
|
|
481
505
|
async findBySyncStatus(
|
|
@@ -524,17 +548,20 @@ export class MailboxRepo implements IMailboxRepository {
|
|
|
524
548
|
.limit(MAIL_DELETE_BATCH_SIZE);
|
|
525
549
|
if (rows.length === 0) break;
|
|
526
550
|
const messageIds = rows.map((row) => row.messageId);
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
551
|
+
const listings = await this.db
|
|
552
|
+
.select({ threadMessageId: threadMessageTable.threadMessageId })
|
|
553
|
+
.from(threadMessageTable)
|
|
554
|
+
.where(inArray(threadMessageTable.messageId, messageIds));
|
|
555
|
+
|
|
556
|
+
// The primitive the rest of the codebase deletes mail with: nine
|
|
557
|
+
// per-message child tables. A bespoke table list would orphan them.
|
|
558
|
+
await runBatch(this.db, [
|
|
559
|
+
...deleteMessageSubtree(this.db, messageIds),
|
|
560
|
+
this.db
|
|
535
561
|
.delete(threadMessageTable)
|
|
536
|
-
.where(inArray(threadMessageTable.messageId, messageIds))
|
|
537
|
-
|
|
562
|
+
.where(inArray(threadMessageTable.messageId, messageIds)),
|
|
563
|
+
]);
|
|
564
|
+
await this.search.sync(listings.map((row) => row.threadMessageId));
|
|
538
565
|
}
|
|
539
566
|
|
|
540
567
|
await this.db
|
|
@@ -23,11 +23,9 @@ function rowToItem(
|
|
|
23
23
|
userId: row.userId,
|
|
24
24
|
state: row.state as OrganizeJobRequestItem["state"],
|
|
25
25
|
filterId: row.filterId,
|
|
26
|
-
anchorMessageId: row.anchorMessageId,
|
|
27
26
|
matchOperator: row.matchOperator as OrganizeJobRequestItem["matchOperator"],
|
|
28
27
|
literalClauses:
|
|
29
28
|
row.literalClauses as OrganizeJobRequestItem["literalClauses"],
|
|
30
|
-
similarityThreshold: row.similarityThreshold,
|
|
31
29
|
actionLabelId: row.actionLabelId,
|
|
32
30
|
actionMailboxId: row.actionMailboxId,
|
|
33
31
|
matchedCount: row.matchedCount,
|
|
@@ -55,10 +53,8 @@ export class OrganizeJobRequestRepo implements IOrganizeJobRequestRepository {
|
|
|
55
53
|
userId: input.userId,
|
|
56
54
|
state: input.state ?? "Pending",
|
|
57
55
|
filterId: input.filterId,
|
|
58
|
-
anchorMessageId: input.anchorMessageId,
|
|
59
56
|
matchOperator: input.matchOperator,
|
|
60
57
|
literalClauses: input.literalClauses,
|
|
61
|
-
similarityThreshold: input.similarityThreshold,
|
|
62
58
|
actionLabelId: input.actionLabelId,
|
|
63
59
|
actionMailboxId: input.actionMailboxId,
|
|
64
60
|
matchedCount: 0,
|
|
@@ -6,11 +6,11 @@ import type {
|
|
|
6
6
|
ReserveOutboxAttachmentResult,
|
|
7
7
|
} from "@remit/data-ports";
|
|
8
8
|
import { holdsRoom } from "@remit/data-ports";
|
|
9
|
-
import { and, eq, inArray, lt } from "drizzle-orm";
|
|
9
|
+
import { and, eq, gt, gte, inArray, lt, or, sum } from "drizzle-orm";
|
|
10
|
+
import { runBatch } from "../batch.js";
|
|
10
11
|
import type { Db } from "../db.js";
|
|
11
12
|
import { NotFoundError } from "../error.js";
|
|
12
13
|
import { outboxAttachmentTable } from "../schema/i4-outbox-attachment.js";
|
|
13
|
-
import { runInTransaction } from "../tx.js";
|
|
14
14
|
|
|
15
15
|
type DB = Db<Record<string, unknown>>;
|
|
16
16
|
|
|
@@ -37,40 +37,30 @@ export class OutboxAttachmentRepo implements IOutboxAttachmentRepository {
|
|
|
37
37
|
constructor(private db: DB) {}
|
|
38
38
|
|
|
39
39
|
/**
|
|
40
|
-
* The per-message cap
|
|
41
|
-
*
|
|
42
|
-
*
|
|
43
|
-
*
|
|
40
|
+
* The per-message cap. The insert, the read of the draft's live rows and a
|
|
41
|
+
* delete of the new row when it breaks the cap run as one batch, which
|
|
42
|
+
* commits as a unit and serializes against every other batch. Of two
|
|
43
|
+
* concurrent reservations the later one sees the earlier one's row, so it
|
|
44
|
+
* is the one removed when the two together do not fit.
|
|
44
45
|
*/
|
|
45
46
|
async reserve(
|
|
46
47
|
input: CreateOutboxAttachmentInput,
|
|
47
48
|
cap: OutboxAttachmentCap,
|
|
48
49
|
): Promise<ReserveOutboxAttachmentResult> {
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
)
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
const usedBytes = live.reduce((total, item) => total + item.sizeBytes, 0);
|
|
64
|
-
|
|
65
|
-
if (live.length >= cap.maxCount) {
|
|
66
|
-
return { outcome: "OverCountCap", usedBytes };
|
|
67
|
-
}
|
|
68
|
-
if (usedBytes + input.sizeBytes > cap.maxTotalBytes) {
|
|
69
|
-
return { outcome: "OverByteCap", usedBytes };
|
|
70
|
-
}
|
|
71
|
-
|
|
72
|
-
const now = Date.now();
|
|
73
|
-
const [row] = await tx
|
|
50
|
+
const now = Date.now();
|
|
51
|
+
const draft = and(
|
|
52
|
+
eq(outboxAttachmentTable.accountConfigId, input.accountConfigId),
|
|
53
|
+
eq(outboxAttachmentTable.outboxMessageId, input.outboxMessageId),
|
|
54
|
+
);
|
|
55
|
+
const live = and(
|
|
56
|
+
draft,
|
|
57
|
+
or(
|
|
58
|
+
eq(outboxAttachmentTable.state, "Stored"),
|
|
59
|
+
gte(outboxAttachmentTable.reservationExpiresAt, cap.nowSeconds),
|
|
60
|
+
),
|
|
61
|
+
);
|
|
62
|
+
const [inserted, rows, removed] = await runBatch(this.db, [
|
|
63
|
+
this.db
|
|
74
64
|
.insert(outboxAttachmentTable)
|
|
75
65
|
.values({
|
|
76
66
|
outboxAttachmentId: input.outboxAttachmentId,
|
|
@@ -86,10 +76,49 @@ export class OutboxAttachmentRepo implements IOutboxAttachmentRepository {
|
|
|
86
76
|
createdAt: now,
|
|
87
77
|
updatedAt: now,
|
|
88
78
|
})
|
|
89
|
-
.returning()
|
|
79
|
+
.returning(),
|
|
80
|
+
this.db.select().from(outboxAttachmentTable).where(draft),
|
|
81
|
+
this.db
|
|
82
|
+
.delete(outboxAttachmentTable)
|
|
83
|
+
.where(
|
|
84
|
+
and(
|
|
85
|
+
eq(
|
|
86
|
+
outboxAttachmentTable.outboxAttachmentId,
|
|
87
|
+
input.outboxAttachmentId,
|
|
88
|
+
),
|
|
89
|
+
or(
|
|
90
|
+
gt(this.db.$count(outboxAttachmentTable, live), cap.maxCount),
|
|
91
|
+
gt(
|
|
92
|
+
this.db
|
|
93
|
+
.select({ total: sum(outboxAttachmentTable.sizeBytes) })
|
|
94
|
+
.from(outboxAttachmentTable)
|
|
95
|
+
.where(live),
|
|
96
|
+
cap.maxTotalBytes,
|
|
97
|
+
),
|
|
98
|
+
),
|
|
99
|
+
),
|
|
100
|
+
)
|
|
101
|
+
.returning({
|
|
102
|
+
outboxAttachmentId: outboxAttachmentTable.outboxAttachmentId,
|
|
103
|
+
}),
|
|
104
|
+
] as const);
|
|
90
105
|
|
|
106
|
+
const others = rows
|
|
107
|
+
.map(rowToOutboxAttachment)
|
|
108
|
+
.filter(
|
|
109
|
+
(item) =>
|
|
110
|
+
item.outboxAttachmentId !== input.outboxAttachmentId &&
|
|
111
|
+
holdsRoom(item, cap.nowSeconds),
|
|
112
|
+
);
|
|
113
|
+
const usedBytes = others.reduce((total, item) => total + item.sizeBytes, 0);
|
|
114
|
+
const [row] = inserted;
|
|
115
|
+
if (removed.length === 0 && row) {
|
|
91
116
|
return { outcome: "Reserved", item: rowToOutboxAttachment(row) };
|
|
92
|
-
}
|
|
117
|
+
}
|
|
118
|
+
if (others.length >= cap.maxCount) {
|
|
119
|
+
return { outcome: "OverCountCap", usedBytes };
|
|
120
|
+
}
|
|
121
|
+
return { outcome: "OverByteCap", usedBytes };
|
|
93
122
|
}
|
|
94
123
|
|
|
95
124
|
async get(
|