@remit/drizzle-service 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/drizzle.config.ts +15 -0
- package/package.json +52 -0
- package/src/db.ts +13 -0
- package/src/dialect.ts +13 -0
- package/src/error.ts +37 -0
- package/src/id.ts +50 -0
- package/src/index.ts +62 -0
- package/src/pagination.ts +29 -0
- package/src/repos/cascade-delete.sqlite.test.ts +159 -0
- package/src/repos/cascade-delete.test.ts +423 -0
- package/src/repos/cascade-delete.ts +219 -0
- package/src/repos/envelope.sqlite.test.ts +94 -0
- package/src/repos/envelope.test.ts +225 -0
- package/src/repos/envelope.ts +342 -0
- package/src/repos/filter-anchor.test.ts +131 -0
- package/src/repos/filter-anchor.ts +105 -0
- package/src/repos/filter.test.ts +279 -0
- package/src/repos/filter.ts +253 -0
- package/src/repos/i4-account-config.test.ts +105 -0
- package/src/repos/i4-account-config.ts +257 -0
- package/src/repos/i4-account-export-request.ts +141 -0
- package/src/repos/i4-account-setting.test.ts +94 -0
- package/src/repos/i4-account-setting.ts +92 -0
- package/src/repos/i4-account.test.ts +223 -0
- package/src/repos/i4-account.ts +380 -0
- package/src/repos/i4-address-wellknown.ts +31 -0
- package/src/repos/i4-address.test.ts +358 -0
- package/src/repos/i4-address.ts +613 -0
- package/src/repos/i4-mailbox-lock.test.ts +368 -0
- package/src/repos/i4-mailbox-lock.ts +140 -0
- package/src/repos/i4-mailbox-special-use.ts +165 -0
- package/src/repos/i4-mailbox.test.ts +188 -0
- package/src/repos/i4-mailbox.ts +347 -0
- package/src/repos/i4-message-flag-push.test.ts +189 -0
- package/src/repos/i4-message-flag-push.ts +173 -0
- package/src/repos/i4-message-placement-move.test.ts +135 -0
- package/src/repos/i4-message-placement-move.ts +144 -0
- package/src/repos/i4-organize-job-request.ts +142 -0
- package/src/repos/i4-outbox-message.test.ts +298 -0
- package/src/repos/i4-outbox-message.ts +299 -0
- package/src/repos/label.conformance.sqlite.test.ts +23 -0
- package/src/repos/label.conformance.test.ts +19 -0
- package/src/repos/label.ts +125 -0
- package/src/repos/mappers.ts +171 -0
- package/src/repos/message-flag.ts +162 -0
- package/src/repos/message-label.test.ts +110 -0
- package/src/repos/message-label.ts +96 -0
- package/src/repos/message.sqlite.test.ts +118 -0
- package/src/repos/message.test.ts +488 -0
- package/src/repos/message.ts +558 -0
- package/src/repos/serialized-writes.sqlite.test.ts +199 -0
- package/src/repos/test-helpers.ts +222 -0
- package/src/repos/thread-message.sqlite.test.ts +198 -0
- package/src/repos/thread-message.test.ts +744 -0
- package/src/repos/thread-message.ts +832 -0
- package/src/repos/thread-search-predicates.ts +79 -0
- package/src/repos/unit-of-work.sqlite.test.ts +138 -0
- package/src/repos/unit-of-work.test.ts +105 -0
- package/src/repos/unit-of-work.ts +31 -0
- package/src/schema/active-entities.ts +19 -0
- package/src/schema/i4-account-config.ts +4 -0
- package/src/schema/i4-account-export-request.ts +3 -0
- package/src/schema/i4-account-setting.ts +3 -0
- package/src/schema/i4-address.ts +3 -0
- package/src/schema/i4-mailbox-lock.ts +3 -0
- package/src/schema/i4-mailbox.ts +4 -0
- package/src/schema/i4-message-flag-push.ts +3 -0
- package/src/schema/i4-message-placement-move.ts +3 -0
- package/src/schema/i4-organize-job-request.ts +1 -0
- package/src/schema/i4-outbox-message.ts +3 -0
- package/src/schema/message-data.ts +44 -0
- package/src/schema/outbox.ts +74 -0
- package/src/schema/thread-message.ts +3 -0
- package/src/schema-full-sqlite.ts +11 -0
- package/src/schema-full.ts +16 -0
- package/src/schema.ts +28 -0
- package/src/sqlite-client.ts +52 -0
- package/src/test-db-sqlite.ts +75 -0
- package/src/test-db.ts +76 -0
- package/src/tx.ts +208 -0
- package/src/vps-migrations-drift.test.ts +34 -0
- package/tsconfig.json +8 -0
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import { type SQL, sql } from "drizzle-orm";
|
|
2
|
+
import { isSqlite } from "../dialect.js";
|
|
3
|
+
|
|
4
|
+
// Accent- and case-insensitive substring match over the whole mailbox, isolated
|
|
5
|
+
// here as the one text-search seam that genuinely differs by dialect (RFC 036
|
|
6
|
+
// D1). The subject and sender predicates match the DynamoDB `contains()`
|
|
7
|
+
// substring contract: LIKE metacharacters (`\`, `%`, `_`) are escaped in JS so
|
|
8
|
+
// the needle arrives as bind-parameter text, and the escaped form is treated
|
|
9
|
+
// literally.
|
|
10
|
+
|
|
11
|
+
const escapeLike = (term: string): string => term.replace(/[\\%_]/g, "\\$&");
|
|
12
|
+
|
|
13
|
+
// ─── Postgres ────────────────────────────────────────────────────────────────
|
|
14
|
+
// The folded expressions must reproduce the indexed expressions in
|
|
15
|
+
// npm-scripts/pg-search-index.sql exactly (immutable unaccent + lower over the
|
|
16
|
+
// coalesced text) so the planner uses the trigram GIN indexes. Matching runs
|
|
17
|
+
// over the whole mailbox — Postgres indexes the text, so there is no
|
|
18
|
+
// recent-window read bound.
|
|
19
|
+
const PG_SUBJECT_FOLDED = sql`remit_immutable_unaccent(lower(coalesce(subject, '')))`;
|
|
20
|
+
const PG_FROM_FOLDED = sql`remit_immutable_unaccent(lower(coalesce(from_name, '') || ' ' || coalesce(from_email, '')))`;
|
|
21
|
+
|
|
22
|
+
const pgLikePattern = (term: string): SQL =>
|
|
23
|
+
sql`'%' || remit_immutable_unaccent(lower(${escapeLike(term)})) || '%'`;
|
|
24
|
+
|
|
25
|
+
const pgSubjectMatch = (term: string): SQL =>
|
|
26
|
+
sql`${PG_SUBJECT_FOLDED} like ${pgLikePattern(term)}`;
|
|
27
|
+
const pgFromMatch = (term: string): SQL =>
|
|
28
|
+
sql`${PG_FROM_FOLDED} like ${pgLikePattern(term)}`;
|
|
29
|
+
|
|
30
|
+
// ─── SQLite ──────────────────────────────────────────────────────────────────
|
|
31
|
+
// Text search on SQLite is the external-content FTS5 trigram index that
|
|
32
|
+
// npm-scripts/sqlite-search-index.sql installs (RFC 036 D4): `thread_message_fts`
|
|
33
|
+
// indexes the folded subject and sender, and MATCH is an accent- and
|
|
34
|
+
// case-insensitive substring search (the tokenizer folds both sides, so the
|
|
35
|
+
// needle is passed through untransformed). The predicate is a `rowid IN
|
|
36
|
+
// (subquery)` over that index — the outer WHERE still narrows by mailbox.
|
|
37
|
+
//
|
|
38
|
+
// Trigram indexing needs three characters, so a one- or two-character term
|
|
39
|
+
// falls back to the unindexed folded LIKE scan D4 names — lower() both sides,
|
|
40
|
+
// substring-match, `escape '\'` making the JS-escaped metacharacters literal.
|
|
41
|
+
// It is case-insensitive for ASCII and does not fold diacritics; the accepted
|
|
42
|
+
// per-target difference from Postgres `unaccent`.
|
|
43
|
+
|
|
44
|
+
// FTS5 treats bare query text as its match grammar (AND/OR/NEAR/`*`/`-`/`:`), so
|
|
45
|
+
// wrap the term as a double-quoted string literal — doubling embedded quotes —
|
|
46
|
+
// to match it verbatim as a trigram substring.
|
|
47
|
+
const ftsPhrase = (term: string): string => `"${term.replace(/"/g, '""')}"`;
|
|
48
|
+
|
|
49
|
+
// Trigram tokenization is by character, so measure the term in code points, not
|
|
50
|
+
// UTF-16 units — a two-astral-character term is still under the three-char
|
|
51
|
+
// index floor.
|
|
52
|
+
const isTrigramIndexable = (term: string): boolean => [...term].length >= 3;
|
|
53
|
+
|
|
54
|
+
const ftsRowidMatch = (matchExpr: string): SQL =>
|
|
55
|
+
sql`"thread_message"."rowid" in (select "rowid" from "thread_message_fts" where "thread_message_fts" match ${matchExpr})`;
|
|
56
|
+
|
|
57
|
+
const SQLITE_SUBJECT_FOLDED = sql`lower(coalesce(subject, ''))`;
|
|
58
|
+
const SQLITE_FROM_FOLDED = sql`lower(coalesce(from_name, '') || ' ' || coalesce(from_email, ''))`;
|
|
59
|
+
|
|
60
|
+
const sqliteLikePattern = (term: string): SQL =>
|
|
61
|
+
sql`'%' || lower(${escapeLike(term)}) || '%'`;
|
|
62
|
+
|
|
63
|
+
const sqliteSubjectMatch = (term: string): SQL =>
|
|
64
|
+
isTrigramIndexable(term)
|
|
65
|
+
? ftsRowidMatch(`subject : ${ftsPhrase(term)}`)
|
|
66
|
+
: sql`${SQLITE_SUBJECT_FOLDED} like ${sqliteLikePattern(term)} escape '\\'`;
|
|
67
|
+
|
|
68
|
+
const sqliteFromMatch = (term: string): SQL =>
|
|
69
|
+
isTrigramIndexable(term)
|
|
70
|
+
? ftsRowidMatch(`sender : ${ftsPhrase(term)}`)
|
|
71
|
+
: sql`${SQLITE_FROM_FOLDED} like ${sqliteLikePattern(term)} escape '\\'`;
|
|
72
|
+
|
|
73
|
+
// ─── Dialect selection ───────────────────────────────────────────────────────
|
|
74
|
+
|
|
75
|
+
export const subjectMatch = (term: string): SQL =>
|
|
76
|
+
isSqlite() ? sqliteSubjectMatch(term) : pgSubjectMatch(term);
|
|
77
|
+
|
|
78
|
+
export const fromMatch = (term: string): SQL =>
|
|
79
|
+
isSqlite() ? sqliteFromMatch(term) : pgFromMatch(term);
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { after, before, describe, test } from "node:test";
|
|
3
|
+
import { eq } from "drizzle-orm";
|
|
4
|
+
import {
|
|
5
|
+
envelopeId as deriveEnvelopeId,
|
|
6
|
+
rootBodyPartId as deriveRootBodyPartId,
|
|
7
|
+
} from "../id.js";
|
|
8
|
+
import { addressTable } from "../schema/i4-address.js";
|
|
9
|
+
import {
|
|
10
|
+
type MessageDataSchema,
|
|
11
|
+
messageDataSchema,
|
|
12
|
+
messageTable,
|
|
13
|
+
} from "../schema/message-data.js";
|
|
14
|
+
import { threadMessageTable } from "../schema/thread-message.js";
|
|
15
|
+
import { createSqliteTestDb, type SqliteTestDb } from "../test-db-sqlite.js";
|
|
16
|
+
import { DrizzleUnitOfWork } from "./unit-of-work.js";
|
|
17
|
+
|
|
18
|
+
// The unit-of-work runs its write set inside one SAVEPOINT on sqlite (RFC 036
|
|
19
|
+
// D1) — and each repo's own `create` opens a nested SAVEPOINT, so this also
|
|
20
|
+
// covers savepoint nesting. Commit persists the whole set; a throw rolls all of
|
|
21
|
+
// it back, message and thread-message together.
|
|
22
|
+
|
|
23
|
+
const SCHEMA = {
|
|
24
|
+
...messageDataSchema,
|
|
25
|
+
threadMessage: threadMessageTable,
|
|
26
|
+
address: addressTable,
|
|
27
|
+
};
|
|
28
|
+
|
|
29
|
+
describe("DrizzleUnitOfWork (sqlite)", () => {
|
|
30
|
+
let db: SqliteTestDb<MessageDataSchema>;
|
|
31
|
+
let close: () => Promise<void>;
|
|
32
|
+
let uow: DrizzleUnitOfWork;
|
|
33
|
+
|
|
34
|
+
before(async () => {
|
|
35
|
+
({ db, close } = await createSqliteTestDb<MessageDataSchema>(SCHEMA));
|
|
36
|
+
uow = new DrizzleUnitOfWork(db);
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
after(async () => {
|
|
40
|
+
await close();
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
const messageInput = (id: string) => ({
|
|
44
|
+
messageId: id,
|
|
45
|
+
mailboxId: "00000000-0000-0000-3333-000000000002",
|
|
46
|
+
uid: 7,
|
|
47
|
+
sequenceNumber: 1,
|
|
48
|
+
rfc822Size: 512,
|
|
49
|
+
internalDate: 1700000000000,
|
|
50
|
+
envelopeId: deriveEnvelopeId(id),
|
|
51
|
+
rootBodyPartId: deriveRootBodyPartId(id),
|
|
52
|
+
status: "active" as const,
|
|
53
|
+
syncStatus: "synced" as const,
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
const threadInput = (id: string) => ({
|
|
57
|
+
accountConfigId: "acct-1",
|
|
58
|
+
threadId: `thread-${id}`,
|
|
59
|
+
messageId: id,
|
|
60
|
+
mailboxId: "mbx-1",
|
|
61
|
+
uid: 7,
|
|
62
|
+
referenceOrder: 0,
|
|
63
|
+
internalDate: 1700000000000,
|
|
64
|
+
sentDate: 1700000000000,
|
|
65
|
+
isRead: false,
|
|
66
|
+
isDeleted: false,
|
|
67
|
+
hasAttachment: false,
|
|
68
|
+
hasStars: false,
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
test("commits message + thread-message together", async () => {
|
|
72
|
+
const id = "00000000-0000-0000-3333-000000000010";
|
|
73
|
+
await uow.transaction(async (repos) => {
|
|
74
|
+
await repos.message.create(messageInput(id));
|
|
75
|
+
await repos.threadMessage.create(threadInput(id));
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
const msg = await db
|
|
79
|
+
.select()
|
|
80
|
+
.from(messageTable)
|
|
81
|
+
.where(eq(messageTable.messageId, id));
|
|
82
|
+
const thread = await db
|
|
83
|
+
.select()
|
|
84
|
+
.from(threadMessageTable)
|
|
85
|
+
.where(eq(threadMessageTable.messageId, id));
|
|
86
|
+
assert.equal(msg.length, 1);
|
|
87
|
+
assert.equal(thread.length, 1);
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
test("a throw rolls the whole set back", async () => {
|
|
91
|
+
const id = "00000000-0000-0000-3333-000000000011";
|
|
92
|
+
await assert.rejects(() =>
|
|
93
|
+
uow.transaction(async (repos) => {
|
|
94
|
+
await repos.message.create(messageInput(id));
|
|
95
|
+
await repos.threadMessage.create(threadInput(id));
|
|
96
|
+
throw new Error("boom");
|
|
97
|
+
}),
|
|
98
|
+
);
|
|
99
|
+
|
|
100
|
+
const msg = await db
|
|
101
|
+
.select()
|
|
102
|
+
.from(messageTable)
|
|
103
|
+
.where(eq(messageTable.messageId, id));
|
|
104
|
+
const thread = await db
|
|
105
|
+
.select()
|
|
106
|
+
.from(threadMessageTable)
|
|
107
|
+
.where(eq(threadMessageTable.messageId, id));
|
|
108
|
+
assert.equal(msg.length, 0, "message insert must roll back");
|
|
109
|
+
assert.equal(thread.length, 0, "thread-message insert must roll back");
|
|
110
|
+
});
|
|
111
|
+
|
|
112
|
+
test("concurrent transactions serialize without corrupting savepoints", async () => {
|
|
113
|
+
const ids = Array.from(
|
|
114
|
+
{ length: 12 },
|
|
115
|
+
(_, i) =>
|
|
116
|
+
`00000000-0000-0000-3333-0000000001${i.toString().padStart(2, "0")}`,
|
|
117
|
+
);
|
|
118
|
+
|
|
119
|
+
// The failure this guards: shared-connection savepoint interleaving when
|
|
120
|
+
// callers run under concurrency (message-sync's pMap). All must commit.
|
|
121
|
+
await Promise.all(
|
|
122
|
+
ids.map((id) =>
|
|
123
|
+
uow.transaction(async (repos) => {
|
|
124
|
+
await repos.message.create(messageInput(id));
|
|
125
|
+
await repos.threadMessage.create(threadInput(id));
|
|
126
|
+
}),
|
|
127
|
+
),
|
|
128
|
+
);
|
|
129
|
+
|
|
130
|
+
for (const id of ids) {
|
|
131
|
+
const msg = await db
|
|
132
|
+
.select()
|
|
133
|
+
.from(messageTable)
|
|
134
|
+
.where(eq(messageTable.messageId, id));
|
|
135
|
+
assert.equal(msg.length, 1, `message ${id} must be committed`);
|
|
136
|
+
}
|
|
137
|
+
});
|
|
138
|
+
});
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { after, before, describe, test } from "node:test";
|
|
3
|
+
import { eq } from "drizzle-orm";
|
|
4
|
+
import {
|
|
5
|
+
envelopeId as deriveEnvelopeId,
|
|
6
|
+
rootBodyPartId as deriveRootBodyPartId,
|
|
7
|
+
} from "../id.js";
|
|
8
|
+
import {
|
|
9
|
+
envelopeTable,
|
|
10
|
+
messageTable,
|
|
11
|
+
outboxTable,
|
|
12
|
+
} from "../schema/message-data.js";
|
|
13
|
+
import { createTestDb, type TestDb } from "./test-helpers.js";
|
|
14
|
+
import { DrizzleUnitOfWork } from "./unit-of-work.js";
|
|
15
|
+
|
|
16
|
+
describe("DrizzleUnitOfWork", () => {
|
|
17
|
+
let db: TestDb;
|
|
18
|
+
let stop: () => Promise<void>;
|
|
19
|
+
let unitOfWork: DrizzleUnitOfWork;
|
|
20
|
+
|
|
21
|
+
const MESSAGE_ID = "00000000-0000-0000-2222-000000000001";
|
|
22
|
+
const MAILBOX_ID = "00000000-0000-0000-2222-000000000002";
|
|
23
|
+
const NOW = 1700000000000;
|
|
24
|
+
|
|
25
|
+
const writeEnvelopeAndMessage = async (
|
|
26
|
+
repos: Parameters<Parameters<DrizzleUnitOfWork["transaction"]>[0]>[0],
|
|
27
|
+
) => {
|
|
28
|
+
await repos.envelope.upsertEnvelope({
|
|
29
|
+
envelopeId: deriveEnvelopeId(MESSAGE_ID),
|
|
30
|
+
messageId: MESSAGE_ID,
|
|
31
|
+
dateValue: NOW,
|
|
32
|
+
dateRaw: "Tue, 14 Nov 2023 22:13:20 +0000",
|
|
33
|
+
subject: "hello",
|
|
34
|
+
messageIdValue: "<hello@example.com>",
|
|
35
|
+
});
|
|
36
|
+
await repos.message.upsertWithStatus({
|
|
37
|
+
messageId: MESSAGE_ID,
|
|
38
|
+
mailboxId: MAILBOX_ID,
|
|
39
|
+
uid: 7,
|
|
40
|
+
sequenceNumber: 1,
|
|
41
|
+
rfc822Size: 1024,
|
|
42
|
+
internalDate: NOW,
|
|
43
|
+
envelopeId: deriveEnvelopeId(MESSAGE_ID),
|
|
44
|
+
rootBodyPartId: deriveRootBodyPartId(MESSAGE_ID),
|
|
45
|
+
});
|
|
46
|
+
};
|
|
47
|
+
|
|
48
|
+
const rows = async () => {
|
|
49
|
+
const envelopes = await db
|
|
50
|
+
.select()
|
|
51
|
+
.from(envelopeTable)
|
|
52
|
+
.where(eq(envelopeTable.messageId, MESSAGE_ID));
|
|
53
|
+
const messages = await db
|
|
54
|
+
.select()
|
|
55
|
+
.from(messageTable)
|
|
56
|
+
.where(eq(messageTable.messageId, MESSAGE_ID));
|
|
57
|
+
const outbox = await db
|
|
58
|
+
.select()
|
|
59
|
+
.from(outboxTable)
|
|
60
|
+
.where(eq(outboxTable.messageId, MESSAGE_ID));
|
|
61
|
+
return { envelopes, messages, outbox };
|
|
62
|
+
};
|
|
63
|
+
|
|
64
|
+
before(async () => {
|
|
65
|
+
({ db, stop } = await createTestDb());
|
|
66
|
+
unitOfWork = new DrizzleUnitOfWork(db);
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
after(async () => {
|
|
70
|
+
await stop();
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
test("a mid-save throw rolls back the whole write set, outbox included", async () => {
|
|
74
|
+
await assert.rejects(
|
|
75
|
+
() =>
|
|
76
|
+
unitOfWork.transaction(async (repos) => {
|
|
77
|
+
await writeEnvelopeAndMessage(repos);
|
|
78
|
+
// A later write in the set fails after the envelope, message and its
|
|
79
|
+
// transactional-outbox row have already been written.
|
|
80
|
+
throw new Error("thread write failed");
|
|
81
|
+
}),
|
|
82
|
+
/thread write failed/,
|
|
83
|
+
);
|
|
84
|
+
|
|
85
|
+
const { envelopes, messages, outbox } = await rows();
|
|
86
|
+
assert.equal(envelopes.length, 0, "envelope must be rolled back");
|
|
87
|
+
assert.equal(messages.length, 0, "message must be rolled back");
|
|
88
|
+
assert.equal(
|
|
89
|
+
outbox.length,
|
|
90
|
+
0,
|
|
91
|
+
"outbox row must be rolled back with the message",
|
|
92
|
+
);
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
test("a successful transaction commits the data rows and the outbox row", async () => {
|
|
96
|
+
await unitOfWork.transaction(writeEnvelopeAndMessage);
|
|
97
|
+
|
|
98
|
+
const { envelopes, messages, outbox } = await rows();
|
|
99
|
+
assert.equal(envelopes.length, 1);
|
|
100
|
+
assert.equal(messages.length, 1);
|
|
101
|
+
assert.equal(outbox.length, 1);
|
|
102
|
+
assert.equal(outbox[0].event, "message.created");
|
|
103
|
+
assert.deepStrictEqual(outbox[0].payload, { messageId: MESSAGE_ID });
|
|
104
|
+
});
|
|
105
|
+
});
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import type { IUnitOfWork, UnitOfWorkRepositories } from "@remit/data-ports";
|
|
2
|
+
import type { Db } from "../db.js";
|
|
3
|
+
import type { MessageDataSchema } from "../schema/message-data.js";
|
|
4
|
+
import { runInTransaction } from "../tx.js";
|
|
5
|
+
import { DrizzleEnvelopeRepository } from "./envelope.js";
|
|
6
|
+
import { AddressRepo } from "./i4-address.js";
|
|
7
|
+
import { DrizzleMessageRepository } from "./message.js";
|
|
8
|
+
import { DrizzleThreadMessageRepository } from "./thread-message.js";
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Runs a write set inside a single Postgres transaction. The repositories handed
|
|
12
|
+
* to the callback are bound to that transaction, so the data rows and the
|
|
13
|
+
* transactional-outbox rows the message write appends commit atomically — a
|
|
14
|
+
* throw anywhere rolls the whole set back, outbox included.
|
|
15
|
+
*/
|
|
16
|
+
export class DrizzleUnitOfWork implements IUnitOfWork {
|
|
17
|
+
constructor(private db: Db<MessageDataSchema>) {}
|
|
18
|
+
|
|
19
|
+
transaction<T>(
|
|
20
|
+
fn: (repos: UnitOfWorkRepositories) => Promise<T>,
|
|
21
|
+
): Promise<T> {
|
|
22
|
+
return runInTransaction(this.db, (tx) =>
|
|
23
|
+
fn({
|
|
24
|
+
message: new DrizzleMessageRepository(tx),
|
|
25
|
+
envelope: new DrizzleEnvelopeRepository(tx),
|
|
26
|
+
address: new AddressRepo(tx),
|
|
27
|
+
threadMessage: new DrizzleThreadMessageRepository(tx),
|
|
28
|
+
}),
|
|
29
|
+
);
|
|
30
|
+
}
|
|
31
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
// The dialect-selected entity table set (RFC 036 D1). Both generated packages
|
|
2
|
+
// export the same table symbols under the same names — they differ only in the
|
|
3
|
+
// column builders (`pgTable`/`jsonb`/`timestamp` vs `sqliteTable`/`text(json)`/
|
|
4
|
+
// `integer`). A process runs one dialect (see ../dialect.ts), so the active set
|
|
5
|
+
// is chosen once here and re-exported through the schema facades the repos
|
|
6
|
+
// import from.
|
|
7
|
+
//
|
|
8
|
+
// The cast to the Postgres module type is the single "typing loosens at the
|
|
9
|
+
// injection boundary" point RFC 036 D1 names: the repos are written once
|
|
10
|
+
// against the Postgres-typed shape, and at runtime the SQLite tables carry
|
|
11
|
+
// their own dialect so the queries generate the correct SQL. The backend
|
|
12
|
+
// already crosses this boundary with a cast on the db handle.
|
|
13
|
+
import * as pgEntities from "@remit/drizzle-pg-schema";
|
|
14
|
+
import * as sqliteEntities from "@remit/drizzle-sqlite-schema";
|
|
15
|
+
import { isSqlite } from "../dialect.js";
|
|
16
|
+
|
|
17
|
+
export const entities: typeof pgEntities = isSqlite()
|
|
18
|
+
? (sqliteEntities as unknown as typeof pgEntities)
|
|
19
|
+
: pgEntities;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { organizeJobRequests as organizeJobRequestTable } from "@remit/drizzle-pg-schema";
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { entities } from "./active-entities.js";
|
|
2
|
+
import { outboxTable } from "./outbox.js";
|
|
3
|
+
|
|
4
|
+
export { outboxTable };
|
|
5
|
+
|
|
6
|
+
const bodyPartContentTable = entities.bodyPartContents;
|
|
7
|
+
const bodyPartParameterTable = entities.bodyPartParameters;
|
|
8
|
+
const bodyPartStorageTable = entities.bodyPartStorages;
|
|
9
|
+
const bodyPartTable = entities.bodyParts;
|
|
10
|
+
const envelopeAddressTable = entities.envelopeAddresses;
|
|
11
|
+
const envelopeTable = entities.envelopes;
|
|
12
|
+
const messageFlagTable = entities.messageFlags;
|
|
13
|
+
const messageReferenceTable = entities.messageReferences;
|
|
14
|
+
const messageTable = entities.messages;
|
|
15
|
+
const rawMessageStorageTable = entities.rawMessageStorages;
|
|
16
|
+
|
|
17
|
+
export {
|
|
18
|
+
bodyPartContentTable,
|
|
19
|
+
bodyPartParameterTable,
|
|
20
|
+
bodyPartStorageTable,
|
|
21
|
+
bodyPartTable,
|
|
22
|
+
envelopeAddressTable,
|
|
23
|
+
envelopeTable,
|
|
24
|
+
messageFlagTable,
|
|
25
|
+
messageReferenceTable,
|
|
26
|
+
messageTable,
|
|
27
|
+
rawMessageStorageTable,
|
|
28
|
+
};
|
|
29
|
+
|
|
30
|
+
export const messageDataSchema = {
|
|
31
|
+
envelope: envelopeTable,
|
|
32
|
+
messageReference: messageReferenceTable,
|
|
33
|
+
envelopeAddress: envelopeAddressTable,
|
|
34
|
+
bodyPart: bodyPartTable,
|
|
35
|
+
bodyPartParameter: bodyPartParameterTable,
|
|
36
|
+
rawMessageStorage: rawMessageStorageTable,
|
|
37
|
+
bodyPartStorage: bodyPartStorageTable,
|
|
38
|
+
bodyPartContent: bodyPartContentTable,
|
|
39
|
+
message: messageTable,
|
|
40
|
+
messageFlag: messageFlagTable,
|
|
41
|
+
outbox: outboxTable,
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
export type MessageDataSchema = typeof messageDataSchema;
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import { sql } from "drizzle-orm";
|
|
2
|
+
import {
|
|
3
|
+
bigint,
|
|
4
|
+
jsonb,
|
|
5
|
+
index as pgIndex,
|
|
6
|
+
pgTable,
|
|
7
|
+
text as pgText,
|
|
8
|
+
timestamp,
|
|
9
|
+
uuid,
|
|
10
|
+
} from "drizzle-orm/pg-core";
|
|
11
|
+
import {
|
|
12
|
+
index as sqliteIndex,
|
|
13
|
+
integer as sqliteInteger,
|
|
14
|
+
sqliteTable,
|
|
15
|
+
text as sqliteText,
|
|
16
|
+
} from "drizzle-orm/sqlite-core";
|
|
17
|
+
import { isSqlite } from "../dialect.js";
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Transactional outbox. It has no TypeSpec entity — it is infrastructure for
|
|
21
|
+
* the search-index worker (append a row per body change / move, drain by id,
|
|
22
|
+
* mark `processed_at`). The partial index selects unprocessed rows for the
|
|
23
|
+
* boot-time backstop scan (Postgres) and the short-cadence poll (SQLite,
|
|
24
|
+
* RFC 036 D2). It is hand-written per dialect because the two column-builder
|
|
25
|
+
* sets share no surface; both keep identical column names so the repos and the
|
|
26
|
+
* drain logic read the same rows on either backend.
|
|
27
|
+
*
|
|
28
|
+
* Both raw tables are exported for committed-migration generation (schema-full
|
|
29
|
+
* per dialect). The runtime `outboxTable` is the dialect-selected one, cast to
|
|
30
|
+
* the Postgres type so the repos keep one static shape (RFC 036 D1).
|
|
31
|
+
*/
|
|
32
|
+
export const pgOutboxTable = pgTable(
|
|
33
|
+
"outbox",
|
|
34
|
+
{
|
|
35
|
+
id: uuid("id").primaryKey(),
|
|
36
|
+
messageId: pgText("message_id").notNull(),
|
|
37
|
+
event: pgText("event").notNull(),
|
|
38
|
+
payload: jsonb("payload").notNull(),
|
|
39
|
+
createdAt: timestamp("created_at", { withTimezone: true })
|
|
40
|
+
.defaultNow()
|
|
41
|
+
.notNull(),
|
|
42
|
+
processedAt: bigint("processed_at", { mode: "number" }),
|
|
43
|
+
},
|
|
44
|
+
(t) => [
|
|
45
|
+
pgIndex("outbox_message_id_idx").on(t.messageId),
|
|
46
|
+
pgIndex("outbox_unprocessed_idx")
|
|
47
|
+
.on(t.createdAt)
|
|
48
|
+
.where(sql`${t.processedAt} IS NULL`),
|
|
49
|
+
],
|
|
50
|
+
);
|
|
51
|
+
|
|
52
|
+
export const sqliteOutboxTable = sqliteTable(
|
|
53
|
+
"outbox",
|
|
54
|
+
{
|
|
55
|
+
id: sqliteText("id").primaryKey(),
|
|
56
|
+
messageId: sqliteText("message_id").notNull(),
|
|
57
|
+
event: sqliteText("event").notNull(),
|
|
58
|
+
payload: sqliteText("payload", { mode: "json" }).notNull(),
|
|
59
|
+
createdAt: sqliteInteger("created_at", { mode: "timestamp_ms" })
|
|
60
|
+
.$defaultFn(() => new Date())
|
|
61
|
+
.notNull(),
|
|
62
|
+
processedAt: sqliteInteger("processed_at", { mode: "number" }),
|
|
63
|
+
},
|
|
64
|
+
(t) => [
|
|
65
|
+
sqliteIndex("outbox_message_id_idx").on(t.messageId),
|
|
66
|
+
sqliteIndex("outbox_unprocessed_idx")
|
|
67
|
+
.on(t.createdAt)
|
|
68
|
+
.where(sql`${t.processedAt} IS NULL`),
|
|
69
|
+
],
|
|
70
|
+
);
|
|
71
|
+
|
|
72
|
+
export const outboxTable: typeof pgOutboxTable = isSqlite()
|
|
73
|
+
? (sqliteOutboxTable as unknown as typeof pgOutboxTable)
|
|
74
|
+
: pgOutboxTable;
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
// Complete SQLite drizzle schema for committed-migration GENERATION only
|
|
2
|
+
// (RFC 036 D5) — consumed by deploy/vps/migrate/drizzle.entities.sqlite.config.ts
|
|
3
|
+
// and the drift guard (npm-scripts/check-vps-migrations.mjs), never at runtime.
|
|
4
|
+
//
|
|
5
|
+
// The SQLite twin of schema-full.ts: the sqlite-dialect entity package
|
|
6
|
+
// wholesale (`sqliteTable`/`text(json)`/`integer`) plus the raw sqlite outbox
|
|
7
|
+
// infra table. Kept separate from the runtime facade (../schema/active-entities)
|
|
8
|
+
// because drizzle-kit's `generate --dialect sqlite` needs the real sqlite table
|
|
9
|
+
// objects, not the pg-cast the repos consume.
|
|
10
|
+
export * from "@remit/drizzle-sqlite-schema";
|
|
11
|
+
export { sqliteOutboxTable as outboxTable } from "./schema/outbox.js";
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
// Complete Postgres drizzle schema for committed-migration GENERATION only —
|
|
2
|
+
// consumed by deploy/vps/migrate/drizzle.entities.config.ts and the drift guard
|
|
3
|
+
// (npm-scripts/check-vps-migrations.mjs), never by `pushSchema`.
|
|
4
|
+
//
|
|
5
|
+
// It pulls the generated entity package in wholesale, so a new TypeSpec entity
|
|
6
|
+
// flows into the committed migration with nothing to hand-maintain — the
|
|
7
|
+
// omission that let eight tables drift out of the deployed schema. The only
|
|
8
|
+
// addition is the `outbox` infra table, which has no entity. This file imports
|
|
9
|
+
// the raw pg outbox directly (not the dialect-selected `outboxTable`), so
|
|
10
|
+
// migration generation never depends on the runtime `DATA_BACKEND`.
|
|
11
|
+
//
|
|
12
|
+
// schema.ts stays the app/dev surface (single `*Table` alias per table, what
|
|
13
|
+
// the repos and `pushSchema` need); this file exposes canonical names, so it
|
|
14
|
+
// must not be fed to `pushSchema` alongside schema.ts (duplicate index names).
|
|
15
|
+
export * from "@remit/drizzle-pg-schema";
|
|
16
|
+
export { pgOutboxTable as outboxTable } from "./schema/outbox.js";
|
package/src/schema.ts
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
// App/dev-facing drizzle schema: every table under a single `*Table` alias,
|
|
2
|
+
// the names the repos and `pushSchema` (test-db.ts) consume. Each table appears
|
|
3
|
+
// exactly once here — `pushSchema` registers a table's indexes per exported
|
|
4
|
+
// binding, so exposing one table under two names creates duplicate index names
|
|
5
|
+
// and breaks `apply()`. The committed-migration `generate` reads schema-full.ts
|
|
6
|
+
// instead (the entity package wholesale), so the migration is driven by the
|
|
7
|
+
// entities, not by this hand-maintained alias list.
|
|
8
|
+
|
|
9
|
+
import { entities } from "./schema/active-entities.js";
|
|
10
|
+
|
|
11
|
+
export const filterAnchorTable = entities.filterAnchors;
|
|
12
|
+
export const filterTable = entities.filters;
|
|
13
|
+
export const labelTable = entities.labels;
|
|
14
|
+
export const mailboxAttributeEntryTable = entities.mailboxAttributeEntries;
|
|
15
|
+
export const mailboxFlagTable = entities.mailboxFlags;
|
|
16
|
+
export const messageLabelTable = entities.messageLabels;
|
|
17
|
+
export * from "./schema/i4-account-config.js";
|
|
18
|
+
export * from "./schema/i4-account-export-request.js";
|
|
19
|
+
export * from "./schema/i4-account-setting.js";
|
|
20
|
+
export * from "./schema/i4-address.js";
|
|
21
|
+
export * from "./schema/i4-mailbox.js";
|
|
22
|
+
export * from "./schema/i4-mailbox-lock.js";
|
|
23
|
+
export * from "./schema/i4-message-flag-push.js";
|
|
24
|
+
export * from "./schema/i4-message-placement-move.js";
|
|
25
|
+
export * from "./schema/i4-organize-job-request.js";
|
|
26
|
+
export * from "./schema/i4-outbox-message.js";
|
|
27
|
+
export * from "./schema/message-data.js";
|
|
28
|
+
export { threadMessageTable } from "./schema/thread-message.js";
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import type { Db } from "./db.js";
|
|
2
|
+
import { serializeSqliteWrites } from "./tx.js";
|
|
3
|
+
|
|
4
|
+
// The one place a SQLite connection is opened for the app (RFC 036 D3). Every
|
|
5
|
+
// writer container (backend, imap-worker, smtp-worker, account-worker) opens the
|
|
6
|
+
// same file here with the same cross-process settings — WAL for concurrent
|
|
7
|
+
// readers, a 5 s busy_timeout so a writer waits out another's short transaction
|
|
8
|
+
// instead of failing, synchronous=NORMAL (durable under WAL), foreign keys on.
|
|
9
|
+
//
|
|
10
|
+
// The handle is wrapped by `serializeSqliteWrites` before it leaves this
|
|
11
|
+
// function, so a repo's insert/update/delete cannot bypass the in-process write
|
|
12
|
+
// serialization (RFC 036 D3). `run`/`transaction`/reads pass through by design —
|
|
13
|
+
// see the wrapper's comment.
|
|
14
|
+
//
|
|
15
|
+
// better-sqlite3 and its drizzle driver are imported dynamically so the Postgres
|
|
16
|
+
// path never loads the native binding, and so the whole module stays out of the
|
|
17
|
+
// DynamoDB Lambda bundle (this package is `external` there — see
|
|
18
|
+
// remit-backend/src/service/dynamodb.ts).
|
|
19
|
+
|
|
20
|
+
export interface SqliteClientOptions {
|
|
21
|
+
filename: string;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export interface SqliteClient<TSchema extends Record<string, unknown>> {
|
|
25
|
+
db: Db<TSchema>;
|
|
26
|
+
close: () => Promise<void>;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export async function createSqliteDatabase<
|
|
30
|
+
TSchema extends Record<string, unknown>,
|
|
31
|
+
>(
|
|
32
|
+
schema: TSchema,
|
|
33
|
+
options: SqliteClientOptions,
|
|
34
|
+
): Promise<SqliteClient<TSchema>> {
|
|
35
|
+
const { default: Database } = await import("better-sqlite3");
|
|
36
|
+
const { drizzle } = await import("drizzle-orm/better-sqlite3");
|
|
37
|
+
|
|
38
|
+
const sqlite = new Database(options.filename);
|
|
39
|
+
sqlite.pragma("journal_mode = WAL");
|
|
40
|
+
sqlite.pragma("busy_timeout = 5000");
|
|
41
|
+
sqlite.pragma("synchronous = NORMAL");
|
|
42
|
+
sqlite.pragma("foreign_keys = ON");
|
|
43
|
+
|
|
44
|
+
const base = drizzle(sqlite, { schema }) as unknown as Db<TSchema>;
|
|
45
|
+
|
|
46
|
+
return {
|
|
47
|
+
db: serializeSqliteWrites(base),
|
|
48
|
+
close: async () => {
|
|
49
|
+
sqlite.close();
|
|
50
|
+
},
|
|
51
|
+
};
|
|
52
|
+
}
|