@coffre/db 0.0.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +19 -3
- package/dist/commits-BuW-NL3n.js +30 -0
- package/dist/connect.d.ts +26 -0
- package/dist/connect.js +156 -0
- package/dist/database-CWpo5aF0.d.ts +39 -0
- package/dist/dialect.d.ts +88 -0
- package/dist/dialect.js +128 -0
- package/dist/hyperdrive.d.ts +32 -0
- package/dist/hyperdrive.js +54 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.js +34 -0
- package/dist/log.d.ts +72 -0
- package/dist/log.js +133 -0
- package/dist/migrate.d.ts +10 -0
- package/dist/migrate.js +130 -0
- package/dist/migrations/postgres/0000_baseline.sql +505 -0
- package/dist/migrations/postgres/meta/0000_snapshot.json +1939 -0
- package/dist/migrations/postgres/meta/_journal.json +13 -0
- package/dist/migrations/sqlite/0000_baseline.sql +293 -0
- package/dist/migrations/sqlite/meta/0000_snapshot.json +1811 -0
- package/dist/migrations/sqlite/meta/_journal.json +13 -0
- package/dist/portable.d.ts +33 -0
- package/dist/portable.js +12 -0
- package/dist/postgres-Bmar8y0q.js +29 -0
- package/dist/relations-DqgxtlKu.js +58 -0
- package/dist/rolldown-runtime-D7D4PA-g.js +13 -0
- package/dist/schema-Cd4WOnB5.d.ts +2503 -0
- package/dist/schema-sqlite.d.ts +2587 -0
- package/dist/schema-sqlite.js +401 -0
- package/dist/schema-version.d.ts +11 -0
- package/dist/schema-version.js +16 -0
- package/dist/schema.d.ts +2 -0
- package/dist/schema.js +448 -0
- package/package.json +92 -5
- package/src/baseline/postgres.sql +241 -0
- package/src/baseline/sqlite.sql +27 -0
- package/src/baseline.ts +104 -0
- package/src/commits.ts +47 -0
- package/src/connect.ts +171 -0
- package/src/database.ts +63 -0
- package/src/dialect.ts +174 -0
- package/src/drizzle.postgres.config.ts +7 -0
- package/src/drizzle.sqlite.config.ts +7 -0
- package/src/hyperdrive.ts +63 -0
- package/src/log.ts +177 -0
- package/src/migrate-main.ts +8 -0
- package/src/migrate.ts +207 -0
- package/src/migrations/postgres/0000_baseline.sql +505 -0
- package/src/migrations/postgres/meta/0000_snapshot.json +1939 -0
- package/src/migrations/postgres/meta/_journal.json +13 -0
- package/src/migrations/sqlite/0000_baseline.sql +293 -0
- package/src/migrations/sqlite/meta/0000_snapshot.json +1811 -0
- package/src/migrations/sqlite/meta/_journal.json +13 -0
- package/src/portable.ts +54 -0
- package/src/postgres.ts +35 -0
- package/src/relations.ts +53 -0
- package/src/schema-version.ts +13 -0
- package/src/schema.sqlite.ts +511 -0
- package/src/schema.ts +548 -0
- package/src/verify-runtime.ts +122 -0
package/src/dialect.ts
ADDED
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
import { entityKind, getTableColumns, sql, type SQL } from 'drizzle-orm';
|
|
2
|
+
import type { PgTable } from 'drizzle-orm/pg-core';
|
|
3
|
+
|
|
4
|
+
import type { Queryable } from './database.ts';
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Differences between Postgres and the SQLite used for tests and local dev.
|
|
8
|
+
* Queries stay in the server's queries.ts; it never asks which database it has.
|
|
9
|
+
*
|
|
10
|
+
* Postgres SQLite
|
|
11
|
+
* isolation READ COMMITTED one writer at a time
|
|
12
|
+
* lock a row SELECT … FOR UPDATE one writer at a time
|
|
13
|
+
* rows changed rowCount rowsAffected
|
|
14
|
+
* duplicate key SQLSTATE 23505 SQLITE_CONSTRAINT_UNIQUE
|
|
15
|
+
* clock CURRENT_TIMESTAMP strftime(…, 'now')
|
|
16
|
+
* clock, in ms clock_timestamp() unixepoch('subsec')
|
|
17
|
+
* a condition read true / false 1 / 0
|
|
18
|
+
*
|
|
19
|
+
* Nothing here imports SQLite code, so the Worker bundles none.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
export type Table = PgTable;
|
|
23
|
+
|
|
24
|
+
export type Engine = 'postgres' | 'sqlite';
|
|
25
|
+
|
|
26
|
+
/** Which database this is, from the Drizzle dialect it was opened with. */
|
|
27
|
+
export function engineOf(db: Queryable): Engine {
|
|
28
|
+
const kind = (db as unknown as { dialect: { constructor: { [entityKind]: string } } }).dialect.constructor[entityKind];
|
|
29
|
+
if (kind.startsWith('Pg')) return 'postgres';
|
|
30
|
+
if (kind.startsWith('SQLite')) return 'sqlite';
|
|
31
|
+
throw new Error(`unsupported database: ${kind}`);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Lock the rows a select reads until the transaction ends. SQLite has no row
|
|
36
|
+
* locks and needs none: connect.ts runs one transaction at a time, and each
|
|
37
|
+
* holds the database's write lock from its first statement.
|
|
38
|
+
*
|
|
39
|
+
* `no key update` is for rows others reference by foreign key: it keeps out
|
|
40
|
+
* every other writer of the row, but not an insert that points at it.
|
|
41
|
+
*/
|
|
42
|
+
export function forUpdate<Query extends { for: (strength: 'update' | 'no key update') => unknown }>(
|
|
43
|
+
db: Queryable,
|
|
44
|
+
query: Query,
|
|
45
|
+
strength: 'update' | 'no key update' = 'update',
|
|
46
|
+
): ReturnType<Query['for']> {
|
|
47
|
+
if (engineOf(db) === 'sqlite') return query as ReturnType<Query['for']>;
|
|
48
|
+
return query.for(strength) as ReturnType<Query['for']>;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** Insert the rows whose unique keys are free; returns how many that was. */
|
|
52
|
+
export async function insertIfAbsent(db: Queryable, table: Table, rows: object[]): Promise<number> {
|
|
53
|
+
return changedRows(await db.insert(table).values(rows as never).onConflictDoNothing());
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Insert the rows, or where one repeats the unique key `target`, overwrite
|
|
58
|
+
* `columns` of the existing row with the values it was about to insert.
|
|
59
|
+
*/
|
|
60
|
+
export async function upsert<T extends Table>(
|
|
61
|
+
db: Queryable,
|
|
62
|
+
table: T,
|
|
63
|
+
rows: object[],
|
|
64
|
+
target: (keyof T['$inferSelect'])[],
|
|
65
|
+
columns: (keyof T['$inferSelect'])[],
|
|
66
|
+
): Promise<void> {
|
|
67
|
+
const all = getTableColumns(table) as Record<string, ReturnType<typeof getTableColumns>[string]>;
|
|
68
|
+
const incoming = (name: string) => sql`excluded.${sql.identifier(all[name].name)}`;
|
|
69
|
+
const set = Object.fromEntries(columns.map((name) => [name, incoming(name as string)]));
|
|
70
|
+
const query = db.insert(table).values(rows as never);
|
|
71
|
+
await query.onConflictDoUpdate({
|
|
72
|
+
target: target.map((name) => all[name as string]) as never,
|
|
73
|
+
set: set as never,
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** A condition read as a boolean: Postgres returns true or false, SQLite 1 or 0. */
|
|
78
|
+
export function truth(condition: SQL): SQL<boolean> {
|
|
79
|
+
return condition.mapWith((value) => Number(value) === 1);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/** How many rows an insert, update or delete touched. */
|
|
83
|
+
export function changedRows(result: unknown): number {
|
|
84
|
+
const header = result as { rowCount?: number | null; rowsAffected?: number };
|
|
85
|
+
return header.rowCount ?? header.rowsAffected ?? 0;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
const DUPLICATE_KEY = new Set(['23505', 'SQLITE_CONSTRAINT_UNIQUE', 'SQLITE_CONSTRAINT_PRIMARYKEY']);
|
|
89
|
+
|
|
90
|
+
/** Whether an insert or update failed on a unique constraint. */
|
|
91
|
+
export function isUniqueViolation(error: unknown): boolean {
|
|
92
|
+
for (let cause = error; typeof cause === 'object' && cause !== null; ) {
|
|
93
|
+
const { code, extendedCode } = cause as { code?: unknown; extendedCode?: unknown };
|
|
94
|
+
if (DUPLICATE_KEY.has(code as string) || DUPLICATE_KEY.has(extendedCode as string)) return true;
|
|
95
|
+
cause = (cause as { cause?: unknown }).cause;
|
|
96
|
+
}
|
|
97
|
+
return false;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* A consistent read of several statements that cannot write, such as
|
|
102
|
+
* verifying the chain. SQLite ignores it: its transactions already see one
|
|
103
|
+
* snapshot, and connect.ts runs them one at a time.
|
|
104
|
+
*/
|
|
105
|
+
export const SNAPSHOT = { isolationLevel: 'repeatable read', accessMode: 'read only' } as const;
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* The database clock as text, for canonicalTimestamp. Entries from several
|
|
109
|
+
* servers order by it, so it is never an application server's.
|
|
110
|
+
*/
|
|
111
|
+
export function clock(db: Queryable): SQL<string> {
|
|
112
|
+
switch (engineOf(db)) {
|
|
113
|
+
case 'postgres':
|
|
114
|
+
return sql<string>`CURRENT_TIMESTAMP`;
|
|
115
|
+
case 'sqlite':
|
|
116
|
+
return sql<string>`strftime('%Y-%m-%dT%H:%M:%fZ', 'now')`;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* The database clock as milliseconds since the epoch, as the statement runs:
|
|
122
|
+
* Postgres's `clock_timestamp()`, not `CURRENT_TIMESTAMP`, which is the
|
|
123
|
+
* transaction's start and would date an entry that waited for the log's lock
|
|
124
|
+
* before one written while it waited. Read it in a statement after the lock.
|
|
125
|
+
*/
|
|
126
|
+
export function clockMillis(db: Queryable): SQL<number> {
|
|
127
|
+
switch (engineOf(db)) {
|
|
128
|
+
case 'postgres':
|
|
129
|
+
return sql`(extract(epoch from clock_timestamp()) * 1000)::bigint`.mapWith(Number);
|
|
130
|
+
case 'sqlite':
|
|
131
|
+
return sql`CAST(unixepoch('subsec') * 1000 AS INTEGER)`.mapWith(Number);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* The migrator's own ledger, read by readiness to tell a database that is
|
|
137
|
+
* up but not yet migrated from one that is ready. Drizzle keeps it in a
|
|
138
|
+
* `drizzle` schema on Postgres, and in a plain table elsewhere.
|
|
139
|
+
*/
|
|
140
|
+
export function migrationLedger(db: Queryable): SQL {
|
|
141
|
+
const table = sql.identifier('__drizzle_migrations');
|
|
142
|
+
return engineOf(db) === 'postgres' ? sql`${sql.identifier('drizzle')}.${table}` : sql`${table}`;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
const TIMESTAMP =
|
|
146
|
+
/^(\d{4})-(\d{2})-(\d{2})[T ](\d{2}):(\d{2}):(\d{2})(?:\.(\d{1,6}))?(Z|[+-]\d{2}(?::?\d{2})?)?$/;
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* The one rendering of a timestamp read as text: UTC, microseconds, `Z`.
|
|
150
|
+
*
|
|
151
|
+
* The audit chain covers occurred_at as text, so reading a row back has to
|
|
152
|
+
* reproduce it byte for byte. Databases hand timestamps back in their own
|
|
153
|
+
* shapes (`2026-09-27 12:34:56.1+02` from Postgres, ISO text from SQLite),
|
|
154
|
+
* and a Date would drop the microseconds, so every value goes through here
|
|
155
|
+
* on the way in and out:
|
|
156
|
+
*
|
|
157
|
+
* 2026-09-27 12:34:56.1+02 -> 2026-09-27T10:34:56.100000Z
|
|
158
|
+
* 2026-09-27T10:34:56.1Z -> 2026-09-27T10:34:56.100000Z
|
|
159
|
+
*/
|
|
160
|
+
export function canonicalTimestamp(raw: string): string {
|
|
161
|
+
const match = TIMESTAMP.exec(raw);
|
|
162
|
+
if (match === null) throw new Error(`unexpected timestamp from the database: ${raw}`);
|
|
163
|
+
const [, year, month, day, hour, minute, second, fraction = '', zone = 'Z'] = match;
|
|
164
|
+
let offsetMinutes = 0;
|
|
165
|
+
if (zone !== 'Z') {
|
|
166
|
+
const sign = zone.startsWith('-') ? -1 : 1;
|
|
167
|
+
const digits = zone.slice(1).replace(':', '');
|
|
168
|
+
offsetMinutes = sign * (Number(digits.slice(0, 2)) * 60 + Number(digits.slice(2) || '0'));
|
|
169
|
+
}
|
|
170
|
+
const utc = new Date(
|
|
171
|
+
Date.UTC(+year, +month - 1, +day, +hour, +minute, +second) - offsetMinutes * 60_000,
|
|
172
|
+
);
|
|
173
|
+
return `${utc.toISOString().slice(0, 19)}.${fraction.padEnd(6, '0')}Z`;
|
|
174
|
+
}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import pg from 'pg';
|
|
2
|
+
|
|
3
|
+
import type { PoolLike } from './database.ts';
|
|
4
|
+
import { postgresConnection } from './postgres.ts';
|
|
5
|
+
|
|
6
|
+
/** Postgres through a Hyperdrive binding: `postgres(env.HYPERDRIVE)`. */
|
|
7
|
+
export type PostgresDatabase = { readonly engine: 'postgres'; readonly hyperdrive: { readonly connectionString: string } };
|
|
8
|
+
|
|
9
|
+
/** Postgres through Hyperdrive, the one database a Worker runs on, the app's and the vault's. */
|
|
10
|
+
export function postgres(hyperdrive: { readonly connectionString: string }): PostgresDatabase {
|
|
11
|
+
if (typeof hyperdrive?.connectionString !== 'string') {
|
|
12
|
+
throw new Error('postgres() takes the Hyperdrive binding, e.g. postgres(env.HYPERDRIVE)');
|
|
13
|
+
}
|
|
14
|
+
return { engine: 'postgres', hyperdrive };
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Request-scoped node-postgres adapter for Hyperdrive.
|
|
19
|
+
*
|
|
20
|
+
* Hyperdrive owns the durable origin pool. The Worker creates edge clients
|
|
21
|
+
* only inside one invocation, which prevents I/O objects from leaking across
|
|
22
|
+
* Worker request contexts while keeping the Pool shape Drizzle expects. The
|
|
23
|
+
* class name must end in `Pool`: that is how Drizzle knows to check out a
|
|
24
|
+
* client for each transaction.
|
|
25
|
+
*/
|
|
26
|
+
export class HyperdrivePool implements Exclude<PoolLike, pg.Pool> {
|
|
27
|
+
readonly #connectionString: string;
|
|
28
|
+
|
|
29
|
+
constructor(connectionString: string) {
|
|
30
|
+
if (connectionString.trim().length === 0) {
|
|
31
|
+
throw new Error('Hyperdrive returned an empty connection string');
|
|
32
|
+
}
|
|
33
|
+
this.#connectionString = connectionString;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
readonly query: pg.Pool['query'] = (async (...args: unknown[]) => {
|
|
37
|
+
const client = await this.#connectClient();
|
|
38
|
+
try {
|
|
39
|
+
return await (client.query as (...queryArgs: unknown[]) => Promise<unknown>)(...args);
|
|
40
|
+
} finally {
|
|
41
|
+
await client.end();
|
|
42
|
+
}
|
|
43
|
+
}) as pg.Pool['query'];
|
|
44
|
+
|
|
45
|
+
async connect(): Promise<Pick<pg.PoolClient, 'query'> & { release: () => void }> {
|
|
46
|
+
const client = await this.#connectClient();
|
|
47
|
+
let released = false;
|
|
48
|
+
return {
|
|
49
|
+
query: client.query.bind(client) as pg.PoolClient['query'],
|
|
50
|
+
release: () => {
|
|
51
|
+
if (released) return;
|
|
52
|
+
released = true;
|
|
53
|
+
client.end().catch(() => {});
|
|
54
|
+
},
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
async #connectClient(): Promise<pg.Client> {
|
|
59
|
+
const client = new pg.Client(postgresConnection(this.#connectionString));
|
|
60
|
+
await client.connect();
|
|
61
|
+
return client;
|
|
62
|
+
}
|
|
63
|
+
}
|
package/src/log.ts
ADDED
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
import { GENESIS_HASH, sealEntry, type LogFields, type LogKey } from '@coffre/core/audit';
|
|
2
|
+
import { desc, eq } from 'drizzle-orm';
|
|
3
|
+
|
|
4
|
+
import { afterCommit } from './commits.ts';
|
|
5
|
+
import { tablesOf, type Transaction } from './database.ts';
|
|
6
|
+
import { clockMillis, forUpdate } from './dialect.ts';
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Appending to the audit log, the one table both the app and the vault
|
|
10
|
+
* write, each under its own key (@coffre/core/audit says what an entry's MAC
|
|
11
|
+
* and hash cover).
|
|
12
|
+
*
|
|
13
|
+
* Every append locks the log's head first, `audit_chain_head`, so entries
|
|
14
|
+
* from any process take their numbers in one order. It then reads the
|
|
15
|
+
* database clock, after the lock, and the log's last entry, which the head
|
|
16
|
+
* must name: a head that does not was moved, or entries were deleted behind
|
|
17
|
+
* it, and the append refuses.
|
|
18
|
+
*
|
|
19
|
+
* This is deliberately not a queue and not best-effort: the entries go in
|
|
20
|
+
* the caller's transaction, so if the append fails, what it records does
|
|
21
|
+
* not happen either.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
/** An entry to append: what it says. The append numbers, dates, signs and chains it. */
|
|
25
|
+
export type NewEntry = {
|
|
26
|
+
actor: string;
|
|
27
|
+
action: string;
|
|
28
|
+
decision: 'allow' | 'deny';
|
|
29
|
+
code?: string | null;
|
|
30
|
+
subjectPrincipal?: string | null;
|
|
31
|
+
projectId?: string | null;
|
|
32
|
+
environmentId?: string | null;
|
|
33
|
+
secretId?: string | null;
|
|
34
|
+
secretVersionId?: string | null;
|
|
35
|
+
operationId?: string | null;
|
|
36
|
+
requestId?: string | null;
|
|
37
|
+
sourceIp?: string | null;
|
|
38
|
+
relatedSeq?: bigint | null;
|
|
39
|
+
/** JSON text. */
|
|
40
|
+
metadata?: string;
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
export type Appended = {
|
|
44
|
+
/** The first entry's number, and the next number after the last. */
|
|
45
|
+
seqStart: bigint;
|
|
46
|
+
nextSeq: bigint;
|
|
47
|
+
headHash: Buffer;
|
|
48
|
+
/** Milliseconds since the epoch, the database's, shared by the batch. */
|
|
49
|
+
occurredAt: number;
|
|
50
|
+
};
|
|
51
|
+
|
|
52
|
+
/** The log's head does not name its last entry: something moved it, or cut the log behind it. */
|
|
53
|
+
export class LogHeadMismatch extends Error {
|
|
54
|
+
constructor(message: string) {
|
|
55
|
+
super(message);
|
|
56
|
+
this.name = 'LogHeadMismatch';
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* The log's head is behind one this process wrote: the database was rolled
|
|
62
|
+
* back, or entries were cut off its end. That cannot be told from a restore,
|
|
63
|
+
* so the append refuses and someone has to look.
|
|
64
|
+
*/
|
|
65
|
+
export class LogRewound extends Error {
|
|
66
|
+
constructor(message: string) {
|
|
67
|
+
super(message);
|
|
68
|
+
this.name = 'LogRewound';
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* The newest head each writer has written, under each of its keys, while
|
|
74
|
+
* the process lives. A head is remembered only once the transaction that
|
|
75
|
+
* wrote it has committed, all the way out (commits.ts): one that rolls back
|
|
76
|
+
* never puts it ahead of the database, and the last batch committed is
|
|
77
|
+
* covered too. When commits race, the furthest head stays.
|
|
78
|
+
*/
|
|
79
|
+
const remembered = new WeakMap<LogKey, { epoch: number; nextSeq: bigint; headHash: Buffer }>();
|
|
80
|
+
let epoch = 0;
|
|
81
|
+
|
|
82
|
+
function remember(key: LogKey, head: { nextSeq: bigint; headHash: Buffer }): void {
|
|
83
|
+
const before = remembered.get(key);
|
|
84
|
+
if (before !== undefined && before.epoch === epoch && before.nextSeq >= head.nextSeq) return;
|
|
85
|
+
remembered.set(key, { epoch, ...head });
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* Forget every head written so far. For tests that empty the database
|
|
90
|
+
* between cases, which is exactly what the check above refuses.
|
|
91
|
+
*/
|
|
92
|
+
export function forgetLogHeads(): void {
|
|
93
|
+
epoch += 1;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* Lock the log's head, and read it. A transaction that appends takes this
|
|
98
|
+
* before any other row, so the lock order is the same everywhere; taking it
|
|
99
|
+
* again in the same transaction is free.
|
|
100
|
+
*/
|
|
101
|
+
export async function lockLogHead(tx: Transaction): Promise<{ nextSeq: bigint; headHash: Buffer }> {
|
|
102
|
+
const { auditChainHead } = tablesOf(tx);
|
|
103
|
+
const [head] = await forUpdate(
|
|
104
|
+
tx,
|
|
105
|
+
tx
|
|
106
|
+
.select({ nextSeq: auditChainHead.nextSeq, headHash: auditChainHead.headHash })
|
|
107
|
+
.from(auditChainHead)
|
|
108
|
+
.where(eq(auditChainHead.onlyRow, true)),
|
|
109
|
+
);
|
|
110
|
+
if (head === undefined) throw new Error('audit_chain_head is missing; refusing to write an unchained entry');
|
|
111
|
+
return head;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
export async function appendEntries(tx: Transaction, key: LogKey, entries: readonly NewEntry[]): Promise<Appended> {
|
|
115
|
+
if (entries.length === 0) throw new Error('appendEntries called with no entries');
|
|
116
|
+
const { auditChainHead, auditLog } = tablesOf(tx);
|
|
117
|
+
|
|
118
|
+
// Every append queues here, if its transaction has not already.
|
|
119
|
+
const head = await lockLogHead(tx);
|
|
120
|
+
const [{ now }] = await tx.select({ now: clockMillis(tx) }).from(auditChainHead);
|
|
121
|
+
const [last] = await tx
|
|
122
|
+
.select({ seq: auditLog.seq, hash: auditLog.hash })
|
|
123
|
+
.from(auditLog)
|
|
124
|
+
.orderBy(desc(auditLog.seq))
|
|
125
|
+
.limit(1);
|
|
126
|
+
|
|
127
|
+
const expected = last === undefined ? { nextSeq: 0n, headHash: GENESIS_HASH } : { nextSeq: last.seq + 1n, headHash: last.hash };
|
|
128
|
+
if (head.nextSeq !== expected.nextSeq || !head.headHash.equals(expected.headHash)) {
|
|
129
|
+
throw new LogHeadMismatch(
|
|
130
|
+
`the log's head says entry ${head.nextSeq} comes next, but the log ends ${last === undefined ? 'empty' : `at entry ${last.seq}`}, or with another hash`,
|
|
131
|
+
);
|
|
132
|
+
}
|
|
133
|
+
const before = remembered.get(key);
|
|
134
|
+
if (
|
|
135
|
+
before !== undefined &&
|
|
136
|
+
before.epoch === epoch &&
|
|
137
|
+
(head.nextSeq < before.nextSeq || (head.nextSeq === before.nextSeq && !head.headHash.equals(before.headHash)))
|
|
138
|
+
) {
|
|
139
|
+
throw new LogRewound(`the log ends before entry ${before.nextSeq}, which this process wrote: it was rolled back`);
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
let seq = head.nextSeq;
|
|
143
|
+
let prevHash = head.headHash;
|
|
144
|
+
const rows: (typeof auditLog.$inferInsert)[] = [];
|
|
145
|
+
for (const entry of entries) {
|
|
146
|
+
const fields: LogFields = {
|
|
147
|
+
seq,
|
|
148
|
+
author: key.author,
|
|
149
|
+
keyId: key.keyId,
|
|
150
|
+
occurredAt: now,
|
|
151
|
+
actor: entry.actor,
|
|
152
|
+
action: entry.action,
|
|
153
|
+
decision: entry.decision,
|
|
154
|
+
code: entry.code ?? null,
|
|
155
|
+
subjectPrincipal: entry.subjectPrincipal ?? null,
|
|
156
|
+
projectId: entry.projectId ?? null,
|
|
157
|
+
environmentId: entry.environmentId ?? null,
|
|
158
|
+
secretId: entry.secretId ?? null,
|
|
159
|
+
secretVersionId: entry.secretVersionId ?? null,
|
|
160
|
+
operationId: entry.operationId ?? null,
|
|
161
|
+
requestId: entry.requestId ?? null,
|
|
162
|
+
sourceIp: entry.sourceIp ?? null,
|
|
163
|
+
relatedSeq: entry.relatedSeq ?? null,
|
|
164
|
+
metadata: entry.metadata ?? '{}',
|
|
165
|
+
};
|
|
166
|
+
const { mac, hash } = sealEntry(key, prevHash, fields);
|
|
167
|
+
rows.push({ ...fields, prevHash, mac, hash });
|
|
168
|
+
prevHash = hash;
|
|
169
|
+
seq += 1n;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
await tx.insert(auditLog).values(rows);
|
|
173
|
+
await tx.update(auditChainHead).set({ nextSeq: seq, headHash: prevHash }).where(eq(auditChainHead.onlyRow, true));
|
|
174
|
+
const written = { nextSeq: seq, headHash: prevHash };
|
|
175
|
+
afterCommit(tx, () => remember(key, written));
|
|
176
|
+
return { seqStart: head.nextSeq, ...written, occurredAt: now };
|
|
177
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
// `pnpm db:migrate`: the workspace's own migrator, for dev
|
|
2
|
+
// and the tests. A deployment runs `coffre-server migrate`.
|
|
3
|
+
import { migrateDatabase } from './migrate.ts';
|
|
4
|
+
|
|
5
|
+
const url = process.env.DATABASE_URL?.trim();
|
|
6
|
+
if (!url) throw new Error('DATABASE_URL is required');
|
|
7
|
+
await migrateDatabase(url);
|
|
8
|
+
console.log('database schema is up to date');
|
package/src/migrate.ts
ADDED
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
import { readFile } from 'node:fs/promises';
|
|
3
|
+
import { fileURLToPath } from 'node:url';
|
|
4
|
+
|
|
5
|
+
import { engineOfUrl } from './connect.ts';
|
|
6
|
+
import type { Engine } from './dialect.ts';
|
|
7
|
+
import { postgresConnection } from './postgres.ts';
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Bring a database up to date with its engine's migration tree:
|
|
11
|
+
* migrations/postgres or migrations/sqlite. Both trees
|
|
12
|
+
* describe one schema; see schema-parity.test.ts.
|
|
13
|
+
*
|
|
14
|
+
* Applied history must be a prefix of the local journal, byte for byte,
|
|
15
|
+
* before and after. Only one migrator runs at a time: Postgres takes an
|
|
16
|
+
* advisory lock, and SQLite's own write lock covers the
|
|
17
|
+
* whole run. Restricted runtime roles are Postgres-only (baseline/postgres.sql).
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
const MIGRATION_LOCK_KEY = '7165058122361679213';
|
|
21
|
+
|
|
22
|
+
interface JournalEntry {
|
|
23
|
+
idx: number;
|
|
24
|
+
when: number;
|
|
25
|
+
tag: string;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
interface ExpectedMigration {
|
|
29
|
+
createdAt: string;
|
|
30
|
+
hash: string;
|
|
31
|
+
tag: string;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
interface AppliedMigration {
|
|
35
|
+
created_at: string;
|
|
36
|
+
hash: string;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** A connection that applies one engine's tree. */
|
|
40
|
+
interface Migrator {
|
|
41
|
+
/** The applied history, oldest first, or null before the first migration. */
|
|
42
|
+
applied(): Promise<AppliedMigration[] | null>;
|
|
43
|
+
lock(): Promise<void>;
|
|
44
|
+
unlock(): Promise<void>;
|
|
45
|
+
migrate(migrationsFolder: string): Promise<void>;
|
|
46
|
+
/** Reassert privileges a one-database backup does not carry. */
|
|
47
|
+
restrict(): Promise<void>;
|
|
48
|
+
close(): Promise<void>;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Beside this module: src/migrations in the workspace, and dist/migrations
|
|
53
|
+
* in the built package, where the build copies them.
|
|
54
|
+
*/
|
|
55
|
+
export function migrationsFolder(engine: Engine): string {
|
|
56
|
+
return fileURLToPath(new URL(`./migrations/${engine}`, import.meta.url));
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
async function expectedMigrations(engine: Engine): Promise<ExpectedMigration[]> {
|
|
60
|
+
const folder = migrationsFolder(engine);
|
|
61
|
+
const journal = JSON.parse(await readFile(`${folder}/meta/_journal.json`, 'utf8')) as { entries?: JournalEntry[] };
|
|
62
|
+
if (!Array.isArray(journal.entries)) {
|
|
63
|
+
throw new Error('invalid Drizzle migration journal');
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
const expected = await Promise.all(
|
|
67
|
+
journal.entries.map(async (entry, position) => {
|
|
68
|
+
if (
|
|
69
|
+
entry.idx !== position
|
|
70
|
+
|| !Number.isSafeInteger(entry.when)
|
|
71
|
+
|| typeof entry.tag !== 'string'
|
|
72
|
+
|| !/^[a-zA-Z0-9_-]+$/.test(entry.tag)
|
|
73
|
+
) {
|
|
74
|
+
throw new Error(`invalid Drizzle migration journal entry at index ${position}`);
|
|
75
|
+
}
|
|
76
|
+
const sqlText = await readFile(`${folder}/${entry.tag}.sql`, 'utf8');
|
|
77
|
+
return {
|
|
78
|
+
createdAt: String(entry.when),
|
|
79
|
+
hash: createHash('sha256').update(sqlText).digest('hex'),
|
|
80
|
+
tag: entry.tag,
|
|
81
|
+
};
|
|
82
|
+
}),
|
|
83
|
+
);
|
|
84
|
+
|
|
85
|
+
for (let index = 1; index < expected.length; index += 1) {
|
|
86
|
+
if (BigInt(expected[index - 1].createdAt) >= BigInt(expected[index].createdAt)) {
|
|
87
|
+
throw new Error('Drizzle migration journal timestamps must be strictly increasing');
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
return expected;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function verifyHistory(
|
|
94
|
+
expected: ExpectedMigration[],
|
|
95
|
+
applied: AppliedMigration[] | null,
|
|
96
|
+
requireComplete: boolean,
|
|
97
|
+
): void {
|
|
98
|
+
const rows = applied ?? [];
|
|
99
|
+
if (rows.length > expected.length) {
|
|
100
|
+
throw new Error('database contains migrations that are absent from this image');
|
|
101
|
+
}
|
|
102
|
+
if (requireComplete && rows.length !== expected.length) {
|
|
103
|
+
throw new Error(`migration history is incomplete: expected ${expected.length}, found ${rows.length}`);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
rows.forEach((row, index) => {
|
|
107
|
+
const local = expected[index];
|
|
108
|
+
if (row.created_at !== local.createdAt || row.hash !== local.hash) {
|
|
109
|
+
throw new Error(
|
|
110
|
+
`migration history diverged at ${local.tag}; applied migrations must never be edited`,
|
|
111
|
+
);
|
|
112
|
+
}
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
async function postgresMigrator(url: string): Promise<Migrator> {
|
|
117
|
+
const [{ default: pg }, { drizzle }, { migrate }] = await Promise.all([
|
|
118
|
+
import('pg'),
|
|
119
|
+
import('drizzle-orm/node-postgres'),
|
|
120
|
+
import('drizzle-orm/node-postgres/migrator'),
|
|
121
|
+
]);
|
|
122
|
+
const pool = new pg.Pool({ ...postgresConnection(url), application_name: 'coffre-migrations', max: 1 });
|
|
123
|
+
const client = await pool.connect();
|
|
124
|
+
return {
|
|
125
|
+
async applied() {
|
|
126
|
+
const relation = await client.query<{ relation: string | null }>(
|
|
127
|
+
"SELECT to_regclass('drizzle.__drizzle_migrations')::text AS relation",
|
|
128
|
+
);
|
|
129
|
+
if (!relation.rows[0]?.relation) return null;
|
|
130
|
+
const result = await client.query<AppliedMigration>(
|
|
131
|
+
'SELECT hash, created_at::text FROM drizzle.__drizzle_migrations ORDER BY created_at, id',
|
|
132
|
+
);
|
|
133
|
+
return result.rows;
|
|
134
|
+
},
|
|
135
|
+
async lock() {
|
|
136
|
+
await client.query("SET lock_timeout = '5min'");
|
|
137
|
+
await client.query('SELECT pg_advisory_lock($1::bigint)', [MIGRATION_LOCK_KEY]);
|
|
138
|
+
},
|
|
139
|
+
async unlock() {
|
|
140
|
+
await client.query('SELECT pg_advisory_unlock($1::bigint)', [MIGRATION_LOCK_KEY]);
|
|
141
|
+
},
|
|
142
|
+
migrate: (migrationsFolder) => migrate(drizzle(client), { migrationsFolder }),
|
|
143
|
+
async restrict() {
|
|
144
|
+
// pg_dump carries table privileges, but not privileges on the database itself.
|
|
145
|
+
await client.query(`DO $$ BEGIN
|
|
146
|
+
EXECUTE format(
|
|
147
|
+
'REVOKE CREATE, TEMPORARY ON DATABASE %I FROM PUBLIC, coffre_app, coffre_runtime, coffre_vault, coffre_vault_runtime',
|
|
148
|
+
current_database()
|
|
149
|
+
);
|
|
150
|
+
END $$`);
|
|
151
|
+
},
|
|
152
|
+
async close() {
|
|
153
|
+
client.release();
|
|
154
|
+
await pool.end();
|
|
155
|
+
},
|
|
156
|
+
};
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
async function sqliteMigrator(url: string): Promise<Migrator> {
|
|
160
|
+
const [{ openDatabase }, { migrate }] = await Promise.all([
|
|
161
|
+
import('./connect.ts'),
|
|
162
|
+
import('drizzle-orm/libsql/migrator'),
|
|
163
|
+
]);
|
|
164
|
+
const { db, close } = await openDatabase(url);
|
|
165
|
+
const sqlite = db as unknown as Parameters<typeof migrate>[0];
|
|
166
|
+
return {
|
|
167
|
+
async applied() {
|
|
168
|
+
const tables = await sqlite.all(
|
|
169
|
+
"SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = '__drizzle_migrations'",
|
|
170
|
+
);
|
|
171
|
+
if (tables.length === 0) return null;
|
|
172
|
+
return sqlite.all<AppliedMigration>(
|
|
173
|
+
'SELECT hash, CAST(created_at AS TEXT) AS created_at FROM __drizzle_migrations ORDER BY created_at, id',
|
|
174
|
+
);
|
|
175
|
+
},
|
|
176
|
+
// The migration runs as one transaction, in the database's write queue.
|
|
177
|
+
async lock() {},
|
|
178
|
+
async unlock() {},
|
|
179
|
+
migrate: (migrationsFolder) => migrate(sqlite, { migrationsFolder }),
|
|
180
|
+
async restrict() {},
|
|
181
|
+
close,
|
|
182
|
+
};
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/** Apply missing migrations, check their history and reassert database privileges. */
|
|
186
|
+
export async function migrateDatabase(url: string): Promise<void> {
|
|
187
|
+
const engine = engineOfUrl(url);
|
|
188
|
+
const expected = await expectedMigrations(engine);
|
|
189
|
+
const migrator = await { postgres: postgresMigrator, sqlite: sqliteMigrator }[engine](url);
|
|
190
|
+
let locked = false;
|
|
191
|
+
|
|
192
|
+
try {
|
|
193
|
+
await migrator.lock();
|
|
194
|
+
locked = true;
|
|
195
|
+
|
|
196
|
+
verifyHistory(expected, await migrator.applied(), false);
|
|
197
|
+
await migrator.migrate(migrationsFolder(engine));
|
|
198
|
+
verifyHistory(expected, await migrator.applied(), true);
|
|
199
|
+
await migrator.restrict();
|
|
200
|
+
} finally {
|
|
201
|
+
try {
|
|
202
|
+
if (locked) await migrator.unlock();
|
|
203
|
+
} finally {
|
|
204
|
+
await migrator.close();
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
}
|