@trylight/app 0.1.1 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -3
- package/dist/db/audit.d.ts +36 -0
- package/dist/db/audit.js +193 -0
- package/dist/db/cli.js +29 -10
- package/dist/db/migrations.d.ts +4 -1
- package/dist/db/migrations.js +6 -4
- package/dist/schema.d.ts +7 -5
- package/dist/schema.js +6 -4
- package/dist/server/entities.js +13 -32
- package/dist/server/env.d.ts +1 -0
- package/dist/server/env.js +3 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -21,8 +21,8 @@ through `entity()`. Each request is parsed with the table's own Zod schema
|
|
|
21
21
|
(drizzle-zod; unknown or platform columns are refused, `field_sources` must be
|
|
22
22
|
`{url, evidence}` per real column), goes through the app's door (members-only:
|
|
23
23
|
a member; public: anyone reads, people signed in write; `write: "agents"`
|
|
24
|
-
tables refuse the app), must come from the app's own pages
|
|
25
|
-
|
|
24
|
+
tables refuse the app), and must come from the app's own pages; each write
|
|
25
|
+
names its person for the table's change trigger (below). An
|
|
26
26
|
edited value loses its source; sources sent are merged by column. Lists sort
|
|
27
27
|
by the asked column, then id. With no sign-in configured only a non-production
|
|
28
28
|
server asked on loopback answers (as "Local developer"); anything else gets 503.
|
|
@@ -46,12 +46,17 @@ sign-in; a stranger to a public app only reads what members kept.
|
|
|
46
46
|
|
|
47
47
|
## Schema sync
|
|
48
48
|
|
|
49
|
+
`light-db sync` and `migrate` connect with `DATABASE_URL_UNPOOLED`, the
|
|
50
|
+
database's direct address, and refuse to run without it: they serialise on
|
|
51
|
+
Postgres advisory locks, which a transaction pooler (the app's `DATABASE_URL`
|
|
52
|
+
on Neon) keeps on a pooled connection after the process exits.
|
|
53
|
+
|
|
49
54
|
`plan()` diffs `db/schema` against the newest migration snapshot with
|
|
50
55
|
drizzle-kit's API, removals first and additions second, so drizzle-kit never
|
|
51
56
|
has to ask whether a column was renamed (it can't without a terminal).
|
|
52
57
|
The change is written as a drizzle migration (`db/migrations`,
|
|
53
58
|
drizzle-kit's own layout) and applied, whatever it does to the rows there (a
|
|
54
|
-
removed column's values
|
|
59
|
+
removed column's values are archived first, below); one the rows refuse is taken back out,
|
|
55
60
|
so the change stays pending rather than blocking every later migrate. Changed
|
|
56
61
|
indexes and constraints are dropped before the columns they cover and enums
|
|
57
62
|
only after, so the two halves apply in order. Syncs and migrates each hold a Postgres advisory lock (with a
|
|
@@ -60,6 +65,22 @@ overlap, and a process that dies releases it. `migrate` applies the journal; in
|
|
|
60
65
|
asks the platform's migrate socket to run it. `check` (in `pnpm build`)
|
|
61
66
|
fails while `db/schema` has changes no migration holds.
|
|
62
67
|
|
|
68
|
+
## Recoverable deletes
|
|
69
|
+
|
|
70
|
+
`src/db/audit.ts`, in Postgres itself. Every entity table has a row trigger
|
|
71
|
+
that records each insert, update, delete and truncate in `light_changes` (the
|
|
72
|
+
new row, `{ column: { from, to } }`, or the old row), whoever wrote it: the
|
|
73
|
+
actor is `light.actor_kind/_id/_name`, which the entities route and Plaza's
|
|
74
|
+
`mii db write` set per transaction, or else the database role. A migration
|
|
75
|
+
that drops a table or column or changes a column's type first copies what it
|
|
76
|
+
destroys into `light_archive` (`light_archive_values`, called at the top of
|
|
77
|
+
the migration, so in its transaction) and the migrate prints "Archived N
|
|
78
|
+
values from table.column". Both tables refuse UPDATE, DELETE and TRUNCATE (a guard against mistakes:
|
|
79
|
+
the owner role can still drop the trigger). DDL an agent runs itself (`drop
|
|
80
|
+
table` through `mii db write`) is not archived.
|
|
81
|
+
`migrate` installs the functions before applying migrations and the triggers
|
|
82
|
+
after; the template's AGENTS.md has the restore SQL.
|
|
83
|
+
|
|
63
84
|
## Develop
|
|
64
85
|
|
|
65
86
|
```bash
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import type postgres from "postgres";
|
|
2
|
+
/** Makes light_archive and the functions migrations and triggers call. Runs before migrating. */
|
|
3
|
+
export declare function prepareAudit(sql: postgres.Sql): Promise<void>;
|
|
4
|
+
/**
|
|
5
|
+
* The entity tables `schema` declares. Read before migrating: the triggers
|
|
6
|
+
* write into light_changes, so a schema that doesn't export it is refused
|
|
7
|
+
* before any migration applies.
|
|
8
|
+
*/
|
|
9
|
+
export declare function entityTables(schema: Record<string, unknown>): string[];
|
|
10
|
+
/**
|
|
11
|
+
* Puts the change trigger on each of `tables` the database has (one a
|
|
12
|
+
* migration hasn't made yet is tracked on the migrate that makes it), and
|
|
13
|
+
* makes light_changes append-only. Runs after migrating.
|
|
14
|
+
*/
|
|
15
|
+
export declare function trackTables(sql: postgres.Sql, tables: string[]): Promise<void>;
|
|
16
|
+
/** A table, or a column of one, that a migration destroys: what it archives first. */
|
|
17
|
+
export type Destroyed = {
|
|
18
|
+
table: string;
|
|
19
|
+
column: string | null;
|
|
20
|
+
};
|
|
21
|
+
/**
|
|
22
|
+
* What going from `previous` to `current` (drizzle-kit snapshots' `tables`)
|
|
23
|
+
* destroys: dropped tables, dropped columns, and columns whose type changes
|
|
24
|
+
* (Postgres converts the values, which may not survive the trip back).
|
|
25
|
+
*/
|
|
26
|
+
export declare function destroyed(previous: Record<string, unknown>, current: Record<string, unknown>): Destroyed[];
|
|
27
|
+
/** The statements that archive what migration `tag` destroys, to run first in it. */
|
|
28
|
+
export declare function archiveStatements(archive: Destroyed[], tag: string): string[];
|
|
29
|
+
/**
|
|
30
|
+
* Collects what a migrate archived, to print once its transaction commits
|
|
31
|
+
* (a migration that fails archives nothing). Other notices are dropped.
|
|
32
|
+
*/
|
|
33
|
+
export declare function archiveNotices(): {
|
|
34
|
+
onnotice(notice: postgres.Notice): void;
|
|
35
|
+
print(): void;
|
|
36
|
+
};
|
package/dist/db/audit.js
ADDED
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
// Recoverable deletes: change triggers into light_changes, an archive step at
|
|
2
|
+
// the top of destructive migrations, both tables append-only (README,
|
|
3
|
+
// "Recoverable deletes"; the restore is the template AGENTS.md's).
|
|
4
|
+
import { getTableName, is } from "drizzle-orm";
|
|
5
|
+
import { PgTable } from "drizzle-orm/pg-core";
|
|
6
|
+
import { z } from "zod/v4";
|
|
7
|
+
import { isEntity, lightChanges } from "../schema.js";
|
|
8
|
+
/** The SQLSTATE of the notice an archive raises, so the CLI prints just those. */
|
|
9
|
+
const ARCHIVED_NOTICE = "LA001";
|
|
10
|
+
/**
|
|
11
|
+
* Installed before migrations run (a migration's archive step calls these),
|
|
12
|
+
* all idempotent. No input reaches this text; the functions quote the
|
|
13
|
+
* identifiers they are given with format('%I').
|
|
14
|
+
*/
|
|
15
|
+
const PREPARE = `
|
|
16
|
+
create table if not exists light_archive (
|
|
17
|
+
id uuid primary key default gen_random_uuid(),
|
|
18
|
+
table_name text not null,
|
|
19
|
+
-- null when the whole row was archived (its table was dropped)
|
|
20
|
+
column_name text,
|
|
21
|
+
-- the row's id; null for a table without a uuid id
|
|
22
|
+
row_id uuid,
|
|
23
|
+
value jsonb not null,
|
|
24
|
+
migration text not null,
|
|
25
|
+
archived_at timestamptz not null default now()
|
|
26
|
+
);
|
|
27
|
+
create index if not exists light_archive_table_idx on light_archive (table_name, column_name);
|
|
28
|
+
|
|
29
|
+
create or replace function light_append_only() returns trigger language plpgsql as $$
|
|
30
|
+
begin
|
|
31
|
+
raise exception '% is append-only: it is the record of what changed and what was removed, so its rows are never changed or deleted.', tg_table_name
|
|
32
|
+
using errcode = 'insufficient_privilege';
|
|
33
|
+
end $$;
|
|
34
|
+
create or replace trigger light_append_only before update or delete or truncate on light_archive
|
|
35
|
+
for each statement execute function light_append_only();
|
|
36
|
+
|
|
37
|
+
create or replace function light_record_change() returns trigger language plpgsql as $$
|
|
38
|
+
declare
|
|
39
|
+
-- A setting set local reads '' (not null) once its transaction has ended.
|
|
40
|
+
actor_kind text := coalesce(nullif(current_setting('light.actor_kind', true), ''), 'database');
|
|
41
|
+
actor_id text := nullif(current_setting('light.actor_id', true), '');
|
|
42
|
+
actor_name text := coalesce(nullif(current_setting('light.actor_name', true), ''), current_user);
|
|
43
|
+
row_id uuid;
|
|
44
|
+
action text;
|
|
45
|
+
changes jsonb;
|
|
46
|
+
begin
|
|
47
|
+
if tg_op = 'TRUNCATE' then
|
|
48
|
+
execute format(
|
|
49
|
+
'insert into light_changes (table_name, row_id, action, changes, actor_kind, actor_id, actor_name)
|
|
50
|
+
select $1, t.id, ''delete'', to_jsonb(t), $2, $3, $4 from %I.%I t', tg_table_schema, tg_table_name)
|
|
51
|
+
using tg_table_name, actor_kind, actor_id, actor_name;
|
|
52
|
+
return null;
|
|
53
|
+
elsif tg_op = 'INSERT' then
|
|
54
|
+
row_id := new.id;
|
|
55
|
+
action := 'create';
|
|
56
|
+
changes := to_jsonb(new);
|
|
57
|
+
elsif tg_op = 'DELETE' then
|
|
58
|
+
row_id := old.id;
|
|
59
|
+
action := 'delete';
|
|
60
|
+
changes := to_jsonb(old);
|
|
61
|
+
else
|
|
62
|
+
row_id := new.id;
|
|
63
|
+
action := 'update';
|
|
64
|
+
select jsonb_object_agg(o.key, jsonb_build_object('from', o.value, 'to', n.value)) into changes
|
|
65
|
+
from jsonb_each(to_jsonb(old)) o join jsonb_each(to_jsonb(new)) n on n.key = o.key
|
|
66
|
+
where o.value is distinct from n.value and o.key <> 'updated_at';
|
|
67
|
+
-- An update that set every value to what it was changed nothing.
|
|
68
|
+
if changes is null then return null; end if;
|
|
69
|
+
end if;
|
|
70
|
+
insert into light_changes (table_name, row_id, action, changes, actor_kind, actor_id, actor_name)
|
|
71
|
+
values (tg_table_name, row_id, action, changes, actor_kind, actor_id, actor_name);
|
|
72
|
+
return null;
|
|
73
|
+
end $$;
|
|
74
|
+
|
|
75
|
+
create or replace function light_track(tbl text) returns void language plpgsql as $$
|
|
76
|
+
begin
|
|
77
|
+
execute format('create or replace trigger light_record_change after insert or update or delete on %I
|
|
78
|
+
for each row execute function light_record_change()', tbl);
|
|
79
|
+
execute format('create or replace trigger light_record_truncate before truncate on %I
|
|
80
|
+
for each statement execute function light_record_change()', tbl);
|
|
81
|
+
end $$;
|
|
82
|
+
|
|
83
|
+
-- col null: every row of tbl, whole (the table is being dropped).
|
|
84
|
+
create or replace function light_archive_values(tbl text, col text, tag text) returns void language plpgsql as $$
|
|
85
|
+
declare
|
|
86
|
+
id_of_row text := case when exists (
|
|
87
|
+
select 1 from pg_attribute
|
|
88
|
+
where attrelid = format('%I', tbl)::regclass and attname = 'id' and atttypid = 'uuid'::regtype and not attisdropped
|
|
89
|
+
) then 't.id' else 'null::uuid' end;
|
|
90
|
+
archived bigint;
|
|
91
|
+
begin
|
|
92
|
+
if col is null then
|
|
93
|
+
execute format(
|
|
94
|
+
'insert into light_archive (table_name, column_name, row_id, value, migration)
|
|
95
|
+
select $1, null, %s, to_jsonb(t), $2 from %I t', id_of_row, tbl)
|
|
96
|
+
using tbl, tag;
|
|
97
|
+
else
|
|
98
|
+
execute format(
|
|
99
|
+
'insert into light_archive (table_name, column_name, row_id, value, migration)
|
|
100
|
+
select $1, $2, %s, to_jsonb(t)->$2, $3 from %I t where %I is not null', id_of_row, tbl, col)
|
|
101
|
+
using tbl, col, tag;
|
|
102
|
+
end if;
|
|
103
|
+
get diagnostics archived = row_count;
|
|
104
|
+
if archived = 0 then return; end if;
|
|
105
|
+
raise notice using errcode = '${ARCHIVED_NOTICE}', message = format('Archived %s %s from %s (migration %s).',
|
|
106
|
+
archived, case when col is null then 'rows' else 'values' end, concat_ws('.', tbl, col), tag);
|
|
107
|
+
end $$;
|
|
108
|
+
`;
|
|
109
|
+
/** light_changes is a drizzle table (db/schema exports it), so it exists only once migrated. */
|
|
110
|
+
const CHANGES_APPEND_ONLY = `
|
|
111
|
+
create or replace trigger light_append_only before update or delete or truncate on light_changes
|
|
112
|
+
for each statement execute function light_append_only();
|
|
113
|
+
`;
|
|
114
|
+
/** Makes light_archive and the functions migrations and triggers call. Runs before migrating. */
|
|
115
|
+
export async function prepareAudit(sql) {
|
|
116
|
+
await sql.unsafe(PREPARE);
|
|
117
|
+
}
|
|
118
|
+
/**
|
|
119
|
+
* The entity tables `schema` declares. Read before migrating: the triggers
|
|
120
|
+
* write into light_changes, so a schema that doesn't export it is refused
|
|
121
|
+
* before any migration applies.
|
|
122
|
+
*/
|
|
123
|
+
export function entityTables(schema) {
|
|
124
|
+
const values = Object.values(schema);
|
|
125
|
+
if (!values.some((value) => is(value, PgTable) && getTableName(value) === getTableName(lightChanges))) {
|
|
126
|
+
throw new Error("db/schema/index.ts must export lightChanges (`export { lightChanges } from \"@trylight/app/schema\";`): it records every change to the app's tables.");
|
|
127
|
+
}
|
|
128
|
+
return values.filter(isEntity).map((entity) => getTableName(entity));
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* Puts the change trigger on each of `tables` the database has (one a
|
|
132
|
+
* migration hasn't made yet is tracked on the migrate that makes it), and
|
|
133
|
+
* makes light_changes append-only. Runs after migrating.
|
|
134
|
+
*/
|
|
135
|
+
export async function trackTables(sql, tables) {
|
|
136
|
+
await sql `select light_track(name) from unnest(${sql.array(tables)}::text[]) as name where to_regclass(quote_ident(name)) is not null`;
|
|
137
|
+
await sql.unsafe(CHANGES_APPEND_ONLY);
|
|
138
|
+
}
|
|
139
|
+
const snapshotTable = z.looseObject({
|
|
140
|
+
name: z.string(),
|
|
141
|
+
columns: z.record(z.string(), z.looseObject({ type: z.string() }))
|
|
142
|
+
});
|
|
143
|
+
/**
|
|
144
|
+
* What going from `previous` to `current` (drizzle-kit snapshots' `tables`)
|
|
145
|
+
* destroys: dropped tables, dropped columns, and columns whose type changes
|
|
146
|
+
* (Postgres converts the values, which may not survive the trip back).
|
|
147
|
+
*/
|
|
148
|
+
export function destroyed(previous, current) {
|
|
149
|
+
const found = [];
|
|
150
|
+
for (const [key, entry] of Object.entries(previous)) {
|
|
151
|
+
const before = snapshotTable.parse(entry);
|
|
152
|
+
if (!(key in current)) {
|
|
153
|
+
found.push({ table: before.name, column: null });
|
|
154
|
+
continue;
|
|
155
|
+
}
|
|
156
|
+
const now = snapshotTable.parse(current[key]);
|
|
157
|
+
for (const [name, column] of Object.entries(before.columns)) {
|
|
158
|
+
const after = now.columns[name];
|
|
159
|
+
if (after === undefined || after.type !== column.type)
|
|
160
|
+
found.push({ table: before.name, column: name });
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
return found;
|
|
164
|
+
}
|
|
165
|
+
/** A SQL string literal, for the migration file drizzle's migrator runs. */
|
|
166
|
+
function literal(text) {
|
|
167
|
+
return `'${text.replaceAll("'", "''")}'`;
|
|
168
|
+
}
|
|
169
|
+
/** The statements that archive what migration `tag` destroys, to run first in it. */
|
|
170
|
+
export function archiveStatements(archive, tag) {
|
|
171
|
+
return archive.map(({ table, column }) => `SELECT light_archive_values(${literal(table)}, ${column === null ? "NULL" : literal(column)}, ${literal(tag)});`);
|
|
172
|
+
}
|
|
173
|
+
const archivedNotice = z.object({ code: z.literal(ARCHIVED_NOTICE), message: z.string() });
|
|
174
|
+
/**
|
|
175
|
+
* Collects what a migrate archived, to print once its transaction commits
|
|
176
|
+
* (a migration that fails archives nothing). Other notices are dropped.
|
|
177
|
+
*/
|
|
178
|
+
export function archiveNotices() {
|
|
179
|
+
const messages = [];
|
|
180
|
+
return {
|
|
181
|
+
onnotice(notice) {
|
|
182
|
+
const archived = archivedNotice.safeParse(notice);
|
|
183
|
+
if (archived.success)
|
|
184
|
+
messages.push(archived.data.message);
|
|
185
|
+
},
|
|
186
|
+
print() {
|
|
187
|
+
if (messages.length === 0)
|
|
188
|
+
return;
|
|
189
|
+
console.log(messages.join("\n"));
|
|
190
|
+
console.log("Archived values stay in light_archive; AGENTS.md, \"Getting deleted data back\", says how to restore them.");
|
|
191
|
+
}
|
|
192
|
+
};
|
|
193
|
+
}
|
package/dist/db/cli.js
CHANGED
|
@@ -9,6 +9,7 @@ import { drizzle } from "drizzle-orm/postgres-js";
|
|
|
9
9
|
import { migrate } from "drizzle-orm/postgres-js/migrator";
|
|
10
10
|
import postgres from "postgres";
|
|
11
11
|
import { z } from "zod/v4";
|
|
12
|
+
import { archiveNotices, entityTables, prepareAudit, trackTables } from "./audit.js";
|
|
12
13
|
import { MIGRATIONS_DIR, SCHEMA_ENTRY, plan, unwriteMigration, writeMigration } from "./migrations.js";
|
|
13
14
|
import { env } from "../server/env.js";
|
|
14
15
|
/**
|
|
@@ -20,8 +21,12 @@ const PLATFORM_MIGRATE = "/run/mii/migrate.sock";
|
|
|
20
21
|
/** Postgres advisory locks: one migrate, and one sync, of a database at a time. */
|
|
21
22
|
const MIGRATE_LOCK_KEY = 4_815_162_342;
|
|
22
23
|
const SYNC_LOCK_KEY = 4_815_162_343;
|
|
24
|
+
/** Exit code of a migrate whose migrations committed but whose change triggers didn't go on. */
|
|
25
|
+
const APPLIED_UNTRACKED = 2;
|
|
23
26
|
// pg_advisory_lock waits for the one before it; a process that dies releases it.
|
|
24
|
-
|
|
27
|
+
// Only on a direct connection: behind a transaction pooler the lock stays on a
|
|
28
|
+
// pooled server connection after the process exits, and every later sync waits.
|
|
29
|
+
const connect = (onnotice = () => undefined) => postgres(unpooledDatabaseUrl(), { max: 1, onnotice });
|
|
25
30
|
const root = process.cwd();
|
|
26
31
|
const envFile = join(root, ".env.local");
|
|
27
32
|
if (existsSync(envFile))
|
|
@@ -30,11 +35,12 @@ const [command] = process.argv.slice(2);
|
|
|
30
35
|
async function loadSchema() {
|
|
31
36
|
return import(pathToFileURL(join(root, SCHEMA_ENTRY)).href);
|
|
32
37
|
}
|
|
33
|
-
function
|
|
34
|
-
const {
|
|
35
|
-
if (
|
|
36
|
-
throw new Error("
|
|
37
|
-
|
|
38
|
+
function unpooledDatabaseUrl() {
|
|
39
|
+
const { unpooledDatabaseUrl } = env();
|
|
40
|
+
if (unpooledDatabaseUrl === undefined) {
|
|
41
|
+
throw new Error("DATABASE_URL_UNPOOLED is not set: light-db sync and migrate need the database's direct (unpooled) connection.");
|
|
42
|
+
}
|
|
43
|
+
return unpooledDatabaseUrl;
|
|
38
44
|
}
|
|
39
45
|
/**
|
|
40
46
|
* Makes the database match `db/schema`, whatever the change does to the rows
|
|
@@ -56,7 +62,7 @@ async function sync() {
|
|
|
56
62
|
console.error(error instanceof Error ? error.message : String(error));
|
|
57
63
|
return 1;
|
|
58
64
|
});
|
|
59
|
-
if (applied !== 0) {
|
|
65
|
+
if (applied !== 0 && applied !== APPLIED_UNTRACKED) {
|
|
60
66
|
unwriteMigration(root, tag);
|
|
61
67
|
console.error(`${MIGRATIONS_DIR}/${tag}.sql didn't apply, so it was taken back; change db/schema and save again.`);
|
|
62
68
|
}
|
|
@@ -74,17 +80,30 @@ async function check() {
|
|
|
74
80
|
console.error(`db/schema has changes with no migration yet; run \`pnpm db:sync\`:\n${planned.statements.join("\n")}`);
|
|
75
81
|
return 1;
|
|
76
82
|
}
|
|
77
|
-
/**
|
|
83
|
+
/**
|
|
84
|
+
* Applies every migration in db/migrations the database doesn't have, with
|
|
85
|
+
* the audit around them (./audit.ts): its functions first, since a
|
|
86
|
+
* migration's archive step calls them; its triggers after, on the tables
|
|
87
|
+
* migrations made.
|
|
88
|
+
*/
|
|
78
89
|
async function applyMigrations() {
|
|
79
90
|
// The agent (not root) in an app's sandbox: the platform migrates for it.
|
|
80
91
|
if (userInfo().uid !== 0 && existsSync(PLATFORM_MIGRATE))
|
|
81
92
|
return migrateThroughPlatform();
|
|
82
|
-
const
|
|
93
|
+
const archived = archiveNotices();
|
|
94
|
+
const client = connect(archived.onnotice);
|
|
83
95
|
try {
|
|
84
96
|
await client `select pg_advisory_lock(${MIGRATE_LOCK_KEY})`;
|
|
97
|
+
const tables = entityTables(await loadSchema());
|
|
98
|
+
await prepareAudit(client);
|
|
85
99
|
await migrate(drizzle(client), { migrationsFolder: join(root, MIGRATIONS_DIR) });
|
|
100
|
+
archived.print();
|
|
86
101
|
console.log("Database is up to date with db/migrations.");
|
|
87
|
-
|
|
102
|
+
// The migrations are committed: a failure here must not take their files back.
|
|
103
|
+
return await trackTables(client, tables).then(() => 0, (error) => {
|
|
104
|
+
console.error(`The change triggers didn't go on (${error instanceof Error ? error.message : String(error)}); the next migrate tries again.`);
|
|
105
|
+
return APPLIED_UNTRACKED;
|
|
106
|
+
});
|
|
88
107
|
}
|
|
89
108
|
finally {
|
|
90
109
|
await client.end();
|
package/dist/db/migrations.d.ts
CHANGED
|
@@ -1,9 +1,12 @@
|
|
|
1
1
|
import { z } from "zod/v4";
|
|
2
|
+
import { type Destroyed } from "./audit.js";
|
|
2
3
|
export declare const SCHEMA_ENTRY = "db/schema/index.ts";
|
|
3
4
|
export declare const MIGRATIONS_DIR = "db/migrations";
|
|
4
5
|
export type Plan = {
|
|
5
6
|
/** The new migration's statements, in order; empty when nothing changed. */
|
|
6
7
|
statements: string[];
|
|
8
|
+
/** What the statements destroy, archived first when the migration runs. */
|
|
9
|
+
archive: Destroyed[];
|
|
7
10
|
snapshot: Snapshot;
|
|
8
11
|
};
|
|
9
12
|
/** The parts of a drizzle-kit snapshot a removal touches; the rest passes through. */
|
|
@@ -30,7 +33,7 @@ type Snapshot = z.infer<typeof snapshotSchema>;
|
|
|
30
33
|
* What `db/schema` needs beyond the last migration, in three diffs that each
|
|
31
34
|
* only remove or only add, so drizzle-kit never has to ask whether something
|
|
32
35
|
* removed was renamed (it can't ask here: there is no terminal). A rename is
|
|
33
|
-
* therefore a drop and an add: the old column's values
|
|
36
|
+
* therefore a drop and an add: the old column's values are archived. Removed enums go last,
|
|
34
37
|
* after the columns that used them have changed.
|
|
35
38
|
*/
|
|
36
39
|
export declare function plan(root: string, schema: Record<string, unknown>): Promise<Plan>;
|
package/dist/db/migrations.js
CHANGED
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
// Writing a schema file is the migration: the change from the last
|
|
2
2
|
// migration's snapshot to `db/schema` becomes a new drizzle migration,
|
|
3
3
|
// written and applied at once, whatever it does to the rows there (a removed
|
|
4
|
-
// column's values
|
|
4
|
+
// column's values are archived first, in light_archive).
|
|
5
5
|
import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
|
6
6
|
import { isDeepStrictEqual } from "node:util";
|
|
7
7
|
import { join } from "node:path";
|
|
8
8
|
import { generateDrizzleJson, generateMigration } from "drizzle-kit/api";
|
|
9
9
|
import { z } from "zod/v4";
|
|
10
|
+
import { archiveStatements, destroyed } from "./audit.js";
|
|
10
11
|
export const SCHEMA_ENTRY = "db/schema/index.ts";
|
|
11
12
|
export const MIGRATIONS_DIR = "db/migrations";
|
|
12
13
|
const entries = z.record(z.string(), z.unknown());
|
|
@@ -33,7 +34,7 @@ const snapshotSchema = z.looseObject({
|
|
|
33
34
|
* What `db/schema` needs beyond the last migration, in three diffs that each
|
|
34
35
|
* only remove or only add, so drizzle-kit never has to ask whether something
|
|
35
36
|
* removed was renamed (it can't ask here: there is no terminal). A rename is
|
|
36
|
-
* therefore a drop and an add: the old column's values
|
|
37
|
+
* therefore a drop and an add: the old column's values are archived. Removed enums go last,
|
|
37
38
|
* after the columns that used them have changed.
|
|
38
39
|
*/
|
|
39
40
|
export async function plan(root, schema) {
|
|
@@ -47,7 +48,7 @@ export async function plan(root, schema) {
|
|
|
47
48
|
...(await generateMigration(kept, withOldEnums)),
|
|
48
49
|
...(await generateMigration(withOldEnums, snapshot))
|
|
49
50
|
];
|
|
50
|
-
return { statements, snapshot };
|
|
51
|
+
return { statements, archive: destroyed(previous.tables, snapshot.tables), snapshot };
|
|
51
52
|
}
|
|
52
53
|
function only(from, keep) {
|
|
53
54
|
return Object.fromEntries(Object.entries(from).filter(([key]) => key in keep));
|
|
@@ -139,7 +140,8 @@ export function writeMigration(root, planned) {
|
|
|
139
140
|
const idx = journal.entries.length;
|
|
140
141
|
const tag = `${String(idx).padStart(4, "0")}_schema`;
|
|
141
142
|
mkdirSync(join(root, MIGRATIONS_DIR, "meta"), { recursive: true });
|
|
142
|
-
|
|
143
|
+
const statements = [...archiveStatements(planned.archive, tag), ...planned.statements];
|
|
144
|
+
writeFileSync(join(root, MIGRATIONS_DIR, `${tag}.sql`), statements.join("\n--> statement-breakpoint\n") + "\n");
|
|
143
145
|
writeFileSync(snapshotFile(root, idx), JSON.stringify(planned.snapshot, null, 2) + "\n");
|
|
144
146
|
journal.entries.push({ idx, version: "7", when: Date.now(), tag, breakpoints: true });
|
|
145
147
|
writeFileSync(join(root, MIGRATIONS_DIR, "meta", "_journal.json"), JSON.stringify(journal, null, 2) + "\n");
|
package/dist/schema.d.ts
CHANGED
|
@@ -75,9 +75,11 @@ export type Entity = PgTable & {
|
|
|
75
75
|
};
|
|
76
76
|
export declare function isEntity(value: unknown): value is Entity;
|
|
77
77
|
/**
|
|
78
|
-
* Every change
|
|
79
|
-
*
|
|
80
|
-
* the old row for a delete, and `{ column: { from, to } }`
|
|
78
|
+
* Every change to an entity table, whoever made it, written by the table's
|
|
79
|
+
* trigger (src/db/audit.ts) in the change's transaction. `changes` is the new
|
|
80
|
+
* row for a create, the old row for a delete, and `{ column: { from, to } }`
|
|
81
|
+
* for an update. A `database` actor is a writer that didn't name itself: its
|
|
82
|
+
* name is the database role.
|
|
81
83
|
*/
|
|
82
84
|
export declare const lightChanges: PgTableWithColumns<{
|
|
83
85
|
name: "light_changes";
|
|
@@ -175,14 +177,14 @@ export declare const lightChanges: PgTableWithColumns<{
|
|
|
175
177
|
tableName: "light_changes";
|
|
176
178
|
dataType: "string";
|
|
177
179
|
columnType: "PgText";
|
|
178
|
-
data: "person" | "agent";
|
|
180
|
+
data: "person" | "agent" | "database";
|
|
179
181
|
driverParam: string;
|
|
180
182
|
notNull: true;
|
|
181
183
|
hasDefault: false;
|
|
182
184
|
isPrimaryKey: false;
|
|
183
185
|
isAutoincrement: false;
|
|
184
186
|
hasRuntimeDefault: false;
|
|
185
|
-
enumValues: ["person", "agent"];
|
|
187
|
+
enumValues: ["person", "agent", "database"];
|
|
186
188
|
baseColumn: never;
|
|
187
189
|
identity: undefined;
|
|
188
190
|
generated: undefined;
|
package/dist/schema.js
CHANGED
|
@@ -64,9 +64,11 @@ export function isEntity(value) {
|
|
|
64
64
|
return is(value, PgTable) && "$access" in value;
|
|
65
65
|
}
|
|
66
66
|
/**
|
|
67
|
-
* Every change
|
|
68
|
-
*
|
|
69
|
-
* the old row for a delete, and `{ column: { from, to } }`
|
|
67
|
+
* Every change to an entity table, whoever made it, written by the table's
|
|
68
|
+
* trigger (src/db/audit.ts) in the change's transaction. `changes` is the new
|
|
69
|
+
* row for a create, the old row for a delete, and `{ column: { from, to } }`
|
|
70
|
+
* for an update. A `database` actor is a writer that didn't name itself: its
|
|
71
|
+
* name is the database role.
|
|
70
72
|
*/
|
|
71
73
|
export const lightChanges = pgTable("light_changes", {
|
|
72
74
|
id: uuid().primaryKey().defaultRandom(),
|
|
@@ -74,7 +76,7 @@ export const lightChanges = pgTable("light_changes", {
|
|
|
74
76
|
row_id: uuid().notNull(),
|
|
75
77
|
action: text({ enum: ["create", "update", "delete"] }).notNull(),
|
|
76
78
|
changes: jsonb().$type().notNull(),
|
|
77
|
-
actor_kind: text({ enum: ["person", "agent"] }).notNull(),
|
|
79
|
+
actor_kind: text({ enum: ["person", "agent", "database"] }).notNull(),
|
|
78
80
|
actor_id: text(),
|
|
79
81
|
actor_name: text().notNull(),
|
|
80
82
|
created_at: timestamp({ withTimezone: true, mode: "string" })
|
package/dist/server/entities.js
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
// The one route behind `app.table(...)`: list, get, create, update and delete
|
|
2
2
|
// on any entity table, each request checked against the table's own Zod
|
|
3
|
-
// schema and the app's door
|
|
4
|
-
//
|
|
5
|
-
import { and, asc, desc, eq, getTableColumns, getTableName, inArray, isNull } from "drizzle-orm";
|
|
3
|
+
// schema and the app's door. Each write names its person for the table's
|
|
4
|
+
// trigger, which records the change in light_changes (src/db/audit.ts).
|
|
5
|
+
import { and, asc, desc, eq, getTableColumns, getTableName, inArray, isNull, sql } from "drizzle-orm";
|
|
6
6
|
import { isDeepStrictEqual } from "node:util";
|
|
7
7
|
import { createSchemaFactory } from "drizzle-zod";
|
|
8
8
|
import { isSameOrigin } from "@shubh90/app-runtime/chat/server";
|
|
9
9
|
import { z } from "zod/v4";
|
|
10
10
|
import { ENTITIES_PATH, LIST_LIMIT } from "../entities.js";
|
|
11
|
-
import { PLATFORM_COLUMNS, fieldSource, isEntity
|
|
11
|
+
import { PLATFORM_COLUMNS, fieldSource, isEntity } from "../schema.js";
|
|
12
12
|
import { env } from "./env.js";
|
|
13
13
|
/** Who writes when the app runs with no sign-in at all: a bare local dev server. */
|
|
14
14
|
const LOCAL_DEVELOPER = { id: null, name: "Local developer" };
|
|
@@ -153,54 +153,39 @@ async function get(db, table, id) {
|
|
|
153
153
|
}
|
|
154
154
|
async function create(db, table, actor, data) {
|
|
155
155
|
return db.transaction(async (tx) => {
|
|
156
|
-
|
|
156
|
+
await nameActor(tx, actor);
|
|
157
|
+
return exactlyOne(await tx
|
|
157
158
|
.insert(table.entity)
|
|
158
159
|
.values({ ...data, created_by: actor.id })
|
|
159
160
|
.returning());
|
|
160
|
-
await audit(tx, table, rowId(row), "create", row, actor);
|
|
161
|
-
return row;
|
|
162
161
|
});
|
|
163
162
|
}
|
|
164
163
|
async function update(db, table, actor, id, patch) {
|
|
165
164
|
return db.transaction(async (tx) => {
|
|
165
|
+
await nameActor(tx, actor);
|
|
166
166
|
const before = atMostOne(await tx.select().from(table.entity).where(eq(table.id, id)).for("update"));
|
|
167
167
|
if (before === null)
|
|
168
168
|
return null;
|
|
169
169
|
const changed = Object.keys(patch).filter((key) => !isDeepStrictEqual(patch[key], before[key]));
|
|
170
170
|
// A value a person sets is no longer what its source said; sources sent are merged in.
|
|
171
171
|
const field_sources = { ...withoutSources(before.field_sources, changed), ...patch.field_sources };
|
|
172
|
-
|
|
172
|
+
return exactlyOne(await tx
|
|
173
173
|
.update(table.entity)
|
|
174
174
|
.set({ ...patch, field_sources })
|
|
175
175
|
.where(eq(table.id, id))
|
|
176
176
|
.returning());
|
|
177
|
-
const changes = Object.fromEntries([...changed, "field_sources"]
|
|
178
|
-
.filter((key) => !isDeepStrictEqual(before[key], row[key]))
|
|
179
|
-
.map((key) => [key, { from: before[key], to: row[key] }]));
|
|
180
|
-
if (Object.keys(changes).length > 0)
|
|
181
|
-
await audit(tx, table, id, "update", changes, actor);
|
|
182
|
-
return row;
|
|
183
177
|
});
|
|
184
178
|
}
|
|
185
179
|
async function remove(db, table, actor, id) {
|
|
186
180
|
return db.transaction(async (tx) => {
|
|
181
|
+
await nameActor(tx, actor);
|
|
187
182
|
const row = atMostOne(await tx.delete(table.entity).where(eq(table.id, id)).returning());
|
|
188
|
-
|
|
189
|
-
return null;
|
|
190
|
-
await audit(tx, table, id, "delete", row, actor);
|
|
191
|
-
return { id };
|
|
183
|
+
return row === null ? null : { id };
|
|
192
184
|
});
|
|
193
185
|
}
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
row_id: rowId,
|
|
198
|
-
action,
|
|
199
|
-
changes,
|
|
200
|
-
actor_kind: "person",
|
|
201
|
-
actor_id: actor.id,
|
|
202
|
-
actor_name: actor.name
|
|
203
|
-
});
|
|
186
|
+
/** Who the table's trigger records in light_changes for the rest of this transaction. */
|
|
187
|
+
function nameActor(tx, actor) {
|
|
188
|
+
return tx.execute(sql `select set_config('light.actor_kind', 'person', true), set_config('light.actor_id', coalesce(${actor.id}, ''), true), set_config('light.actor_name', ${actor.name}, true)`);
|
|
204
189
|
}
|
|
205
190
|
/** The row a statement by primary key touched, or null when there is none. */
|
|
206
191
|
function atMostOne(rows) {
|
|
@@ -218,10 +203,6 @@ function exactlyOne(rows) {
|
|
|
218
203
|
}
|
|
219
204
|
// What's stored may come from an agent's own write: read as it is, never refused here.
|
|
220
205
|
const storedSources = z.record(z.string(), z.unknown());
|
|
221
|
-
const storedId = z.object({ id: z.uuid() });
|
|
222
|
-
function rowId(row) {
|
|
223
|
-
return storedId.parse(row).id;
|
|
224
|
-
}
|
|
225
206
|
function withoutSources(stored, columns) {
|
|
226
207
|
const sources = storedSources.parse(stored);
|
|
227
208
|
return Object.fromEntries(Object.entries(sources).filter(([column]) => !columns.includes(column)));
|
package/dist/server/env.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
/** Read when asked, not at import: a build imports this without a runtime env. */
|
|
2
2
|
export declare function env(): {
|
|
3
3
|
databaseUrl: string | undefined;
|
|
4
|
+
unpooledDatabaseUrl: string | undefined;
|
|
4
5
|
/** Sign-in needs Plaza, this app's id in it, and a database for sessions. */
|
|
5
6
|
configured: boolean;
|
|
6
7
|
/**
|
package/dist/server/env.js
CHANGED
|
@@ -4,12 +4,15 @@ import { z } from "zod/v4";
|
|
|
4
4
|
const serverEnv = z
|
|
5
5
|
.object({
|
|
6
6
|
DATABASE_URL: z.string().min(1).optional(),
|
|
7
|
+
/** The same database without a pooler: schema changes and their locks need a session. */
|
|
8
|
+
DATABASE_URL_UNPOOLED: z.string().min(1).optional(),
|
|
7
9
|
MII_PLAZA_URL: z.url().optional(),
|
|
8
10
|
MII_APP_ID: z.string().min(1).optional(),
|
|
9
11
|
NODE_ENV: z.string().optional()
|
|
10
12
|
})
|
|
11
13
|
.transform((env) => ({
|
|
12
14
|
databaseUrl: env.DATABASE_URL,
|
|
15
|
+
unpooledDatabaseUrl: env.DATABASE_URL_UNPOOLED,
|
|
13
16
|
/** Sign-in needs Plaza, this app's id in it, and a database for sessions. */
|
|
14
17
|
configured: Boolean(env.MII_PLAZA_URL && env.MII_APP_ID && env.DATABASE_URL),
|
|
15
18
|
/**
|
package/package.json
CHANGED