@ultimat3/db 29.4.0 → 30.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +74 -110
- package/README.md +163 -15
- package/package.json +13 -2
- package/src/app-relation.ts +7 -2
- package/src/catalog-fold.ts +5 -0
- package/src/catalog-objects.ts +20 -24
- package/src/catalog-policies.ts +81 -0
- package/src/catalog-relations.ts +31 -10
- package/src/catalog.ts +40 -1
- package/src/check-ddl.ts +34 -7
- package/src/client.ts +132 -48
- package/src/column-alter.ts +42 -1
- package/src/concurrent-index.ts +20 -0
- package/src/concurrent-plan.ts +135 -17
- package/src/connection-url.ts +16 -4
- package/src/db-diagnostics-deadline.ts +43 -0
- package/src/db-diagnostics-redact.ts +58 -0
- package/src/db-diagnostics-rows.ts +166 -0
- package/src/db-diagnostics-sql.ts +92 -0
- package/src/db-diagnostics.ts +178 -0
- package/src/destructive.ts +17 -3
- package/src/diagnostics-entry.ts +15 -0
- package/src/drift-errors.ts +9 -4
- package/src/drift-findings.ts +47 -62
- package/src/drift-missing.ts +203 -0
- package/src/drift-row-security.ts +88 -0
- package/src/drift-triggers.ts +54 -0
- package/src/drift.ts +88 -21
- package/src/entity-shape.ts +67 -0
- package/src/errors.ts +77 -9
- package/src/expected-loop.ts +34 -4
- package/src/failure-store-pg.ts +157 -0
- package/src/foreign-key-plan.ts +75 -23
- package/src/foreign-key.ts +76 -5
- package/src/generate-counter.ts +85 -0
- package/src/generate-history.ts +142 -0
- package/src/generate-row-security.ts +51 -0
- package/src/generate-snapshot.ts +82 -0
- package/src/generate-steps.ts +6 -1
- package/src/generate-triggers.ts +119 -0
- package/src/generate-view.ts +97 -0
- package/src/generate.ts +105 -79
- package/src/index.ts +129 -6
- package/src/introspect-catalog.ts +26 -6
- package/src/introspect-rows.ts +82 -0
- package/src/introspect.ts +108 -50
- package/src/keyset.ts +182 -0
- package/src/lifecycle-errors.ts +15 -0
- package/src/local-settings.ts +38 -0
- package/src/marker-line.ts +12 -4
- package/src/migrate-lock-retry.ts +60 -0
- package/src/migrate-rollback.ts +45 -6
- package/src/migrate.ts +45 -4
- package/src/migration-errors.ts +27 -1
- package/src/migration-ledger.ts +38 -4
- package/src/migration-step.ts +50 -22
- package/src/partition-order.ts +56 -0
- package/src/partitioned-tables.ts +24 -0
- package/src/pool-gate.ts +120 -0
- package/src/pool-listeners.ts +143 -0
- package/src/pool-pins.ts +68 -0
- package/src/pool-profile.ts +23 -7
- package/src/pool-reserve.ts +11 -4
- package/src/pool-retire.ts +22 -0
- package/src/pool-sweep.ts +115 -0
- package/src/pool-timer.ts +10 -0
- package/src/pooler-fix.ts +53 -0
- package/src/pooler.ts +161 -0
- package/src/read-transaction.ts +59 -0
- package/src/readonly-query.ts +4 -4
- package/src/replica-client.ts +54 -22
- package/src/replica-position.ts +89 -0
- package/src/replica-scope.ts +25 -4
- package/src/retired.ts +131 -0
- package/src/retype-dependents.ts +2 -0
- package/src/rolling-errors.ts +97 -0
- package/src/rolling.ts +212 -0
- package/src/row-secure-client.ts +54 -0
- package/src/row-security-catalog.ts +26 -0
- package/src/row-security-migrator.ts +48 -0
- package/src/row-security-plan.ts +58 -0
- package/src/row-security-role.ts +60 -0
- package/src/row-security-roles.ts +64 -0
- package/src/row-security.ts +59 -0
- package/src/same-tenant.ts +173 -0
- package/src/schema-dump-policies.ts +26 -0
- package/src/schema-dump-table.ts +21 -6
- package/src/schema-dump.ts +13 -2
- package/src/snapshot-parse.ts +42 -2
- package/src/sqlstate.ts +4 -0
- package/src/statement-bound.ts +82 -0
- package/src/transaction-errors.ts +14 -0
- package/src/transaction-guard.ts +40 -0
- package/src/transaction-settings.ts +24 -0
- package/src/transaction.ts +74 -19
- package/src/ungeneratable.ts +30 -0
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
// The durable failure-bundle store: `x_failures`, written OUTSIDE the request's transaction (which
|
|
2
|
+
// a 5xx rolled back), read by `x repro` and purged on a window the app chooses. Bundles only:
|
|
3
|
+
// counts per fingerprint are `x_error_groups`' (one table per fact).
|
|
4
|
+
|
|
5
|
+
import type { FailureBundle, FailureStore } from '@ultimat3/core';
|
|
6
|
+
import { finiteCount, isJsonObject } from '@ultimat3/core';
|
|
7
|
+
import type { DbClient } from './client';
|
|
8
|
+
import { db } from './client';
|
|
9
|
+
import { outsideTransaction } from './transaction';
|
|
10
|
+
|
|
11
|
+
export const FAILURES_TABLE = 'x_failures';
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Applied by the boot (`FRAMEWORK_SCHEMA`), never by an app migration, whether or not capture is
|
|
15
|
+
* on — enabling it must not be the deploy that finds the table missing. `bundle` is the whole
|
|
16
|
+
* `FailureBundle`; the columns beside it are the lookups `x repro` makes.
|
|
17
|
+
*/
|
|
18
|
+
export const SQL_FAILURES_TABLE = `
|
|
19
|
+
create table if not exists x_failures (
|
|
20
|
+
id uuid primary key,
|
|
21
|
+
fingerprint text not null,
|
|
22
|
+
code text not null,
|
|
23
|
+
at timestamptz not null,
|
|
24
|
+
request_id text,
|
|
25
|
+
trace_id text,
|
|
26
|
+
build text,
|
|
27
|
+
primitive text not null,
|
|
28
|
+
name text not null,
|
|
29
|
+
bundle jsonb not null
|
|
30
|
+
);
|
|
31
|
+
|
|
32
|
+
create index if not exists x_failures_fingerprint_idx on x_failures (fingerprint, at desc);
|
|
33
|
+
create index if not exists x_failures_request_idx on x_failures (request_id);
|
|
34
|
+
create index if not exists x_failures_trace_idx on x_failures (trace_id);
|
|
35
|
+
create index if not exists x_failures_at_idx on x_failures (at);
|
|
36
|
+
`;
|
|
37
|
+
|
|
38
|
+
/** The most bundles one `list` answers. */
|
|
39
|
+
export const MAX_FAILURE_LIST = 200;
|
|
40
|
+
|
|
41
|
+
export interface FailureSummary {
|
|
42
|
+
readonly id: string;
|
|
43
|
+
readonly fingerprint: string;
|
|
44
|
+
readonly code: string;
|
|
45
|
+
readonly at: string;
|
|
46
|
+
readonly requestId: string | null;
|
|
47
|
+
readonly name: string;
|
|
48
|
+
readonly build: string | null;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export interface PostgresFailureStore extends FailureStore {
|
|
52
|
+
/** By bundle id, request id or trace id — newest first. */
|
|
53
|
+
find(id: string): Promise<FailureBundle | undefined>;
|
|
54
|
+
/** The newest bundle of a group. */
|
|
55
|
+
latest(fingerprint: string): Promise<FailureBundle | undefined>;
|
|
56
|
+
list(options?: {
|
|
57
|
+
readonly code?: string;
|
|
58
|
+
readonly limit?: number;
|
|
59
|
+
}): Promise<readonly FailureSummary[]>;
|
|
60
|
+
purge(beforeMs: number): Promise<number>;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
interface BundleRow {
|
|
64
|
+
readonly bundle: unknown;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
const bundleOf = (rows: readonly BundleRow[]): FailureBundle | undefined => {
|
|
68
|
+
const raw = rows[0]?.bundle;
|
|
69
|
+
const parsed: unknown = typeof raw === 'string' ? JSON.parse(raw) : raw;
|
|
70
|
+
// Written by `save` below from a `FailureBundle`; a row that is not an object is not one.
|
|
71
|
+
return isJsonObject(parsed) ? (parsed as unknown as FailureBundle) : undefined;
|
|
72
|
+
};
|
|
73
|
+
|
|
74
|
+
const iso = (value: unknown): string =>
|
|
75
|
+
value instanceof Date ? value.toISOString() : new Date(String(value)).toISOString();
|
|
76
|
+
|
|
77
|
+
export function postgresFailureStore(
|
|
78
|
+
options: { readonly client?: DbClient } = {},
|
|
79
|
+
): PostgresFailureStore {
|
|
80
|
+
const client = (): DbClient => options.client ?? db();
|
|
81
|
+
// Never inside the caller's transaction: the request that failed rolled it back.
|
|
82
|
+
const send = <T>(text: string, values: readonly unknown[]): Promise<readonly T[]> =>
|
|
83
|
+
outsideTransaction(() => client().query<T>({ text, values: [...values] }));
|
|
84
|
+
return {
|
|
85
|
+
async save(bundle) {
|
|
86
|
+
await send(
|
|
87
|
+
`insert into ${FAILURES_TABLE} (id, fingerprint, code, at, request_id, trace_id, build, primitive, name, bundle) values ($1, $2, $3, $4::timestamptz, $5, $6, $7, $8, $9, $10::text::jsonb) on conflict (id) do nothing`,
|
|
88
|
+
[
|
|
89
|
+
bundle.id,
|
|
90
|
+
bundle.fingerprint,
|
|
91
|
+
bundle.code,
|
|
92
|
+
new Date(bundle.at).toISOString(),
|
|
93
|
+
bundle.requestId,
|
|
94
|
+
bundle.traceId,
|
|
95
|
+
bundle.build,
|
|
96
|
+
bundle.primitive,
|
|
97
|
+
bundle.name,
|
|
98
|
+
JSON.stringify(bundle),
|
|
99
|
+
],
|
|
100
|
+
);
|
|
101
|
+
},
|
|
102
|
+
async find(id) {
|
|
103
|
+
return bundleOf(
|
|
104
|
+
await send<BundleRow>(
|
|
105
|
+
`select bundle from ${FAILURES_TABLE} where id::text = $1 or request_id = $1 or trace_id = $1 order by at desc limit 1`,
|
|
106
|
+
[id],
|
|
107
|
+
),
|
|
108
|
+
);
|
|
109
|
+
},
|
|
110
|
+
async latest(fingerprint) {
|
|
111
|
+
return bundleOf(
|
|
112
|
+
await send<BundleRow>(
|
|
113
|
+
`select bundle from ${FAILURES_TABLE} where fingerprint = $1 order by at desc limit 1`,
|
|
114
|
+
[fingerprint],
|
|
115
|
+
),
|
|
116
|
+
);
|
|
117
|
+
},
|
|
118
|
+
async list(query = {}) {
|
|
119
|
+
const limit = Math.min(
|
|
120
|
+
finiteCount('postgresFailureStore.list', 'limit', query.limit ?? 50, 1),
|
|
121
|
+
MAX_FAILURE_LIST,
|
|
122
|
+
);
|
|
123
|
+
const rows = await send<Record<string, unknown>>(
|
|
124
|
+
`select id::text as id, fingerprint, code, at, request_id, name, build from ${FAILURES_TABLE}${query.code === undefined ? '' : ' where code = $2'} order by at desc limit $1`,
|
|
125
|
+
query.code === undefined ? [limit] : [limit, query.code],
|
|
126
|
+
);
|
|
127
|
+
return rows.map((row) => ({
|
|
128
|
+
id: String(row['id']),
|
|
129
|
+
fingerprint: String(row['fingerprint']),
|
|
130
|
+
code: String(row['code']),
|
|
131
|
+
at: iso(row['at']),
|
|
132
|
+
requestId: typeof row['request_id'] === 'string' ? row['request_id'] : null,
|
|
133
|
+
name: String(row['name']),
|
|
134
|
+
build: typeof row['build'] === 'string' ? row['build'] : null,
|
|
135
|
+
}));
|
|
136
|
+
},
|
|
137
|
+
async purge(beforeMs) {
|
|
138
|
+
const rows = await send<{ removed: number | string }>(
|
|
139
|
+
`with gone as (delete from ${FAILURES_TABLE} where at < $1::timestamptz returning 1) select count(*)::int as removed from gone`,
|
|
140
|
+
[new Date(beforeMs).toISOString()],
|
|
141
|
+
);
|
|
142
|
+
return Number(rows[0]?.removed ?? 0);
|
|
143
|
+
},
|
|
144
|
+
};
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
* A `purge()` target (`@ultimat3/jobs`, structural): bundles older than `keepMs`, measured on the
|
|
149
|
+
* JOB's clock. No default window: how long user input may be kept is the app's decision.
|
|
150
|
+
*/
|
|
151
|
+
export function failurePurgeTarget(
|
|
152
|
+
store: Pick<PostgresFailureStore, 'purge'>,
|
|
153
|
+
options: { readonly keepMs: number },
|
|
154
|
+
): { readonly name: string; purgeExpired(nowMs: number): Promise<number> } {
|
|
155
|
+
const keepMs = finiteCount('failurePurgeTarget', 'keepMs', options.keepMs, 1);
|
|
156
|
+
return { name: FAILURES_TABLE, purgeExpired: (nowMs) => store.purge(nowMs - keepMs) };
|
|
157
|
+
}
|
package/src/foreign-key-plan.ts
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
|
-
// Single responsibility: which foreign keys a migration
|
|
2
|
-
//
|
|
3
|
-
//
|
|
1
|
+
// Single responsibility: which foreign keys a migration adds or drops, and where each statement
|
|
2
|
+
// goes — `constraints` AFTER the table statements, `preDrops` BEFORE them, or the deferred step for
|
|
3
|
+
// a key on a live table, whose add and validate would scan it under a lock that stops writes.
|
|
4
|
+
// `generate.ts` assembles the plan; `foreign-key.ts` writes the SQL.
|
|
4
5
|
|
|
6
|
+
import { type ConcurrentPlan, contractKey, deferKey, targetIndexDeferred } from './concurrent-plan';
|
|
5
7
|
import type { EntityDescriptionLike } from './entity-shape';
|
|
6
8
|
import {
|
|
7
9
|
addForeignKey,
|
|
@@ -9,9 +11,11 @@ import {
|
|
|
9
11
|
foreignKeyTarget,
|
|
10
12
|
keyId,
|
|
11
13
|
onDeleteRule,
|
|
14
|
+
referenceParts,
|
|
12
15
|
unrestorableNote,
|
|
13
16
|
} from './foreign-key';
|
|
14
17
|
import type { ForeignKeyDescription, TableDescription } from './introspect';
|
|
18
|
+
import { sameTenantKeyName } from './same-tenant';
|
|
15
19
|
|
|
16
20
|
/** The two directions of one migration, pushed in `up` order; `down` is reversed at assembly. */
|
|
17
21
|
export interface Plan {
|
|
@@ -19,16 +23,6 @@ export interface Plan {
|
|
|
19
23
|
readonly down: string[];
|
|
20
24
|
}
|
|
21
25
|
|
|
22
|
-
/**
|
|
23
|
-
* The two ends of a `references()`, which entity renders as `"<table>.<column>"`. Read once: the
|
|
24
|
-
* clause that writes the constraint and the snapshot that records it must not disagree about what
|
|
25
|
-
* it points at, or drift reports a key the database holds exactly as declared.
|
|
26
|
-
*/
|
|
27
|
-
export function referenceParts(references: string): readonly [string, string] {
|
|
28
|
-
const [table = references, column = 'id'] = references.split('.');
|
|
29
|
-
return [table, column];
|
|
30
|
-
}
|
|
31
|
-
|
|
32
26
|
/**
|
|
33
27
|
* The keys this entity declares, recorded so drift can see one dropped by hand. Recording
|
|
34
28
|
* `foreignKeys: []` while the same run emitted a constraint was a snapshot claiming a constraint
|
|
@@ -43,12 +37,28 @@ export function referenceParts(references: string): readonly [string, string] {
|
|
|
43
37
|
* `onDelete` is the column's own, and `addForeignKey` spells it: a rule recorded here while no
|
|
44
38
|
* clause declared one would be a claim about the database that is not true, which is what it was
|
|
45
39
|
* until the clause learned to write it out.
|
|
40
|
+
*
|
|
41
|
+
* A column carrying `sameTenant` is the composite `(tenant, column) -> target (tenant, target)`, under
|
|
42
|
+
* a name of its own so the plain key it replaces on a live table can stand until it validates
|
|
43
|
+
* (`sameTenantKeyName`). Its `set null` names the one column: bare, Postgres nulls every column of a
|
|
44
|
+
* key, and that includes the tenant.
|
|
46
45
|
*/
|
|
47
46
|
export function foreignKeysOf(entity: EntityDescriptionLike): ForeignKeyDescription[] {
|
|
48
47
|
return entity.columns
|
|
49
48
|
.filter((column) => column.references !== null)
|
|
50
49
|
.map((column): ForeignKeyDescription => {
|
|
51
50
|
const [table, key] = referenceParts(column.references ?? '');
|
|
51
|
+
const tenant = column.sameTenant;
|
|
52
|
+
if (tenant !== null && tenant !== undefined) {
|
|
53
|
+
const rule = column.onDelete ?? null;
|
|
54
|
+
return {
|
|
55
|
+
name: sameTenantKeyName(entity.table, column.column),
|
|
56
|
+
columns: [tenant.column, column.column],
|
|
57
|
+
referencedTable: table,
|
|
58
|
+
referencedColumns: [tenant.targetColumn, key],
|
|
59
|
+
onDelete: rule === 'set null' ? `set null (${column.column})` : rule,
|
|
60
|
+
};
|
|
61
|
+
}
|
|
52
62
|
return {
|
|
53
63
|
name: `${entity.table}_${column.column}_fkey`,
|
|
54
64
|
columns: [column.column],
|
|
@@ -92,14 +102,35 @@ export function foreignKeysOf(entity: EntityDescriptionLike): ForeignKeyDescript
|
|
|
92
102
|
* every table statement; one the entity dropped is left where the retype left it, because dropping
|
|
93
103
|
* it a second time is `42704`; and one whose `on delete` moved comes back carrying the new rule.
|
|
94
104
|
* Its `down` is the retype's, pushed where reversal puts it after both ends are back.
|
|
105
|
+
*
|
|
106
|
+
* WHICH PHASE. A key on a table in `live` goes to the deferred step (`deferKey`): `not valid`, then
|
|
107
|
+
* `validate`. So does a key on a table this migration creates whose target index only the deferred
|
|
108
|
+
* step builds, since `add constraint` before that index exists is `42830`. One path for every such
|
|
109
|
+
* key, whether or not its target index is new, so the question is never asked twice. Two stay
|
|
110
|
+
* transactional: a key on a new table over a target that already has its index (nothing to scan,
|
|
111
|
+
* byte-identical to what every table's first migration always said), and a retype's re-add, which
|
|
112
|
+
* holds ACCESS EXCLUSIVE for the rewrite already, so a second pass buys nothing and costs a window
|
|
113
|
+
* with no key. A recorded key whose column a deferred key now covers goes in the contract phase;
|
|
114
|
+
* until then the old key still guards every write.
|
|
95
115
|
*/
|
|
96
116
|
export function foreignKeyPlan(
|
|
97
117
|
entity: EntityDescriptionLike,
|
|
98
118
|
live: TableDescription | undefined,
|
|
99
119
|
plans: ConstraintPlans,
|
|
100
120
|
): void {
|
|
101
|
-
const { constraints, preDrops, doomed, predropped } = plans;
|
|
121
|
+
const { constraints, preDrops, doomed, predropped, concurrent } = plans;
|
|
102
122
|
const wanted = foreignKeysOf(entity);
|
|
123
|
+
const movedByRetype = new Set(
|
|
124
|
+
(live?.foreignKeys ?? [])
|
|
125
|
+
.filter((key) => predropped.has(keyId(entity.table, key.name)))
|
|
126
|
+
.map(foreignKeyTarget),
|
|
127
|
+
);
|
|
128
|
+
const deferredColumns = new Set<string>();
|
|
129
|
+
const swapped = new Set<string>();
|
|
130
|
+
const defer = (key: ForeignKeyDescription, recorded?: ForeignKeyDescription): void => {
|
|
131
|
+
deferKey(entity.table, key, recorded, concurrent);
|
|
132
|
+
for (const column of key.columns) deferredColumns.add(column);
|
|
133
|
+
};
|
|
103
134
|
const recordedKeys = (live?.foreignKeys ?? []).filter(
|
|
104
135
|
(key) => !predropped.has(keyId(entity.table, key.name)),
|
|
105
136
|
);
|
|
@@ -114,6 +145,19 @@ export function foreignKeyPlan(
|
|
|
114
145
|
continue;
|
|
115
146
|
}
|
|
116
147
|
if (recorded === undefined) {
|
|
148
|
+
const onLiveTable = live !== undefined && !movedByRetype.has(foreignKeyTarget(key));
|
|
149
|
+
if (
|
|
150
|
+
onLiveTable ||
|
|
151
|
+
targetIndexDeferred(concurrent, key.referencedTable, key.referencedColumns)
|
|
152
|
+
) {
|
|
153
|
+
// A recorded key repointed under its own name: the one ALTER is its contract too.
|
|
154
|
+
const named = recordedKeys.find((each) => each.name === key.name);
|
|
155
|
+
if (named !== undefined) swapped.add(named.name);
|
|
156
|
+
// Moved away from a table this migration drops: `preDrops` takes it first, and there is
|
|
157
|
+
// nothing to restore it onto, so the snapshot and the `down` treat this key as new.
|
|
158
|
+
defer(key, named !== undefined && doomed.has(named.referencedTable) ? undefined : named);
|
|
159
|
+
continue;
|
|
160
|
+
}
|
|
117
161
|
constraints.up.push(addForeignKey(entity.table, key));
|
|
118
162
|
constraints.down.push(dropForeignKey(entity.table, key.name));
|
|
119
163
|
continue;
|
|
@@ -121,15 +165,13 @@ export function foreignKeyPlan(
|
|
|
121
165
|
// The rule is not part of a key's identity, so the same key under a new one is a rebuild —
|
|
122
166
|
// Postgres has no `alter constraint` for it, the same reason `redefineIndex` recreates.
|
|
123
167
|
if (onDeleteRule(recorded.onDelete) === onDeleteRule(key.onDelete)) continue;
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
dropForeignKey(entity.table, key.name),
|
|
132
|
-
);
|
|
168
|
+
// Under its own name the one ALTER swaps it; under another the recorded key holds till contract.
|
|
169
|
+
if (recorded.name === key.name) {
|
|
170
|
+
defer(key, recorded);
|
|
171
|
+
} else {
|
|
172
|
+
defer(key);
|
|
173
|
+
contractKey(entity.table, recorded, concurrent);
|
|
174
|
+
}
|
|
133
175
|
}
|
|
134
176
|
const declared = new Set(wanted.map(foreignKeyTarget));
|
|
135
177
|
const columns = new Set(entity.columns.map((column) => column.column));
|
|
@@ -142,6 +184,11 @@ export function foreignKeyPlan(
|
|
|
142
184
|
unrestorableDrop(entity.table, key.name, key.referencedTable, preDrops);
|
|
143
185
|
continue;
|
|
144
186
|
}
|
|
187
|
+
if (swapped.has(key.name)) continue;
|
|
188
|
+
if (key.columns.some((column) => deferredColumns.has(column))) {
|
|
189
|
+
contractKey(entity.table, key, concurrent);
|
|
190
|
+
continue;
|
|
191
|
+
}
|
|
145
192
|
constraints.up.push(dropForeignKey(entity.table, key.name));
|
|
146
193
|
constraints.down.push(addForeignKey(entity.table, key));
|
|
147
194
|
}
|
|
@@ -160,6 +207,11 @@ export interface ConstraintPlans {
|
|
|
160
207
|
* `add constraint`, and it is this function that writes it.
|
|
161
208
|
*/
|
|
162
209
|
readonly predropped: ReadonlySet<string>;
|
|
210
|
+
/**
|
|
211
|
+
* The deferred step. Filled by every table's `diffTable` BEFORE this function runs for any table,
|
|
212
|
+
* since a key's own table says nothing about whether its target's index is deferred.
|
|
213
|
+
*/
|
|
214
|
+
readonly concurrent: ConcurrentPlan;
|
|
163
215
|
}
|
|
164
216
|
|
|
165
217
|
/**
|
package/src/foreign-key.ts
CHANGED
|
@@ -40,6 +40,38 @@ export function onDeleteRule(raw: string | null): string | null {
|
|
|
40
40
|
|
|
41
41
|
const WRITABLE = new Set(['cascade', 'restrict', 'set null', 'set default']);
|
|
42
42
|
|
|
43
|
+
/**
|
|
44
|
+
* `set null (a, b)`: only those columns go, the form a composite key needs to keep its other half
|
|
45
|
+
* (Postgres 15+). The list is the rule's own text in a description and in the snapshot, so a key
|
|
46
|
+
* that changes it reads as a changed rule, and `introspectSchema()` answers in the same spelling.
|
|
47
|
+
*/
|
|
48
|
+
const SET_NULL_COLUMNS = /^set null \((.+)\)$/;
|
|
49
|
+
|
|
50
|
+
/** The clause for a screened rule: `identifier` for each name, never the text as it arrived. */
|
|
51
|
+
const ruleClause = (rule: string | null): string => {
|
|
52
|
+
if (rule === null) return '';
|
|
53
|
+
const listed = SET_NULL_COLUMNS.exec(rule)?.[1];
|
|
54
|
+
if (listed === undefined) return ` on delete ${rule}`;
|
|
55
|
+
return ` on delete set null (${quoted(listed.split(',').map((name) => name.trim()))})`;
|
|
56
|
+
};
|
|
57
|
+
|
|
58
|
+
/** Whether a rule is one `addClause` may write: a listed column has to be one of the key's own. */
|
|
59
|
+
const isWritable = (rule: string, key: ForeignKeyDescription): boolean => {
|
|
60
|
+
const listed = SET_NULL_COLUMNS.exec(rule)?.[1];
|
|
61
|
+
if (listed === undefined) return WRITABLE.has(rule);
|
|
62
|
+
return listed.split(',').every((name) => key.columns.includes(name.trim()));
|
|
63
|
+
};
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* The two ends of a `references()`, which entity renders as `"<table>.<column>"`. Read once: the
|
|
67
|
+
* clause that writes the constraint and the snapshot that records it must not disagree about what
|
|
68
|
+
* it points at, or drift reports a key the database holds exactly as declared.
|
|
69
|
+
*/
|
|
70
|
+
export function referenceParts(references: string): readonly [string, string] {
|
|
71
|
+
const [table = references, column = 'id'] = references.split('.');
|
|
72
|
+
return [table, column];
|
|
73
|
+
}
|
|
74
|
+
|
|
43
75
|
/**
|
|
44
76
|
* A key's identity: its columns, its target table, its target columns — never its name. Postgres
|
|
45
77
|
* names an inline `references` clause `<table>_<column>_fkey` and a hand-written migration may have
|
|
@@ -91,26 +123,65 @@ const quoted = (names: readonly string[]): string =>
|
|
|
91
123
|
* `X_INVARIANT` rather than DDL, the same discipline `createIndex` applies to an index naming no
|
|
92
124
|
* column: `entity()`'s option is a closed union, so only a hand-built description can get here.
|
|
93
125
|
*/
|
|
94
|
-
export function addForeignKey(
|
|
126
|
+
export function addForeignKey(
|
|
127
|
+
table: string,
|
|
128
|
+
key: ForeignKeyDescription,
|
|
129
|
+
options: { readonly notValid?: boolean } = {},
|
|
130
|
+
): string {
|
|
131
|
+
return (
|
|
132
|
+
`alter table ${identifier(table).text} ${addClause(table, key)}` +
|
|
133
|
+
`${options.notValid === true ? ' not valid' : ''};`
|
|
134
|
+
);
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/** The `add constraint …` clause alone, with the rule screened. Both statement forms end in it. */
|
|
138
|
+
function addClause(table: string, key: ForeignKeyDescription): string {
|
|
95
139
|
const rule = onDeleteRule(key.onDelete);
|
|
96
140
|
assert(
|
|
97
|
-
rule === null ||
|
|
141
|
+
rule === null || isWritable(rule, key),
|
|
98
142
|
`foreign key "${key.name}" on "${table}" declares an unknown on delete rule`,
|
|
99
143
|
`references(() => target.id, { onDelete: 'cascade' }) # cascade | restrict | set null`,
|
|
100
144
|
);
|
|
101
145
|
return (
|
|
102
|
-
`
|
|
103
|
-
`foreign key (${quoted(key.columns)}) ` +
|
|
146
|
+
`add constraint ${identifier(key.name).text} foreign key (${quoted(key.columns)}) ` +
|
|
104
147
|
`references ${identifier(key.referencedTable).text} (${quoted(key.referencedColumns)})` +
|
|
105
|
-
|
|
148
|
+
ruleClause(rule)
|
|
106
149
|
);
|
|
107
150
|
}
|
|
108
151
|
|
|
152
|
+
/**
|
|
153
|
+
* The deferred step's add: `not valid`, so it scans nothing and new writes are held to it at once.
|
|
154
|
+
*
|
|
155
|
+
* One ALTER and not a drop beside an add. `add constraint` has no `if not exists`, so the drop is
|
|
156
|
+
* what makes the step re-runnable — and two statements would each take a transaction of their own
|
|
157
|
+
* (`migration-step.ts`), leaving a same-named rebuild with no key at all between them.
|
|
158
|
+
*/
|
|
159
|
+
export function replaceForeignKeyNotValid(table: string, key: ForeignKeyDescription): string {
|
|
160
|
+
return (
|
|
161
|
+
`alter table ${identifier(table).text} drop constraint if exists ${identifier(key.name).text}, ` +
|
|
162
|
+
`${addClause(table, key)} not valid;`
|
|
163
|
+
);
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
/**
|
|
167
|
+
* Scans the rows a `not valid` constraint let stand, under SHARE UPDATE EXCLUSIVE on the table:
|
|
168
|
+
* reads and writes keep flowing. On a constraint that is already valid it does nothing, so a step
|
|
169
|
+
* failing after this statement re-runs it harmlessly.
|
|
170
|
+
*/
|
|
171
|
+
export function validateConstraint(table: string, name: string): string {
|
|
172
|
+
return `alter table ${identifier(table).text} validate constraint ${identifier(name).text};`;
|
|
173
|
+
}
|
|
174
|
+
|
|
109
175
|
/** The reverse. Dropping a constraint loses nothing the database cannot rebuild. */
|
|
110
176
|
export function dropForeignKey(table: string, constraint: string): string {
|
|
111
177
|
return `alter table ${identifier(table).text} drop constraint ${identifier(constraint).text};`;
|
|
112
178
|
}
|
|
113
179
|
|
|
180
|
+
/** The deferred step's drop, re-runnable: the step re-runs from its first statement on a failure. */
|
|
181
|
+
export function dropForeignKeyIfExists(table: string, constraint: string): string {
|
|
182
|
+
return `alter table ${identifier(table).text} drop constraint if exists ${identifier(constraint).text};`;
|
|
183
|
+
}
|
|
184
|
+
|
|
114
185
|
/**
|
|
115
186
|
* What a `down` says in place of an `add constraint` it cannot run: the key's table or its target
|
|
116
187
|
* is dropped by this migration, so there is nothing to add the constraint back onto.
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
// Single responsibility: the trigger `references(…, { countedIn })` compiles to, and the
|
|
2
|
+
// recount that makes a counter true again. The trigger sits on the CHILD and moves the parent's
|
|
3
|
+
// integer in the same transaction, so `deleteWhere`, `upsertAll` and raw SQL keep it as an insert does.
|
|
4
|
+
|
|
5
|
+
import type { CounterDescriptionLike } from './entity-shape';
|
|
6
|
+
import { triggerInstall } from './generate-history';
|
|
7
|
+
import type { TriggerRecord } from './introspect';
|
|
8
|
+
import { identifier } from './sql';
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* `ultimate_count_<crc32 of parent.column>`: per counter, because each body names its own tables,
|
|
12
|
+
* and short, because a name past 63 bytes is truncated in silence and drift would compare the
|
|
13
|
+
* untruncated one forever. CRC32 is Bun's, stable across processes and platforms.
|
|
14
|
+
*/
|
|
15
|
+
export function counterTriggerName(parent: string, counter: CounterDescriptionLike): string {
|
|
16
|
+
const hash = Bun.hash.crc32(`${parent}.${counter.column}`).toString(16).padStart(8, '0');
|
|
17
|
+
return `ultimate_count_${hash}`;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/** Whether a row image (`old`/`new`) is counted: it names a parent and is not soft-deleted. */
|
|
21
|
+
const counted = (image: 'old' | 'new', counter: CounterDescriptionLike): string =>
|
|
22
|
+
`${image}.${identifier(counter.references).text} is not null` +
|
|
23
|
+
(counter.softDelete === null
|
|
24
|
+
? ''
|
|
25
|
+
: ` and ${image}.${identifier(counter.softDelete).text} is null`);
|
|
26
|
+
|
|
27
|
+
const bump = (parent: string, counter: CounterDescriptionLike, image: string, sign: string) =>
|
|
28
|
+
`update ${identifier(parent).text} set ${identifier(counter.column).text} = ` +
|
|
29
|
+
`${identifier(counter.column).text} ${sign} 1 where ${identifier(counter.key).text} = ` +
|
|
30
|
+
`${image}.${identifier(counter.references).text};`;
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Old image out, new image in: an insert adds one, a delete takes one, a re-parent or a soft delete
|
|
34
|
+
* moves it. Lock order is the PARENT row, so a hot parent serializes its writers — `countBy` plus a
|
|
35
|
+
* cache is the alternative where that matters. `after … for each row`, so a refused write moves
|
|
36
|
+
* nothing, and `update of` the two columns that can change the answer, so no other update pays.
|
|
37
|
+
*/
|
|
38
|
+
export function counterTrigger(parent: string, counter: CounterDescriptionLike): TriggerRecord {
|
|
39
|
+
const name = counterTriggerName(parent, counter);
|
|
40
|
+
const fn =
|
|
41
|
+
`create or replace function ${identifier(name).text}() returns trigger language plpgsql as ` +
|
|
42
|
+
`$counter$ begin if tg_op <> 'INSERT' and ${counted('old', counter)} then ` +
|
|
43
|
+
`${bump(parent, counter, 'old', '-')} end if; if tg_op <> 'DELETE' and ` +
|
|
44
|
+
`${counted('new', counter)} then ${bump(parent, counter, 'new', '+')} end if; ` +
|
|
45
|
+
'return null; end; $counter$;';
|
|
46
|
+
const watched = [counter.references, ...(counter.softDelete === null ? [] : [counter.softDelete])]
|
|
47
|
+
.map((column) => identifier(column).text)
|
|
48
|
+
.join(', ');
|
|
49
|
+
return triggerInstall(
|
|
50
|
+
name,
|
|
51
|
+
{ name, sql: fn },
|
|
52
|
+
`after insert or delete or update of ${watched}`,
|
|
53
|
+
counter.table,
|
|
54
|
+
'',
|
|
55
|
+
);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** The true count of one parent row, as a correlated subquery over `parent` aliased `p`. */
|
|
59
|
+
const truth = (counter: CounterDescriptionLike): string =>
|
|
60
|
+
`(select count(*)::integer from ${identifier(counter.table).text} c where ` +
|
|
61
|
+
`c.${identifier(counter.references).text} = p.${identifier(counter.key).text}` +
|
|
62
|
+
(counter.softDelete === null ? '' : ` and c.${identifier(counter.softDelete).text} is null`) +
|
|
63
|
+
')';
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Sets every drifted counter to the truth. A migration adding a counter runs it once, so a table
|
|
67
|
+
* that already holds rows starts true; `x db recount --write` runs it again, by key range.
|
|
68
|
+
*/
|
|
69
|
+
export function recountSql(parent: string, counter: CounterDescriptionLike): string {
|
|
70
|
+
return (
|
|
71
|
+
`update ${identifier(parent).text} p set ${identifier(counter.column).text} = ${truth(counter)} ` +
|
|
72
|
+
`where p.${identifier(counter.column).text} is distinct from ${truth(counter)};`
|
|
73
|
+
);
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/** The drifted rows, for `x db recount`'s report: key, stored and true count. */
|
|
77
|
+
export function recountDriftSql(parent: string, counter: CounterDescriptionLike): string {
|
|
78
|
+
return (
|
|
79
|
+
`select p.${identifier(counter.key).text}::text as "key", ` +
|
|
80
|
+
`p.${identifier(counter.column).text} as "stored", ${truth(counter)} as "actual" ` +
|
|
81
|
+
`from ${identifier(parent).text} p ` +
|
|
82
|
+
`where p.${identifier(counter.column).text} is distinct from ${truth(counter)} ` +
|
|
83
|
+
`order by p.${identifier(counter.key).text}`
|
|
84
|
+
);
|
|
85
|
+
}
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
// Single responsibility: the two triggers `entity({ history: true })` compiles to. One records a
|
|
2
|
+
// before and an after image of every row change into `<table>_history`; the other refuses every
|
|
3
|
+
// rewrite of that history, and every delete but a purge's. Triggers and not the repository, because
|
|
4
|
+
// only a trigger sees `updateWhere`, a cascade and raw SQL.
|
|
5
|
+
|
|
6
|
+
import type { HistoryDescriptionLike } from './entity-shape';
|
|
7
|
+
import type { TriggerRecord } from './introspect';
|
|
8
|
+
import { identifier, literal } from './sql';
|
|
9
|
+
|
|
10
|
+
/** On the PARENT table. One name everywhere: a trigger name is unique per table, never per schema. */
|
|
11
|
+
export const HISTORY_TRIGGER = 'ultimate_history';
|
|
12
|
+
/** On the HISTORY table. */
|
|
13
|
+
export const HISTORY_GUARD_TRIGGER = 'ultimate_history_guard';
|
|
14
|
+
export const HISTORY_FUNCTION = 'ultimate_record_history';
|
|
15
|
+
export const HISTORY_GUARD_FUNCTION = 'ultimate_guard_history';
|
|
16
|
+
|
|
17
|
+
/** The transaction-local settings the two functions read. `setLocal` is their one writer. */
|
|
18
|
+
export const ACTOR_SETTING = 'x.actor';
|
|
19
|
+
export const REQUEST_SETTING = 'x.request';
|
|
20
|
+
export const PURGE_SETTING = 'x.purge';
|
|
21
|
+
|
|
22
|
+
/** A JSON number's own digits, as text: `->>` spells a jsonb numeric exactly, scale included. */
|
|
23
|
+
const DIGITS = { type: 'number', text: '{image} ->> k' };
|
|
24
|
+
/** The instant `to_jsonb` spelled with the session's offset, re-spelled in UTC as `toISOString`. */
|
|
25
|
+
const INSTANT = {
|
|
26
|
+
type: 'string',
|
|
27
|
+
text: `to_char(({image} ->> k)::timestamptz at time zone 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')`,
|
|
28
|
+
};
|
|
29
|
+
|
|
30
|
+
/** One image's key `k` replaced by its text, when it holds the JSON type `to_jsonb` gave it. */
|
|
31
|
+
const exact = (image: string, how: { readonly type: string; readonly text: string }): string =>
|
|
32
|
+
`if jsonb_typeof(${image} -> k) = '${how.type}' then ${image} := jsonb_set(${image}, array[k], ` +
|
|
33
|
+
`coalesce(to_jsonb(${how.text.replaceAll('{image}', image)}), ${image} -> k)); end if;`;
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* One function for every history table: the trigger's arguments name the table, the key, the tenant
|
|
37
|
+
* column, the columns whose change alone records nothing (the counters the database keeps), and the
|
|
38
|
+
* columns `to_jsonb` spells otherwise than the row: an `int8`/`numeric` stored as its digits (a JSON
|
|
39
|
+
* number is a float to every reader past 2^53 and drops a scale's zeros), a `timestamptz` as UTC ISO
|
|
40
|
+
* to the millisecond (`to_jsonb` writes the SESSION zone's offset) — what the memory driver keeps.
|
|
41
|
+
* An argument a trigger installed before it existed reads NULL, and NULL is the empty list.
|
|
42
|
+
*
|
|
43
|
+
* `jsonb_populate_record` builds the history row, so every value lands in its column's own type —
|
|
44
|
+
* a uuid tenant from a jsonb text, with no cast spliced per table. A statement that changes nothing
|
|
45
|
+
* records nothing; the actor and request are what the WRITE'S transaction set, so a statement with
|
|
46
|
+
* no transaction around it records `null`, never a guess.
|
|
47
|
+
*/
|
|
48
|
+
export const HISTORY_FUNCTION_SQL =
|
|
49
|
+
`create or replace function ${identifier(HISTORY_FUNCTION).text}() returns trigger ` +
|
|
50
|
+
'language plpgsql as $history$ declare ' +
|
|
51
|
+
"source jsonb := case when tg_op = 'DELETE' then to_jsonb(old) else to_jsonb(new) end; " +
|
|
52
|
+
"prior jsonb := case when tg_op = 'INSERT' then null else to_jsonb(old) end; " +
|
|
53
|
+
'ignored text[] := tg_argv[3]::text[]; ' +
|
|
54
|
+
"digits text[] := coalesce(tg_argv[4], '{}')::text[]; " +
|
|
55
|
+
"instants text[] := coalesce(tg_argv[5], '{}')::text[]; " +
|
|
56
|
+
'k text; ' +
|
|
57
|
+
"begin if tg_op = 'UPDATE' and (prior - ignored) = (source - ignored) then return null; end if; " +
|
|
58
|
+
'foreach k in array digits loop ' +
|
|
59
|
+
`${exact('source', DIGITS)} ${exact('prior', DIGITS)} end loop; ` +
|
|
60
|
+
'foreach k in array instants loop ' +
|
|
61
|
+
`${exact('source', INSTANT)} ${exact('prior', INSTANT)} end loop; ` +
|
|
62
|
+
"execute format('insert into %I select * from jsonb_populate_record(null::%I, $1)', " +
|
|
63
|
+
'tg_argv[0], tg_argv[0]) using jsonb_build_object(' +
|
|
64
|
+
"'id', gen_random_uuid(), 'row_id', source ->> tg_argv[1], 'op', lower(tg_op), 'at', now(), " +
|
|
65
|
+
`'actor_id', nullif(current_setting('${ACTOR_SETTING}', true), ''), ` +
|
|
66
|
+
`'request_id', nullif(current_setting('${REQUEST_SETTING}', true), ''), ` +
|
|
67
|
+
"'before', prior, 'after', case when tg_op = 'DELETE' then null else source end) || " +
|
|
68
|
+
"case when tg_argv[2] = '' then '{}'::jsonb " +
|
|
69
|
+
'else jsonb_build_object(tg_argv[2], source -> tg_argv[2]) end; ' +
|
|
70
|
+
'return null; end; $history$;';
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Update refused always; delete refused unless `x.purge` names THIS table — `retainTarget` and a
|
|
74
|
+
* subject erase set it through `setLocal`. As `x.cross` is, this catches forgetting, not hostile SQL
|
|
75
|
+
* in the app's own process. The code leads the message and the SQLSTATE is `23001`, exactly as the
|
|
76
|
+
* append-only trigger's: a history table is append-only to everything but its sweep.
|
|
77
|
+
*/
|
|
78
|
+
export const HISTORY_GUARD_FUNCTION_SQL =
|
|
79
|
+
`create or replace function ${identifier(HISTORY_GUARD_FUNCTION).text}() returns trigger ` +
|
|
80
|
+
"language plpgsql as $history_guard$ begin if tg_op = 'DELETE' and " +
|
|
81
|
+
`current_setting('${PURGE_SETTING}', true) = tg_table_name then return old; end if; ` +
|
|
82
|
+
"raise exception 'X_ENTITY_APPEND_ONLY: % on %.% is refused, the table is history', " +
|
|
83
|
+
"tg_op, tg_table_schema, tg_table_name using errcode = '23001', hint = 'history is written " +
|
|
84
|
+
"by its trigger and removed only by purge({ entities: ''declared'' })'; end; $history_guard$;";
|
|
85
|
+
|
|
86
|
+
/** Function, drop-if-exists, create: idempotent over a missing and a DISABLED trigger alike. */
|
|
87
|
+
export const triggerInstall = (
|
|
88
|
+
name: string,
|
|
89
|
+
fn: { readonly name: string; readonly sql: string },
|
|
90
|
+
timing: string,
|
|
91
|
+
table: string,
|
|
92
|
+
args: string,
|
|
93
|
+
): TriggerRecord => ({
|
|
94
|
+
name,
|
|
95
|
+
install: [
|
|
96
|
+
fn.sql,
|
|
97
|
+
`drop trigger if exists ${identifier(name).text} on ${identifier(table).text};`,
|
|
98
|
+
`create trigger ${identifier(name).text} ${timing} on ${identifier(table).text} for each row ` +
|
|
99
|
+
`execute function ${identifier(fn.name).text}(${args});`,
|
|
100
|
+
],
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
/** A text[] literal of physical names — each one already a validated identifier. */
|
|
104
|
+
const nameArray = (names: readonly string[]): string =>
|
|
105
|
+
literal(`{${names.map((name) => identifier(name).text).join(',')}}`).text;
|
|
106
|
+
|
|
107
|
+
/** The parent's recording trigger and the history table's guard, keyed by the table each sits on. */
|
|
108
|
+
export function historyTriggers(
|
|
109
|
+
parent: string,
|
|
110
|
+
history: HistoryDescriptionLike,
|
|
111
|
+
): readonly (readonly [string, TriggerRecord])[] {
|
|
112
|
+
const args = [
|
|
113
|
+
literal(history.table).text,
|
|
114
|
+
literal(history.key).text,
|
|
115
|
+
literal(history.tenant ?? '').text,
|
|
116
|
+
nameArray(history.ignore),
|
|
117
|
+
nameArray(history.text ?? []),
|
|
118
|
+
nameArray(history.instants ?? []),
|
|
119
|
+
].join(', ');
|
|
120
|
+
return [
|
|
121
|
+
[
|
|
122
|
+
parent,
|
|
123
|
+
triggerInstall(
|
|
124
|
+
HISTORY_TRIGGER,
|
|
125
|
+
{ name: HISTORY_FUNCTION, sql: HISTORY_FUNCTION_SQL },
|
|
126
|
+
'after insert or update or delete',
|
|
127
|
+
parent,
|
|
128
|
+
args,
|
|
129
|
+
),
|
|
130
|
+
],
|
|
131
|
+
[
|
|
132
|
+
history.table,
|
|
133
|
+
triggerInstall(
|
|
134
|
+
HISTORY_GUARD_TRIGGER,
|
|
135
|
+
{ name: HISTORY_GUARD_FUNCTION, sql: HISTORY_GUARD_FUNCTION_SQL },
|
|
136
|
+
'before update or delete',
|
|
137
|
+
history.table,
|
|
138
|
+
'',
|
|
139
|
+
),
|
|
140
|
+
],
|
|
141
|
+
];
|
|
142
|
+
}
|