@everystack/cli 0.4.98 → 0.4.100
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +42 -0
- package/package.json +13 -3
- package/src/cli/apply-execute.ts +60 -1
- package/src/cli/authz-manifest.ts +93 -0
- package/src/cli/authz-redteam.ts +19 -10
- package/src/cli/backfill.ts +10 -4
- package/src/cli/backup.ts +21 -0
- package/src/cli/catalog-rename.ts +115 -0
- package/src/cli/commands/db-backup.ts +56 -17
- package/src/cli/commands/db-check.ts +16 -3
- package/src/cli/commands/db-fork.ts +12 -4
- package/src/cli/commands/db-generate.ts +7 -3
- package/src/cli/commands/db.ts +3 -2
- package/src/cli/commands/runbook.ts +3 -0
- package/src/cli/connection-fence.ts +351 -0
- package/src/cli/db-source.ts +11 -7
- package/src/cli/declared-derived.ts +6 -1
- package/src/cli/derived-apply.ts +20 -5
- package/src/cli/index.ts +1 -1
- package/src/cli/model-render.ts +19 -3
- package/src/cli/mutation-lease.ts +5 -0
- package/src/cli/plain-output.ts +32 -10
- package/src/cli/runbook/model-section.ts +5 -2
- package/src/cli/schema-compile.ts +22 -1
- package/src/cli/schema-diff.ts +18 -10
- package/src/cli/schema-introspect.ts +2 -0
- package/src/cli/select-grant-advisory.ts +132 -0
- package/src/cli/stage-backup.ts +57 -16
package/CHANGELOG.md
CHANGED
|
@@ -4,6 +4,48 @@ All notable changes to `@everystack/cli` are documented here.
|
|
|
4
4
|
|
|
5
5
|
Format based on [Keep a Changelog](https://keepachangelog.com/).
|
|
6
6
|
|
|
7
|
+
## [0.4.100] - 2026-10-16
|
|
8
|
+
|
|
9
|
+
### Added
|
|
10
|
+
|
|
11
|
+
- check() takes an explicit constraint name; db:pull keeps a live name
|
|
12
|
+
|
|
13
|
+
### Fixed
|
|
14
|
+
|
|
15
|
+
- db:apply runs a rename-only plan
|
|
16
|
+
|
|
17
|
+
### Documentation
|
|
18
|
+
|
|
19
|
+
- a check folded into .validate() takes the field check name
|
|
20
|
+
|
|
21
|
+
## [0.4.99] - 2026-10-14
|
|
22
|
+
|
|
23
|
+
### Added
|
|
24
|
+
|
|
25
|
+
- db:backup reads the origin commit in the fingerprint session
|
|
26
|
+
- render the declared-authz manifest from the checkout's models
|
|
27
|
+
- mirror originCommit and authzManifest into the CLI BackupManifest
|
|
28
|
+
- stamp the authz manifest after a proven stage backup
|
|
29
|
+
- db:backup renders the authz manifest and stamps it post-dump
|
|
30
|
+
- db:restore and db:fork write a restore witness from the backup's origin (#15)
|
|
31
|
+
- link table name references through model thunks
|
|
32
|
+
- db:check warns when a role can write a table but not read it
|
|
33
|
+
- the db:check one-line summary counts warnings
|
|
34
|
+
|
|
35
|
+
### Fixed
|
|
36
|
+
|
|
37
|
+
- normalize statement keys in plain reports
|
|
38
|
+
- lease the ops-Lambda write path, no driver fork
|
|
39
|
+
- link names before manifest and runbook rendering
|
|
40
|
+
- the select-grant warning no longer suggests a blanket read
|
|
41
|
+
- probe UPDATE per role in the authz red-team
|
|
42
|
+
- probe red-team UPDATE with SET col = DEFAULT
|
|
43
|
+
- db:doctor prints each fix once and an accurate closing line
|
|
44
|
+
|
|
45
|
+
### Documentation
|
|
46
|
+
|
|
47
|
+
- the select-grant advice is never broader than the write
|
|
48
|
+
|
|
7
49
|
## [0.4.98] - 2026-09-27
|
|
8
50
|
|
|
9
51
|
### Added
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@everystack/cli",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.100",
|
|
4
4
|
"description": "CLI and OTA updates for Expo apps on everystack",
|
|
5
5
|
"license": "AGPL-3.0-only",
|
|
6
6
|
"author": "Scalable Technology, Inc. <licensing@scalable.technology>",
|
|
@@ -58,6 +58,16 @@
|
|
|
58
58
|
"types": "./src/reconcile.ts",
|
|
59
59
|
"default": "./src/reconcile.ts"
|
|
60
60
|
},
|
|
61
|
+
"./mutation-lease": {
|
|
62
|
+
"source": "./src/cli/mutation-lease.ts",
|
|
63
|
+
"types": "./src/cli/mutation-lease.ts",
|
|
64
|
+
"default": "./src/cli/mutation-lease.ts"
|
|
65
|
+
},
|
|
66
|
+
"./connection-fence": {
|
|
67
|
+
"source": "./src/cli/connection-fence.ts",
|
|
68
|
+
"types": "./src/cli/connection-fence.ts",
|
|
69
|
+
"default": "./src/cli/connection-fence.ts"
|
|
70
|
+
},
|
|
61
71
|
"./refresh": {
|
|
62
72
|
"source": "./src/refresh.ts",
|
|
63
73
|
"types": "./src/refresh.ts",
|
|
@@ -135,7 +145,7 @@
|
|
|
135
145
|
"structured-headers": "1.0.1",
|
|
136
146
|
"tsx": "4.21.0",
|
|
137
147
|
"typescript": "5.9.3",
|
|
138
|
-
"@everystack/model": "0.4.
|
|
148
|
+
"@everystack/model": "0.4.22"
|
|
139
149
|
},
|
|
140
150
|
"peerDependencies": {
|
|
141
151
|
"@everystack/server": ">=0.4.0",
|
|
@@ -182,7 +192,7 @@
|
|
|
182
192
|
"react": "19.2.0",
|
|
183
193
|
"ts-jest": "29.4.9",
|
|
184
194
|
"zod": "3.25.67",
|
|
185
|
-
"@everystack/server": "0.4.
|
|
195
|
+
"@everystack/server": "0.4.35"
|
|
186
196
|
},
|
|
187
197
|
"scripts": {
|
|
188
198
|
"test": "jest",
|
package/src/cli/apply-execute.ts
CHANGED
|
@@ -73,6 +73,7 @@ import {
|
|
|
73
73
|
import { validationPendingReason, pendingValidationsAreSafe } from './apply-refusals';
|
|
74
74
|
import { ENSURE_RECONCILER_SQL, renderSchemaLogInsert, renderSchemaLogFingerprintUpdate } from './derived-apply';
|
|
75
75
|
import { checkPlanPrecondition, planHash, defaultAclHash, type EdgePlan } from './edge-plan';
|
|
76
|
+
import { renameEdgeVerdict } from './catalog-rename';
|
|
76
77
|
import type { DescentLedgerRow, LedgerRead } from './git-descent';
|
|
77
78
|
|
|
78
79
|
/**
|
|
@@ -416,6 +417,47 @@ export async function liveBaseAndLedger(session: SessionRunner): Promise<LiveBas
|
|
|
416
417
|
return { hash: fingerprintLive(snapshot, contract).hash, snapshot, contract, ledger, ledgerRead };
|
|
417
418
|
}
|
|
418
419
|
|
|
420
|
+
/**
|
|
421
|
+
* The commit that last deployed this target — the `git_ref` SCHEMA_LOG_ANCHOR_SQL answers, i.e.
|
|
422
|
+
* the newest successful apply that RECORDED a commit. An app-only apply without one never shadows
|
|
423
|
+
* it, exactly as it never shadows descent's anchor.
|
|
424
|
+
*
|
|
425
|
+
* Three answers, because the ledger read has three outcomes:
|
|
426
|
+
* - a string — the anchor's commit;
|
|
427
|
+
* - `null` — a CLEAN read proved there is none (no schema_log, or no commit-carrying apply);
|
|
428
|
+
* - undefined — the read failed, so nothing is known. Never `null`: that would claim "this
|
|
429
|
+
* target has no deploying commit" on the strength of a read that did not happen.
|
|
430
|
+
*/
|
|
431
|
+
export function originCommitFromLedger(ledger: DescentLedgerRow[], ledgerRead: LedgerRead): string | null | undefined {
|
|
432
|
+
if (ledgerRead.kind === 'unreadable') return undefined;
|
|
433
|
+
if (ledgerRead.kind === 'absent' || !ledgerRead.anchored) return null;
|
|
434
|
+
// ledgerFromSlots folds the anchor row into the ledger, so the anchor is the newest row (by id,
|
|
435
|
+
// never by clock — the anchor query's own order) that carries a commit.
|
|
436
|
+
let anchor: DescentLedgerRow | undefined;
|
|
437
|
+
for (const row of ledger) {
|
|
438
|
+
if (row.gitRef == null) continue;
|
|
439
|
+
if (anchor === undefined || BigInt(row.id) > BigInt(anchor.id)) anchor = row;
|
|
440
|
+
}
|
|
441
|
+
return anchor?.gitRef ?? null;
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
/** What a stage backup records about the live target at dump time. */
|
|
445
|
+
export interface BackupLiveRead {
|
|
446
|
+
fingerprint: string;
|
|
447
|
+
/** See originCommitFromLedger — undefined means the ledger read failed. */
|
|
448
|
+
originCommit: string | null | undefined;
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
/**
|
|
452
|
+
* The backup's live read: the base fingerprint AND the origin commit, in the ONE repeatable-read
|
|
453
|
+
* session liveBaseAndLedger already runs — the same fingerprint db:apply's lock computes, and the
|
|
454
|
+
* same anchor its descent proof reads. No second round trip.
|
|
455
|
+
*/
|
|
456
|
+
export async function readBackupLive(session: SessionRunner): Promise<BackupLiveRead> {
|
|
457
|
+
const { hash, ledger, ledgerRead } = await liveBaseAndLedger(session);
|
|
458
|
+
return { fingerprint: hash, originCommit: originCommitFromLedger(ledger, ledgerRead) };
|
|
459
|
+
}
|
|
460
|
+
|
|
419
461
|
/** Verify → apply → verify. Pure orchestration over an injected QueryRunner. */
|
|
420
462
|
export async function executeApplyPlan(
|
|
421
463
|
runner: QueryRunner,
|
|
@@ -436,7 +478,20 @@ export async function executeApplyPlan(
|
|
|
436
478
|
}
|
|
437
479
|
|
|
438
480
|
if (live === plan.toFingerprint) {
|
|
439
|
-
|
|
481
|
+
// The hash is blind to a catalog rename (a check is keyed by predicate, an index or key by
|
|
482
|
+
// content), so a plan of renames alone has from == to == live whether or not it ran. Only
|
|
483
|
+
// then do the names decide, read from the same snapshot as the lock. Not landed: fall
|
|
484
|
+
// through, and every gate below runs. Half landed, or in a shape the plan never predicted:
|
|
485
|
+
// stale. Everything else is today's idempotent re-apply.
|
|
486
|
+
const verdict = plan.fromFingerprint === plan.toFingerprint
|
|
487
|
+
? renameEdgeVerdict(plan.statements, before)
|
|
488
|
+
: { verdict: 'not-renames' as const };
|
|
489
|
+
if (verdict.verdict === 'stale') {
|
|
490
|
+
const reason = `this plan is stale: ${verdict.reason}. Re-mint with db:plan.`;
|
|
491
|
+
await recordRefusal(runner, plan, live, 'rename edge', reason, options);
|
|
492
|
+
return { status: 'refused', liveFingerprint: live, reason };
|
|
493
|
+
}
|
|
494
|
+
if (verdict.verdict !== 'unsettled') return { status: 'already-applied', liveFingerprint: live };
|
|
440
495
|
}
|
|
441
496
|
const pre = checkPlanPrecondition(plan, live);
|
|
442
497
|
if (!pre.ok) {
|
|
@@ -622,3 +677,7 @@ export async function executeApplyPlan(
|
|
|
622
677
|
// notably the ops Lambda's db:apply action — everything it needs to validate
|
|
623
678
|
// and run a plan without reaching into the CLI's internal module layout.
|
|
624
679
|
export { PLAN_VERSION, APPLY_CONTRACT_VERSION, planHash, checkPlanPrecondition, type EdgePlan } from './edge-plan';
|
|
680
|
+
|
|
681
|
+
// The ledger DDL, so the ops Lambda's restore dispatcher can create `everystack.schema_log` on a
|
|
682
|
+
// fresh target from the one copy every apply already runs — never a second copy of the DDL.
|
|
683
|
+
export { ENSURE_RECONCILER_SQL } from './derived-apply';
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The declared-authz manifest a backup records (restore-converges-authz R1).
|
|
3
|
+
*
|
|
4
|
+
* A restore lands the dump's data AND its privileges-free schema; converging authz afterwards needs
|
|
5
|
+
* to know what the Models declared. Compiling a HISTORICAL models tree to learn that was rejected —
|
|
6
|
+
* an old tree can fail to compile under the current CLI — so the backup records the answer at dump
|
|
7
|
+
* time instead, rendered from the models checked out when `db:backup` runs.
|
|
8
|
+
*
|
|
9
|
+
* Pure and a composition only. Every section is the output of the renderer that already owns it:
|
|
10
|
+
* - tables — compileTableContract, per model (what db:plan / db:check converge tables to);
|
|
11
|
+
* - derived — parseGrantAttachments per view/matview/function, plus the schema USAGE rule
|
|
12
|
+
* renderEnsureObjectSchemas applies (declaredObjectSchemaUsage);
|
|
13
|
+
* - owners — extractDeclaredOwnership (what db:provision's ownership convergence enacts/adopts).
|
|
14
|
+
* No logic is re-derived here: a second copy would drift from what the verbs actually converge to.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import { ModelLinkError } from '@everystack/model';
|
|
18
|
+
import type { TableContract } from './authz-contract';
|
|
19
|
+
import { compileTableContract } from './authz-compile';
|
|
20
|
+
import { parseGrantAttachments } from './derived-grants';
|
|
21
|
+
import { declaredObjectSchemaUsage } from './derived-apply';
|
|
22
|
+
import { extractDeclaredOwnership } from './commands/db';
|
|
23
|
+
import type { DeclaredDerived } from './declared-derived';
|
|
24
|
+
import type { DeclaredOwner, Adoptee } from '../wire';
|
|
25
|
+
import type { BackupAuthzManifest } from './backup';
|
|
26
|
+
|
|
27
|
+
/** One view / matview / function's declared grant contract, as parseGrantAttachments reads it. */
|
|
28
|
+
export interface DerivedGrantEntry {
|
|
29
|
+
identity: string;
|
|
30
|
+
kind: string;
|
|
31
|
+
schema: string;
|
|
32
|
+
/** The `ON …` target as the declared statements spell it, or null when none was declared. */
|
|
33
|
+
target: string | null;
|
|
34
|
+
/** grantee → sorted privileges. PUBLIC present-with-[] means explicitly revoked. */
|
|
35
|
+
grants: Record<string, string[]>;
|
|
36
|
+
/** A column-scoped grant is declared (not drift-checked by the reconciler). */
|
|
37
|
+
hasColumnGrants: boolean;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/** Narrows the mirrored BackupManifest envelope, so a rendered manifest is assignable to the field. */
|
|
41
|
+
export interface AuthzManifest extends BackupAuthzManifest {
|
|
42
|
+
version: 1;
|
|
43
|
+
/** compileTableContract(model) for every declared model, in declaration order. */
|
|
44
|
+
tables: TableContract[];
|
|
45
|
+
derived: {
|
|
46
|
+
/** Every derived object that carries grants — triggers and `sql` escape hatches do not. */
|
|
47
|
+
objects: DerivedGrantEntry[];
|
|
48
|
+
/** non-public schema → the roles granted USAGE on it (sorted; [] = the schema, no grant). */
|
|
49
|
+
schemaUsage: Record<string, string[]>;
|
|
50
|
+
};
|
|
51
|
+
/** The ENACT pairs and ADOPT objects db:provision converges ownership to. */
|
|
52
|
+
owners: { declaredOwners: DeclaredOwner[]; declaredObjects: Adoptee[] };
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** Render the declared-authz manifest from the checkout's models and derived descriptors. */
|
|
56
|
+
export function renderAuthzManifest(declared: Pick<DeclaredDerived, 'models' | 'objects'>): AuthzManifest {
|
|
57
|
+
const objects: DerivedGrantEntry[] = declared.objects
|
|
58
|
+
// The kinds the reconciler grant-checks (objectGrantDelta): triggers have no ACL, and an
|
|
59
|
+
// opaque `sql` object's grants are its own business.
|
|
60
|
+
.filter((o) => o.kind !== 'trigger' && o.kind !== 'sql')
|
|
61
|
+
.map((o) => {
|
|
62
|
+
const parsed = parseGrantAttachments(o.attachments);
|
|
63
|
+
return {
|
|
64
|
+
identity: o.identity, kind: o.kind, schema: o.schema,
|
|
65
|
+
target: parsed.target, grants: parsed.grants, hasColumnGrants: parsed.hasColumnGrants,
|
|
66
|
+
};
|
|
67
|
+
});
|
|
68
|
+
return {
|
|
69
|
+
version: 1,
|
|
70
|
+
tables: declared.models.map((m) => compileTableContract(m)),
|
|
71
|
+
derived: { objects, schemaUsage: declaredObjectSchemaUsage(declared.objects) },
|
|
72
|
+
owners: extractDeclaredOwnership(declared.objects),
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* What `db:backup` sends: the manifest rendered from the checkout, loaded through `load`
|
|
78
|
+
* (loadDeclaredDerived in the command). A models-less project has nothing declared, so it sends
|
|
79
|
+
* nothing and says nothing. A name-link failure refuses the backup. Other load failures warn —
|
|
80
|
+
* the dump is the safety net — so that is a warning and the backup proceeds without a manifest.
|
|
81
|
+
*/
|
|
82
|
+
export async function loadBackupAuthzManifest(
|
|
83
|
+
load: () => Promise<Pick<DeclaredDerived, 'models' | 'objects'> | null>,
|
|
84
|
+
): Promise<{ manifest?: AuthzManifest; warning?: string }> {
|
|
85
|
+
try {
|
|
86
|
+
const declared = await load();
|
|
87
|
+
return declared ? { manifest: renderAuthzManifest(declared) } : {};
|
|
88
|
+
} catch (err: unknown) {
|
|
89
|
+
if (err instanceof ModelLinkError) throw err;
|
|
90
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
91
|
+
return { warning: `the authz manifest could not be rendered from your models (${message}), so this backup will not carry one` };
|
|
92
|
+
}
|
|
93
|
+
}
|
package/src/cli/authz-redteam.ts
CHANGED
|
@@ -84,7 +84,7 @@ function pgArrayLiteral(values: string[]): string {
|
|
|
84
84
|
* transaction it always rolls back. Probe statements are privilege-isolated:
|
|
85
85
|
* SELECT 1 FROM t WHERE false — SELECT privilege, reads no rows
|
|
86
86
|
* INSERT INTO t DEFAULT VALUES — INSERT privilege (rolled back; NOT NULL = allowed)
|
|
87
|
-
* UPDATE t SET col =
|
|
87
|
+
* UPDATE t SET col = DEFAULT WHERE false — UPDATE privilege on col only, touches no rows
|
|
88
88
|
* DELETE FROM t WHERE false — DELETE privilege, touches no rows
|
|
89
89
|
* "permission denied" -> denied-privilege; "row-level security" -> denied-rls.
|
|
90
90
|
*
|
|
@@ -93,8 +93,9 @@ function pgArrayLiteral(values: string[]): string {
|
|
|
93
93
|
* privilege was present -> allowed. But a statement rejected at parse/analysis time never
|
|
94
94
|
* reached the permission check at all and proves NOTHING -> inconclusive. Classifying the
|
|
95
95
|
* second kind as `allowed` is what reported `anon` as able to UPDATE a table it cannot
|
|
96
|
-
* touch: the probe's `SET pk = pk` hit 428C9 ("can only be updated to DEFAULT") on an
|
|
97
|
-
* identity PK, and a non-grant then read as "enforcement exceeds declaration".
|
|
96
|
+
* touch: the probe's old `SET pk = pk` hit 428C9 ("can only be updated to DEFAULT") on an
|
|
97
|
+
* identity PK, and a non-grant then read as "enforcement exceeds declaration". The probe
|
|
98
|
+
* now SETs DEFAULT, which identity and generated columns accept.
|
|
98
99
|
*/
|
|
99
100
|
export function buildProbeSql(roles: string[], tables: string[]): string {
|
|
100
101
|
return `
|
|
@@ -119,21 +120,29 @@ BEGIN
|
|
|
119
120
|
WHEN 'UPDATE' THEN NULL
|
|
120
121
|
END;
|
|
121
122
|
IF v_cmd = 'UPDATE' THEN
|
|
122
|
-
--
|
|
123
|
-
--
|
|
124
|
-
--
|
|
125
|
-
--
|
|
123
|
+
-- The probe is "SET col = DEFAULT", which needs the UPDATE privilege on col and
|
|
124
|
+
-- nothing else. "SET col = col" READS col, so it also needs SELECT on col: a role
|
|
125
|
+
-- granted UPDATE (b) without SELECT on b probed as denied. DEFAULT is also legal on
|
|
126
|
+
-- identity (GENERATED ALWAYS too) and generated columns, so any column can be SET
|
|
127
|
+
-- and the statement always reaches the permission check.
|
|
128
|
+
--
|
|
129
|
+
-- The column is chosen PER ROLE. The contract grants UPDATE on the table privilege
|
|
130
|
+
-- OR any column privilege, so the probe must SET a column this role may update —
|
|
131
|
+
-- a per-table pick (the first column) reports a column-scoped grant as denied.
|
|
132
|
+
-- has_column_privilege is true for a table-wide grant too, and resolves
|
|
133
|
+
-- inheritance and PUBLIC. A role with no UPDATE on any column falls through to
|
|
134
|
+
-- the first column and is denied.
|
|
126
135
|
SELECT quote_ident(attname) INTO v_col FROM pg_attribute
|
|
127
136
|
WHERE attrelid = format('%I.%I', split_part(v_tbl,'.',1), split_part(v_tbl,'.',2))::regclass
|
|
128
137
|
AND attnum > 0 AND NOT attisdropped
|
|
129
|
-
AND
|
|
138
|
+
AND has_column_privilege(v_role, attrelid, attnum, 'UPDATE')
|
|
130
139
|
ORDER BY attnum LIMIT 1;
|
|
131
140
|
IF v_col IS NULL THEN
|
|
132
141
|
SELECT quote_ident(attname) INTO v_col FROM pg_attribute
|
|
133
142
|
WHERE attrelid = format('%I.%I', split_part(v_tbl,'.',1), split_part(v_tbl,'.',2))::regclass
|
|
134
143
|
AND attnum > 0 AND NOT attisdropped ORDER BY attnum LIMIT 1;
|
|
135
144
|
END IF;
|
|
136
|
-
v_stmt := format('UPDATE %I.%I SET %s =
|
|
145
|
+
v_stmt := format('UPDATE %I.%I SET %s = DEFAULT WHERE false', split_part(v_tbl,'.',1), split_part(v_tbl,'.',2), v_col);
|
|
137
146
|
END IF;
|
|
138
147
|
|
|
139
148
|
BEGIN
|
|
@@ -150,7 +159,7 @@ BEGIN
|
|
|
150
159
|
-- After (data/constraint): the privilege was present.
|
|
151
160
|
v_outcome := CASE
|
|
152
161
|
WHEN SQLSTATE IN (
|
|
153
|
-
'428C9', -- generated_always: identity/generated column (
|
|
162
|
+
'428C9', -- generated_always: identity/generated column (no longer hit by SET col = DEFAULT)
|
|
154
163
|
'42703', -- undefined_column
|
|
155
164
|
'42P01', -- undefined_table
|
|
156
165
|
'42601', -- syntax_error
|
package/src/cli/backfill.ts
CHANGED
|
@@ -27,6 +27,7 @@ import { createHash } from 'node:crypto';
|
|
|
27
27
|
import { lex, normalizeSql, splitSqlStatements, type SourceFile } from './derived-source';
|
|
28
28
|
import { escapeLiteral } from './derived-apply';
|
|
29
29
|
import type { QueryRunner } from './authz-contract';
|
|
30
|
+
import { ConnectionLostError } from './connection-fence';
|
|
30
31
|
|
|
31
32
|
// ---------------------------------------------------------------------------
|
|
32
33
|
// Identity + advisories (pure).
|
|
@@ -204,10 +205,15 @@ export async function executeBackfills(
|
|
|
204
205
|
}));
|
|
205
206
|
result.ran.push({ file: job.file, identity: job.identity, durationMs });
|
|
206
207
|
} catch (err: any) {
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
208
|
+
try {
|
|
209
|
+
await runner(renderBackfillLogInsert({
|
|
210
|
+
identity: job.identity, name, sql: `-- FAILED: ${err.message}\n${job.sql}`, outcome: 'failed',
|
|
211
|
+
actor: options.actor, gitRef: options.gitRef, durationMs: now() - started,
|
|
212
|
+
}));
|
|
213
|
+
} catch { /* the memoir is best-effort on failure; it must never replace the job's own error */ }
|
|
214
|
+
// A lost connection is not a job that failed and rolled back: a job lost IN FLIGHT may
|
|
215
|
+
// have committed with no log row. Surface the loss itself, which says so.
|
|
216
|
+
if (err instanceof ConnectionLostError) throw err;
|
|
211
217
|
result.failed = { file: job.file, identity: job.identity, error: err.message };
|
|
212
218
|
return result;
|
|
213
219
|
}
|
package/src/cli/backup.ts
CHANGED
|
@@ -81,6 +81,27 @@ export interface BackupManifest {
|
|
|
81
81
|
fingerprintSource?: string;
|
|
82
82
|
bytes?: number;
|
|
83
83
|
key: string;
|
|
84
|
+
/**
|
|
85
|
+
* The commit that last deployed the dumped database — schema_log's anchor (the newest successful
|
|
86
|
+
* apply that recorded a commit), read by the operator in the same session as `fingerprint`.
|
|
87
|
+
* `null` = a clean read proved there is none; absent = unknown (pre-R1, or the read failed).
|
|
88
|
+
*/
|
|
89
|
+
originCommit?: string | null;
|
|
90
|
+
/** What the Models declared for authz when the backup was taken — see BackupAuthzManifest. */
|
|
91
|
+
authzManifest?: BackupAuthzManifest;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* The declared-authz manifest a backup records: rendered by the OPERATOR's CLI from the models
|
|
96
|
+
* checked out at dump time (@everystack/cli authz-manifest.ts, which narrows each section to its
|
|
97
|
+
* real type), so a later restore can converge authz without compiling a historical models tree.
|
|
98
|
+
* The sections are opaque on this side of the mirror: they ride into the `.meta.json` unchanged.
|
|
99
|
+
*/
|
|
100
|
+
export interface BackupAuthzManifest {
|
|
101
|
+
version: number;
|
|
102
|
+
tables: unknown[];
|
|
103
|
+
derived: unknown;
|
|
104
|
+
owners: unknown;
|
|
84
105
|
}
|
|
85
106
|
|
|
86
107
|
export interface SnapshotVerifyInput {
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* catalog-rename — the catalog-only renames the planner emits, their parser, and the verdict
|
|
3
|
+
* db:apply reads from them.
|
|
4
|
+
*
|
|
5
|
+
* A rename of a check, a foreign key or an index changes no fingerprint: a check is hashed by its
|
|
6
|
+
* predicate and an index or key by its content. So a plan whose ONLY change is such a rename has
|
|
7
|
+
* from == to == live, and the endpoint cannot tell a rename that ran from one that did not. The
|
|
8
|
+
* live names can. The renderer and the parser live side by side here so they cannot drift; the
|
|
9
|
+
* parser accepts exactly what the renderer writes and nothing else.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
import type { SchemaSnapshot } from './schema-introspect';
|
|
13
|
+
import { isPureComment } from './sql-statement';
|
|
14
|
+
|
|
15
|
+
/** The same quoting schema-diff uses for every identifier it emits. */
|
|
16
|
+
const quote = (name: string): string => `"${name}"`;
|
|
17
|
+
|
|
18
|
+
/** `public.orders` → `public.`; an unqualified table carries no qualifier. */
|
|
19
|
+
const schemaPrefix = (table: string): string => (table.includes('.') ? `${table.split('.')[0]}.` : '');
|
|
20
|
+
|
|
21
|
+
export interface RenameChange {
|
|
22
|
+
/** Schema-qualified, as the snapshot spells it (`public.orders`). */
|
|
23
|
+
table: string;
|
|
24
|
+
from: string;
|
|
25
|
+
to: string;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** `ALTER TABLE … RENAME CONSTRAINT` — catalog-only and instant; the constraint keeps its identity. */
|
|
29
|
+
export function renderConstraintRename(change: RenameChange): string {
|
|
30
|
+
return `ALTER TABLE ${change.table} RENAME CONSTRAINT ${quote(change.from)} TO ${quote(change.to)};`;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** `ALTER INDEX … RENAME TO` — no rebuild. The schema qualifies the OLD name only. */
|
|
34
|
+
export function renderIndexRename(change: RenameChange): string {
|
|
35
|
+
return `ALTER INDEX ${schemaPrefix(change.table)}${quote(change.from)} RENAME TO ${quote(change.to)};`;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export type CatalogRename =
|
|
39
|
+
| { kind: 'constraint'; table: string; from: string; to: string }
|
|
40
|
+
/** ALTER INDEX names no table — an index name is unique within its schema, so the schema is the scope. */
|
|
41
|
+
| { kind: 'index'; schema: string; from: string; to: string };
|
|
42
|
+
|
|
43
|
+
const IDENT = '"([^"]*)"';
|
|
44
|
+
const CONSTRAINT_RE = new RegExp(`^ALTER TABLE (\\S+) RENAME CONSTRAINT ${IDENT} TO ${IDENT};$`);
|
|
45
|
+
const INDEX_RE = new RegExp(`^ALTER INDEX (?:([^\\s."]+)\\.)?${IDENT} RENAME TO ${IDENT};$`);
|
|
46
|
+
|
|
47
|
+
/** The rename a statement is, if it is exactly one the renderers above produce; otherwise null. */
|
|
48
|
+
export function parseCatalogRename(sql: string): CatalogRename | null {
|
|
49
|
+
const c = CONSTRAINT_RE.exec(sql);
|
|
50
|
+
if (c) return { kind: 'constraint', table: c[1], from: c[2], to: c[3] };
|
|
51
|
+
const i = INDEX_RE.exec(sql);
|
|
52
|
+
// An unqualified ALTER INDEX resolves through search_path; the renderer only writes one for an
|
|
53
|
+
// unqualified table, which the snapshot spells as `public`.
|
|
54
|
+
if (i) return { kind: 'index', schema: i[1] ?? 'public', from: i[2], to: i[3] };
|
|
55
|
+
return null;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export type RenameEdgeVerdict =
|
|
59
|
+
/** Empty, or some statement is not a recognized rename — the names have nothing to say. */
|
|
60
|
+
| { verdict: 'not-renames' }
|
|
61
|
+
/** Every rename already landed: `to` present, `from` absent. */
|
|
62
|
+
| { verdict: 'settled' }
|
|
63
|
+
/** No rename has landed: `from` present, `to` absent. The edge still has to run. */
|
|
64
|
+
| { verdict: 'unsettled' }
|
|
65
|
+
/** Anything else — the plan describes neither the database before it nor after it. */
|
|
66
|
+
| { verdict: 'stale'; reason: string };
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Where each rename in the edge stands in the BEFORE snapshot (the one the lock was computed
|
|
70
|
+
* from). Only an edge made entirely of recognized renames gets a verdict other than
|
|
71
|
+
* `not-renames`; one rename in an odd state, or a mix of landed and not, makes the plan stale.
|
|
72
|
+
*/
|
|
73
|
+
export function renameEdgeVerdict(statements: readonly string[], before: SchemaSnapshot): RenameEdgeVerdict {
|
|
74
|
+
// A comment is never run, and the planner writes them beside real changes (an inert
|
|
75
|
+
// renamedFrom notice, a held drop). They must not hide a rename edge from the names.
|
|
76
|
+
const real = statements.filter((sql) => !isPureComment(sql));
|
|
77
|
+
if (real.length === 0) return { verdict: 'not-renames' };
|
|
78
|
+
const renames: CatalogRename[] = [];
|
|
79
|
+
for (const sql of real) {
|
|
80
|
+
const rename = parseCatalogRename(sql);
|
|
81
|
+
if (!rename) return { verdict: 'not-renames' };
|
|
82
|
+
renames.push(rename);
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
const settled: string[] = [];
|
|
86
|
+
const unsettled: string[] = [];
|
|
87
|
+
for (const rename of renames) {
|
|
88
|
+
let present: Set<string>;
|
|
89
|
+
let where: string;
|
|
90
|
+
if (rename.kind === 'constraint') {
|
|
91
|
+
const table = before.tables.find((t) => t.table === rename.table);
|
|
92
|
+
if (!table) return { verdict: 'stale', reason: `table ${rename.table} does not exist` };
|
|
93
|
+
present = new Set([...table.checks, ...table.foreignKeys, ...table.uniques].map((c) => c.name));
|
|
94
|
+
where = `on ${rename.table}`;
|
|
95
|
+
} else {
|
|
96
|
+
present = new Set(before.tables
|
|
97
|
+
.filter((t) => t.table.startsWith(`${rename.schema}.`))
|
|
98
|
+
.flatMap((t) => t.indexes.map((ix) => ix.name)));
|
|
99
|
+
where = `in schema ${rename.schema}`;
|
|
100
|
+
}
|
|
101
|
+
const hasFrom = present.has(rename.from);
|
|
102
|
+
const hasTo = present.has(rename.to);
|
|
103
|
+
const label = `"${rename.from}" → "${rename.to}"`;
|
|
104
|
+
if (hasTo && !hasFrom) settled.push(label);
|
|
105
|
+
else if (hasFrom && !hasTo) unsettled.push(label);
|
|
106
|
+
else if (hasFrom && hasTo) return { verdict: 'stale', reason: `both "${rename.from}" and "${rename.to}" exist ${where}` };
|
|
107
|
+
else return { verdict: 'stale', reason: `neither "${rename.from}" nor "${rename.to}" exists ${where}` };
|
|
108
|
+
}
|
|
109
|
+
if (unsettled.length === 0) return { verdict: 'settled' };
|
|
110
|
+
if (settled.length === 0) return { verdict: 'unsettled' };
|
|
111
|
+
return {
|
|
112
|
+
verdict: 'stale',
|
|
113
|
+
reason: `some of its renames are already applied (${settled.join(', ')}) and some are not (${unsettled.join(', ')})`,
|
|
114
|
+
};
|
|
115
|
+
}
|
|
@@ -5,14 +5,15 @@
|
|
|
5
5
|
* dispatches and polls. db:backups/db:restore/download are thin invokeAction wrappers. The image's
|
|
6
6
|
* version handshake is the compatibility gate — verify a deploy with `everystack task:probe`.
|
|
7
7
|
*
|
|
8
|
-
* db:backup has a SECOND venue
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
8
|
+
* db:backup has a SECOND venue: `--database-url <local> --stage <target>` dumps a LOCAL database and
|
|
9
|
+
* uploads it into the target stage's backups bucket as a db:backup-shaped object (same key scheme +
|
|
10
|
+
* .meta.json + dump flags as a server backup). The local dump is the only held connection; the
|
|
11
|
+
* upload rides the operator's AWS credentials.
|
|
12
|
+
*
|
|
13
|
+
* Restoring such a backup into a stage is CLOSED. A restore must record the commit the restored
|
|
14
|
+
* state descends from (the witness), and a local database has no verified origin: nothing proves
|
|
15
|
+
* it matches any commit. So the local meta carries no originCommit and db:restore refuses it. It
|
|
16
|
+
* reopens only when a local backup can record an origin it has verified against the models.
|
|
16
17
|
*/
|
|
17
18
|
|
|
18
19
|
import { spawn } from 'node:child_process';
|
|
@@ -43,7 +44,7 @@ export function resolveBackupVenue(flags: Record<string, string>): BackupVenue {
|
|
|
43
44
|
if (!flags.stage) {
|
|
44
45
|
throw new Error(
|
|
45
46
|
'db:backup --database-url <local> also needs --stage <target> — the local dump uploads into '
|
|
46
|
-
+ "that stage's backups bucket, and the stage names the backup id
|
|
47
|
+
+ "that stage's backups bucket, and the stage names the backup id.",
|
|
47
48
|
);
|
|
48
49
|
}
|
|
49
50
|
return { kind: 'local', url, stage: flags.stage };
|
|
@@ -177,7 +178,7 @@ export async function dbBackupCommand(flags: Record<string, string>): Promise<vo
|
|
|
177
178
|
}
|
|
178
179
|
|
|
179
180
|
// Local venue: dump a reachable database here and upload it into the target stage's backups bucket
|
|
180
|
-
// as a db:backup-shaped object, so
|
|
181
|
+
// as a db:backup-shaped object. It carries no origin, so db:restore refuses it (see the header).
|
|
181
182
|
if (venue.kind === 'local') {
|
|
182
183
|
step('Resolving deployed config...');
|
|
183
184
|
let config: CliConfig;
|
|
@@ -192,22 +193,24 @@ export async function dbBackupCommand(flags: Record<string, string>): Promise<vo
|
|
|
192
193
|
process.exit(1);
|
|
193
194
|
}
|
|
194
195
|
info(`Region: ${config.region}, Bucket: ${config.backupsBucket}`);
|
|
195
|
-
//
|
|
196
|
-
//
|
|
197
|
-
// so
|
|
198
|
-
// lossless (it fails before --clean drops anything), but discovered exactly when you need the
|
|
199
|
-
// backup. Surface the local major up front so the operator can match it to the target's PG major.
|
|
196
|
+
// Unlike the stage venue (whose pg binaries are the Task image), this dumps with the OPERATOR's
|
|
197
|
+
// pg_dump. pg_restore refuses an archive from a NEWER pg_dump major ("unsupported version in file
|
|
198
|
+
// header"), so surface the local major for whoever pg_restores the downloaded dump by hand.
|
|
200
199
|
try {
|
|
201
200
|
const { execFileSync } = await import('node:child_process');
|
|
202
201
|
const v = execFileSync('pg_dump', ['--version']).toString().trim();
|
|
203
|
-
info(`Local ${v} —
|
|
202
|
+
info(`Local ${v} — a manual pg_restore of this dump needs pg_restore of the same major or newer.`);
|
|
204
203
|
} catch { /* a missing pg_dump surfaces loudly in the dump step below */ }
|
|
205
204
|
step(`Running pg_dump (full, --exclude-schema=everystack) against --database-url → S3 (this may take a while for large databases)...`);
|
|
206
205
|
try {
|
|
207
206
|
const res = await runLocalBackupUpload({
|
|
208
207
|
region: config.region, bucket: config.backupsBucket, url: venue.url, stage: venue.stage,
|
|
209
208
|
});
|
|
210
|
-
success(`Backup ${res.id} (${fmtBytes(res.bytes)})
|
|
209
|
+
success(`Backup ${res.id} (${fmtBytes(res.bytes)}).`);
|
|
210
|
+
warn(
|
|
211
|
+
'A local backup records no origin commit, so db:restore refuses it: restoring a local backup into a stage is closed ' +
|
|
212
|
+
`until local backups can record a verified origin. It stays downloadable: everystack db:backup:download ${res.id} --stage ${venue.stage}`,
|
|
213
|
+
);
|
|
211
214
|
} catch (err: any) {
|
|
212
215
|
fail(`Backup failed: ${err.message}`);
|
|
213
216
|
process.exit(1);
|
|
@@ -227,7 +230,16 @@ export async function dbBackupCommand(flags: Record<string, string>): Promise<vo
|
|
|
227
230
|
info(`Region: ${config.region}, Function: ${opsFunction(config)}`);
|
|
228
231
|
const fn = opsFunction(config);
|
|
229
232
|
|
|
233
|
+
// The declared authz at dump time, rendered from THIS checkout's models (never a historical
|
|
234
|
+
// tree) and stamped onto the backup once the dump is proven — so a later restore can converge
|
|
235
|
+
// authz to it. Best-effort: a failure here is a warning, never a lost backup.
|
|
236
|
+
const { loadDeclaredDerived } = await import('../declared-derived');
|
|
237
|
+
const { loadBackupAuthzManifest } = await import('../authz-manifest');
|
|
238
|
+
const authz = await loadBackupAuthzManifest(() => loadDeclaredDerived(flags.models));
|
|
239
|
+
if (authz.warning) warn(authz.warning);
|
|
240
|
+
|
|
230
241
|
const result = await runStageBackup(stageBackupDeps(config, fn, flags.stage, {
|
|
242
|
+
...(authz.manifest ? { authzManifest: authz.manifest } : {}),
|
|
231
243
|
onDispatch: (d) => {
|
|
232
244
|
info(`backup ${d.id} — task ${d.taskArn}`);
|
|
233
245
|
info(`run id: ${d.runId} (everystack.task_log)`);
|
|
@@ -284,6 +296,32 @@ export async function dbBackupsCommand(flags: Record<string, string>): Promise<v
|
|
|
284
296
|
success(`${backups.length} backup${backups.length === 1 ? '' : 's'}.`);
|
|
285
297
|
}
|
|
286
298
|
|
|
299
|
+
/**
|
|
300
|
+
* What the dispatcher recorded in the target before the task started: the schema_log witness row
|
|
301
|
+
* and the commit it names (the backup's own origin), plus whether the declared roles were checked.
|
|
302
|
+
*
|
|
303
|
+
* A successful dispatch with no witnessId came from an ops Lambda that predates restore witnesses:
|
|
304
|
+
* it ignored the origin checks and recorded nothing. The restore is already running, so this warns
|
|
305
|
+
* rather than refuses.
|
|
306
|
+
*/
|
|
307
|
+
export function printRestoreWitness(
|
|
308
|
+
dispatched: { witnessId?: string; origin?: string; rolesUnchecked?: boolean },
|
|
309
|
+
stage: string,
|
|
310
|
+
): void {
|
|
311
|
+
if (dispatched.witnessId) {
|
|
312
|
+
info(`witness ${dispatched.witnessId} in everystack.schema_log — origin ${dispatched.origin ?? '(none)'} (the commit the backup recorded)`);
|
|
313
|
+
} else {
|
|
314
|
+
warn(
|
|
315
|
+
`WARNING: the ops Lambda recorded no restore witness. It predates restore witnesses, so it did not check the backup's ` +
|
|
316
|
+
`origin or roles, and nothing in ${stage}'s everystack.schema_log marks this restore. The restore is already running. ` +
|
|
317
|
+
`Fix: redeploy the ops Lambda (sst deploy --stage ${stage}) so restores record the witness.`,
|
|
318
|
+
);
|
|
319
|
+
}
|
|
320
|
+
if (dispatched.rolesUnchecked) {
|
|
321
|
+
warn('declared roles were not checked: this backup carries no authz manifest.');
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
|
|
287
325
|
/**
|
|
288
326
|
* db:restore — pg_restore a backup INTO a stage's existing DB. Destructive (drops + replaces), so
|
|
289
327
|
* it requires --confirm; restoring production-tier data downward or into a production target raises
|
|
@@ -340,6 +378,7 @@ export async function dbRestoreCommand(flags: Record<string, string>): Promise<v
|
|
|
340
378
|
const { runId, taskArn, intentKey, warning } = dispatched as { runId: string; taskArn: string; intentKey: string; warning?: string };
|
|
341
379
|
info(`restore of ${id} → ${toStage} — task ${taskArn}`);
|
|
342
380
|
info(`intent: ${intentKey} (durable S3 record — survives the restore)`);
|
|
381
|
+
printRestoreWitness(dispatched, toStage);
|
|
343
382
|
if (warning) info(`note: ${warning}`);
|
|
344
383
|
|
|
345
384
|
step(`Restoring ${id} into ${toStage} (pg_restore in the Task; this may take a while)...`);
|