@everystack/cli 0.4.32 → 0.4.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -1
- package/src/cli/aws.ts +1 -1
- package/src/cli/commands/db-backup.ts +187 -2
- package/src/cli/commands/db-reconcile.ts +8 -5
- package/src/cli/derived-apply.ts +14 -4
- package/src/cli/derived-compile.ts +13 -2
- package/src/cli/derived-grants.ts +15 -0
- package/src/cli/derived-introspect.ts +4 -0
- package/src/cli/derived-plan.ts +27 -3
- package/src/cli/derived-source.ts +8 -0
- package/src/cli/index.ts +1 -1
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@everystack/cli",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.34",
|
|
4
4
|
"description": "CLI and OTA updates for Expo apps on everystack",
|
|
5
5
|
"license": "AGPL-3.0-only",
|
|
6
6
|
"author": "Scalable Technology, Inc. <licensing@scalable.technology>",
|
|
@@ -100,6 +100,7 @@
|
|
|
100
100
|
"@aws-sdk/client-lambda": "3.1053.0",
|
|
101
101
|
"@aws-sdk/client-s3": "3.1053.0",
|
|
102
102
|
"@aws-sdk/client-ssm": "3.1053.0",
|
|
103
|
+
"@aws-sdk/lib-storage": "3.1053.0",
|
|
103
104
|
"@aws-sdk/client-sts": "3.1053.0",
|
|
104
105
|
"@aws-sdk/signature-v4a": "3.1048.0",
|
|
105
106
|
"glob": "13.0.6",
|
package/src/cli/aws.ts
CHANGED
|
@@ -13,7 +13,7 @@ let lambdaClient: InstanceType<typeof import('@aws-sdk/client-lambda').LambdaCli
|
|
|
13
13
|
let kvsClient: InstanceType<typeof import('@aws-sdk/client-cloudfront-keyvaluestore').CloudFrontKeyValueStoreClient> | null = null;
|
|
14
14
|
let rdsClient: InstanceType<typeof import('@aws-sdk/client-rds').RDSClient> | null = null;
|
|
15
15
|
|
|
16
|
-
async function getS3(region: string) {
|
|
16
|
+
export async function getS3(region: string) {
|
|
17
17
|
if (!s3Client) {
|
|
18
18
|
const { S3Client } = await importOptionalAws(
|
|
19
19
|
() => import('@aws-sdk/client-s3'), '@aws-sdk/client-s3', 'this command',
|
|
@@ -4,14 +4,152 @@
|
|
|
4
4
|
* db:backup/db:export run in the ephemeral Task lane (the ops Lambda holds no pg binaries); the CLI
|
|
5
5
|
* dispatches and polls. db:backups/db:restore/download are thin invokeAction wrappers. The image's
|
|
6
6
|
* version handshake is the compatibility gate — verify a deploy with `everystack task:probe`.
|
|
7
|
+
*
|
|
8
|
+
* db:backup has a SECOND venue for break-glass: `--database-url <local> --stage <target>` dumps a
|
|
9
|
+
* LOCAL database and uploads it into the target stage's backups bucket as a db:backup-shaped object
|
|
10
|
+
* (same key scheme + .meta.json + dump flags as a server backup). The standard credential-free
|
|
11
|
+
* `db:restore --from <id> --stage <target>` then lands it via the Task lane (pg_restore runs as the
|
|
12
|
+
* privileged `migrator` role, which can CREATE SCHEMA + own objects). The local dump is the only
|
|
13
|
+
* held connection; the upload rides the operator's AWS credentials — the same trust boundary as
|
|
14
|
+
* db:fork's cross-stage presign. This is the sanctioned local→stage full replace on a role-managed
|
|
15
|
+
* stage (raw pg_restore can't: the operator/api role can't CREATE SCHEMA, and only the master could).
|
|
7
16
|
*/
|
|
8
17
|
|
|
18
|
+
import { spawn } from 'node:child_process';
|
|
19
|
+
import { createGzip } from 'node:zlib';
|
|
20
|
+
import { pipeline } from 'node:stream/promises';
|
|
21
|
+
import { randomBytes } from 'node:crypto';
|
|
9
22
|
import { resolveConfig, opsFunction, type CliConfig } from '../config.js';
|
|
10
|
-
import { invokeAction, presignGet } from '../aws.js';
|
|
11
|
-
import {
|
|
23
|
+
import { invokeAction, presignGet, getS3 } from '../aws.js';
|
|
24
|
+
import {
|
|
25
|
+
parseBackupRef, keyForId, crossStageGuard, restoreTargetGuard,
|
|
26
|
+
backupKey, backupId, metaKey, utcStamp,
|
|
27
|
+
} from '../backup.js';
|
|
28
|
+
import { pgEnvFromUrl } from './db.js';
|
|
12
29
|
import { pollTaskUntilStopped } from '../task-poll.js';
|
|
13
30
|
import { step, success, fail, info, warn } from '../output.js';
|
|
14
31
|
|
|
32
|
+
/** Which database db:backup dumps. LOCAL requires a target --stage (its bucket receives the upload
|
|
33
|
+
* AND names the backup id). Note this DIVERGES from db:export, where --database-url and --stage are
|
|
34
|
+
* mutually-exclusive venues: here the local venue COMBINES them (dump local → upload to the stage). */
|
|
35
|
+
export type BackupVenue =
|
|
36
|
+
| { kind: 'local'; url: string; stage: string }
|
|
37
|
+
| { kind: 'stage'; stage: string | undefined };
|
|
38
|
+
|
|
39
|
+
export function resolveBackupVenue(flags: Record<string, string>): BackupVenue {
|
|
40
|
+
const url = flags['database-url'];
|
|
41
|
+
if (url) {
|
|
42
|
+
if (!flags.stage) {
|
|
43
|
+
throw new Error(
|
|
44
|
+
'db:backup --database-url <local> also needs --stage <target> — the local dump uploads into '
|
|
45
|
+
+ "that stage's backups bucket, and the stage names the backup id `db:restore --from` consumes.",
|
|
46
|
+
);
|
|
47
|
+
}
|
|
48
|
+
return { kind: 'local', url, stage: flags.stage };
|
|
49
|
+
}
|
|
50
|
+
return { kind: 'stage', stage: flags.stage };
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Full-DB streaming pg_dump args — IDENTICAL to the server's db:backup dump (packages/server
|
|
55
|
+
* backup.ts pgDumpArgs()), minus `-f`: streamed to stdout so it pipes into gzip → S3. Excludes the
|
|
56
|
+
* everystack ops schema (task_log/schema_log/etc — the TARGET's own audit history, not app data;
|
|
57
|
+
* restoring it would rewind that history AND makes pg_restore --clean fail on DROP SCHEMA everystack).
|
|
58
|
+
*/
|
|
59
|
+
export function pgDumpFullArgs(): string[] {
|
|
60
|
+
return ['-Fc', '--no-owner', '--no-privileges', '--no-comments', '--exclude-schema=everystack'];
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/** The .meta.json sidecar — byte-shape identical to the server's runBackup meta (backup-run.ts:156),
|
|
64
|
+
* so db:backups list and the destructive-apply gate read a local upload exactly like a stage backup.
|
|
65
|
+
* `bytes` is undefined-safe (JSON.stringify drops it) exactly as the server leaves it on a failed
|
|
66
|
+
* HeadObject — never coerced to 0. */
|
|
67
|
+
export function localBackupMeta(opts: {
|
|
68
|
+
id: string;
|
|
69
|
+
stage: string;
|
|
70
|
+
createdAt: Date;
|
|
71
|
+
bytes?: number;
|
|
72
|
+
key: string;
|
|
73
|
+
}): Record<string, unknown> {
|
|
74
|
+
return {
|
|
75
|
+
id: opts.id,
|
|
76
|
+
stage: opts.stage,
|
|
77
|
+
createdAt: opts.createdAt.toISOString(),
|
|
78
|
+
bytes: opts.bytes,
|
|
79
|
+
key: opts.key,
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Stream a local database into a stage's backups bucket as a db:backup-shaped object. Mirrors the
|
|
85
|
+
* server's runBackup streaming AND its load-bearing correctness rule: a completed S3 multipart upload
|
|
86
|
+
* does NOT mean pg_dump succeeded (pg_dump can exit non-zero mid-stream while the upload "completes"),
|
|
87
|
+
* so we await the child exit code SEPARATELY and, on any failure, abort the upload and delete the
|
|
88
|
+
* partial object. A backup tool that ships a truncated dump is worse than none.
|
|
89
|
+
*/
|
|
90
|
+
export async function runLocalBackupUpload(deps: {
|
|
91
|
+
region: string;
|
|
92
|
+
bucket: string;
|
|
93
|
+
url: string;
|
|
94
|
+
stage: string;
|
|
95
|
+
}): Promise<{ id: string; key: string; bytes?: number }> {
|
|
96
|
+
const now = new Date();
|
|
97
|
+
const stamp = utcStamp(now);
|
|
98
|
+
const shortId = randomBytes(3).toString('hex');
|
|
99
|
+
const key = backupKey(deps.stage, stamp, shortId);
|
|
100
|
+
const id = backupId(deps.stage, stamp, shortId);
|
|
101
|
+
const mkey = metaKey(key);
|
|
102
|
+
|
|
103
|
+
const env = { ...process.env, ...pgEnvFromUrl(deps.url) };
|
|
104
|
+
const s3 = await getS3(deps.region);
|
|
105
|
+
const { Upload } = await import('@aws-sdk/lib-storage');
|
|
106
|
+
const { DeleteObjectCommand, PutObjectCommand, HeadObjectCommand } = await import('@aws-sdk/client-s3');
|
|
107
|
+
|
|
108
|
+
const child = spawn('pg_dump', pgDumpFullArgs(), { env, stdio: ['ignore', 'pipe', 'pipe'] });
|
|
109
|
+
let stderr = '';
|
|
110
|
+
child.stderr.on('data', (d) => { stderr += d.toString(); });
|
|
111
|
+
const childExit = new Promise<void>((resolve, reject) => {
|
|
112
|
+
child.on('error', reject);
|
|
113
|
+
child.on('close', (code) =>
|
|
114
|
+
code === 0 ? resolve() : reject(new Error(`pg_dump exited ${code}: ${stderr.trim()}`)));
|
|
115
|
+
});
|
|
116
|
+
|
|
117
|
+
const gzip = createGzip();
|
|
118
|
+
const upload = new Upload({
|
|
119
|
+
client: s3,
|
|
120
|
+
params: {
|
|
121
|
+
Bucket: deps.bucket, Key: key, Body: gzip,
|
|
122
|
+
ContentType: 'application/gzip', ServerSideEncryption: 'AES256',
|
|
123
|
+
},
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
try {
|
|
127
|
+
await Promise.all([
|
|
128
|
+
pipeline(child.stdout!, gzip), // pg_dump → gzip; errors propagate, streams destroyed
|
|
129
|
+
upload.done(), // gzip → S3 multipart
|
|
130
|
+
childExit, // the guard: non-zero pg_dump rejects the whole thing
|
|
131
|
+
]);
|
|
132
|
+
} catch (err) {
|
|
133
|
+
try { await upload.abort(); } catch { /* may already have completed */ }
|
|
134
|
+
await s3.send(new DeleteObjectCommand({ Bucket: deps.bucket, Key: key })).catch(() => {});
|
|
135
|
+
throw err;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
let bytes: number | undefined;
|
|
139
|
+
try {
|
|
140
|
+
const head = await s3.send(new HeadObjectCommand({ Bucket: deps.bucket, Key: key }));
|
|
141
|
+
bytes = head.ContentLength;
|
|
142
|
+
} catch { /* size is best-effort */ }
|
|
143
|
+
|
|
144
|
+
await s3.send(new PutObjectCommand({
|
|
145
|
+
Bucket: deps.bucket, Key: mkey,
|
|
146
|
+
Body: JSON.stringify(localBackupMeta({ id, stage: deps.stage, createdAt: now, bytes, key })),
|
|
147
|
+
ContentType: 'application/json', ServerSideEncryption: 'AES256',
|
|
148
|
+
}));
|
|
149
|
+
|
|
150
|
+
return { id, key, bytes };
|
|
151
|
+
}
|
|
152
|
+
|
|
15
153
|
function fmtBytes(n?: number): string {
|
|
16
154
|
if (n == null) return '-';
|
|
17
155
|
if (n < 1024) return `${n} B`;
|
|
@@ -27,6 +165,53 @@ function fmtBytes(n?: number): string {
|
|
|
27
165
|
* the compatibility gate (a too-old image fails the run loudly — no separate layer pre-flight).
|
|
28
166
|
*/
|
|
29
167
|
export async function dbBackupCommand(flags: Record<string, string>): Promise<void> {
|
|
168
|
+
let venue: BackupVenue;
|
|
169
|
+
try {
|
|
170
|
+
venue = resolveBackupVenue(flags);
|
|
171
|
+
} catch (err: any) {
|
|
172
|
+
fail(err.message);
|
|
173
|
+
process.exit(1);
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
// Local venue: dump a reachable database here and upload it into the target stage's backups bucket
|
|
177
|
+
// as a db:backup-shaped object, so `db:restore --from <id> --stage <target>` lands it credential-free.
|
|
178
|
+
if (venue.kind === 'local') {
|
|
179
|
+
step('Resolving deployed config...');
|
|
180
|
+
let config: CliConfig;
|
|
181
|
+
try {
|
|
182
|
+
config = await resolveConfig(venue.stage);
|
|
183
|
+
} catch (err: any) {
|
|
184
|
+
fail(err.message);
|
|
185
|
+
process.exit(1);
|
|
186
|
+
}
|
|
187
|
+
if (!config.backupsBucket) {
|
|
188
|
+
fail('No backupsBucket in the deployed config. Add `backupsBucket: backups.name` to sst.config outputs and redeploy.');
|
|
189
|
+
process.exit(1);
|
|
190
|
+
}
|
|
191
|
+
info(`Region: ${config.region}, Bucket: ${config.backupsBucket}`);
|
|
192
|
+
// Version gate for the local venue: unlike the stage venue (whose pg binaries are the Task image),
|
|
193
|
+
// this dumps with the OPERATOR's pg_dump. pg_restore refuses an archive from a NEWER pg_dump major,
|
|
194
|
+
// so a mismatch surfaces only at restore time ("unsupported version in file header") — loud and
|
|
195
|
+
// lossless (it fails before --clean drops anything), but discovered exactly when you need the
|
|
196
|
+
// backup. Surface the local major up front so the operator can match it to the target's PG major.
|
|
197
|
+
try {
|
|
198
|
+
const { execFileSync } = await import('node:child_process');
|
|
199
|
+
const v = execFileSync('pg_dump', ['--version']).toString().trim();
|
|
200
|
+
info(`Local ${v} — must be ≤ the target stage's PostgreSQL major, or db:restore will reject the dump.`);
|
|
201
|
+
} catch { /* a missing pg_dump surfaces loudly in the dump step below */ }
|
|
202
|
+
step(`Running pg_dump (full, --exclude-schema=everystack) against --database-url → S3 (this may take a while for large databases)...`);
|
|
203
|
+
try {
|
|
204
|
+
const res = await runLocalBackupUpload({
|
|
205
|
+
region: config.region, bucket: config.backupsBucket, url: venue.url, stage: venue.stage,
|
|
206
|
+
});
|
|
207
|
+
success(`Backup ${res.id} (${fmtBytes(res.bytes)}). Restore with: everystack db:restore --from ${res.id} --stage ${venue.stage} --confirm`);
|
|
208
|
+
} catch (err: any) {
|
|
209
|
+
fail(`Backup failed: ${err.message}`);
|
|
210
|
+
process.exit(1);
|
|
211
|
+
}
|
|
212
|
+
return;
|
|
213
|
+
}
|
|
214
|
+
|
|
30
215
|
step('Resolving deployed config...');
|
|
31
216
|
let config: CliConfig;
|
|
32
217
|
try {
|
|
@@ -206,7 +206,7 @@ export async function executeReconcile(
|
|
|
206
206
|
// 'sql'-kind objects are invisible to the catalog: provenance records the source
|
|
207
207
|
// hash on both sides plus HOW TO REMOVE it (the escape hatch's whole contract).
|
|
208
208
|
if (src.kind === 'sql') {
|
|
209
|
-
bookkeeping.push(renderProvenanceUpsert(identity, src.hash, src.hash, 'sql', src.drop));
|
|
209
|
+
bookkeeping.push(renderProvenanceUpsert(identity, src.hash, src.hash, 'sql', src.drop, src.bodyHash));
|
|
210
210
|
continue;
|
|
211
211
|
}
|
|
212
212
|
const liveObj = liveById.get(identity);
|
|
@@ -214,7 +214,7 @@ export async function executeReconcile(
|
|
|
214
214
|
throw new Error(`reconcile applied but ${identity} is not introspectable afterwards — provenance not recorded, investigate`);
|
|
215
215
|
}
|
|
216
216
|
const dropSql = src.kind === 'trigger' && src.table ? triggerDropSql(src.name, src.table) : undefined;
|
|
217
|
-
bookkeeping.push(renderProvenanceUpsert(identity, src.hash, liveObj.defHash, src.kind, dropSql));
|
|
217
|
+
bookkeeping.push(renderProvenanceUpsert(identity, src.hash, liveObj.defHash, src.kind, dropSql, src.bodyHash));
|
|
218
218
|
}
|
|
219
219
|
for (const m of rendered.migrate) bookkeeping.push(renderProvenanceMigrate(m.from, m.to));
|
|
220
220
|
for (const identity of rendered.remove) bookkeeping.push(renderProvenanceDelete(identity));
|
|
@@ -292,7 +292,7 @@ export function formatBytes(bytes: number): string {
|
|
|
292
292
|
}
|
|
293
293
|
|
|
294
294
|
const VERB_GLYPH: Record<string, string> = {
|
|
295
|
-
create: '+', replace: '~', drop: '-', refresh: '↻', baseline: '◦', rebaseline: '≈', prune: '·',
|
|
295
|
+
create: '+', replace: '~', drop: '-', refresh: '↻', baseline: '◦', rebaseline: '≈', prune: '·', backfill: '◌',
|
|
296
296
|
};
|
|
297
297
|
|
|
298
298
|
export function buildReconcileReport(plan: ReconcilePlan): string[] {
|
|
@@ -330,9 +330,12 @@ export function buildReconcileReport(plan: ReconcilePlan): string[] {
|
|
|
330
330
|
}
|
|
331
331
|
|
|
332
332
|
/** Anything that makes `--check` fail: pending work or unresolved findings.
|
|
333
|
-
* Baseline and rebaseline are explicit adoption the operator asked for, not findings
|
|
333
|
+
* Baseline and rebaseline are explicit adoption the operator asked for, not findings; a
|
|
334
|
+
* backfill is record-only bookkeeping (body_hash arming) — neither is a CI failure. An
|
|
335
|
+
* UNAPPLIED authz-only grant change still fails here via `regrants` below (the real signal);
|
|
336
|
+
* an empty regrant delta means live already matches declared, so green is correct. */
|
|
334
337
|
export function checkFails(plan: ReconcilePlan): boolean {
|
|
335
|
-
return plan.actions.some((a) => a.action !== 'baseline' && a.action !== 'rebaseline')
|
|
338
|
+
return plan.actions.some((a) => a.action !== 'baseline' && a.action !== 'rebaseline' && a.action !== 'backfill')
|
|
336
339
|
|| plan.drift.length > 0
|
|
337
340
|
|| plan.needsBaseline.length > 0
|
|
338
341
|
|| plan.blocked.length > 0
|
package/src/cli/derived-apply.ts
CHANGED
|
@@ -41,6 +41,10 @@ export const ENSURE_RECONCILER_SQL: string[] = [
|
|
|
41
41
|
// databases upgrade in place; their old rows (relations/functions) never need these.
|
|
42
42
|
`ALTER TABLE everystack.derived_provenance ADD COLUMN IF NOT EXISTS kind text`,
|
|
43
43
|
`ALTER TABLE everystack.derived_provenance ADD COLUMN IF NOT EXISTS drop_sql text`,
|
|
44
|
+
// body_hash = the source hash MINUS plain grants (authz-only-change discriminator). Idempotent;
|
|
45
|
+
// legacy rows have NULL until the reconciler re-records them, and a NULL body_hash falls back to
|
|
46
|
+
// the rebuild-on-src-change behavior (no regression) — the fix only ARMS once body_hash is recorded.
|
|
47
|
+
`ALTER TABLE everystack.derived_provenance ADD COLUMN IF NOT EXISTS body_hash text`,
|
|
44
48
|
`CREATE TABLE IF NOT EXISTS everystack.schema_log (
|
|
45
49
|
id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
|
|
46
50
|
applied_at timestamptz NOT NULL DEFAULT now(),
|
|
@@ -221,6 +225,12 @@ export function renderReconcileSql(plan: ReconcilePlan, source: SourceObject[]):
|
|
|
221
225
|
record.push(action.identity);
|
|
222
226
|
break;
|
|
223
227
|
}
|
|
228
|
+
// Arm the authz-only fast path on an up-to-date object: no DDL — the upsert just adds the
|
|
229
|
+
// body_hash column value the record loop always writes now. Idempotent; converges in one run.
|
|
230
|
+
case 'backfill': {
|
|
231
|
+
record.push(action.identity);
|
|
232
|
+
break;
|
|
233
|
+
}
|
|
224
234
|
case 'prune': {
|
|
225
235
|
remove.push(action.identity);
|
|
226
236
|
break;
|
|
@@ -235,10 +245,10 @@ export function renderReconcileSql(plan: ReconcilePlan, source: SourceObject[]):
|
|
|
235
245
|
return { statements, record, remove, migrate: [...plan.migrations] };
|
|
236
246
|
}
|
|
237
247
|
|
|
238
|
-
export function renderProvenanceUpsert(identity: string, srcHash: string, defHash: string, kind?: string, dropSql?: string): string {
|
|
239
|
-
return `INSERT INTO everystack.derived_provenance (identity, src_hash, def_hash, kind, drop_sql, applied_at)
|
|
240
|
-
VALUES (${escapeLiteral(identity)}, ${escapeLiteral(srcHash)}, ${escapeLiteral(defHash)}, ${nullable(kind)}, ${nullable(dropSql)}, now())
|
|
241
|
-
ON CONFLICT (identity) DO UPDATE SET src_hash = EXCLUDED.src_hash, def_hash = EXCLUDED.def_hash, kind = EXCLUDED.kind, drop_sql = EXCLUDED.drop_sql, applied_at = now()`;
|
|
248
|
+
export function renderProvenanceUpsert(identity: string, srcHash: string, defHash: string, kind?: string, dropSql?: string, bodyHash?: string): string {
|
|
249
|
+
return `INSERT INTO everystack.derived_provenance (identity, src_hash, def_hash, kind, drop_sql, body_hash, applied_at)
|
|
250
|
+
VALUES (${escapeLiteral(identity)}, ${escapeLiteral(srcHash)}, ${escapeLiteral(defHash)}, ${nullable(kind)}, ${nullable(dropSql)}, ${nullable(bodyHash)}, now())
|
|
251
|
+
ON CONFLICT (identity) DO UPDATE SET src_hash = EXCLUDED.src_hash, def_hash = EXCLUDED.def_hash, kind = EXCLUDED.kind, drop_sql = EXCLUDED.drop_sql, body_hash = EXCLUDED.body_hash, applied_at = now()`;
|
|
242
252
|
}
|
|
243
253
|
|
|
244
254
|
/** A table rename carried its triggers along — same object, new identity; the provenance
|
|
@@ -22,14 +22,22 @@ import type {
|
|
|
22
22
|
FunctionDescriptor, SqlDescriptor, TriggerSpec, Ability, SetofReturn, DependsOnRef,
|
|
23
23
|
} from '@everystack/model';
|
|
24
24
|
import { hashSourceContent, parseQualified, DECLARED_SOURCE_FILE, type Attachment, type SourceObject } from './derived-source.js';
|
|
25
|
+
import { isPlainGrantAttachment } from './derived-grants.js';
|
|
25
26
|
import { findInvokerReachabilityGaps } from './derived-lint.js';
|
|
26
27
|
|
|
27
28
|
/** The provenance marker for descriptor-compiled objects (SourceObject.file). */
|
|
28
29
|
const DECLARED = DECLARED_SOURCE_FILE;
|
|
29
30
|
|
|
30
|
-
/**
|
|
31
|
+
/**
|
|
32
|
+
* Always schema-qualify — the CREATE/GRANT/COMMENT target must be `schema.name` so the object lands
|
|
33
|
+
* in its DECLARED schema regardless of the apply-time search_path order. A bare `public` name landed
|
|
34
|
+
* in search_path[0] instead (derivedSearchPath puts public LAST), so a new public derived object
|
|
35
|
+
* created while a non-public derived schema existed mislanded into the wrong schema and the record
|
|
36
|
+
* loop couldn't find it. (The old bare-public rendering chased hash parity with hand-written db/sql
|
|
37
|
+
* sources; db/sql is retired — B7 — so that rationale is dead. Qualifying is the correctness rule.)
|
|
38
|
+
*/
|
|
31
39
|
function renderName(schema: string, name: string): string {
|
|
32
|
-
return
|
|
40
|
+
return `${schema}.${name}`;
|
|
33
41
|
}
|
|
34
42
|
|
|
35
43
|
/** The declared name of a dependable ref, for rendering (`SETOF posts`) and identity. */
|
|
@@ -243,6 +251,9 @@ function make(kind: SourceObject['kind'], rawName: string, sql: string, attachme
|
|
|
243
251
|
identity: extra.identity ?? `${schema}.${name}`,
|
|
244
252
|
sql, attachments,
|
|
245
253
|
hash: hashSourceContent(sql, attachments),
|
|
254
|
+
// bodyHash excludes PLAIN grants (column-scoped grants stay in — attacl isn't drift-checked, so
|
|
255
|
+
// a column-grant change must still rebuild). Equal across an authz-only plain-grant change.
|
|
256
|
+
bodyHash: hashSourceContent(sql, attachments.filter((a) => !isPlainGrantAttachment(a))),
|
|
246
257
|
file: DECLARED, seq,
|
|
247
258
|
...extra,
|
|
248
259
|
});
|
|
@@ -41,6 +41,21 @@ export interface ParsedGrants {
|
|
|
41
41
|
const GRANT_RE = /^GRANT\s+([A-Z]+)\s*(\([^)]*\))?\s+ON\s+(.+?)\s+TO\s+(.+)$/i;
|
|
42
42
|
const REVOKE_PUBLIC_RE = /^REVOKE\s+ALL\s+ON\s+(.+?)\s+FROM\s+PUBLIC$/i;
|
|
43
43
|
|
|
44
|
+
/**
|
|
45
|
+
* A PLAIN (non-column-scoped) grant/revoke attachment — a table/function-level `GRANT … TO …` with
|
|
46
|
+
* no column list, or a `REVOKE ALL … FROM PUBLIC`. These are the grants `diffObjectGrants` can fully
|
|
47
|
+
* express, so `bodyHash` excludes them: an authz-only change to them applies as a bare regrant, never
|
|
48
|
+
* a rebuild. Column-scoped grants (`GRANT SELECT ("a","b") …`) are NOT plain — attacl is not
|
|
49
|
+
* introspected, so a change to them must still rebuild; they stay in `bodyHash`. Non-grant
|
|
50
|
+
* attachments (index/comment) are structural and always return false.
|
|
51
|
+
*/
|
|
52
|
+
export function isPlainGrantAttachment(a: Attachment): boolean {
|
|
53
|
+
if (a.kind !== 'grant') return false;
|
|
54
|
+
if (REVOKE_PUBLIC_RE.test(a.sql)) return true;
|
|
55
|
+
const g = a.sql.match(GRANT_RE);
|
|
56
|
+
return g !== null && !g[2]; // g[2] is the (columns) group — present => column-scoped => not plain
|
|
57
|
+
}
|
|
58
|
+
|
|
44
59
|
/** The declared grant contract, read back from an object's own grant attachments. */
|
|
45
60
|
export function parseGrantAttachments(attachments: readonly Attachment[]): ParsedGrants {
|
|
46
61
|
const grants: Record<string, Set<string>> = {};
|
|
@@ -81,6 +81,9 @@ export interface ProvenanceRow {
|
|
|
81
81
|
kind?: string;
|
|
82
82
|
/** How to remove what was recorded — the C1 closure: removal never orphans. */
|
|
83
83
|
dropSql?: string;
|
|
84
|
+
/** The source hash MINUS plain grants at record time. Present only once the reconciler has
|
|
85
|
+
* re-recorded the row post-fix; absent on legacy rows → the planner falls back to rebuild. */
|
|
86
|
+
bodyHash?: string;
|
|
84
87
|
}
|
|
85
88
|
|
|
86
89
|
export interface DerivedCatalog {
|
|
@@ -514,6 +517,7 @@ export function assembleDerivedCatalog(rows: DerivedRows): DerivedCatalog {
|
|
|
514
517
|
defHash: String(r.def_hash ?? ''),
|
|
515
518
|
...(r.extra?.kind != null ? { kind: String(r.extra.kind) } : {}),
|
|
516
519
|
...(r.extra?.drop_sql != null ? { dropSql: String(r.extra.drop_sql) } : {}),
|
|
520
|
+
...(r.extra?.body_hash != null ? { bodyHash: String(r.extra.body_hash) } : {}),
|
|
517
521
|
}))
|
|
518
522
|
.sort((a, b) => a.identity.localeCompare(b.identity));
|
|
519
523
|
|
package/src/cli/derived-plan.ts
CHANGED
|
@@ -29,7 +29,7 @@ import type { DerivedCatalog, LiveObject } from './derived-introspect.js';
|
|
|
29
29
|
import { triggerDropSql } from './derived-apply.js';
|
|
30
30
|
import { parseGrantAttachments, diffObjectGrants } from './derived-grants.js';
|
|
31
31
|
|
|
32
|
-
export type ReconcileVerb = 'create' | 'replace' | 'refresh' | 'drop' | 'baseline' | 'rebaseline' | 'prune';
|
|
32
|
+
export type ReconcileVerb = 'create' | 'replace' | 'refresh' | 'drop' | 'baseline' | 'rebaseline' | 'prune' | 'backfill';
|
|
33
33
|
|
|
34
34
|
export interface ReconcileAction {
|
|
35
35
|
action: ReconcileVerb;
|
|
@@ -192,6 +192,9 @@ export function planReconcile(
|
|
|
192
192
|
const baseline: string[] = [];
|
|
193
193
|
const rebaseline: string[] = [];
|
|
194
194
|
const prune: string[] = [];
|
|
195
|
+
/** Up-to-date objects (src+live unchanged) whose provenance predates body_hash — record it once
|
|
196
|
+
* so a FUTURE authz-only change is a regrant, not a rebuild. Record-only, no DDL; arms the fix. */
|
|
197
|
+
const backfill: string[] = [];
|
|
195
198
|
|
|
196
199
|
// -------------------------------------------------------------------------
|
|
197
200
|
// The decision table.
|
|
@@ -289,14 +292,34 @@ export function planReconcile(
|
|
|
289
292
|
continue;
|
|
290
293
|
}
|
|
291
294
|
if (srcChanged) {
|
|
295
|
+
// Authz-only change: the BODY (and live) is untouched, only plain grants moved — bodyHash
|
|
296
|
+
// matches provenance. Route it like a rebaseline: no rebuild, re-record the (new) source +
|
|
297
|
+
// body hash, and the grant-drift pass below (which covers skipped ∪ rebaseline) applies the
|
|
298
|
+
// bare GRANT/REVOKE delta. Gate is strict — any leg missing falls through to today's rebuild:
|
|
299
|
+
// - bodyHash recorded AND equal → the change is grants-only, not body (armed row only)
|
|
300
|
+
// - live grants introspected (≠ undef) → diffObjectGrants can actually run (absent ≠ empty)
|
|
301
|
+
// - no column-scoped grants → attacl isn't drift-applied, so those MUST rebuild
|
|
302
|
+
const parsed = parseGrantAttachments(src.attachments);
|
|
303
|
+
const authzOnly =
|
|
304
|
+
prov.bodyHash != null &&
|
|
305
|
+
src.bodyHash === prov.bodyHash &&
|
|
306
|
+
liveObj.grants !== undefined &&
|
|
307
|
+
!parsed.hasColumnGrants;
|
|
292
308
|
// The live object is verifiably untouched here (liveChanged was handled above), so
|
|
293
309
|
// under rebaseline a source re-render is bookkeeping: re-record the source hash,
|
|
294
310
|
// rebuild nothing, cascade nothing.
|
|
295
|
-
if (options.rebaseline) rebaseline.push(src.identity);
|
|
311
|
+
if (options.rebaseline || authzOnly) rebaseline.push(src.identity);
|
|
296
312
|
else if (isRelation(src.kind)) rebuild.set(src.identity, 'source changed');
|
|
297
313
|
else fnReplace.set(src.identity, 'source changed');
|
|
298
314
|
continue;
|
|
299
315
|
}
|
|
316
|
+
// Up-to-date (src + live both match provenance). If provenance predates body_hash, record it
|
|
317
|
+
// once — a record-only backfill that ARMS the authz-only fast path for a future grant change.
|
|
318
|
+
// Converges: after this run the row has body_hash, so it skips cleanly next time.
|
|
319
|
+
if (prov.bodyHash == null) {
|
|
320
|
+
backfill.push(src.identity);
|
|
321
|
+
continue;
|
|
322
|
+
}
|
|
300
323
|
skipped.push(src.identity);
|
|
301
324
|
}
|
|
302
325
|
|
|
@@ -336,7 +359,7 @@ export function planReconcile(
|
|
|
336
359
|
// Rebaselined objects join the lens: their live catalog rows are as real as a skip's,
|
|
337
360
|
// and running the checks NOW means a migration converges (grants ride the same apply)
|
|
338
361
|
// instead of surfacing on the next plan.
|
|
339
|
-
const lensSet = new Set([...skipped, ...rebaseline]);
|
|
362
|
+
const lensSet = new Set([...skipped, ...rebaseline, ...backfill]);
|
|
340
363
|
for (const src of source.objects) {
|
|
341
364
|
if (!lensSet.has(src.identity)) continue;
|
|
342
365
|
if (src.kind === 'trigger' || src.kind === 'sql') continue; // no grants, no rewrite edges
|
|
@@ -485,6 +508,7 @@ export function planReconcile(
|
|
|
485
508
|
.map(([id, reason]) => actionFor(id, 'refresh', reason)),
|
|
486
509
|
...baseline.sort().map((id) => actionFor(id, 'baseline', 'recording provenance for an existing match (trusted once, explicitly)')),
|
|
487
510
|
...rebaseline.sort().map((id) => actionFor(id, 'rebaseline', 'source re-rendered — live verified untouched (defHash match); provenance re-recorded, no rebuild')),
|
|
511
|
+
...backfill.sort().map((id) => actionFor(id, 'backfill', 'recording body_hash on an up-to-date object — arms the authz-only fast path, no rebuild')),
|
|
488
512
|
...prune.sort().map((id) => ({
|
|
489
513
|
action: 'prune' as const, identity: id, kind: 'view' as DerivedKind,
|
|
490
514
|
reason: 'stale provenance — object gone from both source and database',
|
|
@@ -48,6 +48,14 @@ export interface SourceObject {
|
|
|
48
48
|
attachments: Attachment[];
|
|
49
49
|
/** sha256 of the normalized CREATE + attachments — comment/whitespace edits do not change it. */
|
|
50
50
|
hash: string;
|
|
51
|
+
/**
|
|
52
|
+
* sha256 of the normalized CREATE + attachments EXCEPT plain (non-column-scoped) grants — the
|
|
53
|
+
* "would a rebuild be required" hash. Equal across an authz-only change (plain grants moved, body
|
|
54
|
+
* unchanged), so the reconciler can route that to a bare GRANT delta instead of a matview rebuild.
|
|
55
|
+
* Column-scoped grants stay IN (attacl not introspected → they must rebuild). `hash` stays the
|
|
56
|
+
* full/stable formula for provenance compat; this is the additional discriminator.
|
|
57
|
+
*/
|
|
58
|
+
bodyHash: string;
|
|
51
59
|
/** Source file this object came from (descriptors: the declared-models marker). */
|
|
52
60
|
file: string;
|
|
53
61
|
/** Position in the concatenated source — a valid dependency order by convention. */
|
package/src/cli/index.ts
CHANGED
|
@@ -368,7 +368,7 @@ Usage:
|
|
|
368
368
|
everystack db:provision --stage <name> [--direct | --database-url <url>] Create the least-privilege role chain on an EXISTING database (idempotent; creates no DB). No flag = ops-Lambda venue (auto-falls to direct via the ADMIN_DATABASE_URL secret if the handler has no dbPlugin). --direct = direct connection reading that secret (master never on argv); --database-url = explicit URL. Declared schemas are set as the ROLE default (ALTER ROLE … SET search_path) — the secret stays credentials-only, and the URL is libpq-clean
|
|
369
369
|
everystack db:snapshot [--stage <name>] [--instance <id>] Take a physical RDS snapshot (instant DR point; RDS only — use db:backup for portable logical backups)
|
|
370
370
|
everystack db:snapshots [--stage <name>] [--instance <id>] List manual RDS snapshots for the instance
|
|
371
|
-
everystack db:backup [--stage <name>] Logical pg_dump of the DB → private S3 backups bucket (prints the backup id)
|
|
371
|
+
everystack db:backup [--stage <name>] Logical pg_dump of the DB → private S3 backups bucket (prints the backup id). Add --database-url <local> --stage <target> to dump a LOCAL database and upload it into the target stage's bucket as a restorable backup — the credential-free break-glass local→stage full replace (then: db:restore --from <id> --stage <target>)
|
|
372
372
|
everystack db:backups [--stage <name>] List logical backups (id, size, created)
|
|
373
373
|
everystack db:restore --from <id> [--stage <name>] --confirm Restore a backup INTO the stage's DB (DESTRUCTIVE)
|
|
374
374
|
everystack db:backup:download <id> [--stage <name>] Presigned URL to download a backup's dump (valid 1h)
|