@runeditrun/rer 0.2.73 → 0.2.75

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/cli/contract.mjs +1 -1
  2. package/cli/export.mjs +1 -1
  3. package/cli/parse.mjs +1 -0
  4. package/cli/portability-worker.mjs +6 -7
  5. package/cli/release.mjs +9 -0
  6. package/cli/site-check.mjs +4 -2
  7. package/cli/site-prepare.mjs +6 -0
  8. package/cli/site-render.mjs +7 -1
  9. package/cli/site-runtime.mjs +55 -0
  10. package/cli/site.mjs +45 -17
  11. package/global/CHANGELOG.md +13 -0
  12. package/modules/container/build.mjs +2 -1
  13. package/modules/deploy-kit/README.md +47 -2
  14. package/modules/deploy-kit/owner-cli.mjs +2 -1
  15. package/modules/deploy-kit/site-readback.mjs +12 -2
  16. package/modules/deploy-kit/site-runtime.mjs +33 -0
  17. package/modules/deploy-kit/site.mjs +27 -8
  18. package/modules/operator-auth/scheduled/definition.mjs +3 -1
  19. package/modules/operator-auth/scheduled/expiry.mjs +2 -2
  20. package/modules/operator-auth/scheduled/fence.mjs +2 -1
  21. package/modules/operator-auth/scheduled/runner.mjs +13 -5
  22. package/modules/portability/README.md +27 -8
  23. package/modules/portability/backfills.mjs +22 -14
  24. package/modules/portability/bridge.mjs +9 -6
  25. package/modules/portability/migration-sql.mjs +8 -0
  26. package/modules/portability/migration-statements.mjs +22 -17
  27. package/modules/portability/rebuilds.mjs +106 -0
  28. package/modules/portability/replay-views.mjs +35 -0
  29. package/modules/release/produce.mjs +5 -3
  30. package/modules/release/publication-scan.mjs +5 -4
  31. package/modules/site/docs-search/SearchField.tsx +18 -0
  32. package/modules/site/docs-search/artifacts.mjs +54 -0
  33. package/modules/site/docs-search/client.ts +194 -0
  34. package/modules/site/docs-search/index-build.mjs +63 -0
  35. package/modules/site/docs-search/retrieve.ts +16 -0
  36. package/modules/site/docs-search/search.css +19 -0
  37. package/modules/site/docs-search/sections.ts +24 -0
  38. package/modules/site/docs-search/types.ts +14 -0
  39. package/modules/site/docs-search/worker.ts +39 -0
  40. package/package.json +1 -1
  41. package/scripts/publish-check/README.md +6 -1
  42. package/scripts/publish-check/cli.py +8 -4
  43. package/scripts/publish-check/history_hydration.py +4 -1
  44. package/scripts/publish-check/slots.py +67 -0
  45. package/scripts/publish-check/timeout.json +1 -0
  46. package/scripts/publish-check/timeout.mjs +14 -0
  47. package/scripts/publish-check.mjs +18 -5
@@ -6,11 +6,13 @@ import { uploadAssets, workerOrigin } from './worker.mjs';
6
6
  import { readinessWindow, readinessAPI } from './readiness-window.mjs';
7
7
  import { uploadVersionId } from './uploaded-worker.mjs';
8
8
  import { ownedSiteVersion, waitForSiteDomains, waitForSiteUpload } from './site-readback.mjs';
9
+ import { siteRuntime, siteRuntimeBindings } from './site-runtime.mjs';
9
10
  export { servedVersion, siteReadbackSummary } from './site-readback.mjs';
10
11
 
11
12
  /* A seed's run site (run-<seed>.com): the folder `rer build --site` writes,
12
- * served on the owner's own account by a Base-generated module Worker (fixed
13
- * content, no seed code) whose only binding is its assets. The Worker runs
13
+ * served on the owner's own account by a Base-generated module Worker (no
14
+ * seed code). Its default binding is its assets; opt-in docs search adds AI
15
+ * and an immutable Base-generated search bundle. The Worker runs
14
16
  * first on every request: www.<domain> gets a 301 to the apex with path and
15
17
  * query kept, and everything else is answered from the assets. It has no
16
18
  * secrets and no schedules, workers.dev stays off, and its name can never be
@@ -58,16 +60,32 @@ export function siteWorkerName(project, environment) {
58
60
  }
59
61
 
60
62
  /** Everything a site deploy will do, with no provider request. */
61
- export function sitePlan({ project, environment, compatibilityDate, declaration, assets }) {
63
+ export function sitePlan({ project, environment, compatibilityDate, declaration, assets, runtime }) {
62
64
  const { domain } = siteDeclaration(declaration);
63
65
  assert(/^\d{4}-\d{2}-\d{2}$/.test(compatibilityDate ?? ''), 'invalid-plan');
64
66
  assert(typeof assets === 'string' && assets.startsWith('/'), 'invalid-site-assets');
67
+ runtime = siteRuntime(runtime);
65
68
  return { version: 1, worker: siteWorkerName(project, environment), appWorker: ownerWorkerName(project, environment), module: SITE_MODULE, assets, compatibilityDate,
66
- bindings: ['ASSETS'], secrets: [], crons: [], workersDev: false, runWorkerFirst: true, assetsConfig: { ...SITE_ASSETS_CONFIG },
69
+ bindings: runtime ? [...runtime.bindings] : ['ASSETS'], secrets: [], crons: [], workersDev: false, runWorkerFirst: true, assetsConfig: { ...SITE_ASSETS_CONFIG },
70
+ ...(runtime ? { runtime } : {}),
67
71
  domain, hostnames: [domain, `www.${domain}`],
68
72
  redirects: [{ from: `https://www.${domain}/*`, to: `https://${domain}/`, status: 301, keeps: ['path', 'query'] }] };
69
73
  }
70
74
 
75
+ /** Assets-only plans retain the original fingerprint byte for byte. */
76
+ export function siteFingerprint(plan, files, { preparing = false } = {}) {
77
+ const runtime = siteRuntime(plan.runtime);
78
+ const identity = { files: files.map(({ path, sha256 }) => ({ path, sha256 })), config: plan.assetsConfig,
79
+ module: hash(runtime?.source ?? SITE_WORKER), compatibilityDate: plan.compatibilityDate };
80
+ if (runtime) {
81
+ const vectors = files.find(file => file.path === 'docs-search-vectors.json');
82
+ assert(preparing || vectors && /^[a-f0-9]{64}$/.test(vectors.sha256), 'missing-site-search-vectors');
83
+ identity.runtime = { type: runtime.type, module: runtime.module, endpoint: runtime.endpoint, bindings: runtime.bindings,
84
+ model: runtime.model, pooling: runtime.pooling, corpusHash: runtime.corpusHash, vectors: preparing ? 'prepare-on-deploy' : vectors.sha256 };
85
+ }
86
+ return hash(identity);
87
+ }
88
+
71
89
  async function writeReceipt(path, receipt) {
72
90
  await mkdir(dirname(path), { recursive: true, mode: 0o700 });
73
91
  const temporary = `${path}.${process.pid}.tmp`;
@@ -79,6 +97,8 @@ async function writeReceipt(path, receipt) {
79
97
  export async function siteDeploy({ api, plan, files, receiptPath, readiness = {} }) {
80
98
  assert(typeof receiptPath === 'string' && receiptPath.length, 'missing-receipt-path');
81
99
  assert(plan?.module === SITE_MODULE && Array.isArray(files) && files.some(file => file.path === 'index.html'), 'invalid-site-assets');
100
+ const runtime = siteRuntime(plan.runtime);
101
+ const fingerprint = siteFingerprint(plan, files);
82
102
  const zones = await api.zoneRequest(`/zones?name=${plan.domain}&account.id=${api.accountId}`);
83
103
  const zone = Array.isArray(zones) ? zones.find(row => row?.name === plan.domain && row.account?.id === api.accountId) : null;
84
104
  if (!zone) throw refuse('site-zone-missing', 'zone', plan.domain);
@@ -91,19 +111,18 @@ export async function siteDeploy({ api, plan, files, receiptPath, readiness = {}
91
111
  if (row && (!previous || row.service !== plan.worker)) throw refuse('resource-name-collision', 'domain', hostname);
92
112
  }
93
113
 
94
- const fingerprint = hash({ files: files.map(({ path, sha256 }) => ({ path, sha256 })), config: plan.assetsConfig, module: hash(SITE_WORKER), compatibilityDate: plan.compatibilityDate });
95
114
  const jwt = await uploadAssets(api, plan.worker, files);
96
115
  const body = new FormData();
97
- body.set('metadata', new Blob([JSON.stringify({ main_module: SITE_MODULE, compatibility_date: plan.compatibilityDate, bindings: [{ type: 'assets', name: 'ASSETS' }],
116
+ body.set('metadata', new Blob([JSON.stringify({ main_module: SITE_MODULE, compatibility_date: plan.compatibilityDate, bindings: siteRuntimeBindings(runtime),
98
117
  assets: { jwt, config: { ...plan.assetsConfig, run_worker_first: true } }, annotations: { 'workers/message': `rer-site:${fingerprint}` } })], { type: 'application/json' }));
99
- body.set(SITE_MODULE, new Blob([SITE_WORKER], { type: 'application/javascript+module' }), SITE_MODULE);
118
+ body.set(SITE_MODULE, new Blob([runtime?.source ?? SITE_WORKER], { type: 'application/javascript+module' }), SITE_MODULE);
100
119
  const window = readiness.window ?? readinessWindow(readiness);
101
120
  window.start();
102
121
  api = readinessAPI(api, window, 'site-upload-timeout');
103
122
  const upload = await api.request(`/workers/scripts/${plan.worker}`, { method: 'PUT', body });
104
123
  const versionId = uploadVersionId(upload);
105
124
  assert(versionId, 'invalid-upload-version');
106
- const readback = () => waitForSiteUpload({ api, worker: plan.worker, versionId, fingerprint, previous, window });
125
+ const readback = () => waitForSiteUpload({ api, worker: plan.worker, versionId, fingerprint, previous, window, runtime, module: SITE_MODULE });
107
126
  await readback();
108
127
  await workerOrigin(api, plan.worker, { enable: true, workersDev: false });
109
128
  for (const hostname of plan.hostnames) {
@@ -34,5 +34,7 @@ export async function readPolicy(definitions, id) {
34
34
  if (!Array.isArray(policies) || new Set(policies.map(policy => policy.id)).size !== policies.length) invalid();
35
35
  const input = policies.find(policy => policy.id === id);
36
36
  if (!input) throw new AuthError('policy_not_found', 404);
37
- return policyVersion(input);
37
+ const policy = await policyVersion(input);
38
+ if (policy.definition.id !== id) invalid();
39
+ return policy;
38
40
  }
@@ -3,12 +3,12 @@
3
3
  export const scheduledHandlerLimitMs = 15 * 60000;
4
4
 
5
5
  export async function expireRuns(store, policyId, now) {
6
- const stale = "policy_id=? AND admitted=1 AND status='running' AND started_at<?";
6
+ const stale = "policy_id=? AND status='running' AND started_at<?";
7
7
  const args = [policyId, now - scheduledHandlerLimitMs];
8
8
  // One transaction records each abandoned run exactly once before closing it.
9
9
  await store.batch([
10
10
  [`INSERT INTO oa_auth_audit(id,actor_id,action,value,created_at)
11
- SELECT lower(hex(randomblob(16))),'policy:'||policy_id||'@'||version_hash,'policy:expired',
11
+ SELECT lower(hex(randomblob(16))),'policy:'||policy_id||CASE WHEN version_hash IS NULL THEN '' ELSE '@'||version_hash END,'policy:expired',
12
12
  json_object('policyId',policy_id,'versionHash',version_hash,'approvalId',approval_id,
13
13
  'approvedBy',approved_by,'runId',id,'error','policy_expired'),?
14
14
  FROM oa_policy_runs WHERE ${stale}`, [now, ...args]],
@@ -22,7 +22,8 @@ export function actionFence(approval, runId, action, target, now, maxActions) {
22
22
  return { sql: '(' + sql + ')', args };
23
23
  });
24
24
  return { sql: `${approvalGuard.sql} AND EXISTS(SELECT 1 FROM oa_policy_runs WHERE id=? AND approval_id=? AND status='running' AND admitted=1 AND started_at>=?)
25
- AND (SELECT COUNT(*) FROM oa_auth_audit WHERE CASE WHEN json_valid(value) THEN json_extract(value,'$.runId') END=?)<?
25
+ AND (SELECT COUNT(*) FROM oa_auth_audit WHERE action NOT IN ('policy:refused','policy:expired')
26
+ AND CASE WHEN json_valid(value) THEN json_extract(value,'$.runId') END=?)<?
26
27
  AND role!='owner' AND status!='invited' AND id=? AND (${scopes.map(scope => scope.sql).join(' OR ') || '0'})`,
27
28
  args: [...approvalGuard.args, runId, approval.id, now - scheduledHandlerLimitMs, runId, maxActions, target.id, ...scopes.flatMap(scope => scope.args)] };
28
29
  }
@@ -4,6 +4,11 @@ import { readPolicy } from './definition.mjs';
4
4
  import { actionFence, bindPolicyActor } from './fence.mjs';
5
5
  import { finishRun } from './history.mjs';
6
6
  import { expireRuns } from './expiry.mjs';
7
+ // Expiry is reserved for the audited sweeper, never a callback result.
8
+ function runFailure(error) {
9
+ return error instanceof AuthError && error.code !== 'policy_expired'
10
+ ? { status: 'refused', code: error.code } : { status: 'failed', code: 'policy_failed' };
11
+ }
7
12
  const actionStatuses = Object.freeze({ 'operators:deactivate': 'inactive', 'operators:reactivate': 'active' });
8
13
 
9
14
  export function policyRunner({ definitions, store, clock, currentApproval, check, rolePolicy }) {
@@ -23,13 +28,16 @@ export function policyRunner({ definitions, store, clock, currentApproval, check
23
28
  }
24
29
  }
25
30
  try {
31
+ await expireRuns(store, policyId, startedAt);
26
32
  policy = await readPolicy(definitions, policyId); approval = await currentApproval(policyId);
27
33
  await store.run('UPDATE oa_policy_runs SET version_hash=?,approval_id=?,approved_by=? WHERE id=?', [policy.versionHash, approval?.id ?? null, approval?.ownerId ?? null, id]);
28
34
  await verify(policy);
29
- await expireRuns(store, policyId, startedAt);
30
35
  const admission = await store.first(`UPDATE oa_policy_runs SET admitted=1 WHERE id=? AND NOT EXISTS(
31
- SELECT 1 FROM oa_policy_runs WHERE policy_id=? AND id!=? AND admitted=1
32
- AND (status='running' OR COALESCE(CASE WHEN error='policy_expired' THEN started_at ELSE finished_at END,started_at)>?)) RETURNING id`,
36
+ SELECT 1 FROM oa_policy_runs prior WHERE policy_id=? AND id!=? AND admitted=1
37
+ AND (status='running' OR COALESCE(CASE WHEN EXISTS(
38
+ SELECT 1 FROM oa_auth_audit WHERE action='policy:expired'
39
+ AND CASE WHEN json_valid(value) THEN json_extract(value,'$.runId') END=prior.id
40
+ ) THEN started_at ELSE finished_at END,started_at)>?)) RETURNING id`,
33
41
  [id, policyId, id, startedAt - policy.definition.minIntervalMinutes * 60000]);
34
42
  if (!admission) deny('policy_interval', 429);
35
43
  let pending = null, actionError = null, committed = 0;
@@ -52,7 +60,7 @@ export function policyRunner({ definitions, store, clock, currentApproval, check
52
60
  pending = operation;
53
61
  try { return await operation; } catch (error) {
54
62
  actionError = error;
55
- await store.run('INSERT INTO oa_auth_audit(id,actor_id,action,value,created_at) VALUES(?,?,?,?,?)', [crypto.randomUUID(), capability.actor, 'policy:refused', JSON.stringify({ policyId, versionHash: policy.versionHash, approvalId: approval.id, approvedBy: approval.ownerId, runId: id, operatorId, attemptedAction: action, error: error instanceof AuthError ? error.code : 'policy_failed' }), clock()]);
63
+ await store.run('INSERT INTO oa_auth_audit(id,actor_id,action,value,created_at) VALUES(?,?,?,?,?)', [crypto.randomUUID(), capability.actor, 'policy:refused', JSON.stringify({ policyId, versionHash: policy.versionHash, approvalId: approval.id, approvedBy: approval.ownerId, runId: id, operatorId, attemptedAction: action, error: runFailure(error).code }), clock()]);
56
64
  throw error;
57
65
  } finally { pending = null; }
58
66
  } });
@@ -61,7 +69,7 @@ export function policyRunner({ definitions, store, clock, currentApproval, check
61
69
  await finishRun(store, id, null, clock());
62
70
  return { id };
63
71
  } catch (error) {
64
- await finishRun(store, id, { status: error instanceof AuthError ? 'refused' : 'failed', code: error instanceof AuthError ? error.code : 'policy_failed' }, clock());
72
+ await finishRun(store, id, runFailure(error), clock());
65
73
  throw error;
66
74
  } finally { open = false; }
67
75
  };
@@ -261,10 +261,11 @@ requires version 2 integrity.
261
261
 
262
262
  An archive restores into an installation whose schema differs only by migrations
263
263
  applied after the archive was made, when those migrations are additive: new
264
- tables, new columns that are nullable or have a default, and indexes. The
264
+ tables, new columns that are nullable or have a default, and indexes, or the
265
+ preserving CHECK rebuild described below. The
265
266
  archive's tables restore with their archived columns, later columns take their
266
- defaults, and later tables stay empty (they must be empty, like every restored
267
- table). This applies to normal v2 import and to backup-only v1 recovery.
267
+ defaults, and later tables stay empty unless their eligible seed or backfill is
268
+ replayed below. This applies to normal v2 import and to backup-only v1 recovery.
268
269
 
269
270
  The Base CLI runners (local Workers and container) pass `createPortability` a
270
271
  `migrations` replayer (`migration-history.mjs`, Node only) built from the
@@ -298,7 +299,8 @@ contain only `CREATE TABLE` (not `AS SELECT`), `CREATE INDEX` and
298
299
  `ALTER TABLE ... ADD COLUMN` (SQLite allows only constant defaults there) on the
299
300
  restorable tables. A later migration with any other statement that names a
300
301
  restorable table, such as an `INSERT`, `UPDATE` or `DELETE` that purges or
301
- backfills rows or a trigger, is refused with `schema_mismatch`; statements that
302
+ backfills rows, is refused with `schema_mismatch` unless admitted or replayed
303
+ below. Bounded trigger and view creation/removal changes only schema. Statements that
302
304
  touch only excluded tables are allowed. A data statement (`INSERT`, `REPLACE`,
303
305
  `UPDATE` or `DELETE`, including `INSERT ... SELECT` and a leading `WITH`) is
304
306
  judged by the table it writes: when that table is outside the restorable
@@ -319,12 +321,22 @@ keywords outside table position, function names (also table-valued ones such as
319
321
  literals, numbers and comments never count. Text Base cannot read to a
320
322
  statement boundary, such as an unclosed block comment, is refused. An archived table the archive holds no
321
323
  rows of restores empty, so an `UPDATE` or `DELETE` of it changes nothing and is
322
- allowed unless it also names a table the archive holds rows of (which it could
323
- read into its writes); an `INSERT` into it, or any other statement naming it,
324
- is still refused. For archives without a migration record,
324
+ allowed regardless of what it reads: no row can change or fire a trigger. An
325
+ ordinary `INSERT` into it remains refused. For archives without a migration record,
325
326
  every migration after the earliest prefix that gives the archive's schema is
326
327
  checked, since the archive may predate any of them.
327
328
 
329
+ A complete five-statement `CREATE TABLE copy AS SELECT` / `DROP TABLE` /
330
+ `CREATE TABLE` / `INSERT ... SELECT` / `DROP TABLE copy` sequence within one
331
+ migration is admitted when every column is copied back unchanged and, for a
332
+ populated archived table, its rowid is copied back too. The copy must be local
333
+ to the sequence. Only CHECK clauses may differ; column declarations, defaults,
334
+ collations, other constraints and table options remain unchanged. Manifest
335
+ counts identify empty tables without another row scan. Restore uses the
336
+ destination definition, so a row rejected by its CHECK fails the whole atomic
337
+ restore; no row is skipped or coerced. Other statements still face the usual
338
+ data classifier, and a proven preserving block is not a competing backfill writer.
339
+
328
340
  A backfill of a catalogue table the archive does not carry (one a later
329
341
  migration adds, restored empty) is **replayed** instead of refused: an
330
342
  `INSERT ... SELECT` (or `INSERT ... VALUES`) into that table. Finalize empties
@@ -339,13 +351,18 @@ scans the restored rows. Rows the restore normalizes (import safety) are read
339
351
  as restored. The restore is refused with
340
352
  `schema_mismatch:migration_data:<migration>` instead when the archive carries
341
353
  the table (even with no rows), when a write to it is not a plain `INSERT` or
342
- `REPLACE`, reads any table other than archived ones and itself, selects `*` or
354
+ `REPLACE`, reads any table other than archived ones and itself (including through
355
+ a view resolved from applied migration history), selects `*` or
343
356
  a secret column, when a trigger on it or a foreign key action from it reaches a
344
357
  catalogue table, when any other later statement writes it (directly or through
345
358
  a trigger) or changes its columns other than by `ADD COLUMN` (allowed only when
346
359
  every replay names its columns), or when the statement cannot be bounded.
347
360
  A failed or interrupted apply leaves no replayed row; finalizing again replays
348
361
  from the start.
362
+ Read-only trigger bodies, including RAISE-only guards, add no write dependency.
363
+ Eligible constant seed INSERTs replay too. Statements confined to tables added
364
+ after the archive retain their existing admission when they need no replay;
365
+ an INSERT reading a view must still resolve entirely to carried tables.
349
366
 
350
367
  The same check runs when the archive's schema identity equals the destination's:
351
368
  a later migration that only purges or backfills rows leaves the schema unchanged,
@@ -646,6 +663,8 @@ modules/portability/ledgers.mjs
646
663
  modules/portability/migration-statements.mjs
647
664
  modules/portability/sql-text.mjs
648
665
  modules/portability/backfills.mjs
666
+ modules/portability/rebuilds.mjs
667
+ modules/portability/replay-views.mjs
649
668
  modules/portability/legacy-safety.mjs
650
669
  modules/portability/export.mjs
651
670
  modules/portability/import.mjs
@@ -2,6 +2,8 @@ import { statement, quote, insist } from './storage.mjs';
2
2
  import { tokens, folded } from './generated.mjs';
3
3
  import { changesData, dataStatement, reachingTables, triggerBodies, schemaChange, statementNames } from './migration-statements.mjs';
4
4
  import { splitStatements } from './sql-text.mjs';
5
+ import { preservingRebuilds } from './rebuilds.mjs';
6
+ import { replayViews, viewReads } from './replay-views.mjs';
5
7
 
6
8
  // The data statements of later migrations, judged by the tables they write
7
9
  // (bridge.mjs placement). Restored rows never pass through a later migration.
@@ -10,12 +12,13 @@ import { splitStatements } from './sql-text.mjs';
10
12
  // writes reach no catalogue table through a trigger or foreign key action
11
13
  // (reachingTables) changes no restored row, whatever it reads: admitted.
12
14
  // - One whose target is a catalogue table the archive does not carry (restored
13
- // empty) and that today's rule refuses (it reads archived rows: a backfill)
14
- // is replayed at finalize. Every later write to that table must then be a
15
+ // empty) is replayed at finalize when it inserts seed or backfill rows.
16
+ // Every later write to that table must then be a
15
17
  // replayable INSERT: they run in migration order, exactly as written, after
16
18
  // the archived rows are inserted and with the destination's triggers still
17
19
  // dropped, so the table holds what the live upgrade gave it. A replay may read
18
- // only archived tables (complete by then) and the table itself, by named
20
+ // only archived tables (complete by then), views over those tables and the
21
+ // table itself, by named
19
22
  // columns that are not secret; no trigger on the table may write a catalogue
20
23
  // table; no later statement may write it another way (directly or through a
21
24
  // trigger) or change its columns other than by ADD COLUMN.
@@ -25,14 +28,15 @@ import { splitStatements } from './sql-text.mjs';
25
28
  // Each chunk the runner's splitter handed over is split again at SQLite's own
26
29
  // statement boundaries first (sql-text.mjs), so a trigger glued to the
27
30
  // statements after it is judged as the statements it holds.
28
- // Returns the replays in migration order, or refuses with the first migration
31
+ // Returns the replays in migration order and the proven CHECK rebuilds, or refuses with the first migration
29
32
  // (in `unapplied` order) holding a statement that cannot be admitted.
30
- export function laterData({applied,unapplied,replay,catalogue,archived,empty,tables,secrets}) {
33
+ export function laterData({applied,unapplied,replay,catalogue,archived,empty,tables,secrets,manifest}) {
31
34
  const split=name=>replay.statements(name).flatMap(splitStatements),fold=list=>list.map(folded);
32
35
  const statements=applied.flatMap(split);
33
36
  const restorable=fold(catalogue),carried=fold(archived),absent=restorable.filter(name=>!carried.includes(name));
34
37
  const reached=reachingTables(statements,carried),writable=reachingTables(statements,restorable);
35
38
  const later=unapplied.flatMap(migration=>split(migration).map((sql,index)=>({migration,index,sql,data:dataStatement(sql)})));
39
+ const {admitted,tables:rebuilt}=preservingRebuilds(later,manifest,tables,statements),views=replayViews(statements);
36
40
  const known=fold(tables),outside=later.filter(s=>s.data && !writable.includes(s.data.target)),tainted=new Set();
37
41
  // Tables an admitted write fills from archived rows (directly, through its
38
42
  // target's triggers, or from another such table) hold rows the restore lacks.
@@ -40,29 +44,33 @@ export function laterData({applied,unapplied,replay,catalogue,archived,empty,tab
40
44
  for(let grew=true;grew;)grew=outside.filter(s=>s.data.names.some(name=>carried.includes(name) || tainted.has(name))).map(s=>taint(s.data.target)).some(Boolean);
41
45
  const refused=new Set(),writes=new Map();
42
46
  for(const s of later) {
43
- if(outside.includes(s))continue;
47
+ if(outside.includes(s) || admitted.has(s))continue;
44
48
  s.today=changesData(s.sql,[...reached,...tainted],empty);
45
49
  if(s.data && absent.includes(s.data.target)){if(!writes.has(s.data.target))writes.set(s.data.target,[]);writes.get(s.data.target).push(s);}
46
50
  else if(s.today)refused.add(s);
47
51
  }
48
52
  const replays=[];
49
53
  for(const [table,list] of writes) {
50
- if(!list.some(s=>s.today))continue;
51
- const bad=list.filter(s=>!replayable(s,{table,statements,restorable,writable,carried,tables:known,secrets}));
52
- const reach=reachingTables(statements,[table]),other=later.filter(s=>!list.includes(s) && writesAnother(s,table,reach,list));
54
+ // Seed INSERTs into a new table replay too: the rows already present in a
55
+ // fresh destination must be reset and derived from the restored installation.
56
+ const bad=list.filter(s=>!replayable(s,{table,statements,restorable,writable,carried,tables:known,secrets,views}));
57
+ // Writes confined to tables created after the archive retain their
58
+ // admission. Replay eligible seeds, but do not turn a new-table-only
59
+ // upgrade into a refused backfill. View reads always require resolution.
60
+ const required=list.some(s=>s.today || s.data.names.some(name=>views.has(name)));
61
+ if(!required && bad.length)continue;
62
+ const reach=reachingTables(statements,[table]),other=later.filter(s=>!admitted.has(s) && !list.includes(s) && writesAnother(s,table,reach,list));
53
63
  if(bad.length || other.length){[...bad,...other].forEach(s=>refused.add(s));continue;}
54
64
  replays.push(...list);
55
65
  }
56
66
  const first=later.find(s=>refused.has(s));
57
67
  insist(!first,'schema_mismatch:migration_data:'+first?.migration);
58
68
  const named=Object.fromEntries(catalogue.map(name=>[folded(name),name]));
59
- return later.filter(s=>replays.includes(s)).map(s=>({migration:s.migration,statement:s.index,table:named[s.data.target],sql:s.sql}));
69
+ return {rebuilt,replays:later.filter(s=>replays.includes(s)).map(s=>({migration:s.migration,statement:s.index,table:named[s.data.target],sql:s.sql}))};
60
70
  }
61
- function replayable({data},{table,statements,restorable,writable,carried,tables,secrets}) {
71
+ function replayable({data},{table,statements,restorable,writable,carried,tables,secrets,views}) {
62
72
  if(!['insert','replace'].includes(data.verb) || data.star)return false;
63
- const reads=data.names.filter(name=>tables.includes(name) && name!==table);
64
- if(!reads.every(name=>carried.includes(name)))return false;
65
- if(reads.some(name=>Object.entries(secrets).some(([t,columns])=>folded(t)===name && columns.some(c=>data.words.includes(folded(c))))))return false;
73
+ if(!viewReads(data.names,data.words,{views,tables,carried,table,secrets}))return false;
66
74
  if(triggerBodies(statements,table).some(name=>writable.includes(name)))return false;
67
75
  // Foreign key actions: a row the replay deletes or rekeys (REPLACE, upsert) must reach no other catalogue table.
68
76
  return !reachingTables(statements,restorable.filter(name=>name!==table)).includes(table);
@@ -3,6 +3,7 @@ import { canonicalJson } from './hash.mjs';
3
3
  import { tokens, identifier, folded } from './generated.mjs';
4
4
  import { laterData } from './backfills.mjs';
5
5
  import { destinationLedger } from './ledgers.mjs';
6
+ import { checkRebuild } from './rebuilds.mjs';
6
7
 
7
8
  // Applied product migrations in the ledger's own order (ledgers.mjs). A
8
9
  // state-bearing ledger may hold an unacknowledged migration whose effects are
@@ -51,8 +52,8 @@ async function placement(db,manifest,cat,replay,ledger,history) {
51
52
  const catalogue=cat.tables.map(t=>t.name),archived=catalogue.filter(name=>manifest.tables.some(t=>t.name===name));
52
53
  const empty=archived.filter(name=>!manifest.tables.find(t=>t.name===name).count);
53
54
  const tables=(await all(db,"SELECT name FROM sqlite_master WHERE type IN ('table','view')")).map(t=>t.name);
54
- const replays=laterData({applied,unapplied,replay,catalogue,archived,empty,tables,secrets:manifest.excludedColumns??{}});
55
- return {earlier,replays};
55
+ const {replays,rebuilt}=laterData({applied,unapplied,replay,catalogue,archived,empty,tables,secrets:manifest.excludedColumns??{},manifest:manifest.tables});
56
+ return {earlier,replays,rebuilt};
56
57
  }
57
58
  // An archive without a migration record that carries its source's migration
58
59
  // ledger (an old archive from before that ledger became bookkeeping) records
@@ -84,20 +85,22 @@ export async function sameSchemaMigrations(db,manifest,cat,migrations,history) {
84
85
  await placement(db,manifest,cat,await migrations(),ledger,history);
85
86
  }
86
87
  // An archive made before later migrations restores only when those migrations
87
- // are additive: new tables, new columns that take a default, or indexes (which
88
- // archives never carry). Anything else stays a schema_mismatch refusal.
88
+ // are additive, or a proven row-preserving CHECK rebuild: new tables, new
89
+ // columns that take a default, or indexes (which archives never carry).
90
+ // Anything else stays a schema_mismatch refusal.
89
91
  export async function additiveBridge(db,manifest,cat,migrations,history) {
90
92
  insist(migrations,'schema_mismatch');
91
93
  const replay=await migrations(),ledger=await destinationLedger(db)??[],applied=ledger.map(m=>m.name);
92
94
  insist(!manifest.migrations || manifest.migrations.every(name=>applied.includes(name)),'schema_mismatch:migrations');
93
- const {earlier,replays}=await placement(db,manifest,cat,replay,ledger,history);
95
+ const {earlier,replays,rebuilt}=await placement(db,manifest,cat,replay,ledger,history);
94
96
  const columns={};
95
97
  for(const table of manifest.tables) {
96
98
  const target=cat.tables.find(t=>t.name===table.name),label=':'+table.name;
97
99
  insist(target,'schema_mismatch:table'+label);
98
100
  const kept=target.columns.slice(0,table.columns.length),added=target.columns.slice(table.columns.length);
99
101
  insist(canonicalJson(kept)===canonicalJson(table.columns) && table.rowidAlias===target.rowidAlias && table.dependencies.every(d=>target.dependencies.includes(d)),'schema_mismatch:columns'+label);
100
- if(!added.length){insist(table.sql===target.sql,'schema_mismatch:definition'+label);continue;}
102
+ const rebuiltChecks=rebuilt.has(folded(table.name)) && checkRebuild(table.sql,target.sql);
103
+ if(!added.length){insist(table.sql===target.sql || rebuiltChecks,'schema_mismatch:definition'+label);continue;}
101
104
  const declared=await all(db,`PRAGMA table_xinfo(${quote(table.name)})`);
102
105
  insist(added.every(c=>!c.pk && !c.generated && (!c.notnull || declared.find(d=>d.name===c.name)?.dflt_value!=null)),'schema_mismatch:column_default'+label);
103
106
  insist(appended(table.sql,target.sql,added.map(c=>c.name)),'schema_mismatch:definition'+label);
@@ -0,0 +1,8 @@
1
+ import { splitStatements } from './sql-text.mjs';
2
+
3
+ // Use the installation's migration runner to reproduce the text it stores in
4
+ // sqlite_master (including its comment handling). Then recover SQLite's own
5
+ // boundaries: Wrangler can glue a CASE ... END trigger to later statements.
6
+ export function migrationStatements(sql, runnerSplit) {
7
+ return runnerSplit(sql).flatMap(splitStatements);
8
+ }
@@ -16,8 +16,8 @@ import { tableNames, splitStatements } from './sql-text.mjs';
16
16
  // operator-auth or other excluded tables that no trigger links to restorable ones.
17
17
  // `empty` lists the restorable tables the archive holds no rows of: restore
18
18
  // leaves them empty, and an UPDATE or DELETE of an empty table changes nothing
19
- // (no row fires a trigger either), so such a statement changes data only if it
20
- // names another reached table (one it could write, or read into its writes).
19
+ // (no row fires a trigger either), regardless of the tables it reads. This
20
+ // exemption requires a bounded data statement whose target can be read.
21
21
  // Any other statement naming an empty table still does: an INSERT would leave
22
22
  // rows the restore lacks. A chunk holding several statements (sql-text.mjs
23
23
  // splitStatements) changes data when any of them does.
@@ -27,28 +27,30 @@ export function changesData(sql,reached,empty=[]) {
27
27
  function statementChanges(sql,reached,empty) {
28
28
  const list=tokens(sql),words=list.map(t=>t.quoted?undefined:folded(t.text)),kind=kindOf(words);
29
29
  if(kind==='create table')return topLevelWords(list,words).includes('as');
30
- if(kind==='create index')return false;
30
+ if(['create index','create view','drop view'].includes(kind))return false;
31
31
  if(kind==='alter table' && words.find((w,i)=>i>2 && ['add','rename','drop'].includes(w))==='add')return false;
32
32
  if(kind==='create trigger' && createdTrigger(list))return false;
33
33
  if(kind==='drop trigger' && droppedTrigger(list))return false;
34
- const skipped=new Set(['update','delete'].includes(words[0])?empty.map(folded):[]);
35
- return names(list,new Set(reached.map(folded).filter(name=>!skipped.has(name))));
34
+ const data=dataStatement(sql);
35
+ if(data && ['update','delete'].includes(data.verb) && empty.map(folded).includes(data.target))return false;
36
+ return names(list,new Set(reached.map(folded)));
36
37
  }
37
38
  // The restorable tables and every table whose writes reach one through a
38
39
  // trigger any of `statements` creates, or through a foreign key action (ON
39
40
  // DELETE/UPDATE CASCADE or SET) declared by a reached table: a write to such a
40
41
  // table during a later migration changes restorable rows the archive lacks. A
41
- // trigger counts when its statement names a reached table anywhere; one this
42
- // parser cannot bound adds every table name it holds (sql-text.mjs tableNames:
43
- // never a keyword, function, qualified column or literal).
42
+ // bounded trigger with a writing body counts when its statement names a reached
43
+ // table anywhere; a read-only body (including SELECT RAISE) writes nothing.
44
+ // One this parser cannot bound adds every table name it holds (sql-text.mjs
45
+ // tableNames: never a keyword, function, qualified column or literal).
44
46
  export function reachingTables(statements,restorable) {
45
47
  const reached=new Set(restorable.map(folded)),parsed=statements.flatMap(splitStatements).map(sql=>tokens(sql)).map(list=>({list,words:list.map(t=>t.quoted?undefined:folded(t.text))}));
46
- const triggers=parsed.filter(({words})=>kindOf(words)==='create trigger').map(({list})=>({list,on:createdTrigger(list)?.on}));
48
+ const triggers=parsed.filter(({words})=>kindOf(words)==='create trigger').map(({list})=>({list,created:createdTrigger(list)})).filter(({created})=>!created || created.writes);
47
49
  const actions=parsed.filter(({words})=>['create table','alter table'].includes(kindOf(words)) && words.some((w,i)=>w==='cascade' || (w==='set' && ['delete','update'].includes(words[i-1])))).map(({list})=>referencing(list));
48
50
  for(let grew=true;grew;) {
49
51
  grew=false;
50
52
  const add=name=>{if(!reached.has(name)){reached.add(name);grew=true;}};
51
- for(const {list,on} of triggers)if(names(list,reached))(on===undefined?tableNames(list):[on]).forEach(add);
53
+ for(const {list,created} of triggers)if(names(list,reached))(created===undefined?tableNames(list):[created.on]).forEach(add);
52
54
  for(const {table,parents} of actions)if(table===undefined?parents.some(name=>reached.has(name)):reached.has(table))parents.forEach(add);
53
55
  }
54
56
  return [...reached];
@@ -115,7 +117,7 @@ function closing(parts,start) {
115
117
  for(let i=start;i<parts.length;i++){if(parts[i].quoted)continue;if(parts[i].text==='(')depth++;else if(parts[i].text===')' && !--depth)return i+1;}
116
118
  return parts.length;
117
119
  }
118
- // The table names in the bodies of triggers on `table` that `statements`
120
+ // The table names in writing bodies of triggers on `table` that `statements`
119
121
  // create, and every one in a trigger this parser cannot bound that names the table.
120
122
  export function triggerBodies(statements,table) {
121
123
  const result=[];
@@ -123,7 +125,7 @@ export function triggerBodies(statements,table) {
123
125
  if(kindOf(list.map(t=>t.quoted?undefined:folded(t.text)))!=='create trigger')continue;
124
126
  const created=createdTrigger(list);
125
127
  if(!created){if(tableNames(list).includes(table))result.push(...tableNames(list));}
126
- else if(created.on===table)result.push(...tableNames(atoms(list).slice(created.begin+1)));
128
+ else if(created.on===table && created.writes)result.push(...tableNames(atoms(list).slice(created.begin+1)));
127
129
  }
128
130
  return result;
129
131
  }
@@ -149,9 +151,11 @@ function atoms(list) {
149
151
  }
150
152
  const semicolon=a=>!a.quoted && a.text===';';
151
153
  const word=(a,value)=>a && !a.quoted && folded(a.text)===value;
152
- // {on} for CREATE [TEMP] TRIGGER ... ON table ... BEGIN statement; ... END and
153
- // nothing after it. The body ends at the first END that starts a statement;
154
- // CASE ... END inside a statement does not. Anything else is unbounded.
154
+ // {on,begin,writes} for CREATE [TEMP] TRIGGER ... ON table ... BEGIN statement;
155
+ // ... END and nothing after it. Only body DML counts as writing; WHEN reads
156
+ // and the header's INSERT/UPDATE/DELETE event do not. The body ends at the first
157
+ // END that starts a statement; CASE ... END inside a statement does not.
158
+ // Anything else is unbounded.
155
159
  function createdTrigger(list) {
156
160
  const parts=atoms(list);let depth=0,begin=-1,on;
157
161
  for(let i=0;i<parts.length && begin<0;i++) {
@@ -170,11 +174,12 @@ function createdTrigger(list) {
170
174
  }else if(!depth && word(a,'begin'))begin=i;
171
175
  }
172
176
  if(begin<0 || on===undefined)return;
173
- let start=true,statements=0;
177
+ let start=true,statements=0,writes=false;
174
178
  for(let i=begin+1;i<parts.length;i++) {
175
179
  const a=parts[i];
176
- if(start && word(a,'end'))return i===parts.length-1 && statements && !depth?{on,begin}:undefined;
180
+ if(start && word(a,'end'))return i===parts.length-1 && statements && !depth?{on,begin,writes}:undefined;
177
181
  if(semicolon(a)){if(start || depth)return;start=true;statements++;continue;}
182
+ if(['insert','replace','update','delete'].some(verb=>word(a,verb)))writes=true;
178
183
  if(!a.quoted){if(a.text==='(')depth++;else if(a.text===')' && --depth<0)return;}
179
184
  start=false;
180
185
  }
@@ -0,0 +1,106 @@
1
+ import { tokens, identifier, folded } from './generated.mjs';
2
+ import { statementNames, reachingTables } from './migration-statements.mjs';
3
+
4
+ const word=(t,w)=>t && !t.quoted && folded(t.text)===w;
5
+ const name=t=>t && (t.quoted?t.text[0]!=="'":/^[A-Za-z_\u0080-\uffff][\w$\u0080-\uffff]*$/.test(t.text))?folded(identifier(t)):undefined;
6
+ const same=(a,b)=>JSON.stringify(a)===JSON.stringify(b);
7
+ const lex=sql=>tokens(sql);
8
+ function groups(list) {
9
+ const result=[[]];let depth=0;
10
+ for(const t of list) {
11
+ if(t.text==='(')depth++;
12
+ if(t.text===')')depth--;
13
+ if(t.text===',' && !depth)result.push([]);else result.at(-1).push(t);
14
+ }
15
+ return result;
16
+ }
17
+ // A definition may change CHECKs, while every column declaration, other
18
+ // constraint, collation, default, generated expression and table option stays.
19
+ export function checkRebuild(before,after) {
20
+ const shape=sql=>{
21
+ const list=lex(sql),open=list.findIndex(t=>t.text==='(');let depth=0,close=-1;
22
+ for(let i=open;i<list.length;i++){if(list[i].text==='(')depth++;if(list[i].text===')' && !--depth){close=i;break;}}
23
+ if(open<0 || close<0)return;
24
+ const declarations=groups(list.slice(open+1,close)).map(part=>{
25
+ const kept=[];
26
+ for(let i=0;i<part.length;i++) {
27
+ if(word(part[i],'constraint') && word(part[i+2],'check')){i+=2;}
28
+ if(word(part[i],'check') && part[i+1]?.text==='(') {
29
+ let d=0;i++;
30
+ do {if(part[i].text==='(')d++;if(part[i].text===')')d--;i++;}while(i<part.length && d);
31
+ if(d)return ['unbounded'];
32
+ i--;continue;
33
+ }
34
+ kept.push(part[i]);
35
+ }
36
+ return kept;
37
+ }).filter(part=>part.length);
38
+ return [...declarations.map(part=>part.map(normal)),list.slice(close+1).map(normal)];
39
+ };
40
+ const a=shape(before),b=shape(after);
41
+ return a!==undefined && b!==undefined && same(a,b);
42
+ }
43
+ const normal=t=>t.quoted?(t.text[0]==="'"?t.text:folded(identifier(t))):folded(t.text);
44
+ function declaration(sql) {
45
+ const list=lex(sql);
46
+ if(!word(list[0],'create') || !word(list[1],'table') || !name(list[2]) || list[3]?.text!=='(')return;
47
+ let depth=0,close=-1;
48
+ for(let i=3;i<list.length;i++){if(list[i].text==='(')depth++;if(list[i].text===')' && !--depth){close=i;break;}}
49
+ if(close<0)return;
50
+ const columns=groups(list.slice(4,close)).filter(p=>!['constraint','primary','unique','check','foreign'].some(w=>word(p[0],w))).map(p=>name(p[0]));
51
+ if(columns.some(c=>!c))return;
52
+ return {name:name(list[2]),columns};
53
+ }
54
+ function select(list,columns) {
55
+ if(!word(list[0],'select'))return;
56
+ const from=list.findIndex(t=>word(t,'from'));
57
+ if(from<0 || from!==list.length-2 || !name(list.at(-1)))return;
58
+ const projection=groups(list.slice(1,from)).flatMap(part=>part.length===1 && part[0].text==='*'?columns.map(c=>({source:c,name:c})):
59
+ part.length===1 && name(part[0])?[{source:name(part[0]),name:name(part[0])}]:
60
+ part.length===3 && word(part[1],'as') && name(part[0]) && name(part[2])?[{source:name(part[0]),name:name(part[2])}]:[{source:undefined}]);
61
+ if(projection.some(p=>!p.source))return;
62
+ return {table:name(list.at(-1)),projection};
63
+ }
64
+ const drop=(sql,table)=>{const t=lex(sql);return t.length===3 && word(t[0],'drop') && word(t[1],'table') && name(t[2])===table;};
65
+ function copied(block,table) {
66
+ const [copy,removed,created,insert,cleaned]=block.map(s=>s.sql),head=lex(copy),target=declaration(created);
67
+ if(!target || target.name!==folded(table.name) || !word(head[0],'create') || !word(head[1],'table') || !name(head[2]) || !word(head[3],'as'))return;
68
+ const temporary=name(head[2]),saved=select(head.slice(4),target.columns);
69
+ if(!saved || saved.table!==target.name || temporary===target.name || !drop(removed,target.name) || !drop(cleaned,temporary))return;
70
+ const put=lex(insert);
71
+ if(!word(put[0],'insert') || !word(put[1],'into') || name(put[2])!==target.name)return;
72
+ let at=3,columns=target.columns;
73
+ if(put[at]?.text==='('){const end=put.findIndex((t,i)=>i>at && t.text===')');if(end<0)return;columns=groups(put.slice(at+1,end)).map(p=>p.length===1?name(p[0]):undefined);at=end+1;}
74
+ const restored=select(put.slice(at),saved.projection.map(p=>p.name));
75
+ if(!restored || restored.table!==temporary || restored.projection.length!==columns.length || new Set(columns).size!==columns.length)return;
76
+ const mapping=columns.map((column,i)=>({column,source:saved.projection.find(p=>p.name===restored.projection[i].source)?.source}));
77
+ if(!target.columns.every(c=>mapping.some(m=>m.column===c && m.source===c)))return;
78
+ if(!table.columns.every(c=>target.columns.includes(folded(c.name))) || table.columns.some(c=>c.generated))return;
79
+ // A populated archive requires an explicit rowid round trip, even for a
80
+ // table whose declared key makes its rowid incidental to archive identity.
81
+ if(table.count && !mapping.some(m=>['rowid','_rowid_','oid'].includes(m.column) && ['rowid','_rowid_','oid'].includes(m.source) && !target.columns.includes(m.column) && !target.columns.includes(m.source)))return;
82
+ if(mapping.some(m=>!m.source || (!target.columns.includes(m.column) && !['rowid','_rowid_','oid'].includes(m.column))))return;
83
+ return temporary;
84
+ }
85
+ // Admit only a complete, local copy/drop/create/copy-back/drop sequence. The
86
+ // manifest supplies counts; no restored rows are scanned to classify SQL.
87
+ // Statements outside the block still face the normal data classifier.
88
+ export function preservingRebuilds(later,manifest,known,statements) {
89
+ const admitted=new Set(),tables=new Set();
90
+ for(let i=0;i+4<later.length;i++) {
91
+ const block=later.slice(i,i+5);
92
+ if(block.some(s=>s.migration!==block[0].migration))continue;
93
+ const table=manifest.find(t=>folded(t.name)===declaration(block[2].sql)?.name);
94
+ if(!table)continue;
95
+ const temporary=copied(block,table);
96
+ if(!temporary || known.map(folded).includes(temporary))continue;
97
+ if(later.some(s=>!block.includes(s) && statementNames(s.sql).includes(temporary)))continue;
98
+ const foreignKeys=statements.filter(sql=>{const t=lex(sql);return (word(t[0],'create') || word(t[0],'alter')) && word(t[1],'table');});
99
+ if(table.count && reachingTables(foreignKeys,manifest.filter(t=>t!==table).map(t=>t.name)).includes(folded(table.name)))continue;
100
+ // CHECK-only changes are verified against the archived definition too;
101
+ // dropping/changing a column never earns a definition exemption.
102
+ if(!checkRebuild(table.sql,block[2].sql))continue;
103
+ block.forEach(s=>admitted.add(s));tables.add(folded(table.name));i+=4;
104
+ }
105
+ return {admitted,tables};
106
+ }