@runeditrun/rer 0.2.84 → 0.2.86

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (236) hide show
  1. package/README.md +156 -50
  2. package/cli/README.md +78 -2
  3. package/cli/agent-door.mjs +149 -0
  4. package/cli/agent-fixtures.mjs +16 -0
  5. package/cli/agent-mail.mjs +54 -0
  6. package/cli/agent-network.mjs +64 -0
  7. package/cli/agent-reviewers.mjs +110 -0
  8. package/cli/agent-secrets.mjs +23 -0
  9. package/cli/agent-sponsors.mjs +43 -0
  10. package/cli/agent-surface.mjs +34 -0
  11. package/cli/agent-webhooks.mjs +19 -0
  12. package/cli/authoring.mjs +2 -1
  13. package/cli/bounded.mjs +1 -1
  14. package/cli/business/archive-task.mjs +32 -0
  15. package/cli/business/carry.mjs +151 -0
  16. package/cli/business/catchup-task.mjs +42 -0
  17. package/cli/business/choices.mjs +97 -0
  18. package/cli/business/command.mjs +29 -6
  19. package/cli/business/connect.mjs +20 -8
  20. package/cli/business/detector.mjs +4 -1
  21. package/cli/business/device.mjs +13 -0
  22. package/cli/business/export-inputs.mjs +69 -0
  23. package/cli/business/export-task.mjs +17 -0
  24. package/cli/business/invocation.mjs +8 -1
  25. package/cli/business/keychain.mjs +8 -1
  26. package/cli/business/lease.mjs +11 -1
  27. package/cli/business/local-commands.mjs +80 -30
  28. package/cli/business/outbound.mjs +9 -4
  29. package/cli/business/parsers-config.mjs +8 -7
  30. package/cli/business/parsers-syntax.mjs +7 -3
  31. package/cli/business/permissions.mjs +2 -0
  32. package/cli/business/questions.mjs +89 -6
  33. package/cli/business/setup-command.mjs +49 -0
  34. package/cli/business/setup-company.json +15 -0
  35. package/cli/business/setup-demo.mjs +95 -0
  36. package/cli/business/state.mjs +44 -10
  37. package/cli/business/supervisor.mjs +97 -32
  38. package/cli/business/vendor-door.mjs +230 -0
  39. package/cli/business/vendors/README.md +29 -3
  40. package/cli/business/vendors/common.mjs +8 -6
  41. package/cli/business/vendors/index.mjs +22 -11
  42. package/cli/business/vendors/sorry.mjs +106 -0
  43. package/cli/container-transfer.mjs +2 -2
  44. package/cli/contract.mjs +7 -7
  45. package/cli/demo-deploy-error.mjs +14 -0
  46. package/cli/demo-native.mjs +6 -2
  47. package/cli/deploy-command.mjs +6 -4
  48. package/cli/deploy-config.mjs +1 -1
  49. package/cli/deploy-owner-input.mjs +3 -1
  50. package/cli/deploy-variables.mjs +58 -0
  51. package/cli/deploy.mjs +9 -7
  52. package/cli/door-clauses.mjs +157 -0
  53. package/cli/errors.mjs +1 -1
  54. package/cli/export.mjs +16 -4
  55. package/cli/hosted-output.mjs +17 -7
  56. package/cli/human-error.mjs +20 -0
  57. package/cli/inspect.mjs +4 -0
  58. package/cli/jsonc.mjs +8 -4
  59. package/cli/lifecycle.mjs +26 -6
  60. package/cli/main.mjs +2 -1
  61. package/cli/managed-data-schema.mjs +127 -0
  62. package/cli/managed-data-zip.mjs +116 -0
  63. package/cli/managed-data.mjs +110 -0
  64. package/cli/managed-server.mjs +25 -14
  65. package/cli/native-owner-entry.mjs +86 -0
  66. package/cli/output.mjs +3 -2
  67. package/cli/parse.mjs +2 -2
  68. package/cli/portability-empty.mjs +65 -0
  69. package/cli/portability-migrations.mjs +33 -9
  70. package/cli/portability-policy.mjs +5 -2
  71. package/cli/portability-worker.mjs +15 -6
  72. package/cli/portability.mjs +30 -8
  73. package/cli/production-auth.mjs +1 -1
  74. package/cli/production-mail.mjs +8 -2
  75. package/cli/ready-summary.mjs +9 -0
  76. package/cli/run-transfers.mjs +37 -0
  77. package/cli/runtime-front-door.mjs +4 -4
  78. package/cli/server-proxy.mjs +4 -3
  79. package/cli/server-registry.mjs +13 -8
  80. package/cli/site.mjs +6 -1
  81. package/cli/transfers.mjs +17 -4
  82. package/edition-authoring/archive-source.mjs +59 -0
  83. package/edition-authoring/base-copies.mjs +147 -25
  84. package/edition-authoring/cli.mjs +1 -1
  85. package/edition-authoring/edition-migrations.mjs +7 -3
  86. package/edition-authoring/module-closure.mjs +2 -16
  87. package/edition-authoring/workflow-copies.mjs +1 -1
  88. package/edition-authoring/workflow-scripts.mjs +12 -3
  89. package/foundation/runtime/contract-docs.mjs +218 -34
  90. package/foundation/runtime/contract-prose-patterns.mjs +14 -0
  91. package/foundation/runtime/contract-prose.mjs +88 -0
  92. package/foundation/runtime/run-data.d.mts +1 -0
  93. package/foundation/runtime/run-data.mjs +2 -0
  94. package/global/CHANGELOG.md +126 -0
  95. package/global/README.md +156 -50
  96. package/global/cli/command.mjs +8 -2
  97. package/global/cli/install.mjs +4 -5
  98. package/global/cli/path-link.mjs +3 -2
  99. package/global/cli/repo-command.mjs +3 -7
  100. package/global/door/command.mjs +3 -1
  101. package/global/launch/dev.mjs +12 -2
  102. package/global/launch/facts.mjs +2 -2
  103. package/global/launch/render.mjs +16 -9
  104. package/global/launch/state.mjs +4 -4
  105. package/global/main.mjs +13 -8
  106. package/install.sh +17 -6
  107. package/modules/agent-door/CLAUSES.md +215 -0
  108. package/modules/agent-door/conformance/audit.mjs +66 -0
  109. package/modules/agent-door/conformance/authority.d.mts +2 -0
  110. package/modules/agent-door/conformance/authority.mjs +17 -0
  111. package/modules/agent-door/conformance/crypto.mjs +36 -0
  112. package/modules/agent-door/conformance/discovery.mjs +61 -0
  113. package/modules/agent-door/conformance/index.mjs +41 -0
  114. package/modules/agent-door/conformance/invocation.mjs +32 -0
  115. package/modules/agent-door/conformance/journey.mjs +112 -0
  116. package/modules/agent-door/conformance/narrowing.d.mts +1 -0
  117. package/modules/agent-door/conformance/narrowing.mjs +21 -0
  118. package/modules/agent-door/conformance/negative.mjs +47 -0
  119. package/modules/agent-door/conformance/network-modules.mjs +86 -0
  120. package/modules/agent-door/conformance/network-runtime.mjs +46 -0
  121. package/modules/agent-door/conformance/policy-phases.mjs +75 -0
  122. package/modules/agent-door/conformance/protocol.mjs +145 -0
  123. package/modules/agent-door/conformance/replay.mjs +38 -0
  124. package/modules/agent-door/conformance/schema.mjs +71 -0
  125. package/modules/agent-door/conformance/security.mjs +33 -0
  126. package/modules/agent-door/conformance/sponsors.mjs +68 -0
  127. package/modules/agent-door/conformance/surface.mjs +89 -0
  128. package/modules/agent-door/conformance/target.mjs +18 -0
  129. package/modules/agent-door/conformance/tool-name.d.mts +1 -0
  130. package/modules/agent-door/conformance/tool-name.mjs +7 -0
  131. package/modules/agent-door/conformance/webhooks.mjs +30 -0
  132. package/modules/container/README.md +6 -1
  133. package/modules/container/adapters/d1-limits.mjs +286 -0
  134. package/modules/container/adapters/d1-sqlite.mjs +95 -0
  135. package/modules/container/adapters/d1-workerd-preload.mjs +19 -0
  136. package/modules/container/adapters/d1-workerd-setup.mjs +26 -0
  137. package/modules/container/adapters/d1-workerd.mjs +5 -0
  138. package/modules/container/adapters/sqlite-worker.mjs +5 -3
  139. package/modules/container/build.mjs +1 -0
  140. package/modules/container/image-variables.mjs +14 -3
  141. package/modules/container/transfer.mjs +14 -4
  142. package/modules/container/worker-engine.mjs +3 -3
  143. package/modules/demo-host/README.md +6 -0
  144. package/modules/demo-host/build.mjs +1 -0
  145. package/modules/demo-host/native/prepare.mjs +2 -1
  146. package/modules/demo-host/native/sql.mjs +7 -2
  147. package/modules/demo-host/ownership.mjs +39 -0
  148. package/modules/demo-host/plan.mjs +2 -1
  149. package/modules/demo-host/remove.mjs +6 -1
  150. package/modules/deploy-kit/README.md +22 -2
  151. package/modules/deploy-kit/d1-import-diagnostics.mjs +56 -0
  152. package/modules/deploy-kit/d1-import.mjs +17 -5
  153. package/modules/deploy-kit/diagnostics.mjs +29 -3
  154. package/modules/deploy-kit/owner-cli.mjs +20 -7
  155. package/modules/deploy-kit/provider-transport.mjs +19 -4
  156. package/modules/deploy-kit/provider.mjs +57 -22
  157. package/modules/deploy-kit/readiness-window.mjs +1 -1
  158. package/modules/deploy-kit/site.mjs +8 -3
  159. package/modules/deploy-kit/worker.mjs +4 -4
  160. package/modules/import-kit/schema/importer-verification-receipt.schema.json +171 -0
  161. package/modules/import-kit/schema/importers.schema.json +107 -0
  162. package/modules/import-kit/src/d1/README.md +23 -4
  163. package/modules/import-kit/src/d1/atomic-prefix.ts +2 -2
  164. package/modules/import-kit/src/d1/budget.ts +15 -7
  165. package/modules/import-kit/src/d1/column-layout.ts +5 -0
  166. package/modules/import-kit/src/d1/creation.ts +15 -8
  167. package/modules/import-kit/src/d1/database.ts +3 -3
  168. package/modules/import-kit/src/d1/erasure-pages.ts +2 -1
  169. package/modules/import-kit/src/d1/erasure.ts +2 -2
  170. package/modules/import-kit/src/d1/jobs.ts +10 -6
  171. package/modules/import-kit/src/d1/plans.ts +14 -8
  172. package/modules/import-kit/src/d1/retirement.ts +2 -2
  173. package/modules/import-kit/src/d1/sets.ts +1 -3
  174. package/modules/import-kit/src/d1/staged-creation.ts +3 -3
  175. package/modules/import-kit/src/door/ADOPTING.md +20 -0
  176. package/modules/import-kit/src/door/README.md +2 -1
  177. package/modules/import-kit/src/door/admission.ts +5 -3
  178. package/modules/import-kit/src/door/archive.ts +5 -2
  179. package/modules/import-kit/src/door/authority.ts +1 -0
  180. package/modules/import-kit/src/door/grants.ts +1 -1
  181. package/modules/import-kit/src/door/handler.ts +14 -8
  182. package/modules/import-kit/src/door/index.ts +1 -1
  183. package/modules/import-kit/src/door/jobs.ts +40 -0
  184. package/modules/import-kit/src/door/limits.ts +1 -1
  185. package/modules/import-kit/src/door/operator.ts +3 -3
  186. package/modules/import-kit/src/door/planning.ts +10 -6
  187. package/modules/import-kit/src/door/protocol.ts +2 -2
  188. package/modules/import-kit/src/door/raw.ts +25 -0
  189. package/modules/import-kit/src/door/refusals.ts +2 -1
  190. package/modules/import-kit/src/door/types.ts +26 -6
  191. package/modules/import-kit/src/receipt/encoding.ts +37 -0
  192. package/modules/import-kit/src/receipt/index.ts +69 -0
  193. package/modules/import-kit/src/receipt/policy.ts +24 -0
  194. package/modules/import-kit/src/receipt/types.ts +64 -0
  195. package/modules/import-kit/src/receipt/uri.ts +45 -0
  196. package/modules/import-kit/src/receipt/validation.ts +87 -0
  197. package/modules/import-kit/src/status/encoding.ts +13 -0
  198. package/modules/import-kit/src/status/evidence.ts +218 -0
  199. package/modules/import-kit/src/status/index.ts +56 -0
  200. package/modules/import-kit/src/status/schema.ts +208 -0
  201. package/modules/import-kit/src/status/types.ts +86 -0
  202. package/modules/operator-auth/accounts/delegated.mjs +48 -0
  203. package/modules/operator-auth/index.d.mts +7 -0
  204. package/modules/operator-auth/service.mjs +2 -0
  205. package/modules/portability/README.md +138 -9
  206. package/modules/portability/backfills.mjs +9 -8
  207. package/modules/portability/bridge.mjs +32 -5
  208. package/modules/portability/catalogue.mjs +12 -5
  209. package/modules/portability/copies.mjs +8 -1
  210. package/modules/portability/derived.mjs +33 -0
  211. package/modules/portability/files.mjs +4 -3
  212. package/modules/portability/http-errors.mjs +1 -1
  213. package/modules/portability/http.mjs +3 -3
  214. package/modules/portability/import.mjs +29 -13
  215. package/modules/portability/index.d.mts +4 -3
  216. package/modules/portability/index.mjs +1 -1
  217. package/modules/portability/ledgers.mjs +3 -2
  218. package/modules/portability/legacy-safety.mjs +11 -2
  219. package/modules/portability/migration-history.mjs +2 -2
  220. package/modules/portability/progress.mjs +1 -0
  221. package/modules/portability/restore-empty.mjs +12 -0
  222. package/modules/release/github-verify.mjs +11 -2
  223. package/onboarding/adapter.mjs +35 -6
  224. package/onboarding/app-fetch.mjs +6 -4
  225. package/onboarding/development-runtime.mjs +19 -0
  226. package/onboarding/local/operation.mjs +45 -27
  227. package/onboarding/loopback-fetch.mjs +16 -0
  228. package/onboarding/operator-installer.mjs +2 -4
  229. package/onboarding/process-diagnostics.mjs +6 -2
  230. package/onboarding/production-runtime.mjs +4 -2
  231. package/onboarding/readiness.mjs +1 -0
  232. package/package.json +4 -2
  233. package/scripts/publish-check/copy_identities.py +3 -1
  234. package/scripts/publish-check/github_refs.py +32 -13
  235. package/scripts/publish-check/public_hygiene.py +1 -1
  236. package/cli/business/opt-ins.mjs +0 -33
@@ -15,8 +15,8 @@ import { transientTables } from './transient-tables.mjs';
15
15
  // - An eligible seed or backfill of a catalogue table replays at finalize.
16
16
  // A carried table keeps its archived rows and admits only one later writer,
17
17
  // naming its target columns. An absent table is reset first; each later write is a
18
- // replayable INSERT: they run in migration order, exactly as written, after
19
- // the archived rows are inserted and with the destination's triggers still
18
+ // replayable INSERT or a bounded UPDATE of a newly reconstructed table. They
19
+ // run in migration order, exactly as written, after archived rows are inserted and with the destination's triggers still
20
20
  // dropped, so the table holds what the live upgrade gave it. A replay may read
21
21
  // only archived tables (complete by then), views over those tables and the
22
22
  // table itself, by named
@@ -56,11 +56,12 @@ export function laterData({applied,unapplied,replay,catalogue,archived,empty,tab
56
56
  // Seed INSERTs into a new table replay too: the rows already present in a
57
57
  // fresh destination must be reset and derived from the restored installation.
58
58
  const transient=transients.get(table);
59
- // A transient fill can also read new catalogue tables whose INSERTs the
60
- // same plan reconstructs (or whose reset leaves them empty at this point).
59
+ // A new target can read other new catalogue tables that the same plan
60
+ // reconstructs. Carried targets still read only archived tables.
61
61
  // Dependencies are checked below before any of the plan is selected.
62
- const available=[...transients.keys(),...(transient?[...writes.keys()].filter(name=>!carried.includes(name)):[])];
63
- const options={table,statements,restorable,writable,carried:[...carried,...available],tables:known,secrets,views,manifest};
62
+ const available=[...transients.keys(),...(!carried.includes(table)?[...writes.keys()].filter(name=>!carried.includes(name)):[])];
63
+ const allowUpdate=!transient && !carried.includes(table) && list.some(s=>['insert','replace'].includes(s.data.verb));
64
+ const options={table,allowUpdate,statements,restorable,writable,carried:[...carried,...available],tables:known,secrets,views,manifest};
64
65
  // Carried target rules must use the actual archive, not reconstructed reads.
65
66
  const bad=carried.includes(table) && list.length>1?list:list.filter(s=>!replayable(s,{...options,targetCarried:carried.includes(table)}));
66
67
  // Writes confined to tables created after the archive retain their
@@ -95,8 +96,8 @@ export function laterData({applied,unapplied,replay,catalogue,archived,empty,tab
95
96
  return {migration:s.migration,statement:s.index,table:named[table]??table,sql:s.sql,...(carried.includes(table)?{carried:true}:{}),...(transients.has(table)?{transient:true}:{}),...(schema.has(s)?{schema:true}:{})};
96
97
  })};
97
98
  }
98
- function replayable({data},{table,statements,restorable,writable,carried,targetCarried,tables,secrets,views,manifest}) {
99
- if(!['insert','replace'].includes(data.verb) || data.star)return false;
99
+ function replayable({data},{table,allowUpdate,statements,restorable,writable,carried,targetCarried,tables,secrets,views,manifest}) {
100
+ if(!['insert','replace',...(allowUpdate?['update']:[])].includes(data.verb) || data.star)return false;
100
101
  if(targetCarried) {
101
102
  // sqlite_sequence's high-water mark is not archived, so replay cannot
102
103
  // reproduce a carried AUTOINCREMENT table's live key allocation.
@@ -22,7 +22,7 @@ export async function appliedMigrations(db) {
22
22
  // Restored rows never pass through a later migration, so one that writes or
23
23
  // rewrites restorable data is refused, whether or not it changes the schema,
24
24
  // unless its statement is replayed at finalize (backfills.mjs).
25
- async function placement(db,manifest,cat,replay,ledger,history) {
25
+ async function placement(db,manifest,cat,replay,ledger,history,legacyRecovery,unverified=false) {
26
26
  const files=replay.names.map(name=>({name,sha256:replay.sha256?.(name),aliases:replay.aliases?.(name)}));
27
27
  // An aligned replayer keeps its existing contract: hashes are optional.
28
28
  // Resolve destination aliases only when its ledger uses different names.
@@ -32,10 +32,17 @@ async function placement(db,manifest,cat,replay,ledger,history) {
32
32
  const archived=history?.();
33
33
  const historyEntries=manifest.migrations?.map(name=>archived?.find(entry=>entry.name===name) ?? {name}) ?? archived;
34
34
  const recorded=historyEntries && matchMigrationLedger(historyEntries,files,replay);
35
+ // Only explicitly acknowledged v1 recovery may use the old projection for
36
+ // archive placement. The live destination check above remains complete, and
37
+ // current v2 imports use their authenticated object inventory. Verified legacy
38
+ // v2 uses the replayed complete identity; acknowledged recovery places by tables only.
39
+ const schemaAfter=names=>replay.schemaAfter(names,legacyRecovery && manifest.schemaObjects===undefined?{legacyExcludedObjects:true}:unverified?{tablesOnly:true}:undefined);
40
+ if(unverified && archived && manifest.migrations)insist(canonicalJson(matchMigrationLedger(archived,files,replay))===canonicalJson(recorded),'schema_mismatch:migration_history',409);
35
41
  let earlier,unapplied;
36
42
  if(recorded) {
43
+ if(unverified)insist(recorded.every((name,i)=>applied.indexOf(name)>(i?applied.indexOf(recorded[i-1]):-1)),'schema_mismatch:migration_history',409);
37
44
  earlier=applied.filter(name=>recorded.includes(name));
38
- insist(await replay.schemaAfter(earlier)===manifest.schema,'schema_mismatch');
45
+ insist(await schemaAfter(earlier)===manifest.schema,'schema_mismatch');
39
46
  unapplied=applied.filter(name=>!earlier.includes(name));
40
47
  }else{
41
48
  // Unrecorded: the latest matching prefix. Migrations that change no archived
@@ -43,7 +50,7 @@ async function placement(db,manifest,cat,replay,ledger,history) {
43
50
  // and count as later. The source may have stopped at any matching prefix,
44
51
  // so every migration after the earliest one may be missing from its rows.
45
52
  const matching=[];
46
- for(let k=0;k<=applied.length;k++)if(await replay.schemaAfter(applied.slice(0,k))===manifest.schema)matching.push(k);
53
+ for(let k=0;k<=applied.length;k++)if(await schemaAfter(applied.slice(0,k))===manifest.schema)matching.push(k);
47
54
  insist(matching.length,'schema_mismatch');
48
55
  earlier=applied.slice(0,matching.at(-1));
49
56
  unapplied=applied.slice(matching[0]);
@@ -62,6 +69,26 @@ async function placement(db,manifest,cat,replay,ledger,history) {
62
69
  const {replays,rebuilt}=laterData({applied,unapplied,replay,catalogue,archived:archivedTables,empty,tables,secrets:manifest.excludedColumns??{},manifest:manifest.tables});
63
70
  return {applied,earlier,replays,rebuilt};
64
71
  }
72
+ // Old v2 writers recorded tables and applied migration names, but no behavioral
73
+ // object inventory. Verify both projections: the recorded files must reproduce
74
+ // the complete live schema, and their tables must reproduce the archive. Never
75
+ // guess an unrecorded prefix or install source SQL to fill missing evidence.
76
+ export async function verifiedLegacySchema(db,manifest,cat,migrations,history) {
77
+ if(!migrations)return false;
78
+ const replay=await migrations(),ledger=await destinationLedger(db)??[];
79
+ try {
80
+ const files=replay.names.map(name=>({name,sha256:replay.sha256?.(name),aliases:replay.aliases?.(name)}));
81
+ const archived=history(),entries=manifest.migrations?.map(name=>archived?.find(entry=>entry.name===name)??{name})??archived;
82
+ if(!entries?.length)return false;
83
+ const applied=matchMigrationLedger(ledger,files,replay),recorded=matchMigrationLedger(entries,files,replay);
84
+ if(!recorded.length || canonicalJson(applied)!==canonicalJson(replay.names)
85
+ || !recorded.every((name,i)=>applied.indexOf(name)>(i?applied.indexOf(recorded[i-1]):-1)))return false;
86
+ if(archived && manifest.migrations && canonicalJson(matchMigrationLedger(archived,files,replay))!==canonicalJson(recorded))return false;
87
+ return await replay.schemaAfter(applied)===cat.identity
88
+ && await replay.schemaAfter(recorded)===cat.identity
89
+ && await replay.schemaAfter(recorded,{tablesOnly:true})===manifest.schema;
90
+ }catch{return false;} // Unknown history or a failed replay is not verification.
91
+ }
65
92
  // An archive with the destination's schema identity can still predate later
66
93
  // migrations that changed only data (a purge or a backfill).
67
94
  // Those that change restorable data are refused like any other. Without a
@@ -85,10 +112,10 @@ export async function sameSchemaMigrations(db,manifest,cat,migrations,history) {
85
112
  // are additive, or a proven row-preserving CHECK rebuild: new tables, new
86
113
  // columns that take a default, or later schema objects established by replay.
87
114
  // Anything else stays a schema_mismatch refusal.
88
- export async function additiveBridge(db,manifest,cat,migrations,history) {
115
+ export async function additiveBridge(db,manifest,cat,migrations,history,legacyRecovery,unverified=false) {
89
116
  insist(migrations,'schema_mismatch');
90
117
  const replay=await migrations(),ledger=await destinationLedger(db)??[];
91
- const {applied,earlier,replays,rebuilt}=await placement(db,manifest,cat,replay,ledger,history);
118
+ const {applied,earlier,replays,rebuilt}=await placement(db,manifest,cat,replay,ledger,history,legacyRecovery,unverified);
92
119
  const columns={};
93
120
  for(const table of manifest.tables) {
94
121
  const target=cat.tables.find(t=>t.name===table.name),label=':'+table.name;
@@ -53,19 +53,26 @@ export function policy(input) {
53
53
  /** Platform tables are always excluded; explicit includes override only optional exclusions. */
54
54
  export const excludedTable=(table,input={})=>platformTable(table) || (!(input.includeTables??[]).includes(table) && [...excludedTables,...(input.excludeTables??[])].some(pattern=>matches(table,pattern)));
55
55
  async function archived(db,config) {
56
- const objects=await all(db,"SELECT name,sql,type,tbl_name FROM sqlite_master WHERE sql IS NOT NULL ORDER BY name");
56
+ const objects=await all(db,"SELECT name,sql,type,tbl_name FROM sqlite_master WHERE sql IS NOT NULL AND substr(lower(name),1,7) <> 'sqlite_' AND substr(lower(name),1,4) <> '_cf_' ORDER BY name");
57
57
  const kinds=await all(db,'PRAGMA table_list');
58
58
  const derived=config.derivedTables;
59
59
  const excluded=objects.filter(x=>x.type==='table' && (excludedTable(x.name,config) || (!config.includeTables.includes(x.name) &&
60
60
  derived.some(p=>x.name===p || (kinds.find(k=>k.name===x.name)?.type==='shadow' && x.name.startsWith(p+'_'))))));
61
61
  objects.sort((a,b)=>a.name<b.name?-1:a.name>b.name?1:a.type<b.type?-1:a.type>b.type?1:0);
62
- const schemaObjects=objects.filter(x=>['index','view','trigger'].includes(x.type)
63
- && !excluded.some(t=>t.name===x.tbl_name)).map(({name,type,tbl_name,sql})=>({name,type,table:tbl_name,sql}));
62
+ // Row exclusions never erase behavioral schema evidence, even for objects
63
+ // attached to excluded tables. Only literal engine-internal names are exempt.
64
+ const schemaObjects=objects.filter(x=>['index','view','trigger'].includes(x.type)).map(({name,type,tbl_name,sql})=>({name,type,table:tbl_name,sql}));
64
65
  return {objects,kinds,excluded,schemaObjects,entries:objects.filter(x=>x.type==='table' && !excluded.some(y=>y.name===x.name))};
65
66
  }
66
67
  // The archive schema identity alone, for databases that need not satisfy the policy.
67
- export async function schemaIdentity(db,config) {
68
- const {entries,schemaObjects}=await archived(db,config);return schemaIdentityOf(entries,schemaObjects);
68
+ export async function schemaIdentity(db,config,{legacyExcludedObjects=false,tablesOnly=false}={}) {
69
+ const {entries,excluded,schemaObjects}=await archived(db,config);
70
+ // Acknowledged v1 recovery places an old backup using its historical
71
+ // projection; acknowledged recovery of a version 2 archive without an object
72
+ // inventory compares tables only. Current archives always retain the
73
+ // complete object inventory.
74
+ const objects=tablesOnly?[]:legacyExcludedObjects?schemaObjects.filter(object=>!excluded.some(table=>table.name===object.table)):schemaObjects;
75
+ return schemaIdentityOf(entries,objects);
69
76
  }
70
77
  export async function catalogue(db,config) {
71
78
  const {objects,kinds,excluded,entries,schemaObjects}=await archived(db,config);
@@ -1,4 +1,4 @@
1
- import { all,statement,quote,insist } from './storage.mjs';
1
+ import { all,statement,quote,insist,PortabilityError } from './storage.mjs';
2
2
  import { projectedColumns,encodedRow,mapRow,bindCell,writableColumns } from './catalogue.mjs';
3
3
  import { canonicalJson, utf8 } from './hash.mjs';
4
4
  import { generatedCopySQL } from './generated.mjs';
@@ -43,6 +43,13 @@ export async function readCopy(db,table,copy,offset,limit,encoded=false) {
43
43
  return raw.map((row,i)=>({row,meta:meta[i]}));
44
44
  }
45
45
  export async function normalizeCopy(context,table,copy) {
46
+ try { return await normalize(context,table,copy); }
47
+ catch(error) {
48
+ if(error instanceof PortabilityError)throw error;
49
+ throw new PortabilityError('copy_normalization_failed',409);
50
+ }
51
+ }
52
+ async function normalize(context,table,copy) {
46
53
  const {db,config}=context,secrets=config.secretColumns[table.name]??[];
47
54
  // Generated reads must reflect exactly the base cells a restore will receive.
48
55
  if(table.columns.some(c=>c.generated)&&secrets.length)await db.prepare(`UPDATE ${quote(copy)} SET ${secrets.map(name=>quote(name)+'=NULL').join(',')}`).run();
@@ -0,0 +1,33 @@
1
+ import { all, quote } from './storage.mjs';
2
+ import { sqliteTokens, asciiFold } from './sqlite-tokens.mjs';
3
+
4
+ const value = token => {
5
+ const text = token?.text ?? '', delimiter = text[0];
6
+ return ['"', "'", '`', '['].includes(delimiter)
7
+ ? text.slice(1, -1).replaceAll(delimiter + delimiter, delimiter) : text;
8
+ };
9
+
10
+ // Only external-content FTS5 has SQLite's native rebuild contract: its rows
11
+ // come from the migrated content table. Other derived data remains seed-owned.
12
+ function externalFts5(sql) {
13
+ const tokens = sqliteTokens(sql), words = tokens.map(token => asciiFold(token.text));
14
+ if (words.slice(0, 3).join(' ') !== 'create virtual table') return false;
15
+ const using = words.indexOf('using');
16
+ if (using < 0 || asciiFold(value(tokens[using + 1])) !== 'fts5') return false;
17
+ let depth = 0;
18
+ for (let i = using + 2; i < tokens.length; i++) {
19
+ if (tokens[i].text === '(') depth++;
20
+ else if (tokens[i].text === ')') depth--;
21
+ else if (depth === 1 && ['(', ','].includes(tokens[i - 1]?.text)
22
+ && asciiFold(value(tokens[i])) === 'content' && tokens[i + 1]?.text === '=')
23
+ return value(tokens[i + 2]).length > 0;
24
+ }
25
+ return false;
26
+ }
27
+
28
+ export async function rebuildDerivedStatements(db, config) {
29
+ if (!config.derivedTables.length) return [];
30
+ const objects = await all(db, "SELECT name,sql FROM sqlite_master WHERE type='table' AND sql IS NOT NULL ORDER BY name");
31
+ return objects.filter(object => config.derivedTables.includes(object.name) && !config.includeTables.includes(object.name) && externalFts5(object.sql))
32
+ .map(object => db.prepare(`INSERT INTO ${quote(object.name)}(${quote(object.name)}) VALUES('rebuild')`));
33
+ }
@@ -60,7 +60,7 @@ export async function exportArchive(service, path, transient, progress) {
60
60
  if (id) await service.exports.abort(id);
61
61
  }
62
62
  }
63
- export async function importArchive(service, path, transient, journal, progress) {
63
+ export async function importArchive(service, path, transient, journal, progress, options={}) {
64
64
  let id, manifest, closing, complete = false, staged = 0;
65
65
  try {
66
66
  // Validate both complete archives before creating any restore state. An old
@@ -76,7 +76,7 @@ export async function importArchive(service, path, transient, journal, progress)
76
76
  const ledgers = ledger.ledgers();
77
77
  for await (const record of fileRecords(path)) {
78
78
  if (record.kind === 'manifest') {
79
- manifest = record; id = await service.imports.create(manifest, undefined, ledgers ? { ledgers } : {});
79
+ manifest = record; id = await service.imports.create(manifest, undefined, {...options,...(ledgers?{ledgers}:{})});
80
80
  const { skippedTables } = await service.imports.status(id);
81
81
  if (skippedTables?.length) progress?.({ phase: 'skipped', tables: skippedTables });
82
82
  }
@@ -89,6 +89,7 @@ export async function importArchive(service, path, transient, journal, progress)
89
89
  }
90
90
  progress?.({ phase: 'finishing', total: manifest.partCount });
91
91
  const status = await service.imports.finalize(id, { contentHash: closing.contentHash, archiveHash: closing.archiveHash, ...(later ? { laterJournal: later } : {}) }); complete = true;
92
- return { ...receipt(manifest, closing, transient), ...(status.bridged ? { bridged: status.bridged } : {}), ...(status.skippedTables ? { skippedTables: status.skippedTables } : {}), ...(status.safetyChanges ? { safetyChanges: status.safetyChanges } : {}) };
92
+ if(status.schemaUnverified)progress?.({phase:'schema-unverified',receipt:status.schemaUnverified});
93
+ return { ...receipt(manifest, closing, transient), ...(status.schemaUnverified?{schemaUnverified:status.schemaUnverified}:{}), ...(status.bridged ? { bridged: status.bridged } : {}), ...(status.skippedTables ? { skippedTables: status.skippedTables } : {}), ...(status.safetyChanges ? { safetyChanges: status.safetyChanges } : {}) };
93
94
  } finally { if (id && !complete) await service.imports.abort(id); }
94
95
  }
@@ -5,7 +5,7 @@ import { importRunningMetadata } from './import-lease.mjs';
5
5
  const publicCodes = new Set(`
6
6
  D1_RPC_TRANSPORT_LIMIT advance_limit archive_checksum archive_format archive_incomplete archive_json
7
7
  binary_checksum binary_inventory binary_order binary_size body_too_large bucket_inventory bucket_not_empty
8
- closing_missing column_inventory configuration_inventory confirmation_required content_checksum copy_inventory copy_rowid
8
+ closing_missing column_inventory configuration_inventory confirmation_required content_checksum copy_inventory copy_normalization_failed copy_rowid
9
9
  data_after_closing dev_data erased_row erasure_epoch excluded_destination_dependents excluded_foreign_key
10
10
  export_incomplete export_phase export_row_must_be_pure export_row_not_idempotent extra_part generated_sql
11
11
  import_applying import_complete import_lease_expired import_phase import_row_columns import_row_key import_row_must_be_pure
@@ -11,10 +11,10 @@ export async function boundedJson(request,limit) {
11
11
  try{return JSON.parse(new TextDecoder('utf-8',{fatal:true}).decode(bytes));}catch{throw new PortabilityError('invalid_json');}
12
12
  }
13
13
  const json=(value,status=200)=>new Response(JSON.stringify(value),{status,headers:{'Content-Type':'application/json','Cache-Control':'no-store'}});
14
- export async function createPortabilityHandler({db,buckets,policy,appOrigin,basePath='/api/portability',authorize,maxBodyBytes=MAX_LINE_BYTES}) {
14
+ export async function createPortabilityHandler({db,buckets,policy,migrations,appOrigin,basePath='/api/portability',authorize,maxBodyBytes=MAX_LINE_BYTES}) {
15
15
  insist(typeof authorize==='function' && new URL(appOrigin).origin===appOrigin,'invalid_handler_configuration');
16
16
  insist(/^\/[A-Za-z0-9/_-]+$/.test(basePath) && !basePath.endsWith('/') && Number.isSafeInteger(maxBodyBytes) && maxBodyBytes>0,'invalid_handler_configuration');
17
- const service=await createPortability({db,buckets,policy});
17
+ const service=await createPortability({db,buckets,policy,migrations});
18
18
  return async request=>{
19
19
  const url=new URL(request.url);
20
20
  if(url.pathname!==basePath && !url.pathname.startsWith(basePath+'/'))return null;
@@ -31,7 +31,7 @@ export async function createPortabilityHandler({db,buckets,policy,appOrigin,base
31
31
  if(collection==='imports' && request.method==='GET' && !id)return json({active:await api.active(actor)});
32
32
  if(request.method==='POST' && !id) {
33
33
  const body=await boundedJson(request,maxBodyBytes);
34
- if(collection==='imports'){insist(body.confirmed===true,'confirmation_required');return json({id:await api.create(body.manifest,actor)},201);}
34
+ if(collection==='imports'){insist(body.confirmed===true,'confirmation_required');return json({id:await api.create(body.manifest,actor,{ledgers:body.ledgers,acknowledgeUnverifiedSchemaObjects:body.acknowledgeUnverifiedSchemaObjects===true})},201);}
35
35
  return json({id:await api.create(actor)},201);
36
36
  }
37
37
  if(request.method==='GET' && id && !action)return json(await api.status(id));
@@ -13,26 +13,39 @@ import { fence, mergeJournal, tombstone } from './journal.mjs';
13
13
  import { writeObject } from './payloads.mjs';
14
14
  import { createCopy,sidecar,readCopy,restoreCopy } from './copies.mjs';
15
15
  import { excludedDependentGuards, migrationSeeded, seededCondition } from './restore-safety.mjs';
16
- import { additiveBridge, sameSchemaMigrations, appliedMigrations } from './bridge.mjs';
16
+ import { additiveBridge, sameSchemaMigrations, appliedMigrations, verifiedLegacySchema } from './bridge.mjs';
17
17
  import { replayStatements } from './backfills.mjs';
18
18
  import { checkArchivedLedgers, archivedHistory } from './ledgers.mjs';
19
19
  import { legacyEmptyGuards, legacyAuditStatements, requireLegacyEmpty } from './legacy-safety.mjs';
20
20
  import { checkReferences } from './references.mjs';
21
+ import { rebuildDerivedStatements } from './derived.mjs';
21
22
  // ledgers: the rows of each migration ledger table the archive carries, as the
22
23
  // caller read them from it ({table:rows}); uploadPart binds them to its parts.
23
- export async function createImport(context,manifest,actor='direct',legacyRecovery,ledgers) {
24
+ export async function createImport(context,manifest,actor='direct',legacyRecovery,ledgers,acknowledgeUnverifiedSchemaObjects=false) {
24
25
  const {db,config,buckets}=context;await checkManifest(manifest);checkArchivedLedgers(manifest,ledgers);
25
26
  insist(manifest.product===config.product,'wrong_product');
26
27
  // Explicit schema upgrades can reconcile rows, but cannot attest an interrupted deployment.
27
- await appliedMigrations(db);await state(db);
28
+ await appliedMigrations(db);
28
29
  const original=manifest;manifest=await restorableManifest(original);
29
30
  const skippedTables=original.tables.filter(table=>platformTable(table.name)).map(table=>table.name);
30
31
  const cat=await catalogue(db,config),upgrade=config.acceptedSchemas[original.schema];
31
- // Table SQL and migration names cannot establish missing trigger/view/index
32
- // definitions. Never infer that an old archive's absent inventory is empty.
33
- insist(manifest.schemaObjects!==undefined || legacyRecovery,'schema_mismatch:legacy_schema',409);
34
- const same=manifest.schema===cat.identity,history=()=>archivedHistory(original,ledgers);
35
- const {replays,...bridge}=upgrade?{}:same?(await sameSchemaMigrations(db,manifest,cat,context.migrations,history)??{}):await additiveBridge(db,manifest,cat,context.migrations,history);
32
+ const legacy=original.version===2 && manifest.schemaObjects===undefined;
33
+ // An explicit acknowledgment keeps its existing recovery semantics. Without
34
+ // it, legacy v2 needs positive replay evidence for the complete destination.
35
+ const unverified=legacy && acknowledgeUnverifiedSchemaObjects===true;
36
+ const verifyingLegacy=legacy && !unverified && !legacyRecovery;
37
+ const history=()=>archivedHistory(original,ledgers,{allowOlderShape:unverified || verifyingLegacy});
38
+ const verified=verifyingLegacy && await verifiedLegacySchema(db,manifest,cat,context.migrations,history);
39
+ insist(!legacy || legacyRecovery || unverified || verified,'schema_mismatch:legacy_schema',409);
40
+ if(verified)manifest={...manifest,schema:cat.identity,schemaObjects:cat.schemaObjects};
41
+ if(unverified)for(const table of manifest.tables) {
42
+ const target=cat.tables.find(t=>t.name===table.name);
43
+ insist(target,'schema_mismatch:table:'+table.name,409);
44
+ for(const column of table.columns)insist(target.columns.some(c=>canonicalJson(c)===canonicalJson(column)),'schema_mismatch:column:'+table.name+':'+column.name,409);
45
+ insist(table.dependencies.every(d=>target.dependencies.includes(d)),'schema_mismatch:dependencies:'+table.name,409);
46
+ }
47
+ const same=manifest.schema===cat.identity;
48
+ const {replays,...bridge}=unverified?await additiveBridge(db,manifest,cat,context.migrations,history,legacyRecovery,true):upgrade?{}:same?(await sameSchemaMigrations(db,manifest,cat,context.migrations,history)??{}):await additiveBridge(db,manifest,cat,context.migrations,history,legacyRecovery);
36
49
  // Finalize checks object references on the staged rows; a later migration
37
50
  // replayed into a reference table would publish rows that check never saw.
38
51
  const referenced=replays?.find(r=>Object.values(config.buckets).some(spec=>spec.objectReferences?.table===r.table));
@@ -49,6 +62,7 @@ export async function createImport(context,manifest,actor='direct',legacyRecover
49
62
  insist(canonicalJson(manifest.requiredConfiguration)===canonicalJson(config.requiredConfiguration),'configuration_inventory');
50
63
  for(const o of manifest.objects) insist(buckets[o.bucket] && config.buckets[o.bucket] && o.key.startsWith(config.buckets[o.bucket].prefix??''),'bucket_inventory');
51
64
  insist(canonicalJson(manifest.payloads.map(x=>x.name))===canonicalJson(Object.keys(config.extraPayloads).sort()),'payload_inventory');
65
+ if(unverified)await destinationState(context,{metadata:{targets:cat.tables,manifest:original}});
52
66
  await ensureImportLeases(db);
53
67
  const previous=(await state(db)).import_job;
54
68
  if(previous) {
@@ -56,10 +70,10 @@ export async function createImport(context,manifest,actor='direct',legacyRecover
56
70
  await abortImport(context,previous,{staleOnly:true});
57
71
  }
58
72
  const id=crypto.randomUUID(),copies=Object.fromEntries(targets.map((t,i)=>[t.name,'rer_portability_stage_'+id.replaceAll('-','')+'_'+i]));
59
- const row={id,kind:'import',actor,phase:'staging',completed:0,metadata:{manifest:original,...(skippedTables.length?{restoreManifest:manifest,skippedTables}:{}),copies,targets,triggers:cat.triggers,...(bridge.fromSchema?{bridge}:{}),...(replays?{replays}:{}),...(legacyRecovery?{legacyRecovery}: {}),...(ledgers?{ledgers}:{})}};
73
+ const row={id,kind:'import',actor,phase:'staging',completed:0,metadata:{manifest:original,...(unverified?{schemaUnverified:{reason:'acknowledged recovery of a version 2 archive without schemaObjects',sourceIdentity:original.sourceIdentity,exportedAt:original.exportedAt,acknowledgedAt:new Date().toISOString()}}:{}),...(skippedTables.length?{restoreManifest:manifest,skippedTables}:{}),copies,targets,triggers:cat.triggers,...(bridge.fromSchema?{bridge}:{}),...(replays?{replays}:{}),...(legacyRecovery?{legacyRecovery}: {}),...(ledgers?{ledgers}:{})}};
60
74
  try {
61
75
  await db.batch([guard(db,'(SELECT import_job FROM rer_portability_state WHERE id=1) IS NULL'),
62
- statement(db,'UPDATE rer_portability_state SET import_job=? WHERE id=1',id),
76
+ statement(db,'UPDATE rer_portability_state SET import_job=?,applying=? WHERE id=1',id,unverified?1:0),
63
77
  ...targets.flatMap(t=>createCopy(db,t,copies[t.name])),
64
78
  statement(db,'INSERT INTO rer_portability_jobs(id,kind,actor,phase,metadata) VALUES(?,?,?,?,?)',id,'import',actor,row.phase,canonicalJson(row.metadata)),createImportLease(db,id),db.prepare('DELETE FROM rer_portability_guard')]);
65
79
  }catch(e){const running=(await state(db)).import_job;if(running)throw await importRunning(db,running);throw e;}
@@ -208,14 +222,16 @@ async function finalizeLeasedImport(context,id,{contentHash,archiveHash,laterJou
208
222
  insist(hash.digest()===object.sha256,'object_write_checksum');
209
223
  }
210
224
  for(const payload of manifest.payloads)await config.extraPayloads[payload.name].consume(binaryStream(context,id,payload,binaryIndexes),payload);
225
+ const derived=await rebuildDerivedStatements(db,config);
211
226
  row.phase='complete';row.metadata.contentHash=digest;
227
+ if(row.metadata.schemaUnverified)row.metadata.schemaUnverified.archiveHash=archiveHash;
212
228
  await db.batch([db.prepare('PRAGMA defer_foreign_keys=ON'),...emptyGuards(context,row,seeded),...seeded.map(t=>guard(db,...seededCondition(t))),...dependentGuards,...legacyEmptyGuards(db,row),
213
229
  guard(db,'NOT EXISTS(SELECT 1 FROM rer_portability_epochs WHERE source=? AND epoch>?) AND NOT EXISTS(SELECT 1 FROM rer_portability_state WHERE source_identity=? AND epoch>?)',manifest.sourceIdentity,checked.epoch,manifest.sourceIdentity,checked.epoch),
214
230
  ...row.metadata.targets.map(t=>guard(db,`NOT EXISTS(SELECT 1 FROM ${quote(sidecar(row.metadata.copies[t.name]))} s JOIN rer_portability_journal j ON j.source=? AND j.tombstone=s.rer_portability_tombstone)`,manifest.sourceIdentity)),
215
231
  ...row.metadata.triggers.map(t=>db.prepare(`DROP TRIGGER ${quote(t.name)}`)),
216
232
  ...replaced.map(t=>db.prepare(`DELETE FROM ${quote(t)}`)),
217
233
  ...row.metadata.targets.flatMap(t=>restoreCopy(db,t,row.metadata.copies[t.name])),save(db,row),...replayStatements(db,row),
218
- ...row.metadata.triggers.map(t=>db.prepare(t.sql)),...mergeJournal(db,manifest,checked),...legacyAuditStatements(db,row),
234
+ ...row.metadata.triggers.map(t=>db.prepare(t.sql)),...derived,...mergeJournal(db,manifest,checked),...legacyAuditStatements(db,row),
219
235
  // Publication and removal of this import's scratch commit together. A lost
220
236
  // acknowledgement still finds a complete job with no staging left behind.
221
237
  ...dropCopyStatements(db,row),
@@ -224,14 +240,14 @@ async function finalizeLeasedImport(context,id,{contentHash,archiveHash,laterJou
224
240
  // D1 may have committed even when its acknowledgement failed.
225
241
  if((await job(db,id)).phase==='complete')return importStatus(context,id);
226
242
  if(!await context.importLease.owned())throw new (await import('./storage.mjs')).PortabilityError('import_lease_expired',409);
227
- await removeOwned(context,id);row.phase='staging';await db.batch([save(db,row),db.prepare('UPDATE rer_portability_state SET applying=0 WHERE id=1')]);throw e;
243
+ await removeOwned(context,id);row.phase='staging';await db.batch([save(db,row),statement(db,'UPDATE rer_portability_state SET applying=? WHERE id=1',row.metadata.schemaUnverified?1:0)]);throw e;
228
244
  }
229
245
  return importStatus(context,id);
230
246
  }
231
247
  // Upload receipts stay idempotent; the status door exposes the moving lease.
232
248
  export async function importStatus(context,id,{includeLease=true}={}) {
233
249
  const row=await job(context.db,id,'import');return {id,kind:'import',phase:row.phase,completed:row.completed,total:row.metadata.manifest.partCount,...(includeLease?await importLeaseStatus(context.db,row):{}),
234
- contentHash:row.metadata.contentHash??null,requiredConfiguration:row.phase==='complete'?row.metadata.manifest.requiredConfiguration:[],...(row.metadata.bridge?{bridged:row.metadata.bridge}:{}),...(row.metadata.skippedTables?{skippedTables:row.metadata.skippedTables}:{}),...(row.metadata.safetyChanges?{safetyChanges:row.metadata.safetyChanges}:{})};
250
+ contentHash:row.metadata.contentHash??null,...(row.phase==='complete' && row.metadata.schemaUnverified?{schemaUnverified:row.metadata.schemaUnverified}:{}),requiredConfiguration:row.phase==='complete'?row.metadata.manifest.requiredConfiguration:[],...(row.metadata.bridge?{bridged:row.metadata.bridge}:{}),...(row.metadata.skippedTables?{skippedTables:row.metadata.skippedTables}:{}),...(row.metadata.safetyChanges?{safetyChanges:row.metadata.safetyChanges}:{})};
235
251
  }
236
252
  export async function activeImport(context,actor='direct') {
237
253
  const id=(await state(context.db)).import_job;
@@ -39,11 +39,12 @@ export interface ArchivePolicy {
39
39
  }
40
40
  /** Actual changed rows grouped by table, column and safe replacement value; never original values. */
41
41
  export interface SafetyChange {table:string;column:string;value:Cell;rows:number}
42
- export interface Status {id:string;kind:'export'|'import';phase:'building'|'staging'|'applying'|'complete'|'aborted';completed:number;total:number|null;/** Import only: inactivity deadline and whether it has passed. */abortableAt?:string|null;stale?:boolean;/** Export only: bytes of the parts built so far. */bytes?:number;contentHash:string|null;requiredConfiguration?:string[];bridged?:Bridge;/** Platform bookkeeping ignored in an older archive. */skippedTables?:string[];/** Durable summary of bounded import normalization. */safetyChanges?:SafetyChange[]}
42
+ export interface SchemaUnverifiedReceipt {reason:string;sourceIdentity:string;exportedAt:string;acknowledgedAt:string;archiveHash:string}
43
+ export interface Status {schemaUnverified?:SchemaUnverifiedReceipt;id:string;kind:'export'|'import';phase:'building'|'staging'|'applying'|'complete'|'aborted';completed:number;total:number|null;/** Import only: inactivity deadline and whether it has passed. */abortableAt?:string|null;stale?:boolean;/** Export only: bytes of the parts built so far. */bytes?:number;contentHash:string|null;requiredConfiguration?:string[];bridged?:Bridge;/** Platform bookkeeping ignored in an older archive. */skippedTables?:string[];/** Durable summary of bounded import normalization. */safetyChanges?:SafetyChange[]}
43
44
  /** Later additive migrations an older archive was restored across. */
44
45
  export interface Bridge {fromSchema:string;toSchema:string;migrations:string[];tables:string[];columns:Record<string,string[]>;/** Later backfills of tables the archive lacks, replayed at finalize in this order; rows is null until the apply commits. */replayed?:{migration:string;statement:number;table:string;rows:number|null}[]}
45
46
  /** A runner's replay of the destination's applied migrations: the archive schema identity after a chosen set, and each migration's statements. */
46
- export interface MigrationReplay {names:string[];/** Seed-declared data-only Platform entries without migration files. */dataOnly?:string[];/** Names generated by the seed's deployment specification for this file. */aliases?(name:string):string[]|undefined;statements(name:string):string[];/** The migration file's SHA-256 hex, where the runner read the file. */sha256?(name:string):string|undefined;schemaAfter(names:string[]):Promise<string>}
47
+ export interface MigrationReplay {names:string[];/** Seed-declared data-only Platform entries without migration files. */dataOnly?:string[];/** Names generated by the seed's deployment specification for this file. */aliases?(name:string):string[]|undefined;statements(name:string):string[];/** The migration file's SHA-256 hex, where the runner read the file. */sha256?(name:string):string|undefined;schemaAfter(names:string[],options?:{tablesOnly?:boolean}):Promise<string>}
47
48
  /** Export job creation only; no record data, object keys or file paths. */
48
49
  export interface ExportEvent {id:string;/** Opaque caller id; the direct/CLI sentinel is normalized to null. */actor:string|null;scope:'installation'}
49
50
  export interface PortabilityOptions {
@@ -53,7 +54,7 @@ export interface PortabilityOptions {
53
54
  }
54
55
  export interface Portability {
55
56
  exports: {create(actor?:string):Promise<string>;advance(id:string,options?:{expectedPart?:number;maxParts?:number}):Promise<Status>;status(id:string):Promise<Status>;download(id:string):Promise<ReadableStream<Uint8Array>>;abort(id:string):Promise<Status>};
56
- imports: {active(actor?:string):Promise<Status|null>;/** ledgers: the rows of each migration ledger table an old archive carries, as read from it; they place it in the destination's history and must equal its parts. */create(manifest:Manifest,actor?:string,options?:{ledgers?:Record<string,Row[]>}):Promise<string>;uploadPart(id:string,index:number,part:Part):Promise<Status>;finalize(id:string,options:{archiveHash:string;contentHash?:string;laterJournal?:LaterJournal}):Promise<Status>;status(id:string):Promise<Status>;abort(id:string):Promise<Status>;/** Aborts the import left staged by a crashed restore and returns its id; call only as the destination's sole operator. */abortAbandoned():Promise<string|null>};
57
+ imports: {active(actor?:string):Promise<Status|null>;/** ledgers: the rows of each migration ledger table an old archive carries, as read from it; they place it in the destination's history and must equal its parts. */create(manifest:Manifest,actor?:string,options?:{ledgers?:Record<string,Row[]>;acknowledgeUnverifiedSchemaObjects?:boolean}):Promise<string>;uploadPart(id:string,index:number,part:Part):Promise<Status>;finalize(id:string,options:{archiveHash:string;contentHash?:string;laterJournal?:LaterJournal}):Promise<Status>;status(id:string):Promise<Status>;abort(id:string):Promise<Status>;/** Aborts the import left staged by a crashed restore and returns its id; call only as the destination's sole operator. */abortAbandoned():Promise<string|null>};
57
58
  status(id:string):Promise<Status>;actor(id:string):Promise<string>;
58
59
  erasureStatements(table:string,primaryKey:Record<string,Cell>):Promise<D1PreparedStatement[]>;
59
60
  redactionStatements():Promise<D1PreparedStatement[]>;
@@ -20,7 +20,7 @@ export async function createPortability({db,buckets={},policy:input,migrations,o
20
20
  const context={db,buckets,config,migrations,onExport};
21
21
  return {
22
22
  exports:{create:actor=>createExport(context,actor),advance:(id,options)=>advanceExport(context,id,options),status:id=>exportStatus(context,id),download:id=>downloadExport(context,id),abort:id=>abortExport(context,id)},
23
- imports:{active:actor=>activeImport(context,actor),create:(manifest,actor,options={})=>createImport(context,manifest,actor,undefined,options.ledgers),uploadPart:(id,index,part)=>uploadPart(context,id,index,part),finalize:(id,options)=>finalizeImport(context,id,options),status:id=>importStatus(context,id),abort:id=>abortImport(context,id),abortAbandoned:()=>abortAbandonedImport(context)},
23
+ imports:{active:actor=>activeImport(context,actor),create:(manifest,actor,options={})=>createImport(context,manifest,actor,undefined,options.ledgers,options.acknowledgeUnverifiedSchemaObjects),uploadPart:(id,index,part)=>uploadPart(context,id,index,part),finalize:(id,options)=>finalizeImport(context,id,options),status:id=>importStatus(context,id),abort:id=>abortImport(context,id),abortAbandoned:()=>abortAbandonedImport(context)},
24
24
  status:async id=>(await job(db,id)).kind==='export'?exportStatus(context,id):importStatus(context,id),
25
25
  actor:async id=>(await job(db,id)).actor,
26
26
  erasureStatements:(table,key)=>erasureStatements(db,table,key),redactionStatements:()=>redactionStatements(db),invalidateTransfersStatements:()=>invalidateTransfersStatements(db),
@@ -115,7 +115,7 @@ export function checkArchivedLedgers(manifest,ledgers) {
115
115
  }
116
116
 
117
117
  /** The applied migrations an archive's own ledger records, [{name,sha256?}]; undefined when it carries no ledger. */
118
- export function archivedHistory(manifest,ledgers) {
118
+ export function archivedHistory(manifest,ledgers,{allowOlderShape=false}={}) {
119
119
  const tables=archivedLedgerTables(manifest);
120
120
  // A Platform-origin export records its ledger in the manifest, never as a table.
121
121
  if(manifest.migrationLedger)insist(!tables.length,'schema_mismatch:migration_history');
@@ -123,7 +123,8 @@ export function archivedHistory(manifest,ledgers) {
123
123
  insist(tables.length===1 && ledgers?.[tables[0].name],'schema_mismatch:migration_history');
124
124
  const [table]=tables,rows=ledgers[table.name],columns=table.columns.map(c=>c.name);
125
125
  if(identifier(table.name)==='rer_migrations') {
126
- insist(shape(table.columns)===platformShape,'schema_mismatch:migration_history');
126
+ const older=canonicalJson([{name:'name',type:'TEXT',notnull:0,pk:1},{name:'applied_at',type:'TEXT',notnull:1,pk:0}]);
127
+ insist(shape(table.columns)===platformShape || (allowOlderShape && shape(table.columns)===older),'schema_mismatch:migration_history');
127
128
  insist(!rows.some(row=>partial(row.name)),'schema_mismatch:uncertain_migrations');
128
129
  }else{
129
130
  insist(columns.includes('name') && columns.includes('state')===columns.includes('sha256'),'schema_mismatch:migration_history');
@@ -1,5 +1,6 @@
1
1
  import { all, statement, guard, quote, insist } from './storage.mjs';
2
2
  import { canonicalJson } from './hash.mjs';
3
+ import { excludedTable } from './catalogue.mjs';
3
4
  import { migrationSeeded, seededCondition } from './restore-safety.mjs';
4
5
 
5
6
  // Recovery requires a new installation, including excluded identity tables and
@@ -22,14 +23,22 @@ function shadowTables(tables) {
22
23
  /** What the destination must still hold when the restore commits: `emptyTables` stay empty and
23
24
  * each of `seededTables` keeps exactly the rows its migrations put there. A table with other rows
24
25
  * is listed as to-be-empty, so the preflight refuses it. */
25
- export async function legacyTargetState(db,migrations) {
26
+ export async function legacyTargetState(db,migrations,include=()=>true) {
26
27
  const tables=await all(db,"SELECT name,sql FROM sqlite_master WHERE type='table' AND sql IS NOT NULL ORDER BY name"),shadows=shadowTables(tables);
27
- const names=tables.map(t=>t.name).filter(name=>!bookkeeping(name) && !shadows.has(name.toLowerCase()));
28
+ const names=tables.map(t=>t.name).filter(name=>!bookkeeping(name) && !shadows.has(name.toLowerCase()) && include(name));
28
29
  const filled=[];
29
30
  for(const name of names)if(await statement(db,`SELECT 1 FROM ${quote(name)} LIMIT 1`).first())filled.push(name);
30
31
  const seeded=await migrationSeeded(db,filled,migrations);
31
32
  return {emptyTables:names.filter(name=>!seeded.some(s=>s.name===name)),seededTables:seeded};
32
33
  }
34
+ // Before a runner applies pending migrations, use the normal product emptiness
35
+ // rule against the existing tables. Identity exclusions and replaceable rows
36
+ // retain their normal meaning; only the already-applied history can vouch for seeds.
37
+ export async function requireProductEmpty({db,policy,migrations}) {
38
+ const {emptyTables}=await legacyTargetState(db,migrations,name=>!excludedTable(name,policy)
39
+ && !(policy.derivedTables??[]).includes(name) && !(policy.replaceableTables??[]).includes(name));
40
+ for(const name of emptyTables)insist(!(await statement(db,`SELECT 1 FROM ${quote(name)} LIMIT 1`).first()),'target_not_empty:'+name,409);
41
+ }
33
42
  export async function requireLegacyEmpty({db,buckets},{emptyTables,seededTables=[]}) {
34
43
  for(const table of emptyTables)insist(!(await statement(db,`SELECT 1 FROM ${quote(table)} LIMIT 1`).first()),'legacy_target_not_empty:'+table,409);
35
44
  for(const table of seededTables){const [condition,...values]=seededCondition(table);insist((await statement(db,`SELECT CASE WHEN (${condition}) THEN 1 ELSE 0 END AS ok`,...values).first()).ok===1,'legacy_target_not_empty:'+table.name,409);}
@@ -8,12 +8,12 @@ import { policy, schemaIdentity } from './catalogue.mjs';
8
8
  // sha256 being the migration file's hash where the runner read the file.
9
9
  export function migrationReplay(plan,input,{dataOnly=[]}={}) {
10
10
  const config=policy(input);
11
- return {names:plan.map(migration=>migration.name),dataOnly,aliases:name=>plan.find(m=>m.name===name).aliases,statements:name=>plan.find(m=>m.name===name).statements,sha256:name=>plan.find(m=>m.name===name).sha256,async schemaAfter(names) {
11
+ return {names:plan.map(migration=>migration.name),dataOnly,aliases:name=>plan.find(m=>m.name===name).aliases,statements:name=>plan.find(m=>m.name===name).statements,sha256:name=>plan.find(m=>m.name===name).sha256,async schemaAfter(names,options) {
12
12
  const scratch=new DatabaseSync(':memory:');
13
13
  const db={prepare:sql=>{const query=scratch.prepare(sql);let values=[];return {bind(...args){values=args;return this;},async all(){return {results:query.all(...values)};}};}};
14
14
  try {
15
15
  for(const migration of plan.filter(m=>names.includes(m.name)))for(const sql of migration.statements)scratch.exec(sql);
16
- return await schemaIdentity(db,config);
16
+ return await schemaIdentity(db,config,options);
17
17
  }finally{scratch.close();}
18
18
  },
19
19
  // The rows the whole replay leaves in each requested table, as sorted images (legacy-safety.mjs).
@@ -39,6 +39,7 @@ export function transferProgress(report, { interval = 3000, now = () => performa
39
39
  }
40
40
  };
41
41
  return event => {
42
+ if(event.phase==='schema-unverified') { report('Schema unverified: '+JSON.stringify(event.receipt)); return; }
42
43
  if (event.phase === 'skipped') { report(skippedTablesLine(event.tables)); return; }
43
44
  if (event.phase === 'start') {
44
45
  const noun = event.direction === 'export' ? `Exporting about ${formatBytes(event.bytes)} of local data` : `Restoring a ${formatBytes(event.bytes)} archive`;
@@ -0,0 +1,12 @@
1
+ import { all, quote, insist } from './storage.mjs';
2
+ import { catalogue, policy } from './catalogue.mjs';
3
+ import { migrationSeeded } from './restore-safety.mjs';
4
+
5
+ // Runner preflight before creating a restore job. Finalize repeats its atomic
6
+ // empty-table guards; this early read keeps an occupied destination untouched.
7
+ export async function requireEmptyTables({ db, policy: input, migrations }) {
8
+ const config = policy(input), cat = await catalogue(db, config), filled = [];
9
+ for (const table of cat.tables) if (!config.replaceableTables.includes(table.name) && (await all(db, `SELECT 1 FROM ${quote(table.name)} LIMIT 1`)).length) filled.push(table.name);
10
+ const seeded = await migrationSeeded(db, filled, migrations);
11
+ for (const name of filled) insist(seeded.some(table => table.name === name), 'target_not_empty:' + name, 409);
12
+ }
@@ -1,6 +1,9 @@
1
1
  import { execFileSync } from 'node:child_process';
2
2
  import { git } from './source.mjs';
3
3
  import { verifyWorkflow } from './verify-workflow.mjs';
4
+ import { shared } from '../../cli/shared.mjs';
5
+
6
+ const { createRedaction } = await shared('redaction.mjs');
4
7
 
5
8
  const same = (left, right) => typeof left === 'string' && left.toLowerCase() === right.toLowerCase();
6
9
  function identity(value) {
@@ -101,9 +104,15 @@ export function githubVerify(source, repository, selector) {
101
104
  }
102
105
  if (tagUnavailable) throw Error('tag');
103
106
  result.details = rejected.size ? [...rejected].join('; ') : 'No runs found for this SHA.';
104
- } catch {
107
+ } catch (error) {
105
108
  result.reason = 'github-verify-unavailable';
106
- result.details = 'GitHub Verify could not be resolved. Check gh authentication and repository access, then try again; no local verification was substituted.';
109
+ const fallback = 'GitHub Verify could not be resolved. Check gh authentication and repository access, then try again; no local verification was substituted.';
110
+ const stderr = typeof error.stderr === 'string' || Buffer.isBuffer(error.stderr)
111
+ ? createRedaction().redact(error.stderr.toString()).trim() : '';
112
+ // Redact the complete diagnostic before bounding it, including long tokens.
113
+ const diagnostic = stderr ? stderr.slice(0, 8192) : fallback;
114
+ result.details = Number.isInteger(error.status)
115
+ ? `gh exited with status ${error.status}: ${diagnostic}` : diagnostic;
107
116
  }
108
117
  return result;
109
118
  }