@runeditrun/rer 0.2.99 → 0.2.100

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/cli/README.md +88 -2
  2. package/cli/business/bounded-waves.d.mts +23 -0
  3. package/cli/business/bounded-waves.mjs +57 -0
  4. package/cli/business/catchup-task.mjs +9 -2
  5. package/cli/business/command.mjs +8 -3
  6. package/cli/business/export-inputs.mjs +12 -2
  7. package/cli/business/export-task.mjs +9 -2
  8. package/cli/business/legacy-package.mjs +117 -0
  9. package/cli/business/local-commands.mjs +10 -1
  10. package/cli/business/package-transfer.mjs +127 -0
  11. package/cli/business/state.mjs +3 -3
  12. package/cli/business/vendor-door.mjs +83 -11
  13. package/cli/business/vendors/README.md +41 -2
  14. package/cli/business/vendors/common.mjs +81 -0
  15. package/cli/business/vendors/index.mjs +12 -2
  16. package/cli/business/vendors/zendesk-attachments.mjs +57 -0
  17. package/cli/business/vendors/zendesk-encode.mjs +133 -0
  18. package/cli/business/vendors/zendesk-guide.mjs +34 -0
  19. package/cli/business/vendors/zendesk-package.mjs +28 -0
  20. package/cli/business/vendors/zendesk-pages.mjs +61 -0
  21. package/cli/business/vendors/zendesk-people.mjs +7 -0
  22. package/cli/business/vendors/zendesk-store.mjs +154 -0
  23. package/cli/business/vendors/zendesk-tickets.mjs +21 -0
  24. package/cli/business/vendors/zendesk-token.mjs +31 -0
  25. package/cli/business/vendors/zendesk-transport.mjs +48 -0
  26. package/cli/business/vendors/zendesk.mjs +61 -0
  27. package/cli/deploy-command.mjs +5 -4
  28. package/cli/deploy-inputs.mjs +1 -1
  29. package/cli/deploy.mjs +6 -4
  30. package/cli/production-auth.mjs +16 -3
  31. package/cli/production-mail.mjs +18 -1
  32. package/cli/site.mjs +1 -1
  33. package/global/CHANGELOG.md +21 -0
  34. package/modules/deploy-kit/README.md +10 -0
  35. package/modules/deploy-kit/provider.mjs +18 -2
  36. package/modules/import-kit/src/d1/jobs.ts +3 -1
  37. package/modules/import-kit/src/d1/retirement-scope.ts +2 -2
  38. package/modules/import-kit/src/d1/retirement-tally.ts +1 -0
  39. package/modules/import-kit/src/d1/retirement.ts +15 -5
  40. package/modules/import-kit/src/d1/types.ts +3 -0
  41. package/modules/import-kit/src/d1/wave-boundary.ts +33 -0
  42. package/modules/import-kit/src/d1/wave-retirement.ts +47 -0
  43. package/modules/import-kit/src/door/ADOPTING.md +37 -0
  44. package/modules/import-kit/src/door/README.md +66 -0
  45. package/modules/import-kit/src/door/admission.ts +13 -3
  46. package/modules/import-kit/src/door/authority.ts +1 -0
  47. package/modules/import-kit/src/door/failures.ts +1 -1
  48. package/modules/import-kit/src/door/grants.ts +19 -5
  49. package/modules/import-kit/src/door/handler.ts +12 -1
  50. package/modules/import-kit/src/door/index.ts +6 -0
  51. package/modules/import-kit/src/door/package.ts +121 -0
  52. package/modules/import-kit/src/door/planning.ts +44 -19
  53. package/modules/import-kit/src/door/protocol.ts +4 -2
  54. package/modules/import-kit/src/door/schema.ts +2 -0
  55. package/modules/import-kit/src/door/store.ts +2 -2
  56. package/modules/import-kit/src/door/types.ts +24 -1
  57. package/modules/import-kit/src/door/waves/README.md +158 -0
  58. package/modules/import-kit/src/door/waves/digests.ts +38 -0
  59. package/modules/import-kit/src/door/waves/inventory.ts +84 -0
  60. package/modules/import-kit/src/door/waves/journal.ts +19 -0
  61. package/modules/import-kit/src/door/waves/lifecycle.ts +242 -0
  62. package/modules/import-kit/src/door/waves/preparation.ts +190 -0
  63. package/modules/import-kit/src/door/waves/schema.ts +73 -0
  64. package/modules/import-kit/src/door/waves/store.ts +112 -0
  65. package/modules/import-kit/src/door/waves/types.ts +76 -0
  66. package/onboarding/adapter.mjs +7 -1
  67. package/onboarding/operator-installer.mjs +3 -1
  68. package/package.json +1 -1
  69. package/scripts/publish-check/README.md +20 -0
  70. package/scripts/publish-check/diagnostics.mjs +73 -0
  71. package/scripts/publish-check/slots.py +37 -7
  72. package/scripts/publish-check.mjs +64 -37
@@ -0,0 +1,121 @@
1
+ import { Buffer } from 'node:buffer';
2
+ import { createHash } from 'node:crypto';
3
+ import { sha256 } from '../provenance/identity.ts';
4
+ import { DEFAULT_LIMITS, limitsFor, RecordCollector, safeValue } from '../readers/records.ts';
5
+ import type { ParsedRecord, SourceBundle } from '../types.ts';
6
+ import type { DoorPackageFile, DoorPackageManifest, DoorRequest, ImportDoorPackageUpload } from './types.ts';
7
+ import { DoorError } from './protocol.ts';
8
+ import { bundleBytes, CHUNK_BYTES, type UploadRow } from './uploads.ts';
9
+
10
+ const positive = (n: unknown): n is number => Number.isSafeInteger(n) && (n as number) > 0;
11
+ const count = (n: unknown): n is number => Number.isSafeInteger(n) && (n as number) >= 0;
12
+ const digest = (s: unknown): s is string => typeof s === 'string' && /^[a-f0-9]{64}$/.test(s);
13
+ const filename = (s: unknown): s is string => typeof s === 'string' && /^[A-Za-z0-9_][A-Za-z0-9_./-]*$/.test(s) && !s.split('/').some(p=>!p || p==='.' || p==='..');
14
+ const invalid = (): never => { throw new DoorError('invalid_package',422); };
15
+ const missing = (segments: {bytes: number; sha256: string}[]): never => { throw new DoorError('package_segment_missing',409,{missing:segments}); };
16
+ const json = (bytes: Uint8Array): any => { try { return JSON.parse(new TextDecoder('utf-8',{fatal:true}).decode(bytes)); } catch { return invalid(); } };
17
+
18
+ /** Resolve only verified complete segments in this installation AND namespace. No schema or grant changes. */
19
+ export async function packageUpload(request: DoorRequest, upload: UploadRow, resolved?: (rows: readonly UploadRow[]) => void): Promise<ImportDoorPackageUpload> {
20
+ const declaration = request.options.package;
21
+ if (!declaration) throw new DoorError('seed_package_unsupported',415);
22
+ const raw = await bundleBytes(request,upload), value = json(raw);
23
+ safeValue(value);
24
+ if (value?.format!=='rer-door-package/1' || !declaration.formats.includes(value.source?.format) || typeof value.mode!=='string'
25
+ || !positive(value.stream?.bytes) || value.stream.bytes > declaration.maxBytes || !digest(value.stream.sha256)
26
+ || !Array.isArray(value.segments) || !value.segments.length || value.segments.length > declaration.maxSegments
27
+ || value.index?.path!=='index.ndjson' || !positive(value.index.bytes) || value.index.bytes > request.access.maxBytes || !digest(value.index.sha256)
28
+ || !value.records || !(value.records.total===null || count(value.records.total)) || !value.records.byKind || typeof value.records.byKind!=='object' || Array.isArray(value.records.byKind)
29
+ || Object.values(value.records.byKind).some(n=>!count(n))) invalid();
30
+ const manifest = value as DoorPackageManifest;
31
+ let total = 0;
32
+ for (const [i,segment] of manifest.segments.entries()) {
33
+ if (!positive(segment?.bytes) || segment.bytes > request.access.maxBytes || Math.ceil(segment.bytes / CHUNK_BYTES) > request.access.maxChunks
34
+ || i < manifest.segments.length - 1 && segment.bytes !== request.access.maxBytes || !digest(segment.sha256)) invalid();
35
+ total += segment.bytes;
36
+ }
37
+ if (!Number.isSafeInteger(total) || total!==manifest.stream.bytes || manifest.index.bytes > total) invalid();
38
+ const rows: UploadRow[] = [], absent: {bytes:number;sha256:string}[] = [];
39
+ const streamHash = createHash('sha256');
40
+ for (const segment of manifest.segments) {
41
+ const candidates = await request.options.database.prepare(`SELECT * FROM ik_door_uploads WHERE installation_id=? AND namespace=? AND name='rer-segment' AND bundle_sha256=? AND total_bytes=? ORDER BY id`)
42
+ .bind(request.access.installationId,request.access.namespace,segment.sha256,segment.bytes).all<UploadRow>();
43
+ let complete: UploadRow | undefined, bytes: Uint8Array | undefined;
44
+ for (const row of candidates.results) {
45
+ if (row.total_chunks!==Math.ceil(segment.bytes / CHUNK_BYTES) || row.intake_json) continue;
46
+ try { bytes = await bundleBytes(request,row); complete = row; break; }
47
+ catch (error) { if (!(error instanceof DoorError) || !['upload_incomplete','upload_custody_mismatch','bundle_hash_mismatch'].includes(error.code)) throw error; }
48
+ }
49
+ if (!complete || !bytes) absent.push(segment);
50
+ else { rows.push(complete); streamHash.update(bytes); }
51
+ }
52
+ if (absent.length) missing(absent);
53
+ if (streamHash.digest('hex')!==manifest.stream.sha256) invalid();
54
+ // Planning owns these verified custody identities even if the index or reader refuses.
55
+ resolved?.(rows);
56
+ // Range reads materialize at most one segment plus the bounded requested file.
57
+ async function range(start: number, size: number): Promise<Uint8Array> {
58
+ const result = new Uint8Array(size); let offset = 0, written = 0;
59
+ for (let i = 0; i < rows.length && written < size; i++) {
60
+ const end = offset + manifest.segments[i].bytes;
61
+ if (end > start && offset < start + size) {
62
+ let bytes: Uint8Array;
63
+ try { bytes = await bundleBytes(request,rows[i]); } catch (error) { if (error instanceof DoorError && ['upload_incomplete','upload_custody_mismatch','bundle_hash_mismatch'].includes(error.code)) missing([manifest.segments[i]]); throw error; }
64
+ const from = Math.max(start,offset), to = Math.min(start+size,end);
65
+ result.set(bytes.subarray(from-offset,to-offset),from-start); written += to-from;
66
+ }
67
+ offset = end;
68
+ }
69
+ if (written!==size) invalid();
70
+ return result;
71
+ }
72
+ const indexBytes = await range(0,manifest.index.bytes);
73
+ if (sha256(indexBytes)!==manifest.index.sha256) invalid();
74
+ let indexText: string;
75
+ try { indexText = new TextDecoder('utf-8',{fatal:true}).decode(indexBytes); } catch { return invalid(); }
76
+ if (!indexText.endsWith('\n')) invalid();
77
+ const files: DoorPackageFile[] = [], names = new Set<string>(); let offset = 0;
78
+ for (const line of indexText.slice(0,-1).split('\n')) {
79
+ const row = json(Buffer.from(line));
80
+ if (!filename(row?.path) || row.path==='index.ndjson' || names.has(row.path) || !count(row.bytes) || !digest(row.sha256) || row.offset!==offset) invalid();
81
+ offset += row.bytes;
82
+ if (!Number.isSafeInteger(offset) || offset > total - manifest.index.bytes) invalid();
83
+ names.add(row.path); files.push(Object.freeze({path:row.path,bytes:row.bytes,sha256:row.sha256,offset:row.offset}));
84
+ }
85
+ if (!files.length || offset + manifest.index.bytes!==total) invalid();
86
+ // Detach the accessor's custody data from callback-owned metadata.
87
+ const exposed = JSON.parse(JSON.stringify(manifest)) as DoorPackageManifest;
88
+ const fileLimit = Math.min(declaration.maxBytes,request.options.limits?.maxBytes ?? DEFAULT_LIMITS.maxBytes);
89
+ return {manifest:exposed,files:Object.freeze(files),async read(path, maxBytes = fileLimit) {
90
+ const file = files.find(f=>f.path===path);
91
+ if (!file) throw new DoorError('package_file_missing',422);
92
+ if (!count(maxBytes) || file.bytes > Math.min(maxBytes,fileLimit)) throw new DoorError('package_file_exceeds_limit',413);
93
+ const bytes = await range(manifest.index.bytes+file.offset,file.bytes);
94
+ if (sha256(bytes)!==file.sha256) throw new DoorError('package_file_hash_mismatch',409);
95
+ return bytes;
96
+ }};
97
+ }
98
+
99
+ export async function readPackageSource(request: DoorRequest, upload: UploadRow, namespace: string): Promise<SourceBundle> {
100
+ const pkg = await packageUpload(request,upload), declaration = request.options.package!;
101
+ const limits = limitsFor({...request.options.limits,maxBytes:Math.min(declaration.maxBytes,request.options.limits?.maxBytes ?? DEFAULT_LIMITS.maxBytes),
102
+ maxRecords:Math.min(request.options.limits?.maxRecords ?? request.access.maxRecords,request.access.maxRecords)});
103
+ const expectedTotal = pkg.manifest.records.total, expectedKinds = {...pkg.manifest.records.byKind};
104
+ let records: ParsedRecord[];
105
+ try { records = await declaration.read(pkg,Object.freeze({...limits})); }
106
+ catch (error) { if (error instanceof DoorError) throw error; throw new DoorError('package_reader_failed',422); }
107
+ if (!Array.isArray(records)) invalid();
108
+ // Count the reader's aggregate before mapping or saving ANY plan, including when total is null (legacy).
109
+ if (records.length > request.access.maxRecords) throw new DoorError('plan_exceeds_grant_quota');
110
+ if (expectedTotal!==null && records.length!==expectedTotal) throw new DoorError('package_record_count_mismatch',422);
111
+ if (expectedTotal!==null) {
112
+ const actual: Record<string,number> = Object.create(null);
113
+ for (const record of records) actual[record.kind] = (actual[record.kind] ?? 0) + 1;
114
+ if (Object.keys(actual).length!==Object.keys(expectedKinds).length || Object.entries(actual).some(([kind,n])=>expectedKinds[kind]!==n)) throw new DoorError('package_record_count_mismatch',422);
115
+ }
116
+ const collector = new RecordCollector(namespace,limits);
117
+ collector.add(records,{artifact:upload.name},upload.bundle_sha256);
118
+ collector.bundle.artifacts.push({name:upload.name,sha256:upload.bundle_sha256,size:upload.total_bytes},
119
+ ...pkg.files.map(f=>({name:f.path,sha256:f.sha256,size:f.bytes})));
120
+ return collector.bundle;
121
+ }
@@ -1,3 +1,4 @@
1
+ import { packageUpload, readPackageSource } from './package.ts';
1
2
  import { readRawSource } from './raw.ts';
2
3
  import { randomUUID } from 'node:crypto';
3
4
  import { canonicalJson, sha256 } from '../provenance/identity.ts';
@@ -24,26 +25,36 @@ export function preview(snapshot: DoorSnapshot, created: boolean) {
24
25
  export async function plan(request: DoorRequest, input: Record<string, unknown>) {
25
26
  if (Object.keys(input).some(k=>!['uploadId','catchup'].includes(k)) || (input.catchup !== undefined && typeof input.catchup !== 'boolean')) throw new DoorError('invalid_plan_input',400);
26
27
  const upload = await uploadRow(request,text(input.uploadId,'upload_id'));
27
- try { return await planUpload(request,input,upload); }
28
+ const segments: UploadRow[] = [];
29
+ try {
30
+ if (upload.name === 'rer-package.json') await packageUpload(request,upload,rows=>segments.push(...rows));
31
+ return await planUpload(request,input,upload,segments);
32
+ }
28
33
  catch (error) {
29
34
  // A reader or bundle-check refusal answers a named 4xx, never a bare conflict.
30
35
  const refused = error instanceof DoorError ? error : sourceRefusal(error);
31
36
  // An upload holding a credential the reader cannot remove, or a value it
32
37
  // cannot inspect for one, never plans, so it is not kept.
33
38
  const unreadable = refused?.code === 'source_credential' || refused?.code === 'source_invalid_or_credential_source_data';
34
- if (unreadable) await forget(request,upload);
35
- if (unreadable || refused?.code === 'source_erased') throw refused;
39
+ if (unreadable || refused?.code === 'source_erased') await forget(request,upload,segments);
40
+ if (unreadable || refused?.code === 'source_erased' || refused?.code === 'package_segment_missing') throw refused;
36
41
  // Whatever refuses this plan first (a pending import, interrupted staging, a
37
42
  // withdrawn snapshot, the catch-up shape, a bound or the source itself), an
38
43
  // upload holding an erased record is refused for that instead, and goes. A
39
44
  // failing screen leaves the original refusal standing. No refused plan keeps
40
45
  // credential values the reader removes: their uploaded bytes go.
41
- const withheld = await screen(request,upload).catch(screened => { if (refused && screened instanceof DoorError && screened.code === 'source_erased') throw screened; return false; });
42
- if (withheld) await dropChunks(request,upload);
46
+ const withheld = await screen(request,upload,segments).catch(async screened => {
47
+ if (refused && screened instanceof DoorError && screened.code === 'source_erased') {
48
+ await forget(request,upload,segments);
49
+ throw screened;
50
+ }
51
+ return false;
52
+ });
53
+ if (withheld) await dropChunks(request,upload,segments);
43
54
  throw refused ?? error;
44
55
  }
45
56
  }
46
- async function planUpload(request: DoorRequest, input: Record<string, unknown>, upload: UploadRow) {
57
+ async function planUpload(request: DoorRequest, input: Record<string, unknown>, upload: UploadRow, segments: UploadRow[]) {
47
58
  const current = await lineage(request);
48
59
  // A plan whose apply failed, or whose import was undone or rolled back, has
49
60
  // left the lineage: the same bundle plans afresh instead of reusing it. A
@@ -67,7 +78,7 @@ async function planUpload(request: DoorRequest, input: Record<string, unknown>,
67
78
  if (snapshot.sourceCount > request.access.maxRecords) throw new DoorError('plan_exceeds_grant_quota');
68
79
  if (snapshot.plan.mappingVersion !== request.options.mapping.version) throw new DoorError('snapshot_mapping_changed_use_new_namespace',409);
69
80
  if (request.access.catchup === 'bounded' && (!snapshot.standingApproval || snapshot.standingApproval.id !== request.access.approvalRecordId || snapshot.standingApproval.at !== request.access.approvedAt)) throw new DoorError('standing_approval_mismatch');
70
- await dropCredentialBytes(request,snapshot,upload);
81
+ await dropCredentialBytes(request,snapshot,upload,segments);
71
82
  return preview(snapshot,false);
72
83
  }
73
84
  if (current.pending_id) throw new DoorError('namespace_has_pending_import',409);
@@ -85,7 +96,7 @@ async function planUpload(request: DoorRequest, input: Record<string, unknown>,
85
96
  if (!previous && (input.catchup === true || request.access.catchup === 'bounded')) throw new DoorError('catchup_requires_baseline',409);
86
97
  // A catch-up reads the destination's whole native set, which a large destination
87
98
  // refuses (413); an upload holding an erased record is refused and goes first.
88
- if (previous) await screen(request,upload);
99
+ if (previous) await screen(request,upload,segments);
89
100
  const planId = randomUUID(), options = request.options, {kitNamespace,source,retainedFrom} = await readUpload(request,upload,planId);
90
101
  const prior = new Map((previous?.records ?? []).map(r=>[recordKey(r),r]));
91
102
  const revisions = new Map<string,{revision:string;matches:boolean}>();
@@ -135,10 +146,10 @@ async function planUpload(request: DoorRequest, input: Record<string, unknown>,
135
146
  snapshot.records.push({...old,explanation:'Absent from source snapshot; retained in the seed. No deletion proposed.'});
136
147
  }
137
148
  if (request.access.catchup === 'bounded') assertBounded(snapshot,previous!);
138
- await refuseErased(request,upload,snapshot.plan);
149
+ await refuseErased(request,upload,snapshot.plan,segments);
139
150
  if (retired) await retire(request,retired,snapshot.plan);
140
151
  await saveSnapshot(request,snapshot,upload,retainedFrom);
141
- await dropCredentialBytes(request,snapshot,upload);
152
+ await dropCredentialBytes(request,snapshot,upload,segments);
142
153
  return preview(snapshot,true);
143
154
  }
144
155
  /** Retire an erasure-withdrawn live import as the namespace's baseline. The kit
@@ -160,11 +171,24 @@ async function retire(request: DoorRequest, head: DoorPlanRow, plan: ImportPlan)
160
171
  /** The reader removed credential values: the saved plan holds none and withholds
161
172
  * the original, so the uploaded bytes that still hold them go once it is saved.
162
173
  * The upload's header stays, so repeating this plan finds the saved one. */
163
- async function dropCredentialBytes(request: DoorRequest, snapshot: DoorSnapshot, upload: UploadRow): Promise<void> {
164
- if (withheldCredentials(snapshot.plan.source)) await dropChunks(request,upload);
174
+ async function dropCredentialBytes(request: DoorRequest, snapshot: DoorSnapshot, upload: UploadRow, segments: UploadRow[]): Promise<void> {
175
+ if (withheldCredentials(snapshot.plan.source)) await dropChunks(request,upload,segments);
165
176
  }
166
177
  const withheldCredentials = (source: SourceBundle) => source.artifacts.some(a=>a.original?.status==='unavailable' && a.original.reason==='credentials-removed');
167
- const dropChunks = (request: DoorRequest, upload: UploadRow) => transact(request.options.database,request.authority,[sql('DELETE FROM ik_door_chunks WHERE upload_id=?',upload.id)]);
178
+ /** Shared segment bytes cannot outlive a privacy withdrawal through another
179
+ * upload id. Match all equivalent copies in the authorized scope; manifests
180
+ * sharing them must request missing segments again. Foreign scopes stay intact. */
181
+ function cleanup(request: DoorRequest, upload: UploadRow, segments: readonly UploadRow[], headers: boolean) {
182
+ const {installationId,namespace} = request.access;
183
+ const matches = [sql('id=? AND installation_id=? AND namespace=?',upload.id,installationId,namespace),
184
+ ...segments.map(segment=>sql("installation_id=? AND namespace=? AND name='rer-segment' AND bundle_sha256=? AND total_bytes=?",installationId,namespace,segment.bundle_sha256,segment.total_bytes))];
185
+ return matches.flatMap(match=>[
186
+ sql(`DELETE FROM ik_door_chunks WHERE upload_id IN (SELECT id FROM ik_door_uploads WHERE ${match.sql})`,...match.params ?? []),
187
+ ...(headers ? [sql(`DELETE FROM ik_door_uploads WHERE ${match.sql}`,...match.params ?? [])] : []),
188
+ ]);
189
+ }
190
+ const dropChunks = (request: DoorRequest, upload: UploadRow, segments: readonly UploadRow[] = []) =>
191
+ transact(request.options.database,request.authority,cleanup(request,upload,segments,false));
168
192
  /** Each plan has its own kit namespace, so a replan after a failed apply never
169
193
  * meets the failed job's kit namespace. */
170
194
  async function readUpload(request: DoorRequest, upload: UploadRow, planId: string) {
@@ -174,6 +198,7 @@ async function readUpload(request: DoorRequest, upload: UploadRow, planId: strin
174
198
  const retained = await retainedSource(request,upload);
175
199
  if (retained) return {kitNamespace,retainedFrom:retained.planId,source:{...retained.source,records:retained.source.records.map(record=>({...record,namespace:kitNamespace}))}};
176
200
  // Credential values are removed per record, never imported, and never refuse a record.
201
+ if (upload.name === 'rer-package.json') return {kitNamespace,source:await readPackageSource(request,upload,kitNamespace),retainedFrom:undefined};
177
202
  const bytes = await bundleBytes(request,upload);
178
203
  const limits = {...options.limits,maxBytes:Math.min(options.limits?.maxBytes ?? request.access.maxBytes,request.access.maxBytes),maxRecords:Math.min(options.limits?.maxRecords ?? request.access.maxRecords,request.access.maxRecords)};
179
204
  const source = options.format === 'raw'
@@ -206,20 +231,20 @@ async function retainedSource(request: DoorRequest, upload: UploadRow): Promise<
206
231
  }
207
232
  /** The kit never stages an erased record, and neither does the door: an upload
208
233
  * holding one can never be planned, so it goes too. The epoch fences a later erasure. */
209
- export async function refuseErased(request: DoorRequest, upload: UploadRow, plan: ImportPlan): Promise<void> {
234
+ export async function refuseErased(request: DoorRequest, upload: UploadRow, plan: ImportPlan, segments: readonly UploadRow[] = []): Promise<void> {
210
235
  const db = request.options.database;
211
236
  for (const clear of erasedSourcePredicates(plan)) if ((await statement(db, sql(`SELECT ${clear.sql} AS clear`, ...clear.params ?? [])).first())?.clear !== 1) {
212
- await forget(request, upload);
237
+ await forget(request,upload,segments);
213
238
  throw new DoorError('source_erased', 409);
214
239
  }
215
240
  }
216
- const forget = (request: DoorRequest, upload: UploadRow) => transact(request.options.database, request.authority,
217
- [sql('DELETE FROM ik_door_chunks WHERE upload_id=?', upload.id), sql('DELETE FROM ik_door_uploads WHERE id=?', upload.id)]);
241
+ export const forget = (request: DoorRequest, upload: UploadRow, segments: readonly UploadRow[] = []) =>
242
+ transact(request.options.database,request.authority,cleanup(request,upload,segments,true));
218
243
  /** The upload planned on its own with the plain mapping: one holding an erased
219
244
  * record is refused and goes. Answers whether the reader removed credential values. */
220
- async function screen(request: DoorRequest, upload: UploadRow): Promise<boolean> {
245
+ async function screen(request: DoorRequest, upload: UploadRow, segments: readonly UploadRow[] = []): Promise<boolean> {
221
246
  const id = randomUUID(), {kitNamespace,source} = await readUpload(request,upload,id), mapping = request.options.mapping;
222
- await refuseErased(request,upload,await planD1Import(source,mapping,kitContext(request,{id,kitNamespace,plan:{mappingVersion:mapping.version} as ImportPlan})));
247
+ await refuseErased(request,upload,await planD1Import(source,mapping,kitContext(request,{id,kitNamespace,plan:{mappingVersion:mapping.version} as ImportPlan})),segments);
223
248
  return withheldCredentials(source);
224
249
  }
225
250
  export function assertBounded(snapshot: DoorSnapshot, previous: DoorSnapshot): void {
@@ -1,6 +1,7 @@
1
- export type DoorAction = 'uploads' | 'plan' | 'apply' | 'status' | 'reconcile' | 'undo' | 'cancel' | 'archive' | 'restore-check';
1
+ import type { WaveBinding } from './waves/types.ts';
2
+ export type DoorAction = 'uploads' | 'plan' | 'apply' | 'status' | 'reconcile' | 'undo' | 'cancel' | 'archive' | 'restore-check' | 'prepare' | 'seal-parent' | 'review-parent' | 'apply-child' | 'wave-status' | 'finalize-parent' | 'reverse-parent';
2
3
  /** Grant-only actions: the owner-session mount answers them 404. */
3
- export const GRANT_ONLY_ACTIONS: readonly DoorAction[] = ['cancel', 'archive', 'restore-check'];
4
+ export const GRANT_ONLY_ACTIONS: readonly DoorAction[] = ['cancel', 'archive', 'restore-check', 'prepare', 'seal-parent', 'review-parent', 'apply-child', 'wave-status', 'finalize-parent', 'reverse-parent'];
4
5
 
5
6
  export interface DoorGrant {
6
7
  v: 1;
@@ -20,6 +21,7 @@ export interface DoorGrant {
20
21
  approvalRecordId?: string;
21
22
  approvedAt?: string;
22
23
  catchup?: 'bounded';
24
+ bounded?: WaveBinding;
23
25
  }
24
26
 
25
27
  export class DoorError extends Error {
@@ -1,3 +1,4 @@
1
+ import { importDoorBoundedWavesMigration } from './waves/schema.ts';
1
2
  function planTables(suffix = ''): string[] {
2
3
  return [
3
4
  `CREATE TABLE IF NOT EXISTS ik_door_plans${suffix} (
@@ -98,4 +99,5 @@ export const importDoorSchema = [
98
99
  ...failureSchema,
99
100
  ...importDoorScopesMigration,
100
101
  ...importDoorArchiveMigration,
102
+ ...importDoorBoundedWavesMigration,
101
103
  ];
@@ -8,8 +8,8 @@ import { importDoorSchemaCheck } from './schema.ts';
8
8
  export const sql = (sql: string, ...params: unknown[]): D1Sql => ({ sql, params });
9
9
  export const statement = (db: D1Database, query: D1Sql) => db.prepare(query.sql).bind(...query.params ?? []);
10
10
  export const assertion = (query: D1Sql) => sql(`INSERT INTO ik_d1_assertions(ok) SELECT CASE WHEN (${query.sql}) THEN 1 ELSE 0 END`, ...query.params ?? []);
11
- export async function transact(db: D1Database, authority: D1Sql, queries: D1Sql[]): Promise<void> {
12
- await db.batch([assertion(authority), ...queries, assertion(authority), sql('DELETE FROM ik_d1_assertions')].map(q => statement(db, q)));
11
+ export async function transact(db: D1Database, authority: D1Sql, queries: D1Sql[], final: D1Sql[] = []): Promise<void> {
12
+ await db.batch([assertion(authority), ...queries, assertion(authority), sql('DELETE FROM ik_d1_assertions'), ...[authority,...final].map(predicate=>sql(`SELECT CASE WHEN (${predicate.sql}) THEN 1 ELSE json('IMPORT_ATOMIC_GUARD') END`,...predicate.params ?? []))].map(q => statement(db, q)));
13
13
  }
14
14
  export async function authorized(request: DoorRequest): Promise<void> {
15
15
  const row = await statement(request.options.database, sql(`SELECT CASE WHEN (${request.authority.sql}) THEN 1 ELSE 0 END AS ok`, ...request.authority.params ?? [])).first();
@@ -1,6 +1,7 @@
1
1
  import type { D1Database, D1Sql, D1TargetAdapter } from '../d1/types.ts';
2
2
  import type { D1ReversalPolicy } from '../d1/undo-types.ts';
3
3
  import type { ImportMapping, ImportPlan, Json, ParsedRecord, ReaderLimits } from '../types.ts';
4
+ import type { ImportDoorBoundedSource } from './waves/types.ts';
4
5
  import type { DoorGrant } from './protocol.ts';
5
6
  import type { ImportDoorIntake } from './intake/types.ts';
6
7
  export type { ImportDoorIntake, IntakeFormat } from './intake/types.ts';
@@ -12,8 +13,30 @@ export interface ImportDoorRawUpload {
12
13
  readonly sha256: string;
13
14
  readonly size: number;
14
15
  }
16
+ export interface DoorPackageFile { readonly path: string; readonly bytes: number; readonly sha256: string; readonly offset: number }
17
+ export interface DoorPackageManifest {
18
+ format: 'rer-door-package/1'; source: {format: string; [key: string]: Json}; mode: string;
19
+ stream: {bytes: number; sha256: string}; segments: {bytes: number; sha256: string}[];
20
+ index: {path: 'index.ndjson'; bytes: number; sha256: string};
21
+ records: {total: number | null; byKind: Record<string, number>}; totals: Json;
22
+ }
23
+ export interface ImportDoorPackageUpload {
24
+ readonly manifest: Readonly<DoorPackageManifest>;
25
+ readonly files: readonly Readonly<DoorPackageFile>[];
26
+ /** A verified file, bounded by the declaration and the caller's optional tighter limit. */
27
+ read(path: string, maxBytes?: number): Promise<Uint8Array>;
28
+ }
29
+ export interface ImportDoorPackageSource {
30
+ formats: readonly string[];
31
+ maxBytes: number;
32
+ maxSegments: number;
33
+ /** Return the whole logical snapshot, never one plan per segment or record wave. */
34
+ read(upload: ImportDoorPackageUpload, limits: Readonly<Required<ReaderLimits>>): ParsedRecord[] | Promise<ParsedRecord[]>;
35
+ }
15
36
  /** One decoded source; existing adapters may keep the mount's format. */
16
37
  export interface ImportDoorSource {
38
+ package?: ImportDoorPackageSource;
39
+ bounded?: ImportDoorBoundedSource;
17
40
  mapping: ImportMapping;
18
41
  parse(value: Json, index: number): ParsedRecord | ParsedRecord[];
19
42
  ownerPrincipalId?: string;
@@ -72,7 +95,7 @@ export type ImportDoorOptions = ImportDoorMountOptions & (
72
95
  parse?: undefined; mapping?: undefined; readRaw?: undefined;
73
96
  });
74
97
  export type DoorAccess = Pick<DoorGrant, 'action' | 'installationId' | 'email' | 'namespace'
75
- | 'maxBytes' | 'maxChunks' | 'maxRecords' | 'fingerprint' | 'approvalRecordId' | 'approvedAt' | 'catchup'>;
98
+ | 'bounded' | 'maxBytes' | 'maxChunks' | 'maxRecords' | 'fingerprint' | 'approvalRecordId' | 'approvedAt' | 'catchup'>;
76
99
  export interface ImportOperatorOptions extends ImportDoorRuntimeOptions {
77
100
  installationId: string;
78
101
  appOrigin: string;
@@ -0,0 +1,158 @@
1
+ # Opt-in complete-parent imports
2
+
3
+ The bounded-waves protocol shown below is a grant-only source protocol. Ordinary `/plan`, including
4
+ the logical-package path, keeps its existing aggregate refusal and quotas.
5
+ Transport segments never grant execution authority. This source change does not
6
+ adopt the protocol in an edition, raise a grant, approve an aggregate, or authorize
7
+ a real import. An ordinary whole snapshot above its record grant continues to refuse.
8
+
9
+ Apply `importDoorBoundedWavesMigration` in a new numbered edition migration after
10
+ the existing door migrations. The maintained reference includes the additive
11
+ `0042_import_bounded_waves.sql`; it supplies no bounded source or Business tasks.
12
+ Existing jobs, originals, approvals and migration names stay intact. The schema
13
+ adds root, ordered child, hashed membership and approval tables. Source content
14
+ uses the existing door snapshot chunks, and original/attachment bytes use the
15
+ ordinary D1 job custody. Erasure withdraws snapshots and equivalent authorized
16
+ package segments, marks their roots withdrawn, and retires affected root journals
17
+ with the existing stream/reuse/history closure. Peer native rows are retained
18
+ unless the product's erasure statements select them. Withdrawal is not Undo.
19
+
20
+ The edition supplies `ImportDoorBoundedSource` from `types.ts` as `source.bounded`
21
+ alongside its existing `source.package`. This is an actual typed input boundary;
22
+ no HappyToHelp reader, parser, native grouping or Business approval implementation
23
+ is supplied here. Those upstream integrations must exist and be independently
24
+ qualified before adoption. The ordinary whole-array package `read` is not used
25
+ by preparation.
26
+
27
+ `describe(upload)` independently enumerates the complete source and returns one
28
+ finite `WaveCensus`, including source/native counts by kind/table, originals,
29
+ objects, logical/stored/native/blob/context bytes, accounting version, source
30
+ mode and high water, and complete inventory/relationship/baseline digests.
31
+ `page(upload, ordinal, limits)` returns exactly one indivisible `WaveUnit` with
32
+ all its primary identities, file roles and dependencies on earlier units. The
33
+ adapter must include metadata/context/archive/rejected aliases in its reader's
34
+ erasure recognition, and coalesce cycles and publication closures before paging.
35
+ Files, source identities, additional subject aliases and native projection keys each have exactly one owner.
36
+ `aliases` explicitly indexes context, archive, rejected and metadata subjects that
37
+ are not primary source units; their files must be in the verified package. The
38
+ census reports their separate count and its inventory digest includes every
39
+ alias. The server-owned erasure adapter can supply exact
40
+ `D1ImportPerson.aliases: [{installationId, namespace, kind, sourceId}]`, including
41
+ a subject that never became a native target. It persists scoped digest tombstones
42
+ even before preparation. These selectors are never client input. Wave erasure uses the journaled
43
+ four-argument `retireD1ImportCustody` API and the actual restore checkpoint; the
44
+ separate legacy `operationKey` erasure-step interface refuses wave custody with
45
+ `IMPORT_WAVE_JOURNAL_REQUIRED`. The journal API retains its existing 1000-statement
46
+ and 2 MiB RPC bounds.
47
+ An unsupported indivisible unit refuses; no remainder is silently deferred.
48
+
49
+ `native(plan)` enumerates every actual expanded native row, including joins,
50
+ origin/hold rows and native blob chunks, with its stable key, projection digest
51
+ and stored/blob byte sizes. Its totals cannot be smaller than the kit effects.
52
+ It must describe the installed native adapter, including unchanged reused rows;
53
+ an effect/container is not a substitute record unit. The native adapter's
54
+ `read`, exact `matches` and full-set predicates must cover those physical rows,
55
+ references and byte representations. Kit protocol/journal metadata is separate.
56
+ `batchSize` optionally lowers the ordinary engine batch (1..20), without changing
57
+ the independent source and expanded-native execution quotas.
58
+
59
+ `waveInventoryHash({source,native,files})`, exported from the door, defines the
60
+ complete census digest. Source rows supply kind, stable ID and payload hash;
61
+ native rows supply the full `WaveNativeRow`, and `aliases` supplies additional
62
+ `WaveAlias` records; verified files also supply their
63
+ role and logical bytes. `inventoryMember(type, identity, content)` exposes each
64
+ hashed leaf. Sort its hexadecimal digest leaves in ascending ASCII order and
65
+ hash their canonical JSON array. The server computes this from its immutable
66
+ membership index with bounded keyset pages, without building a full inventory
67
+ JSON value. An adapter can do the same in its paged reader or use the convenience
68
+ function for a small complete census.
69
+
70
+ `relationshipsHash` hashes the ordered array of each unit's relationship digest.
71
+ `baselineHash` is the previous complete parent fingerprint, or SHA-256 of
72
+ canonical `[]` for the initial snapshot. Each child separately commits its exact
73
+ native before-images. The sealed parent commits all child fingerprints, census,
74
+ package, destination/claim, migration/revision/policy, mapping and erasure epoch.
75
+
76
+ `authority(binding, action)` MUST return a same-database scalar predicate for the
77
+ CURRENT installation, owner/claim, migration/revision/policy and active exact
78
+ parent/child approvals. It must consult the edition/Business approval rows;
79
+ returning a cached boolean or constant permission defeats the integration.
80
+ The door combines it with current enablement, trust/key revocation, single-use
81
+ nonce, expiry, namespace/destination binding and erasure epoch before and after
82
+ each existing D1 transaction. A root approval has no apply authority.
83
+
84
+ For example, the trusted signer issues fresh grants using these additional
85
+ payload members (the existing issuer/audience/owner/nonce/expiry/quotas remain):
86
+
87
+ ```ts
88
+ const bounded = {
89
+ protocol: 'rer-bounded-waves/1', migration: 'migration-17',
90
+ revision: 'revision-2', policy: 'policy-1',
91
+ };
92
+ // prepare / seal-parent / wave-status: { bounded }
93
+ // review-parent:
94
+ const reviewedParent = {
95
+ ...bounded, parentFingerprint, parentApprovalRecordId, parentApprovedAt,
96
+ };
97
+ // { bounded: reviewedParent, fingerprint: parentFingerprint,
98
+ // approvalRecordId: parentApprovalRecordId, approvedAt: parentApprovedAt }
99
+ // apply-child:
100
+ // { bounded: {...reviewedParent, childFingerprint}, fingerprint: childFingerprint,
101
+ // approvalRecordId: childApprovalRecordId, approvedAt: childApprovedAt }
102
+ ```
103
+
104
+ The concrete POST bodies are:
105
+
106
+ ```json
107
+ {"uploadId":"server-upload-id","ordinal":0}
108
+ {"parentId":"server-parent-id","ordinal":1}
109
+ {"parentId":"server-parent-id"}
110
+ {"parentId":"server-parent-id","ordinal":0,"reviewedFingerprint":"<child-sha256>"}
111
+ ```
112
+
113
+ The first two are `/prepare`. The third is `/seal-parent`, `/review-parent`,
114
+ `/wave-status`, `/finalize-parent` or `/reverse-parent` under its own action
115
+ grant. The fourth is `/apply-child`. Before approval, `/prepare` with
116
+ `{"parentId":"server-parent-id","cancel":true}` can discard private custody;
117
+ the previous complete head stays authoritative. A cancelled revision is not
118
+ mutated/reused: review a new revision. All routes remain on `/api/rer/import/`.
119
+
120
+ Child responses say `childSuccess` and always `aggregateSuccess:false`. Missing,
121
+ altered, foreign, unapproved, revoked or out-of-order children refuse. Fresh
122
+ grants resolve lost responses through durable inventory/status, operation keys
123
+ and the existing D1 checkpoints. A child uses an ordinary D1 job, preserving
124
+ attachment/original eligibility, strict before-images, apply and rollback. The
125
+ kit's internal source namespace binds the reviewed snapshot revision, not a
126
+ child/upload/attempt; the logical namespace and migration lineage never split.
127
+
128
+ Finalization requires all exact children, current approvals, readable custody,
129
+ complete inventory totals and exact native/provenance reconciliation. Only that
130
+ transaction advances the complete head and releases gone-but-retained reporting.
131
+ Absence never deletes. `wave-status` reports progress/completion history, not a
132
+ fresh aggregate proof; the sender consumes `/finalize-parent` itself.
133
+ Routine grants marked `catchup:'bounded'` require the global strict under-five-
134
+ percent comparison and all existing gone/conflict/rejected/unmapped exclusions.
135
+ A separately reviewed proposal can describe a complete snapshot with absence;
136
+ its previous targets remain attributed and retained. Catch-up Undo preserves
137
+ unchanged original owners. Reverse checks all known conflicts first, then uses
138
+ ordinary strict Undo or partial-job rollback, descending dependants first. It is
139
+ bounded, resumable reversal, not whole-database atomicity. Reversal reserves the
140
+ same pending-parent slot, so concurrent cumulative preparation cannot race its
141
+ preflight or clear another parent during complete-head restoration.
142
+
143
+ Every request retains the existing plan/value/parser/SQL/transport limits. A
144
+ complete reconciliation or baseline read that cannot fit the actual request
145
+ budget refuses before publication; there is no unchecked paged remainder or
146
+ hosted-capacity claim. Independent qualification must establish the capacity of
147
+ the adopting edition and its exact native adapter. The tiny-grant public proof
148
+ does not establish capacity for a production-sized legacy snapshot.
149
+
150
+ The reusable sender is `cli/business/bounded-waves.mjs`, with its actual typed
151
+ `BoundedArrangement` in `bounded-waves.d.mts`. `rer business run` opts in only
152
+ when Business supplies `task.bounded_waves`; ordinary tasks keep their path.
153
+ Business supplies the claimed destination and fresh action grants, then the
154
+ exact server parent fingerprint and ordered immutable child approvals. The
155
+ sender cannot synthesize them or treat task completion as reconciliation.
156
+ An arrangement without approval prepares/seals and pauses truthfully. Approval
157
+ arrangement, grant minting, edition-native grouping, and real adoption remain
158
+ upstream dependencies, not evidence supplied by this Base implementation.
@@ -0,0 +1,38 @@
1
+ import { createHash } from 'node:crypto';
2
+ import type { D1Database } from '../../d1/types.ts';
3
+ import { canonicalHash } from '../../provenance/canonical-stream.ts';
4
+ import type { WaveAlias, WaveNativeRow } from './types.ts';
5
+
6
+ export interface WaveInventoryMember {type: string; identity: string; digest: string}
7
+ export const inventoryMember = (type: string, identity: unknown, content: unknown): WaveInventoryMember => ({
8
+ type,identity:canonicalHash(identity),digest:canonicalHash([type,identity,content]),
9
+ });
10
+ export interface WaveInventoryInput {
11
+ source: readonly {kind:string;sourceId:string;payloadHash:string}[];
12
+ native: readonly WaveNativeRow[];
13
+ aliases?: readonly WaveAlias[];
14
+ files: readonly {path:string;sha256:string;bytes:number;role:'source'|'object'|'context';logicalBytes:number}[];
15
+ }
16
+ /** Adapter convenience for a complete independently enumerated census. An
17
+ * edition can equivalently sort these digest leaves in its paged reader/store. */
18
+ export function waveInventoryHash(input: WaveInventoryInput): string {
19
+ return canonicalHash([
20
+ ...input.source.map(row=>inventoryMember('source',[row.kind,row.sourceId],row.payloadHash)),
21
+ ...(input.aliases ?? []).map(alias=>inventoryMember('alias',[alias.kind,alias.sourceId],alias)),
22
+ ...input.native.map(row=>inventoryMember('native',[row.table,row.key],row)),
23
+ ...input.files.map(file=>inventoryMember('file',file.path,[file.sha256,file.bytes,file.role,file.logicalBytes])),
24
+ ].map(row=>row.digest).sort());
25
+ }
26
+ /** Hash the complete exact inventory without materializing its JSON or all
27
+ * source/native membership rows. The unique identity index seals coverage;
28
+ * the digest index permits bounded keyset reads. */
29
+ export async function storedInventoryHash(db: D1Database, parentId: string): Promise<string> {
30
+ const digest=createHash('sha256');digest.update('[');
31
+ let after='',first=true;
32
+ for (;;) {
33
+ const {results}=await db.prepare('SELECT digest FROM ik_door_wave_members WHERE parent_id=? AND digest>? ORDER BY digest LIMIT 128').bind(parentId,after).all<{digest:string}>();
34
+ for (const row of results) {digest.update((first?'':',')+JSON.stringify(row.digest));first=false;after=row.digest;}
35
+ if(results.length<128)break;
36
+ }
37
+ digest.update(']');return digest.digest('hex');
38
+ }