@runeditrun/rer 0.2.99 → 0.2.100

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/cli/README.md +88 -2
  2. package/cli/business/bounded-waves.d.mts +23 -0
  3. package/cli/business/bounded-waves.mjs +57 -0
  4. package/cli/business/catchup-task.mjs +9 -2
  5. package/cli/business/command.mjs +8 -3
  6. package/cli/business/export-inputs.mjs +12 -2
  7. package/cli/business/export-task.mjs +9 -2
  8. package/cli/business/legacy-package.mjs +117 -0
  9. package/cli/business/local-commands.mjs +10 -1
  10. package/cli/business/package-transfer.mjs +127 -0
  11. package/cli/business/state.mjs +3 -3
  12. package/cli/business/vendor-door.mjs +83 -11
  13. package/cli/business/vendors/README.md +41 -2
  14. package/cli/business/vendors/common.mjs +81 -0
  15. package/cli/business/vendors/index.mjs +12 -2
  16. package/cli/business/vendors/zendesk-attachments.mjs +57 -0
  17. package/cli/business/vendors/zendesk-encode.mjs +133 -0
  18. package/cli/business/vendors/zendesk-guide.mjs +34 -0
  19. package/cli/business/vendors/zendesk-package.mjs +28 -0
  20. package/cli/business/vendors/zendesk-pages.mjs +61 -0
  21. package/cli/business/vendors/zendesk-people.mjs +7 -0
  22. package/cli/business/vendors/zendesk-store.mjs +154 -0
  23. package/cli/business/vendors/zendesk-tickets.mjs +21 -0
  24. package/cli/business/vendors/zendesk-token.mjs +31 -0
  25. package/cli/business/vendors/zendesk-transport.mjs +48 -0
  26. package/cli/business/vendors/zendesk.mjs +61 -0
  27. package/cli/deploy-command.mjs +5 -4
  28. package/cli/deploy-inputs.mjs +1 -1
  29. package/cli/deploy.mjs +6 -4
  30. package/cli/production-auth.mjs +16 -3
  31. package/cli/production-mail.mjs +18 -1
  32. package/cli/site.mjs +1 -1
  33. package/global/CHANGELOG.md +21 -0
  34. package/modules/deploy-kit/README.md +10 -0
  35. package/modules/deploy-kit/provider.mjs +18 -2
  36. package/modules/import-kit/src/d1/jobs.ts +3 -1
  37. package/modules/import-kit/src/d1/retirement-scope.ts +2 -2
  38. package/modules/import-kit/src/d1/retirement-tally.ts +1 -0
  39. package/modules/import-kit/src/d1/retirement.ts +15 -5
  40. package/modules/import-kit/src/d1/types.ts +3 -0
  41. package/modules/import-kit/src/d1/wave-boundary.ts +33 -0
  42. package/modules/import-kit/src/d1/wave-retirement.ts +47 -0
  43. package/modules/import-kit/src/door/ADOPTING.md +37 -0
  44. package/modules/import-kit/src/door/README.md +66 -0
  45. package/modules/import-kit/src/door/admission.ts +13 -3
  46. package/modules/import-kit/src/door/authority.ts +1 -0
  47. package/modules/import-kit/src/door/failures.ts +1 -1
  48. package/modules/import-kit/src/door/grants.ts +19 -5
  49. package/modules/import-kit/src/door/handler.ts +12 -1
  50. package/modules/import-kit/src/door/index.ts +6 -0
  51. package/modules/import-kit/src/door/package.ts +121 -0
  52. package/modules/import-kit/src/door/planning.ts +44 -19
  53. package/modules/import-kit/src/door/protocol.ts +4 -2
  54. package/modules/import-kit/src/door/schema.ts +2 -0
  55. package/modules/import-kit/src/door/store.ts +2 -2
  56. package/modules/import-kit/src/door/types.ts +24 -1
  57. package/modules/import-kit/src/door/waves/README.md +158 -0
  58. package/modules/import-kit/src/door/waves/digests.ts +38 -0
  59. package/modules/import-kit/src/door/waves/inventory.ts +84 -0
  60. package/modules/import-kit/src/door/waves/journal.ts +19 -0
  61. package/modules/import-kit/src/door/waves/lifecycle.ts +242 -0
  62. package/modules/import-kit/src/door/waves/preparation.ts +190 -0
  63. package/modules/import-kit/src/door/waves/schema.ts +73 -0
  64. package/modules/import-kit/src/door/waves/store.ts +112 -0
  65. package/modules/import-kit/src/door/waves/types.ts +76 -0
  66. package/onboarding/adapter.mjs +7 -1
  67. package/onboarding/operator-installer.mjs +3 -1
  68. package/package.json +1 -1
  69. package/scripts/publish-check/README.md +20 -0
  70. package/scripts/publish-check/diagnostics.mjs +73 -0
  71. package/scripts/publish-check/slots.py +37 -7
  72. package/scripts/publish-check.mjs +64 -37
@@ -1,3 +1,4 @@
1
+ import timers from 'node:timers/promises';
1
2
  import { providerPages, providerListShape } from './pagination.mjs';
2
3
  import { deleteR2Object } from './r2-object-transport.mjs';
3
4
  import { isR2ObjectDelete } from './r2-objects.mjs';
@@ -40,7 +41,7 @@ const redactValues = (text, values) => values.filter(Boolean).flatMap(value=>[va
40
41
  return at >= 0 && secret.startsWith(value.slice(at)) ? value.slice(0,at)+'[REDACTED]' : value;
41
42
  },text);
42
43
  /** Only this fixed origin and the explicitly selected account ever receive the token. */
43
- export function cloudflare({ accountId, token, fetch: request = globalThis.fetch, objectDelete = deleteR2Object, timeoutMs = 30000, maxRequests = 500, removal = false, budgetMigrations = false, requestSignal = () => undefined }) {
44
+ export function cloudflare({ accountId, token, fetch: request = globalThis.fetch, objectDelete = deleteR2Object, timeoutMs = 30000, maxRequests = 500, removal = false, budgetMigrations = false, requestSignal = () => undefined, settleLists = false }) {
44
45
  assert(hexId(accountId), 'invalid-cloudflare-account-id');
45
46
  assert(typeof token === 'string' && token.trim() && !/[\r\n]/.test(token), 'invalid-cloudflare-api-token');
46
47
  // Capture an outer deadline once per operation, before transport retries.
@@ -137,7 +138,7 @@ export function cloudflare({ accountId, token, fetch: request = globalThis.fetch
137
138
  throw error;
138
139
  }
139
140
  }
140
- async function collect(path, options) {
141
+ async function collectOnce(path, options) {
141
142
  let result, rows = [];
142
143
  for await (const page of providerPages(envelope, path, options)) {
143
144
  result ??= page.data.result;
@@ -146,6 +147,21 @@ export function cloudflare({ accountId, token, fetch: request = globalThis.fetch
146
147
  }
147
148
  return options?.field ? {...result,[options.field]:rows} : rows;
148
149
  }
150
+ // Site deploy opts in here: both request/zoneRequest and list must discard
151
+ // every page of an inconsistent snapshot. Only this read is replayed; owner
152
+ // and teardown callers retain their existing settling policies and defaults.
153
+ async function collect(path, options) {
154
+ if (!settleLists) return collectOnce(path, options);
155
+ const signal = deadlineSignal(options?.signal);
156
+ for (let attempt = 1; ; attempt++) {
157
+ signal?.throwIfAborted();
158
+ try { return await collectOnce(path, options); }
159
+ catch (error) {
160
+ if (attempt === 3 || !['provider-partial-list','provider-pagination-changed'].includes(error.message)) throw error;
161
+ await timers.setTimeout(20000, undefined, {signal});
162
+ }
163
+ }
164
+ }
149
165
  async function list(path, options = {}) {
150
166
  const shape = {...providerListShape(path),...options};
151
167
  const result = await collect(path,shape);
@@ -2,6 +2,7 @@ import { captureJobLayout, jobLayoutGuards, jobLayoutReads } from './stream-layo
2
2
  import { needsStreamOrigin } from './stream-origin.ts';
3
3
  import { nativeStagingScope } from './native-staging-context.ts';
4
4
  import { streamBoundary } from './stream-boundary.ts';
5
+ import { captureWaveBoundary, waveBoundary } from './wave-boundary.ts';
5
6
  import { requestContext } from './budget.ts';
6
7
  import { loadPlans, storedPlanSnapshot, planBody, planGuards, planChunkReads, retainPlan, creationCodec, LARGE_PLAN_BYTES } from './plans.ts';
7
8
  import { randomUUID } from 'node:crypto';
@@ -38,10 +39,11 @@ export async function stored(id: string, context: D1ImportContext, action: Impor
38
39
  job.erasedRecords = plan.entries.filter(entry => entry.erased).length;
39
40
  attachReversal(job, await loadReversalState(row, plan, context));
40
41
  await captureJobLayout(job, row, context, action);
42
+ await captureWaveBoundary(job, context);
41
43
  return { job, plan };
42
44
  }
43
45
  export function boundary(job: ImportJob, plan: ImportPlan, action: ImportAction = 'read'): D1Sql[] {
44
- return [...jobLayoutGuards(job), ...streamBoundary(job.id, action), ...planGuards(plan, job.id), ...reversalGuards(job.id, reversalState(job)), guard(sql('EXISTS(SELECT 1 FROM ik_jobs WHERE id=? AND fence=? AND checkpoint=? AND state=? AND fingerprint=? AND plan_json=? AND principal_id=? AND destination=? AND namespace=? AND mapping_version=? AND total=? AND error IS ?)', job.id, job.fence, job.checkpoint, job.state, job.fingerprint, planBody(plan), job.principalId, job.destination, job.namespace, job.mappingVersion, job.total, job.error))];
46
+ return [...waveBoundary(job, action), ...jobLayoutGuards(job), ...streamBoundary(job.id, action), ...planGuards(plan, job.id), ...reversalGuards(job.id, reversalState(job)), guard(sql('EXISTS(SELECT 1 FROM ik_jobs WHERE id=? AND fence=? AND checkpoint=? AND state=? AND fingerprint=? AND plan_json=? AND principal_id=? AND destination=? AND namespace=? AND mapping_version=? AND total=? AND error IS ?)', job.id, job.fence, job.checkpoint, job.state, job.fingerprint, planBody(plan), job.principalId, job.destination, job.namespace, job.mappingVersion, job.total, job.error))];
45
47
  }
46
48
  export async function planD1Import(source: SourceBundle, mapping: ImportMapping, context: D1ImportContext): Promise<ImportPlan> {
47
49
  context = requestContext(context);
@@ -21,8 +21,8 @@ export interface RetirementScope { jobs: string[]; streams: string[]; stages: st
21
21
  /** Jobs, streams and private stages whose headers hold one of the exact values,
22
22
  * closed over everything their deletion would break: whole streams, jobs whose
23
23
  * entries reuse a retired job's sources, and preserved history in both directions. */
24
- export async function retirementScope(db: D1Database, destination: string, values: string[]): Promise<RetirementScope> {
25
- const jobs = new Set<string>(), streams = new Set<string>(), stages = new Set<string>(), anchors = new Set<string>(), preparations = new Set<string>();
24
+ export async function retirementScope(db: D1Database, destination: string, values: string[], seedJobs: string[] = []): Promise<RetirementScope> {
25
+ const jobs = new Set<string>(seedJobs), streams = new Set<string>(), stages = new Set<string>(), anchors = new Set<string>(), preparations = new Set<string>();
26
26
  const exact = JSON.stringify(values);
27
27
  const read = async (query: string, ...bound: unknown[]) => (await db.prepare(query).bind(...bound).all<{ id: string }>()).results.map(row => row.id);
28
28
  if (values.length) {
@@ -5,6 +5,7 @@ import { doorRetirementStore } from './retirement-door.ts';
5
5
  // Bookkeeping that every custody change bumps, plus the erasure's own records.
6
6
  const SKIPPED = new Set(['ik_d1_assertions', 'ik_d1_custody_states', 'ik_d1_custody_revisions', 'ik_d1_publication_fences',
7
7
  'ik_d1_erasure_tally', 'ik_d1_erasure_journal', 'ik_d1_erasure_progress', 'ik_d1_retirements', 'ik_d1_erased_sources', 'ik_d1_erased_targets', 'ik_targets',
8
+ 'ik_door_wave_erased_aliases', 'ik_door_wave_origins', 'ik_door_wave_roots', 'ik_door_wave_children', 'ik_door_wave_members', 'ik_door_wave_approvals',
8
9
  'ik_door_settings', // Installation opt-in only; no uploaded record content.
9
10
  'ik_door_revoked_keys', // Persistent key revocations must survive erasure.
10
11
  'ik_door_grants', // Grant nonce replay protection; no uploaded record content.
@@ -1,3 +1,4 @@
1
+ import { waveRetirementSeeds } from './wave-retirement.ts';
1
2
  import { randomUUID } from 'node:crypto';
2
3
  import type { D1CustodyRetirement, D1ErasureJournalEntry, D1ImportContext, D1ImportPerson, D1RetirementOptions, D1RetirementProgress, D1Sql } from './types.ts';
3
4
  import { requestContext, limitStatements, packGuards, remaining, D1RequestLimitError } from './budget.ts';
@@ -37,7 +38,14 @@ export function retireD1ImportCustody(person: D1ImportPerson, nativeStatements:
37
38
  export function retireD1ImportCustody(person: D1ImportPerson, nativeStatements: D1Sql[], input: D1ImportContext, checkpoint: D1Sql[], options: D1ErasureStepOptions): Promise<D1CustodyRetirement | D1ErasurePending>;
38
39
  export function retireD1ImportCustody(person: D1ImportPerson, nativeStatements: D1Sql[], input: D1ImportContext, checkpoint: D1Sql[], options: D1RetirementOptions): Promise<D1CustodyRetirement | D1RetirementProgress>;
39
40
  export async function retireD1ImportCustody(person: D1ImportPerson, nativeStatements: D1Sql[], input: D1ImportContext, checkpoint: D1Sql[], options: D1RetirementOptions | D1ErasureStepOptions = {}): Promise<D1CustodyRetirement | D1RetirementProgress | D1ErasurePending> {
40
- if ('operationKey' in options) return retireD1ImportCustodyStep(person, nativeStatements, input, checkpoint, options);
41
+ if ('operationKey' in options) {
42
+ const admission=requestContext(input);
43
+ await readAtomic(admission,[],undefined,'erase');
44
+ const db=admission.storage.database;
45
+ const installed=await db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='ik_door_wave_roots'").first();
46
+ if (person.aliases?.length || installed && await db.prepare('SELECT 1 FROM ik_door_wave_roots WHERE destination=? LIMIT 1').bind(input.destination).first()) throw new Error('IMPORT_WAVE_JOURNAL_REQUIRED');
47
+ return retireD1ImportCustodyStep(person,nativeStatements,admission,checkpoint,options);
48
+ }
41
49
  person = { ...person, related: person.related ? { ...person.related, fields: fieldRedactions(person.related.fields) } : undefined };
42
50
  const context = limitStatements(requestContext({ ...input, originalErasurePolicy: input.originalErasurePolicy ?? 'withdraw-affected-artifacts' }), 1000);
43
51
  if (context.maxBatchBytes !== undefined) throw new Error('IMPORT_RETIREMENT_OPERATION_KEY_REQUIRED');
@@ -45,16 +53,18 @@ export async function retireD1ImportCustody(person: D1ImportPerson, nativeStatem
45
53
  for (const value of values) requireText(value, 'person value');
46
54
  if (person.target) { requireText(person.target.kind, 'target kind'); requireText(person.target.id, 'target id'); }
47
55
  const fields = person.related?.fields ?? [];
48
- if (!person.target && !values.length && !fields.length) throw new Error('IMPORT_RETIREMENT_PERSON_REQUIRED');
56
+ if (!person.target && !values.length && !fields.length && !person.aliases?.length) throw new Error('IMPORT_RETIREMENT_PERSON_REQUIRED');
57
+ for (const alias of person.aliases ?? []) for (const value of [alias.installationId,alias.namespace,alias.kind,alias.sourceId]) requireText(value,'source alias');
49
58
  // Without a native target only field redaction applies: whole-record erasure
50
59
  // (`source`, `original`) binds what it severs to the erased target.
51
60
  if (person.related && !person.target && (person.related.source || person.related.original || !fields.length)) throw new Error('IMPORT_RETIREMENT_TARGET_REQUIRED');
52
61
  await readAtomic(context, [], undefined, 'erase');
53
62
  const db = context.storage.database, destination = context.destination;
54
- const scope = await retirementScope(db, destination, values), retiring = new Set(scope.jobs);
63
+ const wave = await waveRetirementSeeds(db,destination,person,context.storage.targets.planCodec);
64
+ const scope = await retirementScope(db, destination, values,wave.jobs), retiring = new Set(scope.jobs);
55
65
  // Withdrawn door content is custody too: a values-only erasure that removes it is fenced.
56
66
  let stores: Store[] | undefined;
57
- const fenced = Boolean(person.target) || fields.length > 0 || scope.jobs.length + scope.streams.length + scope.stages.length + scope.anchors.length + scope.preparations.length > 0
67
+ const fenced = Boolean(person.target) || !!person.aliases?.length || fields.length > 0 || scope.jobs.length + scope.streams.length + scope.stages.length + scope.anchors.length + scope.preparations.length > 0
58
68
  || await doorCustodyInScope(db, destination, stores = await tallyStores(db));
59
69
  if (fenced && !checkpoint.length) throw new Error('IMPORT_RETIREMENT_CHECKPOINT_REQUIRED');
60
70
  const fence = fenced ? checkpoint : [];
@@ -94,7 +104,7 @@ export async function retireD1ImportCustody(person: D1ImportPerson, nativeStatem
94
104
  touched.streams = [...new Set([...scope.anchors, ...await pageStreams(db, touched.jobs)])].sort();
95
105
  stores ??= await tallyStores(db);
96
106
  const entry = { recordedAt, recordsErased: erasure.decisions.length, jobsRetired: scope.jobs.length, streamsRetired: scope.anchors.length };
97
- prefix = packGuards([authority(context, 'erase'), ...tallyBefore(stores, receiptId, touched), ...tallyWithdraw(stores, touched), ...erasure.before, ...scopeGuards(scope), ...lineage.fence.map(guard)]);
107
+ prefix = packGuards([authority(context, 'erase'), ...tallyBefore(stores, receiptId, touched), ...(wave.roots.length ? [sql("UPDATE ik_door_wave_roots SET state='withdrawn' WHERE id IN (SELECT value FROM json_each(?))",JSON.stringify(wave.roots))] : []), ...wave.tombstones.map(alias=>sql('INSERT OR IGNORE INTO ik_door_wave_erased_aliases(installation_id,namespace,destination,identity) VALUES(?,?,?,?)',alias.installation,alias.namespace,destination,alias.identity)), ...tallyWithdraw(stores, touched), ...erasure.before, ...scopeGuards(scope), ...wave.guards.map(guard), ...lineage.fence.map(guard)]);
98
108
  suffix = [...packGuards([...erasure.after, ...retirementWrites(scope, destination, receiptId, recordedAt), ...headerAbsence(destination, values),
99
109
  ...tallyJournal(stores, receiptId, touched, entry, fenced), authority(context, 'erase'), sql('DELETE FROM ik_d1_assertions')]), ...readonlyChecks(erasure.final), finalAuthority(context, 'erase')];
100
110
  // One read after the batch returns its journal row.
@@ -123,6 +123,9 @@ export interface D1ProvenanceRow extends ProvenanceRow { predecessorId?: string
123
123
  * sit whole in a job or stream header, an original's name or a private stage.
124
124
  * Nothing is ever matched as a substring. */
125
125
  export interface D1ImportPerson {
126
+ /** Server-derived source subjects, including context/metadata subjects that
127
+ * never became native targets. Never accept this selector from a client. */
128
+ aliases?: readonly { installationId: string; namespace: string; kind: string; sourceId: string }[];
126
129
  target?: { kind: string; id: string };
127
130
  values?: string[];
128
131
  related?: D1RelatedErasure;
@@ -0,0 +1,33 @@
1
+ import type { ImportAction, ImportJob } from '../types.ts';
2
+ import type { D1ImportContext, D1Sql } from './types.ts';
3
+ import { guard, sql } from './database.ts';
4
+
5
+ const contexts = new WeakMap<D1ImportContext['authorization'], {parent:string;mode:string;database:D1ImportContext['storage']['database']}>();
6
+ const jobs = new WeakMap<ImportJob, {parent:string;ordinal:number;allowed:boolean;mode:string}>();
7
+ /** A direct ordinary apply/undo must not promote a prepared wave's child. */
8
+ export function bindWaveContext(context: D1ImportContext, parent: string, mode: string) {
9
+ contexts.set(context.authorization,{parent,mode,database:context.storage.database});
10
+ }
11
+ export async function captureWaveBoundary(job: ImportJob, context: D1ImportContext) {
12
+ if (!job.namespace.startsWith('door-wave:')) return;
13
+ const db=context.storage.database;
14
+ if (!await db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='ik_door_wave_children'").first()) return;
15
+ const row=await db.prepare(`SELECT c.parent_id,c.ordinal FROM ik_door_wave_children c JOIN ik_door_plans p ON p.id=c.plan_id
16
+ WHERE p.job_id=? OR ('door:'||p.id)=(SELECT operation_key FROM ik_jobs WHERE id=?)`).bind(job.id,job.id).first<any>();
17
+ if (!row) {jobs.set(job,{parent:'',ordinal:-1,allowed:false,mode:''});return;}
18
+ const scope=contexts.get(context.authorization);
19
+ jobs.set(job,{parent:row.parent_id,ordinal:row.ordinal,allowed:scope?.parent===row.parent_id && scope?.database===db,mode:scope?.mode ?? ''});
20
+ }
21
+ export function waveBoundary(job: ImportJob, action: ImportAction): D1Sql[] {
22
+ const row=jobs.get(job);
23
+ if (!row || !['apply','undo','rollback'].includes(action)) return [];
24
+ const forward=action==='apply';
25
+ if (!row.allowed || row.mode!==(forward?'apply':'reverse')) return [guard(sql('0'))];
26
+ return [guard(sql(`EXISTS(SELECT 1 FROM ik_door_wave_roots r JOIN ik_door_wave_children c ON c.parent_id=r.id
27
+ JOIN ik_door_plans p ON p.id=c.plan_id WHERE r.id=? AND c.ordinal=? AND p.job_id=?
28
+ AND EXISTS(SELECT 1 FROM ik_door_wave_origins o WHERE o.job_id=p.job_id AND o.parent_id=r.id AND o.ordinal=c.ordinal AND o.plan_id=p.id AND o.fingerprint=p.fingerprint)
29
+ AND r.state IN (${forward?"'applying','completed'":"'reversing'"})
30
+ AND EXISTS(SELECT 1 FROM ik_door_wave_approvals a WHERE a.parent_id=r.id AND a.ordinal=-1 AND a.revoked=0 AND a.fingerprint=r.fingerprint)
31
+ AND NOT EXISTS(SELECT 1 FROM ik_door_wave_children peer JOIN ik_door_plans pp ON pp.id=peer.plan_id LEFT JOIN ik_jobs j ON j.id=pp.job_id
32
+ WHERE peer.parent_id=r.id AND peer.ordinal ${forward?'<':'>'} c.ordinal AND ${forward?"(j.id IS NULL OR j.state<>'completed')":"(j.id IS NOT NULL AND j.state NOT IN ('undone','rolled_back'))"}))`,row.parent,row.ordinal,job.id))];
33
+ }
@@ -0,0 +1,47 @@
1
+ import type { D1Database, D1ImportPerson, D1PlanCodec, D1Sql } from './types.ts';
2
+ import { targetIdentity } from './erasure-index.ts';
3
+ import { sql } from './database.ts';
4
+ import { doorLineage } from './retirement-door.ts';
5
+ import { canonicalJson, sha256 } from '../provenance/identity.ts';
6
+
7
+ /** Wave snapshots are indivisible custody. Retire their actual child journals
8
+ * whole before ordinary erasure tries to rewrite an attachment-bearing entry.
9
+ * The caller still deletes only the selected person's native data. Peer native
10
+ * records remain; this is withdrawal, never successful Undo or reconciliation. */
11
+ export async function waveRetirementSeeds(db: D1Database, destination: string, person: D1ImportPerson, codec?: D1PlanCodec): Promise<{jobs:string[];roots:string[];tombstones:{installation:string;namespace:string;identity:string}[];guards:D1Sql[]}> {
12
+ const empty={jobs:[],roots:[],tombstones:[],guards:[]};
13
+ if (!person.target && !person.aliases?.length) return empty;
14
+ if (!await db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='ik_door_wave_children'").first()) {
15
+ if (person.aliases?.length) throw new Error('IMPORT_WAVE_SCHEMA_REQUIRED');
16
+ return empty;
17
+ }
18
+ if (!person.aliases?.length && !await db.prepare('SELECT 1 FROM ik_door_wave_roots WHERE destination=? LIMIT 1').bind(destination).first()) return empty;
19
+ const lineage=await doorLineage(person,db,destination,codec);
20
+ const tombstones=[...lineage.sources.map(source=>{
21
+ const [installation,namespace,kind,id]=JSON.parse(source);
22
+ return {installation,namespace,identity:sha256(canonicalJson([kind,id]))};
23
+ }),...(person.aliases ?? []).map(a=>({installation:a.installationId,namespace:a.namespace,identity:sha256(canonicalJson([a.kind,a.sourceId]))}))];
24
+ const scopes=JSON.stringify(tombstones);
25
+ const selection=`WITH RECURSIVE affected(parent_id) AS (
26
+ SELECT DISTINCT c.parent_id FROM ik_door_wave_children c JOIN ik_door_plans p ON p.id=c.plan_id
27
+ JOIN ik_jobs j ON j.id=p.job_id OR j.operation_key='door:'||p.id JOIN ik_d1_plan_index i ON i.job_id=j.id
28
+ WHERE j.destination=? AND i.target_identity=?
29
+ UNION SELECT m.parent_id FROM ik_door_wave_members m JOIN ik_door_wave_roots r ON r.id=m.parent_id JOIN json_each(?) s
30
+ ON r.installation_id=json_extract(s.value,'$.installation') AND r.namespace=json_extract(s.value,'$.namespace')
31
+ AND m.identity=json_extract(s.value,'$.identity') WHERE r.destination=? AND m.type IN ('source','alias')
32
+ UNION SELECT r.id FROM ik_door_wave_roots r JOIN affected a ON r.previous_id=a.parent_id JOIN ik_door_wave_roots parent ON parent.id=a.parent_id WHERE r.destination=? AND r.installation_id=parent.installation_id AND r.namespace=parent.namespace
33
+ ) SELECT parent_id FROM affected`;
34
+ const digest=person.target ? targetIdentity(destination,person.target.kind,person.target.id) : null;
35
+ const parameters=[destination,digest,scopes,destination,destination];
36
+ const roots=(await db.prepare(selection).bind(...parameters).all<{parent_id:string}>()).results.map(row=>row.parent_id);
37
+ if (!roots.length) return {jobs:[],roots:[],tombstones,guards:[]};
38
+ const rootJson=JSON.stringify(roots);
39
+ const members=`SELECT j.id FROM ik_door_wave_children c JOIN ik_door_plans p ON p.id=c.plan_id
40
+ JOIN ik_jobs j ON j.id=p.job_id OR j.operation_key='door:'||p.id WHERE j.destination=? AND c.parent_id IN (SELECT value FROM json_each(?))`;
41
+ const jobs=(await db.prepare(members).bind(destination,rootJson).all<{id:string}>()).results.map(row=>row.id);
42
+ return {jobs,roots,tombstones,guards:[
43
+ ...lineage.fence,
44
+ sql(`NOT EXISTS(SELECT 1 FROM (${selection}) WHERE parent_id NOT IN (SELECT value FROM json_each(?)))`,...parameters,rootJson),
45
+ sql(`NOT EXISTS(SELECT 1 FROM (${members}) WHERE id NOT IN (SELECT value FROM json_each(?)))`,destination,rootJson,JSON.stringify(jobs)),
46
+ ]};
47
+ }
@@ -484,3 +484,40 @@ For a concrete grant client and harness see Spool's `tests/business-import.mjs`;
484
484
  its `grant`, `doorRequest`, `nativeSnapshot`, `startTestWorker` and `signIn`
485
485
  provide those network edges. Run with `nice -n 15 node <existing-test-journey>`
486
486
  after the seed's normal build. No `rer check` extension is required.
487
+
488
+ ### Package-aware sources
489
+
490
+ Re-vendor the door together with its ordinary dependency closure, including
491
+ `door/package.ts`, `planning.ts`, `types.ts`, `handler.ts` and `admission.ts`.
492
+ There is no package schema or migration-ledger change: segments use existing
493
+ upload/chunk storage. Include the static `package` declaration in each source
494
+ allow-list result on both mounts. An undeclared namespace remains refused before
495
+ any grant row is saved. See [Logical packages](README.md#logical-packages-across-upload-grants)
496
+ for the stream/index contract and an accessor's bounds.
497
+
498
+ Provide `package.formats`, `maxBytes`, `maxSegments` and `read`. Keep readers
499
+ responsible for format semantics, metadata, per-table counts, attachment bindings
500
+ and explained omissions. Base verifies transport custody and aggregate record
501
+ authority; a reader must return the entire snapshot in one call. It must not
502
+ split a snapshot into independently authorized plans. The legacy CLI deliberately
503
+ declares `records.total: null`: it chooses no record units or waves. HappyToHelp
504
+ must supply D-CAP (package capacity), D-LEG (legacy/delta reader) and D-WAVES
505
+ (aggregate bounded-wave authority) before real oversized legacy imports.
506
+
507
+ The agent registers a locally prepared legacy database export with:
508
+
509
+ ```sh
510
+ rer business legacy-package add DIR
511
+ rer business legacy-package status DIR
512
+ rer business legacy-package remove DIR
513
+ ```
514
+
515
+ Registration verifies all declared file sizes and digests, row chunk/context
516
+ counts, per-table totals, object identities and attachment counts. JSON files pass
517
+ the outbound detector before upload; customer-original media are exempt. It
518
+ stores a private local registration and prints only counts and the manifest SHA.
519
+ A leased `happytohelp-legacy` export checks that registration again, pauses and
520
+ names the local package if it changed, then sends the exact package files through
521
+ the logical package protocol. It needs no vendor key and makes no request to a
522
+ live HappyToHelp account. Disconnect removes the registration and transfer
523
+ journal/spool; removing registration leaves the source directory intact.
@@ -552,3 +552,69 @@ approval. It refuses gone, conflicting, rejected or unmapped records. Additions
552
552
  plus replacements divided by baseline source count must be strictly below 5%;
553
553
  the server derives these counts. A refusal leaves Business to present a new
554
554
  proposal for owner review. No caller-supplied totals relax this boundary.
555
+
556
+ ## Logical packages across upload grants
557
+
558
+ A source may opt in to manifests with `format: "rer-door-package/1"` and
559
+ `package: {formats, maxBytes, maxSegments, read}` on its resolved source. The
560
+ same declaration belongs on the operator and grant mounts. Other sources keep
561
+ their existing readers and upload limits. No additional schema migration or
562
+ larger grant is required. The declaration cannot be combined with `intake`.
563
+
564
+ The sender uploads `rer-package.json` first, within one ordinary upload grant,
565
+ then calls `plan` for that manifest's upload id. The manifest has:
566
+
567
+ ```json
568
+ {
569
+ "format": "rer-door-package/1",
570
+ "source": {"format": "example-snapshot-v1", "version": 1, "origin": "legacy-database", "exportedAt": null, "synthetic": true, "adapter": "example"},
571
+ "mode": "snapshot",
572
+ "stream": {"bytes": 0, "sha256": "<SHA-256 of the entire stream>"},
573
+ "segments": [{"bytes": 2097152, "sha256": "<segment SHA-256>"}],
574
+ "index": {"path": "index.ndjson", "bytes": 0, "sha256": "<index SHA-256>"},
575
+ "records": {"total": null, "byKind": {}},
576
+ "totals": {}
577
+ }
578
+ ```
579
+
580
+ The zero sizes and digest placeholders above describe fields, not an accepted
581
+ manifest. `source.format` must be listed by the seed. Both the index and manifest
582
+ fit one grant. The stream starts with `index.ndjson`, then the source files in
583
+ index order, with no padding or encoding. Each index line is
584
+ `{path, bytes, sha256, offset}`; offsets start at zero immediately after the index.
585
+ Paths are relative, unique, and contain no empty, `.` or `..` components. Source
586
+ files may be record NDJSON, `blobs/<sha256>`, census/tombstone/Guide sidecars, or
587
+ all files of a legacy package byte-for-byte. All stream bytes belong to indexed
588
+ files. Nonfinal segments are exactly the grant's `maxBytes`; the final segment
589
+ may be shorter. Upload each as `rer-segment` in ordinary 64 KiB chunks.
590
+
591
+ Before any plan exists, an aware door returns 409 `package_segment_missing`
592
+ with `details.missing: [{bytes, sha256}]`. Missing, incomplete, altered, wrong-size
593
+ or foreign segments have that same response. Only complete uploads named
594
+ `rer-segment` in the same installation and namespace resolve. The sender stops
595
+ with `SEED_PACKAGE_UNSUPPORTED` if a fresh manifest probe answers otherwise;
596
+ it sends no segments. A resumed, previously accepted probe may find the complete
597
+ package and return its plan. The CLI's private journal saves upload ids before
598
+ requests, uses door inventories to skip durable chunks, and restarts an upload
599
+ whose inventory returns 404. Each request uses a fresh single-use grant.
600
+
601
+ The reader receives `{manifest, files, read(path, maxBytes?)}`. The accessor checks
602
+ segment custody, index layout and each requested file's digest, and enforces the
603
+ declared package bound, the source's `limits.maxBytes` file bound (64 MiB by
604
+ default), and any tighter requested file bound. Read record files
605
+ with a tighter bound where practical. Return the whole snapshot as inert parsed
606
+ records. The door counts the returned aggregate against the current plan grant's
607
+ `maxRecords` before mapping or saving a plan, including legacy packages whose
608
+ `records.total` is null. A numeric declared total and its `byKind` counts must match the returned records.
609
+ A package is one plan, with one lineage comparison and one manifest digest; it
610
+ never authorizes record waves or increases record authority per segment.
611
+
612
+ The package path records manifest/file digest metadata in the plan; it does not
613
+ claim those files as kit-retained originals. Seeds needing durable original
614
+ custody must provide that through their reader/product contract. Grant and D1
615
+ request limits still apply; an oversized read/plan is a refusal, never a partial
616
+ import. HappyToHelp adoption still needs its own package reader, capacity and
617
+ legacy delta semantics. Legacy snapshots above the unchanged 100,000-record
618
+ grant require a separate bounded-wave authority decision before real imports.
619
+
620
+ The opt-in [complete-parent bounded-wave protocol](waves/README.md) uses separately approved children and the existing D1 journal. Ordinary package planning keeps its aggregate refusal.
@@ -1,7 +1,8 @@
1
+ import {importDoorBoundedWavesCheck} from './waves/schema.ts';
1
2
  import type { ImportDoorOptions, ImportDoorRuntime, DoorRequest } from './types.ts';
2
3
  import { runtimeOptions } from './handler.ts';
3
4
  import { DoorError, verifyImportGrant, type DoorAction } from './grants.ts';
4
- import { doorFence, erasures, sql, transact } from './store.ts';
5
+ import { authorized, doorFence, erasures, sql, transact } from './store.ts';
5
6
  import { importDoorSchemaCheck } from './schema.ts';
6
7
 
7
8
  /** The storage key; grants are still verified against `claim.installationId`. */
@@ -21,7 +22,7 @@ async function source(options: ImportDoorOptions, namespace: string): Promise<Im
21
22
  if (!options.source) return runtimeOptions(options);
22
23
  const listed = await options.source(namespace);
23
24
  if (!listed) throw new DoorError('unknown_import_source');
24
- return runtimeOptions({...options, parse: listed.parse, mapping: listed.mapping, intake: listed.intake,
25
+ return runtimeOptions({...options, parse: listed.parse, mapping: listed.mapping, intake: listed.intake, package: listed.package, bounded: listed.bounded,
25
26
  format: listed.format ?? options.format, readRaw: 'readRaw' in listed ? listed.readRaw : undefined,
26
27
  ownerPrincipalId: listed.ownerPrincipalId ?? options.ownerPrincipalId});
27
28
  }
@@ -43,5 +44,14 @@ export async function admit(options: ImportDoorOptions, request: Request, action
43
44
  throw new DoorError('grant_replayed_revoked_or_expired');
44
45
  }
45
46
  const authority = sql(`(${basic.sql}) AND EXISTS(SELECT 1 FROM ik_door_grants WHERE installation_id=? AND jti=? AND action=? AND namespace=? AND kid=? AND expires=?)`, ...basic.params ?? [], key, grant.jti, action, grant.namespace, kid, grant.exp);
46
- return {options: runtime, access: {...grant, installationId: key}, authority, via:'grant', epoch, nonce: {jti: grant.jti, kid}};
47
+ if (grant.bounded) {
48
+ if ((await options.database.prepare(importDoorBoundedWavesCheck).first())?.current!==1) throw new DoorError('bounded_waves_schema_outdated',503);
49
+ if (!runtime.bounded) throw new DoorError('bounded_waves_not_adopted',409);
50
+ const external=runtime.bounded.authority(grant.bounded,action);
51
+ authority.sql=`(${authority.sql}) AND (${external.sql})`;
52
+ authority.params=[...authority.params ?? [],...external.params ?? []];
53
+ }
54
+ const admitted: DoorRequest = {options: runtime, access: {...grant, installationId: key}, authority, via:'grant', epoch, nonce: {jti: grant.jti, kid}};
55
+ if (grant.bounded) await authorized(admitted);
56
+ return admitted;
47
57
  }
@@ -7,6 +7,7 @@ import { approvalCaption } from './failures.ts';
7
7
  import { reserveReturn } from '../d1/budget.ts';
8
8
 
9
9
  const actions: Record<DoorAction, ImportAction[]> = {
10
+ prepare: [], 'seal-parent': [], 'review-parent': [], 'apply-child': [], 'wave-status': [], 'finalize-parent': [], 'reverse-parent': [],
10
11
  uploads: [], plan: ['plan', 'read'], apply: ['create', 'apply', 'read'],
11
12
  status: ['read'], reconcile: ['read'], undo: ['read', 'undo', 'rollback'],
12
13
  cancel: [],
@@ -14,7 +14,7 @@ const transient = new RegExp([
14
14
  ].join('|'),'i');
15
15
  /** Only persist allowlisted classifications, never adapter/SQL messages that
16
16
  * may contain customer records. Providers can wrap SQLite errors in causes. */
17
- function reasonCode(error: unknown): string {
17
+ export function reasonCode(error: unknown): string {
18
18
  const seen = new Set<unknown>();
19
19
  for (let cause = error; cause && typeof cause === 'object' && !seen.has(cause);) {
20
20
  seen.add(cause);
@@ -9,8 +9,8 @@ import type { DoorGrant } from './protocol.ts';
9
9
  function fail(): never { throw new DoorError('invalid_grant'); }
10
10
  const object = (value: unknown): value is Record<string, unknown> => value !== null && typeof value === 'object' && !Array.isArray(value);
11
11
  const requiredKeys = ['v', 'iss', 'aud', 'installationId', 'email', 'jti', 'iat', 'exp', 'action', 'namespace', 'maxBytes', 'maxChunks', 'maxRecords'];
12
- const optionalKeys = ['fingerprint', 'approvalRecordId', 'approvedAt', 'catchup'];
13
- const actions: readonly string[] = ['uploads', 'plan', 'apply', 'status', 'reconcile', 'undo', 'cancel', 'archive', 'restore-check'];
12
+ const optionalKeys = ['fingerprint', 'approvalRecordId', 'approvedAt', 'catchup', 'bounded'];
13
+ const actions: readonly string[] = ['uploads', 'plan', 'apply', 'status', 'reconcile', 'undo', 'cancel', 'archive', 'restore-check', 'prepare', 'seal-parent', 'review-parent', 'apply-child', 'wave-status', 'finalize-parent', 'reverse-parent'];
14
14
 
15
15
  function decode(segment: string): Uint8Array<ArrayBuffer> {
16
16
  if (!/^[A-Za-z0-9_-]+$/.test(segment)) fail();
@@ -80,14 +80,28 @@ function validateGrant(value: Record<string, unknown>, config: Readonly<OwnerCla
80
80
  || !quota(value.maxBytes, 300_000_000) || !quota(value.maxChunks, 1024) || !quota(value.maxRecords, 1_000_000)) fail();
81
81
 
82
82
  const catchup = Object.hasOwn(value, 'catchup');
83
- if (catchup && (value.catchup !== 'bounded' || (value.action !== 'plan' && value.action !== 'apply'))) fail();
83
+ if (catchup && (value.catchup !== 'bounded' || (value.action !== 'plan' && value.action !== 'apply' && !(value.bounded && ['prepare','apply-child'].includes(value.action as string))))) fail();
84
84
  if (catchup && !approved(value)) fail();
85
- if (value.action === 'apply') {
85
+ const waveActions=['prepare','seal-parent','review-parent','apply-child','wave-status','finalize-parent','reverse-parent'];
86
+ const wave=waveActions.includes(value.action as string);
87
+ if (wave !== Object.hasOwn(value,'bounded')) fail();
88
+ if (wave) {
89
+ const b=value.bounded;
90
+ if (!object(b) || b.protocol!=='rer-bounded-waves/1' || !['migration','revision','policy'].every(k=>typeof b[k]==='string' && /^[A-Za-z0-9_-]{1,128}$/.test(b[k] as string))
91
+ || Object.keys(b).some(k=>!['protocol','migration','revision','policy','parentFingerprint','childFingerprint','parentApprovalRecordId','parentApprovedAt'].includes(k))
92
+ || ['parentFingerprint','childFingerprint'].some(k=>Object.hasOwn(b,k) && (typeof b[k]!=='string' || !/^[a-f0-9]{64}$/.test(b[k] as string)))) fail();
93
+ if (['apply-child','finalize-parent','reverse-parent'].includes(value.action as string)
94
+ && (!b.parentFingerprint || typeof b.parentApprovalRecordId!=='string' || !b.parentApprovalRecordId.trim() || b.parentApprovalRecordId.length>256
95
+ || typeof b.parentApprovedAt!=='string' || !Number.isFinite(Date.parse(b.parentApprovedAt)) || new Date(b.parentApprovedAt).toISOString()!==b.parentApprovedAt)) fail();
96
+ if (value.action==='apply-child' && (!b.childFingerprint || value.fingerprint!==b.childFingerprint)) fail();
97
+ if (value.action==='review-parent' && (value.fingerprint!==b.parentFingerprint || value.approvalRecordId!==b.parentApprovalRecordId || value.approvedAt!==b.parentApprovedAt)) fail();
98
+ }
99
+ if (value.action === 'apply' || ['review-parent','apply-child'].includes(value.action as string)) {
86
100
  if (typeof value.fingerprint !== 'string' || !/^[0-9a-f]{64}$/.test(value.fingerprint) || !approved(value)) fail();
87
101
  } else {
88
102
  if (Object.hasOwn(value, 'fingerprint')) fail();
89
103
  const hasApproval = Object.hasOwn(value, 'approvalRecordId') || Object.hasOwn(value, 'approvedAt');
90
- if (hasApproval && (value.action !== 'plan' || !catchup || !approved(value))) fail();
104
+ if (hasApproval && (!['plan','prepare'].includes(value.action as string) || !catchup || !approved(value))) fail();
91
105
  }
92
106
  return Object.freeze(value) as unknown as Readonly<DoorGrant>;
93
107
  }
@@ -11,7 +11,10 @@ import { archive, restoreCheck } from './archive.ts';
11
11
  import { streamUpload } from './intake-upload.ts';
12
12
  import { intakeIdentity } from './intake-policy.ts';
13
13
 
14
- const routes = {uploads,plan,apply,status,reconcile,undo,cancel,archive,'restore-check':restoreCheck};
14
+ import { prepare } from './waves/preparation.ts';
15
+ import { seal, reviewParent, applyChild, waveStatus, finalize, reverseParent } from './waves/lifecycle.ts';
16
+
17
+ const routes = {prepare,'seal-parent':seal,'review-parent':reviewParent,'apply-child':applyChild,'wave-status':waveStatus,'finalize-parent':finalize,'reverse-parent':reverseParent,uploads,plan,apply,status,reconcile,undo,cancel,archive,'restore-check':restoreCheck};
15
18
  const headers = {'Cache-Control':'no-store','Content-Type':'application/json','X-Content-Type-Options':'nosniff'};
16
19
  async function readInput(request: Request): Promise<Record<string,unknown>> {
17
20
  if (request.headers.get('Content-Type')?.split(';')[0] !== 'application/json' || !request.body) throw new DoorError('json_body_required',400);
@@ -46,6 +49,12 @@ export function runtimeOptions<T extends RuntimeConfiguration>(options: T): T &
46
49
  || (options.format === 'raw' ? typeof options.readRaw !== 'function' || options.intake !== undefined
47
50
  : typeof options.parse !== 'function')) throw new DoorError('invalid_configuration',503);
48
51
  if (options.intake !== undefined) intakeIdentity(options.intake);
52
+ const pkg = options.package;
53
+ if (pkg && (options.intake || !Array.isArray(pkg.formats) || !pkg.formats.length || pkg.formats.some(f=>typeof f!=='string' || !f.trim())
54
+ || !Number.isSafeInteger(pkg.maxBytes) || pkg.maxBytes < 1 || !Number.isSafeInteger(pkg.maxSegments) || pkg.maxSegments < 1 || typeof pkg.read !== 'function')) throw new DoorError('invalid_configuration',503);
55
+ if (options.bounded && (!pkg || options.bounded.protocol !== 'rer-bounded-waves/1'
56
+ || options.bounded.batchSize!==undefined && (!Number.isSafeInteger(options.bounded.batchSize) || options.bounded.batchSize<1 || options.bounded.batchSize>20)
57
+ || ['describe','page','native','authority'].some(key=>typeof (options.bounded as any)[key]!=='function'))) throw new DoorError('invalid_configuration',503);
49
58
  return options as T & ImportDoorRuntime;
50
59
  }
51
60
  /** Server-side mount only. A null response belongs to another dispatcher. Every
@@ -78,6 +87,8 @@ export function createDoorHandler(prefix: string, database: D1Database,
78
87
  const input = await readInput(request);
79
88
  admitted = await review(admitted,input);
80
89
  if (['status','reconcile','undo','cancel'].includes(action) && Object.keys(input).some(k=>k!=='planId')) throw new DoorError('invalid_input',400);
90
+ if (['seal-parent','review-parent','wave-status','finalize-parent','reverse-parent'].includes(action) && Object.keys(input).some(k=>k!=='parentId')
91
+ || action==='apply-child' && Object.keys(input).some(k=>!['parentId','ordinal','reviewedFingerprint'].includes(k))) throw new DoorError('invalid_input',400);
81
92
  const result = await routes[action](admitted,input);
82
93
  await authorized(admitted);
83
94
  return Response.json(result,{headers});
@@ -18,3 +18,9 @@ export function createImportDoorHandler(options: ImportDoorOptions): (request: R
18
18
  return createDoorHandler('/api/rer/import/', options.database, (request, action, database) => admit({...options, database}, request, action),
19
19
  undefined, async database => !!options.claim?.publicKeys && Object.keys(options.claim.publicKeys).length > 0 && await enabled({...options, database}));
20
20
  }
21
+
22
+ export { importDoorBoundedWavesMigration } from './waves/schema.ts';
23
+ export type * from './waves/types.ts';
24
+ export { revokeImportDoorWaveApproval } from './waves/store.ts';
25
+ export { waveInventoryHash, inventoryMember } from './waves/digests.ts';
26
+ export type {WaveInventoryInput, WaveInventoryMember} from './waves/digests.ts';