@runeditrun/rer 0.2.118 → 0.2.120

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/cli/README.md +40 -0
  2. package/cli/business/agent-args.mjs +60 -0
  3. package/cli/business/agent-help.mjs +53 -0
  4. package/cli/business/agent-output.mjs +38 -0
  5. package/cli/business/agent-table.mjs +32 -0
  6. package/cli/business/agent-verbs.mjs +132 -0
  7. package/cli/business/agent-wait.mjs +28 -0
  8. package/cli/business/command.mjs +18 -2
  9. package/cli/business/mcp-bridge.mjs +58 -0
  10. package/cli/business/questions.mjs +32 -9
  11. package/cli/business/run-child.mjs +42 -0
  12. package/cli/business/run-grant.mjs +65 -0
  13. package/cli/business/run-mask.mjs +21 -0
  14. package/cli/deploy-command.mjs +13 -7
  15. package/cli/deploy-inputs.mjs +1 -4
  16. package/cli/deploy-settings.mjs +15 -9
  17. package/cli/deploy.mjs +1 -1
  18. package/cli/managed-data-schema.mjs +2 -0
  19. package/cli/portability-worker.mjs +3 -1
  20. package/cli/portability.mjs +1 -1
  21. package/global/CHANGELOG.md +34 -0
  22. package/modules/portability/README.md +8 -0
  23. package/modules/portability/archive-step.mjs +11 -0
  24. package/modules/portability/export.mjs +34 -25
  25. package/modules/portability/import-safety.mjs +4 -3
  26. package/modules/portability/import-staging.mjs +4 -4
  27. package/modules/portability/import.mjs +14 -5
  28. package/modules/portability/index.mjs +2 -1
  29. package/modules/portability/object-inventory.mjs +45 -0
  30. package/modules/portability/storage.mjs +4 -3
  31. package/package.json +1 -1
  32. package/scripts/publish-check/README.md +5 -1
  33. package/scripts/publish-check/checksum_maps.py +9 -5
  34. package/scripts/publish-check/hit_location.py +20 -9
  35. package/scripts/publish-check/json_identity.py +15 -14
@@ -0,0 +1,65 @@
1
+ import { randomUUID } from 'node:crypto';
2
+ import { Api, identifier } from './api.mjs';
3
+ import { paired } from './local-commands.mjs';
4
+ import { runChild, signalCode } from './run-child.mjs';
5
+
6
+ const route = '/api/business/executors/cloudflare-token';
7
+ const usage = 'Usage: rer business run --grant G -- <command> [args...]\n';
8
+ function validLease(value, state, grant, id) {
9
+ return value?.lease_id === id && value.grant_id === grant && value.business_id === state.businessId && value.executor_id === state.executorId
10
+ && /^[a-f0-9]{32}$/.test(value.account_id) && typeof value.credential === 'string' && value.credential.length >= 16 && value.credential.length <= 4096
11
+ && !/[\s\x00-\x1f\x7f]/.test(value.credential) && value.cleanup_status === 'active' && value.provider_credential_revoked === false
12
+ && Number.isFinite(Date.parse(value.credential_expires_at)) && Date.parse(value.credential_expires_at) > Date.now()
13
+ && Date.parse(value.credential_expires_at) <= Date.now() + 900_000;
14
+ }
15
+ async function cleanup(api, id) {
16
+ try {
17
+ const value = await api.post(`${route}/${id}/revoke`, {}, { quiet: true, timeout: 15000 });
18
+ if (value?.lease_id === id && ['revoked', 'expired'].includes(value.cleanup_status)
19
+ && (value.cleanup_status !== 'revoked' || value.provider_credential_revoked === true)) return value.cleanup_status;
20
+ } catch { /* A lost acknowledgement is unconfirmed, never success. */ }
21
+ return 'unconfirmed';
22
+ }
23
+ export async function cleanupCommand(args) {
24
+ if (args.length < 1 || args.length > 2 || args[1] !== undefined && args[1] !== '--status' || !/^[a-f0-9-]{36}$/.test(args[0])) {
25
+ process.stderr.write('Usage: rer business cleanup LEASE [--status]\n'); return 2;
26
+ }
27
+ const {state, credential} = await paired(), api = new Api(state.origin, credential), id = args[0];
28
+ try {
29
+ const value = args[1] ? await api.request('GET', `${route}/${id}`, undefined, { quiet: true }) : null;
30
+ const status = args[1] ? value?.lease_id === id && ['issuing','active','revoked','unconfirmed','expired'].includes(value.cleanup_status) ? value.cleanup_status : 'unconfirmed' : await cleanup(api, id);
31
+ process.stderr.write(`Cloud credential cleanup: ${status}. Lease ${id}.\n`);
32
+ return ['revoked','expired'].includes(status) ? 0 : 1;
33
+ } catch { process.stderr.write(`Cloud credential cleanup: unconfirmed. Lease ${id}.\n`); return 1; }
34
+ }
35
+ export async function grantCommand(args) {
36
+ if (args[0] !== '--grant' || !args[1] || args[2] !== '--' || !args[3]) { process.stderr.write(usage); return 2; }
37
+ try { identifier(args[1]); } catch { process.stderr.write(usage); return 2; }
38
+ const {state, credential} = await paired(), api = new Api(state.origin, credential), id = randomUUID();
39
+ const stop = new AbortController(), handlers = new Map();
40
+ for (const signal of ['SIGINT','SIGTERM','SIGHUP']) {
41
+ const handler = () => { if (!stop.signal.aborted) stop.abort({signal}); };
42
+ handlers.set(signal, handler); process.on(signal, handler);
43
+ }
44
+ let lease, code = 1, exitSignal;
45
+ try {
46
+ // No automatic retry: a lost response may already have minted a child. Cleanup uses the preselected lease id.
47
+ lease = await api.post(route, {grant_id: args[1], lease_id: id}, {quiet: true, signal: stop.signal});
48
+ if (!validLease(lease, state, args[1], id) || lease.credential === credential) throw Error('invalid credential exchange');
49
+ const result = await runChild(args.slice(3), lease, {signal: stop.signal});
50
+ code = result.code; exitSignal = result.signal;
51
+ } catch {
52
+ code = stop.signal.aborted ? signalCode(stop.signal.reason.signal) : 1;
53
+ process.stderr.write('Business command refused or its credential exchange failed. Check pairing, selected account, deploy action, grant and Cloudflare consent in Business.\n');
54
+ } finally {
55
+ const status = await cleanup(api, id);
56
+ const expiry = lease && Number.isFinite(Date.parse(lease.credential_expires_at)) ? new Date(lease.credential_expires_at).toISOString() : 'the server-enforced expiry (at most 15 minutes)';
57
+ process.stderr.write(`Cloud credential cleanup: ${status}. Lease ${id}.${status === 'unconfirmed' ? ` Credential expiry: ${expiry}. Retry: rer business cleanup ${id}.` : ''}\n`);
58
+ lease = undefined;
59
+ for (const [signal, handler] of handlers) process.removeListener(signal, handler);
60
+ }
61
+ // Keep the child's exit status even if cleanup is unconfirmed; cleanup has its own explicit status above.
62
+ exitSignal ??= stop.signal.reason?.signal;
63
+ if (exitSignal && process.platform !== 'win32') { process.kill(process.pid, exitSignal); await new Promise(resolve => setImmediate(resolve)); }
64
+ return code;
65
+ }
@@ -0,0 +1,21 @@
1
+ import { StringDecoder } from 'node:string_decoder';
2
+
3
+ /** Keep a possible credential prefix across writes; neither chunk boundaries nor EOF release it. */
4
+ export function commandMask(values, output) {
5
+ const secrets = [...new Set(values.filter(value => typeof value === 'string' && value).flatMap(value =>
6
+ [value, encodeURIComponent(value), Buffer.from(value).toString('base64'), Buffer.from(value).toString('base64url')]))].sort((a, b) => b.length - a.length);
7
+ const decoder = new StringDecoder('utf8');
8
+ let pending = '';
9
+ function replace(text) { for (const secret of secrets) text = text.split(secret).join('[credential]'); return text; }
10
+ function write(chunk) {
11
+ pending = replace(pending + decoder.write(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)));
12
+ let held = 0;
13
+ for (const secret of secrets) for (let count = Math.min(secret.length - 1, pending.length); count > held; count--)
14
+ if (pending.endsWith(secret.slice(0, count))) { held = count; break; }
15
+ const ready = pending.slice(0, pending.length - held);
16
+ pending = pending.slice(pending.length - held);
17
+ if (ready) output.write(ready);
18
+ }
19
+ return { write, end() { pending = replace(pending + decoder.end()); if (pending) output.write('[credential]'); pending = ''; },
20
+ text: replace };
21
+ }
@@ -26,6 +26,10 @@ import { keepHookSecrets } from './deploy-secrets.mjs';
26
26
  import { withoutLocalSettings, refuseLocalEnvOutput } from './local-env.mjs';
27
27
  import { deployProductDirectory, deployProductArguments, DeployProductError } from './deploy-product.mjs';
28
28
 
29
+ // A hook may wrap this controller. Its needs return to the caller, which renders
30
+ // the combined dry-run summary once after every local plan step has succeeded.
31
+ const deferDryRunSummary = Symbol('defer dry-run summary');
32
+
29
33
  export function validateDeployOptions(options) {
30
34
  const changes = [options.remove && '--remove', options.setSecret && '--set-secret', options.unsetSecret && '--unset-secret', options.rotateSecret && '--rotate-secret', options.reopenInstallationKey && '--reopen-installation-key'].filter(Boolean);
31
35
  if (changes.length > 1 || changes.length && options.dryRun) throw invalid(`Use ${changes.join(', ')}${options.dryRun ? ' and --dry-run' : ''} on separate runs.`);
@@ -88,16 +92,18 @@ export async function deployCommand(context) {
88
92
  const plan = result?.plan ?? (!planned ? undefined : (await owner(['local-plan', '--declaration', path, '--environment', options.environment ?? 'production', ...await deployProductArguments(root, deployProductDirectory(declaration))], production.env, { cwd: root, localOnlyKeys })).plan);
89
93
  result = { ...result, dryRun: true, deployed: false, providerChecked: false, plan };
90
94
  const { signIn, mailOff } = await requireProductionAuth({ root, declaration, env: production.env, setupManifest, setupSeed: context.setupSeed, adapter, declarationPath: path, options }) ?? {};
91
- if (mailOff) store.report(mailOffNotice(mailOff, options.environment ?? 'production'), { names: mailOff.credentialNames });
92
- // The one completion line: rer's own plan step (or its deliberate skip) succeeded. A hook's own line says only that the hook finished.
93
- store.report('deploy: local dry run complete; no credentials were needed, no Cloudflare requests were made and nothing was changed remotely. Target secrets and remote readiness were not checked.');
94
95
  const needed = [...new Set([...neededForRealDeploy({ declaration, env: production.env, signIn, secretsChecked: false }), ...(result.neededForRealDeploy ?? []), ...(extraNeeded ?? [])])];
95
- reportNeeded(store, needed);
96
- if (declaration.secrets?.some(item => item.binding === 'INSTALLATION_KEY')) store.report('deploy: INSTALLATION_KEY is a private generated binding; INSTALLATION_KEY_PREVIOUS is optional and managed only during rotation or an explicit retired-key reopen. Values are withheld.');
96
+ if (!context[deferDryRunSummary]) {
97
+ if (mailOff) store.report(mailOffNotice(mailOff, options.environment ?? 'production'), { names: mailOff.credentialNames });
98
+ // The one completion line: rer's own plan step (or its deliberate skip) succeeded. A hook's own line says only that the hook finished.
99
+ store.report('deploy: local dry run complete; no credentials were needed, no Cloudflare requests were made and nothing was changed remotely. Target secrets and remote readiness were not checked.');
100
+ reportNeeded(store, needed);
101
+ if (declaration.secrets?.some(item => item.binding === 'INSTALLATION_KEY')) store.report('deploy: INSTALLATION_KEY is a private generated binding; INSTALLATION_KEY_PREVIOUS is optional and managed only during rotation or an explicit retired-key reopen. Values are withheld.');
102
+ }
97
103
  return needed.length && result && typeof result === 'object' ? { ...result, neededForRealDeploy: needed } : result;
98
104
  };
99
105
  const runHook = hookContext => options.dryRun ? withLocalDeployPlan(async () => {
100
- try { return await adapter.deploy(hookContext); }
106
+ try { return await adapter.deploy({ ...hookContext, [deferDryRunSummary]: true }); }
101
107
  catch (error) {
102
108
  // Production-only owner/input preflights may refuse before the hook builds.
103
109
  // An offline plan defers those inputs, retaining the refusal as guidance.
@@ -123,7 +129,7 @@ export async function deployCommand(context) {
123
129
  const redaction = createRedaction();
124
130
  const kinds = await settingKinds({ declaration, setupManifest });
125
131
  const addSecret = value => { redaction.addSecrets([value]); store.addSecrets?.([value]); };
126
- for (const value of [...kinds.privateValues(env), env.CLOUDFLARE_ACCOUNT_ID, ...(declaration.secrets ?? []).map(item => env[item.environment])].filter(Boolean)) addSecret(value);
132
+ for (const value of kinds.privateValues(env)) addSecret(value);
127
133
  const environment = options.environment ?? 'production';
128
134
  // The hook uploads every value it is given: a generated secret it gets is the one the Worker keeps.
129
135
  if (!options.dryRun) await keepHookSecrets({ root, environment, declaration, setupManifest, env, generated, reused, rotated, again: deployAgain(options), report: (line, more) => store.report(line, more) });
@@ -77,11 +77,8 @@ export async function withDeployEnvironment(project, options, out, run) {
77
77
  const { declaredSettings, settingKinds, undeclaredNotice } = await import('./deploy-settings.mjs');
78
78
  const declared = await declaredSettings(project), kinds = await settingKinds(declared);
79
79
  const redaction = createRedaction();
80
- // Provider names are public enum values. Masking "resend" would also hide
81
- // RESEND_API_KEY in the preflight's instruction for the missing credential.
82
- const publicProvider = (key, value) => key === 'MAIL_PROVIDER' && /^(resend|sendgrid|cloudflare|rer|later|none)$/.test(value);
83
80
  const supplied = Object.entries(env).filter(([key, value]) => saved[key] !== value);
84
- redaction.addSecrets([...kinds.privateValues(env), ...supplied.filter(([key, value]) => !kinds.shown(key, value) && !publicProvider(key, value)).map(([, value]) => value)]);
81
+ redaction.addSecrets([...kinds.privateValues(env), ...supplied.filter(([key, value]) => !kinds.shown(key, value)).map(([, value]) => value)]);
85
82
  const undeclared = declared.declaration ? kinds.undeclared(supplied.map(([key]) => key)) : [];
86
83
  if (undeclared.length) {
87
84
  const message = undeclaredNotice(undeclared, declared.path);
@@ -3,24 +3,30 @@ import { shared } from './shared.mjs';
3
3
  // What a deployment's declaration says each value is. A value prints in full only
4
4
  // when the declaration names it a non-secret setting: a declared var, the origin
5
5
  // variable and its setting, a setup field that is not a secret, or a name in
6
- // "publicSettings". Anything the declaration calls a secret, any credential-named
7
- // key and any other --env-file value stays redacted.
6
+ // "publicSettings". Secret transport alone does not make an explicitly public
7
+ // setting private. Credential bindings, private setup fields and undeclared
8
+ // --env-file values stay redacted.
8
9
  const PRIVATE_KEYS = ['CLOUDFLARE_ACCOUNT_ID'];
9
10
 
10
11
  export async function settingKinds({ declaration = {}, setupManifest } = {}) {
11
- const { credentialName, credentialUrl, credentialValues } = await shared('redaction.mjs');
12
+ const { credentialName, credentialValue, credentialValues } = await shared('redaction.mjs');
12
13
  const secrets = Array.isArray(declaration.secrets) ? declaration.secrets : [];
13
14
  const names = list => (Array.isArray(list) ? list : []).filter(name => typeof name === 'string');
14
15
  const fields = Array.isArray(setupManifest?.fields) ? setupManifest.fields : [];
15
- const secret = new Set([...PRIVATE_KEYS, ...secrets.flatMap(item => [item?.binding, item?.environment]), ...fields.filter(field => field?.kind === 'secret').map(field => field.name)].filter(Boolean));
16
- const open = new Set([...Object.keys(declaration.vars ?? {}), declaration.originVar, declaration.publicOriginSetting, ...names(declaration.publicSettings), ...fields.filter(field => field?.kind !== 'secret').map(field => field?.name)].filter(name => name && !secret.has(name)));
17
- // A declared setting is exempt from the key-name heuristic (AUTH_ORIGIN is a
18
- // name, not a credential), never when its value carries credentials itself.
19
- const shown = (key, value) => open.has(key) && typeof value === 'string' && !credentialUrl(value);
16
+ const privateFields = new Set([...PRIVATE_KEYS, ...fields.filter(field => field?.kind === 'secret').map(field => field.name)]);
17
+ // Select fields describe public choices. Other transported inputs need an
18
+ // explicit publicSettings declaration; an owner email is still private.
19
+ const publicTransport = new Set([...names(declaration.publicSettings), ...fields.filter(field => field?.kind === 'select').map(field => field.name)]);
20
+ const publicBindings = secrets.filter(item => publicTransport.has(item?.binding) &&
21
+ [item.binding, item.environment].every(name => name && !privateFields.has(name) && !credentialName(name)));
22
+ const secret = new Set([...privateFields, ...secrets.filter(item => !publicBindings.includes(item)).flatMap(item => [item?.binding, item?.environment])].filter(Boolean));
23
+ const open = new Set([...Object.keys(declaration.vars ?? {}), declaration.originVar, declaration.publicOriginSetting, ...names(declaration.publicSettings), ...fields.filter(field => field?.kind !== 'secret').map(field => field?.name), ...publicBindings.flatMap(item => [item.binding, item.environment])].filter(name => name && !secret.has(name)));
24
+ // Public names never exempt a credential-shaped value from masking.
25
+ const shown = (key, value) => open.has(key) && typeof value === 'string' && !credentialValue(value);
20
26
  return {
21
27
  shown,
22
28
  // The values to keep out of output, from every key that is not shown.
23
- privateValues: env => Object.entries(env).flatMap(([key, value]) => typeof value !== 'string' || shown(key, value) ? [] : secret.has(key) ? (value ? [value] : []) : credentialValues({ [key]: value })),
29
+ privateValues: env => Object.entries(env).flatMap(([key, value]) => typeof value !== 'string' || shown(key, value) ? [] : (secret.has(key) || credentialValue(value)) ? (value ? [value] : []) : credentialValues({ [key]: value })),
24
30
  // Keys the deploy hides by default and the person may declare.
25
31
  undeclared: keys => keys.filter(key => !open.has(key) && !secret.has(key) && !credentialName(key)),
26
32
  };
package/cli/deploy.mjs CHANGED
@@ -61,7 +61,7 @@ export function ownerDeploy(declarationPath, lifecycle) {
61
61
  const { main } = await import('../modules/deploy-kit/owner-cli.mjs');
62
62
  const { createRedaction } = await shared('redaction.mjs');
63
63
  const redaction = createRedaction();
64
- redaction.addSecrets([...(await settingKinds({ declaration, setupManifest })).privateValues(env), env.CLOUDFLARE_ACCOUNT_ID, ...(declaration.secrets ?? []).map(item => env[item.environment]).filter(Boolean)]);
64
+ redaction.addSecrets((await settingKinds({ declaration, setupManifest })).privateValues(env));
65
65
  const claim = setupManifest?.steps?.find(item => item.type === 'owner-claim');
66
66
  const declared = binding => (Array.isArray(declaration.secrets) ? declaration.secrets : []).find(item => item?.binding === binding);
67
67
  const owner = async (command, extra = [], values = env, more = {}) => main([command, '--declaration', declarationPath, '--environment', environment, ...(['dry-run', 'local-plan', 'deploy', 'health'].includes(command) ? await deployProductArguments(root, productDirectory) : []), ...extra], values, { cwd: root, localOnlyKeys, ...more });
@@ -5,6 +5,7 @@ import { excludedTable } from '../modules/portability/catalogue.mjs';
5
5
  import { schemaSQL } from '../modules/portability/schema-identity.mjs';
6
6
  import { ensurePortabilitySchema, portabilityPlan } from '../modules/container/portability-schema.mjs';
7
7
  import { ensureImportLeases } from '../modules/portability/import-lease.mjs';
8
+ import { ensureObjectInventory } from '../modules/portability/object-inventory.mjs';
8
9
  import { splitStatements } from '../modules/portability/sql-text.mjs';
9
10
  import { createPortability } from '../modules/portability/index.mjs';
10
11
  import { migrationReplay } from '../modules/portability/migration-history.mjs';
@@ -55,6 +56,7 @@ export async function managedSchema(db, plan, policy, ledger) {
55
56
  const native = nativeDatabase(scratch);
56
57
  await ensurePortabilitySchema(native, await portabilityPlan(splitStatements));
57
58
  await ensureImportLeases(native);
59
+ await ensureObjectInventory(native);
58
60
  previous = new Map((await all(native, query)).map(object => [key(object), object]));
59
61
  const inventory = [], groups = new Map();
60
62
  const account = (object, disposition, migration) => {
@@ -1,3 +1,4 @@
1
+ import { failureStep } from '../modules/portability/archive-step.mjs';
1
2
  import { installD1Preflight } from '../modules/container/adapters/d1-workerd-setup.mjs';
2
3
  import { portabilityPlan, ensurePortabilitySchema } from '../modules/container/portability-schema.mjs';
3
4
  import { exportArchive, importArchive, fileRecords } from '../modules/portability/files.mjs';
@@ -94,7 +95,8 @@ catch (error) {
94
95
  const identityTable = error instanceof PortabilityError && error.code.startsWith('identity_table_undeclared:') ? error.code.slice('identity_table_undeclared:'.length) : undefined;
95
96
  const safety = error instanceof PortabilityError && error.code.startsWith('import_row_') ? error.code.slice(error.code.indexOf(':')+1) : undefined;
96
97
  // Anything that is not a portability refusal names its class and code, never its message: that can hold row data.
98
+ const step=failureStep(error) ?? (process.argv[2] === 'import' ? undefined : process.argv[2]);
97
99
  const native = error instanceof PortabilityError || error.message === 'archive_json' ? undefined : nativeFailure(error);
98
100
  process.exitCode = exitCode;
99
- await reply({ ok: false, code: exitCode === 2 ? 'invalid_archive' : exitCode === 4 ? 'restore_refused' : 'transfer_failed', reason, ...(reExport ? { reExport: true } : {}), ...(migration ? { migration } : {}), ...(unapplied ? { unapplied } : {}), ...(occupied ? { occupied } : {}), ...(identityTable ? { identityTable } : {}), ...(safety ? { safety } : {}), ...(native ? { native } : {}), exitCode });
101
+ await reply({ ok: false, code: exitCode === 2 ? 'invalid_archive' : exitCode === 4 ? 'restore_refused' : 'transfer_failed', reason, step, ...(reExport ? { reExport: true } : {}), ...(migration ? { migration } : {}), ...(unapplied ? { unapplied } : {}), ...(occupied ? { occupied } : {}), ...(identityTable ? { identityTable } : {}), ...(safety ? { safety } : {}), ...(native ? { native } : {}), exitCode });
100
102
  }
@@ -110,7 +110,7 @@ async function nativeWorker(store,direction,input,signal) {
110
110
  : receipt.reason?.startsWith('migration_ledger:') ? ledgerRefusal(receipt)
111
111
  : receipt.migration ? `Restore refused (schema_mismatch:migration_data): migration ${receipt.migration} changed data after this archive was made, so its rows would skip that change. Restore an archive made after it.`
112
112
  : receipt.occupied ? occupiedRefusal(input,receipt.reason)
113
- : `The portability module refused or failed the archive operation (${receipt.reason ?? 'incomplete'}${receipt.native ? `: ${receipt.native.class}${receipt.native.code && receipt.native.code !== receipt.reason ? `, code ${receipt.native.code}` : ''}` : ''}).${receipt.native && receipt.exitCode === 1 ? ' Run the same command again; it starts over.' : ''}`, receipt.exitCode ?? 1);
113
+ : `The portability module refused or failed the archive operation (${receipt.reason ?? 'incomplete'}${receipt.native ? `: ${receipt.native.class}${receipt.native.code && receipt.native.code !== receipt.reason ? `, code ${receipt.native.code}` : ''}` : ''})${receipt.step ? ` during ${receipt.step}` : ''}.${receipt.native && receipt.exitCode === 1 ? ' Run the same command again; it starts over.' : ''}`, receipt.exitCode ?? 1);
114
114
  delete receipt.ok;
115
115
  return receipt;
116
116
  }
@@ -1,5 +1,39 @@
1
1
  # Changelog
2
2
 
3
+ ## 0.2.120
4
+
5
+ ### Commands under an approved Business grant
6
+
7
+ - `rer business run --grant G -- <command> [args...]` exchanges this computer's pairing and explicit grant for a short-lived scoped Cloudflare child credential. It removes inherited Cloudflare credential aliases, passes arguments without a shell and preserves the child's exit code or Unix terminating signal. Existing `run [--once]` check-ins remain available.
8
+ - The wrapper masks the credential and common encoded forms in output, interrupts the command at credential expiry and requests cleanup after completion or interruption. Cleanup reports revoked, expired or unconfirmed separately from the child result; `rer business cleanup LEASE [--status]` retries revocation or reads its status. This environment boundary is not a sandbox: run trusted software, and an unconfirmed cleanup can leave the credential usable until server expiry.
9
+
10
+ ### Agent commands and stdio MCP
11
+
12
+ - Paired agents can call Business tools for installed apps, changes, updates, domains, imports, profiles, plans and owner sign-in actions, subject to the server's available tools and the owner's grants. `rer business tools` discovers tools and `rer business call TOOL` reaches tools without a dedicated verb. Flags accept structured JSON or file input; `--json` returns structured results.
13
+ - Change and update following supports bounded waits. A change request can wait for its preview and, with explicit `--publish`, request publication and wait for live. Refusals, unavailable tools and unfinished waits retain distinct outcomes and actionable guidance.
14
+ - `rer business mcp` forwards newline-delimited JSON-RPC over stdio to the Business MCP. The paired client refreshes its short-lived bearer near expiry and once after a 401; protocol output stays on stdout and diagnostics on stderr.
15
+
16
+ These CLI changes require the matching Business server, pairing, owner approvals and grants. This preparation establishes no live-provider, Managed deployment, seed adoption or production result. The actual source diff changes no portability, import-kit, product migrations or archive format identity; the browser advice change is not included.
17
+
18
+ ## 0.2.119
19
+
20
+ ### Large installation archive inventories
21
+
22
+ - Archive export and restore store large object inventories in bounded internal rows instead of one oversized job-metadata value, addressing the D1 cell limit reached by a company-copy export with 13,879 objects. Small jobs keep their existing metadata representation. Archive parts can group multiple small objects, and restore reuses a shared part while reading its objects. The archive format and version are unchanged.
23
+ - Managed intake accounts for the internal inventory table. Native archive failures name the transfer stage without printing SQL, object keys, row values or filesystem paths.
24
+
25
+ ### Publication scan copy pointers
26
+
27
+ - Publication scans locate Gitleaks full matches within their reported lines when directory-fragment columns are relative to a fragment beginning mid-line. Every possible occurrence must satisfy the existing copy-pointer or checksum content proof; unresolved matches and unproved occurrences retain their finding.
28
+
29
+ ### Deploy output and timing checks
30
+
31
+ - Declared public deployment settings, including `MAIL_PROVIDER`, remain readable even when carried through secret transport. Credential-named bindings and aliases, private setup fields and credential-shaped values stay masked.
32
+ - Nested deploy hooks defer their dry-run completion and needs summary to the outer command, so the combined summary prints once after local planning succeeds.
33
+ - The real 20-second listing-backoff test still uses the native timer and high-resolution timestamps, observes the configured delay and compares elapsed time at Node's millisecond timer precision to avoid a fractional early-wake failure. Runtime backoff behavior is unchanged.
34
+
35
+ These changes become available after adopting the released package or refreshed source. This preparation establishes no company-copy export result, seed adoption, deployment or production result. The actual diff changes portability inventory storage and archive-part grouping, adds no product migrations, changes no import-kit source and preserves the archive format identity.
36
+
3
37
  ## 0.2.118
4
38
 
5
39
  ### Local diagnostics and Move in guidance
@@ -1133,6 +1133,8 @@ modules/portability/sqlite-tokens.mjs
1133
1133
  modules/portability/bookkeeping.mjs
1134
1134
  modules/portability/snapshot-objects.mjs
1135
1135
  modules/portability/storage.mjs
1136
+ modules/portability/object-inventory.mjs
1137
+ modules/portability/archive-step.mjs
1136
1138
  modules/portability/copies.mjs
1137
1139
  modules/portability/derived.mjs
1138
1140
  modules/portability/dev-data.mjs
@@ -1268,3 +1270,9 @@ parameters of at most 2 MiB each. It refuses compressed or encrypted entries,
1268
1270
  unsafe paths, duplicate or unexpected members, missing ledgers, other versions,
1269
1271
  container transport, later-journal and legacy-recovery options. Refusals use
1270
1272
  constant `managed_data_*` reasons and do not print source values.
1273
+
1274
+ Large object inventories are stored in bounded internal rows so D1 cell and row
1275
+ limits do not prevent backups. Native CLI failures name the transfer stage (for
1276
+ example `export_snapshot`) without printing SQL, values or absolute paths. Keep
1277
+ the stopped source and retry after resolving the named stage; changing source
1278
+ objects or deleting data is not a remedy for an inventory storage failure.
@@ -0,0 +1,11 @@
1
+ // Stage names are fixed by the implementation. Never send native messages,
2
+ // SQL, object keys, row values or filesystem paths to the CLI.
3
+ const steps=new Set(['export_object_inventory','export_snapshot','export_part']);
4
+ export async function archiveStep(step,operation) {
5
+ try{return await operation();}
6
+ catch(error){
7
+ if(error && typeof error==='object' && !error.portabilityStep)error.portabilityStep=step;
8
+ throw error;
9
+ }
10
+ }
11
+ export const failureStep=error=>steps.has(error?.portabilityStep)?error.portabilityStep:undefined;
@@ -1,4 +1,6 @@
1
- import { statement, state, guard, job, save, partStatements, parts, readPart, cleanCopies, quote, stream, insist, run } from './storage.mjs';
1
+ import { prepareInventory, metadataJson, inventoryStatements } from './object-inventory.mjs';
2
+ import { archiveStep } from './archive-step.mjs';
3
+ import { statement, state, guard, job, save, partStatements, parts, readPart, cleanCopies, quote, stream, insist } from './storage.mjs';
2
4
  import { catalogue, encodedRow } from './catalogue.mjs';
3
5
  import { checkReferences } from './references.mjs';
4
6
  import { createCopy,readCopy,normalizeCopy } from './copies.mjs';
@@ -16,7 +18,7 @@ export async function createExport(context,actor='direct') {
16
18
  for(const [binding,spec] of Object.entries(config.buckets).sort()) {
17
19
  let cursor;
18
20
  do {
19
- const page=await buckets[binding].list({prefix:spec.prefix??'',cursor,include:['httpMetadata','customMetadata']});
21
+ const page=await archiveStep('export_object_inventory',()=>buckets[binding].list({prefix:spec.prefix??'',cursor,include:['httpMetadata','customMetadata']}));
20
22
  for(const object of page.objects) objects.push({bucket:binding,key:object.key,size:object.size,etag:object.etag,httpMetadata:object.httpMetadata??{},customMetadata:object.customMetadata??{}});
21
23
  cursor=page.truncated?page.cursor:undefined;
22
24
  }while(cursor);
@@ -27,9 +29,10 @@ export async function createExport(context,actor='direct') {
27
29
  const manifest={kind:'manifest',format:FORMAT,version:2,product:config.product,exportedAt:new Date().toISOString(),sourceIdentity:s.source_identity,schema:cat.identity,tables:cat.tables,schemaObjects:cat.schemaObjects,objects,payloads:[],excludedTables:cat.excludedTables,excludedColumns:config.secretColumns,requiredConfiguration:config.requiredConfiguration,partCount:0,epoch:s.epoch,journal:await journal(db,s.source_identity),...(migrations?{migrations}:{}),...(origin?{migrationLedger:origin}:{})};
28
30
  const row={id,kind:'export',actor,phase:'building',completed:0,metadata:{manifest,copies,cursor:{table:0,row:0,binary:0,offset:0},partBytes:config.partBytes}};
29
31
  const audit=context.onExport?context.onExport({id,actor:actor==='direct'?null:actor,scope:'installation'}):[];
30
- await db.batch([guard(db,'EXISTS(SELECT 1 FROM rer_portability_state WHERE id=1 AND applying=0 AND source_identity=? AND epoch=?)',s.source_identity,s.epoch),
31
- ...cat.tables.flatMap(t=>createCopy(db,t,copies[t.name],true)),statement(db,'INSERT INTO rer_portability_jobs(id,kind,actor,phase,metadata) VALUES(?,?,?,?,?)',id,'export',actor,row.phase,canonicalJson(row.metadata)),db.prepare('DELETE FROM rer_portability_guard'),
32
- ...audit.map(([sql,args])=>statement(db,sql,...args))]);
32
+ prepareInventory(row);
33
+ await archiveStep('export_snapshot',()=>db.batch([guard(db,'EXISTS(SELECT 1 FROM rer_portability_state WHERE id=1 AND applying=0 AND source_identity=? AND epoch=?)',s.source_identity,s.epoch),
34
+ ...cat.tables.flatMap(t=>createCopy(db,t,copies[t.name],true)),statement(db,'INSERT INTO rer_portability_jobs(id,kind,actor,phase,metadata) VALUES(?,?,?,?,?)',id,'export',actor,row.phase,metadataJson(row.metadata)),...inventoryStatements(db,row),db.prepare('DELETE FROM rer_portability_guard'),
35
+ ...audit.map(([sql,args])=>statement(db,sql,...args))]));
33
36
  try {
34
37
  await dropDevData(db,cat.tables,copies);
35
38
  await dropProviderData(db,cat.tables,copies);
@@ -38,7 +41,7 @@ export async function createExport(context,actor='direct') {
38
41
  for(const [name,producer] of Object.entries(config.extraPayloads).sort()) {
39
42
  const descriptor=await capturePayload(db,id,manifest.payloads.length,await producer.produce());manifest.payloads.push({name,...descriptor});
40
43
  }
41
- await run(db,'UPDATE rer_portability_jobs SET metadata=? WHERE id=?',canonicalJson(row.metadata),id);
44
+ await save(db,row).run();
42
45
  await checkReferences(context,copies,objects);
43
46
  return id;
44
47
  }catch(e){await abortExport(context,id);throw e;}
@@ -68,24 +71,30 @@ async function nextItems(context,row) {
68
71
  }
69
72
  const entries=[...manifest.objects,...manifest.payloads];
70
73
  if(cursor.binary===entries.length)return null;
71
- const entry=entries[cursor.binary],length=Math.min(entry.size-cursor.offset,Math.max(1,Math.floor((partBytes-200-utf8(canonicalJson({bucket:entry.bucket??null,key:entry.key??null,name:entry.name??null})).length)*3/4)));
72
- let bytes,item;
73
- if(cursor.binary<manifest.objects.length) {
74
- const object=await buckets[entry.bucket].get(entry.key,length?{range:{offset:cursor.offset,length}}:undefined);
75
- insist(object && object.etag===entry.etag,'object_changed:'+entry.bucket+':'+entry.key);
76
- bytes=new Uint8Array(await object.arrayBuffer());insist(bytes.length===length,'object_changed:'+entry.key);
77
- item={kind:'object',bucket:entry.bucket,key:entry.key,offset:cursor.offset,data:base64(bytes)};
78
- }else{
79
- bytes=await payloadRange(db,row.id,cursor.binary-manifest.objects.length,cursor.offset,length);
80
- item={kind:'payload',name:entry.name,offset:cursor.offset,data:base64(bytes)};
74
+ const items=[];let used=200;
75
+ while(cursor.binary<entries.length && items.length<1000 && partBytes-used>512) {
76
+ const entry=entries[cursor.binary],header=utf8(canonicalJson({bucket:entry.bucket??null,key:entry.key??null,name:entry.name??null})).length;
77
+ if(used+200+header>=partBytes){insist(items.length,'binary_part_too_large');break;}
78
+ const length=Math.min(entry.size-cursor.offset,Math.max(1,Math.floor((partBytes-used-200-header)*3/4)));
79
+ let bytes,item;
80
+ if(cursor.binary<manifest.objects.length) {
81
+ const object=await buckets[entry.bucket].get(entry.key,length?{range:{offset:cursor.offset,length}}:undefined);
82
+ insist(object && object.etag===entry.etag,'object_changed:'+entry.bucket+':'+entry.key);
83
+ bytes=new Uint8Array(await object.arrayBuffer());insist(bytes.length===length,'object_changed:'+entry.key);
84
+ item={kind:'object',bucket:entry.bucket,key:entry.key,offset:cursor.offset,data:base64(bytes)};
85
+ }else{
86
+ bytes=await payloadRange(db,row.id,cursor.binary-manifest.objects.length,cursor.offset,length);
87
+ item={kind:'payload',name:entry.name,offset:cursor.offset,data:base64(bytes)};
88
+ }
89
+ const hash=new StreamHash();if(cursor.hash){hash.h=Uint32Array.from(cursor.hash.h);hash.block=Uint8Array.from(cursor.hash.block);hash.size=cursor.hash.size;hash.used=cursor.hash.used;}
90
+ hash.update(bytes);cursor.offset+=length;
91
+ if(cursor.offset===entry.size) {
92
+ const digest=hash.digest();if(entry.sha256)insist(entry.sha256===digest,'payload_changed');entry.sha256=digest;delete entry.etag;
93
+ cursor.binary++;cursor.offset=0;delete cursor.hash;
94
+ }else cursor.hash={h:[...hash.h],block:[...hash.block],size:hash.size,used:hash.used};
95
+ items.push(item);used+=utf8(canonicalJson(item)).length+1;
81
96
  }
82
- const hash=new StreamHash();if(cursor.hash){hash.h=Uint32Array.from(cursor.hash.h);hash.block=Uint8Array.from(cursor.hash.block);hash.size=cursor.hash.size;hash.used=cursor.hash.used;}
83
- hash.update(bytes);cursor.offset+=length;
84
- if(cursor.offset===entry.size) {
85
- const digest=hash.digest();if(entry.sha256)insist(entry.sha256===digest,'payload_changed');entry.sha256=digest;delete entry.etag;
86
- cursor.binary++;cursor.offset=0;delete cursor.hash;
87
- }else cursor.hash={h:[...hash.h],block:[...hash.block],size:hash.size,used:hash.used};
88
- return [item];
97
+ return items;
89
98
  }
90
99
  export async function advanceExport(context,id,{expectedPart,maxParts=1}={}) {
91
100
  insist(Number.isSafeInteger(maxParts) && maxParts>0 && maxParts<=100,'advance_limit');
@@ -94,10 +103,10 @@ export async function advanceExport(context,id,{expectedPart,maxParts=1}={}) {
94
103
  if(row.phase==='complete')return exportStatus(context,id);
95
104
  insist(row.phase==='building','export_phase',409);
96
105
  if(expectedPart!==undefined && row.completed!==expectedPart)return exportStatus(context,id);
97
- const before=row.completed,items=await nextItems(context,row);
106
+ const before=row.completed,binary=row.metadata.cursor.binary,items=await archiveStep('export_part',()=>nextItems(context,row));
98
107
  if(items) {
99
108
  const part=await makePart(before,items),size=await checkPart(part,before,row.metadata.partBytes);row.completed++;row.metadata.bytes=(row.metadata.bytes??0)+size+1;
100
- try{await context.db.batch([guard(context.db,'(SELECT completed FROM rer_portability_jobs WHERE id=?)=?',id,before),...partStatements(context.db,id,part),save(context.db,row),context.db.prepare('DELETE FROM rer_portability_guard')]);}
109
+ try{await context.db.batch([guard(context.db,'(SELECT completed FROM rer_portability_jobs WHERE id=?)=?',id,before),...partStatements(context.db,id,part),...inventoryStatements(context.db,row,binary,row.metadata.cursor.binary),save(context.db,row),context.db.prepare('DELETE FROM rer_portability_guard')]);}
101
110
  catch(e){if((await job(context.db,id)).completed===before)throw e;}
102
111
  if(expectedPart!==undefined)expectedPart++;
103
112
  }else{
@@ -1,3 +1,4 @@
1
+ import { metadataJson } from './object-inventory.mjs';
1
2
  import { encodedRow, validateRow, bindCell } from './catalogue.mjs';
2
3
  import { readCopy, copyRowid } from './copies.mjs';
3
4
  import { canonicalJson } from './hash.mjs';
@@ -44,7 +45,7 @@ const guardSql=`INSERT INTO rer_portability_guard(value) SELECT CASE WHEN (
44
45
  ) THEN 1 ELSE 0 END`;
45
46
  const checkpoint=(row,before,metadata,queries)=>[
46
47
  {sql:guardSql,params:[row.id,before,row.id]},...queries,
47
- {sql:'UPDATE rer_portability_jobs SET phase=?,completed=?,metadata=? WHERE id=?',params:[row.phase,row.completed,canonicalJson(metadata),row.id]},
48
+ {sql:'UPDATE rer_portability_jobs SET phase=?,completed=?,metadata=? WHERE id=?',params:[row.phase,row.completed,metadataJson(metadata),row.id]},
48
49
  {sql:'DELETE FROM rer_portability_guard'},
49
50
  ];
50
51
  // Each page's staged updates, cursor and value-free change counts commit together.
@@ -55,10 +56,10 @@ export async function normalizeStaged(context,row) {
55
56
  const copy=row.metadata.copies[table.name],count=row.metadata.stagedCounts?.[table.name]??0;
56
57
  let offset=row.metadata.safetyProgress?.[table.name]??0;
57
58
  while(offset<count) {
58
- let before=canonicalJson(row.metadata),queries=[];
59
+ let before=metadataJson(row.metadata),queries=[];
59
60
  const commit=async()=>{
60
61
  await db.batch(checkpoint(row,before,row.metadata,queries).map(q=>statement(db,q.sql,...(q.params??[]))));
61
- before=canonicalJson(row.metadata);queries=[];
62
+ before=metadataJson(row.metadata);queries=[];
62
63
  };
63
64
  for(const {row:raw,meta} of await readCopy(db,table,copy,offset,100,true)) {
64
65
  const original=encodedRow(table,raw),after=mapImportRow(config,table,original);
@@ -1,5 +1,5 @@
1
+ import { metadataJson } from './object-inventory.mjs';
1
2
  import { RPC_LIMIT, rpcSize, rpcSizeBytes } from '../d1-transport/index.mjs';
2
- import { canonicalJson } from './hash.mjs';
3
3
  import { statement, save, readPart, insist } from './storage.mjs';
4
4
  import { checkPart } from './format.mjs';
5
5
 
@@ -18,11 +18,11 @@ export async function stagePart(context,row,part,queries,stagedCounts) {
18
18
  insist(!pending || (pending.index===part.index && pending.sha256===part.sha256),'part_conflict',409);
19
19
  let next=pending?.next??0;
20
20
  while(next<queries.length) {
21
- const before=canonicalJson(row.metadata),chunk=[];
21
+ const before=metadataJson(row.metadata),chunk=[];
22
22
  // Reserve the largest cursor for this part before admitting any data.
23
23
  row.metadata.pendingPart={index:part.index,sha256:part.sha256,next:queries.length};
24
24
  const guard={sql:guardSql,params:[row.id,row.completed,before,row.id]};
25
- const checkpoint={sql:'UPDATE rer_portability_jobs SET phase=?,completed=?,metadata=? WHERE id=?',params:[row.phase,row.completed,canonicalJson(row.metadata),row.id]};
25
+ const checkpoint={sql:'UPDATE rer_portability_jobs SET phase=?,completed=?,metadata=? WHERE id=?',params:[row.phase,row.completed,metadataJson(row.metadata),row.id]};
26
26
  let size=rpcSize([guard,checkpoint,{sql:'DELETE FROM rer_portability_guard'}]),end=next;
27
27
  for(;end<queries.length;end++) {
28
28
  const candidate=combine(size,rpcSize([queries[end]]));
@@ -37,7 +37,7 @@ export async function stagePart(context,row,part,queries,stagedCounts) {
37
37
  // Authenticate the reassembled stored bytes before acknowledging the part.
38
38
  const stored=await readPart(db,row.id,part.index);
39
39
  await checkPart(stored,part.index,config.partBytes);insist(stored.sha256===part.sha256,'part_checksum');
40
- const before=canonicalJson(row.metadata),completed=row.completed;
40
+ const before=metadataJson(row.metadata),completed=row.completed;
41
41
  delete row.metadata.pendingPart;row.metadata.stagedCounts=stagedCounts;row.completed++;
42
42
  await db.batch([statement(db,guardSql,row.id,completed,before,row.id),
43
43
  statement(db,'INSERT INTO rer_portability_parts VALUES(?,?,?)',row.id,part.index,part.sha256),
@@ -1,3 +1,4 @@
1
+ import { prepareInventory, metadataJson, inventoryStatements } from './object-inventory.mjs';
1
2
  import { libraryMigrations } from './library-history.mjs';
2
3
  import { all, statement, state, guard, job, save, partQueries, parts, readPart, dropCopyStatements, cleanCopyStatements, quote, stream, insist, run } from './storage.mjs';
3
4
  import { stagePart } from './import-staging.mjs';
@@ -78,11 +79,12 @@ export async function createImport(context,manifest,actor='direct',legacyRecover
78
79
  }
79
80
  const copies=Object.fromEntries(targets.map((t,i)=>[t.name,'rer_portability_stage_'+id.replaceAll('-','')+'_'+i]));
80
81
  const row={id,kind:'import',actor,phase:'staging',completed:0,metadata:{manifest:original,...(unverified?{schemaUnverified:{reason:'acknowledged recovery of a version 2 archive without schemaObjects',sourceIdentity:original.sourceIdentity,exportedAt:original.exportedAt,acknowledgedAt:new Date().toISOString()}}:{}),...(skippedTables.length?{restoreManifest:manifest,skippedTables}:{}),copies,targets,triggers:cat.triggers,...(bridge.fromSchema?{bridge}:{}),...(replays?{replays}:{}),...(legacyRecovery?{legacyRecovery}: {}),...(ledgers?{ledgers}:{})}};
82
+ prepareInventory(row);
81
83
  try {
82
84
  await db.batch([guard(db,'(SELECT import_job FROM rer_portability_state WHERE id=1) IS NULL'),
83
85
  statement(db,'UPDATE rer_portability_state SET import_job=?,applying=? WHERE id=1',id,unverified?1:0),
84
86
  ...targets.flatMap(t=>createCopy(db,t,copies[t.name])),
85
- statement(db,'INSERT INTO rer_portability_jobs(id,kind,actor,phase,metadata) VALUES(?,?,?,?,?)',id,'import',actor,row.phase,canonicalJson(row.metadata)),createImportLease(db,id),db.prepare('DELETE FROM rer_portability_guard')]);
87
+ statement(db,'INSERT INTO rer_portability_jobs(id,kind,actor,phase,metadata) VALUES(?,?,?,?,?)',id,'import',actor,row.phase,metadataJson(row.metadata)),...inventoryStatements(db,row),createImportLease(db,id),db.prepare('DELETE FROM rer_portability_guard')]);
86
88
  }catch(e){const running=(await state(db)).import_job;if(running)throw await importRunning(db,running);throw e;}
87
89
  return id;
88
90
  }
@@ -136,10 +138,10 @@ async function uploadLeasedPart(context,id,index,part) {
136
138
  return importStatus(context,id,{includeLease:false});
137
139
  }
138
140
  const binaryKey=entry=>canonicalJson(entry.bucket===undefined?['payload',entry.name]:['object',entry.bucket,entry.key]);
139
- function binaryStream(context,id,entry,indexes) {
141
+ function binaryStream(entry,indexes,read) {
140
142
  const selected=indexes.get(binaryKey(entry));insist(selected,'binary_inventory');
141
143
  return stream((async function*(){
142
- for(const index of selected)for(const item of (await readPart(context.db,id,index)).items) {
144
+ for(const index of selected)for(const item of (await read(index)).items) {
143
145
  if(entry.bucket===undefined?item.kind==='payload' && item.name===entry.name:item.kind==='object' && item.bucket===entry.bucket && item.key===entry.key)yield unbase64(item.data);
144
146
  }
145
147
  })());
@@ -218,17 +220,24 @@ async function finalizeLeasedImport(context,id,{contentHash,archiveHash,laterJou
218
220
  }
219
221
  row.phase='applying';
220
222
  await db.batch([guard(db,'(SELECT import_job FROM rer_portability_state WHERE id=1)=?',id),db.prepare('UPDATE rer_portability_state SET applying=1 WHERE id=1'),save(db,row),db.prepare('DELETE FROM rer_portability_guard')]);
223
+ // Grouped objects often share a part. Keep one immutable part during this
224
+ // leased finalize, rather than rereading its bytes for every small object.
225
+ let cached;
226
+ const readBinaryPart=async index=>{
227
+ if(cached?.index!==index)cached=await readPart(db,id,index);
228
+ return cached;
229
+ };
221
230
  try {
222
231
  for(const object of manifest.objects) {
223
232
  await run(db,'INSERT OR IGNORE INTO rer_portability_owned VALUES(?,?,?)',id,object.bucket,object.key);
224
- await writeObject(context.buckets[object.bucket],object,binaryStream(context,id,object,binaryIndexes));
233
+ await writeObject(context.buckets[object.bucket],object,binaryStream(object,binaryIndexes,readBinaryPart));
225
234
  const stored=await context.buckets[object.bucket].get(object.key);insist(stored,'object_write_failed');
226
235
  // Read back independently, including multipart results, before the commit.
227
236
  const hash=new (await import('./hash.mjs')).StreamHash();const reader=stored.body.getReader();
228
237
  for(;;){const {value,done}=await reader.read();if(done)break;hash.update(value);}reader.releaseLock();
229
238
  insist(hash.digest()===object.sha256,'object_write_checksum');
230
239
  }
231
- for(const payload of manifest.payloads)await config.extraPayloads[payload.name].consume(binaryStream(context,id,payload,binaryIndexes),payload);
240
+ for(const payload of manifest.payloads)await config.extraPayloads[payload.name].consume(binaryStream(payload,binaryIndexes,readBinaryPart),payload);
232
241
  const derived=await rebuildDerivedStatements(db,config);
233
242
  row.phase='complete';row.metadata.contentHash=digest;
234
243
  if(row.metadata.schemaUnverified)row.metadata.schemaUnverified.archiveHash=archiveHash;
@@ -1,3 +1,4 @@
1
+ import { ensureObjectInventory } from './object-inventory.mjs';
1
2
  import { ensureImportLeases } from './import-lease.mjs';
2
3
  import { policy, catalogue } from './catalogue.mjs';
3
4
  import { wireDatabase } from '../d1-transport/index.mjs';
@@ -14,7 +15,7 @@ export { createPortabilityHandler } from './http.mjs';
14
15
  export async function createPortability({db,buckets={},policy:input,migrations,onExport,migrationRoot}) {
15
16
  insist(onExport===undefined || typeof onExport==='function','invalid_on_export');
16
17
  db=wireDatabase(db);
17
- const config=policy(input);await catalogue(db,config);
18
+ const config=policy(input);await ensureObjectInventory(db);await catalogue(db,config);
18
19
  for(const name of Object.keys(config.buckets))insist(buckets[name] && typeof buckets[name].get==='function','missing_bucket:'+name);
19
20
  await ensureImportLeases(db);
20
21
  const context={db,buckets,config,migrations,onExport};
@@ -0,0 +1,45 @@
1
+ import { canonicalJson, utf8 } from './hash.mjs';
2
+
3
+ const table='rer_portability_object_inventory';
4
+ const chunkSize=50;
5
+ export async function ensureObjectInventory(db) {
6
+ await db.prepare(`CREATE TABLE IF NOT EXISTS ${table}(job TEXT NOT NULL REFERENCES rer_portability_jobs(id) ON DELETE CASCADE, chunk INTEGER NOT NULL, data TEXT NOT NULL, PRIMARY KEY(job,chunk))`).run();
7
+ }
8
+ // Keep large manifests out of D1's per-value and per-row limits. Only the
9
+ // storage representation changes; archive manifests still contain every object.
10
+ export function prepareInventory(row) {
11
+ if(utf8(canonicalJson(row.metadata.manifest.objects)).length>256*1024)
12
+ row.metadata.objectInventory=Math.ceil(row.metadata.manifest.objects.length/chunkSize);
13
+ }
14
+ export function metadataJson(metadata) {
15
+ if(!metadata.objectInventory)return canonicalJson(metadata);
16
+ const compact={...metadata,manifest:{...metadata.manifest,objects:[]}};
17
+ if(metadata.restoreManifest)compact.restoreManifest={...metadata.restoreManifest,objects:[]};
18
+ return canonicalJson(compact);
19
+ }
20
+ export function inventoryStatements(db,row,start=0,end=row.metadata.manifest.objects.length) {
21
+ if(!row.metadata.objectInventory)return [];
22
+ const objects=row.metadata.manifest.objects,statements=[];
23
+ for(let chunk=Math.floor(start/chunkSize);chunk<Math.ceil(end/chunkSize);chunk++)
24
+ statements.push(db.prepare(`INSERT OR REPLACE INTO ${table}(job,chunk,data) VALUES(?,?,?)`).bind(row.id,chunk,canonicalJson(objects.slice(chunk*chunkSize,(chunk+1)*chunkSize))));
25
+ return statements;
26
+ }
27
+ export async function loadInventory(db,row) {
28
+ if(!row.metadata.objectInventory)return row;
29
+ const objects=[];
30
+ // Read bounded pages: a complete inventory can exceed the RPC result limit.
31
+ for(let offset=0;offset<row.metadata.objectInventory;offset+=16) {
32
+ const {results}=await db.prepare(`SELECT data FROM ${table} WHERE job=? ORDER BY chunk LIMIT 16 OFFSET ?`).bind(row.id,offset).all();
33
+ for(const item of results)objects.push(...JSON.parse(item.data));
34
+ }
35
+ row.metadata.manifest.objects=objects;
36
+ if(row.metadata.restoreManifest)row.metadata.restoreManifest.objects=objects;
37
+ return row;
38
+ }
39
+ export function clearInventory(db,row) {
40
+ if(!row.metadata.objectInventory)return [];
41
+ delete row.metadata.objectInventory;
42
+ row.metadata.manifest.objects=[];
43
+ if(row.metadata.restoreManifest)row.metadata.restoreManifest.objects=[];
44
+ return [db.prepare(`DELETE FROM ${table} WHERE job=?`).bind(row.id)];
45
+ }