@runeditrun/rer 0.2.84 → 0.2.86
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +156 -50
- package/cli/README.md +78 -2
- package/cli/agent-door.mjs +149 -0
- package/cli/agent-fixtures.mjs +16 -0
- package/cli/agent-mail.mjs +54 -0
- package/cli/agent-network.mjs +64 -0
- package/cli/agent-reviewers.mjs +110 -0
- package/cli/agent-secrets.mjs +23 -0
- package/cli/agent-sponsors.mjs +43 -0
- package/cli/agent-surface.mjs +34 -0
- package/cli/agent-webhooks.mjs +19 -0
- package/cli/authoring.mjs +2 -1
- package/cli/bounded.mjs +1 -1
- package/cli/business/archive-task.mjs +32 -0
- package/cli/business/carry.mjs +151 -0
- package/cli/business/catchup-task.mjs +42 -0
- package/cli/business/choices.mjs +97 -0
- package/cli/business/command.mjs +29 -6
- package/cli/business/connect.mjs +20 -8
- package/cli/business/detector.mjs +4 -1
- package/cli/business/device.mjs +13 -0
- package/cli/business/export-inputs.mjs +69 -0
- package/cli/business/export-task.mjs +17 -0
- package/cli/business/invocation.mjs +8 -1
- package/cli/business/keychain.mjs +8 -1
- package/cli/business/lease.mjs +11 -1
- package/cli/business/local-commands.mjs +80 -30
- package/cli/business/outbound.mjs +9 -4
- package/cli/business/parsers-config.mjs +8 -7
- package/cli/business/parsers-syntax.mjs +7 -3
- package/cli/business/permissions.mjs +2 -0
- package/cli/business/questions.mjs +89 -6
- package/cli/business/setup-command.mjs +49 -0
- package/cli/business/setup-company.json +15 -0
- package/cli/business/setup-demo.mjs +95 -0
- package/cli/business/state.mjs +44 -10
- package/cli/business/supervisor.mjs +97 -32
- package/cli/business/vendor-door.mjs +230 -0
- package/cli/business/vendors/README.md +29 -3
- package/cli/business/vendors/common.mjs +8 -6
- package/cli/business/vendors/index.mjs +22 -11
- package/cli/business/vendors/sorry.mjs +106 -0
- package/cli/container-transfer.mjs +2 -2
- package/cli/contract.mjs +7 -7
- package/cli/demo-deploy-error.mjs +14 -0
- package/cli/demo-native.mjs +6 -2
- package/cli/deploy-command.mjs +6 -4
- package/cli/deploy-config.mjs +1 -1
- package/cli/deploy-owner-input.mjs +3 -1
- package/cli/deploy-variables.mjs +58 -0
- package/cli/deploy.mjs +9 -7
- package/cli/door-clauses.mjs +157 -0
- package/cli/errors.mjs +1 -1
- package/cli/export.mjs +16 -4
- package/cli/hosted-output.mjs +17 -7
- package/cli/human-error.mjs +20 -0
- package/cli/inspect.mjs +4 -0
- package/cli/jsonc.mjs +8 -4
- package/cli/lifecycle.mjs +26 -6
- package/cli/main.mjs +2 -1
- package/cli/managed-data-schema.mjs +127 -0
- package/cli/managed-data-zip.mjs +116 -0
- package/cli/managed-data.mjs +110 -0
- package/cli/managed-server.mjs +25 -14
- package/cli/native-owner-entry.mjs +86 -0
- package/cli/output.mjs +3 -2
- package/cli/parse.mjs +2 -2
- package/cli/portability-empty.mjs +65 -0
- package/cli/portability-migrations.mjs +33 -9
- package/cli/portability-policy.mjs +5 -2
- package/cli/portability-worker.mjs +15 -6
- package/cli/portability.mjs +30 -8
- package/cli/production-auth.mjs +1 -1
- package/cli/production-mail.mjs +8 -2
- package/cli/ready-summary.mjs +9 -0
- package/cli/run-transfers.mjs +37 -0
- package/cli/runtime-front-door.mjs +4 -4
- package/cli/server-proxy.mjs +4 -3
- package/cli/server-registry.mjs +13 -8
- package/cli/site.mjs +6 -1
- package/cli/transfers.mjs +17 -4
- package/edition-authoring/archive-source.mjs +59 -0
- package/edition-authoring/base-copies.mjs +147 -25
- package/edition-authoring/cli.mjs +1 -1
- package/edition-authoring/edition-migrations.mjs +7 -3
- package/edition-authoring/module-closure.mjs +2 -16
- package/edition-authoring/workflow-copies.mjs +1 -1
- package/edition-authoring/workflow-scripts.mjs +12 -3
- package/foundation/runtime/contract-docs.mjs +218 -34
- package/foundation/runtime/contract-prose-patterns.mjs +14 -0
- package/foundation/runtime/contract-prose.mjs +88 -0
- package/foundation/runtime/run-data.d.mts +1 -0
- package/foundation/runtime/run-data.mjs +2 -0
- package/global/CHANGELOG.md +126 -0
- package/global/README.md +156 -50
- package/global/cli/command.mjs +8 -2
- package/global/cli/install.mjs +4 -5
- package/global/cli/path-link.mjs +3 -2
- package/global/cli/repo-command.mjs +3 -7
- package/global/door/command.mjs +3 -1
- package/global/launch/dev.mjs +12 -2
- package/global/launch/facts.mjs +2 -2
- package/global/launch/render.mjs +16 -9
- package/global/launch/state.mjs +4 -4
- package/global/main.mjs +13 -8
- package/install.sh +17 -6
- package/modules/agent-door/CLAUSES.md +215 -0
- package/modules/agent-door/conformance/audit.mjs +66 -0
- package/modules/agent-door/conformance/authority.d.mts +2 -0
- package/modules/agent-door/conformance/authority.mjs +17 -0
- package/modules/agent-door/conformance/crypto.mjs +36 -0
- package/modules/agent-door/conformance/discovery.mjs +61 -0
- package/modules/agent-door/conformance/index.mjs +41 -0
- package/modules/agent-door/conformance/invocation.mjs +32 -0
- package/modules/agent-door/conformance/journey.mjs +112 -0
- package/modules/agent-door/conformance/narrowing.d.mts +1 -0
- package/modules/agent-door/conformance/narrowing.mjs +21 -0
- package/modules/agent-door/conformance/negative.mjs +47 -0
- package/modules/agent-door/conformance/network-modules.mjs +86 -0
- package/modules/agent-door/conformance/network-runtime.mjs +46 -0
- package/modules/agent-door/conformance/policy-phases.mjs +75 -0
- package/modules/agent-door/conformance/protocol.mjs +145 -0
- package/modules/agent-door/conformance/replay.mjs +38 -0
- package/modules/agent-door/conformance/schema.mjs +71 -0
- package/modules/agent-door/conformance/security.mjs +33 -0
- package/modules/agent-door/conformance/sponsors.mjs +68 -0
- package/modules/agent-door/conformance/surface.mjs +89 -0
- package/modules/agent-door/conformance/target.mjs +18 -0
- package/modules/agent-door/conformance/tool-name.d.mts +1 -0
- package/modules/agent-door/conformance/tool-name.mjs +7 -0
- package/modules/agent-door/conformance/webhooks.mjs +30 -0
- package/modules/container/README.md +6 -1
- package/modules/container/adapters/d1-limits.mjs +286 -0
- package/modules/container/adapters/d1-sqlite.mjs +95 -0
- package/modules/container/adapters/d1-workerd-preload.mjs +19 -0
- package/modules/container/adapters/d1-workerd-setup.mjs +26 -0
- package/modules/container/adapters/d1-workerd.mjs +5 -0
- package/modules/container/adapters/sqlite-worker.mjs +5 -3
- package/modules/container/build.mjs +1 -0
- package/modules/container/image-variables.mjs +14 -3
- package/modules/container/transfer.mjs +14 -4
- package/modules/container/worker-engine.mjs +3 -3
- package/modules/demo-host/README.md +6 -0
- package/modules/demo-host/build.mjs +1 -0
- package/modules/demo-host/native/prepare.mjs +2 -1
- package/modules/demo-host/native/sql.mjs +7 -2
- package/modules/demo-host/ownership.mjs +39 -0
- package/modules/demo-host/plan.mjs +2 -1
- package/modules/demo-host/remove.mjs +6 -1
- package/modules/deploy-kit/README.md +22 -2
- package/modules/deploy-kit/d1-import-diagnostics.mjs +56 -0
- package/modules/deploy-kit/d1-import.mjs +17 -5
- package/modules/deploy-kit/diagnostics.mjs +29 -3
- package/modules/deploy-kit/owner-cli.mjs +20 -7
- package/modules/deploy-kit/provider-transport.mjs +19 -4
- package/modules/deploy-kit/provider.mjs +57 -22
- package/modules/deploy-kit/readiness-window.mjs +1 -1
- package/modules/deploy-kit/site.mjs +8 -3
- package/modules/deploy-kit/worker.mjs +4 -4
- package/modules/import-kit/schema/importer-verification-receipt.schema.json +171 -0
- package/modules/import-kit/schema/importers.schema.json +107 -0
- package/modules/import-kit/src/d1/README.md +23 -4
- package/modules/import-kit/src/d1/atomic-prefix.ts +2 -2
- package/modules/import-kit/src/d1/budget.ts +15 -7
- package/modules/import-kit/src/d1/column-layout.ts +5 -0
- package/modules/import-kit/src/d1/creation.ts +15 -8
- package/modules/import-kit/src/d1/database.ts +3 -3
- package/modules/import-kit/src/d1/erasure-pages.ts +2 -1
- package/modules/import-kit/src/d1/erasure.ts +2 -2
- package/modules/import-kit/src/d1/jobs.ts +10 -6
- package/modules/import-kit/src/d1/plans.ts +14 -8
- package/modules/import-kit/src/d1/retirement.ts +2 -2
- package/modules/import-kit/src/d1/sets.ts +1 -3
- package/modules/import-kit/src/d1/staged-creation.ts +3 -3
- package/modules/import-kit/src/door/ADOPTING.md +20 -0
- package/modules/import-kit/src/door/README.md +2 -1
- package/modules/import-kit/src/door/admission.ts +5 -3
- package/modules/import-kit/src/door/archive.ts +5 -2
- package/modules/import-kit/src/door/authority.ts +1 -0
- package/modules/import-kit/src/door/grants.ts +1 -1
- package/modules/import-kit/src/door/handler.ts +14 -8
- package/modules/import-kit/src/door/index.ts +1 -1
- package/modules/import-kit/src/door/jobs.ts +40 -0
- package/modules/import-kit/src/door/limits.ts +1 -1
- package/modules/import-kit/src/door/operator.ts +3 -3
- package/modules/import-kit/src/door/planning.ts +10 -6
- package/modules/import-kit/src/door/protocol.ts +2 -2
- package/modules/import-kit/src/door/raw.ts +25 -0
- package/modules/import-kit/src/door/refusals.ts +2 -1
- package/modules/import-kit/src/door/types.ts +26 -6
- package/modules/import-kit/src/receipt/encoding.ts +37 -0
- package/modules/import-kit/src/receipt/index.ts +69 -0
- package/modules/import-kit/src/receipt/policy.ts +24 -0
- package/modules/import-kit/src/receipt/types.ts +64 -0
- package/modules/import-kit/src/receipt/uri.ts +45 -0
- package/modules/import-kit/src/receipt/validation.ts +87 -0
- package/modules/import-kit/src/status/encoding.ts +13 -0
- package/modules/import-kit/src/status/evidence.ts +218 -0
- package/modules/import-kit/src/status/index.ts +56 -0
- package/modules/import-kit/src/status/schema.ts +208 -0
- package/modules/import-kit/src/status/types.ts +86 -0
- package/modules/operator-auth/accounts/delegated.mjs +48 -0
- package/modules/operator-auth/index.d.mts +7 -0
- package/modules/operator-auth/service.mjs +2 -0
- package/modules/portability/README.md +138 -9
- package/modules/portability/backfills.mjs +9 -8
- package/modules/portability/bridge.mjs +32 -5
- package/modules/portability/catalogue.mjs +12 -5
- package/modules/portability/copies.mjs +8 -1
- package/modules/portability/derived.mjs +33 -0
- package/modules/portability/files.mjs +4 -3
- package/modules/portability/http-errors.mjs +1 -1
- package/modules/portability/http.mjs +3 -3
- package/modules/portability/import.mjs +29 -13
- package/modules/portability/index.d.mts +4 -3
- package/modules/portability/index.mjs +1 -1
- package/modules/portability/ledgers.mjs +3 -2
- package/modules/portability/legacy-safety.mjs +11 -2
- package/modules/portability/migration-history.mjs +2 -2
- package/modules/portability/progress.mjs +1 -0
- package/modules/portability/restore-empty.mjs +12 -0
- package/modules/release/github-verify.mjs +11 -2
- package/onboarding/adapter.mjs +35 -6
- package/onboarding/app-fetch.mjs +6 -4
- package/onboarding/development-runtime.mjs +19 -0
- package/onboarding/local/operation.mjs +45 -27
- package/onboarding/loopback-fetch.mjs +16 -0
- package/onboarding/operator-installer.mjs +2 -4
- package/onboarding/process-diagnostics.mjs +6 -2
- package/onboarding/production-runtime.mjs +4 -2
- package/onboarding/readiness.mjs +1 -0
- package/package.json +4 -2
- package/scripts/publish-check/copy_identities.py +3 -1
- package/scripts/publish-check/github_refs.py +32 -13
- package/scripts/publish-check/public_hygiene.py +1 -1
- package/cli/business/opt-ins.mjs +0 -33
|
@@ -3,11 +3,11 @@ import { needsStreamOrigin } from './stream-origin.ts';
|
|
|
3
3
|
import { nativeStagingScope } from './native-staging-context.ts';
|
|
4
4
|
import { streamBoundary } from './stream-boundary.ts';
|
|
5
5
|
import { requestContext } from './budget.ts';
|
|
6
|
-
import { loadPlans, planBody, planGuards, planChunkReads, retainPlan, creationCodec, LARGE_PLAN_BYTES } from './plans.ts';
|
|
6
|
+
import { loadPlans, storedPlanSnapshot, planBody, planGuards, planChunkReads, retainPlan, creationCodec, LARGE_PLAN_BYTES } from './plans.ts';
|
|
7
7
|
import { randomUUID } from 'node:crypto';
|
|
8
8
|
import type { ImportJob, ImportPlan, ImportMapping, SourceBundle, ImportAction } from '../types.ts';
|
|
9
9
|
import { buildImportPlan, planFingerprint } from '../plan/index.ts';
|
|
10
|
-
import { creationStatements, validateCreation } from './creation.ts';
|
|
10
|
+
import { creationStatements, preflightD1ImportJob, validateCreation } from './creation.ts';
|
|
11
11
|
import { planStagedCreation, runStagedCreation, stagedSize } from './staged-creation.ts';
|
|
12
12
|
import type { D1ImportContext, D1Sql } from './types.ts';
|
|
13
13
|
import { atomic, readAtomic, authorize, guard, sql } from './database.ts';
|
|
@@ -31,7 +31,7 @@ export async function stored(id: string, context: D1ImportContext, action: Impor
|
|
|
31
31
|
// Unbounded guards run before their transaction's writes: a large plan is
|
|
32
32
|
// fenced by its custody revisions instead of resending it in every request.
|
|
33
33
|
const fenced = !bounded, job = decode(row), plan = (await loadPlans([row], context.storage.database, context.storage.targets.planCodec, undefined, fenced)).get(id)!;
|
|
34
|
-
planReads.set(job, planChunkReads(row.plan_json, fenced));
|
|
34
|
+
planReads.set(job, planChunkReads(row.plan_json, fenced, storedPlanSnapshot(plan, id).chunks));
|
|
35
35
|
if (job.destination !== context.destination || job.namespace !== context.namespace || job.mappingVersion !== context.mappingVersion || job.principalId !== context.authorization.principalId) throw new Error('JOB_BINDING_MISMATCH');
|
|
36
36
|
if (context.reviewedFingerprint !== undefined && context.reviewedFingerprint !== job.fingerprint) throw new Error('REVIEW_FINGERPRINT_MISMATCH');
|
|
37
37
|
if ((verifiedPlanFingerprint(plan) ?? planFingerprint(plan)) !== job.fingerprint || plan.fingerprint !== job.fingerprint || plan.destination !== job.destination || plan.namespace !== job.namespace || plan.mappingVersion !== job.mappingVersion || plan.principalId !== job.principalId || plan.entries.length !== job.total) throw new Error('STORED_PLAN_CHANGED');
|
|
@@ -47,6 +47,10 @@ export async function planD1Import(source: SourceBundle, mapping: ImportMapping,
|
|
|
47
47
|
context = requestContext(context);
|
|
48
48
|
await authorize(context, 'plan');
|
|
49
49
|
const plan = buildImportPlan(source, mapping, context);
|
|
50
|
+
// Admission measures calls, not current create authority. Planning still
|
|
51
|
+
// evaluates only the plan grant; creation checks its own live predicate.
|
|
52
|
+
const { storage, ...admission } = context;
|
|
53
|
+
preflightD1ImportJob(plan, { ...admission, planCodec: storage.targets.planCodec, reviewedFingerprint: plan.fingerprint, authorization: { principalId: context.authorization.principalId, predicate: () => sql('1') } });
|
|
50
54
|
await authorize(context, 'plan');
|
|
51
55
|
return plan;
|
|
52
56
|
}
|
|
@@ -76,16 +80,16 @@ export async function createD1ImportJob(plan: ImportPlan, context: D1ImportConte
|
|
|
76
80
|
await guardErasedSources(plan, db);
|
|
77
81
|
revalidate();
|
|
78
82
|
const codec = creationCodec(plan, context.storage.targets.planCodec), unbounded = context.maxBatchBytes === undefined;
|
|
79
|
-
const returnReads = (body: string) => 1 + planChunkReads(body, unbounded) + Number(context.reversalPolicy === 'preserve-conflicts');
|
|
80
83
|
const id = randomUUID(), retained = retainPlan(plan, id, codec);
|
|
84
|
+
const returnReads = 1 + planChunkReads(retained.body, unbounded, retained.encoded.chunks) + Number(context.reversalPolicy === 'preserve-conflicts');
|
|
81
85
|
// A large plan is created in bounded steps; a retry resumes what was staged.
|
|
82
86
|
if (unbounded && stagedSize(retained.encoded) > LARGE_PLAN_BYTES) {
|
|
83
87
|
const creation = await planStagedCreation(plan, context, operationKey, retained.encoded, () => [randomUUID(), randomUUID(), id]);
|
|
84
88
|
revalidate();
|
|
85
|
-
return (await stored(await runStagedCreation(context, creation, returnReads
|
|
89
|
+
return (await stored(await runStagedCreation(context, creation, returnReads), context, 'create')).job;
|
|
86
90
|
}
|
|
87
91
|
const creation = creationStatements(plan, { ...context, planCodec: codec }, id, operationKey, retained);
|
|
88
|
-
await atomic(context, 'create', creation, returnReads
|
|
92
|
+
await atomic(context, 'create', creation, returnReads);
|
|
89
93
|
return (await stored(id, context, 'create')).job;
|
|
90
94
|
}
|
|
91
95
|
export async function readD1ImportJob(id: string, context: D1ImportContext): Promise<ImportJob> {
|
|
@@ -4,6 +4,7 @@ import { compressedPlan, storedStreamPage } from './stream-plan.ts';
|
|
|
4
4
|
import { expandCompressedEnvelope } from './compressed-plan.ts';
|
|
5
5
|
import { planReadScope } from './verified-plan.ts';
|
|
6
6
|
import { Buffer } from 'node:buffer';
|
|
7
|
+
import { RPC_LIMIT } from '../../../d1-transport/index.mjs';
|
|
7
8
|
import { createHash } from 'node:crypto';
|
|
8
9
|
import type { ImportPlan } from '../types.ts';
|
|
9
10
|
import type { D1Database, D1Sql, D1PlanCodec } from './types.ts';
|
|
@@ -27,13 +28,15 @@ export interface StoredPlan {
|
|
|
27
28
|
export const LARGE_PLAN_BYTES = 4 * 1024 * 1024;
|
|
28
29
|
const large = (envelope: any): boolean => envelope?.d1Plan === 1 && Number.isSafeInteger(envelope.size) && envelope.size > LARGE_PLAN_BYTES;
|
|
29
30
|
const snapshots = new WeakMap<ImportPlan, StoredPlan>();
|
|
30
|
-
//
|
|
31
|
-
// the
|
|
32
|
-
|
|
33
|
-
|
|
31
|
+
// Reserve room for JSON row metadata and D1's whole-result two-byte strings.
|
|
32
|
+
// Measure the actual JSON-encoded row ceiling before choosing a page size:
|
|
33
|
+
// legacy chunks can be much larger than today's 128 KiB canonical chunks.
|
|
34
|
+
const planReadRows = (bytes: number): number => Math.max(1, Math.min(64, Math.floor((RPC_LIMIT / 4) / (bytes + 512))));
|
|
35
|
+
const quotedChunkBytes = (chunks: readonly { body: string }[]): number => chunks.reduce((max, chunk) => Math.max(max, Buffer.byteLength(JSON.stringify(chunk))), 0);
|
|
36
|
+
export function planChunkReads(body: string, fenced: boolean, chunks: readonly { body: string }[]): number {
|
|
34
37
|
const envelope = JSON.parse(body);
|
|
35
38
|
// A fenced large plan's revision read precedes its chunk reads.
|
|
36
|
-
return envelope?.d1Plan === 1 ? Math.ceil((envelope.chunks + 1) /
|
|
39
|
+
return envelope?.d1Plan === 1 ? 1 + Math.ceil((envelope.chunks + 1) / planReadRows(quotedChunkBytes(chunks))) + Number(fenced && large(envelope)) : 0;
|
|
37
40
|
}
|
|
38
41
|
export function encodePlan(plan: ImportPlan, id: string): StoredPlan {
|
|
39
42
|
const chunks: StoredPlan['chunks'] = [], hash = createHash('sha256'); let size = 0;
|
|
@@ -72,10 +75,13 @@ export async function loadPlans(rows: any[], db: D1Database, codec?: D1PlanCodec
|
|
|
72
75
|
const revision = await db.prepare('SELECT revision FROM ik_d1_custody_revisions WHERE job_id=?').bind(row.id).first();
|
|
73
76
|
if (revision && Number.isSafeInteger(revision.revision)) fences.set(row.id, revision.revision);
|
|
74
77
|
}
|
|
75
|
-
|
|
76
|
-
|
|
78
|
+
const ceiling = await db.prepare(`SELECT MAX(length(CAST(json_object('job_id',job_id,'ordinal',ordinal,'body',body) AS BLOB))) AS bytes FROM ik_d1_plan_chunks WHERE job_id=?${fence?` AND (${fence.sql})`:''}`).bind(row.id,...fence?.params??[]).first();
|
|
79
|
+
if (!Number.isSafeInteger(ceiling?.bytes) || (ceiling!.bytes as number) < 2) throw new Error('STORED_PLAN_CHANGED');
|
|
80
|
+
const readRows = planReadRows(ceiling!.bytes as number);
|
|
81
|
+
for (let offset = 0; offset <= envelope.chunks; offset += readRows) {
|
|
82
|
+
const found = await db.prepare(`SELECT * FROM ik_d1_plan_chunks WHERE job_id IN (SELECT value FROM json_each(?))${fence?` AND (${fence.sql})`:''} ORDER BY job_id,ordinal LIMIT ${readRows} OFFSET ${offset}`).bind(JSON.stringify([row.id]),...fence?.params??[]).all();
|
|
77
83
|
chunks.push(...found.results);
|
|
78
|
-
if (found.results.length <
|
|
84
|
+
if (found.results.length < readRows) break;
|
|
79
85
|
}
|
|
80
86
|
}
|
|
81
87
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { randomUUID } from 'node:crypto';
|
|
2
2
|
import type { D1CustodyRetirement, D1ErasureJournalEntry, D1ImportContext, D1ImportPerson, D1RetirementOptions, D1RetirementProgress, D1Sql } from './types.ts';
|
|
3
|
-
import { requestContext, packGuards, remaining, D1RequestLimitError } from './budget.ts';
|
|
3
|
+
import { requestContext, limitStatements, packGuards, remaining, D1RequestLimitError } from './budget.ts';
|
|
4
4
|
import { authority, authorize, finalAuthority, guard, readonlyChecks, prepare, readAtomic, sql } from './database.ts';
|
|
5
5
|
import { requireText } from '../provenance/identity.ts';
|
|
6
6
|
import { targetErasure, type ErasureStep } from './erasure.ts';
|
|
@@ -39,7 +39,7 @@ export function retireD1ImportCustody(person: D1ImportPerson, nativeStatements:
|
|
|
39
39
|
export async function retireD1ImportCustody(person: D1ImportPerson, nativeStatements: D1Sql[], input: D1ImportContext, checkpoint: D1Sql[], options: D1RetirementOptions | D1ErasureStepOptions = {}): Promise<D1CustodyRetirement | D1RetirementProgress | D1ErasurePending> {
|
|
40
40
|
if ('operationKey' in options) return retireD1ImportCustodyStep(person, nativeStatements, input, checkpoint, options);
|
|
41
41
|
person = { ...person, related: person.related ? { ...person.related, fields: fieldRedactions(person.related.fields) } : undefined };
|
|
42
|
-
const context = requestContext({ ...input, originalErasurePolicy: input.originalErasurePolicy ?? 'withdraw-affected-artifacts' });
|
|
42
|
+
const context = limitStatements(requestContext({ ...input, originalErasurePolicy: input.originalErasurePolicy ?? 'withdraw-affected-artifacts' }), 1000);
|
|
43
43
|
if (context.maxBatchBytes !== undefined) throw new Error('IMPORT_RETIREMENT_OPERATION_KEY_REQUIRED');
|
|
44
44
|
const values = [...new Set(person.values ?? [])].sort();
|
|
45
45
|
for (const value of values) requireText(value, 'person value');
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
export { jobColumns, provenanceColumns, claimColumns } from './column-layout.ts';
|
|
1
2
|
import { Buffer } from 'node:buffer';
|
|
2
3
|
import type { D1Database, D1Sql } from './types.ts';
|
|
3
4
|
import { guard, sql } from './database.ts';
|
|
@@ -64,9 +65,6 @@ export function exactRows(table: string, columns: string[], rows: any[], scope?:
|
|
|
64
65
|
if (result.length) result[0].expectedSnapshots = [{table,columns,rows,...(selection?{selection}:{})}];
|
|
65
66
|
return result;
|
|
66
67
|
}
|
|
67
|
-
export const jobColumns = ['id','destination','namespace','mapping_version','principal_id','fingerprint','operation_key','plan_json','state','checkpoint','total','fence','lease_token','lease_expires','error','reversal_policy','reversal_policy_hash','reversal_cursor','reversal_hash'];
|
|
68
|
-
export const provenanceColumns = ['id','job_id','ordinal','effect_ordinal','source_identity','destination','target_kind','target_id','reversed','record_json'];
|
|
69
|
-
export const claimColumns = ['identity','source_hash','mapping_version','effects_hash','job_id'];
|
|
70
68
|
|
|
71
69
|
/** One bound JSON value per statement; SQLite preserves strings as TEXT even
|
|
72
70
|
* in BLOB-affinity columns. Column names are internal constants only. */
|
|
@@ -3,7 +3,7 @@ import type { ImportPlan } from '../types.ts';
|
|
|
3
3
|
import type { D1ImportContext, D1Sql } from './types.ts';
|
|
4
4
|
import type { StoredPlan } from './plans.ts';
|
|
5
5
|
import { atomic, atomicCount, guard, prepare, sql } from './database.ts';
|
|
6
|
-
import { D1RequestLimitError, remaining } from './budget.ts';
|
|
6
|
+
import { batchExecutions, D1RequestLimitError, remaining } from './budget.ts';
|
|
7
7
|
import { stagingStatements } from './staging-statements.ts';
|
|
8
8
|
import { batchBytes } from './transport.ts';
|
|
9
9
|
import { bindingPredicate, stageClear, type StageBinding } from './staging-transport.ts';
|
|
@@ -142,7 +142,7 @@ async function resumable(context: D1ImportContext, prior: any, expected: Layout)
|
|
|
142
142
|
* own fences, and never more than STAGED_BATCH_BYTES on the wire. */
|
|
143
143
|
async function stagedBatch(context: D1ImportContext, statements: D1Sql[]): Promise<void> {
|
|
144
144
|
const packed = stagingStatements(context, 'create', statements);
|
|
145
|
-
if (packed.length > remaining(context)) throw new D1RequestLimitError();
|
|
145
|
+
if (batchExecutions(context, packed.length) > remaining(context)) throw new D1RequestLimitError();
|
|
146
146
|
if (batchBytes(packed) > STAGED_BATCH_BYTES) throw new Error('IMPORT_STAGE_TRANSPORT_LIMIT');
|
|
147
147
|
await context.storage.database.batch(packed.map(s => prepare(context.storage.database, s)));
|
|
148
148
|
}
|
|
@@ -192,7 +192,7 @@ export function preflightStagedCreation(plan: ImportPlan, context: D1ImportConte
|
|
|
192
192
|
}
|
|
193
193
|
/** Statements the creation spends in this request, excluding its return reads. */
|
|
194
194
|
export const stagedCreationCost = (context: D1ImportContext, creation: StagedCreation): number =>
|
|
195
|
-
creation.batches.reduce((n, batch) => n + stagingStatements(context, 'create', batch).length, 0) + atomicCount(context, 'create', creation.publication, 0);
|
|
195
|
+
creation.batches.reduce((n, batch) => n + batchExecutions(context, stagingStatements(context, 'create', batch).length), 0) + atomicCount(context, 'create', creation.publication, 0);
|
|
196
196
|
export async function runStagedCreation(context: D1ImportContext, creation: StagedCreation, returnReads: number): Promise<string> {
|
|
197
197
|
if (stagedCreationCost(context, creation) + returnReads > remaining(context)) throw new D1RequestLimitError();
|
|
198
198
|
for (const batch of creation.batches) await stagedBatch(context, batch);
|
|
@@ -361,6 +361,26 @@ requires replacing that caller with the operator mount and deleting the old
|
|
|
361
361
|
HTTP importer in `src/domain/sentry-import/routes.ts`.
|
|
362
362
|
Do not copy its older vendored door instead of current Base.
|
|
363
363
|
|
|
364
|
+
## Raw source files
|
|
365
|
+
|
|
366
|
+
For ZIP or another binary export, declare `format: 'raw'` and
|
|
367
|
+
`readRaw(upload, limits)` instead of `parse`. The same source declaration serves
|
|
368
|
+
both mounts, including a grant allow-list entry. The reader receives the verified
|
|
369
|
+
`upload.bytes` (`Uint8Array`), `name`, `sha256` and `size`, before any text decoding,
|
|
370
|
+
and returns a `ParsedRecord[]` (or a promise of it). Bound archive expansion while
|
|
371
|
+
reading, inspect every member for credentials, and reject malformed or unsupported
|
|
372
|
+
archives; thrown errors become a non-secret 422 `source_raw_reader_failed`.
|
|
373
|
+
The door validates returned records, retains the exact original archive and binds
|
|
374
|
+
it to the reviewed fingerprint. Raw sources cannot declare `intake`, which changes
|
|
375
|
+
custody bytes. Chunk uploads have at most 1,024 chunks of 64 KiB: a 64 MiB
|
|
376
|
+
(67,108,864-byte) ceiling, also the default owner quota. Signed quotas can reduce
|
|
377
|
+
it. `limits.maxBytes` may lower it further and is checked before the
|
|
378
|
+
reader runs; excess upload declarations are refused, or planning answers 413
|
|
379
|
+
`source_byte_limit` (a previously retained upload above a lowered owner quota
|
|
380
|
+
answers 403 `upload_exceeds_grant_quota`). Record count and per-record byte limits still apply. Export
|
|
381
|
+
exclusion, erasure of complete or partial custody, archive restore-check, catch-up
|
|
382
|
+
and Undo use the existing lifecycle and require the same seed integration proof.
|
|
383
|
+
|
|
364
384
|
## Copyable public HTTP conformance
|
|
365
385
|
|
|
366
386
|
Copy the following into an existing seed test journey and call `checkDoor`.
|
|
@@ -225,11 +225,12 @@ are admission limits, not a hosted capacity promise.
|
|
|
225
225
|
| `uploads` (no intake declaration) | `{ uploadId?, name, bundleSha256, totalBytes, totalChunks, index, chunkSha256, base64 }`; decoded chunks are exactly 64 KiB except the final chunk. `totalChunks` must equal `ceil(totalBytes / 65536)`. Omit `uploadId` on the first chunk to receive a server-generated UUID. Reuse it and the same bundle metadata for subsequent chunks. Both chunk and complete bundle SHA-256 are checked before reporting completion. |
|
|
226
226
|
| `uploads` (declared intake) | Full raw stream with `Content-Type: application/octet-stream`, `X-Import-Format: csv\|ndjson\|json` and `X-Import-Name: <safe file name>`. Returns custody hashes and intake metadata; see below. |
|
|
227
227
|
| `uploads` | `{ uploadId }` returns received-chunk inventory/progress. Without intake, an identical duplicate chunk is idempotent with a fresh grant. |
|
|
228
|
-
| `plan` | `{ uploadId, catchup?: boolean }` returns `planId`, `fingerprint`, server-derived counts, summary, rejected rows with reasons, `diagnostics` and differences including gone source ids. Planning makes no native target writes. |
|
|
228
|
+
| `plan` | `{ uploadId, catchup?: boolean }` returns `planId`, `fingerprint`, `created` (true only when this request created the plan), server-derived counts, summary, rejected rows with reasons, `diagnostics` and differences including gone source ids. Planning makes no native target writes. |
|
|
229
229
|
| `apply` | `{ planId, reviewedFingerprint }` creates or resumes the real kit job in bounded batches. Repeat with a fresh valid grant to continue. A large source (a stored plan over 4 MiB, such as a multi-megabyte Notion export) is created in bounded staged steps within one request, and the answer may be the still `queued` job; an interrupted creation is never a job and the next `apply` resumes it from what was staged. |
|
|
230
230
|
| `status` | `{ planId }` reads current job progress. |
|
|
231
231
|
| `reconcile` | `{ planId }` derives source-id disposition/presence and native/provenance revision, value, attachment and count checks on the server. |
|
|
232
232
|
| `undo` | `{ planId }` reverses the newest snapshot's attributed effects in bounded kit work, preserving native edits. |
|
|
233
|
+
| `cancel` | `{ planId }` removes only an unapproved private plan with no `job_id`, failure or kit job under its operation key. Otherwise answers 409 `plan_cancel_refused`. Grant only; the owner-session mount uses `undo`. |
|
|
233
234
|
| `archive` | `{ uploadId, tool, takenAt }` keeps a fully uploaded bundle as the tool's archive; returns `archiveId`, `key`, `tool`, `takenAt`, `bundleSha256`, `bytes` and `counts`. Grant only. See "Archives". |
|
|
234
235
|
| `restore-check` | `{ archiveId }` parses the archive with the namespace's reader and mapping and writes nothing native; returns the archive's fields with `bundleSha256`, `bytes` and `counts` recomputed from its bytes, and `rejected`. Grant only. |
|
|
235
236
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { ImportDoorOptions,
|
|
1
|
+
import type { ImportDoorOptions, ImportDoorRuntime, DoorRequest } from './types.ts';
|
|
2
2
|
import { runtimeOptions } from './handler.ts';
|
|
3
3
|
import { DoorError, verifyImportGrant, type DoorAction } from './grants.ts';
|
|
4
4
|
import { doorFence, erasures, sql, transact } from './store.ts';
|
|
@@ -17,11 +17,13 @@ export async function enabled(options: ImportDoorOptions): Promise<boolean> {
|
|
|
17
17
|
}
|
|
18
18
|
/** This grant's parser and mapping: the mount's own, or the seed's allow-list entry
|
|
19
19
|
* for the verified namespace. Unlisted namespaces are refused before any write. */
|
|
20
|
-
async function source(options: ImportDoorOptions, namespace: string): Promise<
|
|
20
|
+
async function source(options: ImportDoorOptions, namespace: string): Promise<ImportDoorRuntime> {
|
|
21
21
|
if (!options.source) return runtimeOptions(options);
|
|
22
22
|
const listed = await options.source(namespace);
|
|
23
23
|
if (!listed) throw new DoorError('unknown_import_source');
|
|
24
|
-
return runtimeOptions({...options, parse: listed.parse, mapping: listed.mapping, intake: listed.intake,
|
|
24
|
+
return runtimeOptions({...options, parse: listed.parse, mapping: listed.mapping, intake: listed.intake,
|
|
25
|
+
format: listed.format ?? options.format, readRaw: 'readRaw' in listed ? listed.readRaw : undefined,
|
|
26
|
+
ownerPrincipalId: listed.ownerPrincipalId ?? options.ownerPrincipalId});
|
|
25
27
|
}
|
|
26
28
|
export async function admit(options: ImportDoorOptions, request: Request, action: DoorAction): Promise<DoorRequest> {
|
|
27
29
|
const authorization = request.headers.get('Authorization');
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { readRawSource } from './raw.ts';
|
|
1
2
|
import { Buffer } from 'node:buffer';
|
|
2
3
|
import { randomUUID } from 'node:crypto';
|
|
3
4
|
import { canonicalJson, sha256 } from '../provenance/identity.ts';
|
|
@@ -44,8 +45,10 @@ async function readArchive(request: DoorRequest, name: string, bytes: Uint8Array
|
|
|
44
45
|
if (!!intake !== !!options.intake || (intake && intake.allowlistSha256 !== intakeIdentity(options.intake!))) throw new DoorError('archive_intake_changed',409);
|
|
45
46
|
const kitNamespace = 'door-archive:' + sha256(canonicalJson([access.namespace, bundleSha256, id]));
|
|
46
47
|
try {
|
|
47
|
-
const
|
|
48
|
-
|
|
48
|
+
const limits = {...options.limits, maxBytes: Math.min(options.limits?.maxBytes ?? access.maxBytes, access.maxBytes), maxRecords: Math.min(options.limits?.maxRecords ?? access.maxRecords, access.maxRecords)};
|
|
49
|
+
const {source, liveCredentialsRemoved} = options.format === 'raw'
|
|
50
|
+
? {source: await readRawSource(options,{name,bundle_sha256:bundleSha256,total_bytes:bytes.length},bytes,kitNamespace,limits),liveCredentialsRemoved:false}
|
|
51
|
+
: await readFileSourceRemovals({namespace: kitNamespace, name, bytes, format: intake ? 'ndjson' : options.format, parse: options.parse, credentials: 'remove'},limits);
|
|
49
52
|
if (liveCredentialsRemoved) throw new DoorError('archive_holds_credentials', 422,
|
|
50
53
|
{reason: 'The bundle holds credential values, and an archive keeps its exact bytes, so it is not kept. Export again with credential values removed (rer\'s vendor exports remove them) and archive that.'});
|
|
51
54
|
if (intake) source.intake = intake;
|
|
@@ -9,6 +9,7 @@ import { reserveReturn } from '../d1/budget.ts';
|
|
|
9
9
|
const actions: Record<DoorAction, ImportAction[]> = {
|
|
10
10
|
uploads: [], plan: ['plan', 'read'], apply: ['create', 'apply', 'read'],
|
|
11
11
|
status: ['read'], reconcile: ['read'], undo: ['read', 'undo', 'rollback'],
|
|
12
|
+
cancel: [],
|
|
12
13
|
// Parsing a bundle for its counts is a plan: no native write.
|
|
13
14
|
archive: ['plan'], 'restore-check': ['plan'],
|
|
14
15
|
};
|
|
@@ -10,7 +10,7 @@ function fail(): never { throw new DoorError('invalid_grant'); }
|
|
|
10
10
|
const object = (value: unknown): value is Record<string, unknown> => value !== null && typeof value === 'object' && !Array.isArray(value);
|
|
11
11
|
const requiredKeys = ['v', 'iss', 'aud', 'installationId', 'email', 'jti', 'iat', 'exp', 'action', 'namespace', 'maxBytes', 'maxChunks', 'maxRecords'];
|
|
12
12
|
const optionalKeys = ['fingerprint', 'approvalRecordId', 'approvedAt', 'catchup'];
|
|
13
|
-
const actions: readonly string[] = ['uploads', 'plan', 'apply', 'status', 'reconcile', 'undo', 'archive', 'restore-check'];
|
|
13
|
+
const actions: readonly string[] = ['uploads', 'plan', 'apply', 'status', 'reconcile', 'undo', 'cancel', 'archive', 'restore-check'];
|
|
14
14
|
|
|
15
15
|
function decode(segment: string): Uint8Array<ArrayBuffer> {
|
|
16
16
|
if (!/^[A-Za-z0-9_-]+$/.test(segment)) fail();
|
|
@@ -1,17 +1,17 @@
|
|
|
1
1
|
import type { D1Database } from '../d1/types.ts';
|
|
2
|
-
import type { DoorRequest,
|
|
2
|
+
import type { DoorRequest, ImportDoorRuntime, ImportDoorRawSource } from './types.ts';
|
|
3
3
|
import { DoorError, GRANT_ONLY_ACTIONS, type DoorAction } from './protocol.ts';
|
|
4
4
|
import { authorized, erasedSince } from './store.ts';
|
|
5
5
|
import { sizeLimitError } from './limits.ts';
|
|
6
6
|
import { requestDatabase } from '../d1/budget.ts';
|
|
7
7
|
import { uploads } from './uploads.ts';
|
|
8
8
|
import { plan } from './planning.ts';
|
|
9
|
-
import { apply, status, reconcile, undo } from './jobs.ts';
|
|
9
|
+
import { apply, status, reconcile, undo, cancel } from './jobs.ts';
|
|
10
10
|
import { archive, restoreCheck } from './archive.ts';
|
|
11
11
|
import { streamUpload } from './intake-upload.ts';
|
|
12
12
|
import { intakeIdentity } from './intake-policy.ts';
|
|
13
13
|
|
|
14
|
-
const routes = {uploads,plan,apply,status,reconcile,undo,archive,'restore-check':restoreCheck};
|
|
14
|
+
const routes = {uploads,plan,apply,status,reconcile,undo,cancel,archive,'restore-check':restoreCheck};
|
|
15
15
|
const headers = {'Cache-Control':'no-store','Content-Type':'application/json','X-Content-Type-Options':'nosniff'};
|
|
16
16
|
async function readInput(request: Request): Promise<Record<string,unknown>> {
|
|
17
17
|
if (request.headers.get('Content-Type')?.split(';')[0] !== 'application/json' || !request.body) throw new DoorError('json_body_required',400);
|
|
@@ -31,16 +31,22 @@ async function readInput(request: Request): Promise<Record<string,unknown>> {
|
|
|
31
31
|
if (!value || typeof value!=='object' || Array.isArray(value)) throw new DoorError('invalid_input',400);
|
|
32
32
|
return value;
|
|
33
33
|
}
|
|
34
|
+
type RuntimeConfiguration = {[K in keyof ImportDoorRuntime]?: ImportDoorRuntime[K]} & {
|
|
35
|
+
readRaw?: ImportDoorRawSource['readRaw'];
|
|
36
|
+
};
|
|
37
|
+
|
|
34
38
|
/** The settings every mount needs whatever its source: a preserving undo policy needs the adapter's preserved undo. */
|
|
35
|
-
export function mountValid(options:
|
|
39
|
+
export function mountValid(options: RuntimeConfiguration): boolean {
|
|
36
40
|
return !!options.ownerPrincipalId?.trim() && !!options.destination?.trim() && [undefined,'strict','preserve-conflicts'].includes(options.reversalPolicy)
|
|
37
41
|
&& (options.reversalPolicy!=='preserve-conflicts' || typeof options.targets?.preparePreservedUndo==='function');
|
|
38
42
|
}
|
|
39
43
|
/** Refuses a mount, or a resolved source, missing what the pipeline needs. */
|
|
40
|
-
export function runtimeOptions<T extends
|
|
41
|
-
if (!options.mapping?.version ||
|
|
44
|
+
export function runtimeOptions<T extends RuntimeConfiguration>(options: T): T & ImportDoorRuntime {
|
|
45
|
+
if (!options.mapping?.version || !mountValid(options)
|
|
46
|
+
|| (options.format === 'raw' ? typeof options.readRaw !== 'function' || options.intake !== undefined
|
|
47
|
+
: typeof options.parse !== 'function')) throw new DoorError('invalid_configuration',503);
|
|
42
48
|
if (options.intake !== undefined) intakeIdentity(options.intake);
|
|
43
|
-
return options as T &
|
|
49
|
+
return options as T & ImportDoorRuntime;
|
|
44
50
|
}
|
|
45
51
|
/** Server-side mount only. A null response belongs to another dispatcher. Every
|
|
46
52
|
* statement of one request, the door's own, the adapter's and each kit call's,
|
|
@@ -71,7 +77,7 @@ export function createDoorHandler(prefix: string, database: D1Database,
|
|
|
71
77
|
}
|
|
72
78
|
const input = await readInput(request);
|
|
73
79
|
admitted = await review(admitted,input);
|
|
74
|
-
if (['status','reconcile','undo'].includes(action) && Object.keys(input).some(k=>k!=='planId')) throw new DoorError('invalid_input',400);
|
|
80
|
+
if (['status','reconcile','undo','cancel'].includes(action) && Object.keys(input).some(k=>k!=='planId')) throw new DoorError('invalid_input',400);
|
|
75
81
|
const result = await routes[action](admitted,input);
|
|
76
82
|
await authorized(admitted);
|
|
77
83
|
return Response.json(result,{headers});
|
|
@@ -13,7 +13,7 @@ export { createImportOperatorHandler } from './operator.ts';
|
|
|
13
13
|
export function createImportDoorHandler(options: ImportDoorOptions): (request: Request)=>Promise<Response|null> {
|
|
14
14
|
if (options.installationId !== undefined && (typeof options.installationId !== 'string' || !options.installationId.trim())) throw new DoorError('invalid_configuration',503);
|
|
15
15
|
if (options.source === undefined) runtimeOptions(options);
|
|
16
|
-
else if (typeof options.source !== 'function' || options.parse !== undefined || options.mapping !== undefined
|
|
16
|
+
else if (typeof options.source !== 'function' || options.parse !== undefined || options.mapping !== undefined || options.readRaw !== undefined
|
|
17
17
|
|| !mountValid(options)) throw new DoorError('invalid_configuration',503);
|
|
18
18
|
return createDoorHandler('/api/rer/import/', options.database, (request, action, database) => admit({...options, database}, request, action),
|
|
19
19
|
undefined, async database => !!options.claim?.publicKeys && Object.keys(options.claim.publicKeys).length > 0 && await enabled({...options, database}));
|
|
@@ -102,6 +102,46 @@ export async function reconcile(request: DoorRequest, input: Record<string,unkno
|
|
|
102
102
|
await authorized(request);
|
|
103
103
|
return {planId:row.id,fingerprint:row.fingerprint,bundleSha256:snapshot.bundleSha256,...report};
|
|
104
104
|
}
|
|
105
|
+
/** RER's cleanup for a plan created only while checking an executor's report.
|
|
106
|
+
* It can discard private plan custody, never a reviewed or published import. */
|
|
107
|
+
export async function cancel(request: DoorRequest, input: Record<string,unknown>) {
|
|
108
|
+
const id = text(input.planId,'plan_id'), db = request.options.database;
|
|
109
|
+
const row = await db.prepare(`SELECT p.job_id,
|
|
110
|
+
EXISTS(SELECT 1 FROM ik_jobs j WHERE j.destination=? AND j.principal_id=? AND j.operation_key='door:'||p.id) AS has_job,
|
|
111
|
+
EXISTS(SELECT 1 FROM ik_door_approvals a WHERE a.plan_id=p.id) AS has_approval,
|
|
112
|
+
EXISTS(SELECT 1 FROM ik_door_apply_failures f WHERE f.plan_id=p.id) AS has_failure
|
|
113
|
+
FROM ik_door_plans p WHERE p.id=? AND p.installation_id=? AND p.namespace=?`)
|
|
114
|
+
.bind(request.options.destination,request.options.ownerPrincipalId,id,request.access.installationId,request.access.namespace)
|
|
115
|
+
.first<{job_id:string|null;has_job:number;has_approval:number;has_failure:number}>();
|
|
116
|
+
if (!row) throw new DoorError('plan_not_found',404);
|
|
117
|
+
if (row.job_id || row.has_job || row.has_approval || row.has_failure) throw new DoorError('plan_cancel_refused',409);
|
|
118
|
+
const clear = sql(`EXISTS(SELECT 1 FROM ik_door_plans p WHERE p.id=? AND p.installation_id=? AND p.namespace=? AND p.job_id IS NULL
|
|
119
|
+
AND NOT EXISTS(SELECT 1 FROM ik_jobs j WHERE j.destination=? AND j.principal_id=? AND j.operation_key='door:'||p.id)
|
|
120
|
+
AND NOT EXISTS(SELECT 1 FROM ik_door_approvals a WHERE a.plan_id=p.id)
|
|
121
|
+
AND NOT EXISTS(SELECT 1 FROM ik_door_apply_failures f WHERE f.plan_id=p.id))`,
|
|
122
|
+
id,request.access.installationId,request.access.namespace,request.options.destination,request.options.ownerPrincipalId);
|
|
123
|
+
try {
|
|
124
|
+
await transact(db,request.authority,[
|
|
125
|
+
assertion(clear),
|
|
126
|
+
sql('UPDATE ik_door_namespaces SET pending_id=NULL WHERE installation_id=? AND namespace=? AND pending_id=?',request.access.installationId,request.access.namespace,id),
|
|
127
|
+
sql('DELETE FROM ik_d1_staging WHERE destination=? AND principal_id=? AND operation_key=?',request.options.destination,request.options.ownerPrincipalId,'door:'+id),
|
|
128
|
+
sql('DELETE FROM ik_door_plan_chunks WHERE plan_id=?',id),
|
|
129
|
+
sql(`DELETE FROM ik_door_plans WHERE id=? AND job_id IS NULL
|
|
130
|
+
AND NOT EXISTS(SELECT 1 FROM ik_jobs j WHERE j.destination=? AND j.principal_id=? AND j.operation_key='door:'||ik_door_plans.id)
|
|
131
|
+
AND NOT EXISTS(SELECT 1 FROM ik_door_approvals a WHERE a.plan_id=ik_door_plans.id)
|
|
132
|
+
AND NOT EXISTS(SELECT 1 FROM ik_door_apply_failures f WHERE f.plan_id=ik_door_plans.id)`,id,request.options.destination,request.options.ownerPrincipalId),
|
|
133
|
+
]);
|
|
134
|
+
} catch (error) {
|
|
135
|
+
if (error instanceof DoorError) throw error;
|
|
136
|
+
const blocked = await db.prepare(`SELECT 1 FROM ik_door_plans p WHERE p.id=? AND (p.job_id IS NOT NULL
|
|
137
|
+
OR EXISTS(SELECT 1 FROM ik_jobs j WHERE j.destination=? AND j.principal_id=? AND j.operation_key='door:'||p.id)
|
|
138
|
+
OR EXISTS(SELECT 1 FROM ik_door_approvals a WHERE a.plan_id=p.id)
|
|
139
|
+
OR EXISTS(SELECT 1 FROM ik_door_apply_failures f WHERE f.plan_id=p.id))`).bind(id,request.options.destination,request.options.ownerPrincipalId).first();
|
|
140
|
+
if (blocked) throw new DoorError('plan_cancel_refused',409);
|
|
141
|
+
throw error;
|
|
142
|
+
}
|
|
143
|
+
return {planId:id,state:'cancelled'};
|
|
144
|
+
}
|
|
105
145
|
export async function undo(request: DoorRequest, input: Record<string,unknown>) {
|
|
106
146
|
const id = text(input.planId,'plan_id');
|
|
107
147
|
const candidate = await request.options.database.prepare('SELECT * FROM ik_door_plans WHERE id=? AND installation_id=? AND namespace=?').bind(id,request.access.installationId,request.access.namespace).first<DoorPlanRow>();
|
|
@@ -10,7 +10,7 @@ export function sizeLimitError(error: unknown): DoorError | undefined {
|
|
|
10
10
|
const reason = targetKinds.length
|
|
11
11
|
? `The exact native snapshot of target kind ${targetKinds.map(kind => JSON.stringify(kind)).join(', ')} exceeds one 2 MiB D1 read; the seed's target adapter needs a compact matchesMany.`
|
|
12
12
|
: error instanceof ImportSizeLimitError ? 'The import custody fence exceeds one 2 MiB D1 read.'
|
|
13
|
-
: error.message === 'IMPORT_D1_REQUEST_LIMIT' ? 'This request needs more than its 1,000 D1
|
|
13
|
+
: error.message === 'IMPORT_D1_REQUEST_LIMIT' ? 'This request needs more than its 1,000 D1 queries; the seed\'s target adapter must read its native set in a fixed number of queries.'
|
|
14
14
|
: 'A D1 statement of this import exceeds its size limit.';
|
|
15
15
|
return new DoorError('import_door_too_large', 413, {targetKinds, reason});
|
|
16
16
|
}
|
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
import type { DoorRequest, ImportOperatorOptions } from './types.ts';
|
|
1
|
+
import type { DoorRequest, ImportOperatorOptions, ImportRawOperatorOptions } from './types.ts';
|
|
2
2
|
import { DoorError } from './protocol.ts';
|
|
3
3
|
import { createDoorHandler, runtimeOptions } from './handler.ts';
|
|
4
4
|
import { authorized, doorFence, erasures, schemaCurrent, sql } from './store.ts';
|
|
5
5
|
export { importDoorSchema, importDoorApplyFailureMigration, importDoorScopesMigration, importDoorArchiveMigration, importDoorIntakeMigration } from './schema.ts';
|
|
6
6
|
export { writeImportDoorSettings, readImportDoorHistory } from './authority.ts';
|
|
7
|
-
export type { ImportOperatorOptions } from './types.ts';
|
|
7
|
+
export type { ImportOperatorOptions, ImportRawOperatorOptions } from './types.ts';
|
|
8
8
|
|
|
9
9
|
/** Owner-session entry point. No owner-claim module or grant key ring required. */
|
|
10
|
-
export function createImportOperatorHandler(options: ImportOperatorOptions) {
|
|
10
|
+
export function createImportOperatorHandler(options: ImportOperatorOptions | ImportRawOperatorOptions) {
|
|
11
11
|
if (!options.installationId?.trim() || !options.ownerEmail?.trim() || !options.namespace?.trim()
|
|
12
12
|
|| new URL(options.appOrigin).origin !== options.appOrigin || typeof options.authorize !== 'function') throw new DoorError('invalid_configuration',503);
|
|
13
13
|
runtimeOptions(options);
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { readRawSource } from './raw.ts';
|
|
1
2
|
import { randomUUID } from 'node:crypto';
|
|
2
3
|
import { canonicalJson, sha256 } from '../provenance/identity.ts';
|
|
3
4
|
import { readFileSource } from '../readers/file.ts';
|
|
@@ -13,9 +14,9 @@ import { bundleBytes, text, uploadRow, type UploadRow } from './uploads.ts';
|
|
|
13
14
|
import { sourceRefusal } from './refusals.ts';
|
|
14
15
|
import { intakeState } from './intake-policy.ts';
|
|
15
16
|
|
|
16
|
-
export function preview(snapshot: DoorSnapshot) {
|
|
17
|
+
export function preview(snapshot: DoorSnapshot, created: boolean) {
|
|
17
18
|
const {plan,differences} = snapshot;
|
|
18
|
-
return {planId:snapshot.id,fingerprint:plan.fingerprint,bundleSha256:snapshot.bundleSha256,
|
|
19
|
+
return {planId:snapshot.id,fingerprint:plan.fingerprint,bundleSha256:snapshot.bundleSha256,created,
|
|
19
20
|
summary:{sourceRecords:snapshot.sourceCount,created:differences.additions.length,updated:differences.replacements.length,unchanged:differences.unchanged.length,gone:differences.gone.length},
|
|
20
21
|
counts:plan.counts,rejected:plan.entries.filter(e=>e.rejection).map(e=>({kind:e.source.kind,sourceId:e.source.sourceId,reason:e.rejection})),
|
|
21
22
|
diagnostics:plan.diagnostics,differences};
|
|
@@ -67,7 +68,7 @@ async function planUpload(request: DoorRequest, input: Record<string, unknown>,
|
|
|
67
68
|
if (snapshot.plan.mappingVersion !== request.options.mapping.version) throw new DoorError('snapshot_mapping_changed_use_new_namespace',409);
|
|
68
69
|
if (request.access.catchup === 'bounded' && (!snapshot.standingApproval || snapshot.standingApproval.id !== request.access.approvalRecordId || snapshot.standingApproval.at !== request.access.approvedAt)) throw new DoorError('standing_approval_mismatch');
|
|
69
70
|
await dropCredentialBytes(request,snapshot,upload);
|
|
70
|
-
return preview(snapshot);
|
|
71
|
+
return preview(snapshot,false);
|
|
71
72
|
}
|
|
72
73
|
if (current.pending_id) throw new DoorError('namespace_has_pending_import',409);
|
|
73
74
|
// An erasure withdrew the live import's snapshot, so nothing can be compared
|
|
@@ -138,7 +139,7 @@ async function planUpload(request: DoorRequest, input: Record<string, unknown>,
|
|
|
138
139
|
if (retired) await retire(request,retired,snapshot.plan);
|
|
139
140
|
await saveSnapshot(request,snapshot,upload,retainedFrom);
|
|
140
141
|
await dropCredentialBytes(request,snapshot,upload);
|
|
141
|
-
return preview(snapshot);
|
|
142
|
+
return preview(snapshot,true);
|
|
142
143
|
}
|
|
143
144
|
/** Retire an erasure-withdrawn live import as the namespace's baseline. The kit
|
|
144
145
|
* never lets a new import write a record another live import holds, so while an
|
|
@@ -173,8 +174,11 @@ async function readUpload(request: DoorRequest, upload: UploadRow, planId: strin
|
|
|
173
174
|
const retained = await retainedSource(request,upload);
|
|
174
175
|
if (retained) return {kitNamespace,retainedFrom:retained.planId,source:{...retained.source,records:retained.source.records.map(record=>({...record,namespace:kitNamespace}))}};
|
|
175
176
|
// Credential values are removed per record, never imported, and never refuse a record.
|
|
176
|
-
const
|
|
177
|
-
|
|
177
|
+
const bytes = await bundleBytes(request,upload);
|
|
178
|
+
const limits = {...options.limits,maxBytes:Math.min(options.limits?.maxBytes ?? request.access.maxBytes,request.access.maxBytes),maxRecords:Math.min(options.limits?.maxRecords ?? request.access.maxRecords,request.access.maxRecords)};
|
|
179
|
+
const source = options.format === 'raw'
|
|
180
|
+
? await readRawSource(options,upload,bytes,kitNamespace,limits)
|
|
181
|
+
: await readFileSource({namespace:kitNamespace,name:upload.name,bytes,format:intake ? 'ndjson' : options.format,parse:options.parse,credentials:'remove'},limits);
|
|
178
182
|
if (intake) source.intake = intake;
|
|
179
183
|
return {kitNamespace,source,retainedFrom:undefined};
|
|
180
184
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
export type DoorAction = 'uploads' | 'plan' | 'apply' | 'status' | 'reconcile' | 'undo' | 'archive' | 'restore-check';
|
|
1
|
+
export type DoorAction = 'uploads' | 'plan' | 'apply' | 'status' | 'reconcile' | 'undo' | 'cancel' | 'archive' | 'restore-check';
|
|
2
2
|
/** Grant-only actions: the owner-session mount answers them 404. */
|
|
3
|
-
export const GRANT_ONLY_ACTIONS: readonly DoorAction[] = ['archive', 'restore-check'];
|
|
3
|
+
export const GRANT_ONLY_ACTIONS: readonly DoorAction[] = ['cancel', 'archive', 'restore-check'];
|
|
4
4
|
|
|
5
5
|
export interface DoorGrant {
|
|
6
6
|
v: 1;
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { Buffer } from 'node:buffer';
|
|
2
|
+
import type { ParsedRecord, ReaderLimits, SourceBundle } from '../types.ts';
|
|
3
|
+
import { limitsFor, RecordCollector, safeValue, textId } from '../readers/records.ts';
|
|
4
|
+
import { fail } from '../readers/errors.ts';
|
|
5
|
+
import type { ImportDoorRawRuntimeOptions } from './types.ts';
|
|
6
|
+
import type { UploadRow } from './uploads.ts';
|
|
7
|
+
|
|
8
|
+
/** Upload custody is already verified. Never decode it before the seed reads it. */
|
|
9
|
+
export async function readRawSource(options: ImportDoorRawRuntimeOptions, upload: Pick<UploadRow, 'name' | 'bundle_sha256' | 'total_bytes'>,
|
|
10
|
+
bytes: Uint8Array, namespace: string, limits: ReaderLimits): Promise<SourceBundle> {
|
|
11
|
+
const bounded = limitsFor(limits), location = {artifact: upload.name};
|
|
12
|
+
textId(upload.name); safeValue(upload.name);
|
|
13
|
+
if (bytes.byteLength > bounded.maxBytes) fail('SOURCE_BYTE_LIMIT',location);
|
|
14
|
+
// Snapshot custody before the callback, whose working buffer may be mutated.
|
|
15
|
+
const base64 = Buffer.from(bytes).toString('base64');
|
|
16
|
+
let records: ParsedRecord[];
|
|
17
|
+
try {
|
|
18
|
+
records = await options.readRaw({bytes:Uint8Array.from(bytes),name:upload.name,sha256:upload.bundle_sha256,size:upload.total_bytes},Object.freeze({...bounded}));
|
|
19
|
+
} catch { fail('SOURCE_RAW_READER_FAILED',location); }
|
|
20
|
+
const collector = new RecordCollector(namespace,bounded,[]);
|
|
21
|
+
collector.add(records,location,upload.bundle_sha256);
|
|
22
|
+
collector.bundle.artifacts.push({name:upload.name,sha256:upload.bundle_sha256,size:upload.total_bytes,
|
|
23
|
+
original:{status:'available',base64}});
|
|
24
|
+
return collector.bundle;
|
|
25
|
+
}
|
|
@@ -8,6 +8,7 @@ const reasons: Record<string, string> = {
|
|
|
8
8
|
MALFORMED_JSON: 'The upload is not valid JSON in the declared format.',
|
|
9
9
|
MALFORMED_UTF8: 'The upload is not valid UTF-8.',
|
|
10
10
|
INVALID_OR_CREDENTIAL_SOURCE_DATA: 'A source value is not plain JSON data, holds a URL that cannot be parsed, or is encoded too deeply to inspect for credentials.',
|
|
11
|
+
SOURCE_RAW_READER_FAILED: "The seed's reader refused the raw upload. Check the archive format and contents.",
|
|
11
12
|
SOURCE_PARSER_FAILED: "The seed's parser refused a source value.",
|
|
12
13
|
SOURCE_OBJECT_DROPPED: "The seed's parser returned no record for a source value.",
|
|
13
14
|
INVALID_PARSED_RECORDS: "The seed's parser returned an invalid record list.",
|
|
@@ -46,6 +47,6 @@ export function sourceRefusal(error: unknown): DoorError | undefined {
|
|
|
46
47
|
reason: `The upload holds a credential the door cannot remove: a ${field}. Remove it and upload again.`, ...at});
|
|
47
48
|
}
|
|
48
49
|
if (!(error instanceof ReaderError) && !Object.hasOwn(reasons, code)) return undefined;
|
|
49
|
-
const reason = reasons[code] ?? `The reader refused the upload (${code}).`;
|
|
50
|
+
const reason = error instanceof PlanTooLargeError ? `The import ${error.subject} is ${error.bytes} canonical JSON bytes; the limit is ${error.limit}.` : reasons[code] ?? `The reader refused the upload (${code}).`;
|
|
50
51
|
return new DoorError('source_' + code.toLowerCase().replace(/^source_/, ''), tooLarge.has(code) ? 413 : 422, {reason, ...at});
|
|
51
52
|
}
|