@runeditrun/rer 0.2.84 → 0.2.86
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +156 -50
- package/cli/README.md +78 -2
- package/cli/agent-door.mjs +149 -0
- package/cli/agent-fixtures.mjs +16 -0
- package/cli/agent-mail.mjs +54 -0
- package/cli/agent-network.mjs +64 -0
- package/cli/agent-reviewers.mjs +110 -0
- package/cli/agent-secrets.mjs +23 -0
- package/cli/agent-sponsors.mjs +43 -0
- package/cli/agent-surface.mjs +34 -0
- package/cli/agent-webhooks.mjs +19 -0
- package/cli/authoring.mjs +2 -1
- package/cli/bounded.mjs +1 -1
- package/cli/business/archive-task.mjs +32 -0
- package/cli/business/carry.mjs +151 -0
- package/cli/business/catchup-task.mjs +42 -0
- package/cli/business/choices.mjs +97 -0
- package/cli/business/command.mjs +29 -6
- package/cli/business/connect.mjs +20 -8
- package/cli/business/detector.mjs +4 -1
- package/cli/business/device.mjs +13 -0
- package/cli/business/export-inputs.mjs +69 -0
- package/cli/business/export-task.mjs +17 -0
- package/cli/business/invocation.mjs +8 -1
- package/cli/business/keychain.mjs +8 -1
- package/cli/business/lease.mjs +11 -1
- package/cli/business/local-commands.mjs +80 -30
- package/cli/business/outbound.mjs +9 -4
- package/cli/business/parsers-config.mjs +8 -7
- package/cli/business/parsers-syntax.mjs +7 -3
- package/cli/business/permissions.mjs +2 -0
- package/cli/business/questions.mjs +89 -6
- package/cli/business/setup-command.mjs +49 -0
- package/cli/business/setup-company.json +15 -0
- package/cli/business/setup-demo.mjs +95 -0
- package/cli/business/state.mjs +44 -10
- package/cli/business/supervisor.mjs +97 -32
- package/cli/business/vendor-door.mjs +230 -0
- package/cli/business/vendors/README.md +29 -3
- package/cli/business/vendors/common.mjs +8 -6
- package/cli/business/vendors/index.mjs +22 -11
- package/cli/business/vendors/sorry.mjs +106 -0
- package/cli/container-transfer.mjs +2 -2
- package/cli/contract.mjs +7 -7
- package/cli/demo-deploy-error.mjs +14 -0
- package/cli/demo-native.mjs +6 -2
- package/cli/deploy-command.mjs +6 -4
- package/cli/deploy-config.mjs +1 -1
- package/cli/deploy-owner-input.mjs +3 -1
- package/cli/deploy-variables.mjs +58 -0
- package/cli/deploy.mjs +9 -7
- package/cli/door-clauses.mjs +157 -0
- package/cli/errors.mjs +1 -1
- package/cli/export.mjs +16 -4
- package/cli/hosted-output.mjs +17 -7
- package/cli/human-error.mjs +20 -0
- package/cli/inspect.mjs +4 -0
- package/cli/jsonc.mjs +8 -4
- package/cli/lifecycle.mjs +26 -6
- package/cli/main.mjs +2 -1
- package/cli/managed-data-schema.mjs +127 -0
- package/cli/managed-data-zip.mjs +116 -0
- package/cli/managed-data.mjs +110 -0
- package/cli/managed-server.mjs +25 -14
- package/cli/native-owner-entry.mjs +86 -0
- package/cli/output.mjs +3 -2
- package/cli/parse.mjs +2 -2
- package/cli/portability-empty.mjs +65 -0
- package/cli/portability-migrations.mjs +33 -9
- package/cli/portability-policy.mjs +5 -2
- package/cli/portability-worker.mjs +15 -6
- package/cli/portability.mjs +30 -8
- package/cli/production-auth.mjs +1 -1
- package/cli/production-mail.mjs +8 -2
- package/cli/ready-summary.mjs +9 -0
- package/cli/run-transfers.mjs +37 -0
- package/cli/runtime-front-door.mjs +4 -4
- package/cli/server-proxy.mjs +4 -3
- package/cli/server-registry.mjs +13 -8
- package/cli/site.mjs +6 -1
- package/cli/transfers.mjs +17 -4
- package/edition-authoring/archive-source.mjs +59 -0
- package/edition-authoring/base-copies.mjs +147 -25
- package/edition-authoring/cli.mjs +1 -1
- package/edition-authoring/edition-migrations.mjs +7 -3
- package/edition-authoring/module-closure.mjs +2 -16
- package/edition-authoring/workflow-copies.mjs +1 -1
- package/edition-authoring/workflow-scripts.mjs +12 -3
- package/foundation/runtime/contract-docs.mjs +218 -34
- package/foundation/runtime/contract-prose-patterns.mjs +14 -0
- package/foundation/runtime/contract-prose.mjs +88 -0
- package/foundation/runtime/run-data.d.mts +1 -0
- package/foundation/runtime/run-data.mjs +2 -0
- package/global/CHANGELOG.md +126 -0
- package/global/README.md +156 -50
- package/global/cli/command.mjs +8 -2
- package/global/cli/install.mjs +4 -5
- package/global/cli/path-link.mjs +3 -2
- package/global/cli/repo-command.mjs +3 -7
- package/global/door/command.mjs +3 -1
- package/global/launch/dev.mjs +12 -2
- package/global/launch/facts.mjs +2 -2
- package/global/launch/render.mjs +16 -9
- package/global/launch/state.mjs +4 -4
- package/global/main.mjs +13 -8
- package/install.sh +17 -6
- package/modules/agent-door/CLAUSES.md +215 -0
- package/modules/agent-door/conformance/audit.mjs +66 -0
- package/modules/agent-door/conformance/authority.d.mts +2 -0
- package/modules/agent-door/conformance/authority.mjs +17 -0
- package/modules/agent-door/conformance/crypto.mjs +36 -0
- package/modules/agent-door/conformance/discovery.mjs +61 -0
- package/modules/agent-door/conformance/index.mjs +41 -0
- package/modules/agent-door/conformance/invocation.mjs +32 -0
- package/modules/agent-door/conformance/journey.mjs +112 -0
- package/modules/agent-door/conformance/narrowing.d.mts +1 -0
- package/modules/agent-door/conformance/narrowing.mjs +21 -0
- package/modules/agent-door/conformance/negative.mjs +47 -0
- package/modules/agent-door/conformance/network-modules.mjs +86 -0
- package/modules/agent-door/conformance/network-runtime.mjs +46 -0
- package/modules/agent-door/conformance/policy-phases.mjs +75 -0
- package/modules/agent-door/conformance/protocol.mjs +145 -0
- package/modules/agent-door/conformance/replay.mjs +38 -0
- package/modules/agent-door/conformance/schema.mjs +71 -0
- package/modules/agent-door/conformance/security.mjs +33 -0
- package/modules/agent-door/conformance/sponsors.mjs +68 -0
- package/modules/agent-door/conformance/surface.mjs +89 -0
- package/modules/agent-door/conformance/target.mjs +18 -0
- package/modules/agent-door/conformance/tool-name.d.mts +1 -0
- package/modules/agent-door/conformance/tool-name.mjs +7 -0
- package/modules/agent-door/conformance/webhooks.mjs +30 -0
- package/modules/container/README.md +6 -1
- package/modules/container/adapters/d1-limits.mjs +286 -0
- package/modules/container/adapters/d1-sqlite.mjs +95 -0
- package/modules/container/adapters/d1-workerd-preload.mjs +19 -0
- package/modules/container/adapters/d1-workerd-setup.mjs +26 -0
- package/modules/container/adapters/d1-workerd.mjs +5 -0
- package/modules/container/adapters/sqlite-worker.mjs +5 -3
- package/modules/container/build.mjs +1 -0
- package/modules/container/image-variables.mjs +14 -3
- package/modules/container/transfer.mjs +14 -4
- package/modules/container/worker-engine.mjs +3 -3
- package/modules/demo-host/README.md +6 -0
- package/modules/demo-host/build.mjs +1 -0
- package/modules/demo-host/native/prepare.mjs +2 -1
- package/modules/demo-host/native/sql.mjs +7 -2
- package/modules/demo-host/ownership.mjs +39 -0
- package/modules/demo-host/plan.mjs +2 -1
- package/modules/demo-host/remove.mjs +6 -1
- package/modules/deploy-kit/README.md +22 -2
- package/modules/deploy-kit/d1-import-diagnostics.mjs +56 -0
- package/modules/deploy-kit/d1-import.mjs +17 -5
- package/modules/deploy-kit/diagnostics.mjs +29 -3
- package/modules/deploy-kit/owner-cli.mjs +20 -7
- package/modules/deploy-kit/provider-transport.mjs +19 -4
- package/modules/deploy-kit/provider.mjs +57 -22
- package/modules/deploy-kit/readiness-window.mjs +1 -1
- package/modules/deploy-kit/site.mjs +8 -3
- package/modules/deploy-kit/worker.mjs +4 -4
- package/modules/import-kit/schema/importer-verification-receipt.schema.json +171 -0
- package/modules/import-kit/schema/importers.schema.json +107 -0
- package/modules/import-kit/src/d1/README.md +23 -4
- package/modules/import-kit/src/d1/atomic-prefix.ts +2 -2
- package/modules/import-kit/src/d1/budget.ts +15 -7
- package/modules/import-kit/src/d1/column-layout.ts +5 -0
- package/modules/import-kit/src/d1/creation.ts +15 -8
- package/modules/import-kit/src/d1/database.ts +3 -3
- package/modules/import-kit/src/d1/erasure-pages.ts +2 -1
- package/modules/import-kit/src/d1/erasure.ts +2 -2
- package/modules/import-kit/src/d1/jobs.ts +10 -6
- package/modules/import-kit/src/d1/plans.ts +14 -8
- package/modules/import-kit/src/d1/retirement.ts +2 -2
- package/modules/import-kit/src/d1/sets.ts +1 -3
- package/modules/import-kit/src/d1/staged-creation.ts +3 -3
- package/modules/import-kit/src/door/ADOPTING.md +20 -0
- package/modules/import-kit/src/door/README.md +2 -1
- package/modules/import-kit/src/door/admission.ts +5 -3
- package/modules/import-kit/src/door/archive.ts +5 -2
- package/modules/import-kit/src/door/authority.ts +1 -0
- package/modules/import-kit/src/door/grants.ts +1 -1
- package/modules/import-kit/src/door/handler.ts +14 -8
- package/modules/import-kit/src/door/index.ts +1 -1
- package/modules/import-kit/src/door/jobs.ts +40 -0
- package/modules/import-kit/src/door/limits.ts +1 -1
- package/modules/import-kit/src/door/operator.ts +3 -3
- package/modules/import-kit/src/door/planning.ts +10 -6
- package/modules/import-kit/src/door/protocol.ts +2 -2
- package/modules/import-kit/src/door/raw.ts +25 -0
- package/modules/import-kit/src/door/refusals.ts +2 -1
- package/modules/import-kit/src/door/types.ts +26 -6
- package/modules/import-kit/src/receipt/encoding.ts +37 -0
- package/modules/import-kit/src/receipt/index.ts +69 -0
- package/modules/import-kit/src/receipt/policy.ts +24 -0
- package/modules/import-kit/src/receipt/types.ts +64 -0
- package/modules/import-kit/src/receipt/uri.ts +45 -0
- package/modules/import-kit/src/receipt/validation.ts +87 -0
- package/modules/import-kit/src/status/encoding.ts +13 -0
- package/modules/import-kit/src/status/evidence.ts +218 -0
- package/modules/import-kit/src/status/index.ts +56 -0
- package/modules/import-kit/src/status/schema.ts +208 -0
- package/modules/import-kit/src/status/types.ts +86 -0
- package/modules/operator-auth/accounts/delegated.mjs +48 -0
- package/modules/operator-auth/index.d.mts +7 -0
- package/modules/operator-auth/service.mjs +2 -0
- package/modules/portability/README.md +138 -9
- package/modules/portability/backfills.mjs +9 -8
- package/modules/portability/bridge.mjs +32 -5
- package/modules/portability/catalogue.mjs +12 -5
- package/modules/portability/copies.mjs +8 -1
- package/modules/portability/derived.mjs +33 -0
- package/modules/portability/files.mjs +4 -3
- package/modules/portability/http-errors.mjs +1 -1
- package/modules/portability/http.mjs +3 -3
- package/modules/portability/import.mjs +29 -13
- package/modules/portability/index.d.mts +4 -3
- package/modules/portability/index.mjs +1 -1
- package/modules/portability/ledgers.mjs +3 -2
- package/modules/portability/legacy-safety.mjs +11 -2
- package/modules/portability/migration-history.mjs +2 -2
- package/modules/portability/progress.mjs +1 -0
- package/modules/portability/restore-empty.mjs +12 -0
- package/modules/release/github-verify.mjs +11 -2
- package/onboarding/adapter.mjs +35 -6
- package/onboarding/app-fetch.mjs +6 -4
- package/onboarding/development-runtime.mjs +19 -0
- package/onboarding/local/operation.mjs +45 -27
- package/onboarding/loopback-fetch.mjs +16 -0
- package/onboarding/operator-installer.mjs +2 -4
- package/onboarding/process-diagnostics.mjs +6 -2
- package/onboarding/production-runtime.mjs +4 -2
- package/onboarding/readiness.mjs +1 -0
- package/package.json +4 -2
- package/scripts/publish-check/copy_identities.py +3 -1
- package/scripts/publish-check/github_refs.py +32 -13
- package/scripts/publish-check/public_hygiene.py +1 -1
- package/cli/business/opt-ins.mjs +0 -33
|
@@ -27,6 +27,6 @@ export function readinessAPI(api, window, failure = 'deployment-readiness-timeou
|
|
|
27
27
|
const remaining = window.remaining();
|
|
28
28
|
assert(remaining > 0, failure);
|
|
29
29
|
const deadline = window.timeout(Math.max(1, Math.ceil(remaining)));
|
|
30
|
-
return api.request(path, {...options, signal:options.signal ? AbortSignal.any([options.signal,deadline]) : deadline});
|
|
30
|
+
return api.request(path, {...options, deadlineMs:remaining, signal:options.signal ? AbortSignal.any([options.signal,deadline]) : deadline});
|
|
31
31
|
}};
|
|
32
32
|
}
|
|
@@ -97,6 +97,11 @@ async function writeReceipt(path, receipt) {
|
|
|
97
97
|
export async function siteDeploy({ api, plan, files, receiptPath, readiness = {} }) {
|
|
98
98
|
assert(typeof receiptPath === 'string' && receiptPath.length, 'missing-receipt-path');
|
|
99
99
|
assert(plan?.module === SITE_MODULE && Array.isArray(files) && files.some(file => file.path === 'index.html'), 'invalid-site-assets');
|
|
100
|
+
const provider = api;
|
|
101
|
+
api = {...api,
|
|
102
|
+
zoneRequest:(path,options = {}) => provider.zoneRequest(path,{step:'checking site zone',...options}),
|
|
103
|
+
request:(path,options = {}) => provider.request(path,{step:path.includes('/domains') ? 'checking site custom domains' : 'checking site Worker',...options}),
|
|
104
|
+
};
|
|
100
105
|
const runtime = siteRuntime(plan.runtime);
|
|
101
106
|
const fingerprint = siteFingerprint(plan, files);
|
|
102
107
|
const zones = await api.zoneRequest(`/zones?name=${plan.domain}&account.id=${api.accountId}`);
|
|
@@ -111,7 +116,7 @@ export async function siteDeploy({ api, plan, files, receiptPath, readiness = {}
|
|
|
111
116
|
if (row && (!previous || row.service !== plan.worker)) throw refuse('resource-name-collision', 'domain', hostname);
|
|
112
117
|
}
|
|
113
118
|
|
|
114
|
-
const jwt = await uploadAssets(api, plan.worker, files);
|
|
119
|
+
const jwt = await uploadAssets(api, plan.worker, files, {label:'site assets'});
|
|
115
120
|
const body = new FormData();
|
|
116
121
|
body.set('metadata', new Blob([JSON.stringify({ main_module: SITE_MODULE, compatibility_date: plan.compatibilityDate, bindings: siteRuntimeBindings(runtime),
|
|
117
122
|
assets: { jwt, config: { ...plan.assetsConfig, run_worker_first: true } }, annotations: { 'workers/message': `rer-site:${fingerprint}` } })], { type: 'application/json' }));
|
|
@@ -119,7 +124,7 @@ export async function siteDeploy({ api, plan, files, receiptPath, readiness = {}
|
|
|
119
124
|
const window = readiness.window ?? readinessWindow(readiness);
|
|
120
125
|
window.start();
|
|
121
126
|
api = readinessAPI(api, window, 'site-upload-timeout');
|
|
122
|
-
const upload = await api.request(`/workers/scripts/${plan.worker}`, { method: 'PUT', body });
|
|
127
|
+
const upload = await api.request(`/workers/scripts/${plan.worker}`, { method: 'PUT', body, step:'uploading site Worker with redirect' });
|
|
123
128
|
const versionId = uploadVersionId(upload);
|
|
124
129
|
assert(versionId, 'invalid-upload-version');
|
|
125
130
|
const readback = () => waitForSiteUpload({ api, worker: plan.worker, versionId, fingerprint, previous, window, runtime, module: SITE_MODULE });
|
|
@@ -127,7 +132,7 @@ export async function siteDeploy({ api, plan, files, receiptPath, readiness = {}
|
|
|
127
132
|
await workerOrigin(api, plan.worker, { enable: true, workersDev: false });
|
|
128
133
|
for (const hostname of plan.hostnames) {
|
|
129
134
|
if (attached.some(row => row?.hostname === hostname)) continue;
|
|
130
|
-
const row = await api.request('/workers/domains', { method: 'PUT', body: { environment: 'production', hostname, service: plan.worker, zone_id: zone.id } });
|
|
135
|
+
const row = await api.request('/workers/domains', { method: 'PUT', step:`attaching custom domain ${hostname}`, body: { environment: 'production', hostname, service: plan.worker, zone_id: zone.id } });
|
|
131
136
|
assert(row?.hostname === hostname && row.service === plan.worker && typeof row.id === 'string', 'invalid-domain-attach');
|
|
132
137
|
}
|
|
133
138
|
// Readback: the kit's Worker live and both domains on it.
|
|
@@ -64,26 +64,26 @@ export async function secretBindings(api, worker, plan, env, exists) {
|
|
|
64
64
|
}
|
|
65
65
|
return {bindings,names,unset,missing:plan.secrets.filter(x => !names.includes(x.binding) && !unset.includes(x.binding)).map(x => x.binding)};
|
|
66
66
|
}
|
|
67
|
-
export async function uploadAssets(api, worker, files) {
|
|
67
|
+
export async function uploadAssets(api, worker, files, {label = 'assets'} = {}) {
|
|
68
68
|
const manifest = {}; const byHash = new Map();
|
|
69
69
|
for (const file of files) {
|
|
70
70
|
const digest = hash(file.base64 + extname(file.path).slice(1)).slice(0,32);
|
|
71
71
|
manifest[`/${file.path}`] = {hash:digest,size:Buffer.from(file.base64,'base64').length};
|
|
72
72
|
byHash.set(digest,file);
|
|
73
73
|
}
|
|
74
|
-
const session = await api.request(`/workers/scripts/${worker}/assets-upload-session`,{method:'POST',body:{manifest}})
|
|
74
|
+
const session = await api.request(`/workers/scripts/${worker}/assets-upload-session`,{method:'POST',body:{manifest},step:`creating ${label} upload session`})
|
|
75
75
|
.catch(error => { throw nameAssetFailure(error,files.map(file => file.path)); });
|
|
76
76
|
assert(typeof session.jwt === 'string' && session.jwt && Array.isArray(session.buckets),'invalid-asset-session');
|
|
77
77
|
let completion = session.buckets.length ? null : session.jwt;
|
|
78
78
|
const seen = new Set();
|
|
79
|
-
for (const bucket of session.buckets) {
|
|
79
|
+
for (const [index,bucket] of session.buckets.entries()) {
|
|
80
80
|
assert(Array.isArray(bucket) && bucket.length,'invalid-asset-bucket');
|
|
81
81
|
const data = new FormData();
|
|
82
82
|
for (const key of bucket) {
|
|
83
83
|
assert(byHash.has(key) && !seen.has(key),'invalid-asset-request'); seen.add(key);
|
|
84
84
|
const file = byHash.get(key); data.set(key,new Blob([file.base64],{type:assetType(file.path)}),file.path.split('/').at(-1));
|
|
85
85
|
}
|
|
86
|
-
const uploaded = await api.request('/workers/assets/upload?base64=true',{method:'POST',body:data,assetToken:session.jwt})
|
|
86
|
+
const uploaded = await api.request('/workers/assets/upload?base64=true',{method:'POST',body:data,assetToken:session.jwt,step:`uploading ${label}, batch ${index + 1} of ${session.buckets.length}`})
|
|
87
87
|
.catch(error => { throw nameAssetFailure(error,bucket.map(key => byHash.get(key).path)); });
|
|
88
88
|
if (uploaded.jwt) completion = uploaded.jwt;
|
|
89
89
|
}
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "https://runeditrun.com/schemas/import-kit/importer-verification-receipt-v1.schema.json",
|
|
4
|
+
"title": "Signed real-export importer verification receipt",
|
|
5
|
+
"type": "object",
|
|
6
|
+
"required": ["schemaVersion", "type", "keyId", "algorithm", "payload", "signature"],
|
|
7
|
+
"properties": {
|
|
8
|
+
"schemaVersion": { "const": 1 },
|
|
9
|
+
"type": { "const": "runeditrun.importer-verification-receipt" },
|
|
10
|
+
"keyId": { "type": "string", "minLength": 1 },
|
|
11
|
+
"algorithm": { "const": "Ed25519" },
|
|
12
|
+
"payload": { "$ref": "#/$defs/payload" },
|
|
13
|
+
"signature": {
|
|
14
|
+
"type": "string",
|
|
15
|
+
"description": "Base64 Ed25519 signature over UTF-8 canonical JSON of payload, with object keys sorted recursively.",
|
|
16
|
+
"pattern": "^[A-Za-z0-9+/]+={0,2}$"
|
|
17
|
+
}
|
|
18
|
+
},
|
|
19
|
+
"additionalProperties": false,
|
|
20
|
+
"$defs": {
|
|
21
|
+
"hash": { "type": "string", "pattern": "^[a-f0-9]{64}$" },
|
|
22
|
+
"revision": { "type": "string", "pattern": "^[a-f0-9]{40,64}$" },
|
|
23
|
+
"timestamp": { "type": "string", "format": "date-time" },
|
|
24
|
+
"counts": {
|
|
25
|
+
"type": "object",
|
|
26
|
+
"required": ["input", "unique", "duplicates", "mapped", "unmapped", "rejected", "effects", "byKind"],
|
|
27
|
+
"properties": {
|
|
28
|
+
"input": { "type": "integer", "minimum": 0 },
|
|
29
|
+
"unique": { "type": "integer", "minimum": 0 },
|
|
30
|
+
"duplicates": { "type": "integer", "minimum": 0 },
|
|
31
|
+
"mapped": { "type": "integer", "minimum": 0 },
|
|
32
|
+
"unmapped": { "type": "integer", "minimum": 0 },
|
|
33
|
+
"rejected": { "type": "integer", "minimum": 0 },
|
|
34
|
+
"effects": { "type": "integer", "minimum": 0 },
|
|
35
|
+
"byKind": {
|
|
36
|
+
"type": "object",
|
|
37
|
+
"minProperties": 1,
|
|
38
|
+
"additionalProperties": { "type": "integer", "minimum": 0 }
|
|
39
|
+
}
|
|
40
|
+
},
|
|
41
|
+
"additionalProperties": false
|
|
42
|
+
},
|
|
43
|
+
"manifestComparison": {
|
|
44
|
+
"type": "object",
|
|
45
|
+
"required": ["sourceCount", "targetCount", "sourceManifestSha256", "targetManifestSha256"],
|
|
46
|
+
"properties": {
|
|
47
|
+
"sourceCount": { "type": "integer", "minimum": 0 },
|
|
48
|
+
"targetCount": { "type": "integer", "minimum": 0 },
|
|
49
|
+
"sourceManifestSha256": { "$ref": "#/$defs/hash" },
|
|
50
|
+
"targetManifestSha256": { "$ref": "#/$defs/hash" }
|
|
51
|
+
},
|
|
52
|
+
"additionalProperties": false
|
|
53
|
+
},
|
|
54
|
+
"payload": {
|
|
55
|
+
"type": "object",
|
|
56
|
+
"required": [
|
|
57
|
+
"incumbent", "product", "synthetic", "source", "candidate", "adapter",
|
|
58
|
+
"mapping", "command", "runs", "reconciliation", "unmapped", "research", "review"
|
|
59
|
+
],
|
|
60
|
+
"properties": {
|
|
61
|
+
"incumbent": { "type": "string", "minLength": 1 },
|
|
62
|
+
"product": { "type": "string", "minLength": 1 },
|
|
63
|
+
"synthetic": { "const": false },
|
|
64
|
+
"source": {
|
|
65
|
+
"type": "object",
|
|
66
|
+
"required": ["classification", "origin", "exportedAt", "format", "version", "sha256", "size"],
|
|
67
|
+
"properties": {
|
|
68
|
+
"classification": { "const": "authorized-real-export" },
|
|
69
|
+
"origin": { "type": "string", "minLength": 1 },
|
|
70
|
+
"exportedAt": { "$ref": "#/$defs/timestamp" },
|
|
71
|
+
"format": { "type": "string", "minLength": 1 },
|
|
72
|
+
"version": { "type": "string", "minLength": 1 },
|
|
73
|
+
"sha256": { "$ref": "#/$defs/hash" },
|
|
74
|
+
"size": { "type": "integer", "minimum": 1 }
|
|
75
|
+
},
|
|
76
|
+
"additionalProperties": false
|
|
77
|
+
},
|
|
78
|
+
"candidate": {
|
|
79
|
+
"type": "object",
|
|
80
|
+
"required": ["repository", "revision"],
|
|
81
|
+
"properties": {
|
|
82
|
+
"repository": { "type": "string", "minLength": 1 },
|
|
83
|
+
"revision": { "$ref": "#/$defs/revision" }
|
|
84
|
+
},
|
|
85
|
+
"additionalProperties": false
|
|
86
|
+
},
|
|
87
|
+
"adapter": {
|
|
88
|
+
"type": "object",
|
|
89
|
+
"required": ["repository", "revision"],
|
|
90
|
+
"properties": {
|
|
91
|
+
"repository": { "type": "string", "minLength": 1 },
|
|
92
|
+
"revision": { "$ref": "#/$defs/revision" }
|
|
93
|
+
},
|
|
94
|
+
"additionalProperties": false
|
|
95
|
+
},
|
|
96
|
+
"mapping": {
|
|
97
|
+
"type": "object",
|
|
98
|
+
"required": ["version", "policy", "policySha256"],
|
|
99
|
+
"properties": {
|
|
100
|
+
"version": { "type": "string", "minLength": 1 },
|
|
101
|
+
"policy": { "type": "string", "minLength": 1 },
|
|
102
|
+
"policySha256": { "$ref": "#/$defs/hash" }
|
|
103
|
+
},
|
|
104
|
+
"additionalProperties": false
|
|
105
|
+
},
|
|
106
|
+
"command": { "type": "string", "minLength": 1 },
|
|
107
|
+
"runs": {
|
|
108
|
+
"type": "object",
|
|
109
|
+
"required": ["dryRun", "apply"],
|
|
110
|
+
"properties": {
|
|
111
|
+
"dryRun": { "$ref": "#/$defs/counts" },
|
|
112
|
+
"apply": { "$ref": "#/$defs/counts" }
|
|
113
|
+
},
|
|
114
|
+
"additionalProperties": false
|
|
115
|
+
},
|
|
116
|
+
"reconciliation": {
|
|
117
|
+
"type": "object",
|
|
118
|
+
"required": ["rows", "provenance", "attachments"],
|
|
119
|
+
"properties": {
|
|
120
|
+
"rows": { "$ref": "#/$defs/manifestComparison" },
|
|
121
|
+
"provenance": { "$ref": "#/$defs/manifestComparison" },
|
|
122
|
+
"attachments": { "$ref": "#/$defs/manifestComparison" }
|
|
123
|
+
},
|
|
124
|
+
"additionalProperties": false
|
|
125
|
+
},
|
|
126
|
+
"unmapped": {
|
|
127
|
+
"type": "object",
|
|
128
|
+
"required": ["count", "itemManifestSha256", "items"],
|
|
129
|
+
"properties": {
|
|
130
|
+
"count": { "type": "integer", "minimum": 0 },
|
|
131
|
+
"itemManifestSha256": { "$ref": "#/$defs/hash" },
|
|
132
|
+
"items": {
|
|
133
|
+
"type": "array",
|
|
134
|
+
"items": {
|
|
135
|
+
"type": "object",
|
|
136
|
+
"required": ["sourceIdentitySha256", "sourceKind", "fields"],
|
|
137
|
+
"properties": {
|
|
138
|
+
"sourceIdentitySha256": { "$ref": "#/$defs/hash" },
|
|
139
|
+
"sourceKind": { "type": "string", "minLength": 1 },
|
|
140
|
+
"fields": { "type": "array", "minItems": 1, "items": { "type": "string", "minLength": 1 } }
|
|
141
|
+
},
|
|
142
|
+
"additionalProperties": false
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
},
|
|
146
|
+
"additionalProperties": false
|
|
147
|
+
},
|
|
148
|
+
"research": {
|
|
149
|
+
"type": "object",
|
|
150
|
+
"required": ["state", "sha256"],
|
|
151
|
+
"properties": {
|
|
152
|
+
"state": { "const": "finished" },
|
|
153
|
+
"sha256": { "$ref": "#/$defs/hash" }
|
|
154
|
+
},
|
|
155
|
+
"additionalProperties": false
|
|
156
|
+
},
|
|
157
|
+
"review": {
|
|
158
|
+
"type": "object",
|
|
159
|
+
"required": ["reviewer", "reviewedAt", "result"],
|
|
160
|
+
"properties": {
|
|
161
|
+
"reviewer": { "type": "string", "minLength": 3, "maxLength": 128, "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]+$" },
|
|
162
|
+
"reviewedAt": { "$ref": "#/$defs/timestamp" },
|
|
163
|
+
"result": { "const": "passed" }
|
|
164
|
+
},
|
|
165
|
+
"additionalProperties": false
|
|
166
|
+
}
|
|
167
|
+
},
|
|
168
|
+
"additionalProperties": false
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
}
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "https://runeditrun.com/schemas/import-kit/importers-v1.schema.json",
|
|
4
|
+
"title": "Evidence-bound importer status catalogue",
|
|
5
|
+
"type": "object",
|
|
6
|
+
"required": ["$schema", "schemaVersion", "importers"],
|
|
7
|
+
"properties": {
|
|
8
|
+
"$schema": { "const": "./schema/importers.schema.json" },
|
|
9
|
+
"schemaVersion": { "const": 1 },
|
|
10
|
+
"importers": {
|
|
11
|
+
"type": "array",
|
|
12
|
+
"minItems": 1,
|
|
13
|
+
"items": {
|
|
14
|
+
"oneOf": [
|
|
15
|
+
{ "$ref": "#/$defs/verifiedImporter" },
|
|
16
|
+
{ "$ref": "#/$defs/documentedImporter" },
|
|
17
|
+
{ "$ref": "#/$defs/noneImporter" }
|
|
18
|
+
]
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
},
|
|
22
|
+
"additionalProperties": false,
|
|
23
|
+
"$defs": {
|
|
24
|
+
"artifact": {
|
|
25
|
+
"type": "object",
|
|
26
|
+
"required": ["link", "sha256", "size"],
|
|
27
|
+
"properties": {
|
|
28
|
+
"link": { "type": "string", "minLength": 1 },
|
|
29
|
+
"sha256": { "type": "string", "pattern": "^[a-f0-9]{64}$" },
|
|
30
|
+
"size": { "type": "integer", "minimum": 1 }
|
|
31
|
+
},
|
|
32
|
+
"additionalProperties": false
|
|
33
|
+
},
|
|
34
|
+
"verifiedImporter": {
|
|
35
|
+
"type": "object",
|
|
36
|
+
"required": ["incumbent", "product", "status", "evidence"],
|
|
37
|
+
"properties": {
|
|
38
|
+
"incumbent": { "type": "string", "minLength": 1 },
|
|
39
|
+
"product": { "type": "string", "minLength": 1 },
|
|
40
|
+
"status": { "const": "verified" },
|
|
41
|
+
"evidence": {
|
|
42
|
+
"type": "object",
|
|
43
|
+
"required": ["link", "sha256", "size", "research"],
|
|
44
|
+
"properties": {
|
|
45
|
+
"link": { "type": "string", "minLength": 1 },
|
|
46
|
+
"sha256": { "type": "string", "pattern": "^[a-f0-9]{64}$" },
|
|
47
|
+
"size": { "type": "integer", "minimum": 1 },
|
|
48
|
+
"research": { "$ref": "#/$defs/artifact" }
|
|
49
|
+
},
|
|
50
|
+
"additionalProperties": false
|
|
51
|
+
}
|
|
52
|
+
},
|
|
53
|
+
"additionalProperties": false
|
|
54
|
+
},
|
|
55
|
+
"documentedImporter": {
|
|
56
|
+
"type": "object",
|
|
57
|
+
"required": ["incumbent", "product", "status", "evidence"],
|
|
58
|
+
"properties": {
|
|
59
|
+
"incumbent": { "type": "string", "minLength": 1 },
|
|
60
|
+
"product": { "type": "string", "minLength": 1 },
|
|
61
|
+
"status": { "const": "documented format" },
|
|
62
|
+
"evidence": {
|
|
63
|
+
"type": "object",
|
|
64
|
+
"required": [
|
|
65
|
+
"link", "sha256", "size", "sourceRepository", "sourceRevision",
|
|
66
|
+
"sourcePath", "sourceSha256", "validation", "realExport"
|
|
67
|
+
],
|
|
68
|
+
"properties": {
|
|
69
|
+
"link": { "type": "string", "minLength": 1 },
|
|
70
|
+
"sha256": { "type": "string", "pattern": "^[a-f0-9]{64}$" },
|
|
71
|
+
"size": { "type": "integer", "minimum": 1 },
|
|
72
|
+
"sourceRepository": { "type": "string", "minLength": 1 },
|
|
73
|
+
"sourceRevision": { "type": "string", "pattern": "^[a-f0-9]{40,64}$" },
|
|
74
|
+
"sourcePath": { "type": "string", "minLength": 1 },
|
|
75
|
+
"sourceSha256": { "type": "string", "pattern": "^[a-f0-9]{64}$" },
|
|
76
|
+
"validation": { "const": "synthetic-only" },
|
|
77
|
+
"realExport": { "const": "not-supplied" }
|
|
78
|
+
},
|
|
79
|
+
"additionalProperties": false
|
|
80
|
+
}
|
|
81
|
+
},
|
|
82
|
+
"additionalProperties": false
|
|
83
|
+
},
|
|
84
|
+
"noneImporter": {
|
|
85
|
+
"type": "object",
|
|
86
|
+
"required": ["incumbent", "product", "status", "evidence"],
|
|
87
|
+
"properties": {
|
|
88
|
+
"incumbent": { "type": "string", "minLength": 1 },
|
|
89
|
+
"product": { "type": "string", "minLength": 1 },
|
|
90
|
+
"status": { "const": "none" },
|
|
91
|
+
"evidence": {
|
|
92
|
+
"type": "object",
|
|
93
|
+
"required": ["link", "sha256", "size", "reason", "checkedAt"],
|
|
94
|
+
"properties": {
|
|
95
|
+
"link": { "type": "string", "minLength": 1 },
|
|
96
|
+
"sha256": { "type": "string", "pattern": "^[a-f0-9]{64}$" },
|
|
97
|
+
"size": { "type": "integer", "minimum": 1 },
|
|
98
|
+
"reason": { "type": "string", "minLength": 1 },
|
|
99
|
+
"checkedAt": { "type": "string", "format": "date-time" }
|
|
100
|
+
},
|
|
101
|
+
"additionalProperties": false
|
|
102
|
+
}
|
|
103
|
+
},
|
|
104
|
+
"additionalProperties": false
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
}
|
|
@@ -277,13 +277,32 @@ all layers. An import still applying or reversing cannot be superseded
|
|
|
277
277
|
The bounds below follow [D1 limits](https://developers.cloudflare.com/d1/platform/limits/).
|
|
278
278
|
Bulk text insertion uses the supported [D1 JSON SQL functions](https://developers.cloudflare.com/d1/sql-api/query-json/).
|
|
279
279
|
|
|
280
|
-
The port
|
|
281
|
-
|
|
280
|
+
The port assumes Workers Paid: one Worker invocation allows 1,000 D1 queries;
|
|
281
|
+
Workers Free allows only 50 and is not supported by this budget. Each direct
|
|
282
|
+
read and each `batch()` call counts as one query, independent of batch members.
|
|
283
|
+
Statement length, bound-parameter and value guards still check every member.
|
|
284
|
+
Planning and creation preflight share the production batch-cost function; a
|
|
285
|
+
plan exceeding the fresh creation query ceiling is refused during planning
|
|
286
|
+
with `IMPORT_PLAN_TOO_LARGE`, naming the 1,000-query paid-plan limit.
|
|
287
|
+
`preflightD1ImportJob().totalExecutions` measures those D1 queries;
|
|
288
|
+
`totalStatements` and `mutationStatements` retain the individual statement
|
|
289
|
+
counts, and `mutationExecutions` counts mutation batch calls.
|
|
290
|
+
The door shares one budget across all its kit calls in one HTTP request; later
|
|
291
|
+
apply, resume and undo requests are distinct Worker invocations. Erasure retains
|
|
292
|
+
its existing 1,000-statement work ceiling, counting every batch member as well
|
|
293
|
+
as reads. An explicit `maxStatements` may tighten that separate ceiling.
|
|
294
|
+
|
|
295
|
+
The port counts queries over the entire public invocation (including
|
|
296
|
+
nested resume reads and atomic batches), caps them at 1,000 and
|
|
282
297
|
returns durable partial progress as the budget approaches exhaustion. Explicit
|
|
283
298
|
`IMPORT_D1_REQUEST_LIMIT`, `IMPORT_D1_STATEMENT_LIMIT` and
|
|
284
299
|
`IMPORT_D1_VALUE_LIMIT` failures publish no portion of the rejected batch.
|
|
285
|
-
Create, apply and reversal reserve
|
|
286
|
-
|
|
300
|
+
Create, apply and reversal reserve their return reads before publication,
|
|
301
|
+
including chunked-plan readback. A shared calculation sizes plan pages from the
|
|
302
|
+
maximum JSON-encoded stored chunk row bytes, measured before reading, with space
|
|
303
|
+
reserved under the transport ceiling for row metadata and two-byte strings.
|
|
304
|
+
Today's small chunks use up to 64 rows per page; larger legacy chunks use
|
|
305
|
+
smaller pages. Neither the stored plan format nor the canonical byte cap changes. Each successful batch retains its checked job state;
|
|
287
306
|
if a later batch's preflight or exact SQL count exhausts the same request budget,
|
|
288
307
|
the invocation returns that durable partial progress. Other adapter, integrity
|
|
289
308
|
and authority failures still reject. A first batch that cannot fit rejects before
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { ImportAction } from '../types.ts';
|
|
2
2
|
import type { D1ImportContext, D1Sql } from './types.ts';
|
|
3
3
|
import type { NativeStagingScope } from './native-staging-context.ts';
|
|
4
|
-
import { GuardPacking, valueBytes } from './budget.ts';
|
|
4
|
+
import { batchExecutions, GuardPacking, valueBytes } from './budget.ts';
|
|
5
5
|
import { authority, finalAuthority, postConditions, sql } from './database.ts';
|
|
6
6
|
|
|
7
7
|
/** Unbounded publication has append-only SQL and final checks. Bounded native
|
|
@@ -56,7 +56,7 @@ export function atomicPrefix(context: D1ImportContext, action: ImportAction, sta
|
|
|
56
56
|
const writes = this.writes.count([...tail, authority(context, action), sql('DELETE FROM ik_d1_assertions')]);
|
|
57
57
|
const checks = tail.flatMap(s => s.finalChecks ?? []), post = this.post.count(checks);
|
|
58
58
|
if (this.postLength + checks.length) finalAuthority(context, action);
|
|
59
|
-
return writes + post + Number(this.postLength + checks.length > 0) + returnReads;
|
|
59
|
+
return batchExecutions(context, writes + post + Number(this.postLength + checks.length > 0)) + returnReads;
|
|
60
60
|
}
|
|
61
61
|
}
|
|
62
62
|
const post = postConditions(context, statements, scope, action);
|
|
@@ -2,7 +2,15 @@ import { Buffer } from 'node:buffer';
|
|
|
2
2
|
import { wireDatabase } from '../../../d1-transport/index.mjs';
|
|
3
3
|
import { guard, guardPredicate } from './guards.ts';
|
|
4
4
|
import type { D1Database, D1ImportContext, D1Sql } from './types.ts';
|
|
5
|
-
const
|
|
5
|
+
export const D1_REQUEST_QUERY_LIMIT = 1000;
|
|
6
|
+
const D1_BATCH_QUERY_COST = 1;
|
|
7
|
+
const budgets = new WeakMap<D1Database, { used: number; statements: number; statementLimit?: number }>();
|
|
8
|
+
/** Paid D1 counts one batch call as one query, regardless of its members.
|
|
9
|
+
* An explicit erasure work ceiling still counts each executed statement. */
|
|
10
|
+
export function batchExecutions(context: { storage?: Pick<D1ImportContext['storage'], 'database'> }, statements: number): number {
|
|
11
|
+
const budget = context.storage ? budgets.get(context.storage.database) : undefined;
|
|
12
|
+
return budget?.statementLimit === undefined ? D1_BATCH_QUERY_COST : statements;
|
|
13
|
+
}
|
|
6
14
|
const reservedTail = Symbol('import request return queries');
|
|
7
15
|
type ReservedContext = D1ImportContext & { [reservedTail]?: number };
|
|
8
16
|
/** Distinguish this request's measured exhaustion from adapter/provider errors. */
|
|
@@ -14,13 +22,13 @@ export function checkSql(s: D1Sql): void {
|
|
|
14
22
|
if (Buffer.byteLength(s.sql) > 100000 || (s.params?.length ?? 0) > 100) throw new Error('IMPORT_D1_STATEMENT_LIMIT');
|
|
15
23
|
if ((s.params ?? []).some(p => valueBytes(p) > 2000000) || (s.params ?? []).reduce((n, p) => n + valueBytes(p), 0) > 2000000) throw new Error('IMPORT_D1_VALUE_LIMIT');
|
|
16
24
|
}
|
|
17
|
-
export function remaining(context: D1ImportContext): number { const budget=budgets.get(context.storage.database);return (
|
|
25
|
+
export function remaining(context: D1ImportContext): number { const budget=budgets.get(context.storage.database);return Math.min(D1_REQUEST_QUERY_LIMIT-(budget?.used??0), budget?.statementLimit === undefined ? Infinity : budget.statementLimit-budget.statements)-((context as ReservedContext)[reservedTail]??0); }
|
|
18
26
|
/** A smaller public-operation budget also stops reads, before execution. Nested
|
|
19
27
|
* calls can only tighten the shared request's existing execution ceiling. */
|
|
20
28
|
export function limitStatements(context:D1ImportContext,max:number):D1ImportContext {
|
|
21
29
|
const budget=budgets.get(context.storage.database);
|
|
22
|
-
if(!budget||!Number.isSafeInteger(max)||max<1||max>
|
|
23
|
-
budget.
|
|
30
|
+
if(!budget||!Number.isSafeInteger(max)||max<1||max>D1_REQUEST_QUERY_LIMIT)throw new Error('INVALID_ERASURE_STATEMENT_LIMIT');
|
|
31
|
+
budget.statementLimit=Math.min(budget.statementLimit??D1_REQUEST_QUERY_LIMIT,budget.statements+max);
|
|
24
32
|
return context;
|
|
25
33
|
}
|
|
26
34
|
/** A nested operation leaves the outer operation's exact bounded return path
|
|
@@ -36,11 +44,11 @@ export function requestDatabase(database: D1Database): D1Database {
|
|
|
36
44
|
return requestContext({ storage: { database } } as D1ImportContext).storage.database;
|
|
37
45
|
}
|
|
38
46
|
/** A fresh wrapper belongs to the outer public invocation; nested resume calls
|
|
39
|
-
* reuse it. It counts
|
|
47
|
+
* reuse it. It counts D1 calls; every batch is one query. Statement work limits are separate. */
|
|
40
48
|
export function requestContext(context: D1ImportContext): D1ImportContext {
|
|
41
49
|
if (budgets.has(context.storage.database)) return context;
|
|
42
|
-
const db = wireDatabase(context.storage.database), budget = { used: 0,
|
|
43
|
-
const consume = (n: number) => { if (budget.used + n > budget.
|
|
50
|
+
const db = wireDatabase(context.storage.database), budget: { used: number; statements: number; statementLimit?: number } = { used: 0, statements: 0 };
|
|
51
|
+
const consume = (n: number) => { const queries = D1_BATCH_QUERY_COST; if (budget.used + queries > D1_REQUEST_QUERY_LIMIT || budget.statements + n > (budget.statementLimit ?? Infinity)) throw new D1RequestLimitError(); budget.used += queries; budget.statements += n; };
|
|
44
52
|
const raw = new WeakMap<object, any>();
|
|
45
53
|
const wrap = (statement: any, query: D1Sql): any => {
|
|
46
54
|
checkSql(query);
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
// Static layouts have no engine imports, so either test-port entry order
|
|
2
|
+
// can initialize the erasure planner without a module-evaluation cycle.
|
|
3
|
+
export const jobColumns = ['id','destination','namespace','mapping_version','principal_id','fingerprint','operation_key','plan_json','state','checkpoint','total','fence','lease_token','lease_expires','error','reversal_policy','reversal_policy_hash','reversal_cursor','reversal_hash'];
|
|
4
|
+
export const provenanceColumns = ['id','job_id','ordinal','effect_ordinal','source_identity','destination','target_kind','target_id','reversed','record_json'];
|
|
5
|
+
export const claimColumns = ['identity','source_hash','mapping_version','effects_hash','job_id'];
|
|
@@ -8,13 +8,13 @@ import type { D1ImportContext, D1PlanCodec, D1Sql } from './types.ts';
|
|
|
8
8
|
import { planFingerprint } from '../plan/index.ts';
|
|
9
9
|
import { assertPlanSize } from '../plan/size.ts';
|
|
10
10
|
import { assertPlanCredentials } from '../provenance/credentials.ts';
|
|
11
|
-
import { preparedAtomic, guard, sql } from './database.ts';
|
|
11
|
+
import { atomicCount, preparedAtomic, guard, sql } from './database.ts';
|
|
12
12
|
import { retainPlan, planChunkReads, creationCodec, LARGE_PLAN_BYTES } from './plans.ts';
|
|
13
|
-
import { preflightStagedCreation, stagedSize } from './staged-creation.ts';
|
|
13
|
+
import { preflightStagedCreation, stagedCreationCost, stagedSize } from './staged-creation.ts';
|
|
14
14
|
import { stagingStatements } from './staging-statements.ts';
|
|
15
15
|
import { custodyGuards } from './original-storage.ts';
|
|
16
16
|
import { retainOriginals } from './originals.ts';
|
|
17
|
-
import { checkSql,
|
|
17
|
+
import { batchExecutions, checkSql, D1_REQUEST_QUERY_LIMIT, valueBytes } from './budget.ts';
|
|
18
18
|
import { policy, policyHash, EMPTY_REVERSAL_HASH } from './undo-state.ts';
|
|
19
19
|
|
|
20
20
|
export type D1CreatePreflightContext = Pick<D1ImportContext, 'destination' | 'namespace' | 'mappingVersion' | 'authorization' | 'reviewedFingerprint' | 'operationKey' | 'reversalPolicy' | 'maxBatchBytes'> & {
|
|
@@ -22,10 +22,13 @@ export type D1CreatePreflightContext = Pick<D1ImportContext, 'destination' | 'na
|
|
|
22
22
|
planCodec?: D1PlanCodec;
|
|
23
23
|
};
|
|
24
24
|
export interface D1CreatePreflight {
|
|
25
|
-
/** Statements
|
|
25
|
+
/** Statements across private staging and atomic job, plan and custody publication. */
|
|
26
26
|
mutationStatements: number;
|
|
27
|
-
|
|
27
|
+
mutationExecutions: number;
|
|
28
|
+
/** D1 queries: authority batch, operation-key lookup, mutation and return reads. */
|
|
28
29
|
totalExecutions: number;
|
|
30
|
+
/** All individual statements, including the members of each batch. */
|
|
31
|
+
totalStatements: number;
|
|
29
32
|
maxSqlBytes: number;
|
|
30
33
|
maxBoundBytes: number;
|
|
31
34
|
maxParameters: number;
|
|
@@ -82,13 +85,17 @@ export function preflightD1ImportJob(plan: ImportPlan, context: D1CreatePrefligh
|
|
|
82
85
|
sql('SELECT * FROM ik_jobs WHERE id=?', '00000000-0000-0000-0000-000000000000'),
|
|
83
86
|
sql('SELECT * FROM ik_d1_plan_chunks WHERE job_id IN (SELECT value FROM json_each(?)) ORDER BY job_id,ordinal', JSON.stringify(['00000000-0000-0000-0000-000000000000']))];
|
|
84
87
|
const chunkRead = reads.pop()!;
|
|
85
|
-
|
|
88
|
+
const chunkReads = planChunkReads(retained.body, unbounded, retained.encoded.chunks);
|
|
89
|
+
for (let n = 0; n < chunkReads; n++) reads.push(chunkRead);
|
|
86
90
|
if (policy(context) === 'preserve-conflicts') reads.push(sql('SELECT r.*,p.reversed AS native_reversed FROM ik_d1_reversal_reports r LEFT JOIN ik_provenance p ON p.id=r.effect_id AND p.job_id=r.job_id WHERE r.job_id=? ORDER BY r.position', '00000000-0000-0000-0000-000000000000'));
|
|
87
91
|
const erasedReads = erasedSourcePredicates(plan).map(p => ({ sql: `SELECT ${p.sql} AS clear`, params: p.params }));
|
|
88
92
|
const all = [...initial, ...erasedReads, ...statements, ...reads];
|
|
89
93
|
for (const statement of all) checkSql(statement);
|
|
90
|
-
|
|
91
|
-
|
|
94
|
+
const queryContext = context as D1ImportContext;
|
|
95
|
+
const mutationExecutions = staged ? stagedCreationCost(queryContext, staged) : atomicCount(queryContext, 'create', creation, 0);
|
|
96
|
+
const totalExecutions = batchExecutions(queryContext, initial.length) + erasedReads.length + reads.length + mutationExecutions;
|
|
97
|
+
if (totalExecutions > D1_REQUEST_QUERY_LIMIT) throw new Error(`IMPORT_PLAN_TOO_LARGE: D1 creation needs ${totalExecutions} queries; the paid-plan limit is ${D1_REQUEST_QUERY_LIMIT}`);
|
|
98
|
+
return { mutationStatements: statements.length, totalExecutions, mutationExecutions, totalStatements: all.length,
|
|
92
99
|
maxSqlBytes: Math.max(...all.map(s => Buffer.byteLength(s.sql))),
|
|
93
100
|
maxBoundBytes: Math.max(...all.map(s => (s.params ?? []).reduce((n, p) => n + valueBytes(p), 0))),
|
|
94
101
|
maxParameters: Math.max(...all.map(s => s.params?.length ?? 0)) };
|
|
@@ -8,7 +8,7 @@ import { d1NativeStagingSchema } from './native-staging-schema.ts';
|
|
|
8
8
|
import { d1CustodyStateSchema, d1CustodyStateReplaceSchema } from './custody-state-schema.ts';
|
|
9
9
|
import { d1ReversalFenceSchema } from './reversal-fence-schema.ts';
|
|
10
10
|
import { batchBytes } from './transport.ts';
|
|
11
|
-
import { D1RequestLimitError, packGuards, remaining } from './budget.ts';
|
|
11
|
+
import { batchExecutions, D1RequestLimitError, packGuards, remaining } from './budget.ts';
|
|
12
12
|
import { d1ErasureSchema } from './erasure-schema.ts';
|
|
13
13
|
import { d1RetirementSchema } from './retirement-schema.ts';
|
|
14
14
|
import { d1ErasureProgressSchema } from './erasure-progress-schema.ts';
|
|
@@ -130,7 +130,7 @@ export const atomicCount = (context:Parameters<typeof authority>[0]&Pick<D1Impor
|
|
|
130
130
|
function atomicPreparation(context:Parameters<typeof authority>[0]&Pick<D1ImportContext,'maxBatchBytes'>,action:ImportAction,statements:D1Sql[],scope?:NativeStagingScope,returnReads=0){
|
|
131
131
|
const post=postConditions(context,statements,scope,action);
|
|
132
132
|
const packed=[...atomicStatements(context,action,statements),...readonlyChecks(post),...(post.length?[finalAuthority(context,action)]:[])];
|
|
133
|
-
return {post,packed,count:packed.length+returnReads};
|
|
133
|
+
return {post,packed,count:batchExecutions(context as D1ImportContext,packed.length)+returnReads};
|
|
134
134
|
}
|
|
135
135
|
export { atomicPrefix } from './atomic-prefix.ts';
|
|
136
136
|
/** SELECT-only fence: inspect every packed scalar before publishing snapshots.
|
|
@@ -145,7 +145,7 @@ export async function readAtomic(context: D1ImportContext, guards: D1Sql[], scop
|
|
|
145
145
|
await readBoundedGuards(context, checks, scope, Math.min(limit,2*1024*1024), action);
|
|
146
146
|
return;
|
|
147
147
|
}
|
|
148
|
-
if (packed.length > remaining(context)) throw new D1RequestLimitError();
|
|
148
|
+
if (batchExecutions(context, packed.length) > remaining(context)) throw new D1RequestLimitError();
|
|
149
149
|
const results = await context.storage.database.batch(packed.map(s => prepare(context.storage.database, s)));
|
|
150
150
|
checkBatchResults(results, packed.length, 'IMPORT_ATOMIC_GUARD');
|
|
151
151
|
for (const result of results) {
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { D1Database,D1Sql } from './types.ts';
|
|
2
|
-
import { jsonChunks
|
|
2
|
+
import { jsonChunks } from './sets.ts';
|
|
3
|
+
import { jobColumns, provenanceColumns, claimColumns } from './column-layout.ts';
|
|
3
4
|
|
|
4
5
|
const PAGE_BYTES=2*1024*1024;
|
|
5
6
|
const UNKNOWN_BYTES=Number.MAX_SAFE_INTEGER;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { erasedCustody } from './erased-custody.ts';
|
|
2
2
|
import { targetGuards } from './target-guards.ts';
|
|
3
3
|
import type { D1FieldMatch, D1ImportContext, D1RelatedErasure, D1Sql } from './types.ts';
|
|
4
|
-
import { requestContext, packGuards, remaining, D1RequestLimitError } from './budget.ts';
|
|
4
|
+
import { requestContext, limitStatements, packGuards, remaining, D1RequestLimitError } from './budget.ts';
|
|
5
5
|
import { readAtomic, authority, finalAuthority, readonlyChecks, guard, sql, prepare } from './database.ts';
|
|
6
6
|
import { decode } from './jobs.ts';
|
|
7
7
|
import { retainPlan } from './plans.ts';
|
|
@@ -34,7 +34,7 @@ import { supersededEntries } from './erasure-superseded.ts';
|
|
|
34
34
|
* must redact them. `related.fields` instead sanitizes peer values while
|
|
35
35
|
* preserving their effects, provenance and reversible before-images. */
|
|
36
36
|
export async function eraseD1ImportTarget(kind: string, id: string, nativeStatements: D1Sql[], context: D1ImportContext, related?: D1RelatedErasure): Promise<{ nativeResults: unknown[]; erasedRecords: number; fieldMatches: D1FieldMatch[] }> {
|
|
37
|
-
context = requestContext(context);
|
|
37
|
+
context = limitStatements(requestContext(context), 1000);
|
|
38
38
|
if (related) related = { ...related, fields: fieldRedactions(related.fields) };
|
|
39
39
|
if (context.originalErasurePolicy !== undefined && !['withdraw-affected-artifacts', 'withdraw-all-job-artifacts'].includes(context.originalErasurePolicy)) throw new Error('INVALID_ORIGINAL_ERASURE_POLICY');
|
|
40
40
|
requireText(kind, 'target kind'); requireText(id, 'target id');
|