@runeditrun/rer 0.2.83 → 0.2.85
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -24
- package/cli/README.md +17 -1
- package/cli/authoring.mjs +1 -0
- package/cli/bounded.mjs +1 -1
- package/cli/business/carry.mjs +149 -0
- package/cli/business/command.mjs +6 -1
- package/cli/business/connect.mjs +1 -1
- package/cli/business/local-commands.mjs +2 -0
- package/cli/business/opt-ins.mjs +8 -4
- package/cli/business/questions.mjs +4 -1
- package/cli/business/state.mjs +1 -1
- package/cli/business/supervisor.mjs +7 -1
- package/cli/container-transfer.mjs +2 -2
- package/cli/contract.mjs +7 -7
- package/cli/demo-host.mjs +5 -1
- package/cli/demo-native.mjs +2 -1
- package/cli/demo.mjs +6 -1
- package/cli/deploy-command.mjs +6 -4
- package/cli/deploy.mjs +9 -7
- package/cli/errors.mjs +1 -1
- package/cli/export.mjs +1 -1
- package/cli/human-error.mjs +20 -0
- package/cli/lifecycle.mjs +23 -5
- package/cli/main.mjs +1 -1
- package/cli/managed-data-schema.mjs +114 -0
- package/cli/managed-data-zip.mjs +116 -0
- package/cli/managed-data.mjs +110 -0
- package/cli/managed-server.mjs +25 -14
- package/cli/native-owner-entry.mjs +86 -0
- package/cli/output.mjs +3 -2
- package/cli/parse.mjs +3 -3
- package/cli/portability-empty.mjs +59 -0
- package/cli/portability-migrations.mjs +7 -5
- package/cli/portability-policy.mjs +5 -2
- package/cli/portability-worker.mjs +15 -6
- package/cli/portability.mjs +30 -8
- package/cli/ready-summary.mjs +9 -0
- package/cli/run-transfers.mjs +37 -0
- package/cli/runtime-front-door.mjs +4 -4
- package/cli/server-proxy.mjs +4 -3
- package/cli/server-registry.mjs +13 -8
- package/cli/site.mjs +6 -1
- package/cli/transfers.mjs +17 -4
- package/edition-authoring/base-copies.mjs +125 -30
- package/edition-authoring/copy-checkout.mjs +7 -120
- package/edition-authoring/edition-migrations.mjs +41 -0
- package/edition-authoring/files.mjs +67 -0
- package/edition-authoring/lock.mjs +2 -2
- package/edition-authoring/module-closure.mjs +232 -0
- package/edition-authoring/module-edges.mjs +100 -0
- package/edition-authoring/module-resolution.mjs +66 -0
- package/edition-authoring/module-syntax.mjs +57 -0
- package/edition-authoring/package-target.mjs +92 -0
- package/edition-authoring/runtime-dependencies.mjs +74 -97
- package/edition-authoring/vendor/babel-parser/LICENSE +22 -0
- package/edition-authoring/vendor/babel-parser/README.md +21 -0
- package/edition-authoring/vendor/babel-parser/babel-helper-string-parser/index.mjs +292 -0
- package/edition-authoring/vendor/babel-parser/babel-helper-validator-identifier/identifier.mjs +66 -0
- package/edition-authoring/vendor/babel-parser/babel-helper-validator-identifier/index.mjs +13 -0
- package/edition-authoring/vendor/babel-parser/babel-helper-validator-identifier/keyword.mjs +31 -0
- package/edition-authoring/vendor/babel-parser/babel-parser/index.mjs +99 -0
- package/edition-authoring/vendor/babel-parser/babel-parser/options.mjs +81 -0
- package/edition-authoring/vendor/babel-parser/babel-parser/parse-error/bind-operator-errors.mjs +8 -0
- package/edition-authoring/vendor/babel-parser/babel-parser/parse-error/module-errors.mjs +15 -0
- package/edition-authoring/vendor/babel-parser/babel-parser/parse-error/parse-expression-errors.mjs +10 -0
- package/edition-authoring/vendor/babel-parser/babel-parser/parse-error/pipeline-operator-errors.mjs +29 -0
- package/edition-authoring/vendor/babel-parser/babel-parser/parse-error/standard-errors.mjs +224 -0
- package/edition-authoring/vendor/babel-parser/babel-parser/parse-error/strict-mode-errors.mjs +17 -0
- package/edition-authoring/vendor/babel-parser/babel-parser/parse-error/to-node-description.mjs +31 -0
- package/edition-authoring/vendor/babel-parser/babel-parser/parse-error.mjs +114 -0
- package/edition-authoring/vendor/babel-parser/babel-parser/parser/base.mjs +34 -0
- package/edition-authoring/vendor/babel-parser/babel-parser/parser/comments.mjs +199 -0
- package/edition-authoring/vendor/babel-parser/babel-parser/parser/expression.mjs +1794 -0
- package/edition-authoring/vendor/babel-parser/babel-parser/parser/index.mjs +76 -0
- package/edition-authoring/vendor/babel-parser/babel-parser/parser/lval.mjs +439 -0
- package/edition-authoring/vendor/babel-parser/babel-parser/parser/node.mjs +111 -0
- package/edition-authoring/vendor/babel-parser/babel-parser/parser/statement.mjs +1962 -0
- package/edition-authoring/vendor/babel-parser/babel-parser/parser/util.mjs +244 -0
- package/edition-authoring/vendor/babel-parser/babel-parser/plugin-utils.mjs +116 -0
- package/edition-authoring/vendor/babel-parser/babel-parser/plugins/estree.mjs +417 -0
- package/edition-authoring/vendor/babel-parser/babel-parser/plugins/flow/index.mjs +2412 -0
- package/edition-authoring/vendor/babel-parser/babel-parser/plugins/flow/scope.mjs +37 -0
- package/edition-authoring/vendor/babel-parser/babel-parser/plugins/jsx/index.mjs +420 -0
- package/edition-authoring/vendor/babel-parser/babel-parser/plugins/jsx/xhtml.mjs +261 -0
- package/edition-authoring/vendor/babel-parser/babel-parser/plugins/placeholders.mjs +205 -0
- package/edition-authoring/vendor/babel-parser/babel-parser/plugins/typescript/index.mjs +2889 -0
- package/edition-authoring/vendor/babel-parser/babel-parser/plugins/typescript/scope.mjs +115 -0
- package/edition-authoring/vendor/babel-parser/babel-parser/plugins/v8intrinsic.mjs +26 -0
- package/edition-authoring/vendor/babel-parser/babel-parser/tokenizer/context.mjs +22 -0
- package/edition-authoring/vendor/babel-parser/babel-parser/tokenizer/index.mjs +1037 -0
- package/edition-authoring/vendor/babel-parser/babel-parser/tokenizer/state.mjs +178 -0
- package/edition-authoring/vendor/babel-parser/babel-parser/tokenizer/types.mjs +583 -0
- package/edition-authoring/vendor/babel-parser/babel-parser/types.mjs +1 -0
- package/edition-authoring/vendor/babel-parser/babel-parser/util/class-scope.mjs +82 -0
- package/edition-authoring/vendor/babel-parser/babel-parser/util/expression-scope.mjs +121 -0
- package/edition-authoring/vendor/babel-parser/babel-parser/util/identifier.mjs +23 -0
- package/edition-authoring/vendor/babel-parser/babel-parser/util/location.mjs +34 -0
- package/edition-authoring/vendor/babel-parser/babel-parser/util/production-parameter.mjs +40 -0
- package/edition-authoring/vendor/babel-parser/babel-parser/util/scope.mjs +173 -0
- package/edition-authoring/vendor/babel-parser/babel-parser/util/scopeflags.mjs +66 -0
- package/edition-authoring/vendor/babel-parser/babel-parser/util/whitespace.mjs +61 -0
- package/edition-authoring/vendor/yaml/LICENSE +13 -0
- package/edition-authoring/vendor/yaml/README.md +11 -0
- package/edition-authoring/vendor/yaml/compose/compose-collection.mjs +88 -0
- package/edition-authoring/vendor/yaml/compose/compose-doc.mjs +43 -0
- package/edition-authoring/vendor/yaml/compose/compose-node.mjs +110 -0
- package/edition-authoring/vendor/yaml/compose/compose-scalar.mjs +86 -0
- package/edition-authoring/vendor/yaml/compose/composer.mjs +218 -0
- package/edition-authoring/vendor/yaml/compose/resolve-block-map.mjs +115 -0
- package/edition-authoring/vendor/yaml/compose/resolve-block-scalar.mjs +198 -0
- package/edition-authoring/vendor/yaml/compose/resolve-block-seq.mjs +49 -0
- package/edition-authoring/vendor/yaml/compose/resolve-end.mjs +37 -0
- package/edition-authoring/vendor/yaml/compose/resolve-flow-collection.mjs +207 -0
- package/edition-authoring/vendor/yaml/compose/resolve-flow-scalar.mjs +228 -0
- package/edition-authoring/vendor/yaml/compose/resolve-props.mjs +148 -0
- package/edition-authoring/vendor/yaml/compose/util-contains-newline.mjs +34 -0
- package/edition-authoring/vendor/yaml/compose/util-empty-scalar-position.mjs +27 -0
- package/edition-authoring/vendor/yaml/compose/util-flow-indent-check.mjs +15 -0
- package/edition-authoring/vendor/yaml/compose/util-map-includes.mjs +13 -0
- package/edition-authoring/vendor/yaml/doc/Document.mjs +333 -0
- package/edition-authoring/vendor/yaml/doc/anchors.mjs +74 -0
- package/edition-authoring/vendor/yaml/doc/applyReviver.mjs +54 -0
- package/edition-authoring/vendor/yaml/doc/createNode.mjs +88 -0
- package/edition-authoring/vendor/yaml/doc/directives.mjs +176 -0
- package/edition-authoring/vendor/yaml/errors.mjs +57 -0
- package/edition-authoring/vendor/yaml/index.mjs +17 -0
- package/edition-authoring/vendor/yaml/log.mjs +11 -0
- package/edition-authoring/vendor/yaml/nodes/Alias.mjs +119 -0
- package/edition-authoring/vendor/yaml/nodes/Collection.mjs +147 -0
- package/edition-authoring/vendor/yaml/nodes/Node.mjs +38 -0
- package/edition-authoring/vendor/yaml/nodes/Pair.mjs +36 -0
- package/edition-authoring/vendor/yaml/nodes/Scalar.mjs +24 -0
- package/edition-authoring/vendor/yaml/nodes/YAMLMap.mjs +144 -0
- package/edition-authoring/vendor/yaml/nodes/YAMLSeq.mjs +113 -0
- package/edition-authoring/vendor/yaml/nodes/addPairToJSMap.mjs +62 -0
- package/edition-authoring/vendor/yaml/nodes/identity.mjs +36 -0
- package/edition-authoring/vendor/yaml/nodes/toJS.mjs +35 -0
- package/edition-authoring/vendor/yaml/parse/cst-scalar.mjs +214 -0
- package/edition-authoring/vendor/yaml/parse/cst-stringify.mjs +61 -0
- package/edition-authoring/vendor/yaml/parse/cst-visit.mjs +97 -0
- package/edition-authoring/vendor/yaml/parse/cst.mjs +98 -0
- package/edition-authoring/vendor/yaml/parse/lexer.mjs +722 -0
- package/edition-authoring/vendor/yaml/parse/line-counter.mjs +39 -0
- package/edition-authoring/vendor/yaml/parse/parser.mjs +975 -0
- package/edition-authoring/vendor/yaml/public-api.mjs +102 -0
- package/edition-authoring/vendor/yaml/schema/Schema.mjs +37 -0
- package/edition-authoring/vendor/yaml/schema/common/map.mjs +17 -0
- package/edition-authoring/vendor/yaml/schema/common/null.mjs +15 -0
- package/edition-authoring/vendor/yaml/schema/common/seq.mjs +17 -0
- package/edition-authoring/vendor/yaml/schema/common/string.mjs +14 -0
- package/edition-authoring/vendor/yaml/schema/core/bool.mjs +19 -0
- package/edition-authoring/vendor/yaml/schema/core/float.mjs +43 -0
- package/edition-authoring/vendor/yaml/schema/core/int.mjs +38 -0
- package/edition-authoring/vendor/yaml/schema/core/schema.mjs +23 -0
- package/edition-authoring/vendor/yaml/schema/json/schema.mjs +62 -0
- package/edition-authoring/vendor/yaml/schema/tags.mjs +96 -0
- package/edition-authoring/vendor/yaml/schema/yaml-1.1/binary.mjs +59 -0
- package/edition-authoring/vendor/yaml/schema/yaml-1.1/bool.mjs +26 -0
- package/edition-authoring/vendor/yaml/schema/yaml-1.1/float.mjs +46 -0
- package/edition-authoring/vendor/yaml/schema/yaml-1.1/int.mjs +71 -0
- package/edition-authoring/vendor/yaml/schema/yaml-1.1/merge.mjs +67 -0
- package/edition-authoring/vendor/yaml/schema/yaml-1.1/omap.mjs +74 -0
- package/edition-authoring/vendor/yaml/schema/yaml-1.1/pairs.mjs +78 -0
- package/edition-authoring/vendor/yaml/schema/yaml-1.1/schema.mjs +39 -0
- package/edition-authoring/vendor/yaml/schema/yaml-1.1/set.mjs +93 -0
- package/edition-authoring/vendor/yaml/schema/yaml-1.1/timestamp.mjs +101 -0
- package/edition-authoring/vendor/yaml/stringify/foldFlowLines.mjs +146 -0
- package/edition-authoring/vendor/yaml/stringify/stringify.mjs +130 -0
- package/edition-authoring/vendor/yaml/stringify/stringifyCollection.mjs +155 -0
- package/edition-authoring/vendor/yaml/stringify/stringifyComment.mjs +20 -0
- package/edition-authoring/vendor/yaml/stringify/stringifyDocument.mjs +85 -0
- package/edition-authoring/vendor/yaml/stringify/stringifyNumber.mjs +25 -0
- package/edition-authoring/vendor/yaml/stringify/stringifyPair.mjs +150 -0
- package/edition-authoring/vendor/yaml/stringify/stringifyString.mjs +338 -0
- package/edition-authoring/vendor/yaml/util.mjs +11 -0
- package/edition-authoring/vendor/yaml/visit.mjs +233 -0
- package/edition-authoring/workflow-copies.mjs +125 -0
- package/edition-authoring/workflow-scripts.mjs +112 -0
- package/foundation/runtime/contract-docs.mjs +754 -0
- package/foundation/runtime/run-data.d.mts +1 -0
- package/foundation/runtime/run-data.mjs +2 -0
- package/global/CHANGELOG.md +55 -3
- package/global/README.md +21 -24
- package/global/cli/command.mjs +8 -2
- package/global/cli/install.mjs +4 -5
- package/global/cli/path-link.mjs +3 -2
- package/global/cli/repo-command.mjs +3 -7
- package/global/launch/dev.mjs +12 -2
- package/global/launch/facts.mjs +2 -2
- package/global/launch/render.mjs +16 -9
- package/global/launch/state.mjs +4 -4
- package/global/main.mjs +12 -7
- package/install.sh +17 -6
- package/modules/container/README.md +6 -1
- package/modules/container/adapters/d1-limits.mjs +286 -0
- package/modules/container/adapters/d1-sqlite.mjs +95 -0
- package/modules/container/adapters/d1-workerd-preload.mjs +19 -0
- package/modules/container/adapters/d1-workerd-setup.mjs +26 -0
- package/modules/container/adapters/d1-workerd.mjs +5 -0
- package/modules/container/adapters/sqlite-worker.mjs +5 -3
- package/modules/container/build.mjs +1 -0
- package/modules/container/image-variables.mjs +8 -3
- package/modules/container/transfer.mjs +14 -4
- package/modules/container/worker-engine.mjs +3 -3
- package/modules/demo/declaration.mjs +1 -4
- package/modules/demo/vite.mjs +1 -1
- package/modules/demo-host/build.mjs +1 -0
- package/modules/demo-host/native/prepare.mjs +2 -1
- package/modules/demo-host/native/sql.mjs +7 -2
- package/modules/deploy-kit/README.md +23 -3
- package/modules/deploy-kit/d1-import-diagnostics.mjs +56 -0
- package/modules/deploy-kit/d1-import.mjs +17 -5
- package/modules/deploy-kit/diagnostics.mjs +29 -3
- package/modules/deploy-kit/owner-cli.mjs +20 -7
- package/modules/deploy-kit/provider-transport.mjs +19 -4
- package/modules/deploy-kit/provider.mjs +57 -22
- package/modules/deploy-kit/readiness-window.mjs +1 -1
- package/modules/deploy-kit/site.mjs +8 -3
- package/modules/deploy-kit/templates.mjs +3 -3
- package/modules/deploy-kit/worker.mjs +4 -4
- package/modules/import-kit/src/d1/README.md +23 -4
- package/modules/import-kit/src/d1/atomic-prefix.ts +2 -2
- package/modules/import-kit/src/d1/budget.ts +15 -7
- package/modules/import-kit/src/d1/column-layout.ts +5 -0
- package/modules/import-kit/src/d1/creation.ts +15 -8
- package/modules/import-kit/src/d1/database.ts +3 -3
- package/modules/import-kit/src/d1/erasure-pages.ts +2 -1
- package/modules/import-kit/src/d1/erasure.ts +2 -2
- package/modules/import-kit/src/d1/jobs.ts +10 -6
- package/modules/import-kit/src/d1/plans.ts +14 -8
- package/modules/import-kit/src/d1/retirement.ts +2 -2
- package/modules/import-kit/src/d1/sets.ts +1 -3
- package/modules/import-kit/src/d1/staged-creation.ts +3 -3
- package/modules/import-kit/src/door/limits.ts +1 -1
- package/modules/import-kit/src/door/refusals.ts +1 -1
- package/modules/portability/README.md +112 -4
- package/modules/portability/backfills.mjs +9 -8
- package/modules/portability/bridge.mjs +12 -5
- package/modules/portability/catalogue.mjs +12 -5
- package/modules/portability/files.mjs +4 -3
- package/modules/portability/http.mjs +3 -3
- package/modules/portability/import.mjs +18 -9
- package/modules/portability/index.d.mts +4 -3
- package/modules/portability/index.mjs +1 -1
- package/modules/portability/ledgers.mjs +3 -2
- package/modules/portability/legacy-safety.mjs +11 -2
- package/modules/portability/migration-history.mjs +2 -2
- package/modules/portability/progress.mjs +1 -0
- package/modules/portability/restore-empty.mjs +12 -0
- package/modules/release/github-verify.mjs +11 -2
- package/onboarding/adapter.mjs +35 -6
- package/onboarding/app-fetch.mjs +6 -4
- package/onboarding/development-runtime.mjs +19 -0
- package/onboarding/local/operation.mjs +45 -27
- package/onboarding/loopback-fetch.mjs +16 -0
- package/onboarding/operator-installer.mjs +4 -6
- package/onboarding/process-diagnostics.mjs +6 -2
- package/onboarding/production-runtime.mjs +4 -2
- package/onboarding/readiness.mjs +1 -0
- package/package.json +1 -1
- package/scripts/publish-check/README.md +6 -0
- package/scripts/publish-check/history_scan.py +5 -3
- package/scripts/publish-check/hygiene_config.py +2 -1
- package/scripts/publish-check/public_hygiene.py +1 -1
- package/scripts/publish-check/report.py +9 -3
- package/scripts/publish-check/third-party-notices.json +13 -0
- package/scripts/publish-check/third_party_notices.py +38 -0
- package/scripts/publish-check/tree_scan.py +6 -0
- package/scripts/publish-check.mjs +1 -1
|
@@ -1,9 +1,10 @@
|
|
|
1
|
-
import { access, mkdir,
|
|
1
|
+
import { access, mkdir, copyFile, writeFile } from 'node:fs/promises';
|
|
2
2
|
import { constants } from 'node:fs';
|
|
3
3
|
import { execFileSync } from 'node:child_process';
|
|
4
4
|
import { resolve, join, dirname } from 'node:path';
|
|
5
5
|
import { fileURLToPath } from 'node:url';
|
|
6
6
|
import { assert } from './validation.mjs';
|
|
7
|
+
import { toolFiles } from '../../edition-authoring/files.mjs';
|
|
7
8
|
const base = resolve(dirname(fileURLToPath(import.meta.url)),'../..');
|
|
8
9
|
/** Generate alongside an edition. The existing edition tree is never an output directory. */
|
|
9
10
|
export async function createTemplates({destination,project,sourcePaths,commands,loops=[]}) {
|
|
@@ -13,9 +14,8 @@ export async function createTemplates({destination,project,sourcePaths,commands,
|
|
|
13
14
|
await mkdir(root,{recursive:false});
|
|
14
15
|
// A new directory only: callers add actual edition sources then run the lock command.
|
|
15
16
|
const scripts = join(root,'scripts'); await mkdir(scripts);
|
|
16
|
-
await mkdir(join(scripts,'edition-authoring'));
|
|
17
17
|
for (const name of ['operations.mjs','project-operations.mjs']) await copyFile(join(base,'operations',name),join(scripts,name),constants.COPYFILE_EXCL);
|
|
18
|
-
for (const name of
|
|
18
|
+
for (const name of toolFiles(join(base,'edition-authoring'))) { await mkdir(dirname(join(scripts,'edition-authoring',name)),{recursive:true}); await copyFile(join(base,'edition-authoring',name),join(scripts,'edition-authoring',name),constants.COPYFILE_EXCL); }
|
|
19
19
|
await writeFile(join(root,'OPERATIONS.json'),JSON.stringify({version:2,project,runtime:{requirements:[{name:'node',version:'>=22.12.0'}]},requiredEnvironment:[],commands,loops},null,2)+'\n',{flag:'wx'});
|
|
20
20
|
await writeFile(join(root,'edition.json'),JSON.stringify({schemaVersion:1,id:project,toolchain:{id:'runeditrun/edition-authoring',version:'1.1.0'},parent:null,components:[{id:project,role:'edition',source:{kind:'local',paths:sourcePaths}}]},null,2)+'\n',{flag:'wx'});
|
|
21
21
|
return {root,lock:['node','scripts/edition-authoring/cli.mjs','lock'],check:['node','scripts/edition-authoring/cli.mjs','check']};
|
|
@@ -64,26 +64,26 @@ export async function secretBindings(api, worker, plan, env, exists) {
|
|
|
64
64
|
}
|
|
65
65
|
return {bindings,names,unset,missing:plan.secrets.filter(x => !names.includes(x.binding) && !unset.includes(x.binding)).map(x => x.binding)};
|
|
66
66
|
}
|
|
67
|
-
export async function uploadAssets(api, worker, files) {
|
|
67
|
+
export async function uploadAssets(api, worker, files, {label = 'assets'} = {}) {
|
|
68
68
|
const manifest = {}; const byHash = new Map();
|
|
69
69
|
for (const file of files) {
|
|
70
70
|
const digest = hash(file.base64 + extname(file.path).slice(1)).slice(0,32);
|
|
71
71
|
manifest[`/${file.path}`] = {hash:digest,size:Buffer.from(file.base64,'base64').length};
|
|
72
72
|
byHash.set(digest,file);
|
|
73
73
|
}
|
|
74
|
-
const session = await api.request(`/workers/scripts/${worker}/assets-upload-session`,{method:'POST',body:{manifest}})
|
|
74
|
+
const session = await api.request(`/workers/scripts/${worker}/assets-upload-session`,{method:'POST',body:{manifest},step:`creating ${label} upload session`})
|
|
75
75
|
.catch(error => { throw nameAssetFailure(error,files.map(file => file.path)); });
|
|
76
76
|
assert(typeof session.jwt === 'string' && session.jwt && Array.isArray(session.buckets),'invalid-asset-session');
|
|
77
77
|
let completion = session.buckets.length ? null : session.jwt;
|
|
78
78
|
const seen = new Set();
|
|
79
|
-
for (const bucket of session.buckets) {
|
|
79
|
+
for (const [index,bucket] of session.buckets.entries()) {
|
|
80
80
|
assert(Array.isArray(bucket) && bucket.length,'invalid-asset-bucket');
|
|
81
81
|
const data = new FormData();
|
|
82
82
|
for (const key of bucket) {
|
|
83
83
|
assert(byHash.has(key) && !seen.has(key),'invalid-asset-request'); seen.add(key);
|
|
84
84
|
const file = byHash.get(key); data.set(key,new Blob([file.base64],{type:assetType(file.path)}),file.path.split('/').at(-1));
|
|
85
85
|
}
|
|
86
|
-
const uploaded = await api.request('/workers/assets/upload?base64=true',{method:'POST',body:data,assetToken:session.jwt})
|
|
86
|
+
const uploaded = await api.request('/workers/assets/upload?base64=true',{method:'POST',body:data,assetToken:session.jwt,step:`uploading ${label}, batch ${index + 1} of ${session.buckets.length}`})
|
|
87
87
|
.catch(error => { throw nameAssetFailure(error,bucket.map(key => byHash.get(key).path)); });
|
|
88
88
|
if (uploaded.jwt) completion = uploaded.jwt;
|
|
89
89
|
}
|
|
@@ -277,13 +277,32 @@ all layers. An import still applying or reversing cannot be superseded
|
|
|
277
277
|
The bounds below follow [D1 limits](https://developers.cloudflare.com/d1/platform/limits/).
|
|
278
278
|
Bulk text insertion uses the supported [D1 JSON SQL functions](https://developers.cloudflare.com/d1/sql-api/query-json/).
|
|
279
279
|
|
|
280
|
-
The port
|
|
281
|
-
|
|
280
|
+
The port assumes Workers Paid: one Worker invocation allows 1,000 D1 queries;
|
|
281
|
+
Workers Free allows only 50 and is not supported by this budget. Each direct
|
|
282
|
+
read and each `batch()` call counts as one query, independent of batch members.
|
|
283
|
+
Statement length, bound-parameter and value guards still check every member.
|
|
284
|
+
Planning and creation preflight share the production batch-cost function; a
|
|
285
|
+
plan exceeding the fresh creation query ceiling is refused during planning
|
|
286
|
+
with `IMPORT_PLAN_TOO_LARGE`, naming the 1,000-query paid-plan limit.
|
|
287
|
+
`preflightD1ImportJob().totalExecutions` measures those D1 queries;
|
|
288
|
+
`totalStatements` and `mutationStatements` retain the individual statement
|
|
289
|
+
counts, and `mutationExecutions` counts mutation batch calls.
|
|
290
|
+
The door shares one budget across all its kit calls in one HTTP request; later
|
|
291
|
+
apply, resume and undo requests are distinct Worker invocations. Erasure retains
|
|
292
|
+
its existing 1,000-statement work ceiling, counting every batch member as well
|
|
293
|
+
as reads. An explicit `maxStatements` may tighten that separate ceiling.
|
|
294
|
+
|
|
295
|
+
The port counts queries over the entire public invocation (including
|
|
296
|
+
nested resume reads and atomic batches), caps them at 1,000 and
|
|
282
297
|
returns durable partial progress as the budget approaches exhaustion. Explicit
|
|
283
298
|
`IMPORT_D1_REQUEST_LIMIT`, `IMPORT_D1_STATEMENT_LIMIT` and
|
|
284
299
|
`IMPORT_D1_VALUE_LIMIT` failures publish no portion of the rejected batch.
|
|
285
|
-
Create, apply and reversal reserve
|
|
286
|
-
|
|
300
|
+
Create, apply and reversal reserve their return reads before publication,
|
|
301
|
+
including chunked-plan readback. A shared calculation sizes plan pages from the
|
|
302
|
+
maximum JSON-encoded stored chunk row bytes, measured before reading, with space
|
|
303
|
+
reserved under the transport ceiling for row metadata and two-byte strings.
|
|
304
|
+
Today's small chunks use up to 64 rows per page; larger legacy chunks use
|
|
305
|
+
smaller pages. Neither the stored plan format nor the canonical byte cap changes. Each successful batch retains its checked job state;
|
|
287
306
|
if a later batch's preflight or exact SQL count exhausts the same request budget,
|
|
288
307
|
the invocation returns that durable partial progress. Other adapter, integrity
|
|
289
308
|
and authority failures still reject. A first batch that cannot fit rejects before
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { ImportAction } from '../types.ts';
|
|
2
2
|
import type { D1ImportContext, D1Sql } from './types.ts';
|
|
3
3
|
import type { NativeStagingScope } from './native-staging-context.ts';
|
|
4
|
-
import { GuardPacking, valueBytes } from './budget.ts';
|
|
4
|
+
import { batchExecutions, GuardPacking, valueBytes } from './budget.ts';
|
|
5
5
|
import { authority, finalAuthority, postConditions, sql } from './database.ts';
|
|
6
6
|
|
|
7
7
|
/** Unbounded publication has append-only SQL and final checks. Bounded native
|
|
@@ -56,7 +56,7 @@ export function atomicPrefix(context: D1ImportContext, action: ImportAction, sta
|
|
|
56
56
|
const writes = this.writes.count([...tail, authority(context, action), sql('DELETE FROM ik_d1_assertions')]);
|
|
57
57
|
const checks = tail.flatMap(s => s.finalChecks ?? []), post = this.post.count(checks);
|
|
58
58
|
if (this.postLength + checks.length) finalAuthority(context, action);
|
|
59
|
-
return writes + post + Number(this.postLength + checks.length > 0) + returnReads;
|
|
59
|
+
return batchExecutions(context, writes + post + Number(this.postLength + checks.length > 0)) + returnReads;
|
|
60
60
|
}
|
|
61
61
|
}
|
|
62
62
|
const post = postConditions(context, statements, scope, action);
|
|
@@ -2,7 +2,15 @@ import { Buffer } from 'node:buffer';
|
|
|
2
2
|
import { wireDatabase } from '../../../d1-transport/index.mjs';
|
|
3
3
|
import { guard, guardPredicate } from './guards.ts';
|
|
4
4
|
import type { D1Database, D1ImportContext, D1Sql } from './types.ts';
|
|
5
|
-
const
|
|
5
|
+
export const D1_REQUEST_QUERY_LIMIT = 1000;
|
|
6
|
+
const D1_BATCH_QUERY_COST = 1;
|
|
7
|
+
const budgets = new WeakMap<D1Database, { used: number; statements: number; statementLimit?: number }>();
|
|
8
|
+
/** Paid D1 counts one batch call as one query, regardless of its members.
|
|
9
|
+
* An explicit erasure work ceiling still counts each executed statement. */
|
|
10
|
+
export function batchExecutions(context: { storage?: Pick<D1ImportContext['storage'], 'database'> }, statements: number): number {
|
|
11
|
+
const budget = context.storage ? budgets.get(context.storage.database) : undefined;
|
|
12
|
+
return budget?.statementLimit === undefined ? D1_BATCH_QUERY_COST : statements;
|
|
13
|
+
}
|
|
6
14
|
const reservedTail = Symbol('import request return queries');
|
|
7
15
|
type ReservedContext = D1ImportContext & { [reservedTail]?: number };
|
|
8
16
|
/** Distinguish this request's measured exhaustion from adapter/provider errors. */
|
|
@@ -14,13 +22,13 @@ export function checkSql(s: D1Sql): void {
|
|
|
14
22
|
if (Buffer.byteLength(s.sql) > 100000 || (s.params?.length ?? 0) > 100) throw new Error('IMPORT_D1_STATEMENT_LIMIT');
|
|
15
23
|
if ((s.params ?? []).some(p => valueBytes(p) > 2000000) || (s.params ?? []).reduce((n, p) => n + valueBytes(p), 0) > 2000000) throw new Error('IMPORT_D1_VALUE_LIMIT');
|
|
16
24
|
}
|
|
17
|
-
export function remaining(context: D1ImportContext): number { const budget=budgets.get(context.storage.database);return (
|
|
25
|
+
export function remaining(context: D1ImportContext): number { const budget=budgets.get(context.storage.database);return Math.min(D1_REQUEST_QUERY_LIMIT-(budget?.used??0), budget?.statementLimit === undefined ? Infinity : budget.statementLimit-budget.statements)-((context as ReservedContext)[reservedTail]??0); }
|
|
18
26
|
/** A smaller public-operation budget also stops reads, before execution. Nested
|
|
19
27
|
* calls can only tighten the shared request's existing execution ceiling. */
|
|
20
28
|
export function limitStatements(context:D1ImportContext,max:number):D1ImportContext {
|
|
21
29
|
const budget=budgets.get(context.storage.database);
|
|
22
|
-
if(!budget||!Number.isSafeInteger(max)||max<1||max>
|
|
23
|
-
budget.
|
|
30
|
+
if(!budget||!Number.isSafeInteger(max)||max<1||max>D1_REQUEST_QUERY_LIMIT)throw new Error('INVALID_ERASURE_STATEMENT_LIMIT');
|
|
31
|
+
budget.statementLimit=Math.min(budget.statementLimit??D1_REQUEST_QUERY_LIMIT,budget.statements+max);
|
|
24
32
|
return context;
|
|
25
33
|
}
|
|
26
34
|
/** A nested operation leaves the outer operation's exact bounded return path
|
|
@@ -36,11 +44,11 @@ export function requestDatabase(database: D1Database): D1Database {
|
|
|
36
44
|
return requestContext({ storage: { database } } as D1ImportContext).storage.database;
|
|
37
45
|
}
|
|
38
46
|
/** A fresh wrapper belongs to the outer public invocation; nested resume calls
|
|
39
|
-
* reuse it. It counts
|
|
47
|
+
* reuse it. It counts D1 calls; every batch is one query. Statement work limits are separate. */
|
|
40
48
|
export function requestContext(context: D1ImportContext): D1ImportContext {
|
|
41
49
|
if (budgets.has(context.storage.database)) return context;
|
|
42
|
-
const db = wireDatabase(context.storage.database), budget = { used: 0,
|
|
43
|
-
const consume = (n: number) => { if (budget.used + n > budget.
|
|
50
|
+
const db = wireDatabase(context.storage.database), budget: { used: number; statements: number; statementLimit?: number } = { used: 0, statements: 0 };
|
|
51
|
+
const consume = (n: number) => { const queries = D1_BATCH_QUERY_COST; if (budget.used + queries > D1_REQUEST_QUERY_LIMIT || budget.statements + n > (budget.statementLimit ?? Infinity)) throw new D1RequestLimitError(); budget.used += queries; budget.statements += n; };
|
|
44
52
|
const raw = new WeakMap<object, any>();
|
|
45
53
|
const wrap = (statement: any, query: D1Sql): any => {
|
|
46
54
|
checkSql(query);
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
// Static layouts have no engine imports, so either test-port entry order
|
|
2
|
+
// can initialize the erasure planner without a module-evaluation cycle.
|
|
3
|
+
export const jobColumns = ['id','destination','namespace','mapping_version','principal_id','fingerprint','operation_key','plan_json','state','checkpoint','total','fence','lease_token','lease_expires','error','reversal_policy','reversal_policy_hash','reversal_cursor','reversal_hash'];
|
|
4
|
+
export const provenanceColumns = ['id','job_id','ordinal','effect_ordinal','source_identity','destination','target_kind','target_id','reversed','record_json'];
|
|
5
|
+
export const claimColumns = ['identity','source_hash','mapping_version','effects_hash','job_id'];
|
|
@@ -8,13 +8,13 @@ import type { D1ImportContext, D1PlanCodec, D1Sql } from './types.ts';
|
|
|
8
8
|
import { planFingerprint } from '../plan/index.ts';
|
|
9
9
|
import { assertPlanSize } from '../plan/size.ts';
|
|
10
10
|
import { assertPlanCredentials } from '../provenance/credentials.ts';
|
|
11
|
-
import { preparedAtomic, guard, sql } from './database.ts';
|
|
11
|
+
import { atomicCount, preparedAtomic, guard, sql } from './database.ts';
|
|
12
12
|
import { retainPlan, planChunkReads, creationCodec, LARGE_PLAN_BYTES } from './plans.ts';
|
|
13
|
-
import { preflightStagedCreation, stagedSize } from './staged-creation.ts';
|
|
13
|
+
import { preflightStagedCreation, stagedCreationCost, stagedSize } from './staged-creation.ts';
|
|
14
14
|
import { stagingStatements } from './staging-statements.ts';
|
|
15
15
|
import { custodyGuards } from './original-storage.ts';
|
|
16
16
|
import { retainOriginals } from './originals.ts';
|
|
17
|
-
import { checkSql,
|
|
17
|
+
import { batchExecutions, checkSql, D1_REQUEST_QUERY_LIMIT, valueBytes } from './budget.ts';
|
|
18
18
|
import { policy, policyHash, EMPTY_REVERSAL_HASH } from './undo-state.ts';
|
|
19
19
|
|
|
20
20
|
export type D1CreatePreflightContext = Pick<D1ImportContext, 'destination' | 'namespace' | 'mappingVersion' | 'authorization' | 'reviewedFingerprint' | 'operationKey' | 'reversalPolicy' | 'maxBatchBytes'> & {
|
|
@@ -22,10 +22,13 @@ export type D1CreatePreflightContext = Pick<D1ImportContext, 'destination' | 'na
|
|
|
22
22
|
planCodec?: D1PlanCodec;
|
|
23
23
|
};
|
|
24
24
|
export interface D1CreatePreflight {
|
|
25
|
-
/** Statements
|
|
25
|
+
/** Statements across private staging and atomic job, plan and custody publication. */
|
|
26
26
|
mutationStatements: number;
|
|
27
|
-
|
|
27
|
+
mutationExecutions: number;
|
|
28
|
+
/** D1 queries: authority batch, operation-key lookup, mutation and return reads. */
|
|
28
29
|
totalExecutions: number;
|
|
30
|
+
/** All individual statements, including the members of each batch. */
|
|
31
|
+
totalStatements: number;
|
|
29
32
|
maxSqlBytes: number;
|
|
30
33
|
maxBoundBytes: number;
|
|
31
34
|
maxParameters: number;
|
|
@@ -82,13 +85,17 @@ export function preflightD1ImportJob(plan: ImportPlan, context: D1CreatePrefligh
|
|
|
82
85
|
sql('SELECT * FROM ik_jobs WHERE id=?', '00000000-0000-0000-0000-000000000000'),
|
|
83
86
|
sql('SELECT * FROM ik_d1_plan_chunks WHERE job_id IN (SELECT value FROM json_each(?)) ORDER BY job_id,ordinal', JSON.stringify(['00000000-0000-0000-0000-000000000000']))];
|
|
84
87
|
const chunkRead = reads.pop()!;
|
|
85
|
-
|
|
88
|
+
const chunkReads = planChunkReads(retained.body, unbounded, retained.encoded.chunks);
|
|
89
|
+
for (let n = 0; n < chunkReads; n++) reads.push(chunkRead);
|
|
86
90
|
if (policy(context) === 'preserve-conflicts') reads.push(sql('SELECT r.*,p.reversed AS native_reversed FROM ik_d1_reversal_reports r LEFT JOIN ik_provenance p ON p.id=r.effect_id AND p.job_id=r.job_id WHERE r.job_id=? ORDER BY r.position', '00000000-0000-0000-0000-000000000000'));
|
|
87
91
|
const erasedReads = erasedSourcePredicates(plan).map(p => ({ sql: `SELECT ${p.sql} AS clear`, params: p.params }));
|
|
88
92
|
const all = [...initial, ...erasedReads, ...statements, ...reads];
|
|
89
93
|
for (const statement of all) checkSql(statement);
|
|
90
|
-
|
|
91
|
-
|
|
94
|
+
const queryContext = context as D1ImportContext;
|
|
95
|
+
const mutationExecutions = staged ? stagedCreationCost(queryContext, staged) : atomicCount(queryContext, 'create', creation, 0);
|
|
96
|
+
const totalExecutions = batchExecutions(queryContext, initial.length) + erasedReads.length + reads.length + mutationExecutions;
|
|
97
|
+
if (totalExecutions > D1_REQUEST_QUERY_LIMIT) throw new Error(`IMPORT_PLAN_TOO_LARGE: D1 creation needs ${totalExecutions} queries; the paid-plan limit is ${D1_REQUEST_QUERY_LIMIT}`);
|
|
98
|
+
return { mutationStatements: statements.length, totalExecutions, mutationExecutions, totalStatements: all.length,
|
|
92
99
|
maxSqlBytes: Math.max(...all.map(s => Buffer.byteLength(s.sql))),
|
|
93
100
|
maxBoundBytes: Math.max(...all.map(s => (s.params ?? []).reduce((n, p) => n + valueBytes(p), 0))),
|
|
94
101
|
maxParameters: Math.max(...all.map(s => s.params?.length ?? 0)) };
|
|
@@ -8,7 +8,7 @@ import { d1NativeStagingSchema } from './native-staging-schema.ts';
|
|
|
8
8
|
import { d1CustodyStateSchema, d1CustodyStateReplaceSchema } from './custody-state-schema.ts';
|
|
9
9
|
import { d1ReversalFenceSchema } from './reversal-fence-schema.ts';
|
|
10
10
|
import { batchBytes } from './transport.ts';
|
|
11
|
-
import { D1RequestLimitError, packGuards, remaining } from './budget.ts';
|
|
11
|
+
import { batchExecutions, D1RequestLimitError, packGuards, remaining } from './budget.ts';
|
|
12
12
|
import { d1ErasureSchema } from './erasure-schema.ts';
|
|
13
13
|
import { d1RetirementSchema } from './retirement-schema.ts';
|
|
14
14
|
import { d1ErasureProgressSchema } from './erasure-progress-schema.ts';
|
|
@@ -130,7 +130,7 @@ export const atomicCount = (context:Parameters<typeof authority>[0]&Pick<D1Impor
|
|
|
130
130
|
function atomicPreparation(context:Parameters<typeof authority>[0]&Pick<D1ImportContext,'maxBatchBytes'>,action:ImportAction,statements:D1Sql[],scope?:NativeStagingScope,returnReads=0){
|
|
131
131
|
const post=postConditions(context,statements,scope,action);
|
|
132
132
|
const packed=[...atomicStatements(context,action,statements),...readonlyChecks(post),...(post.length?[finalAuthority(context,action)]:[])];
|
|
133
|
-
return {post,packed,count:packed.length+returnReads};
|
|
133
|
+
return {post,packed,count:batchExecutions(context as D1ImportContext,packed.length)+returnReads};
|
|
134
134
|
}
|
|
135
135
|
export { atomicPrefix } from './atomic-prefix.ts';
|
|
136
136
|
/** SELECT-only fence: inspect every packed scalar before publishing snapshots.
|
|
@@ -145,7 +145,7 @@ export async function readAtomic(context: D1ImportContext, guards: D1Sql[], scop
|
|
|
145
145
|
await readBoundedGuards(context, checks, scope, Math.min(limit,2*1024*1024), action);
|
|
146
146
|
return;
|
|
147
147
|
}
|
|
148
|
-
if (packed.length > remaining(context)) throw new D1RequestLimitError();
|
|
148
|
+
if (batchExecutions(context, packed.length) > remaining(context)) throw new D1RequestLimitError();
|
|
149
149
|
const results = await context.storage.database.batch(packed.map(s => prepare(context.storage.database, s)));
|
|
150
150
|
checkBatchResults(results, packed.length, 'IMPORT_ATOMIC_GUARD');
|
|
151
151
|
for (const result of results) {
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { D1Database,D1Sql } from './types.ts';
|
|
2
|
-
import { jsonChunks
|
|
2
|
+
import { jsonChunks } from './sets.ts';
|
|
3
|
+
import { jobColumns, provenanceColumns, claimColumns } from './column-layout.ts';
|
|
3
4
|
|
|
4
5
|
const PAGE_BYTES=2*1024*1024;
|
|
5
6
|
const UNKNOWN_BYTES=Number.MAX_SAFE_INTEGER;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { erasedCustody } from './erased-custody.ts';
|
|
2
2
|
import { targetGuards } from './target-guards.ts';
|
|
3
3
|
import type { D1FieldMatch, D1ImportContext, D1RelatedErasure, D1Sql } from './types.ts';
|
|
4
|
-
import { requestContext, packGuards, remaining, D1RequestLimitError } from './budget.ts';
|
|
4
|
+
import { requestContext, limitStatements, packGuards, remaining, D1RequestLimitError } from './budget.ts';
|
|
5
5
|
import { readAtomic, authority, finalAuthority, readonlyChecks, guard, sql, prepare } from './database.ts';
|
|
6
6
|
import { decode } from './jobs.ts';
|
|
7
7
|
import { retainPlan } from './plans.ts';
|
|
@@ -34,7 +34,7 @@ import { supersededEntries } from './erasure-superseded.ts';
|
|
|
34
34
|
* must redact them. `related.fields` instead sanitizes peer values while
|
|
35
35
|
* preserving their effects, provenance and reversible before-images. */
|
|
36
36
|
export async function eraseD1ImportTarget(kind: string, id: string, nativeStatements: D1Sql[], context: D1ImportContext, related?: D1RelatedErasure): Promise<{ nativeResults: unknown[]; erasedRecords: number; fieldMatches: D1FieldMatch[] }> {
|
|
37
|
-
context = requestContext(context);
|
|
37
|
+
context = limitStatements(requestContext(context), 1000);
|
|
38
38
|
if (related) related = { ...related, fields: fieldRedactions(related.fields) };
|
|
39
39
|
if (context.originalErasurePolicy !== undefined && !['withdraw-affected-artifacts', 'withdraw-all-job-artifacts'].includes(context.originalErasurePolicy)) throw new Error('INVALID_ORIGINAL_ERASURE_POLICY');
|
|
40
40
|
requireText(kind, 'target kind'); requireText(id, 'target id');
|
|
@@ -3,11 +3,11 @@ import { needsStreamOrigin } from './stream-origin.ts';
|
|
|
3
3
|
import { nativeStagingScope } from './native-staging-context.ts';
|
|
4
4
|
import { streamBoundary } from './stream-boundary.ts';
|
|
5
5
|
import { requestContext } from './budget.ts';
|
|
6
|
-
import { loadPlans, planBody, planGuards, planChunkReads, retainPlan, creationCodec, LARGE_PLAN_BYTES } from './plans.ts';
|
|
6
|
+
import { loadPlans, storedPlanSnapshot, planBody, planGuards, planChunkReads, retainPlan, creationCodec, LARGE_PLAN_BYTES } from './plans.ts';
|
|
7
7
|
import { randomUUID } from 'node:crypto';
|
|
8
8
|
import type { ImportJob, ImportPlan, ImportMapping, SourceBundle, ImportAction } from '../types.ts';
|
|
9
9
|
import { buildImportPlan, planFingerprint } from '../plan/index.ts';
|
|
10
|
-
import { creationStatements, validateCreation } from './creation.ts';
|
|
10
|
+
import { creationStatements, preflightD1ImportJob, validateCreation } from './creation.ts';
|
|
11
11
|
import { planStagedCreation, runStagedCreation, stagedSize } from './staged-creation.ts';
|
|
12
12
|
import type { D1ImportContext, D1Sql } from './types.ts';
|
|
13
13
|
import { atomic, readAtomic, authorize, guard, sql } from './database.ts';
|
|
@@ -31,7 +31,7 @@ export async function stored(id: string, context: D1ImportContext, action: Impor
|
|
|
31
31
|
// Unbounded guards run before their transaction's writes: a large plan is
|
|
32
32
|
// fenced by its custody revisions instead of resending it in every request.
|
|
33
33
|
const fenced = !bounded, job = decode(row), plan = (await loadPlans([row], context.storage.database, context.storage.targets.planCodec, undefined, fenced)).get(id)!;
|
|
34
|
-
planReads.set(job, planChunkReads(row.plan_json, fenced));
|
|
34
|
+
planReads.set(job, planChunkReads(row.plan_json, fenced, storedPlanSnapshot(plan, id).chunks));
|
|
35
35
|
if (job.destination !== context.destination || job.namespace !== context.namespace || job.mappingVersion !== context.mappingVersion || job.principalId !== context.authorization.principalId) throw new Error('JOB_BINDING_MISMATCH');
|
|
36
36
|
if (context.reviewedFingerprint !== undefined && context.reviewedFingerprint !== job.fingerprint) throw new Error('REVIEW_FINGERPRINT_MISMATCH');
|
|
37
37
|
if ((verifiedPlanFingerprint(plan) ?? planFingerprint(plan)) !== job.fingerprint || plan.fingerprint !== job.fingerprint || plan.destination !== job.destination || plan.namespace !== job.namespace || plan.mappingVersion !== job.mappingVersion || plan.principalId !== job.principalId || plan.entries.length !== job.total) throw new Error('STORED_PLAN_CHANGED');
|
|
@@ -47,6 +47,10 @@ export async function planD1Import(source: SourceBundle, mapping: ImportMapping,
|
|
|
47
47
|
context = requestContext(context);
|
|
48
48
|
await authorize(context, 'plan');
|
|
49
49
|
const plan = buildImportPlan(source, mapping, context);
|
|
50
|
+
// Admission measures calls, not current create authority. Planning still
|
|
51
|
+
// evaluates only the plan grant; creation checks its own live predicate.
|
|
52
|
+
const { storage, ...admission } = context;
|
|
53
|
+
preflightD1ImportJob(plan, { ...admission, planCodec: storage.targets.planCodec, reviewedFingerprint: plan.fingerprint, authorization: { principalId: context.authorization.principalId, predicate: () => sql('1') } });
|
|
50
54
|
await authorize(context, 'plan');
|
|
51
55
|
return plan;
|
|
52
56
|
}
|
|
@@ -76,16 +80,16 @@ export async function createD1ImportJob(plan: ImportPlan, context: D1ImportConte
|
|
|
76
80
|
await guardErasedSources(plan, db);
|
|
77
81
|
revalidate();
|
|
78
82
|
const codec = creationCodec(plan, context.storage.targets.planCodec), unbounded = context.maxBatchBytes === undefined;
|
|
79
|
-
const returnReads = (body: string) => 1 + planChunkReads(body, unbounded) + Number(context.reversalPolicy === 'preserve-conflicts');
|
|
80
83
|
const id = randomUUID(), retained = retainPlan(plan, id, codec);
|
|
84
|
+
const returnReads = 1 + planChunkReads(retained.body, unbounded, retained.encoded.chunks) + Number(context.reversalPolicy === 'preserve-conflicts');
|
|
81
85
|
// A large plan is created in bounded steps; a retry resumes what was staged.
|
|
82
86
|
if (unbounded && stagedSize(retained.encoded) > LARGE_PLAN_BYTES) {
|
|
83
87
|
const creation = await planStagedCreation(plan, context, operationKey, retained.encoded, () => [randomUUID(), randomUUID(), id]);
|
|
84
88
|
revalidate();
|
|
85
|
-
return (await stored(await runStagedCreation(context, creation, returnReads
|
|
89
|
+
return (await stored(await runStagedCreation(context, creation, returnReads), context, 'create')).job;
|
|
86
90
|
}
|
|
87
91
|
const creation = creationStatements(plan, { ...context, planCodec: codec }, id, operationKey, retained);
|
|
88
|
-
await atomic(context, 'create', creation, returnReads
|
|
92
|
+
await atomic(context, 'create', creation, returnReads);
|
|
89
93
|
return (await stored(id, context, 'create')).job;
|
|
90
94
|
}
|
|
91
95
|
export async function readD1ImportJob(id: string, context: D1ImportContext): Promise<ImportJob> {
|
|
@@ -4,6 +4,7 @@ import { compressedPlan, storedStreamPage } from './stream-plan.ts';
|
|
|
4
4
|
import { expandCompressedEnvelope } from './compressed-plan.ts';
|
|
5
5
|
import { planReadScope } from './verified-plan.ts';
|
|
6
6
|
import { Buffer } from 'node:buffer';
|
|
7
|
+
import { RPC_LIMIT } from '../../../d1-transport/index.mjs';
|
|
7
8
|
import { createHash } from 'node:crypto';
|
|
8
9
|
import type { ImportPlan } from '../types.ts';
|
|
9
10
|
import type { D1Database, D1Sql, D1PlanCodec } from './types.ts';
|
|
@@ -27,13 +28,15 @@ export interface StoredPlan {
|
|
|
27
28
|
export const LARGE_PLAN_BYTES = 4 * 1024 * 1024;
|
|
28
29
|
const large = (envelope: any): boolean => envelope?.d1Plan === 1 && Number.isSafeInteger(envelope.size) && envelope.size > LARGE_PLAN_BYTES;
|
|
29
30
|
const snapshots = new WeakMap<ImportPlan, StoredPlan>();
|
|
30
|
-
//
|
|
31
|
-
// the
|
|
32
|
-
|
|
33
|
-
|
|
31
|
+
// Reserve room for JSON row metadata and D1's whole-result two-byte strings.
|
|
32
|
+
// Measure the actual JSON-encoded row ceiling before choosing a page size:
|
|
33
|
+
// legacy chunks can be much larger than today's 128 KiB canonical chunks.
|
|
34
|
+
const planReadRows = (bytes: number): number => Math.max(1, Math.min(64, Math.floor((RPC_LIMIT / 4) / (bytes + 512))));
|
|
35
|
+
const quotedChunkBytes = (chunks: readonly { body: string }[]): number => chunks.reduce((max, chunk) => Math.max(max, Buffer.byteLength(JSON.stringify(chunk))), 0);
|
|
36
|
+
export function planChunkReads(body: string, fenced: boolean, chunks: readonly { body: string }[]): number {
|
|
34
37
|
const envelope = JSON.parse(body);
|
|
35
38
|
// A fenced large plan's revision read precedes its chunk reads.
|
|
36
|
-
return envelope?.d1Plan === 1 ? Math.ceil((envelope.chunks + 1) /
|
|
39
|
+
return envelope?.d1Plan === 1 ? 1 + Math.ceil((envelope.chunks + 1) / planReadRows(quotedChunkBytes(chunks))) + Number(fenced && large(envelope)) : 0;
|
|
37
40
|
}
|
|
38
41
|
export function encodePlan(plan: ImportPlan, id: string): StoredPlan {
|
|
39
42
|
const chunks: StoredPlan['chunks'] = [], hash = createHash('sha256'); let size = 0;
|
|
@@ -72,10 +75,13 @@ export async function loadPlans(rows: any[], db: D1Database, codec?: D1PlanCodec
|
|
|
72
75
|
const revision = await db.prepare('SELECT revision FROM ik_d1_custody_revisions WHERE job_id=?').bind(row.id).first();
|
|
73
76
|
if (revision && Number.isSafeInteger(revision.revision)) fences.set(row.id, revision.revision);
|
|
74
77
|
}
|
|
75
|
-
|
|
76
|
-
|
|
78
|
+
const ceiling = await db.prepare(`SELECT MAX(length(CAST(json_object('job_id',job_id,'ordinal',ordinal,'body',body) AS BLOB))) AS bytes FROM ik_d1_plan_chunks WHERE job_id=?${fence?` AND (${fence.sql})`:''}`).bind(row.id,...fence?.params??[]).first();
|
|
79
|
+
if (!Number.isSafeInteger(ceiling?.bytes) || (ceiling!.bytes as number) < 2) throw new Error('STORED_PLAN_CHANGED');
|
|
80
|
+
const readRows = planReadRows(ceiling!.bytes as number);
|
|
81
|
+
for (let offset = 0; offset <= envelope.chunks; offset += readRows) {
|
|
82
|
+
const found = await db.prepare(`SELECT * FROM ik_d1_plan_chunks WHERE job_id IN (SELECT value FROM json_each(?))${fence?` AND (${fence.sql})`:''} ORDER BY job_id,ordinal LIMIT ${readRows} OFFSET ${offset}`).bind(JSON.stringify([row.id]),...fence?.params??[]).all();
|
|
77
83
|
chunks.push(...found.results);
|
|
78
|
-
if (found.results.length <
|
|
84
|
+
if (found.results.length < readRows) break;
|
|
79
85
|
}
|
|
80
86
|
}
|
|
81
87
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { randomUUID } from 'node:crypto';
|
|
2
2
|
import type { D1CustodyRetirement, D1ErasureJournalEntry, D1ImportContext, D1ImportPerson, D1RetirementOptions, D1RetirementProgress, D1Sql } from './types.ts';
|
|
3
|
-
import { requestContext, packGuards, remaining, D1RequestLimitError } from './budget.ts';
|
|
3
|
+
import { requestContext, limitStatements, packGuards, remaining, D1RequestLimitError } from './budget.ts';
|
|
4
4
|
import { authority, authorize, finalAuthority, guard, readonlyChecks, prepare, readAtomic, sql } from './database.ts';
|
|
5
5
|
import { requireText } from '../provenance/identity.ts';
|
|
6
6
|
import { targetErasure, type ErasureStep } from './erasure.ts';
|
|
@@ -39,7 +39,7 @@ export function retireD1ImportCustody(person: D1ImportPerson, nativeStatements:
|
|
|
39
39
|
export async function retireD1ImportCustody(person: D1ImportPerson, nativeStatements: D1Sql[], input: D1ImportContext, checkpoint: D1Sql[], options: D1RetirementOptions | D1ErasureStepOptions = {}): Promise<D1CustodyRetirement | D1RetirementProgress | D1ErasurePending> {
|
|
40
40
|
if ('operationKey' in options) return retireD1ImportCustodyStep(person, nativeStatements, input, checkpoint, options);
|
|
41
41
|
person = { ...person, related: person.related ? { ...person.related, fields: fieldRedactions(person.related.fields) } : undefined };
|
|
42
|
-
const context = requestContext({ ...input, originalErasurePolicy: input.originalErasurePolicy ?? 'withdraw-affected-artifacts' });
|
|
42
|
+
const context = limitStatements(requestContext({ ...input, originalErasurePolicy: input.originalErasurePolicy ?? 'withdraw-affected-artifacts' }), 1000);
|
|
43
43
|
if (context.maxBatchBytes !== undefined) throw new Error('IMPORT_RETIREMENT_OPERATION_KEY_REQUIRED');
|
|
44
44
|
const values = [...new Set(person.values ?? [])].sort();
|
|
45
45
|
for (const value of values) requireText(value, 'person value');
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
export { jobColumns, provenanceColumns, claimColumns } from './column-layout.ts';
|
|
1
2
|
import { Buffer } from 'node:buffer';
|
|
2
3
|
import type { D1Database, D1Sql } from './types.ts';
|
|
3
4
|
import { guard, sql } from './database.ts';
|
|
@@ -64,9 +65,6 @@ export function exactRows(table: string, columns: string[], rows: any[], scope?:
|
|
|
64
65
|
if (result.length) result[0].expectedSnapshots = [{table,columns,rows,...(selection?{selection}:{})}];
|
|
65
66
|
return result;
|
|
66
67
|
}
|
|
67
|
-
export const jobColumns = ['id','destination','namespace','mapping_version','principal_id','fingerprint','operation_key','plan_json','state','checkpoint','total','fence','lease_token','lease_expires','error','reversal_policy','reversal_policy_hash','reversal_cursor','reversal_hash'];
|
|
68
|
-
export const provenanceColumns = ['id','job_id','ordinal','effect_ordinal','source_identity','destination','target_kind','target_id','reversed','record_json'];
|
|
69
|
-
export const claimColumns = ['identity','source_hash','mapping_version','effects_hash','job_id'];
|
|
70
68
|
|
|
71
69
|
/** One bound JSON value per statement; SQLite preserves strings as TEXT even
|
|
72
70
|
* in BLOB-affinity columns. Column names are internal constants only. */
|
|
@@ -3,7 +3,7 @@ import type { ImportPlan } from '../types.ts';
|
|
|
3
3
|
import type { D1ImportContext, D1Sql } from './types.ts';
|
|
4
4
|
import type { StoredPlan } from './plans.ts';
|
|
5
5
|
import { atomic, atomicCount, guard, prepare, sql } from './database.ts';
|
|
6
|
-
import { D1RequestLimitError, remaining } from './budget.ts';
|
|
6
|
+
import { batchExecutions, D1RequestLimitError, remaining } from './budget.ts';
|
|
7
7
|
import { stagingStatements } from './staging-statements.ts';
|
|
8
8
|
import { batchBytes } from './transport.ts';
|
|
9
9
|
import { bindingPredicate, stageClear, type StageBinding } from './staging-transport.ts';
|
|
@@ -142,7 +142,7 @@ async function resumable(context: D1ImportContext, prior: any, expected: Layout)
|
|
|
142
142
|
* own fences, and never more than STAGED_BATCH_BYTES on the wire. */
|
|
143
143
|
async function stagedBatch(context: D1ImportContext, statements: D1Sql[]): Promise<void> {
|
|
144
144
|
const packed = stagingStatements(context, 'create', statements);
|
|
145
|
-
if (packed.length > remaining(context)) throw new D1RequestLimitError();
|
|
145
|
+
if (batchExecutions(context, packed.length) > remaining(context)) throw new D1RequestLimitError();
|
|
146
146
|
if (batchBytes(packed) > STAGED_BATCH_BYTES) throw new Error('IMPORT_STAGE_TRANSPORT_LIMIT');
|
|
147
147
|
await context.storage.database.batch(packed.map(s => prepare(context.storage.database, s)));
|
|
148
148
|
}
|
|
@@ -192,7 +192,7 @@ export function preflightStagedCreation(plan: ImportPlan, context: D1ImportConte
|
|
|
192
192
|
}
|
|
193
193
|
/** Statements the creation spends in this request, excluding its return reads. */
|
|
194
194
|
export const stagedCreationCost = (context: D1ImportContext, creation: StagedCreation): number =>
|
|
195
|
-
creation.batches.reduce((n, batch) => n + stagingStatements(context, 'create', batch).length, 0) + atomicCount(context, 'create', creation.publication, 0);
|
|
195
|
+
creation.batches.reduce((n, batch) => n + batchExecutions(context, stagingStatements(context, 'create', batch).length), 0) + atomicCount(context, 'create', creation.publication, 0);
|
|
196
196
|
export async function runStagedCreation(context: D1ImportContext, creation: StagedCreation, returnReads: number): Promise<string> {
|
|
197
197
|
if (stagedCreationCost(context, creation) + returnReads > remaining(context)) throw new D1RequestLimitError();
|
|
198
198
|
for (const batch of creation.batches) await stagedBatch(context, batch);
|
|
@@ -10,7 +10,7 @@ export function sizeLimitError(error: unknown): DoorError | undefined {
|
|
|
10
10
|
const reason = targetKinds.length
|
|
11
11
|
? `The exact native snapshot of target kind ${targetKinds.map(kind => JSON.stringify(kind)).join(', ')} exceeds one 2 MiB D1 read; the seed's target adapter needs a compact matchesMany.`
|
|
12
12
|
: error instanceof ImportSizeLimitError ? 'The import custody fence exceeds one 2 MiB D1 read.'
|
|
13
|
-
: error.message === 'IMPORT_D1_REQUEST_LIMIT' ? 'This request needs more than its 1,000 D1
|
|
13
|
+
: error.message === 'IMPORT_D1_REQUEST_LIMIT' ? 'This request needs more than its 1,000 D1 queries; the seed\'s target adapter must read its native set in a fixed number of queries.'
|
|
14
14
|
: 'A D1 statement of this import exceeds its size limit.';
|
|
15
15
|
return new DoorError('import_door_too_large', 413, {targetKinds, reason});
|
|
16
16
|
}
|
|
@@ -46,6 +46,6 @@ export function sourceRefusal(error: unknown): DoorError | undefined {
|
|
|
46
46
|
reason: `The upload holds a credential the door cannot remove: a ${field}. Remove it and upload again.`, ...at});
|
|
47
47
|
}
|
|
48
48
|
if (!(error instanceof ReaderError) && !Object.hasOwn(reasons, code)) return undefined;
|
|
49
|
-
const reason = reasons[code] ?? `The reader refused the upload (${code}).`;
|
|
49
|
+
const reason = error instanceof PlanTooLargeError ? `The import ${error.subject} is ${error.bytes} canonical JSON bytes; the limit is ${error.limit}.` : reasons[code] ?? `The reader refused the upload (${code}).`;
|
|
50
50
|
return new DoorError('source_' + code.toLowerCase().replace(/^source_/, ''), tooLarge.has(code) ? 413 : 422, {reason, ...at});
|
|
51
51
|
}
|