@runeditrun/rer 0.2.76 → 0.2.78
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -0
- package/cli/README.md +55 -0
- package/cli/contract.mjs +1 -1
- package/cli/deploy-command.mjs +4 -2
- package/cli/deploy-inputs.mjs +15 -3
- package/cli/deploy-installation-key.mjs +71 -0
- package/cli/deploy-secrets.mjs +2 -0
- package/cli/deploy.mjs +13 -2
- package/cli/deployment-record.mjs +2 -0
- package/cli/export.mjs +1 -1
- package/cli/main.mjs +1 -1
- package/cli/parse.mjs +3 -2
- package/cli/portability-migrations.mjs +35 -14
- package/cli/portability-worker.mjs +2 -2
- package/cli/portability.mjs +2 -2
- package/cli/runtime-front-door.mjs +1 -1
- package/global/CHANGELOG.md +14 -0
- package/global/README.md +32 -0
- package/modules/dependencies/README.md +71 -1
- package/modules/dependencies/auth.mjs +4 -4
- package/modules/dependencies/crypto.mjs +6 -1
- package/modules/dependencies/custody.mjs +16 -0
- package/modules/dependencies/index.d.mts +1 -1
- package/modules/dependencies/index.mjs +6 -5
- package/modules/dependencies/schema/keys-sealed.sql +4 -0
- package/modules/dependencies/sealing.mjs +55 -0
- package/modules/dependencies/store.mjs +62 -9
- package/modules/deploy-kit/README.md +1 -0
- package/modules/deploy-kit/owner-cli.mjs +19 -7
- package/modules/deploy-kit/owner-rekey.mjs +69 -0
- package/modules/deploy-kit/provider.mjs +9 -4
- package/modules/import-kit/src/d1/README.md +1322 -0
- package/modules/import-kit/src/d1/apply.ts +115 -0
- package/modules/import-kit/src/d1/budget.ts +89 -0
- package/modules/import-kit/src/d1/compressed-plan.ts +63 -0
- package/modules/import-kit/src/d1/creation.ts +95 -0
- package/modules/import-kit/src/d1/custody-fence.ts +27 -0
- package/modules/import-kit/src/d1/custody-state-schema.ts +67 -0
- package/modules/import-kit/src/d1/database.ts +163 -0
- package/modules/import-kit/src/d1/entry-locator.ts +61 -0
- package/modules/import-kit/src/d1/erased-artifacts.ts +38 -0
- package/modules/import-kit/src/d1/erased-custody.ts +36 -0
- package/modules/import-kit/src/d1/erased-plan.ts +47 -0
- package/modules/import-kit/src/d1/erased-reversal.ts +19 -0
- package/modules/import-kit/src/d1/erasure-bounded.ts +22 -0
- package/modules/import-kit/src/d1/erasure-claims.ts +27 -0
- package/modules/import-kit/src/d1/erasure-compiler.ts +67 -0
- package/modules/import-kit/src/d1/erasure-disposition.ts +16 -0
- package/modules/import-kit/src/d1/erasure-fragments.ts +86 -0
- package/modules/import-kit/src/d1/erasure-index.ts +53 -0
- package/modules/import-kit/src/d1/erasure-pages.ts +62 -0
- package/modules/import-kit/src/d1/erasure-progress-schema.ts +8 -0
- package/modules/import-kit/src/d1/erasure-progress.ts +61 -0
- package/modules/import-kit/src/d1/erasure-related.ts +124 -0
- package/modules/import-kit/src/d1/erasure-report.ts +31 -0
- package/modules/import-kit/src/d1/erasure-schema.ts +24 -0
- package/modules/import-kit/src/d1/erasure-selection.ts +77 -0
- package/modules/import-kit/src/d1/erasure-snapshot.ts +15 -0
- package/modules/import-kit/src/d1/erasure-stage.ts +146 -0
- package/modules/import-kit/src/d1/erasure-step.ts +12 -0
- package/modules/import-kit/src/d1/erasure-superseded.ts +98 -0
- package/modules/import-kit/src/d1/erasure-values.ts +83 -0
- package/modules/import-kit/src/d1/erasure-writes.ts +46 -0
- package/modules/import-kit/src/d1/erasure.ts +170 -0
- package/modules/import-kit/src/d1/evolutions/0001-preserved-undo.ts +13 -0
- package/modules/import-kit/src/d1/expected-state.ts +91 -0
- package/modules/import-kit/src/d1/field-redaction.ts +86 -0
- package/modules/import-kit/src/d1/guards.ts +37 -0
- package/modules/import-kit/src/d1/history.ts +71 -0
- package/modules/import-kit/src/d1/index.ts +88 -0
- package/modules/import-kit/src/d1/integrity.ts +127 -0
- package/modules/import-kit/src/d1/inventory-snapshot.ts +84 -0
- package/modules/import-kit/src/d1/jobs.ts +97 -0
- package/modules/import-kit/src/d1/native-set.ts +48 -0
- package/modules/import-kit/src/d1/native-staging-admission.ts +19 -0
- package/modules/import-kit/src/d1/native-staging-context.ts +44 -0
- package/modules/import-kit/src/d1/native-staging-lifecycle.ts +33 -0
- package/modules/import-kit/src/d1/native-staging-offset.ts +74 -0
- package/modules/import-kit/src/d1/native-staging-publication.ts +18 -0
- package/modules/import-kit/src/d1/native-staging-rows.ts +72 -0
- package/modules/import-kit/src/d1/native-staging-schema.ts +28 -0
- package/modules/import-kit/src/d1/native-staging-state.ts +22 -0
- package/modules/import-kit/src/d1/native-staging.ts +64 -0
- package/modules/import-kit/src/d1/original-copies.ts +53 -0
- package/modules/import-kit/src/d1/original-storage.ts +56 -0
- package/modules/import-kit/src/d1/originals.ts +74 -0
- package/modules/import-kit/src/d1/plan-dictionary.ts +80 -0
- package/modules/import-kit/src/d1/plans.ts +121 -0
- package/modules/import-kit/src/d1/preparation-fingerprint.ts +57 -0
- package/modules/import-kit/src/d1/preparation-header.ts +33 -0
- package/modules/import-kit/src/d1/preparation-reader.ts +98 -0
- package/modules/import-kit/src/d1/preparation-schema.ts +16 -0
- package/modules/import-kit/src/d1/preparation-state.ts +56 -0
- package/modules/import-kit/src/d1/preparation-step-combine.ts +80 -0
- package/modules/import-kit/src/d1/preparation-step-control.ts +48 -0
- package/modules/import-kit/src/d1/preparation-step-hash.ts +131 -0
- package/modules/import-kit/src/d1/preparation-step-pages.ts +77 -0
- package/modules/import-kit/src/d1/preparation-steps-schema.ts +11 -0
- package/modules/import-kit/src/d1/preparation-steps.ts +102 -0
- package/modules/import-kit/src/d1/preparations.ts +87 -0
- package/modules/import-kit/src/d1/preserved-preview.ts +89 -0
- package/modules/import-kit/src/d1/preserved-reverse.ts +91 -0
- package/modules/import-kit/src/d1/provenance-set.ts +87 -0
- package/modules/import-kit/src/d1/publication-fence.ts +22 -0
- package/modules/import-kit/src/d1/publication-receipt-schema.ts +15 -0
- package/modules/import-kit/src/d1/published-stream.ts +11 -0
- package/modules/import-kit/src/d1/read-fence.ts +64 -0
- package/modules/import-kit/src/d1/read-results.ts +16 -0
- package/modules/import-kit/src/d1/redacted-custody.ts +76 -0
- package/modules/import-kit/src/d1/redaction-step.ts +125 -0
- package/modules/import-kit/src/d1/referenced-plans.ts +136 -0
- package/modules/import-kit/src/d1/retained-values.ts +35 -0
- package/modules/import-kit/src/d1/retirement-bounded.ts +71 -0
- package/modules/import-kit/src/d1/retirement-door.ts +105 -0
- package/modules/import-kit/src/d1/retirement-schema.ts +39 -0
- package/modules/import-kit/src/d1/retirement-scope.ts +94 -0
- package/modules/import-kit/src/d1/retirement-steps.ts +76 -0
- package/modules/import-kit/src/d1/retirement-tally.ts +74 -0
- package/modules/import-kit/src/d1/retirement.ts +136 -0
- package/modules/import-kit/src/d1/reversal-conversion-record.ts +19 -0
- package/modules/import-kit/src/d1/reversal-conversion-schema.ts +11 -0
- package/modules/import-kit/src/d1/reversal-conversion.ts +40 -0
- package/modules/import-kit/src/d1/reversal-fence-schema.ts +14 -0
- package/modules/import-kit/src/d1/reversal-fence.ts +106 -0
- package/modules/import-kit/src/d1/reversal-journal.ts +24 -0
- package/modules/import-kit/src/d1/reverse.ts +103 -0
- package/modules/import-kit/src/d1/reviewed-source.ts +23 -0
- package/modules/import-kit/src/d1/sets.ts +78 -0
- package/modules/import-kit/src/d1/staged-creation.ts +201 -0
- package/modules/import-kit/src/d1/staging-admission.ts +45 -0
- package/modules/import-kit/src/d1/staging-control.ts +43 -0
- package/modules/import-kit/src/d1/staging-erasure.ts +28 -0
- package/modules/import-kit/src/d1/staging-lifecycle.ts +49 -0
- package/modules/import-kit/src/d1/staging-plan.ts +15 -0
- package/modules/import-kit/src/d1/staging-privacy.ts +22 -0
- package/modules/import-kit/src/d1/staging-publication.ts +79 -0
- package/modules/import-kit/src/d1/staging-schema.ts +37 -0
- package/modules/import-kit/src/d1/staging-statements.ts +8 -0
- package/modules/import-kit/src/d1/staging-transport.ts +70 -0
- package/modules/import-kit/src/d1/stream-boundary.ts +11 -0
- package/modules/import-kit/src/d1/stream-create.ts +55 -0
- package/modules/import-kit/src/d1/stream-identity-admission.ts +37 -0
- package/modules/import-kit/src/d1/stream-input.ts +18 -0
- package/modules/import-kit/src/d1/stream-layout.ts +43 -0
- package/modules/import-kit/src/d1/stream-origin.ts +44 -0
- package/modules/import-kit/src/d1/stream-page-integrity.ts +18 -0
- package/modules/import-kit/src/d1/stream-partition.ts +50 -0
- package/modules/import-kit/src/d1/stream-plan.ts +133 -0
- package/modules/import-kit/src/d1/stream-receipt.ts +45 -0
- package/modules/import-kit/src/d1/stream-revision-schema.ts +35 -0
- package/modules/import-kit/src/d1/stream-snapshot.ts +26 -0
- package/modules/import-kit/src/d1/stream-stage.ts +79 -0
- package/modules/import-kit/src/d1/stream-state.ts +25 -0
- package/modules/import-kit/src/d1/stream-working-set.ts +30 -0
- package/modules/import-kit/src/d1/streams.ts +39 -0
- package/modules/import-kit/src/d1/target-guards.ts +9 -0
- package/modules/import-kit/src/d1/target-identities.ts +17 -0
- package/modules/import-kit/src/d1/transport.ts +10 -0
- package/modules/import-kit/src/d1/types.ts +162 -0
- package/modules/import-kit/src/d1/undo-observations.ts +39 -0
- package/modules/import-kit/src/d1/undo-report.ts +22 -0
- package/modules/import-kit/src/d1/undo-snapshot.ts +87 -0
- package/modules/import-kit/src/d1/undo-state.ts +84 -0
- package/modules/import-kit/src/d1/undo-types.ts +110 -0
- package/modules/import-kit/src/journal/authority.ts +12 -0
- package/modules/import-kit/src/plan/budget.ts +60 -0
- package/modules/import-kit/src/plan/index.ts +123 -0
- package/modules/import-kit/src/plan/size.ts +122 -0
- package/modules/import-kit/src/provenance/canonical-hash.ts +130 -0
- package/modules/import-kit/src/provenance/canonical-stream.ts +85 -0
- package/modules/import-kit/src/provenance/credential-names.ts +131 -0
- package/modules/import-kit/src/provenance/credentials.ts +317 -0
- package/modules/import-kit/src/provenance/identity.ts +51 -0
- package/modules/import-kit/src/provenance/json-snapshot.ts +21 -0
- package/modules/import-kit/src/provenance/resumable-sha256.ts +75 -0
- package/modules/import-kit/src/provenance/validate-json.ts +19 -0
- package/modules/import-kit/src/storage/originals.ts +87 -0
- package/modules/import-kit/src/storage/provenance.ts +64 -0
- package/modules/import-kit/src/storage/schema.ts +42 -0
- package/modules/import-kit/src/types.ts +47 -0
- package/modules/operator-auth/docs/native-accounts.md +4 -1
- package/modules/operator-auth/identity/oidc/aes-gcm.mjs +21 -0
- package/modules/operator-auth/identity/oidc/crypto.mjs +5 -12
- package/modules/operator-auth/index.d.mts +1 -1
- package/modules/portability/README.md +98 -2
- package/modules/portability/backfills.mjs +42 -15
- package/modules/portability/ledgers.mjs +7 -1
- package/modules/portability/transient-tables.mjs +40 -0
- package/onboarding/local/environment.mjs +2 -2
- package/package.json +3 -1
- package/scripts/publish-check/reviewed.py +3 -1
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import { retainedSql } from './retained-values.ts';
|
|
2
|
+
import { nativeStagingScope, readStagedTarget, expectedStagedTarget, type NativeStagingScope } from './native-staging-context.ts';
|
|
3
|
+
import { targetGuards } from './target-guards.ts';
|
|
4
|
+
import { D1RequestLimitError, requestContext, remaining } from './budget.ts';
|
|
5
|
+
import type { ImportJob, PlanEntry, ProvenanceRow, TargetRow } from '../types.ts';
|
|
6
|
+
import { canonicalJson, sha256, attachmentHash } from '../provenance/identity.ts';
|
|
7
|
+
import { validateEffect } from '../plan/index.ts';
|
|
8
|
+
import { effectIdentity } from '../storage/provenance.ts';
|
|
9
|
+
import type { D1ImportContext, D1Sql } from './types.ts';
|
|
10
|
+
import { atomic, atomicCount, authorize, guard, limits, sql, fitsBatch } from './database.ts';
|
|
11
|
+
import { stored, boundary, storedReads } from './jobs.ts';
|
|
12
|
+
import { custodyGuards, validatedProvenance, inventoryGuards, reusedSourceGuards } from './integrity.ts';
|
|
13
|
+
import { supersedeTarget } from './history.ts';
|
|
14
|
+
import { readOriginals } from './originals.ts';
|
|
15
|
+
import { changed, putRow, updateRows } from './expected-state.ts';
|
|
16
|
+
|
|
17
|
+
export async function applyD1ImportBatches(jobOrId: ImportJob | string, context: D1ImportContext): Promise<ImportJob> {
|
|
18
|
+
context = requestContext(context);
|
|
19
|
+
const id = typeof jobOrId === 'string' ? jobOrId : jobOrId.id, { batchSize, maxBatches } = limits(context);
|
|
20
|
+
await authorize(context, 'apply');
|
|
21
|
+
let current = await stored(id, context, 'apply'), published: ImportJob | undefined;
|
|
22
|
+
try {
|
|
23
|
+
for (let count = 0; count < maxBatches; count++) {
|
|
24
|
+
const { job, plan } = current;
|
|
25
|
+
if (typeof jobOrId !== 'string' && jobOrId.fingerprint !== job.fingerprint) throw new Error('JOB_FINGERPRINT_MISMATCH');
|
|
26
|
+
await readOriginals(plan, id, context);
|
|
27
|
+
if (job.state === 'completed') { await atomic(context, 'apply', [...boundary(job, plan, 'apply'), ...custodyGuards(plan, id), ...await inventoryGuards(job, plan, context)], 0, nativeStagingScope(job, plan)); return job; }
|
|
28
|
+
if (!['queued', 'applying', 'failed'].includes(job.state)) throw new Error('IMPORT_ALREADY_REVERSING');
|
|
29
|
+
const statements: D1Sql[] = [...boundary(job, plan, 'apply'), ...custodyGuards(plan, id), ...await inventoryGuards(job, plan, context)], entries: PlanEntry[] = [], returnReads = storedReads(job);
|
|
30
|
+
let effects = 0, scope = nativeStagingScope(job, plan), planned = cost(context, publication(job, 0, statements, scope), returnReads);
|
|
31
|
+
for (const entry of plan.entries.slice(job.checkpoint, job.checkpoint + batchSize)) {
|
|
32
|
+
if (entries.length && effects + entry.effects.length > 1000) break;
|
|
33
|
+
if (entry.effects.length > 1000) throw new Error('SOURCE_EFFECT_LIMIT');
|
|
34
|
+
// Stop before an entry's own reads could exhaust the request. The exact
|
|
35
|
+
// request, as atomic will send it, is counted again once the entry is built.
|
|
36
|
+
const reserve = 40 + entry.effects.length * 16;
|
|
37
|
+
if (remaining(context) - planned < reserve) { if (!entries.length) { if (count > 0) return job; throw new D1RequestLimitError(); } break; }
|
|
38
|
+
const candidate = [...statements], nextScope = { ...scope, nativeGuards:[...scope.nativeGuards], postGuards:[...scope.postGuards], targets:[...scope.targets] };
|
|
39
|
+
await publishEntry(id, entry, job.checkpoint + entries.length, context, candidate, nextScope);
|
|
40
|
+
const next = publication(job, entries.length + 1, candidate, nextScope), spent = cost(context, next, returnReads);
|
|
41
|
+
if (spent > remaining(context)) { if (entries.length) break; if (count > 0) return job; throw new D1RequestLimitError(); }
|
|
42
|
+
const fits = fitsBatch(context, 'apply', next.statements, next.scope);
|
|
43
|
+
if (!fits && entries.length) break;
|
|
44
|
+
statements.splice(0, statements.length, ...candidate);
|
|
45
|
+
scope = nextScope; entries.push(entry); effects += entry.effects.length; planned = spent;
|
|
46
|
+
if (!fits) break;
|
|
47
|
+
}
|
|
48
|
+
const request = publication(job, entries.length, statements, scope), done = job.checkpoint + entries.length === job.total;
|
|
49
|
+
await atomic(context, 'apply', request.statements, returnReads, request.scope);
|
|
50
|
+
current = await stored(id, context, 'apply'); published = current.job;
|
|
51
|
+
if (done || remaining(context) < 200) return published;
|
|
52
|
+
}
|
|
53
|
+
} catch (error) {
|
|
54
|
+
// Later preflight has no native effects. Its measured budget exhaustion
|
|
55
|
+
// returns the last checked publication; integrity/authority errors still fail.
|
|
56
|
+
if (error instanceof D1RequestLimitError && published) return published;
|
|
57
|
+
throw error;
|
|
58
|
+
}
|
|
59
|
+
return current.job;
|
|
60
|
+
}
|
|
61
|
+
async function publishEntry(id: string, entry: PlanEntry, ordinal: number, context: D1ImportContext, statements: D1Sql[], scope: NativeStagingScope): Promise<void> {
|
|
62
|
+
const { database: db, targets } = context.storage, effectsHash = sha256(canonicalJson(entry.effects));
|
|
63
|
+
const prior = await db.prepare('SELECT * FROM ik_sources WHERE identity=?').bind(entry.identity).first();
|
|
64
|
+
if (prior) {
|
|
65
|
+
if (prior.source_hash !== entry.source.payloadHash || prior.mapping_version !== context.mappingVersion || prior.effects_hash !== effectsHash) throw new Error('SOURCE_ID_CONTENT_CONFLICT');
|
|
66
|
+
await reusedSourceGuards(entry, prior.job_id, context, statements);
|
|
67
|
+
statements.push(guard(sql('EXISTS(SELECT 1 FROM ik_sources WHERE identity=? AND source_hash=? AND mapping_version=? AND effects_hash=? AND job_id=?)', entry.identity, prior.source_hash, prior.mapping_version, prior.effects_hash, prior.job_id)));
|
|
68
|
+
for (const effect of entry.effects) {
|
|
69
|
+
const raw = await db.prepare('SELECT * FROM ik_provenance WHERE source_identity=? AND destination=? AND target_kind=? AND target_id=? AND reversed=0').bind(entry.identity, context.destination, effect.targetKind, effect.targetId).first();
|
|
70
|
+
if (!raw) throw new Error('PROVENANCE_MISSING');
|
|
71
|
+
const p = await validatedProvenance(raw, context, statements);
|
|
72
|
+
const current = await readStagedTarget(context, scope, effect.targetKind, effect.targetId);
|
|
73
|
+
if (!current || current.revision !== p.appliedRevision || sha256(canonicalJson(current.value)) !== p.valueHash || attachmentHash(current.attachment) !== p.attachmentHash) throw new Error('NATIVE_EDIT_CONFLICT');
|
|
74
|
+
const post = targetGuards(targets,current,context.destination,effect.targetKind,effect.targetId);
|
|
75
|
+
scope.postGuards.push(...post);
|
|
76
|
+
statements.push(guard(retainedSql(['EXISTS(SELECT 1 FROM ik_provenance WHERE id=', ' AND record_json=', ' AND reversed=0)'], [raw.id, raw.record_json], [1])), ...post);
|
|
77
|
+
}
|
|
78
|
+
statements.push(entryWrite(id,ordinal,entry.identity,prior.job_id,effectsHash));
|
|
79
|
+
return;
|
|
80
|
+
}
|
|
81
|
+
statements.push(guard(sql('NOT EXISTS(SELECT 1 FROM ik_sources WHERE identity=?)', entry.identity)));
|
|
82
|
+
for (const [index, effect] of entry.effects.entries()) {
|
|
83
|
+
validateEffect(effect);
|
|
84
|
+
const before = await readStagedTarget(context, scope, effect.targetKind, effect.targetId);
|
|
85
|
+
if (before ? effect.expectedRevision !== before.revision : effect.expectedRevision !== undefined) throw new Error('TARGET_REVISION_CONFLICT');
|
|
86
|
+
const appliedRevision = sha256(canonicalJson([id, entry.identity, index, 'applied']));
|
|
87
|
+
const row: TargetRow = { destination: context.destination, targetKind: effect.targetKind, targetId: effect.targetId, value: effect.value, revision: appliedRevision, ...(effect.attachment ? { attachment: effect.attachment } : {}) };
|
|
88
|
+
expectedStagedTarget(scope,effect.targetKind,effect.targetId,true);
|
|
89
|
+
const post = targetGuards(targets, row, context.destination, effect.targetKind, effect.targetId);
|
|
90
|
+
scope.postGuards.push(...post);
|
|
91
|
+
const p: ProvenanceRow = { id: effectIdentity(id, entry.identity, index), jobId: id, sourceIdentity: entry.identity, sourceKind: entry.source.kind, sourceId: entry.source.sourceId, sourceHash: entry.source.payloadHash, artifactHash: entry.source.artifactHash, location: entry.source.location, destination: context.destination, targetKind: effect.targetKind, targetId: effect.targetId, valueHash: sha256(canonicalJson(effect.value)), attachmentHash: attachmentHash(effect.attachment), appliedRevision, before: before ?? null, reversed: false };
|
|
92
|
+
await supersedeTarget(p, before, context, statements);
|
|
93
|
+
const record_json=canonicalJson(p);
|
|
94
|
+
statements.push(...targetGuards(targets,before ?? null, context.destination, effect.targetKind, effect.targetId), ...targets.write(row), ...post, changed(retainedSql(['INSERT INTO ik_provenance(id,job_id,ordinal,effect_ordinal,source_identity,destination,target_kind,target_id,record_json) VALUES(', ',', ',', ',', ',', ',', ',', ',', ',', ')'], [p.id, id, ordinal, index, entry.identity, context.destination, effect.targetKind, effect.targetId, record_json], [8]),putRow('ik_provenance',{id:p.id,job_id:id,ordinal,effect_ordinal:index,source_identity:entry.identity,destination:context.destination,target_kind:effect.targetKind,target_id:effect.targetId,reversed:0,record_json})));
|
|
95
|
+
}
|
|
96
|
+
statements.push(entryWrite(id,ordinal,entry.identity,id,effectsHash));
|
|
97
|
+
statements.push(changed(sql('INSERT INTO ik_sources(identity,source_hash,mapping_version,effects_hash,job_id) VALUES(?,?,?,?,?)', entry.identity, entry.source.payloadHash, context.mappingVersion, effectsHash, id),putRow('ik_sources',{identity:entry.identity,source_hash:entry.source.payloadHash,mapping_version:context.mappingVersion,effects_hash:effectsHash,job_id:id})));
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
function entryWrite(job_id:string,ordinal:number,source_identity:string,owner_job_id:string,effects_hash:string):D1Sql{
|
|
101
|
+
return changed(sql('INSERT INTO ik_d1_entries(job_id,ordinal,source_identity,owner_job_id,effects_hash) VALUES(?,?,?,?,?)',job_id,ordinal,source_identity,owner_job_id,effects_hash),putRow('ik_d1_entries',{job_id,ordinal,source_identity,owner_job_id,effects_hash}));
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/** The request that publishes the first `count` gathered entries: their SQL,
|
|
105
|
+
* the progress write and the guard that the progress committed. */
|
|
106
|
+
function publication(job: ImportJob, count: number, statements: D1Sql[], scope: NativeStagingScope): { statements: D1Sql[]; scope: NativeStagingScope } {
|
|
107
|
+
const end = job.checkpoint + count;
|
|
108
|
+
return { statements: [...statements, ...progress(job, count)], scope: { ...scope, postGuards: [...scope.postGuards, guard(sql('EXISTS(SELECT 1 FROM ik_jobs WHERE id=? AND checkpoint=? AND state=? AND fence=? AND error IS NULL)', job.id, end, end === job.total ? 'completed' : 'applying', job.fence + 1))] } };
|
|
109
|
+
}
|
|
110
|
+
const cost = (context: D1ImportContext, request: { statements: D1Sql[]; scope: NativeStagingScope }, returnReads: number): number => atomicCount(context, 'apply', request.statements, returnReads, request.scope);
|
|
111
|
+
function progress(job: ImportJob, count: number): D1Sql[] {
|
|
112
|
+
const end = job.checkpoint + count;
|
|
113
|
+
const state=end===job.total?'completed':'applying';
|
|
114
|
+
return [changed(sql("INSERT INTO ik_batches(job_id,action,position,end_position,fence) VALUES(?,'apply',?,?,?)", job.id, job.checkpoint, end, job.fence + 1),putRow('ik_batches',{job_id:job.id,action:'apply',position:job.checkpoint,end_position:end,fence:job.fence+1})), changed(sql('UPDATE ik_jobs SET checkpoint=?,state=?,fence=fence+1,error=NULL WHERE id=?', end, state, job.id),updateRows('ik_jobs',row=>row.id===job.id,row=>({...row,checkpoint:end,state,fence:job.fence+1,error:null})))];
|
|
115
|
+
}
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import { Buffer } from 'node:buffer';
|
|
2
|
+
import { wireDatabase } from '../../../d1-transport/index.mjs';
|
|
3
|
+
import { guard, guardPredicate } from './guards.ts';
|
|
4
|
+
import type { D1Database, D1ImportContext, D1Sql } from './types.ts';
|
|
5
|
+
const budgets = new WeakMap<D1Database, { used: number; limit: number }>();
|
|
6
|
+
const reservedTail = Symbol('import request return queries');
|
|
7
|
+
type ReservedContext = D1ImportContext & { [reservedTail]?: number };
|
|
8
|
+
/** Distinguish this request's measured exhaustion from adapter/provider errors. */
|
|
9
|
+
export class D1RequestLimitError extends Error {
|
|
10
|
+
constructor() { super('IMPORT_D1_REQUEST_LIMIT'); }
|
|
11
|
+
}
|
|
12
|
+
export const valueBytes = (v: any): number => typeof v === 'string' ? Buffer.byteLength(v) : v instanceof ArrayBuffer || ArrayBuffer.isView(v) ? v.byteLength : 8;
|
|
13
|
+
export function checkSql(s: D1Sql): void {
|
|
14
|
+
if (Buffer.byteLength(s.sql) > 100000 || (s.params?.length ?? 0) > 100) throw new Error('IMPORT_D1_STATEMENT_LIMIT');
|
|
15
|
+
if ((s.params ?? []).some(p => valueBytes(p) > 2000000) || (s.params ?? []).reduce((n, p) => n + valueBytes(p), 0) > 2000000) throw new Error('IMPORT_D1_VALUE_LIMIT');
|
|
16
|
+
}
|
|
17
|
+
export function remaining(context: D1ImportContext): number { const budget=budgets.get(context.storage.database);return (budget?.limit??1000)-(budget?.used??0)-((context as ReservedContext)[reservedTail]??0); }
|
|
18
|
+
/** A smaller public-operation budget also stops reads, before execution. Nested
|
|
19
|
+
* calls can only tighten the shared request's existing execution ceiling. */
|
|
20
|
+
export function limitStatements(context:D1ImportContext,max:number):D1ImportContext {
|
|
21
|
+
const budget=budgets.get(context.storage.database);
|
|
22
|
+
if(!budget||!Number.isSafeInteger(max)||max<1||max>1000)throw new Error('INVALID_ERASURE_STATEMENT_LIMIT');
|
|
23
|
+
budget.limit=Math.min(budget.limit,budget.used+max);
|
|
24
|
+
return context;
|
|
25
|
+
}
|
|
26
|
+
/** A nested operation leaves the outer operation's exact bounded return path
|
|
27
|
+
* available. Spreads preserve the symbol; the original outer context releases
|
|
28
|
+
* the reservation simply by continuing its own already admitted reads. */
|
|
29
|
+
export function reserveReturn(context: D1ImportContext, queries: number): D1ImportContext {
|
|
30
|
+
if(!Number.isSafeInteger(queries)||queries<0||queries>remaining(context))throw new D1RequestLimitError();
|
|
31
|
+
return {...context,[reservedTail]:((context as ReservedContext)[reservedTail]??0)+queries} as ReservedContext;
|
|
32
|
+
}
|
|
33
|
+
/** One shared request budget for a caller (such as the import door) whose own
|
|
34
|
+
* statements and every kit call it makes must count against the same ceiling. */
|
|
35
|
+
export function requestDatabase(database: D1Database): D1Database {
|
|
36
|
+
return requestContext({ storage: { database } } as D1ImportContext).storage.database;
|
|
37
|
+
}
|
|
38
|
+
/** A fresh wrapper belongs to the outer public invocation; nested resume calls
|
|
39
|
+
* reuse it. It counts individual executed SQL statements, including batches. */
|
|
40
|
+
export function requestContext(context: D1ImportContext): D1ImportContext {
|
|
41
|
+
if (budgets.has(context.storage.database)) return context;
|
|
42
|
+
const db = wireDatabase(context.storage.database), budget = { used: 0, limit: 1000 };
|
|
43
|
+
const consume = (n: number) => { if (budget.used + n > budget.limit) throw new D1RequestLimitError(); budget.used += n; };
|
|
44
|
+
const raw = new WeakMap<object, any>();
|
|
45
|
+
const wrap = (statement: any, query: D1Sql): any => {
|
|
46
|
+
checkSql(query);
|
|
47
|
+
const result = { bind(...params: any[]) { return wrap(statement.bind(...params), { sql: query.sql, params }); }, async first(...args: any[]) { consume(1); return statement.first(...args); }, async all(...args: any[]) { consume(1); return statement.all(...args); } };
|
|
48
|
+
raw.set(result, statement); return result;
|
|
49
|
+
};
|
|
50
|
+
const database: D1Database = { prepare(query) { return wrap(db.prepare(query), { sql: query }); }, async batch(statements) { consume(statements.length); return db.batch(statements.map(s => raw.get(s))); } };
|
|
51
|
+
budgets.set(database, budget); return { ...context, storage: { ...context.storage, database } };
|
|
52
|
+
}
|
|
53
|
+
function conjunction(predicates: string[], start = 0, end = predicates.length): string {
|
|
54
|
+
if (end - start === 1) return predicates[start];
|
|
55
|
+
const middle = start + Math.floor((end - start) / 2);
|
|
56
|
+
return `(${conjunction(predicates, start, middle)} AND ${conjunction(predicates, middle, end)})`;
|
|
57
|
+
}
|
|
58
|
+
/** Adjacent assertions can share one CHECK without changing transactional order.
|
|
59
|
+
* Native statements remain between their original pre/postconditions. */
|
|
60
|
+
export function packGuards(statements: D1Sql[], readOnly = false): D1Sql[] {
|
|
61
|
+
const prefix = readOnly ? 'SELECT CASE WHEN (' : 'INSERT INTO ik_d1_assertions(ok) SELECT CASE WHEN (';
|
|
62
|
+
const suffix = readOnly ? ') THEN 1 ELSE 0 END AS ok' : ') THEN 1 ELSE 0 END';
|
|
63
|
+
const result: D1Sql[] = []; const seen = new Map<string, D1Sql[]>(); let predicates: string[] = [], params: any[] = [], bytes = 0;
|
|
64
|
+
const flush = () => {
|
|
65
|
+
if (predicates.length) {
|
|
66
|
+
// A flat AND list becomes a left-deep SQLite expression tree. Keep the
|
|
67
|
+
// added depth logarithmic without changing predicates or binding order.
|
|
68
|
+
const text=conjunction(predicates),packed = readOnly ? { sql: `${prefix}${text}${suffix}`, params } : guard({sql:text,params});
|
|
69
|
+
checkSql(packed); result.push(packed);
|
|
70
|
+
}
|
|
71
|
+
predicates = []; params = []; bytes = 0;
|
|
72
|
+
};
|
|
73
|
+
for (const statement of statements) {
|
|
74
|
+
checkSql(statement);
|
|
75
|
+
const predicate = guardPredicate(statement);
|
|
76
|
+
if (!predicate) {
|
|
77
|
+
if (readOnly) throw new Error('IMPORT_READ_GUARD_REQUIRED');
|
|
78
|
+
flush(); seen.clear(); result.push(statement); continue;
|
|
79
|
+
}
|
|
80
|
+
const prior = seen.get(statement.sql) ?? [];
|
|
81
|
+
if (prior.some(s => (s.params?.length ?? 0) === (statement.params?.length ?? 0) && (s.params ?? []).every((p, i) => p === statement.params![i]))) continue;
|
|
82
|
+
prior.push(statement); seen.set(statement.sql, prior);
|
|
83
|
+
const p = predicate.params ?? [], text = `(${predicate.sql})`;
|
|
84
|
+
const nextBytes = Buffer.byteLength(text);
|
|
85
|
+
if (predicates.length && (params.length + p.length > 100 || bytes + nextBytes > 90000 || params.reduce((n, p) => n + valueBytes(p), 0) + p.reduce((n, p) => n + valueBytes(p), 0) > 1500000 || predicates.length >= 50)) flush();
|
|
86
|
+
predicates.push(text); params.push(...p); bytes += nextBytes;
|
|
87
|
+
}
|
|
88
|
+
flush(); return result;
|
|
89
|
+
}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import { dictionaryEncode, dictionaryDecode } from './plan-dictionary.ts';
|
|
2
|
+
import { deflateRawSync, inflateRawSync, brotliCompressSync, brotliDecompressSync, constants } from 'node:zlib';
|
|
3
|
+
import { Buffer } from 'node:buffer';
|
|
4
|
+
import type { ImportPlan, Json } from '../types.ts';
|
|
5
|
+
import type { D1PlanCodec } from './types.ts';
|
|
6
|
+
import { planFingerprint } from '../plan/index.ts';
|
|
7
|
+
import { canonicalHash, canonicalSize, canonicalChunks, cloneJson } from '../provenance/canonical-stream.ts';
|
|
8
|
+
import { canonicalJson, sha256 } from '../provenance/identity.ts';
|
|
9
|
+
|
|
10
|
+
function validateRoundTrip(plan: ImportPlan, payload: Json, codec: D1PlanCodec): void {
|
|
11
|
+
const roundtrip = codec.decode(cloneJson(payload));
|
|
12
|
+
if (roundtrip.fingerprint !== plan.fingerprint || planFingerprint(roundtrip) !== plan.fingerprint) throw new Error('IMPORT_PLAN_CODEC_MISMATCH');
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export function compressedPlan(plan: ImportPlan, codec?: D1PlanCodec): string {
|
|
16
|
+
const expandedSize = canonicalSize(plan), payload = codec ? codec.encode(plan) : plan as unknown as Json;
|
|
17
|
+
if (codec) validateRoundTrip(plan, payload, codec);
|
|
18
|
+
const payloadSize = codec ? canonicalSize(payload) : undefined;
|
|
19
|
+
const body = codec ? JSON.stringify(dictionaryEncode(payload)) : [...canonicalChunks(plan)].join('');
|
|
20
|
+
const compressed = codec ? brotliCompressSync(body,{params:{[constants.BROTLI_PARAM_QUALITY]:4,[constants.BROTLI_PARAM_LGWIN]:20}}) : deflateRawSync(body);
|
|
21
|
+
return canonicalJson({ d1Plan: codec ? 3 : 2, ...(codec ? {codec:codec.id,dictionary:2,compression:'brotli',fingerprint:plan.fingerprint,expandedSize,payloadSize} : {}), size: Buffer.byteLength(body), sha256: sha256(body), data: Buffer.from(compressed.buffer,compressed.byteOffset,compressed.byteLength).toString('base64') });
|
|
22
|
+
}
|
|
23
|
+
export function expandCompressedPlan(body: string, codec?: D1PlanCodec): ImportPlan {
|
|
24
|
+
return expandCompressedEnvelope(body, JSON.parse(body), codec);
|
|
25
|
+
}
|
|
26
|
+
/** Custody loading already parses the envelope to distinguish storage formats.
|
|
27
|
+
* Reuse that object, still comparing its complete canonical bytes with body. */
|
|
28
|
+
export function expandCompressedEnvelope(body: string, value: any, codec?: D1PlanCodec): ImportPlan {
|
|
29
|
+
if (!value || typeof value !== 'object' || Array.isArray(value)) throw new Error('STORED_PLAN_CHANGED');
|
|
30
|
+
if (canonicalHash(value) !== sha256(body)) throw new Error('STORED_PLAN_CHANGED');
|
|
31
|
+
if (![2,3].includes(value.d1Plan) || !Number.isSafeInteger(value.size) || value.size < 0) throw new Error('STORED_PLAN_CHANGED');
|
|
32
|
+
const keys = value.d1Plan === 3 ? ['codec','compression','d1Plan','data','dictionary','expandedSize','fingerprint','sha256','size'] : ['d1Plan','data','sha256','size'];
|
|
33
|
+
if (value.d1Plan === 3 && value.dictionary === 2) keys.push('payloadSize');
|
|
34
|
+
if (canonicalJson(Object.keys(value).sort()) !== canonicalJson(keys.sort()) || value.d1Plan === 3 && (![1,2].includes(value.dictionary) || value.compression !== 'brotli')) throw new Error('STORED_PLAN_CHANGED');
|
|
35
|
+
if (value.compression !== undefined && value.compression !== 'brotli') throw new Error('STORED_PLAN_CHANGED');
|
|
36
|
+
if (value.d1Plan === 3 && (!Number.isSafeInteger(value.expandedSize) || value.expandedSize < 0)) throw new Error('STORED_PLAN_CHANGED');
|
|
37
|
+
if (value.d1Plan === 3 && value.dictionary === 2 && (!Number.isSafeInteger(value.payloadSize) || value.payloadSize < 0)) throw new Error('STORED_PLAN_CHANGED');
|
|
38
|
+
const payload = decodePayload(value, codec);
|
|
39
|
+
const payloadHash = value.d1Plan === 3 ? canonicalHash(payload) : undefined;
|
|
40
|
+
const plan = (value.d1Plan === 3 ? codec!.decode(payload) : payload) as ImportPlan;
|
|
41
|
+
if (value.d1Plan === 3 && canonicalHash(codec!.encode(plan)) !== payloadHash) throw new Error('STORED_PLAN_CHANGED');
|
|
42
|
+
if (value.d1Plan === 3 && plan.fingerprint !== value.fingerprint) throw new Error('STORED_PLAN_CHANGED');
|
|
43
|
+
if (value.d1Plan === 3 && canonicalSize(plan) !== value.expandedSize) throw new Error('STORED_PLAN_CHANGED');
|
|
44
|
+
if (planFingerprint(plan) !== plan.fingerprint) throw new Error('STORED_PLAN_CHANGED');
|
|
45
|
+
return plan;
|
|
46
|
+
}
|
|
47
|
+
/** Release compressed bytes and the dictionary tree before native decoding. */
|
|
48
|
+
function decodePayload(value: any, codec?: D1PlanCodec): Json {
|
|
49
|
+
// workerd accounts Brotli decoder workspace against maxOutputLength too.
|
|
50
|
+
// Keep that allowance fixed and still require the exact declared output.
|
|
51
|
+
const input = Buffer.from(value.data, 'base64');
|
|
52
|
+
const result: any = (value.compression === 'brotli' ? brotliDecompressSync : inflateRawSync)(input, { info: true, maxOutputLength: value.size + (value.compression === 'brotli' ? 2 * 1024 * 1024 : 0) });
|
|
53
|
+
if (input.toString('base64') !== value.data || result.engine.bytesWritten !== input.length) throw new Error('STORED_PLAN_CHANGED');
|
|
54
|
+
const decoded: Buffer = result.buffer;
|
|
55
|
+
if (decoded.length !== value.size || sha256(decoded) !== value.sha256) throw new Error('STORED_PLAN_CHANGED');
|
|
56
|
+
if (value.d1Plan === 3 && (!codec || codec.id !== value.codec)) throw new Error('IMPORT_PLAN_CODEC_REQUIRED');
|
|
57
|
+
const packed = JSON.parse(Buffer.from(decoded.buffer,decoded.byteOffset,decoded.byteLength).toString('utf8'));
|
|
58
|
+
if (canonicalHash(packed) !== value.sha256) throw new Error('STORED_PLAN_CHANGED');
|
|
59
|
+
if (value.d1Plan === 3 && (value.dictionary === 2 ? packed?.version !== 2 : packed?.version !== undefined)) throw new Error('STORED_PLAN_CHANGED');
|
|
60
|
+
const payload = value.d1Plan === 3 ? dictionaryDecode(packed, value.dictionary === 2 ? value.payloadSize : undefined) : packed;
|
|
61
|
+
if (value.d1Plan === 3 && value.dictionary === 2 && canonicalSize(payload) !== value.payloadSize) throw new Error('STORED_PLAN_CHANGED');
|
|
62
|
+
return payload;
|
|
63
|
+
}
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
import { erasedSourcePredicates } from './erasure-claims.ts';
|
|
2
|
+
import { retainPlanIndex } from './erasure-index.ts';
|
|
3
|
+
import { Buffer } from 'node:buffer';
|
|
4
|
+
import { exactRows } from './sets.ts';
|
|
5
|
+
import { planIndex } from './erasure-index.ts';
|
|
6
|
+
import type { ImportPlan } from '../types.ts';
|
|
7
|
+
import type { D1ImportContext, D1PlanCodec, D1Sql } from './types.ts';
|
|
8
|
+
import { planFingerprint } from '../plan/index.ts';
|
|
9
|
+
import { assertPlanSize } from '../plan/size.ts';
|
|
10
|
+
import { assertPlanCredentials } from '../provenance/credentials.ts';
|
|
11
|
+
import { preparedAtomic, guard, sql } from './database.ts';
|
|
12
|
+
import { retainPlan, planChunkReads, creationCodec, LARGE_PLAN_BYTES } from './plans.ts';
|
|
13
|
+
import { preflightStagedCreation, stagedSize } from './staged-creation.ts';
|
|
14
|
+
import { stagingStatements } from './staging-statements.ts';
|
|
15
|
+
import { custodyGuards } from './original-storage.ts';
|
|
16
|
+
import { retainOriginals } from './originals.ts';
|
|
17
|
+
import { checkSql, D1RequestLimitError, valueBytes } from './budget.ts';
|
|
18
|
+
import { policy, policyHash, EMPTY_REVERSAL_HASH } from './undo-state.ts';
|
|
19
|
+
|
|
20
|
+
export type D1CreatePreflightContext = Pick<D1ImportContext, 'destination' | 'namespace' | 'mappingVersion' | 'authorization' | 'reviewedFingerprint' | 'operationKey' | 'reversalPolicy' | 'maxBatchBytes'> & {
|
|
21
|
+
/** The target adapter's plan codec, when it has one: creation retains the plan with it. */
|
|
22
|
+
planCodec?: D1PlanCodec;
|
|
23
|
+
};
|
|
24
|
+
export interface D1CreatePreflight {
|
|
25
|
+
/** Statements in the one atomic job, plan and original-custody batch. */
|
|
26
|
+
mutationStatements: number;
|
|
27
|
+
/** Initial authority batch, operation-key lookup, mutation and return reads. */
|
|
28
|
+
totalExecutions: number;
|
|
29
|
+
maxSqlBytes: number;
|
|
30
|
+
maxBoundBytes: number;
|
|
31
|
+
maxParameters: number;
|
|
32
|
+
}
|
|
33
|
+
/** `credentials: false` re-validates a plan already scanned: its unchanged
|
|
34
|
+
* fingerprint proves unchanged content, so the whole-plan scan is not repeated. */
|
|
35
|
+
export function validateCreation(plan: ImportPlan, context: D1CreatePreflightContext, credentials = true): string {
|
|
36
|
+
// Before any whole-plan encoding (fingerprint, retained body, chunks).
|
|
37
|
+
assertPlanSize(plan);
|
|
38
|
+
if (!context.reviewedFingerprint || context.reviewedFingerprint !== plan.fingerprint || planFingerprint(plan) !== plan.fingerprint) throw new Error('REVIEW_FINGERPRINT_MISMATCH');
|
|
39
|
+
if (plan.version !== 1 || plan.destination !== context.destination || plan.namespace !== context.namespace || plan.mappingVersion !== context.mappingVersion || plan.principalId !== context.authorization.principalId) throw new Error('PLAN_BINDING_MISMATCH');
|
|
40
|
+
if (credentials) assertPlanCredentials(plan);
|
|
41
|
+
policy(context);
|
|
42
|
+
const operationKey = context.operationKey ?? plan.fingerprint;
|
|
43
|
+
if (!operationKey || operationKey.length > 4096) throw new Error('INVALID_OPERATION_KEY');
|
|
44
|
+
return operationKey;
|
|
45
|
+
}
|
|
46
|
+
/** This is also the production serialization path, not an estimate. */
|
|
47
|
+
export function creationStatements(plan: ImportPlan, context: D1CreatePreflightContext, id: string, operationKey: string, retained = retainPlan(plan, id, creationCodec(plan, context.planCodec))): D1Sql[] {
|
|
48
|
+
const jobGuard=guard(sql('EXISTS(SELECT 1 FROM ik_jobs WHERE id=? AND destination=? AND namespace=? AND mapping_version=? AND principal_id=? AND fingerprint=? AND operation_key=? AND plan_json=? AND state=\'queued\' AND checkpoint=0 AND fence=0 AND total=? AND error IS NULL AND lease_token IS NULL AND lease_expires IS NULL AND reversal_policy=? AND reversal_policy_hash=? AND reversal_cursor=0 AND reversal_hash=?)', id, plan.destination, plan.namespace, plan.mappingVersion, plan.principalId, plan.fingerprint, operationKey, retained.body, plan.entries.length, policy(context), policyHash(id, plan.fingerprint, policy(context)), EMPTY_REVERSAL_HASH));
|
|
49
|
+
const statements=[...erasedSourcePredicates(plan).map(guard), sql("INSERT INTO ik_jobs(id,destination,namespace,mapping_version,principal_id,fingerprint,operation_key,plan_json,state,total,reversal_policy,reversal_policy_hash) VALUES(?,?,?,?,?,?,?,?,'queued',?,?,?)", id, plan.destination, plan.namespace, plan.mappingVersion, plan.principalId, plan.fingerprint, operationKey, retained.body, plan.entries.length, policy(context), policyHash(id, plan.fingerprint, policy(context))), ...retained.statements, ...retainOriginals(plan, id, retained.encoded),
|
|
50
|
+
jobGuard,
|
|
51
|
+
guard(sql('(SELECT COUNT(*) FROM ik_d1_creation_plans WHERE job_id=? AND released=0)=? AND (SELECT COUNT(*) FROM ik_d1_plan_chunks WHERE job_id=?)=? AND NOT EXISTS(SELECT 1 FROM ik_d1_creation_plans s WHERE s.job_id=? AND NOT EXISTS(SELECT 1 FROM ik_d1_plan_chunks p WHERE p.job_id=s.job_id AND p.ordinal=s.ordinal AND p.body IS s.body))', id, retained.encoded.chunks.length, id, retained.encoded.chunks.length, id)),
|
|
52
|
+
...custodyGuards(plan, id, retained.encoded), ...retainPlanIndex(plan, id), guard(sql('(SELECT COUNT(*) FROM ik_d1_reversal_reports WHERE job_id=?)=0', id)),
|
|
53
|
+
sql('UPDATE ik_d1_creation_plans SET released=1 WHERE job_id=?', id), sql('DELETE FROM ik_d1_creation_plans WHERE job_id=?', id), guard(sql('(SELECT COUNT(*) FROM ik_d1_creation_plans WHERE job_id=?)=0', id))];
|
|
54
|
+
if(context.maxBatchBytes!==undefined){
|
|
55
|
+
statements[0].finalChecks=[jobGuard,...custodyGuards(plan,id,retained.encoded),
|
|
56
|
+
...exactRows('ik_d1_plan_chunks',['job_id','ordinal','body'],retained.encoded.chunks,sql('job_id=?',id)),
|
|
57
|
+
...exactRows('ik_d1_plan_index',['job_id','ordinal','source_identity','target_identity'],planIndex(plan,id),sql('job_id=?',id)),
|
|
58
|
+
guard(sql('EXISTS(SELECT 1 FROM ik_d1_indexed_jobs WHERE job_id=? AND fingerprint=?) AND NOT EXISTS(SELECT 1 FROM ik_d1_creation_plans WHERE job_id=?) AND NOT EXISTS(SELECT 1 FROM ik_d1_reversal_reports WHERE job_id=?)',id,plan.fingerprint,id,id))];
|
|
59
|
+
}
|
|
60
|
+
return statements;
|
|
61
|
+
}
|
|
62
|
+
/** Pure admission for a NEW job: no database, writes, random IDs or cached
|
|
63
|
+
* authority decisions. The consumer's predicate must itself be pure. Use the
|
|
64
|
+
* exact reviewed plan, operation key and production authorization expression.
|
|
65
|
+
* Current authority is still evaluated inside each production transaction.
|
|
66
|
+
* This sizes custody creation, not the consumer's later native effect batches. */
|
|
67
|
+
export function preflightD1ImportJob(plan: ImportPlan, context: D1CreatePreflightContext): D1CreatePreflight {
|
|
68
|
+
const operationKey = validateCreation(plan, context);
|
|
69
|
+
// UUIDs have identical UTF-8/JSON size. The generated ID is the only value
|
|
70
|
+
// different from the actual creation path.
|
|
71
|
+
// The codec production creation uses: given directly, or the adapter's when a full context is passed.
|
|
72
|
+
const codec = creationCodec(plan, context.planCodec ?? (context as Partial<D1ImportContext>).storage?.targets?.planCodec);
|
|
73
|
+
const placeholder = '00000000-0000-0000-0000-000000000000', retained = retainPlan(plan, placeholder, codec), unbounded = context.maxBatchBytes === undefined;
|
|
74
|
+
// A large plan is created through bounded staging: admit every staged batch,
|
|
75
|
+
// its publication and the prior-generation lookup.
|
|
76
|
+
const staged = unbounded && stagedSize(retained.encoded) > LARGE_PLAN_BYTES ? preflightStagedCreation(plan, context as D1ImportContext, operationKey, retained.encoded, placeholder) : undefined;
|
|
77
|
+
const creation = staged ? staged.publication : creationStatements(plan, context, placeholder, operationKey, retained);
|
|
78
|
+
const statements = staged ? [...staged.batches.flatMap(batch => stagingStatements(context as D1ImportContext, 'create', batch)), ...preparedAtomic(context, 'create', creation)] : preparedAtomic(context, 'create', creation);
|
|
79
|
+
const initial = preparedAtomic(context, 'create', []);
|
|
80
|
+
const reads = [sql('SELECT * FROM ik_jobs WHERE destination=? AND principal_id=? AND operation_key=?', context.destination, context.authorization.principalId, operationKey),
|
|
81
|
+
...(staged ? [sql('SELECT * FROM ik_d1_staging WHERE destination=? AND principal_id=? AND operation_key=?', context.destination, context.authorization.principalId, operationKey)] : []),
|
|
82
|
+
sql('SELECT * FROM ik_jobs WHERE id=?', '00000000-0000-0000-0000-000000000000'),
|
|
83
|
+
sql('SELECT * FROM ik_d1_plan_chunks WHERE job_id IN (SELECT value FROM json_each(?)) ORDER BY job_id,ordinal', JSON.stringify(['00000000-0000-0000-0000-000000000000']))];
|
|
84
|
+
const chunkRead = reads.pop()!;
|
|
85
|
+
for (let n = 0; n < planChunkReads(retained.body, unbounded); n++) reads.push(chunkRead);
|
|
86
|
+
if (policy(context) === 'preserve-conflicts') reads.push(sql('SELECT r.*,p.reversed AS native_reversed FROM ik_d1_reversal_reports r LEFT JOIN ik_provenance p ON p.id=r.effect_id AND p.job_id=r.job_id WHERE r.job_id=? ORDER BY r.position', '00000000-0000-0000-0000-000000000000'));
|
|
87
|
+
const erasedReads = erasedSourcePredicates(plan).map(p => ({ sql: `SELECT ${p.sql} AS clear`, params: p.params }));
|
|
88
|
+
const all = [...initial, ...erasedReads, ...statements, ...reads];
|
|
89
|
+
for (const statement of all) checkSql(statement);
|
|
90
|
+
if (all.length > 1000) throw new D1RequestLimitError();
|
|
91
|
+
return { mutationStatements: statements.length, totalExecutions: all.length,
|
|
92
|
+
maxSqlBytes: Math.max(...all.map(s => Buffer.byteLength(s.sql))),
|
|
93
|
+
maxBoundBytes: Math.max(...all.map(s => (s.params ?? []).reduce((n, p) => n + valueBytes(p), 0))),
|
|
94
|
+
maxParameters: Math.max(...all.map(s => s.params?.length ?? 0)) };
|
|
95
|
+
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import type { D1ImportContext, D1Sql } from './types.ts';
|
|
2
|
+
import { guard, sql } from './database.ts';
|
|
3
|
+
import { jsonChunks } from './sets.ts';
|
|
4
|
+
|
|
5
|
+
/** Captures the destination's job custody revisions before any custody is read.
|
|
6
|
+
* Every write to a job's custody (its job row, plan chunks, originals, claims,
|
|
7
|
+
* provenance, history, reports, journal, decisions and indexed tombstones)
|
|
8
|
+
* advances its `ik_d1_custody_states` revision in the writing transaction; the
|
|
9
|
+
* revision never decreases and leaves only with its job. So an unchanged
|
|
10
|
+
* revision in the final read proves that every row of that job read in between
|
|
11
|
+
* came from one state, at a constant cost per job instead of its whole plan. */
|
|
12
|
+
export async function captureCustody(context: D1ImportContext): Promise<(jobs: Iterable<string>) => D1Sql[]> {
|
|
13
|
+
const grant = context.authorization.predicate('read', context.destination);
|
|
14
|
+
const { results } = await context.storage.database.prepare(`SELECT j.id,s.revision FROM ik_jobs j LEFT JOIN ik_d1_custody_states s ON s.job_id=j.id WHERE j.destination=? AND (${grant.sql})`)
|
|
15
|
+
.bind(context.destination, ...grant.params ?? []).all<{ id: string; revision: number | null }>();
|
|
16
|
+
const captured = new Map(results.map(row => [row.id, row.revision]));
|
|
17
|
+
return jobs => {
|
|
18
|
+
const fenced = [...new Set(jobs)].sort().map(id => {
|
|
19
|
+
// A job created after the capture was read without a fence.
|
|
20
|
+
if (!captured.has(id)) throw new Error('IMPORT_ATOMIC_GUARD');
|
|
21
|
+
const revision = captured.get(id);
|
|
22
|
+
if (!Number.isSafeInteger(revision) || revision! < 0) throw new Error('IMPORT_CUSTODY_STATE_REQUIRED');
|
|
23
|
+
return [id, revision];
|
|
24
|
+
});
|
|
25
|
+
return jsonChunks(fenced).map(body => guard(sql("NOT EXISTS(SELECT 1 FROM json_each(?) e WHERE NOT EXISTS(SELECT 1 FROM ik_d1_custody_states s WHERE s.job_id IS json_extract(e.value,'$[0]') AND s.revision IS json_extract(e.value,'$[1]')))", body)));
|
|
26
|
+
};
|
|
27
|
+
}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
/** Every table holding a job's custody, with the column naming its job. */
|
|
2
|
+
const custodyTables: [table: string, column: string][] = [['ik_jobs','id'],...['ik_d1_plan_chunks','ik_d1_creation_plans','ik_d1_plan_index','ik_originals','ik_d1_original_chunks','ik_original_sets','ik_d1_indexed_jobs','ik_d1_reversal_reports','ik_provenance','ik_sources','ik_d1_entries','ik_d1_history','ik_batches','ik_d1_stream_pages'].map((table):[string,string]=>[table,'job_id'])];
|
|
3
|
+
/** Tombstone tables, with the plan index column through which they reach their jobs. */
|
|
4
|
+
const tombstoneTables: [table: string, column: string][] = [['ik_d1_erased_sources','source_identity'],['ik_d1_erased_targets','target_identity']];
|
|
5
|
+
|
|
6
|
+
/** A compact monotonic snapshot fence for bounded multi-job privacy work.
|
|
7
|
+
* It covers all custody fields, independently of the plan revision used by
|
|
8
|
+
* ordinary apply. Backfilling never rewrites existing public custody. */
|
|
9
|
+
export const d1CustodyStateSchema = [
|
|
10
|
+
'CREATE TABLE IF NOT EXISTS ik_d1_custody_states (job_id TEXT PRIMARY KEY REFERENCES ik_jobs(id) ON DELETE CASCADE,revision INTEGER NOT NULL DEFAULT 0)',
|
|
11
|
+
'INSERT OR IGNORE INTO ik_d1_custody_states(job_id) SELECT id FROM ik_jobs',
|
|
12
|
+
"CREATE TRIGGER IF NOT EXISTS ik_d1_custody_state_monotonic BEFORE UPDATE ON ik_d1_custody_states WHEN NEW.job_id IS NOT OLD.job_id OR NEW.revision<=OLD.revision BEGIN SELECT RAISE(ABORT,'IMPORT_CUSTODY_STATE_CHANGED'); END",
|
|
13
|
+
"CREATE TRIGGER IF NOT EXISTS ik_d1_custody_state_delete BEFORE DELETE ON ik_d1_custody_states WHEN EXISTS(SELECT 1 FROM ik_jobs WHERE id=OLD.job_id) BEGIN SELECT RAISE(ABORT,'IMPORT_CUSTODY_STATE_CHANGED'); END",
|
|
14
|
+
"CREATE TRIGGER IF NOT EXISTS ik_d1_custody_state_publication BEFORE UPDATE ON ik_d1_custody_states WHEN EXISTS(SELECT 1 FROM ik_d1_publication_fences f JOIN ik_d1_custody_revisions r ON r.job_id=f.job_id WHERE (f.job_id=OLD.job_id OR f.job_id=NEW.job_id) AND r.seal IS NOT NULL) BEGIN SELECT RAISE(ABORT,'IMPORT_PUBLICATION_CHANGED'); END",
|
|
15
|
+
'CREATE TRIGGER IF NOT EXISTS ik_d1_custody_state_receipt AFTER UPDATE ON ik_d1_custody_states BEGIN UPDATE ik_d1_custody_revisions SET seal=NULL WHERE job_id=NEW.job_id; END',
|
|
16
|
+
'CREATE TRIGGER IF NOT EXISTS ik_d1_custody_state_create AFTER INSERT ON ik_jobs BEGIN INSERT INTO ik_d1_custody_states(job_id) VALUES(NEW.id); END',
|
|
17
|
+
...custodyTables.flatMap(([table,column])=>['INSERT','UPDATE','DELETE'].map(event=>{
|
|
18
|
+
const refs=event==='INSERT'?`job_id=NEW.${column}`:event==='DELETE'?`job_id=OLD.${column}`:`job_id=OLD.${column} OR job_id=NEW.${column}`;
|
|
19
|
+
return `CREATE TRIGGER IF NOT EXISTS ik_d1_custody_state_${table}_${event.toLowerCase()} AFTER ${event} ON ${table} BEGIN UPDATE ik_d1_custody_states SET revision=revision+1 WHERE ${refs}; END`;
|
|
20
|
+
})),
|
|
21
|
+
...['UPDATE','DELETE'].map(event=>`CREATE TRIGGER IF NOT EXISTS ik_d1_custody_state_stream_${event.toLowerCase()} BEFORE ${event} ON ik_d1_streams BEGIN UPDATE ik_d1_custody_states SET revision=revision+1 WHERE job_id IN (SELECT job_id FROM ik_d1_stream_pages WHERE stream_id=OLD.id); END`),
|
|
22
|
+
// Tombstones participate in a SELECT-only multi-batch custody snapshot too.
|
|
23
|
+
...tombstoneTables.flatMap(([table,column])=>['INSERT','UPDATE','DELETE'].map(event=>{
|
|
24
|
+
const match=(row:string)=>`i.${column}=${row}.identity AND j.destination=${row}.destination`;
|
|
25
|
+
const predicate=event==='INSERT'?match('NEW'):event==='DELETE'?match('OLD'):`(${match('OLD')}) OR (${match('NEW')})`;
|
|
26
|
+
return `CREATE TRIGGER IF NOT EXISTS ik_d1_custody_state_${table}_${event.toLowerCase()} AFTER ${event} ON ${table} BEGIN UPDATE ik_d1_custody_states SET revision=revision+1 WHERE job_id IN (SELECT i.job_id FROM ik_d1_plan_index i JOIN ik_jobs j ON j.id=i.job_id WHERE ${predicate}); END`;
|
|
27
|
+
})),
|
|
28
|
+
];
|
|
29
|
+
|
|
30
|
+
/** Owners of the rows an insert or update would replace through a key other than its own job's.
|
|
31
|
+
* SQLite's rowid is a unique key of every one of these tables: an insert or update that names an
|
|
32
|
+
* existing rowid replaces that row too, so each table also matches the colliding row by rowid. */
|
|
33
|
+
const byRowid = (table: string, column = 'job_id') => `SELECT ${column} FROM ${table} WHERE rowid=NEW.rowid`;
|
|
34
|
+
const namedKeyOwners: [table: string, owners: string][] = [
|
|
35
|
+
['ik_sources', `SELECT job_id FROM ik_sources WHERE identity=NEW.identity UNION ALL ${byRowid('ik_sources')}`],
|
|
36
|
+
['ik_provenance', `SELECT job_id FROM ik_provenance WHERE id=NEW.id UNION ALL ${byRowid('ik_provenance')} UNION ALL SELECT job_id FROM ik_provenance WHERE NEW.reversed=0 AND reversed=0 AND destination=NEW.destination AND target_kind=NEW.target_kind AND target_id=NEW.target_id`],
|
|
37
|
+
['ik_d1_history', `SELECT job_id FROM ik_d1_history WHERE id=NEW.id UNION ALL ${byRowid('ik_d1_history')} UNION ALL SELECT job_id FROM ik_d1_history WHERE successor_id=NEW.successor_id`],
|
|
38
|
+
['ik_d1_stream_pages', `SELECT job_id FROM ik_d1_stream_pages WHERE stream_id=NEW.stream_id AND ordinal=NEW.ordinal UNION ALL ${byRowid('ik_d1_stream_pages')}`],
|
|
39
|
+
['ik_d1_streams', 'SELECT job_id FROM ik_d1_stream_pages WHERE stream_id=NEW.id OR stream_id IN (SELECT id FROM ik_d1_streams WHERE rowid=NEW.rowid) UNION ALL SELECT p.job_id FROM ik_d1_streams s JOIN ik_d1_stream_pages p ON p.stream_id=s.id WHERE s.destination=NEW.destination AND s.principal_id=NEW.principal_id AND s.operation_key=NEW.operation_key'],
|
|
40
|
+
];
|
|
41
|
+
const replacedOwners: [table: string, owners: string][] = [
|
|
42
|
+
...namedKeyOwners,
|
|
43
|
+
// Every other custody table by rowid; then each tombstone by rowid, its owners found through the plan index as its AFTER triggers find them.
|
|
44
|
+
...custodyTables.filter(([table]) => !namedKeyOwners.some(([named]) => named === table)).map(([table, column]): [string, string] => [table, byRowid(table, column)]),
|
|
45
|
+
...tombstoneTables.map(([table, column]): [string, string] => [table, `SELECT i.job_id FROM ${table} e JOIN ik_jobs j ON j.destination=e.destination JOIN ik_d1_plan_index i ON i.job_id=j.id AND i.${column}=e.identity WHERE e.rowid=NEW.rowid`]),
|
|
46
|
+
];
|
|
47
|
+
|
|
48
|
+
/** Additive step after `d1CustodyStateSchema` and `d1RetirementSchema`. SQLite's
|
|
49
|
+
* REPLACE deletes the row it conflicts with without firing DELETE triggers, so
|
|
50
|
+
* without it one statement could rewind a revision or move custody between jobs
|
|
51
|
+
* while every revision the persisted read fenced stays the same.
|
|
52
|
+
* - An insert over an existing counter, by job id or by rowid, is ignored; the
|
|
53
|
+
* backfill's INSERT OR IGNORE still adds missing ones. A counter's rowid never
|
|
54
|
+
* changes (`IMPORT_CUSTODY_STATE_CHANGED`).
|
|
55
|
+
* - A job id that exists or was retired is never inserted again (`IMPORT_JOB_ID_REUSED`).
|
|
56
|
+
* - An insert or update that would replace a claim, provenance, history, stream
|
|
57
|
+
* page or stream row by a named key, or any custody or tombstone row by its
|
|
58
|
+
* rowid, first advances the revision of every job owning the replaced row
|
|
59
|
+
* (`UPDATE OR REPLACE` deletes the row its new key collides with the same way).
|
|
60
|
+
* An ordinary write replaces nothing beyond its own row, whose job its AFTER
|
|
61
|
+
* triggers advance anyway, and a refused one rolls the advance back. */
|
|
62
|
+
export const d1CustodyStateReplaceSchema = [
|
|
63
|
+
'CREATE TRIGGER IF NOT EXISTS ik_d1_custody_state_insert BEFORE INSERT ON ik_d1_custody_states WHEN EXISTS(SELECT 1 FROM ik_d1_custody_states WHERE job_id=NEW.job_id OR rowid=NEW.rowid) BEGIN SELECT RAISE(IGNORE); END',
|
|
64
|
+
"CREATE TRIGGER IF NOT EXISTS ik_d1_custody_state_rowid BEFORE UPDATE ON ik_d1_custody_states WHEN NEW.rowid IS NOT OLD.rowid BEGIN SELECT RAISE(ABORT,'IMPORT_CUSTODY_STATE_CHANGED'); END",
|
|
65
|
+
"CREATE TRIGGER IF NOT EXISTS ik_d1_custody_state_job_reuse BEFORE INSERT ON ik_jobs WHEN EXISTS(SELECT 1 FROM ik_jobs WHERE id=NEW.id) OR EXISTS(SELECT 1 FROM ik_d1_retirements WHERE id=NEW.id AND kind='job') BEGIN SELECT RAISE(ABORT,'IMPORT_JOB_ID_REUSED'); END",
|
|
66
|
+
...[['INSERT', 'replace'], ['UPDATE', 'update_replace']].flatMap(([event, name]) => replacedOwners.map(([table, owners]) => `CREATE TRIGGER IF NOT EXISTS ik_d1_custody_state_${table}_${name} BEFORE ${event} ON ${table} BEGIN UPDATE ik_d1_custody_states SET revision=revision+1 WHERE job_id IN (${owners}); END`)),
|
|
67
|
+
];
|