@runeditrun/rer 0.2.76 → 0.2.78
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -0
- package/cli/README.md +55 -0
- package/cli/contract.mjs +1 -1
- package/cli/deploy-command.mjs +4 -2
- package/cli/deploy-inputs.mjs +15 -3
- package/cli/deploy-installation-key.mjs +71 -0
- package/cli/deploy-secrets.mjs +2 -0
- package/cli/deploy.mjs +13 -2
- package/cli/deployment-record.mjs +2 -0
- package/cli/export.mjs +1 -1
- package/cli/main.mjs +1 -1
- package/cli/parse.mjs +3 -2
- package/cli/portability-migrations.mjs +35 -14
- package/cli/portability-worker.mjs +2 -2
- package/cli/portability.mjs +2 -2
- package/cli/runtime-front-door.mjs +1 -1
- package/global/CHANGELOG.md +14 -0
- package/global/README.md +32 -0
- package/modules/dependencies/README.md +71 -1
- package/modules/dependencies/auth.mjs +4 -4
- package/modules/dependencies/crypto.mjs +6 -1
- package/modules/dependencies/custody.mjs +16 -0
- package/modules/dependencies/index.d.mts +1 -1
- package/modules/dependencies/index.mjs +6 -5
- package/modules/dependencies/schema/keys-sealed.sql +4 -0
- package/modules/dependencies/sealing.mjs +55 -0
- package/modules/dependencies/store.mjs +62 -9
- package/modules/deploy-kit/README.md +1 -0
- package/modules/deploy-kit/owner-cli.mjs +19 -7
- package/modules/deploy-kit/owner-rekey.mjs +69 -0
- package/modules/deploy-kit/provider.mjs +9 -4
- package/modules/import-kit/src/d1/README.md +1322 -0
- package/modules/import-kit/src/d1/apply.ts +115 -0
- package/modules/import-kit/src/d1/budget.ts +89 -0
- package/modules/import-kit/src/d1/compressed-plan.ts +63 -0
- package/modules/import-kit/src/d1/creation.ts +95 -0
- package/modules/import-kit/src/d1/custody-fence.ts +27 -0
- package/modules/import-kit/src/d1/custody-state-schema.ts +67 -0
- package/modules/import-kit/src/d1/database.ts +163 -0
- package/modules/import-kit/src/d1/entry-locator.ts +61 -0
- package/modules/import-kit/src/d1/erased-artifacts.ts +38 -0
- package/modules/import-kit/src/d1/erased-custody.ts +36 -0
- package/modules/import-kit/src/d1/erased-plan.ts +47 -0
- package/modules/import-kit/src/d1/erased-reversal.ts +19 -0
- package/modules/import-kit/src/d1/erasure-bounded.ts +22 -0
- package/modules/import-kit/src/d1/erasure-claims.ts +27 -0
- package/modules/import-kit/src/d1/erasure-compiler.ts +67 -0
- package/modules/import-kit/src/d1/erasure-disposition.ts +16 -0
- package/modules/import-kit/src/d1/erasure-fragments.ts +86 -0
- package/modules/import-kit/src/d1/erasure-index.ts +53 -0
- package/modules/import-kit/src/d1/erasure-pages.ts +62 -0
- package/modules/import-kit/src/d1/erasure-progress-schema.ts +8 -0
- package/modules/import-kit/src/d1/erasure-progress.ts +61 -0
- package/modules/import-kit/src/d1/erasure-related.ts +124 -0
- package/modules/import-kit/src/d1/erasure-report.ts +31 -0
- package/modules/import-kit/src/d1/erasure-schema.ts +24 -0
- package/modules/import-kit/src/d1/erasure-selection.ts +77 -0
- package/modules/import-kit/src/d1/erasure-snapshot.ts +15 -0
- package/modules/import-kit/src/d1/erasure-stage.ts +146 -0
- package/modules/import-kit/src/d1/erasure-step.ts +12 -0
- package/modules/import-kit/src/d1/erasure-superseded.ts +98 -0
- package/modules/import-kit/src/d1/erasure-values.ts +83 -0
- package/modules/import-kit/src/d1/erasure-writes.ts +46 -0
- package/modules/import-kit/src/d1/erasure.ts +170 -0
- package/modules/import-kit/src/d1/evolutions/0001-preserved-undo.ts +13 -0
- package/modules/import-kit/src/d1/expected-state.ts +91 -0
- package/modules/import-kit/src/d1/field-redaction.ts +86 -0
- package/modules/import-kit/src/d1/guards.ts +37 -0
- package/modules/import-kit/src/d1/history.ts +71 -0
- package/modules/import-kit/src/d1/index.ts +88 -0
- package/modules/import-kit/src/d1/integrity.ts +127 -0
- package/modules/import-kit/src/d1/inventory-snapshot.ts +84 -0
- package/modules/import-kit/src/d1/jobs.ts +97 -0
- package/modules/import-kit/src/d1/native-set.ts +48 -0
- package/modules/import-kit/src/d1/native-staging-admission.ts +19 -0
- package/modules/import-kit/src/d1/native-staging-context.ts +44 -0
- package/modules/import-kit/src/d1/native-staging-lifecycle.ts +33 -0
- package/modules/import-kit/src/d1/native-staging-offset.ts +74 -0
- package/modules/import-kit/src/d1/native-staging-publication.ts +18 -0
- package/modules/import-kit/src/d1/native-staging-rows.ts +72 -0
- package/modules/import-kit/src/d1/native-staging-schema.ts +28 -0
- package/modules/import-kit/src/d1/native-staging-state.ts +22 -0
- package/modules/import-kit/src/d1/native-staging.ts +64 -0
- package/modules/import-kit/src/d1/original-copies.ts +53 -0
- package/modules/import-kit/src/d1/original-storage.ts +56 -0
- package/modules/import-kit/src/d1/originals.ts +74 -0
- package/modules/import-kit/src/d1/plan-dictionary.ts +80 -0
- package/modules/import-kit/src/d1/plans.ts +121 -0
- package/modules/import-kit/src/d1/preparation-fingerprint.ts +57 -0
- package/modules/import-kit/src/d1/preparation-header.ts +33 -0
- package/modules/import-kit/src/d1/preparation-reader.ts +98 -0
- package/modules/import-kit/src/d1/preparation-schema.ts +16 -0
- package/modules/import-kit/src/d1/preparation-state.ts +56 -0
- package/modules/import-kit/src/d1/preparation-step-combine.ts +80 -0
- package/modules/import-kit/src/d1/preparation-step-control.ts +48 -0
- package/modules/import-kit/src/d1/preparation-step-hash.ts +131 -0
- package/modules/import-kit/src/d1/preparation-step-pages.ts +77 -0
- package/modules/import-kit/src/d1/preparation-steps-schema.ts +11 -0
- package/modules/import-kit/src/d1/preparation-steps.ts +102 -0
- package/modules/import-kit/src/d1/preparations.ts +87 -0
- package/modules/import-kit/src/d1/preserved-preview.ts +89 -0
- package/modules/import-kit/src/d1/preserved-reverse.ts +91 -0
- package/modules/import-kit/src/d1/provenance-set.ts +87 -0
- package/modules/import-kit/src/d1/publication-fence.ts +22 -0
- package/modules/import-kit/src/d1/publication-receipt-schema.ts +15 -0
- package/modules/import-kit/src/d1/published-stream.ts +11 -0
- package/modules/import-kit/src/d1/read-fence.ts +64 -0
- package/modules/import-kit/src/d1/read-results.ts +16 -0
- package/modules/import-kit/src/d1/redacted-custody.ts +76 -0
- package/modules/import-kit/src/d1/redaction-step.ts +125 -0
- package/modules/import-kit/src/d1/referenced-plans.ts +136 -0
- package/modules/import-kit/src/d1/retained-values.ts +35 -0
- package/modules/import-kit/src/d1/retirement-bounded.ts +71 -0
- package/modules/import-kit/src/d1/retirement-door.ts +105 -0
- package/modules/import-kit/src/d1/retirement-schema.ts +39 -0
- package/modules/import-kit/src/d1/retirement-scope.ts +94 -0
- package/modules/import-kit/src/d1/retirement-steps.ts +76 -0
- package/modules/import-kit/src/d1/retirement-tally.ts +74 -0
- package/modules/import-kit/src/d1/retirement.ts +136 -0
- package/modules/import-kit/src/d1/reversal-conversion-record.ts +19 -0
- package/modules/import-kit/src/d1/reversal-conversion-schema.ts +11 -0
- package/modules/import-kit/src/d1/reversal-conversion.ts +40 -0
- package/modules/import-kit/src/d1/reversal-fence-schema.ts +14 -0
- package/modules/import-kit/src/d1/reversal-fence.ts +106 -0
- package/modules/import-kit/src/d1/reversal-journal.ts +24 -0
- package/modules/import-kit/src/d1/reverse.ts +103 -0
- package/modules/import-kit/src/d1/reviewed-source.ts +23 -0
- package/modules/import-kit/src/d1/sets.ts +78 -0
- package/modules/import-kit/src/d1/staged-creation.ts +201 -0
- package/modules/import-kit/src/d1/staging-admission.ts +45 -0
- package/modules/import-kit/src/d1/staging-control.ts +43 -0
- package/modules/import-kit/src/d1/staging-erasure.ts +28 -0
- package/modules/import-kit/src/d1/staging-lifecycle.ts +49 -0
- package/modules/import-kit/src/d1/staging-plan.ts +15 -0
- package/modules/import-kit/src/d1/staging-privacy.ts +22 -0
- package/modules/import-kit/src/d1/staging-publication.ts +79 -0
- package/modules/import-kit/src/d1/staging-schema.ts +37 -0
- package/modules/import-kit/src/d1/staging-statements.ts +8 -0
- package/modules/import-kit/src/d1/staging-transport.ts +70 -0
- package/modules/import-kit/src/d1/stream-boundary.ts +11 -0
- package/modules/import-kit/src/d1/stream-create.ts +55 -0
- package/modules/import-kit/src/d1/stream-identity-admission.ts +37 -0
- package/modules/import-kit/src/d1/stream-input.ts +18 -0
- package/modules/import-kit/src/d1/stream-layout.ts +43 -0
- package/modules/import-kit/src/d1/stream-origin.ts +44 -0
- package/modules/import-kit/src/d1/stream-page-integrity.ts +18 -0
- package/modules/import-kit/src/d1/stream-partition.ts +50 -0
- package/modules/import-kit/src/d1/stream-plan.ts +133 -0
- package/modules/import-kit/src/d1/stream-receipt.ts +45 -0
- package/modules/import-kit/src/d1/stream-revision-schema.ts +35 -0
- package/modules/import-kit/src/d1/stream-snapshot.ts +26 -0
- package/modules/import-kit/src/d1/stream-stage.ts +79 -0
- package/modules/import-kit/src/d1/stream-state.ts +25 -0
- package/modules/import-kit/src/d1/stream-working-set.ts +30 -0
- package/modules/import-kit/src/d1/streams.ts +39 -0
- package/modules/import-kit/src/d1/target-guards.ts +9 -0
- package/modules/import-kit/src/d1/target-identities.ts +17 -0
- package/modules/import-kit/src/d1/transport.ts +10 -0
- package/modules/import-kit/src/d1/types.ts +162 -0
- package/modules/import-kit/src/d1/undo-observations.ts +39 -0
- package/modules/import-kit/src/d1/undo-report.ts +22 -0
- package/modules/import-kit/src/d1/undo-snapshot.ts +87 -0
- package/modules/import-kit/src/d1/undo-state.ts +84 -0
- package/modules/import-kit/src/d1/undo-types.ts +110 -0
- package/modules/import-kit/src/journal/authority.ts +12 -0
- package/modules/import-kit/src/plan/budget.ts +60 -0
- package/modules/import-kit/src/plan/index.ts +123 -0
- package/modules/import-kit/src/plan/size.ts +122 -0
- package/modules/import-kit/src/provenance/canonical-hash.ts +130 -0
- package/modules/import-kit/src/provenance/canonical-stream.ts +85 -0
- package/modules/import-kit/src/provenance/credential-names.ts +131 -0
- package/modules/import-kit/src/provenance/credentials.ts +317 -0
- package/modules/import-kit/src/provenance/identity.ts +51 -0
- package/modules/import-kit/src/provenance/json-snapshot.ts +21 -0
- package/modules/import-kit/src/provenance/resumable-sha256.ts +75 -0
- package/modules/import-kit/src/provenance/validate-json.ts +19 -0
- package/modules/import-kit/src/storage/originals.ts +87 -0
- package/modules/import-kit/src/storage/provenance.ts +64 -0
- package/modules/import-kit/src/storage/schema.ts +42 -0
- package/modules/import-kit/src/types.ts +47 -0
- package/modules/operator-auth/docs/native-accounts.md +4 -1
- package/modules/operator-auth/identity/oidc/aes-gcm.mjs +21 -0
- package/modules/operator-auth/identity/oidc/crypto.mjs +5 -12
- package/modules/operator-auth/index.d.mts +1 -1
- package/modules/portability/README.md +98 -2
- package/modules/portability/backfills.mjs +42 -15
- package/modules/portability/ledgers.mjs +7 -1
- package/modules/portability/transient-tables.mjs +40 -0
- package/onboarding/local/environment.mjs +2 -2
- package/package.json +3 -1
- package/scripts/publish-check/reviewed.py +3 -1
|
@@ -0,0 +1,1322 @@
|
|
|
1
|
+
# Cloudflare D1 port
|
|
2
|
+
|
|
3
|
+
To bring existing data into a seed, sign in as the owner and use its import
|
|
4
|
+
screen. First step: open this path on the destination app:
|
|
5
|
+
|
|
6
|
+
```text
|
|
7
|
+
/app/import
|
|
8
|
+
```
|
|
9
|
+
|
|
10
|
+
Upload a supported source export, review the plan and its differences, then
|
|
11
|
+
approve that exact plan to apply it. Reconcile the result against native data;
|
|
12
|
+
use Undo from the same history when needed. Base's reference accepts people JSON
|
|
13
|
+
here, not installation `.rer` archives. Editions adopt this same door
|
|
14
|
+
with their own supported readers and native adapters.
|
|
15
|
+
|
|
16
|
+
For seed integration, follow [the door adoption guide](../door/ADOPTING.md)
|
|
17
|
+
first. The D1 primitives below back that shared pipeline; do not mount separate
|
|
18
|
+
job creation/apply routes beside it. Keep native readers and target adapters.
|
|
19
|
+
|
|
20
|
+
Import `@runeditrun/rer/modules/import-kit/d1` from a supplying rer release
|
|
21
|
+
after 0.2.77 in TypeScript Workers with `nodejs_compat` (or `src/d1/index.ts`
|
|
22
|
+
when working in Base source). The package carries TypeScript source for the
|
|
23
|
+
Worker bundler and includes the shared D1 transport. This entry point does
|
|
24
|
+
not import `node:sqlite` or the optional file/API readers. Products that need the
|
|
25
|
+
readers can explicitly import `src/readers/index.ts`. The synchronous Node API and its guarantees remain
|
|
26
|
+
unchanged. D1 uses the same reviewed plans, source identities, retained originals,
|
|
27
|
+
provenance and inert-effect policy through an asynchronous native SQL port.
|
|
28
|
+
|
|
29
|
+
When copying Base source directly, copy the shared [D1 transport](../../../d1-transport/index.mjs) and its
|
|
30
|
+
[types](../../../d1-transport/index.d.mts), preserving their sibling location
|
|
31
|
+
beside the import-kit directory.
|
|
32
|
+
The D1 entry uses this shared transport automatically. It accounts for actual
|
|
33
|
+
UTF-8 bytes and workerd's whole-call two-byte strings, escapes sparse wide text
|
|
34
|
+
for SQLite to decode, and keeps each mutation in its original transaction.
|
|
35
|
+
Retained plans are read in bounded pages, including their reserved return-query
|
|
36
|
+
budget. Calls that still exceed the provider transport ceiling fail explicitly.
|
|
37
|
+
|
|
38
|
+
Apply `d1Schema`, `d1ErasureSchema`, `d1StreamOriginSchema`,
|
|
39
|
+
`d1LegacyStreamReceiptUpgrade`, `d1StagingSchema`,
|
|
40
|
+
`d1NativeStagingSchema`, `d1CustodyStateSchema`, `d1PreparationSchema`, `d1ReversalFenceSchema`, `d1RetirementSchema` and `d1CustodyStateReplaceSchema` in product-owned numbered migrations, or call
|
|
41
|
+
`initializeD1ImportStorage(database)` during setup. The schema includes the common
|
|
42
|
+
kit tables and D1 ownership/assertion tables. Do not mix the synchronous engine
|
|
43
|
+
and D1 engine for the same jobs: each engine owns its checkpoint protocol.
|
|
44
|
+
For reviewed reversal-policy conversion, also apply the additive
|
|
45
|
+
`d1ReversalPolicyConversionSchema` after these existing schemas in a new
|
|
46
|
+
product-owned numbered migration. `initializeD1ImportStorage` installs it
|
|
47
|
+
automatically. Do not rewrite an applied migration.
|
|
48
|
+
|
|
49
|
+
Apply stream-origin backfill and legacy receipt conversion in one transaction.
|
|
50
|
+
Origins record actual page, parent and job bindings and remain immutable until
|
|
51
|
+
the job is deleted. Missing or substituted parents cannot turn a known stream
|
|
52
|
+
child into an ordinary job. Ordinary version-three custody remains usable;
|
|
53
|
+
neither a nullable reversal-policy hash nor an operation-key shape identifies a
|
|
54
|
+
stream. Legacy custody whose entire structural origin was already lost cannot
|
|
55
|
+
be distinguished from ordinary custody by this upgrade.
|
|
56
|
+
|
|
57
|
+
A `D1ImportContext` contains the destination, namespace, mappingVersion,
|
|
58
|
+
`storage: { database, targets }`, and
|
|
59
|
+
`authorization: { principalId, predicate(action, destination) }`.
|
|
60
|
+
`predicate` must return a scalar SQL boolean expression and bound `params` that
|
|
61
|
+
checks the current session, actor, destination grant and expiry in D1. Returning
|
|
62
|
+
a cached boolean or constant SQL permission defeats this contract. The engine
|
|
63
|
+
evaluates the expression at both ends of every atomic mutation batch. A false
|
|
64
|
+
predicate raises `IMPORT_ATOMIC_GUARD` and rolls back every native and kit write.
|
|
65
|
+
Read operations also require current authority. Jobs remain bound to their actor.
|
|
66
|
+
The plan and its fingerprint carry that `principalId`, so a job can never be rebound
|
|
67
|
+
to another one, and an archive carries the jobs but not the operators who created
|
|
68
|
+
them: after an export, reset, import and setup the owner is a new operator row. A
|
|
69
|
+
product that wants restored jobs to stay usable passes a principal that survives a
|
|
70
|
+
restore, such as the operator's role (`'role:' + role`), and lets `predicate` check
|
|
71
|
+
that the acting operator is still active in that role. A product that passes the
|
|
72
|
+
operator's row id keeps its jobs bound to that operator for good.
|
|
73
|
+
|
|
74
|
+
`readD1ImportJob`, `readD1ImportOriginals`, `readD1PersistedImport` and
|
|
75
|
+
`readD1ImportUndoReport` execute SELECT-only SQL, including their initial authority
|
|
76
|
+
check and final atomic fence. Their guards retain scalar predicate data and pack
|
|
77
|
+
balanced `SELECT CASE WHEN (...) THEN 1 ELSE 0 END AS ok` statements into one
|
|
78
|
+
D1 batch. Every scalar result must pass before any candidate is returned; false,
|
|
79
|
+
NULL, missing, malformed or failed guard results reject with `IMPORT_ATOMIC_GUARD`.
|
|
80
|
+
The final fence still covers current authority, exact binding/plan/custody,
|
|
81
|
+
inventory and full report/provenance/native snapshots at the existing boundaries.
|
|
82
|
+
Bounded job, original and report reads split oversized custody checks into
|
|
83
|
+
SELECT-only batches at the smaller of the caller limit and 2 MiB. Each batch
|
|
84
|
+
checks the same monotonic custody revisions, including tombstone changes;
|
|
85
|
+
the original plan revision is captured before decoding. This requires the
|
|
86
|
+
additive custody-state schema. Ordinary reads without a transport policy retain
|
|
87
|
+
their existing atomic admission. `readD1PersistedImport` keeps one complete
|
|
88
|
+
atomic native snapshot capped at 2 MiB even when its caller omits a limit. It
|
|
89
|
+
reads each destination job's `ik_d1_custody_states` revision (the same additive
|
|
90
|
+
schema) before any custody and fences those revisions instead of resending
|
|
91
|
+
plans, so its read grows by one revision per job, not by a plan per catch-up.
|
|
92
|
+
Under that fence it checks each owner job by its stored plan header and
|
|
93
|
+
fingerprint and returns only the plan entries its live provenance references,
|
|
94
|
+
parsed inside D1: from an inline plan, or from the run of whole entries a chunked
|
|
95
|
+
plan's entry locator names (the `entries` member of its `d1Plan` 1 envelope,
|
|
96
|
+
which every full plan load checks against the chunks). No SQL value grows past
|
|
97
|
+
one run, so each owner job costs fixed statements and bytes however long a
|
|
98
|
+
catch-up chain grows. A plan without a locator (compressed, staged, or stored
|
|
99
|
+
before locators) and a preserved job with a reversal report still decode their
|
|
100
|
+
whole plan. An oversized snapshot raises
|
|
101
|
+
`ImportSizeLimitError` (its message is the size code) naming `targetKinds`.
|
|
102
|
+
Reads change neither tables nor SQLite `total_changes()` and work with SQLite
|
|
103
|
+
`PRAGMA query_only=ON`. Mutation batches retain inside-transaction CHECK assertions
|
|
104
|
+
and rollback. Planning uses its existing `plan` authorization path. The ordinary-job SELECT fence needs no migration; the bounded stream receipt
|
|
105
|
+
requires the additive stream revisions described below. The assertion table
|
|
106
|
+
remains required for writes.
|
|
107
|
+
|
|
108
|
+
`readD1ImportStream` reads its parent, ordered pages and ordered child statuses
|
|
109
|
+
in one SELECT-only batch. It checks current `read` authority on both sides of
|
|
110
|
+
that snapshot. Creation/replay uses `create`; resume uses `apply`; undo uses
|
|
111
|
+
`undo` and also checks the selected child's reversal action. Each return is
|
|
112
|
+
admitted using that same builder before a child can publish. SQL inputs contain
|
|
113
|
+
only scalar identity and revision bindings, never a JSON copy of every page or
|
|
114
|
+
job. The complete batch includes SQL, JSON escaping and authority parameters in
|
|
115
|
+
the existing 2 MiB ceiling (or a lower selected `maxBatchBytes`). Output arrays
|
|
116
|
+
still grow with page count; this is not a total Worker-memory qualification.
|
|
117
|
+
|
|
118
|
+
The additive `ik_d1_stream_revisions` schema is included at the end of the existing
|
|
119
|
+
`d1StreamOriginSchema` export, so the migration export order remains unchanged.
|
|
120
|
+
Install it in a new numbered migration; do not rewrite an applied migration.
|
|
121
|
+
Initialization captures existing parent identity once, after origin backfill,
|
|
122
|
+
and is idempotent. Reads still recompute the canonical layout hash and verify
|
|
123
|
+
contiguous ordinals, page totals, child binding and both origin directions.
|
|
124
|
+
A permanent parent-identity anchor and monotonic layout/progress revisions
|
|
125
|
+
survive deletion and reinsertion. Parent identity and layout, page membership,
|
|
126
|
+
child binding/operation and origin mutations invalidate the layout revision for
|
|
127
|
+
both old and new parents. Normal checkpoint, fence, state and erasure-count
|
|
128
|
+
writes advance progress only. A child fences immutable layout and its current
|
|
129
|
+
sibling order; the parent undo-direction transaction fences exact progress as
|
|
130
|
+
well and verifies its single intended increment after all cleanup mutations.
|
|
131
|
+
A fresh direct child ID also reads and validates the complete parent layout
|
|
132
|
+
before capturing its revision. Nested stream calls reuse only their own validated
|
|
133
|
+
request receipt, and every pre/postcondition still checks that revision and current
|
|
134
|
+
authority in SQL. Revisions convey snapshot identity, never authorization.
|
|
135
|
+
|
|
136
|
+
The read boundary accepts only Kit-constructed guards with associated scalar
|
|
137
|
+
predicates, never arbitrary DML or SQL text rewritten into SELECTs. Trusted
|
|
138
|
+
adapters must still provide inert scalar SQL and side-effect-free read methods;
|
|
139
|
+
this is not a sandbox for arbitrary adapter JavaScript or SQL. Tests cover actual
|
|
140
|
+
SQLite and local workerd D1 result shapes, zero changes, query-only reads, late
|
|
141
|
+
revocation/tampering, complete packed groups and request-budget rejection. These
|
|
142
|
+
are local runtime receipts, not hosted capacity evidence.
|
|
143
|
+
|
|
144
|
+
The product implements `D1TargetAdapter`:
|
|
145
|
+
|
|
146
|
+
- `read(database, destination, kind, id)` returns the complete native `TargetRow`.
|
|
147
|
+
- `matches(rowOrNull, destination, kind, id)` returns an exact SQL boolean snapshot
|
|
148
|
+
predicate covering the native values, revision, attachment and related evidence,
|
|
149
|
+
or complete absence for null. This is the optimistic concurrency fence. Checking
|
|
150
|
+
only an ID or revision is insufficient; edits without revision changes must fail.
|
|
151
|
+
- Optional `matchesParts(rowOrNull, destination, kind, id)` splits the same exact
|
|
152
|
+
snapshot into nonempty conjunctive predicates for large native entities. Every
|
|
153
|
+
part and the scoped row counts are required in the same publication transaction.
|
|
154
|
+
This preserves the existing scalar `matches` API for other consumers.
|
|
155
|
+
- `write(row)` and `remove(destination, kind, id)` return inert native SQL statements.
|
|
156
|
+
- `list(database, destination)` returns native rows for independent readback.
|
|
157
|
+
- Export requires either `matchesMany(rows, destination)`, with exact predicates
|
|
158
|
+
covering the whole native set, or `matchesCount(count, destination)`, with a
|
|
159
|
+
scalar SQL predicate for its exact cardinality. With `matchesCount`, the engine
|
|
160
|
+
also guards every listed row through `matches`. The count must cover the same
|
|
161
|
+
destination and native entities as `list`, including when the snapshot is empty.
|
|
162
|
+
`matchesMany` must return at least one predicate even for an empty snapshot.
|
|
163
|
+
Missing both methods or returning an empty predicate array fails export with
|
|
164
|
+
`IMPORT_NATIVE_SET_GUARD_REQUIRED`.
|
|
165
|
+
|
|
166
|
+
`D1Sql` is `{ sql: string, params?: any[] }`; parameters are D1 bind values.
|
|
167
|
+
Adapters supply SQL; they must not send, enqueue, activate access, run lifecycle
|
|
168
|
+
hooks, mutate the import tables or contact external services. Use one effect to
|
|
169
|
+
represent a complete atomic native entity and its evidence. Contact normalization,
|
|
170
|
+
source consolidation, suppression precedence and historical consent interpretation
|
|
171
|
+
are product responsibilities. The planner rejects duplicate native target keys.
|
|
172
|
+
|
|
173
|
+
Planning checks current `plan` authority using SELECT-only fences before mapping
|
|
174
|
+
and before returning the reviewed plan. It does not require `read` authority and
|
|
175
|
+
does not write assertions or any other database state. Stream planning uses the
|
|
176
|
+
same boundary.
|
|
177
|
+
|
|
178
|
+
Call `await planD1Import(source, mapping, context)`, show its diagnostics and
|
|
179
|
+
fingerprint for review, then pass that exact `reviewedFingerprint` to
|
|
180
|
+
`createD1ImportJob`. `applyD1ImportBatches(jobOrId, context)` publishes bounded
|
|
181
|
+
batches. `batchSize` defaults to 100 source records, maximum 1000; each batch has
|
|
182
|
+
at most 1000 effects. Choose smaller batches for the product's native statement
|
|
183
|
+
count, D1 binding/statement limits and Worker CPU budget. `maxBatches` bounds work
|
|
184
|
+
per request. The port does not retry failed SQL, conflicts or revoked authority.
|
|
185
|
+
A failed batch leaves its prior committed checkpoint intact; the product can fix
|
|
186
|
+
the condition and explicitly call `resumeD1Import`, or `rollbackD1Import`.
|
|
187
|
+
|
|
188
|
+
Each batch guards the immutable plan, actor binding, checkpoint and durable fence
|
|
189
|
+
inside the same D1 transaction as native rows, originals, source ownership,
|
|
190
|
+
provenance and journal. Successful batches increment the fence. Unlike Node's
|
|
191
|
+
long-lived synchronous lease, D1 has no work outside its optimistic batch which
|
|
192
|
+
needs a lease: concurrent requests have one winner, and stale snapshots abort.
|
|
193
|
+
Terminating a request leaves either the entire D1 batch committed or none of it.
|
|
194
|
+
A later Worker instance resumes from the retained plan without parser, mapping,
|
|
195
|
+
credentials or source download. The native adapter and current auth context must
|
|
196
|
+
be provided again.
|
|
197
|
+
|
|
198
|
+
Reusing a source claim validates its owner's immutable plan, committed checkpoint
|
|
199
|
+
and recorded decision, including entries with no native effects. The owner,
|
|
200
|
+
plan chunks, decision and claim are fenced in the publication transaction, so
|
|
201
|
+
an unrelated owner or a concurrent edit cannot publish a reused decision.
|
|
202
|
+
Previously reused owners' complete selected decision, claim, active provenance and
|
|
203
|
+
history sets also carry cardinality guards, including empty sets, so an insertion
|
|
204
|
+
after preflight cannot escape their full-field snapshots.
|
|
205
|
+
Previously reused source-only entries also revalidate their claim's existence,
|
|
206
|
+
owner, source hash, mapping version and effects hash before each publication;
|
|
207
|
+
snapshotting an already corrupted claim cannot authorize another checkpoint.
|
|
208
|
+
An owner's reversal may release its source-only claims while retaining valid
|
|
209
|
+
historical replay decisions.
|
|
210
|
+
|
|
211
|
+
`undoD1Import` requires a completed job; `rollbackD1Import` reverses partial work.
|
|
212
|
+
Reversal intent is committed first, preventing subsequent forward apply even if
|
|
213
|
+
a native edit blocks undo. Effects reverse in exact reverse plan order. Each
|
|
214
|
+
bounded transaction compares native values and revisions, restores before images,
|
|
215
|
+
releases only owned source claims, and records reversal. A stale native snapshot
|
|
216
|
+
inside the batch produces `IMPORT_ATOMIC_GUARD`; an edit visible at preflight
|
|
217
|
+
produces `NATIVE_EDIT_CONFLICT`. Both preserve the edit. Resume dispatches to the
|
|
218
|
+
persisted reversal mode. Completed reversal is idempotent. Originals and all
|
|
219
|
+
provenance, including reversed before images, remain durable.
|
|
220
|
+
|
|
221
|
+
Strict reversal without a transport policy (`maxBatchBytes` omitted) verifies the
|
|
222
|
+
complete inventory, originals and plan in the intent transaction or in the first
|
|
223
|
+
batch it runs, then records in `ik_d1_reversal_fences` the job's
|
|
224
|
+
`ik_d1_custody_states` revision after that batch's own writes. While the job's
|
|
225
|
+
revision, fence and binding still equal that row, later batches and later requests
|
|
226
|
+
read only their own provenance rows, native targets and predecessors, and fence
|
|
227
|
+
the revision instead, so a batch costs the same in a large import as in a small
|
|
228
|
+
one. Any change to the job's custody, plan, originals, decisions, claims, history,
|
|
229
|
+
journal or indexed tombstones advances the revision, and the next request verifies
|
|
230
|
+
the complete inventory again before it writes. Jobs that reuse another job's
|
|
231
|
+
entries, jobs without a current erasure index, preserved-conflict undo and bounded
|
|
232
|
+
transport keep the complete per-batch verification. A fenced batch releases only
|
|
233
|
+
its own sources' claims; the final batch releases the job's remaining claims,
|
|
234
|
+
including sources with zero effects. Install the additive `d1ReversalFenceSchema`
|
|
235
|
+
(the fence table and indexes on provenance by job and by source, history by
|
|
236
|
+
source and claims by job) in a new numbered migration after `d1CustodyStateSchema`.
|
|
237
|
+
|
|
238
|
+
`readD1ImportJob`, `readD1ImportOriginals`, and `readD1PersistedImport` expose status,
|
|
239
|
+
digest-checked original bytes and native/provenance readback. Custody is also
|
|
240
|
+
rechecked inside each publication batch. The D1 tests use actual Miniflare/workerd
|
|
241
|
+
D1, durable runtime disposal/reopen, concurrent invocations, native contact and
|
|
242
|
+
evidence tables, same-revision edits, atomic SQL failure, and authority revocation
|
|
243
|
+
between preflight and commit. A Worker entry test loads this graph in workerd.
|
|
244
|
+
These tests establish local provider-runtime behavior, not a live deployment.
|
|
245
|
+
|
|
246
|
+
## Sequential imports and ownership history
|
|
247
|
+
|
|
248
|
+
A distinct source may replace a target attributed to a completed D1 import when
|
|
249
|
+
its reviewed effect contains the exact current `expectedRevision`. The engine
|
|
250
|
+
compares the complete native value and attachment with the previous provenance,
|
|
251
|
+
then atomically moves that unchanged provenance into `ik_d1_history` while
|
|
252
|
+
publishing the successor and its full before-image. The active-target unique
|
|
253
|
+
index remains enforced. Changed content under the SAME source identity still
|
|
254
|
+
fails; products must use stable identities that distinguish distinct exports.
|
|
255
|
+
|
|
256
|
+
The additive `ik_d1_history` table and index are included in `d1Schema`; apply
|
|
257
|
+
these CREATE IF NOT EXISTS statements in the next product-owned migration for
|
|
258
|
+
existing D1 installations. No existing schema or provenance needs rewriting.
|
|
259
|
+
|
|
260
|
+
Undo or rollback of a superseded job fails with `IMPORT_SUPERSEDED` before
|
|
261
|
+
changing its state. Undo the newest layer first: this restores the exact native
|
|
262
|
+
before-image AND the predecessor's active provenance and source ownership. Then
|
|
263
|
+
the predecessor can itself be undone. Native edits and concurrent successors
|
|
264
|
+
still fail their exact snapshot guards. Partial reversal resumes across requests,
|
|
265
|
+
and original source bytes, before-images and attribution remain retained across
|
|
266
|
+
all layers. An import still applying or reversing cannot be superseded
|
|
267
|
+
(`TARGET_OWNER_BUSY`). The synchronous SQLite API retains its original behavior.
|
|
268
|
+
|
|
269
|
+
## D1 request and storage limits
|
|
270
|
+
|
|
271
|
+
The bounds below follow [D1 limits](https://developers.cloudflare.com/d1/platform/limits/).
|
|
272
|
+
Bulk text insertion uses the supported [D1 JSON SQL functions](https://developers.cloudflare.com/d1/sql-api/query-json/).
|
|
273
|
+
|
|
274
|
+
The port counts SQL executions over the entire public invocation (including
|
|
275
|
+
nested resume reads and every atomic batch statement), caps them at 1,000 and
|
|
276
|
+
returns durable partial progress as the budget approaches exhaustion. Explicit
|
|
277
|
+
`IMPORT_D1_REQUEST_LIMIT`, `IMPORT_D1_STATEMENT_LIMIT` and
|
|
278
|
+
`IMPORT_D1_VALUE_LIMIT` failures publish no portion of the rejected batch.
|
|
279
|
+
Create, apply and reversal reserve both return reads before publication, including
|
|
280
|
+
the read of a chunked plan. Each successful batch retains its checked job state;
|
|
281
|
+
if a later batch's preflight or exact SQL count exhausts the same request budget,
|
|
282
|
+
the invocation returns that durable partial progress. Other adapter, integrity
|
|
283
|
+
and authority failures still reject. A first batch that cannot fit rejects before
|
|
284
|
+
native publication; reversal intent may already be persisted to prevent forward
|
|
285
|
+
apply, but no native effect or reversal journal advances.
|
|
286
|
+
Adapters must make bounded reads: implement `list` with set queries, not one
|
|
287
|
+
query per entity. Optional `matchesMany(rows, destination)` supplies exact bounded
|
|
288
|
+
set predicates for export, including native cardinality and every related field;
|
|
289
|
+
otherwise `matchesCount` supplies the cardinality fence and the engine packs the
|
|
290
|
+
individual `matches` predicates. A count alone does not protect native values,
|
|
291
|
+
and individual row predicates alone cannot detect insertions into the set.
|
|
292
|
+
Native writes must still provide their exact preconditions and postconditions.
|
|
293
|
+
Adjacent atomic guards are packed into balanced boolean trees so packing adds
|
|
294
|
+
logarithmic expression depth. Every adapter predicate must itself be valid D1
|
|
295
|
+
SQL; packing preserves its complete contents and bound-parameter order. Binding,
|
|
296
|
+
SQL-byte and value-byte ceilings still apply to each packed statement.
|
|
297
|
+
|
|
298
|
+
Install the additive `ik_d1_plan_chunks` and `ik_d1_original_chunks` CREATE
|
|
299
|
+
statements from `d1Schema` in a new product migration. Existing inline plans and
|
|
300
|
+
version-1 inline and version-2 binary-chunk original sets remain readable.
|
|
301
|
+
Plans larger than 128 KiB are stored as a digest/count/size manifest and ordered
|
|
302
|
+
UTF-8 chunks. New original sets use version 3: metadata rows have NULL bytes,
|
|
303
|
+
and canonical base64 TEXT is retained in ordered 128 KiB chunks in the existing
|
|
304
|
+
`ik_d1_original_chunks.bytes` column. SQLite BLOB affinity preserves bound TEXT;
|
|
305
|
+
no SQL binary decoder is required. Empty retained originals have one empty TEXT
|
|
306
|
+
chunk. Unavailable originals have no chunks. Version 3 needs no new migration
|
|
307
|
+
when both chunk tables already exist. It does require this reader version; do
|
|
308
|
+
not roll back to a binary-only reader after creating version-3 jobs.
|
|
309
|
+
|
|
310
|
+
Plan chunks are inserted with bounded `json_each` batches, each bound JSON
|
|
311
|
+
string at most 1,900,000 UTF-8 bytes including escaping and array punctuation.
|
|
312
|
+
For chunked plans, original chunks are copied from that exact base64 text using
|
|
313
|
+
bounded positional descriptors and SQLite `substr`, inside the same transaction.
|
|
314
|
+
Offsets account for Unicode code points and split boundaries. Original bytes
|
|
315
|
+
remain independently stored; they are not reconstructed from parsed source data.
|
|
316
|
+
Inline plans bind their small original chunks directly. Publication guards first
|
|
317
|
+
compare all plan chunks to exact snapshots, then compare original chunks with
|
|
318
|
+
those same guarded positions, plus complete metadata and cardinality checks.
|
|
319
|
+
This avoids sending two complete copies of base64 in one D1 batch request. Reconstruction
|
|
320
|
+
verifies complete bytes, ordinals, counts and digests. Publication also compares
|
|
321
|
+
the exact manifest and every chunk, including original bytes; a stored digest is
|
|
322
|
+
never the concurrency fence. Erasure integrations must delete both new chunk
|
|
323
|
+
tables along with existing job custody. These tables retain audit data through
|
|
324
|
+
undo and rollback.
|
|
325
|
+
|
|
326
|
+
### Large plans: staged creation and revision fences
|
|
327
|
+
|
|
328
|
+
A stored plan larger than `LARGE_PLAN_BYTES` (4 MiB) is never sent whole in one
|
|
329
|
+
request. Without a transport policy (`maxBatchBytes` omitted, as the import door
|
|
330
|
+
runs), `createD1ImportJob` stages it: the plan's chunks go into the existing
|
|
331
|
+
private staging tables (`d1StagingSchema`) in batches of at most
|
|
332
|
+
`STAGED_BATCH_BYTES` (4 MiB) on the wire, behind the hash-only privacy index
|
|
333
|
+
that erasure and retirement already reach. Every batch asserts current
|
|
334
|
+
authority, the stage's binding and its exact revision before and after. One
|
|
335
|
+
small final transaction then publishes the job: it copies the staged chunks and
|
|
336
|
+
index inside D1, copies each original from those chunks by position (a
|
|
337
|
+
canonical plan holds it literally, so its bytes travel once), compares every
|
|
338
|
+
copy with its staged source there, checks tombstones and releases the creation
|
|
339
|
+
capture. Only identities, positions and metadata cross the transport. A partly
|
|
340
|
+
staged creation is never a job: the next call resumes from the rows already
|
|
341
|
+
staged (verified by reading them back), or discards a generation that no longer
|
|
342
|
+
matches, and `cancelD1ImportStaging`, erasure and retirement remove it. Every
|
|
343
|
+
remaining batch is admitted against the request budget before the first write.
|
|
344
|
+
`preflightD1ImportJob` sizes the same staged requests.
|
|
345
|
+
|
|
346
|
+
After creation, an unbounded load of a large plan reads its
|
|
347
|
+
`ik_d1_custody_revisions` revision before any chunk, then verifies the chunks
|
|
348
|
+
against the envelope digest and fingerprint as before. Its guards fence that
|
|
349
|
+
revision and the exact `plan_json` envelope instead of resending every chunk:
|
|
350
|
+
every chunk insert, update or delete and every `plan_json` update advances the
|
|
351
|
+
revision, so the guard holds exactly when the verified chunks are unchanged.
|
|
352
|
+
Originals keep their position comparisons. Smaller plans, and every bounded
|
|
353
|
+
(`maxBatchBytes`) path, keep exact byte guards. Original chunk and plan chunk
|
|
354
|
+
reads are paged.
|
|
355
|
+
|
|
356
|
+
Creation retains a normalized plan with the target adapter's `planCodec`, as
|
|
357
|
+
streams and erasure re-retention do. A plan holding original bytes (an
|
|
358
|
+
available artifact or an attachment) stays canonical, so its originals are
|
|
359
|
+
copied from and compared by position with its chunks.
|
|
360
|
+
|
|
361
|
+
## Erasing normalized personal records
|
|
362
|
+
|
|
363
|
+
`eraseD1ImportTarget(kind, id, nativeStatements, context)` erases an entity's
|
|
364
|
+
normalized source payload, effect, record-scoped diagnostics and provenance
|
|
365
|
+
before-images across all destination jobs, including inline and chunked plans.
|
|
366
|
+
It executes the supplied inert native SQL and the kit reconciliation in one D1
|
|
367
|
+
batch. Supply the native erasure and anonymized audit statements; the adapter's
|
|
368
|
+
complete current snapshot and absent postcondition must cover all dependent
|
|
369
|
+
native rows. Current `erase` authority is checked at both ends of the batch;
|
|
370
|
+
this destination-level permission is independent of the original importing actor.
|
|
371
|
+
The result contains the native statement results and number of erased decisions.
|
|
372
|
+
Job status reports `erasedRecords`, so reversal can disclose records already erased.
|
|
373
|
+
|
|
374
|
+
By default this primitive covers one entity per source record with unavailable
|
|
375
|
+
original artifacts and no retained attachments. Shared source records and retained
|
|
376
|
+
original bytes fail explicitly before any mutation. A product can deliberately
|
|
377
|
+
select `context.originalErasurePolicy = 'withdraw-affected-artifacts'` for erasure.
|
|
378
|
+
This privacy policy withdraws the entire original artifact containing an erased
|
|
379
|
+
source, including other people's raw records in the same file/page; their normalized
|
|
380
|
+
records, native rows and resumable job remain. It also withdraws byte-identical
|
|
381
|
+
artifact copies and originals referenced by duplicate occurrences of that source.
|
|
382
|
+
Other original artifacts retain their exact bytes. For a mapping assembled from
|
|
383
|
+
multiple pages, select `context.originalErasurePolicy = 'withdraw-all-job-artifacts'`
|
|
384
|
+
to withdraw every artifact in each affected job. Use that conservative policy when
|
|
385
|
+
one record location cannot describe all contributing raw pages, as with event and
|
|
386
|
+
invitee snapshots. All normalized peers and their native rows still remain.
|
|
387
|
+
The first explicit withdrawal binds the job's policy; selecting a different
|
|
388
|
+
explicit policy later fails with `IMPORT_ORIGINAL_ERASURE_POLICY_CONFLICT`.
|
|
389
|
+
Attachments and composite source effects remain unsupported and fail before native mutation.
|
|
390
|
+
|
|
391
|
+
The selected policy is snapshotted for the request and recorded in the changed
|
|
392
|
+
job's fingerprint-bound `plan.erasure.originalPolicy`. `readD1ImportOriginals`
|
|
393
|
+
returns affected artifacts with `{ status: 'unavailable', reason:
|
|
394
|
+
'unavailable-after-erasure', policy: <selected policy> }`. Artifact
|
|
395
|
+
names/keys and associated source/provenance locations become opaque, hash-derived
|
|
396
|
+
metadata; bytes are deleted, never rewritten into a claimed original. Original
|
|
397
|
+
hash and size remain as non-content custody evidence. Artifact-scoped and unscoped
|
|
398
|
+
diagnostics are removed when withdrawal occurs. Unaffected artifacts, mapping
|
|
399
|
+
values and ordinary peer undo semantics remain intact. No migration is needed.
|
|
400
|
+
|
|
401
|
+
This is lineage-based erasure: the consuming reader must account for every
|
|
402
|
+
personal record in the retained artifact and associate it with its artifact hash
|
|
403
|
+
and location, including repeated occurrences. The kit cannot infer a person's
|
|
404
|
+
identity in arbitrary raw text, other records' payloads or unrelated native
|
|
405
|
+
stores. The caller must select every affected native entity and supply its full
|
|
406
|
+
native deletion/anonymization SQL. Original privacy policy does not authorize
|
|
407
|
+
reading or rewriting any external source. Both ordinary and transport-bounded
|
|
408
|
+
D1 erasure use these policies; size/admission limits still fail before publication.
|
|
409
|
+
Scope personal diagnostic text to its source
|
|
410
|
+
kind and ID or source location so the same erasure removes it. Do not place personal
|
|
411
|
+
record data in unscoped diagnostics or other records' metadata.
|
|
412
|
+
|
|
413
|
+
### Related copies of the erased person
|
|
414
|
+
|
|
415
|
+
`eraseD1ImportTarget(kind, id, nativeStatements, context, related)` also erases
|
|
416
|
+
retained copies the product recognizes outside the target's own lineage.
|
|
417
|
+
`related.source(record)` is asked about every unerased source record in the
|
|
418
|
+
destination, including rejected rows and records that map to other targets;
|
|
419
|
+
`related.original(bytes, { name, sha256 })` is asked about every retained original
|
|
420
|
+
artifact. Matching records are erased like the target's own sources, bound to the
|
|
421
|
+
erased target's identity, with their effects severed: their provenance and
|
|
422
|
+
before-images are deleted, so undo leaves the native rows they produced in place.
|
|
423
|
+
`nativeStatements` must redact those rows in the same batch. Matching originals
|
|
424
|
+
are withdrawn under the selected `originalErasurePolicy`, which is required. The
|
|
425
|
+
exact destination job set and fingerprints read by the scan are fenced, so a job
|
|
426
|
+
created or changed before the batch aborts it with `IMPORT_ATOMIC_GUARD`. The scan
|
|
427
|
+
reads every retained plan and original in the destination; bounded erasure
|
|
428
|
+
(`maxBatchBytes`) refuses `related` with `IMPORT_RELATED_ERASURE_UNBOUNDED_ONLY`.
|
|
429
|
+
Matchers receive decoded personal data: they must be pure and must not log it.
|
|
430
|
+
|
|
431
|
+
Erasure replaces the record with an explicit zero-effect tombstone, keeps source
|
|
432
|
+
identity and ordinal, and rehashes the plan, source claims and committed decisions.
|
|
433
|
+
The changed plan fingerprint fences any apply or reversal prepared before erasure.
|
|
434
|
+
It preserves other records and their exact undo guards. Original operation-key
|
|
435
|
+
replay returns the same redacted job, while a new operation involving an erased
|
|
436
|
+
source or native target is refused before retaining its plan. Hash-only tombstones prevent
|
|
437
|
+
replay even after undo releases transient source claims. Resume continues ordinary
|
|
438
|
+
remaining work; undo skips the erased effect and remains idempotent. Never remove
|
|
439
|
+
retained jobs as an erasure shortcut.
|
|
440
|
+
|
|
441
|
+
Existing installations must apply the separate additive `d1ErasureSchema` (the
|
|
442
|
+
original `d1Schema` is unchanged), then call `upgradeD1ImportErasureIndex(context)`
|
|
443
|
+
in bounded pages until `complete`. This indexes prior inline and chunked plans
|
|
444
|
+
without changing them. Each page requires current erase authority and fences the
|
|
445
|
+
exact source plan. Erasure fails before any native change while that destination
|
|
446
|
+
has unindexed jobs. New jobs write their index atomically, so subsequent erasure
|
|
447
|
+
looks up only affected copies and their inventory owners. Indexes contain hashed
|
|
448
|
+
source/target identities, ordinals, job IDs and plan fingerprints, never personal
|
|
449
|
+
answers or raw source/native IDs. Set reads and writes share one publication
|
|
450
|
+
snapshot across repeated imports, preserving all existing inventory guards.
|
|
451
|
+
|
|
452
|
+
Every insertion, update or deletion of a derived index row invalidates that job's
|
|
453
|
+
completeness receipt through database triggers. The bounded upgrade rebuilds the
|
|
454
|
+
rows from the exact source plan and writes its receipt last in the same transaction.
|
|
455
|
+
Apply, resumed create and reversal also check live source and target tombstones
|
|
456
|
+
independently of the derived index; only explicit erased zero-effect entries are
|
|
457
|
+
exempt. A stale or corrupted private plan can never use a missing index row to
|
|
458
|
+
resurrect a tombstoned source.
|
|
459
|
+
|
|
460
|
+
|
|
461
|
+
## Erasing a person from import custody
|
|
462
|
+
|
|
463
|
+
`eraseD1ImportTarget` removes a person's records but leaves any header the
|
|
464
|
+
product filled from that person: a job or stream namespace, an operation key,
|
|
465
|
+
an original's name, a private stage's header. `retireD1ImportCustody(person,
|
|
466
|
+
nativeStatements, context, checkpoint)` erases the person from all of import
|
|
467
|
+
custody and writes one erasure journal row, in a single D1 batch with the
|
|
468
|
+
supplied native statements, the restore checkpoint and current `erase`
|
|
469
|
+
authority at both ends.
|
|
470
|
+
|
|
471
|
+
```ts
|
|
472
|
+
const native = await erasure.nativeStatements('person', { id }); // modules/erasure
|
|
473
|
+
const checkpoint = (await portability.redactionStatements()).map(toD1Sql);
|
|
474
|
+
const entry = await retireD1ImportCustody(
|
|
475
|
+
{ target: { kind: 'person', id }, values: [person.email] }, native.statements, context, checkpoint);
|
|
476
|
+
await erasure.deleteBlobs(native.operationId);
|
|
477
|
+
```
|
|
478
|
+
|
|
479
|
+
Matching uses exact identifiers only, never substrings. `target` names the
|
|
480
|
+
native entity the person's records were imported into; those records become
|
|
481
|
+
zero-effect tombstones in every job exactly as `eraseD1ImportTarget` makes them,
|
|
482
|
+
so other people's records in the same job still resume and undo. `values` are
|
|
483
|
+
exact strings, typically the person's email: every job, stream, stream origin,
|
|
484
|
+
revision anchor, private stage and original whose header cell equals one of
|
|
485
|
+
them is found. Such a job cannot be renamed without changing the source
|
|
486
|
+
identities that undo and resume rely on, so its whole custody is retired: plans,
|
|
487
|
+
originals, provenance, history, reversal reports, entries, sources, stages,
|
|
488
|
+
indexes and the job row are deleted, and `ik_d1_retirements` keeps a plain row
|
|
489
|
+
(job or stream id, destination, receipt id, time). Retirement closes over every
|
|
490
|
+
stream the job belongs to, jobs whose entries reuse its sources and preserved
|
|
491
|
+
history in both directions, and the commit refuses if that closure changed after
|
|
492
|
+
it was read. Native rows a retired job already wrote stay; it can no longer be
|
|
493
|
+
resumed or undone, and `readD1ImportJob`, `resumeD1Import` and `undoD1Import`
|
|
494
|
+
throw `IMPORT_JOB_RETIRED` (streams: `IMPORT_STREAM_RETIRED`) carrying the
|
|
495
|
+
`receiptId`. A retired stream's revision anchor keeps its counters and layout
|
|
496
|
+
fields and drops its namespace and operation key, so its id can never be reused.
|
|
497
|
+
The batch then proves that no header cell in the destination still equals a
|
|
498
|
+
named value. Originals holding the person are withdrawn under
|
|
499
|
+
`context.originalErasurePolicy`, which defaults to `withdraw-affected-artifacts`
|
|
500
|
+
here. Pass `related` as for `eraseD1ImportTarget` to sever other records that
|
|
501
|
+
mention the person; a name is not an identifier, so do not pass one as a value.
|
|
502
|
+
|
|
503
|
+
Rewriting a job into tombstones keeps its row keys, so per-row restore
|
|
504
|
+
tombstones cannot fence it, and an archive made before the erasure may hold the
|
|
505
|
+
person only in import custody (still pending, or only in a header). `checkpoint`
|
|
506
|
+
is the product's restore checkpoint for rewritten rows: portability's
|
|
507
|
+
`redactionStatements()`, as `D1Sql`. The batch carries it whenever it has a
|
|
508
|
+
`target` or retires anything, so `rer import --journal LATER_ARCHIVE` refuses any
|
|
509
|
+
older archive whole; it is required then (`IMPORT_RETIREMENT_CHECKPOINT_REQUIRED`).
|
|
510
|
+
A batch without it (nothing matched, so nothing is fenced) must also carry no
|
|
511
|
+
`nativeStatements`: `IMPORT_RETIREMENT_NATIVE_REQUIRES_CHECKPOINT` refuses before any
|
|
512
|
+
write, because SQL the kit cannot fence could rewrite a row it does not count, such as
|
|
513
|
+
another import's job. With the checkpoint, and nothing else to do, it must leave every
|
|
514
|
+
row of the retired scope unchanged, or it aborts.
|
|
515
|
+
|
|
516
|
+
The journal (`ik_d1_erasure_journal`) and `ik_d1_retirements` are append-only:
|
|
517
|
+
database triggers refuse updates, deletes and any insert over an existing key,
|
|
518
|
+
so `INSERT OR REPLACE` cannot rewrite a row. Each row holds the receipt id, destination, time, the number
|
|
519
|
+
of records erased, jobs and streams retired, and per store the number of rows
|
|
520
|
+
the erasure deleted or overwrote (`stores`). Counts come from row images taken
|
|
521
|
+
and compared inside the same transaction in `ik_d1_erasure_tally`, which is
|
|
522
|
+
empty again before the batch commits; bookkeeping counters, tombstones and the
|
|
523
|
+
journal itself are not counted. It holds no erased value. Read it with
|
|
524
|
+
`readD1ErasureJournal(context)`; the function returns the row it wrote.
|
|
525
|
+
|
|
526
|
+
Apply `d1RetirementSchema` as a new numbered migration after the other kit
|
|
527
|
+
schemas (existing applied migrations stay unchanged); `initializeD1ImportStorage`
|
|
528
|
+
applies it too. It adds the three tables and replaces the stream revision update
|
|
529
|
+
trigger with one that also admits the retirement transition. A store that
|
|
530
|
+
already applied an earlier `d1RetirementSchema` applies `d1RetirementAppendOnly`
|
|
531
|
+
(the two insert triggers) as its next numbered migration. Keep the journal and
|
|
532
|
+
retirements in archives. Restoring an archive made before the erasure brings
|
|
533
|
+
nothing back: the native rows are fenced by the erasure module's
|
|
534
|
+
`nativeStatements`, and import custody by the checkpoint. Without an operation
|
|
535
|
+
key this remains a single transaction. A bounded call supplies an operation key
|
|
536
|
+
as described below; omitting it with `maxBatchBytes` raises
|
|
537
|
+
`IMPORT_RETIREMENT_OPERATION_KEY_REQUIRED`.
|
|
538
|
+
|
|
539
|
+
`person.related.fields` can stand without a `target` (a person who appears only
|
|
540
|
+
inside other people's records), and `{ resumable: true }` spreads a redaction
|
|
541
|
+
across requests in bounded steps; see the kit README, "Erasing fields in
|
|
542
|
+
retained shared records".
|
|
543
|
+
|
|
544
|
+
|
|
545
|
+
### Resuming bounded retirement and erasure
|
|
546
|
+
|
|
547
|
+
Apply `d1ErasureProgressSchema` as a new numbered migration after the existing
|
|
548
|
+
kit schemas. `initializeD1ImportStorage` includes it. Keep completed progress
|
|
549
|
+
in archives alongside the erasure journal and retirement rows.
|
|
550
|
+
|
|
551
|
+
`retireD1ImportCustodyStep` (also `retireD1ImportCustody` with an `operationKey`
|
|
552
|
+
option) prepares oversized legacy custody over multiple requests. Reuse the
|
|
553
|
+
product's durable operation key, person, native SQL and checkpoint each time:
|
|
554
|
+
|
|
555
|
+
```ts
|
|
556
|
+
const result = await retireD1ImportCustodyStep(person, native, context, checkpoint, {
|
|
557
|
+
operationKey: operation.id,
|
|
558
|
+
maxStatements: 800,
|
|
559
|
+
});
|
|
560
|
+
return Response.json(result, { status: result.phase === 'pending' ? 202 : 200 });
|
|
561
|
+
```
|
|
562
|
+
|
|
563
|
+
Each invocation executes at most `maxStatements` SQL statements, including reads
|
|
564
|
+
(default and maximum 1000). Mutation batches fit the smaller of
|
|
565
|
+
`context.maxBatchBytes` and 2 MiB; omission chooses 2 MiB. Admission uses the shared
|
|
566
|
+
D1 transport's UTF-8, JSON escaping and whole-call string-width accounting. It
|
|
567
|
+
admits a useful future step and the indivisible publication, without charging
|
|
568
|
+
all future staging batches to this request. `admissionOnly: true` compiles using
|
|
569
|
+
real reads and writes nothing. Its pending descriptor does not create a job.
|
|
570
|
+
|
|
571
|
+
`phase: 'pending'` reports the operation id and verified index/value counts. A
|
|
572
|
+
later invocation recompiles the custody generation and obtains its offset from
|
|
573
|
+
`ik_d1_erasure_progress`. Each insertion verifies exact rows after triggers and
|
|
574
|
+
advances that journal in the same transaction. A failed transaction leaves the
|
|
575
|
+
previous offset; a lost acknowledgement resumes from the committed offset.
|
|
576
|
+
Changed custody starts a newly guarded generation. An offset disagreeing with
|
|
577
|
+
the stage binding, revision or physical counts is refused. Request options do
|
|
578
|
+
not accept numeric offsets. Progress stores digests and counts, never selected
|
|
579
|
+
values; private normalized values remain under native staging privacy guards.
|
|
580
|
+
|
|
581
|
+
Only final publication changes native rows, rewrites plans, retires the header
|
|
582
|
+
closure, advances the restore checkpoint and writes the receipt. It also removes
|
|
583
|
+
the private stage. Reusing a completed operation returns the same receipt without
|
|
584
|
+
applying its writes again. `nativeResults` is present on the publishing response
|
|
585
|
+
and empty on a completed retry. Different request bindings under the same key
|
|
586
|
+
raise `IMPORT_ERASURE_OPERATION_CHANGED`. Keep identities distinct for distinct
|
|
587
|
+
erasures. Provider errors preserve progress because a batch may have committed.
|
|
588
|
+
`cancelD1ErasureStaging` removes private staging under current erase authority;
|
|
589
|
+
a later invocation can begin preparation again.
|
|
590
|
+
|
|
591
|
+
`eraseD1ImportTargetStep(kind, id, native, context, options)` provides the same
|
|
592
|
+
journaled continuation for target-only erasure. Use retirement when headers and
|
|
593
|
+
restore checkpoints also apply. Existing one-invocation erasure remains usable.
|
|
594
|
+
|
|
595
|
+
For additional product custody rewrites, such as removing legacy mapper
|
|
596
|
+
`source.options`, use `stageD1ImportErasure(context, options, binding, build)`.
|
|
597
|
+
`build(id, boundedContext)` returns `D1ErasureStageSpec`: deterministic,
|
|
598
|
+
authenticated fragments, native snapshot guards, target publication declarations
|
|
599
|
+
and final predicates. Always use its supplied context for database work. The
|
|
600
|
+
binding identifies immutable product erasure intent. Optional `publication`
|
|
601
|
+
statements surround the compiled writes; its `tail` runs after retained values
|
|
602
|
+
are consumed and the stage is discarded, in the same transaction. This supports
|
|
603
|
+
product journals, restore fences and work-item completion. `survivingJobs`
|
|
604
|
+
excludes jobs deleted by that tail from final custody seals. Exact replay,
|
|
605
|
+
privacy membership and native publication checks still apply.
|
|
606
|
+
|
|
607
|
+
`NativeStageRows` accepts an opaque `VerifiedNativeStageOffset` from
|
|
608
|
+
`verifyNativeStageOffset`. Supply the journal's counts and authentic journal
|
|
609
|
+
guards. Verification binds it to the context, generation, baseline, table and
|
|
610
|
+
canonical columns. Replay starts at the beginning and skips only that verified
|
|
611
|
+
prefix. Its optional checkpoint callback runs atomically with each insertion;
|
|
612
|
+
a forged object or truncated replay is refused.
|
|
613
|
+
|
|
614
|
+
## Normalized paged imports
|
|
615
|
+
|
|
616
|
+
`planD1ImportStream(source, mapping, context)` supports normalized sources with
|
|
617
|
+
unique record identities, unavailable original bytes, no attachments and no
|
|
618
|
+
unscoped source diagnostics. Other sources continue to use `planD1Import`.
|
|
619
|
+
Input records may be an array-compatible lazy view. Mapping runs once per record;
|
|
620
|
+
normalization uses the shared planner, then partitions at 100 records, 100
|
|
621
|
+
native units (one per effect, at least one per record) and about 512 KiB of
|
|
622
|
+
normalized source/effect content. Undo cost grows with a child's units, so many
|
|
623
|
+
tiny effects split into more children. Partitioning counts shared literals
|
|
624
|
+
once, including short statements and small dictionaries, so repeated immutable
|
|
625
|
+
metadata does not multiply a dictionary across pages. An individual native entity remains
|
|
626
|
+
atomic. Distinct source identities with equal values remain distinct records.
|
|
627
|
+
|
|
628
|
+
The returned compressed pages are lossless normalized plans. A one-page view
|
|
629
|
+
computes the same complete canonical fingerprint as the eager planner, including
|
|
630
|
+
source order, diagnostics and counts. During initial review, fresh bounded source
|
|
631
|
+
copies are checked against the canonical hashes observed during normalization;
|
|
632
|
+
this avoids reconstructing native tables twice just to read their source records.
|
|
633
|
+
The returned stream retains no original source or mapping closure. Canonical size and integrity comparisons
|
|
634
|
+
do not allocate complete expanded JSON strings. Creation can reuse that computed
|
|
635
|
+
fingerprint only while every page byte, length and header field matches its
|
|
636
|
+
private in-memory snapshot; changed input receives full canonical verification. Review this fingerprint, then call
|
|
637
|
+
`createD1ImportStream` with `reviewedFingerprint`. Creation publishes all pages, original availability, hash-only erasure indexes
|
|
638
|
+
and the sealed parent layout atomically after bounded private custody staging. No native effect is published during creation. Publication groups each child
|
|
639
|
+
identity once with its index rows and derives shared original-availability
|
|
640
|
+
receipts from the atomic page membership in SQL, avoiding repeated metadata
|
|
641
|
+
in the D1 transport. The normal request budget
|
|
642
|
+
and exact authority checks apply; insufficient budget publishes no parent or page.
|
|
643
|
+
The input and compressed pages still occupy memory in the creating request;
|
|
644
|
+
products must qualify their actual admitted source limits. Publication transport
|
|
645
|
+
retains only one bounded row set, not every SQL value for the complete stream.
|
|
646
|
+
|
|
647
|
+
`readD1ImportStream`, `resumeD1ImportStream` and `undoD1ImportStream` accept the
|
|
648
|
+
parent ID. Each resume or undo invocation operates on one bounded child plan,
|
|
649
|
+
using the existing native engine, custody, source reuse and optimistic guards.
|
|
650
|
+
The parent retains the reviewed fingerprint; each child retains its own canonical
|
|
651
|
+
plan fingerprint. Resume needs neither uploaded files nor the original mapper.
|
|
652
|
+
Reversal proceeds in reverse page order and persists its direction before native
|
|
653
|
+
changes. Direct child entry points also enforce parent direction and ordering.
|
|
654
|
+
Version-4 original availability receipts bind normalized child custody to a live
|
|
655
|
+
parent even if its membership and parent rows are both removed. Ordinary version-1/2/3
|
|
656
|
+
receipts keep their existing interpretation and arbitrary operation keys.
|
|
657
|
+
Page membership mutations invalidate the parent; ordinary erased-record progress
|
|
658
|
+
does not alter the sealed layout. Erasure redacts every affected child and reports
|
|
659
|
+
aggregate erased decisions in parent status. Original-operation replay returns
|
|
660
|
+
that same parent; new operations still honor source and target tombstones.
|
|
661
|
+
|
|
662
|
+
Large child pages use digest-checked compressed encoding inside the existing exact
|
|
663
|
+
chunk manifests. The manifest and every compressed byte remain publication
|
|
664
|
+
preconditions. Compression does not replace snapshot fencing or source identity.
|
|
665
|
+
Eager create keeps its pure admission and one atomic original-custody
|
|
666
|
+
publication; a large plan stages that publication's input first (see "Large
|
|
667
|
+
plans: staged creation and revision fences"). Private normalized stream staging
|
|
668
|
+
is a separate operation.
|
|
669
|
+
|
|
670
|
+
|
|
671
|
+
A native adapter may provide a versioned `planCodec` with pure `encode(plan)` and
|
|
672
|
+
`decode(payload)` operations. This is a lossless storage representation, not a
|
|
673
|
+
mapping callback: every source record, native field and order must reconstruct
|
|
674
|
+
exactly. The kit verifies the full canonical plan fingerprint on encode roundtrip
|
|
675
|
+
and every decode, rejects an absent or mismatched codec, and applies the same
|
|
676
|
+
manifest/chunk byte fences. Consumers must retain every shipped decoder version.
|
|
677
|
+
Compact derived-value encodings can keep complete normalized custody small even
|
|
678
|
+
when native tables expand accepted input substantially. All encoded pages and
|
|
679
|
+
indexes still publish atomically before native apply; no original input or mapper
|
|
680
|
+
closure is needed after an interruption. Erasure replaces the entire affected
|
|
681
|
+
encoded page and removes its old chunks, including all encoded source copies.
|
|
682
|
+
|
|
683
|
+
Normalized streams account for all repeated literals when choosing retained
|
|
684
|
+
pages, including short Unicode statements. When repeated metadata dominates a
|
|
685
|
+
plan's expanded JSON, adjacent normalized pages share a bounded dictionary
|
|
686
|
+
working set before compression, bounded by two input pages and 8 MiB of
|
|
687
|
+
expanded JSON; a larger individual input page is processed on its own. Mapping still runs once per source record;
|
|
688
|
+
canonical fingerprints, exact native values and transactional guards are
|
|
689
|
+
unchanged. Ordinary pages retain their expanded-byte boundary. Erasure rebuilds
|
|
690
|
+
the dictionary from the redacted plan and retains compressed custody, so
|
|
691
|
+
removed values cannot survive in unused dictionary entries and a privacy action
|
|
692
|
+
does not materialize the entire expanded plan in SQL buffers.
|
|
693
|
+
|
|
694
|
+
Dictionary version two also shares literals inside exactly serialized JSON
|
|
695
|
+
strings. A native JSON answer and its normalized source can therefore share an
|
|
696
|
+
immutable statement while decoding to their original distinct JSON types. The
|
|
697
|
+
decoder reconstructs each serialized string once per page, validates every
|
|
698
|
+
reference, and rejects unused or duplicate entries. The envelope records the
|
|
699
|
+
exact encoded payload size; embedded text is size-checked before reconstruction
|
|
700
|
+
so a small reference graph cannot allocate text beyond that bound. Strings with significant
|
|
701
|
+
formatting retain their original literal bytes. First-version dictionaries
|
|
702
|
+
remain readable; erasure rebuilds all tables from the remaining values.
|
|
703
|
+
|
|
704
|
+
Canonical hashing uses a 64 KiB byte buffer and a per-call cache bounded to
|
|
705
|
+
4,096 strings and 1 MiB of encoded literals. It traverses mutable containers on
|
|
706
|
+
every visit and preserves the same canonical UTF-8 bytes and fingerprint.
|
|
707
|
+
### Pure creation admission before product changes
|
|
708
|
+
|
|
709
|
+
Call `preflightD1ImportJob(plan, context)` before adopting native schema or
|
|
710
|
+
permissions. It is synchronous and needs no database or target adapter. The
|
|
711
|
+
`D1CreatePreflightContext` includes destination, namespace, mappingVersion,
|
|
712
|
+
authorization, reviewedFingerprint and optional operationKey. Use the complete
|
|
713
|
+
actual reviewed plan and operation key and the same pure authorization predicate
|
|
714
|
+
used by production creation; the predicate builds SQL and must have no side effects.
|
|
715
|
+
Admission does not execute that predicate or authorize anything. Production
|
|
716
|
+
evaluates current authority inside the initial and final atomic transactions.
|
|
717
|
+
|
|
718
|
+
The returned `D1CreatePreflight` contains `mutationStatements`,
|
|
719
|
+
`totalExecutions`, `maxSqlBytes`, `maxBoundBytes` and `maxParameters`. It uses the
|
|
720
|
+
production serialization, chunking and guard packing, reserving initial authority,
|
|
721
|
+
operation-key lookup, job return and chunked-plan return reads. For a large
|
|
722
|
+
plan without a transport policy it admits every staged batch, the publication
|
|
723
|
+
and the prior-generation lookup instead, assuming nothing is staged yet. Pass
|
|
724
|
+
the adapter's `planCodec` (or a full context) so it sizes the codec-retained
|
|
725
|
+
plan production creates. The 36-byte
|
|
726
|
+
placeholder UUID has exactly the encoded size of the production-generated UUID.
|
|
727
|
+
Invalid plans/bindings and `IMPORT_D1_REQUEST_LIMIT`,
|
|
728
|
+
`IMPORT_D1_STATEMENT_LIMIT`, `IMPORT_D1_VALUE_LIMIT` throw before any product
|
|
729
|
+
mutation. The limit is the actual encoded work, not a synthetic artifact-count or
|
|
730
|
+
file-byte ceiling. Admission covers fresh job creation; replay depends on the
|
|
731
|
+
existing job inventory, and later native effect batches require their own bounds.
|
|
732
|
+
It does not predict provider storage availability, Worker memory/CPU allowance or
|
|
733
|
+
validity of consumer SQL. The product must check those independent boundaries.
|
|
734
|
+
New plans or submitted creation plans over the inclusive 300,000,000 canonical
|
|
735
|
+
content-byte `PLAN_BYTES_LIMIT` are refused with `IMPORT_PLAN_TOO_LARGE` before
|
|
736
|
+
encoding/cloning mapper effects or encoding the retained plan. Content excludes
|
|
737
|
+
the fingerprint field; the complete stored plan reserves 81 more bytes. Source,
|
|
738
|
+
repeated entry source, effects, metadata, diagnostics, base64 and JSON escaping
|
|
739
|
+
all count. The error carries numeric measured `bytes`, `limit` and `subject`;
|
|
740
|
+
show these in preview before writes. The cumulative planner stops at the mapping
|
|
741
|
+
that exceeds the cap. Creation guards again after awaited boundaries, including
|
|
742
|
+
operation-key replay, without replacing current-authority or reviewed-fingerprint
|
|
743
|
+
checks. See "Plan-size cap" in the kit README for current memory/request evidence:
|
|
744
|
+
the 4 GiB Node heap ceiling does not bound total RSS, and hosted capacity remains
|
|
745
|
+
unproven.
|
|
746
|
+
|
|
747
|
+
|
|
748
|
+
The default D1 suite includes twenty distinct 10 MiB originals with native
|
|
749
|
+
publication, replay, undo and exact retained-byte readback. The local
|
|
750
|
+
Miniflare/workerd capacity probe uses the real D1 binding and meters each entire
|
|
751
|
+
public invocation. This is local D1 SQL/custody evidence: a 200 MiB source and
|
|
752
|
+
its complete plan have substantial Node/runtime memory costs. It does not
|
|
753
|
+
establish hosted Worker memory, CPU, multipart upload or product parser limits.
|
|
754
|
+
Nor does it establish capacity for twenty maximum-content CSVs: a valid example
|
|
755
|
+
with 105 rows/table, two fields and 10 MiB/file requires 910,382,498 plan content
|
|
756
|
+
bytes. The kit cap is an additional preview/storage bound; retain the product's
|
|
757
|
+
separate table/field/file/row/cell limits and never strip custody or parsed data to
|
|
758
|
+
make admission pass.
|
|
759
|
+
|
|
760
|
+
## Adopting preserved-conflict undo (evolution 0001)
|
|
761
|
+
|
|
762
|
+
Existing installations must apply every statement exported as
|
|
763
|
+
`preservedUndoEvolution` from `src/d1/evolutions/0001-preserved-undo.ts` in a NEW
|
|
764
|
+
product-owned numbered migration. Do not edit an applied migration. The evolution
|
|
765
|
+
adds the durable policy, its job/fingerprint binding, report cursor and report
|
|
766
|
+
hash to `ik_jobs`, and adds `ik_d1_reversal_reports` and the transaction-local
|
|
767
|
+
creation snapshot table/triggers `ik_d1_creation_plans`. Existing jobs default to
|
|
768
|
+
`strict`; their provenance and custody are unchanged. `d1Schema` contains the
|
|
769
|
+
complete fresh-install schema including this evolution (its ALTER statements are
|
|
770
|
+
one-time DDL). `initializeD1ImportStorage` applies the evolution on first setup.
|
|
771
|
+
The synchronous SQLite engine is unchanged and does not use these D1 columns.
|
|
772
|
+
|
|
773
|
+
New jobs may select `context.reversalPolicy = 'preserve-conflicts'` at creation.
|
|
774
|
+
Omission creates a strict job. Later calls may omit the option to use the durable
|
|
775
|
+
policy, or pass the same value; a different explicit value rejects with
|
|
776
|
+
`REVERSAL_POLICY_CONFLICT`. Only the explicit reviewed conversion described
|
|
777
|
+
below can change an eligible existing strict job's policy; no ordinary call
|
|
778
|
+
switches it automatically. The policy and its binding are fenced alongside the
|
|
779
|
+
job, original custody and plan. Creation admission measures these statements,
|
|
780
|
+
all exact custody guards, initial authorization, and the actual return reads.
|
|
781
|
+
|
|
782
|
+
Strict undo retains its full native revision/value/attachment comparison and
|
|
783
|
+
`NATIVE_EDIT_CONFLICT`. Preserved undo still requires completed imports and the
|
|
784
|
+
current ordinary `undo` authorization in every transaction, including batches
|
|
785
|
+
which only record conflicts and the terminal batch. The same selected policy applies to rollback of partially published work;
|
|
786
|
+
its report covers only the effects already committed. Undo and rollback cannot change an existing reversal mode.
|
|
787
|
+
A job beneath any successor still rejects with `IMPORT_SUPERSEDED`, except
|
|
788
|
+
where the successor's preserved undo kept an effect for the owner: that undo
|
|
789
|
+
releases, in the same batch, the history beneath the kept effect (each
|
|
790
|
+
predecessor returns to its own job's provenance as reversed, and that job's
|
|
791
|
+
claim goes once nothing else backs it). The predecessor job's own undo then
|
|
792
|
+
reports each such effect `retained` with reason `superseded` (a strict job just
|
|
793
|
+
skips it), so a lineage with a catch-up undoes fully and the owner's edit stays.
|
|
794
|
+
|
|
795
|
+
To support preserved undo, implement `targets.preparePreservedUndo(db,
|
|
796
|
+
destination, effects)` and, only when native effects own disjoint parts of a
|
|
797
|
+
shared snapshot, `targets.ownedContent(row)`. The types are exported from
|
|
798
|
+
`undo-types.ts`. `effects` supplies each active effect's immutable applied row,
|
|
799
|
+
original provenance and whether its retention is already durable. Return exactly
|
|
800
|
+
one observation per effect, its complete current row or null, native entity
|
|
801
|
+
labels, optional domain evidence, explicit dependencies, and nonempty scalar SQL
|
|
802
|
+
predicates guarding the complete current native/dependency snapshot. These
|
|
803
|
+
predicates must cover values, revisions, ownership markers, related evidence,
|
|
804
|
+
outside incoming dependencies, absent edges and cardinalities. The engine also
|
|
805
|
+
guards each observation through `targets.matches`; an observation may supply an
|
|
806
|
+
exact `guard` for domain-specific absence such as a deleted native row whose
|
|
807
|
+
ownership marker remains. These guards are evaluated inside the same atomic
|
|
808
|
+
transaction as native reversal, attribution, dispositions and progress.
|
|
809
|
+
|
|
810
|
+
`ownedContent` is a pure projection which Kit applies to both expected and
|
|
811
|
+
observed rows; its semantics must remain stable for existing jobs and reports.
|
|
812
|
+
It may exclude unrelated wrapper revisions but must retain every owned value,
|
|
813
|
+
attachment and owned revision. The complete current snapshot is still fenced.
|
|
814
|
+
An effect with a before-image uses the full revision/value/attachment comparison
|
|
815
|
+
to preserve exact predecessor restoration. Products own the relation, required
|
|
816
|
+
field and label rules; Kit does not infer them from table or field names.
|
|
817
|
+
|
|
818
|
+
A dependency says `requiredEffectId` must remain while `dependentEffectId`
|
|
819
|
+
remains, and the dependent must reverse first. An omitted dependent identifies
|
|
820
|
+
an outside live reference which protects the required effect. Retention is
|
|
821
|
+
transitive. `order: false` declares a retention-only edge, for example a retained
|
|
822
|
+
owner requiring a separately owned MANY join while removal must still order that
|
|
823
|
+
join before its owner. Kit validates references and rejects unsupported removal
|
|
824
|
+
cycles. It neither disables foreign keys nor deletes schemas or cascades.
|
|
825
|
+
|
|
826
|
+
`snapshot.atomicGroups` can declare effects that must reverse together, such as
|
|
827
|
+
an owner and its required MANY joins. Overlapping groups form one unit. Retaining
|
|
828
|
+
any member protects the unit, with explicit atomic-dependency evidence. Kit
|
|
829
|
+
orders both units and their members by dependencies. A batch normally processes
|
|
830
|
+
at most `batchSize` effects; an indivisible first unit may exceed that count and
|
|
831
|
+
is still bounded by the exact D1 statement/request limits. It is never split to
|
|
832
|
+
make a batch fit. This prevents an owner edit between batches from retaining a
|
|
833
|
+
required-empty owner after its join was already reversed. Edited
|
|
834
|
+
effects and protected dependencies get durable retained dispositions while their
|
|
835
|
+
original provenance, applied hash/revision, source claim, originals and any
|
|
836
|
+
predecessor history remain unchanged. Unchanged independent effects reverse,
|
|
837
|
+
and exact predecessors reactivate only for effects actually reversed. Conflicts
|
|
838
|
+
advance the durable cursor and fence, so mixed and all-conflicted imports reach
|
|
839
|
+
terminal `undone`. A repeated undo does not reconsider a retained decision.
|
|
840
|
+
|
|
841
|
+
`readD1ImportUndoReport(id, context, { offset, limit })` returns the policy, job,
|
|
842
|
+
full cursor/total, a disposition page and `nextOffset` (null at the end). The
|
|
843
|
+
default page has at most 100 entries; the explicit maximum is 1000. Every page
|
|
844
|
+
validates the full custody, provenance, source, history and report integrity
|
|
845
|
+
closure before returning. Each disposition identifies job/effect, native entity,
|
|
846
|
+
field/target where supplied, edited/dependency reason, original and observed
|
|
847
|
+
revision/value/attachment/owned-content hashes and values, domain evidence and
|
|
848
|
+
protecting causes. The original mapping plan remains retained and unchanged.
|
|
849
|
+
Report rows are bounded by the ordinary D1 statement/value limits; the engine
|
|
850
|
+
counts all snapshot reads and guards against the request's remaining budget.
|
|
851
|
+
Accumulated dispositions use byte-bounded JSON set comparisons for their exact
|
|
852
|
+
provenance IDs, owner IDs and reversal flags, including SQL storage types. Exact
|
|
853
|
+
report bytes, positions and a scoped row count remain fenced in the same batch;
|
|
854
|
+
the report hash is not a substitute for those comparisons. Resuming undo and
|
|
855
|
+
reading terminal pages therefore do not add one SQL predicate per prior effect.
|
|
856
|
+
For large native sets, a snapshot can supply `nativeRows`, the complete
|
|
857
|
+
destination set returned by `targets.list`. Kit requires `targets.matchesMany`,
|
|
858
|
+
rejects duplicate or differently bound rows, and compares each observation's
|
|
859
|
+
complete canonical value to that set (including absence). Kit then uses the
|
|
860
|
+
adapter's exact full-set predicates instead of repeating `matches` per effect.
|
|
861
|
+
Custom observation guards and the complete dependency guards still apply.
|
|
862
|
+
The native dependency snapshot and integrity closure are complete per batch,
|
|
863
|
+
so adapters must use bounded set queries; `batchSize` bounds mutations rather
|
|
864
|
+
than promising constant-memory reads of an arbitrarily large native graph.
|
|
865
|
+
|
|
866
|
+
### Reviewed conversion of existing strict jobs
|
|
867
|
+
|
|
868
|
+
The public D1 entry exports two operations:
|
|
869
|
+
|
|
870
|
+
```ts
|
|
871
|
+
const preview = await previewD1PreservedUndo(jobId, context);
|
|
872
|
+
// { job, policy, reviewedFingerprint, dispositions }
|
|
873
|
+
|
|
874
|
+
// Show the exact preview for review before submitting its fingerprint.
|
|
875
|
+
const job = await convertD1ReversalPolicy(jobId, context, {
|
|
876
|
+
to: 'preserve-conflicts',
|
|
877
|
+
reviewedFingerprint: preview.reviewedFingerprint,
|
|
878
|
+
}); // ImportJob
|
|
879
|
+
```
|
|
880
|
+
|
|
881
|
+
`previewD1PreservedUndo` requires current `undo` authorization and the existing
|
|
882
|
+
job binding. It executes SELECT-only SQL and checks the complete native and
|
|
883
|
+
dependency snapshot supplied by `targets.preparePreservedUndo`, including its
|
|
884
|
+
guards. It does not change policy, start reversal or store an approval.
|
|
885
|
+
|
|
886
|
+
`convertD1ReversalPolicy` accepts only a strict job in `completed` or `undoing`
|
|
887
|
+
state, including one stopped by `NATIVE_EDIT_CONFLICT`. It requires the same
|
|
888
|
+
current undo authority and binding as an undo, and the exact
|
|
889
|
+
`reviewedFingerprint` returned for the reviewed snapshot. Conversion atomically
|
|
890
|
+
commits the new policy and its binding, an append-only conversion journal record
|
|
891
|
+
with the policy hash, and baseline report dispositions for effects already
|
|
892
|
+
reversed by strict undo. It does not repeat those native reversals or rewrite
|
|
893
|
+
provenance, originals, source claims or history. Refusal leaves no partial
|
|
894
|
+
conversion state; there is no automatic policy switch or silent re-conversion.
|
|
895
|
+
An earlier strict reversal must still have its exact restored before-image or
|
|
896
|
+
absence. A recreated or superseding target makes that baseline unfenceable;
|
|
897
|
+
conversion refuses rather than misreporting retained later work as reversed.
|
|
898
|
+
|
|
899
|
+
| Job or review condition | Conversion result |
|
|
900
|
+
| --- | --- |
|
|
901
|
+
| Strict `completed` or `undoing`, current undo authority and binding, exact reviewed snapshot, complete fenceable adapter observations | Converts atomically to `preserve-conflicts`. |
|
|
902
|
+
| Already converted | `IMPORT_CONVERSION_ALREADY_CONVERTED`; no new conversion. |
|
|
903
|
+
| Any other state or an ineligible policy | `IMPORT_CONVERSION_UNAVAILABLE`, with the state/policy reason. |
|
|
904
|
+
| Superseded job | `IMPORT_CONVERSION_UNAVAILABLE`, with the superseded reason. |
|
|
905
|
+
| Missing or incomplete adapter guards, an unfenceable later reversal, or a conversion that would release predecessor history | `IMPORT_CONVERSION_UNAVAILABLE`, with the unfenceable reason. |
|
|
906
|
+
| The reviewed fingerprint no longer matches the complete snapshot | `IMPORT_CONVERSION_REVIEW_CHANGED`; obtain and review a fresh preview. |
|
|
907
|
+
| Missing or revoked undo authority, or a different job binding | The ordinary authorization/binding refusal; conversion grants no additional authority. |
|
|
908
|
+
|
|
909
|
+
Conversion-specific refusals are typed `D1ReversalPolicyConversionError`; inspect
|
|
910
|
+
its `code` and the reason distinguishing state, superseded and unfenceable jobs.
|
|
911
|
+
An already preserved job cannot use this operation to obtain another conversion.
|
|
912
|
+
|
|
913
|
+
Later preserved reversal rechecks the complete current native/dependency
|
|
914
|
+
snapshot in every batch. Converted jobs cannot release predecessor history to
|
|
915
|
+
make reversal proceed. No later work, money or signed document may be detached
|
|
916
|
+
or deleted to satisfy conversion. A seed implementing reviewed compensation
|
|
917
|
+
must still supply an approval-bound adapter that fences every reversal batch to
|
|
918
|
+
its own approved snapshot; reviewing conversion does not replace that product
|
|
919
|
+
approval contract.
|
|
920
|
+
|
|
921
|
+
`readD1ImportUndoReport(jobId, context)` exposes optional `conversion` evidence
|
|
922
|
+
and an `outcome` of `pending`, `undone` or `compensated`. The engine checkpoint
|
|
923
|
+
still ends at `job.state === 'undone'`. Consumers must use the report's `outcome`
|
|
924
|
+
for the product result and must never present `compensated` as `undone`.
|
|
925
|
+
|
|
926
|
+
Existing installations must apply `d1ReversalPolicyConversionSchema` after the
|
|
927
|
+
existing D1 schemas in a new product-owned numbered migration before using
|
|
928
|
+
these operations. `initializeD1ImportStorage` installs it automatically. The
|
|
929
|
+
earlier preserved-undo evolution alone does not install conversion storage.
|
|
930
|
+
Existing plan/apply/read/undo calls continue to work on that earlier schema;
|
|
931
|
+
the new conversion and preview operations refuse with the `schema` reason until
|
|
932
|
+
their additive migration is installed. Updating kit source does not silently
|
|
933
|
+
adopt conversion or require an edit to an existing seed migration.
|
|
934
|
+
|
|
935
|
+
Current rer 0.2.77 lacks this API. A seed adopts a reviewed vendored Base revision
|
|
936
|
+
containing the capability, or a future rer release that supplies it, and proves
|
|
937
|
+
its migration, current authority, conversion and later reversal through its own
|
|
938
|
+
public door. Base's tests do not prove that adoption. Until then, existing strict
|
|
939
|
+
jobs retain their policy and the seed's compensation-unavailable refusal remains
|
|
940
|
+
in force. Adoption, seed release and deployment are separate from this Base
|
|
941
|
+
source change.
|
|
942
|
+
|
|
943
|
+
`tests/d1-preserved-scale.test.ts` exercises 34,000 distinct effects in real
|
|
944
|
+
SQLite and local workerd D1. It measures complete public resume, report,
|
|
945
|
+
original-read and native-export calls, reopens durable databases, and injects
|
|
946
|
+
late changes in the final report/native chunks. The fixtures retain their plans
|
|
947
|
+
and originals through public creation, then seed completed history directly;
|
|
948
|
+
they test reversal and read scale without replaying 34,000 prior apply writes.
|
|
949
|
+
Separate small public apply/undo cases exercise mixed present, absent, edited,
|
|
950
|
+
unrelated and unchanged rows, typed values, authorization and publication
|
|
951
|
+
rollback. These are local provider and custody checks; hosted Worker resource
|
|
952
|
+
limits and a consumer's native adapter remain separate acceptance work.
|
|
953
|
+
The Node 26.8.1 SELECT-only read regression run measured at most 211 executed statements per
|
|
954
|
+
34,000-effect public invocation, 39 parameters per statement, 9,233 SQL bytes
|
|
955
|
+
and 1,832,989 aggregate bound bytes. A report page validated all 34,000 durable
|
|
956
|
+
dispositions in 103 statements; the complete native export returned 33,999
|
|
957
|
+
remaining rows and their provenance in 61 statements after one retained edit
|
|
958
|
+
and one actual reversal. Exact retained originals survived both reopen paths.
|
|
959
|
+
|
|
960
|
+
Creation binds each exact plan chunk once. A Kit trigger captures the original
|
|
961
|
+
INSERT `NEW.body` in `ik_d1_creation_plans`; its update/delete triggers reject
|
|
962
|
+
changes before release. Final guards compare every plan ordinal/body and count
|
|
963
|
+
against that protected snapshot and every original byte against its exact plan
|
|
964
|
+
positions. Only after all guards pass does the engine release and delete the
|
|
965
|
+
snapshot, in the same D1 batch. A malformed plan/original INSERT trigger therefore
|
|
966
|
+
rolls back job, originals and snapshots together. No staging survives successful
|
|
967
|
+
creation, and no public custody crosses transactions. This avoids a second full plan
|
|
968
|
+
copy in D1's serialized batch request while retaining exact byte comparisons.
|
|
969
|
+
A large plan's chunks reach that transaction from private staging instead of
|
|
970
|
+
its own request; the same guards then compare them inside D1.
|
|
971
|
+
|
|
972
|
+
## Bounded private stream custody
|
|
973
|
+
|
|
974
|
+
### Acknowledged preparation and cold verification
|
|
975
|
+
|
|
976
|
+
Consumers may split normalized creation across two requests. Review still calls
|
|
977
|
+
`planD1ImportStream` and computes the complete canonical fingerprint without
|
|
978
|
+
writing custody. After the operator acknowledges that report,
|
|
979
|
+
`prepareD1ImportStream(source, mapping, context)` requires its fingerprint as
|
|
980
|
+
`reviewedFingerprint` and stores independently checked normalized pages under a
|
|
981
|
+
private generation. Its `ImportPreparation` result has `state: 'preparing'`,
|
|
982
|
+
`checkpoint: 0` and `expectedFingerprint`; it is explicitly unverified and is
|
|
983
|
+
never accepted as a public import job. No native form or response exists yet.
|
|
984
|
+
|
|
985
|
+
`publishD1ImportPreparation(id, context)` reads bounded private chunks and hashes
|
|
986
|
+
every value in the original canonical order, checks the expected fingerprint,
|
|
987
|
+
header, codec, complete page and privacy inventories, and current authority.
|
|
988
|
+
Only then does the existing atomic publication create the public stream. The
|
|
989
|
+
consumer can call `resumeD1ImportStream` with the same `d1RequestContext` to apply
|
|
990
|
+
the first bounded native batch without resetting the request budget. Cold
|
|
991
|
+
verification needs no uploaded files, parser or mapper.
|
|
992
|
+
|
|
993
|
+
`readD1ImportPreparation` is owner scoped and distinguishes complete private
|
|
994
|
+
preparation from `preparation-interrupted`. Interrupted writes can be retried
|
|
995
|
+
with the reviewed source or removed by `cancelD1ImportStaging(operationKey,
|
|
996
|
+
context)`. A sealed private generation cannot be overwritten by a stale retry.
|
|
997
|
+
Cancellation, erasure and verified publication cascade private headers and
|
|
998
|
+
receipts. An authority failure can leave private custody for an authorized
|
|
999
|
+
owner to inspect or cancel; it cannot confer publication permission.
|
|
1000
|
+
|
|
1001
|
+
### Preparation across requests
|
|
1002
|
+
|
|
1003
|
+
A large normalized file costs more CPU than one Worker request allows when it
|
|
1004
|
+
is reviewed, prepared and verified in one go (a 100,000-row Typeform export
|
|
1005
|
+
took about 55 s of hosted CPU to review and 62 s to prepare; Workers Paid allows
|
|
1006
|
+
30 s). `prepareD1ImportSteps(source, mapping, context)` does the same work in
|
|
1007
|
+
bounded steps instead. Call it with the same source, mapping and
|
|
1008
|
+
`operationKey`, once per request, until its result is `reviewable`:
|
|
1009
|
+
|
|
1010
|
+
- `preparing`: each step maps, normalizes and stages 5,000 records as a private,
|
|
1011
|
+
sealed generation under the usual privacy admission (registered identities
|
|
1012
|
+
before payload, erasure tombstones checked at every write).
|
|
1013
|
+
- `verifying`: each step reads 125 staged pages, makes cold verification's checks
|
|
1014
|
+
(page projection, privacy index, index and policy hashes, counts) and writes
|
|
1015
|
+
them into the complete canonical fingerprint with a SHA-256 whose state is
|
|
1016
|
+
kept in D1 (`ik_d1_preparation_steps`), fenced by every generation's
|
|
1017
|
+
revision. A changed, missing or reordered page fails the step or changes the
|
|
1018
|
+
fingerprint; nothing the caller sends can supply hash state. The last step
|
|
1019
|
+
copies the generations into one sealed preparation in SQL.
|
|
1020
|
+
- `reviewable`: carries `fingerprint`, `counts` and `diagnostics`, the review.
|
|
1021
|
+
The fingerprint equals the one `planD1ImportStream` computes for the same
|
|
1022
|
+
source. Show it, and after the owner acknowledges call
|
|
1023
|
+
`publishD1ImportPreparation(id, { ...context, reviewedFingerprint })`, which
|
|
1024
|
+
publishes atomically without reading the pages again while the preparation is
|
|
1025
|
+
still at the revision the steps verified (any change sends it through full
|
|
1026
|
+
cold verification). Then apply with `resumeD1ImportStream` as before.
|
|
1027
|
+
|
|
1028
|
+
Unlike `planD1ImportStream`, this review is computed from private custody, so
|
|
1029
|
+
the steps write before the owner acknowledges; that custody is private,
|
|
1030
|
+
owner-scoped and removed by `cancelD1ImportStaging(operationKey, context)`,
|
|
1031
|
+
destination field erasure, or header retirement of its operation key. A step
|
|
1032
|
+
generation erased or changed between steps ends the preparation with
|
|
1033
|
+
`IMPORT_PREPARATION_INTERRUPTED`: cancel and prepare again. A step whose request
|
|
1034
|
+
ended after it staged its slice but before recording it is staged again by the
|
|
1035
|
+
next call, and cancellation or retirement removes every generation the steps
|
|
1036
|
+
staged, recorded or not. A different upload under the same operation key is
|
|
1037
|
+
refused (`OPERATION_KEY_CONFLICT`), before and after publication: the published
|
|
1038
|
+
stream keeps the steps' source digest (`ik_d1_stream_sources`), so only the same
|
|
1039
|
+
upload returns `published`, and a key an import published in one request is
|
|
1040
|
+
refused here. Measured on default-seed's people
|
|
1041
|
+
adapter: 100,000 rows in 40 requests, none above 2 s of Node CPU against
|
|
1042
|
+
15 s for the one-request review (see the kit test
|
|
1043
|
+
`tests/d1/preparation-steps.test.ts`, run at full size in `verify:scale`).
|
|
1044
|
+
Apply `d1PreparationStepsSchema` as a new numbered migration after
|
|
1045
|
+
`d1PreparationSchema`; `initializeD1ImportStorage` applies it.
|
|
1046
|
+
|
|
1047
|
+
Apply the additive `d1PreparationSchema` after `d1CustodyStateSchema`.
|
|
1048
|
+
`initializeD1ImportStorage` applies that order and preserves existing jobs.
|
|
1049
|
+
The new tables contain safe normalized header custody and its exact receipt;
|
|
1050
|
+
consumers must include both in their archive and erasure inventories. The
|
|
1051
|
+
ordinary `createD1ImportStream` and legacy job APIs retain their original
|
|
1052
|
+
verification and publication behavior.
|
|
1053
|
+
|
|
1054
|
+
Apply the separate additive `d1StagingSchema` after the erasure and preserved
|
|
1055
|
+
undo evolutions. It adds private parents, pages, chunks, original availability
|
|
1056
|
+
metadata and a hash-only identity index. It does not modify an applied migration,
|
|
1057
|
+
the synchronous SQLite protocol, or ordinary D1 original-byte custody.
|
|
1058
|
+
|
|
1059
|
+
Each attempted stream is bound to the exact reviewed fingerprint, principal,
|
|
1060
|
+
destination, namespace, mapping version, operation key and random generation.
|
|
1061
|
+
The whole request reserves its statement budget before private writes. Every
|
|
1062
|
+
stage transaction checks current authority at both ends, the exact binding,
|
|
1063
|
+
expected mutation revision and source/target tombstones. Complete page identity
|
|
1064
|
+
indexes are registered before payload storage. Any page/index mutation
|
|
1065
|
+
invalidates registration; changing an index also removes its private payload
|
|
1066
|
+
so interrupted custody cannot lose privacy discovery. Any child mutation
|
|
1067
|
+
invalidates sealing. Final seal
|
|
1068
|
+
checks page membership, ordinals, chunk sizes and counts against the reviewed
|
|
1069
|
+
manifests. Each inserted row is read back inside its transaction and compared
|
|
1070
|
+
column-for-column with the intended bytes. Exact revision fences then bind
|
|
1071
|
+
those verified rows and hashes to publication,
|
|
1072
|
+
including mutations that restore the original value.
|
|
1073
|
+
|
|
1074
|
+
Each transport contains at most 2 MiB of conservatively counted serialized JSON,
|
|
1075
|
+
including SQL, escaping, binary binding expansion and actual authority guards.
|
|
1076
|
+
A row set is at most 768 KiB before the outer transport encoding; payload chunks
|
|
1077
|
+
are at most 64 Ki UTF-16 code units and do not split surrogate pairs. Statement
|
|
1078
|
+
binding/value limits and the shared 1,000-execution request budget still apply.
|
|
1079
|
+
One decoded normalized page and one pending row set are retained at a time.
|
|
1080
|
+
|
|
1081
|
+
Publication is a small guarded SQL-only transaction. It copies private custody
|
|
1082
|
+
into existing job, stream, original and index tables with `INSERT ... SELECT`,
|
|
1083
|
+
compares every copied set against its exact unchanged source, releases the
|
|
1084
|
+
transaction-local creation snapshots, and removes the private generation.
|
|
1085
|
+
There is no public job, stream or native effect before publication. Existing
|
|
1086
|
+
published read, apply, replay and undo paths keep their custody and native guards.
|
|
1087
|
+
|
|
1088
|
+
Fresh creation, cold publication and exact replay return the public status from
|
|
1089
|
+
one read-only SQL transaction under current create authority. Parent binding,
|
|
1090
|
+
ordered complete pages, child status and immutable origin membership belong to
|
|
1091
|
+
that same snapshot, with authority checked again at its end. No assertion cleanup
|
|
1092
|
+
or other write follows those reads. The receipt's transaction commit is the
|
|
1093
|
+
return boundary; a later change does not invalidate that completed snapshot.
|
|
1094
|
+
Admission reserves the actual packed receipt and the reachable generation-only
|
|
1095
|
+
cleanup, winner lookup and second authorized receipt before publication. Fresh
|
|
1096
|
+
creation includes that same reservation before its first private write.
|
|
1097
|
+
|
|
1098
|
+
A fatal interruption may leave incomplete private custody. It is not a resumable
|
|
1099
|
+
job and exposes no payload through public job APIs. A fresh authorized create
|
|
1100
|
+
with the same binding replaces that private generation; stale writers and
|
|
1101
|
+
cleanup cannot modify the replacement. `cancelD1ImportStaging(operationKey,
|
|
1102
|
+
context)` removes only the current operator's exact private generation under
|
|
1103
|
+
current create authority. An optional reviewed fingerprint fences cancellation.
|
|
1104
|
+
The same owner can cancel an older mapping version in that namespace without
|
|
1105
|
+
reopening its payload or adopting the older mapping implementation.
|
|
1106
|
+
Cancellation and retry never remove a published import. Failed cleanup after
|
|
1107
|
+
lost authority leaves inaccessible custody for an authorized retry, cancellation
|
|
1108
|
+
or destination privacy erasure; it is not silently treated as a completed import.
|
|
1109
|
+
No time-based automatic deletion is introduced.
|
|
1110
|
+
|
|
1111
|
+
Destination privacy erasure discovers staged target/source identities across
|
|
1112
|
+
importing principals. The same atomic erasure transaction writes permanent
|
|
1113
|
+
tombstones and cascades deletion through every intersecting private generation,
|
|
1114
|
+
including partial payloads. Registration after an earlier erasure fails on those
|
|
1115
|
+
tombstones. Other destinations remain outside the operation.
|
|
1116
|
+
|
|
1117
|
+
Ordinary version-three original-byte receipts without structural stream origin
|
|
1118
|
+
remain usable and unchanged. Every initialization, including fresh installations,
|
|
1119
|
+
atomically runs `d1StreamOriginSchema` and `d1LegacyStreamReceiptUpgrade`:
|
|
1120
|
+
actual stream-page membership is captured durably, and only actual members move
|
|
1121
|
+
from version three to version four. Captured origins and version-four receipts
|
|
1122
|
+
reject missing or changed parent membership. If all structural evidence for a
|
|
1123
|
+
legacy version-three job was already lost before origin capture, its stream
|
|
1124
|
+
history cannot be classified; IDs, payloads and nullable reversal-policy bindings
|
|
1125
|
+
are never used to guess it. Setup and conversion are idempotent; products retain
|
|
1126
|
+
ownership of numbered migrations.
|
|
1127
|
+
|
|
1128
|
+
Preserved-conflict undo reports can contain cross-effect native evidence.
|
|
1129
|
+
Privacy erasure validates their existing report and policy bindings, fences the
|
|
1130
|
+
exact prior report rows, then records explicit redacted dispositions. Erased
|
|
1131
|
+
effects retain only source/effect digests and their prior disposition. Remaining
|
|
1132
|
+
destination reports lose free-form evidence, native IDs, values and value hashes;
|
|
1133
|
+
their disposition/reason remains visible and their report hash is rebound in the
|
|
1134
|
+
same transaction. Erasure never trusts a newly read hash as authority for changed
|
|
1135
|
+
report contents. Strict undo keeps its existing native edit conflict behavior.
|
|
1136
|
+
|
|
1137
|
+
Canonical string escaping emits slices of at most one Mi UTF-16 code units,
|
|
1138
|
+
with surrogate pairs kept together. Fingerprints and retained chunks consume
|
|
1139
|
+
those slices without joining a whole large literal. Planning size walks share
|
|
1140
|
+
only immutable string byte counts, bounded to 4,096 entries and 512 Ki UTF-16
|
|
1141
|
+
code units per plan; container reads and size-limit checks remain fresh.
|
|
1142
|
+
Mapper properties are captured once in canonical order before admission,
|
|
1143
|
+
credential checks and retained cloning; shared container graphs are not expanded
|
|
1144
|
+
before their logical size is admitted. Ordinary source cloning keeps its eager
|
|
1145
|
+
canonical JSON semantics, while stream pages share immutable source strings.
|
|
1146
|
+
|
|
1147
|
+
The eager planner's content-size cap applies to each normalized child plan.
|
|
1148
|
+
The complete stream is fingerprinted as one ordered logical plan without
|
|
1149
|
+
applying the eager representation's aggregate cap to the compressed stream.
|
|
1150
|
+
Products must still qualify their complete upload, expansion, CPU and Worker
|
|
1151
|
+
memory limits. The bounded transport is not a claim of measured total isolate
|
|
1152
|
+
high-water or hosted resource acceptance.
|
|
1153
|
+
|
|
1154
|
+
|
|
1155
|
+
## Optional bounded native transport
|
|
1156
|
+
|
|
1157
|
+
`maxBatchBytes` is an explicit consumer policy. Omission preserves ordinary
|
|
1158
|
+
D1 transport behavior and original-custody semantics. With a policy, apply and
|
|
1159
|
+
strict reversal gather fewer whole source entries when their complete prepared
|
|
1160
|
+
SQL, guards and progress writes would exceed the serialized batch ceiling. The
|
|
1161
|
+
configured record cap remains an upper bound; one native entity stays atomic.
|
|
1162
|
+
|
|
1163
|
+
A native entity can exceed that transport ceiling even when its reviewed source
|
|
1164
|
+
is admitted. An adapter supporting this case supplies `retainedSql(parts, params,
|
|
1165
|
+
indexes)` for safe normalized text values and exact native-value predicates.
|
|
1166
|
+
The literal SQL parts define the placeholders structurally. The engine never
|
|
1167
|
+
rewrites arbitrary SQL, quoted question marks, comments or numbered parameters.
|
|
1168
|
+
Original uploads, credentials, session tokens, capability bindings and arbitrary
|
|
1169
|
+
environment parameters must never be marked as retainable.
|
|
1170
|
+
|
|
1171
|
+
Install the separate additive `d1NativeStagingSchema`. Its private parent, complete
|
|
1172
|
+
hashed source/target index and ordered value chunks are bound to the current job,
|
|
1173
|
+
plan fingerprint, principal, destination, namespace, action, fence and generation.
|
|
1174
|
+
Related retained plan snapshots contribute their entire privacy membership. The
|
|
1175
|
+
complete index is verified before any value bytes are accepted. Every bounded
|
|
1176
|
+
write checks current authority, the originally observed custody/native revisions,
|
|
1177
|
+
and tombstones; inserted columns are read back exactly before the next write.
|
|
1178
|
+
All statements, readbacks, cleanup and return reads are admitted against the one
|
|
1179
|
+
request SQL budget before a private write. Every stage batch has at most 2 MiB
|
|
1180
|
+
of serialized transport, including SQL, JSON escaping and all guard parameters.
|
|
1181
|
+
|
|
1182
|
+
`stagingRead` captures a mutation stamp before reading native values. Its guards
|
|
1183
|
+
must cover all fields and absence, including edits that retain a public revision.
|
|
1184
|
+
`stagingPublication` protects the exact post-state while private bytes are removed.
|
|
1185
|
+
It installs a known receipt only after full native checks, releases its temporary
|
|
1186
|
+
fences, and supplies scalar final predicates for that expected state. Every native
|
|
1187
|
+
mutation, including direct stamp changes, must invalidate the receipt; a receipt
|
|
1188
|
+
cannot be installed without its matching publication fence. The engine does the
|
|
1189
|
+
same for custody. Read-only final checks and current authority follow every
|
|
1190
|
+
cleanup mutation. There is no assertion-table cleanup after those checks.
|
|
1191
|
+
|
|
1192
|
+
Final effects read retained values through SQL and commit native rows, provenance
|
|
1193
|
+
and checkpoint in one bounded atomic transaction. No partially written native
|
|
1194
|
+
entity or private staging object is returned as a public job. Fatal requests may
|
|
1195
|
+
leave indexed private values; cold retry replaces only that owner's generation.
|
|
1196
|
+
`cancelD1NativeStaging(jobId, context)` requires current owner authority and removes
|
|
1197
|
+
private values without changing native progress. Reversal also cancels abandoned
|
|
1198
|
+
forward stages. Destination privacy erasure discovers stages across importing
|
|
1199
|
+
principals and removes every affected private copy together with its tombstones.
|
|
1200
|
+
An old writer cannot republish after cancellation, erasure or a winning retry.
|
|
1201
|
+
|
|
1202
|
+
The additive custody revision table backfills existing jobs without rewriting their
|
|
1203
|
+
plans, original versions, provenance or reversal policy. Product adapters must
|
|
1204
|
+
ship their own additive native mutation-stamp and publication-fence schema in the
|
|
1205
|
+
same evolution. Existing published reads, ordinary original custody and default
|
|
1206
|
+
adapters keep their existing public contracts.
|
|
1207
|
+
|
|
1208
|
+
Stream page admission computes exact expanded size and dictionary weight once
|
|
1209
|
+
per unchanged private page and reuses those local measurements for coalescing.
|
|
1210
|
+
It does not cache mutable plans or change ordinary planner admission limits.
|
|
1211
|
+
|
|
1212
|
+
With the transport policy enabled, maintained native mutations carry explicit
|
|
1213
|
+
intended row changes. Final checks compare the complete known provenance, source
|
|
1214
|
+
claims, decisions, journal, job fields and custody metadata against the validated
|
|
1215
|
+
preflight plus those changes. A receipt is installed only after these exact
|
|
1216
|
+
comparisons. Transactions that fit directly perform read-only comparisons and
|
|
1217
|
+
current-authority checks after assertion cleanup as well. Ordinary consumers
|
|
1218
|
+
without the policy retain their existing transaction contract.
|
|
1219
|
+
|
|
1220
|
+
Privacy erasure uses the same bounded value transport when replacement plans or
|
|
1221
|
+
native guards make its complete transaction larger than the selected ceiling.
|
|
1222
|
+
Private preparation does not delete native data or redact public custody. Its
|
|
1223
|
+
index covers all retained preflight and replacement values, including related
|
|
1224
|
+
reports, before payload writes. Destination privacy authority is checked for
|
|
1225
|
+
every write. The final SQL transaction removes native data, replaces/redacts all
|
|
1226
|
+
affected custody and reports, installs tombstones and removes private copies
|
|
1227
|
+
together. Interrupted private erasure can be retried under current privacy
|
|
1228
|
+
authority. `cancelD1ErasureStaging(jobId, context)` removes only that privacy
|
|
1229
|
+
operator's private erasure generation; it does not require ownership of the
|
|
1230
|
+
other operator's public import or change that import. Original upload and
|
|
1231
|
+
attachment bytes are refused by default. Explicit `withdraw-affected-artifacts`
|
|
1232
|
+
erasure may stage the original-bearing plan under its complete privacy identity
|
|
1233
|
+
index (either explicit withdrawal policy); the final transaction withdraws the selected originals and clears every
|
|
1234
|
+
private copy. Attachment bytes remain refused. Ordinary apply/undo staging does
|
|
1235
|
+
not adopt this erasure-only exception.
|
|
1236
|
+
|
|
1237
|
+
Stream resume and undo reserve their outer authority and complete snapshot
|
|
1238
|
+
return before a child can retain or publish values. Nested calls share the
|
|
1239
|
+
same 1,000-statement request counter, including cleanup and final read-only checks.
|
|
1240
|
+
|
|
1241
|
+
|
|
1242
|
+
### Bounded privacy publication
|
|
1243
|
+
|
|
1244
|
+
For a consumer that sets `maxBatchBytes`, erasure groups small retained jobs
|
|
1245
|
+
using numeric stored-byte and expanded-plan measurements of their complete
|
|
1246
|
+
ownership closure. Set reads share a bounded page; larger closures occupy a
|
|
1247
|
+
page alone. Every payload read requires the monotonic custody revisions captured
|
|
1248
|
+
with those measurements, so a concurrent write cannot enlarge a measured page.
|
|
1249
|
+
Decoded and rewritten plans are released before advancing to the next group.
|
|
1250
|
+
Only small job headers, ownership IDs, ordered hashes, compact SQL references
|
|
1251
|
+
and numeric transport reservations span those pages. A
|
|
1252
|
+
direct transaction is retained only while it fits the configured whole batch
|
|
1253
|
+
bound; a larger transaction uses the existing private native value tables.
|
|
1254
|
+
There is no new default transport limit on ordinary omitted-policy APIs.
|
|
1255
|
+
|
|
1256
|
+
The additive `d1CustodyStateSchema` installs `ik_d1_custody_states` and monotonic
|
|
1257
|
+
mutation triggers covering job fields, plan and original copies, provenance,
|
|
1258
|
+
claims, decisions, history, journal, reversal reports and stream membership.
|
|
1259
|
+
Prior jobs receive counters without changes to their public bytes. SQLite's
|
|
1260
|
+
REPLACE deletes the row it conflicts with without firing DELETE triggers, so apply
|
|
1261
|
+
the additive `d1CustodyStateReplaceSchema` as a new numbered migration after
|
|
1262
|
+
`d1CustodyStateSchema` and `d1RetirementSchema` (`initializeD1ImportStorage`
|
|
1263
|
+
applies it last). It ignores an insert over an existing counter, by job id or
|
|
1264
|
+
by rowid (the backfill's `INSERT OR IGNORE` still adds missing ones), refuses to
|
|
1265
|
+
change a counter's rowid with `IMPORT_CUSTODY_STATE_CHANGED`, refuses an `ik_jobs`
|
|
1266
|
+
id that exists or was retired with `IMPORT_JOB_ID_REUSED`, and makes an insert or
|
|
1267
|
+
update (`UPDATE OR REPLACE` deletes a colliding row the same way) that would
|
|
1268
|
+
replace a claim, provenance row (by id or live target), history row (by id or
|
|
1269
|
+
successor), stream page or stream advance the revision of every job owning the
|
|
1270
|
+
replaced row. SQLite's rowid is one more unique key of every custody table and
|
|
1271
|
+
tombstone, so a row replaced by rowid advances its owners too (a tombstone's
|
|
1272
|
+
owners are found through the plan index, as its own triggers find them).
|
|
1273
|
+
A counter is
|
|
1274
|
+
captured before each retained report is validated; old cross-effect evidence is
|
|
1275
|
+
never copied into private transport. The report's existing policy and content
|
|
1276
|
+
hashes are validated before its explicitly redacted replacement is constructed.
|
|
1277
|
+
|
|
1278
|
+
The first read pass reserves both later read passes and all exact writes,
|
|
1279
|
+
authority guards, readbacks, publication and cleanup before any private write.
|
|
1280
|
+
A second pass reproduces the reviewed page hashes and registers all privacy
|
|
1281
|
+
identities. Only then can a third pass retain bounded value chunks. Each batch
|
|
1282
|
+
checks current authority and the original job, custody and native revisions;
|
|
1283
|
+
each stored row is compared exactly with the intended bytes before registration
|
|
1284
|
+
or sealing can progress. Private transport batches, including SQL, JSON escaping
|
|
1285
|
+
and guards, never exceed 2 MiB or a lower configured bound. All passes share the
|
|
1286
|
+
same 1,000-query request budget.
|
|
1287
|
+
|
|
1288
|
+
The final SQL-only transaction removes the native entity and all retained source
|
|
1289
|
+
copies together, updates fingerprints and redacted reversal bindings, writes
|
|
1290
|
+
privacy tombstones and discards private generations. Exact expected state is
|
|
1291
|
+
checked before sealing. Read-only receipt, tombstone, absence and current
|
|
1292
|
+
authority checks follow the last cleanup mutation. A fatal request may retain an
|
|
1293
|
+
indexed, unpublished private generation; a cold retry rebuilds it against current
|
|
1294
|
+
state. `cancelD1ErasureStaging(jobId, context)` removes only the current principal's
|
|
1295
|
+
private erasure generations in that destination and namespace, without changing
|
|
1296
|
+
public jobs or native values. Destination privacy erasure discovers matching
|
|
1297
|
+
private generations across principals and deletes every retained copy atomically.
|
|
1298
|
+
|
|
1299
|
+
Configured bounded creation repeats its complete job, original custody and plan
|
|
1300
|
+
index membership checks after assertion cleanup. The same final read-only
|
|
1301
|
+
checks are included in its query admission. Intended-state tracking preserves
|
|
1302
|
+
explicit absence checks for deleted provenance and history primary keys,
|
|
1303
|
+
including related owners, until final publication has been sealed.
|
|
1304
|
+
|
|
1305
|
+
Erasing a formerly effectful entry retains its original target digest in the
|
|
1306
|
+
fingerprinted plan as `erasedTargetIdentity`. Cold bounded reads, apply and undo
|
|
1307
|
+
compare the exact original source/target index membership and require both
|
|
1308
|
+
privacy tombstones. A source that never produced an effect keeps its separate
|
|
1309
|
+
empty-target membership. An earlier unaccepted erased snapshot without the
|
|
1310
|
+
original digest cannot be promoted into a new bounded binding by guessing or
|
|
1311
|
+
trusting a replacement index row; it fails closed. Prior ordinary live custody
|
|
1312
|
+
needs no conversion: erasure captures its reviewed target before removing values.
|
|
1313
|
+
Omitted-policy ordinary APIs retain their behavior.
|
|
1314
|
+
|
|
1315
|
+
All privacy replay passes reuse the original numeric page/owner descriptors and
|
|
1316
|
+
revision predicates. They never rediscover a changed ownership graph. A page's
|
|
1317
|
+
first successful snapshot also records its exact read count; later snapshots
|
|
1318
|
+
refuse an extra read before executing it. Admission conservatively reserves two
|
|
1319
|
+
full first-pass read costs, including initial measurements that replay does not
|
|
1320
|
+
repeat, plus every private write, final publication and failure discard. Thus a
|
|
1321
|
+
concurrent native import can invalidate erasure without consuming its cleanup
|
|
1322
|
+
reservation by expanding the newly visible history graph.
|