@runeditrun/rer 0.2.101 → 0.2.102
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/global/CHANGELOG.md +8 -0
- package/modules/deploy-kit/d1-import.mjs +11 -1
- package/modules/deploy-kit/migrations.mjs +14 -1
- package/modules/import-kit/src/d1/README.md +2 -1
- package/modules/import-kit/src/d1/apply-reads.ts +38 -12
- package/modules/import-kit/src/d1/apply.ts +15 -5
- package/modules/import-kit/src/d1/redaction-step.ts +17 -7
- package/package.json +1 -1
package/global/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,14 @@
|
|
|
2
2
|
|
|
3
3
|
## Unreleased
|
|
4
4
|
|
|
5
|
+
## 0.2.102
|
|
6
|
+
|
|
7
|
+
### Fixes
|
|
8
|
+
|
|
9
|
+
- Deploy no longer reports a completed D1 migration as failed when a poll after an active ingest answers "Not currently importing anything." It accepts that answer only when the migration ledger has exactly one row for that migration, marked `applied` with the expected SHA-256. A missing, unapplied or mismatched row, or a failed ledger read, keeps the `d1-import-failed` refusal.
|
|
10
|
+
- Import apply prefetch reads bounded windows of entries that can still publish in the current call, sizing each window against the 1,000-effect cap and a quarter of the remaining publication budget, always including the next entry. It resizes later windows against actual read usage. Request-budget exhaustion inside a bulk read ends the gather like the existing reserve stop; other bulk-read errors still discard the snapshot and use individual reads. The documented budget-boundary behavior from 0.2.101 is unchanged: saved reads can advance a call's checkpoint or surface a later entry's refusal one call earlier, with the same error code and text; that refusal publishes none of the gathered entries and leaves the last consistent durable checkpoint (`modules/import-kit/src/d1/README.md`).
|
|
11
|
+
- Resumable erasure defers a redaction step to a fresh request when earlier attempts have spent the current request's budget and the step cannot shrink further. It returns a `redacting` continuation instead of throwing `IMPORT_D1_REQUEST_LIMIT`, so a People erasure whose step fits a fresh request can continue. A smallest step that cannot fit on its first attempt still refuses.
|
|
12
|
+
|
|
5
13
|
## 0.2.101
|
|
6
14
|
|
|
7
15
|
### Person erasure keeps other people's agent-door records
|
|
@@ -5,6 +5,10 @@ import { importDiagnostic } from './d1-import-diagnostics.mjs';
|
|
|
5
5
|
|
|
6
6
|
export const D1_IMPORT_POLL_LIMIT = 100;
|
|
7
7
|
|
|
8
|
+
// D1 answers a poll with this once the import it was running has finished: a
|
|
9
|
+
// short import can commit, completion marker included, before the next poll.
|
|
10
|
+
const NOT_IMPORTING = 'Not currently importing anything.';
|
|
11
|
+
|
|
8
12
|
// Wrangler's remote --file protocol. D1 imports execute the whole file atomically,
|
|
9
13
|
// including trigger bodies; the query endpoint's semicolon splitting cannot.
|
|
10
14
|
// Cloudflare's own reason for an import that failed or did not finish, kept on the
|
|
@@ -47,7 +51,13 @@ export async function importD1({ envelope, upload, redact = String }, id, sql) {
|
|
|
47
51
|
}
|
|
48
52
|
for (let polls = 0; polls < D1_IMPORT_POLL_LIMIT; polls++) {
|
|
49
53
|
// REST places errors on the envelope; validate nested errors when supplied.
|
|
50
|
-
if (!(response?.success === true && response.type === 'import' && response.status !== 'error' && (!Object.hasOwn(response, 'errors') || Array.isArray(response.errors) && !response.errors.length)))
|
|
54
|
+
if (!(response?.success === true && response.type === 'import' && response.status !== 'error' && (!Object.hasOwn(response, 'errors') || Array.isArray(response.errors) && !response.errors.length))) {
|
|
55
|
+
const failure = fail('d1-import-failed');
|
|
56
|
+
// Only a poll that follows an active import can mean it finished; the caller
|
|
57
|
+
// that knows the migration decides from the D1 ledger, never from this answer.
|
|
58
|
+
if (step === 'poll' && response?.success === false && response.error === NOT_IMPORTING && !response.errors?.length) failure.importNotActive = true;
|
|
59
|
+
throw failure;
|
|
60
|
+
}
|
|
51
61
|
if (response.status === 'complete') {
|
|
52
62
|
if (!(Number.isSafeInteger(response.result?.num_queries) && response.result.num_queries > 0 && typeof response.result.final_bookmark === 'string' && response.result.final_bookmark.length > 0)) throw fail('d1-import-invalid-result');
|
|
53
63
|
return;
|
|
@@ -53,6 +53,16 @@ export async function query(api, id, sql) {
|
|
|
53
53
|
assert(Array.isArray(rows) && rows.length > 0 && rows.every(x => x.success === true && Array.isArray(x.results)), 'd1-query-failed');
|
|
54
54
|
return rows;
|
|
55
55
|
}
|
|
56
|
+
/** D1 answered a poll "not currently importing" after an active ingest: the import may
|
|
57
|
+
* have committed before the poll. Complete only when the ledger holds this migration
|
|
58
|
+
* applied on exactly its bytes; any other answer, or a read that fails, leaves the
|
|
59
|
+
* import's own failure standing. */
|
|
60
|
+
async function ledgerAppliesMigration(api, id, migration) {
|
|
61
|
+
try {
|
|
62
|
+
const rows = (await query(api,id,`SELECT name, sha256, state FROM ${ledger} WHERE name=${literal(migration.name)}`))[0].results;
|
|
63
|
+
return rows.length === 1 && rows[0].state === 'applied' && rows[0].sha256 === migration.sha256;
|
|
64
|
+
} catch { return false; }
|
|
65
|
+
}
|
|
56
66
|
/** `locked`: the caller holds the install's deploy lock (owner.mjs withStateLock). */
|
|
57
67
|
export async function migrate(api, id, migrations, { apply = true, upgrade = apply, locked = false } = {}) {
|
|
58
68
|
validateMigrations(migrations);
|
|
@@ -80,7 +90,10 @@ export async function migrate(api, id, migrations, { apply = true, upgrade = app
|
|
|
80
90
|
? `UPDATE ${ledger} SET sha256=${literal(migration.sha256)}, state='applied' WHERE name=${literal(migration.name)} AND sha256=${literal(claimedFrom)} AND state='applying'`
|
|
81
91
|
: `UPDATE ${ledger} SET state='applied' WHERE name=${literal(migration.name)} AND sha256=${literal(migration.sha256)}`;
|
|
82
92
|
await api.importSql(id,`${migration.sql}\n${separator}${done};`)
|
|
83
|
-
.catch(error => {
|
|
93
|
+
.catch(async error => {
|
|
94
|
+
if (error?.importNotActive && await ledgerAppliesMigration(api,id,migration)) return;
|
|
95
|
+
error.migration ??= migration.name; throw error;
|
|
96
|
+
});
|
|
84
97
|
}
|
|
85
98
|
if (pending.length) {
|
|
86
99
|
// By name and bytes: rows applied after ones that sort above them keep their ids.
|
|
@@ -158,7 +158,8 @@ The product implements `D1TargetAdapter`:
|
|
|
158
158
|
Query only selected keys and preserve native decoding. With private transport,
|
|
159
159
|
adapters that implement `stagingRead` can also provide `stagingReadMany`, with
|
|
160
160
|
the same exact before-read guards for each key. Missing bulk methods retain
|
|
161
|
-
individual reads; any bulk-read error
|
|
161
|
+
individual reads; any bulk-read error except request-budget exhaustion discards the
|
|
162
|
+
snapshot and uses that path. Exhaustion ends the batch like the budget reserve does.
|
|
162
163
|
An inherited bulk method must preserve an overridden single read or throw.
|
|
163
164
|
Bulk observations belong only to the current publication.
|
|
164
165
|
- `matches(rowOrNull, destination, kind, id)` returns an exact SQL boolean snapshot
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { d1NativeLimits } from '../../../d1-transport/limits.mjs';
|
|
2
2
|
import type { PlanEntry, TargetRow } from '../types.ts';
|
|
3
|
+
import { D1RequestLimitError } from './budget.ts';
|
|
3
4
|
import type { D1ImportContext, D1Sql } from './types.ts';
|
|
4
5
|
|
|
5
6
|
const key = (kind: string, id: string): string => JSON.stringify([kind, id]);
|
|
@@ -10,39 +11,64 @@ export interface ApplyReads {
|
|
|
10
11
|
}
|
|
11
12
|
export const targetReadKey = key;
|
|
12
13
|
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
14
|
+
const emptyReads = (): ApplyReads => ({ sources: new Map(), predecessors: new Map(), targets: new Map() });
|
|
15
|
+
const providerQueries = Math.floor((d1NativeLimits.variableNumber - 1) / 2);
|
|
16
|
+
|
|
17
|
+
/** How far the next bulk read may reach: the end (exclusive) of the entries from
|
|
18
|
+
* `from` that can still publish in this call. It stops at the 1000-effect batch
|
|
19
|
+
* cap, and where the reads, counted as the contract's worst case of one query per
|
|
20
|
+
* target key, would take more than a quarter of what the request has left after the
|
|
21
|
+
* publication so far. The entry at `from` is always included; later windows are
|
|
22
|
+
* sized again against the budget those reads actually used. */
|
|
23
|
+
export function readEnd(pending: PlanEntry[], from: number, gathered: number, budget: number, reads?: ApplyReads): number {
|
|
24
|
+
const identities = new Set<string>(), keys = new Set<string>();
|
|
25
|
+
let effects = 0, end = from;
|
|
26
|
+
for (; end < pending.length; end++) {
|
|
27
|
+
const entry = pending[end];
|
|
28
|
+
if (end > from && gathered + effects + entry.effects.length > 1000) break;
|
|
29
|
+
identities.add(entry.identity);
|
|
30
|
+
for (const effect of entry.effects) { const k = key(effect.targetKind, effect.targetId); if (!reads?.targets.has(k)) keys.add(k); }
|
|
31
|
+
const queries = Math.ceil(identities.size / d1NativeLimits.variableNumber) + keys.size + Math.ceil(keys.size / providerQueries);
|
|
32
|
+
if (end > from && queries > budget / 4) break;
|
|
33
|
+
effects += entry.effects.length;
|
|
34
|
+
}
|
|
35
|
+
return end;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/** Reads this window's rows into `reads`, which holds only this publication's
|
|
39
|
+
* observations; no observation survives atomic(). A budget exhaustion propagates
|
|
40
|
+
* so the gather can stop the way its own reserve does. Any other failure discards
|
|
41
|
+
* the entire snapshot, including any earlier successful window. */
|
|
42
|
+
export async function applyReads(entries: PlanEntry[], context: D1ImportContext, reads: ApplyReads = emptyReads()): Promise<ApplyReads | undefined> {
|
|
16
43
|
const { database: db, targets } = context.storage;
|
|
17
44
|
const staged = context.maxBatchBytes !== undefined && !!targets.stagingRead;
|
|
18
45
|
if (staged ? !targets.stagingReadMany : !targets.readMany) return;
|
|
19
46
|
try {
|
|
20
|
-
const sources
|
|
47
|
+
const { sources, predecessors, targets: native } = reads;
|
|
21
48
|
const identities = [...new Set(entries.map(entry => entry.identity))];
|
|
22
49
|
for (let i = 0; i < identities.length; i += d1NativeLimits.variableNumber) {
|
|
23
50
|
const part = identities.slice(i, i + d1NativeLimits.variableNumber);
|
|
24
51
|
const { results } = await db.prepare(`SELECT * FROM ik_sources WHERE identity IN (${part.map(() => '?').join(',')})`).bind(...part).all();
|
|
25
52
|
for (const row of results) sources.set(row.identity, row);
|
|
26
53
|
}
|
|
27
|
-
const keys = [...new Map(entries.flatMap(entry => entry.effects.map(effect => [key(effect.targetKind, effect.targetId), { kind: effect.targetKind, id: effect.targetId }] as const))).values()];
|
|
28
|
-
const snapshots: { row: TargetRow | undefined; guards?: D1Sql[] }[] = staged
|
|
54
|
+
const keys = [...new Map(entries.flatMap(entry => entry.effects.map(effect => [key(effect.targetKind, effect.targetId), { kind: effect.targetKind, id: effect.targetId }] as const)).filter(([k]) => !native.has(k))).values()];
|
|
55
|
+
const snapshots: { row: TargetRow | undefined; guards?: D1Sql[] }[] = !keys.length ? [] : staged
|
|
29
56
|
? await targets.stagingReadMany!(db, context.destination, keys)
|
|
30
57
|
: (await targets.readMany!(db, context.destination, keys)).map(row => ({ row }));
|
|
31
58
|
if (snapshots.length !== keys.length) throw new Error('IMPORT_NATIVE_STAGING_GUARD_REQUIRED');
|
|
32
|
-
const native = new Map<string, { row: TargetRow | undefined; guards?: D1Sql[] }>();
|
|
33
59
|
for (const [i, selected] of keys.entries()) {
|
|
34
60
|
const snapshot = snapshots[i];
|
|
35
61
|
if (staged && !snapshot.guards?.length) throw new Error('IMPORT_NATIVE_STAGING_GUARD_REQUIRED');
|
|
36
62
|
native.set(key(selected.kind, selected.id), snapshot);
|
|
37
63
|
}
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
const part = keys.slice(i, i + size);
|
|
64
|
+
for (let i = 0; i < keys.length; i += providerQueries) {
|
|
65
|
+
const part = keys.slice(i, i + providerQueries);
|
|
41
66
|
const { results } = await db.prepare(`SELECT * FROM ik_provenance WHERE destination=? AND (target_kind,target_id) IN (${part.map(() => '(?,?)').join(',')}) AND reversed=0`).bind(context.destination, ...part.flatMap(k => [k.kind, k.id])).all();
|
|
42
67
|
for (const row of results) predecessors.set(key(row.target_kind, row.target_id), row);
|
|
43
68
|
}
|
|
44
|
-
return
|
|
45
|
-
} catch {
|
|
69
|
+
return reads;
|
|
70
|
+
} catch (error) {
|
|
71
|
+
if (error instanceof D1RequestLimitError) throw error;
|
|
46
72
|
return undefined;
|
|
47
73
|
}
|
|
48
74
|
}
|
|
@@ -15,7 +15,7 @@ import { custodyGuards, validatedProvenance, inventoryGuards, reusedSourceGuards
|
|
|
15
15
|
import { supersedeTarget } from './history.ts';
|
|
16
16
|
import { readOriginals } from './originals.ts';
|
|
17
17
|
import { changed, putRow, updateRows } from './expected-state.ts';
|
|
18
|
-
import { applyReads, targetReadKey, type ApplyReads } from './apply-reads.ts';
|
|
18
|
+
import { applyReads, readEnd, targetReadKey, type ApplyReads } from './apply-reads.ts';
|
|
19
19
|
|
|
20
20
|
export async function applyD1ImportBatches(jobOrId: ImportJob | string, context: D1ImportContext): Promise<ImportJob> {
|
|
21
21
|
context = requestContext(context);
|
|
@@ -37,15 +37,25 @@ export async function applyD1ImportBatches(jobOrId: ImportJob | string, context:
|
|
|
37
37
|
let prefix = atomicPrefix(context, 'apply', statements, scope);
|
|
38
38
|
let planned = prefix ? prefix.count(progress(job, 0), returnReads) : cost(context, publication(job, 0, statements, scope), returnReads);
|
|
39
39
|
const pending = plan.entries.slice(job.checkpoint, job.checkpoint + batchSize);
|
|
40
|
-
let reads: ApplyReads | undefined,
|
|
41
|
-
for (
|
|
40
|
+
let reads: ApplyReads | undefined, covered = 0, bulk = true;
|
|
41
|
+
for (let index = 0; index < pending.length; index++) {
|
|
42
|
+
const entry = pending[index];
|
|
42
43
|
if (entries.length && effects + entry.effects.length > 1000) break;
|
|
43
44
|
if (entry.effects.length > 1000) throw new Error('SOURCE_EFFECT_LIMIT');
|
|
44
45
|
// Stop before an entry's own reads could exhaust the request. The exact
|
|
45
46
|
// request, as atomic will send it, is counted again once the entry is built.
|
|
46
47
|
const reserve = 40 + entry.effects.length * 16;
|
|
47
|
-
|
|
48
|
-
if (!
|
|
48
|
+
let starved = remaining(context) - planned < reserve;
|
|
49
|
+
if (!starved && index >= covered) {
|
|
50
|
+
// Read only the next entries that can still publish; a budget exhaustion
|
|
51
|
+
// inside the bulk read stops the gather exactly like the reserve above.
|
|
52
|
+
try {
|
|
53
|
+
const end: number = bulk ? readEnd(pending, index, effects, remaining(context) - planned, reads) : pending.length;
|
|
54
|
+
reads = bulk ? await applyReads(pending.slice(index, end), context, reads) : undefined;
|
|
55
|
+
bulk = !!reads; covered = end;
|
|
56
|
+
} catch (error) { if (!(error instanceof D1RequestLimitError)) throw error; starved = true; }
|
|
57
|
+
}
|
|
58
|
+
if (starved) { if (!entries.length) { if (count > 0) return job; throw new D1RequestLimitError(); } break; }
|
|
49
59
|
const candidate: D1Sql[] = [], nextScope = { ...scope, nativeGuards:[...scope.nativeGuards], postGuards:[...scope.postGuards], targets:[...scope.targets] };
|
|
50
60
|
const candidateUndo: D1UndoEffect[] | undefined = undoEffects && [];
|
|
51
61
|
await publishEntry(id, entry, job.checkpoint + entries.length, context, candidate, nextScope, candidateUndo, reads);
|
|
@@ -34,16 +34,16 @@ export interface LineageSever { target: string; select: (record: SourceRecord) =
|
|
|
34
34
|
* step stays inside the limit and the erasure reaches its final step. With
|
|
35
35
|
* `sever`, each step also severs the door lineage's copies in the jobs it reads. */
|
|
36
36
|
export async function redactionSteps(fields: readonly D1FieldRedaction[], context: D1ImportContext, fence: D1Sql[], progress: RedactionProgress, pending: JobRevision[], failed?: Overflow, sever?: LineageSever): Promise<D1RetirementProgress> {
|
|
37
|
-
let { span, page } = progress, unsized = failed;
|
|
37
|
+
let { span, page } = progress, unsized = failed, attempts = failed ? 1 : 0;
|
|
38
38
|
for (;;) {
|
|
39
39
|
let job = page && pending.find(item => item.id === page!.job);
|
|
40
40
|
// A job changed since its last page starts again from its first entry; one no longer pending is done.
|
|
41
41
|
if (page && (!job || job.revision !== page.revision)) page = job ? { ...page, from: 0, revision: job.revision } : undefined;
|
|
42
42
|
if (unsized) {
|
|
43
|
-
const next = smaller(span, page, pending, unsized.cost);
|
|
43
|
+
const next = smaller(span, page, pending, unsized.cost, attempts === 1);
|
|
44
44
|
({ span, page } = next); job = page && pending.find(item => item.id === page!.job);
|
|
45
45
|
// Try it now only while this request has room for it; otherwise commit nothing and let the next request take it.
|
|
46
|
-
if (unsized.cost === undefined || unsized.cost * next.fraction > remaining(context)) {
|
|
46
|
+
if (next.defer || unsized.cost === undefined || unsized.cost * next.fraction > remaining(context)) {
|
|
47
47
|
return { phase: 'redacting', continuation: writeProgress({ ...progress, span, page }), jobsRedacted: 0, jobsPending: pending.length, fieldMatches: progress.matched };
|
|
48
48
|
}
|
|
49
49
|
}
|
|
@@ -53,6 +53,7 @@ export async function redactionSteps(fields: readonly D1FieldRedaction[], contex
|
|
|
53
53
|
try { done = await redactionStep(fields, context, fence, progress, jobs, range, sever); }
|
|
54
54
|
catch (error) {
|
|
55
55
|
if (!(error instanceof D1RequestLimitError)) throw error;
|
|
56
|
+
attempts++;
|
|
56
57
|
const needed = (error as { needed?: number }).needed;
|
|
57
58
|
unsized = { cost: needed === undefined ? undefined : start - remaining(context) + needed };
|
|
58
59
|
continue;
|
|
@@ -72,18 +73,27 @@ export async function redactionSteps(fields: readonly D1FieldRedaction[], contex
|
|
|
72
73
|
|
|
73
74
|
/** The next, strictly smaller step after one cost `cost` statements: in proportion,
|
|
74
75
|
* with headroom, or half when its reads alone ran out. Fewer jobs first, then
|
|
75
|
-
* pages of one job's entries; `fraction` is its expected share of that cost.
|
|
76
|
-
|
|
76
|
+
* pages of one job's entries; `fraction` is its expected share of that cost.
|
|
77
|
+
* When one entry is already the step and that attempt was `first` in its request,
|
|
78
|
+
* nothing fits and the step is refused. A request that earlier attempts had spent
|
|
79
|
+
* defers the same step instead (`defer`): a fresh request has its whole budget. */
|
|
80
|
+
function smaller(span: number, page: RedactionPage | undefined, pending: JobRevision[], cost: number | undefined, first: boolean): { span: number; page?: RedactionPage; fraction: number; defer?: boolean } {
|
|
77
81
|
const size = (current: number) => Math.min(current - 1, cost === undefined ? Math.floor(current / 2) : Math.floor(current * BUDGET * 0.7 / cost));
|
|
78
82
|
if (page) {
|
|
79
83
|
const count = size(page.count);
|
|
80
|
-
if (count < 1)
|
|
84
|
+
if (count < 1) {
|
|
85
|
+
if (!first) return { span, page, fraction: 1, defer: true };
|
|
86
|
+
throw new D1RequestLimitError(); // One entry alone is more than a request.
|
|
87
|
+
}
|
|
81
88
|
return { span, page: { ...page, count }, fraction: count / page.count };
|
|
82
89
|
}
|
|
83
90
|
const jobs = Math.min(span, pending.length);
|
|
84
91
|
if (jobs > 1) { const fewer = Math.max(1, size(jobs)); return { span: fewer, fraction: fewer / jobs }; }
|
|
85
92
|
const [job] = pending, count = size(job.total);
|
|
86
|
-
if (count < 1)
|
|
93
|
+
if (count < 1) {
|
|
94
|
+
if (!first) return { span, fraction: 1, defer: true };
|
|
95
|
+
throw new D1RequestLimitError();
|
|
96
|
+
}
|
|
87
97
|
return { span: 1, page: { job: job.id, from: 0, count, revision: job.revision }, fraction: count / job.total };
|
|
88
98
|
}
|
|
89
99
|
|