@runeditrun/rer 0.2.101 → 0.2.102

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -2,6 +2,14 @@
2
2
 
3
3
  ## Unreleased
4
4
 
5
+ ## 0.2.102
6
+
7
+ ### Fixes
8
+
9
+ - Deploy no longer reports a completed D1 migration as failed when a poll after an active ingest answers "Not currently importing anything." It accepts that answer only when the migration ledger has exactly one row for that migration, marked `applied` with the expected SHA-256. A missing, unapplied or mismatched row, or a failed ledger read, keeps the `d1-import-failed` refusal.
10
+ - Import apply prefetch reads bounded windows of entries that can still publish in the current call, sizing each window against the 1,000-effect cap and a quarter of the remaining publication budget, always including the next entry. It resizes later windows against actual read usage. Request-budget exhaustion inside a bulk read ends the gather like the existing reserve stop; other bulk-read errors still discard the snapshot and use individual reads. The documented budget-boundary behavior from 0.2.101 is unchanged: saved reads can advance a call's checkpoint or surface a later entry's refusal one call earlier, with the same error code and text; that refusal publishes none of the gathered entries and leaves the last consistent durable checkpoint (`modules/import-kit/src/d1/README.md`).
11
+ - Resumable erasure defers a redaction step to a fresh request when earlier attempts have spent the current request's budget and the step cannot shrink further. It returns a `redacting` continuation instead of throwing `IMPORT_D1_REQUEST_LIMIT`, so a People erasure whose step fits a fresh request can continue. A smallest step that cannot fit on its first attempt still refuses.
12
+
5
13
  ## 0.2.101
6
14
 
7
15
  ### Person erasure keeps other people's agent-door records
@@ -5,6 +5,10 @@ import { importDiagnostic } from './d1-import-diagnostics.mjs';
5
5
 
6
6
  export const D1_IMPORT_POLL_LIMIT = 100;
7
7
 
8
+ // D1 answers a poll with this once the import it was running has finished: a
9
+ // short import can commit, completion marker included, before the next poll.
10
+ const NOT_IMPORTING = 'Not currently importing anything.';
11
+
8
12
  // Wrangler's remote --file protocol. D1 imports execute the whole file atomically,
9
13
  // including trigger bodies; the query endpoint's semicolon splitting cannot.
10
14
  // Cloudflare's own reason for an import that failed or did not finish, kept on the
@@ -47,7 +51,13 @@ export async function importD1({ envelope, upload, redact = String }, id, sql) {
47
51
  }
48
52
  for (let polls = 0; polls < D1_IMPORT_POLL_LIMIT; polls++) {
49
53
  // REST places errors on the envelope; validate nested errors when supplied.
50
- if (!(response?.success === true && response.type === 'import' && response.status !== 'error' && (!Object.hasOwn(response, 'errors') || Array.isArray(response.errors) && !response.errors.length))) throw fail('d1-import-failed');
54
+ if (!(response?.success === true && response.type === 'import' && response.status !== 'error' && (!Object.hasOwn(response, 'errors') || Array.isArray(response.errors) && !response.errors.length))) {
55
+ const failure = fail('d1-import-failed');
56
+ // Only a poll that follows an active import can mean it finished; the caller
57
+ // that knows the migration decides from the D1 ledger, never from this answer.
58
+ if (step === 'poll' && response?.success === false && response.error === NOT_IMPORTING && !response.errors?.length) failure.importNotActive = true;
59
+ throw failure;
60
+ }
51
61
  if (response.status === 'complete') {
52
62
  if (!(Number.isSafeInteger(response.result?.num_queries) && response.result.num_queries > 0 && typeof response.result.final_bookmark === 'string' && response.result.final_bookmark.length > 0)) throw fail('d1-import-invalid-result');
53
63
  return;
@@ -53,6 +53,16 @@ export async function query(api, id, sql) {
53
53
  assert(Array.isArray(rows) && rows.length > 0 && rows.every(x => x.success === true && Array.isArray(x.results)), 'd1-query-failed');
54
54
  return rows;
55
55
  }
56
+ /** D1 answered a poll "not currently importing" after an active ingest: the import may
57
+ * have committed before the poll. Complete only when the ledger holds this migration
58
+ * applied on exactly its bytes; any other answer, or a read that fails, leaves the
59
+ * import's own failure standing. */
60
+ async function ledgerAppliesMigration(api, id, migration) {
61
+ try {
62
+ const rows = (await query(api,id,`SELECT name, sha256, state FROM ${ledger} WHERE name=${literal(migration.name)}`))[0].results;
63
+ return rows.length === 1 && rows[0].state === 'applied' && rows[0].sha256 === migration.sha256;
64
+ } catch { return false; }
65
+ }
56
66
  /** `locked`: the caller holds the install's deploy lock (owner.mjs withStateLock). */
57
67
  export async function migrate(api, id, migrations, { apply = true, upgrade = apply, locked = false } = {}) {
58
68
  validateMigrations(migrations);
@@ -80,7 +90,10 @@ export async function migrate(api, id, migrations, { apply = true, upgrade = app
80
90
  ? `UPDATE ${ledger} SET sha256=${literal(migration.sha256)}, state='applied' WHERE name=${literal(migration.name)} AND sha256=${literal(claimedFrom)} AND state='applying'`
81
91
  : `UPDATE ${ledger} SET state='applied' WHERE name=${literal(migration.name)} AND sha256=${literal(migration.sha256)}`;
82
92
  await api.importSql(id,`${migration.sql}\n${separator}${done};`)
83
- .catch(error => { error.migration ??= migration.name; throw error; });
93
+ .catch(async error => {
94
+ if (error?.importNotActive && await ledgerAppliesMigration(api,id,migration)) return;
95
+ error.migration ??= migration.name; throw error;
96
+ });
84
97
  }
85
98
  if (pending.length) {
86
99
  // By name and bytes: rows applied after ones that sort above them keep their ids.
@@ -158,7 +158,8 @@ The product implements `D1TargetAdapter`:
158
158
  Query only selected keys and preserve native decoding. With private transport,
159
159
  adapters that implement `stagingRead` can also provide `stagingReadMany`, with
160
160
  the same exact before-read guards for each key. Missing bulk methods retain
161
- individual reads; any bulk-read error discards the snapshot and uses that path.
161
+ individual reads; any bulk-read error except request-budget exhaustion discards the
162
+ snapshot and uses that path. Exhaustion ends the batch like the budget reserve does.
162
163
  An inherited bulk method must preserve an overridden single read or throw.
163
164
  Bulk observations belong only to the current publication.
164
165
  - `matches(rowOrNull, destination, kind, id)` returns an exact SQL boolean snapshot
@@ -1,5 +1,6 @@
1
1
  import { d1NativeLimits } from '../../../d1-transport/limits.mjs';
2
2
  import type { PlanEntry, TargetRow } from '../types.ts';
3
+ import { D1RequestLimitError } from './budget.ts';
3
4
  import type { D1ImportContext, D1Sql } from './types.ts';
4
5
 
5
6
  const key = (kind: string, id: string): string => JSON.stringify([kind, id]);
@@ -10,39 +11,64 @@ export interface ApplyReads {
10
11
  }
11
12
  export const targetReadKey = key;
12
13
 
13
- /** Only this publication's candidate entries. No observation survives atomic().
14
- * Failure discards the entire snapshot, including any earlier successful set. */
15
- export async function applyReads(entries: PlanEntry[], context: D1ImportContext): Promise<ApplyReads | undefined> {
14
+ const emptyReads = (): ApplyReads => ({ sources: new Map(), predecessors: new Map(), targets: new Map() });
15
+ const providerQueries = Math.floor((d1NativeLimits.variableNumber - 1) / 2);
16
+
17
+ /** How far the next bulk read may reach: the end (exclusive) of the entries from
18
+ * `from` that can still publish in this call. It stops at the 1000-effect batch
19
+ * cap, and where the reads, counted as the contract's worst case of one query per
20
+ * target key, would take more than a quarter of what the request has left after the
21
+ * publication so far. The entry at `from` is always included; later windows are
22
+ * sized again against the budget those reads actually used. */
23
+ export function readEnd(pending: PlanEntry[], from: number, gathered: number, budget: number, reads?: ApplyReads): number {
24
+ const identities = new Set<string>(), keys = new Set<string>();
25
+ let effects = 0, end = from;
26
+ for (; end < pending.length; end++) {
27
+ const entry = pending[end];
28
+ if (end > from && gathered + effects + entry.effects.length > 1000) break;
29
+ identities.add(entry.identity);
30
+ for (const effect of entry.effects) { const k = key(effect.targetKind, effect.targetId); if (!reads?.targets.has(k)) keys.add(k); }
31
+ const queries = Math.ceil(identities.size / d1NativeLimits.variableNumber) + keys.size + Math.ceil(keys.size / providerQueries);
32
+ if (end > from && queries > budget / 4) break;
33
+ effects += entry.effects.length;
34
+ }
35
+ return end;
36
+ }
37
+
38
+ /** Reads this window's rows into `reads`, which holds only this publication's
39
+ * observations; no observation survives atomic(). A budget exhaustion propagates
40
+ * so the gather can stop the way its own reserve does. Any other failure discards
41
+ * the entire snapshot, including any earlier successful window. */
42
+ export async function applyReads(entries: PlanEntry[], context: D1ImportContext, reads: ApplyReads = emptyReads()): Promise<ApplyReads | undefined> {
16
43
  const { database: db, targets } = context.storage;
17
44
  const staged = context.maxBatchBytes !== undefined && !!targets.stagingRead;
18
45
  if (staged ? !targets.stagingReadMany : !targets.readMany) return;
19
46
  try {
20
- const sources = new Map<string, any>(), predecessors = new Map<string, any>();
47
+ const { sources, predecessors, targets: native } = reads;
21
48
  const identities = [...new Set(entries.map(entry => entry.identity))];
22
49
  for (let i = 0; i < identities.length; i += d1NativeLimits.variableNumber) {
23
50
  const part = identities.slice(i, i + d1NativeLimits.variableNumber);
24
51
  const { results } = await db.prepare(`SELECT * FROM ik_sources WHERE identity IN (${part.map(() => '?').join(',')})`).bind(...part).all();
25
52
  for (const row of results) sources.set(row.identity, row);
26
53
  }
27
- const keys = [...new Map(entries.flatMap(entry => entry.effects.map(effect => [key(effect.targetKind, effect.targetId), { kind: effect.targetKind, id: effect.targetId }] as const))).values()];
28
- const snapshots: { row: TargetRow | undefined; guards?: D1Sql[] }[] = staged
54
+ const keys = [...new Map(entries.flatMap(entry => entry.effects.map(effect => [key(effect.targetKind, effect.targetId), { kind: effect.targetKind, id: effect.targetId }] as const)).filter(([k]) => !native.has(k))).values()];
55
+ const snapshots: { row: TargetRow | undefined; guards?: D1Sql[] }[] = !keys.length ? [] : staged
29
56
  ? await targets.stagingReadMany!(db, context.destination, keys)
30
57
  : (await targets.readMany!(db, context.destination, keys)).map(row => ({ row }));
31
58
  if (snapshots.length !== keys.length) throw new Error('IMPORT_NATIVE_STAGING_GUARD_REQUIRED');
32
- const native = new Map<string, { row: TargetRow | undefined; guards?: D1Sql[] }>();
33
59
  for (const [i, selected] of keys.entries()) {
34
60
  const snapshot = snapshots[i];
35
61
  if (staged && !snapshot.guards?.length) throw new Error('IMPORT_NATIVE_STAGING_GUARD_REQUIRED');
36
62
  native.set(key(selected.kind, selected.id), snapshot);
37
63
  }
38
- const size = Math.floor((d1NativeLimits.variableNumber - 1) / 2);
39
- for (let i = 0; i < keys.length; i += size) {
40
- const part = keys.slice(i, i + size);
64
+ for (let i = 0; i < keys.length; i += providerQueries) {
65
+ const part = keys.slice(i, i + providerQueries);
41
66
  const { results } = await db.prepare(`SELECT * FROM ik_provenance WHERE destination=? AND (target_kind,target_id) IN (${part.map(() => '(?,?)').join(',')}) AND reversed=0`).bind(context.destination, ...part.flatMap(k => [k.kind, k.id])).all();
42
67
  for (const row of results) predecessors.set(key(row.target_kind, row.target_id), row);
43
68
  }
44
- return { sources, predecessors, targets: native };
45
- } catch {
69
+ return reads;
70
+ } catch (error) {
71
+ if (error instanceof D1RequestLimitError) throw error;
46
72
  return undefined;
47
73
  }
48
74
  }
@@ -15,7 +15,7 @@ import { custodyGuards, validatedProvenance, inventoryGuards, reusedSourceGuards
15
15
  import { supersedeTarget } from './history.ts';
16
16
  import { readOriginals } from './originals.ts';
17
17
  import { changed, putRow, updateRows } from './expected-state.ts';
18
- import { applyReads, targetReadKey, type ApplyReads } from './apply-reads.ts';
18
+ import { applyReads, readEnd, targetReadKey, type ApplyReads } from './apply-reads.ts';
19
19
 
20
20
  export async function applyD1ImportBatches(jobOrId: ImportJob | string, context: D1ImportContext): Promise<ImportJob> {
21
21
  context = requestContext(context);
@@ -37,15 +37,25 @@ export async function applyD1ImportBatches(jobOrId: ImportJob | string, context:
37
37
  let prefix = atomicPrefix(context, 'apply', statements, scope);
38
38
  let planned = prefix ? prefix.count(progress(job, 0), returnReads) : cost(context, publication(job, 0, statements, scope), returnReads);
39
39
  const pending = plan.entries.slice(job.checkpoint, job.checkpoint + batchSize);
40
- let reads: ApplyReads | undefined, loaded = false;
41
- for (const entry of pending) {
40
+ let reads: ApplyReads | undefined, covered = 0, bulk = true;
41
+ for (let index = 0; index < pending.length; index++) {
42
+ const entry = pending[index];
42
43
  if (entries.length && effects + entry.effects.length > 1000) break;
43
44
  if (entry.effects.length > 1000) throw new Error('SOURCE_EFFECT_LIMIT');
44
45
  // Stop before an entry's own reads could exhaust the request. The exact
45
46
  // request, as atomic will send it, is counted again once the entry is built.
46
47
  const reserve = 40 + entry.effects.length * 16;
47
- if (remaining(context) - planned < reserve) { if (!entries.length) { if (count > 0) return job; throw new D1RequestLimitError(); } break; }
48
- if (!loaded) { reads = await applyReads(pending, context); loaded = true; }
48
+ let starved = remaining(context) - planned < reserve;
49
+ if (!starved && index >= covered) {
50
+ // Read only the next entries that can still publish; a budget exhaustion
51
+ // inside the bulk read stops the gather exactly like the reserve above.
52
+ try {
53
+ const end: number = bulk ? readEnd(pending, index, effects, remaining(context) - planned, reads) : pending.length;
54
+ reads = bulk ? await applyReads(pending.slice(index, end), context, reads) : undefined;
55
+ bulk = !!reads; covered = end;
56
+ } catch (error) { if (!(error instanceof D1RequestLimitError)) throw error; starved = true; }
57
+ }
58
+ if (starved) { if (!entries.length) { if (count > 0) return job; throw new D1RequestLimitError(); } break; }
49
59
  const candidate: D1Sql[] = [], nextScope = { ...scope, nativeGuards:[...scope.nativeGuards], postGuards:[...scope.postGuards], targets:[...scope.targets] };
50
60
  const candidateUndo: D1UndoEffect[] | undefined = undoEffects && [];
51
61
  await publishEntry(id, entry, job.checkpoint + entries.length, context, candidate, nextScope, candidateUndo, reads);
@@ -34,16 +34,16 @@ export interface LineageSever { target: string; select: (record: SourceRecord) =
34
34
  * step stays inside the limit and the erasure reaches its final step. With
35
35
  * `sever`, each step also severs the door lineage's copies in the jobs it reads. */
36
36
  export async function redactionSteps(fields: readonly D1FieldRedaction[], context: D1ImportContext, fence: D1Sql[], progress: RedactionProgress, pending: JobRevision[], failed?: Overflow, sever?: LineageSever): Promise<D1RetirementProgress> {
37
- let { span, page } = progress, unsized = failed;
37
+ let { span, page } = progress, unsized = failed, attempts = failed ? 1 : 0;
38
38
  for (;;) {
39
39
  let job = page && pending.find(item => item.id === page!.job);
40
40
  // A job changed since its last page starts again from its first entry; one no longer pending is done.
41
41
  if (page && (!job || job.revision !== page.revision)) page = job ? { ...page, from: 0, revision: job.revision } : undefined;
42
42
  if (unsized) {
43
- const next = smaller(span, page, pending, unsized.cost);
43
+ const next = smaller(span, page, pending, unsized.cost, attempts === 1);
44
44
  ({ span, page } = next); job = page && pending.find(item => item.id === page!.job);
45
45
  // Try it now only while this request has room for it; otherwise commit nothing and let the next request take it.
46
- if (unsized.cost === undefined || unsized.cost * next.fraction > remaining(context)) {
46
+ if (next.defer || unsized.cost === undefined || unsized.cost * next.fraction > remaining(context)) {
47
47
  return { phase: 'redacting', continuation: writeProgress({ ...progress, span, page }), jobsRedacted: 0, jobsPending: pending.length, fieldMatches: progress.matched };
48
48
  }
49
49
  }
@@ -53,6 +53,7 @@ export async function redactionSteps(fields: readonly D1FieldRedaction[], contex
53
53
  try { done = await redactionStep(fields, context, fence, progress, jobs, range, sever); }
54
54
  catch (error) {
55
55
  if (!(error instanceof D1RequestLimitError)) throw error;
56
+ attempts++;
56
57
  const needed = (error as { needed?: number }).needed;
57
58
  unsized = { cost: needed === undefined ? undefined : start - remaining(context) + needed };
58
59
  continue;
@@ -72,18 +73,27 @@ export async function redactionSteps(fields: readonly D1FieldRedaction[], contex
72
73
 
73
74
  /** The next, strictly smaller step after one cost `cost` statements: in proportion,
74
75
  * with headroom, or half when its reads alone ran out. Fewer jobs first, then
75
- * pages of one job's entries; `fraction` is its expected share of that cost. */
76
- function smaller(span: number, page: RedactionPage | undefined, pending: JobRevision[], cost: number | undefined): { span: number; page?: RedactionPage; fraction: number } {
76
+ * pages of one job's entries; `fraction` is its expected share of that cost.
77
+ * When one entry is already the step and that attempt was `first` in its request,
78
+ * nothing fits and the step is refused. A request that earlier attempts had spent
79
+ * defers the same step instead (`defer`): a fresh request has its whole budget. */
80
+ function smaller(span: number, page: RedactionPage | undefined, pending: JobRevision[], cost: number | undefined, first: boolean): { span: number; page?: RedactionPage; fraction: number; defer?: boolean } {
77
81
  const size = (current: number) => Math.min(current - 1, cost === undefined ? Math.floor(current / 2) : Math.floor(current * BUDGET * 0.7 / cost));
78
82
  if (page) {
79
83
  const count = size(page.count);
80
- if (count < 1) throw new D1RequestLimitError(); // One entry alone is more than a request.
84
+ if (count < 1) {
85
+ if (!first) return { span, page, fraction: 1, defer: true };
86
+ throw new D1RequestLimitError(); // One entry alone is more than a request.
87
+ }
81
88
  return { span, page: { ...page, count }, fraction: count / page.count };
82
89
  }
83
90
  const jobs = Math.min(span, pending.length);
84
91
  if (jobs > 1) { const fewer = Math.max(1, size(jobs)); return { span: fewer, fraction: fewer / jobs }; }
85
92
  const [job] = pending, count = size(job.total);
86
- if (count < 1) throw new D1RequestLimitError();
93
+ if (count < 1) {
94
+ if (!first) return { span, fraction: 1, defer: true };
95
+ throw new D1RequestLimitError();
96
+ }
87
97
  return { span: 1, page: { job: job.id, from: 0, count, revision: job.revision }, fraction: count / job.total };
88
98
  }
89
99
 
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@runeditrun/rer",
3
- "version": "0.2.101",
3
+ "version": "0.2.102",
4
4
  "description": "The rer command: install, set up, run and launch Run Edit Run checkouts.",
5
5
  "license": "MIT",
6
6
  "type": "module",