@runeditrun/rer 0.1.1 → 0.2.0

package/modules/deploy-kit/site.mjs

@@ -0,0 +1,135 @@
+import { mkdir, rename, writeFile } from 'node:fs/promises';
+import { dirname } from 'node:path';
+import { assert, hash, resourceName } from './validation.mjs';
+import { ownerWorkerName, workerSettings } from './owner.mjs';
+import { uploadAssets, workerOrigin } from './worker.mjs';
+
+/* A seed's run site (run-<seed>.com): the folder `rer build --site` writes,
+ * served on the owner's own account by a Base-generated module Worker (fixed
+ * content, no seed code) whose only binding is its assets. The Worker runs
+ * first on every request: www.<domain> gets a 301 to the apex with path and
+ * query kept, and everything else is answered from the assets. It has no
+ * secrets and no schedules, workers.dev stays off, and its name can never be
+ * the installed app's Worker name, so a site deploy never touches the app or
+ * its resources. The apex and www.<domain> are Workers Custom Domains; no zone
+ * ruleset is read or written.
+ *
+ * Ownership is proven from the provider alone, so a deploy from a fresh
+ * checkout (a CI runner) recognises the site it deployed before: the Worker is
+ * the kit's when its latest deployment serves, at 100%, the Worker's newest
+ * version and that version carries the kit's `rer-site:<fingerprint>`
+ * annotation; a Custom Domain is the kit's when its service is the kit's
+ * Worker. Anything of the same name that is not the kit's is refused, never
+ * adopted. A local receipt is written for the person and never read. */
+
+const RECEIPT_KIND = 'rer-deploy-site-receipt';
+const APEX = /^(?!www\.)(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z][a-z0-9-]{0,62}$/;
+const ANNOTATION = /^rer-site:[0-9a-f]{64}$/;
+export const SITE_ASSETS_CONFIG = Object.freeze({ html_handling: 'auto-trailing-slash', not_found_handling: '404-page' });
+export const SITE_MODULE = 'site-worker.mjs';
+/** The site Worker. It names no domain: only the site's two Custom Domains route to it. */
+export const SITE_WORKER = `// Run site Worker, generated by Base's deploy kit (rer deploy --site). Do not edit.
+export default {
+  async fetch(request, env) {
+    const url = new URL(request.url);
+    if (url.hostname.startsWith('www.')) return new Response(null, { status: 301, headers: { location: \`https://\${url.hostname.slice(4)}\${url.pathname}\${url.search}\` } });
+    return env.ASSETS.fetch(request);
+  },
+};
+`;
+
+function refuse(code, kind, name) { const error = new Error(code); error.resource = `${kind}:${name}`; return error; }
+
+/** deploy-site.json: `{ "version": 1, "domain": "run-cradle.com" }`, the apex hostname only. */
+export function siteDeclaration(value) {
+  assert(value && typeof value === 'object' && !Array.isArray(value) && value.version === 1 && Object.keys(value).sort().join() === 'domain,version', 'invalid-site-declaration');
+  assert(typeof value.domain === 'string' && value.domain.length <= 253 && APEX.test(value.domain), 'invalid-site-domain');
+  return value;
+}
+
+/** The app is `<project>-<environment>`; owner names never hold `--`, so the site Worker cannot be it. */
+export function siteWorkerName(project, environment) {
+  const worker = `${ownerWorkerName(project, environment)}--site`;
+  assert(resourceName(worker), 'owner-name-too-long');
+  return worker;
+}
+
+/** Everything a site deploy will do, with no provider request. */
+export function sitePlan({ project, environment, compatibilityDate, declaration, assets }) {
+  const { domain } = siteDeclaration(declaration);
+  assert(/^\d{4}-\d{2}-\d{2}$/.test(compatibilityDate ?? ''), 'invalid-plan');
+  assert(typeof assets === 'string' && assets.startsWith('/'), 'invalid-site-assets');
+  return { version: 1, worker: siteWorkerName(project, environment), appWorker: ownerWorkerName(project, environment), module: SITE_MODULE, assets, compatibilityDate,
+    bindings: ['ASSETS'], secrets: [], crons: [], workersDev: false, runWorkerFirst: true, assetsConfig: { ...SITE_ASSETS_CONFIG },
+    domain, hostnames: [domain, `www.${domain}`],
+    redirects: [{ from: `https://www.${domain}/*`, to: `https://${domain}/`, status: 301, keeps: ['path', 'query'] }] };
+}
+
+/** The version the Worker's latest deployment serves, as the kit reads ownership: null when there is no Worker. */
+async function servedVersion(api, worker) {
+  if (!await workerSettings(api, worker)) return null;
+  const { deployments } = await api.request(`/workers/scripts/${worker}/deployments`);
+  assert(Array.isArray(deployments) && deployments.length > 0, 'invalid-deployments');
+  const [latest] = deployments, single = latest.versions?.length === 1 && latest.versions[0].percentage === 100 ? latest.versions[0].version_id : null;
+  const newest = (await api.request(`/workers/scripts/${worker}/versions`))?.items?.[0]?.id;
+  const version = single ? await api.request(`/workers/scripts/${worker}/versions/${single}`) : null;
+  const annotation = version?.annotations?.['workers/message'];
+  return { kit: typeof annotation === 'string' && ANNOTATION.test(annotation), annotation, current: Boolean(single) && single === newest };
+}
+/** The Worker is absent (false) or the kit's (true); anything else is refused. */
+async function ownWorker(api, worker) {
+  const served = await servedVersion(api, worker);
+  if (!served) return false;
+  // Someone else's Worker of this name, or one whose live version the kit did not upload.
+  if (!served.kit) throw refuse('resource-name-collision', 'worker', worker);
+  // A kit version that is no longer the newest one served alone: a dashboard rollback or a split deployment.
+  if (!served.current) throw refuse('owned-resource-replaced', 'worker', worker);
+  return true;
+}
+
+async function writeReceipt(path, receipt) {
+  await mkdir(dirname(path), { recursive: true, mode: 0o700 });
+  const temporary = `${path}.${process.pid}.tmp`;
+  await writeFile(temporary, JSON.stringify({ version: 1, kind: RECEIPT_KIND, ...receipt }, null, 2) + '\n', { flag: 'wx', mode: 0o600 });
+  await rename(temporary, path);
+}
+
+/** Read-only preflight, then the site Worker upload and both Custom Domains, then readback. */
+export async function siteDeploy({ api, plan, files, receiptPath }) {
+  assert(typeof receiptPath === 'string' && receiptPath.length, 'missing-receipt-path');
+  assert(plan?.module === SITE_MODULE && Array.isArray(files) && files.some(file => file.path === 'index.html'), 'invalid-site-assets');
+  const zones = await api.zoneRequest(`/zones?name=${plan.domain}&account.id=${api.accountId}`);
+  const zone = Array.isArray(zones) ? zones.find(row => row?.name === plan.domain && row.account?.id === api.accountId) : null;
+  if (!zone) throw refuse('site-zone-missing', 'zone', plan.domain);
+  // Preflight: the Worker and each hostname are the kit's own or absent before any write.
+  const existed = await ownWorker(api, plan.worker);
+  const attached = await api.request(`/workers/domains?zone_id=${zone.id}`);
+  assert(Array.isArray(attached), 'invalid-domain-list');
+  for (const hostname of plan.hostnames) {
+    const row = attached.find(item => item?.hostname === hostname);
+    if (row && (!existed || row.service !== plan.worker)) throw refuse('resource-name-collision', 'domain', hostname);
+  }
+
+  const fingerprint = hash({ files: files.map(({ path, sha256 }) => ({ path, sha256 })), config: plan.assetsConfig, module: hash(SITE_WORKER), compatibilityDate: plan.compatibilityDate });
+  const jwt = await uploadAssets(api, plan.worker, files);
+  const body = new FormData();
+  body.set('metadata', new Blob([JSON.stringify({ main_module: SITE_MODULE, compatibility_date: plan.compatibilityDate, bindings: [{ type: 'assets', name: 'ASSETS' }],
+    assets: { jwt, config: { ...plan.assetsConfig, run_worker_first: true } }, annotations: { 'workers/message': `rer-site:${fingerprint}` } })], { type: 'application/json' }));
+  body.set(SITE_MODULE, new Blob([SITE_WORKER], { type: 'application/javascript+module' }), SITE_MODULE);
+  await api.request(`/workers/scripts/${plan.worker}`, { method: 'PUT', body });
+  const uploaded = await servedVersion(api, plan.worker);
+  assert(uploaded?.current && uploaded.annotation === `rer-site:${fingerprint}`, 'site-upload-not-live');
+  await workerOrigin(api, plan.worker, { enable: true, workersDev: false });
+  for (const hostname of plan.hostnames) {
+    if (attached.some(row => row?.hostname === hostname)) continue;
+    const row = await api.request('/workers/domains', { method: 'PUT', body: { environment: 'production', hostname, service: plan.worker, zone_id: zone.id } });
+    assert(row?.hostname === hostname && row.service === plan.worker && typeof row.id === 'string', 'invalid-domain-attach');
+  }
+  // Readback: the kit's Worker live and both domains on it.
+  assert(await ownWorker(api, plan.worker), 'site-worker-missing');
+  const domains = await api.request(`/workers/domains?zone_id=${zone.id}`);
+  for (const hostname of plan.hostnames) assert(domains.some(row => row.hostname === hostname && row.service === plan.worker), 'domain-readback-mismatch');
+  const receipt = { worker: plan.worker, domain: plan.domain, zoneId: zone.id, fingerprint, files: files.length, hostnames: plan.hostnames, redirect: plan.redirects[0] };
+  await writeReceipt(receiptPath, receipt);
+  return { origin: `https://${plan.domain}`, receipt };
+}

package/modules/erasure/CONTRACT.md

@@ -31,8 +31,11 @@ outside D1 and R2.
   leaves-first, the subject row last) is deleted in a D1 batch that also holds
   portability's `erasureStatements(table, key)` for that row and
   `invalidateTransfersStatements()`. A batch holds at most `rows` dependents, so
-  deletion work is bounded. Scope/audit and completeness checks also read the
-  declared closure; their cost grows with that closure. A guard on the operation's step
+  deletion work is bounded. The first step reads and orders the declared closure
+  and queues what it leaves; every later step reads, verifies and deletes only its
+  own slice of that queue, and the last step reads the closure once more from the
+  subject and every remembered parent, so the work of an erasure is linear in its
+  rows. A guard on the operation's step
   counter makes a concurrent runner fail with `erasure_busy` (409) instead of
   committing twice.
 - **ERASE-005 — Blobs after the commit.** For a declared `blob` edge, each row's
@@ -51,7 +54,7 @@ outside D1 and R2.
   replaced with one label per erasure, `erased:<128 random bits>`. The label is
   not derived from the subject, so nobody who holds the database or an older
   archive can recompute it or test a guess against it. Declared `clear` columns
-  are overwritten. Audit rows for selected dependents are relabelled in every batch. Temporary
+  are overwritten. Audit rows for a batch's dependents are relabelled in that batch, and every remembered dependent's in the final one. Temporary
   scope preserves earlier keys until the final pass, catching audit rows written
   after those dependents disappeared; that scope is then removed atomically.
 - **ERASE-008 — Value-free receipt.** The receipt row holds the entity name,

package/modules/erasure/README.md

@@ -62,7 +62,14 @@ if (receipt.phase !== 'complete') await env.JOBS.send({ erasure: opened.id }); /
 `commit(id,{rows,until})` runs only the D1 phase: bounded atomic batches of at
 most `rows` dependents (default 500), leaves first, with the subject row last.
 Each deleted row's `erasureStatements` and `invalidateTransfersStatements()` go
-in the same batch. `deleteBlobs(id,{until})` runs only the blob phase, after the
+in the same batch.
+The first step collects and orders the whole plan and queues what it does not delete
+in the private scope table; each later step reads and verifies only its own slice, so the
+work of an erasure is linear in its rows, however many steps it takes. The queue is
+not trusted for completeness: once it is empty, one more step collects the graph again
+from the subject and every remembered parent, and anything written meanwhile is
+erased before the receipt says `complete`. Index the child columns an edge
+matches on (`on`), as foreign keys usually are, or each step scans that table. `deleteBlobs(id,{until})` runs only the blob phase, after the
 commit. `run` does both within the time budget. Every phase is safe to repeat,
 and a concurrent runner gets `erasure_busy` (409). The receipt says `complete`
 only after the completion transaction re-reads every declared dependent and audit path

package/modules/erasure/audit.mjs

@@ -4,7 +4,7 @@ import { statement, quote, chunks, randomHex } from './sql.mjs';
  * nobody holding the database (or an old archive) can recompute or test it. */
 export const newLabel = () => 'erased:' + randomHex(16);
 
-export function auditStatements(db, d, plan, label, { only } = {}) {
+export function auditStatements(db, d, plan, label, { only, relabel = true } = {}) {
   const out = [];
   for (const item of d.audit) {
     if (only && item.of !== only) continue;
@@ -13,7 +13,7 @@ export function auditStatements(db, d, plan, label, { only } = {}) {
     const set = [`${quote(item.column)}=?`, ...clear.map(([c]) => `${quote(c)}=?`)].join(',');
     // A previously relabelled row is still this operation's audit: a writer
     // may have changed its detail after the first cleanup.
-    out.push(statement(db, `UPDATE ${quote(item.table)} SET ${set} WHERE ${quote(item.column)}=?${item.where ? ` AND (${item.where})` : ''}`, label, ...clear.map(([, v]) => v), label));
+    if (relabel) out.push(statement(db, `UPDATE ${quote(item.table)} SET ${set} WHERE ${quote(item.column)}=?${item.where ? ` AND (${item.where})` : ''}`, label, ...clear.map(([, v]) => v), label));
     for (const part of chunks(keys, 200)) {
       out.push(statement(db, `UPDATE ${quote(item.table)} SET ${set} WHERE ${quote(item.column)} IN (SELECT value FROM json_each(?))${item.where ? ` AND (${item.where})` : ''}`,
         label, ...clear.map(([, v]) => v), JSON.stringify(part)));

package/modules/erasure/cascade.mjs

@@ -5,6 +5,7 @@ import { verificationStatements, childAbsenceStatements } from './verification.m
 import { collect, matchSql, checkDeclared } from './plan.mjs';
 import { auditStatements } from './audit.mjs';
 import { progressStatement, committedStatement } from './receipt.mjs';
+import { queueStatements, readQueue, consumeStatement, clearQueue, slicePlan, rootScopeExists } from './queue.mjs';
 
 const where = (keys, table) => keys.get(table).pk.map(c => `${c === 'rowid' ? 'rowid' : quote(c)}=?`).join(' AND ');
 const values = e => Object.values(e.key);
@@ -37,7 +38,8 @@ export async function deletionStatements(ctx, d, plan, entries, id, deleted) {
 export async function firstStatements(ctx, d, plan, label) {
   return d.beforeStatements ? await d.beforeStatements({ db: ctx.db, subject: plan.subject, label }) : [];
 }
-export const cleanupStatements = (ctx, d, plan, label) => [...redactStatements(ctx.db, d, plan), ...auditStatements(ctx.db, d, plan, label)];
+// `relabel: false` (a slice of the plan) leaves the table-wide relabel of earlier audit rows to the final batch.
+export const cleanupStatements = (ctx, d, plan, label, options) => [...redactStatements(ctx.db, d, plan), ...auditStatements(ctx.db, d, plan, label, options)];
 /** Commits with the subject row: a second redaction/audit pass catches rows written
  * during a long erasure. Verify before removing the root (so triggers cannot hide
  * new children through CASCADE), then after final policy effects. */
@@ -53,8 +55,43 @@ export async function finalStatements(ctx, d, plan, id, label, now, deleted) {
   return out;
 }
 
+const lost = error => {
+  if (/erasure_retained_missing/.test(String(error?.message ?? error))) throw new ErasureError('retained_row_missing', 409);
+  if (/CHECK constraint failed/.test(String(error?.message ?? error))) throw new ErasureError('erasure_busy', 409);
+  throw error;
+};
+
+/** A non-final step: deletes `entries` (a slice of the plan in deletion order)
+ * and leaves `rest` queued. Everything the batch verifies is bounded by the
+ * slice. Completeness is the final step's job, which collects the whole graph
+ * again from the subject and every remembered parent. */
+async function sliceStep(ctx, d, op, entries, { first = null, rest = null } = {}) {
+  const { db, fence } = ctx, id = op.id, deleted = JSON.parse(op.deleted);
+  const plan = await slicePlan(db, d, JSON.parse(op.subject), entries);
+  const batch = [db.prepare('PRAGMA defer_foreign_keys=ON'),
+    guard(db, "EXISTS(SELECT 1 FROM rer_erasure_operations WHERE id=? AND step=? AND phase='deleting')", id, op.step),
+    ...await fence.transfers()];
+  batch.push(...await scopeStatements(db, d, first ?? plan, id, first ? [...entries, first.root] : entries, { whole: Boolean(first) }));
+  if (first) batch.push(...await firstStatements(ctx, d, plan, op.label));
+  batch.push(...await verificationStatements(db, d, plan, id, op.label));
+  batch.push(...cleanupStatements(ctx, d, plan, op.label, { relabel: false }));
+  batch.push(...await deletionStatements(ctx, d, plan, entries, id, deleted));
+  batch.push(...await verificationStatements(db, d, plan, id, op.label, { intake: false }));
+  if (rest) batch.push(...queueStatements(db, id, rest));
+  else batch.push(consumeStatement(db, id, entries.at(-1).seq));
+  batch.push(progressStatement(db, id, op.step, deleted), db.prepare('DELETE FROM rer_erasure_guard'));
+  try { await db.batch(batch); }
+  catch (error) {
+    if (/erasure_scope_changed/.test(String(error?.message ?? error))) { await clearQueue(db, id).run(); return 'changed'; }
+    return lost(error);
+  }
+  return true;
+}
+
 /** One bounded, atomic D1 commit of up to `rows` dependents, leaves first. The
- * subject row commits with the last step. A concurrent runner loses the guard. */
+ * subject row commits with the last step. A concurrent runner loses the guard.
+ * The first step collects and orders the whole plan and queues what it leaves;
+ * later steps take their slice from the queue until it is empty. */
 export async function step(ctx, d, id, { rows = 500 } = {}) {
   const { db, fence } = ctx;
   const op = await statement(db, 'SELECT * FROM rer_erasure_operations WHERE id=?', id).first();
@@ -63,29 +100,33 @@ export async function step(ctx, d, id, { rows = 500 } = {}) {
   const now = new Date().toISOString(), deleted = JSON.parse(op.deleted);
   await checkDeclared(db, d);
   insist(Number.isInteger(rows) && rows > 0, 'batch_rows');
+  if (op.step > 0) {
+    const slice = await readQueue(db, id, rows);
+    if (slice.length) {
+      insist(await rootScopeExists(db, id, d), 'erasure_scope_missing', 409);
+      return sliceStep(ctx, d, op, slice);
+    }
+  }
   const remembered = await readScope(db, id);
   insist(op.step === 0 || remembered.some(e => e.table === d.root.table), 'erasure_scope_missing', 409);
   const plan = await collect(db, d, JSON.parse(op.subject), remembered);
   insist(plan && plan.root, 'subject_removed_outside_erasure', 409);
+  const take = plan.order.slice(0, rows), final = take.length === plan.order.length;
+  if (!final) return sliceStep(ctx, d, op, take, { first: op.step === 0 ? plan : null, rest: plan.order.slice(rows) });
   const batch = [db.prepare('PRAGMA defer_foreign_keys=ON'),
     guard(db, "EXISTS(SELECT 1 FROM rer_erasure_operations WHERE id=? AND step=? AND phase='deleting')", id, op.step),
     ...await fence.transfers()];
-  const take = plan.order.slice(0, rows), final = take.length === plan.order.length;
   batch.push(...await scopeStatements(db, d, plan, id, [...take, plan.root]));
   if (op.step === 0) batch.push(...await firstStatements(ctx, d, plan, op.label));
   batch.push(...await verificationStatements(db, d, plan, id, op.label));
   batch.push(...cleanupStatements(ctx, d, plan, op.label));
   batch.push(...await deletionStatements(ctx, d, plan, take, id, deleted));
-  if (final) batch.push(...await finalStatements(ctx, d, plan, id, op.label, now, deleted));
-  else batch.push(...await verificationStatements(db, d, plan, id, op.label, { intake: false }));
-
-  batch.push(progressStatement(db, id, op.step, deleted), db.prepare('DELETE FROM rer_erasure_guard'));
+  batch.push(...await finalStatements(ctx, d, plan, id, op.label, now, deleted));
+  batch.push(clearQueue(db, id), progressStatement(db, id, op.step, deleted), db.prepare('DELETE FROM rer_erasure_guard'));
   try { await db.batch(batch); }
   catch (error) {
     if (/erasure_scope_changed/.test(String(error?.message ?? error))) return 'changed';
-    if (/erasure_retained_missing/.test(String(error?.message ?? error))) throw new ErasureError('retained_row_missing', 409);
-    if (/CHECK constraint failed/.test(String(error?.message ?? error))) throw new ErasureError('erasure_busy', 409);
-    throw error;
+    return lost(error);
   }
-  return !final;
+  return false;
 }

package/modules/erasure/order.mjs

@@ -1,6 +1,24 @@
 import { all, canonicalJson, insist, quote } from './sql.mjs';
 import { matching } from './plan.mjs';
 
+/** Every foreign key between two deleted tables, grouped per constraint. Schema
+ * only: it never reads a row, so a later step can use it without a plan. */
+export async function foreignKeyLinks(db, d) {
+  const links = [];
+  for (const table of d.deleted) {
+    const groups = new Map();
+    for (const fk of await all(db, `PRAGMA foreign_key_list(${quote(table)})`)) {
+      if (!groups.has(fk.id)) groups.set(fk.id, []);
+      groups.get(fk.id).push(fk);
+    }
+    for (const group of groups.values()) {
+      const parentTable = d.deleted.find(name => name.toLowerCase() === group[0].table.toLowerCase());
+      if (parentTable) links.push({ table, parentTable, group });
+    }
+  }
+  return links;
+}
+
 /** Topological row order across every path, not the depth of the first visit.
  * Actual FK links also constrain order, even if a row was reached by another edge. */
 export async function deletionOrder(db, d, plan) {
@@ -18,28 +36,17 @@ export async function deletionOrder(db, d, plan) {
       if (child) link(parent, child);
     }
   }
-  plan.foreignKeys = [];
-  for (const table of d.deleted) {
-    const groups = new Map();
-    for (const fk of await all(db, `PRAGMA foreign_key_list(${quote(table)})`)) {
-      if (!groups.has(fk.id)) groups.set(fk.id, []);
-      groups.get(fk.id).push(fk);
-    }
-    for (const group of groups.values()) {
-      const parentTable = d.deleted.find(name => name.toLowerCase() === group[0].table.toLowerCase());
-      const parents = parentTable && plan.rows.get(parentTable);
-      if (parentTable) plan.foreignKeys.push({ table, parentTable, group });
-      if (!parents) continue;
-      for (const parent of parents.values()) {
-        const parentKeys = plan.keys.get(parent.table).pk;
-        // Read the actual join: FK columns need not be part of a declared edge's projection.
-        const cols = plan.keys.get(table).pk;
-        const found = await all(db, `SELECT ${cols.map(c => c === 'rowid' ? 'rowid AS rowid' : quote(c)).join(',')} FROM ${quote(table)} WHERE ${group.map(f => `${quote(f.from)}=(SELECT ${quote(f.to || parentKeys[f.seq])} FROM ${quote(parent.table)} WHERE ${parentKeys.map(c => `${quote(c)}=?`).join(' AND ')})`).join(' AND ')}`, ...group.flatMap(() => parentKeys.map(c => parent.key[c])));
-        for (const row of found) {
-          const child = entries.get(table + '\0' + canonicalJson(Object.fromEntries(cols.map(c => [c, row[c]]))));
-          insist(child, `unreached_dependent:${table}:${parent.table}`, 409);
-          link(parent, child);
-        }
+  plan.foreignKeys = await foreignKeyLinks(db, d);
+  for (const { table, parentTable, group } of plan.foreignKeys) {
+    for (const parent of plan.rows.get(parentTable).values()) {
+      const parentKeys = plan.keys.get(parent.table).pk;
+      // Read the actual join: FK columns need not be part of a declared edge's projection.
+      const cols = plan.keys.get(table).pk;
+      const found = await all(db, `SELECT ${cols.map(c => c === 'rowid' ? 'rowid AS rowid' : quote(c)).join(',')} FROM ${quote(table)} WHERE ${group.map(f => `${quote(f.from)}=(SELECT ${quote(f.to || parentKeys[f.seq])} FROM ${quote(parent.table)} WHERE ${parentKeys.map(c => `${quote(c)}=?`).join(' AND ')})`).join(' AND ')}`, ...group.flatMap(() => parentKeys.map(c => parent.key[c])));
+      for (const row of found) {
+        const child = entries.get(table + '\0' + canonicalJson(Object.fromEntries(cols.map(c => [c, row[c]]))));
+        insist(child, `unreached_dependent:${table}:${parent.table}`, 409);
+        link(parent, child);
       }
     }
   }

package/modules/erasure/plan.mjs

@@ -40,7 +40,9 @@ export function matchSql(item, parents) {
   if (!item.on) return parents.map(p => ({ where: `(${item.where})`, values: (item.params ?? []).map(c => p[c]) }));
   const pairs = Object.entries(item.on);
   const tuples = parents.map(p => pairs.map(([, parent]) => p[parent])).filter(t => t.every(v => v !== null && v !== undefined));
-  const where = `EXISTS(SELECT 1 FROM json_each(?) j WHERE ${pairs.map(([child], i) => `${quote(item.table)}.${quote(child)}=json_extract(j.value,'$[${i}]')`).join(' AND ')})`;
+  // A set lookup, so an index on the child columns serves it; json_each's own
+  // columns (id, key, value...) are never in scope beside the child's.
+  const where = `(${pairs.map(([child]) => `${quote(item.table)}.${quote(child)}`).join(',')}) IN (SELECT ${pairs.map((_, i) => `json_extract(j.value,'$[${i}]')`).join(',')} FROM json_each(?) j)`;
   return chunks(tuples, 200).map(part => ({ where, values: [JSON.stringify(part)] }));
 }
 
@@ -71,7 +73,7 @@ export async function collect(db, d, subject, remembered = []) {
   for (const [table, old] of known) {
     const pk = keys.get(table).pk;
     for (const part of chunks([...old.values()], 200)) {
-      const found = await all(db, `SELECT ${projection(columns.get(table))} FROM ${quote(table)} WHERE EXISTS(SELECT 1 FROM json_each(?) j WHERE ${pk.map((c, i) => `${quote(c)} IS json_extract(j.value,'$[${i}]')`).join(' AND ')})`, JSON.stringify(part.map(row => pk.map(c => row[c]))));
+      const found = await all(db, `SELECT ${projection(columns.get(table))} FROM ${quote(table)} WHERE EXISTS(SELECT 1 FROM json_each(?) j WHERE ${pk.map((c, i) => `${quote(table)}.${c === 'rowid' ? 'rowid' : quote(c)} IS json_extract(j.value,'$[${i}]')`).join(' AND ')})`, JSON.stringify(part.map(row => pk.map(c => row[c]))));
       for (const row of found) { remember(table, row); const key = keyOf(table, row); rows.get(table).set(canonicalJson(key), { table, key, row, depth: 1 }); }
     }
   }

package/modules/erasure/queue.mjs

@@ -0,0 +1,41 @@
+import { all, statement, canonicalJson } from './sql.mjs';
+import { tableKeys } from './plan.mjs';
+import { foreignKeyLinks } from './order.mjs';
+
+/** What is left of a plan after its first step, in deletion order, held in the
+ * private scope table (no schema of its own). Each later step reads only its
+ * slice, so no step re-reads or re-checks the rows still waiting: the cost of
+ * the whole erasure stays linear in its rows. The slice is verified inside the
+ * step's batch; anything the queue does not know about (rows written after it
+ * was built) is found by the full collection that runs once the queue is empty. */
+const seq = n => String(n).padStart(10, '0');
+
+export const clearQueue = (db, id) => statement(db, "DELETE FROM rer_erasure_scope WHERE operation=? AND kind='queue'", id);
+
+/** Replaces the operation's queue with `entries`; chunks stay small enough for one bound value. */
+export function queueStatements(db, id, entries) {
+  const out = [clearQueue(db, id)], texts = entries.map((e, i) => canonicalJson({ i, t: e.table, k: e.key, r: e.row }));
+  for (let from = 0, to = 0; from < texts.length; from = to) {
+    for (let size = 0; to < texts.length && size + texts[to].length < 60000 && to - from < 200; to++) size += texts[to].length;
+    if (to === from) to++;
+    out.push(statement(db, "INSERT INTO rer_erasure_scope SELECT ?,'queue',printf('%010d',json_extract(j.value,'$.i')),'{}',j.value FROM json_each(?) j", id, `[${texts.slice(from, to).join(',')}]`));
+  }
+  return out;
+}
+
+export async function readQueue(db, id, limit) {
+  const rows = await all(db, "SELECT table_name AS seq,row_json FROM rer_erasure_scope WHERE operation=? AND kind='queue' ORDER BY table_name LIMIT ?", id, limit);
+  return rows.map(r => { const e = JSON.parse(r.row_json); return { seq: r.seq, table: e.t, key: e.k, row: e.r, depth: 1 }; });
+}
+export const consumeStatement = (db, id, last) => statement(db, "DELETE FROM rer_erasure_scope WHERE operation=? AND kind='queue' AND table_name<=?", id, last);
+
+/** The part of a plan one step needs: only the rows it deletes are tracked, so
+ * every check in its batch is bounded by the slice, not by the rows waiting. */
+export async function slicePlan(db, d, subject, entries) {
+  const keys = new Map();
+  for (const t of new Set([...d.tracked, ...d.redact.map(r => r.table)])) keys.set(t, await tableKeys(db, t));
+  const rows = new Map(d.tracked.map(t => [t, new Map()])), known = new Map(d.tracked.map(t => [t, new Map()]));
+  for (const e of entries) { rows.get(e.table).set(canonicalJson(e.key), e); known.get(e.table).set(canonicalJson(e.row), e.row); }
+  return { subject, root: null, keys, rows, entries, known, order: entries, foreignKeys: await foreignKeyLinks(db, d) };
+}
+export const rootScopeExists = async (db, id, d) => (await all(db, "SELECT 1 FROM rer_erasure_scope WHERE operation=? AND kind='graph' AND table_name=? LIMIT 1", id, d.root.table)).length > 0;

package/modules/erasure/scope.mjs

@@ -6,17 +6,17 @@ import { tableKeys } from './plan.mjs';
 export async function readScope(db, id) {
   return (await all(db, "SELECT table_name,key_json,row_json FROM rer_erasure_scope WHERE operation=? AND kind='graph'", id)).map(r => ({ table: r.table_name, key: JSON.parse(r.key_json), row: JSON.parse(r.row_json) }));
 }
-export async function scopeStatements(db, d, plan, id, entries = plan.entries) {
+export async function scopeStatements(db, d, plan, id, entries = plan.entries, { whole = true } = {}) {
   const out = [];
   for (const { table, row, key } of entries) {
     out.push(statement(db, "INSERT OR IGNORE INTO rer_erasure_scope VALUES(?,'graph',?,?,?)", id, table, canonicalJson(key), canonicalJson(row)));
   }
-  for (const { table } of d.retain) {
+  for (const { table } of whole ? d.retain : []) {
     const { pk } = await tableKeys(db, table);
     const key = `json_object(${pk.flatMap(c => ["'" + c.replaceAll("'", "''") + "'", quote(c)]).join(',')})`;
     out.push(statement(db, `INSERT OR IGNORE INTO rer_erasure_scope SELECT ?,'retain',?,${key},${key} FROM ${quote(table)}`, id, table));
   }
-  if (d.root.redact && plan.root) {
+  if (whole && d.root.redact && plan.root) {
     const identity = Object.fromEntries((d.root.redact.identityColumns ?? []).map(c => [c, plan.root.row[c]]));
     out.push(statement(db, "INSERT INTO rer_erasure_scope SELECT ?,'identity',?,?,? WHERE NOT EXISTS(SELECT 1 FROM rer_erasure_scope WHERE operation=? AND kind='identity')", id, d.root.table, canonicalJson(plan.subject), canonicalJson(identity), id));
   }

package/modules/erasure/verification.mjs

@@ -19,9 +19,11 @@ export async function verificationStatements(db, d, plan, id, label, { final = f
       const pk = plan.keys.get(table).pk, parentPk = plan.keys.get(parentTable).pk;
       const parents = [...plan.rows.get(parentTable).values()].map(e => parentPk.map(c => e.key[c]));
       const children = [...plan.rows.get(table).values()].map(e => pk.map(c => e.key[c]));
-      const keyMatch = (alias, columns) => columns.map((c, i) => `${alias}.${quote(c)} IS json_extract(j.value,'$[${i}]')`).join(' AND ');
+      if (!parents.length) continue;
+      const keyMatch = (alias, columns, json = 'j') => columns.map((c, i) => `${alias}.${quote(c)} IS json_extract(${json}.value,'$[${i}]')`).join(' AND ');
       const joined = group.map(f => `c.${quote(f.from)}=p.${quote(f.to || parentPk[f.seq])}`).join(' AND ');
-      out.push(verify(db, 'scope', `NOT EXISTS(SELECT 1 FROM ${quote(table)} c JOIN ${quote(parentTable)} p ON ${joined} WHERE EXISTS(SELECT 1 FROM json_each(?) j WHERE ${keyMatch('p', parentPk)}) AND NOT EXISTS(SELECT 1 FROM json_each(?) j WHERE ${keyMatch('c', pk)}))`, JSON.stringify(parents), JSON.stringify(children)));
+      // Parents first, then their children through the key: cost follows the parents, not the child table.
+      out.push(verify(db, 'scope', `NOT EXISTS(SELECT 1 FROM json_each(?) j CROSS JOIN ${quote(parentTable)} p CROSS JOIN ${quote(table)} c WHERE ${keyMatch('p', parentPk)} AND ${joined} AND NOT EXISTS(SELECT 1 FROM json_each(?) k WHERE ${keyMatch('c', pk, 'k')}))`, JSON.stringify(parents), JSON.stringify(children)));
     }
     for (const edge of d.edges) for (const m of matchSql(edge, [...plan.known.get(edge.from).values()])) {
       const pk = plan.keys.get(edge.table).pk, keys = [...plan.rows.get(edge.table).values()].map(e => pk.map(c => e.key[c]));

package/modules/mail/README.md

@@ -149,7 +149,7 @@ Network loss, redirects, HTTP 408 and 5xx are conservatively `uncertain`; 429 is
 
 ## Development and unsubscribe
 
-Development requires all three: `provider:'development'`, `mode:'development'`, and explicit `dependencies.outbox.put`. It captures the prepared message with its raw attachment bytes and returns `development`, never remote acceptance. Configure storage only in your development host; no route is mounted automatically and no network replay is provided. Treat captured messages and sign-in links as sensitive local data.
+Development requires all three: `provider:'development'`, `mode:'development'`, and explicit `dependencies.outbox.put`. It captures the prepared message with its raw attachment bytes and returns `development`, never remote acceptance. A storage failure throws `DevelopmentOutboxError` (`code: 'development_outbox_failed'`, `retryable: true`): the message is valid, so a queue consumer retries it. Invalid input throws `MailValidationError`, which is permanent and is dropped. Configure storage only in your development host; no route is mounted automatically and no network replay is provided. Treat captured messages and sign-in links as sensitive local data.
 
 ### Adopting the local outbox
 

package/modules/mail/index.mjs

@@ -1,6 +1,6 @@
 export { createMailSender } from './src/sender.mjs';
 
-export { MailValidationError } from './src/core/errors.mjs';
+export { MailValidationError, DevelopmentOutboxError } from './src/core/errors.mjs';
 export { serializeSendGridMessage } from './src/adapters/sendgrid/index.mjs';
 export { verifyWebhook, MAX_WEBHOOK_BYTES, createDeliveryEventIntake, createD1EventStore } from './src/events/index.mjs';
 export { normalizeInbound, createResendAttachmentFetcher } from './src/inbound/index.mjs';

package/modules/mail/src/core/errors.mjs

@@ -7,3 +7,12 @@ export class MailValidationError extends Error {
   }
 }
 export function fail(code) { throw new MailValidationError(code); }
+/** The development outbox could not store the message. The message is fine: send it again. */
+export class DevelopmentOutboxError extends Error {
+  constructor() {
+    super('Development outbox storage failed; the same message may succeed when sent again.');
+    this.name = 'DevelopmentOutboxError';
+    this.code = 'development_outbox_failed';
+    this.retryable = true;
+  }
+}

package/modules/mail/src/development/index.d.mts

@@ -27,6 +27,12 @@ export interface DevelopmentD1Database {
   prepare(sql: string): DevelopmentD1Statement;
   batch(statements: DevelopmentD1Statement[]): Promise<Array<{ success: boolean; results: unknown[] }>>;
 }
+/** Storage failure: the message is valid, so a queue consumer retries instead of dropping it. Invalid input throws `MailValidationError`, which is permanent. */
+export class DevelopmentOutboxError extends Error {
+  readonly name: 'DevelopmentOutboxError';
+  readonly code: 'development_outbox_failed';
+  readonly retryable: true;
+}
 export function createD1DevelopmentOutbox(options: { db: DevelopmentD1Database; maxMessages?: number; maxAgeMs?: number; clock?: () => number }): DevelopmentOutbox;
 export function createDevelopmentOutboxHandler(options: {
   local: boolean; origin: string; outbox: Pick<DevelopmentOutbox, 'list' | 'get'>; path?: string;

package/modules/mail/src/development/index.mjs

@@ -1,9 +1,10 @@
-import { MailValidationError, fail } from '../core/errors.mjs';
+import { MailValidationError, DevelopmentOutboxError, fail } from '../core/errors.mjs';
 import { validateConfig, validateMessage } from '../core/validation.mjs';
 import { sendResult } from '../core/transport.mjs';
 
 export { createD1DevelopmentOutbox } from './outbox.mjs';
 export { createDevelopmentOutboxHandler } from './viewer.mjs';
+export { DevelopmentOutboxError } from '../core/errors.mjs';
 
 function validate(operation, errorCode) {
   try {
@@ -27,8 +28,10 @@ export async function sendDevelopment(config, message, dependencies = {}) {
 
   try {
     await dependencies.outbox.put(snapshot);
-  } catch {
-    fail('development_outbox_failed');
+  } catch (error) {
+    // Input the outbox itself refuses stays permanent; a storage failure is worth a retry.
+    if (error instanceof MailValidationError) throw error;
+    throw new DevelopmentOutboxError();
   }
 
   return sendResult('development', 'development', null);

package/modules/mail/src/development/viewer-html.mjs

@@ -1,4 +1,4 @@
-export const style = `*{box-sizing:border-box}body{margin:0;background:#f5f7fa;color:#17212e;font:16px/1.6 system-ui,sans-serif}main{max-width:52rem;margin:0 auto;padding:24px 16px}h1{font-size:1.75rem;line-height:1.2}article{margin:20px 0;padding:20px;background:white;border:1px solid #cad2dc;border-radius:12px}h2{font-size:1.2rem}h1,h2,p,a,pre{overflow-wrap:anywhere;word-break:normal}pre{white-space:pre-wrap;font:inherit}a{color:#174fb5}a.open{display:block;padding:16px;margin:20px 0;border-radius:8px;background:#174fb5;color:white;font-weight:700;text-align:center;text-decoration:none;min-height:56px}small{color:#465569}`;
+export const style = `*{box-sizing:border-box}body{margin:0;background:#f5f7fa;color:#17212e;font:16px/1.6 system-ui,sans-serif}main{max-width:52rem;margin:0 auto;padding:24px 16px}h1{font-size:1.75rem;line-height:1.2}article{margin:20px 0;padding:20px;background:white;border:1px solid #cad2dc;border-radius:12px}h2{font-size:1.2rem}h1,h2,p,a,pre{overflow-wrap:anywhere;word-break:normal}pre{white-space:pre-wrap;font:inherit}a{color:#174fb5}a.open{display:block;padding:16px;margin:20px 0;border-radius:8px;background:#174fb5;color:white;font-weight:700;text-align:center;text-decoration:none;min-height:56px}a.nav{display:inline-block;padding:4px 0}small{color:#465569}`;
 
 // Added only when the seed declares a theme: its linked rer-theme.css supplies the
 // --rer-* tokens. Each falls back to the plain sheet's value, and the focus ring is
@@ -33,10 +33,10 @@ export function document(body, theme) {
 }
 
 export function listing(rows, path) {
-  return rows.length ? rows.map(row => `<article>${metadata(row)}<p><a href="${escape(`${path}?message=${encodeURIComponent(row.id)}`)}">Read message</a></p></article>`).join('') : '<p>No messages yet.</p>';
+  return rows.length ? rows.map(row => `<article>${metadata(row)}<p><a href="${escape(`${path}?message=${encodeURIComponent(row.id)}`)}" class="nav">Read message</a></p></article>`).join('') : '<p>No messages yet.</p>';
 }
 
 export function detail(row, path) {
   const link = firstLink(row.message);
-  return `<p><a href="${escape(path)}">All messages</a></p><article>${metadata(row)}${link ? `<a class="open" href="${escape(link)}" rel="noreferrer noopener">Open this link</a>` : ''}<h2>Message text</h2><pre>${escape(row.message.text || row.message.html || '')}</pre>${row.message.attachments?.length ? `<p>${row.message.attachments.length} attachment(s) stored with this message.</p>` : ''}</article>`;
+  return `<p><a href="${escape(path)}" class="nav">All messages</a></p><article>${metadata(row)}${link ? `<a class="open" href="${escape(link)}" rel="noreferrer noopener">Open this link</a>` : ''}<h2>Message text</h2><pre>${escape(row.message.text || row.message.html || '')}</pre>${row.message.attachments?.length ? `<p>${row.message.attachments.length} attachment(s) stored with this message.</p>` : ''}</article>`;
 }

package/modules/mail/src/development/viewer.mjs

@@ -1,11 +1,18 @@
 import { detail, document, listing, style, themedStyle } from './viewer-html.mjs';
 
+// The mail module runs as its own module root (in workerd too), so it cannot
+// import operator-auth's shared predicate (modules/operator-auth/loopback.mjs);
+// this is the same rule: the loopback hosts, 127/8, or a name under .localhost
+// (the front door's http://NAME.localhost:PORT).
+const localhostName = /^(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+localhost$/;
+const isLoopbackHost = hostname => hostname === 'localhost' || hostname === '127.0.0.1' || hostname === '[::1]' || (hostname.length <= 253 && localhostName.test(hostname));
+
 function localOrigin(origin) {
   if (typeof origin !== 'string') return null;
   try {
     const url = new URL(origin);
     const ipv4 = url.hostname.split('.');
-    const loopback = url.hostname === 'localhost' || url.hostname === '[::1]' ||
+    const loopback = isLoopbackHost(url.hostname) ||
       (ipv4.length === 4 && ipv4[0] === '127' && ipv4.every(part => /^(0|[1-9]\d{0,2})$/.test(part) && Number(part) <= 255));
     return loopback && ['http:', 'https:'].includes(url.protocol) && url.origin === origin ? url : null;
   } catch { return null; }