@substrat-run/adapter-sqlite 0.114.0 → 0.116.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,8 +1,8 @@
1
1
  import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
2
2
  import { dirname, join } from 'node:path';
3
3
  import Database from 'better-sqlite3';
4
- import { accessLogEntry, adminLogEntry, opsFailureEntry, opsFailureFingerprint, issueEntry, FRESHNESS_HEARTBEAT_MINUTES, sweepRunEntry, modelUsageEntry, ATTACHMENT_ADDED, ATTACHMENT_REMOVED, attachmentRecord, createTenantInput, domainEvent, domainEventInput, entitlementGrant, entitlementGrantInput, eventId, meterReading, platformRequestInput, platformRequestId, platformRequest, MAX_PENDING_PLATFORM_REQUESTS, identityLink, identityPool, instant, connection, capabilityGrant, connectionGrant, connectionGrantRecord, connectionSecret, systemGrant, subjectRef, requestFingerprint, createConnectionInput, projectedConnectionGrant, projectedConnectionKey, moduleManifest, createOrgInput, promotionAcknowledgement, bindHostnameInput, channelHistoryEntry, hostnameBinding, publishVersionInput, AUTO_ADMISSION_NOTE, routeTarget, registerVerticalInput, vertical as verticalSchema, verticalChannel, verticalVersion, objectRef, grantRefFromProof, org as orgSchema, orgMembership, principalId, resolvedIdentity, roleDefinition, scope as scopeSchema, tenant as tenantSchema, tenantRole, subjectShredReceipt, SCOPE_TABLE_PAGE_DEFAULT, SCOPE_TABLE_PAGE_MAX, SCOPE_QUERY_ROW_MAX, verticalServingState, outboundOfManifestJson, listLimitOf, substratError, assertReplayableDump, delegatedReadRecord, redrainEventsInput, REDRAIN_BATCH, } from '@substrat-run/contracts';
5
- import { asPrincipal, assertAllowed, assertReadOnlyQuery, attachmentBlobKey, entitlementDenial, foldMeterReading, guardSpine, parseValidationRecords, resolveScopeRecord, ulid, ISSUE_RETENTION_DAYS, OPS_FAILURE_RETENTION_DAYS, SWEEP_RUN_RETENTION_DAYS, MODEL_USAGE_RETENTION_DAYS, foldModelUsage, backoffAt, platformRequestHistoryQuery, PLATFORM_REQUEST_COLUMNS, denialListQuery, denialSummaryQuery, denialTotalsQuery, DENIAL_WINDOW_QUERY, mapDenialRow, mapDenialSummaryBuckets, IMPERSONATION_COLUMNS, IMPERSONATION_DDL, ImpersonationRefused, assertImpersonationWrites, assertModuleEnqueueableKind, assertSessionUsable, impersonationByIdQuery, impersonationListQuery, impersonationRowValues, impersonationStampOf, mapImpersonationRow, newImpersonationSession, resolveRetryPolicy, isSecretBoxConfigured, unconfiguredSecretBox, createSubjectKeys, generateSealingKeyPair, noSealingKeyMessage, openSealed, sealTo, ConnectionSealingKeyUnavailableError, PermissionDenied, createAtomic, NotSearchable, NotListable, listIndexMigrations, listIndexPlans, listQuery, cursorOf, isSearchIndexTable, searchIndexDdl, searchIndexMigrations, searchIndexPlans, searchLimit, searchMatchExpression, searchQuery, entityVersionQuery, entityVersionOf, assertIfMatch, OUTBOX_ENTITY_INDEX, IDEMPOTENCY_DDL, assertIdempotencyKey, idempotencyLookupQuery, idempotencyPruneStatement, idempotencyRecordStatement, idempotencyOptedOutMessage, replayFor, globalFetch, createUlid, facetEvents, readHistory, walkEventCause, walkEventEffects, assertRedrainWindow, readInvocation, readDeadLetters, } from '@substrat-run/kernel';
4
+ import { accessLogEntry, adminLogEntry, opsFailureEntry, opsFailureFingerprint, issueEntry, FRESHNESS_HEARTBEAT_MINUTES, sweepRunEntry, modelUsageEntry, ATTACHMENT_ADDED, ATTACHMENT_REMOVED, attachmentRecord, createTenantInput, domainEvent, domainEventInput, entitlementGrant, entitlementGrantInput, eventId, meterReading, platformRequestInput, platformRequestId, platformRequest, MAX_PENDING_PLATFORM_REQUESTS, identityLink, identityPool, instant, connection, capabilityGrant, connectionGrant, connectionGrantRecord, connectionSecret, systemGrant, subjectRef, requestFingerprint, createConnectionInput, projectedConnectionGrant, projectedConnectionKey, moduleManifest, createOrgInput, promotionAcknowledgement, bindHostnameInput, channelHistoryEntry, hostnameBinding, publishVersionInput, AUTO_ADMISSION_NOTE, routeTarget, registerVerticalInput, vertical as verticalSchema, verticalChannel, verticalVersion, objectRef, grantRefFromProof, org as orgSchema, orgMembership, principalId, resolvedIdentity, identityMembership, roleDefinition, scope as scopeSchema, tenant as tenantSchema, tenantRole, subjectShredReceipt, SCOPE_TABLE_PAGE_DEFAULT, SCOPE_TABLE_PAGE_MAX, SCOPE_QUERY_ROW_MAX, verticalServingState, outboundOfManifestJson, listLimitOf, substratError, assertReplayableDump, delegatedReadRecord, redrainEventsInput, REDRAIN_BATCH, } from '@substrat-run/contracts';
5
+ import { asPrincipal, assertAllowed, assertReadOnlyQuery, attachmentBlobKey, entitlementDenial, foldMeterReading, guardSpine, parseValidationRecords, resolveScopeRecord, ulid, ISSUE_RETENTION_DAYS, OPS_FAILURE_RETENTION_DAYS, SWEEP_RUN_RETENTION_DAYS, MODEL_USAGE_RETENTION_DAYS, foldModelUsage, backoffAt, platformRequestHistoryQuery, PLATFORM_REQUEST_COLUMNS, denialListQuery, denialSummaryQuery, denialTotalsQuery, DENIAL_WINDOW_QUERY, mapDenialRow, mapDenialSummaryBuckets, IMPERSONATION_COLUMNS, IMPERSONATION_DDL, ImpersonationRefused, assertImpersonationWrites, assertModuleEnqueueableKind, assertSessionUsable, impersonationByIdQuery, impersonationListQuery, impersonationRowValues, impersonationStampOf, mapImpersonationRow, newImpersonationSession, resolveRetryPolicy, isSecretBoxConfigured, unconfiguredSecretBox, createSubjectKeys, generateSealingKeyPair, noSealingKeyMessage, openSealed, sealTo, ConnectionSealingKeyUnavailableError, PermissionDenied, createAtomic, NotSearchable, NotListable, listIndexMigrations, listIndexPlans, listQuery, cursorOf, isSearchIndexTable, searchIndexDdl, searchIndexMigrations, searchIndexPlans, searchLimit, searchMatchExpression, searchQuery, entityVersionQuery, entityVersionOf, assertIfMatch, OUTBOX_ENTITY_INDEX, SCHEDULE_STATE_DDL, SCHEDULE_STATE_REBUILD, scheduleStateHasKind, JOB_RUN_DDL, jobRunListLimit, jobRunOf, runDueJobRuns, startJobRun, IDEMPOTENCY_DDL, assertIdempotencyKey, idempotencyLookupQuery, idempotencyPruneStatement, idempotencyRecordStatement, idempotencyOptedOutMessage, replayFor, globalFetch, createUlid, facetEvents, readHistory, walkEventCause, walkEventEffects, assertRedrainWindow, readInvocation, readDeadLetters, } from '@substrat-run/kernel';
6
6
  import { ScopeActor } from './actor.js';
7
7
  import { createTupleChecker } from './checker.js';
8
8
  /**
@@ -119,10 +119,35 @@ const KERNEL_DDL = `
119
119
  operation TEXT,
120
120
  -- K-42: WHICH of the two actors was refused is exactly what this log is for.
121
121
  impersonation TEXT,
122
+ -- #1525: the INVOCATION this refusal happened during, the same id #1237 stamps on
123
+ -- every event of a call. A denial joins to nothing but actor and time otherwise —
124
+ -- the operation column names what was attempted, never WHICH attempt — so "what
125
+ -- else did this request do" cannot reach a refusal, which is the one thing an
126
+ -- incident asks about first. NULL = the transport carried no id (a seed, a test,
127
+ -- an internal call, an attachment RPC), or the row predates the column.
128
+ invocation_id TEXT,
122
129
  at TEXT NOT NULL,
123
130
  drained_at TEXT
124
131
  );
125
132
  ${IDEMPOTENCY_DDL}
133
+ -- #383 / #1232 / #1288: the platform sweep's per-scope gating state, holding two
134
+ -- families of row that the kind COLUMN — not the spelling of a key — tells apart.
135
+ -- Spine (kernel-written), never a module migration. Shared with the DO adapter from
136
+ -- @substrat-run/kernel, so the shape a rebuild produces and the shape a fresh store
137
+ -- gets cannot part company; the column comments are in there.
138
+ --
139
+ -- HERE since #1288, rather than lazily in runDueSchedules/checkFreshness as it was:
140
+ -- a table only one adapter's KERNEL_DDL declares is a one-sided NOTE to
141
+ -- lint:spine-ddl, never a failure, so the copies this issue had to keep in step were
142
+ -- the copies the gate could not see. Every scope db passes through runtime(), which
143
+ -- runs this, so the lazy creates were load-bearing for nothing.
144
+ ${SCHEDULE_STATE_DDL}
145
+ -- #1577: the resumable-run driver's record and the step ledger of the pass it
146
+ -- currently has in flight. Spine (kernel-written), never a module migration.
147
+ -- Shared with the DO adapter from @substrat-run/kernel so the shape a self-host
148
+ -- builds and the shape production builds cannot part company; the column
149
+ -- comments, and the reason coalescing is NOT a unique index, are in there.
150
+ ${JOB_RUN_DDL}
126
151
  CREATE TABLE IF NOT EXISTS _substrat_migrations (
127
152
  module_id TEXT NOT NULL,
128
153
  version TEXT NOT NULL,
@@ -155,6 +180,17 @@ const KERNEL_DDL = `
155
180
  -- next_attempt_at IS NULL -> terminal: error IS NULL delivered, else dead
156
181
  attempts INTEGER NOT NULL DEFAULT 0,
157
182
  next_attempt_at TEXT,
183
+ -- #1525: the invocation THIS ATTEMPT ran in, the same id #1237 stamps on every
184
+ -- event of a call. Not the emitting event's — that one is already on the outbox
185
+ -- and a delivery joins to it through event_id. What that join cannot say is
186
+ -- which call attempted the DELIVERY, and for an executor the two differ by
187
+ -- design: attempt one runs in the emitting call's post-commit tail, every retry
188
+ -- afterwards in a drain that is a different call or none. Moves with the row on
189
+ -- an upsert, like delivered_at, error and attempts — it describes the
190
+ -- latest attempt, not the first. NULL = that attempt carried no call (a
191
+ -- scheduled drain, an alarm, a seed, an attachment RPC), or the row predates
192
+ -- the column.
193
+ invocation_id TEXT,
158
194
  PRIMARY KEY (event_id, consumer_module)
159
195
  );
160
196
  -- Attachment metadata facts (#473): one row per object in the per-tenant blob store,
@@ -350,6 +386,32 @@ const admitByDelivery = (rt, eventId, row) => {
350
386
  };
351
387
  export class SqliteScopeHost {
352
388
  admin;
389
+ /**
390
+ * Live reads are not available on this host (#938) — `never`, not `undefined`.
391
+ *
392
+ * This is the `clock?: never` seam (`CloudflareScopeHostOptions`) pointing the other
393
+ * way. There, the hosted adapter refuses an option it could only honour for half of
394
+ * the host's judgements; here, the pure adapter refuses a surface it cannot honour at
395
+ * all. A live read needs something that outlives a request and can be woken when an
396
+ * event lands. On Cloudflare that is the scope's own Durable Object, which the runtime
397
+ * keeps addressable between requests. This host is a function call inside somebody
398
+ * else's process: nothing here survives the return, so the only honest answer is that
399
+ * there is no subscription to hand out.
400
+ *
401
+ * `never` rather than an omission, and rather than a `subscribe` that throws, because
402
+ * those two fail in the places this one does not: an omission makes
403
+ * `host.liveReads?.subscribe(…)` compile and silently do nothing, and a thrower moves
404
+ * the discovery to runtime, in production, on the one path nobody drove in a test.
405
+ * The field is declared so that reaching for it is refused by the compiler at the call
406
+ * site — the same argument the clock seam makes in its own comment.
407
+ *
408
+ * What this costs, stated rather than hidden: nothing in the fan-out is held to a
409
+ * CONTRACT suite, because a contract suite asserts the two adapters agree and on this
410
+ * they deliberately do not. The tests live on `adapter-cloudflare` instead, beside the
411
+ * only host that can run them. The day a pure host grows a long-lived transport, this
412
+ * comment and those tests' home are the whole change.
413
+ */
414
+ liveReads;
353
415
  dir;
354
416
  checker;
355
417
  directory;
@@ -399,6 +461,13 @@ export class SqliteScopeHost {
399
461
  roles = new Map(); // 'tenantId/roleKey'
400
462
  /** Executor id → {eventType, handler} (K-22 §4.2). Host code, not module code. */
401
463
  executors = new Map();
464
+ /**
465
+ * `<moduleId>/<job>` → the pass body and its default step policy (#1577). Host
466
+ * code like `executors`, and keyed the way a run row is: the coalescing key's
467
+ * first two thirds, so a run picked off the table finds its handler by the
468
+ * columns it already carries.
469
+ */
470
+ jobs = new Map();
402
471
  /**
403
472
  * The event currently being effected by an executor, stamped onto any admin rows
404
473
  * it writes. Ambient rather than threaded through every HostAdmin signature: it is
@@ -1627,12 +1696,16 @@ export class SqliteScopeHost {
1627
1696
  }
1628
1697
  catch (err) {
1629
1698
  rt.db.exec('ROLLBACK');
1699
+ // #1525: null. This task is the attachment RPC's own — no invoke set an id on
1700
+ // the runtime, and the actor guarantees no other task is mid-flight holding one.
1630
1701
  if (err instanceof PermissionDenied)
1631
- this.recordDenial(rt, subject, operation, err);
1702
+ this.recordDenial(rt, subject, operation, err, null);
1632
1703
  throw err;
1633
1704
  }
1634
- await this.dispatch(rt);
1635
- await this.dispatchExecutors(rt);
1705
+ // #1525: null, for the reason the denial two lines up records null — this task
1706
+ // is the attachment RPC's own and no invoke set an id on the runtime.
1707
+ await this.dispatch(rt, null);
1708
+ await this.dispatchExecutors(rt, null);
1636
1709
  return result;
1637
1710
  });
1638
1711
  /**
@@ -1653,8 +1726,12 @@ export class SqliteScopeHost {
1653
1726
  return await fn(this.operationContext(rt, subject, undefined, undefined, undefined, operation));
1654
1727
  }
1655
1728
  catch (err) {
1729
+ // #1525: the ENCLOSING call's id, deliberately. This runner takes no turn of its
1730
+ // own — it is the connector's dispatch-time read, already inside that call's
1731
+ // actor task — so its refusal belongs to that call, for the reason a consumer's
1732
+ // emit does (#1237): dispatch runs in the same post-commit tail and IS the call.
1656
1733
  if (err instanceof PermissionDenied)
1657
- this.recordDenial(rt, subject, operation, err);
1734
+ this.recordDenial(rt, subject, operation, err, rt.invocationId);
1658
1735
  throw err;
1659
1736
  }
1660
1737
  };
@@ -2148,22 +2225,11 @@ export class SqliteScopeHost {
2148
2225
  return report;
2149
2226
  const rt = this.runtime(tenantId, scopeId);
2150
2227
  await this.applyPendingMigrations(rt);
2151
- // Same lazy create as runDueSchedules — a freshness-only module must not hit
2152
- // `no such table` on a scope no schedule ever swept. Despite the table's name it
2153
- // holds two kinds of row (#1232): schedule operations keyed `module/verb`, and
2154
- // the rows written just below, keyed `freshness:<eventType>`. For those,
2155
- // `last_run_at`/`last_status` are the last RECORDED at and verdict — nothing ran.
2156
- // Only half the no-collision claim is enforced: an event type has passed
2157
- // contracts' `eventType` regex (lowercase `ns.verb`, no colon), so a freshness
2158
- // key can never look like an operation — but `scheduleSpec.operation` is
2159
- // `z.string().min(1)`, so a schedule literally named `freshness:orders.placed`
2160
- // would share this row and nothing would refuse it. #1288 tracks the `kind`
2161
- // column that would make the distinction real instead of conventional.
2162
- rt.db.exec(`CREATE TABLE IF NOT EXISTS _substrat_schedule_state (
2163
- schedule_op TEXT PRIMARY KEY,
2164
- last_run_at TEXT,
2165
- last_status TEXT
2166
- )`);
2228
+ // The gating-state table is KERNEL_DDL's since #1288 — `runtime()` above ran it,
2229
+ // so a freshness-only module on a scope no schedule ever swept finds the table
2230
+ // there rather than creating it here. The rows written below are the evaluator's,
2231
+ // `kind = 'freshness'`, keyed `freshness:<eventType>`: for those `last_run_at` /
2232
+ // `last_status` are the last RECORDED at and verdict — nothing ran.
2167
2233
  // Collapse duplicate event types to the tightest window, ACROSS modules.
2168
2234
  const windows = new Map();
2169
2235
  for (const m of this.modules.values()) {
@@ -2194,8 +2260,12 @@ export class SqliteScopeHost {
2194
2260
  ? 'ok'
2195
2261
  : 'failed';
2196
2262
  const stateKey = `freshness:${eventType}`;
2263
+ // #1288: the evaluator's own row, named by kind. Before the column, a module
2264
+ // declaring a schedule called `freshness:<this type>` shared it — and the
2265
+ // verdict read back here was whichever of the two wrote last.
2197
2266
  const state = rt.db
2198
- .prepare('SELECT last_run_at, last_status FROM _substrat_schedule_state WHERE schedule_op = ?')
2267
+ .prepare(`SELECT last_run_at, last_status FROM _substrat_schedule_state
2268
+ WHERE kind = 'freshness' AND schedule_op = ?`)
2199
2269
  .get(stateKey);
2200
2270
  const changed = state?.last_status !== outcome;
2201
2271
  const heartbeatDue = !state?.last_run_at ||
@@ -2203,10 +2273,10 @@ export class SqliteScopeHost {
2203
2273
  if (!changed && !heartbeatDue)
2204
2274
  continue;
2205
2275
  rt.db
2206
- .prepare(`INSERT INTO _substrat_schedule_state (schedule_op, last_run_at, last_status)
2207
- VALUES (?, ?, ?)
2208
- ON CONFLICT(schedule_op) DO UPDATE SET last_run_at = excluded.last_run_at,
2209
- last_status = excluded.last_status`)
2276
+ .prepare(`INSERT INTO _substrat_schedule_state (kind, schedule_op, last_run_at, last_status)
2277
+ VALUES ('freshness', ?, ?, ?)
2278
+ ON CONFLICT(kind, schedule_op) DO UPDATE SET last_run_at = excluded.last_run_at,
2279
+ last_status = excluded.last_status`)
2210
2280
  .run(stateKey, nowIso, outcome);
2211
2281
  report.checks.push({ eventType, outcome, observedAt, withinHours });
2212
2282
  }
@@ -2240,21 +2310,20 @@ export class SqliteScopeHost {
2240
2310
  .get(`system:${moduleId}`, nowIso);
2241
2311
  if (!hasGrant)
2242
2312
  return report;
2243
- // The spine state table is created lazily on first sweep of a scope — it is
2244
- // kernel-owned (`_substrat_*`), never in a module migration. It holds this
2245
- // loop's rows, keyed by the schedule operation (`module/verb`, where
2246
- // `last_run_at`/`last_status` are when it RAN and how it ended), and — since
2247
- // #1232 — the freshness evaluator's, keyed `freshness:<eventType>`. See
2248
- // checkFreshness for what the columns mean there; #1288 tracks the widening.
2249
- rt.db.exec(`CREATE TABLE IF NOT EXISTS _substrat_schedule_state (
2250
- schedule_op TEXT PRIMARY KEY,
2251
- last_run_at TEXT,
2252
- last_status TEXT
2253
- )`);
2313
+ // The spine state table is KERNEL_DDL's (#1288) — kernel-owned (`_substrat_*`),
2314
+ // never a module migration, and created by `runtime()` above rather than lazily
2315
+ // here. This loop's rows are `kind = 'schedule'`, keyed by the operation
2316
+ // (`module/verb`, where `last_run_at`/`last_status` are when it RAN and how it
2317
+ // ended); the freshness evaluator's are `kind = 'freshness'` — see checkFreshness
2318
+ // for what the same two columns mean there.
2254
2319
  const now = Date.parse(nowIso);
2255
2320
  for (const schedule of mod.schedules) {
2321
+ // #1288: this schedule's own row. `kind` is what makes that true even when the
2322
+ // operation is spelled `freshness:<eventType>` — without it, the cadence gate
2323
+ // below read the evaluator's last recorded verdict as a run of this operation.
2256
2324
  const row = rt.db
2257
- .prepare('SELECT last_run_at FROM _substrat_schedule_state WHERE schedule_op = ?')
2325
+ .prepare(`SELECT last_run_at FROM _substrat_schedule_state
2326
+ WHERE kind = 'schedule' AND schedule_op = ?`)
2258
2327
  .get(schedule.operation);
2259
2328
  const lastRun = row?.last_run_at ? Date.parse(row.last_run_at) : null;
2260
2329
  const dueAt = lastRun === null ? -Infinity : lastRun + schedule.cadence.everyMinutes * 60_000;
@@ -2280,10 +2349,10 @@ export class SqliteScopeHost {
2280
2349
  });
2281
2350
  }
2282
2351
  rt.db
2283
- .prepare(`INSERT INTO _substrat_schedule_state (schedule_op, last_run_at, last_status)
2284
- VALUES (?, ?, ?)
2285
- ON CONFLICT(schedule_op) DO UPDATE SET last_run_at = excluded.last_run_at,
2286
- last_status = excluded.last_status`)
2352
+ .prepare(`INSERT INTO _substrat_schedule_state (kind, schedule_op, last_run_at, last_status)
2353
+ VALUES ('schedule', ?, ?, ?)
2354
+ ON CONFLICT(kind, schedule_op) DO UPDATE SET last_run_at = excluded.last_run_at,
2355
+ last_status = excluded.last_status`)
2287
2356
  .run(schedule.operation, new Date(now).toISOString(), status);
2288
2357
  report.runs.push({ operation: schedule.operation, outcome: status === 'ok' ? 'ok' : 'failed' });
2289
2358
  }
@@ -2481,7 +2550,8 @@ export class SqliteScopeHost {
2481
2550
  // K-35: a refused check rolled the operation back. Record it now — a fresh
2482
2551
  // statement in autocommit, AFTER the rollback, so the denial survives it.
2483
2552
  if (err instanceof PermissionDenied)
2484
- this.recordDenial(rt, subject, operation, err, session);
2553
+ // Inside the actor task that set it, so the field is this call's own (#1237).
2554
+ this.recordDenial(rt, subject, operation, err, rt.invocationId, session);
2485
2555
  throw err;
2486
2556
  }
2487
2557
  // Post-commit, still inside the actor task: drain outbox → consumers,
@@ -2492,8 +2562,10 @@ export class SqliteScopeHost {
2492
2562
  // drain — and draining anyway would run consumers as a side effect of a
2493
2563
  // session that may not have side effects.
2494
2564
  if (session?.mode !== 'read-only') {
2495
- await this.dispatch(rt);
2496
- await this.dispatchExecutors(rt);
2565
+ // #1525: still inside the actor task that set it, so these deliveries are
2566
+ // this call's own work — the same tail its consumers' emits are stamped in.
2567
+ await this.dispatch(rt, rt.invocationId);
2568
+ await this.dispatchExecutors(rt, rt.invocationId);
2497
2569
  }
2498
2570
  return structuredClone(result);
2499
2571
  }
@@ -2594,7 +2666,15 @@ export class SqliteScopeHost {
2594
2666
  * At-least-once still requires idempotent handlers. Retry is the backstop, not a
2595
2667
  * substitute.
2596
2668
  */
2597
- async dispatchExecutors(rt) {
2669
+ async dispatchExecutors(rt,
2670
+ /**
2671
+ * #1525: the call this drain pass is running in, or null — PASSED, not read off
2672
+ * `rt.invocationId`, for the reason `recordDenial` states. The Cloudflare twin
2673
+ * has no ambient field to read here at all: its executors run on the coordinator
2674
+ * and journal over RPC, so the id has to travel as an argument on that side, and
2675
+ * a recorded fact whose correctness argument differs per adapter drifts.
2676
+ */
2677
+ invocationId) {
2598
2678
  const report = {
2599
2679
  attempted: 0,
2600
2680
  delivered: 0,
@@ -2628,11 +2708,11 @@ export class SqliteScopeHost {
2628
2708
  else {
2629
2709
  await executor.handler(this.admin, event);
2630
2710
  }
2631
- this.recordExecutorDelivery(rt, row.id, deliveryId, null, executor.retry);
2711
+ this.recordExecutorDelivery(rt, row.id, deliveryId, null, executor.retry, invocationId);
2632
2712
  report.delivered += 1;
2633
2713
  }
2634
2714
  catch (err) {
2635
- const dead = this.recordExecutorDelivery(rt, row.id, deliveryId, err instanceof Error ? (err.stack ?? err.message) : String(err), executor.retry);
2715
+ const dead = this.recordExecutorDelivery(rt, row.id, deliveryId, err instanceof Error ? (err.stack ?? err.message) : String(err), executor.retry, invocationId);
2636
2716
  if (dead)
2637
2717
  report.deadLettered += 1;
2638
2718
  else
@@ -2653,7 +2733,9 @@ export class SqliteScopeHost {
2653
2733
  * silently marking success. Claiming first would make delivery at-most-once and
2654
2734
  * lose an effect on any crash in between.
2655
2735
  */
2656
- recordExecutorDelivery(rt, eventId, deliveryId, error, retry) {
2736
+ recordExecutorDelivery(rt, eventId, deliveryId, error, retry,
2737
+ /** #1525: the call THIS attempt ran in, or null. Last, mirroring the DO's RPC. */
2738
+ invocationId) {
2657
2739
  const prior = rt.db
2658
2740
  .prepare('SELECT attempts FROM _substrat_deliveries WHERE event_id = ? AND consumer_module = ?')
2659
2741
  .get(eventId, deliveryId)?.attempts ?? 0;
@@ -2663,20 +2745,160 @@ export class SqliteScopeHost {
2663
2745
  const nextAttemptAt = error === null || exhausted ? null : backoffAt(attempts, retry, new Date());
2664
2746
  rt.db
2665
2747
  .prepare(`INSERT INTO _substrat_deliveries
2666
- (event_id, consumer_module, delivered_at, error, attempts, next_attempt_at)
2667
- VALUES (?, ?, ?, ?, ?, ?)
2748
+ (event_id, consumer_module, delivered_at, error, attempts, next_attempt_at,
2749
+ invocation_id)
2750
+ VALUES (?, ?, ?, ?, ?, ?, ?)
2668
2751
  ON CONFLICT (event_id, consumer_module) DO UPDATE SET
2669
2752
  delivered_at = excluded.delivered_at,
2670
2753
  error = excluded.error,
2671
2754
  attempts = excluded.attempts,
2672
- next_attempt_at = excluded.next_attempt_at`)
2673
- .run(eventId, deliveryId, new Date().toISOString(), error, attempts, nextAttemptAt);
2755
+ next_attempt_at = excluded.next_attempt_at,
2756
+ -- #1525: overwritten, like the four above. The row describes the LATEST
2757
+ -- attempt, so a retry drained by a different call (or by none) must not
2758
+ -- keep claiming the call that made the first one.
2759
+ invocation_id = excluded.invocation_id`)
2760
+ .run(eventId, deliveryId, new Date().toISOString(), error, attempts, nextAttemptAt,
2761
+ // #1525: the call this attempt ran in, as the caller named it.
2762
+ invocationId);
2674
2763
  return error !== null && exhausted;
2675
2764
  }
2676
2765
  async drainDue(tenantId, scopeId) {
2677
2766
  const rt = this.runtime(tenantId, scopeId);
2678
2767
  await this.applyPendingMigrations(rt);
2679
- return rt.actor.enqueue(() => this.dispatchExecutors(rt));
2768
+ // #1525: null, and honestly so — a sweep is not a call. An attempt this pass makes
2769
+ // records no invocation, which is what distinguishes it from the first attempt the
2770
+ // emitting operation's own tail made.
2771
+ return rt.actor.enqueue(() => this.dispatchExecutors(rt, null));
2772
+ }
2773
+ registerJob(moduleId, name, handler, retry) {
2774
+ const key = `${moduleId}/${name}`;
2775
+ if (this.jobs.has(key))
2776
+ throw new Error(`job '${key}' is already registered`);
2777
+ this.jobs.set(key, { handler, retry });
2778
+ }
2779
+ /**
2780
+ * D-14's in-process driver: the run store is direct SQL on the scope db.
2781
+ *
2782
+ * **The PASS stays outside the actor; every STORE OPERATION takes a turn on it.**
2783
+ * That split is the whole of the concurrency design here, and the first cut got
2784
+ * the second half wrong — it ran these statements on `rt.db` with no turn at all,
2785
+ * under a comment claiming they "interleave exactly as any other write does".
2786
+ * They do not. `invoke` opens a raw `BEGIN IMMEDIATE` on this same connection and
2787
+ * HOLDS IT ACROSS AWAITS (its guards, its handler), so a job-store statement
2788
+ * issued while some unrelated request was mid-operation landed INSIDE that
2789
+ * request's transaction — and rolled back with it. A step whose effect had
2790
+ * already happened would lose its ledger row and run a second time, for no reason
2791
+ * connected to the job at all.
2792
+ *
2793
+ * A turn each is the fix, and it keeps the property the original comment wanted:
2794
+ * a pass may take an hour, and holding the scope's lock for it would stop every
2795
+ * request on that scope, so the pass itself is never enqueued. Only the short
2796
+ * writes are. Nothing nests — a store operation never invokes, and the `invoke`s
2797
+ * a pass makes through `pass.scope()` take their own separate turns — so there is
2798
+ * no path by which this can deadlock against itself.
2799
+ */
2800
+ jobStore(rt) {
2801
+ const db = rt.db;
2802
+ const row = (v) => v ?? null;
2803
+ const findLive = db.prepare(`SELECT * FROM _substrat_job_runs
2804
+ WHERE module_id = ? AND job = ? AND instance = ? AND status = 'running'
2805
+ ORDER BY id DESC LIMIT 1`);
2806
+ const insertRun = db.prepare(`INSERT INTO _substrat_job_runs
2807
+ (id, module_id, job, instance, payload, status, cursor, counters, attempts,
2808
+ last_error, started_at, updated_at, next_attempt_at, ended_at)
2809
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
2810
+ const patchRun = db.prepare(`UPDATE _substrat_job_runs
2811
+ SET status = ?, cursor = ?, counters = ?, attempts = ?, last_error = ?,
2812
+ updated_at = ?, next_attempt_at = ?, ended_at = ?
2813
+ WHERE id = ?`);
2814
+ const patchArgs = (id, p) => [p.status, p.cursor, p.counters, p.attempts, p.lastError, p.updatedAt, p.nextAttemptAt, p.endedAt, id];
2815
+ // `db.transaction` on better-sqlite3 runs its body SYNCHRONOUSLY inside a real
2816
+ // SQLite transaction, so nothing — not another turn of the event loop, not
2817
+ // another caller of this host — can interleave between the read and the write.
2818
+ // That is what makes the two operations below atomic rather than merely ordered.
2819
+ const startOrJoinTx = db.transaction((key, r) => {
2820
+ const live = row(findLive.get(key.moduleId, key.job, key.instance));
2821
+ if (live)
2822
+ return live;
2823
+ insertRun.run(r.id, r.module_id, r.job, r.instance, r.payload, r.status, r.cursor, r.counters, r.attempts, r.last_error, r.started_at, r.updated_at, r.next_attempt_at, r.ended_at);
2824
+ return r;
2825
+ });
2826
+ const commitPassTx = db.transaction((id, p) => {
2827
+ patchRun.run(...patchArgs(id, p));
2828
+ db.prepare('DELETE FROM _substrat_job_steps WHERE run_id = ?').run(id);
2829
+ });
2830
+ // Every store operation takes its own short turn on the scope actor, so no
2831
+ // statement here can land inside an `invoke`'s open transaction. `turn` is the
2832
+ // one place that happens; adding a method without it reintroduces the bug.
2833
+ const turn = (fn) => rt.actor.enqueue(fn);
2834
+ return {
2835
+ startOrJoin: (key, r) => turn(() => startOrJoinTx(key, r)),
2836
+ get: (id) => turn(() => row(db.prepare('SELECT * FROM _substrat_job_runs WHERE id = ?').get(id))),
2837
+ due: (now, limit, afterId) => turn(() => db
2838
+ .prepare(`SELECT * FROM _substrat_job_runs
2839
+ WHERE status = 'running' AND (next_attempt_at IS NULL OR next_attempt_at <= ?)
2840
+ AND (? IS NULL OR id > ?)
2841
+ ORDER BY id LIMIT ?`)
2842
+ .all(now, afterId ?? null, afterId ?? null, limit)),
2843
+ list: (filter) => turn(() => {
2844
+ const where = [];
2845
+ const params = [];
2846
+ for (const [column, value] of [
2847
+ ['module_id', filter.moduleId],
2848
+ ['job', filter.job],
2849
+ ['instance', filter.instance],
2850
+ ['status', filter.status],
2851
+ ]) {
2852
+ if (value !== undefined) {
2853
+ where.push(`${column} = ?`);
2854
+ params.push(value);
2855
+ }
2856
+ }
2857
+ params.push(jobRunListLimit(filter.limit));
2858
+ return db
2859
+ .prepare(`SELECT * FROM _substrat_job_runs
2860
+ ${where.length ? `WHERE ${where.join(' AND ')}` : ''}
2861
+ ORDER BY id DESC LIMIT ?`)
2862
+ .all(...params);
2863
+ }),
2864
+ patch: (id, p) => turn(() => {
2865
+ patchRun.run(...patchArgs(id, p));
2866
+ }),
2867
+ commitPass: (id, p) => turn(() => commitPassTx(id, p)),
2868
+ step: (runId, name) => turn(() => db
2869
+ .prepare('SELECT step, result, attempts, last_error FROM _substrat_job_steps WHERE run_id = ? AND step = ?')
2870
+ .get(runId, name) ?? null),
2871
+ recordStep: (runId, name, result, attempts, lastError, at) => turn(() => {
2872
+ db.prepare(`INSERT INTO _substrat_job_steps (run_id, step, result, attempts, last_error, recorded_at)
2873
+ VALUES (?, ?, ?, ?, ?, ?)
2874
+ ON CONFLICT (run_id, step) DO UPDATE SET result = excluded.result,
2875
+ attempts = excluded.attempts,
2876
+ last_error = excluded.last_error,
2877
+ recorded_at = excluded.recorded_at`).run(runId, name, result, attempts, lastError, at);
2878
+ }),
2879
+ };
2880
+ }
2881
+ async startJobRun(tenantId, scopeId, input) {
2882
+ const rt = this.runtime(tenantId, scopeId);
2883
+ await this.applyPendingMigrations(rt);
2884
+ return jobRunOf(await startJobRun(this.jobStore(rt), input, ulid, this.clock));
2885
+ }
2886
+ async runDueJobs(tenantId, scopeId, options) {
2887
+ const rt = this.runtime(tenantId, scopeId);
2888
+ await this.applyPendingMigrations(rt);
2889
+ return runDueJobRuns({
2890
+ store: this.jobStore(rt),
2891
+ handlerFor: (run) => this.jobs.get(`${run.module_id}/${run.job}`),
2892
+ now: this.clock,
2893
+ openScope: (run) => this.getSystemScope(run.module_id, tenantId, scopeId),
2894
+ maxPasses: options?.maxPasses,
2895
+ limit: options?.limit,
2896
+ });
2897
+ }
2898
+ async jobRuns(tenantId, scopeId, filter) {
2899
+ const rt = this.runtime(tenantId, scopeId);
2900
+ await this.applyPendingMigrations(rt);
2901
+ return (await this.jobStore(rt).list(filter ?? {})).map(jobRunOf);
2680
2902
  }
2681
2903
  async dispatchConnector(tenantId, scopeId, handler, event, options) {
2682
2904
  // The platform half of a routed `connector:<provider>` intent (#574 phase 3): run
@@ -2839,7 +3061,13 @@ export class SqliteScopeHost {
2839
3061
  .run(outcome.status, outcome.result === undefined ? null : JSON.stringify(outcome.result), outcome.lastError ?? null, outcome.failure == null ? null : JSON.stringify(outcome.failure), outcome.status === 'pending' ? null : new Date().toISOString(), id);
2840
3062
  });
2841
3063
  }
2842
- async dispatch(rt) {
3064
+ async dispatch(rt,
3065
+ /**
3066
+ * #1525: the call this drain is running in, or null — PASSED, for the reason
3067
+ * `dispatchExecutors` states. Every round of the loop belongs to it: a consumer's
3068
+ * own emit is delivered in the same tail, so the whole cascade is one call's work.
3069
+ */
3070
+ invocationId) {
2843
3071
  for (let round = 0; round < 50; round++) {
2844
3072
  let deliveredAny = false;
2845
3073
  for (const mod of this.modules.values()) {
@@ -2867,9 +3095,10 @@ export class SqliteScopeHost {
2867
3095
  try {
2868
3096
  await consumer.handler(ctx, event);
2869
3097
  rt.db
2870
- .prepare(`INSERT INTO _substrat_deliveries (event_id, consumer_module, delivered_at)
2871
- VALUES (?, ?, ?)`)
2872
- .run(event.id, mod.id, new Date().toISOString());
3098
+ .prepare(`INSERT INTO _substrat_deliveries
3099
+ (event_id, consumer_module, delivered_at, invocation_id)
3100
+ VALUES (?, ?, ?, ?)`)
3101
+ .run(event.id, mod.id, new Date().toISOString(), invocationId);
2873
3102
  rt.db.exec('COMMIT');
2874
3103
  deliveredAny = true;
2875
3104
  }
@@ -2878,9 +3107,10 @@ export class SqliteScopeHost {
2878
3107
  // Dead-letter (v0): journal the failure so one poison event
2879
3108
  // can't wedge the loop. Real redelivery/backoff is a later cut.
2880
3109
  rt.db
2881
- .prepare(`INSERT INTO _substrat_deliveries (event_id, consumer_module, delivered_at, error)
2882
- VALUES (?, ?, ?, ?)`)
2883
- .run(event.id, mod.id, new Date().toISOString(), String(err));
3110
+ .prepare(`INSERT INTO _substrat_deliveries
3111
+ (event_id, consumer_module, delivered_at, error, invocation_id)
3112
+ VALUES (?, ?, ?, ?, ?)`)
3113
+ .run(event.id, mod.id, new Date().toISOString(), String(err), invocationId);
2884
3114
  }
2885
3115
  finally {
2886
3116
  // Cleared on BOTH paths. Left set, the id would leak onto every later
@@ -2902,6 +3132,19 @@ export class SqliteScopeHost {
2902
3132
  * point, since the denial is exactly the write the rolled-back operation could not make.
2903
3133
  */
2904
3134
  recordDenial(rt, subject, operation, err,
3135
+ /**
3136
+ * #1525: the invocation this refusal belongs to, or null — PASSED, not read off
3137
+ * `rt.invocationId` here.
3138
+ *
3139
+ * Every denial path on THIS adapter runs inside the scope's actor task, so reading
3140
+ * `rt.invocationId` would in fact be right at each one. It is passed anyway, to keep
3141
+ * one discipline across both adapters: the Cloudflare side has two denial paths that
3142
+ * run outside its queue, where the ambient read rests on an argument about workerd's
3143
+ * input gate rather than on anything local. A recorded fact whose correctness
3144
+ * argument differs per adapter is the kind that drifts, so on both sides the caller
3145
+ * says what it knows.
3146
+ */
3147
+ invocationId,
2905
3148
  /** K-42: the session the refused call ran under, when it ran under one. */
2906
3149
  impersonation) {
2907
3150
  // Only an ENFORCED denial (assertAllowed, which attaches the checked permission +
@@ -2916,9 +3159,12 @@ export class SqliteScopeHost {
2916
3159
  : subject.id;
2917
3160
  rt.db
2918
3161
  .prepare(`INSERT INTO _substrat_denials
2919
- (id, actor, permission, tenant_id, scope_id, operation, impersonation, at)
2920
- VALUES (?, ?, ?, ?, ?, ?, ?, ?)`)
2921
- .run(ulid(), JSON.stringify(actor), err.permission, err.node.tenantId, err.node.scopeId ?? null, operation, impersonation ? JSON.stringify(impersonationStampOf(impersonation)) : null, new Date().toISOString());
3162
+ (id, actor, permission, tenant_id, scope_id, operation, impersonation,
3163
+ invocation_id, at)
3164
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`)
3165
+ .run(ulid(), JSON.stringify(actor), err.permission, err.node.tenantId, err.node.scopeId ?? null, operation, impersonation ? JSON.stringify(impersonationStampOf(impersonation)) : null,
3166
+ // #1525: the call this refusal belongs to, as the caller named it.
3167
+ invocationId, new Date().toISOString());
2922
3168
  }
2923
3169
  parseOutboxRow(row) {
2924
3170
  return domainEvent.parse({
@@ -3787,7 +4033,7 @@ export class SqliteScopeHost {
3787
4033
  const parsed = publishVersionInput.parse(input);
3788
4034
  const owning = readVertical(parsed.verticalSlug);
3789
4035
  if (!owning) {
3790
- throw new Error(`unknown vertical '${parsed.verticalSlug}'`);
4036
+ throw substratError('not_found', `unknown vertical '${parsed.verticalSlug}'`);
3791
4037
  }
3792
4038
  // Lands PENDING — a push is not a deploy — except for a PRIVATE vertical
3793
4039
  // (tenant-owned, not listed), whose blast radius is its own tenant: there the
@@ -3830,7 +4076,7 @@ export class SqliteScopeHost {
3830
4076
  setVerticalListed: async (actor, slug, listed) => {
3831
4077
  const existing = readVertical(slug);
3832
4078
  if (!existing)
3833
- throw new Error(`unknown vertical '${slug}'`);
4079
+ throw substratError('not_found', `unknown vertical '${slug}'`);
3834
4080
  // Listing is the moment other tenants start trusting this code, so the
3835
4081
  // version they would install must carry a real staff vouch — an auto-admitted
3836
4082
  // prod version has never been read by anyone but its author.
@@ -3853,14 +4099,14 @@ export class SqliteScopeHost {
3853
4099
  requestPublish: async (actor, slug) => {
3854
4100
  const existing = readVertical(slug);
3855
4101
  if (!existing)
3856
- throw new Error(`unknown vertical '${slug}'`);
4102
+ throw substratError('not_found', `unknown vertical '${slug}'`);
3857
4103
  this.directory.prepare('UPDATE verticals SET publish_requested_at = ? WHERE slug = ?').run(new Date().toISOString(), slug);
3858
4104
  this.recordAdmin(actor, 'requestPublish', { tenantId: null }, null, { slug });
3859
4105
  },
3860
4106
  setVerticalInstallsBlocked: async (actor, slug, blocked) => {
3861
4107
  const existing = readVertical(slug);
3862
4108
  if (!existing)
3863
- throw new Error(`unknown vertical '${slug}'`);
4109
+ throw substratError('not_found', `unknown vertical '${slug}'`);
3864
4110
  this.directory
3865
4111
  .prepare('UPDATE verticals SET installs_blocked = ? WHERE slug = ?')
3866
4112
  .run(blocked ? 1 : 0, slug);
@@ -3869,7 +4115,7 @@ export class SqliteScopeHost {
3869
4115
  setVerticalTenantProvisioner: async (actor, slug, granted) => {
3870
4116
  const existing = readVertical(slug);
3871
4117
  if (!existing)
3872
- throw new Error(`unknown vertical '${slug}'`);
4118
+ throw substratError('not_found', `unknown vertical '${slug}'`);
3873
4119
  this.directory
3874
4120
  .prepare('UPDATE verticals SET tenant_provisioner = ? WHERE slug = ?')
3875
4121
  .run(granted ? 1 : 0, slug);
@@ -3878,7 +4124,7 @@ export class SqliteScopeHost {
3878
4124
  setVerticalEmailSender: async (actor, slug, granted) => {
3879
4125
  const existing = readVertical(slug);
3880
4126
  if (!existing)
3881
- throw new Error(`unknown vertical '${slug}'`);
4127
+ throw substratError('not_found', `unknown vertical '${slug}'`);
3882
4128
  this.directory
3883
4129
  .prepare('UPDATE verticals SET email_sender = ? WHERE slug = ?')
3884
4130
  .run(granted ? 1 : 0, slug);
@@ -3887,7 +4133,7 @@ export class SqliteScopeHost {
3887
4133
  deleteVertical: async (actor, slug) => {
3888
4134
  const existing = readVertical(slug);
3889
4135
  if (!existing)
3890
- throw new Error(`unknown vertical '${slug}'`);
4136
+ throw substratError('not_found', `unknown vertical '${slug}'`);
3891
4137
  // Refuse while any restorable scope is bound: a deleted registry row would strand
3892
4138
  // those scopes' version pins and routing. An `archived` scope (a deleted app) still
3893
4139
  // blocks — unarchive can bring it back — but the refusal names reap/restore, not
@@ -3900,10 +4146,10 @@ export class SqliteScopeHost {
3900
4146
  'FROM scopes WHERE vertical = ?')
3901
4147
  .get(slug);
3902
4148
  if (bound.live > 0) {
3903
- throw new Error(`vertical '${slug}' still backs ${bound.live} scope(s) — delete or rebind them first`);
4149
+ throw substratError('conflict', `vertical '${slug}' still backs ${bound.live} scope(s) — delete or rebind them first`);
3904
4150
  }
3905
4151
  if (bound.archived > 0) {
3906
- throw new Error(`vertical '${slug}' still backs ${bound.archived} archived scope(s) — reap or restore them first`);
4152
+ throw substratError('conflict', `vertical '${slug}' still backs ${bound.archived} archived scope(s) — reap or restore them first`);
3907
4153
  }
3908
4154
  // Deployed dispatch scripts are NOT reaped here — they become orphans for the
3909
4155
  // cleanup script (#248), never destroyed alongside a registry row.
@@ -3916,7 +4162,7 @@ export class SqliteScopeHost {
3916
4162
  admitVersion: async (actor, versionId) => {
3917
4163
  const v = readVersion(versionId);
3918
4164
  if (!v)
3919
- throw new Error(`unknown version ${versionId}`);
4165
+ throw substratError('not_found', `unknown version ${versionId}`);
3920
4166
  if (v.admission === 'admitted') {
3921
4167
  // Idempotent — except an AUTO-admitted version, which this upgrades to a
3922
4168
  // manual vouch by clearing the note (what the publish seam requires).
@@ -3932,7 +4178,7 @@ export class SqliteScopeHost {
3932
4178
  return;
3933
4179
  }
3934
4180
  if (v.admission === 'rejected') {
3935
- throw new Error(`version ${versionId} was rejected — publish a new one`);
4181
+ throw substratError('conflict', `version ${versionId} was rejected — publish a new one`);
3936
4182
  }
3937
4183
  this.directory
3938
4184
  .prepare("UPDATE vertical_versions SET admission = 'admitted' WHERE id = ?")
@@ -3944,7 +4190,7 @@ export class SqliteScopeHost {
3944
4190
  rejectVersion: async (actor, versionId, note) => {
3945
4191
  const v = readVersion(versionId);
3946
4192
  if (!v)
3947
- throw new Error(`unknown version ${versionId}`);
4193
+ throw substratError('not_found', `unknown version ${versionId}`);
3948
4194
  if (v.admission === 'admitted') {
3949
4195
  throw new Error(`version ${versionId} is already admitted — it may be bound`);
3950
4196
  }
@@ -3961,7 +4207,7 @@ export class SqliteScopeHost {
3961
4207
  promoteVersion: async (actor, verticalSlug, channel, versionId, acknowledge) => {
3962
4208
  const incoming = readVersion(versionId);
3963
4209
  if (!incoming)
3964
- throw new Error(`unknown version ${versionId}`);
4210
+ throw substratError('not_found', `unknown version ${versionId}`);
3965
4211
  if (incoming.verticalSlug !== verticalSlug) {
3966
4212
  throw new Error(`version ${versionId} belongs to '${incoming.verticalSlug}'`);
3967
4213
  }
@@ -4091,7 +4337,7 @@ export class SqliteScopeHost {
4091
4337
  bindScopeVersion: async (actor, tenantId, scopeId, versionId, opts) => {
4092
4338
  const v = readVersion(versionId);
4093
4339
  if (!v)
4094
- throw new Error(`unknown version ${versionId}`);
4340
+ throw substratError('not_found', `unknown version ${versionId}`);
4095
4341
  const scope = this.directory
4096
4342
  .prepare('SELECT tenant_id, kind, vertical_version_id FROM scopes WHERE scope_id = ?')
4097
4343
  .get(scopeId);
@@ -4147,7 +4393,7 @@ export class SqliteScopeHost {
4147
4393
  .prepare('SELECT * FROM verticals WHERE slug = ?')
4148
4394
  .get(verticalSlug);
4149
4395
  if (!r)
4150
- throw new Error(`unknown vertical '${verticalSlug}'`);
4396
+ throw substratError('not_found', `unknown vertical '${verticalSlug}'`);
4151
4397
  this.recordAccess(actor, 'verticalServing', {}, { verticalSlug }, r.serving_ref ? 1 : 0);
4152
4398
  if (!r.serving_ref || !r.serving_version_id || !r.serving_migration_tag)
4153
4399
  return null;
@@ -4164,7 +4410,7 @@ export class SqliteScopeHost {
4164
4410
  .prepare('SELECT * FROM verticals WHERE slug = ?')
4165
4411
  .get(verticalSlug);
4166
4412
  if (!r)
4167
- throw new Error(`unknown vertical '${verticalSlug}'`);
4413
+ throw substratError('not_found', `unknown vertical '${verticalSlug}'`);
4168
4414
  this.directory
4169
4415
  .prepare(`UPDATE verticals SET serving_ref = ?, serving_version_id = ?,
4170
4416
  serving_do_classes = ?, serving_migration_tag = ? WHERE slug = ?`)
@@ -4176,7 +4422,7 @@ export class SqliteScopeHost {
4176
4422
  .prepare('SELECT * FROM vertical_versions WHERE id = ?')
4177
4423
  .get(versionId);
4178
4424
  if (!v || v.vertical_slug !== verticalSlug) {
4179
- throw new Error(`unknown version ${versionId} for vertical '${verticalSlug}'`);
4425
+ throw substratError('not_found', `unknown version ${versionId} for vertical '${verticalSlug}'`);
4180
4426
  }
4181
4427
  this.recordAccess(actor, 'versionManifest', {}, { verticalSlug, versionId }, v.manifest_json ? 1 : 0);
4182
4428
  return v.manifest_json;
@@ -4265,12 +4511,34 @@ export class SqliteScopeHost {
4265
4511
  // Directory check first, on mark's reasoning: "nothing to reopen" and "you may not
4266
4512
  // address this scope" are different answers.
4267
4513
  this.assertScopeReachable(tenantId, scopeId);
4268
- const { drainedBefore } = redrainEventsInput.parse(input);
4514
+ const { drainedBefore, countOnly } = redrainEventsInput.parse(input);
4269
4515
  // The window rule, enforced HERE and not only at the control-plane door: this verb
4270
4516
  // is public `HostAdmin`, so the fleet script reaches it without passing that route.
4271
- // The injected clock, so a test pins the boundary rather than racing it.
4517
+ // The injected clock, so a test pins the boundary rather than racing it. A count is
4518
+ // held to it too — the two answers come from one verb and one door.
4272
4519
  assertRedrainWindow(drainedBefore, this.clock());
4273
4520
  const db = this.scopeDbFor(tenantId, scopeId);
4521
+ // A COUNT reopens nothing, writes nothing, and records nothing (#1545). The two
4522
+ // receipts below are there because a reopen is a second egress of a tenant's
4523
+ // payloads; a count egresses none, and an admin row claiming a redrain on a scope
4524
+ // that was only counted is a false statement in the one log that is evidence.
4525
+ //
4526
+ // UNBOUNDED, unlike the reopen under it: the batch exists because an UPDATE over
4527
+ // the whole window must fit one Durable-Object request on the other adapter, and an
4528
+ // aggregate materialises no rows. A capped count would answer "5000, or more",
4529
+ // which is not an answer.
4530
+ //
4531
+ // It IS a read, and K-24 takes all reads rather than a chosen subset, so it leaves
4532
+ // an access row — the count itself as the result count, which is the fact an
4533
+ // auditor asking "who counted this tenant's outbox, and how much was there" wants.
4534
+ if (countOnly) {
4535
+ const redrainable = db
4536
+ .prepare(`SELECT COUNT(*) AS c FROM _substrat_outbox
4537
+ WHERE drained_at IS NOT NULL AND drained_at < ?`)
4538
+ .get(drainedBefore).c;
4539
+ this.recordAccess(actor, 'redrainEvents', { tenantId, scopeId }, { drainedBefore, countOnly: true }, redrainable);
4540
+ return redrainable;
4541
+ }
4274
4542
  // Audit FIRST, matching the Cloudflare adapter and `rewindScope` (K-33): the row is
4275
4543
  // a separate statement outside the update's transaction, so a failure between them
4276
4544
  // would leave a reopen with no receipt — and the retry cannot repair it, because the
@@ -5121,6 +5389,40 @@ export class SqliteScopeHost {
5121
5389
  this.recordAccess(actor, 'listIdentityTenants', {}, { provider }, tenants.length);
5122
5390
  return tenants;
5123
5391
  },
5392
+ listIdentityMemberships: async (actor, provider, externalId) => {
5393
+ const pool = readPool(provider);
5394
+ if (!pool)
5395
+ throw new Error(`identity pool '${provider}' is not registered`);
5396
+ if (pool.topology !== 'central') {
5397
+ throw new Error(`identity pool '${provider}' is tenant-bound — enumerating tenants is only ` +
5398
+ `meaningful on a central pool, where the same externalId is the same person`);
5399
+ }
5400
+ const rows = this.directory
5401
+ .prepare(`SELECT i.tenant_id, i.principal_id, s.scope_id, s.vertical, s.status AS scope_status
5402
+ FROM _substrat_identities i
5403
+ LEFT JOIN scopes s ON s.scope_id = i.scope_id AND s.tenant_id = i.tenant_id
5404
+ WHERE i.provider = ? AND i.external_id = ?
5405
+ ORDER BY i.tenant_id`)
5406
+ .all(provider, externalId);
5407
+ const memberships = rows.flatMap((r) => {
5408
+ // A link whose tenant row is gone names nothing a caller could land in.
5409
+ const tenant = readTenant(r.tenant_id);
5410
+ if (!tenant)
5411
+ return [];
5412
+ return [
5413
+ identityMembership.parse({
5414
+ tenant,
5415
+ principal: r.principal_id,
5416
+ scope: r.scope_id === null
5417
+ ? null
5418
+ : { id: r.scope_id, vertical: r.vertical, status: r.scope_status ?? 'active' },
5419
+ }),
5420
+ ];
5421
+ });
5422
+ // One row for the whole answer — the read this replaces logged one per tenant.
5423
+ this.recordAccess(actor, 'listIdentityMemberships', {}, { provider }, memberships.length);
5424
+ return memberships;
5425
+ },
5124
5426
  listIdentityLinks: async (actor, tid) => {
5125
5427
  // The projection read (#406): what the platform gathers to deliver a tenant's
5126
5428
  // links with provisioning/reconcile — the same trust line entitlements ride
@@ -5826,6 +6128,26 @@ export class SqliteScopeHost {
5826
6128
  if (!existing)
5827
6129
  db.exec(`ALTER TABLE ${table} ADD COLUMN ${ddl}`);
5828
6130
  }
6131
+ /**
6132
+ * #1288: `_substrat_schedule_state`, rebuilt with `kind` in its key on a scope db
6133
+ * whose table predates the column. The statements are the kernel's, so the DO
6134
+ * adapter rebuilds byte-identically, and the detection is the `sqlite_master.sql`
6135
+ * read `ensureIdentityKey` already uses — PRAGMA would do here, but not there.
6136
+ */
6137
+ ensureScheduleStateKind(db) {
6138
+ const row = db
6139
+ .prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?")
6140
+ .get('_substrat_schedule_state');
6141
+ if (!row || scheduleStateHasKind(row.sql))
6142
+ return;
6143
+ // In a transaction, for the reason the kernel constant spells out: create-copy-
6144
+ // drop-rename has two intermediate states and BOTH are unrecoverable on the next
6145
+ // wake — one dies on the leftover scratch table, one silently orphans the rows
6146
+ // behind an empty table `CREATE TABLE IF NOT EXISTS` put back. `db.transaction`
6147
+ // nests as a SAVEPOINT, which is what makes this safe on the `loadDump` path too,
6148
+ // where the whole replay is already inside one.
6149
+ db.transaction(() => db.exec(SCHEDULE_STATE_REBUILD))();
6150
+ }
5829
6151
  /**
5830
6152
  * Rebuild `_substrat_identities` when it still carries the pre-K-22 global key.
5831
6153
  * A PRIMARY KEY cannot be ALTERed, so this is create-copy-drop-rename.
@@ -6516,6 +6838,11 @@ export class SqliteScopeHost {
6516
6838
  // consumer dead-letter.
6517
6839
  this.ensureColumn(db, '_substrat_deliveries', 'attempts', 'attempts INTEGER NOT NULL DEFAULT 0');
6518
6840
  this.ensureColumn(db, '_substrat_deliveries', 'next_attempt_at', 'next_attempt_at TEXT');
6841
+ // #1525: the invocation an attempt ran in, on a scope DB created before the column.
6842
+ // Nullable, and the null is honestly "no call was carried" — which attempt produced
6843
+ // a delivery already journalled cannot be decided afterwards, exactly as #1237's
6844
+ // outbox column argued.
6845
+ this.ensureColumn(db, '_substrat_deliveries', 'invocation_id', 'invocation_id TEXT');
6519
6846
  // K-34: the authorization column on a scope DB created before it existed. Nullable,
6520
6847
  // so legacy outbox rows read as "unrecorded" — the honest value. (_substrat_denials
6521
6848
  // is a whole new table, so KERNEL_DDL's IF NOT EXISTS covers it with no ALTER.)
@@ -6531,6 +6858,10 @@ export class SqliteScopeHost {
6531
6858
  this.ensureColumn(db, '_substrat_outbox', 'impersonation', 'impersonation TEXT');
6532
6859
  this.ensureColumn(db, '_substrat_platform_requests', 'impersonation', 'impersonation TEXT');
6533
6860
  this.ensureColumn(db, '_substrat_denials', 'impersonation', 'impersonation TEXT');
6861
+ // #1525: the invocation a refusal happened during, on a scope DB created before the
6862
+ // column. Nullable, and the null is honestly "no id was carried" — a past denial's
6863
+ // call cannot be decided afterwards, exactly as #1237's outbox column argued.
6864
+ this.ensureColumn(db, '_substrat_denials', 'invocation_id', 'invocation_id TEXT');
6534
6865
  // #1231: the emitting operation, on a scope DB created before the column. Nullable
6535
6866
  // so every legacy row reads as unrecorded rather than claiming a name nobody stamped.
6536
6867
  this.ensureColumn(db, '_substrat_outbox', 'operation', 'operation TEXT');
@@ -6544,6 +6875,9 @@ export class SqliteScopeHost {
6544
6875
  // #1237: existing scopes get it on next wake. Rows already written keep NULL, which
6545
6876
  // honestly means "no invocation id was carried" — nothing can decide one afterwards.
6546
6877
  this.ensureColumn(db, '_substrat_outbox', 'invocation_id', 'invocation_id TEXT');
6878
+ // #1288: not an ensureColumn, because `kind` joins the schedule-state PRIMARY KEY
6879
+ // and no ALTER can widen a key. Rebuilt instead, from the kernel's statements.
6880
+ this.ensureScheduleStateKind(db);
6547
6881
  // #1237: `readInvocation`'s lookup — WHERE invocation_id = ? ORDER BY id — over an outbox
6548
6882
  // that is never pruned. No index leads with invocation_id, so without this one SQLite
6549
6883
  // walks the PRIMARY KEY from the oldest event until it reaches the call, and reading a