@substrat-run/adapter-cloudflare 0.113.0 → 0.116.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/scope-do.js CHANGED
@@ -1,9 +1,10 @@
1
1
  import { DurableObject } from 'cloudflare:workers';
2
- import { ATTACHMENT_ADDED, ATTACHMENT_REMOVED, attachmentRecord, domainEvent, domainEventInput, eventId, instant, objectRef, toWireFailure, grantRefFromProof, principalId, platformRequestInput, platformRequestId, MAX_PENDING_PLATFORM_REQUESTS, MAX_PENDING_SWEEP_RUNS, SWEEP_RUNS_KIND, platformRequest, SCOPE_TABLE_PAGE_MAX, SCOPE_QUERY_ROW_MAX, listLimitOf, requestFingerprint, substratError, assertReplayableDump, REDRAIN_BATCH, } from '@substrat-run/contracts';
3
- import { ulid, createUlid, assertAllowed, ConnectionSealingKeyUnavailableError, noSealingKeyMessage, sealTo, assertReadOnlyQuery, entitlementDenial, platformRequestHistoryQuery, PLATFORM_REQUEST_COLUMNS, denialListQuery, denialSummaryQuery, denialTotalsQuery, DENIAL_WINDOW_QUERY, mapDenialRow, mapDenialSummaryBuckets, PermissionDenied, assertImpersonationWrites, assertModuleEnqueueableKind, impersonationStampOf, createAtomic, NotSearchable, isSearchIndexTable, searchIndexDdl, searchIndexMigrations, searchIndexPlans, NotListable, listIndexMigrations, listIndexPlans, listQuery, cursorOf, searchLimit, searchMatchExpression, searchQuery, IDEMPOTENCY_DDL, assertIdempotencyKey, idempotencyLookupQuery, idempotencyPruneStatement, idempotencyRecordStatement, idempotencyOptedOutMessage, replayFor, entityVersionQuery, entityVersionOf, assertIfMatch, OUTBOX_ENTITY_INDEX, } from '@substrat-run/kernel';
2
+ import { ATTACHMENT_ADDED, ATTACHMENT_REMOVED, attachmentRecord, domainEvent, domainEventInput, eventId, instant, objectRef, toWireFailure, grantRefFromProof, principalId, scopeId as scopeIdOf, tenantId as tenantIdOf, platformRequestInput, platformRequestId, MAX_PENDING_PLATFORM_REQUESTS, MAX_PENDING_SWEEP_RUNS, SWEEP_RUNS_KIND, platformRequest, SCOPE_TABLE_PAGE_MAX, SCOPE_QUERY_ROW_MAX, listLimitOf, requestFingerprint, substratError, assertReplayableDump, REDRAIN_BATCH, } from '@substrat-run/contracts';
3
+ import { ulid, createUlid, assertAllowed, ConnectionSealingKeyUnavailableError, noSealingKeyMessage, sealTo, assertReadOnlyQuery, entitlementDenial, platformRequestHistoryQuery, PLATFORM_REQUEST_COLUMNS, denialListQuery, denialSummaryQuery, denialTotalsQuery, DENIAL_WINDOW_QUERY, mapDenialRow, mapDenialSummaryBuckets, PermissionDenied, assertImpersonationWrites, assertModuleEnqueueableKind, impersonationStampOf, createAtomic, NotSearchable, isSearchIndexTable, searchIndexDdl, searchIndexMigrations, searchIndexPlans, NotListable, listIndexMigrations, listIndexPlans, listQuery, cursorOf, searchLimit, searchMatchExpression, searchQuery, IDEMPOTENCY_DDL, assertIdempotencyKey, idempotencyLookupQuery, idempotencyPruneStatement, idempotencyRecordStatement, idempotencyOptedOutMessage, replayFor, entityVersionQuery, entityVersionOf, assertIfMatch, OUTBOX_ENTITY_INDEX, SCHEDULE_STATE_DDL, SCHEDULE_STATE_REBUILD, scheduleStateHasKind, JOB_RUN_DDL, jobRunListLimit, } from '@substrat-run/kernel';
4
+ import { isUpgradeRequest, readSubscription, LIVE_FANOUT_LIMIT, LIVE_MODE_HEADER, LIVE_PRINCIPAL_HEADER, LIVE_SCOPE_HEADER, LIVE_SUBSCRIBE_PATH, LIVE_TENANT_HEADER, } from './live-reads.js';
4
5
  import { OperationQueue } from './serialization.js';
5
6
  import { doScopedSql } from './sql.js';
6
- import { facetEvents, readHistory, readInvocation, walkEventCause, walkEventEffects } from '@substrat-run/kernel';
7
+ import { facetEvents, readDeadLetters, readHistory, readInvocation, walkEventCause, walkEventEffects } from '@substrat-run/kernel';
7
8
  import { createDoTupleChecker, createLocalControlPlaneReader } from './checker.js';
8
9
  /**
9
10
  * The key marking a scope DO whose storage was destroyed (`destroyStorage`).
@@ -127,31 +128,28 @@ const KERNEL_DDL = `
127
128
  operation TEXT,
128
129
  -- K-42: WHICH of the two actors was refused is exactly what this log is for.
129
130
  impersonation TEXT,
131
+ -- #1525: the INVOCATION this refusal happened during, the same id #1237 stamps on
132
+ -- every event of a call. A denial joins to nothing but actor and time otherwise --
133
+ -- the operation column names what was attempted, never WHICH attempt -- so "what
134
+ -- else did this request do" cannot reach a refusal, which is the one thing an
135
+ -- incident asks about first. NULL = the transport carried no id (a seed, a test,
136
+ -- an internal call, an attachment RPC), or the row predates the column.
137
+ invocation_id TEXT,
130
138
  at TEXT NOT NULL,
131
139
  drained_at TEXT
132
140
  );
133
- -- #383: the platform sweep's per-scope gating state. Despite the name, this table
134
- -- holds TWO kinds of row since #1232, told apart by the shape of schedule_op:
135
- -- * a schedule operation, spelled module/verb -- last_run_at / last_status are
136
- -- when that operation last RAN here and how it ended;
137
- -- * a freshness key, spelled freshness:<eventType> -- last_run_at / last_status
138
- -- are when the evaluator last RECORDED a verdict for that event type and what
139
- -- the verdict was. Nothing ran; the row gates what the sweep records.
140
- -- Only HALF the no-collision claim is enforced, which is worth knowing before
141
- -- trusting it: a freshness key is always "freshness:" followed by a value the
142
- -- contracts eventType schema accepted (lowercase ns.verb, no colon and no slash),
143
- -- so no freshness key can ever look like an operation. The other direction is
144
- -- CONVENTION only -- scheduleSpec.operation is z.string().min(1), so a module
145
- -- that declared a schedule operation literally named "freshness:orders.placed"
146
- -- would share a row with the evaluator and nothing today would reject it.
147
- -- #1288 tracks giving the table a column (or a name) that says this outright,
148
- -- which is also what would let the collision be refused rather than avoided.
149
- -- Spine (kernel-written), never a module migration.
150
- CREATE TABLE IF NOT EXISTS _substrat_schedule_state (
151
- schedule_op TEXT PRIMARY KEY,
152
- last_run_at TEXT,
153
- last_status TEXT
154
- );
141
+ -- #383 / #1232 / #1288: the platform sweep's per-scope gating state, holding two
142
+ -- families of row that the kind COLUMN -- not the spelling of a key -- tells
143
+ -- apart. Spine (kernel-written), never a module migration. Shared with the pure
144
+ -- adapter from @substrat-run/kernel, so the shape a rebuild produces and the shape
145
+ -- a fresh store gets cannot part company; the column comments are in there.
146
+ ${SCHEDULE_STATE_DDL}
147
+ -- #1577: the resumable-run driver's record and the step ledger of the pass it
148
+ -- currently has in flight. Spine (kernel-written), never a module migration.
149
+ -- Shared with the pure adapter from @substrat-run/kernel so the shape production
150
+ -- builds and the shape a self-host builds cannot part company; the column
151
+ -- comments, and the reason coalescing is NOT a unique index, are in there.
152
+ ${JOB_RUN_DDL}
155
153
  CREATE TABLE IF NOT EXISTS _substrat_migrations (
156
154
  module_id TEXT NOT NULL,
157
155
  version TEXT NOT NULL,
@@ -191,6 +189,17 @@ const KERNEL_DDL = `
191
189
  -- next_attempt_at IS NULL -> terminal: error IS NULL delivered, else dead
192
190
  attempts INTEGER NOT NULL DEFAULT 0,
193
191
  next_attempt_at TEXT,
192
+ -- #1525: the invocation THIS ATTEMPT ran in, the same id #1237 stamps on every
193
+ -- event of a call. Not the emitting event's -- that one is already on the outbox
194
+ -- and a delivery joins to it through event_id. What that join cannot say is
195
+ -- which call attempted the DELIVERY, and for an executor the two differ by
196
+ -- design: attempt one runs in the emitting call's post-commit tail, every retry
197
+ -- afterwards in a drain that is a different call or none. Moves with the row on
198
+ -- an upsert, like delivered_at, error and attempts -- it describes the
199
+ -- latest attempt, not the first. NULL = that attempt carried no call (a
200
+ -- scheduled drain, an alarm, a seed, an attachment RPC), or the row predates
201
+ -- the column.
202
+ invocation_id TEXT,
194
203
  PRIMARY KEY (event_id, consumer_module)
195
204
  );
196
205
  -- Scope-local permissions (docs/architecture/scope-local-permissions.md): the
@@ -505,6 +514,12 @@ export function defineScopeDO(modules, bareOps) {
505
514
  listPlans = new Map();
506
515
  /** entityType → the declared attachment gate (#473): read key + write key (default: read). */
507
516
  attachmentTargets = new Map();
517
+ /**
518
+ * entityType → the declared live-read gate (#938): the key a subscriber must hold
519
+ * ON THAT ENTITY before a change to it is announced. Absent = announced to nobody;
520
+ * `fanOutLive` treats a miss as silence, never as "unguarded".
521
+ */
522
+ liveTargets = new Map();
508
523
  checker;
509
524
  systemPrincipal = principalId.parse(ulid());
510
525
  /**
@@ -622,6 +637,20 @@ export function defineScopeDO(modules, bareOps) {
622
637
  }
623
638
  this.attachmentTargets.set(target.entityType, gate);
624
639
  }
640
+ // Live-read targets (#938): entityType → the read key a subscriber must pass ON
641
+ // THAT ENTITY before a change to it is announced. Validated exactly like the
642
+ // attachment gate above, and for a sharper reason: two modules disagreeing about
643
+ // which key guards an entity type would make "who may watch this" depend on
644
+ // registration order, and the losing declaration would be the one that was meant
645
+ // to be stricter. An entity type nobody declares is announced to nobody.
646
+ for (const target of manifest.liveTargets ?? []) {
647
+ const existing = this.liveTargets.get(target.entityType);
648
+ if (existing && existing !== target.readPermission) {
649
+ throw new Error(`conflicting liveTargets for '${target.entityType}': ` +
650
+ `(${existing}) vs (${target.readPermission})`);
651
+ }
652
+ this.liveTargets.set(target.entityType, target.readPermission);
653
+ }
625
654
  for (const name of manifest.withdraws ?? []) {
626
655
  this.withdrawn.set(name, manifest.id);
627
656
  this.operations.delete(name);
@@ -843,6 +872,24 @@ export function defineScopeDO(modules, bareOps) {
843
872
  return ids.length;
844
873
  });
845
874
  }
875
+ /**
876
+ * How many rows `redrainEvents` WOULD reopen for the same instant (#1545), reopening
877
+ * none of them. A separate verb rather than a flag on the one above, deliberately: the
878
+ * two answers are indistinguishable once they are numbers, so the caller that asks for
879
+ * a count must not be able to reach the reopen by losing an argument on the way.
880
+ *
881
+ * UNBOUNDED where the reopen is batched. The batch exists because an UPDATE over the
882
+ * whole window rewrites every row and its index entries inside ONE Durable Object
883
+ * request, against a fixed budget. An aggregate materialises no rows, so that budget is
884
+ * not the binding constraint — and a partial count would be worse than useless: the
885
+ * number is the whole point, and "5000, or possibly more" answers nothing.
886
+ */
887
+ async redrainCount(drainedBefore) {
888
+ return this.sql
889
+ .exec(`SELECT COUNT(*) AS c FROM _substrat_outbox
890
+ WHERE drained_at IS NOT NULL AND drained_at < ?`, drainedBefore)
891
+ .toArray()[0].c;
892
+ }
846
893
  /**
847
894
  * Facet this scope's own outbox (#1239) — `facetEvents`, which is the
848
895
  * sanctioned read: an erased payload yields the same NULL a missing field
@@ -879,6 +926,10 @@ export function defineScopeDO(modules, bareOps) {
879
926
  invocationEvents(input) {
880
927
  return readInvocation({ sql: doScopedSql(this.sql) }, input.invocationId, input.limit);
881
928
  }
929
+ /** #1525: every delivery in this scope that gave up, inside the DO where both tables live. */
930
+ deadLetters(input) {
931
+ return readDeadLetters({ sql: doScopedSql(this.sql) }, { limit: input.limit, cursor: input.cursor });
932
+ }
882
933
  migrationBookmarks(limit = 20) {
883
934
  return this.sql
884
935
  .exec(`SELECT bookmark, taken_at, pending FROM _substrat_migration_bookmarks
@@ -1159,6 +1210,21 @@ export function defineScopeDO(modules, bareOps) {
1159
1210
  // one call. Same placement as the SQLite adapter's actor task, for this reason.
1160
1211
  this.invocationId = invokeOptions?.invocationId ?? null;
1161
1212
  try {
1213
+ /**
1214
+ * #938: the outbox's high-water mark BEFORE this call wrote anything, so the
1215
+ * post-commit fan-out can name exactly the events this call (and the consumers
1216
+ * it set off) added. Read here — inside the queued body, before the
1217
+ * transaction — because that is the region where this call holds the DO to
1218
+ * itself, the same reason `invocationId` is set here.
1219
+ *
1220
+ * A socket that connects between here and the fan-out is served whatever this
1221
+ * call committed, and one that connects while `null` was decided hears nothing
1222
+ * about it. Both are harmless and neither is worth a lock: a frame is an
1223
+ * invalidation, so hearing about a change from just before you subscribed costs
1224
+ * one redundant re-read, and missing one costs a wait for the client's poll —
1225
+ * which is the floor this whole surface sits on.
1226
+ */
1227
+ const liveSince = this.liveHighWaterMark();
1162
1228
  let result;
1163
1229
  let committedVersion = null;
1164
1230
  // #116: set when this invocation was answered from a recording rather
@@ -1270,7 +1336,10 @@ export function defineScopeDO(modules, bareOps) {
1270
1336
  ? { kind: 'system', id: systemModuleId }
1271
1337
  : connectionId
1272
1338
  ? { kind: 'connection', id: connectionId }
1273
- : { kind: 'principal', id: principal }, tenantId, operation, err, impersonation);
1339
+ : { kind: 'principal', id: principal }, tenantId, operation, err,
1340
+ // Inside the queued body, which is the one region where this call holds
1341
+ // the DO to itself — so the field is this call's own (#1237).
1342
+ this.invocationId, impersonation);
1274
1343
  }
1275
1344
  // The ORIGINAL error, deliberately: `invoke` flattens it for the envelope
1276
1345
  // (which keeps its code and extensions) or rewraps it for the legacy throw
@@ -1281,8 +1350,14 @@ export function defineScopeDO(modules, bareOps) {
1281
1350
  // Skipped on a replay: nothing was written, so there is nothing this
1282
1351
  // invocation added to drain. Anything the ORIGINAL left undrained is the
1283
1352
  // outbox's own retry backstop, which is what that backstop is for.
1353
+ // Drain to consumers, then announce what landed (#938). Skipped whole on a
1354
+ // replay: nothing was written, so there is nothing this invocation added to
1355
+ // drain or to announce. Anything the ORIGINAL left undrained is the outbox's
1356
+ // own retry backstop, which is what that backstop is for.
1357
+ // #1525: still inside the queued body that set it, so these deliveries are this
1358
+ // call's own work — the same tail its consumers' emits are stamped in.
1284
1359
  if (!replayed)
1285
- await this.dispatch(tenantId, scopeId);
1360
+ await this.settleCommitted(tenantId, scopeId, liveSince, this.invocationId);
1286
1361
  return {
1287
1362
  result,
1288
1363
  platformRequests: signals.platformRequests,
@@ -1315,6 +1390,288 @@ export function defineScopeDO(modules, bareOps) {
1315
1390
  }
1316
1391
  });
1317
1392
  }
1393
+ // -- live reads (#938): the subscription half of the change feed ------------
1394
+ // The ONLY part of this DO addressed as a fetch target rather than over RPC, and
1395
+ // only because a WebSocket cannot cross RPC — a socket is not serializable, so the
1396
+ // one way to hand one back is a `Response` carrying a `webSocket`. Everything the
1397
+ // coordinator asserts on the way in is named in `live-reads.ts`, shared with
1398
+ // `host.ts` so the two ends cannot drift.
1399
+ /**
1400
+ * Accept a subscription to this scope's changes.
1401
+ *
1402
+ * The coordinator has already decided that this connection can carry a push at all
1403
+ * (the O2O check) and WHO is asking. What is decided here is nothing about
1404
+ * authority: a subscription is not an authorization, and accepting one grants the
1405
+ * subscriber no read it did not already have. Every frame is checked on its way
1406
+ * out, individually, against the tuple state at that moment — so a grant revoked
1407
+ * while the socket is open stops the frames it used to allow, which a
1408
+ * subscription-time check would not.
1409
+ *
1410
+ * **This adds no authority to a holder of the stub, and the question is worth
1411
+ * answering rather than leaving to be asked.** Being a public method, anything with
1412
+ * the `SCOPE` binding can call it and assert whatever principal it likes in the
1413
+ * headers. That is already true of `invoke`, which takes the principal as an
1414
+ * argument and acts on it: a stub is the key to the scope, which is precisely why
1415
+ * the router is not given one. The trust boundary is who holds the binding, not
1416
+ * what this method checks — and what it does NOT do is let a stub-holder read
1417
+ * anything the asserted principal could not, because the filter downstream re-checks
1418
+ * that principal against every frame.
1419
+ */
1420
+ async fetch(request) {
1421
+ const url = new URL(request.url);
1422
+ if (url.pathname !== LIVE_SUBSCRIBE_PATH) {
1423
+ // The DO has exactly one fetch surface. Anything else reaching here is a
1424
+ // coordinator bug, and a 404 says so without guessing at an intent.
1425
+ return new Response('this scope has no such surface', { status: 404 });
1426
+ }
1427
+ if (!isUpgradeRequest(request)) {
1428
+ return new Response('live reads are a WebSocket surface', {
1429
+ status: 426,
1430
+ headers: { [LIVE_MODE_HEADER]: 'not-an-upgrade' },
1431
+ });
1432
+ }
1433
+ const principal = request.headers.get(LIVE_PRINCIPAL_HEADER);
1434
+ const tenantId = request.headers.get(LIVE_TENANT_HEADER);
1435
+ const scopeId = request.headers.get(LIVE_SCOPE_HEADER);
1436
+ if (!principal || !tenantId || !scopeId) {
1437
+ // Fail closed and loudly. An unnamed subscriber is one whose permissions
1438
+ // cannot be evaluated, and the only safe thing to do with a channel we cannot
1439
+ // filter is to refuse to open it. 500, not 400: the caller is the coordinator
1440
+ // in this same package, so a missing assertion is OUR bug, not the client's.
1441
+ return new Response('live reads require an asserted principal, tenant and scope', {
1442
+ status: 500,
1443
+ });
1444
+ }
1445
+ // A subscriber arriving before the scope's migrations have run would be told
1446
+ // about events against a schema it cannot read back through. Same gate every
1447
+ // other entry point takes, for the same reason.
1448
+ await this.ensureMigrations();
1449
+ const pair = new WebSocketPair();
1450
+ const [client, server] = Object.values(pair);
1451
+ // HIBERNATABLE, not `server.accept()`. A scope with a watcher open would
1452
+ // otherwise be pinned in memory for as long as somebody has a tab open, which is
1453
+ // the cost model inverted: a support desk being WATCHED is the normal state.
1454
+ // Hibernation also fixes the worse half — an in-memory roster does not survive
1455
+ // eviction, so the socket would stay open and silently stop receiving, which is
1456
+ // indistinguishable from a quiet scope.
1457
+ this.ctx.acceptWebSocket(server);
1458
+ // `.parse`, not a cast: these three arrived as header strings, and the branded
1459
+ // ids are what every check downstream is keyed on. A malformed one would
1460
+ // otherwise be carried all the way to a `ctx.check` that quietly matches nothing —
1461
+ // which reads as "this subscriber may see nothing" and is indistinguishable from
1462
+ // a correct denial. Refused here instead, where it is still one subscriber's
1463
+ // problem. Throwing is right: the coordinator built this request.
1464
+ server.serializeAttachment({
1465
+ principal: principalId.parse(principal),
1466
+ tenantId: tenantIdOf.parse(tenantId),
1467
+ scopeId: scopeIdOf.parse(scopeId),
1468
+ since: new Date().toISOString(),
1469
+ });
1470
+ return new Response(null, { status: 101, webSocket: client });
1471
+ }
1472
+ /**
1473
+ * A subscriber said something.
1474
+ *
1475
+ * The channel is one-way by design — the server announces, the client re-reads
1476
+ * through the ordinary operation — so there is no client message that can cause a
1477
+ * read, a write, or a change of subscription. `ping`/`pong` is the whole protocol,
1478
+ * and it exists so an idle connection can be kept alive by either end.
1479
+ *
1480
+ * Deliberately NOT a place to let a client narrow or widen what it receives: a
1481
+ * filter the client chooses is a filter the client can choose wrongly, and the
1482
+ * only filter that matters here is the one it does not control.
1483
+ */
1484
+ webSocketMessage(ws, message) {
1485
+ if (message === 'ping')
1486
+ ws.send('pong');
1487
+ }
1488
+ /**
1489
+ * The client hung up. Close our end so the runtime stops holding the subscription.
1490
+ *
1491
+ * The runtime passes `reason` and `wasClean` too; neither is read, so neither is
1492
+ * named — there is nothing to do differently for an unclean close, because the
1493
+ * socket is going away either way and the client's poll is what covers the gap.
1494
+ */
1495
+ webSocketClose(ws, code) {
1496
+ // 1006 is reserved: it is what the runtime REPORTS for an abnormal closure and
1497
+ // is not a code anything may SEND, so echoing it back throws — on precisely the
1498
+ // path where the connection is already in trouble.
1499
+ try {
1500
+ ws.close(code === 1006 ? 1000 : code, 'scope closing the subscription');
1501
+ }
1502
+ catch {
1503
+ // Already gone. Nothing to do, and nothing worth reporting.
1504
+ }
1505
+ }
1506
+ /**
1507
+ * A socket failed. Logged and not rethrown: there is no caller to fail, and the
1508
+ * runtime delivers `webSocketClose` after this, which is what does the cleanup.
1509
+ */
1510
+ webSocketError(_ws, error) {
1511
+ console.error('substrat: live-read socket error', error);
1512
+ }
1513
+ /**
1514
+ * The outbox's high-water mark before a committing path runs (#938), or `null`
1515
+ * when nobody is listening.
1516
+ *
1517
+ * `null` and `''` are different answers: `null` means no subscriber, so nothing is
1518
+ * read at all and a scope pays nothing for a feature it is not using; `''` is the
1519
+ * honest empty-outbox answer, and every ULID sorts above it.
1520
+ *
1521
+ * Read BEFORE the transaction by every caller, which is what makes the pair below
1522
+ * able to name exactly the events that path added.
1523
+ */
1524
+ liveHighWaterMark() {
1525
+ if (this.ctx.getWebSockets().length === 0)
1526
+ return null;
1527
+ return (this.sql.exec('SELECT MAX(id) AS id FROM _substrat_outbox').toArray()[0]?.id ?? '');
1528
+ }
1529
+ /**
1530
+ * What every committing path does after its transaction closes: drain the outbox
1531
+ * to consumers, then announce what landed to whoever may see it.
1532
+ *
1533
+ * **One step, because there is one rule.** The fan-out was wired into `invoke`
1534
+ * alone at first, and `attachmentAdd`/`attachmentRemove` commit and emit too —
1535
+ * `attachment.added` and `attachment.removed`, about a real entity. A watcher of
1536
+ * that entity would have missed them: no error, no gap it could see, just a screen
1537
+ * that did not update for one kind of change. That is the exact failure the
1538
+ * hibernation design exists to prevent, arriving through a different door, so the
1539
+ * answer is a step both doors take rather than a second call both must remember.
1540
+ *
1541
+ * Order is load-bearing: drain FIRST, announce after. A consumer's own emits are
1542
+ * changes too, and they land in the outbox above `liveSince` — so announcing first
1543
+ * would tell a subscriber about the cause and not the effect, and it would re-read
1544
+ * too early.
1545
+ */
1546
+ async settleCommitted(tenantId, scopeId, liveSince,
1547
+ /**
1548
+ * #1525: the call whose commit this is settling, or null — PASSED rather than
1549
+ * read off `this.invocationId` inside `dispatch`, so each of the three doors
1550
+ * says what it actually knows: `invoke` its own id, the two attachment verbs
1551
+ * null. Same discipline `recordDenial` adopted, and for the same reason.
1552
+ */
1553
+ invocationId) {
1554
+ await this.dispatch(tenantId, scopeId, invocationId);
1555
+ if (liveSince === null)
1556
+ return;
1557
+ try {
1558
+ await this.fanOutLive(liveSince, tenantId, scopeId);
1559
+ }
1560
+ catch (err) {
1561
+ // The write has COMMITTED and the caller is owed its answer. A failure to
1562
+ // announce is a failure of a hint, and the client's poll is the floor
1563
+ // underneath it — so this is logged and never rethrown. Rethrowing would turn
1564
+ // a delivered write into a 500 the caller would reasonably retry.
1565
+ console.error('substrat: live-read fan-out failed after commit', err);
1566
+ }
1567
+ }
1568
+ /**
1569
+ * Announce what just committed, to whoever may see it (#938).
1570
+ *
1571
+ * **Three properties, and each is load-bearing.**
1572
+ *
1573
+ * *Post-commit.* Called after the operation's transaction has closed and after the
1574
+ * consumer drain, so nothing is announced that a rollback could take back and
1575
+ * nothing a consumer emitted is missed. A subscriber told about a row that then
1576
+ * vanished would re-read, find nothing, and have no way to tell that from a
1577
+ * deletion.
1578
+ *
1579
+ * *Filtered per subscriber, per event, after the check.* The declared
1580
+ * `liveTargets` key is checked ON THE EVENT'S OWN ENTITY, through `ctx.check` —
1581
+ * the same evaluator, the same entity-narrowed grants, the same parent walk as the
1582
+ * read the client is about to make. An entity type no module declared is announced
1583
+ * to nobody. Knowing that a row exists and changed at 14:02 is information about
1584
+ * that row, so the empty payload is not what makes this safe; this is.
1585
+ *
1586
+ * *Never able to fail the operation.* The write has committed and the caller has
1587
+ * its answer. A socket that has gone away mid-fan-out, or a check that cannot be
1588
+ * evaluated, costs a subscriber its live update — which it survives, because the
1589
+ * client's contract is that a push is a hint and the poll is the floor.
1590
+ */
1591
+ async fanOutLive(sinceEventId, tenantId, scopeId) {
1592
+ const sockets = this.ctx.getWebSockets();
1593
+ if (sockets.length === 0)
1594
+ return;
1595
+ const rows = this.sql
1596
+ .exec(`SELECT id, type, entity_type, entity_id, occurred_at FROM _substrat_outbox
1597
+ WHERE id > ? ORDER BY id LIMIT ?`, sinceEventId, LIVE_FANOUT_LIMIT)
1598
+ .toArray();
1599
+ // Drop the undeclared entity types BEFORE any per-subscriber work. Not an
1600
+ // optimisation: it is the fail-closed rule stated once, in the one place that
1601
+ // decides, rather than relied upon inside the loop below.
1602
+ const announceable = rows.filter((r) => this.liveTargets.has(r.entity_type));
1603
+ if (announceable.length === 0)
1604
+ return;
1605
+ for (const ws of sockets) {
1606
+ let subscription = null;
1607
+ try {
1608
+ subscription = readSubscription(ws.deserializeAttachment());
1609
+ }
1610
+ catch {
1611
+ subscription = null;
1612
+ }
1613
+ // A socket we cannot name is a socket we cannot filter for. Skipped, never
1614
+ // sent to — see `readSubscription` for why every unusable shape fails closed.
1615
+ if (!subscription)
1616
+ continue;
1617
+ // A socket that outlived a scope rebind, or was somehow accepted for another
1618
+ // node, must not be fed this scope's events. Cheap, and it makes the identity
1619
+ // the frames are filtered against an explicit precondition rather than an
1620
+ // assumption about how the subscription was created.
1621
+ if (subscription.tenantId !== tenantId || subscription.scopeId !== scopeId)
1622
+ continue;
1623
+ // One context per subscriber, not per event: `ctx.check` is the expensive part
1624
+ // and the context is only the subject it is evaluated for.
1625
+ //
1626
+ // A frame this subscriber does not pass is NOT recorded as a denial (K-35),
1627
+ // and that is deliberate: `recordDenial` is called on a refused REQUEST, where
1628
+ // somebody asked for something and was told no. Nobody asked for these. Logging
1629
+ // one row per unentitled subscriber per event would bury the denials that mean
1630
+ // something — a broken screen, or somebody walking the surface — under the
1631
+ // ordinary, correct working of a filter.
1632
+ //
1633
+ // The operation name is carried anyway, for the events a fan-out cannot emit
1634
+ // but a future reader of this context might.
1635
+ const ctx = this.operationContext(subscription.principal, tenantId, scopeId, undefined, undefined, undefined, undefined, undefined, 'live.subscribe');
1636
+ for (const row of announceable) {
1637
+ // Non-null: `announceable` is exactly the rows whose type is in the map.
1638
+ const permission = this.liveTargets.get(row.entity_type);
1639
+ let allowed = false;
1640
+ try {
1641
+ const decision = await ctx.check(permission, {
1642
+ entityType: row.entity_type,
1643
+ entityId: row.entity_id,
1644
+ });
1645
+ allowed = decision.allowed;
1646
+ }
1647
+ catch {
1648
+ // A check that cannot answer is a check that refuses. The alternative —
1649
+ // treating an evaluator failure as an allow — turns an outage in the
1650
+ // permission path into a disclosure, which is the one failure mode this
1651
+ // surface must not have.
1652
+ allowed = false;
1653
+ }
1654
+ if (!allowed)
1655
+ continue;
1656
+ const frame = {
1657
+ kind: 'change',
1658
+ id: row.id,
1659
+ type: row.type,
1660
+ entityType: row.entity_type,
1661
+ entityId: row.entity_id,
1662
+ at: row.occurred_at,
1663
+ };
1664
+ try {
1665
+ ws.send(JSON.stringify(frame));
1666
+ }
1667
+ catch {
1668
+ // The socket went away between `getWebSockets()` and here. Stop writing to
1669
+ // this one and move on; the runtime will deliver `webSocketClose`.
1670
+ break;
1671
+ }
1672
+ }
1673
+ }
1674
+ }
1318
1675
  // -- attachments (#473): the metadata half of the attachment surface --------
1319
1676
  // The coordinator (worker) holds the bytes and the per-tenant R2 binding; this
1320
1677
  // DO holds the permission gate and the metadata fact, under the same per-scope
@@ -1355,6 +1712,10 @@ export function defineScopeDO(modules, bareOps) {
1355
1712
  const parsed = attachmentRecord.parse(record);
1356
1713
  const gate = this.attachmentGate(parsed.entity.entityType);
1357
1714
  return this.queue.enqueue(async () => {
1715
+ // #938: this path commits and emits too, so it takes the same mark-then-settle
1716
+ // pair `invoke` takes. Inside the queued body and before the transaction, for
1717
+ // the reason it is read there: that is where this call holds the DO to itself.
1718
+ const liveSince = this.liveHighWaterMark();
1358
1719
  try {
1359
1720
  await this.ctx.storage.transaction(async () => {
1360
1721
  const ctx = this.operationContext(principal, tenantId, scopeId, undefined, connectionId, undefined, undefined, undefined, 'attachments.upload');
@@ -1374,11 +1735,13 @@ export function defineScopeDO(modules, bareOps) {
1374
1735
  }
1375
1736
  catch (err) {
1376
1737
  if (err instanceof PermissionDenied) {
1377
- this.recordDenial(attachSubject(principal, connectionId), tenantId, 'attachments.upload', err);
1738
+ this.recordDenial(attachSubject(principal, connectionId), tenantId, 'attachments.upload', err, null);
1378
1739
  }
1379
1740
  throw toRpcError(err);
1380
1741
  }
1381
- await this.dispatch(tenantId, scopeId);
1742
+ // #1525: null. An attachment RPC carries no invocation, exactly as its denial
1743
+ // records none — so the deliveries it drains name no call rather than a wrong one.
1744
+ await this.settleCommitted(tenantId, scopeId, liveSince, null);
1382
1745
  return parsed;
1383
1746
  });
1384
1747
  }
@@ -1392,7 +1755,7 @@ export function defineScopeDO(modules, bareOps) {
1392
1755
  }
1393
1756
  catch (err) {
1394
1757
  if (err instanceof PermissionDenied) {
1395
- this.recordDenial(attachSubject(principal, connectionId), tenantId, 'attachments.list', err);
1758
+ this.recordDenial(attachSubject(principal, connectionId), tenantId, 'attachments.list', err, null);
1396
1759
  }
1397
1760
  throw toRpcError(err);
1398
1761
  }
@@ -1423,7 +1786,7 @@ export function defineScopeDO(modules, bareOps) {
1423
1786
  }
1424
1787
  catch (err) {
1425
1788
  if (err instanceof PermissionDenied) {
1426
- this.recordDenial(attachSubject(principal, connectionId), tenantId, 'attachments.open', err);
1789
+ this.recordDenial(attachSubject(principal, connectionId), tenantId, 'attachments.open', err, null);
1427
1790
  }
1428
1791
  throw toRpcError(err);
1429
1792
  }
@@ -1436,7 +1799,7 @@ export function defineScopeDO(modules, bareOps) {
1436
1799
  }
1437
1800
  catch (err) {
1438
1801
  if (err instanceof PermissionDenied) {
1439
- this.recordDenial(attachSubject(principal, connectionId), tenantId, 'attachments.open', err);
1802
+ this.recordDenial(attachSubject(principal, connectionId), tenantId, 'attachments.open', err, null);
1440
1803
  }
1441
1804
  throw toRpcError(err);
1442
1805
  }
@@ -1450,6 +1813,8 @@ export function defineScopeDO(modules, bareOps) {
1450
1813
  if (!record)
1451
1814
  return null;
1452
1815
  const gate = this.attachmentGate(record.entity.entityType);
1816
+ // #938: same mark-then-settle pair as the upload path above.
1817
+ const liveSince = this.liveHighWaterMark();
1453
1818
  try {
1454
1819
  await this.ctx.storage.transaction(async () => {
1455
1820
  const ctx = this.operationContext(principal, tenantId, scopeId, undefined, connectionId, undefined, undefined, undefined, 'attachments.remove');
@@ -1466,11 +1831,13 @@ export function defineScopeDO(modules, bareOps) {
1466
1831
  }
1467
1832
  catch (err) {
1468
1833
  if (err instanceof PermissionDenied) {
1469
- this.recordDenial(attachSubject(principal, connectionId), tenantId, 'attachments.remove', err);
1834
+ this.recordDenial(attachSubject(principal, connectionId), tenantId, 'attachments.remove', err, null);
1470
1835
  }
1471
1836
  throw toRpcError(err);
1472
1837
  }
1473
- await this.dispatch(tenantId, scopeId);
1838
+ // #1525: null. An attachment RPC carries no invocation, exactly as its denial
1839
+ // records none — so the deliveries it drains name no call rather than a wrong one.
1840
+ await this.settleCommitted(tenantId, scopeId, liveSince, null);
1474
1841
  return record;
1475
1842
  });
1476
1843
  }
@@ -1488,10 +1855,17 @@ export function defineScopeDO(modules, bareOps) {
1488
1855
  .toArray()[0];
1489
1856
  return row !== undefined;
1490
1857
  }
1491
- /** The last time a schedule's operation ran on this scope (#383), or null. */
1858
+ /**
1859
+ * The last time a schedule's operation ran on this scope (#383), or null.
1860
+ *
1861
+ * `kind = 'schedule'` is not decoration (#1288): an operation may legally be
1862
+ * named `freshness:<something>`, and before the column this read answered with
1863
+ * the EVALUATOR's last recorded time for that event type — a cadence gate
1864
+ * driven by a verdict nothing ran.
1865
+ */
1492
1866
  async scheduleLastRun(operation) {
1493
1867
  const row = this.sql
1494
- .exec(`SELECT last_run_at FROM _substrat_schedule_state WHERE schedule_op = ?`, operation)
1868
+ .exec(`SELECT last_run_at FROM _substrat_schedule_state WHERE kind = 'schedule' AND schedule_op = ?`, operation)
1495
1869
  .toArray()[0];
1496
1870
  return row?.last_run_at ?? null;
1497
1871
  }
@@ -1499,8 +1873,9 @@ export function defineScopeDO(modules, bareOps) {
1499
1873
  * #1232: everything the freshness evaluator needs about this scope, one round
1500
1874
  * trip regardless of how many types are declared — the newest matching event
1501
1875
  * per type, plus each type's recorded evaluator state (last recorded at +
1502
- * outcome, kept in `_substrat_schedule_state` under a `freshness:` prefix that
1503
- * cannot collide with operation names).
1876
+ * outcome, kept in `_substrat_schedule_state` under `kind = 'freshness'`, which
1877
+ * since #1288 is what separates these rows from the schedule rows beside them —
1878
+ * the `freshness:` prefix on the key is retained but no longer load-bearing).
1504
1879
  */
1505
1880
  async freshnessProbe(types) {
1506
1881
  const out = {};
@@ -1517,7 +1892,8 @@ export function defineScopeDO(modules, bareOps) {
1517
1892
  }
1518
1893
  const keys = types.map((t) => `freshness:${t}`);
1519
1894
  for (const row of this.sql
1520
- .exec(`SELECT schedule_op, last_run_at, last_status FROM _substrat_schedule_state WHERE schedule_op IN (${marks})`, ...keys)
1895
+ .exec(`SELECT schedule_op, last_run_at, last_status FROM _substrat_schedule_state
1896
+ WHERE kind = 'freshness' AND schedule_op IN (${marks})`, ...keys)
1521
1897
  .toArray()) {
1522
1898
  const t = row.schedule_op.slice('freshness:'.length);
1523
1899
  if (out[t]) {
@@ -1529,16 +1905,173 @@ export function defineScopeDO(modules, bareOps) {
1529
1905
  }
1530
1906
  /**
1531
1907
  * Write one `_substrat_schedule_state` row (#383). Spine, kernel-written.
1532
- * `unit` is the row's key in either of the two shapes the table holds: a schedule
1908
+ *
1909
+ * `kind` is PASSED, never derived from the shape of `unit` (#1288) — deriving it
1910
+ * is precisely the convention this column replaced, and it is wrong for the one
1911
+ * input that matters: a schedule operation named `freshness:orders.placed` is a
1912
+ * schedule row, whatever its key looks like. `unit` is that row's key: a schedule
1533
1913
  * operation (`module/verb`, and then `at`/`status` are when it ran and how it
1534
1914
  * ended) or a freshness key (`freshness:<eventType>`, and then they are when the
1535
1915
  * verdict was recorded and what it was — nothing ran). See the bootstrap DDL.
1916
+ *
1917
+ * `kind` is LAST because this is a positional RPC — see the interface's own note
1918
+ * in `host.ts`. An old DO drops a trailing argument; a leading one it binds, and
1919
+ * every value after it lands one column to the left, silently.
1920
+ */
1921
+ async recordScheduleRun(unit, at, status, kind) {
1922
+ this.sql.exec(`INSERT INTO _substrat_schedule_state (kind, schedule_op, last_run_at, last_status)
1923
+ VALUES (?, ?, ?, ?)
1924
+ ON CONFLICT(kind, schedule_op) DO UPDATE SET last_run_at = excluded.last_run_at,
1925
+ last_status = excluded.last_status`, kind, unit, at, status);
1926
+ }
1927
+ // -- the resumable-run driver's store (#1577) -----------------------------
1928
+ //
1929
+ // D-14's DURABLE driver, and the only half of it that lives in the DO: the run
1930
+ // record and the step ledger. Every decision — what coalescing means, when a
1931
+ // step is skipped, when a run fails — is in the kernel's `JobRunStore` callers,
1932
+ // which the coordinator drives; these are reads and writes and nothing else, so
1933
+ // the two drivers cannot disagree about any of it.
1934
+ //
1935
+ // Each step commits on its OWN round trip rather than the pass batching them at
1936
+ // the end. That is what a mid-pass eviction keeps, and a DO is evicted and
1937
+ // revived constantly — batching would lose exactly the work resume exists for.
1938
+ /**
1939
+ * Coalescing, as ONE round trip: the live (`running`) run for this key, or this
1940
+ * row inserted and returned.
1941
+ *
1942
+ * **The single RPC is the atomicity**, and that is the whole reason it is shaped
1943
+ * this way rather than as a `jobRunLive` the coordinator follows with a
1944
+ * `jobRunInsert`. A Durable Object serializes its RPCs, so the lookup and the
1945
+ * insert cannot be interleaved by another caller; split across two calls they
1946
+ * can, and two concurrent starts then both find nothing and both insert. The
1947
+ * schema carries no unique constraint to catch that, deliberately — a crashed
1948
+ * run must stay restartable — so the indivisibility has to come from here.
1949
+ */
1950
+ async jobRunStartOrJoin(moduleId, job, instance, row) {
1951
+ const live = this.sql
1952
+ .exec(`SELECT * FROM _substrat_job_runs
1953
+ WHERE module_id = ? AND job = ? AND instance = ? AND status = 'running'
1954
+ ORDER BY id DESC LIMIT 1`, moduleId, job, instance)
1955
+ .toArray()[0] ?? null;
1956
+ if (live)
1957
+ return live;
1958
+ // `transactionSync`, and the two writes inlined rather than reached through
1959
+ // `jobRunInsert`: an `await` between them is an output-gate boundary, and the
1960
+ // point of doing this in one RPC is that there is no boundary to be evicted
1961
+ // at. Same reason `SCHEDULE_STATE_REBUILD` insists on it.
1962
+ this.ctx.storage.transactionSync(() => {
1963
+ this.sql.exec(`INSERT INTO _substrat_job_runs
1964
+ (id, module_id, job, instance, payload, status, cursor, counters, attempts,
1965
+ last_error, started_at, updated_at, next_attempt_at, ended_at)
1966
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, row.id, row.module_id, row.job, row.instance, row.payload, row.status, row.cursor, row.counters, row.attempts, row.last_error, row.started_at, row.updated_at, row.next_attempt_at, row.ended_at);
1967
+ });
1968
+ return row;
1969
+ }
1970
+ /** One run by id, whatever its status. */
1971
+ async jobRunById(id) {
1972
+ return (this.sql.exec('SELECT * FROM _substrat_job_runs WHERE id = ?', id).toArray()[0] ?? null);
1973
+ }
1974
+ /** Insert a fresh run. The coordinator has already refused a non-queue-safe payload. */
1975
+ async jobRunInsert(row) {
1976
+ this.sql.exec(`INSERT INTO _substrat_job_runs
1977
+ (id, module_id, job, instance, payload, status, cursor, counters, attempts,
1978
+ last_error, started_at, updated_at, next_attempt_at, ended_at)
1979
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, row.id, row.module_id, row.job, row.instance, row.payload, row.status, row.cursor, row.counters, row.attempts, row.last_error, row.started_at, row.updated_at, row.next_attempt_at, row.ended_at);
1980
+ }
1981
+ /**
1982
+ * `running` runs whose backoff has elapsed, oldest first, after `afterId`.
1983
+ *
1984
+ * The cursor is what lets the coordinator page past runs it cannot drive: it
1985
+ * skips any whose job this deployment does not register, and without a cursor
1986
+ * those rows head every batch forever (`runDueJobRuns`). `afterId` is LAST, as
1987
+ * every argument added to an RPC on this interface must be.
1536
1988
  */
1537
- async recordScheduleRun(unit, at, status) {
1538
- this.sql.exec(`INSERT INTO _substrat_schedule_state (schedule_op, last_run_at, last_status)
1539
- VALUES (?, ?, ?)
1540
- ON CONFLICT(schedule_op) DO UPDATE SET last_run_at = excluded.last_run_at,
1541
- last_status = excluded.last_status`, unit, at, status);
1989
+ async jobRunsDue(now, limit, afterId) {
1990
+ return this.sql
1991
+ .exec(
1992
+ // `afterId` bound TWICE rather than as `?2`: mixing anonymous and numbered
1993
+ // parameters makes the anonymous ones resume from the highest index used,
1994
+ // which is a footgun for the next person to add a clause. Spelled exactly
1995
+ // as the pure adapter spells it.
1996
+ `SELECT * FROM _substrat_job_runs
1997
+ WHERE status = 'running' AND (next_attempt_at IS NULL OR next_attempt_at <= ?)
1998
+ AND (? IS NULL OR id > ?)
1999
+ ORDER BY id LIMIT ?`, now, afterId ?? null, afterId ?? null, limit)
2000
+ .toArray();
2001
+ }
2002
+ /** The operator read, newest first. */
2003
+ async jobRunList(filter) {
2004
+ const where = [];
2005
+ const params = [];
2006
+ for (const [column, value] of [
2007
+ ['module_id', filter.moduleId],
2008
+ ['job', filter.job],
2009
+ ['instance', filter.instance],
2010
+ ['status', filter.status],
2011
+ ]) {
2012
+ if (value !== undefined) {
2013
+ where.push(`${column} = ?`);
2014
+ params.push(value);
2015
+ }
2016
+ }
2017
+ params.push(jobRunListLimit(filter.limit));
2018
+ return this.sql
2019
+ .exec(`SELECT * FROM _substrat_job_runs
2020
+ ${where.length ? `WHERE ${where.join(' AND ')}` : ''}
2021
+ ORDER BY id DESC LIMIT ?`, ...params)
2022
+ .toArray();
2023
+ }
2024
+ /**
2025
+ * Write a pass outcome onto the run row — every column at once.
2026
+ *
2027
+ * Whole rather than partial, and that is the shape not an accident: a commit
2028
+ * writes cursor AND counters AND a cleared error AND a cleared backoff, and a
2029
+ * partial update that wrote three of the four would leave a run carrying the
2030
+ * last failure's error beside the new cursor, which reads as broken forever.
2031
+ */
2032
+ async jobRunPatch(id, patch) {
2033
+ this.sql.exec(`UPDATE _substrat_job_runs
2034
+ SET status = ?, cursor = ?, counters = ?, attempts = ?, last_error = ?,
2035
+ updated_at = ?, next_attempt_at = ?, ended_at = ?
2036
+ WHERE id = ?`, patch.status, patch.cursor, patch.counters, patch.attempts, patch.lastError, patch.updatedAt, patch.nextAttemptAt, patch.endedAt, id);
2037
+ }
2038
+ /** One step's ledger row — a non-null `result` is what means completed. */
2039
+ async jobStepRow(runId, step) {
2040
+ return (this.sql
2041
+ .exec('SELECT step, result, attempts, last_error FROM _substrat_job_steps WHERE run_id = ? AND step = ?', runId, step)
2042
+ .toArray()[0] ?? null);
2043
+ }
2044
+ /** Record one step attempt. `result` non-null = it completed and must not re-run. */
2045
+ async jobStepRecord(runId, step, result, attempts, lastError, at) {
2046
+ this.sql.exec(`INSERT INTO _substrat_job_steps (run_id, step, result, attempts, last_error, recorded_at)
2047
+ VALUES (?, ?, ?, ?, ?, ?)
2048
+ ON CONFLICT (run_id, step) DO UPDATE SET result = excluded.result,
2049
+ attempts = excluded.attempts,
2050
+ last_error = excluded.last_error,
2051
+ recorded_at = excluded.recorded_at`, runId, step, result, attempts, lastError, at);
2052
+ }
2053
+ /**
2054
+ * A COMMITTED pass: the run's new state and the dropping of its step ledger, in
2055
+ * ONE RPC so they cannot come apart.
2056
+ *
2057
+ * Same reasoning as `jobRunStartOrJoin` — the single round trip is the
2058
+ * atomicity. As two calls, a stop in between leaves the advanced cursor beside
2059
+ * the finished pass's memo rows, and a handler that reuses a step name across
2060
+ * passes (legal: the determinism rule binds names to the payload and prior
2061
+ * results, not to the cursor) then skips work it never did. The kernel's
2062
+ * `runJobPass` carries the full argument.
2063
+ */
2064
+ async jobCommitPass(id, patch) {
2065
+ // `transactionSync` with both statements inline — NOT `await
2066
+ // this.jobRunPatch(...)` then the delete. The await is an output-gate
2067
+ // boundary, which is precisely the gap this method exists to close.
2068
+ this.ctx.storage.transactionSync(() => {
2069
+ this.sql.exec(`UPDATE _substrat_job_runs
2070
+ SET status = ?, cursor = ?, counters = ?, attempts = ?, last_error = ?,
2071
+ updated_at = ?, next_attempt_at = ?, ended_at = ?
2072
+ WHERE id = ?`, patch.status, patch.cursor, patch.counters, patch.attempts, patch.lastError, patch.updatedAt, patch.nextAttemptAt, patch.endedAt, id);
2073
+ this.sql.exec('DELETE FROM _substrat_job_steps WHERE run_id = ?', id);
2074
+ });
1542
2075
  }
1543
2076
  // -- guards (K-17) --------------------------------------------------------
1544
2077
  /** This scope's spine, read under whatever transaction the caller already opened. */
@@ -1661,20 +2194,44 @@ export function defineScopeDO(modules, bareOps) {
1661
2194
  * Written AFTER the effect, so a crash mid-effect retries rather than silently
1662
2195
  * marking success. The coordinator computes the backoff because it owns the
1663
2196
  * per-executor policy; the DO owns the state.
2197
+ *
2198
+ * `invocationId` is LAST because this is a positional RPC — the same rule
2199
+ * `recordScheduleRun` states. An old DO drops a trailing argument; a leading one
2200
+ * it binds, and every value after it lands one column to the left, silently.
2201
+ */
2202
+ recordExecutorAttempt(eventId, deliveryId, error, nextAttemptAt,
2203
+ /**
2204
+ * #1525: the call THIS attempt ran in, or null. Passed rather than read off
2205
+ * `this.invocationId`, and here there is no choice about it: executors run on
2206
+ * the COORDINATOR, so by the time this RPC arrives the queued body that held
2207
+ * the id has long returned and the field reads null. The coordinator is the
2208
+ * only side that knows, which is why the SQLite twin passes it too — a recorded
2209
+ * fact whose correctness argument differs per adapter is the kind that drifts.
2210
+ *
2211
+ * Defaulted, for the other half of the skew: a coordinator too old to pass it
2212
+ * calls this with four arguments, and `undefined` is not a value SQLite binds.
2213
+ * Null is the honest answer there anyway — that coordinator recorded no call.
1664
2214
  */
1665
- recordExecutorAttempt(eventId, deliveryId, error, nextAttemptAt) {
2215
+ invocationId = null) {
1666
2216
  const prior = this.sql
1667
2217
  .exec('SELECT attempts FROM _substrat_deliveries WHERE event_id = ? AND consumer_module = ?', eventId, deliveryId)
1668
2218
  .toArray()[0];
1669
2219
  const attempts = (prior?.attempts ?? 0) + 1;
1670
2220
  this.sql.exec(`INSERT INTO _substrat_deliveries
1671
- (event_id, consumer_module, delivered_at, error, attempts, next_attempt_at)
1672
- VALUES (?, ?, ?, ?, ?, ?)
2221
+ (event_id, consumer_module, delivered_at, error, attempts, next_attempt_at,
2222
+ invocation_id)
2223
+ VALUES (?, ?, ?, ?, ?, ?, ?)
1673
2224
  ON CONFLICT (event_id, consumer_module) DO UPDATE SET
1674
2225
  delivered_at = excluded.delivered_at,
1675
2226
  error = excluded.error,
1676
2227
  attempts = excluded.attempts,
1677
- next_attempt_at = excluded.next_attempt_at`, eventId, deliveryId, new Date().toISOString(), error, attempts, nextAttemptAt);
2228
+ next_attempt_at = excluded.next_attempt_at,
2229
+ -- #1525: overwritten, like the four above. The row describes the LATEST
2230
+ -- attempt, so a retry drained by a different call (or by none) must not
2231
+ -- keep claiming the call that made the first one.
2232
+ invocation_id = excluded.invocation_id`, eventId, deliveryId, new Date().toISOString(), error, attempts, nextAttemptAt,
2233
+ // #1525: the call this attempt ran in, as the coordinator named it.
2234
+ invocationId);
1678
2235
  return attempts;
1679
2236
  }
1680
2237
  /** How many attempts this delivery has already had — the backoff input. */
@@ -1732,8 +2289,18 @@ export function defineScopeDO(modules, bareOps) {
1732
2289
  * the event on the next drain). Backpressure throws BEFORE any write — the caller
1733
2290
  * records a failed attempt and the delivery retries on its own backoff, exactly as
1734
2291
  * a throwing handler would.
2292
+ *
2293
+ * `invocationId` is LAST, and defaulted, for the RPC-skew reason `recordExecutorAttempt`
2294
+ * states at length.
2295
+ */
2296
+ routeExecutorEventToPlatform(eventId, deliveryId, kind, payload, requestedBy,
2297
+ /**
2298
+ * #1525: the call this routing ran in, or null. On the hosted path this is the
2299
+ * dominant executor journal — every vertical is CP-less, so a connector delivery
2300
+ * becomes an intent here rather than running in `recordExecutorAttempt`'s caller —
2301
+ * so leaving it null would make the column read "no call" for the common case.
1735
2302
  */
1736
- routeExecutorEventToPlatform(eventId, deliveryId, kind, payload, requestedBy) {
2303
+ invocationId = null) {
1737
2304
  const pending = Number(this.sql
1738
2305
  .exec(`SELECT COUNT(*) AS c FROM _substrat_platform_requests WHERE status = 'pending'`)
1739
2306
  .toArray()[0].c);
@@ -1753,7 +2320,7 @@ export function defineScopeDO(modules, bareOps) {
1753
2320
  this.sql.exec(`INSERT INTO _substrat_platform_requests
1754
2321
  (id, kind, payload, requested_by, impersonation, status, attempts, requested_at)
1755
2322
  VALUES (?, ?, ?, ?, ?, 'pending', 0, ?)`, id, kind, payload, requestedBy, source?.impersonation ?? null, instant.parse(new Date().toISOString()));
1756
- this.recordExecutorAttempt(eventId, deliveryId, null, null);
2323
+ this.recordExecutorAttempt(eventId, deliveryId, null, null, invocationId);
1757
2324
  return id;
1758
2325
  }
1759
2326
  /**
@@ -1947,6 +2514,11 @@ export function defineScopeDO(modules, bareOps) {
1947
2514
  // consumer dead-letter.
1948
2515
  'ALTER TABLE _substrat_deliveries ADD COLUMN attempts INTEGER NOT NULL DEFAULT 0',
1949
2516
  'ALTER TABLE _substrat_deliveries ADD COLUMN next_attempt_at TEXT',
2517
+ // #1525: the invocation an attempt ran in, on a scope DO created before the
2518
+ // column. Nullable, and the null is honestly "no call was carried" — which
2519
+ // attempt produced a delivery already journalled cannot be decided afterwards,
2520
+ // exactly as #1237's outbox column argued.
2521
+ 'ALTER TABLE _substrat_deliveries ADD COLUMN invocation_id TEXT',
1950
2522
  // K-34: the authorization column on a scope DO created before it existed. Nullable,
1951
2523
  // so legacy outbox rows read as "unrecorded". (_substrat_denials is a new table,
1952
2524
  // covered by KERNEL_DDL's IF NOT EXISTS with no ALTER.)
@@ -1962,6 +2534,10 @@ export function defineScopeDO(modules, bareOps) {
1962
2534
  'ALTER TABLE _substrat_outbox ADD COLUMN impersonation TEXT',
1963
2535
  'ALTER TABLE _substrat_platform_requests ADD COLUMN impersonation TEXT',
1964
2536
  'ALTER TABLE _substrat_denials ADD COLUMN impersonation TEXT',
2537
+ // #1525: the invocation a refusal happened during, on a scope DO created before
2538
+ // the column. Nullable, and the null is honestly "no id was carried" — a past
2539
+ // denial's call cannot be decided afterwards, as #1237's outbox column argued.
2540
+ 'ALTER TABLE _substrat_denials ADD COLUMN invocation_id TEXT',
1965
2541
  // #1231: the emitting operation, on a scope DO created before the column.
1966
2542
  // Nullable so every legacy row reads as unrecorded rather than named.
1967
2543
  'ALTER TABLE _substrat_outbox ADD COLUMN operation TEXT',
@@ -1997,6 +2573,33 @@ export function defineScopeDO(modules, bareOps) {
1997
2573
  // boot. `lint:spine-ddl` compares KERNEL_DDL's indexes only, so this one is held to
1998
2574
  // both adapters by the query-plan test rather than by that gate.
1999
2575
  this.sql.exec('CREATE INDEX IF NOT EXISTS _substrat_outbox_invocation ON _substrat_outbox (invocation_id, id)');
2576
+ this.ensureScheduleStateKind();
2577
+ }
2578
+ /**
2579
+ * #1288: `_substrat_schedule_state`, rebuilt with `kind` in its key on a scope DO
2580
+ * whose table predates the column. Not in the ALTER list above, because `kind`
2581
+ * joins the PRIMARY KEY and no ALTER can widen a key — the statements are the
2582
+ * kernel's, so the pure adapter rebuilds byte-identically.
2583
+ *
2584
+ * Detected from `sqlite_master.sql`: DO SQLite restricts `PRAGMA`, and reading
2585
+ * the stored DDL is the one probe both adapters can make.
2586
+ */
2587
+ ensureScheduleStateKind() {
2588
+ const row = this.sql
2589
+ .exec(`SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?`, '_substrat_schedule_state')
2590
+ .toArray()[0];
2591
+ if (!row || scheduleStateHasKind(row.sql))
2592
+ return;
2593
+ // In a transaction, for the reason the kernel constant spells out: create-copy-
2594
+ // drop-rename has two intermediate states and both are unrecoverable on the next
2595
+ // wake. `transactionSync`, not the async one every operation uses — the DO
2596
+ // runtime forbids a manual BEGIN through `sql.exec`, and this body is wholly
2597
+ // synchronous, which is the one case the sync API is for (it commits at the
2598
+ // first await, and there is none). It also has to be sync because the caller is.
2599
+ this.ctx.storage.transactionSync(() => {
2600
+ for (const stmt of splitSqlStatements(SCHEDULE_STATE_REBUILD))
2601
+ this.sql.exec(stmt);
2602
+ });
2000
2603
  }
2001
2604
  async importDump(tables, destScopeId) {
2002
2605
  // The WHOLE drop-then-replay runs under deferred foreign keys, in one transaction.
@@ -2196,7 +2799,13 @@ export function defineScopeDO(modules, bareOps) {
2196
2799
  * later alarm-driven drain with a call it had nothing to do with.
2197
2800
  */
2198
2801
  invocationId = null;
2199
- async dispatch(tenantId, scopeId) {
2802
+ async dispatch(tenantId, scopeId,
2803
+ /**
2804
+ * #1525: the call this drain is running in, or null. Every round of the loop
2805
+ * belongs to it: a consumer's own emit is delivered in the same tail, so the
2806
+ * whole cascade is one call's work.
2807
+ */
2808
+ invocationId) {
2200
2809
  for (let round = 0; round < 50; round++) {
2201
2810
  let deliveredAny = false;
2202
2811
  for (const mod of this.modules.values()) {
@@ -2222,16 +2831,18 @@ export function defineScopeDO(modules, bareOps) {
2222
2831
  system: mod.id,
2223
2832
  });
2224
2833
  await consumer.handler(ctx, event);
2225
- this.sql.exec(`INSERT INTO _substrat_deliveries (event_id, consumer_module, delivered_at)
2226
- VALUES (?, ?, ?)`, event.id, mod.id, new Date().toISOString());
2834
+ this.sql.exec(`INSERT INTO _substrat_deliveries
2835
+ (event_id, consumer_module, delivered_at, invocation_id)
2836
+ VALUES (?, ?, ?, ?)`, event.id, mod.id, new Date().toISOString(), invocationId);
2227
2837
  });
2228
2838
  deliveredAny = true;
2229
2839
  }
2230
2840
  catch (err) {
2231
2841
  // Dead-letter (v0): journal the failure so one poison event
2232
2842
  // can't wedge the loop. Written outside the rolled-back txn.
2233
- this.sql.exec(`INSERT INTO _substrat_deliveries (event_id, consumer_module, delivered_at, error)
2234
- VALUES (?, ?, ?, ?)`, event.id, mod.id, new Date().toISOString(), String(err));
2843
+ this.sql.exec(`INSERT INTO _substrat_deliveries
2844
+ (event_id, consumer_module, delivered_at, error, invocation_id)
2845
+ VALUES (?, ?, ?, ?, ?)`, event.id, mod.id, new Date().toISOString(), String(err), invocationId);
2235
2846
  }
2236
2847
  finally {
2237
2848
  // Cleared on BOTH paths. Left set, the id leaks onto every later emit
@@ -2253,7 +2864,28 @@ export function defineScopeDO(modules, bareOps) {
2253
2864
  * survives — the whole point, since the denial is the write the operation could not make.
2254
2865
  */
2255
2866
  /** K-42: the session a refused call ran under travels with the denial row. */
2256
- recordDenial(subject, tenantId, operation, err, impersonation) {
2867
+ recordDenial(subject, tenantId, operation, err,
2868
+ /**
2869
+ * #1525: the invocation this refusal belongs to, or null — PASSED, never read off
2870
+ * `this.invocationId` here.
2871
+ *
2872
+ * Reading the ambient field is only self-evidently right where one call holds the
2873
+ * DO to itself, and two denial paths do not: `attachmentList` and
2874
+ * `attachmentAuthorize` run OUTSIDE `this.queue`, and both await before they
2875
+ * record (`ensureMigrations`, `ctx.check`). Whether the input gate can actually
2876
+ * reopen far enough for one of them to observe an in-flight call's id is NOT
2877
+ * settled here — a probe that raced twelve attachment refusals against invokes
2878
+ * holding an id (including one awaiting the control plane) recorded null every
2879
+ * time, so the gate evidently holds more than the shape of the code promises.
2880
+ *
2881
+ * Passed anyway, because the argument for the ambient read is an argument about
2882
+ * workerd's gate semantics, and the argument for a parameter is local: the invoke
2883
+ * path passes its own id, every attachment path passes null, and each says what it
2884
+ * actually knows. That is the property worth having on a recorded fact, and it
2885
+ * costs one argument. No test accompanies it — the condition could not be
2886
+ * reproduced, and a test that passes either way would be worse than none.
2887
+ */
2888
+ invocationId, impersonation) {
2257
2889
  // Only an ENFORCED denial (assertAllowed, which attaches the checked permission +
2258
2890
  // node) is recorded. A module's own hand-thrown `new PermissionDenied('…')` carries
2259
2891
  // no permission key and is left to the module.
@@ -2265,8 +2897,11 @@ export function defineScopeDO(modules, bareOps) {
2265
2897
  ? { connection: subject.id }
2266
2898
  : subject.id;
2267
2899
  this.sql.exec(`INSERT INTO _substrat_denials
2268
- (id, actor, permission, tenant_id, scope_id, operation, impersonation, at)
2269
- VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, ulid(), JSON.stringify(actor), err.permission, err.node.tenantId, err.node.scopeId ?? null, operation, impersonation ? JSON.stringify(impersonationStampOf(impersonation)) : null, new Date().toISOString());
2900
+ (id, actor, permission, tenant_id, scope_id, operation, impersonation,
2901
+ invocation_id, at)
2902
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, ulid(), JSON.stringify(actor), err.permission, err.node.tenantId, err.node.scopeId ?? null, operation, impersonation ? JSON.stringify(impersonationStampOf(impersonation)) : null,
2903
+ // #1525: the call this refusal belongs to, as the caller named it.
2904
+ invocationId, new Date().toISOString());
2270
2905
  }
2271
2906
  parseOutboxRow(row) {
2272
2907
  return domainEvent.parse({