@substrat-run/adapter-cloudflare 0.114.0 → 0.117.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/scope-do.js CHANGED
@@ -1,9 +1,10 @@
1
1
  import { DurableObject } from 'cloudflare:workers';
2
- import { ATTACHMENT_ADDED, ATTACHMENT_REMOVED, attachmentRecord, domainEvent, domainEventInput, eventId, instant, objectRef, toWireFailure, grantRefFromProof, principalId, platformRequestInput, platformRequestId, MAX_PENDING_PLATFORM_REQUESTS, MAX_PENDING_SWEEP_RUNS, SWEEP_RUNS_KIND, platformRequest, SCOPE_TABLE_PAGE_MAX, SCOPE_QUERY_ROW_MAX, listLimitOf, requestFingerprint, substratError, assertReplayableDump, REDRAIN_BATCH, } from '@substrat-run/contracts';
3
- import { ulid, createUlid, assertAllowed, ConnectionSealingKeyUnavailableError, noSealingKeyMessage, sealTo, assertReadOnlyQuery, entitlementDenial, platformRequestHistoryQuery, PLATFORM_REQUEST_COLUMNS, denialListQuery, denialSummaryQuery, denialTotalsQuery, DENIAL_WINDOW_QUERY, mapDenialRow, mapDenialSummaryBuckets, PermissionDenied, assertImpersonationWrites, assertModuleEnqueueableKind, impersonationStampOf, createAtomic, NotSearchable, isSearchIndexTable, searchIndexDdl, searchIndexMigrations, searchIndexPlans, NotListable, listIndexMigrations, listIndexPlans, listQuery, cursorOf, searchLimit, searchMatchExpression, searchQuery, IDEMPOTENCY_DDL, assertIdempotencyKey, idempotencyLookupQuery, idempotencyPruneStatement, idempotencyRecordStatement, idempotencyOptedOutMessage, replayFor, entityVersionQuery, entityVersionOf, assertIfMatch, OUTBOX_ENTITY_INDEX, } from '@substrat-run/kernel';
2
+ import { ATTACHMENT_ADDED, ATTACHMENT_REMOVED, attachmentRecord, domainEvent, domainEventInput, eventId, instant, objectRef, toWireFailure, grantRefFromProof, principalId, scopeId as scopeIdOf, tenantId as tenantIdOf, platformRequestInput, platformRequestId, MAX_PENDING_PLATFORM_REQUESTS, MAX_PENDING_SWEEP_RUNS, SWEEP_RUNS_KIND, SCOPE_TABLE_PAGE_MAX, SCOPE_QUERY_ROW_MAX, listLimitOf, requestFingerprint, substratError, assertReplayableDump, REDRAIN_BATCH, } from '@substrat-run/contracts';
3
+ import { ulid, createUlid, assertAllowed, ConnectionSealingKeyUnavailableError, noSealingKeyMessage, sealTo, assertReadOnlyQuery, entitlementDenial, platformRequestHistoryQuery, platformRequestOf, PLATFORM_REQUEST_COLUMNS, PLATFORM_REQUEST_REDACTION_SQL, platformRequestRedactionParams, platformRequestRedactionQuery, intentPayloadCarriesSubject, seatScopeTuple, effectiveRoleGrantQuery, switchSystemSchedules, systemScheduleState, systemSwitchedOff, denialListQuery, denialSummaryQuery, denialTotalsQuery, DENIAL_WINDOW_QUERY, mapDenialRow, mapDenialSummaryBuckets, PermissionDenied, assertImpersonationWrites, assertModuleEnqueueableKind, impersonationStampOf, createAtomic, NotSearchable, isSearchIndexTable, searchIndexDdl, searchIndexMigrations, searchIndexPlans, NotListable, listIndexMigrations, listIndexPlans, listQuery, cursorOf, searchLimit, searchMatchExpression, searchQuery, IDEMPOTENCY_DDL, assertIdempotencyKey, assertPermissionKey, idempotencyLookupQuery, idempotencyPruneStatement, idempotencyRecordStatement, idempotencyOptedOutMessage, replayFor, entityVersionQuery, entityVersionOf, assertIfMatch, OUTBOX_ENTITY_INDEX, SCHEDULE_STATE_DDL, SCHEDULE_STATE_REBUILD, scheduleStateHasKind, JOB_RUN_DDL, jobRunListLimit, } from '@substrat-run/kernel';
4
+ import { isUpgradeRequest, readSubscription, LIVE_FANOUT_LIMIT, LIVE_MODE_HEADER, LIVE_PRINCIPAL_HEADER, LIVE_SCOPE_HEADER, LIVE_SUBSCRIBE_PATH, LIVE_TENANT_HEADER, } from './live-reads.js';
4
5
  import { OperationQueue } from './serialization.js';
5
- import { doScopedSql } from './sql.js';
6
- import { facetEvents, readDeadLetters, readHistory, readInvocation, walkEventCause, walkEventEffects } from '@substrat-run/kernel';
6
+ import { doScopedSql, doSpineSql } from './sql.js';
7
+ import { actorOf, assertNoSecret, CAPABILITY_DDL, CAPABILITY_EXCHANGE_OPERATION, createCapabilityVerbs, exchangeCapability, guardSecrets, mintBecomeCapability, redactSecrets, resolveCapabilitySession, revokeCapabilityAsPlatform, domainEventOf, facetEvents, readDeadLetters, readHistory, readInvocation, readUndrainedOutbox, walkEventCause, walkEventEffects, } from '@substrat-run/kernel';
7
8
  import { createDoTupleChecker, createLocalControlPlaneReader } from './checker.js';
8
9
  /**
9
10
  * The key marking a scope DO whose storage was destroyed (`destroyStorage`).
@@ -127,31 +128,28 @@ const KERNEL_DDL = `
127
128
  operation TEXT,
128
129
  -- K-42: WHICH of the two actors was refused is exactly what this log is for.
129
130
  impersonation TEXT,
131
+ -- #1525: the INVOCATION this refusal happened during, the same id #1237 stamps on
132
+ -- every event of a call. A denial joins to nothing but actor and time otherwise --
133
+ -- the operation column names what was attempted, never WHICH attempt -- so "what
134
+ -- else did this request do" cannot reach a refusal, which is the one thing an
135
+ -- incident asks about first. NULL = the transport carried no id (a seed, a test,
136
+ -- an internal call, an attachment RPC), or the row predates the column.
137
+ invocation_id TEXT,
130
138
  at TEXT NOT NULL,
131
139
  drained_at TEXT
132
140
  );
133
- -- #383: the platform sweep's per-scope gating state. Despite the name, this table
134
- -- holds TWO kinds of row since #1232, told apart by the shape of schedule_op:
135
- -- * a schedule operation, spelled module/verb -- last_run_at / last_status are
136
- -- when that operation last RAN here and how it ended;
137
- -- * a freshness key, spelled freshness:<eventType> -- last_run_at / last_status
138
- -- are when the evaluator last RECORDED a verdict for that event type and what
139
- -- the verdict was. Nothing ran; the row gates what the sweep records.
140
- -- Only HALF the no-collision claim is enforced, which is worth knowing before
141
- -- trusting it: a freshness key is always "freshness:" followed by a value the
142
- -- contracts eventType schema accepted (lowercase ns.verb, no colon and no slash),
143
- -- so no freshness key can ever look like an operation. The other direction is
144
- -- CONVENTION only -- scheduleSpec.operation is z.string().min(1), so a module
145
- -- that declared a schedule operation literally named "freshness:orders.placed"
146
- -- would share a row with the evaluator and nothing today would reject it.
147
- -- #1288 tracks giving the table a column (or a name) that says this outright,
148
- -- which is also what would let the collision be refused rather than avoided.
149
- -- Spine (kernel-written), never a module migration.
150
- CREATE TABLE IF NOT EXISTS _substrat_schedule_state (
151
- schedule_op TEXT PRIMARY KEY,
152
- last_run_at TEXT,
153
- last_status TEXT
154
- );
141
+ -- #383 / #1232 / #1288: the platform sweep's per-scope gating state, holding two
142
+ -- families of row that the kind COLUMN -- not the spelling of a key -- tells
143
+ -- apart. Spine (kernel-written), never a module migration. Shared with the pure
144
+ -- adapter from @substrat-run/kernel, so the shape a rebuild produces and the shape
145
+ -- a fresh store gets cannot part company; the column comments are in there.
146
+ ${SCHEDULE_STATE_DDL}
147
+ -- #1577: the resumable-run driver's record and the step ledger of the pass it
148
+ -- currently has in flight. Spine (kernel-written), never a module migration.
149
+ -- Shared with the pure adapter from @substrat-run/kernel so the shape production
150
+ -- builds and the shape a self-host builds cannot part company; the column
151
+ -- comments, and the reason coalescing is NOT a unique index, are in there.
152
+ ${JOB_RUN_DDL}
155
153
  CREATE TABLE IF NOT EXISTS _substrat_migrations (
156
154
  module_id TEXT NOT NULL,
157
155
  version TEXT NOT NULL,
@@ -191,6 +189,17 @@ const KERNEL_DDL = `
191
189
  -- next_attempt_at IS NULL -> terminal: error IS NULL delivered, else dead
192
190
  attempts INTEGER NOT NULL DEFAULT 0,
193
191
  next_attempt_at TEXT,
192
+ -- #1525: the invocation THIS ATTEMPT ran in, the same id #1237 stamps on every
193
+ -- event of a call. Not the emitting event's -- that one is already on the outbox
194
+ -- and a delivery joins to it through event_id. What that join cannot say is
195
+ -- which call attempted the DELIVERY, and for an executor the two differ by
196
+ -- design: attempt one runs in the emitting call's post-commit tail, every retry
197
+ -- afterwards in a drain that is a different call or none. Moves with the row on
198
+ -- an upsert, like delivered_at, error and attempts -- it describes the
199
+ -- latest attempt, not the first. NULL = that attempt carried no call (a
200
+ -- scheduled drain, an alarm, a seed, an attachment RPC), or the row predates
201
+ -- the column.
202
+ invocation_id TEXT,
194
203
  PRIMARY KEY (event_id, consumer_module)
195
204
  );
196
205
  -- Scope-local permissions (docs/architecture/scope-local-permissions.md): the
@@ -301,6 +310,10 @@ const KERNEL_DDL = `
301
310
  ${OUTBOX_ENTITY_INDEX}
302
311
  -- #116: the request-dedupe table, kernel-owned so no vertical migrates for it.
303
312
  ${IDEMPOTENCY_DDL}
313
+ -- #1672: capabilities — authority carried by a secret (a link share), and the sessions
314
+ -- an exchange trades that secret for. Shared with the pure adapter from
315
+ -- @substrat-run/kernel so the two cannot part company; the column comments are there.
316
+ ${CAPABILITY_DDL}
304
317
  `;
305
318
  /**
306
319
  * Workers RPC carries a plain `Error`'s MESSAGE faithfully and nothing else. A custom
@@ -359,27 +372,6 @@ function attachSubject(principal, connectionId) {
359
372
  function isSystemTable(name) {
360
373
  return name.startsWith('_substrat') || name.startsWith('sqlite_');
361
374
  }
362
- /**
363
- * A stored row → the `PlatformRequest` contract shape (JSON columns parsed). The coordinator
364
- * maps the RPC's raw rows the same way in `host.ts`; this copy exists because `ctx.platformRequests`
365
- * (#618) answers INSIDE the DO, where the row never crosses an RPC boundary at all.
366
- */
367
- function rowToPlatformRequest(r) {
368
- return platformRequest.parse({
369
- id: r.id,
370
- kind: r.kind,
371
- payload: JSON.parse(r.payload),
372
- requestedBy: JSON.parse(r.requested_by),
373
- impersonation: r.impersonation == null ? null : JSON.parse(r.impersonation),
374
- status: r.status,
375
- attempts: r.attempts,
376
- lastError: r.last_error,
377
- failure: r.last_failure == null ? null : JSON.parse(r.last_failure),
378
- result: r.result === null ? null : JSON.parse(r.result),
379
- requestedAt: r.requested_at,
380
- settledAt: r.settled_at,
381
- });
382
- }
383
375
  /** SQLite cell → a JSON-safe value: bigints stringify, blobs (ArrayBuffer) read as null. */
384
376
  function cellToJson(v) {
385
377
  if (v == null)
@@ -505,6 +497,12 @@ export function defineScopeDO(modules, bareOps) {
505
497
  listPlans = new Map();
506
498
  /** entityType → the declared attachment gate (#473): read key + write key (default: read). */
507
499
  attachmentTargets = new Map();
500
+ /**
501
+ * entityType → the declared live-read gate (#938): the key a subscriber must hold
502
+ * ON THAT ENTITY before a change to it is announced. Absent = announced to nobody;
503
+ * `fanOutLive` treats a miss as silence, never as "unguarded".
504
+ */
505
+ liveTargets = new Map();
508
506
  checker;
509
507
  systemPrincipal = principalId.parse(ulid());
510
508
  /**
@@ -622,6 +620,20 @@ export function defineScopeDO(modules, bareOps) {
622
620
  }
623
621
  this.attachmentTargets.set(target.entityType, gate);
624
622
  }
623
+ // Live-read targets (#938): entityType → the read key a subscriber must pass ON
624
+ // THAT ENTITY before a change to it is announced. Validated exactly like the
625
+ // attachment gate above, and for a sharper reason: two modules disagreeing about
626
+ // which key guards an entity type would make "who may watch this" depend on
627
+ // registration order, and the losing declaration would be the one that was meant
628
+ // to be stricter. An entity type nobody declares is announced to nobody.
629
+ for (const target of manifest.liveTargets ?? []) {
630
+ const existing = this.liveTargets.get(target.entityType);
631
+ if (existing && existing !== target.readPermission) {
632
+ throw new Error(`conflicting liveTargets for '${target.entityType}': ` +
633
+ `(${existing}) vs (${target.readPermission})`);
634
+ }
635
+ this.liveTargets.set(target.entityType, target.readPermission);
636
+ }
625
637
  for (const name of manifest.withdraws ?? []) {
626
638
  this.withdrawn.set(name, manifest.id);
627
639
  this.operations.delete(name);
@@ -680,9 +692,11 @@ export function defineScopeDO(modules, bareOps) {
680
692
  * and skips the write when nothing changed.
681
693
  *
682
694
  * `ensureMigrations` memoises its promise, so every later call on a warm DO
683
- * resolves to the SAME `true` without applying anything. Reporting on each of
684
- * those would bill a control-plane RPC per stub mint to store a number that
685
- * has not moved — hence the once-per-instance latch.
695
+ * resolves to the SAME `true` without applying anything — until something
696
+ * clears the memo (`retryMigrations`, `importDump`), which is a fresh pass and
697
+ * may apply. Reporting on each of the cached ones would bill a control-plane
698
+ * RPC per stub mint to store a number that has not moved — hence the
699
+ * once-per-instance latch.
686
700
  */
687
701
  async migrate() {
688
702
  const applied = await this.ensureMigrations();
@@ -738,6 +752,14 @@ export function defineScopeDO(modules, bareOps) {
738
752
  migrationFailure() {
739
753
  return this.lastFailure ? { ...this.lastFailure, applied: this.applied.size } : null;
740
754
  }
755
+ /**
756
+ * This scope's database size in bytes (#1524): `SqlStorage.databaseSize`, which Cloudflare
757
+ * bills on. Its one caller is an on-demand storage reading, never a sweep, because reaching
758
+ * it wakes this DO.
759
+ */
760
+ databaseSize() {
761
+ return this.sql.databaseSize;
762
+ }
741
763
  /**
742
764
  * The PITR bookmarks this scope recorded before migration passes (#286),
743
765
  * newest first — what a backout UI offers as rewind points. Rows taken after
@@ -762,26 +784,26 @@ export function defineScopeDO(modules, bareOps) {
762
784
  }));
763
785
  }
764
786
  /**
765
- * The events not yet shipped to Tier 2 (#1334), oldest first. `ORDER BY id`
766
- * is chronological (ULID) and stable, so a drain resumes where it stopped.
787
+ * The events not yet shipped to Tier 2 (#1334), oldest first — and what the read
788
+ * stepped over (#1636). `ORDER BY id` is chronological (ULID) and stable, so a drain
789
+ * resumes where it stopped.
790
+ *
791
+ * The kernel's read, shared with the pure adapter: a row that will not decode is
792
+ * neither returned nor stamped, and the rows behind it still come back. An object
793
+ * rather than an array because it crosses the RPC — a property on an array would not.
794
+ */
795
+ undrainedEventsRead(limit) {
796
+ return readUndrainedOutbox((offset, count) => this.sql
797
+ .exec(`SELECT * FROM _substrat_outbox WHERE drained_at IS NULL ORDER BY id LIMIT ? OFFSET ?`, count, offset)
798
+ .toArray(), limit);
799
+ }
800
+ /**
801
+ * The same read as a bare array, for a coordinator deployed before
802
+ * `undrainedEventsRead` (#1636) — kept so that pairing still drains, and still steps
803
+ * over a bad row rather than stalling on it. It just cannot say that it did.
767
804
  */
768
805
  undrainedEvents(limit) {
769
- const rows = this.sql
770
- .exec(`SELECT * FROM _substrat_outbox WHERE drained_at IS NULL ORDER BY id LIMIT ?`, limit)
771
- .toArray();
772
- return rows.map((r) => ({
773
- ...this.parseOutboxRow(r),
774
- operation: r.operation ?? null,
775
- version: r.version ?? null,
776
- // #1237 — lifted like the two above, and for the same reason: the column
777
- // exists on the outbox but not on the envelope `parseOutboxRow` returns,
778
- // whose `domainEvent.parse` strips anything it does not declare.
779
- causedBy: r.caused_by ?? null,
780
- // …and the invocation, for the same reason again: a lake that kept cause and
781
- // dropped the call could say what set an event off and never which request did
782
- // it, which is the grouping a trace is built on.
783
- invocationId: r.invocation_id ?? null,
784
- }));
806
+ return this.undrainedEventsRead(limit).events;
785
807
  }
786
808
  /**
787
809
  * Stamp `drained_at` on shipped events (#1334). Idempotent — a re-mark is a no-op,
@@ -843,6 +865,24 @@ export function defineScopeDO(modules, bareOps) {
843
865
  return ids.length;
844
866
  });
845
867
  }
868
+ /**
869
+ * How many rows `redrainEvents` WOULD reopen for the same instant (#1545), reopening
870
+ * none of them. A separate verb rather than a flag on the one above, deliberately: the
871
+ * two answers are indistinguishable once they are numbers, so the caller that asks for
872
+ * a count must not be able to reach the reopen by losing an argument on the way.
873
+ *
874
+ * UNBOUNDED where the reopen is batched. The batch exists because an UPDATE over the
875
+ * whole window rewrites every row and its index entries inside ONE Durable Object
876
+ * request, against a fixed budget. An aggregate materialises no rows, so that budget is
877
+ * not the binding constraint — and a partial count would be worse than useless: the
878
+ * number is the whole point, and "5000, or possibly more" answers nothing.
879
+ */
880
+ async redrainCount(drainedBefore) {
881
+ return this.sql
882
+ .exec(`SELECT COUNT(*) AS c FROM _substrat_outbox
883
+ WHERE drained_at IS NOT NULL AND drained_at < ?`, drainedBefore)
884
+ .toArray()[0].c;
885
+ }
846
886
  /**
847
887
  * Facet this scope's own outbox (#1239) — `facetEvents`, which is the
848
888
  * sanctioned read: an erased payload yields the same NULL a missing field
@@ -960,13 +1000,30 @@ export function defineScopeDO(modules, bareOps) {
960
1000
  return rows.sort((a, b) => a.subject.localeCompare(b.subject) || a.relation.localeCompare(b.relation));
961
1001
  });
962
1002
  }
963
- /** Admin scope-tuple write (role assignment / grant scoped to this scope). */
1003
+ /**
1004
+ * Admin scope-tuple write (role assignment / grant scoped to this scope) — the
1005
+ * EXPLICIT grant. `INSERT OR REPLACE`, so it clears a tombstone: a re-grant grants.
1006
+ * Provisioning does not come through here; it seats with `seatTuple` (#1659).
1007
+ */
964
1008
  async writeTuple(subject, relation, object, expiresAt) {
965
1009
  await this.queue.enqueue(() => {
966
1010
  this.sql.exec(`INSERT OR REPLACE INTO _substrat_tuples (subject, relation, object, expires_at)
967
1011
  VALUES (?, ?, ?, ?)`, subject, relation, object, expiresAt);
968
1012
  });
969
1013
  }
1014
+ /**
1015
+ * Provisioning's scope-tuple write (#1659): create the row if it is missing, follow the
1016
+ * platform's expiry if it is live, and leave it alone if it was revoked — so a re-run
1017
+ * provision cannot undo an operator's revoke — and seat nothing at all for a module
1018
+ * whose schedule kill switch is off (#1666). `seatScopeTuple` is the statement, shared
1019
+ * with `applyProjection`'s `scopeTuples` and with the pure adapter.
1020
+ */
1021
+ async seatTuple(subject, relation, object, expiresAt) {
1022
+ await this.queue.enqueue(() => {
1023
+ const seat = seatScopeTuple(subject, relation, object, expiresAt);
1024
+ this.sql.exec(seat.sql, ...seat.params);
1025
+ });
1026
+ }
970
1027
  /**
971
1028
  * The dispatch capability's admission (#726 remedy B): this delivery may read the
972
1029
  * attachments of the entity its own spine row names, and no others.
@@ -1056,19 +1113,28 @@ export function defineScopeDO(modules, bareOps) {
1056
1113
  * this scope; the DO uses it for the two-actor stamp and for the read-only
1057
1114
  * bound, and nothing here can be reached without it having been minted.
1058
1115
  */
1059
- impersonation) {
1116
+ impersonation,
1117
+ /**
1118
+ * #1672: the HASH of a capability session token — the coordinator hashed the token
1119
+ * and the plaintext never crosses. Resolved to its capability INSIDE the queued body,
1120
+ * on every call, so a revoke between two calls refuses the second. `principal` is
1121
+ * then a random placeholder that holds nothing, and the reply carries
1122
+ * `capability.honoured`: an old DO that ignored this argument would run the call as
1123
+ * that placeholder, and the coordinator refuses a success without the acknowledgement.
1124
+ */
1125
+ capabilitySession) {
1060
1126
  if (!failureEnvelope) {
1061
1127
  // Legacy path, byte-for-byte what it was: rewrapped so a non-plain error (a
1062
1128
  // ZodError, whose `message` is a getter) still arrives with its message.
1063
1129
  try {
1064
- return await this.invokeOrThrow(operation, input, principal, tenantId, scopeId, connectionId, requiredEntitlement, systemModuleId, invokeOptions, impersonation);
1130
+ return await this.invokeOrThrow(operation, input, principal, tenantId, scopeId, connectionId, requiredEntitlement, systemModuleId, invokeOptions, impersonation, capabilitySession);
1065
1131
  }
1066
1132
  catch (err) {
1067
1133
  throw toRpcError(err);
1068
1134
  }
1069
1135
  }
1070
1136
  try {
1071
- return await this.invokeOrThrow(operation, input, principal, tenantId, scopeId, connectionId, requiredEntitlement, systemModuleId, invokeOptions, impersonation);
1137
+ return await this.invokeOrThrow(operation, input, principal, tenantId, scopeId, connectionId, requiredEntitlement, systemModuleId, invokeOptions, impersonation, capabilitySession);
1072
1138
  }
1073
1139
  catch (err) {
1074
1140
  // The ONE place the error keeps its structure: flattened here, rebuilt by the
@@ -1080,7 +1146,9 @@ export function defineScopeDO(modules, bareOps) {
1080
1146
  /** The operation path itself. Throws; `invoke` decides how that reaches the caller. */
1081
1147
  async invokeOrThrow(operation, input, principal, tenantId, scopeId, connectionId, requiredEntitlement, systemModuleId, invokeOptions,
1082
1148
  /** K-42: the session, resolved coordinator-side. See `invoke` above. */
1083
- impersonation) {
1149
+ impersonation,
1150
+ /** #1672: a capability session's hash. See `invoke` above. */
1151
+ capabilitySession) {
1084
1152
  await this.ensureMigrations();
1085
1153
  const handler = this.operations.get(operation);
1086
1154
  // `not_found`, not a bare throw (#113): every vertical hand-matched this message
@@ -1139,7 +1207,7 @@ export function defineScopeDO(modules, bareOps) {
1139
1207
  // The subject a key is scoped to — the same three-way read `recordDenial`
1140
1208
  // makes below, hoisted because both need it. A key belongs to whoever sent
1141
1209
  // it: two principals choosing `1` must not reach each other's response.
1142
- const idempotencySubjectRef = systemModuleId
1210
+ let idempotencySubjectRef = systemModuleId
1143
1211
  ? { kind: 'system', id: systemModuleId }
1144
1212
  : connectionId
1145
1213
  ? { kind: 'connection', id: connectionId }
@@ -1163,6 +1231,34 @@ export function defineScopeDO(modules, bareOps) {
1163
1231
  // one call. Same placement as the SQLite adapter's actor task, for this reason.
1164
1232
  this.invocationId = invokeOptions?.invocationId ?? null;
1165
1233
  try {
1234
+ // #1672: the capability session, resolved on EVERY call and here — inside the queued
1235
+ // body, the one region where this call holds the DO to itself — so nothing can
1236
+ // revoke between this read and the transaction. Refuses a stale session, a revoked
1237
+ // or expired capability and an operation off its allowlist before anything opens;
1238
+ // none of those is a K-35 denial (no key was checked).
1239
+ let capabilityId;
1240
+ if (capabilitySession !== undefined) {
1241
+ capabilityId = resolveCapabilitySession(doSpineSql(this.sql), capabilitySession, instant.parse(new Date().toISOString()), operation);
1242
+ idempotencySubjectRef = { kind: 'capability', id: capabilityId };
1243
+ }
1244
+ // #1672: the secrets this call mints — withheld from its idempotency recording, and
1245
+ // what the tripwire on its writes looks for.
1246
+ const minted = [];
1247
+ /**
1248
+ * #938: the outbox's high-water mark BEFORE this call wrote anything, so the
1249
+ * post-commit fan-out can name exactly the events this call (and the consumers
1250
+ * it set off) added. Read here — inside the queued body, before the
1251
+ * transaction — because that is the region where this call holds the DO to
1252
+ * itself, the same reason `invocationId` is set here.
1253
+ *
1254
+ * A socket that connects between here and the fan-out is served whatever this
1255
+ * call committed, and one that connects while `null` was decided hears nothing
1256
+ * about it. Both are harmless and neither is worth a lock: a frame is an
1257
+ * invalidation, so hearing about a change from just before you subscribed costs
1258
+ * one redundant re-read, and missing one costs a wait for the client's poll —
1259
+ * which is the floor this whole surface sits on.
1260
+ */
1261
+ const liveSince = this.liveHighWaterMark();
1166
1262
  let result;
1167
1263
  let committedVersion = null;
1168
1264
  // #116: set when this invocation was answered from a recording rather
@@ -1190,7 +1286,7 @@ export function defineScopeDO(modules, bareOps) {
1190
1286
  // emitted events back as one — verified across `await` in workerd.
1191
1287
  try {
1192
1288
  await this.ctx.storage.transaction(async () => {
1193
- const ctx = this.operationContext(principal, tenantId, scopeId, undefined, connectionId, systemModuleId, signals, impersonation, operation);
1289
+ const ctx = this.operationContext(principal, tenantId, scopeId, undefined, connectionId, systemModuleId, signals, impersonation, operation, capabilityId, minted);
1194
1290
  // #116: a retry is answered from the recording, and nothing else runs
1195
1291
  // — not the guards, not the handler, not the permission check inside
1196
1292
  // it. Keyed by SUBJECT, so a caller only ever reaches its own
@@ -1231,7 +1327,9 @@ export function defineScopeDO(modules, bareOps) {
1231
1327
  // it describes. The prune rides along, on the only path that adds a row.
1232
1328
  if (idempotencyKey !== undefined && fingerprint !== undefined) {
1233
1329
  const at = new Date().toISOString();
1234
- const record = idempotencyRecordStatement(idempotencySubjectRef, idempotencyKey, operation, fingerprint, result, committedVersion, at);
1330
+ const record = idempotencyRecordStatement(idempotencySubjectRef, idempotencyKey, operation, fingerprint,
1331
+ // #1672: a replay of a mint returns the placeholder, never the secret.
1332
+ redactSecrets(result, minted), committedVersion, at);
1235
1333
  this.sql.exec(record.sql, ...record.params);
1236
1334
  const prune = idempotencyPruneStatement(at);
1237
1335
  this.sql.exec(prune.sql, ...prune.params);
@@ -1254,6 +1352,7 @@ export function defineScopeDO(modules, bareOps) {
1254
1352
  result,
1255
1353
  platformRequests: 0,
1256
1354
  impersonation: { honoured: true },
1355
+ ...(capabilitySession !== undefined ? { capability: { honoured: true } } : {}),
1257
1356
  ...(idempotencyKey !== undefined
1258
1357
  ? { idempotency: { keyHonoured: true, replayed } }
1259
1358
  : {}),
@@ -1270,11 +1369,12 @@ export function defineScopeDO(modules, bareOps) {
1270
1369
  // K-35: the transaction has rolled back; record a refused check now, as its own
1271
1370
  // write (outside that transaction), so the denial survives the rollback.
1272
1371
  if (err instanceof PermissionDenied) {
1273
- this.recordDenial(systemModuleId
1274
- ? { kind: 'system', id: systemModuleId }
1275
- : connectionId
1276
- ? { kind: 'connection', id: connectionId }
1277
- : { kind: 'principal', id: principal }, tenantId, operation, err, impersonation);
1372
+ this.recordDenial(
1373
+ // The subject the call acted as — the capability's, once resolved above.
1374
+ idempotencySubjectRef, tenantId, operation, err,
1375
+ // Inside the queued body, which is the one region where this call holds
1376
+ // the DO to itself — so the field is this call's own (#1237).
1377
+ this.invocationId, impersonation);
1278
1378
  }
1279
1379
  // The ORIGINAL error, deliberately: `invoke` flattens it for the envelope
1280
1380
  // (which keeps its code and extensions) or rewraps it for the legacy throw
@@ -1285,12 +1385,20 @@ export function defineScopeDO(modules, bareOps) {
1285
1385
  // Skipped on a replay: nothing was written, so there is nothing this
1286
1386
  // invocation added to drain. Anything the ORIGINAL left undrained is the
1287
1387
  // outbox's own retry backstop, which is what that backstop is for.
1388
+ // Drain to consumers, then announce what landed (#938). Skipped whole on a
1389
+ // replay: nothing was written, so there is nothing this invocation added to
1390
+ // drain or to announce. Anything the ORIGINAL left undrained is the outbox's
1391
+ // own retry backstop, which is what that backstop is for.
1392
+ // #1525: still inside the queued body that set it, so these deliveries are this
1393
+ // call's own work — the same tail its consumers' emits are stamped in.
1288
1394
  if (!replayed)
1289
- await this.dispatch(tenantId, scopeId);
1395
+ await this.settleCommitted(tenantId, scopeId, liveSince, this.invocationId);
1290
1396
  return {
1291
1397
  result,
1292
1398
  platformRequests: signals.platformRequests,
1293
1399
  ...(impersonation ? { impersonation: { honoured: true } } : {}),
1400
+ // #1672: the acknowledgement the coordinator's skew check reads — see `invoke`.
1401
+ ...(capabilitySession !== undefined ? { capability: { honoured: true } } : {}),
1294
1402
  // The acknowledgement the coordinator's skew check reads (#116), on the
1295
1403
  // same reasoning as `ifMatchChecked` below and with a sharper failure: a
1296
1404
  // DO too old to know about keys would EXECUTE THE OPERATION AGAIN and
@@ -1319,6 +1427,288 @@ export function defineScopeDO(modules, bareOps) {
1319
1427
  }
1320
1428
  });
1321
1429
  }
1430
+ // -- live reads (#938): the subscription half of the change feed ------------
1431
+ // The ONLY part of this DO addressed as a fetch target rather than over RPC, and
1432
+ // only because a WebSocket cannot cross RPC — a socket is not serializable, so the
1433
+ // one way to hand one back is a `Response` carrying a `webSocket`. Everything the
1434
+ // coordinator asserts on the way in is named in `live-reads.ts`, shared with
1435
+ // `host.ts` so the two ends cannot drift.
1436
+ /**
1437
+ * Accept a subscription to this scope's changes.
1438
+ *
1439
+ * The coordinator has already decided that this connection can carry a push at all
1440
+ * (the O2O check) and WHO is asking. What is decided here is nothing about
1441
+ * authority: a subscription is not an authorization, and accepting one grants the
1442
+ * subscriber no read it did not already have. Every frame is checked on its way
1443
+ * out, individually, against the tuple state at that moment — so a grant revoked
1444
+ * while the socket is open stops the frames it used to allow, which a
1445
+ * subscription-time check would not.
1446
+ *
1447
+ * **This adds no authority to a holder of the stub, and the question is worth
1448
+ * answering rather than leaving to be asked.** Being a public method, anything with
1449
+ * the `SCOPE` binding can call it and assert whatever principal it likes in the
1450
+ * headers. That is already true of `invoke`, which takes the principal as an
1451
+ * argument and acts on it: a stub is the key to the scope, which is precisely why
1452
+ * the router is not given one. The trust boundary is who holds the binding, not
1453
+ * what this method checks — and what it does NOT do is let a stub-holder read
1454
+ * anything the asserted principal could not, because the filter downstream re-checks
1455
+ * that principal against every frame.
1456
+ */
1457
+ async fetch(request) {
1458
+ const url = new URL(request.url);
1459
+ if (url.pathname !== LIVE_SUBSCRIBE_PATH) {
1460
+ // The DO has exactly one fetch surface. Anything else reaching here is a
1461
+ // coordinator bug, and a 404 says so without guessing at an intent.
1462
+ return new Response('this scope has no such surface', { status: 404 });
1463
+ }
1464
+ if (!isUpgradeRequest(request)) {
1465
+ return new Response('live reads are a WebSocket surface', {
1466
+ status: 426,
1467
+ headers: { [LIVE_MODE_HEADER]: 'not-an-upgrade' },
1468
+ });
1469
+ }
1470
+ const principal = request.headers.get(LIVE_PRINCIPAL_HEADER);
1471
+ const tenantId = request.headers.get(LIVE_TENANT_HEADER);
1472
+ const scopeId = request.headers.get(LIVE_SCOPE_HEADER);
1473
+ if (!principal || !tenantId || !scopeId) {
1474
+ // Fail closed and loudly. An unnamed subscriber is one whose permissions
1475
+ // cannot be evaluated, and the only safe thing to do with a channel we cannot
1476
+ // filter is to refuse to open it. 500, not 400: the caller is the coordinator
1477
+ // in this same package, so a missing assertion is OUR bug, not the client's.
1478
+ return new Response('live reads require an asserted principal, tenant and scope', {
1479
+ status: 500,
1480
+ });
1481
+ }
1482
+ // A subscriber arriving before the scope's migrations have run would be told
1483
+ // about events against a schema it cannot read back through. Same gate every
1484
+ // other entry point takes, for the same reason.
1485
+ await this.ensureMigrations();
1486
+ const pair = new WebSocketPair();
1487
+ const [client, server] = Object.values(pair);
1488
+ // HIBERNATABLE, not `server.accept()`. A scope with a watcher open would
1489
+ // otherwise be pinned in memory for as long as somebody has a tab open, which is
1490
+ // the cost model inverted: a support desk being WATCHED is the normal state.
1491
+ // Hibernation also fixes the worse half — an in-memory roster does not survive
1492
+ // eviction, so the socket would stay open and silently stop receiving, which is
1493
+ // indistinguishable from a quiet scope.
1494
+ this.ctx.acceptWebSocket(server);
1495
+ // `.parse`, not a cast: these three arrived as header strings, and the branded
1496
+ // ids are what every check downstream is keyed on. A malformed one would
1497
+ // otherwise be carried all the way to a `ctx.check` that quietly matches nothing —
1498
+ // which reads as "this subscriber may see nothing" and is indistinguishable from
1499
+ // a correct denial. Refused here instead, where it is still one subscriber's
1500
+ // problem. Throwing is right: the coordinator built this request.
1501
+ server.serializeAttachment({
1502
+ principal: principalId.parse(principal),
1503
+ tenantId: tenantIdOf.parse(tenantId),
1504
+ scopeId: scopeIdOf.parse(scopeId),
1505
+ since: new Date().toISOString(),
1506
+ });
1507
+ return new Response(null, { status: 101, webSocket: client });
1508
+ }
1509
+ /**
1510
+ * A subscriber said something.
1511
+ *
1512
+ * The channel is one-way by design — the server announces, the client re-reads
1513
+ * through the ordinary operation — so there is no client message that can cause a
1514
+ * read, a write, or a change of subscription. `ping`/`pong` is the whole protocol,
1515
+ * and it exists so an idle connection can be kept alive by either end.
1516
+ *
1517
+ * Deliberately NOT a place to let a client narrow or widen what it receives: a
1518
+ * filter the client chooses is a filter the client can choose wrongly, and the
1519
+ * only filter that matters here is the one it does not control.
1520
+ */
1521
+ webSocketMessage(ws, message) {
1522
+ if (message === 'ping')
1523
+ ws.send('pong');
1524
+ }
1525
+ /**
1526
+ * The client hung up. Close our end so the runtime stops holding the subscription.
1527
+ *
1528
+ * The runtime passes `reason` and `wasClean` too; neither is read, so neither is
1529
+ * named — there is nothing to do differently for an unclean close, because the
1530
+ * socket is going away either way and the client's poll is what covers the gap.
1531
+ */
1532
+ webSocketClose(ws, code) {
1533
+ // 1006 is reserved: it is what the runtime REPORTS for an abnormal closure and
1534
+ // is not a code anything may SEND, so echoing it back throws — on precisely the
1535
+ // path where the connection is already in trouble.
1536
+ try {
1537
+ ws.close(code === 1006 ? 1000 : code, 'scope closing the subscription');
1538
+ }
1539
+ catch {
1540
+ // Already gone. Nothing to do, and nothing worth reporting.
1541
+ }
1542
+ }
1543
+ /**
1544
+ * A socket failed. Logged and not rethrown: there is no caller to fail, and the
1545
+ * runtime delivers `webSocketClose` after this, which is what does the cleanup.
1546
+ */
1547
+ webSocketError(_ws, error) {
1548
+ console.error('substrat: live-read socket error', error);
1549
+ }
1550
+ /**
1551
+ * The outbox's high-water mark before a committing path runs (#938), or `null`
1552
+ * when nobody is listening.
1553
+ *
1554
+ * `null` and `''` are different answers: `null` means no subscriber, so nothing is
1555
+ * read at all and a scope pays nothing for a feature it is not using; `''` is the
1556
+ * honest empty-outbox answer, and every ULID sorts above it.
1557
+ *
1558
+ * Read BEFORE the transaction by every caller, which is what makes the pair below
1559
+ * able to name exactly the events that path added.
1560
+ */
1561
+ liveHighWaterMark() {
1562
+ if (this.ctx.getWebSockets().length === 0)
1563
+ return null;
1564
+ return (this.sql.exec('SELECT MAX(id) AS id FROM _substrat_outbox').toArray()[0]?.id ?? '');
1565
+ }
1566
+ /**
1567
+ * What every committing path does after its transaction closes: drain the outbox
1568
+ * to consumers, then announce what landed to whoever may see it.
1569
+ *
1570
+ * **One step, because there is one rule.** The fan-out was wired into `invoke`
1571
+ * alone at first, and `attachmentAdd`/`attachmentRemove` commit and emit too —
1572
+ * `attachment.added` and `attachment.removed`, about a real entity. A watcher of
1573
+ * that entity would have missed them: no error, no gap it could see, just a screen
1574
+ * that did not update for one kind of change. That is the exact failure the
1575
+ * hibernation design exists to prevent, arriving through a different door, so the
1576
+ * answer is a step both doors take rather than a second call both must remember.
1577
+ *
1578
+ * Order is load-bearing: drain FIRST, announce after. A consumer's own emits are
1579
+ * changes too, and they land in the outbox above `liveSince` — so announcing first
1580
+ * would tell a subscriber about the cause and not the effect, and it would re-read
1581
+ * too early.
1582
+ */
1583
+ async settleCommitted(tenantId, scopeId, liveSince,
1584
+ /**
1585
+ * #1525: the call whose commit this is settling, or null — PASSED rather than
1586
+ * read off `this.invocationId` inside `dispatch`, so each of the three doors
1587
+ * says what it actually knows: `invoke` its own id, the two attachment verbs
1588
+ * null. Same discipline `recordDenial` adopted, and for the same reason.
1589
+ */
1590
+ invocationId) {
1591
+ await this.dispatch(tenantId, scopeId, invocationId);
1592
+ if (liveSince === null)
1593
+ return;
1594
+ try {
1595
+ await this.fanOutLive(liveSince, tenantId, scopeId);
1596
+ }
1597
+ catch (err) {
1598
+ // The write has COMMITTED and the caller is owed its answer. A failure to
1599
+ // announce is a failure of a hint, and the client's poll is the floor
1600
+ // underneath it — so this is logged and never rethrown. Rethrowing would turn
1601
+ // a delivered write into a 500 the caller would reasonably retry.
1602
+ console.error('substrat: live-read fan-out failed after commit', err);
1603
+ }
1604
+ }
1605
+ /**
1606
+ * Announce what just committed, to whoever may see it (#938).
1607
+ *
1608
+ * **Three properties, and each is load-bearing.**
1609
+ *
1610
+ * *Post-commit.* Called after the operation's transaction has closed and after the
1611
+ * consumer drain, so nothing is announced that a rollback could take back and
1612
+ * nothing a consumer emitted is missed. A subscriber told about a row that then
1613
+ * vanished would re-read, find nothing, and have no way to tell that from a
1614
+ * deletion.
1615
+ *
1616
+ * *Filtered per subscriber, per event, after the check.* The declared
1617
+ * `liveTargets` key is checked ON THE EVENT'S OWN ENTITY, through `ctx.check` —
1618
+ * the same evaluator, the same entity-narrowed grants, the same parent walk as the
1619
+ * read the client is about to make. An entity type no module declared is announced
1620
+ * to nobody. Knowing that a row exists and changed at 14:02 is information about
1621
+ * that row, so the empty payload is not what makes this safe; this is.
1622
+ *
1623
+ * *Never able to fail the operation.* The write has committed and the caller has
1624
+ * its answer. A socket that has gone away mid-fan-out, or a check that cannot be
1625
+ * evaluated, costs a subscriber its live update — which it survives, because the
1626
+ * client's contract is that a push is a hint and the poll is the floor.
1627
+ */
1628
+ async fanOutLive(sinceEventId, tenantId, scopeId) {
1629
+ const sockets = this.ctx.getWebSockets();
1630
+ if (sockets.length === 0)
1631
+ return;
1632
+ const rows = this.sql
1633
+ .exec(`SELECT id, type, entity_type, entity_id, occurred_at FROM _substrat_outbox
1634
+ WHERE id > ? ORDER BY id LIMIT ?`, sinceEventId, LIVE_FANOUT_LIMIT)
1635
+ .toArray();
1636
+ // Drop the undeclared entity types BEFORE any per-subscriber work. Not an
1637
+ // optimisation: it is the fail-closed rule stated once, in the one place that
1638
+ // decides, rather than relied upon inside the loop below.
1639
+ const announceable = rows.filter((r) => this.liveTargets.has(r.entity_type));
1640
+ if (announceable.length === 0)
1641
+ return;
1642
+ for (const ws of sockets) {
1643
+ let subscription = null;
1644
+ try {
1645
+ subscription = readSubscription(ws.deserializeAttachment());
1646
+ }
1647
+ catch {
1648
+ subscription = null;
1649
+ }
1650
+ // A socket we cannot name is a socket we cannot filter for. Skipped, never
1651
+ // sent to — see `readSubscription` for why every unusable shape fails closed.
1652
+ if (!subscription)
1653
+ continue;
1654
+ // A socket that outlived a scope rebind, or was somehow accepted for another
1655
+ // node, must not be fed this scope's events. Cheap, and it makes the identity
1656
+ // the frames are filtered against an explicit precondition rather than an
1657
+ // assumption about how the subscription was created.
1658
+ if (subscription.tenantId !== tenantId || subscription.scopeId !== scopeId)
1659
+ continue;
1660
+ // One context per subscriber, not per event: `ctx.check` is the expensive part
1661
+ // and the context is only the subject it is evaluated for.
1662
+ //
1663
+ // A frame this subscriber does not pass is NOT recorded as a denial (K-35),
1664
+ // and that is deliberate: `recordDenial` is called on a refused REQUEST, where
1665
+ // somebody asked for something and was told no. Nobody asked for these. Logging
1666
+ // one row per unentitled subscriber per event would bury the denials that mean
1667
+ // something — a broken screen, or somebody walking the surface — under the
1668
+ // ordinary, correct working of a filter.
1669
+ //
1670
+ // The operation name is carried anyway, for the events a fan-out cannot emit
1671
+ // but a future reader of this context might.
1672
+ const ctx = this.operationContext(subscription.principal, tenantId, scopeId, undefined, undefined, undefined, undefined, undefined, 'live.subscribe');
1673
+ for (const row of announceable) {
1674
+ // Non-null: `announceable` is exactly the rows whose type is in the map.
1675
+ const permission = this.liveTargets.get(row.entity_type);
1676
+ let allowed = false;
1677
+ try {
1678
+ const decision = await ctx.check(permission, {
1679
+ entityType: row.entity_type,
1680
+ entityId: row.entity_id,
1681
+ });
1682
+ allowed = decision.allowed;
1683
+ }
1684
+ catch {
1685
+ // A check that cannot answer is a check that refuses. The alternative —
1686
+ // treating an evaluator failure as an allow — turns an outage in the
1687
+ // permission path into a disclosure, which is the one failure mode this
1688
+ // surface must not have.
1689
+ allowed = false;
1690
+ }
1691
+ if (!allowed)
1692
+ continue;
1693
+ const frame = {
1694
+ kind: 'change',
1695
+ id: row.id,
1696
+ type: row.type,
1697
+ entityType: row.entity_type,
1698
+ entityId: row.entity_id,
1699
+ at: row.occurred_at,
1700
+ };
1701
+ try {
1702
+ ws.send(JSON.stringify(frame));
1703
+ }
1704
+ catch {
1705
+ // The socket went away between `getWebSockets()` and here. Stop writing to
1706
+ // this one and move on; the runtime will deliver `webSocketClose`.
1707
+ break;
1708
+ }
1709
+ }
1710
+ }
1711
+ }
1322
1712
  // -- attachments (#473): the metadata half of the attachment surface --------
1323
1713
  // The coordinator (worker) holds the bytes and the per-tenant R2 binding; this
1324
1714
  // DO holds the permission gate and the metadata fact, under the same per-scope
@@ -1359,6 +1749,10 @@ export function defineScopeDO(modules, bareOps) {
1359
1749
  const parsed = attachmentRecord.parse(record);
1360
1750
  const gate = this.attachmentGate(parsed.entity.entityType);
1361
1751
  return this.queue.enqueue(async () => {
1752
+ // #938: this path commits and emits too, so it takes the same mark-then-settle
1753
+ // pair `invoke` takes. Inside the queued body and before the transaction, for
1754
+ // the reason it is read there: that is where this call holds the DO to itself.
1755
+ const liveSince = this.liveHighWaterMark();
1362
1756
  try {
1363
1757
  await this.ctx.storage.transaction(async () => {
1364
1758
  const ctx = this.operationContext(principal, tenantId, scopeId, undefined, connectionId, undefined, undefined, undefined, 'attachments.upload');
@@ -1378,11 +1772,13 @@ export function defineScopeDO(modules, bareOps) {
1378
1772
  }
1379
1773
  catch (err) {
1380
1774
  if (err instanceof PermissionDenied) {
1381
- this.recordDenial(attachSubject(principal, connectionId), tenantId, 'attachments.upload', err);
1775
+ this.recordDenial(attachSubject(principal, connectionId), tenantId, 'attachments.upload', err, null);
1382
1776
  }
1383
1777
  throw toRpcError(err);
1384
1778
  }
1385
- await this.dispatch(tenantId, scopeId);
1779
+ // #1525: null. An attachment RPC carries no invocation, exactly as its denial
1780
+ // records none — so the deliveries it drains name no call rather than a wrong one.
1781
+ await this.settleCommitted(tenantId, scopeId, liveSince, null);
1386
1782
  return parsed;
1387
1783
  });
1388
1784
  }
@@ -1396,7 +1792,7 @@ export function defineScopeDO(modules, bareOps) {
1396
1792
  }
1397
1793
  catch (err) {
1398
1794
  if (err instanceof PermissionDenied) {
1399
- this.recordDenial(attachSubject(principal, connectionId), tenantId, 'attachments.list', err);
1795
+ this.recordDenial(attachSubject(principal, connectionId), tenantId, 'attachments.list', err, null);
1400
1796
  }
1401
1797
  throw toRpcError(err);
1402
1798
  }
@@ -1427,7 +1823,7 @@ export function defineScopeDO(modules, bareOps) {
1427
1823
  }
1428
1824
  catch (err) {
1429
1825
  if (err instanceof PermissionDenied) {
1430
- this.recordDenial(attachSubject(principal, connectionId), tenantId, 'attachments.open', err);
1826
+ this.recordDenial(attachSubject(principal, connectionId), tenantId, 'attachments.open', err, null);
1431
1827
  }
1432
1828
  throw toRpcError(err);
1433
1829
  }
@@ -1440,7 +1836,7 @@ export function defineScopeDO(modules, bareOps) {
1440
1836
  }
1441
1837
  catch (err) {
1442
1838
  if (err instanceof PermissionDenied) {
1443
- this.recordDenial(attachSubject(principal, connectionId), tenantId, 'attachments.open', err);
1839
+ this.recordDenial(attachSubject(principal, connectionId), tenantId, 'attachments.open', err, null);
1444
1840
  }
1445
1841
  throw toRpcError(err);
1446
1842
  }
@@ -1454,6 +1850,8 @@ export function defineScopeDO(modules, bareOps) {
1454
1850
  if (!record)
1455
1851
  return null;
1456
1852
  const gate = this.attachmentGate(record.entity.entityType);
1853
+ // #938: same mark-then-settle pair as the upload path above.
1854
+ const liveSince = this.liveHighWaterMark();
1457
1855
  try {
1458
1856
  await this.ctx.storage.transaction(async () => {
1459
1857
  const ctx = this.operationContext(principal, tenantId, scopeId, undefined, connectionId, undefined, undefined, undefined, 'attachments.remove');
@@ -1470,32 +1868,128 @@ export function defineScopeDO(modules, bareOps) {
1470
1868
  }
1471
1869
  catch (err) {
1472
1870
  if (err instanceof PermissionDenied) {
1473
- this.recordDenial(attachSubject(principal, connectionId), tenantId, 'attachments.remove', err);
1871
+ this.recordDenial(attachSubject(principal, connectionId), tenantId, 'attachments.remove', err, null);
1474
1872
  }
1475
1873
  throw toRpcError(err);
1476
1874
  }
1477
- await this.dispatch(tenantId, scopeId);
1875
+ // #1525: null. An attachment RPC carries no invocation, exactly as its denial
1876
+ // records none — so the deliveries it drains name no call rather than a wrong one.
1877
+ await this.settleCommitted(tenantId, scopeId, liveSince, null);
1478
1878
  return record;
1479
1879
  });
1480
1880
  }
1881
+ /** The kernel's schedule-switch SQL (#1666), over this DO's storage. */
1882
+ switchSql() {
1883
+ return {
1884
+ all: (sql, ...params) => this.sql.exec(sql, ...params).toArray(),
1885
+ run: (sql, ...params) => {
1886
+ this.sql.exec(sql, ...params);
1887
+ },
1888
+ };
1889
+ }
1890
+ /**
1891
+ * Where a module's schedules stand on this scope (#383, #1666) — the kernel's
1892
+ * `systemScheduleState`, the predicate the pure adapter runs too: `on` with a live
1893
+ * `system:<moduleId>` grant, `off` while the kill switch's marker is live, whatever
1894
+ * else is, and `ungranted` on a scope that never ran the module (a foreign vertical's,
1895
+ * which the sweep skips quietly).
1896
+ */
1897
+ async systemScheduleState(moduleId) {
1898
+ return systemScheduleState(this.switchSql(), moduleId, new Date().toISOString());
1899
+ }
1481
1900
  /**
1482
- * Whether this scope holds a live `system:<moduleId>` grant (#383) — the switch
1483
- * that decides if a module's schedules run here at all. Absent on a foreign
1484
- * vertical's scope, or after a per-tenant revoke, so the sweep skips it quietly.
1901
+ * The pre-#1666 read, kept for ONE reason: a coordinator a deploy behind this DO still
1902
+ * calls it. It answers the new question, not the old one — the old predicate counted any
1903
+ * live `system:` tuple, and the kill switch's OFF marker is one, so an old coordinator
1904
+ * asking the old question would run a switched-off scope's schedules.
1485
1905
  */
1486
1906
  async hasSystemGrant(moduleId) {
1487
- const now = new Date().toISOString();
1488
- const row = this.sql
1489
- .exec(`SELECT 1 FROM _substrat_tuples
1490
- WHERE subject = ? AND revoked_at IS NULL AND (expires_at IS NULL OR expires_at > ?)
1491
- LIMIT 1`, `system:${moduleId}`, now)
1492
- .toArray()[0];
1493
- return row !== undefined;
1907
+ return (await this.systemScheduleState(moduleId)) === 'on';
1908
+ }
1909
+ /**
1910
+ * `grantToSystem`'s scope-level write (#1666): the explicit grant — `INSERT OR REPLACE`,
1911
+ * as `writeTuple` — EXCEPT while the module's schedule kill switch is off, when it writes
1912
+ * nothing and answers `false`. The check and the write are one queued unit, so no switch
1913
+ * can move between them. Restore is the lever; a grant is not.
1914
+ */
1915
+ async writeSystemGrant(moduleId, relation, object, expiresAt) {
1916
+ return this.queue.enqueue(() => {
1917
+ if (systemSwitchedOff(this.switchSql(), moduleId))
1918
+ return false;
1919
+ this.sql.exec(`INSERT OR REPLACE INTO _substrat_tuples (subject, relation, object, expires_at)
1920
+ VALUES (?, ?, ?, ?)`, `system:${moduleId}`, relation, object, expiresAt);
1921
+ return true;
1922
+ });
1494
1923
  }
1495
- /** The last time a schedule's operation ran on this scope (#383), or null. */
1924
+ /**
1925
+ * Move a module's schedule switch on this scope (#1666) — the kernel's
1926
+ * `switchSystemSchedules`, serialized on the queue with every other tuple write and run
1927
+ * as one `transactionSync`, so its reads and writes are one unit.
1928
+ */
1929
+ async switchSystemSchedules(moduleId, scopeId, to, at) {
1930
+ return this.queue.enqueue(() => this.ctx.storage.transactionSync(() => switchSystemSchedules(this.switchSql(), { moduleId, scopeId, to, at })));
1931
+ }
1932
+ /**
1933
+ * The exchange (#1672) — a capability's secret traded for a session, or for the
1934
+ * principal a `become` capability yields. Runs the kernel's `exchangeCapability`, the
1935
+ * function the pure adapter runs, in this DO: one queued body and one storage
1936
+ * transaction, so the use it takes and the `capability.exercised` event recording it
1937
+ * commit together or not at all; the event's consumers then settle as an invoke's do.
1938
+ *
1939
+ * The secret itself reaches this DO — it must, to be hashed and looked up — and goes
1940
+ * no further: nothing here stores or returns it.
1941
+ */
1942
+ async exchangeCapability(secret, tenantId, scopeId, mode) {
1943
+ await this.ensureMigrations();
1944
+ return await this.queue.enqueue(async () => {
1945
+ const liveSince = this.liveHighWaterMark();
1946
+ // ONE instant for the whole exchange: the row's `last_used_at`, the session's times
1947
+ // and the event's `occurredAt` are one fact, and two clock reads could disagree.
1948
+ const now = instant.parse(new Date().toISOString());
1949
+ let outcome = null;
1950
+ await this.ctx.storage.transaction(async () => {
1951
+ outcome = await exchangeCapability({
1952
+ sql: doSpineSql(this.sql),
1953
+ now,
1954
+ // Stamped `{ capability }` by the ordinary emit path, under the exchange's own
1955
+ // pseudo-operation name — the actor is the capability, as on every event it
1956
+ // goes on to cause.
1957
+ emit: (capability, event) => this.operationContext(principalId.parse(ulid()), tenantId, scopeId, undefined, undefined, undefined, undefined, undefined, CAPABILITY_EXCHANGE_OPERATION, capability, [], now).emit(event),
1958
+ }, secret, mode);
1959
+ });
1960
+ if (outcome)
1961
+ await this.settleCommitted(tenantId, scopeId, liveSince, null);
1962
+ return outcome;
1963
+ });
1964
+ }
1965
+ /**
1966
+ * The platform's mint (#1672, `HostAdmin.mintCapability`) — a `become` capability,
1967
+ * serialized on the queue with every other spine write. One INSERT, so it needs no
1968
+ * transaction of its own; the hash is computed before it.
1969
+ */
1970
+ async mintBecomeCapability(input, actor) {
1971
+ await this.ensureMigrations();
1972
+ return await this.queue.enqueue(() => mintBecomeCapability(doSpineSql(this.sql), input, actor, instant.parse(new Date().toISOString())));
1973
+ }
1974
+ /**
1975
+ * The platform's revoke (#1672, `HostAdmin.revokeCapability`) — of any capability in
1976
+ * this scope. Returns the record as it stood before, for the admin log, or `null`.
1977
+ */
1978
+ async revokeCapabilityAsPlatform(id, actor) {
1979
+ await this.ensureMigrations();
1980
+ return await this.queue.enqueue(() => this.ctx.storage.transactionSync(() => revokeCapabilityAsPlatform(doSpineSql(this.sql), id, actor, instant.parse(new Date().toISOString()))) ?? null);
1981
+ }
1982
+ /**
1983
+ * The last time a schedule's operation ran on this scope (#383), or null.
1984
+ *
1985
+ * `kind = 'schedule'` is not decoration (#1288): an operation may legally be
1986
+ * named `freshness:<something>`, and before the column this read answered with
1987
+ * the EVALUATOR's last recorded time for that event type — a cadence gate
1988
+ * driven by a verdict nothing ran.
1989
+ */
1496
1990
  async scheduleLastRun(operation) {
1497
1991
  const row = this.sql
1498
- .exec(`SELECT last_run_at FROM _substrat_schedule_state WHERE schedule_op = ?`, operation)
1992
+ .exec(`SELECT last_run_at FROM _substrat_schedule_state WHERE kind = 'schedule' AND schedule_op = ?`, operation)
1499
1993
  .toArray()[0];
1500
1994
  return row?.last_run_at ?? null;
1501
1995
  }
@@ -1503,8 +1997,9 @@ export function defineScopeDO(modules, bareOps) {
1503
1997
  * #1232: everything the freshness evaluator needs about this scope, one round
1504
1998
  * trip regardless of how many types are declared — the newest matching event
1505
1999
  * per type, plus each type's recorded evaluator state (last recorded at +
1506
- * outcome, kept in `_substrat_schedule_state` under a `freshness:` prefix that
1507
- * cannot collide with operation names).
2000
+ * outcome, kept in `_substrat_schedule_state` under `kind = 'freshness'`, which
2001
+ * since #1288 is what separates these rows from the schedule rows beside them —
2002
+ * the `freshness:` prefix on the key is retained but no longer load-bearing).
1508
2003
  */
1509
2004
  async freshnessProbe(types) {
1510
2005
  const out = {};
@@ -1521,7 +2016,8 @@ export function defineScopeDO(modules, bareOps) {
1521
2016
  }
1522
2017
  const keys = types.map((t) => `freshness:${t}`);
1523
2018
  for (const row of this.sql
1524
- .exec(`SELECT schedule_op, last_run_at, last_status FROM _substrat_schedule_state WHERE schedule_op IN (${marks})`, ...keys)
2019
+ .exec(`SELECT schedule_op, last_run_at, last_status FROM _substrat_schedule_state
2020
+ WHERE kind = 'freshness' AND schedule_op IN (${marks})`, ...keys)
1525
2021
  .toArray()) {
1526
2022
  const t = row.schedule_op.slice('freshness:'.length);
1527
2023
  if (out[t]) {
@@ -1533,16 +2029,173 @@ export function defineScopeDO(modules, bareOps) {
1533
2029
  }
1534
2030
  /**
1535
2031
  * Write one `_substrat_schedule_state` row (#383). Spine, kernel-written.
1536
- * `unit` is the row's key in either of the two shapes the table holds: a schedule
2032
+ *
2033
+ * `kind` is PASSED, never derived from the shape of `unit` (#1288) — deriving it
2034
+ * is precisely the convention this column replaced, and it is wrong for the one
2035
+ * input that matters: a schedule operation named `freshness:orders.placed` is a
2036
+ * schedule row, whatever its key looks like. `unit` is that row's key: a schedule
1537
2037
  * operation (`module/verb`, and then `at`/`status` are when it ran and how it
1538
2038
  * ended) or a freshness key (`freshness:<eventType>`, and then they are when the
1539
2039
  * verdict was recorded and what it was — nothing ran). See the bootstrap DDL.
2040
+ *
2041
+ * `kind` is LAST because this is a positional RPC — see the interface's own note
2042
+ * in `host.ts`. An old DO drops a trailing argument; a leading one it binds, and
2043
+ * every value after it lands one column to the left, silently.
1540
2044
  */
1541
- async recordScheduleRun(unit, at, status) {
1542
- this.sql.exec(`INSERT INTO _substrat_schedule_state (schedule_op, last_run_at, last_status)
1543
- VALUES (?, ?, ?)
1544
- ON CONFLICT(schedule_op) DO UPDATE SET last_run_at = excluded.last_run_at,
1545
- last_status = excluded.last_status`, unit, at, status);
2045
+ async recordScheduleRun(unit, at, status, kind) {
2046
+ this.sql.exec(`INSERT INTO _substrat_schedule_state (kind, schedule_op, last_run_at, last_status)
2047
+ VALUES (?, ?, ?, ?)
2048
+ ON CONFLICT(kind, schedule_op) DO UPDATE SET last_run_at = excluded.last_run_at,
2049
+ last_status = excluded.last_status`, kind, unit, at, status);
2050
+ }
2051
+ // -- the resumable-run driver's store (#1577) -----------------------------
2052
+ //
2053
+ // D-14's DURABLE driver, and the only half of it that lives in the DO: the run
2054
+ // record and the step ledger. Every decision — what coalescing means, when a
2055
+ // step is skipped, when a run fails — is in the kernel's `JobRunStore` callers,
2056
+ // which the coordinator drives; these are reads and writes and nothing else, so
2057
+ // the two drivers cannot disagree about any of it.
2058
+ //
2059
+ // Each step commits on its OWN round trip rather than the pass batching them at
2060
+ // the end. That is what a mid-pass eviction keeps, and a DO is evicted and
2061
+ // revived constantly — batching would lose exactly the work resume exists for.
2062
+ /**
2063
+ * Coalescing, as ONE round trip: the live (`running`) run for this key, or this
2064
+ * row inserted and returned.
2065
+ *
2066
+ * **The single RPC is the atomicity**, and that is the whole reason it is shaped
2067
+ * this way rather than as a `jobRunLive` the coordinator follows with a
2068
+ * `jobRunInsert`. A Durable Object serializes its RPCs, so the lookup and the
2069
+ * insert cannot be interleaved by another caller; split across two calls they
2070
+ * can, and two concurrent starts then both find nothing and both insert. The
2071
+ * schema carries no unique constraint to catch that, deliberately — a crashed
2072
+ * run must stay restartable — so the indivisibility has to come from here.
2073
+ */
2074
+ async jobRunStartOrJoin(moduleId, job, instance, row) {
2075
+ const live = this.sql
2076
+ .exec(`SELECT * FROM _substrat_job_runs
2077
+ WHERE module_id = ? AND job = ? AND instance = ? AND status = 'running'
2078
+ ORDER BY id DESC LIMIT 1`, moduleId, job, instance)
2079
+ .toArray()[0] ?? null;
2080
+ if (live)
2081
+ return live;
2082
+ // `transactionSync`, and the two writes inlined rather than reached through
2083
+ // `jobRunInsert`: an `await` between them is an output-gate boundary, and the
2084
+ // point of doing this in one RPC is that there is no boundary to be evicted
2085
+ // at. Same reason `SCHEDULE_STATE_REBUILD` insists on it.
2086
+ this.ctx.storage.transactionSync(() => {
2087
+ this.sql.exec(`INSERT INTO _substrat_job_runs
2088
+ (id, module_id, job, instance, payload, status, cursor, counters, attempts,
2089
+ last_error, started_at, updated_at, next_attempt_at, ended_at)
2090
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, row.id, row.module_id, row.job, row.instance, row.payload, row.status, row.cursor, row.counters, row.attempts, row.last_error, row.started_at, row.updated_at, row.next_attempt_at, row.ended_at);
2091
+ });
2092
+ return row;
2093
+ }
2094
+ /** One run by id, whatever its status. */
2095
+ async jobRunById(id) {
2096
+ return (this.sql.exec('SELECT * FROM _substrat_job_runs WHERE id = ?', id).toArray()[0] ?? null);
2097
+ }
2098
+ /** Insert a fresh run. The coordinator has already refused a non-queue-safe payload. */
2099
+ async jobRunInsert(row) {
2100
+ this.sql.exec(`INSERT INTO _substrat_job_runs
2101
+ (id, module_id, job, instance, payload, status, cursor, counters, attempts,
2102
+ last_error, started_at, updated_at, next_attempt_at, ended_at)
2103
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, row.id, row.module_id, row.job, row.instance, row.payload, row.status, row.cursor, row.counters, row.attempts, row.last_error, row.started_at, row.updated_at, row.next_attempt_at, row.ended_at);
2104
+ }
2105
+ /**
2106
+ * `running` runs whose backoff has elapsed, oldest first, after `afterId`.
2107
+ *
2108
+ * The cursor is what lets the coordinator page past runs it cannot drive: it
2109
+ * skips any whose job this deployment does not register, and without a cursor
2110
+ * those rows head every batch forever (`runDueJobRuns`). `afterId` is LAST, as
2111
+ * every argument added to an RPC on this interface must be.
2112
+ */
2113
+ async jobRunsDue(now, limit, afterId) {
2114
+ return this.sql
2115
+ .exec(
2116
+ // `afterId` bound TWICE rather than as `?2`: mixing anonymous and numbered
2117
+ // parameters makes the anonymous ones resume from the highest index used,
2118
+ // which is a footgun for the next person to add a clause. Spelled exactly
2119
+ // as the pure adapter spells it.
2120
+ `SELECT * FROM _substrat_job_runs
2121
+ WHERE status = 'running' AND (next_attempt_at IS NULL OR next_attempt_at <= ?)
2122
+ AND (? IS NULL OR id > ?)
2123
+ ORDER BY id LIMIT ?`, now, afterId ?? null, afterId ?? null, limit)
2124
+ .toArray();
2125
+ }
2126
+ /** The operator read, newest first. */
2127
+ async jobRunList(filter) {
2128
+ const where = [];
2129
+ const params = [];
2130
+ for (const [column, value] of [
2131
+ ['module_id', filter.moduleId],
2132
+ ['job', filter.job],
2133
+ ['instance', filter.instance],
2134
+ ['status', filter.status],
2135
+ ]) {
2136
+ if (value !== undefined) {
2137
+ where.push(`${column} = ?`);
2138
+ params.push(value);
2139
+ }
2140
+ }
2141
+ params.push(jobRunListLimit(filter.limit));
2142
+ return this.sql
2143
+ .exec(`SELECT * FROM _substrat_job_runs
2144
+ ${where.length ? `WHERE ${where.join(' AND ')}` : ''}
2145
+ ORDER BY id DESC LIMIT ?`, ...params)
2146
+ .toArray();
2147
+ }
2148
+ /**
2149
+ * Write a pass outcome onto the run row — every column at once.
2150
+ *
2151
+ * Whole rather than partial, and that is the shape not an accident: a commit
2152
+ * writes cursor AND counters AND a cleared error AND a cleared backoff, and a
2153
+ * partial update that wrote three of the four would leave a run carrying the
2154
+ * last failure's error beside the new cursor, which reads as broken forever.
2155
+ */
2156
+ async jobRunPatch(id, patch) {
2157
+ this.sql.exec(`UPDATE _substrat_job_runs
2158
+ SET status = ?, cursor = ?, counters = ?, attempts = ?, last_error = ?,
2159
+ updated_at = ?, next_attempt_at = ?, ended_at = ?
2160
+ WHERE id = ?`, patch.status, patch.cursor, patch.counters, patch.attempts, patch.lastError, patch.updatedAt, patch.nextAttemptAt, patch.endedAt, id);
2161
+ }
2162
+ /** One step's ledger row — a non-null `result` is what means completed. */
2163
+ async jobStepRow(runId, step) {
2164
+ return (this.sql
2165
+ .exec('SELECT step, result, attempts, last_error FROM _substrat_job_steps WHERE run_id = ? AND step = ?', runId, step)
2166
+ .toArray()[0] ?? null);
2167
+ }
2168
+ /** Record one step attempt. `result` non-null = it completed and must not re-run. */
2169
+ async jobStepRecord(runId, step, result, attempts, lastError, at) {
2170
+ this.sql.exec(`INSERT INTO _substrat_job_steps (run_id, step, result, attempts, last_error, recorded_at)
2171
+ VALUES (?, ?, ?, ?, ?, ?)
2172
+ ON CONFLICT (run_id, step) DO UPDATE SET result = excluded.result,
2173
+ attempts = excluded.attempts,
2174
+ last_error = excluded.last_error,
2175
+ recorded_at = excluded.recorded_at`, runId, step, result, attempts, lastError, at);
2176
+ }
2177
+ /**
2178
+ * A COMMITTED pass: the run's new state and the dropping of its step ledger, in
2179
+ * ONE RPC so they cannot come apart.
2180
+ *
2181
+ * Same reasoning as `jobRunStartOrJoin` — the single round trip is the
2182
+ * atomicity. As two calls, a stop in between leaves the advanced cursor beside
2183
+ * the finished pass's memo rows, and a handler that reuses a step name across
2184
+ * passes (legal: the determinism rule binds names to the payload and prior
2185
+ * results, not to the cursor) then skips work it never did. The kernel's
2186
+ * `runJobPass` carries the full argument.
2187
+ */
2188
+ async jobCommitPass(id, patch) {
2189
+ // `transactionSync` with both statements inline — NOT `await
2190
+ // this.jobRunPatch(...)` then the delete. The await is an output-gate
2191
+ // boundary, which is precisely the gap this method exists to close.
2192
+ this.ctx.storage.transactionSync(() => {
2193
+ this.sql.exec(`UPDATE _substrat_job_runs
2194
+ SET status = ?, cursor = ?, counters = ?, attempts = ?, last_error = ?,
2195
+ updated_at = ?, next_attempt_at = ?, ended_at = ?
2196
+ WHERE id = ?`, patch.status, patch.cursor, patch.counters, patch.attempts, patch.lastError, patch.updatedAt, patch.nextAttemptAt, patch.endedAt, id);
2197
+ this.sql.exec('DELETE FROM _substrat_job_steps WHERE run_id = ?', id);
2198
+ });
1546
2199
  }
1547
2200
  // -- guards (K-17) --------------------------------------------------------
1548
2201
  /** This scope's spine, read under whatever transaction the caller already opened. */
@@ -1644,8 +2297,15 @@ export function defineScopeDO(modules, bareOps) {
1644
2297
  *
1645
2298
  * "Not yet consumed" means never attempted, or retrying and now due (#100).
1646
2299
  * Terminal rows — delivered or dead-lettered — are excluded by the join.
2300
+ *
2301
+ * Decoded per row (#1636). A row that will not decode comes back in `undecodable`,
2302
+ * never as an event: the coordinator journals it as a dead letter, and its handler is
2303
+ * never handed an event built from stand-ins. The whole list used to be one
2304
+ * `rows.map(decode)`, so one bad row threw the executor's pending list on every pass.
2305
+ * The journal write stays on the coordinator, beside every other attempt it records —
2306
+ * this stays a read.
1647
2307
  */
1648
- pendingExecutorEvents(deliveryId, eventType) {
2308
+ pendingExecutorDeliveries(deliveryId, eventType) {
1649
2309
  const rows = this.sql
1650
2310
  .exec(`SELECT o.* FROM _substrat_outbox o
1651
2311
  LEFT JOIN _substrat_deliveries d
@@ -1655,7 +2315,26 @@ export function defineScopeDO(modules, bareOps) {
1655
2315
  OR (d.next_attempt_at IS NOT NULL AND d.next_attempt_at <= ?))
1656
2316
  ORDER BY o.id`, deliveryId, eventType, new Date().toISOString())
1657
2317
  .toArray();
1658
- return rows.map((r) => this.parseOutboxRow(r));
2318
+ const events = [];
2319
+ const undecodable = [];
2320
+ for (const r of rows) {
2321
+ try {
2322
+ events.push(domainEventOf(r));
2323
+ }
2324
+ catch (err) {
2325
+ undecodable.push({ eventId: r.id, error: String(err) });
2326
+ }
2327
+ }
2328
+ return { events, undecodable };
2329
+ }
2330
+ /**
2331
+ * The same read as a bare list of events, for a coordinator deployed before
2332
+ * `pendingExecutorDeliveries` (#1636). An undecodable row is left out rather than
2333
+ * thrown, so that pairing still delivers the rows behind it; the next coordinator
2334
+ * dead-letters it.
2335
+ */
2336
+ pendingExecutorEvents(deliveryId, eventType) {
2337
+ return this.pendingExecutorDeliveries(deliveryId, eventType).events;
1659
2338
  }
1660
2339
  /**
1661
2340
  * Journal one executor attempt (#100). `error` null means delivered;
@@ -1665,20 +2344,44 @@ export function defineScopeDO(modules, bareOps) {
1665
2344
  * Written AFTER the effect, so a crash mid-effect retries rather than silently
1666
2345
  * marking success. The coordinator computes the backoff because it owns the
1667
2346
  * per-executor policy; the DO owns the state.
2347
+ *
2348
+ * `invocationId` is LAST because this is a positional RPC — the same rule
2349
+ * `recordScheduleRun` states. An old DO drops a trailing argument; a leading one
2350
+ * it binds, and every value after it lands one column to the left, silently.
1668
2351
  */
1669
- recordExecutorAttempt(eventId, deliveryId, error, nextAttemptAt) {
2352
+ recordExecutorAttempt(eventId, deliveryId, error, nextAttemptAt,
2353
+ /**
2354
+ * #1525: the call THIS attempt ran in, or null. Passed rather than read off
2355
+ * `this.invocationId`, and here there is no choice about it: executors run on
2356
+ * the COORDINATOR, so by the time this RPC arrives the queued body that held
2357
+ * the id has long returned and the field reads null. The coordinator is the
2358
+ * only side that knows, which is why the SQLite twin passes it too — a recorded
2359
+ * fact whose correctness argument differs per adapter is the kind that drifts.
2360
+ *
2361
+ * Defaulted, for the other half of the skew: a coordinator too old to pass it
2362
+ * calls this with four arguments, and `undefined` is not a value SQLite binds.
2363
+ * Null is the honest answer there anyway — that coordinator recorded no call.
2364
+ */
2365
+ invocationId = null) {
1670
2366
  const prior = this.sql
1671
2367
  .exec('SELECT attempts FROM _substrat_deliveries WHERE event_id = ? AND consumer_module = ?', eventId, deliveryId)
1672
2368
  .toArray()[0];
1673
2369
  const attempts = (prior?.attempts ?? 0) + 1;
1674
2370
  this.sql.exec(`INSERT INTO _substrat_deliveries
1675
- (event_id, consumer_module, delivered_at, error, attempts, next_attempt_at)
1676
- VALUES (?, ?, ?, ?, ?, ?)
2371
+ (event_id, consumer_module, delivered_at, error, attempts, next_attempt_at,
2372
+ invocation_id)
2373
+ VALUES (?, ?, ?, ?, ?, ?, ?)
1677
2374
  ON CONFLICT (event_id, consumer_module) DO UPDATE SET
1678
2375
  delivered_at = excluded.delivered_at,
1679
2376
  error = excluded.error,
1680
2377
  attempts = excluded.attempts,
1681
- next_attempt_at = excluded.next_attempt_at`, eventId, deliveryId, new Date().toISOString(), error, attempts, nextAttemptAt);
2378
+ next_attempt_at = excluded.next_attempt_at,
2379
+ -- #1525: overwritten, like the four above. The row describes the LATEST
2380
+ -- attempt, so a retry drained by a different call (or by none) must not
2381
+ -- keep claiming the call that made the first one.
2382
+ invocation_id = excluded.invocation_id`, eventId, deliveryId, new Date().toISOString(), error, attempts, nextAttemptAt,
2383
+ // #1525: the call this attempt ran in, as the coordinator named it.
2384
+ invocationId);
1682
2385
  return attempts;
1683
2386
  }
1684
2387
  /** How many attempts this delivery has already had — the backoff input. */
@@ -1720,12 +2423,21 @@ export function defineScopeDO(modules, bareOps) {
1720
2423
  * `result` is COALESCE'd so a value written on an earlier pass (e.g. a minted sibling scope id,
1721
2424
  * for two-phase idempotency) survives a null on retry. `attempts` bumps each settle; `settled_at`
1722
2425
  * is set only on a terminal outcome.
2426
+ *
2427
+ * **Compare-and-set on `pending` (#1600 review).** The drain reads pending rows, runs a
2428
+ * handler, then settles — and between the read and the settle a subject erasure can redact
2429
+ * the row. Settling by `id` alone let that stale pass overwrite the redaction and write a
2430
+ * provider's reply, which can quote the person, back into `last_error`. Nothing legitimate
2431
+ * is refused: `pendingPlatformRequests` returns only pending rows, so every settle targets
2432
+ * one that was pending when it was read. A settle that finds the row already terminal does
2433
+ * nothing, deliberately silently — throwing would make the drain's blanket catch retry a
2434
+ * row that is correctly over.
1723
2435
  */
1724
2436
  settlePlatformRequest(id, status, result, lastError, lastFailure = null) {
1725
2437
  this.sql.exec(`UPDATE _substrat_platform_requests
1726
2438
  SET status = ?, result = COALESCE(?, result), last_error = ?, last_failure = ?,
1727
2439
  attempts = attempts + 1, settled_at = ?
1728
- WHERE id = ?`, status, result, lastError, lastFailure, status === 'pending' ? null : new Date().toISOString(), id);
2440
+ WHERE id = ? AND status = 'pending'`, status, result, lastError, lastFailure, status === 'pending' ? null : new Date().toISOString(), id);
1729
2441
  }
1730
2442
  /**
1731
2443
  * Turn one connector delivery into a `connector:<provider>` platform intent (#574
@@ -1736,8 +2448,18 @@ export function defineScopeDO(modules, bareOps) {
1736
2448
  * the event on the next drain). Backpressure throws BEFORE any write — the caller
1737
2449
  * records a failed attempt and the delivery retries on its own backoff, exactly as
1738
2450
  * a throwing handler would.
2451
+ *
2452
+ * `invocationId` is LAST, and defaulted, for the RPC-skew reason `recordExecutorAttempt`
2453
+ * states at length.
1739
2454
  */
1740
- routeExecutorEventToPlatform(eventId, deliveryId, kind, payload, requestedBy) {
2455
+ routeExecutorEventToPlatform(eventId, deliveryId, kind, payload, requestedBy,
2456
+ /**
2457
+ * #1525: the call this routing ran in, or null. On the hosted path this is the
2458
+ * dominant executor journal — every vertical is CP-less, so a connector delivery
2459
+ * becomes an intent here rather than running in `recordExecutorAttempt`'s caller —
2460
+ * so leaving it null would make the column read "no call" for the common case.
2461
+ */
2462
+ invocationId = null) {
1741
2463
  const pending = Number(this.sql
1742
2464
  .exec(`SELECT COUNT(*) AS c FROM _substrat_platform_requests WHERE status = 'pending'`)
1743
2465
  .toArray()[0].c);
@@ -1757,7 +2479,7 @@ export function defineScopeDO(modules, bareOps) {
1757
2479
  this.sql.exec(`INSERT INTO _substrat_platform_requests
1758
2480
  (id, kind, payload, requested_by, impersonation, status, attempts, requested_at)
1759
2481
  VALUES (?, ?, ?, ?, ?, 'pending', 0, ?)`, id, kind, payload, requestedBy, source?.impersonation ?? null, instant.parse(new Date().toISOString()));
1760
- this.recordExecutorAttempt(eventId, deliveryId, null, null);
2482
+ this.recordExecutorAttempt(eventId, deliveryId, null, null, invocationId);
1761
2483
  return id;
1762
2484
  }
1763
2485
  /**
@@ -1933,6 +2655,9 @@ export function defineScopeDO(modules, bareOps) {
1933
2655
  * `destScopeId` is the scope being written INTO. A dump carries scope-level tuples
1934
2656
  * naming the scope it was captured from, so restoring one anywhere else needs them
1935
2657
  * re-pointed — see `rewriteScopeTuples`.
2658
+ *
2659
+ * What the dump did NOT carry is rebuilt by the next migration pass, which is why
2660
+ * this ends by forgetting the memoised one (#1589) — see the tail of the method.
1936
2661
  */
1937
2662
  /**
1938
2663
  * The additive spine-column migrations. KERNEL_DDL is all IF NOT EXISTS, so a
@@ -1951,6 +2676,11 @@ export function defineScopeDO(modules, bareOps) {
1951
2676
  // consumer dead-letter.
1952
2677
  'ALTER TABLE _substrat_deliveries ADD COLUMN attempts INTEGER NOT NULL DEFAULT 0',
1953
2678
  'ALTER TABLE _substrat_deliveries ADD COLUMN next_attempt_at TEXT',
2679
+ // #1525: the invocation an attempt ran in, on a scope DO created before the
2680
+ // column. Nullable, and the null is honestly "no call was carried" — which
2681
+ // attempt produced a delivery already journalled cannot be decided afterwards,
2682
+ // exactly as #1237's outbox column argued.
2683
+ 'ALTER TABLE _substrat_deliveries ADD COLUMN invocation_id TEXT',
1954
2684
  // K-34: the authorization column on a scope DO created before it existed. Nullable,
1955
2685
  // so legacy outbox rows read as "unrecorded". (_substrat_denials is a new table,
1956
2686
  // covered by KERNEL_DDL's IF NOT EXISTS with no ALTER.)
@@ -1966,6 +2696,10 @@ export function defineScopeDO(modules, bareOps) {
1966
2696
  'ALTER TABLE _substrat_outbox ADD COLUMN impersonation TEXT',
1967
2697
  'ALTER TABLE _substrat_platform_requests ADD COLUMN impersonation TEXT',
1968
2698
  'ALTER TABLE _substrat_denials ADD COLUMN impersonation TEXT',
2699
+ // #1525: the invocation a refusal happened during, on a scope DO created before
2700
+ // the column. Nullable, and the null is honestly "no id was carried" — a past
2701
+ // denial's call cannot be decided afterwards, as #1237's outbox column argued.
2702
+ 'ALTER TABLE _substrat_denials ADD COLUMN invocation_id TEXT',
1969
2703
  // #1231: the emitting operation, on a scope DO created before the column.
1970
2704
  // Nullable so every legacy row reads as unrecorded rather than named.
1971
2705
  'ALTER TABLE _substrat_outbox ADD COLUMN operation TEXT',
@@ -2001,6 +2735,33 @@ export function defineScopeDO(modules, bareOps) {
2001
2735
  // boot. `lint:spine-ddl` compares KERNEL_DDL's indexes only, so this one is held to
2002
2736
  // both adapters by the query-plan test rather than by that gate.
2003
2737
  this.sql.exec('CREATE INDEX IF NOT EXISTS _substrat_outbox_invocation ON _substrat_outbox (invocation_id, id)');
2738
+ this.ensureScheduleStateKind();
2739
+ }
2740
+ /**
2741
+ * #1288: `_substrat_schedule_state`, rebuilt with `kind` in its key on a scope DO
2742
+ * whose table predates the column. Not in the ALTER list above, because `kind`
2743
+ * joins the PRIMARY KEY and no ALTER can widen a key — the statements are the
2744
+ * kernel's, so the pure adapter rebuilds byte-identically.
2745
+ *
2746
+ * Detected from `sqlite_master.sql`: DO SQLite restricts `PRAGMA`, and reading
2747
+ * the stored DDL is the one probe both adapters can make.
2748
+ */
2749
+ ensureScheduleStateKind() {
2750
+ const row = this.sql
2751
+ .exec(`SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?`, '_substrat_schedule_state')
2752
+ .toArray()[0];
2753
+ if (!row || scheduleStateHasKind(row.sql))
2754
+ return;
2755
+ // In a transaction, for the reason the kernel constant spells out: create-copy-
2756
+ // drop-rename has two intermediate states and both are unrecoverable on the next
2757
+ // wake. `transactionSync`, not the async one every operation uses — the DO
2758
+ // runtime forbids a manual BEGIN through `sql.exec`, and this body is wholly
2759
+ // synchronous, which is the one case the sync API is for (it commits at the
2760
+ // first await, and there is none). It also has to be sync because the caller is.
2761
+ this.ctx.storage.transactionSync(() => {
2762
+ for (const stmt of splitSqlStatements(SCHEDULE_STATE_REBUILD))
2763
+ this.sql.exec(stmt);
2764
+ });
2004
2765
  }
2005
2766
  async importDump(tables, destScopeId) {
2006
2767
  // The WHOLE drop-then-replay runs under deferred foreign keys, in one transaction.
@@ -2100,6 +2861,29 @@ export function defineScopeDO(modules, bareOps) {
2100
2861
  .toArray()) {
2101
2862
  this.applied.add(`${row.module_id}@${row.version}`);
2102
2863
  }
2864
+ // …and forget that this INSTANCE ever ran a migration pass (#1589). Refreshing
2865
+ // the set above is not enough on its own: `ensureMigrations` memoises its
2866
+ // promise, so a warm DO answers "already migrated" from the cache and never
2867
+ // reads the set again. A restore replays only what the dump carries, so a dump
2868
+ // that omits a module's tables — a world that keeps part of the spine
2869
+ // elsewhere, a targeted repair supplying only the tables being fixed — leaves
2870
+ // them dropped and never rebuilt, and the next operation touching one fails
2871
+ // with a bare `no such table` until an eviction or `retryMigrations` resets the
2872
+ // latch. The pure host has no such memo (it re-reads `appliedMigrations` on
2873
+ // every pass), so this is the line that makes the two adapters agree; without
2874
+ // it the divergence is invisible to dev, CI and self-host, which is the #969
2875
+ // class exactly. Same two fields `retryMigrations` clears, for the same reason:
2876
+ // the next pass has to be a fresh one. Already-journaled versions are skipped
2877
+ // by the `applied` set and the in-transaction re-check, so a dump that DID
2878
+ // carry its tables re-applies nothing.
2879
+ //
2880
+ // `schemaVersionReported` is deliberately left set, as `retryMigrations` leaves
2881
+ // it. `migrate()` reports only when a pass APPLIED something, and a pass that
2882
+ // applies after this reset ends with every code-defined migration journaled —
2883
+ // the same `applied.size` the first pass already reported. A dump whose frontier
2884
+ // is complete applies nothing, so there is nothing to report either way.
2885
+ this.migrationPromise = undefined;
2886
+ this.lastFailure = null;
2103
2887
  }
2104
2888
  /**
2105
2889
  * Re-point scope-level tuples at the scope they now live in.
@@ -2161,12 +2945,23 @@ export function defineScopeDO(modules, bareOps) {
2161
2945
  * and transaction facts remain". A timeline still shows that something happened, to
2162
2946
  * what, and when; it no longer shows who, or what was said about them.
2163
2947
  *
2948
+ * **Two tables, one verb (#1600).** The outbox is not the only place the spine holds
2949
+ * an event's payload: this host is the CP-less one, so every connector delivery it
2950
+ * cannot run becomes a `connector:<provider>` intent carrying the whole event, and
2951
+ * nothing ever deletes those rows. Redacting one table and not the other left the
2952
+ * name in the live database and in every copy taken from it afterwards. One RPC
2953
+ * rather than two for `routeExecutorEventToPlatform`'s reason — a crash cannot land
2954
+ * half an erasure — and both halves are idempotent anyway, so a retry converges.
2955
+ *
2164
2956
  * This is the one sanctioned write that mutates the outbox. It is kernel code, not
2165
2957
  * module code, and an erasure request is precisely the case the append-only rule has
2166
2958
  * to yield to — the alternative is telling a data subject that the spine's convenience
2167
2959
  * outranks their Article 17 right.
2168
2960
  */
2169
2961
  async redactSubject(subjectId) {
2962
+ // One instant for the whole erasure — the intent tombstones must not disagree with
2963
+ // each other about when a person was erased.
2964
+ const at = new Date().toISOString();
2170
2965
  const doomed = this.sql
2171
2966
  .exec(`SELECT id FROM _substrat_outbox
2172
2967
  WHERE subject_id = ? AND pii_class != 'none' AND payload IS NOT NULL`, subjectId)
@@ -2174,7 +2969,30 @@ export function defineScopeDO(modules, bareOps) {
2174
2969
  for (const id of doomed) {
2175
2970
  this.sql.exec('UPDATE _substrat_outbox SET payload = NULL WHERE id = ?', id);
2176
2971
  }
2177
- return doomed.length;
2972
+ return { events: doomed.length, intents: this.redactSubjectIntents(subjectId, at) };
2973
+ }
2974
+ /**
2975
+ * The intent-journal half of `redactSubject` (#1600) — the SQLite adapter's twin.
2976
+ *
2977
+ * Row-by-row rather than one `UPDATE … WHERE`, because the decision is structural and
2978
+ * SQL cannot make it: an intent payload is opaque JSON with no `pii_class` column to
2979
+ * test, so the SQL narrows to rows that could possibly match and the kernel predicate
2980
+ * decides. Query, predicate and statement all come from the kernel, so this adapter
2981
+ * and the pure one cannot drift about what a redacted intent is.
2982
+ */
2983
+ redactSubjectIntents(subjectId, at) {
2984
+ const q = platformRequestRedactionQuery(subjectId);
2985
+ const candidates = this.sql
2986
+ .exec(q.sql, ...q.params)
2987
+ .toArray();
2988
+ let redacted = 0;
2989
+ for (const row of candidates) {
2990
+ if (!intentPayloadCarriesSubject(row.payload, subjectId))
2991
+ continue;
2992
+ this.sql.exec(PLATFORM_REQUEST_REDACTION_SQL, ...platformRequestRedactionParams(row.id, subjectId, at));
2993
+ redacted += 1;
2994
+ }
2995
+ return redacted;
2178
2996
  }
2179
2997
  // -- event dispatch (port of dispatch) ------------------------------------
2180
2998
  /**
@@ -2200,7 +3018,13 @@ export function defineScopeDO(modules, bareOps) {
2200
3018
  * later alarm-driven drain with a call it had nothing to do with.
2201
3019
  */
2202
3020
  invocationId = null;
2203
- async dispatch(tenantId, scopeId) {
3021
+ async dispatch(tenantId, scopeId,
3022
+ /**
3023
+ * #1525: the call this drain is running in, or null. Every round of the loop
3024
+ * belongs to it: a consumer's own emit is delivered in the same tail, so the
3025
+ * whole cascade is one call's work.
3026
+ */
3027
+ invocationId) {
2204
3028
  for (let round = 0; round < 50; round++) {
2205
3029
  let deliveredAny = false;
2206
3030
  for (const mod of this.modules.values()) {
@@ -2215,7 +3039,21 @@ export function defineScopeDO(modules, bareOps) {
2215
3039
  ORDER BY o.id`, consumer.eventType, mod.id)
2216
3040
  .toArray();
2217
3041
  for (const row of rows) {
2218
- const event = this.parseOutboxRow(row);
3042
+ let event;
3043
+ try {
3044
+ event = domainEventOf(row);
3045
+ }
3046
+ catch (err) {
3047
+ // #1636: dead-letter an event that does not decode, exactly as a failed
3048
+ // handler is — the decode sat ABOVE the `try` below, so one bad row halted
3049
+ // every event of this type behind it, on every pass. The consumer is never
3050
+ // handed it: an event built from stand-ins is not one it may act on. Only
3051
+ // the decode is caught; the journal write is not.
3052
+ this.sql.exec(`INSERT INTO _substrat_deliveries
3053
+ (event_id, consumer_module, delivered_at, error, invocation_id)
3054
+ VALUES (?, ?, ?, ?, ?)`, row.id, mod.id, new Date().toISOString(), String(err), invocationId);
3055
+ continue;
3056
+ }
2219
3057
  // #1237: anything this consumer emits was emitted BECAUSE of this event
2220
3058
  // — the step a backwards walk used to stop dead at, since a consumer
2221
3059
  // emit records no operation either.
@@ -2226,16 +3064,18 @@ export function defineScopeDO(modules, bareOps) {
2226
3064
  system: mod.id,
2227
3065
  });
2228
3066
  await consumer.handler(ctx, event);
2229
- this.sql.exec(`INSERT INTO _substrat_deliveries (event_id, consumer_module, delivered_at)
2230
- VALUES (?, ?, ?)`, event.id, mod.id, new Date().toISOString());
3067
+ this.sql.exec(`INSERT INTO _substrat_deliveries
3068
+ (event_id, consumer_module, delivered_at, invocation_id)
3069
+ VALUES (?, ?, ?, ?)`, event.id, mod.id, new Date().toISOString(), invocationId);
2231
3070
  });
2232
3071
  deliveredAny = true;
2233
3072
  }
2234
3073
  catch (err) {
2235
3074
  // Dead-letter (v0): journal the failure so one poison event
2236
3075
  // can't wedge the loop. Written outside the rolled-back txn.
2237
- this.sql.exec(`INSERT INTO _substrat_deliveries (event_id, consumer_module, delivered_at, error)
2238
- VALUES (?, ?, ?, ?)`, event.id, mod.id, new Date().toISOString(), String(err));
3076
+ this.sql.exec(`INSERT INTO _substrat_deliveries
3077
+ (event_id, consumer_module, delivered_at, error, invocation_id)
3078
+ VALUES (?, ?, ?, ?, ?)`, event.id, mod.id, new Date().toISOString(), String(err), invocationId);
2239
3079
  }
2240
3080
  finally {
2241
3081
  // Cleared on BOTH paths. Left set, the id leaks onto every later emit
@@ -2257,43 +3097,40 @@ export function defineScopeDO(modules, bareOps) {
2257
3097
  * survives — the whole point, since the denial is the write the operation could not make.
2258
3098
  */
2259
3099
  /** K-42: the session a refused call ran under travels with the denial row. */
2260
- recordDenial(subject, tenantId, operation, err, impersonation) {
3100
+ recordDenial(subject, tenantId, operation, err,
3101
+ /**
3102
+ * #1525: the invocation this refusal belongs to, or null — PASSED, never read off
3103
+ * `this.invocationId` here.
3104
+ *
3105
+ * Reading the ambient field is only self-evidently right where one call holds the
3106
+ * DO to itself, and two denial paths do not: `attachmentList` and
3107
+ * `attachmentAuthorize` run OUTSIDE `this.queue`, and both await before they
3108
+ * record (`ensureMigrations`, `ctx.check`). Whether the input gate can actually
3109
+ * reopen far enough for one of them to observe an in-flight call's id is NOT
3110
+ * settled here — a probe that raced twelve attachment refusals against invokes
3111
+ * holding an id (including one awaiting the control plane) recorded null every
3112
+ * time, so the gate evidently holds more than the shape of the code promises.
3113
+ *
3114
+ * Passed anyway, because the argument for the ambient read is an argument about
3115
+ * workerd's gate semantics, and the argument for a parameter is local: the invoke
3116
+ * path passes its own id, every attachment path passes null, and each says what it
3117
+ * actually knows. That is the property worth having on a recorded fact, and it
3118
+ * costs one argument. No test accompanies it — the condition could not be
3119
+ * reproduced, and a test that passes either way would be worse than none.
3120
+ */
3121
+ invocationId, impersonation) {
2261
3122
  // Only an ENFORCED denial (assertAllowed, which attaches the checked permission +
2262
3123
  // node) is recorded. A module's own hand-thrown `new PermissionDenied('…')` carries
2263
3124
  // no permission key and is left to the module.
2264
3125
  if (!err.permission || !err.node)
2265
3126
  return;
2266
- const actor = subject.kind === 'system'
2267
- ? { system: subject.id }
2268
- : subject.kind === 'connection'
2269
- ? { connection: subject.id }
2270
- : subject.id;
3127
+ const actor = actorOf(subject);
2271
3128
  this.sql.exec(`INSERT INTO _substrat_denials
2272
- (id, actor, permission, tenant_id, scope_id, operation, impersonation, at)
2273
- VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, ulid(), JSON.stringify(actor), err.permission, err.node.tenantId, err.node.scopeId ?? null, operation, impersonation ? JSON.stringify(impersonationStampOf(impersonation)) : null, new Date().toISOString());
2274
- }
2275
- parseOutboxRow(row) {
2276
- return domainEvent.parse({
2277
- id: row.id,
2278
- type: row.type,
2279
- schemaVersion: row.schema_version,
2280
- occurredAt: row.occurred_at,
2281
- tenantId: row.tenant_id,
2282
- scopeId: row.scope_id,
2283
- actor: JSON.parse(row.actor),
2284
- entity: { entityType: row.entity_type, entityId: row.entity_id },
2285
- piiClass: row.pii_class,
2286
- ...(row.subject_id ? { subjectId: row.subject_id } : {}),
2287
- ...(row.authorization ? { authorization: JSON.parse(row.authorization) } : {}),
2288
- // K-42: the stamp survives the read, so a consumer's event and an executor's
2289
- // are the same fact the stored row is. Absent rather than null when nobody
2290
- // was impersonating, because `DomainEvent.impersonation` is optional — the
2291
- // shape module code never sees is also the shape it cannot branch on.
2292
- ...(row.impersonation ? { impersonation: JSON.parse(row.impersonation) } : {}),
2293
- // #1231: absent rather than null, the same shape rule as the stamp above.
2294
- ...(row.operation ? { operation: row.operation } : {}),
2295
- payload: row.payload === null ? undefined : JSON.parse(row.payload),
2296
- });
3129
+ (id, actor, permission, tenant_id, scope_id, operation, impersonation,
3130
+ invocation_id, at)
3131
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, ulid(), JSON.stringify(actor), err.permission, err.node.tenantId, err.node.scopeId ?? null, operation, impersonation ? JSON.stringify(impersonationStampOf(impersonation)) : null,
3132
+ // #1525: the call this refusal belongs to, as the caller named it.
3133
+ invocationId, new Date().toISOString());
2297
3134
  }
2298
3135
  // -- operation context (port of operationContext) -------------------------
2299
3136
  operationContext(principal, tenantId, scopeId, systemActor, connectionId, systemModuleId,
@@ -2312,7 +3149,28 @@ export function defineScopeDO(modules, bareOps) {
2312
3149
  * onto every event it emits. Absent for consumer dispatch: a consumer runs on
2313
3150
  * behalf of no operation, and the emitted row's NULL says so.
2314
3151
  */
2315
- operation) {
3152
+ operation,
3153
+ /**
3154
+ * #1672: set when the caller holds a CAPABILITY session — already resolved from its
3155
+ * hash inside the queued body, before the transaction. Mutually exclusive with
3156
+ * `connectionId` and `systemModuleId`; `principal` is then a placeholder that the
3157
+ * subject below never reads.
3158
+ */
3159
+ capabilityId,
3160
+ /**
3161
+ * #1672: the secrets `ctx.capabilities.mint` hands out during this invocation. The
3162
+ * caller owns the array (it withholds them from the idempotency recording); this
3163
+ * context appends to it and holds `ctx.sql`, `ctx.emit` and `ctx.requestPlatform` to
3164
+ * it — the tripwire that catches a module persisting one by accident (not a boundary
3165
+ * against one that means to; see `assertNoSecret`).
3166
+ */
3167
+ minted = [],
3168
+ /**
3169
+ * #1672: the instant to stamp, when the caller already read one. The exchange passes
3170
+ * the instant its row write used, so the `capability.exercised` event's `occurredAt`
3171
+ * and the row's `last_used_at` are the same value — a second read could disagree.
3172
+ */
3173
+ instantOverride) {
2316
3174
  const checker = this.checker;
2317
3175
  const relations = this.relations;
2318
3176
  const searchPlans = this.searchPlans;
@@ -2326,20 +3184,18 @@ export function defineScopeDO(modules, bareOps) {
2326
3184
  * on, and it holds identically here: `ctx.now()`, every `occurredAt` and
2327
3185
  * every `requested_at` in one operation are the same value.
2328
3186
  */
2329
- const at = instant.parse(new Date().toISOString());
3187
+ const at = instantOverride ?? instant.parse(new Date().toISOString());
2330
3188
  // The permission subject and the derived event actor for a NON-override caller
2331
3189
  // (#383/#97): a scheduled module, a connection, or a person. `systemActor` (the
2332
3190
  // override, used only by consumer dispatch) stays a separate bypass path below.
2333
- const subject = systemModuleId
2334
- ? { kind: 'system', id: systemModuleId }
2335
- : connectionId
2336
- ? { kind: 'connection', id: connectionId }
2337
- : { kind: 'principal', id: principal };
2338
- const derivedActor = subject.kind === 'system'
2339
- ? { system: subject.id }
2340
- : subject.kind === 'connection'
2341
- ? { connection: subject.id }
2342
- : principal;
3191
+ const subject = capabilityId
3192
+ ? { kind: 'capability', id: capabilityId }
3193
+ : systemModuleId
3194
+ ? { kind: 'system', id: systemModuleId }
3195
+ : connectionId
3196
+ ? { kind: 'connection', id: connectionId }
3197
+ : { kind: 'principal', id: principal };
3198
+ const derivedActor = actorOf(subject);
2343
3199
  // #304: entitlement reads pick the same local-vs-RPC reader the permission checker
2344
3200
  // uses (projected scope → local table; console-managed → CP over RPC), resolved per
2345
3201
  // call so a scope that flips to 'local' is picked up without rebuilding the context.
@@ -2368,7 +3224,10 @@ export function defineScopeDO(modules, bareOps) {
2368
3224
  // Lifted so `grant` reuses the SAME check the operation itself passes —
2369
3225
  // a delegation check that could differ from the operation's would be a
2370
3226
  // second opinion about what the caller holds.
2371
- const runCheck = async (permission, entity) => {
3227
+ const runCheck = async (unparsed, entity) => {
3228
+ // #1642: parsed before the system actor's early return, which never reaches
3229
+ // the checker — a cast key would otherwise become that path's proof relation.
3230
+ const permission = assertPermissionKey(unparsed);
2372
3231
  if (systemActor) {
2373
3232
  return {
2374
3233
  allowed: true,
@@ -2403,15 +3262,19 @@ export function defineScopeDO(modules, bareOps) {
2403
3262
  }
2404
3263
  return checker.covers(subject, role.permissions, { tenantId, scopeId });
2405
3264
  };
2406
- return {
3265
+ const ctxRef = {
2407
3266
  tenantId,
2408
3267
  scopeId,
2409
- principal,
2410
- sql: doScopedSql(sql),
3268
+ // #1672: a capability's own id stands in so the type holds — it is not a person, and
3269
+ // the event actor says what it is instead. Every other door passes its own value.
3270
+ principal: capabilityId ? capabilityId : principal,
3271
+ sql: guardSecrets(doScopedSql(sql), minted),
2411
3272
  now: () => at,
2412
3273
  emit: (event) => {
2413
3274
  assertImpersonationWrites(impersonation, 'ctx.emit');
2414
3275
  const parsed = domainEventInput.parse(event);
3276
+ // #1672: the COMPLETE parsed event — entity id, type and subject as well as payload.
3277
+ assertNoSecret('ctx.emit', parsed, minted);
2415
3278
  const full = domainEvent.parse({
2416
3279
  ...parsed,
2417
3280
  // #956: from the operation's instant, not a second reading of the clock.
@@ -2448,6 +3311,8 @@ export function defineScopeDO(modules, bareOps) {
2448
3311
  requestPlatform: (request) => {
2449
3312
  assertImpersonationWrites(impersonation, 'ctx.requestPlatform');
2450
3313
  const input = platformRequestInput.parse(request);
3314
+ // #1672: the COMPLETE parsed request — its `kind` is persisted as surely as its payload.
3315
+ assertNoSecret('ctx.requestPlatform', input, minted);
2451
3316
  // #1474: a platform-authored kind (`sweep-runs`) never comes from module code —
2452
3317
  // the sweeper enqueues it through `enqueueSweepRuns`, which does not pass here.
2453
3318
  assertModuleEnqueueableKind(input.kind);
@@ -2472,7 +3337,9 @@ export function defineScopeDO(modules, bareOps) {
2472
3337
  // predicate is needed or possible: the DO IS the scope.
2473
3338
  platformRequests: (filter) => {
2474
3339
  const q = platformRequestHistoryQuery(filter);
2475
- return sql.exec(q.sql, ...q.params).toArray().map(rowToPlatformRequest);
3340
+ // The kernel's decoder, the one the coordinator maps the RPC's rows with (#1588):
3341
+ // tolerant, so one undecodable row cannot hide this scope's other intents from it.
3342
+ return sql.exec(q.sql, ...q.params).toArray().map(platformRequestOf);
2476
3343
  },
2477
3344
  // #901. Mirror of the pure adapter, and the reason the contract suite
2478
3345
  // runs on both: the query is ordinary SQL, but it is only a seek rather
@@ -2551,6 +3418,22 @@ export function defineScopeDO(modules, bareOps) {
2551
3418
  sql.exec(`DELETE FROM _substrat_tuples WHERE subject = ? AND relation = ? AND object = ?`, `principal:${principal}`, `granted:${permission}`, `${entity.entityType}:${entity.entityId}`);
2552
3419
  },
2553
3420
  atomic: createAtomic(runSub, { passed, signals }),
3421
+ // #1672: mint / revoke / list, written once in the kernel — the pure adapter hands
3422
+ // the same function the same four things. The raw spine seam (the kernel's own write
3423
+ // to `_substrat_capabilities`), the operation's OWN check, and `ctx.emit`. A
3424
+ // consumer's override actor is passed as the system actor it is, so a consumer
3425
+ // cannot mint: its checks allow unconditionally, which would make "the minter holds
3426
+ // it" vacuous.
3427
+ capabilities: createCapabilityVerbs({
3428
+ sql: doSpineSql(sql),
3429
+ subject: systemActor ? { kind: 'system', id: systemActor.system } : subject,
3430
+ now: at,
3431
+ check: runCheck,
3432
+ emit: (event) => ctxRef.emit(event),
3433
+ isOperation: (name) => this.operations.has(name),
3434
+ assertWrites: (verb) => assertImpersonationWrites(impersonation, verb),
3435
+ minted,
3436
+ }),
2554
3437
  link: (child, parent) => {
2555
3438
  assertImpersonationWrites(impersonation, 'ctx.link');
2556
3439
  const allowed = relations.get(child.entityType);
@@ -2584,6 +3467,7 @@ export function defineScopeDO(modules, bareOps) {
2584
3467
  return sealTo({ keyId: row.key_id, publicKey: row.public_key }, plaintext);
2585
3468
  },
2586
3469
  };
3470
+ return ctxRef;
2587
3471
  }
2588
3472
  /** True once this scope has had entitlements projected at least once (#304) — the switch
2589
3473
  * from trust-upstream to strict fail-closed entitlement enforcement on the local path. */
@@ -2686,11 +3570,16 @@ export function defineScopeDO(modules, bareOps) {
2686
3570
  * while passing a list — even `[]` — full-replaces them. This keeps pre-#304 callers
2687
3571
  * from silently wiping a scope's entitlements. */
2688
3572
  entitlements,
2689
- /** Scope-level tuples (e.g. the owner's role grant at provision) upserted into
3573
+ /** Scope-level tuples (e.g. the owner's role grant at provision) seated into
2690
3574
  * `_substrat_tuples` in this SAME transaction, additively (#332). Preserve-on-undefined:
2691
3575
  * omitting it leaves existing scope tuples untouched, so a role-only re-projection keeps
2692
3576
  * the owner grant. Passing them here (rather than a follow-up `writeTuple`) is what makes
2693
- * provision atomic — the grant and the enforcement flip land together or not at all. */
3577
+ * provision atomic — the grant and the enforcement flip land together or not at all.
3578
+ *
3579
+ * Seated, not replaced (#1659): a missing tuple is created, a revoked one stays revoked.
3580
+ * `lockout_reseat` marks the one exception — the owner-of-record's seat, which is
3581
+ * re-seated even over a revoke when the scope would otherwise hold no effective role
3582
+ * grant (`hasEffectiveRoleGrant`). */
2694
3583
  scopeTuples,
2695
3584
  /** The tenant's identity links (#406) — projected alongside the rest so a CP-less
2696
3585
  * vertical's auth adapter resolves logins locally. Same preserve-on-undefined
@@ -2765,18 +3654,39 @@ export function defineScopeDO(modules, bareOps) {
2765
3654
  // — NOT a full replace — so existing scope tuples are preserved. This is what keeps a
2766
3655
  // scope from ever being left "roles projected, permission_source=local, zero tuples" by
2767
3656
  // a write that lands the projection but drops before a follow-up owner grant.
3657
+ //
3658
+ // #1659: SEATED, so a reconcile creates what is missing and leaves a revoke alone. It
3659
+ // used to be `INSERT OR REPLACE … revoked_at = NULL`, which undid an operator's revoke
3660
+ // of the owner seat or of a `system:` schedule grant on the next reconcile. And a
3661
+ // module switched off (#1666) gets no `system:` grant seated, new or old.
2768
3662
  for (const st of scopeTuples ?? []) {
2769
- this.sql.exec(`INSERT OR REPLACE INTO _substrat_tuples (subject, relation, object, expires_at, revoked_at)
2770
- VALUES (?, ?, ?, ?, NULL)`, st.subject, st.relation, st.object, st.expires_at);
3663
+ const seat = seatScopeTuple(st.subject, st.relation, st.object, st.expires_at);
3664
+ this.sql.exec(seat.sql, ...seat.params);
3665
+ }
3666
+ // #1659's one exception: the owner-of-record's seat comes back over a revoke when
3667
+ // NOTHING else would let anyone act here — roles projected, no effective role grant.
3668
+ // That is the #332 lockout this path exists to repair, and it is decided by the same
3669
+ // predicate as the flip guard below, so "locked out" means one thing in this unit.
3670
+ // With any other effective holder, the revoke stands: a hand-over that seats a
3671
+ // successor before unseating the owner is not undone by the next promote. A holder of
3672
+ // a role the vertical no longer defines is NOT one — it passes no check, so it must
3673
+ // not stand in for the holder this repair exists to restore.
3674
+ if (roles.length > 0 && !this.hasEffectiveRoleGrant(tenantId)) {
3675
+ for (const st of scopeTuples ?? []) {
3676
+ if (!st.lockout_reseat)
3677
+ continue;
3678
+ this.sql.exec(`INSERT OR REPLACE INTO _substrat_tuples (subject, relation, object, expires_at, revoked_at)
3679
+ VALUES (?, ?, ?, ?, NULL)`, st.subject, st.relation, st.object, st.expires_at);
3680
+ }
2771
3681
  }
2772
3682
  // #332: only switch on strict local enforcement when SOMEONE actually holds a role.
2773
- // A projection that leaves role definitions but no live principal→role grant would make
2774
- // every check fail closed — a scope serving nothing but denials, unfixable from inside.
2775
- // Leave `permission_source` as-is instead; a reconcile that restores the owner grant
2776
- // re-runs this and flips safely. (A CP-less vertical uses the local reader regardless of
2777
- // this flag, so the owner grant is written above in the same unit — this guard is the
2778
- // belt to that suspenders, and it protects the CP-backed flip outright.)
2779
- if (roles.length > 0 && !this.hasLiveRoleGrant(tenantId))
3683
+ // A projection that leaves role definitions but no effective principal→role grant would
3684
+ // make every check fail closed — a scope serving nothing but denials, unfixable from
3685
+ // inside. Leave `permission_source` as-is instead; a reconcile that restores the owner
3686
+ // grant re-runs this and flips safely. (A CP-less vertical uses the local reader
3687
+ // regardless of this flag, so the owner grant is written above in the same unit — this
3688
+ // guard is the belt to that suspenders, and it protects the CP-backed flip outright.)
3689
+ if (roles.length > 0 && !this.hasEffectiveRoleGrant(tenantId))
2780
3690
  return;
2781
3691
  this.sql.exec(`INSERT OR REPLACE INTO _substrat_meta (key, value) VALUES ('permission_source', 'local')`);
2782
3692
  });
@@ -2797,23 +3707,17 @@ export function defineScopeDO(modules, bareOps) {
2797
3707
  return undefined;
2798
3708
  return { principal: row.principal_id, scopeId: row.scope_id };
2799
3709
  }
2800
- /** True if any live (non-revoked, unexpired) principal→role grant exists for this tenant,
2801
- * at scope OR tenant level — the precondition for switching on strict local enforcement so a
2802
- * projection never enables fail-closed evaluation against an empty tuple table (#332). */
2803
- hasLiveRoleGrant(tenantId) {
2804
- const now = new Date().toISOString();
2805
- const scope = this.sql
2806
- .exec(`SELECT 1 FROM _substrat_tuples
2807
- WHERE relation LIKE 'role:%' AND revoked_at IS NULL AND (expires_at IS NULL OR expires_at > ?)
2808
- LIMIT 1`, now)
2809
- .toArray();
2810
- if (scope.length > 0)
2811
- return true;
2812
- return (this.sql
2813
- .exec(`SELECT 1 FROM _substrat_tenant_tuples
2814
- WHERE tenant_id = ? AND relation LIKE 'role:%' AND revoked_at IS NULL AND (expires_at IS NULL OR expires_at > ?)
2815
- LIMIT 1`, tenantId, now)
2816
- .toArray().length > 0);
3710
+ /** True if some principal holds a role this scope can actually EXPAND — a live
3711
+ * (non-revoked, unexpired) `role:<key>` tuple, at scope OR tenant level, whose key names a
3712
+ * current, non-revoked role definition for this tenant. The one predicate behind both the
3713
+ * #332 flip guard and #1659's owner re-seat in `applyProjection`; the query is the
3714
+ * kernel's `effectiveRoleGrantQuery`, where it is tested against a real SQLite. A tuple
3715
+ * for a role the vertical no longer defines counts for nothing, exactly as in the local
3716
+ * checker, which expands a role only through its definition. */
3717
+ hasEffectiveRoleGrant(tenantId) {
3718
+ const q = effectiveRoleGrantQuery(tenantId, new Date().toISOString());
3719
+ const row = this.sql.exec(q.sql, ...q.params).toArray()[0];
3720
+ return row?.effective === 1;
2817
3721
  }
2818
3722
  };
2819
3723
  }