@substrat-run/adapter-cloudflare 0.114.0 → 0.117.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/host.js CHANGED
@@ -1,6 +1,7 @@
1
- import { fromWireFailure, accessLogEntry, adminLogEntry, opsFailureEntry, opsFailureFingerprint, issueEntry, sweepRunEntry, FRESHNESS_HEARTBEAT_MINUTES, sweepRunsPayload, modelUsageEntry, attachmentRecord, createTenantInput, identityLink, identityPool, createOrgInput, promotionAcknowledgement, bindHostnameInput, channelHistoryEntry, hostnameBinding, publishVersionInput, AUTO_ADMISSION_NOTE, registerVerticalInput, vertical as verticalSchema, verticalServingState, verticalChannel, verticalVersion, connection, capabilityGrant, connectionGrant, connectionGrantRecord, connectionSecret, systemGrant, entitlementGrant, entitlementGrantInput, instant, meterReading, subjectRef, createConnectionInput, projectedConnectionGrant, projectedConnectionKey, moduleManifest, org as orgSchema, orgMembership, resolvedIdentity, roleDefinition, scope as scopeSchema, tenant as tenantSchema, tenantRole, delegatedReadRecord, subjectShredReceipt, platformRequest, connectorDispatchKind, outboundOfManifestJson, substratError, redrainEventsInput, } from '@substrat-run/contracts';
1
+ import { fromWireFailure, accessLogEntry, adminLogEntry, opsFailureEntry, opsFailureFingerprint, issueEntry, sweepRunEntry, FRESHNESS_HEARTBEAT_MINUTES, sweepRunsPayload, modelUsageEntry, attachmentRecord, createTenantInput, identityLink, identityPool, createOrgInput, promotionAcknowledgement, bindHostnameInput, channelHistoryEntry, hostnameBinding, publishVersionInput, AUTO_ADMISSION_NOTE, registerVerticalInput, vertical as verticalSchema, verticalServingState, verticalChannel, verticalVersion, connection, capabilityExchange, capabilityGrant, principalId, connectionGrant, connectionGrantRecord, connectionSecret, systemGrant, systemSwitch, systemSwitchOutcome, entitlementGrant, entitlementGrantInput, instant, meterReading, subjectRef, createConnectionInput, projectedConnectionGrant, projectedConnectionKey, moduleManifest, org as orgSchema, orgMembership, resolvedIdentity, identityMembership, roleDefinition, scope as scopeSchema, tenant as tenantSchema, tenantRole, delegatedReadRecord, subjectShredReceipt, connectorDispatchKind, outboundOfManifestJson, substratError, redrainEventsInput, } from '@substrat-run/contracts';
2
2
  import { normalizeHostname, toRouteTarget } from './route-resolver.js';
3
- import { attachmentBlobKey, entitlementDenial, foldMeterReading, parseValidationRecords, resolveScopeRecord, ulid, foldModelUsage, backoffAt, resolveRetryPolicy, isSecretBoxConfigured, unconfiguredSecretBox, createSubjectKeys, generateSealingKeyPair, openSealed, ImpersonationRefused, assertSessionUsable, impersonationRowValues, mapImpersonationRow, newImpersonationSession, globalFetch, assertRedrainWindow, } from '@substrat-run/kernel';
3
+ import { attachmentBlobKey, entitlementDenial, foldMeterReading, parseValidationRecords, resolveScopeRecord, ulid, capabilityTokenHash, checkBecomeInput, plausibleSessionToken, foldModelUsage, backoffAt, resolveRetryPolicy, isSecretBoxConfigured, unconfiguredSecretBox, createSubjectKeys, generateSealingKeyPair, openSealed, ImpersonationRefused, assertSessionUsable, impersonationRowValues, mapImpersonationRow, newImpersonationSession, jobRunOf, runDueJobRuns, startJobRun, globalFetch, assertRedrainWindow, platformRequestOf, undrainedEventsOf, systemSwitchedOffMessage, } from '@substrat-run/kernel';
4
+ import { isOrangeToOrange, isUpgradeRequest, LIVE_MODE_HEADER, LIVE_PRINCIPAL_HEADER, LIVE_SCOPE_HEADER, LIVE_SUBSCRIBE_PATH, LIVE_TENANT_HEADER, } from './live-reads.js';
4
5
  import { tenantStoreDatabaseName } from './d1.js';
5
6
  import { blobStoreBucketName, r2TenantBlobStore } from './r2.js';
6
7
  /** DO row → contract shape. Never reads the secrets table — that is the split. */
@@ -21,23 +22,6 @@ const toConnection = (r) => connection.parse({
21
22
  createdAt: r.created_at,
22
23
  revokedAt: r.revoked_at,
23
24
  });
24
- /** Map a stored platform-request row to the `PlatformRequest` contract shape (JSON columns parsed). */
25
- function rowToPlatformRequest(r) {
26
- return platformRequest.parse({
27
- id: r.id,
28
- kind: r.kind,
29
- payload: JSON.parse(r.payload),
30
- requestedBy: JSON.parse(r.requested_by),
31
- impersonation: r.impersonation == null ? null : JSON.parse(r.impersonation),
32
- status: r.status,
33
- attempts: r.attempts,
34
- lastError: r.last_error,
35
- failure: r.last_failure == null ? null : JSON.parse(r.last_failure),
36
- result: r.result === null ? null : JSON.parse(r.result),
37
- requestedAt: r.requested_at,
38
- settledAt: r.settled_at,
39
- });
40
- }
41
25
  /**
42
26
  * A control-plane stand-in for a CP-less vertical (scope-local-permissions.md Phase 3).
43
27
  * The hot path a served scope actually touches becomes trust-the-upstream:
@@ -99,6 +83,14 @@ export class CloudflareScopeHost {
99
83
  /** Worker-side attachment-bucket resolver (#473); undefined ⇒ attachments() refuses. */
100
84
  attachmentBuckets;
101
85
  executors = new Map();
86
+ /**
87
+ * `<moduleId>/<job>` → the pass body and its default step policy (#1577). Host
88
+ * code like `executors`, and keyed the way a run row is: the coalescing key's
89
+ * first two thirds, so a run read off the DO finds its handler by the columns it
90
+ * already carries. The HANDLER stays on the coordinator — it holds credentials
91
+ * and calls the internet, which is why the DO never sees it.
92
+ */
93
+ jobs = new Map();
102
94
  /**
103
95
  * The event currently being effected, stamped onto admin rows the executor writes.
104
96
  * Ambient rather than threaded through every HostAdmin signature: set and cleared
@@ -112,6 +104,8 @@ export class CloudflareScopeHost {
112
104
  connectorDelegation;
113
105
  /** #1334: the Tier-2 drain's reach into the deployment serving a scope. */
114
106
  eventDrainDelegation;
107
+ /** #1666: the schedule kill switch's reach into the deployment serving a scope. */
108
+ systemSwitchDelegation;
115
109
  /**
116
110
  * MUST be constructed per request. Never cache an instance across requests.
117
111
  *
@@ -137,6 +131,7 @@ export class CloudflareScopeHost {
137
131
  : nullControlPlane();
138
132
  this.connectorDelegation = options.connectorDelegation;
139
133
  this.eventDrainDelegation = options.eventDrainDelegation;
134
+ this.systemSwitchDelegation = options.systemSwitchDelegation;
140
135
  this.admin = this.buildAdmin();
141
136
  }
142
137
  // -- registration mechanics (validation only) -----------------------------
@@ -245,7 +240,16 @@ export class CloudflareScopeHost {
245
240
  * delivery cannot wedge the ones behind it. At-least-once still requires
246
241
  * idempotent handlers.
247
242
  */
248
- async drainExecutors(tenantId, scopeId) {
243
+ async drainExecutors(tenantId, scopeId,
244
+ /**
245
+ * #1525: the call this drain pass is running in, or null.
246
+ *
247
+ * The coordinator is the ONLY side that knows. Executors run here, not in the DO,
248
+ * so by the time the journal RPC arrives the DO's queued body has returned and its
249
+ * own `invocationId` field reads null — an ambient read there would record "no
250
+ * call" for every attempt an operation's own tail made.
251
+ */
252
+ invocationId) {
249
253
  const report = {
250
254
  attempted: 0,
251
255
  delivered: 0,
@@ -258,7 +262,16 @@ export class CloudflareScopeHost {
258
262
  const stub = this.scopeStub(scopeId);
259
263
  for (const [id, executor] of this.executors) {
260
264
  const deliveryId = `executor:${id}`;
261
- const events = await stub.pendingExecutorEvents(deliveryId, executor.eventType);
265
+ const { events, undecodable } = await stub.pendingExecutorDeliveries(deliveryId, executor.eventType);
266
+ // #1636: an event the DO could not decode is dead-lettered for this executor at once,
267
+ // and its handler never sees it. Terminal on the FIRST failure, unlike a handler's:
268
+ // the decode is pure, so a retry cannot succeed. The rows behind it are delivered
269
+ // below — the decode used to throw the whole list, on every pass.
270
+ for (const bad of undecodable) {
271
+ report.attempted += 1;
272
+ await stub.recordExecutorAttempt(bad.eventId, deliveryId, bad.error, null, invocationId);
273
+ report.deadLettered += 1;
274
+ }
262
275
  for (const event of events) {
263
276
  report.attempted += 1;
264
277
  this.causedBy = event.id;
@@ -272,17 +285,17 @@ export class CloudflareScopeHost {
272
285
  // the authority this host lacks. The intent row carries the retry
273
286
  // state from here on; the handler's own idempotency ledger absorbs
274
287
  // the at-least-once residue, as it already must in-process.
275
- await stub.routeExecutorEventToPlatform(event.id, deliveryId, connectorDispatchKind(executor.provider), JSON.stringify({ executorId: id, event }), JSON.stringify({ system: 'connector-dispatch' }));
288
+ await stub.routeExecutorEventToPlatform(event.id, deliveryId, connectorDispatchKind(executor.provider), JSON.stringify({ executorId: id, event }), JSON.stringify({ system: 'connector-dispatch' }), invocationId);
276
289
  report.routedToPlatform += 1;
277
290
  }
278
291
  else if (executor.kind === 'connector') {
279
292
  await executor.handler(await this.connectorContext(tenantId, scopeId, executor.timeoutMs, event.id), event);
280
- await stub.recordExecutorAttempt(event.id, deliveryId, null, null);
293
+ await stub.recordExecutorAttempt(event.id, deliveryId, null, null, invocationId);
281
294
  report.delivered += 1;
282
295
  }
283
296
  else {
284
297
  await executor.handler(this.admin, event);
285
- await stub.recordExecutorAttempt(event.id, deliveryId, null, null);
298
+ await stub.recordExecutorAttempt(event.id, deliveryId, null, null, invocationId);
286
299
  report.delivered += 1;
287
300
  }
288
301
  }
@@ -293,7 +306,7 @@ export class CloudflareScopeHost {
293
306
  const prior = await stub.executorAttempts(event.id, deliveryId);
294
307
  const attempts = prior + 1;
295
308
  const exhausted = attempts >= executor.retry.maxAttempts;
296
- await stub.recordExecutorAttempt(event.id, deliveryId, message, exhausted ? null : backoffAt(attempts, executor.retry, new Date()));
309
+ await stub.recordExecutorAttempt(event.id, deliveryId, message, exhausted ? null : backoffAt(attempts, executor.retry, new Date()), invocationId);
297
310
  if (exhausted)
298
311
  report.deadLettered += 1;
299
312
  else
@@ -311,7 +324,65 @@ export class CloudflareScopeHost {
311
324
  // does not get its effects driven either.
312
325
  await this.cp.validateScopeAccess(tenantId, scopeId);
313
326
  await this.migrateAndRecord(scopeId);
314
- return this.drainExecutors(tenantId, scopeId);
327
+ // #1525: null, and honestly so — a sweep is not a call. An attempt this pass makes
328
+ // records no invocation, which is what distinguishes it from the first attempt the
329
+ // emitting operation's own tail made.
330
+ return this.drainExecutors(tenantId, scopeId, null);
331
+ }
332
+ registerJob(moduleId, name, handler, retry) {
333
+ const key = `${moduleId}/${name}`;
334
+ if (this.jobs.has(key))
335
+ throw new Error(`job '${key}' is already registered`);
336
+ this.jobs.set(key, { handler, retry });
337
+ }
338
+ /**
339
+ * D-14's DURABLE driver: the run store is RPC to the scope DO, and nothing else.
340
+ *
341
+ * The pass engine stays on the coordinator — it is the same kernel code the pure
342
+ * adapter runs, so the two drivers cannot disagree about when a step is skipped or
343
+ * when a run fails. What crosses into the DO is a row read and a row write, which
344
+ * is also the only part that has to be durable.
345
+ *
346
+ * Every step costs a round trip, deliberately. Batching a pass's steps into one
347
+ * write at the end would be cheaper and would lose exactly the work an eviction
348
+ * mid-pass is supposed to keep — and a DO is evicted and revived constantly.
349
+ */
350
+ jobStore(scopeId) {
351
+ const stub = this.scopeStub(scopeId);
352
+ return {
353
+ startOrJoin: (key, row) => stub.jobRunStartOrJoin(key.moduleId, key.job, key.instance, row),
354
+ get: (id) => stub.jobRunById(id),
355
+ due: (now, limit, afterId) => stub.jobRunsDue(now, limit, afterId),
356
+ list: (filter) => stub.jobRunList(filter),
357
+ patch: (id, patch) => stub.jobRunPatch(id, patch),
358
+ commitPass: (id, patch) => stub.jobCommitPass(id, patch),
359
+ step: (runId, name) => stub.jobStepRow(runId, name),
360
+ recordStep: (runId, name, result, attempts, lastError, at) => stub.jobStepRecord(runId, name, result, attempts, lastError, at),
361
+ };
362
+ }
363
+ async startJobRun(tenantId, scopeId, input) {
364
+ await this.cp.validateScopeAccess(tenantId, scopeId);
365
+ await this.migrateAndRecord(scopeId);
366
+ return jobRunOf(await startJobRun(this.jobStore(scopeId), input, ulid, () => new Date().toISOString()));
367
+ }
368
+ async runDueJobs(tenantId, scopeId, options) {
369
+ // Same lifecycle gate as `drainDue`: a suspended scope's runs wait rather than
370
+ // advance, and an archived one's never move again.
371
+ await this.cp.validateScopeAccess(tenantId, scopeId);
372
+ await this.migrateAndRecord(scopeId);
373
+ return runDueJobRuns({
374
+ store: this.jobStore(scopeId),
375
+ handlerFor: (run) => this.jobs.get(`${run.module_id}/${run.job}`),
376
+ now: () => new Date().toISOString(),
377
+ openScope: (run) => this.getSystemScope(run.module_id, tenantId, scopeId),
378
+ maxPasses: options?.maxPasses,
379
+ limit: options?.limit,
380
+ });
381
+ }
382
+ async jobRuns(tenantId, scopeId, filter) {
383
+ await this.cp.validateScopeAccess(tenantId, scopeId);
384
+ await this.migrateAndRecord(scopeId);
385
+ return (await this.jobStore(scopeId).list(filter ?? {})).map(jobRunOf);
315
386
  }
316
387
  async dispatchConnector(tenantId, scopeId, handler, event, options) {
317
388
  // The platform half of a routed delivery (#574 phase 3). The same lifecycle gate as
@@ -344,12 +415,13 @@ export class CloudflareScopeHost {
344
415
  async listPlatformRequests(tenantId, scopeId) {
345
416
  await this.cp.validateScopeAccess(tenantId, scopeId);
346
417
  await this.migrateAndRecord(scopeId);
347
- return (await this.scopeStub(scopeId).pendingPlatformRequests()).map(rowToPlatformRequest);
418
+ // Tolerant (#1588): one undecodable row comes back naming why, never throws for the list.
419
+ return (await this.scopeStub(scopeId).pendingPlatformRequests()).map(platformRequestOf);
348
420
  }
349
421
  async listPlatformRequestHistory(tenantId, scopeId, filter) {
350
422
  await this.cp.validateScopeAccess(tenantId, scopeId);
351
423
  await this.migrateAndRecord(scopeId);
352
- return (await this.scopeStub(scopeId).platformRequestHistory(filter)).map(rowToPlatformRequest);
424
+ return (await this.scopeStub(scopeId).platformRequestHistory(filter)).map(platformRequestOf);
353
425
  }
354
426
  async settlePlatformRequest(tenantId, scopeId, id, outcome) {
355
427
  await this.cp.validateScopeAccess(tenantId, scopeId);
@@ -528,13 +600,17 @@ export class CloudflareScopeHost {
528
600
  async appliedMigrationsLocal(scopeId) {
529
601
  return this.scopeStub(scopeId).appliedMigrations();
530
602
  }
603
+ /** This host's own scope's database size in bytes (#1524), the vertical-host read. */
604
+ async databaseSizeLocal(scopeId) {
605
+ return this.scopeStub(scopeId).databaseSize();
606
+ }
531
607
  /**
532
608
  * The oldest not-yet-drained events of this host's own scope (#1334) — the far end of
533
609
  * the control plane's `EventDrainDelegation`. Bounded the same way the audited verb is,
534
610
  * so a caller cannot ask this side for more than the other would.
535
611
  */
536
612
  async undrainedEventsLocal(scopeId, limit) {
537
- return this.scopeStub(scopeId).undrainedEvents(Math.min(Math.max(limit, 1), 1000));
613
+ return undrainedEventsOf(await this.scopeStub(scopeId).undrainedEventsRead(Math.min(Math.max(limit, 1), 1000)));
538
614
  }
539
615
  /**
540
616
  * Stamp `drained_at` on this host's own scope (#1334) — the delegation's other half.
@@ -552,6 +628,16 @@ export class CloudflareScopeHost {
552
628
  async redrainEventsLocal(scopeId, drainedBefore) {
553
629
  return this.scopeStub(scopeId).redrainEvents(drainedBefore);
554
630
  }
631
+ /**
632
+ * How many rows that reopen would touch, in this host's own scope (#1545) — the
633
+ * read-only half a dry run asks for. A separate method rather than a flag on the one
634
+ * above: the two return the same shape, so a lost argument would turn a count into a
635
+ * reopen silently, and a name cannot be lost that way. No audit, like the reopen: the
636
+ * platform's `redrainEvents` is the door, and a count egresses nothing to record.
637
+ */
638
+ async redrainCountLocal(scopeId, drainedBefore) {
639
+ return this.scopeStub(scopeId).redrainCount(drainedBefore);
640
+ }
555
641
  /**
556
642
  * The PITR bookmarks one scope recorded before its migration passes (#286) — the
557
643
  * rewind points a backout UI offers. Behind the vertical's platform-gated
@@ -651,14 +737,18 @@ export class CloudflareScopeHost {
651
737
  // holds exactly the permissions its schedules declared, on this scope, so
652
738
  // `ctx.check` resolves for scheduled work (the gate stays the check). Written to
653
739
  // the scope's own tuples, where the checker reads them — the same place the owner
654
- // grant and connection grants land. Idempotent, so a re-provision re-asserts them.
740
+ // grant and connection grants land. Idempotent, so a re-provision re-asserts them —
741
+ // SEATED (#1659): a missing grant is recreated, a revoked one stays revoked. The
742
+ // per-scope schedule kill switch is `revokeFromSystem` (#1666), and its OFF marker is
743
+ // not a grant, so no reconcile can seat it away; `restoreToSystem` is the only way back.
744
+ // (`grantToSystem` still clears a tuple's tombstone, but it does not move the switch.)
655
745
  for (const [moduleId, schedules] of this.moduleSchedules) {
656
746
  const perms = new Set();
657
747
  for (const s of schedules)
658
748
  for (const p of s.permissions)
659
749
  perms.add(p);
660
750
  for (const perm of perms) {
661
- await this.scopeStub(input.scopeId).writeTuple(`system:${moduleId}`, `granted:${perm}`, `scope:${input.scopeId}`, null);
751
+ await this.scopeStub(input.scopeId).seatTuple(`system:${moduleId}`, `granted:${perm}`, `scope:${input.scopeId}`, null);
662
752
  }
663
753
  }
664
754
  // Audit a real provision only; an idempotent re-provision changed nothing.
@@ -1137,6 +1227,59 @@ export class CloudflareScopeHost {
1137
1227
  await this.migrateAndRecord(scopeId);
1138
1228
  return this.buildStub(tenantId, scopeId, undefined, undefined, moduleId);
1139
1229
  }
1230
+ /**
1231
+ * The exchange (#1672) — the same fail-closed (tenant, scope) and lifecycle gate as
1232
+ * `getScope`, then the kernel's `exchangeCapability` inside the ScopeDO, where the
1233
+ * capability row lives. The answer is re-parsed on this side of the RPC.
1234
+ */
1235
+ async exchangeCapability(tenantId, scopeId, secret, options) {
1236
+ await this.cp.validateScopeAccess(tenantId, scopeId);
1237
+ await this.migrateAndRecord(scopeId);
1238
+ const outcome = await this.scopeStub(scopeId).exchangeCapability(secret, tenantId, scopeId, options?.mode);
1239
+ return capabilityExchange.nullable().parse(outcome);
1240
+ }
1241
+ /**
1242
+ * The capability door (#1672) — mirror of the connection, system and impersonation doors.
1243
+ * The token is shape-checked and hashed HERE; only its hash reaches the ScopeDO, which
1244
+ * re-resolves it to its capability on every invoke and acknowledges that it did.
1245
+ */
1246
+ async getCapabilityScope(sessionToken, tenantId, scopeId, options) {
1247
+ if (!plausibleSessionToken(sessionToken)) {
1248
+ throw substratError('unauthenticated', 'not a capability session token');
1249
+ }
1250
+ await this.cp.validateScopeAccess(tenantId, scopeId);
1251
+ await this.migrateAndRecord(scopeId);
1252
+ const hash = await capabilityTokenHash(sessionToken);
1253
+ return this.buildStub(tenantId, scopeId, undefined, undefined, undefined, options, undefined, hash);
1254
+ }
1255
+ /**
1256
+ * Is this the SHARED control plane's host — the one whose own `SCOPE` namespace holds no
1257
+ * hosted scope's storage, and which routes scope writes to the serving deployment? The
1258
+ * delegations are set on that host and no other (their own docs say so), so any one of
1259
+ * them being present is the signal.
1260
+ */
1261
+ get servesScopesElsewhere() {
1262
+ return Boolean(this.connectorDelegation || this.systemSwitchDelegation || this.eventDrainDelegation);
1263
+ }
1264
+ /**
1265
+ * Where a platform capability verb (#1672) may write: this host's own ScopeDO — refused,
1266
+ * loudly, on the shared control plane for a scope bound to a vertical, whose storage lives
1267
+ * in that vertical's deployment. Writing through `this.scopeStub` there would mint into
1268
+ * an empty DO of the wrong namespace, a capability nobody could ever exchange. Delegating
1269
+ * these verbs to the serving deployment (as the schedule switch is) is a follow-up; the
1270
+ * claim-link migration is the first caller that needs it.
1271
+ */
1272
+ async capabilityScopeStub(tenantId, scopeId, verb) {
1273
+ const rec = await this.cp.getScopeRecord(tenantId, scopeId);
1274
+ if (!rec)
1275
+ throw substratError('not_found', `unknown scope for tenant: (${tenantId}, ${scopeId})`);
1276
+ if (this.servesScopesElsewhere && rec.vertical !== null) {
1277
+ throw substratError('unavailable', `${verb} cannot reach scope ${scopeId}: it is served by the '${rec.vertical}' ` +
1278
+ 'deployment, and platform capability verbs are not delegated there yet');
1279
+ }
1280
+ await this.migrateAndRecord(scopeId);
1281
+ return { stub: this.scopeStub(scopeId), vertical: rec.vertical };
1282
+ }
1140
1283
  registeredSchedules() {
1141
1284
  const out = [];
1142
1285
  for (const [moduleId, schedules] of this.moduleSchedules) {
@@ -1196,7 +1339,8 @@ export class CloudflareScopeHost {
1196
1339
  const heartbeatDue = p.stateAt === null || now - Date.parse(p.stateAt) > FRESHNESS_HEARTBEAT_MINUTES * 60_000;
1197
1340
  if (!changed && !heartbeatDue)
1198
1341
  continue;
1199
- await stub.recordScheduleRun(`freshness:${eventType}`, nowIso, outcome);
1342
+ // #1288: 'freshness', said rather than inferred from the key's prefix.
1343
+ await stub.recordScheduleRun(`freshness:${eventType}`, nowIso, outcome, 'freshness');
1200
1344
  report.checks.push({ eventType, outcome, observedAt: p.observedAt, withinHours });
1201
1345
  }
1202
1346
  return report;
@@ -1217,11 +1361,22 @@ export class CloudflareScopeHost {
1217
1361
  return report;
1218
1362
  }
1219
1363
  const stub = this.scopeStub(scopeId);
1220
- // The grant IS the switch (#383): run only where the scope holds a live
1221
- // `system:<moduleId>` grant. Skips a foreign-vertical scope and a per-tenant
1222
- // revoke quietly — no run, no error.
1223
- if (!(await stub.hasSystemGrant(moduleId)))
1364
+ // The grant IS the switch (#383), and the kill switch is its lever (#1666): the
1365
+ // kernel's `systemScheduleState`, the one predicate both adapters run. A scope that
1366
+ // never held the module's grant (a foreign vertical's) is a quiet no-op, exactly as
1367
+ // before. A scope switched OFF reports every schedule `skipped`, never `failed`, and
1368
+ // does not touch its cadence rows — so a restore fires a due schedule on the next pass.
1369
+ const state = await stub.systemScheduleState(moduleId);
1370
+ if (state === 'ungranted')
1224
1371
  return report;
1372
+ if (state === 'off') {
1373
+ for (const schedule of schedules) {
1374
+ report.skipped += 1;
1375
+ report.runs.push({ operation: schedule.operation, outcome: 'skipped' });
1376
+ }
1377
+ report.switchedOff = true;
1378
+ return report;
1379
+ }
1225
1380
  const now = Date.now();
1226
1381
  for (const schedule of schedules) {
1227
1382
  const last = await stub.scheduleLastRun(schedule.operation);
@@ -1246,7 +1401,9 @@ export class CloudflareScopeHost {
1246
1401
  error: err instanceof Error ? err.message : String(err),
1247
1402
  });
1248
1403
  }
1249
- await stub.recordScheduleRun(schedule.operation, new Date(now).toISOString(), status);
1404
+ // #1288: 'schedule', whatever this operation happens to be called — including
1405
+ // `freshness:<something>`, which is exactly the row the evaluator no longer eats.
1406
+ await stub.recordScheduleRun(schedule.operation, new Date(now).toISOString(), status, 'schedule');
1250
1407
  report.runs.push({ operation: schedule.operation, outcome: status === 'ok' ? 'ok' : 'failed' });
1251
1408
  }
1252
1409
  return report;
@@ -1260,16 +1417,25 @@ export class CloudflareScopeHost {
1260
1417
  * time box that never runs out for the one caller holding it, and
1261
1418
  * `endImpersonation` would stop nothing.
1262
1419
  */
1263
- sessionId) {
1420
+ sessionId,
1421
+ /**
1422
+ * #1672: the HASH of a capability session token — the capability door hashed it and
1423
+ * the plaintext goes no further. The ScopeDO resolves it on every invoke, inside its
1424
+ * queue, and acknowledges it; see the refusal below for why the acknowledgement matters.
1425
+ */
1426
+ capabilitySession) {
1264
1427
  const stub = this.scopeStub(scopeId);
1265
1428
  const cp = this.cp;
1266
1429
  const operationEntitlement = this.operationEntitlement;
1267
1430
  // The DO needs SOME principal-shaped value for `ctx.principal`; for a
1268
1431
  // connection it is the connection id, for a schedule the module id, and the
1269
- // honest attribution rides on the event actor instead.
1432
+ // honest attribution rides on the event actor instead. For a capability it is a
1433
+ // FRESH id nobody holds anything under (#1672): the DO replaces it with the resolved
1434
+ // capability, and a DO too old to know that would act as nobody rather than as someone.
1270
1435
  const asPrincipalId = (principal ??
1271
1436
  connectionId ??
1272
- systemModuleId);
1437
+ systemModuleId ??
1438
+ (capabilitySession !== undefined ? principalId.parse(ulid()) : undefined));
1273
1439
  return {
1274
1440
  tenantId,
1275
1441
  scopeId,
@@ -1291,7 +1457,7 @@ export class CloudflareScopeHost {
1291
1457
  }
1292
1458
  // K-42: resolved per invoke, so expiry and `endImpersonation` both bite.
1293
1459
  const session = sessionId === undefined ? undefined : await this.resolveImpersonation(sessionId, tenantId, scopeId);
1294
- const envelope = await stub.invoke(operation, input, asPrincipalId, tenantId, scopeId, connectionId, requiredKey, systemModuleId, true, invokeOptions, session);
1460
+ const envelope = await stub.invoke(operation, input, asPrincipalId, tenantId, scopeId, connectionId, requiredKey, systemModuleId, true, invokeOptions, session, capabilitySession);
1295
1461
  // The operation failed and the DO handed the error back as DATA — so it still
1296
1462
  // has its code and extensions, which a throw across this boundary would have
1297
1463
  // stripped down to a message (#113 §3). Rethrown here, where the caller expects
@@ -1325,6 +1491,14 @@ export class CloudflareScopeHost {
1325
1491
  // principal with nothing stamped and no read-only bound. The write has
1326
1492
  // committed; what this refuses is the SUCCESS, because a support session
1327
1493
  // believed to be recorded and bounded is worse than no support session.
1494
+ // #1672, the same shape: a DO too old to know about capability sessions ignored the
1495
+ // hash and ran the call as the fresh placeholder principal, who holds nothing — so
1496
+ // the likely outcome is a refusal, but a SUCCESS here would have been decided
1497
+ // without the capability's grant, its allowlist or its liveness, and is refused.
1498
+ if (capabilitySession !== undefined && envelope.capability?.honoured !== true) {
1499
+ throw substratError('unavailable', `${operation} ran on a scope host that did not understand capability sessions. ` +
1500
+ 'Retry once the scope has been migrated');
1501
+ }
1328
1502
  if (session !== undefined && envelope.impersonation?.honoured !== true) {
1329
1503
  throw substratError('unavailable', `${operation} ran on a scope host that did not understand impersonation — the ` +
1330
1504
  'operation may have run unrecorded and unbounded. Retry once the scope has ' +
@@ -1337,7 +1511,9 @@ export class CloudflareScopeHost {
1337
1511
  // Whatever the outbox already held is the drain sweep's own backstop.
1338
1512
  const drained = session?.mode === 'read-only'
1339
1513
  ? { attempted: 0, delivered: 0, retrying: 0, deadLettered: 0, routedToPlatform: 0 }
1340
- : await this.drainExecutors(tenantId, scopeId);
1514
+ // #1525: this drain is part of the call that emitted the events — the
1515
+ // coordinator's half of the post-commit tail the DO ran the consumers in.
1516
+ : await this.drainExecutors(tenantId, scopeId, invokeOptions?.invocationId ?? null);
1341
1517
  // #458: the operation committed having enqueued platform intents — tell the
1342
1518
  // caller's harness so it can flag the response for the router kick (#381).
1343
1519
  // Routed connector deliveries (#574 phase 3) count too: the inline drain just
@@ -1479,6 +1655,74 @@ export class CloudflareScopeHost {
1479
1655
  await this.cp.writeTenantTuple(node.tenantId, subject, `granted:${permission}`, `tenant:${node.tenantId}`, expiresAt ?? null);
1480
1656
  }
1481
1657
  };
1658
+ /**
1659
+ * #1666: move one module's schedule switch on one scope — see `HostAdmin.revokeFromSystem`.
1660
+ *
1661
+ * Where the write lands is the whole point of the delegation branch. The shared control
1662
+ * plane's own `SCOPE` namespace is the module-less placeholder, so a hosted scope's
1663
+ * switch has to be moved in the deployment serving it; writing it here would report a
1664
+ * switch pulled while every schedule kept firing. The audit row is written HERE either
1665
+ * way — the deployment's host is CP-less, and its `recordAdmin` is a no-op.
1666
+ */
1667
+ const switchSystem = async (actor, raw, to) => {
1668
+ const input = systemSwitch.parse(raw);
1669
+ const { tenantId, scopeId } = input.node;
1670
+ let vertical = null;
1671
+ if (!this.cpLess) {
1672
+ const rec = await this.cp.getScopeRecord(tenantId, scopeId);
1673
+ if (!rec)
1674
+ throw substratError('not_found', `unknown scope for tenant: (${tenantId}, ${scopeId})`);
1675
+ vertical = rec.vertical;
1676
+ }
1677
+ // AUDIT FIRST (#1666 review): the intent row lands before anything moves, and the
1678
+ // outcome row after — every attempt, a repeat included. The scope's store and the
1679
+ // admin log are separate, so no order makes the pair atomic; this one fails toward
1680
+ // "an intent with no recorded outcome" and never toward "a switch that moved with no
1681
+ // audit row". A retry after a crash re-audits even though it answers `changed: false`.
1682
+ const operationId = ulid();
1683
+ const action = to === 'off' ? 'revokeFromSystem' : 'restoreToSystem';
1684
+ const target = { tenantId, scopeId, vertical };
1685
+ const base = { operationId, moduleId: input.moduleId, schedules: to };
1686
+ await this.recordAdmin(actor, action, target, null, { ...base, phase: 'intent', reason: input.reason });
1687
+ // A scope bound to no vertical (#1666 review) has no deployment to delegate to: its
1688
+ // store is the DO here, so the switch moves (or answers `held: false`) here too.
1689
+ // Delegating it would throw "no deployment serving scope" instead. A scope WITH a
1690
+ // vertical still delegates, and still fails loudly when none serves it.
1691
+ const delegation = this.cpLess || vertical !== null ? this.systemSwitchDelegation : undefined;
1692
+ let outcome;
1693
+ try {
1694
+ outcome = delegation
1695
+ ? await delegation.switch({ tenantId, scopeId, moduleId: input.moduleId, to })
1696
+ : await this.scopeStub(scopeId).switchSystemSchedules(input.moduleId, scopeId, to, new Date().toISOString());
1697
+ }
1698
+ catch (err) {
1699
+ // Best effort: the original error is what the caller must see, and the intent row
1700
+ // already says an attempt was made.
1701
+ await this.recordAdmin(actor, action, target, null, {
1702
+ ...base,
1703
+ phase: 'failed',
1704
+ error: err instanceof Error ? err.message : String(err),
1705
+ }).catch(() => undefined);
1706
+ throw err;
1707
+ }
1708
+ await this.recordAdmin(actor, action, target, null, {
1709
+ ...base,
1710
+ phase: outcome.held ? 'applied' : 'refused',
1711
+ changed: outcome.changed,
1712
+ permissions: outcome.permissions,
1713
+ });
1714
+ if (!outcome.held) {
1715
+ throw substratError('not_found', `scope ${scopeId} holds no system grant for module '${input.moduleId}' — nothing to switch ${to} ` +
1716
+ `(check the module id: it is the module's manifest id, e.g. '@substrat-run/engine-absence')`);
1717
+ }
1718
+ return {
1719
+ operationId,
1720
+ moduleId: input.moduleId,
1721
+ schedules: to,
1722
+ changed: outcome.changed,
1723
+ permissions: outcome.permissions,
1724
+ };
1725
+ };
1482
1726
  return {
1483
1727
  // #603: fixed at construction — a worker deployed without SECRET_BOX_KEY can never
1484
1728
  // store a credential, and saying so is what lets a transport answer 503 instead of 500.
@@ -1617,11 +1861,59 @@ export class CloudflareScopeHost {
1617
1861
  // The scheduler's grant (#383) — mirror of grantToConnection. Narrow: one
1618
1862
  // module, one permission; tombstones on revoke; shows in the permission diff.
1619
1863
  const grant = systemGrant.parse(raw);
1620
- await writeGrant(subjectRef({ kind: 'system', id: grant.moduleId }), grant.permission, grant.node, undefined, grant.expiresAt);
1864
+ if (grant.node.scopeId) {
1865
+ // #1666: refused while the module is switched off on this scope — checked and
1866
+ // written in one DO unit. Restore is the lever; a grant is not.
1867
+ const written = await this.scopeStub(grant.node.scopeId).writeSystemGrant(grant.moduleId, `granted:${grant.permission}`, `scope:${grant.node.scopeId}`, grant.expiresAt ?? null);
1868
+ if (!written) {
1869
+ throw substratError('conflict', systemSwitchedOffMessage(grant.moduleId, grant.node.scopeId));
1870
+ }
1871
+ }
1872
+ else {
1873
+ await writeGrant(subjectRef({ kind: 'system', id: grant.moduleId }), grant.permission, grant.node, undefined, grant.expiresAt);
1874
+ }
1621
1875
  await this.recordAdmin(actor, 'grantToSystem', { tenantId: grant.node.tenantId, scopeId: grant.node.scopeId }, null, { moduleId: grant.moduleId, permission: grant.permission, node: grant.node });
1622
1876
  if (!grant.node.scopeId)
1623
1877
  await this.fanOut(grant.node.tenantId);
1624
1878
  },
1879
+ // #1666: the schedule kill switch and its lever back — `system-switch.ts` is the
1880
+ // whole rule, shared with the pure adapter; this is the directory check, the reach
1881
+ // into the scope's storage, and the audit row around it.
1882
+ revokeFromSystem: async (actor, raw) => switchSystem(actor, raw, 'off'),
1883
+ restoreToSystem: async (actor, raw) => switchSystem(actor, raw, 'on'),
1884
+ // #1672 — the platform's two capability verbs. Audited AFTER the write, on both, and
1885
+ // the failure each leaves is the safe one: a mint whose audit row did not land never
1886
+ // returned its secret, so nobody can ever exchange it; a revoke whose row did not land
1887
+ // has still revoked. Neither the secret nor its hash is ever in before/after.
1888
+ mintCapability: async (actor, tenantId, scopeId, input) => {
1889
+ // Checked HERE as well as in the DO: a typed refusal thrown across the RPC arrives as
1890
+ // a bare message, and the caller would lose `validation_failed`. One function, so the
1891
+ // two sides cannot disagree about what a valid mint is.
1892
+ checkBecomeInput(input, new Date().toISOString());
1893
+ await this.cp.validateScopeAccess(tenantId, scopeId);
1894
+ const { stub, vertical } = await this.capabilityScopeStub(tenantId, scopeId, 'mintCapability');
1895
+ const minted = await stub.mintBecomeCapability(input, actor);
1896
+ await this.recordAdmin(actor, 'mintCapability', { tenantId, scopeId, vertical }, null, {
1897
+ capabilityId: minted.id,
1898
+ mode: 'become',
1899
+ principal: input.principal,
1900
+ expiresAt: input.expiresAt,
1901
+ maxUses: input.maxUses,
1902
+ label: input.label ?? null,
1903
+ });
1904
+ return minted;
1905
+ },
1906
+ revokeCapability: async (actor, tenantId, scopeId, capabilityId) => {
1907
+ const { stub, vertical } = await this.capabilityScopeStub(tenantId, scopeId, 'revokeCapability');
1908
+ const before = await stub.revokeCapabilityAsPlatform(capabilityId, actor);
1909
+ if (!before) {
1910
+ throw substratError('not_found', `no capability ${capabilityId} in scope ${scopeId}`);
1911
+ }
1912
+ await this.recordAdmin(actor, 'revokeCapability', { tenantId, scopeId, vertical }, before, {
1913
+ capabilityId,
1914
+ revoked: true,
1915
+ });
1916
+ },
1625
1917
  grantToOrg: async (actor, orgId, permission, node, entity) => {
1626
1918
  // The org must exist in the node's tenant. A grant to a phantom org looks
1627
1919
  // applied, resolves for nobody, and still shows up in the permission diff.
@@ -1781,7 +2073,7 @@ export class CloudflareScopeHost {
1781
2073
  const parsed = publishVersionInput.parse(input);
1782
2074
  const owning = await this.cp.readVertical(parsed.verticalSlug);
1783
2075
  if (!owning) {
1784
- throw new Error(`unknown vertical '${parsed.verticalSlug}'`);
2076
+ throw substratError('not_found', `unknown vertical '${parsed.verticalSlug}'`);
1785
2077
  }
1786
2078
  // Lands PENDING — a push is not a deploy — except for a PRIVATE vertical
1787
2079
  // (tenant-owned, not listed), whose blast radius is its own tenant: there the
@@ -1820,7 +2112,7 @@ export class CloudflareScopeHost {
1820
2112
  setVerticalListed: async (actor, slug, listed) => {
1821
2113
  const existing = await this.cp.readVertical(slug);
1822
2114
  if (!existing)
1823
- throw new Error(`unknown vertical '${slug}'`);
2115
+ throw substratError('not_found', `unknown vertical '${slug}'`);
1824
2116
  // Listing is the moment other tenants start trusting this code, so the
1825
2117
  // version they would install must carry a real staff vouch — an auto-admitted
1826
2118
  // prod version has never been read by anyone but its author.
@@ -1838,35 +2130,35 @@ export class CloudflareScopeHost {
1838
2130
  requestPublish: async (actor, slug) => {
1839
2131
  const existing = await this.cp.readVertical(slug);
1840
2132
  if (!existing)
1841
- throw new Error(`unknown vertical '${slug}'`);
2133
+ throw substratError('not_found', `unknown vertical '${slug}'`);
1842
2134
  await this.cp.updateVerticalPublishRequest(slug, new Date().toISOString());
1843
2135
  await this.recordAdmin(actor, 'requestPublish', { tenantId: null }, null, { slug });
1844
2136
  },
1845
2137
  setVerticalInstallsBlocked: async (actor, slug, blocked) => {
1846
2138
  const existing = await this.cp.readVertical(slug);
1847
2139
  if (!existing)
1848
- throw new Error(`unknown vertical '${slug}'`);
2140
+ throw substratError('not_found', `unknown vertical '${slug}'`);
1849
2141
  await this.cp.updateVerticalInstallsBlocked(slug, blocked ? 1 : 0);
1850
2142
  await this.recordAdmin(actor, 'setVerticalInstallsBlocked', { tenantId: null }, { installsBlocked: !!existing.installs_blocked }, { installsBlocked: blocked });
1851
2143
  },
1852
2144
  setVerticalTenantProvisioner: async (actor, slug, granted) => {
1853
2145
  const existing = await this.cp.readVertical(slug);
1854
2146
  if (!existing)
1855
- throw new Error(`unknown vertical '${slug}'`);
2147
+ throw substratError('not_found', `unknown vertical '${slug}'`);
1856
2148
  await this.cp.updateVerticalTenantProvisioner(slug, granted ? 1 : 0);
1857
2149
  await this.recordAdmin(actor, 'setVerticalTenantProvisioner', { tenantId: null }, { tenantProvisioner: !!existing.tenant_provisioner }, { tenantProvisioner: granted });
1858
2150
  },
1859
2151
  setVerticalEmailSender: async (actor, slug, granted) => {
1860
2152
  const existing = await this.cp.readVertical(slug);
1861
2153
  if (!existing)
1862
- throw new Error(`unknown vertical '${slug}'`);
2154
+ throw substratError('not_found', `unknown vertical '${slug}'`);
1863
2155
  await this.cp.updateVerticalEmailSender(slug, granted ? 1 : 0);
1864
2156
  await this.recordAdmin(actor, 'setVerticalEmailSender', { tenantId: null }, { emailSender: !!existing.email_sender }, { emailSender: granted });
1865
2157
  },
1866
2158
  deleteVertical: async (actor, slug) => {
1867
2159
  const existing = await this.cp.readVertical(slug);
1868
2160
  if (!existing)
1869
- throw new Error(`unknown vertical '${slug}'`);
2161
+ throw substratError('not_found', `unknown vertical '${slug}'`);
1870
2162
  // Refuse while any restorable scope is bound: a deleted registry row would strand
1871
2163
  // those scopes' version pins and routing. An `archived` scope (a deleted app) still
1872
2164
  // blocks — unarchive can bring it back — but the refusal names reap/restore, not
@@ -1875,10 +2167,10 @@ export class CloudflareScopeHost {
1875
2167
  // for the cleanup script (#248).
1876
2168
  const bound = await this.cp.countScopesForVertical(slug);
1877
2169
  if (bound.live > 0) {
1878
- throw new Error(`vertical '${slug}' still backs ${bound.live} scope(s) — delete or rebind them first`);
2170
+ throw substratError('conflict', `vertical '${slug}' still backs ${bound.live} scope(s) — delete or rebind them first`);
1879
2171
  }
1880
2172
  if (bound.archived > 0) {
1881
- throw new Error(`vertical '${slug}' still backs ${bound.archived} archived scope(s) — reap or restore them first`);
2173
+ throw substratError('conflict', `vertical '${slug}' still backs ${bound.archived} archived scope(s) — reap or restore them first`);
1882
2174
  }
1883
2175
  await this.cp.deleteVertical(slug);
1884
2176
  await this.recordAdmin(actor, 'deleteVertical', { tenantId: null }, { slug, source: existing.source, ownerTenant: existing.owner_tenant }, null);
@@ -1886,7 +2178,7 @@ export class CloudflareScopeHost {
1886
2178
  admitVersion: async (actor, versionId) => {
1887
2179
  const v = await this.cp.readVersion(versionId);
1888
2180
  if (!v)
1889
- throw new Error(`unknown version ${versionId}`);
2181
+ throw substratError('not_found', `unknown version ${versionId}`);
1890
2182
  if (v.admission === 'admitted') {
1891
2183
  // Idempotent — except an AUTO-admitted version, which this upgrades to a
1892
2184
  // manual vouch by clearing the note (what the publish seam requires).
@@ -1897,7 +2189,7 @@ export class CloudflareScopeHost {
1897
2189
  return;
1898
2190
  }
1899
2191
  if (v.admission === 'rejected') {
1900
- throw new Error(`version ${versionId} was rejected — publish a new one`);
2192
+ throw substratError('conflict', `version ${versionId} was rejected — publish a new one`);
1901
2193
  }
1902
2194
  await this.cp.setAdmission(versionId, 'admitted', null);
1903
2195
  await this.recordAdmin(actor, 'admitVersion', { tenantId: null }, { admission: v.admission }, { admission: 'admitted' });
@@ -1905,7 +2197,7 @@ export class CloudflareScopeHost {
1905
2197
  rejectVersion: async (actor, versionId, note) => {
1906
2198
  const v = await this.cp.readVersion(versionId);
1907
2199
  if (!v)
1908
- throw new Error(`unknown version ${versionId}`);
2200
+ throw substratError('not_found', `unknown version ${versionId}`);
1909
2201
  if (v.admission === 'admitted') {
1910
2202
  throw new Error(`version ${versionId} is already admitted — it may be bound`);
1911
2203
  }
@@ -1917,7 +2209,7 @@ export class CloudflareScopeHost {
1917
2209
  promoteVersion: async (actor, verticalSlug, channel, versionId, acknowledge) => {
1918
2210
  const incoming = await this.cp.readVersion(versionId);
1919
2211
  if (!incoming)
1920
- throw new Error(`unknown version ${versionId}`);
2212
+ throw substratError('not_found', `unknown version ${versionId}`);
1921
2213
  if (incoming.vertical_slug !== verticalSlug) {
1922
2214
  throw new Error(`version ${versionId} belongs to '${incoming.vertical_slug}'`);
1923
2215
  }
@@ -2022,7 +2314,7 @@ export class CloudflareScopeHost {
2022
2314
  bindScopeVersion: async (actor, tenantId, scopeId, versionId, opts) => {
2023
2315
  const v = await this.cp.readVersion(versionId);
2024
2316
  if (!v)
2025
- throw new Error(`unknown version ${versionId}`);
2317
+ throw substratError('not_found', `unknown version ${versionId}`);
2026
2318
  const scope = await this.cp.getScopeRecord(tenantId, scopeId);
2027
2319
  if (!scope)
2028
2320
  throw new Error(`unknown scope ${scopeId} in tenant ${tenantId}`);
@@ -2068,7 +2360,7 @@ export class CloudflareScopeHost {
2068
2360
  verticalServing: async (actor, verticalSlug) => {
2069
2361
  const r = await this.cp.readVertical(verticalSlug);
2070
2362
  if (!r)
2071
- throw new Error(`unknown vertical '${verticalSlug}'`);
2363
+ throw substratError('not_found', `unknown vertical '${verticalSlug}'`);
2072
2364
  await this.recordAccess(actor, 'verticalServing', {}, { verticalSlug }, r.serving_ref ? 1 : 0);
2073
2365
  if (!r.serving_ref || !r.serving_version_id || !r.serving_migration_tag)
2074
2366
  return null;
@@ -2083,7 +2375,7 @@ export class CloudflareScopeHost {
2083
2375
  const parsed = verticalServingState.parse(state);
2084
2376
  const r = await this.cp.readVertical(verticalSlug);
2085
2377
  if (!r)
2086
- throw new Error(`unknown vertical '${verticalSlug}'`);
2378
+ throw substratError('not_found', `unknown vertical '${verticalSlug}'`);
2087
2379
  await this.cp.setVerticalServing(verticalSlug, {
2088
2380
  ref: parsed.ref,
2089
2381
  versionId: parsed.versionId,
@@ -2097,7 +2389,7 @@ export class CloudflareScopeHost {
2097
2389
  versionManifest: async (actor, verticalSlug, versionId) => {
2098
2390
  const v = await this.cp.readVersion(versionId);
2099
2391
  if (!v || v.vertical_slug !== verticalSlug) {
2100
- throw new Error(`unknown version ${versionId} for vertical '${verticalSlug}'`);
2392
+ throw substratError('not_found', `unknown version ${versionId} for vertical '${verticalSlug}'`);
2101
2393
  }
2102
2394
  await this.recordAccess(actor, 'versionManifest', {}, { verticalSlug, versionId }, v.manifest_json ? 1 : 0);
2103
2395
  return v.manifest_json;
@@ -2132,6 +2424,13 @@ export class CloudflareScopeHost {
2132
2424
  await this.recordAccess(actor, 'scopeMigrationBookmarks', { tenantId, scopeId }, null, bookmarks.length);
2133
2425
  return bookmarks;
2134
2426
  },
2427
+ scopeDatabaseSize: async (actor, tenantId, scopeId) => {
2428
+ // Reaped is refused, not read: addressing the deleted DO would recreate it.
2429
+ await this.scopeRecordForRead(tenantId, scopeId);
2430
+ const bytes = await this.scopeStub(scopeId).databaseSize();
2431
+ await this.recordAccess(actor, 'scopeDatabaseSize', { tenantId, scopeId }, null, 1);
2432
+ return bytes;
2433
+ },
2135
2434
  rewindScope: async (actor, tenantId, scopeId, bookmark, opts) => {
2136
2435
  const scope = await this.cp.getScopeRecord(tenantId, scopeId);
2137
2436
  if (!scope)
@@ -2305,7 +2604,7 @@ export class CloudflareScopeHost {
2305
2604
  vertical: record.vertical,
2306
2605
  limit: bounded,
2307
2606
  })
2308
- : await this.scopeStub(scopeId).undrainedEvents(bounded);
2607
+ : undrainedEventsOf(await this.scopeStub(scopeId).undrainedEventsRead(bounded));
2309
2608
  await this.recordAccess(actor, 'readUndrainedEvents', { tenantId, scopeId }, { limit }, events.length);
2310
2609
  return events;
2311
2610
  },
@@ -2342,11 +2641,39 @@ export class CloudflareScopeHost {
2342
2641
  // The same refusal the stamp carries: a reaped scope's storage is gone, and
2343
2642
  // addressing its DO would construct an empty one and report nothing reopened.
2344
2643
  const record = await this.scopeRecordForRead(tenantId, scopeId);
2345
- const { drainedBefore } = redrainEventsInput.parse(input);
2644
+ const { drainedBefore, countOnly } = redrainEventsInput.parse(input);
2346
2645
  // The window rule at the HostAdmin boundary, not only at the control-plane door:
2347
2646
  // this verb is public, so an in-process caller reaches it without that route. Host
2348
2647
  // code, so the real clock is the right one to read (the DO host injects none).
2648
+ // Applied to a count too: a future instant is as meaningless to count as it is
2649
+ // dangerous to reopen, and one answer from this verb should not be reachable
2650
+ // through a door the other is refused at.
2349
2651
  assertRedrainWindow(drainedBefore, new Date().toISOString());
2652
+ // A COUNT reopens nothing, so it writes no receipt (#1545). Both rows below exist
2653
+ // for a second egress of a tenant's payloads: the intent because a reopen that
2654
+ // crashed before its outcome row would otherwise leave no trace, the outcome
2655
+ // because something moved. A count moves nothing and egresses nothing, and an
2656
+ // admin row saying a redrain was intended on a scope where none was is a false
2657
+ // statement about a tenant's data — the opposite of what the log is for.
2658
+ //
2659
+ // It IS a read, though, and K-24 admits no curated subset: every `HostAdmin` read
2660
+ // records actor, method, target and result count in the access log, so "who counted
2661
+ // every tenant's outbox" has an answer. The count itself is the result count — the
2662
+ // read returns one row, and the number in it is the fact worth having. The row lands
2663
+ // on THIS host whichever branch answered, like the drain's read.
2664
+ if (countOnly) {
2665
+ const redrainable = this.eventDrainDelegation && record.vertical
2666
+ ? await this.eventDrainDelegation.redrain({
2667
+ tenantId,
2668
+ scopeId,
2669
+ vertical: record.vertical,
2670
+ drainedBefore,
2671
+ countOnly: true,
2672
+ })
2673
+ : await this.scopeStub(scopeId).redrainCount(drainedBefore);
2674
+ await this.recordAccess(actor, 'redrainEvents', { tenantId, scopeId }, { drainedBefore, countOnly: true }, redrainable);
2675
+ return redrainable;
2676
+ }
2350
2677
  // Audit FIRST, on `rewindScope`'s rule (K-33), because this has the same shape: the
2351
2678
  // mutation commits in a DO and the row is a separate write afterwards, so a failure
2352
2679
  // between them left a reopen that had happened with no receipt — and the retry could
@@ -2575,12 +2902,29 @@ export class CloudflareScopeHost {
2575
2902
  // half-done state harms the person: dying after the redaction leaves ciphertext in
2576
2903
  // a backup that no key opens; destroying the key first would leave their PII in the
2577
2904
  // live database while the audit log already claims they were erased.
2578
- const eventsRedacted = await this.scopeStub(scopeId).redactSubject(subjectId);
2905
+ // Both spine copies (#1600): the outbox row AND any platform intent this event was
2906
+ // routed into. One RPC, so a crash cannot land half of it.
2907
+ const redacted = await this.scopeStub(scopeId).redactSubject(subjectId);
2908
+ // An OLD ScopeDO answers with a bare number — it redacted the outbox and never
2909
+ // looked at the intent journal. Refused here, BEFORE the key is destroyed, and
2910
+ // that order is the whole point: the key is the irreversible half, so proceeding
2911
+ // would leave the subject's platform-retained copies permanently unreadable, their
2912
+ // name still sitting in `_substrat_platform_requests`, and no admin-log row at all
2913
+ // (the log is written after this). Refusing leaves an erasure that can simply be
2914
+ // re-run once the scope is redeployed. Loud rather than partial, the way this
2915
+ // interface's reverse skew is left loud on `recordScheduleRun`.
2916
+ if (typeof redacted === 'number') {
2917
+ throw substratError('unavailable', `scope ${scopeId} runs a ScopeDO from before #1600, whose redaction does not reach ` +
2918
+ `_substrat_platform_requests — erasing now would destroy the subject key while ` +
2919
+ `leaving their payloads in the intent journal. Redeploy the vertical and re-run.`);
2920
+ }
2921
+ const { events: eventsRedacted, intents: intentsRedacted } = redacted;
2579
2922
  const at = new Date().toISOString();
2580
2923
  const { existed } = await this.subjectKeysFor(tenantId, scopeId).destroy(subjectId, at);
2581
2924
  const receipt = subjectShredReceipt.parse({
2582
2925
  subjectId,
2583
2926
  eventsRedacted,
2927
+ intentsRedacted,
2584
2928
  keyDestroyed: existed,
2585
2929
  tombstoned: true,
2586
2930
  });
@@ -2588,7 +2932,9 @@ export class CloudflareScopeHost {
2588
2932
  // because it destroys evidence. An erasure is the one action where "who asked for
2589
2933
  // this to disappear" is itself part of the record.
2590
2934
  await this.recordAdmin(actor, 'shredSubject', { tenantId, scopeId }, null, receipt);
2591
- await this.recordAccess(actor, 'shredSubject', { tenantId, scopeId }, { subjectId }, eventsRedacted);
2935
+ // BOTH counts: the access log's number is "how much evidence this destroyed", and
2936
+ // an intent payload is a whole event's worth of it.
2937
+ await this.recordAccess(actor, 'shredSubject', { tenantId, scopeId }, { subjectId }, eventsRedacted + intentsRedacted);
2592
2938
  return receipt;
2593
2939
  },
2594
2940
  // -- impersonation (K-42, #868) ----------------------------------------
@@ -2727,6 +3073,23 @@ export class CloudflareScopeHost {
2727
3073
  await this.recordAccess(actor, 'listIdentityTenants', {}, { provider }, tenants.length);
2728
3074
  return tenants;
2729
3075
  },
3076
+ listIdentityMemberships: async (actor, provider, externalId) => {
3077
+ // ONE round trip: the pool check, the join and the K-24 row all happen inside
3078
+ // the directory's own call. The id and timestamp are minted here, as
3079
+ // `recordAccess` mints them, so the log row is shaped exactly like its siblings.
3080
+ const { topology, memberships } = await this.cp.identityMemberships(provider, externalId, {
3081
+ id: ulid(),
3082
+ actor,
3083
+ at: new Date().toISOString(),
3084
+ });
3085
+ if (topology === null)
3086
+ throw new Error(`identity pool '${provider}' is not registered`);
3087
+ if (topology !== 'central') {
3088
+ throw new Error(`identity pool '${provider}' is tenant-bound — enumerating tenants is only ` +
3089
+ `meaningful on a central pool, where the same externalId is the same person`);
3090
+ }
3091
+ return memberships.map((m) => identityMembership.parse(m));
3092
+ },
2730
3093
  listIdentityLinks: async (actor, tenantId) => {
2731
3094
  const rows = await this.cp.dumpTenantIdentities(tenantId);
2732
3095
  const links = rows.map((r) => identityLink.parse({
@@ -3309,6 +3672,104 @@ export class CloudflareScopeHost {
3309
3672
  // via newUniqueId (K-7) and stores the mapping in the directory — deferred.
3310
3673
  return this.scopeNs.get(this.scopeNs.idFromName(scopeId));
3311
3674
  }
3675
+ // -- live reads (#938): the door ------------------------------------------
3676
+ // The coordinator's half of the live-read path. It decides two things and no more:
3677
+ // whether this CONNECTION can carry a push at all, and who is asking. What a
3678
+ // subscriber may then be told is decided in the scope DO, per event, per frame.
3679
+ /**
3680
+ * Subscribe a principal to this scope's changes — `ScopeHost.liveReads`.
3681
+ *
3682
+ * Present on this host and declared `never` on `SqliteScopeHost`: a live read needs
3683
+ * something that outlives a request and can be woken when an event lands, and on
3684
+ * Cloudflare that is the scope's own Durable Object. The seam's full reasoning is on
3685
+ * the contract (`ScopeHost.liveReads`), beside the `clock?: never` precedent it
3686
+ * mirrors.
3687
+ *
3688
+ * **Two kinds of refusal, and they are deliberately different shapes.** A request
3689
+ * that cannot carry a socket is answered with a `Response` (426, 501) — that is a
3690
+ * fact about the connection, addressed to the client. A scope that must not be
3691
+ * reached at all THROWS, exactly as `getScope` and the attachment door do, because
3692
+ * it is the same refusal an ordinary read of that scope would get and it should not
3693
+ * arrive as a different class of answer just because the caller asked for a socket.
3694
+ */
3695
+ liveReads = {
3696
+ subscribe: async ({ tenantId, scopeId, principal, request }) => {
3697
+ if (!isUpgradeRequest(request)) {
3698
+ return new Response('live reads are a WebSocket surface', {
3699
+ status: 426,
3700
+ headers: { [LIVE_MODE_HEADER]: 'not-an-upgrade' },
3701
+ });
3702
+ }
3703
+ /**
3704
+ * The orange-to-orange refusal (#938).
3705
+ *
3706
+ * Cloudflare does not carry WebSockets across an O2O hop — the customer's own
3707
+ * proxied zone in front of ours — so an upgrade offered here would fail
3708
+ * somewhere the vertical cannot see, and the client would be left holding a
3709
+ * socket that never delivers. Refused at the door instead, with a reason the
3710
+ * client can read, so the fallback to polling is a thing it KNOWS it is doing
3711
+ * rather than a silence it has to infer.
3712
+ *
3713
+ * Per request, never per hostname: whether a tenant is O2O is decided by the
3714
+ * tenant's own DNS, which is theirs to change without telling us. Anything
3715
+ * cached would be a fact with an invisible expiry date; this header is correct
3716
+ * on the request after they change it.
3717
+ *
3718
+ * 501, not 426: 426 means "upgrade and try again", and trying again is exactly
3719
+ * what will not work. This connection cannot carry the thing that was asked for.
3720
+ */
3721
+ if (isOrangeToOrange(request)) {
3722
+ return new Response('live reads are not available over this connection — the request arrived ' +
3723
+ 'through a proxied customer zone, which does not carry WebSockets. Poll instead.', { status: 501, headers: { [LIVE_MODE_HEADER]: 'poll' } });
3724
+ }
3725
+ /**
3726
+ * The same fail-closed lifecycle gate + lazy migration every other door takes
3727
+ * (`getScope`, `attachments`; control-plane.md §4.1/§4.2, K-3).
3728
+ *
3729
+ * A subscription is a new way INTO a scope, and the permission filter downstream
3730
+ * answers a different question: it decides what a subscriber may see, not whether
3731
+ * this scope should be reachable at all. Without this, an unknown, cross-tenant,
3732
+ * suspended or archiving scope could still be addressed and handed a 101 — a
3733
+ * scope that refuses every ordinary read while quietly holding an open socket.
3734
+ *
3735
+ * Costs a CP-less vertical nothing: `validateScopeAccess` is a **no-op** on the
3736
+ * null control plane (the router already gated lifecycle and tenancy from the
3737
+ * shared directory), so this is the hosted-vertical shape unchanged.
3738
+ *
3739
+ * `migrateAndRecord`, not the DO's own `ensureMigrations`: the DO migrates
3740
+ * itself when the socket opens either way, but only this reports the applied
3741
+ * count to the directory — so a scope whose first contact after a deploy is a
3742
+ * subscription does not go dark in the migration fleet view.
3743
+ */
3744
+ await this.cp.validateScopeAccess(tenantId, scopeId);
3745
+ await this.migrateAndRecord(scopeId);
3746
+ // Asserted, not carried through from the client: the principal is the
3747
+ // vertical's own resolution of its session, and the tenant and scope are the
3748
+ // node the router resolved. Every inbound copy is replaced, for the reason the
3749
+ // router strips `x-substrat-*` before setting its own.
3750
+ const headers = new Headers(request.headers);
3751
+ headers.set(LIVE_PRINCIPAL_HEADER, principal);
3752
+ headers.set(LIVE_TENANT_HEADER, tenantId);
3753
+ headers.set(LIVE_SCOPE_HEADER, scopeId);
3754
+ const forwarded = new Request(new URL(LIVE_SUBSCRIBE_PATH, 'https://scope.substrat.internal'),
3755
+ // `new Request(url, { …, headers })` rather than `new Request(request, …)`:
3756
+ // the DO is addressed by its own path, not the client's, and the upgrade
3757
+ // headers that matter travel in `headers` above. Verified in workerd — a
3758
+ // reconstructed request keeps `Upgrade`, `Connection` and the
3759
+ // `Sec-WebSocket-*` pair, which is what makes the router's own
3760
+ // strip-and-assert safe on this path too.
3761
+ { method: 'GET', headers });
3762
+ // Through `scopeStub`, deliberately, rather than a second `idFromName` here:
3763
+ // that method's own comment says the id derivation is milestone-one and that
3764
+ // production will mint per-jurisdiction ids from the directory. A copy of the
3765
+ // derivation would keep compiling on the day it changes and address a DIFFERENT
3766
+ // object — a subscriber watching an empty scope while its writes land elsewhere.
3767
+ // `ScopeStubRpc` describes the RPC methods and not `fetch`, which every stub has;
3768
+ // the cast widens the view of one object, it does not mint a second one.
3769
+ const stub = this.scopeStub(scopeId);
3770
+ return stub.fetch(forwarded);
3771
+ },
3772
+ };
3312
3773
  // -- scope-local projection (docs/architecture/scope-local-permissions.md, Phase 2) --
3313
3774
  // The write side of the local reader (Phase 1): after any tenant-level change,
3314
3775
  // the coordinator PROJECTS the tenant's current roles + tenant-level tuples into
@@ -3455,15 +3916,22 @@ export class CloudflareScopeHost {
3455
3916
  // used to leave the scope "roles projected, source=local, zero tuples" — enforcing nothing
3456
3917
  // but denials, with no builder-facing lever to fix it. Atomic now: grant and flip land
3457
3918
  // together, and the empty-tuple guard in `applyProjection` refuses the flip if they don't.
3919
+ //
3920
+ // #1659: every tuple here is SEATED — created if missing, left alone if revoked — so a
3921
+ // reconcile no longer undoes an operator's revoke. The owner's seat alone carries
3922
+ // `lockout_reseat`: it comes back over a revoke only when the scope would otherwise hold
3923
+ // no effective role grant — none whose role the vertical still defines — which is the
3924
+ // #332 lockout a reconcile exists to repair.
3458
3925
  [
3459
3926
  {
3460
3927
  subject: `principal:${input.owner}`,
3461
3928
  relation: `role:${input.ownerRoleKey}`,
3462
3929
  object: `scope:${input.scopeId}`,
3463
3930
  expires_at: null,
3931
+ lockout_reseat: true,
3464
3932
  },
3465
3933
  // #461: each registered module's SCHEDULE grants (#383) ride the same unit —
3466
- // the CP-less mirror of `provisionScope`'s writeTuple loop. Without them the
3934
+ // the CP-less mirror of `provisionScope`'s seatTuple loop. Without them the
3467
3935
  // grant-is-the-switch check makes every schedule a silent no-op (`fired: 0`,
3468
3936
  // no error — the #49 unfalsifiable zero).
3469
3937
  ...[...this.moduleSchedules].flatMap(([modId, schedules]) => {
@@ -3550,16 +4018,23 @@ export class CloudflareScopeHost {
3550
4018
  * seat is the one that knows what to announce, and it emits from its own operation.
3551
4019
  * Guarded like `assignScopeRole`: at the harness route, not at this seam.
3552
4020
  *
3553
- * Two things a caller has to know, because the tombstone is only as durable as the next
3554
- * `INSERT OR REPLACE` on the same row:
3555
- * - Anything that re-projects the scope's tuples clears it — `provisionScopeLocal` on a
3556
- * reconcile re-seats the owner and the `system:` grants, and a vertical's `onProvision`
3557
- * hook that re-issues `assignScopeRole` re-seats whatever it names. A revoke of a role
3558
- * such a path grants is undone on the next reconcile, silently. Revoke the seats your
3559
- * own flow granted, not the ones provisioning did.
3560
- * - Revoking the LAST live role tuple in a scope leaves nobody who passes a check, and
3561
- * the local checker has no way back in (#332 guards the flip to local on the way in
3562
- * only). Seat the successor before unseating the last holder.
4021
+ * What survives a reconcile (#1659), because the tombstone is only as durable as the next
4022
+ * write that is allowed to replace it:
4023
+ * - Provisioning does not clear it. `provisionScopeLocal` SEATS its tuples (the owner's
4024
+ * role, the `system:` grants, the connection grants): it recreates a missing one and
4025
+ * leaves a revoked one revoked, so a reconcile — every listed promote runs one — keeps
4026
+ * your revoke.
4027
+ * - With ONE exception: the owner-of-record's seat is re-seated over a revoke when the
4028
+ * scope would otherwise hold no effective role grant at all — a holder of a role the
4029
+ * vertical no longer defines passes no check, so it does not count. A scope nobody can
4030
+ * act in is the #332 lockout the reconcile exists to repair, so revoking the LAST holder
4031
+ * is undone at the next reconcile. Seat the successor (in a role the vertical defines)
4032
+ * before unseating the owner, and the revoke stands. The owner re-seated is the one `owner_of_record` names, which is first-write-
4033
+ * wins — if a successor is later revoked too, the ORIGINAL owner comes back. To lock a
4034
+ * compromised owner out, suspend the scope; a seat revoke is not that lever.
4035
+ * - An explicit grant does clear it: `assignScopeRole` is `INSERT OR REPLACE`, and so is
4036
+ * a vertical's `onProvision` hook that re-issues it — which re-seats whatever it names
4037
+ * on every reconcile. Revoke the seats your own flow granted and does not re-grant.
3563
4038
  * On a CP-less host this records no admin-log row (there is no control plane to hold
3564
4039
  * one), so the row's `revoked_at` is the only evidence, and a re-assign replaces it.
3565
4040
  */
@@ -3623,6 +4098,19 @@ export class CloudflareScopeHost {
3623
4098
  async connectorGrantLocal(connectionId, scopeId, permission, expiresAt) {
3624
4099
  await this.writeScopeTuple(scopeId, subjectRef({ kind: 'connection', id: connectionId }), `granted:${permission}`, `scope:${scopeId}`, expiresAt ?? null);
3625
4100
  }
4101
+ /**
4102
+ * The far end of the schedule kill switch for a scope served HERE (#1666):
4103
+ * `/internal/system-switch` lands on this, from the shared control plane's
4104
+ * `revokeFromSystem` / `restoreToSystem`. It moves the switch in the scope's own DO and
4105
+ * answers what it did; it audits nothing, because the control plane that asked holds the
4106
+ * admin log and writes the row once this returns. `held: false` is an answer, not a
4107
+ * throw — see `systemSwitchOutcome`.
4108
+ */
4109
+ async systemSwitchLocal(scopeId, moduleId, to) {
4110
+ // Parsed on the way out: this is the wire answer the platform reads, and the DO's
4111
+ // plain strings become the published shape here rather than on trust.
4112
+ return systemSwitchOutcome.parse(await this.scopeStub(scopeId).switchSystemSchedules(moduleId, scopeId, to, new Date().toISOString()));
4113
+ }
3626
4114
  }
3627
4115
  /** A ledger row -> the wire entry, the attribution re-nested (#1054). */
3628
4116
  function modelUsageEntryOf(r) {