@substrat-run/adapter-sqlite 0.136.0 → 0.138.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,13 +1,13 @@
1
1
  import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
2
2
  import { dirname, join } from 'node:path';
3
3
  import Database from 'better-sqlite3';
4
- import { accessLogEntry, adminLogEntry, delegatedReadParams, opsFailureEntry, opsFailureFingerprint, issueEntry, FRESHNESS_HEARTBEAT_MINUTES, sweepRunEntry, modelUsageEntry, ATTACHMENT_ADDED, ATTACHMENT_REMOVED, attachmentRecord, createTenantInput, domainEvent, domainEventInput, exportReadInput, importBatch, importCursorMove, exportsOfManifestJson, importsOfManifestJson, entitlementGrant, entitlementGrantInput, eventId, meterReading, platformRequestInput, platformRequestId, MAX_PENDING_PLATFORM_REQUESTS, identityLink, identityPool, instant, connection, capabilityGrant, connectionGrant, connectionGrantRecord, connectionSecret, systemGrant, systemSwitch, systemSwitchRecord, peerSwitch, verticalCaller, verticalSlug as verticalSlugOf, subjectRef, requestFingerprint, createConnectionInput, projectedConnectionGrant, projectedConnectionKey, moduleManifest, createOrgInput, promotionAcknowledgement, bindAcknowledgement, bindHostnameInput, channelHistoryEntry, hostnameBinding, publishVersionInput, AUTO_ADMISSION_NOTE, routeTarget, ROUTE_SCOPE_HOSTNAMES_MAX, registerVerticalInput, vertical as verticalSchema, verticalChannel, verticalVersion, objectRef, entityObjectRef, assertModuleEmittableType, assertKernelAuthoredType, grantRefFromProof, org as orgSchema, orgMembership, principalId, resolvedIdentity, identityMembership, roleDefinition, scope as scopeSchema, tenant as tenantSchema, tenantRole, subjectShredReceipt, SCOPE_TABLE_PAGE_DEFAULT, SCOPE_TABLE_PAGE_MAX, SCOPE_QUERY_ROW_MAX, verticalServingState, callsOfManifestJson, outboundOfManifestJson, listLimitOf, substratError, assertReplayableDump, delegatedReadRecord, ownerTransferAudit, copyMarkAudit, redrainEventsInput, REDRAIN_BATCH, } from '@substrat-run/contracts';
5
- import { actorOf, asPrincipal, assertAllowed, assertNoSecret, assertPermissionKey, CAPABILITY_DDL, COPY_ORIGIN_DDL, CAPABILITY_EXCHANGE_OPERATION, capabilityAttachmentWriteRefused, capabilityTokenHash, createCapabilityVerbs, createEntityEdgeVerbs, exchangeCapability as exchangeCapabilitySecret, guardSecrets, mintBecomeCapability, plausibleSessionToken, readCapabilityPage, redactSecrets, redactSecretText, moduleLog, consoleLogSink, asyncInvocationId, asyncLinePass, consoleInvocationLineSink, resolveCapabilitySession, revokeCapabilityAsPlatform, assertReadOnlyQuery, attachmentBlobKey, attachmentSha256, entitlementDenial, requiredEntitlementFor, foldMeterReading, guardSpine, guardSqlLimits, DO_SQL_LIMITS, tooManyResultColumns, tooManyTableColumns, parseValidationRecords, resolveScopeRecord, ulid, ISSUE_RETENTION_DAYS, telemetryRetentionStatements, assertRowLimit, assertRowOffset, OPS_FAILURE_RETENTION_DAYS, SWEEP_RUN_RETENTION_DAYS, SWEEP_RUNS_INTENT_INDEX, sweepRunsIntentHasKind, MODEL_USAGE_RETENTION_DAYS, foldModelUsage, backoffAt, platformRequestHistoryQuery, platformRequestOf, PLATFORM_REQUEST_COLUMNS, PLATFORM_REQUEST_REDACTION_SQL, platformRequestRedactionParams, platformRequestRedactionQuery, intentPayloadCarriesSubject, redactSubjectJobRuns, redactSubjectScopeText, redactSubjectDirectoryText, ISSUE_EXEMPLAR_OWNER_BACKFILL_SQL, issueExemplarOwner, JOB_RUN_PATCH_SQL, JOB_STEP_RECORD_SQL, DELIVERY_ERROR_REDACTION_SQL, REDACTED_DELIVERY_NOTE, seatScopeTuple, switchSystemSchedules, admitPeer, collectPeers, peerSeats, resolveVerticalInstanceFrom, switchPeer, peerGrantsStatus, systemGrantsStatus, systemScheduleState, systemSwitchedOff, systemSwitchedOffMessage, tenantSystemSwitchedOffMessage, SYSTEM_SWITCHES_BACKFILL_SQL, SYSTEM_SWITCHES_DDL, forgetSystemSwitchesOf, listSystemSwitchRecords, recordSystemSwitchedOff, recordSystemSwitchedOn, restoreSystemSwitchRecord, scopesSwitchedOffFor, switchedOffModulesOf, inUnitMovesToAudit, staleCarryRevertRow, staleCarryReverts, systemSwitchRecordsOf, systemSwitchesTableExists, withRecorded, VERSION_MIGRATIONS_DDL, splitManifestMigrations, splitVersionMigrationsBatch, versionMigrationsOf, writeVersionMigrations, denialListQuery, refusalListQuery, mapRefusalRow, denialSummaryQuery, denialTotalsQuery, DENIAL_WINDOW_QUERY, mapDenialRow, mapDenialSummaryBuckets, IMPERSONATION_COLUMNS, IMPERSONATION_DDL, ImpersonationRefused, assertImpersonationWrites, assertModuleEnqueueableKind, assertSessionUsable, impersonationByIdQuery, impersonationListQuery, impersonationRowValues, impersonationStampOf, mapImpersonationRow, newImpersonationSession, resolveRetryPolicy, isSecretBoxConfigured, unconfiguredSecretBox, createSubjectKeys, generateSealingKeyPair, noSealingKeyMessage, openSealed, sealTo, ConnectionSealingKeyUnavailableError, PermissionDenied, createAtomic, NotSearchable, NotListable, listIndexMigrations, listIndexPlans, listQuery, cursorOf, moduleMigrations, isSearchIndexTable, searchIndexDdl, searchIndexMigrations, searchIndexPlans, searchLimit, searchMatchExpression, searchQuery, ATTACHMENT_TEXT_DDL, assertAttachmentExtractors, assertJobRegistrable, attachmentRecordOfRow, attachmentTextJob, enqueueAttachmentText, isAttachmentTextRun, reconcileAttachmentText, recordAttachmentText, searchAttachments, entityVersionQuery, entityVersionOf, assertIfMatch, OUTBOX_ENTITY_INDEX, SCHEDULE_STATE_DDL, SCHEDULE_STATE_REBUILD, scheduleStateHasKind, VERTICAL_EVENTS_DDL, EXPORT_HOPS_SQL, IMPORT_CURSORS_SQL, IMPORT_CURSOR_OF_SQL, OUTBOX_MARK_SQL, emittedSinceQuery, emittedReportOf, EMITTED_REPORT_CAP, exportedSinceQuery, IMPORT_CURSOR_ADVANCE_SQL, IMPORT_RECORD_SQL, moveImportCursor, importCursorSourceOf, exportBreaksOf, exportBreakRefusal, bindExportBreaksOf, bindExportBreakRefusal, CrossVerticalRegistry, exportReadPlan, exportReadQuery, planExportBatch, withheldNote, JOB_RUN_DDL, jobRunListLimit, jobRunOf, runDueJobRuns, startJobRun, IDEMPOTENCY_DDL, REFUSALS_DDL, refusalInsert, refusedTransitionOf, assertIdempotencyKey, idempotencyLookupQuery, idempotencyPruneStatement, idempotencyRecordStatement, idempotencyOptedOutMessage, replayFor, globalFetch, createUlid, facetEvents, readHistory, walkEventCause, walkEventEffects, assertRedrainWindow, readInvocation, readDeadLetters, readLifecycleFlow, domainEventOf, readUndrainedOutbox, undrainedEventsOf, connectorCallRecord, noopConnectorCallRecorder, recordConnectorCall, settleConnectionUse, unknownRoleError, } from '@substrat-run/kernel';
4
+ import { accessLogEntry, adminLogEntry, delegatedReadParams, operationSeriesCount, opsFailureEntry, opsFailureFingerprint, issueEntry, FRESHNESS_HEARTBEAT_MINUTES, sweepRunEntry, modelUsageEntry, ATTACHMENT_ADDED, ATTACHMENT_REMOVED, attachmentRecord, createTenantInput, domainEvent, domainEventInput, exportReadInput, importBatch, importCursorMove, exportsOfManifestJson, importsOfManifestJson, entitlementGrant, entitlementGrantInput, eventId, meterReading, platformRequestInput, platformRequestId, MAX_PENDING_PLATFORM_REQUESTS, identityLink, identityPool, instant, connection, capabilityGrant, connectionGrant, connectionGrantRecord, connectionSecret, systemGrant, systemSwitch, systemSwitchRecord, peerSwitch, verticalCaller, verticalSlug as verticalSlugOf, subjectRef, requestFingerprint, createConnectionInput, projectedConnectionGrant, projectedConnectionKey, moduleManifest, createOrgInput, promotionAcknowledgement, bindAcknowledgement, bindHostnameInput, channelHistoryEntry, hostnameBinding, publishVersionInput, AUTO_ADMISSION_NOTE, routeTarget, ROUTE_SCOPE_HOSTNAMES_MAX, registerVerticalInput, vertical as verticalSchema, verticalChannel, verticalVersion, objectRef, entityObjectRef, assertModuleEmittableType, assertKernelAuthoredType, grantRefFromProof, org as orgSchema, orgMembership, principalId, resolvedIdentity, identityMembership, roleDefinition, scope as scopeSchema, tenant as tenantSchema, tenantRole, subjectShredReceipt, SCOPE_TABLE_PAGE_DEFAULT, SCOPE_TABLE_PAGE_MAX, SCOPE_QUERY_ROW_MAX, verticalServingState, callsOfManifestJson, outboundOfManifestJson, listLimitOf, SCOPE_GATE_REASONS, substratError, assertReplayableDump, delegatedReadRecord, ownerTransferAudit, copyMarkAudit, redrainEventsInput, REDRAIN_BATCH, } from '@substrat-run/contracts';
5
+ import { actorOf, asPrincipal, assertAllowed, assertNoSecret, assertPermissionKey, CAPABILITY_DDL, COPY_ORIGIN_DDL, CAPABILITY_EXCHANGE_OPERATION, capabilityAttachmentWriteRefused, capabilityTokenHash, createCapabilityVerbs, createEntityEdgeVerbs, exchangeCapability as exchangeCapabilitySecret, guardSecrets, mintBecomeCapability, plausibleSessionToken, readCapabilityPage, redactSecrets, redactSecretText, moduleLog, consoleLogSink, asyncInvocationId, asyncLinePass, consoleInvocationLineSink, resolveCapabilitySession, revokeCapabilityAsPlatform, assertReadOnlyQuery, attachmentBlobKey, attachmentSha256, entitlementDenial, requiredEntitlementFor, foldMeterReading, guardSpine, guardSqlLimits, DO_SQL_LIMITS, tooManyResultColumns, tooManyTableColumns, parseValidationRecords, resolveScopeRecord, ulid, ISSUE_RETENTION_DAYS, telemetryRetentionStatements, assertRowLimit, assertRowOffset, OPS_FAILURE_RETENTION_DAYS, SWEEP_RUN_RETENTION_DAYS, SWEEP_RUNS_INTENT_INDEX, sweepRunsIntentHasKind, MODEL_USAGE_RETENTION_DAYS, foldModelUsage, backoffAt, platformRequestHistoryQuery, platformRequestOf, PLATFORM_REQUEST_COLUMNS, PLATFORM_REQUEST_REDACTION_SQL, platformRequestRedactionParams, platformRequestRedactionQuery, intentPayloadCarriesSubject, redactSubjectJobRuns, redactSubjectScopeText, redactSubjectDirectoryText, ISSUE_EXEMPLAR_OWNER_BACKFILL_SQL, issueExemplarOwner, JOB_RUN_PATCH_SQL, JOB_RUN_CLAIM_SQL, JOB_RUN_RENEW_SQL, JOB_RUN_BEGIN_SQL, JOB_RUN_MISS_SQL, JOB_RUN_MISS_SETTLE_SQL, admissionMissOutcome, JOB_STEP_RECORD_SQL, JOB_LEASE_EXPIRED_NOTE, assertLeaseMs, DELIVERY_ERROR_REDACTION_SQL, REDACTED_DELIVERY_NOTE, seatScopeTuple, admitPeer, collectPeers, peerSeats, resolveVerticalInstanceFrom, peerGrantsStatus, systemGrantsStatus, systemScheduleState, systemSwitchedOff, systemSwitchedOffMessage, tenantSystemSwitchedOffMessage, PEER_SWITCHES_DDL, SWITCH_OWED_DDL, SWITCH_FENCES_DDL, recordWriteSuperseded, switchFencesOf, switchSupersededMessage, SWITCH_KINDS, SYSTEM_SWITCHES_DDL, forgetSwitchesOf, listSystemSwitchRecords, moveSwitch, reassertActionOf, reassertEntry, reassertOffRow, recordSwitchedOff, recordSwitchedOn, restoreSwitchRecord, scopesSwitchedOffFor, switchActionOf, switchAuditSubject, switchNotFoundMessage, switchRecordsOf, switchedOffOf, switchesBackfillSqlOf, switchesTableExists, tenantHeldOf, tenantHoldsGrant, inUnitMovesToAudit, staleCarryRevertRow, staleCarryReverts, withRecorded, VERSION_MIGRATIONS_DDL, splitManifestMigrations, splitVersionMigrationsBatch, versionMigrationsOf, writeVersionMigrations, denialListQuery, refusalListQuery, mapRefusalRow, denialSummaryQuery, denialTotalsQuery, DENIAL_WINDOW_QUERY, mapDenialRow, mapDenialSummaryBuckets, IMPERSONATION_COLUMNS, IMPERSONATION_DDL, ImpersonationRefused, assertImpersonationWrites, assertModuleEnqueueableKind, assertSessionUsable, impersonationByIdQuery, impersonationListQuery, impersonationRowValues, impersonationStampOf, mapImpersonationRow, newImpersonationSession, resolveRetryPolicy, executorOutcomeOf, isDeliveryRefusal, refusalJournalText, membershipFencesBackfillSql, MEMBERSHIP_FENCES_DDL, membershipFencesTableExists, MEMBERSHIP_FENCE_SINCE_SQL, RAISE_MEMBERSHIP_FENCE_SQL, tenantCoverage, isSecretBoxConfigured, unconfiguredSecretBox, createSubjectKeys, generateSealingKeyPair, noSealingKeyMessage, openSealed, sealTo, ConnectionSealingKeyUnavailableError, PermissionDenied, createAtomic, NotSearchable, NotListable, listIndexMigrations, listIndexPlans, listQuery, cursorOf, moduleMigrations, isSearchIndexTable, searchIndexDdl, searchIndexMigrations, searchIndexPlans, searchLimit, searchMatchExpression, searchQuery, ATTACHMENT_TEXT_DDL, assertAttachmentExtractors, assertJobRegistrable, attachmentRecordOfRow, attachmentTextJob, enqueueAttachmentText, isAttachmentTextRun, reconcileAttachmentText, recordAttachmentText, searchAttachments, entityVersionQuery, entityVersionOf, assertIfMatch, OUTBOX_ENTITY_INDEX, SCHEDULE_STATE_DDL, SCHEDULE_STATE_REBUILD, scheduleStateHasKind, VERTICAL_EVENTS_DDL, EXPORT_HOPS_SQL, emptyImportResult, IMPORT_CURSORS_SQL, IMPORT_CURSOR_OF_SQL, OUTBOX_MARK_SQL, emittedSinceQuery, emittedReportOf, EMITTED_REPORT_CAP, exportedSinceQuery, IMPORT_CURSOR_ADVANCE_SQL, IMPORT_RECORD_SQL, moveImportCursor, importCursorSourceOf, exportBreaksOf, exportBreakRefusal, bindExportBreaksOf, bindExportBreakRefusal, CrossVerticalRegistry, exportReadPlan, exportReadQuery, planExportBatch, withheldNote, JOB_RUN_DDL, JOB_RUN_DUE_AT, jobRunListLimit, jobRunOf, runDueJobRuns, startJobRun, IDEMPOTENCY_DDL, REFUSALS_TABLE_DDL, REFUSALS_INDEX, refusalInsert, refusalOf, markGuardRefusal, REFUSALS_REBUILD, refusalsAdmitGuards, assertIdempotencyKey, idempotencyLookupQuery, idempotencyPruneStatement, idempotencyRecordStatement, idempotencyOptedOutMessage, replayFor, globalFetch, createUlid, facetEvents, readHistory, walkEventCause, walkEventEffects, assertRedrainWindow, readInvocation, readDeadLetters, readLifecycleFlow, readOperationSeries, domainEventOf, readUndrainedOutbox, undrainedEventsOf, connectorCallRecord, noopConnectorCallRecorder, recordConnectorCall, settleConnectionUse, unknownRoleError, } from '@substrat-run/kernel';
6
6
  import { attributedHost } from '@substrat-run/kernel';
7
7
  import { INERT_SCOPE_REASON, isPrimaryScopeRow } from '@substrat-run/kernel';
8
8
  import { LEGACY_SCOPE_ROWS_BACKFILL, assertNoSpineReference, assertSpineTablesBuilt, capabilitiesForLoad, dumpRowsInsert, isSpineTable, loadDirectoryDump, repointScopeGrants, settleCopiedWork, emittedHere, spineColumnAdditions } from '@substrat-run/kernel';
9
9
  import { ScopeActor } from './actor.js';
10
- import { createTupleChecker } from './checker.js';
10
+ import { createTupleChecker, directoryTenantReader } from './checker.js';
11
11
  /**
12
12
  * A fault in the database itself (full, corrupt, I/O), as opposed to one statement failing.
13
13
  * The #1764 backfill logs and opens past a failed batch, but never past one of these.
@@ -204,7 +204,8 @@ const KERNEL_DDL = `
204
204
  drained_at TEXT
205
205
  );
206
206
  ${IDEMPOTENCY_DDL}
207
- ${REFUSALS_DDL}
207
+ ${REFUSALS_TABLE_DDL}
208
+ ${REFUSALS_INDEX}
208
209
  -- #1672: capabilities — authority carried by a secret (a link share), and the sessions
209
210
  -- an exchange trades that secret for. Spine (kernel-written), shared with the DO adapter
210
211
  -- from @substrat-run/kernel so the two cannot part company; the column comments are there.
@@ -242,6 +243,7 @@ const KERNEL_DDL = `
242
243
  rows_changed INTEGER,
243
244
  PRIMARY KEY (module_id, version)
244
245
  );
246
+ ${SWITCH_FENCES_DDL}
245
247
  CREATE TABLE IF NOT EXISTS _substrat_tuples (
246
248
  subject TEXT NOT NULL,
247
249
  relation TEXT NOT NULL,
@@ -1062,6 +1064,9 @@ export class SqliteScopeHost {
1062
1064
  );
1063
1065
  ${IMPERSONATION_DDL}
1064
1066
  ${SYSTEM_SWITCHES_DDL}
1067
+ ${MEMBERSHIP_FENCES_DDL}
1068
+ ${PEER_SWITCHES_DDL}
1069
+ ${SWITCH_OWED_DDL}
1065
1070
  CREATE TABLE IF NOT EXISTS _substrat_admin_log (
1066
1071
  id TEXT PRIMARY KEY,
1067
1072
  actor TEXT NOT NULL,
@@ -1236,10 +1241,16 @@ export class SqliteScopeHost {
1236
1241
  */
1237
1242
  ensureDirectorySchema() {
1238
1243
  this.directory.transaction(() => {
1239
- const switchRecordIsNew = !systemSwitchesTableExists(switchSqlOf(this.directory));
1244
+ // Each kill switch's record (#1674, #2029) is backfilled on the pass that creates its table.
1245
+ const newRecords = SWITCH_KINDS.filter((kind) => !switchesTableExists(switchSqlOf(this.directory), kind));
1246
+ const fencesAreNew = !membershipFencesTableExists(switchSqlOf(this.directory));
1240
1247
  this.buildDirectorySchema();
1241
- if (switchRecordIsNew)
1242
- this.directory.exec(SYSTEM_SWITCHES_BACKFILL_SQL);
1248
+ for (const kind of newRecords)
1249
+ this.directory.exec(switchesBackfillSqlOf(kind));
1250
+ // #1184: the removal fence, for everyone removed before it existed — the same once-only
1251
+ // gate — stamped now, by the host clock (`membershipFencesBackfillSql` says why).
1252
+ if (fencesAreNew)
1253
+ this.directory.exec(membershipFencesBackfillSql(this.clock()));
1243
1254
  })();
1244
1255
  this.splitVersionMigrations();
1245
1256
  }
@@ -1577,10 +1588,10 @@ export class SqliteScopeHost {
1577
1588
  .prepare('SELECT status FROM tenants WHERE tenant_id = ?')
1578
1589
  .get(input.tenantId);
1579
1590
  if (!tenantRow) {
1580
- throw new Error(`cannot provision scope under unknown tenant: ${input.tenantId}`);
1591
+ throw substratError('conflict', `cannot provision scope under unknown tenant: ${input.tenantId}`);
1581
1592
  }
1582
1593
  if (tenantRow.status !== 'active') {
1583
- throw new Error(`cannot provision scope under non-active tenant (status: ${tenantRow.status}): ${input.tenantId}`);
1594
+ throw substratError('conflict', `cannot provision scope under non-active tenant (status: ${tenantRow.status}): ${input.tenantId}`);
1584
1595
  }
1585
1596
  const record = resolveScopeRecord(input);
1586
1597
  // Idempotency is on the scope_id (§3.3: provisioning is idempotent and
@@ -1599,7 +1610,7 @@ export class SqliteScopeHost {
1599
1610
  .prepare("SELECT scope_id FROM scopes WHERE tenant_id = ? AND slug = ? AND status NOT IN ('archived', 'reaped')")
1600
1611
  .get(input.tenantId, record.slug);
1601
1612
  if (slugOwner) {
1602
- throw new Error(`scope slug '${record.slug}' already taken under tenant ${input.tenantId} ` +
1613
+ throw substratError('conflict', `scope slug '${record.slug}' already taken under tenant ${input.tenantId} ` +
1603
1614
  `by ${slugOwner.scope_id} (slugs are unique within a tenant)`);
1604
1615
  }
1605
1616
  this.directory
@@ -1665,48 +1676,52 @@ export class SqliteScopeHost {
1665
1676
  }
1666
1677
  /**
1667
1678
  * `HostAdmin.reassertSystemSwitches`' body (#1674), run inside a turn the caller already
1668
- * holds on the scope's actor: every module the directory records OFF on the scope is
1669
- * switched off again. Already off answers `changed: false` and writes nothing, and a
1670
- * module the scope does not hold is left alone. Audited only when something moved.
1679
+ * holds on the scope's actor: every module, and (#2029) every peer, the directory records OFF
1680
+ * on the scope is switched off again. Already off answers `changed: false` and writes nothing,
1681
+ * and a subject with no authority reaching the scope (no grant here, no tenant-level grant) is
1682
+ * left alone. Audited only when something moved, under each kind's re-assert action.
1671
1683
  *
1672
1684
  * `audit` receives each row. By default it writes to the admin log at once. A caller running
1673
1685
  * this inside a scope transaction that may still roll back (a restore's replay) collects
1674
1686
  * the rows and writes them once the transaction commits. The admin log is another
1675
1687
  * database, and a row must not outlive a move that was rolled back.
1676
1688
  */
1677
- reassertSwitchesInTurn(rt, actor, tenantId, scopeId, opts, audit = (after) => this.recordAdmin(actor, 'reassertSystemSwitch', { tenantId, scopeId }, null, after)) {
1689
+ reassertSwitchesInTurn(rt, actor, tenantId, scopeId, opts, audit = (action, after) => this.recordAdmin(actor, action, { tenantId, scopeId }, null, after)) {
1678
1690
  const at = new Date().toISOString();
1679
- // #1742 review: a move the deployment made from a stale list — the module was restored
1680
- // ON after the list was read — is undone first, so the operator's ON stands.
1681
- const recorded = systemSwitchRecordsOf(switchSqlOf(this.directory), tenantId, scopeId);
1682
- // One synchronous turn: no switch can land between this read and the moves below.
1683
- const reverted = new Set(staleCarryReverts(recorded, opts?.appliedInUnit));
1684
- for (const moduleId of reverted) {
1685
- const outcome = rt.db.transaction(() => switchSystemSchedules(switchSqlOf(rt.db), { moduleId, scopeId, to: 'on', at }))();
1686
- if (outcome.changed)
1687
- audit({ operationId: ulid(), ...staleCarryRevertRow(moduleId, outcome) });
1688
- }
1689
- const recordedOff = switchedOffModulesOf(switchSqlOf(this.directory), tenantId, scopeId);
1690
- // #1742: what a deployment already switched off inside its own unit, audited here — the
1691
- // switch below answers `changed: false` for it and would write no row.
1692
- // A move the revert above undid is not credited as an in-unit OFF.
1693
- const applied = opts?.appliedInUnit?.filter((a) => !reverted.has(a.moduleId));
1694
- for (const row of inUnitMovesToAudit(recordedOff, applied))
1695
- audit({ operationId: ulid(), ...row });
1696
- return recordedOff.map((moduleId) => {
1697
- const outcome = rt.db.transaction(() => switchSystemSchedules(switchSqlOf(rt.db), { moduleId, scopeId, to: 'off', at }))();
1698
- if (outcome.changed) {
1699
- audit({
1700
- operationId: ulid(),
1701
- moduleId,
1702
- schedules: 'off',
1703
- phase: 'applied',
1704
- changed: true,
1705
- permissions: outcome.permissions,
1706
- });
1691
+ const directorySql = switchSqlOf(this.directory);
1692
+ const results = [];
1693
+ for (const kind of SWITCH_KINDS) {
1694
+ const action = reassertActionOf(kind);
1695
+ // #2045: every move carries the fence of the call the record holds, so a scope moved since by
1696
+ // a newer call stays as that call put it.
1697
+ const fences = switchFencesOf(directorySql, kind, tenantId, scopeId);
1698
+ const move = (key, to, tenantHeld) => rt.db.transaction(() => moveSwitch(switchSqlOf(rt.db), kind, { key, scopeId, to, at, tenantHeld, fence: fences.get(key) }))();
1699
+ // #1742 review: a move the deployment made from a stale list — the subject was restored
1700
+ // ON after the list was read — is undone first, so the operator's ON stands.
1701
+ const recorded = switchRecordsOf(directorySql, kind, tenantId, scopeId);
1702
+ // One synchronous turn: no switch can land between this read and the moves below.
1703
+ const reverted = new Set(staleCarryReverts(kind, recorded, opts?.appliedInUnit));
1704
+ for (const key of reverted) {
1705
+ const outcome = move(key, 'on');
1706
+ if (outcome.changed)
1707
+ audit(action, { operationId: ulid(), ...staleCarryRevertRow(kind, key, outcome) });
1707
1708
  }
1708
- return { moduleId, held: outcome.held, changed: outcome.changed };
1709
- });
1709
+ const recordedOff = switchedOffOf(directorySql, kind, tenantId, scopeId);
1710
+ // #1742: what a deployment already switched off inside its own unit, audited here — the
1711
+ // switch below answers `changed: false` for it and would write no row. A move the revert
1712
+ // above undid is not credited as an in-unit OFF.
1713
+ for (const row of inUnitMovesToAudit(kind, recordedOff, opts?.appliedInUnit, reverted)) {
1714
+ audit(action, { operationId: ulid(), ...row });
1715
+ }
1716
+ for (const key of recordedOff) {
1717
+ // #1823 (#2030): a subject whose only authority here is a tenant-level grant is held too.
1718
+ const outcome = move(key, 'off', tenantHoldsGrant(directorySql, kind, tenantId, key, this.clock()));
1719
+ if (outcome.changed)
1720
+ audit(action, { operationId: ulid(), ...reassertOffRow(kind, key, outcome.permissions) });
1721
+ results.push(reassertEntry(kind, key, outcome));
1722
+ }
1723
+ }
1724
+ return results;
1710
1725
  }
1711
1726
  async provisionTenantStore(actor, input) {
1712
1727
  // Fail closed on an unknown/inactive tenant, exactly as provisionScope does: a store
@@ -1716,10 +1731,10 @@ export class SqliteScopeHost {
1716
1731
  .prepare('SELECT status FROM tenants WHERE tenant_id = ?')
1717
1732
  .get(input.tenantId);
1718
1733
  if (!tenantRow) {
1719
- throw new Error(`cannot provision tenant store under unknown tenant: ${input.tenantId}`);
1734
+ throw substratError('not_found', `cannot provision tenant store under unknown tenant: ${input.tenantId}`);
1720
1735
  }
1721
1736
  if (tenantRow.status !== 'active') {
1722
- throw new Error(`cannot provision tenant store under non-active tenant (status: ${tenantRow.status}): ${input.tenantId}`);
1737
+ throw substratError('conflict', `cannot provision tenant store under non-active tenant (status: ${tenantRow.status}): ${input.tenantId}`);
1723
1738
  }
1724
1739
  // Idempotent on (tenant, vertical, binding): a retried provision re-resolves the SAME
1725
1740
  // store rather than minting a second database (the K-31 ready-gate retries the whole
@@ -1787,10 +1802,10 @@ export class SqliteScopeHost {
1787
1802
  .prepare('SELECT status FROM tenants WHERE tenant_id = ?')
1788
1803
  .get(input.tenantId);
1789
1804
  if (!tenantRow) {
1790
- throw new Error(`cannot provision blob store under unknown tenant: ${input.tenantId}`);
1805
+ throw substratError('not_found', `cannot provision blob store under unknown tenant: ${input.tenantId}`);
1791
1806
  }
1792
1807
  if (tenantRow.status !== 'active') {
1793
- throw new Error(`cannot provision blob store under non-active tenant (status: ${tenantRow.status}): ${input.tenantId}`);
1808
+ throw substratError('conflict', `cannot provision blob store under non-active tenant (status: ${tenantRow.status}): ${input.tenantId}`);
1794
1809
  }
1795
1810
  const existing = this.directory
1796
1811
  .prepare('SELECT ref FROM blob_stores WHERE tenant_id = ? AND vertical = ? AND binding = ?')
@@ -2312,9 +2327,9 @@ export class SqliteScopeHost {
2312
2327
  const rows = [];
2313
2328
  // #1869: the re-point's source is the separate hint when given; the dump is otherwise as sent.
2314
2329
  const source = { tables: dump.tables, scopeId: opts?.sourceScopeId ?? dump.scopeId };
2315
- await this.loadDump(tenantId, scopeId, source, (rt) => this.reassertSwitchesInTurn(rt, actor, tenantId, scopeId, undefined, (after) => rows.push(after)));
2316
- for (const after of rows)
2317
- this.recordAdmin(actor, 'reassertSystemSwitch', { tenantId, scopeId }, null, after);
2330
+ await this.loadDump(tenantId, scopeId, source, (rt) => this.reassertSwitchesInTurn(rt, actor, tenantId, scopeId, undefined, (action, after) => rows.push([action, after])));
2331
+ for (const [action, after] of rows)
2332
+ this.recordAdmin(actor, action, { tenantId, scopeId }, null, after);
2318
2333
  this.recordAdmin(actor, 'restoreScope', { tenantId, scopeId }, null, { sourceScopeId: dump.scopeId, tables: dump.tables.length, capturedAt: dump.capturedAt });
2319
2334
  }
2320
2335
  async snapshotScope(actor, tenantId, scopeId, opts) {
@@ -2347,7 +2362,7 @@ export class SqliteScopeHost {
2347
2362
  if (!rec)
2348
2363
  throw substratError('not_found', `unknown scope ${scopeId} in tenant ${tenantId}`);
2349
2364
  if (!rec.forkedFrom && rec.kind !== 'preview') {
2350
- throw new Error(`scope ${scopeId} is not a fork or preview — only previews may be deleted; ` +
2365
+ throw substratError('forbidden', `scope ${scopeId} is not a fork or preview — only previews may be deleted; ` +
2351
2366
  `archive a primary scope instead`);
2352
2367
  }
2353
2368
  // Close and evict the runtime handle before touching the file.
@@ -2360,7 +2375,7 @@ export class SqliteScopeHost {
2360
2375
  // orphaned bytes with no record (the §9 hazard).
2361
2376
  this.directory.prepare('DELETE FROM hostnames WHERE scope_id = ?').run(scopeId);
2362
2377
  rmSync(join(this.dir, `${tenantId}__${scopeId}.sqlite`), { force: true });
2363
- forgetSystemSwitchesOf(switchSqlOf(this.directory), scopeId);
2378
+ forgetSwitchesOf(switchSqlOf(this.directory), scopeId);
2364
2379
  this.directory.prepare('DELETE FROM scopes WHERE scope_id = ?').run(scopeId);
2365
2380
  this.recordAdmin(actor, 'deleteSnapshot', { tenantId, scopeId }, null, {
2366
2381
  forkedFrom: rec.forkedFrom,
@@ -2385,10 +2400,12 @@ export class SqliteScopeHost {
2385
2400
  .prepare('SELECT status FROM tenants WHERE tenant_id = ?')
2386
2401
  .get(tenantId);
2387
2402
  if (!tenantRow) {
2388
- throw new Error(`scope has no tenant record: (${tenantId}, ${scopeId})`);
2403
+ throw substratError('not_found', `scope has no tenant record: (${tenantId}, ${scopeId})`, {
2404
+ reason: SCOPE_GATE_REASONS.unrecorded,
2405
+ });
2389
2406
  }
2390
2407
  if (tenantRow.status !== 'active') {
2391
- throw new Error(`tenant not active (status: ${tenantRow.status}): ${tenantId}`);
2408
+ throw substratError('conflict', `tenant not active (status: ${tenantRow.status}): ${tenantId}`, { reason: SCOPE_GATE_REASONS.notActive });
2392
2409
  }
2393
2410
  // `provisioning` is handled BELOW rather than here, because a scope that never
2394
2411
  // finished setting up should still retry its migrations when touched — that lazy
@@ -2397,7 +2414,7 @@ export class SqliteScopeHost {
2397
2414
  // deliberate states, and running migrations for them would be work on behalf of a
2398
2415
  // request that is going to be refused anyway.
2399
2416
  if (row.status !== 'active' && row.status !== 'provisioning') {
2400
- throw new Error(`scope not active (status: ${row.status}): ${scopeId}`);
2417
+ throw substratError('conflict', `scope not active (status: ${row.status}): ${scopeId}`, { reason: SCOPE_GATE_REASONS.notActive });
2401
2418
  }
2402
2419
  const rt = this.runtime(tenantId, scopeId);
2403
2420
  await this.applyPendingMigrations(rt);
@@ -2406,7 +2423,7 @@ export class SqliteScopeHost {
2406
2423
  // the scope exists on the vertical's side (K-31). Refused, but only after the
2407
2424
  // retry above has had its chance — and if THAT is what failed, it threw with
2408
2425
  // the migration's own message, which is the one an operator needs.
2409
- throw new Error(`scope not active (status: ${row.status}): ${scopeId}`);
2426
+ throw substratError('conflict', `scope not active (status: ${row.status}): ${scopeId}`, { reason: SCOPE_GATE_REASONS.notActive });
2410
2427
  }
2411
2428
  return this.buildStub(tenantId, scopeId, rt, asPrincipal(principal), options);
2412
2429
  }
@@ -2442,7 +2459,7 @@ export class SqliteScopeHost {
2442
2459
  `runs '${scope.vertical ?? 'none'}'`);
2443
2460
  }
2444
2461
  if (scope.status !== 'active') {
2445
- throw new Error(`scope not active (status: ${scope.status}): ${scopeId}`);
2462
+ throw substratError('conflict', `scope not active (status: ${scope.status}): ${scopeId}`, { reason: SCOPE_GATE_REASONS.notActive });
2446
2463
  }
2447
2464
  const rt = this.runtime(conn.tenant_id, scopeId);
2448
2465
  await this.applyPendingMigrations(rt);
@@ -2470,7 +2487,7 @@ export class SqliteScopeHost {
2470
2487
  `runs '${scope.vertical ?? 'none'}'`);
2471
2488
  }
2472
2489
  if (scope.status !== 'active') {
2473
- throw new Error(`scope not active (status: ${scope.status}): ${scopeId}`);
2490
+ throw substratError('conflict', `scope not active (status: ${scope.status}): ${scopeId}`, { reason: SCOPE_GATE_REASONS.notActive });
2474
2491
  }
2475
2492
  const rt = this.runtime(conn.tenant_id, scopeId);
2476
2493
  await this.applyPendingMigrations(rt);
@@ -2505,7 +2522,7 @@ export class SqliteScopeHost {
2505
2522
  throw substratError('not_found', `unknown scope for tenant: (${tenantId}, ${scopeId})`);
2506
2523
  }
2507
2524
  if (scope.status !== 'active') {
2508
- throw new Error(`scope not active (status: ${scope.status}): ${scopeId}`);
2525
+ throw substratError('conflict', `scope not active (status: ${scope.status}): ${scopeId}`, { reason: SCOPE_GATE_REASONS.notActive });
2509
2526
  }
2510
2527
  const rt = this.runtime(tenantId, scopeId);
2511
2528
  await this.applyPendingMigrations(rt);
@@ -2517,7 +2534,7 @@ export class SqliteScopeHost {
2517
2534
  // and the scope must be active. Authority is then an ordinary check against
2518
2535
  // `system:<moduleId>` grants inside the stub; nothing here is a person.
2519
2536
  if (!this.modules.has(moduleId)) {
2520
- throw new Error(`module not registered on this host: ${moduleId}`);
2537
+ throw substratError('not_found', `module not registered on this host: ${moduleId}`);
2521
2538
  }
2522
2539
  const scope = this.directory
2523
2540
  .prepare('SELECT tenant_id, status FROM scopes WHERE scope_id = ?')
@@ -2526,7 +2543,7 @@ export class SqliteScopeHost {
2526
2543
  throw substratError('not_found', `unknown scope: ${scopeId}`);
2527
2544
  }
2528
2545
  if (scope.status !== 'active') {
2529
- throw new Error(`scope not active (status: ${scope.status}): ${scopeId}`);
2546
+ throw substratError('conflict', `scope not active (status: ${scope.status}): ${scopeId}`, { reason: SCOPE_GATE_REASONS.notActive });
2530
2547
  }
2531
2548
  const rt = this.runtime(tenantId, scopeId);
2532
2549
  await this.applyPendingMigrations(rt);
@@ -2596,15 +2613,11 @@ export class SqliteScopeHost {
2596
2613
  const source = batch.source;
2597
2614
  const rt = await this.openActiveScope(tenantId, scopeId);
2598
2615
  return rt.actor.enqueue(async () => {
2599
- const result = {
2600
- delivered: 0,
2601
- deadLettered: 0,
2602
- duplicates: 0,
2603
- withheld: 0,
2604
- cursor: batch.after,
2605
- stale: false,
2606
- paused: null,
2607
- };
2616
+ const result = emptyImportResult(batch);
2617
+ // #2004: a copy consumes no other vertical's events, at the door as in the sweep
2618
+ // (`scope-copy.ts`). A pause: nothing runs and the watermark stays.
2619
+ if (!this.isPrimaryInDirectory(scopeId))
2620
+ return { ...result, paused: { reason: INERT_SCOPE_REASON } };
2608
2621
  // #1706's door, for a delivery (`operation: null`): the producer must be a declared peer
2609
2622
  // of this vertical with its switch on. Admitted inside the actor task, before anything
2610
2623
  // runs, so a switch pulled between two passes refuses the second. A refusal is a PAUSE:
@@ -2793,10 +2806,10 @@ export class SqliteScopeHost {
2793
2806
  .prepare('SELECT status FROM tenants WHERE tenant_id = ?')
2794
2807
  .get(tenantId);
2795
2808
  if (!tenant || tenant.status !== 'active') {
2796
- throw new Error(`tenant not active (status: ${tenant?.status ?? 'missing'}): ${tenantId}`);
2809
+ throw substratError('conflict', `tenant not active (status: ${tenant?.status ?? 'missing'}): ${tenantId}`, { reason: SCOPE_GATE_REASONS.notActive });
2797
2810
  }
2798
2811
  if (scope.status !== 'active') {
2799
- throw new Error(`scope not active (status: ${scope.status}): ${scopeId}`);
2812
+ throw substratError('conflict', `scope not active (status: ${scope.status}): ${scopeId}`, { reason: SCOPE_GATE_REASONS.notActive });
2800
2813
  }
2801
2814
  const rt = this.runtime(tenantId, scopeId);
2802
2815
  await this.applyPendingMigrations(rt);
@@ -2925,7 +2938,9 @@ export class SqliteScopeHost {
2925
2938
  * assignment would actually confer, and that is what has to be bounded. The one
2926
2939
  * resolution both `ctx.canAssign` and the host's `canAssign` answer from.
2927
2940
  */
2928
- async assignmentBound(subject, tenantId, scopeId, roleKey) {
2941
+ async assignmentBound(subject, tenantId,
2942
+ /** Null for the tenant node — the executor's `level: 'tenant'` (#1184). */
2943
+ scopeId, roleKey) {
2929
2944
  const role = this.roles.get(`${tenantId}/${roleKey}`);
2930
2945
  if (!role) {
2931
2946
  throw unknownRoleError(roleKey);
@@ -3276,6 +3291,8 @@ export class SqliteScopeHost {
3276
3291
  let replayed = false;
3277
3292
  // #1746: what this call itself emitted, for `onEmitted`. Set only after a commit.
3278
3293
  let emittedReport;
3294
+ // #1184: what this call's executors did inline, for `onExecutorOutcomes`.
3295
+ const executorOutcomes = invokeOptions?.onExecutorOutcomes ? [] : undefined;
3279
3296
  const invoked = await rt.actor.enqueue(async () => {
3280
3297
  // #1237: the invocation this call belongs to, for the duration of it. Set
3281
3298
  // INSIDE the actor task — the actor serializes invoke and dispatch alike, so
@@ -3445,7 +3462,7 @@ export class SqliteScopeHost {
3445
3462
  // #1525: still inside the actor task that set it, so these deliveries are
3446
3463
  // this call's own work — the same tail its consumers' emits are stamped in.
3447
3464
  await this.dispatch(rt, rt.invocationId);
3448
- await this.dispatchExecutors(rt, rt.invocationId);
3465
+ await this.dispatchExecutors(rt, rt.invocationId, executorOutcomes);
3449
3466
  }
3450
3467
  if (exportMark !== null && !replayed) {
3451
3468
  const q = exportedSinceQuery(exportTypes, exportMark);
@@ -3472,6 +3489,8 @@ export class SqliteScopeHost {
3472
3489
  invokeOptions?.onIdempotentReplay?.();
3473
3490
  if (emittedReport)
3474
3491
  invokeOptions?.onEmitted?.(emittedReport);
3492
+ if (executorOutcomes && !replayed && session?.mode !== 'read-only')
3493
+ invokeOptions?.onExecutorOutcomes?.(executorOutcomes);
3475
3494
  return invoked;
3476
3495
  },
3477
3496
  };
@@ -3519,7 +3538,15 @@ export class SqliteScopeHost {
3519
3538
  throw new Error(`unknown guard predicate: '${guard.predicate}' — declared by ${guard.declaredBy} ` +
3520
3539
  `before '${operation}'; no registered module contributes it (operation blocked)`);
3521
3540
  }
3522
- await predicate.handler(ctx, guard.config, input);
3541
+ try {
3542
+ await predicate.handler(ctx, guard.config, input);
3543
+ }
3544
+ catch (err) {
3545
+ // #1745: a guard refusing is recorded after the rollback, as a transition is —
3546
+ // marked here, where which guard threw is still known. Always rethrown.
3547
+ markGuardRefusal(err, guard.predicate, operation);
3548
+ throw err;
3549
+ }
3523
3550
  }
3524
3551
  }
3525
3552
  async close() {
@@ -3552,6 +3579,17 @@ export class SqliteScopeHost {
3552
3579
  * At-least-once still requires idempotent handlers. Retry is the backstop, not a
3553
3580
  * substitute.
3554
3581
  */
3582
+ /**
3583
+ * #1184: the scope reads an executor's handler may make, built over `rt` directly. The
3584
+ * handler runs INSIDE this scope's actor task, so a host read that re-enqueued — the
3585
+ * public `canAssign`, `admin.entityHistory` — would wait on the task holding it.
3586
+ */
3587
+ executorScope(rt) {
3588
+ return {
3589
+ history: async (entity, page) => readHistory({ sql: scopedSql(rt.db) }, entity, page),
3590
+ covers: (principal, roleKey, level) => this.assignmentBound(asPrincipal(principalId.parse(principal)), rt.tenantId, level === 'scope' ? rt.scopeId : null, roleKey),
3591
+ };
3592
+ }
3555
3593
  async dispatchExecutors(rt,
3556
3594
  /**
3557
3595
  * #1525: the call this drain pass is running in, or null — PASSED, not read off
@@ -3560,7 +3598,9 @@ export class SqliteScopeHost {
3560
3598
  * and journal over RPC, so the id has to travel as an argument on that side, and
3561
3599
  * a recorded fact whose correctness argument differs per adapter drifts.
3562
3600
  */
3563
- invocationId) {
3601
+ invocationId,
3602
+ /** #1184: what each attempt did, for the emitting call's `onExecutorOutcomes`. */
3603
+ outcomes) {
3564
3604
  const report = {
3565
3605
  attempted: 0,
3566
3606
  delivered: 0,
@@ -3571,15 +3611,16 @@ export class SqliteScopeHost {
3571
3611
  return report;
3572
3612
  const lines = asyncLinePass(this.invocationLineSink);
3573
3613
  try {
3574
- await this.dispatchExecutorsPass(rt, invocationId, report, lines);
3614
+ await this.dispatchExecutorsPass(rt, invocationId, report, lines, outcomes);
3575
3615
  }
3576
3616
  finally {
3577
3617
  lines.end();
3578
3618
  }
3579
3619
  return report;
3580
3620
  }
3581
- async dispatchExecutorsPass(rt, invocationId, report, lines) {
3621
+ async dispatchExecutorsPass(rt, invocationId, report, lines, outcomes) {
3582
3622
  const now = new Date().toISOString();
3623
+ const scope = this.executorScope(rt);
3583
3624
  // #2005: a non-primary scope (a fork, a snapshot, a preview of either kind) causes no
3584
3625
  // outbound effects. Every executor is host code acting with platform authority — a
3585
3626
  // connector with the tenant's credential, a plain executor with `HostAdmin` — so its
@@ -3631,26 +3672,43 @@ export class SqliteScopeHost {
3631
3672
  lines.write({ ...unit, attempt: attempts, outcome: 'dead-lettered' });
3632
3673
  continue;
3633
3674
  }
3675
+ const outcomeOf = (outcome, error) => {
3676
+ outcomes?.push(executorOutcomeOf(id, event, outcome, error));
3677
+ };
3634
3678
  if (isInert()) {
3635
3679
  const { attempts } = this.recordExecutorDelivery(rt, row.id, deliveryId, INERT_SCOPE_REASON, TERMINAL_RETRY, invocationId);
3636
3680
  report.inert = (report.inert ?? 0) + 1;
3681
+ outcomeOf('inert', INERT_SCOPE_REASON);
3637
3682
  // Its own outcome, never `delivered`: no handler ran (#2005).
3638
3683
  lines.write({ ...unit, attempt: attempts, outcome: 'inert' });
3639
3684
  continue;
3640
3685
  }
3641
3686
  this.causedBy = event.id;
3642
3687
  try {
3688
+ let result;
3643
3689
  if (executor.kind === 'connector') {
3644
3690
  // `true`: dispatchExecutors is only ever reached from inside
3645
3691
  // `rt.actor.enqueue` (invoke's post-commit tail, or drainDue).
3646
3692
  await executor.handler(this.connectorContext(rt, executor.timeoutMs, true, event.id), event);
3647
3693
  }
3648
3694
  else {
3649
- await executor.handler(this.admin, event);
3695
+ result = await executor.handler(this.admin, event, scope);
3696
+ }
3697
+ // #1184: a refusal is the handler's own terminal decision — journaled with its
3698
+ // reason, never retried, and listed by `executorDeadLetters` beside an exhausted one.
3699
+ const refused = isDeliveryRefusal(result) ? result : null;
3700
+ const text = refused && refusalJournalText(refused);
3701
+ const { attempts } = this.recordExecutorDelivery(rt, row.id, deliveryId, text, refused ? TERMINAL_RETRY : executor.retry, invocationId);
3702
+ if (refused) {
3703
+ report.deadLettered += 1;
3704
+ outcomeOf('refused', refused.reason);
3705
+ lines.write({ ...unit, attempt: attempts, outcome: 'dead-lettered', error: text });
3706
+ }
3707
+ else {
3708
+ report.delivered += 1;
3709
+ outcomeOf('delivered');
3710
+ lines.write({ ...unit, attempt: attempts, outcome: 'delivered' });
3650
3711
  }
3651
- const { attempts } = this.recordExecutorDelivery(rt, row.id, deliveryId, null, executor.retry, invocationId);
3652
- report.delivered += 1;
3653
- lines.write({ ...unit, attempt: attempts, outcome: 'delivered' });
3654
3712
  }
3655
3713
  catch (err) {
3656
3714
  const { dead, attempts } = this.recordExecutorDelivery(rt, row.id, deliveryId, err instanceof Error ? (err.stack ?? err.message) : String(err), executor.retry, invocationId);
@@ -3658,6 +3716,7 @@ export class SqliteScopeHost {
3658
3716
  report.deadLettered += 1;
3659
3717
  else
3660
3718
  report.retrying += 1;
3719
+ outcomeOf(dead ? 'dead-lettered' : 'retrying', err);
3661
3720
  lines.write({ ...unit, attempt: attempts, outcome: dead ? 'dead-lettered' : 'retrying', error: err });
3662
3721
  }
3663
3722
  finally {
@@ -3715,13 +3774,14 @@ export class SqliteScopeHost {
3715
3774
  // emitting operation's own tail made.
3716
3775
  return rt.actor.enqueue(() => this.dispatchExecutors(rt, null));
3717
3776
  }
3718
- registerJob(moduleId, name, handler, retry) {
3777
+ registerJob(moduleId, name, handler, retry, options) {
3719
3778
  // #1575: the kernel's own jobs are dispatched before this registry is read.
3720
3779
  assertJobRegistrable(moduleId, name);
3780
+ assertLeaseMs(options?.leaseMs);
3721
3781
  const key = `${moduleId}/${name}`;
3722
3782
  if (this.jobs.has(key))
3723
3783
  throw new Error(`job '${key}' is already registered`);
3724
- this.jobs.set(key, { handler, retry });
3784
+ this.jobs.set(key, { handler, retry, leaseMs: options?.leaseMs });
3725
3785
  }
3726
3786
  /**
3727
3787
  * D-14's in-process driver: the run store is direct SQL on the scope db.
@@ -3754,11 +3814,24 @@ export class SqliteScopeHost {
3754
3814
  (id, module_id, job, instance, payload, status, cursor, counters, attempts,
3755
3815
  last_error, started_at, updated_at, next_attempt_at, ended_at, subject_id)
3756
3816
  VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
3757
- // A compare-and-set on `running` (#1632): a pass that outlived an erasure must not
3758
- // write its cursor back over the redaction. The kernel's statement, so both adapters
3759
- // hold the same line.
3817
+ // A compare-and-set on `running` (#1632) and on the pass's lease (#2034): a pass that
3818
+ // outlived an erasure, or lost its run to a takeover, must not write back over it. The
3819
+ // kernel's statement, so both adapters hold the same line.
3760
3820
  const patchRun = db.prepare(JOB_RUN_PATCH_SQL);
3761
- const patchArgs = (id, p) => [p.status, p.cursor, p.counters, p.attempts, p.lastError, p.updatedAt, p.nextAttemptAt, p.endedAt, id];
3821
+ const patchArgs = (id, p, owner) => [p.status, p.cursor, p.counters, p.attempts, p.lastError, p.updatedAt, p.nextAttemptAt, p.endedAt, id, owner];
3822
+ const beganAt = db.prepare('SELECT lease_began_at FROM _substrat_job_runs WHERE id = ?');
3823
+ const beginRun = db.prepare(JOB_RUN_BEGIN_SQL);
3824
+ const missRun = db.prepare(JOB_RUN_MISS_SQL);
3825
+ const missSettle = db.prepare(JOB_RUN_MISS_SETTLE_SQL);
3826
+ // #2042 r4: every lease time is this host's clock as the statement runs, inside its turn —
3827
+ // never a time the drive computed, which a skew or a wait in the queue would make wrong.
3828
+ const now = () => this.clock();
3829
+ const plus = (at, ms) => new Date(Date.parse(at) + ms).toISOString();
3830
+ const claimRun = db.prepare(JOB_RUN_CLAIM_SQL);
3831
+ const renewRun = db.prepare(JOB_RUN_RENEW_SQL);
3832
+ const recordStepRow = db.prepare(JOB_STEP_RECORD_SQL);
3833
+ const dropSteps = db.prepare('DELETE FROM _substrat_job_steps WHERE run_id = ?');
3834
+ const stepRow = db.prepare('SELECT step, result, attempts, last_error FROM _substrat_job_steps WHERE run_id = ? AND step = ?');
3762
3835
  // `db.transaction` on better-sqlite3 runs its body SYNCHRONOUSLY inside a real
3763
3836
  // SQLite transaction, so nothing — not another turn of the event loop, not
3764
3837
  // another caller of this host — can interleave between the read and the write.
@@ -3770,9 +3843,37 @@ export class SqliteScopeHost {
3770
3843
  insertRun.run(r.id, r.module_id, r.job, r.instance, r.payload, r.status, r.cursor, r.counters, r.attempts, r.last_error, r.started_at, r.updated_at, r.next_attempt_at, r.ended_at, r.subject_id ?? null);
3771
3844
  return r;
3772
3845
  });
3773
- const commitPassTx = db.transaction((id, p) => {
3774
- patchRun.run(...patchArgs(id, p));
3775
- db.prepare('DELETE FROM _substrat_job_steps WHERE run_id = ?').run(id);
3846
+ const commitPassTx = db.transaction((id, p, owner) => {
3847
+ // #2034: the ledger goes only with a patch that applied — a stale holder's commit must
3848
+ // not empty the ledger of the pass that took the run over.
3849
+ if (patchRun.run(...patchArgs(id, p, owner)).changes === 0)
3850
+ return false;
3851
+ dropSteps.run(id);
3852
+ return true;
3853
+ });
3854
+ // #2034: the claim, and whether it took over a lease, read in the same transaction.
3855
+ const claimTx = db.transaction((id, owner, leaseMs) => {
3856
+ // #2042 r2, r4: a takeover is charged only when the lease it takes had BEGUN its pass.
3857
+ const before = beganAt.get(id);
3858
+ const at = now();
3859
+ const claimed = row(claimRun.get(owner, plus(at, leaseMs), at, JOB_LEASE_EXPIRED_NOTE, id, at));
3860
+ return claimed && { run: claimed, takeover: (before?.lease_began_at ?? null) !== null };
3861
+ });
3862
+ // #2042 r3, r4: an admission miss — the relative count, then the backoff or failure it calls for.
3863
+ const missTx = db.transaction((id, owner, note) => {
3864
+ const at = now();
3865
+ const counted = missRun.get(at, id, owner);
3866
+ if (!counted)
3867
+ return null;
3868
+ const o = admissionMissOutcome(counted.admission_misses, at, note);
3869
+ missSettle.run(o.status, o.nextAttemptAt, o.endedAt, o.lastError, id, counted.admission_misses);
3870
+ return { misses: counted.admission_misses, failed: o.status === 'failed' };
3871
+ });
3872
+ const recordStepTx = db.transaction((runId, name, result, attempts, lastError, at, owner, leaseMs) => {
3873
+ if (renewRun.run(plus(now(), leaseMs), runId, owner).changes === 0)
3874
+ return false;
3875
+ recordStepRow.run(runId, name, result, attempts, lastError, at, runId, owner);
3876
+ return true;
3776
3877
  });
3777
3878
  // Every store operation takes its own short turn on the scope actor, so no
3778
3879
  // statement here can land inside an `invoke`'s open transaction. `turn` is the
@@ -3780,13 +3881,19 @@ export class SqliteScopeHost {
3780
3881
  const turn = (fn) => rt.actor.enqueue(fn);
3781
3882
  return {
3782
3883
  startOrJoin: (key, r) => turn(() => startOrJoinTx(key, r)),
3783
- get: (id) => turn(() => row(db.prepare('SELECT * FROM _substrat_job_runs WHERE id = ?').get(id))),
3784
- due: (now, limit, afterId) => turn(() => db
3785
- .prepare(`SELECT * FROM _substrat_job_runs
3884
+ claim: (id, owner, leaseMs) => turn(() => claimTx(id, owner, leaseMs)),
3885
+ // #2042 r4: BEGIN, the commitment point, judged by this host's clock as it runs.
3886
+ begin: (id, owner, marginMs) => turn(() => {
3887
+ const at = now();
3888
+ return beginRun.run(at, id, owner, plus(at, marginMs)).changes > 0;
3889
+ }),
3890
+ miss: (id, owner, note) => turn(() => missTx(id, owner, note)),
3891
+ // #1834: the drive's one snapshot — keys only, in the order runs became due.
3892
+ dueKeys: (now, max) => turn(() => db
3893
+ .prepare(`SELECT id, module_id, job FROM _substrat_job_runs
3786
3894
  WHERE status = 'running' AND (next_attempt_at IS NULL OR next_attempt_at <= ?)
3787
- AND (? IS NULL OR id > ?)
3788
- ORDER BY id LIMIT ?`)
3789
- .all(now, afterId ?? null, afterId ?? null, limit)),
3895
+ ORDER BY ${JOB_RUN_DUE_AT}, id LIMIT ?`)
3896
+ .all(now, max)),
3790
3897
  list: (filter) => turn(() => {
3791
3898
  const where = [];
3792
3899
  const params = [];
@@ -3808,17 +3915,18 @@ export class SqliteScopeHost {
3808
3915
  ORDER BY id DESC LIMIT ?`)
3809
3916
  .all(...params);
3810
3917
  }),
3811
- patch: (id, p) => turn(() => {
3812
- patchRun.run(...patchArgs(id, p));
3813
- }),
3814
- commitPass: (id, p) => turn(() => commitPassTx(id, p)),
3815
- step: (runId, name) => turn(() => db
3816
- .prepare('SELECT step, result, attempts, last_error FROM _substrat_job_steps WHERE run_id = ? AND step = ?')
3817
- .get(runId, name) ?? null),
3818
- recordStep: (runId, name, result, attempts, lastError, at) => turn(() => {
3819
- // Only while the run is still `running` (#1632) — see `JOB_STEP_RECORD_SQL`.
3820
- db.prepare(JOB_STEP_RECORD_SQL).run(runId, name, result, attempts, lastError, at, runId);
3918
+ patch: (id, p, owner) => turn(() => patchRun.run(...patchArgs(id, p, owner)).changes > 0),
3919
+ commitPass: (id, p, owner) => turn(() => commitPassTx(id, p, owner)),
3920
+ // #2034: a step boundary renews the lease; a pass that lost it reads nothing.
3921
+ beginStep: (runId, name, owner, leaseMs) => turn(() => {
3922
+ if (renewRun.run(plus(now(), leaseMs), runId, owner).changes === 0)
3923
+ return { held: false, row: null };
3924
+ return { held: true, row: stepRow.get(runId, name) ?? null };
3821
3925
  }),
3926
+ recordStep: (runId, name, result, attempts, lastError, at, owner, leaseMs) =>
3927
+ // Only while the run is still `running` (#1632) and this pass holds it (#2034), renewing
3928
+ // the lease in the same transaction — see `JOB_STEP_RECORD_SQL`.
3929
+ turn(() => recordStepTx(runId, name, result, attempts, lastError, at, owner, leaseMs)),
3822
3930
  };
3823
3931
  }
3824
3932
  /**
@@ -3947,7 +4055,7 @@ export class SqliteScopeHost {
3947
4055
  // `provisioning` allowed — a scope stuck there on a failed migration is a
3948
4056
  // sweep target. Suspended/archived are deliberate states; not disturbed.
3949
4057
  if (row.status !== 'active' && row.status !== 'provisioning') {
3950
- throw new Error(`scope not migratable (status: ${row.status}): ${scopeId}`);
4058
+ throw substratError('conflict', `scope not migratable (status: ${row.status}): ${scopeId}`);
3951
4059
  }
3952
4060
  const rt = this.runtime(tenantId, scopeId);
3953
4061
  // Nothing pending FOR THIS HOST → noop, and deliberately no state write: a
@@ -4157,11 +4265,12 @@ export class SqliteScopeHost {
4157
4265
  * point, since the denial is exactly the write the rolled-back operation could not make.
4158
4266
  */
4159
4267
  /**
4160
- * #1745: record a refused transition the operation failed with, after its rollback — the
4161
- * denial's discipline (above), for the lifecycle's refusals. Anything else is not one.
4268
+ * #1745: record a refused transition or refusing guard the operation failed with, after its
4269
+ * rollback — the denial's discipline (above), for the lifecycle's and the guards' refusals.
4270
+ * Anything else is not one.
4162
4271
  */
4163
4272
  recordRefusal(rt, subject, operation, err, invocationId, impersonation) {
4164
- const refused = refusedTransitionOf(err);
4273
+ const refused = refusalOf(err);
4165
4274
  if (!refused)
4166
4275
  return;
4167
4276
  const q = refusalInsert({
@@ -4423,14 +4532,16 @@ export class SqliteScopeHost {
4423
4532
  throw new Error(`connection not found: ${id}`);
4424
4533
  return row;
4425
4534
  }
4426
- recordAdmin(actor, action, target, before, after) {
4535
+ recordAdmin(actor, action, target, before, after,
4536
+ /** When the row says it happened; the wall clock unless the write took its time elsewhere (#1184). */
4537
+ at = new Date().toISOString()) {
4427
4538
  this.directory
4428
4539
  .prepare(`INSERT INTO _substrat_admin_log
4429
4540
  (id, actor, action, tenant_id, scope_id, vertical, before, after, caused_by, on_behalf_of, at)
4430
4541
  VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
4431
4542
  .run(ulid(), actor, action, target.tenantId ?? null, target.scopeId ?? null, target.vertical ?? null, before == null ? null : JSON.stringify(before), after == null ? null : JSON.stringify(after), this.causedBy,
4432
4543
  // #977: set only on an attributed view (`attributed`), never on the host itself.
4433
- this.onBehalfOf === null ? null : JSON.stringify(this.onBehalfOf), new Date().toISOString());
4544
+ this.onBehalfOf === null ? null : JSON.stringify(this.onBehalfOf), at);
4434
4545
  }
4435
4546
  tenantHoldsEntitlement(tenantId, key) {
4436
4547
  // An expired grant fails closed, exactly as if revoked (#33) — evaluated
@@ -4490,12 +4601,12 @@ export class SqliteScopeHost {
4490
4601
  const requirePoolServes = (provider, tenant) => {
4491
4602
  const pool = readPool(provider);
4492
4603
  if (!pool) {
4493
- throw new Error(`identity pool '${provider}' is not registered — a pool must declare its ` +
4604
+ throw substratError('conflict', `identity pool '${provider}' is not registered — a pool must declare its ` +
4494
4605
  `topology before it may link (central vs tenant-bound decides whether the ` +
4495
4606
  `same externalId in two tenants is one person or two)`);
4496
4607
  }
4497
4608
  if (pool.topology === 'tenant-bound' && pool.tenantId !== tenant) {
4498
- throw new Error(`identity pool '${provider}' is bound to tenant ${pool.tenantId} and cannot link into ${tenant}`);
4609
+ throw substratError('conflict', `identity pool '${provider}' is bound to tenant ${pool.tenantId} and cannot link into ${tenant}`);
4499
4610
  }
4500
4611
  };
4501
4612
  const mapVertical = (r) => verticalSchema.parse({
@@ -4565,7 +4676,7 @@ export class SqliteScopeHost {
4565
4676
  */
4566
4677
  const requireOrg = (tenant, id) => {
4567
4678
  if (!readOrg(tenant, id)) {
4568
- throw new Error(`unknown org ${id} in tenant ${tenant}`);
4679
+ throw substratError('not_found', `unknown org ${id} in tenant ${tenant}`);
4569
4680
  }
4570
4681
  };
4571
4682
  // The directory row → the `scope` contract. Parsed, not cast: the columns are
@@ -4637,7 +4748,7 @@ export class SqliteScopeHost {
4637
4748
  throw substratError('not_found', `unknown scope for tenant: (${tenantId}, ${scopeId})`);
4638
4749
  }
4639
4750
  if (!from.includes(row.status)) {
4640
- throw new Error(`illegal scope transition for ${action}: ${row.status} → ${to} ` +
4751
+ throw substratError('conflict', `illegal scope transition for ${action}: ${row.status} → ${to} ` +
4641
4752
  `(allowed from: ${from.join('|')})`);
4642
4753
  }
4643
4754
  // One directory transaction: the status, the audit row and — on a reap — the scope's
@@ -4661,7 +4772,7 @@ export class SqliteScopeHost {
4661
4772
  this.directory.prepare('UPDATE scopes SET status = ? WHERE scope_id = ?').run(to, scopeId);
4662
4773
  }
4663
4774
  if (to === 'reaped')
4664
- forgetSystemSwitchesOf(switchSqlOf(this.directory), scopeId);
4775
+ forgetSwitchesOf(switchSqlOf(this.directory), scopeId);
4665
4776
  // The audit target carries the scope's vertical (control-plane.md §4.4:
4666
4777
  // "vertical stays null until §4.2 lifecycle actions that name one"). It is
4667
4778
  // read from the scope rather than passed in, so the trail cannot disagree
@@ -4674,6 +4785,35 @@ export class SqliteScopeHost {
4674
4785
  (tenant_id, subject, relation, object, expires_at)
4675
4786
  VALUES (?, ?, ?, ?, ?)`)
4676
4787
  .run(tenantId, subject, relation, object, expiresAt ?? null);
4788
+ // #1184: a tenant-level removal, for inside a directory transaction — the tombstone
4789
+ // (K-21), the removal fence, raised even when nothing was held so an add still on its way
4790
+ // cannot land after this, and the audit row (`audit`), which K-21 writes only when
4791
+ // something changed. The shape of the Durable-Object adapter's `revokeAndFence`.
4792
+ //
4793
+ // All three are stamped by the HOST clock, the one that stamps a scope event's `occurredAt`:
4794
+ // the fence is compared with an add's `occurredAt`, and two clocks would let a host clock
4795
+ // running ahead of the wall push a removal under the skew window (Codex round 3).
4796
+ let revokeStmt;
4797
+ let fenceStmt;
4798
+ const revokeAndFence = (tenantId, principal, relation, object, audit) => {
4799
+ const now = this.clock();
4800
+ const changed = (revokeStmt ??= this.directory.prepare(`UPDATE _substrat_tenant_tuples SET revoked_at = ?
4801
+ WHERE tenant_id = ? AND subject = ? AND relation = ? AND object = ? AND revoked_at IS NULL`)).run(now, tenantId, `principal:${principal}`, relation, object).changes > 0;
4802
+ (fenceStmt ??= this.directory.prepare(RAISE_MEMBERSHIP_FENCE_SQL)).run(tenantId, principal, now);
4803
+ if (changed)
4804
+ audit(now);
4805
+ return changed;
4806
+ };
4807
+ const revokeTenantRoleAndFence = (actor, assignment) => {
4808
+ const { tenantId } = assignment.node;
4809
+ return revokeAndFence(tenantId, assignment.principalId, `role:${assignment.roleKey}`, `tenant:${tenantId}`, (at) => this.recordAdmin(actor, 'unassignRole', { tenantId, scopeId: null }, assignment, null, at));
4810
+ };
4811
+ // The tenant-level reads the checker makes, synchronous, for the bound inside a unit.
4812
+ const directoryReader = directoryTenantReader({
4813
+ directory: this.directory,
4814
+ getRole: (tenantId, key) => this.roles.get(`${tenantId}/${key}`),
4815
+ clock: () => this.clock(),
4816
+ });
4677
4817
  // The EXPLICIT grant: `INSERT OR REPLACE` clears a tombstone, because a re-grant must
4678
4818
  // grant. Provisioning seats with `seatScopeTuple` instead, which never un-revokes
4679
4819
  // (#1659) — keep the two apart.
@@ -4709,10 +4849,14 @@ export class SqliteScopeHost {
4709
4849
  const onNode = (node, body) => node.scopeId
4710
4850
  ? this.runtime(node.tenantId, node.scopeId).actor.turn(body)
4711
4851
  : Promise.resolve().then(body);
4712
- /** #1666: move one module's schedule switch on one scope — see `HostAdmin.revokeFromSystem`. */
4713
- const switchSystem = async (actor, raw, to) => {
4714
- const input = systemSwitch.parse(raw);
4715
- const { tenantId, scopeId } = input.node;
4852
+ /**
4853
+ * One kill switch moved on one scope — the ONE body `revokeFromSystem` / `restoreToSystem`
4854
+ * (#1666, a module's `system:` authority) and `revokeFromPeer` / `restoreToPeer` (#1706, a
4855
+ * peer vertical's) share, so the two levers cannot disagree about the record, the audit or
4856
+ * the undo. `kind` says which; `key` is the module id or the peer's slug.
4857
+ */
4858
+ const switchSubject = async (actor, kind, node, key, reason, to) => {
4859
+ const { tenantId, scopeId } = node;
4716
4860
  const scope = this.directory
4717
4861
  .prepare('SELECT tenant_id FROM scopes WHERE scope_id = ?')
4718
4862
  .get(scopeId);
@@ -4724,41 +4868,47 @@ export class SqliteScopeHost {
4724
4868
  // scope's file are separate databases, so a crash between them leaves an intent with
4725
4869
  // no recorded outcome — never a moved switch with no audit row.
4726
4870
  const operationId = ulid();
4727
- const action = to === 'off' ? 'revokeFromSystem' : 'restoreToSystem';
4871
+ const action = switchActionOf(kind, to);
4728
4872
  const target = { tenantId, scopeId };
4729
- const base = { operationId, moduleId: input.moduleId, schedules: to };
4730
- this.recordAdmin(actor, action, target, null, { ...base, phase: 'intent', reason: input.reason });
4731
- // The directory's record (#1674): ON before the scope moves, OFF after it held — see
4732
- // `recordSystemSwitchedOn` for why that order is the safe one.
4873
+ const base = { operationId, ...switchAuditSubject(kind, key, to) };
4874
+ this.recordAdmin(actor, action, target, null, { ...base, phase: 'intent', reason });
4875
+ // The directory's record (#1674; #2029 for a peer), written BEFORE the scope moves, both
4876
+ // ways, and undone if the move throws or holds nothing — see `recordSwitchedOn` and
4877
+ // (#1823) `recordSwitchedOff` for why that order is the safe one.
4733
4878
  const at = new Date().toISOString();
4734
4879
  const directorySql = switchSqlOf(this.directory);
4735
- const record = { tenantId, scopeId, moduleId: input.moduleId, actor, reason: input.reason, operationId, at };
4880
+ const record = { kind, tenantId, scopeId, key, actor, reason, operationId, at };
4736
4881
  const errorOf = (err) => (err instanceof Error ? err.message : String(err));
4737
- let prior = null;
4738
- if (to === 'on') {
4882
+ let prior;
4883
+ try {
4884
+ prior = to === 'on' ? recordSwitchedOn(directorySql, record) : recordSwitchedOff(directorySql, record);
4885
+ }
4886
+ catch (err) {
4887
+ // Nothing has moved: fail the call here, audited — the Cloudflare adapter's posture.
4739
4888
  try {
4740
- prior = recordSystemSwitchedOn(directorySql, record);
4889
+ this.recordAdmin(actor, action, target, null, { ...base, phase: 'failed', error: errorOf(err) });
4741
4890
  }
4742
- catch (err) {
4743
- // Nothing has moved: fail the call here, audited — the Cloudflare adapter's posture.
4744
- try {
4745
- this.recordAdmin(actor, action, target, null, { ...base, phase: 'failed', error: errorOf(err) });
4746
- }
4747
- catch {
4748
- // Best effort: the original error is what the caller must see.
4749
- }
4750
- throw err;
4891
+ catch {
4892
+ // Best effort: the original error is what the caller must see.
4751
4893
  }
4894
+ throw err;
4895
+ }
4896
+ // #2045: a newer call on this subject has recorded its position already: this one writes
4897
+ // nothing, here or in the scope, and says so.
4898
+ if (recordWriteSuperseded(prior, record)) {
4899
+ this.recordAdmin(actor, action, target, null, { ...base, phase: 'refused', superseded: true });
4900
+ throw substratError('conflict', switchSupersededMessage(kind, scopeId, key, to));
4752
4901
  }
4753
- /** A record write after the scope moved (review #5): retried once, never swallowed. */
4754
- const recordWrite = (write) => {
4902
+ /** The record's undo, after the scope's move threw or held nothing: retried once, never swallowed. */
4903
+ const undoRecord = () => {
4904
+ const undo = () => restoreSwitchRecord(directorySql, record, prior);
4755
4905
  try {
4756
- write();
4906
+ undo();
4757
4907
  return null;
4758
4908
  }
4759
4909
  catch {
4760
4910
  try {
4761
- write();
4911
+ undo();
4762
4912
  return null;
4763
4913
  }
4764
4914
  catch (err) {
@@ -4773,11 +4923,18 @@ export class SqliteScopeHost {
4773
4923
  // `db.transaction` issued mid-invoke became a SAVEPOINT inside it and rolled back
4774
4924
  // with it — after this verb had already audited `applied`. `turn` is re-entrant, so a
4775
4925
  // caller already inside one of this scope's tasks joins it rather than deadlocking.
4926
+ //
4927
+ // #1823 (#2030 for a peer): read in the same turn, so no tenant grant moves between the
4928
+ // read and the switch. The directory is this process's own database, so the read is
4929
+ // synchronous.
4776
4930
  const rt = this.runtime(tenantId, scopeId);
4777
- outcome = await rt.actor.turn(() => rt.db.transaction(() => switchSystemSchedules(switchSqlOf(rt.db), { moduleId: input.moduleId, scopeId, to, at }))());
4931
+ outcome = await rt.actor.turn(() => {
4932
+ const tenantHeld = tenantHoldsGrant(directorySql, kind, tenantId, key, this.clock());
4933
+ return rt.db.transaction(() => moveSwitch(switchSqlOf(rt.db), kind, { key, scopeId, to, at, tenantHeld, fence: operationId }))();
4934
+ });
4778
4935
  }
4779
4936
  catch (err) {
4780
- const recordError = recordWrite(() => restoreSystemSwitchRecord(directorySql, record, prior));
4937
+ const recordError = undoRecord();
4781
4938
  try {
4782
4939
  this.recordAdmin(actor, action, target, null, {
4783
4940
  ...base,
@@ -4791,14 +4948,17 @@ export class SqliteScopeHost {
4791
4948
  }
4792
4949
  throw err;
4793
4950
  }
4794
- // OFF is recorded once the scope held it. A refused ON moved nothing, so its record
4795
- // write is undone: left `on`, the next reconcile of a wiped scope would leave the
4796
- // module running.
4797
- const recordError = to === 'off' && outcome.held
4798
- ? recordWrite(() => recordSystemSwitchedOff(directorySql, record))
4799
- : to === 'on' && !outcome.held
4800
- ? recordWrite(() => restoreSystemSwitchRecord(directorySql, record, prior))
4801
- : null;
4951
+ // #2045: the scope has applied a newer call on this subject, so this move wrote nothing. The
4952
+ // record is that newer call's too (its write overwrote this one's), so nothing is undone.
4953
+ if (outcome.superseded) {
4954
+ this.recordAdmin(actor, action, target, null, { ...base, phase: 'refused', superseded: true });
4955
+ throw substratError('conflict', switchSupersededMessage(kind, scopeId, key, to));
4956
+ }
4957
+ // A call that held nothing moved nothing (the subject has no authority reaching the
4958
+ // scope, its own or the tenant's), so its record write is undone: left `off`, it would
4959
+ // switch the subject off the day it is installed; left `on`, the next reconcile of a
4960
+ // wiped scope would leave it running.
4961
+ const recordError = outcome.held ? null : undoRecord();
4802
4962
  this.recordAdmin(actor, action, target, null, {
4803
4963
  ...base,
4804
4964
  phase: outcome.held ? 'applied' : 'refused',
@@ -4807,15 +4967,15 @@ export class SqliteScopeHost {
4807
4967
  ...(recordError ? { recordError } : {}),
4808
4968
  });
4809
4969
  if (!outcome.held) {
4810
- throw substratError('not_found', `scope ${scopeId} holds no system grant for module '${input.moduleId}' — nothing to switch ${to} ` +
4811
- `(check the module id: it is the module's manifest id, e.g. '@substrat-run/engine-absence')` +
4970
+ throw substratError('not_found', switchNotFoundMessage(kind, scopeId, key, to) +
4812
4971
  (recordError ? `; and its directory record could not be put back (${recordError})` : ''));
4813
4972
  }
4814
- if (recordError) {
4815
- throw substratError('unavailable', `module '${input.moduleId}' is switched ${to} on scope ${scopeId}, but the directory's record of it ` +
4816
- `could not be written (${recordError}) — repeat the call to record it; a wipe of this scope would ` +
4817
- `otherwise lose the switch`);
4818
- }
4973
+ return { operationId, outcome };
4974
+ };
4975
+ /** #1666: move one module's schedule switch on one scope — see `HostAdmin.revokeFromSystem`. */
4976
+ const switchSystem = async (actor, raw, to) => {
4977
+ const input = systemSwitch.parse(raw);
4978
+ const { operationId, outcome } = await switchSubject(actor, 'system', input.node, input.moduleId, input.reason, to);
4819
4979
  return {
4820
4980
  operationId,
4821
4981
  moduleId: input.moduleId,
@@ -4824,58 +4984,10 @@ export class SqliteScopeHost {
4824
4984
  permissions: outcome.permissions,
4825
4985
  };
4826
4986
  };
4827
- /**
4828
- * #1706: move one PEER's kill switch on one scope — see `HostAdmin.revokeFromPeer`. The
4829
- * schedule switch's body with the subject swapped (`switchPeer` → `switchSubjectGrants`):
4830
- * audit first, one turn on the scope actor, outcome after, `not_found` when nothing was held.
4831
- */
4987
+ /** #1706: move one PEER's kill switch on one scope — see `HostAdmin.revokeFromPeer`. */
4832
4988
  const switchPeerAt = async (actor, raw, to) => {
4833
4989
  const input = peerSwitch.parse(raw);
4834
- const { tenantId, scopeId } = input.node;
4835
- const scope = this.directory
4836
- .prepare('SELECT tenant_id FROM scopes WHERE scope_id = ?')
4837
- .get(scopeId);
4838
- if (!scope || scope.tenant_id !== tenantId) {
4839
- throw substratError('not_found', `unknown scope for tenant: (${tenantId}, ${scopeId})`);
4840
- }
4841
- const operationId = ulid();
4842
- const action = to === 'off' ? 'revokeFromPeer' : 'restoreToPeer';
4843
- const target = { tenantId, scopeId };
4844
- const base = { operationId, vertical: input.vertical, calls: to };
4845
- this.recordAdmin(actor, action, target, null, { ...base, phase: 'intent', reason: input.reason });
4846
- let outcome;
4847
- try {
4848
- const rt = this.runtime(tenantId, scopeId);
4849
- outcome = await rt.actor.turn(() => rt.db.transaction(() => switchPeer(switchSqlOf(rt.db), {
4850
- vertical: input.vertical,
4851
- scopeId,
4852
- to,
4853
- at: new Date().toISOString(),
4854
- }))());
4855
- }
4856
- catch (err) {
4857
- try {
4858
- this.recordAdmin(actor, action, target, null, {
4859
- ...base,
4860
- phase: 'failed',
4861
- error: err instanceof Error ? err.message : String(err),
4862
- });
4863
- }
4864
- catch {
4865
- // Best effort: the original error is what the caller must see.
4866
- }
4867
- throw err;
4868
- }
4869
- this.recordAdmin(actor, action, target, null, {
4870
- ...base,
4871
- phase: outcome.held ? 'applied' : 'refused',
4872
- changed: outcome.changed,
4873
- permissions: outcome.permissions,
4874
- });
4875
- if (!outcome.held) {
4876
- throw substratError('not_found', `scope ${scopeId} holds no grant for peer '${input.vertical}' — nothing to switch ${to} ` +
4877
- `(check the slug: it is the calling vertical's registry id, as the target's \`peers\` names it)`);
4878
- }
4990
+ const { operationId, outcome } = await switchSubject(actor, 'peer', input.node, input.vertical, input.reason, to);
4879
4991
  return {
4880
4992
  operationId,
4881
4993
  vertical: input.vertical,
@@ -4950,7 +5062,7 @@ export class SqliteScopeHost {
4950
5062
  const states = await rt.actor.turn(() => systemGrantsStatus(switchSqlOf(rt.db), new Date().toISOString()));
4951
5063
  const offModules = new Set(states.filter((s) => s.schedules === 'off').map((s) => s.moduleId));
4952
5064
  const explanations = offModules.size > 0 ? lastSwitchedOff(tenantId, scopeId, offModules) : new Map();
4953
- const recorded = systemSwitchRecordsOf(switchSqlOf(this.directory), tenantId, scopeId);
5065
+ const recorded = switchRecordsOf(switchSqlOf(this.directory), 'system', tenantId, scopeId);
4954
5066
  const result = withRecorded(states, recorded).map((s) => ({
4955
5067
  moduleId: s.moduleId,
4956
5068
  schedules: s.schedules,
@@ -5059,29 +5171,22 @@ export class SqliteScopeHost {
5059
5171
  // Tombstone (K-21), never DELETE — the checker skips revoked_at rows, so the
5060
5172
  // assignment stops resolving while staying visible to audit. Guarded on
5061
5173
  // `revoked_at IS NULL` so a repeat unassign is a silent no-op.
5062
- const subject = `principal:${assignment.principalId}`;
5063
- const relation = `role:${assignment.roleKey}`;
5064
- const now = new Date().toISOString();
5065
- let changes;
5066
- if (assignment.node.scopeId) {
5067
- // On the scope actor (#1678): a revoke that joined a stranger's transaction and
5068
- // rolled back with it left the role LIVE after the caller was told it was gone.
5069
- const scopeNode = assignment.node.scopeId;
5070
- const rt = this.runtime(assignment.node.tenantId, scopeNode);
5071
- changes = await rt.actor.turn(() => rt.db
5072
- .prepare(`UPDATE _substrat_tuples SET revoked_at = ?
5073
- WHERE subject = ? AND relation = ? AND object = ? AND revoked_at IS NULL`)
5074
- .run(now, subject, relation, `scope:${scopeNode}`).changes);
5075
- }
5076
- else {
5077
- changes = this.directory
5078
- .prepare(`UPDATE _substrat_tenant_tuples SET revoked_at = ?
5079
- WHERE tenant_id = ? AND subject = ? AND relation = ? AND object = ? AND revoked_at IS NULL`)
5080
- .run(now, assignment.node.tenantId, subject, relation, `tenant:${assignment.node.tenantId}`).changes;
5174
+ const scopeNode = assignment.node.scopeId;
5175
+ if (!scopeNode) {
5176
+ // One directory transaction (#1184): the revoke, the removal fence and the audit row.
5177
+ this.directory.transaction(() => revokeTenantRoleAndFence(actor, assignment))();
5178
+ return;
5081
5179
  }
5180
+ // On the scope actor (#1678): a revoke that joined a stranger's transaction and
5181
+ // rolled back with it left the role LIVE after the caller was told it was gone.
5182
+ const rt = this.runtime(assignment.node.tenantId, scopeNode);
5183
+ const changes = await rt.actor.turn(() => rt.db
5184
+ .prepare(`UPDATE _substrat_tuples SET revoked_at = ?
5185
+ WHERE subject = ? AND relation = ? AND object = ? AND revoked_at IS NULL`)
5186
+ .run(new Date().toISOString(), `principal:${assignment.principalId}`, `role:${assignment.roleKey}`, `scope:${scopeNode}`).changes);
5082
5187
  if (changes === 0)
5083
5188
  return; // never assigned, or already revoked — idempotent, unaudited
5084
- this.recordAdmin(actor, 'unassignRole', { tenantId: assignment.node.tenantId, scopeId: assignment.node.scopeId }, assignment, null);
5189
+ this.recordAdmin(actor, 'unassignRole', { tenantId: assignment.node.tenantId, scopeId: scopeNode }, assignment, null);
5085
5190
  },
5086
5191
  grant: async (actor, raw) => {
5087
5192
  // Parsed like its `grantToConnection`/`grantToSystem` siblings, not taken on
@@ -5200,6 +5305,21 @@ export class SqliteScopeHost {
5200
5305
  this.recordAccess(actor, 'listSystemSwitches', { tenantId: filter?.tenantId ?? null }, filter ?? null, rows.length);
5201
5306
  return rows.map((r) => systemSwitchRecord.parse(r));
5202
5307
  },
5308
+ tenantHeldSystemModules: async (actor, tenantId, moduleIds) => {
5309
+ const now = this.clock();
5310
+ const held = tenantHeldOf(switchSqlOf(this.directory), 'system', tenantId, moduleIds, now);
5311
+ this.recordAccess(actor, 'tenantHeldSystemModules', { tenantId }, { moduleIds: [...moduleIds] }, held.length);
5312
+ return held;
5313
+ },
5314
+ peerSwitchCarry: async (actor, node) => {
5315
+ const directorySql = switchSqlOf(this.directory);
5316
+ const switchedOffPeers = switchedOffOf(directorySql, 'peer', node.tenantId, node.scopeId);
5317
+ const tenantHeldPeers = tenantHeldOf(directorySql, 'peer', node.tenantId, switchedOffPeers, this.clock());
5318
+ const all = switchFencesOf(directorySql, 'peer', node.tenantId, node.scopeId);
5319
+ const fences = Object.fromEntries(switchedOffPeers.map((v) => [v, all.get(v)]));
5320
+ this.recordAccess(actor, 'peerSwitchCarry', node, null, switchedOffPeers.length);
5321
+ return { switchedOffPeers, tenantHeldPeers, fences };
5322
+ },
5203
5323
  reassertSystemSwitches: async (actor, node, opts) => {
5204
5324
  this.assertScope(node.tenantId, node.scopeId);
5205
5325
  const rt = this.runtime(node.tenantId, node.scopeId);
@@ -5283,7 +5403,7 @@ export class SqliteScopeHost {
5283
5403
  // rebinding it would move another tenant's traffic. Exception: the holder is
5284
5404
  // ARCHIVED or REAPED (a deleted app, storage since wiped) — it released the
5285
5405
  // name, so the rebind reclaims it.
5286
- throw new Error(`hostname '${parsed.hostname}' is already bound to another scope`);
5406
+ throw substratError('conflict', `hostname '${parsed.hostname}' is already bound to another scope`);
5287
5407
  }
5288
5408
  // Exactly one canonical per (scope, surface): "which one do certs and
5289
5409
  // redirects use" has to have one answer, so a new canonical demotes the old.
@@ -5309,7 +5429,7 @@ export class SqliteScopeHost {
5309
5429
  .prepare('SELECT tenant_id, scope_id, status FROM hostnames WHERE hostname = ?')
5310
5430
  .get(hostname);
5311
5431
  if (!row)
5312
- throw new Error(`unknown hostname '${hostname}'`);
5432
+ throw substratError('not_found', `unknown hostname '${hostname}'`);
5313
5433
  if (row.status === status)
5314
5434
  return; // idempotent, and a no-op is not audited
5315
5435
  this.directory
@@ -5323,7 +5443,7 @@ export class SqliteScopeHost {
5323
5443
  .prepare('SELECT tenant_id, scope_id, status, custom_hostname_id, validation_records FROM hostnames WHERE hostname = ?')
5324
5444
  .get(hostname);
5325
5445
  if (!row)
5326
- throw new Error(`unknown hostname '${hostname}'`);
5446
+ throw substratError('not_found', `unknown hostname '${hostname}'`);
5327
5447
  const recordsJson = fields.validationRecords.length
5328
5448
  ? JSON.stringify(fields.validationRecords)
5329
5449
  : null;
@@ -5512,7 +5632,7 @@ export class SqliteScopeHost {
5512
5632
  if (existing.ownerTenant !== parsed.ownerTenant) {
5513
5633
  throw substratError('conflict', `vertical '${parsed.slug}' is owned by ${existing.ownerTenant ?? 'the platform'}, not ${parsed.ownerTenant ?? 'the platform'}`);
5514
5634
  }
5515
- throw new Error(`vertical '${parsed.slug}' is already registered as ${existing.source}`);
5635
+ throw substratError('conflict', `vertical '${parsed.slug}' is already registered as ${existing.source}`);
5516
5636
  }
5517
5637
  this.directory
5518
5638
  .prepare('INSERT INTO verticals (slug, name, source, owner_tenant, env_spec, install_spec, listed, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)')
@@ -5745,11 +5865,11 @@ export class SqliteScopeHost {
5745
5865
  // gate is about change, not about existence.
5746
5866
  if (outgoing) {
5747
5867
  if (outgoing.permissionDigest !== incoming.permissionDigest && !ack.permissionChange) {
5748
- throw new Error(`promotion changes the permission surface (${outgoing.permissionDigest} → ` +
5868
+ throw substratError('conflict', `promotion changes the permission surface (${outgoing.permissionDigest} → ` +
5749
5869
  `${incoming.permissionDigest}) — acknowledge it explicitly to promote`);
5750
5870
  }
5751
5871
  if (outgoing.migrationDigest !== incoming.migrationDigest && !ack.migrationChange) {
5752
- throw new Error(`promotion changes migrations (${outgoing.migrationDigest} → ` +
5872
+ throw substratError('conflict', `promotion changes migrations (${outgoing.migrationDigest} → ` +
5753
5873
  `${incoming.migrationDigest}) — acknowledge it explicitly to promote`);
5754
5874
  }
5755
5875
  // #1705 PR 3: an export an installed consumer imports, dropped or re-versioned.
@@ -6247,6 +6367,13 @@ export class SqliteScopeHost {
6247
6367
  this.recordAccess(actor, 'lifecycleFlow', { tenantId, scopeId }, delegatedReadParams.lifecycleFlow(input), flow.observation.events);
6248
6368
  return flow;
6249
6369
  },
6370
+ operationSeries: async (actor, tenantId, scopeId, input) => {
6371
+ // #1750: business volumes per bucket, for Pulse.
6372
+ const db = this.scopeReadDbFor(tenantId, scopeId);
6373
+ const series = readOperationSeries({ sql: scopedSql(db) }, input);
6374
+ this.recordAccess(actor, 'operationSeries', { tenantId, scopeId }, delegatedReadParams.operationSeries(input), operationSeriesCount(series));
6375
+ return series;
6376
+ },
6250
6377
  scopeAppliedMigrations: async (actor, tenantId, scopeId) => {
6251
6378
  const db = this.scopeReadDbFor(tenantId, scopeId);
6252
6379
  const rows = db
@@ -6298,7 +6425,7 @@ export class SqliteScopeHost {
6298
6425
  .prepare('SELECT org_id FROM orgs WHERE tenant_id = ? AND slug = ?')
6299
6426
  .get(parsed.tenantId, parsed.slug);
6300
6427
  if (slugOwner) {
6301
- throw new Error(`org slug '${parsed.slug}' already taken by ${slugOwner.org_id} (slugs are unique per tenant)`);
6428
+ throw substratError('conflict', `org slug '${parsed.slug}' already taken by ${slugOwner.org_id} (slugs are unique per tenant)`);
6302
6429
  }
6303
6430
  this.directory
6304
6431
  .prepare('INSERT INTO orgs (org_id, tenant_id, slug, name, created_at) VALUES (?, ?, ?, ?, ?)')
@@ -6326,18 +6453,39 @@ export class SqliteScopeHost {
6326
6453
  writeTenantTuple(tenantId, `principal:${principal}`, 'member', `org:${orgId}`);
6327
6454
  this.recordAdmin(actor, 'addMember', { tenantId }, null, { principal, orgId });
6328
6455
  },
6456
+ applyMembership: async (actor, change) => {
6457
+ // ONE directory transaction, synchronous throughout (#1184): the fence (an add's), the
6458
+ // bound and the write. Nothing else on this host lands between them, and a failure
6459
+ // leaves none.
6460
+ const { tenantId, principal, roleKey, boundedBy } = change;
6461
+ const assignment = { principalId: principal, roleKey, node: { tenantId, scopeId: null } };
6462
+ return this.directory.transaction(() => {
6463
+ if (change.op === 'add') {
6464
+ const fence = this.directory
6465
+ .prepare(MEMBERSHIP_FENCE_SINCE_SQL)
6466
+ .get(tenantId, principal, change.unlessRemovedSince);
6467
+ if (fence)
6468
+ return { applied: false, removedAt: fence.removed_at };
6469
+ }
6470
+ const role = directoryReader.getRole(tenantId, roleKey);
6471
+ if (!role)
6472
+ return { applied: false, unknownRole: roleKey };
6473
+ const bound = tenantCoverage(directoryReader, tenantId, boundedBy, role.permissions);
6474
+ if (!bound.covered)
6475
+ return { applied: false, missing: bound.missing };
6476
+ if (change.op === 'remove')
6477
+ return { applied: true, changed: revokeTenantRoleAndFence(actor, assignment) };
6478
+ writeTenantTuple(tenantId, `principal:${principal}`, `role:${roleKey}`, `tenant:${tenantId}`);
6479
+ this.recordAdmin(actor, 'assignRole', { tenantId, scopeId: null }, null, assignment);
6480
+ return { applied: true };
6481
+ })();
6482
+ },
6329
6483
  removeMember: async (actor, tenantId, principal, orgId) => {
6330
6484
  requireOrg(tenantId, orgId);
6331
- // Tombstone (K-21), never DELETE. Guarded on `revoked_at IS NULL` so a
6332
- // repeat revoke neither moves the timestamp nor writes a second audit row.
6333
- const info = this.directory
6334
- .prepare(`UPDATE _substrat_tenant_tuples SET revoked_at = ?
6335
- WHERE tenant_id = ? AND subject = ? AND relation = 'member' AND object = ?
6336
- AND revoked_at IS NULL`)
6337
- .run(new Date().toISOString(), tenantId, `principal:${principal}`, `org:${orgId}`);
6338
- if (info.changes === 0)
6339
- return; // never a member, or already revoked
6340
- this.recordAdmin(actor, 'removeMember', { tenantId }, { principal, orgId }, null);
6485
+ // Tombstone (K-21), never DELETE: a repeat revoke neither moves the timestamp nor writes
6486
+ // a second audit row. One transaction with the removal fence (#1184), which a no-op
6487
+ // raises too.
6488
+ this.directory.transaction(() => revokeAndFence(tenantId, principal, 'member', `org:${orgId}`, (at) => this.recordAdmin(actor, 'removeMember', { tenantId }, { principal, orgId }, null, at)))();
6341
6489
  },
6342
6490
  listMembers: async (actor, tenantId, orgId, options) => {
6343
6491
  requireOrg(tenantId, orgId);
@@ -6368,7 +6516,7 @@ export class SqliteScopeHost {
6368
6516
  .prepare('SELECT tenant_id FROM tenants WHERE slug = ?')
6369
6517
  .get(parsed.slug);
6370
6518
  if (slugOwner) {
6371
- throw new Error(`tenant slug '${parsed.slug}' already taken by ${slugOwner.tenant_id} (slugs are unique)`);
6519
+ throw substratError('conflict', `tenant slug '${parsed.slug}' already taken by ${slugOwner.tenant_id} (slugs are unique)`);
6372
6520
  }
6373
6521
  this.directory
6374
6522
  .prepare(`INSERT INTO tenants (tenant_id, slug, name, status, created_at, provisioned_by_tenant)
@@ -6379,12 +6527,12 @@ export class SqliteScopeHost {
6379
6527
  setTenantStatus: async (actor, tenantId, status) => {
6380
6528
  const before = readTenant(tenantId);
6381
6529
  if (!before)
6382
- throw new Error(`unknown tenant: ${tenantId}`);
6530
+ throw substratError('not_found', `unknown tenant: ${tenantId}`);
6383
6531
  // `reaped` is terminal and destroys data — it is unreachable here and only
6384
6532
  // ever set by reapTenant, so a plain status flip cannot forge a tombstone
6385
6533
  // over live data (§4.8, the tenant analogue of reapScope's archived-only gate).
6386
6534
  if (status === 'reaped') {
6387
- throw new Error(`tenant ${tenantId} cannot be set to 'reaped' via setTenantStatus — reap goes through reapTenant (control-plane.md §4.8)`);
6535
+ throw substratError('validation_failed', `tenant ${tenantId} cannot be set to 'reaped' via setTenantStatus — reap goes through reapTenant (control-plane.md §4.8)`);
6388
6536
  }
6389
6537
  // Stamp/clear deleting_at so the grace-window sweep can age tenants (§4.8):
6390
6538
  // entering `deleting` stamps, leaving it (an un-delete) clears — exactly the
@@ -6404,7 +6552,7 @@ export class SqliteScopeHost {
6404
6552
  setTenantName: async (actor, tenantId, name) => {
6405
6553
  const before = readTenant(tenantId);
6406
6554
  if (!before)
6407
- throw new Error(`unknown tenant: ${tenantId}`);
6555
+ throw substratError('not_found', `unknown tenant: ${tenantId}`);
6408
6556
  if (before.name === name)
6409
6557
  return; // no-op is not audited — nothing changed
6410
6558
  this.directory.prepare('UPDATE tenants SET name = ? WHERE tenant_id = ?').run(name, tenantId);
@@ -6536,7 +6684,7 @@ export class SqliteScopeHost {
6536
6684
  // the query, so the quoted identifier below carries no user input.
6537
6685
  const known = new Set(db.prepare(`SELECT name FROM sqlite_master WHERE type = 'table'`).all().map((r) => r.name));
6538
6686
  if (!known.has(input.table))
6539
- throw new Error(`unknown table '${input.table}'`);
6687
+ throw substratError('not_found', `unknown table '${input.table}'`);
6540
6688
  // The ceiling clamps; a bound SQLite would misread (NaN, non-finite, fractional,
6541
6689
  // negative) is refused instead of reaching LIMIT / OFFSET (#1632).
6542
6690
  const limit = Math.min(assertRowLimit('limit', input.limit ?? SCOPE_TABLE_PAGE_DEFAULT), SCOPE_TABLE_PAGE_MAX);
@@ -6557,7 +6705,7 @@ export class SqliteScopeHost {
6557
6705
  // itself rejects multi-statement strings and bad SQL with the driver's message.
6558
6706
  const stmt = db.prepare(sql);
6559
6707
  if (!stmt.readonly)
6560
- throw new Error('read-only console: statement is not read-only');
6708
+ throw substratError('validation_failed', 'read-only console: statement is not read-only');
6561
6709
  if (!stmt.reader)
6562
6710
  return { columns: [], rows: [], truncated: false };
6563
6711
  stmt.raw(true);
@@ -6703,6 +6851,7 @@ export class SqliteScopeHost {
6703
6851
  loadDirectoryDump(dump.tables, {
6704
6852
  columnsOf: (name) => builtColumnsOf(this.directory, name),
6705
6853
  exec: (sql) => this.directory.exec(sql),
6854
+ now: () => this.clock(),
6706
6855
  insert: (sql, rows) => {
6707
6856
  const insert = this.directory.prepare(sql);
6708
6857
  for (const row of rows)
@@ -6754,7 +6903,7 @@ export class SqliteScopeHost {
6754
6903
  throw substratError('not_found', `unknown scope for tenant: (${tenantId}, ${scopeId})`);
6755
6904
  }
6756
6905
  if (rec.status !== 'archived') {
6757
- throw new Error(`scope ${scopeId} is ${rec.status}, not archived — only an archived scope may be reaped`);
6906
+ throw substratError('conflict', `scope ${scopeId} is ${rec.status}, not archived — only an archived scope may be reaped`);
6758
6907
  }
6759
6908
  // A serving scope always holds ≥1 bound hostname; a truly-dead one has been
6760
6909
  // unbound. The dashboard delete path unbinds AT archive, but a bare console
@@ -6770,7 +6919,7 @@ export class SqliteScopeHost {
6770
6919
  .prepare('SELECT hostname FROM hostnames WHERE scope_id = ? LIMIT 1')
6771
6920
  .get(scopeId);
6772
6921
  if (bound) {
6773
- throw new Error(`scope ${scopeId} still resolves hostname '${bound.hostname}' — ` +
6922
+ throw substratError('conflict', `scope ${scopeId} still resolves hostname '${bound.hostname}' — ` +
6774
6923
  `unbind it before reaping (reap wipes storage and cannot be undone)`);
6775
6924
  }
6776
6925
  // The reader goes with the writer (#1624): an open handle on a removed file keeps
@@ -6921,9 +7070,9 @@ export class SqliteScopeHost {
6921
7070
  // reaped — an illegal source fails closed, like reapScope's archived-only gate.
6922
7071
  const before = readTenant(tenantId);
6923
7072
  if (!before)
6924
- throw new Error(`unknown tenant: ${tenantId}`);
7073
+ throw substratError('not_found', `unknown tenant: ${tenantId}`);
6925
7074
  if (before.status !== 'deleting') {
6926
- throw new Error(`tenant ${tenantId} is ${before.status}, not deleting — only a deleting tenant may be reaped`);
7075
+ throw substratError('conflict', `tenant ${tenantId} is ${before.status}, not deleting — only a deleting tenant may be reaped`);
6927
7076
  }
6928
7077
  // Clear the tenant's directory-side rows. Idempotent (set-to-empty), so a crash
6929
7078
  // mid-reap converges on retry. KEPT as the tombstone: the `tenants` row itself
@@ -6937,6 +7086,9 @@ export class SqliteScopeHost {
6937
7086
  '_substrat_entitlements', // per-tenant SKU flags
6938
7087
  'orgs', // K-22 org records
6939
7088
  '_substrat_system_switches', // #1674: the schedule switch's record, per scope
7089
+ '_substrat_membership_fences', // #1184: the latest removal, per principal
7090
+ '_substrat_peer_switches', // #2029: the peer switch's record, per scope
7091
+ '_substrat_switch_owed', // #2045: subjects owed a re-assert, per scope
6940
7092
  ];
6941
7093
  const clear = this.directory.transaction(() => {
6942
7094
  for (const table of tables) {
@@ -7055,7 +7207,7 @@ export class SqliteScopeHost {
7055
7207
  if (existing.topology === parsed.topology && existing.tenantId === parsed.tenantId) {
7056
7208
  return;
7057
7209
  }
7058
- throw new Error(`identity pool '${parsed.provider}' is already registered as ${existing.topology}` +
7210
+ throw substratError('conflict', `identity pool '${parsed.provider}' is already registered as ${existing.topology}` +
7059
7211
  `${existing.tenantId ? ` for tenant ${existing.tenantId}` : ''}`);
7060
7212
  }
7061
7213
  this.directory
@@ -7074,9 +7226,9 @@ export class SqliteScopeHost {
7074
7226
  listIdentityTenants: async (actor, provider, externalId) => {
7075
7227
  const pool = readPool(provider);
7076
7228
  if (!pool)
7077
- throw new Error(`identity pool '${provider}' is not registered`);
7229
+ throw substratError('not_found', `identity pool '${provider}' is not registered`);
7078
7230
  if (pool.topology !== 'central') {
7079
- throw new Error(`identity pool '${provider}' is tenant-bound — enumerating tenants is only ` +
7231
+ throw substratError('forbidden', `identity pool '${provider}' is tenant-bound — enumerating tenants is only ` +
7080
7232
  `meaningful on a central pool, where the same externalId is the same person`);
7081
7233
  }
7082
7234
  const tenants = this.directory
@@ -7090,9 +7242,9 @@ export class SqliteScopeHost {
7090
7242
  listIdentityMemberships: async (actor, provider, externalId) => {
7091
7243
  const pool = readPool(provider);
7092
7244
  if (!pool)
7093
- throw new Error(`identity pool '${provider}' is not registered`);
7245
+ throw substratError('not_found', `identity pool '${provider}' is not registered`);
7094
7246
  if (pool.topology !== 'central') {
7095
- throw new Error(`identity pool '${provider}' is tenant-bound — enumerating tenants is only ` +
7247
+ throw substratError('forbidden', `identity pool '${provider}' is tenant-bound — enumerating tenants is only ` +
7096
7248
  `meaningful on a central pool, where the same externalId is the same person`);
7097
7249
  }
7098
7250
  const rows = this.directory
@@ -7391,7 +7543,7 @@ export class SqliteScopeHost {
7391
7543
  if (existing) {
7392
7544
  if (existing.principal_id === parsed.principal)
7393
7545
  return; // idempotent, unaudited
7394
- throw new Error(`identity ${parsed.provider}:${parsed.externalId} in tenant ${parsed.tenantId} ` +
7546
+ throw substratError('conflict', `identity ${parsed.provider}:${parsed.externalId} in tenant ${parsed.tenantId} ` +
7395
7547
  `is already bound to ${existing.principal_id}`);
7396
7548
  }
7397
7549
  this.directory
@@ -7878,18 +8030,32 @@ export class SqliteScopeHost {
7878
8030
  * read `ensureIdentityKey` already uses — PRAGMA would do here, but not there.
7879
8031
  */
7880
8032
  ensureScheduleStateKind(db) {
8033
+ this.rebuildIfStale(db, '_substrat_schedule_state', scheduleStateHasKind, SCHEDULE_STATE_REBUILD);
8034
+ }
8035
+ /**
8036
+ * #1745: `_substrat_refusals`, rebuilt to admit a guard row (a nullable `from_state`, plus
8037
+ * `guard` and `reason`) on a scope db whose table predates it.
8038
+ */
8039
+ ensureRefusalsAdmitGuards(db) {
8040
+ this.rebuildIfStale(db, '_substrat_refusals', refusalsAdmitGuards, REFUSALS_REBUILD);
8041
+ }
8042
+ /**
8043
+ * Run a kernel create-copy-drop-rename `script` over `table` when the stored DDL fails
8044
+ * `isCurrent` — the DO adapter's `rebuildIfStale`, so the two rebuild byte-identically.
8045
+ */
8046
+ rebuildIfStale(db, table, isCurrent, script) {
7881
8047
  const row = db
7882
8048
  .prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?")
7883
- .get('_substrat_schedule_state');
7884
- if (!row || scheduleStateHasKind(row.sql))
8049
+ .get(table);
8050
+ if (!row || isCurrent(row.sql))
7885
8051
  return;
7886
- // In a transaction, for the reason the kernel constant spells out: create-copy-
8052
+ // In a transaction, for the reason the kernel constants spell out: create-copy-
7887
8053
  // drop-rename has two intermediate states and BOTH are unrecoverable on the next
7888
8054
  // wake — one dies on the leftover scratch table, one silently orphans the rows
7889
8055
  // behind an empty table `CREATE TABLE IF NOT EXISTS` put back. `db.transaction`
7890
8056
  // nests as a SAVEPOINT, which is what makes this safe on the `loadDump` path too,
7891
8057
  // where the whole replay is already inside one.
7892
- db.transaction(() => db.exec(SCHEDULE_STATE_REBUILD))();
8058
+ db.transaction(() => db.exec(script))();
7893
8059
  }
7894
8060
  /**
7895
8061
  * #1573: run a create-copy-drop-rename against the directory as ONE transaction.
@@ -8658,7 +8824,7 @@ export class SqliteScopeHost {
8658
8824
  if (!r || r.tenant_id !== tenantId)
8659
8825
  throw substratError('not_found', `unknown scope for tenant: (${tenantId}, ${scopeId})`);
8660
8826
  if (r.status === 'reaped') {
8661
- throw new Error(`scope ${scopeId} is reaped — its storage is gone and cannot be read`);
8827
+ throw substratError('conflict', `scope ${scopeId} is reaped — its storage is gone and cannot be read`);
8662
8828
  }
8663
8829
  }
8664
8830
  /**
@@ -8817,6 +8983,8 @@ export class SqliteScopeHost {
8817
8983
  // #1288: not an ensureColumn, because `kind` joins the schedule-state PRIMARY KEY
8818
8984
  // and no ALTER can widen a key. Rebuilt instead, from the kernel's statements.
8819
8985
  this.ensureScheduleStateKind(db);
8986
+ // #1745: likewise a rebuild — `from_state` loses its NOT NULL, which no ALTER can do.
8987
+ this.ensureRefusalsAdmitGuards(db);
8820
8988
  // #1525: the invocation a fired schedule ran in, on a scope DB created before the
8821
8989
  // column. Nullable, and the null is honestly "no call was carried" for every legacy
8822
8990
  // row — a past run's id cannot be recovered afterwards, exactly as the other #1525
@@ -8826,6 +8994,13 @@ export class SqliteScopeHost {
8826
8994
  // executes, the table always already has `kind` in its key, never `invocation_id`.
8827
8995
  this.ensureColumn(db, '_substrat_schedule_state', 'invocation_id', 'invocation_id TEXT');
8828
8996
  this.ensureColumn(db, '_substrat_job_runs', 'subject_id', 'subject_id TEXT');
8997
+ // #2034: the lease, on a scope DB built before it. NULL = nobody holds the run.
8998
+ this.ensureColumn(db, '_substrat_job_runs', 'lease_owner', 'lease_owner TEXT');
8999
+ // #2042 r2, r4: whether the holder BEGAN its pass. NULL = it has not, which is right for any lease
9000
+ // already there: a takeover of it costs nothing.
9001
+ this.ensureColumn(db, '_substrat_job_runs', 'lease_began_at', 'lease_began_at TEXT');
9002
+ // #2042 r3: consecutive admission misses. NULL = none, right for every run already there.
9003
+ this.ensureColumn(db, '_substrat_job_runs', 'admission_misses', 'admission_misses INTEGER');
8829
9004
  // #2009: the copy classification on a scope DB built before it (NULL reads as a copy; see
8830
9005
  // `COPY_ORIGIN_DDL`).
8831
9006
  this.ensureColumn(db, '_substrat_copy_origin', 'is_copy', 'is_copy INTEGER');