@substrat-run/adapter-sqlite 0.137.0 → 0.138.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,13 +1,13 @@
1
1
  import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
2
2
  import { dirname, join } from 'node:path';
3
3
  import Database from 'better-sqlite3';
4
- import { accessLogEntry, adminLogEntry, delegatedReadParams, opsFailureEntry, opsFailureFingerprint, issueEntry, FRESHNESS_HEARTBEAT_MINUTES, sweepRunEntry, modelUsageEntry, ATTACHMENT_ADDED, ATTACHMENT_REMOVED, attachmentRecord, createTenantInput, domainEvent, domainEventInput, exportReadInput, importBatch, importCursorMove, exportsOfManifestJson, importsOfManifestJson, entitlementGrant, entitlementGrantInput, eventId, meterReading, platformRequestInput, platformRequestId, MAX_PENDING_PLATFORM_REQUESTS, identityLink, identityPool, instant, connection, capabilityGrant, connectionGrant, connectionGrantRecord, connectionSecret, systemGrant, systemSwitch, systemSwitchRecord, peerSwitch, verticalCaller, verticalSlug as verticalSlugOf, subjectRef, requestFingerprint, createConnectionInput, projectedConnectionGrant, projectedConnectionKey, moduleManifest, createOrgInput, promotionAcknowledgement, bindAcknowledgement, bindHostnameInput, channelHistoryEntry, hostnameBinding, publishVersionInput, AUTO_ADMISSION_NOTE, routeTarget, ROUTE_SCOPE_HOSTNAMES_MAX, registerVerticalInput, vertical as verticalSchema, verticalChannel, verticalVersion, objectRef, entityObjectRef, assertModuleEmittableType, assertKernelAuthoredType, grantRefFromProof, org as orgSchema, orgMembership, principalId, resolvedIdentity, identityMembership, roleDefinition, scope as scopeSchema, tenant as tenantSchema, tenantRole, subjectShredReceipt, SCOPE_TABLE_PAGE_DEFAULT, SCOPE_TABLE_PAGE_MAX, SCOPE_QUERY_ROW_MAX, verticalServingState, callsOfManifestJson, outboundOfManifestJson, listLimitOf, substratError, assertReplayableDump, delegatedReadRecord, ownerTransferAudit, copyMarkAudit, redrainEventsInput, REDRAIN_BATCH, } from '@substrat-run/contracts';
5
- import { actorOf, asPrincipal, assertAllowed, assertNoSecret, assertPermissionKey, CAPABILITY_DDL, COPY_ORIGIN_DDL, CAPABILITY_EXCHANGE_OPERATION, capabilityAttachmentWriteRefused, capabilityTokenHash, createCapabilityVerbs, createEntityEdgeVerbs, exchangeCapability as exchangeCapabilitySecret, guardSecrets, mintBecomeCapability, plausibleSessionToken, readCapabilityPage, redactSecrets, redactSecretText, moduleLog, consoleLogSink, asyncInvocationId, asyncLinePass, consoleInvocationLineSink, resolveCapabilitySession, revokeCapabilityAsPlatform, assertReadOnlyQuery, attachmentBlobKey, attachmentSha256, entitlementDenial, requiredEntitlementFor, foldMeterReading, guardSpine, guardSqlLimits, DO_SQL_LIMITS, tooManyResultColumns, tooManyTableColumns, parseValidationRecords, resolveScopeRecord, ulid, ISSUE_RETENTION_DAYS, telemetryRetentionStatements, assertRowLimit, assertRowOffset, OPS_FAILURE_RETENTION_DAYS, SWEEP_RUN_RETENTION_DAYS, SWEEP_RUNS_INTENT_INDEX, sweepRunsIntentHasKind, MODEL_USAGE_RETENTION_DAYS, foldModelUsage, backoffAt, platformRequestHistoryQuery, platformRequestOf, PLATFORM_REQUEST_COLUMNS, PLATFORM_REQUEST_REDACTION_SQL, platformRequestRedactionParams, platformRequestRedactionQuery, intentPayloadCarriesSubject, redactSubjectJobRuns, redactSubjectScopeText, redactSubjectDirectoryText, ISSUE_EXEMPLAR_OWNER_BACKFILL_SQL, issueExemplarOwner, JOB_RUN_PATCH_SQL, JOB_STEP_RECORD_SQL, DELIVERY_ERROR_REDACTION_SQL, REDACTED_DELIVERY_NOTE, seatScopeTuple, switchSystemSchedules, admitPeer, collectPeers, peerSeats, resolveVerticalInstanceFrom, switchPeer, peerGrantsStatus, systemGrantsStatus, systemScheduleState, systemSwitchedOff, systemSwitchedOffMessage, tenantSystemSwitchedOffMessage, SYSTEM_SWITCHES_BACKFILL_SQL, SYSTEM_SWITCHES_DDL, forgetSystemSwitchesOf, listSystemSwitchRecords, recordSystemSwitchedOff, recordSystemSwitchedOn, restoreSystemSwitchRecord, scopesSwitchedOffFor, tenantHoldsSystemGrant, switchedOffModulesOf, inUnitMovesToAudit, staleCarryRevertRow, staleCarryReverts, systemSwitchRecordsOf, systemSwitchesTableExists, withRecorded, VERSION_MIGRATIONS_DDL, splitManifestMigrations, splitVersionMigrationsBatch, versionMigrationsOf, writeVersionMigrations, denialListQuery, refusalListQuery, mapRefusalRow, denialSummaryQuery, denialTotalsQuery, DENIAL_WINDOW_QUERY, mapDenialRow, mapDenialSummaryBuckets, IMPERSONATION_COLUMNS, IMPERSONATION_DDL, ImpersonationRefused, assertImpersonationWrites, assertModuleEnqueueableKind, assertSessionUsable, impersonationByIdQuery, impersonationListQuery, impersonationRowValues, impersonationStampOf, mapImpersonationRow, newImpersonationSession, resolveRetryPolicy, isSecretBoxConfigured, unconfiguredSecretBox, createSubjectKeys, generateSealingKeyPair, noSealingKeyMessage, openSealed, sealTo, ConnectionSealingKeyUnavailableError, PermissionDenied, createAtomic, NotSearchable, NotListable, listIndexMigrations, listIndexPlans, listQuery, cursorOf, moduleMigrations, isSearchIndexTable, searchIndexDdl, searchIndexMigrations, searchIndexPlans, searchLimit, searchMatchExpression, searchQuery, ATTACHMENT_TEXT_DDL, assertAttachmentExtractors, assertJobRegistrable, attachmentRecordOfRow, attachmentTextJob, enqueueAttachmentText, isAttachmentTextRun, reconcileAttachmentText, recordAttachmentText, searchAttachments, entityVersionQuery, entityVersionOf, assertIfMatch, OUTBOX_ENTITY_INDEX, SCHEDULE_STATE_DDL, SCHEDULE_STATE_REBUILD, scheduleStateHasKind, VERTICAL_EVENTS_DDL, EXPORT_HOPS_SQL, IMPORT_CURSORS_SQL, IMPORT_CURSOR_OF_SQL, OUTBOX_MARK_SQL, emittedSinceQuery, emittedReportOf, EMITTED_REPORT_CAP, exportedSinceQuery, IMPORT_CURSOR_ADVANCE_SQL, IMPORT_RECORD_SQL, moveImportCursor, importCursorSourceOf, exportBreaksOf, exportBreakRefusal, bindExportBreaksOf, bindExportBreakRefusal, CrossVerticalRegistry, exportReadPlan, exportReadQuery, planExportBatch, withheldNote, JOB_RUN_DDL, JOB_RUN_DUE_AT, jobRunListLimit, jobRunOf, runDueJobRuns, startJobRun, IDEMPOTENCY_DDL, REFUSALS_TABLE_DDL, REFUSALS_INDEX, refusalInsert, refusalOf, markGuardRefusal, REFUSALS_REBUILD, refusalsAdmitGuards, assertIdempotencyKey, idempotencyLookupQuery, idempotencyPruneStatement, idempotencyRecordStatement, idempotencyOptedOutMessage, replayFor, globalFetch, createUlid, facetEvents, readHistory, walkEventCause, walkEventEffects, assertRedrainWindow, readInvocation, readDeadLetters, readLifecycleFlow, domainEventOf, readUndrainedOutbox, undrainedEventsOf, connectorCallRecord, noopConnectorCallRecorder, recordConnectorCall, settleConnectionUse, unknownRoleError, } from '@substrat-run/kernel';
4
+ import { accessLogEntry, adminLogEntry, delegatedReadParams, operationSeriesCount, opsFailureEntry, opsFailureFingerprint, issueEntry, FRESHNESS_HEARTBEAT_MINUTES, sweepRunEntry, modelUsageEntry, ATTACHMENT_ADDED, ATTACHMENT_REMOVED, attachmentRecord, createTenantInput, domainEvent, domainEventInput, exportReadInput, importBatch, importCursorMove, exportsOfManifestJson, importsOfManifestJson, entitlementGrant, entitlementGrantInput, eventId, meterReading, platformRequestInput, platformRequestId, MAX_PENDING_PLATFORM_REQUESTS, identityLink, identityPool, instant, connection, capabilityGrant, connectionGrant, connectionGrantRecord, connectionSecret, systemGrant, systemSwitch, systemSwitchRecord, peerSwitch, verticalCaller, verticalSlug as verticalSlugOf, subjectRef, requestFingerprint, createConnectionInput, projectedConnectionGrant, projectedConnectionKey, moduleManifest, createOrgInput, promotionAcknowledgement, bindAcknowledgement, bindHostnameInput, channelHistoryEntry, hostnameBinding, publishVersionInput, AUTO_ADMISSION_NOTE, routeTarget, ROUTE_SCOPE_HOSTNAMES_MAX, registerVerticalInput, vertical as verticalSchema, verticalChannel, verticalVersion, objectRef, entityObjectRef, assertModuleEmittableType, assertKernelAuthoredType, grantRefFromProof, org as orgSchema, orgMembership, principalId, resolvedIdentity, identityMembership, roleDefinition, scope as scopeSchema, tenant as tenantSchema, tenantRole, subjectShredReceipt, SCOPE_TABLE_PAGE_DEFAULT, SCOPE_TABLE_PAGE_MAX, SCOPE_QUERY_ROW_MAX, verticalServingState, callsOfManifestJson, outboundOfManifestJson, listLimitOf, SCOPE_GATE_REASONS, substratError, assertReplayableDump, delegatedReadRecord, ownerTransferAudit, copyMarkAudit, redrainEventsInput, REDRAIN_BATCH, } from '@substrat-run/contracts';
5
+ import { actorOf, asPrincipal, assertAllowed, assertNoSecret, assertPermissionKey, CAPABILITY_DDL, COPY_ORIGIN_DDL, CAPABILITY_EXCHANGE_OPERATION, capabilityAttachmentWriteRefused, capabilityTokenHash, createCapabilityVerbs, createEntityEdgeVerbs, exchangeCapability as exchangeCapabilitySecret, guardSecrets, mintBecomeCapability, plausibleSessionToken, readCapabilityPage, redactSecrets, redactSecretText, moduleLog, consoleLogSink, asyncInvocationId, asyncLinePass, consoleInvocationLineSink, resolveCapabilitySession, revokeCapabilityAsPlatform, assertReadOnlyQuery, attachmentBlobKey, attachmentSha256, entitlementDenial, requiredEntitlementFor, foldMeterReading, guardSpine, guardSqlLimits, DO_SQL_LIMITS, tooManyResultColumns, tooManyTableColumns, parseValidationRecords, resolveScopeRecord, ulid, ISSUE_RETENTION_DAYS, telemetryRetentionStatements, assertRowLimit, assertRowOffset, OPS_FAILURE_RETENTION_DAYS, SWEEP_RUN_RETENTION_DAYS, SWEEP_RUNS_INTENT_INDEX, sweepRunsIntentHasKind, MODEL_USAGE_RETENTION_DAYS, foldModelUsage, backoffAt, platformRequestHistoryQuery, platformRequestOf, PLATFORM_REQUEST_COLUMNS, PLATFORM_REQUEST_REDACTION_SQL, platformRequestRedactionParams, platformRequestRedactionQuery, intentPayloadCarriesSubject, redactSubjectJobRuns, redactSubjectScopeText, redactSubjectDirectoryText, ISSUE_EXEMPLAR_OWNER_BACKFILL_SQL, issueExemplarOwner, JOB_RUN_PATCH_SQL, JOB_RUN_CLAIM_SQL, JOB_RUN_RENEW_SQL, JOB_RUN_BEGIN_SQL, JOB_RUN_MISS_SQL, JOB_RUN_MISS_SETTLE_SQL, admissionMissOutcome, JOB_STEP_RECORD_SQL, JOB_LEASE_EXPIRED_NOTE, assertLeaseMs, DELIVERY_ERROR_REDACTION_SQL, REDACTED_DELIVERY_NOTE, seatScopeTuple, admitPeer, collectPeers, peerSeats, resolveVerticalInstanceFrom, peerGrantsStatus, systemGrantsStatus, systemScheduleState, systemSwitchedOff, systemSwitchedOffMessage, tenantSystemSwitchedOffMessage, PEER_SWITCHES_DDL, SWITCH_OWED_DDL, SWITCH_FENCES_DDL, recordWriteSuperseded, switchFencesOf, switchSupersededMessage, SWITCH_KINDS, SYSTEM_SWITCHES_DDL, forgetSwitchesOf, listSystemSwitchRecords, moveSwitch, reassertActionOf, reassertEntry, reassertOffRow, recordSwitchedOff, recordSwitchedOn, restoreSwitchRecord, scopesSwitchedOffFor, switchActionOf, switchAuditSubject, switchNotFoundMessage, switchRecordsOf, switchedOffOf, switchesBackfillSqlOf, switchesTableExists, tenantHeldOf, tenantHoldsGrant, inUnitMovesToAudit, staleCarryRevertRow, staleCarryReverts, withRecorded, VERSION_MIGRATIONS_DDL, splitManifestMigrations, splitVersionMigrationsBatch, versionMigrationsOf, writeVersionMigrations, denialListQuery, refusalListQuery, mapRefusalRow, denialSummaryQuery, denialTotalsQuery, DENIAL_WINDOW_QUERY, mapDenialRow, mapDenialSummaryBuckets, IMPERSONATION_COLUMNS, IMPERSONATION_DDL, ImpersonationRefused, assertImpersonationWrites, assertModuleEnqueueableKind, assertSessionUsable, impersonationByIdQuery, impersonationListQuery, impersonationRowValues, impersonationStampOf, mapImpersonationRow, newImpersonationSession, resolveRetryPolicy, executorOutcomeOf, isDeliveryRefusal, refusalJournalText, membershipFencesBackfillSql, MEMBERSHIP_FENCES_DDL, membershipFencesTableExists, MEMBERSHIP_FENCE_SINCE_SQL, RAISE_MEMBERSHIP_FENCE_SQL, tenantCoverage, isSecretBoxConfigured, unconfiguredSecretBox, createSubjectKeys, generateSealingKeyPair, noSealingKeyMessage, openSealed, sealTo, ConnectionSealingKeyUnavailableError, PermissionDenied, createAtomic, NotSearchable, NotListable, listIndexMigrations, listIndexPlans, listQuery, cursorOf, moduleMigrations, isSearchIndexTable, searchIndexDdl, searchIndexMigrations, searchIndexPlans, searchLimit, searchMatchExpression, searchQuery, ATTACHMENT_TEXT_DDL, assertAttachmentExtractors, assertJobRegistrable, attachmentRecordOfRow, attachmentTextJob, enqueueAttachmentText, isAttachmentTextRun, reconcileAttachmentText, recordAttachmentText, searchAttachments, entityVersionQuery, entityVersionOf, assertIfMatch, OUTBOX_ENTITY_INDEX, SCHEDULE_STATE_DDL, SCHEDULE_STATE_REBUILD, scheduleStateHasKind, VERTICAL_EVENTS_DDL, EXPORT_HOPS_SQL, emptyImportResult, IMPORT_CURSORS_SQL, IMPORT_CURSOR_OF_SQL, OUTBOX_MARK_SQL, emittedSinceQuery, emittedReportOf, EMITTED_REPORT_CAP, exportedSinceQuery, IMPORT_CURSOR_ADVANCE_SQL, IMPORT_RECORD_SQL, moveImportCursor, importCursorSourceOf, exportBreaksOf, exportBreakRefusal, bindExportBreaksOf, bindExportBreakRefusal, CrossVerticalRegistry, exportReadPlan, exportReadQuery, planExportBatch, withheldNote, JOB_RUN_DDL, JOB_RUN_DUE_AT, jobRunListLimit, jobRunOf, runDueJobRuns, startJobRun, IDEMPOTENCY_DDL, REFUSALS_TABLE_DDL, REFUSALS_INDEX, refusalInsert, refusalOf, markGuardRefusal, REFUSALS_REBUILD, refusalsAdmitGuards, assertIdempotencyKey, idempotencyLookupQuery, idempotencyPruneStatement, idempotencyRecordStatement, idempotencyOptedOutMessage, replayFor, globalFetch, createUlid, facetEvents, readHistory, walkEventCause, walkEventEffects, assertRedrainWindow, readInvocation, readDeadLetters, readLifecycleFlow, readOperationSeries, domainEventOf, readUndrainedOutbox, undrainedEventsOf, connectorCallRecord, noopConnectorCallRecorder, recordConnectorCall, settleConnectionUse, unknownRoleError, } from '@substrat-run/kernel';
6
6
  import { attributedHost } from '@substrat-run/kernel';
7
7
  import { INERT_SCOPE_REASON, isPrimaryScopeRow } from '@substrat-run/kernel';
8
8
  import { LEGACY_SCOPE_ROWS_BACKFILL, assertNoSpineReference, assertSpineTablesBuilt, capabilitiesForLoad, dumpRowsInsert, isSpineTable, loadDirectoryDump, repointScopeGrants, settleCopiedWork, emittedHere, spineColumnAdditions } from '@substrat-run/kernel';
9
9
  import { ScopeActor } from './actor.js';
10
- import { createTupleChecker } from './checker.js';
10
+ import { createTupleChecker, directoryTenantReader } from './checker.js';
11
11
  /**
12
12
  * A fault in the database itself (full, corrupt, I/O), as opposed to one statement failing.
13
13
  * The #1764 backfill logs and opens past a failed batch, but never past one of these.
@@ -243,6 +243,7 @@ const KERNEL_DDL = `
243
243
  rows_changed INTEGER,
244
244
  PRIMARY KEY (module_id, version)
245
245
  );
246
+ ${SWITCH_FENCES_DDL}
246
247
  CREATE TABLE IF NOT EXISTS _substrat_tuples (
247
248
  subject TEXT NOT NULL,
248
249
  relation TEXT NOT NULL,
@@ -1063,6 +1064,9 @@ export class SqliteScopeHost {
1063
1064
  );
1064
1065
  ${IMPERSONATION_DDL}
1065
1066
  ${SYSTEM_SWITCHES_DDL}
1067
+ ${MEMBERSHIP_FENCES_DDL}
1068
+ ${PEER_SWITCHES_DDL}
1069
+ ${SWITCH_OWED_DDL}
1066
1070
  CREATE TABLE IF NOT EXISTS _substrat_admin_log (
1067
1071
  id TEXT PRIMARY KEY,
1068
1072
  actor TEXT NOT NULL,
@@ -1237,10 +1241,16 @@ export class SqliteScopeHost {
1237
1241
  */
1238
1242
  ensureDirectorySchema() {
1239
1243
  this.directory.transaction(() => {
1240
- const switchRecordIsNew = !systemSwitchesTableExists(switchSqlOf(this.directory));
1244
+ // Each kill switch's record (#1674, #2029) is backfilled on the pass that creates its table.
1245
+ const newRecords = SWITCH_KINDS.filter((kind) => !switchesTableExists(switchSqlOf(this.directory), kind));
1246
+ const fencesAreNew = !membershipFencesTableExists(switchSqlOf(this.directory));
1241
1247
  this.buildDirectorySchema();
1242
- if (switchRecordIsNew)
1243
- this.directory.exec(SYSTEM_SWITCHES_BACKFILL_SQL);
1248
+ for (const kind of newRecords)
1249
+ this.directory.exec(switchesBackfillSqlOf(kind));
1250
+ // #1184: the removal fence, for everyone removed before it existed — the same once-only
1251
+ // gate — stamped now, by the host clock (`membershipFencesBackfillSql` says why).
1252
+ if (fencesAreNew)
1253
+ this.directory.exec(membershipFencesBackfillSql(this.clock()));
1244
1254
  })();
1245
1255
  this.splitVersionMigrations();
1246
1256
  }
@@ -1578,10 +1588,10 @@ export class SqliteScopeHost {
1578
1588
  .prepare('SELECT status FROM tenants WHERE tenant_id = ?')
1579
1589
  .get(input.tenantId);
1580
1590
  if (!tenantRow) {
1581
- throw new Error(`cannot provision scope under unknown tenant: ${input.tenantId}`);
1591
+ throw substratError('conflict', `cannot provision scope under unknown tenant: ${input.tenantId}`);
1582
1592
  }
1583
1593
  if (tenantRow.status !== 'active') {
1584
- throw new Error(`cannot provision scope under non-active tenant (status: ${tenantRow.status}): ${input.tenantId}`);
1594
+ throw substratError('conflict', `cannot provision scope under non-active tenant (status: ${tenantRow.status}): ${input.tenantId}`);
1585
1595
  }
1586
1596
  const record = resolveScopeRecord(input);
1587
1597
  // Idempotency is on the scope_id (§3.3: provisioning is idempotent and
@@ -1600,7 +1610,7 @@ export class SqliteScopeHost {
1600
1610
  .prepare("SELECT scope_id FROM scopes WHERE tenant_id = ? AND slug = ? AND status NOT IN ('archived', 'reaped')")
1601
1611
  .get(input.tenantId, record.slug);
1602
1612
  if (slugOwner) {
1603
- throw new Error(`scope slug '${record.slug}' already taken under tenant ${input.tenantId} ` +
1613
+ throw substratError('conflict', `scope slug '${record.slug}' already taken under tenant ${input.tenantId} ` +
1604
1614
  `by ${slugOwner.scope_id} (slugs are unique within a tenant)`);
1605
1615
  }
1606
1616
  this.directory
@@ -1666,51 +1676,52 @@ export class SqliteScopeHost {
1666
1676
  }
1667
1677
  /**
1668
1678
  * `HostAdmin.reassertSystemSwitches`' body (#1674), run inside a turn the caller already
1669
- * holds on the scope's actor: every module the directory records OFF on the scope is
1670
- * switched off again. Already off answers `changed: false` and writes nothing, and a
1671
- * module with no authority reaching the scope (no grant here, no tenant-level grant) is left
1672
- * alone. Audited only when something moved.
1679
+ * holds on the scope's actor: every module, and (#2029) every peer, the directory records OFF
1680
+ * on the scope is switched off again. Already off answers `changed: false` and writes nothing,
1681
+ * and a subject with no authority reaching the scope (no grant here, no tenant-level grant) is
1682
+ * left alone. Audited only when something moved, under each kind's re-assert action.
1673
1683
  *
1674
1684
  * `audit` receives each row. By default it writes to the admin log at once. A caller running
1675
1685
  * this inside a scope transaction that may still roll back (a restore's replay) collects
1676
1686
  * the rows and writes them once the transaction commits. The admin log is another
1677
1687
  * database, and a row must not outlive a move that was rolled back.
1678
1688
  */
1679
- reassertSwitchesInTurn(rt, actor, tenantId, scopeId, opts, audit = (after) => this.recordAdmin(actor, 'reassertSystemSwitch', { tenantId, scopeId }, null, after)) {
1689
+ reassertSwitchesInTurn(rt, actor, tenantId, scopeId, opts, audit = (action, after) => this.recordAdmin(actor, action, { tenantId, scopeId }, null, after)) {
1680
1690
  const at = new Date().toISOString();
1681
- // #1742 review: a move the deployment made from a stale list — the module was restored
1682
- // ON after the list was read — is undone first, so the operator's ON stands.
1683
- const recorded = systemSwitchRecordsOf(switchSqlOf(this.directory), tenantId, scopeId);
1684
- // One synchronous turn: no switch can land between this read and the moves below.
1685
- const reverted = new Set(staleCarryReverts(recorded, opts?.appliedInUnit));
1686
- for (const moduleId of reverted) {
1687
- const outcome = rt.db.transaction(() => switchSystemSchedules(switchSqlOf(rt.db), { moduleId, scopeId, to: 'on', at }))();
1688
- if (outcome.changed)
1689
- audit({ operationId: ulid(), ...staleCarryRevertRow(moduleId, outcome) });
1690
- }
1691
- const recordedOff = switchedOffModulesOf(switchSqlOf(this.directory), tenantId, scopeId);
1692
- // #1742: what a deployment already switched off inside its own unit, audited here — the
1693
- // switch below answers `changed: false` for it and would write no row.
1694
- // A move the revert above undid is not credited as an in-unit OFF.
1695
- const applied = opts?.appliedInUnit?.filter((a) => !reverted.has(a.moduleId));
1696
- for (const row of inUnitMovesToAudit(recordedOff, applied))
1697
- audit({ operationId: ulid(), ...row });
1698
- return recordedOff.map((moduleId) => {
1699
- // #1823: a module whose only authority here is a tenant-level grant is held too.
1700
- const tenantHeld = tenantHoldsSystemGrant(switchSqlOf(this.directory), tenantId, moduleId, this.clock());
1701
- const outcome = rt.db.transaction(() => switchSystemSchedules(switchSqlOf(rt.db), { moduleId, scopeId, to: 'off', at, tenantHeld }))();
1702
- if (outcome.changed) {
1703
- audit({
1704
- operationId: ulid(),
1705
- moduleId,
1706
- schedules: 'off',
1707
- phase: 'applied',
1708
- changed: true,
1709
- permissions: outcome.permissions,
1710
- });
1691
+ const directorySql = switchSqlOf(this.directory);
1692
+ const results = [];
1693
+ for (const kind of SWITCH_KINDS) {
1694
+ const action = reassertActionOf(kind);
1695
+ // #2045: every move carries the fence of the call the record holds, so a scope moved since by
1696
+ // a newer call stays as that call put it.
1697
+ const fences = switchFencesOf(directorySql, kind, tenantId, scopeId);
1698
+ const move = (key, to, tenantHeld) => rt.db.transaction(() => moveSwitch(switchSqlOf(rt.db), kind, { key, scopeId, to, at, tenantHeld, fence: fences.get(key) }))();
1699
+ // #1742 review: a move the deployment made from a stale list — the subject was restored
1700
+ // ON after the list was read — is undone first, so the operator's ON stands.
1701
+ const recorded = switchRecordsOf(directorySql, kind, tenantId, scopeId);
1702
+ // One synchronous turn: no switch can land between this read and the moves below.
1703
+ const reverted = new Set(staleCarryReverts(kind, recorded, opts?.appliedInUnit));
1704
+ for (const key of reverted) {
1705
+ const outcome = move(key, 'on');
1706
+ if (outcome.changed)
1707
+ audit(action, { operationId: ulid(), ...staleCarryRevertRow(kind, key, outcome) });
1711
1708
  }
1712
- return { moduleId, held: outcome.held, changed: outcome.changed };
1713
- });
1709
+ const recordedOff = switchedOffOf(directorySql, kind, tenantId, scopeId);
1710
+ // #1742: what a deployment already switched off inside its own unit, audited here — the
1711
+ // switch below answers `changed: false` for it and would write no row. A move the revert
1712
+ // above undid is not credited as an in-unit OFF.
1713
+ for (const row of inUnitMovesToAudit(kind, recordedOff, opts?.appliedInUnit, reverted)) {
1714
+ audit(action, { operationId: ulid(), ...row });
1715
+ }
1716
+ for (const key of recordedOff) {
1717
+ // #1823 (#2030): a subject whose only authority here is a tenant-level grant is held too.
1718
+ const outcome = move(key, 'off', tenantHoldsGrant(directorySql, kind, tenantId, key, this.clock()));
1719
+ if (outcome.changed)
1720
+ audit(action, { operationId: ulid(), ...reassertOffRow(kind, key, outcome.permissions) });
1721
+ results.push(reassertEntry(kind, key, outcome));
1722
+ }
1723
+ }
1724
+ return results;
1714
1725
  }
1715
1726
  async provisionTenantStore(actor, input) {
1716
1727
  // Fail closed on an unknown/inactive tenant, exactly as provisionScope does: a store
@@ -1720,10 +1731,10 @@ export class SqliteScopeHost {
1720
1731
  .prepare('SELECT status FROM tenants WHERE tenant_id = ?')
1721
1732
  .get(input.tenantId);
1722
1733
  if (!tenantRow) {
1723
- throw new Error(`cannot provision tenant store under unknown tenant: ${input.tenantId}`);
1734
+ throw substratError('not_found', `cannot provision tenant store under unknown tenant: ${input.tenantId}`);
1724
1735
  }
1725
1736
  if (tenantRow.status !== 'active') {
1726
- throw new Error(`cannot provision tenant store under non-active tenant (status: ${tenantRow.status}): ${input.tenantId}`);
1737
+ throw substratError('conflict', `cannot provision tenant store under non-active tenant (status: ${tenantRow.status}): ${input.tenantId}`);
1727
1738
  }
1728
1739
  // Idempotent on (tenant, vertical, binding): a retried provision re-resolves the SAME
1729
1740
  // store rather than minting a second database (the K-31 ready-gate retries the whole
@@ -1791,10 +1802,10 @@ export class SqliteScopeHost {
1791
1802
  .prepare('SELECT status FROM tenants WHERE tenant_id = ?')
1792
1803
  .get(input.tenantId);
1793
1804
  if (!tenantRow) {
1794
- throw new Error(`cannot provision blob store under unknown tenant: ${input.tenantId}`);
1805
+ throw substratError('not_found', `cannot provision blob store under unknown tenant: ${input.tenantId}`);
1795
1806
  }
1796
1807
  if (tenantRow.status !== 'active') {
1797
- throw new Error(`cannot provision blob store under non-active tenant (status: ${tenantRow.status}): ${input.tenantId}`);
1808
+ throw substratError('conflict', `cannot provision blob store under non-active tenant (status: ${tenantRow.status}): ${input.tenantId}`);
1798
1809
  }
1799
1810
  const existing = this.directory
1800
1811
  .prepare('SELECT ref FROM blob_stores WHERE tenant_id = ? AND vertical = ? AND binding = ?')
@@ -2316,9 +2327,9 @@ export class SqliteScopeHost {
2316
2327
  const rows = [];
2317
2328
  // #1869: the re-point's source is the separate hint when given; the dump is otherwise as sent.
2318
2329
  const source = { tables: dump.tables, scopeId: opts?.sourceScopeId ?? dump.scopeId };
2319
- await this.loadDump(tenantId, scopeId, source, (rt) => this.reassertSwitchesInTurn(rt, actor, tenantId, scopeId, undefined, (after) => rows.push(after)));
2320
- for (const after of rows)
2321
- this.recordAdmin(actor, 'reassertSystemSwitch', { tenantId, scopeId }, null, after);
2330
+ await this.loadDump(tenantId, scopeId, source, (rt) => this.reassertSwitchesInTurn(rt, actor, tenantId, scopeId, undefined, (action, after) => rows.push([action, after])));
2331
+ for (const [action, after] of rows)
2332
+ this.recordAdmin(actor, action, { tenantId, scopeId }, null, after);
2322
2333
  this.recordAdmin(actor, 'restoreScope', { tenantId, scopeId }, null, { sourceScopeId: dump.scopeId, tables: dump.tables.length, capturedAt: dump.capturedAt });
2323
2334
  }
2324
2335
  async snapshotScope(actor, tenantId, scopeId, opts) {
@@ -2351,7 +2362,7 @@ export class SqliteScopeHost {
2351
2362
  if (!rec)
2352
2363
  throw substratError('not_found', `unknown scope ${scopeId} in tenant ${tenantId}`);
2353
2364
  if (!rec.forkedFrom && rec.kind !== 'preview') {
2354
- throw new Error(`scope ${scopeId} is not a fork or preview — only previews may be deleted; ` +
2365
+ throw substratError('forbidden', `scope ${scopeId} is not a fork or preview — only previews may be deleted; ` +
2355
2366
  `archive a primary scope instead`);
2356
2367
  }
2357
2368
  // Close and evict the runtime handle before touching the file.
@@ -2364,7 +2375,7 @@ export class SqliteScopeHost {
2364
2375
  // orphaned bytes with no record (the §9 hazard).
2365
2376
  this.directory.prepare('DELETE FROM hostnames WHERE scope_id = ?').run(scopeId);
2366
2377
  rmSync(join(this.dir, `${tenantId}__${scopeId}.sqlite`), { force: true });
2367
- forgetSystemSwitchesOf(switchSqlOf(this.directory), scopeId);
2378
+ forgetSwitchesOf(switchSqlOf(this.directory), scopeId);
2368
2379
  this.directory.prepare('DELETE FROM scopes WHERE scope_id = ?').run(scopeId);
2369
2380
  this.recordAdmin(actor, 'deleteSnapshot', { tenantId, scopeId }, null, {
2370
2381
  forkedFrom: rec.forkedFrom,
@@ -2389,10 +2400,12 @@ export class SqliteScopeHost {
2389
2400
  .prepare('SELECT status FROM tenants WHERE tenant_id = ?')
2390
2401
  .get(tenantId);
2391
2402
  if (!tenantRow) {
2392
- throw new Error(`scope has no tenant record: (${tenantId}, ${scopeId})`);
2403
+ throw substratError('not_found', `scope has no tenant record: (${tenantId}, ${scopeId})`, {
2404
+ reason: SCOPE_GATE_REASONS.unrecorded,
2405
+ });
2393
2406
  }
2394
2407
  if (tenantRow.status !== 'active') {
2395
- throw new Error(`tenant not active (status: ${tenantRow.status}): ${tenantId}`);
2408
+ throw substratError('conflict', `tenant not active (status: ${tenantRow.status}): ${tenantId}`, { reason: SCOPE_GATE_REASONS.notActive });
2396
2409
  }
2397
2410
  // `provisioning` is handled BELOW rather than here, because a scope that never
2398
2411
  // finished setting up should still retry its migrations when touched — that lazy
@@ -2401,7 +2414,7 @@ export class SqliteScopeHost {
2401
2414
  // deliberate states, and running migrations for them would be work on behalf of a
2402
2415
  // request that is going to be refused anyway.
2403
2416
  if (row.status !== 'active' && row.status !== 'provisioning') {
2404
- throw new Error(`scope not active (status: ${row.status}): ${scopeId}`);
2417
+ throw substratError('conflict', `scope not active (status: ${row.status}): ${scopeId}`, { reason: SCOPE_GATE_REASONS.notActive });
2405
2418
  }
2406
2419
  const rt = this.runtime(tenantId, scopeId);
2407
2420
  await this.applyPendingMigrations(rt);
@@ -2410,7 +2423,7 @@ export class SqliteScopeHost {
2410
2423
  // the scope exists on the vertical's side (K-31). Refused, but only after the
2411
2424
  // retry above has had its chance — and if THAT is what failed, it threw with
2412
2425
  // the migration's own message, which is the one an operator needs.
2413
- throw new Error(`scope not active (status: ${row.status}): ${scopeId}`);
2426
+ throw substratError('conflict', `scope not active (status: ${row.status}): ${scopeId}`, { reason: SCOPE_GATE_REASONS.notActive });
2414
2427
  }
2415
2428
  return this.buildStub(tenantId, scopeId, rt, asPrincipal(principal), options);
2416
2429
  }
@@ -2446,7 +2459,7 @@ export class SqliteScopeHost {
2446
2459
  `runs '${scope.vertical ?? 'none'}'`);
2447
2460
  }
2448
2461
  if (scope.status !== 'active') {
2449
- throw new Error(`scope not active (status: ${scope.status}): ${scopeId}`);
2462
+ throw substratError('conflict', `scope not active (status: ${scope.status}): ${scopeId}`, { reason: SCOPE_GATE_REASONS.notActive });
2450
2463
  }
2451
2464
  const rt = this.runtime(conn.tenant_id, scopeId);
2452
2465
  await this.applyPendingMigrations(rt);
@@ -2474,7 +2487,7 @@ export class SqliteScopeHost {
2474
2487
  `runs '${scope.vertical ?? 'none'}'`);
2475
2488
  }
2476
2489
  if (scope.status !== 'active') {
2477
- throw new Error(`scope not active (status: ${scope.status}): ${scopeId}`);
2490
+ throw substratError('conflict', `scope not active (status: ${scope.status}): ${scopeId}`, { reason: SCOPE_GATE_REASONS.notActive });
2478
2491
  }
2479
2492
  const rt = this.runtime(conn.tenant_id, scopeId);
2480
2493
  await this.applyPendingMigrations(rt);
@@ -2509,7 +2522,7 @@ export class SqliteScopeHost {
2509
2522
  throw substratError('not_found', `unknown scope for tenant: (${tenantId}, ${scopeId})`);
2510
2523
  }
2511
2524
  if (scope.status !== 'active') {
2512
- throw new Error(`scope not active (status: ${scope.status}): ${scopeId}`);
2525
+ throw substratError('conflict', `scope not active (status: ${scope.status}): ${scopeId}`, { reason: SCOPE_GATE_REASONS.notActive });
2513
2526
  }
2514
2527
  const rt = this.runtime(tenantId, scopeId);
2515
2528
  await this.applyPendingMigrations(rt);
@@ -2521,7 +2534,7 @@ export class SqliteScopeHost {
2521
2534
  // and the scope must be active. Authority is then an ordinary check against
2522
2535
  // `system:<moduleId>` grants inside the stub; nothing here is a person.
2523
2536
  if (!this.modules.has(moduleId)) {
2524
- throw new Error(`module not registered on this host: ${moduleId}`);
2537
+ throw substratError('not_found', `module not registered on this host: ${moduleId}`);
2525
2538
  }
2526
2539
  const scope = this.directory
2527
2540
  .prepare('SELECT tenant_id, status FROM scopes WHERE scope_id = ?')
@@ -2530,7 +2543,7 @@ export class SqliteScopeHost {
2530
2543
  throw substratError('not_found', `unknown scope: ${scopeId}`);
2531
2544
  }
2532
2545
  if (scope.status !== 'active') {
2533
- throw new Error(`scope not active (status: ${scope.status}): ${scopeId}`);
2546
+ throw substratError('conflict', `scope not active (status: ${scope.status}): ${scopeId}`, { reason: SCOPE_GATE_REASONS.notActive });
2534
2547
  }
2535
2548
  const rt = this.runtime(tenantId, scopeId);
2536
2549
  await this.applyPendingMigrations(rt);
@@ -2600,15 +2613,11 @@ export class SqliteScopeHost {
2600
2613
  const source = batch.source;
2601
2614
  const rt = await this.openActiveScope(tenantId, scopeId);
2602
2615
  return rt.actor.enqueue(async () => {
2603
- const result = {
2604
- delivered: 0,
2605
- deadLettered: 0,
2606
- duplicates: 0,
2607
- withheld: 0,
2608
- cursor: batch.after,
2609
- stale: false,
2610
- paused: null,
2611
- };
2616
+ const result = emptyImportResult(batch);
2617
+ // #2004: a copy consumes no other vertical's events, at the door as in the sweep
2618
+ // (`scope-copy.ts`). A pause: nothing runs and the watermark stays.
2619
+ if (!this.isPrimaryInDirectory(scopeId))
2620
+ return { ...result, paused: { reason: INERT_SCOPE_REASON } };
2612
2621
  // #1706's door, for a delivery (`operation: null`): the producer must be a declared peer
2613
2622
  // of this vertical with its switch on. Admitted inside the actor task, before anything
2614
2623
  // runs, so a switch pulled between two passes refuses the second. A refusal is a PAUSE:
@@ -2797,10 +2806,10 @@ export class SqliteScopeHost {
2797
2806
  .prepare('SELECT status FROM tenants WHERE tenant_id = ?')
2798
2807
  .get(tenantId);
2799
2808
  if (!tenant || tenant.status !== 'active') {
2800
- throw new Error(`tenant not active (status: ${tenant?.status ?? 'missing'}): ${tenantId}`);
2809
+ throw substratError('conflict', `tenant not active (status: ${tenant?.status ?? 'missing'}): ${tenantId}`, { reason: SCOPE_GATE_REASONS.notActive });
2801
2810
  }
2802
2811
  if (scope.status !== 'active') {
2803
- throw new Error(`scope not active (status: ${scope.status}): ${scopeId}`);
2812
+ throw substratError('conflict', `scope not active (status: ${scope.status}): ${scopeId}`, { reason: SCOPE_GATE_REASONS.notActive });
2804
2813
  }
2805
2814
  const rt = this.runtime(tenantId, scopeId);
2806
2815
  await this.applyPendingMigrations(rt);
@@ -2929,7 +2938,9 @@ export class SqliteScopeHost {
2929
2938
  * assignment would actually confer, and that is what has to be bounded. The one
2930
2939
  * resolution both `ctx.canAssign` and the host's `canAssign` answer from.
2931
2940
  */
2932
- async assignmentBound(subject, tenantId, scopeId, roleKey) {
2941
+ async assignmentBound(subject, tenantId,
2942
+ /** Null for the tenant node — the executor's `level: 'tenant'` (#1184). */
2943
+ scopeId, roleKey) {
2933
2944
  const role = this.roles.get(`${tenantId}/${roleKey}`);
2934
2945
  if (!role) {
2935
2946
  throw unknownRoleError(roleKey);
@@ -3280,6 +3291,8 @@ export class SqliteScopeHost {
3280
3291
  let replayed = false;
3281
3292
  // #1746: what this call itself emitted, for `onEmitted`. Set only after a commit.
3282
3293
  let emittedReport;
3294
+ // #1184: what this call's executors did inline, for `onExecutorOutcomes`.
3295
+ const executorOutcomes = invokeOptions?.onExecutorOutcomes ? [] : undefined;
3283
3296
  const invoked = await rt.actor.enqueue(async () => {
3284
3297
  // #1237: the invocation this call belongs to, for the duration of it. Set
3285
3298
  // INSIDE the actor task — the actor serializes invoke and dispatch alike, so
@@ -3449,7 +3462,7 @@ export class SqliteScopeHost {
3449
3462
  // #1525: still inside the actor task that set it, so these deliveries are
3450
3463
  // this call's own work — the same tail its consumers' emits are stamped in.
3451
3464
  await this.dispatch(rt, rt.invocationId);
3452
- await this.dispatchExecutors(rt, rt.invocationId);
3465
+ await this.dispatchExecutors(rt, rt.invocationId, executorOutcomes);
3453
3466
  }
3454
3467
  if (exportMark !== null && !replayed) {
3455
3468
  const q = exportedSinceQuery(exportTypes, exportMark);
@@ -3476,6 +3489,8 @@ export class SqliteScopeHost {
3476
3489
  invokeOptions?.onIdempotentReplay?.();
3477
3490
  if (emittedReport)
3478
3491
  invokeOptions?.onEmitted?.(emittedReport);
3492
+ if (executorOutcomes && !replayed && session?.mode !== 'read-only')
3493
+ invokeOptions?.onExecutorOutcomes?.(executorOutcomes);
3479
3494
  return invoked;
3480
3495
  },
3481
3496
  };
@@ -3564,6 +3579,17 @@ export class SqliteScopeHost {
3564
3579
  * At-least-once still requires idempotent handlers. Retry is the backstop, not a
3565
3580
  * substitute.
3566
3581
  */
3582
+ /**
3583
+ * #1184: the scope reads an executor's handler may make, built over `rt` directly. The
3584
+ * handler runs INSIDE this scope's actor task, so a host read that re-enqueued — the
3585
+ * public `canAssign`, `admin.entityHistory` — would wait on the task holding it.
3586
+ */
3587
+ executorScope(rt) {
3588
+ return {
3589
+ history: async (entity, page) => readHistory({ sql: scopedSql(rt.db) }, entity, page),
3590
+ covers: (principal, roleKey, level) => this.assignmentBound(asPrincipal(principalId.parse(principal)), rt.tenantId, level === 'scope' ? rt.scopeId : null, roleKey),
3591
+ };
3592
+ }
3567
3593
  async dispatchExecutors(rt,
3568
3594
  /**
3569
3595
  * #1525: the call this drain pass is running in, or null — PASSED, not read off
@@ -3572,7 +3598,9 @@ export class SqliteScopeHost {
3572
3598
  * and journal over RPC, so the id has to travel as an argument on that side, and
3573
3599
  * a recorded fact whose correctness argument differs per adapter drifts.
3574
3600
  */
3575
- invocationId) {
3601
+ invocationId,
3602
+ /** #1184: what each attempt did, for the emitting call's `onExecutorOutcomes`. */
3603
+ outcomes) {
3576
3604
  const report = {
3577
3605
  attempted: 0,
3578
3606
  delivered: 0,
@@ -3583,15 +3611,16 @@ export class SqliteScopeHost {
3583
3611
  return report;
3584
3612
  const lines = asyncLinePass(this.invocationLineSink);
3585
3613
  try {
3586
- await this.dispatchExecutorsPass(rt, invocationId, report, lines);
3614
+ await this.dispatchExecutorsPass(rt, invocationId, report, lines, outcomes);
3587
3615
  }
3588
3616
  finally {
3589
3617
  lines.end();
3590
3618
  }
3591
3619
  return report;
3592
3620
  }
3593
- async dispatchExecutorsPass(rt, invocationId, report, lines) {
3621
+ async dispatchExecutorsPass(rt, invocationId, report, lines, outcomes) {
3594
3622
  const now = new Date().toISOString();
3623
+ const scope = this.executorScope(rt);
3595
3624
  // #2005: a non-primary scope (a fork, a snapshot, a preview of either kind) causes no
3596
3625
  // outbound effects. Every executor is host code acting with platform authority — a
3597
3626
  // connector with the tenant's credential, a plain executor with `HostAdmin` — so its
@@ -3643,26 +3672,43 @@ export class SqliteScopeHost {
3643
3672
  lines.write({ ...unit, attempt: attempts, outcome: 'dead-lettered' });
3644
3673
  continue;
3645
3674
  }
3675
+ const outcomeOf = (outcome, error) => {
3676
+ outcomes?.push(executorOutcomeOf(id, event, outcome, error));
3677
+ };
3646
3678
  if (isInert()) {
3647
3679
  const { attempts } = this.recordExecutorDelivery(rt, row.id, deliveryId, INERT_SCOPE_REASON, TERMINAL_RETRY, invocationId);
3648
3680
  report.inert = (report.inert ?? 0) + 1;
3681
+ outcomeOf('inert', INERT_SCOPE_REASON);
3649
3682
  // Its own outcome, never `delivered`: no handler ran (#2005).
3650
3683
  lines.write({ ...unit, attempt: attempts, outcome: 'inert' });
3651
3684
  continue;
3652
3685
  }
3653
3686
  this.causedBy = event.id;
3654
3687
  try {
3688
+ let result;
3655
3689
  if (executor.kind === 'connector') {
3656
3690
  // `true`: dispatchExecutors is only ever reached from inside
3657
3691
  // `rt.actor.enqueue` (invoke's post-commit tail, or drainDue).
3658
3692
  await executor.handler(this.connectorContext(rt, executor.timeoutMs, true, event.id), event);
3659
3693
  }
3660
3694
  else {
3661
- await executor.handler(this.admin, event);
3695
+ result = await executor.handler(this.admin, event, scope);
3696
+ }
3697
+ // #1184: a refusal is the handler's own terminal decision — journaled with its
3698
+ // reason, never retried, and listed by `executorDeadLetters` beside an exhausted one.
3699
+ const refused = isDeliveryRefusal(result) ? result : null;
3700
+ const text = refused && refusalJournalText(refused);
3701
+ const { attempts } = this.recordExecutorDelivery(rt, row.id, deliveryId, text, refused ? TERMINAL_RETRY : executor.retry, invocationId);
3702
+ if (refused) {
3703
+ report.deadLettered += 1;
3704
+ outcomeOf('refused', refused.reason);
3705
+ lines.write({ ...unit, attempt: attempts, outcome: 'dead-lettered', error: text });
3706
+ }
3707
+ else {
3708
+ report.delivered += 1;
3709
+ outcomeOf('delivered');
3710
+ lines.write({ ...unit, attempt: attempts, outcome: 'delivered' });
3662
3711
  }
3663
- const { attempts } = this.recordExecutorDelivery(rt, row.id, deliveryId, null, executor.retry, invocationId);
3664
- report.delivered += 1;
3665
- lines.write({ ...unit, attempt: attempts, outcome: 'delivered' });
3666
3712
  }
3667
3713
  catch (err) {
3668
3714
  const { dead, attempts } = this.recordExecutorDelivery(rt, row.id, deliveryId, err instanceof Error ? (err.stack ?? err.message) : String(err), executor.retry, invocationId);
@@ -3670,6 +3716,7 @@ export class SqliteScopeHost {
3670
3716
  report.deadLettered += 1;
3671
3717
  else
3672
3718
  report.retrying += 1;
3719
+ outcomeOf(dead ? 'dead-lettered' : 'retrying', err);
3673
3720
  lines.write({ ...unit, attempt: attempts, outcome: dead ? 'dead-lettered' : 'retrying', error: err });
3674
3721
  }
3675
3722
  finally {
@@ -3727,13 +3774,14 @@ export class SqliteScopeHost {
3727
3774
  // emitting operation's own tail made.
3728
3775
  return rt.actor.enqueue(() => this.dispatchExecutors(rt, null));
3729
3776
  }
3730
- registerJob(moduleId, name, handler, retry) {
3777
+ registerJob(moduleId, name, handler, retry, options) {
3731
3778
  // #1575: the kernel's own jobs are dispatched before this registry is read.
3732
3779
  assertJobRegistrable(moduleId, name);
3780
+ assertLeaseMs(options?.leaseMs);
3733
3781
  const key = `${moduleId}/${name}`;
3734
3782
  if (this.jobs.has(key))
3735
3783
  throw new Error(`job '${key}' is already registered`);
3736
- this.jobs.set(key, { handler, retry });
3784
+ this.jobs.set(key, { handler, retry, leaseMs: options?.leaseMs });
3737
3785
  }
3738
3786
  /**
3739
3787
  * D-14's in-process driver: the run store is direct SQL on the scope db.
@@ -3766,11 +3814,24 @@ export class SqliteScopeHost {
3766
3814
  (id, module_id, job, instance, payload, status, cursor, counters, attempts,
3767
3815
  last_error, started_at, updated_at, next_attempt_at, ended_at, subject_id)
3768
3816
  VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
3769
- // A compare-and-set on `running` (#1632): a pass that outlived an erasure must not
3770
- // write its cursor back over the redaction. The kernel's statement, so both adapters
3771
- // hold the same line.
3817
+ // A compare-and-set on `running` (#1632) and on the pass's lease (#2034): a pass that
3818
+ // outlived an erasure, or lost its run to a takeover, must not write back over it. The
3819
+ // kernel's statement, so both adapters hold the same line.
3772
3820
  const patchRun = db.prepare(JOB_RUN_PATCH_SQL);
3773
- const patchArgs = (id, p) => [p.status, p.cursor, p.counters, p.attempts, p.lastError, p.updatedAt, p.nextAttemptAt, p.endedAt, id];
3821
+ const patchArgs = (id, p, owner) => [p.status, p.cursor, p.counters, p.attempts, p.lastError, p.updatedAt, p.nextAttemptAt, p.endedAt, id, owner];
3822
+ const beganAt = db.prepare('SELECT lease_began_at FROM _substrat_job_runs WHERE id = ?');
3823
+ const beginRun = db.prepare(JOB_RUN_BEGIN_SQL);
3824
+ const missRun = db.prepare(JOB_RUN_MISS_SQL);
3825
+ const missSettle = db.prepare(JOB_RUN_MISS_SETTLE_SQL);
3826
+ // #2042 r4: every lease time is this host's clock as the statement runs, inside its turn —
3827
+ // never a time the drive computed, which a skew or a wait in the queue would make wrong.
3828
+ const now = () => this.clock();
3829
+ const plus = (at, ms) => new Date(Date.parse(at) + ms).toISOString();
3830
+ const claimRun = db.prepare(JOB_RUN_CLAIM_SQL);
3831
+ const renewRun = db.prepare(JOB_RUN_RENEW_SQL);
3832
+ const recordStepRow = db.prepare(JOB_STEP_RECORD_SQL);
3833
+ const dropSteps = db.prepare('DELETE FROM _substrat_job_steps WHERE run_id = ?');
3834
+ const stepRow = db.prepare('SELECT step, result, attempts, last_error FROM _substrat_job_steps WHERE run_id = ? AND step = ?');
3774
3835
  // `db.transaction` on better-sqlite3 runs its body SYNCHRONOUSLY inside a real
3775
3836
  // SQLite transaction, so nothing — not another turn of the event loop, not
3776
3837
  // another caller of this host — can interleave between the read and the write.
@@ -3782,9 +3843,37 @@ export class SqliteScopeHost {
3782
3843
  insertRun.run(r.id, r.module_id, r.job, r.instance, r.payload, r.status, r.cursor, r.counters, r.attempts, r.last_error, r.started_at, r.updated_at, r.next_attempt_at, r.ended_at, r.subject_id ?? null);
3783
3844
  return r;
3784
3845
  });
3785
- const commitPassTx = db.transaction((id, p) => {
3786
- patchRun.run(...patchArgs(id, p));
3787
- db.prepare('DELETE FROM _substrat_job_steps WHERE run_id = ?').run(id);
3846
+ const commitPassTx = db.transaction((id, p, owner) => {
3847
+ // #2034: the ledger goes only with a patch that applied — a stale holder's commit must
3848
+ // not empty the ledger of the pass that took the run over.
3849
+ if (patchRun.run(...patchArgs(id, p, owner)).changes === 0)
3850
+ return false;
3851
+ dropSteps.run(id);
3852
+ return true;
3853
+ });
3854
+ // #2034: the claim, and whether it took over a lease, read in the same transaction.
3855
+ const claimTx = db.transaction((id, owner, leaseMs) => {
3856
+ // #2042 r2, r4: a takeover is charged only when the lease it takes had BEGUN its pass.
3857
+ const before = beganAt.get(id);
3858
+ const at = now();
3859
+ const claimed = row(claimRun.get(owner, plus(at, leaseMs), at, JOB_LEASE_EXPIRED_NOTE, id, at));
3860
+ return claimed && { run: claimed, takeover: (before?.lease_began_at ?? null) !== null };
3861
+ });
3862
+ // #2042 r3, r4: an admission miss — the relative count, then the backoff or failure it calls for.
3863
+ const missTx = db.transaction((id, owner, note) => {
3864
+ const at = now();
3865
+ const counted = missRun.get(at, id, owner);
3866
+ if (!counted)
3867
+ return null;
3868
+ const o = admissionMissOutcome(counted.admission_misses, at, note);
3869
+ missSettle.run(o.status, o.nextAttemptAt, o.endedAt, o.lastError, id, counted.admission_misses);
3870
+ return { misses: counted.admission_misses, failed: o.status === 'failed' };
3871
+ });
3872
+ const recordStepTx = db.transaction((runId, name, result, attempts, lastError, at, owner, leaseMs) => {
3873
+ if (renewRun.run(plus(now(), leaseMs), runId, owner).changes === 0)
3874
+ return false;
3875
+ recordStepRow.run(runId, name, result, attempts, lastError, at, runId, owner);
3876
+ return true;
3788
3877
  });
3789
3878
  // Every store operation takes its own short turn on the scope actor, so no
3790
3879
  // statement here can land inside an `invoke`'s open transaction. `turn` is the
@@ -3792,7 +3881,13 @@ export class SqliteScopeHost {
3792
3881
  const turn = (fn) => rt.actor.enqueue(fn);
3793
3882
  return {
3794
3883
  startOrJoin: (key, r) => turn(() => startOrJoinTx(key, r)),
3795
- get: (id) => turn(() => row(db.prepare('SELECT * FROM _substrat_job_runs WHERE id = ?').get(id))),
3884
+ claim: (id, owner, leaseMs) => turn(() => claimTx(id, owner, leaseMs)),
3885
+ // #2042 r4: BEGIN, the commitment point, judged by this host's clock as it runs.
3886
+ begin: (id, owner, marginMs) => turn(() => {
3887
+ const at = now();
3888
+ return beginRun.run(at, id, owner, plus(at, marginMs)).changes > 0;
3889
+ }),
3890
+ miss: (id, owner, note) => turn(() => missTx(id, owner, note)),
3796
3891
  // #1834: the drive's one snapshot — keys only, in the order runs became due.
3797
3892
  dueKeys: (now, max) => turn(() => db
3798
3893
  .prepare(`SELECT id, module_id, job FROM _substrat_job_runs
@@ -3820,17 +3915,18 @@ export class SqliteScopeHost {
3820
3915
  ORDER BY id DESC LIMIT ?`)
3821
3916
  .all(...params);
3822
3917
  }),
3823
- patch: (id, p) => turn(() => {
3824
- patchRun.run(...patchArgs(id, p));
3825
- }),
3826
- commitPass: (id, p) => turn(() => commitPassTx(id, p)),
3827
- step: (runId, name) => turn(() => db
3828
- .prepare('SELECT step, result, attempts, last_error FROM _substrat_job_steps WHERE run_id = ? AND step = ?')
3829
- .get(runId, name) ?? null),
3830
- recordStep: (runId, name, result, attempts, lastError, at) => turn(() => {
3831
- // Only while the run is still `running` (#1632) — see `JOB_STEP_RECORD_SQL`.
3832
- db.prepare(JOB_STEP_RECORD_SQL).run(runId, name, result, attempts, lastError, at, runId);
3918
+ patch: (id, p, owner) => turn(() => patchRun.run(...patchArgs(id, p, owner)).changes > 0),
3919
+ commitPass: (id, p, owner) => turn(() => commitPassTx(id, p, owner)),
3920
+ // #2034: a step boundary renews the lease; a pass that lost it reads nothing.
3921
+ beginStep: (runId, name, owner, leaseMs) => turn(() => {
3922
+ if (renewRun.run(plus(now(), leaseMs), runId, owner).changes === 0)
3923
+ return { held: false, row: null };
3924
+ return { held: true, row: stepRow.get(runId, name) ?? null };
3833
3925
  }),
3926
+ recordStep: (runId, name, result, attempts, lastError, at, owner, leaseMs) =>
3927
+ // Only while the run is still `running` (#1632) and this pass holds it (#2034), renewing
3928
+ // the lease in the same transaction — see `JOB_STEP_RECORD_SQL`.
3929
+ turn(() => recordStepTx(runId, name, result, attempts, lastError, at, owner, leaseMs)),
3834
3930
  };
3835
3931
  }
3836
3932
  /**
@@ -3959,7 +4055,7 @@ export class SqliteScopeHost {
3959
4055
  // `provisioning` allowed — a scope stuck there on a failed migration is a
3960
4056
  // sweep target. Suspended/archived are deliberate states; not disturbed.
3961
4057
  if (row.status !== 'active' && row.status !== 'provisioning') {
3962
- throw new Error(`scope not migratable (status: ${row.status}): ${scopeId}`);
4058
+ throw substratError('conflict', `scope not migratable (status: ${row.status}): ${scopeId}`);
3963
4059
  }
3964
4060
  const rt = this.runtime(tenantId, scopeId);
3965
4061
  // Nothing pending FOR THIS HOST → noop, and deliberately no state write: a
@@ -4436,14 +4532,16 @@ export class SqliteScopeHost {
4436
4532
  throw new Error(`connection not found: ${id}`);
4437
4533
  return row;
4438
4534
  }
4439
- recordAdmin(actor, action, target, before, after) {
4535
+ recordAdmin(actor, action, target, before, after,
4536
+ /** When the row says it happened; the wall clock unless the write took its time elsewhere (#1184). */
4537
+ at = new Date().toISOString()) {
4440
4538
  this.directory
4441
4539
  .prepare(`INSERT INTO _substrat_admin_log
4442
4540
  (id, actor, action, tenant_id, scope_id, vertical, before, after, caused_by, on_behalf_of, at)
4443
4541
  VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
4444
4542
  .run(ulid(), actor, action, target.tenantId ?? null, target.scopeId ?? null, target.vertical ?? null, before == null ? null : JSON.stringify(before), after == null ? null : JSON.stringify(after), this.causedBy,
4445
4543
  // #977: set only on an attributed view (`attributed`), never on the host itself.
4446
- this.onBehalfOf === null ? null : JSON.stringify(this.onBehalfOf), new Date().toISOString());
4544
+ this.onBehalfOf === null ? null : JSON.stringify(this.onBehalfOf), at);
4447
4545
  }
4448
4546
  tenantHoldsEntitlement(tenantId, key) {
4449
4547
  // An expired grant fails closed, exactly as if revoked (#33) — evaluated
@@ -4503,12 +4601,12 @@ export class SqliteScopeHost {
4503
4601
  const requirePoolServes = (provider, tenant) => {
4504
4602
  const pool = readPool(provider);
4505
4603
  if (!pool) {
4506
- throw new Error(`identity pool '${provider}' is not registered — a pool must declare its ` +
4604
+ throw substratError('conflict', `identity pool '${provider}' is not registered — a pool must declare its ` +
4507
4605
  `topology before it may link (central vs tenant-bound decides whether the ` +
4508
4606
  `same externalId in two tenants is one person or two)`);
4509
4607
  }
4510
4608
  if (pool.topology === 'tenant-bound' && pool.tenantId !== tenant) {
4511
- throw new Error(`identity pool '${provider}' is bound to tenant ${pool.tenantId} and cannot link into ${tenant}`);
4609
+ throw substratError('conflict', `identity pool '${provider}' is bound to tenant ${pool.tenantId} and cannot link into ${tenant}`);
4512
4610
  }
4513
4611
  };
4514
4612
  const mapVertical = (r) => verticalSchema.parse({
@@ -4578,7 +4676,7 @@ export class SqliteScopeHost {
4578
4676
  */
4579
4677
  const requireOrg = (tenant, id) => {
4580
4678
  if (!readOrg(tenant, id)) {
4581
- throw new Error(`unknown org ${id} in tenant ${tenant}`);
4679
+ throw substratError('not_found', `unknown org ${id} in tenant ${tenant}`);
4582
4680
  }
4583
4681
  };
4584
4682
  // The directory row → the `scope` contract. Parsed, not cast: the columns are
@@ -4650,7 +4748,7 @@ export class SqliteScopeHost {
4650
4748
  throw substratError('not_found', `unknown scope for tenant: (${tenantId}, ${scopeId})`);
4651
4749
  }
4652
4750
  if (!from.includes(row.status)) {
4653
- throw new Error(`illegal scope transition for ${action}: ${row.status} → ${to} ` +
4751
+ throw substratError('conflict', `illegal scope transition for ${action}: ${row.status} → ${to} ` +
4654
4752
  `(allowed from: ${from.join('|')})`);
4655
4753
  }
4656
4754
  // One directory transaction: the status, the audit row and — on a reap — the scope's
@@ -4674,7 +4772,7 @@ export class SqliteScopeHost {
4674
4772
  this.directory.prepare('UPDATE scopes SET status = ? WHERE scope_id = ?').run(to, scopeId);
4675
4773
  }
4676
4774
  if (to === 'reaped')
4677
- forgetSystemSwitchesOf(switchSqlOf(this.directory), scopeId);
4775
+ forgetSwitchesOf(switchSqlOf(this.directory), scopeId);
4678
4776
  // The audit target carries the scope's vertical (control-plane.md §4.4:
4679
4777
  // "vertical stays null until §4.2 lifecycle actions that name one"). It is
4680
4778
  // read from the scope rather than passed in, so the trail cannot disagree
@@ -4687,6 +4785,35 @@ export class SqliteScopeHost {
4687
4785
  (tenant_id, subject, relation, object, expires_at)
4688
4786
  VALUES (?, ?, ?, ?, ?)`)
4689
4787
  .run(tenantId, subject, relation, object, expiresAt ?? null);
4788
+ // #1184: a tenant-level removal, for inside a directory transaction — the tombstone
4789
+ // (K-21), the removal fence, raised even when nothing was held so an add still on its way
4790
+ // cannot land after this, and the audit row (`audit`), which K-21 writes only when
4791
+ // something changed. The shape of the Durable-Object adapter's `revokeAndFence`.
4792
+ //
4793
+ // All three are stamped by the HOST clock, the one that stamps a scope event's `occurredAt`:
4794
+ // the fence is compared with an add's `occurredAt`, and two clocks would let a host clock
4795
+ // running ahead of the wall push a removal under the skew window (Codex round 3).
4796
+ let revokeStmt;
4797
+ let fenceStmt;
4798
+ const revokeAndFence = (tenantId, principal, relation, object, audit) => {
4799
+ const now = this.clock();
4800
+ const changed = (revokeStmt ??= this.directory.prepare(`UPDATE _substrat_tenant_tuples SET revoked_at = ?
4801
+ WHERE tenant_id = ? AND subject = ? AND relation = ? AND object = ? AND revoked_at IS NULL`)).run(now, tenantId, `principal:${principal}`, relation, object).changes > 0;
4802
+ (fenceStmt ??= this.directory.prepare(RAISE_MEMBERSHIP_FENCE_SQL)).run(tenantId, principal, now);
4803
+ if (changed)
4804
+ audit(now);
4805
+ return changed;
4806
+ };
4807
+ const revokeTenantRoleAndFence = (actor, assignment) => {
4808
+ const { tenantId } = assignment.node;
4809
+ return revokeAndFence(tenantId, assignment.principalId, `role:${assignment.roleKey}`, `tenant:${tenantId}`, (at) => this.recordAdmin(actor, 'unassignRole', { tenantId, scopeId: null }, assignment, null, at));
4810
+ };
4811
+ // The tenant-level reads the checker makes, synchronous, for the bound inside a unit.
4812
+ const directoryReader = directoryTenantReader({
4813
+ directory: this.directory,
4814
+ getRole: (tenantId, key) => this.roles.get(`${tenantId}/${key}`),
4815
+ clock: () => this.clock(),
4816
+ });
4690
4817
  // The EXPLICIT grant: `INSERT OR REPLACE` clears a tombstone, because a re-grant must
4691
4818
  // grant. Provisioning seats with `seatScopeTuple` instead, which never un-revokes
4692
4819
  // (#1659) — keep the two apart.
@@ -4722,10 +4849,14 @@ export class SqliteScopeHost {
4722
4849
  const onNode = (node, body) => node.scopeId
4723
4850
  ? this.runtime(node.tenantId, node.scopeId).actor.turn(body)
4724
4851
  : Promise.resolve().then(body);
4725
- /** #1666: move one module's schedule switch on one scope — see `HostAdmin.revokeFromSystem`. */
4726
- const switchSystem = async (actor, raw, to) => {
4727
- const input = systemSwitch.parse(raw);
4728
- const { tenantId, scopeId } = input.node;
4852
+ /**
4853
+ * One kill switch moved on one scope — the ONE body `revokeFromSystem` / `restoreToSystem`
4854
+ * (#1666, a module's `system:` authority) and `revokeFromPeer` / `restoreToPeer` (#1706, a
4855
+ * peer vertical's) share, so the two levers cannot disagree about the record, the audit or
4856
+ * the undo. `kind` says which; `key` is the module id or the peer's slug.
4857
+ */
4858
+ const switchSubject = async (actor, kind, node, key, reason, to) => {
4859
+ const { tenantId, scopeId } = node;
4729
4860
  const scope = this.directory
4730
4861
  .prepare('SELECT tenant_id FROM scopes WHERE scope_id = ?')
4731
4862
  .get(scopeId);
@@ -4737,20 +4868,20 @@ export class SqliteScopeHost {
4737
4868
  // scope's file are separate databases, so a crash between them leaves an intent with
4738
4869
  // no recorded outcome — never a moved switch with no audit row.
4739
4870
  const operationId = ulid();
4740
- const action = to === 'off' ? 'revokeFromSystem' : 'restoreToSystem';
4871
+ const action = switchActionOf(kind, to);
4741
4872
  const target = { tenantId, scopeId };
4742
- const base = { operationId, moduleId: input.moduleId, schedules: to };
4743
- this.recordAdmin(actor, action, target, null, { ...base, phase: 'intent', reason: input.reason });
4744
- // The directory's record (#1674), written BEFORE the scope moves, both ways, and undone
4745
- // if the move throws or holds nothing — see `recordSystemSwitchedOn` and (#1823)
4746
- // `recordSystemSwitchedOff` for why that order is the safe one.
4873
+ const base = { operationId, ...switchAuditSubject(kind, key, to) };
4874
+ this.recordAdmin(actor, action, target, null, { ...base, phase: 'intent', reason });
4875
+ // The directory's record (#1674; #2029 for a peer), written BEFORE the scope moves, both
4876
+ // ways, and undone if the move throws or holds nothing — see `recordSwitchedOn` and
4877
+ // (#1823) `recordSwitchedOff` for why that order is the safe one.
4747
4878
  const at = new Date().toISOString();
4748
4879
  const directorySql = switchSqlOf(this.directory);
4749
- const record = { tenantId, scopeId, moduleId: input.moduleId, actor, reason: input.reason, operationId, at };
4880
+ const record = { kind, tenantId, scopeId, key, actor, reason, operationId, at };
4750
4881
  const errorOf = (err) => (err instanceof Error ? err.message : String(err));
4751
4882
  let prior;
4752
4883
  try {
4753
- prior = to === 'on' ? recordSystemSwitchedOn(directorySql, record) : recordSystemSwitchedOff(directorySql, record);
4884
+ prior = to === 'on' ? recordSwitchedOn(directorySql, record) : recordSwitchedOff(directorySql, record);
4754
4885
  }
4755
4886
  catch (err) {
4756
4887
  // Nothing has moved: fail the call here, audited — the Cloudflare adapter's posture.
@@ -4762,9 +4893,15 @@ export class SqliteScopeHost {
4762
4893
  }
4763
4894
  throw err;
4764
4895
  }
4896
+ // #2045: a newer call on this subject has recorded its position already: this one writes
4897
+ // nothing, here or in the scope, and says so.
4898
+ if (recordWriteSuperseded(prior, record)) {
4899
+ this.recordAdmin(actor, action, target, null, { ...base, phase: 'refused', superseded: true });
4900
+ throw substratError('conflict', switchSupersededMessage(kind, scopeId, key, to));
4901
+ }
4765
4902
  /** The record's undo, after the scope's move threw or held nothing: retried once, never swallowed. */
4766
4903
  const undoRecord = () => {
4767
- const undo = () => restoreSystemSwitchRecord(directorySql, record, prior);
4904
+ const undo = () => restoreSwitchRecord(directorySql, record, prior);
4768
4905
  try {
4769
4906
  undo();
4770
4907
  return null;
@@ -4787,12 +4924,13 @@ export class SqliteScopeHost {
4787
4924
  // with it — after this verb had already audited `applied`. `turn` is re-entrant, so a
4788
4925
  // caller already inside one of this scope's tasks joins it rather than deadlocking.
4789
4926
  //
4790
- // #1823: read in the same turn, so no tenant grant moves between the read and the switch.
4791
- // The directory is this process's own database, so the read is synchronous.
4927
+ // #1823 (#2030 for a peer): read in the same turn, so no tenant grant moves between the
4928
+ // read and the switch. The directory is this process's own database, so the read is
4929
+ // synchronous.
4792
4930
  const rt = this.runtime(tenantId, scopeId);
4793
4931
  outcome = await rt.actor.turn(() => {
4794
- const tenantHeld = tenantHoldsSystemGrant(directorySql, tenantId, input.moduleId, this.clock());
4795
- return rt.db.transaction(() => switchSystemSchedules(switchSqlOf(rt.db), { moduleId: input.moduleId, scopeId, to, at, tenantHeld }))();
4932
+ const tenantHeld = tenantHoldsGrant(directorySql, kind, tenantId, key, this.clock());
4933
+ return rt.db.transaction(() => moveSwitch(switchSqlOf(rt.db), kind, { key, scopeId, to, at, tenantHeld, fence: operationId }))();
4796
4934
  });
4797
4935
  }
4798
4936
  catch (err) {
@@ -4810,10 +4948,16 @@ export class SqliteScopeHost {
4810
4948
  }
4811
4949
  throw err;
4812
4950
  }
4813
- // A call that held nothing moved nothing (the module has no authority reaching the scope,
4814
- // its own or the tenant's), so its record write is undone: left `off`, it
4815
- // would switch the module off the day it is installed; left `on`, the next reconcile of
4816
- // a wiped scope would leave the module running.
4951
+ // #2045: the scope has applied a newer call on this subject, so this move wrote nothing. The
4952
+ // record is that newer call's too (its write overwrote this one's), so nothing is undone.
4953
+ if (outcome.superseded) {
4954
+ this.recordAdmin(actor, action, target, null, { ...base, phase: 'refused', superseded: true });
4955
+ throw substratError('conflict', switchSupersededMessage(kind, scopeId, key, to));
4956
+ }
4957
+ // A call that held nothing moved nothing (the subject has no authority reaching the
4958
+ // scope, its own or the tenant's), so its record write is undone: left `off`, it would
4959
+ // switch the subject off the day it is installed; left `on`, the next reconcile of a
4960
+ // wiped scope would leave it running.
4817
4961
  const recordError = outcome.held ? null : undoRecord();
4818
4962
  this.recordAdmin(actor, action, target, null, {
4819
4963
  ...base,
@@ -4823,10 +4967,15 @@ export class SqliteScopeHost {
4823
4967
  ...(recordError ? { recordError } : {}),
4824
4968
  });
4825
4969
  if (!outcome.held) {
4826
- throw substratError('not_found', `scope ${scopeId} holds no system grant for module '${input.moduleId}' — nothing to switch ${to} ` +
4827
- `(check the module id: it is the module's manifest id, e.g. '@substrat-run/engine-absence')` +
4970
+ throw substratError('not_found', switchNotFoundMessage(kind, scopeId, key, to) +
4828
4971
  (recordError ? `; and its directory record could not be put back (${recordError})` : ''));
4829
4972
  }
4973
+ return { operationId, outcome };
4974
+ };
4975
+ /** #1666: move one module's schedule switch on one scope — see `HostAdmin.revokeFromSystem`. */
4976
+ const switchSystem = async (actor, raw, to) => {
4977
+ const input = systemSwitch.parse(raw);
4978
+ const { operationId, outcome } = await switchSubject(actor, 'system', input.node, input.moduleId, input.reason, to);
4830
4979
  return {
4831
4980
  operationId,
4832
4981
  moduleId: input.moduleId,
@@ -4835,58 +4984,10 @@ export class SqliteScopeHost {
4835
4984
  permissions: outcome.permissions,
4836
4985
  };
4837
4986
  };
4838
- /**
4839
- * #1706: move one PEER's kill switch on one scope — see `HostAdmin.revokeFromPeer`. The
4840
- * schedule switch's body with the subject swapped (`switchPeer` → `switchSubjectGrants`):
4841
- * audit first, one turn on the scope actor, outcome after, `not_found` when nothing was held.
4842
- */
4987
+ /** #1706: move one PEER's kill switch on one scope — see `HostAdmin.revokeFromPeer`. */
4843
4988
  const switchPeerAt = async (actor, raw, to) => {
4844
4989
  const input = peerSwitch.parse(raw);
4845
- const { tenantId, scopeId } = input.node;
4846
- const scope = this.directory
4847
- .prepare('SELECT tenant_id FROM scopes WHERE scope_id = ?')
4848
- .get(scopeId);
4849
- if (!scope || scope.tenant_id !== tenantId) {
4850
- throw substratError('not_found', `unknown scope for tenant: (${tenantId}, ${scopeId})`);
4851
- }
4852
- const operationId = ulid();
4853
- const action = to === 'off' ? 'revokeFromPeer' : 'restoreToPeer';
4854
- const target = { tenantId, scopeId };
4855
- const base = { operationId, vertical: input.vertical, calls: to };
4856
- this.recordAdmin(actor, action, target, null, { ...base, phase: 'intent', reason: input.reason });
4857
- let outcome;
4858
- try {
4859
- const rt = this.runtime(tenantId, scopeId);
4860
- outcome = await rt.actor.turn(() => rt.db.transaction(() => switchPeer(switchSqlOf(rt.db), {
4861
- vertical: input.vertical,
4862
- scopeId,
4863
- to,
4864
- at: new Date().toISOString(),
4865
- }))());
4866
- }
4867
- catch (err) {
4868
- try {
4869
- this.recordAdmin(actor, action, target, null, {
4870
- ...base,
4871
- phase: 'failed',
4872
- error: err instanceof Error ? err.message : String(err),
4873
- });
4874
- }
4875
- catch {
4876
- // Best effort: the original error is what the caller must see.
4877
- }
4878
- throw err;
4879
- }
4880
- this.recordAdmin(actor, action, target, null, {
4881
- ...base,
4882
- phase: outcome.held ? 'applied' : 'refused',
4883
- changed: outcome.changed,
4884
- permissions: outcome.permissions,
4885
- });
4886
- if (!outcome.held) {
4887
- throw substratError('not_found', `scope ${scopeId} holds no grant for peer '${input.vertical}' — nothing to switch ${to} ` +
4888
- `(check the slug: it is the calling vertical's registry id, as the target's \`peers\` names it)`);
4889
- }
4990
+ const { operationId, outcome } = await switchSubject(actor, 'peer', input.node, input.vertical, input.reason, to);
4890
4991
  return {
4891
4992
  operationId,
4892
4993
  vertical: input.vertical,
@@ -4961,7 +5062,7 @@ export class SqliteScopeHost {
4961
5062
  const states = await rt.actor.turn(() => systemGrantsStatus(switchSqlOf(rt.db), new Date().toISOString()));
4962
5063
  const offModules = new Set(states.filter((s) => s.schedules === 'off').map((s) => s.moduleId));
4963
5064
  const explanations = offModules.size > 0 ? lastSwitchedOff(tenantId, scopeId, offModules) : new Map();
4964
- const recorded = systemSwitchRecordsOf(switchSqlOf(this.directory), tenantId, scopeId);
5065
+ const recorded = switchRecordsOf(switchSqlOf(this.directory), 'system', tenantId, scopeId);
4965
5066
  const result = withRecorded(states, recorded).map((s) => ({
4966
5067
  moduleId: s.moduleId,
4967
5068
  schedules: s.schedules,
@@ -5070,29 +5171,22 @@ export class SqliteScopeHost {
5070
5171
  // Tombstone (K-21), never DELETE — the checker skips revoked_at rows, so the
5071
5172
  // assignment stops resolving while staying visible to audit. Guarded on
5072
5173
  // `revoked_at IS NULL` so a repeat unassign is a silent no-op.
5073
- const subject = `principal:${assignment.principalId}`;
5074
- const relation = `role:${assignment.roleKey}`;
5075
- const now = new Date().toISOString();
5076
- let changes;
5077
- if (assignment.node.scopeId) {
5078
- // On the scope actor (#1678): a revoke that joined a stranger's transaction and
5079
- // rolled back with it left the role LIVE after the caller was told it was gone.
5080
- const scopeNode = assignment.node.scopeId;
5081
- const rt = this.runtime(assignment.node.tenantId, scopeNode);
5082
- changes = await rt.actor.turn(() => rt.db
5083
- .prepare(`UPDATE _substrat_tuples SET revoked_at = ?
5084
- WHERE subject = ? AND relation = ? AND object = ? AND revoked_at IS NULL`)
5085
- .run(now, subject, relation, `scope:${scopeNode}`).changes);
5086
- }
5087
- else {
5088
- changes = this.directory
5089
- .prepare(`UPDATE _substrat_tenant_tuples SET revoked_at = ?
5090
- WHERE tenant_id = ? AND subject = ? AND relation = ? AND object = ? AND revoked_at IS NULL`)
5091
- .run(now, assignment.node.tenantId, subject, relation, `tenant:${assignment.node.tenantId}`).changes;
5174
+ const scopeNode = assignment.node.scopeId;
5175
+ if (!scopeNode) {
5176
+ // One directory transaction (#1184): the revoke, the removal fence and the audit row.
5177
+ this.directory.transaction(() => revokeTenantRoleAndFence(actor, assignment))();
5178
+ return;
5092
5179
  }
5180
+ // On the scope actor (#1678): a revoke that joined a stranger's transaction and
5181
+ // rolled back with it left the role LIVE after the caller was told it was gone.
5182
+ const rt = this.runtime(assignment.node.tenantId, scopeNode);
5183
+ const changes = await rt.actor.turn(() => rt.db
5184
+ .prepare(`UPDATE _substrat_tuples SET revoked_at = ?
5185
+ WHERE subject = ? AND relation = ? AND object = ? AND revoked_at IS NULL`)
5186
+ .run(new Date().toISOString(), `principal:${assignment.principalId}`, `role:${assignment.roleKey}`, `scope:${scopeNode}`).changes);
5093
5187
  if (changes === 0)
5094
5188
  return; // never assigned, or already revoked — idempotent, unaudited
5095
- this.recordAdmin(actor, 'unassignRole', { tenantId: assignment.node.tenantId, scopeId: assignment.node.scopeId }, assignment, null);
5189
+ this.recordAdmin(actor, 'unassignRole', { tenantId: assignment.node.tenantId, scopeId: scopeNode }, assignment, null);
5096
5190
  },
5097
5191
  grant: async (actor, raw) => {
5098
5192
  // Parsed like its `grantToConnection`/`grantToSystem` siblings, not taken on
@@ -5213,10 +5307,19 @@ export class SqliteScopeHost {
5213
5307
  },
5214
5308
  tenantHeldSystemModules: async (actor, tenantId, moduleIds) => {
5215
5309
  const now = this.clock();
5216
- const held = moduleIds.filter((m) => tenantHoldsSystemGrant(switchSqlOf(this.directory), tenantId, m, now));
5310
+ const held = tenantHeldOf(switchSqlOf(this.directory), 'system', tenantId, moduleIds, now);
5217
5311
  this.recordAccess(actor, 'tenantHeldSystemModules', { tenantId }, { moduleIds: [...moduleIds] }, held.length);
5218
5312
  return held;
5219
5313
  },
5314
+ peerSwitchCarry: async (actor, node) => {
5315
+ const directorySql = switchSqlOf(this.directory);
5316
+ const switchedOffPeers = switchedOffOf(directorySql, 'peer', node.tenantId, node.scopeId);
5317
+ const tenantHeldPeers = tenantHeldOf(directorySql, 'peer', node.tenantId, switchedOffPeers, this.clock());
5318
+ const all = switchFencesOf(directorySql, 'peer', node.tenantId, node.scopeId);
5319
+ const fences = Object.fromEntries(switchedOffPeers.map((v) => [v, all.get(v)]));
5320
+ this.recordAccess(actor, 'peerSwitchCarry', node, null, switchedOffPeers.length);
5321
+ return { switchedOffPeers, tenantHeldPeers, fences };
5322
+ },
5220
5323
  reassertSystemSwitches: async (actor, node, opts) => {
5221
5324
  this.assertScope(node.tenantId, node.scopeId);
5222
5325
  const rt = this.runtime(node.tenantId, node.scopeId);
@@ -5300,7 +5403,7 @@ export class SqliteScopeHost {
5300
5403
  // rebinding it would move another tenant's traffic. Exception: the holder is
5301
5404
  // ARCHIVED or REAPED (a deleted app, storage since wiped) — it released the
5302
5405
  // name, so the rebind reclaims it.
5303
- throw new Error(`hostname '${parsed.hostname}' is already bound to another scope`);
5406
+ throw substratError('conflict', `hostname '${parsed.hostname}' is already bound to another scope`);
5304
5407
  }
5305
5408
  // Exactly one canonical per (scope, surface): "which one do certs and
5306
5409
  // redirects use" has to have one answer, so a new canonical demotes the old.
@@ -5326,7 +5429,7 @@ export class SqliteScopeHost {
5326
5429
  .prepare('SELECT tenant_id, scope_id, status FROM hostnames WHERE hostname = ?')
5327
5430
  .get(hostname);
5328
5431
  if (!row)
5329
- throw new Error(`unknown hostname '${hostname}'`);
5432
+ throw substratError('not_found', `unknown hostname '${hostname}'`);
5330
5433
  if (row.status === status)
5331
5434
  return; // idempotent, and a no-op is not audited
5332
5435
  this.directory
@@ -5340,7 +5443,7 @@ export class SqliteScopeHost {
5340
5443
  .prepare('SELECT tenant_id, scope_id, status, custom_hostname_id, validation_records FROM hostnames WHERE hostname = ?')
5341
5444
  .get(hostname);
5342
5445
  if (!row)
5343
- throw new Error(`unknown hostname '${hostname}'`);
5446
+ throw substratError('not_found', `unknown hostname '${hostname}'`);
5344
5447
  const recordsJson = fields.validationRecords.length
5345
5448
  ? JSON.stringify(fields.validationRecords)
5346
5449
  : null;
@@ -5529,7 +5632,7 @@ export class SqliteScopeHost {
5529
5632
  if (existing.ownerTenant !== parsed.ownerTenant) {
5530
5633
  throw substratError('conflict', `vertical '${parsed.slug}' is owned by ${existing.ownerTenant ?? 'the platform'}, not ${parsed.ownerTenant ?? 'the platform'}`);
5531
5634
  }
5532
- throw new Error(`vertical '${parsed.slug}' is already registered as ${existing.source}`);
5635
+ throw substratError('conflict', `vertical '${parsed.slug}' is already registered as ${existing.source}`);
5533
5636
  }
5534
5637
  this.directory
5535
5638
  .prepare('INSERT INTO verticals (slug, name, source, owner_tenant, env_spec, install_spec, listed, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)')
@@ -5762,11 +5865,11 @@ export class SqliteScopeHost {
5762
5865
  // gate is about change, not about existence.
5763
5866
  if (outgoing) {
5764
5867
  if (outgoing.permissionDigest !== incoming.permissionDigest && !ack.permissionChange) {
5765
- throw new Error(`promotion changes the permission surface (${outgoing.permissionDigest} → ` +
5868
+ throw substratError('conflict', `promotion changes the permission surface (${outgoing.permissionDigest} → ` +
5766
5869
  `${incoming.permissionDigest}) — acknowledge it explicitly to promote`);
5767
5870
  }
5768
5871
  if (outgoing.migrationDigest !== incoming.migrationDigest && !ack.migrationChange) {
5769
- throw new Error(`promotion changes migrations (${outgoing.migrationDigest} → ` +
5872
+ throw substratError('conflict', `promotion changes migrations (${outgoing.migrationDigest} → ` +
5770
5873
  `${incoming.migrationDigest}) — acknowledge it explicitly to promote`);
5771
5874
  }
5772
5875
  // #1705 PR 3: an export an installed consumer imports, dropped or re-versioned.
@@ -6264,6 +6367,13 @@ export class SqliteScopeHost {
6264
6367
  this.recordAccess(actor, 'lifecycleFlow', { tenantId, scopeId }, delegatedReadParams.lifecycleFlow(input), flow.observation.events);
6265
6368
  return flow;
6266
6369
  },
6370
+ operationSeries: async (actor, tenantId, scopeId, input) => {
6371
+ // #1750: business volumes per bucket, for Pulse.
6372
+ const db = this.scopeReadDbFor(tenantId, scopeId);
6373
+ const series = readOperationSeries({ sql: scopedSql(db) }, input);
6374
+ this.recordAccess(actor, 'operationSeries', { tenantId, scopeId }, delegatedReadParams.operationSeries(input), operationSeriesCount(series));
6375
+ return series;
6376
+ },
6267
6377
  scopeAppliedMigrations: async (actor, tenantId, scopeId) => {
6268
6378
  const db = this.scopeReadDbFor(tenantId, scopeId);
6269
6379
  const rows = db
@@ -6315,7 +6425,7 @@ export class SqliteScopeHost {
6315
6425
  .prepare('SELECT org_id FROM orgs WHERE tenant_id = ? AND slug = ?')
6316
6426
  .get(parsed.tenantId, parsed.slug);
6317
6427
  if (slugOwner) {
6318
- throw new Error(`org slug '${parsed.slug}' already taken by ${slugOwner.org_id} (slugs are unique per tenant)`);
6428
+ throw substratError('conflict', `org slug '${parsed.slug}' already taken by ${slugOwner.org_id} (slugs are unique per tenant)`);
6319
6429
  }
6320
6430
  this.directory
6321
6431
  .prepare('INSERT INTO orgs (org_id, tenant_id, slug, name, created_at) VALUES (?, ?, ?, ?, ?)')
@@ -6343,18 +6453,39 @@ export class SqliteScopeHost {
6343
6453
  writeTenantTuple(tenantId, `principal:${principal}`, 'member', `org:${orgId}`);
6344
6454
  this.recordAdmin(actor, 'addMember', { tenantId }, null, { principal, orgId });
6345
6455
  },
6456
+ applyMembership: async (actor, change) => {
6457
+ // ONE directory transaction, synchronous throughout (#1184): the fence (an add's), the
6458
+ // bound and the write. Nothing else on this host lands between them, and a failure
6459
+ // leaves none.
6460
+ const { tenantId, principal, roleKey, boundedBy } = change;
6461
+ const assignment = { principalId: principal, roleKey, node: { tenantId, scopeId: null } };
6462
+ return this.directory.transaction(() => {
6463
+ if (change.op === 'add') {
6464
+ const fence = this.directory
6465
+ .prepare(MEMBERSHIP_FENCE_SINCE_SQL)
6466
+ .get(tenantId, principal, change.unlessRemovedSince);
6467
+ if (fence)
6468
+ return { applied: false, removedAt: fence.removed_at };
6469
+ }
6470
+ const role = directoryReader.getRole(tenantId, roleKey);
6471
+ if (!role)
6472
+ return { applied: false, unknownRole: roleKey };
6473
+ const bound = tenantCoverage(directoryReader, tenantId, boundedBy, role.permissions);
6474
+ if (!bound.covered)
6475
+ return { applied: false, missing: bound.missing };
6476
+ if (change.op === 'remove')
6477
+ return { applied: true, changed: revokeTenantRoleAndFence(actor, assignment) };
6478
+ writeTenantTuple(tenantId, `principal:${principal}`, `role:${roleKey}`, `tenant:${tenantId}`);
6479
+ this.recordAdmin(actor, 'assignRole', { tenantId, scopeId: null }, null, assignment);
6480
+ return { applied: true };
6481
+ })();
6482
+ },
6346
6483
  removeMember: async (actor, tenantId, principal, orgId) => {
6347
6484
  requireOrg(tenantId, orgId);
6348
- // Tombstone (K-21), never DELETE. Guarded on `revoked_at IS NULL` so a
6349
- // repeat revoke neither moves the timestamp nor writes a second audit row.
6350
- const info = this.directory
6351
- .prepare(`UPDATE _substrat_tenant_tuples SET revoked_at = ?
6352
- WHERE tenant_id = ? AND subject = ? AND relation = 'member' AND object = ?
6353
- AND revoked_at IS NULL`)
6354
- .run(new Date().toISOString(), tenantId, `principal:${principal}`, `org:${orgId}`);
6355
- if (info.changes === 0)
6356
- return; // never a member, or already revoked
6357
- this.recordAdmin(actor, 'removeMember', { tenantId }, { principal, orgId }, null);
6485
+ // Tombstone (K-21), never DELETE: a repeat revoke neither moves the timestamp nor writes
6486
+ // a second audit row. One transaction with the removal fence (#1184), which a no-op
6487
+ // raises too.
6488
+ this.directory.transaction(() => revokeAndFence(tenantId, principal, 'member', `org:${orgId}`, (at) => this.recordAdmin(actor, 'removeMember', { tenantId }, { principal, orgId }, null, at)))();
6358
6489
  },
6359
6490
  listMembers: async (actor, tenantId, orgId, options) => {
6360
6491
  requireOrg(tenantId, orgId);
@@ -6385,7 +6516,7 @@ export class SqliteScopeHost {
6385
6516
  .prepare('SELECT tenant_id FROM tenants WHERE slug = ?')
6386
6517
  .get(parsed.slug);
6387
6518
  if (slugOwner) {
6388
- throw new Error(`tenant slug '${parsed.slug}' already taken by ${slugOwner.tenant_id} (slugs are unique)`);
6519
+ throw substratError('conflict', `tenant slug '${parsed.slug}' already taken by ${slugOwner.tenant_id} (slugs are unique)`);
6389
6520
  }
6390
6521
  this.directory
6391
6522
  .prepare(`INSERT INTO tenants (tenant_id, slug, name, status, created_at, provisioned_by_tenant)
@@ -6396,12 +6527,12 @@ export class SqliteScopeHost {
6396
6527
  setTenantStatus: async (actor, tenantId, status) => {
6397
6528
  const before = readTenant(tenantId);
6398
6529
  if (!before)
6399
- throw new Error(`unknown tenant: ${tenantId}`);
6530
+ throw substratError('not_found', `unknown tenant: ${tenantId}`);
6400
6531
  // `reaped` is terminal and destroys data — it is unreachable here and only
6401
6532
  // ever set by reapTenant, so a plain status flip cannot forge a tombstone
6402
6533
  // over live data (§4.8, the tenant analogue of reapScope's archived-only gate).
6403
6534
  if (status === 'reaped') {
6404
- throw new Error(`tenant ${tenantId} cannot be set to 'reaped' via setTenantStatus — reap goes through reapTenant (control-plane.md §4.8)`);
6535
+ throw substratError('validation_failed', `tenant ${tenantId} cannot be set to 'reaped' via setTenantStatus — reap goes through reapTenant (control-plane.md §4.8)`);
6405
6536
  }
6406
6537
  // Stamp/clear deleting_at so the grace-window sweep can age tenants (§4.8):
6407
6538
  // entering `deleting` stamps, leaving it (an un-delete) clears — exactly the
@@ -6421,7 +6552,7 @@ export class SqliteScopeHost {
6421
6552
  setTenantName: async (actor, tenantId, name) => {
6422
6553
  const before = readTenant(tenantId);
6423
6554
  if (!before)
6424
- throw new Error(`unknown tenant: ${tenantId}`);
6555
+ throw substratError('not_found', `unknown tenant: ${tenantId}`);
6425
6556
  if (before.name === name)
6426
6557
  return; // no-op is not audited — nothing changed
6427
6558
  this.directory.prepare('UPDATE tenants SET name = ? WHERE tenant_id = ?').run(name, tenantId);
@@ -6553,7 +6684,7 @@ export class SqliteScopeHost {
6553
6684
  // the query, so the quoted identifier below carries no user input.
6554
6685
  const known = new Set(db.prepare(`SELECT name FROM sqlite_master WHERE type = 'table'`).all().map((r) => r.name));
6555
6686
  if (!known.has(input.table))
6556
- throw new Error(`unknown table '${input.table}'`);
6687
+ throw substratError('not_found', `unknown table '${input.table}'`);
6557
6688
  // The ceiling clamps; a bound SQLite would misread (NaN, non-finite, fractional,
6558
6689
  // negative) is refused instead of reaching LIMIT / OFFSET (#1632).
6559
6690
  const limit = Math.min(assertRowLimit('limit', input.limit ?? SCOPE_TABLE_PAGE_DEFAULT), SCOPE_TABLE_PAGE_MAX);
@@ -6574,7 +6705,7 @@ export class SqliteScopeHost {
6574
6705
  // itself rejects multi-statement strings and bad SQL with the driver's message.
6575
6706
  const stmt = db.prepare(sql);
6576
6707
  if (!stmt.readonly)
6577
- throw new Error('read-only console: statement is not read-only');
6708
+ throw substratError('validation_failed', 'read-only console: statement is not read-only');
6578
6709
  if (!stmt.reader)
6579
6710
  return { columns: [], rows: [], truncated: false };
6580
6711
  stmt.raw(true);
@@ -6720,6 +6851,7 @@ export class SqliteScopeHost {
6720
6851
  loadDirectoryDump(dump.tables, {
6721
6852
  columnsOf: (name) => builtColumnsOf(this.directory, name),
6722
6853
  exec: (sql) => this.directory.exec(sql),
6854
+ now: () => this.clock(),
6723
6855
  insert: (sql, rows) => {
6724
6856
  const insert = this.directory.prepare(sql);
6725
6857
  for (const row of rows)
@@ -6771,7 +6903,7 @@ export class SqliteScopeHost {
6771
6903
  throw substratError('not_found', `unknown scope for tenant: (${tenantId}, ${scopeId})`);
6772
6904
  }
6773
6905
  if (rec.status !== 'archived') {
6774
- throw new Error(`scope ${scopeId} is ${rec.status}, not archived — only an archived scope may be reaped`);
6906
+ throw substratError('conflict', `scope ${scopeId} is ${rec.status}, not archived — only an archived scope may be reaped`);
6775
6907
  }
6776
6908
  // A serving scope always holds ≥1 bound hostname; a truly-dead one has been
6777
6909
  // unbound. The dashboard delete path unbinds AT archive, but a bare console
@@ -6787,7 +6919,7 @@ export class SqliteScopeHost {
6787
6919
  .prepare('SELECT hostname FROM hostnames WHERE scope_id = ? LIMIT 1')
6788
6920
  .get(scopeId);
6789
6921
  if (bound) {
6790
- throw new Error(`scope ${scopeId} still resolves hostname '${bound.hostname}' — ` +
6922
+ throw substratError('conflict', `scope ${scopeId} still resolves hostname '${bound.hostname}' — ` +
6791
6923
  `unbind it before reaping (reap wipes storage and cannot be undone)`);
6792
6924
  }
6793
6925
  // The reader goes with the writer (#1624): an open handle on a removed file keeps
@@ -6938,9 +7070,9 @@ export class SqliteScopeHost {
6938
7070
  // reaped — an illegal source fails closed, like reapScope's archived-only gate.
6939
7071
  const before = readTenant(tenantId);
6940
7072
  if (!before)
6941
- throw new Error(`unknown tenant: ${tenantId}`);
7073
+ throw substratError('not_found', `unknown tenant: ${tenantId}`);
6942
7074
  if (before.status !== 'deleting') {
6943
- throw new Error(`tenant ${tenantId} is ${before.status}, not deleting — only a deleting tenant may be reaped`);
7075
+ throw substratError('conflict', `tenant ${tenantId} is ${before.status}, not deleting — only a deleting tenant may be reaped`);
6944
7076
  }
6945
7077
  // Clear the tenant's directory-side rows. Idempotent (set-to-empty), so a crash
6946
7078
  // mid-reap converges on retry. KEPT as the tombstone: the `tenants` row itself
@@ -6954,6 +7086,9 @@ export class SqliteScopeHost {
6954
7086
  '_substrat_entitlements', // per-tenant SKU flags
6955
7087
  'orgs', // K-22 org records
6956
7088
  '_substrat_system_switches', // #1674: the schedule switch's record, per scope
7089
+ '_substrat_membership_fences', // #1184: the latest removal, per principal
7090
+ '_substrat_peer_switches', // #2029: the peer switch's record, per scope
7091
+ '_substrat_switch_owed', // #2045: subjects owed a re-assert, per scope
6957
7092
  ];
6958
7093
  const clear = this.directory.transaction(() => {
6959
7094
  for (const table of tables) {
@@ -7072,7 +7207,7 @@ export class SqliteScopeHost {
7072
7207
  if (existing.topology === parsed.topology && existing.tenantId === parsed.tenantId) {
7073
7208
  return;
7074
7209
  }
7075
- throw new Error(`identity pool '${parsed.provider}' is already registered as ${existing.topology}` +
7210
+ throw substratError('conflict', `identity pool '${parsed.provider}' is already registered as ${existing.topology}` +
7076
7211
  `${existing.tenantId ? ` for tenant ${existing.tenantId}` : ''}`);
7077
7212
  }
7078
7213
  this.directory
@@ -7091,9 +7226,9 @@ export class SqliteScopeHost {
7091
7226
  listIdentityTenants: async (actor, provider, externalId) => {
7092
7227
  const pool = readPool(provider);
7093
7228
  if (!pool)
7094
- throw new Error(`identity pool '${provider}' is not registered`);
7229
+ throw substratError('not_found', `identity pool '${provider}' is not registered`);
7095
7230
  if (pool.topology !== 'central') {
7096
- throw new Error(`identity pool '${provider}' is tenant-bound — enumerating tenants is only ` +
7231
+ throw substratError('forbidden', `identity pool '${provider}' is tenant-bound — enumerating tenants is only ` +
7097
7232
  `meaningful on a central pool, where the same externalId is the same person`);
7098
7233
  }
7099
7234
  const tenants = this.directory
@@ -7107,9 +7242,9 @@ export class SqliteScopeHost {
7107
7242
  listIdentityMemberships: async (actor, provider, externalId) => {
7108
7243
  const pool = readPool(provider);
7109
7244
  if (!pool)
7110
- throw new Error(`identity pool '${provider}' is not registered`);
7245
+ throw substratError('not_found', `identity pool '${provider}' is not registered`);
7111
7246
  if (pool.topology !== 'central') {
7112
- throw new Error(`identity pool '${provider}' is tenant-bound — enumerating tenants is only ` +
7247
+ throw substratError('forbidden', `identity pool '${provider}' is tenant-bound — enumerating tenants is only ` +
7113
7248
  `meaningful on a central pool, where the same externalId is the same person`);
7114
7249
  }
7115
7250
  const rows = this.directory
@@ -7408,7 +7543,7 @@ export class SqliteScopeHost {
7408
7543
  if (existing) {
7409
7544
  if (existing.principal_id === parsed.principal)
7410
7545
  return; // idempotent, unaudited
7411
- throw new Error(`identity ${parsed.provider}:${parsed.externalId} in tenant ${parsed.tenantId} ` +
7546
+ throw substratError('conflict', `identity ${parsed.provider}:${parsed.externalId} in tenant ${parsed.tenantId} ` +
7412
7547
  `is already bound to ${existing.principal_id}`);
7413
7548
  }
7414
7549
  this.directory
@@ -8689,7 +8824,7 @@ export class SqliteScopeHost {
8689
8824
  if (!r || r.tenant_id !== tenantId)
8690
8825
  throw substratError('not_found', `unknown scope for tenant: (${tenantId}, ${scopeId})`);
8691
8826
  if (r.status === 'reaped') {
8692
- throw new Error(`scope ${scopeId} is reaped — its storage is gone and cannot be read`);
8827
+ throw substratError('conflict', `scope ${scopeId} is reaped — its storage is gone and cannot be read`);
8693
8828
  }
8694
8829
  }
8695
8830
  /**
@@ -8859,6 +8994,13 @@ export class SqliteScopeHost {
8859
8994
  // executes, the table always already has `kind` in its key, never `invocation_id`.
8860
8995
  this.ensureColumn(db, '_substrat_schedule_state', 'invocation_id', 'invocation_id TEXT');
8861
8996
  this.ensureColumn(db, '_substrat_job_runs', 'subject_id', 'subject_id TEXT');
8997
+ // #2034: the lease, on a scope DB built before it. NULL = nobody holds the run.
8998
+ this.ensureColumn(db, '_substrat_job_runs', 'lease_owner', 'lease_owner TEXT');
8999
+ // #2042 r2, r4: whether the holder BEGAN its pass. NULL = it has not, which is right for any lease
9000
+ // already there: a takeover of it costs nothing.
9001
+ this.ensureColumn(db, '_substrat_job_runs', 'lease_began_at', 'lease_began_at TEXT');
9002
+ // #2042 r3: consecutive admission misses. NULL = none, right for every run already there.
9003
+ this.ensureColumn(db, '_substrat_job_runs', 'admission_misses', 'admission_misses INTEGER');
8862
9004
  // #2009: the copy classification on a scope DB built before it (NULL reads as a copy; see
8863
9005
  // `COPY_ORIGIN_DDL`).
8864
9006
  this.ensureColumn(db, '_substrat_copy_origin', 'is_copy', 'is_copy INTEGER');