@substrat-run/adapter-sqlite 0.135.0 → 0.137.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,10 +1,11 @@
1
1
  import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
2
2
  import { dirname, join } from 'node:path';
3
3
  import Database from 'better-sqlite3';
4
- import { accessLogEntry, adminLogEntry, delegatedReadParams, opsFailureEntry, opsFailureFingerprint, issueEntry, FRESHNESS_HEARTBEAT_MINUTES, sweepRunEntry, modelUsageEntry, ATTACHMENT_ADDED, ATTACHMENT_REMOVED, attachmentRecord, createTenantInput, domainEvent, domainEventInput, exportReadInput, importBatch, importCursorMove, exportsOfManifestJson, importsOfManifestJson, entitlementGrant, entitlementGrantInput, eventId, meterReading, platformRequestInput, platformRequestId, MAX_PENDING_PLATFORM_REQUESTS, identityLink, identityPool, instant, connection, capabilityGrant, connectionGrant, connectionGrantRecord, connectionSecret, systemGrant, systemSwitch, systemSwitchRecord, peerSwitch, verticalCaller, verticalSlug as verticalSlugOf, subjectRef, requestFingerprint, createConnectionInput, projectedConnectionGrant, projectedConnectionKey, moduleManifest, createOrgInput, promotionAcknowledgement, bindAcknowledgement, bindHostnameInput, channelHistoryEntry, hostnameBinding, publishVersionInput, AUTO_ADMISSION_NOTE, routeTarget, registerVerticalInput, vertical as verticalSchema, verticalChannel, verticalVersion, objectRef, entityObjectRef, assertModuleEmittableType, assertKernelAuthoredType, grantRefFromProof, org as orgSchema, orgMembership, principalId, resolvedIdentity, identityMembership, roleDefinition, scope as scopeSchema, tenant as tenantSchema, tenantRole, subjectShredReceipt, SCOPE_TABLE_PAGE_DEFAULT, SCOPE_TABLE_PAGE_MAX, SCOPE_QUERY_ROW_MAX, verticalServingState, callsOfManifestJson, outboundOfManifestJson, listLimitOf, substratError, assertReplayableDump, delegatedReadRecord, ownerTransferAudit, redrainEventsInput, REDRAIN_BATCH, } from '@substrat-run/contracts';
5
- import { actorOf, asPrincipal, assertAllowed, assertNoSecret, assertPermissionKey, CAPABILITY_DDL, CAPABILITY_EXCHANGE_OPERATION, capabilityAttachmentWriteRefused, capabilityTokenHash, createCapabilityVerbs, createEntityEdgeVerbs, exchangeCapability as exchangeCapabilitySecret, guardSecrets, mintBecomeCapability, plausibleSessionToken, readCapabilityPage, redactSecrets, redactSecretText, moduleLog, consoleLogSink, resolveCapabilitySession, revokeCapabilityAsPlatform, assertReadOnlyQuery, attachmentBlobKey, attachmentSha256, entitlementDenial, requiredEntitlementFor, foldMeterReading, guardSpine, guardSqlLimits, DO_SQL_LIMITS, tooManyResultColumns, tooManyTableColumns, parseValidationRecords, resolveScopeRecord, ulid, ISSUE_RETENTION_DAYS, telemetryRetentionStatements, assertRowLimit, assertRowOffset, OPS_FAILURE_RETENTION_DAYS, SWEEP_RUN_RETENTION_DAYS, SWEEP_RUNS_INTENT_INDEX, sweepRunsIntentHasKind, MODEL_USAGE_RETENTION_DAYS, foldModelUsage, backoffAt, platformRequestHistoryQuery, platformRequestOf, PLATFORM_REQUEST_COLUMNS, PLATFORM_REQUEST_REDACTION_SQL, platformRequestRedactionParams, platformRequestRedactionQuery, intentPayloadCarriesSubject, redactSubjectJobRuns, redactSubjectScopeText, redactSubjectDirectoryText, ISSUE_EXEMPLAR_OWNER_BACKFILL_SQL, issueExemplarOwner, JOB_RUN_PATCH_SQL, JOB_STEP_RECORD_SQL, DELIVERY_ERROR_REDACTION_SQL, REDACTED_DELIVERY_NOTE, seatScopeTuple, switchSystemSchedules, admitPeer, collectPeers, peerSeats, resolveVerticalInstanceFrom, switchPeer, peerGrantsStatus, systemGrantsStatus, systemScheduleState, systemSwitchedOff, systemSwitchedOffMessage, tenantSystemSwitchedOffMessage, SYSTEM_SWITCHES_BACKFILL_SQL, SYSTEM_SWITCHES_DDL, forgetSystemSwitchesOf, listSystemSwitchRecords, recordSystemSwitchedOff, recordSystemSwitchedOn, restoreSystemSwitchRecord, scopesSwitchedOffFor, switchedOffModulesOf, inUnitMovesToAudit, staleCarryRevertRow, staleCarryReverts, systemSwitchRecordsOf, systemSwitchesTableExists, withRecorded, VERSION_MIGRATIONS_DDL, splitManifestMigrations, splitVersionMigrationsBatch, versionMigrationsOf, writeVersionMigrations, denialListQuery, refusalListQuery, mapRefusalRow, denialSummaryQuery, denialTotalsQuery, DENIAL_WINDOW_QUERY, mapDenialRow, mapDenialSummaryBuckets, IMPERSONATION_COLUMNS, IMPERSONATION_DDL, ImpersonationRefused, assertImpersonationWrites, assertModuleEnqueueableKind, assertSessionUsable, impersonationByIdQuery, impersonationListQuery, impersonationRowValues, impersonationStampOf, mapImpersonationRow, newImpersonationSession, resolveRetryPolicy, isSecretBoxConfigured, unconfiguredSecretBox, createSubjectKeys, generateSealingKeyPair, noSealingKeyMessage, openSealed, sealTo, ConnectionSealingKeyUnavailableError, PermissionDenied, createAtomic, NotSearchable, NotListable, listIndexMigrations, listIndexPlans, listQuery, cursorOf, moduleMigrations, isSearchIndexTable, searchIndexDdl, searchIndexMigrations, searchIndexPlans, searchLimit, searchMatchExpression, searchQuery, ATTACHMENT_TEXT_DDL, assertAttachmentExtractors, assertJobRegistrable, attachmentRecordOfRow, attachmentTextJob, enqueueAttachmentText, isAttachmentTextRun, reconcileAttachmentText, recordAttachmentText, searchAttachments, entityVersionQuery, entityVersionOf, assertIfMatch, OUTBOX_ENTITY_INDEX, SCHEDULE_STATE_DDL, SCHEDULE_STATE_REBUILD, scheduleStateHasKind, VERTICAL_EVENTS_DDL, EXPORT_HOPS_SQL, IMPORT_CURSORS_SQL, IMPORT_CURSOR_OF_SQL, OUTBOX_MARK_SQL, emittedSinceQuery, emittedReportOf, EMITTED_REPORT_CAP, exportedSinceQuery, IMPORT_CURSOR_ADVANCE_SQL, IMPORT_RECORD_SQL, moveImportCursor, importCursorSourceOf, exportBreaksOf, exportBreakRefusal, bindExportBreaksOf, bindExportBreakRefusal, CrossVerticalRegistry, exportReadPlan, exportReadQuery, planExportBatch, withheldNote, JOB_RUN_DDL, jobRunListLimit, jobRunOf, runDueJobRuns, startJobRun, IDEMPOTENCY_DDL, REFUSALS_DDL, refusalInsert, refusedTransitionOf, assertIdempotencyKey, idempotencyLookupQuery, idempotencyPruneStatement, idempotencyRecordStatement, idempotencyOptedOutMessage, replayFor, globalFetch, createUlid, facetEvents, readHistory, walkEventCause, walkEventEffects, assertRedrainWindow, readInvocation, readDeadLetters, readLifecycleFlow, domainEventOf, readUndrainedOutbox, undrainedEventsOf, connectorCallRecord, noopConnectorCallRecorder, recordConnectorCall, settleConnectionUse, unknownRoleError, } from '@substrat-run/kernel';
4
+ import { accessLogEntry, adminLogEntry, delegatedReadParams, opsFailureEntry, opsFailureFingerprint, issueEntry, FRESHNESS_HEARTBEAT_MINUTES, sweepRunEntry, modelUsageEntry, ATTACHMENT_ADDED, ATTACHMENT_REMOVED, attachmentRecord, createTenantInput, domainEvent, domainEventInput, exportReadInput, importBatch, importCursorMove, exportsOfManifestJson, importsOfManifestJson, entitlementGrant, entitlementGrantInput, eventId, meterReading, platformRequestInput, platformRequestId, MAX_PENDING_PLATFORM_REQUESTS, identityLink, identityPool, instant, connection, capabilityGrant, connectionGrant, connectionGrantRecord, connectionSecret, systemGrant, systemSwitch, systemSwitchRecord, peerSwitch, verticalCaller, verticalSlug as verticalSlugOf, subjectRef, requestFingerprint, createConnectionInput, projectedConnectionGrant, projectedConnectionKey, moduleManifest, createOrgInput, promotionAcknowledgement, bindAcknowledgement, bindHostnameInput, channelHistoryEntry, hostnameBinding, publishVersionInput, AUTO_ADMISSION_NOTE, routeTarget, ROUTE_SCOPE_HOSTNAMES_MAX, registerVerticalInput, vertical as verticalSchema, verticalChannel, verticalVersion, objectRef, entityObjectRef, assertModuleEmittableType, assertKernelAuthoredType, grantRefFromProof, org as orgSchema, orgMembership, principalId, resolvedIdentity, identityMembership, roleDefinition, scope as scopeSchema, tenant as tenantSchema, tenantRole, subjectShredReceipt, SCOPE_TABLE_PAGE_DEFAULT, SCOPE_TABLE_PAGE_MAX, SCOPE_QUERY_ROW_MAX, verticalServingState, callsOfManifestJson, outboundOfManifestJson, listLimitOf, substratError, assertReplayableDump, delegatedReadRecord, ownerTransferAudit, copyMarkAudit, redrainEventsInput, REDRAIN_BATCH, } from '@substrat-run/contracts';
5
+ import { actorOf, asPrincipal, assertAllowed, assertNoSecret, assertPermissionKey, CAPABILITY_DDL, COPY_ORIGIN_DDL, CAPABILITY_EXCHANGE_OPERATION, capabilityAttachmentWriteRefused, capabilityTokenHash, createCapabilityVerbs, createEntityEdgeVerbs, exchangeCapability as exchangeCapabilitySecret, guardSecrets, mintBecomeCapability, plausibleSessionToken, readCapabilityPage, redactSecrets, redactSecretText, moduleLog, consoleLogSink, asyncInvocationId, asyncLinePass, consoleInvocationLineSink, resolveCapabilitySession, revokeCapabilityAsPlatform, assertReadOnlyQuery, attachmentBlobKey, attachmentSha256, entitlementDenial, requiredEntitlementFor, foldMeterReading, guardSpine, guardSqlLimits, DO_SQL_LIMITS, tooManyResultColumns, tooManyTableColumns, parseValidationRecords, resolveScopeRecord, ulid, ISSUE_RETENTION_DAYS, telemetryRetentionStatements, assertRowLimit, assertRowOffset, OPS_FAILURE_RETENTION_DAYS, SWEEP_RUN_RETENTION_DAYS, SWEEP_RUNS_INTENT_INDEX, sweepRunsIntentHasKind, MODEL_USAGE_RETENTION_DAYS, foldModelUsage, backoffAt, platformRequestHistoryQuery, platformRequestOf, PLATFORM_REQUEST_COLUMNS, PLATFORM_REQUEST_REDACTION_SQL, platformRequestRedactionParams, platformRequestRedactionQuery, intentPayloadCarriesSubject, redactSubjectJobRuns, redactSubjectScopeText, redactSubjectDirectoryText, ISSUE_EXEMPLAR_OWNER_BACKFILL_SQL, issueExemplarOwner, JOB_RUN_PATCH_SQL, JOB_STEP_RECORD_SQL, DELIVERY_ERROR_REDACTION_SQL, REDACTED_DELIVERY_NOTE, seatScopeTuple, switchSystemSchedules, admitPeer, collectPeers, peerSeats, resolveVerticalInstanceFrom, switchPeer, peerGrantsStatus, systemGrantsStatus, systemScheduleState, systemSwitchedOff, systemSwitchedOffMessage, tenantSystemSwitchedOffMessage, SYSTEM_SWITCHES_BACKFILL_SQL, SYSTEM_SWITCHES_DDL, forgetSystemSwitchesOf, listSystemSwitchRecords, recordSystemSwitchedOff, recordSystemSwitchedOn, restoreSystemSwitchRecord, scopesSwitchedOffFor, tenantHoldsSystemGrant, switchedOffModulesOf, inUnitMovesToAudit, staleCarryRevertRow, staleCarryReverts, systemSwitchRecordsOf, systemSwitchesTableExists, withRecorded, VERSION_MIGRATIONS_DDL, splitManifestMigrations, splitVersionMigrationsBatch, versionMigrationsOf, writeVersionMigrations, denialListQuery, refusalListQuery, mapRefusalRow, denialSummaryQuery, denialTotalsQuery, DENIAL_WINDOW_QUERY, mapDenialRow, mapDenialSummaryBuckets, IMPERSONATION_COLUMNS, IMPERSONATION_DDL, ImpersonationRefused, assertImpersonationWrites, assertModuleEnqueueableKind, assertSessionUsable, impersonationByIdQuery, impersonationListQuery, impersonationRowValues, impersonationStampOf, mapImpersonationRow, newImpersonationSession, resolveRetryPolicy, isSecretBoxConfigured, unconfiguredSecretBox, createSubjectKeys, generateSealingKeyPair, noSealingKeyMessage, openSealed, sealTo, ConnectionSealingKeyUnavailableError, PermissionDenied, createAtomic, NotSearchable, NotListable, listIndexMigrations, listIndexPlans, listQuery, cursorOf, moduleMigrations, isSearchIndexTable, searchIndexDdl, searchIndexMigrations, searchIndexPlans, searchLimit, searchMatchExpression, searchQuery, ATTACHMENT_TEXT_DDL, assertAttachmentExtractors, assertJobRegistrable, attachmentRecordOfRow, attachmentTextJob, enqueueAttachmentText, isAttachmentTextRun, reconcileAttachmentText, recordAttachmentText, searchAttachments, entityVersionQuery, entityVersionOf, assertIfMatch, OUTBOX_ENTITY_INDEX, SCHEDULE_STATE_DDL, SCHEDULE_STATE_REBUILD, scheduleStateHasKind, VERTICAL_EVENTS_DDL, EXPORT_HOPS_SQL, IMPORT_CURSORS_SQL, IMPORT_CURSOR_OF_SQL, OUTBOX_MARK_SQL, emittedSinceQuery, emittedReportOf, EMITTED_REPORT_CAP, exportedSinceQuery, IMPORT_CURSOR_ADVANCE_SQL, IMPORT_RECORD_SQL, moveImportCursor, importCursorSourceOf, exportBreaksOf, exportBreakRefusal, bindExportBreaksOf, bindExportBreakRefusal, CrossVerticalRegistry, exportReadPlan, exportReadQuery, planExportBatch, withheldNote, JOB_RUN_DDL, JOB_RUN_DUE_AT, jobRunListLimit, jobRunOf, runDueJobRuns, startJobRun, IDEMPOTENCY_DDL, REFUSALS_TABLE_DDL, REFUSALS_INDEX, refusalInsert, refusalOf, markGuardRefusal, REFUSALS_REBUILD, refusalsAdmitGuards, assertIdempotencyKey, idempotencyLookupQuery, idempotencyPruneStatement, idempotencyRecordStatement, idempotencyOptedOutMessage, replayFor, globalFetch, createUlid, facetEvents, readHistory, walkEventCause, walkEventEffects, assertRedrainWindow, readInvocation, readDeadLetters, readLifecycleFlow, domainEventOf, readUndrainedOutbox, undrainedEventsOf, connectorCallRecord, noopConnectorCallRecorder, recordConnectorCall, settleConnectionUse, unknownRoleError, } from '@substrat-run/kernel';
6
6
  import { attributedHost } from '@substrat-run/kernel';
7
- import { LEGACY_SCOPE_ROWS_BACKFILL, assertNoSpineReference, assertSpineTablesBuilt, dumpRowsInsert, isSpineTable, loadDirectoryDump, repointScopeGrants, spineColumnAdditions } from '@substrat-run/kernel';
7
+ import { INERT_SCOPE_REASON, isPrimaryScopeRow } from '@substrat-run/kernel';
8
+ import { LEGACY_SCOPE_ROWS_BACKFILL, assertNoSpineReference, assertSpineTablesBuilt, capabilitiesForLoad, dumpRowsInsert, isSpineTable, loadDirectoryDump, repointScopeGrants, settleCopiedWork, emittedHere, spineColumnAdditions } from '@substrat-run/kernel';
8
9
  import { ScopeActor } from './actor.js';
9
10
  import { createTupleChecker } from './checker.js';
10
11
  /**
@@ -203,11 +204,15 @@ const KERNEL_DDL = `
203
204
  drained_at TEXT
204
205
  );
205
206
  ${IDEMPOTENCY_DDL}
206
- ${REFUSALS_DDL}
207
+ ${REFUSALS_TABLE_DDL}
208
+ ${REFUSALS_INDEX}
207
209
  -- #1672: capabilities — authority carried by a secret (a link share), and the sessions
208
210
  -- an exchange trades that secret for. Spine (kernel-written), shared with the DO adapter
209
211
  -- from @substrat-run/kernel so the two cannot part company; the column comments are there.
210
212
  ${CAPABILITY_DDL}
213
+ -- #1686: where a copied scope's data came from, and the event id its own events start above.
214
+ -- Shared with the other adapter from @substrat-run/kernel; the column comments are there.
215
+ ${COPY_ORIGIN_DDL}
211
216
  -- #383 / #1232 / #1288: the platform sweep's per-scope gating state, holding two
212
217
  -- families of row that the kind COLUMN — not the spelling of a key — tells apart.
213
218
  -- Spine (kernel-written), never a module migration. Shared with the DO adapter from
@@ -418,11 +423,12 @@ function modelUsageEntryOf(r) {
418
423
  });
419
424
  }
420
425
  /**
421
- * The policy an undecodable event's executor delivery is journaled under (#1636): one
422
- * attempt, so the first failure is the dead letter. A decode is pure — retrying it cannot
423
- * succeed — and the Cloudflare coordinator records the same row with no next attempt.
426
+ * The policy a delivery that can never succeed is journaled under: one attempt, so the first
427
+ * is the dead letter, and the Cloudflare coordinator records the same row with no next
428
+ * attempt. An undecodable event (#1636) — a decode is pure, so a retry cannot succeed — and an
429
+ * inert scope's delivery (#2005) — a scope does not become primary, so a retry decides the same.
424
430
  */
425
- const UNDECODABLE_RETRY = { maxAttempts: 1, baseDelayMs: 0, maxDelayMs: 0 };
431
+ const TERMINAL_RETRY = { maxAttempts: 1, baseDelayMs: 0, maxDelayMs: 0 };
426
432
  /**
427
433
  * The dispatch capability (#726 remedy B): a connector delivery may read the attachments
428
434
  * of the entity the DELIVERED EVENT names, and nothing else.
@@ -572,6 +578,8 @@ export class SqliteScopeHost {
572
578
  versionId;
573
579
  /** #1746: where `ctx.log` lines go — the console unless a caller (a test) says otherwise. */
574
580
  logSink;
581
+ /** #1901: where async work's invocation lines go. */
582
+ invocationLineSink;
575
583
  // The mint for event ids (#956) is NOT here: it lives on `ScopeRuntime`, one per
576
584
  // scope, seeded from that scope's persisted maximum (#1335). Its own monotonic
577
585
  // floor, because the timestamp it stamps comes from `this.clock` — a scripted
@@ -586,6 +594,7 @@ export class SqliteScopeHost {
586
594
  this.clock = options.clock ?? (() => instant.parse(new Date().toISOString()));
587
595
  this.versionId = options.versionId ?? null;
588
596
  this.logSink = options.logSink ?? consoleLogSink;
597
+ this.invocationLineSink = options.invocationLineSink ?? consoleInvocationLineSink;
589
598
  this.dir = options.dir;
590
599
  mkdirSync(this.dir, { recursive: true });
591
600
  this.directory = new Database(join(this.dir, '_directory.sqlite'));
@@ -1659,7 +1668,8 @@ export class SqliteScopeHost {
1659
1668
  * `HostAdmin.reassertSystemSwitches`' body (#1674), run inside a turn the caller already
1660
1669
  * holds on the scope's actor: every module the directory records OFF on the scope is
1661
1670
  * switched off again. Already off answers `changed: false` and writes nothing, and a
1662
- * module the scope does not hold is left alone. Audited only when something moved.
1671
+ * module with no authority reaching the scope (no grant here, no tenant-level grant) is left
1672
+ * alone. Audited only when something moved.
1663
1673
  *
1664
1674
  * `audit` receives each row. By default it writes to the admin log at once. A caller running
1665
1675
  * this inside a scope transaction that may still roll back (a restore's replay) collects
@@ -1686,7 +1696,9 @@ export class SqliteScopeHost {
1686
1696
  for (const row of inUnitMovesToAudit(recordedOff, applied))
1687
1697
  audit({ operationId: ulid(), ...row });
1688
1698
  return recordedOff.map((moduleId) => {
1689
- const outcome = rt.db.transaction(() => switchSystemSchedules(switchSqlOf(rt.db), { moduleId, scopeId, to: 'off', at }))();
1699
+ // #1823: a module whose only authority here is a tenant-level grant is held too.
1700
+ const tenantHeld = tenantHoldsSystemGrant(switchSqlOf(this.directory), tenantId, moduleId, this.clock());
1701
+ const outcome = rt.db.transaction(() => switchSystemSchedules(switchSqlOf(rt.db), { moduleId, scopeId, to: 'off', at, tenantHeld }))();
1690
1702
  if (outcome.changed) {
1691
1703
  audit({
1692
1704
  operationId: ulid(),
@@ -2246,6 +2258,10 @@ export class SqliteScopeHost {
2246
2258
  // #1575: attachment text is not in a dump, so a load left it as it was. Drop the
2247
2259
  // text of attachments the dump did not bring back, and queue extraction for those
2248
2260
  // it brought back without text — the bytes decide what that run finds.
2261
+ const switchSql = switchSqlOf(db);
2262
+ // #1686: nothing the source queued runs in a copy; a return leaves it all queued. BEFORE
2263
+ // the extraction queue below, which is this scope's own work, not the source's.
2264
+ settleCopiedWork(switchSql, scopeId, dump.scopeId, this.clock());
2249
2265
  reconcileAttachmentText(spineSql(db), ulid, this.clock());
2250
2266
  // Re-point scope-level grants at the scope they now live in. They are written as
2251
2267
  // `object = scope:<scopeId>`, so a fork, a restore into a different scope, or #286's
@@ -2257,7 +2273,7 @@ export class SqliteScopeHost {
2257
2273
  // (subject, relation, object) is the primary key, so a moved row can meet one the dump
2258
2274
  // already holds here; which one survives is that function's rule too (#1882), judged
2259
2275
  // at this host's clock as the checker judges expiry.
2260
- repointScopeGrants(switchSqlOf(db), scopeId, { scopeId: dump.scopeId, exact: dump.exact }, this.clock());
2276
+ repointScopeGrants(switchSql, scopeId, { scopeId: dump.scopeId, exact: dump.exact }, this.clock());
2261
2277
  // #1742: inside the replay's transaction, so a failure here rolls the whole load back and
2262
2278
  // the dump's grants never commit without the switch that should cover them.
2263
2279
  afterLoad?.(rt);
@@ -2266,7 +2282,20 @@ export class SqliteScopeHost {
2266
2282
  // load was a SAVEPOINT inside it, and that invoke's rollback undid the restore after
2267
2283
  // this verb had audited it.
2268
2284
  await rt.actor.turn(() => {
2269
- load(dump.tables);
2285
+ // #1686: a copy into another scope id loads no capability rows; a same-scope restore keeps them.
2286
+ load(capabilitiesForLoad(dump.tables, scopeId, dump.scopeId));
2287
+ // #1335 / #1686: ids resume above the outbox the dump brought, as on a reopen, so a copy's
2288
+ // own events sort above every copied one (`emittedHere()`). A top id that is no ULID leaves
2289
+ // the floor where it was.
2290
+ const top = db.prepare('SELECT MAX(id) AS id FROM _substrat_outbox').get()?.id;
2291
+ if (top) {
2292
+ try {
2293
+ rt.mintEventId.seedFrom(top);
2294
+ }
2295
+ catch {
2296
+ // Not a ULID: nothing this mint could have written, so there is no floor to keep.
2297
+ }
2298
+ }
2270
2299
  // The frontier came in with the dump — refresh the cached applied-migration set so
2271
2300
  // a later bind/migrate builds on the loaded state, not the previous one.
2272
2301
  rt.appliedMigrations.clear();
@@ -2602,12 +2631,29 @@ export class SqliteScopeHost {
2602
2631
  const deadLetter = rt.db.prepare(`INSERT OR IGNORE INTO _substrat_deliveries
2603
2632
  (event_id, consumer_module, delivered_at, error, invocation_id)
2604
2633
  VALUES (?, ?, ?, ?, NULL)`);
2634
+ // #1901: one line per (event, importing module), as for the scope's own consumers. The
2635
+ // handler runs as the producer's peer, so that is who the line says it ran as.
2636
+ const lines = asyncLinePass(this.invocationLineSink);
2637
+ const unitOf = (moduleId, eventId, eventType) => ({
2638
+ kind: 'consumer',
2639
+ tenantId: rt.tenantId,
2640
+ scopeId: rt.scopeId,
2641
+ invocationId: asyncInvocationId(null),
2642
+ operation: moduleId,
2643
+ eventType,
2644
+ eventId,
2645
+ attempt: 1,
2646
+ startedAt: Date.now(),
2647
+ principalKind: subject.kind,
2648
+ versionId: this.versionId,
2649
+ });
2605
2650
  // Named and not carried: a dead letter per importing module, so the operator who reads
2606
2651
  // this scope's dead letters sees what the producer sent and why it did not arrive.
2607
2652
  for (const w of batch.withheld) {
2608
2653
  record.run(w.id, source.scopeId, source.vertical, w.type, w.schemaVersion, w.occurredAt, w.entity.entityType, w.entity.entityId, 0, w.reason, at);
2609
2654
  for (const moduleId of this.crossVertical.modulesImporting(source.vertical, w.type)) {
2610
2655
  deadLetter.run(w.id, moduleId, at, withheldNote(w.reason, source.vertical));
2656
+ lines.write({ ...unitOf(moduleId, w.id, w.type), outcome: 'dead-lettered' });
2611
2657
  }
2612
2658
  result.withheld += 1;
2613
2659
  }
@@ -2623,14 +2669,17 @@ export class SqliteScopeHost {
2623
2669
  // this scope's code changed between the two, so it is refused by version (K-39)
2624
2670
  // exactly as the producer would have refused it.
2625
2671
  deadLetter.run(e.id, imp.moduleId, at, withheldNote('version', source.vertical));
2672
+ lines.write({ ...unitOf(imp.moduleId, e.id, e.type), outcome: 'dead-lettered' });
2626
2673
  result.deadLettered += 1;
2627
2674
  continue;
2628
2675
  }
2629
2676
  const { hops: _hops, ...fact } = e;
2630
2677
  const event = structuredClone({ ...fact, source });
2678
+ const unit = unitOf(imp.moduleId, e.id, e.type);
2631
2679
  // What the handler emits was emitted BECAUSE of the producer's event (#1237). The id
2632
2680
  // resolves through `_substrat_imports`, which the row above has just written.
2633
2681
  rt.causedBy = e.id;
2682
+ rt.unitInvocationId = unit.invocationId;
2634
2683
  rt.db.exec('BEGIN IMMEDIATE');
2635
2684
  try {
2636
2685
  // As the producer's principal, admitted above: the handler's checks are real ones,
@@ -2643,6 +2692,7 @@ export class SqliteScopeHost {
2643
2692
  .run(e.id, imp.moduleId, new Date().toISOString());
2644
2693
  rt.db.exec('COMMIT');
2645
2694
  result.delivered += 1;
2695
+ lines.write({ ...unit, outcome: 'delivered' });
2646
2696
  }
2647
2697
  catch (err) {
2648
2698
  rt.db.exec('ROLLBACK');
@@ -2650,9 +2700,11 @@ export class SqliteScopeHost {
2650
2700
  // behind it still arrive.
2651
2701
  deadLetter.run(e.id, imp.moduleId, new Date().toISOString(), String(err));
2652
2702
  result.deadLettered += 1;
2703
+ lines.write({ ...unit, outcome: 'dead-lettered', error: err });
2653
2704
  }
2654
2705
  finally {
2655
2706
  rt.causedBy = null;
2707
+ rt.unitInvocationId = null;
2656
2708
  }
2657
2709
  }
2658
2710
  if (!ran)
@@ -2662,6 +2714,7 @@ export class SqliteScopeHost {
2662
2714
  // rows above absorb it.
2663
2715
  rt.db.prepare(IMPORT_CURSOR_ADVANCE_SQL).run(source.scopeId, source.vertical, batch.next, at);
2664
2716
  result.cursor = batch.next;
2717
+ lines.end();
2665
2718
  // What the handlers emitted reaches this scope's own consumers and executors in the same
2666
2719
  // tail, under the ordinary cascade cap.
2667
2720
  await this.dispatch(rt, null);
@@ -3061,6 +3114,7 @@ export class SqliteScopeHost {
3061
3114
  // ended); the freshness evaluator's are `kind = 'freshness'` — see checkFreshness
3062
3115
  // for what the same two columns mean there.
3063
3116
  const now = Date.parse(nowIso);
3117
+ const lines = asyncLinePass(this.invocationLineSink);
3064
3118
  for (const schedule of mod.schedules) {
3065
3119
  const lastRunAt = gate.lastRuns.get(schedule.operation) ?? null;
3066
3120
  const lastRun = lastRunAt ? Date.parse(lastRunAt) : null;
@@ -3077,19 +3131,39 @@ export class SqliteScopeHost {
3077
3131
  // produces either way carries it, and so does the row below.
3078
3132
  const invocationId = ulid();
3079
3133
  let status = 'ok';
3134
+ // #1901: the run's line, under the call's own id — the one its `ctx.log` lines carry.
3135
+ const startedAt = Date.now();
3136
+ let emitted;
3137
+ let failure;
3080
3138
  try {
3081
3139
  const stub = await this.getSystemScope(moduleId, tenantId, scopeId);
3082
- await stub.invoke(schedule.operation, schedule.input, { invocationId });
3140
+ await stub.invoke(schedule.operation, schedule.input, { invocationId, onEmitted: (r) => (emitted = r) });
3083
3141
  report.fired += 1;
3084
3142
  }
3085
3143
  catch (err) {
3086
3144
  status = 'failed';
3145
+ failure = { error: err };
3087
3146
  report.failed += 1;
3088
3147
  report.errors.push({
3089
3148
  operation: schedule.operation,
3090
3149
  error: err instanceof Error ? err.message : String(err),
3091
3150
  });
3092
3151
  }
3152
+ lines.write({
3153
+ kind: 'schedule',
3154
+ tenantId,
3155
+ scopeId,
3156
+ invocationId,
3157
+ operation: schedule.operation,
3158
+ startedAt,
3159
+ outcome: status,
3160
+ ...failure,
3161
+ // Against the host's clock, which is what decided the run was due.
3162
+ dueAt: lastRun === null ? null : new Date(dueAt).toISOString(),
3163
+ latenessMs: lastRun === null ? null : now - dueAt,
3164
+ ...(emitted ? { emitted } : {}),
3165
+ versionId: this.versionId,
3166
+ });
3093
3167
  // On the scope actor (#1678): the cadence row joined whatever transaction an invoke
3094
3168
  // held open, so its rollback forgot the run and the next pass fired it AGAIN.
3095
3169
  await rt.actor.turn(() => rt.db
@@ -3101,6 +3175,7 @@ export class SqliteScopeHost {
3101
3175
  .run(schedule.operation, new Date(now).toISOString(), status, invocationId));
3102
3176
  report.runs.push({ operation: schedule.operation, outcome: status === 'ok' ? 'ok' : 'failed' });
3103
3177
  }
3178
+ lines.end();
3104
3179
  return report;
3105
3180
  }
3106
3181
  /** The stub body, shared by the principal, connection, system, impersonation and capability doors. */
@@ -3448,7 +3523,15 @@ export class SqliteScopeHost {
3448
3523
  throw new Error(`unknown guard predicate: '${guard.predicate}' — declared by ${guard.declaredBy} ` +
3449
3524
  `before '${operation}'; no registered module contributes it (operation blocked)`);
3450
3525
  }
3451
- await predicate.handler(ctx, guard.config, input);
3526
+ try {
3527
+ await predicate.handler(ctx, guard.config, input);
3528
+ }
3529
+ catch (err) {
3530
+ // #1745: a guard refusing is recorded after the rollback, as a transition is —
3531
+ // marked here, where which guard threw is still known. Always rethrown.
3532
+ markGuardRefusal(err, guard.predicate, operation);
3533
+ throw err;
3534
+ }
3452
3535
  }
3453
3536
  }
3454
3537
  async close() {
@@ -3496,7 +3579,26 @@ export class SqliteScopeHost {
3496
3579
  retrying: 0,
3497
3580
  deadLettered: 0,
3498
3581
  };
3582
+ if (this.executors.size === 0)
3583
+ return report;
3584
+ const lines = asyncLinePass(this.invocationLineSink);
3585
+ try {
3586
+ await this.dispatchExecutorsPass(rt, invocationId, report, lines);
3587
+ }
3588
+ finally {
3589
+ lines.end();
3590
+ }
3591
+ return report;
3592
+ }
3593
+ async dispatchExecutorsPass(rt, invocationId, report, lines) {
3499
3594
  const now = new Date().toISOString();
3595
+ // #2005: a non-primary scope (a fork, a snapshot, a preview of either kind) causes no
3596
+ // outbound effects. Every executor is host code acting with platform authority — a
3597
+ // connector with the tenant's credential, a plain executor with `HostAdmin` — so its
3598
+ // deliveries are journaled terminal with the reason and the handler never runs.
3599
+ // Asked on the first due event, once per pass: most passes have none.
3600
+ let inert;
3601
+ const isInert = () => (inert ??= !this.isPrimaryInDirectory(rt.scopeId));
3500
3602
  for (const [id, executor] of this.executors) {
3501
3603
  const deliveryId = `executor:${id}`;
3502
3604
  // Due = never attempted, or retrying and past its next attempt time.
@@ -3505,13 +3607,26 @@ export class SqliteScopeHost {
3505
3607
  .prepare(`SELECT o.* FROM _substrat_outbox o
3506
3608
  LEFT JOIN _substrat_deliveries d
3507
3609
  ON d.event_id = o.id AND d.consumer_module = ?
3508
- WHERE o.type = ?
3610
+ WHERE o.type = ? AND ${emittedHere('o.')}
3509
3611
  AND (d.event_id IS NULL
3510
3612
  OR (d.next_attempt_at IS NOT NULL AND d.next_attempt_at <= ?))
3511
3613
  ORDER BY o.id`)
3512
3614
  .all(deliveryId, executor.eventType, now);
3513
3615
  for (const row of rows) {
3514
3616
  report.attempted += 1;
3617
+ // #1901: this attempt's line. Its number is the one the journal records, which
3618
+ // `recordExecutorDelivery` answers — the Durable Object's RPC does the same.
3619
+ const unit = {
3620
+ kind: 'consumer',
3621
+ tenantId: rt.tenantId,
3622
+ scopeId: rt.scopeId,
3623
+ invocationId: asyncInvocationId(invocationId),
3624
+ operation: deliveryId,
3625
+ eventType: executor.eventType,
3626
+ eventId: row.id,
3627
+ startedAt: Date.now(),
3628
+ versionId: this.versionId,
3629
+ };
3515
3630
  let event;
3516
3631
  try {
3517
3632
  event = domainEventOf(row);
@@ -3523,8 +3638,16 @@ export class SqliteScopeHost {
3523
3638
  // on the FIRST failure, unlike a handler's: the decode is pure, of text already
3524
3639
  // in hand, so a retry cannot succeed — and only the decode is caught here, so a
3525
3640
  // handler's transient failure keeps its backoff below.
3526
- this.recordExecutorDelivery(rt, row.id, deliveryId, String(err), UNDECODABLE_RETRY, invocationId);
3641
+ const { attempts } = this.recordExecutorDelivery(rt, row.id, deliveryId, String(err), TERMINAL_RETRY, invocationId);
3527
3642
  report.deadLettered += 1;
3643
+ lines.write({ ...unit, attempt: attempts, outcome: 'dead-lettered' });
3644
+ continue;
3645
+ }
3646
+ if (isInert()) {
3647
+ const { attempts } = this.recordExecutorDelivery(rt, row.id, deliveryId, INERT_SCOPE_REASON, TERMINAL_RETRY, invocationId);
3648
+ report.inert = (report.inert ?? 0) + 1;
3649
+ // Its own outcome, never `delivered`: no handler ran (#2005).
3650
+ lines.write({ ...unit, attempt: attempts, outcome: 'inert' });
3528
3651
  continue;
3529
3652
  }
3530
3653
  this.causedBy = event.id;
@@ -3537,26 +3660,27 @@ export class SqliteScopeHost {
3537
3660
  else {
3538
3661
  await executor.handler(this.admin, event);
3539
3662
  }
3540
- this.recordExecutorDelivery(rt, row.id, deliveryId, null, executor.retry, invocationId);
3663
+ const { attempts } = this.recordExecutorDelivery(rt, row.id, deliveryId, null, executor.retry, invocationId);
3541
3664
  report.delivered += 1;
3665
+ lines.write({ ...unit, attempt: attempts, outcome: 'delivered' });
3542
3666
  }
3543
3667
  catch (err) {
3544
- const dead = this.recordExecutorDelivery(rt, row.id, deliveryId, err instanceof Error ? (err.stack ?? err.message) : String(err), executor.retry, invocationId);
3668
+ const { dead, attempts } = this.recordExecutorDelivery(rt, row.id, deliveryId, err instanceof Error ? (err.stack ?? err.message) : String(err), executor.retry, invocationId);
3545
3669
  if (dead)
3546
3670
  report.deadLettered += 1;
3547
3671
  else
3548
3672
  report.retrying += 1;
3673
+ lines.write({ ...unit, attempt: attempts, outcome: dead ? 'dead-lettered' : 'retrying', error: err });
3549
3674
  }
3550
3675
  finally {
3551
3676
  this.causedBy = null;
3552
3677
  }
3553
3678
  }
3554
3679
  }
3555
- return report;
3556
3680
  }
3557
3681
  /**
3558
- * Journal one executor attempt. Returns true when this attempt was the last one
3559
- * — i.e. the delivery is now dead-lettered.
3682
+ * Journal one executor attempt. Answers which attempt it was (#1901), and `dead` when
3683
+ * it was the last one — i.e. the delivery is now dead-lettered.
3560
3684
  *
3561
3685
  * Written AFTER the handler ran, so a crash mid-effect retries rather than
3562
3686
  * silently marking success. Claiming first would make delivery at-most-once and
@@ -3589,7 +3713,11 @@ export class SqliteScopeHost {
3589
3713
  .run(eventId, deliveryId, new Date().toISOString(), error, attempts, nextAttemptAt,
3590
3714
  // #1525: the call this attempt ran in, as the caller named it.
3591
3715
  invocationId);
3592
- return error !== null && exhausted;
3716
+ return { dead: error !== null && exhausted, attempts };
3717
+ }
3718
+ /** Whether the directory says `scopeId` is the real install (`isPrimaryScopeRow`, #2005). */
3719
+ isPrimaryInDirectory(scopeId) {
3720
+ return isPrimaryScopeRow(this.directory.prepare('SELECT kind, forked_from FROM scopes WHERE scope_id = ?').get(scopeId));
3593
3721
  }
3594
3722
  async drainDue(tenantId, scopeId) {
3595
3723
  const rt = this.runtime(tenantId, scopeId);
@@ -3665,12 +3793,12 @@ export class SqliteScopeHost {
3665
3793
  return {
3666
3794
  startOrJoin: (key, r) => turn(() => startOrJoinTx(key, r)),
3667
3795
  get: (id) => turn(() => row(db.prepare('SELECT * FROM _substrat_job_runs WHERE id = ?').get(id))),
3668
- due: (now, limit, afterId) => turn(() => db
3669
- .prepare(`SELECT * FROM _substrat_job_runs
3796
+ // #1834: the drive's one snapshot — keys only, in the order runs became due.
3797
+ dueKeys: (now, max) => turn(() => db
3798
+ .prepare(`SELECT id, module_id, job FROM _substrat_job_runs
3670
3799
  WHERE status = 'running' AND (next_attempt_at IS NULL OR next_attempt_at <= ?)
3671
- AND (? IS NULL OR id > ?)
3672
- ORDER BY id LIMIT ?`)
3673
- .all(now, afterId ?? null, afterId ?? null, limit)),
3800
+ ORDER BY ${JOB_RUN_DUE_AT}, id LIMIT ?`)
3801
+ .all(now, max)),
3674
3802
  list: (filter) => turn(() => {
3675
3803
  const where = [];
3676
3804
  const params = [];
@@ -3929,13 +4057,23 @@ export class SqliteScopeHost {
3929
4057
  * own emit is delivered in the same tail, so the whole cascade is one call's work.
3930
4058
  */
3931
4059
  invocationId) {
4060
+ // #1901: one line per delivery, capped per drain.
4061
+ const lines = asyncLinePass(this.invocationLineSink);
4062
+ try {
4063
+ await this.dispatchRounds(rt, invocationId, lines);
4064
+ }
4065
+ finally {
4066
+ lines.end();
4067
+ }
4068
+ }
4069
+ async dispatchRounds(rt, invocationId, lines) {
3932
4070
  for (let round = 0; round < 50; round++) {
3933
4071
  let deliveredAny = false;
3934
4072
  for (const mod of this.modules.values()) {
3935
4073
  for (const consumer of mod.consumers) {
3936
4074
  const rows = rt.db
3937
4075
  .prepare(`SELECT * FROM _substrat_outbox o
3938
- WHERE o.type = ?
4076
+ WHERE o.type = ? AND ${emittedHere('o.')}
3939
4077
  AND NOT EXISTS (
3940
4078
  SELECT 1 FROM _substrat_deliveries d
3941
4079
  WHERE d.event_id = o.id AND d.consumer_module = ?
@@ -3943,6 +4081,20 @@ export class SqliteScopeHost {
3943
4081
  ORDER BY o.id`)
3944
4082
  .all(consumer.eventType, mod.id);
3945
4083
  for (const row of rows) {
4084
+ // #1901: this delivery's line — the call's id when it runs in one, else its own.
4085
+ const unit = {
4086
+ kind: 'consumer',
4087
+ tenantId: rt.tenantId,
4088
+ scopeId: rt.scopeId,
4089
+ invocationId: asyncInvocationId(invocationId),
4090
+ operation: mod.id,
4091
+ eventType: consumer.eventType,
4092
+ eventId: row.id,
4093
+ // A module consumer is tried once: a throw dead-letters it (v0).
4094
+ attempt: 1,
4095
+ startedAt: Date.now(),
4096
+ versionId: this.versionId,
4097
+ };
3946
4098
  let event;
3947
4099
  try {
3948
4100
  event = domainEventOf(row);
@@ -3959,6 +4111,8 @@ export class SqliteScopeHost {
3959
4111
  (event_id, consumer_module, delivered_at, error, invocation_id)
3960
4112
  VALUES (?, ?, ?, ?, ?)`)
3961
4113
  .run(row.id, mod.id, new Date().toISOString(), String(err), invocationId);
4114
+ // Never reached the handler, so nothing threw in it: a warning, not an error.
4115
+ lines.write({ ...unit, outcome: 'dead-lettered' });
3962
4116
  continue;
3963
4117
  }
3964
4118
  const ctx = this.operationContext(rt, asPrincipal(this.systemPrincipal), {
@@ -3969,6 +4123,8 @@ export class SqliteScopeHost {
3969
4123
  // consumers never did, which is exactly where a backwards walk stopped.
3970
4124
  // On the runtime rather than the host — see `ScopeRuntime.causedBy`.
3971
4125
  rt.causedBy = event.id;
4126
+ // #1901: the handler's `ctx.log` lines join this delivery's line.
4127
+ rt.unitInvocationId = unit.invocationId;
3972
4128
  rt.db.exec('BEGIN IMMEDIATE');
3973
4129
  try {
3974
4130
  await consumer.handler(ctx, event);
@@ -3979,6 +4135,7 @@ export class SqliteScopeHost {
3979
4135
  .run(event.id, mod.id, new Date().toISOString(), invocationId);
3980
4136
  rt.db.exec('COMMIT');
3981
4137
  deliveredAny = true;
4138
+ lines.write({ ...unit, outcome: 'delivered' });
3982
4139
  }
3983
4140
  catch (err) {
3984
4141
  rt.db.exec('ROLLBACK');
@@ -3989,8 +4146,10 @@ export class SqliteScopeHost {
3989
4146
  (event_id, consumer_module, delivered_at, error, invocation_id)
3990
4147
  VALUES (?, ?, ?, ?, ?)`)
3991
4148
  .run(event.id, mod.id, new Date().toISOString(), String(err), invocationId);
4149
+ lines.write({ ...unit, outcome: 'dead-lettered', error: err });
3992
4150
  }
3993
4151
  finally {
4152
+ rt.unitInvocationId = null;
3994
4153
  // Cleared on BOTH paths. Left set, the id would leak onto every later
3995
4154
  // emit in this scope — an operation's own event stamped as caused by
3996
4155
  // whatever happened to be delivered last, which is worse than no cause
@@ -4010,11 +4169,12 @@ export class SqliteScopeHost {
4010
4169
  * point, since the denial is exactly the write the rolled-back operation could not make.
4011
4170
  */
4012
4171
  /**
4013
- * #1745: record a refused transition the operation failed with, after its rollback — the
4014
- * denial's discipline (above), for the lifecycle's refusals. Anything else is not one.
4172
+ * #1745: record a refused transition or refusing guard the operation failed with, after its
4173
+ * rollback — the denial's discipline (above), for the lifecycle's and the guards' refusals.
4174
+ * Anything else is not one.
4015
4175
  */
4016
4176
  recordRefusal(rt, subject, operation, err, invocationId, impersonation) {
4017
- const refused = refusedTransitionOf(err);
4177
+ const refused = refusalOf(err);
4018
4178
  if (!refused)
4019
4179
  return;
4020
4180
  const q = refusalInsert({
@@ -4581,37 +4741,37 @@ export class SqliteScopeHost {
4581
4741
  const target = { tenantId, scopeId };
4582
4742
  const base = { operationId, moduleId: input.moduleId, schedules: to };
4583
4743
  this.recordAdmin(actor, action, target, null, { ...base, phase: 'intent', reason: input.reason });
4584
- // The directory's record (#1674): ON before the scope moves, OFF after it held — see
4585
- // `recordSystemSwitchedOn` for why that order is the safe one.
4744
+ // The directory's record (#1674), written BEFORE the scope moves, both ways, and undone
4745
+ // if the move throws or holds nothing — see `recordSystemSwitchedOn` and (#1823)
4746
+ // `recordSystemSwitchedOff` for why that order is the safe one.
4586
4747
  const at = new Date().toISOString();
4587
4748
  const directorySql = switchSqlOf(this.directory);
4588
4749
  const record = { tenantId, scopeId, moduleId: input.moduleId, actor, reason: input.reason, operationId, at };
4589
4750
  const errorOf = (err) => (err instanceof Error ? err.message : String(err));
4590
- let prior = null;
4591
- if (to === 'on') {
4751
+ let prior;
4752
+ try {
4753
+ prior = to === 'on' ? recordSystemSwitchedOn(directorySql, record) : recordSystemSwitchedOff(directorySql, record);
4754
+ }
4755
+ catch (err) {
4756
+ // Nothing has moved: fail the call here, audited — the Cloudflare adapter's posture.
4592
4757
  try {
4593
- prior = recordSystemSwitchedOn(directorySql, record);
4758
+ this.recordAdmin(actor, action, target, null, { ...base, phase: 'failed', error: errorOf(err) });
4594
4759
  }
4595
- catch (err) {
4596
- // Nothing has moved: fail the call here, audited — the Cloudflare adapter's posture.
4597
- try {
4598
- this.recordAdmin(actor, action, target, null, { ...base, phase: 'failed', error: errorOf(err) });
4599
- }
4600
- catch {
4601
- // Best effort: the original error is what the caller must see.
4602
- }
4603
- throw err;
4760
+ catch {
4761
+ // Best effort: the original error is what the caller must see.
4604
4762
  }
4763
+ throw err;
4605
4764
  }
4606
- /** A record write after the scope moved (review #5): retried once, never swallowed. */
4607
- const recordWrite = (write) => {
4765
+ /** The record's undo, after the scope's move threw or held nothing: retried once, never swallowed. */
4766
+ const undoRecord = () => {
4767
+ const undo = () => restoreSystemSwitchRecord(directorySql, record, prior);
4608
4768
  try {
4609
- write();
4769
+ undo();
4610
4770
  return null;
4611
4771
  }
4612
4772
  catch {
4613
4773
  try {
4614
- write();
4774
+ undo();
4615
4775
  return null;
4616
4776
  }
4617
4777
  catch (err) {
@@ -4626,11 +4786,17 @@ export class SqliteScopeHost {
4626
4786
  // `db.transaction` issued mid-invoke became a SAVEPOINT inside it and rolled back
4627
4787
  // with it — after this verb had already audited `applied`. `turn` is re-entrant, so a
4628
4788
  // caller already inside one of this scope's tasks joins it rather than deadlocking.
4789
+ //
4790
+ // #1823: read in the same turn, so no tenant grant moves between the read and the switch.
4791
+ // The directory is this process's own database, so the read is synchronous.
4629
4792
  const rt = this.runtime(tenantId, scopeId);
4630
- outcome = await rt.actor.turn(() => rt.db.transaction(() => switchSystemSchedules(switchSqlOf(rt.db), { moduleId: input.moduleId, scopeId, to, at }))());
4793
+ outcome = await rt.actor.turn(() => {
4794
+ const tenantHeld = tenantHoldsSystemGrant(directorySql, tenantId, input.moduleId, this.clock());
4795
+ return rt.db.transaction(() => switchSystemSchedules(switchSqlOf(rt.db), { moduleId: input.moduleId, scopeId, to, at, tenantHeld }))();
4796
+ });
4631
4797
  }
4632
4798
  catch (err) {
4633
- const recordError = recordWrite(() => restoreSystemSwitchRecord(directorySql, record, prior));
4799
+ const recordError = undoRecord();
4634
4800
  try {
4635
4801
  this.recordAdmin(actor, action, target, null, {
4636
4802
  ...base,
@@ -4644,14 +4810,11 @@ export class SqliteScopeHost {
4644
4810
  }
4645
4811
  throw err;
4646
4812
  }
4647
- // OFF is recorded once the scope held it. A refused ON moved nothing, so its record
4648
- // write is undone: left `on`, the next reconcile of a wiped scope would leave the
4649
- // module running.
4650
- const recordError = to === 'off' && outcome.held
4651
- ? recordWrite(() => recordSystemSwitchedOff(directorySql, record))
4652
- : to === 'on' && !outcome.held
4653
- ? recordWrite(() => restoreSystemSwitchRecord(directorySql, record, prior))
4654
- : null;
4813
+ // A call that held nothing moved nothing (the module has no authority reaching the scope,
4814
+ // its own or the tenant's), so its record write is undone: left `off`, it
4815
+ // would switch the module off the day it is installed; left `on`, the next reconcile of
4816
+ // a wiped scope would leave the module running.
4817
+ const recordError = outcome.held ? null : undoRecord();
4655
4818
  this.recordAdmin(actor, action, target, null, {
4656
4819
  ...base,
4657
4820
  phase: outcome.held ? 'applied' : 'refused',
@@ -4664,11 +4827,6 @@ export class SqliteScopeHost {
4664
4827
  `(check the module id: it is the module's manifest id, e.g. '@substrat-run/engine-absence')` +
4665
4828
  (recordError ? `; and its directory record could not be put back (${recordError})` : ''));
4666
4829
  }
4667
- if (recordError) {
4668
- throw substratError('unavailable', `module '${input.moduleId}' is switched ${to} on scope ${scopeId}, but the directory's record of it ` +
4669
- `could not be written (${recordError}) — repeat the call to record it; a wipe of this scope would ` +
4670
- `otherwise lose the switch`);
4671
- }
4672
4830
  return {
4673
4831
  operationId,
4674
4832
  moduleId: input.moduleId,
@@ -5053,6 +5211,12 @@ export class SqliteScopeHost {
5053
5211
  this.recordAccess(actor, 'listSystemSwitches', { tenantId: filter?.tenantId ?? null }, filter ?? null, rows.length);
5054
5212
  return rows.map((r) => systemSwitchRecord.parse(r));
5055
5213
  },
5214
+ tenantHeldSystemModules: async (actor, tenantId, moduleIds) => {
5215
+ const now = this.clock();
5216
+ const held = moduleIds.filter((m) => tenantHoldsSystemGrant(switchSqlOf(this.directory), tenantId, m, now));
5217
+ this.recordAccess(actor, 'tenantHeldSystemModules', { tenantId }, { moduleIds: [...moduleIds] }, held.length);
5218
+ return held;
5219
+ },
5056
5220
  reassertSystemSwitches: async (actor, node, opts) => {
5057
5221
  this.assertScope(node.tenantId, node.scopeId);
5058
5222
  const rt = this.runtime(node.tenantId, node.scopeId);
@@ -5270,7 +5434,16 @@ export class SqliteScopeHost {
5270
5434
  COALESCE(s.serving_ref, vv.deployment_ref) AS deployment_ref,
5271
5435
  CASE WHEN s.serving_ref IS NOT NULL
5272
5436
  THEN json_extract(sv.manifest_json, '$.outbound')
5273
- ELSE json_extract(vv.manifest_json, '$.outbound') END AS outbound_json
5437
+ ELSE json_extract(vv.manifest_json, '$.outbound') END AS outbound_json,
5438
+ s.kind, s.forked_from,
5439
+ -- #2005: the scope's own addresses, bounded as the Cloudflare read bounds them.
5440
+ (SELECT json_group_array(sh.hostname) FROM (
5441
+ -- Pinned like the Cloudflare read: a range on the scope, never every
5442
+ -- active hostname through hostnames_status.
5443
+ SELECT hostname FROM hostnames INDEXED BY hostnames_scope
5444
+ WHERE scope_id = s.scope_id AND status = 'active'
5445
+ ORDER BY hostname LIMIT ${ROUTE_SCOPE_HOSTNAMES_MAX}
5446
+ ) sh) AS scope_hostnames_json
5274
5447
  FROM hostnames h
5275
5448
  JOIN scopes s ON s.scope_id = h.scope_id AND s.tenant_id = h.tenant_id
5276
5449
  JOIN tenants t ON t.tenant_id = s.tenant_id
@@ -5302,6 +5475,9 @@ export class SqliteScopeHost {
5302
5475
  surface: r.surface,
5303
5476
  region: r.region,
5304
5477
  outboundHosts,
5478
+ // #2005: the egress worker holds a fork or a preview to no third-party egress.
5479
+ primary: isPrimaryScopeRow(r),
5480
+ hostnames: JSON.parse(r.scope_hostnames_json),
5305
5481
  });
5306
5482
  },
5307
5483
  registerVertical: async (actor, input) => {
@@ -5828,6 +6004,15 @@ export class SqliteScopeHost {
5828
6004
  .run(servingRef, scopeId);
5829
6005
  this.recordAdmin(actor, 'setScopeServingRef', { tenantId, scopeId }, { servingRef: scope.serving_ref }, { servingRef, ...(ack.exportBreak ? { acknowledged: ack } : {}) });
5830
6006
  },
6007
+ recordKeptCopyResolution: async (actor, tenantId, scopeId, r) => {
6008
+ const scope = this.directory
6009
+ .prepare('SELECT tenant_id FROM scopes WHERE scope_id = ?')
6010
+ .get(scopeId);
6011
+ if (!scope || scope.tenant_id !== tenantId) {
6012
+ throw substratError('not_found', `unknown scope ${scopeId} in tenant ${tenantId}`);
6013
+ }
6014
+ this.recordAdmin(actor, 'resolveKeptCopy', { tenantId, scopeId }, { script: r.script, keptAt: r.keptAt, revision: r.revisionBefore }, { action: r.action, liveScript: r.liveScript, revision: r.revisionAfter });
6015
+ },
5831
6016
  setScopeExpiresAt: async (actor, tenantId, scopeId, expiresAt) => {
5832
6017
  const scope = this.directory
5833
6018
  .prepare('SELECT tenant_id, expires_at FROM scopes WHERE scope_id = ?')
@@ -5846,7 +6031,7 @@ export class SqliteScopeHost {
5846
6031
  // — which is append-only — holding an event the scope never had. The mark that
5847
6032
  // follows needs no such care: it stamps only rows that exist and are undrained.
5848
6033
  const db = this.scopeReadDbFor(tenantId, scopeId);
5849
- const page = db.prepare(`SELECT * FROM _substrat_outbox WHERE drained_at IS NULL ORDER BY id LIMIT ? OFFSET ?`);
6034
+ const page = db.prepare(`SELECT * FROM _substrat_outbox WHERE drained_at IS NULL AND ${emittedHere()} ORDER BY id LIMIT ? OFFSET ?`);
5850
6035
  // The kernel's read, shared with the DO: the envelope and the columns lifted beside
5851
6036
  // it (#1242, #1237) are decoded once, and a row that will not decode is stepped over
5852
6037
  // rather than stalling the scope on it (#1636).
@@ -7270,6 +7455,11 @@ export class SqliteScopeHost {
7270
7455
  const { tenantId, scopeId, ...after } = ownerTransferAudit.parse(entry);
7271
7456
  this.recordAdmin(actor, 'transferOwner', { tenantId, scopeId }, null, after);
7272
7457
  },
7458
+ /** #2005: one change to a scope's copy marker, written around the vertical's own change. */
7459
+ recordCopyMark: async (actor, entry) => {
7460
+ const { tenantId, scopeId, action, ...after } = copyMarkAudit.parse(entry);
7461
+ this.recordAdmin(actor, action === 'mark' ? 'markScopeCopy' : 'clearScopeCopyMark', { tenantId, scopeId }, null, after);
7462
+ },
7273
7463
  accessLog: async (actor, filter) => {
7274
7464
  const where = [];
7275
7465
  const params = [];
@@ -7705,18 +7895,32 @@ export class SqliteScopeHost {
7705
7895
  * read `ensureIdentityKey` already uses — PRAGMA would do here, but not there.
7706
7896
  */
7707
7897
  ensureScheduleStateKind(db) {
7898
+ this.rebuildIfStale(db, '_substrat_schedule_state', scheduleStateHasKind, SCHEDULE_STATE_REBUILD);
7899
+ }
7900
+ /**
7901
+ * #1745: `_substrat_refusals`, rebuilt to admit a guard row (a nullable `from_state`, plus
7902
+ * `guard` and `reason`) on a scope db whose table predates it.
7903
+ */
7904
+ ensureRefusalsAdmitGuards(db) {
7905
+ this.rebuildIfStale(db, '_substrat_refusals', refusalsAdmitGuards, REFUSALS_REBUILD);
7906
+ }
7907
+ /**
7908
+ * Run a kernel create-copy-drop-rename `script` over `table` when the stored DDL fails
7909
+ * `isCurrent` — the DO adapter's `rebuildIfStale`, so the two rebuild byte-identically.
7910
+ */
7911
+ rebuildIfStale(db, table, isCurrent, script) {
7708
7912
  const row = db
7709
7913
  .prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?")
7710
- .get('_substrat_schedule_state');
7711
- if (!row || scheduleStateHasKind(row.sql))
7914
+ .get(table);
7915
+ if (!row || isCurrent(row.sql))
7712
7916
  return;
7713
- // In a transaction, for the reason the kernel constant spells out: create-copy-
7917
+ // In a transaction, for the reason the kernel constants spell out: create-copy-
7714
7918
  // drop-rename has two intermediate states and BOTH are unrecoverable on the next
7715
7919
  // wake — one dies on the leftover scratch table, one silently orphans the rows
7716
7920
  // behind an empty table `CREATE TABLE IF NOT EXISTS` put back. `db.transaction`
7717
7921
  // nests as a SAVEPOINT, which is what makes this safe on the `loadDump` path too,
7718
7922
  // where the whole replay is already inside one.
7719
- db.transaction(() => db.exec(SCHEDULE_STATE_REBUILD))();
7923
+ db.transaction(() => db.exec(script))();
7720
7924
  }
7721
7925
  /**
7722
7926
  * #1573: run a create-copy-drop-rename against the directory as ONE transaction.
@@ -8155,7 +8359,7 @@ export class SqliteScopeHost {
8155
8359
  tenantId: rt.tenantId,
8156
8360
  scopeId: rt.scopeId,
8157
8361
  operation: operation ?? null,
8158
- invocationId: () => rt.invocationId,
8362
+ invocationId: () => rt.invocationId ?? rt.unitInvocationId,
8159
8363
  // A consumer runs under the system override, so it logs as `system`.
8160
8364
  principalKind: overrideActor ? 'system' : subject.kind,
8161
8365
  // The string-safe redaction: `redactSecrets` parses its serialization back, and a
@@ -8243,7 +8447,7 @@ export class SqliteScopeHost {
8243
8447
  // as `pageOf`, applied here because the cursor is a COLUMN's value (plus
8244
8448
  // the tie-break) rather than a field the caller could name.
8245
8449
  const last = rows.length >= limit ? rows[rows.length - 1] : undefined;
8246
- const nextCursor = last === undefined ? null : cursorOf(last, q.sortColumn, plan.idColumn);
8450
+ const nextCursor = last === undefined ? null : cursorOf(last, q.sortColumn, plan.idColumn, q.order);
8247
8451
  const page = { entries: rows, nextCursor };
8248
8452
  if (!params.total)
8249
8453
  return page;
@@ -8644,6 +8848,8 @@ export class SqliteScopeHost {
8644
8848
  // #1288: not an ensureColumn, because `kind` joins the schedule-state PRIMARY KEY
8645
8849
  // and no ALTER can widen a key. Rebuilt instead, from the kernel's statements.
8646
8850
  this.ensureScheduleStateKind(db);
8851
+ // #1745: likewise a rebuild — `from_state` loses its NOT NULL, which no ALTER can do.
8852
+ this.ensureRefusalsAdmitGuards(db);
8647
8853
  // #1525: the invocation a fired schedule ran in, on a scope DB created before the
8648
8854
  // column. Nullable, and the null is honestly "no call was carried" for every legacy
8649
8855
  // row — a past run's id cannot be recovered afterwards, exactly as the other #1525
@@ -8653,6 +8859,9 @@ export class SqliteScopeHost {
8653
8859
  // executes, the table always already has `kind` in its key, never `invocation_id`.
8654
8860
  this.ensureColumn(db, '_substrat_schedule_state', 'invocation_id', 'invocation_id TEXT');
8655
8861
  this.ensureColumn(db, '_substrat_job_runs', 'subject_id', 'subject_id TEXT');
8862
+ // #2009: the copy classification on a scope DB built before it (NULL reads as a copy; see
8863
+ // `COPY_ORIGIN_DDL`).
8864
+ this.ensureColumn(db, '_substrat_copy_origin', 'is_copy', 'is_copy INTEGER');
8656
8865
  // #1237: `readInvocation`'s lookup — WHERE invocation_id = ? ORDER BY id — over an outbox
8657
8866
  // that is never pruned. No index leads with invocation_id, so without this one SQLite
8658
8867
  // walks the PRIMARY KEY from the oldest event until it reaches the call, and reading a
@@ -8700,6 +8909,7 @@ export class SqliteScopeHost {
8700
8909
  mintEventId,
8701
8910
  causedBy: null,
8702
8911
  invocationId: null,
8912
+ unitInvocationId: null,
8703
8913
  };
8704
8914
  this.scopes.set(key, created);
8705
8915
  this.scopesById.set(scopeId, created);