@substrat-run/adapter-sqlite 0.135.0 → 0.137.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/checker.d.ts.map +1 -1
- package/dist/checker.js +5 -1
- package/dist/checker.js.map +1 -1
- package/dist/index.d.ts +29 -6
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +282 -72
- package/dist/index.js.map +1 -1
- package/package.json +6 -6
package/dist/index.js
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
|
2
2
|
import { dirname, join } from 'node:path';
|
|
3
3
|
import Database from 'better-sqlite3';
|
|
4
|
-
import { accessLogEntry, adminLogEntry, delegatedReadParams, opsFailureEntry, opsFailureFingerprint, issueEntry, FRESHNESS_HEARTBEAT_MINUTES, sweepRunEntry, modelUsageEntry, ATTACHMENT_ADDED, ATTACHMENT_REMOVED, attachmentRecord, createTenantInput, domainEvent, domainEventInput, exportReadInput, importBatch, importCursorMove, exportsOfManifestJson, importsOfManifestJson, entitlementGrant, entitlementGrantInput, eventId, meterReading, platformRequestInput, platformRequestId, MAX_PENDING_PLATFORM_REQUESTS, identityLink, identityPool, instant, connection, capabilityGrant, connectionGrant, connectionGrantRecord, connectionSecret, systemGrant, systemSwitch, systemSwitchRecord, peerSwitch, verticalCaller, verticalSlug as verticalSlugOf, subjectRef, requestFingerprint, createConnectionInput, projectedConnectionGrant, projectedConnectionKey, moduleManifest, createOrgInput, promotionAcknowledgement, bindAcknowledgement, bindHostnameInput, channelHistoryEntry, hostnameBinding, publishVersionInput, AUTO_ADMISSION_NOTE, routeTarget, registerVerticalInput, vertical as verticalSchema, verticalChannel, verticalVersion, objectRef, entityObjectRef, assertModuleEmittableType, assertKernelAuthoredType, grantRefFromProof, org as orgSchema, orgMembership, principalId, resolvedIdentity, identityMembership, roleDefinition, scope as scopeSchema, tenant as tenantSchema, tenantRole, subjectShredReceipt, SCOPE_TABLE_PAGE_DEFAULT, SCOPE_TABLE_PAGE_MAX, SCOPE_QUERY_ROW_MAX, verticalServingState, callsOfManifestJson, outboundOfManifestJson, listLimitOf, substratError, assertReplayableDump, delegatedReadRecord, ownerTransferAudit, redrainEventsInput, REDRAIN_BATCH, } from '@substrat-run/contracts';
|
|
5
|
-
import { actorOf, asPrincipal, assertAllowed, assertNoSecret, assertPermissionKey, CAPABILITY_DDL, CAPABILITY_EXCHANGE_OPERATION, capabilityAttachmentWriteRefused, capabilityTokenHash, createCapabilityVerbs, createEntityEdgeVerbs, exchangeCapability as exchangeCapabilitySecret, guardSecrets, mintBecomeCapability, plausibleSessionToken, readCapabilityPage, redactSecrets, redactSecretText, moduleLog, consoleLogSink, resolveCapabilitySession, revokeCapabilityAsPlatform, assertReadOnlyQuery, attachmentBlobKey, attachmentSha256, entitlementDenial, requiredEntitlementFor, foldMeterReading, guardSpine, guardSqlLimits, DO_SQL_LIMITS, tooManyResultColumns, tooManyTableColumns, parseValidationRecords, resolveScopeRecord, ulid, ISSUE_RETENTION_DAYS, telemetryRetentionStatements, assertRowLimit, assertRowOffset, OPS_FAILURE_RETENTION_DAYS, SWEEP_RUN_RETENTION_DAYS, SWEEP_RUNS_INTENT_INDEX, sweepRunsIntentHasKind, MODEL_USAGE_RETENTION_DAYS, foldModelUsage, backoffAt, platformRequestHistoryQuery, platformRequestOf, PLATFORM_REQUEST_COLUMNS, PLATFORM_REQUEST_REDACTION_SQL, platformRequestRedactionParams, platformRequestRedactionQuery, intentPayloadCarriesSubject, redactSubjectJobRuns, redactSubjectScopeText, redactSubjectDirectoryText, ISSUE_EXEMPLAR_OWNER_BACKFILL_SQL, issueExemplarOwner, JOB_RUN_PATCH_SQL, JOB_STEP_RECORD_SQL, DELIVERY_ERROR_REDACTION_SQL, REDACTED_DELIVERY_NOTE, seatScopeTuple, switchSystemSchedules, admitPeer, collectPeers, peerSeats, resolveVerticalInstanceFrom, switchPeer, peerGrantsStatus, systemGrantsStatus, systemScheduleState, systemSwitchedOff, systemSwitchedOffMessage, tenantSystemSwitchedOffMessage, SYSTEM_SWITCHES_BACKFILL_SQL, SYSTEM_SWITCHES_DDL, forgetSystemSwitchesOf, listSystemSwitchRecords, recordSystemSwitchedOff, recordSystemSwitchedOn, restoreSystemSwitchRecord, scopesSwitchedOffFor, switchedOffModulesOf, inUnitMovesToAudit, staleCarryRevertRow, staleCarryReverts, systemSwitchRecordsOf, systemSwitchesTableExists, withRecorded, VERSION_MIGRATIONS_DDL, splitManifestMigrations, splitVersionMigrationsBatch, versionMigrationsOf, writeVersionMigrations, denialListQuery, refusalListQuery, mapRefusalRow, denialSummaryQuery, denialTotalsQuery, DENIAL_WINDOW_QUERY, mapDenialRow, mapDenialSummaryBuckets, IMPERSONATION_COLUMNS, IMPERSONATION_DDL, ImpersonationRefused, assertImpersonationWrites, assertModuleEnqueueableKind, assertSessionUsable, impersonationByIdQuery, impersonationListQuery, impersonationRowValues, impersonationStampOf, mapImpersonationRow, newImpersonationSession, resolveRetryPolicy, isSecretBoxConfigured, unconfiguredSecretBox, createSubjectKeys, generateSealingKeyPair, noSealingKeyMessage, openSealed, sealTo, ConnectionSealingKeyUnavailableError, PermissionDenied, createAtomic, NotSearchable, NotListable, listIndexMigrations, listIndexPlans, listQuery, cursorOf, moduleMigrations, isSearchIndexTable, searchIndexDdl, searchIndexMigrations, searchIndexPlans, searchLimit, searchMatchExpression, searchQuery, ATTACHMENT_TEXT_DDL, assertAttachmentExtractors, assertJobRegistrable, attachmentRecordOfRow, attachmentTextJob, enqueueAttachmentText, isAttachmentTextRun, reconcileAttachmentText, recordAttachmentText, searchAttachments, entityVersionQuery, entityVersionOf, assertIfMatch, OUTBOX_ENTITY_INDEX, SCHEDULE_STATE_DDL, SCHEDULE_STATE_REBUILD, scheduleStateHasKind, VERTICAL_EVENTS_DDL, EXPORT_HOPS_SQL, IMPORT_CURSORS_SQL, IMPORT_CURSOR_OF_SQL, OUTBOX_MARK_SQL, emittedSinceQuery, emittedReportOf, EMITTED_REPORT_CAP, exportedSinceQuery, IMPORT_CURSOR_ADVANCE_SQL, IMPORT_RECORD_SQL, moveImportCursor, importCursorSourceOf, exportBreaksOf, exportBreakRefusal, bindExportBreaksOf, bindExportBreakRefusal, CrossVerticalRegistry, exportReadPlan, exportReadQuery, planExportBatch, withheldNote, JOB_RUN_DDL, jobRunListLimit, jobRunOf, runDueJobRuns, startJobRun, IDEMPOTENCY_DDL,
|
|
4
|
+
import { accessLogEntry, adminLogEntry, delegatedReadParams, opsFailureEntry, opsFailureFingerprint, issueEntry, FRESHNESS_HEARTBEAT_MINUTES, sweepRunEntry, modelUsageEntry, ATTACHMENT_ADDED, ATTACHMENT_REMOVED, attachmentRecord, createTenantInput, domainEvent, domainEventInput, exportReadInput, importBatch, importCursorMove, exportsOfManifestJson, importsOfManifestJson, entitlementGrant, entitlementGrantInput, eventId, meterReading, platformRequestInput, platformRequestId, MAX_PENDING_PLATFORM_REQUESTS, identityLink, identityPool, instant, connection, capabilityGrant, connectionGrant, connectionGrantRecord, connectionSecret, systemGrant, systemSwitch, systemSwitchRecord, peerSwitch, verticalCaller, verticalSlug as verticalSlugOf, subjectRef, requestFingerprint, createConnectionInput, projectedConnectionGrant, projectedConnectionKey, moduleManifest, createOrgInput, promotionAcknowledgement, bindAcknowledgement, bindHostnameInput, channelHistoryEntry, hostnameBinding, publishVersionInput, AUTO_ADMISSION_NOTE, routeTarget, ROUTE_SCOPE_HOSTNAMES_MAX, registerVerticalInput, vertical as verticalSchema, verticalChannel, verticalVersion, objectRef, entityObjectRef, assertModuleEmittableType, assertKernelAuthoredType, grantRefFromProof, org as orgSchema, orgMembership, principalId, resolvedIdentity, identityMembership, roleDefinition, scope as scopeSchema, tenant as tenantSchema, tenantRole, subjectShredReceipt, SCOPE_TABLE_PAGE_DEFAULT, SCOPE_TABLE_PAGE_MAX, SCOPE_QUERY_ROW_MAX, verticalServingState, callsOfManifestJson, outboundOfManifestJson, listLimitOf, substratError, assertReplayableDump, delegatedReadRecord, ownerTransferAudit, copyMarkAudit, redrainEventsInput, REDRAIN_BATCH, } from '@substrat-run/contracts';
|
|
5
|
+
import { actorOf, asPrincipal, assertAllowed, assertNoSecret, assertPermissionKey, CAPABILITY_DDL, COPY_ORIGIN_DDL, CAPABILITY_EXCHANGE_OPERATION, capabilityAttachmentWriteRefused, capabilityTokenHash, createCapabilityVerbs, createEntityEdgeVerbs, exchangeCapability as exchangeCapabilitySecret, guardSecrets, mintBecomeCapability, plausibleSessionToken, readCapabilityPage, redactSecrets, redactSecretText, moduleLog, consoleLogSink, asyncInvocationId, asyncLinePass, consoleInvocationLineSink, resolveCapabilitySession, revokeCapabilityAsPlatform, assertReadOnlyQuery, attachmentBlobKey, attachmentSha256, entitlementDenial, requiredEntitlementFor, foldMeterReading, guardSpine, guardSqlLimits, DO_SQL_LIMITS, tooManyResultColumns, tooManyTableColumns, parseValidationRecords, resolveScopeRecord, ulid, ISSUE_RETENTION_DAYS, telemetryRetentionStatements, assertRowLimit, assertRowOffset, OPS_FAILURE_RETENTION_DAYS, SWEEP_RUN_RETENTION_DAYS, SWEEP_RUNS_INTENT_INDEX, sweepRunsIntentHasKind, MODEL_USAGE_RETENTION_DAYS, foldModelUsage, backoffAt, platformRequestHistoryQuery, platformRequestOf, PLATFORM_REQUEST_COLUMNS, PLATFORM_REQUEST_REDACTION_SQL, platformRequestRedactionParams, platformRequestRedactionQuery, intentPayloadCarriesSubject, redactSubjectJobRuns, redactSubjectScopeText, redactSubjectDirectoryText, ISSUE_EXEMPLAR_OWNER_BACKFILL_SQL, issueExemplarOwner, JOB_RUN_PATCH_SQL, JOB_STEP_RECORD_SQL, DELIVERY_ERROR_REDACTION_SQL, REDACTED_DELIVERY_NOTE, seatScopeTuple, switchSystemSchedules, admitPeer, collectPeers, peerSeats, resolveVerticalInstanceFrom, switchPeer, peerGrantsStatus, systemGrantsStatus, systemScheduleState, systemSwitchedOff, systemSwitchedOffMessage, tenantSystemSwitchedOffMessage, SYSTEM_SWITCHES_BACKFILL_SQL, SYSTEM_SWITCHES_DDL, forgetSystemSwitchesOf, listSystemSwitchRecords, recordSystemSwitchedOff, recordSystemSwitchedOn, restoreSystemSwitchRecord, scopesSwitchedOffFor, tenantHoldsSystemGrant, switchedOffModulesOf, inUnitMovesToAudit, staleCarryRevertRow, staleCarryReverts, systemSwitchRecordsOf, systemSwitchesTableExists, withRecorded, VERSION_MIGRATIONS_DDL, splitManifestMigrations, splitVersionMigrationsBatch, versionMigrationsOf, writeVersionMigrations, denialListQuery, refusalListQuery, mapRefusalRow, denialSummaryQuery, denialTotalsQuery, DENIAL_WINDOW_QUERY, mapDenialRow, mapDenialSummaryBuckets, IMPERSONATION_COLUMNS, IMPERSONATION_DDL, ImpersonationRefused, assertImpersonationWrites, assertModuleEnqueueableKind, assertSessionUsable, impersonationByIdQuery, impersonationListQuery, impersonationRowValues, impersonationStampOf, mapImpersonationRow, newImpersonationSession, resolveRetryPolicy, isSecretBoxConfigured, unconfiguredSecretBox, createSubjectKeys, generateSealingKeyPair, noSealingKeyMessage, openSealed, sealTo, ConnectionSealingKeyUnavailableError, PermissionDenied, createAtomic, NotSearchable, NotListable, listIndexMigrations, listIndexPlans, listQuery, cursorOf, moduleMigrations, isSearchIndexTable, searchIndexDdl, searchIndexMigrations, searchIndexPlans, searchLimit, searchMatchExpression, searchQuery, ATTACHMENT_TEXT_DDL, assertAttachmentExtractors, assertJobRegistrable, attachmentRecordOfRow, attachmentTextJob, enqueueAttachmentText, isAttachmentTextRun, reconcileAttachmentText, recordAttachmentText, searchAttachments, entityVersionQuery, entityVersionOf, assertIfMatch, OUTBOX_ENTITY_INDEX, SCHEDULE_STATE_DDL, SCHEDULE_STATE_REBUILD, scheduleStateHasKind, VERTICAL_EVENTS_DDL, EXPORT_HOPS_SQL, IMPORT_CURSORS_SQL, IMPORT_CURSOR_OF_SQL, OUTBOX_MARK_SQL, emittedSinceQuery, emittedReportOf, EMITTED_REPORT_CAP, exportedSinceQuery, IMPORT_CURSOR_ADVANCE_SQL, IMPORT_RECORD_SQL, moveImportCursor, importCursorSourceOf, exportBreaksOf, exportBreakRefusal, bindExportBreaksOf, bindExportBreakRefusal, CrossVerticalRegistry, exportReadPlan, exportReadQuery, planExportBatch, withheldNote, JOB_RUN_DDL, JOB_RUN_DUE_AT, jobRunListLimit, jobRunOf, runDueJobRuns, startJobRun, IDEMPOTENCY_DDL, REFUSALS_TABLE_DDL, REFUSALS_INDEX, refusalInsert, refusalOf, markGuardRefusal, REFUSALS_REBUILD, refusalsAdmitGuards, assertIdempotencyKey, idempotencyLookupQuery, idempotencyPruneStatement, idempotencyRecordStatement, idempotencyOptedOutMessage, replayFor, globalFetch, createUlid, facetEvents, readHistory, walkEventCause, walkEventEffects, assertRedrainWindow, readInvocation, readDeadLetters, readLifecycleFlow, domainEventOf, readUndrainedOutbox, undrainedEventsOf, connectorCallRecord, noopConnectorCallRecorder, recordConnectorCall, settleConnectionUse, unknownRoleError, } from '@substrat-run/kernel';
|
|
6
6
|
import { attributedHost } from '@substrat-run/kernel';
|
|
7
|
-
import {
|
|
7
|
+
import { INERT_SCOPE_REASON, isPrimaryScopeRow } from '@substrat-run/kernel';
|
|
8
|
+
import { LEGACY_SCOPE_ROWS_BACKFILL, assertNoSpineReference, assertSpineTablesBuilt, capabilitiesForLoad, dumpRowsInsert, isSpineTable, loadDirectoryDump, repointScopeGrants, settleCopiedWork, emittedHere, spineColumnAdditions } from '@substrat-run/kernel';
|
|
8
9
|
import { ScopeActor } from './actor.js';
|
|
9
10
|
import { createTupleChecker } from './checker.js';
|
|
10
11
|
/**
|
|
@@ -203,11 +204,15 @@ const KERNEL_DDL = `
|
|
|
203
204
|
drained_at TEXT
|
|
204
205
|
);
|
|
205
206
|
${IDEMPOTENCY_DDL}
|
|
206
|
-
${
|
|
207
|
+
${REFUSALS_TABLE_DDL}
|
|
208
|
+
${REFUSALS_INDEX}
|
|
207
209
|
-- #1672: capabilities — authority carried by a secret (a link share), and the sessions
|
|
208
210
|
-- an exchange trades that secret for. Spine (kernel-written), shared with the DO adapter
|
|
209
211
|
-- from @substrat-run/kernel so the two cannot part company; the column comments are there.
|
|
210
212
|
${CAPABILITY_DDL}
|
|
213
|
+
-- #1686: where a copied scope's data came from, and the event id its own events start above.
|
|
214
|
+
-- Shared with the other adapter from @substrat-run/kernel; the column comments are there.
|
|
215
|
+
${COPY_ORIGIN_DDL}
|
|
211
216
|
-- #383 / #1232 / #1288: the platform sweep's per-scope gating state, holding two
|
|
212
217
|
-- families of row that the kind COLUMN — not the spelling of a key — tells apart.
|
|
213
218
|
-- Spine (kernel-written), never a module migration. Shared with the DO adapter from
|
|
@@ -418,11 +423,12 @@ function modelUsageEntryOf(r) {
|
|
|
418
423
|
});
|
|
419
424
|
}
|
|
420
425
|
/**
|
|
421
|
-
* The policy
|
|
422
|
-
*
|
|
423
|
-
*
|
|
426
|
+
* The policy a delivery that can never succeed is journaled under: one attempt, so the first
|
|
427
|
+
* is the dead letter, and the Cloudflare coordinator records the same row with no next
|
|
428
|
+
* attempt. An undecodable event (#1636) — a decode is pure, so a retry cannot succeed — and an
|
|
429
|
+
* inert scope's delivery (#2005) — a scope does not become primary, so a retry decides the same.
|
|
424
430
|
*/
|
|
425
|
-
const
|
|
431
|
+
const TERMINAL_RETRY = { maxAttempts: 1, baseDelayMs: 0, maxDelayMs: 0 };
|
|
426
432
|
/**
|
|
427
433
|
* The dispatch capability (#726 remedy B): a connector delivery may read the attachments
|
|
428
434
|
* of the entity the DELIVERED EVENT names, and nothing else.
|
|
@@ -572,6 +578,8 @@ export class SqliteScopeHost {
|
|
|
572
578
|
versionId;
|
|
573
579
|
/** #1746: where `ctx.log` lines go — the console unless a caller (a test) says otherwise. */
|
|
574
580
|
logSink;
|
|
581
|
+
/** #1901: where async work's invocation lines go. */
|
|
582
|
+
invocationLineSink;
|
|
575
583
|
// The mint for event ids (#956) is NOT here: it lives on `ScopeRuntime`, one per
|
|
576
584
|
// scope, seeded from that scope's persisted maximum (#1335). Its own monotonic
|
|
577
585
|
// floor, because the timestamp it stamps comes from `this.clock` — a scripted
|
|
@@ -586,6 +594,7 @@ export class SqliteScopeHost {
|
|
|
586
594
|
this.clock = options.clock ?? (() => instant.parse(new Date().toISOString()));
|
|
587
595
|
this.versionId = options.versionId ?? null;
|
|
588
596
|
this.logSink = options.logSink ?? consoleLogSink;
|
|
597
|
+
this.invocationLineSink = options.invocationLineSink ?? consoleInvocationLineSink;
|
|
589
598
|
this.dir = options.dir;
|
|
590
599
|
mkdirSync(this.dir, { recursive: true });
|
|
591
600
|
this.directory = new Database(join(this.dir, '_directory.sqlite'));
|
|
@@ -1659,7 +1668,8 @@ export class SqliteScopeHost {
|
|
|
1659
1668
|
* `HostAdmin.reassertSystemSwitches`' body (#1674), run inside a turn the caller already
|
|
1660
1669
|
* holds on the scope's actor: every module the directory records OFF on the scope is
|
|
1661
1670
|
* switched off again. Already off answers `changed: false` and writes nothing, and a
|
|
1662
|
-
* module
|
|
1671
|
+
* module with no authority reaching the scope (no grant here, no tenant-level grant) is left
|
|
1672
|
+
* alone. Audited only when something moved.
|
|
1663
1673
|
*
|
|
1664
1674
|
* `audit` receives each row. By default it writes to the admin log at once. A caller running
|
|
1665
1675
|
* this inside a scope transaction that may still roll back (a restore's replay) collects
|
|
@@ -1686,7 +1696,9 @@ export class SqliteScopeHost {
|
|
|
1686
1696
|
for (const row of inUnitMovesToAudit(recordedOff, applied))
|
|
1687
1697
|
audit({ operationId: ulid(), ...row });
|
|
1688
1698
|
return recordedOff.map((moduleId) => {
|
|
1689
|
-
|
|
1699
|
+
// #1823: a module whose only authority here is a tenant-level grant is held too.
|
|
1700
|
+
const tenantHeld = tenantHoldsSystemGrant(switchSqlOf(this.directory), tenantId, moduleId, this.clock());
|
|
1701
|
+
const outcome = rt.db.transaction(() => switchSystemSchedules(switchSqlOf(rt.db), { moduleId, scopeId, to: 'off', at, tenantHeld }))();
|
|
1690
1702
|
if (outcome.changed) {
|
|
1691
1703
|
audit({
|
|
1692
1704
|
operationId: ulid(),
|
|
@@ -2246,6 +2258,10 @@ export class SqliteScopeHost {
|
|
|
2246
2258
|
// #1575: attachment text is not in a dump, so a load left it as it was. Drop the
|
|
2247
2259
|
// text of attachments the dump did not bring back, and queue extraction for those
|
|
2248
2260
|
// it brought back without text — the bytes decide what that run finds.
|
|
2261
|
+
const switchSql = switchSqlOf(db);
|
|
2262
|
+
// #1686: nothing the source queued runs in a copy; a return leaves it all queued. BEFORE
|
|
2263
|
+
// the extraction queue below, which is this scope's own work, not the source's.
|
|
2264
|
+
settleCopiedWork(switchSql, scopeId, dump.scopeId, this.clock());
|
|
2249
2265
|
reconcileAttachmentText(spineSql(db), ulid, this.clock());
|
|
2250
2266
|
// Re-point scope-level grants at the scope they now live in. They are written as
|
|
2251
2267
|
// `object = scope:<scopeId>`, so a fork, a restore into a different scope, or #286's
|
|
@@ -2257,7 +2273,7 @@ export class SqliteScopeHost {
|
|
|
2257
2273
|
// (subject, relation, object) is the primary key, so a moved row can meet one the dump
|
|
2258
2274
|
// already holds here; which one survives is that function's rule too (#1882), judged
|
|
2259
2275
|
// at this host's clock as the checker judges expiry.
|
|
2260
|
-
repointScopeGrants(
|
|
2276
|
+
repointScopeGrants(switchSql, scopeId, { scopeId: dump.scopeId, exact: dump.exact }, this.clock());
|
|
2261
2277
|
// #1742: inside the replay's transaction, so a failure here rolls the whole load back and
|
|
2262
2278
|
// the dump's grants never commit without the switch that should cover them.
|
|
2263
2279
|
afterLoad?.(rt);
|
|
@@ -2266,7 +2282,20 @@ export class SqliteScopeHost {
|
|
|
2266
2282
|
// load was a SAVEPOINT inside it, and that invoke's rollback undid the restore after
|
|
2267
2283
|
// this verb had audited it.
|
|
2268
2284
|
await rt.actor.turn(() => {
|
|
2269
|
-
|
|
2285
|
+
// #1686: a copy into another scope id loads no capability rows; a same-scope restore keeps them.
|
|
2286
|
+
load(capabilitiesForLoad(dump.tables, scopeId, dump.scopeId));
|
|
2287
|
+
// #1335 / #1686: ids resume above the outbox the dump brought, as on a reopen, so a copy's
|
|
2288
|
+
// own events sort above every copied one (`emittedHere()`). A top id that is no ULID leaves
|
|
2289
|
+
// the floor where it was.
|
|
2290
|
+
const top = db.prepare('SELECT MAX(id) AS id FROM _substrat_outbox').get()?.id;
|
|
2291
|
+
if (top) {
|
|
2292
|
+
try {
|
|
2293
|
+
rt.mintEventId.seedFrom(top);
|
|
2294
|
+
}
|
|
2295
|
+
catch {
|
|
2296
|
+
// Not a ULID: nothing this mint could have written, so there is no floor to keep.
|
|
2297
|
+
}
|
|
2298
|
+
}
|
|
2270
2299
|
// The frontier came in with the dump — refresh the cached applied-migration set so
|
|
2271
2300
|
// a later bind/migrate builds on the loaded state, not the previous one.
|
|
2272
2301
|
rt.appliedMigrations.clear();
|
|
@@ -2602,12 +2631,29 @@ export class SqliteScopeHost {
|
|
|
2602
2631
|
const deadLetter = rt.db.prepare(`INSERT OR IGNORE INTO _substrat_deliveries
|
|
2603
2632
|
(event_id, consumer_module, delivered_at, error, invocation_id)
|
|
2604
2633
|
VALUES (?, ?, ?, ?, NULL)`);
|
|
2634
|
+
// #1901: one line per (event, importing module), as for the scope's own consumers. The
|
|
2635
|
+
// handler runs as the producer's peer, so that is who the line says it ran as.
|
|
2636
|
+
const lines = asyncLinePass(this.invocationLineSink);
|
|
2637
|
+
const unitOf = (moduleId, eventId, eventType) => ({
|
|
2638
|
+
kind: 'consumer',
|
|
2639
|
+
tenantId: rt.tenantId,
|
|
2640
|
+
scopeId: rt.scopeId,
|
|
2641
|
+
invocationId: asyncInvocationId(null),
|
|
2642
|
+
operation: moduleId,
|
|
2643
|
+
eventType,
|
|
2644
|
+
eventId,
|
|
2645
|
+
attempt: 1,
|
|
2646
|
+
startedAt: Date.now(),
|
|
2647
|
+
principalKind: subject.kind,
|
|
2648
|
+
versionId: this.versionId,
|
|
2649
|
+
});
|
|
2605
2650
|
// Named and not carried: a dead letter per importing module, so the operator who reads
|
|
2606
2651
|
// this scope's dead letters sees what the producer sent and why it did not arrive.
|
|
2607
2652
|
for (const w of batch.withheld) {
|
|
2608
2653
|
record.run(w.id, source.scopeId, source.vertical, w.type, w.schemaVersion, w.occurredAt, w.entity.entityType, w.entity.entityId, 0, w.reason, at);
|
|
2609
2654
|
for (const moduleId of this.crossVertical.modulesImporting(source.vertical, w.type)) {
|
|
2610
2655
|
deadLetter.run(w.id, moduleId, at, withheldNote(w.reason, source.vertical));
|
|
2656
|
+
lines.write({ ...unitOf(moduleId, w.id, w.type), outcome: 'dead-lettered' });
|
|
2611
2657
|
}
|
|
2612
2658
|
result.withheld += 1;
|
|
2613
2659
|
}
|
|
@@ -2623,14 +2669,17 @@ export class SqliteScopeHost {
|
|
|
2623
2669
|
// this scope's code changed between the two, so it is refused by version (K-39)
|
|
2624
2670
|
// exactly as the producer would have refused it.
|
|
2625
2671
|
deadLetter.run(e.id, imp.moduleId, at, withheldNote('version', source.vertical));
|
|
2672
|
+
lines.write({ ...unitOf(imp.moduleId, e.id, e.type), outcome: 'dead-lettered' });
|
|
2626
2673
|
result.deadLettered += 1;
|
|
2627
2674
|
continue;
|
|
2628
2675
|
}
|
|
2629
2676
|
const { hops: _hops, ...fact } = e;
|
|
2630
2677
|
const event = structuredClone({ ...fact, source });
|
|
2678
|
+
const unit = unitOf(imp.moduleId, e.id, e.type);
|
|
2631
2679
|
// What the handler emits was emitted BECAUSE of the producer's event (#1237). The id
|
|
2632
2680
|
// resolves through `_substrat_imports`, which the row above has just written.
|
|
2633
2681
|
rt.causedBy = e.id;
|
|
2682
|
+
rt.unitInvocationId = unit.invocationId;
|
|
2634
2683
|
rt.db.exec('BEGIN IMMEDIATE');
|
|
2635
2684
|
try {
|
|
2636
2685
|
// As the producer's principal, admitted above: the handler's checks are real ones,
|
|
@@ -2643,6 +2692,7 @@ export class SqliteScopeHost {
|
|
|
2643
2692
|
.run(e.id, imp.moduleId, new Date().toISOString());
|
|
2644
2693
|
rt.db.exec('COMMIT');
|
|
2645
2694
|
result.delivered += 1;
|
|
2695
|
+
lines.write({ ...unit, outcome: 'delivered' });
|
|
2646
2696
|
}
|
|
2647
2697
|
catch (err) {
|
|
2648
2698
|
rt.db.exec('ROLLBACK');
|
|
@@ -2650,9 +2700,11 @@ export class SqliteScopeHost {
|
|
|
2650
2700
|
// behind it still arrive.
|
|
2651
2701
|
deadLetter.run(e.id, imp.moduleId, new Date().toISOString(), String(err));
|
|
2652
2702
|
result.deadLettered += 1;
|
|
2703
|
+
lines.write({ ...unit, outcome: 'dead-lettered', error: err });
|
|
2653
2704
|
}
|
|
2654
2705
|
finally {
|
|
2655
2706
|
rt.causedBy = null;
|
|
2707
|
+
rt.unitInvocationId = null;
|
|
2656
2708
|
}
|
|
2657
2709
|
}
|
|
2658
2710
|
if (!ran)
|
|
@@ -2662,6 +2714,7 @@ export class SqliteScopeHost {
|
|
|
2662
2714
|
// rows above absorb it.
|
|
2663
2715
|
rt.db.prepare(IMPORT_CURSOR_ADVANCE_SQL).run(source.scopeId, source.vertical, batch.next, at);
|
|
2664
2716
|
result.cursor = batch.next;
|
|
2717
|
+
lines.end();
|
|
2665
2718
|
// What the handlers emitted reaches this scope's own consumers and executors in the same
|
|
2666
2719
|
// tail, under the ordinary cascade cap.
|
|
2667
2720
|
await this.dispatch(rt, null);
|
|
@@ -3061,6 +3114,7 @@ export class SqliteScopeHost {
|
|
|
3061
3114
|
// ended); the freshness evaluator's are `kind = 'freshness'` — see checkFreshness
|
|
3062
3115
|
// for what the same two columns mean there.
|
|
3063
3116
|
const now = Date.parse(nowIso);
|
|
3117
|
+
const lines = asyncLinePass(this.invocationLineSink);
|
|
3064
3118
|
for (const schedule of mod.schedules) {
|
|
3065
3119
|
const lastRunAt = gate.lastRuns.get(schedule.operation) ?? null;
|
|
3066
3120
|
const lastRun = lastRunAt ? Date.parse(lastRunAt) : null;
|
|
@@ -3077,19 +3131,39 @@ export class SqliteScopeHost {
|
|
|
3077
3131
|
// produces either way carries it, and so does the row below.
|
|
3078
3132
|
const invocationId = ulid();
|
|
3079
3133
|
let status = 'ok';
|
|
3134
|
+
// #1901: the run's line, under the call's own id — the one its `ctx.log` lines carry.
|
|
3135
|
+
const startedAt = Date.now();
|
|
3136
|
+
let emitted;
|
|
3137
|
+
let failure;
|
|
3080
3138
|
try {
|
|
3081
3139
|
const stub = await this.getSystemScope(moduleId, tenantId, scopeId);
|
|
3082
|
-
await stub.invoke(schedule.operation, schedule.input, { invocationId });
|
|
3140
|
+
await stub.invoke(schedule.operation, schedule.input, { invocationId, onEmitted: (r) => (emitted = r) });
|
|
3083
3141
|
report.fired += 1;
|
|
3084
3142
|
}
|
|
3085
3143
|
catch (err) {
|
|
3086
3144
|
status = 'failed';
|
|
3145
|
+
failure = { error: err };
|
|
3087
3146
|
report.failed += 1;
|
|
3088
3147
|
report.errors.push({
|
|
3089
3148
|
operation: schedule.operation,
|
|
3090
3149
|
error: err instanceof Error ? err.message : String(err),
|
|
3091
3150
|
});
|
|
3092
3151
|
}
|
|
3152
|
+
lines.write({
|
|
3153
|
+
kind: 'schedule',
|
|
3154
|
+
tenantId,
|
|
3155
|
+
scopeId,
|
|
3156
|
+
invocationId,
|
|
3157
|
+
operation: schedule.operation,
|
|
3158
|
+
startedAt,
|
|
3159
|
+
outcome: status,
|
|
3160
|
+
...failure,
|
|
3161
|
+
// Against the host's clock, which is what decided the run was due.
|
|
3162
|
+
dueAt: lastRun === null ? null : new Date(dueAt).toISOString(),
|
|
3163
|
+
latenessMs: lastRun === null ? null : now - dueAt,
|
|
3164
|
+
...(emitted ? { emitted } : {}),
|
|
3165
|
+
versionId: this.versionId,
|
|
3166
|
+
});
|
|
3093
3167
|
// On the scope actor (#1678): the cadence row joined whatever transaction an invoke
|
|
3094
3168
|
// held open, so its rollback forgot the run and the next pass fired it AGAIN.
|
|
3095
3169
|
await rt.actor.turn(() => rt.db
|
|
@@ -3101,6 +3175,7 @@ export class SqliteScopeHost {
|
|
|
3101
3175
|
.run(schedule.operation, new Date(now).toISOString(), status, invocationId));
|
|
3102
3176
|
report.runs.push({ operation: schedule.operation, outcome: status === 'ok' ? 'ok' : 'failed' });
|
|
3103
3177
|
}
|
|
3178
|
+
lines.end();
|
|
3104
3179
|
return report;
|
|
3105
3180
|
}
|
|
3106
3181
|
/** The stub body, shared by the principal, connection, system, impersonation and capability doors. */
|
|
@@ -3448,7 +3523,15 @@ export class SqliteScopeHost {
|
|
|
3448
3523
|
throw new Error(`unknown guard predicate: '${guard.predicate}' — declared by ${guard.declaredBy} ` +
|
|
3449
3524
|
`before '${operation}'; no registered module contributes it (operation blocked)`);
|
|
3450
3525
|
}
|
|
3451
|
-
|
|
3526
|
+
try {
|
|
3527
|
+
await predicate.handler(ctx, guard.config, input);
|
|
3528
|
+
}
|
|
3529
|
+
catch (err) {
|
|
3530
|
+
// #1745: a guard refusing is recorded after the rollback, as a transition is —
|
|
3531
|
+
// marked here, where which guard threw is still known. Always rethrown.
|
|
3532
|
+
markGuardRefusal(err, guard.predicate, operation);
|
|
3533
|
+
throw err;
|
|
3534
|
+
}
|
|
3452
3535
|
}
|
|
3453
3536
|
}
|
|
3454
3537
|
async close() {
|
|
@@ -3496,7 +3579,26 @@ export class SqliteScopeHost {
|
|
|
3496
3579
|
retrying: 0,
|
|
3497
3580
|
deadLettered: 0,
|
|
3498
3581
|
};
|
|
3582
|
+
if (this.executors.size === 0)
|
|
3583
|
+
return report;
|
|
3584
|
+
const lines = asyncLinePass(this.invocationLineSink);
|
|
3585
|
+
try {
|
|
3586
|
+
await this.dispatchExecutorsPass(rt, invocationId, report, lines);
|
|
3587
|
+
}
|
|
3588
|
+
finally {
|
|
3589
|
+
lines.end();
|
|
3590
|
+
}
|
|
3591
|
+
return report;
|
|
3592
|
+
}
|
|
3593
|
+
async dispatchExecutorsPass(rt, invocationId, report, lines) {
|
|
3499
3594
|
const now = new Date().toISOString();
|
|
3595
|
+
// #2005: a non-primary scope (a fork, a snapshot, a preview of either kind) causes no
|
|
3596
|
+
// outbound effects. Every executor is host code acting with platform authority — a
|
|
3597
|
+
// connector with the tenant's credential, a plain executor with `HostAdmin` — so its
|
|
3598
|
+
// deliveries are journaled terminal with the reason and the handler never runs.
|
|
3599
|
+
// Asked on the first due event, once per pass: most passes have none.
|
|
3600
|
+
let inert;
|
|
3601
|
+
const isInert = () => (inert ??= !this.isPrimaryInDirectory(rt.scopeId));
|
|
3500
3602
|
for (const [id, executor] of this.executors) {
|
|
3501
3603
|
const deliveryId = `executor:${id}`;
|
|
3502
3604
|
// Due = never attempted, or retrying and past its next attempt time.
|
|
@@ -3505,13 +3607,26 @@ export class SqliteScopeHost {
|
|
|
3505
3607
|
.prepare(`SELECT o.* FROM _substrat_outbox o
|
|
3506
3608
|
LEFT JOIN _substrat_deliveries d
|
|
3507
3609
|
ON d.event_id = o.id AND d.consumer_module = ?
|
|
3508
|
-
WHERE o.type = ?
|
|
3610
|
+
WHERE o.type = ? AND ${emittedHere('o.')}
|
|
3509
3611
|
AND (d.event_id IS NULL
|
|
3510
3612
|
OR (d.next_attempt_at IS NOT NULL AND d.next_attempt_at <= ?))
|
|
3511
3613
|
ORDER BY o.id`)
|
|
3512
3614
|
.all(deliveryId, executor.eventType, now);
|
|
3513
3615
|
for (const row of rows) {
|
|
3514
3616
|
report.attempted += 1;
|
|
3617
|
+
// #1901: this attempt's line. Its number is the one the journal records, which
|
|
3618
|
+
// `recordExecutorDelivery` answers — the Durable Object's RPC does the same.
|
|
3619
|
+
const unit = {
|
|
3620
|
+
kind: 'consumer',
|
|
3621
|
+
tenantId: rt.tenantId,
|
|
3622
|
+
scopeId: rt.scopeId,
|
|
3623
|
+
invocationId: asyncInvocationId(invocationId),
|
|
3624
|
+
operation: deliveryId,
|
|
3625
|
+
eventType: executor.eventType,
|
|
3626
|
+
eventId: row.id,
|
|
3627
|
+
startedAt: Date.now(),
|
|
3628
|
+
versionId: this.versionId,
|
|
3629
|
+
};
|
|
3515
3630
|
let event;
|
|
3516
3631
|
try {
|
|
3517
3632
|
event = domainEventOf(row);
|
|
@@ -3523,8 +3638,16 @@ export class SqliteScopeHost {
|
|
|
3523
3638
|
// on the FIRST failure, unlike a handler's: the decode is pure, of text already
|
|
3524
3639
|
// in hand, so a retry cannot succeed — and only the decode is caught here, so a
|
|
3525
3640
|
// handler's transient failure keeps its backoff below.
|
|
3526
|
-
this.recordExecutorDelivery(rt, row.id, deliveryId, String(err),
|
|
3641
|
+
const { attempts } = this.recordExecutorDelivery(rt, row.id, deliveryId, String(err), TERMINAL_RETRY, invocationId);
|
|
3527
3642
|
report.deadLettered += 1;
|
|
3643
|
+
lines.write({ ...unit, attempt: attempts, outcome: 'dead-lettered' });
|
|
3644
|
+
continue;
|
|
3645
|
+
}
|
|
3646
|
+
if (isInert()) {
|
|
3647
|
+
const { attempts } = this.recordExecutorDelivery(rt, row.id, deliveryId, INERT_SCOPE_REASON, TERMINAL_RETRY, invocationId);
|
|
3648
|
+
report.inert = (report.inert ?? 0) + 1;
|
|
3649
|
+
// Its own outcome, never `delivered`: no handler ran (#2005).
|
|
3650
|
+
lines.write({ ...unit, attempt: attempts, outcome: 'inert' });
|
|
3528
3651
|
continue;
|
|
3529
3652
|
}
|
|
3530
3653
|
this.causedBy = event.id;
|
|
@@ -3537,26 +3660,27 @@ export class SqliteScopeHost {
|
|
|
3537
3660
|
else {
|
|
3538
3661
|
await executor.handler(this.admin, event);
|
|
3539
3662
|
}
|
|
3540
|
-
this.recordExecutorDelivery(rt, row.id, deliveryId, null, executor.retry, invocationId);
|
|
3663
|
+
const { attempts } = this.recordExecutorDelivery(rt, row.id, deliveryId, null, executor.retry, invocationId);
|
|
3541
3664
|
report.delivered += 1;
|
|
3665
|
+
lines.write({ ...unit, attempt: attempts, outcome: 'delivered' });
|
|
3542
3666
|
}
|
|
3543
3667
|
catch (err) {
|
|
3544
|
-
const dead = this.recordExecutorDelivery(rt, row.id, deliveryId, err instanceof Error ? (err.stack ?? err.message) : String(err), executor.retry, invocationId);
|
|
3668
|
+
const { dead, attempts } = this.recordExecutorDelivery(rt, row.id, deliveryId, err instanceof Error ? (err.stack ?? err.message) : String(err), executor.retry, invocationId);
|
|
3545
3669
|
if (dead)
|
|
3546
3670
|
report.deadLettered += 1;
|
|
3547
3671
|
else
|
|
3548
3672
|
report.retrying += 1;
|
|
3673
|
+
lines.write({ ...unit, attempt: attempts, outcome: dead ? 'dead-lettered' : 'retrying', error: err });
|
|
3549
3674
|
}
|
|
3550
3675
|
finally {
|
|
3551
3676
|
this.causedBy = null;
|
|
3552
3677
|
}
|
|
3553
3678
|
}
|
|
3554
3679
|
}
|
|
3555
|
-
return report;
|
|
3556
3680
|
}
|
|
3557
3681
|
/**
|
|
3558
|
-
* Journal one executor attempt.
|
|
3559
|
-
* — i.e. the delivery is now dead-lettered.
|
|
3682
|
+
* Journal one executor attempt. Answers which attempt it was (#1901), and `dead` when
|
|
3683
|
+
* it was the last one — i.e. the delivery is now dead-lettered.
|
|
3560
3684
|
*
|
|
3561
3685
|
* Written AFTER the handler ran, so a crash mid-effect retries rather than
|
|
3562
3686
|
* silently marking success. Claiming first would make delivery at-most-once and
|
|
@@ -3589,7 +3713,11 @@ export class SqliteScopeHost {
|
|
|
3589
3713
|
.run(eventId, deliveryId, new Date().toISOString(), error, attempts, nextAttemptAt,
|
|
3590
3714
|
// #1525: the call this attempt ran in, as the caller named it.
|
|
3591
3715
|
invocationId);
|
|
3592
|
-
return error !== null && exhausted;
|
|
3716
|
+
return { dead: error !== null && exhausted, attempts };
|
|
3717
|
+
}
|
|
3718
|
+
/** Whether the directory says `scopeId` is the real install (`isPrimaryScopeRow`, #2005). */
|
|
3719
|
+
isPrimaryInDirectory(scopeId) {
|
|
3720
|
+
return isPrimaryScopeRow(this.directory.prepare('SELECT kind, forked_from FROM scopes WHERE scope_id = ?').get(scopeId));
|
|
3593
3721
|
}
|
|
3594
3722
|
async drainDue(tenantId, scopeId) {
|
|
3595
3723
|
const rt = this.runtime(tenantId, scopeId);
|
|
@@ -3665,12 +3793,12 @@ export class SqliteScopeHost {
|
|
|
3665
3793
|
return {
|
|
3666
3794
|
startOrJoin: (key, r) => turn(() => startOrJoinTx(key, r)),
|
|
3667
3795
|
get: (id) => turn(() => row(db.prepare('SELECT * FROM _substrat_job_runs WHERE id = ?').get(id))),
|
|
3668
|
-
|
|
3669
|
-
|
|
3796
|
+
// #1834: the drive's one snapshot — keys only, in the order runs became due.
|
|
3797
|
+
dueKeys: (now, max) => turn(() => db
|
|
3798
|
+
.prepare(`SELECT id, module_id, job FROM _substrat_job_runs
|
|
3670
3799
|
WHERE status = 'running' AND (next_attempt_at IS NULL OR next_attempt_at <= ?)
|
|
3671
|
-
|
|
3672
|
-
|
|
3673
|
-
.all(now, afterId ?? null, afterId ?? null, limit)),
|
|
3800
|
+
ORDER BY ${JOB_RUN_DUE_AT}, id LIMIT ?`)
|
|
3801
|
+
.all(now, max)),
|
|
3674
3802
|
list: (filter) => turn(() => {
|
|
3675
3803
|
const where = [];
|
|
3676
3804
|
const params = [];
|
|
@@ -3929,13 +4057,23 @@ export class SqliteScopeHost {
|
|
|
3929
4057
|
* own emit is delivered in the same tail, so the whole cascade is one call's work.
|
|
3930
4058
|
*/
|
|
3931
4059
|
invocationId) {
|
|
4060
|
+
// #1901: one line per delivery, capped per drain.
|
|
4061
|
+
const lines = asyncLinePass(this.invocationLineSink);
|
|
4062
|
+
try {
|
|
4063
|
+
await this.dispatchRounds(rt, invocationId, lines);
|
|
4064
|
+
}
|
|
4065
|
+
finally {
|
|
4066
|
+
lines.end();
|
|
4067
|
+
}
|
|
4068
|
+
}
|
|
4069
|
+
async dispatchRounds(rt, invocationId, lines) {
|
|
3932
4070
|
for (let round = 0; round < 50; round++) {
|
|
3933
4071
|
let deliveredAny = false;
|
|
3934
4072
|
for (const mod of this.modules.values()) {
|
|
3935
4073
|
for (const consumer of mod.consumers) {
|
|
3936
4074
|
const rows = rt.db
|
|
3937
4075
|
.prepare(`SELECT * FROM _substrat_outbox o
|
|
3938
|
-
WHERE o.type = ?
|
|
4076
|
+
WHERE o.type = ? AND ${emittedHere('o.')}
|
|
3939
4077
|
AND NOT EXISTS (
|
|
3940
4078
|
SELECT 1 FROM _substrat_deliveries d
|
|
3941
4079
|
WHERE d.event_id = o.id AND d.consumer_module = ?
|
|
@@ -3943,6 +4081,20 @@ export class SqliteScopeHost {
|
|
|
3943
4081
|
ORDER BY o.id`)
|
|
3944
4082
|
.all(consumer.eventType, mod.id);
|
|
3945
4083
|
for (const row of rows) {
|
|
4084
|
+
// #1901: this delivery's line — the call's id when it runs in one, else its own.
|
|
4085
|
+
const unit = {
|
|
4086
|
+
kind: 'consumer',
|
|
4087
|
+
tenantId: rt.tenantId,
|
|
4088
|
+
scopeId: rt.scopeId,
|
|
4089
|
+
invocationId: asyncInvocationId(invocationId),
|
|
4090
|
+
operation: mod.id,
|
|
4091
|
+
eventType: consumer.eventType,
|
|
4092
|
+
eventId: row.id,
|
|
4093
|
+
// A module consumer is tried once: a throw dead-letters it (v0).
|
|
4094
|
+
attempt: 1,
|
|
4095
|
+
startedAt: Date.now(),
|
|
4096
|
+
versionId: this.versionId,
|
|
4097
|
+
};
|
|
3946
4098
|
let event;
|
|
3947
4099
|
try {
|
|
3948
4100
|
event = domainEventOf(row);
|
|
@@ -3959,6 +4111,8 @@ export class SqliteScopeHost {
|
|
|
3959
4111
|
(event_id, consumer_module, delivered_at, error, invocation_id)
|
|
3960
4112
|
VALUES (?, ?, ?, ?, ?)`)
|
|
3961
4113
|
.run(row.id, mod.id, new Date().toISOString(), String(err), invocationId);
|
|
4114
|
+
// Never reached the handler, so nothing threw in it: a warning, not an error.
|
|
4115
|
+
lines.write({ ...unit, outcome: 'dead-lettered' });
|
|
3962
4116
|
continue;
|
|
3963
4117
|
}
|
|
3964
4118
|
const ctx = this.operationContext(rt, asPrincipal(this.systemPrincipal), {
|
|
@@ -3969,6 +4123,8 @@ export class SqliteScopeHost {
|
|
|
3969
4123
|
// consumers never did, which is exactly where a backwards walk stopped.
|
|
3970
4124
|
// On the runtime rather than the host — see `ScopeRuntime.causedBy`.
|
|
3971
4125
|
rt.causedBy = event.id;
|
|
4126
|
+
// #1901: the handler's `ctx.log` lines join this delivery's line.
|
|
4127
|
+
rt.unitInvocationId = unit.invocationId;
|
|
3972
4128
|
rt.db.exec('BEGIN IMMEDIATE');
|
|
3973
4129
|
try {
|
|
3974
4130
|
await consumer.handler(ctx, event);
|
|
@@ -3979,6 +4135,7 @@ export class SqliteScopeHost {
|
|
|
3979
4135
|
.run(event.id, mod.id, new Date().toISOString(), invocationId);
|
|
3980
4136
|
rt.db.exec('COMMIT');
|
|
3981
4137
|
deliveredAny = true;
|
|
4138
|
+
lines.write({ ...unit, outcome: 'delivered' });
|
|
3982
4139
|
}
|
|
3983
4140
|
catch (err) {
|
|
3984
4141
|
rt.db.exec('ROLLBACK');
|
|
@@ -3989,8 +4146,10 @@ export class SqliteScopeHost {
|
|
|
3989
4146
|
(event_id, consumer_module, delivered_at, error, invocation_id)
|
|
3990
4147
|
VALUES (?, ?, ?, ?, ?)`)
|
|
3991
4148
|
.run(event.id, mod.id, new Date().toISOString(), String(err), invocationId);
|
|
4149
|
+
lines.write({ ...unit, outcome: 'dead-lettered', error: err });
|
|
3992
4150
|
}
|
|
3993
4151
|
finally {
|
|
4152
|
+
rt.unitInvocationId = null;
|
|
3994
4153
|
// Cleared on BOTH paths. Left set, the id would leak onto every later
|
|
3995
4154
|
// emit in this scope — an operation's own event stamped as caused by
|
|
3996
4155
|
// whatever happened to be delivered last, which is worse than no cause
|
|
@@ -4010,11 +4169,12 @@ export class SqliteScopeHost {
|
|
|
4010
4169
|
* point, since the denial is exactly the write the rolled-back operation could not make.
|
|
4011
4170
|
*/
|
|
4012
4171
|
/**
|
|
4013
|
-
* #1745: record a refused transition the operation failed with, after its
|
|
4014
|
-
* denial's discipline (above), for the lifecycle's
|
|
4172
|
+
* #1745: record a refused transition or refusing guard the operation failed with, after its
|
|
4173
|
+
* rollback — the denial's discipline (above), for the lifecycle's and the guards' refusals.
|
|
4174
|
+
* Anything else is not one.
|
|
4015
4175
|
*/
|
|
4016
4176
|
recordRefusal(rt, subject, operation, err, invocationId, impersonation) {
|
|
4017
|
-
const refused =
|
|
4177
|
+
const refused = refusalOf(err);
|
|
4018
4178
|
if (!refused)
|
|
4019
4179
|
return;
|
|
4020
4180
|
const q = refusalInsert({
|
|
@@ -4581,37 +4741,37 @@ export class SqliteScopeHost {
|
|
|
4581
4741
|
const target = { tenantId, scopeId };
|
|
4582
4742
|
const base = { operationId, moduleId: input.moduleId, schedules: to };
|
|
4583
4743
|
this.recordAdmin(actor, action, target, null, { ...base, phase: 'intent', reason: input.reason });
|
|
4584
|
-
// The directory's record (#1674)
|
|
4585
|
-
//
|
|
4744
|
+
// The directory's record (#1674), written BEFORE the scope moves, both ways, and undone
|
|
4745
|
+
// if the move throws or holds nothing — see `recordSystemSwitchedOn` and (#1823)
|
|
4746
|
+
// `recordSystemSwitchedOff` for why that order is the safe one.
|
|
4586
4747
|
const at = new Date().toISOString();
|
|
4587
4748
|
const directorySql = switchSqlOf(this.directory);
|
|
4588
4749
|
const record = { tenantId, scopeId, moduleId: input.moduleId, actor, reason: input.reason, operationId, at };
|
|
4589
4750
|
const errorOf = (err) => (err instanceof Error ? err.message : String(err));
|
|
4590
|
-
let prior
|
|
4591
|
-
|
|
4751
|
+
let prior;
|
|
4752
|
+
try {
|
|
4753
|
+
prior = to === 'on' ? recordSystemSwitchedOn(directorySql, record) : recordSystemSwitchedOff(directorySql, record);
|
|
4754
|
+
}
|
|
4755
|
+
catch (err) {
|
|
4756
|
+
// Nothing has moved: fail the call here, audited — the Cloudflare adapter's posture.
|
|
4592
4757
|
try {
|
|
4593
|
-
|
|
4758
|
+
this.recordAdmin(actor, action, target, null, { ...base, phase: 'failed', error: errorOf(err) });
|
|
4594
4759
|
}
|
|
4595
|
-
catch
|
|
4596
|
-
//
|
|
4597
|
-
try {
|
|
4598
|
-
this.recordAdmin(actor, action, target, null, { ...base, phase: 'failed', error: errorOf(err) });
|
|
4599
|
-
}
|
|
4600
|
-
catch {
|
|
4601
|
-
// Best effort: the original error is what the caller must see.
|
|
4602
|
-
}
|
|
4603
|
-
throw err;
|
|
4760
|
+
catch {
|
|
4761
|
+
// Best effort: the original error is what the caller must see.
|
|
4604
4762
|
}
|
|
4763
|
+
throw err;
|
|
4605
4764
|
}
|
|
4606
|
-
/**
|
|
4607
|
-
const
|
|
4765
|
+
/** The record's undo, after the scope's move threw or held nothing: retried once, never swallowed. */
|
|
4766
|
+
const undoRecord = () => {
|
|
4767
|
+
const undo = () => restoreSystemSwitchRecord(directorySql, record, prior);
|
|
4608
4768
|
try {
|
|
4609
|
-
|
|
4769
|
+
undo();
|
|
4610
4770
|
return null;
|
|
4611
4771
|
}
|
|
4612
4772
|
catch {
|
|
4613
4773
|
try {
|
|
4614
|
-
|
|
4774
|
+
undo();
|
|
4615
4775
|
return null;
|
|
4616
4776
|
}
|
|
4617
4777
|
catch (err) {
|
|
@@ -4626,11 +4786,17 @@ export class SqliteScopeHost {
|
|
|
4626
4786
|
// `db.transaction` issued mid-invoke became a SAVEPOINT inside it and rolled back
|
|
4627
4787
|
// with it — after this verb had already audited `applied`. `turn` is re-entrant, so a
|
|
4628
4788
|
// caller already inside one of this scope's tasks joins it rather than deadlocking.
|
|
4789
|
+
//
|
|
4790
|
+
// #1823: read in the same turn, so no tenant grant moves between the read and the switch.
|
|
4791
|
+
// The directory is this process's own database, so the read is synchronous.
|
|
4629
4792
|
const rt = this.runtime(tenantId, scopeId);
|
|
4630
|
-
outcome = await rt.actor.turn(() =>
|
|
4793
|
+
outcome = await rt.actor.turn(() => {
|
|
4794
|
+
const tenantHeld = tenantHoldsSystemGrant(directorySql, tenantId, input.moduleId, this.clock());
|
|
4795
|
+
return rt.db.transaction(() => switchSystemSchedules(switchSqlOf(rt.db), { moduleId: input.moduleId, scopeId, to, at, tenantHeld }))();
|
|
4796
|
+
});
|
|
4631
4797
|
}
|
|
4632
4798
|
catch (err) {
|
|
4633
|
-
const recordError =
|
|
4799
|
+
const recordError = undoRecord();
|
|
4634
4800
|
try {
|
|
4635
4801
|
this.recordAdmin(actor, action, target, null, {
|
|
4636
4802
|
...base,
|
|
@@ -4644,14 +4810,11 @@ export class SqliteScopeHost {
|
|
|
4644
4810
|
}
|
|
4645
4811
|
throw err;
|
|
4646
4812
|
}
|
|
4647
|
-
//
|
|
4648
|
-
//
|
|
4649
|
-
// module
|
|
4650
|
-
|
|
4651
|
-
|
|
4652
|
-
: to === 'on' && !outcome.held
|
|
4653
|
-
? recordWrite(() => restoreSystemSwitchRecord(directorySql, record, prior))
|
|
4654
|
-
: null;
|
|
4813
|
+
// A call that held nothing moved nothing (the module has no authority reaching the scope,
|
|
4814
|
+
// its own or the tenant's), so its record write is undone: left `off`, it
|
|
4815
|
+
// would switch the module off the day it is installed; left `on`, the next reconcile of
|
|
4816
|
+
// a wiped scope would leave the module running.
|
|
4817
|
+
const recordError = outcome.held ? null : undoRecord();
|
|
4655
4818
|
this.recordAdmin(actor, action, target, null, {
|
|
4656
4819
|
...base,
|
|
4657
4820
|
phase: outcome.held ? 'applied' : 'refused',
|
|
@@ -4664,11 +4827,6 @@ export class SqliteScopeHost {
|
|
|
4664
4827
|
`(check the module id: it is the module's manifest id, e.g. '@substrat-run/engine-absence')` +
|
|
4665
4828
|
(recordError ? `; and its directory record could not be put back (${recordError})` : ''));
|
|
4666
4829
|
}
|
|
4667
|
-
if (recordError) {
|
|
4668
|
-
throw substratError('unavailable', `module '${input.moduleId}' is switched ${to} on scope ${scopeId}, but the directory's record of it ` +
|
|
4669
|
-
`could not be written (${recordError}) — repeat the call to record it; a wipe of this scope would ` +
|
|
4670
|
-
`otherwise lose the switch`);
|
|
4671
|
-
}
|
|
4672
4830
|
return {
|
|
4673
4831
|
operationId,
|
|
4674
4832
|
moduleId: input.moduleId,
|
|
@@ -5053,6 +5211,12 @@ export class SqliteScopeHost {
|
|
|
5053
5211
|
this.recordAccess(actor, 'listSystemSwitches', { tenantId: filter?.tenantId ?? null }, filter ?? null, rows.length);
|
|
5054
5212
|
return rows.map((r) => systemSwitchRecord.parse(r));
|
|
5055
5213
|
},
|
|
5214
|
+
tenantHeldSystemModules: async (actor, tenantId, moduleIds) => {
|
|
5215
|
+
const now = this.clock();
|
|
5216
|
+
const held = moduleIds.filter((m) => tenantHoldsSystemGrant(switchSqlOf(this.directory), tenantId, m, now));
|
|
5217
|
+
this.recordAccess(actor, 'tenantHeldSystemModules', { tenantId }, { moduleIds: [...moduleIds] }, held.length);
|
|
5218
|
+
return held;
|
|
5219
|
+
},
|
|
5056
5220
|
reassertSystemSwitches: async (actor, node, opts) => {
|
|
5057
5221
|
this.assertScope(node.tenantId, node.scopeId);
|
|
5058
5222
|
const rt = this.runtime(node.tenantId, node.scopeId);
|
|
@@ -5270,7 +5434,16 @@ export class SqliteScopeHost {
|
|
|
5270
5434
|
COALESCE(s.serving_ref, vv.deployment_ref) AS deployment_ref,
|
|
5271
5435
|
CASE WHEN s.serving_ref IS NOT NULL
|
|
5272
5436
|
THEN json_extract(sv.manifest_json, '$.outbound')
|
|
5273
|
-
ELSE json_extract(vv.manifest_json, '$.outbound') END AS outbound_json
|
|
5437
|
+
ELSE json_extract(vv.manifest_json, '$.outbound') END AS outbound_json,
|
|
5438
|
+
s.kind, s.forked_from,
|
|
5439
|
+
-- #2005: the scope's own addresses, bounded as the Cloudflare read bounds them.
|
|
5440
|
+
(SELECT json_group_array(sh.hostname) FROM (
|
|
5441
|
+
-- Pinned like the Cloudflare read: a range on the scope, never every
|
|
5442
|
+
-- active hostname through hostnames_status.
|
|
5443
|
+
SELECT hostname FROM hostnames INDEXED BY hostnames_scope
|
|
5444
|
+
WHERE scope_id = s.scope_id AND status = 'active'
|
|
5445
|
+
ORDER BY hostname LIMIT ${ROUTE_SCOPE_HOSTNAMES_MAX}
|
|
5446
|
+
) sh) AS scope_hostnames_json
|
|
5274
5447
|
FROM hostnames h
|
|
5275
5448
|
JOIN scopes s ON s.scope_id = h.scope_id AND s.tenant_id = h.tenant_id
|
|
5276
5449
|
JOIN tenants t ON t.tenant_id = s.tenant_id
|
|
@@ -5302,6 +5475,9 @@ export class SqliteScopeHost {
|
|
|
5302
5475
|
surface: r.surface,
|
|
5303
5476
|
region: r.region,
|
|
5304
5477
|
outboundHosts,
|
|
5478
|
+
// #2005: the egress worker holds a fork or a preview to no third-party egress.
|
|
5479
|
+
primary: isPrimaryScopeRow(r),
|
|
5480
|
+
hostnames: JSON.parse(r.scope_hostnames_json),
|
|
5305
5481
|
});
|
|
5306
5482
|
},
|
|
5307
5483
|
registerVertical: async (actor, input) => {
|
|
@@ -5828,6 +6004,15 @@ export class SqliteScopeHost {
|
|
|
5828
6004
|
.run(servingRef, scopeId);
|
|
5829
6005
|
this.recordAdmin(actor, 'setScopeServingRef', { tenantId, scopeId }, { servingRef: scope.serving_ref }, { servingRef, ...(ack.exportBreak ? { acknowledged: ack } : {}) });
|
|
5830
6006
|
},
|
|
6007
|
+
recordKeptCopyResolution: async (actor, tenantId, scopeId, r) => {
|
|
6008
|
+
const scope = this.directory
|
|
6009
|
+
.prepare('SELECT tenant_id FROM scopes WHERE scope_id = ?')
|
|
6010
|
+
.get(scopeId);
|
|
6011
|
+
if (!scope || scope.tenant_id !== tenantId) {
|
|
6012
|
+
throw substratError('not_found', `unknown scope ${scopeId} in tenant ${tenantId}`);
|
|
6013
|
+
}
|
|
6014
|
+
this.recordAdmin(actor, 'resolveKeptCopy', { tenantId, scopeId }, { script: r.script, keptAt: r.keptAt, revision: r.revisionBefore }, { action: r.action, liveScript: r.liveScript, revision: r.revisionAfter });
|
|
6015
|
+
},
|
|
5831
6016
|
setScopeExpiresAt: async (actor, tenantId, scopeId, expiresAt) => {
|
|
5832
6017
|
const scope = this.directory
|
|
5833
6018
|
.prepare('SELECT tenant_id, expires_at FROM scopes WHERE scope_id = ?')
|
|
@@ -5846,7 +6031,7 @@ export class SqliteScopeHost {
|
|
|
5846
6031
|
// — which is append-only — holding an event the scope never had. The mark that
|
|
5847
6032
|
// follows needs no such care: it stamps only rows that exist and are undrained.
|
|
5848
6033
|
const db = this.scopeReadDbFor(tenantId, scopeId);
|
|
5849
|
-
const page = db.prepare(`SELECT * FROM _substrat_outbox WHERE drained_at IS NULL ORDER BY id LIMIT ? OFFSET ?`);
|
|
6034
|
+
const page = db.prepare(`SELECT * FROM _substrat_outbox WHERE drained_at IS NULL AND ${emittedHere()} ORDER BY id LIMIT ? OFFSET ?`);
|
|
5850
6035
|
// The kernel's read, shared with the DO: the envelope and the columns lifted beside
|
|
5851
6036
|
// it (#1242, #1237) are decoded once, and a row that will not decode is stepped over
|
|
5852
6037
|
// rather than stalling the scope on it (#1636).
|
|
@@ -7270,6 +7455,11 @@ export class SqliteScopeHost {
|
|
|
7270
7455
|
const { tenantId, scopeId, ...after } = ownerTransferAudit.parse(entry);
|
|
7271
7456
|
this.recordAdmin(actor, 'transferOwner', { tenantId, scopeId }, null, after);
|
|
7272
7457
|
},
|
|
7458
|
+
/** #2005: one change to a scope's copy marker, written around the vertical's own change. */
|
|
7459
|
+
recordCopyMark: async (actor, entry) => {
|
|
7460
|
+
const { tenantId, scopeId, action, ...after } = copyMarkAudit.parse(entry);
|
|
7461
|
+
this.recordAdmin(actor, action === 'mark' ? 'markScopeCopy' : 'clearScopeCopyMark', { tenantId, scopeId }, null, after);
|
|
7462
|
+
},
|
|
7273
7463
|
accessLog: async (actor, filter) => {
|
|
7274
7464
|
const where = [];
|
|
7275
7465
|
const params = [];
|
|
@@ -7705,18 +7895,32 @@ export class SqliteScopeHost {
|
|
|
7705
7895
|
* read `ensureIdentityKey` already uses — PRAGMA would do here, but not there.
|
|
7706
7896
|
*/
|
|
7707
7897
|
ensureScheduleStateKind(db) {
|
|
7898
|
+
this.rebuildIfStale(db, '_substrat_schedule_state', scheduleStateHasKind, SCHEDULE_STATE_REBUILD);
|
|
7899
|
+
}
|
|
7900
|
+
/**
|
|
7901
|
+
* #1745: `_substrat_refusals`, rebuilt to admit a guard row (a nullable `from_state`, plus
|
|
7902
|
+
* `guard` and `reason`) on a scope db whose table predates it.
|
|
7903
|
+
*/
|
|
7904
|
+
ensureRefusalsAdmitGuards(db) {
|
|
7905
|
+
this.rebuildIfStale(db, '_substrat_refusals', refusalsAdmitGuards, REFUSALS_REBUILD);
|
|
7906
|
+
}
|
|
7907
|
+
/**
|
|
7908
|
+
* Run a kernel create-copy-drop-rename `script` over `table` when the stored DDL fails
|
|
7909
|
+
* `isCurrent` — the DO adapter's `rebuildIfStale`, so the two rebuild byte-identically.
|
|
7910
|
+
*/
|
|
7911
|
+
rebuildIfStale(db, table, isCurrent, script) {
|
|
7708
7912
|
const row = db
|
|
7709
7913
|
.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?")
|
|
7710
|
-
.get(
|
|
7711
|
-
if (!row ||
|
|
7914
|
+
.get(table);
|
|
7915
|
+
if (!row || isCurrent(row.sql))
|
|
7712
7916
|
return;
|
|
7713
|
-
// In a transaction, for the reason the kernel
|
|
7917
|
+
// In a transaction, for the reason the kernel constants spell out: create-copy-
|
|
7714
7918
|
// drop-rename has two intermediate states and BOTH are unrecoverable on the next
|
|
7715
7919
|
// wake — one dies on the leftover scratch table, one silently orphans the rows
|
|
7716
7920
|
// behind an empty table `CREATE TABLE IF NOT EXISTS` put back. `db.transaction`
|
|
7717
7921
|
// nests as a SAVEPOINT, which is what makes this safe on the `loadDump` path too,
|
|
7718
7922
|
// where the whole replay is already inside one.
|
|
7719
|
-
db.transaction(() => db.exec(
|
|
7923
|
+
db.transaction(() => db.exec(script))();
|
|
7720
7924
|
}
|
|
7721
7925
|
/**
|
|
7722
7926
|
* #1573: run a create-copy-drop-rename against the directory as ONE transaction.
|
|
@@ -8155,7 +8359,7 @@ export class SqliteScopeHost {
|
|
|
8155
8359
|
tenantId: rt.tenantId,
|
|
8156
8360
|
scopeId: rt.scopeId,
|
|
8157
8361
|
operation: operation ?? null,
|
|
8158
|
-
invocationId: () => rt.invocationId,
|
|
8362
|
+
invocationId: () => rt.invocationId ?? rt.unitInvocationId,
|
|
8159
8363
|
// A consumer runs under the system override, so it logs as `system`.
|
|
8160
8364
|
principalKind: overrideActor ? 'system' : subject.kind,
|
|
8161
8365
|
// The string-safe redaction: `redactSecrets` parses its serialization back, and a
|
|
@@ -8243,7 +8447,7 @@ export class SqliteScopeHost {
|
|
|
8243
8447
|
// as `pageOf`, applied here because the cursor is a COLUMN's value (plus
|
|
8244
8448
|
// the tie-break) rather than a field the caller could name.
|
|
8245
8449
|
const last = rows.length >= limit ? rows[rows.length - 1] : undefined;
|
|
8246
|
-
const nextCursor = last === undefined ? null : cursorOf(last, q.sortColumn, plan.idColumn);
|
|
8450
|
+
const nextCursor = last === undefined ? null : cursorOf(last, q.sortColumn, plan.idColumn, q.order);
|
|
8247
8451
|
const page = { entries: rows, nextCursor };
|
|
8248
8452
|
if (!params.total)
|
|
8249
8453
|
return page;
|
|
@@ -8644,6 +8848,8 @@ export class SqliteScopeHost {
|
|
|
8644
8848
|
// #1288: not an ensureColumn, because `kind` joins the schedule-state PRIMARY KEY
|
|
8645
8849
|
// and no ALTER can widen a key. Rebuilt instead, from the kernel's statements.
|
|
8646
8850
|
this.ensureScheduleStateKind(db);
|
|
8851
|
+
// #1745: likewise a rebuild — `from_state` loses its NOT NULL, which no ALTER can do.
|
|
8852
|
+
this.ensureRefusalsAdmitGuards(db);
|
|
8647
8853
|
// #1525: the invocation a fired schedule ran in, on a scope DB created before the
|
|
8648
8854
|
// column. Nullable, and the null is honestly "no call was carried" for every legacy
|
|
8649
8855
|
// row — a past run's id cannot be recovered afterwards, exactly as the other #1525
|
|
@@ -8653,6 +8859,9 @@ export class SqliteScopeHost {
|
|
|
8653
8859
|
// executes, the table always already has `kind` in its key, never `invocation_id`.
|
|
8654
8860
|
this.ensureColumn(db, '_substrat_schedule_state', 'invocation_id', 'invocation_id TEXT');
|
|
8655
8861
|
this.ensureColumn(db, '_substrat_job_runs', 'subject_id', 'subject_id TEXT');
|
|
8862
|
+
// #2009: the copy classification on a scope DB built before it (NULL reads as a copy; see
|
|
8863
|
+
// `COPY_ORIGIN_DDL`).
|
|
8864
|
+
this.ensureColumn(db, '_substrat_copy_origin', 'is_copy', 'is_copy INTEGER');
|
|
8656
8865
|
// #1237: `readInvocation`'s lookup — WHERE invocation_id = ? ORDER BY id — over an outbox
|
|
8657
8866
|
// that is never pruned. No index leads with invocation_id, so without this one SQLite
|
|
8658
8867
|
// walks the PRIMARY KEY from the oldest event until it reaches the call, and reading a
|
|
@@ -8700,6 +8909,7 @@ export class SqliteScopeHost {
|
|
|
8700
8909
|
mintEventId,
|
|
8701
8910
|
causedBy: null,
|
|
8702
8911
|
invocationId: null,
|
|
8912
|
+
unitInvocationId: null,
|
|
8703
8913
|
};
|
|
8704
8914
|
this.scopes.set(key, created);
|
|
8705
8915
|
this.scopesById.set(scopeId, created);
|