@smthrs/engine-store 0.0.0-stage → 1.0.0-rc.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +247 -0
- package/LICENSE +21 -0
- package/README.md +146 -2
- package/dist/cjs/ArtifactGc.d.ts +235 -0
- package/dist/cjs/ArtifactGc.d.ts.map +1 -0
- package/dist/cjs/ArtifactGc.js +183 -0
- package/dist/cjs/ArtifactGc.js.map +7 -0
- package/dist/cjs/ArtifactSync.d.ts +164 -0
- package/dist/cjs/ArtifactSync.d.ts.map +1 -0
- package/dist/cjs/ArtifactSync.js +144 -0
- package/dist/cjs/ArtifactSync.js.map +7 -0
- package/dist/cjs/CacheSync.d.ts +112 -0
- package/dist/cjs/CacheSync.d.ts.map +1 -0
- package/dist/cjs/CacheSync.js +64 -0
- package/dist/cjs/CacheSync.js.map +7 -0
- package/dist/cjs/DeferredClockFold.d.ts +354 -0
- package/dist/cjs/DeferredClockFold.d.ts.map +1 -0
- package/dist/cjs/DeferredClockFold.js +335 -0
- package/dist/cjs/DeferredClockFold.js.map +7 -0
- package/dist/cjs/DisasterRecovery.d.ts +431 -0
- package/dist/cjs/DisasterRecovery.d.ts.map +1 -0
- package/dist/cjs/DisasterRecovery.js +453 -0
- package/dist/cjs/DisasterRecovery.js.map +7 -0
- package/dist/cjs/DurableEngineState.d.ts +776 -0
- package/dist/cjs/DurableEngineState.d.ts.map +1 -0
- package/dist/cjs/DurableEngineState.js +1536 -0
- package/dist/cjs/DurableEngineState.js.map +7 -0
- package/dist/cjs/EngineStore.d.ts +158 -0
- package/dist/cjs/EngineStore.d.ts.map +1 -0
- package/dist/cjs/EngineStore.js +398 -0
- package/dist/cjs/EngineStore.js.map +7 -0
- package/dist/cjs/EngineStoreMetrics.d.ts +276 -0
- package/dist/cjs/EngineStoreMetrics.d.ts.map +1 -0
- package/dist/cjs/EngineStoreMetrics.js +135 -0
- package/dist/cjs/EngineStoreMetrics.js.map +7 -0
- package/dist/cjs/Errors.d.ts +22 -0
- package/dist/cjs/Errors.d.ts.map +1 -0
- package/dist/cjs/Errors.js +39 -0
- package/dist/cjs/Errors.js.map +7 -0
- package/dist/cjs/EventTypes.d.ts +27 -0
- package/dist/cjs/EventTypes.d.ts.map +1 -0
- package/dist/cjs/EventTypes.js +37 -0
- package/dist/cjs/EventTypes.js.map +7 -0
- package/dist/cjs/ExecutionFacts.d.ts +59 -0
- package/dist/cjs/ExecutionFacts.d.ts.map +1 -0
- package/dist/cjs/ExecutionFacts.js +123 -0
- package/dist/cjs/ExecutionFacts.js.map +7 -0
- package/dist/cjs/ExecutionSnapshot.d.ts +117 -0
- package/dist/cjs/ExecutionSnapshot.d.ts.map +1 -0
- package/dist/cjs/ExecutionSnapshot.js +111 -0
- package/dist/cjs/ExecutionSnapshot.js.map +7 -0
- package/dist/cjs/Inconsistency.d.ts +154 -0
- package/dist/cjs/Inconsistency.d.ts.map +1 -0
- package/dist/cjs/Inconsistency.js +135 -0
- package/dist/cjs/Inconsistency.js.map +7 -0
- package/dist/cjs/Migrations.d.ts +56 -0
- package/dist/cjs/Migrations.d.ts.map +1 -0
- package/dist/cjs/Migrations.js +78 -0
- package/dist/cjs/Migrations.js.map +7 -0
- package/dist/cjs/OwnerIdentity.d.ts +91 -0
- package/dist/cjs/OwnerIdentity.d.ts.map +1 -0
- package/dist/cjs/OwnerIdentity.js +65 -0
- package/dist/cjs/OwnerIdentity.js.map +7 -0
- package/dist/cjs/PlanInputStore.d.ts +110 -0
- package/dist/cjs/PlanInputStore.d.ts.map +1 -0
- package/dist/cjs/PlanInputStore.js +224 -0
- package/dist/cjs/PlanInputStore.js.map +7 -0
- package/dist/cjs/PlanMergeStore.d.ts +113 -0
- package/dist/cjs/PlanMergeStore.d.ts.map +1 -0
- package/dist/cjs/PlanMergeStore.js +255 -0
- package/dist/cjs/PlanMergeStore.js.map +7 -0
- package/dist/cjs/PlanScheduler.d.ts +363 -0
- package/dist/cjs/PlanScheduler.d.ts.map +1 -0
- package/dist/cjs/PlanScheduler.js +1090 -0
- package/dist/cjs/PlanScheduler.js.map +7 -0
- package/dist/cjs/Reconciliation.d.ts +175 -0
- package/dist/cjs/Reconciliation.d.ts.map +1 -0
- package/dist/cjs/Reconciliation.js +119 -0
- package/dist/cjs/Reconciliation.js.map +7 -0
- package/dist/cjs/ReplayOnly.d.ts +91 -0
- package/dist/cjs/ReplayOnly.d.ts.map +1 -0
- package/dist/cjs/ReplayOnly.js +52 -0
- package/dist/cjs/ReplayOnly.js.map +7 -0
- package/dist/cjs/Retention.d.ts +113 -0
- package/dist/cjs/Retention.d.ts.map +1 -0
- package/dist/cjs/Retention.js +99 -0
- package/dist/cjs/Retention.js.map +7 -0
- package/dist/cjs/RunCatalogRead.d.ts +17 -0
- package/dist/cjs/RunCatalogRead.d.ts.map +1 -0
- package/dist/cjs/RunCatalogRead.js +33 -0
- package/dist/cjs/RunCatalogRead.js.map +7 -0
- package/dist/cjs/RunChangeFeed.d.ts +75 -0
- package/dist/cjs/RunChangeFeed.d.ts.map +1 -0
- package/dist/cjs/RunChangeFeed.js +96 -0
- package/dist/cjs/RunChangeFeed.js.map +7 -0
- package/dist/cjs/RunState.d.ts +116 -0
- package/dist/cjs/RunState.d.ts.map +1 -0
- package/dist/cjs/RunState.js +76 -0
- package/dist/cjs/RunState.js.map +7 -0
- package/dist/cjs/Selection.d.ts +495 -0
- package/dist/cjs/Selection.d.ts.map +1 -0
- package/dist/cjs/Selection.js +288 -0
- package/dist/cjs/Selection.js.map +7 -0
- package/dist/cjs/SelectionStore.d.ts +141 -0
- package/dist/cjs/SelectionStore.d.ts.map +1 -0
- package/dist/cjs/SelectionStore.js +229 -0
- package/dist/cjs/SelectionStore.js.map +7 -0
- package/dist/cjs/StepBoundary.d.ts +501 -0
- package/dist/cjs/StepBoundary.d.ts.map +1 -0
- package/dist/cjs/StepBoundary.js +619 -0
- package/dist/cjs/StepBoundary.js.map +7 -0
- package/dist/cjs/StepCacheFold.d.ts +148 -0
- package/dist/cjs/StepCacheFold.d.ts.map +1 -0
- package/dist/cjs/StepCacheFold.js +131 -0
- package/dist/cjs/StepCacheFold.js.map +7 -0
- package/dist/cjs/StepSandbox.d.ts +78 -0
- package/dist/cjs/StepSandbox.d.ts.map +1 -0
- package/dist/cjs/StepSandbox.js +75 -0
- package/dist/cjs/StepSandbox.js.map +7 -0
- package/dist/cjs/WakeBus.d.ts +116 -0
- package/dist/cjs/WakeBus.d.ts.map +1 -0
- package/dist/cjs/WakeBus.js +89 -0
- package/dist/cjs/WakeBus.js.map +7 -0
- package/dist/cjs/WorkspaceSandbox.d.ts +619 -0
- package/dist/cjs/WorkspaceSandbox.d.ts.map +1 -0
- package/dist/cjs/WorkspaceSandbox.js +711 -0
- package/dist/cjs/WorkspaceSandbox.js.map +7 -0
- package/dist/cjs/index.d.ts +157 -0
- package/dist/cjs/index.d.ts.map +1 -0
- package/dist/cjs/index.js +95 -0
- package/dist/cjs/index.js.map +7 -0
- package/dist/cjs/internal/ActionPersistence.d.ts +276 -0
- package/dist/cjs/internal/ActionPersistence.d.ts.map +1 -0
- package/dist/cjs/internal/ActionPersistence.js +1395 -0
- package/dist/cjs/internal/ActionPersistence.js.map +7 -0
- package/dist/cjs/internal/ArtifactRoots.d.ts +38 -0
- package/dist/cjs/internal/ArtifactRoots.d.ts.map +1 -0
- package/dist/cjs/internal/ArtifactRoots.js +89 -0
- package/dist/cjs/internal/ArtifactRoots.js.map +7 -0
- package/dist/cjs/internal/AttemptAdmission.d.ts +53 -0
- package/dist/cjs/internal/AttemptAdmission.d.ts.map +1 -0
- package/dist/cjs/internal/AttemptAdmission.js +52 -0
- package/dist/cjs/internal/AttemptAdmission.js.map +7 -0
- package/dist/cjs/internal/AttemptLifecycle.d.ts +245 -0
- package/dist/cjs/internal/AttemptLifecycle.d.ts.map +1 -0
- package/dist/cjs/internal/AttemptLifecycle.js +112 -0
- package/dist/cjs/internal/AttemptLifecycle.js.map +7 -0
- package/dist/cjs/internal/AttemptProbe.d.ts +53 -0
- package/dist/cjs/internal/AttemptProbe.d.ts.map +1 -0
- package/dist/cjs/internal/AttemptProbe.js +66 -0
- package/dist/cjs/internal/AttemptProbe.js.map +7 -0
- package/dist/cjs/internal/CacheAdmission.d.ts +79 -0
- package/dist/cjs/internal/CacheAdmission.d.ts.map +1 -0
- package/dist/cjs/internal/CacheAdmission.js +71 -0
- package/dist/cjs/internal/CacheAdmission.js.map +7 -0
- package/dist/cjs/internal/CacheAgeHistory.d.ts +27 -0
- package/dist/cjs/internal/CacheAgeHistory.d.ts.map +1 -0
- package/dist/cjs/internal/CacheAgeHistory.js +61 -0
- package/dist/cjs/internal/CacheAgeHistory.js.map +7 -0
- package/dist/cjs/internal/CacheAgeVerdicts.d.ts +15 -0
- package/dist/cjs/internal/CacheAgeVerdicts.d.ts.map +1 -0
- package/dist/cjs/internal/CacheAgeVerdicts.js +73 -0
- package/dist/cjs/internal/CacheAgeVerdicts.js.map +7 -0
- package/dist/cjs/internal/CacheOutputPolicy.d.ts +24 -0
- package/dist/cjs/internal/CacheOutputPolicy.d.ts.map +1 -0
- package/dist/cjs/internal/CacheOutputPolicy.js +75 -0
- package/dist/cjs/internal/CacheOutputPolicy.js.map +7 -0
- package/dist/cjs/internal/CachePublication.d.ts +97 -0
- package/dist/cjs/internal/CachePublication.d.ts.map +1 -0
- package/dist/cjs/internal/CachePublication.js +80 -0
- package/dist/cjs/internal/CachePublication.js.map +7 -0
- package/dist/cjs/internal/CallFacts.d.ts +15 -0
- package/dist/cjs/internal/CallFacts.d.ts.map +1 -0
- package/dist/cjs/internal/CallFacts.js +97 -0
- package/dist/cjs/internal/CallFacts.js.map +7 -0
- package/dist/cjs/internal/CopiedRecord.d.ts +61 -0
- package/dist/cjs/internal/CopiedRecord.d.ts.map +1 -0
- package/dist/cjs/internal/CopiedRecord.js +134 -0
- package/dist/cjs/internal/CopiedRecord.js.map +7 -0
- package/dist/cjs/internal/DeferredPersistence.d.ts +108 -0
- package/dist/cjs/internal/DeferredPersistence.d.ts.map +1 -0
- package/dist/cjs/internal/DeferredPersistence.js +270 -0
- package/dist/cjs/internal/DeferredPersistence.js.map +7 -0
- package/dist/cjs/internal/EffectRecords.d.ts +92 -0
- package/dist/cjs/internal/EffectRecords.d.ts.map +1 -0
- package/dist/cjs/internal/EffectRecords.js +63 -0
- package/dist/cjs/internal/EffectRecords.js.map +7 -0
- package/dist/cjs/internal/EngineJj.d.ts +30 -0
- package/dist/cjs/internal/EngineJj.d.ts.map +1 -0
- package/dist/cjs/internal/EngineJj.js +42 -0
- package/dist/cjs/internal/EngineJj.js.map +7 -0
- package/dist/cjs/internal/EngineStateSchema.d.ts +68 -0
- package/dist/cjs/internal/EngineStateSchema.d.ts.map +1 -0
- package/dist/cjs/internal/EngineStateSchema.js +58 -0
- package/dist/cjs/internal/EngineStateSchema.js.map +7 -0
- package/dist/cjs/internal/ExecutionSnapshotRead.d.ts +162 -0
- package/dist/cjs/internal/ExecutionSnapshotRead.d.ts.map +1 -0
- package/dist/cjs/internal/ExecutionSnapshotRead.js +199 -0
- package/dist/cjs/internal/ExecutionSnapshotRead.js.map +7 -0
- package/dist/cjs/internal/ExecutionSnapshotRelations.d.ts +39 -0
- package/dist/cjs/internal/ExecutionSnapshotRelations.d.ts.map +1 -0
- package/dist/cjs/internal/ExecutionSnapshotRelations.js +105 -0
- package/dist/cjs/internal/ExecutionSnapshotRelations.js.map +7 -0
- package/dist/cjs/internal/ExitEncoding.d.ts +180 -0
- package/dist/cjs/internal/ExitEncoding.d.ts.map +1 -0
- package/dist/cjs/internal/ExitEncoding.js +236 -0
- package/dist/cjs/internal/ExitEncoding.js.map +7 -0
- package/dist/cjs/internal/FileBoundarySnapshot.d.ts +14 -0
- package/dist/cjs/internal/FileBoundarySnapshot.d.ts.map +1 -0
- package/dist/cjs/internal/FileBoundarySnapshot.js +51 -0
- package/dist/cjs/internal/FileBoundarySnapshot.js.map +7 -0
- package/dist/cjs/internal/FileEnumeration.d.ts +127 -0
- package/dist/cjs/internal/FileEnumeration.d.ts.map +1 -0
- package/dist/cjs/internal/FileEnumeration.js +194 -0
- package/dist/cjs/internal/FileEnumeration.js.map +7 -0
- package/dist/cjs/internal/HostReflection.d.ts +80 -0
- package/dist/cjs/internal/HostReflection.d.ts.map +1 -0
- package/dist/cjs/internal/HostReflection.js +43 -0
- package/dist/cjs/internal/HostReflection.js.map +7 -0
- package/dist/cjs/internal/JournalRecords.d.ts +325 -0
- package/dist/cjs/internal/JournalRecords.d.ts.map +1 -0
- package/dist/cjs/internal/JournalRecords.js +122 -0
- package/dist/cjs/internal/JournalRecords.js.map +7 -0
- package/dist/cjs/internal/NodeJournal.d.ts +157 -0
- package/dist/cjs/internal/NodeJournal.d.ts.map +1 -0
- package/dist/cjs/internal/NodeJournal.js +192 -0
- package/dist/cjs/internal/NodeJournal.js.map +7 -0
- package/dist/cjs/internal/Ordering.d.ts +14 -0
- package/dist/cjs/internal/Ordering.d.ts.map +1 -0
- package/dist/cjs/internal/Ordering.js +25 -0
- package/dist/cjs/internal/Ordering.js.map +7 -0
- package/dist/cjs/internal/PlanInputs.d.ts +14 -0
- package/dist/cjs/internal/PlanInputs.d.ts.map +1 -0
- package/dist/cjs/internal/PlanInputs.js +78 -0
- package/dist/cjs/internal/PlanInputs.js.map +7 -0
- package/dist/cjs/internal/PlanMerges.d.ts +178 -0
- package/dist/cjs/internal/PlanMerges.d.ts.map +1 -0
- package/dist/cjs/internal/PlanMerges.js +125 -0
- package/dist/cjs/internal/PlanMerges.js.map +7 -0
- package/dist/cjs/internal/ProvenanceSlot.d.ts +30 -0
- package/dist/cjs/internal/ProvenanceSlot.d.ts.map +1 -0
- package/dist/cjs/internal/ProvenanceSlot.js +53 -0
- package/dist/cjs/internal/ProvenanceSlot.js.map +7 -0
- package/dist/cjs/internal/ResultEnvelope.d.ts +88 -0
- package/dist/cjs/internal/ResultEnvelope.d.ts.map +1 -0
- package/dist/cjs/internal/ResultEnvelope.js +90 -0
- package/dist/cjs/internal/ResultEnvelope.js.map +7 -0
- package/dist/cjs/internal/RetentionOps.d.ts +414 -0
- package/dist/cjs/internal/RetentionOps.d.ts.map +1 -0
- package/dist/cjs/internal/RetentionOps.js +354 -0
- package/dist/cjs/internal/RetentionOps.js.map +7 -0
- package/dist/cjs/internal/RunCatalogOps.d.ts +118 -0
- package/dist/cjs/internal/RunCatalogOps.d.ts.map +1 -0
- package/dist/cjs/internal/RunCatalogOps.js +85 -0
- package/dist/cjs/internal/RunCatalogOps.js.map +7 -0
- package/dist/cjs/internal/RunCoordinator.d.ts +78 -0
- package/dist/cjs/internal/RunCoordinator.d.ts.map +1 -0
- package/dist/cjs/internal/RunCoordinator.js +123 -0
- package/dist/cjs/internal/RunCoordinator.js.map +7 -0
- package/dist/cjs/internal/RunDriver.d.ts +192 -0
- package/dist/cjs/internal/RunDriver.d.ts.map +1 -0
- package/dist/cjs/internal/RunDriver.js +1584 -0
- package/dist/cjs/internal/RunDriver.js.map +7 -0
- package/dist/cjs/internal/RunLifecycle.d.ts +22 -0
- package/dist/cjs/internal/RunLifecycle.d.ts.map +1 -0
- package/dist/cjs/internal/RunLifecycle.js +63 -0
- package/dist/cjs/internal/RunLifecycle.js.map +7 -0
- package/dist/cjs/internal/RunListing.d.ts +78 -0
- package/dist/cjs/internal/RunListing.d.ts.map +1 -0
- package/dist/cjs/internal/RunListing.js +171 -0
- package/dist/cjs/internal/RunListing.js.map +7 -0
- package/dist/cjs/internal/RunListingError.d.ts +35 -0
- package/dist/cjs/internal/RunListingError.d.ts.map +1 -0
- package/dist/cjs/internal/RunListingError.js +47 -0
- package/dist/cjs/internal/RunListingError.js.map +7 -0
- package/dist/cjs/internal/RuntimeGraph.d.ts +73 -0
- package/dist/cjs/internal/RuntimeGraph.d.ts.map +1 -0
- package/dist/cjs/internal/RuntimeGraph.js +180 -0
- package/dist/cjs/internal/RuntimeGraph.js.map +7 -0
- package/dist/cjs/internal/SandboxedExecution.d.ts +80 -0
- package/dist/cjs/internal/SandboxedExecution.d.ts.map +1 -0
- package/dist/cjs/internal/SandboxedExecution.js +104 -0
- package/dist/cjs/internal/SandboxedExecution.js.map +7 -0
- package/dist/cjs/internal/StateTransaction.d.ts +44 -0
- package/dist/cjs/internal/StateTransaction.d.ts.map +1 -0
- package/dist/cjs/internal/StateTransaction.js +52 -0
- package/dist/cjs/internal/StateTransaction.js.map +7 -0
- package/dist/cjs/internal/StepFacts.d.ts +14 -0
- package/dist/cjs/internal/StepFacts.d.ts.map +1 -0
- package/dist/cjs/internal/StepFacts.js +58 -0
- package/dist/cjs/internal/StepFacts.js.map +7 -0
- package/dist/cjs/internal/TypedEvents.d.ts +14 -0
- package/dist/cjs/internal/TypedEvents.d.ts.map +1 -0
- package/dist/cjs/internal/TypedEvents.js +37 -0
- package/dist/cjs/internal/TypedEvents.js.map +7 -0
- package/dist/cjs/migrations/0001_initial.d.ts +17 -0
- package/dist/cjs/migrations/0001_initial.d.ts.map +1 -0
- package/dist/cjs/migrations/0001_initial.js +61 -0
- package/dist/cjs/migrations/0001_initial.js.map +7 -0
- package/dist/cjs/migrations/0002_selection_store.d.ts +15 -0
- package/dist/cjs/migrations/0002_selection_store.d.ts.map +1 -0
- package/dist/cjs/migrations/0002_selection_store.js +56 -0
- package/dist/cjs/migrations/0002_selection_store.js.map +7 -0
- package/dist/cjs/migrations/0003_plan_inputs.d.ts +13 -0
- package/dist/cjs/migrations/0003_plan_inputs.d.ts.map +1 -0
- package/dist/cjs/migrations/0003_plan_inputs.js +90 -0
- package/dist/cjs/migrations/0003_plan_inputs.js.map +7 -0
- package/dist/cjs/migrations/0004_plan_environment.d.ts +14 -0
- package/dist/cjs/migrations/0004_plan_environment.d.ts.map +1 -0
- package/dist/cjs/migrations/0004_plan_environment.js +56 -0
- package/dist/cjs/migrations/0004_plan_environment.js.map +7 -0
- package/dist/cjs/migrations/0005_plan_merges.d.ts +14 -0
- package/dist/cjs/migrations/0005_plan_merges.d.ts.map +1 -0
- package/dist/cjs/migrations/0005_plan_merges.js +82 -0
- package/dist/cjs/migrations/0005_plan_merges.js.map +7 -0
- package/dist/cjs/migrations/0006_execution_listing.d.ts +15 -0
- package/dist/cjs/migrations/0006_execution_listing.d.ts.map +1 -0
- package/dist/cjs/migrations/0006_execution_listing.js +101 -0
- package/dist/cjs/migrations/0006_execution_listing.js.map +7 -0
- package/dist/cjs/migrations/0007_run_parent_sequence.d.ts +15 -0
- package/dist/cjs/migrations/0007_run_parent_sequence.d.ts.map +1 -0
- package/dist/cjs/migrations/0007_run_parent_sequence.js +40 -0
- package/dist/cjs/migrations/0007_run_parent_sequence.js.map +7 -0
- package/dist/cjs/migrations/0008_deferred_consumption.d.ts +16 -0
- package/dist/cjs/migrations/0008_deferred_consumption.d.ts.map +1 -0
- package/dist/cjs/migrations/0008_deferred_consumption.js +45 -0
- package/dist/cjs/migrations/0008_deferred_consumption.js.map +7 -0
- package/dist/cjs/migrations/0009_deferred_clock_facts.d.ts +24 -0
- package/dist/cjs/migrations/0009_deferred_clock_facts.d.ts.map +1 -0
- package/dist/cjs/migrations/0009_deferred_clock_facts.js +141 -0
- package/dist/cjs/migrations/0009_deferred_clock_facts.js.map +7 -0
- package/dist/cjs/migrations/0010_pending_cancellation.d.ts +15 -0
- package/dist/cjs/migrations/0010_pending_cancellation.d.ts.map +1 -0
- package/dist/cjs/migrations/0010_pending_cancellation.js +41 -0
- package/dist/cjs/migrations/0010_pending_cancellation.js.map +7 -0
- package/dist/cjs/package.json +1 -0
- package/dist/cjs/test/TestStores.d.ts +89 -0
- package/dist/cjs/test/TestStores.d.ts.map +1 -0
- package/dist/cjs/test/TestStores.js +98 -0
- package/dist/cjs/test/TestStores.js.map +7 -0
- package/dist/esm/ArtifactGc.d.ts +235 -0
- package/dist/esm/ArtifactGc.d.ts.map +1 -0
- package/dist/esm/ArtifactGc.js +268 -0
- package/dist/esm/ArtifactGc.js.map +1 -0
- package/dist/esm/ArtifactSync.d.ts +164 -0
- package/dist/esm/ArtifactSync.d.ts.map +1 -0
- package/dist/esm/ArtifactSync.js +176 -0
- package/dist/esm/ArtifactSync.js.map +1 -0
- package/dist/esm/CacheSync.d.ts +112 -0
- package/dist/esm/CacheSync.d.ts.map +1 -0
- package/dist/esm/CacheSync.js +94 -0
- package/dist/esm/CacheSync.js.map +1 -0
- package/dist/esm/DeferredClockFold.d.ts +354 -0
- package/dist/esm/DeferredClockFold.d.ts.map +1 -0
- package/dist/esm/DeferredClockFold.js +472 -0
- package/dist/esm/DeferredClockFold.js.map +1 -0
- package/dist/esm/DisasterRecovery.d.ts +431 -0
- package/dist/esm/DisasterRecovery.d.ts.map +1 -0
- package/dist/esm/DisasterRecovery.js +523 -0
- package/dist/esm/DisasterRecovery.js.map +1 -0
- package/dist/esm/DurableEngineState.d.ts +776 -0
- package/dist/esm/DurableEngineState.d.ts.map +1 -0
- package/dist/esm/DurableEngineState.js +1648 -0
- package/dist/esm/DurableEngineState.js.map +1 -0
- package/dist/esm/EngineStore.d.ts +158 -0
- package/dist/esm/EngineStore.d.ts.map +1 -0
- package/dist/esm/EngineStore.js +403 -0
- package/dist/esm/EngineStore.js.map +1 -0
- package/dist/esm/EngineStoreMetrics.d.ts +276 -0
- package/dist/esm/EngineStoreMetrics.d.ts.map +1 -0
- package/dist/esm/EngineStoreMetrics.js +260 -0
- package/dist/esm/EngineStoreMetrics.js.map +1 -0
- package/dist/esm/Errors.d.ts +22 -0
- package/dist/esm/Errors.d.ts.map +1 -0
- package/dist/esm/Errors.js +22 -0
- package/dist/esm/Errors.js.map +1 -0
- package/dist/esm/EventTypes.d.ts +27 -0
- package/dist/esm/EventTypes.d.ts.map +1 -0
- package/dist/esm/EventTypes.js +27 -0
- package/dist/esm/EventTypes.js.map +1 -0
- package/dist/esm/ExecutionFacts.d.ts +59 -0
- package/dist/esm/ExecutionFacts.d.ts.map +1 -0
- package/dist/esm/ExecutionFacts.js +96 -0
- package/dist/esm/ExecutionFacts.js.map +1 -0
- package/dist/esm/ExecutionSnapshot.d.ts +117 -0
- package/dist/esm/ExecutionSnapshot.d.ts.map +1 -0
- package/dist/esm/ExecutionSnapshot.js +95 -0
- package/dist/esm/ExecutionSnapshot.js.map +1 -0
- package/dist/esm/Inconsistency.d.ts +154 -0
- package/dist/esm/Inconsistency.d.ts.map +1 -0
- package/dist/esm/Inconsistency.js +133 -0
- package/dist/esm/Inconsistency.js.map +1 -0
- package/dist/esm/Migrations.d.ts +56 -0
- package/dist/esm/Migrations.d.ts.map +1 -0
- package/dist/esm/Migrations.js +91 -0
- package/dist/esm/Migrations.js.map +1 -0
- package/dist/esm/OwnerIdentity.d.ts +91 -0
- package/dist/esm/OwnerIdentity.d.ts.map +1 -0
- package/dist/esm/OwnerIdentity.js +99 -0
- package/dist/esm/OwnerIdentity.js.map +1 -0
- package/dist/esm/PlanInputStore.d.ts +110 -0
- package/dist/esm/PlanInputStore.d.ts.map +1 -0
- package/dist/esm/PlanInputStore.js +207 -0
- package/dist/esm/PlanInputStore.js.map +1 -0
- package/dist/esm/PlanMergeStore.d.ts +113 -0
- package/dist/esm/PlanMergeStore.d.ts.map +1 -0
- package/dist/esm/PlanMergeStore.js +244 -0
- package/dist/esm/PlanMergeStore.js.map +1 -0
- package/dist/esm/PlanScheduler.d.ts +363 -0
- package/dist/esm/PlanScheduler.d.ts.map +1 -0
- package/dist/esm/PlanScheduler.js +1321 -0
- package/dist/esm/PlanScheduler.js.map +1 -0
- package/dist/esm/Reconciliation.d.ts +175 -0
- package/dist/esm/Reconciliation.d.ts.map +1 -0
- package/dist/esm/Reconciliation.js +175 -0
- package/dist/esm/Reconciliation.js.map +1 -0
- package/dist/esm/ReplayOnly.d.ts +91 -0
- package/dist/esm/ReplayOnly.d.ts.map +1 -0
- package/dist/esm/ReplayOnly.js +59 -0
- package/dist/esm/ReplayOnly.js.map +1 -0
- package/dist/esm/Retention.d.ts +113 -0
- package/dist/esm/Retention.d.ts.map +1 -0
- package/dist/esm/Retention.js +139 -0
- package/dist/esm/Retention.js.map +1 -0
- package/dist/esm/RunCatalogRead.d.ts +17 -0
- package/dist/esm/RunCatalogRead.d.ts.map +1 -0
- package/dist/esm/RunCatalogRead.js +17 -0
- package/dist/esm/RunCatalogRead.js.map +1 -0
- package/dist/esm/RunChangeFeed.d.ts +75 -0
- package/dist/esm/RunChangeFeed.d.ts.map +1 -0
- package/dist/esm/RunChangeFeed.js +81 -0
- package/dist/esm/RunChangeFeed.js.map +1 -0
- package/dist/esm/RunState.d.ts +116 -0
- package/dist/esm/RunState.d.ts.map +1 -0
- package/dist/esm/RunState.js +109 -0
- package/dist/esm/RunState.js.map +1 -0
- package/dist/esm/Selection.d.ts +495 -0
- package/dist/esm/Selection.d.ts.map +1 -0
- package/dist/esm/Selection.js +482 -0
- package/dist/esm/Selection.js.map +1 -0
- package/dist/esm/SelectionStore.d.ts +141 -0
- package/dist/esm/SelectionStore.d.ts.map +1 -0
- package/dist/esm/SelectionStore.js +215 -0
- package/dist/esm/SelectionStore.js.map +1 -0
- package/dist/esm/StepBoundary.d.ts +501 -0
- package/dist/esm/StepBoundary.d.ts.map +1 -0
- package/dist/esm/StepBoundary.js +919 -0
- package/dist/esm/StepBoundary.js.map +1 -0
- package/dist/esm/StepCacheFold.d.ts +148 -0
- package/dist/esm/StepCacheFold.d.ts.map +1 -0
- package/dist/esm/StepCacheFold.js +201 -0
- package/dist/esm/StepCacheFold.js.map +1 -0
- package/dist/esm/StepSandbox.d.ts +78 -0
- package/dist/esm/StepSandbox.d.ts.map +1 -0
- package/dist/esm/StepSandbox.js +70 -0
- package/dist/esm/StepSandbox.js.map +1 -0
- package/dist/esm/WakeBus.d.ts +116 -0
- package/dist/esm/WakeBus.d.ts.map +1 -0
- package/dist/esm/WakeBus.js +121 -0
- package/dist/esm/WakeBus.js.map +1 -0
- package/dist/esm/WorkspaceSandbox.d.ts +619 -0
- package/dist/esm/WorkspaceSandbox.d.ts.map +1 -0
- package/dist/esm/WorkspaceSandbox.js +985 -0
- package/dist/esm/WorkspaceSandbox.js.map +1 -0
- package/dist/esm/index.d.ts +157 -0
- package/dist/esm/index.d.ts.map +1 -0
- package/dist/esm/index.js +157 -0
- package/dist/esm/index.js.map +1 -0
- package/dist/esm/internal/ActionPersistence.d.ts +276 -0
- package/dist/esm/internal/ActionPersistence.d.ts.map +1 -0
- package/dist/esm/internal/ActionPersistence.js +2306 -0
- package/dist/esm/internal/ActionPersistence.js.map +1 -0
- package/dist/esm/internal/ArtifactRoots.d.ts +38 -0
- package/dist/esm/internal/ArtifactRoots.d.ts.map +1 -0
- package/dist/esm/internal/ArtifactRoots.js +76 -0
- package/dist/esm/internal/ArtifactRoots.js.map +1 -0
- package/dist/esm/internal/AttemptAdmission.d.ts +53 -0
- package/dist/esm/internal/AttemptAdmission.d.ts.map +1 -0
- package/dist/esm/internal/AttemptAdmission.js +52 -0
- package/dist/esm/internal/AttemptAdmission.js.map +1 -0
- package/dist/esm/internal/AttemptLifecycle.d.ts +245 -0
- package/dist/esm/internal/AttemptLifecycle.d.ts.map +1 -0
- package/dist/esm/internal/AttemptLifecycle.js +104 -0
- package/dist/esm/internal/AttemptLifecycle.js.map +1 -0
- package/dist/esm/internal/AttemptProbe.d.ts +53 -0
- package/dist/esm/internal/AttemptProbe.d.ts.map +1 -0
- package/dist/esm/internal/AttemptProbe.js +83 -0
- package/dist/esm/internal/AttemptProbe.js.map +1 -0
- package/dist/esm/internal/CacheAdmission.d.ts +79 -0
- package/dist/esm/internal/CacheAdmission.d.ts.map +1 -0
- package/dist/esm/internal/CacheAdmission.js +72 -0
- package/dist/esm/internal/CacheAdmission.js.map +1 -0
- package/dist/esm/internal/CacheAgeHistory.d.ts +27 -0
- package/dist/esm/internal/CacheAgeHistory.d.ts.map +1 -0
- package/dist/esm/internal/CacheAgeHistory.js +49 -0
- package/dist/esm/internal/CacheAgeHistory.js.map +1 -0
- package/dist/esm/internal/CacheAgeVerdicts.d.ts +15 -0
- package/dist/esm/internal/CacheAgeVerdicts.d.ts.map +1 -0
- package/dist/esm/internal/CacheAgeVerdicts.js +54 -0
- package/dist/esm/internal/CacheAgeVerdicts.js.map +1 -0
- package/dist/esm/internal/CacheOutputPolicy.d.ts +24 -0
- package/dist/esm/internal/CacheOutputPolicy.d.ts.map +1 -0
- package/dist/esm/internal/CacheOutputPolicy.js +73 -0
- package/dist/esm/internal/CacheOutputPolicy.js.map +1 -0
- package/dist/esm/internal/CachePublication.d.ts +97 -0
- package/dist/esm/internal/CachePublication.d.ts.map +1 -0
- package/dist/esm/internal/CachePublication.js +116 -0
- package/dist/esm/internal/CachePublication.js.map +1 -0
- package/dist/esm/internal/CallFacts.d.ts +15 -0
- package/dist/esm/internal/CallFacts.d.ts.map +1 -0
- package/dist/esm/internal/CallFacts.js +83 -0
- package/dist/esm/internal/CallFacts.js.map +1 -0
- package/dist/esm/internal/CopiedRecord.d.ts +61 -0
- package/dist/esm/internal/CopiedRecord.d.ts.map +1 -0
- package/dist/esm/internal/CopiedRecord.js +165 -0
- package/dist/esm/internal/CopiedRecord.js.map +1 -0
- package/dist/esm/internal/DeferredPersistence.d.ts +108 -0
- package/dist/esm/internal/DeferredPersistence.d.ts.map +1 -0
- package/dist/esm/internal/DeferredPersistence.js +240 -0
- package/dist/esm/internal/DeferredPersistence.js.map +1 -0
- package/dist/esm/internal/EffectRecords.d.ts +92 -0
- package/dist/esm/internal/EffectRecords.d.ts.map +1 -0
- package/dist/esm/internal/EffectRecords.js +94 -0
- package/dist/esm/internal/EffectRecords.js.map +1 -0
- package/dist/esm/internal/EngineJj.d.ts +30 -0
- package/dist/esm/internal/EngineJj.d.ts.map +1 -0
- package/dist/esm/internal/EngineJj.js +29 -0
- package/dist/esm/internal/EngineJj.js.map +1 -0
- package/dist/esm/internal/EngineStateSchema.d.ts +68 -0
- package/dist/esm/internal/EngineStateSchema.d.ts.map +1 -0
- package/dist/esm/internal/EngineStateSchema.js +53 -0
- package/dist/esm/internal/EngineStateSchema.js.map +1 -0
- package/dist/esm/internal/ExecutionSnapshotRead.d.ts +162 -0
- package/dist/esm/internal/ExecutionSnapshotRead.d.ts.map +1 -0
- package/dist/esm/internal/ExecutionSnapshotRead.js +218 -0
- package/dist/esm/internal/ExecutionSnapshotRead.js.map +1 -0
- package/dist/esm/internal/ExecutionSnapshotRelations.d.ts +39 -0
- package/dist/esm/internal/ExecutionSnapshotRelations.d.ts.map +1 -0
- package/dist/esm/internal/ExecutionSnapshotRelations.js +81 -0
- package/dist/esm/internal/ExecutionSnapshotRelations.js.map +1 -0
- package/dist/esm/internal/ExitEncoding.d.ts +180 -0
- package/dist/esm/internal/ExitEncoding.d.ts.map +1 -0
- package/dist/esm/internal/ExitEncoding.js +361 -0
- package/dist/esm/internal/ExitEncoding.js.map +1 -0
- package/dist/esm/internal/FileBoundarySnapshot.d.ts +14 -0
- package/dist/esm/internal/FileBoundarySnapshot.d.ts.map +1 -0
- package/dist/esm/internal/FileBoundarySnapshot.js +29 -0
- package/dist/esm/internal/FileBoundarySnapshot.js.map +1 -0
- package/dist/esm/internal/FileEnumeration.d.ts +127 -0
- package/dist/esm/internal/FileEnumeration.d.ts.map +1 -0
- package/dist/esm/internal/FileEnumeration.js +236 -0
- package/dist/esm/internal/FileEnumeration.js.map +1 -0
- package/dist/esm/internal/HostReflection.d.ts +80 -0
- package/dist/esm/internal/HostReflection.d.ts.map +1 -0
- package/dist/esm/internal/HostReflection.js +98 -0
- package/dist/esm/internal/HostReflection.js.map +1 -0
- package/dist/esm/internal/JournalRecords.d.ts +325 -0
- package/dist/esm/internal/JournalRecords.d.ts.map +1 -0
- package/dist/esm/internal/JournalRecords.js +306 -0
- package/dist/esm/internal/JournalRecords.js.map +1 -0
- package/dist/esm/internal/NodeJournal.d.ts +157 -0
- package/dist/esm/internal/NodeJournal.d.ts.map +1 -0
- package/dist/esm/internal/NodeJournal.js +298 -0
- package/dist/esm/internal/NodeJournal.js.map +1 -0
- package/dist/esm/internal/Ordering.d.ts +14 -0
- package/dist/esm/internal/Ordering.d.ts.map +1 -0
- package/dist/esm/internal/Ordering.js +14 -0
- package/dist/esm/internal/Ordering.js.map +1 -0
- package/dist/esm/internal/PlanInputs.d.ts +14 -0
- package/dist/esm/internal/PlanInputs.d.ts.map +1 -0
- package/dist/esm/internal/PlanInputs.js +57 -0
- package/dist/esm/internal/PlanInputs.js.map +1 -0
- package/dist/esm/internal/PlanMerges.d.ts +178 -0
- package/dist/esm/internal/PlanMerges.d.ts.map +1 -0
- package/dist/esm/internal/PlanMerges.js +113 -0
- package/dist/esm/internal/PlanMerges.js.map +1 -0
- package/dist/esm/internal/ProvenanceSlot.d.ts +30 -0
- package/dist/esm/internal/ProvenanceSlot.d.ts.map +1 -0
- package/dist/esm/internal/ProvenanceSlot.js +39 -0
- package/dist/esm/internal/ProvenanceSlot.js.map +1 -0
- package/dist/esm/internal/ResultEnvelope.d.ts +88 -0
- package/dist/esm/internal/ResultEnvelope.d.ts.map +1 -0
- package/dist/esm/internal/ResultEnvelope.js +83 -0
- package/dist/esm/internal/ResultEnvelope.js.map +1 -0
- package/dist/esm/internal/RetentionOps.d.ts +414 -0
- package/dist/esm/internal/RetentionOps.d.ts.map +1 -0
- package/dist/esm/internal/RetentionOps.js +576 -0
- package/dist/esm/internal/RetentionOps.js.map +1 -0
- package/dist/esm/internal/RunCatalogOps.d.ts +118 -0
- package/dist/esm/internal/RunCatalogOps.d.ts.map +1 -0
- package/dist/esm/internal/RunCatalogOps.js +120 -0
- package/dist/esm/internal/RunCatalogOps.js.map +1 -0
- package/dist/esm/internal/RunCoordinator.d.ts +78 -0
- package/dist/esm/internal/RunCoordinator.d.ts.map +1 -0
- package/dist/esm/internal/RunCoordinator.js +115 -0
- package/dist/esm/internal/RunCoordinator.js.map +1 -0
- package/dist/esm/internal/RunDriver.d.ts +192 -0
- package/dist/esm/internal/RunDriver.d.ts.map +1 -0
- package/dist/esm/internal/RunDriver.js +2334 -0
- package/dist/esm/internal/RunDriver.js.map +1 -0
- package/dist/esm/internal/RunLifecycle.d.ts +22 -0
- package/dist/esm/internal/RunLifecycle.d.ts.map +1 -0
- package/dist/esm/internal/RunLifecycle.js +49 -0
- package/dist/esm/internal/RunLifecycle.js.map +1 -0
- package/dist/esm/internal/RunListing.d.ts +78 -0
- package/dist/esm/internal/RunListing.d.ts.map +1 -0
- package/dist/esm/internal/RunListing.js +146 -0
- package/dist/esm/internal/RunListing.js.map +1 -0
- package/dist/esm/internal/RunListingError.d.ts +35 -0
- package/dist/esm/internal/RunListingError.d.ts.map +1 -0
- package/dist/esm/internal/RunListingError.js +27 -0
- package/dist/esm/internal/RunListingError.js.map +1 -0
- package/dist/esm/internal/RuntimeGraph.d.ts +73 -0
- package/dist/esm/internal/RuntimeGraph.d.ts.map +1 -0
- package/dist/esm/internal/RuntimeGraph.js +182 -0
- package/dist/esm/internal/RuntimeGraph.js.map +1 -0
- package/dist/esm/internal/SandboxedExecution.d.ts +80 -0
- package/dist/esm/internal/SandboxedExecution.d.ts.map +1 -0
- package/dist/esm/internal/SandboxedExecution.js +107 -0
- package/dist/esm/internal/SandboxedExecution.js.map +1 -0
- package/dist/esm/internal/StateTransaction.d.ts +44 -0
- package/dist/esm/internal/StateTransaction.d.ts.map +1 -0
- package/dist/esm/internal/StateTransaction.js +54 -0
- package/dist/esm/internal/StateTransaction.js.map +1 -0
- package/dist/esm/internal/StepFacts.d.ts +14 -0
- package/dist/esm/internal/StepFacts.d.ts.map +1 -0
- package/dist/esm/internal/StepFacts.js +35 -0
- package/dist/esm/internal/StepFacts.js.map +1 -0
- package/dist/esm/internal/TypedEvents.d.ts +14 -0
- package/dist/esm/internal/TypedEvents.d.ts.map +1 -0
- package/dist/esm/internal/TypedEvents.js +14 -0
- package/dist/esm/internal/TypedEvents.js.map +1 -0
- package/dist/esm/migrations/0001_initial.d.ts +17 -0
- package/dist/esm/migrations/0001_initial.d.ts.map +1 -0
- package/dist/esm/migrations/0001_initial.js +41 -0
- package/dist/esm/migrations/0001_initial.js.map +1 -0
- package/dist/esm/migrations/0002_selection_store.d.ts +15 -0
- package/dist/esm/migrations/0002_selection_store.d.ts.map +1 -0
- package/dist/esm/migrations/0002_selection_store.js +34 -0
- package/dist/esm/migrations/0002_selection_store.js.map +1 -0
- package/dist/esm/migrations/0003_plan_inputs.d.ts +13 -0
- package/dist/esm/migrations/0003_plan_inputs.d.ts.map +1 -0
- package/dist/esm/migrations/0003_plan_inputs.js +70 -0
- package/dist/esm/migrations/0003_plan_inputs.js.map +1 -0
- package/dist/esm/migrations/0004_plan_environment.d.ts +14 -0
- package/dist/esm/migrations/0004_plan_environment.d.ts.map +1 -0
- package/dist/esm/migrations/0004_plan_environment.js +33 -0
- package/dist/esm/migrations/0004_plan_environment.js.map +1 -0
- package/dist/esm/migrations/0005_plan_merges.d.ts +14 -0
- package/dist/esm/migrations/0005_plan_merges.d.ts.map +1 -0
- package/dist/esm/migrations/0005_plan_merges.js +59 -0
- package/dist/esm/migrations/0005_plan_merges.js.map +1 -0
- package/dist/esm/migrations/0006_execution_listing.d.ts +15 -0
- package/dist/esm/migrations/0006_execution_listing.d.ts.map +1 -0
- package/dist/esm/migrations/0006_execution_listing.js +81 -0
- package/dist/esm/migrations/0006_execution_listing.js.map +1 -0
- package/dist/esm/migrations/0007_run_parent_sequence.d.ts +15 -0
- package/dist/esm/migrations/0007_run_parent_sequence.d.ts.map +1 -0
- package/dist/esm/migrations/0007_run_parent_sequence.js +18 -0
- package/dist/esm/migrations/0007_run_parent_sequence.js.map +1 -0
- package/dist/esm/migrations/0008_deferred_consumption.d.ts +16 -0
- package/dist/esm/migrations/0008_deferred_consumption.d.ts.map +1 -0
- package/dist/esm/migrations/0008_deferred_consumption.js +24 -0
- package/dist/esm/migrations/0008_deferred_consumption.js.map +1 -0
- package/dist/esm/migrations/0009_deferred_clock_facts.d.ts +24 -0
- package/dist/esm/migrations/0009_deferred_clock_facts.d.ts.map +1 -0
- package/dist/esm/migrations/0009_deferred_clock_facts.js +127 -0
- package/dist/esm/migrations/0009_deferred_clock_facts.js.map +1 -0
- package/dist/esm/migrations/0010_pending_cancellation.d.ts +15 -0
- package/dist/esm/migrations/0010_pending_cancellation.d.ts.map +1 -0
- package/dist/esm/migrations/0010_pending_cancellation.js +19 -0
- package/dist/esm/migrations/0010_pending_cancellation.js.map +1 -0
- package/dist/esm/test/TestStores.d.ts +89 -0
- package/dist/esm/test/TestStores.d.ts.map +1 -0
- package/dist/esm/test/TestStores.js +103 -0
- package/dist/esm/test/TestStores.js.map +1 -0
- package/docs/README.md +113 -0
- package/docs/api.md +1494 -0
- package/docs/concepts/attempts-and-replay.md +149 -0
- package/docs/concepts/cache-admission.md +265 -0
- package/docs/concepts/call-facts.md +86 -0
- package/docs/concepts/durable-waits.md +175 -0
- package/docs/concepts/execution-facts.md +90 -0
- package/docs/concepts/ownership-and-fencing.md +128 -0
- package/docs/concepts/step-boundaries.md +148 -0
- package/docs/concepts/workspace-transactions.md +199 -0
- package/docs/event-types.md +14 -0
- package/docs/guides/back-up-and-restore.md +164 -0
- package/docs/guides/collect-unreferenced-artifacts.md +128 -0
- package/docs/guides/compose-a-durable-engine.md +146 -0
- package/docs/guides/coordinate-two-processes.md +120 -0
- package/docs/guides/defer-work-with-selection.md +171 -0
- package/docs/guides/delete-old-run-history.md +154 -0
- package/docs/guides/drive-a-plan.md +314 -0
- package/docs/guides/observe-engine-metrics.md +98 -0
- package/docs/guides/observe-executions.md +187 -0
- package/docs/guides/reclaim-runs-from-a-dead-host.md +133 -0
- package/docs/guides/share-a-cache-across-machines.md +134 -0
- package/docs/guides/testing.md +123 -0
- package/docs/installation.md +102 -0
- package/docs/quickstart.md +215 -0
- package/docs/troubleshooting.md +347 -0
- package/package.json +438 -3
- package/src/ArtifactGc.ts +408 -0
- package/src/ArtifactSync.ts +278 -0
- package/src/CacheSync.ts +131 -0
- package/src/DeferredClockFold.ts +586 -0
- package/src/DisasterRecovery.ts +888 -0
- package/src/DurableEngineState.ts +2656 -0
- package/src/EngineStore.ts +656 -0
- package/src/EngineStoreMetrics.ts +331 -0
- package/src/Errors.ts +28 -0
- package/src/EventTypes.ts +27 -0
- package/src/ExecutionFacts.ts +125 -0
- package/src/ExecutionSnapshot.ts +186 -0
- package/src/Inconsistency.ts +253 -0
- package/src/Migrations.ts +95 -0
- package/src/OwnerIdentity.ts +125 -0
- package/src/PlanInputStore.ts +292 -0
- package/src/PlanMergeStore.ts +330 -0
- package/src/PlanScheduler.ts +1806 -0
- package/src/Reconciliation.ts +224 -0
- package/src/ReplayOnly.ts +95 -0
- package/src/Retention.ts +214 -0
- package/src/RunCatalogRead.ts +27 -0
- package/src/RunChangeFeed.ts +125 -0
- package/src/RunState.ts +137 -0
- package/src/Selection.ts +683 -0
- package/src/SelectionStore.ts +353 -0
- package/src/StepBoundary.ts +1165 -0
- package/src/StepCacheFold.ts +244 -0
- package/src/StepSandbox.ts +110 -0
- package/src/WakeBus.ts +167 -0
- package/src/WorkspaceSandbox.ts +1604 -0
- package/src/index.ts +185 -0
- package/src/internal/ActionPersistence.ts +2687 -0
- package/src/internal/ArtifactRoots.ts +92 -0
- package/src/internal/AttemptAdmission.ts +76 -0
- package/src/internal/AttemptLifecycle.ts +163 -0
- package/src/internal/AttemptProbe.ts +90 -0
- package/src/internal/CacheAdmission.ts +133 -0
- package/src/internal/CacheAgeHistory.ts +58 -0
- package/src/internal/CacheAgeVerdicts.ts +61 -0
- package/src/internal/CacheOutputPolicy.ts +82 -0
- package/src/internal/CachePublication.ts +153 -0
- package/src/internal/CallFacts.ts +87 -0
- package/src/internal/CopiedRecord.ts +201 -0
- package/src/internal/DeferredPersistence.ts +459 -0
- package/src/internal/EffectRecords.ts +136 -0
- package/src/internal/EngineJj.ts +30 -0
- package/src/internal/EngineStateSchema.ts +92 -0
- package/src/internal/ExecutionSnapshotRead.ts +298 -0
- package/src/internal/ExecutionSnapshotRelations.ts +121 -0
- package/src/internal/ExitEncoding.ts +453 -0
- package/src/internal/FileBoundarySnapshot.ts +39 -0
- package/src/internal/FileEnumeration.ts +334 -0
- package/src/internal/HostReflection.ts +128 -0
- package/src/internal/JournalRecords.ts +396 -0
- package/src/internal/NodeJournal.ts +340 -0
- package/src/internal/Ordering.ts +14 -0
- package/src/internal/PlanInputs.ts +62 -0
- package/src/internal/PlanMerges.ts +123 -0
- package/src/internal/ProvenanceSlot.ts +52 -0
- package/src/internal/ResultEnvelope.ts +118 -0
- package/src/internal/RetentionOps.ts +865 -0
- package/src/internal/RunCatalogOps.ts +174 -0
- package/src/internal/RunCoordinator.ts +215 -0
- package/src/internal/RunDriver.ts +3152 -0
- package/src/internal/RunLifecycle.ts +53 -0
- package/src/internal/RunListing.ts +210 -0
- package/src/internal/RunListingError.ts +37 -0
- package/src/internal/RuntimeGraph.ts +253 -0
- package/src/internal/SandboxedExecution.ts +165 -0
- package/src/internal/StateTransaction.ts +71 -0
- package/src/internal/StepFacts.ts +36 -0
- package/src/internal/TypedEvents.ts +29 -0
- package/src/migrations/0001_initial.ts +51 -0
- package/src/migrations/0002_selection_store.ts +38 -0
- package/src/migrations/0003_plan_inputs.ts +75 -0
- package/src/migrations/0004_plan_environment.ts +34 -0
- package/src/migrations/0005_plan_merges.ts +60 -0
- package/src/migrations/0006_execution_listing.ts +86 -0
- package/src/migrations/0007_run_parent_sequence.ts +19 -0
- package/src/migrations/0008_deferred_consumption.ts +27 -0
- package/src/migrations/0009_deferred_clock_facts.ts +163 -0
- package/src/migrations/0010_pending_cancellation.ts +20 -0
- package/src/test/TestStores.ts +145 -0
|
@@ -0,0 +1,3152 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Claim-gated durable flow run lifecycle.
|
|
3
|
+
*
|
|
4
|
+
* @since 0.1.0
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import * as CapabilitySet from "@smthrs/capability/CapabilitySet"
|
|
8
|
+
import { DatabaseError, DurableWriter } from "@smthrs/database/DurableWriter"
|
|
9
|
+
import { FlowEngine } from "@smthrs/engine"
|
|
10
|
+
import { Flow, FlowRuntime } from "@smthrs/flow"
|
|
11
|
+
import { Journal } from "@smthrs/journal"
|
|
12
|
+
import { Ownership, RunStore } from "@smthrs/run-store"
|
|
13
|
+
import * as Cause from "effect/Cause"
|
|
14
|
+
import * as Clock from "effect/Clock"
|
|
15
|
+
import type * as Crypto from "effect/Crypto"
|
|
16
|
+
import * as Deferred from "effect/Deferred"
|
|
17
|
+
import * as Duration from "effect/Duration"
|
|
18
|
+
import * as Effect from "effect/Effect"
|
|
19
|
+
import * as Exit from "effect/Exit"
|
|
20
|
+
import * as Metric from "effect/Metric"
|
|
21
|
+
import * as Option from "effect/Option"
|
|
22
|
+
import * as Result from "effect/Result"
|
|
23
|
+
import * as Schema from "effect/Schema"
|
|
24
|
+
import * as Scope from "effect/Scope"
|
|
25
|
+
import * as DurableEngineState from "../DurableEngineState.ts"
|
|
26
|
+
import * as EngineStoreMetrics from "../EngineStoreMetrics.ts"
|
|
27
|
+
import { EventTypes } from "../EventTypes.ts"
|
|
28
|
+
import * as ExecutionFacts from "../ExecutionFacts.ts"
|
|
29
|
+
import { type OnParentExit, type ReleaseCause, RunState } from "../RunState.ts"
|
|
30
|
+
import * as WakeBus from "../WakeBus.ts"
|
|
31
|
+
import * as ActionPersistence from "./ActionPersistence.ts"
|
|
32
|
+
import * as EffectRecords from "./EffectRecords.ts"
|
|
33
|
+
import * as ExitEncoding from "./ExitEncoding.ts"
|
|
34
|
+
import * as JournalRecords from "./JournalRecords.ts"
|
|
35
|
+
import * as RunCoordinator from "./RunCoordinator.ts"
|
|
36
|
+
import * as RunLifecycle from "./RunLifecycle.ts"
|
|
37
|
+
import * as StateTransaction from "./StateTransaction.ts"
|
|
38
|
+
|
|
39
|
+
const RunStateJson = Schema.fromJsonString(RunState)
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Re-channels every typed reason of a cause as a defect, identity intact.
|
|
43
|
+
*
|
|
44
|
+
* A boundary whose declared error type cannot describe what happened has two
|
|
45
|
+
* honest options: widen the type, or stop calling the failure typed. Casting
|
|
46
|
+
* the cause is the third, dishonest one — it left a raw `DatabaseError`
|
|
47
|
+
* escaping an `Effect<void, CancelRequestFailed>`. Demotion keeps the error
|
|
48
|
+
* object, every other reason and every interruption, and no caller can
|
|
49
|
+
* mistake the result for a failure its handler declared.
|
|
50
|
+
*/
|
|
51
|
+
const asDefects = <E>(cause: Cause.Cause<E>): Cause.Cause<never> =>
|
|
52
|
+
Cause.fromReasons(
|
|
53
|
+
cause.reasons.map((reason) =>
|
|
54
|
+
Cause.isFailReason(reason)
|
|
55
|
+
? Cause.makeDieReason(reason.error).annotate(Cause.reasonAnnotations(reason))
|
|
56
|
+
: reason
|
|
57
|
+
)
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Raised when a flow (directly or through mutual ancestry) attempts to
|
|
62
|
+
* execute an execution id that already appears in its own persisted
|
|
63
|
+
* `parentExecutionId` chain.
|
|
64
|
+
*
|
|
65
|
+
* Detection walks the already-persisted parent chain from the requesting
|
|
66
|
+
* parent upward — an O(depth) check, not a dependency-graph DFS — because
|
|
67
|
+
* `parentExecutionId` is the only edge our runtime model can express.
|
|
68
|
+
*
|
|
69
|
+
* The class is declared by `@smthrs/flow` (it is part of the `execute`
|
|
70
|
+
* contract) and re-exported here for the detector's callers.
|
|
71
|
+
*
|
|
72
|
+
* @since 0.1.0
|
|
73
|
+
* @category errors
|
|
74
|
+
*/
|
|
75
|
+
export const FlowCycleDetected = FlowRuntime.FlowCycleDetected
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* The value form of {@link FlowCycleDetected}.
|
|
79
|
+
*
|
|
80
|
+
* @since 0.1.0
|
|
81
|
+
* @category errors
|
|
82
|
+
*/
|
|
83
|
+
export type FlowCycleDetected = FlowRuntime.FlowCycleDetected
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* Why the engine itself asked a parked execution to resume.
|
|
87
|
+
*
|
|
88
|
+
* The operator's own resume is absent on purpose: it is the one wake a host
|
|
89
|
+
* has already claimed before the engine hears about it.
|
|
90
|
+
*
|
|
91
|
+
* @since 0.1.0
|
|
92
|
+
* @category models
|
|
93
|
+
*/
|
|
94
|
+
export type RequestedResumeReason = "deferred" | "clock" | "parent"
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* Why a wake is scheduled for an execution.
|
|
98
|
+
*
|
|
99
|
+
* Only `operator` is recovery consent for a quarantine park. `delegated` is a
|
|
100
|
+
* host taking up a control delegation it has already claimed, so, like
|
|
101
|
+
* `operator`, it is not recorded back to the host as a request.
|
|
102
|
+
*
|
|
103
|
+
* @since 1.0.0
|
|
104
|
+
* @category models
|
|
105
|
+
*/
|
|
106
|
+
export type ScheduledResumeReason = RequestedResumeReason | "operator" | "delegated"
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* Dependencies for the run driver.
|
|
110
|
+
*
|
|
111
|
+
* @since 0.1.0
|
|
112
|
+
* @category models
|
|
113
|
+
*/
|
|
114
|
+
export interface Dependencies {
|
|
115
|
+
readonly owner: Ownership.OwnerId
|
|
116
|
+
readonly journalSource: string
|
|
117
|
+
/**
|
|
118
|
+
* Liveness arbitration before a steal, consulted only for a run whose lease
|
|
119
|
+
* has already expired. Optional: the default is
|
|
120
|
+
* `Ownership.leaseLiveness(Ownership.heartbeatStaleAfter)`, which has no
|
|
121
|
+
* evidence beyond the lease and therefore reports the owner gone, so a
|
|
122
|
+
* composition that supplies nothing still reclaims hard-killed runs.
|
|
123
|
+
*/
|
|
124
|
+
readonly isAlive?: Ownership.LivenessCheck | undefined
|
|
125
|
+
readonly canExecute?: ((row: RunStore.RunRow) => Effect.Effect<boolean>) | undefined
|
|
126
|
+
readonly canActivate?: ((row: RunStore.RunRow) => Effect.Effect<boolean>) | undefined
|
|
127
|
+
/**
|
|
128
|
+
* Records, for a host that keeps one, that this driver has asked a
|
|
129
|
+
* SUSPENDED execution to resume.
|
|
130
|
+
*
|
|
131
|
+
* Every wake the engine itself initiates passes through here: a durable
|
|
132
|
+
* clock firing, a durable deferred completing (a signal, a queue offer, a
|
|
133
|
+
* registration-time sweep of completions taken while the host was down),
|
|
134
|
+
* and a child settling under a parent that parked on it. A host that guards
|
|
135
|
+
* re-entry into a parked run — `AgentSession` refuses a round for a parked
|
|
136
|
+
* control row that nobody asked for — otherwise cannot tell those from its
|
|
137
|
+
* own heartbeat sweep, and a run that slept never settled.
|
|
138
|
+
*
|
|
139
|
+
* An operator resume is NOT recorded: `Control.resume` and
|
|
140
|
+
* `ControlLive.decide` claim the control row themselves, and recording a
|
|
141
|
+
* second request for the resume a host is already taking up would buy that
|
|
142
|
+
* park a second re-drive.
|
|
143
|
+
*
|
|
144
|
+
* Optional. A composition with no such host leaves it undefined and the
|
|
145
|
+
* wake is exactly what it was.
|
|
146
|
+
*/
|
|
147
|
+
readonly requestResume?:
|
|
148
|
+
| ((executionId: string, reason: RequestedResumeReason) => Effect.Effect<void>)
|
|
149
|
+
| undefined
|
|
150
|
+
readonly engine: Effect.Effect<FlowRuntime.FlowRuntime["Service"]>
|
|
151
|
+
/**
|
|
152
|
+
* In-process wake bus announced to whenever a durable write makes a run
|
|
153
|
+
* runnable — a scheduled resume (deferred, clock, operator) and a run
|
|
154
|
+
* settling terminally. Optional so a direct construction without one keeps
|
|
155
|
+
* the pre-existing polling-only behavior: the default drops every wake,
|
|
156
|
+
* which the engine's suspension polling schedule already covers.
|
|
157
|
+
*/
|
|
158
|
+
readonly wakeBus?: WakeBus.Service | undefined
|
|
159
|
+
/**
|
|
160
|
+
* Runs inside the uninterruptible finalizer that retains a suspended run's
|
|
161
|
+
* flow scope, immediately after the entry is inserted and before the round's
|
|
162
|
+
* settlement can start.
|
|
163
|
+
*
|
|
164
|
+
* Private to `packages/smithers/flows/engine-store/test`. That boundary is where a pending
|
|
165
|
+
* interruption first becomes observable again, and no public seam reaches
|
|
166
|
+
* it, so the regression proving the retention is released there — rather
|
|
167
|
+
* than stranded until the driver's own scope closes — raises the
|
|
168
|
+
* interruption from here. This module is not exported from the package
|
|
169
|
+
* (`"./internal/*": null`), and production compositions leave the hook
|
|
170
|
+
* undefined, which costs one `undefined` check per suspension.
|
|
171
|
+
*/
|
|
172
|
+
readonly unsafeOnScopeRetained?: ((runId: string) => Effect.Effect<void>) | undefined
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/**
|
|
176
|
+
* Claim-gated operations composed into the encoded flow engine.
|
|
177
|
+
*
|
|
178
|
+
* @since 0.1.0
|
|
179
|
+
* @category models
|
|
180
|
+
*/
|
|
181
|
+
export interface Service {
|
|
182
|
+
readonly register: FlowEngine.Encoded["register"]
|
|
183
|
+
/**
|
|
184
|
+
* The encoded `execute`, widened for direct driver callers: an absent
|
|
185
|
+
* `round` admits the execution as round zero of its own lineage. Nested
|
|
186
|
+
* direct callers observe its latest round; encoded callers read one round.
|
|
187
|
+
* To decode a terminal result, direct nested callers must register the
|
|
188
|
+
* terminal flow's codec in this driver, even if another worker ran it.
|
|
189
|
+
* A missing codec is a wiring defect, as with the typed Trampoline; it
|
|
190
|
+
* fails fast rather than suspending terminal work with no future wake.
|
|
191
|
+
*/
|
|
192
|
+
readonly execute: <const Discard extends boolean>(
|
|
193
|
+
flow: Flow.Any,
|
|
194
|
+
options: {
|
|
195
|
+
readonly executionId: string
|
|
196
|
+
readonly payload: object
|
|
197
|
+
readonly discard: Discard
|
|
198
|
+
readonly parent?: FlowRuntime.FlowInstance["Service"] | undefined
|
|
199
|
+
readonly round?: Parameters<FlowEngine.Encoded["execute"]>[1]["round"] | undefined
|
|
200
|
+
readonly follow?: boolean | undefined
|
|
201
|
+
}
|
|
202
|
+
) => Effect.Effect<Discard extends true ? void : Flow.Result<unknown, unknown>, FlowCycleDetected>
|
|
203
|
+
readonly poll: FlowEngine.Encoded["poll"]
|
|
204
|
+
readonly interrupt: FlowEngine.Encoded["interrupt"]
|
|
205
|
+
readonly interruptUnsafe: FlowEngine.Encoded["interruptUnsafe"]
|
|
206
|
+
readonly resume: FlowEngine.Encoded["resume"]
|
|
207
|
+
readonly scheduleResume: (
|
|
208
|
+
flowName: string,
|
|
209
|
+
executionId: string,
|
|
210
|
+
reason: ScheduledResumeReason,
|
|
211
|
+
sourceId?: string | undefined
|
|
212
|
+
) => Effect.Effect<void>
|
|
213
|
+
readonly active: Effect.Effect<ReadonlySet<string>>
|
|
214
|
+
/**
|
|
215
|
+
* The runs whose parked flow scope this driver still holds. Cancellation
|
|
216
|
+
* and re-drive both release; the set is the leak assertion's observation
|
|
217
|
+
* point.
|
|
218
|
+
*/
|
|
219
|
+
readonly retainedRuns: Effect.Effect<ReadonlySet<string>>
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
interface Registration {
|
|
223
|
+
readonly flow: Flow.Any
|
|
224
|
+
readonly execute: (
|
|
225
|
+
payload: object,
|
|
226
|
+
executionId: string
|
|
227
|
+
) => Effect.Effect<unknown, unknown, FlowRuntime.FlowInstance | FlowRuntime.FlowRuntime>
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/**
|
|
231
|
+
* The effect kind a detached child spawn is journaled under, and therefore the
|
|
232
|
+
* kind an engine composition registers a spawn-compensation handler for.
|
|
233
|
+
*
|
|
234
|
+
* @since 0.1.0
|
|
235
|
+
* @category constants
|
|
236
|
+
*/
|
|
237
|
+
export const spawnEffectKind = EventTypes.childSpawnKind
|
|
238
|
+
|
|
239
|
+
const snapshot = (row: RunStore.RunRow): RunStore.RunSnapshot => ({
|
|
240
|
+
status: row.status,
|
|
241
|
+
owner: row.owner,
|
|
242
|
+
heartbeatAtMs: row.heartbeatAtMs
|
|
243
|
+
})
|
|
244
|
+
|
|
245
|
+
const isPlainObject = (value: unknown): value is Record<string, unknown> => {
|
|
246
|
+
if (value === null || typeof value !== "object") return false
|
|
247
|
+
const prototype = Object.getPrototypeOf(value)
|
|
248
|
+
return prototype === Object.prototype || prototype === null
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
/**
|
|
252
|
+
* Compares encoded JSON payloads structurally. Object key order is not part
|
|
253
|
+
* of JSON identity, while array order and array length are.
|
|
254
|
+
*
|
|
255
|
+
* Values outside JSON compare unequal unless both arguments are the same
|
|
256
|
+
* reference.
|
|
257
|
+
*
|
|
258
|
+
* @since 1.0.0
|
|
259
|
+
* @category guards
|
|
260
|
+
* @private
|
|
261
|
+
*/
|
|
262
|
+
export const samePayload = (left: unknown, right: unknown): boolean => {
|
|
263
|
+
if (Object.is(left, right)) return true
|
|
264
|
+
if (typeof left === "number" && typeof right === "number") {
|
|
265
|
+
return left === 0 && right === 0
|
|
266
|
+
}
|
|
267
|
+
if (Array.isArray(left)) {
|
|
268
|
+
if (!Array.isArray(right) || left.length !== right.length) return false
|
|
269
|
+
for (let index = 0; index < left.length; index++) {
|
|
270
|
+
if (!samePayload(left[index], right[index])) return false
|
|
271
|
+
}
|
|
272
|
+
return true
|
|
273
|
+
}
|
|
274
|
+
if (Array.isArray(right) || !isPlainObject(left) || !isPlainObject(right)) return false
|
|
275
|
+
const leftKeys = Object.keys(left)
|
|
276
|
+
const rightKeys = Object.keys(right)
|
|
277
|
+
if (leftKeys.length !== rightKeys.length) return false
|
|
278
|
+
for (const key of leftKeys) {
|
|
279
|
+
if (!Object.hasOwn(right, key) || !samePayload(left[key], right[key])) return false
|
|
280
|
+
}
|
|
281
|
+
return true
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
/**
|
|
285
|
+
* Sentinel produced when the cancel-request poll observes a durable
|
|
286
|
+
* cancellation before the flow settles.
|
|
287
|
+
*/
|
|
288
|
+
const cancelRequested = { _tag: "CancelRequested" } as const
|
|
289
|
+
|
|
290
|
+
/**
|
|
291
|
+
* How many stale-running rows one heartbeat tick may wake (issue #79).
|
|
292
|
+
* Oldest heartbeats surface first, so a backlog larger than the batch
|
|
293
|
+
* drains across ticks; each successful steal removes the row from the
|
|
294
|
+
* stale window, and losing drivers see a shrinking batch next tick.
|
|
295
|
+
*/
|
|
296
|
+
const staleRunningSweepBatch = 64
|
|
297
|
+
|
|
298
|
+
/**
|
|
299
|
+
* How long a run waits before its live owner is probed again, and how far
|
|
300
|
+
* that wait doubles.
|
|
301
|
+
*
|
|
302
|
+
* A refusal is evidence: the recorded owner answered, so the one thing the
|
|
303
|
+
* sweep knows is that re-asking immediately is waste. Without a wait the
|
|
304
|
+
* oldest batch is re-probed every heartbeat forever and nothing behind it is
|
|
305
|
+
* ever reached (issue #79 follow-up). The first wait is two ticks, which is
|
|
306
|
+
* already enough for the batch to move on, and it doubles up to the lease
|
|
307
|
+
* window: an owner that stops heartbeating is therefore re-probed within one
|
|
308
|
+
* `heartbeatStaleAfter` of the last refusal, which is the same window every
|
|
309
|
+
* other reclaim decision is made against.
|
|
310
|
+
*/
|
|
311
|
+
const stealRefusalBackoffFirst = Duration.times(Ownership.heartbeatInterval, 2)
|
|
312
|
+
const stealRefusalBackoffMax = Ownership.heartbeatStaleAfter
|
|
313
|
+
|
|
314
|
+
/**
|
|
315
|
+
* A refusal this driver remembers: which lease it was refused against, when
|
|
316
|
+
* the owner may be probed again, and how many consecutive refusals that
|
|
317
|
+
* lease has produced (the backoff exponent).
|
|
318
|
+
*
|
|
319
|
+
* Keyed by run id and holding the heartbeat it was collected against, so an
|
|
320
|
+
* owner that resumes heartbeating — a new lease — restarts the ladder rather
|
|
321
|
+
* than inheriting a long wait from the stall that preceded it.
|
|
322
|
+
*/
|
|
323
|
+
interface StealRefusal {
|
|
324
|
+
readonly heartbeatAtMs: number | null
|
|
325
|
+
readonly nextProbeAtMs: number
|
|
326
|
+
readonly attempt: number
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
/**
|
|
330
|
+
* The type of {@link cancelRequested}: the settlement a round takes when the
|
|
331
|
+
* cancel-request poll wins the race against the flow.
|
|
332
|
+
*/
|
|
333
|
+
type CancelRequested = typeof cancelRequested
|
|
334
|
+
|
|
335
|
+
const withoutResult = (state: RunState): RunState => {
|
|
336
|
+
const { cancellation: _, result: __, ...rest } = state
|
|
337
|
+
return rest
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
/**
|
|
341
|
+
* Constructs a scoped run driver.
|
|
342
|
+
*
|
|
343
|
+
* Every start and wake enters the same keyed coordinator and then the same
|
|
344
|
+
* exact-snapshot claim/activation path.
|
|
345
|
+
*
|
|
346
|
+
* @since 0.1.0
|
|
347
|
+
* @category constructors
|
|
348
|
+
*/
|
|
349
|
+
export const make = (
|
|
350
|
+
dependencies: Dependencies
|
|
351
|
+
): Effect.Effect<
|
|
352
|
+
Service,
|
|
353
|
+
never,
|
|
354
|
+
| Crypto.Crypto
|
|
355
|
+
| DurableEngineState.DurableEngineState
|
|
356
|
+
| DurableWriter
|
|
357
|
+
| Journal.Journal
|
|
358
|
+
| RunStore.RunStore
|
|
359
|
+
| Scope.Scope
|
|
360
|
+
> =>
|
|
361
|
+
Effect.gen(function*() {
|
|
362
|
+
const journal = yield* Journal.Journal
|
|
363
|
+
const writer = yield* DurableWriter
|
|
364
|
+
const store = yield* RunStore.RunStore
|
|
365
|
+
const engineState = yield* DurableEngineState.DurableEngineState
|
|
366
|
+
const lifecycle = RunLifecycle.make(store, engineState)
|
|
367
|
+
const wakeBus = dependencies.wakeBus ?? WakeBus.makeNoop()
|
|
368
|
+
/**
|
|
369
|
+
* The deployment's own liveness answer, if it gave one.
|
|
370
|
+
*
|
|
371
|
+
* Kept separate from {@link isAlive} because the two differ in what they
|
|
372
|
+
* PROVE, and the journal says which was used: a supplied probe reports on
|
|
373
|
+
* the process (or the host it lives on), while the default reports only
|
|
374
|
+
* that the lease it stopped renewing has expired.
|
|
375
|
+
*/
|
|
376
|
+
const suppliedIsAlive = dependencies.isAlive
|
|
377
|
+
const isAlive: Ownership.LivenessCheck = suppliedIsAlive ??
|
|
378
|
+
Ownership.leaseLiveness(Ownership.heartbeatStaleAfter)
|
|
379
|
+
/**
|
|
380
|
+
* The evidence kind a steal admitted by {@link isAlive} carries.
|
|
381
|
+
*
|
|
382
|
+
* `steal` is only reached for a run whose lease has already expired, so
|
|
383
|
+
* `lease-expired` is always true here; what the supplied-probe kinds add
|
|
384
|
+
* is WHY the deployment believes the owner is gone, which is the
|
|
385
|
+
* distinction `RunStore` binds to the host relation.
|
|
386
|
+
*/
|
|
387
|
+
const livenessEvidenceKind = (
|
|
388
|
+
expectedOwner: Ownership.OwnerId
|
|
389
|
+
): Ownership.LivenessEvidence["kind"] =>
|
|
390
|
+
suppliedIsAlive === undefined
|
|
391
|
+
? "lease-expired"
|
|
392
|
+
: Ownership.sameHostIncarnation(expectedOwner, dependencies.owner)
|
|
393
|
+
? "same-host-pid-dead"
|
|
394
|
+
: "cross-host-unreachable-stale"
|
|
395
|
+
/** Owner identity equality, the fence's own notion of "the same process". */
|
|
396
|
+
const sameOwner = (left: Ownership.OwnerId, right: Ownership.OwnerId): boolean =>
|
|
397
|
+
left.hostId === right.hostId && left.pid === right.pid && left.nonce === right.nonce
|
|
398
|
+
const registrations = new Map<string, Array<Registration>>()
|
|
399
|
+
/**
|
|
400
|
+
* Runs already warned about waking without a registered flow (issue
|
|
401
|
+
* #62): the sweep retries every heartbeat, so the warning is emitted
|
|
402
|
+
* once per run, not once per tick. Cleared on every registration — a
|
|
403
|
+
* newly registered flow makes previously dropped runs drivable again.
|
|
404
|
+
*/
|
|
405
|
+
const warnedUnregistered = new Set<string>()
|
|
406
|
+
/**
|
|
407
|
+
* Runs whose owner answered a probe (or held a fresh lease) and are
|
|
408
|
+
* therefore not worth re-arbitrating yet. See {@link StealRefusal}.
|
|
409
|
+
*
|
|
410
|
+
* Bounded by the stale set the sweep can see: every tick drops the entries
|
|
411
|
+
* whose rows are no longer in the window it read, so a run that is stolen,
|
|
412
|
+
* released, or settled takes its entry with it.
|
|
413
|
+
*/
|
|
414
|
+
const stealRefusals = new Map<string, StealRefusal>()
|
|
415
|
+
|
|
416
|
+
/**
|
|
417
|
+
* Records that a run's recorded owner was found alive, and when it may be
|
|
418
|
+
* asked about again.
|
|
419
|
+
*/
|
|
420
|
+
const deferSteal = (
|
|
421
|
+
runId: string,
|
|
422
|
+
heartbeatAtMs: number | null,
|
|
423
|
+
nowMs: number
|
|
424
|
+
): void => {
|
|
425
|
+
const previous = stealRefusals.get(runId)
|
|
426
|
+
// A moved lease is new evidence about a different state of the world, so
|
|
427
|
+
// it starts the ladder over instead of inheriting the old wait.
|
|
428
|
+
const attempt = previous !== undefined && previous.heartbeatAtMs === heartbeatAtMs ? previous.attempt + 1 : 1
|
|
429
|
+
const waitMs = Math.min(
|
|
430
|
+
Duration.toMillis(stealRefusalBackoffFirst) * Math.pow(2, attempt - 1),
|
|
431
|
+
Duration.toMillis(stealRefusalBackoffMax)
|
|
432
|
+
)
|
|
433
|
+
stealRefusals.set(runId, { heartbeatAtMs, nextProbeAtMs: nowMs + waitMs, attempt })
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
/**
|
|
437
|
+
* The journal address of one refusal.
|
|
438
|
+
*
|
|
439
|
+
* Refusals were emitted through the driver's default source id with no
|
|
440
|
+
* `sourceSeq`, so nothing deduplicated them: a live owner cost one durable
|
|
441
|
+
* journal row per refused run per heartbeat, indefinitely. Addressing the
|
|
442
|
+
* record by the run, the owner, and the lease it was refused against makes
|
|
443
|
+
* the journal's own first-writer admission the deduplication — the same
|
|
444
|
+
* mechanism the registration-time wake records use — so a repeated refusal
|
|
445
|
+
* about an unchanged lease says nothing twice.
|
|
446
|
+
*
|
|
447
|
+
* The evidence is part of the address because one lease can be refused for
|
|
448
|
+
* two different reasons, and they are different records. A wake that
|
|
449
|
+
* arrives while the owner is still pulsing is refused by the lease alone
|
|
450
|
+
* (`lease-fresh`); the same owner, still alive but stalled past the
|
|
451
|
+
* window, is refused by the probe (`probe`). Addressing both to one key
|
|
452
|
+
* made the second write a different payload under the first one's address,
|
|
453
|
+
* which the journal rejects as `idempotency_conflict` — a defect that
|
|
454
|
+
* killed the drive fiber, dropped the probe refusal, and took down any
|
|
455
|
+
* `execute` joined to that run. One address per (run, owner, lease,
|
|
456
|
+
* evidence) keeps every reason sayable and still admits each of them once.
|
|
457
|
+
*
|
|
458
|
+
* That makes this driver's own payload a pure function of its address, so
|
|
459
|
+
* it can no longer conflict with itself. It is not the only writer at
|
|
460
|
+
* these addresses, and a conflict raised by another one is no longer fatal
|
|
461
|
+
* either: refusals are emitted through {@link emitEvidence}.
|
|
462
|
+
*/
|
|
463
|
+
const stealRefusedSourceId = (
|
|
464
|
+
owner: Ownership.OwnerId,
|
|
465
|
+
heartbeatAtMs: number | null,
|
|
466
|
+
evidence: "lease-fresh" | "probe"
|
|
467
|
+
): string =>
|
|
468
|
+
`${dependencies.journalSource}:steal-refused:${evidence}:${
|
|
469
|
+
JSON.stringify([owner.hostId, owner.pid, owner.nonce])
|
|
470
|
+
}:${heartbeatAtMs}`
|
|
471
|
+
const liveInstances = new Map<string, FlowRuntime.FlowInstance["Service"]>()
|
|
472
|
+
/**
|
|
473
|
+
* Flow scopes retained past a round's settlement.
|
|
474
|
+
*
|
|
475
|
+
* `Flow.intoResult` closes the flow scope for every settlement EXCEPT a
|
|
476
|
+
* suspension — a parked flow keeps its scope so the resources its body
|
|
477
|
+
* acquired outlive the park (`packages/smithers/flows/flow/src/Flow/Runtime.ts`). Nothing
|
|
478
|
+
* then owned that scope: the instance left `liveInstances` on the way out
|
|
479
|
+
* and the `Scope.Closeable` became unreachable, so a cancelled parked flow
|
|
480
|
+
* never ran its finalizers and every park leaked one scope for the
|
|
481
|
+
* process's lifetime.
|
|
482
|
+
*
|
|
483
|
+
* The map is that owner. It holds at most one scope per run — an entry is
|
|
484
|
+
* a state, not a queue — and every removal goes through
|
|
485
|
+
* `releaseRetainedScope`, which deletes before it closes. That makes the
|
|
486
|
+
* close idempotent by construction rather than by a flag: a second caller
|
|
487
|
+
* finds no entry and does nothing, so concurrent cancel, resume, and
|
|
488
|
+
* shutdown paths cannot double-finalize.
|
|
489
|
+
*/
|
|
490
|
+
const retainedScopes = new Map<string, Scope.Closeable>()
|
|
491
|
+
|
|
492
|
+
/**
|
|
493
|
+
* Closes a run's retained flow scope exactly once, if it has one.
|
|
494
|
+
*
|
|
495
|
+
* `exit` decides what the finalizers see. A cancellation closes with an
|
|
496
|
+
* interrupt exit so `Flow.withRollback` compensations run; a supersede
|
|
497
|
+
* (the run re-entered execution under a fresh instance and scope) closes
|
|
498
|
+
* with `Exit.void` so ordinary finalizers release resources while
|
|
499
|
+
* rollbacks correctly stay out — a re-driven round is not an unsuccessful
|
|
500
|
+
* exit.
|
|
501
|
+
*/
|
|
502
|
+
const releaseRetainedScope = (
|
|
503
|
+
runId: string,
|
|
504
|
+
exit: Exit.Exit<unknown, unknown>
|
|
505
|
+
): Effect.Effect<void> =>
|
|
506
|
+
Effect.suspend(() => {
|
|
507
|
+
const scope = retainedScopes.get(runId)
|
|
508
|
+
if (scope === undefined) return Effect.void
|
|
509
|
+
retainedScopes.delete(runId)
|
|
510
|
+
return Scope.close(scope, exit).pipe(
|
|
511
|
+
Effect.catchCause((cause) =>
|
|
512
|
+
Effect.logWarning(
|
|
513
|
+
`engine-store: finalizers of parked run ${runId} failed while closing its retained flow scope`,
|
|
514
|
+
cause
|
|
515
|
+
)
|
|
516
|
+
)
|
|
517
|
+
)
|
|
518
|
+
})
|
|
519
|
+
|
|
520
|
+
/**
|
|
521
|
+
* Snapshots the runs whose flow scope this driver is currently holding.
|
|
522
|
+
* Exposed for the leak assertions in `packages/smithers/flows/engine-store/test`, which
|
|
523
|
+
* would otherwise have to reach into driver internals.
|
|
524
|
+
*/
|
|
525
|
+
const retainedRuns = Effect.sync(() => new Set(retainedScopes.keys()))
|
|
526
|
+
|
|
527
|
+
// Process shutdown must not strand parked scopes either: closing the
|
|
528
|
+
// driver's own scope releases every scope still retained. `Exit.void`,
|
|
529
|
+
// because a shutdown is a reclaimable release (issue #26) and not a
|
|
530
|
+
// cancellation, so compensations must not fire.
|
|
531
|
+
yield* Effect.addFinalizer(() =>
|
|
532
|
+
Effect.forEach(
|
|
533
|
+
Array.from(retainedScopes.keys()),
|
|
534
|
+
(runId) => releaseRetainedScope(runId, Exit.void),
|
|
535
|
+
{ discard: true }
|
|
536
|
+
)
|
|
537
|
+
)
|
|
538
|
+
const encodeState = (state: RunState): Effect.Effect<string> =>
|
|
539
|
+
Schema.encodeEffect(RunStateJson)(state).pipe(Effect.orDie)
|
|
540
|
+
|
|
541
|
+
const decodeState = (stateJson: string): Effect.Effect<RunState> =>
|
|
542
|
+
Schema.decodeUnknownEffect(RunStateJson)(stateJson).pipe(Effect.orDie)
|
|
543
|
+
|
|
544
|
+
/**
|
|
545
|
+
* Tells the host that this driver is asking a PARKED execution to resume.
|
|
546
|
+
*
|
|
547
|
+
* Only a parked one. A wake against a run that is still running is
|
|
548
|
+
* already covered by the round it is in, and a request recorded for it
|
|
549
|
+
* would be a delegation no park ever takes up: the host polls it for as
|
|
550
|
+
* long as the run lives, and every poll pays a park wait for a run that
|
|
551
|
+
* is not parked.
|
|
552
|
+
*
|
|
553
|
+
* See {@link Dependencies.requestResume} for which wakes reach here.
|
|
554
|
+
*/
|
|
555
|
+
const recordResume = (
|
|
556
|
+
executionId: string,
|
|
557
|
+
parked: boolean,
|
|
558
|
+
reason: RequestedResumeReason
|
|
559
|
+
): Effect.Effect<void> =>
|
|
560
|
+
dependencies.requestResume === undefined || !parked
|
|
561
|
+
? Effect.void
|
|
562
|
+
: dependencies.requestResume(executionId, reason)
|
|
563
|
+
|
|
564
|
+
/**
|
|
565
|
+
* {@link recordResume} for the parent a settling child is waking.
|
|
566
|
+
*
|
|
567
|
+
* The waiting row is the park: it is written in the same transaction as
|
|
568
|
+
* the suspended transition and cleared by the claim that ends it, so a
|
|
569
|
+
* parent that has one is parked and a parent mid-round has none. It is
|
|
570
|
+
* read only for a host that records, so a composition without one pays
|
|
571
|
+
* nothing per child settlement.
|
|
572
|
+
*/
|
|
573
|
+
const recordParentResume = (parentExecutionId: string): Effect.Effect<void> =>
|
|
574
|
+
dependencies.requestResume === undefined ? Effect.void : Effect.flatMap(
|
|
575
|
+
engineState.waiting(parentExecutionId),
|
|
576
|
+
(waiting) =>
|
|
577
|
+
recordResume(
|
|
578
|
+
parentExecutionId,
|
|
579
|
+
Option.isSome(waiting) && waiting.value.reason !== "quarantine",
|
|
580
|
+
"parent"
|
|
581
|
+
)
|
|
582
|
+
)
|
|
583
|
+
|
|
584
|
+
/**
|
|
585
|
+
* Run decisions are lifecycle records: they take the journal's durable
|
|
586
|
+
* channel so a saturated lossy queue can never drop them (issue #10).
|
|
587
|
+
* They stay ownerless because several decisions — `claim-lost`,
|
|
588
|
+
* `steal-refused-owner-alive`, post-transition `transitioned` — are
|
|
589
|
+
* legitimately recorded by a process that does not (or no longer does)
|
|
590
|
+
* own the run; the ownership fence for these paths is the run-row CAS
|
|
591
|
+
* that precedes each emit.
|
|
592
|
+
*
|
|
593
|
+
* `meta.lineageId` is a JOURNAL lineage id (`FlowEngine.Lineage`,
|
|
594
|
+
* `<runId>/root`), because that is the space a time-travel frame addresses:
|
|
595
|
+
* time travel (`packages/smithers/flows/time-travel/docs/README.md`) makes a frame `(lineageId, seq)`, and
|
|
596
|
+
* replay skips an entry whose `meta.lineageId` names a different lineage.
|
|
597
|
+
* The run row's `lineageId` column is a different space — the TRAMPOLINE
|
|
598
|
+
* lineage of `packages/smithers/flows/engine/docs/concepts/trampoline-rounds.md`, round 0's execution
|
|
599
|
+
* id.
|
|
600
|
+
*
|
|
601
|
+
* DECIDED (2026-08-12): decisions address the journal lineage of the run
|
|
602
|
+
* that records them, one per round, the same lineage that run's attempt
|
|
603
|
+
* and snapshot records already carry. Reading the run row's trampoline
|
|
604
|
+
* lineage here put a run's decisions in a lineage its own attempts do not
|
|
605
|
+
* address, so a rewind of that run skipped them. The trampoline lineage is
|
|
606
|
+
* not lost: `created` and `handed-off` carry `lineageId` and
|
|
607
|
+
* `roundOrdinal` in the decision payload, which is what walks a whole
|
|
608
|
+
* trampoline chain.
|
|
609
|
+
*/
|
|
610
|
+
const executionFacts = ExecutionFacts.make({
|
|
611
|
+
runs: store,
|
|
612
|
+
state: engineState,
|
|
613
|
+
journal,
|
|
614
|
+
sourceId: dependencies.journalSource
|
|
615
|
+
})
|
|
616
|
+
const requestCancellation = (runId: string, at: number) =>
|
|
617
|
+
executionFacts.requestCancel(runId, at).pipe(
|
|
618
|
+
Effect.catchIf(
|
|
619
|
+
(error) => error._tag === "@smthrs/journal/JournalError" || error instanceof DatabaseError,
|
|
620
|
+
Effect.die
|
|
621
|
+
)
|
|
622
|
+
)
|
|
623
|
+
|
|
624
|
+
const decisionRecord = (runId: string, payload: unknown, sourceId: string) =>
|
|
625
|
+
// Lifecycle decisions normally use the unfenced channel: they commit in the SAME transaction
|
|
626
|
+
// as the store-level owner CAS that is its fence (`transitionOwned`,
|
|
627
|
+
// `activate`, `claim`/`steal` outcomes), and by then the run is often no
|
|
628
|
+
// longer `running` under this owner, which is the exact predicate the
|
|
629
|
+
// journal fence asserts. Several call sites (claim-lost,
|
|
630
|
+
// steal-refused-owner-alive, activation-lost) also record decisions for
|
|
631
|
+
// runs this driver never owned at all — first-writer-wins evidence,
|
|
632
|
+
// which is what the unfenced channel exists for. Lease reconfirmation
|
|
633
|
+
// retains ownership and emits this record through the fenced channel.
|
|
634
|
+
JournalRecords.runDecision({
|
|
635
|
+
runId,
|
|
636
|
+
lineageId: FlowEngine.Lineage.root(runId),
|
|
637
|
+
sourceId,
|
|
638
|
+
...(sourceId === dependencies.journalSource ? {} : { sourceSeq: 0 })
|
|
639
|
+
}, payload)
|
|
640
|
+
|
|
641
|
+
/**
|
|
642
|
+
* Completes every pending clock of a settled run and records each
|
|
643
|
+
* completion, in the caller's transaction. A run that settles before its
|
|
644
|
+
* timer fires still has to leave the journal saying the clock is done, or
|
|
645
|
+
* a rebuilt deadline index would arm it again (issue #2037).
|
|
646
|
+
*/
|
|
647
|
+
const completeRunClocks = (runId: string, completedAtMs: number) =>
|
|
648
|
+
Effect.flatMap(engineState.completeRunClocks(runId, completedAtMs), (completed) =>
|
|
649
|
+
Effect.forEach(completed, (clock) =>
|
|
650
|
+
journal.emitDurableUnfenced(
|
|
651
|
+
JournalRecords.clockCompletedFor(
|
|
652
|
+
dependencies.journalSource,
|
|
653
|
+
FlowEngine.Lineage.root(runId),
|
|
654
|
+
clock,
|
|
655
|
+
completedAtMs
|
|
656
|
+
)
|
|
657
|
+
), { discard: true }))
|
|
658
|
+
|
|
659
|
+
/**
|
|
660
|
+
* Records a decision this driver's own write transaction is responsible
|
|
661
|
+
* for: the `transitioned` record that commits with its run-row CAS, and
|
|
662
|
+
* the `claimed-and-activated` / `stolen-and-activated` record that commits
|
|
663
|
+
* with the activation CAS.
|
|
664
|
+
*
|
|
665
|
+
* These stay fatal. They take the driver's default source id, so the
|
|
666
|
+
* journal allocates them a fresh `sourceSeq` and no address can collide;
|
|
667
|
+
* a failure here is the storage layer failing under a transaction whose
|
|
668
|
+
* other half is a run-row write, and the transaction has to take the
|
|
669
|
+
* driver down with it rather than commit a terminal row whose decision
|
|
670
|
+
* never reached the journal.
|
|
671
|
+
*/
|
|
672
|
+
const emitDecision = (
|
|
673
|
+
runId: string,
|
|
674
|
+
payload: unknown,
|
|
675
|
+
sourceId = dependencies.journalSource
|
|
676
|
+
): Effect.Effect<void> =>
|
|
677
|
+
Effect.gen(function*() {
|
|
678
|
+
const decision = payload as Readonly<Record<string, unknown>>
|
|
679
|
+
const authoritative = sourceId === dependencies.journalSource && decision.decision !== "wake-scheduled" &&
|
|
680
|
+
decision.decision !== "child-policy-applied"
|
|
681
|
+
const recorded = authoritative
|
|
682
|
+
? {
|
|
683
|
+
...decision,
|
|
684
|
+
executionFact: yield* executionFacts.fact(runId, decision.decision === "created" ? "created" : "legacy")
|
|
685
|
+
}
|
|
686
|
+
: payload
|
|
687
|
+
yield* journal.emitDurableUnfenced(decisionRecord(runId, recorded, sourceId))
|
|
688
|
+
}).pipe(Effect.orDie)
|
|
689
|
+
|
|
690
|
+
/**
|
|
691
|
+
* Records evidence ABOUT a run this driver does not own, and never raises
|
|
692
|
+
* on the drive fiber.
|
|
693
|
+
*
|
|
694
|
+
* `claim-lost`, `activation-lost` and `steal-refused-owner-alive` are the
|
|
695
|
+
* three decisions taken about a run another process holds. They are
|
|
696
|
+
* observations, not commitments: the arbitration they describe already
|
|
697
|
+
* happened in the store, the caller has already lost, and the very next
|
|
698
|
+
* thing every one of these call sites does is answer `false`. Routing them
|
|
699
|
+
* through {@link emitDecision} put an audit write on the failure path of
|
|
700
|
+
* the loop that drives every run in the process — an `idempotency_conflict`
|
|
701
|
+
* on a shared refusal address killed the drive fiber, dropped the record
|
|
702
|
+
* anyway, and took down any `execute` joined to that run.
|
|
703
|
+
*
|
|
704
|
+
* So the receipt is folded into a log, the shape `ActionPersistence`'s
|
|
705
|
+
* `emitConverging` uses for the same reason on the replay branches:
|
|
706
|
+
*
|
|
707
|
+
* - `Accepted`: the observation is durable. Nothing to say.
|
|
708
|
+
* - `Duplicate`: an identical observation is already durable — the
|
|
709
|
+
* per-heartbeat deduplication the refusal address exists for. Debug.
|
|
710
|
+
* - `idempotency_conflict`: this address already holds a DIFFERENT
|
|
711
|
+
* observation, so someone (another process, an earlier evidence shape,
|
|
712
|
+
* a copied journal after a time-travel fork) said something else about
|
|
713
|
+
* the same (run, owner, lease, evidence). The durable record wins and
|
|
714
|
+
* the divergence is worth a warning, but it is still only evidence.
|
|
715
|
+
*
|
|
716
|
+
* Any other journal failure is folded the same way and for the same
|
|
717
|
+
* reason: an audit record must never be the thing that stops a driver
|
|
718
|
+
* from driving. A journal that is genuinely broken still fails the very
|
|
719
|
+
* next decision write that IS a fence, which is where it belongs.
|
|
720
|
+
*/
|
|
721
|
+
const emitEvidence = (
|
|
722
|
+
runId: string,
|
|
723
|
+
payload: unknown,
|
|
724
|
+
sourceId = dependencies.journalSource
|
|
725
|
+
): Effect.Effect<void> =>
|
|
726
|
+
journal.emitDurableUnfenced(decisionRecord(runId, payload, sourceId)).pipe(
|
|
727
|
+
Effect.flatMap((receipt) =>
|
|
728
|
+
receipt._tag === "Duplicate"
|
|
729
|
+
? Effect.logDebug(
|
|
730
|
+
`engine-store: run ${runId} already recorded this evidence`,
|
|
731
|
+
{ runId, sourceId, seq: receipt.seq }
|
|
732
|
+
)
|
|
733
|
+
: Effect.void
|
|
734
|
+
),
|
|
735
|
+
Effect.catch((error) =>
|
|
736
|
+
Effect.logWarning(
|
|
737
|
+
`engine-store: evidence about run ${runId} was not admitted to the journal (${error.code}); the run is unaffected`,
|
|
738
|
+
{ runId, sourceId, code: error.code, payload }
|
|
739
|
+
)
|
|
740
|
+
)
|
|
741
|
+
)
|
|
742
|
+
|
|
743
|
+
/**
|
|
744
|
+
* The package's ordered state-and-journal boundary: engine state first,
|
|
745
|
+
* the journal's write transaction inside it. See
|
|
746
|
+
* {@link StateTransaction.make} for the order and what it guarantees.
|
|
747
|
+
*/
|
|
748
|
+
const transactState = StateTransaction.make(engineState, journal)
|
|
749
|
+
|
|
750
|
+
/**
|
|
751
|
+
* Commits a run-row transition and the decision describing it in ONE write
|
|
752
|
+
* transaction, reporting the store outcome.
|
|
753
|
+
*
|
|
754
|
+
* `RunStore` and the journal write through the same `DurableWriter`, so the CAS
|
|
755
|
+
* becomes a savepoint of this transaction: a crash can no longer leave a
|
|
756
|
+
* terminal run row whose `transitioned` decision never reached the
|
|
757
|
+
* journal, nor a decision for a CAS that lost. Temporal commits mutable
|
|
758
|
+
* state and its history events as one persistence request
|
|
759
|
+
* (`reference/temporal/service/history/workflow/transaction_impl.go`);
|
|
760
|
+
* this is the same unit of work for a run transition.
|
|
761
|
+
*
|
|
762
|
+
* The decision is emitted only for a `Transitioned` outcome: a lost CAS
|
|
763
|
+
* changed nothing, and its `claim-lost`/`activation-lost` records are
|
|
764
|
+
* emitted by the caller, outside the transaction, so they survive.
|
|
765
|
+
*
|
|
766
|
+
* `waiting` parks before the CAS releases ownership, inside this same
|
|
767
|
+
* transaction. A refused park or transition FAILS the transaction, so SQL
|
|
768
|
+
* rolls back and the memory state's serialized snapshot is restored too.
|
|
769
|
+
* Only outside both boundaries is the refusal returned as an outcome.
|
|
770
|
+
* No former owner reads or clears a replacement's waiting marker.
|
|
771
|
+
*
|
|
772
|
+
* `afterTransitioned` is the seam for work that must commit WITH the
|
|
773
|
+
* transition or not at all: today, ending the run's linked children on a
|
|
774
|
+
* terminal exit. It runs only after a successful CAS, so a lost fence
|
|
775
|
+
* touches nothing, and inside the transaction, so a crash between the two
|
|
776
|
+
* writes rolls both back.
|
|
777
|
+
*/
|
|
778
|
+
const transitionAndRecord = (
|
|
779
|
+
runId: string,
|
|
780
|
+
toStatus: RunStore.RunStatus,
|
|
781
|
+
stateJson: string,
|
|
782
|
+
decision: unknown,
|
|
783
|
+
guard?: RunStore.TransitionGuard | undefined,
|
|
784
|
+
afterTransitioned?: Effect.Effect<void> | undefined,
|
|
785
|
+
waiting?: DurableEngineState.Waiting | null | undefined
|
|
786
|
+
): Effect.Effect<RunStore.TransitionOutcome> =>
|
|
787
|
+
transactState(
|
|
788
|
+
Effect.gen(function*() {
|
|
789
|
+
if (waiting !== undefined && waiting !== null) {
|
|
790
|
+
const parked = yield* engineState.park(runId, waiting, dependencies.owner)
|
|
791
|
+
if (parked._tag === "NotFound") {
|
|
792
|
+
return yield* Effect.fail({ _tag: "TransitionRefused" as const, outcome: parked })
|
|
793
|
+
}
|
|
794
|
+
}
|
|
795
|
+
const transitioned = yield* store.transitionOwned(
|
|
796
|
+
runId,
|
|
797
|
+
dependencies.owner,
|
|
798
|
+
toStatus,
|
|
799
|
+
stateJson,
|
|
800
|
+
guard
|
|
801
|
+
).pipe(Effect.orDie)
|
|
802
|
+
if (transitioned._tag !== "Transitioned") {
|
|
803
|
+
return yield* Effect.fail({ _tag: "TransitionRefused" as const, outcome: transitioned })
|
|
804
|
+
}
|
|
805
|
+
if (waiting === null) {
|
|
806
|
+
yield* engineState.wake(runId)
|
|
807
|
+
}
|
|
808
|
+
if (afterTransitioned !== undefined) {
|
|
809
|
+
yield* afterTransitioned
|
|
810
|
+
}
|
|
811
|
+
// Every terminal round closes its own deadlines, including a
|
|
812
|
+
// handoff whose successor and linked children continue separately.
|
|
813
|
+
if (toStatus === "completed" || toStatus === "failed") {
|
|
814
|
+
yield* completeRunClocks(runId, yield* Clock.currentTimeMillis.pipe(Effect.map(Math.floor)))
|
|
815
|
+
}
|
|
816
|
+
// The decision carries the state it committed, so run state at a
|
|
817
|
+
// frame is DERIVED by replaying decisions rather than read off the
|
|
818
|
+
// run row's current `state_json`
|
|
819
|
+
// (Temporal's
|
|
820
|
+
// `ndc/state_rebuilder.go` is the model). Without it a fork at an
|
|
821
|
+
// early frame silently inherited the parent's *latest* state.
|
|
822
|
+
yield* emitDecision(runId, { ...(decision as object), state: JSON.parse(stateJson) })
|
|
823
|
+
return transitioned
|
|
824
|
+
})
|
|
825
|
+
).pipe(
|
|
826
|
+
Effect.catchTag("TransitionRefused", ({ outcome }) =>
|
|
827
|
+
Effect.succeed(outcome)),
|
|
828
|
+
Effect.orDie
|
|
829
|
+
)
|
|
830
|
+
|
|
831
|
+
const abandon = (runId: string, claimedAtMs: number): Effect.Effect<void> =>
|
|
832
|
+
store.abandonClaim(runId, dependencies.owner, claimedAtMs).pipe(
|
|
833
|
+
Effect.asVoid,
|
|
834
|
+
Effect.orDie
|
|
835
|
+
)
|
|
836
|
+
|
|
837
|
+
const claimAndActivate: (row: RunStore.RunRow) => Effect.Effect<boolean> = Effect.fn(
|
|
838
|
+
"RunDriver.claimAndActivate"
|
|
839
|
+
)(function*(row: RunStore.RunRow) {
|
|
840
|
+
return yield* Effect.gen(function*() {
|
|
841
|
+
yield* Effect.annotateCurrentSpan({ runId: row.runId, status: row.status })
|
|
842
|
+
if (dependencies.canExecute !== undefined && !(yield* dependencies.canExecute(row))) {
|
|
843
|
+
return false
|
|
844
|
+
}
|
|
845
|
+
if (row.status === "completed" || row.status === "failed" || row.status === "cancelled") {
|
|
846
|
+
yield* Effect.annotateCurrentSpan({ outcome: "terminal" })
|
|
847
|
+
yield* Metric.update(EngineStoreMetrics.claim.Terminal, 1)
|
|
848
|
+
return false
|
|
849
|
+
}
|
|
850
|
+
|
|
851
|
+
// Every drive, including a fresh execute or a queued wake, must
|
|
852
|
+
// preserve quarantine. Cancellation still claims the row to close it.
|
|
853
|
+
if (row.cancelRequestedAtMs === null) {
|
|
854
|
+
const waiting = yield* engineState.waiting(row.runId)
|
|
855
|
+
if (Option.isSome(waiting) && waiting.value.reason === "quarantine") {
|
|
856
|
+
return false
|
|
857
|
+
}
|
|
858
|
+
}
|
|
859
|
+
|
|
860
|
+
const expected = snapshot(row)
|
|
861
|
+
const nowMs = yield* Clock.currentTimeMillis.pipe(Effect.map(Math.floor))
|
|
862
|
+
let claim: RunStore.StealOutcome
|
|
863
|
+
let stealEvidenceKind: Ownership.LivenessEvidence["kind"] | undefined
|
|
864
|
+
let recoveredClaim: { readonly claimant: Ownership.OwnerId; readonly claimedAtMs: number } | undefined
|
|
865
|
+
|
|
866
|
+
if (row.status === "running") {
|
|
867
|
+
if (
|
|
868
|
+
row.owner === null ||
|
|
869
|
+
row.heartbeatAtMs === null ||
|
|
870
|
+
row.heartbeatAtMs >= nowMs - Duration.toMillis(Ownership.heartbeatStaleAfter)
|
|
871
|
+
) {
|
|
872
|
+
// The lease is the refusal, and no probe is consulted for it: a
|
|
873
|
+
// probe may be a remote call, and a fresh heartbeat already
|
|
874
|
+
// answers the question. It is journaled anyway — but only when
|
|
875
|
+
// another owner holds the run, because refusing to steal from
|
|
876
|
+
// ourselves is routine re-entry rather than arbitration — so an
|
|
877
|
+
// operator whose `resume` did nothing can see that a live owner
|
|
878
|
+
// was in the way (issue #53 follow-up).
|
|
879
|
+
if (row.owner !== null && !sameOwner(row.owner, dependencies.owner)) {
|
|
880
|
+
deferSteal(row.runId, row.heartbeatAtMs, nowMs)
|
|
881
|
+
yield* emitEvidence(row.runId, {
|
|
882
|
+
decision: "steal-refused-owner-alive",
|
|
883
|
+
evidence: "lease-fresh",
|
|
884
|
+
expectedOwner: row.owner,
|
|
885
|
+
heartbeatAtMs: row.heartbeatAtMs
|
|
886
|
+
}, stealRefusedSourceId(row.owner, row.heartbeatAtMs, "lease-fresh"))
|
|
887
|
+
}
|
|
888
|
+
yield* Effect.annotateCurrentSpan({ outcome: "heartbeat_fresh" })
|
|
889
|
+
yield* Metric.update(EngineStoreMetrics.claim.HeartbeatFresh, 1)
|
|
890
|
+
return false
|
|
891
|
+
}
|
|
892
|
+
if (yield* isAlive(row.owner, { claimant: dependencies.owner, heartbeatAtMs: row.heartbeatAtMs, nowMs })) {
|
|
893
|
+
deferSteal(row.runId, row.heartbeatAtMs, nowMs)
|
|
894
|
+
yield* emitEvidence(row.runId, {
|
|
895
|
+
decision: "steal-refused-owner-alive",
|
|
896
|
+
// Always a supplied probe. The default check is the lease, and
|
|
897
|
+
// the lease already refused above for every row that could still
|
|
898
|
+
// be fresh, so reaching this arm means the deployment answered
|
|
899
|
+
// for the owner with something the store cannot see.
|
|
900
|
+
evidence: "probe",
|
|
901
|
+
expectedOwner: row.owner,
|
|
902
|
+
heartbeatAtMs: row.heartbeatAtMs
|
|
903
|
+
}, stealRefusedSourceId(row.owner, row.heartbeatAtMs, "probe"))
|
|
904
|
+
yield* Effect.annotateCurrentSpan({ outcome: "steal_refused_owner_alive" })
|
|
905
|
+
yield* Metric.update(EngineStoreMetrics.claim.StealRefusedOwnerAlive, 1)
|
|
906
|
+
yield* Effect.logDebug("run steal refused, recorded owner is alive", {
|
|
907
|
+
runId: row.runId,
|
|
908
|
+
heartbeatAtMs: row.heartbeatAtMs
|
|
909
|
+
})
|
|
910
|
+
return false
|
|
911
|
+
}
|
|
912
|
+
stealEvidenceKind = livenessEvidenceKind(row.owner)
|
|
913
|
+
claim = yield* store.steal(
|
|
914
|
+
row.runId,
|
|
915
|
+
expected,
|
|
916
|
+
dependencies.owner,
|
|
917
|
+
nowMs,
|
|
918
|
+
{
|
|
919
|
+
expectedOwner: row.owner,
|
|
920
|
+
checkedAtMs: nowMs,
|
|
921
|
+
kind: stealEvidenceKind
|
|
922
|
+
}
|
|
923
|
+
).pipe(Effect.orDie)
|
|
924
|
+
} else {
|
|
925
|
+
// A process that died between `claim` and `activate` leaves its
|
|
926
|
+
// claim on a parked or pending row, and every later claim loses to
|
|
927
|
+
// it: the run could never be driven again. A claim older than the
|
|
928
|
+
// lease whose claimant is gone is released first, by the same
|
|
929
|
+
// liveness rule a steal applies to a running owner.
|
|
930
|
+
if (
|
|
931
|
+
row.claim !== null && row.claimedAtMs !== null && !sameOwner(row.claim, dependencies.owner) &&
|
|
932
|
+
row.claimedAtMs < nowMs - Duration.toMillis(Ownership.heartbeatStaleAfter) &&
|
|
933
|
+
!(yield* isAlive(row.claim, { claimant: dependencies.owner, heartbeatAtMs: row.claimedAtMs, nowMs }))
|
|
934
|
+
) {
|
|
935
|
+
const recovered = yield* store.recoverClaim(
|
|
936
|
+
row.runId,
|
|
937
|
+
row.claim,
|
|
938
|
+
row.claimedAtMs,
|
|
939
|
+
dependencies.owner,
|
|
940
|
+
nowMs,
|
|
941
|
+
{ expectedOwner: row.claim, checkedAtMs: nowMs, kind: livenessEvidenceKind(row.claim) }
|
|
942
|
+
).pipe(Effect.orDie)
|
|
943
|
+
if (recovered._tag === "Recovered") {
|
|
944
|
+
recoveredClaim = { claimant: row.claim, claimedAtMs: row.claimedAtMs }
|
|
945
|
+
}
|
|
946
|
+
}
|
|
947
|
+
claim = yield* store.claim(
|
|
948
|
+
row.runId,
|
|
949
|
+
expected,
|
|
950
|
+
dependencies.owner,
|
|
951
|
+
nowMs
|
|
952
|
+
).pipe(Effect.orDie)
|
|
953
|
+
}
|
|
954
|
+
|
|
955
|
+
if (claim._tag !== "Claimed") {
|
|
956
|
+
yield* emitEvidence(row.runId, {
|
|
957
|
+
decision: "claim-lost",
|
|
958
|
+
outcome: claim._tag,
|
|
959
|
+
expected
|
|
960
|
+
})
|
|
961
|
+
yield* Effect.annotateCurrentSpan({ outcome: "claim_lost" })
|
|
962
|
+
yield* Metric.update(EngineStoreMetrics.claim.ClaimLost, 1)
|
|
963
|
+
yield* Effect.logDebug("run claim lost", { runId: row.runId, outcome: claim._tag })
|
|
964
|
+
return false
|
|
965
|
+
}
|
|
966
|
+
|
|
967
|
+
// The activation CAS and the decision recording it commit together:
|
|
968
|
+
// a crash between them left a run durably running under this owner
|
|
969
|
+
// with no journal entry saying who took it.
|
|
970
|
+
const activation = yield* transactState(
|
|
971
|
+
Effect.gen(function*() {
|
|
972
|
+
// Admission can change without changing the ownership snapshot
|
|
973
|
+
// (a released→running→released ABA, rewind, or consent change).
|
|
974
|
+
// Recheck its read-only native fence under the writer transaction;
|
|
975
|
+
// loading modules and probing external hosts remain preflight work.
|
|
976
|
+
if (
|
|
977
|
+
dependencies.canActivate !== undefined &&
|
|
978
|
+
!(yield* dependencies.canActivate(yield* store.get(row.runId).pipe(Effect.orDie)))
|
|
979
|
+
) {
|
|
980
|
+
return { _tag: "SnapshotChanged" as const }
|
|
981
|
+
}
|
|
982
|
+
// Another owner can quarantine and return to the same suspended
|
|
983
|
+
// snapshot between preflight and claim. Fence that park in the
|
|
984
|
+
// activation transaction too; the rejected claim is abandoned below.
|
|
985
|
+
if (row.cancelRequestedAtMs === null) {
|
|
986
|
+
const waiting = yield* engineState.waiting(row.runId)
|
|
987
|
+
if (Option.isSome(waiting) && waiting.value.reason === "quarantine") {
|
|
988
|
+
return { _tag: "SnapshotChanged" as const }
|
|
989
|
+
}
|
|
990
|
+
}
|
|
991
|
+
const activation = yield* store.activate(
|
|
992
|
+
row.runId,
|
|
993
|
+
dependencies.owner,
|
|
994
|
+
claim.claimedAtMs,
|
|
995
|
+
expected
|
|
996
|
+
).pipe(Effect.orDie)
|
|
997
|
+
if (activation._tag !== "Activated") {
|
|
998
|
+
return activation
|
|
999
|
+
}
|
|
1000
|
+
yield* emitDecision(row.runId, {
|
|
1001
|
+
decision: row.status === "running" ? "stolen-and-activated" : "claimed-and-activated",
|
|
1002
|
+
previousStatus: row.status,
|
|
1003
|
+
owner: dependencies.owner,
|
|
1004
|
+
// What admitted the takeover, so an operator reading the journal
|
|
1005
|
+
// can tell a lease that simply expired from a probe that
|
|
1006
|
+
// positively reported the owner gone.
|
|
1007
|
+
...(stealEvidenceKind === undefined ? {} : { evidence: stealEvidenceKind }),
|
|
1008
|
+
...(recoveredClaim === undefined ? {} : { recoveredClaim })
|
|
1009
|
+
})
|
|
1010
|
+
return activation
|
|
1011
|
+
})
|
|
1012
|
+
).pipe(Effect.orDie)
|
|
1013
|
+
if (activation._tag !== "Activated") {
|
|
1014
|
+
yield* abandon(row.runId, claim.claimedAtMs)
|
|
1015
|
+
yield* emitEvidence(row.runId, {
|
|
1016
|
+
decision: "activation-lost",
|
|
1017
|
+
outcome: activation._tag,
|
|
1018
|
+
expected
|
|
1019
|
+
})
|
|
1020
|
+
yield* Effect.annotateCurrentSpan({ outcome: "activation_lost" })
|
|
1021
|
+
yield* Metric.update(EngineStoreMetrics.claim.ActivationLost, 1)
|
|
1022
|
+
yield* Effect.logDebug("run activation lost", { runId: row.runId, outcome: activation._tag })
|
|
1023
|
+
return false
|
|
1024
|
+
}
|
|
1025
|
+
yield* Effect.annotateCurrentSpan({ outcome: "activated" })
|
|
1026
|
+
yield* Metric.update(EngineStoreMetrics.claim.Activated, 1)
|
|
1027
|
+
// The run is this driver's now, so whatever it was told about the
|
|
1028
|
+
// previous owner is spent.
|
|
1029
|
+
stealRefusals.delete(row.runId)
|
|
1030
|
+
return true
|
|
1031
|
+
})
|
|
1032
|
+
})
|
|
1033
|
+
|
|
1034
|
+
/**
|
|
1035
|
+
* Collects other rounds and transitive descendants of the logical run.
|
|
1036
|
+
*
|
|
1037
|
+
* Round membership and the edge table represent the logical job that every
|
|
1038
|
+
* owner process and every restart can see (issues #40/#41), which is
|
|
1039
|
+
* exactly why the cascade reads it instead of an in-process instance map:
|
|
1040
|
+
* a cross-process cancellation is observed by a driver that never spawned
|
|
1041
|
+
* the children and holds no `FlowInstance` for any of them.
|
|
1042
|
+
*
|
|
1043
|
+
* Deduplicated by run id, so the diamond the edge set permits is visited
|
|
1044
|
+
* once and a (rejected, but defensively handled) cycle terminates. Work and
|
|
1045
|
+
* memory are bounded by the finite durable edge set; there is deliberately
|
|
1046
|
+
* no arbitrary fan-out cap, because truncating a wide generation would
|
|
1047
|
+
* leave linked children uncancelled forever.
|
|
1048
|
+
*/
|
|
1049
|
+
const descendantsOf = lifecycle.descendants
|
|
1050
|
+
|
|
1051
|
+
/**
|
|
1052
|
+
* Records a durable cancellation request against every linked descendant
|
|
1053
|
+
* of a cancelled run.
|
|
1054
|
+
*
|
|
1055
|
+
* `requestCancel` is unfenced and first-writer-wins, so this is idempotent
|
|
1056
|
+
* by construction: re-running the cascade — a re-drive, a second operator
|
|
1057
|
+
* cancel, a child that also cascades from its own cancellation — writes
|
|
1058
|
+
* nothing the second time and answers `AlreadyRequested`. A row that no
|
|
1059
|
+
* longer exists answers `NotFound`, which is equally fine.
|
|
1060
|
+
*
|
|
1061
|
+
* `cancelOwned` calls it inside the same transaction as the parent's
|
|
1062
|
+
* terminal `cancelled` transition, so a crash cannot commit a cancelled
|
|
1063
|
+
* parent whose children were never asked to stop.
|
|
1064
|
+
*
|
|
1065
|
+
* Both callers record the run's OWN cancellation BEFORE this walk —
|
|
1066
|
+
* `interrupt` writes the request, `cancelOwned` commits the terminal
|
|
1067
|
+
* `cancelled` transition — and that order is load-bearing rather than
|
|
1068
|
+
* incidental. A late admission inherits by reading the parent row after it
|
|
1069
|
+
* has created the child row (`inheritParentCancellation`), so a walk that
|
|
1070
|
+
* ran before the parent was marked could miss an edge whose admission also
|
|
1071
|
+
* missed the mark. Both the SQL engine state and
|
|
1072
|
+
* `DurableEngineState.makeMemory` serialize their transactions. The memory
|
|
1073
|
+
* implementation snapshots and restores its maps on failure; the shared
|
|
1074
|
+
* writer transaction supplies the run-store write boundary. The ordering
|
|
1075
|
+
* also lets a child admitted within the cascade inherit cancellation.
|
|
1076
|
+
*/
|
|
1077
|
+
const requestCancelDescendants = (
|
|
1078
|
+
runId: string,
|
|
1079
|
+
nowMs: number
|
|
1080
|
+
): Effect.Effect<ReadonlyArray<string>, RunStore.RunStoreError> =>
|
|
1081
|
+
Effect.gen(function*() {
|
|
1082
|
+
const descendants = yield* descendantsOf(runId)
|
|
1083
|
+
yield* Effect.forEach(
|
|
1084
|
+
descendants,
|
|
1085
|
+
(childId) =>
|
|
1086
|
+
requestCancellation(childId, nowMs),
|
|
1087
|
+
{ discard: true }
|
|
1088
|
+
)
|
|
1089
|
+
return descendants
|
|
1090
|
+
})
|
|
1091
|
+
|
|
1092
|
+
/** The exit policy a child recorded for itself when it was spawned. */
|
|
1093
|
+
const exitPolicyOf = (row: RunStore.RunRow): Effect.Effect<OnParentExit> =>
|
|
1094
|
+
decodeState(row.stateJson).pipe(
|
|
1095
|
+
// A row written before the policy existed, or created outside
|
|
1096
|
+
// `ensureRun` (a seeded edge, an external admission), names none. It
|
|
1097
|
+
// is a LINKED child either way, and the contract for a linked child is
|
|
1098
|
+
// that it ends with the run it belongs to; `detach` is the deliberate
|
|
1099
|
+
// exception and has to be recorded to hold.
|
|
1100
|
+
Effect.map((state) => state.onParentExit ?? "cancel")
|
|
1101
|
+
)
|
|
1102
|
+
|
|
1103
|
+
/**
|
|
1104
|
+
* Applies the per-child exit policy when a run ends on its own terms.
|
|
1105
|
+
*
|
|
1106
|
+
* Cancellation is not this path. A cancelled run cascades to every linked
|
|
1107
|
+
* descendant unconditionally (`requestCancelDescendants`): an operator who
|
|
1108
|
+
* cancels a run is asking for the subtree to stop, and a child cannot opt
|
|
1109
|
+
* out of that. What this closes is the other half — a parent that
|
|
1110
|
+
* COMPLETED or FAILED used to leave every linked child running, so a
|
|
1111
|
+
* `.child()` boundary whose parent died on an unrelated branch left a
|
|
1112
|
+
* durable orphan nothing would ever wait for or cancel.
|
|
1113
|
+
*
|
|
1114
|
+
* Three shapes are left alone, and each for its own reason:
|
|
1115
|
+
*
|
|
1116
|
+
* - A child that recorded `detach` was spawned fire-and-forget. Outliving
|
|
1117
|
+
* the parent is the whole point, so it is reported, not touched, and its
|
|
1118
|
+
* own subtree is not walked — those runs hang off a run that is still
|
|
1119
|
+
* going.
|
|
1120
|
+
* - A child that already settled has nothing to cancel, and it applied
|
|
1121
|
+
* this same policy to its own children when it settled.
|
|
1122
|
+
* - A child whose row is gone leaves only an edge.
|
|
1123
|
+
*
|
|
1124
|
+
* The requests are written by the caller's transaction — the parent's
|
|
1125
|
+
* terminal compare-and-swap — so a crash cannot commit a completed parent
|
|
1126
|
+
* whose children were never asked to stop.
|
|
1127
|
+
*/
|
|
1128
|
+
const endLinkedChildren = (
|
|
1129
|
+
runId: string,
|
|
1130
|
+
nowMs: number
|
|
1131
|
+
): Effect.Effect<
|
|
1132
|
+
{
|
|
1133
|
+
readonly cancelled: ReadonlyArray<string>
|
|
1134
|
+
readonly detached: ReadonlyArray<string>
|
|
1135
|
+
},
|
|
1136
|
+
RunStore.RunStoreError
|
|
1137
|
+
> =>
|
|
1138
|
+
Effect.gen(function*() {
|
|
1139
|
+
const roots = (yield* lifecycle.rounds(runId)).map((row) => row.runId)
|
|
1140
|
+
const seen = new Set<string>(roots)
|
|
1141
|
+
const cancelled: Array<string> = []
|
|
1142
|
+
const detached: Array<string> = []
|
|
1143
|
+
let frontier: ReadonlyArray<string> = roots
|
|
1144
|
+
while (frontier.length > 0) {
|
|
1145
|
+
const next: Array<string> = []
|
|
1146
|
+
for (const parentId of frontier) {
|
|
1147
|
+
for (const edge of yield* engineState.runChildren(parentId)) {
|
|
1148
|
+
if (seen.has(edge.childId)) continue
|
|
1149
|
+
const rounds = yield* lifecycle.rounds(edge.childId)
|
|
1150
|
+
for (const round of rounds) seen.add(round.runId)
|
|
1151
|
+
seen.add(edge.childId)
|
|
1152
|
+
const row = rounds.find((round) =>
|
|
1153
|
+
round.status !== "completed" && round.status !== "failed" && round.status !== "cancelled"
|
|
1154
|
+
)
|
|
1155
|
+
if (row === undefined) continue
|
|
1156
|
+
// The edge belongs to the creating round. Its ownership policy
|
|
1157
|
+
// survives the child's handoffs; do not infer it from a later
|
|
1158
|
+
// round written by a version that omitted onParentExit.
|
|
1159
|
+
// A nonempty lineage read includes its requested member.
|
|
1160
|
+
if ((yield* exitPolicyOf(rounds.find((round) => round.runId === edge.childId)!)) === "detach") {
|
|
1161
|
+
detached.push(edge.childId)
|
|
1162
|
+
continue
|
|
1163
|
+
}
|
|
1164
|
+
for (const round of rounds) {
|
|
1165
|
+
if (round.status !== "completed" && round.status !== "failed" && round.status !== "cancelled") {
|
|
1166
|
+
cancelled.push(round.runId)
|
|
1167
|
+
}
|
|
1168
|
+
next.push(round.runId)
|
|
1169
|
+
}
|
|
1170
|
+
}
|
|
1171
|
+
}
|
|
1172
|
+
frontier = next
|
|
1173
|
+
}
|
|
1174
|
+
yield* Effect.forEach(
|
|
1175
|
+
cancelled,
|
|
1176
|
+
(childId) => requestCancellation(childId, nowMs),
|
|
1177
|
+
{ discard: true }
|
|
1178
|
+
)
|
|
1179
|
+
return { cancelled, detached }
|
|
1180
|
+
})
|
|
1181
|
+
|
|
1182
|
+
/**
|
|
1183
|
+
* The effect a terminal transition runs inside its own transaction to end
|
|
1184
|
+
* the children it is responsible for, journaling what it decided.
|
|
1185
|
+
*
|
|
1186
|
+
* The record is emitted only when there was a linked child to decide
|
|
1187
|
+
* about, so an ordinary childless run adds nothing to its journal.
|
|
1188
|
+
*/
|
|
1189
|
+
const applyChildExitPolicy = (runId: string): Effect.Effect<void> =>
|
|
1190
|
+
Effect.gen(function*() {
|
|
1191
|
+
const nowMs = yield* Clock.currentTimeMillis.pipe(Effect.map(Math.floor))
|
|
1192
|
+
const outcome = yield* endLinkedChildren(runId, nowMs).pipe(Effect.orDie)
|
|
1193
|
+
if (outcome.cancelled.length === 0 && outcome.detached.length === 0) return
|
|
1194
|
+
yield* emitDecision(runId, {
|
|
1195
|
+
decision: "child-policy-applied",
|
|
1196
|
+
appliedAtMs: nowMs,
|
|
1197
|
+
cancelled: outcome.cancelled,
|
|
1198
|
+
detached: outcome.detached
|
|
1199
|
+
})
|
|
1200
|
+
})
|
|
1201
|
+
|
|
1202
|
+
/**
|
|
1203
|
+
* Carries a parent's durable cancellation onto a child that was linked
|
|
1204
|
+
* AFTER the parent's cascade had already run.
|
|
1205
|
+
*
|
|
1206
|
+
* Admission and cancellation are two separate serialized write
|
|
1207
|
+
* transactions over the same store, so they interleave in both orders and
|
|
1208
|
+
* neither order may leak a live child (issue #83):
|
|
1209
|
+
*
|
|
1210
|
+
* - Admission first. The edge is committed before the cancellation
|
|
1211
|
+
* transaction opens, so `requestCancelDescendants` sees the child and
|
|
1212
|
+
* requests it. Nothing more is needed here.
|
|
1213
|
+
* - Cancellation first. The cascade walked `flows_run_parents` while the
|
|
1214
|
+
* edge did not exist yet, so it could not have seen this child — and
|
|
1215
|
+
* the parent's drive may already be gone, so nothing will walk the
|
|
1216
|
+
* edge table again on its behalf. The child would be created fresh and
|
|
1217
|
+
* uncancelled, and would run to completion under a cancelled parent.
|
|
1218
|
+
*
|
|
1219
|
+
* The second order is closed by reading the parent row inside the SAME
|
|
1220
|
+
* outer storage transaction that recorded the edge and created the child
|
|
1221
|
+
* row, after the child row exists. Either the cancellation transaction
|
|
1222
|
+
* committed before this read, in which case the request is visible here
|
|
1223
|
+
* and is inherited atomically with the child's own creation, or it has
|
|
1224
|
+
* not committed yet, in which case it commits after this transaction and
|
|
1225
|
+
* its own cascade sees the edge. There is no third interleaving, so no
|
|
1226
|
+
* ephemeral hand-off flag and no timing-based retry is involved. The
|
|
1227
|
+
* argument rests on the store's serialized writes and the order inside
|
|
1228
|
+
* `requestCancelDescendants`: a cancellation marks its own run before it
|
|
1229
|
+
* walks the edge table. Memory durable-state transactions also serialize
|
|
1230
|
+
* access and restore their snapshots on failure; they do not provide
|
|
1231
|
+
* persistence across process crashes.
|
|
1232
|
+
*
|
|
1233
|
+
* Nesting falls out of this: the inherited request is written to the
|
|
1234
|
+
* child's own row, so a grandchild admitted later reads a cancel-requested
|
|
1235
|
+
* parent and inherits in turn.
|
|
1236
|
+
*
|
|
1237
|
+
* `requestCancel` is unfenced and first-writer-wins, so a repeated
|
|
1238
|
+
* admission writes nothing the second time; a parent row that is gone,
|
|
1239
|
+
* or one with no rounds at all, inherits nothing. Both ways a parent can
|
|
1240
|
+
* exit are inherited here, and they differ in whether the child's own
|
|
1241
|
+
* `onParentExit` can spare it. An explicit cancellation in ANY round is
|
|
1242
|
+
* inherited unconditionally, whether it is a pending
|
|
1243
|
+
* `cancel_requested_at_ms` or a round already settled terminally
|
|
1244
|
+
* `cancelled` whose request column a compaction could have cleared. A
|
|
1245
|
+
* parent that instead stopped normally, with EVERY round `completed` or
|
|
1246
|
+
* `failed`, cancels a late child that was admitted attached and leaves an
|
|
1247
|
+
* `onParentExit === "detach"` child to outlive it.
|
|
1248
|
+
*
|
|
1249
|
+
* The child does not need its own cascade here: it was just created, so
|
|
1250
|
+
* it has no descendants of its own, and once it observes the inherited
|
|
1251
|
+
* request `cancelOwned` cascades from it like any other cancellation.
|
|
1252
|
+
*/
|
|
1253
|
+
const inheritParentCancellation = (
|
|
1254
|
+
parentId: string,
|
|
1255
|
+
childId: string
|
|
1256
|
+
): Effect.Effect<void> =>
|
|
1257
|
+
Effect.gen(function*() {
|
|
1258
|
+
const rounds = yield* lifecycle.rounds(parentId).pipe(Effect.orDie)
|
|
1259
|
+
if (rounds.length === 0) return
|
|
1260
|
+
const cancelled = rounds.find((round) => round.cancelRequestedAtMs !== null || round.status === "cancelled")
|
|
1261
|
+
if (cancelled !== undefined) {
|
|
1262
|
+
// The parent's own request timestamp, so the inherited request reads
|
|
1263
|
+
// as the same operator intent rather than as a later, independent one.
|
|
1264
|
+
const nowMs = cancelled.cancelRequestedAtMs ?? (yield* Clock.currentTimeMillis.pipe(Effect.map(Math.floor)))
|
|
1265
|
+
yield* requestCancellation(childId, nowMs).pipe(Effect.orDie)
|
|
1266
|
+
return
|
|
1267
|
+
}
|
|
1268
|
+
// The same interleaving, for the other way a parent stops: a parent
|
|
1269
|
+
// that had already COMPLETED or FAILED ran `applyChildExitPolicy`
|
|
1270
|
+
// before this edge existed, so its walk could not have seen this
|
|
1271
|
+
// child and nothing will walk again on its behalf. The child applies
|
|
1272
|
+
// its own recorded policy here instead, in the transaction that made
|
|
1273
|
+
// it a child, so an attached child admitted a moment too late cannot
|
|
1274
|
+
// run on under a parent that is already gone.
|
|
1275
|
+
if (rounds.some((round) => round.status !== "completed" && round.status !== "failed")) return
|
|
1276
|
+
// Admission reads the child's recorded policy in this transaction.
|
|
1277
|
+
// A later caller awaiting a detached execution cannot change the
|
|
1278
|
+
// policy chosen by its original spawn.
|
|
1279
|
+
if ((yield* exitPolicyOf(yield* store.get(childId).pipe(Effect.orDie))) === "detach") {
|
|
1280
|
+
return
|
|
1281
|
+
}
|
|
1282
|
+
yield* requestCancellation(childId, yield* Clock.currentTimeMillis.pipe(Effect.map(Math.floor))).pipe(
|
|
1283
|
+
Effect.orDie
|
|
1284
|
+
)
|
|
1285
|
+
})
|
|
1286
|
+
|
|
1287
|
+
const cancelOwned = (
|
|
1288
|
+
runId: string,
|
|
1289
|
+
state: RunState
|
|
1290
|
+
): Effect.Effect<void> =>
|
|
1291
|
+
Effect.gen(function*() {
|
|
1292
|
+
const interruptedAtMs = yield* Clock.currentTimeMillis.pipe(Effect.map(Math.floor))
|
|
1293
|
+
const stateJson = yield* encodeState({
|
|
1294
|
+
...withoutResult(state),
|
|
1295
|
+
cancellation: { interruptedAtMs }
|
|
1296
|
+
})
|
|
1297
|
+
let cascaded: ReadonlyArray<string> = []
|
|
1298
|
+
let settled = false
|
|
1299
|
+
yield* transactState(
|
|
1300
|
+
Effect.gen(function*() {
|
|
1301
|
+
yield* store.acknowledgeCancel(runId, dependencies.owner, interruptedAtMs).pipe(Effect.orDie)
|
|
1302
|
+
const transitioned = yield* store.transitionOwned(
|
|
1303
|
+
runId,
|
|
1304
|
+
dependencies.owner,
|
|
1305
|
+
"cancelled",
|
|
1306
|
+
stateJson
|
|
1307
|
+
).pipe(Effect.orDie)
|
|
1308
|
+
if (transitioned._tag !== "Transitioned") return
|
|
1309
|
+
settled = true
|
|
1310
|
+
// Cancellation cascades to linked children (`Flow.interrupt` is
|
|
1311
|
+
// documented to preserve "child-flow handling"). It is recorded
|
|
1312
|
+
// durably, in this transaction, off the durable edge table, so the
|
|
1313
|
+
// cascade is identical whether the cancellation was requested in
|
|
1314
|
+
// this process or observed from another one — the pre-existing
|
|
1315
|
+
// in-process linkage in `FlowEngine.make` fires off the ephemeral
|
|
1316
|
+
// `instance.interrupted` flag, which a durable observation never
|
|
1317
|
+
// sets and a second driver instance never has.
|
|
1318
|
+
cascaded = yield* requestCancelDescendants(runId, interruptedAtMs)
|
|
1319
|
+
// The run has no future, so neither do its timers. Completing the
|
|
1320
|
+
// clock rows in this transaction is what stops a durable timer
|
|
1321
|
+
// from outliving the run it belongs to: the sweep queries already
|
|
1322
|
+
// skip a settled run's rows, and this stops them being pending at
|
|
1323
|
+
// all — including under an in-memory state with no run view to
|
|
1324
|
+
// join against.
|
|
1325
|
+
yield* completeRunClocks(runId, interruptedAtMs)
|
|
1326
|
+
// A cancel can race the final poll after the run already parked
|
|
1327
|
+
// (park precedes the guarded terminal CAS). Clear the waiting row so
|
|
1328
|
+
// the terminally cancelled run never surfaces to a sweeper again
|
|
1329
|
+
// (issue #28). It shares the transition's transaction, so a crash
|
|
1330
|
+
// can no longer land between them.
|
|
1331
|
+
yield* engineState.wake(runId).pipe(Effect.asVoid)
|
|
1332
|
+
// Durable channel (issue #10): the interruption record must survive
|
|
1333
|
+
// the process exiting right after cancellation. Unfenced because the
|
|
1334
|
+
// `cancelled` transition above has already released ownership; the
|
|
1335
|
+
// fence is the transition CAS itself, which now commits with this
|
|
1336
|
+
// record rather than before it.
|
|
1337
|
+
yield* journal.emitDurableUnfenced(
|
|
1338
|
+
JournalRecords.interrupted({
|
|
1339
|
+
runId,
|
|
1340
|
+
lineageId: FlowEngine.Lineage.root(runId),
|
|
1341
|
+
sourceId: dependencies.journalSource
|
|
1342
|
+
}, {
|
|
1343
|
+
outcome: "cancelled",
|
|
1344
|
+
interruptedAtMs,
|
|
1345
|
+
owner: dependencies.owner,
|
|
1346
|
+
executionFact: yield* executionFacts.fact(runId),
|
|
1347
|
+
...(cascaded.length === 0 ? {} : { cascadedTo: cascaded })
|
|
1348
|
+
})
|
|
1349
|
+
).pipe(Effect.orDie)
|
|
1350
|
+
})
|
|
1351
|
+
).pipe(Effect.orDie)
|
|
1352
|
+
// Post-commit, and only reachable after a transition that actually
|
|
1353
|
+
// happened. Closing the parked flow scope runs the finalizers the
|
|
1354
|
+
// suspension retained, with an interrupt exit so `Flow.withRollback`
|
|
1355
|
+
// compensations fire — cancellation is exactly the unsuccessful exit
|
|
1356
|
+
// they exist for. The run is durably cancelled first, so a crash
|
|
1357
|
+
// inside a finalizer cannot resurrect it.
|
|
1358
|
+
yield* releaseRetainedScope(runId, Exit.failCause(Cause.interrupt()))
|
|
1359
|
+
// Waking a cascaded child is in-process scheduling: a child this
|
|
1360
|
+
// process owns or has parked re-enters claim/activate now and the
|
|
1361
|
+
// activation cancel guard closes it, instead of waiting out a poll
|
|
1362
|
+
// interval. A child owned elsewhere ignores the wake and is reached by
|
|
1363
|
+
// its own owner's cancel poll or by that owner's parked-run sweep.
|
|
1364
|
+
if (cascaded.length > 0) {
|
|
1365
|
+
const activeCoordinator = yield* Deferred.await(coordinatorDeferred)
|
|
1366
|
+
yield* Effect.forEach(cascaded, (childId) => activeCoordinator.wake(childId), { discard: true })
|
|
1367
|
+
}
|
|
1368
|
+
// A cancelled round is a terminal settlement like a completed or
|
|
1369
|
+
// failed one, so every parent already parked on it — through any round
|
|
1370
|
+
// of its lineage — is woken to read the cancellation (#2758). Only
|
|
1371
|
+
// after the transition this call committed: a lost fence woke nobody.
|
|
1372
|
+
if (settled) {
|
|
1373
|
+
yield* announceSettled(runId, state)
|
|
1374
|
+
}
|
|
1375
|
+
})
|
|
1376
|
+
|
|
1377
|
+
/**
|
|
1378
|
+
* Releases an interrupted run reclaimably instead of closing it
|
|
1379
|
+
* (issue #26). A drive-fiber interruption is not evidence of operator
|
|
1380
|
+
* cancellation — process shutdown closes the coordinator scope, and the
|
|
1381
|
+
* heartbeat loop self-interrupts on any heartbeat error — so the run
|
|
1382
|
+
* transitions back to `suspended` while the fence is still validly held,
|
|
1383
|
+
* leaving it claimable by any worker (Temporal worker-shutdown
|
|
1384
|
+
* semantics). On genuine fence loss the owned transition fails
|
|
1385
|
+
* harmlessly.
|
|
1386
|
+
*/
|
|
1387
|
+
/**
|
|
1388
|
+
* The waiting reason an interrupted round parks under.
|
|
1389
|
+
*
|
|
1390
|
+
* A flow that has ALREADY asked to suspend is parked, not released. The
|
|
1391
|
+
* round's own settlement derives the same three answers a few statements
|
|
1392
|
+
* later ({@link drive}'s `settleRound`), and losing the race to a
|
|
1393
|
+
* shutdown must not lose the classification with it: a `released` row
|
|
1394
|
+
* carries no wake time and no token, so a run that suspended on a
|
|
1395
|
+
* 150-second clock came back as a run whose owner merely went away, and
|
|
1396
|
+
* the sweeper had to re-drive it blind instead of waking it when its
|
|
1397
|
+
* deadline fell due (release rehearsal).
|
|
1398
|
+
*
|
|
1399
|
+
* A flow that has NOT asked to suspend is a genuine shutdown or heartbeat
|
|
1400
|
+
* self-interrupt, and `released` is exactly what it is (issue #39).
|
|
1401
|
+
*/
|
|
1402
|
+
const interruptedWaiting = (
|
|
1403
|
+
runId: string,
|
|
1404
|
+
instance: FlowRuntime.FlowInstance["Service"] | undefined
|
|
1405
|
+
): Effect.Effect<DurableEngineState.Waiting> =>
|
|
1406
|
+
Effect.gen(function*() {
|
|
1407
|
+
if (instance === undefined || !instance.suspended) return { reason: "released" } as const
|
|
1408
|
+
const declared = instance.waiting
|
|
1409
|
+
if (declared !== undefined) return declared
|
|
1410
|
+
const pendingClocks = yield* engineState.pendingClocks({ executionId: runId })
|
|
1411
|
+
return pendingClocks.length > 0
|
|
1412
|
+
? {
|
|
1413
|
+
reason: "timer",
|
|
1414
|
+
wakeAt: Math.min(...pendingClocks.map((clock) => clock.dueAtMs))
|
|
1415
|
+
} as const
|
|
1416
|
+
: { reason: "event" } as const
|
|
1417
|
+
})
|
|
1418
|
+
|
|
1419
|
+
const releaseOwned = (
|
|
1420
|
+
runId: string,
|
|
1421
|
+
state: RunState,
|
|
1422
|
+
cause: ReleaseCause,
|
|
1423
|
+
round?: {
|
|
1424
|
+
readonly flow: Flow.Any
|
|
1425
|
+
readonly instance: FlowRuntime.FlowInstance["Service"]
|
|
1426
|
+
} | undefined
|
|
1427
|
+
): Effect.Effect<void> =>
|
|
1428
|
+
Effect.gen(function*() {
|
|
1429
|
+
// Park atomically with releasing ownership (`park` is owner-fenced). The
|
|
1430
|
+
// durable waiting row is what makes a released run visible to the
|
|
1431
|
+
// parked-run sweeper: without it nothing ever re-drives the run and
|
|
1432
|
+
// a durable `requestCancel` against it is write-only forever
|
|
1433
|
+
// (issue #39). On genuine fence loss the park reports NotFound
|
|
1434
|
+
// harmlessly, exactly like the transition below.
|
|
1435
|
+
const waiting = yield* interruptedWaiting(runId, round?.instance)
|
|
1436
|
+
// A run whose flow asked to suspend carries the suspension on its row,
|
|
1437
|
+
// exactly as the round's own settlement writes it. `poll` publishes a
|
|
1438
|
+
// state with no `result` as "not settled yet", so a park recorded
|
|
1439
|
+
// without one was invisible to every resumer: `AgentSession`'s
|
|
1440
|
+
// `awaitParked` polls `poll` and answers `unknown` for anything but a
|
|
1441
|
+
// published `Suspended`, so `smithers approve` and `smithers run
|
|
1442
|
+
// --resume` accepted the request and then drove nothing in the release rehearsal.
|
|
1443
|
+
const suspension = round !== undefined && waiting.reason !== "released"
|
|
1444
|
+
? (yield* encodeResult(
|
|
1445
|
+
round.flow,
|
|
1446
|
+
new Flow.Suspended({ cause: round.instance.cause })
|
|
1447
|
+
)).encoded
|
|
1448
|
+
: undefined
|
|
1449
|
+
yield* transitionAndRecord(
|
|
1450
|
+
runId,
|
|
1451
|
+
"suspended",
|
|
1452
|
+
yield* encodeState(
|
|
1453
|
+
suspension === undefined
|
|
1454
|
+
? withoutResult(state)
|
|
1455
|
+
: { ...withoutResult(state), result: suspension }
|
|
1456
|
+
),
|
|
1457
|
+
// Why the run was released (#3328): an operator reading a parked run
|
|
1458
|
+
// must be able to tell a lapsed lease on a live host from a shutdown.
|
|
1459
|
+
{ decision: "interrupt-released", owner: dependencies.owner, cause },
|
|
1460
|
+
undefined,
|
|
1461
|
+
undefined,
|
|
1462
|
+
waiting
|
|
1463
|
+
)
|
|
1464
|
+
})
|
|
1465
|
+
|
|
1466
|
+
/**
|
|
1467
|
+
* Discriminates an interruption cause by durable state: only an
|
|
1468
|
+
* interruption backed by a recorded cancel request closes the run;
|
|
1469
|
+
* anything else releases it for reclaim (issue #26), under the waiting
|
|
1470
|
+
* reason the flow had already declared for itself when it has one.
|
|
1471
|
+
*/
|
|
1472
|
+
const settleInterrupted = (
|
|
1473
|
+
runId: string,
|
|
1474
|
+
state: RunState,
|
|
1475
|
+
cause: ReleaseCause,
|
|
1476
|
+
round?: {
|
|
1477
|
+
readonly flow: Flow.Any
|
|
1478
|
+
readonly instance: FlowRuntime.FlowInstance["Service"]
|
|
1479
|
+
} | undefined
|
|
1480
|
+
): Effect.Effect<void> =>
|
|
1481
|
+
store.get(runId).pipe(
|
|
1482
|
+
Effect.flatMap((row) => {
|
|
1483
|
+
// Shutdown can arrive after the claim CAS but before activation, or
|
|
1484
|
+
// after activation but before/after the execution race. Release only
|
|
1485
|
+
// this incarnation's persisted fence; a settled or foreign row stays
|
|
1486
|
+
// untouched, including its waiting reason.
|
|
1487
|
+
if (row.claim !== null && row.claimedAtMs !== null && sameOwner(row.claim, dependencies.owner)) {
|
|
1488
|
+
return abandon(runId, row.claimedAtMs)
|
|
1489
|
+
}
|
|
1490
|
+
if (row.status !== "running" || row.owner === null || !sameOwner(row.owner, dependencies.owner)) {
|
|
1491
|
+
return Effect.void
|
|
1492
|
+
}
|
|
1493
|
+
return row.cancelRequestedAtMs !== null ? cancelOwned(runId, state) : releaseOwned(runId, state, cause, round)
|
|
1494
|
+
}),
|
|
1495
|
+
Effect.catch((error) => error.code === "not_found_row" ? Effect.void : Effect.die(error))
|
|
1496
|
+
)
|
|
1497
|
+
|
|
1498
|
+
/**
|
|
1499
|
+
* Encodes a settlement for the row, and never fails.
|
|
1500
|
+
*
|
|
1501
|
+
* The encode used to be guarded with `Effect.orDie`, which made a
|
|
1502
|
+
* settlement the flow's own codec rejects fatal for the drain rather than
|
|
1503
|
+
* terminal for the run. `agent/run` declares both channels
|
|
1504
|
+
* `Schema.Unknown`, so every `Data.TaggedError` an agent body fails with
|
|
1505
|
+
* is such a settlement, and the release validation watched the whole
|
|
1506
|
+
* chain: the drain died with `SchemaError: Expected JSON value at
|
|
1507
|
+
* ["exit"]["cause"][0]["error"]`, `engine.db` kept the run `running`
|
|
1508
|
+
* under a pid that had exited while `control.db` said `failed`, and the
|
|
1509
|
+
* next process's stale-running sweep stole the row and billed the
|
|
1510
|
+
* provider seat a second time. `ExitEncoding.encode` fails closed
|
|
1511
|
+
* instead: on rejection it answers a JSON projection of the settlement
|
|
1512
|
+
* and a `note`, and a noted settlement is written `failed`.
|
|
1513
|
+
*/
|
|
1514
|
+
const encodeResult = (
|
|
1515
|
+
flow: Flow.Any,
|
|
1516
|
+
result: Flow.Result<unknown, unknown>
|
|
1517
|
+
): Effect.Effect<ExitEncoding.EncodedResult> => ExitEncoding.encode(flow, result)
|
|
1518
|
+
|
|
1519
|
+
/**
|
|
1520
|
+
* Re-encodes a handoff payload through the target flow's own codec, so the
|
|
1521
|
+
* bytes the next round's row holds are the bytes `ensureRun` would write
|
|
1522
|
+
* for the same invocation.
|
|
1523
|
+
*/
|
|
1524
|
+
const normalizePayload = (
|
|
1525
|
+
flow: Flow.Any,
|
|
1526
|
+
payload: unknown
|
|
1527
|
+
): Effect.Effect<unknown> => {
|
|
1528
|
+
const codec = Schema.toCodecJson(flow.payloadSchema)
|
|
1529
|
+
return (Schema.decodeUnknownEffect(codec)(payload).pipe(
|
|
1530
|
+
Effect.orDie,
|
|
1531
|
+
Effect.flatMap((decoded) => Schema.encodeEffect(codec)(decoded)),
|
|
1532
|
+
Effect.orDie
|
|
1533
|
+
) as Effect.Effect<unknown>)
|
|
1534
|
+
}
|
|
1535
|
+
|
|
1536
|
+
/**
|
|
1537
|
+
* Creates one run row, tolerating only an existing row with identical
|
|
1538
|
+
* execution identity and encoded invocation data.
|
|
1539
|
+
*/
|
|
1540
|
+
const ensureCreatedRun = (options: {
|
|
1541
|
+
readonly flowName: string
|
|
1542
|
+
readonly executionId: string
|
|
1543
|
+
readonly stateJson: string
|
|
1544
|
+
readonly payload: unknown
|
|
1545
|
+
readonly lineageId: string
|
|
1546
|
+
readonly roundOrdinal: number
|
|
1547
|
+
readonly parentRunId?: string | undefined
|
|
1548
|
+
/** The authority the new row would persist, which a join must cover. */
|
|
1549
|
+
readonly capabilityCeilings: NonNullable<RunState["capabilityCeilings"]>
|
|
1550
|
+
readonly onCreated: Effect.Effect<void, never, never>
|
|
1551
|
+
}): Effect.Effect<void, never, never> =>
|
|
1552
|
+
Effect.gen(function*() {
|
|
1553
|
+
const created = yield* store.create(options.executionId, options.stateJson, {
|
|
1554
|
+
...(options.parentRunId === undefined ? {} : { parentRunId: options.parentRunId }),
|
|
1555
|
+
lineageId: options.lineageId,
|
|
1556
|
+
roundOrdinal: options.roundOrdinal
|
|
1557
|
+
}).pipe(Effect.exit)
|
|
1558
|
+
if (Exit.isSuccess(created)) {
|
|
1559
|
+
return yield* options.onCreated
|
|
1560
|
+
}
|
|
1561
|
+
|
|
1562
|
+
// A cause carrying no `Fail` reason — an interrupt-only cause, or a
|
|
1563
|
+
// bare defect — used to reach `Option.getOrThrow(Exit.findErrorOption(...))`,
|
|
1564
|
+
// which threw a raw `NoSuchElementError` defect and discarded the
|
|
1565
|
+
// original cause. A caller that interrupted the fiber while
|
|
1566
|
+
// `store.create` was inside its write transaction saw a crash rather
|
|
1567
|
+
// than the cancellation it asked for. `Cause.findFail` hands back
|
|
1568
|
+
// that residual cause typed `Cause<never>`, so re-raising it verbatim
|
|
1569
|
+
// keeps an interrupt an interrupt (issue #151).
|
|
1570
|
+
const failure = Cause.findFail(created.cause)
|
|
1571
|
+
if (Result.isFailure(failure)) {
|
|
1572
|
+
return yield* Effect.failCause(failure.failure)
|
|
1573
|
+
}
|
|
1574
|
+
const error = failure.success.error
|
|
1575
|
+
if (!(error instanceof RunStore.RunStoreError) || error.code !== "constraint") {
|
|
1576
|
+
return yield* Effect.die(error)
|
|
1577
|
+
}
|
|
1578
|
+
const existing = yield* store.get(options.executionId).pipe(Effect.orDie)
|
|
1579
|
+
const persisted = yield* decodeState(existing.stateJson)
|
|
1580
|
+
const sameParent = options.parentRunId === undefined ||
|
|
1581
|
+
existing.parentRunId === options.parentRunId
|
|
1582
|
+
const conflict = persisted.flowName !== options.flowName
|
|
1583
|
+
? {
|
|
1584
|
+
field: "flow" as const,
|
|
1585
|
+
expected: persisted.flowName,
|
|
1586
|
+
actual: options.flowName
|
|
1587
|
+
}
|
|
1588
|
+
: !samePayload(persisted.payload, options.payload)
|
|
1589
|
+
? {
|
|
1590
|
+
field: "payload" as const,
|
|
1591
|
+
expected: "the encoded payload the execution was admitted with",
|
|
1592
|
+
actual: "a different encoded payload"
|
|
1593
|
+
}
|
|
1594
|
+
: !FlowEngine.joinable(
|
|
1595
|
+
// A row with no persisted authority ran unrestricted, and is
|
|
1596
|
+
// refused further dispatch; only an unrestricted caller joins it.
|
|
1597
|
+
persisted.capabilityCeilings ?? [],
|
|
1598
|
+
options.capabilityCeilings
|
|
1599
|
+
)
|
|
1600
|
+
? {
|
|
1601
|
+
// A join answers the result the admitted authority produced.
|
|
1602
|
+
field: "capabilities" as const,
|
|
1603
|
+
expected: "a capability ceiling covering the one the execution was admitted with",
|
|
1604
|
+
actual: "a capability ceiling that does not cover it"
|
|
1605
|
+
}
|
|
1606
|
+
: existing.lineageId !== options.lineageId
|
|
1607
|
+
? {
|
|
1608
|
+
field: "lineage" as const,
|
|
1609
|
+
expected: existing.lineageId ?? "no lineage",
|
|
1610
|
+
actual: options.lineageId
|
|
1611
|
+
}
|
|
1612
|
+
: existing.roundOrdinal !== options.roundOrdinal
|
|
1613
|
+
? {
|
|
1614
|
+
field: "round" as const,
|
|
1615
|
+
expected: String(existing.roundOrdinal),
|
|
1616
|
+
actual: String(options.roundOrdinal)
|
|
1617
|
+
}
|
|
1618
|
+
: !sameParent
|
|
1619
|
+
? {
|
|
1620
|
+
field: "parent" as const,
|
|
1621
|
+
expected: existing.parentRunId ?? "no parent",
|
|
1622
|
+
// `!sameParent` implies the requested parent is present: an
|
|
1623
|
+
// omitted parent is deliberately compatible with any persisted
|
|
1624
|
+
// parent when joining an already-created root.
|
|
1625
|
+
actual: options.parentRunId
|
|
1626
|
+
}
|
|
1627
|
+
: undefined
|
|
1628
|
+
if (conflict !== undefined) {
|
|
1629
|
+
return yield* Effect.die(
|
|
1630
|
+
new FlowEngine.ExecutionIdentityConflict({
|
|
1631
|
+
executionId: options.executionId,
|
|
1632
|
+
status: existing.status,
|
|
1633
|
+
...conflict,
|
|
1634
|
+
message: `execution ${options.executionId} already belongs to ${conflict.field} identity ` +
|
|
1635
|
+
`${conflict.expected}; it cannot be reused for ${conflict.actual}`
|
|
1636
|
+
})
|
|
1637
|
+
)
|
|
1638
|
+
}
|
|
1639
|
+
})
|
|
1640
|
+
|
|
1641
|
+
const coordinatorDeferred = yield* Deferred.make<RunCoordinator.RunCoordinator<string, never>>()
|
|
1642
|
+
|
|
1643
|
+
/**
|
|
1644
|
+
* Observes a durably recorded cancellation request
|
|
1645
|
+
* (`RunStore.requestCancel` / `cancel_requested_at_ms`) from another
|
|
1646
|
+
* process. Polls on the heartbeat cadence — the request is unfenced, so
|
|
1647
|
+
* only the owner can act on it, and it must act within a poll interval
|
|
1648
|
+
* (issue #11). Completes when a request is observed; races against the
|
|
1649
|
+
* flow like the heartbeat loop.
|
|
1650
|
+
*/
|
|
1651
|
+
const cancelPollLoop = (executionId: string): Effect.Effect<CancelRequested> =>
|
|
1652
|
+
Effect.gen(function*() {
|
|
1653
|
+
// Check-first: a request that raced in just before activation is
|
|
1654
|
+
// observed without waiting out a full heartbeat (issue #27).
|
|
1655
|
+
while (true) {
|
|
1656
|
+
const requested = yield* store.get(executionId).pipe(
|
|
1657
|
+
Effect.map((row) => row.cancelRequestedAtMs !== null),
|
|
1658
|
+
Effect.catch(() => Effect.succeed(false))
|
|
1659
|
+
)
|
|
1660
|
+
if (requested) {
|
|
1661
|
+
yield* store.acknowledgeCancel(
|
|
1662
|
+
executionId,
|
|
1663
|
+
dependencies.owner,
|
|
1664
|
+
yield* Clock.currentTimeMillis.pipe(Effect.map(Math.floor))
|
|
1665
|
+
).pipe(Effect.orDie)
|
|
1666
|
+
return cancelRequested
|
|
1667
|
+
}
|
|
1668
|
+
yield* Effect.sleep(Ownership.heartbeatInterval)
|
|
1669
|
+
}
|
|
1670
|
+
})
|
|
1671
|
+
|
|
1672
|
+
/**
|
|
1673
|
+
* What one handoff has to settle: the round that produced it, the row it
|
|
1674
|
+
* runs under (which carries the lineage columns), the state it was
|
|
1675
|
+
* activated with, its declaration, and the invocation it named.
|
|
1676
|
+
*/
|
|
1677
|
+
interface HandoffSeam {
|
|
1678
|
+
readonly executionId: string
|
|
1679
|
+
readonly row: RunStore.RunRow
|
|
1680
|
+
readonly state: RunState
|
|
1681
|
+
readonly flow: Flow.Any
|
|
1682
|
+
readonly handoff: Flow.Handoff
|
|
1683
|
+
}
|
|
1684
|
+
|
|
1685
|
+
/**
|
|
1686
|
+
* Opens the next round, and closes this one, in ONE transaction.
|
|
1687
|
+
*
|
|
1688
|
+
* `ensureRun`'s parent-edge/run-row pairing (issue #80) is the precedent
|
|
1689
|
+
* and the reason: a crash between the two writes would leave a terminal
|
|
1690
|
+
* round whose successor was never created, and nothing would ever look for
|
|
1691
|
+
* it again — the lineage would end silently at a round that says it handed
|
|
1692
|
+
* off. The stores' own writes become savepoints of this transaction, so
|
|
1693
|
+
* either both commit or neither.
|
|
1694
|
+
*
|
|
1695
|
+
* The next round is a run row chained through the RESERVED
|
|
1696
|
+
* `parent_run_id` column, not a `flows_run_parents` edge: that table is
|
|
1697
|
+
* the subflow DAG cycle detection walks, and a round is the same run
|
|
1698
|
+
* continuing rather than a child being spawned
|
|
1699
|
+
* (`packages/smithers/flows/engine/docs/concepts/trampoline-rounds.md`).
|
|
1700
|
+
*/
|
|
1701
|
+
const continueLineage = (
|
|
1702
|
+
seam: HandoffSeam,
|
|
1703
|
+
advanced: { readonly round: FlowEngine.Round.Round; readonly executionId: string }
|
|
1704
|
+
): Effect.Effect<void> =>
|
|
1705
|
+
Effect.gen(function*() {
|
|
1706
|
+
// The handoff payload travels encoded, and the next round's row has to
|
|
1707
|
+
// hold the same value `ensureRun` would write for it: the root caller
|
|
1708
|
+
// re-enters `execute` for this round, and its identical-create check
|
|
1709
|
+
// compares encoded payloads structurally, so object key order cannot
|
|
1710
|
+
// separate two encodings of the same value. The execution id itself
|
|
1711
|
+
// uses canonical RFC 8785 JSON, so the identity and payload checks
|
|
1712
|
+
// agree by construction.
|
|
1713
|
+
//
|
|
1714
|
+
// A target this process does not run has no codec to normalize
|
|
1715
|
+
// through, and the round is still created with the payload verbatim —
|
|
1716
|
+
// the same posture the wake path takes for an unregistered flow, and
|
|
1717
|
+
// the reason a lineage survives a worker that only knows some of its
|
|
1718
|
+
// legs.
|
|
1719
|
+
const target = registrations.get(seam.handoff.flow)?.at(-1)
|
|
1720
|
+
const payload = target === undefined
|
|
1721
|
+
? seam.handoff.payload
|
|
1722
|
+
: yield* normalizePayload(target.flow, seam.handoff.payload)
|
|
1723
|
+
const nextCeilings = [
|
|
1724
|
+
// The execution guard rejects missing persisted authority before a handler can hand off.
|
|
1725
|
+
...seam.state.capabilityCeilings!,
|
|
1726
|
+
...(seam.handoff.capabilityCeilings ?? [[]]),
|
|
1727
|
+
// The successor records its own declaration, as admission would, so
|
|
1728
|
+
// a later join compares against the authority it actually runs
|
|
1729
|
+
// under. A target this process does not run has none here; the root
|
|
1730
|
+
// caller's re-entry then carries a declaration the row lacks and is
|
|
1731
|
+
// refused rather than trusted.
|
|
1732
|
+
...(target === undefined
|
|
1733
|
+
? []
|
|
1734
|
+
: Flow.parseCapabilityCeilings(Flow.capabilityCeilings(target.flow.annotations)))
|
|
1735
|
+
]
|
|
1736
|
+
const nextStateJson = yield* encodeState({
|
|
1737
|
+
version: 1,
|
|
1738
|
+
flowName: seam.handoff.flow,
|
|
1739
|
+
payload,
|
|
1740
|
+
capabilityCeilings: nextCeilings,
|
|
1741
|
+
...(seam.state.parentExecutionId === undefined
|
|
1742
|
+
? {}
|
|
1743
|
+
: { parentExecutionId: seam.state.parentExecutionId }),
|
|
1744
|
+
...(seam.state.onParentExit === undefined ? {} : { onParentExit: seam.state.onParentExit }),
|
|
1745
|
+
...(seam.state.maxRounds === undefined ? {} : { maxRounds: seam.state.maxRounds }),
|
|
1746
|
+
...(seam.handoff.deadline === undefined ? {} : { deadline: seam.handoff.deadline })
|
|
1747
|
+
})
|
|
1748
|
+
const settledStateJson = yield* encodeState({
|
|
1749
|
+
...seam.state,
|
|
1750
|
+
result: (yield* encodeResult(seam.flow, seam.handoff)).encoded
|
|
1751
|
+
})
|
|
1752
|
+
const cancelled = { _tag: "HandoffCancelled" } as const
|
|
1753
|
+
const fenceLost = { _tag: "HandoffFenceLost" } as const
|
|
1754
|
+
const committed = yield* Effect.result(
|
|
1755
|
+
transactState(Effect.gen(function*() {
|
|
1756
|
+
// The next round's id is DERIVED from (lineage, ordinal), so a
|
|
1757
|
+
// re-drive finds the exact row it already opened. Only an
|
|
1758
|
+
// identical create is tolerated; a collision in flow, payload,
|
|
1759
|
+
// parent, lineage, or ordinal is a defect.
|
|
1760
|
+
yield* ensureCreatedRun({
|
|
1761
|
+
flowName: seam.handoff.flow,
|
|
1762
|
+
executionId: advanced.executionId,
|
|
1763
|
+
stateJson: nextStateJson,
|
|
1764
|
+
payload,
|
|
1765
|
+
lineageId: advanced.round.rootExecutionId,
|
|
1766
|
+
roundOrdinal: advanced.round.ordinal,
|
|
1767
|
+
parentRunId: seam.executionId,
|
|
1768
|
+
capabilityCeilings: nextCeilings,
|
|
1769
|
+
onCreated: emitDecision(advanced.executionId, {
|
|
1770
|
+
decision: "created",
|
|
1771
|
+
state: JSON.parse(nextStateJson),
|
|
1772
|
+
lineageId: advanced.round.rootExecutionId,
|
|
1773
|
+
roundOrdinal: advanced.round.ordinal,
|
|
1774
|
+
parentExecutionId: seam.executionId
|
|
1775
|
+
})
|
|
1776
|
+
})
|
|
1777
|
+
// DECIDED (2026-08-11, pending review): a handed-off round settles
|
|
1778
|
+
// `completed`. The round did finish, and adding a `Continued`
|
|
1779
|
+
// status would widen every status reader for a distinction the
|
|
1780
|
+
// `handed-off` decision and lineage columns already record.
|
|
1781
|
+
//
|
|
1782
|
+
// DECIDED (2026-08-11, pending review): cancellation guards the
|
|
1783
|
+
// handoff transition. If it raced the last poll, failing the outer
|
|
1784
|
+
// transaction rolls successor creation back before the ordinary
|
|
1785
|
+
// cancellation path closes the owned round.
|
|
1786
|
+
const transitioned = yield* transitionAndRecord(
|
|
1787
|
+
seam.executionId,
|
|
1788
|
+
"completed",
|
|
1789
|
+
settledStateJson,
|
|
1790
|
+
{
|
|
1791
|
+
decision: "handed-off",
|
|
1792
|
+
status: "completed",
|
|
1793
|
+
flow: seam.handoff.flow,
|
|
1794
|
+
lineageId: advanced.round.rootExecutionId,
|
|
1795
|
+
roundOrdinal: advanced.round.ordinal,
|
|
1796
|
+
nextExecutionId: advanced.executionId,
|
|
1797
|
+
owner: dependencies.owner
|
|
1798
|
+
},
|
|
1799
|
+
{ cancelRequested: "absent" }
|
|
1800
|
+
)
|
|
1801
|
+
if (transitioned._tag === "Transitioned") return
|
|
1802
|
+
return yield* Effect.fail(
|
|
1803
|
+
transitioned._tag === "GuardFailed" ? cancelled : fenceLost
|
|
1804
|
+
)
|
|
1805
|
+
})).pipe(Effect.catchTag("@smthrs/journal/JournalError", Effect.die))
|
|
1806
|
+
)
|
|
1807
|
+
if (Result.isFailure(committed)) {
|
|
1808
|
+
if (committed.failure._tag === "HandoffCancelled") {
|
|
1809
|
+
yield* cancelOwned(seam.executionId, seam.state)
|
|
1810
|
+
}
|
|
1811
|
+
return
|
|
1812
|
+
}
|
|
1813
|
+
// The root caller follows the lineage itself, but a discarded (or
|
|
1814
|
+
// orphaned) one does not exist to follow it, so the successor is woken
|
|
1815
|
+
// here rather than left for a sweep that has no reason to look at it.
|
|
1816
|
+
const activeCoordinator = yield* Deferred.await(coordinatorDeferred)
|
|
1817
|
+
yield* activeCoordinator.wake(advanced.executionId)
|
|
1818
|
+
})
|
|
1819
|
+
|
|
1820
|
+
/**
|
|
1821
|
+
* Announces a committed terminal settlement: to any in-process caller
|
|
1822
|
+
* parked on this run's poll loop, and to every parent attached to the
|
|
1823
|
+
* logical run, which is runnable now. That wake is the only thing that
|
|
1824
|
+
* says so: a child settlement never passes through `scheduleResume`.
|
|
1825
|
+
*
|
|
1826
|
+
* A diamond's second parent lives only in its durable edge (issue #2500),
|
|
1827
|
+
* and `ensureRun` records an edge under the round the parent joined, so
|
|
1828
|
+
* the edges are gathered from every round of the lineage, deduplicated
|
|
1829
|
+
* with the creating parent `state_json` carries. Each is recorded for the
|
|
1830
|
+
* same reason `scheduleResume` records: the round this wake schedules is
|
|
1831
|
+
* the one a host's park guard has to recognize.
|
|
1832
|
+
*/
|
|
1833
|
+
const announceSettled = (executionId: string, state: RunState): Effect.Effect<void> =>
|
|
1834
|
+
Effect.gen(function*() {
|
|
1835
|
+
yield* wakeBus.wake(executionId)
|
|
1836
|
+
const parents = new Set<string>()
|
|
1837
|
+
if (state.parentExecutionId !== undefined) parents.add(state.parentExecutionId)
|
|
1838
|
+
const rounds = yield* lifecycle.rounds(executionId).pipe(Effect.orDie)
|
|
1839
|
+
for (const roundId of new Set([executionId, ...rounds.map((round) => round.runId)])) {
|
|
1840
|
+
for (const edge of yield* engineState.runParents(roundId)) parents.add(edge.parentId)
|
|
1841
|
+
}
|
|
1842
|
+
if (parents.size === 0) return
|
|
1843
|
+
const activeCoordinator = yield* Deferred.await(coordinatorDeferred)
|
|
1844
|
+
for (const parentId of parents) {
|
|
1845
|
+
yield* recordParentResume(parentId)
|
|
1846
|
+
yield* activeCoordinator.wake(parentId)
|
|
1847
|
+
yield* wakeBus.wake(parentId)
|
|
1848
|
+
}
|
|
1849
|
+
})
|
|
1850
|
+
|
|
1851
|
+
/**
|
|
1852
|
+
* Ends a lineage that asked for one round past its declared budget.
|
|
1853
|
+
*
|
|
1854
|
+
* The round itself ran to completion; what is refused is the handoff, so
|
|
1855
|
+
* the round settles `failed` carrying the typed refusal and no successor
|
|
1856
|
+
* is created (`packages/smithers/flows/engine/docs/concepts/trampoline-rounds.md`).
|
|
1857
|
+
*/
|
|
1858
|
+
const endLineage = (
|
|
1859
|
+
seam: HandoffSeam,
|
|
1860
|
+
error: Flow.MaxRoundsExceeded
|
|
1861
|
+
): Effect.Effect<void> =>
|
|
1862
|
+
Effect.gen(function*() {
|
|
1863
|
+
const stateJson = yield* encodeState({
|
|
1864
|
+
...seam.state,
|
|
1865
|
+
result: (yield* encodeResult(seam.flow, new Flow.Complete({ exit: Exit.die(error) }))).encoded
|
|
1866
|
+
})
|
|
1867
|
+
const transitioned = yield* transitionAndRecord(
|
|
1868
|
+
seam.executionId,
|
|
1869
|
+
"failed",
|
|
1870
|
+
stateJson,
|
|
1871
|
+
{
|
|
1872
|
+
decision: "lineage-exhausted",
|
|
1873
|
+
status: "failed",
|
|
1874
|
+
lineageId: error.lineageId,
|
|
1875
|
+
maxRounds: error.maxRounds,
|
|
1876
|
+
owner: dependencies.owner
|
|
1877
|
+
},
|
|
1878
|
+
{ cancelRequested: "absent" },
|
|
1879
|
+
// The lineage is over, not continuing: this round is the last one,
|
|
1880
|
+
// so it ends its children like any other terminal exit. A round that
|
|
1881
|
+
// HANDS OFF settles through `continueLineage` instead and deliberately
|
|
1882
|
+
// does not, because the lineage — and therefore the parent the
|
|
1883
|
+
// children are linked to — carries on.
|
|
1884
|
+
applyChildExitPolicy(seam.executionId)
|
|
1885
|
+
)
|
|
1886
|
+
if (transitioned._tag === "GuardFailed") {
|
|
1887
|
+
yield* cancelOwned(seam.executionId, seam.state)
|
|
1888
|
+
} else if (transitioned._tag === "Transitioned") {
|
|
1889
|
+
yield* announceSettled(seam.executionId, seam.state)
|
|
1890
|
+
}
|
|
1891
|
+
})
|
|
1892
|
+
|
|
1893
|
+
/** Fails a run whose persisted round identity or budget is malformed. */
|
|
1894
|
+
const endInvalidRound = (
|
|
1895
|
+
seam: HandoffSeam,
|
|
1896
|
+
error: FlowEngine.Round.InvalidRound
|
|
1897
|
+
): Effect.Effect<void> =>
|
|
1898
|
+
Effect.gen(function*() {
|
|
1899
|
+
const stateJson = yield* encodeState({
|
|
1900
|
+
...seam.state,
|
|
1901
|
+
result: (yield* encodeResult(seam.flow, new Flow.Complete({ exit: Exit.die(error) }))).encoded
|
|
1902
|
+
})
|
|
1903
|
+
const transitioned = yield* transitionAndRecord(
|
|
1904
|
+
seam.executionId,
|
|
1905
|
+
"failed",
|
|
1906
|
+
stateJson,
|
|
1907
|
+
{
|
|
1908
|
+
decision: "round-invalid",
|
|
1909
|
+
status: "failed",
|
|
1910
|
+
message: error.message,
|
|
1911
|
+
owner: dependencies.owner
|
|
1912
|
+
},
|
|
1913
|
+
{ cancelRequested: "absent" },
|
|
1914
|
+
applyChildExitPolicy(seam.executionId)
|
|
1915
|
+
)
|
|
1916
|
+
if (transitioned._tag === "GuardFailed") {
|
|
1917
|
+
yield* cancelOwned(seam.executionId, seam.state)
|
|
1918
|
+
} else if (transitioned._tag === "Transitioned") {
|
|
1919
|
+
yield* announceSettled(seam.executionId, seam.state)
|
|
1920
|
+
}
|
|
1921
|
+
})
|
|
1922
|
+
|
|
1923
|
+
/**
|
|
1924
|
+
* The handoff seam: the one place a round's terminal settlement and its
|
|
1925
|
+
* successor are decided together.
|
|
1926
|
+
*/
|
|
1927
|
+
const handOff = (seam: HandoffSeam): Effect.Effect<void, never, Crypto.Crypto> =>
|
|
1928
|
+
FlowEngine.Round.next(
|
|
1929
|
+
{
|
|
1930
|
+
// The row's lineage_id was written from a round's rootExecutionId,
|
|
1931
|
+
// so this rehydrates the brand; `Round.next` still validates it.
|
|
1932
|
+
rootExecutionId: seam.row.lineageId! as FlowEngine.Round.RootExecutionId,
|
|
1933
|
+
ordinal: seam.row.roundOrdinal!
|
|
1934
|
+
},
|
|
1935
|
+
// The origin persisted its budget into every round's state, so a
|
|
1936
|
+
// multi-flow handoff cannot reset the lineage by changing targets.
|
|
1937
|
+
{ flowName: seam.flow._tag, maxRounds: seam.state.maxRounds }
|
|
1938
|
+
).pipe(
|
|
1939
|
+
Effect.flatMap((advanced) => continueLineage(seam, advanced)),
|
|
1940
|
+
Effect.catch((error) =>
|
|
1941
|
+
error instanceof FlowEngine.Round.InvalidRound
|
|
1942
|
+
? endInvalidRound(seam, error)
|
|
1943
|
+
: endLineage(seam, error)
|
|
1944
|
+
)
|
|
1945
|
+
)
|
|
1946
|
+
|
|
1947
|
+
const drive = (executionId: string): Effect.Effect<void, never, Crypto.Crypto> =>
|
|
1948
|
+
Effect.suspend(() => {
|
|
1949
|
+
let interruptState: RunState | undefined
|
|
1950
|
+
let round: { readonly flow: Flow.Any; readonly instance: FlowRuntime.FlowInstance["Service"] } | undefined
|
|
1951
|
+
let durableParkAccepted = false
|
|
1952
|
+
// Set by the heartbeat loop when this round's lease lapses, so the
|
|
1953
|
+
// release it causes records why (#3328).
|
|
1954
|
+
let lapsedMs: number | undefined
|
|
1955
|
+
return Effect.gen(function*() {
|
|
1956
|
+
const initial = yield* store.get(executionId).pipe(
|
|
1957
|
+
Effect.catch((error) =>
|
|
1958
|
+
error.code === "not_found_row"
|
|
1959
|
+
? Effect.succeed(undefined)
|
|
1960
|
+
: Effect.die(error)
|
|
1961
|
+
)
|
|
1962
|
+
)
|
|
1963
|
+
if (initial === undefined) return
|
|
1964
|
+
|
|
1965
|
+
const state = yield* decodeState(initial.stateJson)
|
|
1966
|
+
interruptState = state
|
|
1967
|
+
const registration = registrations.get(state.flowName)?.at(-1)
|
|
1968
|
+
if (registration === undefined) {
|
|
1969
|
+
// A coalesced wake can outlive terminal settlement. There is no
|
|
1970
|
+
// parked work left for a handler to reclaim.
|
|
1971
|
+
if (initial.status === "cancelled" || initial.status === "completed" || initial.status === "failed") return
|
|
1972
|
+
// Cancelling needs no handler. A parked run whose cancellation was
|
|
1973
|
+
// durably requested used to be dropped here with everything else,
|
|
1974
|
+
// and nothing else could ever reach it: the sweep woke the row every
|
|
1975
|
+
// heartbeat and every wake bailed at this line, so a cancel written
|
|
1976
|
+
// by a control-only process — the operator CLI, a second engine that
|
|
1977
|
+
// registers other flows — stayed write-only forever while the run's
|
|
1978
|
+
// linked children kept going. The terminal transition, the cascade
|
|
1979
|
+
// over the durable edge table, and the interruption record are all
|
|
1980
|
+
// written from the run ROW, so a process that could never execute
|
|
1981
|
+
// this flow can still close it. Claiming first is what makes the
|
|
1982
|
+
// close fenced: `cancelOwned` is owner-fenced, so exactly one of the
|
|
1983
|
+
// sweeping processes wins and the rest see a lost CAS.
|
|
1984
|
+
//
|
|
1985
|
+
// `running` is here for the same reason `suspended` is. The row a
|
|
1986
|
+
// hard-killed owner leaves behind is `running` with a frozen
|
|
1987
|
+
// heartbeat and no waiting row, so the stale-running sweep is the
|
|
1988
|
+
// only thing that reaches it, and it arrives through this same
|
|
1989
|
+
// function: without this arm a control-only process could close a
|
|
1990
|
+
// PARKED run of an unregistered flow and not a hard-killed one,
|
|
1991
|
+
// though the two closes are the identical durable write. Nothing is
|
|
1992
|
+
// weakened by admitting it, because the arbitration is
|
|
1993
|
+
// `claimAndActivate`'s and not this guard's: a fresh lease or a live
|
|
1994
|
+
// owner refuses the takeover and the run is left exactly as it was.
|
|
1995
|
+
if (
|
|
1996
|
+
(initial.status === "pending" || initial.status === "suspended" || initial.status === "running") &&
|
|
1997
|
+
initial.cancelRequestedAtMs !== null
|
|
1998
|
+
) {
|
|
1999
|
+
if (!(yield* claimAndActivate(initial))) {
|
|
2000
|
+
return
|
|
2001
|
+
}
|
|
2002
|
+
return yield* cancelOwned(executionId, withoutResult(state))
|
|
2003
|
+
}
|
|
2004
|
+
// A wake for a flow this process has not registered — after a full
|
|
2005
|
+
// restart the sweep re-drives released rows before (or without)
|
|
2006
|
+
// the flow ever registering here. Dropping the wake silently made
|
|
2007
|
+
// the #39 reclaim guarantee invisibly conditional on registration
|
|
2008
|
+
// (issue #62): warn (once per run, the sweep retries every
|
|
2009
|
+
// heartbeat) and leave the durable waiting row untouched so any
|
|
2010
|
+
// process that does register the flow still reclaims the run.
|
|
2011
|
+
if (!warnedUnregistered.has(executionId)) {
|
|
2012
|
+
warnedUnregistered.add(executionId)
|
|
2013
|
+
yield* Effect.logWarning(
|
|
2014
|
+
`engine-store: run ${executionId} woke for flow ${state.flowName}, which is not registered in this process; leaving it parked for a worker that registers the flow`,
|
|
2015
|
+
{ runId: executionId, flowName: state.flowName }
|
|
2016
|
+
)
|
|
2017
|
+
}
|
|
2018
|
+
return
|
|
2019
|
+
}
|
|
2020
|
+
if (!(yield* claimAndActivate(initial))) {
|
|
2021
|
+
return
|
|
2022
|
+
}
|
|
2023
|
+
yield* Effect.logDebug("run activated", { flowName: state.flowName })
|
|
2024
|
+
|
|
2025
|
+
const activeState = withoutResult(state)
|
|
2026
|
+
// The activation transition carries the cancel guard: a run whose
|
|
2027
|
+
// cancellation was durably requested while it was parked must cancel
|
|
2028
|
+
// here instead of re-executing flow side effects (issue #27).
|
|
2029
|
+
const cleared = yield* transitionAndRecord(
|
|
2030
|
+
executionId,
|
|
2031
|
+
"running",
|
|
2032
|
+
yield* encodeState(activeState),
|
|
2033
|
+
{ decision: "resumed", status: "running" },
|
|
2034
|
+
{ cancelRequested: "absent" },
|
|
2035
|
+
undefined,
|
|
2036
|
+
null
|
|
2037
|
+
)
|
|
2038
|
+
if (cleared._tag === "GuardFailed") {
|
|
2039
|
+
return yield* cancelOwned(executionId, withoutResult(state))
|
|
2040
|
+
}
|
|
2041
|
+
if (cleared._tag !== "Transitioned") return
|
|
2042
|
+
|
|
2043
|
+
const payload = yield* (Schema.decodeUnknownEffect(
|
|
2044
|
+
Schema.toCodecJson(registration.flow.payloadSchema)
|
|
2045
|
+
)(activeState.payload).pipe(Effect.orDie) as Effect.Effect<unknown>)
|
|
2046
|
+
// A previous round of this run may have parked and retained its flow
|
|
2047
|
+
// scope. That instance is now superseded — this round runs under the
|
|
2048
|
+
// fresh instance below, with a scope of its own — so release the old
|
|
2049
|
+
// one instead of orphaning it. `Exit.void`: a re-drive is a successful
|
|
2050
|
+
// continuation, so ordinary finalizers release resources while
|
|
2051
|
+
// `Flow.withRollback` compensations correctly stay out.
|
|
2052
|
+
yield* releaseRetainedScope(executionId, Exit.void)
|
|
2053
|
+
const instance = FlowEngine.makeInstance(
|
|
2054
|
+
registration.flow,
|
|
2055
|
+
executionId,
|
|
2056
|
+
activeState.deadline
|
|
2057
|
+
)
|
|
2058
|
+
round = { flow: registration.flow, instance }
|
|
2059
|
+
const flowEngine = yield* dependencies.engine
|
|
2060
|
+
// ONE cleanup region, registered before the race starts and spanning
|
|
2061
|
+
// everything that can retain a scope or make its park durable: the
|
|
2062
|
+
// suspension surfacing, the retention insertion in the race's exit,
|
|
2063
|
+
// `encodeResult`, the pending-clock read, `engineState.park`, the
|
|
2064
|
+
// guarded transition and the journal write it commits with, and the
|
|
2065
|
+
// durable-park decision itself. Registering it here is what closes the
|
|
2066
|
+
// last window: retention is inserted by an uninterruptible finalizer,
|
|
2067
|
+
// and the fiber becomes interruptible again the instant that finalizer
|
|
2068
|
+
// returns, so any cleanup piped onto the settlement alone would never
|
|
2069
|
+
// be registered when an interrupt is already pending at that boundary —
|
|
2070
|
+
// the scope would stay retained until the driver's own scope closed,
|
|
2071
|
+
// which for a long-lived driver is not a release at all.
|
|
2072
|
+
//
|
|
2073
|
+
// `Exit.void` is the release exit, for the reason shutdown and a
|
|
2074
|
+
// supersede use it (issue #26): a park that never committed leaves the
|
|
2075
|
+
// run owned-but-stale or already re-owned, so it is reclaimed and
|
|
2076
|
+
// replayed, and the replay re-registers its `Flow.withRollback`
|
|
2077
|
+
// compensations under the reclaiming owner. Compensating here would
|
|
2078
|
+
// undo work the journal still reports as done — and would then do it
|
|
2079
|
+
// twice. Only a cancellation closes with an interrupt exit, and only
|
|
2080
|
+
// `cancelOwned` does that.
|
|
2081
|
+
yield* Effect.gen(function*() {
|
|
2082
|
+
// Published inside the region for the same reason: the entry used to
|
|
2083
|
+
// be written before `dependencies.engine` was awaited, one
|
|
2084
|
+
// interruptible step ahead of the `ensuring` that removes it, so an
|
|
2085
|
+
// interrupt landing there left this round's instance in the map for
|
|
2086
|
+
// the driver's lifetime — a slow leak, and a stale instance for
|
|
2087
|
+
// `interrupt` to mark interrupted.
|
|
2088
|
+
liveInstances.set(executionId, instance)
|
|
2089
|
+
const result = yield* Effect.scoped(
|
|
2090
|
+
Effect.raceFirst(
|
|
2091
|
+
Effect.raceFirst(
|
|
2092
|
+
Effect.suspend(() =>
|
|
2093
|
+
activeState.capabilityCeilings === undefined
|
|
2094
|
+
? Effect.die(new Error("Missing persisted capabilityCeilings; execution cannot safely resume"))
|
|
2095
|
+
: registration.execute(payload as object, executionId)
|
|
2096
|
+
).pipe(
|
|
2097
|
+
CapabilitySet.attenuateGroups(activeState.capabilityCeilings ?? [[]]),
|
|
2098
|
+
Flow.intoResult,
|
|
2099
|
+
Effect.provideService(FlowRuntime.FlowInstance, instance),
|
|
2100
|
+
Effect.provideService(FlowRuntime.FlowRuntime, flowEngine)
|
|
2101
|
+
),
|
|
2102
|
+
Ownership.heartbeatLoop(executionId, dependencies.owner, {
|
|
2103
|
+
onReconfirm: (unconfirmedMs) => {
|
|
2104
|
+
const record = decisionRecord(executionId, {
|
|
2105
|
+
decision: "lease-reconfirmed",
|
|
2106
|
+
detail: { unconfirmedMs }
|
|
2107
|
+
}, dependencies.journalSource)
|
|
2108
|
+
return journal.emitDurable(record, dependencies.owner).pipe(Effect.asVoid, Effect.orDie)
|
|
2109
|
+
},
|
|
2110
|
+
onLapse: (unconfirmedMs) =>
|
|
2111
|
+
Effect.sync(() => {
|
|
2112
|
+
lapsedMs = unconfirmedMs
|
|
2113
|
+
})
|
|
2114
|
+
}).pipe(
|
|
2115
|
+
Effect.provideService(RunStore.RunStore, store)
|
|
2116
|
+
)
|
|
2117
|
+
),
|
|
2118
|
+
cancelPollLoop(executionId)
|
|
2119
|
+
)
|
|
2120
|
+
).pipe(
|
|
2121
|
+
Effect.ensuring(Effect.sync(() => liveInstances.delete(executionId))),
|
|
2122
|
+
// A suspension is the ONE settlement `Flow.intoResult` leaves the
|
|
2123
|
+
// flow scope open for, so from the instant it surfaces the scope
|
|
2124
|
+
// needs an owner. Retention is taken HERE, inside the race's own
|
|
2125
|
+
// exit processing — `Effect.onExit` finalizers run uninterruptibly —
|
|
2126
|
+
// because taking it any later crosses an interruptible boundary: an
|
|
2127
|
+
// interrupt landing there would end the drive with the scope never
|
|
2128
|
+
// retained, unreachable, and its finalizers never run at all. The
|
|
2129
|
+
// entry is inserted into a cleanup region that was registered before
|
|
2130
|
+
// this race started, so it is owned from this instant on: every
|
|
2131
|
+
// ending, including an interrupt observed at the boundary between
|
|
2132
|
+
// this finalizer and the settlement below, releases it there.
|
|
2133
|
+
Effect.onExit((exit) =>
|
|
2134
|
+
Effect.suspend(() => {
|
|
2135
|
+
if (!(Exit.isSuccess(exit) && exit.value._tag === "Suspended")) return Effect.void
|
|
2136
|
+
retainedScopes.set(executionId, instance.scope)
|
|
2137
|
+
return dependencies.unsafeOnScopeRetained?.(executionId) ?? Effect.void
|
|
2138
|
+
})
|
|
2139
|
+
)
|
|
2140
|
+
)
|
|
2141
|
+
/**
|
|
2142
|
+
* Settles this round durably.
|
|
2143
|
+
*
|
|
2144
|
+
* A suspension that the guarded terminal transition durably accepts
|
|
2145
|
+
* sets `durableParkAccepted`, and that flag is the only thing that lets
|
|
2146
|
+
* a retained flow scope outlive this round. Every other ending — an
|
|
2147
|
+
* alternative settlement, a cancellation, a lost fence, a defect, an
|
|
2148
|
+
* interruption — leaves it unset, so the cleanup region registered
|
|
2149
|
+
* around the round releases the scope.
|
|
2150
|
+
*/
|
|
2151
|
+
const settleRound = Effect.gen(function*() {
|
|
2152
|
+
if (result._tag === "CancelRequested") {
|
|
2153
|
+
yield* cancelOwned(executionId, activeState)
|
|
2154
|
+
return
|
|
2155
|
+
}
|
|
2156
|
+
|
|
2157
|
+
// Corrupt evidence on a SUCCEEDED attempt row is an operator-visible
|
|
2158
|
+
// event, not a terminal run failure (issue #171): the row cannot be
|
|
2159
|
+
// evicted and re-executed like a corrupt cache row (#164) without
|
|
2160
|
+
// breaking exactly-once. ActionPersistence has already journalled
|
|
2161
|
+
// the corruption and quarantined only its boundary evidence off the
|
|
2162
|
+
// row. Park this first strict detection so it remains visible; the
|
|
2163
|
+
// next explicit resume returns the durable outcome without replaying
|
|
2164
|
+
// the poison or re-executing the action.
|
|
2165
|
+
const quarantine = result._tag === "Complete" && Exit.isFailure(result.exit)
|
|
2166
|
+
? ActionPersistence.evidenceQuarantined(result.exit.cause)
|
|
2167
|
+
: undefined
|
|
2168
|
+
if (quarantine !== undefined) {
|
|
2169
|
+
const parked = yield* transitionAndRecord(
|
|
2170
|
+
executionId,
|
|
2171
|
+
"suspended",
|
|
2172
|
+
yield* encodeState(withoutResult(activeState)),
|
|
2173
|
+
{
|
|
2174
|
+
decision: "quarantined",
|
|
2175
|
+
status: "suspended",
|
|
2176
|
+
keyDigest: quarantine.keyDigest,
|
|
2177
|
+
owner: dependencies.owner
|
|
2178
|
+
},
|
|
2179
|
+
{ cancelRequested: "absent" },
|
|
2180
|
+
undefined,
|
|
2181
|
+
{ reason: "quarantine", token: quarantine.keyDigest }
|
|
2182
|
+
)
|
|
2183
|
+
if (parked._tag === "GuardFailed") {
|
|
2184
|
+
yield* cancelOwned(executionId, activeState)
|
|
2185
|
+
}
|
|
2186
|
+
return
|
|
2187
|
+
}
|
|
2188
|
+
|
|
2189
|
+
// A round that handed off settles through the seam instead: its
|
|
2190
|
+
// terminal transition and its successor's creation are one write, so
|
|
2191
|
+
// it cannot share the ordinary terminal path below.
|
|
2192
|
+
if (result._tag === "Handoff") {
|
|
2193
|
+
yield* handOff({
|
|
2194
|
+
executionId,
|
|
2195
|
+
row: initial,
|
|
2196
|
+
state: activeState,
|
|
2197
|
+
flow: registration.flow,
|
|
2198
|
+
handoff: result
|
|
2199
|
+
})
|
|
2200
|
+
return
|
|
2201
|
+
}
|
|
2202
|
+
|
|
2203
|
+
const encodedResult = yield* encodeResult(registration.flow, result)
|
|
2204
|
+
const nextState: RunState = { ...activeState, result: encodedResult.encoded }
|
|
2205
|
+
// A settlement the flow's codec rejected is written `failed`
|
|
2206
|
+
// whatever it claimed to be: the row must reach a terminal status
|
|
2207
|
+
// in this process, because the alternative — the `running` row the
|
|
2208
|
+
// release validation found — is re-executed by the next one.
|
|
2209
|
+
const status: RunStore.RunStatus = encodedResult.note !== undefined
|
|
2210
|
+
? "failed"
|
|
2211
|
+
: result._tag === "Suspended"
|
|
2212
|
+
? "suspended"
|
|
2213
|
+
: Exit.isSuccess(result.exit)
|
|
2214
|
+
? "completed"
|
|
2215
|
+
: "failed"
|
|
2216
|
+
let waiting: DurableEngineState.Waiting | null = null
|
|
2217
|
+
if (status === "suspended") {
|
|
2218
|
+
// Park atomically with the suspended transition while this
|
|
2219
|
+
// process still owns the row (`park` is owner-fenced). The
|
|
2220
|
+
// reason is derived from durable state: a pending
|
|
2221
|
+
// clock row means a timer wake with a known deadline; anything
|
|
2222
|
+
// else waits on an external event (deferred completion). This is
|
|
2223
|
+
// what makes `waitingRuns` sweepers and the 0004 partial index
|
|
2224
|
+
// match real suspensions (issue #12).
|
|
2225
|
+
// A flow-declared classification (FlowRuntime.annotateWaiting) wins:
|
|
2226
|
+
// it is the only way an approval or quota wait — and its wake
|
|
2227
|
+
// token — reaches the parked row (issue #31). The durable-state
|
|
2228
|
+
// derivation stays the fallback.
|
|
2229
|
+
const declared = instance.waiting
|
|
2230
|
+
const pendingClocks = yield* engineState.pendingClocks({ executionId })
|
|
2231
|
+
waiting = declared !== undefined
|
|
2232
|
+
? declared
|
|
2233
|
+
: pendingClocks.length > 0
|
|
2234
|
+
? {
|
|
2235
|
+
reason: "timer",
|
|
2236
|
+
wakeAt: Math.min(...pendingClocks.map((clock) => clock.dueAtMs))
|
|
2237
|
+
}
|
|
2238
|
+
: { reason: "event" }
|
|
2239
|
+
}
|
|
2240
|
+
// Finalize is guarded on `cancel_requested_at_ms` inside the same
|
|
2241
|
+
// CAS: a cancellation request that raced past the last poll turns
|
|
2242
|
+
// the terminal transition into GuardFailed, and the run cancels
|
|
2243
|
+
// instead of finalizing (issue #11).
|
|
2244
|
+
const transitioned = yield* transitionAndRecord(
|
|
2245
|
+
executionId,
|
|
2246
|
+
status,
|
|
2247
|
+
yield* encodeState(nextState),
|
|
2248
|
+
{ decision: "transitioned", status, owner: dependencies.owner },
|
|
2249
|
+
{ cancelRequested: "absent" },
|
|
2250
|
+
// A suspension is not an exit: the run is parked and will settle
|
|
2251
|
+
// later, so its children keep going and its timers stay armed. A
|
|
2252
|
+
// `completed` or `failed` run is done, so its attached children
|
|
2253
|
+
// end with it. The transition closes its own clock rows.
|
|
2254
|
+
status === "suspended" ? undefined : applyChildExitPolicy(executionId),
|
|
2255
|
+
waiting
|
|
2256
|
+
).pipe(
|
|
2257
|
+
// The park becomes durable the instant this transition commits, so
|
|
2258
|
+
// the flag that records it is set in the transition's own exit
|
|
2259
|
+
// processing. `Effect.onExit` finalizers run uninterruptibly and
|
|
2260
|
+
// before the fiber can observe an interruption again, so no
|
|
2261
|
+
// interrupt can land between the durable commit and the flag and
|
|
2262
|
+
// make the cleanup region discard a scope the park still needs.
|
|
2263
|
+
Effect.onExit((exit) =>
|
|
2264
|
+
Effect.sync(() => {
|
|
2265
|
+
if (status === "suspended" && Exit.isSuccess(exit) && exit.value._tag === "Transitioned") {
|
|
2266
|
+
durableParkAccepted = true
|
|
2267
|
+
}
|
|
2268
|
+
})
|
|
2269
|
+
)
|
|
2270
|
+
)
|
|
2271
|
+
if (transitioned._tag === "GuardFailed") {
|
|
2272
|
+
// `cancelOwned` closes the retained scope itself, with the
|
|
2273
|
+
// interrupt exit cancellation compensations exist for. Leaving
|
|
2274
|
+
// `durableParkAccepted` unset therefore releases nothing a second
|
|
2275
|
+
// time: the entry is already gone from the map.
|
|
2276
|
+
yield* cancelOwned(executionId, activeState)
|
|
2277
|
+
return
|
|
2278
|
+
}
|
|
2279
|
+
// A lost fence (`NotFound`, a mismatched CAS) settles nothing: the
|
|
2280
|
+
// run is someone else's now, so a suspension holds no durable park
|
|
2281
|
+
// and the cleanup region releases the scope rather than holding it
|
|
2282
|
+
// until shutdown.
|
|
2283
|
+
if (transitioned._tag !== "Transitioned") return
|
|
2284
|
+
if (status !== "suspended") {
|
|
2285
|
+
// The settle is durable; tell any in-process caller parked on this
|
|
2286
|
+
// run's poll loop, so a run driven to completion by a sweep or a
|
|
2287
|
+
// coordinator wake is observed now rather than on the next tick.
|
|
2288
|
+
yield* announceSettled(executionId, activeState)
|
|
2289
|
+
return
|
|
2290
|
+
}
|
|
2291
|
+
})
|
|
2292
|
+
|
|
2293
|
+
yield* settleRound
|
|
2294
|
+
})
|
|
2295
|
+
}).pipe(
|
|
2296
|
+
// Install ownership cleanup before the claim itself. Limiting it to
|
|
2297
|
+
// the execution race strands an activated row if shutdown lands in
|
|
2298
|
+
// setup or settlement; a replacement host in the same process then
|
|
2299
|
+
// correctly refuses to steal from the still-live PID forever.
|
|
2300
|
+
Effect.onInterrupt(() =>
|
|
2301
|
+
interruptState === undefined
|
|
2302
|
+
? Effect.void
|
|
2303
|
+
: settleInterrupted(
|
|
2304
|
+
executionId,
|
|
2305
|
+
interruptState,
|
|
2306
|
+
lapsedMs === undefined
|
|
2307
|
+
? { kind: "interrupted" }
|
|
2308
|
+
: { kind: "lease-lapsed", unconfirmedMs: Math.max(0, Math.floor(lapsedMs)) },
|
|
2309
|
+
round
|
|
2310
|
+
)
|
|
2311
|
+
),
|
|
2312
|
+
Effect.onExit(() =>
|
|
2313
|
+
Effect.suspend(() => {
|
|
2314
|
+
liveInstances.delete(executionId)
|
|
2315
|
+
// Cancellation settles first so its compensations still see the
|
|
2316
|
+
// retained scope. A successful durable park alone retains it.
|
|
2317
|
+
return round === undefined || durableParkAccepted
|
|
2318
|
+
? Effect.void
|
|
2319
|
+
: releaseRetainedScope(executionId, Exit.void)
|
|
2320
|
+
})
|
|
2321
|
+
),
|
|
2322
|
+
Effect.annotateLogs({ runId: executionId })
|
|
2323
|
+
)
|
|
2324
|
+
})
|
|
2325
|
+
|
|
2326
|
+
const coordinator = yield* RunCoordinator.make<string, never, Crypto.Crypto>({
|
|
2327
|
+
drain: drive
|
|
2328
|
+
})
|
|
2329
|
+
yield* Deferred.succeed(coordinatorDeferred, coordinator)
|
|
2330
|
+
|
|
2331
|
+
/**
|
|
2332
|
+
* Delivers cancellation to parked runs (issue #27). A suspended run has
|
|
2333
|
+
* no owner and therefore no cancel poll, so `requestCancel` against it
|
|
2334
|
+
* is write-only until something re-drives the run — a run parked on a
|
|
2335
|
+
* deferred that never completes could otherwise never be cancelled. The
|
|
2336
|
+
* sweep lists parked rows, and wakes any whose cancel was durably
|
|
2337
|
+
* requested; the re-activation cancel guard then closes the run without
|
|
2338
|
+
* re-executing the flow.
|
|
2339
|
+
*
|
|
2340
|
+
* The same sweep reclaims interrupt-released runs (issue #39): a run
|
|
2341
|
+
* parked with reason `released` was interrupted mid-action by shutdown
|
|
2342
|
+
* or a heartbeat self-interrupt, has no pending clock and no completed
|
|
2343
|
+
* deferred, and would otherwise never be re-driven. Waking it re-enters
|
|
2344
|
+
* the ordinary claim/activate path, which also delivers any pending
|
|
2345
|
+
* cancel via the activation guard.
|
|
2346
|
+
*/
|
|
2347
|
+
const sweepCancelRequested: Effect.Effect<void> = Effect.gen(function*() {
|
|
2348
|
+
// Fetch only actionable rows (issue #68): the sweep acts solely on
|
|
2349
|
+
// released rows and rows whose cancellation was durably requested, so
|
|
2350
|
+
// a large quota/event-parked fleet must cost it nothing per tick. The
|
|
2351
|
+
// per-row `store.get` below is a status guard over the (small)
|
|
2352
|
+
// actionable set, not a probe over every parked run — the in-memory
|
|
2353
|
+
// implementation without a `runs` view stays permissive on the cancel
|
|
2354
|
+
// predicate and relies on exactly this guard.
|
|
2355
|
+
const released = yield* engineState.waitingRuns({ reason: "released" })
|
|
2356
|
+
const cancelRequestedRows = yield* engineState.waitingRuns({ cancelRequested: true })
|
|
2357
|
+
const candidates = new Map<string, DurableEngineState.WaitingRow>()
|
|
2358
|
+
for (const waiting of released) candidates.set(waiting.runId, waiting)
|
|
2359
|
+
for (const waiting of cancelRequestedRows) candidates.set(waiting.runId, waiting)
|
|
2360
|
+
for (const waiting of candidates.values()) {
|
|
2361
|
+
const row = yield* store.get(waiting.runId).pipe(
|
|
2362
|
+
Effect.catch(() => Effect.succeed(undefined))
|
|
2363
|
+
)
|
|
2364
|
+
if (
|
|
2365
|
+
row !== undefined &&
|
|
2366
|
+
row.status === "suspended" &&
|
|
2367
|
+
(row.cancelRequestedAtMs !== null || waiting.reason === "released")
|
|
2368
|
+
) {
|
|
2369
|
+
yield* coordinator.wake(row.runId)
|
|
2370
|
+
}
|
|
2371
|
+
}
|
|
2372
|
+
})
|
|
2373
|
+
/**
|
|
2374
|
+
* A timer can fire while a dead control owner is still inside its stale
|
|
2375
|
+
* cutoff. The clock completion is durable, but that first coordinator
|
|
2376
|
+
* wake cannot claim yet (#3408). Revisit only due timer waits with an
|
|
2377
|
+
* unconsumed completion; the normal drive still checks admission and CAS.
|
|
2378
|
+
* Released actions, future timers and ordinary event waits are excluded.
|
|
2379
|
+
*/
|
|
2380
|
+
const sweepReadyTimers: Effect.Effect<void> = Effect.gen(function*() {
|
|
2381
|
+
const nowMs = yield* Clock.currentTimeMillis
|
|
2382
|
+
const due = yield* engineState.waitingRuns({ reason: "timer", dueBeforeMs: nowMs })
|
|
2383
|
+
const completions = new Map<string, ReadonlySet<string>>()
|
|
2384
|
+
for (const waiting of due) {
|
|
2385
|
+
const row = yield* store.get(waiting.runId).pipe(Effect.catch(() => Effect.succeed(undefined)))
|
|
2386
|
+
if (row === undefined || row.status !== "suspended") continue
|
|
2387
|
+
const state = yield* decodeState(row.stateJson)
|
|
2388
|
+
if (!registrations.has(state.flowName)) continue
|
|
2389
|
+
let ready = completions.get(state.flowName)
|
|
2390
|
+
if (ready === undefined) {
|
|
2391
|
+
ready = new Set((yield* engineState.completedDeferreds(state.flowName)).map((address) => address.executionId))
|
|
2392
|
+
completions.set(state.flowName, ready)
|
|
2393
|
+
}
|
|
2394
|
+
if (!ready.has(row.runId)) continue
|
|
2395
|
+
yield* coordinator.schedule(row.runId)
|
|
2396
|
+
}
|
|
2397
|
+
})
|
|
2398
|
+
|
|
2399
|
+
// Admissions committed before a host crash have no lease to go stale.
|
|
2400
|
+
// Reuse the normal coordinator and claim/activate path for these rows.
|
|
2401
|
+
const pendingCycles = new Map<string, {
|
|
2402
|
+
readonly through: DurableEngineState.PendingRunCursor
|
|
2403
|
+
readonly eligibleBeforeMs: number
|
|
2404
|
+
after?: DurableEngineState.PendingRunCursor
|
|
2405
|
+
}>()
|
|
2406
|
+
let pendingCancellationCycle: {
|
|
2407
|
+
readonly through: DurableEngineState.PendingRunCursor
|
|
2408
|
+
after?: DurableEngineState.PendingRunCursor
|
|
2409
|
+
} | undefined
|
|
2410
|
+
// Explicit cancellation needs no registration or admission grace. A finite
|
|
2411
|
+
// tail and cursor keep a denied prefix from starving eligible later rows,
|
|
2412
|
+
// even when new cancellations arrive continuously.
|
|
2413
|
+
const sweepPendingCancellation = (): Effect.Effect<void> =>
|
|
2414
|
+
Effect.gen(function*() {
|
|
2415
|
+
if (pendingCancellationCycle === undefined) {
|
|
2416
|
+
const tail = yield* engineState.pendingCancellationTail()
|
|
2417
|
+
if (Option.isNone(tail)) return
|
|
2418
|
+
pendingCancellationCycle = { through: tail.value }
|
|
2419
|
+
}
|
|
2420
|
+
const cycle = pendingCancellationCycle
|
|
2421
|
+
const candidates = yield* engineState.pendingCancellationRuns(
|
|
2422
|
+
staleRunningSweepBatch,
|
|
2423
|
+
cycle.after,
|
|
2424
|
+
cycle.through
|
|
2425
|
+
)
|
|
2426
|
+
for (const cursor of candidates) {
|
|
2427
|
+
const row = yield* store.get(cursor.runId).pipe(
|
|
2428
|
+
Effect.catch((error) =>
|
|
2429
|
+
error.code === "not_found_row" ? Effect.succeed(undefined) : error.code === "decode_failed"
|
|
2430
|
+
? Effect.logWarning("engine-store: unreadable pending cancellation skipped", {
|
|
2431
|
+
runId: cursor.runId,
|
|
2432
|
+
error
|
|
2433
|
+
})
|
|
2434
|
+
.pipe(Effect.as(undefined))
|
|
2435
|
+
: Effect.die(error)
|
|
2436
|
+
)
|
|
2437
|
+
)
|
|
2438
|
+
// Permanent misses advance; a transient read failure retries this slot.
|
|
2439
|
+
cycle.after = cursor
|
|
2440
|
+
if (
|
|
2441
|
+
row === undefined || row.status !== "pending" || row.cancelRequestedAtMs === null ||
|
|
2442
|
+
(yield* coordinator.active).has(cursor.runId)
|
|
2443
|
+
) continue
|
|
2444
|
+
if (dependencies.canExecute !== undefined && !(yield* dependencies.canExecute(row))) continue
|
|
2445
|
+
if (!(yield* coordinator.active).has(cursor.runId)) yield* coordinator.schedule(cursor.runId)
|
|
2446
|
+
}
|
|
2447
|
+
if (
|
|
2448
|
+
candidates.length < staleRunningSweepBatch ||
|
|
2449
|
+
candidates.at(-1)?.runId === cycle.through.runId
|
|
2450
|
+
) pendingCancellationCycle = undefined
|
|
2451
|
+
})
|
|
2452
|
+
let pendingFlowOffset = 0
|
|
2453
|
+
const sweepPending = (): Effect.Effect<void> =>
|
|
2454
|
+
Effect.gen(function*() {
|
|
2455
|
+
let remaining = staleRunningSweepBatch
|
|
2456
|
+
const allNames = Array.from(registrations.keys())
|
|
2457
|
+
const offset = allNames.length === 0 ? 0 : pendingFlowOffset % allNames.length
|
|
2458
|
+
const names = [...allNames.slice(offset), ...allNames.slice(0, offset)]
|
|
2459
|
+
pendingFlowOffset++
|
|
2460
|
+
for (const name of names) {
|
|
2461
|
+
const registration = registrations.get(name)?.at(-1)
|
|
2462
|
+
if (registration === undefined || remaining === 0) continue
|
|
2463
|
+
let cycle = pendingCycles.get(name)
|
|
2464
|
+
if (cycle === undefined) {
|
|
2465
|
+
const eligibleBeforeMs = (yield* Clock.currentTimeMillis) - Duration.toMillis(Ownership.heartbeatStaleAfter)
|
|
2466
|
+
const tail = yield* engineState.pendingRunTail(name, eligibleBeforeMs)
|
|
2467
|
+
if (Option.isNone(tail)) continue
|
|
2468
|
+
cycle = { through: tail.value, eligibleBeforeMs }
|
|
2469
|
+
pendingCycles.set(name, cycle)
|
|
2470
|
+
}
|
|
2471
|
+
const requested = remaining
|
|
2472
|
+
const pending = yield* engineState.pendingRuns(
|
|
2473
|
+
name,
|
|
2474
|
+
requested,
|
|
2475
|
+
cycle.after,
|
|
2476
|
+
cycle.through,
|
|
2477
|
+
cycle.eligibleBeforeMs
|
|
2478
|
+
)
|
|
2479
|
+
for (const cursor of pending) {
|
|
2480
|
+
if (remaining === 0 || registrations.get(name)?.at(-1) !== registration) break
|
|
2481
|
+
remaining--
|
|
2482
|
+
const row = yield* store.get(cursor.runId).pipe(
|
|
2483
|
+
Effect.catch((error) =>
|
|
2484
|
+
error.code === "not_found_row" ? Effect.succeed(undefined) : error.code === "decode_failed"
|
|
2485
|
+
? Effect.logWarning("engine-store: unreadable pending admission skipped", {
|
|
2486
|
+
runId: cursor.runId,
|
|
2487
|
+
flowName: name,
|
|
2488
|
+
error
|
|
2489
|
+
}).pipe(Effect.as(undefined))
|
|
2490
|
+
: Effect.die(error)
|
|
2491
|
+
)
|
|
2492
|
+
)
|
|
2493
|
+
// Advance after successful reads or permanent row errors; transient failures retry this
|
|
2494
|
+
// position, while a retained query cursor skips deleted rows safely.
|
|
2495
|
+
cycle.after = cursor
|
|
2496
|
+
if (row === undefined || row.status !== "pending" || (yield* coordinator.active).has(cursor.runId)) {
|
|
2497
|
+
continue
|
|
2498
|
+
}
|
|
2499
|
+
if (dependencies.canExecute !== undefined && !(yield* dependencies.canExecute(row))) {
|
|
2500
|
+
continue
|
|
2501
|
+
}
|
|
2502
|
+
if (registrations.get(name)?.at(-1) !== registration || (yield* coordinator.active).has(cursor.runId)) {
|
|
2503
|
+
continue
|
|
2504
|
+
}
|
|
2505
|
+
yield* coordinator.schedule(cursor.runId)
|
|
2506
|
+
}
|
|
2507
|
+
const last = pending.at(-1)
|
|
2508
|
+
if (pending.length < requested || (last !== undefined && last.runId === cycle.through.runId)) {
|
|
2509
|
+
pendingCycles.delete(name)
|
|
2510
|
+
}
|
|
2511
|
+
}
|
|
2512
|
+
})
|
|
2513
|
+
|
|
2514
|
+
/**
|
|
2515
|
+
* Reclaims hard-killed runs (issue #53). An owner that dies without
|
|
2516
|
+
* releasing (SIGKILL, OOM, power loss) leaves a `running` row with a
|
|
2517
|
+
* frozen heartbeat and no waiting row, so the parked-run sweep above
|
|
2518
|
+
* never sees it, and the steal path — reachable only through `drive()` —
|
|
2519
|
+
* is never entered. Enumerate stale-running rows and re-drive them: the
|
|
2520
|
+
* ordinary claim/steal path (liveness check, exact-snapshot steal CAS)
|
|
2521
|
+
* then decides whether the owner is genuinely dead — the analog of
|
|
2522
|
+
* Temporal's task-timeout re-dispatch. A pending durable cancel is
|
|
2523
|
+
* delivered by the re-activation guard, same as for parked runs.
|
|
2524
|
+
*/
|
|
2525
|
+
const sweepStaleRunning: Effect.Effect<void> = Effect.gen(function*() {
|
|
2526
|
+
const nowMs = yield* Clock.currentTimeMillis.pipe(Effect.map(Math.floor))
|
|
2527
|
+
// Rows this driver has already arbitrated and been refused. They are the
|
|
2528
|
+
// oldest heartbeats in the window, so they sort first and would fill the
|
|
2529
|
+
// batch every tick; the read asks for enough extra rows to see past
|
|
2530
|
+
// exactly the ones it is going to skip.
|
|
2531
|
+
const deferred = Array.from(stealRefusals.values()).filter((entry) => entry.nextProbeAtMs > nowMs).length
|
|
2532
|
+
// Capped per tick (issue #79): oldest heartbeats come back first, so a
|
|
2533
|
+
// mass owner death drains across successive ticks — batch after batch
|
|
2534
|
+
// as each stolen run's heartbeat leaves the stale window — instead of
|
|
2535
|
+
// every surviving driver waking every stale run every second and
|
|
2536
|
+
// contending N-drivers × M-runs on the claim/steal CAS.
|
|
2537
|
+
const stale = yield* engineState.staleRunningRuns(
|
|
2538
|
+
nowMs - Duration.toMillis(Ownership.heartbeatStaleAfter),
|
|
2539
|
+
staleRunningSweepBatch + deferred
|
|
2540
|
+
)
|
|
2541
|
+
// A refusal is remembered only while its row is still in the window this
|
|
2542
|
+
// read covers. A run that was stolen, released, or settled leaves the
|
|
2543
|
+
// stale set, and its entry leaves with it, so the map cannot outgrow the
|
|
2544
|
+
// backlog it describes.
|
|
2545
|
+
const visible = new Set(stale.map((row) => row.runId))
|
|
2546
|
+
for (const runId of stealRefusals.keys()) {
|
|
2547
|
+
if (!visible.has(runId)) stealRefusals.delete(runId)
|
|
2548
|
+
}
|
|
2549
|
+
let woken = 0
|
|
2550
|
+
for (const { heartbeatAtMs, runId } of stale) {
|
|
2551
|
+
if (woken >= staleRunningSweepBatch) break
|
|
2552
|
+
// A refusal defers only the lease it was refused against. An owner
|
|
2553
|
+
// that pulsed and stalled again between two ticks never left this
|
|
2554
|
+
// read's window, so the lease is what says the stall is a new one.
|
|
2555
|
+
const refusal = stealRefusals.get(runId)
|
|
2556
|
+
if (refusal !== undefined && refusal.heartbeatAtMs === heartbeatAtMs && refusal.nextProbeAtMs > nowMs) continue
|
|
2557
|
+
woken = woken + 1
|
|
2558
|
+
yield* coordinator.wake(runId)
|
|
2559
|
+
}
|
|
2560
|
+
})
|
|
2561
|
+
const resilientSweep = (sweep: Effect.Effect<void>, name: string) =>
|
|
2562
|
+
sweep.pipe(
|
|
2563
|
+
Effect.sandbox,
|
|
2564
|
+
Effect.catchCause((cause) => Effect.logWarning(`engine-store: ${name} sweep failed; retrying next tick`, cause))
|
|
2565
|
+
)
|
|
2566
|
+
yield* Effect.forkScoped(
|
|
2567
|
+
Effect.forever(
|
|
2568
|
+
Effect.sleep(Ownership.heartbeatInterval).pipe(
|
|
2569
|
+
Effect.andThen(
|
|
2570
|
+
// One transient defect (a `SQLITE_BUSY` escaping `waitingRuns()`'s
|
|
2571
|
+
// `orDie`, a wake failure) must not kill the sweeper for the rest
|
|
2572
|
+
// of the process lifetime (issue #44) — that would silently revert
|
|
2573
|
+
// to pre-#27 behavior where cancel of parked runs is never
|
|
2574
|
+
// delivered. Mirror `armClock`'s hardening: expose the full cause,
|
|
2575
|
+
// log it, and keep ticking.
|
|
2576
|
+
resilientSweep(sweepCancelRequested, "parked-run cancel").pipe(
|
|
2577
|
+
Effect.andThen(resilientSweep(sweepStaleRunning, "stale-running")),
|
|
2578
|
+
Effect.andThen(resilientSweep(sweepReadyTimers, "ready timer")),
|
|
2579
|
+
Effect.andThen(resilientSweep(sweepPendingCancellation(), "pending cancellation")),
|
|
2580
|
+
Effect.andThen(resilientSweep(sweepPending(), "pending admission"))
|
|
2581
|
+
)
|
|
2582
|
+
)
|
|
2583
|
+
)
|
|
2584
|
+
)
|
|
2585
|
+
)
|
|
2586
|
+
|
|
2587
|
+
const ensureRun = (
|
|
2588
|
+
flow: Flow.Any,
|
|
2589
|
+
options: {
|
|
2590
|
+
readonly executionId: string
|
|
2591
|
+
readonly payload: object
|
|
2592
|
+
/**
|
|
2593
|
+
* Whether the caller discarded this execution's result. It is what
|
|
2594
|
+
* separates the two child shapes: a caller that waits for the child's
|
|
2595
|
+
* value holds it attached, and a caller that threw the value away
|
|
2596
|
+
* spawned it detached. See {@link RunState.OnParentExit}.
|
|
2597
|
+
*/
|
|
2598
|
+
readonly discard?: boolean | undefined
|
|
2599
|
+
readonly parent?: FlowRuntime.FlowInstance["Service"] | undefined
|
|
2600
|
+
readonly round?:
|
|
2601
|
+
| (FlowEngine.Round.Round & {
|
|
2602
|
+
readonly previousExecutionId?: string | undefined
|
|
2603
|
+
})
|
|
2604
|
+
| undefined
|
|
2605
|
+
}
|
|
2606
|
+
): Effect.Effect<void, FlowCycleDetected> =>
|
|
2607
|
+
Effect.gen(function*() {
|
|
2608
|
+
const payload = yield* (Schema.encodeEffect(
|
|
2609
|
+
Schema.toCodecJson(flow.payloadSchema)
|
|
2610
|
+
)(
|
|
2611
|
+
options.payload
|
|
2612
|
+
).pipe(Effect.orDie) as Effect.Effect<unknown>)
|
|
2613
|
+
// Every requested parent — first (creating) parent and a diamond's
|
|
2614
|
+
// second parent alike — is recorded as a durable edge BEFORE the run
|
|
2615
|
+
// row is created. `recordRunParent` inserts the edge and checks for
|
|
2616
|
+
// a cycle inside one storage transaction (issues #29/#40/#54/#56),
|
|
2617
|
+
// so a rejected request fails here and never creates a run row — no
|
|
2618
|
+
// `state_json` parent can outlive a rejected edge (issue #55). The
|
|
2619
|
+
// edge table is the single source of truth for cycle detection in
|
|
2620
|
+
// every owner process and across restarts (issues #40/#41).
|
|
2621
|
+
//
|
|
2622
|
+
// The edge record and the run-row creation share one outer storage
|
|
2623
|
+
// transaction (issue #80): a crash between them used to commit a
|
|
2624
|
+
// durable orphan edge for a child run that never existed — never
|
|
2625
|
+
// GC'd, permanently walked by cycle detection, and able to force a
|
|
2626
|
+
// false FlowCycleDetected if the execution id was later reused
|
|
2627
|
+
// under an inverted topology. The stores' own writes become
|
|
2628
|
+
// savepoints of this transaction, so either both commit or neither.
|
|
2629
|
+
yield* transactState(Effect.gen(function*() {
|
|
2630
|
+
let parentRecorded = false
|
|
2631
|
+
if (options.parent !== undefined) {
|
|
2632
|
+
const parentEdge = yield* engineState.recordRunParent(
|
|
2633
|
+
options.executionId,
|
|
2634
|
+
options.parent.executionId
|
|
2635
|
+
).pipe(
|
|
2636
|
+
Effect.catch((error) =>
|
|
2637
|
+
Effect.fail(
|
|
2638
|
+
new FlowCycleDetected({ code: "flow_cycle_detected", path: error.path })
|
|
2639
|
+
)
|
|
2640
|
+
)
|
|
2641
|
+
)
|
|
2642
|
+
parentRecorded = parentEdge._tag === "Recorded"
|
|
2643
|
+
}
|
|
2644
|
+
const round = options.round ?? FlowEngine.Round.initial(options.executionId)
|
|
2645
|
+
const previousExecutionId = options.round?.previousExecutionId
|
|
2646
|
+
// Decided once, at spawn, from what the caller did with the result:
|
|
2647
|
+
// a caller waiting for the value keeps the child attached, and a
|
|
2648
|
+
// caller that discarded it detached the child on purpose. Nothing
|
|
2649
|
+
// later rewrites this.
|
|
2650
|
+
const onParentExit: OnParentExit = options.discard === true ? "detach" : "cancel"
|
|
2651
|
+
const state: RunState = {
|
|
2652
|
+
version: 1,
|
|
2653
|
+
flowName: flow._tag,
|
|
2654
|
+
payload,
|
|
2655
|
+
capabilityCeilings: yield* FlowEngine.requestedAuthority(flow),
|
|
2656
|
+
onParentExit,
|
|
2657
|
+
...(options.parent === undefined
|
|
2658
|
+
? {}
|
|
2659
|
+
: { parentExecutionId: options.parent.executionId }),
|
|
2660
|
+
...(flow.maxRounds === undefined ? {} : { maxRounds: flow.maxRounds })
|
|
2661
|
+
}
|
|
2662
|
+
const createdStateJson = yield* encodeState(state)
|
|
2663
|
+
yield* ensureCreatedRun({
|
|
2664
|
+
flowName: flow._tag,
|
|
2665
|
+
capabilityCeilings: state.capabilityCeilings!,
|
|
2666
|
+
executionId: options.executionId,
|
|
2667
|
+
stateJson: createdStateJson,
|
|
2668
|
+
payload,
|
|
2669
|
+
lineageId: round.rootExecutionId,
|
|
2670
|
+
roundOrdinal: round.ordinal,
|
|
2671
|
+
...(previousExecutionId === undefined
|
|
2672
|
+
? {}
|
|
2673
|
+
: { parentRunId: previousExecutionId }),
|
|
2674
|
+
onCreated: Effect.gen(function*() {
|
|
2675
|
+
// The run's opening frame. `store.create` used to be the one
|
|
2676
|
+
// durable write with no journal record at all, which left the
|
|
2677
|
+
// replay-derived state projection with no base to fold onto —
|
|
2678
|
+
// `flowName` and `payload` exist nowhere else in the journal.
|
|
2679
|
+
// The state travels ENCODED, exactly as every later decision
|
|
2680
|
+
// carries it: `stateAt` folds these payloads and hands the winner
|
|
2681
|
+
// back as the run row's own `state_json`, so a base recorded in the
|
|
2682
|
+
// decoded shape would be a schema the caller cannot decode.
|
|
2683
|
+
yield* emitDecision(options.executionId, {
|
|
2684
|
+
decision: "created",
|
|
2685
|
+
state: JSON.parse(createdStateJson),
|
|
2686
|
+
...(options.parent === undefined ? {} : { parentExecutionId: options.parent.executionId })
|
|
2687
|
+
})
|
|
2688
|
+
if (options.parent !== undefined) {
|
|
2689
|
+
// A spawn is a lineage edge, and a DETACHED spawn is a tier-3
|
|
2690
|
+
// effect: nothing the parent's rewind can undo, because the child
|
|
2691
|
+
// is its own claim and its own journal. The record
|
|
2692
|
+
// is boundary-shaped so the same assessment that classifies a
|
|
2693
|
+
// sent webhook classifies an orphaned child, and it is emitted at
|
|
2694
|
+
// `succeeded` because by this point the child run durably exists.
|
|
2695
|
+
// Unfenced: the creator of a detached child is not necessarily
|
|
2696
|
+
// the parent's current owner (an external spawn admission
|
|
2697
|
+
// creates child rows without holding the parent's fence), so
|
|
2698
|
+
// this lineage-edge record is first-writer-wins evidence keyed
|
|
2699
|
+
// by its spawn identity.
|
|
2700
|
+
yield* journal.emitDurableUnfenced(
|
|
2701
|
+
EffectRecords.boundary(
|
|
2702
|
+
{
|
|
2703
|
+
id: `${options.parent.executionId}:spawn:${options.executionId}`,
|
|
2704
|
+
kind: spawnEffectKind,
|
|
2705
|
+
tier: "irreversible",
|
|
2706
|
+
runId: options.parent.executionId,
|
|
2707
|
+
lineageId: FlowEngine.Lineage.root(options.parent.executionId),
|
|
2708
|
+
sourceId: dependencies.journalSource,
|
|
2709
|
+
attempt: 1,
|
|
2710
|
+
residue:
|
|
2711
|
+
`Child run ${options.executionId} exists and keeps its own journal; rewinding past its spawn orphans it.`
|
|
2712
|
+
},
|
|
2713
|
+
"succeeded",
|
|
2714
|
+
// `attached` is written even though it is always false: the
|
|
2715
|
+
// lineage-tree bridge in `@smthrs/time-travel` reads it off
|
|
2716
|
+
// this payload, and an absent field there would make "this
|
|
2717
|
+
// spawn is detached" indistinguishable from "this producer
|
|
2718
|
+
// predates the field". A run created with a parent is a
|
|
2719
|
+
// separate run row with its own claim, which is what detached
|
|
2720
|
+
// means; attached nesting
|
|
2721
|
+
// never reaches `create` because it is one journal.
|
|
2722
|
+
{ childRunId: options.executionId, flowName: flow._tag, attached: false }
|
|
2723
|
+
)
|
|
2724
|
+
).pipe(Effect.orDie)
|
|
2725
|
+
}
|
|
2726
|
+
})
|
|
2727
|
+
})
|
|
2728
|
+
// The row already exists; the durable edge recorded above is the
|
|
2729
|
+
// only place a diamond's second parent lives (issues #41/#48): a
|
|
2730
|
+
// driver-local side table would be invisible to other owners over
|
|
2731
|
+
// the same store, lost across restart, and would grow without
|
|
2732
|
+
// bound for the driver's lifetime.
|
|
2733
|
+
//
|
|
2734
|
+
// Which is also why the parent's cancellation is inherited HERE,
|
|
2735
|
+
// last and inside this same transaction: the child row must already
|
|
2736
|
+
// exist for `requestCancel` to have anything to write to, and the
|
|
2737
|
+
// inheritance must commit or roll back with the edge that made this
|
|
2738
|
+
// run a child in the first place. The inherited request is what the
|
|
2739
|
+
// activation guard reads, so the child cancels instead of executing
|
|
2740
|
+
// the flow body.
|
|
2741
|
+
if (options.parent !== undefined) {
|
|
2742
|
+
yield* inheritParentCancellation(
|
|
2743
|
+
options.parent.executionId,
|
|
2744
|
+
options.executionId
|
|
2745
|
+
)
|
|
2746
|
+
if (parentRecorded) {
|
|
2747
|
+
yield* emitDecision(options.executionId, {
|
|
2748
|
+
decision: "parent-recorded",
|
|
2749
|
+
parentExecutionId: options.parent.executionId
|
|
2750
|
+
})
|
|
2751
|
+
}
|
|
2752
|
+
}
|
|
2753
|
+
})).pipe(
|
|
2754
|
+
Effect.catchIf((error) => error instanceof Journal.JournalError || error instanceof DatabaseError, Effect.die)
|
|
2755
|
+
)
|
|
2756
|
+
})
|
|
2757
|
+
|
|
2758
|
+
const readResult = (flow: Flow.Any, executionId: string, followLineage = false) =>
|
|
2759
|
+
Effect.annotateCurrentSpan({ executionId, flow: flow._tag }).pipe(
|
|
2760
|
+
Effect.andThen(FlowEngine.requestedAuthority(flow)),
|
|
2761
|
+
Effect.flatMap((requested) =>
|
|
2762
|
+
Effect.map(followLineage ? store.latestRound(executionId) : store.get(executionId), (row) => ({
|
|
2763
|
+
row,
|
|
2764
|
+
requested
|
|
2765
|
+
}))
|
|
2766
|
+
),
|
|
2767
|
+
Effect.catch((error) =>
|
|
2768
|
+
error.code === "not_found_row"
|
|
2769
|
+
? Effect.succeed(undefined)
|
|
2770
|
+
: Effect.die(error)
|
|
2771
|
+
),
|
|
2772
|
+
Effect.flatMap((found) => {
|
|
2773
|
+
if (found === undefined) {
|
|
2774
|
+
// No run row at all is a typed not-found; `Option.none` is
|
|
2775
|
+
// reserved for a known run that has not settled yet.
|
|
2776
|
+
return Effect.fail(
|
|
2777
|
+
new FlowRuntime.FlowExecutionNotFound({
|
|
2778
|
+
code: "execution_not_found",
|
|
2779
|
+
executionId
|
|
2780
|
+
})
|
|
2781
|
+
)
|
|
2782
|
+
}
|
|
2783
|
+
const { row, requested } = found
|
|
2784
|
+
return decodeState(row.stateJson).pipe(
|
|
2785
|
+
Effect.flatMap((state) => {
|
|
2786
|
+
if (
|
|
2787
|
+
(!followLineage && state.flowName !== flow._tag) ||
|
|
2788
|
+
state.result === undefined
|
|
2789
|
+
) {
|
|
2790
|
+
return Effect.succeedNone
|
|
2791
|
+
}
|
|
2792
|
+
// A result is what the admitted authority produced; a caller
|
|
2793
|
+
// whose authority does not cover it never reads it.
|
|
2794
|
+
if (!FlowEngine.joinable(state.capabilityCeilings ?? [], requested)) {
|
|
2795
|
+
return Effect.die(FlowEngine.capabilityConflict(executionId))
|
|
2796
|
+
}
|
|
2797
|
+
const resultFlow = state.flowName === flow._tag ? flow : registrations.get(state.flowName)?.at(-1)?.flow
|
|
2798
|
+
if (resultFlow === undefined) {
|
|
2799
|
+
// Direct nested callers must register the terminal flow's
|
|
2800
|
+
// codec here, even when another worker executed that flow.
|
|
2801
|
+
// This is the typed Trampoline's fail-fast wiring contract:
|
|
2802
|
+
// terminal work has no future wake to justify Suspended.
|
|
2803
|
+
return Effect.die(new FlowEngine.FlowNotRegistered({
|
|
2804
|
+
flowName: state.flowName,
|
|
2805
|
+
message: `Flow ${state.flowName} is not registered`
|
|
2806
|
+
}))
|
|
2807
|
+
}
|
|
2808
|
+
return (Schema.decodeUnknownEffect(
|
|
2809
|
+
Schema.toCodecJson(Flow.Result({
|
|
2810
|
+
success: resultFlow.successSchema,
|
|
2811
|
+
error: resultFlow.errorSchema
|
|
2812
|
+
}))
|
|
2813
|
+
)(state.result).pipe(
|
|
2814
|
+
Effect.orDie,
|
|
2815
|
+
Effect.map(Option.some)
|
|
2816
|
+
) as Effect.Effect<Option.Option<Flow.Result<unknown, unknown>>>)
|
|
2817
|
+
}),
|
|
2818
|
+
Effect.map((result) => ({ result, status: row.status }))
|
|
2819
|
+
)
|
|
2820
|
+
})
|
|
2821
|
+
)
|
|
2822
|
+
|
|
2823
|
+
/**
|
|
2824
|
+
* Whether a resume of `executionId` for `flow` may drive anything: only an
|
|
2825
|
+
* admitted row of this declaration whose recorded authority the caller's
|
|
2826
|
+
* covers. An unknown id or a row another declaration owns drives nothing,
|
|
2827
|
+
* so a row admitted after this read is never woken on this request; a
|
|
2828
|
+
* caller whose authority does not cover the row is refused.
|
|
2829
|
+
*/
|
|
2830
|
+
const authorizeResume = (flow: Flow.Any, executionId: string): Effect.Effect<boolean> =>
|
|
2831
|
+
Effect.gen(function*() {
|
|
2832
|
+
const row = yield* store.get(executionId).pipe(
|
|
2833
|
+
Effect.catch((error) => error.code === "not_found_row" ? Effect.succeed(undefined) : Effect.die(error))
|
|
2834
|
+
)
|
|
2835
|
+
if (row === undefined) return false
|
|
2836
|
+
const state = yield* decodeState(row.stateJson)
|
|
2837
|
+
if (state.flowName !== flow._tag) return false
|
|
2838
|
+
if (!FlowEngine.joinable(state.capabilityCeilings ?? [], yield* FlowEngine.requestedAuthority(flow))) {
|
|
2839
|
+
return yield* Effect.die(FlowEngine.capabilityConflict(executionId))
|
|
2840
|
+
}
|
|
2841
|
+
return true
|
|
2842
|
+
})
|
|
2843
|
+
|
|
2844
|
+
const poll: Service["poll"] = Effect.fn("FlowEngine.poll")((flow, executionId) =>
|
|
2845
|
+
Effect.map(readResult(flow, executionId), (observed) => observed.result)
|
|
2846
|
+
)
|
|
2847
|
+
|
|
2848
|
+
const execute: Service["execute"] = Effect.fn("FlowEngine.execute")(
|
|
2849
|
+
function*<const Discard extends boolean>(
|
|
2850
|
+
flow: Flow.Any,
|
|
2851
|
+
options: {
|
|
2852
|
+
readonly executionId: string
|
|
2853
|
+
readonly payload: object
|
|
2854
|
+
readonly discard: Discard
|
|
2855
|
+
readonly parent?: FlowRuntime.FlowInstance["Service"] | undefined
|
|
2856
|
+
readonly round?:
|
|
2857
|
+
| (FlowEngine.Round.Round & {
|
|
2858
|
+
readonly previousExecutionId?: string | undefined
|
|
2859
|
+
})
|
|
2860
|
+
| undefined
|
|
2861
|
+
readonly follow?: boolean | undefined
|
|
2862
|
+
}
|
|
2863
|
+
) {
|
|
2864
|
+
yield* Effect.annotateCurrentSpan({ executionId: options.executionId, flow: flow._tag })
|
|
2865
|
+
if (!registrations.has(flow._tag)) {
|
|
2866
|
+
return yield* Effect.die(
|
|
2867
|
+
new FlowEngine.FlowNotRegistered({
|
|
2868
|
+
flowName: flow._tag,
|
|
2869
|
+
message: `Flow ${flow._tag} is not registered`
|
|
2870
|
+
})
|
|
2871
|
+
)
|
|
2872
|
+
}
|
|
2873
|
+
// Cycle rejection happens atomically inside `ensureRun`'s call to
|
|
2874
|
+
// `DurableEngineState.recordRunParent`: the storage transaction that
|
|
2875
|
+
// inserts the edge also walks the parent chain and rolls back on a
|
|
2876
|
+
// hit, so no in-process gate, cross-owner arbitration, or withdrawal
|
|
2877
|
+
// protocol is needed here (issues #29/#40/#54/#55/#56) and the
|
|
2878
|
+
// mutual `coordinator.run` deadlock cannot form.
|
|
2879
|
+
if (options.discard) {
|
|
2880
|
+
// Keep durable admission and scheduling together across caller cancellation.
|
|
2881
|
+
// The coordinator owns the drive in the engine scope, so a
|
|
2882
|
+
// detached child outlives the caller's action and execution.
|
|
2883
|
+
yield* Effect.uninterruptible(
|
|
2884
|
+
ensureRun(flow, options).pipe(Effect.andThen(coordinator.schedule(options.executionId)))
|
|
2885
|
+
)
|
|
2886
|
+
return undefined as Discard extends true ? void : never
|
|
2887
|
+
}
|
|
2888
|
+
yield* ensureRun(flow, options)
|
|
2889
|
+
// Admission, handoff, or a wake normally scheduled this drive. A
|
|
2890
|
+
// replacement host may instead find an admitted pending row with no
|
|
2891
|
+
// drive, so schedule that row before joining. An idle suspended row
|
|
2892
|
+
// stays passive until its actual wake or elapsed poll resumes it.
|
|
2893
|
+
if (options.follow === true) {
|
|
2894
|
+
yield* coordinator.scheduleIf(
|
|
2895
|
+
options.executionId,
|
|
2896
|
+
store.get(options.executionId).pipe(
|
|
2897
|
+
Effect.orDie,
|
|
2898
|
+
Effect.map((row) => row.status === "pending")
|
|
2899
|
+
)
|
|
2900
|
+
)
|
|
2901
|
+
yield* coordinator.join(options.executionId)
|
|
2902
|
+
} else {
|
|
2903
|
+
yield* coordinator.run(options.executionId)
|
|
2904
|
+
}
|
|
2905
|
+
// `ensureRun` created the row above, so a not-found here is a broken
|
|
2906
|
+
// store invariant, not a caller-recoverable state.
|
|
2907
|
+
// The trampoline supplies `round` and consumes each raw handoff itself.
|
|
2908
|
+
// A nested direct caller instead needs the child's current lineage row,
|
|
2909
|
+
// including its own codec and cancellation status. `follow` schedules an
|
|
2910
|
+
// undriven pending row before joining; it does not select the result's scope.
|
|
2911
|
+
const observed = yield* Effect.orDie(readResult(
|
|
2912
|
+
flow,
|
|
2913
|
+
options.executionId,
|
|
2914
|
+
options.parent !== undefined && options.round === undefined
|
|
2915
|
+
))
|
|
2916
|
+
if (Option.isSome(observed.result)) {
|
|
2917
|
+
return observed.result.value as Discard extends true ? void : Flow.Result<unknown, unknown>
|
|
2918
|
+
}
|
|
2919
|
+
// A cancelled run has no `Flow.Result` on its row — cancellation is
|
|
2920
|
+
// recorded as `cancellation`, not as a settlement — so answering the
|
|
2921
|
+
// absence with `Suspended` told the caller to wait for a run that was
|
|
2922
|
+
// over. The engine's suspended-retry loop then did what that answer
|
|
2923
|
+
// means: slept, resumed, re-drove, and asked again, for as long as the
|
|
2924
|
+
// caller lived. A run cancelled underneath its caller interrupts the
|
|
2925
|
+
// caller instead, which is what the memory engine already does
|
|
2926
|
+
// (`layerMemory.ts` `interrupt` settles the round `Complete` with an
|
|
2927
|
+
// interrupt cause) and what a cancellation means: the work the caller
|
|
2928
|
+
// asked for is not going to happen.
|
|
2929
|
+
// Result and cancellation status come from the same row snapshot.
|
|
2930
|
+
return (observed.status === "cancelled"
|
|
2931
|
+
? new Flow.Complete({ exit: Exit.failCause(Cause.interrupt()) })
|
|
2932
|
+
: new Flow.Suspended({})) as Discard extends true ? void
|
|
2933
|
+
: Flow.Result<unknown, unknown>
|
|
2934
|
+
}
|
|
2935
|
+
)
|
|
2936
|
+
|
|
2937
|
+
const interrupt = Effect.fn("FlowEngine.interrupt")((
|
|
2938
|
+
_flow: Flow.Any,
|
|
2939
|
+
executionId: string
|
|
2940
|
+
): Effect.Effect<void, FlowRuntime.CancelRequestFailed> =>
|
|
2941
|
+
Effect.gen(function*() {
|
|
2942
|
+
yield* Effect.annotateCurrentSpan({ executionId })
|
|
2943
|
+
// Operator intent is recorded durably before the fiber interrupt so
|
|
2944
|
+
// the interruption handler can tell cancellation apart from shutdown
|
|
2945
|
+
// (issue #26), and so the request survives if this process dies
|
|
2946
|
+
// before the interrupt lands.
|
|
2947
|
+
//
|
|
2948
|
+
// The write used to be `Effect.ignore`d, which answered success for a
|
|
2949
|
+
// cancellation that was never recorded: the fiber interrupt that
|
|
2950
|
+
// follows is then read as a shutdown by `settleInterrupted`, the run
|
|
2951
|
+
// is RELEASED for reclaim rather than cancelled, another worker picks
|
|
2952
|
+
// it up, and the caller was told the run was cancelled. The failure is
|
|
2953
|
+
// typed instead, so the caller can retry against a state that is still
|
|
2954
|
+
// truthful — the run is still running and still cancellable.
|
|
2955
|
+
const nowMs = yield* Clock.currentTimeMillis.pipe(Effect.map(Math.floor))
|
|
2956
|
+
// The package's one lock order, the same one every settlement takes:
|
|
2957
|
+
// engine state first, then the journal. The descendant walk reads
|
|
2958
|
+
// engine state while holding the SQL writer, so the alternative would
|
|
2959
|
+
// deadlock against any fiber that ordered the two boundaries the
|
|
2960
|
+
// other way.
|
|
2961
|
+
const requested = yield* transactState(
|
|
2962
|
+
Effect.gen(function*() {
|
|
2963
|
+
yield* requestCancellation(executionId, nowMs)
|
|
2964
|
+
return [executionId, ...yield* requestCancelDescendants(executionId, nowMs)]
|
|
2965
|
+
}),
|
|
2966
|
+
// Capture the shared writer directly. SqlJournal.transact catches a
|
|
2967
|
+
// selected failure and can discard the rest of a composite cause.
|
|
2968
|
+
(write) => Effect.flatMap(Effect.exit(writer.write(write)), (exit) => exit)
|
|
2969
|
+
).pipe(
|
|
2970
|
+
Effect.catchCause((cause) => {
|
|
2971
|
+
// A composite is never a retryable CancelRequestFailed: retain
|
|
2972
|
+
// every original reason, including defect identity and interruption.
|
|
2973
|
+
const failure = (error: { readonly message: string }) =>
|
|
2974
|
+
new FlowRuntime.CancelRequestFailed({
|
|
2975
|
+
code: "cancel_request_failed",
|
|
2976
|
+
executionId,
|
|
2977
|
+
reason: error.message
|
|
2978
|
+
})
|
|
2979
|
+
if (cause.reasons.length === 1) {
|
|
2980
|
+
const reason = cause.reasons[0]!
|
|
2981
|
+
if (Cause.isDieReason(reason) && reason.defect instanceof DatabaseError) {
|
|
2982
|
+
return Effect.fail(failure(reason.defect))
|
|
2983
|
+
}
|
|
2984
|
+
// The typed channel is the normalized writer/store/journal union.
|
|
2985
|
+
if (Cause.isFailReason(reason)) {
|
|
2986
|
+
return Effect.fail(failure(reason.error))
|
|
2987
|
+
}
|
|
2988
|
+
}
|
|
2989
|
+
// A composite is not the retryable request failure the typed
|
|
2990
|
+
// channel promises, so every typed reason is re-channelled as a
|
|
2991
|
+
// defect rather than cast through `CancelRequestFailed`. The error
|
|
2992
|
+
// objects, the accompanying defects and the interruptions all
|
|
2993
|
+
// survive; only the channel changes, and it changes to the one the
|
|
2994
|
+
// declared type actually admits.
|
|
2995
|
+
return Effect.failCause(asDefects(cause))
|
|
2996
|
+
})
|
|
2997
|
+
)
|
|
2998
|
+
// Only after the durable write succeeds may the ephemeral surface say
|
|
2999
|
+
// this instance was interrupted or stop its drive fiber. If the write
|
|
3000
|
+
// failed, changing either would make a typed failure observably mutate
|
|
3001
|
+
// the run (and could spuriously trigger the in-process child path).
|
|
3002
|
+
// The transaction above records the parent and every durable
|
|
3003
|
+
// descendant together. A run this process does not drive — parked
|
|
3004
|
+
// here, owned elsewhere, or already settled — therefore receives the
|
|
3005
|
+
// same child-flow handling without a partial-cascade state if any
|
|
3006
|
+
// request write fails. `requestCancel` is first-writer-wins, so the
|
|
3007
|
+
// owning driver's own cascade later writes nothing.
|
|
3008
|
+
yield* journal.whenCommitted(Effect.gen(function*() {
|
|
3009
|
+
for (const targetId of requested) {
|
|
3010
|
+
const instance = liveInstances.get(targetId)
|
|
3011
|
+
if (instance !== undefined) instance.interrupted = true
|
|
3012
|
+
// Publication only sends the request. Awaiting user finalizers
|
|
3013
|
+
// here would make an outer commit uninterruptibly wait on them.
|
|
3014
|
+
yield* coordinator.requestInterrupt(targetId)
|
|
3015
|
+
}
|
|
3016
|
+
}))
|
|
3017
|
+
})
|
|
3018
|
+
)
|
|
3019
|
+
|
|
3020
|
+
const scheduleResume: Service["scheduleResume"] = Effect.fn("FlowEngine.scheduleResume")((
|
|
3021
|
+
flowName,
|
|
3022
|
+
executionId,
|
|
3023
|
+
reason,
|
|
3024
|
+
sourceId
|
|
3025
|
+
) =>
|
|
3026
|
+
transactState(Effect.gen(function*() {
|
|
3027
|
+
yield* Effect.annotateCurrentSpan({ executionId, flow: flowName, reason })
|
|
3028
|
+
const row = yield* store.get(executionId).pipe(
|
|
3029
|
+
Effect.catch((error) =>
|
|
3030
|
+
error.code === "not_found_row"
|
|
3031
|
+
? Effect.succeed(undefined)
|
|
3032
|
+
: Effect.die(error)
|
|
3033
|
+
)
|
|
3034
|
+
)
|
|
3035
|
+
if (row === undefined) return
|
|
3036
|
+
// A settled run is not woken and then refused: it is not woken. The
|
|
3037
|
+
// terminal reject used to live one layer down, in `claimAndActivate`,
|
|
3038
|
+
// so a timer armed before its run was cancelled and an external
|
|
3039
|
+
// trigger arriving after the run finished both wrote a
|
|
3040
|
+
// `wake-scheduled` decision into the journal of a run that can never
|
|
3041
|
+
// wake again — one per event, forever, on a durable channel.
|
|
3042
|
+
if (row.status === "completed" || row.status === "failed" || row.status === "cancelled") return
|
|
3043
|
+
const state = yield* decodeState(row.stateJson)
|
|
3044
|
+
if (state.flowName !== flowName) return
|
|
3045
|
+
const waiting = yield* engineState.waiting(executionId)
|
|
3046
|
+
if (Option.isSome(waiting) && waiting.value.reason === "quarantine") {
|
|
3047
|
+
if (reason !== "operator") return
|
|
3048
|
+
// Recovery consent and its journal receipt commit together. A
|
|
3049
|
+
// queued drive cannot consume quarantined evidence before consent.
|
|
3050
|
+
yield* engineState.wake(executionId)
|
|
3051
|
+
}
|
|
3052
|
+
yield* emitDecision(executionId, {
|
|
3053
|
+
decision: "wake-scheduled",
|
|
3054
|
+
reason
|
|
3055
|
+
}, sourceId)
|
|
3056
|
+
return { parked: row.status === "suspended" }
|
|
3057
|
+
})).pipe(
|
|
3058
|
+
Effect.orDie,
|
|
3059
|
+
// A host can write a separate database. Release the engine transaction
|
|
3060
|
+
// before calling it, but record its request before publishing the wake.
|
|
3061
|
+
Effect.flatMap((scheduled) =>
|
|
3062
|
+
scheduled === undefined
|
|
3063
|
+
? Effect.void
|
|
3064
|
+
: (reason === "operator" || reason === "delegated"
|
|
3065
|
+
? Effect.void
|
|
3066
|
+
: recordResume(executionId, scheduled.parked, reason)).pipe(
|
|
3067
|
+
Effect.andThen(coordinator.wake(executionId)),
|
|
3068
|
+
Effect.andThen(wakeBus.wake(executionId))
|
|
3069
|
+
)
|
|
3070
|
+
)
|
|
3071
|
+
)
|
|
3072
|
+
)
|
|
3073
|
+
|
|
3074
|
+
return {
|
|
3075
|
+
register: Effect.fn("FlowEngine.register")((flow, handler) =>
|
|
3076
|
+
Effect.acquireRelease(
|
|
3077
|
+
Effect.sync(() => {
|
|
3078
|
+
const registration = { flow, execute: handler }
|
|
3079
|
+
const entries = registrations.get(flow._tag) ?? []
|
|
3080
|
+
entries.push(registration)
|
|
3081
|
+
registrations.set(flow._tag, entries)
|
|
3082
|
+
pendingCycles.delete(flow._tag)
|
|
3083
|
+
warnedUnregistered.clear()
|
|
3084
|
+
return { registration, entries }
|
|
3085
|
+
}),
|
|
3086
|
+
({ registration, entries }) =>
|
|
3087
|
+
Effect.sync(() => {
|
|
3088
|
+
const wasLatest = entries.at(-1) === registration
|
|
3089
|
+
entries.splice(entries.indexOf(registration), 1)
|
|
3090
|
+
if (entries.length === 0) registrations.delete(flow._tag)
|
|
3091
|
+
if (wasLatest) {
|
|
3092
|
+
pendingCycles.delete(flow._tag)
|
|
3093
|
+
// Restoring a live predecessor changes which handler may
|
|
3094
|
+
// drive pending work, just like a fresh registration does.
|
|
3095
|
+
if (entries.length > 0) warnedUnregistered.clear()
|
|
3096
|
+
}
|
|
3097
|
+
})
|
|
3098
|
+
).pipe(Effect.asVoid)
|
|
3099
|
+
),
|
|
3100
|
+
execute,
|
|
3101
|
+
poll,
|
|
3102
|
+
interrupt,
|
|
3103
|
+
// The durable engine has ONE cancellation path (the release policy). The
|
|
3104
|
+
// port promises forced cancellation without cleanup; nothing here can
|
|
3105
|
+
// deliver that — a durable run is closed by a fenced transition whose
|
|
3106
|
+
// finalizers, cascade, and interruption record are the point — and
|
|
3107
|
+
// answering with `interrupt` silently reinterpreted the caller's
|
|
3108
|
+
// request as the safe one. Refusing is what keeps the excluded feature
|
|
3109
|
+
// from appearing to half-work; `interrupt` is the supported request and
|
|
3110
|
+
// is still available to the same caller. The memory engine, which does
|
|
3111
|
+
// have two behaviors, is unchanged.
|
|
3112
|
+
//
|
|
3113
|
+
// The reason is the release policy "Durable interruptUnsafe" verbatim, not a
|
|
3114
|
+
// paraphrase of it: the release note and the failure a caller catches
|
|
3115
|
+
// are the same statement, so an operator who reads one and a developer
|
|
3116
|
+
// who logs the other are reading the same sentence.
|
|
3117
|
+
interruptUnsafe: Effect.fn("FlowEngine.interruptUnsafe")((
|
|
3118
|
+
_flow: Flow.Any,
|
|
3119
|
+
executionId: string
|
|
3120
|
+
) =>
|
|
3121
|
+
Effect.fail(
|
|
3122
|
+
new FlowRuntime.CancelRequestFailed({
|
|
3123
|
+
code: "unsafe_interrupt_unsupported",
|
|
3124
|
+
executionId,
|
|
3125
|
+
reason:
|
|
3126
|
+
"The durable engine has one cancellation path, interrupt, which is durable and cascades to linked children. FlowRuntime.interruptUnsafe on the durable engine fails with unsafe_interrupt_unsupported instead of forcing cancellation without cleanup."
|
|
3127
|
+
})
|
|
3128
|
+
)
|
|
3129
|
+
),
|
|
3130
|
+
resume: Effect.fn("FlowEngine.resume")((flow, executionId, options) =>
|
|
3131
|
+
Effect.annotateCurrentSpan({ executionId, flow: flow._tag }).pipe(
|
|
3132
|
+
// A resume drives the run under its admitted authority, so a caller
|
|
3133
|
+
// whose authority does not cover it may not wake or join it, and a
|
|
3134
|
+
// request that names no admitted run of this flow drives nothing.
|
|
3135
|
+
Effect.andThen(authorizeResume(flow, executionId)),
|
|
3136
|
+
Effect.flatMap((authorized) =>
|
|
3137
|
+
authorized
|
|
3138
|
+
? Effect.andThen(
|
|
3139
|
+
options?.poll === true
|
|
3140
|
+
? Effect.void
|
|
3141
|
+
: scheduleResume(flow._tag, executionId, options?.delegated === true ? "delegated" : "operator"),
|
|
3142
|
+
coordinator.run(executionId)
|
|
3143
|
+
)
|
|
3144
|
+
: Effect.void
|
|
3145
|
+
)
|
|
3146
|
+
)
|
|
3147
|
+
),
|
|
3148
|
+
scheduleResume,
|
|
3149
|
+
active: Effect.fn("FlowEngine.active")(() => coordinator.active)(),
|
|
3150
|
+
retainedRuns
|
|
3151
|
+
}
|
|
3152
|
+
})
|