@smthrs/engine-store 0.0.0-stage → 1.0.0-rc.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +247 -0
- package/LICENSE +21 -0
- package/README.md +146 -2
- package/dist/cjs/ArtifactGc.d.ts +235 -0
- package/dist/cjs/ArtifactGc.d.ts.map +1 -0
- package/dist/cjs/ArtifactGc.js +183 -0
- package/dist/cjs/ArtifactGc.js.map +7 -0
- package/dist/cjs/ArtifactSync.d.ts +164 -0
- package/dist/cjs/ArtifactSync.d.ts.map +1 -0
- package/dist/cjs/ArtifactSync.js +144 -0
- package/dist/cjs/ArtifactSync.js.map +7 -0
- package/dist/cjs/CacheSync.d.ts +112 -0
- package/dist/cjs/CacheSync.d.ts.map +1 -0
- package/dist/cjs/CacheSync.js +64 -0
- package/dist/cjs/CacheSync.js.map +7 -0
- package/dist/cjs/DeferredClockFold.d.ts +354 -0
- package/dist/cjs/DeferredClockFold.d.ts.map +1 -0
- package/dist/cjs/DeferredClockFold.js +335 -0
- package/dist/cjs/DeferredClockFold.js.map +7 -0
- package/dist/cjs/DisasterRecovery.d.ts +431 -0
- package/dist/cjs/DisasterRecovery.d.ts.map +1 -0
- package/dist/cjs/DisasterRecovery.js +453 -0
- package/dist/cjs/DisasterRecovery.js.map +7 -0
- package/dist/cjs/DurableEngineState.d.ts +776 -0
- package/dist/cjs/DurableEngineState.d.ts.map +1 -0
- package/dist/cjs/DurableEngineState.js +1536 -0
- package/dist/cjs/DurableEngineState.js.map +7 -0
- package/dist/cjs/EngineStore.d.ts +158 -0
- package/dist/cjs/EngineStore.d.ts.map +1 -0
- package/dist/cjs/EngineStore.js +398 -0
- package/dist/cjs/EngineStore.js.map +7 -0
- package/dist/cjs/EngineStoreMetrics.d.ts +276 -0
- package/dist/cjs/EngineStoreMetrics.d.ts.map +1 -0
- package/dist/cjs/EngineStoreMetrics.js +135 -0
- package/dist/cjs/EngineStoreMetrics.js.map +7 -0
- package/dist/cjs/Errors.d.ts +22 -0
- package/dist/cjs/Errors.d.ts.map +1 -0
- package/dist/cjs/Errors.js +39 -0
- package/dist/cjs/Errors.js.map +7 -0
- package/dist/cjs/EventTypes.d.ts +27 -0
- package/dist/cjs/EventTypes.d.ts.map +1 -0
- package/dist/cjs/EventTypes.js +37 -0
- package/dist/cjs/EventTypes.js.map +7 -0
- package/dist/cjs/ExecutionFacts.d.ts +59 -0
- package/dist/cjs/ExecutionFacts.d.ts.map +1 -0
- package/dist/cjs/ExecutionFacts.js +123 -0
- package/dist/cjs/ExecutionFacts.js.map +7 -0
- package/dist/cjs/ExecutionSnapshot.d.ts +117 -0
- package/dist/cjs/ExecutionSnapshot.d.ts.map +1 -0
- package/dist/cjs/ExecutionSnapshot.js +111 -0
- package/dist/cjs/ExecutionSnapshot.js.map +7 -0
- package/dist/cjs/Inconsistency.d.ts +154 -0
- package/dist/cjs/Inconsistency.d.ts.map +1 -0
- package/dist/cjs/Inconsistency.js +135 -0
- package/dist/cjs/Inconsistency.js.map +7 -0
- package/dist/cjs/Migrations.d.ts +56 -0
- package/dist/cjs/Migrations.d.ts.map +1 -0
- package/dist/cjs/Migrations.js +78 -0
- package/dist/cjs/Migrations.js.map +7 -0
- package/dist/cjs/OwnerIdentity.d.ts +91 -0
- package/dist/cjs/OwnerIdentity.d.ts.map +1 -0
- package/dist/cjs/OwnerIdentity.js +65 -0
- package/dist/cjs/OwnerIdentity.js.map +7 -0
- package/dist/cjs/PlanInputStore.d.ts +110 -0
- package/dist/cjs/PlanInputStore.d.ts.map +1 -0
- package/dist/cjs/PlanInputStore.js +224 -0
- package/dist/cjs/PlanInputStore.js.map +7 -0
- package/dist/cjs/PlanMergeStore.d.ts +113 -0
- package/dist/cjs/PlanMergeStore.d.ts.map +1 -0
- package/dist/cjs/PlanMergeStore.js +255 -0
- package/dist/cjs/PlanMergeStore.js.map +7 -0
- package/dist/cjs/PlanScheduler.d.ts +363 -0
- package/dist/cjs/PlanScheduler.d.ts.map +1 -0
- package/dist/cjs/PlanScheduler.js +1090 -0
- package/dist/cjs/PlanScheduler.js.map +7 -0
- package/dist/cjs/Reconciliation.d.ts +175 -0
- package/dist/cjs/Reconciliation.d.ts.map +1 -0
- package/dist/cjs/Reconciliation.js +119 -0
- package/dist/cjs/Reconciliation.js.map +7 -0
- package/dist/cjs/ReplayOnly.d.ts +91 -0
- package/dist/cjs/ReplayOnly.d.ts.map +1 -0
- package/dist/cjs/ReplayOnly.js +52 -0
- package/dist/cjs/ReplayOnly.js.map +7 -0
- package/dist/cjs/Retention.d.ts +113 -0
- package/dist/cjs/Retention.d.ts.map +1 -0
- package/dist/cjs/Retention.js +99 -0
- package/dist/cjs/Retention.js.map +7 -0
- package/dist/cjs/RunCatalogRead.d.ts +17 -0
- package/dist/cjs/RunCatalogRead.d.ts.map +1 -0
- package/dist/cjs/RunCatalogRead.js +33 -0
- package/dist/cjs/RunCatalogRead.js.map +7 -0
- package/dist/cjs/RunChangeFeed.d.ts +75 -0
- package/dist/cjs/RunChangeFeed.d.ts.map +1 -0
- package/dist/cjs/RunChangeFeed.js +96 -0
- package/dist/cjs/RunChangeFeed.js.map +7 -0
- package/dist/cjs/RunState.d.ts +116 -0
- package/dist/cjs/RunState.d.ts.map +1 -0
- package/dist/cjs/RunState.js +76 -0
- package/dist/cjs/RunState.js.map +7 -0
- package/dist/cjs/Selection.d.ts +495 -0
- package/dist/cjs/Selection.d.ts.map +1 -0
- package/dist/cjs/Selection.js +288 -0
- package/dist/cjs/Selection.js.map +7 -0
- package/dist/cjs/SelectionStore.d.ts +141 -0
- package/dist/cjs/SelectionStore.d.ts.map +1 -0
- package/dist/cjs/SelectionStore.js +229 -0
- package/dist/cjs/SelectionStore.js.map +7 -0
- package/dist/cjs/StepBoundary.d.ts +501 -0
- package/dist/cjs/StepBoundary.d.ts.map +1 -0
- package/dist/cjs/StepBoundary.js +619 -0
- package/dist/cjs/StepBoundary.js.map +7 -0
- package/dist/cjs/StepCacheFold.d.ts +148 -0
- package/dist/cjs/StepCacheFold.d.ts.map +1 -0
- package/dist/cjs/StepCacheFold.js +131 -0
- package/dist/cjs/StepCacheFold.js.map +7 -0
- package/dist/cjs/StepSandbox.d.ts +78 -0
- package/dist/cjs/StepSandbox.d.ts.map +1 -0
- package/dist/cjs/StepSandbox.js +75 -0
- package/dist/cjs/StepSandbox.js.map +7 -0
- package/dist/cjs/WakeBus.d.ts +116 -0
- package/dist/cjs/WakeBus.d.ts.map +1 -0
- package/dist/cjs/WakeBus.js +89 -0
- package/dist/cjs/WakeBus.js.map +7 -0
- package/dist/cjs/WorkspaceSandbox.d.ts +619 -0
- package/dist/cjs/WorkspaceSandbox.d.ts.map +1 -0
- package/dist/cjs/WorkspaceSandbox.js +711 -0
- package/dist/cjs/WorkspaceSandbox.js.map +7 -0
- package/dist/cjs/index.d.ts +157 -0
- package/dist/cjs/index.d.ts.map +1 -0
- package/dist/cjs/index.js +95 -0
- package/dist/cjs/index.js.map +7 -0
- package/dist/cjs/internal/ActionPersistence.d.ts +276 -0
- package/dist/cjs/internal/ActionPersistence.d.ts.map +1 -0
- package/dist/cjs/internal/ActionPersistence.js +1395 -0
- package/dist/cjs/internal/ActionPersistence.js.map +7 -0
- package/dist/cjs/internal/ArtifactRoots.d.ts +38 -0
- package/dist/cjs/internal/ArtifactRoots.d.ts.map +1 -0
- package/dist/cjs/internal/ArtifactRoots.js +89 -0
- package/dist/cjs/internal/ArtifactRoots.js.map +7 -0
- package/dist/cjs/internal/AttemptAdmission.d.ts +53 -0
- package/dist/cjs/internal/AttemptAdmission.d.ts.map +1 -0
- package/dist/cjs/internal/AttemptAdmission.js +52 -0
- package/dist/cjs/internal/AttemptAdmission.js.map +7 -0
- package/dist/cjs/internal/AttemptLifecycle.d.ts +245 -0
- package/dist/cjs/internal/AttemptLifecycle.d.ts.map +1 -0
- package/dist/cjs/internal/AttemptLifecycle.js +112 -0
- package/dist/cjs/internal/AttemptLifecycle.js.map +7 -0
- package/dist/cjs/internal/AttemptProbe.d.ts +53 -0
- package/dist/cjs/internal/AttemptProbe.d.ts.map +1 -0
- package/dist/cjs/internal/AttemptProbe.js +66 -0
- package/dist/cjs/internal/AttemptProbe.js.map +7 -0
- package/dist/cjs/internal/CacheAdmission.d.ts +79 -0
- package/dist/cjs/internal/CacheAdmission.d.ts.map +1 -0
- package/dist/cjs/internal/CacheAdmission.js +71 -0
- package/dist/cjs/internal/CacheAdmission.js.map +7 -0
- package/dist/cjs/internal/CacheAgeHistory.d.ts +27 -0
- package/dist/cjs/internal/CacheAgeHistory.d.ts.map +1 -0
- package/dist/cjs/internal/CacheAgeHistory.js +61 -0
- package/dist/cjs/internal/CacheAgeHistory.js.map +7 -0
- package/dist/cjs/internal/CacheAgeVerdicts.d.ts +15 -0
- package/dist/cjs/internal/CacheAgeVerdicts.d.ts.map +1 -0
- package/dist/cjs/internal/CacheAgeVerdicts.js +73 -0
- package/dist/cjs/internal/CacheAgeVerdicts.js.map +7 -0
- package/dist/cjs/internal/CacheOutputPolicy.d.ts +24 -0
- package/dist/cjs/internal/CacheOutputPolicy.d.ts.map +1 -0
- package/dist/cjs/internal/CacheOutputPolicy.js +75 -0
- package/dist/cjs/internal/CacheOutputPolicy.js.map +7 -0
- package/dist/cjs/internal/CachePublication.d.ts +97 -0
- package/dist/cjs/internal/CachePublication.d.ts.map +1 -0
- package/dist/cjs/internal/CachePublication.js +80 -0
- package/dist/cjs/internal/CachePublication.js.map +7 -0
- package/dist/cjs/internal/CallFacts.d.ts +15 -0
- package/dist/cjs/internal/CallFacts.d.ts.map +1 -0
- package/dist/cjs/internal/CallFacts.js +97 -0
- package/dist/cjs/internal/CallFacts.js.map +7 -0
- package/dist/cjs/internal/CopiedRecord.d.ts +61 -0
- package/dist/cjs/internal/CopiedRecord.d.ts.map +1 -0
- package/dist/cjs/internal/CopiedRecord.js +134 -0
- package/dist/cjs/internal/CopiedRecord.js.map +7 -0
- package/dist/cjs/internal/DeferredPersistence.d.ts +108 -0
- package/dist/cjs/internal/DeferredPersistence.d.ts.map +1 -0
- package/dist/cjs/internal/DeferredPersistence.js +270 -0
- package/dist/cjs/internal/DeferredPersistence.js.map +7 -0
- package/dist/cjs/internal/EffectRecords.d.ts +92 -0
- package/dist/cjs/internal/EffectRecords.d.ts.map +1 -0
- package/dist/cjs/internal/EffectRecords.js +63 -0
- package/dist/cjs/internal/EffectRecords.js.map +7 -0
- package/dist/cjs/internal/EngineJj.d.ts +30 -0
- package/dist/cjs/internal/EngineJj.d.ts.map +1 -0
- package/dist/cjs/internal/EngineJj.js +42 -0
- package/dist/cjs/internal/EngineJj.js.map +7 -0
- package/dist/cjs/internal/EngineStateSchema.d.ts +68 -0
- package/dist/cjs/internal/EngineStateSchema.d.ts.map +1 -0
- package/dist/cjs/internal/EngineStateSchema.js +58 -0
- package/dist/cjs/internal/EngineStateSchema.js.map +7 -0
- package/dist/cjs/internal/ExecutionSnapshotRead.d.ts +162 -0
- package/dist/cjs/internal/ExecutionSnapshotRead.d.ts.map +1 -0
- package/dist/cjs/internal/ExecutionSnapshotRead.js +199 -0
- package/dist/cjs/internal/ExecutionSnapshotRead.js.map +7 -0
- package/dist/cjs/internal/ExecutionSnapshotRelations.d.ts +39 -0
- package/dist/cjs/internal/ExecutionSnapshotRelations.d.ts.map +1 -0
- package/dist/cjs/internal/ExecutionSnapshotRelations.js +105 -0
- package/dist/cjs/internal/ExecutionSnapshotRelations.js.map +7 -0
- package/dist/cjs/internal/ExitEncoding.d.ts +180 -0
- package/dist/cjs/internal/ExitEncoding.d.ts.map +1 -0
- package/dist/cjs/internal/ExitEncoding.js +236 -0
- package/dist/cjs/internal/ExitEncoding.js.map +7 -0
- package/dist/cjs/internal/FileBoundarySnapshot.d.ts +14 -0
- package/dist/cjs/internal/FileBoundarySnapshot.d.ts.map +1 -0
- package/dist/cjs/internal/FileBoundarySnapshot.js +51 -0
- package/dist/cjs/internal/FileBoundarySnapshot.js.map +7 -0
- package/dist/cjs/internal/FileEnumeration.d.ts +127 -0
- package/dist/cjs/internal/FileEnumeration.d.ts.map +1 -0
- package/dist/cjs/internal/FileEnumeration.js +194 -0
- package/dist/cjs/internal/FileEnumeration.js.map +7 -0
- package/dist/cjs/internal/HostReflection.d.ts +80 -0
- package/dist/cjs/internal/HostReflection.d.ts.map +1 -0
- package/dist/cjs/internal/HostReflection.js +43 -0
- package/dist/cjs/internal/HostReflection.js.map +7 -0
- package/dist/cjs/internal/JournalRecords.d.ts +325 -0
- package/dist/cjs/internal/JournalRecords.d.ts.map +1 -0
- package/dist/cjs/internal/JournalRecords.js +122 -0
- package/dist/cjs/internal/JournalRecords.js.map +7 -0
- package/dist/cjs/internal/NodeJournal.d.ts +157 -0
- package/dist/cjs/internal/NodeJournal.d.ts.map +1 -0
- package/dist/cjs/internal/NodeJournal.js +192 -0
- package/dist/cjs/internal/NodeJournal.js.map +7 -0
- package/dist/cjs/internal/Ordering.d.ts +14 -0
- package/dist/cjs/internal/Ordering.d.ts.map +1 -0
- package/dist/cjs/internal/Ordering.js +25 -0
- package/dist/cjs/internal/Ordering.js.map +7 -0
- package/dist/cjs/internal/PlanInputs.d.ts +14 -0
- package/dist/cjs/internal/PlanInputs.d.ts.map +1 -0
- package/dist/cjs/internal/PlanInputs.js +78 -0
- package/dist/cjs/internal/PlanInputs.js.map +7 -0
- package/dist/cjs/internal/PlanMerges.d.ts +178 -0
- package/dist/cjs/internal/PlanMerges.d.ts.map +1 -0
- package/dist/cjs/internal/PlanMerges.js +125 -0
- package/dist/cjs/internal/PlanMerges.js.map +7 -0
- package/dist/cjs/internal/ProvenanceSlot.d.ts +30 -0
- package/dist/cjs/internal/ProvenanceSlot.d.ts.map +1 -0
- package/dist/cjs/internal/ProvenanceSlot.js +53 -0
- package/dist/cjs/internal/ProvenanceSlot.js.map +7 -0
- package/dist/cjs/internal/ResultEnvelope.d.ts +88 -0
- package/dist/cjs/internal/ResultEnvelope.d.ts.map +1 -0
- package/dist/cjs/internal/ResultEnvelope.js +90 -0
- package/dist/cjs/internal/ResultEnvelope.js.map +7 -0
- package/dist/cjs/internal/RetentionOps.d.ts +414 -0
- package/dist/cjs/internal/RetentionOps.d.ts.map +1 -0
- package/dist/cjs/internal/RetentionOps.js +354 -0
- package/dist/cjs/internal/RetentionOps.js.map +7 -0
- package/dist/cjs/internal/RunCatalogOps.d.ts +118 -0
- package/dist/cjs/internal/RunCatalogOps.d.ts.map +1 -0
- package/dist/cjs/internal/RunCatalogOps.js +85 -0
- package/dist/cjs/internal/RunCatalogOps.js.map +7 -0
- package/dist/cjs/internal/RunCoordinator.d.ts +78 -0
- package/dist/cjs/internal/RunCoordinator.d.ts.map +1 -0
- package/dist/cjs/internal/RunCoordinator.js +123 -0
- package/dist/cjs/internal/RunCoordinator.js.map +7 -0
- package/dist/cjs/internal/RunDriver.d.ts +192 -0
- package/dist/cjs/internal/RunDriver.d.ts.map +1 -0
- package/dist/cjs/internal/RunDriver.js +1584 -0
- package/dist/cjs/internal/RunDriver.js.map +7 -0
- package/dist/cjs/internal/RunLifecycle.d.ts +22 -0
- package/dist/cjs/internal/RunLifecycle.d.ts.map +1 -0
- package/dist/cjs/internal/RunLifecycle.js +63 -0
- package/dist/cjs/internal/RunLifecycle.js.map +7 -0
- package/dist/cjs/internal/RunListing.d.ts +78 -0
- package/dist/cjs/internal/RunListing.d.ts.map +1 -0
- package/dist/cjs/internal/RunListing.js +171 -0
- package/dist/cjs/internal/RunListing.js.map +7 -0
- package/dist/cjs/internal/RunListingError.d.ts +35 -0
- package/dist/cjs/internal/RunListingError.d.ts.map +1 -0
- package/dist/cjs/internal/RunListingError.js +47 -0
- package/dist/cjs/internal/RunListingError.js.map +7 -0
- package/dist/cjs/internal/RuntimeGraph.d.ts +73 -0
- package/dist/cjs/internal/RuntimeGraph.d.ts.map +1 -0
- package/dist/cjs/internal/RuntimeGraph.js +180 -0
- package/dist/cjs/internal/RuntimeGraph.js.map +7 -0
- package/dist/cjs/internal/SandboxedExecution.d.ts +80 -0
- package/dist/cjs/internal/SandboxedExecution.d.ts.map +1 -0
- package/dist/cjs/internal/SandboxedExecution.js +104 -0
- package/dist/cjs/internal/SandboxedExecution.js.map +7 -0
- package/dist/cjs/internal/StateTransaction.d.ts +44 -0
- package/dist/cjs/internal/StateTransaction.d.ts.map +1 -0
- package/dist/cjs/internal/StateTransaction.js +52 -0
- package/dist/cjs/internal/StateTransaction.js.map +7 -0
- package/dist/cjs/internal/StepFacts.d.ts +14 -0
- package/dist/cjs/internal/StepFacts.d.ts.map +1 -0
- package/dist/cjs/internal/StepFacts.js +58 -0
- package/dist/cjs/internal/StepFacts.js.map +7 -0
- package/dist/cjs/internal/TypedEvents.d.ts +14 -0
- package/dist/cjs/internal/TypedEvents.d.ts.map +1 -0
- package/dist/cjs/internal/TypedEvents.js +37 -0
- package/dist/cjs/internal/TypedEvents.js.map +7 -0
- package/dist/cjs/migrations/0001_initial.d.ts +17 -0
- package/dist/cjs/migrations/0001_initial.d.ts.map +1 -0
- package/dist/cjs/migrations/0001_initial.js +61 -0
- package/dist/cjs/migrations/0001_initial.js.map +7 -0
- package/dist/cjs/migrations/0002_selection_store.d.ts +15 -0
- package/dist/cjs/migrations/0002_selection_store.d.ts.map +1 -0
- package/dist/cjs/migrations/0002_selection_store.js +56 -0
- package/dist/cjs/migrations/0002_selection_store.js.map +7 -0
- package/dist/cjs/migrations/0003_plan_inputs.d.ts +13 -0
- package/dist/cjs/migrations/0003_plan_inputs.d.ts.map +1 -0
- package/dist/cjs/migrations/0003_plan_inputs.js +90 -0
- package/dist/cjs/migrations/0003_plan_inputs.js.map +7 -0
- package/dist/cjs/migrations/0004_plan_environment.d.ts +14 -0
- package/dist/cjs/migrations/0004_plan_environment.d.ts.map +1 -0
- package/dist/cjs/migrations/0004_plan_environment.js +56 -0
- package/dist/cjs/migrations/0004_plan_environment.js.map +7 -0
- package/dist/cjs/migrations/0005_plan_merges.d.ts +14 -0
- package/dist/cjs/migrations/0005_plan_merges.d.ts.map +1 -0
- package/dist/cjs/migrations/0005_plan_merges.js +82 -0
- package/dist/cjs/migrations/0005_plan_merges.js.map +7 -0
- package/dist/cjs/migrations/0006_execution_listing.d.ts +15 -0
- package/dist/cjs/migrations/0006_execution_listing.d.ts.map +1 -0
- package/dist/cjs/migrations/0006_execution_listing.js +101 -0
- package/dist/cjs/migrations/0006_execution_listing.js.map +7 -0
- package/dist/cjs/migrations/0007_run_parent_sequence.d.ts +15 -0
- package/dist/cjs/migrations/0007_run_parent_sequence.d.ts.map +1 -0
- package/dist/cjs/migrations/0007_run_parent_sequence.js +40 -0
- package/dist/cjs/migrations/0007_run_parent_sequence.js.map +7 -0
- package/dist/cjs/migrations/0008_deferred_consumption.d.ts +16 -0
- package/dist/cjs/migrations/0008_deferred_consumption.d.ts.map +1 -0
- package/dist/cjs/migrations/0008_deferred_consumption.js +45 -0
- package/dist/cjs/migrations/0008_deferred_consumption.js.map +7 -0
- package/dist/cjs/migrations/0009_deferred_clock_facts.d.ts +24 -0
- package/dist/cjs/migrations/0009_deferred_clock_facts.d.ts.map +1 -0
- package/dist/cjs/migrations/0009_deferred_clock_facts.js +141 -0
- package/dist/cjs/migrations/0009_deferred_clock_facts.js.map +7 -0
- package/dist/cjs/migrations/0010_pending_cancellation.d.ts +15 -0
- package/dist/cjs/migrations/0010_pending_cancellation.d.ts.map +1 -0
- package/dist/cjs/migrations/0010_pending_cancellation.js +41 -0
- package/dist/cjs/migrations/0010_pending_cancellation.js.map +7 -0
- package/dist/cjs/package.json +1 -0
- package/dist/cjs/test/TestStores.d.ts +89 -0
- package/dist/cjs/test/TestStores.d.ts.map +1 -0
- package/dist/cjs/test/TestStores.js +98 -0
- package/dist/cjs/test/TestStores.js.map +7 -0
- package/dist/esm/ArtifactGc.d.ts +235 -0
- package/dist/esm/ArtifactGc.d.ts.map +1 -0
- package/dist/esm/ArtifactGc.js +268 -0
- package/dist/esm/ArtifactGc.js.map +1 -0
- package/dist/esm/ArtifactSync.d.ts +164 -0
- package/dist/esm/ArtifactSync.d.ts.map +1 -0
- package/dist/esm/ArtifactSync.js +176 -0
- package/dist/esm/ArtifactSync.js.map +1 -0
- package/dist/esm/CacheSync.d.ts +112 -0
- package/dist/esm/CacheSync.d.ts.map +1 -0
- package/dist/esm/CacheSync.js +94 -0
- package/dist/esm/CacheSync.js.map +1 -0
- package/dist/esm/DeferredClockFold.d.ts +354 -0
- package/dist/esm/DeferredClockFold.d.ts.map +1 -0
- package/dist/esm/DeferredClockFold.js +472 -0
- package/dist/esm/DeferredClockFold.js.map +1 -0
- package/dist/esm/DisasterRecovery.d.ts +431 -0
- package/dist/esm/DisasterRecovery.d.ts.map +1 -0
- package/dist/esm/DisasterRecovery.js +523 -0
- package/dist/esm/DisasterRecovery.js.map +1 -0
- package/dist/esm/DurableEngineState.d.ts +776 -0
- package/dist/esm/DurableEngineState.d.ts.map +1 -0
- package/dist/esm/DurableEngineState.js +1648 -0
- package/dist/esm/DurableEngineState.js.map +1 -0
- package/dist/esm/EngineStore.d.ts +158 -0
- package/dist/esm/EngineStore.d.ts.map +1 -0
- package/dist/esm/EngineStore.js +403 -0
- package/dist/esm/EngineStore.js.map +1 -0
- package/dist/esm/EngineStoreMetrics.d.ts +276 -0
- package/dist/esm/EngineStoreMetrics.d.ts.map +1 -0
- package/dist/esm/EngineStoreMetrics.js +260 -0
- package/dist/esm/EngineStoreMetrics.js.map +1 -0
- package/dist/esm/Errors.d.ts +22 -0
- package/dist/esm/Errors.d.ts.map +1 -0
- package/dist/esm/Errors.js +22 -0
- package/dist/esm/Errors.js.map +1 -0
- package/dist/esm/EventTypes.d.ts +27 -0
- package/dist/esm/EventTypes.d.ts.map +1 -0
- package/dist/esm/EventTypes.js +27 -0
- package/dist/esm/EventTypes.js.map +1 -0
- package/dist/esm/ExecutionFacts.d.ts +59 -0
- package/dist/esm/ExecutionFacts.d.ts.map +1 -0
- package/dist/esm/ExecutionFacts.js +96 -0
- package/dist/esm/ExecutionFacts.js.map +1 -0
- package/dist/esm/ExecutionSnapshot.d.ts +117 -0
- package/dist/esm/ExecutionSnapshot.d.ts.map +1 -0
- package/dist/esm/ExecutionSnapshot.js +95 -0
- package/dist/esm/ExecutionSnapshot.js.map +1 -0
- package/dist/esm/Inconsistency.d.ts +154 -0
- package/dist/esm/Inconsistency.d.ts.map +1 -0
- package/dist/esm/Inconsistency.js +133 -0
- package/dist/esm/Inconsistency.js.map +1 -0
- package/dist/esm/Migrations.d.ts +56 -0
- package/dist/esm/Migrations.d.ts.map +1 -0
- package/dist/esm/Migrations.js +91 -0
- package/dist/esm/Migrations.js.map +1 -0
- package/dist/esm/OwnerIdentity.d.ts +91 -0
- package/dist/esm/OwnerIdentity.d.ts.map +1 -0
- package/dist/esm/OwnerIdentity.js +99 -0
- package/dist/esm/OwnerIdentity.js.map +1 -0
- package/dist/esm/PlanInputStore.d.ts +110 -0
- package/dist/esm/PlanInputStore.d.ts.map +1 -0
- package/dist/esm/PlanInputStore.js +207 -0
- package/dist/esm/PlanInputStore.js.map +1 -0
- package/dist/esm/PlanMergeStore.d.ts +113 -0
- package/dist/esm/PlanMergeStore.d.ts.map +1 -0
- package/dist/esm/PlanMergeStore.js +244 -0
- package/dist/esm/PlanMergeStore.js.map +1 -0
- package/dist/esm/PlanScheduler.d.ts +363 -0
- package/dist/esm/PlanScheduler.d.ts.map +1 -0
- package/dist/esm/PlanScheduler.js +1321 -0
- package/dist/esm/PlanScheduler.js.map +1 -0
- package/dist/esm/Reconciliation.d.ts +175 -0
- package/dist/esm/Reconciliation.d.ts.map +1 -0
- package/dist/esm/Reconciliation.js +175 -0
- package/dist/esm/Reconciliation.js.map +1 -0
- package/dist/esm/ReplayOnly.d.ts +91 -0
- package/dist/esm/ReplayOnly.d.ts.map +1 -0
- package/dist/esm/ReplayOnly.js +59 -0
- package/dist/esm/ReplayOnly.js.map +1 -0
- package/dist/esm/Retention.d.ts +113 -0
- package/dist/esm/Retention.d.ts.map +1 -0
- package/dist/esm/Retention.js +139 -0
- package/dist/esm/Retention.js.map +1 -0
- package/dist/esm/RunCatalogRead.d.ts +17 -0
- package/dist/esm/RunCatalogRead.d.ts.map +1 -0
- package/dist/esm/RunCatalogRead.js +17 -0
- package/dist/esm/RunCatalogRead.js.map +1 -0
- package/dist/esm/RunChangeFeed.d.ts +75 -0
- package/dist/esm/RunChangeFeed.d.ts.map +1 -0
- package/dist/esm/RunChangeFeed.js +81 -0
- package/dist/esm/RunChangeFeed.js.map +1 -0
- package/dist/esm/RunState.d.ts +116 -0
- package/dist/esm/RunState.d.ts.map +1 -0
- package/dist/esm/RunState.js +109 -0
- package/dist/esm/RunState.js.map +1 -0
- package/dist/esm/Selection.d.ts +495 -0
- package/dist/esm/Selection.d.ts.map +1 -0
- package/dist/esm/Selection.js +482 -0
- package/dist/esm/Selection.js.map +1 -0
- package/dist/esm/SelectionStore.d.ts +141 -0
- package/dist/esm/SelectionStore.d.ts.map +1 -0
- package/dist/esm/SelectionStore.js +215 -0
- package/dist/esm/SelectionStore.js.map +1 -0
- package/dist/esm/StepBoundary.d.ts +501 -0
- package/dist/esm/StepBoundary.d.ts.map +1 -0
- package/dist/esm/StepBoundary.js +919 -0
- package/dist/esm/StepBoundary.js.map +1 -0
- package/dist/esm/StepCacheFold.d.ts +148 -0
- package/dist/esm/StepCacheFold.d.ts.map +1 -0
- package/dist/esm/StepCacheFold.js +201 -0
- package/dist/esm/StepCacheFold.js.map +1 -0
- package/dist/esm/StepSandbox.d.ts +78 -0
- package/dist/esm/StepSandbox.d.ts.map +1 -0
- package/dist/esm/StepSandbox.js +70 -0
- package/dist/esm/StepSandbox.js.map +1 -0
- package/dist/esm/WakeBus.d.ts +116 -0
- package/dist/esm/WakeBus.d.ts.map +1 -0
- package/dist/esm/WakeBus.js +121 -0
- package/dist/esm/WakeBus.js.map +1 -0
- package/dist/esm/WorkspaceSandbox.d.ts +619 -0
- package/dist/esm/WorkspaceSandbox.d.ts.map +1 -0
- package/dist/esm/WorkspaceSandbox.js +985 -0
- package/dist/esm/WorkspaceSandbox.js.map +1 -0
- package/dist/esm/index.d.ts +157 -0
- package/dist/esm/index.d.ts.map +1 -0
- package/dist/esm/index.js +157 -0
- package/dist/esm/index.js.map +1 -0
- package/dist/esm/internal/ActionPersistence.d.ts +276 -0
- package/dist/esm/internal/ActionPersistence.d.ts.map +1 -0
- package/dist/esm/internal/ActionPersistence.js +2306 -0
- package/dist/esm/internal/ActionPersistence.js.map +1 -0
- package/dist/esm/internal/ArtifactRoots.d.ts +38 -0
- package/dist/esm/internal/ArtifactRoots.d.ts.map +1 -0
- package/dist/esm/internal/ArtifactRoots.js +76 -0
- package/dist/esm/internal/ArtifactRoots.js.map +1 -0
- package/dist/esm/internal/AttemptAdmission.d.ts +53 -0
- package/dist/esm/internal/AttemptAdmission.d.ts.map +1 -0
- package/dist/esm/internal/AttemptAdmission.js +52 -0
- package/dist/esm/internal/AttemptAdmission.js.map +1 -0
- package/dist/esm/internal/AttemptLifecycle.d.ts +245 -0
- package/dist/esm/internal/AttemptLifecycle.d.ts.map +1 -0
- package/dist/esm/internal/AttemptLifecycle.js +104 -0
- package/dist/esm/internal/AttemptLifecycle.js.map +1 -0
- package/dist/esm/internal/AttemptProbe.d.ts +53 -0
- package/dist/esm/internal/AttemptProbe.d.ts.map +1 -0
- package/dist/esm/internal/AttemptProbe.js +83 -0
- package/dist/esm/internal/AttemptProbe.js.map +1 -0
- package/dist/esm/internal/CacheAdmission.d.ts +79 -0
- package/dist/esm/internal/CacheAdmission.d.ts.map +1 -0
- package/dist/esm/internal/CacheAdmission.js +72 -0
- package/dist/esm/internal/CacheAdmission.js.map +1 -0
- package/dist/esm/internal/CacheAgeHistory.d.ts +27 -0
- package/dist/esm/internal/CacheAgeHistory.d.ts.map +1 -0
- package/dist/esm/internal/CacheAgeHistory.js +49 -0
- package/dist/esm/internal/CacheAgeHistory.js.map +1 -0
- package/dist/esm/internal/CacheAgeVerdicts.d.ts +15 -0
- package/dist/esm/internal/CacheAgeVerdicts.d.ts.map +1 -0
- package/dist/esm/internal/CacheAgeVerdicts.js +54 -0
- package/dist/esm/internal/CacheAgeVerdicts.js.map +1 -0
- package/dist/esm/internal/CacheOutputPolicy.d.ts +24 -0
- package/dist/esm/internal/CacheOutputPolicy.d.ts.map +1 -0
- package/dist/esm/internal/CacheOutputPolicy.js +73 -0
- package/dist/esm/internal/CacheOutputPolicy.js.map +1 -0
- package/dist/esm/internal/CachePublication.d.ts +97 -0
- package/dist/esm/internal/CachePublication.d.ts.map +1 -0
- package/dist/esm/internal/CachePublication.js +116 -0
- package/dist/esm/internal/CachePublication.js.map +1 -0
- package/dist/esm/internal/CallFacts.d.ts +15 -0
- package/dist/esm/internal/CallFacts.d.ts.map +1 -0
- package/dist/esm/internal/CallFacts.js +83 -0
- package/dist/esm/internal/CallFacts.js.map +1 -0
- package/dist/esm/internal/CopiedRecord.d.ts +61 -0
- package/dist/esm/internal/CopiedRecord.d.ts.map +1 -0
- package/dist/esm/internal/CopiedRecord.js +165 -0
- package/dist/esm/internal/CopiedRecord.js.map +1 -0
- package/dist/esm/internal/DeferredPersistence.d.ts +108 -0
- package/dist/esm/internal/DeferredPersistence.d.ts.map +1 -0
- package/dist/esm/internal/DeferredPersistence.js +240 -0
- package/dist/esm/internal/DeferredPersistence.js.map +1 -0
- package/dist/esm/internal/EffectRecords.d.ts +92 -0
- package/dist/esm/internal/EffectRecords.d.ts.map +1 -0
- package/dist/esm/internal/EffectRecords.js +94 -0
- package/dist/esm/internal/EffectRecords.js.map +1 -0
- package/dist/esm/internal/EngineJj.d.ts +30 -0
- package/dist/esm/internal/EngineJj.d.ts.map +1 -0
- package/dist/esm/internal/EngineJj.js +29 -0
- package/dist/esm/internal/EngineJj.js.map +1 -0
- package/dist/esm/internal/EngineStateSchema.d.ts +68 -0
- package/dist/esm/internal/EngineStateSchema.d.ts.map +1 -0
- package/dist/esm/internal/EngineStateSchema.js +53 -0
- package/dist/esm/internal/EngineStateSchema.js.map +1 -0
- package/dist/esm/internal/ExecutionSnapshotRead.d.ts +162 -0
- package/dist/esm/internal/ExecutionSnapshotRead.d.ts.map +1 -0
- package/dist/esm/internal/ExecutionSnapshotRead.js +218 -0
- package/dist/esm/internal/ExecutionSnapshotRead.js.map +1 -0
- package/dist/esm/internal/ExecutionSnapshotRelations.d.ts +39 -0
- package/dist/esm/internal/ExecutionSnapshotRelations.d.ts.map +1 -0
- package/dist/esm/internal/ExecutionSnapshotRelations.js +81 -0
- package/dist/esm/internal/ExecutionSnapshotRelations.js.map +1 -0
- package/dist/esm/internal/ExitEncoding.d.ts +180 -0
- package/dist/esm/internal/ExitEncoding.d.ts.map +1 -0
- package/dist/esm/internal/ExitEncoding.js +361 -0
- package/dist/esm/internal/ExitEncoding.js.map +1 -0
- package/dist/esm/internal/FileBoundarySnapshot.d.ts +14 -0
- package/dist/esm/internal/FileBoundarySnapshot.d.ts.map +1 -0
- package/dist/esm/internal/FileBoundarySnapshot.js +29 -0
- package/dist/esm/internal/FileBoundarySnapshot.js.map +1 -0
- package/dist/esm/internal/FileEnumeration.d.ts +127 -0
- package/dist/esm/internal/FileEnumeration.d.ts.map +1 -0
- package/dist/esm/internal/FileEnumeration.js +236 -0
- package/dist/esm/internal/FileEnumeration.js.map +1 -0
- package/dist/esm/internal/HostReflection.d.ts +80 -0
- package/dist/esm/internal/HostReflection.d.ts.map +1 -0
- package/dist/esm/internal/HostReflection.js +98 -0
- package/dist/esm/internal/HostReflection.js.map +1 -0
- package/dist/esm/internal/JournalRecords.d.ts +325 -0
- package/dist/esm/internal/JournalRecords.d.ts.map +1 -0
- package/dist/esm/internal/JournalRecords.js +306 -0
- package/dist/esm/internal/JournalRecords.js.map +1 -0
- package/dist/esm/internal/NodeJournal.d.ts +157 -0
- package/dist/esm/internal/NodeJournal.d.ts.map +1 -0
- package/dist/esm/internal/NodeJournal.js +298 -0
- package/dist/esm/internal/NodeJournal.js.map +1 -0
- package/dist/esm/internal/Ordering.d.ts +14 -0
- package/dist/esm/internal/Ordering.d.ts.map +1 -0
- package/dist/esm/internal/Ordering.js +14 -0
- package/dist/esm/internal/Ordering.js.map +1 -0
- package/dist/esm/internal/PlanInputs.d.ts +14 -0
- package/dist/esm/internal/PlanInputs.d.ts.map +1 -0
- package/dist/esm/internal/PlanInputs.js +57 -0
- package/dist/esm/internal/PlanInputs.js.map +1 -0
- package/dist/esm/internal/PlanMerges.d.ts +178 -0
- package/dist/esm/internal/PlanMerges.d.ts.map +1 -0
- package/dist/esm/internal/PlanMerges.js +113 -0
- package/dist/esm/internal/PlanMerges.js.map +1 -0
- package/dist/esm/internal/ProvenanceSlot.d.ts +30 -0
- package/dist/esm/internal/ProvenanceSlot.d.ts.map +1 -0
- package/dist/esm/internal/ProvenanceSlot.js +39 -0
- package/dist/esm/internal/ProvenanceSlot.js.map +1 -0
- package/dist/esm/internal/ResultEnvelope.d.ts +88 -0
- package/dist/esm/internal/ResultEnvelope.d.ts.map +1 -0
- package/dist/esm/internal/ResultEnvelope.js +83 -0
- package/dist/esm/internal/ResultEnvelope.js.map +1 -0
- package/dist/esm/internal/RetentionOps.d.ts +414 -0
- package/dist/esm/internal/RetentionOps.d.ts.map +1 -0
- package/dist/esm/internal/RetentionOps.js +576 -0
- package/dist/esm/internal/RetentionOps.js.map +1 -0
- package/dist/esm/internal/RunCatalogOps.d.ts +118 -0
- package/dist/esm/internal/RunCatalogOps.d.ts.map +1 -0
- package/dist/esm/internal/RunCatalogOps.js +120 -0
- package/dist/esm/internal/RunCatalogOps.js.map +1 -0
- package/dist/esm/internal/RunCoordinator.d.ts +78 -0
- package/dist/esm/internal/RunCoordinator.d.ts.map +1 -0
- package/dist/esm/internal/RunCoordinator.js +115 -0
- package/dist/esm/internal/RunCoordinator.js.map +1 -0
- package/dist/esm/internal/RunDriver.d.ts +192 -0
- package/dist/esm/internal/RunDriver.d.ts.map +1 -0
- package/dist/esm/internal/RunDriver.js +2334 -0
- package/dist/esm/internal/RunDriver.js.map +1 -0
- package/dist/esm/internal/RunLifecycle.d.ts +22 -0
- package/dist/esm/internal/RunLifecycle.d.ts.map +1 -0
- package/dist/esm/internal/RunLifecycle.js +49 -0
- package/dist/esm/internal/RunLifecycle.js.map +1 -0
- package/dist/esm/internal/RunListing.d.ts +78 -0
- package/dist/esm/internal/RunListing.d.ts.map +1 -0
- package/dist/esm/internal/RunListing.js +146 -0
- package/dist/esm/internal/RunListing.js.map +1 -0
- package/dist/esm/internal/RunListingError.d.ts +35 -0
- package/dist/esm/internal/RunListingError.d.ts.map +1 -0
- package/dist/esm/internal/RunListingError.js +27 -0
- package/dist/esm/internal/RunListingError.js.map +1 -0
- package/dist/esm/internal/RuntimeGraph.d.ts +73 -0
- package/dist/esm/internal/RuntimeGraph.d.ts.map +1 -0
- package/dist/esm/internal/RuntimeGraph.js +182 -0
- package/dist/esm/internal/RuntimeGraph.js.map +1 -0
- package/dist/esm/internal/SandboxedExecution.d.ts +80 -0
- package/dist/esm/internal/SandboxedExecution.d.ts.map +1 -0
- package/dist/esm/internal/SandboxedExecution.js +107 -0
- package/dist/esm/internal/SandboxedExecution.js.map +1 -0
- package/dist/esm/internal/StateTransaction.d.ts +44 -0
- package/dist/esm/internal/StateTransaction.d.ts.map +1 -0
- package/dist/esm/internal/StateTransaction.js +54 -0
- package/dist/esm/internal/StateTransaction.js.map +1 -0
- package/dist/esm/internal/StepFacts.d.ts +14 -0
- package/dist/esm/internal/StepFacts.d.ts.map +1 -0
- package/dist/esm/internal/StepFacts.js +35 -0
- package/dist/esm/internal/StepFacts.js.map +1 -0
- package/dist/esm/internal/TypedEvents.d.ts +14 -0
- package/dist/esm/internal/TypedEvents.d.ts.map +1 -0
- package/dist/esm/internal/TypedEvents.js +14 -0
- package/dist/esm/internal/TypedEvents.js.map +1 -0
- package/dist/esm/migrations/0001_initial.d.ts +17 -0
- package/dist/esm/migrations/0001_initial.d.ts.map +1 -0
- package/dist/esm/migrations/0001_initial.js +41 -0
- package/dist/esm/migrations/0001_initial.js.map +1 -0
- package/dist/esm/migrations/0002_selection_store.d.ts +15 -0
- package/dist/esm/migrations/0002_selection_store.d.ts.map +1 -0
- package/dist/esm/migrations/0002_selection_store.js +34 -0
- package/dist/esm/migrations/0002_selection_store.js.map +1 -0
- package/dist/esm/migrations/0003_plan_inputs.d.ts +13 -0
- package/dist/esm/migrations/0003_plan_inputs.d.ts.map +1 -0
- package/dist/esm/migrations/0003_plan_inputs.js +70 -0
- package/dist/esm/migrations/0003_plan_inputs.js.map +1 -0
- package/dist/esm/migrations/0004_plan_environment.d.ts +14 -0
- package/dist/esm/migrations/0004_plan_environment.d.ts.map +1 -0
- package/dist/esm/migrations/0004_plan_environment.js +33 -0
- package/dist/esm/migrations/0004_plan_environment.js.map +1 -0
- package/dist/esm/migrations/0005_plan_merges.d.ts +14 -0
- package/dist/esm/migrations/0005_plan_merges.d.ts.map +1 -0
- package/dist/esm/migrations/0005_plan_merges.js +59 -0
- package/dist/esm/migrations/0005_plan_merges.js.map +1 -0
- package/dist/esm/migrations/0006_execution_listing.d.ts +15 -0
- package/dist/esm/migrations/0006_execution_listing.d.ts.map +1 -0
- package/dist/esm/migrations/0006_execution_listing.js +81 -0
- package/dist/esm/migrations/0006_execution_listing.js.map +1 -0
- package/dist/esm/migrations/0007_run_parent_sequence.d.ts +15 -0
- package/dist/esm/migrations/0007_run_parent_sequence.d.ts.map +1 -0
- package/dist/esm/migrations/0007_run_parent_sequence.js +18 -0
- package/dist/esm/migrations/0007_run_parent_sequence.js.map +1 -0
- package/dist/esm/migrations/0008_deferred_consumption.d.ts +16 -0
- package/dist/esm/migrations/0008_deferred_consumption.d.ts.map +1 -0
- package/dist/esm/migrations/0008_deferred_consumption.js +24 -0
- package/dist/esm/migrations/0008_deferred_consumption.js.map +1 -0
- package/dist/esm/migrations/0009_deferred_clock_facts.d.ts +24 -0
- package/dist/esm/migrations/0009_deferred_clock_facts.d.ts.map +1 -0
- package/dist/esm/migrations/0009_deferred_clock_facts.js +127 -0
- package/dist/esm/migrations/0009_deferred_clock_facts.js.map +1 -0
- package/dist/esm/migrations/0010_pending_cancellation.d.ts +15 -0
- package/dist/esm/migrations/0010_pending_cancellation.d.ts.map +1 -0
- package/dist/esm/migrations/0010_pending_cancellation.js +19 -0
- package/dist/esm/migrations/0010_pending_cancellation.js.map +1 -0
- package/dist/esm/test/TestStores.d.ts +89 -0
- package/dist/esm/test/TestStores.d.ts.map +1 -0
- package/dist/esm/test/TestStores.js +103 -0
- package/dist/esm/test/TestStores.js.map +1 -0
- package/docs/README.md +113 -0
- package/docs/api.md +1494 -0
- package/docs/concepts/attempts-and-replay.md +149 -0
- package/docs/concepts/cache-admission.md +265 -0
- package/docs/concepts/call-facts.md +86 -0
- package/docs/concepts/durable-waits.md +175 -0
- package/docs/concepts/execution-facts.md +90 -0
- package/docs/concepts/ownership-and-fencing.md +128 -0
- package/docs/concepts/step-boundaries.md +148 -0
- package/docs/concepts/workspace-transactions.md +199 -0
- package/docs/event-types.md +14 -0
- package/docs/guides/back-up-and-restore.md +164 -0
- package/docs/guides/collect-unreferenced-artifacts.md +128 -0
- package/docs/guides/compose-a-durable-engine.md +146 -0
- package/docs/guides/coordinate-two-processes.md +120 -0
- package/docs/guides/defer-work-with-selection.md +171 -0
- package/docs/guides/delete-old-run-history.md +154 -0
- package/docs/guides/drive-a-plan.md +314 -0
- package/docs/guides/observe-engine-metrics.md +98 -0
- package/docs/guides/observe-executions.md +187 -0
- package/docs/guides/reclaim-runs-from-a-dead-host.md +133 -0
- package/docs/guides/share-a-cache-across-machines.md +134 -0
- package/docs/guides/testing.md +123 -0
- package/docs/installation.md +102 -0
- package/docs/quickstart.md +215 -0
- package/docs/troubleshooting.md +347 -0
- package/package.json +438 -3
- package/src/ArtifactGc.ts +408 -0
- package/src/ArtifactSync.ts +278 -0
- package/src/CacheSync.ts +131 -0
- package/src/DeferredClockFold.ts +586 -0
- package/src/DisasterRecovery.ts +888 -0
- package/src/DurableEngineState.ts +2656 -0
- package/src/EngineStore.ts +656 -0
- package/src/EngineStoreMetrics.ts +331 -0
- package/src/Errors.ts +28 -0
- package/src/EventTypes.ts +27 -0
- package/src/ExecutionFacts.ts +125 -0
- package/src/ExecutionSnapshot.ts +186 -0
- package/src/Inconsistency.ts +253 -0
- package/src/Migrations.ts +95 -0
- package/src/OwnerIdentity.ts +125 -0
- package/src/PlanInputStore.ts +292 -0
- package/src/PlanMergeStore.ts +330 -0
- package/src/PlanScheduler.ts +1806 -0
- package/src/Reconciliation.ts +224 -0
- package/src/ReplayOnly.ts +95 -0
- package/src/Retention.ts +214 -0
- package/src/RunCatalogRead.ts +27 -0
- package/src/RunChangeFeed.ts +125 -0
- package/src/RunState.ts +137 -0
- package/src/Selection.ts +683 -0
- package/src/SelectionStore.ts +353 -0
- package/src/StepBoundary.ts +1165 -0
- package/src/StepCacheFold.ts +244 -0
- package/src/StepSandbox.ts +110 -0
- package/src/WakeBus.ts +167 -0
- package/src/WorkspaceSandbox.ts +1604 -0
- package/src/index.ts +185 -0
- package/src/internal/ActionPersistence.ts +2687 -0
- package/src/internal/ArtifactRoots.ts +92 -0
- package/src/internal/AttemptAdmission.ts +76 -0
- package/src/internal/AttemptLifecycle.ts +163 -0
- package/src/internal/AttemptProbe.ts +90 -0
- package/src/internal/CacheAdmission.ts +133 -0
- package/src/internal/CacheAgeHistory.ts +58 -0
- package/src/internal/CacheAgeVerdicts.ts +61 -0
- package/src/internal/CacheOutputPolicy.ts +82 -0
- package/src/internal/CachePublication.ts +153 -0
- package/src/internal/CallFacts.ts +87 -0
- package/src/internal/CopiedRecord.ts +201 -0
- package/src/internal/DeferredPersistence.ts +459 -0
- package/src/internal/EffectRecords.ts +136 -0
- package/src/internal/EngineJj.ts +30 -0
- package/src/internal/EngineStateSchema.ts +92 -0
- package/src/internal/ExecutionSnapshotRead.ts +298 -0
- package/src/internal/ExecutionSnapshotRelations.ts +121 -0
- package/src/internal/ExitEncoding.ts +453 -0
- package/src/internal/FileBoundarySnapshot.ts +39 -0
- package/src/internal/FileEnumeration.ts +334 -0
- package/src/internal/HostReflection.ts +128 -0
- package/src/internal/JournalRecords.ts +396 -0
- package/src/internal/NodeJournal.ts +340 -0
- package/src/internal/Ordering.ts +14 -0
- package/src/internal/PlanInputs.ts +62 -0
- package/src/internal/PlanMerges.ts +123 -0
- package/src/internal/ProvenanceSlot.ts +52 -0
- package/src/internal/ResultEnvelope.ts +118 -0
- package/src/internal/RetentionOps.ts +865 -0
- package/src/internal/RunCatalogOps.ts +174 -0
- package/src/internal/RunCoordinator.ts +215 -0
- package/src/internal/RunDriver.ts +3152 -0
- package/src/internal/RunLifecycle.ts +53 -0
- package/src/internal/RunListing.ts +210 -0
- package/src/internal/RunListingError.ts +37 -0
- package/src/internal/RuntimeGraph.ts +253 -0
- package/src/internal/SandboxedExecution.ts +165 -0
- package/src/internal/StateTransaction.ts +71 -0
- package/src/internal/StepFacts.ts +36 -0
- package/src/internal/TypedEvents.ts +29 -0
- package/src/migrations/0001_initial.ts +51 -0
- package/src/migrations/0002_selection_store.ts +38 -0
- package/src/migrations/0003_plan_inputs.ts +75 -0
- package/src/migrations/0004_plan_environment.ts +34 -0
- package/src/migrations/0005_plan_merges.ts +60 -0
- package/src/migrations/0006_execution_listing.ts +86 -0
- package/src/migrations/0007_run_parent_sequence.ts +19 -0
- package/src/migrations/0008_deferred_consumption.ts +27 -0
- package/src/migrations/0009_deferred_clock_facts.ts +163 -0
- package/src/migrations/0010_pending_cancellation.ts +20 -0
- package/src/test/TestStores.ts +145 -0
|
@@ -0,0 +1,2687 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Durable action dispatch at the engine encoded seam.
|
|
3
|
+
*
|
|
4
|
+
* Governing design: `packages/smithers/flows/plan/docs/concepts/step-keys.md`.
|
|
5
|
+
*
|
|
6
|
+
* @since 0.1.0
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import { Sha256 } from "@smthrs/crypto"
|
|
10
|
+
import { FlowEngine } from "@smthrs/engine"
|
|
11
|
+
import { Action, FlowRuntime } from "@smthrs/flow"
|
|
12
|
+
import * as CacheEnvironment from "@smthrs/flow/CacheEnvironment"
|
|
13
|
+
import type { FileBoundary } from "@smthrs/flow/FileBoundary"
|
|
14
|
+
import { Journal, type JournalEvent, Redaction } from "@smthrs/journal"
|
|
15
|
+
import { Jj } from "@smthrs/kernel"
|
|
16
|
+
import { DerivedKey } from "@smthrs/keys"
|
|
17
|
+
import { AttemptStore, Ownership, RunStore } from "@smthrs/run-store"
|
|
18
|
+
import { CacheStore } from "@smthrs/step-cache"
|
|
19
|
+
import * as Cause from "effect/Cause"
|
|
20
|
+
import * as Clock from "effect/Clock"
|
|
21
|
+
import * as Context from "effect/Context"
|
|
22
|
+
import * as Effect from "effect/Effect"
|
|
23
|
+
import * as Exit from "effect/Exit"
|
|
24
|
+
import * as Metric from "effect/Metric"
|
|
25
|
+
import * as Option from "effect/Option"
|
|
26
|
+
import * as Schema from "effect/Schema"
|
|
27
|
+
import * as EngineStoreMetrics from "../EngineStoreMetrics.ts"
|
|
28
|
+
import * as Inconsistency from "../Inconsistency.ts"
|
|
29
|
+
import * as ReplayOnly from "../ReplayOnly.ts"
|
|
30
|
+
import * as StepBoundary from "../StepBoundary.ts"
|
|
31
|
+
import * as StepSandbox from "../StepSandbox.ts"
|
|
32
|
+
import * as WorkspaceSandbox from "../WorkspaceSandbox.ts"
|
|
33
|
+
import * as AttemptAdmission from "./AttemptAdmission.ts"
|
|
34
|
+
import * as CacheAdmission from "./CacheAdmission.ts"
|
|
35
|
+
import * as CacheAgeHistory from "./CacheAgeHistory.ts"
|
|
36
|
+
import * as CacheAgeVerdicts from "./CacheAgeVerdicts.ts"
|
|
37
|
+
import * as CacheOutputPolicy from "./CacheOutputPolicy.ts"
|
|
38
|
+
import * as CachePublication from "./CachePublication.ts"
|
|
39
|
+
import * as CallFacts from "./CallFacts.ts"
|
|
40
|
+
import * as CopiedRecord from "./CopiedRecord.ts"
|
|
41
|
+
import * as EffectRecords from "./EffectRecords.ts"
|
|
42
|
+
import * as HostReflection from "./HostReflection.ts"
|
|
43
|
+
import * as JournalRecords from "./JournalRecords.ts"
|
|
44
|
+
import * as ProvenanceSlot from "./ProvenanceSlot.ts"
|
|
45
|
+
import * as SandboxedExecution from "./SandboxedExecution.ts"
|
|
46
|
+
import * as StepFacts from "./StepFacts.ts"
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* The boundary declaration an action may carry alongside its input.
|
|
50
|
+
*
|
|
51
|
+
* Aliased to `FileBoundary` rather than re-declared so the dispatch path and
|
|
52
|
+
* the `@smthrs/flow` declaration cannot drift apart; it is a distinct name only
|
|
53
|
+
* because "metadata" is how the action input refers to it.
|
|
54
|
+
*
|
|
55
|
+
* @since 0.1.0
|
|
56
|
+
* @category models
|
|
57
|
+
*/
|
|
58
|
+
export type BoundaryMetadata = FileBoundary
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Everything one dispatch of a durable action needs: the opaque `action`
|
|
62
|
+
* body, which `attempt` this is, the step `key` it is cached under, its trust
|
|
63
|
+
* `tier`, and the boundary it declared, if any.
|
|
64
|
+
*
|
|
65
|
+
* The `key` is a digest of the caller's declaration, so two dispatches sharing
|
|
66
|
+
* one carry the same claim about their inputs — which is what makes the
|
|
67
|
+
* attempt row and the cache row addressable by it.
|
|
68
|
+
*
|
|
69
|
+
* @since 0.1.0
|
|
70
|
+
* @category models
|
|
71
|
+
*/
|
|
72
|
+
export interface ActionInput {
|
|
73
|
+
readonly action: unknown
|
|
74
|
+
readonly attempt: number
|
|
75
|
+
readonly key: string
|
|
76
|
+
readonly tier: Action.Tier
|
|
77
|
+
/**
|
|
78
|
+
* The caller declares this sealed step's recorded result is one of many
|
|
79
|
+
* legitimate results; a same-key divergence is expected, not a hermeticity
|
|
80
|
+
* violation.
|
|
81
|
+
*/
|
|
82
|
+
readonly nondeterministic?: true | undefined
|
|
83
|
+
readonly metadata?: BoundaryMetadata | undefined
|
|
84
|
+
/**
|
|
85
|
+
* The engine's `SnapshotBoundary` preparation for a compensable attempt:
|
|
86
|
+
* restore the retry sequence's original handle, then take this attempt's.
|
|
87
|
+
* Evaluated only when the attempt actually executes, after the journal and
|
|
88
|
+
* cache lookups, and its handle is persisted in the attempt row before the
|
|
89
|
+
* body runs.
|
|
90
|
+
*/
|
|
91
|
+
readonly snapshot?: Effect.Effect<unknown> | undefined
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* The attempt stopped without settling — it is durably parked, not failed.
|
|
96
|
+
*
|
|
97
|
+
* The distinction matters to the driver: a suspended attempt keeps its row and
|
|
98
|
+
* its attempt number, so resuming continues the same attempt rather than
|
|
99
|
+
* burning a new one against the retry budget.
|
|
100
|
+
*
|
|
101
|
+
* @since 0.1.0
|
|
102
|
+
* @category errors
|
|
103
|
+
*/
|
|
104
|
+
export class AttemptSuspended extends Schema.TaggedError<AttemptSuspended>()(
|
|
105
|
+
"@smthrs/engine-store/AttemptSuspended",
|
|
106
|
+
{
|
|
107
|
+
code: Schema.Literal("attempt_suspended"),
|
|
108
|
+
runId: Schema.String,
|
|
109
|
+
keyDigest: Schema.String,
|
|
110
|
+
attempt: Schema.Int
|
|
111
|
+
}
|
|
112
|
+
) {}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* The attempt was not admitted, so its body never ran.
|
|
116
|
+
*
|
|
117
|
+
* `outcome` names which admission check refused — a superseded fence, a live
|
|
118
|
+
* same-key attempt, an already-settled row. Because the body did not execute,
|
|
119
|
+
* this failure is always safe to surface without compensation.
|
|
120
|
+
*
|
|
121
|
+
* @since 0.1.0
|
|
122
|
+
* @category errors
|
|
123
|
+
*/
|
|
124
|
+
export class AttemptAdmissionRejected extends Schema.TaggedError<AttemptAdmissionRejected>()(
|
|
125
|
+
"@smthrs/engine-store/AttemptAdmissionRejected",
|
|
126
|
+
{
|
|
127
|
+
code: Schema.Literal("attempt_admission_rejected"),
|
|
128
|
+
keyDigest: Schema.String,
|
|
129
|
+
outcome: Schema.String
|
|
130
|
+
}
|
|
131
|
+
) {}
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* Two different runs recorded results under the same step key.
|
|
135
|
+
*
|
|
136
|
+
* The key is a digest of the declaration, so a conflict means the declaration
|
|
137
|
+
* does not fully describe what the step depends on: same key, different
|
|
138
|
+
* answer. `recordedRunId` names the run that got there first, so the
|
|
139
|
+
* divergence can be investigated rather than silently resolved.
|
|
140
|
+
*
|
|
141
|
+
* @since 0.1.0
|
|
142
|
+
* @category errors
|
|
143
|
+
*/
|
|
144
|
+
export class CacheConflictDetected extends Schema.TaggedError<CacheConflictDetected>()(
|
|
145
|
+
"@smthrs/engine-store/CacheConflictDetected",
|
|
146
|
+
{
|
|
147
|
+
code: Schema.Literal("cache_conflict_detected"),
|
|
148
|
+
keyDigest: Schema.String,
|
|
149
|
+
recordedRunId: Schema.String
|
|
150
|
+
}
|
|
151
|
+
) {}
|
|
152
|
+
|
|
153
|
+
/**
|
|
154
|
+
* A cached output's bytes no longer hash to the digest recorded for them.
|
|
155
|
+
*
|
|
156
|
+
* Unlike a succeeded attempt row, a shared cache row is evictable: the entry is
|
|
157
|
+
* dropped and the next dispatch re-executes and re-captures cleanly (issue
|
|
158
|
+
* #164). Reported rather than swallowed so a failing disk is visible.
|
|
159
|
+
*
|
|
160
|
+
* @since 0.1.0
|
|
161
|
+
* @category errors
|
|
162
|
+
*/
|
|
163
|
+
export class CacheCorruptionDetected extends Schema.TaggedError<CacheCorruptionDetected>()(
|
|
164
|
+
"@smthrs/engine-store/CacheCorruptionDetected",
|
|
165
|
+
{
|
|
166
|
+
code: Schema.Literal("cache_corruption_detected"),
|
|
167
|
+
keyDigest: Schema.String,
|
|
168
|
+
path: Schema.String,
|
|
169
|
+
recordedDigest: Schema.String,
|
|
170
|
+
measuredDigest: Schema.String
|
|
171
|
+
}
|
|
172
|
+
) {}
|
|
173
|
+
|
|
174
|
+
/**
|
|
175
|
+
* Corrupt recorded evidence on a SUCCEEDED durable attempt row under the
|
|
176
|
+
* strict verdict (issue #171).
|
|
177
|
+
*
|
|
178
|
+
* Distinct from {@link CacheCorruptionDetected} on purpose: a corrupt shared
|
|
179
|
+
* cache row is evictable — the next dispatch re-executes and re-captures
|
|
180
|
+
* cleanly (issue #164) — but a succeeded attempt row records that this run's
|
|
181
|
+
* side effects already ran, so eviction/re-execution would violate
|
|
182
|
+
* exactly-once for irreversible actions. The corrupt boundary evidence is
|
|
183
|
+
* quarantined off the succeeded row instead: the driver parks the first
|
|
184
|
+
* detection in the `quarantine` waiting state, and the next explicit resume
|
|
185
|
+
* returns the durable outcome without re-materializing the poisoned evidence
|
|
186
|
+
* or re-executing the action.
|
|
187
|
+
*
|
|
188
|
+
* @since 0.1.0
|
|
189
|
+
* @category errors
|
|
190
|
+
*/
|
|
191
|
+
export class AttemptEvidenceQuarantined extends Schema.TaggedError<AttemptEvidenceQuarantined>()(
|
|
192
|
+
"@smthrs/engine-store/AttemptEvidenceQuarantined",
|
|
193
|
+
{
|
|
194
|
+
code: Schema.Literal("attempt_evidence_quarantined"),
|
|
195
|
+
keyDigest: Schema.String,
|
|
196
|
+
attempt: Schema.Int,
|
|
197
|
+
path: Schema.String,
|
|
198
|
+
recordedDigest: Schema.String,
|
|
199
|
+
measuredDigest: Schema.String
|
|
200
|
+
}
|
|
201
|
+
) {}
|
|
202
|
+
|
|
203
|
+
/**
|
|
204
|
+
* Extracts an {@link AttemptEvidenceQuarantined} carried anywhere in a flow
|
|
205
|
+
* result's cause — as a typed failure or squashed into a defect — so the
|
|
206
|
+
* driver can park the run instead of settling it `failed` (issue #171).
|
|
207
|
+
*
|
|
208
|
+
* @since 0.1.0
|
|
209
|
+
* @category errors
|
|
210
|
+
*/
|
|
211
|
+
export const evidenceQuarantined = (
|
|
212
|
+
cause: Cause.Cause<unknown>
|
|
213
|
+
): AttemptEvidenceQuarantined | undefined => {
|
|
214
|
+
for (const reason of cause.reasons) {
|
|
215
|
+
const carried = Cause.isFailReason(reason)
|
|
216
|
+
? reason.error
|
|
217
|
+
: Cause.isDieReason(reason)
|
|
218
|
+
? reason.defect
|
|
219
|
+
: undefined
|
|
220
|
+
if (carried instanceof AttemptEvidenceQuarantined) {
|
|
221
|
+
return carried
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
return undefined
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
/**
|
|
228
|
+
* The one classification of a `replayOutputs` failure (issue #150): a
|
|
229
|
+
* `BoundaryCorruption` in the cause is on-disk corruption of recorded
|
|
230
|
+
* evidence — the store's strongest invariant violated — while anything else
|
|
231
|
+
* is a transient host refusal that stays retryable. The two previously
|
|
232
|
+
* journalled identically, so a failing disk corrupting many blobs was
|
|
233
|
+
* indistinguishable from a one-off EIO.
|
|
234
|
+
*/
|
|
235
|
+
const replayCorruption = (
|
|
236
|
+
cause: Cause.Cause<unknown>
|
|
237
|
+
): StepBoundary.BoundaryCorruption | undefined => {
|
|
238
|
+
for (const reason of cause.reasons) {
|
|
239
|
+
if (Cause.isFailReason(reason) && reason.error instanceof StepBoundary.BoundaryCorruption) {
|
|
240
|
+
return reason.error
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
return undefined
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
/**
|
|
247
|
+
* The stable effect kind a compensation handler is registered under.
|
|
248
|
+
*
|
|
249
|
+
* The action name is the adapter's own identity and is what an engine
|
|
250
|
+
* composition registers its handler by; a dispatch whose action carries no
|
|
251
|
+
* name at all (the plan scheduler's synthetic node dispatch) falls back to a
|
|
252
|
+
* constant, which resolves to no handler and therefore assesses as blocking —
|
|
253
|
+
* the safe direction.
|
|
254
|
+
*/
|
|
255
|
+
const actionKind = (action: unknown): string =>
|
|
256
|
+
typeof action === "object" && action !== null && typeof (action as { name?: unknown }).name === "string"
|
|
257
|
+
? (action as { readonly name: string }).name
|
|
258
|
+
: "flows/engine-store/action"
|
|
259
|
+
|
|
260
|
+
/**
|
|
261
|
+
* Whether a failing execution failed because it broke its declared boundary.
|
|
262
|
+
*
|
|
263
|
+
* The isolated execution path raises the boundary's own `UndeclaredWrite`, so
|
|
264
|
+
* a violation detected while the body ran classifies exactly like one detected
|
|
265
|
+
* at settle time: the row records `hardViolation` and the journal gets the
|
|
266
|
+
* violation record, rather than the failure passing as an ordinary action
|
|
267
|
+
* error the retry policy might happily retry.
|
|
268
|
+
*/
|
|
269
|
+
const declarationViolated = (cause: Cause.Cause<unknown>): boolean =>
|
|
270
|
+
cause.reasons.some((reason) =>
|
|
271
|
+
Cause.isFailReason(reason) &&
|
|
272
|
+
(reason.error instanceof StepBoundary.UndeclaredWrite || reason.error instanceof StepSandbox.UndeclaredRead)
|
|
273
|
+
)
|
|
274
|
+
|
|
275
|
+
/**
|
|
276
|
+
* Whether a failed settle carries one of the boundary's own contract
|
|
277
|
+
* violations, as opposed to a host refusal. Classification for the
|
|
278
|
+
* `boundarySettlements` counter only — the journal record stays the source
|
|
279
|
+
* of truth.
|
|
280
|
+
*/
|
|
281
|
+
const settlementViolated = (cause: Cause.Cause<unknown>): boolean =>
|
|
282
|
+
cause.reasons.some((reason) =>
|
|
283
|
+
Cause.isFailReason(reason) &&
|
|
284
|
+
(reason.error instanceof StepBoundary.UndeclaredWrite ||
|
|
285
|
+
reason.error instanceof StepBoundary.MissingDeclaredOutput ||
|
|
286
|
+
reason.error instanceof StepBoundary.SurvivingDeclaredRemoval)
|
|
287
|
+
)
|
|
288
|
+
|
|
289
|
+
/**
|
|
290
|
+
* What the action dispatcher is constructed with: the run it belongs to, the
|
|
291
|
+
* ownership fence it writes under, and the `execute` function that actually
|
|
292
|
+
* runs an action body.
|
|
293
|
+
*
|
|
294
|
+
* Everything durable — the attempt row, the cache row, the journal record — is
|
|
295
|
+
* this module's job; `execute` is the only part it delegates, which is what
|
|
296
|
+
* keeps the persistence discipline in one place regardless of what a flow
|
|
297
|
+
* runtime does with the body.
|
|
298
|
+
*
|
|
299
|
+
* @since 0.1.0
|
|
300
|
+
* @category models
|
|
301
|
+
*/
|
|
302
|
+
export interface Dependencies {
|
|
303
|
+
readonly runId: string
|
|
304
|
+
readonly owner: Ownership.OwnerId
|
|
305
|
+
readonly sourceId: string
|
|
306
|
+
/** Runs one action body. The only part of a dispatch this module delegates. */
|
|
307
|
+
readonly execute: (input: ActionInput) => Effect.Effect<unknown, unknown>
|
|
308
|
+
/** Repository authority reserved for engine bookkeeping. */
|
|
309
|
+
readonly engineJj?: Jj.Jj | undefined
|
|
310
|
+
/**
|
|
311
|
+
* Makes a retry of an irreversible action recognizable downstream. Its
|
|
312
|
+
* absence is what
|
|
313
|
+
* {@link Action.IrreversibleRetryRequiresIdempotencyKey} reports.
|
|
314
|
+
*/
|
|
315
|
+
readonly idempotencyKey?: string | undefined
|
|
316
|
+
/**
|
|
317
|
+
* The incarnation-wide admission mutex (issues #102, #103). `EngineStore`
|
|
318
|
+
* passes one shared instance per store incarnation so every dispatch of a
|
|
319
|
+
* given attempt key in this process serializes through it; when omitted, a
|
|
320
|
+
* fresh mutex private to this `make` call is used — correct only when all
|
|
321
|
+
* same-key dispatches share the returned executor.
|
|
322
|
+
*/
|
|
323
|
+
readonly admission?: AttemptAdmission.Service | undefined
|
|
324
|
+
/** Shared by callers that construct an executor for every dispatch in a run. */
|
|
325
|
+
readonly cacheAgeVerdict?: ReturnType<typeof CacheAgeVerdicts.make> | undefined
|
|
326
|
+
/**
|
|
327
|
+
* Present on a replay-only engine: every settled dispatch is reported to it,
|
|
328
|
+
* and a dispatch that would execute a body dies with `WouldExecute` first.
|
|
329
|
+
*/
|
|
330
|
+
readonly replayOnly?: ReplayOnly.Service | undefined
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
const AttemptMeta = Schema.Struct({
|
|
334
|
+
tier: Schema.Literals(["sealed", "compensable", "irreversible"]),
|
|
335
|
+
// Only a genuine action idempotency key permits automatic released retry.
|
|
336
|
+
// Old rows without this evidence remain fail-closed (issues #2982/#3409).
|
|
337
|
+
keyed: Schema.optional(Schema.Literal(true)),
|
|
338
|
+
/**
|
|
339
|
+
* The sealed declaration admits multiple legitimate recorded results under
|
|
340
|
+
* this key. Absence remains the durable determinism claim.
|
|
341
|
+
*/
|
|
342
|
+
nondeterministic: Schema.optional(Schema.Literal(true)),
|
|
343
|
+
boundary: Schema.optional(StepBoundary.BoundaryEvidence),
|
|
344
|
+
/**
|
|
345
|
+
* The prepare-time measurement matched the caller-declared read set the
|
|
346
|
+
* step key was derived from (issue #106). Only such completions may enter
|
|
347
|
+
* the shared cache: a stale declaration executes against the *real*
|
|
348
|
+
* content, so recording its result under the declaration's key would hand
|
|
349
|
+
* a later, genuinely accurate run the wrong value as a verified hit.
|
|
350
|
+
*/
|
|
351
|
+
readSetVerified: Schema.optional(Schema.Literal(true)),
|
|
352
|
+
/**
|
|
353
|
+
* The row's boundary evidence failed integrity verification and was removed
|
|
354
|
+
* after its detailed corruption record reached the journal (issue #171).
|
|
355
|
+
* The succeeded outcome remains authoritative, but this row may never be
|
|
356
|
+
* converged into the shared cache again.
|
|
357
|
+
*/
|
|
358
|
+
boundaryQuarantined: Schema.optional(Schema.Literal(true)),
|
|
359
|
+
/**
|
|
360
|
+
* The failed row records a boundary violation (a prepare or settle
|
|
361
|
+
* failure), so the failed replay branch can re-emit the `hardViolation`
|
|
362
|
+
* journal record idempotently after a crash in the finish→emit window
|
|
363
|
+
* (issue #109) — the violation kind is not recoverable from the persisted
|
|
364
|
+
* cause alone.
|
|
365
|
+
*/
|
|
366
|
+
hardViolation: Schema.optional(Schema.Literal(true)),
|
|
367
|
+
snapshotId: Schema.optional(Schema.String),
|
|
368
|
+
/**
|
|
369
|
+
* The jj operation that recorded `snapshotId`, when the host reports one.
|
|
370
|
+
* A whole-repository rewind restores it to undo bookmark moves, rebases,
|
|
371
|
+
* `describe`, and `abandon` that a tree restore leaves in place.
|
|
372
|
+
*/
|
|
373
|
+
snapshotOperationId: Schema.optional(Schema.String),
|
|
374
|
+
/**
|
|
375
|
+
* The engine `SnapshotBoundary` handle taken before this attempt executed.
|
|
376
|
+
* Wrapped so an opaque `null` or `undefined` handle stays distinguishable
|
|
377
|
+
* from no handle; `handle` is absent when the handle was `undefined`.
|
|
378
|
+
*/
|
|
379
|
+
boundarySnapshot: Schema.optional(Schema.Struct({ handle: Schema.optionalKey(Schema.Unknown) })),
|
|
380
|
+
// The incarnation that admitted the running row. Since issues #102/#103
|
|
381
|
+
// the adoption decision rests on the admission permit rather than this
|
|
382
|
+
// nonce — a live same-key fiber of this process would be holding the
|
|
383
|
+
// permit, which distinguishes a dead fiber from a live dispatch in a way
|
|
384
|
+
// the recorded incarnation cannot — but the field is kept as durable
|
|
385
|
+
// forensic evidence of which incarnation last drove the attempt.
|
|
386
|
+
admittedBy: Schema.optional(Ownership.OwnerId),
|
|
387
|
+
/**
|
|
388
|
+
* How far this attempt's effect boundary got, in EXECUTABLE state.
|
|
389
|
+
*
|
|
390
|
+
* The journal's boundary records already say this, and adoption cannot use
|
|
391
|
+
* them: `@smthrs/journal` puts every payload it writes through redaction,
|
|
392
|
+
* deliberately, so a recorded `output` may never be re-entered as a flow's
|
|
393
|
+
* result (issue #72), and the journal offers no read addressed by producer
|
|
394
|
+
* identity — only a full scan of the run. So the same fact is written here,
|
|
395
|
+
* where the attempt's `outcome` column is already replayed verbatim, in the
|
|
396
|
+
* SAME transaction as the record that describes it.
|
|
397
|
+
*
|
|
398
|
+
* - `intended`: the body was dispatched and its outcome is not durably
|
|
399
|
+
* known. Whether the effect reached the outside world is genuinely
|
|
400
|
+
* unanswerable from durable state.
|
|
401
|
+
* - `parked`: the body reached a durable wait and suspended, so it never
|
|
402
|
+
* finished and nothing crossed. Recorded explicitly because a park leaves
|
|
403
|
+
* the same `running` row a crash does.
|
|
404
|
+
* - `succeeded`: the body completed and the row carries its result.
|
|
405
|
+
*/
|
|
406
|
+
effectCrossing: Schema.optional(Schema.Literals(["intended", "parked", "succeeded"]))
|
|
407
|
+
})
|
|
408
|
+
|
|
409
|
+
type AttemptMeta = typeof AttemptMeta.Type
|
|
410
|
+
|
|
411
|
+
const decodeMeta = (value: unknown): AttemptMeta | undefined => {
|
|
412
|
+
const decoded = Schema.decodeUnknownResult(AttemptMeta)(value)
|
|
413
|
+
return decoded._tag === "Success" ? decoded.success : undefined
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
/**
|
|
417
|
+
* The earliest boundary snapshot handle persisted for an action key's
|
|
418
|
+
* surviving attempts, walking `earliest` through `latest`. An attempt row
|
|
419
|
+
* without one (it crashed before its snapshot was persisted) is skipped.
|
|
420
|
+
*
|
|
421
|
+
* @since 0.1.0
|
|
422
|
+
* @category combinators
|
|
423
|
+
*/
|
|
424
|
+
export const earliestBoundarySnapshot = (
|
|
425
|
+
attempts: AttemptStore.Service,
|
|
426
|
+
runId: string,
|
|
427
|
+
stepKeyDigest: string,
|
|
428
|
+
range: { readonly earliest: number; readonly latest: number }
|
|
429
|
+
): Effect.Effect<Option.Option<unknown>> =>
|
|
430
|
+
Effect.gen(function*() {
|
|
431
|
+
for (let attempt = range.earliest; attempt <= range.latest; attempt++) {
|
|
432
|
+
const row = yield* attempts.get({ runId, stepKeyDigest, attempt }).pipe(Effect.orDie)
|
|
433
|
+
const persisted = Option.isSome(row) ? decodeMeta(row.value.meta)?.boundarySnapshot : undefined
|
|
434
|
+
if (persisted !== undefined) return Option.some(persisted.handle)
|
|
435
|
+
}
|
|
436
|
+
return Option.none()
|
|
437
|
+
})
|
|
438
|
+
|
|
439
|
+
const CauseJson = Schema.Struct({
|
|
440
|
+
reasons: Schema.Array(Schema.Union([
|
|
441
|
+
Schema.Struct({ _tag: Schema.Literal("Fail"), error: Schema.Unknown }),
|
|
442
|
+
Schema.Struct({ _tag: Schema.Literal("Die"), defect: Schema.Unknown }),
|
|
443
|
+
Schema.Struct({
|
|
444
|
+
_tag: Schema.Literal("Interrupt"),
|
|
445
|
+
fiberId: Schema.optional(Schema.NullOr(Schema.Number))
|
|
446
|
+
})
|
|
447
|
+
]))
|
|
448
|
+
})
|
|
449
|
+
|
|
450
|
+
/**
|
|
451
|
+
* Maximum nesting copied from one live failure value.
|
|
452
|
+
*
|
|
453
|
+
* The bounds here are the store's own admission limits, less the envelope
|
|
454
|
+
* `persistCause` wraps around a reduced value, and never tighter: a value the
|
|
455
|
+
* store would have persisted must not be thrown away here. `AttemptStore`
|
|
456
|
+
* admits 128 levels, and a reduced value sits three levels down, inside
|
|
457
|
+
* `{ reasons: [{ error }] }`.
|
|
458
|
+
*
|
|
459
|
+
* @since 1.0.0
|
|
460
|
+
* @category constants
|
|
461
|
+
*/
|
|
462
|
+
export const maxInertJsonDepth = AttemptStore.maximumJsonDepth - 4
|
|
463
|
+
|
|
464
|
+
/**
|
|
465
|
+
* Maximum JSON values copied from one persisted cause.
|
|
466
|
+
*
|
|
467
|
+
* The budget is spent across the cause object, its reasons array, every reason
|
|
468
|
+
* envelope, and every reduced leaf. A cause that cannot fit collapses to one
|
|
469
|
+
* bounded reason before it reaches `AttemptStore`. Own-key enumeration has a
|
|
470
|
+
* separate allowance of the same size so ignored keys cannot make traversal
|
|
471
|
+
* unbounded.
|
|
472
|
+
*
|
|
473
|
+
* @since 1.0.0
|
|
474
|
+
* @category constants
|
|
475
|
+
*/
|
|
476
|
+
export const maxInertJsonNodes = AttemptStore.maximumJsonNodes - 8
|
|
477
|
+
|
|
478
|
+
/**
|
|
479
|
+
* Maximum UTF-16 units copied from one persisted cause, keys included.
|
|
480
|
+
*
|
|
481
|
+
* Spent across every caller-supplied key and string leaf. This bounds text the
|
|
482
|
+
* reduction inspects and retains, not the serialized byte length: numeric
|
|
483
|
+
* spellings, punctuation, and the fixed cause envelope are outside this
|
|
484
|
+
* character count and are bounded separately by {@link maxInertJsonNodes}.
|
|
485
|
+
*
|
|
486
|
+
* @since 1.0.0
|
|
487
|
+
* @category constants
|
|
488
|
+
*/
|
|
489
|
+
export const maxInertJsonCharacters = 256 * 1024
|
|
490
|
+
|
|
491
|
+
type InertJsonValue =
|
|
492
|
+
| null
|
|
493
|
+
| boolean
|
|
494
|
+
| number
|
|
495
|
+
| string
|
|
496
|
+
| Array<InertJsonValue>
|
|
497
|
+
| { [key: string]: InertJsonValue }
|
|
498
|
+
|
|
499
|
+
const inertJsonRejected = Symbol("inertJsonRejected")
|
|
500
|
+
const inertJsonOmitted = Symbol("inertJsonOmitted")
|
|
501
|
+
const arrayBufferViewMarker = "[ArrayBufferView]"
|
|
502
|
+
|
|
503
|
+
type InertJsonReduction = InertJsonValue | typeof inertJsonRejected | typeof inertJsonOmitted
|
|
504
|
+
|
|
505
|
+
/** What one cause's reduction has already spent of its shared bounds. */
|
|
506
|
+
interface InertJsonBudget {
|
|
507
|
+
nodes: number
|
|
508
|
+
characters: number
|
|
509
|
+
enumeratedKeys: number
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
/** Charges JSON values against the cause-wide node allowance. */
|
|
513
|
+
const spendNodes = (budget: InertJsonBudget, nodes: number): boolean => {
|
|
514
|
+
if (budget.nodes + nodes > maxInertJsonNodes) return false
|
|
515
|
+
budget.nodes += nodes
|
|
516
|
+
return true
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
/** The store rejects unpaired surrogates, so an ill-formed string is refused. */
|
|
520
|
+
const isWellFormedInertString = (value: string): boolean => {
|
|
521
|
+
for (let index = 0; index < value.length; index++) {
|
|
522
|
+
const unit = value.charCodeAt(index)
|
|
523
|
+
if (unit >= 0xd800 && unit <= 0xdbff) {
|
|
524
|
+
const low = value.charCodeAt(++index)
|
|
525
|
+
if (!(low >= 0xdc00 && low <= 0xdfff)) return false
|
|
526
|
+
} else if (unit >= 0xdc00 && unit <= 0xdfff) return false
|
|
527
|
+
}
|
|
528
|
+
return true
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
/**
|
|
532
|
+
* Charges one string, a key or a value, against the cause's character budget.
|
|
533
|
+
* Answers whether the string is admitted.
|
|
534
|
+
*/
|
|
535
|
+
const spend = (budget: InertJsonBudget, value: string): boolean => {
|
|
536
|
+
budget.characters += value.length
|
|
537
|
+
return budget.characters <= maxInertJsonCharacters && isWellFormedInertString(value)
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
/** Charges every own key before its type or descriptor can make it ignorable. */
|
|
541
|
+
const spendEnumeratedKey = (budget: InertJsonBudget, key: PropertyKey): boolean => {
|
|
542
|
+
budget.enumeratedKeys += 1
|
|
543
|
+
if (budget.enumeratedKeys > maxInertJsonNodes) return false
|
|
544
|
+
return typeof key !== "string" || spend(budget, key)
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
/**
|
|
548
|
+
* Projects one failure value into the inert JSON data `AttemptStore` admits.
|
|
549
|
+
*
|
|
550
|
+
* A `Fail` reason raised by a service rather than by `Action.executeEncoded`
|
|
551
|
+
* carries a live class instance — `StepBoundary.UnsupportedBoundary`, a
|
|
552
|
+
* `StepSandbox` refusal, any `Data.TaggedError` — and a `Die` reason carries
|
|
553
|
+
* an arbitrary defect. `@smthrs/run-store`'s attempt boundary refuses a
|
|
554
|
+
* non-plain object outright rather than serializing it, so the projection has
|
|
555
|
+
* to happen on the write side, here, where the durable shape is already owned.
|
|
556
|
+
*
|
|
557
|
+
* The reduction reads only own data-property descriptors. It never invokes a
|
|
558
|
+
* getter or `toJSON`, and a host-detected proxy is rejected before any of its
|
|
559
|
+
* traps can run. An array-buffer view becomes the fixed
|
|
560
|
+
* `"[ArrayBufferView]"` marker before its indexed own keys can be materialized;
|
|
561
|
+
* its bytes and attached properties are deliberately omitted. JSON leaves,
|
|
562
|
+
* arrays, and enumerable string-keyed object data otherwise keep their ordinary
|
|
563
|
+
* JSON shape. Unsupported object members are omitted and unsupported array
|
|
564
|
+
* members become `null`, matching JSON's container rules. Every key returned
|
|
565
|
+
* by an own-key enumeration spends from a cause-wide allowance before its type,
|
|
566
|
+
* enumerability, or descriptor is inspected.
|
|
567
|
+
*
|
|
568
|
+
* Depth, node, and character bounds apply before the attempt store's admission
|
|
569
|
+
* boundary, and `budget` carries the allowances across every reason of one
|
|
570
|
+
* cause. The caller charges the root value as part of its reason envelope; this
|
|
571
|
+
* walk charges every descendant. A cycle, proxy, inspection refusal, or
|
|
572
|
+
* exceeded bound rejects the whole value as `null`, so persistence on the
|
|
573
|
+
* failure path neither executes user code nor leaves an attempt running because
|
|
574
|
+
* its error was too large for the store.
|
|
575
|
+
*/
|
|
576
|
+
const inertJson = (value: unknown, budget: InertJsonBudget): unknown => {
|
|
577
|
+
const active = new WeakSet<object>()
|
|
578
|
+
|
|
579
|
+
const reduce = (current: unknown, depth: number): InertJsonReduction => {
|
|
580
|
+
if (depth > maxInertJsonDepth) return inertJsonRejected
|
|
581
|
+
if (depth > 0 && !spendNodes(budget, 1)) return inertJsonRejected
|
|
582
|
+
|
|
583
|
+
if (current === null) return null
|
|
584
|
+
switch (typeof current) {
|
|
585
|
+
case "boolean":
|
|
586
|
+
return current
|
|
587
|
+
case "number":
|
|
588
|
+
return Number.isFinite(current) ? current : null
|
|
589
|
+
case "string":
|
|
590
|
+
return spend(budget, current) ? current : inertJsonRejected
|
|
591
|
+
case "undefined":
|
|
592
|
+
case "symbol":
|
|
593
|
+
case "function":
|
|
594
|
+
case "bigint":
|
|
595
|
+
return inertJsonOmitted
|
|
596
|
+
case "object": {
|
|
597
|
+
if (HostReflection.host.isProxy(current) || active.has(current)) return inertJsonRejected
|
|
598
|
+
if (ArrayBuffer.isView(current)) return arrayBufferViewMarker
|
|
599
|
+
active.add(current)
|
|
600
|
+
try {
|
|
601
|
+
if (Array.isArray(current)) {
|
|
602
|
+
const lengthDescriptor = Object.getOwnPropertyDescriptor(current, "length")
|
|
603
|
+
/* v8 ignore next 3 -- A non-proxy Array always has an own data-property uint32 length. */
|
|
604
|
+
if (
|
|
605
|
+
lengthDescriptor === undefined || !("value" in lengthDescriptor) ||
|
|
606
|
+
typeof lengthDescriptor.value !== "number"
|
|
607
|
+
) return inertJsonRejected
|
|
608
|
+
const output: Array<InertJsonValue> = []
|
|
609
|
+
for (let index = 0; index < lengthDescriptor.value; index++) {
|
|
610
|
+
const descriptor = Object.getOwnPropertyDescriptor(current, String(index))
|
|
611
|
+
const reduced = reduce(
|
|
612
|
+
descriptor !== undefined && "value" in descriptor ? descriptor.value : undefined,
|
|
613
|
+
depth + 1
|
|
614
|
+
)
|
|
615
|
+
if (reduced === inertJsonRejected) return inertJsonRejected
|
|
616
|
+
output.push(reduced === inertJsonOmitted ? null : reduced)
|
|
617
|
+
}
|
|
618
|
+
return output
|
|
619
|
+
}
|
|
620
|
+
|
|
621
|
+
const output: Record<string, InertJsonValue> = {}
|
|
622
|
+
// This engine-owned error's native Error.message is non-enumerable,
|
|
623
|
+
// but is part of the durable schema used to classify a landing
|
|
624
|
+
// conflict. Preserve that one data field, never getters, stacks, or
|
|
625
|
+
// arbitrary hidden properties. It spends the same bounded budget.
|
|
626
|
+
const tag = Object.getOwnPropertyDescriptor(current, "_tag")
|
|
627
|
+
const conflictMessage = tag !== undefined && "value" in tag &&
|
|
628
|
+
tag.value === WorkspaceSandbox.MaterializationConflict.identifier
|
|
629
|
+
for (const key of Reflect.ownKeys(current)) {
|
|
630
|
+
if (!spendEnumeratedKey(budget, key)) return inertJsonRejected
|
|
631
|
+
if (typeof key !== "string") continue
|
|
632
|
+
const descriptor = Object.getOwnPropertyDescriptor(current, key)
|
|
633
|
+
/* v8 ignore next -- A key returned by Reflect.ownKeys exists on a non-proxy object. */
|
|
634
|
+
if (descriptor === undefined) continue
|
|
635
|
+
if ((!descriptor.enumerable && !(key === "message" && conflictMessage)) || !("value" in descriptor)) {
|
|
636
|
+
continue
|
|
637
|
+
}
|
|
638
|
+
// `message` and `stack` are observability text, never a field replay
|
|
639
|
+
// classifies on, and a git or HTTP failure embeds its remote URL,
|
|
640
|
+
// credentials included. They meet the journal's own rules; every
|
|
641
|
+
// other member, `_tag` included, stays byte-exact for replay.
|
|
642
|
+
const member = (key === "message" || key === "stack") && typeof descriptor.value === "string"
|
|
643
|
+
? Redaction.redact(descriptor.value)
|
|
644
|
+
: descriptor.value
|
|
645
|
+
const reduced = reduce(member, depth + 1)
|
|
646
|
+
if (reduced === inertJsonRejected) return inertJsonRejected
|
|
647
|
+
if (reduced === inertJsonOmitted) continue
|
|
648
|
+
Object.defineProperty(output, key, {
|
|
649
|
+
value: reduced,
|
|
650
|
+
enumerable: true,
|
|
651
|
+
configurable: true,
|
|
652
|
+
writable: true
|
|
653
|
+
})
|
|
654
|
+
}
|
|
655
|
+
return output
|
|
656
|
+
} finally {
|
|
657
|
+
active.delete(current)
|
|
658
|
+
}
|
|
659
|
+
}
|
|
660
|
+
}
|
|
661
|
+
/* v8 ignore next -- The typeof switch covers every JavaScript value category. */
|
|
662
|
+
return inertJsonRejected
|
|
663
|
+
}
|
|
664
|
+
|
|
665
|
+
try {
|
|
666
|
+
const reduced = reduce(value, 0)
|
|
667
|
+
return reduced === inertJsonRejected || reduced === inertJsonOmitted ? null : reduced
|
|
668
|
+
} catch {
|
|
669
|
+
return null
|
|
670
|
+
}
|
|
671
|
+
}
|
|
672
|
+
|
|
673
|
+
/**
|
|
674
|
+
* Encodes a live `Cause` into the plain tagged-reason JSON `rehydrateCause`
|
|
675
|
+
* decodes. Persisting the `Cause` object itself left the durable shape to
|
|
676
|
+
* whatever the ambient serializer produced (`Cause.toJSON` emits
|
|
677
|
+
* `{_id, failures}`, a structural walk emits `{reasons}`), so a change in
|
|
678
|
+
* the store's encoding silently broke failed-attempt replay. The write side
|
|
679
|
+
* now owns the shape explicitly. {@link inertJson} makes every live leaf
|
|
680
|
+
* inert and bounded before the store sees it, against one budget shared by
|
|
681
|
+
* the fixed envelope and every reason. The loop stops at the first reason that
|
|
682
|
+
* cannot fit and returns one `Die(null)` reason, which {@link rehydrateCause}
|
|
683
|
+
* decodes on replay, so even an adversarial reason count remains admissible.
|
|
684
|
+
*/
|
|
685
|
+
const persistCause = (cause: Cause.Cause<unknown>): typeof CauseJson.Type => {
|
|
686
|
+
// The cause object and its `reasons` array are the two fixed envelope nodes.
|
|
687
|
+
const budget: InertJsonBudget = { nodes: 2, characters: 0, enumeratedKeys: 0 }
|
|
688
|
+
const reasons: Array<typeof CauseJson.Type.reasons[number]> = []
|
|
689
|
+
for (const reason of cause.reasons) {
|
|
690
|
+
// One object, its `_tag` string, and its error, defect, or fiber-id value.
|
|
691
|
+
if (!spendNodes(budget, 3)) {
|
|
692
|
+
return { reasons: [{ _tag: "Die", defect: null }] }
|
|
693
|
+
}
|
|
694
|
+
if (Cause.isFailReason(reason)) {
|
|
695
|
+
reasons.push({ _tag: "Fail", error: inertJson(reason.error, budget) })
|
|
696
|
+
} else if (Cause.isDieReason(reason)) {
|
|
697
|
+
reasons.push({ _tag: "Die", defect: inertJson(reason.defect, budget) })
|
|
698
|
+
} else {
|
|
699
|
+
reasons.push({ _tag: "Interrupt", fiberId: reason.fiberId ?? null })
|
|
700
|
+
}
|
|
701
|
+
}
|
|
702
|
+
return { reasons }
|
|
703
|
+
}
|
|
704
|
+
|
|
705
|
+
/**
|
|
706
|
+
* Presents a payload this module already reduced to JSON as the attempt
|
|
707
|
+
* store's `JsonValue`.
|
|
708
|
+
*
|
|
709
|
+
* `@smthrs/run-store` types the `meta`, `error`, and `outcome` columns as
|
|
710
|
+
* `JsonValue` and admits every write through its own inert-JSON boundary,
|
|
711
|
+
* which refuses anything that boundary rejects with `AttemptStoreError`. The
|
|
712
|
+
* values handed here are already reduced: `AttemptMeta` is a schema whose
|
|
713
|
+
* optional fields are omitted rather than written as `undefined`,
|
|
714
|
+
* {@link persistCause} passes every leaf through {@link inertJson}, and an
|
|
715
|
+
* action `outcome` is the encoded result the caller's own codec produced.
|
|
716
|
+
* TypeScript still refuses the assignment because an optional property's type
|
|
717
|
+
* includes `undefined` and a JSON index signature does not, and because a
|
|
718
|
+
* schema-level `unknown` leaf cannot be proven JSON statically. The store's
|
|
719
|
+
* admission is the check; this is the type-level statement of what it checks.
|
|
720
|
+
*/
|
|
721
|
+
const attemptPayload = (value: unknown): AttemptStore.JsonValue => value as AttemptStore.JsonValue
|
|
722
|
+
|
|
723
|
+
/**
|
|
724
|
+
* Rebuilds the persisted failure of a `failed` attempt row so replay can
|
|
725
|
+
* rethrow the original domain error (issue #59). The row's `error` column
|
|
726
|
+
* holds the {@link persistCause} encoding of the failing `Cause` — a plain
|
|
727
|
+
* object whose `reasons` carry the tagged `Fail`/`Die`/`Interrupt` material.
|
|
728
|
+
* `Fail` errors are already schema-encoded by `Action.executeEncoded`, so
|
|
729
|
+
* their `_tag` survives and `RetryPolicy` non-retryable matching applies on
|
|
730
|
+
* replay exactly as it did on the live attempt. Live reasons are rebuilt
|
|
731
|
+
* unconditionally from the tagged material; anything unrecognizable becomes
|
|
732
|
+
* a defect carrying the raw persisted value.
|
|
733
|
+
*/
|
|
734
|
+
const rehydrateCause = (error: unknown): Cause.Cause<unknown> => {
|
|
735
|
+
const decoded = Schema.decodeUnknownResult(CauseJson)(error)
|
|
736
|
+
if (decoded._tag === "Success" && decoded.success.reasons.length > 0) {
|
|
737
|
+
return Cause.fromReasons(decoded.success.reasons.map((reason) => {
|
|
738
|
+
switch (reason._tag) {
|
|
739
|
+
case "Fail":
|
|
740
|
+
return Cause.makeFailReason(reason.error)
|
|
741
|
+
case "Die":
|
|
742
|
+
return Cause.makeDieReason(reason.defect)
|
|
743
|
+
case "Interrupt":
|
|
744
|
+
return Cause.makeInterruptReason(reason.fiberId ?? undefined)
|
|
745
|
+
}
|
|
746
|
+
}))
|
|
747
|
+
}
|
|
748
|
+
return Cause.die(error)
|
|
749
|
+
}
|
|
750
|
+
|
|
751
|
+
/**
|
|
752
|
+
* The {@link module:CacheEnvironment.CachePolicy} the dispatched declaration
|
|
753
|
+
* annotated, if any.
|
|
754
|
+
*
|
|
755
|
+
* The action stays opaque to this module — it is `unknown` on
|
|
756
|
+
* {@link ActionInput} on purpose — so the policy is read out of its annotation
|
|
757
|
+
* bag rather than off a field. An action without one, or a caller that hands
|
|
758
|
+
* the seam a bare object (every unit test of this module does), simply has no
|
|
759
|
+
* policy and keeps the pre-policy behavior.
|
|
760
|
+
*
|
|
761
|
+
* @since 0.1.0
|
|
762
|
+
* @private
|
|
763
|
+
*/
|
|
764
|
+
const declaredCachePolicy = (action: unknown): CacheEnvironment.CachePolicy | undefined => {
|
|
765
|
+
const annotations = (action as { readonly annotations?: unknown } | null | undefined)?.annotations
|
|
766
|
+
return Context.isContext(annotations) ? CacheEnvironment.cachePolicyOf(annotations) : undefined
|
|
767
|
+
}
|
|
768
|
+
|
|
769
|
+
/**
|
|
770
|
+
* Constructs the encoded action executor. The action itself stays opaque;
|
|
771
|
+
* the supplied dispatcher is the only physical execution point.
|
|
772
|
+
*
|
|
773
|
+
* @since 0.1.0
|
|
774
|
+
* @category constructors
|
|
775
|
+
*/
|
|
776
|
+
export const make = (deps: Dependencies) => {
|
|
777
|
+
/**
|
|
778
|
+
* Runs one action body, and tells whoever drove this dispatch that it ran.
|
|
779
|
+
*
|
|
780
|
+
* The report is the only way a node above this knows the difference between
|
|
781
|
+
* work and replay. Every execution path funnels through here, so a body
|
|
782
|
+
* that starts says so exactly once; a path that returns a durable record
|
|
783
|
+
* instead reports `replayed` at its own return. A dispatch that reports
|
|
784
|
+
* nothing is read as work, which is the safe reading: a node never claims
|
|
785
|
+
* it rebuilt nothing when it might have rebuilt something.
|
|
786
|
+
*
|
|
787
|
+
* The digest travels with the report because it is what joins the two
|
|
788
|
+
* halves of a node's evidence: the attempt records this dispatch writes are
|
|
789
|
+
* keyed by it, and the node record above has no other way to name them.
|
|
790
|
+
*/
|
|
791
|
+
const execute = (input: ActionInput, stepKeyDigest: string) =>
|
|
792
|
+
Effect.andThen(
|
|
793
|
+
Action.reportDispatch({ outcome: "executed", stepKeyDigest, attempt: input.attempt }),
|
|
794
|
+
deps.execute(input)
|
|
795
|
+
)
|
|
796
|
+
const admission = deps.admission ?? AttemptAdmission.makeUnsafe()
|
|
797
|
+
const recordedAgeVerdict = deps.cacheAgeVerdict ?? CacheAgeVerdicts.make(deps.runId)
|
|
798
|
+
// The lineage every record this executor writes addresses itself to.
|
|
799
|
+
// An action is a node inside its run's root lineage, not a lineage of
|
|
800
|
+
// its own: a lineage segment is minted only where a separate run is.
|
|
801
|
+
const lineageId = FlowEngine.Lineage.root(deps.runId)
|
|
802
|
+
return Effect.fn("ActionPersistence.execute")((input: ActionInput) => {
|
|
803
|
+
// What a replay-only engine reports for this dispatch: the digest once it
|
|
804
|
+
// exists, and whether the dispatch stopped at the execution gate.
|
|
805
|
+
const replay: { stepKeyDigest?: string; gated: boolean } = { gated: false }
|
|
806
|
+
return Effect.gen(function*() {
|
|
807
|
+
yield* Effect.annotateCurrentSpan({
|
|
808
|
+
runId: deps.runId,
|
|
809
|
+
attempt: input.attempt,
|
|
810
|
+
tier: input.tier
|
|
811
|
+
})
|
|
812
|
+
const attempts = yield* AttemptStore.AttemptStore
|
|
813
|
+
const cache = yield* CacheStore.CacheStore
|
|
814
|
+
const journal = yield* Journal.Journal
|
|
815
|
+
const runs = yield* RunStore.RunStore
|
|
816
|
+
const policy = declaredCachePolicy(input.action)
|
|
817
|
+
// SCOPE NARROWS THE CACHE ADDRESS. The engine derives a cross-run key
|
|
818
|
+
// from the inputs and the environment, which is the widest reach a
|
|
819
|
+
// content address can have. A declaration that wants a narrower one
|
|
820
|
+
// folds the identity it is narrowing to into the address of its cache
|
|
821
|
+
// row, so a sibling run derives a different address and never finds the
|
|
822
|
+
// row: narrowing by construction rather than by a read-time filter that
|
|
823
|
+
// a second reader could disagree with. `shared` is the unnarrowed
|
|
824
|
+
// default and folds nothing, so every key predating the policy keeps
|
|
825
|
+
// its bytes.
|
|
826
|
+
const scopeTag = policy?.scope === "run"
|
|
827
|
+
? `run:${deps.runId}`
|
|
828
|
+
: policy?.scope === "flow"
|
|
829
|
+
// The flow tag comes from the executing instance, which the engine
|
|
830
|
+
// provides around every dispatch. A caller that reaches this seam
|
|
831
|
+
// without one — the unit tests of this module — cannot name a flow, so
|
|
832
|
+
// the key narrows to the run instead: never wider than the caller
|
|
833
|
+
// asked for.
|
|
834
|
+
? `flow:${
|
|
835
|
+
Option.getOrUndefined(
|
|
836
|
+
Option.map(yield* Effect.serviceOption(FlowRuntime.FlowInstance), (instance) => instance.flow._tag)
|
|
837
|
+
) ?? `run:${deps.runId}`
|
|
838
|
+
}`
|
|
839
|
+
: undefined
|
|
840
|
+
// THE STEP KEY DIGEST IS NOT NARROWED. It is the identity the rest of the
|
|
841
|
+
// engine addresses this dispatch by: `flows_attempts` rows, which
|
|
842
|
+
// `EngineStore.actionRetryOrigin` and `actionLatestAttempt` probe under
|
|
843
|
+
// `sha256(input.key)`, and the `stepKeyDigest` on every attempt record,
|
|
844
|
+
// which `PlanScheduler` maps back to the node that dispatched it. Both
|
|
845
|
+
// derive the digest from the key alone and neither can see an
|
|
846
|
+
// annotation, so narrowing this one would hide a scoped step's attempts
|
|
847
|
+
// from its own retry counter and its deviations from the reconciler.
|
|
848
|
+
const stepKeyDigest = yield* Schema.decodeUnknownEffect(Sha256)(input.key).pipe(Effect.orDie)
|
|
849
|
+
replay.stepKeyDigest = stepKeyDigest
|
|
850
|
+
// What the scope narrows is the CACHE ROW ADDRESS, which is the only
|
|
851
|
+
// thing a scope is about: an unscoped declaration addresses the row by
|
|
852
|
+
// the step key alone, and a narrowed one folds the identity it is
|
|
853
|
+
// narrowing to into the address, so a sibling derives a different one and
|
|
854
|
+
// never finds the row. The two digests are the same string for every
|
|
855
|
+
// declaration that narrows nothing.
|
|
856
|
+
const keyDigest = scopeTag === undefined
|
|
857
|
+
? stepKeyDigest
|
|
858
|
+
: yield* Schema.decodeUnknownEffect(Sha256)(`${input.key}\u0000scope:${scopeTag}`).pipe(Effect.orDie)
|
|
859
|
+
yield* Effect.annotateCurrentSpan({ stepKeyDigest, keyDigest })
|
|
860
|
+
const attemptId = { runId: deps.runId, stepKeyDigest, attempt: input.attempt }
|
|
861
|
+
/**
|
|
862
|
+
* Lifecycle events take the journal's durable channel, fenced to the
|
|
863
|
+
* owning process: the write commits inside the SQL sink while
|
|
864
|
+
* `flows_runs` still records `deps.owner`, so a reclaimed (zombie) run
|
|
865
|
+
* fails with `fence_lost` — surfaced as self-interruption, matching the
|
|
866
|
+
* store-level fence outcomes — and a saturated lossy queue can never
|
|
867
|
+
* drop an attempt lifecycle record (issue #10).
|
|
868
|
+
*/
|
|
869
|
+
const emitLifecycle = (record: JournalEvent.Input) =>
|
|
870
|
+
journal.emitDurable(record, deps.owner).pipe(
|
|
871
|
+
Effect.catch((error) => error.code === "fence_lost" ? Effect.interrupt : Effect.fail(error))
|
|
872
|
+
)
|
|
873
|
+
const callFacts = CallFacts.make(input.action, deps.runId)
|
|
874
|
+
const stepFacts = StepFacts.make(input.action, deps.runId)
|
|
875
|
+
const emitStepSettled = (outcome: unknown) =>
|
|
876
|
+
Effect.flatMap(
|
|
877
|
+
stepFacts.settled(outcome),
|
|
878
|
+
(record) => record === undefined ? Effect.void : Effect.asVoid(emitLifecycle(record))
|
|
879
|
+
)
|
|
880
|
+
const emitCallInvoked = Effect.flatMap(
|
|
881
|
+
callFacts.invoked,
|
|
882
|
+
(record) => record === undefined ? Effect.void : Effect.asVoid(emitLifecycle(record))
|
|
883
|
+
)
|
|
884
|
+
const emitCallSettled = (outcome: unknown) =>
|
|
885
|
+
Effect.flatMap(
|
|
886
|
+
callFacts.settled(outcome),
|
|
887
|
+
(record) => record === undefined ? Effect.void : Effect.asVoid(emitLifecycle(record))
|
|
888
|
+
)
|
|
889
|
+
/**
|
|
890
|
+
* Commits an attempt/cache state transition and the lifecycle records
|
|
891
|
+
* describing it in ONE write transaction.
|
|
892
|
+
*
|
|
893
|
+
* The store writes below (`attempts.put`/`patch`/`finish`, `cache.put`)
|
|
894
|
+
* and `emitLifecycle` all go through the same `DurableWriter`, so the inner
|
|
895
|
+
* writes become savepoints of this transaction: either the row and its
|
|
896
|
+
* journal entry are both durable, or neither is. Before this, a crash in
|
|
897
|
+
* the interstitial left an attempt row the journal could not explain —
|
|
898
|
+
* `attemptStarted` with no `attemptFinished`, forever. Temporal's
|
|
899
|
+
* mutable state and history events commit as one persistence request
|
|
900
|
+
* (`reference/temporal/service/history/workflow/transaction_impl.go`);
|
|
901
|
+
* this is that unit of work, scoped to one attempt.
|
|
902
|
+
*
|
|
903
|
+
* Nothing that is not storage work belongs inside: the action body,
|
|
904
|
+
* the Jj snapshot, and the boundary prepare/settle all stay outside so
|
|
905
|
+
* the write transaction is never held across a host call.
|
|
906
|
+
*/
|
|
907
|
+
const atomically = <A, E, R>(effect: Effect.Effect<A, E, R>) => journal.transact(effect)
|
|
908
|
+
/**
|
|
909
|
+
* Commits an attempt's terminal row and the lifecycle records describing
|
|
910
|
+
* it as one unit, reporting whether the fenced write landed.
|
|
911
|
+
*
|
|
912
|
+
* Returning a boolean rather than interrupting inside keeps the
|
|
913
|
+
* transaction's failure paths to genuine storage failures: a lost fence
|
|
914
|
+
* is an ordinary "someone else owns this attempt now" outcome, and its
|
|
915
|
+
* self-interruption belongs to the caller, outside the transaction.
|
|
916
|
+
*/
|
|
917
|
+
const settleAttempt = (
|
|
918
|
+
row: AttemptStore.FinishAttempt,
|
|
919
|
+
records: ReadonlyArray<JournalEvent.Input>
|
|
920
|
+
) =>
|
|
921
|
+
atomically(Effect.gen(function*() {
|
|
922
|
+
const finished = yield* attempts.finish(row, deps.owner)
|
|
923
|
+
if (finished._tag !== "Finished") {
|
|
924
|
+
return false
|
|
925
|
+
}
|
|
926
|
+
// The checkpoint fact precedes its completion boundary so a rewind
|
|
927
|
+
// retaining that completion also retains the observation it owns.
|
|
928
|
+
if (row.state === "succeeded" && stepFacts.annotated) {
|
|
929
|
+
const committed = yield* attempts.get(attemptId)
|
|
930
|
+
if (Option.isNone(committed)) {
|
|
931
|
+
return yield* Effect.fail(
|
|
932
|
+
new AttemptStore.AttemptStoreError({
|
|
933
|
+
code: "persistence_failed",
|
|
934
|
+
method: "get",
|
|
935
|
+
message: "Finished checkpoint outcome disappeared",
|
|
936
|
+
cause: attemptId
|
|
937
|
+
})
|
|
938
|
+
)
|
|
939
|
+
}
|
|
940
|
+
yield* emitStepSettled(committed.value.outcome)
|
|
941
|
+
}
|
|
942
|
+
yield* Effect.forEach(records, (record) => emitLifecycle(record), { discard: true })
|
|
943
|
+
if (row.state === "succeeded" && callFacts.settles) {
|
|
944
|
+
const committed = yield* attempts.get(attemptId)
|
|
945
|
+
if (Option.isNone(committed)) {
|
|
946
|
+
return yield* Effect.die(new Error("Finished call outcome disappeared"))
|
|
947
|
+
}
|
|
948
|
+
yield* emitCallSettled(committed.value.outcome)
|
|
949
|
+
}
|
|
950
|
+
return true
|
|
951
|
+
}))
|
|
952
|
+
const fencedAtMs = yield* Clock.currentTimeMillis.pipe(Effect.map(Math.floor))
|
|
953
|
+
const heartbeat = yield* runs.heartbeat(deps.runId, deps.owner, fencedAtMs)
|
|
954
|
+
if (heartbeat._tag !== "Updated") {
|
|
955
|
+
return yield* Effect.interrupt
|
|
956
|
+
}
|
|
957
|
+
|
|
958
|
+
if (input.tier === "irreversible" && input.attempt > 1 && deps.idempotencyKey === undefined) {
|
|
959
|
+
return yield* Effect.fail(
|
|
960
|
+
new Action.IrreversibleRetryRequiresIdempotencyKey({
|
|
961
|
+
actionName: actionKind(input.action),
|
|
962
|
+
attempt: input.attempt
|
|
963
|
+
})
|
|
964
|
+
)
|
|
965
|
+
}
|
|
966
|
+
|
|
967
|
+
// A source glob is cacheable only after the scheduler pins and replaces
|
|
968
|
+
// it with exact measured inputs. Direct actions retain the pattern, so
|
|
969
|
+
// their key cannot prove which expansion it names and must stay local.
|
|
970
|
+
const cacheDeclaration = CacheAdmission.declaration(input)
|
|
971
|
+
const declarationMeta = {
|
|
972
|
+
tier: input.tier,
|
|
973
|
+
...(deps.idempotencyKey === undefined ? {} : { keyed: true as const }),
|
|
974
|
+
...(input.nondeterministic === undefined ? {} : { nondeterministic: input.nondeterministic })
|
|
975
|
+
} satisfies AttemptMeta
|
|
976
|
+
|
|
977
|
+
/**
|
|
978
|
+
* Cache-provenance producer identity (issue #124): the plain
|
|
979
|
+
* `{runId, sourceId}` identity carried no `sourceSeq`, so every re-drive of
|
|
980
|
+
* the same observation — the same replay refusal against the same
|
|
981
|
+
* recorded row, on every later dispatch of the key — appended a fresh
|
|
982
|
+
* journal row forever. A per-(key, action, recorded-provenance)
|
|
983
|
+
* identity with `sourceSeq 0` collapses the identical re-observation
|
|
984
|
+
* into a `Duplicate`, while folding the recorded row's provenance into
|
|
985
|
+
* the identity keeps a genuinely new observation — the same action
|
|
986
|
+
* against a *different* recorded row — a distinct producer that still
|
|
987
|
+
* journals.
|
|
988
|
+
*
|
|
989
|
+
* The identity names the run, the step-key digest, the decision, and the
|
|
990
|
+
* recorded row's provenance — everything the decision is ABOUT — and
|
|
991
|
+
* deliberately nothing about WHO TOOK IT. `deps.sourceId` is the host's
|
|
992
|
+
* configured `journalSource` (`EngineStore.Options.journalSource`), which
|
|
993
|
+
* a run outlives: the process that admits a row at 900 ms is routinely
|
|
994
|
+
* not the process that resumes the run at 1100 ms. Folding it in made the
|
|
995
|
+
* `ttl` verdict below re-decidable — the resuming incarnation got
|
|
996
|
+
* `Accepted` for a fresh verdict under its own identity and re-judged the
|
|
997
|
+
* age against its own clock, which is the one thing a recorded verdict
|
|
998
|
+
* exists to prevent — and made every other record here re-appendable once
|
|
999
|
+
* per incarnation rather than once per observation.
|
|
1000
|
+
*
|
|
1001
|
+
* The sibling `recorded` identity below still carries `deps.sourceId`, on
|
|
1002
|
+
* purpose: nothing reads a decision back out of it. It exists so a
|
|
1003
|
+
* convergence re-record collapses into a `Duplicate`, and a resume under
|
|
1004
|
+
* another journal source costs one extra provenance row there, never a
|
|
1005
|
+
* different answer.
|
|
1006
|
+
*/
|
|
1007
|
+
const cacheSource = (
|
|
1008
|
+
action: string,
|
|
1009
|
+
recorded?: { readonly runId: string; readonly eventSeq: number }
|
|
1010
|
+
): JournalRecords.EventOptions => ({
|
|
1011
|
+
runId: deps.runId,
|
|
1012
|
+
lineageId,
|
|
1013
|
+
sourceId: `cache:${keyDigest}:${action}${
|
|
1014
|
+
recorded === undefined ? "" : `:${recorded.runId}:${recorded.eventSeq}`
|
|
1015
|
+
}`,
|
|
1016
|
+
sourceSeq: 0
|
|
1017
|
+
})
|
|
1018
|
+
const noteUnshareable = (unshareable: CachePublication.Unshareable) =>
|
|
1019
|
+
Effect.gen(function*() {
|
|
1020
|
+
const reason = yield* Schema.decodeUnknownEffect(Sha256)(`${unshareable.stage}:${unshareable.message}`).pipe(
|
|
1021
|
+
Effect.orDie
|
|
1022
|
+
)
|
|
1023
|
+
yield* emitLifecycle(JournalRecords.cacheProvenance(cacheSource(`unpublished:${reason}`), {
|
|
1024
|
+
keyDigest,
|
|
1025
|
+
action: "unpublished",
|
|
1026
|
+
stage: unshareable.stage,
|
|
1027
|
+
message: unshareable.message
|
|
1028
|
+
}))
|
|
1029
|
+
})
|
|
1030
|
+
/**
|
|
1031
|
+
* Records the sealed completion into the shared cache with provenance,
|
|
1032
|
+
* failing (by default) on a divergent first-recorded row. Used by both
|
|
1033
|
+
* the fresh completion path and succeeded-attempt replay, so a crash
|
|
1034
|
+
* between `attempts.finish` and `cache.put` converges on restart
|
|
1035
|
+
* instead of leaving the cache permanently behind the journal.
|
|
1036
|
+
*/
|
|
1037
|
+
const recordCache = (options: {
|
|
1038
|
+
readonly result: unknown
|
|
1039
|
+
readonly admission: CacheAdmission.PublishCompletion<AttemptMeta>
|
|
1040
|
+
readonly createdAtMs: number
|
|
1041
|
+
}) =>
|
|
1042
|
+
Effect.gen(function*() {
|
|
1043
|
+
// BLOBS BEFORE METADATA (issue #172). Every artifact this evidence
|
|
1044
|
+
// references is made durable in the shared tier *before* the entry
|
|
1045
|
+
// that references it is written, so a sibling machine can never see
|
|
1046
|
+
// a hit whose outputs it cannot materialize. Bazel's REAPI ordering
|
|
1047
|
+
// constraint, stated at `UploadManifest.java:630-633`; the whole
|
|
1048
|
+
// protocol lives in `CachePublication`, which is a no-op when no
|
|
1049
|
+
// shared tier is configured. It runs OUTSIDE the write transaction
|
|
1050
|
+
// below, like every other host call.
|
|
1051
|
+
//
|
|
1052
|
+
// A refusal withholds the SHARED entry, never the local row and never
|
|
1053
|
+
// the run: this point is reached after `attempts.finish`, so the work
|
|
1054
|
+
// is already done and durably recorded, and failing here would throw
|
|
1055
|
+
// a real result away because an optional accelerator was unreachable.
|
|
1056
|
+
// It is journalled below instead — the same "visible, not silent"
|
|
1057
|
+
// treatment an unverified read set gets (issue #106).
|
|
1058
|
+
const meta = options.admission.meta
|
|
1059
|
+
let unshareable = yield* CachePublication.publishArtifacts(meta.boundary)
|
|
1060
|
+
// The producer identity folds a digest of the recorded content
|
|
1061
|
+
// (issue #129): a constant per-key identity made a post-eviction
|
|
1062
|
+
// re-record collapse into a `Duplicate` carrying the EVICTED
|
|
1063
|
+
// generation's seq, so the fresh row inherited the evicted row's
|
|
1064
|
+
// exact provenance and a laggard's #119 `ifRecordedBy` fence could
|
|
1065
|
+
// delete the valid new row. With the content folded in, the #124
|
|
1066
|
+
// convergence re-record (identical content, rebuilt from the same
|
|
1067
|
+
// persisted attempt row, so the serialization is byte-stable)
|
|
1068
|
+
// still collapses into a `Duplicate` whose receipt carries the
|
|
1069
|
+
// original emission's canonical seq, while a re-record with
|
|
1070
|
+
// different content is a distinct producer that journals fresh
|
|
1071
|
+
// provenance. A re-record whose content happens to equal the
|
|
1072
|
+
// evicted generation's shares its provenance by design: the two
|
|
1073
|
+
// rows are indistinguishable, so a fenced evict of one is exactly
|
|
1074
|
+
// a fenced evict of the other.
|
|
1075
|
+
//
|
|
1076
|
+
// The digest goes through `DerivedKey`, the repo's one hashing
|
|
1077
|
+
// chokepoint: RFC 8785 canonical JSON, then SHA-256. Hashing
|
|
1078
|
+
// `JSON.stringify` output made "byte stable" depend on key order,
|
|
1079
|
+
// and the two paths do not build `meta` the same way — the fresh
|
|
1080
|
+
// path spreads an object, the convergence path decodes through
|
|
1081
|
+
// `Schema.decodeUnknownEffect(AttemptMeta)`, which emits keys in
|
|
1082
|
+
// schema declaration order. Today the two orders coincide, so the
|
|
1083
|
+
// break was latent; adding one optional field to `AttemptMeta` or
|
|
1084
|
+
// `BoundaryEvidence` above an existing one, or reordering the spread
|
|
1085
|
+
// at line 1405, would have made the convergence re-record compute a
|
|
1086
|
+
// different `generation`, append fresh provenance on every later
|
|
1087
|
+
// dispatch, and reopen the unbounded-append regression issue #124
|
|
1088
|
+
// closed. Canonical JSON makes the stability structural.
|
|
1089
|
+
const generation = yield* Schema.decodeUnknownEffect(DerivedKey)({
|
|
1090
|
+
kind: "cache-generation",
|
|
1091
|
+
meta,
|
|
1092
|
+
result: options.result
|
|
1093
|
+
}).pipe(Effect.orDie)
|
|
1094
|
+
// The provenance record and the row it describes commit together:
|
|
1095
|
+
// the row carries the record's canonical seq as its provenance, so a
|
|
1096
|
+
// crash between them left either a row pointing at a sequence that
|
|
1097
|
+
// does not exist or a `recorded` entry for a row nobody wrote.
|
|
1098
|
+
// Losing the first-writer race is not that crash — it is a decision,
|
|
1099
|
+
// and it stays journalled: the `recorded` entry says what this run
|
|
1100
|
+
// tried to record and the `cache-conflict` entry below says how it
|
|
1101
|
+
// resolved.
|
|
1102
|
+
const recording = yield* atomically(
|
|
1103
|
+
Effect.gen(function*() {
|
|
1104
|
+
const receipt = yield* emitLifecycle(
|
|
1105
|
+
JournalRecords.cacheProvenance({
|
|
1106
|
+
runId: deps.runId,
|
|
1107
|
+
lineageId,
|
|
1108
|
+
cacheKey: keyDigest,
|
|
1109
|
+
sourceId: `${deps.sourceId}:cache:${keyDigest}:recorded:${generation}`,
|
|
1110
|
+
sourceSeq: 0
|
|
1111
|
+
}, { keyDigest, action: "recorded" })
|
|
1112
|
+
)
|
|
1113
|
+
const entry = {
|
|
1114
|
+
keyDigest,
|
|
1115
|
+
result: options.result,
|
|
1116
|
+
meta,
|
|
1117
|
+
createdAtMs: options.createdAtMs,
|
|
1118
|
+
recordedRunId: deps.runId,
|
|
1119
|
+
recordedEventSeq: receipt.seq
|
|
1120
|
+
}
|
|
1121
|
+
const outcome = yield* cache.put(entry)
|
|
1122
|
+
// Only an inserting put creates a head, so the insertion is
|
|
1123
|
+
// journalled in the same transaction. `StepCacheFold` rebuilds the
|
|
1124
|
+
// heads from these entries and never from `recorded`, which a
|
|
1125
|
+
// duplicate or a losing conflict also writes (issue #2053).
|
|
1126
|
+
if (outcome._tag === "Inserted") {
|
|
1127
|
+
const recorded = { runId: deps.runId, eventSeq: receipt.seq }
|
|
1128
|
+
yield* emitLifecycle(JournalRecords.cacheProvenance(cacheSource("admitted", recorded), {
|
|
1129
|
+
keyDigest,
|
|
1130
|
+
action: "admitted",
|
|
1131
|
+
recordedRunId: recorded.runId,
|
|
1132
|
+
recordedEventSeq: recorded.eventSeq
|
|
1133
|
+
}))
|
|
1134
|
+
}
|
|
1135
|
+
return { entry, outcome }
|
|
1136
|
+
})
|
|
1137
|
+
).pipe(Effect.catchTag("@smthrs/step-cache/CacheStoreError", (error) =>
|
|
1138
|
+
// Only availability of the local transaction is optional. A known
|
|
1139
|
+
// deterministic Conflict below still belongs to Inconsistency.
|
|
1140
|
+
noteUnshareable({ stage: "entry", message: `local cache publication failed: ${error.message}` }).pipe(
|
|
1141
|
+
Effect.as(undefined)
|
|
1142
|
+
)))
|
|
1143
|
+
if (recording === undefined) {
|
|
1144
|
+
return
|
|
1145
|
+
}
|
|
1146
|
+
// ENTRY LAST, AND OUTSIDE THE TRANSACTION. The local row and its
|
|
1147
|
+
// provenance record are now durable together; only here does the
|
|
1148
|
+
// entry become observable to other machines, which is the second half
|
|
1149
|
+
// of the REAPI ordering constraint. It is deliberately not inside the
|
|
1150
|
+
// transaction above: `CacheSync` speaks HTTP, and nothing that is not
|
|
1151
|
+
// storage work may be held across a `DurableWriter` write — a stalled
|
|
1152
|
+
// shared cache would block every other writer and roll back a row
|
|
1153
|
+
// that has nothing to do with it. A `Conflict` skips publication for
|
|
1154
|
+
// the reason the local tier reported it: this machine does not agree
|
|
1155
|
+
// with itself about the key yet.
|
|
1156
|
+
if (unshareable === undefined && recording.outcome._tag !== "Conflict") {
|
|
1157
|
+
unshareable = yield* CachePublication.publishEntry(recording.entry)
|
|
1158
|
+
}
|
|
1159
|
+
if (unshareable !== undefined) {
|
|
1160
|
+
// The refusal is journalled so a missing shared entry is
|
|
1161
|
+
// explainable, never inferred from its absence. The identity folds
|
|
1162
|
+
// a digest of the reason for the same reason the `recorded` record
|
|
1163
|
+
// above folds one of its content (issue #129): a convergence
|
|
1164
|
+
// re-record hitting the identical refusal is an exact producer
|
|
1165
|
+
// retry the journal collapses into a `Duplicate`, while a
|
|
1166
|
+
// *different* refusal is a genuinely new observation that journals
|
|
1167
|
+
// fresh — and neither can ever be the same identity carrying
|
|
1168
|
+
// different content, which is what an idempotency conflict is.
|
|
1169
|
+
yield* noteUnshareable(unshareable)
|
|
1170
|
+
}
|
|
1171
|
+
if (recording.outcome._tag === "Conflict") {
|
|
1172
|
+
const conflicting = yield* cache.get(keyDigest)
|
|
1173
|
+
if (meta.nondeterministic === true) {
|
|
1174
|
+
const recorded = Option.map(conflicting, (entry) => ({
|
|
1175
|
+
runId: entry.recordedRunId,
|
|
1176
|
+
eventSeq: entry.recordedEventSeq
|
|
1177
|
+
}))
|
|
1178
|
+
// Declared output nondeterminism is not a hermeticity violation,
|
|
1179
|
+
// so it does not enter the Inconsistency receiver. The key folds
|
|
1180
|
+
// the declaration, making first-writer-wins safe for both sides.
|
|
1181
|
+
yield* emitLifecycle(
|
|
1182
|
+
JournalRecords.cacheProvenance(
|
|
1183
|
+
cacheSource("conflict_first_writer", Option.getOrUndefined(recorded)),
|
|
1184
|
+
{
|
|
1185
|
+
keyDigest,
|
|
1186
|
+
action: "conflict_first_writer",
|
|
1187
|
+
recordedRunId: Option.getOrNull(Option.map(recorded, (value) => value.runId)),
|
|
1188
|
+
recordedEventSeq: Option.getOrNull(Option.map(recorded, (value) => value.eventSeq))
|
|
1189
|
+
}
|
|
1190
|
+
)
|
|
1191
|
+
)
|
|
1192
|
+
return
|
|
1193
|
+
}
|
|
1194
|
+
const receiverOption = yield* Effect.serviceOption(Inconsistency.Inconsistency)
|
|
1195
|
+
// Core default is STRICT: journal the conflict and fail the run,
|
|
1196
|
+
// which is Skyframe's throwing `GraphInconsistencyReceiver`.
|
|
1197
|
+
// Providing `Inconsistency.layerTolerant` opts out.
|
|
1198
|
+
const receiver = Option.isSome(receiverOption)
|
|
1199
|
+
? receiverOption.value
|
|
1200
|
+
: Inconsistency.make({ journal, verdict: "fail", owner: deps.owner })
|
|
1201
|
+
const verdict = yield* receiver.note({
|
|
1202
|
+
key: keyDigest,
|
|
1203
|
+
existing: Option.getOrUndefined(conflicting),
|
|
1204
|
+
attempted: recording.entry
|
|
1205
|
+
})
|
|
1206
|
+
if (verdict === "fail") {
|
|
1207
|
+
return yield* Effect.fail(
|
|
1208
|
+
new CacheConflictDetected({
|
|
1209
|
+
code: "cache_conflict_detected",
|
|
1210
|
+
keyDigest,
|
|
1211
|
+
recordedRunId: Option.isSome(conflicting) ? conflicting.value.recordedRunId : "unknown"
|
|
1212
|
+
})
|
|
1213
|
+
)
|
|
1214
|
+
}
|
|
1215
|
+
}
|
|
1216
|
+
})
|
|
1217
|
+
// Everything from the cache-hit verification to the terminal transition
|
|
1218
|
+
// runs under this process's exclusive permit for the attempt key
|
|
1219
|
+
// (issues #102, #103, #118): the adoption decision below is taken from
|
|
1220
|
+
// a read no in-process racer can invalidate before the claim lands, a
|
|
1221
|
+
// concurrent same-key dispatch waits here until the winner's terminal
|
|
1222
|
+
// row is visible and replays it instead of re-executing the body, and
|
|
1223
|
+
// the cache block's read-verify-materialize-evict span can never
|
|
1224
|
+
// interleave with another dispatch's execution. A verified hit returns
|
|
1225
|
+
// from inside the permit. The permit is keyed by (runId, stepKeyDigest)
|
|
1226
|
+
// alone (issue #133): the cache row is addressed with no attempt
|
|
1227
|
+
// material, so folding the attempt counter in let sanctioned keyed
|
|
1228
|
+
// dispatches at skewed attempt counters (#111/#116) acquire different
|
|
1229
|
+
// permits and interleave the very span the permit serializes. The step
|
|
1230
|
+
// key, not the narrowed cache address, is what serializes here, because
|
|
1231
|
+
// the attempt rows two scopes of one key share are what they race over.
|
|
1232
|
+
return yield* admission.withPermit(`${deps.runId}|${stepKeyDigest}`)(
|
|
1233
|
+
Effect.gen(function*() {
|
|
1234
|
+
// Lifecycle announcements take a per-attempt producer identity
|
|
1235
|
+
// (issue #91): adoption — or a replay after a crash in the
|
|
1236
|
+
// finish→emit window (issue #109) — re-emits records a dead
|
|
1237
|
+
// incarnation may already have announced, and lifecycle records
|
|
1238
|
+
// without a `sourceSeq` allocate a fresh journal row on every
|
|
1239
|
+
// emission. A dedicated `(sourceId, sourceSeq 0)` per record makes
|
|
1240
|
+
// the re-emission an exact producer retry the journal collapses
|
|
1241
|
+
// into a `Duplicate`.
|
|
1242
|
+
const attemptSource = (record: string): JournalRecords.EventOptions => ({
|
|
1243
|
+
runId: deps.runId,
|
|
1244
|
+
lineageId,
|
|
1245
|
+
// A sealed dispatch's result lives in the step cache, so the record
|
|
1246
|
+
// carries the digest that addresses it: replay hands the projection
|
|
1247
|
+
// the sealed value instead of re-deriving it, and a cache miss is
|
|
1248
|
+
// simply an absent value rather than a broken fold.
|
|
1249
|
+
...(input.tier === "sealed" ? { cacheKey: keyDigest } : {}),
|
|
1250
|
+
sourceId: `${deps.sourceId}:attempt:${stepKeyDigest}:${input.attempt}:${record}`,
|
|
1251
|
+
sourceSeq: 0
|
|
1252
|
+
})
|
|
1253
|
+
/**
|
|
1254
|
+
* Journal-convergence emit for the replay branches (issue #109):
|
|
1255
|
+
* an identical re-emission collapses into a `Duplicate`, and an
|
|
1256
|
+
* `idempotency_conflict` may mean the journal already holds this
|
|
1257
|
+
* record under this producer identity as another lineage recorded
|
|
1258
|
+
* it — a time-travel fork copies the parent's journal rows, so the
|
|
1259
|
+
* copied record names the parent run. A cache-provenance record
|
|
1260
|
+
* may also restate a fact this run already journalled with fresh
|
|
1261
|
+
* measurements (an expiry re-measured after a resume). Those are
|
|
1262
|
+
* the only conflicts this emit tolerates, and only after reading
|
|
1263
|
+
* the occupying record and validating it: an attempt-scoped record
|
|
1264
|
+
* must be this record as a retained fork ancestor emitted it — same
|
|
1265
|
+
* event type, attempt coordinates and terminal state
|
|
1266
|
+
* (`CopiedRecord.accept`); a provenance record must state the same
|
|
1267
|
+
* fact about the same row (`ProvenanceSlot.accept`). An unrelated
|
|
1268
|
+
* or contradictory record in the slot — corrupt or imported
|
|
1269
|
+
* history, a producer collision — surfaces the journal's own
|
|
1270
|
+
* conflict instead of a replay that reports no inconsistency.
|
|
1271
|
+
*/
|
|
1272
|
+
const emitConverging = (record: JournalEvent.Input) =>
|
|
1273
|
+
emitLifecycle(record).pipe(
|
|
1274
|
+
Effect.catch((error) =>
|
|
1275
|
+
error.code !== "idempotency_conflict"
|
|
1276
|
+
? Effect.fail(error)
|
|
1277
|
+
: record.eventType === "flows.engine.cache-provenance"
|
|
1278
|
+
? ProvenanceSlot.accept({ journal, runId: deps.runId, record, conflict: error })
|
|
1279
|
+
: CopiedRecord.accept({ journal, runs, runId: deps.runId, record, conflict: error })
|
|
1280
|
+
)
|
|
1281
|
+
)
|
|
1282
|
+
/**
|
|
1283
|
+
* Records and clears a row the declared time to live has aged out.
|
|
1284
|
+
*
|
|
1285
|
+
* The age judgement is journalled BEFORE the row is dropped, and
|
|
1286
|
+
* with the age and the bound in the payload, so a reader can tell an
|
|
1287
|
+
* expiry from a plain miss and a replay reads the recorded verdict
|
|
1288
|
+
* instead of re-judging the age against a fresh clock. The eviction
|
|
1289
|
+
* is what keeps `CacheStore.put`'s insert-or-nothing head from
|
|
1290
|
+
* turning the re-execution's genuinely newer result into a
|
|
1291
|
+
* `Conflict` — the same repair the stale-read-set branch performs
|
|
1292
|
+
* (issue #99) — and it is fenced on the expired row's own provenance
|
|
1293
|
+
* (issue #119) so a fresh row another process landed in the meantime
|
|
1294
|
+
* survives.
|
|
1295
|
+
*/
|
|
1296
|
+
const expireRow = (ttlMs: number, stale: CacheStore.CacheEntry) =>
|
|
1297
|
+
Effect.gen(function*() {
|
|
1298
|
+
const recorded = {
|
|
1299
|
+
runId: stale.recordedRunId,
|
|
1300
|
+
eventSeq: stale.recordedEventSeq
|
|
1301
|
+
}
|
|
1302
|
+
const nowMs = yield* Clock.currentTimeMillis.pipe(Effect.map(Math.floor))
|
|
1303
|
+
yield* emitConverging(
|
|
1304
|
+
JournalRecords.cacheProvenance(cacheSource("expired", recorded), {
|
|
1305
|
+
keyDigest,
|
|
1306
|
+
action: "expired",
|
|
1307
|
+
ttlMs,
|
|
1308
|
+
ageMs: nowMs - stale.createdAtMs,
|
|
1309
|
+
recordedRunId: recorded.runId,
|
|
1310
|
+
recordedEventSeq: recorded.eventSeq
|
|
1311
|
+
})
|
|
1312
|
+
)
|
|
1313
|
+
yield* cache.evict(keyDigest, { ifRecordedBy: recorded })
|
|
1314
|
+
})
|
|
1315
|
+
/**
|
|
1316
|
+
* Decides — once, durably — whether a row is inside the declared time
|
|
1317
|
+
* to live, and answers the DECISION THIS RUN RECORDED rather than the
|
|
1318
|
+
* one this clock reading would make.
|
|
1319
|
+
*
|
|
1320
|
+
* A time to live is the one cache input that changes answer on its
|
|
1321
|
+
* own. A dispatch that served a row at 900 ms and then lost its
|
|
1322
|
+
* process would, on the plain reading, re-dispatch at 1100 ms, expire
|
|
1323
|
+
* the row it already served, and execute a body whose result the run
|
|
1324
|
+
* had already consumed. The verdict is therefore journalled before it
|
|
1325
|
+
* is acted on, under a producer identity that names the step key, the
|
|
1326
|
+
* `ttl` decision, and the row's own recorded provenance
|
|
1327
|
+
* (`cacheSource`), with `sourceSeq` 0. That identity belongs to the
|
|
1328
|
+
* RUN, not to the incarnation driving it, so the engine that resumes
|
|
1329
|
+
* the run reads the same verdict rather than taking a second one.
|
|
1330
|
+
*
|
|
1331
|
+
* `Accepted` records the first decision; `Duplicate` confirms it.
|
|
1332
|
+
* A conflict alone does not identify the recorded verdict: TTL is
|
|
1333
|
+
* not part of every action key, and copied history can carry other
|
|
1334
|
+
* lineage metadata. Only an exact duplicate of the opposite verdict
|
|
1335
|
+
* proves a clock change. Otherwise fail before serving or expiring
|
|
1336
|
+
* the row, preserving the incompatible history for the caller.
|
|
1337
|
+
*
|
|
1338
|
+
* A row re-recorded under a provenance this run already expired keeps
|
|
1339
|
+
* that verdict, which is the same rule read from the other side.
|
|
1340
|
+
*/
|
|
1341
|
+
const admitByAge = (ttlMs: number, row: CacheStore.CacheEntry) =>
|
|
1342
|
+
Effect.gen(function*() {
|
|
1343
|
+
const recorded = { runId: row.recordedRunId, eventSeq: row.recordedEventSeq }
|
|
1344
|
+
const nowMs = yield* Clock.currentTimeMillis.pipe(Effect.map(Math.floor))
|
|
1345
|
+
const measured: "admitted" | "expired" = nowMs - row.createdAtMs <= ttlMs ? "admitted" : "expired"
|
|
1346
|
+
const decision = (verdict: "admitted" | "expired") =>
|
|
1347
|
+
JournalRecords.cacheProvenance(cacheSource("ttl", recorded), {
|
|
1348
|
+
keyDigest,
|
|
1349
|
+
action: "ttl",
|
|
1350
|
+
ttlMs,
|
|
1351
|
+
verdict,
|
|
1352
|
+
recordedRunId: recorded.runId,
|
|
1353
|
+
recordedEventSeq: recorded.eventSeq
|
|
1354
|
+
})
|
|
1355
|
+
// A copied producer whose source fields changed must not look
|
|
1356
|
+
// like a new first decision merely because emit would accept it.
|
|
1357
|
+
const expectedSource = cacheSource("ttl", recorded).sourceId
|
|
1358
|
+
const prior = yield* CacheAgeHistory.find(journal, deps.runId, (entry) => {
|
|
1359
|
+
const payload = entry.payload as {
|
|
1360
|
+
action?: unknown
|
|
1361
|
+
keyDigest?: unknown
|
|
1362
|
+
recordedRunId?: unknown
|
|
1363
|
+
recordedEventSeq?: unknown
|
|
1364
|
+
} | null
|
|
1365
|
+
return entry.sourceId === expectedSource ||
|
|
1366
|
+
(payload?.action === "ttl" && payload.keyDigest === keyDigest &&
|
|
1367
|
+
payload.recordedRunId === recorded.runId && payload.recordedEventSeq === recorded.eventSeq)
|
|
1368
|
+
})
|
|
1369
|
+
if (prior !== undefined && (prior.sourceId !== expectedSource || prior.sourceSeq !== 0)) {
|
|
1370
|
+
yield* emitConverging(JournalRecords.cacheProvenance(cacheSource("replay_failed", recorded), {
|
|
1371
|
+
keyDigest,
|
|
1372
|
+
action: "replay_failed",
|
|
1373
|
+
reason: "incompatible-age-history",
|
|
1374
|
+
recordedRunId: recorded.runId,
|
|
1375
|
+
recordedEventSeq: recorded.eventSeq
|
|
1376
|
+
}))
|
|
1377
|
+
return yield* Effect.fail(
|
|
1378
|
+
new Journal.JournalError({
|
|
1379
|
+
code: "idempotency_conflict",
|
|
1380
|
+
message: "incompatible recorded cache-age decision: producer identity changed",
|
|
1381
|
+
cause: prior
|
|
1382
|
+
})
|
|
1383
|
+
)
|
|
1384
|
+
}
|
|
1385
|
+
return yield* emitLifecycle(decision(measured)).pipe(
|
|
1386
|
+
Effect.as(measured),
|
|
1387
|
+
Effect.catch((error) => {
|
|
1388
|
+
if (error.code !== "idempotency_conflict") {
|
|
1389
|
+
return Effect.fail(error)
|
|
1390
|
+
}
|
|
1391
|
+
const conflict = new Journal.JournalError({
|
|
1392
|
+
code: "idempotency_conflict",
|
|
1393
|
+
message:
|
|
1394
|
+
"incompatible recorded cache-age decision: restore the original TTL and history identity " +
|
|
1395
|
+
"or use a new action identity; the cache row was not served or expired",
|
|
1396
|
+
cause: error
|
|
1397
|
+
})
|
|
1398
|
+
const opposite = measured === "admitted" ? "expired" : "admitted"
|
|
1399
|
+
return emitLifecycle(decision(opposite)).pipe(
|
|
1400
|
+
Effect.catch((oppositeError) =>
|
|
1401
|
+
oppositeError.code === "idempotency_conflict"
|
|
1402
|
+
? CacheAgeHistory.copiedVerdict({
|
|
1403
|
+
journal,
|
|
1404
|
+
runs,
|
|
1405
|
+
runId: deps.runId,
|
|
1406
|
+
decision: decision(measured),
|
|
1407
|
+
conflict
|
|
1408
|
+
}).pipe(
|
|
1409
|
+
Effect.tapError((failure) =>
|
|
1410
|
+
failure.code === "idempotency_conflict"
|
|
1411
|
+
? emitConverging(JournalRecords.cacheProvenance(cacheSource("replay_failed", recorded), {
|
|
1412
|
+
keyDigest,
|
|
1413
|
+
action: "replay_failed",
|
|
1414
|
+
reason: "incompatible-age-history",
|
|
1415
|
+
recordedRunId: recorded.runId,
|
|
1416
|
+
recordedEventSeq: recorded.eventSeq
|
|
1417
|
+
})).pipe(Effect.asVoid)
|
|
1418
|
+
: Effect.void
|
|
1419
|
+
),
|
|
1420
|
+
Effect.map((verdict) => ({ _tag: "Historical" as const, verdict }))
|
|
1421
|
+
)
|
|
1422
|
+
: Effect.fail(oppositeError)
|
|
1423
|
+
),
|
|
1424
|
+
Effect.flatMap((receipt) =>
|
|
1425
|
+
receipt._tag === "Historical" ?
|
|
1426
|
+
Effect.succeed(receipt.verdict) :
|
|
1427
|
+
receipt._tag === "Duplicate"
|
|
1428
|
+
? Effect.succeed(opposite)
|
|
1429
|
+
: Effect.fail(conflict)
|
|
1430
|
+
)
|
|
1431
|
+
)
|
|
1432
|
+
})
|
|
1433
|
+
)
|
|
1434
|
+
})
|
|
1435
|
+
/**
|
|
1436
|
+
* The row this dispatch may serve, after the declared age bound.
|
|
1437
|
+
*
|
|
1438
|
+
* The bound is applied here rather than inside `cache.get` so the
|
|
1439
|
+
* refusal is a decision this run journalled and owns, not a read
|
|
1440
|
+
* policy the store re-derives from a fresh clock on every lookup.
|
|
1441
|
+
* The no-TTL guard refreshes a per-executor verdict index through
|
|
1442
|
+
* the journal's current tail before answering absence. Its cursor
|
|
1443
|
+
* covers only fully processed pages in the same journal generation;
|
|
1444
|
+
* resume or a changed history rebuilds it once, never per dispatch.
|
|
1445
|
+
*/
|
|
1446
|
+
const admissible = (
|
|
1447
|
+
ttlMs: number | undefined,
|
|
1448
|
+
row: Option.Option<CacheStore.CacheEntry>
|
|
1449
|
+
): Effect.Effect<Option.Option<CacheStore.CacheEntry>, Journal.JournalError | CacheStore.CacheStoreError> =>
|
|
1450
|
+
Effect.gen(function*() {
|
|
1451
|
+
// Removing TTL cannot bypass a verdict this run already consumed,
|
|
1452
|
+
// even if the head was subsequently evicted or replaced.
|
|
1453
|
+
if (ttlMs === undefined) {
|
|
1454
|
+
const recorded = yield* recordedAgeVerdict(journal, keyDigest)
|
|
1455
|
+
if (recorded !== undefined) {
|
|
1456
|
+
return yield* Effect.fail(
|
|
1457
|
+
new Journal.JournalError({
|
|
1458
|
+
code: "idempotency_conflict",
|
|
1459
|
+
message:
|
|
1460
|
+
"incompatible recorded cache-age decision: ttlMs cannot be removed after a recorded verdict",
|
|
1461
|
+
cause: recorded
|
|
1462
|
+
})
|
|
1463
|
+
)
|
|
1464
|
+
}
|
|
1465
|
+
return row
|
|
1466
|
+
}
|
|
1467
|
+
if (Option.isNone(row)) {
|
|
1468
|
+
return row
|
|
1469
|
+
}
|
|
1470
|
+
if ((yield* admitByAge(ttlMs, row.value)) === "admitted") {
|
|
1471
|
+
return row
|
|
1472
|
+
}
|
|
1473
|
+
yield* expireRow(ttlMs, row.value)
|
|
1474
|
+
return Option.none<CacheStore.CacheEntry>()
|
|
1475
|
+
})
|
|
1476
|
+
// A run's admitted attempt is replay authority. Shared cache rows
|
|
1477
|
+
// are replaceable and cannot overwrite its result or failure.
|
|
1478
|
+
const existing = yield* attempts.get(attemptId)
|
|
1479
|
+
if (cacheDeclaration._tag === "Eligible") {
|
|
1480
|
+
const observed = yield* cache.get(keyDigest).pipe(
|
|
1481
|
+
Effect.catch((error) =>
|
|
1482
|
+
noteUnshareable({ stage: "entry", message: `cache lookup failed: ${error.message}` }).pipe(
|
|
1483
|
+
Effect.as(Option.none<CacheStore.CacheEntry>())
|
|
1484
|
+
)
|
|
1485
|
+
)
|
|
1486
|
+
)
|
|
1487
|
+
const evidenceDecision = CacheAdmission.candidate(
|
|
1488
|
+
Option.isSome(observed) ? decodeMeta(observed.value.meta) : undefined
|
|
1489
|
+
)
|
|
1490
|
+
const outputDecision = evidenceDecision._tag === "CandidateEvidence"
|
|
1491
|
+
? CacheOutputPolicy.replay(input.metadata, evidenceDecision.evidence)
|
|
1492
|
+
: undefined
|
|
1493
|
+
const candidate = outputDecision?._tag === "Refused" ? outputDecision : evidenceDecision
|
|
1494
|
+
// A descriptor refusal cannot be turned into TTL eviction. Validate
|
|
1495
|
+
// replay authority before any age decision is allowed to prune a head.
|
|
1496
|
+
const cached =
|
|
1497
|
+
Option.isNone(observed) || candidate._tag === "CandidateEvidence" || policy?.ttlMs === undefined
|
|
1498
|
+
? yield* admissible(policy?.ttlMs, observed)
|
|
1499
|
+
: observed
|
|
1500
|
+
if (Option.isSome(cached)) {
|
|
1501
|
+
if (candidate._tag === "Refused") {
|
|
1502
|
+
yield* emitConverging(JournalRecords.cacheProvenance(
|
|
1503
|
+
cacheSource("replay_failed", {
|
|
1504
|
+
runId: cached.value.recordedRunId,
|
|
1505
|
+
eventSeq: cached.value.recordedEventSeq
|
|
1506
|
+
}),
|
|
1507
|
+
{
|
|
1508
|
+
keyDigest,
|
|
1509
|
+
action: "replay_failed",
|
|
1510
|
+
reason: candidate.reason,
|
|
1511
|
+
recordedRunId: cached.value.recordedRunId,
|
|
1512
|
+
recordedEventSeq: cached.value.recordedEventSeq
|
|
1513
|
+
}
|
|
1514
|
+
))
|
|
1515
|
+
}
|
|
1516
|
+
if (candidate._tag === "CandidateEvidence" && Option.isNone(existing)) {
|
|
1517
|
+
const boundary = yield* StepBoundary.StepBoundary
|
|
1518
|
+
// Skyframe's dirty check, not "the declaration changed" (issue
|
|
1519
|
+
// #90): the read-set digests folded into the step key are caller
|
|
1520
|
+
// metadata, so reuse is justified only once the host has
|
|
1521
|
+
// measured them and agreed. A stale declaration falls through to
|
|
1522
|
+
// a real execution instead of replaying a pre-edit result, and
|
|
1523
|
+
// the refusal is journalled so it is visible rather than silent.
|
|
1524
|
+
// A boundary the host cannot enforce is likewise not a hit; the
|
|
1525
|
+
// dispatch path below re-prepares and fails the attempt properly.
|
|
1526
|
+
const measured = yield* boundary.prepare(cacheDeclaration.metadata).pipe(Effect.option)
|
|
1527
|
+
const verified = Option.isSome(measured) && StepBoundary.readSetMatches(measured.value)
|
|
1528
|
+
if (verified) {
|
|
1529
|
+
const evidence = candidate.evidence
|
|
1530
|
+
// Cache materialization is host work too. The authorized
|
|
1531
|
+
// invocation is durable before it, even without a new attempt.
|
|
1532
|
+
yield* emitCallInvoked
|
|
1533
|
+
let materialized = yield* boundary.replayOutputs(evidence).pipe(Effect.exit)
|
|
1534
|
+
if (
|
|
1535
|
+
Exit.isFailure(materialized) &&
|
|
1536
|
+
CachePublication.replayMissingArtifact(materialized.cause) !== undefined
|
|
1537
|
+
) {
|
|
1538
|
+
// LAZY DOWNLOAD (issue #172). A shared cache row is
|
|
1539
|
+
// routinely recorded on a machine whose artifacts this one
|
|
1540
|
+
// has never seen, so "the blob is not here" is the normal
|
|
1541
|
+
// first answer, not a defect — and it is the one replay
|
|
1542
|
+
// refusal a shared artifact tier can repair. Fetch, verify,
|
|
1543
|
+
// write back, and retry the replay ONCE. Only once: a
|
|
1544
|
+
// second failure means the tier cannot serve it either, and
|
|
1545
|
+
// the fall-through below (a real execution) is strictly
|
|
1546
|
+
// better than looping. With no shared tier configured
|
|
1547
|
+
// `hydrateArtifacts` reports `false` and this costs one
|
|
1548
|
+
// cheap branch.
|
|
1549
|
+
if (yield* CachePublication.hydrateArtifacts(evidence)) {
|
|
1550
|
+
materialized = yield* boundary.replayOutputs(evidence).pipe(Effect.exit)
|
|
1551
|
+
}
|
|
1552
|
+
}
|
|
1553
|
+
if (Exit.isSuccess(materialized)) {
|
|
1554
|
+
const recorded = {
|
|
1555
|
+
runId: cached.value.recordedRunId,
|
|
1556
|
+
eventSeq: cached.value.recordedEventSeq
|
|
1557
|
+
}
|
|
1558
|
+
yield* emitConverging(JournalRecords.cacheProvenance(cacheSource("hit", recorded), {
|
|
1559
|
+
keyDigest,
|
|
1560
|
+
recordedRunId: cached.value.recordedRunId,
|
|
1561
|
+
recordedEventSeq: cached.value.recordedEventSeq
|
|
1562
|
+
}))
|
|
1563
|
+
// Counted after the provenance emit: `verified_hit` means
|
|
1564
|
+
// the cached result was served, and a journal failure on
|
|
1565
|
+
// the emit fails the dispatch before any result is
|
|
1566
|
+
// returned. A dispatch that dies mid-decision records no
|
|
1567
|
+
// decision; its exit lands in `flows_engine_dispatches`.
|
|
1568
|
+
yield* Metric.update(EngineStoreMetrics.stepCacheDecision.VerifiedHit, 1)
|
|
1569
|
+
// Served from a record: the body never ran, and the node
|
|
1570
|
+
// above this dispatch is told so rather than left to
|
|
1571
|
+
// report work nobody did.
|
|
1572
|
+
yield* Action.reportDispatch({
|
|
1573
|
+
outcome: "replayed",
|
|
1574
|
+
stepKeyDigest,
|
|
1575
|
+
attempt: input.attempt
|
|
1576
|
+
})
|
|
1577
|
+
return cached.value.result
|
|
1578
|
+
}
|
|
1579
|
+
// Evidence the host cannot re-materialize — a transient
|
|
1580
|
+
// filesystem error, or a row recorded by a foreign boundary
|
|
1581
|
+
// implementation — is not a hit and not a run failure
|
|
1582
|
+
// (issue #107): failing here while the verified row survived
|
|
1583
|
+
// repeated refuse→fail on every later run, the exact
|
|
1584
|
+
// permanent-failure loop #99 closed one branch later. The
|
|
1585
|
+
// refusal is journalled and the dispatch path below executes
|
|
1586
|
+
// for real; the row survives for hosts that can replay it.
|
|
1587
|
+
// The refusal record carries its classification (issue
|
|
1588
|
+
// #150): `corruption` for a digest mismatch at the
|
|
1589
|
+
// content-addressed blob path, `host` for everything else —
|
|
1590
|
+
// a failing disk corrupting many blobs must never journal
|
|
1591
|
+
// identically to a one-off EIO.
|
|
1592
|
+
const corruption = replayCorruption(materialized.cause)
|
|
1593
|
+
yield* emitConverging(
|
|
1594
|
+
JournalRecords.cacheProvenance(
|
|
1595
|
+
cacheSource("replay_failed", {
|
|
1596
|
+
runId: cached.value.recordedRunId,
|
|
1597
|
+
eventSeq: cached.value.recordedEventSeq
|
|
1598
|
+
}),
|
|
1599
|
+
{
|
|
1600
|
+
keyDigest,
|
|
1601
|
+
action: "replay_failed",
|
|
1602
|
+
reason: corruption === undefined ? "host" : "corruption",
|
|
1603
|
+
recordedRunId: cached.value.recordedRunId,
|
|
1604
|
+
recordedEventSeq: cached.value.recordedEventSeq
|
|
1605
|
+
}
|
|
1606
|
+
)
|
|
1607
|
+
)
|
|
1608
|
+
if (corruption !== undefined) {
|
|
1609
|
+
// Corruption is an integrity violation, not a retryable
|
|
1610
|
+
// refusal: it routes to the Inconsistency receiver like
|
|
1611
|
+
// a cache-key conflict does. The core default is STRICT
|
|
1612
|
+
// (fail); `Inconsistency.layerTolerant` (or a plugin
|
|
1613
|
+
// verdict) lets the dispatch fall back to the real
|
|
1614
|
+
// execution below, whose re-capture heals the address.
|
|
1615
|
+
const receiverOption = yield* Effect.serviceOption(Inconsistency.Inconsistency)
|
|
1616
|
+
const receiver = Option.isSome(receiverOption)
|
|
1617
|
+
? receiverOption.value
|
|
1618
|
+
: Inconsistency.make({ journal, verdict: "fail", owner: deps.owner })
|
|
1619
|
+
const verdict = yield* receiver.noteCorruption({
|
|
1620
|
+
runId: deps.runId,
|
|
1621
|
+
keyDigest,
|
|
1622
|
+
path: corruption.path,
|
|
1623
|
+
recordedDigest: corruption.recordedDigest,
|
|
1624
|
+
measuredDigest: corruption.measuredDigest,
|
|
1625
|
+
recordedRunId: cached.value.recordedRunId,
|
|
1626
|
+
recordedEventSeq: cached.value.recordedEventSeq
|
|
1627
|
+
})
|
|
1628
|
+
// Quarantine is journal AND evict (issue #164). The
|
|
1629
|
+
// receiver's durable record preserved the evidence, but
|
|
1630
|
+
// leaving the row in place made the poison permanent for
|
|
1631
|
+
// INLINE evidence: `CacheStore.put` is insert-or-nothing,
|
|
1632
|
+
// so a tolerant re-execution never replaced the corrupt
|
|
1633
|
+
// bytes and re-detected them on every later run, while
|
|
1634
|
+
// strict mode re-failed the key forever. Evicting under
|
|
1635
|
+
// both verdicts lets the next dispatch — the tolerant
|
|
1636
|
+
// fall-through below, or the run after a strict failure —
|
|
1637
|
+
// execute and record cleanly. The evict is fenced on the
|
|
1638
|
+
// poisoned row's own provenance like the stale-read-set
|
|
1639
|
+
// branch (issue #119): a fresh row landed by a concurrent
|
|
1640
|
+
// run between this dispatch's `get` and the `evict` makes
|
|
1641
|
+
// the compare-and-swap a no-op instead of deleting valid
|
|
1642
|
+
// evidence.
|
|
1643
|
+
yield* cache.evict(keyDigest, {
|
|
1644
|
+
ifRecordedBy: {
|
|
1645
|
+
runId: cached.value.recordedRunId,
|
|
1646
|
+
eventSeq: cached.value.recordedEventSeq
|
|
1647
|
+
}
|
|
1648
|
+
})
|
|
1649
|
+
if (verdict === "fail") {
|
|
1650
|
+
return yield* Effect.fail(
|
|
1651
|
+
new CacheCorruptionDetected({
|
|
1652
|
+
code: "cache_corruption_detected",
|
|
1653
|
+
keyDigest,
|
|
1654
|
+
path: corruption.path,
|
|
1655
|
+
recordedDigest: corruption.recordedDigest,
|
|
1656
|
+
measuredDigest: corruption.measuredDigest
|
|
1657
|
+
})
|
|
1658
|
+
)
|
|
1659
|
+
}
|
|
1660
|
+
}
|
|
1661
|
+
// `replay_failed` is a fall-through decision, so it is
|
|
1662
|
+
// counted only once the refusal emit landed and the strict
|
|
1663
|
+
// corruption verdict above has NOT terminated the dispatch:
|
|
1664
|
+
// a dispatch that fails instead of re-executing records no
|
|
1665
|
+
// decision.
|
|
1666
|
+
yield* Metric.update(EngineStoreMetrics.stepCacheDecision.ReplayFailed, 1)
|
|
1667
|
+
} else if (Option.isSome(measured)) {
|
|
1668
|
+
// Only a *measured* mismatch is evidence the inputs changed
|
|
1669
|
+
// (issue #110): a host that cannot measure right now — a
|
|
1670
|
+
// transient EIO/EACCES on any declared read path — says
|
|
1671
|
+
// nothing about the read set, so the hit is merely refused for
|
|
1672
|
+
// this dispatch (the path below re-prepares and surfaces the
|
|
1673
|
+
// host failure as an ordinary attempt failure) and the valid
|
|
1674
|
+
// shared row survives for every run whose host is healthy.
|
|
1675
|
+
//
|
|
1676
|
+
// The durable emit is also the fence: `emitDurable` fails with
|
|
1677
|
+
// `fence_lost` for a zombie that lost the run, surfacing as
|
|
1678
|
+
// self-interruption before the eviction below can run.
|
|
1679
|
+
yield* emitConverging(
|
|
1680
|
+
JournalRecords.cacheProvenance(
|
|
1681
|
+
cacheSource("stale_read_set", {
|
|
1682
|
+
runId: cached.value.recordedRunId,
|
|
1683
|
+
eventSeq: cached.value.recordedEventSeq
|
|
1684
|
+
}),
|
|
1685
|
+
{
|
|
1686
|
+
keyDigest,
|
|
1687
|
+
action: "stale_read_set",
|
|
1688
|
+
recordedRunId: cached.value.recordedRunId,
|
|
1689
|
+
recordedEventSeq: cached.value.recordedEventSeq
|
|
1690
|
+
}
|
|
1691
|
+
)
|
|
1692
|
+
)
|
|
1693
|
+
// Skyframe invalidation, not just refusal (issue #99): a stale
|
|
1694
|
+
// read set means the inputs changed, so the re-execution's
|
|
1695
|
+
// result is *expected* to differ — left in place, the poisoned
|
|
1696
|
+
// row turns the fresh `cache.put` into a Conflict, the strict
|
|
1697
|
+
// verdict fails the run, and nothing ever removes the row, so
|
|
1698
|
+
// every later run repeats the refuse → re-execute → conflict →
|
|
1699
|
+
// fail cycle. The refusal is journalled above; evicting here
|
|
1700
|
+
// lets the re-execution record cleanly under the same key.
|
|
1701
|
+
// The eviction is fenced on the poisoned row's own
|
|
1702
|
+
// provenance (issue #119): a fresh entry recorded by a
|
|
1703
|
+
// concurrent run between this dispatch's `get` and its
|
|
1704
|
+
// `evict` must not be deleted with the poison (issue #110).
|
|
1705
|
+
// The permit (issue #118) only closes the *in-process*
|
|
1706
|
+
// window, so the guard rides inside the DELETE rather than
|
|
1707
|
+
// in a preceding read — a foreign process landing a fresh
|
|
1708
|
+
// row simply makes the compare-and-swap a no-op.
|
|
1709
|
+
yield* cache.evict(keyDigest, {
|
|
1710
|
+
ifRecordedBy: {
|
|
1711
|
+
runId: cached.value.recordedRunId,
|
|
1712
|
+
eventSeq: cached.value.recordedEventSeq
|
|
1713
|
+
}
|
|
1714
|
+
})
|
|
1715
|
+
// The fall-through decision is counted once the refusal is
|
|
1716
|
+
// durable and the poisoned row is gone: a fenced-out zombie
|
|
1717
|
+
// self-interrupts at the emit above and records no decision.
|
|
1718
|
+
yield* Metric.update(EngineStoreMetrics.stepCacheDecision.StaleReadSet, 1)
|
|
1719
|
+
} else {
|
|
1720
|
+
// The host could not measure the read set at all, so the
|
|
1721
|
+
// hit is merely refused for this dispatch; the row survives.
|
|
1722
|
+
yield* Metric.update(EngineStoreMetrics.stepCacheDecision.Unmeasurable, 1)
|
|
1723
|
+
}
|
|
1724
|
+
} else if (candidate._tag === "Refused") {
|
|
1725
|
+
// A row whose recorded evidence cannot justify reuse — a
|
|
1726
|
+
// foreign tier, an unverified capture, a recorded deviation.
|
|
1727
|
+
yield* Metric.update(EngineStoreMetrics.stepCacheDecision.UnverifiableEvidence, 1)
|
|
1728
|
+
}
|
|
1729
|
+
} else {
|
|
1730
|
+
yield* Metric.update(EngineStoreMetrics.stepCacheDecision.Miss, 1)
|
|
1731
|
+
}
|
|
1732
|
+
}
|
|
1733
|
+
|
|
1734
|
+
if (Option.isSome(existing)) {
|
|
1735
|
+
const row = existing.value
|
|
1736
|
+
if (row.state === "succeeded") {
|
|
1737
|
+
yield* emitCallInvoked
|
|
1738
|
+
const meta = decodeMeta(row.meta)
|
|
1739
|
+
let corruptEvidence = false
|
|
1740
|
+
const outputDecision = meta?.boundary === undefined
|
|
1741
|
+
? undefined
|
|
1742
|
+
: CacheOutputPolicy.replay(input.metadata, meta.boundary)
|
|
1743
|
+
const storedDecision = CacheAdmission.candidate(meta)
|
|
1744
|
+
const refused = storedDecision._tag === "Refused" &&
|
|
1745
|
+
(storedDecision.reason === "contradictory-evidence" ||
|
|
1746
|
+
storedDecision.reason === "quarantined-evidence")
|
|
1747
|
+
? storedDecision
|
|
1748
|
+
: outputDecision
|
|
1749
|
+
if (refused?._tag === "Refused") {
|
|
1750
|
+
yield* emitConverging(JournalRecords.cacheProvenance(cacheSource("replay_failed"), {
|
|
1751
|
+
keyDigest,
|
|
1752
|
+
action: "replay_failed",
|
|
1753
|
+
reason: refused.reason
|
|
1754
|
+
}))
|
|
1755
|
+
}
|
|
1756
|
+
if (meta?.boundary !== undefined && refused?._tag !== "Refused") {
|
|
1757
|
+
const boundary = yield* StepBoundary.StepBoundary
|
|
1758
|
+
const materialized = yield* boundary.replayOutputs(meta.boundary).pipe(Effect.exit)
|
|
1759
|
+
if (Exit.isFailure(materialized)) {
|
|
1760
|
+
// The attempt durably succeeded: its recorded outcome is
|
|
1761
|
+
// the truth, and re-materializing the workspace outputs is
|
|
1762
|
+
// best-effort — failing the dispatch here while the
|
|
1763
|
+
// terminal row survived recreated the #99 permanent
|
|
1764
|
+
// refuse→fail loop one branch earlier (issue #107). The
|
|
1765
|
+
// refusal is journalled so a missing output is
|
|
1766
|
+
// explainable rather than silent, and it carries its
|
|
1767
|
+
// classification (issue #150): corruption of the recorded
|
|
1768
|
+
// evidence routes to the Inconsistency receiver instead of
|
|
1769
|
+
// passing as an ordinary transient host refusal.
|
|
1770
|
+
const corruption = replayCorruption(materialized.cause)
|
|
1771
|
+
corruptEvidence = corruption !== undefined
|
|
1772
|
+
yield* emitConverging(
|
|
1773
|
+
JournalRecords.cacheProvenance(cacheSource("replay_failed"), {
|
|
1774
|
+
keyDigest,
|
|
1775
|
+
action: "replay_failed",
|
|
1776
|
+
reason: corruption === undefined ? "host" : "corruption"
|
|
1777
|
+
})
|
|
1778
|
+
)
|
|
1779
|
+
if (corruption !== undefined) {
|
|
1780
|
+
const receiverOption = yield* Effect.serviceOption(Inconsistency.Inconsistency)
|
|
1781
|
+
const receiver = Option.isSome(receiverOption)
|
|
1782
|
+
? receiverOption.value
|
|
1783
|
+
: Inconsistency.make({ journal, verdict: "fail", owner: deps.owner })
|
|
1784
|
+
const verdict = yield* receiver.noteCorruption({
|
|
1785
|
+
runId: deps.runId,
|
|
1786
|
+
// The attempt row, not a cache row: this evidence is
|
|
1787
|
+
// addressed by (runId, stepKeyDigest, attempt).
|
|
1788
|
+
keyDigest: stepKeyDigest,
|
|
1789
|
+
path: corruption.path,
|
|
1790
|
+
recordedDigest: corruption.recordedDigest,
|
|
1791
|
+
measuredDigest: corruption.measuredDigest
|
|
1792
|
+
})
|
|
1793
|
+
// Quarantine is journal AND take a state action (issues
|
|
1794
|
+
// #164, #171). Unlike a shared cache row, this succeeded
|
|
1795
|
+
// attempt cannot be evicted: its side effects already ran,
|
|
1796
|
+
// so a miss would re-execute potentially irreversible work.
|
|
1797
|
+
// Remove only the corrupt boundary evidence and mark the
|
|
1798
|
+
// row quarantined. The durable outcome stays intact and the
|
|
1799
|
+
// next dispatch returns it without materializing the poison
|
|
1800
|
+
// or publishing the row back into the shared cache.
|
|
1801
|
+
//
|
|
1802
|
+
// The owner heartbeat and patch share one write
|
|
1803
|
+
// transaction. The patch is owner-fenced itself — it only
|
|
1804
|
+
// lands while `flows_runs` still records `deps.owner` —
|
|
1805
|
+
// and the heartbeat both refreshes the lease and reports
|
|
1806
|
+
// the loss as a run-store outcome before the patch runs.
|
|
1807
|
+
// `boundary` is dropped by omitting the key, never by
|
|
1808
|
+
// writing `undefined` over it: the attempt store admits
|
|
1809
|
+
// inert JSON, and an own property holding `undefined` is
|
|
1810
|
+
// not JSON, so the patch would be refused outright.
|
|
1811
|
+
// New quarantine writes clear both reusable proof fields;
|
|
1812
|
+
// contradictory legacy rows remain readable without repair.
|
|
1813
|
+
const { boundary: _quarantined, readSetVerified: _unverified, ...retainedMeta } = meta
|
|
1814
|
+
const quarantinedMeta: AttemptMeta = {
|
|
1815
|
+
...retainedMeta,
|
|
1816
|
+
boundaryQuarantined: true
|
|
1817
|
+
}
|
|
1818
|
+
const quarantined = yield* atomically(Effect.gen(function*() {
|
|
1819
|
+
const quarantineAtMs = yield* Clock.currentTimeMillis.pipe(Effect.map(Math.floor))
|
|
1820
|
+
const fence = yield* runs.heartbeat(deps.runId, deps.owner, quarantineAtMs)
|
|
1821
|
+
if (fence._tag !== "Updated") {
|
|
1822
|
+
return false
|
|
1823
|
+
}
|
|
1824
|
+
const patched = yield* attempts.patch(
|
|
1825
|
+
attemptId,
|
|
1826
|
+
{ meta: attemptPayload(quarantinedMeta) },
|
|
1827
|
+
deps.owner
|
|
1828
|
+
)
|
|
1829
|
+
return patched._tag === "Patched"
|
|
1830
|
+
}))
|
|
1831
|
+
if (!quarantined) {
|
|
1832
|
+
return yield* Effect.interrupt
|
|
1833
|
+
}
|
|
1834
|
+
if (verdict === "fail") {
|
|
1835
|
+
// The strict verdict still makes the integrity violation
|
|
1836
|
+
// visible by parking this dispatch. The row repair above
|
|
1837
|
+
// is what makes the park resumable without an out-of-band
|
|
1838
|
+
// byte repair. This is a defect, not a declared action
|
|
1839
|
+
// business error: routing it through the failure channel
|
|
1840
|
+
// would make the action's error schema replace it with
|
|
1841
|
+
// a SchemaError.
|
|
1842
|
+
return yield* Effect.die(
|
|
1843
|
+
new AttemptEvidenceQuarantined({
|
|
1844
|
+
code: "attempt_evidence_quarantined",
|
|
1845
|
+
keyDigest: stepKeyDigest,
|
|
1846
|
+
attempt: input.attempt,
|
|
1847
|
+
path: corruption.path,
|
|
1848
|
+
recordedDigest: corruption.recordedDigest,
|
|
1849
|
+
measuredDigest: corruption.measuredDigest
|
|
1850
|
+
})
|
|
1851
|
+
)
|
|
1852
|
+
}
|
|
1853
|
+
}
|
|
1854
|
+
}
|
|
1855
|
+
}
|
|
1856
|
+
// Converge the cache with the durable completion: a crash between
|
|
1857
|
+
// `attempts.finish` and `cache.put` otherwise leaves the sealed
|
|
1858
|
+
// result permanently missing from the shared cache (issue #24).
|
|
1859
|
+
// Reaching this branch with an eligible declaration means `cache.get`
|
|
1860
|
+
// above missed or was unfit for replay.
|
|
1861
|
+
// Only a row whose recorded read set was verified at prepare
|
|
1862
|
+
// time may converge into the shared cache (issue #106): an
|
|
1863
|
+
// unverified result was computed against content the key does
|
|
1864
|
+
// not describe.
|
|
1865
|
+
// A row whose boundary evidence was just measured corrupt is
|
|
1866
|
+
// quarantined, never converged (issue #160): a `tolerate`
|
|
1867
|
+
// verdict keeps the durable outcome as this run's truth, but
|
|
1868
|
+
// publishing the known-corrupt evidence into the shared cache
|
|
1869
|
+
// would hand sibling runs a poisoned hit under this run's
|
|
1870
|
+
// provenance.
|
|
1871
|
+
const admission = CacheAdmission.completion(
|
|
1872
|
+
cacheDeclaration,
|
|
1873
|
+
meta,
|
|
1874
|
+
corruptEvidence || refused?._tag === "Refused"
|
|
1875
|
+
)
|
|
1876
|
+
if (admission._tag === "PublishCompletion") {
|
|
1877
|
+
yield* recordCache({
|
|
1878
|
+
result: row.outcome,
|
|
1879
|
+
admission,
|
|
1880
|
+
createdAtMs: row.finishedAtMs ?? (yield* Clock.currentTimeMillis.pipe(Effect.map(Math.floor)))
|
|
1881
|
+
})
|
|
1882
|
+
}
|
|
1883
|
+
// Converge the journal with the durable completion (issue
|
|
1884
|
+
// #109): a crash between `attempts.finish` and the terminal
|
|
1885
|
+
// emits left `attemptStarted` without `attemptFinished`
|
|
1886
|
+
// forever. The per-attempt producer identity collapses the
|
|
1887
|
+
// re-emission into a `Duplicate` on ordinary replays.
|
|
1888
|
+
if (meta?.boundary?.deviation !== undefined) {
|
|
1889
|
+
yield* emitConverging(
|
|
1890
|
+
JournalRecords.expectedSetDeviation(attemptSource("deviation"), {
|
|
1891
|
+
...attemptId,
|
|
1892
|
+
...meta.boundary.deviation
|
|
1893
|
+
})
|
|
1894
|
+
)
|
|
1895
|
+
}
|
|
1896
|
+
// Replays publish the saved first observation, never a newly
|
|
1897
|
+
// computed timestamp or the current history generation.
|
|
1898
|
+
yield* emitStepSettled(row.outcome)
|
|
1899
|
+
yield* emitConverging(
|
|
1900
|
+
JournalRecords.attemptFinished(attemptSource("finished"), { ...attemptId, state: "succeeded" })
|
|
1901
|
+
)
|
|
1902
|
+
// Upgrade an older durable controller record without rerunning
|
|
1903
|
+
// its handler; current records collapse to the original receipt.
|
|
1904
|
+
yield* emitCallSettled(row.outcome)
|
|
1905
|
+
// A durable attempt row answered the dispatch, so nothing ran.
|
|
1906
|
+
yield* Action.reportDispatch({ outcome: "replayed", stepKeyDigest, attempt: input.attempt })
|
|
1907
|
+
return row.outcome
|
|
1908
|
+
}
|
|
1909
|
+
if (row.state === "failed") {
|
|
1910
|
+
yield* emitCallInvoked
|
|
1911
|
+
// Converge the journal before rethrowing (issue #109): the
|
|
1912
|
+
// violation kind survives in the row meta because the
|
|
1913
|
+
// persisted cause alone cannot distinguish a boundary
|
|
1914
|
+
// violation from an ordinary execution failure.
|
|
1915
|
+
if (decodeMeta(row.meta)?.hardViolation === true) {
|
|
1916
|
+
yield* emitConverging(
|
|
1917
|
+
JournalRecords.hardViolation(attemptSource("hard-violation"), {
|
|
1918
|
+
...attemptId,
|
|
1919
|
+
error: rehydrateCause(row.error)
|
|
1920
|
+
})
|
|
1921
|
+
)
|
|
1922
|
+
}
|
|
1923
|
+
yield* emitConverging(
|
|
1924
|
+
JournalRecords.attemptFinished(attemptSource("finished"), { ...attemptId, state: "failed" })
|
|
1925
|
+
)
|
|
1926
|
+
// A durably failed attempt is replayed by rethrowing the persisted
|
|
1927
|
+
// domain failure — never by readmission (issue #59). Falling
|
|
1928
|
+
// through to `attempts.put` here surfaced the row as
|
|
1929
|
+
// `AttemptAdmissionRejected`, whose tag can never match a
|
|
1930
|
+
// policy-declared `nonRetryable` classification, so a durably
|
|
1931
|
+
// failed no-retry action earned an extra real dispatch after
|
|
1932
|
+
// resume. Temporal's prior art: mutable state persists the attempt
|
|
1933
|
+
// failure and `ExecutionInfo.Attempt`, and its no-retry decision
|
|
1934
|
+
// (`service/history/workflow/retry.go`) is re-evaluated from that
|
|
1935
|
+
// persisted failure — the failure itself is durable, not just the
|
|
1936
|
+
// fact that an attempt happened.
|
|
1937
|
+
return yield* Effect.failCause(rehydrateCause(row.error))
|
|
1938
|
+
}
|
|
1939
|
+
if (row.state === "suspended") {
|
|
1940
|
+
return yield* Effect.fail(
|
|
1941
|
+
new AttemptSuspended({
|
|
1942
|
+
code: "attempt_suspended",
|
|
1943
|
+
runId: deps.runId,
|
|
1944
|
+
keyDigest: stepKeyDigest,
|
|
1945
|
+
attempt: input.attempt
|
|
1946
|
+
})
|
|
1947
|
+
)
|
|
1948
|
+
}
|
|
1949
|
+
}
|
|
1950
|
+
|
|
1951
|
+
/**
|
|
1952
|
+
* A persisted `running` row read while this owner holds the run fence
|
|
1953
|
+
* is crash evidence, not a live admission (issue #71): the incarnation
|
|
1954
|
+
* that admitted the attempt died before finishing (SIGKILL, OOM), and
|
|
1955
|
+
* the #53 stale-running sweep re-drove the run to a new owner — or the
|
|
1956
|
+
* same owner after restart. The attempt never completed, so it must
|
|
1957
|
+
* re-execute under its original number rather than fall through to
|
|
1958
|
+
* `attempts.put` (whose `Conflict` on the differing `startedAtMs`
|
|
1959
|
+
* surfaced as `AttemptAdmissionRejected`, permanently failing a
|
|
1960
|
+
* no-policy run with an infrastructure tag). The row is adopted; the
|
|
1961
|
+
* ordinary fenced `attempts.finish` transition below records the
|
|
1962
|
+
* re-execution's outcome.
|
|
1963
|
+
*
|
|
1964
|
+
* Adoption requires liveness evidence (issue #86), and that evidence is
|
|
1965
|
+
* the admission permit held around this whole span (issues #102, #103):
|
|
1966
|
+
* a live same-key dispatch of this process would be holding the permit,
|
|
1967
|
+
* so a `running` row observed here cannot belong to a live in-process
|
|
1968
|
+
* fiber — it is a dead fiber of this incarnation (an in-process
|
|
1969
|
+
* re-drive after an interrupt, the #71 mode the recorded-nonce guard
|
|
1970
|
+
* wrongly refused) or a superseded incarnation (which provably lost the
|
|
1971
|
+
* run fence this owner holds). Deciding from the recorded `admittedBy`
|
|
1972
|
+
* nonce could not tell those apart, and comparing it against a stale
|
|
1973
|
+
* pre-claim read left a TOCTOU window where two concurrent dispatches
|
|
1974
|
+
* both saw a dead owner and both executed an irreversible body.
|
|
1975
|
+
*/
|
|
1976
|
+
const runningRow = Option.isSome(existing) && existing.value.state === "running"
|
|
1977
|
+
? existing.value
|
|
1978
|
+
: undefined
|
|
1979
|
+
const runningMeta = runningRow === undefined ? undefined : decodeMeta(runningRow.meta)
|
|
1980
|
+
/**
|
|
1981
|
+
* ADOPTION READS THE CROSSING BACK. A `running` row is crash
|
|
1982
|
+
* evidence, but "the attempt did not settle" is not "the body did
|
|
1983
|
+
* not run": an effect whose terminal boundary is already durable was
|
|
1984
|
+
* still re-dispatched here, because the row alone cannot say so and
|
|
1985
|
+
* nothing ever read the boundary record back. That made a keyless
|
|
1986
|
+
* irreversible action charge twice on a SIGKILL landing in the
|
|
1987
|
+
* millisecond between the `succeeded` boundary and `attempts.finish`
|
|
1988
|
+
* — the exact window the boundary record exists to close.
|
|
1989
|
+
*
|
|
1990
|
+
* A recorded `succeeded` crossing settles the adopted attempt from
|
|
1991
|
+
* the outcome the crossing wrote, in this attempt's own number, and
|
|
1992
|
+
* the body never runs again. The outcome comes from the ROW, not
|
|
1993
|
+
* from the journal record: journal payloads go through redaction on
|
|
1994
|
+
* write, so the record's `output` is observability and the row's
|
|
1995
|
+
* `outcome` is the executable copy (see `AttemptMeta.effectCrossing`).
|
|
1996
|
+
*/
|
|
1997
|
+
if (runningRow !== undefined && runningMeta?.effectCrossing === "succeeded") {
|
|
1998
|
+
yield* emitCallInvoked
|
|
1999
|
+
const finishedAtMs = yield* Clock.currentTimeMillis.pipe(Effect.map(Math.floor))
|
|
2000
|
+
// `outcome` and `meta` are deliberately omitted: `FinishAttempt`
|
|
2001
|
+
// leaves an omitted field as recorded, so the terminal transition
|
|
2002
|
+
// is the crossing's own result being sealed rather than a value
|
|
2003
|
+
// this incarnation invented.
|
|
2004
|
+
const finished = yield* settleAttempt(
|
|
2005
|
+
{ ...attemptId, state: "succeeded", finishedAtMs },
|
|
2006
|
+
[JournalRecords.attemptFinished(attemptSource("finished"), { ...attemptId, state: "succeeded" })]
|
|
2007
|
+
)
|
|
2008
|
+
if (!finished) {
|
|
2009
|
+
return yield* Effect.interrupt
|
|
2010
|
+
}
|
|
2011
|
+
return runningRow.outcome
|
|
2012
|
+
}
|
|
2013
|
+
/**
|
|
2014
|
+
* AN UNRESOLVED CROSSING IS NOT SILENTLY RE-DISPATCHED. `intended`
|
|
2015
|
+
* without a terminal crossing means the body was dispatched and
|
|
2016
|
+
* nothing durable says how it ended: the charge may have gone
|
|
2017
|
+
* through, and no record can settle the question. An idempotency key
|
|
2018
|
+
* is the one thing that makes the repeat safe, so with a key the
|
|
2019
|
+
* attempt re-executes exactly as before, and without one this
|
|
2020
|
+
* refuses under the same error a numbered retry refuses under. The
|
|
2021
|
+
* run stops with a typed, durable "we cannot tell whether this
|
|
2022
|
+
* happened" for an operator, which is the honest outcome; charging a
|
|
2023
|
+
* second time to keep the run moving is not.
|
|
2024
|
+
*
|
|
2025
|
+
* Only tier 3 refuses. A compensable body is re-executed on a
|
|
2026
|
+
* restored pre-image (issue #87), which is what makes its repeat
|
|
2027
|
+
* safe by construction.
|
|
2028
|
+
*/
|
|
2029
|
+
if (
|
|
2030
|
+
input.tier === "irreversible" &&
|
|
2031
|
+
runningMeta?.effectCrossing === "intended" &&
|
|
2032
|
+
deps.idempotencyKey === undefined
|
|
2033
|
+
) {
|
|
2034
|
+
return yield* Effect.fail(
|
|
2035
|
+
new Action.IrreversibleRetryRequiresIdempotencyKey({
|
|
2036
|
+
actionName: actionKind(input.action),
|
|
2037
|
+
attempt: input.attempt
|
|
2038
|
+
})
|
|
2039
|
+
)
|
|
2040
|
+
}
|
|
2041
|
+
// THE REPLAY-ONLY GATE. Every durable record that could serve this
|
|
2042
|
+
// dispatch has been consulted above; what remains is admitting an
|
|
2043
|
+
// attempt and running the body. A replay-only engine stops here,
|
|
2044
|
+
// before the attempt row, the snapshot, the boundary and the body.
|
|
2045
|
+
if (deps.replayOnly !== undefined) {
|
|
2046
|
+
replay.gated = true
|
|
2047
|
+
const refused = {
|
|
2048
|
+
runId: deps.runId,
|
|
2049
|
+
stepKeyDigest,
|
|
2050
|
+
attempt: input.attempt,
|
|
2051
|
+
action: actionKind(input.action)
|
|
2052
|
+
}
|
|
2053
|
+
yield* deps.replayOnly.observe({
|
|
2054
|
+
...refused,
|
|
2055
|
+
tier: input.tier,
|
|
2056
|
+
outcome: runningRow === undefined ? "would-execute" : "resumes"
|
|
2057
|
+
})
|
|
2058
|
+
return yield* Effect.die(new ReplayOnly.WouldExecute(refused))
|
|
2059
|
+
}
|
|
2060
|
+
const adopted = runningRow !== undefined
|
|
2061
|
+
// The admission row and its announcement commit as one unit: an
|
|
2062
|
+
// `attemptStarted` never describes a row that rolled back, and an
|
|
2063
|
+
// admitted attempt is never invisible to the journal.
|
|
2064
|
+
yield* atomically(Effect.gen(function*() {
|
|
2065
|
+
if (!adopted) {
|
|
2066
|
+
const now = yield* Clock.currentTimeMillis.pipe(Effect.map(Math.floor))
|
|
2067
|
+
const initialMeta: AttemptMeta = { ...declarationMeta, admittedBy: deps.owner }
|
|
2068
|
+
const put = yield* attempts.put(
|
|
2069
|
+
{ ...attemptId, state: "running", startedAtMs: now, meta: attemptPayload(initialMeta) },
|
|
2070
|
+
deps.owner
|
|
2071
|
+
)
|
|
2072
|
+
if (put._tag === "FenceLost" || put._tag === "RunNotFound") {
|
|
2073
|
+
return yield* Effect.interrupt
|
|
2074
|
+
}
|
|
2075
|
+
if (put._tag !== "Inserted") {
|
|
2076
|
+
return yield* Effect.fail(
|
|
2077
|
+
new AttemptAdmissionRejected({
|
|
2078
|
+
code: "attempt_admission_rejected",
|
|
2079
|
+
keyDigest: stepKeyDigest,
|
|
2080
|
+
outcome: put._tag
|
|
2081
|
+
})
|
|
2082
|
+
)
|
|
2083
|
+
}
|
|
2084
|
+
} else {
|
|
2085
|
+
// The claim is fenced at the moment it lands (issue #102): re-verify
|
|
2086
|
+
// run ownership immediately before re-homing the row, so a process
|
|
2087
|
+
// that lost the fence while waiting on the permit parks instead of
|
|
2088
|
+
// patching a run it no longer owns. The patch below carries the
|
|
2089
|
+
// owner fence itself; the heartbeat additionally refreshes the
|
|
2090
|
+
// lease, and the permit excludes in-process racers.
|
|
2091
|
+
const claimAtMs = yield* Clock.currentTimeMillis.pipe(Effect.map(Math.floor))
|
|
2092
|
+
const claimFence = yield* runs.heartbeat(deps.runId, deps.owner, claimAtMs)
|
|
2093
|
+
if (claimFence._tag !== "Updated") {
|
|
2094
|
+
return yield* Effect.interrupt
|
|
2095
|
+
}
|
|
2096
|
+
// Re-home the adopted row to the current incarnation; the patch
|
|
2097
|
+
// keeps the dead incarnation's other meta (tier, pre-image
|
|
2098
|
+
// snapshot) intact. A vanished row or a lost fence means the
|
|
2099
|
+
// durable state moved under us — surface it as self-interruption
|
|
2100
|
+
// like the other fence losses.
|
|
2101
|
+
const rehomed = yield* attempts.patch(attemptId, {
|
|
2102
|
+
meta: attemptPayload(
|
|
2103
|
+
{ ...runningMeta, ...declarationMeta, admittedBy: deps.owner } satisfies AttemptMeta
|
|
2104
|
+
)
|
|
2105
|
+
}, deps.owner)
|
|
2106
|
+
if (rehomed._tag !== "Patched") {
|
|
2107
|
+
return yield* Effect.interrupt
|
|
2108
|
+
}
|
|
2109
|
+
}
|
|
2110
|
+
yield* emitLifecycle(
|
|
2111
|
+
JournalRecords.attemptStarted(attemptSource("started"), { ...attemptId, tier: input.tier })
|
|
2112
|
+
)
|
|
2113
|
+
// Shares attempt admission's transaction. Refusal rolls admission
|
|
2114
|
+
// back before a handler, snapshot, or sandbox can execute.
|
|
2115
|
+
yield* emitCallInvoked
|
|
2116
|
+
}))
|
|
2117
|
+
|
|
2118
|
+
/** The pre-image as the attempt row carries it: the jj operation only when reported. */
|
|
2119
|
+
const snapshotFields = (id: string, operationId: string | undefined) =>
|
|
2120
|
+
operationId === undefined
|
|
2121
|
+
? { meta: { snapshotId: id }, announced: { snapshotId: id } }
|
|
2122
|
+
: {
|
|
2123
|
+
meta: { snapshotId: id, snapshotOperationId: operationId },
|
|
2124
|
+
announced: { snapshotId: id, operationId }
|
|
2125
|
+
}
|
|
2126
|
+
const announceSnapshot = (fields: ReturnType<typeof snapshotFields>) =>
|
|
2127
|
+
emitLifecycle(
|
|
2128
|
+
JournalRecords.snapshotIdentified(attemptSource("snapshot"), { ...attemptId, ...fields.announced })
|
|
2129
|
+
)
|
|
2130
|
+
let snapshotId: string | undefined
|
|
2131
|
+
let snapshotMeta: Pick<AttemptMeta, "snapshotId" | "snapshotOperationId" | "boundarySnapshot"> = {}
|
|
2132
|
+
if (input.tier !== "compensable") {
|
|
2133
|
+
/**
|
|
2134
|
+
* THE TIER-2 ANCHOR FOR AN ORDINARY FRAME.
|
|
2135
|
+
*
|
|
2136
|
+
* Time travel (`packages/smithers/flows/time-travel/docs/README.md`) requires the jj pointer
|
|
2137
|
+
* current when a seq was journaled to be recorded at the frame,
|
|
2138
|
+
* because replay cannot derive it. Only compensable work took a
|
|
2139
|
+
* fresh snapshot, so every other frame had no anchor at all and a
|
|
2140
|
+
* rewind to it restored the workspace to whatever the nearest
|
|
2141
|
+
* *compensable* attempt happened to leave behind.
|
|
2142
|
+
*
|
|
2143
|
+
* The anchor is `carried`: it asserts "the same pointer as the
|
|
2144
|
+
* previous anchor in this lineage" rather than naming one. That is
|
|
2145
|
+
* the cheap half of the obligation — no jj call, no host round
|
|
2146
|
+
* trip, one journal row — and the snapshot projector resolves it by
|
|
2147
|
+
* copying the last change id forward. A lineage that has taken no
|
|
2148
|
+
* snapshot yet carries nothing forward, which is honest: there is
|
|
2149
|
+
* no pointer to restore, and a rewind reports none rather than
|
|
2150
|
+
* inventing one.
|
|
2151
|
+
*/
|
|
2152
|
+
yield* emitLifecycle(
|
|
2153
|
+
JournalRecords.snapshotIdentified(attemptSource("snapshot"), { ...attemptId, carried: true })
|
|
2154
|
+
)
|
|
2155
|
+
}
|
|
2156
|
+
if (input.tier === "compensable") {
|
|
2157
|
+
const jj = deps.engineJj === undefined ? yield* Jj.Jj : deps.engineJj
|
|
2158
|
+
/**
|
|
2159
|
+
* THE ENGINE'S BOUNDARY HANDLE (issue #1805). The engine restores
|
|
2160
|
+
* the retry sequence's original handle inside `input.snapshot` and
|
|
2161
|
+
* hands back this attempt's pre-image. It runs only here, past the
|
|
2162
|
+
* journal and cache lookups, so a replayed attempt never touches
|
|
2163
|
+
* the boundary. An adopted attempt keeps the handle its dead
|
|
2164
|
+
* incarnation persisted: that one predates any partial mutation.
|
|
2165
|
+
* Every meta write below carries it, so it reaches the row before
|
|
2166
|
+
* the body can run.
|
|
2167
|
+
*/
|
|
2168
|
+
const boundaryHandle = input.snapshot === undefined ? undefined : yield* input.snapshot
|
|
2169
|
+
const boundarySnapshot = runningMeta?.boundarySnapshot ?? (
|
|
2170
|
+
input.snapshot === undefined
|
|
2171
|
+
? undefined
|
|
2172
|
+
: boundaryHandle === undefined
|
|
2173
|
+
? {}
|
|
2174
|
+
: { handle: boundaryHandle }
|
|
2175
|
+
)
|
|
2176
|
+
const boundaryMeta = boundarySnapshot === undefined ? {} : { boundarySnapshot }
|
|
2177
|
+
if (adopted && runningMeta?.snapshotId !== undefined) {
|
|
2178
|
+
// The dead incarnation persisted this attempt's own pre-image
|
|
2179
|
+
// before mutating the workspace (issue #87): restore it so the
|
|
2180
|
+
// re-execution runs on the clean tree, and keep it as the attempt's
|
|
2181
|
+
// compensation baseline instead of snapshotting the dirty state.
|
|
2182
|
+
yield* jj.restore(runningMeta.snapshotId)
|
|
2183
|
+
snapshotId = runningMeta.snapshotId
|
|
2184
|
+
const fields = snapshotFields(snapshotId, runningMeta.snapshotOperationId)
|
|
2185
|
+
snapshotMeta = { ...fields.meta, ...boundaryMeta }
|
|
2186
|
+
// Re-announcing a pre-image the row already records durably:
|
|
2187
|
+
// there is no state write to pair this announcement with.
|
|
2188
|
+
yield* announceSnapshot(fields)
|
|
2189
|
+
} else {
|
|
2190
|
+
if (input.attempt > 1) {
|
|
2191
|
+
const previous = yield* attempts.get({ ...attemptId, attempt: input.attempt - 1 })
|
|
2192
|
+
if (
|
|
2193
|
+
Option.isSome(previous) &&
|
|
2194
|
+
decodeMeta(previous.value.meta)?.snapshotId !== undefined
|
|
2195
|
+
) {
|
|
2196
|
+
yield* jj.restore(decodeMeta(previous.value.meta)!.snapshotId!)
|
|
2197
|
+
}
|
|
2198
|
+
}
|
|
2199
|
+
const snapshot = yield* jj.snapshot(`smithers action ${stepKeyDigest} attempt ${input.attempt}`)
|
|
2200
|
+
// The immutable commit id: a retry or an adoption restores this
|
|
2201
|
+
// pointer, and a change id would follow an agent's rewrite of the
|
|
2202
|
+
// pre-image change (`jj squash`) and restore the step's edits.
|
|
2203
|
+
snapshotId = snapshot.commitId
|
|
2204
|
+
const fields = snapshotFields(snapshotId, snapshot.operationId)
|
|
2205
|
+
snapshotMeta = { ...fields.meta, ...boundaryMeta }
|
|
2206
|
+
// Persist the pre-image into the running row before announcing it
|
|
2207
|
+
// (issue #87): a SIGKILL mid-attempt must not lose the only
|
|
2208
|
+
// reference to the clean tree, or adoption re-executes on top of
|
|
2209
|
+
// the dead incarnation's partial mutations. The announcement
|
|
2210
|
+
// shares the patch's transaction, so the journal never names a
|
|
2211
|
+
// pre-image the row does not carry. The Jj snapshot itself stays
|
|
2212
|
+
// outside: a host call must never run inside a write transaction.
|
|
2213
|
+
yield* atomically(
|
|
2214
|
+
attempts.patch(attemptId, {
|
|
2215
|
+
meta: attemptPayload(
|
|
2216
|
+
{ ...declarationMeta, admittedBy: deps.owner, ...snapshotMeta } satisfies AttemptMeta
|
|
2217
|
+
)
|
|
2218
|
+
}, deps.owner).pipe(Effect.andThen(announceSnapshot(fields)))
|
|
2219
|
+
)
|
|
2220
|
+
}
|
|
2221
|
+
}
|
|
2222
|
+
|
|
2223
|
+
/**
|
|
2224
|
+
* Settles an attempt its boundary refused and rethrows the refusal.
|
|
2225
|
+
* Preparation, sandbox open, and settlement can each refuse, and all
|
|
2226
|
+
* three are the same durable transition: a `failed` row marked
|
|
2227
|
+
* `hardViolation`, committed with the hard-violation record and the
|
|
2228
|
+
* failed finish. A lost fence self-interrupts instead.
|
|
2229
|
+
*
|
|
2230
|
+
* A boundary exists only for sealed work, while snapshots are
|
|
2231
|
+
* created only for compensable work. The two capabilities are
|
|
2232
|
+
* disjoint, so a refused attempt never carries a snapshot.
|
|
2233
|
+
*/
|
|
2234
|
+
const failBoundaryAttempt = (cause: Cause.Cause<unknown>) =>
|
|
2235
|
+
Effect.gen(function*() {
|
|
2236
|
+
const finishedAtMs = yield* Clock.currentTimeMillis.pipe(Effect.map(Math.floor))
|
|
2237
|
+
const finished = yield* settleAttempt({
|
|
2238
|
+
...attemptId,
|
|
2239
|
+
state: "failed",
|
|
2240
|
+
finishedAtMs,
|
|
2241
|
+
error: attemptPayload(persistCause(cause)),
|
|
2242
|
+
meta: { ...declarationMeta, hardViolation: true }
|
|
2243
|
+
}, [
|
|
2244
|
+
JournalRecords.hardViolation(attemptSource("hard-violation"), { ...attemptId, error: cause }),
|
|
2245
|
+
JournalRecords.attemptFinished(attemptSource("finished"), { ...attemptId, state: "failed" })
|
|
2246
|
+
])
|
|
2247
|
+
if (!finished) {
|
|
2248
|
+
return yield* Effect.interrupt
|
|
2249
|
+
}
|
|
2250
|
+
return yield* Effect.failCause(cause)
|
|
2251
|
+
})
|
|
2252
|
+
const boundary = input.tier === "sealed" && input.metadata !== undefined
|
|
2253
|
+
? yield* StepBoundary.StepBoundary
|
|
2254
|
+
: undefined
|
|
2255
|
+
const preparedResult = boundary === undefined || input.metadata === undefined
|
|
2256
|
+
? undefined
|
|
2257
|
+
: yield* boundary.prepare(input.metadata).pipe(Effect.exit)
|
|
2258
|
+
if (preparedResult !== undefined && Exit.isFailure(preparedResult)) {
|
|
2259
|
+
return yield* failBoundaryAttempt(preparedResult.cause)
|
|
2260
|
+
}
|
|
2261
|
+
const prepared = preparedResult === undefined ? undefined : preparedResult.value
|
|
2262
|
+
|
|
2263
|
+
/**
|
|
2264
|
+
* THE ISOLATED EXECUTION (this lane). A sealed action carrying a
|
|
2265
|
+
* boundary descriptor runs inside a workspace transaction when one
|
|
2266
|
+
* is composed: the body observes only its declared read set, its
|
|
2267
|
+
* writes become a diff bundle, and the host is untouched until
|
|
2268
|
+
* copy-back. That is what makes whole-tree write verification
|
|
2269
|
+
* structural rather than inferred — and therefore what lets a
|
|
2270
|
+
* production-composed result enter the shared cache at all.
|
|
2271
|
+
*
|
|
2272
|
+
* The service is optional. Without it the body runs directly
|
|
2273
|
+
* against the host, exactly as before, and its evidence keeps the
|
|
2274
|
+
* honest omission that withholds it from the shared cache.
|
|
2275
|
+
*/
|
|
2276
|
+
const stepSandbox = boundary === undefined || input.metadata === undefined
|
|
2277
|
+
? Option.none<StepSandbox.Service>()
|
|
2278
|
+
: yield* Effect.serviceOption(StepSandbox.StepSandbox)
|
|
2279
|
+
const opened = Option.isSome(stepSandbox) ? yield* stepSandbox.value.open.pipe(Effect.exit) : undefined
|
|
2280
|
+
if (opened !== undefined && Exit.isFailure(opened)) {
|
|
2281
|
+
// A host that cannot isolate (`layerNoop`, a refusing forest) is a
|
|
2282
|
+
// typed refusal, not a crash: settle the attempt, or the row stays
|
|
2283
|
+
// "running" and reads as an abandoned attempt to the reclaim
|
|
2284
|
+
// machinery.
|
|
2285
|
+
return yield* failBoundaryAttempt(opened.cause)
|
|
2286
|
+
}
|
|
2287
|
+
const sandbox = boundary === undefined || input.metadata === undefined
|
|
2288
|
+
? undefined
|
|
2289
|
+
: opened !== undefined
|
|
2290
|
+
? opened.value
|
|
2291
|
+
: Option.getOrUndefined(yield* Effect.serviceOption(WorkspaceSandbox.WorkspaceSandbox))
|
|
2292
|
+
const isolated = sandbox === undefined || input.metadata === undefined
|
|
2293
|
+
? undefined
|
|
2294
|
+
: yield* SandboxedExecution.execute({
|
|
2295
|
+
sandbox,
|
|
2296
|
+
descriptor: input.metadata,
|
|
2297
|
+
workflow: execute(input, stepKeyDigest)
|
|
2298
|
+
}).pipe(Effect.exit)
|
|
2299
|
+
// The settlement is the isolated execution's whole story; the
|
|
2300
|
+
// attempt's outcome is only its `result`, so the ordinary failure
|
|
2301
|
+
// handling below is unchanged by which path produced it.
|
|
2302
|
+
const settlement = isolated !== undefined && Exit.isSuccess(isolated) ? isolated.value : undefined
|
|
2303
|
+
/**
|
|
2304
|
+
* THE EFFECT BOUNDARY. An irreversible dispatch can change the world
|
|
2305
|
+
* outside this journal, and a compensable one mutates the workspace
|
|
2306
|
+
* a rewind must restore — both are wrapped: `intended` commits
|
|
2307
|
+
* before the body starts, and the terminal record commits after it
|
|
2308
|
+
* settles — `succeeded` with the recorded result, `unknown` for a
|
|
2309
|
+
* failure, defect, or interruption whose external outcome nobody can
|
|
2310
|
+
* testify to.
|
|
2311
|
+
*
|
|
2312
|
+
* The compensable record is what makes the tier-2 restore REAL: a
|
|
2313
|
+
* rewind classifies the doomed suffix by its boundary rows, so a
|
|
2314
|
+
* compensable action that recorded only its pre-image snapshot left
|
|
2315
|
+
* nothing for the rewind to restore against — the suffix archived
|
|
2316
|
+
* "completed" while the tree kept the discarded future's bytes. The
|
|
2317
|
+
* record names the attempt's anchored `changeId` so the evidence and
|
|
2318
|
+
* the pointer travel together.
|
|
2319
|
+
*
|
|
2320
|
+
* The settlement is uninterruptible: cancellation must not strand an
|
|
2321
|
+
* effect that has already crossed without at least attempting to say
|
|
2322
|
+
* so. Sealed work is deliberately outside this — a sealed result is
|
|
2323
|
+
* cache evidence, and replay answers it from the recorded cache
|
|
2324
|
+
* entry rather than an operator decision.
|
|
2325
|
+
*/
|
|
2326
|
+
const effect = input.tier === "irreversible" || input.tier === "compensable"
|
|
2327
|
+
? {
|
|
2328
|
+
id: `${deps.runId}:${stepKeyDigest}:${input.attempt}`,
|
|
2329
|
+
kind: actionKind(input.action),
|
|
2330
|
+
tier: input.tier,
|
|
2331
|
+
runId: deps.runId,
|
|
2332
|
+
lineageId,
|
|
2333
|
+
sourceId: deps.sourceId,
|
|
2334
|
+
attempt: input.attempt,
|
|
2335
|
+
...(deps.idempotencyKey === undefined ? {} : { idempotencyKey: deps.idempotencyKey }),
|
|
2336
|
+
...(snapshotId === undefined ? {} : { changeId: snapshotId })
|
|
2337
|
+
} satisfies EffectRecords.Descriptor
|
|
2338
|
+
: undefined
|
|
2339
|
+
/**
|
|
2340
|
+
* Whether a failing exit ends this dispatch in a durable PARK rather
|
|
2341
|
+
* than a settlement (N-08). A body that reaches a wait point marks
|
|
2342
|
+
* its instance suspended and interrupts, so an interrupt-only exit
|
|
2343
|
+
* under a suspended instance is a run that parked. Quarantine from
|
|
2344
|
+
* a nested action also parks its wrapper: resume must re-enter it
|
|
2345
|
+
* to consume the inner action's preserved outcome. Both the effect
|
|
2346
|
+
* boundary and the attempt row read it: a park closes neither.
|
|
2347
|
+
*/
|
|
2348
|
+
const parked = (cause: Cause.Cause<unknown>) =>
|
|
2349
|
+
Effect.map(
|
|
2350
|
+
Effect.serviceOption(FlowRuntime.FlowInstance),
|
|
2351
|
+
(instance) =>
|
|
2352
|
+
evidenceQuarantined(cause) !== undefined ||
|
|
2353
|
+
(Option.getOrUndefined(instance)?.suspended === true && Cause.hasInterruptsOnly(cause))
|
|
2354
|
+
)
|
|
2355
|
+
/**
|
|
2356
|
+
* The row's meta as this dispatch last wrote it. The crossing writes
|
|
2357
|
+
* below patch `meta` wholesale, so everything the attempt already
|
|
2358
|
+
* carries — the adopted incarnation's fields, the tier declaration,
|
|
2359
|
+
* the compensable pre-image — has to be carried across with it.
|
|
2360
|
+
*/
|
|
2361
|
+
const dispatchMeta = {
|
|
2362
|
+
...runningMeta,
|
|
2363
|
+
...declarationMeta,
|
|
2364
|
+
admittedBy: deps.owner,
|
|
2365
|
+
...snapshotMeta
|
|
2366
|
+
} satisfies AttemptMeta
|
|
2367
|
+
/**
|
|
2368
|
+
* Commits a crossing into the attempt row, with the boundary record
|
|
2369
|
+
* that describes it when there is one, as ONE transaction.
|
|
2370
|
+
*
|
|
2371
|
+
* The row is what recovery reads (see `AttemptMeta.effectCrossing`),
|
|
2372
|
+
* so a crash must never leave the two disagreeing: a record without
|
|
2373
|
+
* the row would re-dispatch a crossed effect, and a row without the
|
|
2374
|
+
* record would settle a run from evidence the audit cannot show. A
|
|
2375
|
+
* refused patch is a lost fence, surfaced as self-interruption like
|
|
2376
|
+
* every other fence outcome here, and reported from OUTSIDE the
|
|
2377
|
+
* transaction so the transaction's own failures stay storage
|
|
2378
|
+
* failures.
|
|
2379
|
+
*/
|
|
2380
|
+
const recordCrossing = (
|
|
2381
|
+
crossing: NonNullable<AttemptMeta["effectCrossing"]>,
|
|
2382
|
+
patch: AttemptStore.AttemptPatch,
|
|
2383
|
+
record?: JournalEvent.Input
|
|
2384
|
+
) =>
|
|
2385
|
+
Effect.flatMap(
|
|
2386
|
+
atomically(Effect.gen(function*() {
|
|
2387
|
+
if (record !== undefined) {
|
|
2388
|
+
yield* emitLifecycle(record)
|
|
2389
|
+
}
|
|
2390
|
+
const patched = yield* attempts.patch(
|
|
2391
|
+
attemptId,
|
|
2392
|
+
{
|
|
2393
|
+
...patch,
|
|
2394
|
+
meta: attemptPayload({ ...dispatchMeta, effectCrossing: crossing } satisfies AttemptMeta)
|
|
2395
|
+
},
|
|
2396
|
+
deps.owner
|
|
2397
|
+
)
|
|
2398
|
+
return patched._tag === "Patched"
|
|
2399
|
+
})),
|
|
2400
|
+
(recorded) => recorded ? Effect.void : Effect.interrupt
|
|
2401
|
+
)
|
|
2402
|
+
const dispatch = effect === undefined
|
|
2403
|
+
? execute(input, stepKeyDigest)
|
|
2404
|
+
: Effect.uninterruptibleMask((restore) =>
|
|
2405
|
+
Effect.gen(function*() {
|
|
2406
|
+
yield* recordCrossing("intended", {}, EffectRecords.boundary(effect, "intended"))
|
|
2407
|
+
const exit = yield* Effect.exit(restore(execute(input, stepKeyDigest)))
|
|
2408
|
+
yield* Exit.isSuccess(exit)
|
|
2409
|
+
? recordCrossing(
|
|
2410
|
+
"succeeded",
|
|
2411
|
+
{ outcome: attemptPayload(exit.value) },
|
|
2412
|
+
EffectRecords.boundary(effect, "succeeded", exit.value)
|
|
2413
|
+
)
|
|
2414
|
+
// A park leaves the boundary OPEN. The terminal record says
|
|
2415
|
+
// the effect finished and nobody can testify to how; a parked
|
|
2416
|
+
// dispatch has not finished, and the next drive re-enters the
|
|
2417
|
+
// SAME attempt. Writing it here made a routine park and
|
|
2418
|
+
// resume read `intended, unknown, intended, succeeded` in the
|
|
2419
|
+
// journal a rewind classifies a doomed suffix from.
|
|
2420
|
+
//
|
|
2421
|
+
// The park is still recorded in the ROW, with no record to
|
|
2422
|
+
// pair it with: a park and a SIGKILL mid-body leave the same
|
|
2423
|
+
// `running` row and the same open boundary, and only this
|
|
2424
|
+
// says which one happened. Without it the resumed wait looked
|
|
2425
|
+
// like an unresolved crossing and the refusal above made every
|
|
2426
|
+
// keyless irreversible wait unresumable.
|
|
2427
|
+
: Effect.flatMap(
|
|
2428
|
+
parked(exit.cause),
|
|
2429
|
+
(isPark) =>
|
|
2430
|
+
isPark
|
|
2431
|
+
? recordCrossing("parked", {})
|
|
2432
|
+
: emitLifecycle(EffectRecords.boundary(effect, "unknown")).pipe(
|
|
2433
|
+
Effect.catchCause((cause) =>
|
|
2434
|
+
Effect.logWarning("engine-store: failed to persist unknown effect boundary", cause).pipe(
|
|
2435
|
+
Effect.annotateLogs({ runId: deps.runId, step: input.key })
|
|
2436
|
+
)
|
|
2437
|
+
)
|
|
2438
|
+
)
|
|
2439
|
+
)
|
|
2440
|
+
return yield* exit
|
|
2441
|
+
})
|
|
2442
|
+
)
|
|
2443
|
+
const outcome = isolated === undefined
|
|
2444
|
+
? yield* dispatch.pipe(Effect.exit)
|
|
2445
|
+
: Exit.map(isolated, (settled) => settled.result)
|
|
2446
|
+
if (Exit.isFailure(outcome)) {
|
|
2447
|
+
/**
|
|
2448
|
+
* A DURABLE PARK IS NOT A SETTLEMENT (N-08). A body that reaches a
|
|
2449
|
+
* wait point ends its fiber by self-interruption — `Flow.suspend`
|
|
2450
|
+
* marks the instance and interrupts — so an interrupt-only exit
|
|
2451
|
+
* from a dispatch whose instance is suspended describes a run that
|
|
2452
|
+
* parked, not an attempt that failed. Settling it below wrote a
|
|
2453
|
+
* `failed` row carrying the interrupt cause, and the replay branch
|
|
2454
|
+
* above then rethrew that cause on every later drive: the run died
|
|
2455
|
+
* with "All fibers interrupted without error" instead of resuming.
|
|
2456
|
+
* The memory engine kept no attempt row and so never saw it, which
|
|
2457
|
+
* is why `HumanTask.timeoutMs`, `Action.raceAll`, and every durable
|
|
2458
|
+
* wait raced against a timer worked in memory and not on SQLite.
|
|
2459
|
+
*
|
|
2460
|
+
* The row therefore stays `running`, which is exactly the shape the
|
|
2461
|
+
* adoption path above re-enters: the next drive re-executes the
|
|
2462
|
+
* body under the SAME attempt number, so the park costs nothing
|
|
2463
|
+
* against the retry budget and the raced deferreds re-register
|
|
2464
|
+
* against their persisted completions. `running` is also the only
|
|
2465
|
+
* in-progress state the attempt store admits, so a later settlement
|
|
2466
|
+
* of this attempt is an ordinary fenced `finish`.
|
|
2467
|
+
*/
|
|
2468
|
+
if (yield* parked(outcome.cause)) {
|
|
2469
|
+
return yield* Effect.failCause(outcome.cause)
|
|
2470
|
+
}
|
|
2471
|
+
const finishedAtMs = yield* Clock.currentTimeMillis.pipe(Effect.map(Math.floor))
|
|
2472
|
+
// A boundary violation raised while the body ran is classified
|
|
2473
|
+
// like one raised at settle time (issue #109): the row records it
|
|
2474
|
+
// so a post-crash replay can re-emit the violation record, which
|
|
2475
|
+
// the persisted cause alone cannot distinguish.
|
|
2476
|
+
const violation = declarationViolated(outcome.cause)
|
|
2477
|
+
const finished = yield* settleAttempt({
|
|
2478
|
+
...attemptId,
|
|
2479
|
+
state: "failed",
|
|
2480
|
+
finishedAtMs,
|
|
2481
|
+
error: attemptPayload(persistCause(outcome.cause)),
|
|
2482
|
+
meta: attemptPayload(
|
|
2483
|
+
{
|
|
2484
|
+
...declarationMeta,
|
|
2485
|
+
...(violation ? { hardViolation: true as const } : {}),
|
|
2486
|
+
...snapshotMeta
|
|
2487
|
+
} satisfies AttemptMeta
|
|
2488
|
+
)
|
|
2489
|
+
}, [
|
|
2490
|
+
...(violation
|
|
2491
|
+
? [
|
|
2492
|
+
JournalRecords.hardViolation(attemptSource("hard-violation"), {
|
|
2493
|
+
...attemptId,
|
|
2494
|
+
error: outcome.cause
|
|
2495
|
+
})
|
|
2496
|
+
]
|
|
2497
|
+
: []),
|
|
2498
|
+
JournalRecords.attemptFinished(attemptSource("finished"), { ...attemptId, state: "failed" })
|
|
2499
|
+
])
|
|
2500
|
+
if (!finished) {
|
|
2501
|
+
return yield* Effect.interrupt
|
|
2502
|
+
}
|
|
2503
|
+
return yield* Effect.failCause(outcome.cause)
|
|
2504
|
+
}
|
|
2505
|
+
if (settlement !== undefined) {
|
|
2506
|
+
// Forensics requires both halves as journal facts, never inferred
|
|
2507
|
+
// from an absence: what the
|
|
2508
|
+
// transaction proposed, and that it reached the host.
|
|
2509
|
+
yield* emitConverging(
|
|
2510
|
+
JournalRecords.diffBundleCaptured(attemptSource("diff-bundle"), {
|
|
2511
|
+
...attemptId,
|
|
2512
|
+
bundleIdentity: settlement.bundleIdentity,
|
|
2513
|
+
changedPaths: settlement.files.map((change) => change.path),
|
|
2514
|
+
deviations: settlement.deviations
|
|
2515
|
+
})
|
|
2516
|
+
)
|
|
2517
|
+
// THE DISPATCH STAGE. Queued effects fire here — after copy-back
|
|
2518
|
+
// settled, outside the transaction, deduplicated by idempotency
|
|
2519
|
+
// key so a body that queued the same key twice, and a bundle that
|
|
2520
|
+
// rebased before it landed, both send exactly once.
|
|
2521
|
+
const dispatcher = Option.getOrUndefined(
|
|
2522
|
+
yield* Effect.serviceOption(WorkspaceSandbox.EffectDispatcher)
|
|
2523
|
+
)
|
|
2524
|
+
const deduped = new Set<string>()
|
|
2525
|
+
for (const queued of settlement.effects) {
|
|
2526
|
+
if (deduped.has(queued.idempotencyKey)) continue
|
|
2527
|
+
deduped.add(queued.idempotencyKey)
|
|
2528
|
+
if (dispatcher !== undefined) yield* dispatcher.dispatch(queued)
|
|
2529
|
+
}
|
|
2530
|
+
yield* emitConverging(
|
|
2531
|
+
JournalRecords.copyBackSettled(attemptSource("copy-back"), {
|
|
2532
|
+
...attemptId,
|
|
2533
|
+
bundleIdentity: settlement.bundleIdentity,
|
|
2534
|
+
rebases: settlement.rebases,
|
|
2535
|
+
queued: [...deduped],
|
|
2536
|
+
dispatched: dispatcher === undefined ? [] : [...deduped]
|
|
2537
|
+
})
|
|
2538
|
+
)
|
|
2539
|
+
}
|
|
2540
|
+
|
|
2541
|
+
const settled = prepared === undefined || boundary === undefined
|
|
2542
|
+
? undefined
|
|
2543
|
+
: yield* boundary.settle(prepared).pipe(Effect.exit)
|
|
2544
|
+
yield* settled === undefined ? Effect.void : Metric.update(
|
|
2545
|
+
EngineStoreMetrics.boundarySettlement[
|
|
2546
|
+
Exit.isSuccess(settled)
|
|
2547
|
+
? settled.value.deviation === undefined ? "Clean" : "Deviation"
|
|
2548
|
+
: settlementViolated(settled.cause)
|
|
2549
|
+
? "Violation"
|
|
2550
|
+
: "Refused"
|
|
2551
|
+
],
|
|
2552
|
+
1
|
|
2553
|
+
)
|
|
2554
|
+
if (settled !== undefined && Exit.isFailure(settled)) {
|
|
2555
|
+
return yield* failBoundaryAttempt(settled.cause)
|
|
2556
|
+
}
|
|
2557
|
+
const settledEvidence = settled === undefined ? undefined : settled.value
|
|
2558
|
+
/**
|
|
2559
|
+
* THE RETIRED LIMITATION. `StepBoundary`'s filesystem layer omits
|
|
2560
|
+
* `wholeTreeWritesVerified` because it can only re-measure paths it
|
|
2561
|
+
* was told about, and only `layerTest` ever set it — so under the
|
|
2562
|
+
* production composition nothing could enter the shared cache.
|
|
2563
|
+
*
|
|
2564
|
+
* An isolated execution answers the question the boundary could not:
|
|
2565
|
+
* the transaction *is* the tree, so a write outside the declared set
|
|
2566
|
+
* is a map comparison rather than an inference. The claim is made
|
|
2567
|
+
* here, by the code that knows the body ran in isolation, and only
|
|
2568
|
+
* when the whole-tree diff found no deviation — the same whole-tree
|
|
2569
|
+
* view also supplies a deviation the boundary's declared-read scan
|
|
2570
|
+
* would have missed entirely.
|
|
2571
|
+
*/
|
|
2572
|
+
const evidence = settledEvidence === undefined || settlement === undefined
|
|
2573
|
+
? settledEvidence
|
|
2574
|
+
: {
|
|
2575
|
+
...settledEvidence,
|
|
2576
|
+
...(settlement.deviations.length === 0
|
|
2577
|
+
? {
|
|
2578
|
+
wholeTreeWritesVerified: true as const,
|
|
2579
|
+
// `StepSandbox` is the injectable façade; the
|
|
2580
|
+
// `WorkspaceSandbox` service is the same isolated
|
|
2581
|
+
// transaction backend and proves the same read property.
|
|
2582
|
+
hermeticReadsVerified: true as const
|
|
2583
|
+
}
|
|
2584
|
+
: {
|
|
2585
|
+
deviation: {
|
|
2586
|
+
_tag: "ExpectedSetDeviation" as const,
|
|
2587
|
+
paths: settlement.deviations,
|
|
2588
|
+
diffIdentity: settlement.bundleIdentity
|
|
2589
|
+
}
|
|
2590
|
+
})
|
|
2591
|
+
}
|
|
2592
|
+
const finishedAtMs = yield* Clock.currentTimeMillis.pipe(Effect.map(Math.floor))
|
|
2593
|
+
// The declared read set is the key input; the prepare-time
|
|
2594
|
+
// measurement is the evidence it described reality when the body
|
|
2595
|
+
// ran (issue #106). A mismatch means the result was computed from
|
|
2596
|
+
// different inputs than the key claims — the attempt itself is
|
|
2597
|
+
// fine, but the completion must never enter the shared cache.
|
|
2598
|
+
const readSetVerified = prepared !== undefined && StepBoundary.readSetMatches(prepared)
|
|
2599
|
+
const meta: AttemptMeta = {
|
|
2600
|
+
...declarationMeta,
|
|
2601
|
+
...snapshotMeta,
|
|
2602
|
+
...(evidence === undefined ? {} : { boundary: evidence }),
|
|
2603
|
+
...(readSetVerified ? { readSetVerified: true as const } : {})
|
|
2604
|
+
}
|
|
2605
|
+
const finished = yield* settleAttempt(
|
|
2606
|
+
{
|
|
2607
|
+
...attemptId,
|
|
2608
|
+
state: "succeeded",
|
|
2609
|
+
finishedAtMs,
|
|
2610
|
+
outcome: attemptPayload(outcome.value),
|
|
2611
|
+
meta: attemptPayload(meta)
|
|
2612
|
+
},
|
|
2613
|
+
[
|
|
2614
|
+
...(evidence?.deviation === undefined ? [] : [
|
|
2615
|
+
JournalRecords.expectedSetDeviation(attemptSource("deviation"), {
|
|
2616
|
+
...attemptId,
|
|
2617
|
+
...evidence.deviation
|
|
2618
|
+
})
|
|
2619
|
+
]),
|
|
2620
|
+
JournalRecords.attemptFinished(attemptSource("finished"), { ...attemptId, state: "succeeded" })
|
|
2621
|
+
]
|
|
2622
|
+
)
|
|
2623
|
+
if (!finished) return yield* Effect.interrupt
|
|
2624
|
+
|
|
2625
|
+
const admission = CacheAdmission.completion(cacheDeclaration, meta, false)
|
|
2626
|
+
if (admission._tag === "PublishCompletion") {
|
|
2627
|
+
yield* recordCache({ result: outcome.value, admission, createdAtMs: finishedAtMs })
|
|
2628
|
+
} else if (admission.reason === "unverified-recorded-reads") {
|
|
2629
|
+
// Visible, not silent (issue #106): the run continues on its
|
|
2630
|
+
// own result, but the stale declaration is journalled so the
|
|
2631
|
+
// missing cache entry is explainable.
|
|
2632
|
+
yield* emitConverging(
|
|
2633
|
+
JournalRecords.cacheProvenance(cacheSource("unverified_read_set"), {
|
|
2634
|
+
keyDigest,
|
|
2635
|
+
action: "unverified_read_set"
|
|
2636
|
+
})
|
|
2637
|
+
)
|
|
2638
|
+
}
|
|
2639
|
+
return outcome.value
|
|
2640
|
+
})
|
|
2641
|
+
)
|
|
2642
|
+
}).pipe(
|
|
2643
|
+
// The operation's own span is annotated above once the digest exists;
|
|
2644
|
+
// this ambient context gives every child store, boundary, and sandbox
|
|
2645
|
+
// span the dispatch identity as it opens.
|
|
2646
|
+
//
|
|
2647
|
+
// `stepKey`, never `key`: `@smthrs/journal`'s `Redaction.isSensitiveKey`
|
|
2648
|
+
// treats a standalone trailing `key` word as a credential name and
|
|
2649
|
+
// replaces the value, and `RedactedLogger` applies that rule to every
|
|
2650
|
+
// log annotation under a real host. The span uses the same name so a
|
|
2651
|
+
// trace and a log line name the dispatch identically.
|
|
2652
|
+
Effect.annotateSpans({
|
|
2653
|
+
runId: deps.runId,
|
|
2654
|
+
stepKey: input.key,
|
|
2655
|
+
attempt: input.attempt,
|
|
2656
|
+
tier: input.tier
|
|
2657
|
+
}),
|
|
2658
|
+
Effect.annotateLogs({
|
|
2659
|
+
runId: deps.runId,
|
|
2660
|
+
stepKey: input.key,
|
|
2661
|
+
attempt: input.attempt,
|
|
2662
|
+
tier: input.tier
|
|
2663
|
+
}),
|
|
2664
|
+
EngineStoreMetrics.observe({
|
|
2665
|
+
timer: EngineStoreMetrics.dispatchDuration,
|
|
2666
|
+
counter: EngineStoreMetrics.dispatch
|
|
2667
|
+
}),
|
|
2668
|
+
// A dispatch that settled without reaching the gate was served from a
|
|
2669
|
+
// durable record: a recorded success, or a recorded failure it rethrew.
|
|
2670
|
+
// An interruption or a defect settled nothing and reports nothing.
|
|
2671
|
+
Effect.onExit((exit) => {
|
|
2672
|
+
const replayOnly = deps.replayOnly
|
|
2673
|
+
const stepKeyDigest = replay.stepKeyDigest
|
|
2674
|
+
if (replayOnly === undefined || replay.gated || stepKeyDigest === undefined) return Effect.void
|
|
2675
|
+
if (Exit.isFailure(exit) && (Cause.hasInterrupts(exit.cause) || Cause.hasDies(exit.cause))) return Effect.void
|
|
2676
|
+
return replayOnly.observe({
|
|
2677
|
+
runId: deps.runId,
|
|
2678
|
+
stepKeyDigest,
|
|
2679
|
+
attempt: input.attempt,
|
|
2680
|
+
action: actionKind(input.action),
|
|
2681
|
+
tier: input.tier,
|
|
2682
|
+
outcome: "replayed"
|
|
2683
|
+
})
|
|
2684
|
+
})
|
|
2685
|
+
)
|
|
2686
|
+
})
|
|
2687
|
+
}
|