@sema-agent/server 7.43.0 → 7.44.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/USAGE.md +18 -0
- package/dist/adoption/plan.js +31 -192
- package/dist/adoption/quiesce.js +3 -86
- package/dist/adoption/runner.js +3 -140
- package/dist/adoption/sql.js +0 -74
- package/dist/adoption/wire.js +0 -80
- package/dist/approval-ask-machine.js +0 -75
- package/dist/approval-card.js +0 -323
- package/dist/approval-deny-reasons.js +0 -42
- package/dist/approval-hmac.js +0 -35
- package/dist/approval-reconciler.js +8 -166
- package/dist/approval.js +4 -67
- package/dist/audit.js +1 -44
- package/dist/auth-bridge.js +4 -47
- package/dist/auth-keys.js +0 -23
- package/dist/bake-runner/main.js +4 -65
- package/dist/bake-runner/protocol.js +3 -58
- package/dist/bake-runner/runner.js +5 -91
- package/dist/bench/l8/artifact.js +2 -61
- package/dist/bench/l8/escape.js +0 -25
- package/dist/bench/l8/index.js +0 -14
- package/dist/bench/l8/probes.js +0 -60
- package/dist/bench/l8/run-probes.js +2 -45
- package/dist/bench/s1/arms.js +14 -189
- package/dist/bench/s1/live-deps.js +11 -253
- package/dist/bench/s1/oracle.js +1 -36
- package/dist/bench/s1/repair-oracle-adapter.js +0 -30
- package/dist/bench/s1/reviewer.js +0 -34
- package/dist/bench/s1/row.js +0 -62
- package/dist/bench/s1/run-firm.js +4 -70
- package/dist/bench/s1/runner-ctx.js +0 -40
- package/dist/bench/s1/tasks.js +0 -115
- package/dist/boot/adoption.js +0 -20
- package/dist/boot/budget-tracing.js +3 -55
- package/dist/boot/config-center.js +43 -551
- package/dist/boot/coordinators.js +1 -69
- package/dist/boot/crash-last.js +0 -18
- package/dist/boot/deferred-sandbox-path-env.js +3 -119
- package/dist/boot/execution-env.js +5 -171
- package/dist/boot/governance-seams.js +0 -92
- package/dist/boot/leader.js +0 -69
- package/dist/boot/limit-sync.js +0 -2
- package/dist/boot/memory-boundary.js +3 -91
- package/dist/boot/org-memory.js +1 -25
- package/dist/boot/parked-revive-gate.js +0 -150
- package/dist/boot/permission-rules-audit.js +1 -56
- package/dist/boot/reapers.js +15 -273
- package/dist/boot/resolve-spec.js +9 -768
- package/dist/boot/retention-lane.js +4 -87
- package/dist/boot/runner-deps.js +3 -297
- package/dist/boot/runtime-caps.js +0 -40
- package/dist/boot/session-faces.js +2 -133
- package/dist/boot/shutdown.js +9 -83
- package/dist/boot/side-query-lane.js +2 -137
- package/dist/boot/stores.js +15 -215
- package/dist/boot/task-list-lane.js +0 -18
- package/dist/boot/webfetch-summarize-lane.js +0 -47
- package/dist/boot/workflow-orchestration.js +3 -99
- package/dist/boot-reclaim.js +0 -20
- package/dist/bounded-session-map.js +0 -19
- package/dist/brain.js +2 -139
- package/dist/budget.js +9 -244
- package/dist/capabilities/builtin-tools.js +0 -3
- package/dist/capabilities/center-plugins.js +6 -51
- package/dist/capabilities/center-prompts.js +6 -71
- package/dist/capabilities/code-review-council.js +7 -40
- package/dist/capabilities/collab-workflows.js +1 -44
- package/dist/capabilities/hands-lane.js +0 -65
- package/dist/capabilities/memory-notice.js +0 -70
- package/dist/capabilities/prompt.js +0 -14
- package/dist/capabilities/prompts/code-review.js +0 -14
- package/dist/capabilities/prompts/identity.js +0 -6
- package/dist/capabilities/prompts/team.js +0 -4
- package/dist/capabilities/repo-tools.js +0 -34
- package/dist/capabilities/sandbox-file-send.js +3 -92
- package/dist/capabilities/scenarios.d.ts +0 -1
- package/dist/capabilities/scenarios.js +18 -279
- package/dist/capabilities/select-environment-tool.js +0 -32
- package/dist/capabilities/send-user-file-tool.js +2 -66
- package/dist/capabilities/skills.d.ts +1 -2
- package/dist/capabilities/skills.js +3 -21
- package/dist/capabilities/team.d.ts +3 -11
- package/dist/capabilities/team.js +3 -51
- package/dist/capabilities/tool-defer.js +0 -3
- package/dist/config-center/apply-effective.js +15 -330
- package/dist/config-center/apply-ledger.js +5 -38
- package/dist/config-center/facade.js +0 -41
- package/dist/config-center/hot-keys-registry.js +0 -20
- package/dist/config-center/http-client.js +2 -125
- package/dist/config-center/mcp-revocation.js +2 -34
- package/dist/config-center/read-face.js +0 -59
- package/dist/config-center/restart-signal.js +1 -85
- package/dist/config-center/skills-mcp.d.ts +1 -1
- package/dist/config-center/skills-mcp.js +9 -76
- package/dist/config-center/stage-limits.js +8 -35
- package/dist/config-invariants.js +0 -16
- package/dist/config-lkg.js +0 -42
- package/dist/config-provider.js +3 -186
- package/dist/config-types.js +0 -5
- package/dist/config.js +107 -1145
- package/dist/degenerate-instrument.js +3 -67
- package/dist/deployment-governance.js +0 -124
- package/dist/digest-form.js +0 -12
- package/dist/elicitation.js +3 -86
- package/dist/env-facts.js +7 -75
- package/dist/fleet/fleet-bus.js +34 -507
- package/dist/fleet/fleet-reconciler.js +9 -149
- package/dist/fleet/fleet-terminal-window.js +10 -178
- package/dist/fleet/subagent-tail-bus.js +3 -72
- package/dist/fleet-client.js +10 -70
- package/dist/fleet-lease.js +5 -79
- package/dist/git-api-kind.js +0 -3
- package/dist/governance-ask-marks.js +2 -78
- package/dist/hooks/branch-transcript.js +0 -74
- package/dist/hooks/cc-agent-hook-prompt.js +0 -29
- package/dist/hooks/cc-stop-prompt.js +1 -46
- package/dist/hooks/hook-llm.js +1 -53
- package/dist/hooks/hook-runner.js +20 -414
- package/dist/http/active-run-conflict.js +4 -129
- package/dist/http/cursor-fingerprint.d.ts +5 -0
- package/dist/http/cursor-fingerprint.js +5 -0
- package/dist/http/idempotency.js +0 -37
- package/dist/http/principal-gate.js +3 -40
- package/dist/http/route-ctx.js +0 -9
- package/dist/http/routes/a2a-serve.js +7 -319
- package/dist/http/routes/admin-config-refresh.js +0 -4
- package/dist/http/routes/admin-drain.js +0 -10
- package/dist/http/routes/adoption.js +1 -29
- package/dist/http/routes/agents-roster.js +1 -47
- package/dist/http/routes/approvals-assistant.js +32 -395
- package/dist/http/routes/attachments.js +4 -22
- package/dist/http/routes/capabilities.js +3 -420
- package/dist/http/routes/diagnostics.js +1 -81
- package/dist/http/routes/fleet.js +9 -185
- package/dist/http/routes/images.js +14 -239
- package/dist/http/routes/leader.js +0 -13
- package/dist/http/routes/memory-bundle.js +1 -61
- package/dist/http/routes/memory-policy.js +9 -99
- package/dist/http/routes/notify-wake.js +3 -37
- package/dist/http/routes/observability.js +2 -19
- package/dist/http/routes/retention-ops.js +2 -34
- package/dist/http/routes/rules.js +1 -93
- package/dist/http/routes/runs.js +64 -873
- package/dist/http/routes/session-sync.js +19 -258
- package/dist/http/routes/sessions-list.js +17 -43
- package/dist/http/routes/sessions.js +34 -226
- package/dist/http/routes/shared-memory.js +5 -36
- package/dist/http/routes/side-query.js +1 -87
- package/dist/http/routes/tasks.js +52 -722
- package/dist/http/routes/trace-usage.js +32 -195
- package/dist/http/routes/workflows.js +19 -195
- package/dist/http/run-meta.js +0 -6
- package/dist/http/send.js +0 -32
- package/dist/http/server.js +92 -1597
- package/dist/http/sse-lifecycle.js +2 -13
- package/dist/http/sse-log.js +3 -48
- package/dist/http/tar.js +5 -21
- package/dist/http/verify-rounds.js +0 -5
- package/dist/http/wire-gate.js +0 -9
- package/dist/http/workspace-content.js +0 -10
- package/dist/images/bake-validate.js +1 -70
- package/dist/images/manifest.js +1 -6
- package/dist/index.js +0 -21
- package/dist/key-resolver.js +2 -17
- package/dist/leader/diffout.js +1 -20
- package/dist/leader/diffup.js +0 -47
- package/dist/leader/endpoint.js +2 -57
- package/dist/leader/fanout.js +3 -45
- package/dist/leader/grader-env-factory.js +3 -72
- package/dist/leader/leader.js +5 -156
- package/dist/leader/merge.js +8 -115
- package/dist/leader/planner.js +3 -54
- package/dist/leader/repair-oracle.js +1 -60
- package/dist/leader/repair-wire.js +2 -79
- package/dist/leader/wire.js +8 -307
- package/dist/lsp/e2b-bridge.js +4 -64
- package/dist/lsp/e2b-manager.js +6 -94
- package/dist/lsp/lsp-frames.js +0 -12
- package/dist/lsp/manager.js +4 -96
- package/dist/lsp/ws-transport.js +5 -55
- package/dist/lsp-evict.js +1 -15
- package/dist/main.js +57 -765
- package/dist/memory-bundle-engine.js +0 -55
- package/dist/memory-export.js +0 -4
- package/dist/memory-posture.js +1 -15
- package/dist/memory-scope.js +11 -148
- package/dist/memory-sync-client.js +2 -44
- package/dist/memory-sync.js +1 -80
- package/dist/model-select.js +3 -80
- package/dist/observability/cost-quota.js +1 -17
- package/dist/observability/cost-taxonomy.js +0 -34
- package/dist/observability/fail-open.js +7 -86
- package/dist/observability/logger.js +0 -6
- package/dist/observability/metrics.js +0 -94
- package/dist/observability/otel-exporter.js +3 -13
- package/dist/observability/principal-context.js +0 -9
- package/dist/observability/prompt-manifest.js +1 -37
- package/dist/observability/rate-limit.js +0 -4
- package/dist/observability/secret-env-scrub.js +2 -56
- package/dist/observability/tool-trace.js +1 -70
- package/dist/orchestration/hardened-vm-runner.js +4 -118
- package/dist/orchestration/hardened-vm-worker-runner.js +1 -26
- package/dist/orchestration/hardened-vm-worker.js +0 -27
- package/dist/orchestration/subagent-steer.js +1 -45
- package/dist/orchestration/workflow-agent-steer.js +1 -80
- package/dist/orchestration/workflow-completion-inbox.js +32 -285
- package/dist/orchestration/workflow-notify-journal.js +16 -259
- package/dist/org-memory-admission.js +3 -47
- package/dist/parent-watch.js +2 -48
- package/dist/parked-decide.js +1 -109
- package/dist/per-task-image.js +0 -57
- package/dist/plan-cache-probe.js +3 -27
- package/dist/plugins/adoption-log-sql.js +2 -119
- package/dist/plugins/approval-ask-store-memory.js +3 -38
- package/dist/plugins/approval-ask-store-sql.js +6 -188
- package/dist/plugins/approval-exemption-store.js +2 -28
- package/dist/plugins/background-agent-store-sql.js +4 -105
- package/dist/plugins/background-shell-support.js +14 -122
- package/dist/plugins/blob-backend.js +6 -169
- package/dist/plugins/breaker-state-sql.js +8 -46
- package/dist/plugins/caching-session-store.js +4 -106
- package/dist/plugins/checkpoint-store-sql.js +18 -505
- package/dist/plugins/e2b-orphan-reclaim.js +0 -45
- package/dist/plugins/file-outcome-sink.js +0 -9
- package/dist/plugins/file-resume-anchor-store.js +4 -43
- package/dist/plugins/file-run-store.js +26 -364
- package/dist/plugins/file-snapshot-store-sql.js +10 -181
- package/dist/plugins/fork-routing-session-store.js +8 -111
- package/dist/plugins/host-platform.js +2 -91
- package/dist/plugins/image-bake-store-sql.js +7 -250
- package/dist/plugins/image-index-sql.js +4 -123
- package/dist/plugins/k8s-bg-scripts.js +4 -88
- package/dist/plugins/k8s-exec-protocol.js +0 -41
- package/dist/plugins/leader-run-store-sql.js +0 -101
- package/dist/plugins/local-checkpoint-store.js +8 -128
- package/dist/plugins/local-session-store.js +34 -296
- package/dist/plugins/local-task-attachment-store.js +2 -16
- package/dist/plugins/mailbox-store-sql.js +8 -65
- package/dist/plugins/memory-embedder-fingerprint.js +5 -166
- package/dist/plugins/memory-embedder.js +4 -70
- package/dist/plugins/memory-engine-pg.js +6 -170
- package/dist/plugins/memory-engine-tidb.js +7 -155
- package/dist/plugins/memory-engine-vector-util.js +0 -10
- package/dist/plugins/memory-key-guards.js +0 -34
- package/dist/plugins/memory-origin-law.js +0 -187
- package/dist/plugins/memory-resume-anchor-store.js +0 -17
- package/dist/plugins/memory-run-store.js +12 -89
- package/dist/plugins/memory-session-policy-store.js +0 -17
- package/dist/plugins/memory-sync-store-pg.js +4 -49
- package/dist/plugins/memory-sync-store-tidb.js +3 -35
- package/dist/plugins/outcome-ledger-sql.js +3 -97
- package/dist/plugins/permission-rule-store-file.js +5 -133
- package/dist/plugins/permission-rule-store-sql.d.ts +3 -0
- package/dist/plugins/permission-rule-store-sql.js +50 -307
- package/dist/plugins/pg-cost-quota.js +0 -7
- package/dist/plugins/pg-pool.js +0 -92
- package/dist/plugins/pg-rate-limiter.js +2 -13
- package/dist/plugins/pg-safe-json.js +4 -40
- package/dist/plugins/pg-session-storage.js +25 -189
- package/dist/plugins/posix-shell-fs.js +1 -31
- package/dist/plugins/remote-env-adb.js +12 -101
- package/dist/plugins/remote-env-e2b.js +40 -371
- package/dist/plugins/remote-env-file-error.js +0 -37
- package/dist/plugins/remote-env-host.js +63 -483
- package/dist/plugins/remote-env-k8s.js +32 -326
- package/dist/plugins/remote-env-local-docker.js +24 -157
- package/dist/plugins/remote-env-ssh.js +20 -128
- package/dist/plugins/remote-scratchpad.js +2 -32
- package/dist/plugins/remote-shell.js +1 -32
- package/dist/plugins/resume-anchor-store-sql.js +0 -11
- package/dist/plugins/retention-lane-store-sql.js +0 -108
- package/dist/plugins/retention-store-sql.js +4 -383
- package/dist/plugins/roster-store-sql.js +0 -55
- package/dist/plugins/run-store-sql.js +14 -251
- package/dist/plugins/s3-presign.js +2 -49
- package/dist/plugins/scheduler-support.js +3 -80
- package/dist/plugins/send-file-ledger.js +4 -53
- package/dist/plugins/send-user-file.js +4 -94
- package/dist/plugins/session-placement.js +1 -89
- package/dist/plugins/session-policy-store-sql.js +3 -81
- package/dist/plugins/session-store.js +0 -57
- package/dist/plugins/shared-memory-store-sql.js +4 -167
- package/dist/plugins/sql-driver.js +0 -17
- package/dist/plugins/sql-errors.js +0 -7
- package/dist/plugins/sql-escape.js +0 -8
- package/dist/plugins/sql-row-helpers.js +0 -25
- package/dist/plugins/store-backend.js +42 -222
- package/dist/plugins/store-contracts.js +2 -46
- package/dist/plugins/task-attachment-store.js +3 -49
- package/dist/plugins/task-list-store-sql.js +0 -79
- package/dist/plugins/tidb-cost-quota.js +1 -4
- package/dist/plugins/tidb-pool.js +1 -207
- package/dist/plugins/tidb-rate-limiter.js +3 -9
- package/dist/plugins/tidb-session-storage.js +4 -70
- package/dist/plugins/tidb-session-store.js +19 -341
- package/dist/plugins/tool-result-store-sql.js +4 -179
- package/dist/plugins/usage-window-store-sql.js +0 -8
- package/dist/plugins/web-search.js +10 -132
- package/dist/plugins/workflow-journal-store-sql.js +2 -58
- package/dist/plugins/workflow-run-store-sql.js +6 -91
- package/dist/plugins/worktree-isolation.js +6 -126
- package/dist/plugins/write-behind-counter.js +16 -75
- package/dist/principal-jwt.js +5 -60
- package/dist/project-memory.js +15 -146
- package/dist/prompts-domain-validate.js +1 -51
- package/dist/question.js +2 -111
- package/dist/resource-suspend.js +0 -18
- package/dist/router/route-orchestration.js +0 -77
- package/dist/rules-consent.d.ts +44 -5
- package/dist/rules-consent.js +75 -215
- package/dist/run-local.js +14 -381
- package/dist/runs.js +29 -617
- package/dist/runtime-caps-resolver.js +6 -132
- package/dist/runtime-governance.js +1 -232
- package/dist/sandbox-pkg-source.js +0 -37
- package/dist/sealed-key.js +3 -68
- package/dist/security.js +6 -291
- package/dist/session-leaf-bus.js +0 -32
- package/dist/session-sync-content.js +1 -69
- package/dist/session-sync-kernel.js +3 -60
- package/dist/session-sync.js +3 -66
- package/dist/session-titler.js +7 -41
- package/dist/session-watch.js +9 -73
- package/dist/shared-memory-scope-authorizer.js +0 -18
- package/dist/sighup-idle.js +1 -12
- package/dist/spec-fields.js +4 -143
- package/dist/store-live-probe.js +3 -49
- package/dist/task-a2a.js +1 -126
- package/dist/task-cwd.js +1 -103
- package/dist/task-mcp.js +1 -89
- package/dist/task-settings.js +10 -300
- package/dist/task-workflow.js +8 -75
- package/dist/tool-approval.d.ts +28 -1
- package/dist/tool-approval.js +70 -1430
- package/dist/trace/artifacts.js +5 -20
- package/dist/trace/engine-notice-wire.js +6 -138
- package/dist/trace/ledger-sink.js +7 -96
- package/dist/trace/project.js +4 -448
- package/dist/trace/redact.js +9 -77
- package/dist/turn-activity.js +1 -27
- package/dist/usage-analytics.js +4 -34
- package/dist/wall-clock-jump-guard.js +1 -68
- package/package.json +3 -3
- package/dist/capabilities/scenario-alias.d.ts +0 -27
- package/dist/capabilities/scenario-alias.js +0 -61
|
@@ -1,52 +1,7 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* SVC-1 (core contract) — the AT-LEAST-ONCE workflow completion-notify TRUST GATE.
|
|
3
|
-
*
|
|
4
|
-
* ## Why this exists
|
|
5
|
-
* core's `run_workflow` tool already does the BACKGROUND-RUN half of SVC-1: `startWorkflow` returns a `runId`
|
|
6
|
-
* synchronously (the caller never blocks), the workflow runs to terminal in the background, and on terminal the
|
|
7
|
-
* tool fires a single in-process completion notify through the {@link import("@sema-agent/core").WorkflowCompletionNotifier}
|
|
8
|
-
* seam (see core `run-workflow-tool.js`). That notify is **at-MOST-once**: it lives only in the process that
|
|
9
|
-
* started the run. If the replica CRASHES between the workflow reaching terminal and the receiver acking the
|
|
10
|
-
* notification — or if it crashes before the run even finishes — the notification is LOST. Nothing re-derives it.
|
|
11
|
-
*
|
|
12
|
-
* SVC-1 makes the completion notify **at-LEAST-once**, the half that is service's (design boundary
|
|
13
|
-
* [[core-service-boundary]] — the durable wiring + the deployment notify route is TOB, not core):
|
|
14
|
-
* 1. **terminal → notify** (steady state): every started run is journaled as `pending`; when its in-process
|
|
15
|
-
* notify fires, we record the terminal status durably, DELIVER to the wrapped notifier, and mark the journal
|
|
16
|
-
* entry `acked`. Delivery before ack ⇒ a crash mid-delivery re-delivers on recovery (never a silent drop).
|
|
17
|
-
* 2. **crash-recovery → re-derive + notify** (startup): on boot we read the journal for every NOT-`acked` entry
|
|
18
|
-
* and RE-FETCH the run from the durable {@link import("@sema-agent/core").WorkflowRunStore}. A run that is now
|
|
19
|
-
* TERMINAL (its in-process notify was lost to the crash, or it finished while we were down) is RE-DELIVERED
|
|
20
|
-
* and acked. A run still `running` is left pending — the live process owns its terminal notify, and the next
|
|
21
|
-
* recovery sweep (or that process's own delivery) will catch it. A run MISSING from the store (reaped /
|
|
22
|
-
* never persisted) is acked-as-abandoned so the journal does not grow unbounded chasing a ghost.
|
|
23
|
-
* 3. **idempotent**: the journal de-dups by `runId` (a delivered+acked run is never re-delivered), so the
|
|
24
|
-
* wrapped notifier — and the downstream receiver — may safely be invoked twice for the same run across a
|
|
25
|
-
* crash boundary (the SVC-1 contract: the receiver is idempotent on `runId`). We dedup on OUR side too, so a
|
|
26
|
-
* double recovery sweep, or a recovery racing the live notify, delivers at most one extra time, not N.
|
|
27
|
-
*
|
|
28
|
-
* ## Durability model
|
|
29
|
-
* The journal is itself a {@link WorkflowNotifyJournalStore} — a tiny append-only `(runId → entry)` durable map,
|
|
30
|
-
* deliberately SEPARATE from the `WorkflowRunStore`: the run store is core-owned (its schema/CAS is core's), and
|
|
31
|
-
* critically it has **no cross-scope enumeration** (`listByScope` needs a scope; there is no "list all runs" /
|
|
32
|
-
* "list all scopes"), so it cannot by itself answer "which runs might still owe a notify?" after a restart. The
|
|
33
|
-
* journal records `(runId, scope)` for exactly that: a bounded, service-owned to-do list of notifications, scoped
|
|
34
|
-
* to the deployment, that the recovery sweep walks. It is the SINGLE source of truth for "has this run's
|
|
35
|
-
* completion been delivered?".
|
|
36
|
-
*
|
|
37
|
-
* The default {@link FileWorkflowNotifyJournalStore} is zero-dependency + crash-safe (atomic-append ledger, same
|
|
38
|
-
* posture as core's `FileWorkflowRunStore`) so the no-DB / TOC-local path is durable out of the box. A durable
|
|
39
|
-
* cross-replica backend (TiDB/PG) is a thin port of the same interface (a follow-on; the recovery sweep + the
|
|
40
|
-
* gate logic here are backend-agnostic).
|
|
41
|
-
*/
|
|
42
1
|
import { mkdirSync, readFileSync, existsSync, openSync, writeSync, fsyncSync, closeSync } from "node:fs";
|
|
43
2
|
import { join } from "node:path";
|
|
44
3
|
import { buildFleetWorkflowRow } from "../fleet/fleet-bus.js";
|
|
45
4
|
import { isTerminalWorkflowRowStatus, seedTerminalWorkflowRow } from "../fleet/fleet-terminal-window.js";
|
|
46
|
-
/** Build a bounded + redacted completion summary from a durable run's terminal state (the recovery path — the
|
|
47
|
-
* live path uses core's own `boundedSummary`). We never have the raw result here (the run store keeps the run,
|
|
48
|
-
* not the originator's payload), so we synthesize a conservative one: status + the run's already-bounded error.
|
|
49
|
-
* Mirrors core `failureSummary`/`redactSummary` posture — strip URLs/tokens/paths defensively. */
|
|
50
5
|
const SUMMARY_MAX = 500;
|
|
51
6
|
export function recoverySummary(status, error) {
|
|
52
7
|
if (status === "completed")
|
|
@@ -57,24 +12,9 @@ export function recoverySummary(status, error) {
|
|
|
57
12
|
.replace(/\b(?:Bearer|Basic)\s+[A-Za-z0-9._~+/=-]+/gi, "[redacted-token]")
|
|
58
13
|
.replace(/\b(?:api[_-]?key|secret|token|password|authorization)\b\s*[:=]\s*"?[^\s"',}]+/gi, "[redacted-secret]")
|
|
59
14
|
.replace(/(?:\/[A-Za-z0-9_.-]+){2,}/g, "[redacted-path]")
|
|
60
|
-
// Windows/UNC paths too (SVC-1 review): `C:\dir\file` + `\\host\share\…` would otherwise surface VERBATIM — a
|
|
61
|
-
// host-path/identity leak the forward-slash rule above misses. `\\{1,2}` covers the UNC double-leading-backslash.
|
|
62
15
|
.replace(/(?:[A-Za-z]:)?(?:\\{1,2}[A-Za-z0-9_.$-]+){2,}/g, "[redacted-path]");
|
|
63
16
|
return redacted.length > SUMMARY_MAX ? `${redacted.slice(0, SUMMARY_MAX)}…[+${redacted.length - SUMMARY_MAX} chars]` : redacted;
|
|
64
17
|
}
|
|
65
|
-
/**
|
|
66
|
-
* The AT-LEAST-ONCE completion-notify gate. Wrap the deployment's real delivery once at boot, then:
|
|
67
|
-
* - {@link buildNotifier} hands core a {@link import("@sema-agent/core").WorkflowCompletionNotifier} whose `notify`
|
|
68
|
-
* runs the LIVE deliver-then-ack path (the in-process at-most-once notify, now journaled into at-least-once).
|
|
69
|
-
* - {@link onWorkflowStart} journals a run the moment it starts (so a crash BEFORE its terminal notify still
|
|
70
|
-
* leaves a pending entry the recovery sweep finds). Call it from wherever the runId becomes known.
|
|
71
|
-
* - {@link recover} runs ONCE at boot: re-derives every pending run's terminal state from the
|
|
72
|
-
* {@link WorkflowRunStore} and re-delivers + acks the ones that are terminal.
|
|
73
|
-
*
|
|
74
|
-
* The gate is the SINGLE writer of `acked`, and dedups every delivery against the journal, so the wrapped
|
|
75
|
-
* delivery sink is invoked at most once per run in steady state (and at most one extra time across a crash —
|
|
76
|
-
* the at-least-once / idempotent contract).
|
|
77
|
-
*/
|
|
78
18
|
export class WorkflowNotifyGate {
|
|
79
19
|
journal;
|
|
80
20
|
runStore;
|
|
@@ -86,35 +26,18 @@ export class WorkflowNotifyGate {
|
|
|
86
26
|
this.deliver = deliver;
|
|
87
27
|
this.opts = opts;
|
|
88
28
|
}
|
|
89
|
-
/**
|
|
90
|
-
* The runIds whose journal entry THIS process recorded — the boot-orphan judgment's authoritative anchor
|
|
91
|
-
* (round-1 review, MEDIUM). A wall-clock cutoff alone is not sound: after a BACKWARD clock step the entries
|
|
92
|
-
* this process records land BELOW the boot cutoff, and the sweep would finalize runs whose in-process
|
|
93
|
-
* executor is alive and running (a false `failed` notify for a workflow that then keeps going — worse than
|
|
94
|
-
* the absent row it was fixing). Membership here is a fact about THIS incarnation, unforgeable by any clock.
|
|
95
|
-
* Retired on ack AND the moment a sweep observes the run terminal (the anchor only ever gates the `running`
|
|
96
|
-
* arm), so a delivery/ack outage cannot pile up entries for runs that are no longer executing.
|
|
97
|
-
*/
|
|
98
29
|
recordedThisIncarnation = new Set();
|
|
99
30
|
now() {
|
|
100
31
|
return this.opts.now ? this.opts.now() : Date.now();
|
|
101
32
|
}
|
|
102
|
-
/** Record a journal entry AND remember that this incarnation is the one that recorded it (see the field). */
|
|
103
33
|
async recordPending(entry) {
|
|
104
34
|
await this.journal.record(entry);
|
|
105
35
|
this.recordedThisIncarnation.add(entry.runId);
|
|
106
36
|
}
|
|
107
|
-
/** Ack an entry + drop its incarnation mark (acked ⇒ never scanned again, so the mark has no further use). */
|
|
108
37
|
async ackDelivered(runId) {
|
|
109
38
|
await this.journal.ack(runId, this.now());
|
|
110
39
|
this.recordedThisIncarnation.delete(runId);
|
|
111
40
|
}
|
|
112
|
-
/**
|
|
113
|
-
* Journal a STARTED run as pending-notify. Call this with the synchronous `runId` from `startWorkflow` /
|
|
114
|
-
* `run_workflow` — BEFORE the workflow can reach terminal — so a crash mid-run still leaves a recoverable
|
|
115
|
-
* to-do. Best-effort + idempotent: a journal throw is reported, never propagated (it must not break the
|
|
116
|
-
* workflow start; worst case the recovery sweep won't know about this run — degraded, not corrupting).
|
|
117
|
-
*/
|
|
118
41
|
async onWorkflowStart(input) {
|
|
119
42
|
try {
|
|
120
43
|
await this.recordPending({
|
|
@@ -126,15 +49,9 @@ export class WorkflowNotifyGate {
|
|
|
126
49
|
});
|
|
127
50
|
}
|
|
128
51
|
catch (err) {
|
|
129
|
-
this.opts.onError?.("record", input.runId, err);
|
|
52
|
+
this.opts.onError?.("record", input.runId, err);
|
|
130
53
|
}
|
|
131
54
|
}
|
|
132
|
-
/**
|
|
133
|
-
* The core-facing notifier. core calls this exactly once per run on terminal (its own at-most-once, runId-dedup
|
|
134
|
-
* `fire`). We turn it into at-least-once: ensure the run is journaled (the start hook may have been skipped),
|
|
135
|
-
* then deliver-then-ack. Idempotent: if the run is ALREADY acked (a recovery sweep beat us, or a duplicate
|
|
136
|
-
* core fire), skip — at most one delivery.
|
|
137
|
-
*/
|
|
138
55
|
buildNotifier() {
|
|
139
56
|
return {
|
|
140
57
|
notify: async (input) => {
|
|
@@ -144,24 +61,16 @@ export class WorkflowNotifyGate {
|
|
|
144
61
|
summary: input.summary,
|
|
145
62
|
...(input.sourceTaskId ? { sourceTaskId: input.sourceTaskId } : {}),
|
|
146
63
|
...(input.principal ? { principal: input.principal } : {}),
|
|
147
|
-
...(input.originatingSessionId ? { originatingSessionId: input.originatingSessionId } : {}),
|
|
64
|
+
...(input.originatingSessionId ? { originatingSessionId: input.originatingSessionId } : {}),
|
|
148
65
|
}, input.scope ?? input.principal ?? "default");
|
|
149
66
|
},
|
|
150
67
|
};
|
|
151
68
|
}
|
|
152
|
-
/**
|
|
153
|
-
* Deliver a run's completion exactly once (idempotent on the journal). Ensures the entry exists (record is a
|
|
154
|
-
* no-op if it already does), SKIPS if already acked, otherwise delivers THEN acks (a crash between deliver and
|
|
155
|
-
* ack re-delivers on recovery — at-least-once). A delivery throw is reported and re-thrown to the LIVE caller
|
|
156
|
-
* only as a swallowed report (core's `fire` already `.catch`es), leaving the entry pending for the next sweep.
|
|
157
|
-
*/
|
|
158
69
|
async deliverOnce(p, scope) {
|
|
159
70
|
const existing = await this.journal.get(p.runId);
|
|
160
71
|
if (existing?.acked)
|
|
161
|
-
return;
|
|
72
|
+
return;
|
|
162
73
|
if (!existing) {
|
|
163
|
-
// The start hook didn't journal it (e.g. a run started before this gate existed, or the hook was skipped).
|
|
164
|
-
// Record it now so the deliver-then-ack ordering still holds and a crash mid-delivery is recoverable.
|
|
165
74
|
await this.recordPending({
|
|
166
75
|
runId: p.runId,
|
|
167
76
|
scope,
|
|
@@ -174,59 +83,17 @@ export class WorkflowNotifyGate {
|
|
|
174
83
|
await this.deliver(p);
|
|
175
84
|
}
|
|
176
85
|
catch (err) {
|
|
177
|
-
// Stays PENDING (not acked) → the next recovery sweep re-delivers. Report, never throw back into core.
|
|
178
86
|
this.opts.onError?.("deliver", p.runId, err);
|
|
179
87
|
return;
|
|
180
88
|
}
|
|
181
89
|
await this.ackDelivered(p.runId);
|
|
182
90
|
}
|
|
183
|
-
/**
|
|
184
|
-
* RECOVERY sweep — run at boot AND PERIODICALLY (wired into the service reaper), BEFORE/while serving traffic.
|
|
185
|
-
* For every pending (not-acked) journal entry, re-fetch the run from the durable {@link WorkflowRunStore}:
|
|
186
|
-
* - TERMINAL (completed/failed): its in-process notify was lost to a crash (or it finished while we were
|
|
187
|
-
* down) → re-deliver + ack (idempotent on the receiver).
|
|
188
|
-
* - `running` but STALE (run.startedAt older than `orphanGraceMs`): a crash-orphaned run — core never resumes
|
|
189
|
-
* a prior `running` row (resume mints a NEW runId) and never reaps a `running` row, so NOTHING will ever
|
|
190
|
-
* flip it terminal. Finalize-as-ABANDONED: deliver a `failed` completion (so the originator learns the
|
|
191
|
-
* workflow died) + ack, instead of leaking the entry forever. This closes the exact crash topology SVC-1
|
|
192
|
-
* exists for (a replica SIGKILLed mid-run).
|
|
193
|
-
* - `running` and FRESH (within the grace window): genuinely in flight → leave pending (the owning process
|
|
194
|
-
* delivers its terminal notify, or the next sweep catches it once it goes terminal or stale). The sweep
|
|
195
|
-
* publishes NOTHING here — see the single-writer invariant below.
|
|
196
|
-
* - MISSING (reaped / never persisted): ack-as-abandoned so the journal doesn't chase a ghost forever.
|
|
197
|
-
* `orphanGraceMs` MUST exceed the max expected workflow runtime (the run store has no cross-replica liveness
|
|
198
|
-
* signal, so age is the only orphan proxy). Returns a tally. A per-entry throw is isolated so one bad entry
|
|
199
|
-
* can't abort the sweep.
|
|
200
|
-
*
|
|
201
|
-
* BOTH abandoned arms (boot-orphan + stale-past-grace) flip the DURABLE row to `failed` FIRST (CAS on rev),
|
|
202
|
-
* so the run store, the fleet panel and the delivered notify give ONE answer — previously the notify said
|
|
203
|
-
* `failed` while `/workflows` kept saying `running` forever ([2999]: republishing without finalizing would
|
|
204
|
-
* have turned "panel empty" into "panel shows a row that never moves"). A lost CAS means the run moved under
|
|
205
|
-
* the sweep (e.g. its real terminal landed concurrently) — skip this pass; the entry stays pending and the
|
|
206
|
-
* next sweep handles the NEW state.
|
|
207
|
-
*
|
|
208
|
-
* 🔴 SINGLE-WRITER INVARIANT for the fleet row (round-1 review, two HIGH findings): the sweep only ever
|
|
209
|
-
* publishes a **TERMINAL** fleet frame (the flip above, via `publishTerminalFleetRow` → final frame + remove).
|
|
210
|
-
* It NEVER publishes a `running` row. Two reasons, both "a row we mint here can become one nobody can retire":
|
|
211
|
-
* 1. cross-replica (SQL journal): a pending `running` entry may belong to ANOTHER replica. Its terminal
|
|
212
|
-
* update lands on that replica's own (replica-local) fleet bus, and its ack removes the entry from the
|
|
213
|
-
* shared journal — so this replica would never see the run again and its minted row would sit `running`
|
|
214
|
-
* forever. (The terminal-redelivery arm deliberately doesn't publish either — it is a notify path.)
|
|
215
|
-
* 2. same-replica: `runStore.get` is a READ-TIME SNAPSHOT. If the live run commits its terminal (and the
|
|
216
|
-
* wrapper removes the row) between that read and the publish, a `running` republish resurrects a row
|
|
217
|
-
* that will never be removed again.
|
|
218
|
-
* A live row's ONE writer is this replica's `put`/`update` observation point ({@link JournalingWorkflowRunStore}),
|
|
219
|
-
* which by construction sees every transition including the terminal one.
|
|
220
|
-
*/
|
|
221
91
|
async recover(opts = {}) {
|
|
222
|
-
const orphanGraceMs = opts.orphanGraceMs ?? 24 * 60 * 60 * 1000;
|
|
92
|
+
const orphanGraceMs = opts.orphanGraceMs ?? 24 * 60 * 60 * 1000;
|
|
223
93
|
const pending = await this.journal.listPending();
|
|
224
94
|
let redelivered = 0;
|
|
225
95
|
let stillRunning = 0;
|
|
226
96
|
let abandoned = 0;
|
|
227
|
-
/** Was this entry recorded by a PREVIOUS incarnation of this process? Only meaningful when the caller granted
|
|
228
|
-
* the replica-local finalize authority (a shared/SQL journal carries other replicas' entries). Both halves
|
|
229
|
-
* are required — see `finalizeStartedBeforeMs` + {@link recordedThisIncarnation}. */
|
|
230
97
|
const isPreBootEntry = (entry) => opts.finalizeStartedBeforeMs !== undefined &&
|
|
231
98
|
!this.recordedThisIncarnation.has(entry.runId) &&
|
|
232
99
|
entry.createdAt < opts.finalizeStartedBeforeMs;
|
|
@@ -234,69 +101,33 @@ export class WorkflowNotifyGate {
|
|
|
234
101
|
try {
|
|
235
102
|
const run = await this.runStore.get(entry.runId);
|
|
236
103
|
if (!run) {
|
|
237
|
-
// No durable run — USUALLY a ghost (the row was reaped, or its `put` threw). But it is ALSO the
|
|
238
|
-
// transient shape of a run being started RIGHT NOW: JournalingWorkflowRunStore journals BEFORE it
|
|
239
|
-
// persists (deliberately — a persist that throws must still leave a recoverable entry), so there is a
|
|
240
|
-
// window where the entry exists and the row does not. Acking that window is unrecoverable: the owner's
|
|
241
|
-
// own terminal `deliverOnce` then sees `acked` and drops the completion for good — the exact loss SVC-1
|
|
242
|
-
// exists to prevent (round-2 review, HIGH). So only ack a ghost that PROVABLY isn't that window: one
|
|
243
|
-
// that predates this incarnation, or one older than the grace the whole sweep already trusts.
|
|
244
104
|
if (!isPreBootEntry(entry) && this.now() - entry.createdAt <= orphanGraceMs) {
|
|
245
|
-
stillRunning++;
|
|
105
|
+
stillRunning++;
|
|
246
106
|
continue;
|
|
247
107
|
}
|
|
248
108
|
await this.ackDelivered(entry.runId);
|
|
249
109
|
abandoned++;
|
|
250
110
|
continue;
|
|
251
111
|
}
|
|
252
|
-
// principal fallback: the start-hook (JournalingWorkflowRunStore.put) records only runId+scope — scope IS
|
|
253
|
-
// the creating principal, so it's the sound owner attribution when entry.principal is absent (M6: a
|
|
254
|
-
// recovered notify must still carry an owner).
|
|
255
112
|
const principal = entry.principal ?? entry.scope;
|
|
256
|
-
// #3 G (core 1.187): sourceTaskId used to be "genuinely unrecoverable" here (the durable WorkflowRun didn't
|
|
257
|
-
// carry it). core now threads it onto WorkflowRun.sourceTaskId (startWorkflow stores it / RunWorkflow passes the
|
|
258
|
-
// host taskId), so fall back to run's copy when the journal entry lost it — the recovered notify can finally
|
|
259
|
-
// attribute the originating host task.
|
|
260
113
|
const sourceTaskId = entry.sourceTaskId ?? run.sourceTaskId;
|
|
261
|
-
// core 1.210: the durable WorkflowRun now records `originatingSessionId` (same story as
|
|
262
|
-
// sourceTaskId's 1.187) — thread it onto the RECOVERED payload so crash-recovery routes lookup-free,
|
|
263
|
-
// symmetric with the live notify (resolveCompletionRoute reads it first).
|
|
264
114
|
const originatingSessionId = run.originatingSessionId;
|
|
265
115
|
if (run.status === "running") {
|
|
266
116
|
const startedAt = run.startedAt || entry.createdAt;
|
|
267
|
-
// [2995] boot-orphan judgment (replica-local journal only): the entry predates this process, so the
|
|
268
|
-
// in-process executor that owned it died with the previous incarnation — nothing will ever flip it.
|
|
269
|
-
// (Conditions in `isPreBootEntry`: replica-local authority + not recorded by US + predates boot.)
|
|
270
117
|
const bootOrphan = isPreBootEntry(entry);
|
|
271
118
|
if (!bootOrphan && this.now() - startedAt <= orphanGraceMs) {
|
|
272
119
|
stillRunning++;
|
|
273
|
-
// fresh → genuinely in flight; the owner (or a later sweep) delivers terminal. Publish NOTHING here:
|
|
274
|
-
// a `running` row minted from a read-time snapshot can outlive every chance to retire it (the
|
|
275
|
-
// single-writer invariant on the doc-comment above).
|
|
276
120
|
continue;
|
|
277
121
|
}
|
|
278
|
-
// Orphaned `running` — boot-orphan (executor died with the previous process) or stale past grace
|
|
279
|
-
// (nothing will ever flip it terminal). Finalize as abandoned/failed: flip the DURABLE row first
|
|
280
|
-
// ([2999] honesty half — store/fleet/notify must give one answer), then deliver + ack.
|
|
281
122
|
const abandonSummary = bootOrphan
|
|
282
123
|
? "workflow abandoned — the engine restarted while it was running (its in-process executor did not survive the restart)"
|
|
283
124
|
: "workflow abandoned — still `running` past the orphan grace window (replica likely crashed mid-run)";
|
|
284
125
|
const flipped = { ...run, status: "failed", error: abandonSummary, endedAt: this.now() };
|
|
285
126
|
const flippedOk = await this.runStore.update(entry.runId, run.scope, flipped, run.rev !== undefined ? { rev: run.rev } : undefined);
|
|
286
127
|
if (!flippedOk)
|
|
287
|
-
continue;
|
|
288
|
-
this.recordedThisIncarnation.delete(entry.runId);
|
|
289
|
-
|
|
290
|
-
// Same order as the live leg (the wrapper publishes the terminal frame, then core fires its notify), and
|
|
291
|
-
// it is the only order where "a finalized run leaves the active fleet" survives every delivery outcome
|
|
292
|
-
// (rounds 4+5): the delivery may throw at its ack, or never settle at all, and a `finally` cannot help
|
|
293
|
-
// with the latter — while the NEXT sweep takes the terminal-redelivery arm, which deliberately publishes
|
|
294
|
-
// nothing, so a row skipped here would be stranded `running` with nobody left to retire it.
|
|
295
|
-
// The converse risk is bounded and self-healing: if the projection throws, this pass delivers nothing
|
|
296
|
-
// and the entry stays pending — the next sweep's terminal arm delivers it. Deferred by one sweep, never
|
|
297
|
-
// lost. (Subscriber throws are already isolated inside the fleet bus — per-callback try/catch with a
|
|
298
|
-
// registered fail-open tag — so a projection throw here means the publish machinery itself broke.)
|
|
299
|
-
opts.publishTerminalFleetRow?.(entry.runId, flipped); // final frame then remove
|
|
128
|
+
continue;
|
|
129
|
+
this.recordedThisIncarnation.delete(entry.runId);
|
|
130
|
+
opts.publishTerminalFleetRow?.(entry.runId, flipped);
|
|
300
131
|
await this.deliverOnce({
|
|
301
132
|
runId: entry.runId,
|
|
302
133
|
status: "failed",
|
|
@@ -309,10 +140,6 @@ export class WorkflowNotifyGate {
|
|
|
309
140
|
abandoned++;
|
|
310
141
|
continue;
|
|
311
142
|
}
|
|
312
|
-
// Terminal but not acked ⇒ the crash dropped its notify. Re-derive the bounded summary + re-deliver.
|
|
313
|
-
// Drop the incarnation mark FIRST (round-2 review, MEDIUM): the anchor only ever gates the `running` arm,
|
|
314
|
-
// so a terminal run has no use for it — retiring here (not only on a successful ack) keeps the registry
|
|
315
|
-
// tracking runs that are actually in flight even through a prolonged delivery outage.
|
|
316
143
|
this.recordedThisIncarnation.delete(entry.runId);
|
|
317
144
|
await this.deliverOnce({
|
|
318
145
|
runId: entry.runId,
|
|
@@ -322,7 +149,6 @@ export class WorkflowNotifyGate {
|
|
|
322
149
|
...(principal ? { principal } : {}),
|
|
323
150
|
...(originatingSessionId ? { originatingSessionId } : {}),
|
|
324
151
|
}, entry.scope);
|
|
325
|
-
// deliverOnce only acks on a clean delivery; count a redelivery whether or not it stuck (it logs on fail).
|
|
326
152
|
const after = await this.journal.get(entry.runId);
|
|
327
153
|
if (after?.acked)
|
|
328
154
|
redelivered++;
|
|
@@ -334,16 +160,6 @@ export class WorkflowNotifyGate {
|
|
|
334
160
|
return { scanned: pending.length, redelivered, stillRunning, abandoned };
|
|
335
161
|
}
|
|
336
162
|
}
|
|
337
|
-
/**
|
|
338
|
-
* Default zero-dependency, crash-safe {@link WorkflowNotifyJournalStore}. An append-only JSONL ledger replayed
|
|
339
|
-
* at boot (last-writer-wins per `runId`), the same durable posture as core's `FileWorkflowRunStore` — fsync'd
|
|
340
|
-
* appends, atomic single-line records. Single-instance / TOC-local: a cross-replica TiDB/PG port is a follow-on
|
|
341
|
-
* (the recovery sweep + gate are backend-agnostic).
|
|
342
|
-
*
|
|
343
|
-
* The ledger only ever grows by one line per `record`/`ack`; an acked entry's line is not deleted (the replay
|
|
344
|
-
* folds it), so retention is a future compaction concern — the backlog `listPending` walks is bounded by
|
|
345
|
-
* IN-FLIGHT + un-swept runs, not history.
|
|
346
|
-
*/
|
|
347
163
|
export class FileWorkflowNotifyJournalStore {
|
|
348
164
|
fsyncEnabled;
|
|
349
165
|
ledgerPath;
|
|
@@ -352,15 +168,10 @@ export class FileWorkflowNotifyJournalStore {
|
|
|
352
168
|
constructor(root, fsyncEnabled = true) {
|
|
353
169
|
this.fsyncEnabled = fsyncEnabled;
|
|
354
170
|
const dir = join(root, "workflow-notify");
|
|
355
|
-
// 0o700 dir + 0o600 ledger: the journal persists tenant identity (scope/principal/sourceTaskId) — it must not
|
|
356
|
-
// be group/world-readable (parity with the durable-secret posture; a default-umask file would leak identity).
|
|
357
171
|
mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
358
172
|
this.ledgerPath = join(dir, "journal.jsonl");
|
|
359
173
|
this.replay();
|
|
360
174
|
}
|
|
361
|
-
/** Replay the ledger into the authoritative map (last-writer-wins by line order). Tolerates a torn final line
|
|
362
|
-
* (a crash mid-append) by dropping an unparsable trailing record — the same crash-safety FileWorkflowRunStore
|
|
363
|
-
* relies on (a partial append is simply not yet committed state). */
|
|
364
175
|
replay() {
|
|
365
176
|
if (!existsSync(this.ledgerPath))
|
|
366
177
|
return;
|
|
@@ -373,17 +184,15 @@ export class FileWorkflowNotifyJournalStore {
|
|
|
373
184
|
ev = JSON.parse(line);
|
|
374
185
|
}
|
|
375
186
|
catch {
|
|
376
|
-
continue;
|
|
187
|
+
continue;
|
|
377
188
|
}
|
|
378
189
|
if (ev && typeof ev.runId === "string")
|
|
379
190
|
this.entries.set(ev.runId, ev);
|
|
380
191
|
}
|
|
381
192
|
}
|
|
382
|
-
/** Commit one entry: append the JSON line (fsync) THEN flip the in-memory map (crash-safe ordering — the
|
|
383
|
-
* durable record lands before the in-memory state the caller observes). */
|
|
384
193
|
commit(entry) {
|
|
385
194
|
if (this.fd === undefined)
|
|
386
|
-
this.fd = openSync(this.ledgerPath, "a", 0o600);
|
|
195
|
+
this.fd = openSync(this.ledgerPath, "a", 0o600);
|
|
387
196
|
writeSync(this.fd, `${JSON.stringify(entry)}\n`);
|
|
388
197
|
if (this.fsyncEnabled)
|
|
389
198
|
fsyncSync(this.fd);
|
|
@@ -391,7 +200,6 @@ export class FileWorkflowNotifyJournalStore {
|
|
|
391
200
|
}
|
|
392
201
|
async record(input) {
|
|
393
202
|
const existing = this.entries.get(input.runId);
|
|
394
|
-
// Idempotent: never reset an existing entry (esp. an acked one → it would re-deliver forever).
|
|
395
203
|
if (existing)
|
|
396
204
|
return;
|
|
397
205
|
this.commit({
|
|
@@ -406,7 +214,7 @@ export class FileWorkflowNotifyJournalStore {
|
|
|
406
214
|
async ack(runId, ackedAt) {
|
|
407
215
|
const existing = this.entries.get(runId);
|
|
408
216
|
if (!existing || existing.acked)
|
|
409
|
-
return;
|
|
217
|
+
return;
|
|
410
218
|
this.commit({ ...existing, acked: true, ackedAt });
|
|
411
219
|
}
|
|
412
220
|
async listPending() {
|
|
@@ -415,32 +223,25 @@ export class FileWorkflowNotifyJournalStore {
|
|
|
415
223
|
async get(runId) {
|
|
416
224
|
return this.entries.get(runId) ?? null;
|
|
417
225
|
}
|
|
418
|
-
/** Release the append handle (best-effort) — called on shutdown. */
|
|
419
226
|
close() {
|
|
420
227
|
if (this.fd !== undefined) {
|
|
421
228
|
try {
|
|
422
229
|
closeSync(this.fd);
|
|
423
230
|
}
|
|
424
231
|
catch {
|
|
425
|
-
/* best-effort */
|
|
426
232
|
}
|
|
427
233
|
this.fd = undefined;
|
|
428
234
|
}
|
|
429
235
|
}
|
|
430
|
-
/** Test/inspection: total entries (acked + pending). */
|
|
431
236
|
get size() {
|
|
432
237
|
return this.entries.size;
|
|
433
238
|
}
|
|
434
239
|
}
|
|
435
|
-
/** In-process {@link WorkflowNotifyJournalStore} — tests / single-instance ephemeral. Does NOT survive a
|
|
436
|
-
* restart (so it provides NO crash-recovery on its own); use {@link FileWorkflowNotifyJournalStore} or a
|
|
437
|
-
* durable backend for the real at-least-once guarantee. Useful to unit-test the gate's live + idempotency
|
|
438
|
-
* paths without touching disk, and to simulate a crash by handing a FRESH store to a new gate. */
|
|
439
240
|
export class InMemoryWorkflowNotifyJournalStore {
|
|
440
241
|
entries = new Map();
|
|
441
242
|
async record(input) {
|
|
442
243
|
if (this.entries.has(input.runId))
|
|
443
|
-
return;
|
|
244
|
+
return;
|
|
444
245
|
this.entries.set(input.runId, {
|
|
445
246
|
runId: input.runId,
|
|
446
247
|
scope: input.scope,
|
|
@@ -466,53 +267,19 @@ export class InMemoryWorkflowNotifyJournalStore {
|
|
|
466
267
|
return this.entries.size;
|
|
467
268
|
}
|
|
468
269
|
}
|
|
469
|
-
/**
|
|
470
|
-
* A {@link WorkflowRunStore} DECORATOR that journals a run the instant core persists it at START. core's
|
|
471
|
-
* `startWorkflow` `put`s the run row (`status: "running"`) synchronously before the body runs, carrying both the
|
|
472
|
-
* `runId` and the `scope` — the one synchronous start-time hook the service gets WITHOUT a new core seam (core
|
|
473
|
-
* exposes no `onWorkflowStart`). Wrapping the run store here observes that `put`, journals the started run as
|
|
474
|
-
* pending-notify, then delegates to the wrapped store. Every other method is a pure pass-through (the run store
|
|
475
|
-
* stays core's source of truth for `/workflows`). This is the [[core-service-boundary]]-clean wiring: reuse the
|
|
476
|
-
* core store seam, observe writes through it, never fork a parallel run model.
|
|
477
|
-
*
|
|
478
|
-
* Journaling on `put` (not on notify) is what makes recovery cover a crash BEFORE terminal: the entry exists the
|
|
479
|
-
* moment the run starts, so the boot sweep finds it even if the process died mid-run.
|
|
480
|
-
*/
|
|
481
270
|
export class JournalingWorkflowRunStore {
|
|
482
271
|
inner;
|
|
483
272
|
gate;
|
|
484
273
|
fleetBus;
|
|
485
|
-
constructor(inner,
|
|
486
|
-
/** [1262] optional since the fleet decoupling: undefined ⇒ no start-time journal (ephemeral memory
|
|
487
|
-
* backend has nothing to recover), the FLEET half still publishes — the panel is replica-local UI
|
|
488
|
-
* state and must not be gated on the durable-notify machinery (the regression that emptied clay's
|
|
489
|
-
* workflow panel on memory-backend deployments). */
|
|
490
|
-
gate,
|
|
491
|
-
/** MF-Fleet (data contract): publish the workflow's fleet row on every put/update (the same write-observation
|
|
492
|
-
* point as the notify journal). Optional — undefined ⇒ no fleet publish. */
|
|
493
|
-
fleetBus) {
|
|
274
|
+
constructor(inner, gate, fleetBus) {
|
|
494
275
|
this.inner = inner;
|
|
495
276
|
this.gate = gate;
|
|
496
277
|
this.fleetBus = fleetBus;
|
|
497
278
|
}
|
|
498
|
-
/** Derive + publish the MF-Fleet workflow row from a WorkflowRun (doneCount/totalCount from agents, tokens from
|
|
499
|
-
* stats); a terminal workflow LEAVES the fleet (publish terminal then remove), a running one stays/updates. */
|
|
500
279
|
publishFleet(id, run) {
|
|
501
280
|
if (!this.fleetBus)
|
|
502
281
|
return;
|
|
503
|
-
// A-002.3(#194 病族 F1 同源性断裂):终态判据走 fleet-terminal-window 的共享谓词(其穷举表对
|
|
504
|
-
// core WorkflowRunStatus 编译期闭合)——此前手写两值比较是第二份独立定义,core 加终态词时穷举表
|
|
505
|
-
// 红、这里恒 false ⇒ 行既不 seed 进终态窗也不 removeWorkflow,面板留幽灵 running 行。
|
|
506
282
|
const terminal = isTerminalWorkflowRowStatus(run.status);
|
|
507
|
-
// 撤行**不挂在发帧成功上**(复审第 2 轮 MEDIUM):「终态行必须离场」是不变量、发终帧是尽力而为。
|
|
508
|
-
// try/finally ⇒ 发布本体抛错照样撤行(异常仍向上抛,不吞:吞了就是一条未登记的 fail-open 臂)。
|
|
509
|
-
// 订阅方抛错已在 fleet bus 的 subscribe 隔离层被吞并记 fail-open——本 finally 守的是发布机器本身
|
|
510
|
-
// 的抛错(理论缝级,fresh 复审盘点后判近死防御但保留:防未来发布本体改动回退)。两腿一处收口。
|
|
511
|
-
// 行投影本体 = {@link buildFleetWorkflowRow}(#189:快照的终态行窗与本写路径共用同一份投影,
|
|
512
|
-
// 单源不漂;本函数的发布/撤行语义逐字不变)。
|
|
513
|
-
// [3156] 轮2 修:终态行**写路径顺手 seed** 进程内终态窗缓存——重启-重连剧本里(boot recover 判死),
|
|
514
|
-
// fleetWide 连接的 durable pull 没有正确的查询键可用(caller principal ≠ 引擎铸行 scope),seed 是
|
|
515
|
-
// 唯一同源可达径。放在 publish 之前:seed 是纯内存 push,不会抛;终态行入窗与发帧同拍。
|
|
516
283
|
if (terminal)
|
|
517
284
|
seedTerminalWorkflowRow(buildFleetWorkflowRow(id, run));
|
|
518
285
|
try {
|
|
@@ -520,34 +287,24 @@ export class JournalingWorkflowRunStore {
|
|
|
520
287
|
}
|
|
521
288
|
finally {
|
|
522
289
|
if (terminal)
|
|
523
|
-
this.fleetBus.removeWorkflow(id);
|
|
290
|
+
this.fleetBus.removeWorkflow(id);
|
|
524
291
|
}
|
|
525
292
|
}
|
|
526
|
-
/** [2995] recovery projection seam: the same fleet projection as the put/update observation points, exposed so
|
|
527
|
-
* the recovery sweep can RETIRE a row for a run it just finalized (terminal input ⇒ final frame, then remove).
|
|
528
|
-
* Wire it as `recover`'s `publishTerminalFleetRow` hook — and only ever hand it a TERMINAL run: a `running`
|
|
529
|
-
* row published from outside the wrapper's own write path has no guaranteed retirement (see the invariant on
|
|
530
|
-
* {@link WorkflowNotifyGate.recover}). */
|
|
531
293
|
republishFleet(id, run) {
|
|
532
294
|
this.publishFleet(id, run);
|
|
533
295
|
}
|
|
534
296
|
async put(id, run) {
|
|
535
|
-
// Journal FIRST (best-effort, swallowed inside onWorkflowStart) so a started run is always recoverable, THEN
|
|
536
|
-
// persist. Order matters: if persist throws, the journal entry still lets recovery re-check the (absent) run
|
|
537
|
-
// and ack-as-abandoned — no leak, no lost notify owed.
|
|
538
297
|
await this.gate?.onWorkflowStart({
|
|
539
298
|
runId: id,
|
|
540
299
|
scope: run.scope,
|
|
541
|
-
// core threads sourceTaskId/principal to the NOTIFIER, not onto the run row, so they're unavailable here;
|
|
542
|
-
// the notifier path (deliverOnce) fills them in when it records/delivers. Recovery still has runId+scope.
|
|
543
300
|
});
|
|
544
301
|
await this.inner.put(id, run);
|
|
545
|
-
this.publishFleet(id, run);
|
|
302
|
+
this.publishFleet(id, run);
|
|
546
303
|
}
|
|
547
304
|
async update(id, scope, run, expect) {
|
|
548
305
|
const ok = await this.inner.update(id, scope, run, expect);
|
|
549
306
|
if (ok)
|
|
550
|
-
this.publishFleet(id, run);
|
|
307
|
+
this.publishFleet(id, run);
|
|
551
308
|
return ok;
|
|
552
309
|
}
|
|
553
310
|
get(id) {
|