@sema-agent/server 7.43.0 → 7.44.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/USAGE.md +18 -0
- package/dist/adoption/plan.js +31 -192
- package/dist/adoption/quiesce.js +3 -86
- package/dist/adoption/runner.js +3 -140
- package/dist/adoption/sql.js +0 -74
- package/dist/adoption/wire.js +0 -80
- package/dist/approval-ask-machine.js +0 -75
- package/dist/approval-card.js +0 -323
- package/dist/approval-deny-reasons.js +0 -42
- package/dist/approval-hmac.js +0 -35
- package/dist/approval-reconciler.js +8 -166
- package/dist/approval.js +4 -67
- package/dist/audit.js +1 -44
- package/dist/auth-bridge.js +4 -47
- package/dist/auth-keys.js +0 -23
- package/dist/bake-runner/main.js +4 -65
- package/dist/bake-runner/protocol.js +3 -58
- package/dist/bake-runner/runner.js +5 -91
- package/dist/bench/l8/artifact.js +2 -61
- package/dist/bench/l8/escape.js +0 -25
- package/dist/bench/l8/index.js +0 -14
- package/dist/bench/l8/probes.js +0 -60
- package/dist/bench/l8/run-probes.js +2 -45
- package/dist/bench/s1/arms.js +14 -189
- package/dist/bench/s1/live-deps.js +11 -253
- package/dist/bench/s1/oracle.js +1 -36
- package/dist/bench/s1/repair-oracle-adapter.js +0 -30
- package/dist/bench/s1/reviewer.js +0 -34
- package/dist/bench/s1/row.js +0 -62
- package/dist/bench/s1/run-firm.js +4 -70
- package/dist/bench/s1/runner-ctx.js +0 -40
- package/dist/bench/s1/tasks.js +0 -115
- package/dist/boot/adoption.js +0 -20
- package/dist/boot/budget-tracing.js +3 -55
- package/dist/boot/config-center.js +43 -551
- package/dist/boot/coordinators.js +1 -69
- package/dist/boot/crash-last.js +0 -18
- package/dist/boot/deferred-sandbox-path-env.js +3 -119
- package/dist/boot/execution-env.js +5 -171
- package/dist/boot/governance-seams.js +0 -92
- package/dist/boot/leader.js +0 -69
- package/dist/boot/limit-sync.js +0 -2
- package/dist/boot/memory-boundary.js +3 -91
- package/dist/boot/org-memory.js +1 -25
- package/dist/boot/parked-revive-gate.js +0 -150
- package/dist/boot/permission-rules-audit.js +1 -56
- package/dist/boot/reapers.js +15 -273
- package/dist/boot/resolve-spec.js +9 -768
- package/dist/boot/retention-lane.js +4 -87
- package/dist/boot/runner-deps.js +3 -297
- package/dist/boot/runtime-caps.js +0 -40
- package/dist/boot/session-faces.js +2 -133
- package/dist/boot/shutdown.js +9 -83
- package/dist/boot/side-query-lane.js +2 -137
- package/dist/boot/stores.js +15 -215
- package/dist/boot/task-list-lane.js +0 -18
- package/dist/boot/webfetch-summarize-lane.js +0 -47
- package/dist/boot/workflow-orchestration.js +3 -99
- package/dist/boot-reclaim.js +0 -20
- package/dist/bounded-session-map.js +0 -19
- package/dist/brain.js +2 -139
- package/dist/budget.js +9 -244
- package/dist/capabilities/builtin-tools.js +0 -3
- package/dist/capabilities/center-plugins.js +6 -51
- package/dist/capabilities/center-prompts.js +6 -71
- package/dist/capabilities/code-review-council.js +7 -40
- package/dist/capabilities/collab-workflows.js +1 -44
- package/dist/capabilities/hands-lane.js +0 -65
- package/dist/capabilities/memory-notice.js +0 -70
- package/dist/capabilities/prompt.js +0 -14
- package/dist/capabilities/prompts/code-review.js +0 -14
- package/dist/capabilities/prompts/identity.js +0 -6
- package/dist/capabilities/prompts/team.js +0 -4
- package/dist/capabilities/repo-tools.js +0 -34
- package/dist/capabilities/sandbox-file-send.js +3 -92
- package/dist/capabilities/scenarios.d.ts +0 -1
- package/dist/capabilities/scenarios.js +18 -279
- package/dist/capabilities/select-environment-tool.js +0 -32
- package/dist/capabilities/send-user-file-tool.js +2 -66
- package/dist/capabilities/skills.d.ts +1 -2
- package/dist/capabilities/skills.js +3 -21
- package/dist/capabilities/team.d.ts +3 -11
- package/dist/capabilities/team.js +3 -51
- package/dist/capabilities/tool-defer.js +0 -3
- package/dist/config-center/apply-effective.js +15 -330
- package/dist/config-center/apply-ledger.js +5 -38
- package/dist/config-center/facade.js +0 -41
- package/dist/config-center/hot-keys-registry.js +0 -20
- package/dist/config-center/http-client.js +2 -125
- package/dist/config-center/mcp-revocation.js +2 -34
- package/dist/config-center/read-face.js +0 -59
- package/dist/config-center/restart-signal.js +1 -85
- package/dist/config-center/skills-mcp.d.ts +1 -1
- package/dist/config-center/skills-mcp.js +9 -76
- package/dist/config-center/stage-limits.js +8 -35
- package/dist/config-invariants.js +0 -16
- package/dist/config-lkg.js +0 -42
- package/dist/config-provider.js +3 -186
- package/dist/config-types.js +0 -5
- package/dist/config.js +107 -1145
- package/dist/degenerate-instrument.js +3 -67
- package/dist/deployment-governance.js +0 -124
- package/dist/digest-form.js +0 -12
- package/dist/elicitation.js +3 -86
- package/dist/env-facts.js +7 -75
- package/dist/fleet/fleet-bus.js +34 -507
- package/dist/fleet/fleet-reconciler.js +9 -149
- package/dist/fleet/fleet-terminal-window.js +10 -178
- package/dist/fleet/subagent-tail-bus.js +3 -72
- package/dist/fleet-client.js +10 -70
- package/dist/fleet-lease.js +5 -79
- package/dist/git-api-kind.js +0 -3
- package/dist/governance-ask-marks.js +2 -78
- package/dist/hooks/branch-transcript.js +0 -74
- package/dist/hooks/cc-agent-hook-prompt.js +0 -29
- package/dist/hooks/cc-stop-prompt.js +1 -46
- package/dist/hooks/hook-llm.js +1 -53
- package/dist/hooks/hook-runner.js +20 -414
- package/dist/http/active-run-conflict.js +4 -129
- package/dist/http/cursor-fingerprint.d.ts +5 -0
- package/dist/http/cursor-fingerprint.js +5 -0
- package/dist/http/idempotency.js +0 -37
- package/dist/http/principal-gate.js +3 -40
- package/dist/http/route-ctx.js +0 -9
- package/dist/http/routes/a2a-serve.js +7 -319
- package/dist/http/routes/admin-config-refresh.js +0 -4
- package/dist/http/routes/admin-drain.js +0 -10
- package/dist/http/routes/adoption.js +1 -29
- package/dist/http/routes/agents-roster.js +1 -47
- package/dist/http/routes/approvals-assistant.js +32 -395
- package/dist/http/routes/attachments.js +4 -22
- package/dist/http/routes/capabilities.js +3 -420
- package/dist/http/routes/diagnostics.js +1 -81
- package/dist/http/routes/fleet.js +9 -185
- package/dist/http/routes/images.js +14 -239
- package/dist/http/routes/leader.js +0 -13
- package/dist/http/routes/memory-bundle.js +1 -61
- package/dist/http/routes/memory-policy.js +9 -99
- package/dist/http/routes/notify-wake.js +3 -37
- package/dist/http/routes/observability.js +2 -19
- package/dist/http/routes/retention-ops.js +2 -34
- package/dist/http/routes/rules.js +1 -93
- package/dist/http/routes/runs.js +64 -873
- package/dist/http/routes/session-sync.js +19 -258
- package/dist/http/routes/sessions-list.js +17 -43
- package/dist/http/routes/sessions.js +34 -226
- package/dist/http/routes/shared-memory.js +5 -36
- package/dist/http/routes/side-query.js +1 -87
- package/dist/http/routes/tasks.js +52 -722
- package/dist/http/routes/trace-usage.js +32 -195
- package/dist/http/routes/workflows.js +19 -195
- package/dist/http/run-meta.js +0 -6
- package/dist/http/send.js +0 -32
- package/dist/http/server.js +92 -1597
- package/dist/http/sse-lifecycle.js +2 -13
- package/dist/http/sse-log.js +3 -48
- package/dist/http/tar.js +5 -21
- package/dist/http/verify-rounds.js +0 -5
- package/dist/http/wire-gate.js +0 -9
- package/dist/http/workspace-content.js +0 -10
- package/dist/images/bake-validate.js +1 -70
- package/dist/images/manifest.js +1 -6
- package/dist/index.js +0 -21
- package/dist/key-resolver.js +2 -17
- package/dist/leader/diffout.js +1 -20
- package/dist/leader/diffup.js +0 -47
- package/dist/leader/endpoint.js +2 -57
- package/dist/leader/fanout.js +3 -45
- package/dist/leader/grader-env-factory.js +3 -72
- package/dist/leader/leader.js +5 -156
- package/dist/leader/merge.js +8 -115
- package/dist/leader/planner.js +3 -54
- package/dist/leader/repair-oracle.js +1 -60
- package/dist/leader/repair-wire.js +2 -79
- package/dist/leader/wire.js +8 -307
- package/dist/lsp/e2b-bridge.js +4 -64
- package/dist/lsp/e2b-manager.js +6 -94
- package/dist/lsp/lsp-frames.js +0 -12
- package/dist/lsp/manager.js +4 -96
- package/dist/lsp/ws-transport.js +5 -55
- package/dist/lsp-evict.js +1 -15
- package/dist/main.js +57 -765
- package/dist/memory-bundle-engine.js +0 -55
- package/dist/memory-export.js +0 -4
- package/dist/memory-posture.js +1 -15
- package/dist/memory-scope.js +11 -148
- package/dist/memory-sync-client.js +2 -44
- package/dist/memory-sync.js +1 -80
- package/dist/model-select.js +3 -80
- package/dist/observability/cost-quota.js +1 -17
- package/dist/observability/cost-taxonomy.js +0 -34
- package/dist/observability/fail-open.js +7 -86
- package/dist/observability/logger.js +0 -6
- package/dist/observability/metrics.js +0 -94
- package/dist/observability/otel-exporter.js +3 -13
- package/dist/observability/principal-context.js +0 -9
- package/dist/observability/prompt-manifest.js +1 -37
- package/dist/observability/rate-limit.js +0 -4
- package/dist/observability/secret-env-scrub.js +2 -56
- package/dist/observability/tool-trace.js +1 -70
- package/dist/orchestration/hardened-vm-runner.js +4 -118
- package/dist/orchestration/hardened-vm-worker-runner.js +1 -26
- package/dist/orchestration/hardened-vm-worker.js +0 -27
- package/dist/orchestration/subagent-steer.js +1 -45
- package/dist/orchestration/workflow-agent-steer.js +1 -80
- package/dist/orchestration/workflow-completion-inbox.js +32 -285
- package/dist/orchestration/workflow-notify-journal.js +16 -259
- package/dist/org-memory-admission.js +3 -47
- package/dist/parent-watch.js +2 -48
- package/dist/parked-decide.js +1 -109
- package/dist/per-task-image.js +0 -57
- package/dist/plan-cache-probe.js +3 -27
- package/dist/plugins/adoption-log-sql.js +2 -119
- package/dist/plugins/approval-ask-store-memory.js +3 -38
- package/dist/plugins/approval-ask-store-sql.js +6 -188
- package/dist/plugins/approval-exemption-store.js +2 -28
- package/dist/plugins/background-agent-store-sql.js +4 -105
- package/dist/plugins/background-shell-support.js +14 -122
- package/dist/plugins/blob-backend.js +6 -169
- package/dist/plugins/breaker-state-sql.js +8 -46
- package/dist/plugins/caching-session-store.js +4 -106
- package/dist/plugins/checkpoint-store-sql.js +18 -505
- package/dist/plugins/e2b-orphan-reclaim.js +0 -45
- package/dist/plugins/file-outcome-sink.js +0 -9
- package/dist/plugins/file-resume-anchor-store.js +4 -43
- package/dist/plugins/file-run-store.js +26 -364
- package/dist/plugins/file-snapshot-store-sql.js +10 -181
- package/dist/plugins/fork-routing-session-store.js +8 -111
- package/dist/plugins/host-platform.js +2 -91
- package/dist/plugins/image-bake-store-sql.js +7 -250
- package/dist/plugins/image-index-sql.js +4 -123
- package/dist/plugins/k8s-bg-scripts.js +4 -88
- package/dist/plugins/k8s-exec-protocol.js +0 -41
- package/dist/plugins/leader-run-store-sql.js +0 -101
- package/dist/plugins/local-checkpoint-store.js +8 -128
- package/dist/plugins/local-session-store.js +34 -296
- package/dist/plugins/local-task-attachment-store.js +2 -16
- package/dist/plugins/mailbox-store-sql.js +8 -65
- package/dist/plugins/memory-embedder-fingerprint.js +5 -166
- package/dist/plugins/memory-embedder.js +4 -70
- package/dist/plugins/memory-engine-pg.js +6 -170
- package/dist/plugins/memory-engine-tidb.js +7 -155
- package/dist/plugins/memory-engine-vector-util.js +0 -10
- package/dist/plugins/memory-key-guards.js +0 -34
- package/dist/plugins/memory-origin-law.js +0 -187
- package/dist/plugins/memory-resume-anchor-store.js +0 -17
- package/dist/plugins/memory-run-store.js +12 -89
- package/dist/plugins/memory-session-policy-store.js +0 -17
- package/dist/plugins/memory-sync-store-pg.js +4 -49
- package/dist/plugins/memory-sync-store-tidb.js +3 -35
- package/dist/plugins/outcome-ledger-sql.js +3 -97
- package/dist/plugins/permission-rule-store-file.js +5 -133
- package/dist/plugins/permission-rule-store-sql.d.ts +3 -0
- package/dist/plugins/permission-rule-store-sql.js +50 -307
- package/dist/plugins/pg-cost-quota.js +0 -7
- package/dist/plugins/pg-pool.js +0 -92
- package/dist/plugins/pg-rate-limiter.js +2 -13
- package/dist/plugins/pg-safe-json.js +4 -40
- package/dist/plugins/pg-session-storage.js +25 -189
- package/dist/plugins/posix-shell-fs.js +1 -31
- package/dist/plugins/remote-env-adb.js +12 -101
- package/dist/plugins/remote-env-e2b.js +40 -371
- package/dist/plugins/remote-env-file-error.js +0 -37
- package/dist/plugins/remote-env-host.js +63 -483
- package/dist/plugins/remote-env-k8s.js +32 -326
- package/dist/plugins/remote-env-local-docker.js +24 -157
- package/dist/plugins/remote-env-ssh.js +20 -128
- package/dist/plugins/remote-scratchpad.js +2 -32
- package/dist/plugins/remote-shell.js +1 -32
- package/dist/plugins/resume-anchor-store-sql.js +0 -11
- package/dist/plugins/retention-lane-store-sql.js +0 -108
- package/dist/plugins/retention-store-sql.js +4 -383
- package/dist/plugins/roster-store-sql.js +0 -55
- package/dist/plugins/run-store-sql.js +14 -251
- package/dist/plugins/s3-presign.js +2 -49
- package/dist/plugins/scheduler-support.js +3 -80
- package/dist/plugins/send-file-ledger.js +4 -53
- package/dist/plugins/send-user-file.js +4 -94
- package/dist/plugins/session-placement.js +1 -89
- package/dist/plugins/session-policy-store-sql.js +3 -81
- package/dist/plugins/session-store.js +0 -57
- package/dist/plugins/shared-memory-store-sql.js +4 -167
- package/dist/plugins/sql-driver.js +0 -17
- package/dist/plugins/sql-errors.js +0 -7
- package/dist/plugins/sql-escape.js +0 -8
- package/dist/plugins/sql-row-helpers.js +0 -25
- package/dist/plugins/store-backend.js +42 -222
- package/dist/plugins/store-contracts.js +2 -46
- package/dist/plugins/task-attachment-store.js +3 -49
- package/dist/plugins/task-list-store-sql.js +0 -79
- package/dist/plugins/tidb-cost-quota.js +1 -4
- package/dist/plugins/tidb-pool.js +1 -207
- package/dist/plugins/tidb-rate-limiter.js +3 -9
- package/dist/plugins/tidb-session-storage.js +4 -70
- package/dist/plugins/tidb-session-store.js +19 -341
- package/dist/plugins/tool-result-store-sql.js +4 -179
- package/dist/plugins/usage-window-store-sql.js +0 -8
- package/dist/plugins/web-search.js +10 -132
- package/dist/plugins/workflow-journal-store-sql.js +2 -58
- package/dist/plugins/workflow-run-store-sql.js +6 -91
- package/dist/plugins/worktree-isolation.js +6 -126
- package/dist/plugins/write-behind-counter.js +16 -75
- package/dist/principal-jwt.js +5 -60
- package/dist/project-memory.js +15 -146
- package/dist/prompts-domain-validate.js +1 -51
- package/dist/question.js +2 -111
- package/dist/resource-suspend.js +0 -18
- package/dist/router/route-orchestration.js +0 -77
- package/dist/rules-consent.d.ts +44 -5
- package/dist/rules-consent.js +75 -215
- package/dist/run-local.js +14 -381
- package/dist/runs.js +29 -617
- package/dist/runtime-caps-resolver.js +6 -132
- package/dist/runtime-governance.js +1 -232
- package/dist/sandbox-pkg-source.js +0 -37
- package/dist/sealed-key.js +3 -68
- package/dist/security.js +6 -291
- package/dist/session-leaf-bus.js +0 -32
- package/dist/session-sync-content.js +1 -69
- package/dist/session-sync-kernel.js +3 -60
- package/dist/session-sync.js +3 -66
- package/dist/session-titler.js +7 -41
- package/dist/session-watch.js +9 -73
- package/dist/shared-memory-scope-authorizer.js +0 -18
- package/dist/sighup-idle.js +1 -12
- package/dist/spec-fields.js +4 -143
- package/dist/store-live-probe.js +3 -49
- package/dist/task-a2a.js +1 -126
- package/dist/task-cwd.js +1 -103
- package/dist/task-mcp.js +1 -89
- package/dist/task-settings.js +10 -300
- package/dist/task-workflow.js +8 -75
- package/dist/tool-approval.d.ts +28 -1
- package/dist/tool-approval.js +70 -1430
- package/dist/trace/artifacts.js +5 -20
- package/dist/trace/engine-notice-wire.js +6 -138
- package/dist/trace/ledger-sink.js +7 -96
- package/dist/trace/project.js +4 -448
- package/dist/trace/redact.js +9 -77
- package/dist/turn-activity.js +1 -27
- package/dist/usage-analytics.js +4 -34
- package/dist/wall-clock-jump-guard.js +1 -68
- package/package.json +3 -3
- package/dist/capabilities/scenario-alias.d.ts +0 -27
- package/dist/capabilities/scenario-alias.js +0 -61
|
@@ -1,77 +1,28 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* File-backed async run registry + replayable event log — the DURABLE local twin of {@link MemoryRunStore} (and the
|
|
3
|
-
* file-backed sibling of {@link TiDBRunStore}/{@link PgRunStore}). P0.5 variant-1 (clay 2026-06-26): the file-backed
|
|
4
|
-
* follow-on the Memory twin's header promised — a locally-run HTTP service now keeps its runs ledger + trace event log
|
|
5
|
-
* across a process restart, not just in memory. Behaviour is byte-identical to MemoryRunStore (the shared parity oracle
|
|
6
|
-
* — same single-active claim, the suspend/needs_review parks that KEEP the claim, the markResuming CAS that resets
|
|
7
|
-
* cancel/preempt, the owner null-safe guard, the keyset list order, the (taskId,seq) event dedup, and the structuredClone
|
|
8
|
-
* write/read isolation); the only difference is durability.
|
|
9
|
-
*
|
|
10
|
-
* THREE on-disk data planes (all under `<root>/runs/`), each using core's crash-safe fs primitives — never hand-rolled
|
|
11
|
-
* fs (zero drift, no zero-length/torn corruption):
|
|
12
|
-
* 1. run registry — `<root>/runs/<taskId>/run.json` = the RunRow, replaced WHOLE on every status change via
|
|
13
|
-
* `atomicWriteFile` (tmp→fsync→rename→fsync-dir, the 5-step atomic replace).
|
|
14
|
-
* 2. single-active claim — `<root>/runs/active/<sanitizePathComponent(sessionId)>` carrying the active taskId, created
|
|
15
|
-
* with `writeThenLink` (atomic create; EEXIST ⇒ the session already has an active run → the
|
|
16
|
-
* createRun loses the claim, returns `{ok:false, activeTaskId}`). The file version of task_active's
|
|
17
|
-
* unique key.
|
|
18
|
-
* 3. per-run event log — `<root>/runs/<taskId>/events.jsonl` = an `AppendLog` (one line per event, fsync:false — events
|
|
19
|
-
* are high-frequency, the torn-tail-drop on replay covers a crash tail; only the run.json status
|
|
20
|
-
* commit + the active-claim create use fsync). (taskId,seq) is deduped on replay.
|
|
21
|
-
*
|
|
22
|
-
* An in-memory INDEX (the same Maps MemoryRunStore holds) is hydrated on construction by scanning each runs/<taskId>/
|
|
23
|
-
* run.json + the active claim files — so listRuns/listSessions/getActiveTaskId are O(in-mem) with byte-identical keyset sort to the SQL/Memory
|
|
24
|
-
* twins. Every write touches BOTH the disk and the index on one path. Single-process (the FileStorageBackend boot lock
|
|
25
|
-
* guarantees one writer per data dir), so there is no per-op lock — with ONE exception: #213's claim acquisition
|
|
26
|
-
* (`withClaimLock`) runs under a per-session in-process critical section, because judging a claim stale involves
|
|
27
|
-
* awaits (the checkpoint probe) and the claim + run row must commit as one unit (the SQL twin's transaction). The
|
|
28
|
-
* checkpoint-coupled suspended-run reapers run
|
|
29
|
-
* off an injected {@link RunStoreCheckpointProbe} ([868]; wired by the local backend where checkpoint + run store
|
|
30
|
-
* share one root) and degrade to honest no-ops when the probe is absent — exactly like MemoryRunStore.
|
|
31
|
-
*/
|
|
32
1
|
import { projectUsageStats, USAGE_SCAN_LIMIT } from "../usage-analytics.js";
|
|
33
2
|
import { join } from "node:path";
|
|
34
3
|
import { existsSync, mkdirSync, readdirSync, renameSync, rmSync } from "node:fs";
|
|
35
4
|
import { randomBytes } from "node:crypto";
|
|
36
5
|
import { AppendLog, atomicWriteFile, ensureDir, readJsonlRecords, sanitizePathComponent, writeThenLink } from "@sema-agent/core";
|
|
37
6
|
import { recordFailOpen } from "../observability/fail-open.js";
|
|
38
|
-
/** Null-safe owner equality — the JS twin of the SQL `owner <=> ?` guard (a legacy null-owner run matches owner=null). */
|
|
39
7
|
const ownerEq = (a, b) => (a ?? null) === (b ?? null);
|
|
40
|
-
/** 进程事实(取证列),模块加载时算一次。 */
|
|
41
8
|
const PROCESS_FACTS = {
|
|
42
9
|
pid: process.pid,
|
|
43
10
|
startedAtMs: Date.now() - Math.round(process.uptime() * 1000),
|
|
44
11
|
};
|
|
45
|
-
/** 接管的有界重试次数:一次判定 + 一次接管后重铸 + 一次输掉竞争后的复判,3 次足够收敛(论证见 takeOverClaim)。 */
|
|
46
12
|
const CLAIM_TAKEOVER_ATTEMPTS = 3;
|
|
47
|
-
/**
|
|
48
|
-
* A1/codex R4 —— claim 记录的**有效形**读法(判定/接管/收口三处共用,与 hydrate 的收编守卫同一把尺)。
|
|
49
|
-
* 「可解析但缺键」(如 `{}` 整行)与零长/被撕是同一种损坏 —— 只认 `claim === undefined` 当撕裂会让
|
|
50
|
-
* 这形走可解析路径、持有者又从陈旧索引里猜(R3 修掉的遮蔽原样复发)。会话对不上号的记录同判无效
|
|
51
|
-
* (错位/拷贝进来的文件):认主一律回落盘上非终局行,那才是权威。
|
|
52
|
-
*/
|
|
53
|
-
/** core `sanitizePathComponent`(fs-atomic SAFE_COMPONENT)同一条规则的**布尔形**(codex R5):
|
|
54
|
-
* 它本体对不安全分量是 throw —— 判定腿要的是「无效 ⇒ 走恢复路」,不是「无效 ⇒ 每发 createRun 500」。
|
|
55
|
-
* 规则只可能比 core 更保守地漂(判无效 ⇒ 回落盘上认行,方向安全)。 */
|
|
56
13
|
const SAFE_PATH_COMPONENT = /^[A-Za-z0-9_.-]+$/;
|
|
57
14
|
const isSafePathComponent = (s) => s !== "" && s !== "." && s !== ".." && SAFE_PATH_COMPONENT.test(s);
|
|
58
15
|
function readClaimRecord(recs, sessionId) {
|
|
59
16
|
const c = recs[recs.length - 1];
|
|
60
17
|
if (!c || typeof c.sessionId !== "string" || typeof c.taskId !== "string" || c.sessionId !== sessionId)
|
|
61
18
|
return undefined;
|
|
62
|
-
// 不安全的 taskId(`../x` 之类)当有效读会让 authoritativeRow 在 sanitize 处炸(codex R5)——同判无效。
|
|
63
19
|
if (!isSafePathComponent(c.taskId))
|
|
64
20
|
return undefined;
|
|
65
21
|
return c;
|
|
66
22
|
}
|
|
67
|
-
/**
|
|
68
|
-
* 「这个 pid 现在还在吗」—— 信号 0 探活,与 core 剪 `root/LOCK` 陈旧锁用的是同一条判据。
|
|
69
|
-
* 🔴 方向:只有 **ESRCH**(确凿不在)才回 false。EPERM = 进程存在但不归我管 ⇒ 回 true(在);
|
|
70
|
-
* 任何其它异常同样回 true —— 探测失败不许被读成「它死了」(那才是会错杀活主的方向)。
|
|
71
|
-
*/
|
|
72
23
|
function isPidAlive(pid) {
|
|
73
24
|
if (!Number.isInteger(pid) || pid <= 0)
|
|
74
|
-
return true;
|
|
25
|
+
return true;
|
|
75
26
|
try {
|
|
76
27
|
process.kill(pid, 0);
|
|
77
28
|
return true;
|
|
@@ -80,45 +31,29 @@ function isPidAlive(pid) {
|
|
|
80
31
|
return e.code !== "ESRCH";
|
|
81
32
|
}
|
|
82
33
|
}
|
|
83
|
-
/**
|
|
84
|
-
* park 孤儿的**静置线**(防竞态余量,不是判据本身 —— 判据是「无 checkpoint 可续」)。
|
|
85
|
-
* 见 judgeClaim 里 park 族那一段的红字注:它盖的是 decide 腿「解 checkpoint」与「markResuming」之间那一拍。
|
|
86
|
-
*/
|
|
87
34
|
const PARK_ORPHAN_SETTLE_MS = 30_000;
|
|
88
35
|
export class FileRunStore {
|
|
89
36
|
runsDir;
|
|
90
37
|
activeDir;
|
|
91
38
|
tmpDir;
|
|
92
|
-
/** taskId → RunRow (the run registry index). */
|
|
93
39
|
runs = new Map();
|
|
94
|
-
/** sessionId → taskId — the single-active-run claim (the task_active table; index of active/*). */
|
|
95
40
|
active = new Map();
|
|
96
|
-
/** taskId → events (kept seq-sorted on read), the event-log index. */
|
|
97
41
|
events = new Map();
|
|
98
|
-
/** taskId → the open append log (one O_APPEND fd per run, lazily opened, held for lifetime). */
|
|
99
42
|
eventLogs = new Map();
|
|
100
|
-
/** [868] the checkpoint-table stand-in (see memory-run-store.ts {@link RunStoreCheckpointProbe} — the full
|
|
101
|
-
* rationale lives on the interface); absent ⇒ the suspended-run reapers stay honest NO-OPs. */
|
|
102
43
|
checkpointProbe;
|
|
103
|
-
/** #213 —— 本实例的 claim 身份(见 {@link ClaimOwner} 的选型注)。 */
|
|
104
44
|
claimOwner = { nonce: randomBytes(12).toString("hex"), ...PROCESS_FACTS };
|
|
105
|
-
/** #213 —— sessionId → 该会话 claim 获取的临界区链尾(见 {@link withClaimLock})。 */
|
|
106
45
|
claimLocks = new Map();
|
|
107
|
-
/** 件6(#43 案 [4281]):启动期告警座(可选)。既有调用方不传 ⇒ 只走 fail-open 遥测那一半,不炸。 */
|
|
108
46
|
logger;
|
|
109
47
|
constructor(root, logger) {
|
|
110
48
|
this.logger = logger;
|
|
111
49
|
this.runsDir = join(root, "runs");
|
|
112
50
|
this.activeDir = join(root, "runs", "active");
|
|
113
|
-
this.tmpDir = join(root, "tmp");
|
|
51
|
+
this.tmpDir = join(root, "tmp");
|
|
114
52
|
ensureDir(this.runsDir);
|
|
115
53
|
ensureDir(this.activeDir);
|
|
116
54
|
ensureDir(this.tmpDir);
|
|
117
55
|
this.hydrate();
|
|
118
|
-
// NB: orphan reclaim is NOT auto-run here — main.ts boot drives it AFTER the checkpointStore is wired, so a
|
|
119
|
-
// resume-in-progress (markResuming) row with a pending checkpoint is excluded (see reclaimOrphanedAtBoot).
|
|
120
56
|
}
|
|
121
|
-
// ── disk paths ──
|
|
122
57
|
runDir(taskId) {
|
|
123
58
|
return join(this.runsDir, sanitizePathComponent(taskId));
|
|
124
59
|
}
|
|
@@ -154,22 +89,12 @@ export class FileRunStore {
|
|
|
154
89
|
updatedAt: r.updatedAt.toISOString(),
|
|
155
90
|
};
|
|
156
91
|
}
|
|
157
|
-
/** Replay the on-disk registry/claims/event-logs into the in-memory index on boot. Each plane is torn-tail-/
|
|
158
|
-
* partial-write-safe (run.json via atomicWriteFile can never be half-written; the active file via writeThenLink is
|
|
159
|
-
* all-or-nothing; events.jsonl via readJsonlRecords drops a crash tail). A corrupt run.json (failed parse) is
|
|
160
|
-
* skipped — that run is lost, not a poison that fails the boot. */
|
|
161
92
|
hydrate() {
|
|
162
93
|
let entries;
|
|
163
94
|
try {
|
|
164
95
|
entries = readdirSync(this.runsDir);
|
|
165
96
|
}
|
|
166
97
|
catch (err) {
|
|
167
|
-
// 🔴 件6(#43 案 [4281],静默降级纪律):旧形是**空 catch + return** —— data root 不可读时引擎以
|
|
168
|
-
// 空账起动、零告警,此后每一次 getRun 恒 miss,而「恒 miss」正是 #43 现场看到的形。方向错在这里:
|
|
169
|
-
// 吞掉之后被读成的是「**证不出有行 ⇒ 没有行**」,而这两句话不等价,一次环境故障就此被伪装成
|
|
170
|
-
// 「本来就没有这条 run」。
|
|
171
|
-
// **行为不变**(仍不拒启 —— 拒启是行为变更,另立),只从静默变响亮:一行 error 级日志带 runs 目录
|
|
172
|
-
// **绝对路径** + 异常文案(运维一眼能判是权限还是路径),外加登记进 fail-open 遥测让这条债显形。
|
|
173
98
|
const detail = err instanceof Error ? err.message : String(err);
|
|
174
99
|
this.logger?.error?.("file_run_store_hydrate_unreadable", {
|
|
175
100
|
runsDir: this.runsDir,
|
|
@@ -181,24 +106,19 @@ export class FileRunStore {
|
|
|
181
106
|
}
|
|
182
107
|
for (const name of entries) {
|
|
183
108
|
if (name === "active")
|
|
184
|
-
continue;
|
|
109
|
+
continue;
|
|
185
110
|
const dir = join(this.runsDir, name);
|
|
186
111
|
const jsonPath = join(dir, "run.json");
|
|
187
112
|
if (!existsSync(jsonPath))
|
|
188
113
|
continue;
|
|
189
|
-
const rows = readJsonlRecords(jsonPath);
|
|
190
|
-
const disk = rows[rows.length - 1];
|
|
114
|
+
const rows = readJsonlRecords(jsonPath);
|
|
115
|
+
const disk = rows[rows.length - 1];
|
|
191
116
|
if (!disk || typeof disk.taskId !== "string")
|
|
192
117
|
continue;
|
|
193
|
-
// R8 修:atomicWriteFile 防的是半写,防不了 schema 半态(旧版残留/手工改动的 run.json 缺时间戳字段)。
|
|
194
|
-
// 不挡在这里的话,坏行会在**读路径**上毒发:`toRecord` 对 Invalid Date 调 toISOString ⇒ RangeError ⇒
|
|
195
|
-
// 一条坏文件打死整个 listRuns(GET /v1/runs 全 500)。按顶注自述的隔离承诺同款处置:跳过该行,
|
|
196
|
-
// 一条坏行最多只影响它自己。
|
|
197
118
|
if (typeof disk.createdAt !== "string" || Number.isNaN(Date.parse(disk.createdAt)) ||
|
|
198
119
|
typeof disk.updatedAt !== "string" || Number.isNaN(Date.parse(disk.updatedAt)))
|
|
199
120
|
continue;
|
|
200
121
|
this.runs.set(disk.taskId, this.toRow(disk));
|
|
201
|
-
// event log
|
|
202
122
|
const eventsPath = join(dir, "events.jsonl");
|
|
203
123
|
const recs = readJsonlRecords(eventsPath);
|
|
204
124
|
if (recs.length) {
|
|
@@ -206,14 +126,13 @@ export class FileRunStore {
|
|
|
206
126
|
const seen = new Set();
|
|
207
127
|
for (const e of recs) {
|
|
208
128
|
if (!e || typeof e.seq !== "number" || seen.has(e.seq))
|
|
209
|
-
continue;
|
|
129
|
+
continue;
|
|
210
130
|
seen.add(e.seq);
|
|
211
131
|
list.push({ seq: e.seq, type: String(e.type), data: e.data ?? null, ts: String(e.ts) });
|
|
212
132
|
}
|
|
213
133
|
this.events.set(disk.taskId, list);
|
|
214
134
|
}
|
|
215
135
|
}
|
|
216
|
-
// active claims
|
|
217
136
|
let claims;
|
|
218
137
|
try {
|
|
219
138
|
claims = readdirSync(this.activeDir);
|
|
@@ -221,22 +140,14 @@ export class FileRunStore {
|
|
|
221
140
|
catch {
|
|
222
141
|
claims = [];
|
|
223
142
|
}
|
|
224
|
-
// A1 幸存者索引(codex R1 [high]):撕裂 claim 只剩文件名可认 —— 文件名 = sanitize 后的会话名,
|
|
225
|
-
// 不可逆,所以对已 hydrate 的**非终局**行正向重算 sanitize 比对。撕的是文件,不是会话的占用语义:
|
|
226
|
-
// park 行跨重启刻意保锁、可续的 running 行 boot 回收也刻意保 —— 这些会话必须**收编回索引**而不是
|
|
227
|
-
// 当残骸解锁(解锁 = 新 run 与将来 resume 的旧 run 并飞,单活不变式被 boot 自己撕开)。
|
|
228
143
|
const nonTerminalBySanitizedSession = new Map();
|
|
229
144
|
for (const r of this.runs.values()) {
|
|
230
145
|
if (r.status !== "running" && r.status !== "suspended" && r.status !== "needs_review")
|
|
231
146
|
continue;
|
|
232
|
-
// sanitizePathComponent 对不安全分量是 THROW——坏 run.json(sessionId 非 string/带空格)在此
|
|
233
|
-
// 会把整个构造器炸死(boot 亡),违背文件头「坏行跳过,不毒 boot」承诺(增量重扫 [med] 拦下)。
|
|
234
|
-
// 同尺跳过:这种行进不了幸存者索引,其 claim 文件(若有)走无主隔离路——方向仍 fail-closed。
|
|
235
147
|
if (typeof r.sessionId !== "string" || !isSafePathComponent(r.sessionId))
|
|
236
148
|
continue;
|
|
237
149
|
const key = sanitizePathComponent(r.sessionId);
|
|
238
150
|
const prev = nonTerminalBySanitizedSession.get(key);
|
|
239
|
-
// 同会话多条非终局行不该存在(单活);真撞上取最新的一条,方向仍是 fail-closed(保锁)。
|
|
240
151
|
if (!prev || r.createdAt.getTime() > prev.createdAt.getTime())
|
|
241
152
|
nonTerminalBySanitizedSession.set(key, r);
|
|
242
153
|
}
|
|
@@ -244,38 +155,22 @@ export class FileRunStore {
|
|
|
244
155
|
const path = join(this.activeDir, sanitized);
|
|
245
156
|
const recs = readJsonlRecords(path);
|
|
246
157
|
const claim = recs[recs.length - 1];
|
|
247
|
-
// 与判定腿同一把尺(codex R5):不安全 taskId(`../x` 之类)不得进索引 —— 进了,后续任何
|
|
248
|
-
// authoritativeRow(taskId) 都在 sanitize 处炸(runDir throw ⇒ createRun 每发 500)。这种记录
|
|
249
|
-
// 与零长/缺键同判无效,落到下面的幸存者收编/隔离路。
|
|
250
158
|
if (claim && typeof claim.sessionId === "string" && typeof claim.taskId === "string" && isSafePathComponent(claim.taskId)) {
|
|
251
159
|
this.active.set(claim.sessionId, claim.taskId);
|
|
252
160
|
continue;
|
|
253
161
|
}
|
|
254
162
|
const survivor = nonTerminalBySanitizedSession.get(sanitized);
|
|
255
163
|
if (survivor) {
|
|
256
|
-
// 幸存者在 ⇒ 只收编、不裁死活:死不死轮不到 hydrate 裁 —— checkpoint 面(isResumable/probe)
|
|
257
|
-
// 此刻还没接线,裁了必错杀可续行。收编之后一切既有腿照常工作:EEXIST ⇒ torn 裁定的
|
|
258
|
-
// 「索引里有」臂挡住并报真持有者;孤儿 running ⇒ reclaimOrphanedAtBoot(checkpoint 接好后
|
|
259
|
-
// 由 main.ts 驱动)裁;终局 ⇒ releaseClaim 连撕裂文件一并收走。文件本身不重写 —— 索引才是
|
|
260
|
-
// 运行期的判据面,重写一个正在被 EEXIST 依赖的文件反而引入新的失败形。
|
|
261
164
|
this.active.set(survivor.sessionId, survivor.taskId);
|
|
262
165
|
continue;
|
|
263
166
|
}
|
|
264
|
-
// A1(rescan 2026-08-11):解析不出记录(零长/被撕/缺键)**且无任何非终局行认领**的 claim 文件,
|
|
265
|
-
// 在 boot 这一刻可以无条件判残骸 —— 单写者锁作证没有并存的活写者(与 reclaimOrphanedAtBoot
|
|
266
|
-
// 同一条论证),而 writeThenLink 的原子性保证活写者留不下这种形。不收的话它进不了索引;
|
|
267
|
-
// sweepClaims 只扫索引,这枚文件从此无人能删,EEXIST 把该会话永远拦死(createRun 的 torn
|
|
268
|
-
// 裁定救得了新提交,救不了「残骸文件常驻 activeDir」本身)。隔离到 tmp(同 takeOverClaim 的
|
|
269
|
-
// 姿势):留证据,且绝不留在 activeDir 里等下次 hydrate 又把它当 claim 读一遍。
|
|
270
167
|
const quarantine = join(this.tmpDir, `torn-claim.${sanitized}.${process.pid}.${randomBytes(6).toString("hex")}`);
|
|
271
168
|
try {
|
|
272
169
|
renameSync(path, quarantine);
|
|
273
170
|
}
|
|
274
171
|
catch (e) {
|
|
275
172
|
if (e.code === "ENOENT")
|
|
276
|
-
continue;
|
|
277
|
-
// 搬不动(EPERM/EIO…)再试直删;仍失败 = 残骸文件继续拦人,按 F 类留痕(同 tag:都是
|
|
278
|
-
// 「陈旧 claim 的清理没做成」这一种形),下次 boot 重试。
|
|
173
|
+
continue;
|
|
279
174
|
try {
|
|
280
175
|
rmSync(path, { force: true });
|
|
281
176
|
}
|
|
@@ -285,7 +180,6 @@ export class FileRunStore {
|
|
|
285
180
|
}
|
|
286
181
|
}
|
|
287
182
|
}
|
|
288
|
-
/** Commit a RunRow to disk (whole-file atomic replace) AND the index — the single write path for every status change. */
|
|
289
183
|
persist(r) {
|
|
290
184
|
mkdirSync(this.runDir(r.taskId), { recursive: true });
|
|
291
185
|
atomicWriteFile(this.tmpDir, this.runJsonPath(r.taskId), `${JSON.stringify(this.toDisk(r))}\n`);
|
|
@@ -300,16 +194,7 @@ export class FileRunStore {
|
|
|
300
194
|
}
|
|
301
195
|
return log;
|
|
302
196
|
}
|
|
303
|
-
/** Create a run iff the session has no active run (the task_active unique-key claim). The claim file is created with
|
|
304
|
-
* writeThenLink (atomic): if it already exists (EEXIST) the session has an active run → lose, return the holder. */
|
|
305
197
|
async createRun(taskId, sessionId, owner, instanceId, meta = {}) {
|
|
306
|
-
// Atomic single-active claim: writeThenLink throws EEXIST if the claim already exists (the file twin of the
|
|
307
|
-
// task_active unique key). The index is the fast-path read; the file is the authority on a concurrent create.
|
|
308
|
-
// #213: an EEXIST no longer ends the story — the holder is ADJUDICATED (live vs dead-owner debris) and a
|
|
309
|
-
// provably-stale claim is TAKEN OVER.
|
|
310
|
-
// 🔴 **claim 与 run 行整段都在临界区里**(不只是取 claim 那一步)——SQL 孪生把两件事放在**一个事务**里,
|
|
311
|
-
// 本地车道的等价物就是这段临界区。只锁取 claim 会留下一个真窗:A 拿到 claim、行还没落盘时 B 就进来了,
|
|
312
|
-
// B 读到「有 claim 无行」⇒ 按残骸把 A 刚拿到的 claim 接管走 —— L7 那一格第二次红就是这条路径。
|
|
313
198
|
return this.withClaimLock(sessionId, async () => {
|
|
314
199
|
const acquired = await this.acquireClaim(sessionId, taskId);
|
|
315
200
|
if (!acquired.ok)
|
|
@@ -317,7 +202,6 @@ export class FileRunStore {
|
|
|
317
202
|
return this.commitNewRun(taskId, sessionId, owner, instanceId, meta);
|
|
318
203
|
});
|
|
319
204
|
}
|
|
320
|
-
/** #213 —— createRun 临界区的后半段:落 claim 索引 + run 行(两者一体,行写失败即回滚 claim)。 */
|
|
321
205
|
commitNewRun(taskId, sessionId, owner, instanceId, meta) {
|
|
322
206
|
this.active.set(sessionId, taskId);
|
|
323
207
|
const now = new Date();
|
|
@@ -341,15 +225,11 @@ export class FileRunStore {
|
|
|
341
225
|
});
|
|
342
226
|
}
|
|
343
227
|
catch (e) {
|
|
344
|
-
// R9 修:claim 与 run 行是一体的 —— 行没落下(如超长 taskId 的 ENAMETOOLONG)就把刚落的 claim 撤掉再
|
|
345
|
-
// rethrow。否则 session 被一个 getRun=undefined 的幽灵 taskId 占用:起不了新 run、也取消不了"活跃 run",
|
|
346
|
-
// 直到进程重启 sweepClaims 才解(SQL 孪生天然免疫:createRun 整体在一个事务里)。
|
|
347
228
|
this.active.delete(sessionId);
|
|
348
229
|
try {
|
|
349
230
|
rmSync(this.activePath(sessionId), { force: true });
|
|
350
231
|
}
|
|
351
232
|
catch {
|
|
352
|
-
/* best-effort:claim 文件撤不掉时至少 index 已撤;重启 sweep 兜底 */
|
|
353
233
|
}
|
|
354
234
|
throw e;
|
|
355
235
|
}
|
|
@@ -388,14 +268,8 @@ export class FileRunStore {
|
|
|
388
268
|
}
|
|
389
269
|
async appendEvent(taskId, seq, type, data) {
|
|
390
270
|
const list = this.events.get(taskId) ?? [];
|
|
391
|
-
// Mirror the (task_id, seq) PK: a duplicate seq is a caller bug (callers use ++seq); ignore a re-append of the
|
|
392
|
-
// same seq rather than corrupt the log (the SQL twin throws ER_DUP_ENTRY — callers never hit it).
|
|
393
271
|
if (list.some((e) => e.seq === seq))
|
|
394
272
|
return;
|
|
395
|
-
// Deep-clone `data` on WRITE so the stored event is isolated from later caller mutation — the SQL twin
|
|
396
|
-
// JSON.stringify's it into the column, giving the same write-isolation (parity oracle). Persist BEFORE indexing
|
|
397
|
-
// so a crash after the append still replays the event (fsync:false — events are high-frequency, the torn-tail
|
|
398
|
-
// drop on replay covers a crash tail; only run.json status + the active claim need fsync).
|
|
399
273
|
const clone = data == null ? null : structuredClone(data);
|
|
400
274
|
const ts = new Date().toISOString();
|
|
401
275
|
this.eventLogFor(taskId).append({ seq, type, data: clone, ts }, false);
|
|
@@ -412,21 +286,14 @@ export class FileRunStore {
|
|
|
412
286
|
}
|
|
413
287
|
async getEvents(taskId, afterSeq) {
|
|
414
288
|
const list = this.events.get(taskId) ?? [];
|
|
415
|
-
// Deep-clone `data` on READ so each call yields a fresh, mutation-isolated object — the SQL twin re-parses the
|
|
416
|
-
// JSON column per read (parity oracle); a shallow `{...e}` would alias `data` into the stored list (review footgun).
|
|
417
289
|
return list
|
|
418
290
|
.filter((e) => e.seq > afterSeq)
|
|
419
291
|
.sort((a, b) => a.seq - b.seq)
|
|
420
292
|
.map((e) => ({ ...e, data: e.data == null ? null : structuredClone(e.data) }));
|
|
421
293
|
}
|
|
422
|
-
/** [888] FIRST terminal writer wins(SQL 孪生同款正向 CAS on running/suspended/needs_review,
|
|
423
|
-
* [1.207 codex M1] 负向形漏 blocked/timeout);claim 释放保持无条件(幂等)。 */
|
|
424
294
|
async setTerminal(taskId, status, result, error) {
|
|
425
295
|
const r = this.runs.get(taskId);
|
|
426
296
|
if (!r) {
|
|
427
|
-
// [2255]③ claim 孤儿修:行不在(跨进程窗 —— claim 落盘、行未落/已失;R9 只兜同进程写失败)时,
|
|
428
|
-
// 终局动词仍必须能放锁:cancel 是文案承诺的用户自救路,它对孤儿失效=会话钉死到 reaper/boot。
|
|
429
|
-
// 反查 active 索引找持有该 taskId 的 session(O(claims),claims 数=活跃会话数,小)。
|
|
430
297
|
for (const [sessionId, tid] of this.active)
|
|
431
298
|
if (tid === taskId) {
|
|
432
299
|
this.releaseClaim(sessionId);
|
|
@@ -442,11 +309,9 @@ export class FileRunStore {
|
|
|
442
309
|
r.updatedAt = new Date();
|
|
443
310
|
this.persist(r);
|
|
444
311
|
}
|
|
445
|
-
// Release the single-active-run claim (DELETE FROM task_active WHERE task_id = ?) — unlink the claim file + index.
|
|
446
312
|
if (this.active.get(r.sessionId) === taskId)
|
|
447
313
|
this.releaseClaim(r.sessionId);
|
|
448
314
|
}
|
|
449
|
-
/** Durable F4: park NON-terminal `suspended`, KEEP the task_active claim. CAS on running/suspended (reaper-revert guard). */
|
|
450
315
|
async setSuspended(taskId) {
|
|
451
316
|
const r = this.runs.get(taskId);
|
|
452
317
|
if (r && (r.status === "running" || r.status === "suspended")) {
|
|
@@ -455,7 +320,6 @@ export class FileRunStore {
|
|
|
455
320
|
this.persist(r);
|
|
456
321
|
}
|
|
457
322
|
}
|
|
458
|
-
/** design/80 D-B: park NON-terminal `needs_review`, KEEP the claim. CAS on running/needs_review. */
|
|
459
323
|
async setNeedsReview(taskId) {
|
|
460
324
|
const r = this.runs.get(taskId);
|
|
461
325
|
if (r && (r.status === "running" || r.status === "needs_review")) {
|
|
@@ -464,7 +328,6 @@ export class FileRunStore {
|
|
|
464
328
|
this.persist(r);
|
|
465
329
|
}
|
|
466
330
|
}
|
|
467
|
-
/** Durable F4 resume: flip suspended/needs_review → running (CAS) + reset cancel/preempt flags. */
|
|
468
331
|
async markResuming(taskId) {
|
|
469
332
|
const r = this.runs.get(taskId);
|
|
470
333
|
if (!r || (r.status !== "suspended" && r.status !== "needs_review"))
|
|
@@ -483,33 +346,27 @@ export class FileRunStore {
|
|
|
483
346
|
const r = this.runs.get(taskId);
|
|
484
347
|
return r ? this.toRecord(r) : undefined;
|
|
485
348
|
}
|
|
486
|
-
/** Runs newest-first, keyset on (createdAt, taskId) DESC; optional exact-match filters (owner is exact, like the SQL). */
|
|
487
349
|
async listRuns(opts) {
|
|
488
|
-
// 夜测 N1(2026-07-09,LOW):a non-date cursor.createdAt made the four backends diverge (PG threw a raw
|
|
489
|
-
// DB error; the other three returned an empty page). HTTP's decodeCursor pre-validates, so this is
|
|
490
|
-
// defence-in-depth for direct store callers — normalize ALL backends to the empty page.
|
|
491
350
|
if (opts.cursor && Number.isNaN(Date.parse(opts.cursor.createdAt)))
|
|
492
351
|
return [];
|
|
493
352
|
const cursorAt = opts.cursor ? Date.parse(opts.cursor.createdAt) : undefined;
|
|
494
353
|
const rows = [...this.runs.values()]
|
|
495
354
|
.filter((r) => (opts.status ? r.status === opts.status : true))
|
|
496
|
-
.filter((r) => (opts.jobId !== undefined ? r.jobId === opts.jobId : true))
|
|
497
|
-
.filter((r) => (opts.source !== undefined ? r.source === opts.source : true))
|
|
498
|
-
.filter((r) => (opts.owner !== undefined ? r.owner === opts.owner : true))
|
|
355
|
+
.filter((r) => (opts.jobId !== undefined ? r.jobId === opts.jobId : true))
|
|
356
|
+
.filter((r) => (opts.source !== undefined ? r.source === opts.source : true))
|
|
357
|
+
.filter((r) => (opts.owner !== undefined ? r.owner === opts.owner : true))
|
|
499
358
|
.filter((r) => {
|
|
500
359
|
if (cursorAt === undefined)
|
|
501
360
|
return true;
|
|
502
361
|
const t = r.createdAt.getTime();
|
|
503
|
-
return t < cursorAt || (t === cursorAt && r.taskId < opts.cursor.taskId);
|
|
362
|
+
return t < cursorAt || (t === cursorAt && r.taskId < opts.cursor.taskId);
|
|
504
363
|
})
|
|
505
364
|
.sort((a, b) => b.createdAt.getTime() - a.createdAt.getTime() || (a.taskId < b.taskId ? 1 : a.taskId > b.taskId ? -1 : 0));
|
|
506
365
|
return rows.slice(0, opts.limit).map((r) => this.toRecord(r));
|
|
507
366
|
}
|
|
508
|
-
/** DISTINCT sessions newest-first by last activity (the CC /resume picker): latest run's preview/status + first/last/count. */
|
|
509
367
|
async listSessions(opts) {
|
|
510
368
|
const bySession = new Map();
|
|
511
369
|
for (const r of this.runs.values()) {
|
|
512
|
-
// exact owner filter (SQL `owner = ?`); "" is a valid exact owner (B10). `includeUnowned` = [4198] 窄臂第三条件(SQL `OR owner IS NULL`)。
|
|
513
370
|
if (opts.owner !== undefined && r.owner !== opts.owner && !(opts.includeUnowned === true && r.owner === null))
|
|
514
371
|
continue;
|
|
515
372
|
const arr = bySession.get(r.sessionId) ?? [];
|
|
@@ -517,14 +374,11 @@ export class FileRunStore {
|
|
|
517
374
|
bySession.set(r.sessionId, arr);
|
|
518
375
|
}
|
|
519
376
|
const cursorAt = opts.cursor ? Date.parse(opts.cursor.lastActivityAt) : undefined;
|
|
520
|
-
const qlc = opts.q?.toLowerCase();
|
|
521
|
-
// (this lister is UNREACHABLE via GET /v1/sessions today [local backend's sessionStorage lister wins], but a
|
|
522
|
-
// carried lister must not silently ignore a filter key — the workflow-audit hygiene item, 2026-07-13).
|
|
377
|
+
const qlc = opts.q?.toLowerCase();
|
|
523
378
|
const summaries = [];
|
|
524
379
|
for (const [sessionId, arr] of bySession) {
|
|
525
380
|
if (qlc && !arr.some((r) => r.objectivePreview?.toLowerCase().includes(qlc)))
|
|
526
381
|
continue;
|
|
527
|
-
// rn=1 = latest run (createdAt DESC, taskId DESC) → preview/status/owner.
|
|
528
382
|
const sorted = [...arr].sort((a, b) => b.createdAt.getTime() - a.createdAt.getTime() || (a.taskId < b.taskId ? 1 : a.taskId > b.taskId ? -1 : 0));
|
|
529
383
|
const latest = sorted[0];
|
|
530
384
|
const times = arr.map((r) => r.createdAt.getTime());
|
|
@@ -538,8 +392,8 @@ export class FileRunStore {
|
|
|
538
392
|
runCount: arr.length,
|
|
539
393
|
objectivePreview: latest.objectivePreview,
|
|
540
394
|
lastStatus: latest.status,
|
|
541
|
-
lastRunId: latest.taskId,
|
|
542
|
-
title: null,
|
|
395
|
+
lastRunId: latest.taskId,
|
|
396
|
+
title: null,
|
|
543
397
|
});
|
|
544
398
|
}
|
|
545
399
|
return summaries
|
|
@@ -552,12 +406,8 @@ export class FileRunStore {
|
|
|
552
406
|
.sort((a, b) => Date.parse(b.lastActivityAt) - Date.parse(a.lastActivityAt) || (a.sessionId < b.sessionId ? 1 : a.sessionId > b.sessionId ? -1 : 0))
|
|
553
407
|
.slice(0, opts.limit);
|
|
554
408
|
}
|
|
555
|
-
/** E21 (§0.5 delete) — purge the runs ledger for one session, scoped by `session_id` ALONE (per-row owner is the
|
|
556
|
-
* submitting principal and diverges from the session owner — full 判据 see the SQL twin's head-note, F-A);
|
|
557
|
-
* aborts (returns {active}) if a run is live。`expectedSessionOwner` = SQL 双生的化身围栏入参,本孪生执行不了
|
|
558
|
-
* (账本里没有 session_meta),理由与替代围栏逐字见 {@link MemoryRunStore.deleteBySession}。 */
|
|
559
409
|
async deleteBySession(sessionId, expectedSessionOwner) {
|
|
560
|
-
void expectedSessionOwner;
|
|
410
|
+
void expectedSessionOwner;
|
|
561
411
|
const claimed = this.active.get(sessionId);
|
|
562
412
|
if (claimed !== undefined)
|
|
563
413
|
return { active: claimed };
|
|
@@ -570,7 +420,6 @@ export class FileRunStore {
|
|
|
570
420
|
}
|
|
571
421
|
return { removed };
|
|
572
422
|
}
|
|
573
|
-
/** 用量扫窗——in-memory 腿:遍历窗内行,投影同 usage-analytics 纯函数(SQL 孪生 parity)。 */
|
|
574
423
|
async usageScan(fromMs, toMs, opts) {
|
|
575
424
|
const limit = Math.min(opts?.limit ?? USAGE_SCAN_LIMIT, USAGE_SCAN_LIMIT);
|
|
576
425
|
const all = [];
|
|
@@ -591,16 +440,13 @@ export class FileRunStore {
|
|
|
591
440
|
for (const r of this.runs.values()) {
|
|
592
441
|
if (r.createdAt.getTime() < sinceMs)
|
|
593
442
|
continue;
|
|
594
|
-
const key = JSON.stringify([r.source, r.status]);
|
|
443
|
+
const key = JSON.stringify([r.source, r.status]);
|
|
595
444
|
const cur = counts.get(key) ?? { source: r.source, status: r.status, count: 0 };
|
|
596
445
|
cur.count += 1;
|
|
597
446
|
counts.set(key, cur);
|
|
598
447
|
}
|
|
599
448
|
return [...counts.values()];
|
|
600
449
|
}
|
|
601
|
-
/** Fail stale `running` rows (crashed-run backstop) + release task_active for any now-terminal row. */
|
|
602
|
-
/** Release any claim whose row is now terminal; KEEP suspended/needs_review (resume re-finds them). One place for
|
|
603
|
-
* the claim-release invariant, shared by reapRunning + reclaimOrphanedAtBoot. */
|
|
604
450
|
sweepClaims() {
|
|
605
451
|
for (const [sessionId, taskId] of [...this.active]) {
|
|
606
452
|
const r = this.runs.get(taskId);
|
|
@@ -608,7 +454,6 @@ export class FileRunStore {
|
|
|
608
454
|
this.releaseClaim(sessionId);
|
|
609
455
|
}
|
|
610
456
|
}
|
|
611
|
-
/** Flip every `running` row matching `pred` → failed (with `reason`), then sweep terminal claims. */
|
|
612
457
|
reapRunning(pred, reason) {
|
|
613
458
|
let reaped = 0;
|
|
614
459
|
for (const r of this.runs.values()) {
|
|
@@ -627,18 +472,6 @@ export class FileRunStore {
|
|
|
627
472
|
const cutoff = Date.now() - olderThanMs;
|
|
628
473
|
return this.reapRunning((r) => r.updatedAt.getTime() < cutoff, "run stalled (instance lost?)");
|
|
629
474
|
}
|
|
630
|
-
/** TOC integration ask③ (INTEGRATION-DESIGN §11): a host-lane engine restart leaves the PREVIOUS process's
|
|
631
|
-
* `running` rows orphaned — their in-process driver (runInBackground, in the dead process's memory) is gone, but the
|
|
632
|
-
* `active/<sessionId>` claim persists on disk and EEXIST-blocks a re-submit (the shell reads the session as stuck).
|
|
633
|
-
* The boot flock (store-backend `LOCK`) already proves no concurrent live owner reached here: a still-alive old
|
|
634
|
-
* engine holds it so the new one FAILS FAST; a dead one had it kernel-released. So a `running` row here is orphaned
|
|
635
|
-
* → flip failed + release its claim UNCONDITIONALLY (no TTL wait, unlike reapStale's steady-state poll).
|
|
636
|
-
* 🔴 EXCEPT a resume-in-progress row: `markResuming` flips a suspended run (suspended→running) while its pending
|
|
637
|
-
* checkpoint still exists, and the resume picker will re-drive it — reclaiming it would STRAND a resumable session
|
|
638
|
-
* as failed. `isResumable(sessionId)` (wired by main.ts boot, which holds the checkpointStore) excludes those; with
|
|
639
|
-
* no callback every running row is treated as orphaned (conservative default when no checkpoint plane is wired).
|
|
640
|
-
* HOST/FILE LANE ONLY — the cross-replica tidb/pg store uses heartbeat→reapStale (a running row there may be alive
|
|
641
|
-
* on another replica; an unconditional reclaim would kill a live fleet run). */
|
|
642
475
|
async reclaimOrphanedAtBoot(isResumable) {
|
|
643
476
|
const orphans = [];
|
|
644
477
|
for (const r of this.runs.values()) {
|
|
@@ -654,12 +487,9 @@ export class FileRunStore {
|
|
|
654
487
|
this.sweepClaims();
|
|
655
488
|
return orphans.length;
|
|
656
489
|
}
|
|
657
|
-
/** [868] wire the checkpoint probe (LocalBackend.checkpoint(), where checkpoint + run store share one root). */
|
|
658
490
|
setCheckpointProbe(probe) {
|
|
659
491
|
this.checkpointProbe = probe;
|
|
660
492
|
}
|
|
661
|
-
/** [868] shared tail of both suspended-run reapers: flip one parked row → failed('approval.expired') and
|
|
662
|
-
* PERSIST it — the file twin of the SQL UPDATE (`error`/`error_code` set directly, no result blob). */
|
|
663
493
|
failExpiredPark(r) {
|
|
664
494
|
r.status = "failed";
|
|
665
495
|
r.error = "approval expired before decision";
|
|
@@ -667,22 +497,12 @@ export class FileRunStore {
|
|
|
667
497
|
r.updatedAt = new Date();
|
|
668
498
|
this.persist(r);
|
|
669
499
|
}
|
|
670
|
-
/**
|
|
671
|
-
* [868] Durable F4 expiry — the file twin of TiDBRunStore.reapSuspended, driven by the injected checkpoint
|
|
672
|
-
* probe instead of a SQL JOIN (formerly a NO-OP — one of the five closed doors in the "suspended run locks the
|
|
673
|
-
* session forever" incident). Semantics verbatim from the SQL twin: suspended ∧ updatedAt < cutoff ∧ NOT
|
|
674
|
-
* probe.hasPending(session) → failed 'approval expired before decision' / errorCode 'approval.expired' +
|
|
675
|
-
* release the task_active claim. The NOT-pending clause is the design/80 D-D adversarial-blocker guard
|
|
676
|
-
* (SQL 孪生现在的真身:`run-store-sql.ts` 的 `reapSuspended`;`tidb-run-store.ts` 该壳文件已删(5.0.0)): a row whose checkpoint is STILL pending belongs to the checkpoint-state sweeps —
|
|
677
|
-
* time-reaping it would orphan a model leg on a now-unlocked session. Probe absent ⇒ honest NO-OP.
|
|
678
|
-
* MemoryRunStore is the parity oracle (byte-identical predicate; only the persist differs).
|
|
679
|
-
*/
|
|
680
500
|
async reapSuspended(olderThanMs) {
|
|
681
501
|
const probe = this.checkpointProbe;
|
|
682
502
|
if (!probe) {
|
|
683
503
|
this.sweepClaims();
|
|
684
504
|
return 0;
|
|
685
|
-
}
|
|
505
|
+
}
|
|
686
506
|
const cutoff = Date.now() - olderThanMs;
|
|
687
507
|
let reaped = 0;
|
|
688
508
|
for (const r of this.runs.values()) {
|
|
@@ -690,10 +510,7 @@ export class FileRunStore {
|
|
|
690
510
|
continue;
|
|
691
511
|
const snap = r.updatedAt.getTime();
|
|
692
512
|
if (await probe.hasPending(r.sessionId))
|
|
693
|
-
continue;
|
|
694
|
-
// codex H2:probe 的 await 让出事件循环——markResuming(approve 恢复)/cancel/新一轮 park 可能已改行。
|
|
695
|
-
// 复核与改写之间无 await(Node 单线程 ⇒ 原子);状态或 updatedAt 变了就放手,绝不覆盖一条已被别腿
|
|
696
|
-
// 认领的行(SQL 孪生靠单条条件 UPDATE 天然原子,这里的 snap 复核就是它的本地等价)。
|
|
513
|
+
continue;
|
|
697
514
|
if (r.status !== "suspended" || r.updatedAt.getTime() !== snap)
|
|
698
515
|
continue;
|
|
699
516
|
this.failExpiredPark(r);
|
|
@@ -702,20 +519,12 @@ export class FileRunStore {
|
|
|
702
519
|
this.sweepClaims();
|
|
703
520
|
return reaped;
|
|
704
521
|
}
|
|
705
|
-
/**
|
|
706
|
-
* [868] design/80 §3 inv#3 crash-safe backstop — the file twin of TiDBRunStore.failSuspendedWithExpiredCheckpoint
|
|
707
|
-
* (checkpoint-STATE-driven, not time-driven; scheduled UNCONDITIONALLY by the main.ts reaper, no
|
|
708
|
-
* APPROVAL_TIMEOUT_SEC gate). Semantics verbatim from the SQL twin: (suspended ∨ needs_review) ∧
|
|
709
|
-
* probe.hasExpired(session) ∧ NOT probe.hasPending(session) → same fail + claim release (wq64gmm5e: an
|
|
710
|
-
* abandoned needs_review park leaks exactly like a suspended one; a re-suspended session that minted a NEW
|
|
711
|
-
* pending gate is excluded by the NOT-pending clause).
|
|
712
|
-
*/
|
|
713
522
|
async failSuspendedWithExpiredCheckpoint() {
|
|
714
523
|
const probe = this.checkpointProbe;
|
|
715
524
|
if (!probe) {
|
|
716
525
|
this.sweepClaims();
|
|
717
526
|
return 0;
|
|
718
|
-
}
|
|
527
|
+
}
|
|
719
528
|
let reaped = 0;
|
|
720
529
|
for (const r of this.runs.values()) {
|
|
721
530
|
if (r.status !== "suspended" && r.status !== "needs_review")
|
|
@@ -726,7 +535,6 @@ export class FileRunStore {
|
|
|
726
535
|
continue;
|
|
727
536
|
if (await probe.hasPending(r.sessionId))
|
|
728
537
|
continue;
|
|
729
|
-
// codex H2 同款 snap 复核(两个 await 窗都盖住):行被别腿改过就放手。
|
|
730
538
|
if (r.status !== parkedAs || r.updatedAt.getTime() !== snap)
|
|
731
539
|
continue;
|
|
732
540
|
this.failExpiredPark(r);
|
|
@@ -735,21 +543,9 @@ export class FileRunStore {
|
|
|
735
543
|
this.sweepClaims();
|
|
736
544
|
return reaped;
|
|
737
545
|
}
|
|
738
|
-
// ── claim / run file lifecycle helpers ──
|
|
739
|
-
/**
|
|
740
|
-
* #213 —— 裁定/接管路径上唯一许可的取行方式:**盘优先**,盘上没有才回退内存索引。
|
|
741
|
-
*
|
|
742
|
-
* 复审 R2 [high]:反过来(索引优先)会在「同进程两只实例、两边索引都热」时出事 —— A 把 park 行
|
|
743
|
-
* `markResuming` 成 running 并消费掉 checkpoint,B 的索引里还留着**旧的 park 对象**;B 于是读不到
|
|
744
|
-
* 变化、拿自己那份陈旧对象通过复核,把 A 正在跑的行盖成 failed 再放锁。盘是唯一权威,判据只能读盘。
|
|
745
|
-
* 回退内存索引只为「行还没落盘」这一种形(理论上不该出现,留着不让判据凭空缺行)。
|
|
746
|
-
* 代价:只在 EEXIST 争用路径上多一次读文件,常路零开销。
|
|
747
|
-
*/
|
|
748
546
|
authoritativeRow(taskId) {
|
|
749
547
|
return this.readRowFromDisk(taskId) ?? this.runs.get(taskId);
|
|
750
548
|
}
|
|
751
|
-
/** #213 —— 越过内存索引直接读盘上的那一行(索引可能比另一实例的写晚一步;判据必须以盘为准)。
|
|
752
|
-
* 读不出/坏行 ⇒ undefined,与 hydrate 的跳过口径一致。 */
|
|
753
549
|
readRowFromDisk(taskId) {
|
|
754
550
|
const path = this.runJsonPath(taskId);
|
|
755
551
|
if (!existsSync(path))
|
|
@@ -763,24 +559,12 @@ export class FileRunStore {
|
|
|
763
559
|
return undefined;
|
|
764
560
|
return this.toRow(disk);
|
|
765
561
|
}
|
|
766
|
-
/**
|
|
767
|
-
* A1/codex R2 —— 盘上扫该会话最新的非终局行:撕裂 claim(解析不出记录)唯一还能认主的通道。
|
|
768
|
-
* 内存索引不可用作判据 —— 同进程冷索引姊妹实例的行可能晚于本实例 hydrate 才落盘(L9/[1684] 同族),
|
|
769
|
-
* 判据必须以盘为准。O(盘上 run 数),只在「claim 解析不出记录」这条罕见路径上走,常路零开销。
|
|
770
|
-
* 行的读法与 hydrate/readRowFromDisk 同一套宽容口径(坏行跳过,不毒判定)。
|
|
771
|
-
*/
|
|
772
562
|
newestNonTerminalRowOnDisk(sessionId) {
|
|
773
|
-
// readdir 失败**不吞**(静默降级棘轮三问):吞成「无行」会把「证不出有行」读成「可接管」——
|
|
774
|
-
// 在单活轴上方向反了。真 fs 故障就让 createRun 响亮地抛(takeOverClaim 对非 ENOENT 同姿势)。
|
|
775
563
|
const names = readdirSync(this.runsDir);
|
|
776
564
|
let best;
|
|
777
565
|
for (const name of names) {
|
|
778
566
|
if (name === "active")
|
|
779
567
|
continue;
|
|
780
|
-
// 与 hydrate(:283)/readRowFromDisk 同一把尺:条目不是目录 / run.json 不在 ⇒ 跳过。
|
|
781
|
-
// 少了这行守卫时,runs/ 里任何非目录杂物(.DS_Store/._* 备份残件)会让 readJsonlRecords 抛
|
|
782
|
-
// ENOTDIR(core 只吞 ENOENT)⇒ torn 裁定路 500 且 torn claim 永不清理——「会话永钉」以更响的
|
|
783
|
-
// 形回归(7.13.0 发车前增量重扫 [high] 拦下)。上面「readdir 失败不吞」只管 readdirSync 本体。
|
|
784
568
|
const path = join(this.runsDir, name, "run.json");
|
|
785
569
|
if (!existsSync(path))
|
|
786
570
|
continue;
|
|
@@ -799,25 +583,9 @@ export class FileRunStore {
|
|
|
799
583
|
}
|
|
800
584
|
return best;
|
|
801
585
|
}
|
|
802
|
-
/** 本进程要落盘的 claim 形(带持有者身份 —— #213 的活性证据)。 */
|
|
803
586
|
claimRecord(sessionId, taskId) {
|
|
804
587
|
return { sessionId, taskId, owner: { ...this.claimOwner } };
|
|
805
588
|
}
|
|
806
|
-
/**
|
|
807
|
-
* #213 —— **每会话**的 claim 获取临界区。
|
|
808
|
-
*
|
|
809
|
-
* 为什么必须有:接管的判定链里有 `await`(checkpoint probe 的两条 EXISTS 谓词),两条并发 `createRun`
|
|
810
|
-
* 会在同一枚陈旧 claim 上**各判各的**、然后**各接管各的** —— 第二条的 `rename` 搬走的是第一条刚写下的
|
|
811
|
-
* **新** claim,于是两条都拿到 ok:true,单活不变式被并发撕开。(这不是纸面推演:L7 那一格先红,就是
|
|
812
|
-
* 这条路径。`rename` 只保证「同一个源路径只成功一次」,它**不是**对被判定那份内容的 CAS。)
|
|
813
|
-
*
|
|
814
|
-
* 为什么进程内互斥就够:这条车道是「一个数据根一个写者」——`FileStorageBackend` 的 `root/LOCK` 让第二个
|
|
815
|
-
* 活进程 fail fast。跨进程并发在本车道**不存在**;而这正是接管判据本身所依赖的同一条不变式
|
|
816
|
-
* (`reclaimOrphanedAtBoot` 也全靠它),所以这里没有引入新的假设。`rename` 仍然留着当廉价二道闸:
|
|
817
|
-
* 判定与接管之间 claim 若已被别腿正常释放,它给 ENOENT,整轮重来。
|
|
818
|
-
*
|
|
819
|
-
* 形状:每 session 一条 promise 链,后来者 await 前一位;链尾归零时删表项(不无界增长)。
|
|
820
|
-
*/
|
|
821
589
|
async withClaimLock(sessionId, fn) {
|
|
822
590
|
const prev = this.claimLocks.get(sessionId);
|
|
823
591
|
let release = () => undefined;
|
|
@@ -826,7 +594,7 @@ export class FileRunStore {
|
|
|
826
594
|
});
|
|
827
595
|
this.claimLocks.set(sessionId, mine);
|
|
828
596
|
if (prev)
|
|
829
|
-
await prev;
|
|
597
|
+
await prev;
|
|
830
598
|
try {
|
|
831
599
|
return await fn();
|
|
832
600
|
}
|
|
@@ -836,7 +604,6 @@ export class FileRunStore {
|
|
|
836
604
|
this.claimLocks.delete(sessionId);
|
|
837
605
|
}
|
|
838
606
|
}
|
|
839
|
-
/** #213 —— 临界区内的 claim 获取本体:铸不下就裁定持有者,判陈旧则接管后重铸。有界重试(论证见 takeOverClaim)。 */
|
|
840
607
|
async acquireClaim(sessionId, taskId) {
|
|
841
608
|
for (let attempt = 0; attempt < CLAIM_TAKEOVER_ATTEMPTS; attempt += 1) {
|
|
842
609
|
try {
|
|
@@ -846,20 +613,14 @@ export class FileRunStore {
|
|
|
846
613
|
catch (e) {
|
|
847
614
|
if (e.code !== "EEXIST")
|
|
848
615
|
throw e;
|
|
849
|
-
// Recover the current holder from the claim file (the index may lag a concurrent winner, but in this
|
|
850
|
-
// single-process model the index is authoritative; read disk for robustness anyway).
|
|
851
|
-
// 有效形读法(readClaimRecord):缺键/错会话的记录读成 undefined ⇒ 走撕裂裁定的扫盘认主。
|
|
852
616
|
const claim = readClaimRecord(readJsonlRecords(this.activePath(sessionId)), sessionId);
|
|
853
617
|
const holder = claim?.taskId ?? this.active.get(sessionId) ?? "";
|
|
854
618
|
const verdict = await this.judgeClaim(sessionId, claim, holder);
|
|
855
619
|
if (verdict.kind === "live")
|
|
856
620
|
return { ok: false, activeTaskId: verdict.holderTaskId ?? holder };
|
|
857
|
-
this.takeOverClaim(sessionId, verdict.reason, verdict.snapshot);
|
|
621
|
+
this.takeOverClaim(sessionId, verdict.reason, verdict.snapshot);
|
|
858
622
|
}
|
|
859
623
|
}
|
|
860
|
-
// 预算用尽的收口(复审 R2 [medium]):最后一轮里**接管刚刚成功**时,循环已经没有回合去铸新 claim 了。
|
|
861
|
-
// 此时直接回 409 是**撒谎** —— 会话其实是空的,且 activeTaskId 只能给出空串,调用方连 attach 到谁都不知道。
|
|
862
|
-
// 所以先诚实地再铸一次;只有真的还被人占着(EEXIST)才报冲突,而且报的是**盘上真持有者**。
|
|
863
624
|
try {
|
|
864
625
|
writeThenLink(this.activePath(sessionId), `${JSON.stringify(this.claimRecord(sessionId, taskId))}\n`);
|
|
865
626
|
return { ok: true };
|
|
@@ -871,52 +632,18 @@ export class FileRunStore {
|
|
|
871
632
|
const finalClaim = readClaimRecord(readJsonlRecords(this.activePath(sessionId)), sessionId);
|
|
872
633
|
return { ok: false, activeTaskId: finalClaim?.taskId ?? this.active.get(sessionId) ?? "" };
|
|
873
634
|
}
|
|
874
|
-
/**
|
|
875
|
-
* #213 —— 撞上 EEXIST 之后,对**现持有者**的裁定。默认方向是 `live`(挡住):只有能**证明**
|
|
876
|
-
* 持有者已死的形才判 `stale`。证不出来一律 fail-closed —— 单活不变式比自愈更重要。
|
|
877
|
-
*
|
|
878
|
-
* 三族:
|
|
879
|
-
* ① 行不在 / 行已终局 —— claim 是残骸(R9 回滚失手、跨进程窗)。`sweepClaims` 本该收掉,
|
|
880
|
-
* 它只在 boot 与几条 reaper 上跑;这里就地补齐。
|
|
881
|
-
* ② 行 `running` —— 有没有在飞的驱动腿,取决于**谁**写的这枚 claim:本进程写的 ⇒ 驱动腿就在
|
|
882
|
-
* 本进程内存里 ⇒ 真活主;别的进程写的 ⇒ 那个进程已死(boot 单写者锁作证)⇒ 驱动腿随它一起没了,
|
|
883
|
-
* 行永远不会自己走到终局。这与 `reclaimOrphanedAtBoot` 的判决**逐字同源**,只是时点不同。
|
|
884
|
-
* ③ 行 `suspended`/`needs_review`(park)—— park 按契约**没有在飞的腿**(腿返回 suspended 后就退出了,
|
|
885
|
-
* resume 腿会先 `markResuming` 翻回 running),所以这一族与「谁写的」无关,只问一件事:
|
|
886
|
-
* **还有东西可续吗**。有 pending ⇒ 可续(409 体里的 pendingGate 就是出路);有 expired ⇒ 归
|
|
887
|
-
* [868] `failSuspendedWithExpiredCheckpoint` 那条腿收(它要落 `approval.expired` 这个终局语义,
|
|
888
|
-
* 本函数不许抢);两者皆无 ⇒ 这条 park 谁也续不动,而**没有任何一条排期腿够得着它**
|
|
889
|
-
* (reapStale 只碰 running;boot 回收刻意跳过 park;时间腿 reapSuspended 挂 APPROVAL_TIMEOUT_SEC,
|
|
890
|
-
* 默认 0 = 不排期)⇒ 永久占用,判 stale。probe 缺席 ⇒ 证不出「无可续」⇒ fail-closed 判 live。
|
|
891
|
-
*/
|
|
892
635
|
async judgeClaim(sessionId, claim, holderTaskId) {
|
|
893
636
|
const owner = claim?.owner;
|
|
894
637
|
const who = owner
|
|
895
638
|
? `pid ${owner.pid} started ${new Date(owner.startedAtMs).toISOString()}`
|
|
896
639
|
: "an owner-less claim (written before the liveness stamp existed)";
|
|
897
640
|
const torn = claim === undefined;
|
|
898
|
-
// 🔴 索引里没有 ≠ 行不存在:另一个实例在本实例 hydrate **之后**建的 run,只活在盘上。拿索引当判据
|
|
899
|
-
// 会把它读成「幽灵 claim」并抢走一个**真活主**的锁([1684] 的两实例并发格就是这条路)。以盘为准。
|
|
900
|
-
// 撕裂形跳过这次急切取行(holder 是索引猜的,可能陈旧甚至不安全)——撕裂臂里无条件扫盘认主。
|
|
901
641
|
let row = !torn && holderTaskId ? this.authoritativeRow(holderTaskId) : undefined;
|
|
902
642
|
let holder = holderTaskId;
|
|
903
|
-
// ── A1(rescan 2026-08-11 + codex R2 两条 [high]):claim 文件在(EEXIST 撞上了它)却解析不出任何
|
|
904
|
-
// 记录 = 「被撕的写」(readJsonlRecords 契约:零长=[],无尾换行的末行被丢)。撕的是**文件**,不是
|
|
905
|
-
// 会话的占用语义 —— 修前这形恒 fail-closed live 且 activeTaskId 为空、任何收割腿都够不着,会话被一枚
|
|
906
|
-
// 谁都删不掉的空文件永久钉死。裁定:唯一还能认主的通道是**行**,而行必须**以盘为准**(同进程冷索引
|
|
907
|
-
// 姊妹实例的行可能不在本实例内存里 —— L9/[1684] 冷索引形 × 外力截断的组合,codex R2 第二条)。
|
|
908
|
-
// · 认出非终局行 ⇒ 把持有者当它,走下面与可解析 claim **同一套**裁定(park 族问可续性、running 族要
|
|
909
|
-
// 死亡证据;文件不可读 ⇒ 无 owner 证据 ⇒ running 族自然 fail-closed)。blanket-live 是反例:它会把
|
|
910
|
-
// 「无可续的老 park」这一 L2 形永久锁死,而它的可解析孪生早就该被接管(codex R2 第一条)。
|
|
911
|
-
// · 认不出行:本实例索引在飞(claim 已铸、行未落的半拍)⇒ live;索引也没有 ⇒ 单写者不变式作证
|
|
912
|
-
// (活写者只能是本进程,而本进程持有中的 claim 必在索引里)⇒ 死进程残骸,判 stale。
|
|
913
643
|
if (torn) {
|
|
914
|
-
// codex R3 [high]:认主**无条件**以盘上最新非终局行为准 —— 索引猜出来的 holder 可能是姊妹实例
|
|
915
|
-
// 缓存的**旧**持有者(行已终局):拿它当行会遮住接班的 park(409 报一个终局名字,替班行照 live
|
|
916
|
-
// 挡住,永锁)。索引只在「盘上无非终局行」时兜「claim 已铸、行未落」的本进程半拍。
|
|
917
644
|
row = this.newestNonTerminalRowOnDisk(sessionId);
|
|
918
645
|
if (row) {
|
|
919
|
-
holder = row.taskId;
|
|
646
|
+
holder = row.taskId;
|
|
920
647
|
}
|
|
921
648
|
else if (this.active.has(sessionId)) {
|
|
922
649
|
return { kind: "live", holderTaskId: this.active.get(sessionId) };
|
|
@@ -929,10 +656,6 @@ export class FileRunStore {
|
|
|
929
656
|
};
|
|
930
657
|
}
|
|
931
658
|
}
|
|
932
|
-
// 判定链里的每个 await 都会让出事件循环 —— 复审(codex R1 [critical])的靶心:probe 期间 resume 腿
|
|
933
|
-
// 可能已经 `markResuming`(park→running)、终局腿可能已经放锁重铸。所以这里先取一份快照,接管前
|
|
934
|
-
// 必须**逐字复核**(见 takeOverClaim);任何一项动了就放手重来。同一条纪律在 reapSuspended 里
|
|
935
|
-
// 已经有先例(那处的 `snap` 复核),本函数照抄并把 **claim 身份**也一并纳入。
|
|
936
659
|
const snapshot = {
|
|
937
660
|
taskId: holder,
|
|
938
661
|
ownerNonce: owner?.nonce,
|
|
@@ -940,105 +663,53 @@ export class FileRunStore {
|
|
|
940
663
|
updatedAtMs: row?.updatedAt.getTime(),
|
|
941
664
|
...(torn ? { torn: true } : {}),
|
|
942
665
|
};
|
|
943
|
-
/** live 判决统一带上真持有者(撕裂形的 holder 是从行认出来的,调用方的 claim 读数里没有它)。 */
|
|
944
666
|
const live = () => ({ kind: "live", ...(holder ? { holderTaskId: holder } : {}) });
|
|
945
|
-
// ── ③ park 族:与「谁写的」无关,只问「还有东西可续吗」 ──
|
|
946
667
|
if (row && (row.status === "suspended" || row.status === "needs_review")) {
|
|
947
668
|
const probe = this.checkpointProbe;
|
|
948
669
|
if (!probe)
|
|
949
|
-
return live();
|
|
670
|
+
return live();
|
|
950
671
|
if (await probe.hasPending(row.sessionId))
|
|
951
|
-
return live();
|
|
672
|
+
return live();
|
|
952
673
|
if (await probe.hasExpired(row.sessionId))
|
|
953
|
-
return live();
|
|
954
|
-
// 🔴 复审 R1 [critical] 的残余窗:decide 腿「解掉 checkpoint」与「markResuming」之间,行还是 park、
|
|
955
|
-
// checkpoint 已不 pending —— 快照复核看不出区别。用一道**静置线**盖住它:只收「已经躺了
|
|
956
|
-
// PARK_ORPHAN_SETTLE_MS 没人动过」的 park。这不是把 TTL 当修法(修法是「无 checkpoint 可续」
|
|
957
|
-
// 这条新谓词 —— 今天没有任何一条腿有它);静置线只是防竞态的余量,而它要清的占用本来是**无界**的,
|
|
958
|
-
// 多等 30 秒零代价。真要跨过它,得有一条 resume 腿在解 checkpoint 与 markResuming 之间卡满 30 秒。
|
|
674
|
+
return live();
|
|
959
675
|
if (Date.now() - row.updatedAt.getTime() < PARK_ORPHAN_SETTLE_MS)
|
|
960
676
|
return live();
|
|
961
677
|
return { kind: "stale", reason: `parked run (${row.status}) has no resumable checkpoint — claim reclaimed`, snapshot };
|
|
962
678
|
}
|
|
963
|
-
// ── ①② 非 park 族:判死之前必须有**正面死亡证据** ──
|
|
964
|
-
// 证据只有两种:(a) 这枚 claim 是**本实例**写的 —— 那我对它的驱动腿有完整的内存知识;
|
|
965
|
-
// (b) 它是外来的,且 owner 的 pid **确凿不在了**(ESRCH)。
|
|
966
|
-
// 为什么不能只看 nonce:`root/LOCK` 保证的是「一个数据根一个活**进程**」,同一进程里再造第二只
|
|
967
|
-
// FileRunStore(测试/误配)并不违反它,而那第二只的 claim 背后是一个**活着的**驱动腿 —— 只看 nonce
|
|
968
|
-
// 就会把真活主抢走([1684] 两实例并发格 + 「claim 已落、行还没落」那半拍窗,两次都真红过)。
|
|
969
|
-
// 🔴 pid 复用在这里**不会造成误判**:方向是 fail-closed —— 只有「确凿不在」才判陈旧;pid 被复用 ⇒
|
|
970
|
-
// 探到「在」⇒ 判 live ⇒ 维持今天的 409(少愈一次,绝不错杀),boot 回收仍是兜底。EPERM(属于别的
|
|
971
|
-
// 用户)同样表示进程存在,按 live 处理。缺 owner 戳的老 claim 无证可查,同样 live —— 升级本身就是
|
|
972
|
-
// 一次重启,`reclaimOrphanedAtBoot`(它靠单写者锁,不需要逐 claim 的死亡证据)才是它的正经收口。
|
|
973
679
|
const mine = owner?.nonce === this.claimOwner.nonce;
|
|
974
680
|
const ownerGone = owner !== undefined && !isPidAlive(owner.pid);
|
|
975
681
|
if (!mine && !ownerGone)
|
|
976
|
-
return live();
|
|
682
|
+
return live();
|
|
977
683
|
if (!row)
|
|
978
684
|
return { kind: "stale", reason: `claim reclaimed: no run row for ${holder || "(unknown task)"} — ${who}`, snapshot };
|
|
979
685
|
if (row.status !== "running")
|
|
980
686
|
return { kind: "stale", reason: `claim reclaimed: holder already terminal (${row.status})`, snapshot };
|
|
981
|
-
// running:本实例写的 ⇒ 驱动腿就在本进程内存里 ⇒ 真活主(单活不变式在此守住)。
|
|
982
687
|
if (mine)
|
|
983
688
|
return live();
|
|
984
|
-
// 🔴 复审 R1 [high]:外来且 owner 已死 **仍不够** —— `reclaimOrphanedAtBoot` 刻意用
|
|
985
|
-
// `isResumable(sessionId)`(= 该会话有 pending checkpoint)把一类 running 行留下来:
|
|
986
|
-
// `markResuming` 会把 park 翻成 running,而它的 checkpoint 还没被消费,resume 挑选器会重新驱动它。
|
|
987
|
-
// 本函数是同一条判决的「晚一点的时点」,判据必须**逐字同源**,否则新提交会把一条 boot 特意保住的
|
|
988
|
-
// 行打成 failed、把用户还没决议的审批晾在那儿。
|
|
989
689
|
const probe = this.checkpointProbe;
|
|
990
690
|
if (probe && (await probe.hasPending(row.sessionId)))
|
|
991
691
|
return live();
|
|
992
|
-
// 外来 + owner 已死 + 无可续 ⇒ 驱动腿随那个进程没了,这行永远走不到终局 ⇒ 孤儿。
|
|
993
692
|
return { kind: "stale", reason: `run orphaned by engine restart (claim held by ${who}, which is gone)`, snapshot };
|
|
994
693
|
}
|
|
995
|
-
/**
|
|
996
|
-
* #213 —— 接管一枚已判定陈旧的 claim。返回 false = 现场已经变了 / 抢输了,调用方整轮重来。
|
|
997
|
-
* **整段同步**(无 await)⇒ 进程内不可被打断;跨进程由 rename 与单写者锁兜。
|
|
998
|
-
*
|
|
999
|
-
* ① **复核先行**(复审 codex R1 [critical]):judgeClaim 的 probe await 期间,resume 腿可能已经
|
|
1000
|
-
* `markResuming`、终局腿可能已经放锁重铸。所以第一件事是拿判定时的快照逐字比对**盘上现状**
|
|
1001
|
-
* (claim 的 taskId+owner nonce、行的 status+updatedAt)。任何一项动了 ⇒ 判决作废,不许动手。
|
|
1002
|
-
* ② **先把行写死,再拆 claim**(复审 codex R1 [medium]):次序反过来的话,「claim 已拆、行还 park」
|
|
1003
|
-
* 这半拍崩溃会留下一条**永远 park** 的行 —— boot 回收刻意跳过 park、sweepClaims 又没有 claim 可循,
|
|
1004
|
-
* 没人收得掉。现在的次序里,同一处崩溃留下的是「行已终局 + claim 还在」,而那正是 `sweepClaims`
|
|
1005
|
-
* 每次 boot / 每条 reaper 都会扫掉的形 —— 崩在哪一步都自愈。
|
|
1006
|
-
* ③ 拆 claim 用 `renameSync(claim → tmp/隔离名)`:POSIX `rename` 对同一个源路径只可能**成功一次**,
|
|
1007
|
-
* 并发的第二个 racer 拿 ENOENT。隔离件落在 `tmp/` 而不是 `runs/active/` —— `hydrate()` 把 activeDir 里的
|
|
1008
|
-
* **每个文件**都当 claim 读,隔离件留在那儿就等于把刚拆掉的幽灵在下次 boot 原样复活。
|
|
1009
|
-
*/
|
|
1010
694
|
takeOverClaim(sessionId, reason, snapshot) {
|
|
1011
|
-
// ① 复核:盘上的 claim 还是我判过的那一枚吗?(同一把有效形尺:缺键/错会话读成 undefined)
|
|
1012
695
|
const nowClaim = readClaimRecord(readJsonlRecords(this.activePath(sessionId)), sessionId);
|
|
1013
696
|
if (snapshot.torn) {
|
|
1014
|
-
// A1:被撕的形没有可比的字段 —— 复核=「现在仍然解析不出记录」。解析得出 ⇒ 判定的 await
|
|
1015
|
-
// 间隙里这枚 claim 被正常释放又被活主重铸了,判决作废(下一轮对新 claim 重新裁定)。
|
|
1016
697
|
if (nowClaim !== undefined)
|
|
1017
698
|
return false;
|
|
1018
699
|
}
|
|
1019
700
|
else if (!nowClaim || nowClaim.taskId !== snapshot.taskId || nowClaim.owner?.nonce !== snapshot.ownerNonce)
|
|
1020
701
|
return false;
|
|
1021
|
-
// 行的键统一取自快照(裁定认出的持有者)——撕裂 claim 的调用方读数里没有 taskId,快照才有(codex R2)。
|
|
1022
702
|
const row = snapshot.taskId ? this.authoritativeRow(snapshot.taskId) : undefined;
|
|
1023
703
|
if (row?.status !== snapshot.status || row?.updatedAt.getTime() !== snapshot.updatedAtMs)
|
|
1024
704
|
return false;
|
|
1025
|
-
// ② 先落终局行:留一条永远 running/park 的行会让 listRuns/fleet 显示一个永不结束的假活 run。
|
|
1026
|
-
// errorCode 刻意留空 —— 与 reclaimOrphanedAtBoot 同族(它也只写 error),不新铸 wire 错误码。
|
|
1027
|
-
// 🔴 **副本写,不原地改**:本文件其它写路径是「改 row 再 persist」,那在 persist 抛出时会留下
|
|
1028
|
-
// 「内存已 failed、盘上还 park」的分叉。接管这条路径必须是全有或全无(下一步就要拆 claim),
|
|
1029
|
-
// 所以交给 persist 去 `runs.set` —— 写盘失败就什么都没发生,claim 原封不动,下一轮重来。
|
|
1030
705
|
if (row && (row.status === "running" || row.status === "suspended" || row.status === "needs_review")) {
|
|
1031
706
|
this.persist({ ...row, status: "failed", error: reason, updatedAt: new Date() });
|
|
1032
707
|
}
|
|
1033
|
-
// ③ 再拆 claim。
|
|
1034
708
|
const quarantine = join(this.tmpDir, `stale-claim.${sanitizePathComponent(sessionId)}.${process.pid}.${randomBytes(6).toString("hex")}`);
|
|
1035
709
|
try {
|
|
1036
710
|
renameSync(this.activePath(sessionId), quarantine);
|
|
1037
711
|
}
|
|
1038
712
|
catch (e) {
|
|
1039
|
-
// ENOENT = 输掉了这一枚 claim 的接管权(或它已被正常释放);行已终局,sweepClaims 会收尾。
|
|
1040
|
-
// 其它错误(EPERM/EIO…)不是「输掉竞争」而是真实 fs 故障——吞成 false 会把故障演成永远的
|
|
1041
|
-
// 「接管失败→重试→再失败」静默循环,rethrow 让它像其它写路径一样响亮。
|
|
1042
713
|
if (e.code === "ENOENT")
|
|
1043
714
|
return false;
|
|
1044
715
|
throw e;
|
|
@@ -1047,15 +718,12 @@ export class FileRunStore {
|
|
|
1047
718
|
rmSync(quarantine, { force: true });
|
|
1048
719
|
}
|
|
1049
720
|
catch (e) {
|
|
1050
|
-
// F 类兜底:隔离件是死物,且不在 activeDir 下,留着不会被 hydrate 当成 claim 复活;
|
|
1051
|
-
// 但删不掉仍要留痕(计数+一次性 warn),否则 tmp 静默膨胀没人知道。
|
|
1052
721
|
recordFailOpen("server.run-store.stale-claim-quarantine-unremoved", e instanceof Error ? e.message : String(e));
|
|
1053
722
|
}
|
|
1054
723
|
if (this.active.get(sessionId) === snapshot.taskId)
|
|
1055
724
|
this.active.delete(sessionId);
|
|
1056
725
|
return true;
|
|
1057
726
|
}
|
|
1058
|
-
/** Release a session's single-active claim: unlink the claim file + drop the index entry (idempotent). */
|
|
1059
727
|
releaseClaim(sessionId) {
|
|
1060
728
|
this.active.delete(sessionId);
|
|
1061
729
|
const path = this.activePath(sessionId);
|
|
@@ -1064,11 +732,9 @@ export class FileRunStore {
|
|
|
1064
732
|
rmSync(path, { force: true });
|
|
1065
733
|
}
|
|
1066
734
|
catch {
|
|
1067
|
-
/* best-effort — a stale claim file is harmless (the index entry is already gone) */
|
|
1068
735
|
}
|
|
1069
736
|
}
|
|
1070
737
|
}
|
|
1071
|
-
/** Drop a run entirely (registry + event log + open fd + on-disk dir) — used by deleteBySession. */
|
|
1072
738
|
dropRun(taskId) {
|
|
1073
739
|
const log = this.eventLogs.get(taskId);
|
|
1074
740
|
if (log) {
|
|
@@ -1076,7 +742,6 @@ export class FileRunStore {
|
|
|
1076
742
|
log.close();
|
|
1077
743
|
}
|
|
1078
744
|
catch {
|
|
1079
|
-
/* best-effort */
|
|
1080
745
|
}
|
|
1081
746
|
this.eventLogs.delete(taskId);
|
|
1082
747
|
}
|
|
@@ -1088,18 +753,15 @@ export class FileRunStore {
|
|
|
1088
753
|
rmSync(dir, { recursive: true, force: true });
|
|
1089
754
|
}
|
|
1090
755
|
catch {
|
|
1091
|
-
/* best-effort */
|
|
1092
756
|
}
|
|
1093
757
|
}
|
|
1094
758
|
}
|
|
1095
|
-
/** Release every open event-log fd (called by LocalBackend.close so a graceful restart can re-open the data dir). */
|
|
1096
759
|
dispose() {
|
|
1097
760
|
for (const log of this.eventLogs.values()) {
|
|
1098
761
|
try {
|
|
1099
762
|
log.close();
|
|
1100
763
|
}
|
|
1101
764
|
catch {
|
|
1102
|
-
/* best-effort */
|
|
1103
765
|
}
|
|
1104
766
|
}
|
|
1105
767
|
this.eventLogs.clear();
|