@fyeeme/pi-dynamic-workflows 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +373 -0
- package/README.zh-CN.md +359 -0
- package/index.ts +650 -0
- package/package.json +58 -0
- package/sessions/spawn.ts +15 -0
- package/src/agent/dispatch.ts +76 -0
- package/src/budget/caps.ts +42 -0
- package/src/budget/index.ts +8 -0
- package/src/budget/pool.ts +118 -0
- package/src/cache/index.ts +7 -0
- package/src/cache/journal.ts +184 -0
- package/src/cache/key.ts +97 -0
- package/src/determinism/ast-guard.ts +196 -0
- package/src/errors.ts +55 -0
- package/src/format.ts +27 -0
- package/src/index.ts +28 -0
- package/src/inspect.ts +237 -0
- package/src/lifecycle.ts +75 -0
- package/src/loader.ts +50 -0
- package/src/outcomes.ts +113 -0
- package/src/planner.ts +66 -0
- package/src/runner/index.ts +188 -0
- package/src/runner/stage-executor.ts +1078 -0
- package/src/state/index.ts +1 -0
- package/src/state/names.ts +33 -0
- package/src/types.ts +332 -0
- package/src/ui-groups.ts +43 -0
package/package.json
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@fyeeme/pi-dynamic-workflows",
|
|
3
|
+
"version": "0.1.1",
|
|
4
|
+
"description": "Deterministic TypeScript workflow orchestration for pi. Fuses the pi-dynamic-workflows design (declarative graph, 10 step primitives, heuristic planner, outcome collectors) with Claude Code's workflow engine coordination mechanisms (deterministic sandbox, cache-key resume, per-agent abort map, dynamic budget, runaway caps).",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"license": "MIT",
|
|
7
|
+
"author": "fyeeme",
|
|
8
|
+
"engines": {
|
|
9
|
+
"node": ">=18"
|
|
10
|
+
},
|
|
11
|
+
"keywords": [
|
|
12
|
+
"pi-package",
|
|
13
|
+
"pi",
|
|
14
|
+
"dynamic-workflow",
|
|
15
|
+
"workflow",
|
|
16
|
+
"orchestration",
|
|
17
|
+
"fan-out",
|
|
18
|
+
"pipeline",
|
|
19
|
+
"multi-agent"
|
|
20
|
+
],
|
|
21
|
+
"files": [
|
|
22
|
+
"*.ts",
|
|
23
|
+
"src/**/*.ts",
|
|
24
|
+
"sessions/**/*.ts",
|
|
25
|
+
"README.md",
|
|
26
|
+
"README.zh-CN.md",
|
|
27
|
+
"LICENSE"
|
|
28
|
+
],
|
|
29
|
+
"pi": {
|
|
30
|
+
"extensions": [
|
|
31
|
+
"./index.ts"
|
|
32
|
+
]
|
|
33
|
+
},
|
|
34
|
+
"scripts": {
|
|
35
|
+
"test": "vitest --run",
|
|
36
|
+
"typecheck": "tsc"
|
|
37
|
+
},
|
|
38
|
+
"dependencies": {
|
|
39
|
+
"@fyeeme/pi-subagent-core": "^0.3.2"
|
|
40
|
+
},
|
|
41
|
+
"peerDependencies": {
|
|
42
|
+
"@earendil-works/pi-ai": ">=0.84.1",
|
|
43
|
+
"@earendil-works/pi-coding-agent": ">=0.84.1",
|
|
44
|
+
"@earendil-works/pi-tui": ">=0.84.1",
|
|
45
|
+
"jiti": ">=2.0.0",
|
|
46
|
+
"typebox": ">=1.0.0",
|
|
47
|
+
"typescript": ">=5.0.0"
|
|
48
|
+
},
|
|
49
|
+
"devDependencies": {
|
|
50
|
+
"@earendil-works/pi-ai": "0.84.1",
|
|
51
|
+
"@earendil-works/pi-coding-agent": "0.84.1",
|
|
52
|
+
"@earendil-works/pi-tui": "0.84.1",
|
|
53
|
+
"@types/node": "22.19.19",
|
|
54
|
+
"jiti": "2.7.0",
|
|
55
|
+
"typebox": "1.1.38",
|
|
56
|
+
"typescript": "5.9.3"
|
|
57
|
+
}
|
|
58
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* sessions/spawn.ts — public per-agent abort/skip API (README §7).
|
|
3
|
+
*
|
|
4
|
+
* Thin barrel over src/agent/dispatch.ts: the spawn registry + per-agent
|
|
5
|
+
* abort primitives. The core spawn implementation lives in
|
|
6
|
+
* `@fyeeme/pi-subagent-core`; skip/retry (workflows-specific) stay in
|
|
7
|
+
* dispatch.ts.
|
|
8
|
+
*/
|
|
9
|
+
export {
|
|
10
|
+
abortAgent,
|
|
11
|
+
createSpawnRegistry,
|
|
12
|
+
retryAgent,
|
|
13
|
+
skipAgent,
|
|
14
|
+
type AgentSpawnRegistry,
|
|
15
|
+
} from "../src/agent/dispatch.ts";
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* src/agent/dispatch.ts — agent dispatch底层 (Task 2)
|
|
3
|
+
*
|
|
4
|
+
* The core spawn primitive (`spawnAgent`, `mapWithConcurrencyLimit`,
|
|
5
|
+
* `createSpawnRegistry`, `abortAgent`, `getPiInvocation` + the registry/
|
|
6
|
+
* options/result types) lives in the shared `@fyeeme/pi-subagent-core`
|
|
7
|
+
* package — extracted from the duplicate copies that used to live here and
|
|
8
|
+
* in pi-review. This module keeps the workflows-specific layer on top:
|
|
9
|
+
* `skipAgent`/`retryAgent` (with `AbortReason` semantics) and the lifecycle
|
|
10
|
+
* notifications.
|
|
11
|
+
*
|
|
12
|
+
* Core semantics: one `pi --mode json -p --no-session` subprocess per agent
|
|
13
|
+
* call, stdout parsed for {message_end, tool_result_end} events,
|
|
14
|
+
* AbortSignal → SIGTERM with a 5s SIGKILL escalation. Each call owns a
|
|
15
|
+
* per-call AbortController registered in an AgentAbortMap, paired with
|
|
16
|
+
* Map<callId, ChildProcess>. A single callId can be aborted (retry/skip)
|
|
17
|
+
* without disturbing its batch siblings, because abort is translated to a
|
|
18
|
+
* SIGTERM on exactly one process.
|
|
19
|
+
*/
|
|
20
|
+
import type { AgentLifecycleListeners } from "../lifecycle.ts";
|
|
21
|
+
import { notifyRetry, notifySkip } from "../lifecycle.ts";
|
|
22
|
+
import type { AgentSpawnRegistry } from "@fyeeme/pi-subagent-core";
|
|
23
|
+
|
|
24
|
+
// Re-export the core dispatch surface so existing importers of this module
|
|
25
|
+
// (`../agent/dispatch.ts`) keep working unchanged.
|
|
26
|
+
export {
|
|
27
|
+
abortAgent,
|
|
28
|
+
createSpawnRegistry,
|
|
29
|
+
getPiInvocation,
|
|
30
|
+
mapWithConcurrencyLimit,
|
|
31
|
+
spawnAgent,
|
|
32
|
+
} from "@fyeeme/pi-subagent-core";
|
|
33
|
+
export type {
|
|
34
|
+
AgentAbortMap,
|
|
35
|
+
AgentCallId,
|
|
36
|
+
AgentSpawnOptions,
|
|
37
|
+
AgentSpawnRegistry,
|
|
38
|
+
AgentSpawnResult,
|
|
39
|
+
AgentUsage,
|
|
40
|
+
} from "@fyeeme/pi-subagent-core";
|
|
41
|
+
|
|
42
|
+
export type AbortReason = "user-skip" | "user-retry";
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Abort one call as skipped. The call settles skipped (runner will not
|
|
46
|
+
* re-dispatch it); batch siblings are untouched. Fires `onAgentSkip`.
|
|
47
|
+
*/
|
|
48
|
+
export function skipAgent(
|
|
49
|
+
registry: AgentSpawnRegistry,
|
|
50
|
+
callId: string,
|
|
51
|
+
listeners?: AgentLifecycleListeners,
|
|
52
|
+
): boolean {
|
|
53
|
+
const controller = registry.controllers.get(callId);
|
|
54
|
+
if (!controller) return false;
|
|
55
|
+
controller.abort("user-skip");
|
|
56
|
+
notifySkip(listeners, callId);
|
|
57
|
+
return true;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Abort one call so the runner can re-dispatch it. Only this callId is
|
|
62
|
+
* aborted; batch siblings keep running. Fires `onAgentRetry`. The actual
|
|
63
|
+
* re-dispatch is the runner's job (it sees the call settle aborted and
|
|
64
|
+
* decides whether to spawn again).
|
|
65
|
+
*/
|
|
66
|
+
export function retryAgent(
|
|
67
|
+
registry: AgentSpawnRegistry,
|
|
68
|
+
callId: string,
|
|
69
|
+
listeners?: AgentLifecycleListeners,
|
|
70
|
+
): boolean {
|
|
71
|
+
const controller = registry.controllers.get(callId);
|
|
72
|
+
if (!controller) return false;
|
|
73
|
+
controller.abort("user-retry");
|
|
74
|
+
notifyRetry(listeners, callId);
|
|
75
|
+
return true;
|
|
76
|
+
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Hard runaway caps — the pi port of CC's workflow backstops.
|
|
3
|
+
*
|
|
4
|
+
* CC caps: `min(16, cpu-2)` concurrent per workflow, 1000 agents per workflow
|
|
5
|
+
* lifetime (runaway backstop), 4096 items per `parallel()`/`pipeline()` call
|
|
6
|
+
* (explicit error on exceed — never silent truncation). Pi ports the lifetime
|
|
7
|
+
* and batch caps here; the concurrency cap is enforced by
|
|
8
|
+
* `mapWithConcurrencyLimit` in src/agent/dispatch.ts.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import { WorkflowError } from "../errors.ts";
|
|
12
|
+
|
|
13
|
+
export const MAX_LIFETIME_AGENTS = 1000;
|
|
14
|
+
export const MAX_BATCH = 4096;
|
|
15
|
+
|
|
16
|
+
/** Budget exhaustion (maxAgents / maxTokens / maxDuration) or a hard cap hit
|
|
17
|
+
* (MAX_BATCH / MAX_LIFETIME_AGENTS). Extends WorkflowError so it carries the
|
|
18
|
+
* `budget-exceeded` category for retry policy (terminal by default). */
|
|
19
|
+
export class BudgetExceededError extends WorkflowError {
|
|
20
|
+
constructor(message: string) {
|
|
21
|
+
super(message, { category: "budget-exceeded" });
|
|
22
|
+
this.name = "BudgetExceededError";
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/** Assert a fan-out batch fits the hard cap. Throws — no silent truncation. */
|
|
27
|
+
export function assertBatchSize(n: number): void {
|
|
28
|
+
if (n > MAX_BATCH) {
|
|
29
|
+
throw new BudgetExceededError(
|
|
30
|
+
`fan-out batch ${n} exceeds MAX_BATCH (${MAX_BATCH}); pass fewer items or split the input`,
|
|
31
|
+
);
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** Assert cumulative agent count hasn't hit the runaway backstop. */
|
|
36
|
+
export function assertLifetimeAgents(spawned: number): void {
|
|
37
|
+
if (spawned >= MAX_LIFETIME_AGENTS) {
|
|
38
|
+
throw new BudgetExceededError(
|
|
39
|
+
`workflow spawned ${spawned} agents — MAX_LIFETIME_AGENTS (${MAX_LIFETIME_AGENTS}) runaway backstop reached`,
|
|
40
|
+
);
|
|
41
|
+
}
|
|
42
|
+
}
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Budget pool — CC's `budget.{total, spent(), remaining()}` re-expressed for pi.
|
|
3
|
+
*
|
|
4
|
+
* A static `Budget` (types.ts) describes caps; a `BudgetPool` is the live,
|
|
5
|
+
* queryable tracker the runtime mutates as agents settle. Caps are enforced
|
|
6
|
+
* as SPAWN-GATE SOFT LIMITS: the runtime checks `isExhausted()` before
|
|
7
|
+
* committing each new spawn (guardSpawn/guardBatch) and stops spawning when
|
|
8
|
+
* exhausted — an in-flight agent is allowed to complete and may push spend
|
|
9
|
+
* past the cap. fan_out concurrency is the step's static `parallelism`; the
|
|
10
|
+
* budget does not dynamically resize it.
|
|
11
|
+
*
|
|
12
|
+
* Determinism: identity (runId, journal timestamps, cache keys) never reads
|
|
13
|
+
* Date.now() — `now` is the run inception timestamp. The DURATION dimension,
|
|
14
|
+
* however, is wall-clock and the engine reads Date.now() at the guard points
|
|
15
|
+
* (guardSpawn/guardBatch) to enforce maxDurationMs; engine code is not
|
|
16
|
+
* AST-guarded. Token/agent dimensions are passed `now` but don't depend on
|
|
17
|
+
* it (elapsed only matters for duration).
|
|
18
|
+
*
|
|
19
|
+
* Scope: per-run (one pool per workflow execution), not global.
|
|
20
|
+
*
|
|
21
|
+
* P1-1 fix — reservation model: the agents dimension is committed synchronously
|
|
22
|
+
* via `reserve(n)` (check + increment in one step, no `await` gap), closing the
|
|
23
|
+
* TOCTOU where N concurrent fan_out workers all read `remaining.agents >= 1`
|
|
24
|
+
* before any settles. `reserve` returns a release handle; a failed dispatch
|
|
25
|
+
* (spawn rejected) calls it to return the slot. Tokens can't be reserved
|
|
26
|
+
* (cost is unknown until the agent settles), so maxTokens stays an
|
|
27
|
+
* after-the-fact track + isExhausted check on the next guard.
|
|
28
|
+
*/
|
|
29
|
+
import type { Budget } from "../types.ts";
|
|
30
|
+
import { BudgetExceededError } from "./caps.ts";
|
|
31
|
+
|
|
32
|
+
export interface BudgetRemaining {
|
|
33
|
+
/** Tokens left before maxTokens is hit. Infinity if uncapped. */
|
|
34
|
+
readonly tokens: number;
|
|
35
|
+
/** Milliseconds left before maxDurationMs. Infinity if uncapped. */
|
|
36
|
+
readonly durationMs: number;
|
|
37
|
+
/** Agent slots left before maxAgents. Infinity if uncapped. */
|
|
38
|
+
readonly agents: number;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export class BudgetPool {
|
|
42
|
+
private spentTokens = 0;
|
|
43
|
+
/** Agents reserved via reserve() — synchronously committed, so concurrent
|
|
44
|
+
* callers can't all pass the check before any settles (TOCTOU). */
|
|
45
|
+
private reservedAgents = 0;
|
|
46
|
+
private readonly originMs: number;
|
|
47
|
+
private readonly config: Budget;
|
|
48
|
+
|
|
49
|
+
// Note: no parameter properties (e.g. `private readonly config` in the param
|
|
50
|
+
// list) — erasableSyntaxOnly forbids them. Declare the field, assign in body.
|
|
51
|
+
constructor(config: Budget, originMs: number) {
|
|
52
|
+
this.config = config;
|
|
53
|
+
this.originMs = originMs;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Atomically reserve `n` agent slots against maxAgents. Throws
|
|
58
|
+
* BudgetExceededError if the reservation would exceed the cap. Returns a
|
|
59
|
+
* release handle — call it ONLY if the dispatch fails (the slot was never
|
|
60
|
+
* used); a settled agent keeps its slot (it consumed budget).
|
|
61
|
+
*/
|
|
62
|
+
reserve(n: number): () => void {
|
|
63
|
+
if (n <= 0) return () => {};
|
|
64
|
+
const cap = this.config.maxAgents;
|
|
65
|
+
if (cap !== undefined && this.reservedAgents + n > cap) {
|
|
66
|
+
throw new BudgetExceededError(
|
|
67
|
+
`budget exhausted: ${this.reservedAgents + n} agent(s) would exceed maxAgents ${cap}`,
|
|
68
|
+
);
|
|
69
|
+
}
|
|
70
|
+
this.reservedAgents += n;
|
|
71
|
+
let released = false;
|
|
72
|
+
return () => {
|
|
73
|
+
if (released) return;
|
|
74
|
+
released = true;
|
|
75
|
+
this.reservedAgents -= n;
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** Record token spend by a settled agent. (Agents are counted via reserve.) */
|
|
80
|
+
track(input: { tokens?: number }): void {
|
|
81
|
+
if (input.tokens) this.spentTokens += input.tokens;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/** The configured cap set (CC's budget.total surface). */
|
|
85
|
+
get total(): Budget {
|
|
86
|
+
return this.config;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/** Remaining headroom per dimension at time `now`. */
|
|
90
|
+
remaining(now: number): BudgetRemaining {
|
|
91
|
+
return {
|
|
92
|
+
tokens:
|
|
93
|
+
this.config.maxTokens !== undefined
|
|
94
|
+
? Math.max(0, this.config.maxTokens - this.spentTokens)
|
|
95
|
+
: Number.POSITIVE_INFINITY,
|
|
96
|
+
durationMs:
|
|
97
|
+
this.config.maxDurationMs !== undefined
|
|
98
|
+
? Math.max(0, this.config.maxDurationMs - (now - this.originMs))
|
|
99
|
+
: Number.POSITIVE_INFINITY,
|
|
100
|
+
agents:
|
|
101
|
+
this.config.maxAgents !== undefined
|
|
102
|
+
? Math.max(0, this.config.maxAgents - this.reservedAgents)
|
|
103
|
+
: Number.POSITIVE_INFINITY,
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** True once any capped dimension hits zero. */
|
|
108
|
+
isExhausted(now: number): boolean {
|
|
109
|
+
const r = this.remaining(now);
|
|
110
|
+
return r.tokens === 0 || r.durationMs === 0 || r.agents === 0;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/** Can n more agents be spawned under the agents cap? (Pre-check; actual
|
|
114
|
+
* commit is via reserve().) */
|
|
115
|
+
canSpawn(n: number, now: number): boolean {
|
|
116
|
+
return this.remaining(now).agents >= n;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-run journal of agent call results — the pi port of CC.s LocalFileJournal.
|
|
3
|
+
*
|
|
4
|
+
* One JSONL file per run (<dir>/journal.jsonl). Each line is either a
|
|
5
|
+
* `started` marker (an agent dispatched) or a `result` (an agent settled with
|
|
6
|
+
* a value). On resume, the journal is loaded into an in-memory map keyed by
|
|
7
|
+
* CacheKey; `result` entries supersede earlier `started` entries for the same
|
|
8
|
+
* key. The runner consults `lookup()` before dispatching: a hit means replay,
|
|
9
|
+
* skipping the subprocess entirely.
|
|
10
|
+
*
|
|
11
|
+
* Append-only within a run; resume reads the existing file, then the resumed
|
|
12
|
+
* run continues appending entries for cache-miss agents.
|
|
13
|
+
*
|
|
14
|
+
* Determinism note: `at` is an opaque counter/timestamp supplied by the caller
|
|
15
|
+
* (from the run's deterministic inception time, never Date.now() inside the
|
|
16
|
+
* workflow body — the Task 3 sandbox forbids that).
|
|
17
|
+
*/
|
|
18
|
+
import * as fs from "node:fs";
|
|
19
|
+
import * as path from "node:path";
|
|
20
|
+
import type { CacheKey } from "../types.ts";
|
|
21
|
+
|
|
22
|
+
export type JournalEntry<T = unknown> =
|
|
23
|
+
| { readonly type: "started"; readonly key: CacheKey; readonly at: number }
|
|
24
|
+
| { readonly type: "result"; readonly key: CacheKey; readonly at: number; readonly ok: boolean; readonly value: T };
|
|
25
|
+
|
|
26
|
+
/** Staged-resume manifest: identifies the last completed run. Only `runId` is
|
|
27
|
+
* consumed downstream (resume.previousRunId); cache-hit accounting is observed
|
|
28
|
+
* live during the run, so no key list is persisted here. */
|
|
29
|
+
export interface RunManifest {
|
|
30
|
+
/** Run ID that produced this manifest. */
|
|
31
|
+
readonly runId: string;
|
|
32
|
+
/** Inception timestamp. */
|
|
33
|
+
readonly at: number;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export interface JournalOptions {
|
|
37
|
+
/** Directory holding journal.jsonl (typically <cwd>/.pi/workflows/runs/<runId>). */
|
|
38
|
+
readonly dir: string;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** Monotonic counter for unique manifest temp-file names (crash-safe atomic writes). */
|
|
42
|
+
let manifestSeq = 0;
|
|
43
|
+
|
|
44
|
+
export class Journal {
|
|
45
|
+
private readonly filePath: string;
|
|
46
|
+
/** key → latest entry. result entries supersede started entries. */
|
|
47
|
+
private readonly results = new Map<CacheKey, JournalEntry>();
|
|
48
|
+
private loaded = false;
|
|
49
|
+
|
|
50
|
+
constructor(opts: JournalOptions) {
|
|
51
|
+
this.filePath = path.join(opts.dir, "journal.jsonl");
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** Read the JSONL file into the in-memory map. A missing file (fresh run) is a no-op. */
|
|
55
|
+
async load(): Promise<void> {
|
|
56
|
+
let content: string;
|
|
57
|
+
try {
|
|
58
|
+
content = await fs.promises.readFile(this.filePath, "utf-8");
|
|
59
|
+
} catch (e) {
|
|
60
|
+
if (!isENOENT(e)) throw e;
|
|
61
|
+
this.loaded = true;
|
|
62
|
+
return;
|
|
63
|
+
}
|
|
64
|
+
for (const line of content.split("\n")) {
|
|
65
|
+
if (!line.trim()) continue;
|
|
66
|
+
let entry: JournalEntry;
|
|
67
|
+
try {
|
|
68
|
+
entry = JSON.parse(line) as JournalEntry;
|
|
69
|
+
} catch {
|
|
70
|
+
continue; // skip malformed line, keep the rest
|
|
71
|
+
}
|
|
72
|
+
if (entry.type === "result") this.results.set(entry.key, entry);
|
|
73
|
+
}
|
|
74
|
+
this.loaded = true;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** Look up a settled result by cache key. Undefined = cache miss → must dispatch. */
|
|
78
|
+
lookup(key: CacheKey): JournalEntry | undefined {
|
|
79
|
+
return this.results.get(key);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/** Append an entry to disk and (for result entries) the in-memory map. */
|
|
83
|
+
private appendChain: Promise<void> = Promise.resolve();
|
|
84
|
+
private lastWriteError: unknown;
|
|
85
|
+
async append(entry: JournalEntry): Promise<void> {
|
|
86
|
+
if (entry.type === "result") this.results.set(entry.key, entry);
|
|
87
|
+
// Serialize appends: concurrent appendFile calls interleave bytes for
|
|
88
|
+
// entries larger than PIPE_BUF (4KB on Linux), corrupting JSONL lines that
|
|
89
|
+
// Journal.load then silently drops. Chain each write onto the previous.
|
|
90
|
+
//
|
|
91
|
+
// P0 fix: catch on the chain so a single disk failure (ENOSPC/EACCES/EIO)
|
|
92
|
+
// does NOT poison it forever — without the catch, every later `.then`
|
|
93
|
+
// never runs its callback, so all subsequent writes become permanent
|
|
94
|
+
// silent no-ops while `lookup()` keeps returning the in-memory entries.
|
|
95
|
+
// The failed write is recorded in `lastWriteError` for the caller.
|
|
96
|
+
const line = JSON.stringify(entry) + "\n";
|
|
97
|
+
this.appendChain = this.appendChain
|
|
98
|
+
.then(() => fs.promises.appendFile(this.filePath, line, "utf-8"))
|
|
99
|
+
.catch((e) => {
|
|
100
|
+
this.lastWriteError = e;
|
|
101
|
+
});
|
|
102
|
+
return this.appendChain;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/** Number of cached results currently in memory. */
|
|
106
|
+
get size(): number {
|
|
107
|
+
return this.results.size;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
get file(): string {
|
|
111
|
+
return this.filePath;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
get isLoaded(): boolean {
|
|
115
|
+
return this.loaded;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/** The most recent disk-write error, if any. A non-null value means at least
|
|
119
|
+
* one entry may be in memory but not on disk → resume could lose it. */
|
|
120
|
+
get writeError(): unknown {
|
|
121
|
+
return this.lastWriteError;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/** Iterator over all entries in the in-memory cache. */
|
|
125
|
+
allEntries(): IterableIterator<JournalEntry> {
|
|
126
|
+
return this.results.values();
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/** Write the staged-resume manifest for the next run to consume.
|
|
130
|
+
* Call after the run completes successfully. Written atomically
|
|
131
|
+
* (write-temp + rename) so a crash between truncate and write can never
|
|
132
|
+
* leave a 0-byte / truncated manifest that would crash future runs. */
|
|
133
|
+
async writeManifest(manifest: RunManifest): Promise<void> {
|
|
134
|
+
const dir = path.dirname(this.filePath);
|
|
135
|
+
const manifestPath = path.join(dir, "manifest.json");
|
|
136
|
+
const tmp = path.join(dir, `.manifest.${process.pid}.${manifestSeq++}.tmp`);
|
|
137
|
+
// Sweep stale tmp files from crashed prior writes BEFORE writing ours — a
|
|
138
|
+
// post-write sweep would match (and delete) the file we are about to
|
|
139
|
+
// rename from.
|
|
140
|
+
await this.sweepStaleTmp(dir);
|
|
141
|
+
await fs.promises.writeFile(tmp, JSON.stringify(manifest, null, 2), "utf-8");
|
|
142
|
+
await fs.promises.rename(tmp, manifestPath);
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
private async sweepStaleTmp(dir: string): Promise<void> {
|
|
146
|
+
try {
|
|
147
|
+
const entries = await fs.promises.readdir(dir);
|
|
148
|
+
await Promise.all(
|
|
149
|
+
entries
|
|
150
|
+
.filter((e) => e.startsWith(".manifest.") && e.endsWith(".tmp"))
|
|
151
|
+
.map((e) => fs.promises.unlink(path.join(dir, e)).catch(() => {})),
|
|
152
|
+
);
|
|
153
|
+
} catch {
|
|
154
|
+
/* best-effort */
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/** Load the staged-resume manifest from the last completed run, if present.
|
|
159
|
+
* Best-effort and crash-resilient: a missing file is the normal
|
|
160
|
+
* pre-first-run state; a corrupt/truncated file (e.g. after a crash
|
|
161
|
+
* mid-write) is treated as "no manifest" with a warning — it MUST never
|
|
162
|
+
* crash the run. The next successful run overwrites it. */
|
|
163
|
+
async loadManifest(): Promise<RunManifest | undefined> {
|
|
164
|
+
const manifestPath = path.join(path.dirname(this.filePath), "manifest.json");
|
|
165
|
+
let raw: string;
|
|
166
|
+
try {
|
|
167
|
+
raw = await fs.promises.readFile(manifestPath, "utf-8");
|
|
168
|
+
} catch (e) {
|
|
169
|
+
if (isENOENT(e)) return undefined; // normal pre-first-run state
|
|
170
|
+
console.warn(`[pi-dynamic-workflows] manifest read failed, ignoring: ${(e as Error).message}`);
|
|
171
|
+
return undefined;
|
|
172
|
+
}
|
|
173
|
+
try {
|
|
174
|
+
return JSON.parse(raw) as RunManifest;
|
|
175
|
+
} catch (e) {
|
|
176
|
+
console.warn(`[pi-dynamic-workflows] manifest is corrupt, ignoring: ${(e as Error).message}`);
|
|
177
|
+
return undefined;
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
function isENOENT(e: unknown): boolean {
|
|
183
|
+
return e !== null && typeof e === "object" && (e as { code?: string }).code === "ENOENT";
|
|
184
|
+
}
|
package/src/cache/key.ts
ADDED
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cache key for agent invocations — the pi port of Claude Code.s workflow cache-key
|
|
3
|
+
*
|
|
4
|
+
* A workflow run that is re-executed (resume after a script edit, or a second
|
|
5
|
+
* identical run) should NOT pay to re-dispatch agents whose (prompt,
|
|
6
|
+
* signature) is unchanged. The key captures everything that affects an
|
|
7
|
+
* agent's output: workflow name (scope), prompt (the task), and a normalized
|
|
8
|
+
* signature (model/tools/systemPrompt). runId is deliberately excluded —
|
|
9
|
+
* resume changes the runId but must still hit the cache.
|
|
10
|
+
*
|
|
11
|
+
* Normalization (CC.s signature normalizer): drop functions, sort object keys, so the same
|
|
12
|
+
* logical signature produces the same JSON regardless of field declaration
|
|
13
|
+
* order or attached closures. sha256 makes the key opaque and fixed-length.
|
|
14
|
+
*/
|
|
15
|
+
import { createHash } from "node:crypto";
|
|
16
|
+
import type { CacheKey } from "../types.ts";
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* The subset of agent options that affect output and therefore belong in the
|
|
20
|
+
* key. Caller constructs this explicitly so callId/signal/cwd (run-time
|
|
21
|
+
* control, not output semantics) never leak into the key by accident.
|
|
22
|
+
*/
|
|
23
|
+
export interface AgentCacheSignature {
|
|
24
|
+
readonly model?: string;
|
|
25
|
+
readonly tools?: readonly string[];
|
|
26
|
+
readonly systemPrompt?: string;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Serialize a signature to a stable canonical form: object keys sorted, no
|
|
31
|
+
* functions, arrays in order. Same signature → identical string.
|
|
32
|
+
*
|
|
33
|
+
* Picks only the known data fields (model/tools/systemPrompt) — CC.s signature normalizer
|
|
34
|
+
* pattern. A caller may pass a wider object (e.g. an AgentCallSpec that also
|
|
35
|
+
* carries a `prompt` function); picking avoids both hashing the prompt
|
|
36
|
+
* (which is a separate cache-key dimension) and tripping the function-guard
|
|
37
|
+
* on the prompt closure.
|
|
38
|
+
*/
|
|
39
|
+
const SIGNATURE_FIELDS = ["model", "tools", "systemPrompt"] as const;
|
|
40
|
+
|
|
41
|
+
export function normalizeSignature(signature: AgentCacheSignature | undefined): string {
|
|
42
|
+
if (!signature) return "{}";
|
|
43
|
+
const src = signature as Record<string, unknown>;
|
|
44
|
+
const picked: Record<string, unknown> = {};
|
|
45
|
+
for (const f of SIGNATURE_FIELDS) {
|
|
46
|
+
const v = src[f];
|
|
47
|
+
if (v !== undefined) picked[f] = v;
|
|
48
|
+
}
|
|
49
|
+
return JSON.stringify(stabilize(picked));
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function stabilize(value: unknown): unknown {
|
|
53
|
+
if (typeof value === "function") {
|
|
54
|
+
// Functions are not stable across runs (closures capture mutable scope),
|
|
55
|
+
// so silently dropping them would let two signatures that differ only in a
|
|
56
|
+
// function field hash identically → resume returns a result computed under
|
|
57
|
+
// function A to a caller that supplied function B. Throw instead — a
|
|
58
|
+
// determinism subsystem must not accept un-hashable inputs.
|
|
59
|
+
throw new Error(
|
|
60
|
+
"workflow signature contains a function — functions are not stable across runs; pass only data (string/number/array/plain object)",
|
|
61
|
+
);
|
|
62
|
+
}
|
|
63
|
+
if (Array.isArray(value)) return value.map(stabilize);
|
|
64
|
+
if (value !== null && typeof value === "object") {
|
|
65
|
+
const out: Record<string, unknown> = {};
|
|
66
|
+
const src = value as Record<string, unknown>;
|
|
67
|
+
for (const key of Object.keys(src).sort()) out[key] = stabilize(src[key]);
|
|
68
|
+
return out;
|
|
69
|
+
}
|
|
70
|
+
return value;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
export interface CacheKeyInput {
|
|
74
|
+
readonly workflowName: string;
|
|
75
|
+
readonly prompt: string;
|
|
76
|
+
readonly signature?: AgentCacheSignature;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** Cache-key namespace prefix. Bump it (wf3 → wf4 → …) whenever the key
|
|
80
|
+
* derivation OR the engine's prompt-affecting behavior changes — old journal
|
|
81
|
+
* entries then cleanly miss instead of being replayed as stale results (the
|
|
82
|
+
* entries stay on disk but are never looked up). Derivation history:
|
|
83
|
+
* `wf:` — initial; `wf2:` — B1+B2 base system prompt joined into the key;
|
|
84
|
+
* `wf3:` — JSON-tuple encoding; `wf4:` — current revision. */
|
|
85
|
+
export const CACHE_KEY_PREFIX = "wf4";
|
|
86
|
+
|
|
87
|
+
/** Compute the deterministic cache key for one agent call.
|
|
88
|
+
* The tuple is JSON-encoded so field boundaries are unambiguous: a `\x00`
|
|
89
|
+
* (or any char) inside workflowName/prompt cannot shift the parse and collide
|
|
90
|
+
* with another (name, prompt) pair. Production paths sanitize these inputs,
|
|
91
|
+
* but this is a self-defending public function. */
|
|
92
|
+
export function computeCacheKey(input: CacheKeyInput): CacheKey {
|
|
93
|
+
const hash = createHash("sha256");
|
|
94
|
+
const blob = JSON.stringify([input.workflowName, input.prompt, normalizeSignature(input.signature)]);
|
|
95
|
+
hash.update(blob);
|
|
96
|
+
return `${CACHE_KEY_PREFIX}:${hash.digest("hex")}`;
|
|
97
|
+
}
|