@osolmaz/pi-workflows 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +71 -5
- package/dist/builtins/monitor.workflow.d.ts +71 -0
- package/dist/builtins/monitor.workflow.js +234 -0
- package/dist/builtins/monitor.workflow.js.map +1 -0
- package/dist/controllers/conditions.d.ts +6 -0
- package/dist/controllers/conditions.js +68 -0
- package/dist/controllers/conditions.js.map +1 -0
- package/dist/controllers/definition.d.ts +6 -0
- package/dist/controllers/definition.js +45 -0
- package/dist/controllers/definition.js.map +1 -0
- package/dist/controllers/effects.d.ts +14 -0
- package/dist/controllers/effects.js +104 -0
- package/dist/controllers/effects.js.map +1 -0
- package/dist/controllers/errors.d.ts +12 -0
- package/dist/controllers/errors.js +25 -0
- package/dist/controllers/errors.js.map +1 -0
- package/dist/controllers/index.d.ts +13 -0
- package/dist/controllers/index.js +13 -0
- package/dist/controllers/index.js.map +1 -0
- package/dist/controllers/json.d.ts +5 -0
- package/dist/controllers/json.js +57 -0
- package/dist/controllers/json.js.map +1 -0
- package/dist/controllers/loader.d.ts +23 -0
- package/dist/controllers/loader.js +74 -0
- package/dist/controllers/loader.js.map +1 -0
- package/dist/controllers/manager.d.ts +58 -0
- package/dist/controllers/manager.js +399 -0
- package/dist/controllers/manager.js.map +1 -0
- package/dist/controllers/results.d.ts +5 -0
- package/dist/controllers/results.js +32 -0
- package/dist/controllers/results.js.map +1 -0
- package/dist/controllers/sqlite.d.ts +212 -0
- package/dist/controllers/sqlite.js +1009 -0
- package/dist/controllers/sqlite.js.map +1 -0
- package/dist/controllers/store.d.ts +112 -0
- package/dist/controllers/store.js +32 -0
- package/dist/controllers/store.js.map +1 -0
- package/dist/controllers/types.d.ts +159 -0
- package/dist/controllers/types.js +2 -0
- package/dist/controllers/types.js.map +1 -0
- package/dist/controllers/workflow-engine-scheduler.d.ts +25 -0
- package/dist/controllers/workflow-engine-scheduler.js +93 -0
- package/dist/controllers/workflow-engine-scheduler.js.map +1 -0
- package/dist/controllers/workflows.d.ts +27 -0
- package/dist/controllers/workflows.js +109 -0
- package/dist/controllers/workflows.js.map +1 -0
- package/dist/extension/controller-host.d.ts +47 -0
- package/dist/extension/controller-host.js +110 -0
- package/dist/extension/controller-host.js.map +1 -0
- package/dist/extension/executor.js +1 -1
- package/dist/extension/executor.js.map +1 -1
- package/dist/extension/index.d.ts +7 -0
- package/dist/extension/index.js +1053 -96
- package/dist/extension/index.js.map +1 -1
- package/dist/extension/recorder.d.ts +6 -0
- package/dist/extension/recorder.js +20 -8
- package/dist/extension/recorder.js.map +1 -1
- package/dist/extension/workflow-tool.d.ts +28 -0
- package/dist/extension/workflow-tool.js +33 -0
- package/dist/extension/workflow-tool.js.map +1 -0
- package/dist/host/processes.d.ts +24 -0
- package/dist/host/processes.js +114 -0
- package/dist/host/processes.js.map +1 -0
- package/dist/host/rpc-bridge.d.ts +9 -0
- package/dist/host/rpc-bridge.js +39 -0
- package/dist/host/rpc-bridge.js.map +1 -0
- package/dist/host/rpc-executor.d.ts +38 -0
- package/dist/host/rpc-executor.js +254 -0
- package/dist/host/rpc-executor.js.map +1 -0
- package/dist/host/runner.d.ts +49 -0
- package/dist/host/runner.js +350 -0
- package/dist/host/runner.js.map +1 -0
- package/dist/viewer/cli.d.ts +7 -3
- package/dist/viewer/cli.js +150 -19
- package/dist/viewer/cli.js.map +1 -1
- package/dist/workflows/engine.d.ts +36 -0
- package/dist/workflows/engine.js +244 -14
- package/dist/workflows/engine.js.map +1 -1
- package/dist/workflows/errors.d.ts +23 -0
- package/dist/workflows/errors.js +38 -0
- package/dist/workflows/errors.js.map +1 -1
- package/dist/workflows/graph.js +0 -5
- package/dist/workflows/graph.js.map +1 -1
- package/dist/workflows/loader.d.ts +5 -3
- package/dist/workflows/loader.js +10 -1
- package/dist/workflows/loader.js.map +1 -1
- package/dist/workflows/schema.js +1 -1
- package/dist/workflows/schema.js.map +1 -1
- package/dist/workflows/store.d.ts +50 -6
- package/dist/workflows/store.js +446 -51
- package/dist/workflows/store.js.map +1 -1
- package/dist/workflows/types.d.ts +10 -0
- package/docs/CONTROLLERS.md +215 -0
- package/docs/development.md +12 -9
- package/docs/plans/2026-08-04-controller-runtime-plan.md +169 -0
- package/docs/plans/2026-08-05-always-on-workflows-plan.md +125 -0
- package/docs/plans/2026-08-10-agent-managed-monitor-workflows-plan.md +184 -0
- package/docs/run-bundles.md +66 -27
- package/docs/workflows.md +131 -11
- package/examples/controllers/pull-request.controller.ts +215 -0
- package/package.json +10 -2
- package/src/builtins/monitor.workflow.ts +278 -0
- package/src/controllers/conditions.ts +110 -0
- package/src/controllers/definition.ts +65 -0
- package/src/controllers/effects.ts +123 -0
- package/src/controllers/errors.ts +27 -0
- package/src/controllers/index.ts +90 -0
- package/src/controllers/json.ts +62 -0
- package/src/controllers/loader.ts +104 -0
- package/src/controllers/manager.ts +533 -0
- package/src/controllers/results.ts +46 -0
- package/src/controllers/sqlite.ts +1427 -0
- package/src/controllers/store.ts +160 -0
- package/src/controllers/types.ts +183 -0
- package/src/controllers/workflow-engine-scheduler.ts +145 -0
- package/src/controllers/workflows.ts +152 -0
- package/src/extension/controller-host.ts +163 -0
- package/src/extension/executor.ts +1 -1
- package/src/extension/index.ts +1243 -117
- package/src/extension/recorder.ts +65 -36
- package/src/extension/workflow-tool.ts +59 -0
- package/src/host/processes.ts +119 -0
- package/src/host/rpc-bridge.ts +44 -0
- package/src/host/rpc-executor.ts +299 -0
- package/src/host/runner.ts +406 -0
- package/src/viewer/cli.ts +167 -21
- package/src/workflows/engine.ts +327 -13
- package/src/workflows/errors.ts +45 -0
- package/src/workflows/graph.ts +0 -5
- package/src/workflows/loader.ts +13 -3
- package/src/workflows/schema.ts +1 -1
- package/src/workflows/store.ts +555 -46
- package/src/workflows/types.ts +10 -0
package/dist/workflows/store.js
CHANGED
|
@@ -20,6 +20,28 @@ const SESSION_BINDING_PATH = `${SESSION_DIR}/binding.json`;
|
|
|
20
20
|
const SESSION_ENTRIES_PATH = `${SESSION_DIR}/entries.ndjson`;
|
|
21
21
|
const SESSION_EVENTS_PATH = `${SESSION_DIR}/events.ndjson`;
|
|
22
22
|
const SESSION_CAPTURE_PATH = `${SESSION_DIR}/capture.json`;
|
|
23
|
+
const SESSION_SEGMENTS_DIR = `${SESSION_DIR}/segments`;
|
|
24
|
+
/** Capture file layout for one recorder attempt; "" is the legacy flat stream. */
|
|
25
|
+
function sessionStreamPaths(attemptId) {
|
|
26
|
+
if (attemptId === undefined) {
|
|
27
|
+
return {
|
|
28
|
+
dir: SESSION_DIR,
|
|
29
|
+
binding: SESSION_BINDING_PATH,
|
|
30
|
+
entries: SESSION_ENTRIES_PATH,
|
|
31
|
+
events: SESSION_EVENTS_PATH,
|
|
32
|
+
capture: SESSION_CAPTURE_PATH,
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
assertValidRunId(attemptId);
|
|
36
|
+
const dir = `${SESSION_SEGMENTS_DIR}/${attemptId}`;
|
|
37
|
+
return {
|
|
38
|
+
dir,
|
|
39
|
+
binding: `${dir}/binding.json`,
|
|
40
|
+
entries: `${dir}/entries.ndjson`,
|
|
41
|
+
events: `${dir}/events.ndjson`,
|
|
42
|
+
capture: `${dir}/capture.json`,
|
|
43
|
+
};
|
|
44
|
+
}
|
|
23
45
|
/** Runs directory: `$PI_WORKFLOWS_RUNS_DIR` or `~/.pi/agent/workflows/runs`. */
|
|
24
46
|
export function workflowRunsBaseDir(homeDir = os.homedir()) {
|
|
25
47
|
const override = process.env.PI_WORKFLOWS_RUNS_DIR;
|
|
@@ -40,39 +62,75 @@ export function createRunId(workflowName, now = new Date()) {
|
|
|
40
62
|
.slice(0, 40);
|
|
41
63
|
return `${stamp}-${slug || "workflow"}-${randomUUID().slice(0, 8)}`;
|
|
42
64
|
}
|
|
43
|
-
/**
|
|
44
|
-
* Persists run bundles (see docs/run-bundles.md). `trace.ndjson` is the
|
|
45
|
-
* append-only source of truth; every transition appends the trace event
|
|
46
|
-
* first, then atomically replaces `state.json` (carrying `traceSeq`) and
|
|
47
|
-
* `manifest.json`. Large string leaves in persisted values are externalized
|
|
48
|
-
* into content-addressed `artifacts/`. Bundles are private: directories are
|
|
49
|
-
* 0700 and files 0600.
|
|
50
|
-
*/
|
|
51
65
|
export class WorkflowRunStore {
|
|
52
66
|
outputRoot;
|
|
67
|
+
fenceProvider;
|
|
53
68
|
contexts = new Map();
|
|
54
|
-
constructor(outputRoot = workflowRunsBaseDir()) {
|
|
69
|
+
constructor(outputRoot = workflowRunsBaseDir(), options = {}) {
|
|
55
70
|
this.outputRoot = outputRoot;
|
|
71
|
+
this.fenceProvider = options.fenceProvider;
|
|
56
72
|
}
|
|
57
73
|
runDirFor(runId) {
|
|
74
|
+
assertValidRunId(runId);
|
|
58
75
|
return path.join(this.outputRoot, runId);
|
|
59
76
|
}
|
|
77
|
+
async quarantineIncompleteRun(runId) {
|
|
78
|
+
const runDir = this.runDirFor(runId);
|
|
79
|
+
let runStat;
|
|
80
|
+
try {
|
|
81
|
+
runStat = await fs.lstat(runDir);
|
|
82
|
+
}
|
|
83
|
+
catch (error) {
|
|
84
|
+
if (isMissingPath(error)) {
|
|
85
|
+
return undefined;
|
|
86
|
+
}
|
|
87
|
+
throw error;
|
|
88
|
+
}
|
|
89
|
+
if (!runStat.isDirectory() || runStat.isSymbolicLink()) {
|
|
90
|
+
throw new Error(`Reserved workflow run path is not a directory: ${runDir}`);
|
|
91
|
+
}
|
|
92
|
+
try {
|
|
93
|
+
await fs.lstat(path.join(runDir, MANIFEST_PATH));
|
|
94
|
+
throw new Error(`Reserved workflow run has an unreadable manifest: ${runId}`);
|
|
95
|
+
}
|
|
96
|
+
catch (error) {
|
|
97
|
+
if (!isMissingPath(error)) {
|
|
98
|
+
throw error;
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
const quarantineDir = path.join(this.outputRoot, `.${runId}.incomplete-${randomUUID().slice(0, 8)}`);
|
|
102
|
+
await fs.rename(runDir, quarantineDir);
|
|
103
|
+
this.contexts.delete(runDir);
|
|
104
|
+
return quarantineDir;
|
|
105
|
+
}
|
|
60
106
|
contextFor(runDir) {
|
|
61
107
|
let context = this.contexts.get(runDir);
|
|
62
108
|
if (!context) {
|
|
63
109
|
context = {
|
|
64
110
|
traceSeq: 0,
|
|
111
|
+
artifacts: new ArtifactWriter(runDir),
|
|
112
|
+
lock: Promise.resolve(),
|
|
113
|
+
streams: new Map(),
|
|
114
|
+
};
|
|
115
|
+
this.contexts.set(runDir, context);
|
|
116
|
+
}
|
|
117
|
+
return context;
|
|
118
|
+
}
|
|
119
|
+
streamFor(runDir, attemptId) {
|
|
120
|
+
const context = this.contextFor(runDir);
|
|
121
|
+
const key = attemptId ?? "";
|
|
122
|
+
let stream = context.streams.get(key);
|
|
123
|
+
if (!stream) {
|
|
124
|
+
stream = {
|
|
65
125
|
sessionSeq: 0,
|
|
66
126
|
sessionEventSeq: 0,
|
|
67
127
|
sessionBound: false,
|
|
68
128
|
sessionEventsStopped: false,
|
|
69
|
-
artifacts: new ArtifactWriter(runDir),
|
|
70
129
|
lock: Promise.resolve(),
|
|
71
|
-
sessionEventLock: Promise.resolve(),
|
|
72
130
|
};
|
|
73
|
-
|
|
131
|
+
context.streams.set(key, stream);
|
|
74
132
|
}
|
|
75
|
-
return
|
|
133
|
+
return stream;
|
|
76
134
|
}
|
|
77
135
|
/**
|
|
78
136
|
* Run `task` exclusively for this bundle. Sequence numbers are assigned
|
|
@@ -80,27 +138,181 @@ export class WorkflowRunStore {
|
|
|
80
138
|
*/
|
|
81
139
|
withRunLock(runDir, task) {
|
|
82
140
|
const context = this.contextFor(runDir);
|
|
83
|
-
const result = context.lock.then(
|
|
141
|
+
const result = context.lock.then(async () => {
|
|
142
|
+
this.fenceProvider?.(runDir)?.();
|
|
143
|
+
return await task();
|
|
144
|
+
});
|
|
84
145
|
context.lock = result.then(() => undefined, () => undefined);
|
|
85
146
|
return result;
|
|
86
147
|
}
|
|
87
|
-
withSessionEventLock(runDir, task) {
|
|
88
|
-
const
|
|
89
|
-
const result =
|
|
90
|
-
|
|
148
|
+
withSessionEventLock(runDir, attemptId, task) {
|
|
149
|
+
const stream = this.streamFor(runDir, attemptId);
|
|
150
|
+
const result = stream.lock.then(async () => {
|
|
151
|
+
this.fenceProvider?.(runDir)?.();
|
|
152
|
+
return await task();
|
|
153
|
+
});
|
|
154
|
+
stream.lock = result.then(() => undefined, () => undefined);
|
|
91
155
|
return result;
|
|
92
156
|
}
|
|
93
157
|
async initializeRunBundle(workflow, state) {
|
|
94
158
|
const runDir = this.runDirFor(state.runId);
|
|
95
|
-
this.contexts.delete(runDir);
|
|
96
159
|
return await this.withRunLock(runDir, async () => {
|
|
97
|
-
await fs.mkdir(
|
|
160
|
+
await fs.mkdir(this.outputRoot, { recursive: true, mode: 0o700 });
|
|
161
|
+
await fs.mkdir(runDir, { recursive: false, mode: 0o700 });
|
|
98
162
|
await writeJsonAtomic(path.join(runDir, WORKFLOW_SNAPSHOT_PATH), createDefinitionSnapshot(workflow));
|
|
99
163
|
await appendLine(path.join(runDir, TRACE_PATH), null);
|
|
100
164
|
await this.writeProjections(runDir, state);
|
|
101
165
|
return runDir;
|
|
102
166
|
});
|
|
103
167
|
}
|
|
168
|
+
/**
|
|
169
|
+
* Prepare an interrupted bundle for resume. Repairs a torn trace tail,
|
|
170
|
+
* drops trace events the state projection never recorded, and seeds the
|
|
171
|
+
* in-process context so new events continue the sequence. The caller must
|
|
172
|
+
* hold the run's queue claim; the fence is verified before any write.
|
|
173
|
+
*/
|
|
174
|
+
async prepareRunResume(runId) {
|
|
175
|
+
const runDir = this.runDirFor(runId);
|
|
176
|
+
this.fenceProvider?.(runDir)?.();
|
|
177
|
+
const bundle = await readRunBundle(runDir);
|
|
178
|
+
if (bundle === null) {
|
|
179
|
+
throw new Error(`Cannot resume unreadable workflow run: ${runId}`);
|
|
180
|
+
}
|
|
181
|
+
if (bundle.state.status !== "running") {
|
|
182
|
+
throw new Error(`Cannot resume workflow run ${runId} with status ${bundle.state.status}`);
|
|
183
|
+
}
|
|
184
|
+
const tracePath = resolveBundlePath(runDir, bundle.manifest.paths.trace, TRACE_PATH);
|
|
185
|
+
await repairTraceFile(tracePath, bundle.state.traceSeq, () => {
|
|
186
|
+
// The repair rewrites the trace; re-verify ownership immediately
|
|
187
|
+
// before the rename so a stalled runner cannot truncate a new claim
|
|
188
|
+
// holder's appended events.
|
|
189
|
+
this.fenceProvider?.(runDir)?.();
|
|
190
|
+
});
|
|
191
|
+
// A crashed session never finalizes its capture, and resuming recorders
|
|
192
|
+
// always write new segments — so dangling "recording" captures end here
|
|
193
|
+
// instead of reporting the run capture-corrupt forever.
|
|
194
|
+
await this.finalizeRecordingCaptures(runDir, "Workflow host stopped before the run finished");
|
|
195
|
+
const counts = await this.sessionCounts(runDir);
|
|
196
|
+
const captureFinished = bundle.sessionCapture?.status === "complete" || bundle.sessionCapture?.status === "failed";
|
|
197
|
+
this.contexts.set(runDir, {
|
|
198
|
+
traceSeq: bundle.state.traceSeq,
|
|
199
|
+
artifacts: new ArtifactWriter(runDir),
|
|
200
|
+
lock: Promise.resolve(),
|
|
201
|
+
streams: seededStreams({
|
|
202
|
+
sessionSeq: counts.entryCount,
|
|
203
|
+
sessionEventSeq: counts.lastEventSeq,
|
|
204
|
+
sessionBound: bundle.manifest.paths.session !== undefined,
|
|
205
|
+
sessionEventsStopped: captureFinished,
|
|
206
|
+
}),
|
|
207
|
+
});
|
|
208
|
+
const prepared = await readRunBundle(runDir);
|
|
209
|
+
if (prepared === null) {
|
|
210
|
+
throw new Error(`Workflow run ${runId} became unreadable during resume preparation`);
|
|
211
|
+
}
|
|
212
|
+
return prepared;
|
|
213
|
+
}
|
|
214
|
+
/**
|
|
215
|
+
* Finalize captures left "recording" by a session that is gone, so they
|
|
216
|
+
* report failed with the reason instead of dangling forever.
|
|
217
|
+
*/
|
|
218
|
+
async finalizeRecordingCaptures(runDir, reason, options = {}) {
|
|
219
|
+
if (options.skipFlat !== true) {
|
|
220
|
+
const flatCapture = await readJsonFile(path.join(runDir, SESSION_CAPTURE_PATH));
|
|
221
|
+
if (flatCapture?.status === "recording") {
|
|
222
|
+
const counts = await this.sessionCounts(runDir);
|
|
223
|
+
await this.writeSessionCapture(runDir, {
|
|
224
|
+
schema: SESSION_CAPTURE_SCHEMA,
|
|
225
|
+
eventSchema: SESSION_EVENT_SCHEMA,
|
|
226
|
+
status: "failed",
|
|
227
|
+
...counts,
|
|
228
|
+
failure: {
|
|
229
|
+
failedAt: new Date().toISOString(),
|
|
230
|
+
code: "host_interrupted",
|
|
231
|
+
message: reason,
|
|
232
|
+
},
|
|
233
|
+
});
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
for (const segmentId of await this.listSessionSegments(runDir)) {
|
|
237
|
+
const segmentCapture = await readJsonFile(path.join(runDir, sessionStreamPaths(segmentId).capture));
|
|
238
|
+
if (segmentCapture?.status !== "recording") {
|
|
239
|
+
continue;
|
|
240
|
+
}
|
|
241
|
+
const segmentCounts = await this.sessionCounts(runDir, segmentId);
|
|
242
|
+
await this.writeSessionCapture(runDir, {
|
|
243
|
+
schema: SESSION_CAPTURE_SCHEMA,
|
|
244
|
+
eventSchema: SESSION_EVENT_SCHEMA,
|
|
245
|
+
status: "failed",
|
|
246
|
+
...segmentCounts,
|
|
247
|
+
failure: {
|
|
248
|
+
failedAt: new Date().toISOString(),
|
|
249
|
+
code: "host_interrupted",
|
|
250
|
+
message: reason,
|
|
251
|
+
},
|
|
252
|
+
}, segmentId);
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
/** Mark a nonterminal bundle failed and append an interruption event. */
|
|
256
|
+
async markRunInterrupted(runId, reason = "Workflow host stopped before the run finished") {
|
|
257
|
+
const runDir = this.runDirFor(runId);
|
|
258
|
+
const bundle = await readRunBundle(runDir);
|
|
259
|
+
if (bundle === null || bundle.state.status !== "running") {
|
|
260
|
+
return bundle;
|
|
261
|
+
}
|
|
262
|
+
const lastTraceEvent = await readLastTraceEvent(runDir, bundle.manifest.paths.trace);
|
|
263
|
+
const counts = await this.sessionCounts(runDir);
|
|
264
|
+
const sessionBound = bundle.manifest.paths.session !== undefined;
|
|
265
|
+
const captureFinished = bundle.sessionCapture?.status === "complete" || bundle.sessionCapture?.status === "failed";
|
|
266
|
+
this.contexts.set(runDir, {
|
|
267
|
+
traceSeq: Math.max(bundle.state.traceSeq, lastTraceEvent?.seq ?? 0),
|
|
268
|
+
artifacts: new ArtifactWriter(runDir),
|
|
269
|
+
lock: Promise.resolve(),
|
|
270
|
+
streams: seededStreams({
|
|
271
|
+
sessionSeq: counts.entryCount,
|
|
272
|
+
sessionEventSeq: counts.lastEventSeq,
|
|
273
|
+
sessionBound,
|
|
274
|
+
sessionEventsStopped: captureFinished,
|
|
275
|
+
}),
|
|
276
|
+
});
|
|
277
|
+
const state = bundle.state;
|
|
278
|
+
if (lastTraceEvent !== null && recoverTerminalProjection(state, lastTraceEvent)) {
|
|
279
|
+
await this.writeLoadedProjections(runDir, state);
|
|
280
|
+
return await readRunBundle(runDir);
|
|
281
|
+
}
|
|
282
|
+
if (sessionBound && !captureFinished) {
|
|
283
|
+
await this.writeSessionCapture(runDir, {
|
|
284
|
+
schema: SESSION_CAPTURE_SCHEMA,
|
|
285
|
+
eventSchema: SESSION_EVENT_SCHEMA,
|
|
286
|
+
status: "failed",
|
|
287
|
+
...counts,
|
|
288
|
+
failure: {
|
|
289
|
+
failedAt: new Date().toISOString(),
|
|
290
|
+
code: "host_interrupted",
|
|
291
|
+
message: reason,
|
|
292
|
+
},
|
|
293
|
+
});
|
|
294
|
+
}
|
|
295
|
+
await this.finalizeRecordingCaptures(runDir, reason, { skipFlat: true });
|
|
296
|
+
state.status = "failed";
|
|
297
|
+
state.finishedAt = new Date().toISOString();
|
|
298
|
+
state.error = reason;
|
|
299
|
+
delete state.currentNode;
|
|
300
|
+
delete state.currentAttemptId;
|
|
301
|
+
delete state.currentNodeStartedAt;
|
|
302
|
+
delete state.statusDetail;
|
|
303
|
+
delete state.paused;
|
|
304
|
+
await this.withRunLock(runDir, async () => {
|
|
305
|
+
const traceEvent = await this.appendTraceEvent(runDir, state.runId, {
|
|
306
|
+
scope: "run",
|
|
307
|
+
type: "run_interrupted",
|
|
308
|
+
payload: { error: reason },
|
|
309
|
+
});
|
|
310
|
+
state.traceSeq = traceEvent.seq;
|
|
311
|
+
state.updatedAt = traceEvent.at;
|
|
312
|
+
await this.writeLoadedProjections(runDir, state);
|
|
313
|
+
});
|
|
314
|
+
return await readRunBundle(runDir);
|
|
315
|
+
}
|
|
104
316
|
/**
|
|
105
317
|
* Persist one transition: append the trace event, then rewrite the
|
|
106
318
|
* projections reflecting it.
|
|
@@ -119,48 +331,77 @@ export class WorkflowRunStore {
|
|
|
119
331
|
* append a `session_bound` trace event. Projections catch up on the next
|
|
120
332
|
* snapshot.
|
|
121
333
|
*/
|
|
122
|
-
|
|
334
|
+
/** True when a session binding already exists for this bundle. */
|
|
335
|
+
async hasSessionBinding(runDir) {
|
|
336
|
+
try {
|
|
337
|
+
await fs.lstat(path.join(runDir, SESSION_BINDING_PATH));
|
|
338
|
+
return true;
|
|
339
|
+
}
|
|
340
|
+
catch {
|
|
341
|
+
return false;
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
/** List capture segment attempt ids under `session/segments/`. */
|
|
345
|
+
async listSessionSegments(runDir) {
|
|
346
|
+
try {
|
|
347
|
+
const entries = await fs.readdir(path.join(runDir, SESSION_SEGMENTS_DIR), {
|
|
348
|
+
withFileTypes: true,
|
|
349
|
+
});
|
|
350
|
+
return entries
|
|
351
|
+
.filter((entry) => entry.isDirectory())
|
|
352
|
+
.map((entry) => entry.name)
|
|
353
|
+
.sort();
|
|
354
|
+
}
|
|
355
|
+
catch {
|
|
356
|
+
return [];
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
async writeSessionBinding(runDir, binding, attemptId) {
|
|
123
360
|
await this.withRunLock(runDir, async () => {
|
|
124
|
-
const
|
|
125
|
-
if (
|
|
361
|
+
const stream = this.streamFor(runDir, attemptId);
|
|
362
|
+
if (stream.sessionBound) {
|
|
126
363
|
return;
|
|
127
364
|
}
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
await
|
|
365
|
+
stream.sessionBound = true;
|
|
366
|
+
const paths = sessionStreamPaths(attemptId);
|
|
367
|
+
await fs.mkdir(path.join(runDir, paths.dir), { recursive: true, mode: 0o700 });
|
|
368
|
+
await writeJsonAtomic(path.join(runDir, paths.binding), binding);
|
|
131
369
|
await this.appendTraceEvent(runDir, binding.runId, {
|
|
132
370
|
scope: "session",
|
|
133
371
|
type: "session_bound",
|
|
134
|
-
payload: {
|
|
372
|
+
payload: {
|
|
373
|
+
piSessionId: binding.piSessionId,
|
|
374
|
+
...(attemptId !== undefined ? { captureAttemptId: attemptId } : {}),
|
|
375
|
+
},
|
|
135
376
|
});
|
|
136
377
|
});
|
|
137
378
|
}
|
|
138
379
|
/** Append one verbatim Pi session entry to `session/entries.ndjson`. */
|
|
139
|
-
async appendSessionEntry(runDir, entry) {
|
|
380
|
+
async appendSessionEntry(runDir, entry, attemptId) {
|
|
140
381
|
return await this.withRunLock(runDir, async () => {
|
|
141
|
-
const
|
|
142
|
-
|
|
382
|
+
const stream = this.streamFor(runDir, attemptId);
|
|
383
|
+
stream.sessionSeq += 1;
|
|
143
384
|
const record = {
|
|
144
|
-
seq:
|
|
385
|
+
seq: stream.sessionSeq,
|
|
145
386
|
at: new Date().toISOString(),
|
|
146
387
|
entry,
|
|
147
388
|
};
|
|
148
|
-
await appendLine(path.join(runDir,
|
|
389
|
+
await appendLine(path.join(runDir, sessionStreamPaths(attemptId).entries), record);
|
|
149
390
|
return record.seq;
|
|
150
391
|
});
|
|
151
392
|
}
|
|
152
393
|
/** Append a fully stamped ordered batch to `session/events.ndjson`. */
|
|
153
|
-
async appendSessionEventBatch(runDir, records) {
|
|
394
|
+
async appendSessionEventBatch(runDir, records, attemptId) {
|
|
154
395
|
if (records.length === 0) {
|
|
155
396
|
return;
|
|
156
397
|
}
|
|
157
|
-
await this.withSessionEventLock(runDir, async () => {
|
|
158
|
-
const
|
|
159
|
-
if (
|
|
398
|
+
await this.withSessionEventLock(runDir, attemptId, async () => {
|
|
399
|
+
const stream = this.streamFor(runDir, attemptId);
|
|
400
|
+
if (stream.sessionEventsStopped) {
|
|
160
401
|
throw new Error("Session event capture has stopped");
|
|
161
402
|
}
|
|
162
403
|
try {
|
|
163
|
-
let expected =
|
|
404
|
+
let expected = stream.sessionEventSeq + 1;
|
|
164
405
|
for (const record of records) {
|
|
165
406
|
validateSessionEventRecord(record);
|
|
166
407
|
if (record.seq !== expected) {
|
|
@@ -173,7 +414,7 @@ export class WorkflowRunStore {
|
|
|
173
414
|
payload: record.type === "tool_execution_started" ||
|
|
174
415
|
record.type === "tool_execution_finished" ||
|
|
175
416
|
(record.type === "assistant_event" && record.payload.type === "toolcall_end")
|
|
176
|
-
? (await encodeValue(record.payload,
|
|
417
|
+
? (await encodeValue(record.payload, this.contextFor(runDir).artifacts))
|
|
177
418
|
: record.payload,
|
|
178
419
|
})));
|
|
179
420
|
for (const record of encoded) {
|
|
@@ -181,30 +422,31 @@ export class WorkflowRunStore {
|
|
|
181
422
|
throw new Error(`session event exceeded ${SESSION_EVENT_MAX_BYTES} bytes`);
|
|
182
423
|
}
|
|
183
424
|
}
|
|
184
|
-
await appendLines(path.join(runDir,
|
|
185
|
-
|
|
425
|
+
await appendLines(path.join(runDir, sessionStreamPaths(attemptId).events), encoded);
|
|
426
|
+
stream.sessionEventSeq = records.at(-1)?.seq ?? stream.sessionEventSeq;
|
|
186
427
|
}
|
|
187
428
|
catch (error) {
|
|
188
|
-
|
|
429
|
+
stream.sessionEventsStopped = true;
|
|
189
430
|
throw error;
|
|
190
431
|
}
|
|
191
432
|
});
|
|
192
433
|
}
|
|
193
434
|
/** Atomically replace the temporal capture integrity projection. */
|
|
194
|
-
async writeSessionCapture(runDir, capture) {
|
|
435
|
+
async writeSessionCapture(runDir, capture, attemptId) {
|
|
195
436
|
validateSessionCapture(capture);
|
|
196
|
-
await this.withSessionEventLock(runDir, async () => {
|
|
197
|
-
const
|
|
437
|
+
await this.withSessionEventLock(runDir, attemptId, async () => {
|
|
438
|
+
const stream = this.streamFor(runDir, attemptId);
|
|
198
439
|
if (capture.status !== "recording") {
|
|
199
|
-
|
|
440
|
+
stream.sessionEventsStopped = true;
|
|
200
441
|
}
|
|
201
|
-
await writeJsonAtomic(path.join(runDir,
|
|
442
|
+
await writeJsonAtomic(path.join(runDir, sessionStreamPaths(attemptId).capture), capture);
|
|
202
443
|
});
|
|
203
444
|
}
|
|
204
445
|
/** Count complete durable session records after both writers have drained. */
|
|
205
|
-
async sessionCounts(runDir) {
|
|
206
|
-
const
|
|
207
|
-
const
|
|
446
|
+
async sessionCounts(runDir, attemptId) {
|
|
447
|
+
const paths = sessionStreamPaths(attemptId);
|
|
448
|
+
const events = await readCompleteNdjson(path.join(runDir, paths.events));
|
|
449
|
+
const entries = await readCompleteNdjson(path.join(runDir, paths.entries));
|
|
208
450
|
return {
|
|
209
451
|
eventCount: events.length,
|
|
210
452
|
entryCount: entries.length,
|
|
@@ -229,10 +471,118 @@ export class WorkflowRunStore {
|
|
|
229
471
|
const encoded = await encodeRunState(state, context.artifacts);
|
|
230
472
|
await writeJsonAtomic(path.join(runDir, STATE_PATH), encoded);
|
|
231
473
|
await writeJsonAtomic(path.join(runDir, MANIFEST_PATH), createManifest(state, {
|
|
232
|
-
session: context.sessionBound,
|
|
474
|
+
session: [...context.streams.values()].some((stream) => stream.sessionBound),
|
|
475
|
+
}));
|
|
476
|
+
}
|
|
477
|
+
async writeLoadedProjections(runDir, state) {
|
|
478
|
+
const context = this.contextFor(runDir);
|
|
479
|
+
await writeJsonAtomic(path.join(runDir, STATE_PATH), state);
|
|
480
|
+
await writeJsonAtomic(path.join(runDir, MANIFEST_PATH), createManifest(state, {
|
|
481
|
+
session: [...context.streams.values()].some((stream) => stream.sessionBound),
|
|
233
482
|
}));
|
|
234
483
|
}
|
|
235
484
|
}
|
|
485
|
+
/**
|
|
486
|
+
* Truncate a trace file to the longest contiguous valid prefix, then to the
|
|
487
|
+
* event count the state projection recorded. A crash can leave a partial
|
|
488
|
+
* final line or one event appended before its projection write; the repair
|
|
489
|
+
* keeps state and trace consistent so resume can continue the sequence. The
|
|
490
|
+
* rewrite is atomic: a stale writer appending to the old inode cannot
|
|
491
|
+
* interleave with the repaired file.
|
|
492
|
+
*/
|
|
493
|
+
async function repairTraceFile(tracePath, keepSeq, beforeWrite) {
|
|
494
|
+
let raw;
|
|
495
|
+
try {
|
|
496
|
+
raw = await fs.readFile(tracePath, "utf8");
|
|
497
|
+
}
|
|
498
|
+
catch {
|
|
499
|
+
return;
|
|
500
|
+
}
|
|
501
|
+
const lines = raw.split("\n");
|
|
502
|
+
if (lines.at(-1) === "") {
|
|
503
|
+
lines.pop();
|
|
504
|
+
}
|
|
505
|
+
const good = [];
|
|
506
|
+
let expectedSeq = 1;
|
|
507
|
+
for (const line of lines) {
|
|
508
|
+
if (line.trim().length === 0) {
|
|
509
|
+
break;
|
|
510
|
+
}
|
|
511
|
+
try {
|
|
512
|
+
const event = JSON.parse(line);
|
|
513
|
+
if (event.seq !== expectedSeq) {
|
|
514
|
+
break;
|
|
515
|
+
}
|
|
516
|
+
good.push(line);
|
|
517
|
+
expectedSeq += 1;
|
|
518
|
+
}
|
|
519
|
+
catch {
|
|
520
|
+
break;
|
|
521
|
+
}
|
|
522
|
+
}
|
|
523
|
+
const kept = good.slice(0, keepSeq);
|
|
524
|
+
if (kept.length === lines.length) {
|
|
525
|
+
return;
|
|
526
|
+
}
|
|
527
|
+
beforeWrite?.();
|
|
528
|
+
const tempPath = `${tracePath}.${process.pid}.${randomUUID()}.tmp`;
|
|
529
|
+
await fs.writeFile(tempPath, kept.length === 0 ? "" : `${kept.join("\n")}\n`, {
|
|
530
|
+
encoding: "utf8",
|
|
531
|
+
mode: 0o600,
|
|
532
|
+
});
|
|
533
|
+
await fs.rename(tempPath, tracePath);
|
|
534
|
+
}
|
|
535
|
+
function seededStreams(flat) {
|
|
536
|
+
return new Map([["", { ...flat, lock: Promise.resolve() }]]);
|
|
537
|
+
}
|
|
538
|
+
function recoverTerminalProjection(state, event) {
|
|
539
|
+
const status = terminalStatusForEvent(event.type);
|
|
540
|
+
if (status === undefined) {
|
|
541
|
+
return false;
|
|
542
|
+
}
|
|
543
|
+
state.traceSeq = event.seq;
|
|
544
|
+
state.status = status;
|
|
545
|
+
state.updatedAt = event.at;
|
|
546
|
+
state.finishedAt = event.at;
|
|
547
|
+
if (typeof event.payload.error === "string") {
|
|
548
|
+
state.error = event.payload.error;
|
|
549
|
+
}
|
|
550
|
+
if (typeof event.payload.waitingOn === "string") {
|
|
551
|
+
state.waitingOn = event.payload.waitingOn;
|
|
552
|
+
}
|
|
553
|
+
if (Object.hasOwn(event.payload, "finalOutput")) {
|
|
554
|
+
state.finalOutput = event.payload.finalOutput;
|
|
555
|
+
}
|
|
556
|
+
delete state.currentNode;
|
|
557
|
+
delete state.currentAttemptId;
|
|
558
|
+
delete state.currentNodeStartedAt;
|
|
559
|
+
return true;
|
|
560
|
+
}
|
|
561
|
+
function terminalStatusForEvent(type) {
|
|
562
|
+
switch (type) {
|
|
563
|
+
case "run_waiting":
|
|
564
|
+
return "waiting";
|
|
565
|
+
case "run_completed":
|
|
566
|
+
return "completed";
|
|
567
|
+
case "run_failed":
|
|
568
|
+
case "run_interrupted":
|
|
569
|
+
return "failed";
|
|
570
|
+
case "run_timed_out":
|
|
571
|
+
return "timed_out";
|
|
572
|
+
case "run_cancelled":
|
|
573
|
+
return "cancelled";
|
|
574
|
+
default:
|
|
575
|
+
return undefined;
|
|
576
|
+
}
|
|
577
|
+
}
|
|
578
|
+
function assertValidRunId(runId) {
|
|
579
|
+
if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,199}$/.test(runId)) {
|
|
580
|
+
throw new Error(`Invalid workflow run id: ${JSON.stringify(runId)}`);
|
|
581
|
+
}
|
|
582
|
+
}
|
|
583
|
+
function isMissingPath(error) {
|
|
584
|
+
return error instanceof Error && "code" in error && error.code === "ENOENT";
|
|
585
|
+
}
|
|
236
586
|
async function appendLine(filePath, value) {
|
|
237
587
|
await fs.mkdir(path.dirname(filePath), { recursive: true, mode: 0o700 });
|
|
238
588
|
await fs.appendFile(filePath, value === null ? "" : `${JSON.stringify(value)}\n`, {
|
|
@@ -426,6 +776,11 @@ async function encodeRunState(state, artifacts) {
|
|
|
426
776
|
: {}),
|
|
427
777
|
};
|
|
428
778
|
}
|
|
779
|
+
/** Read the final trace record without loading the rest of a run bundle. */
|
|
780
|
+
export async function readLastTraceEvent(runDir, tracePath) {
|
|
781
|
+
const events = await readNdjsonFile(resolveBundlePath(runDir, tracePath, TRACE_PATH));
|
|
782
|
+
return events.records.at(-1) ?? null;
|
|
783
|
+
}
|
|
429
784
|
/** Read a run bundle from disk. Returns null when the bundle is unreadable. */
|
|
430
785
|
export async function readRunBundle(runDir) {
|
|
431
786
|
const manifest = await readJsonFile(path.join(runDir, MANIFEST_PATH));
|
|
@@ -445,13 +800,52 @@ export async function readRunBundle(runDir) {
|
|
|
445
800
|
const entries = await readNdjsonFile(path.join(sessionDir, "entries.ndjson"));
|
|
446
801
|
const events = await readNdjsonFile(path.join(sessionDir, "events.ndjson"));
|
|
447
802
|
const sessionCapture = await readJsonFile(path.join(sessionDir, "capture.json"));
|
|
448
|
-
const
|
|
803
|
+
const flatIntegrity = assessSessionIntegrity({
|
|
449
804
|
binding: sessionBinding,
|
|
450
805
|
entries,
|
|
451
806
|
events,
|
|
452
807
|
capture: sessionCapture,
|
|
453
808
|
runTerminal: state.status !== "running",
|
|
454
809
|
});
|
|
810
|
+
const sessionSegments = [];
|
|
811
|
+
let segmentIds = [];
|
|
812
|
+
try {
|
|
813
|
+
segmentIds = (await fs.readdir(path.join(sessionDir, "segments"), { withFileTypes: true }))
|
|
814
|
+
.filter((entry) => entry.isDirectory())
|
|
815
|
+
.map((entry) => entry.name)
|
|
816
|
+
.sort();
|
|
817
|
+
}
|
|
818
|
+
catch {
|
|
819
|
+
// No segments directory means only the flat stream can exist.
|
|
820
|
+
}
|
|
821
|
+
for (const attemptId of segmentIds) {
|
|
822
|
+
const segmentDir = path.join(sessionDir, "segments", attemptId);
|
|
823
|
+
const binding = await readJsonFile(path.join(segmentDir, "binding.json"));
|
|
824
|
+
const segmentEntries = await readNdjsonFile(path.join(segmentDir, "entries.ndjson"));
|
|
825
|
+
const segmentEvents = await readNdjsonFile(path.join(segmentDir, "events.ndjson"));
|
|
826
|
+
const capture = await readJsonFile(path.join(segmentDir, "capture.json"));
|
|
827
|
+
sessionSegments.push({
|
|
828
|
+
attemptId,
|
|
829
|
+
binding,
|
|
830
|
+
entries: segmentEntries.records,
|
|
831
|
+
events: segmentEvents.records,
|
|
832
|
+
capture,
|
|
833
|
+
integrity: assessSessionIntegrity({
|
|
834
|
+
binding,
|
|
835
|
+
entries: segmentEntries,
|
|
836
|
+
events: segmentEvents,
|
|
837
|
+
capture,
|
|
838
|
+
runTerminal: state.status !== "running",
|
|
839
|
+
}),
|
|
840
|
+
});
|
|
841
|
+
}
|
|
842
|
+
// The headline integrity is the flat stream's when present; otherwise the
|
|
843
|
+
// chronologically latest capture segment speaks for the run (segment ids
|
|
844
|
+
// are random, so directory order says nothing about time).
|
|
845
|
+
sessionSegments.sort((a, b) => (a.binding?.boundAt ?? "").localeCompare(b.binding?.boundAt ?? ""));
|
|
846
|
+
const sessionIntegrity = flatIntegrity.status !== "unavailable" || sessionSegments.length === 0
|
|
847
|
+
? flatIntegrity
|
|
848
|
+
: (sessionSegments.at(-1)?.integrity ?? flatIntegrity);
|
|
455
849
|
return {
|
|
456
850
|
runDir,
|
|
457
851
|
manifest,
|
|
@@ -462,6 +856,7 @@ export async function readRunBundle(runDir) {
|
|
|
462
856
|
sessionEvents: events.records,
|
|
463
857
|
sessionCapture,
|
|
464
858
|
sessionIntegrity,
|
|
859
|
+
sessionSegments,
|
|
465
860
|
};
|
|
466
861
|
}
|
|
467
862
|
/**
|