@llblab/pi-actors 0.43.1 → 0.44.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +6 -6
- package/BACKLOG.md +1 -189
- package/CHANGELOG.md +412 -402
- package/README.md +5 -5
- package/dist/lib/async-runs.js +8 -30
- package/dist/lib/file-state.d.ts +3 -1
- package/dist/lib/file-state.js +25 -7
- package/dist/lib/inspector-overlay.js +46 -30
- package/dist/lib/limits.d.ts +10 -0
- package/dist/lib/limits.js +10 -0
- package/dist/lib/observability.d.ts +4 -2
- package/dist/lib/observability.js +43 -36
- package/dist/lib/run-evidence-policy.d.ts +95 -0
- package/dist/lib/run-evidence-policy.js +177 -0
- package/dist/lib/run-ui-runtime.js +2 -0
- package/dist/lib/runs-controls.d.ts +7 -5
- package/dist/lib/runs-controls.js +186 -49
- package/dist/lib/runs-retention.js +27 -14
- package/dist/lib/runs-trace.d.ts +25 -1
- package/dist/lib/runs-trace.js +410 -21
- package/dist/lib/runtime-triage.js +6 -22
- package/dist/lib/tools-inspect.js +48 -14
- package/dist/lib/trace-projection.js +90 -44
- package/dist/scripts/conformance.mjs +5 -0
- package/dist/scripts/locker.mjs +31 -74
- package/dist/scripts/music-player.mjs +41 -129
- package/dist/scripts/release-gates.mjs +26 -4
- package/dist/skills/actors/SKILL.md +7 -6
- package/dist/skills/swarm/SKILL.md +2 -2
- package/docs/README.md +0 -1
- package/docs/actor-inspector.md +3 -2
- package/docs/async-runs.md +10 -8
- package/docs/recipe-library.md +2 -2
- package/lib/async-runs.ts +8 -45
- package/lib/file-state.ts +28 -7
- package/lib/inspector-overlay.ts +29 -17
- package/lib/limits.ts +10 -0
- package/lib/observability.ts +55 -57
- package/lib/run-evidence-policy.ts +242 -0
- package/lib/run-ui-runtime.ts +2 -0
- package/lib/runs-controls.ts +177 -108
- package/lib/runs-retention.ts +28 -20
- package/lib/runs-trace.ts +496 -21
- package/lib/runtime-triage.ts +11 -25
- package/lib/tools-inspect.ts +45 -15
- package/lib/trace-projection.ts +137 -75
- package/package.json +1 -1
- package/scripts/conformance.mjs +5 -0
- package/scripts/locker.mjs +31 -74
- package/scripts/music-player.mjs +41 -129
- package/scripts/release-gates.mjs +26 -4
- package/skills/actors/SKILL.md +7 -6
- package/skills/swarm/SKILL.md +2 -2
- package/dist/lib/runtime-notifier.d.ts +0 -48
- package/dist/lib/runtime-notifier.js +0 -138
- package/docs/0.43-baseline.md +0 -39
- package/lib/runtime-notifier.ts +0 -211
package/lib/tools-inspect.ts
CHANGED
|
@@ -4,12 +4,14 @@
|
|
|
4
4
|
* Owns exact inspect target/view dispatch; source projection stays in domain modules.
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
|
+
import { statSync } from "node:fs";
|
|
7
8
|
import { join } from "node:path";
|
|
8
9
|
|
|
9
10
|
import * as AsyncRuns from "./async-runs.ts";
|
|
10
11
|
import * as ControlProjection from "./control-projection.ts";
|
|
11
12
|
import * as Inspector from "./inspector.ts";
|
|
12
13
|
import * as Limits from "./limits.ts";
|
|
14
|
+
import { computeRunControlCapacity } from "./run-evidence-policy.ts";
|
|
13
15
|
import * as Paths from "./paths.ts";
|
|
14
16
|
import * as RecipesDiscovery from "./recipes-discovery.ts";
|
|
15
17
|
import * as ReviewDiagnostics from "./review-diagnostics.ts";
|
|
@@ -75,20 +77,26 @@ function runtimeTriage(
|
|
|
75
77
|
const failed = runs.filter((run) => run.status === "failed");
|
|
76
78
|
const pendingControls: RuntimeTriage.RuntimePendingControl[] = [];
|
|
77
79
|
const staleControls: RuntimeTriage.RuntimePendingControl[] = [];
|
|
80
|
+
const backpressuredRuns: Array<Record<string, unknown>> = [];
|
|
78
81
|
const controlDiagnostics: Array<Record<string, unknown>> = [];
|
|
82
|
+
const traceDiagnostics: Array<Record<string, unknown>> = [];
|
|
79
83
|
const nowMs = Date.now();
|
|
80
84
|
for (const run of runs) {
|
|
81
85
|
const stateDir = typeof run.state_dir === "string" ? run.state_dir : undefined;
|
|
82
86
|
if (!stateDir) continue;
|
|
83
87
|
const runId = String(run.run ?? "unknown");
|
|
84
88
|
const journal = RunsControls.readRunControlJournalFromStateDir(stateDir);
|
|
89
|
+
const capacity = computeRunControlCapacity(journal.records);
|
|
85
90
|
controlDiagnostics.push(...journal.diagnostics.map((diagnostic) => ({
|
|
86
91
|
...(diagnostic.line === undefined ? {} : { line: diagnostic.line }),
|
|
87
|
-
reason: diagnostic.line === undefined
|
|
88
|
-
? "unreadable_control_journal"
|
|
89
|
-
: "invalid_control_json",
|
|
90
|
-
run: runId,
|
|
92
|
+
reason: diagnostic.line === undefined ? "unreadable_control_journal" : "invalid_control_json", run: runId,
|
|
91
93
|
})));
|
|
94
|
+
if (capacity.backpressured) backpressuredRuns.push({
|
|
95
|
+
run: runId, pending: capacity.pending, pending_limit: capacity.limit,
|
|
96
|
+
journal_bytes: statSync(RunsControls.runControlsFile(stateDir)).size,
|
|
97
|
+
});
|
|
98
|
+
const traceSummary = RunsTrace.summarizeRunTraceJournal(RunsTrace.readRunTraceJournal(stateDir));
|
|
99
|
+
if (!traceSummary.history_complete) traceDiagnostics.push({ run: runId, ...traceSummary });
|
|
92
100
|
for (const control of journal.records) {
|
|
93
101
|
const classified = RuntimeTriage.classifyRuntimeControl(
|
|
94
102
|
{
|
|
@@ -137,14 +145,15 @@ function runtimeTriage(
|
|
|
137
145
|
pending_controls: boundedPendingControls,
|
|
138
146
|
stale_control_count: staleControls.length,
|
|
139
147
|
stale_controls: boundedStaleControls,
|
|
140
|
-
|
|
141
|
-
control_diagnostics: controlDiagnostics.slice(0, 40),
|
|
148
|
+
backpressured_run_count: backpressuredRuns.length, backpressured_runs: backpressuredRuns.slice(0, 40),
|
|
149
|
+
control_diagnostic_count: controlDiagnostics.length, control_diagnostics: controlDiagnostics.slice(0, 40),
|
|
150
|
+
trace_diagnostic_count: traceDiagnostics.length, trace_diagnostics: traceDiagnostics.slice(0, 40),
|
|
142
151
|
attention_events: attentionEvents.slice(-40).reverse(),
|
|
143
152
|
next_actions: [
|
|
144
153
|
...(failed.length ? ["inspect target=runtime view=runs status=failed"] : []),
|
|
145
|
-
...staleControls.slice(0, 3).map((entry) =>
|
|
154
|
+
...new Set([...staleControls, ...backpressuredRuns].slice(0, 3).map((entry) =>
|
|
146
155
|
`inspect target=run:${entry.run} view=control`,
|
|
147
|
-
),
|
|
156
|
+
)),
|
|
148
157
|
...(attentionEvents.length
|
|
149
158
|
? attentionEvents.slice(-3).map((entry) =>
|
|
150
159
|
`inspect target=run:${String(entry.run)} view=trace source=runtime`,
|
|
@@ -159,7 +168,7 @@ function compactRuntime(view: string, details: Record<string, unknown>): string
|
|
|
159
168
|
return `\nruntime runs=${(details.runs as unknown[] | undefined)?.length ?? 0}`;
|
|
160
169
|
}
|
|
161
170
|
if (view === "triage") {
|
|
162
|
-
return `\nruntime failed=${(details.failed_runs as unknown[] | undefined)?.length ?? 0} pending_controls=${Number(details.pending_control_count ?? 0)} stale_controls=${Number(details.stale_control_count ?? 0)} attention=${(details.attention_events as unknown[] | undefined)?.length ?? 0}`;
|
|
171
|
+
return `\nruntime failed=${(details.failed_runs as unknown[] | undefined)?.length ?? 0} pending_controls=${Number(details.pending_control_count ?? 0)} stale_controls=${Number(details.stale_control_count ?? 0)} backpressured=${Number(details.backpressured_run_count ?? 0)} incomplete_trace=${Number(details.trace_diagnostic_count ?? 0)} attention=${(details.attention_events as unknown[] | undefined)?.length ?? 0}`;
|
|
163
172
|
}
|
|
164
173
|
return `\nruntime version=${String(details.version)} state_schema=${String(details.state_schema)} automatic_review=${String(details.automatic_review)}`;
|
|
165
174
|
}
|
|
@@ -252,8 +261,8 @@ function inspectRun(
|
|
|
252
261
|
throw new Error(`unsupported Trace source: ${source}`);
|
|
253
262
|
}
|
|
254
263
|
return {
|
|
255
|
-
run,
|
|
256
|
-
|
|
264
|
+
run, source,
|
|
265
|
+
summary: RunsTrace.summarizeRunTraceJournal(RunsTrace.readRunTraceJournal(stateDir)),
|
|
257
266
|
items: TraceProjection.projectRunTrace(stateDir, {
|
|
258
267
|
artifacts: status.artifacts as
|
|
259
268
|
| Record<string, AsyncRuns.RunArtifactDeclaration>
|
|
@@ -264,19 +273,40 @@ function inspectRun(
|
|
|
264
273
|
};
|
|
265
274
|
}
|
|
266
275
|
const runInstanceId = String(status.run_instance_id ?? "");
|
|
276
|
+
const journal = RunsControls.readRunControlJournalFromStateDir(stateDir);
|
|
277
|
+
const capacity = computeRunControlCapacity(journal.records);
|
|
278
|
+
const structuralDiagnostics = journal.records.flatMap((value) => {
|
|
279
|
+
const classified = RuntimeTriage.classifyRuntimeControl({ run,
|
|
280
|
+
runInstanceId: runInstanceId || undefined, status: String(status.status ?? "unknown") }, value, Date.now());
|
|
281
|
+
return classified.diagnostic ? [classified.diagnostic.reason] : [];
|
|
282
|
+
});
|
|
283
|
+
let journalBytes = 0;
|
|
284
|
+
try { journalBytes = statSync(RunsControls.runControlsFile(stateDir)).size; } catch {}
|
|
285
|
+
const diagnostics = [...journal.diagnostics.map((item) => ({
|
|
286
|
+
...(item.line === undefined ? {} : { line: item.line }),
|
|
287
|
+
reason: item.line === undefined ? "unreadable_control_journal" : "invalid_control_json",
|
|
288
|
+
})), ...structuralDiagnostics.map((reason) => ({ reason }))].slice(0, 40);
|
|
289
|
+
const nowMs = Date.now();
|
|
290
|
+
const stalePending = journal.records.reduce<number>((count, control) => count + Number(Boolean(
|
|
291
|
+
RuntimeTriage.classifyRuntimeControl({ run, runInstanceId: runInstanceId || undefined,
|
|
292
|
+
status: String(status.status ?? "unknown") }, control, nowMs).stale)), 0);
|
|
267
293
|
return {
|
|
268
294
|
run,
|
|
269
295
|
run_instance_id: runInstanceId,
|
|
270
296
|
status: status.status,
|
|
297
|
+
pending: capacity.pending, pending_limit: capacity.limit,
|
|
298
|
+
available: capacity.available, backpressured: capacity.backpressured,
|
|
299
|
+
journal_bytes: journalBytes, journal_limit: Limits.RUN_CONTROL_JOURNAL_MAX_BYTES,
|
|
300
|
+
...(capacity.oldest_pending_at ? { oldest_pending_at: capacity.oldest_pending_at } : {}),
|
|
301
|
+
stale_pending: stalePending,
|
|
302
|
+
diagnostics,
|
|
271
303
|
actor_actions: Array.isArray(status.control) ? status.control : [],
|
|
272
304
|
runtime_actions: status.status === "running" ? ["kill"] : ["archive", "prune"],
|
|
273
305
|
endpoint: runInstanceId
|
|
274
306
|
? RunControlDelivery.readRunControlEndpoint(stateDir, runInstanceId)
|
|
275
307
|
: undefined,
|
|
276
|
-
recent_controls:
|
|
277
|
-
.
|
|
278
|
-
.reverse()
|
|
279
|
-
.map(ControlProjection.projectRunControl),
|
|
308
|
+
recent_controls: journal.records.slice(-Math.max(1, Number(input.lines || 20))).reverse()
|
|
309
|
+
.map((control) => ControlProjection.projectRunControl(control as RunsControls.RunControlRecord)),
|
|
280
310
|
};
|
|
281
311
|
}
|
|
282
312
|
|
package/lib/trace-projection.ts
CHANGED
|
@@ -14,13 +14,13 @@ import {
|
|
|
14
14
|
resolveArtifactManifest,
|
|
15
15
|
type RunArtifactDeclaration,
|
|
16
16
|
} from "./runs-artifacts.ts";
|
|
17
|
+
import * as RunsControls from "./runs-controls.ts";
|
|
17
18
|
import type { RunControlRecord } from "./runs-controls.ts";
|
|
18
19
|
import { tailFile } from "./runs-status.ts";
|
|
19
|
-
import
|
|
20
|
+
import * as RunsTrace from "./runs-trace.ts";
|
|
20
21
|
import * as SessionEvidence from "./session-evidence.ts";
|
|
21
22
|
import {
|
|
22
23
|
readJsonFileResilient,
|
|
23
|
-
readJsonlFileResilient,
|
|
24
24
|
type StateReadDiagnostic,
|
|
25
25
|
} from "./state-readers.ts";
|
|
26
26
|
|
|
@@ -44,19 +44,52 @@ export interface TraceItem {
|
|
|
44
44
|
detail: unknown;
|
|
45
45
|
}
|
|
46
46
|
|
|
47
|
+
interface OrderedTraceItem {
|
|
48
|
+
item: TraceItem;
|
|
49
|
+
ordinal: number;
|
|
50
|
+
sourceKey: string;
|
|
51
|
+
}
|
|
52
|
+
|
|
47
53
|
const LIFECYCLE_PREFIXES = ["run.", "command."];
|
|
48
54
|
const SOURCE_LIMIT = 100;
|
|
55
|
+
const SOURCE_RANK: Record<TraceSource, number> = {
|
|
56
|
+
lifecycle: 6,
|
|
57
|
+
control: 5,
|
|
58
|
+
process: 4,
|
|
59
|
+
agent: 3,
|
|
60
|
+
artifact: 2,
|
|
61
|
+
runtime: 1,
|
|
62
|
+
};
|
|
49
63
|
|
|
50
64
|
function redact(value: unknown): unknown {
|
|
51
65
|
return SessionEvidence.redactSessionEvidenceValue(value);
|
|
52
66
|
}
|
|
53
67
|
|
|
54
|
-
function sourceForEvent(event: TraceEvent): TraceSource {
|
|
68
|
+
function sourceForEvent(event: RunsTrace.TraceEvent): TraceSource {
|
|
55
69
|
return LIFECYCLE_PREFIXES.some((prefix) => event.kind.startsWith(prefix))
|
|
56
70
|
? "lifecycle"
|
|
57
71
|
: "runtime";
|
|
58
72
|
}
|
|
59
73
|
|
|
74
|
+
function ordered(
|
|
75
|
+
item: TraceItem,
|
|
76
|
+
sourceKey: string,
|
|
77
|
+
ordinal: number,
|
|
78
|
+
): OrderedTraceItem {
|
|
79
|
+
return { item, ordinal, sourceKey };
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
function compareTraceItems(left: OrderedTraceItem, right: OrderedTraceItem): number {
|
|
83
|
+
const timestamp = right.item.ts.localeCompare(left.item.ts);
|
|
84
|
+
if (timestamp) return timestamp;
|
|
85
|
+
if (left.sourceKey === right.sourceKey) {
|
|
86
|
+
const ordinal = right.ordinal - left.ordinal;
|
|
87
|
+
if (ordinal) return ordinal;
|
|
88
|
+
}
|
|
89
|
+
return SOURCE_RANK[right.item.source] - SOURCE_RANK[left.item.source] ||
|
|
90
|
+
right.item.id.localeCompare(left.item.id);
|
|
91
|
+
}
|
|
92
|
+
|
|
60
93
|
function latestControlTimestamp(control: RunControlRecord): string {
|
|
61
94
|
return (
|
|
62
95
|
control.failed_at ??
|
|
@@ -98,125 +131,154 @@ function controlDiagnosticItems(
|
|
|
98
131
|
}));
|
|
99
132
|
}
|
|
100
133
|
|
|
101
|
-
function traceEventItems(stateDir: string):
|
|
102
|
-
const read =
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
134
|
+
function traceEventItems(stateDir: string): OrderedTraceItem[] {
|
|
135
|
+
const read = RunsTrace.readRunTraceJournal(stateDir);
|
|
136
|
+
const retained = read.events.slice(-SOURCE_LIMIT);
|
|
137
|
+
const projected = retained.map(({ event, ordinal }) => ordered({
|
|
138
|
+
id: event.id,
|
|
139
|
+
ts: event.ts,
|
|
140
|
+
source: sourceForEvent(event),
|
|
141
|
+
kind: event.kind,
|
|
142
|
+
summary: event.summary ?? event.kind,
|
|
143
|
+
...(event.level ? { level: event.level } : {}),
|
|
144
|
+
detail: redact({
|
|
145
|
+
...(event.attention ? { attention: event.attention } : {}),
|
|
146
|
+
...(event.data !== undefined ? { data: event.data } : {}),
|
|
147
|
+
}),
|
|
148
|
+
}, "trace", ordinal));
|
|
149
|
+
const diagnostics = diagnosticItems(read.diagnostics).map((item, index) =>
|
|
150
|
+
ordered(item, "trace", read.events.length + index + 1)
|
|
151
|
+
);
|
|
152
|
+
const hasMarker = read.events.some(
|
|
153
|
+
({ event }) => event.kind === "runtime.trace_compacted",
|
|
154
|
+
);
|
|
155
|
+
if (read.omittedPrefixBytes > 0 && !hasMarker) {
|
|
156
|
+
diagnostics.push(ordered({
|
|
157
|
+
id: "runtime:trace-history-incomplete",
|
|
158
|
+
ts: new Date(0).toISOString(),
|
|
159
|
+
source: "runtime",
|
|
160
|
+
kind: "runtime.trace_history_incomplete",
|
|
161
|
+
summary: "Trace history is incomplete",
|
|
162
|
+
level: "warning",
|
|
163
|
+
detail: {
|
|
164
|
+
omitted_prefix_bytes: read.omittedPrefixBytes,
|
|
165
|
+
read_bytes: read.readBytes,
|
|
166
|
+
retained_events: read.events.length,
|
|
167
|
+
},
|
|
168
|
+
}, "trace", 0));
|
|
169
|
+
}
|
|
170
|
+
return [...projected, ...diagnostics];
|
|
118
171
|
}
|
|
119
172
|
|
|
120
|
-
function controlItems(stateDir: string):
|
|
121
|
-
const read =
|
|
122
|
-
|
|
123
|
-
)
|
|
173
|
+
function controlItems(stateDir: string): OrderedTraceItem[] {
|
|
174
|
+
const read = RunsControls.readRunControlJournalFromStateDir(stateDir);
|
|
175
|
+
const start = Math.max(0, read.records.length - SOURCE_LIMIT);
|
|
176
|
+
const projected = read.records.slice(start).map((control, index) => {
|
|
177
|
+
const value = ControlProjection.projectRunControl(control as RunControlRecord);
|
|
178
|
+
return ordered({
|
|
179
|
+
id: `control:${value.id}`,
|
|
180
|
+
ts: latestControlTimestamp(value),
|
|
181
|
+
source: "control",
|
|
182
|
+
kind: `control.${value.status}`,
|
|
183
|
+
summary: `${value.action} ${value.status}`,
|
|
184
|
+
...(value.status === "failed" ? { level: "error" as const } : {}),
|
|
185
|
+
detail: value,
|
|
186
|
+
}, "control", start + index + 1);
|
|
187
|
+
});
|
|
124
188
|
return [
|
|
125
|
-
...
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
ts: latestControlTimestamp(projected),
|
|
130
|
-
source: "control" as const,
|
|
131
|
-
kind: `control.${projected.status}`,
|
|
132
|
-
summary: `${projected.action} ${projected.status}`,
|
|
133
|
-
...(projected.status === "failed" ? { level: "error" as const } : {}),
|
|
134
|
-
detail: projected,
|
|
135
|
-
};
|
|
136
|
-
}),
|
|
137
|
-
...controlDiagnosticItems(read.diagnostics),
|
|
189
|
+
...projected,
|
|
190
|
+
...controlDiagnosticItems(read.diagnostics).map((item, index) =>
|
|
191
|
+
ordered(item, "control", read.records.length + index + 1)
|
|
192
|
+
),
|
|
138
193
|
];
|
|
139
194
|
}
|
|
140
195
|
|
|
141
|
-
function agentItems(stateDir: string):
|
|
196
|
+
function agentItems(stateDir: string): OrderedTraceItem[] {
|
|
142
197
|
return ExecutionSessions.readExecutionTurns(stateDir)
|
|
143
198
|
.slice(-SOURCE_LIMIT)
|
|
144
199
|
.map((turn) => {
|
|
145
200
|
const text = turn.assistantText ?? turn.userText ?? "Agent turn";
|
|
146
201
|
const summary = text.replaceAll(/\s+/g, " ").trim().slice(0, 160);
|
|
147
|
-
return {
|
|
202
|
+
return ordered({
|
|
148
203
|
id: `agent:${turn.commandId}:${turn.sessionFile}:${turn.index}`,
|
|
149
204
|
ts: turn.timestamp ?? new Date(0).toISOString(),
|
|
150
|
-
source: "agent"
|
|
205
|
+
source: "agent",
|
|
151
206
|
kind: "agent.turn",
|
|
152
207
|
summary: summary || "Agent turn",
|
|
153
208
|
...(turn.error ? { level: "error" as const } : {}),
|
|
154
209
|
detail: redact(turn),
|
|
155
|
-
};
|
|
210
|
+
}, "agent", turn.index);
|
|
156
211
|
});
|
|
157
212
|
}
|
|
158
213
|
|
|
159
|
-
function processItems(stateDir: string):
|
|
214
|
+
function processItems(stateDir: string): OrderedTraceItem[] {
|
|
160
215
|
return ["stdout", "stderr"].flatMap((stream) => {
|
|
161
216
|
const path = join(stateDir, `${stream}.log`);
|
|
162
217
|
if (!existsSync(path)) return [];
|
|
163
218
|
const stat = statSync(path);
|
|
164
219
|
const tail = tailFile(path, Limits.DEFAULT_INSPECT_LINES);
|
|
165
220
|
if (!tail && stat.size === 0) return [];
|
|
166
|
-
return [{
|
|
221
|
+
return [ordered({
|
|
167
222
|
id: `process:${stream}`,
|
|
168
223
|
ts: stat.mtime.toISOString(),
|
|
169
|
-
source: "process"
|
|
224
|
+
source: "process",
|
|
170
225
|
kind: `process.${stream}`,
|
|
171
226
|
summary: `${stream} (${stat.size} bytes)`,
|
|
172
227
|
...(stream === "stderr" ? { level: "warning" as const } : {}),
|
|
173
228
|
detail: redact({ bytes: stat.size, path, tail }),
|
|
174
|
-
}];
|
|
229
|
+
}, `process:${stream}`, 1)];
|
|
175
230
|
});
|
|
176
231
|
}
|
|
177
232
|
|
|
178
|
-
function resultItems(stateDir: string):
|
|
233
|
+
function resultItems(stateDir: string): OrderedTraceItem[] {
|
|
179
234
|
const path = join(stateDir, "result.json");
|
|
180
235
|
const read = readJsonFileResilient<Record<string, unknown>>(path, {});
|
|
181
|
-
if (Object.keys(read.value).length === 0)
|
|
236
|
+
if (Object.keys(read.value).length === 0) {
|
|
237
|
+
return diagnosticItems(read.diagnostics).map((item, index) =>
|
|
238
|
+
ordered(item, "process:result", index)
|
|
239
|
+
);
|
|
240
|
+
}
|
|
182
241
|
const stat = statSync(path);
|
|
183
242
|
const failed = Number(read.value.code ?? 0) !== 0;
|
|
184
|
-
return [{
|
|
243
|
+
return [ordered({
|
|
185
244
|
id: "process:result",
|
|
186
|
-
ts:
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
: stat.mtime.toISOString(),
|
|
245
|
+
ts: typeof read.value.completedAt === "string"
|
|
246
|
+
? read.value.completedAt
|
|
247
|
+
: stat.mtime.toISOString(),
|
|
190
248
|
source: "process",
|
|
191
249
|
kind: failed ? "process.failed" : "process.result",
|
|
192
250
|
summary: failed ? "Run process failed" : "Run process completed",
|
|
193
251
|
...(failed ? { level: "error" as const } : {}),
|
|
194
252
|
detail: redact(read.value),
|
|
195
|
-
}, ...diagnosticItems(read.diagnostics)
|
|
253
|
+
}, "process:result", 1), ...diagnosticItems(read.diagnostics).map(
|
|
254
|
+
(item, index) => ordered(item, "process:result", index + 2),
|
|
255
|
+
)];
|
|
196
256
|
}
|
|
197
257
|
|
|
198
258
|
function artifactItems(
|
|
199
259
|
stateDir: string,
|
|
200
260
|
declarations: Record<string, RunArtifactDeclaration> | undefined,
|
|
201
|
-
):
|
|
261
|
+
): OrderedTraceItem[] {
|
|
202
262
|
const manifest = resolveArtifactManifest(declarations);
|
|
203
263
|
if (!manifest) return [];
|
|
204
|
-
return Object.entries(manifest).slice(0, SOURCE_LIMIT).map(
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
264
|
+
return Object.entries(manifest).slice(0, SOURCE_LIMIT).map(
|
|
265
|
+
([name, artifact], index) => ordered({
|
|
266
|
+
id: `artifact:${name}`,
|
|
267
|
+
ts: artifact.exists && existsSync(artifact.path)
|
|
268
|
+
? statSync(artifact.path).mtime.toISOString()
|
|
269
|
+
: new Date(0).toISOString(),
|
|
270
|
+
source: "artifact",
|
|
271
|
+
kind: artifact.exists ? "artifact.ready" : "artifact.missing",
|
|
272
|
+
summary: `${name}: ${artifact.exists ? "ready" : "missing"}`,
|
|
273
|
+
...(artifact.required && !artifact.exists ? { level: "warning" as const } : {}),
|
|
274
|
+
detail: redact({
|
|
275
|
+
exists: artifact.exists,
|
|
276
|
+
path: relative(stateDir, artifact.path).replaceAll("\\", "/"),
|
|
277
|
+
required: artifact.required,
|
|
278
|
+
...(artifact.size !== undefined ? { size: artifact.size } : {}),
|
|
279
|
+
}),
|
|
280
|
+
}, "artifact", index + 1),
|
|
281
|
+
);
|
|
220
282
|
}
|
|
221
283
|
|
|
222
284
|
export function projectRunTrace(
|
|
@@ -236,9 +298,9 @@ export function projectRunTrace(
|
|
|
236
298
|
...resultItems(stateDir),
|
|
237
299
|
...artifactItems(stateDir, options.artifacts),
|
|
238
300
|
]
|
|
239
|
-
.filter((item) => source === "all" || item.source === source)
|
|
240
|
-
.sort(
|
|
241
|
-
|
|
242
|
-
)
|
|
243
|
-
|
|
301
|
+
.filter(({ item }) => source === "all" || item.source === source)
|
|
302
|
+
.sort(compareTraceItems);
|
|
303
|
+
return items
|
|
304
|
+
.slice(0, Math.max(1, Math.min(options.limit ?? 100, 200)))
|
|
305
|
+
.map(({ item }) => item);
|
|
244
306
|
}
|
package/package.json
CHANGED
package/scripts/conformance.mjs
CHANGED
|
@@ -15,6 +15,11 @@ const conformanceSuites = [
|
|
|
15
15
|
"tests/control.test.ts",
|
|
16
16
|
"tests/runs-controls.test.ts",
|
|
17
17
|
"tests/runs-trace.test.ts",
|
|
18
|
+
"tests/observability.test.ts",
|
|
19
|
+
"tests/async-run-control.test.ts",
|
|
20
|
+
"tests/run-kernel-dogfood.test.ts",
|
|
21
|
+
"tests/installed.scripts.test.ts",
|
|
22
|
+
"tests/inspector-overlay.test.ts",
|
|
18
23
|
"tests/tools-inspect-kernel.test.ts",
|
|
19
24
|
"tests/recipes-discovery.test.ts",
|
|
20
25
|
"tests/review-swarm-dogfood.test.ts",
|
package/scripts/locker.mjs
CHANGED
|
@@ -11,7 +11,6 @@
|
|
|
11
11
|
import { spawnSync } from "node:child_process";
|
|
12
12
|
import { createHash } from "node:crypto";
|
|
13
13
|
import {
|
|
14
|
-
appendFileSync,
|
|
15
14
|
existsSync,
|
|
16
15
|
mkdirSync,
|
|
17
16
|
readFileSync,
|
|
@@ -32,8 +31,12 @@ async function importRuntimeModule(name) {
|
|
|
32
31
|
return await import(pathToFileURL(existsSync(compiled) ? compiled : source).href);
|
|
33
32
|
}
|
|
34
33
|
|
|
35
|
-
const { acquireFileMutationLock, writeJsonAtomic, writeTextAtomic } =
|
|
36
|
-
|
|
34
|
+
const { acquireFileMutationLock, writeJsonAtomic, writeTextAtomic } = await importRuntimeModule("file-state");
|
|
35
|
+
const { readJsonlFileResilient } = await importRuntimeModule("state-readers");
|
|
36
|
+
const LOCKER_JOURNAL_MAX_RECORDS = 512;
|
|
37
|
+
const LOCKER_JOURNAL_MAX_BYTES = 1024 * 1024;
|
|
38
|
+
const { claimRunControlByIdInStateDir, updateRunControlStatusInStateDir } =
|
|
39
|
+
await importRuntimeModule("runs-controls");
|
|
37
40
|
const { appendRunTraceEvent } = await importRuntimeModule("runs-trace");
|
|
38
41
|
|
|
39
42
|
function parseArgs(argv) {
|
|
@@ -59,7 +62,6 @@ async function runLocker(argv = process.argv.slice(2)) {
|
|
|
59
62
|
const queuePath = join(stateDir, "queue.json");
|
|
60
63
|
const locksPath = join(stateDir, "locks.json");
|
|
61
64
|
const journalPath = join(stateDir, "journal.jsonl");
|
|
62
|
-
const controlsPath = join(stateDir, "controls.jsonl");
|
|
63
65
|
const controlPath = join(stateDir, "control.fifo");
|
|
64
66
|
let runInstanceId;
|
|
65
67
|
mkdirSync(stateDir, { recursive: true });
|
|
@@ -96,10 +98,22 @@ async function runLocker(argv = process.argv.slice(2)) {
|
|
|
96
98
|
}
|
|
97
99
|
|
|
98
100
|
function journal(event, data = {}) {
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
101
|
+
const release = acquireFileMutationLock(journalPath);
|
|
102
|
+
try {
|
|
103
|
+
const records = [...readJsonlFileResilient(journalPath).records,
|
|
104
|
+
{ event, ts: new Date().toISOString(), ...data }]
|
|
105
|
+
.slice(-LOCKER_JOURNAL_MAX_RECORDS);
|
|
106
|
+
let content = encodeJournal(records);
|
|
107
|
+
while (records.length && Buffer.byteLength(content) > LOCKER_JOURNAL_MAX_BYTES) {
|
|
108
|
+
records.shift();
|
|
109
|
+
content = encodeJournal(records);
|
|
110
|
+
}
|
|
111
|
+
writeTextAtomic(journalPath, content);
|
|
112
|
+
} finally { release(); }
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
function encodeJournal(records) {
|
|
116
|
+
return records.length ? `${records.map((record) => JSON.stringify(record)).join("\n")}\n` : "";
|
|
103
117
|
}
|
|
104
118
|
|
|
105
119
|
function emitTrace(kind, summary, data = {}, level = "info") {
|
|
@@ -147,69 +161,6 @@ async function runLocker(argv = process.argv.slice(2)) {
|
|
|
147
161
|
}
|
|
148
162
|
}
|
|
149
163
|
|
|
150
|
-
function acquireControlsLock() {
|
|
151
|
-
return acquireFileMutationLock(controlsPath);
|
|
152
|
-
}
|
|
153
|
-
|
|
154
|
-
function readControls() {
|
|
155
|
-
if (!existsSync(controlsPath)) return [];
|
|
156
|
-
return readFileSync(controlsPath, "utf8")
|
|
157
|
-
.split("\n")
|
|
158
|
-
.filter((line) => line.trim())
|
|
159
|
-
.map((line) => {
|
|
160
|
-
try {
|
|
161
|
-
return JSON.parse(line);
|
|
162
|
-
} catch {
|
|
163
|
-
return undefined;
|
|
164
|
-
}
|
|
165
|
-
})
|
|
166
|
-
.filter(Boolean);
|
|
167
|
-
}
|
|
168
|
-
|
|
169
|
-
function writeControls(controls) {
|
|
170
|
-
writeTextAtomic(
|
|
171
|
-
controlsPath,
|
|
172
|
-
controls.length
|
|
173
|
-
? `${controls.map((control) => JSON.stringify(control)).join("\n")}\n`
|
|
174
|
-
: "",
|
|
175
|
-
);
|
|
176
|
-
}
|
|
177
|
-
|
|
178
|
-
function claimControl(id) {
|
|
179
|
-
const release = acquireControlsLock();
|
|
180
|
-
try {
|
|
181
|
-
const controls = readControls();
|
|
182
|
-
const control = controls.find((item) => item.id === id);
|
|
183
|
-
if (
|
|
184
|
-
!control ||
|
|
185
|
-
control.run_instance_id !== runInstanceId ||
|
|
186
|
-
(control.status !== "queued" && control.status !== "delivered")
|
|
187
|
-
) {
|
|
188
|
-
return undefined;
|
|
189
|
-
}
|
|
190
|
-
control.claimed_at = new Date().toISOString();
|
|
191
|
-
control.status = "claimed";
|
|
192
|
-
writeControls(controls);
|
|
193
|
-
return { ...control };
|
|
194
|
-
} finally {
|
|
195
|
-
release();
|
|
196
|
-
}
|
|
197
|
-
}
|
|
198
|
-
|
|
199
|
-
function finalizeControl(id, status, error) {
|
|
200
|
-
const release = acquireControlsLock();
|
|
201
|
-
try {
|
|
202
|
-
const controls = readControls();
|
|
203
|
-
const control = controls.find((item) => item.id === id);
|
|
204
|
-
if (!control || control.status !== "claimed") return;
|
|
205
|
-
control.status = status;
|
|
206
|
-
control[`${status}_at`] = new Date().toISOString();
|
|
207
|
-
if (error) control.error = error;
|
|
208
|
-
writeControls(controls);
|
|
209
|
-
} finally {
|
|
210
|
-
release();
|
|
211
|
-
}
|
|
212
|
-
}
|
|
213
164
|
|
|
214
165
|
function tailJournal(count) {
|
|
215
166
|
if (!existsSync(journalPath)) return [];
|
|
@@ -397,7 +348,9 @@ async function runLocker(argv = process.argv.slice(2)) {
|
|
|
397
348
|
function handleLine(line) {
|
|
398
349
|
const control = normalizeControl(line);
|
|
399
350
|
if (!control) return false;
|
|
400
|
-
const claimed =
|
|
351
|
+
const claimed = claimRunControlByIdInStateDir(
|
|
352
|
+
stateDir, runInstanceId, control.id,
|
|
353
|
+
);
|
|
401
354
|
if (!claimed) {
|
|
402
355
|
emitTrace(
|
|
403
356
|
"lock.control_rejected",
|
|
@@ -409,11 +362,15 @@ async function runLocker(argv = process.argv.slice(2)) {
|
|
|
409
362
|
}
|
|
410
363
|
try {
|
|
411
364
|
const stopping = handle(claimed) === true;
|
|
412
|
-
|
|
365
|
+
updateRunControlStatusInStateDir(
|
|
366
|
+
stateDir, claimed.id, "handled", {}, ["claimed"],
|
|
367
|
+
);
|
|
413
368
|
return stopping;
|
|
414
369
|
} catch (error) {
|
|
415
370
|
const text = error instanceof Error ? error.message : String(error);
|
|
416
|
-
|
|
371
|
+
updateRunControlStatusInStateDir(
|
|
372
|
+
stateDir, claimed.id, "failed", { error: text }, ["claimed"],
|
|
373
|
+
);
|
|
417
374
|
journal("lock.error", { error: text });
|
|
418
375
|
emitTrace("lock.error", text, { error: text }, "error");
|
|
419
376
|
return false;
|