@llblab/pi-actors 0.43.1 → 0.44.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +6 -6
- package/BACKLOG.md +1 -189
- package/CHANGELOG.md +412 -402
- package/README.md +5 -5
- package/dist/lib/async-runs.js +8 -30
- package/dist/lib/file-state.d.ts +3 -1
- package/dist/lib/file-state.js +25 -7
- package/dist/lib/inspector-overlay.js +46 -30
- package/dist/lib/limits.d.ts +10 -0
- package/dist/lib/limits.js +10 -0
- package/dist/lib/observability.d.ts +4 -2
- package/dist/lib/observability.js +43 -36
- package/dist/lib/run-evidence-policy.d.ts +95 -0
- package/dist/lib/run-evidence-policy.js +177 -0
- package/dist/lib/run-ui-runtime.js +2 -0
- package/dist/lib/runs-controls.d.ts +7 -5
- package/dist/lib/runs-controls.js +186 -49
- package/dist/lib/runs-retention.js +27 -14
- package/dist/lib/runs-trace.d.ts +25 -1
- package/dist/lib/runs-trace.js +410 -21
- package/dist/lib/runtime-triage.js +6 -22
- package/dist/lib/tools-inspect.js +48 -14
- package/dist/lib/trace-projection.js +90 -44
- package/dist/scripts/conformance.mjs +5 -0
- package/dist/scripts/locker.mjs +31 -74
- package/dist/scripts/music-player.mjs +41 -129
- package/dist/scripts/release-gates.mjs +26 -4
- package/dist/skills/actors/SKILL.md +7 -6
- package/dist/skills/swarm/SKILL.md +2 -2
- package/docs/README.md +0 -1
- package/docs/actor-inspector.md +3 -2
- package/docs/async-runs.md +10 -8
- package/docs/recipe-library.md +2 -2
- package/lib/async-runs.ts +8 -45
- package/lib/file-state.ts +28 -7
- package/lib/inspector-overlay.ts +29 -17
- package/lib/limits.ts +10 -0
- package/lib/observability.ts +55 -57
- package/lib/run-evidence-policy.ts +242 -0
- package/lib/run-ui-runtime.ts +2 -0
- package/lib/runs-controls.ts +177 -108
- package/lib/runs-retention.ts +28 -20
- package/lib/runs-trace.ts +496 -21
- package/lib/runtime-triage.ts +11 -25
- package/lib/tools-inspect.ts +45 -15
- package/lib/trace-projection.ts +137 -75
- package/package.json +1 -1
- package/scripts/conformance.mjs +5 -0
- package/scripts/locker.mjs +31 -74
- package/scripts/music-player.mjs +41 -129
- package/scripts/release-gates.mjs +26 -4
- package/skills/actors/SKILL.md +7 -6
- package/skills/swarm/SKILL.md +2 -2
- package/dist/lib/runtime-notifier.d.ts +0 -48
- package/dist/lib/runtime-notifier.js +0 -138
- package/docs/0.43-baseline.md +0 -39
- package/lib/runtime-notifier.ts +0 -211
package/README.md
CHANGED
|
@@ -121,13 +121,13 @@ String command-template leaves execute directly without shell interpretation. Us
|
|
|
121
121
|
}
|
|
122
122
|
```
|
|
123
123
|
|
|
124
|
-
Trace fields are exact: `id`, `ts`, `kind`, and optional `summary`, `data`, `level`, `attention`. Address, sender, recipient, reply, and routing fields fail validation.
|
|
124
|
+
Trace fields are exact: `id`, `ts`, `kind`, and optional `summary`, `data`, `level`, `attention`. Address, sender, recipient, reply, and routing fields fail validation. Trace is a bounded retained suffix, not an audit archive: the canonical authority appends within 2,048 events and 4 MiB or atomically keeps the newest suffix plus one warning-only `runtime.trace_compacted` marker. That marker means older history was discarded; `result.json`, `execution.json`, terminal state, and declared artifacts retain their own authority. `inspect view=trace` reports whether retained history is complete. Reads are newline-safe and deterministic; first-party scripts never write this file directly.
|
|
125
125
|
|
|
126
|
-
Use `attention: "notify"` for visible status and `attention: "followup"` only when the coordinator needs semantic follow-up context. Store large evidence in artifacts or bounded execution captures.
|
|
126
|
+
Attention is a live wake hint, not a durable queue: persist recovery state or an artifact first. Use `attention: "notify"` for visible status and `attention: "followup"` only when the coordinator needs semantic follow-up context; compaction may discard older hints. Store large evidence in artifacts or bounded execution captures.
|
|
127
127
|
|
|
128
128
|
## Control
|
|
129
129
|
|
|
130
|
-
Controls persist to `controls.jsonl` before transport.
|
|
130
|
+
Controls persist to `controls.jsonl` before transport. One token-owned lock rejects a 65th pending Control or 1 MiB rewrite before admission, fails closed on malformed or stale-generation evidence, and atomically admits one record. Canonical transitions retain a 128-terminal tail, expected-state fencing, and 4 KiB errors. Admitted nonterminal Controls never expire automatically. Every record carries immutable `run_instance_id`. `inspect view=control` reports pending capacity, saturation, stale work, journal bytes, and bounded diagnostics. If a stuck Run is saturated, use runtime-owned `kill`; it bypasses actor-local capacity and creates no synthetic Control.
|
|
131
131
|
|
|
132
132
|
Long-lived services publish `control-endpoint.json` only when ready:
|
|
133
133
|
|
|
@@ -159,7 +159,7 @@ Core files:
|
|
|
159
159
|
- `execution.json` — command/session provenance and complete-capture references;
|
|
160
160
|
- `result.json`, command logs, progress, and declared artifacts.
|
|
161
161
|
|
|
162
|
-
The runtime preserves owner filtering, process-identity verification, lifecycle locking, shutdown kill, terminal reconciliation, bounded captures, owned Pi sessions, path containment, and redaction.
|
|
162
|
+
The runtime preserves owner filtering, process-identity verification, lifecycle locking, shutdown kill independent of actor-local Control capacity, terminal reconciliation, bounded captures, owned Pi sessions, path containment, and redaction. Trace/Control quotas do not constrain user-declared artifacts, repositories, media sources, complete captures, or actor-owned workload state. Restart clears generation-local Trace, Control, endpoint, execution, and terminal evidence; archive preserves the bounded tree, while prune preserves only requested artifacts.
|
|
163
163
|
|
|
164
164
|
## Actor Inspector
|
|
165
165
|
|
|
@@ -197,7 +197,7 @@ npm run validate
|
|
|
197
197
|
npm run test:preservation
|
|
198
198
|
```
|
|
199
199
|
|
|
200
|
-
See the [documentation index](./docs/README.md), [Run lifecycle](./docs/async-runs.md), [Recipe library](./docs/recipe-library.md)
|
|
200
|
+
See the [documentation index](./docs/README.md), [Run lifecycle](./docs/async-runs.md), and [Recipe library](./docs/recipe-library.md).
|
|
201
201
|
|
|
202
202
|
Project context: [AGENTS.md](./AGENTS.md) · [BACKLOG.md](./BACKLOG.md) · [CHANGELOG.md](./CHANGELOG.md).
|
|
203
203
|
|
package/dist/lib/async-runs.js
CHANGED
|
@@ -23,7 +23,6 @@ import * as RunsStart from "./runs-start.js";
|
|
|
23
23
|
import { appendRunTraceEvent } from "./runs-trace.js";
|
|
24
24
|
import * as RunsIndex from "./runs-index.js";
|
|
25
25
|
import * as RunsParentTeardown from "./runs-parent-teardown.js";
|
|
26
|
-
import { appendRunControlInStateDir, updateRunControlStatusInStateDir, } from "./runs-controls.js";
|
|
27
26
|
import { deliverRunControl, } from "./runs-control-delivery.js";
|
|
28
27
|
import { buildRunStatus, tailFile, } from "./runs-status.js";
|
|
29
28
|
import { readJsonFileResilient } from "./state-readers.js";
|
|
@@ -576,57 +575,36 @@ function stopRun(runOrDir, signal, event, expected = {}) {
|
|
|
576
575
|
status.run_instance_id !== expected.runInstanceId) {
|
|
577
576
|
return { stopped: false, reason: "run generation changed", status };
|
|
578
577
|
}
|
|
579
|
-
const control = event === "run.kill" && typeof status.run_instance_id === "string"
|
|
580
|
-
? appendRunControlInStateDir(stateDir, {
|
|
581
|
-
action: "kill",
|
|
582
|
-
run_instance_id: status.run_instance_id,
|
|
583
|
-
})
|
|
584
|
-
: undefined;
|
|
585
|
-
if (control) {
|
|
586
|
-
updateRunControlStatusInStateDir(stateDir, control.id, "claimed", {}, ["queued"]);
|
|
587
|
-
}
|
|
588
|
-
const finish = (result) => {
|
|
589
|
-
if (!control)
|
|
590
|
-
return result;
|
|
591
|
-
const handled = result.stopped === true;
|
|
592
|
-
updateRunControlStatusInStateDir(stateDir, control.id, handled ? "handled" : "failed", handled ? {} : { error: String(result.reason ?? "kill rejected") }, ["claimed"]);
|
|
593
|
-
return { ...result, control_id: control.id };
|
|
594
|
-
};
|
|
595
578
|
const pid = Number(status.pid || 0);
|
|
596
579
|
if (status.status !== "running" && status.status !== "exited") {
|
|
597
|
-
return
|
|
580
|
+
return { stopped: false, reason: "not running", status };
|
|
598
581
|
}
|
|
599
582
|
const identity = verifyRunProcessIdentity(pid, status.process_identity);
|
|
600
583
|
if (status.status === "exited") {
|
|
601
584
|
if (identity.status === "owner_mismatch" ||
|
|
602
585
|
identity.status === "unsupported_proof") {
|
|
603
|
-
return
|
|
586
|
+
return {
|
|
604
587
|
stopped: false,
|
|
605
588
|
reason: identity.status.replaceAll("_", " "),
|
|
606
589
|
process_identity_status: identity.status,
|
|
607
590
|
status,
|
|
608
|
-
}
|
|
591
|
+
};
|
|
609
592
|
}
|
|
610
|
-
return
|
|
593
|
+
return { stopped: false, reason: "not running", status };
|
|
611
594
|
}
|
|
612
595
|
if (!identity.valid) {
|
|
613
|
-
return
|
|
596
|
+
return {
|
|
614
597
|
stopped: false,
|
|
615
598
|
reason: identity.status.replaceAll("_", " "),
|
|
616
599
|
process_identity_status: identity.status,
|
|
617
600
|
status,
|
|
618
|
-
}
|
|
601
|
+
};
|
|
619
602
|
}
|
|
620
603
|
let signalResult;
|
|
621
604
|
try {
|
|
622
605
|
signalResult = signalOwnedRunProcess(pid, signal, status.process_identity);
|
|
623
606
|
}
|
|
624
607
|
catch (error) {
|
|
625
|
-
if (control) {
|
|
626
|
-
updateRunControlStatusInStateDir(stateDir, control.id, "failed", {
|
|
627
|
-
error: error instanceof Error ? error.message : String(error),
|
|
628
|
-
});
|
|
629
|
-
}
|
|
630
608
|
throw error;
|
|
631
609
|
}
|
|
632
610
|
appendRunTraceEvent(stateDir, {
|
|
@@ -643,13 +621,13 @@ function stopRun(runOrDir, signal, event, expected = {}) {
|
|
|
643
621
|
finalizeInterruptedExecution(stateDir, "cancelled", signal);
|
|
644
622
|
markTerminalProgress(stateDir, "cancelled");
|
|
645
623
|
}
|
|
646
|
-
return
|
|
624
|
+
return {
|
|
647
625
|
stopped: true,
|
|
648
626
|
pid,
|
|
649
627
|
signal,
|
|
650
628
|
...signalResult,
|
|
651
629
|
state_dir: stateDir,
|
|
652
|
-
}
|
|
630
|
+
};
|
|
653
631
|
}
|
|
654
632
|
finally {
|
|
655
633
|
releaseControlLock();
|
package/dist/lib/file-state.d.ts
CHANGED
|
@@ -12,5 +12,7 @@ export interface FileMutationLockOptions {
|
|
|
12
12
|
}
|
|
13
13
|
export declare function acquireFileMutationLock(path: string, options?: FileMutationLockOptions): () => void;
|
|
14
14
|
export declare function withFileMutationLock<T>(path: string, mutate: () => T, options?: FileMutationLockOptions): T;
|
|
15
|
-
export declare function writeTextAtomic(path: string, content: string
|
|
15
|
+
export declare function writeTextAtomic(path: string, content: string, options?: {
|
|
16
|
+
onBeforeReplace?(): void;
|
|
17
|
+
}): void;
|
|
16
18
|
export declare function writeJsonAtomic(path: string, value: unknown): void;
|
package/dist/lib/file-state.js
CHANGED
|
@@ -8,8 +8,11 @@ import { createHash, randomUUID } from "node:crypto";
|
|
|
8
8
|
import { existsSync, lstatSync, mkdirSync, readFileSync, realpathSync, renameSync, rmSync, statSync, unlinkSync, writeFileSync } from "node:fs";
|
|
9
9
|
import { tmpdir } from "node:os";
|
|
10
10
|
import { basename, dirname, join, parse, resolve } from "node:path";
|
|
11
|
-
const FILE_MUTATION_LOCK_TIMEOUT_MS = process.platform === "win32" ?
|
|
11
|
+
const FILE_MUTATION_LOCK_TIMEOUT_MS = process.platform === "win32" ? 30000 : 15000;
|
|
12
12
|
const FILE_MUTATION_LOCK_STALE_MS = 30000;
|
|
13
|
+
const FILE_MUTATION_LOCK_RECLAIM_POLL_MS = 100;
|
|
14
|
+
const FILE_MUTATION_LOCK_REMOVAL_GRACE_MS = 250;
|
|
15
|
+
const FILE_MUTATION_LOCK_MAX_WAIT_MS = 50;
|
|
13
16
|
const FILE_MUTATION_LOCK_ROOT = join(tmpdir(), "pi-actors-file-locks");
|
|
14
17
|
function canonicalMutationPath(path) {
|
|
15
18
|
const absolute = resolve(path);
|
|
@@ -65,7 +68,8 @@ function lockOwnerStatus(lockPath) {
|
|
|
65
68
|
return "unknown";
|
|
66
69
|
try {
|
|
67
70
|
process.kill(pid, 0);
|
|
68
|
-
if (Date.now() - statSync(lockPath).mtimeMs >
|
|
71
|
+
if (Date.now() - statSync(lockPath).mtimeMs > FILE_MUTATION_LOCK_REMOVAL_GRACE_MS &&
|
|
72
|
+
isZombieProcess(pid))
|
|
69
73
|
return "dead";
|
|
70
74
|
return "alive";
|
|
71
75
|
}
|
|
@@ -132,8 +136,10 @@ function tryReclaimRemovalBoundary(reclaimPath) {
|
|
|
132
136
|
if (!existsSync(reclaimPath))
|
|
133
137
|
return;
|
|
134
138
|
try {
|
|
135
|
-
const inspectedToken = readLockToken(reclaimPath);
|
|
136
139
|
const age = Date.now() - statSync(reclaimPath).mtimeMs;
|
|
140
|
+
if (age <= FILE_MUTATION_LOCK_REMOVAL_GRACE_MS)
|
|
141
|
+
return;
|
|
142
|
+
const inspectedToken = readLockToken(reclaimPath);
|
|
137
143
|
const ownerStatus = lockOwnerStatus(reclaimPath);
|
|
138
144
|
if ((ownerStatus === "dead" ||
|
|
139
145
|
(ownerStatus === "unknown" && age > FILE_MUTATION_LOCK_STALE_MS)) &&
|
|
@@ -200,6 +206,8 @@ export function acquireFileMutationLock(path, options = {}) {
|
|
|
200
206
|
const token = randomUUID();
|
|
201
207
|
const pendingPath = prepareLockBoundary(lockPath, token, options.onBeforeLockPublish);
|
|
202
208
|
let contentionReported = false;
|
|
209
|
+
let nextReclaimAt = 0;
|
|
210
|
+
let waitMs = 10;
|
|
203
211
|
try {
|
|
204
212
|
for (;;) {
|
|
205
213
|
try {
|
|
@@ -211,13 +219,22 @@ export function acquireFileMutationLock(path, options = {}) {
|
|
|
211
219
|
contentionReported = true;
|
|
212
220
|
options.onContention?.();
|
|
213
221
|
}
|
|
214
|
-
|
|
215
|
-
if (
|
|
222
|
+
const now = Date.now();
|
|
223
|
+
if (now >= nextReclaimAt) {
|
|
224
|
+
nextReclaimAt = now + FILE_MUTATION_LOCK_RECLAIM_POLL_MS;
|
|
225
|
+
if (tryReclaimMutationLock(lockPath, options)) {
|
|
226
|
+
waitMs = 10;
|
|
227
|
+
continue;
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
if (now >= deadline) {
|
|
216
231
|
throw new Error(`Timed out waiting for file mutation lock: ${canonicalMutationPath(path)}`, {
|
|
217
232
|
cause: error,
|
|
218
233
|
});
|
|
219
234
|
}
|
|
220
|
-
|
|
235
|
+
const jitter = Math.floor(Math.random() * Math.max(1, Math.floor(waitMs / 2)));
|
|
236
|
+
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, waitMs + jitter);
|
|
237
|
+
waitMs = Math.min(FILE_MUTATION_LOCK_MAX_WAIT_MS, waitMs + 5);
|
|
221
238
|
}
|
|
222
239
|
}
|
|
223
240
|
}
|
|
@@ -256,11 +273,12 @@ export function withFileMutationLock(path, mutate, options = {}) {
|
|
|
256
273
|
release();
|
|
257
274
|
}
|
|
258
275
|
}
|
|
259
|
-
export function writeTextAtomic(path, content) {
|
|
276
|
+
export function writeTextAtomic(path, content, options = {}) {
|
|
260
277
|
mkdirSync(dirname(path), { recursive: true });
|
|
261
278
|
const tempPath = `${path}.${process.pid}.${Date.now()}.${randomUUID()}.tmp`;
|
|
262
279
|
try {
|
|
263
280
|
writeFileSync(tempPath, content, "utf8");
|
|
281
|
+
options.onBeforeReplace?.();
|
|
264
282
|
renameSync(tempPath, path);
|
|
265
283
|
}
|
|
266
284
|
catch (error) {
|
|
@@ -3,13 +3,17 @@
|
|
|
3
3
|
* Zones: owned actor selection, Recipe/Trace/Control tabs, safe Run Kill confirmation
|
|
4
4
|
* Owns manual kernel navigation; evidence parsing and lifecycle mutation stay in ports.
|
|
5
5
|
*/
|
|
6
|
-
import { readFileSync } from "node:fs";
|
|
6
|
+
import { readFileSync, statSync } from "node:fs";
|
|
7
7
|
import * as path from "node:path";
|
|
8
8
|
import { matchesKey, truncateToWidth, visibleWidth, } from "@earendil-works/pi-tui";
|
|
9
9
|
import * as ActorInspector from "./inspector.js";
|
|
10
10
|
import * as ControlProjection from "./control-projection.js";
|
|
11
11
|
import * as RunsControls from "./runs-controls.js";
|
|
12
12
|
import * as RunControlDelivery from "./runs-control-delivery.js";
|
|
13
|
+
import * as RunsTrace from "./runs-trace.js";
|
|
14
|
+
import * as Limits from "./limits.js";
|
|
15
|
+
import { computeRunControlCapacity } from "./run-evidence-policy.js";
|
|
16
|
+
import * as RuntimeTriage from "./runtime-triage.js";
|
|
13
17
|
import * as TraceProjection from "./trace-projection.js";
|
|
14
18
|
const TABS = ["recipe", "trace", "control"];
|
|
15
19
|
const TRACE_SOURCES = [
|
|
@@ -21,9 +25,6 @@ const TRACE_SOURCES = [
|
|
|
21
25
|
"artifact",
|
|
22
26
|
"runtime",
|
|
23
27
|
];
|
|
24
|
-
function newestTraceItems(items) {
|
|
25
|
-
return [...items].sort((left, right) => right.ts.localeCompare(left.ts) || right.id.localeCompare(left.id));
|
|
26
|
-
}
|
|
27
28
|
export class ActorInspectorOverlay {
|
|
28
29
|
done;
|
|
29
30
|
killRun;
|
|
@@ -410,7 +411,7 @@ export class ActorInspectorOverlay {
|
|
|
410
411
|
`${this.theme.fg("muted", "Run:")} ${this.theme.fg("accent", `run:${confirmation.run}`)}`,
|
|
411
412
|
`${this.theme.fg("muted", "Current status:")} ${this.theme.fg("warning", confirmation.status)}`,
|
|
412
413
|
"",
|
|
413
|
-
this.theme.fg("text", "This
|
|
414
|
+
this.theme.fg("text", "This invokes generation-fenced runtime kill."),
|
|
414
415
|
"",
|
|
415
416
|
`${cancel} ${kill}`,
|
|
416
417
|
"",
|
|
@@ -501,8 +502,10 @@ export class ActorInspectorOverlay {
|
|
|
501
502
|
}
|
|
502
503
|
renderTraceList(run) {
|
|
503
504
|
const items = this.traceItems(run);
|
|
505
|
+
const summary = RunsTrace.summarizeRunTraceJournal(RunsTrace.readRunTraceJournal(path.join(this.stateRoot, run.run)));
|
|
506
|
+
const banner = summary.history_complete ? [] : [this.theme.fg("warning", ` ! Trace history incomplete: ${summary.compacted ? `${summary.compactions_total} compaction(s), ` : ""}${summary.dropped_event_count_exact ? summary.dropped_events : "unknown"} dropped event(s), ${summary.dropped_bytes} bytes`)];
|
|
504
507
|
if (items.length === 0)
|
|
505
|
-
return [this.theme.fg("muted", " No Trace evidence")];
|
|
508
|
+
return [...banner, this.theme.fg("muted", " No Trace evidence")];
|
|
506
509
|
if (this.selectedTraceId) {
|
|
507
510
|
const selectedIndex = items.findIndex((item) => item.id === this.selectedTraceId);
|
|
508
511
|
if (selectedIndex >= 0)
|
|
@@ -520,24 +523,24 @@ export class ActorInspectorOverlay {
|
|
|
520
523
|
this.contentStripeIndices = items
|
|
521
524
|
.slice(start, start + viewportRows)
|
|
522
525
|
.map((_item, offset) => start + offset);
|
|
523
|
-
return items.slice(start, start + viewportRows).map((item, offset) => {
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
526
|
+
return [...banner, ...items.slice(start, start + viewportRows - banner.length).map((item, offset) => {
|
|
527
|
+
const index = start + offset;
|
|
528
|
+
const detail = item.detail && typeof item.detail === "object"
|
|
529
|
+
? item.detail
|
|
530
|
+
: {};
|
|
531
|
+
const marker = item.level === "error"
|
|
532
|
+
? this.theme.fg("error", "!")
|
|
533
|
+
: detail.attention
|
|
534
|
+
? this.theme.fg("warning", "A")
|
|
535
|
+
: " ";
|
|
536
|
+
const prefix = this.focus === "list" && index === this.rowIndex
|
|
537
|
+
? this.theme.fg("accent", " ▶ ")
|
|
538
|
+
: " ";
|
|
539
|
+
const row = `${prefix}${this.theme.fg("text", `#${items.length - index}`)} ${marker} ${this.theme.fg("muted", item.source)}/${this.theme.fg(item.level === "error" ? "error" : "accent", item.kind)} ${this.theme.fg("text", item.summary)}`;
|
|
540
|
+
return this.focus === "list" && index === this.rowIndex
|
|
541
|
+
? this.theme.fg("accent", row)
|
|
542
|
+
: row;
|
|
543
|
+
})];
|
|
541
544
|
}
|
|
542
545
|
renderTraceDetail(width) {
|
|
543
546
|
if (!this.traceDetail)
|
|
@@ -553,16 +556,29 @@ export class ActorInspectorOverlay {
|
|
|
553
556
|
? RunControlDelivery.readRunControlEndpoint(stateDir, run.runInstanceId)
|
|
554
557
|
: undefined;
|
|
555
558
|
const meta = this.readRunMeta(stateDir);
|
|
559
|
+
const journal = RunsControls.readRunControlJournalFromStateDir(stateDir);
|
|
560
|
+
const capacity = computeRunControlCapacity(journal.records);
|
|
561
|
+
const stalePending = journal.records.reduce((count, control) => count + Number(Boolean(RuntimeTriage.classifyRuntimeControl({ run: run.run,
|
|
562
|
+
runInstanceId: run.runInstanceId, status: run.status }, control, Date.now()).stale)), 0);
|
|
556
563
|
return {
|
|
557
564
|
status: run.status,
|
|
558
565
|
run_instance_id: run.runInstanceId,
|
|
566
|
+
pending: capacity.pending, pending_limit: capacity.limit,
|
|
567
|
+
available: capacity.available, backpressured: capacity.backpressured,
|
|
568
|
+
journal_bytes: (() => { try {
|
|
569
|
+
return statSync(RunsControls.runControlsFile(stateDir)).size;
|
|
570
|
+
}
|
|
571
|
+
catch {
|
|
572
|
+
return 0;
|
|
573
|
+
} })(),
|
|
574
|
+
journal_limit: Limits.RUN_CONTROL_JOURNAL_MAX_BYTES,
|
|
575
|
+
stale_pending: stalePending,
|
|
576
|
+
diagnostics: journal.diagnostics.map((item) => item.line === undefined
|
|
577
|
+
? { reason: "unreadable_control_journal" } : { line: item.line, reason: "invalid_control_json" }),
|
|
559
578
|
actor_actions: Array.isArray(meta.control) ? meta.control : [],
|
|
560
579
|
runtime_actions: run.status === "running" ? ["kill"] : ["archive", "prune"],
|
|
561
580
|
endpoint,
|
|
562
|
-
recent_controls:
|
|
563
|
-
.slice(-20)
|
|
564
|
-
.reverse()
|
|
565
|
-
.map(ControlProjection.projectRunControl),
|
|
581
|
+
recent_controls: journal.records.slice(-20).reverse().map((control) => ControlProjection.projectRunControl(control)),
|
|
566
582
|
};
|
|
567
583
|
}
|
|
568
584
|
allTraceItems(run) {
|
|
@@ -572,10 +588,10 @@ export class ActorInspectorOverlay {
|
|
|
572
588
|
this.traceCache.runInstanceId === run.runInstanceId) {
|
|
573
589
|
return this.traceCache.items;
|
|
574
590
|
}
|
|
575
|
-
const items =
|
|
591
|
+
const items = this.readTrace(path.join(this.stateRoot, run.run), {
|
|
576
592
|
limit: 100,
|
|
577
593
|
source: "all",
|
|
578
|
-
})
|
|
594
|
+
});
|
|
579
595
|
this.traceCache = { run: run.run, runInstanceId: run.runInstanceId, items };
|
|
580
596
|
return items;
|
|
581
597
|
}
|
package/dist/lib/limits.d.ts
CHANGED
|
@@ -16,3 +16,13 @@ export declare const SESSION_EVIDENCE_TEXT_CHARS = 4000;
|
|
|
16
16
|
export declare const SESSION_EVIDENCE_MAX_TOOL_CALLS = 100;
|
|
17
17
|
export declare const TRACE_EVENT_MAX_BYTES: number;
|
|
18
18
|
export declare const TRACE_EVENT_MAX_READ = 200;
|
|
19
|
+
export declare const TRACE_JOURNAL_MAX_EVENTS = 2048;
|
|
20
|
+
export declare const TRACE_JOURNAL_TARGET_EVENTS = 1536;
|
|
21
|
+
export declare const TRACE_JOURNAL_MAX_BYTES: number;
|
|
22
|
+
export declare const TRACE_JOURNAL_TARGET_BYTES: number;
|
|
23
|
+
export declare const RUN_CONTROL_PENDING_LIMIT = 64;
|
|
24
|
+
export declare const RUN_CONTROL_TERMINAL_LIMIT = 128;
|
|
25
|
+
export declare const RUN_CONTROL_ERROR_MAX_BYTES: number;
|
|
26
|
+
export declare const RUN_CONTROL_JOURNAL_MAX_BYTES: number;
|
|
27
|
+
export declare const RUN_RETENTION_MAX_RECORDS = 256;
|
|
28
|
+
export declare const RUN_RETENTION_MAX_BYTES: number;
|
package/dist/lib/limits.js
CHANGED
|
@@ -16,3 +16,13 @@ export const SESSION_EVIDENCE_TEXT_CHARS = 4_000;
|
|
|
16
16
|
export const SESSION_EVIDENCE_MAX_TOOL_CALLS = 100;
|
|
17
17
|
export const TRACE_EVENT_MAX_BYTES = 64 * 1024;
|
|
18
18
|
export const TRACE_EVENT_MAX_READ = 200;
|
|
19
|
+
export const TRACE_JOURNAL_MAX_EVENTS = 2_048;
|
|
20
|
+
export const TRACE_JOURNAL_TARGET_EVENTS = 1_536;
|
|
21
|
+
export const TRACE_JOURNAL_MAX_BYTES = 4 * 1024 * 1024;
|
|
22
|
+
export const TRACE_JOURNAL_TARGET_BYTES = 3 * 1024 * 1024;
|
|
23
|
+
export const RUN_CONTROL_PENDING_LIMIT = 64;
|
|
24
|
+
export const RUN_CONTROL_TERMINAL_LIMIT = 128;
|
|
25
|
+
export const RUN_CONTROL_ERROR_MAX_BYTES = 4 * 1024;
|
|
26
|
+
export const RUN_CONTROL_JOURNAL_MAX_BYTES = 1024 * 1024;
|
|
27
|
+
export const RUN_RETENTION_MAX_RECORDS = 256;
|
|
28
|
+
export const RUN_RETENTION_MAX_BYTES = 1024 * 1024;
|
|
@@ -42,7 +42,7 @@ export interface RunSummary {
|
|
|
42
42
|
}
|
|
43
43
|
export interface RunUiObservationState {
|
|
44
44
|
attentionEventIds: Map<string, Set<string>>;
|
|
45
|
-
|
|
45
|
+
legacyEventLines: Map<string, number>;
|
|
46
46
|
frame: number;
|
|
47
47
|
observed: Map<string, RunObservedStatus>;
|
|
48
48
|
}
|
|
@@ -62,6 +62,7 @@ export interface RunUiNotificationSink {
|
|
|
62
62
|
}): void;
|
|
63
63
|
}
|
|
64
64
|
export declare function createRunUiObservationState(): RunUiObservationState;
|
|
65
|
+
export declare function primeRunAttentionState(state: RunUiObservationState, ownerId: string, stateRoot?: string): void;
|
|
65
66
|
export declare function readRunUiSnapshot(state: RunUiObservationState, ownerId: string, options?: {
|
|
66
67
|
includeAttention?: boolean;
|
|
67
68
|
stateRoot?: string;
|
|
@@ -74,6 +75,7 @@ export declare function reconcileRunTerminalNotifications(input: {
|
|
|
74
75
|
sink: RunUiNotificationSink;
|
|
75
76
|
state: RunUiObservationState;
|
|
76
77
|
stateRoot?: string;
|
|
78
|
+
includeAttention?: boolean;
|
|
77
79
|
}): RunUiSnapshot;
|
|
78
80
|
export declare function deliverRunAttentionNotifications(events: RunAttentionEvent[], sink: RunUiNotificationSink): void;
|
|
79
81
|
export interface RunRetirementCandidate {
|
|
@@ -173,7 +175,7 @@ export declare function findRunRetirementCandidates(summary: RunSummary): RunRet
|
|
|
173
175
|
export declare function executeRunRetirements(summary: RunSummary, options: RunRetirementExecutorOptions): Promise<RunRetirementExecution[]>;
|
|
174
176
|
export declare function detectRunTransitions(previous: Map<string, RunObservedStatus>, summary: RunSummary): RunTransition[];
|
|
175
177
|
export declare function pruneRunObservationState(previousStatuses: Map<string, RunObservedStatus>, previousLineCounts: Map<string, number>, summary: RunSummary, terminalRuns?: Iterable<string>, seenEventIds?: Map<string, Set<string>>): void;
|
|
176
|
-
export declare function detectRunAttentionEvents(
|
|
178
|
+
export declare function detectRunAttentionEvents(legacyLineCounts: Map<string, number>, summary: RunSummary, seenEventIds?: Map<string, Set<string>>, prime?: boolean): RunAttentionEvent[];
|
|
177
179
|
export declare function getRunAttentionNotificationType(event: RunAttentionEvent): RunTransitionNotificationType;
|
|
178
180
|
export declare function shouldNotifyRunAttentionEvent(event: RunAttentionEvent): boolean;
|
|
179
181
|
export declare function shouldSendRunAttentionFollowUp(event: RunAttentionEvent): boolean;
|
|
@@ -7,29 +7,33 @@ import { closeSync, existsSync, fstatSync, openSync, readdirSync, readFileSync,
|
|
|
7
7
|
import { basename, dirname, isAbsolute, join, relative, } from "node:path";
|
|
8
8
|
import * as AsyncRuns from "./async-runs.js";
|
|
9
9
|
import * as Paths from "./paths.js";
|
|
10
|
+
import * as RunsTrace from "./runs-trace.js";
|
|
10
11
|
import { readJsonlFileResilient } from "./state-readers.js";
|
|
11
12
|
export function createRunUiObservationState() {
|
|
12
13
|
return {
|
|
13
14
|
attentionEventIds: new Map(),
|
|
14
|
-
|
|
15
|
+
legacyEventLines: new Map(),
|
|
15
16
|
frame: 0,
|
|
16
17
|
observed: new Map(),
|
|
17
18
|
};
|
|
18
19
|
}
|
|
20
|
+
export function primeRunAttentionState(state, ownerId, stateRoot) {
|
|
21
|
+
detectRunAttentionEvents(state.legacyEventLines, summarizeRuns(stateRoot, ownerId), state.attentionEventIds, true);
|
|
22
|
+
}
|
|
19
23
|
export function readRunUiSnapshot(state, ownerId, options = {}) {
|
|
20
24
|
const summary = summarizeRuns(options.stateRoot, ownerId);
|
|
21
25
|
const status = renderRunStatus(summary, state.frame++);
|
|
22
26
|
return {
|
|
23
27
|
attentionEvents: options.includeAttention === false
|
|
24
28
|
? []
|
|
25
|
-
: detectRunAttentionEvents(state.
|
|
29
|
+
: detectRunAttentionEvents(state.legacyEventLines, summary, state.attentionEventIds),
|
|
26
30
|
status,
|
|
27
31
|
summary,
|
|
28
32
|
transitions: detectRunTransitions(state.observed, summary),
|
|
29
33
|
};
|
|
30
34
|
}
|
|
31
35
|
export function pruneRunUiObservationState(state, snapshot) {
|
|
32
|
-
pruneRunObservationState(state.observed, state.
|
|
36
|
+
pruneRunObservationState(state.observed, state.legacyEventLines, snapshot.summary, snapshot.transitions.map((transition) => transition.stateDir ?? transition.run), state.attentionEventIds);
|
|
33
37
|
}
|
|
34
38
|
export function deliverRunTransitionNotifications(transitions, sink, inFlight = new Set()) {
|
|
35
39
|
for (const transition of transitions) {
|
|
@@ -68,9 +72,11 @@ export function deliverRunTransitionNotifications(transitions, sink, inFlight =
|
|
|
68
72
|
}
|
|
69
73
|
export function reconcileRunTerminalNotifications(input) {
|
|
70
74
|
const snapshot = readRunUiSnapshot(input.state, input.ownerId, {
|
|
71
|
-
includeAttention:
|
|
75
|
+
includeAttention: input.includeAttention,
|
|
72
76
|
stateRoot: input.stateRoot,
|
|
73
77
|
});
|
|
78
|
+
if (input.includeAttention)
|
|
79
|
+
deliverRunAttentionNotifications(snapshot.attentionEvents, input.sink);
|
|
74
80
|
deliverRunTransitionNotifications(snapshot.transitions, input.sink, input.inFlight);
|
|
75
81
|
pruneRunUiObservationState(input.state, snapshot);
|
|
76
82
|
return snapshot;
|
|
@@ -769,12 +775,13 @@ function parseAttentionRecord(raw, run, index) {
|
|
|
769
775
|
}
|
|
770
776
|
function readTraceAttentionRecords(run) {
|
|
771
777
|
if (!run.stateDir)
|
|
772
|
-
return [];
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
+
return { canonical: true, records: [] };
|
|
779
|
+
if (existsSync(join(run.stateDir, "trace.jsonl")))
|
|
780
|
+
return {
|
|
781
|
+
canonical: true,
|
|
782
|
+
records: RunsTrace.readRunTraceJournal(run.stateDir).events.map(({ event }) => event),
|
|
783
|
+
};
|
|
784
|
+
return { canonical: false, records: readJsonlFileResilient(join(run.stateDir, "outbox.jsonl")).records };
|
|
778
785
|
}
|
|
779
786
|
export function pruneRunObservationState(previousStatuses, previousLineCounts, summary, terminalRuns = [], seenEventIds = new Map()) {
|
|
780
787
|
const activeRuns = new Set(summary.runs.map((run) => runObservationKey(run)));
|
|
@@ -785,43 +792,43 @@ export function pruneRunObservationState(previousStatuses, previousLineCounts, s
|
|
|
785
792
|
const activeLineKeys = new Set(summary.runs.map((run) => run.stateDir ?? run.run));
|
|
786
793
|
for (const run of terminalRunSet)
|
|
787
794
|
previousStatuses.delete(run);
|
|
788
|
-
for (const run of previousStatuses.keys())
|
|
795
|
+
for (const run of previousStatuses.keys())
|
|
789
796
|
if (!activeRuns.has(run))
|
|
790
797
|
previousStatuses.delete(run);
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
if (terminalLineKeys.has(key) || !activeLineKeys.has(key)) {
|
|
798
|
+
for (const key of previousLineCounts.keys())
|
|
799
|
+
if (terminalLineKeys.has(key) || !activeLineKeys.has(key))
|
|
794
800
|
previousLineCounts.delete(key);
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
for (const key of seenEventIds.keys()) {
|
|
798
|
-
if (terminalLineKeys.has(key) || !activeLineKeys.has(key)) {
|
|
801
|
+
for (const key of seenEventIds.keys())
|
|
802
|
+
if (terminalLineKeys.has(key) || !activeLineKeys.has(key))
|
|
799
803
|
seenEventIds.delete(key);
|
|
800
|
-
}
|
|
801
|
-
}
|
|
802
804
|
}
|
|
803
|
-
export function detectRunAttentionEvents(
|
|
805
|
+
export function detectRunAttentionEvents(legacyLineCounts, summary, seenEventIds = new Map(), prime = false) {
|
|
804
806
|
const events = [];
|
|
805
807
|
for (const run of summary.runs) {
|
|
806
808
|
const key = run.stateDir ?? run.run;
|
|
807
|
-
const
|
|
808
|
-
const
|
|
809
|
-
if (run.notificationPolicy === "silent") {
|
|
810
|
-
previousLineCounts.set(key, records.length);
|
|
811
|
-
seenEventIds.set(key, seenEventIds.get(key) ?? new Set());
|
|
812
|
-
continue;
|
|
813
|
-
}
|
|
814
|
-
const start = Math.min(previousCount, records.length);
|
|
809
|
+
const read = readTraceAttentionRecords(run);
|
|
810
|
+
const retained = new Set();
|
|
815
811
|
const seen = seenEventIds.get(key) ?? new Set();
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
812
|
+
const start = read.canonical ? 0
|
|
813
|
+
: Math.min(legacyLineCounts.get(key) ?? 0, read.records.length);
|
|
814
|
+
for (const [index, record] of read.records.entries()) {
|
|
815
|
+
const event = parseAttentionRecord(record, run, index);
|
|
816
|
+
if (!event || !shouldNotifyRunAttentionEvent(event) ||
|
|
817
|
+
event.kind === "runtime.trace_compacted")
|
|
819
818
|
continue;
|
|
820
|
-
|
|
821
|
-
|
|
819
|
+
retained.add(event.id);
|
|
820
|
+
if (prime || run.notificationPolicy === "silent")
|
|
821
|
+
seen.add(event.id);
|
|
822
|
+
else if (index >= start && !seen.has(event.id)) {
|
|
823
|
+
events.push(event);
|
|
824
|
+
seen.add(event.id);
|
|
825
|
+
}
|
|
822
826
|
}
|
|
823
|
-
|
|
824
|
-
|
|
827
|
+
seenEventIds.set(key, new Set([...retained].filter((id) => seen.has(id))));
|
|
828
|
+
if (read.canonical)
|
|
829
|
+
legacyLineCounts.delete(key);
|
|
830
|
+
else
|
|
831
|
+
legacyLineCounts.set(key, read.records.length);
|
|
825
832
|
}
|
|
826
833
|
return events;
|
|
827
834
|
}
|