@llblab/pi-actors 0.43.1 → 0.44.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +6 -6
- package/BACKLOG.md +1 -189
- package/CHANGELOG.md +412 -402
- package/README.md +5 -5
- package/dist/lib/async-runs.js +8 -30
- package/dist/lib/file-state.d.ts +3 -1
- package/dist/lib/file-state.js +25 -7
- package/dist/lib/inspector-overlay.js +46 -30
- package/dist/lib/limits.d.ts +10 -0
- package/dist/lib/limits.js +10 -0
- package/dist/lib/observability.d.ts +4 -2
- package/dist/lib/observability.js +43 -36
- package/dist/lib/run-evidence-policy.d.ts +95 -0
- package/dist/lib/run-evidence-policy.js +177 -0
- package/dist/lib/run-ui-runtime.js +2 -0
- package/dist/lib/runs-controls.d.ts +7 -5
- package/dist/lib/runs-controls.js +186 -49
- package/dist/lib/runs-retention.js +27 -14
- package/dist/lib/runs-trace.d.ts +25 -1
- package/dist/lib/runs-trace.js +410 -21
- package/dist/lib/runtime-triage.js +6 -22
- package/dist/lib/tools-inspect.js +48 -14
- package/dist/lib/trace-projection.js +90 -44
- package/dist/scripts/conformance.mjs +5 -0
- package/dist/scripts/locker.mjs +31 -74
- package/dist/scripts/music-player.mjs +41 -129
- package/dist/scripts/release-gates.mjs +26 -4
- package/dist/skills/actors/SKILL.md +7 -6
- package/dist/skills/swarm/SKILL.md +2 -2
- package/docs/README.md +0 -1
- package/docs/actor-inspector.md +3 -2
- package/docs/async-runs.md +10 -8
- package/docs/recipe-library.md +2 -2
- package/lib/async-runs.ts +8 -45
- package/lib/file-state.ts +28 -7
- package/lib/inspector-overlay.ts +29 -17
- package/lib/limits.ts +10 -0
- package/lib/observability.ts +55 -57
- package/lib/run-evidence-policy.ts +242 -0
- package/lib/run-ui-runtime.ts +2 -0
- package/lib/runs-controls.ts +177 -108
- package/lib/runs-retention.ts +28 -20
- package/lib/runs-trace.ts +496 -21
- package/lib/runtime-triage.ts +11 -25
- package/lib/tools-inspect.ts +45 -15
- package/lib/trace-projection.ts +137 -75
- package/package.json +1 -1
- package/scripts/conformance.mjs +5 -0
- package/scripts/locker.mjs +31 -74
- package/scripts/music-player.mjs +41 -129
- package/scripts/release-gates.mjs +26 -4
- package/skills/actors/SKILL.md +7 -6
- package/skills/swarm/SKILL.md +2 -2
- package/dist/lib/runtime-notifier.d.ts +0 -48
- package/dist/lib/runtime-notifier.js +0 -138
- package/docs/0.43-baseline.md +0 -39
- package/lib/runtime-notifier.ts +0 -211
package/lib/inspector-overlay.ts
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* Owns manual kernel navigation; evidence parsing and lifecycle mutation stay in ports.
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
|
-
import { readFileSync } from "node:fs";
|
|
7
|
+
import { readFileSync, statSync } from "node:fs";
|
|
8
8
|
import * as path from "node:path";
|
|
9
9
|
|
|
10
10
|
import type { Theme } from "@earendil-works/pi-coding-agent";
|
|
@@ -19,6 +19,10 @@ import * as ActorInspector from "./inspector.ts";
|
|
|
19
19
|
import * as ControlProjection from "./control-projection.ts";
|
|
20
20
|
import * as RunsControls from "./runs-controls.ts";
|
|
21
21
|
import * as RunControlDelivery from "./runs-control-delivery.ts";
|
|
22
|
+
import * as RunsTrace from "./runs-trace.ts";
|
|
23
|
+
import * as Limits from "./limits.ts";
|
|
24
|
+
import { computeRunControlCapacity } from "./run-evidence-policy.ts";
|
|
25
|
+
import * as RuntimeTriage from "./runtime-triage.ts";
|
|
22
26
|
import * as TraceProjection from "./trace-projection.ts";
|
|
23
27
|
|
|
24
28
|
export type ActorInspectorTab = "recipe" | "trace" | "control";
|
|
@@ -59,12 +63,6 @@ const TRACE_SOURCES: TraceProjection.TraceSourceFilter[] = [
|
|
|
59
63
|
"runtime",
|
|
60
64
|
];
|
|
61
65
|
|
|
62
|
-
function newestTraceItems(items: readonly TraceProjection.TraceItem[]): TraceProjection.TraceItem[] {
|
|
63
|
-
return [...items].sort((left, right) =>
|
|
64
|
-
right.ts.localeCompare(left.ts) || right.id.localeCompare(left.id),
|
|
65
|
-
);
|
|
66
|
-
}
|
|
67
|
-
|
|
68
66
|
export class ActorInspectorOverlay {
|
|
69
67
|
private readonly done: () => void;
|
|
70
68
|
private readonly killRun?: (run: string, runInstanceId: string) => ActorInspectorActionResult;
|
|
@@ -442,7 +440,7 @@ export class ActorInspectorOverlay {
|
|
|
442
440
|
`${this.theme.fg("muted", "Run:")} ${this.theme.fg("accent", `run:${confirmation.run}`)}`,
|
|
443
441
|
`${this.theme.fg("muted", "Current status:")} ${this.theme.fg("warning", confirmation.status)}`,
|
|
444
442
|
"",
|
|
445
|
-
this.theme.fg("text", "This
|
|
443
|
+
this.theme.fg("text", "This invokes generation-fenced runtime kill."),
|
|
446
444
|
"",
|
|
447
445
|
`${cancel} ${kill}`,
|
|
448
446
|
"",
|
|
@@ -544,7 +542,11 @@ export class ActorInspectorOverlay {
|
|
|
544
542
|
|
|
545
543
|
private renderTraceList(run: ActorInspector.ActorInspectorRunItem): string[] {
|
|
546
544
|
const items = this.traceItems(run);
|
|
547
|
-
|
|
545
|
+
const summary = RunsTrace.summarizeRunTraceJournal(
|
|
546
|
+
RunsTrace.readRunTraceJournal(path.join(this.stateRoot, run.run)));
|
|
547
|
+
const banner = summary.history_complete ? [] : [this.theme.fg("warning",
|
|
548
|
+
` ! Trace history incomplete: ${summary.compacted ? `${summary.compactions_total} compaction(s), ` : ""}${summary.dropped_event_count_exact ? summary.dropped_events : "unknown"} dropped event(s), ${summary.dropped_bytes} bytes`)];
|
|
549
|
+
if (items.length === 0) return [...banner, this.theme.fg("muted", " No Trace evidence")];
|
|
548
550
|
if (this.selectedTraceId) {
|
|
549
551
|
const selectedIndex = items.findIndex((item) => item.id === this.selectedTraceId);
|
|
550
552
|
if (selectedIndex >= 0) this.rowIndex = selectedIndex;
|
|
@@ -560,7 +562,7 @@ export class ActorInspectorOverlay {
|
|
|
560
562
|
this.contentStripeIndices = items
|
|
561
563
|
.slice(start, start + viewportRows)
|
|
562
564
|
.map((_item, offset) => start + offset);
|
|
563
|
-
return items.slice(start, start + viewportRows).map((item, offset) => {
|
|
565
|
+
return [...banner, ...items.slice(start, start + viewportRows - banner.length).map((item, offset) => {
|
|
564
566
|
const index = start + offset;
|
|
565
567
|
const detail = item.detail && typeof item.detail === "object"
|
|
566
568
|
? item.detail as Record<string, unknown>
|
|
@@ -577,7 +579,7 @@ export class ActorInspectorOverlay {
|
|
|
577
579
|
return this.focus === "list" && index === this.rowIndex
|
|
578
580
|
? this.theme.fg("accent", row)
|
|
579
581
|
: row;
|
|
580
|
-
});
|
|
582
|
+
})];
|
|
581
583
|
}
|
|
582
584
|
|
|
583
585
|
private renderTraceDetail(width: number): string[] {
|
|
@@ -603,16 +605,26 @@ export class ActorInspectorOverlay {
|
|
|
603
605
|
? RunControlDelivery.readRunControlEndpoint(stateDir, run.runInstanceId)
|
|
604
606
|
: undefined;
|
|
605
607
|
const meta = this.readRunMeta(stateDir);
|
|
608
|
+
const journal = RunsControls.readRunControlJournalFromStateDir(stateDir);
|
|
609
|
+
const capacity = computeRunControlCapacity(journal.records);
|
|
610
|
+
const stalePending = journal.records.reduce<number>((count, control) => count + Number(Boolean(
|
|
611
|
+
RuntimeTriage.classifyRuntimeControl({ run: run.run,
|
|
612
|
+
runInstanceId: run.runInstanceId, status: run.status }, control, Date.now()).stale)), 0);
|
|
606
613
|
return {
|
|
607
614
|
status: run.status,
|
|
608
615
|
run_instance_id: run.runInstanceId,
|
|
616
|
+
pending: capacity.pending, pending_limit: capacity.limit,
|
|
617
|
+
available: capacity.available, backpressured: capacity.backpressured,
|
|
618
|
+
journal_bytes: (() => { try { return statSync(RunsControls.runControlsFile(stateDir)).size; } catch { return 0; } })(),
|
|
619
|
+
journal_limit: Limits.RUN_CONTROL_JOURNAL_MAX_BYTES,
|
|
620
|
+
stale_pending: stalePending,
|
|
621
|
+
diagnostics: journal.diagnostics.map((item) => item.line === undefined
|
|
622
|
+
? { reason: "unreadable_control_journal" } : { line: item.line, reason: "invalid_control_json" }),
|
|
609
623
|
actor_actions: Array.isArray(meta.control) ? meta.control : [],
|
|
610
624
|
runtime_actions: run.status === "running" ? ["kill"] : ["archive", "prune"],
|
|
611
625
|
endpoint,
|
|
612
|
-
recent_controls:
|
|
613
|
-
.
|
|
614
|
-
.reverse()
|
|
615
|
-
.map(ControlProjection.projectRunControl),
|
|
626
|
+
recent_controls: journal.records.slice(-20).reverse().map((control) =>
|
|
627
|
+
ControlProjection.projectRunControl(control as RunsControls.RunControlRecord)),
|
|
616
628
|
};
|
|
617
629
|
}
|
|
618
630
|
|
|
@@ -624,10 +636,10 @@ export class ActorInspectorOverlay {
|
|
|
624
636
|
) {
|
|
625
637
|
return this.traceCache.items;
|
|
626
638
|
}
|
|
627
|
-
const items =
|
|
639
|
+
const items = this.readTrace(path.join(this.stateRoot, run.run), {
|
|
628
640
|
limit: 100,
|
|
629
641
|
source: "all",
|
|
630
|
-
})
|
|
642
|
+
});
|
|
631
643
|
this.traceCache = { run: run.run, runInstanceId: run.runInstanceId, items };
|
|
632
644
|
return items;
|
|
633
645
|
}
|
package/lib/limits.ts
CHANGED
|
@@ -17,3 +17,13 @@ export const SESSION_EVIDENCE_TEXT_CHARS = 4_000;
|
|
|
17
17
|
export const SESSION_EVIDENCE_MAX_TOOL_CALLS = 100;
|
|
18
18
|
export const TRACE_EVENT_MAX_BYTES = 64 * 1024;
|
|
19
19
|
export const TRACE_EVENT_MAX_READ = 200;
|
|
20
|
+
export const TRACE_JOURNAL_MAX_EVENTS = 2_048;
|
|
21
|
+
export const TRACE_JOURNAL_TARGET_EVENTS = 1_536;
|
|
22
|
+
export const TRACE_JOURNAL_MAX_BYTES = 4 * 1024 * 1024;
|
|
23
|
+
export const TRACE_JOURNAL_TARGET_BYTES = 3 * 1024 * 1024;
|
|
24
|
+
export const RUN_CONTROL_PENDING_LIMIT = 64;
|
|
25
|
+
export const RUN_CONTROL_TERMINAL_LIMIT = 128;
|
|
26
|
+
export const RUN_CONTROL_ERROR_MAX_BYTES = 4 * 1024;
|
|
27
|
+
export const RUN_CONTROL_JOURNAL_MAX_BYTES = 1024 * 1024;
|
|
28
|
+
export const RUN_RETENTION_MAX_RECORDS = 256;
|
|
29
|
+
export const RUN_RETENTION_MAX_BYTES = 1024 * 1024;
|
package/lib/observability.ts
CHANGED
|
@@ -25,6 +25,7 @@ import {
|
|
|
25
25
|
|
|
26
26
|
import * as AsyncRuns from "./async-runs.ts";
|
|
27
27
|
import * as Paths from "./paths.ts";
|
|
28
|
+
import * as RunsTrace from "./runs-trace.ts";
|
|
28
29
|
import { readJsonlFileResilient } from "./state-readers.ts";
|
|
29
30
|
|
|
30
31
|
export type RunObservedStatus =
|
|
@@ -73,7 +74,7 @@ export interface RunSummary {
|
|
|
73
74
|
|
|
74
75
|
export interface RunUiObservationState {
|
|
75
76
|
attentionEventIds: Map<string, Set<string>>;
|
|
76
|
-
|
|
77
|
+
legacyEventLines: Map<string, number>;
|
|
77
78
|
frame: number;
|
|
78
79
|
observed: Map<string, RunObservedStatus>;
|
|
79
80
|
}
|
|
@@ -98,12 +99,19 @@ export interface RunUiNotificationSink {
|
|
|
98
99
|
export function createRunUiObservationState(): RunUiObservationState {
|
|
99
100
|
return {
|
|
100
101
|
attentionEventIds: new Map<string, Set<string>>(),
|
|
101
|
-
|
|
102
|
+
legacyEventLines: new Map<string, number>(),
|
|
102
103
|
frame: 0,
|
|
103
104
|
observed: new Map<string, RunObservedStatus>(),
|
|
104
105
|
};
|
|
105
106
|
}
|
|
106
107
|
|
|
108
|
+
export function primeRunAttentionState(
|
|
109
|
+
state: RunUiObservationState, ownerId: string, stateRoot?: string,
|
|
110
|
+
): void {
|
|
111
|
+
detectRunAttentionEvents(state.legacyEventLines, summarizeRuns(stateRoot, ownerId),
|
|
112
|
+
state.attentionEventIds, true);
|
|
113
|
+
}
|
|
114
|
+
|
|
107
115
|
export function readRunUiSnapshot(
|
|
108
116
|
state: RunUiObservationState,
|
|
109
117
|
ownerId: string,
|
|
@@ -116,7 +124,7 @@ export function readRunUiSnapshot(
|
|
|
116
124
|
options.includeAttention === false
|
|
117
125
|
? []
|
|
118
126
|
: detectRunAttentionEvents(
|
|
119
|
-
state.
|
|
127
|
+
state.legacyEventLines,
|
|
120
128
|
summary,
|
|
121
129
|
state.attentionEventIds,
|
|
122
130
|
),
|
|
@@ -132,7 +140,7 @@ export function pruneRunUiObservationState(
|
|
|
132
140
|
): void {
|
|
133
141
|
pruneRunObservationState(
|
|
134
142
|
state.observed,
|
|
135
|
-
state.
|
|
143
|
+
state.legacyEventLines,
|
|
136
144
|
snapshot.summary,
|
|
137
145
|
snapshot.transitions.map(
|
|
138
146
|
(transition) => transition.stateDir ?? transition.run,
|
|
@@ -192,16 +200,15 @@ export function reconcileRunTerminalNotifications(input: {
|
|
|
192
200
|
sink: RunUiNotificationSink;
|
|
193
201
|
state: RunUiObservationState;
|
|
194
202
|
stateRoot?: string;
|
|
203
|
+
includeAttention?: boolean;
|
|
195
204
|
}): RunUiSnapshot {
|
|
196
205
|
const snapshot = readRunUiSnapshot(input.state, input.ownerId, {
|
|
197
|
-
includeAttention:
|
|
206
|
+
includeAttention: input.includeAttention,
|
|
198
207
|
stateRoot: input.stateRoot,
|
|
199
208
|
});
|
|
200
|
-
|
|
201
|
-
snapshot.
|
|
202
|
-
|
|
203
|
-
input.inFlight,
|
|
204
|
-
);
|
|
209
|
+
if (input.includeAttention)
|
|
210
|
+
deliverRunAttentionNotifications(snapshot.attentionEvents, input.sink);
|
|
211
|
+
deliverRunTransitionNotifications(snapshot.transitions, input.sink, input.inFlight);
|
|
205
212
|
pruneRunUiObservationState(input.state, snapshot);
|
|
206
213
|
return snapshot;
|
|
207
214
|
}
|
|
@@ -1114,15 +1121,17 @@ function parseAttentionRecord(
|
|
|
1114
1121
|
};
|
|
1115
1122
|
}
|
|
1116
1123
|
|
|
1117
|
-
function readTraceAttentionRecords(run: RunObservation):
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
if (
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1124
|
+
function readTraceAttentionRecords(run: RunObservation): {
|
|
1125
|
+
canonical: boolean; records: Record<string, unknown>[];
|
|
1126
|
+
} {
|
|
1127
|
+
if (!run.stateDir) return { canonical: true, records: [] };
|
|
1128
|
+
if (existsSync(join(run.stateDir, "trace.jsonl"))) return {
|
|
1129
|
+
canonical: true,
|
|
1130
|
+
records: RunsTrace.readRunTraceJournal(run.stateDir).events.map(
|
|
1131
|
+
({ event }) => event as unknown as Record<string, unknown>),
|
|
1132
|
+
};
|
|
1133
|
+
return { canonical: false, records: readJsonlFileResilient<Record<string, unknown>>(
|
|
1134
|
+
join(run.stateDir, "outbox.jsonl")).records };
|
|
1126
1135
|
}
|
|
1127
1136
|
|
|
1128
1137
|
export function pruneRunObservationState(
|
|
@@ -1134,55 +1143,44 @@ export function pruneRunObservationState(
|
|
|
1134
1143
|
): void {
|
|
1135
1144
|
const activeRuns = new Set(summary.runs.map((run) => runObservationKey(run)));
|
|
1136
1145
|
const terminalRunSet = new Set(terminalRuns);
|
|
1137
|
-
const terminalLineKeys = new Set(
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
);
|
|
1142
|
-
const activeLineKeys = new Set(
|
|
1143
|
-
summary.runs.map((run) => run.stateDir ?? run.run),
|
|
1144
|
-
);
|
|
1146
|
+
const terminalLineKeys = new Set(summary.runs
|
|
1147
|
+
.filter((run) => terminalRunSet.has(runObservationKey(run)))
|
|
1148
|
+
.map((run) => runObservationKey(run)));
|
|
1149
|
+
const activeLineKeys = new Set(summary.runs.map((run) => run.stateDir ?? run.run));
|
|
1145
1150
|
for (const run of terminalRunSet) previousStatuses.delete(run);
|
|
1146
|
-
for (const run of previousStatuses.keys())
|
|
1151
|
+
for (const run of previousStatuses.keys())
|
|
1147
1152
|
if (!activeRuns.has(run)) previousStatuses.delete(run);
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
}
|
|
1153
|
-
}
|
|
1154
|
-
for (const key of seenEventIds.keys()) {
|
|
1155
|
-
if (terminalLineKeys.has(key) || !activeLineKeys.has(key)) {
|
|
1156
|
-
seenEventIds.delete(key);
|
|
1157
|
-
}
|
|
1158
|
-
}
|
|
1153
|
+
for (const key of previousLineCounts.keys())
|
|
1154
|
+
if (terminalLineKeys.has(key) || !activeLineKeys.has(key)) previousLineCounts.delete(key);
|
|
1155
|
+
for (const key of seenEventIds.keys())
|
|
1156
|
+
if (terminalLineKeys.has(key) || !activeLineKeys.has(key)) seenEventIds.delete(key);
|
|
1159
1157
|
}
|
|
1160
1158
|
|
|
1161
1159
|
export function detectRunAttentionEvents(
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
seenEventIds: Map<string, Set<string>> = new Map(),
|
|
1160
|
+
legacyLineCounts: Map<string, number>, summary: RunSummary,
|
|
1161
|
+
seenEventIds: Map<string, Set<string>> = new Map(), prime = false,
|
|
1165
1162
|
): RunAttentionEvent[] {
|
|
1166
1163
|
const events: RunAttentionEvent[] = [];
|
|
1167
1164
|
for (const run of summary.runs) {
|
|
1168
1165
|
const key = run.stateDir ?? run.run;
|
|
1169
|
-
const
|
|
1170
|
-
const
|
|
1171
|
-
if (run.notificationPolicy === "silent") {
|
|
1172
|
-
previousLineCounts.set(key, records.length);
|
|
1173
|
-
seenEventIds.set(key, seenEventIds.get(key) ?? new Set<string>());
|
|
1174
|
-
continue;
|
|
1175
|
-
}
|
|
1176
|
-
const start = Math.min(previousCount, records.length);
|
|
1166
|
+
const read = readTraceAttentionRecords(run);
|
|
1167
|
+
const retained = new Set<string>();
|
|
1177
1168
|
const seen = seenEventIds.get(key) ?? new Set<string>();
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1169
|
+
const start = read.canonical ? 0
|
|
1170
|
+
: Math.min(legacyLineCounts.get(key) ?? 0, read.records.length);
|
|
1171
|
+
for (const [index, record] of read.records.entries()) {
|
|
1172
|
+
const event = parseAttentionRecord(record, run, index);
|
|
1173
|
+
if (!event || !shouldNotifyRunAttentionEvent(event) ||
|
|
1174
|
+
event.kind === "runtime.trace_compacted") continue;
|
|
1175
|
+
retained.add(event.id);
|
|
1176
|
+
if (prime || run.notificationPolicy === "silent") seen.add(event.id);
|
|
1177
|
+
else if (index >= start && !seen.has(event.id)) {
|
|
1178
|
+
events.push(event); seen.add(event.id);
|
|
1179
|
+
}
|
|
1183
1180
|
}
|
|
1184
|
-
|
|
1185
|
-
|
|
1181
|
+
seenEventIds.set(key, new Set([...retained].filter((id) => seen.has(id))));
|
|
1182
|
+
if (read.canonical) legacyLineCounts.delete(key);
|
|
1183
|
+
else legacyLineCounts.set(key, read.records.length);
|
|
1186
1184
|
}
|
|
1187
1185
|
return events;
|
|
1188
1186
|
}
|
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pure bounded Run-evidence policy.
|
|
3
|
+
* Zones: Trace retention, compaction accounting, Control capacity and admission
|
|
4
|
+
* Owns deterministic limits policy; journal I/O, locks, lifecycle authorization, and projection stay in adapters.
|
|
5
|
+
* Trace keeps the newest physical suffix because semantic priority would silently
|
|
6
|
+
* redefine causal history. Admitted Controls never expire: only a real terminal
|
|
7
|
+
* transition or generation replacement proves that durable work is finished.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import * as Limits from "./limits.ts";
|
|
11
|
+
|
|
12
|
+
export const TRACE_COMPACTION_KIND = "runtime.trace_compacted";
|
|
13
|
+
export const TRACE_COMPACTION_VERSION = 1;
|
|
14
|
+
export type TraceJournalUsage = { bytes: number; events: number };
|
|
15
|
+
export type TraceRetentionCandidate<T> = { encodedBytes: number; value: T };
|
|
16
|
+
export type TraceRetentionSelection<T> = {
|
|
17
|
+
droppedEvents: number; retained: TraceRetentionCandidate<T>[];
|
|
18
|
+
retainedBytes: number; retainedEvents: number;
|
|
19
|
+
};
|
|
20
|
+
export interface TraceCompactionData {
|
|
21
|
+
version: typeof TRACE_COMPACTION_VERSION;
|
|
22
|
+
compactions_total: number;
|
|
23
|
+
dropped_valid_events_total: number;
|
|
24
|
+
dropped_malformed_lines_total: number;
|
|
25
|
+
dropped_bytes_total: number;
|
|
26
|
+
dropped_event_count_exact: boolean;
|
|
27
|
+
retained_events: number;
|
|
28
|
+
retained_bytes: number;
|
|
29
|
+
history_complete: false;
|
|
30
|
+
}
|
|
31
|
+
export interface TraceCompactionDelta {
|
|
32
|
+
dropped_valid_events: number;
|
|
33
|
+
dropped_malformed_lines: number;
|
|
34
|
+
dropped_bytes: number;
|
|
35
|
+
dropped_event_count_exact: boolean;
|
|
36
|
+
retained_events: number;
|
|
37
|
+
retained_bytes: number;
|
|
38
|
+
}
|
|
39
|
+
export type RunControlStatus = "queued" | "delivered" | "claimed" | "handled" | "failed";
|
|
40
|
+
export type RunControlCapacity = {
|
|
41
|
+
available: number; backpressured: boolean; limit: number;
|
|
42
|
+
oldest_pending_at?: string; pending: number;
|
|
43
|
+
};
|
|
44
|
+
export type RunControlAdmissionIntegrity =
|
|
45
|
+
| "valid" | "unreadable" | "malformed" | "noncanonical"
|
|
46
|
+
| "invalid_record" | "invalid_status_timestamp" | "generation_mismatch"
|
|
47
|
+
| "journal_bytes";
|
|
48
|
+
export interface RunControlAdmissionErrorDetails {
|
|
49
|
+
reason: "control_backpressure" | "control_journal_integrity";
|
|
50
|
+
pending: number;
|
|
51
|
+
limit: number;
|
|
52
|
+
journal_bytes: number;
|
|
53
|
+
journal_limit: number;
|
|
54
|
+
oldest_pending_at?: string;
|
|
55
|
+
run_instance_id: string;
|
|
56
|
+
next_actions: string[];
|
|
57
|
+
integrity_reason?: string;
|
|
58
|
+
}
|
|
59
|
+
export type RunControlAdmissionDecision =
|
|
60
|
+
{ admitted: true; capacity: RunControlCapacity } |
|
|
61
|
+
{ admitted: false; capacity: RunControlCapacity; error: RunControlAdmissionErrorDetails };
|
|
62
|
+
export type RunControlAdmissionInput = {
|
|
63
|
+
integrity: RunControlAdmissionIntegrity; journalBytes: number;
|
|
64
|
+
newRecordBytes: number; records: readonly unknown[];
|
|
65
|
+
retainedJournalBytes: number; runInstanceId: string;
|
|
66
|
+
capacity?: RunControlCapacity;
|
|
67
|
+
};
|
|
68
|
+
|
|
69
|
+
const PENDING_CONTROL_STATUSES = new Set<RunControlStatus>(["queued", "delivered", "claimed"]);
|
|
70
|
+
const TERMINAL_CONTROL_STATUSES = new Set<RunControlStatus>(["handled", "failed"]);
|
|
71
|
+
const CONTROL_ADMISSION_NEXT_ACTIONS = [
|
|
72
|
+
"inspect target=run:<id> view=control",
|
|
73
|
+
"inspect target=runtime view=triage",
|
|
74
|
+
];
|
|
75
|
+
|
|
76
|
+
function isCount(value: number): boolean {
|
|
77
|
+
return Number.isSafeInteger(value) && value >= 0;
|
|
78
|
+
}
|
|
79
|
+
function assertCount(value: number, name: string): void {
|
|
80
|
+
if (!isCount(value)) throw new RangeError(`${name} must be a non-negative safe integer`);
|
|
81
|
+
}
|
|
82
|
+
function boundedAdd(left: number, right: number): number {
|
|
83
|
+
return Math.min(Number.MAX_SAFE_INTEGER, left + right);
|
|
84
|
+
}
|
|
85
|
+
export function traceAppendFits(usage: TraceJournalUsage, encodedEventBytes: number): boolean {
|
|
86
|
+
if (!isCount(usage.events) || !isCount(usage.bytes) || !isCount(encodedEventBytes)) return false;
|
|
87
|
+
return usage.events + 1 <= Limits.TRACE_JOURNAL_MAX_EVENTS &&
|
|
88
|
+
usage.bytes + encodedEventBytes <= Limits.TRACE_JOURNAL_MAX_BYTES;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
export function selectNewestTraceSuffix<T>(
|
|
92
|
+
candidates: readonly TraceRetentionCandidate<T>[],
|
|
93
|
+
reserve: { bytes?: number; events?: number } = {},
|
|
94
|
+
): TraceRetentionSelection<T> {
|
|
95
|
+
const reservedBytes = reserve.bytes ?? 0;
|
|
96
|
+
const reservedEvents = reserve.events ?? 0;
|
|
97
|
+
assertCount(reservedBytes, "reserved Trace bytes");
|
|
98
|
+
assertCount(reservedEvents, "reserved Trace events");
|
|
99
|
+
const byteBudget = Math.max(0, Limits.TRACE_JOURNAL_TARGET_BYTES - reservedBytes);
|
|
100
|
+
const eventBudget = Math.max(0, Limits.TRACE_JOURNAL_TARGET_EVENTS - reservedEvents);
|
|
101
|
+
let retainedBytes = 0;
|
|
102
|
+
let start = candidates.length;
|
|
103
|
+
while (start > 0 && candidates.length - start < eventBudget) {
|
|
104
|
+
const candidate = candidates[start - 1]!;
|
|
105
|
+
assertCount(candidate.encodedBytes, "encoded Trace bytes");
|
|
106
|
+
if (retainedBytes + candidate.encodedBytes > byteBudget) break;
|
|
107
|
+
retainedBytes += candidate.encodedBytes;
|
|
108
|
+
start -= 1;
|
|
109
|
+
}
|
|
110
|
+
const retained = candidates.slice(start);
|
|
111
|
+
return {
|
|
112
|
+
droppedEvents: candidates.length - retained.length,
|
|
113
|
+
retained,
|
|
114
|
+
retainedBytes,
|
|
115
|
+
retainedEvents: retained.length,
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
export function accumulateTraceCompactionStatistics(
|
|
120
|
+
previous: TraceCompactionData | undefined,
|
|
121
|
+
delta: TraceCompactionDelta,
|
|
122
|
+
): TraceCompactionData {
|
|
123
|
+
for (const [name, value] of Object.entries(delta)) {
|
|
124
|
+
if (name !== "dropped_event_count_exact") assertCount(value as number, name);
|
|
125
|
+
}
|
|
126
|
+
const prior = previous ?? {
|
|
127
|
+
compactions_total: 0,
|
|
128
|
+
dropped_bytes_total: 0,
|
|
129
|
+
dropped_event_count_exact: true,
|
|
130
|
+
dropped_malformed_lines_total: 0,
|
|
131
|
+
dropped_valid_events_total: 0,
|
|
132
|
+
};
|
|
133
|
+
const eventCountOverflow = prior.dropped_valid_events_total +
|
|
134
|
+
delta.dropped_valid_events > Number.MAX_SAFE_INTEGER;
|
|
135
|
+
return {
|
|
136
|
+
version: TRACE_COMPACTION_VERSION,
|
|
137
|
+
compactions_total: boundedAdd(prior.compactions_total, 1),
|
|
138
|
+
dropped_valid_events_total: boundedAdd(prior.dropped_valid_events_total, delta.dropped_valid_events),
|
|
139
|
+
dropped_malformed_lines_total: boundedAdd(prior.dropped_malformed_lines_total, delta.dropped_malformed_lines),
|
|
140
|
+
dropped_bytes_total: boundedAdd(prior.dropped_bytes_total, delta.dropped_bytes),
|
|
141
|
+
dropped_event_count_exact: prior.dropped_event_count_exact &&
|
|
142
|
+
delta.dropped_event_count_exact && !eventCountOverflow,
|
|
143
|
+
retained_events: delta.retained_events,
|
|
144
|
+
retained_bytes: delta.retained_bytes,
|
|
145
|
+
history_complete: false,
|
|
146
|
+
};
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
export function traceCompactionMarkerInput(data: TraceCompactionData): {
|
|
150
|
+
data: TraceCompactionData; kind: typeof TRACE_COMPACTION_KIND; level: "warning";
|
|
151
|
+
} {
|
|
152
|
+
return { data, kind: TRACE_COMPACTION_KIND, level: "warning" };
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
export function classifyRunControlRecord(value: unknown): "pending" | "terminal" | undefined {
|
|
156
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return undefined;
|
|
157
|
+
const status = (value as { status?: unknown }).status as RunControlStatus;
|
|
158
|
+
if (PENDING_CONTROL_STATUSES.has(status)) return "pending";
|
|
159
|
+
if (TERMINAL_CONTROL_STATUSES.has(status)) return "terminal";
|
|
160
|
+
return undefined;
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
export function runControlStatusTimestamp(
|
|
164
|
+
record: Record<string, unknown>, status: RunControlStatus,
|
|
165
|
+
): unknown {
|
|
166
|
+
if (status === "failed") return record.failed_at;
|
|
167
|
+
if (status === "handled") return record.handled_at;
|
|
168
|
+
if (status === "claimed") return record.claimed_at;
|
|
169
|
+
if (status === "delivered") return record.delivered_at;
|
|
170
|
+
return record.queued_at;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
export function computeRunControlCapacity(records: readonly unknown[]): RunControlCapacity {
|
|
174
|
+
let oldest: { timestamp: string; timestampMs: number } | undefined;
|
|
175
|
+
let pending = 0;
|
|
176
|
+
for (const value of records) {
|
|
177
|
+
if (classifyRunControlRecord(value) !== "pending") continue;
|
|
178
|
+
pending += 1;
|
|
179
|
+
const record = value as Record<string, unknown>;
|
|
180
|
+
const timestamp = runControlStatusTimestamp(record, record.status as RunControlStatus);
|
|
181
|
+
const timestampMs = typeof timestamp === "string" ? Date.parse(timestamp) : Number.NaN;
|
|
182
|
+
if (typeof timestamp === "string" && Number.isFinite(timestampMs) &&
|
|
183
|
+
(!oldest || timestampMs < oldest.timestampMs)) {
|
|
184
|
+
oldest = { timestamp, timestampMs };
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
return {
|
|
188
|
+
available: Math.max(0, Limits.RUN_CONTROL_PENDING_LIMIT - pending),
|
|
189
|
+
backpressured: pending >= Limits.RUN_CONTROL_PENDING_LIMIT,
|
|
190
|
+
limit: Limits.RUN_CONTROL_PENDING_LIMIT,
|
|
191
|
+
...(oldest ? { oldest_pending_at: oldest.timestamp } : {}),
|
|
192
|
+
pending,
|
|
193
|
+
};
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
export function decideRunControlAdmission(
|
|
197
|
+
input: RunControlAdmissionInput,
|
|
198
|
+
): RunControlAdmissionDecision {
|
|
199
|
+
const capacity = input.capacity ?? computeRunControlCapacity(input.records);
|
|
200
|
+
const reject = (
|
|
201
|
+
reason: RunControlAdmissionErrorDetails["reason"],
|
|
202
|
+
integrityReason?: string,
|
|
203
|
+
): RunControlAdmissionDecision => ({
|
|
204
|
+
admitted: false,
|
|
205
|
+
capacity,
|
|
206
|
+
error: {
|
|
207
|
+
reason,
|
|
208
|
+
pending: capacity.pending,
|
|
209
|
+
limit: capacity.limit,
|
|
210
|
+
journal_bytes: input.journalBytes,
|
|
211
|
+
journal_limit: Limits.RUN_CONTROL_JOURNAL_MAX_BYTES,
|
|
212
|
+
...(capacity.oldest_pending_at ? { oldest_pending_at: capacity.oldest_pending_at } : {}),
|
|
213
|
+
run_instance_id: input.runInstanceId,
|
|
214
|
+
next_actions: [...CONTROL_ADMISSION_NEXT_ACTIONS],
|
|
215
|
+
...(integrityReason ? { integrity_reason: integrityReason } : {}),
|
|
216
|
+
},
|
|
217
|
+
});
|
|
218
|
+
if (input.integrity !== "valid") return reject("control_journal_integrity", input.integrity);
|
|
219
|
+
if (!isCount(input.journalBytes) || !isCount(input.retainedJournalBytes) ||
|
|
220
|
+
!isCount(input.newRecordBytes) ||
|
|
221
|
+
input.journalBytes > Limits.RUN_CONTROL_JOURNAL_MAX_BYTES) {
|
|
222
|
+
return reject("control_journal_integrity", "journal_bytes");
|
|
223
|
+
}
|
|
224
|
+
for (const value of input.records) {
|
|
225
|
+
const classification = classifyRunControlRecord(value);
|
|
226
|
+
if (!classification) return reject("control_journal_integrity", "invalid_record");
|
|
227
|
+
if (classification !== "pending") continue;
|
|
228
|
+
const record = value as Record<string, unknown>;
|
|
229
|
+
const timestamp = runControlStatusTimestamp(record, record.status as RunControlStatus);
|
|
230
|
+
if (typeof timestamp !== "string" || !Number.isFinite(Date.parse(timestamp))) {
|
|
231
|
+
return reject("control_journal_integrity", "invalid_status_timestamp");
|
|
232
|
+
}
|
|
233
|
+
if (record.run_instance_id !== input.runInstanceId) {
|
|
234
|
+
return reject("control_journal_integrity", "generation_mismatch");
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
if (capacity.backpressured ||
|
|
238
|
+
input.retainedJournalBytes + input.newRecordBytes > Limits.RUN_CONTROL_JOURNAL_MAX_BYTES) {
|
|
239
|
+
return reject("control_backpressure");
|
|
240
|
+
}
|
|
241
|
+
return { admitted: true, capacity };
|
|
242
|
+
}
|
package/lib/run-ui-runtime.ts
CHANGED
|
@@ -106,6 +106,7 @@ export function createRunUiRuntime(deps: RunUiRuntimeDeps): RunUiRuntime {
|
|
|
106
106
|
ownerId: deps.getRunOwnerId(ctx),
|
|
107
107
|
sink: Pi.createNotificationSink(deps.pi, ctx),
|
|
108
108
|
state: observation,
|
|
109
|
+
includeAttention: true,
|
|
109
110
|
});
|
|
110
111
|
reportDiagnostics(ctx);
|
|
111
112
|
},
|
|
@@ -141,6 +142,7 @@ export function createRunUiRuntime(deps: RunUiRuntimeDeps): RunUiRuntime {
|
|
|
141
142
|
},
|
|
142
143
|
start(ctx) {
|
|
143
144
|
close();
|
|
145
|
+
Observability.primeRunAttentionState(observation, deps.getRunOwnerId(ctx));
|
|
144
146
|
update(ctx, true, true);
|
|
145
147
|
watcher.refresh();
|
|
146
148
|
reconciliation.start();
|