@llblab/pi-actors 0.43.1 → 0.44.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +6 -6
- package/BACKLOG.md +1 -189
- package/CHANGELOG.md +412 -402
- package/README.md +5 -5
- package/dist/lib/async-runs.js +8 -30
- package/dist/lib/file-state.d.ts +3 -1
- package/dist/lib/file-state.js +25 -7
- package/dist/lib/inspector-overlay.js +46 -30
- package/dist/lib/limits.d.ts +10 -0
- package/dist/lib/limits.js +10 -0
- package/dist/lib/observability.d.ts +4 -2
- package/dist/lib/observability.js +43 -36
- package/dist/lib/run-evidence-policy.d.ts +95 -0
- package/dist/lib/run-evidence-policy.js +177 -0
- package/dist/lib/run-ui-runtime.js +2 -0
- package/dist/lib/runs-controls.d.ts +7 -5
- package/dist/lib/runs-controls.js +186 -49
- package/dist/lib/runs-retention.js +27 -14
- package/dist/lib/runs-trace.d.ts +25 -1
- package/dist/lib/runs-trace.js +410 -21
- package/dist/lib/runtime-triage.js +6 -22
- package/dist/lib/tools-inspect.js +48 -14
- package/dist/lib/trace-projection.js +90 -44
- package/dist/scripts/conformance.mjs +5 -0
- package/dist/scripts/locker.mjs +31 -74
- package/dist/scripts/music-player.mjs +41 -129
- package/dist/scripts/release-gates.mjs +26 -4
- package/dist/skills/actors/SKILL.md +7 -6
- package/dist/skills/swarm/SKILL.md +2 -2
- package/docs/README.md +0 -1
- package/docs/actor-inspector.md +3 -2
- package/docs/async-runs.md +10 -8
- package/docs/recipe-library.md +2 -2
- package/lib/async-runs.ts +8 -45
- package/lib/file-state.ts +28 -7
- package/lib/inspector-overlay.ts +29 -17
- package/lib/limits.ts +10 -0
- package/lib/observability.ts +55 -57
- package/lib/run-evidence-policy.ts +242 -0
- package/lib/run-ui-runtime.ts +2 -0
- package/lib/runs-controls.ts +177 -108
- package/lib/runs-retention.ts +28 -20
- package/lib/runs-trace.ts +496 -21
- package/lib/runtime-triage.ts +11 -25
- package/lib/tools-inspect.ts +45 -15
- package/lib/trace-projection.ts +137 -75
- package/package.json +1 -1
- package/scripts/conformance.mjs +5 -0
- package/scripts/locker.mjs +31 -74
- package/scripts/music-player.mjs +41 -129
- package/scripts/release-gates.mjs +26 -4
- package/skills/actors/SKILL.md +7 -6
- package/skills/swarm/SKILL.md +2 -2
- package/dist/lib/runtime-notifier.d.ts +0 -48
- package/dist/lib/runtime-notifier.js +0 -138
- package/docs/0.43-baseline.md +0 -39
- package/lib/runtime-notifier.ts +0 -211
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pure bounded Run-evidence policy.
|
|
3
|
+
* Zones: Trace retention, compaction accounting, Control capacity and admission
|
|
4
|
+
* Owns deterministic limits policy; journal I/O, locks, lifecycle authorization, and projection stay in adapters.
|
|
5
|
+
* Trace keeps the newest physical suffix because semantic priority would silently
|
|
6
|
+
* redefine causal history. Admitted Controls never expire: only a real terminal
|
|
7
|
+
* transition or generation replacement proves that durable work is finished.
|
|
8
|
+
*/
|
|
9
|
+
export declare const TRACE_COMPACTION_KIND = "runtime.trace_compacted";
|
|
10
|
+
export declare const TRACE_COMPACTION_VERSION = 1;
|
|
11
|
+
export type TraceJournalUsage = {
|
|
12
|
+
bytes: number;
|
|
13
|
+
events: number;
|
|
14
|
+
};
|
|
15
|
+
export type TraceRetentionCandidate<T> = {
|
|
16
|
+
encodedBytes: number;
|
|
17
|
+
value: T;
|
|
18
|
+
};
|
|
19
|
+
export type TraceRetentionSelection<T> = {
|
|
20
|
+
droppedEvents: number;
|
|
21
|
+
retained: TraceRetentionCandidate<T>[];
|
|
22
|
+
retainedBytes: number;
|
|
23
|
+
retainedEvents: number;
|
|
24
|
+
};
|
|
25
|
+
export interface TraceCompactionData {
|
|
26
|
+
version: typeof TRACE_COMPACTION_VERSION;
|
|
27
|
+
compactions_total: number;
|
|
28
|
+
dropped_valid_events_total: number;
|
|
29
|
+
dropped_malformed_lines_total: number;
|
|
30
|
+
dropped_bytes_total: number;
|
|
31
|
+
dropped_event_count_exact: boolean;
|
|
32
|
+
retained_events: number;
|
|
33
|
+
retained_bytes: number;
|
|
34
|
+
history_complete: false;
|
|
35
|
+
}
|
|
36
|
+
export interface TraceCompactionDelta {
|
|
37
|
+
dropped_valid_events: number;
|
|
38
|
+
dropped_malformed_lines: number;
|
|
39
|
+
dropped_bytes: number;
|
|
40
|
+
dropped_event_count_exact: boolean;
|
|
41
|
+
retained_events: number;
|
|
42
|
+
retained_bytes: number;
|
|
43
|
+
}
|
|
44
|
+
export type RunControlStatus = "queued" | "delivered" | "claimed" | "handled" | "failed";
|
|
45
|
+
export type RunControlCapacity = {
|
|
46
|
+
available: number;
|
|
47
|
+
backpressured: boolean;
|
|
48
|
+
limit: number;
|
|
49
|
+
oldest_pending_at?: string;
|
|
50
|
+
pending: number;
|
|
51
|
+
};
|
|
52
|
+
export type RunControlAdmissionIntegrity = "valid" | "unreadable" | "malformed" | "noncanonical" | "invalid_record" | "invalid_status_timestamp" | "generation_mismatch" | "journal_bytes";
|
|
53
|
+
export interface RunControlAdmissionErrorDetails {
|
|
54
|
+
reason: "control_backpressure" | "control_journal_integrity";
|
|
55
|
+
pending: number;
|
|
56
|
+
limit: number;
|
|
57
|
+
journal_bytes: number;
|
|
58
|
+
journal_limit: number;
|
|
59
|
+
oldest_pending_at?: string;
|
|
60
|
+
run_instance_id: string;
|
|
61
|
+
next_actions: string[];
|
|
62
|
+
integrity_reason?: string;
|
|
63
|
+
}
|
|
64
|
+
export type RunControlAdmissionDecision = {
|
|
65
|
+
admitted: true;
|
|
66
|
+
capacity: RunControlCapacity;
|
|
67
|
+
} | {
|
|
68
|
+
admitted: false;
|
|
69
|
+
capacity: RunControlCapacity;
|
|
70
|
+
error: RunControlAdmissionErrorDetails;
|
|
71
|
+
};
|
|
72
|
+
export type RunControlAdmissionInput = {
|
|
73
|
+
integrity: RunControlAdmissionIntegrity;
|
|
74
|
+
journalBytes: number;
|
|
75
|
+
newRecordBytes: number;
|
|
76
|
+
records: readonly unknown[];
|
|
77
|
+
retainedJournalBytes: number;
|
|
78
|
+
runInstanceId: string;
|
|
79
|
+
capacity?: RunControlCapacity;
|
|
80
|
+
};
|
|
81
|
+
export declare function traceAppendFits(usage: TraceJournalUsage, encodedEventBytes: number): boolean;
|
|
82
|
+
export declare function selectNewestTraceSuffix<T>(candidates: readonly TraceRetentionCandidate<T>[], reserve?: {
|
|
83
|
+
bytes?: number;
|
|
84
|
+
events?: number;
|
|
85
|
+
}): TraceRetentionSelection<T>;
|
|
86
|
+
export declare function accumulateTraceCompactionStatistics(previous: TraceCompactionData | undefined, delta: TraceCompactionDelta): TraceCompactionData;
|
|
87
|
+
export declare function traceCompactionMarkerInput(data: TraceCompactionData): {
|
|
88
|
+
data: TraceCompactionData;
|
|
89
|
+
kind: typeof TRACE_COMPACTION_KIND;
|
|
90
|
+
level: "warning";
|
|
91
|
+
};
|
|
92
|
+
export declare function classifyRunControlRecord(value: unknown): "pending" | "terminal" | undefined;
|
|
93
|
+
export declare function runControlStatusTimestamp(record: Record<string, unknown>, status: RunControlStatus): unknown;
|
|
94
|
+
export declare function computeRunControlCapacity(records: readonly unknown[]): RunControlCapacity;
|
|
95
|
+
export declare function decideRunControlAdmission(input: RunControlAdmissionInput): RunControlAdmissionDecision;
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pure bounded Run-evidence policy.
|
|
3
|
+
* Zones: Trace retention, compaction accounting, Control capacity and admission
|
|
4
|
+
* Owns deterministic limits policy; journal I/O, locks, lifecycle authorization, and projection stay in adapters.
|
|
5
|
+
* Trace keeps the newest physical suffix because semantic priority would silently
|
|
6
|
+
* redefine causal history. Admitted Controls never expire: only a real terminal
|
|
7
|
+
* transition or generation replacement proves that durable work is finished.
|
|
8
|
+
*/
|
|
9
|
+
import * as Limits from "./limits.js";
|
|
10
|
+
export const TRACE_COMPACTION_KIND = "runtime.trace_compacted";
|
|
11
|
+
export const TRACE_COMPACTION_VERSION = 1;
|
|
12
|
+
const PENDING_CONTROL_STATUSES = new Set(["queued", "delivered", "claimed"]);
|
|
13
|
+
const TERMINAL_CONTROL_STATUSES = new Set(["handled", "failed"]);
|
|
14
|
+
const CONTROL_ADMISSION_NEXT_ACTIONS = [
|
|
15
|
+
"inspect target=run:<id> view=control",
|
|
16
|
+
"inspect target=runtime view=triage",
|
|
17
|
+
];
|
|
18
|
+
function isCount(value) {
|
|
19
|
+
return Number.isSafeInteger(value) && value >= 0;
|
|
20
|
+
}
|
|
21
|
+
function assertCount(value, name) {
|
|
22
|
+
if (!isCount(value))
|
|
23
|
+
throw new RangeError(`${name} must be a non-negative safe integer`);
|
|
24
|
+
}
|
|
25
|
+
function boundedAdd(left, right) {
|
|
26
|
+
return Math.min(Number.MAX_SAFE_INTEGER, left + right);
|
|
27
|
+
}
|
|
28
|
+
export function traceAppendFits(usage, encodedEventBytes) {
|
|
29
|
+
if (!isCount(usage.events) || !isCount(usage.bytes) || !isCount(encodedEventBytes))
|
|
30
|
+
return false;
|
|
31
|
+
return usage.events + 1 <= Limits.TRACE_JOURNAL_MAX_EVENTS &&
|
|
32
|
+
usage.bytes + encodedEventBytes <= Limits.TRACE_JOURNAL_MAX_BYTES;
|
|
33
|
+
}
|
|
34
|
+
export function selectNewestTraceSuffix(candidates, reserve = {}) {
|
|
35
|
+
const reservedBytes = reserve.bytes ?? 0;
|
|
36
|
+
const reservedEvents = reserve.events ?? 0;
|
|
37
|
+
assertCount(reservedBytes, "reserved Trace bytes");
|
|
38
|
+
assertCount(reservedEvents, "reserved Trace events");
|
|
39
|
+
const byteBudget = Math.max(0, Limits.TRACE_JOURNAL_TARGET_BYTES - reservedBytes);
|
|
40
|
+
const eventBudget = Math.max(0, Limits.TRACE_JOURNAL_TARGET_EVENTS - reservedEvents);
|
|
41
|
+
let retainedBytes = 0;
|
|
42
|
+
let start = candidates.length;
|
|
43
|
+
while (start > 0 && candidates.length - start < eventBudget) {
|
|
44
|
+
const candidate = candidates[start - 1];
|
|
45
|
+
assertCount(candidate.encodedBytes, "encoded Trace bytes");
|
|
46
|
+
if (retainedBytes + candidate.encodedBytes > byteBudget)
|
|
47
|
+
break;
|
|
48
|
+
retainedBytes += candidate.encodedBytes;
|
|
49
|
+
start -= 1;
|
|
50
|
+
}
|
|
51
|
+
const retained = candidates.slice(start);
|
|
52
|
+
return {
|
|
53
|
+
droppedEvents: candidates.length - retained.length,
|
|
54
|
+
retained,
|
|
55
|
+
retainedBytes,
|
|
56
|
+
retainedEvents: retained.length,
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
export function accumulateTraceCompactionStatistics(previous, delta) {
|
|
60
|
+
for (const [name, value] of Object.entries(delta)) {
|
|
61
|
+
if (name !== "dropped_event_count_exact")
|
|
62
|
+
assertCount(value, name);
|
|
63
|
+
}
|
|
64
|
+
const prior = previous ?? {
|
|
65
|
+
compactions_total: 0,
|
|
66
|
+
dropped_bytes_total: 0,
|
|
67
|
+
dropped_event_count_exact: true,
|
|
68
|
+
dropped_malformed_lines_total: 0,
|
|
69
|
+
dropped_valid_events_total: 0,
|
|
70
|
+
};
|
|
71
|
+
const eventCountOverflow = prior.dropped_valid_events_total +
|
|
72
|
+
delta.dropped_valid_events > Number.MAX_SAFE_INTEGER;
|
|
73
|
+
return {
|
|
74
|
+
version: TRACE_COMPACTION_VERSION,
|
|
75
|
+
compactions_total: boundedAdd(prior.compactions_total, 1),
|
|
76
|
+
dropped_valid_events_total: boundedAdd(prior.dropped_valid_events_total, delta.dropped_valid_events),
|
|
77
|
+
dropped_malformed_lines_total: boundedAdd(prior.dropped_malformed_lines_total, delta.dropped_malformed_lines),
|
|
78
|
+
dropped_bytes_total: boundedAdd(prior.dropped_bytes_total, delta.dropped_bytes),
|
|
79
|
+
dropped_event_count_exact: prior.dropped_event_count_exact &&
|
|
80
|
+
delta.dropped_event_count_exact && !eventCountOverflow,
|
|
81
|
+
retained_events: delta.retained_events,
|
|
82
|
+
retained_bytes: delta.retained_bytes,
|
|
83
|
+
history_complete: false,
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
export function traceCompactionMarkerInput(data) {
|
|
87
|
+
return { data, kind: TRACE_COMPACTION_KIND, level: "warning" };
|
|
88
|
+
}
|
|
89
|
+
export function classifyRunControlRecord(value) {
|
|
90
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
91
|
+
return undefined;
|
|
92
|
+
const status = value.status;
|
|
93
|
+
if (PENDING_CONTROL_STATUSES.has(status))
|
|
94
|
+
return "pending";
|
|
95
|
+
if (TERMINAL_CONTROL_STATUSES.has(status))
|
|
96
|
+
return "terminal";
|
|
97
|
+
return undefined;
|
|
98
|
+
}
|
|
99
|
+
export function runControlStatusTimestamp(record, status) {
|
|
100
|
+
if (status === "failed")
|
|
101
|
+
return record.failed_at;
|
|
102
|
+
if (status === "handled")
|
|
103
|
+
return record.handled_at;
|
|
104
|
+
if (status === "claimed")
|
|
105
|
+
return record.claimed_at;
|
|
106
|
+
if (status === "delivered")
|
|
107
|
+
return record.delivered_at;
|
|
108
|
+
return record.queued_at;
|
|
109
|
+
}
|
|
110
|
+
export function computeRunControlCapacity(records) {
|
|
111
|
+
let oldest;
|
|
112
|
+
let pending = 0;
|
|
113
|
+
for (const value of records) {
|
|
114
|
+
if (classifyRunControlRecord(value) !== "pending")
|
|
115
|
+
continue;
|
|
116
|
+
pending += 1;
|
|
117
|
+
const record = value;
|
|
118
|
+
const timestamp = runControlStatusTimestamp(record, record.status);
|
|
119
|
+
const timestampMs = typeof timestamp === "string" ? Date.parse(timestamp) : Number.NaN;
|
|
120
|
+
if (typeof timestamp === "string" && Number.isFinite(timestampMs) &&
|
|
121
|
+
(!oldest || timestampMs < oldest.timestampMs)) {
|
|
122
|
+
oldest = { timestamp, timestampMs };
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
return {
|
|
126
|
+
available: Math.max(0, Limits.RUN_CONTROL_PENDING_LIMIT - pending),
|
|
127
|
+
backpressured: pending >= Limits.RUN_CONTROL_PENDING_LIMIT,
|
|
128
|
+
limit: Limits.RUN_CONTROL_PENDING_LIMIT,
|
|
129
|
+
...(oldest ? { oldest_pending_at: oldest.timestamp } : {}),
|
|
130
|
+
pending,
|
|
131
|
+
};
|
|
132
|
+
}
|
|
133
|
+
export function decideRunControlAdmission(input) {
|
|
134
|
+
const capacity = input.capacity ?? computeRunControlCapacity(input.records);
|
|
135
|
+
const reject = (reason, integrityReason) => ({
|
|
136
|
+
admitted: false,
|
|
137
|
+
capacity,
|
|
138
|
+
error: {
|
|
139
|
+
reason,
|
|
140
|
+
pending: capacity.pending,
|
|
141
|
+
limit: capacity.limit,
|
|
142
|
+
journal_bytes: input.journalBytes,
|
|
143
|
+
journal_limit: Limits.RUN_CONTROL_JOURNAL_MAX_BYTES,
|
|
144
|
+
...(capacity.oldest_pending_at ? { oldest_pending_at: capacity.oldest_pending_at } : {}),
|
|
145
|
+
run_instance_id: input.runInstanceId,
|
|
146
|
+
next_actions: [...CONTROL_ADMISSION_NEXT_ACTIONS],
|
|
147
|
+
...(integrityReason ? { integrity_reason: integrityReason } : {}),
|
|
148
|
+
},
|
|
149
|
+
});
|
|
150
|
+
if (input.integrity !== "valid")
|
|
151
|
+
return reject("control_journal_integrity", input.integrity);
|
|
152
|
+
if (!isCount(input.journalBytes) || !isCount(input.retainedJournalBytes) ||
|
|
153
|
+
!isCount(input.newRecordBytes) ||
|
|
154
|
+
input.journalBytes > Limits.RUN_CONTROL_JOURNAL_MAX_BYTES) {
|
|
155
|
+
return reject("control_journal_integrity", "journal_bytes");
|
|
156
|
+
}
|
|
157
|
+
for (const value of input.records) {
|
|
158
|
+
const classification = classifyRunControlRecord(value);
|
|
159
|
+
if (!classification)
|
|
160
|
+
return reject("control_journal_integrity", "invalid_record");
|
|
161
|
+
if (classification !== "pending")
|
|
162
|
+
continue;
|
|
163
|
+
const record = value;
|
|
164
|
+
const timestamp = runControlStatusTimestamp(record, record.status);
|
|
165
|
+
if (typeof timestamp !== "string" || !Number.isFinite(Date.parse(timestamp))) {
|
|
166
|
+
return reject("control_journal_integrity", "invalid_status_timestamp");
|
|
167
|
+
}
|
|
168
|
+
if (record.run_instance_id !== input.runInstanceId) {
|
|
169
|
+
return reject("control_journal_integrity", "generation_mismatch");
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
if (capacity.backpressured ||
|
|
173
|
+
input.retainedJournalBytes + input.newRecordBytes > Limits.RUN_CONTROL_JOURNAL_MAX_BYTES) {
|
|
174
|
+
return reject("control_backpressure");
|
|
175
|
+
}
|
|
176
|
+
return { admitted: true, capacity };
|
|
177
|
+
}
|
|
@@ -79,6 +79,7 @@ export function createRunUiRuntime(deps) {
|
|
|
79
79
|
ownerId: deps.getRunOwnerId(ctx),
|
|
80
80
|
sink: Pi.createNotificationSink(deps.pi, ctx),
|
|
81
81
|
state: observation,
|
|
82
|
+
includeAttention: true,
|
|
82
83
|
});
|
|
83
84
|
reportDiagnostics(ctx);
|
|
84
85
|
},
|
|
@@ -110,6 +111,7 @@ export function createRunUiRuntime(deps) {
|
|
|
110
111
|
},
|
|
111
112
|
start(ctx) {
|
|
112
113
|
close();
|
|
114
|
+
Observability.primeRunAttentionState(observation, deps.getRunOwnerId(ctx));
|
|
113
115
|
update(ctx, true, true);
|
|
114
116
|
watcher.refresh();
|
|
115
117
|
reconciliation.start();
|
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Run Control journal.
|
|
3
|
-
* Zones: durable
|
|
4
|
-
* Owns Run-local Control persistence;
|
|
3
|
+
* Zones: durable records, atomic admission, claims, transitions, compaction
|
|
4
|
+
* Owns Run-local Control persistence; lifecycle authorization and transport stay in adapters.
|
|
5
5
|
*/
|
|
6
|
-
import { type
|
|
7
|
-
|
|
8
|
-
export
|
|
6
|
+
import { type RunControlStatus } from "./run-evidence-policy.ts";
|
|
7
|
+
import type { JsonlReadResult } from "./state-readers.ts";
|
|
8
|
+
export { RUN_CONTROL_TERMINAL_LIMIT } from "./limits.ts";
|
|
9
|
+
export type { RunControlStatus } from "./run-evidence-policy.ts";
|
|
9
10
|
export interface RunControlRecord {
|
|
10
11
|
id: string;
|
|
11
12
|
run_instance_id: string;
|
|
@@ -34,4 +35,5 @@ export declare function appendRunControlInStateDir(stateDir: string, request: {
|
|
|
34
35
|
}): RunControlRecord;
|
|
35
36
|
export declare function updateRunControlStatusInStateDir(stateDir: string, id: string, nextStatus: RunControlStatus, metadata?: Pick<RunControlRecord, "error">, expectedStatuses?: readonly RunControlStatus[]): boolean;
|
|
36
37
|
export declare function claimRunControlInStateDir(stateDir: string, runInstanceId: string): RunControlRecord | undefined;
|
|
38
|
+
export declare function claimRunControlByIdInStateDir(stateDir: string, runInstanceId: string, controlId: string): RunControlRecord | undefined;
|
|
37
39
|
export declare function processRunControlsInStateDir(stateDir: string, runInstanceId: string, handler: (control: RunControlRecord) => Promise<void> | void, limit?: number): Promise<ProcessRunControlsResult>;
|
|
@@ -1,20 +1,73 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Run Control journal.
|
|
3
|
-
* Zones: durable
|
|
4
|
-
* Owns Run-local Control persistence;
|
|
3
|
+
* Zones: durable records, atomic admission, claims, transitions, compaction
|
|
4
|
+
* Owns Run-local Control persistence; lifecycle authorization and transport stay in adapters.
|
|
5
5
|
*/
|
|
6
6
|
import { randomUUID } from "node:crypto";
|
|
7
|
-
import {
|
|
7
|
+
import { closeSync, fstatSync, openSync, readSync, statSync } from "node:fs";
|
|
8
8
|
import { join } from "node:path";
|
|
9
|
+
import * as Control from "./control.js";
|
|
9
10
|
import { acquireFileMutationLock, writeTextAtomic } from "./file-state.js";
|
|
10
|
-
import
|
|
11
|
-
|
|
12
|
-
|
|
11
|
+
import * as Limits from "./limits.js";
|
|
12
|
+
import { classifyRunControlRecord, computeRunControlCapacity, decideRunControlAdmission, runControlStatusTimestamp, } from "./run-evidence-policy.js";
|
|
13
|
+
export { RUN_CONTROL_TERMINAL_LIMIT } from "./limits.js";
|
|
14
|
+
function admissionError(details) {
|
|
15
|
+
return Object.assign(new Error(details.reason === "control_backpressure"
|
|
16
|
+
? `Run Control admission backpressured: pending=${details.pending}/${details.limit}`
|
|
17
|
+
: `Run Control journal integrity failed: ${details.integrity_reason ?? "unknown"}`), details);
|
|
18
|
+
}
|
|
19
|
+
const STATUSES = new Set([
|
|
20
|
+
"queued", "delivered", "claimed", "handled", "failed",
|
|
21
|
+
]);
|
|
13
22
|
export function runControlsFile(stateDir) {
|
|
14
23
|
return join(stateDir, "controls.jsonl");
|
|
15
24
|
}
|
|
16
25
|
export function readRunControlJournalFromStateDir(stateDir) {
|
|
17
|
-
|
|
26
|
+
const path = runControlsFile(stateDir);
|
|
27
|
+
let fd;
|
|
28
|
+
try {
|
|
29
|
+
fd = openSync(path, "r");
|
|
30
|
+
}
|
|
31
|
+
catch (error) {
|
|
32
|
+
if (error.code === "ENOENT")
|
|
33
|
+
return { diagnostics: [], records: [] };
|
|
34
|
+
return { diagnostics: [{ message: String(error), path }], records: [] };
|
|
35
|
+
}
|
|
36
|
+
try {
|
|
37
|
+
const bytes = fstatSync(fd).size;
|
|
38
|
+
if (!Number.isSafeInteger(bytes) || bytes < 0 || bytes > Limits.RUN_CONTROL_JOURNAL_MAX_BYTES) {
|
|
39
|
+
return { diagnostics: [{ message: "Run Control journal exceeds bounded read limit", path }], records: [] };
|
|
40
|
+
}
|
|
41
|
+
const buffer = Buffer.allocUnsafe(bytes);
|
|
42
|
+
let consumed = 0;
|
|
43
|
+
while (consumed < bytes) {
|
|
44
|
+
const count = readSync(fd, buffer, consumed, bytes - consumed, consumed);
|
|
45
|
+
if (count === 0)
|
|
46
|
+
return { diagnostics: [{ message: "Run Control journal changed during bounded read", path }], records: [] };
|
|
47
|
+
consumed += count;
|
|
48
|
+
}
|
|
49
|
+
if (fstatSync(fd).size !== bytes)
|
|
50
|
+
return { diagnostics: [{ message: "Run Control journal changed during bounded read", path }], records: [] };
|
|
51
|
+
const records = [];
|
|
52
|
+
const diagnostics = [];
|
|
53
|
+
for (const [index, line] of buffer.toString("utf8").split("\n").entries()) {
|
|
54
|
+
if (!line.trim())
|
|
55
|
+
continue;
|
|
56
|
+
try {
|
|
57
|
+
records.push(JSON.parse(line));
|
|
58
|
+
}
|
|
59
|
+
catch (error) {
|
|
60
|
+
diagnostics.push({ line: index + 1, message: String(error), path });
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
return { diagnostics, records };
|
|
64
|
+
}
|
|
65
|
+
catch (error) {
|
|
66
|
+
return { diagnostics: [{ message: String(error), path }], records: [] };
|
|
67
|
+
}
|
|
68
|
+
finally {
|
|
69
|
+
closeSync(fd);
|
|
70
|
+
}
|
|
18
71
|
}
|
|
19
72
|
export function readRunControlsFromStateDir(stateDir) {
|
|
20
73
|
return readRunControlJournalFromStateDir(stateDir).records;
|
|
@@ -23,47 +76,135 @@ function acquireRunControlsLock(stateDir) {
|
|
|
23
76
|
return acquireFileMutationLock(runControlsFile(stateDir));
|
|
24
77
|
}
|
|
25
78
|
function compactRunControls(records) {
|
|
26
|
-
const active = records.filter((record) =>
|
|
27
|
-
const terminal = records
|
|
28
|
-
.
|
|
29
|
-
.slice(-RUN_CONTROL_TERMINAL_LIMIT);
|
|
79
|
+
const active = records.filter((record) => classifyRunControlRecord(record) !== "terminal");
|
|
80
|
+
const terminal = records.filter((record) => classifyRunControlRecord(record) === "terminal")
|
|
81
|
+
.slice(-Limits.RUN_CONTROL_TERMINAL_LIMIT);
|
|
30
82
|
const retained = new Set([...active, ...terminal]);
|
|
31
|
-
return records.filter((record) => retained.has(record))
|
|
83
|
+
return records.filter((record) => retained.has(record)).map((record) => typeof record.error === "string" && Buffer.byteLength(record.error) > Limits.RUN_CONTROL_ERROR_MAX_BYTES
|
|
84
|
+
? { ...record, error: boundedControlError(record.error) } : record);
|
|
85
|
+
}
|
|
86
|
+
function encodeRunControls(records) {
|
|
87
|
+
return records.length ? `${records.map((record) => JSON.stringify(record)).join("\n")}\n` : "";
|
|
88
|
+
}
|
|
89
|
+
function boundedControlError(value) {
|
|
90
|
+
if (Buffer.byteLength(value) <= Limits.RUN_CONTROL_ERROR_MAX_BYTES)
|
|
91
|
+
return value;
|
|
92
|
+
const suffix = "… [truncated]";
|
|
93
|
+
const limit = Limits.RUN_CONTROL_ERROR_MAX_BYTES - Buffer.byteLength(suffix);
|
|
94
|
+
const bytes = Buffer.from(value);
|
|
95
|
+
let end = limit;
|
|
96
|
+
while (end > 0 && (bytes[end] & 0xc0) === 0x80)
|
|
97
|
+
end -= 1;
|
|
98
|
+
return `${bytes.subarray(0, end).toString("utf8")}${suffix}`;
|
|
32
99
|
}
|
|
33
100
|
function writeRunControls(stateDir, records) {
|
|
34
|
-
const
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
101
|
+
const encoded = encodeRunControls(compactRunControls(records));
|
|
102
|
+
if (Buffer.byteLength(encoded) > Limits.RUN_CONTROL_JOURNAL_MAX_BYTES) {
|
|
103
|
+
throw new Error("Run Control journal transition exceeds bounded byte limit");
|
|
104
|
+
}
|
|
105
|
+
writeTextAtomic(runControlsFile(stateDir), encoded);
|
|
106
|
+
}
|
|
107
|
+
function readMutableRunControls(stateDir, runInstanceId) {
|
|
108
|
+
const journal = readRunControlJournalFromStateDir(stateDir);
|
|
109
|
+
if (journal.diagnostics.length)
|
|
110
|
+
return undefined;
|
|
111
|
+
const records = journal.records;
|
|
112
|
+
if (runInstanceId && records.some((record) => controlIntegrity(record, runInstanceId) !== "valid"))
|
|
113
|
+
return undefined;
|
|
114
|
+
return records;
|
|
115
|
+
}
|
|
116
|
+
function controlIntegrity(value, runInstanceId) {
|
|
117
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
118
|
+
return "invalid_record";
|
|
119
|
+
const record = value;
|
|
120
|
+
if (typeof record.id !== "string" || !record.id ||
|
|
121
|
+
typeof record.run_instance_id !== "string" || !record.run_instance_id ||
|
|
122
|
+
typeof record.status !== "string" || !STATUSES.has(record.status) ||
|
|
123
|
+
(record.error !== undefined && typeof record.error !== "string"))
|
|
124
|
+
return "invalid_record";
|
|
125
|
+
try {
|
|
126
|
+
Control.normalizeControlAction(record.action);
|
|
127
|
+
if (record.input !== undefined)
|
|
128
|
+
Control.normalizeControlInput(record.input);
|
|
129
|
+
}
|
|
130
|
+
catch {
|
|
131
|
+
return "invalid_record";
|
|
132
|
+
}
|
|
133
|
+
const timestamp = runControlStatusTimestamp(record, record.status);
|
|
134
|
+
if (typeof timestamp !== "string" || !Number.isFinite(Date.parse(timestamp)))
|
|
135
|
+
return "invalid_status_timestamp";
|
|
136
|
+
if (classifyRunControlRecord(record) === "pending" &&
|
|
137
|
+
record.run_instance_id !== runInstanceId)
|
|
138
|
+
return "generation_mismatch";
|
|
139
|
+
return "valid";
|
|
38
140
|
}
|
|
39
141
|
export function appendRunControlInStateDir(stateDir, request) {
|
|
142
|
+
const action = Control.normalizeControlAction(request.action);
|
|
143
|
+
const input = request.input === undefined ? undefined : Control.normalizeControlInput(request.input);
|
|
40
144
|
const record = {
|
|
41
145
|
id: randomUUID(),
|
|
42
146
|
run_instance_id: request.run_instance_id,
|
|
43
|
-
action
|
|
44
|
-
...(
|
|
147
|
+
action,
|
|
148
|
+
...(input !== undefined ? { input } : {}),
|
|
45
149
|
status: "queued",
|
|
46
150
|
queued_at: new Date().toISOString(),
|
|
47
151
|
};
|
|
152
|
+
const encodedRecordBytes = Buffer.byteLength(`${JSON.stringify(record)}\n`);
|
|
48
153
|
const releaseLock = acquireRunControlsLock(stateDir);
|
|
49
154
|
try {
|
|
50
|
-
|
|
51
|
-
|
|
155
|
+
const path = runControlsFile(stateDir);
|
|
156
|
+
let bytes = 0;
|
|
157
|
+
try {
|
|
158
|
+
bytes = statSync(path).size;
|
|
159
|
+
}
|
|
160
|
+
catch (error) {
|
|
161
|
+
if (error.code !== "ENOENT")
|
|
162
|
+
bytes = Number.MAX_SAFE_INTEGER;
|
|
163
|
+
}
|
|
164
|
+
const journal = bytes > Limits.RUN_CONTROL_JOURNAL_MAX_BYTES
|
|
165
|
+
? { diagnostics: [], records: [] } : readRunControlJournalFromStateDir(stateDir);
|
|
166
|
+
let integrity = bytes > Limits.RUN_CONTROL_JOURNAL_MAX_BYTES
|
|
167
|
+
? "journal_bytes"
|
|
168
|
+
: journal.diagnostics.some((item) => item.line !== undefined)
|
|
169
|
+
? "malformed"
|
|
170
|
+
: journal.diagnostics.length ? "unreadable" : "valid";
|
|
171
|
+
for (const value of journal.records) {
|
|
172
|
+
if (integrity !== "valid")
|
|
173
|
+
break;
|
|
174
|
+
integrity = controlIntegrity(value, request.run_instance_id);
|
|
175
|
+
}
|
|
176
|
+
const records = journal.records;
|
|
177
|
+
const retained = encodeRunControls(compactRunControls(records));
|
|
178
|
+
const decision = decideRunControlAdmission({
|
|
179
|
+
integrity,
|
|
180
|
+
journalBytes: bytes,
|
|
181
|
+
newRecordBytes: encodedRecordBytes,
|
|
182
|
+
records,
|
|
183
|
+
retainedJournalBytes: Buffer.byteLength(retained),
|
|
184
|
+
runInstanceId: request.run_instance_id,
|
|
185
|
+
capacity: computeRunControlCapacity(records),
|
|
52
186
|
});
|
|
187
|
+
if (!decision.admitted)
|
|
188
|
+
throw admissionError(decision.error);
|
|
189
|
+
writeTextAtomic(path, `${retained}${JSON.stringify(record)}\n`);
|
|
190
|
+
return record;
|
|
53
191
|
}
|
|
54
192
|
finally {
|
|
55
193
|
releaseLock();
|
|
56
194
|
}
|
|
57
|
-
return record;
|
|
58
195
|
}
|
|
59
196
|
export function updateRunControlStatusInStateDir(stateDir, id, nextStatus, metadata = {}, expectedStatuses) {
|
|
60
197
|
const releaseLock = acquireRunControlsLock(stateDir);
|
|
61
198
|
try {
|
|
62
|
-
const records =
|
|
199
|
+
const records = readMutableRunControls(stateDir);
|
|
200
|
+
if (!records)
|
|
201
|
+
return false;
|
|
63
202
|
const index = records.findIndex((record) => record.id === id);
|
|
64
203
|
if (index < 0)
|
|
65
204
|
return false;
|
|
66
205
|
const record = records[index];
|
|
206
|
+
if (records.some((value) => controlIntegrity(value, record.run_instance_id) !== "valid"))
|
|
207
|
+
return false;
|
|
67
208
|
if (expectedStatuses && !expectedStatuses.includes(record.status))
|
|
68
209
|
return false;
|
|
69
210
|
if (record.status === nextStatus)
|
|
@@ -72,28 +213,19 @@ export function updateRunControlStatusInStateDir(stateDir, id, nextStatus, metad
|
|
|
72
213
|
if (nextStatus === "delivered") {
|
|
73
214
|
if (record.delivered_at)
|
|
74
215
|
return false;
|
|
75
|
-
records[index] = {
|
|
76
|
-
...record
|
|
77
|
-
delivered_at: timestamp,
|
|
78
|
-
...(record.status === "queued" ? { status: "delivered" } : {}),
|
|
79
|
-
};
|
|
216
|
+
records[index] = { ...record, delivered_at: timestamp,
|
|
217
|
+
...(record.status === "queued" ? { status: "delivered" } : {}) };
|
|
80
218
|
}
|
|
81
219
|
else {
|
|
82
|
-
if (
|
|
220
|
+
if (classifyRunControlRecord(record) === "terminal" || nextStatus === "queued")
|
|
83
221
|
return false;
|
|
84
|
-
if ((nextStatus === "claimed" &&
|
|
85
|
-
record.status !== "queued" &&
|
|
222
|
+
if ((nextStatus === "claimed" && record.status !== "queued" &&
|
|
86
223
|
record.status !== "delivered") ||
|
|
87
|
-
(nextStatus === "handled" && record.status !== "claimed"))
|
|
224
|
+
(nextStatus === "handled" && record.status !== "claimed"))
|
|
88
225
|
return false;
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
...record,
|
|
93
|
-
...metadata,
|
|
94
|
-
[timestampKey]: timestamp,
|
|
95
|
-
status: nextStatus,
|
|
96
|
-
};
|
|
226
|
+
records[index] = { ...record,
|
|
227
|
+
...(metadata.error !== undefined ? { error: boundedControlError(metadata.error) } : {}),
|
|
228
|
+
[`${nextStatus}_at`]: timestamp, status: nextStatus };
|
|
97
229
|
}
|
|
98
230
|
writeRunControls(stateDir, records);
|
|
99
231
|
return true;
|
|
@@ -102,18 +234,19 @@ export function updateRunControlStatusInStateDir(stateDir, id, nextStatus, metad
|
|
|
102
234
|
releaseLock();
|
|
103
235
|
}
|
|
104
236
|
}
|
|
105
|
-
|
|
237
|
+
function claimRunControl(stateDir, runInstanceId, controlId) {
|
|
106
238
|
const releaseLock = acquireRunControlsLock(stateDir);
|
|
107
239
|
try {
|
|
108
|
-
const records =
|
|
109
|
-
|
|
240
|
+
const records = readMutableRunControls(stateDir, runInstanceId);
|
|
241
|
+
if (!records)
|
|
242
|
+
return undefined;
|
|
243
|
+
const index = records.findIndex((record) => (controlId === undefined || record.id === controlId) &&
|
|
244
|
+
record.run_instance_id === runInstanceId &&
|
|
110
245
|
(record.status === "queued" || record.status === "delivered"));
|
|
111
246
|
if (index < 0)
|
|
112
247
|
return undefined;
|
|
113
248
|
const claimed = {
|
|
114
|
-
...records[index],
|
|
115
|
-
claimed_at: new Date().toISOString(),
|
|
116
|
-
status: "claimed",
|
|
249
|
+
...records[index], claimed_at: new Date().toISOString(), status: "claimed"
|
|
117
250
|
};
|
|
118
251
|
records[index] = claimed;
|
|
119
252
|
writeRunControls(stateDir, records);
|
|
@@ -123,6 +256,12 @@ export function claimRunControlInStateDir(stateDir, runInstanceId) {
|
|
|
123
256
|
releaseLock();
|
|
124
257
|
}
|
|
125
258
|
}
|
|
259
|
+
export function claimRunControlInStateDir(stateDir, runInstanceId) {
|
|
260
|
+
return claimRunControl(stateDir, runInstanceId);
|
|
261
|
+
}
|
|
262
|
+
export function claimRunControlByIdInStateDir(stateDir, runInstanceId, controlId) {
|
|
263
|
+
return claimRunControl(stateDir, runInstanceId, controlId);
|
|
264
|
+
}
|
|
126
265
|
export async function processRunControlsInStateDir(stateDir, runInstanceId, handler, limit = 1) {
|
|
127
266
|
const result = { claimed: 0, failed: 0, handled: 0 };
|
|
128
267
|
for (let index = 0; index < Math.max(1, limit); index += 1) {
|
|
@@ -132,14 +271,12 @@ export async function processRunControlsInStateDir(stateDir, runInstanceId, hand
|
|
|
132
271
|
result.claimed += 1;
|
|
133
272
|
try {
|
|
134
273
|
await handler(control);
|
|
135
|
-
if (updateRunControlStatusInStateDir(stateDir, control.id, "handled", {}, ["claimed"]))
|
|
274
|
+
if (updateRunControlStatusInStateDir(stateDir, control.id, "handled", {}, ["claimed"]))
|
|
136
275
|
result.handled += 1;
|
|
137
|
-
}
|
|
138
276
|
}
|
|
139
277
|
catch (error) {
|
|
140
|
-
if (updateRunControlStatusInStateDir(stateDir, control.id, "failed", { error: error instanceof Error ? error.message : String(error) }, ["claimed"]))
|
|
278
|
+
if (updateRunControlStatusInStateDir(stateDir, control.id, "failed", { error: error instanceof Error ? error.message : String(error) }, ["claimed"]))
|
|
141
279
|
result.failed += 1;
|
|
142
|
-
}
|
|
143
280
|
}
|
|
144
281
|
}
|
|
145
282
|
return result;
|