@pasko70/pibo 1.12.0 → 1.12.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/apps/chat/chat-trace-helpers.js +59 -2
- package/dist/apps/chat/data/timeline-query-service.js +1 -1
- package/dist/apps/chat/web-app.js +16 -8
- package/dist/apps/chat-ui/assets/{dist-DiPvo4rq.js → dist-3bD-92Wo.js} +1 -1
- package/dist/apps/chat-ui/assets/{dist-DxJWicji.js → dist-BFjy5Y59.js} +1 -1
- package/dist/apps/chat-ui/assets/{dist-D1U9ck8z.js → dist-BKwQlbhS.js} +1 -1
- package/dist/apps/chat-ui/assets/{dist-B2Pozq2c.js → dist-Cp68y_h5.js} +1 -1
- package/dist/apps/chat-ui/assets/{dist-DYdgxFZX.js → dist-DE0y_EI7.js} +1 -1
- package/dist/apps/chat-ui/assets/{dist-Ci4lft6y.js → dist-DPb0yDi4.js} +1 -1
- package/dist/apps/chat-ui/assets/{dist-CkLit_da.js → dist-D_dQ5GaA.js} +1 -1
- package/dist/apps/chat-ui/assets/{dist-Bgv5Fm3v.js → dist-Daq2KRpq.js} +1 -1
- package/dist/apps/chat-ui/assets/{dist-D4qbF8wy.js → dist-aZnuCezL.js} +1 -1
- package/dist/apps/chat-ui/assets/{dist-la7ZfDGk.js → dist-q6Vpm5yH.js} +1 -1
- package/dist/apps/chat-ui/assets/index-DXPKuGfs.js +237 -0
- package/dist/apps/chat-ui/index.html +1 -1
- package/dist/apps/chat-vscode-web/assets/index-B3dSrp-L.js +41 -0
- package/dist/apps/chat-vscode-web/index.html +1 -1
- package/dist/apps/vscode-artifacts/latest.vsix +0 -0
- package/dist/apps/vscode-artifacts/pibo-vscode-ext-1.12.1.vsix +0 -0
- package/dist/core/gateway-resource-guard.js +46 -15
- package/dist/core/routed-session.js +40 -1
- package/dist/core/runtime.js +2 -0
- package/dist/core/session-errors.js +3 -0
- package/dist/core/session-router.js +25 -2
- package/dist/core/transcript-integrity.js +431 -0
- package/dist/data/pibo-store.js +2 -0
- package/dist/data/telemetry.js +148 -0
- package/dist/debug/index.js +11 -0
- package/dist/gateway/server.js +1 -0
- package/dist/reliability/store.js +11 -6
- package/dist/runs/registry.js +38 -1
- package/dist/runs/resource-isolation.js +437 -0
- package/dist/runs/tools.js +6 -3
- package/dist/sessions/pibo-data-store.js +114 -0
- package/dist/shared/trace-engine.js +3 -3
- package/dist/shared/trace-event-projection.js +165 -36
- package/dist/shared/trace-nodes.js +6 -4
- package/dist/shared/trace-page-merge.js +106 -2
- package/dist/shared/trace-transcript.js +194 -36
- package/package.json +1 -1
- package/dist/apps/chat-ui/assets/index-CYxPvrxL.js +0 -237
- package/dist/apps/chat-vscode-web/assets/index-CFSHKXsQ.js +0 -41
- package/dist/apps/vscode-artifacts/pibo-vscode-ext-1.12.0.vsix +0 -0
package/dist/runs/registry.js
CHANGED
|
@@ -34,6 +34,8 @@ function snapshot(record) {
|
|
|
34
34
|
output.timeoutPhase = record.timeoutPhase;
|
|
35
35
|
if (record.serviceWarning)
|
|
36
36
|
output.serviceWarning = record.serviceWarning;
|
|
37
|
+
if (record.resources)
|
|
38
|
+
output.resources = structuredClone(record.resources);
|
|
37
39
|
if (record.completedAt)
|
|
38
40
|
output.completedAt = record.completedAt;
|
|
39
41
|
return output;
|
|
@@ -46,6 +48,7 @@ export class PiboRunRegistry {
|
|
|
46
48
|
runs = new Map();
|
|
47
49
|
waiters = new Map();
|
|
48
50
|
listeners = new Set();
|
|
51
|
+
recoveredRuns = [];
|
|
49
52
|
workerId;
|
|
50
53
|
subscribe(listener) {
|
|
51
54
|
this.listeners.add(listener);
|
|
@@ -55,12 +58,17 @@ export class PiboRunRegistry {
|
|
|
55
58
|
this.options = options;
|
|
56
59
|
this.workerId = options.workerId ?? `run-registry:${process.pid}:${randomUUID()}`;
|
|
57
60
|
if (this.options.store) {
|
|
58
|
-
this.options.store.recoverInterruptedRuns(this.workerId)
|
|
61
|
+
for (const recovered of this.options.store.recoverInterruptedRuns(this.workerId)) {
|
|
62
|
+
this.recoveredRuns.push(snapshot(recordFromStored(recovered)));
|
|
63
|
+
}
|
|
59
64
|
for (const record of this.options.store.listRuns({ includeConsumed: true, includeDetached: true })) {
|
|
60
65
|
this.runs.set(record.runId, recordFromStored(record));
|
|
61
66
|
}
|
|
62
67
|
}
|
|
63
68
|
}
|
|
69
|
+
listRecoveredRuns() {
|
|
70
|
+
return this.recoveredRuns.map((run) => ({ ...run }));
|
|
71
|
+
}
|
|
64
72
|
startToolRun(input) {
|
|
65
73
|
this.prune();
|
|
66
74
|
if (this.options.store) {
|
|
@@ -73,6 +81,7 @@ export class PiboRunRegistry {
|
|
|
73
81
|
maxAttempts: input.maxAttempts ?? 1,
|
|
74
82
|
timeoutMs: input.timeoutMs,
|
|
75
83
|
serviceWarning: input.serviceWarning,
|
|
84
|
+
resources: input.resources,
|
|
76
85
|
workerId: this.workerId,
|
|
77
86
|
});
|
|
78
87
|
const record = recordFromStored(stored);
|
|
@@ -98,12 +107,22 @@ export class PiboRunRegistry {
|
|
|
98
107
|
maxAttempts: Math.max(1, input.maxAttempts ?? 1),
|
|
99
108
|
...(input.timeoutMs !== undefined ? { timeoutMs: input.timeoutMs, timeoutAt: runTimeoutAt(timestamp, input.timeoutMs) } : {}),
|
|
100
109
|
...(input.serviceWarning ? { serviceWarning: input.serviceWarning } : {}),
|
|
110
|
+
...(input.resources ? { resources: structuredClone(input.resources) } : {}),
|
|
101
111
|
};
|
|
102
112
|
this.runs.set(runId, record);
|
|
103
113
|
const output = snapshot(record);
|
|
104
114
|
this.notify({ type: "run_started", run: output });
|
|
105
115
|
return output;
|
|
106
116
|
}
|
|
117
|
+
updateResources(runId, resources) {
|
|
118
|
+
const record = this.runs.get(runId);
|
|
119
|
+
if (!record)
|
|
120
|
+
return undefined;
|
|
121
|
+
record.resources = structuredClone(resources);
|
|
122
|
+
record.updatedAt = now();
|
|
123
|
+
this.options.store?.updateRun(runId, record);
|
|
124
|
+
return snapshot(record);
|
|
125
|
+
}
|
|
107
126
|
complete(runId, result) {
|
|
108
127
|
const record = this.runs.get(runId);
|
|
109
128
|
if (!record || terminal(record.status))
|
|
@@ -120,6 +139,23 @@ export class PiboRunRegistry {
|
|
|
120
139
|
this.notify({ type: "run_changed", run: output, previousStatus });
|
|
121
140
|
return output;
|
|
122
141
|
}
|
|
142
|
+
resourceLimit(runId, error, resources) {
|
|
143
|
+
const record = this.runs.get(runId);
|
|
144
|
+
if (!record || terminal(record.status))
|
|
145
|
+
return undefined;
|
|
146
|
+
const previousStatus = record.status;
|
|
147
|
+
record.status = "failed";
|
|
148
|
+
record.error = error;
|
|
149
|
+
record.resources = structuredClone(resources);
|
|
150
|
+
record.summary = `${record.toolName} run was stopped by yielded-run resource limits.`;
|
|
151
|
+
this.finish(record);
|
|
152
|
+
this.options.store?.updateRun(runId, record);
|
|
153
|
+
if (record.jobId)
|
|
154
|
+
this.options.store?.fail(record.jobId, this.workerId, error);
|
|
155
|
+
const output = snapshot(record);
|
|
156
|
+
this.notify({ type: "run_changed", run: output, previousStatus, reason: error });
|
|
157
|
+
return output;
|
|
158
|
+
}
|
|
123
159
|
fail(runId, error) {
|
|
124
160
|
const record = this.runs.get(runId);
|
|
125
161
|
if (!record || terminal(record.status))
|
|
@@ -424,5 +460,6 @@ function recordFromStored(record) {
|
|
|
424
460
|
timeoutAt: record.timeoutAt,
|
|
425
461
|
timeoutPhase: record.timeoutPhase,
|
|
426
462
|
serviceWarning: record.serviceWarning,
|
|
463
|
+
resources: record.resources,
|
|
427
464
|
};
|
|
428
465
|
}
|
|
@@ -0,0 +1,437 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { execFile } from "node:child_process";
|
|
3
|
+
import { existsSync, readFileSync, unlinkSync } from "node:fs";
|
|
4
|
+
import { promisify } from "node:util";
|
|
5
|
+
const execFileAsync = promisify(execFile);
|
|
6
|
+
export class PiboRunResourceLimitError extends Error {
|
|
7
|
+
resources;
|
|
8
|
+
constructor(message, resources) {
|
|
9
|
+
super(message);
|
|
10
|
+
this.resources = resources;
|
|
11
|
+
this.name = "PiboRunResourceLimitError";
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
const DEFAULT_POLICY = Object.freeze({
|
|
15
|
+
mode: "systemd",
|
|
16
|
+
memoryHighBytes: 1280 * 1024 * 1024,
|
|
17
|
+
memoryMaxBytes: 1792 * 1024 * 1024,
|
|
18
|
+
tasksMax: 128,
|
|
19
|
+
cpuQuotaPercent: 200,
|
|
20
|
+
ioWeight: 100,
|
|
21
|
+
monitorIntervalMs: 500,
|
|
22
|
+
minHostAvailableBytes: 1024 * 1024 * 1024,
|
|
23
|
+
maxMemoryFullPsiAvg10: 5,
|
|
24
|
+
maxIoFullPsiAvg10: 10,
|
|
25
|
+
});
|
|
26
|
+
export function resolveYieldedRunResourcePolicy(env = process.env) {
|
|
27
|
+
return {
|
|
28
|
+
mode: parseIsolationMode(env.PIBO_YIELDED_RUN_ISOLATION, DEFAULT_POLICY.mode),
|
|
29
|
+
memoryHighBytes: parseNonNegativeInteger(env.PIBO_YIELDED_RUN_MEMORY_HIGH_BYTES, DEFAULT_POLICY.memoryHighBytes),
|
|
30
|
+
memoryMaxBytes: parseNonNegativeInteger(env.PIBO_YIELDED_RUN_MEMORY_MAX_BYTES, DEFAULT_POLICY.memoryMaxBytes),
|
|
31
|
+
tasksMax: parsePositiveInteger(env.PIBO_YIELDED_RUN_TASKS_MAX, DEFAULT_POLICY.tasksMax),
|
|
32
|
+
cpuQuotaPercent: parsePositiveNumber(env.PIBO_YIELDED_RUN_CPU_QUOTA_PERCENT, DEFAULT_POLICY.cpuQuotaPercent),
|
|
33
|
+
ioWeight: Math.min(10_000, parsePositiveInteger(env.PIBO_YIELDED_RUN_IO_WEIGHT, DEFAULT_POLICY.ioWeight)),
|
|
34
|
+
monitorIntervalMs: Math.max(100, parsePositiveInteger(env.PIBO_YIELDED_RUN_MONITOR_INTERVAL_MS, DEFAULT_POLICY.monitorIntervalMs)),
|
|
35
|
+
minHostAvailableBytes: parseNonNegativeInteger(env.PIBO_YIELDED_RUN_MIN_HOST_AVAILABLE_BYTES, DEFAULT_POLICY.minHostAvailableBytes),
|
|
36
|
+
maxMemoryFullPsiAvg10: parseNonNegativeNumber(env.PIBO_YIELDED_RUN_MAX_MEMORY_FULL_PSI_AVG10, DEFAULT_POLICY.maxMemoryFullPsiAvg10),
|
|
37
|
+
maxIoFullPsiAvg10: parseNonNegativeNumber(env.PIBO_YIELDED_RUN_MAX_IO_FULL_PSI_AVG10, DEFAULT_POLICY.maxIoFullPsiAvg10),
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
export function collectYieldedRunHostResourceSnapshot(options = {}) {
|
|
41
|
+
const memory = parseLinuxMeminfo(options.meminfo ?? readOptionalFile("/proc/meminfo"));
|
|
42
|
+
return {
|
|
43
|
+
capturedAt: (options.now ?? new Date()).toISOString(),
|
|
44
|
+
memoryFreeBytes: memory.freeBytes,
|
|
45
|
+
memoryAvailableBytes: memory.availableBytes,
|
|
46
|
+
memoryPressure: parseLinuxPressure(options.memoryPressure ?? readOptionalFile("/proc/pressure/memory")),
|
|
47
|
+
ioPressure: parseLinuxPressure(options.ioPressure ?? readOptionalFile("/proc/pressure/io")),
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
export function parseLinuxMeminfo(input) {
|
|
51
|
+
const values = new Map();
|
|
52
|
+
for (const line of input.split("\n")) {
|
|
53
|
+
const match = line.match(/^([A-Za-z_()]+):\s+(\d+)\s+kB$/);
|
|
54
|
+
if (match)
|
|
55
|
+
values.set(match[1], Number(match[2]) * 1024);
|
|
56
|
+
}
|
|
57
|
+
const freeBytes = values.get("MemFree") ?? 0;
|
|
58
|
+
return { freeBytes, availableBytes: values.get("MemAvailable") ?? freeBytes };
|
|
59
|
+
}
|
|
60
|
+
export function parseLinuxPressure(input) {
|
|
61
|
+
const output = {};
|
|
62
|
+
for (const line of input.split("\n")) {
|
|
63
|
+
const match = line.match(/^(some|full)\s+.*\bavg10=([0-9.]+)/);
|
|
64
|
+
if (!match)
|
|
65
|
+
continue;
|
|
66
|
+
const value = Number(match[2]);
|
|
67
|
+
if (!Number.isFinite(value))
|
|
68
|
+
continue;
|
|
69
|
+
if (match[1] === "some")
|
|
70
|
+
output.someAvg10 = value;
|
|
71
|
+
else
|
|
72
|
+
output.fullAvg10 = value;
|
|
73
|
+
}
|
|
74
|
+
return output;
|
|
75
|
+
}
|
|
76
|
+
export function prepareYieldedRunExecution(toolName, params, options = {}) {
|
|
77
|
+
const policy = resolveYieldedRunResourcePolicy(options.env);
|
|
78
|
+
const command = bashCommand(params);
|
|
79
|
+
const shouldIsolate = policy.mode === "systemd" && toolName === "bash" && command !== undefined;
|
|
80
|
+
const unitName = shouldIsolate ? options.unitName ?? yieldedRunUnitName() : undefined;
|
|
81
|
+
const metricsPath = unitName ? `/tmp/${unitName}.metrics` : undefined;
|
|
82
|
+
const resources = {
|
|
83
|
+
isolationMode: shouldIsolate ? "systemd" : "off",
|
|
84
|
+
...(unitName ? { unitName } : {}),
|
|
85
|
+
policy,
|
|
86
|
+
admission: collectYieldedRunHostResourceSnapshot({ now: options.now }),
|
|
87
|
+
};
|
|
88
|
+
const preparedParams = shouldIsolate
|
|
89
|
+
? { ...params, command: systemdRunCommand(command, unitName, policy, metricsPath) }
|
|
90
|
+
: params;
|
|
91
|
+
return {
|
|
92
|
+
params: preparedParams,
|
|
93
|
+
resources,
|
|
94
|
+
async execute(operation) {
|
|
95
|
+
if (!shouldIsolate || !unitName)
|
|
96
|
+
return await operation();
|
|
97
|
+
if (process.platform !== "linux" || !existsSync("/run/systemd/system")) {
|
|
98
|
+
resources.limitReason = "systemd isolation is unavailable on this host";
|
|
99
|
+
throw new PiboRunResourceLimitError("resource_limited: systemd isolation is unavailable for yielded Bash execution", resources);
|
|
100
|
+
}
|
|
101
|
+
resources.startedAt = new Date().toISOString();
|
|
102
|
+
const monitor = monitorYieldedRunResources(unitName, policy, resources, metricsPath);
|
|
103
|
+
try {
|
|
104
|
+
const value = await operation();
|
|
105
|
+
await monitor.finish();
|
|
106
|
+
if (resources.limitReason)
|
|
107
|
+
throw new PiboRunResourceLimitError(`resource_limited: ${resources.limitReason}`, resources);
|
|
108
|
+
return value;
|
|
109
|
+
}
|
|
110
|
+
catch (error) {
|
|
111
|
+
if (!(error instanceof PiboRunResourceLimitError))
|
|
112
|
+
await terminateSystemdUnit(unitName);
|
|
113
|
+
await monitor.finish();
|
|
114
|
+
if (error instanceof PiboRunResourceLimitError)
|
|
115
|
+
throw error;
|
|
116
|
+
if (resources.limitReason || cgroupReachedResourceLimit(resources.cgroup)) {
|
|
117
|
+
resources.limitReason ??= cgroupLimitReason(resources.cgroup) ?? "the yielded-run cgroup reached a configured resource limit";
|
|
118
|
+
throw new PiboRunResourceLimitError(`resource_limited: ${resources.limitReason}`, resources);
|
|
119
|
+
}
|
|
120
|
+
throw error;
|
|
121
|
+
}
|
|
122
|
+
},
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
export function systemdRunCommand(command, unitName, policy, metricsPath = `/tmp/${unitName}.metrics`) {
|
|
126
|
+
const captureScript = [
|
|
127
|
+
'/bin/bash -c "$1"',
|
|
128
|
+
"status=$?",
|
|
129
|
+
"cgroup_path=$(awk -F: '$1 == \"0\" { print $3 }' /proc/self/cgroup)",
|
|
130
|
+
'cgroup_root="/sys/fs/cgroup${cgroup_path}"',
|
|
131
|
+
'{',
|
|
132
|
+
'printf "ControlGroup=%s\\n" "$cgroup_path"',
|
|
133
|
+
'printf "CgroupRoot=%s\\n" "$cgroup_root"',
|
|
134
|
+
'for file in memory.current memory.peak memory.high memory.max memory.swap.peak memory.swap.max pids.current pids.peak; do value=$(cat "$cgroup_root/$file" 2>/dev/null || true); printf "%s=%s\\n" "$file" "$value"; done',
|
|
135
|
+
'cat "$cgroup_root/memory.events" 2>/dev/null | awk \'{ print "memory.events." $1 "=" $2 }\' || true',
|
|
136
|
+
'cat "$cgroup_root/cpu.stat" 2>/dev/null | awk \'{ print "cpu.stat." $1 "=" $2 }\' || true',
|
|
137
|
+
'cat "$cgroup_root/io.stat" 2>/dev/null | sed "s/^/io.stat./" || true',
|
|
138
|
+
'} > "$2"',
|
|
139
|
+
"exit $status",
|
|
140
|
+
].join("\n");
|
|
141
|
+
const args = [
|
|
142
|
+
"systemd-run",
|
|
143
|
+
"--quiet",
|
|
144
|
+
"--wait",
|
|
145
|
+
"--pipe",
|
|
146
|
+
"--expand-environment=no",
|
|
147
|
+
`--unit=${unitName}`,
|
|
148
|
+
"--service-type=exec",
|
|
149
|
+
"--working-directory=$PWD",
|
|
150
|
+
"--slice=pibo-yielded.slice",
|
|
151
|
+
"--property=KillMode=control-group",
|
|
152
|
+
"--property=OOMPolicy=stop",
|
|
153
|
+
`--property=MemoryHigh=${policy.memoryHighBytes}`,
|
|
154
|
+
`--property=MemoryMax=${policy.memoryMaxBytes}`,
|
|
155
|
+
"--property=MemorySwapMax=0",
|
|
156
|
+
"--property=MemoryZSwapMax=0",
|
|
157
|
+
`--property=TasksMax=${policy.tasksMax}`,
|
|
158
|
+
`--property=CPUQuota=${policy.cpuQuotaPercent}%`,
|
|
159
|
+
`--property=IOWeight=${policy.ioWeight}`,
|
|
160
|
+
"--",
|
|
161
|
+
"/bin/bash",
|
|
162
|
+
"-c",
|
|
163
|
+
captureScript,
|
|
164
|
+
"pibo-yielded",
|
|
165
|
+
command,
|
|
166
|
+
metricsPath,
|
|
167
|
+
];
|
|
168
|
+
return args.map((value) => value === "--working-directory=$PWD" ? value : shellQuote(value)).join(" ");
|
|
169
|
+
}
|
|
170
|
+
function monitorYieldedRunResources(unitName, policy, resources, metricsPath) {
|
|
171
|
+
let finished = false;
|
|
172
|
+
let pollInProgress = false;
|
|
173
|
+
let killRequested = false;
|
|
174
|
+
let unitObserved = false;
|
|
175
|
+
const poll = async () => {
|
|
176
|
+
if (finished || pollInProgress)
|
|
177
|
+
return;
|
|
178
|
+
pollInProgress = true;
|
|
179
|
+
try {
|
|
180
|
+
if (!unitObserved) {
|
|
181
|
+
unitObserved = await systemdUnitExists(unitName);
|
|
182
|
+
if (!unitObserved)
|
|
183
|
+
return;
|
|
184
|
+
}
|
|
185
|
+
const host = collectYieldedRunHostResourceSnapshot();
|
|
186
|
+
resources.minimumHostAvailableBytes = Math.min(resources.minimumHostAvailableBytes ?? host.memoryAvailableBytes, host.memoryAvailableBytes);
|
|
187
|
+
resources.peakMemoryFullPsiAvg10 = Math.max(resources.peakMemoryFullPsiAvg10 ?? 0, host.memoryPressure.fullAvg10 ?? 0);
|
|
188
|
+
resources.peakIoFullPsiAvg10 = Math.max(resources.peakIoFullPsiAvg10 ?? 0, host.ioPressure.fullAvg10 ?? 0);
|
|
189
|
+
const reason = hostLimitReason(host, policy);
|
|
190
|
+
if (reason) {
|
|
191
|
+
resources.limitReason ??= reason;
|
|
192
|
+
if (!killRequested)
|
|
193
|
+
killRequested = await stopSystemdUnit(unitName);
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
finally {
|
|
197
|
+
pollInProgress = false;
|
|
198
|
+
}
|
|
199
|
+
};
|
|
200
|
+
const timer = setInterval(() => { void poll(); }, policy.monitorIntervalMs);
|
|
201
|
+
timer.unref?.();
|
|
202
|
+
void poll();
|
|
203
|
+
return {
|
|
204
|
+
async finish() {
|
|
205
|
+
if (finished)
|
|
206
|
+
return;
|
|
207
|
+
finished = true;
|
|
208
|
+
clearInterval(timer);
|
|
209
|
+
while (pollInProgress)
|
|
210
|
+
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
211
|
+
resources.cgroup = await collectSystemdUnitResources(unitName, metricsPath);
|
|
212
|
+
resources.completedAt = new Date().toISOString();
|
|
213
|
+
if (!resources.limitReason)
|
|
214
|
+
resources.limitReason = cgroupLimitReason(resources.cgroup);
|
|
215
|
+
await resetSystemdUnit(unitName);
|
|
216
|
+
},
|
|
217
|
+
};
|
|
218
|
+
}
|
|
219
|
+
function hostLimitReason(snapshot, policy) {
|
|
220
|
+
if (snapshot.memoryAvailableBytes < policy.minHostAvailableBytes) {
|
|
221
|
+
return `host MemAvailable ${snapshot.memoryAvailableBytes} fell below ${policy.minHostAvailableBytes}`;
|
|
222
|
+
}
|
|
223
|
+
if ((snapshot.memoryPressure.fullAvg10 ?? 0) >= policy.maxMemoryFullPsiAvg10) {
|
|
224
|
+
return `host memory full PSI avg10 ${snapshot.memoryPressure.fullAvg10} reached ${policy.maxMemoryFullPsiAvg10}`;
|
|
225
|
+
}
|
|
226
|
+
if ((snapshot.ioPressure.fullAvg10 ?? 0) >= policy.maxIoFullPsiAvg10) {
|
|
227
|
+
return `host I/O full PSI avg10 ${snapshot.ioPressure.fullAvg10} reached ${policy.maxIoFullPsiAvg10}`;
|
|
228
|
+
}
|
|
229
|
+
return undefined;
|
|
230
|
+
}
|
|
231
|
+
async function collectSystemdUnitResources(unitName, metricsPath) {
|
|
232
|
+
const fileSnapshot = metricsPath ? readCgroupMetrics(metricsPath, unitName) : { unitName };
|
|
233
|
+
const properties = [
|
|
234
|
+
"ControlGroup",
|
|
235
|
+
"ActiveState",
|
|
236
|
+
"SubState",
|
|
237
|
+
"Result",
|
|
238
|
+
"ExecMainStatus",
|
|
239
|
+
"OOMKilled",
|
|
240
|
+
"MemoryCurrent",
|
|
241
|
+
"MemoryPeak",
|
|
242
|
+
"MemoryHigh",
|
|
243
|
+
"MemoryMax",
|
|
244
|
+
"MemorySwapPeak",
|
|
245
|
+
"MemorySwapMax",
|
|
246
|
+
"CPUUsageNSec",
|
|
247
|
+
"TasksCurrent",
|
|
248
|
+
"TasksPeak",
|
|
249
|
+
"IOReadBytes",
|
|
250
|
+
"IOWriteBytes",
|
|
251
|
+
];
|
|
252
|
+
try {
|
|
253
|
+
const { stdout } = await execFileAsync("systemctl", ["show", unitName, ...properties.map((property) => `--property=${property}`)], { timeout: 5_000 });
|
|
254
|
+
const values = new Map(stdout.split("\n").flatMap((line) => {
|
|
255
|
+
const index = line.indexOf("=");
|
|
256
|
+
return index > 0 ? [[line.slice(0, index), line.slice(index + 1)]] : [];
|
|
257
|
+
}));
|
|
258
|
+
const output = { ...fileSnapshot, unitName };
|
|
259
|
+
assignString(output, "controlGroup", values.get("ControlGroup"));
|
|
260
|
+
assignString(output, "activeState", values.get("ActiveState"));
|
|
261
|
+
assignString(output, "subState", values.get("SubState"));
|
|
262
|
+
assignString(output, "result", values.get("Result"));
|
|
263
|
+
assignNumber(output, "execMainStatus", values.get("ExecMainStatus"));
|
|
264
|
+
if (values.get("OOMKilled") === "yes")
|
|
265
|
+
output.oomKilled = true;
|
|
266
|
+
assignNumber(output, "memoryCurrentBytes", values.get("MemoryCurrent"));
|
|
267
|
+
assignNumber(output, "memoryPeakBytes", values.get("MemoryPeak"));
|
|
268
|
+
assignNumber(output, "memoryHighBytes", values.get("MemoryHigh"));
|
|
269
|
+
assignNumber(output, "memoryMaxBytes", values.get("MemoryMax"));
|
|
270
|
+
assignNumber(output, "memorySwapPeakBytes", values.get("MemorySwapPeak"));
|
|
271
|
+
assignNumber(output, "memorySwapMaxBytes", values.get("MemorySwapMax"));
|
|
272
|
+
assignNumber(output, "cpuUsageNs", values.get("CPUUsageNSec"));
|
|
273
|
+
assignNumber(output, "tasksCurrent", values.get("TasksCurrent"));
|
|
274
|
+
assignNumber(output, "tasksPeak", values.get("TasksPeak"));
|
|
275
|
+
assignNumber(output, "ioReadBytes", values.get("IOReadBytes"));
|
|
276
|
+
assignNumber(output, "ioWriteBytes", values.get("IOWriteBytes"));
|
|
277
|
+
return output;
|
|
278
|
+
}
|
|
279
|
+
catch {
|
|
280
|
+
return fileSnapshot;
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
function readCgroupMetrics(path, unitName) {
|
|
284
|
+
try {
|
|
285
|
+
const values = new Map();
|
|
286
|
+
let ioReadBytes = 0;
|
|
287
|
+
let ioWriteBytes = 0;
|
|
288
|
+
for (const line of readFileSync(path, "utf8").split("\n")) {
|
|
289
|
+
if (line.startsWith("io.stat.")) {
|
|
290
|
+
for (const match of line.matchAll(/\b(rbytes|wbytes)=(\d+)/g)) {
|
|
291
|
+
if (match[1] === "rbytes")
|
|
292
|
+
ioReadBytes += Number(match[2]);
|
|
293
|
+
else
|
|
294
|
+
ioWriteBytes += Number(match[2]);
|
|
295
|
+
}
|
|
296
|
+
continue;
|
|
297
|
+
}
|
|
298
|
+
const equals = line.indexOf("=");
|
|
299
|
+
if (equals > 0)
|
|
300
|
+
values.set(line.slice(0, equals), line.slice(equals + 1));
|
|
301
|
+
}
|
|
302
|
+
return {
|
|
303
|
+
unitName,
|
|
304
|
+
controlGroup: nonEmpty(values.get("ControlGroup")),
|
|
305
|
+
oomKilled: (finiteNumber(values.get("memory.events.oom_kill")) ?? 0) > 0,
|
|
306
|
+
memoryCurrentBytes: finiteNumber(values.get("memory.current")),
|
|
307
|
+
memoryPeakBytes: finiteNumber(values.get("memory.peak")),
|
|
308
|
+
memoryHighBytes: finiteNumber(values.get("memory.high")),
|
|
309
|
+
memoryMaxBytes: finiteNumber(values.get("memory.max")),
|
|
310
|
+
memorySwapPeakBytes: finiteNumber(values.get("memory.swap.peak")),
|
|
311
|
+
memorySwapMaxBytes: finiteNumber(values.get("memory.swap.max")),
|
|
312
|
+
cpuUsageNs: (finiteNumber(values.get("cpu.stat.usage_usec")) ?? 0) * 1_000,
|
|
313
|
+
tasksCurrent: finiteNumber(values.get("pids.current")),
|
|
314
|
+
tasksPeak: finiteNumber(values.get("pids.peak")),
|
|
315
|
+
ioReadBytes,
|
|
316
|
+
ioWriteBytes,
|
|
317
|
+
};
|
|
318
|
+
}
|
|
319
|
+
catch {
|
|
320
|
+
return { unitName };
|
|
321
|
+
}
|
|
322
|
+
finally {
|
|
323
|
+
try {
|
|
324
|
+
unlinkSync(path);
|
|
325
|
+
}
|
|
326
|
+
catch { /* best-effort transient metrics cleanup */ }
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
async function systemdUnitExists(unitName) {
|
|
330
|
+
try {
|
|
331
|
+
const { stdout } = await execFileAsync("systemctl", ["show", unitName, "--property=LoadState", "--value"], { timeout: 5_000 });
|
|
332
|
+
return stdout.trim() !== "" && stdout.trim() !== "not-found";
|
|
333
|
+
}
|
|
334
|
+
catch {
|
|
335
|
+
return false;
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
async function stopSystemdUnit(unitName) {
|
|
339
|
+
try {
|
|
340
|
+
await execFileAsync("systemctl", ["kill", "--kill-whom=all", "--signal=SIGKILL", unitName], { timeout: 5_000 });
|
|
341
|
+
return true;
|
|
342
|
+
}
|
|
343
|
+
catch {
|
|
344
|
+
return false;
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
async function terminateSystemdUnit(unitName) {
|
|
348
|
+
await stopSystemdUnit(unitName);
|
|
349
|
+
await execFileAsync("systemctl", ["stop", unitName], { timeout: 5_000 }).catch(() => undefined);
|
|
350
|
+
}
|
|
351
|
+
async function resetSystemdUnit(unitName) {
|
|
352
|
+
await execFileAsync("systemctl", ["reset-failed", unitName], { timeout: 5_000 }).catch(() => undefined);
|
|
353
|
+
}
|
|
354
|
+
function cgroupReachedResourceLimit(snapshot) {
|
|
355
|
+
if (!snapshot)
|
|
356
|
+
return false;
|
|
357
|
+
if (snapshot.oomKilled || snapshot.result === "oom-kill")
|
|
358
|
+
return true;
|
|
359
|
+
return snapshot.memoryPeakBytes !== undefined
|
|
360
|
+
&& snapshot.memoryMaxBytes !== undefined
|
|
361
|
+
&& snapshot.memoryMaxBytes > 0
|
|
362
|
+
&& snapshot.memoryPeakBytes >= snapshot.memoryMaxBytes;
|
|
363
|
+
}
|
|
364
|
+
function cgroupLimitReason(snapshot) {
|
|
365
|
+
if (!snapshot)
|
|
366
|
+
return undefined;
|
|
367
|
+
if (snapshot.oomKilled || snapshot.result === "oom-kill")
|
|
368
|
+
return `yielded-run cgroup ${snapshot.unitName} was OOM-killed`;
|
|
369
|
+
if (snapshot.memoryPeakBytes !== undefined
|
|
370
|
+
&& snapshot.memoryMaxBytes !== undefined
|
|
371
|
+
&& snapshot.memoryMaxBytes > 0
|
|
372
|
+
&& snapshot.memoryPeakBytes >= snapshot.memoryMaxBytes) {
|
|
373
|
+
return `yielded-run cgroup memory peak ${snapshot.memoryPeakBytes} reached MemoryMax ${snapshot.memoryMaxBytes}`;
|
|
374
|
+
}
|
|
375
|
+
return undefined;
|
|
376
|
+
}
|
|
377
|
+
function bashCommand(params) {
|
|
378
|
+
if (!params || typeof params !== "object" || Array.isArray(params))
|
|
379
|
+
return undefined;
|
|
380
|
+
const command = params.command;
|
|
381
|
+
return typeof command === "string" && command.length > 0 ? command : undefined;
|
|
382
|
+
}
|
|
383
|
+
function yieldedRunUnitName() {
|
|
384
|
+
return `pibo-yielded-${randomUUID().replaceAll("-", "").slice(0, 24)}.service`;
|
|
385
|
+
}
|
|
386
|
+
function shellQuote(value) {
|
|
387
|
+
return `'${value.replaceAll("'", `'"'"'`)}'`;
|
|
388
|
+
}
|
|
389
|
+
function readOptionalFile(path) {
|
|
390
|
+
try {
|
|
391
|
+
return readFileSync(path, "utf8");
|
|
392
|
+
}
|
|
393
|
+
catch {
|
|
394
|
+
return "";
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
function nonEmpty(value) {
|
|
398
|
+
return value && value !== "[not set]" ? value : undefined;
|
|
399
|
+
}
|
|
400
|
+
function assignString(output, key, value) {
|
|
401
|
+
const normalized = nonEmpty(value);
|
|
402
|
+
if (normalized !== undefined)
|
|
403
|
+
output[key] = normalized;
|
|
404
|
+
}
|
|
405
|
+
function assignNumber(output, key, value) {
|
|
406
|
+
const normalized = finiteNumber(value);
|
|
407
|
+
if (normalized !== undefined)
|
|
408
|
+
output[key] = normalized;
|
|
409
|
+
}
|
|
410
|
+
function finiteNumber(value) {
|
|
411
|
+
if (!value || value === "infinity")
|
|
412
|
+
return undefined;
|
|
413
|
+
const parsed = Number(value);
|
|
414
|
+
return Number.isFinite(parsed) ? parsed : undefined;
|
|
415
|
+
}
|
|
416
|
+
function parseIsolationMode(value, fallback) {
|
|
417
|
+
const normalized = value?.trim().toLowerCase();
|
|
418
|
+
if (!normalized)
|
|
419
|
+
return fallback;
|
|
420
|
+
return normalized === "off" || normalized === "0" || normalized === "false" ? "off" : "systemd";
|
|
421
|
+
}
|
|
422
|
+
function parsePositiveInteger(value, fallback) {
|
|
423
|
+
const parsed = Number(value);
|
|
424
|
+
return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;
|
|
425
|
+
}
|
|
426
|
+
function parseNonNegativeInteger(value, fallback) {
|
|
427
|
+
const parsed = Number(value);
|
|
428
|
+
return Number.isFinite(parsed) && parsed >= 0 ? Math.floor(parsed) : fallback;
|
|
429
|
+
}
|
|
430
|
+
function parsePositiveNumber(value, fallback) {
|
|
431
|
+
const parsed = Number(value);
|
|
432
|
+
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
|
433
|
+
}
|
|
434
|
+
function parseNonNegativeNumber(value, fallback) {
|
|
435
|
+
const parsed = Number(value);
|
|
436
|
+
return Number.isFinite(parsed) && parsed >= 0 ? parsed : fallback;
|
|
437
|
+
}
|
package/dist/runs/tools.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { StringEnum, Type } from "@earendil-works/pi-ai";
|
|
2
2
|
import { defineTool } from "@earendil-works/pi-coding-agent";
|
|
3
3
|
import { foregroundServiceWarning, hasMeaningfulTimeoutOutput, isConfiguredTimeoutError, PiboRunExecutionTimeoutError, resolveRunTimeoutMs } from "./lifecycle.js";
|
|
4
|
+
import { PiboRunResourceLimitError, prepareYieldedRunExecution } from "./resource-isolation.js";
|
|
4
5
|
function resultText(prefix, value) {
|
|
5
6
|
return `${prefix}\n${JSON.stringify(value, null, 2)}`;
|
|
6
7
|
}
|
|
@@ -46,6 +47,7 @@ export function createRunToolDefinitions(yieldableTools, controller) {
|
|
|
46
47
|
const tool = requireTool(yieldableTools, params.toolName);
|
|
47
48
|
const timeoutMs = resolveRunTimeoutMs(tool.name, params.arguments);
|
|
48
49
|
const serviceWarning = foregroundServiceWarning(tool.name, params.arguments, timeoutMs);
|
|
50
|
+
const prepared = prepareYieldedRunExecution(tool.name, params.arguments);
|
|
49
51
|
let observedOutput = false;
|
|
50
52
|
const run = controller.startToolRun({
|
|
51
53
|
toolName: tool.name,
|
|
@@ -53,12 +55,13 @@ export function createRunToolDefinitions(yieldableTools, controller) {
|
|
|
53
55
|
completionPolicy: params.completionPolicy,
|
|
54
56
|
timeoutMs,
|
|
55
57
|
serviceWarning,
|
|
58
|
+
resources: prepared.resources,
|
|
56
59
|
async execute() {
|
|
57
60
|
try {
|
|
58
|
-
const result = await tool.execute(toolCallId, params
|
|
61
|
+
const result = await prepared.execute(() => tool.execute(toolCallId, prepared.params, signal, (update) => {
|
|
59
62
|
observedOutput ||= hasMeaningfulTimeoutOutput(update);
|
|
60
63
|
onUpdate?.(update);
|
|
61
|
-
}, ctx);
|
|
64
|
+
}, ctx));
|
|
62
65
|
const resultObject = result;
|
|
63
66
|
const text = textFromToolResult(resultObject);
|
|
64
67
|
if (resultObject.isError === true) {
|
|
@@ -69,7 +72,7 @@ export function createRunToolDefinitions(yieldableTools, controller) {
|
|
|
69
72
|
return { text, details: resultObject.details ?? result };
|
|
70
73
|
}
|
|
71
74
|
catch (error) {
|
|
72
|
-
if (error instanceof PiboRunExecutionTimeoutError)
|
|
75
|
+
if (error instanceof PiboRunExecutionTimeoutError || error instanceof PiboRunResourceLimitError)
|
|
73
76
|
throw error;
|
|
74
77
|
if (timeoutMs !== undefined && isConfiguredTimeoutError(error))
|
|
75
78
|
throw new PiboRunExecutionTimeoutError(error instanceof Error ? error.message : String(error), observedOutput ? "lifetime" : "startup");
|