@ouro.bot/cli 0.1.0-alpha.935 → 0.1.0-alpha.937
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/changelog.json +21 -0
- package/deploy/unraid/sanctuary.ouro/bundle-meta.json +1 -1
- package/deploy/unraid/sanctuary.xml +1 -1
- package/dist/heart/daemon/cli-defaults.js +5 -0
- package/dist/heart/daemon/cli-exec.js +9 -0
- package/dist/heart/daemon/cli-help.js +2 -2
- package/dist/heart/daemon/cli-parse.js +2 -1
- package/dist/heart/daemon/daemon.js +14 -0
- package/dist/heart/daemon/ouro-entry.js +4 -1
- package/dist/heart/daemon/runtime-logging.js +1 -1
- package/dist/heart/daemon/sense-manager.js +53 -0
- package/dist/heart/mailbox/mailbox-http-hooks.js +1 -0
- package/dist/heart/mailbox/mailbox-http-routes.js +8 -0
- package/dist/heart/mailbox/mailbox-read.js +3 -1
- package/dist/heart/mailbox/mailbox-supervision.js +131 -0
- package/dist/heart/shepherd/custody-types.js +2 -0
- package/dist/heart/shepherd/mandates.js +352 -0
- package/dist/heart/shepherd/operation-commands.js +19 -0
- package/dist/heart/shepherd/operations.js +135 -0
- package/dist/heart/shepherd/supervision.js +114 -0
- package/dist/heart/shepherd/types.js +2 -0
- package/dist/mailbox-ui/assets/index-Dbska1tZ.js +1 -0
- package/dist/mailbox-ui/assets/index-DsiKaPyH.css +1 -0
- package/dist/mailbox-ui/index.html +2 -2
- package/dist/nerves/coverage/file-completeness.js +4 -0
- package/dist/senses/shepherd/checkin-store.js +209 -41
- package/dist/senses/shepherd/cmux.js +69 -23
- package/dist/senses/shepherd/guidance.js +5 -2
- package/dist/senses/shepherd/herdr.js +84 -2
- package/dist/senses/shepherd/judge-evaluation-command.js +180 -0
- package/dist/senses/shepherd/judge-evaluation-entry.js +13 -0
- package/dist/senses/shepherd/judge-evaluation.js +150 -0
- package/dist/senses/shepherd/judge.js +12 -5
- package/dist/senses/shepherd/observation-publisher.js +320 -0
- package/dist/senses/shepherd/operations-defaults.js +86 -0
- package/dist/senses/shepherd/operations.js +159 -0
- package/dist/senses/shepherd/recovery.js +543 -0
- package/dist/senses/shepherd/sense.js +14 -9
- package/dist/senses/shepherd/shepherd-judge-corpus-v1.json +167 -0
- package/npm-shrinkwrap.json +2 -2
- package/package.json +2 -1
- package/dist/mailbox-ui/assets/index-Cyv4mw3Y.js +0 -1
- package/dist/mailbox-ui/assets/index-Du_9G9WO.css +0 -1
|
@@ -132,31 +132,46 @@ function createCmuxHost(client, options = {}) {
|
|
|
132
132
|
(0, runtime_1.emitNervesEvent)({ level: "warn", component: "senses", event: "senses.shepherd_cmux_status_error", message: "could not clear a Shepherd status", meta: { error: error.message } });
|
|
133
133
|
});
|
|
134
134
|
};
|
|
135
|
-
async function surfaceEntries() {
|
|
135
|
+
async function surfaceEntries(strict = false) {
|
|
136
136
|
const tree = await client.call("system.tree", { all_windows: true });
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
137
|
+
const records = (value) => {
|
|
138
|
+
if (strict && (!Array.isArray(value) || !value.every(isRecord)))
|
|
139
|
+
throw new Error("invalid cmux terminal inventory");
|
|
140
|
+
return list(value);
|
|
141
|
+
};
|
|
142
|
+
const seen = new Set();
|
|
143
|
+
return records(tree.windows).flatMap((window) => records(window.workspaces).map((workspace) => ({ window, workspace }))).flatMap(({ window, workspace }) => records(workspace.panes).flatMap((pane) => records(pane.surfaces)).map((surface) => {
|
|
144
|
+
if (strict && (typeof surface.id !== "string" || !surface.id.trim() || typeof surface.type !== "string"
|
|
145
|
+
|| !["terminal", "browser"].includes(surface.type) || seen.has(surface.id)))
|
|
146
|
+
throw new Error("invalid cmux terminal inventory identity");
|
|
147
|
+
seen.add(str(surface.id) ?? "");
|
|
148
|
+
return {
|
|
149
|
+
id: str(surface.id) ?? "",
|
|
150
|
+
ref: str(surface.ref) ?? "",
|
|
151
|
+
workspace: str(workspace.ref),
|
|
152
|
+
workspaceTitle: str(workspace.title),
|
|
153
|
+
workspaceId: str(workspace.id) ?? "",
|
|
154
|
+
title: str(surface.title) ?? "",
|
|
155
|
+
tty: str(surface.tty),
|
|
156
|
+
type: str(surface.type) ?? "",
|
|
157
|
+
agent: hooked.get(str(surface.id) ?? "")?.agent ?? null,
|
|
158
|
+
cwd: hooked.get(str(surface.id) ?? "")?.cwd ?? null,
|
|
159
|
+
// The human's terminal: the focused surface of the selected workspace in a visible window.
|
|
160
|
+
focused: surface.focused === true && workspace.selected === true && window.visible !== false,
|
|
161
|
+
...(surface.focused === true && workspace.selected === true && window.visible !== false && window.key === true
|
|
162
|
+
? { selected: true } : {}),
|
|
163
|
+
};
|
|
164
|
+
}));
|
|
165
|
+
}
|
|
166
|
+
async function retirementInventory() {
|
|
167
|
+
return (await surfaceEntries(true)).filter(surface => surface.type === "terminal").map(surface => surface.id);
|
|
153
168
|
}
|
|
154
|
-
async function surfaces(freshFocus = false) {
|
|
155
|
-
const entries = await surfaceEntries();
|
|
169
|
+
async function surfaces(freshFocus = false, strictInventory = false) {
|
|
170
|
+
const entries = await surfaceEntries(strictInventory);
|
|
156
171
|
const identities = await (options.identifyTerminals ?? terminal_process_1.identifyTerminalProcesses)(entries.filter((entry) => entry.type === "terminal" && entry.tty).map((entry) => entry.tty));
|
|
157
172
|
// Process lookup is awaited. On inspection paths, do not return focus from
|
|
158
173
|
// the tree sampled before that await. A moved/rebound TTY loses authority.
|
|
159
|
-
const current = freshFocus ? await surfaceEntries() : entries;
|
|
174
|
+
const current = freshFocus ? await surfaceEntries(strictInventory) : entries;
|
|
160
175
|
return current.map((entry) => {
|
|
161
176
|
const bound = entries.some(before => before.id === entry.id && before.tty === entry.tty && before.workspaceId === entry.workspaceId && before.type === entry.type);
|
|
162
177
|
const identity = entry.tty && bound ? identities.get(entry.tty.replace(/^\/dev\//, "")) : undefined;
|
|
@@ -216,6 +231,7 @@ function createCmuxHost(client, options = {}) {
|
|
|
216
231
|
async list() {
|
|
217
232
|
return (await surfaces()).filter((surface) => surface.type === "terminal").map(({ workspaceId: _workspaceId, type: _type, ...session }) => session);
|
|
218
233
|
},
|
|
234
|
+
retirementInventory,
|
|
219
235
|
read,
|
|
220
236
|
inspect,
|
|
221
237
|
async promptIfUnchanged(sessionId, line, expected, canSend) {
|
|
@@ -390,16 +406,40 @@ function createCmuxHost(client, options = {}) {
|
|
|
390
406
|
},
|
|
391
407
|
});
|
|
392
408
|
};
|
|
393
|
-
|
|
409
|
+
let lastObserved = [];
|
|
410
|
+
/** Inventory reads stay outside the judge queue; idle-hook terminals are observed but never rejudged here. */
|
|
394
411
|
const poll = async () => {
|
|
395
412
|
try {
|
|
396
|
-
const terminals = (await surfaces()).filter((surface) => surface.type === "terminal"
|
|
413
|
+
const terminals = (await surfaces(false, !!handlers.observed)).filter((surface) => surface.type === "terminal");
|
|
414
|
+
if (closed)
|
|
415
|
+
return;
|
|
397
416
|
const seen = new Set(terminals.map((surface) => surface.id));
|
|
398
417
|
for (const id of screens.keys())
|
|
399
418
|
if (!seen.has(id))
|
|
400
419
|
screens.delete(id);
|
|
420
|
+
const rows = [];
|
|
401
421
|
for (const surface of terminals) {
|
|
402
|
-
const
|
|
422
|
+
const idle = hooked.get(surface.id)?.idle === true;
|
|
423
|
+
if (idle)
|
|
424
|
+
screens.delete(surface.id);
|
|
425
|
+
if (idle && !handlers.observed)
|
|
426
|
+
continue;
|
|
427
|
+
let text;
|
|
428
|
+
try {
|
|
429
|
+
text = await read(surface.id, POLL_LINES);
|
|
430
|
+
}
|
|
431
|
+
catch (error) {
|
|
432
|
+
rows.push({ session: surface, activity: "unknown", observedAt: null, error: "read_failed" });
|
|
433
|
+
(0, runtime_1.emitNervesEvent)({ level: "warn", component: "senses", event: "senses.shepherd_cmux_poll_error",
|
|
434
|
+
message: "could not read one cmux terminal; other terminals remain observable", meta: { error: (0, redact_1.redactSecrets)(String(error)) } });
|
|
435
|
+
continue;
|
|
436
|
+
}
|
|
437
|
+
if (closed)
|
|
438
|
+
return;
|
|
439
|
+
rows.push({ session: surface, activity: surface.agent ? (0, prompt_state_1.inspectCodingPrompt)(surface.agent, text).activity : "unknown", observedAt: now() });
|
|
440
|
+
if (idle)
|
|
441
|
+
continue;
|
|
442
|
+
const hash = (0, node_crypto_1.createHash)("sha256").update(text).digest("hex");
|
|
403
443
|
const last = screens.get(surface.id);
|
|
404
444
|
const at = now();
|
|
405
445
|
if (!last)
|
|
@@ -413,9 +453,15 @@ function createCmuxHost(client, options = {}) {
|
|
|
413
453
|
handlers.returned({ sessionId: surface.id, transitionId: `cmux:quiet:${surface.id}:${last.changedAt}`, agent: surface.agent, cwd: surface.cwd ?? null, title: surface.title, workspaceTitle: surface.workspaceTitle, lastBody: null, backgroundWork: null });
|
|
414
454
|
}
|
|
415
455
|
}
|
|
456
|
+
if (closed)
|
|
457
|
+
return;
|
|
458
|
+
lastObserved = rows;
|
|
459
|
+
handlers.observed?.({ at: now(), sessions: rows });
|
|
416
460
|
}
|
|
417
461
|
catch (error) {
|
|
418
462
|
(0, runtime_1.emitNervesEvent)({ level: "warn", component: "senses", event: "senses.shepherd_cmux_poll_error", message: "could not poll cmux terminals; the next poll retries", meta: { error: error.message } });
|
|
463
|
+
if (!closed)
|
|
464
|
+
handlers.observed?.({ at: now(), sessions: lastObserved, error: "host_unavailable" });
|
|
419
465
|
}
|
|
420
466
|
if (!closed)
|
|
421
467
|
cancelPoll = schedule(() => void poll(), exports.POLL_MS);
|
|
@@ -145,8 +145,8 @@ function bounded(value, name, max) {
|
|
|
145
145
|
function knownTarget(root, target, live, record) {
|
|
146
146
|
if (!validTarget(target))
|
|
147
147
|
throw new Error("invalid Shepherd session target");
|
|
148
|
-
const known = record ??
|
|
149
|
-
??
|
|
148
|
+
const known = record ?? live.find(t => t.host === target.host && t.session === target.session)
|
|
149
|
+
?? [...(0, returns_1.readReturns)(root).filter(validReturn)].reverse().find(r => r.host === target.host && r.session === target.session);
|
|
150
150
|
if (!known)
|
|
151
151
|
throw new Error("unknown session; no recorded or verified live Shepherd target");
|
|
152
152
|
if (known.host !== target.host || known.session !== target.session || known.cwd !== target.cwd || known.task !== target.task)
|
|
@@ -191,6 +191,9 @@ async function mutate(root, op, input, apply) {
|
|
|
191
191
|
throw new Error("request ID was already used with a different body");
|
|
192
192
|
return prior.receipt;
|
|
193
193
|
}
|
|
194
|
+
if (input.expectedRevision !== undefined && (!Number.isSafeInteger(input.expectedRevision) || input.expectedRevision < 0 || input.expectedRevision !== state.revision)) {
|
|
195
|
+
throw new Error("Shepherd guidance revision conflict");
|
|
196
|
+
}
|
|
194
197
|
const clean = { actorId: input.actorId, requestId: input.requestId, at: new Date().toISOString(), kind: "operator instruction", returnId: null };
|
|
195
198
|
const result = apply(state, clean);
|
|
196
199
|
state.revision++;
|
|
@@ -114,12 +114,26 @@ function herdrRequest(socketPath, method, params, timeoutMs = TIMEOUT_MS) {
|
|
|
114
114
|
function createHerdrHost(connection, options = {}) {
|
|
115
115
|
const schedule = options.schedule ?? cmux_1.realSchedule;
|
|
116
116
|
const call = (method, params) => herdrRequest(connection.socketPath, method, params);
|
|
117
|
-
async function panes() {
|
|
117
|
+
async function panes(strict = false) {
|
|
118
118
|
const result = await call("pane.list", {});
|
|
119
|
+
if (strict) {
|
|
120
|
+
if (!Array.isArray(result.panes))
|
|
121
|
+
throw new Error("invalid Herdr terminal inventory");
|
|
122
|
+
const seen = new Set();
|
|
123
|
+
for (const pane of result.panes) {
|
|
124
|
+
if (!isRecord(pane) || typeof pane.pane_id !== "string" || !pane.pane_id.trim() || seen.has(pane.pane_id)) {
|
|
125
|
+
throw new Error("invalid Herdr terminal inventory identity");
|
|
126
|
+
}
|
|
127
|
+
seen.add(pane.pane_id);
|
|
128
|
+
}
|
|
129
|
+
}
|
|
119
130
|
return Array.isArray(result.panes) ? result.panes.filter(isRecord) : [];
|
|
120
131
|
}
|
|
121
132
|
return {
|
|
122
133
|
name: "herdr",
|
|
134
|
+
async retirementInventory() {
|
|
135
|
+
return (await panes(true)).map(pane => pane.pane_id);
|
|
136
|
+
},
|
|
123
137
|
async list() {
|
|
124
138
|
return (await panes()).map((pane) => ({
|
|
125
139
|
id: str(pane.pane_id) ?? "",
|
|
@@ -187,6 +201,50 @@ function createHerdrHost(connection, options = {}) {
|
|
|
187
201
|
let refresh = false;
|
|
188
202
|
const status = new Map();
|
|
189
203
|
const pending = new Map();
|
|
204
|
+
const inventory = new Map();
|
|
205
|
+
let cancelObservation = null;
|
|
206
|
+
let inventoryGeneration = 0;
|
|
207
|
+
const publishObservation = (error) => {
|
|
208
|
+
handlers.observed?.({ at: Date.now(), sessions: [...inventory.values()], ...(error ? { error } : {}) });
|
|
209
|
+
};
|
|
210
|
+
const inventoryRow = (pane) => {
|
|
211
|
+
const id = str(pane.pane_id);
|
|
212
|
+
if (!id)
|
|
213
|
+
return;
|
|
214
|
+
const state = str(pane.agent_status);
|
|
215
|
+
inventory.set(id, {
|
|
216
|
+
session: { id, ref: id, workspace: str(pane.workspace_id),
|
|
217
|
+
title: str(pane.title) ?? str(pane.terminal_title_stripped) ?? str(pane.label) ?? "",
|
|
218
|
+
agent: str(pane.agent), cwd: str(pane.foreground_cwd) ?? str(pane.cwd), focused: pane.focused === true },
|
|
219
|
+
activity: state === "working" ? "working" : RETURN_STATES.has(state ?? "") ? "idle" : "unknown",
|
|
220
|
+
observedAt: Date.now(),
|
|
221
|
+
});
|
|
222
|
+
};
|
|
223
|
+
const replaceInventory = (current) => {
|
|
224
|
+
const ids = new Set(current.flatMap(pane => str(pane.pane_id) ?? []));
|
|
225
|
+
for (const id of inventory.keys())
|
|
226
|
+
if (!ids.has(id))
|
|
227
|
+
inventory.delete(id);
|
|
228
|
+
for (const pane of current)
|
|
229
|
+
inventoryRow(pane);
|
|
230
|
+
publishObservation();
|
|
231
|
+
};
|
|
232
|
+
// Read-only refresh belongs to this host watcher, never to the judge or the browser.
|
|
233
|
+
const refreshObservation = async () => {
|
|
234
|
+
cancelObservation = null;
|
|
235
|
+
const generation = ++inventoryGeneration;
|
|
236
|
+
try {
|
|
237
|
+
const current = await panes(true);
|
|
238
|
+
if (!closed && generation === inventoryGeneration)
|
|
239
|
+
replaceInventory(current);
|
|
240
|
+
}
|
|
241
|
+
catch {
|
|
242
|
+
if (!closed && generation === inventoryGeneration)
|
|
243
|
+
publishObservation("host_unavailable");
|
|
244
|
+
}
|
|
245
|
+
if (!closed && generation === inventoryGeneration && !cancelObservation)
|
|
246
|
+
cancelObservation = schedule(() => void refreshObservation(), 5_000);
|
|
247
|
+
};
|
|
190
248
|
const cancelPending = (paneId) => {
|
|
191
249
|
pending.get(paneId)?.();
|
|
192
250
|
pending.delete(paneId);
|
|
@@ -263,12 +321,22 @@ function createHerdrHost(connection, options = {}) {
|
|
|
263
321
|
if (name === "pane_closed") {
|
|
264
322
|
cancelPending(paneId);
|
|
265
323
|
status.delete(paneId);
|
|
324
|
+
inventory.delete(paneId);
|
|
266
325
|
refresh = true;
|
|
267
326
|
stream?.close();
|
|
268
327
|
return;
|
|
269
328
|
}
|
|
270
329
|
if (name !== "pane_agent_status_changed")
|
|
271
330
|
return;
|
|
331
|
+
const row = inventory.get(paneId);
|
|
332
|
+
if (row) {
|
|
333
|
+
const next = str(data.agent_status);
|
|
334
|
+
row.activity = next === "working" ? "working" : RETURN_STATES.has(next ?? "") ? "idle" : "unknown";
|
|
335
|
+
row.observedAt = Date.now();
|
|
336
|
+
publishObservation();
|
|
337
|
+
if (closed)
|
|
338
|
+
return;
|
|
339
|
+
}
|
|
272
340
|
observed(paneId, str(data.agent_status) ?? "unknown");
|
|
273
341
|
};
|
|
274
342
|
const reconnect = (error) => {
|
|
@@ -282,13 +350,17 @@ function createHerdrHost(connection, options = {}) {
|
|
|
282
350
|
const delay = RECONNECT_DELAYS_MS[Math.min(attempt, RECONNECT_DELAYS_MS.length - 1)];
|
|
283
351
|
attempt += 1;
|
|
284
352
|
(0, runtime_1.emitNervesEvent)({ level: "warn", component: "senses", event: "senses.shepherd_herdr_closed", message: "Herdr event subscription closed; resubscribing", meta: { delay, error: error?.message ?? null } });
|
|
353
|
+
publishObservation("host_unavailable");
|
|
354
|
+
if (closed)
|
|
355
|
+
return;
|
|
285
356
|
cancelReconnect = schedule(() => void subscribe(), delay);
|
|
286
357
|
};
|
|
287
358
|
const subscribe = async () => {
|
|
288
359
|
cancelReconnect = null;
|
|
289
360
|
let current;
|
|
361
|
+
const generation = ++inventoryGeneration;
|
|
290
362
|
try {
|
|
291
|
-
current = await panes();
|
|
363
|
+
current = await panes(!!handlers.observed);
|
|
292
364
|
}
|
|
293
365
|
catch (error) {
|
|
294
366
|
return reconnect(error);
|
|
@@ -314,11 +386,19 @@ function createHerdrHost(connection, options = {}) {
|
|
|
314
386
|
else if (next !== "unknown")
|
|
315
387
|
status.set(id, next);
|
|
316
388
|
}
|
|
389
|
+
if (generation === inventoryGeneration)
|
|
390
|
+
replaceInventory(current);
|
|
391
|
+
if (closed)
|
|
392
|
+
return;
|
|
393
|
+
if (handlers.observed && !cancelObservation)
|
|
394
|
+
cancelObservation = schedule(() => void refreshObservation(), 5_000);
|
|
317
395
|
const subscriptions = [
|
|
318
396
|
{ type: "pane.created" }, { type: "pane.closed" }, { type: "pane.focused" }, { type: "pane.moved" },
|
|
319
397
|
...current.flatMap((pane) => str(pane.pane_id) ? [{ type: "pane.agent_status_changed", pane_id: pane.pane_id }] : []),
|
|
320
398
|
];
|
|
321
399
|
stream = connectLines(connection.socketPath, (line) => {
|
|
400
|
+
if (closed)
|
|
401
|
+
return;
|
|
322
402
|
if (isRecord(line.result) && line.result.type === "subscription_started") {
|
|
323
403
|
attempt = 0;
|
|
324
404
|
(0, runtime_1.emitNervesEvent)({ component: "senses", event: "senses.shepherd_herdr_connected", message: "Herdr event subscription started", meta: { panes: subscriptions.length - 4 } });
|
|
@@ -334,6 +414,8 @@ function createHerdrHost(connection, options = {}) {
|
|
|
334
414
|
return {
|
|
335
415
|
close() {
|
|
336
416
|
closed = true;
|
|
417
|
+
inventoryGeneration += 1;
|
|
418
|
+
cancelObservation?.();
|
|
337
419
|
cancelReconnect?.();
|
|
338
420
|
for (const cancel of pending.values())
|
|
339
421
|
cancel();
|
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
36
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
37
|
+
};
|
|
38
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
39
|
+
exports.openShepherdJudgeEvaluationOutput = openShepherdJudgeEvaluationOutput;
|
|
40
|
+
exports.loadConfiguredShepherdEvaluationJudge = loadConfiguredShepherdEvaluationJudge;
|
|
41
|
+
exports.runShepherdJudgeEvaluationCommand = runShepherdJudgeEvaluationCommand;
|
|
42
|
+
const fs = __importStar(require("node:fs"));
|
|
43
|
+
const path = __importStar(require("node:path"));
|
|
44
|
+
const node_crypto_1 = require("node:crypto");
|
|
45
|
+
const runtime_1 = require("../../nerves/runtime");
|
|
46
|
+
const judge_1 = require("./judge");
|
|
47
|
+
const judge_evaluation_1 = require("./judge-evaluation");
|
|
48
|
+
const redact_1 = require("./redact");
|
|
49
|
+
const shepherd_judge_corpus_v1_json_1 = __importDefault(require("./shepherd-judge-corpus-v1.json"));
|
|
50
|
+
function openShepherdJudgeEvaluationOutput(file) {
|
|
51
|
+
const fd = fs.openSync(file, "wx", 0o600);
|
|
52
|
+
return {
|
|
53
|
+
write: report => {
|
|
54
|
+
const bytes = Buffer.from(JSON.stringify(report, null, 2) + "\n", "utf8");
|
|
55
|
+
let offset = 0;
|
|
56
|
+
while (offset < bytes.length) {
|
|
57
|
+
const written = fs.writeSync(fd, bytes, offset, bytes.length - offset, offset);
|
|
58
|
+
if (written === 0)
|
|
59
|
+
throw new Error("Evaluation output write made no progress");
|
|
60
|
+
offset += written;
|
|
61
|
+
}
|
|
62
|
+
fs.ftruncateSync(fd, bytes.length);
|
|
63
|
+
fs.fsyncSync(fd);
|
|
64
|
+
},
|
|
65
|
+
close: () => fs.closeSync(fd),
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
async function loadConfiguredShepherdEvaluationJudge(agent) {
|
|
69
|
+
const { getAgentRoot } = await Promise.resolve().then(() => __importStar(require("../../heart/identity")));
|
|
70
|
+
const root = getAgentRoot(agent);
|
|
71
|
+
if (!fs.statSync(path.join(root, "agent.json")).isFile())
|
|
72
|
+
throw new Error("The selected agent needs an existing agent.json");
|
|
73
|
+
const { getProviderRuntime } = await Promise.resolve().then(() => __importStar(require("../../heart/core")));
|
|
74
|
+
const runtime = await getProviderRuntime("agent", { agentName: agent, agentRoot: root });
|
|
75
|
+
const metadata = JSON.parse(fs.readFileSync(path.resolve(__dirname, "../../../package.json"), "utf8"));
|
|
76
|
+
if (!metadata || typeof metadata !== "object" || !("version" in metadata) || typeof metadata.version !== "string") {
|
|
77
|
+
throw new Error("Could not identify the evaluation runtime package version");
|
|
78
|
+
}
|
|
79
|
+
const digest = (name) => (0, node_crypto_1.createHash)("sha256")
|
|
80
|
+
.update(fs.readFileSync(path.join(__dirname, `${name}${path.extname(__filename)}`))).digest("hex");
|
|
81
|
+
const observed = {
|
|
82
|
+
resetTurnState: messages => runtime.resetTurnState?.(messages),
|
|
83
|
+
streamTurn: async (request) => {
|
|
84
|
+
const response = await runtime.streamTurn(request);
|
|
85
|
+
if (!(0, judge_1.isShepherdJudgeAnswerValid)(response.content ?? ""))
|
|
86
|
+
throw new Error("Judge returned malformed or nonconforming JSON; no classification credit");
|
|
87
|
+
return response;
|
|
88
|
+
},
|
|
89
|
+
};
|
|
90
|
+
return {
|
|
91
|
+
judge: (0, judge_1.createShepherdJudge)(async () => observed, 20_000),
|
|
92
|
+
provenance: {
|
|
93
|
+
provider: runtime.id, model: runtime.model, runtimeVersion: metadata.version,
|
|
94
|
+
judgeSha256: digest("judge"), evaluatorSha256: digest("judge-evaluation"),
|
|
95
|
+
},
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
function parseArgs(argv) {
|
|
99
|
+
const values = new Map();
|
|
100
|
+
for (let i = 0; i < argv.length; i += 2) {
|
|
101
|
+
const name = argv[i], value = argv[i + 1];
|
|
102
|
+
if (!["--agent", "--output"].includes(name) || values.has(name) || !value?.trim() || value.startsWith("--")) {
|
|
103
|
+
throw new Error("Usage: shepherd:judge-eval -- --agent NAME --output NEW_PRIVATE_REPORT.json");
|
|
104
|
+
}
|
|
105
|
+
values.set(name, value);
|
|
106
|
+
}
|
|
107
|
+
const agent = values.get("--agent"), output = values.get("--output");
|
|
108
|
+
if (!agent || !/^[a-zA-Z0-9][a-zA-Z0-9_-]{0,99}$/.test(agent) || !output) {
|
|
109
|
+
throw new Error("Explicit --agent NAME and --output NEW_PRIVATE_REPORT.json are required");
|
|
110
|
+
}
|
|
111
|
+
return { agent, output };
|
|
112
|
+
}
|
|
113
|
+
async function runShepherdJudgeEvaluationCommand(argv, dependencies = {
|
|
114
|
+
loadJudge: loadConfiguredShepherdEvaluationJudge, openOutput: openShepherdJudgeEvaluationOutput,
|
|
115
|
+
// eslint-disable-next-line no-console -- explicit evaluation command terminal output
|
|
116
|
+
print: message => console.log(message),
|
|
117
|
+
}) {
|
|
118
|
+
let output = null;
|
|
119
|
+
let exitCode = 1;
|
|
120
|
+
let summary = null;
|
|
121
|
+
let outputClosed = false;
|
|
122
|
+
(0, runtime_1.emitNervesEvent)({ component: "senses", event: "senses.shepherd_judge_evaluation_command_start",
|
|
123
|
+
message: "starting explicit tool-free classification command", meta: {} });
|
|
124
|
+
try {
|
|
125
|
+
const args = parseArgs(argv);
|
|
126
|
+
(0, judge_evaluation_1.validateShepherdJudgeCorpus)(shepherd_judge_corpus_v1_json_1.default);
|
|
127
|
+
output = dependencies.openOutput(args.output);
|
|
128
|
+
const { judge, provenance } = await dependencies.loadJudge(args.agent);
|
|
129
|
+
const destination = output;
|
|
130
|
+
const report = await (0, judge_evaluation_1.runShepherdJudgeEvaluation)(shepherd_judge_corpus_v1_json_1.default, judge, provenance, results => {
|
|
131
|
+
if (results.length % 20 !== 0)
|
|
132
|
+
return;
|
|
133
|
+
destination.write({ schemaVersion: 1, phase: "judge-classification-only", state: "running",
|
|
134
|
+
...provenance, completedCalls: results.length, results, admissionExercised: false, nativeHostQualified: false, v1Qualified: false });
|
|
135
|
+
dependencies.print(JSON.stringify({ phase: "judge-classification-only", completedCalls: results.length, v1Qualified: false }));
|
|
136
|
+
});
|
|
137
|
+
output.write(report);
|
|
138
|
+
summary = {
|
|
139
|
+
phase: report.phase, classificationPassed: report.classificationPassed, v1Qualified: false,
|
|
140
|
+
uniqueCases: report.uniqueCases, calls: report.results.length,
|
|
141
|
+
unsafeProposedActions: report.unsafeProposedActions, unattendedKeys: report.unattendedKeys, errors: report.errors,
|
|
142
|
+
report: args.output, boundary: "Classification evidence only; not v1 qualification.",
|
|
143
|
+
};
|
|
144
|
+
exitCode = report.classificationPassed ? 0 : 1;
|
|
145
|
+
}
|
|
146
|
+
catch (error) {
|
|
147
|
+
const detail = (0, redact_1.redactSecrets)(String(error)).slice(0, 2000);
|
|
148
|
+
dependencies.print(JSON.stringify({ error: detail, v1Qualified: false }));
|
|
149
|
+
if (output) {
|
|
150
|
+
try {
|
|
151
|
+
output.write({ schemaVersion: 1, phase: "judge-classification-only", state: "error", error: detail, v1Qualified: false });
|
|
152
|
+
}
|
|
153
|
+
catch (writeError) {
|
|
154
|
+
dependencies.print(JSON.stringify({ error: `Could not persist evaluation error: ${(0, redact_1.redactSecrets)(String(writeError)).slice(0, 2000)}` }));
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
(0, runtime_1.emitNervesEvent)({ component: "senses", level: "error", event: "senses.shepherd_judge_evaluation_command_error",
|
|
158
|
+
message: "classification command failed; no v1 qualification",
|
|
159
|
+
meta: { phase: "judge-classification-only", outputReserved: output !== null, error: detail } });
|
|
160
|
+
}
|
|
161
|
+
finally {
|
|
162
|
+
try {
|
|
163
|
+
output?.close();
|
|
164
|
+
outputClosed = output !== null;
|
|
165
|
+
}
|
|
166
|
+
catch (error) {
|
|
167
|
+
exitCode = 1;
|
|
168
|
+
const detail = (0, redact_1.redactSecrets)(String(error)).slice(0, 2000);
|
|
169
|
+
dependencies.print(JSON.stringify({ error: `Could not close evaluation output: ${detail}`, v1Qualified: false }));
|
|
170
|
+
(0, runtime_1.emitNervesEvent)({ component: "senses", level: "error", event: "senses.shepherd_judge_evaluation_output_close_error",
|
|
171
|
+
message: "evaluation output close failed", meta: { phase: "judge-classification-only", error: detail } });
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
if (summary && outputClosed) {
|
|
175
|
+
dependencies.print(JSON.stringify(summary));
|
|
176
|
+
(0, runtime_1.emitNervesEvent)({ component: "senses", event: "senses.shepherd_judge_evaluation_command_end",
|
|
177
|
+
message: "classification report persisted and closed", meta: { classificationPassed: summary.classificationPassed, v1Qualified: false } });
|
|
178
|
+
}
|
|
179
|
+
return exitCode;
|
|
180
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
const judge_evaluation_command_1 = require("./judge-evaluation-command");
|
|
4
|
+
const runtime_1 = require("../../nerves/runtime");
|
|
5
|
+
const redact_1 = require("./redact");
|
|
6
|
+
void (0, judge_evaluation_command_1.runShepherdJudgeEvaluationCommand)(process.argv.slice(2)).then(code => { process.exitCode = code; }).catch(error => {
|
|
7
|
+
const detail = (0, redact_1.redactSecrets)(String(error)).slice(0, 2000);
|
|
8
|
+
(0, runtime_1.emitNervesEvent)({ component: "senses", level: "error", event: "senses.shepherd_judge_evaluation_entry_error",
|
|
9
|
+
message: "classification command rejected unexpectedly", meta: { phase: "judge-classification-only", error: detail } });
|
|
10
|
+
// eslint-disable-next-line no-console -- terminal UX: last-resort nonzero evaluation failure
|
|
11
|
+
console.error(JSON.stringify({ error: detail, v1Qualified: false }));
|
|
12
|
+
process.exitCode = 1;
|
|
13
|
+
});
|