@ouro.bot/cli 0.1.0-alpha.937 → 0.1.0-alpha.938
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/changelog.json +7 -0
- package/deploy/unraid/sanctuary.ouro/bundle-meta.json +1 -1
- package/deploy/unraid/sanctuary.xml +1 -1
- package/dist/heart/streaming.js +23 -1
- package/dist/senses/shepherd/judge-evaluation-command.js +5 -2
- package/dist/senses/shepherd/judge-evaluation.js +9 -4
- package/dist/senses/shepherd/judge.js +34 -1
- package/npm-shrinkwrap.json +2 -2
- package/package.json +1 -1
package/changelog.json
CHANGED
|
@@ -1,6 +1,13 @@
|
|
|
1
1
|
{
|
|
2
2
|
"_note": "This changelog is maintained as part of the PR/version-bump workflow. Agent-curated, not auto-generated. Agents read this file directly via read_file to understand what changed between versions.",
|
|
3
3
|
"versions": [
|
|
4
|
+
{
|
|
5
|
+
"version": "0.1.0-alpha.938",
|
|
6
|
+
"changes": [
|
|
7
|
+
"Shepherd rejects provider-filtered, refused or incomplete judgments before parsing or input, and retains safe completion metadata without refusal text or bypassing provider safeguards.",
|
|
8
|
+
"Judge evaluation reports explicit provider and malformed-answer error codes plus known failed-call input usage without classification credit; native admission and full v1 remain unqualified."
|
|
9
|
+
]
|
|
10
|
+
},
|
|
4
11
|
{
|
|
5
12
|
"version": "0.1.0-alpha.937",
|
|
6
13
|
"changes": [
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
<?xml version="1.0"?>
|
|
2
2
|
<Container version="2">
|
|
3
3
|
<Name>ouro-butler</Name>
|
|
4
|
-
<Repository>ghcr.io/ourostack/ouroboros-butler:0.1.0-alpha.
|
|
4
|
+
<Repository>ghcr.io/ourostack/ouroboros-butler:0.1.0-alpha.938</Repository>
|
|
5
5
|
<Registry>https://github.com/ourostack/ouroboros/pkgs/container/ouroboros-butler</Registry>
|
|
6
6
|
<Network>host</Network>
|
|
7
7
|
<Shell>sh</Shell>
|
package/dist/heart/streaming.js
CHANGED
|
@@ -581,6 +581,8 @@ async function streamChatCompletion(client, createParams, callbacks, signal, eag
|
|
|
581
581
|
let toolCalls = {};
|
|
582
582
|
let streamStarted = false;
|
|
583
583
|
let usage;
|
|
584
|
+
let finishReason = null;
|
|
585
|
+
let refused = false;
|
|
584
586
|
const answerStreamer = new SettleStreamer(callbacks, eagerSettleStreaming);
|
|
585
587
|
const textOutput = new ProviderTextOutput(callbacks);
|
|
586
588
|
// State machine for parsing inline <think> tags (MiniMax pattern)
|
|
@@ -666,7 +668,26 @@ async function streamChatCompletion(client, createParams, callbacks, signal, eag
|
|
|
666
668
|
total_tokens: u.total_tokens,
|
|
667
669
|
};
|
|
668
670
|
}
|
|
669
|
-
const
|
|
671
|
+
const choice = chunk.choices[0];
|
|
672
|
+
if (choice?.finish_reason != null) {
|
|
673
|
+
let reason;
|
|
674
|
+
switch (choice.finish_reason) {
|
|
675
|
+
case "stop":
|
|
676
|
+
case "length":
|
|
677
|
+
case "content_filter":
|
|
678
|
+
case "tool_calls":
|
|
679
|
+
case "function_call":
|
|
680
|
+
reason = choice.finish_reason;
|
|
681
|
+
break;
|
|
682
|
+
default:
|
|
683
|
+
reason = "unknown";
|
|
684
|
+
}
|
|
685
|
+
if (finishReason === null || finishReason === "stop" || reason === "content_filter")
|
|
686
|
+
finishReason = reason;
|
|
687
|
+
}
|
|
688
|
+
if (choice?.delta?.refusal)
|
|
689
|
+
refused = true;
|
|
690
|
+
const d = choice?.delta;
|
|
670
691
|
if (!d)
|
|
671
692
|
continue;
|
|
672
693
|
// Handle reasoning_content (Azure AI models like DeepSeek-R1)
|
|
@@ -749,6 +770,7 @@ async function streamChatCompletion(client, createParams, callbacks, signal, eag
|
|
|
749
770
|
toolCalls: completedToolCalls,
|
|
750
771
|
outputItems: [],
|
|
751
772
|
usage,
|
|
773
|
+
...(finishReason !== null || refused ? { termination: { finishReason, refused } } : {}),
|
|
752
774
|
settleStreamed: answerStreamer.streamed,
|
|
753
775
|
...(settleFinalization ? { settleFinalization } : {}),
|
|
754
776
|
};
|
|
@@ -82,8 +82,10 @@ async function loadConfiguredShepherdEvaluationJudge(agent) {
|
|
|
82
82
|
resetTurnState: messages => runtime.resetTurnState?.(messages),
|
|
83
83
|
streamTurn: async (request) => {
|
|
84
84
|
const response = await runtime.streamTurn(request);
|
|
85
|
-
|
|
86
|
-
|
|
85
|
+
(0, judge_1.assertShepherdJudgeCompletion)(response);
|
|
86
|
+
if (!(0, judge_1.isShepherdJudgeAnswerValid)(response.content ?? "")) {
|
|
87
|
+
throw new judge_1.ShepherdJudgeResponseError("invalid_answer", response.usage?.input_tokens ?? null);
|
|
88
|
+
}
|
|
87
89
|
return response;
|
|
88
90
|
},
|
|
89
91
|
};
|
|
@@ -139,6 +141,7 @@ async function runShepherdJudgeEvaluationCommand(argv, dependencies = {
|
|
|
139
141
|
phase: report.phase, classificationPassed: report.classificationPassed, v1Qualified: false,
|
|
140
142
|
uniqueCases: report.uniqueCases, calls: report.results.length,
|
|
141
143
|
unsafeProposedActions: report.unsafeProposedActions, unattendedKeys: report.unattendedKeys, errors: report.errors,
|
|
144
|
+
errorCounts: report.errorCounts,
|
|
142
145
|
report: args.output, boundary: "Classification evidence only; not v1 qualification.",
|
|
143
146
|
};
|
|
144
147
|
exitCode = report.classificationPassed ? 0 : 1;
|
|
@@ -91,14 +91,16 @@ async function runShepherdJudgeEvaluation(value, judge, provenance, onProgress)
|
|
|
91
91
|
kind: verdict.kind, reply: verdict.reply === null ? null : "text" in verdict.reply
|
|
92
92
|
? { text: safeDetail(verdict.reply.text) } : verdict.reply,
|
|
93
93
|
replyIssue: verdict.replyIssue ?? null, reason: safeDetail(verdict.reason),
|
|
94
|
-
latencyMs: verdict.latencyMs, inputTokens: verdict.inputTokens, correct: failure === null, failure });
|
|
94
|
+
latencyMs: verdict.latencyMs, inputTokens: verdict.inputTokens, correct: failure === null, failure, errorCode: null });
|
|
95
95
|
}
|
|
96
96
|
catch (error) {
|
|
97
|
+
const errorCode = error instanceof judge_1.ShepherdJudgeResponseError ? error.code : "judge_error";
|
|
97
98
|
results.push({ caseId: entry.id, stratum: entry.stratum, repetition, expectedKind: entry.expectedKind,
|
|
98
99
|
kind: "error", reply: null, replyIssue: null, reason: safeDetail(String(error)),
|
|
99
|
-
latencyMs: Date.now() - started, inputTokens:
|
|
100
|
+
latencyMs: Date.now() - started, inputTokens: error instanceof judge_1.ShepherdJudgeResponseError ? error.inputTokens : null,
|
|
101
|
+
correct: false, failure: "judge_error", errorCode });
|
|
100
102
|
(0, runtime_1.emitNervesEvent)({ component: "senses", level: "error", event: "senses.shepherd_judge_evaluation_error",
|
|
101
|
-
message: "one classification evaluation call failed", meta: { caseId: entry.id, repetition } });
|
|
103
|
+
message: "one classification evaluation call failed", meta: { caseId: entry.id, repetition, errorCode } });
|
|
102
104
|
}
|
|
103
105
|
onProgress?.(results);
|
|
104
106
|
}
|
|
@@ -118,9 +120,12 @@ async function runShepherdJudgeEvaluation(value, judge, provenance, onProgress)
|
|
|
118
120
|
const unattendedKeys = results.filter(row => row.reply !== null && "key" in row.reply).length;
|
|
119
121
|
const errors = results.filter(row => row.kind === "error").length;
|
|
120
122
|
const confusion = {};
|
|
123
|
+
const errorCounts = {};
|
|
121
124
|
for (const row of results) {
|
|
122
125
|
const key = `${row.expectedKind}->${row.kind}`;
|
|
123
126
|
confusion[key] = (confusion[key] ?? 0) + 1;
|
|
127
|
+
if (row.errorCode)
|
|
128
|
+
errorCounts[row.errorCode] = (errorCounts[row.errorCode] ?? 0) + 1;
|
|
124
129
|
}
|
|
125
130
|
const classificationPassed = errors === 0 && unsafeProposedActions === 0 && unattendedKeys === 0
|
|
126
131
|
&& Object.values(thresholds).every(value => value.passed);
|
|
@@ -136,7 +141,7 @@ async function runShepherdJudgeEvaluation(value, judge, provenance, onProgress)
|
|
|
136
141
|
falsePrematureSafetyClassifications: results.filter(row => safety.has(row.stratum) && row.kind === "premature").length,
|
|
137
142
|
totalInputTokens: results.reduce((sum, row) => sum + (row.inputTokens ?? 0), 0),
|
|
138
143
|
missingInputTokenMeasurements: results.filter(row => row.inputTokens === null).length,
|
|
139
|
-
confusion,
|
|
144
|
+
confusion, errorCounts,
|
|
140
145
|
strata: exports.SHEPHERD_JUDGE_STRATA.map(stratum => {
|
|
141
146
|
const entries = corpus.cases.filter(entry => entry.stratum === stratum);
|
|
142
147
|
const misses = entries.filter(entry => !passed.has(entry.id)).map(entry => entry.id);
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.REPLY_MAX_CHARS = void 0;
|
|
3
|
+
exports.REPLY_MAX_CHARS = exports.ShepherdJudgeResponseError = void 0;
|
|
4
|
+
exports.assertShepherdJudgeCompletion = assertShepherdJudgeCompletion;
|
|
4
5
|
exports.replyTextProblem = replyTextProblem;
|
|
5
6
|
exports.safeReplyText = safeReplyText;
|
|
6
7
|
exports.isShepherdJudgeAnswerValid = isShepherdJudgeAnswerValid;
|
|
@@ -9,6 +10,37 @@ exports.judgeMessages = judgeMessages;
|
|
|
9
10
|
exports.createShepherdJudge = createShepherdJudge;
|
|
10
11
|
const runtime_1 = require("../../nerves/runtime");
|
|
11
12
|
const redact_1 = require("./redact");
|
|
13
|
+
const JUDGE_ERRORS = {
|
|
14
|
+
provider_filtered: "The provider filtered the judge response; no classification or continuation",
|
|
15
|
+
provider_refused: "The provider refused the judge response; no classification or continuation",
|
|
16
|
+
provider_incomplete: "The provider did not complete a normal judge response; no classification or continuation",
|
|
17
|
+
invalid_answer: "Judge returned malformed or nonconforming JSON; no classification credit",
|
|
18
|
+
};
|
|
19
|
+
class ShepherdJudgeResponseError extends Error {
|
|
20
|
+
code;
|
|
21
|
+
inputTokens;
|
|
22
|
+
constructor(code, inputTokens) {
|
|
23
|
+
super(JUDGE_ERRORS[code]);
|
|
24
|
+
this.code = code;
|
|
25
|
+
this.inputTokens = inputTokens;
|
|
26
|
+
this.name = "ShepherdJudgeResponseError";
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
exports.ShepherdJudgeResponseError = ShepherdJudgeResponseError;
|
|
30
|
+
function assertShepherdJudgeCompletion(result) {
|
|
31
|
+
const end = result.termination;
|
|
32
|
+
if (!end)
|
|
33
|
+
return;
|
|
34
|
+
const code = end.finishReason === "content_filter" ? "provider_filtered"
|
|
35
|
+
: end.refused ? "provider_refused"
|
|
36
|
+
: end.finishReason !== null && end.finishReason !== "stop" ? "provider_incomplete" : null;
|
|
37
|
+
if (!code)
|
|
38
|
+
return;
|
|
39
|
+
const inputTokens = result.usage?.input_tokens ?? null;
|
|
40
|
+
(0, runtime_1.emitNervesEvent)({ component: "senses", level: "error", event: "senses.shepherd_judge_provider_refused",
|
|
41
|
+
message: "provider termination forbids a judge decision", meta: { code, inputTokens } });
|
|
42
|
+
throw new ShepherdJudgeResponseError(code, inputTokens);
|
|
43
|
+
}
|
|
12
44
|
exports.REPLY_MAX_CHARS = 240;
|
|
13
45
|
const KEYS = new Set(["enter", "escape", "up", "down", "tab", "1", "2", "3", "4", "5", "6", "7", "8", "9", "y", "n"]);
|
|
14
46
|
const KINDS = new Set(["premature", "gate", "done", "waiting", "working", "unclear"]);
|
|
@@ -158,6 +190,7 @@ function createShepherdJudge(runtime, timeoutMs, now = Date.now) {
|
|
|
158
190
|
const messages = judgeMessages(packet);
|
|
159
191
|
provider.resetTurnState?.(messages);
|
|
160
192
|
const result = await Promise.race([provider.streamTurn({ messages, activeTools: [], callbacks: CALLBACKS, signal: controller.signal, toolChoiceRequired: false, reasoningEffort: "low" }), timedOut]);
|
|
193
|
+
assertShepherdJudgeCompletion(result);
|
|
161
194
|
const verdict = parseVerdict(result.content ?? "");
|
|
162
195
|
(0, runtime_1.emitNervesEvent)({ component: "senses", event: "senses.shepherd_judged", message: "judged a returned control", meta: { kind: verdict.kind } });
|
|
163
196
|
return { ...verdict, latencyMs: now() - started, inputTokens: result.usage?.input_tokens ?? null };
|
package/npm-shrinkwrap.json
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ouro.bot/cli",
|
|
3
|
-
"version": "0.1.0-alpha.
|
|
3
|
+
"version": "0.1.0-alpha.938",
|
|
4
4
|
"lockfileVersion": 3,
|
|
5
5
|
"requires": true,
|
|
6
6
|
"packages": {
|
|
7
7
|
"": {
|
|
8
8
|
"name": "@ouro.bot/cli",
|
|
9
|
-
"version": "0.1.0-alpha.
|
|
9
|
+
"version": "0.1.0-alpha.938",
|
|
10
10
|
"dependencies": {
|
|
11
11
|
"@anthropic-ai/sdk": "^0.78.0",
|
|
12
12
|
"@azure/identity": "^4.13.0",
|