@openclawbrain/cli 0.4.23 → 0.4.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/src/cli.js
CHANGED
|
@@ -25,7 +25,7 @@ import { summarizePackVectorEmbeddingState } from "./embedding-status.js";
|
|
|
25
25
|
import { buildTracedLearningBridgePayloadFromRuntime, buildTracedLearningStatusSurface, persistTracedLearningBridgeState } from "./traced-learning-bridge.js";
|
|
26
26
|
import { discoverOpenClawSessionStores, loadOpenClawSessionIndex, readOpenClawSessionFile } from "./session-store.js";
|
|
27
27
|
import { readOpenClawBrainProviderDefaults, readOpenClawBrainProviderConfig, readOpenClawBrainProviderConfigFromSources, resolveOpenClawBrainProviderDefaultsPath } from "./provider-config.js";
|
|
28
|
-
import { formatOperatorLearningAttributionSummary, formatOperatorLearningPathSummary } from "./status-learning-path.js";
|
|
28
|
+
import { formatOperatorAttributionCoverageSummary, formatOperatorFeedbackSummary, formatOperatorLearningAttributionSummary, formatOperatorLearningPathSummary } from "./status-learning-path.js";
|
|
29
29
|
import { buildProofCommandForOpenClawHome, buildProofCommandHelpSection, captureOperatorProofBundle, formatOperatorProofResult, parseProofCliArgs } from "./proof-command.js";
|
|
30
30
|
const OPENCLAWBRAIN_EMBEDDER_BASE_URL_ENV = "OPENCLAWBRAIN_EMBEDDER_BASE_URL";
|
|
31
31
|
const OPENCLAWBRAIN_EMBEDDER_PROVIDER_ENV = "OPENCLAWBRAIN_EMBEDDER_PROVIDER";
|
|
@@ -1500,7 +1500,9 @@ function formatCurrentProfileStatusSummary(status, report, targetInspection, opt
|
|
|
1500
1500
|
learningPath: report.learningPath,
|
|
1501
1501
|
tracedLearning
|
|
1502
1502
|
})}`,
|
|
1503
|
+
`feedback ${formatOperatorFeedbackSummary({ tracedLearning })}`,
|
|
1503
1504
|
`attribution ${formatOperatorLearningAttributionSummary({ status })}`,
|
|
1505
|
+
`attrCover ${formatOperatorAttributionCoverageSummary({ tracedLearning })}`,
|
|
1504
1506
|
`learning state=${report.learning.backlogState} bootstrapped=${yesNo(report.learning.bootstrapped)} mode=${report.learning.mode} next=${report.learning.nextPriorityLane} priority=${report.learning.nextPriorityBucket} pending=${report.learning.pendingLive ?? "none"}/${report.learning.pendingBackfill ?? "none"} buckets=${formatLearningBuckets(report)} warn=${formatLearningWarnings(report)} lastPack=${report.learning.lastMaterializedPackId ?? "none"} detail=${report.learning.detail}`,
|
|
1505
1507
|
`traced ${formatTracedLearningSurface(tracedLearning)}`,
|
|
1506
1508
|
`teacherProof ${formatTeacherLoopSummary(report)}`,
|
|
@@ -76,6 +76,41 @@ function writeJson(filePath, value) {
|
|
|
76
76
|
writeText(filePath, `${JSON.stringify(value, null, 2)}\n`);
|
|
77
77
|
}
|
|
78
78
|
|
|
79
|
+
function readJsonObject(value) {
|
|
80
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) {
|
|
81
|
+
return null;
|
|
82
|
+
}
|
|
83
|
+
return value;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function parseJsonObjectText(text) {
|
|
87
|
+
if (typeof text !== "string" || text.trim().length === 0) {
|
|
88
|
+
return null;
|
|
89
|
+
}
|
|
90
|
+
try {
|
|
91
|
+
return readJsonObject(JSON.parse(text));
|
|
92
|
+
}
|
|
93
|
+
catch {
|
|
94
|
+
return null;
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
function normalizeOptionalBoolean(value) {
|
|
99
|
+
return value === true ? true : value === false ? false : null;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
function extractInstallRestartState(text) {
|
|
103
|
+
const parsed = parseJsonObjectText(text);
|
|
104
|
+
const restart = readJsonObject(parsed?.restart ?? null);
|
|
105
|
+
if (restart === null) {
|
|
106
|
+
return null;
|
|
107
|
+
}
|
|
108
|
+
return {
|
|
109
|
+
required: normalizeOptionalBoolean(restart.required),
|
|
110
|
+
performed: normalizeOptionalBoolean(restart.performed),
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
|
|
79
114
|
function buildCurrentCliInvocation(cliEntryPath = process.argv[1]) {
|
|
80
115
|
const normalizedEntryPath = normalizeOptionalCliString(cliEntryPath);
|
|
81
116
|
if (normalizedEntryPath === null) {
|
|
@@ -511,7 +546,7 @@ function buildVerdict({ steps, gatewayStatus, pluginInspect, statusSignals, brea
|
|
|
511
546
|
};
|
|
512
547
|
}
|
|
513
548
|
|
|
514
|
-
function buildSummary({ options, steps, verdict, gatewayStatusText, pluginInspectText, statusSignals, breadcrumbs, runtimeLoadProofSnapshot, guardLine, attributionLine, learningPathLine, coverageSnapshot, hardeningSnapshot }) {
|
|
549
|
+
function buildSummary({ options, steps, verdict, gatewayStatusText, pluginInspectText, statusSignals, breadcrumbs, runtimeLoadProofSnapshot, guardLine, feedbackLine, attributionLine, attributionCoverageLine, learningPathLine, coverageSnapshot, hardeningSnapshot }) {
|
|
515
550
|
const passed = [];
|
|
516
551
|
const missing = [];
|
|
517
552
|
const warnings = Array.isArray(verdict.warnings) ? verdict.warnings : [];
|
|
@@ -573,9 +608,15 @@ function buildSummary({ options, steps, verdict, gatewayStatusText, pluginInspec
|
|
|
573
608
|
: [`- ${guardLine}`]),
|
|
574
609
|
"",
|
|
575
610
|
"## Learning Attribution",
|
|
611
|
+
...(feedbackLine === null
|
|
612
|
+
? ["- feedback line not reported by detailed status"]
|
|
613
|
+
: [`- ${feedbackLine}`]),
|
|
576
614
|
...(attributionLine === null
|
|
577
615
|
? ["- attribution line not reported by detailed status"]
|
|
578
616
|
: [`- ${attributionLine}`]),
|
|
617
|
+
...(attributionCoverageLine === null
|
|
618
|
+
? ["- attribution coverage line not reported by detailed status"]
|
|
619
|
+
: [`- ${attributionCoverageLine}`]),
|
|
579
620
|
...(learningPathLine === null
|
|
580
621
|
? []
|
|
581
622
|
: [`- ${learningPathLine}`]),
|
|
@@ -802,7 +843,7 @@ export function captureOperatorProofBundle(options) {
|
|
|
802
843
|
mkdirSync(bundleDir, { recursive: true });
|
|
803
844
|
const steps = [];
|
|
804
845
|
const gatewayProfile = readOpenClawProfileName(options.openclawHome);
|
|
805
|
-
function addStep(stepId, label, command, args, { skipped = false } = {}) {
|
|
846
|
+
function addStep(stepId, label, command, args, { skipped = false, skipSummary = "step intentionally skipped" } = {}) {
|
|
806
847
|
if (skipped) {
|
|
807
848
|
steps.push({
|
|
808
849
|
stepId,
|
|
@@ -811,7 +852,7 @@ export function captureOperatorProofBundle(options) {
|
|
|
811
852
|
skipped: true,
|
|
812
853
|
captureState: "complete",
|
|
813
854
|
resultClass: "success",
|
|
814
|
-
summary:
|
|
855
|
+
summary: skipSummary,
|
|
815
856
|
stdoutPath: null,
|
|
816
857
|
stderrPath: null,
|
|
817
858
|
});
|
|
@@ -844,8 +885,27 @@ export function captureOperatorProofBundle(options) {
|
|
|
844
885
|
});
|
|
845
886
|
return capture;
|
|
846
887
|
}
|
|
847
|
-
addStep("01-install", "install", cliInvocation.command, [...cliInvocation.args, "install", "--openclaw-home", options.openclawHome], { skipped: options.skipInstall === true });
|
|
848
|
-
|
|
888
|
+
const installCapture = addStep("01-install", "install", cliInvocation.command, [...cliInvocation.args, "install", "--openclaw-home", options.openclawHome, "--json"], { skipped: options.skipInstall === true });
|
|
889
|
+
const installRestartState = options.skipInstall === true || installCapture.exitCode !== 0 || installCapture.error
|
|
890
|
+
? null
|
|
891
|
+
: extractInstallRestartState(installCapture.stdout);
|
|
892
|
+
let restartSkipSummary = null;
|
|
893
|
+
if (options.skipRestart === true) {
|
|
894
|
+
restartSkipSummary = "step intentionally skipped";
|
|
895
|
+
}
|
|
896
|
+
else if (installRestartState?.performed === true) {
|
|
897
|
+
restartSkipSummary = "step intentionally skipped because install already performed the gateway restart";
|
|
898
|
+
}
|
|
899
|
+
else if (installRestartState?.required === false) {
|
|
900
|
+
restartSkipSummary = "step intentionally skipped because install reported no gateway restart was required";
|
|
901
|
+
}
|
|
902
|
+
else if (gatewayProfile === null) {
|
|
903
|
+
restartSkipSummary = "skipped because exact OpenClaw profile token could not be inferred; avoiding shared-gateway self-interrupt during proof capture";
|
|
904
|
+
}
|
|
905
|
+
addStep("02-restart", "gateway restart", "openclaw", buildGatewayArgs("restart", gatewayProfile), {
|
|
906
|
+
skipped: restartSkipSummary !== null,
|
|
907
|
+
skipSummary: restartSkipSummary ?? undefined
|
|
908
|
+
});
|
|
849
909
|
const gatewayStatusCapture = addStep("03-gateway-status", "gateway status", "openclaw", buildGatewayStatusArgs(
|
|
850
910
|
gatewayProfile,
|
|
851
911
|
normalizeOptionalCliString(options.gatewayUrl ?? null),
|
|
@@ -861,7 +921,9 @@ export function captureOperatorProofBundle(options) {
|
|
|
861
921
|
const serveLine = extractDetailedStatusLine(statusCapture.stdout, "serve");
|
|
862
922
|
const routeFnLine = extractDetailedStatusLine(statusCapture.stdout, "routeFn");
|
|
863
923
|
const guardLine = extractDetailedStatusLine(statusCapture.stdout, "guard");
|
|
924
|
+
const feedbackLine = extractDetailedStatusLine(statusCapture.stdout, "feedback");
|
|
864
925
|
const attributionLine = extractDetailedStatusLine(statusCapture.stdout, "attribution");
|
|
926
|
+
const attributionCoverageLine = extractDetailedStatusLine(statusCapture.stdout, "attrCover");
|
|
865
927
|
const learningPathLine = extractDetailedStatusLine(statusCapture.stdout, "path");
|
|
866
928
|
const runtimeLoadProofPath = normalizeReportedProofPath(statusSignals.proofPath)
|
|
867
929
|
?? path.join(activationRoot, "attachment-truth", "runtime-load-proofs.json");
|
|
@@ -917,7 +979,9 @@ export function captureOperatorProofBundle(options) {
|
|
|
917
979
|
runtimeLoadProofPath,
|
|
918
980
|
runtimeLoadProofError: runtimeLoadProofSnapshot.error,
|
|
919
981
|
guardLine,
|
|
982
|
+
feedbackLine,
|
|
920
983
|
attributionLine,
|
|
984
|
+
attributionCoverageLine,
|
|
921
985
|
learningPathLine,
|
|
922
986
|
});
|
|
923
987
|
writeJson(path.join(bundleDir, "hardening-snapshot.json"), hardeningSnapshot);
|
|
@@ -931,7 +995,9 @@ export function captureOperatorProofBundle(options) {
|
|
|
931
995
|
breadcrumbs,
|
|
932
996
|
runtimeLoadProofSnapshot,
|
|
933
997
|
guardLine,
|
|
998
|
+
feedbackLine,
|
|
934
999
|
attributionLine,
|
|
1000
|
+
attributionCoverageLine,
|
|
935
1001
|
learningPathLine,
|
|
936
1002
|
coverageSnapshot,
|
|
937
1003
|
hardeningSnapshot,
|
|
@@ -950,7 +1016,9 @@ export function captureOperatorProofBundle(options) {
|
|
|
950
1016
|
verdict,
|
|
951
1017
|
statusSignals,
|
|
952
1018
|
guardLine,
|
|
1019
|
+
feedbackLine,
|
|
953
1020
|
attributionLine,
|
|
1021
|
+
attributionCoverageLine,
|
|
954
1022
|
learningPathLine,
|
|
955
1023
|
steps,
|
|
956
1024
|
summaryPath: path.join(bundleDir, "summary.md"),
|
|
@@ -7,6 +7,29 @@ function isSeedAwaitingFirstPromotion(status) {
|
|
|
7
7
|
function normalizeOptionalString(value) {
|
|
8
8
|
return typeof value === "string" && value.trim().length > 0 ? value : null;
|
|
9
9
|
}
|
|
10
|
+
function formatOptionalFeedbackLatest(tracedLearning) {
|
|
11
|
+
const latestLabel = normalizeOptionalString(tracedLearning?.feedbackSummary?.latestLabel);
|
|
12
|
+
return latestLabel === null ? "" : ` latest=${latestLabel}`;
|
|
13
|
+
}
|
|
14
|
+
function formatOperatorFeedbackSummary({ tracedLearning }) {
|
|
15
|
+
const routeTraceCount = tracedLearning?.feedbackSummary?.routeTraceCount ?? tracedLearning?.routeTraceCount ?? 0;
|
|
16
|
+
const supervisedTraceCount = tracedLearning?.feedbackSummary?.supervisedTraceCount ?? tracedLearning?.supervisionCount ?? 0;
|
|
17
|
+
return [
|
|
18
|
+
`helpful=${tracedLearning?.feedbackSummary?.helpfulCount ?? 0}`,
|
|
19
|
+
`irrelevant=${tracedLearning?.feedbackSummary?.irrelevantCount ?? 0}`,
|
|
20
|
+
`harmful=${tracedLearning?.feedbackSummary?.harmfulCount ?? 0}`,
|
|
21
|
+
`supervisedTraceCount=${supervisedTraceCount}`,
|
|
22
|
+
`routeTraceCount=${routeTraceCount}`
|
|
23
|
+
].join(" ") + formatOptionalFeedbackLatest(tracedLearning);
|
|
24
|
+
}
|
|
25
|
+
function formatOperatorAttributionCoverageSummary({ tracedLearning }) {
|
|
26
|
+
return [
|
|
27
|
+
`completedWithoutEvaluation=${tracedLearning?.attributionCoverage?.completedWithoutEvaluationCount ?? 0}`,
|
|
28
|
+
`ready=${tracedLearning?.attributionCoverage?.readyCount ?? 0}`,
|
|
29
|
+
`delayed=${tracedLearning?.attributionCoverage?.delayedCount ?? 0}`,
|
|
30
|
+
`budgetDeferred=${tracedLearning?.attributionCoverage?.budgetDeferredCount ?? 0}`
|
|
31
|
+
].join(" ");
|
|
32
|
+
}
|
|
10
33
|
function formatOperatorLearningAttributionSummary({ status }) {
|
|
11
34
|
const attribution = status?.learningAttribution ?? null;
|
|
12
35
|
if (!attribution) {
|
|
@@ -55,4 +78,4 @@ export function formatOperatorLearningPathSummary({ status, learningPath, traced
|
|
|
55
78
|
...detailParts
|
|
56
79
|
].join(" ");
|
|
57
80
|
}
|
|
58
|
-
export { formatOperatorLearningAttributionSummary };
|
|
81
|
+
export { formatOperatorAttributionCoverageSummary, formatOperatorFeedbackSummary, formatOperatorLearningAttributionSummary };
|
|
@@ -15,12 +15,249 @@ function normalizeCount(value) {
|
|
|
15
15
|
function normalizeOptionalString(value) {
|
|
16
16
|
return typeof value === "string" && value.trim().length > 0 ? value : null;
|
|
17
17
|
}
|
|
18
|
+
function parseJsonValue(value, fallback) {
|
|
19
|
+
if (typeof value !== "string" || value.trim().length === 0) {
|
|
20
|
+
return fallback;
|
|
21
|
+
}
|
|
22
|
+
try {
|
|
23
|
+
return JSON.parse(value);
|
|
24
|
+
}
|
|
25
|
+
catch {
|
|
26
|
+
return fallback;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
18
29
|
function normalizeUnitInterval(value) {
|
|
19
30
|
return Number.isFinite(value) ? Math.max(0, Math.min(1, Number(value))) : 0;
|
|
20
31
|
}
|
|
21
32
|
function normalizeSource(value) {
|
|
22
33
|
return value !== null && typeof value === "object" && !Array.isArray(value) ? value : null;
|
|
23
34
|
}
|
|
35
|
+
function toRecord(value) {
|
|
36
|
+
return value !== null && typeof value === "object" && !Array.isArray(value) ? value : null;
|
|
37
|
+
}
|
|
38
|
+
function normalizeAgentIdentity(value) {
|
|
39
|
+
const record = toRecord(value);
|
|
40
|
+
const agentId = normalizeOptionalString(record?.agentId);
|
|
41
|
+
const lane = normalizeOptionalString(record?.lane);
|
|
42
|
+
return agentId === null || lane === null ? null : { agentId, lane };
|
|
43
|
+
}
|
|
44
|
+
function formatAgentIdentity(identity) {
|
|
45
|
+
if (identity === null) {
|
|
46
|
+
return null;
|
|
47
|
+
}
|
|
48
|
+
return identity.lane === "main" ? identity.agentId : `${identity.agentId}:${identity.lane}`;
|
|
49
|
+
}
|
|
50
|
+
function defaultFeedbackSummary(routeTraceCount = 0, supervisedTraceCount = 0, detail = "feedback truth is not visible in the current status surface") {
|
|
51
|
+
return {
|
|
52
|
+
visible: false,
|
|
53
|
+
helpfulCount: 0,
|
|
54
|
+
irrelevantCount: 0,
|
|
55
|
+
harmfulCount: 0,
|
|
56
|
+
supervisedTraceCount,
|
|
57
|
+
routeTraceCount,
|
|
58
|
+
latestAgentIdentity: null,
|
|
59
|
+
latestLabel: null,
|
|
60
|
+
detail
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
function normalizeFeedbackSummary(value, counts = {}) {
|
|
64
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) {
|
|
65
|
+
return defaultFeedbackSummary(normalizeCount(counts.routeTraceCount), normalizeCount(counts.supervisedTraceCount));
|
|
66
|
+
}
|
|
67
|
+
const routeTraceCount = normalizeCount(value.routeTraceCount ?? counts.routeTraceCount);
|
|
68
|
+
const supervisedTraceCount = normalizeCount(value.supervisedTraceCount ?? counts.supervisedTraceCount);
|
|
69
|
+
const latestAgentIdentity = normalizeAgentIdentity(value.latestAgentIdentity);
|
|
70
|
+
return {
|
|
71
|
+
visible: value.visible === true,
|
|
72
|
+
helpfulCount: normalizeCount(value.helpfulCount),
|
|
73
|
+
irrelevantCount: normalizeCount(value.irrelevantCount),
|
|
74
|
+
harmfulCount: normalizeCount(value.harmfulCount),
|
|
75
|
+
supervisedTraceCount,
|
|
76
|
+
routeTraceCount,
|
|
77
|
+
latestAgentIdentity,
|
|
78
|
+
latestLabel: normalizeOptionalString(value.latestLabel) ?? formatAgentIdentity(latestAgentIdentity),
|
|
79
|
+
detail: normalizeOptionalString(value.detail)
|
|
80
|
+
?? (routeTraceCount === 0
|
|
81
|
+
? "no traced routes recorded yet"
|
|
82
|
+
: `${supervisedTraceCount}/${routeTraceCount} traced routes are covered by live verdicts`)
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
function defaultAttributionCoverage(detail = "teacher gating truth is not visible in the current status surface") {
|
|
86
|
+
return {
|
|
87
|
+
visible: false,
|
|
88
|
+
gatingVisible: false,
|
|
89
|
+
completedWithoutEvaluationCount: 0,
|
|
90
|
+
readyCount: 0,
|
|
91
|
+
delayedCount: 0,
|
|
92
|
+
budgetDeferredCount: 0,
|
|
93
|
+
detail
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
function normalizeAttributionCoverage(value) {
|
|
97
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) {
|
|
98
|
+
return defaultAttributionCoverage();
|
|
99
|
+
}
|
|
100
|
+
return {
|
|
101
|
+
visible: value.visible === true,
|
|
102
|
+
gatingVisible: value.gatingVisible === true,
|
|
103
|
+
completedWithoutEvaluationCount: normalizeCount(value.completedWithoutEvaluationCount),
|
|
104
|
+
readyCount: normalizeCount(value.readyCount),
|
|
105
|
+
delayedCount: normalizeCount(value.delayedCount),
|
|
106
|
+
budgetDeferredCount: normalizeCount(value.budgetDeferredCount),
|
|
107
|
+
detail: normalizeOptionalString(value.detail)
|
|
108
|
+
?? "teacher gating truth is not visible in the current status surface"
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
function hasNonEmptyToolResults(value) {
|
|
112
|
+
if (Array.isArray(value)) {
|
|
113
|
+
return value.length > 0;
|
|
114
|
+
}
|
|
115
|
+
return typeof value === "string" && value.trim().length > 0 && value.trim() !== "[]";
|
|
116
|
+
}
|
|
117
|
+
function hasTeacherBindingMode(rawEvaluation) {
|
|
118
|
+
const evaluation = parseJsonValue(rawEvaluation, null);
|
|
119
|
+
switch (evaluation?.bindingMode) {
|
|
120
|
+
case "exact_decision_id":
|
|
121
|
+
case "exact_selection_digest":
|
|
122
|
+
case "turn_compile_event_id":
|
|
123
|
+
case "trace_id":
|
|
124
|
+
case "legacy_heuristic":
|
|
125
|
+
case "unbound":
|
|
126
|
+
return true;
|
|
127
|
+
default:
|
|
128
|
+
return false;
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
function classifyContextFeedbackVerdict(score) {
|
|
132
|
+
if (Number(score) >= 0.25) {
|
|
133
|
+
return "helpful";
|
|
134
|
+
}
|
|
135
|
+
if (Number(score) <= -0.25) {
|
|
136
|
+
return "harmful";
|
|
137
|
+
}
|
|
138
|
+
return "irrelevant";
|
|
139
|
+
}
|
|
140
|
+
function isObservationReadyForTeacher(row, readyBefore) {
|
|
141
|
+
return row.status === "pending_teacher"
|
|
142
|
+
|| normalizeOptionalString(row.follow_up_text) !== null
|
|
143
|
+
|| hasNonEmptyToolResults(row.tool_results_json)
|
|
144
|
+
|| Number(row.created_at ?? 0) <= readyBefore;
|
|
145
|
+
}
|
|
146
|
+
function buildDerivedFeedbackSummary(db, routeTraceCount, defaultSupervisionCount) {
|
|
147
|
+
const traceRows = db.prepare(`
|
|
148
|
+
SELECT id, route_trace_json
|
|
149
|
+
FROM brain_traces
|
|
150
|
+
`).all();
|
|
151
|
+
const traceAgentIdentityById = new Map();
|
|
152
|
+
for (const row of traceRows) {
|
|
153
|
+
const routeTrace = parseJsonValue(row?.route_trace_json, null);
|
|
154
|
+
const agentIdentity = normalizeAgentIdentity(routeTrace?.agentIdentity);
|
|
155
|
+
if (agentIdentity !== null && typeof row?.id === "string") {
|
|
156
|
+
traceAgentIdentityById.set(row.id, agentIdentity);
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
const verdictCounts = {
|
|
160
|
+
helpfulCount: 0,
|
|
161
|
+
irrelevantCount: 0,
|
|
162
|
+
harmfulCount: 0
|
|
163
|
+
};
|
|
164
|
+
const latestTraceIds = new Set();
|
|
165
|
+
let latestAgentIdentity = null;
|
|
166
|
+
const supervisionRows = db.prepare(`
|
|
167
|
+
SELECT trace_id, metadata, value
|
|
168
|
+
FROM brain_trace_supervision
|
|
169
|
+
WHERE resolution = 'promoted_to_label'
|
|
170
|
+
ORDER BY created_at DESC
|
|
171
|
+
`).all();
|
|
172
|
+
for (const row of supervisionRows) {
|
|
173
|
+
const traceId = normalizeOptionalString(row?.trace_id);
|
|
174
|
+
if (traceId === null || latestTraceIds.has(traceId)) {
|
|
175
|
+
continue;
|
|
176
|
+
}
|
|
177
|
+
latestTraceIds.add(traceId);
|
|
178
|
+
const metadata = parseJsonValue(row?.metadata, {});
|
|
179
|
+
const agentIdentity = normalizeAgentIdentity(metadata?.agentIdentity)
|
|
180
|
+
?? traceAgentIdentityById.get(traceId)
|
|
181
|
+
?? null;
|
|
182
|
+
if (latestAgentIdentity === null) {
|
|
183
|
+
latestAgentIdentity = agentIdentity;
|
|
184
|
+
}
|
|
185
|
+
const verdict = classifyContextFeedbackVerdict(Number(row?.value ?? 0));
|
|
186
|
+
if (verdict === "helpful") {
|
|
187
|
+
verdictCounts.helpfulCount += 1;
|
|
188
|
+
}
|
|
189
|
+
else if (verdict === "harmful") {
|
|
190
|
+
verdictCounts.harmfulCount += 1;
|
|
191
|
+
}
|
|
192
|
+
else {
|
|
193
|
+
verdictCounts.irrelevantCount += 1;
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
const supervisedTraceCount = latestTraceIds.size || normalizeCount(defaultSupervisionCount);
|
|
197
|
+
return {
|
|
198
|
+
visible: true,
|
|
199
|
+
...verdictCounts,
|
|
200
|
+
supervisedTraceCount,
|
|
201
|
+
routeTraceCount,
|
|
202
|
+
latestAgentIdentity,
|
|
203
|
+
latestLabel: formatAgentIdentity(latestAgentIdentity),
|
|
204
|
+
detail: routeTraceCount === 0
|
|
205
|
+
? "no traced routes recorded yet"
|
|
206
|
+
: `${verdictCounts.helpfulCount} helpful, ${verdictCounts.irrelevantCount} irrelevant, ${verdictCounts.harmfulCount} harmful; ${supervisedTraceCount}/${routeTraceCount} traced routes are supervised`
|
|
207
|
+
};
|
|
208
|
+
}
|
|
209
|
+
function buildDerivedAttributionCoverage(db) {
|
|
210
|
+
const completedRows = db.prepare(`
|
|
211
|
+
SELECT teacher_evaluation_json
|
|
212
|
+
FROM brain_observations
|
|
213
|
+
WHERE status = 'completed'
|
|
214
|
+
`).all();
|
|
215
|
+
const completedWithoutEvaluationCount = completedRows.reduce((sum, row) => sum + (hasTeacherBindingMode(row?.teacher_evaluation_json) ? 0 : 1), 0);
|
|
216
|
+
const evaluationCycle = loadTrainingStateJson(db, "last_teacher_evaluation_cycle_json");
|
|
217
|
+
const budgetPerTick = Number.isFinite(evaluationCycle.value?.budgetPerTick)
|
|
218
|
+
? Math.max(0, Math.trunc(evaluationCycle.value.budgetPerTick))
|
|
219
|
+
: null;
|
|
220
|
+
const delayMs = Number.isFinite(evaluationCycle.value?.delayMs)
|
|
221
|
+
? Math.max(0, Math.trunc(evaluationCycle.value.delayMs))
|
|
222
|
+
: null;
|
|
223
|
+
if (budgetPerTick === null || delayMs === null) {
|
|
224
|
+
return {
|
|
225
|
+
visible: true,
|
|
226
|
+
gatingVisible: false,
|
|
227
|
+
completedWithoutEvaluationCount,
|
|
228
|
+
readyCount: 0,
|
|
229
|
+
delayedCount: 0,
|
|
230
|
+
budgetDeferredCount: 0,
|
|
231
|
+
detail: "teacher gating truth is not visible in the current status surface"
|
|
232
|
+
};
|
|
233
|
+
}
|
|
234
|
+
const pendingRows = db.prepare(`
|
|
235
|
+
SELECT status, follow_up_text, tool_results_json, created_at
|
|
236
|
+
FROM brain_observations
|
|
237
|
+
WHERE status IN ('pending_followup', 'pending_teacher')
|
|
238
|
+
ORDER BY created_at ASC
|
|
239
|
+
`).all();
|
|
240
|
+
const readyBefore = Date.now() - delayMs;
|
|
241
|
+
let readyCount = 0;
|
|
242
|
+
for (const row of pendingRows) {
|
|
243
|
+
if (isObservationReadyForTeacher(row, readyBefore)) {
|
|
244
|
+
readyCount += 1;
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
const delayedCount = Math.max(0, pendingRows.length - readyCount);
|
|
248
|
+
const budgetDeferredCount = Math.max(0, readyCount - budgetPerTick);
|
|
249
|
+
return {
|
|
250
|
+
visible: true,
|
|
251
|
+
gatingVisible: true,
|
|
252
|
+
completedWithoutEvaluationCount,
|
|
253
|
+
readyCount,
|
|
254
|
+
delayedCount,
|
|
255
|
+
budgetDeferredCount,
|
|
256
|
+
detail: pendingRows.length === 0
|
|
257
|
+
? "no teacher observations are pending"
|
|
258
|
+
: `completed_without_evaluation=${completedWithoutEvaluationCount}; ready=${readyCount}, delayed=${delayedCount}, budget_deferred=${budgetDeferredCount}`
|
|
259
|
+
};
|
|
260
|
+
}
|
|
24
261
|
function normalizeLastInterruptionSummary(value) {
|
|
25
262
|
if (value === null || typeof value !== "object" || Array.isArray(value)) {
|
|
26
263
|
return null;
|
|
@@ -95,11 +332,13 @@ function normalizeBridgePayload(payload) {
|
|
|
95
332
|
if (payload === null || typeof payload !== "object" || Array.isArray(payload)) {
|
|
96
333
|
throw new Error("expected traced-learning bridge payload object");
|
|
97
334
|
}
|
|
335
|
+
const routeTraceCount = normalizeCount(payload.routeTraceCount);
|
|
336
|
+
const supervisionCount = normalizeCount(payload.supervisionCount);
|
|
98
337
|
return {
|
|
99
338
|
contract: TRACED_LEARNING_BRIDGE_CONTRACT,
|
|
100
339
|
updatedAt: normalizeOptionalString(payload.updatedAt) ?? new Date().toISOString(),
|
|
101
|
-
routeTraceCount
|
|
102
|
-
supervisionCount
|
|
340
|
+
routeTraceCount,
|
|
341
|
+
supervisionCount,
|
|
103
342
|
routerUpdateCount: normalizeCount(payload.routerUpdateCount),
|
|
104
343
|
teacherArtifactCount: normalizeCount(payload.teacherArtifactCount),
|
|
105
344
|
pgVersionRequested: normalizeOptionalString(payload.pgVersionRequested),
|
|
@@ -111,6 +350,11 @@ function normalizeBridgePayload(payload) {
|
|
|
111
350
|
promoted: payload.promoted === true,
|
|
112
351
|
baselinePersisted: payload.baselinePersisted === true,
|
|
113
352
|
lastInterruptionSummary: normalizeLastInterruptionSummary(payload.lastInterruptionSummary),
|
|
353
|
+
feedbackSummary: normalizeFeedbackSummary(payload.feedbackSummary, {
|
|
354
|
+
routeTraceCount,
|
|
355
|
+
supervisedTraceCount: supervisionCount
|
|
356
|
+
}),
|
|
357
|
+
attributionCoverage: normalizeAttributionCoverage(payload.attributionCoverage),
|
|
114
358
|
source: normalizeSource(payload.source)
|
|
115
359
|
};
|
|
116
360
|
}
|
|
@@ -122,11 +366,13 @@ function normalizePersistedStatusSurface(payload) {
|
|
|
122
366
|
if (source === null) {
|
|
123
367
|
throw new Error("expected traced-learning status surface source");
|
|
124
368
|
}
|
|
369
|
+
const routeTraceCount = normalizeCount(payload.routeTraceCount);
|
|
370
|
+
const supervisionCount = normalizeCount(payload.supervisionCount);
|
|
125
371
|
return {
|
|
126
372
|
contract: TRACED_LEARNING_STATUS_SURFACE_CONTRACT,
|
|
127
373
|
updatedAt: normalizeOptionalString(payload.updatedAt) ?? new Date().toISOString(),
|
|
128
|
-
routeTraceCount
|
|
129
|
-
supervisionCount
|
|
374
|
+
routeTraceCount,
|
|
375
|
+
supervisionCount,
|
|
130
376
|
routerUpdateCount: normalizeCount(payload.routerUpdateCount),
|
|
131
377
|
teacherArtifactCount: normalizeCount(payload.teacherArtifactCount),
|
|
132
378
|
pgVersionRequested: normalizeOptionalString(payload.pgVersionRequested),
|
|
@@ -138,6 +384,11 @@ function normalizePersistedStatusSurface(payload) {
|
|
|
138
384
|
promoted: payload.promoted === true,
|
|
139
385
|
baselinePersisted: payload.baselinePersisted === true,
|
|
140
386
|
lastInterruptionSummary: normalizeLastInterruptionSummary(payload.lastInterruptionSummary),
|
|
387
|
+
feedbackSummary: normalizeFeedbackSummary(payload.feedbackSummary, {
|
|
388
|
+
routeTraceCount,
|
|
389
|
+
supervisedTraceCount: supervisionCount
|
|
390
|
+
}),
|
|
391
|
+
attributionCoverage: normalizeAttributionCoverage(payload.attributionCoverage),
|
|
141
392
|
source
|
|
142
393
|
};
|
|
143
394
|
}
|
|
@@ -157,6 +408,8 @@ function defaultSurface(pathname, detail, error = null) {
|
|
|
157
408
|
promoted: false,
|
|
158
409
|
baselinePersisted: false,
|
|
159
410
|
lastInterruptionSummary: null,
|
|
411
|
+
feedbackSummary: defaultFeedbackSummary(),
|
|
412
|
+
attributionCoverage: defaultAttributionCoverage(),
|
|
160
413
|
source: null,
|
|
161
414
|
detail,
|
|
162
415
|
error
|
|
@@ -278,6 +531,8 @@ function buildPersistedStatusSurfaceBridge(summary, context) {
|
|
|
278
531
|
promoted: summary.promoted,
|
|
279
532
|
baselinePersisted: summary.baselinePersisted,
|
|
280
533
|
lastInterruptionSummary: summary.lastInterruptionSummary,
|
|
534
|
+
feedbackSummary: summary.feedbackSummary,
|
|
535
|
+
attributionCoverage: summary.attributionCoverage,
|
|
281
536
|
source: {
|
|
282
537
|
command: "brain-store",
|
|
283
538
|
bridge: TRACED_LEARNING_STATUS_SURFACE_BRIDGE,
|
|
@@ -321,6 +576,20 @@ function buildDerivedBrainStoreBridge(db, context, lastInterruptionSummary = nul
|
|
|
321
576
|
? null
|
|
322
577
|
: JSON.parse(candidateUpdateRaw);
|
|
323
578
|
const candidatePackVersion = Number.parseInt(candidatePackVersionRaw ?? "", 10);
|
|
579
|
+
let feedbackSummary = defaultFeedbackSummary(routeTraceCount, supervisionCount);
|
|
580
|
+
try {
|
|
581
|
+
feedbackSummary = buildDerivedFeedbackSummary(db, routeTraceCount, supervisionCount);
|
|
582
|
+
}
|
|
583
|
+
catch {
|
|
584
|
+
feedbackSummary = defaultFeedbackSummary(routeTraceCount, supervisionCount);
|
|
585
|
+
}
|
|
586
|
+
let attributionCoverage = defaultAttributionCoverage();
|
|
587
|
+
try {
|
|
588
|
+
attributionCoverage = buildDerivedAttributionCoverage(db);
|
|
589
|
+
}
|
|
590
|
+
catch {
|
|
591
|
+
attributionCoverage = defaultAttributionCoverage();
|
|
592
|
+
}
|
|
324
593
|
return normalizeBridgePayload({
|
|
325
594
|
updatedAt: toIsoTimestamp(candidateUpdate?.generatedAt),
|
|
326
595
|
routeTraceCount,
|
|
@@ -336,6 +605,8 @@ function buildDerivedBrainStoreBridge(db, context, lastInterruptionSummary = nul
|
|
|
336
605
|
promoted: false,
|
|
337
606
|
baselinePersisted: false,
|
|
338
607
|
lastInterruptionSummary,
|
|
608
|
+
feedbackSummary,
|
|
609
|
+
attributionCoverage,
|
|
339
610
|
source: {
|
|
340
611
|
command: "brain-store",
|
|
341
612
|
bridge: "brain_store_state",
|
|
@@ -360,6 +631,13 @@ function hasMeaningfulTracedLearningSignal(bridge) {
|
|
|
360
631
|
bridge.pgVersionUsed !== null ||
|
|
361
632
|
bridge.fallbackReason !== null ||
|
|
362
633
|
bridge.routerNoOpReason !== null ||
|
|
634
|
+
bridge.feedbackSummary.helpfulCount > 0 ||
|
|
635
|
+
bridge.feedbackSummary.irrelevantCount > 0 ||
|
|
636
|
+
bridge.feedbackSummary.harmfulCount > 0 ||
|
|
637
|
+
bridge.attributionCoverage.completedWithoutEvaluationCount > 0 ||
|
|
638
|
+
bridge.attributionCoverage.readyCount > 0 ||
|
|
639
|
+
bridge.attributionCoverage.delayedCount > 0 ||
|
|
640
|
+
bridge.attributionCoverage.budgetDeferredCount > 0 ||
|
|
363
641
|
Number.isFinite(bridge.source?.candidatePackVersion) ||
|
|
364
642
|
normalizeCount(bridge.source?.candidateUpdateCount) > 0;
|
|
365
643
|
}
|
|
@@ -479,27 +757,41 @@ export function loadBrainStoreTracedLearningBridge(options = {}) {
|
|
|
479
757
|
try {
|
|
480
758
|
db = new sqlite.DatabaseSync(dbPath, { readOnly: true });
|
|
481
759
|
const lastInterruptionSummary = loadLastAssemblyInterruptionSummary(db);
|
|
760
|
+
let derived = null;
|
|
761
|
+
try {
|
|
762
|
+
derived = buildDerivedBrainStoreBridge(db, {
|
|
763
|
+
brainRoot,
|
|
764
|
+
dbPath
|
|
765
|
+
}, lastInterruptionSummary);
|
|
766
|
+
}
|
|
767
|
+
catch {
|
|
768
|
+
derived = null;
|
|
769
|
+
}
|
|
482
770
|
const persisted = loadPersistedStatusSurface(db, {
|
|
483
771
|
brainRoot,
|
|
484
772
|
dbPath
|
|
485
773
|
});
|
|
486
774
|
if (persisted.bridge !== null) {
|
|
487
|
-
const bridge =
|
|
488
|
-
|
|
489
|
-
:
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
775
|
+
const bridge = normalizeBridgePayload({
|
|
776
|
+
...persisted.bridge,
|
|
777
|
+
lastInterruptionSummary: lastInterruptionSummary ?? persisted.bridge.lastInterruptionSummary,
|
|
778
|
+
feedbackSummary: derived?.feedbackSummary ?? persisted.bridge.feedbackSummary,
|
|
779
|
+
attributionCoverage: derived?.attributionCoverage ?? persisted.bridge.attributionCoverage
|
|
780
|
+
});
|
|
493
781
|
return {
|
|
494
782
|
path: dbPath,
|
|
495
783
|
bridge,
|
|
496
784
|
error: null
|
|
497
785
|
};
|
|
498
786
|
}
|
|
499
|
-
const bridge =
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
787
|
+
const bridge = derived;
|
|
788
|
+
if (bridge === null) {
|
|
789
|
+
return {
|
|
790
|
+
path: dbPath,
|
|
791
|
+
bridge: null,
|
|
792
|
+
error: persisted.error
|
|
793
|
+
};
|
|
794
|
+
}
|
|
503
795
|
if (!hasMeaningfulTracedLearningSignal(bridge)) {
|
|
504
796
|
return {
|
|
505
797
|
path: dbPath,
|
|
@@ -565,6 +857,8 @@ function buildStatusSurface(pathname, bridge, options = {}) {
|
|
|
565
857
|
promoted: bridge.promoted,
|
|
566
858
|
baselinePersisted: bridge.baselinePersisted,
|
|
567
859
|
lastInterruptionSummary: bridge.lastInterruptionSummary,
|
|
860
|
+
feedbackSummary: bridge.feedbackSummary,
|
|
861
|
+
attributionCoverage: bridge.attributionCoverage,
|
|
568
862
|
source: bridge.source,
|
|
569
863
|
detail: detailParts.join(" "),
|
|
570
864
|
error: options.error ?? null
|
|
@@ -611,6 +905,8 @@ function mergeCanonicalStatusBridge(canonicalBridge, runtimeLoaded) {
|
|
|
611
905
|
promoted: canonicalBridge.promoted,
|
|
612
906
|
baselinePersisted: canonicalBridge.baselinePersisted,
|
|
613
907
|
lastInterruptionSummary: canonicalBridge.lastInterruptionSummary ?? runtimeBridge?.lastInterruptionSummary ?? null,
|
|
908
|
+
feedbackSummary: canonicalBridge.feedbackSummary,
|
|
909
|
+
attributionCoverage: canonicalBridge.attributionCoverage,
|
|
614
910
|
fallbackReason: canonicalBridge.fallbackReason,
|
|
615
911
|
routerNoOpReason: canonicalBridge.routerNoOpReason,
|
|
616
912
|
source: runtimeMaterialized === null
|
|
@@ -634,6 +930,8 @@ function mergeCanonicalStatusBridge(canonicalBridge, runtimeLoaded) {
|
|
|
634
930
|
promoted: runtimeBridge?.promoted ?? canonicalBridge.promoted,
|
|
635
931
|
baselinePersisted: runtimeBridge?.baselinePersisted ?? canonicalBridge.baselinePersisted,
|
|
636
932
|
lastInterruptionSummary: canonicalBridge.lastInterruptionSummary ?? runtimeBridge?.lastInterruptionSummary ?? null,
|
|
933
|
+
feedbackSummary: canonicalBridge.feedbackSummary,
|
|
934
|
+
attributionCoverage: canonicalBridge.attributionCoverage,
|
|
637
935
|
fallbackReason: runtimeBridge?.fallbackReason ?? canonicalBridge.fallbackReason ?? null,
|
|
638
936
|
routerNoOpReason: runtimeBridge?.routerNoOpReason ?? canonicalBridge.routerNoOpReason ?? null,
|
|
639
937
|
source: runtimeMaterialized === null
|
|
@@ -655,11 +953,16 @@ export function mergeTracedLearningBridgePayload(payload, persisted) {
|
|
|
655
953
|
const routerUpdateCount = Math.max(current.routerUpdateCount, persistedBridge.routerUpdateCount);
|
|
656
954
|
const teacherArtifactCount = Math.max(current.teacherArtifactCount, persistedBridge.teacherArtifactCount);
|
|
657
955
|
const lastInterruptionSummary = current.lastInterruptionSummary ?? persistedBridge.lastInterruptionSummary ?? null;
|
|
956
|
+
const feedbackSummary = current.feedbackSummary.visible ? current.feedbackSummary : persistedBridge.feedbackSummary;
|
|
957
|
+
const attributionCoverage = current.attributionCoverage.visible ? current.attributionCoverage : persistedBridge.attributionCoverage;
|
|
658
958
|
const usedBridge = routeTraceCount !== current.routeTraceCount ||
|
|
659
959
|
supervisionCount !== current.supervisionCount ||
|
|
660
960
|
routerUpdateCount !== current.routerUpdateCount ||
|
|
661
961
|
teacherArtifactCount !== current.teacherArtifactCount ||
|
|
662
|
-
lastInterruptionSummary !== current.lastInterruptionSummary
|
|
962
|
+
lastInterruptionSummary !== current.lastInterruptionSummary ||
|
|
963
|
+
feedbackSummary.visible !== current.feedbackSummary.visible ||
|
|
964
|
+
attributionCoverage.visible !== current.attributionCoverage.visible ||
|
|
965
|
+
attributionCoverage.gatingVisible !== current.attributionCoverage.gatingVisible;
|
|
663
966
|
if (!usedBridge) {
|
|
664
967
|
return current;
|
|
665
968
|
}
|
|
@@ -670,6 +973,8 @@ export function mergeTracedLearningBridgePayload(payload, persisted) {
|
|
|
670
973
|
routerUpdateCount,
|
|
671
974
|
teacherArtifactCount,
|
|
672
975
|
lastInterruptionSummary,
|
|
976
|
+
feedbackSummary,
|
|
977
|
+
attributionCoverage,
|
|
673
978
|
routerNoOpReason: supervisionCount > 0 || routerUpdateCount > 0 ? null : current.routerNoOpReason,
|
|
674
979
|
source: {
|
|
675
980
|
...(current.source ?? {}),
|
package/package.json
CHANGED