@wrongstack/cli 1.0.19 → 1.0.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/data/providers.json +44 -0
- package/dist/{acp-INCNOH42.js → acp-LI665LO2.js} +18 -7
- package/dist/arg-parser.d.ts +1 -0
- package/dist/ascii-mode-SG4PRNLP.js +7 -0
- package/dist/{auth-YXK3EDUL.js → auth-TE55JF4Z.js} +68 -22
- package/dist/auth-menu/direct.d.ts +1 -0
- package/dist/auth-menu/local.d.ts +2 -0
- package/dist/auth-menu/oauth-menu.d.ts +3 -1
- package/dist/auth-menu/panel-service.d.ts +20 -0
- package/dist/{bench-YBU6BVQP.js → bench-GTLPM22B.js} +2 -2
- package/dist/boot/append-system-prompt.d.ts +10 -0
- package/dist/boot/ascii-mode.d.ts +12 -0
- package/dist/boot/config-backup-recovery.d.ts +2 -1
- package/dist/boot/container-wiring.d.ts +4 -2
- package/dist/boot/dispatch-singleshot.d.ts +4 -1
- package/dist/boot/dispatch-webui.d.ts +11 -0
- package/dist/boot/mcp-config-flag.d.ts +22 -0
- package/dist/boot/piped-stdin.d.ts +40 -0
- package/dist/boot/restricted-mode.d.ts +15 -0
- package/dist/boot/safe-mode.d.ts +30 -0
- package/dist/boot/short-circuit-desktop.d.ts +1 -0
- package/dist/boot/single-shot-budget.d.ts +30 -0
- package/dist/boot/stream-json.d.ts +36 -0
- package/dist/boot/structured-output.d.ts +16 -0
- package/dist/boot/system-prompt-builder.d.ts +2 -0
- package/dist/boot/tool-restriction-flags.d.ts +36 -0
- package/dist/boot/tui-project-picker-callback.d.ts +1 -1
- package/dist/boot/tui-settings-adapter.d.ts +15 -2
- package/dist/boot/worktree-sessions.d.ts +21 -0
- package/dist/{boot-auto-thin-MANHOYHE.js → boot-auto-thin-6Q56FY3F.js} +3 -2
- package/dist/boot-provider-selection.d.ts +67 -0
- package/dist/boot.d.ts +1 -38
- package/dist/{chunk-JY52HWL6.js → chunk-2EBVTYJR.js} +2 -2
- package/dist/{chunk-RDHFXOJM.js → chunk-2ERX3RPN.js} +112 -65
- package/dist/chunk-4TVYGG6G.js +62 -0
- package/dist/chunk-5JYDKG5B.js +154 -0
- package/dist/{chunk-KEPABQYT.js → chunk-5SEYGOCH.js} +8 -82
- package/dist/{chunk-3VXYY5JC.js → chunk-5UZBUV2H.js} +3 -19
- package/dist/chunk-BBHWW6LR.js +549 -0
- package/dist/chunk-BEHGRNOQ.js +58 -0
- package/dist/chunk-CZTXF6HY.js +140 -0
- package/dist/{chunk-JX6KL7OF.js → chunk-EPR42SWM.js} +39 -8
- package/dist/chunk-F5DCFAKI.js +36 -0
- package/dist/chunk-HLSHFXTO.js +97 -0
- package/dist/{chunk-CJTBLTS4.js → chunk-KJS5543T.js} +4 -3
- package/dist/{chunk-DE7BZDNK.js → chunk-KOEO4CJG.js} +27 -4
- package/dist/{chunk-SIS37L3W.js → chunk-L2RVEQWO.js} +211 -193
- package/dist/chunk-LBPLTTZ6.js +105 -0
- package/dist/chunk-LDCGZJTC.js +46 -0
- package/dist/chunk-LKPEBZBE.js +143 -0
- package/dist/chunk-LUMETJFO.js +171 -0
- package/dist/{chunk-KK2LB3ZS.js → chunk-OAVBM55Q.js} +9 -5
- package/dist/chunk-PZQBGUVZ.js +18 -0
- package/dist/{chunk-C3Z4N6A6.js → chunk-Q4INH2DG.js} +51 -3
- package/dist/chunk-R4QGL6GB.js +10 -0
- package/dist/{chunk-YLQEGYV4.js → chunk-RZ4ZDRYA.js} +14 -4
- package/dist/chunk-S4FFHDJG.js +93 -0
- package/dist/chunk-SFBFYIOW.js +271 -0
- package/dist/{chunk-RMQNCHGO.js → chunk-TI7KLHYX.js} +136 -11
- package/dist/{chunk-2ENZU66X.js → chunk-WQV2NP57.js} +5 -5
- package/dist/{chunk-B4JLSXZB.js → chunk-WSTVGC2C.js} +73 -45
- package/dist/chunk-X756ZS72.js +329 -0
- package/dist/{chunk-XJ7K67PP.js → chunk-YHYINL47.js} +63 -24
- package/dist/{chunk-2VMJ3E5Z.js → chunk-YTEOAWOJ.js} +6 -2
- package/dist/{chunk-GPPRNAKW.js → chunk-ZMX5HRLI.js} +2 -2
- package/dist/{cli-context-33F74CSZ.js → cli-context-UVGAN3NO.js} +496 -384
- package/dist/cli-entry-point.d.ts +2 -0
- package/dist/{cli-main-FNGMKPBA.js → cli-main-TN3KZ2P6.js} +3561 -2724
- package/dist/cli-update-notice.d.ts +2 -1
- package/dist/config-doctor.d.ts +10 -2
- package/dist/{config-transfer-2YF3CRTN.js → config-transfer-RUWLDCNK.js} +34 -13
- package/dist/{diag-doctor-ANPRUYFM.js → diag-doctor-BIZAFIAT.js} +2 -2
- package/dist/execute-deps.d.ts +2 -1
- package/dist/{execution-A6WJLSOC.js → execution-7QIP6GRY.js} +2384 -2055
- package/dist/fleet/host-types.d.ts +3 -1
- package/dist/goal-commands.d.ts +47 -0
- package/dist/goal-host.d.ts +1 -14
- package/dist/{hq-XQWQTRDA.js → hq-GLZ7QESM.js} +277 -261
- package/dist/hq-command-controller.d.ts +1 -1
- package/dist/{hq-server-IIBZQZU4.js → hq-server-YTKDVQPY.js} +2 -2
- package/dist/{hq-service-L5INQFM4.js → hq-service-BMMJAHRT.js} +11 -4
- package/dist/import-claude-code-32PCSIQI.js +236 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.js +19 -10
- package/dist/kanban-supervisor-67ECABTD.js +7 -0
- package/dist/live-settings-input.d.ts +7 -0
- package/dist/{mailbox-serve-3GFOK2I7.js → mailbox-serve-NW5VZG7R.js} +3 -3
- package/dist/{mcp-DC5WSBOO.js → mcp-LBS45HWZ.js} +4 -4
- package/dist/modeldiag-36VODVIH.js +828 -0
- package/dist/{per-subcommand-help-DHTYU524.js → per-subcommand-help-QDAQASPZ.js} +3 -3
- package/dist/permission-prompt-mirror.d.ts +1 -1
- package/dist/permission-prompt.d.ts +16 -2
- package/dist/permission-rules-view.d.ts +18 -0
- package/dist/{permissions-CFDLXETM.js → permissions-PL27UHYD.js} +67 -45
- package/dist/plugin-config.d.ts +12 -0
- package/dist/plugin-management-types.d.ts +31 -0
- package/dist/plugin-management.d.ts +2 -30
- package/dist/plugin-package-install.d.ts +3 -0
- package/dist/{plugin-usage-M6KET4OZ.js → plugin-usage-CIGOOTX3.js} +6 -5
- package/dist/{plugins-WJLW6ULV.js → plugins-HQTXUD6A.js} +6 -3
- package/dist/preflight.d.ts +1 -1
- package/dist/{projects-LMNAGGEU.js → projects-T4CZCA25.js} +3 -6
- package/dist/provider-config-utils.d.ts +1 -6
- package/dist/provider-helpers.d.ts +7 -11
- package/dist/{providers-models-LFNZ5WI7.js → providers-models-5Z2PJTEW.js} +44 -34
- package/dist/release-asset-download.d.ts +13 -0
- package/dist/release-attestation.d.ts +28 -0
- package/dist/remote/remote-session.d.ts +64 -0
- package/dist/remote/remote-target.d.ts +69 -0
- package/dist/remote-O5WXHLWV.js +561 -0
- package/dist/{replay-Z3FTJEMM.js → replay-DEI566CW.js} +2 -2
- package/dist/services/commit-message.d.ts +2 -1
- package/dist/services/dispatch-classifier.d.ts +39 -5
- package/dist/session-move-P2EC2COA.js +44 -0
- package/dist/session-move.d.ts +20 -0
- package/dist/session-stats.d.ts +10 -0
- package/dist/{sessions-config-UL45XKX7.js → sessions-config-SXTTLNXA.js} +47 -9
- package/dist/settings-menu.d.ts +2 -0
- package/dist/short-circuit-flags-74Y5N6CF.js +10 -0
- package/dist/{simpleui-dist-CXSIDAXZ.js → simpleui-dist-7BQISSEP.js} +7 -3
- package/dist/skill-suggest-L2EVWB6R.js +290 -0
- package/dist/slash-commands/command-context.d.ts +15 -5
- package/dist/slash-commands/effort.d.ts +1 -1
- package/dist/slash-commands/fallback.d.ts +1 -1
- package/dist/slash-commands/gitid.d.ts +1 -1
- package/dist/slash-commands/goal-refiner.d.ts +4 -49
- package/dist/slash-commands/jev.d.ts +4 -0
- package/dist/slash-commands/mcp.d.ts +2 -1
- package/dist/slash-commands/openai-quota.d.ts +15 -0
- package/dist/slash-commands/permissions.d.ts +4 -0
- package/dist/slash-commands/sdd-task-mutations.d.ts +4 -0
- package/dist/slash-commands/settings-context-mutations.d.ts +5 -0
- package/dist/slash-commands/settings-keys.d.ts +2 -0
- package/dist/slash-commands/settings-limits-mutations.d.ts +12 -0
- package/dist/slash-commands/settings-reset.d.ts +6 -0
- package/dist/standalone-auto-update-TOTRGCSA.js +249 -0
- package/dist/standalone-auto-update.d.ts +65 -0
- package/dist/standalone-update-LZO3MNLH.js +24 -0
- package/dist/standalone-update.d.ts +55 -0
- package/dist/subcommands/handlers/hq-alerts.d.ts +3 -0
- package/dist/subcommands/handlers/hq-audit.d.ts +46 -0
- package/dist/subcommands/handlers/hq-data-dir.d.ts +13 -0
- package/dist/subcommands/handlers/import-claude-code.d.ts +42 -0
- package/dist/subcommands/handlers/permissions.d.ts +9 -3
- package/dist/subcommands/handlers/remote.d.ts +6 -0
- package/dist/subcommands/handlers/skill-suggest.d.ts +21 -0
- package/dist/subcommands/handlers/typesafe-brain-replay.d.ts +26 -0
- package/dist/subcommands/handlers/typesafe-judgment-check.d.ts +3 -0
- package/dist/subcommands/handlers/typesafe-topic-replay.d.ts +19 -0
- package/dist/subcommands/handlers/typesafe-triage-replay.d.ts +18 -0
- package/dist/subcommands/handlers/typesafe.d.ts +23 -0
- package/dist/subcommands/handlers/version-help.d.ts +5 -0
- package/dist/{subcommands-XSLXMIDP.js → subcommands-Q6ORSB2E.js} +2 -2
- package/dist/tui-resume-callback.d.ts +18 -0
- package/dist/typesafe-U7YB2RRV.js +316 -0
- package/dist/typesafe-brain-replay-C5276FVR.js +222 -0
- package/dist/typesafe-judgment-check-GGYYV5UF.js +29 -0
- package/dist/typesafe-topic-replay-ZILIU3WS.js +234 -0
- package/dist/typesafe-triage-replay-FGAW7QM7.js +215 -0
- package/dist/{update-5DPB4YD4.js → update-TDZX4IL2.js} +4 -3
- package/dist/{update-check-BS3DHTSI.js → update-check-J5ZVKS73.js} +3 -2
- package/dist/update-check.d.ts +1 -1
- package/dist/version-JB2UAV2A.js +11 -0
- package/dist/version-help-XJJQ7YSH.js +10 -0
- package/dist/version.d.ts +3 -2
- package/dist/webui-client-observability.d.ts +7 -0
- package/dist/webui-server/route-contexts.d.ts +1 -1
- package/dist/webui-server/session-start-payload.d.ts +5 -1
- package/dist/{webui-server-MRPGTTKL.js → webui-server-YTHLRFD2.js} +577 -370
- package/dist/webui-server-lifecycle.d.ts +57 -0
- package/dist/webui-server-options.d.ts +11 -0
- package/dist/webui-server.d.ts +1 -1
- package/dist/wiring/brain-and-orchestration.d.ts +3 -1
- package/dist/wiring/cli-prompt-and-tools-setup.d.ts +9 -0
- package/dist/wiring/cli-slash-commands-setup.d.ts +1 -0
- package/dist/wiring/domain-glossary.d.ts +5 -4
- package/dist/wiring/lifecycle-plugins.d.ts +4 -2
- package/dist/wiring/metrics.d.ts +9 -1
- package/dist/wiring/pipeline.d.ts +2 -7
- package/dist/wiring/plugins.d.ts +4 -1
- package/dist/wiring/provider-persisters.d.ts +24 -0
- package/dist/wiring/provider-runtime.d.ts +9 -1
- package/dist/wiring/provider-status.d.ts +1 -0
- package/dist/wiring/provider-utility-tools.d.ts +3 -1
- package/dist/wiring/runtime-dispatch-state.d.ts +3 -1
- package/package.json +28 -28
- package/dist/chunk-2CZEORLV.js +0 -54
- package/dist/chunk-672UXYL3.js +0 -65
- package/dist/chunk-6WRKAACA.js +0 -96
- package/dist/chunk-DR2OXU46.js +0 -174
- package/dist/chunk-JACOMTIH.js +0 -275
- package/dist/chunk-XJXDOF63.js +0 -28
- package/dist/modeldiag-CJSFHZ3J.js +0 -1356
- package/dist/short-circuit-flags-7GQIGMVD.js +0 -10
- package/dist/version-help-WXI42PZ3.js +0 -10
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
|
|
2
|
+
// src/subcommands/handlers/typesafe-brain-replay.ts
|
|
3
|
+
import { readFile } from "node:fs/promises";
|
|
4
|
+
import * as path from "node:path";
|
|
5
|
+
import {
|
|
6
|
+
probeSystemOneBrain,
|
|
7
|
+
systemOneBrainSettles
|
|
8
|
+
} from "@wrongstack/core/execution";
|
|
9
|
+
import { resolveTypeSafeAccount } from "@wrongstack/core/typesafe";
|
|
10
|
+
import { color } from "@wrongstack/core/utils";
|
|
11
|
+
var MONITOR_OPTIONS = [
|
|
12
|
+
{
|
|
13
|
+
id: "steer",
|
|
14
|
+
label: "Steer the agent with corrective guidance",
|
|
15
|
+
consequence: "A steer message is injected before its next step."
|
|
16
|
+
},
|
|
17
|
+
{ id: "continue", label: "Let the agent continue unaided" }
|
|
18
|
+
];
|
|
19
|
+
function monitorKind(question) {
|
|
20
|
+
if (/has been edited \d+ times within/.test(question)) return "file_churn";
|
|
21
|
+
if (/has failed \d+ times in a row/.test(question)) return "tool_failure";
|
|
22
|
+
if (/no observable progress/.test(question)) return "stall";
|
|
23
|
+
if (/errors occurred within/.test(question)) return "error_storm";
|
|
24
|
+
return void 0;
|
|
25
|
+
}
|
|
26
|
+
function monitorContext(kind, question) {
|
|
27
|
+
if (kind === "file_churn") {
|
|
28
|
+
const m = /file "([^"]+)" has been edited (\d+) times within (\d+) minutes/.exec(question);
|
|
29
|
+
if (m) return `File: ${m[1]}
|
|
30
|
+
Edits in window: ${m[2]}
|
|
31
|
+
Window: ${Number(m[3]) * 60}s`;
|
|
32
|
+
}
|
|
33
|
+
if (kind === "tool_failure") {
|
|
34
|
+
const m = /tool "([^"]+)" has failed (\d+) times/.exec(question);
|
|
35
|
+
if (m) return `Tool: ${m[1]}
|
|
36
|
+
Consecutive failures: ${m[2]}`;
|
|
37
|
+
}
|
|
38
|
+
return question;
|
|
39
|
+
}
|
|
40
|
+
function parseFlag(value, fallback) {
|
|
41
|
+
const n = Number(value);
|
|
42
|
+
return Number.isFinite(n) && n > 0 ? Math.floor(n) : fallback;
|
|
43
|
+
}
|
|
44
|
+
async function replayBrainLedger(deps, write) {
|
|
45
|
+
const flags = deps.flags ?? {};
|
|
46
|
+
const account = resolveTypeSafeAccount({ config: deps.config, restGate: null });
|
|
47
|
+
if (account.status !== "ready") {
|
|
48
|
+
write(`${color.red("\u2717")} ${account.reason}. Run \`wstack typesafe login\`.`);
|
|
49
|
+
return 2;
|
|
50
|
+
}
|
|
51
|
+
const ledgerPath = typeof flags["ledger"] === "string" ? path.resolve(flags["ledger"]) : path.join(deps.paths.projectDir, "brain-ledger.jsonl");
|
|
52
|
+
const perKind = parseFlag(flags["limit"], 40);
|
|
53
|
+
const onlyKind = typeof flags["kind"] === "string" ? flags["kind"] : void 0;
|
|
54
|
+
let raw;
|
|
55
|
+
try {
|
|
56
|
+
raw = await readFile(ledgerPath, "utf8");
|
|
57
|
+
} catch (err) {
|
|
58
|
+
write(`${color.red("\u2717")} cannot read ${ledgerPath}: ${err.message}`);
|
|
59
|
+
return 2;
|
|
60
|
+
}
|
|
61
|
+
const rows = [];
|
|
62
|
+
for (const line of raw.split("\n")) {
|
|
63
|
+
if (!line.trim()) continue;
|
|
64
|
+
try {
|
|
65
|
+
rows.push(JSON.parse(line));
|
|
66
|
+
} catch {
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
const outcomes = /* @__PURE__ */ new Map();
|
|
70
|
+
for (const r of rows) if (r.kind === "outcome" && r.outcome) outcomes.set(r.requestId, r.outcome);
|
|
71
|
+
const byKind = /* @__PURE__ */ new Map();
|
|
72
|
+
const seen = /* @__PURE__ */ new Set();
|
|
73
|
+
for (const r of rows) {
|
|
74
|
+
if (r.kind !== "answered" || !r.question || !r.optionId) continue;
|
|
75
|
+
if (r.optionId !== "steer" && r.optionId !== "continue") continue;
|
|
76
|
+
const kind = monitorKind(r.question);
|
|
77
|
+
if (!kind || onlyKind && kind !== onlyKind || seen.has(r.question)) continue;
|
|
78
|
+
seen.add(r.question);
|
|
79
|
+
const list = byKind.get(kind) ?? [];
|
|
80
|
+
list.push({
|
|
81
|
+
kind,
|
|
82
|
+
question: r.question,
|
|
83
|
+
recorded: r.optionId,
|
|
84
|
+
recordedBy: r.detail?.startsWith("Council") ? "council" : "other",
|
|
85
|
+
outcome: outcomes.get(r.requestId)
|
|
86
|
+
});
|
|
87
|
+
byKind.set(kind, list);
|
|
88
|
+
}
|
|
89
|
+
const cases = [];
|
|
90
|
+
for (const list of byKind.values()) {
|
|
91
|
+
const step = Math.max(1, list.length / perKind);
|
|
92
|
+
for (let i = 0; i < list.length && cases.length < 1e4; i += step) {
|
|
93
|
+
const picked = list[Math.floor(i)];
|
|
94
|
+
if (picked) cases.push(picked);
|
|
95
|
+
if (cases.filter((c) => c.kind === picked?.kind).length >= perKind) break;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
if (cases.length === 0) {
|
|
99
|
+
write("No replayable BrainMonitor decisions in the ledger.");
|
|
100
|
+
return 0;
|
|
101
|
+
}
|
|
102
|
+
if (flags["json"] !== true) {
|
|
103
|
+
write(color.dim(`Replaying ${cases.length} decision(s) from ${ledgerPath}\u2026`));
|
|
104
|
+
}
|
|
105
|
+
const judge = { client: account.client, model: account.model, feature: "brain" };
|
|
106
|
+
const CONCURRENCY = 8;
|
|
107
|
+
for (let i = 0; i < cases.length; i += CONCURRENCY) {
|
|
108
|
+
await Promise.all(
|
|
109
|
+
cases.slice(i, i + CONCURRENCY).map(async (c) => {
|
|
110
|
+
const request = {
|
|
111
|
+
id: `replay-${i}`,
|
|
112
|
+
source: "system",
|
|
113
|
+
question: c.question,
|
|
114
|
+
context: monitorContext(c.kind, c.question),
|
|
115
|
+
risk: "medium",
|
|
116
|
+
fallback: "ask_human",
|
|
117
|
+
options: MONITOR_OPTIONS
|
|
118
|
+
};
|
|
119
|
+
try {
|
|
120
|
+
c.probe = await probeSystemOneBrain(judge, request, { timeoutMs: 15e3 });
|
|
121
|
+
if (!c.probe) c.error = "malformed answer";
|
|
122
|
+
} catch (err) {
|
|
123
|
+
c.error = err instanceof Error ? err.message : String(err);
|
|
124
|
+
}
|
|
125
|
+
})
|
|
126
|
+
);
|
|
127
|
+
}
|
|
128
|
+
const report = summarize(cases);
|
|
129
|
+
if (flags["json"] === true) {
|
|
130
|
+
write(JSON.stringify({ ledger: ledgerPath, ...report }, null, 2));
|
|
131
|
+
return 0;
|
|
132
|
+
}
|
|
133
|
+
const pct = (a, b) => b === 0 ? "\u2014" : `${Math.round(a / b * 100)}%`;
|
|
134
|
+
for (const k of report.kinds) {
|
|
135
|
+
write(color.bold(`${k.kind}`) + color.dim(` n=${k.n} (council ${k.council})`));
|
|
136
|
+
write(
|
|
137
|
+
` recorded steer ${k.recordedSteer} / continue ${k.n - k.recordedSteer} Jev steer ${k.jevSteer} / continue ${k.judged - k.jevSteer}`
|
|
138
|
+
);
|
|
139
|
+
write(` agreement ${pct(k.agree, k.judged)} of ${k.judged} judged`);
|
|
140
|
+
write(
|
|
141
|
+
` settles ${pct(k.settled, k.judged)} at current thresholds, agreeing ${pct(k.settledAgree, k.settled)}`
|
|
142
|
+
);
|
|
143
|
+
if (k.futileSteers > 0) {
|
|
144
|
+
write(
|
|
145
|
+
` futile steers (signal re-fired): ${k.futileSteers}, Jev said continue on ${k.futileSteersJevContinue}`
|
|
146
|
+
);
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
write("");
|
|
150
|
+
write(color.bold("Sweep (decidable threshold; confidence/probability at defaults)"));
|
|
151
|
+
for (const s of report.sweep) {
|
|
152
|
+
write(
|
|
153
|
+
` decidable \u2265 ${s.threshold.toFixed(2)} settles ${pct(s.settled, report.judged)} agreeing ${pct(s.agree, s.settled)}`
|
|
154
|
+
);
|
|
155
|
+
}
|
|
156
|
+
if (report.errors > 0) write(color.amber(`${report.errors} case(s) not judged (host error).`));
|
|
157
|
+
write(color.dim(`model ${report.model ?? account.model}`));
|
|
158
|
+
write(
|
|
159
|
+
color.dim(
|
|
160
|
+
"Replayed contexts lack the tool output the live monitor attaches, so tool_failure reads as signal-only here."
|
|
161
|
+
)
|
|
162
|
+
);
|
|
163
|
+
return 0;
|
|
164
|
+
}
|
|
165
|
+
function summarize(cases) {
|
|
166
|
+
const kinds = [...new Set(cases.map((c) => c.kind))].map((kind) => {
|
|
167
|
+
const list = cases.filter((c) => c.kind === kind);
|
|
168
|
+
const judged2 = list.filter((c) => c.probe);
|
|
169
|
+
const settled = judged2.filter((c) => systemOneBrainSettles(c.probe));
|
|
170
|
+
const futile = list.filter((c) => c.recorded === "steer" && c.outcome === "failure");
|
|
171
|
+
return {
|
|
172
|
+
kind,
|
|
173
|
+
n: list.length,
|
|
174
|
+
council: list.filter((c) => c.recordedBy === "council").length,
|
|
175
|
+
recordedSteer: list.filter((c) => c.recorded === "steer").length,
|
|
176
|
+
judged: judged2.length,
|
|
177
|
+
jevSteer: judged2.filter((c) => c.probe.optionId === "steer").length,
|
|
178
|
+
agree: judged2.filter((c) => c.probe.optionId === c.recorded).length,
|
|
179
|
+
settled: settled.length,
|
|
180
|
+
settledAgree: settled.filter((c) => c.probe.optionId === c.recorded).length,
|
|
181
|
+
futileSteers: futile.length,
|
|
182
|
+
futileSteersJevContinue: futile.filter((c) => c.probe?.optionId === "continue").length
|
|
183
|
+
};
|
|
184
|
+
});
|
|
185
|
+
const judged = cases.filter((c) => c.probe);
|
|
186
|
+
const sweep = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6].map((threshold) => {
|
|
187
|
+
const settled = judged.filter(
|
|
188
|
+
(c) => c.probe.decidable >= threshold && systemOneBrainSettles(c.probe, { minDecidable: 0 })
|
|
189
|
+
);
|
|
190
|
+
return {
|
|
191
|
+
threshold,
|
|
192
|
+
settled: settled.length,
|
|
193
|
+
agree: settled.filter((c) => c.probe.optionId === c.recorded).length
|
|
194
|
+
};
|
|
195
|
+
});
|
|
196
|
+
return {
|
|
197
|
+
kinds,
|
|
198
|
+
sweep,
|
|
199
|
+
judged: judged.length,
|
|
200
|
+
errors: cases.filter((c) => c.error).length,
|
|
201
|
+
model: judged[0]?.probe?.model,
|
|
202
|
+
cases: judged.map((c) => ({
|
|
203
|
+
kind: c.kind,
|
|
204
|
+
recorded: c.recorded,
|
|
205
|
+
jev: c.probe.optionId,
|
|
206
|
+
p: Number(c.probe.probability.toFixed(3)),
|
|
207
|
+
decidable: Number(c.probe.decidable.toFixed(3))
|
|
208
|
+
})),
|
|
209
|
+
disagreements: judged.filter((c) => c.probe.optionId !== c.recorded).slice(0, 20).map((c) => ({
|
|
210
|
+
kind: c.kind,
|
|
211
|
+
question: c.question.slice(0, 200),
|
|
212
|
+
recorded: c.recorded,
|
|
213
|
+
jev: c.probe.optionId,
|
|
214
|
+
p: Number(c.probe.probability.toFixed(3)),
|
|
215
|
+
decidable: Number(c.probe.decidable.toFixed(3))
|
|
216
|
+
}))
|
|
217
|
+
};
|
|
218
|
+
}
|
|
219
|
+
export {
|
|
220
|
+
replayBrainLedger
|
|
221
|
+
};
|
|
222
|
+
//# sourceMappingURL=typesafe-brain-replay-C5276FVR.js.map
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
|
|
2
|
+
// src/subcommands/handlers/typesafe-judgment-check.ts
|
|
3
|
+
import { checkJevJudgments } from "@wrongstack/runtime/jev-checks";
|
|
4
|
+
async function checkJudgments(deps, write) {
|
|
5
|
+
try {
|
|
6
|
+
const report = await checkJevJudgments(deps.config);
|
|
7
|
+
let feature = "";
|
|
8
|
+
for (const result of report.cases) {
|
|
9
|
+
if (feature !== result.feature) {
|
|
10
|
+
feature = result.feature;
|
|
11
|
+
write(feature);
|
|
12
|
+
}
|
|
13
|
+
write(
|
|
14
|
+
`${result.ok ? "\u2713" : "\u2717"} ${result.name} (${result.ms}ms) \u2014 expected ${result.expected}, got ${result.actual}${result.note ? ` [${result.note}]` : ""}`
|
|
15
|
+
);
|
|
16
|
+
}
|
|
17
|
+
write(
|
|
18
|
+
`${report.passed}/${report.total} as expected \u2014 ${report.route} route, model ${report.model}`
|
|
19
|
+
);
|
|
20
|
+
return report.passed === report.total ? 0 : 1;
|
|
21
|
+
} catch {
|
|
22
|
+
write("Jev checks unavailable. Configure an account with wstack typesafe login.");
|
|
23
|
+
return 2;
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
export {
|
|
27
|
+
checkJudgments
|
|
28
|
+
};
|
|
29
|
+
//# sourceMappingURL=typesafe-judgment-check-GGYYV5UF.js.map
|
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
import {
|
|
2
|
+
resolveRuntimeMaxContext
|
|
3
|
+
} from "./chunk-CZTXF6HY.js";
|
|
4
|
+
import {
|
|
5
|
+
createProviderForId
|
|
6
|
+
} from "./chunk-BBHWW6LR.js";
|
|
7
|
+
|
|
8
|
+
// src/subcommands/handlers/typesafe-topic-replay.ts
|
|
9
|
+
import { readdir, readFile } from "node:fs/promises";
|
|
10
|
+
import * as path from "node:path";
|
|
11
|
+
import { promisify } from "node:util";
|
|
12
|
+
import { gunzip } from "node:zlib";
|
|
13
|
+
import { TopicShiftAdvisor } from "@wrongstack/core/execution";
|
|
14
|
+
import { resolveTypeSafeAccount } from "@wrongstack/core/typesafe";
|
|
15
|
+
import { color } from "@wrongstack/core/utils";
|
|
16
|
+
var gunzipAsync = promisify(gunzip);
|
|
17
|
+
function textOf(content) {
|
|
18
|
+
if (typeof content === "string") return content;
|
|
19
|
+
if (!Array.isArray(content)) return "";
|
|
20
|
+
return content.filter((b) => b?.type === "text").map((b) => b.text).join("\n");
|
|
21
|
+
}
|
|
22
|
+
async function sessionFiles(root) {
|
|
23
|
+
const out = [];
|
|
24
|
+
let days;
|
|
25
|
+
try {
|
|
26
|
+
days = await readdir(root);
|
|
27
|
+
} catch {
|
|
28
|
+
return out;
|
|
29
|
+
}
|
|
30
|
+
for (const day of days.sort()) {
|
|
31
|
+
let entries;
|
|
32
|
+
try {
|
|
33
|
+
entries = await readdir(path.join(root, day));
|
|
34
|
+
} catch {
|
|
35
|
+
continue;
|
|
36
|
+
}
|
|
37
|
+
for (const name of entries.sort()) {
|
|
38
|
+
if (/^sess_[^.]+\.jsonl(\.gz)?$/.test(name)) out.push(path.join(root, day, name));
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
return out;
|
|
42
|
+
}
|
|
43
|
+
async function promptPoints(file) {
|
|
44
|
+
const buf = await readFile(file);
|
|
45
|
+
const raw = file.endsWith(".gz") ? (await gunzipAsync(buf)).toString("utf8") : buf.toString("utf8");
|
|
46
|
+
const messages = [];
|
|
47
|
+
const points = [];
|
|
48
|
+
let contextTokens;
|
|
49
|
+
let previousPrompt = "";
|
|
50
|
+
for (const line of raw.split("\n")) {
|
|
51
|
+
if (!line.trim()) continue;
|
|
52
|
+
let e;
|
|
53
|
+
try {
|
|
54
|
+
e = JSON.parse(line);
|
|
55
|
+
} catch {
|
|
56
|
+
continue;
|
|
57
|
+
}
|
|
58
|
+
if (e["type"] === "llm_request" && typeof e["estimatedInputTokens"] === "number") {
|
|
59
|
+
contextTokens = e["estimatedInputTokens"];
|
|
60
|
+
} else if (e["type"] === "user_input") {
|
|
61
|
+
const prompt = textOf(e["content"]).trim();
|
|
62
|
+
if (prompt) {
|
|
63
|
+
points.push({
|
|
64
|
+
session: path.basename(file).replace(/\.jsonl(\.gz)?$/, ""),
|
|
65
|
+
prompt,
|
|
66
|
+
previousPrompt,
|
|
67
|
+
messages: [...messages],
|
|
68
|
+
contextTokens
|
|
69
|
+
});
|
|
70
|
+
previousPrompt = prompt;
|
|
71
|
+
}
|
|
72
|
+
} else if (e["type"] === "message_appended") {
|
|
73
|
+
const m = e["message"];
|
|
74
|
+
if (m && (m.role === "user" || m.role === "assistant")) {
|
|
75
|
+
const text = textOf(m.content).trim();
|
|
76
|
+
if (text) messages.push({ role: m.role, content: text });
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
return points;
|
|
81
|
+
}
|
|
82
|
+
async function replayTopicShift(deps, write) {
|
|
83
|
+
const flags = deps.flags ?? {};
|
|
84
|
+
const json = flags["json"] === true;
|
|
85
|
+
const account = resolveTypeSafeAccount({ config: deps.config, restGate: null });
|
|
86
|
+
if (account.status !== "ready") {
|
|
87
|
+
write(`${color.red("\u2717")} ${account.reason}. Run \`wstack typesafe login\`.`);
|
|
88
|
+
return 2;
|
|
89
|
+
}
|
|
90
|
+
const limit = Math.max(1, Number(flags["limit"]) || 40);
|
|
91
|
+
const sessionsRoot = typeof flags["sessions"] === "string" ? path.resolve(flags["sessions"]) : path.join(deps.paths.projectDir, "sessions");
|
|
92
|
+
const providerId = typeof flags["provider"] === "string" ? flags["provider"] : deps.config.provider;
|
|
93
|
+
const model = typeof flags["model"] === "string" ? flags["model"] : deps.config.model;
|
|
94
|
+
const flagMaxContext = Number(flags["max-context"]);
|
|
95
|
+
const maxContext = Number.isFinite(flagMaxContext) && flagMaxContext > 0 ? Math.floor(flagMaxContext) : await resolveRuntimeMaxContext({
|
|
96
|
+
modelsRegistry: deps.modelsRegistry,
|
|
97
|
+
config: deps.config,
|
|
98
|
+
provider: { capabilities: { maxContext: 0 } },
|
|
99
|
+
providerId,
|
|
100
|
+
modelId: model
|
|
101
|
+
}).catch(() => 0);
|
|
102
|
+
if (maxContext <= 0) {
|
|
103
|
+
write(
|
|
104
|
+
`${color.red("\u2717")} Context window of ${providerId}/${model} is unknown. Pass --max-context <tokens>.`
|
|
105
|
+
);
|
|
106
|
+
return 2;
|
|
107
|
+
}
|
|
108
|
+
const provider = flags["no-llm"] === true ? void 0 : createProviderForId(providerId, deps.config);
|
|
109
|
+
const judge = { client: account.client, model: account.model, feature: "topicShift" };
|
|
110
|
+
const reached = [];
|
|
111
|
+
let scanned = 0;
|
|
112
|
+
const files = await sessionFiles(sessionsRoot);
|
|
113
|
+
for (const file of files.reverse()) {
|
|
114
|
+
if (reached.length >= limit) break;
|
|
115
|
+
let points;
|
|
116
|
+
try {
|
|
117
|
+
points = await promptPoints(file);
|
|
118
|
+
} catch {
|
|
119
|
+
continue;
|
|
120
|
+
}
|
|
121
|
+
for (const point of points) {
|
|
122
|
+
if (reached.length >= limit) break;
|
|
123
|
+
scanned++;
|
|
124
|
+
let gate = false;
|
|
125
|
+
let jevNoul;
|
|
126
|
+
let jevError;
|
|
127
|
+
const jevAdvisor = new TopicShiftAdvisor({
|
|
128
|
+
getJudge: () => judge,
|
|
129
|
+
onDiagnostic: (d) => {
|
|
130
|
+
if (d.stage === "gate") gate = true;
|
|
131
|
+
if (d.stage === "system-one") {
|
|
132
|
+
jevNoul = d.noul;
|
|
133
|
+
jevError = d.error;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
});
|
|
137
|
+
await jevAdvisor.advise({
|
|
138
|
+
prompt: point.prompt,
|
|
139
|
+
messages: point.messages,
|
|
140
|
+
contextTokens: point.contextTokens,
|
|
141
|
+
maxContext
|
|
142
|
+
});
|
|
143
|
+
if (!gate) continue;
|
|
144
|
+
const c = { ...point, jevNoul, jevError };
|
|
145
|
+
if (provider?.complete) {
|
|
146
|
+
let failure;
|
|
147
|
+
const llmAdvisor = new TopicShiftAdvisor({
|
|
148
|
+
onDiagnostic: (d) => {
|
|
149
|
+
if (d.stage === "model" && !d.parsed) {
|
|
150
|
+
failure = d.error ?? (d.text?.trim() ? "unparseable reply" : "empty reply");
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
});
|
|
154
|
+
const advice = await llmAdvisor.advise({
|
|
155
|
+
prompt: point.prompt,
|
|
156
|
+
messages: point.messages,
|
|
157
|
+
provider,
|
|
158
|
+
model,
|
|
159
|
+
contextTokens: point.contextTokens,
|
|
160
|
+
maxContext
|
|
161
|
+
});
|
|
162
|
+
if (advice.source === "model") {
|
|
163
|
+
c.llm = advice.suggestNewContext ? "new" : "same";
|
|
164
|
+
c.llmConfidence = advice.confidence;
|
|
165
|
+
} else {
|
|
166
|
+
c.llmFailure = failure ?? `no model answer (${advice.source})`;
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
reached.push(c);
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
const bucket = (n) => n === void 0 ? "none" : n >= 0.8 ? "new" : n <= 0.3 ? "same" : "unsure";
|
|
173
|
+
if (json) {
|
|
174
|
+
write(
|
|
175
|
+
JSON.stringify(
|
|
176
|
+
{
|
|
177
|
+
sessionsRoot,
|
|
178
|
+
scanned,
|
|
179
|
+
llm: provider ? `${providerId}/${model}` : null,
|
|
180
|
+
cases: reached.map((c) => ({
|
|
181
|
+
session: c.session,
|
|
182
|
+
previousPrompt: c.previousPrompt.slice(0, 200),
|
|
183
|
+
prompt: c.prompt.slice(0, 300),
|
|
184
|
+
jev: c.jevNoul === void 0 ? null : Number(c.jevNoul.toFixed(3)),
|
|
185
|
+
jevBucket: bucket(c.jevNoul),
|
|
186
|
+
jevError: c.jevError,
|
|
187
|
+
llm: c.llm ?? null,
|
|
188
|
+
llmConfidence: c.llmConfidence,
|
|
189
|
+
llmFailure: c.llmFailure
|
|
190
|
+
}))
|
|
191
|
+
},
|
|
192
|
+
null,
|
|
193
|
+
2
|
|
194
|
+
)
|
|
195
|
+
);
|
|
196
|
+
return 0;
|
|
197
|
+
}
|
|
198
|
+
const pct = (a, b) => b === 0 ? "\u2014" : `${Math.round(a / b * 100)}%`;
|
|
199
|
+
const count = (b) => reached.filter((c) => bucket(c.jevNoul) === b).length;
|
|
200
|
+
write(
|
|
201
|
+
color.dim(
|
|
202
|
+
`${scanned} prompt(s) scanned in ${files.length} session file(s); ${reached.length} passed the local gate and reached a classifier`
|
|
203
|
+
)
|
|
204
|
+
);
|
|
205
|
+
write(color.bold("Jev"));
|
|
206
|
+
write(
|
|
207
|
+
` new ${count("new")} same ${count("same")} unsure\u2192provider ${count("unsure")} no answer ${count("none")}`
|
|
208
|
+
);
|
|
209
|
+
if (provider) {
|
|
210
|
+
const answered = reached.filter((c) => c.llm);
|
|
211
|
+
write(color.bold(`Provider classifier (${providerId}/${model}, production request shape)`));
|
|
212
|
+
write(
|
|
213
|
+
` answered ${answered.length}/${reached.length} new ${answered.filter((c) => c.llm === "new").length} same ${answered.filter((c) => c.llm === "same").length}`
|
|
214
|
+
);
|
|
215
|
+
const failures = /* @__PURE__ */ new Map();
|
|
216
|
+
for (const c of reached) {
|
|
217
|
+
if (c.llmFailure) failures.set(c.llmFailure, (failures.get(c.llmFailure) ?? 0) + 1);
|
|
218
|
+
}
|
|
219
|
+
for (const [reason, n] of failures) write(color.amber(` ${n}\xD7 ${reason.slice(0, 100)}`));
|
|
220
|
+
const both = reached.filter((c) => c.llm && ["new", "same"].includes(bucket(c.jevNoul)));
|
|
221
|
+
write(
|
|
222
|
+
` agreement where Jev is decisive: ${pct(
|
|
223
|
+
both.filter((c) => bucket(c.jevNoul) === c.llm).length,
|
|
224
|
+
both.length
|
|
225
|
+
)} of ${both.length}`
|
|
226
|
+
);
|
|
227
|
+
}
|
|
228
|
+
write(color.dim(`model ${account.model}`));
|
|
229
|
+
return 0;
|
|
230
|
+
}
|
|
231
|
+
export {
|
|
232
|
+
replayTopicShift
|
|
233
|
+
};
|
|
234
|
+
//# sourceMappingURL=typesafe-topic-replay-ZILIU3WS.js.map
|
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
import {
|
|
2
|
+
createProviderForId
|
|
3
|
+
} from "./chunk-BBHWW6LR.js";
|
|
4
|
+
|
|
5
|
+
// src/subcommands/handlers/typesafe-triage-replay.ts
|
|
6
|
+
import * as path from "node:path";
|
|
7
|
+
import { resolveTypeSafeAccount } from "@wrongstack/core/typesafe";
|
|
8
|
+
import { color } from "@wrongstack/core/utils";
|
|
9
|
+
import { loadRuntimeDatabaseSync } from "@wrongstack/persistence";
|
|
10
|
+
import {
|
|
11
|
+
computeValueScore,
|
|
12
|
+
evaluateMemory,
|
|
13
|
+
preFilterBatch,
|
|
14
|
+
probeMemoryValue
|
|
15
|
+
} from "@wrongstack/sage";
|
|
16
|
+
var SYSTEM_ONE_MIN_CONFIDENCE = 0.55;
|
|
17
|
+
var SYSTEM_ONE_MIN_PROBABILITY = 0.5;
|
|
18
|
+
async function loadMemories(dbPath) {
|
|
19
|
+
const Database = loadRuntimeDatabaseSync();
|
|
20
|
+
const db = new Database(dbPath, { readOnly: true });
|
|
21
|
+
try {
|
|
22
|
+
const rows = db.prepare("SELECT data FROM memories WHERE status IN ('active', 'stale')").all();
|
|
23
|
+
const out = [];
|
|
24
|
+
for (const row of rows) {
|
|
25
|
+
try {
|
|
26
|
+
out.push(JSON.parse(row.data));
|
|
27
|
+
} catch {
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
return out;
|
|
31
|
+
} finally {
|
|
32
|
+
db.close();
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
async function actionFor(memory, score) {
|
|
36
|
+
const vs = computeValueScore(memory);
|
|
37
|
+
const result = await evaluateMemory(memory, vs, async () => `${score} | calibration`);
|
|
38
|
+
return result.action === "keep_llm_override" ? "keep" : result.action;
|
|
39
|
+
}
|
|
40
|
+
async function replayMemoryTriage(deps, write) {
|
|
41
|
+
const flags = deps.flags ?? {};
|
|
42
|
+
const account = resolveTypeSafeAccount({ config: deps.config, restGate: null });
|
|
43
|
+
if (account.status !== "ready") {
|
|
44
|
+
write(`${color.red("\u2717")} ${account.reason}. Run \`wstack typesafe login\`.`);
|
|
45
|
+
return 2;
|
|
46
|
+
}
|
|
47
|
+
const limit = Math.max(1, Number(flags["limit"]) || 40);
|
|
48
|
+
const useLlm = flags["no-llm"] !== true;
|
|
49
|
+
const storageDir = deps.config.Sage?.storage?.directory;
|
|
50
|
+
const dbPath = path.resolve(
|
|
51
|
+
deps.projectRoot,
|
|
52
|
+
storageDir && typeof storageDir === "string" ? storageDir : path.join(".wrongstack", "memories"),
|
|
53
|
+
"sage.db"
|
|
54
|
+
);
|
|
55
|
+
let memories;
|
|
56
|
+
try {
|
|
57
|
+
memories = await loadMemories(dbPath);
|
|
58
|
+
} catch (err) {
|
|
59
|
+
write(`${color.red("\u2717")} cannot open ${dbPath}: ${err.message}`);
|
|
60
|
+
return 2;
|
|
61
|
+
}
|
|
62
|
+
const { uncertain } = preFilterBatch(memories);
|
|
63
|
+
const gray = uncertain.map((memory) => ({ memory, vs: computeValueScore(memory) })).filter((x) => x.vs.band === "gray");
|
|
64
|
+
const step = Math.max(1, gray.length / limit);
|
|
65
|
+
const cases = [];
|
|
66
|
+
for (let i = 0; i < gray.length && cases.length < limit; i += step) {
|
|
67
|
+
const pick = gray[Math.floor(i)];
|
|
68
|
+
if (pick) cases.push({ memory: pick.memory, vsTotal: pick.vs.total });
|
|
69
|
+
}
|
|
70
|
+
if (flags["json"] !== true)
|
|
71
|
+
write(
|
|
72
|
+
color.dim(
|
|
73
|
+
`${memories.length} memories, ${gray.length} in the gray zone; replaying ${cases.length}` + (useLlm ? " with the LLM reference" : " (Jev only)")
|
|
74
|
+
)
|
|
75
|
+
);
|
|
76
|
+
if (cases.length === 0) return 0;
|
|
77
|
+
let callLlm;
|
|
78
|
+
let llmLabel = "";
|
|
79
|
+
if (useLlm) {
|
|
80
|
+
const providerId = typeof flags["provider"] === "string" ? flags["provider"] : deps.config.provider;
|
|
81
|
+
const model = typeof flags["model"] === "string" ? flags["model"] : deps.config.model;
|
|
82
|
+
const provider = createProviderForId(
|
|
83
|
+
providerId,
|
|
84
|
+
deps.config
|
|
85
|
+
);
|
|
86
|
+
if (!provider?.complete) {
|
|
87
|
+
write(
|
|
88
|
+
color.amber(
|
|
89
|
+
`! cannot build provider "${providerId}" here; use --provider/--model or --no-llm.`
|
|
90
|
+
)
|
|
91
|
+
);
|
|
92
|
+
return 2;
|
|
93
|
+
}
|
|
94
|
+
llmLabel = `${providerId}/${model}`;
|
|
95
|
+
callLlm = async (system, user) => {
|
|
96
|
+
const response = await provider.complete(
|
|
97
|
+
{
|
|
98
|
+
model,
|
|
99
|
+
system: [{ type: "text", text: system }],
|
|
100
|
+
messages: [{ role: "user", content: user }],
|
|
101
|
+
// 60 is what `/memory triage` sends; --llm-max-tokens shows whether
|
|
102
|
+
// a reasoning model needs more room to produce any answer at all.
|
|
103
|
+
maxTokens: Math.max(16, Number(flags["llm-max-tokens"]) || 60),
|
|
104
|
+
temperature: 0.1
|
|
105
|
+
},
|
|
106
|
+
{ signal: AbortSignal.timeout(6e4) }
|
|
107
|
+
);
|
|
108
|
+
return response.content.filter((b) => b.type === "text").map((b) => b.text).join("").trim();
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
const judge = { client: account.client, model: account.model, feature: "memoryTriage" };
|
|
112
|
+
const CONCURRENCY = 6;
|
|
113
|
+
for (let i = 0; i < cases.length; i += CONCURRENCY) {
|
|
114
|
+
await Promise.all(
|
|
115
|
+
cases.slice(i, i + CONCURRENCY).map(async (c) => {
|
|
116
|
+
try {
|
|
117
|
+
c.jev = await probeMemoryValue(judge, c.memory);
|
|
118
|
+
if (c.jev) c.jevAction = await actionFor(c.memory, c.jev.score);
|
|
119
|
+
} catch (err) {
|
|
120
|
+
c.error = `jev: ${err instanceof Error ? err.message : String(err)}`;
|
|
121
|
+
}
|
|
122
|
+
if (!callLlm) return;
|
|
123
|
+
try {
|
|
124
|
+
const r = await evaluateMemory(c.memory, computeValueScore(c.memory), callLlm);
|
|
125
|
+
if (r.evaluation.ok) {
|
|
126
|
+
c.llmScore = r.evaluation.score;
|
|
127
|
+
c.llmAction = r.action === "keep_llm_override" ? "keep" : r.action;
|
|
128
|
+
} else {
|
|
129
|
+
c.error = `llm: ${r.evaluation.error ?? "no score"}`;
|
|
130
|
+
}
|
|
131
|
+
} catch (err) {
|
|
132
|
+
c.error = `llm: ${err instanceof Error ? err.message : String(err)}`;
|
|
133
|
+
}
|
|
134
|
+
})
|
|
135
|
+
);
|
|
136
|
+
}
|
|
137
|
+
const judged = cases.filter((c) => c.jev);
|
|
138
|
+
const both = judged.filter((c) => c.llmScore !== void 0);
|
|
139
|
+
const settles = (c, minConfidence) => !!c.jev && c.jev.confidence >= minConfidence && c.jev.probability >= SYSTEM_ONE_MIN_PROBABILITY;
|
|
140
|
+
const pct = (a, b) => b === 0 ? "\u2014" : `${Math.round(a / b * 100)}%`;
|
|
141
|
+
if (flags["json"] === true) {
|
|
142
|
+
write(
|
|
143
|
+
JSON.stringify(
|
|
144
|
+
{
|
|
145
|
+
db: dbPath,
|
|
146
|
+
llm: llmLabel || null,
|
|
147
|
+
model: judged[0]?.jev?.model,
|
|
148
|
+
cases: cases.map((c) => ({
|
|
149
|
+
id: c.memory.id,
|
|
150
|
+
text: c.memory.text.slice(0, 160),
|
|
151
|
+
phase2: c.vsTotal,
|
|
152
|
+
jev: c.jev ? {
|
|
153
|
+
score: c.jev.score,
|
|
154
|
+
p: Number(c.jev.probability.toFixed(3)),
|
|
155
|
+
confidence: Number(c.jev.confidence.toFixed(3)),
|
|
156
|
+
action: c.jevAction
|
|
157
|
+
} : null,
|
|
158
|
+
llm: c.llmScore !== void 0 ? { score: c.llmScore, action: c.llmAction } : null,
|
|
159
|
+
error: c.error
|
|
160
|
+
}))
|
|
161
|
+
},
|
|
162
|
+
null,
|
|
163
|
+
2
|
|
164
|
+
)
|
|
165
|
+
);
|
|
166
|
+
return 0;
|
|
167
|
+
}
|
|
168
|
+
write("");
|
|
169
|
+
write(color.bold("Jev rating distribution"));
|
|
170
|
+
const dist = [1, 2, 3, 4, 5].map((s) => judged.filter((c) => c.jev.score === s).length);
|
|
171
|
+
write(` 1:${dist[0]} 2:${dist[1]} 3:${dist[2]} 4:${dist[3]} 5:${dist[4]}`);
|
|
172
|
+
if (both.length > 0) {
|
|
173
|
+
write(color.bold(`Jev vs LLM (${llmLabel}), n=${both.length}`));
|
|
174
|
+
write(
|
|
175
|
+
` rating exact ${pct(both.filter((c) => c.jev.score === c.llmScore).length, both.length)}`
|
|
176
|
+
);
|
|
177
|
+
write(
|
|
178
|
+
` rating \xB11 ${pct(both.filter((c) => Math.abs(c.jev.score - c.llmScore) <= 1).length, both.length)}`
|
|
179
|
+
);
|
|
180
|
+
write(
|
|
181
|
+
` same action ${pct(both.filter((c) => c.jevAction === c.llmAction).length, both.length)}`
|
|
182
|
+
);
|
|
183
|
+
write(" matrix (rows Jev 1-5, cols LLM 1-5)");
|
|
184
|
+
for (let j = 1; j <= 5; j++) {
|
|
185
|
+
const row = [1, 2, 3, 4, 5].map(
|
|
186
|
+
(l) => both.filter((c) => c.jev.score === j && c.llmScore === l).length
|
|
187
|
+
);
|
|
188
|
+
write(` ${j}: ${row.map((n) => String(n).padStart(3)).join("")}`);
|
|
189
|
+
}
|
|
190
|
+
write("");
|
|
191
|
+
write(color.bold("Sweep (confidence threshold, p \u2265 0.5): settles / same action"));
|
|
192
|
+
for (const t of [0.3, 0.4, 0.5, 0.55, 0.6, 0.7, 0.8]) {
|
|
193
|
+
const s = both.filter((c) => settles(c, t));
|
|
194
|
+
write(
|
|
195
|
+
` conf \u2265 ${t.toFixed(2)}${t === SYSTEM_ONE_MIN_CONFIDENCE ? " (current)" : " "} ${pct(s.length, both.length).padStart(4)} ${pct(
|
|
196
|
+
s.filter((c) => c.jevAction === c.llmAction).length,
|
|
197
|
+
s.length
|
|
198
|
+
)}`
|
|
199
|
+
);
|
|
200
|
+
}
|
|
201
|
+
} else {
|
|
202
|
+
const s = judged.filter((c) => settles(c, SYSTEM_ONE_MIN_CONFIDENCE));
|
|
203
|
+
write(` settles at current thresholds: ${pct(s.length, judged.length)}`);
|
|
204
|
+
}
|
|
205
|
+
const errors = cases.filter((c) => c.error);
|
|
206
|
+
if (errors.length > 0) {
|
|
207
|
+
write(color.amber(`${errors.length} case(s) with errors, e.g. ${errors[0]?.error}`));
|
|
208
|
+
}
|
|
209
|
+
write(color.dim(`model ${judged[0]?.jev?.model ?? account.model}`));
|
|
210
|
+
return 0;
|
|
211
|
+
}
|
|
212
|
+
export {
|
|
213
|
+
replayMemoryTriage
|
|
214
|
+
};
|
|
215
|
+
//# sourceMappingURL=typesafe-triage-replay-FGAW7QM7.js.map
|