crewhaus 0.1.8 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -3
- package/dist/advice-apply.d.ts +182 -0
- package/dist/advice-apply.js +286 -0
- package/dist/advise-rules.d.ts +348 -0
- package/dist/advise-rules.js +905 -0
- package/dist/alert-sink.d.ts +48 -0
- package/dist/alert-sink.js +86 -0
- package/dist/approval-gate.d.ts +127 -0
- package/dist/approval-gate.js +254 -0
- package/dist/audit-verify.d.ts +69 -0
- package/dist/audit-verify.js +97 -0
- package/dist/autodistill.d.ts +113 -0
- package/dist/autodistill.js +256 -0
- package/dist/channel-provision.d.ts +360 -0
- package/dist/channel-provision.js +881 -0
- package/dist/ci-scaffold.d.ts +31 -0
- package/dist/ci-scaffold.js +343 -0
- package/dist/compile-check.d.ts +90 -0
- package/dist/compile-check.js +285 -0
- package/dist/compliance-schedule.d.ts +35 -0
- package/dist/compliance-schedule.js +36 -0
- package/dist/context-pressure.d.ts +80 -0
- package/dist/context-pressure.js +166 -0
- package/dist/dataset-mine.d.ts +172 -0
- package/dist/dataset-mine.js +403 -0
- package/dist/datasets.d.ts +124 -0
- package/dist/datasets.js +260 -0
- package/dist/deploy-canary.d.ts +83 -0
- package/dist/deploy-canary.js +87 -0
- package/dist/doctor-checks.d.ts +33 -0
- package/dist/doctor-checks.js +92 -0
- package/dist/doctor-detect.d.ts +108 -0
- package/dist/doctor-detect.js +214 -0
- package/dist/doctor-fix.d.ts +81 -0
- package/dist/doctor-fix.js +164 -0
- package/dist/egress-triage.d.ts +121 -0
- package/dist/egress-triage.js +261 -0
- package/dist/eval-bridge.d.ts +114 -0
- package/dist/eval-bridge.js +158 -0
- package/dist/eval-coverage.d.ts +140 -0
- package/dist/eval-coverage.js +428 -0
- package/dist/eval-history.d.ts +48 -0
- package/dist/eval-history.js +157 -0
- package/dist/eval-matrix.d.ts +80 -0
- package/dist/eval-matrix.js +182 -0
- package/dist/eval-sentinel.d.ts +65 -0
- package/dist/eval-sentinel.js +132 -0
- package/dist/faq.d.ts +68 -0
- package/dist/faq.js +168 -0
- package/dist/feedback.d.ts +9 -2
- package/dist/feedback.js +17 -7
- package/dist/fewshot.d.ts +83 -0
- package/dist/fewshot.js +158 -0
- package/dist/fleet.d.ts +207 -0
- package/dist/fleet.js +488 -0
- package/dist/flywheel.d.ts +193 -0
- package/dist/flywheel.js +519 -0
- package/dist/graders-suggest.d.ts +186 -0
- package/dist/graders-suggest.js +658 -0
- package/dist/incident.d.ts +99 -0
- package/dist/incident.js +217 -0
- package/dist/index.d.ts +9 -1
- package/dist/index.js +11601 -964
- package/dist/init-interactive.d.ts +105 -0
- package/dist/init-interactive.js +208 -0
- package/dist/intents.d.ts +105 -0
- package/dist/intents.js +292 -0
- package/dist/judge-calibrate.d.ts +137 -0
- package/dist/judge-calibrate.js +247 -0
- package/dist/justification-calibrate.d.ts +150 -0
- package/dist/justification-calibrate.js +262 -0
- package/dist/justification-gate.d.ts +27 -6
- package/dist/justification-gate.js +30 -6
- package/dist/knowledge-sync.d.ts +179 -0
- package/dist/knowledge-sync.js +551 -0
- package/dist/lessons.d.ts +87 -0
- package/dist/lessons.js +207 -0
- package/dist/lint.d.ts +127 -0
- package/dist/lint.js +226 -0
- package/dist/loadtest.d.ts +114 -0
- package/dist/loadtest.js +196 -0
- package/dist/marketplace-cli.d.ts +110 -0
- package/dist/marketplace-cli.js +250 -0
- package/dist/mcp-doctor.d.ts +121 -0
- package/dist/mcp-doctor.js +249 -0
- package/dist/model-scan.d.ts +116 -0
- package/dist/model-scan.js +226 -0
- package/dist/onchain-tune.d.ts +164 -0
- package/dist/onchain-tune.js +346 -0
- package/dist/permissions-suggest.d.ts +126 -0
- package/dist/permissions-suggest.js +333 -0
- package/dist/pii-tune.d.ts +107 -0
- package/dist/pii-tune.js +122 -0
- package/dist/propose.d.ts +117 -0
- package/dist/propose.js +184 -0
- package/dist/refresh-goldens.d.ts +82 -0
- package/dist/refresh-goldens.js +221 -0
- package/dist/regression-pin.d.ts +160 -0
- package/dist/regression-pin.js +281 -0
- package/dist/retention.d.ts +193 -0
- package/dist/retention.js +607 -0
- package/dist/retire.d.ts +118 -0
- package/dist/retire.js +291 -0
- package/dist/right-size.d.ts +100 -0
- package/dist/right-size.js +123 -0
- package/dist/scaffold-evals.d.ts +138 -0
- package/dist/scaffold-evals.js +410 -0
- package/dist/scope-audit-drift.d.ts +139 -0
- package/dist/scope-audit-drift.js +260 -0
- package/dist/security-corpus.d.ts +237 -0
- package/dist/security-corpus.js +516 -0
- package/dist/security-digest.d.ts +173 -0
- package/dist/security-digest.js +650 -0
- package/dist/sessions-index.d.ts +27 -0
- package/dist/sessions-index.js +51 -0
- package/dist/slo-doctor.d.ts +67 -0
- package/dist/slo-doctor.js +119 -0
- package/dist/slo-sink.d.ts +96 -0
- package/dist/slo-sink.js +107 -0
- package/dist/spec-changelog.d.ts +102 -0
- package/dist/spec-changelog.js +237 -0
- package/dist/state-backup.d.ts +223 -0
- package/dist/state-backup.js +648 -0
- package/dist/tools-cli.d.ts +170 -0
- package/dist/tools-cli.js +298 -0
- package/dist/triage.d.ts +202 -0
- package/dist/triage.js +403 -0
- package/dist/upgrade.d.ts +57 -0
- package/dist/upgrade.js +113 -0
- package/dist/version.d.ts +6 -0
- package/dist/version.js +27 -0
- package/dist/voice-eval.d.ts +138 -0
- package/dist/voice-eval.js +309 -0
- package/dist/watch.d.ts +58 -0
- package/dist/watch.js +97 -0
- package/package.json +89 -65
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Ops item 38 — `crewhaus mcp doctor` core, in a side-effect-free module (this
|
|
3
|
+
* entry file runs an argv switch on import) mirroring `doctor-checks.ts` /
|
|
4
|
+
* `eval-matrix.ts`. Three capabilities, all pure — the CLI owns the file I/O
|
|
5
|
+
* and the live `listTools` probe; this module does the aggregation + diffing:
|
|
6
|
+
*
|
|
7
|
+
* 1. HEALTH SCORING — fold the durable `mcp_stats` session-log records
|
|
8
|
+
* (`{ server, toolName, durationMs, isError }`, mirrored from the
|
|
9
|
+
* trace-bus-only `mcp_call_end` by runtime-core) into per-server
|
|
10
|
+
* error-rate + latency + a chronic-failure verdict. `mcp_call_*` events
|
|
11
|
+
* are trace-bus-ONLY, so this durable mirror is the only cross-session
|
|
12
|
+
* history there is to score.
|
|
13
|
+
*
|
|
14
|
+
* 2. DRIFT WATCH — snapshot each server's `listTools` (tool name + a stable
|
|
15
|
+
* JSON-schema hash) and diff it against the LAST snapshot on disk, so an
|
|
16
|
+
* added / removed / schema-changed tool is reported BEFORE a production
|
|
17
|
+
* call fails (tool lists are fetched once at connect today; drift is
|
|
18
|
+
* otherwise invisible until runtime failure).
|
|
19
|
+
*
|
|
20
|
+
* 3. QUARANTINE DECISION — decide which chronically-failing servers' tools
|
|
21
|
+
* should be withdrawn from the ToolCatalog (the runtime injects a synthetic
|
|
22
|
+
* "unavailable" notice so the model routes around them) and which have
|
|
23
|
+
* recovered enough to restore. The catalog mutation lives in the runtime /
|
|
24
|
+
* CLI; this module only decides.
|
|
25
|
+
*/
|
|
26
|
+
import { createHash } from "node:crypto";
|
|
27
|
+
import { basename } from "node:path";
|
|
28
|
+
/**
|
|
29
|
+
* F5 — sanitise an mcp_servers KEY into a single safe filename segment before it
|
|
30
|
+
* is joined onto `.crewhaus/mcp/` (the drift-snapshot write). The mcp_servers
|
|
31
|
+
* key schema is `z.string().min(1)` (NOT `safeName`), so a server named
|
|
32
|
+
* `../../evil` — or one carrying a slash / NUL — would let
|
|
33
|
+
* `join(mcpDir, `${name}.json`)` escape the directory. We `basename` to strip
|
|
34
|
+
* path components, then replace every character outside the conservative
|
|
35
|
+
* `[A-Za-z0-9._-]` set with `_` (the same class the eval-runner + graders use),
|
|
36
|
+
* and floor an empty / dot-only result to a stable placeholder. The result is
|
|
37
|
+
* always a bare filename that stays inside the snapshot dir.
|
|
38
|
+
*/
|
|
39
|
+
export function safeMcpFileName(name) {
|
|
40
|
+
const base = basename(name).replace(/[^A-Za-z0-9._-]+/g, "_");
|
|
41
|
+
if (base === "" || base === "." || base === ".." || /^\.+$/.test(base))
|
|
42
|
+
return "_server";
|
|
43
|
+
return base;
|
|
44
|
+
}
|
|
45
|
+
/** Thresholds a server must cross to be judged chronically failing → quarantine
|
|
46
|
+
* candidate. Kept together so the report and the quarantine decision agree. */
|
|
47
|
+
export const QUARANTINE_POLICY = {
|
|
48
|
+
/** Minimum calls before an error rate is trustworthy (cold-start guard). */
|
|
49
|
+
minCalls: 5,
|
|
50
|
+
/** Error-rate ceiling; above it (with enough calls) the server is chronic. */
|
|
51
|
+
errorRate: 0.5,
|
|
52
|
+
/** OR: this many consecutive errors is chronic regardless of overall rate
|
|
53
|
+
* (a server that just went hard-down mid-session). */
|
|
54
|
+
errorStreak: 4,
|
|
55
|
+
};
|
|
56
|
+
/** Nearest-rank percentile (0-based floor). Empty ⇒ 0. */
|
|
57
|
+
export function percentile(values, p) {
|
|
58
|
+
if (values.length === 0)
|
|
59
|
+
return 0;
|
|
60
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
61
|
+
const idx = Math.floor(p * (sorted.length - 1));
|
|
62
|
+
return sorted[idx] ?? 0;
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Fold a flat stream of `mcp_stats` payloads (already read from session logs,
|
|
66
|
+
* oldest-first) into per-server health. The `maxErrorStreak` is computed over
|
|
67
|
+
* the input order, so callers should pass records in chronological order (the
|
|
68
|
+
* session-log reader yields insertion order — which is chronological).
|
|
69
|
+
*/
|
|
70
|
+
export function scoreMcpHealth(records) {
|
|
71
|
+
const byServer = new Map();
|
|
72
|
+
for (const r of records) {
|
|
73
|
+
let acc = byServer.get(r.server);
|
|
74
|
+
if (acc === undefined) {
|
|
75
|
+
acc = { calls: 0, errors: 0, durations: [], curStreak: 0, maxStreak: 0 };
|
|
76
|
+
byServer.set(r.server, acc);
|
|
77
|
+
}
|
|
78
|
+
acc.calls += 1;
|
|
79
|
+
acc.durations.push(r.durationMs);
|
|
80
|
+
if (r.isError) {
|
|
81
|
+
acc.errors += 1;
|
|
82
|
+
acc.curStreak += 1;
|
|
83
|
+
if (acc.curStreak > acc.maxStreak)
|
|
84
|
+
acc.maxStreak = acc.curStreak;
|
|
85
|
+
}
|
|
86
|
+
else {
|
|
87
|
+
acc.curStreak = 0;
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
const out = [];
|
|
91
|
+
for (const [server, acc] of byServer) {
|
|
92
|
+
const errorRate = acc.calls > 0 ? acc.errors / acc.calls : 0;
|
|
93
|
+
const chronic = isChronic(acc.calls, errorRate, acc.maxStreak);
|
|
94
|
+
out.push({
|
|
95
|
+
server,
|
|
96
|
+
calls: acc.calls,
|
|
97
|
+
errors: acc.errors,
|
|
98
|
+
errorRate,
|
|
99
|
+
p50Ms: percentile(acc.durations, 0.5),
|
|
100
|
+
p95Ms: percentile(acc.durations, 0.95),
|
|
101
|
+
maxErrorStreak: acc.maxStreak,
|
|
102
|
+
chronic,
|
|
103
|
+
});
|
|
104
|
+
}
|
|
105
|
+
// Sickest first (chronic, then by error rate desc) so the report leads with
|
|
106
|
+
// the servers that need attention.
|
|
107
|
+
return out.sort((a, b) => {
|
|
108
|
+
if (a.chronic !== b.chronic)
|
|
109
|
+
return a.chronic ? -1 : 1;
|
|
110
|
+
return b.errorRate - a.errorRate;
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
/** Chronic-failure verdict shared by scoring + quarantine decision. */
|
|
114
|
+
export function isChronic(calls, errorRate, maxErrorStreak) {
|
|
115
|
+
if (maxErrorStreak >= QUARANTINE_POLICY.errorStreak)
|
|
116
|
+
return true;
|
|
117
|
+
return calls >= QUARANTINE_POLICY.minCalls && errorRate >= QUARANTINE_POLICY.errorRate;
|
|
118
|
+
}
|
|
119
|
+
/**
|
|
120
|
+
* Stable hash of a JSON-schema value: recursively sort object keys so
|
|
121
|
+
* `{a,b}` and `{b,a}` hash identically, then sha256 the canonical form. A
|
|
122
|
+
* server that reorders its schema keys between connects is NOT drift; a server
|
|
123
|
+
* that changes a field's type IS. Returns the first 16 hex chars (collision-
|
|
124
|
+
* safe enough for a per-tool change signal).
|
|
125
|
+
*/
|
|
126
|
+
export function schemaHash(schema) {
|
|
127
|
+
return createHash("sha256").update(canonicalJson(schema)).digest("hex").slice(0, 16);
|
|
128
|
+
}
|
|
129
|
+
function canonicalJson(value) {
|
|
130
|
+
if (value === null || value === undefined)
|
|
131
|
+
return "null";
|
|
132
|
+
const t = typeof value;
|
|
133
|
+
if (t === "string" || t === "number" || t === "boolean")
|
|
134
|
+
return JSON.stringify(value);
|
|
135
|
+
if (Array.isArray(value))
|
|
136
|
+
return `[${value.map(canonicalJson).join(",")}]`;
|
|
137
|
+
if (t === "object") {
|
|
138
|
+
const obj = value;
|
|
139
|
+
const keys = Object.keys(obj).sort();
|
|
140
|
+
return `{${keys.map((k) => `${JSON.stringify(k)}:${canonicalJson(obj[k])}`).join(",")}}`;
|
|
141
|
+
}
|
|
142
|
+
return JSON.stringify(String(value));
|
|
143
|
+
}
|
|
144
|
+
/** Build a fresh snapshot from a server's live `listTools` result. */
|
|
145
|
+
export function buildSnapshot(server, tools, ts) {
|
|
146
|
+
return {
|
|
147
|
+
version: 1,
|
|
148
|
+
server,
|
|
149
|
+
ts,
|
|
150
|
+
tools: tools
|
|
151
|
+
.map((t) => ({ name: t.name, schemaHash: schemaHash(t.inputSchema) }))
|
|
152
|
+
.sort((a, b) => a.name.localeCompare(b.name)),
|
|
153
|
+
};
|
|
154
|
+
}
|
|
155
|
+
/** True when nothing changed. */
|
|
156
|
+
export function driftIsEmpty(drift) {
|
|
157
|
+
return drift.added.length === 0 && drift.removed.length === 0 && drift.schemaChanged.length === 0;
|
|
158
|
+
}
|
|
159
|
+
/**
|
|
160
|
+
* Diff a previous snapshot against the current one. `previous` undefined (no
|
|
161
|
+
* snapshot on disk yet) ⇒ empty drift (the current becomes the new baseline;
|
|
162
|
+
* a first-ever connect is not "drift"). Added = in current not previous;
|
|
163
|
+
* removed = in previous not current; schemaChanged = in both, different hash.
|
|
164
|
+
*/
|
|
165
|
+
export function diffSnapshots(previous, current) {
|
|
166
|
+
if (previous === undefined)
|
|
167
|
+
return { added: [], removed: [], schemaChanged: [] };
|
|
168
|
+
const prev = new Map(previous.tools.map((t) => [t.name, t.schemaHash]));
|
|
169
|
+
const cur = new Map(current.tools.map((t) => [t.name, t.schemaHash]));
|
|
170
|
+
const added = [];
|
|
171
|
+
const removed = [];
|
|
172
|
+
const schemaChanged = [];
|
|
173
|
+
for (const [name, hash] of cur) {
|
|
174
|
+
const prevHash = prev.get(name);
|
|
175
|
+
if (prevHash === undefined)
|
|
176
|
+
added.push(name);
|
|
177
|
+
else if (prevHash !== hash)
|
|
178
|
+
schemaChanged.push(name);
|
|
179
|
+
}
|
|
180
|
+
for (const name of prev.keys()) {
|
|
181
|
+
if (!cur.has(name))
|
|
182
|
+
removed.push(name);
|
|
183
|
+
}
|
|
184
|
+
return {
|
|
185
|
+
added: added.sort(),
|
|
186
|
+
removed: removed.sort(),
|
|
187
|
+
schemaChanged: schemaChanged.sort(),
|
|
188
|
+
};
|
|
189
|
+
}
|
|
190
|
+
/**
|
|
191
|
+
* Decide catalog mutations from the current health + the set of servers whose
|
|
192
|
+
* tools are ALREADY quarantined. A chronic server not yet quarantined →
|
|
193
|
+
* quarantine; a quarantined server that is no longer chronic (its recent
|
|
194
|
+
* window is healthy — e.g. a probe succeeded) → restore. A server absent from
|
|
195
|
+
* `health` entirely (no recent calls) is left as-is: no signal either way.
|
|
196
|
+
*/
|
|
197
|
+
export function decideQuarantine(health, alreadyQuarantined) {
|
|
198
|
+
const out = new Set(alreadyQuarantined);
|
|
199
|
+
const byServer = new Map(health.map((h) => [h.server, h]));
|
|
200
|
+
const quarantine = [];
|
|
201
|
+
const restore = [];
|
|
202
|
+
for (const h of health) {
|
|
203
|
+
if (h.chronic && !out.has(h.server))
|
|
204
|
+
quarantine.push(h.server);
|
|
205
|
+
}
|
|
206
|
+
for (const server of out) {
|
|
207
|
+
const h = byServer.get(server);
|
|
208
|
+
if (h !== undefined && !h.chronic)
|
|
209
|
+
restore.push(server);
|
|
210
|
+
}
|
|
211
|
+
return { quarantine: quarantine.sort(), restore: restore.sort() };
|
|
212
|
+
}
|
|
213
|
+
/**
|
|
214
|
+
* The synthetic notice injected in place of a quarantined server's tools so the
|
|
215
|
+
* model routes around them — mirrors loop-detection's warning injection. Kept
|
|
216
|
+
* here so the report and the runtime injection render the identical text.
|
|
217
|
+
*/
|
|
218
|
+
export function quarantineNotice(server, reason) {
|
|
219
|
+
return `[mcp] tools from server "${server}" are temporarily unavailable (${reason}). Route around them; they auto-restore once the server is healthy again.`;
|
|
220
|
+
}
|
|
221
|
+
// -------- report rendering --------
|
|
222
|
+
/** Render the per-server health table lines for `crewhaus mcp doctor`. */
|
|
223
|
+
export function formatHealthReport(health) {
|
|
224
|
+
if (health.length === 0) {
|
|
225
|
+
return ["no MCP call history yet — run the agent against its MCP servers first"];
|
|
226
|
+
}
|
|
227
|
+
const lines = ["per-server MCP health (from durable mcp_stats):"];
|
|
228
|
+
for (const h of health) {
|
|
229
|
+
const mark = h.chronic ? "✗" : h.errorRate > 0 ? "~" : "✓";
|
|
230
|
+
lines.push(` ${mark} ${h.server}: ${h.calls} call(s), ${(h.errorRate * 100).toFixed(1)}% errors, ` +
|
|
231
|
+
`p50 ${Math.round(h.p50Ms)}ms / p95 ${Math.round(h.p95Ms)}ms, max error streak ${h.maxErrorStreak}` +
|
|
232
|
+
`${h.chronic ? " — CHRONIC (quarantine candidate)" : ""}`);
|
|
233
|
+
}
|
|
234
|
+
return lines;
|
|
235
|
+
}
|
|
236
|
+
/** Render the drift lines for one server. Empty array when no drift. */
|
|
237
|
+
export function formatDriftReport(server, drift) {
|
|
238
|
+
if (driftIsEmpty(drift))
|
|
239
|
+
return [];
|
|
240
|
+
const lines = [` drift on "${server}":`];
|
|
241
|
+
if (drift.added.length > 0)
|
|
242
|
+
lines.push(` + added: ${drift.added.join(", ")}`);
|
|
243
|
+
if (drift.removed.length > 0)
|
|
244
|
+
lines.push(` - removed: ${drift.removed.join(", ")}`);
|
|
245
|
+
if (drift.schemaChanged.length > 0) {
|
|
246
|
+
lines.push(` ~ schema-changed: ${drift.schemaChanged.join(", ")}`);
|
|
247
|
+
}
|
|
248
|
+
return lines;
|
|
249
|
+
}
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
import { type CandidateProvider, type CapabilityRequirement, type PricingTable } from "@crewhaus/cost-tracker";
|
|
2
|
+
import type { Spec } from "@crewhaus/spec";
|
|
3
|
+
/** Thrown on a malformed model-scan / pricing-sync invocation. */
|
|
4
|
+
export declare class ModelScanError extends Error {
|
|
5
|
+
readonly name = "ModelScanError";
|
|
6
|
+
}
|
|
7
|
+
/** A spec model string → the `(provider, modelId)` the cost-tracker tables
|
|
8
|
+
* key on. `undefined` when unparseable or routed to a provider the tables
|
|
9
|
+
* don't cover (local/, azure/, named hosts). */
|
|
10
|
+
export declare function parsedModelForTables(modelString: string): {
|
|
11
|
+
readonly provider: CandidateProvider;
|
|
12
|
+
readonly modelId: string;
|
|
13
|
+
} | undefined;
|
|
14
|
+
/** One evaluated candidate cell — the model-scan input from the matrix run. */
|
|
15
|
+
export type ScanCell = {
|
|
16
|
+
readonly model: string;
|
|
17
|
+
readonly passRate?: number;
|
|
18
|
+
readonly meanScore?: number;
|
|
19
|
+
readonly costPer1kSamplesUsd?: number;
|
|
20
|
+
readonly error?: string;
|
|
21
|
+
};
|
|
22
|
+
export type ScanProposal = {
|
|
23
|
+
readonly currentModel: string;
|
|
24
|
+
readonly candidateModel: string;
|
|
25
|
+
readonly scoreDelta: number;
|
|
26
|
+
readonly costDeltaPer1kUsd: number;
|
|
27
|
+
/** True when the candidate strictly beats current: >= score AND < cost. */
|
|
28
|
+
readonly recommended: boolean;
|
|
29
|
+
readonly reason: string;
|
|
30
|
+
};
|
|
31
|
+
export type MarketScanResult = {
|
|
32
|
+
readonly currentModel: string;
|
|
33
|
+
readonly current?: ScanCell;
|
|
34
|
+
readonly candidates: ReadonlyArray<string>;
|
|
35
|
+
readonly proposals: ReadonlyArray<ScanProposal>;
|
|
36
|
+
/** The single best recommended proposal (highest score at lowest cost), if any. */
|
|
37
|
+
readonly best?: ScanProposal;
|
|
38
|
+
};
|
|
39
|
+
export type BuildScanCandidatesOptions = {
|
|
40
|
+
readonly pricing?: PricingTable;
|
|
41
|
+
readonly require?: CapabilityRequirement;
|
|
42
|
+
/** Cross-provider replacements are in scope for a human-reviewed proposal. */
|
|
43
|
+
readonly sameProviderOnly?: boolean;
|
|
44
|
+
/** Cap the number of candidates evaled (cheapest-first). Default 6. */
|
|
45
|
+
readonly limit?: number;
|
|
46
|
+
};
|
|
47
|
+
/**
|
|
48
|
+
* Enumerate the replacement models model-scan will eval. Cheapest-blended
|
|
49
|
+
* first, current family excluded, capped. Returns spec model strings.
|
|
50
|
+
*/
|
|
51
|
+
export declare function buildScanCandidates(currentModel: string, opts?: BuildScanCandidatesOptions): string[];
|
|
52
|
+
/**
|
|
53
|
+
* Fold matrix cells into scan proposals. `current` is the cell for the spec's
|
|
54
|
+
* existing model; each candidate cell becomes a proposal comparing score +
|
|
55
|
+
* projected cost. A candidate is RECOMMENDED when it holds score (>= current)
|
|
56
|
+
* AND costs less — the strict "beats current on score at lower cost" bar. The
|
|
57
|
+
* `best` proposal maximizes score then minimizes cost among recommended.
|
|
58
|
+
*/
|
|
59
|
+
export declare function buildMarketScan(currentModel: string, cells: ReadonlyArray<ScanCell>): MarketScanResult;
|
|
60
|
+
/** The machine-readable proposal artifact model-scan writes next to a report. */
|
|
61
|
+
export type ScanProposalArtifact = {
|
|
62
|
+
readonly kind: "model-scan-proposal";
|
|
63
|
+
readonly generatedAt: string;
|
|
64
|
+
readonly currentModel: string;
|
|
65
|
+
readonly recommendedModel: string;
|
|
66
|
+
readonly scoreDelta: number;
|
|
67
|
+
readonly costDeltaPer1kUsd: number;
|
|
68
|
+
/** The spec edit a human (or `--write`) applies: replace agent.model. */
|
|
69
|
+
readonly patch: {
|
|
70
|
+
readonly op: "replace";
|
|
71
|
+
readonly path: ReadonlyArray<string>;
|
|
72
|
+
readonly value: string;
|
|
73
|
+
};
|
|
74
|
+
};
|
|
75
|
+
/** Build the patch.json artifact for the best proposal (caller guards `best`). */
|
|
76
|
+
export declare function buildProposalArtifact(result: MarketScanResult, now?: () => Date): ScanProposalArtifact | undefined;
|
|
77
|
+
export type ModelDoctorCheck = {
|
|
78
|
+
readonly label: string;
|
|
79
|
+
readonly pass: boolean;
|
|
80
|
+
readonly warn?: boolean;
|
|
81
|
+
readonly reason?: string;
|
|
82
|
+
};
|
|
83
|
+
export type BuildModelChecksOptions = {
|
|
84
|
+
readonly pricing?: PricingTable;
|
|
85
|
+
readonly now?: Date;
|
|
86
|
+
/** Warn when the pricing table is older than this many days. Default 120. */
|
|
87
|
+
readonly maxAgeDays?: number;
|
|
88
|
+
/** Extra model strings to check (compaction.model, judge model, sub-agents). */
|
|
89
|
+
readonly auxModels?: ReadonlyArray<{
|
|
90
|
+
readonly slot: string;
|
|
91
|
+
readonly model: string;
|
|
92
|
+
}>;
|
|
93
|
+
};
|
|
94
|
+
/**
|
|
95
|
+
* doctor --models: (a) every spec model missing from the pricing table
|
|
96
|
+
* (silently billed $0 today — surfaces the gap), (b) pricing-table staleness,
|
|
97
|
+
* (c) known-sunset model ids. Pure — the CLI reads the spec + prints.
|
|
98
|
+
*/
|
|
99
|
+
export declare function buildModelChecks(agentModel: string | undefined, opts?: BuildModelChecksOptions): ModelDoctorCheck[];
|
|
100
|
+
/** ~/.crewhaus/pricing/<version>.json target path for a feed version. */
|
|
101
|
+
export declare function pricingFeedPath(homeDir: string, version: string): string;
|
|
102
|
+
/**
|
|
103
|
+
* Model fields are OUTSIDE `OPTIMIZABLE_PATHS` by design, so the normal
|
|
104
|
+
* optimizer apply path (`validatePatch` → `applySpecPatch`) REFUSES a model
|
|
105
|
+
* patch. `crewhaus model-scan --write` deliberately bypasses the whitelist
|
|
106
|
+
* gate: it calls `applySpecPatch` DIRECTLY (which never consults
|
|
107
|
+
* OPTIMIZABLE_PATHS — that check lives only in the separately-invoked
|
|
108
|
+
* `validatePatch`), reusing spec-patch's yaml-document CST technique to
|
|
109
|
+
* rewrite the value while preserving every comment, key order, and blank
|
|
110
|
+
* line. The resulting YAML is re-parsed by `applySpecPatch` (so an invalid
|
|
111
|
+
* model still fails loudly), and every `model-scan --write` is a
|
|
112
|
+
* human-initiated action on a human-reviewed proposal — never automatic.
|
|
113
|
+
*
|
|
114
|
+
* Returns the rewritten YAML text (caller writes it back to disk).
|
|
115
|
+
*/
|
|
116
|
+
export declare function writeModelField(yamlText: string, target: Spec["target"], path: ReadonlyArray<string>, value: string): string;
|
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
import { DEFAULT_CAPABILITIES, DEFAULT_PRICING, classifyPricingStaleness, enumerateCandidates, findSunset, resolvePricing, } from "@crewhaus/cost-tracker";
|
|
2
|
+
/**
|
|
3
|
+
* Item 24 — `crewhaus model-scan`, `crewhaus doctor --models`, and
|
|
4
|
+
* `crewhaus pricing sync`, in a side-effect-free module (the CLI entry file
|
|
5
|
+
* runs an argv switch on import) mirroring `eval-matrix.ts` / `doctor-checks.ts`.
|
|
6
|
+
*
|
|
7
|
+
* All three read the pricing/capability tables that ship in `@crewhaus/cost-tracker`:
|
|
8
|
+
* - model-scan enumerates capability-compatible replacement candidates for
|
|
9
|
+
* the cwd spec's `agent.model`, evals each on the spec's dataset (the same
|
|
10
|
+
* `--models` matrix machinery, injected as `runMatrix`), and emits a
|
|
11
|
+
* proposal + patch.json when a candidate beats current on score at lower
|
|
12
|
+
* cost. NEVER auto-applied — model fields are outside OPTIMIZABLE_PATHS by
|
|
13
|
+
* design, so `--write` does its OWN comment-preserving CST edit
|
|
14
|
+
* (`writeModelField`, reusing spec-patch's yaml-document technique).
|
|
15
|
+
* - doctor --models flags pricing misses / staleness / known sunsets.
|
|
16
|
+
* - pricing sync loads a versioned feed into ~/.crewhaus/pricing/.
|
|
17
|
+
*/
|
|
18
|
+
import { parseModelString } from "@crewhaus/model-router";
|
|
19
|
+
import { applySpecPatch } from "@crewhaus/spec-patch";
|
|
20
|
+
/** Thrown on a malformed model-scan / pricing-sync invocation. */
|
|
21
|
+
export class ModelScanError extends Error {
|
|
22
|
+
name = "ModelScanError";
|
|
23
|
+
}
|
|
24
|
+
// ---------- provider mapping ----------
|
|
25
|
+
const CANDIDATE_PROVIDERS = ["anthropic", "openai", "gemini", "bedrock"];
|
|
26
|
+
/** A spec model string → the `(provider, modelId)` the cost-tracker tables
|
|
27
|
+
* key on. `undefined` when unparseable or routed to a provider the tables
|
|
28
|
+
* don't cover (local/, azure/, named hosts). */
|
|
29
|
+
export function parsedModelForTables(modelString) {
|
|
30
|
+
let parsed;
|
|
31
|
+
try {
|
|
32
|
+
parsed = parseModelString(modelString);
|
|
33
|
+
}
|
|
34
|
+
catch {
|
|
35
|
+
return undefined;
|
|
36
|
+
}
|
|
37
|
+
// local/azure/named-host all parse to providerId "openai" but with a
|
|
38
|
+
// baseUrl/azure/hostId — those aren't in the pricing table, so skip.
|
|
39
|
+
if (parsed.providerId === "openai") {
|
|
40
|
+
if (parsed.baseUrl !== undefined || parsed.azure !== undefined || parsed.hostId !== undefined) {
|
|
41
|
+
return undefined;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
if (!CANDIDATE_PROVIDERS.includes(parsed.providerId))
|
|
45
|
+
return undefined;
|
|
46
|
+
return { provider: parsed.providerId, modelId: parsed.modelId };
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Enumerate the replacement models model-scan will eval. Cheapest-blended
|
|
50
|
+
* first, current family excluded, capped. Returns spec model strings.
|
|
51
|
+
*/
|
|
52
|
+
export function buildScanCandidates(currentModel, opts = {}) {
|
|
53
|
+
const parsed = parsedModelForTables(currentModel);
|
|
54
|
+
if (parsed === undefined)
|
|
55
|
+
return [];
|
|
56
|
+
const cands = enumerateCandidates(parsed, {
|
|
57
|
+
pricing: opts.pricing ?? DEFAULT_PRICING,
|
|
58
|
+
capabilities: DEFAULT_CAPABILITIES,
|
|
59
|
+
...(opts.require !== undefined ? { require: opts.require } : {}),
|
|
60
|
+
sameProviderOnly: opts.sameProviderOnly === true,
|
|
61
|
+
excludeCurrent: true,
|
|
62
|
+
});
|
|
63
|
+
const limit = opts.limit ?? 6;
|
|
64
|
+
return cands.slice(0, limit).map((c) => c.modelString);
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Fold matrix cells into scan proposals. `current` is the cell for the spec's
|
|
68
|
+
* existing model; each candidate cell becomes a proposal comparing score +
|
|
69
|
+
* projected cost. A candidate is RECOMMENDED when it holds score (>= current)
|
|
70
|
+
* AND costs less — the strict "beats current on score at lower cost" bar. The
|
|
71
|
+
* `best` proposal maximizes score then minimizes cost among recommended.
|
|
72
|
+
*/
|
|
73
|
+
export function buildMarketScan(currentModel, cells) {
|
|
74
|
+
const current = cells.find((c) => c.model === currentModel);
|
|
75
|
+
const candidateCells = cells.filter((c) => c.model !== currentModel);
|
|
76
|
+
const curScore = current?.meanScore ?? 0;
|
|
77
|
+
const curCost = current?.costPer1kSamplesUsd;
|
|
78
|
+
const proposals = candidateCells.map((cell) => {
|
|
79
|
+
if (cell.error !== undefined || cell.meanScore === undefined) {
|
|
80
|
+
return {
|
|
81
|
+
currentModel,
|
|
82
|
+
candidateModel: cell.model,
|
|
83
|
+
scoreDelta: 0,
|
|
84
|
+
costDeltaPer1kUsd: 0,
|
|
85
|
+
recommended: false,
|
|
86
|
+
reason: cell.error !== undefined ? `cell errored: ${cell.error}` : "no score",
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
const scoreDelta = cell.meanScore - curScore;
|
|
90
|
+
const candCost = cell.costPer1kSamplesUsd;
|
|
91
|
+
const costDelta = candCost !== undefined && curCost !== undefined ? candCost - curCost : Number.NaN;
|
|
92
|
+
const cheaper = Number.isFinite(costDelta) && costDelta < 0;
|
|
93
|
+
const holdsScore = scoreDelta >= 0;
|
|
94
|
+
const recommended = holdsScore && cheaper;
|
|
95
|
+
return {
|
|
96
|
+
currentModel,
|
|
97
|
+
candidateModel: cell.model,
|
|
98
|
+
scoreDelta,
|
|
99
|
+
costDeltaPer1kUsd: Number.isFinite(costDelta) ? costDelta : 0,
|
|
100
|
+
recommended,
|
|
101
|
+
reason: recommended
|
|
102
|
+
? `+${scoreDelta.toFixed(3)} score at ${costDelta < 0 ? "" : "+"}${costDelta.toFixed(4)} $/1k`
|
|
103
|
+
: !Number.isFinite(costDelta)
|
|
104
|
+
? "cost unknown (pricing miss) — cannot compare spend"
|
|
105
|
+
: !holdsScore
|
|
106
|
+
? `score dropped ${scoreDelta.toFixed(3)}`
|
|
107
|
+
: "not cheaper than current",
|
|
108
|
+
};
|
|
109
|
+
});
|
|
110
|
+
const recommended = proposals.filter((p) => p.recommended);
|
|
111
|
+
// Best: max score delta, then min (most-negative) cost delta.
|
|
112
|
+
const best = recommended.sort((a, b) => {
|
|
113
|
+
if (b.scoreDelta !== a.scoreDelta)
|
|
114
|
+
return b.scoreDelta - a.scoreDelta;
|
|
115
|
+
return a.costDeltaPer1kUsd - b.costDeltaPer1kUsd;
|
|
116
|
+
})[0];
|
|
117
|
+
return {
|
|
118
|
+
currentModel,
|
|
119
|
+
...(current !== undefined ? { current } : {}),
|
|
120
|
+
candidates: candidateCells.map((c) => c.model),
|
|
121
|
+
proposals,
|
|
122
|
+
...(best !== undefined ? { best } : {}),
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
/** Build the patch.json artifact for the best proposal (caller guards `best`). */
|
|
126
|
+
export function buildProposalArtifact(result, now = () => new Date()) {
|
|
127
|
+
if (result.best === undefined)
|
|
128
|
+
return undefined;
|
|
129
|
+
return {
|
|
130
|
+
kind: "model-scan-proposal",
|
|
131
|
+
generatedAt: now().toISOString(),
|
|
132
|
+
currentModel: result.currentModel,
|
|
133
|
+
recommendedModel: result.best.candidateModel,
|
|
134
|
+
scoreDelta: result.best.scoreDelta,
|
|
135
|
+
costDeltaPer1kUsd: result.best.costDeltaPer1kUsd,
|
|
136
|
+
patch: { op: "replace", path: ["agent", "model"], value: result.best.candidateModel },
|
|
137
|
+
};
|
|
138
|
+
}
|
|
139
|
+
/**
|
|
140
|
+
* doctor --models: (a) every spec model missing from the pricing table
|
|
141
|
+
* (silently billed $0 today — surfaces the gap), (b) pricing-table staleness,
|
|
142
|
+
* (c) known-sunset model ids. Pure — the CLI reads the spec + prints.
|
|
143
|
+
*/
|
|
144
|
+
export function buildModelChecks(agentModel, opts = {}) {
|
|
145
|
+
const pricing = opts.pricing ?? DEFAULT_PRICING;
|
|
146
|
+
const now = opts.now ?? new Date();
|
|
147
|
+
const checks = [];
|
|
148
|
+
const models = [];
|
|
149
|
+
if (agentModel !== undefined)
|
|
150
|
+
models.push({ slot: "agent.model", model: agentModel });
|
|
151
|
+
for (const aux of opts.auxModels ?? [])
|
|
152
|
+
models.push({ slot: aux.slot, model: aux.model });
|
|
153
|
+
for (const { slot, model } of models) {
|
|
154
|
+
const parsed = parsedModelForTables(model);
|
|
155
|
+
if (parsed === undefined) {
|
|
156
|
+
checks.push({
|
|
157
|
+
label: `${slot} pricing (${model})`,
|
|
158
|
+
pass: true,
|
|
159
|
+
warn: true,
|
|
160
|
+
reason: "routed to a provider outside the pricing table (local/azure/named-host or unparseable) — cost tracking reports $0 for this model",
|
|
161
|
+
});
|
|
162
|
+
continue;
|
|
163
|
+
}
|
|
164
|
+
const row = resolvePricing(pricing, parsed.provider, parsed.modelId);
|
|
165
|
+
if (row === undefined) {
|
|
166
|
+
checks.push({
|
|
167
|
+
label: `${slot} pricing (${model})`,
|
|
168
|
+
pass: false,
|
|
169
|
+
reason: `no pricing row for ${parsed.provider}/${parsed.modelId} — cost-tracker silently bills it $0; add a row to DEFAULT_PRICING or \`crewhaus pricing sync\` a feed`,
|
|
170
|
+
});
|
|
171
|
+
}
|
|
172
|
+
else {
|
|
173
|
+
checks.push({ label: `${slot} pricing (${model})`, pass: true });
|
|
174
|
+
}
|
|
175
|
+
// Sunset watch.
|
|
176
|
+
const sunset = findSunset(parsed.provider, parsed.modelId);
|
|
177
|
+
if (sunset !== undefined) {
|
|
178
|
+
checks.push({
|
|
179
|
+
label: `${slot} sunset (${model})`,
|
|
180
|
+
pass: true,
|
|
181
|
+
warn: true,
|
|
182
|
+
reason: `retires ${sunset.retiresOn} — migrate to ${sunset.replacement}${sunset.note !== undefined ? ` (${sunset.note})` : ""}`,
|
|
183
|
+
});
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
// Pricing-table freshness.
|
|
187
|
+
const staleness = classifyPricingStaleness(pricing, now, opts.maxAgeDays ?? 120);
|
|
188
|
+
checks.push({
|
|
189
|
+
label: `pricing table freshness (v${staleness.version})`,
|
|
190
|
+
pass: true,
|
|
191
|
+
...(staleness.stale ? { warn: true, reason: staleness.reason } : {}),
|
|
192
|
+
});
|
|
193
|
+
return checks;
|
|
194
|
+
}
|
|
195
|
+
// ---------- pricing sync ----------
|
|
196
|
+
/** ~/.crewhaus/pricing/<version>.json target path for a feed version. */
|
|
197
|
+
export function pricingFeedPath(homeDir, version) {
|
|
198
|
+
// Basename-safe version (dates are already safe; be defensive).
|
|
199
|
+
const safe = version.replace(/[^A-Za-z0-9._-]+/g, "_");
|
|
200
|
+
return `${homeDir}/.crewhaus/pricing/${safe}.json`;
|
|
201
|
+
}
|
|
202
|
+
// ---------- --write: direct comment-preserving CST edit of agent.model ----------
|
|
203
|
+
/**
|
|
204
|
+
* Model fields are OUTSIDE `OPTIMIZABLE_PATHS` by design, so the normal
|
|
205
|
+
* optimizer apply path (`validatePatch` → `applySpecPatch`) REFUSES a model
|
|
206
|
+
* patch. `crewhaus model-scan --write` deliberately bypasses the whitelist
|
|
207
|
+
* gate: it calls `applySpecPatch` DIRECTLY (which never consults
|
|
208
|
+
* OPTIMIZABLE_PATHS — that check lives only in the separately-invoked
|
|
209
|
+
* `validatePatch`), reusing spec-patch's yaml-document CST technique to
|
|
210
|
+
* rewrite the value while preserving every comment, key order, and blank
|
|
211
|
+
* line. The resulting YAML is re-parsed by `applySpecPatch` (so an invalid
|
|
212
|
+
* model still fails loudly), and every `model-scan --write` is a
|
|
213
|
+
* human-initiated action on a human-reviewed proposal — never automatic.
|
|
214
|
+
*
|
|
215
|
+
* Returns the rewritten YAML text (caller writes it back to disk).
|
|
216
|
+
*/
|
|
217
|
+
export function writeModelField(yamlText, target, path, value) {
|
|
218
|
+
const { yaml } = applySpecPatch(yamlText, {
|
|
219
|
+
target,
|
|
220
|
+
path: [...path],
|
|
221
|
+
op: "replace",
|
|
222
|
+
value,
|
|
223
|
+
rationale: "model-scan --write: human-reviewed model replacement (outside OPTIMIZABLE_PATHS)",
|
|
224
|
+
});
|
|
225
|
+
return yaml;
|
|
226
|
+
}
|