crewhaus 0.1.7 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/feedback.d.ts +239 -0
- package/dist/feedback.js +602 -0
- package/dist/index.js +372 -22
- package/package.json +65 -64
package/dist/index.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env bun
|
|
2
|
-
import {
|
|
2
|
+
import { randomBytes } from "node:crypto";
|
|
3
|
+
import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
|
|
3
4
|
import { basename, dirname, join, relative, resolve } from "node:path";
|
|
4
5
|
import { SpecParseError, compile, lower } from "@crewhaus/compiler";
|
|
5
6
|
import { buildContextBundle, discoverRoots } from "@crewhaus/context-bundle";
|
|
@@ -9,6 +10,7 @@ import { parseGradersConfig } from "@crewhaus/eval-grader";
|
|
|
9
10
|
import { optimizeSpec } from "@crewhaus/eval-optimizer-orchestrator";
|
|
10
11
|
import { diffReports, loadRun, renderReport } from "@crewhaus/eval-report";
|
|
11
12
|
import { runEval as runEvalLib } from "@crewhaus/eval-runner";
|
|
13
|
+
import { openEventLog } from "@crewhaus/event-log";
|
|
12
14
|
import { loadHooks } from "@crewhaus/hooks-engine";
|
|
13
15
|
import { ArgParseError, parseArgs, } from "@crewhaus/infra-utils";
|
|
14
16
|
import { createLogger } from "@crewhaus/logging";
|
|
@@ -33,6 +35,10 @@ import { buildCredentialChecks, extractSpecModel } from "./doctor-checks";
|
|
|
33
35
|
// imports the optional semantic package only when "semantic" is selected, so
|
|
34
36
|
// the default path pulls in no embedding dependency.
|
|
35
37
|
import { DEFAULT_EGRESS_EMBEDDER_MODEL, InvalidEgressMatcherChoiceError, createEgressMatcher, resolveEgressMatcherChoice, } from "./egress-matcher";
|
|
38
|
+
// Response-feedback core — pure, side-effect-free so it is unit-testable
|
|
39
|
+
// (this entry file runs an argv switch on import). Powers `rate`/`feedback`
|
|
40
|
+
// (capture) and `distill` (ratings → eval dataset + graders).
|
|
41
|
+
import { FEEDBACK_EVENT_KIND, buildFeedbackRecord, deriveTurns, distill as distillFeedback, extractFeedbackRecords, gradersConfigToYaml, samplesToJsonl, } from "./feedback";
|
|
36
42
|
// FR-004 — Pillar 3 intent-gate judge + durable audit-sink resolution, in a
|
|
37
43
|
// side-effect-free module so it is unit-testable (this entry file runs an
|
|
38
44
|
// argv switch on import).
|
|
@@ -134,6 +140,11 @@ const OPTIMIZE_SCHEMA = {
|
|
|
134
140
|
flags: [
|
|
135
141
|
{ name: "dataset", takesValue: true },
|
|
136
142
|
{ name: "graders", takesValue: true },
|
|
143
|
+
// Inline-distill user ratings into the training set (Pillar 2 — close the
|
|
144
|
+
// loop from real feedback). Value: a session id (sess_<16 hex>) or "all".
|
|
145
|
+
// Synthesizes the dataset (and, when --graders is omitted, the graders too).
|
|
146
|
+
{ name: "ratings", takesValue: true },
|
|
147
|
+
{ name: "min-score", takesValue: true },
|
|
137
148
|
{ name: "mutator", takesValue: true },
|
|
138
149
|
{ name: "iterations", takesValue: true },
|
|
139
150
|
{ name: "seed", takesValue: true },
|
|
@@ -171,6 +182,40 @@ const COST_SUMMARY_SCHEMA = {
|
|
|
171
182
|
{ name: "help", short: "h" },
|
|
172
183
|
],
|
|
173
184
|
};
|
|
185
|
+
const RATE_SCHEMA = {
|
|
186
|
+
flags: [
|
|
187
|
+
{ name: "session", takesValue: true },
|
|
188
|
+
{ name: "turn", takesValue: true },
|
|
189
|
+
{ name: "thumbs", takesValue: true },
|
|
190
|
+
{ name: "stars", takesValue: true },
|
|
191
|
+
{ name: "score", takesValue: true },
|
|
192
|
+
{ name: "comment", takesValue: true },
|
|
193
|
+
{ name: "rater", takesValue: true },
|
|
194
|
+
{ name: "help", short: "h" },
|
|
195
|
+
],
|
|
196
|
+
};
|
|
197
|
+
const FEEDBACK_SCHEMA = {
|
|
198
|
+
flags: [
|
|
199
|
+
{ name: "session", takesValue: true },
|
|
200
|
+
{ name: "turn", takesValue: true },
|
|
201
|
+
{ name: "text", takesValue: true },
|
|
202
|
+
{ name: "correction", takesValue: true },
|
|
203
|
+
{ name: "rater", takesValue: true },
|
|
204
|
+
{ name: "help", short: "h" },
|
|
205
|
+
],
|
|
206
|
+
};
|
|
207
|
+
const DISTILL_SCHEMA = {
|
|
208
|
+
flags: [
|
|
209
|
+
{ name: "session", takesValue: true },
|
|
210
|
+
{ name: "all-sessions", takesValue: false },
|
|
211
|
+
{ name: "out", short: "o", takesValue: true },
|
|
212
|
+
{ name: "graders-out", takesValue: true },
|
|
213
|
+
{ name: "min-score", takesValue: true },
|
|
214
|
+
{ name: "judge", takesValue: false },
|
|
215
|
+
{ name: "judge-model", takesValue: true },
|
|
216
|
+
{ name: "help", short: "h" },
|
|
217
|
+
],
|
|
218
|
+
};
|
|
174
219
|
const SANDBOX_SCHEMA = {
|
|
175
220
|
flags: [
|
|
176
221
|
{ name: "probe", takesValue: false },
|
|
@@ -269,6 +314,7 @@ function usageText() {
|
|
|
269
314
|
" [-o <out-dir>]",
|
|
270
315
|
" optimize <spec.yaml> --dataset <data> --graders <graders.yaml>",
|
|
271
316
|
" [--mutator rule-based|claude] [--iterations N] [--seed N]",
|
|
317
|
+
" [--ratings <session>|all] distill user ratings into the training set (Pillar 2)",
|
|
272
318
|
" [--budget-usd N] stop a model-driven run before it exceeds $N (FR-003)",
|
|
273
319
|
" [--write-back] [-o <out-dir>] active eval-driven optimization (Pillar 2)",
|
|
274
320
|
" init [name] scaffold a new crewhaus.yaml",
|
|
@@ -276,6 +322,12 @@ function usageText() {
|
|
|
276
322
|
" context --bundle [-o <file>] emit a single-markdown orientation manifest",
|
|
277
323
|
" [--factory-root <p>] [--docs-root <p>] [--demos-root <p>]",
|
|
278
324
|
" cost-summary --session <id> summarize cost_accrual events for a session",
|
|
325
|
+
" rate --session <id> [--turn N] rate an assistant turn 👍/👎, ⭐, or 0–1",
|
|
326
|
+
" (--thumbs up|down | --stars 1-5 | --score 0-1) [--comment <t>]",
|
|
327
|
+
" feedback --session <id> --text <msg> attach a comment/correction to a turn",
|
|
328
|
+
" [--turn N] [--correction <better answer>]",
|
|
329
|
+
" distill --session <id> -o <ds.jsonl> turn ratings into an eval dataset + graders",
|
|
330
|
+
" [--all-sessions] [--graders-out <g.yaml>] [--min-score F]",
|
|
279
331
|
" secrets doctor list known secrets via the configured backend",
|
|
280
332
|
" secrets rotate <name> [--value V] rotate a named secret (file backend)",
|
|
281
333
|
" spec put|list|get|pin|alias ... versioned spec storage (Section 28 spec-registry)",
|
|
@@ -1332,8 +1384,8 @@ async function runDoctorPhilosophyAlignment() {
|
|
|
1332
1384
|
*/
|
|
1333
1385
|
async function runOptimize(args) {
|
|
1334
1386
|
if (args.flags["help"]) {
|
|
1335
|
-
process.stdout.write("usage: crewhaus optimize <spec.yaml> --dataset <data> --graders <graders.yaml> " +
|
|
1336
|
-
"[--mutator rule-based|claude] [--iterations N] [--seed N] [--concurrency N] " +
|
|
1387
|
+
process.stdout.write("usage: crewhaus optimize <spec.yaml> (--dataset <data> --graders <graders.yaml> | --ratings <session>) " +
|
|
1388
|
+
"[--min-score F] [--mutator rule-based|claude] [--iterations N] [--seed N] [--concurrency N] " +
|
|
1337
1389
|
"[--improvement-threshold F] [--budget-usd N] [--write-back] [-o <out-dir>]\n");
|
|
1338
1390
|
return;
|
|
1339
1391
|
}
|
|
@@ -1342,10 +1394,18 @@ async function runOptimize(args) {
|
|
|
1342
1394
|
die("missing <spec.yaml>");
|
|
1343
1395
|
const datasetPath = args.flags["dataset"];
|
|
1344
1396
|
const gradersPath = args.flags["graders"];
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1397
|
+
const ratingsArg = args.flags["ratings"];
|
|
1398
|
+
if (typeof datasetPath !== "string" && typeof ratingsArg !== "string") {
|
|
1399
|
+
die("missing --dataset <data> (or --ratings <session> to distill one from feedback)");
|
|
1400
|
+
}
|
|
1401
|
+
if (typeof gradersPath !== "string" && typeof ratingsArg !== "string") {
|
|
1402
|
+
die("missing --graders <graders.yaml> (or --ratings to synthesize one from feedback)");
|
|
1403
|
+
}
|
|
1404
|
+
const ratingsMinScore = floatFlag(args, "min-score") ?? 0.7;
|
|
1405
|
+
if (ratingsMinScore < 0 || ratingsMinScore > 1) {
|
|
1406
|
+
die(`invalid --min-score "${ratingsMinScore}" — must be in [0,1]`);
|
|
1407
|
+
}
|
|
1408
|
+
const ratingsDistill = typeof ratingsArg === "string" ? distillRatings(ratingsArg, ratingsMinScore) : undefined;
|
|
1349
1409
|
const iterationsFlag = args.flags["iterations"];
|
|
1350
1410
|
const seedFlag = args.flags["seed"];
|
|
1351
1411
|
const thresholdFlag = args.flags["improvement-threshold"];
|
|
@@ -1383,25 +1443,40 @@ async function runOptimize(args) {
|
|
|
1383
1443
|
? resolve(outDirArg)
|
|
1384
1444
|
: resolve(join(".crewhaus", "optimize", runId));
|
|
1385
1445
|
let gradersYaml;
|
|
1386
|
-
|
|
1387
|
-
|
|
1446
|
+
if (typeof gradersPath === "string") {
|
|
1447
|
+
try {
|
|
1448
|
+
gradersYaml = readFileSync(resolve(gradersPath), "utf-8");
|
|
1449
|
+
}
|
|
1450
|
+
catch (err) {
|
|
1451
|
+
die(`could not read ${gradersPath}: ${err.message}`);
|
|
1452
|
+
}
|
|
1388
1453
|
}
|
|
1389
|
-
|
|
1390
|
-
|
|
1454
|
+
else {
|
|
1455
|
+
// No --graders: use the grader synthesized from the ratings (guaranteed
|
|
1456
|
+
// present here because the missing-flags gate required --ratings).
|
|
1457
|
+
gradersYaml = ratingsDistill.gradersYaml;
|
|
1391
1458
|
}
|
|
1392
1459
|
const { compiled } = parseGradersConfig(gradersYaml);
|
|
1393
|
-
|
|
1394
|
-
//
|
|
1460
|
+
// Materialize the training set once — we'll re-iterate per fitness call. It
|
|
1461
|
+
// is the union of the file dataset (if any) and the distilled ratings (if
|
|
1462
|
+
// any); at least one is present per the missing-flags gate above.
|
|
1395
1463
|
const samples = [];
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
|
|
1464
|
+
let datasetName = "ratings";
|
|
1465
|
+
if (typeof datasetPath === "string") {
|
|
1466
|
+
const dataset = await loadDataset(resolve(datasetPath));
|
|
1467
|
+
datasetName = dataset.name;
|
|
1468
|
+
for await (const s of dataset.samples) {
|
|
1469
|
+
samples.push({
|
|
1470
|
+
id: s.id,
|
|
1471
|
+
input: s.input,
|
|
1472
|
+
...(s.expected_output !== undefined ? { expected_output: s.expected_output } : {}),
|
|
1473
|
+
});
|
|
1474
|
+
}
|
|
1402
1475
|
}
|
|
1476
|
+
if (ratingsDistill !== undefined)
|
|
1477
|
+
samples.push(...ratingsDistill.samples);
|
|
1403
1478
|
if (samples.length === 0)
|
|
1404
|
-
die(`dataset "${
|
|
1479
|
+
die(`dataset "${datasetName}" yielded zero samples`);
|
|
1405
1480
|
// Train/dev split: 70/30 deterministic split by sample id ordering.
|
|
1406
1481
|
const splitIdx = Math.max(1, Math.floor(samples.length * 0.7));
|
|
1407
1482
|
const trainSet = samples.slice(0, splitIdx);
|
|
@@ -1449,7 +1524,7 @@ async function runOptimize(args) {
|
|
|
1449
1524
|
}
|
|
1450
1525
|
const summary = await runEvalLib({
|
|
1451
1526
|
ir,
|
|
1452
|
-
dataset: { name:
|
|
1527
|
+
dataset: { name: datasetName, samples: makeAsyncIterable(devSet) },
|
|
1453
1528
|
compiledGraders: compiled,
|
|
1454
1529
|
opts: {
|
|
1455
1530
|
outDir: join(outDir, "evals", `${prompt.length}_${ir.agent.instructions.length}`),
|
|
@@ -1489,7 +1564,7 @@ async function runOptimize(args) {
|
|
|
1489
1564
|
else if (mutator !== undefined && mutator !== "rule-based") {
|
|
1490
1565
|
die(`unknown --mutator "${mutator}" — supported: rule-based, claude`);
|
|
1491
1566
|
}
|
|
1492
|
-
process.stdout.write(`[optimize] runId=${runId} spec=${specPath} dataset=${
|
|
1567
|
+
process.stdout.write(`[optimize] runId=${runId} spec=${specPath} dataset=${datasetName} ` +
|
|
1493
1568
|
`(${trainSet.length} train / ${devSet.length} dev) iterations=${iterations} ` +
|
|
1494
1569
|
`mutator=${mutator ?? "rule-based"}\n`);
|
|
1495
1570
|
// FR-003 — thread a real trace bus into the optimize run so the per-call
|
|
@@ -1736,6 +1811,272 @@ async function runCostSummary(args) {
|
|
|
1736
1811
|
}
|
|
1737
1812
|
}
|
|
1738
1813
|
}
|
|
1814
|
+
// -------- response feedback: rate / feedback / distill --------
|
|
1815
|
+
const SESSIONS_SUBDIR = join(".crewhaus", "sessions");
|
|
1816
|
+
const FEEDBACK_SUBDIR = join(".crewhaus", "feedback");
|
|
1817
|
+
function sessionJsonlPath(session) {
|
|
1818
|
+
return join(process.cwd(), SESSIONS_SUBDIR, `${session}.jsonl`);
|
|
1819
|
+
}
|
|
1820
|
+
/** Parse a JSONL blob into objects, skipping blank/malformed lines. */
|
|
1821
|
+
function parseJsonlObjects(text) {
|
|
1822
|
+
const out = [];
|
|
1823
|
+
for (const line of text.split("\n")) {
|
|
1824
|
+
if (line.trim() === "")
|
|
1825
|
+
continue;
|
|
1826
|
+
try {
|
|
1827
|
+
out.push(JSON.parse(line));
|
|
1828
|
+
}
|
|
1829
|
+
catch {
|
|
1830
|
+
// A single malformed line must not abort a read.
|
|
1831
|
+
}
|
|
1832
|
+
}
|
|
1833
|
+
return out;
|
|
1834
|
+
}
|
|
1835
|
+
function readSessionEvents(session) {
|
|
1836
|
+
const file = sessionJsonlPath(session);
|
|
1837
|
+
if (!existsSync(file))
|
|
1838
|
+
die(`session log not found at ${file}`);
|
|
1839
|
+
return parseJsonlObjects(readFileSync(file, "utf-8"));
|
|
1840
|
+
}
|
|
1841
|
+
function strFlag(args, name) {
|
|
1842
|
+
const v = args.flags[name];
|
|
1843
|
+
return typeof v === "string" ? v : undefined;
|
|
1844
|
+
}
|
|
1845
|
+
function intFlag(args, name) {
|
|
1846
|
+
const v = args.flags[name];
|
|
1847
|
+
if (typeof v !== "string")
|
|
1848
|
+
return undefined;
|
|
1849
|
+
const n = Number.parseInt(v, 10);
|
|
1850
|
+
if (Number.isNaN(n))
|
|
1851
|
+
die(`invalid --${name} "${v}" — must be an integer`);
|
|
1852
|
+
return n;
|
|
1853
|
+
}
|
|
1854
|
+
function floatFlag(args, name) {
|
|
1855
|
+
const v = args.flags[name];
|
|
1856
|
+
if (typeof v !== "string")
|
|
1857
|
+
return undefined;
|
|
1858
|
+
const n = Number.parseFloat(v);
|
|
1859
|
+
if (Number.isNaN(n))
|
|
1860
|
+
die(`invalid --${name} "${v}" — must be a number`);
|
|
1861
|
+
return n;
|
|
1862
|
+
}
|
|
1863
|
+
/** Resolve --turn against the transcript-derived turns; default to the last. */
|
|
1864
|
+
function resolveTurn(args, turns) {
|
|
1865
|
+
const flag = args.flags["turn"];
|
|
1866
|
+
const last = turns[turns.length - 1];
|
|
1867
|
+
if (typeof flag !== "string")
|
|
1868
|
+
return last.turnNumber;
|
|
1869
|
+
const n = Number.parseInt(flag, 10);
|
|
1870
|
+
if (Number.isNaN(n))
|
|
1871
|
+
die(`invalid --turn "${flag}" — must be an integer`);
|
|
1872
|
+
if (!turns.some((t) => t.turnNumber === n)) {
|
|
1873
|
+
die(`turn ${n} not found — session has turns 1..${turns.length}`);
|
|
1874
|
+
}
|
|
1875
|
+
return n;
|
|
1876
|
+
}
|
|
1877
|
+
/** Shared capture path for `rate` and `feedback`: validate the session, derive
|
|
1878
|
+
* turns, build a FeedbackRecord, and append it as a `user_feedback` event. */
|
|
1879
|
+
async function captureFeedback(args, source, fields) {
|
|
1880
|
+
const session = args.flags["session"];
|
|
1881
|
+
if (typeof session !== "string")
|
|
1882
|
+
die("missing --session <id>");
|
|
1883
|
+
if (!SESSION_ID_REGEX.test(session))
|
|
1884
|
+
die(`invalid --session "${session}" — expected sess_<16 hex>`);
|
|
1885
|
+
const turns = deriveTurns(readSessionEvents(session));
|
|
1886
|
+
if (turns.length === 0)
|
|
1887
|
+
die(`session ${session} has no user turns to rate`);
|
|
1888
|
+
const turnNumber = resolveTurn(args, turns);
|
|
1889
|
+
let record;
|
|
1890
|
+
try {
|
|
1891
|
+
record = buildFeedbackRecord({
|
|
1892
|
+
id: `fb_${randomBytes(6).toString("hex")}`,
|
|
1893
|
+
sessionId: session,
|
|
1894
|
+
turnNumber,
|
|
1895
|
+
ts: new Date().toISOString(),
|
|
1896
|
+
source,
|
|
1897
|
+
...fields,
|
|
1898
|
+
});
|
|
1899
|
+
}
|
|
1900
|
+
catch (err) {
|
|
1901
|
+
die(err instanceof Error ? err.message : String(err));
|
|
1902
|
+
}
|
|
1903
|
+
const log = await openEventLog(session, { rootDir: join(process.cwd(), SESSIONS_SUBDIR) });
|
|
1904
|
+
await log.append({ kind: FEEDBACK_EVENT_KIND, payload: record });
|
|
1905
|
+
process.stdout.write(`recorded ${record.modality} feedback on ${session} turn ${turnNumber}\n`);
|
|
1906
|
+
}
|
|
1907
|
+
async function runRate(args) {
|
|
1908
|
+
if (args.flags["help"]) {
|
|
1909
|
+
process.stdout.write("usage: crewhaus rate --session <id> [--turn N] " +
|
|
1910
|
+
"(--thumbs up|down | --stars 1-5 | --score 0-1) [--comment <text>] [--rater <who>]\n");
|
|
1911
|
+
return;
|
|
1912
|
+
}
|
|
1913
|
+
const thumbsFlag = args.flags["thumbs"];
|
|
1914
|
+
let thumbs;
|
|
1915
|
+
if (typeof thumbsFlag === "string") {
|
|
1916
|
+
if (thumbsFlag !== "up" && thumbsFlag !== "down")
|
|
1917
|
+
die(`--thumbs must be "up" or "down" (got "${thumbsFlag}")`);
|
|
1918
|
+
thumbs = thumbsFlag;
|
|
1919
|
+
}
|
|
1920
|
+
const stars = intFlag(args, "stars");
|
|
1921
|
+
const score = floatFlag(args, "score");
|
|
1922
|
+
if (thumbs === undefined && stars === undefined && score === undefined) {
|
|
1923
|
+
die("give one of --thumbs up|down, --stars 1-5, or --score 0-1");
|
|
1924
|
+
}
|
|
1925
|
+
await captureFeedback(args, "cli", {
|
|
1926
|
+
...(thumbs !== undefined ? { thumbs } : {}),
|
|
1927
|
+
...(stars !== undefined ? { stars } : {}),
|
|
1928
|
+
...(score !== undefined ? { score } : {}),
|
|
1929
|
+
...(strFlag(args, "comment") !== undefined ? { comment: strFlag(args, "comment") } : {}),
|
|
1930
|
+
...(strFlag(args, "rater") !== undefined ? { rater: strFlag(args, "rater") } : {}),
|
|
1931
|
+
});
|
|
1932
|
+
}
|
|
1933
|
+
async function runFeedbackCmd(args) {
|
|
1934
|
+
if (args.flags["help"]) {
|
|
1935
|
+
process.stdout.write("usage: crewhaus feedback --session <id> [--turn N] --text <msg> [--correction <better answer>] [--rater <who>]\n");
|
|
1936
|
+
return;
|
|
1937
|
+
}
|
|
1938
|
+
const text = strFlag(args, "text");
|
|
1939
|
+
const correction = strFlag(args, "correction");
|
|
1940
|
+
if (text === undefined && correction === undefined) {
|
|
1941
|
+
die("give --text <msg> and/or --correction <better answer>");
|
|
1942
|
+
}
|
|
1943
|
+
await captureFeedback(args, "cli", {
|
|
1944
|
+
...(text !== undefined ? { comment: text } : {}),
|
|
1945
|
+
...(correction !== undefined ? { correction } : {}),
|
|
1946
|
+
...(strFlag(args, "rater") !== undefined ? { rater: strFlag(args, "rater") } : {}),
|
|
1947
|
+
});
|
|
1948
|
+
}
|
|
1949
|
+
/** List `sess_*` ids that have a transcript under `.crewhaus/sessions`. */
|
|
1950
|
+
function listSessionIds(sessionsDir) {
|
|
1951
|
+
if (!existsSync(sessionsDir))
|
|
1952
|
+
return [];
|
|
1953
|
+
return readdirSync(sessionsDir)
|
|
1954
|
+
.filter((f) => f.endsWith(".jsonl"))
|
|
1955
|
+
.map((f) => f.slice(0, -".jsonl".length))
|
|
1956
|
+
.filter((id) => SESSION_ID_REGEX.test(id));
|
|
1957
|
+
}
|
|
1958
|
+
/** Read bare FeedbackRecords from `.crewhaus/feedback/*.jsonl` (the web-UI host
|
|
1959
|
+
* sink, which has no event-log handle). */
|
|
1960
|
+
function readFeedbackDir(feedbackDir) {
|
|
1961
|
+
if (!existsSync(feedbackDir))
|
|
1962
|
+
return [];
|
|
1963
|
+
const objects = [];
|
|
1964
|
+
for (const f of readdirSync(feedbackDir)) {
|
|
1965
|
+
if (!f.endsWith(".jsonl"))
|
|
1966
|
+
continue;
|
|
1967
|
+
objects.push(...parseJsonlObjects(readFileSync(join(feedbackDir, f), "utf-8")));
|
|
1968
|
+
}
|
|
1969
|
+
return extractFeedbackRecords(objects);
|
|
1970
|
+
}
|
|
1971
|
+
async function runDistill(args) {
|
|
1972
|
+
if (args.flags["help"]) {
|
|
1973
|
+
process.stdout.write("usage: crewhaus distill (--session <id> | --all-sessions) -o <dataset.jsonl> " +
|
|
1974
|
+
"[--graders-out <graders.yaml>] [--min-score F] [--judge] [--judge-model <model>]\n");
|
|
1975
|
+
return;
|
|
1976
|
+
}
|
|
1977
|
+
const outPath = args.flags["out"];
|
|
1978
|
+
if (typeof outPath !== "string")
|
|
1979
|
+
die("missing -o <dataset.jsonl>");
|
|
1980
|
+
const allSessions = args.flags["all-sessions"] === true;
|
|
1981
|
+
const session = args.flags["session"];
|
|
1982
|
+
if (!allSessions && typeof session !== "string")
|
|
1983
|
+
die("missing --session <id> (or --all-sessions)");
|
|
1984
|
+
if (typeof session === "string" && !SESSION_ID_REGEX.test(session)) {
|
|
1985
|
+
die(`invalid --session "${session}" — expected sess_<16 hex>`);
|
|
1986
|
+
}
|
|
1987
|
+
const minScore = floatFlag(args, "min-score") ?? 0.7;
|
|
1988
|
+
if (minScore < 0 || minScore > 1)
|
|
1989
|
+
die(`invalid --min-score "${minScore}" — must be in [0,1]`);
|
|
1990
|
+
const useJudge = args.flags["judge"] === true;
|
|
1991
|
+
const judgeModel = args.flags["judge-model"];
|
|
1992
|
+
const sessionsDir = join(process.cwd(), SESSIONS_SUBDIR);
|
|
1993
|
+
const sessionIds = allSessions ? listSessionIds(sessionsDir) : [session];
|
|
1994
|
+
if (sessionIds.length === 0)
|
|
1995
|
+
die(`no sessions found under ${sessionsDir}`);
|
|
1996
|
+
// Derive turns per session (tagged with the sessionId join key) and gather
|
|
1997
|
+
// feedback from both the in-transcript events and the web-UI feedback dir.
|
|
1998
|
+
const turns = [];
|
|
1999
|
+
const feedback = [];
|
|
2000
|
+
for (const id of sessionIds) {
|
|
2001
|
+
const events = readSessionEvents(id);
|
|
2002
|
+
for (const t of deriveTurns(events))
|
|
2003
|
+
turns.push({ ...t, sessionId: id });
|
|
2004
|
+
feedback.push(...extractFeedbackRecords(events));
|
|
2005
|
+
}
|
|
2006
|
+
feedback.push(...readFeedbackDir(join(process.cwd(), FEEDBACK_SUBDIR)));
|
|
2007
|
+
if (feedback.length === 0) {
|
|
2008
|
+
die("no feedback found — record some with `crewhaus rate` / `crewhaus feedback` first");
|
|
2009
|
+
}
|
|
2010
|
+
const result = distillFeedback(turns, feedback, {
|
|
2011
|
+
minScore,
|
|
2012
|
+
...(useJudge ? { judge: true } : {}),
|
|
2013
|
+
...(typeof judgeModel === "string" ? { judgeModel } : {}),
|
|
2014
|
+
});
|
|
2015
|
+
for (const w of result.warnings)
|
|
2016
|
+
process.stderr.write(`[distill] warning: ${w}\n`);
|
|
2017
|
+
if (result.samples.length === 0)
|
|
2018
|
+
die("no rated turns could be matched to the transcript(s)");
|
|
2019
|
+
const absOut = resolve(outPath);
|
|
2020
|
+
mkdirSync(dirname(absOut), { recursive: true });
|
|
2021
|
+
writeFileSync(absOut, samplesToJsonl(result.samples), { mode: 0o600 });
|
|
2022
|
+
const gradersOut = args.flags["graders-out"];
|
|
2023
|
+
if (typeof gradersOut === "string") {
|
|
2024
|
+
const absGraders = resolve(gradersOut);
|
|
2025
|
+
mkdirSync(dirname(absGraders), { recursive: true });
|
|
2026
|
+
writeFileSync(absGraders, gradersConfigToYaml(result.graders), { mode: 0o600 });
|
|
2027
|
+
}
|
|
2028
|
+
const { stats } = result;
|
|
2029
|
+
process.stdout.write(`[distill] ${stats.matchedTurns} rated turn(s) → ${result.samples.length} sample(s) ` +
|
|
2030
|
+
`(${stats.positives} positive, ${stats.negatives} low-rated) → ${absOut}\n`);
|
|
2031
|
+
if (typeof gradersOut === "string") {
|
|
2032
|
+
const g = result.graders.graders[0];
|
|
2033
|
+
process.stdout.write(`[distill] grader: ${g?.name} (${g?.type}) → ${resolve(gradersOut)}\n`);
|
|
2034
|
+
}
|
|
2035
|
+
if (stats.unmatchedFeedback > 0) {
|
|
2036
|
+
process.stdout.write(`[distill] ${stats.unmatchedFeedback} rating(s) had no matching turn (skipped)\n`);
|
|
2037
|
+
}
|
|
2038
|
+
}
|
|
2039
|
+
/**
|
|
2040
|
+
* Inline-distill ratings for `crewhaus optimize --ratings`. `ratingsArg` is a
|
|
2041
|
+
* session id or "all". Returns optimizer-shaped samples plus a synthesized
|
|
2042
|
+
* graders.yaml (used only when the user did not pass their own --graders).
|
|
2043
|
+
*/
|
|
2044
|
+
function distillRatings(ratingsArg, minScore) {
|
|
2045
|
+
const sessionsDir = join(process.cwd(), SESSIONS_SUBDIR);
|
|
2046
|
+
let sessionIds;
|
|
2047
|
+
if (ratingsArg === "all") {
|
|
2048
|
+
sessionIds = listSessionIds(sessionsDir);
|
|
2049
|
+
}
|
|
2050
|
+
else {
|
|
2051
|
+
if (!SESSION_ID_REGEX.test(ratingsArg)) {
|
|
2052
|
+
die(`invalid --ratings "${ratingsArg}" — expected a session id (sess_<16 hex>) or "all"`);
|
|
2053
|
+
}
|
|
2054
|
+
sessionIds = [ratingsArg];
|
|
2055
|
+
}
|
|
2056
|
+
if (sessionIds.length === 0)
|
|
2057
|
+
die(`no sessions found under ${sessionsDir}`);
|
|
2058
|
+
const turns = [];
|
|
2059
|
+
const feedback = [];
|
|
2060
|
+
for (const id of sessionIds) {
|
|
2061
|
+
const events = readSessionEvents(id);
|
|
2062
|
+
for (const t of deriveTurns(events))
|
|
2063
|
+
turns.push({ ...t, sessionId: id });
|
|
2064
|
+
feedback.push(...extractFeedbackRecords(events));
|
|
2065
|
+
}
|
|
2066
|
+
feedback.push(...readFeedbackDir(join(process.cwd(), FEEDBACK_SUBDIR)));
|
|
2067
|
+
if (feedback.length === 0) {
|
|
2068
|
+
die("no feedback found for --ratings — record some with `crewhaus rate` / `crewhaus feedback` first");
|
|
2069
|
+
}
|
|
2070
|
+
const result = distillFeedback(turns, feedback, { minScore });
|
|
2071
|
+
for (const w of result.warnings)
|
|
2072
|
+
process.stderr.write(`[optimize] ratings warning: ${w}\n`);
|
|
2073
|
+
const samples = result.samples.map((s) => ({
|
|
2074
|
+
id: s.id,
|
|
2075
|
+
input: s.input,
|
|
2076
|
+
...(s.expected_output !== undefined ? { expected_output: s.expected_output } : {}),
|
|
2077
|
+
}));
|
|
2078
|
+
return { samples, gradersYaml: gradersConfigToYaml(result.graders) };
|
|
2079
|
+
}
|
|
1739
2080
|
/**
|
|
1740
2081
|
* Section 27 — `crewhaus secrets <action> <name> [opts]`. Two actions:
|
|
1741
2082
|
* doctor list configured secrets and report missing
|
|
@@ -2310,6 +2651,15 @@ switch (subcommand) {
|
|
|
2310
2651
|
case "cost-summary":
|
|
2311
2652
|
await runCostSummary(parseFor(rest, COST_SUMMARY_SCHEMA));
|
|
2312
2653
|
break;
|
|
2654
|
+
case "rate":
|
|
2655
|
+
await runRate(parseFor(rest, RATE_SCHEMA));
|
|
2656
|
+
break;
|
|
2657
|
+
case "feedback":
|
|
2658
|
+
await runFeedbackCmd(parseFor(rest, FEEDBACK_SCHEMA));
|
|
2659
|
+
break;
|
|
2660
|
+
case "distill":
|
|
2661
|
+
await runDistill(parseFor(rest, DISTILL_SCHEMA));
|
|
2662
|
+
break;
|
|
2313
2663
|
case "secrets": {
|
|
2314
2664
|
const action = rest[0] ?? "";
|
|
2315
2665
|
if (action !== "doctor" && action !== "rotate") {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "crewhaus",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.8",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "CrewHaus — the meta-harness compiler for AI agents. Compile one crewhaus.yaml spec into a CLI agent, channel bot, RAG pipeline, multi-agent crew, eval harness, voice or browser agent, and more.",
|
|
6
6
|
"keywords": [
|
|
@@ -31,69 +31,70 @@
|
|
|
31
31
|
"test": "bun test src"
|
|
32
32
|
},
|
|
33
33
|
"dependencies": {
|
|
34
|
-
"@crewhaus/adapter-anthropic": "0.1.
|
|
35
|
-
"@crewhaus/agent-context-isolation": "0.1.
|
|
36
|
-
"@crewhaus/audit-log": "0.1.
|
|
37
|
-
"@crewhaus/eval-optimizer-orchestrator": "0.1.
|
|
38
|
-
"@crewhaus/prompt-optimizer": "0.1.
|
|
39
|
-
"@crewhaus/prompt-optimizer-claude": "0.1.
|
|
40
|
-
"@crewhaus/justification-judge-claude": "0.1.
|
|
41
|
-
"@crewhaus/spec-patch": "0.1.
|
|
42
|
-
"@crewhaus/canary-controller": "0.1.
|
|
43
|
-
"@crewhaus/compiler": "0.1.
|
|
44
|
-
"@crewhaus/computer-use-driver": "0.1.
|
|
45
|
-
"@crewhaus/context-bundle": "0.1.
|
|
46
|
-
"@crewhaus/deployment-controller": "0.1.
|
|
47
|
-
"@crewhaus/egress-classifier": "0.1.
|
|
48
|
-
"@crewhaus/egress-matcher-semantic": "0.1.
|
|
49
|
-
"@crewhaus/embedder": "0.1.
|
|
50
|
-
"@crewhaus/errors": "0.1.
|
|
51
|
-
"@crewhaus/eval-dataset": "0.1.
|
|
52
|
-
"@crewhaus/eval-grader": "0.1.
|
|
53
|
-
"@crewhaus/eval-report": "0.1.
|
|
54
|
-
"@crewhaus/eval-runner": "0.1.
|
|
55
|
-
"@crewhaus/
|
|
56
|
-
"@crewhaus/
|
|
57
|
-
"@crewhaus/
|
|
58
|
-
"@crewhaus/
|
|
59
|
-
"@crewhaus/
|
|
60
|
-
"@crewhaus/
|
|
61
|
-
"@crewhaus/
|
|
62
|
-
"@crewhaus/migration-
|
|
63
|
-
"@crewhaus/
|
|
64
|
-
"@crewhaus/
|
|
65
|
-
"@crewhaus/
|
|
66
|
-
"@crewhaus/
|
|
67
|
-
"@crewhaus/
|
|
68
|
-
"@crewhaus/
|
|
69
|
-
"@crewhaus/
|
|
70
|
-
"@crewhaus/
|
|
71
|
-
"@crewhaus/
|
|
72
|
-
"@crewhaus/
|
|
73
|
-
"@crewhaus/
|
|
74
|
-
"@crewhaus/
|
|
75
|
-
"@crewhaus/tool-
|
|
76
|
-
"@crewhaus/tool-
|
|
77
|
-
"@crewhaus/tool-
|
|
78
|
-
"@crewhaus/tool-
|
|
79
|
-
"@crewhaus/tool-
|
|
80
|
-
"@crewhaus/tool-
|
|
81
|
-
"@crewhaus/tool-image
|
|
82
|
-
"@crewhaus/tool-
|
|
83
|
-
"@crewhaus/tool-
|
|
84
|
-
"@crewhaus/tool-
|
|
85
|
-
"@crewhaus/tool-
|
|
86
|
-
"@crewhaus/tool-
|
|
87
|
-
"@crewhaus/tool-
|
|
88
|
-
"@crewhaus/tool-
|
|
89
|
-
"@crewhaus/tool-
|
|
90
|
-
"@crewhaus/tool-
|
|
91
|
-
"@crewhaus/
|
|
92
|
-
"@crewhaus/
|
|
93
|
-
"@crewhaus/
|
|
94
|
-
"@crewhaus/
|
|
95
|
-
"@crewhaus/sandbox
|
|
96
|
-
"@crewhaus/
|
|
34
|
+
"@crewhaus/adapter-anthropic": "0.1.8",
|
|
35
|
+
"@crewhaus/agent-context-isolation": "0.1.8",
|
|
36
|
+
"@crewhaus/audit-log": "0.1.8",
|
|
37
|
+
"@crewhaus/eval-optimizer-orchestrator": "0.1.8",
|
|
38
|
+
"@crewhaus/prompt-optimizer": "0.1.8",
|
|
39
|
+
"@crewhaus/prompt-optimizer-claude": "0.1.8",
|
|
40
|
+
"@crewhaus/justification-judge-claude": "0.1.8",
|
|
41
|
+
"@crewhaus/spec-patch": "0.1.8",
|
|
42
|
+
"@crewhaus/canary-controller": "0.1.8",
|
|
43
|
+
"@crewhaus/compiler": "0.1.8",
|
|
44
|
+
"@crewhaus/computer-use-driver": "0.1.8",
|
|
45
|
+
"@crewhaus/context-bundle": "0.1.8",
|
|
46
|
+
"@crewhaus/deployment-controller": "0.1.8",
|
|
47
|
+
"@crewhaus/egress-classifier": "0.1.8",
|
|
48
|
+
"@crewhaus/egress-matcher-semantic": "0.1.8",
|
|
49
|
+
"@crewhaus/embedder": "0.1.8",
|
|
50
|
+
"@crewhaus/errors": "0.1.8",
|
|
51
|
+
"@crewhaus/eval-dataset": "0.1.8",
|
|
52
|
+
"@crewhaus/eval-grader": "0.1.8",
|
|
53
|
+
"@crewhaus/eval-report": "0.1.8",
|
|
54
|
+
"@crewhaus/eval-runner": "0.1.8",
|
|
55
|
+
"@crewhaus/event-log": "0.1.8",
|
|
56
|
+
"@crewhaus/hooks-engine": "0.1.8",
|
|
57
|
+
"@crewhaus/infra-utils": "0.1.8",
|
|
58
|
+
"@crewhaus/ir": "0.1.8",
|
|
59
|
+
"@crewhaus/logging": "0.1.8",
|
|
60
|
+
"@crewhaus/mcp-host": "0.1.8",
|
|
61
|
+
"@crewhaus/model-router": "0.1.8",
|
|
62
|
+
"@crewhaus/migration-engine": "0.1.8",
|
|
63
|
+
"@crewhaus/migration-runner": "0.1.8",
|
|
64
|
+
"@crewhaus/permission-engine": "0.1.8",
|
|
65
|
+
"@crewhaus/run-context": "0.1.8",
|
|
66
|
+
"@crewhaus/runtime-core": "0.1.8",
|
|
67
|
+
"@crewhaus/secrets-manager": "0.1.8",
|
|
68
|
+
"@crewhaus/session-store": "0.1.8",
|
|
69
|
+
"@crewhaus/spec-registry": "0.1.8",
|
|
70
|
+
"@crewhaus/skills-registry": "0.1.8",
|
|
71
|
+
"@crewhaus/slash-commands": "0.1.8",
|
|
72
|
+
"@crewhaus/spec": "0.1.8",
|
|
73
|
+
"@crewhaus/sub-agent-spawner": "0.1.8",
|
|
74
|
+
"@crewhaus/trace-event-bus": "0.1.8",
|
|
75
|
+
"@crewhaus/tool-bash": "0.1.8",
|
|
76
|
+
"@crewhaus/tool-builder": "0.1.8",
|
|
77
|
+
"@crewhaus/tool-catalog": "0.1.8",
|
|
78
|
+
"@crewhaus/tool-codegraph": "0.1.8",
|
|
79
|
+
"@crewhaus/tool-fetch": "0.1.8",
|
|
80
|
+
"@crewhaus/tool-fs": "0.1.8",
|
|
81
|
+
"@crewhaus/tool-image": "0.1.8",
|
|
82
|
+
"@crewhaus/tool-image-generation": "0.1.8",
|
|
83
|
+
"@crewhaus/tool-document-ingest": "0.1.8",
|
|
84
|
+
"@crewhaus/tool-mcp": "0.1.8",
|
|
85
|
+
"@crewhaus/tool-mouse-keyboard": "0.1.8",
|
|
86
|
+
"@crewhaus/tool-navigate": "0.1.8",
|
|
87
|
+
"@crewhaus/tool-screen-capture": "0.1.8",
|
|
88
|
+
"@crewhaus/tool-task": "0.1.8",
|
|
89
|
+
"@crewhaus/tool-todo": "0.1.8",
|
|
90
|
+
"@crewhaus/tool-vision-grounding": "0.1.8",
|
|
91
|
+
"@crewhaus/tool-web": "0.1.8",
|
|
92
|
+
"@crewhaus/docker-images": "0.1.8",
|
|
93
|
+
"@crewhaus/crewhaus-cloud": "0.1.8",
|
|
94
|
+
"@crewhaus/federation-discovery": "0.1.8",
|
|
95
|
+
"@crewhaus/sandbox": "0.1.8",
|
|
96
|
+
"@crewhaus/sandbox-image-registry": "0.1.8",
|
|
97
|
+
"@crewhaus/compliance-controls": "0.1.8"
|
|
97
98
|
},
|
|
98
99
|
"devDependencies": {
|
|
99
100
|
"zod": "^3.23.8"
|