crewhaus 0.1.7 → 0.1.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,5 +1,6 @@
1
1
  #!/usr/bin/env bun
2
- import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
2
+ import { randomBytes } from "node:crypto";
3
+ import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
3
4
  import { basename, dirname, join, relative, resolve } from "node:path";
4
5
  import { SpecParseError, compile, lower } from "@crewhaus/compiler";
5
6
  import { buildContextBundle, discoverRoots } from "@crewhaus/context-bundle";
@@ -9,6 +10,7 @@ import { parseGradersConfig } from "@crewhaus/eval-grader";
9
10
  import { optimizeSpec } from "@crewhaus/eval-optimizer-orchestrator";
10
11
  import { diffReports, loadRun, renderReport } from "@crewhaus/eval-report";
11
12
  import { runEval as runEvalLib } from "@crewhaus/eval-runner";
13
+ import { openEventLog } from "@crewhaus/event-log";
12
14
  import { loadHooks } from "@crewhaus/hooks-engine";
13
15
  import { ArgParseError, parseArgs, } from "@crewhaus/infra-utils";
14
16
  import { createLogger } from "@crewhaus/logging";
@@ -33,6 +35,10 @@ import { buildCredentialChecks, extractSpecModel } from "./doctor-checks";
33
35
  // imports the optional semantic package only when "semantic" is selected, so
34
36
  // the default path pulls in no embedding dependency.
35
37
  import { DEFAULT_EGRESS_EMBEDDER_MODEL, InvalidEgressMatcherChoiceError, createEgressMatcher, resolveEgressMatcherChoice, } from "./egress-matcher";
38
+ // Response-feedback core — pure, side-effect-free so it is unit-testable
39
+ // (this entry file runs an argv switch on import). Powers `rate`/`feedback`
40
+ // (capture) and `distill` (ratings → eval dataset + graders).
41
+ import { FEEDBACK_EVENT_KIND, buildFeedbackRecord, deriveTurns, distill as distillFeedback, extractFeedbackRecords, gradersConfigToYaml, samplesToJsonl, } from "./feedback";
36
42
  // FR-004 — Pillar 3 intent-gate judge + durable audit-sink resolution, in a
37
43
  // side-effect-free module so it is unit-testable (this entry file runs an
38
44
  // argv switch on import).
@@ -134,6 +140,11 @@ const OPTIMIZE_SCHEMA = {
134
140
  flags: [
135
141
  { name: "dataset", takesValue: true },
136
142
  { name: "graders", takesValue: true },
143
+ // Inline-distill user ratings into the training set (Pillar 2 — close the
144
+ // loop from real feedback). Value: a session id (sess_<16 hex>) or "all".
145
+ // Synthesizes the dataset (and, when --graders is omitted, the graders too).
146
+ { name: "ratings", takesValue: true },
147
+ { name: "min-score", takesValue: true },
137
148
  { name: "mutator", takesValue: true },
138
149
  { name: "iterations", takesValue: true },
139
150
  { name: "seed", takesValue: true },
@@ -171,6 +182,40 @@ const COST_SUMMARY_SCHEMA = {
171
182
  { name: "help", short: "h" },
172
183
  ],
173
184
  };
185
+ const RATE_SCHEMA = {
186
+ flags: [
187
+ { name: "session", takesValue: true },
188
+ { name: "turn", takesValue: true },
189
+ { name: "thumbs", takesValue: true },
190
+ { name: "stars", takesValue: true },
191
+ { name: "score", takesValue: true },
192
+ { name: "comment", takesValue: true },
193
+ { name: "rater", takesValue: true },
194
+ { name: "help", short: "h" },
195
+ ],
196
+ };
197
+ const FEEDBACK_SCHEMA = {
198
+ flags: [
199
+ { name: "session", takesValue: true },
200
+ { name: "turn", takesValue: true },
201
+ { name: "text", takesValue: true },
202
+ { name: "correction", takesValue: true },
203
+ { name: "rater", takesValue: true },
204
+ { name: "help", short: "h" },
205
+ ],
206
+ };
207
+ const DISTILL_SCHEMA = {
208
+ flags: [
209
+ { name: "session", takesValue: true },
210
+ { name: "all-sessions", takesValue: false },
211
+ { name: "out", short: "o", takesValue: true },
212
+ { name: "graders-out", takesValue: true },
213
+ { name: "min-score", takesValue: true },
214
+ { name: "judge", takesValue: false },
215
+ { name: "judge-model", takesValue: true },
216
+ { name: "help", short: "h" },
217
+ ],
218
+ };
174
219
  const SANDBOX_SCHEMA = {
175
220
  flags: [
176
221
  { name: "probe", takesValue: false },
@@ -269,6 +314,7 @@ function usageText() {
269
314
  " [-o <out-dir>]",
270
315
  " optimize <spec.yaml> --dataset <data> --graders <graders.yaml>",
271
316
  " [--mutator rule-based|claude] [--iterations N] [--seed N]",
317
+ " [--ratings <session>|all] distill user ratings into the training set (Pillar 2)",
272
318
  " [--budget-usd N] stop a model-driven run before it exceeds $N (FR-003)",
273
319
  " [--write-back] [-o <out-dir>] active eval-driven optimization (Pillar 2)",
274
320
  " init [name] scaffold a new crewhaus.yaml",
@@ -276,6 +322,12 @@ function usageText() {
276
322
  " context --bundle [-o <file>] emit a single-markdown orientation manifest",
277
323
  " [--factory-root <p>] [--docs-root <p>] [--demos-root <p>]",
278
324
  " cost-summary --session <id> summarize cost_accrual events for a session",
325
+ " rate --session <id> [--turn N] rate an assistant turn 👍/👎, ⭐, or 0–1",
326
+ " (--thumbs up|down | --stars 1-5 | --score 0-1) [--comment <t>]",
327
+ " feedback --session <id> --text <msg> attach a comment/correction to a turn",
328
+ " [--turn N] [--correction <better answer>]",
329
+ " distill --session <id> -o <ds.jsonl> turn ratings into an eval dataset + graders",
330
+ " [--all-sessions] [--graders-out <g.yaml>] [--min-score F]",
279
331
  " secrets doctor list known secrets via the configured backend",
280
332
  " secrets rotate <name> [--value V] rotate a named secret (file backend)",
281
333
  " spec put|list|get|pin|alias ... versioned spec storage (Section 28 spec-registry)",
@@ -1332,8 +1384,8 @@ async function runDoctorPhilosophyAlignment() {
1332
1384
  */
1333
1385
  async function runOptimize(args) {
1334
1386
  if (args.flags["help"]) {
1335
- process.stdout.write("usage: crewhaus optimize <spec.yaml> --dataset <data> --graders <graders.yaml> " +
1336
- "[--mutator rule-based|claude] [--iterations N] [--seed N] [--concurrency N] " +
1387
+ process.stdout.write("usage: crewhaus optimize <spec.yaml> (--dataset <data> --graders <graders.yaml> | --ratings <session>) " +
1388
+ "[--min-score F] [--mutator rule-based|claude] [--iterations N] [--seed N] [--concurrency N] " +
1337
1389
  "[--improvement-threshold F] [--budget-usd N] [--write-back] [-o <out-dir>]\n");
1338
1390
  return;
1339
1391
  }
@@ -1342,10 +1394,18 @@ async function runOptimize(args) {
1342
1394
  die("missing <spec.yaml>");
1343
1395
  const datasetPath = args.flags["dataset"];
1344
1396
  const gradersPath = args.flags["graders"];
1345
- if (typeof datasetPath !== "string")
1346
- die("missing --dataset <data>");
1347
- if (typeof gradersPath !== "string")
1348
- die("missing --graders <graders.yaml>");
1397
+ const ratingsArg = args.flags["ratings"];
1398
+ if (typeof datasetPath !== "string" && typeof ratingsArg !== "string") {
1399
+ die("missing --dataset <data> (or --ratings <session> to distill one from feedback)");
1400
+ }
1401
+ if (typeof gradersPath !== "string" && typeof ratingsArg !== "string") {
1402
+ die("missing --graders <graders.yaml> (or --ratings to synthesize one from feedback)");
1403
+ }
1404
+ const ratingsMinScore = floatFlag(args, "min-score") ?? 0.7;
1405
+ if (ratingsMinScore < 0 || ratingsMinScore > 1) {
1406
+ die(`invalid --min-score "${ratingsMinScore}" — must be in [0,1]`);
1407
+ }
1408
+ const ratingsDistill = typeof ratingsArg === "string" ? distillRatings(ratingsArg, ratingsMinScore) : undefined;
1349
1409
  const iterationsFlag = args.flags["iterations"];
1350
1410
  const seedFlag = args.flags["seed"];
1351
1411
  const thresholdFlag = args.flags["improvement-threshold"];
@@ -1383,25 +1443,40 @@ async function runOptimize(args) {
1383
1443
  ? resolve(outDirArg)
1384
1444
  : resolve(join(".crewhaus", "optimize", runId));
1385
1445
  let gradersYaml;
1386
- try {
1387
- gradersYaml = readFileSync(resolve(gradersPath), "utf-8");
1446
+ if (typeof gradersPath === "string") {
1447
+ try {
1448
+ gradersYaml = readFileSync(resolve(gradersPath), "utf-8");
1449
+ }
1450
+ catch (err) {
1451
+ die(`could not read ${gradersPath}: ${err.message}`);
1452
+ }
1388
1453
  }
1389
- catch (err) {
1390
- die(`could not read ${gradersPath}: ${err.message}`);
1454
+ else {
1455
+ // No --graders: use the grader synthesized from the ratings (guaranteed
1456
+ // present here because the missing-flags gate required --ratings).
1457
+ gradersYaml = ratingsDistill.gradersYaml;
1391
1458
  }
1392
1459
  const { compiled } = parseGradersConfig(gradersYaml);
1393
- const dataset = await loadDataset(resolve(datasetPath));
1394
- // Materialize the dataset once we'll re-iterate per fitness call.
1460
+ // Materialize the training set once — we'll re-iterate per fitness call. It
1461
+ // is the union of the file dataset (if any) and the distilled ratings (if
1462
+ // any); at least one is present per the missing-flags gate above.
1395
1463
  const samples = [];
1396
- for await (const s of dataset.samples) {
1397
- samples.push({
1398
- id: s.id,
1399
- input: s.input,
1400
- ...(s.expected_output !== undefined ? { expected_output: s.expected_output } : {}),
1401
- });
1464
+ let datasetName = "ratings";
1465
+ if (typeof datasetPath === "string") {
1466
+ const dataset = await loadDataset(resolve(datasetPath));
1467
+ datasetName = dataset.name;
1468
+ for await (const s of dataset.samples) {
1469
+ samples.push({
1470
+ id: s.id,
1471
+ input: s.input,
1472
+ ...(s.expected_output !== undefined ? { expected_output: s.expected_output } : {}),
1473
+ });
1474
+ }
1402
1475
  }
1476
+ if (ratingsDistill !== undefined)
1477
+ samples.push(...ratingsDistill.samples);
1403
1478
  if (samples.length === 0)
1404
- die(`dataset "${dataset.name}" yielded zero samples`);
1479
+ die(`dataset "${datasetName}" yielded zero samples`);
1405
1480
  // Train/dev split: 70/30 deterministic split by sample id ordering.
1406
1481
  const splitIdx = Math.max(1, Math.floor(samples.length * 0.7));
1407
1482
  const trainSet = samples.slice(0, splitIdx);
@@ -1449,7 +1524,7 @@ async function runOptimize(args) {
1449
1524
  }
1450
1525
  const summary = await runEvalLib({
1451
1526
  ir,
1452
- dataset: { name: dataset.name, samples: makeAsyncIterable(devSet) },
1527
+ dataset: { name: datasetName, samples: makeAsyncIterable(devSet) },
1453
1528
  compiledGraders: compiled,
1454
1529
  opts: {
1455
1530
  outDir: join(outDir, "evals", `${prompt.length}_${ir.agent.instructions.length}`),
@@ -1489,7 +1564,7 @@ async function runOptimize(args) {
1489
1564
  else if (mutator !== undefined && mutator !== "rule-based") {
1490
1565
  die(`unknown --mutator "${mutator}" — supported: rule-based, claude`);
1491
1566
  }
1492
- process.stdout.write(`[optimize] runId=${runId} spec=${specPath} dataset=${dataset.name} ` +
1567
+ process.stdout.write(`[optimize] runId=${runId} spec=${specPath} dataset=${datasetName} ` +
1493
1568
  `(${trainSet.length} train / ${devSet.length} dev) iterations=${iterations} ` +
1494
1569
  `mutator=${mutator ?? "rule-based"}\n`);
1495
1570
  // FR-003 — thread a real trace bus into the optimize run so the per-call
@@ -1736,6 +1811,272 @@ async function runCostSummary(args) {
1736
1811
  }
1737
1812
  }
1738
1813
  }
1814
+ // -------- response feedback: rate / feedback / distill --------
1815
+ const SESSIONS_SUBDIR = join(".crewhaus", "sessions");
1816
+ const FEEDBACK_SUBDIR = join(".crewhaus", "feedback");
1817
+ function sessionJsonlPath(session) {
1818
+ return join(process.cwd(), SESSIONS_SUBDIR, `${session}.jsonl`);
1819
+ }
1820
+ /** Parse a JSONL blob into objects, skipping blank/malformed lines. */
1821
+ function parseJsonlObjects(text) {
1822
+ const out = [];
1823
+ for (const line of text.split("\n")) {
1824
+ if (line.trim() === "")
1825
+ continue;
1826
+ try {
1827
+ out.push(JSON.parse(line));
1828
+ }
1829
+ catch {
1830
+ // A single malformed line must not abort a read.
1831
+ }
1832
+ }
1833
+ return out;
1834
+ }
1835
+ function readSessionEvents(session) {
1836
+ const file = sessionJsonlPath(session);
1837
+ if (!existsSync(file))
1838
+ die(`session log not found at ${file}`);
1839
+ return parseJsonlObjects(readFileSync(file, "utf-8"));
1840
+ }
1841
+ function strFlag(args, name) {
1842
+ const v = args.flags[name];
1843
+ return typeof v === "string" ? v : undefined;
1844
+ }
1845
+ function intFlag(args, name) {
1846
+ const v = args.flags[name];
1847
+ if (typeof v !== "string")
1848
+ return undefined;
1849
+ const n = Number.parseInt(v, 10);
1850
+ if (Number.isNaN(n))
1851
+ die(`invalid --${name} "${v}" — must be an integer`);
1852
+ return n;
1853
+ }
1854
+ function floatFlag(args, name) {
1855
+ const v = args.flags[name];
1856
+ if (typeof v !== "string")
1857
+ return undefined;
1858
+ const n = Number.parseFloat(v);
1859
+ if (Number.isNaN(n))
1860
+ die(`invalid --${name} "${v}" — must be a number`);
1861
+ return n;
1862
+ }
1863
+ /** Resolve --turn against the transcript-derived turns; default to the last. */
1864
+ function resolveTurn(args, turns) {
1865
+ const flag = args.flags["turn"];
1866
+ const last = turns[turns.length - 1];
1867
+ if (typeof flag !== "string")
1868
+ return last.turnNumber;
1869
+ const n = Number.parseInt(flag, 10);
1870
+ if (Number.isNaN(n))
1871
+ die(`invalid --turn "${flag}" — must be an integer`);
1872
+ if (!turns.some((t) => t.turnNumber === n)) {
1873
+ die(`turn ${n} not found — session has turns 1..${turns.length}`);
1874
+ }
1875
+ return n;
1876
+ }
1877
+ /** Shared capture path for `rate` and `feedback`: validate the session, derive
1878
+ * turns, build a FeedbackRecord, and append it as a `user_feedback` event. */
1879
+ async function captureFeedback(args, source, fields) {
1880
+ const session = args.flags["session"];
1881
+ if (typeof session !== "string")
1882
+ die("missing --session <id>");
1883
+ if (!SESSION_ID_REGEX.test(session))
1884
+ die(`invalid --session "${session}" — expected sess_<16 hex>`);
1885
+ const turns = deriveTurns(readSessionEvents(session));
1886
+ if (turns.length === 0)
1887
+ die(`session ${session} has no user turns to rate`);
1888
+ const turnNumber = resolveTurn(args, turns);
1889
+ let record;
1890
+ try {
1891
+ record = buildFeedbackRecord({
1892
+ id: `fb_${randomBytes(6).toString("hex")}`,
1893
+ sessionId: session,
1894
+ turnNumber,
1895
+ ts: new Date().toISOString(),
1896
+ source,
1897
+ ...fields,
1898
+ });
1899
+ }
1900
+ catch (err) {
1901
+ die(err instanceof Error ? err.message : String(err));
1902
+ }
1903
+ const log = await openEventLog(session, { rootDir: join(process.cwd(), SESSIONS_SUBDIR) });
1904
+ await log.append({ kind: FEEDBACK_EVENT_KIND, payload: record });
1905
+ process.stdout.write(`recorded ${record.modality} feedback on ${session} turn ${turnNumber}\n`);
1906
+ }
1907
+ async function runRate(args) {
1908
+ if (args.flags["help"]) {
1909
+ process.stdout.write("usage: crewhaus rate --session <id> [--turn N] " +
1910
+ "(--thumbs up|down | --stars 1-5 | --score 0-1) [--comment <text>] [--rater <who>]\n");
1911
+ return;
1912
+ }
1913
+ const thumbsFlag = args.flags["thumbs"];
1914
+ let thumbs;
1915
+ if (typeof thumbsFlag === "string") {
1916
+ if (thumbsFlag !== "up" && thumbsFlag !== "down")
1917
+ die(`--thumbs must be "up" or "down" (got "${thumbsFlag}")`);
1918
+ thumbs = thumbsFlag;
1919
+ }
1920
+ const stars = intFlag(args, "stars");
1921
+ const score = floatFlag(args, "score");
1922
+ if (thumbs === undefined && stars === undefined && score === undefined) {
1923
+ die("give one of --thumbs up|down, --stars 1-5, or --score 0-1");
1924
+ }
1925
+ await captureFeedback(args, "cli", {
1926
+ ...(thumbs !== undefined ? { thumbs } : {}),
1927
+ ...(stars !== undefined ? { stars } : {}),
1928
+ ...(score !== undefined ? { score } : {}),
1929
+ ...(strFlag(args, "comment") !== undefined ? { comment: strFlag(args, "comment") } : {}),
1930
+ ...(strFlag(args, "rater") !== undefined ? { rater: strFlag(args, "rater") } : {}),
1931
+ });
1932
+ }
1933
+ async function runFeedbackCmd(args) {
1934
+ if (args.flags["help"]) {
1935
+ process.stdout.write("usage: crewhaus feedback --session <id> [--turn N] --text <msg> [--correction <better answer>] [--rater <who>]\n");
1936
+ return;
1937
+ }
1938
+ const text = strFlag(args, "text");
1939
+ const correction = strFlag(args, "correction");
1940
+ if (text === undefined && correction === undefined) {
1941
+ die("give --text <msg> and/or --correction <better answer>");
1942
+ }
1943
+ await captureFeedback(args, "cli", {
1944
+ ...(text !== undefined ? { comment: text } : {}),
1945
+ ...(correction !== undefined ? { correction } : {}),
1946
+ ...(strFlag(args, "rater") !== undefined ? { rater: strFlag(args, "rater") } : {}),
1947
+ });
1948
+ }
1949
+ /** List `sess_*` ids that have a transcript under `.crewhaus/sessions`. */
1950
+ function listSessionIds(sessionsDir) {
1951
+ if (!existsSync(sessionsDir))
1952
+ return [];
1953
+ return readdirSync(sessionsDir)
1954
+ .filter((f) => f.endsWith(".jsonl"))
1955
+ .map((f) => f.slice(0, -".jsonl".length))
1956
+ .filter((id) => SESSION_ID_REGEX.test(id));
1957
+ }
1958
+ /** Read bare FeedbackRecords from `.crewhaus/feedback/*.jsonl` (the web-UI host
1959
+ * sink, which has no event-log handle). */
1960
+ function readFeedbackDir(feedbackDir) {
1961
+ if (!existsSync(feedbackDir))
1962
+ return [];
1963
+ const objects = [];
1964
+ for (const f of readdirSync(feedbackDir)) {
1965
+ if (!f.endsWith(".jsonl"))
1966
+ continue;
1967
+ objects.push(...parseJsonlObjects(readFileSync(join(feedbackDir, f), "utf-8")));
1968
+ }
1969
+ return extractFeedbackRecords(objects);
1970
+ }
1971
+ async function runDistill(args) {
1972
+ if (args.flags["help"]) {
1973
+ process.stdout.write("usage: crewhaus distill (--session <id> | --all-sessions) -o <dataset.jsonl> " +
1974
+ "[--graders-out <graders.yaml>] [--min-score F] [--judge] [--judge-model <model>]\n");
1975
+ return;
1976
+ }
1977
+ const outPath = args.flags["out"];
1978
+ if (typeof outPath !== "string")
1979
+ die("missing -o <dataset.jsonl>");
1980
+ const allSessions = args.flags["all-sessions"] === true;
1981
+ const session = args.flags["session"];
1982
+ if (!allSessions && typeof session !== "string")
1983
+ die("missing --session <id> (or --all-sessions)");
1984
+ if (typeof session === "string" && !SESSION_ID_REGEX.test(session)) {
1985
+ die(`invalid --session "${session}" — expected sess_<16 hex>`);
1986
+ }
1987
+ const minScore = floatFlag(args, "min-score") ?? 0.7;
1988
+ if (minScore < 0 || minScore > 1)
1989
+ die(`invalid --min-score "${minScore}" — must be in [0,1]`);
1990
+ const useJudge = args.flags["judge"] === true;
1991
+ const judgeModel = args.flags["judge-model"];
1992
+ const sessionsDir = join(process.cwd(), SESSIONS_SUBDIR);
1993
+ const sessionIds = allSessions ? listSessionIds(sessionsDir) : [session];
1994
+ if (sessionIds.length === 0)
1995
+ die(`no sessions found under ${sessionsDir}`);
1996
+ // Derive turns per session (tagged with the sessionId join key) and gather
1997
+ // feedback from both the in-transcript events and the web-UI feedback dir.
1998
+ const turns = [];
1999
+ const feedback = [];
2000
+ for (const id of sessionIds) {
2001
+ const events = readSessionEvents(id);
2002
+ for (const t of deriveTurns(events))
2003
+ turns.push({ ...t, sessionId: id });
2004
+ feedback.push(...extractFeedbackRecords(events));
2005
+ }
2006
+ feedback.push(...readFeedbackDir(join(process.cwd(), FEEDBACK_SUBDIR)));
2007
+ if (feedback.length === 0) {
2008
+ die("no feedback found — record some with `crewhaus rate` / `crewhaus feedback` first");
2009
+ }
2010
+ const result = distillFeedback(turns, feedback, {
2011
+ minScore,
2012
+ ...(useJudge ? { judge: true } : {}),
2013
+ ...(typeof judgeModel === "string" ? { judgeModel } : {}),
2014
+ });
2015
+ for (const w of result.warnings)
2016
+ process.stderr.write(`[distill] warning: ${w}\n`);
2017
+ if (result.samples.length === 0)
2018
+ die("no rated turns could be matched to the transcript(s)");
2019
+ const absOut = resolve(outPath);
2020
+ mkdirSync(dirname(absOut), { recursive: true });
2021
+ writeFileSync(absOut, samplesToJsonl(result.samples), { mode: 0o600 });
2022
+ const gradersOut = args.flags["graders-out"];
2023
+ if (typeof gradersOut === "string") {
2024
+ const absGraders = resolve(gradersOut);
2025
+ mkdirSync(dirname(absGraders), { recursive: true });
2026
+ writeFileSync(absGraders, gradersConfigToYaml(result.graders), { mode: 0o600 });
2027
+ }
2028
+ const { stats } = result;
2029
+ process.stdout.write(`[distill] ${stats.matchedTurns} rated turn(s) → ${result.samples.length} sample(s) ` +
2030
+ `(${stats.positives} positive, ${stats.negatives} low-rated) → ${absOut}\n`);
2031
+ if (typeof gradersOut === "string") {
2032
+ const g = result.graders.graders[0];
2033
+ process.stdout.write(`[distill] grader: ${g?.name} (${g?.type}) → ${resolve(gradersOut)}\n`);
2034
+ }
2035
+ if (stats.unmatchedFeedback > 0) {
2036
+ process.stdout.write(`[distill] ${stats.unmatchedFeedback} rating(s) had no matching turn (skipped)\n`);
2037
+ }
2038
+ }
2039
+ /**
2040
+ * Inline-distill ratings for `crewhaus optimize --ratings`. `ratingsArg` is a
2041
+ * session id or "all". Returns optimizer-shaped samples plus a synthesized
2042
+ * graders.yaml (used only when the user did not pass their own --graders).
2043
+ */
2044
+ function distillRatings(ratingsArg, minScore) {
2045
+ const sessionsDir = join(process.cwd(), SESSIONS_SUBDIR);
2046
+ let sessionIds;
2047
+ if (ratingsArg === "all") {
2048
+ sessionIds = listSessionIds(sessionsDir);
2049
+ }
2050
+ else {
2051
+ if (!SESSION_ID_REGEX.test(ratingsArg)) {
2052
+ die(`invalid --ratings "${ratingsArg}" — expected a session id (sess_<16 hex>) or "all"`);
2053
+ }
2054
+ sessionIds = [ratingsArg];
2055
+ }
2056
+ if (sessionIds.length === 0)
2057
+ die(`no sessions found under ${sessionsDir}`);
2058
+ const turns = [];
2059
+ const feedback = [];
2060
+ for (const id of sessionIds) {
2061
+ const events = readSessionEvents(id);
2062
+ for (const t of deriveTurns(events))
2063
+ turns.push({ ...t, sessionId: id });
2064
+ feedback.push(...extractFeedbackRecords(events));
2065
+ }
2066
+ feedback.push(...readFeedbackDir(join(process.cwd(), FEEDBACK_SUBDIR)));
2067
+ if (feedback.length === 0) {
2068
+ die("no feedback found for --ratings — record some with `crewhaus rate` / `crewhaus feedback` first");
2069
+ }
2070
+ const result = distillFeedback(turns, feedback, { minScore });
2071
+ for (const w of result.warnings)
2072
+ process.stderr.write(`[optimize] ratings warning: ${w}\n`);
2073
+ const samples = result.samples.map((s) => ({
2074
+ id: s.id,
2075
+ input: s.input,
2076
+ ...(s.expected_output !== undefined ? { expected_output: s.expected_output } : {}),
2077
+ }));
2078
+ return { samples, gradersYaml: gradersConfigToYaml(result.graders) };
2079
+ }
1739
2080
  /**
1740
2081
  * Section 27 — `crewhaus secrets <action> <name> [opts]`. Two actions:
1741
2082
  * doctor list configured secrets and report missing
@@ -2310,6 +2651,15 @@ switch (subcommand) {
2310
2651
  case "cost-summary":
2311
2652
  await runCostSummary(parseFor(rest, COST_SUMMARY_SCHEMA));
2312
2653
  break;
2654
+ case "rate":
2655
+ await runRate(parseFor(rest, RATE_SCHEMA));
2656
+ break;
2657
+ case "feedback":
2658
+ await runFeedbackCmd(parseFor(rest, FEEDBACK_SCHEMA));
2659
+ break;
2660
+ case "distill":
2661
+ await runDistill(parseFor(rest, DISTILL_SCHEMA));
2662
+ break;
2313
2663
  case "secrets": {
2314
2664
  const action = rest[0] ?? "";
2315
2665
  if (action !== "doctor" && action !== "rotate") {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "crewhaus",
3
- "version": "0.1.7",
3
+ "version": "0.1.8",
4
4
  "type": "module",
5
5
  "description": "CrewHaus — the meta-harness compiler for AI agents. Compile one crewhaus.yaml spec into a CLI agent, channel bot, RAG pipeline, multi-agent crew, eval harness, voice or browser agent, and more.",
6
6
  "keywords": [
@@ -31,69 +31,70 @@
31
31
  "test": "bun test src"
32
32
  },
33
33
  "dependencies": {
34
- "@crewhaus/adapter-anthropic": "0.1.7",
35
- "@crewhaus/agent-context-isolation": "0.1.7",
36
- "@crewhaus/audit-log": "0.1.7",
37
- "@crewhaus/eval-optimizer-orchestrator": "0.1.7",
38
- "@crewhaus/prompt-optimizer": "0.1.7",
39
- "@crewhaus/prompt-optimizer-claude": "0.1.7",
40
- "@crewhaus/justification-judge-claude": "0.1.7",
41
- "@crewhaus/spec-patch": "0.1.7",
42
- "@crewhaus/canary-controller": "0.1.7",
43
- "@crewhaus/compiler": "0.1.7",
44
- "@crewhaus/computer-use-driver": "0.1.7",
45
- "@crewhaus/context-bundle": "0.1.7",
46
- "@crewhaus/deployment-controller": "0.1.7",
47
- "@crewhaus/egress-classifier": "0.1.7",
48
- "@crewhaus/egress-matcher-semantic": "0.1.7",
49
- "@crewhaus/embedder": "0.1.7",
50
- "@crewhaus/errors": "0.1.7",
51
- "@crewhaus/eval-dataset": "0.1.7",
52
- "@crewhaus/eval-grader": "0.1.7",
53
- "@crewhaus/eval-report": "0.1.7",
54
- "@crewhaus/eval-runner": "0.1.7",
55
- "@crewhaus/hooks-engine": "0.1.7",
56
- "@crewhaus/infra-utils": "0.1.7",
57
- "@crewhaus/ir": "0.1.7",
58
- "@crewhaus/logging": "0.1.7",
59
- "@crewhaus/mcp-host": "0.1.7",
60
- "@crewhaus/model-router": "0.1.7",
61
- "@crewhaus/migration-engine": "0.1.7",
62
- "@crewhaus/migration-runner": "0.1.7",
63
- "@crewhaus/permission-engine": "0.1.7",
64
- "@crewhaus/run-context": "0.1.7",
65
- "@crewhaus/runtime-core": "0.1.7",
66
- "@crewhaus/secrets-manager": "0.1.7",
67
- "@crewhaus/session-store": "0.1.7",
68
- "@crewhaus/spec-registry": "0.1.7",
69
- "@crewhaus/skills-registry": "0.1.7",
70
- "@crewhaus/slash-commands": "0.1.7",
71
- "@crewhaus/spec": "0.1.7",
72
- "@crewhaus/sub-agent-spawner": "0.1.7",
73
- "@crewhaus/trace-event-bus": "0.1.7",
74
- "@crewhaus/tool-bash": "0.1.7",
75
- "@crewhaus/tool-builder": "0.1.7",
76
- "@crewhaus/tool-catalog": "0.1.7",
77
- "@crewhaus/tool-codegraph": "0.1.7",
78
- "@crewhaus/tool-fetch": "0.1.7",
79
- "@crewhaus/tool-fs": "0.1.7",
80
- "@crewhaus/tool-image": "0.1.7",
81
- "@crewhaus/tool-image-generation": "0.1.7",
82
- "@crewhaus/tool-document-ingest": "0.1.7",
83
- "@crewhaus/tool-mcp": "0.1.7",
84
- "@crewhaus/tool-mouse-keyboard": "0.1.7",
85
- "@crewhaus/tool-navigate": "0.1.7",
86
- "@crewhaus/tool-screen-capture": "0.1.7",
87
- "@crewhaus/tool-task": "0.1.7",
88
- "@crewhaus/tool-todo": "0.1.7",
89
- "@crewhaus/tool-vision-grounding": "0.1.7",
90
- "@crewhaus/tool-web": "0.1.7",
91
- "@crewhaus/docker-images": "0.1.7",
92
- "@crewhaus/crewhaus-cloud": "0.1.7",
93
- "@crewhaus/federation-discovery": "0.1.7",
94
- "@crewhaus/sandbox": "0.1.7",
95
- "@crewhaus/sandbox-image-registry": "0.1.7",
96
- "@crewhaus/compliance-controls": "0.1.7"
34
+ "@crewhaus/adapter-anthropic": "0.1.8",
35
+ "@crewhaus/agent-context-isolation": "0.1.8",
36
+ "@crewhaus/audit-log": "0.1.8",
37
+ "@crewhaus/eval-optimizer-orchestrator": "0.1.8",
38
+ "@crewhaus/prompt-optimizer": "0.1.8",
39
+ "@crewhaus/prompt-optimizer-claude": "0.1.8",
40
+ "@crewhaus/justification-judge-claude": "0.1.8",
41
+ "@crewhaus/spec-patch": "0.1.8",
42
+ "@crewhaus/canary-controller": "0.1.8",
43
+ "@crewhaus/compiler": "0.1.8",
44
+ "@crewhaus/computer-use-driver": "0.1.8",
45
+ "@crewhaus/context-bundle": "0.1.8",
46
+ "@crewhaus/deployment-controller": "0.1.8",
47
+ "@crewhaus/egress-classifier": "0.1.8",
48
+ "@crewhaus/egress-matcher-semantic": "0.1.8",
49
+ "@crewhaus/embedder": "0.1.8",
50
+ "@crewhaus/errors": "0.1.8",
51
+ "@crewhaus/eval-dataset": "0.1.8",
52
+ "@crewhaus/eval-grader": "0.1.8",
53
+ "@crewhaus/eval-report": "0.1.8",
54
+ "@crewhaus/eval-runner": "0.1.8",
55
+ "@crewhaus/event-log": "0.1.8",
56
+ "@crewhaus/hooks-engine": "0.1.8",
57
+ "@crewhaus/infra-utils": "0.1.8",
58
+ "@crewhaus/ir": "0.1.8",
59
+ "@crewhaus/logging": "0.1.8",
60
+ "@crewhaus/mcp-host": "0.1.8",
61
+ "@crewhaus/model-router": "0.1.8",
62
+ "@crewhaus/migration-engine": "0.1.8",
63
+ "@crewhaus/migration-runner": "0.1.8",
64
+ "@crewhaus/permission-engine": "0.1.8",
65
+ "@crewhaus/run-context": "0.1.8",
66
+ "@crewhaus/runtime-core": "0.1.8",
67
+ "@crewhaus/secrets-manager": "0.1.8",
68
+ "@crewhaus/session-store": "0.1.8",
69
+ "@crewhaus/spec-registry": "0.1.8",
70
+ "@crewhaus/skills-registry": "0.1.8",
71
+ "@crewhaus/slash-commands": "0.1.8",
72
+ "@crewhaus/spec": "0.1.8",
73
+ "@crewhaus/sub-agent-spawner": "0.1.8",
74
+ "@crewhaus/trace-event-bus": "0.1.8",
75
+ "@crewhaus/tool-bash": "0.1.8",
76
+ "@crewhaus/tool-builder": "0.1.8",
77
+ "@crewhaus/tool-catalog": "0.1.8",
78
+ "@crewhaus/tool-codegraph": "0.1.8",
79
+ "@crewhaus/tool-fetch": "0.1.8",
80
+ "@crewhaus/tool-fs": "0.1.8",
81
+ "@crewhaus/tool-image": "0.1.8",
82
+ "@crewhaus/tool-image-generation": "0.1.8",
83
+ "@crewhaus/tool-document-ingest": "0.1.8",
84
+ "@crewhaus/tool-mcp": "0.1.8",
85
+ "@crewhaus/tool-mouse-keyboard": "0.1.8",
86
+ "@crewhaus/tool-navigate": "0.1.8",
87
+ "@crewhaus/tool-screen-capture": "0.1.8",
88
+ "@crewhaus/tool-task": "0.1.8",
89
+ "@crewhaus/tool-todo": "0.1.8",
90
+ "@crewhaus/tool-vision-grounding": "0.1.8",
91
+ "@crewhaus/tool-web": "0.1.8",
92
+ "@crewhaus/docker-images": "0.1.8",
93
+ "@crewhaus/crewhaus-cloud": "0.1.8",
94
+ "@crewhaus/federation-discovery": "0.1.8",
95
+ "@crewhaus/sandbox": "0.1.8",
96
+ "@crewhaus/sandbox-image-registry": "0.1.8",
97
+ "@crewhaus/compliance-controls": "0.1.8"
97
98
  },
98
99
  "devDependencies": {
99
100
  "zod": "^3.23.8"