crewhaus 0.1.6 → 0.1.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (2) hide show
  1. package/dist/index.js +32 -5
  2. package/package.json +64 -64
package/dist/index.js CHANGED
@@ -137,6 +137,7 @@ const OPTIMIZE_SCHEMA = {
137
137
  { name: "mutator", takesValue: true },
138
138
  { name: "iterations", takesValue: true },
139
139
  { name: "seed", takesValue: true },
140
+ { name: "concurrency", takesValue: true },
140
141
  { name: "improvement-threshold", takesValue: true },
141
142
  // FR-003 — dollar ceiling for model-driven runs (composes with --iterations).
142
143
  { name: "budget-usd", takesValue: true },
@@ -1332,7 +1333,7 @@ async function runDoctorPhilosophyAlignment() {
1332
1333
  async function runOptimize(args) {
1333
1334
  if (args.flags["help"]) {
1334
1335
  process.stdout.write("usage: crewhaus optimize <spec.yaml> --dataset <data> --graders <graders.yaml> " +
1335
- "[--mutator rule-based|claude] [--iterations N] [--seed N] " +
1336
+ "[--mutator rule-based|claude] [--iterations N] [--seed N] [--concurrency N] " +
1336
1337
  "[--improvement-threshold F] [--budget-usd N] [--write-back] [-o <out-dir>]\n");
1337
1338
  return;
1338
1339
  }
@@ -1354,6 +1355,15 @@ async function runOptimize(args) {
1354
1355
  if (Number.isNaN(iterations) || iterations < 1) {
1355
1356
  die(`invalid --iterations "${iterationsFlag}" — must be positive integer`);
1356
1357
  }
1358
+ // Per-candidate eval concurrency. Each iteration runs a full eval pass on
1359
+ // the dev set; on a low provider rate-limit tier a high fan-out trips 429s,
1360
+ // so this is exposed (mirroring `crewhaus eval --concurrency`) and defaults
1361
+ // to 4. The nightly flywheel sets `--concurrency 1` on constrained tiers.
1362
+ const concurrencyFlag = args.flags["concurrency"];
1363
+ const concurrency = typeof concurrencyFlag === "string" ? Number.parseInt(concurrencyFlag, 10) : 4;
1364
+ if (Number.isNaN(concurrency) || concurrency < 1) {
1365
+ die(`invalid --concurrency "${concurrencyFlag}" — must be a positive integer`);
1366
+ }
1357
1367
  // FR-003 — optional dollar budget for model-driven runs. Omit → today's
1358
1368
  // behaviour (iterations cap only). On a rule-based run the gate is inert
1359
1369
  // (no model calls → $0), so passing it is harmless.
@@ -1399,8 +1409,16 @@ async function runOptimize(args) {
1399
1409
  if (devSet.length === 0) {
1400
1410
  die(`dataset has ${samples.length} samples — need at least 2 (70/30 split needs a dev split)`);
1401
1411
  }
1412
+ // Index the dev set by id so the fitness fn can join each graded
1413
+ // sample-result back to the input + reference it was scored against.
1414
+ const devById = new Map(devSet.map((s) => [s.id, s]));
1402
1415
  // Fitness fn: patch the spec with the candidate prompt, lower to IR,
1403
- // run eval-runner, return passRate. Each call is one full eval pass.
1416
+ // run eval-runner, and return the pass-rate PLUS per-sample grades. The
1417
+ // aggregate `passRate` still drives the search (unchanged scoring); the
1418
+ // grades are additive — they carry each sample's overall score and the
1419
+ // grader's rationale to the mutator (via OptimizerState.bestGrades) so
1420
+ // a model-driven rewrite can target the samples the prompt actually
1421
+ // fails and the reason it fails them. Each call is one full eval pass.
1404
1422
  const fitness = async (prompt) => {
1405
1423
  const yamlText = readFileSync(absSpec, "utf-8");
1406
1424
  // Re-parse to capture spec.target without depending on the
@@ -1422,7 +1440,7 @@ async function runOptimize(args) {
1422
1440
  catch (err) {
1423
1441
  if (err instanceof SpecParseError) {
1424
1442
  process.stderr.write("[optimize] candidate compiled invalid spec, skipping\n");
1425
- return 0;
1443
+ return { score: 0 };
1426
1444
  }
1427
1445
  throw err;
1428
1446
  }
@@ -1435,11 +1453,20 @@ async function runOptimize(args) {
1435
1453
  compiledGraders: compiled,
1436
1454
  opts: {
1437
1455
  outDir: join(outDir, "evals", `${prompt.length}_${ir.agent.instructions.length}`),
1438
- concurrency: 4,
1456
+ concurrency,
1439
1457
  seed,
1440
1458
  },
1441
1459
  });
1442
- return summary.aggregates.passRate;
1460
+ const grades = summary.samples.map((r) => {
1461
+ const dev = devById.get(r.sampleId);
1462
+ return {
1463
+ input: dev?.input ?? r.sampleId,
1464
+ score: r.grades.overall.score,
1465
+ ...(dev?.expected_output !== undefined ? { expected: dev.expected_output } : {}),
1466
+ rationale: r.grades.overall.rationale,
1467
+ };
1468
+ });
1469
+ return { score: summary.aggregates.passRate, grades };
1443
1470
  };
1444
1471
  const mutator = args.flags["mutator"];
1445
1472
  let mutatorImpl;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "crewhaus",
3
- "version": "0.1.6",
3
+ "version": "0.1.7",
4
4
  "type": "module",
5
5
  "description": "CrewHaus — the meta-harness compiler for AI agents. Compile one crewhaus.yaml spec into a CLI agent, channel bot, RAG pipeline, multi-agent crew, eval harness, voice or browser agent, and more.",
6
6
  "keywords": [
@@ -31,69 +31,69 @@
31
31
  "test": "bun test src"
32
32
  },
33
33
  "dependencies": {
34
- "@crewhaus/adapter-anthropic": "0.1.6",
35
- "@crewhaus/agent-context-isolation": "0.1.6",
36
- "@crewhaus/audit-log": "0.1.6",
37
- "@crewhaus/eval-optimizer-orchestrator": "0.1.6",
38
- "@crewhaus/prompt-optimizer": "0.1.6",
39
- "@crewhaus/prompt-optimizer-claude": "0.1.6",
40
- "@crewhaus/justification-judge-claude": "0.1.6",
41
- "@crewhaus/spec-patch": "0.1.6",
42
- "@crewhaus/canary-controller": "0.1.6",
43
- "@crewhaus/compiler": "0.1.6",
44
- "@crewhaus/computer-use-driver": "0.1.6",
45
- "@crewhaus/context-bundle": "0.1.6",
46
- "@crewhaus/deployment-controller": "0.1.6",
47
- "@crewhaus/egress-classifier": "0.1.6",
48
- "@crewhaus/egress-matcher-semantic": "0.1.6",
49
- "@crewhaus/embedder": "0.1.6",
50
- "@crewhaus/errors": "0.1.6",
51
- "@crewhaus/eval-dataset": "0.1.6",
52
- "@crewhaus/eval-grader": "0.1.6",
53
- "@crewhaus/eval-report": "0.1.6",
54
- "@crewhaus/eval-runner": "0.1.6",
55
- "@crewhaus/hooks-engine": "0.1.6",
56
- "@crewhaus/infra-utils": "0.1.6",
57
- "@crewhaus/ir": "0.1.6",
58
- "@crewhaus/logging": "0.1.6",
59
- "@crewhaus/mcp-host": "0.1.6",
60
- "@crewhaus/model-router": "0.1.6",
61
- "@crewhaus/migration-engine": "0.1.6",
62
- "@crewhaus/migration-runner": "0.1.6",
63
- "@crewhaus/permission-engine": "0.1.6",
64
- "@crewhaus/run-context": "0.1.6",
65
- "@crewhaus/runtime-core": "0.1.6",
66
- "@crewhaus/secrets-manager": "0.1.6",
67
- "@crewhaus/session-store": "0.1.6",
68
- "@crewhaus/spec-registry": "0.1.6",
69
- "@crewhaus/skills-registry": "0.1.6",
70
- "@crewhaus/slash-commands": "0.1.6",
71
- "@crewhaus/spec": "0.1.6",
72
- "@crewhaus/sub-agent-spawner": "0.1.6",
73
- "@crewhaus/trace-event-bus": "0.1.6",
74
- "@crewhaus/tool-bash": "0.1.6",
75
- "@crewhaus/tool-builder": "0.1.6",
76
- "@crewhaus/tool-catalog": "0.1.6",
77
- "@crewhaus/tool-codegraph": "0.1.6",
78
- "@crewhaus/tool-fetch": "0.1.6",
79
- "@crewhaus/tool-fs": "0.1.6",
80
- "@crewhaus/tool-image": "0.1.6",
81
- "@crewhaus/tool-image-generation": "0.1.6",
82
- "@crewhaus/tool-document-ingest": "0.1.6",
83
- "@crewhaus/tool-mcp": "0.1.6",
84
- "@crewhaus/tool-mouse-keyboard": "0.1.6",
85
- "@crewhaus/tool-navigate": "0.1.6",
86
- "@crewhaus/tool-screen-capture": "0.1.6",
87
- "@crewhaus/tool-task": "0.1.6",
88
- "@crewhaus/tool-todo": "0.1.6",
89
- "@crewhaus/tool-vision-grounding": "0.1.6",
90
- "@crewhaus/tool-web": "0.1.6",
91
- "@crewhaus/docker-images": "0.1.6",
92
- "@crewhaus/crewhaus-cloud": "0.1.6",
93
- "@crewhaus/federation-discovery": "0.1.6",
94
- "@crewhaus/sandbox": "0.1.6",
95
- "@crewhaus/sandbox-image-registry": "0.1.6",
96
- "@crewhaus/compliance-controls": "0.1.6"
34
+ "@crewhaus/adapter-anthropic": "0.1.7",
35
+ "@crewhaus/agent-context-isolation": "0.1.7",
36
+ "@crewhaus/audit-log": "0.1.7",
37
+ "@crewhaus/eval-optimizer-orchestrator": "0.1.7",
38
+ "@crewhaus/prompt-optimizer": "0.1.7",
39
+ "@crewhaus/prompt-optimizer-claude": "0.1.7",
40
+ "@crewhaus/justification-judge-claude": "0.1.7",
41
+ "@crewhaus/spec-patch": "0.1.7",
42
+ "@crewhaus/canary-controller": "0.1.7",
43
+ "@crewhaus/compiler": "0.1.7",
44
+ "@crewhaus/computer-use-driver": "0.1.7",
45
+ "@crewhaus/context-bundle": "0.1.7",
46
+ "@crewhaus/deployment-controller": "0.1.7",
47
+ "@crewhaus/egress-classifier": "0.1.7",
48
+ "@crewhaus/egress-matcher-semantic": "0.1.7",
49
+ "@crewhaus/embedder": "0.1.7",
50
+ "@crewhaus/errors": "0.1.7",
51
+ "@crewhaus/eval-dataset": "0.1.7",
52
+ "@crewhaus/eval-grader": "0.1.7",
53
+ "@crewhaus/eval-report": "0.1.7",
54
+ "@crewhaus/eval-runner": "0.1.7",
55
+ "@crewhaus/hooks-engine": "0.1.7",
56
+ "@crewhaus/infra-utils": "0.1.7",
57
+ "@crewhaus/ir": "0.1.7",
58
+ "@crewhaus/logging": "0.1.7",
59
+ "@crewhaus/mcp-host": "0.1.7",
60
+ "@crewhaus/model-router": "0.1.7",
61
+ "@crewhaus/migration-engine": "0.1.7",
62
+ "@crewhaus/migration-runner": "0.1.7",
63
+ "@crewhaus/permission-engine": "0.1.7",
64
+ "@crewhaus/run-context": "0.1.7",
65
+ "@crewhaus/runtime-core": "0.1.7",
66
+ "@crewhaus/secrets-manager": "0.1.7",
67
+ "@crewhaus/session-store": "0.1.7",
68
+ "@crewhaus/spec-registry": "0.1.7",
69
+ "@crewhaus/skills-registry": "0.1.7",
70
+ "@crewhaus/slash-commands": "0.1.7",
71
+ "@crewhaus/spec": "0.1.7",
72
+ "@crewhaus/sub-agent-spawner": "0.1.7",
73
+ "@crewhaus/trace-event-bus": "0.1.7",
74
+ "@crewhaus/tool-bash": "0.1.7",
75
+ "@crewhaus/tool-builder": "0.1.7",
76
+ "@crewhaus/tool-catalog": "0.1.7",
77
+ "@crewhaus/tool-codegraph": "0.1.7",
78
+ "@crewhaus/tool-fetch": "0.1.7",
79
+ "@crewhaus/tool-fs": "0.1.7",
80
+ "@crewhaus/tool-image": "0.1.7",
81
+ "@crewhaus/tool-image-generation": "0.1.7",
82
+ "@crewhaus/tool-document-ingest": "0.1.7",
83
+ "@crewhaus/tool-mcp": "0.1.7",
84
+ "@crewhaus/tool-mouse-keyboard": "0.1.7",
85
+ "@crewhaus/tool-navigate": "0.1.7",
86
+ "@crewhaus/tool-screen-capture": "0.1.7",
87
+ "@crewhaus/tool-task": "0.1.7",
88
+ "@crewhaus/tool-todo": "0.1.7",
89
+ "@crewhaus/tool-vision-grounding": "0.1.7",
90
+ "@crewhaus/tool-web": "0.1.7",
91
+ "@crewhaus/docker-images": "0.1.7",
92
+ "@crewhaus/crewhaus-cloud": "0.1.7",
93
+ "@crewhaus/federation-discovery": "0.1.7",
94
+ "@crewhaus/sandbox": "0.1.7",
95
+ "@crewhaus/sandbox-image-registry": "0.1.7",
96
+ "@crewhaus/compliance-controls": "0.1.7"
97
97
  },
98
98
  "devDependencies": {
99
99
  "zod": "^3.23.8"