crewhaus 0.1.5 → 0.1.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (2) hide show
  1. package/dist/index.js +41 -8
  2. package/package.json +64 -64
package/dist/index.js CHANGED
@@ -1,6 +1,6 @@
1
1
  #!/usr/bin/env bun
2
2
  import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
3
- import { basename, dirname, join, resolve } from "node:path";
3
+ import { basename, dirname, join, relative, resolve } from "node:path";
4
4
  import { SpecParseError, compile, lower } from "@crewhaus/compiler";
5
5
  import { buildContextBundle, discoverRoots } from "@crewhaus/context-bundle";
6
6
  import { CrewhausError } from "@crewhaus/errors";
@@ -137,6 +137,7 @@ const OPTIMIZE_SCHEMA = {
137
137
  { name: "mutator", takesValue: true },
138
138
  { name: "iterations", takesValue: true },
139
139
  { name: "seed", takesValue: true },
140
+ { name: "concurrency", takesValue: true },
140
141
  { name: "improvement-threshold", takesValue: true },
141
142
  // FR-003 — dollar ceiling for model-driven runs (composes with --iterations).
142
143
  { name: "budget-usd", takesValue: true },
@@ -537,7 +538,12 @@ agent:
537
538
  `;
538
539
  writeFileSync(targetFile, yamlText);
539
540
  process.stdout.write(`wrote ${targetFile}\n`);
540
- process.stdout.write(`next: bun crewhaus run ${targetFile}\n`);
541
+ // The runtime resolves the spec and the `.crewhaus/` session store from
542
+ // the current working directory, so guide the user to run from inside
543
+ // the harness directory (where crewhaus.yaml lives), not from here.
544
+ const rel = relative(process.cwd(), targetDir);
545
+ const cd = rel === "" ? "" : `cd ${rel} && `;
546
+ process.stdout.write(`next: ${cd}crewhaus run crewhaus.yaml\n`);
541
547
  logger.debug("init.success", { target: targetFile });
542
548
  }
543
549
  /**
@@ -753,7 +759,8 @@ async function runRunCli(args, ir, specPath) {
753
759
  const sessions = await store.list();
754
760
  const match = sessions.find((s) => s.name === ir.name);
755
761
  if (match === undefined) {
756
- die(`no prior session for spec "${ir.name}" in ${process.cwd()}/.crewhaus/sessions/. Start one with: crewhaus run ${specPath}`);
762
+ const absSpec = resolve(specPath);
763
+ die(`no prior session for spec "${ir.name}" in ${process.cwd()}/.crewhaus/sessions/. Sessions are stored under the directory you run from — start one from the harness directory with: cd ${dirname(absSpec)} && crewhaus run ${basename(absSpec)}`);
757
764
  }
758
765
  resumeId = match.id;
759
766
  process.stdout.write(`[continue] resuming session ${match.id} (last updated ${match.updatedAt})\n`);
@@ -1326,7 +1333,7 @@ async function runDoctorPhilosophyAlignment() {
1326
1333
  async function runOptimize(args) {
1327
1334
  if (args.flags["help"]) {
1328
1335
  process.stdout.write("usage: crewhaus optimize <spec.yaml> --dataset <data> --graders <graders.yaml> " +
1329
- "[--mutator rule-based|claude] [--iterations N] [--seed N] " +
1336
+ "[--mutator rule-based|claude] [--iterations N] [--seed N] [--concurrency N] " +
1330
1337
  "[--improvement-threshold F] [--budget-usd N] [--write-back] [-o <out-dir>]\n");
1331
1338
  return;
1332
1339
  }
@@ -1348,6 +1355,15 @@ async function runOptimize(args) {
1348
1355
  if (Number.isNaN(iterations) || iterations < 1) {
1349
1356
  die(`invalid --iterations "${iterationsFlag}" — must be positive integer`);
1350
1357
  }
1358
+ // Per-candidate eval concurrency. Each iteration runs a full eval pass on
1359
+ // the dev set; on a low provider rate-limit tier a high fan-out trips 429s,
1360
+ // so this is exposed (mirroring `crewhaus eval --concurrency`) and defaults
1361
+ // to 4. The nightly flywheel sets `--concurrency 1` on constrained tiers.
1362
+ const concurrencyFlag = args.flags["concurrency"];
1363
+ const concurrency = typeof concurrencyFlag === "string" ? Number.parseInt(concurrencyFlag, 10) : 4;
1364
+ if (Number.isNaN(concurrency) || concurrency < 1) {
1365
+ die(`invalid --concurrency "${concurrencyFlag}" — must be a positive integer`);
1366
+ }
1351
1367
  // FR-003 — optional dollar budget for model-driven runs. Omit → today's
1352
1368
  // behaviour (iterations cap only). On a rule-based run the gate is inert
1353
1369
  // (no model calls → $0), so passing it is harmless.
@@ -1393,8 +1409,16 @@ async function runOptimize(args) {
1393
1409
  if (devSet.length === 0) {
1394
1410
  die(`dataset has ${samples.length} samples — need at least 2 (70/30 split needs a dev split)`);
1395
1411
  }
1412
+ // Index the dev set by id so the fitness fn can join each graded
1413
+ // sample-result back to the input + reference it was scored against.
1414
+ const devById = new Map(devSet.map((s) => [s.id, s]));
1396
1415
  // Fitness fn: patch the spec with the candidate prompt, lower to IR,
1397
- // run eval-runner, return passRate. Each call is one full eval pass.
1416
+ // run eval-runner, and return the pass-rate PLUS per-sample grades. The
1417
+ // aggregate `passRate` still drives the search (unchanged scoring); the
1418
+ // grades are additive — they carry each sample's overall score and the
1419
+ // grader's rationale to the mutator (via OptimizerState.bestGrades) so
1420
+ // a model-driven rewrite can target the samples the prompt actually
1421
+ // fails and the reason it fails them. Each call is one full eval pass.
1398
1422
  const fitness = async (prompt) => {
1399
1423
  const yamlText = readFileSync(absSpec, "utf-8");
1400
1424
  // Re-parse to capture spec.target without depending on the
@@ -1416,7 +1440,7 @@ async function runOptimize(args) {
1416
1440
  catch (err) {
1417
1441
  if (err instanceof SpecParseError) {
1418
1442
  process.stderr.write("[optimize] candidate compiled invalid spec, skipping\n");
1419
- return 0;
1443
+ return { score: 0 };
1420
1444
  }
1421
1445
  throw err;
1422
1446
  }
@@ -1429,11 +1453,20 @@ async function runOptimize(args) {
1429
1453
  compiledGraders: compiled,
1430
1454
  opts: {
1431
1455
  outDir: join(outDir, "evals", `${prompt.length}_${ir.agent.instructions.length}`),
1432
- concurrency: 4,
1456
+ concurrency,
1433
1457
  seed,
1434
1458
  },
1435
1459
  });
1436
- return summary.aggregates.passRate;
1460
+ const grades = summary.samples.map((r) => {
1461
+ const dev = devById.get(r.sampleId);
1462
+ return {
1463
+ input: dev?.input ?? r.sampleId,
1464
+ score: r.grades.overall.score,
1465
+ ...(dev?.expected_output !== undefined ? { expected: dev.expected_output } : {}),
1466
+ rationale: r.grades.overall.rationale,
1467
+ };
1468
+ });
1469
+ return { score: summary.aggregates.passRate, grades };
1437
1470
  };
1438
1471
  const mutator = args.flags["mutator"];
1439
1472
  let mutatorImpl;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "crewhaus",
3
- "version": "0.1.5",
3
+ "version": "0.1.7",
4
4
  "type": "module",
5
5
  "description": "CrewHaus — the meta-harness compiler for AI agents. Compile one crewhaus.yaml spec into a CLI agent, channel bot, RAG pipeline, multi-agent crew, eval harness, voice or browser agent, and more.",
6
6
  "keywords": [
@@ -31,69 +31,69 @@
31
31
  "test": "bun test src"
32
32
  },
33
33
  "dependencies": {
34
- "@crewhaus/adapter-anthropic": "0.1.5",
35
- "@crewhaus/agent-context-isolation": "0.1.5",
36
- "@crewhaus/audit-log": "0.1.5",
37
- "@crewhaus/eval-optimizer-orchestrator": "0.1.5",
38
- "@crewhaus/prompt-optimizer": "0.1.5",
39
- "@crewhaus/prompt-optimizer-claude": "0.1.5",
40
- "@crewhaus/justification-judge-claude": "0.1.5",
41
- "@crewhaus/spec-patch": "0.1.5",
42
- "@crewhaus/canary-controller": "0.1.5",
43
- "@crewhaus/compiler": "0.1.5",
44
- "@crewhaus/computer-use-driver": "0.1.5",
45
- "@crewhaus/context-bundle": "0.1.5",
46
- "@crewhaus/deployment-controller": "0.1.5",
47
- "@crewhaus/egress-classifier": "0.1.5",
48
- "@crewhaus/egress-matcher-semantic": "0.1.5",
49
- "@crewhaus/embedder": "0.1.5",
50
- "@crewhaus/errors": "0.1.5",
51
- "@crewhaus/eval-dataset": "0.1.5",
52
- "@crewhaus/eval-grader": "0.1.5",
53
- "@crewhaus/eval-report": "0.1.5",
54
- "@crewhaus/eval-runner": "0.1.5",
55
- "@crewhaus/hooks-engine": "0.1.5",
56
- "@crewhaus/infra-utils": "0.1.5",
57
- "@crewhaus/ir": "0.1.5",
58
- "@crewhaus/logging": "0.1.5",
59
- "@crewhaus/mcp-host": "0.1.5",
60
- "@crewhaus/model-router": "0.1.5",
61
- "@crewhaus/migration-engine": "0.1.5",
62
- "@crewhaus/migration-runner": "0.1.5",
63
- "@crewhaus/permission-engine": "0.1.5",
64
- "@crewhaus/run-context": "0.1.5",
65
- "@crewhaus/runtime-core": "0.1.5",
66
- "@crewhaus/secrets-manager": "0.1.5",
67
- "@crewhaus/session-store": "0.1.5",
68
- "@crewhaus/spec-registry": "0.1.5",
69
- "@crewhaus/skills-registry": "0.1.5",
70
- "@crewhaus/slash-commands": "0.1.5",
71
- "@crewhaus/spec": "0.1.5",
72
- "@crewhaus/sub-agent-spawner": "0.1.5",
73
- "@crewhaus/trace-event-bus": "0.1.5",
74
- "@crewhaus/tool-bash": "0.1.5",
75
- "@crewhaus/tool-builder": "0.1.5",
76
- "@crewhaus/tool-catalog": "0.1.5",
77
- "@crewhaus/tool-codegraph": "0.1.5",
78
- "@crewhaus/tool-fetch": "0.1.5",
79
- "@crewhaus/tool-fs": "0.1.5",
80
- "@crewhaus/tool-image": "0.1.5",
81
- "@crewhaus/tool-image-generation": "0.1.5",
82
- "@crewhaus/tool-document-ingest": "0.1.5",
83
- "@crewhaus/tool-mcp": "0.1.5",
84
- "@crewhaus/tool-mouse-keyboard": "0.1.5",
85
- "@crewhaus/tool-navigate": "0.1.5",
86
- "@crewhaus/tool-screen-capture": "0.1.5",
87
- "@crewhaus/tool-task": "0.1.5",
88
- "@crewhaus/tool-todo": "0.1.5",
89
- "@crewhaus/tool-vision-grounding": "0.1.5",
90
- "@crewhaus/tool-web": "0.1.5",
91
- "@crewhaus/docker-images": "0.1.5",
92
- "@crewhaus/crewhaus-cloud": "0.1.5",
93
- "@crewhaus/federation-discovery": "0.1.5",
94
- "@crewhaus/sandbox": "0.1.5",
95
- "@crewhaus/sandbox-image-registry": "0.1.5",
96
- "@crewhaus/compliance-controls": "0.1.5"
34
+ "@crewhaus/adapter-anthropic": "0.1.7",
35
+ "@crewhaus/agent-context-isolation": "0.1.7",
36
+ "@crewhaus/audit-log": "0.1.7",
37
+ "@crewhaus/eval-optimizer-orchestrator": "0.1.7",
38
+ "@crewhaus/prompt-optimizer": "0.1.7",
39
+ "@crewhaus/prompt-optimizer-claude": "0.1.7",
40
+ "@crewhaus/justification-judge-claude": "0.1.7",
41
+ "@crewhaus/spec-patch": "0.1.7",
42
+ "@crewhaus/canary-controller": "0.1.7",
43
+ "@crewhaus/compiler": "0.1.7",
44
+ "@crewhaus/computer-use-driver": "0.1.7",
45
+ "@crewhaus/context-bundle": "0.1.7",
46
+ "@crewhaus/deployment-controller": "0.1.7",
47
+ "@crewhaus/egress-classifier": "0.1.7",
48
+ "@crewhaus/egress-matcher-semantic": "0.1.7",
49
+ "@crewhaus/embedder": "0.1.7",
50
+ "@crewhaus/errors": "0.1.7",
51
+ "@crewhaus/eval-dataset": "0.1.7",
52
+ "@crewhaus/eval-grader": "0.1.7",
53
+ "@crewhaus/eval-report": "0.1.7",
54
+ "@crewhaus/eval-runner": "0.1.7",
55
+ "@crewhaus/hooks-engine": "0.1.7",
56
+ "@crewhaus/infra-utils": "0.1.7",
57
+ "@crewhaus/ir": "0.1.7",
58
+ "@crewhaus/logging": "0.1.7",
59
+ "@crewhaus/mcp-host": "0.1.7",
60
+ "@crewhaus/model-router": "0.1.7",
61
+ "@crewhaus/migration-engine": "0.1.7",
62
+ "@crewhaus/migration-runner": "0.1.7",
63
+ "@crewhaus/permission-engine": "0.1.7",
64
+ "@crewhaus/run-context": "0.1.7",
65
+ "@crewhaus/runtime-core": "0.1.7",
66
+ "@crewhaus/secrets-manager": "0.1.7",
67
+ "@crewhaus/session-store": "0.1.7",
68
+ "@crewhaus/spec-registry": "0.1.7",
69
+ "@crewhaus/skills-registry": "0.1.7",
70
+ "@crewhaus/slash-commands": "0.1.7",
71
+ "@crewhaus/spec": "0.1.7",
72
+ "@crewhaus/sub-agent-spawner": "0.1.7",
73
+ "@crewhaus/trace-event-bus": "0.1.7",
74
+ "@crewhaus/tool-bash": "0.1.7",
75
+ "@crewhaus/tool-builder": "0.1.7",
76
+ "@crewhaus/tool-catalog": "0.1.7",
77
+ "@crewhaus/tool-codegraph": "0.1.7",
78
+ "@crewhaus/tool-fetch": "0.1.7",
79
+ "@crewhaus/tool-fs": "0.1.7",
80
+ "@crewhaus/tool-image": "0.1.7",
81
+ "@crewhaus/tool-image-generation": "0.1.7",
82
+ "@crewhaus/tool-document-ingest": "0.1.7",
83
+ "@crewhaus/tool-mcp": "0.1.7",
84
+ "@crewhaus/tool-mouse-keyboard": "0.1.7",
85
+ "@crewhaus/tool-navigate": "0.1.7",
86
+ "@crewhaus/tool-screen-capture": "0.1.7",
87
+ "@crewhaus/tool-task": "0.1.7",
88
+ "@crewhaus/tool-todo": "0.1.7",
89
+ "@crewhaus/tool-vision-grounding": "0.1.7",
90
+ "@crewhaus/tool-web": "0.1.7",
91
+ "@crewhaus/docker-images": "0.1.7",
92
+ "@crewhaus/crewhaus-cloud": "0.1.7",
93
+ "@crewhaus/federation-discovery": "0.1.7",
94
+ "@crewhaus/sandbox": "0.1.7",
95
+ "@crewhaus/sandbox-image-registry": "0.1.7",
96
+ "@crewhaus/compliance-controls": "0.1.7"
97
97
  },
98
98
  "devDependencies": {
99
99
  "zod": "^3.23.8"