darwin-agents 0.8.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +113 -0
- package/README.md +77 -0
- package/dist/src/cli/canary.d.ts +13 -0
- package/dist/src/cli/canary.d.ts.map +1 -0
- package/dist/src/cli/canary.js +66 -0
- package/dist/src/cli/canary.js.map +1 -0
- package/dist/src/cli/evolution-flags.d.ts +2 -0
- package/dist/src/cli/evolution-flags.d.ts.map +1 -1
- package/dist/src/cli/evolution-flags.js +36 -1
- package/dist/src/cli/evolution-flags.js.map +1 -1
- package/dist/src/cli/evolve.d.ts +2 -0
- package/dist/src/cli/evolve.d.ts.map +1 -1
- package/dist/src/cli/evolve.js +10 -0
- package/dist/src/cli/evolve.js.map +1 -1
- package/dist/src/cli/index.js +7 -0
- package/dist/src/cli/index.js.map +1 -1
- package/dist/src/cli/run.d.ts.map +1 -1
- package/dist/src/cli/run.js +16 -1
- package/dist/src/cli/run.js.map +1 -1
- package/dist/src/evolution/canary.d.ts +164 -0
- package/dist/src/evolution/canary.d.ts.map +1 -0
- package/dist/src/evolution/canary.js +244 -0
- package/dist/src/evolution/canary.js.map +1 -0
- package/dist/src/evolution/critic-families.d.ts +51 -0
- package/dist/src/evolution/critic-families.d.ts.map +1 -0
- package/dist/src/evolution/critic-families.js +51 -0
- package/dist/src/evolution/critic-families.js.map +1 -0
- package/dist/src/evolution/demos.d.ts +128 -0
- package/dist/src/evolution/demos.d.ts.map +1 -0
- package/dist/src/evolution/demos.js +196 -0
- package/dist/src/evolution/demos.js.map +1 -0
- package/dist/src/evolution/enabled-state.d.ts +5 -5
- package/dist/src/evolution/enabled-state.d.ts.map +1 -1
- package/dist/src/evolution/enabled-state.js +10 -5
- package/dist/src/evolution/enabled-state.js.map +1 -1
- package/dist/src/evolution/loop.d.ts +36 -0
- package/dist/src/evolution/loop.d.ts.map +1 -1
- package/dist/src/evolution/loop.js +118 -19
- package/dist/src/evolution/loop.js.map +1 -1
- package/dist/src/evolution/selection.d.ts +73 -0
- package/dist/src/evolution/selection.d.ts.map +1 -0
- package/dist/src/evolution/selection.js +121 -0
- package/dist/src/evolution/selection.js.map +1 -0
- package/dist/src/index.d.ts +3 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +16 -0
- package/dist/src/index.js.map +1 -1
- package/dist/src/types.d.ts +67 -0
- package/dist/src/types.d.ts.map +1 -1
- package/dist/src/types.js.map +1 -1
- package/package.json +4 -1
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,119 @@
|
|
|
2
2
|
|
|
3
3
|
## [Unreleased]
|
|
4
4
|
|
|
5
|
+
## [0.10.0] — 2026-07-03
|
|
6
|
+
|
|
7
|
+
Two research-driven, opt-in evolution surfaces — a new challenger *source*
|
|
8
|
+
(demo injection) and a new parent-*selection* strategy for the existing
|
|
9
|
+
reflective challenger — both validated against the current state of the field
|
|
10
|
+
(GEPA upstream docs, DSPy SIMBA) before a line was written. Default behaviour
|
|
11
|
+
is byte-for-byte unchanged with the flags off.
|
|
12
|
+
|
|
13
|
+
### Added
|
|
14
|
+
|
|
15
|
+
- **SIMBA-style demo injection** (`src/evolution/demos.ts` + `evolution.useDemos`).
|
|
16
|
+
DSPy's SIMBA optimizer improves programs two ways: appending self-reflective
|
|
17
|
+
*rules* (Darwin's reflector already covers that ground) and appending
|
|
18
|
+
successful past examples as *demonstrations*. This release adapts the second
|
|
19
|
+
strategy to the online loop: on every `demoEveryK`-th evolution cycle (default
|
|
20
|
+
4) the loop harvests the agent's own highest-scoring past runs (score ≥
|
|
21
|
+
`demoScoreThreshold`, default 8; at most one demo per task type for
|
|
22
|
+
diversity; `maxDemos` cap, default 2) and appends them as a marker-delimited
|
|
23
|
+
"Demonstrations" section. The demo-augmented prompt is a normal challenger —
|
|
24
|
+
same alignment guard, same A/B test, same safety gate; if demos don't help
|
|
25
|
+
this agent, the incumbent wins. **Zero LLM cost** (pure selection +
|
|
26
|
+
rendering on data Darwin already persists), works with or without `useGepa`,
|
|
27
|
+
idempotent via `<!-- darwin:demos:start/end -->` markers (a later cycle
|
|
28
|
+
*refreshes* the section, never stacks a second one). Pure helpers
|
|
29
|
+
(`selectDemoCandidates` / `buildDemoSection` / `applyDemoSection` /
|
|
30
|
+
`stripDemoSection`) are exported from the package root. CLI: `--demos` /
|
|
31
|
+
`--no-demos`.
|
|
32
|
+
|
|
33
|
+
- **Parent-selection strategies** (`src/evolution/selection.ts` +
|
|
34
|
+
`evolution.candidateSelection`) — GEPA `candidate_selection_strategy` parity
|
|
35
|
+
for the online loop. Historically the loop always reflected from the
|
|
36
|
+
currently-active prompt (a hill-climb that can sit on a local optimum).
|
|
37
|
+
Opt-in strategies pick the reflection parent from the agent's *scored
|
|
38
|
+
version history* instead: `'best'` (GEPA `current_best` — highest
|
|
39
|
+
scalarised composite), `'pareto'` (GEPA default — uniform sample from the
|
|
40
|
+
non-dominated front, keeping lineages alive that win on different
|
|
41
|
+
objectives), `'epsilon-greedy'` (explore with probability
|
|
42
|
+
`explorationEpsilon`, default 0.1, exploit otherwise). The RNG is
|
|
43
|
+
injectable via the new `DarwinLoopDeps.rng` for deterministic tests.
|
|
44
|
+
Precedence: `useCoverage` (GEPA Algorithm 2, the more specific selector)
|
|
45
|
+
wins when it finds a coverage parent. Only consulted when `useGepa` is on.
|
|
46
|
+
CLI: `--candidate-selection <active|best|pareto|epsilon-greedy>`.
|
|
47
|
+
|
|
48
|
+
### Internal
|
|
49
|
+
|
|
50
|
+
- `tryMergeVariant`'s version-history scoring extracted into a shared
|
|
51
|
+
`buildScoredHistory` (used by merge *and* parent selection) — behaviour
|
|
52
|
+
unchanged.
|
|
53
|
+
- `package.json` now lists Claude (Anthropic) as a contributor — see the
|
|
54
|
+
README's new **Credits** section for how this project is actually built.
|
|
55
|
+
|
|
56
|
+
## [0.9.0] — 2026-06-21
|
|
57
|
+
|
|
58
|
+
### Added
|
|
59
|
+
|
|
60
|
+
- **Validate-by-reproduce drift-detection canary** (Phase 2 A5) — new
|
|
61
|
+
`src/evolution/canary.ts` + a `darwin canary <agent>` command. The A/B gate
|
|
62
|
+
guards prompt *quality*; the canary guards *behaviour*. It compares an agent's
|
|
63
|
+
recent execution trajectories (captured in A1) against a frozen baseline using
|
|
64
|
+
tolerance-based metrics — unordered tool-set Jaccard, ordered tool-sequence
|
|
65
|
+
similarity, turn-count ratio, error-rate delta — and flags drift the score can
|
|
66
|
+
miss (a model update or a broken tool changing *how* the agent works while the
|
|
67
|
+
quality score stays flat). Exact-hash equivalence is deliberately avoided (LLM
|
|
68
|
+
runs are non-deterministic). Drift requires a *pattern* (default ≥2 of N runs),
|
|
69
|
+
and the baseline is pinned to the active prompt version so an intentional
|
|
70
|
+
evolution reports `insufficient-data` (re-baseline) rather than a false alarm.
|
|
71
|
+
Pure and zero-dep; `--json` + `--exit-on-drift` for CI. The metrics and the
|
|
72
|
+
`runCanaryOverExperiments` orchestrator are exported from the package root so
|
|
73
|
+
consumers can run the same check on their own captured trajectories.
|
|
74
|
+
|
|
75
|
+
- **Cross-family critic diversity check** (`src/evolution/critic-families.ts`).
|
|
76
|
+
Multi-critic evaluation spreads critics across model families to reduce
|
|
77
|
+
LLM-as-judge bias — but only when more than one provider key is present.
|
|
78
|
+
Otherwise all three critics collapse onto a single family (`claude-cli` and
|
|
79
|
+
`anthropic-api` are the *same* family, differing only in latency) with no
|
|
80
|
+
signal to the operator. The run path now warns when the critics share one
|
|
81
|
+
family, and hard-fails when `DARWIN_REQUIRE_CROSS_FAMILY` is set (CI / strict
|
|
82
|
+
setups). Default behaviour is unchanged apart from the new warning.
|
|
83
|
+
|
|
84
|
+
## [0.8.0] — 2026-06-21
|
|
85
|
+
|
|
86
|
+
**Evolution wired end-to-end through the CLI, plus three correctness fixes.** The
|
|
87
|
+
v0.7 evolution surfaces existed but the `darwin evolve` command couldn't fully
|
|
88
|
+
drive them; this release closes that gap and hardens score parsing. The
|
|
89
|
+
automatic loop is behaviour-preserving — the gated path still passes the
|
|
90
|
+
existing loop/GEPA suites unchanged.
|
|
91
|
+
|
|
92
|
+
### Added
|
|
93
|
+
|
|
94
|
+
- **`darwin evolve <agent> --force`** now runs the loop's real
|
|
95
|
+
variant-generation + A/B-start path on demand (`DarwinLoop.forceEvolve()`),
|
|
96
|
+
bypassing the automatic "enough runs / actionable patterns / data quality"
|
|
97
|
+
gates while still refusing the impossible cases (no active prompt, no recorded
|
|
98
|
+
experiments, an A/B test already running). It was previously a stub that
|
|
99
|
+
printed "not yet available" while the help text advertised it.
|
|
100
|
+
- **`src/evolution/build-loop.ts`** — shared loop wiring (legacy optimizer +
|
|
101
|
+
opt-in GEPA) used by both `cli/run.ts` and the new `--force` command.
|
|
102
|
+
|
|
103
|
+
### Fixed
|
|
104
|
+
|
|
105
|
+
- **`darwin evolve <agent> --enable|--disable` now persists.** The flag mutated
|
|
106
|
+
only the in-memory agent singleton, so it was lost on process exit and `darwin
|
|
107
|
+
run` read the static source default again. A new `DarwinState.evolutionEnabled`
|
|
108
|
+
override map (round-tripped by every backend's existing JSON state blob) is set
|
|
109
|
+
atomically by the evolve command and wins over the static default in the run
|
|
110
|
+
path. Persisting an enable on an agent with no evolution config is a no-op, not
|
|
111
|
+
a crash; the field is read defensively so pre-existing state rows keep their
|
|
112
|
+
prior behaviour. New shared helper `src/evolution/enabled-state.ts`.
|
|
113
|
+
- **Robust critic-score parsing** via a shared `parseCriticScore` helper —
|
|
114
|
+
handles `===SCORE===`, `N/10`, "N out of 10", "rating: N", "I'd rate this N", …
|
|
115
|
+
(clamped 1–10). Previously only `===SCORE===` and a bare `N/10` were read,
|
|
116
|
+
silently dropping every other phrasing from evolution.
|
|
117
|
+
|
|
5
118
|
## [0.7.1] — 2026-06-20
|
|
6
119
|
|
|
7
120
|
**Documentation honesty + a reproducible evolution benchmark.** No library code
|
package/README.md
CHANGED
|
@@ -195,6 +195,66 @@ const loop = new DarwinLoop({
|
|
|
195
195
|
- **Style-bias-free judging** (`normalizeForJudging`) — strip markdown before the
|
|
196
196
|
critic scores, so it measures content not formatting.
|
|
197
197
|
|
|
198
|
+
### Drift detection — validate-by-reproduce canary (v0.9)
|
|
199
|
+
|
|
200
|
+
The A/B gate guards prompt *quality*. The canary guards *behaviour*: a model
|
|
201
|
+
update or a broken tool can change *how* an agent reaches its answer (different
|
|
202
|
+
tools, more turns, more errors) while the quality score stays flat. `darwin
|
|
203
|
+
canary` re-checks recent runs against a frozen baseline and flags that drift.
|
|
204
|
+
|
|
205
|
+
```bash
|
|
206
|
+
darwin canary writer # Stable / drift / insufficient-data report
|
|
207
|
+
darwin canary writer --json # Machine-readable, for dashboards
|
|
208
|
+
darwin canary writer --exit-on-drift # Non-zero exit on drift, for CI
|
|
209
|
+
```
|
|
210
|
+
|
|
211
|
+
It compares execution *trajectories* (captured opt-in since v0.5) with
|
|
212
|
+
tolerance-based metrics — unordered tool-set Jaccard, ordered sequence
|
|
213
|
+
similarity, turn-count ratio, error-rate delta — never an exact hash (LLM runs
|
|
214
|
+
are non-deterministic). Drift has to be a *pattern* (≥2 of N runs), and the
|
|
215
|
+
baseline is pinned to the active prompt version, so an intentional evolution
|
|
216
|
+
reports `insufficient-data` (re-baseline), not a false alarm. The metrics and
|
|
217
|
+
the `runCanaryOverExperiments` orchestrator are exported for your own pipelines.
|
|
218
|
+
|
|
219
|
+
**Cross-family judging:** with more than one provider key present, Darwin
|
|
220
|
+
already spreads the three critics across model families to cut LLM-as-judge
|
|
221
|
+
bias. With only one key they collapse onto a single family (note: `claude-cli`
|
|
222
|
+
and `anthropic-api` are the *same* family) — Darwin now warns, and hard-fails
|
|
223
|
+
under `DARWIN_REQUIRE_CROSS_FAMILY=1` for strict / CI setups.
|
|
224
|
+
|
|
225
|
+
### Demo injection + parent selection (v0.10, opt-in)
|
|
226
|
+
|
|
227
|
+
Two more challenger surfaces, both borrowed from the strongest ideas in the
|
|
228
|
+
field and adapted to the online loop. Both are off by default.
|
|
229
|
+
|
|
230
|
+
**Demo injection** (`useDemos`) is DSPy SIMBA's `append_a_demo` strategy: on
|
|
231
|
+
every `demoEveryK`-th cycle (default 4) the loop harvests the agent's *own
|
|
232
|
+
highest-scoring past runs* (score ≥ 8, one per task type, capped at
|
|
233
|
+
`maxDemos`) and appends them to the prompt as a marker-delimited
|
|
234
|
+
"Demonstrations" section. Zero LLM cost — pure selection over data Darwin
|
|
235
|
+
already stores — and the demo-augmented prompt is a normal challenger: same
|
|
236
|
+
alignment guard, same A/B test, same rollback. If showing the agent its own
|
|
237
|
+
best work doesn't measurably help, the incumbent wins and nothing changes.
|
|
238
|
+
|
|
239
|
+
**Parent selection** (`candidateSelection`) is GEPA's
|
|
240
|
+
`candidate_selection_strategy`: instead of always mutating the currently-active
|
|
241
|
+
prompt (a hill-climb), reflect from `'best'` (highest composite in the version
|
|
242
|
+
history), `'pareto'` (uniform sample from the history's Pareto front — keeps
|
|
243
|
+
lineages alive that win on *different* objectives), or `'epsilon-greedy'`
|
|
244
|
+
(explore with probability `explorationEpsilon`). Requires `useGepa`.
|
|
245
|
+
|
|
246
|
+
```typescript
|
|
247
|
+
evolution: {
|
|
248
|
+
enabled: true,
|
|
249
|
+
useDemos: true, // SIMBA-style demos as a challenger source
|
|
250
|
+
demoEveryK: 4, // cadence (offset from mergeEveryK: 3)
|
|
251
|
+
maxDemos: 2,
|
|
252
|
+
useGepa: true,
|
|
253
|
+
candidateSelection: 'pareto', // or 'best' | 'epsilon-greedy' | 'active'
|
|
254
|
+
explorationEpsilon: 0.1, // only for 'epsilon-greedy'
|
|
255
|
+
},
|
|
256
|
+
```
|
|
257
|
+
|
|
198
258
|
## Built-in Agents
|
|
199
259
|
|
|
200
260
|
| Agent | What it does | Needs |
|
|
@@ -407,6 +467,8 @@ darwin run analyst --path ./src # Analyze a codebase
|
|
|
407
467
|
darwin status # Overview of all agents
|
|
408
468
|
darwin status writer # Detailed agent stats + evolution history
|
|
409
469
|
|
|
470
|
+
darwin canary writer # Behavioural drift vs a frozen baseline (--json, --exit-on-drift)
|
|
471
|
+
|
|
410
472
|
darwin evolve writer --enable # Enable self-evolution (persisted)
|
|
411
473
|
darwin evolve writer --disable # Disable self-evolution (persisted)
|
|
412
474
|
darwin evolve writer --reset # Reset to v1
|
|
@@ -436,6 +498,8 @@ darwin run writer "Explain consensus" --gepa --pareto-gate
|
|
|
436
498
|
| `--pareto-gate` / `--no-pareto-gate` | Reject an A/B winner that regressed on any objective |
|
|
437
499
|
| `--coverage` / `--no-coverage` | Pick the reflection parent by per-task-type coverage breadth (GEPA Algorithm 2) |
|
|
438
500
|
| `--reflection-model <id>` | Use a stronger model for GEPA reflection (the documented leverage point) |
|
|
501
|
+
| `--demos` / `--no-demos` | SIMBA-style demo injection: the agent's best past runs as a challenger (v0.10) |
|
|
502
|
+
| `--candidate-selection <s>` | Reflection parent strategy: `active` \| `best` \| `pareto` \| `epsilon-greedy` (v0.10) |
|
|
439
503
|
|
|
440
504
|
All default to **off** — the baseline single-objective evolution loop is
|
|
441
505
|
unchanged unless you opt in.
|
|
@@ -497,6 +561,19 @@ The safety gate prevents regressions. If a new variant scores >20% lower, Darwin
|
|
|
497
561
|
|
|
498
562
|
PRs welcome. See [CONTRIBUTING.md](CONTRIBUTING.md).
|
|
499
563
|
|
|
564
|
+
## Credits
|
|
565
|
+
|
|
566
|
+
Darwin is pair-built by a human and an AI, and we think that should be said
|
|
567
|
+
plainly rather than hidden. Matthias Meyer (StudioMeyer) sets direction, makes
|
|
568
|
+
the calls and reviews everything; most of the code is written by
|
|
569
|
+
[Claude](https://www.anthropic.com/claude) — currently **Claude Fable 5** —
|
|
570
|
+
working from research against the primary sources (the GEPA paper and
|
|
571
|
+
reference implementation, DSPy, the MCP spec) and gated by multi-agent code
|
|
572
|
+
review plus this repo's own test suite before anything ships. Claude is listed
|
|
573
|
+
as a contributor in `package.json`, and the commits it co-writes carry a
|
|
574
|
+
`Co-Authored-By` trailer. A framework about agents that improve themselves,
|
|
575
|
+
built with an agent — we find that fitting.
|
|
576
|
+
|
|
500
577
|
## About StudioMeyer
|
|
501
578
|
|
|
502
579
|
[StudioMeyer](https://studiomeyer.io) is an AI and design studio based in Palma de Mallorca, working with clients worldwide. We build custom websites and AI infrastructure for small and medium businesses. Production stack on Claude Agent SDK, MCP and n8n, with Sentry, Langfuse and LangGraph for observability and an in-house guard layer.
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* darwin canary <agent> [--json] [--exit-on-drift]
|
|
3
|
+
*
|
|
4
|
+
* Validate-by-reproduce drift check (Phase 2 A5). Loads the agent's recent
|
|
5
|
+
* experiments, baselines within the active prompt version, and reports whether
|
|
6
|
+
* the recent trajectories have drifted from that baseline (different tools,
|
|
7
|
+
* more turns, more errors) — behavioural drift the A/B *quality* gate can miss.
|
|
8
|
+
*
|
|
9
|
+
* Read-only. Exits 0 by default even on drift (the signal is the printed report
|
|
10
|
+
* / JSON, mirroring the staleness monitor); pass `--exit-on-drift` for CI.
|
|
11
|
+
*/
|
|
12
|
+
export declare function canaryCommand(args: string[]): Promise<void>;
|
|
13
|
+
//# sourceMappingURL=canary.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"canary.d.ts","sourceRoot":"","sources":["../../../src/cli/canary.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAUH,wBAAsB,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CA+BjE"}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* darwin canary <agent> [--json] [--exit-on-drift]
|
|
3
|
+
*
|
|
4
|
+
* Validate-by-reproduce drift check (Phase 2 A5). Loads the agent's recent
|
|
5
|
+
* experiments, baselines within the active prompt version, and reports whether
|
|
6
|
+
* the recent trajectories have drifted from that baseline (different tools,
|
|
7
|
+
* more turns, more errors) — behavioural drift the A/B *quality* gate can miss.
|
|
8
|
+
*
|
|
9
|
+
* Read-only. Exits 0 by default even on drift (the signal is the printed report
|
|
10
|
+
* / JSON, mirroring the staleness monitor); pass `--exit-on-drift` for CI.
|
|
11
|
+
*/
|
|
12
|
+
import { createMemory } from '../memory/index.js';
|
|
13
|
+
import { loadConfig } from '../core/agent.js';
|
|
14
|
+
import { runCanaryOverExperiments } from '../evolution/canary.js';
|
|
15
|
+
/** How many recent experiments to pull for the drift window — wide enough that
|
|
16
|
+
* several same-version runs land in it after an evolution. */
|
|
17
|
+
const CANARY_WINDOW = 100;
|
|
18
|
+
export async function canaryCommand(args) {
|
|
19
|
+
const agentName = args.find((a) => !a.startsWith('--'));
|
|
20
|
+
if (!agentName) {
|
|
21
|
+
throw new Error('Usage: darwin canary <agent> [--json] [--exit-on-drift]');
|
|
22
|
+
}
|
|
23
|
+
const asJson = args.includes('--json');
|
|
24
|
+
const exitOnDrift = args.includes('--exit-on-drift');
|
|
25
|
+
const config = await loadConfig();
|
|
26
|
+
const memory = createMemory(config);
|
|
27
|
+
await memory.init();
|
|
28
|
+
try {
|
|
29
|
+
const state = await memory.getState();
|
|
30
|
+
const activeVersion = state.activeVersions[agentName] ?? null;
|
|
31
|
+
const experiments = await memory.loadExperiments(agentName, CANARY_WINDOW);
|
|
32
|
+
const report = runCanaryOverExperiments(agentName, experiments, activeVersion);
|
|
33
|
+
if (asJson) {
|
|
34
|
+
console.log(JSON.stringify(report, null, 2));
|
|
35
|
+
}
|
|
36
|
+
else {
|
|
37
|
+
printReport(report);
|
|
38
|
+
}
|
|
39
|
+
if (exitOnDrift && report.status === 'drift') {
|
|
40
|
+
process.exitCode = 1;
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
finally {
|
|
44
|
+
await memory.close();
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
function printReport(report) {
|
|
48
|
+
const icon = report.status === 'drift' ? '⚠' : report.status === 'ok' ? '✓' : '·';
|
|
49
|
+
console.log('');
|
|
50
|
+
console.log(`[darwin] Canary: ${report.agentName}${report.promptVersion ? ` / ${report.promptVersion}` : ''}`);
|
|
51
|
+
if (report.baselineId)
|
|
52
|
+
console.log(` Baseline: ${report.baselineId}`);
|
|
53
|
+
console.log(` Evaluated: ${report.evaluated} candidate run(s)`);
|
|
54
|
+
console.log(` Status: ${icon} ${report.status.toUpperCase()}`);
|
|
55
|
+
if (report.result) {
|
|
56
|
+
const r = report.result;
|
|
57
|
+
console.log(` Drifted: ${r.driftedCount}/${r.total} runs`);
|
|
58
|
+
console.log(` Medians: tool-Jaccard ${r.medianToolJaccard.toFixed(2)} | turn-ratio ${r.medianTurnRatio.toFixed(2)}`);
|
|
59
|
+
if (r.reasons.length > 0) {
|
|
60
|
+
console.log(` Reasons: ${r.reasons.join('; ')}`);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
console.log(` ${report.message}`);
|
|
64
|
+
console.log('');
|
|
65
|
+
}
|
|
66
|
+
//# sourceMappingURL=canary.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"canary.js","sourceRoot":"","sources":["../../../src/cli/canary.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,wBAAwB,EAAqB,MAAM,wBAAwB,CAAC;AAErF;+DAC+D;AAC/D,MAAM,aAAa,GAAG,GAAG,CAAC;AAE1B,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,IAAc;IAChD,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC;IACxD,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;IAC7E,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACvC,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC;IAErD,MAAM,MAAM,GAAG,MAAM,UAAU,EAAE,CAAC;IAClC,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;IACpC,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;IAEpB,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,QAAQ,EAAE,CAAC;QACtC,MAAM,aAAa,GAAG,KAAK,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC;QAC9D,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,eAAe,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;QAE3E,MAAM,MAAM,GAAG,wBAAwB,CAAC,SAAS,EAAE,WAAW,EAAE,aAAa,CAAC,CAAC;QAE/E,IAAI,MAAM,EAAE,CAAC;YACX,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC/C,CAAC;aAAM,CAAC;YACN,WAAW,CAAC,MAAM,CAAC,CAAC;QACtB,CAAC;QAED,IAAI,WAAW,IAAI,MAAM,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;YAC7C,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;YAAS,CAAC;QACT,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;AACH,CAAC;AAED,SAAS,WAAW,CAAC,MAAoB;IACvC,MAAM,IAAI,GACR,MAAM,CAAC,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;IAEvE,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAChB,OAAO,CAAC,GAAG,CAAC,oBAAoB,MAAM,CAAC,SAAS,GAAG,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC/G,IAAI,MAAM,CAAC,UAAU;QAAE,OAAO,CAAC,GAAG,CAAC,gBAAgB,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC;IACxE,OAAO,CAAC,GAAG,CAAC,gBAAgB,MAAM,CAAC,SAAS,mBAAmB,CAAC,CAAC;IACjE,OAAO,CAAC,GAAG,CAAC,gBAAgB,IAAI,IAAI,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;IAEnE,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAClB,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;QACxB,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC,KAAK,OAAO,CAAC,CAAC;QAC9D,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACxH,IAAI,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzB,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACtD,CAAC;IACH,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,KAAK,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;IACnC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;AAClB,CAAC"}
|
|
@@ -12,6 +12,8 @@
|
|
|
12
12
|
* --pareto-gate / --no-pareto-gate
|
|
13
13
|
* --coverage / --no-coverage
|
|
14
14
|
* --reflection-model <model-id>
|
|
15
|
+
* --demos / --no-demos (v0.10.0)
|
|
16
|
+
* --candidate-selection <active|best|pareto|epsilon-greedy> (v0.10.0)
|
|
15
17
|
*/
|
|
16
18
|
import type { EvolutionConfigOverride } from '../types.js';
|
|
17
19
|
/** True when `arg` is one of the evolution-config flags this module handles. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"evolution-flags.d.ts","sourceRoot":"","sources":["../../../src/cli/evolution-flags.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"evolution-flags.d.ts","sourceRoot":"","sources":["../../../src/cli/evolution-flags.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,aAAa,CAAC;AAU3D,gFAAgF;AAChF,wBAAgB,qBAAqB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAkB1D;AAED;;;;;;;GAOG;AACH,wBAAgB,kBAAkB,CAChC,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,MAAM,GAAG,SAAS,EAC3B,MAAM,EAAE,uBAAuB,GAC9B,MAAM,CAyDR;AAED;;;;;GAKG;AACH,wBAAgB,yBAAyB,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG;IACzD,QAAQ,EAAE,uBAAuB,CAAC;IAClC,IAAI,EAAE,MAAM,EAAE,CAAC;CAChB,CAaA;AAED,4DAA4D;AAC5D,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,uBAAuB,GAAG,OAAO,CAU9E"}
|
|
@@ -12,7 +12,16 @@
|
|
|
12
12
|
* --pareto-gate / --no-pareto-gate
|
|
13
13
|
* --coverage / --no-coverage
|
|
14
14
|
* --reflection-model <model-id>
|
|
15
|
+
* --demos / --no-demos (v0.10.0)
|
|
16
|
+
* --candidate-selection <active|best|pareto|epsilon-greedy> (v0.10.0)
|
|
15
17
|
*/
|
|
18
|
+
/** Valid values for `--candidate-selection` (v0.10.0). */
|
|
19
|
+
const CANDIDATE_SELECTION_VALUES = new Set([
|
|
20
|
+
'active',
|
|
21
|
+
'best',
|
|
22
|
+
'pareto',
|
|
23
|
+
'epsilon-greedy',
|
|
24
|
+
]);
|
|
16
25
|
/** True when `arg` is one of the evolution-config flags this module handles. */
|
|
17
26
|
export function isEvolutionConfigFlag(arg) {
|
|
18
27
|
switch (arg) {
|
|
@@ -25,6 +34,9 @@ export function isEvolutionConfigFlag(arg) {
|
|
|
25
34
|
case '--coverage':
|
|
26
35
|
case '--no-coverage':
|
|
27
36
|
case '--reflection-model':
|
|
37
|
+
case '--demos':
|
|
38
|
+
case '--no-demos':
|
|
39
|
+
case '--candidate-selection':
|
|
28
40
|
return true;
|
|
29
41
|
default:
|
|
30
42
|
return false;
|
|
@@ -70,6 +82,27 @@ export function applyEvolutionFlag(arg, nextArg, target) {
|
|
|
70
82
|
return 1;
|
|
71
83
|
}
|
|
72
84
|
return 0;
|
|
85
|
+
case '--demos':
|
|
86
|
+
target.useDemos = true;
|
|
87
|
+
return 0;
|
|
88
|
+
case '--no-demos':
|
|
89
|
+
target.useDemos = false;
|
|
90
|
+
return 0;
|
|
91
|
+
case '--candidate-selection':
|
|
92
|
+
// Value is consumed either way (it was clearly meant as this flag's
|
|
93
|
+
// argument); it is only APPLIED when it names a known strategy — an
|
|
94
|
+
// unknown value warns instead of silently persisting a config the loop
|
|
95
|
+
// would then silently treat as 'active'.
|
|
96
|
+
if (nextArg !== undefined) {
|
|
97
|
+
if (CANDIDATE_SELECTION_VALUES.has(nextArg)) {
|
|
98
|
+
target.candidateSelection = nextArg;
|
|
99
|
+
}
|
|
100
|
+
else {
|
|
101
|
+
console.warn(`[darwin] --candidate-selection "${nextArg}" is not one of: ${[...CANDIDATE_SELECTION_VALUES].join(', ')} — ignored.`);
|
|
102
|
+
}
|
|
103
|
+
return 1;
|
|
104
|
+
}
|
|
105
|
+
return 0;
|
|
73
106
|
default:
|
|
74
107
|
return 0;
|
|
75
108
|
}
|
|
@@ -101,6 +134,8 @@ export function hasAnyEvolutionFlag(override) {
|
|
|
101
134
|
override.useMerge !== undefined ||
|
|
102
135
|
override.paretoGate !== undefined ||
|
|
103
136
|
override.useCoverage !== undefined ||
|
|
104
|
-
override.reflectionModel !== undefined
|
|
137
|
+
override.reflectionModel !== undefined ||
|
|
138
|
+
override.useDemos !== undefined ||
|
|
139
|
+
override.candidateSelection !== undefined);
|
|
105
140
|
}
|
|
106
141
|
//# sourceMappingURL=evolution-flags.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"evolution-flags.js","sourceRoot":"","sources":["../../../src/cli/evolution-flags.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"evolution-flags.js","sourceRoot":"","sources":["../../../src/cli/evolution-flags.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAIH,0DAA0D;AAC1D,MAAM,0BAA0B,GAAwB,IAAI,GAAG,CAAC;IAC9D,QAAQ;IACR,MAAM;IACN,QAAQ;IACR,gBAAgB;CACjB,CAAC,CAAC;AAEH,gFAAgF;AAChF,MAAM,UAAU,qBAAqB,CAAC,GAAW;IAC/C,QAAQ,GAAG,EAAE,CAAC;QACZ,KAAK,QAAQ,CAAC;QACd,KAAK,WAAW,CAAC;QACjB,KAAK,SAAS,CAAC;QACf,KAAK,YAAY,CAAC;QAClB,KAAK,eAAe,CAAC;QACrB,KAAK,kBAAkB,CAAC;QACxB,KAAK,YAAY,CAAC;QAClB,KAAK,eAAe,CAAC;QACrB,KAAK,oBAAoB,CAAC;QAC1B,KAAK,SAAS,CAAC;QACf,KAAK,YAAY,CAAC;QAClB,KAAK,uBAAuB;YAC1B,OAAO,IAAI,CAAC;QACd;YACE,OAAO,KAAK,CAAC;IACjB,CAAC;AACH,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,kBAAkB,CAChC,GAAW,EACX,OAA2B,EAC3B,MAA+B;IAE/B,QAAQ,GAAG,EAAE,CAAC;QACZ,KAAK,QAAQ;YACX,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC;YACtB,OAAO,CAAC,CAAC;QACX,KAAK,WAAW;YACd,MAAM,CAAC,OAAO,GAAG,KAAK,CAAC;YACvB,OAAO,CAAC,CAAC;QACX,KAAK,SAAS;YACZ,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC;YACvB,OAAO,CAAC,CAAC;QACX,KAAK,YAAY;YACf,MAAM,CAAC,QAAQ,GAAG,KAAK,CAAC;YACxB,OAAO,CAAC,CAAC;QACX,KAAK,eAAe;YAClB,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC;YACzB,OAAO,CAAC,CAAC;QACX,KAAK,kBAAkB;YACrB,MAAM,CAAC,UAAU,GAAG,KAAK,CAAC;YAC1B,OAAO,CAAC,CAAC;QACX,KAAK,YAAY;YACf,MAAM,CAAC,WAAW,GAAG,IAAI,CAAC;YAC1B,OAAO,CAAC,CAAC;QACX,KAAK,eAAe;YAClB,MAAM,CAAC,WAAW,GAAG,KAAK,CAAC;YAC3B,OAAO,CAAC,CAAC;QACX,KAAK,oBAAoB;YACvB,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;gBAC1B,MAAM,CAAC,eAAe,GAAG,OAAO,CAAC;gBACjC,OAAO,CAAC,CAAC;YACX,CAAC;YACD,OAAO,CAAC,CAAC;QACX,KAAK,SAAS;YACZ,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC;YACvB,OAAO,CAAC,CAAC;QACX,KAAK,YAAY;YACf,MAAM,CAAC,QAAQ,GAAG,KAAK,CAAC;YACxB,OAAO,CAAC,CAAC;QACX,KAAK,uBAAuB;YAC1B,oEAAoE;YACpE,oEAAoE;YACpE,uEAAuE;YACvE,yCAAyC;YACzC,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;gBAC1B,IAAI,0BAA0B,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;oBAC5C,MAAM,CAAC,kBAAkB,GAAG,OAAwD,CAAC;gBACvF,CAAC;qBAAM,CAAC;oBACN,OAAO,CAAC,IAAI,CACV,mCAAmC,OAAO,oBAAoB,CAAC,GAAG,0BAA0B,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CACtH,CAAC;gBACJ,CAAC;gBACD,OAAO,CAAC,CAAC;YACX,CAAC;YACD,OAAO,CAAC,CAAC;QACX;YACE,OAAO,CAAC,CAAC;IACb,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,yBAAyB,CAAC,IAAc;IAItD,MAAM,QAAQ,GAA4B,EAAE,CAAC;IAC7C,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAE,CAAC;QACrB,IAAI,qBAAqB,CAAC,GAAG,CAAC,EAAE,CAAC;YAC/B,MAAM,QAAQ,GAAG,kBAAkB,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;YAChE,CAAC,IAAI,QAAQ,CAAC;QAChB,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACjB,CAAC;IACH,CAAC;IACD,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;AAC5B,CAAC;AAED,4DAA4D;AAC5D,MAAM,UAAU,mBAAmB,CAAC,QAAiC;IACnE,OAAO,CACL,QAAQ,CAAC,OAAO,KAAK,SAAS;QAC9B,QAAQ,CAAC,QAAQ,KAAK,SAAS;QAC/B,QAAQ,CAAC,UAAU,KAAK,SAAS;QACjC,QAAQ,CAAC,WAAW,KAAK,SAAS;QAClC,QAAQ,CAAC,eAAe,KAAK,SAAS;QACtC,QAAQ,CAAC,QAAQ,KAAK,SAAS;QAC/B,QAAQ,CAAC,kBAAkB,KAAK,SAAS,CAC1C,CAAC;AACJ,CAAC"}
|
package/dist/src/cli/evolve.d.ts
CHANGED
|
@@ -17,6 +17,8 @@
|
|
|
17
17
|
* --pareto-gate / --no-pareto-gate multi-objective A/B activation gate
|
|
18
18
|
* --coverage / --no-coverage instance-wise coverage selection
|
|
19
19
|
* --reflection-model <id> stronger reflection model for GEPA
|
|
20
|
+
* --demos / --no-demos SIMBA-style demo injection (v0.10)
|
|
21
|
+
* --candidate-selection <s> reflection parent: active|best|pareto|epsilon-greedy
|
|
20
22
|
*/
|
|
21
23
|
export declare function evolveCommand(args: string[]): Promise<void>;
|
|
22
24
|
//# sourceMappingURL=evolve.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"evolve.d.ts","sourceRoot":"","sources":["../../../src/cli/evolve.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"evolve.d.ts","sourceRoot":"","sources":["../../../src/cli/evolve.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAeH,wBAAsB,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAkGjE"}
|
package/dist/src/cli/evolve.js
CHANGED
|
@@ -17,6 +17,8 @@
|
|
|
17
17
|
* --pareto-gate / --no-pareto-gate multi-objective A/B activation gate
|
|
18
18
|
* --coverage / --no-coverage instance-wise coverage selection
|
|
19
19
|
* --reflection-model <id> stronger reflection model for GEPA
|
|
20
|
+
* --demos / --no-demos SIMBA-style demo injection (v0.10)
|
|
21
|
+
* --candidate-selection <s> reflection parent: active|best|pareto|epsilon-greedy
|
|
20
22
|
*/
|
|
21
23
|
import { createMemory } from '../memory/index.js';
|
|
22
24
|
import { loadConfig } from '../core/agent.js';
|
|
@@ -135,6 +137,10 @@ function describeOverride(o) {
|
|
|
135
137
|
parts.push(`coverage=${o.useCoverage}`);
|
|
136
138
|
if (o.reflectionModel !== undefined)
|
|
137
139
|
parts.push(`reflectionModel=${o.reflectionModel}`);
|
|
140
|
+
if (o.useDemos !== undefined)
|
|
141
|
+
parts.push(`demos=${o.useDemos}`);
|
|
142
|
+
if (o.candidateSelection !== undefined)
|
|
143
|
+
parts.push(`candidateSelection=${o.candidateSelection}`);
|
|
138
144
|
return parts.length > 0 ? parts.join(', ') : '(none)';
|
|
139
145
|
}
|
|
140
146
|
/** One-line summary of the effective advanced config for the status view. */
|
|
@@ -146,9 +152,13 @@ function describeConfig(evo) {
|
|
|
146
152
|
`merge=${evo.useMerge ?? false}`,
|
|
147
153
|
`paretoGate=${evo.paretoGate ?? false}`,
|
|
148
154
|
`coverage=${evo.useCoverage ?? false}`,
|
|
155
|
+
`demos=${evo.useDemos ?? false}`,
|
|
149
156
|
];
|
|
150
157
|
if (evo.reflectionModel)
|
|
151
158
|
parts.push(`reflectionModel=${evo.reflectionModel}`);
|
|
159
|
+
if (evo.candidateSelection && evo.candidateSelection !== 'active') {
|
|
160
|
+
parts.push(`candidateSelection=${evo.candidateSelection}`);
|
|
161
|
+
}
|
|
152
162
|
return parts.join(', ');
|
|
153
163
|
}
|
|
154
164
|
//# sourceMappingURL=evolve.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"evolve.js","sourceRoot":"","sources":["../../../src/cli/evolve.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"evolve.js","sourceRoot":"","sources":["../../../src/cli/evolve.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EACL,uBAAuB,EACvB,sBAAsB,EACtB,mBAAmB,EACnB,2BAA2B,GAC5B,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAChE,OAAO,EAAE,yBAAyB,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAGtF,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,IAAc;IAChD,MAAM,SAAS,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;IAC1B,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,uFAAuF,CAAC,CAAC;IAC3G,CAAC;IAED,MAAM,KAAK,GAAG,aAAa,CAAC,SAAS,CAAC,CAAC;IACvC,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,mBAAmB,SAAS,iBAAiB,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACxG,CAAC;IAED,sEAAsE;IACtE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,yBAAyB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3E,MAAM,MAAM,GAAG,MAAM,UAAU,EAAE,CAAC;IAClC,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;IACpC,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;IAEpB,4EAA4E;IAC5E,mEAAmE;IACnE,IAAI,mBAAmB,CAAC,QAAQ,CAAC,EAAE,CAAC;QAClC,MAAM,2BAA2B,CAAC,MAAM,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;QAC/D,OAAO,CAAC,GAAG,CAAC,yCAAyC,SAAS,KAAK,gBAAgB,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IACnG,CAAC;IAED,IAAI,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;QAC/B,qEAAqE;QACrE,wEAAwE;QACxE,sEAAsE;QACtE,gCAAgC;QAChC,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;YACpB,KAAK,CAAC,SAAS,CAAC,OAAO,GAAG,IAAI,CAAC;QACjC,CAAC;QACD,MAAM,mBAAmB,CAAC,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;QACnD,OAAO,CAAC,GAAG,CAAC,kCAAkC,SAAS,EAAE,CAAC,CAAC;QAC3D,OAAO,CAAC,GAAG,CAAC,wFAAwF,CAAC,CAAC;IACxG,CAAC;SAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;QACvC,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;YACpB,KAAK,CAAC,SAAS,CAAC,OAAO,GAAG,KAAK,CAAC;QAClC,CAAC;QACD,MAAM,mBAAmB,CAAC,MAAM,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;QACpD,OAAO,CAAC,GAAG,CAAC,mCAAmC,SAAS,EAAE,CAAC,CAAC;IAC9D,CAAC;SAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,QAAQ,EAAE,CAAC;QACtC,KAAK,CAAC,cAAc,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC;QACvC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC;QAChC,KAAK,CAAC,mBAAmB,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QACzC,MAAM,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QAC9B,OAAO,CAAC,GAAG,CAAC,gCAAgC,SAAS,eAAe,CAAC,CAAC;IACxE,CAAC;SAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;QACrC,0EAA0E;QAC1E,qEAAqE;QACrE,0EAA0E;QAC1E,uEAAuE;QACvE,yDAAyD;QACzD,OAAO,CAAC,GAAG,CAAC,kCAAkC,SAAS,KAAK,CAAC,CAAC;QAC9D,0EAA0E;QAC1E,uEAAuE;QACvE,+BAA+B;QAC/B,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,QAAQ,EAAE,CAAC;QACtC,MAAM,iBAAiB,GAAG,sBAAsB,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QAC/D,MAAM,cAAc,GAAoB,iBAAiB;YACvD,CAAC,CAAC,EAAE,GAAG,KAAK,EAAE,SAAS,EAAE,iBAAiB,EAAE;YAC5C,CAAC,CAAC,KAAK,CAAC;QACV,MAAM,IAAI,GAAG,kBAAkB,CAAC,cAAc,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;QAChE,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;QACpD,IAAI,SAAS,CAAC,aAAa,EAAE,CAAC;YAC5B,OAAO,CAAC,GAAG,CAAC,qBAAqB,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC;QACxD,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,YAAY,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC;QAC/C,CAAC;QACD,IAAI,SAAS,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvC,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAC;YAC3C,KAAK,MAAM,CAAC,IAAI,SAAS,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;gBACpD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;YAC/C,CAAC;QACH,CAAC;IACH,CAAC;SAAM,IAAI,CAAC,mBAAmB,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1C,wEAAwE;QACxE,2DAA2D;QAC3D,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,QAAQ,EAAE,CAAC;QACtC,MAAM,OAAO,GAAG,KAAK,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC;QACxD,MAAM,IAAI,GAAG,KAAK,CAAC,gBAAgB,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QACpD,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACxC,wEAAwE;QACxE,2EAA2E;QAC3E,MAAM,OAAO,GAAG,uBAAuB,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACtD,MAAM,GAAG,GAAG,sBAAsB,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QAEjD,OAAO,CAAC,GAAG,CAAC,4BAA4B,SAAS,GAAG,CAAC,CAAC;QACtD,OAAO,CAAC,GAAG,CAAC,gBAAgB,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACtD,OAAO,CAAC,GAAG,CAAC,gBAAgB,OAAO,EAAE,CAAC,CAAC;QACvC,OAAO,CAAC,GAAG,CAAC,gBAAgB,IAAI,EAAE,CAAC,CAAC;QACpC,OAAO,CAAC,GAAG,CAAC,gBAAgB,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,QAAQ,OAAO,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;QAC5F,OAAO,CAAC,GAAG,CAAC,gBAAgB,KAAK,CAAC,SAAS,EAAE,OAAO,IAAI,CAAC,EAAE,CAAC,CAAC;QAC7D,OAAO,CAAC,GAAG,CAAC,gBAAgB,cAAc,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACrD,CAAC;IAED,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;AACvB,CAAC;AAED,mFAAmF;AACnF,SAAS,gBAAgB,CAAC,CAA0B;IAClD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,CAAC,CAAC,OAAO,KAAK,SAAS;QAAE,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;IAC7D,IAAI,CAAC,CAAC,QAAQ,KAAK,SAAS;QAAE,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;IAChE,IAAI,CAAC,CAAC,UAAU,KAAK,SAAS;QAAE,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC;IACzE,IAAI,CAAC,CAAC,WAAW,KAAK,SAAS;QAAE,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;IACzE,IAAI,CAAC,CAAC,eAAe,KAAK,SAAS;QAAE,KAAK,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;IACxF,IAAI,CAAC,CAAC,QAAQ,KAAK,SAAS;QAAE,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;IAChE,IAAI,CAAC,CAAC,kBAAkB,KAAK,SAAS;QAAE,KAAK,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC,kBAAkB,EAAE,CAAC,CAAC;IACjG,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;AACxD,CAAC;AAED,6EAA6E;AAC7E,SAAS,cAAc,CAAC,GAAgC;IACtD,IAAI,CAAC,GAAG;QAAE,OAAO,uBAAuB,CAAC;IACzC,MAAM,KAAK,GAAG;QACZ,QAAQ,GAAG,CAAC,OAAO,IAAI,KAAK,EAAE;QAC9B,SAAS,GAAG,CAAC,QAAQ,IAAI,KAAK,EAAE;QAChC,cAAc,GAAG,CAAC,UAAU,IAAI,KAAK,EAAE;QACvC,YAAY,GAAG,CAAC,WAAW,IAAI,KAAK,EAAE;QACtC,SAAS,GAAG,CAAC,QAAQ,IAAI,KAAK,EAAE;KACjC,CAAC;IACF,IAAI,GAAG,CAAC,eAAe;QAAE,KAAK,CAAC,IAAI,CAAC,mBAAmB,GAAG,CAAC,eAAe,EAAE,CAAC,CAAC;IAC9E,IAAI,GAAG,CAAC,kBAAkB,IAAI,GAAG,CAAC,kBAAkB,KAAK,QAAQ,EAAE,CAAC;QAClE,KAAK,CAAC,IAAI,CAAC,sBAAsB,GAAG,CAAC,kBAAkB,EAAE,CAAC,CAAC;IAC7D,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}
|
package/dist/src/cli/index.js
CHANGED
|
@@ -14,12 +14,14 @@ import { statusCommand } from './status.js';
|
|
|
14
14
|
import { evolveCommand } from './evolve.js';
|
|
15
15
|
import { initCommand } from './init.js';
|
|
16
16
|
import { createCommand } from './create.js';
|
|
17
|
+
import { canaryCommand } from './canary.js';
|
|
17
18
|
const HELP = `
|
|
18
19
|
darwin — AI agents that improve themselves.
|
|
19
20
|
|
|
20
21
|
Usage:
|
|
21
22
|
darwin run <agent> "task" Run an agent on a task
|
|
22
23
|
darwin status [agent] Show evolution status & metrics
|
|
24
|
+
darwin canary <agent> Check for behavioural drift vs a frozen baseline
|
|
23
25
|
darwin evolve <agent> Manage evolution settings
|
|
24
26
|
darwin create <name> Scaffold a new agent
|
|
25
27
|
darwin init Initialize darwin in current project
|
|
@@ -57,6 +59,8 @@ const HELP = `
|
|
|
57
59
|
--pareto-gate / --no-pareto-gate multi-objective A/B activation gate
|
|
58
60
|
--coverage / --no-coverage instance-wise coverage selection
|
|
59
61
|
--reflection-model <id> stronger reflection model for GEPA
|
|
62
|
+
--demos / --no-demos SIMBA-style demo injection (v0.10)
|
|
63
|
+
--candidate-selection <s> reflection parent: active|best|pareto|epsilon-greedy
|
|
60
64
|
`;
|
|
61
65
|
async function main() {
|
|
62
66
|
const args = process.argv.slice(2);
|
|
@@ -73,6 +77,9 @@ async function main() {
|
|
|
73
77
|
case 'status':
|
|
74
78
|
await statusCommand(args.slice(1));
|
|
75
79
|
break;
|
|
80
|
+
case 'canary':
|
|
81
|
+
await canaryCommand(args.slice(1));
|
|
82
|
+
break;
|
|
76
83
|
case 'evolve':
|
|
77
84
|
await evolveCommand(args.slice(1));
|
|
78
85
|
break;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/cli/index.ts"],"names":[],"mappings":";AACA;;;;;;;;;GASG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C,MAAM,IAAI,GAAG
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/cli/index.ts"],"names":[],"mappings":";AACA;;;;;;;;;GASG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C,MAAM,IAAI,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA8CZ,CAAC;AAEF,KAAK,UAAU,IAAI;IACjB,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAEnC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACxE,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAClB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;IAExB,IAAI,CAAC;QACH,QAAQ,OAAO,EAAE,CAAC;YAChB,KAAK,KAAK;gBACR,MAAM,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;gBAChC,MAAM;YACR,KAAK,QAAQ;gBACX,MAAM,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;gBACnC,MAAM;YACR,KAAK,QAAQ;gBACX,MAAM,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;gBACnC,MAAM;YACR,KAAK,QAAQ;gBACX,MAAM,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;gBACnC,MAAM;YACR,KAAK,QAAQ;gBACX,MAAM,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;gBACnC,MAAM;YACR,KAAK,MAAM;gBACT,MAAM,WAAW,EAAE,CAAC;gBACpB,MAAM;YACR;gBACE,OAAO,CAAC,KAAK,CAAC,oBAAoB,OAAO,EAAE,CAAC,CAAC;gBAC7C,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBAClB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACpB,CAAC;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,KAAK,CAAC,qBAAsB,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;QAC7D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC;AAED,IAAI,EAAE,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/cli/run.ts"],"names":[],"mappings":"AAAA;;;;GAIG;
|
|
1
|
+
{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/cli/run.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAmKH,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAoB9D"}
|
package/dist/src/cli/run.js
CHANGED
|
@@ -11,6 +11,7 @@ import { runMultiCritic, getCriticPrompts } from '../evolution/multi-critic.js';
|
|
|
11
11
|
import { createProvider } from '../providers/index.js';
|
|
12
12
|
import { resolveEvolutionEnabled } from '../evolution/enabled-state.js';
|
|
13
13
|
import { parseCriticScore } from '../evolution/parse-score.js';
|
|
14
|
+
import { checkCriticFamilies, crossFamilyRequired, singleFamilyHint } from '../evolution/critic-families.js';
|
|
14
15
|
import { resolveEvolutionConfig } from '../evolution/enabled-state.js';
|
|
15
16
|
import { buildEvolutionLoop } from '../evolution/build-loop.js';
|
|
16
17
|
import { isEvolutionConfigFlag, applyEvolutionFlag } from './evolution-flags.js';
|
|
@@ -29,7 +30,7 @@ function resolveCriticProviders(agentName) {
|
|
|
29
30
|
const defaults = {};
|
|
30
31
|
// Initialize all critics with CLI default
|
|
31
32
|
for (const { name } of prompts) {
|
|
32
|
-
defaults[name] = { model: 'claude-sonnet-4-6', label: 'claude-cli' };
|
|
33
|
+
defaults[name] = { model: 'claude-sonnet-4-6', label: 'claude-cli', family: 'anthropic' };
|
|
33
34
|
}
|
|
34
35
|
const criticNames = prompts.map(p => p.name);
|
|
35
36
|
// GPT-5.4 for first critic (model diversity — different training, different biases)
|
|
@@ -40,6 +41,7 @@ function resolveCriticProviders(agentName) {
|
|
|
40
41
|
provider: openaiProvider,
|
|
41
42
|
model: 'gpt-5.4',
|
|
42
43
|
label: 'openai/gpt-5.4',
|
|
44
|
+
family: 'openai',
|
|
43
45
|
};
|
|
44
46
|
}
|
|
45
47
|
catch {
|
|
@@ -54,6 +56,7 @@ function resolveCriticProviders(agentName) {
|
|
|
54
56
|
provider: anthropicProvider,
|
|
55
57
|
model: 'claude-sonnet-4-6',
|
|
56
58
|
label: 'anthropic-api',
|
|
59
|
+
family: 'anthropic',
|
|
57
60
|
};
|
|
58
61
|
}
|
|
59
62
|
catch {
|
|
@@ -253,6 +256,18 @@ async function runCommandInner(flags, agent, config, memory) {
|
|
|
253
256
|
if (evaluatorName === 'multi-critic') {
|
|
254
257
|
// ── Multi-Critic Mode — Multi-Model ───────────
|
|
255
258
|
const criticProviders = resolveCriticProviders(agent.name);
|
|
259
|
+
// Cross-family bias check: if every critic collapsed onto a single model
|
|
260
|
+
// family (e.g. only a Claude key present, so claude-cli + anthropic-api),
|
|
261
|
+
// the LLM-as-judge diversity guarantee is gone. Warn by default; hard-fail
|
|
262
|
+
// when DARWIN_REQUIRE_CROSS_FAMILY is set (CI / strict setups).
|
|
263
|
+
const familyCheck = checkCriticFamilies(criticProviders);
|
|
264
|
+
if (familyCheck.singleFamily) {
|
|
265
|
+
const hint = singleFamilyHint(familyCheck);
|
|
266
|
+
if (crossFamilyRequired()) {
|
|
267
|
+
throw new Error(`[darwin] DARWIN_REQUIRE_CROSS_FAMILY is set. ${hint}`);
|
|
268
|
+
}
|
|
269
|
+
console.warn(`[darwin] ⚠ ${hint}`);
|
|
270
|
+
}
|
|
256
271
|
const providerLabels = Object.entries(criticProviders)
|
|
257
272
|
.map(([name, info]) => `${name}→${info.label}`)
|
|
258
273
|
.join(', ');
|