darwin-agents 0.9.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +105 -0
- package/README.md +87 -0
- package/dist/src/cli/evolution-flags.d.ts +4 -0
- package/dist/src/cli/evolution-flags.d.ts.map +1 -1
- package/dist/src/cli/evolution-flags.js +69 -1
- package/dist/src/cli/evolution-flags.js.map +1 -1
- package/dist/src/cli/evolve.d.ts +4 -0
- package/dist/src/cli/evolve.d.ts.map +1 -1
- package/dist/src/cli/evolve.js +21 -0
- package/dist/src/cli/evolve.js.map +1 -1
- package/dist/src/cli/index.js +2 -0
- package/dist/src/cli/index.js.map +1 -1
- package/dist/src/evolution/demos.d.ts +128 -0
- package/dist/src/evolution/demos.d.ts.map +1 -0
- package/dist/src/evolution/demos.js +196 -0
- package/dist/src/evolution/demos.js.map +1 -0
- package/dist/src/evolution/enabled-state.d.ts +5 -5
- package/dist/src/evolution/enabled-state.d.ts.map +1 -1
- package/dist/src/evolution/enabled-state.js +17 -5
- package/dist/src/evolution/enabled-state.js.map +1 -1
- package/dist/src/evolution/feedback-filter.d.ts +54 -0
- package/dist/src/evolution/feedback-filter.d.ts.map +1 -0
- package/dist/src/evolution/feedback-filter.js +65 -0
- package/dist/src/evolution/feedback-filter.js.map +1 -0
- package/dist/src/evolution/loop.d.ts +70 -0
- package/dist/src/evolution/loop.d.ts.map +1 -1
- package/dist/src/evolution/loop.js +198 -20
- package/dist/src/evolution/loop.js.map +1 -1
- package/dist/src/evolution/selection.d.ts +73 -0
- package/dist/src/evolution/selection.d.ts.map +1 -0
- package/dist/src/evolution/selection.js +121 -0
- package/dist/src/evolution/selection.js.map +1 -0
- package/dist/src/index.d.ts +3 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +15 -0
- package/dist/src/index.js.map +1 -1
- package/dist/src/types.d.ts +132 -6
- package/dist/src/types.d.ts.map +1 -1
- package/dist/src/types.js.map +1 -1
- package/package.json +4 -1
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Demo injection (v0.10.0) — SIMBA-style "append a demo" challenger source.
|
|
3
|
+
*
|
|
4
|
+
* DSPy's SIMBA optimizer (Stochastic Introspective Mini-Batch Ascent) improves
|
|
5
|
+
* programs through two complementary strategies: appending self-reflective
|
|
6
|
+
* RULES (Darwin's reflector already covers that ground) and appending
|
|
7
|
+
* successful past examples as DEMONSTRATIONS. This module is Darwin's
|
|
8
|
+
* adaptation of the second strategy to the online evolution loop:
|
|
9
|
+
*
|
|
10
|
+
* 1. Harvest the agent's own highest-scoring past runs from memory
|
|
11
|
+
* (score ≥ threshold, successful, non-trivial output).
|
|
12
|
+
* 2. Prefer diversity: at most one demo per task type, best-scoring first.
|
|
13
|
+
* 3. Render them as a clearly-delimited "Demonstrations" section and
|
|
14
|
+
* append it to (or refresh it inside) the current prompt.
|
|
15
|
+
*
|
|
16
|
+
* The demo-augmented prompt is a CHALLENGER like any other — it goes through
|
|
17
|
+
* the same alignment guard, the same A/B test, and the same safety gate as a
|
|
18
|
+
* reflective or merged mutation. If demos don't actually help this agent, the
|
|
19
|
+
* incumbent wins and nothing changes.
|
|
20
|
+
*
|
|
21
|
+
* Design notes:
|
|
22
|
+
* - Zero LLM cost. Selection and rendering are pure string/array work on
|
|
23
|
+
* data Darwin already persists (task, output, score, taskType). This makes
|
|
24
|
+
* demos the cheapest challenger source in the loop.
|
|
25
|
+
* - Idempotent via markers. The section is wrapped in
|
|
26
|
+
* `<!-- darwin:demos:start/end -->` comments so a later demo cycle
|
|
27
|
+
* REPLACES the previous section instead of stacking a second one, and
|
|
28
|
+
* {@link stripDemoSection} can remove it cleanly.
|
|
29
|
+
* - Demo text comes from the agent's OWN prior outputs (already produced
|
|
30
|
+
* under this prompt lineage), not from an external input channel. Excerpts
|
|
31
|
+
* are length-capped and fenced, but callers embedding third-party data in
|
|
32
|
+
* tasks should be aware demos quote past tasks verbatim.
|
|
33
|
+
* - Determinism. Given the same experiment list and options, selection and
|
|
34
|
+
* rendering are fully deterministic (stable sort, no randomness) — the
|
|
35
|
+
* loop's no-op check ("would this produce the same prompt?") stays cheap.
|
|
36
|
+
*/
|
|
37
|
+
// ─── Defaults & markers ─────────────────────────────────────────────────────
|
|
38
|
+
export const DEFAULT_MAX_DEMOS = 2;
|
|
39
|
+
export const DEFAULT_DEMO_SCORE_THRESHOLD = 8;
|
|
40
|
+
export const DEFAULT_DEMO_MIN_OUTPUT_CHARS = 200;
|
|
41
|
+
export const DEFAULT_DEMO_TASK_CHARS = 240;
|
|
42
|
+
export const DEFAULT_DEMO_OUTPUT_CHARS = 900;
|
|
43
|
+
/** Start marker — everything between the markers is Darwin-managed. */
|
|
44
|
+
export const DEMO_SECTION_START = "<!-- darwin:demos:start -->";
|
|
45
|
+
/** End marker. */
|
|
46
|
+
export const DEMO_SECTION_END = "<!-- darwin:demos:end -->";
|
|
47
|
+
const DEMO_SECTION_INTRO = "## Demonstrations\n" +
|
|
48
|
+
"The following are excerpts from this agent's own highest-rated past " +
|
|
49
|
+
"responses. Match their quality, depth and structure — do NOT copy their " +
|
|
50
|
+
"content or treat their subject matter as part of the current task.";
|
|
51
|
+
// ─── Selection ──────────────────────────────────────────────────────────────
|
|
52
|
+
/** Clamp helper: finite number ≥ min, else fallback. */
|
|
53
|
+
function clampOption(value, min, fallback) {
|
|
54
|
+
return typeof value === "number" && Number.isFinite(value) && value >= min
|
|
55
|
+
? Math.floor(value)
|
|
56
|
+
: fallback;
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Harvest demo candidates from an agent's experiment history.
|
|
60
|
+
*
|
|
61
|
+
* Filter: successful runs with a critic score ≥ `scoreThreshold` and an
|
|
62
|
+
* output of at least `minOutputChars`. Diversity: at most ONE demo per task
|
|
63
|
+
* type (the best-scoring run of that type, recency as tie-break) so two demos
|
|
64
|
+
* never showcase the same narrow strength. Order: score descending, then
|
|
65
|
+
* most-recent first — fully deterministic.
|
|
66
|
+
*/
|
|
67
|
+
export function selectDemoCandidates(experiments, options = {}) {
|
|
68
|
+
const maxDemos = clampOption(options.maxDemos, 1, DEFAULT_MAX_DEMOS);
|
|
69
|
+
const threshold = typeof options.scoreThreshold === "number" && Number.isFinite(options.scoreThreshold)
|
|
70
|
+
? options.scoreThreshold
|
|
71
|
+
: DEFAULT_DEMO_SCORE_THRESHOLD;
|
|
72
|
+
const minChars = clampOption(options.minOutputChars, 0, DEFAULT_DEMO_MIN_OUTPUT_CHARS);
|
|
73
|
+
const qualified = experiments.filter((exp) => {
|
|
74
|
+
if (!exp.success)
|
|
75
|
+
return false;
|
|
76
|
+
const score = exp.feedback?.score;
|
|
77
|
+
if (typeof score !== "number" || !Number.isFinite(score) || score < threshold)
|
|
78
|
+
return false;
|
|
79
|
+
const output = exp.output;
|
|
80
|
+
if (typeof output !== "string" || output.trim().length < minChars)
|
|
81
|
+
return false;
|
|
82
|
+
return typeof exp.task === "string" && exp.task.trim().length > 0;
|
|
83
|
+
});
|
|
84
|
+
// Best run per task type (score desc, then completedAt desc as tie-break).
|
|
85
|
+
const bestPerType = new Map();
|
|
86
|
+
for (const exp of qualified) {
|
|
87
|
+
const key = exp.taskType || "general";
|
|
88
|
+
const incumbent = bestPerType.get(key);
|
|
89
|
+
if (incumbent === undefined) {
|
|
90
|
+
bestPerType.set(key, exp);
|
|
91
|
+
continue;
|
|
92
|
+
}
|
|
93
|
+
const expScore = exp.feedback.score;
|
|
94
|
+
const incScore = incumbent.feedback.score;
|
|
95
|
+
if (expScore > incScore ||
|
|
96
|
+
(expScore === incScore && exp.completedAt > incumbent.completedAt)) {
|
|
97
|
+
bestPerType.set(key, exp);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
const picked = [...bestPerType.values()].sort((a, b) => {
|
|
101
|
+
const scoreDelta = b.feedback.score - a.feedback.score;
|
|
102
|
+
if (scoreDelta !== 0)
|
|
103
|
+
return scoreDelta;
|
|
104
|
+
// Recency tie-break; ISO timestamps sort lexicographically.
|
|
105
|
+
if (a.completedAt < b.completedAt)
|
|
106
|
+
return 1;
|
|
107
|
+
if (a.completedAt > b.completedAt)
|
|
108
|
+
return -1;
|
|
109
|
+
return 0;
|
|
110
|
+
});
|
|
111
|
+
return picked.slice(0, maxDemos).map((exp) => ({
|
|
112
|
+
task: exp.task.trim(),
|
|
113
|
+
output: exp.output.trim(),
|
|
114
|
+
score: exp.feedback.score,
|
|
115
|
+
taskType: exp.taskType || "general",
|
|
116
|
+
}));
|
|
117
|
+
}
|
|
118
|
+
// ─── Rendering ──────────────────────────────────────────────────────────────
|
|
119
|
+
/**
|
|
120
|
+
* Strip the demo-section marker literals out of quoted demo text. A past
|
|
121
|
+
* output that itself contains `<!-- darwin:demos:end -->` (e.g. an agent
|
|
122
|
+
* writing ABOUT Darwin) would otherwise plant an early end-marker inside the
|
|
123
|
+
* section — the next refresh cycle's lazy marker-to-marker match would then
|
|
124
|
+
* cut at the wrong span and leave half the old section dangling in the prompt.
|
|
125
|
+
*/
|
|
126
|
+
function stripMarkerLiterals(text) {
|
|
127
|
+
return text.split(DEMO_SECTION_START).join("").split(DEMO_SECTION_END).join("");
|
|
128
|
+
}
|
|
129
|
+
/** Truncate at the last sentence/newline boundary within `maxLength`. */
|
|
130
|
+
function truncateAtSentenceBoundary(text, maxLength) {
|
|
131
|
+
if (text.length <= maxLength)
|
|
132
|
+
return text;
|
|
133
|
+
const truncated = text.slice(0, maxLength);
|
|
134
|
+
const lastPeriod = truncated.lastIndexOf(".");
|
|
135
|
+
const lastNewline = truncated.lastIndexOf("\n");
|
|
136
|
+
const cutPoint = Math.max(lastPeriod, lastNewline);
|
|
137
|
+
// Only honour the boundary when it keeps a meaningful chunk (> 40%).
|
|
138
|
+
const kept = cutPoint > maxLength * 0.4 ? truncated.slice(0, cutPoint + 1) : truncated;
|
|
139
|
+
return `${kept.trimEnd()} [...]`;
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* Render demo candidates as a marker-delimited prompt section.
|
|
143
|
+
*
|
|
144
|
+
* Returns an empty string for an empty demo list so callers can treat
|
|
145
|
+
* "nothing to inject" uniformly.
|
|
146
|
+
*/
|
|
147
|
+
export function buildDemoSection(demos, options = {}) {
|
|
148
|
+
if (demos.length === 0)
|
|
149
|
+
return "";
|
|
150
|
+
const maxTask = clampOption(options.maxTaskChars, 20, DEFAULT_DEMO_TASK_CHARS);
|
|
151
|
+
const maxOutput = clampOption(options.maxOutputChars, 80, DEFAULT_DEMO_OUTPUT_CHARS);
|
|
152
|
+
const blocks = demos.map((demo, i) => {
|
|
153
|
+
const task = truncateAtSentenceBoundary(stripMarkerLiterals(demo.task), maxTask);
|
|
154
|
+
const excerpt = truncateAtSentenceBoundary(stripMarkerLiterals(demo.output), maxOutput);
|
|
155
|
+
return (`### Demonstration ${i + 1} — task type "${demo.taskType}" (scored ${demo.score}/10)\n` +
|
|
156
|
+
`Task: ${task}\n` +
|
|
157
|
+
`Response excerpt:\n"""\n${excerpt}\n"""`);
|
|
158
|
+
});
|
|
159
|
+
return [DEMO_SECTION_START, DEMO_SECTION_INTRO, ...blocks, DEMO_SECTION_END].join("\n\n");
|
|
160
|
+
}
|
|
161
|
+
// ─── Application ────────────────────────────────────────────────────────────
|
|
162
|
+
/** Regex matching one marker-delimited demo section (including markers). */
|
|
163
|
+
const DEMO_SECTION_PATTERN = new RegExp(`${DEMO_SECTION_START.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}[\\s\\S]*?${DEMO_SECTION_END.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}`);
|
|
164
|
+
/**
|
|
165
|
+
* Apply a rendered demo section to a prompt.
|
|
166
|
+
*
|
|
167
|
+
* If the prompt already contains a marker-delimited demo section it is
|
|
168
|
+
* REPLACED in place (demos stay fresh, never stack); otherwise the section is
|
|
169
|
+
* appended after a blank line. An empty `section` returns the prompt with any
|
|
170
|
+
* existing demo section stripped — passing "no demos" acts as removal.
|
|
171
|
+
*
|
|
172
|
+
* The replacement uses a FUNCTION replacer: with a string replacer,
|
|
173
|
+
* `$&` / `$'` / `` $` `` / `$$` inside the section (an agent output quoting
|
|
174
|
+
* JS-regex or shell docs contains exactly these) would be interpreted as
|
|
175
|
+
* replacement patterns and garble the prompt on every refresh cycle.
|
|
176
|
+
*/
|
|
177
|
+
export function applyDemoSection(prompt, section) {
|
|
178
|
+
if (section === "")
|
|
179
|
+
return stripDemoSection(prompt);
|
|
180
|
+
if (DEMO_SECTION_PATTERN.test(prompt)) {
|
|
181
|
+
return prompt.replace(DEMO_SECTION_PATTERN, () => section);
|
|
182
|
+
}
|
|
183
|
+
const trimmed = prompt.replace(/\s+$/, "");
|
|
184
|
+
return `${trimmed}\n\n${section}`;
|
|
185
|
+
}
|
|
186
|
+
/**
|
|
187
|
+
* Remove the marker-delimited demo section (if any), collapsing the blank
|
|
188
|
+
* line that {@link applyDemoSection} inserted. Prompts without a section are
|
|
189
|
+
* returned unchanged.
|
|
190
|
+
*/
|
|
191
|
+
export function stripDemoSection(prompt) {
|
|
192
|
+
if (!DEMO_SECTION_PATTERN.test(prompt))
|
|
193
|
+
return prompt;
|
|
194
|
+
return prompt.replace(DEMO_SECTION_PATTERN, "").replace(/\n{3,}/g, "\n\n").replace(/\s+$/, "");
|
|
195
|
+
}
|
|
196
|
+
//# sourceMappingURL=demos.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"demos.js","sourceRoot":"","sources":["../../../src/evolution/demos.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AAqDH,+EAA+E;AAE/E,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC;AACnC,MAAM,CAAC,MAAM,4BAA4B,GAAG,CAAC,CAAC;AAC9C,MAAM,CAAC,MAAM,6BAA6B,GAAG,GAAG,CAAC;AACjD,MAAM,CAAC,MAAM,uBAAuB,GAAG,GAAG,CAAC;AAC3C,MAAM,CAAC,MAAM,yBAAyB,GAAG,GAAG,CAAC;AAE7C,uEAAuE;AACvE,MAAM,CAAC,MAAM,kBAAkB,GAAG,6BAA6B,CAAC;AAChE,kBAAkB;AAClB,MAAM,CAAC,MAAM,gBAAgB,GAAG,2BAA2B,CAAC;AAE5D,MAAM,kBAAkB,GACtB,qBAAqB;IACrB,sEAAsE;IACtE,0EAA0E;IAC1E,oEAAoE,CAAC;AAEvE,+EAA+E;AAE/E,wDAAwD;AACxD,SAAS,WAAW,CAAC,KAAc,EAAE,GAAW,EAAE,QAAgB;IAChE,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,GAAG;QACxE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC;QACnB,CAAC,CAAC,QAAQ,CAAC;AACf,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,oBAAoB,CAClC,WAA4C,EAC5C,UAAgC,EAAE;IAElC,MAAM,QAAQ,GAAG,WAAW,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,EAAE,iBAAiB,CAAC,CAAC;IACrE,MAAM,SAAS,GACb,OAAO,OAAO,CAAC,cAAc,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC;QACnF,CAAC,CAAC,OAAO,CAAC,cAAc;QACxB,CAAC,CAAC,4BAA4B,CAAC;IACnC,MAAM,QAAQ,GAAG,WAAW,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC,EAAE,6BAA6B,CAAC,CAAC;IAEvF,MAAM,SAAS,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE;QAC3C,IAAI,CAAC,GAAG,CAAC,OAAO;YAAE,OAAO,KAAK,CAAC;QAC/B,MAAM,KAAK,GAAG,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC;QAClC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,SAAS;YAAE,OAAO,KAAK,CAAC;QAC5F,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC;QAC1B,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,QAAQ;YAAE,OAAO,KAAK,CAAC;QAChF,OAAO,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;IACpE,CAAC,CAAC,CAAC;IAEH,2EAA2E;IAC3E,MAAM,WAAW,GAAG,IAAI,GAAG,EAA4B,CAAC;IACxD,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;QAC5B,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,IAAI,SAAS,CAAC;QACtC,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACvC,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;YAC5B,WAAW,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;YAC1B,SAAS;QACX,CAAC;QACD,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAS,CAAC,KAAK,CAAC;QACrC,MAAM,QAAQ,GAAG,SAAS,CAAC,QAAS,CAAC,KAAK,CAAC;QAC3C,IACE,QAAQ,GAAG,QAAQ;YACnB,CAAC,QAAQ,KAAK,QAAQ,IAAI,GAAG,CAAC,WAAW,GAAG,SAAS,CAAC,WAAW,CAAC,EAClE,CAAC;YACD,WAAW,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACrD,MAAM,UAAU,GAAG,CAAC,CAAC,QAAS,CAAC,KAAK,GAAG,CAAC,CAAC,QAAS,CAAC,KAAK,CAAC;QACzD,IAAI,UAAU,KAAK,CAAC;YAAE,OAAO,UAAU,CAAC;QACxC,4DAA4D;QAC5D,IAAI,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,WAAW;YAAE,OAAO,CAAC,CAAC;QAC5C,IAAI,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,WAAW;YAAE,OAAO,CAAC,CAAC,CAAC;QAC7C,OAAO,CAAC,CAAC;IACX,CAAC,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QAC7C,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE;QACrB,MAAM,EAAE,GAAG,CAAC,MAAO,CAAC,IAAI,EAAE;QAC1B,KAAK,EAAE,GAAG,CAAC,QAAS,CAAC,KAAK;QAC1B,QAAQ,EAAE,GAAG,CAAC,QAAQ,IAAI,SAAS;KACpC,CAAC,CAAC,CAAC;AACN,CAAC;AAED,+EAA+E;AAE/E;;;;;;GAMG;AACH,SAAS,mBAAmB,CAAC,IAAY;IACvC,OAAO,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;AAClF,CAAC;AAED,yEAAyE;AACzE,SAAS,0BAA0B,CAAC,IAAY,EAAE,SAAiB;IACjE,IAAI,IAAI,CAAC,MAAM,IAAI,SAAS;QAAE,OAAO,IAAI,CAAC;IAC1C,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;IAC3C,MAAM,UAAU,GAAG,SAAS,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAC9C,MAAM,WAAW,GAAG,SAAS,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;IAChD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;IACnD,qEAAqE;IACrE,MAAM,IAAI,GAAG,QAAQ,GAAG,SAAS,GAAG,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACvF,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC;AACnC,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,gBAAgB,CAC9B,KAAmC,EACnC,UAA6B,EAAE;IAE/B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAClC,MAAM,OAAO,GAAG,WAAW,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,EAAE,uBAAuB,CAAC,CAAC;IAC/E,MAAM,SAAS,GAAG,WAAW,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,EAAE,yBAAyB,CAAC,CAAC;IAErF,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE;QACnC,MAAM,IAAI,GAAG,0BAA0B,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;QACjF,MAAM,OAAO,GAAG,0BAA0B,CAAC,mBAAmB,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,CAAC,CAAC;QACxF,OAAO,CACL,qBAAqB,CAAC,GAAG,CAAC,iBAAiB,IAAI,CAAC,QAAQ,aAAa,IAAI,CAAC,KAAK,QAAQ;YACvF,SAAS,IAAI,IAAI;YACjB,2BAA2B,OAAO,OAAO,CAC1C,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,OAAO,CAAC,kBAAkB,EAAE,kBAAkB,EAAE,GAAG,MAAM,EAAE,gBAAgB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC5F,CAAC;AAED,+EAA+E;AAE/E,4EAA4E;AAC5E,MAAM,oBAAoB,GAAG,IAAI,MAAM,CACrC,GAAG,kBAAkB,CAAC,OAAO,CAAC,qBAAqB,EAAE,MAAM,CAAC,aAAa,gBAAgB,CAAC,OAAO,CAAC,qBAAqB,EAAE,MAAM,CAAC,EAAE,CACnI,CAAC;AAEF;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,gBAAgB,CAAC,MAAc,EAAE,OAAe;IAC9D,IAAI,OAAO,KAAK,EAAE;QAAE,OAAO,gBAAgB,CAAC,MAAM,CAAC,CAAC;IACpD,IAAI,oBAAoB,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;QACtC,OAAO,MAAM,CAAC,OAAO,CAAC,oBAAoB,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,CAAC;IAC7D,CAAC;IACD,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAC3C,OAAO,GAAG,OAAO,OAAO,OAAO,EAAE,CAAC;AACpC,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,gBAAgB,CAAC,MAAc;IAC7C,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,MAAM,CAAC;QAAE,OAAO,MAAM,CAAC;IACtD,OAAO,MAAM,CAAC,OAAO,CAAC,oBAAoB,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;AACjG,CAAC"}
|
|
@@ -39,11 +39,11 @@ export declare function setEvolutionEnabled(memory: MemoryProvider, agentName: s
|
|
|
39
39
|
* ({@link DarwinState.evolutionConfigOverrides}) merged ON TOP, then any
|
|
40
40
|
* in-this-process CLI override merged last (e.g. `darwin run … --gepa`).
|
|
41
41
|
*
|
|
42
|
-
* Only the
|
|
43
|
-
* (`enabled`, `metrics`, `minRuns`, …) is taken verbatim from the
|
|
44
|
-
* config. Returns `undefined` when the agent has no `evolution` block
|
|
45
|
-
* persisted/CLI override would create one — overriding a flag on an
|
|
46
|
-
* never opted into evolution is meaningless.
|
|
42
|
+
* Only the advanced flags in {@link OVERRIDE_KEYS} are overridable; every
|
|
43
|
+
* other field (`enabled`, `metrics`, `minRuns`, …) is taken verbatim from the
|
|
44
|
+
* static config. Returns `undefined` when the agent has no `evolution` block
|
|
45
|
+
* AND no persisted/CLI override would create one — overriding a flag on an
|
|
46
|
+
* agent that never opted into evolution is meaningless.
|
|
47
47
|
*
|
|
48
48
|
* Read defensively: undefined override values are skipped so they never clobber
|
|
49
49
|
* a static default with `undefined`.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"enabled-state.d.ts","sourceRoot":"","sources":["../../../src/evolution/enabled-state.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,EACV,eAAe,EACf,WAAW,EACX,eAAe,EACf,uBAAuB,EACvB,cAAc,EACf,MAAM,aAAa,CAAC;AAErB;;;;;;;;;;;;GAYG;AACH,wBAAgB,uBAAuB,CACrC,KAAK,EAAE,eAAe,EACtB,KAAK,EAAE,IAAI,CAAC,WAAW,EAAE,kBAAkB,CAAC,GAC3C,OAAO,CAUT;AAED;;;;;;GAMG;AACH,wBAAsB,mBAAmB,CACvC,MAAM,EAAE,cAAc,EACtB,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,OAAO,GACf,OAAO,CAAC,IAAI,CAAC,CAQf;
|
|
1
|
+
{"version":3,"file":"enabled-state.d.ts","sourceRoot":"","sources":["../../../src/evolution/enabled-state.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,EACV,eAAe,EACf,WAAW,EACX,eAAe,EACf,uBAAuB,EACvB,cAAc,EACf,MAAM,aAAa,CAAC;AAErB;;;;;;;;;;;;GAYG;AACH,wBAAgB,uBAAuB,CACrC,KAAK,EAAE,eAAe,EACtB,KAAK,EAAE,IAAI,CAAC,WAAW,EAAE,kBAAkB,CAAC,GAC3C,OAAO,CAUT;AAED;;;;;;GAMG;AACH,wBAAsB,mBAAmB,CACvC,MAAM,EAAE,cAAc,EACtB,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,OAAO,GACf,OAAO,CAAC,IAAI,CAAC,CAQf;AAiBD;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,sBAAsB,CACpC,KAAK,EAAE,eAAe,EACtB,KAAK,EAAE,IAAI,CAAC,WAAW,EAAE,0BAA0B,CAAC,EACpD,WAAW,CAAC,EAAE,uBAAuB,GACpC,eAAe,GAAG,SAAS,CAoC7B;AAED;;;;;GAKG;AACH,wBAAsB,2BAA2B,CAC/C,MAAM,EAAE,cAAc,EACtB,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,uBAAuB,GAC/B,OAAO,CAAC,IAAI,CAAC,CASf"}
|
|
@@ -58,6 +58,10 @@ const OVERRIDE_KEYS = [
|
|
|
58
58
|
'paretoGate',
|
|
59
59
|
'useCoverage',
|
|
60
60
|
'reflectionModel',
|
|
61
|
+
'useDemos',
|
|
62
|
+
'candidateSelection',
|
|
63
|
+
'skipPerfectFeedback',
|
|
64
|
+
'maxMergeInvocations',
|
|
61
65
|
];
|
|
62
66
|
/**
|
|
63
67
|
* Resolve the effective {@link EvolutionConfig} for an agent: the static
|
|
@@ -65,11 +69,11 @@ const OVERRIDE_KEYS = [
|
|
|
65
69
|
* ({@link DarwinState.evolutionConfigOverrides}) merged ON TOP, then any
|
|
66
70
|
* in-this-process CLI override merged last (e.g. `darwin run … --gepa`).
|
|
67
71
|
*
|
|
68
|
-
* Only the
|
|
69
|
-
* (`enabled`, `metrics`, `minRuns`, …) is taken verbatim from the
|
|
70
|
-
* config. Returns `undefined` when the agent has no `evolution` block
|
|
71
|
-
* persisted/CLI override would create one — overriding a flag on an
|
|
72
|
-
* never opted into evolution is meaningless.
|
|
72
|
+
* Only the advanced flags in {@link OVERRIDE_KEYS} are overridable; every
|
|
73
|
+
* other field (`enabled`, `metrics`, `minRuns`, …) is taken verbatim from the
|
|
74
|
+
* static config. Returns `undefined` when the agent has no `evolution` block
|
|
75
|
+
* AND no persisted/CLI override would create one — overriding a flag on an
|
|
76
|
+
* agent that never opted into evolution is meaningless.
|
|
73
77
|
*
|
|
74
78
|
* Read defensively: undefined override values are skipped so they never clobber
|
|
75
79
|
* a static default with `undefined`.
|
|
@@ -95,7 +99,15 @@ export function resolveEvolutionConfig(agent, state, cliOverride) {
|
|
|
95
99
|
if (key === 'reflectionModel') {
|
|
96
100
|
resolved.reflectionModel = value;
|
|
97
101
|
}
|
|
102
|
+
else if (key === 'candidateSelection') {
|
|
103
|
+
resolved.candidateSelection = value;
|
|
104
|
+
}
|
|
105
|
+
else if (key === 'maxMergeInvocations') {
|
|
106
|
+
resolved.maxMergeInvocations = value;
|
|
107
|
+
}
|
|
98
108
|
else {
|
|
109
|
+
// Remaining override keys are all boolean flags (useGepa / useMerge /
|
|
110
|
+
// paretoGate / useCoverage / useDemos / skipPerfectFeedback).
|
|
99
111
|
resolved[key] = value;
|
|
100
112
|
}
|
|
101
113
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"enabled-state.js","sourceRoot":"","sources":["../../../src/evolution/enabled-state.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAUH;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,uBAAuB,CACrC,KAAsB,EACtB,KAA4C;IAE5C,MAAM,SAAS,GAAG,KAAK,CAAC,gBAAgB,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACvD,IAAI,OAAO,SAAS,KAAK,SAAS,EAAE,CAAC;QACnC,0EAA0E;QAC1E,wEAAwE;QACxE,yEAAyE;QACzE,2EAA2E;QAC3E,OAAO,SAAS,IAAI,KAAK,CAAC,SAAS,KAAK,SAAS,CAAC;IACpD,CAAC;IACD,OAAO,KAAK,CAAC,SAAS,EAAE,OAAO,IAAI,KAAK,CAAC;AAC3C,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,MAAsB,EACtB,SAAiB,EACjB,OAAgB;IAEhB,MAAM,MAAM,CAAC,WAAW,CAAC,CAAC,KAAK,EAAE,EAAE;QACjC,IAAI,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC;YAC5B,KAAK,CAAC,gBAAgB,GAAG,EAAE,CAAC;QAC9B,CAAC;QACD,KAAK,CAAC,gBAAgB,CAAC,SAAS,CAAC,GAAG,OAAO,CAAC;QAC5C,OAAO,KAAK,CAAC;IACf,CAAC,CAAC,CAAC;AACL,CAAC;AAED,uEAAuE;AAEvE,yDAAyD;AACzD,MAAM,aAAa,GAAG;IACpB,SAAS;IACT,UAAU;IACV,YAAY;IACZ,aAAa;IACb,iBAAiB;
|
|
1
|
+
{"version":3,"file":"enabled-state.js","sourceRoot":"","sources":["../../../src/evolution/enabled-state.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAUH;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,uBAAuB,CACrC,KAAsB,EACtB,KAA4C;IAE5C,MAAM,SAAS,GAAG,KAAK,CAAC,gBAAgB,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACvD,IAAI,OAAO,SAAS,KAAK,SAAS,EAAE,CAAC;QACnC,0EAA0E;QAC1E,wEAAwE;QACxE,yEAAyE;QACzE,2EAA2E;QAC3E,OAAO,SAAS,IAAI,KAAK,CAAC,SAAS,KAAK,SAAS,CAAC;IACpD,CAAC;IACD,OAAO,KAAK,CAAC,SAAS,EAAE,OAAO,IAAI,KAAK,CAAC;AAC3C,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,MAAsB,EACtB,SAAiB,EACjB,OAAgB;IAEhB,MAAM,MAAM,CAAC,WAAW,CAAC,CAAC,KAAK,EAAE,EAAE;QACjC,IAAI,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC;YAC5B,KAAK,CAAC,gBAAgB,GAAG,EAAE,CAAC;QAC9B,CAAC;QACD,KAAK,CAAC,gBAAgB,CAAC,SAAS,CAAC,GAAG,OAAO,CAAC;QAC5C,OAAO,KAAK,CAAC;IACf,CAAC,CAAC,CAAC;AACL,CAAC;AAED,uEAAuE;AAEvE,yDAAyD;AACzD,MAAM,aAAa,GAAG;IACpB,SAAS;IACT,UAAU;IACV,YAAY;IACZ,aAAa;IACb,iBAAiB;IACjB,UAAU;IACV,oBAAoB;IACpB,qBAAqB;IACrB,qBAAqB;CACb,CAAC;AAEX;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,sBAAsB,CACpC,KAAsB,EACtB,KAAoD,EACpD,WAAqC;IAErC,MAAM,SAAS,GAAG,KAAK,CAAC,wBAAwB,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC/D,MAAM,IAAI,GAAG,KAAK,CAAC,SAAS,CAAC;IAE7B,IAAI,CAAC,IAAI,IAAI,CAAC,SAAS,IAAI,CAAC,WAAW,EAAE,CAAC;QACxC,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,2EAA2E;IAC3E,2EAA2E;IAC3E,6EAA6E;IAC7E,MAAM,QAAQ,GAAoB,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;IAE1E,KAAK,MAAM,KAAK,IAAI,CAAC,SAAS,EAAE,WAAW,CAAC,EAAE,CAAC;QAC7C,IAAI,CAAC,KAAK;YAAE,SAAS;QACrB,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;YAChC,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC;YACzB,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;gBACxB,oEAAoE;gBACpE,kCAAkC;gBAClC,IAAI,GAAG,KAAK,iBAAiB,EAAE,CAAC;oBAC9B,QAAQ,CAAC,eAAe,GAAG,KAAe,CAAC;gBAC7C,CAAC;qBAAM,IAAI,GAAG,KAAK,oBAAoB,EAAE,CAAC;oBACxC,QAAQ,CAAC,kBAAkB,GAAG,KAA8C,CAAC;gBAC/E,CAAC;qBAAM,IAAI,GAAG,KAAK,qBAAqB,EAAE,CAAC;oBACzC,QAAQ,CAAC,mBAAmB,GAAG,KAAe,CAAC;gBACjD,CAAC;qBAAM,CAAC;oBACN,sEAAsE;oBACtE,8DAA8D;oBAC9D,QAAQ,CAAC,GAAG,CAAC,GAAG,KAAgB,CAAC;gBACnC,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAC/C,MAAsB,EACtB,SAAiB,EACjB,OAAgC;IAEhC,MAAM,MAAM,CAAC,WAAW,CAAC,CAAC,KAAK,EAAE,EAAE;QACjC,IAAI,CAAC,KAAK,CAAC,wBAAwB,EAAE,CAAC;YACpC,KAAK,CAAC,wBAAwB,GAAG,EAAE,CAAC;QACtC,CAAC;QACD,MAAM,QAAQ,GAAG,KAAK,CAAC,wBAAwB,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;QACjE,KAAK,CAAC,wBAAwB,CAAC,SAAS,CAAC,GAAG,EAAE,GAAG,QAAQ,EAAE,GAAG,OAAO,EAAE,CAAC;QACxE,OAAO,KAAK,CAAC;IACf,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Perfect-score feedback filtering (v0.11.0) — adapted from GEPA's
|
|
3
|
+
* `skip_perfect_score` for Darwin's online evolution loop.
|
|
4
|
+
*
|
|
5
|
+
* What upstream actually does (gepa-ai `reflective_mutation.py`): after
|
|
6
|
+
* sampling a minibatch and scoring the candidate on it, GEPA skips the WHOLE
|
|
7
|
+
* reflection iteration when EVERY minibatch score is perfect — it does not drop
|
|
8
|
+
* individual instances, and a perfect instance inside a mixed minibatch is kept
|
|
9
|
+
* (positive contrast for the reflector).
|
|
10
|
+
*
|
|
11
|
+
* Darwin generalizes this to per-report filtering, which upstream can't do
|
|
12
|
+
* cheaply but Darwin can: the critic scores on real runs are already paid for,
|
|
13
|
+
* so there is no eval-budget reason to keep perfect ("nothing to fix") reports
|
|
14
|
+
* in the pool the reflector / legacy optimizer learns from. Dropping them
|
|
15
|
+
* concentrates the feedback on runs that actually went wrong. (The perfect runs
|
|
16
|
+
* are not wasted — with `useDemos` they are harvested as demonstrations, so
|
|
17
|
+
* failures feed reflection and successes feed demos.)
|
|
18
|
+
*
|
|
19
|
+
* Purity: no I/O, no LLM calls, deterministic. The predicate is generic over
|
|
20
|
+
* any `{ score: number }` so both feedback assemblers in the loop
|
|
21
|
+
* (`getReflectiveFeedback` → `ReflectiveFeedback`, `getRecentFeedback` →
|
|
22
|
+
* built-inline) share one source of truth and cannot drift. The loop filters
|
|
23
|
+
* inline during the feedback-window fill (so skipped items don't count toward
|
|
24
|
+
* the window `limit`); `filterPerfectFeedback` is exported as a convenience for
|
|
25
|
+
* consumers assembling their own feedback lists.
|
|
26
|
+
*/
|
|
27
|
+
/**
|
|
28
|
+
* Critic scores are on the 1–10 scale (see `parse-score.ts`). The default
|
|
29
|
+
* perfect threshold is a literal top score — only 10/10 is skipped unless the
|
|
30
|
+
* caller lowers it.
|
|
31
|
+
*/
|
|
32
|
+
export declare const DEFAULT_PERFECT_FEEDBACK_SCORE = 10;
|
|
33
|
+
/**
|
|
34
|
+
* Resolve a perfect-score threshold to a usable value: finite and within the
|
|
35
|
+
* critic 1–10 scale, else the default. A non-finite / out-of-range config
|
|
36
|
+
* value (hand-edited file, `NaN`) must never silently disable or invert the
|
|
37
|
+
* filter — it falls back to {@link DEFAULT_PERFECT_FEEDBACK_SCORE}.
|
|
38
|
+
*/
|
|
39
|
+
export declare function resolvePerfectFeedbackScore(value: number | undefined): number;
|
|
40
|
+
/**
|
|
41
|
+
* True when a run's critic score is at or above the perfect threshold and so
|
|
42
|
+
* carries no improvement gradient. A non-finite score (missing / unparsed) is
|
|
43
|
+
* never "perfect" — it is kept so a genuinely broken run still surfaces to the
|
|
44
|
+
* optimizer rather than being silently dropped as if it were flawless.
|
|
45
|
+
*/
|
|
46
|
+
export declare function isPerfectScore(score: number, threshold: number): boolean;
|
|
47
|
+
/**
|
|
48
|
+
* Drop perfect-score items from a feedback list. Generic over `{ score }` so
|
|
49
|
+
* the exact feedback shape is the caller's; order is preserved.
|
|
50
|
+
*/
|
|
51
|
+
export declare function filterPerfectFeedback<T extends {
|
|
52
|
+
score: number;
|
|
53
|
+
}>(items: readonly T[], threshold: number): T[];
|
|
54
|
+
//# sourceMappingURL=feedback-filter.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"feedback-filter.d.ts","sourceRoot":"","sources":["../../../src/evolution/feedback-filter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH;;;;GAIG;AACH,eAAO,MAAM,8BAA8B,KAAK,CAAC;AAEjD;;;;;GAKG;AACH,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,CAM7E;AAED;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAGxE;AAED;;;GAGG;AACH,wBAAgB,qBAAqB,CAAC,CAAC,SAAS;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,EAC/D,KAAK,EAAE,SAAS,CAAC,EAAE,EACnB,SAAS,EAAE,MAAM,GAChB,CAAC,EAAE,CAEL"}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Perfect-score feedback filtering (v0.11.0) — adapted from GEPA's
|
|
3
|
+
* `skip_perfect_score` for Darwin's online evolution loop.
|
|
4
|
+
*
|
|
5
|
+
* What upstream actually does (gepa-ai `reflective_mutation.py`): after
|
|
6
|
+
* sampling a minibatch and scoring the candidate on it, GEPA skips the WHOLE
|
|
7
|
+
* reflection iteration when EVERY minibatch score is perfect — it does not drop
|
|
8
|
+
* individual instances, and a perfect instance inside a mixed minibatch is kept
|
|
9
|
+
* (positive contrast for the reflector).
|
|
10
|
+
*
|
|
11
|
+
* Darwin generalizes this to per-report filtering, which upstream can't do
|
|
12
|
+
* cheaply but Darwin can: the critic scores on real runs are already paid for,
|
|
13
|
+
* so there is no eval-budget reason to keep perfect ("nothing to fix") reports
|
|
14
|
+
* in the pool the reflector / legacy optimizer learns from. Dropping them
|
|
15
|
+
* concentrates the feedback on runs that actually went wrong. (The perfect runs
|
|
16
|
+
* are not wasted — with `useDemos` they are harvested as demonstrations, so
|
|
17
|
+
* failures feed reflection and successes feed demos.)
|
|
18
|
+
*
|
|
19
|
+
* Purity: no I/O, no LLM calls, deterministic. The predicate is generic over
|
|
20
|
+
* any `{ score: number }` so both feedback assemblers in the loop
|
|
21
|
+
* (`getReflectiveFeedback` → `ReflectiveFeedback`, `getRecentFeedback` →
|
|
22
|
+
* built-inline) share one source of truth and cannot drift. The loop filters
|
|
23
|
+
* inline during the feedback-window fill (so skipped items don't count toward
|
|
24
|
+
* the window `limit`); `filterPerfectFeedback` is exported as a convenience for
|
|
25
|
+
* consumers assembling their own feedback lists.
|
|
26
|
+
*/
|
|
27
|
+
/**
|
|
28
|
+
* Critic scores are on the 1–10 scale (see `parse-score.ts`). The default
|
|
29
|
+
* perfect threshold is a literal top score — only 10/10 is skipped unless the
|
|
30
|
+
* caller lowers it.
|
|
31
|
+
*/
|
|
32
|
+
export const DEFAULT_PERFECT_FEEDBACK_SCORE = 10;
|
|
33
|
+
/**
|
|
34
|
+
* Resolve a perfect-score threshold to a usable value: finite and within the
|
|
35
|
+
* critic 1–10 scale, else the default. A non-finite / out-of-range config
|
|
36
|
+
* value (hand-edited file, `NaN`) must never silently disable or invert the
|
|
37
|
+
* filter — it falls back to {@link DEFAULT_PERFECT_FEEDBACK_SCORE}.
|
|
38
|
+
*/
|
|
39
|
+
export function resolvePerfectFeedbackScore(value) {
|
|
40
|
+
if (typeof value !== 'number' || !Number.isFinite(value)) {
|
|
41
|
+
return DEFAULT_PERFECT_FEEDBACK_SCORE;
|
|
42
|
+
}
|
|
43
|
+
if (value < 1 || value > 10)
|
|
44
|
+
return DEFAULT_PERFECT_FEEDBACK_SCORE;
|
|
45
|
+
return value;
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* True when a run's critic score is at or above the perfect threshold and so
|
|
49
|
+
* carries no improvement gradient. A non-finite score (missing / unparsed) is
|
|
50
|
+
* never "perfect" — it is kept so a genuinely broken run still surfaces to the
|
|
51
|
+
* optimizer rather than being silently dropped as if it were flawless.
|
|
52
|
+
*/
|
|
53
|
+
export function isPerfectScore(score, threshold) {
|
|
54
|
+
if (!Number.isFinite(score))
|
|
55
|
+
return false;
|
|
56
|
+
return score >= threshold;
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Drop perfect-score items from a feedback list. Generic over `{ score }` so
|
|
60
|
+
* the exact feedback shape is the caller's; order is preserved.
|
|
61
|
+
*/
|
|
62
|
+
export function filterPerfectFeedback(items, threshold) {
|
|
63
|
+
return items.filter((item) => !isPerfectScore(item.score, threshold));
|
|
64
|
+
}
|
|
65
|
+
//# sourceMappingURL=feedback-filter.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"feedback-filter.js","sourceRoot":"","sources":["../../../src/evolution/feedback-filter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH;;;;GAIG;AACH,MAAM,CAAC,MAAM,8BAA8B,GAAG,EAAE,CAAC;AAEjD;;;;;GAKG;AACH,MAAM,UAAU,2BAA2B,CAAC,KAAyB;IACnE,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QACzD,OAAO,8BAA8B,CAAC;IACxC,CAAC;IACD,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,EAAE;QAAE,OAAO,8BAA8B,CAAC;IACnE,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,cAAc,CAAC,KAAa,EAAE,SAAiB;IAC7D,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAC1C,OAAO,KAAK,IAAI,SAAS,CAAC;AAC5B,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,qBAAqB,CACnC,KAAmB,EACnB,SAAiB;IAEjB,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC;AACxE,CAAC"}
|
|
@@ -58,6 +58,13 @@ interface DarwinLoopDeps {
|
|
|
58
58
|
* (only used when `embed` is set). Default 0.82.
|
|
59
59
|
*/
|
|
60
60
|
alignmentSimilarityThreshold?: number;
|
|
61
|
+
/**
|
|
62
|
+
* v0.10.0 — Random source in [0, 1) for the stochastic parent-selection
|
|
63
|
+
* strategies (`candidateSelection: 'pareto' | 'epsilon-greedy'`). Injected
|
|
64
|
+
* for deterministic tests; default `Math.random`. Never consulted on the
|
|
65
|
+
* default `'active'` path.
|
|
66
|
+
*/
|
|
67
|
+
rng?: () => number;
|
|
61
68
|
}
|
|
62
69
|
export declare class DarwinLoop {
|
|
63
70
|
private memory;
|
|
@@ -70,6 +77,7 @@ export declare class DarwinLoop {
|
|
|
70
77
|
private gepa?;
|
|
71
78
|
private embed?;
|
|
72
79
|
private alignmentSimilarityThreshold?;
|
|
80
|
+
private rng?;
|
|
73
81
|
constructor(deps: DarwinLoopDeps);
|
|
74
82
|
/**
|
|
75
83
|
* Called AFTER every agent run. Drives the evolution cycle.
|
|
@@ -140,6 +148,12 @@ export declare class DarwinLoop {
|
|
|
140
148
|
* from loadExperiments(), so we just filter for ones with feedback.
|
|
141
149
|
*/
|
|
142
150
|
private getRecentFeedback;
|
|
151
|
+
/**
|
|
152
|
+
* v0.11.0 — Resolved perfect-score threshold for {@link
|
|
153
|
+
* EvolutionConfig.skipPerfectFeedback} (finite, within the critic 1–10
|
|
154
|
+
* scale, else default 10).
|
|
155
|
+
*/
|
|
156
|
+
private perfectFeedbackScore;
|
|
143
157
|
/**
|
|
144
158
|
* v0.6.0 — Generate the next prompt variant via the GEPA reflective path.
|
|
145
159
|
*
|
|
@@ -173,6 +187,62 @@ export declare class DarwinLoop {
|
|
|
173
187
|
* v0.7.0 — Merge cadence (every K-th cycle). Default 3, clamped ≥ 1.
|
|
174
188
|
*/
|
|
175
189
|
private mergeEveryK;
|
|
190
|
+
/**
|
|
191
|
+
* v0.11.0 — Resolved lifetime merge cap (GEPA `max_merge_invocations`), or
|
|
192
|
+
* `null` when uncapped. A non-finite / negative value is treated as "no cap"
|
|
193
|
+
* rather than silently disabling merge; a non-integer cap is floored (mirrors
|
|
194
|
+
* `mergeEveryK`) so a hand-edited `2.5` behaves as 2, not 3.
|
|
195
|
+
*/
|
|
196
|
+
private mergeCap;
|
|
197
|
+
/**
|
|
198
|
+
* v0.11.0 — Lifetime merge budget check (GEPA `max_merge_invocations`).
|
|
199
|
+
* Returns `true` (merge may fire) when no cap is configured, or when the
|
|
200
|
+
* per-agent count of merge-derived challengers is still below the cap.
|
|
201
|
+
*/
|
|
202
|
+
private mergeBudgetAvailable;
|
|
203
|
+
/**
|
|
204
|
+
* v0.11.0 — Increment this agent's lifetime merge-invocation count. Called
|
|
205
|
+
* once per merge challenger that passes the alignment guard and is carried
|
|
206
|
+
* into an A/B test, and ONLY when a cap is configured — an uncapped useMerge
|
|
207
|
+
* agent never writes the counter, so its persisted state is unchanged from
|
|
208
|
+
* v0.10. Initialises the map lazily for state rows that predate the field.
|
|
209
|
+
*
|
|
210
|
+
* The check (`mergeBudgetAvailable`) and this increment are separate state
|
|
211
|
+
* reads, so two cycles running concurrently for the SAME agent could each
|
|
212
|
+
* observe `used = cap-1` and overshoot by one. The overshoot is bounded by
|
|
213
|
+
* the concurrency and sits inside the pre-existing concurrent-A/B-start
|
|
214
|
+
* envelope (afterRun's "an A/B test is already running" guard is likewise
|
|
215
|
+
* check-then-act) — acceptable for a soft lifetime budget, not a hard limit.
|
|
216
|
+
*/
|
|
217
|
+
private recordMergeInvocation;
|
|
218
|
+
/**
|
|
219
|
+
* v0.10.0 — Demo-injection cadence (every K-th cycle). Default 4, clamped
|
|
220
|
+
* ≥ 1. Deliberately offset from the merge default (3) so the two
|
|
221
|
+
* non-reflective challenger sources don't collide on the same cycles.
|
|
222
|
+
*/
|
|
223
|
+
private demoEveryK;
|
|
224
|
+
/**
|
|
225
|
+
* v0.10.0 — Build a SIMBA-style demo challenger: current prompt + a
|
|
226
|
+
* marker-delimited "Demonstrations" section harvested from this agent's
|
|
227
|
+
* highest-scoring past runs. Pure selection + rendering — no LLM call.
|
|
228
|
+
*
|
|
229
|
+
* Returns `null` (→ caller falls through to GEPA/legacy generation) when
|
|
230
|
+
* no run qualifies (score/threshold/length filters) or when the rendered
|
|
231
|
+
* demo set is byte-identical to what the prompt already carries — an A/B
|
|
232
|
+
* test of a version against itself would be pointless.
|
|
233
|
+
*/
|
|
234
|
+
private tryDemoVariant;
|
|
235
|
+
/**
|
|
236
|
+
* v0.10.0 — Scored version history for parent selection AND merge: every
|
|
237
|
+
* prompt version that has text and at least one run's worth of averaged
|
|
238
|
+
* metrics, as {@link ScoredVariant}s. One getAverageMetrics
|
|
239
|
+
* (→ loadExperiments) call per version. O(N) backend round-trips, but N is
|
|
240
|
+
* the agent's prompt-version count (≤ ~20 in practice) and this only runs
|
|
241
|
+
* on selection/merge cadences, so it is cheap — same pattern handleABTest
|
|
242
|
+
* already uses for the Pareto gate. (Extracted from tryMergeVariant,
|
|
243
|
+
* behaviour unchanged.)
|
|
244
|
+
*/
|
|
245
|
+
private buildScoredHistory;
|
|
176
246
|
/**
|
|
177
247
|
* v0.7.0 — GEPA system-aware MERGE (paper Appendix-D). Builds scored
|
|
178
248
|
* variants from this agent's prompt-version history (each version's prompt
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"loop.d.ts","sourceRoot":"","sources":["../../../src/evolution/loop.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,EAEV,eAAe,EACf,gBAAgB,EAChB,aAAa,EACb,cAAc,EAGf,MAAM,aAAa,CAAC;AACrB,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AACtD,OAAO,KAAK,EAAE,eAAe,EAAoB,MAAM,gBAAgB,CAAC;AACxE,OAAO,KAAK,EAAE,UAAU,EAAiB,MAAM,aAAa,CAAC;AAC7D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAE7D,OAAO,KAAK,EAAE,aAAa,EAAiB,MAAM,qBAAqB,CAAC;AAGxE,OAAO,EAAkE,KAAK,OAAO,EAAE,MAAM,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"loop.d.ts","sourceRoot":"","sources":["../../../src/evolution/loop.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,EAEV,eAAe,EACf,gBAAgB,EAChB,aAAa,EACb,cAAc,EAGf,MAAM,aAAa,CAAC;AACrB,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AACtD,OAAO,KAAK,EAAE,eAAe,EAAoB,MAAM,gBAAgB,CAAC;AACxE,OAAO,KAAK,EAAE,UAAU,EAAiB,MAAM,aAAa,CAAC;AAC7D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAE7D,OAAO,KAAK,EAAE,aAAa,EAAiB,MAAM,qBAAqB,CAAC;AAGxE,OAAO,EAAkE,KAAK,OAAO,EAAE,MAAM,gBAAgB,CAAC;AAQ9G,MAAM,WAAW,eAAe;IAC9B,aAAa,EAAE,aAAa,EAAE,CAAC;IAC/B,aAAa,EAAE,OAAO,CAAC;IACvB,aAAa,EAAE,OAAO,CAAC;IACvB,eAAe,EAAE,OAAO,CAAC;IACzB,UAAU,EAAE,OAAO,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;CACjB;AAID,UAAU,cAAc;IACtB,MAAM,EAAE,cAAc,CAAC;IACvB,OAAO,EAAE,iBAAiB,CAAC;IAC3B,SAAS,EAAE,eAAe,CAAC;IAC3B,MAAM,EAAE,UAAU,CAAC;IACnB,QAAQ,EAAE,eAAe,CAAC;IAC1B,gEAAgE;IAChE,KAAK,CAAC,EAAE,eAAe,CAAC;IACxB,8EAA8E;IAC9E,aAAa,CAAC,EAAE,kBAAkB,CAAC;IACnC;;;;;;OAMG;IACH,IAAI,CAAC,EAAE,aAAa,CAAC;IACrB;;;;;;;OAOG;IACH,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB;;;OAGG;IACH,4BAA4B,CAAC,EAAE,MAAM,CAAC;IACtC;;;;;OAKG;IACH,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;CACpB;AAYD,qBAAa,UAAU;IACrB,OAAO,CAAC,MAAM,CAAiB;IAC/B,OAAO,CAAC,OAAO,CAAoB;IACnC,OAAO,CAAC,SAAS,CAAkB;IACnC,OAAO,CAAC,MAAM,CAAa;IAC3B,OAAO,CAAC,QAAQ,CAAkB;IAClC,OAAO,CAAC,KAAK,CAAC,CAAkB;IAChC,OAAO,CAAC,aAAa,CAAqB;IAC1C,OAAO,CAAC,IAAI,CAAC,CAAgB;IAC7B,OAAO,CAAC,KAAK,CAAC,CAAU;IACxB,OAAO,CAAC,4BAA4B,CAAC,CAAS;IAC9C,OAAO,CAAC,GAAG,CAAC,CAAe;gBAEf,IAAI,EAAE,cAAc;IAchC;;;;;;;;;;OAUG;IACG,QAAQ,CAAC,UAAU,EAAE,gBAAgB,GAAG,OAAO,CAAC,eAAe,CAAC;IAuJtE;;;;;;;;;;;;;;;;;;OAkBG;IACG,WAAW,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IAqC9D;;;;;;OAMG;YACW,sBAAsB;YA8ItB,YAAY;IAwJ1B;;;OAGG;YACW,QAAQ;IA8BtB;;OAEG;YACW,eAAe;IAe7B;;;;OAIG;IACH,OAAO,CAAC,cAAc;IAMtB;;;OAGG;IACH,OAAO,CAAC,UAAU;IAKlB;;;;;;OAMG;YACW,iBAAiB;IAqB/B;;;;OAIG;IACH,OAAO,CAAC,oBAAoB;IAI5B;;;;;;;;;;;;;;;;;OAiBG;YACW,mBAAmB;IAyHjC;;;;;;;;OAQG;YACW,iBAAiB;IAa/B;;OAEG;IACH,OAAO,CAAC,WAAW;IAMnB;;;;;OAKG;IACH,OAAO,CAAC,QAAQ;IAMhB;;;;OAIG;YACW,oBAAoB;IAQlC;;;;;;;;;;;;;OAaG;YACW,qBAAqB;IASnC;;;;OAIG;IACH,OAAO,CAAC,UAAU;IAMlB;;;;;;;;;OASG;YACW,cAAc;IAa5B;;;;;;;;;OASG;YACW,kBAAkB;IAYhC;;;;;;;;;;;;OAYG;YACW,eAAe;IAgC7B;;;;;;;;;;;;;;;;OAgBG;YACW,oBAAoB;IAyBlC;;;;;;OAMG;YACW,qBAAqB;IA4BnC;;OAEG;IACH,OAAO,CAAC,WAAW;IAWnB;;;;;;OAMG;IACH,OAAO,CAAC,eAAe;IAKvB;;;OAGG;YACW,mBAAmB;IAiDjC;;OAEG;IACH,OAAO,CAAC,iBAAiB;CAsB1B"}
|