@tangle-network/agent-eval 0.51.0 → 0.53.0

package/dist/{run-improvement-loop-BPMjNKMJ.d.ts → run-improvement-loop-Cc7oZlRP.d.ts}

@@ -79,25 +79,48 @@ declare function evolutionaryDriver<TFindings = unknown>(opts: EvolutionaryDrive
  *
  * `gepaDriver` — a reflective `ImprovementDriver` for prompt-tier surfaces.
  * Each generation it reflects on the prior best candidate's per-scenario
- * scores + weakest dimensions (the `GenerationCandidate` evidence from
- * `runOptimization`), asks an LLM to propose targeted rewrites of the current
- * surface, and returns them as the next population.
+ * scores + weakest dimensions, asks an LLM to propose targeted rewrites of
+ * the current surface, and returns them as the next population.
  *
- * This is the substrate's best-in-class prompt optimizer: surface-agnostic, so
- * ANY string surface in ANY consumer opts in by selecting it — system prompts,
- * prompt addenda, judge/reviewer prompts, even a driver's own reflection
- * prompt. It reuses the generic reflection primitive (`buildReflectionPrompt` /
- * `parseReflectionResponse`) and the router client; it has NO dependency on the
- * legacy `runMultiShotOptimization` / `prompt-evolution` orchestration.
+ * Honest scope vs the GEPA paper (Agrawal et al., arXiv:2507.19457):
+ * this driver implements the *reflection* primitive — it does NOT implement
+ * GEPA's Pareto frontier of candidates, multi-objective non-dominated
+ * tracking, or the combine-complementary-lessons step. We use "best by
+ * composite" as the parent each generation; the paper retains a Pareto set
+ * and combines lessons across non-dominated candidates. Tracked as #101 in
+ * the substrate roadmap. See `docs/specs/driver-honest-spec.md`.
  *
- * It earns its keep where there is real per-instance signal (which the
+ * Optional `constraints` move structured-doc guards into the driver
+ * (preserve H2 section headings, cap sentence-level edits) — useful when
+ * the surface IS a structured procedure like a SKILL.md / runbook /
+ * judge rubric. When `constraints` is omitted, behavior is unchanged.
+ *
+ * The driver is surface-agnostic — any string surface in any consumer opts
+ * in by selecting it. Reuses the generic reflection primitive
+ * (`buildReflectionPrompt` / `parseReflectionResponse`) and the router
+ * client; no dependency on the legacy `runMultiShotOptimization` /
+ * `prompt-evolution` orchestration.
+ *
+ * Earns its keep where there is real per-instance signal (which the
  * dimensional + per-scenario evidence + the `LabeledScenarioStore` flywheel
- * now provide). For thin-signal surfaces it degrades to plain reflection — so
- * it is a SELECTABLE driver, never a forced default. On generation 0 (no
- * history) it reflects on the current surface against the mutation primitives
- * alone.
+ * now provide). For thin-signal surfaces it degrades to plain reflection.
+ * On generation 0 (no history) it reflects on the current surface against
+ * the mutation primitives alone.
  */
 
+interface GepaDriverConstraints {
+    /** H2 section headings that MUST appear unchanged in every candidate.
+     *  When set, the driver auto-detects current H2s if this is empty AND
+     *  rejects any candidate that drops or renames a preserved heading.
+     *  Use when the surface is a structured doc (SKILL.md, runbook,
+     *  sectioned system prompt, judge rubric). */
+    preserveSections?: string[];
+    /** Maximum sentence-level edits per candidate vs the parent surface.
+     *  Rejection threshold = maxSentenceEdits × 2 (counts adds + removes).
+     *  Inspired by SkillOpt's edit-budget as a "textual learning rate."
+     *  Cap prevents an LLM rewrite from overwriting useful prior rules. */
+    maxSentenceEdits?: number;
+}
 interface GepaDriverOptions {
     /** Router transport (apiKey/baseUrl). */
     llm: LlmClientOptions;
@@ -113,8 +136,18 @@ interface GepaDriverOptions {
     temperature?: number;
     /** Reflection max tokens. Default 6000. */
     maxTokens?: number;
+    /** Structured-doc constraints. Candidates violating any are rejected
+     *  post-parse and dropped from the returned population. */
+    constraints?: GepaDriverConstraints;
 }
 declare function gepaDriver(opts: GepaDriverOptions): ImprovementDriver;
+/** Extract H2 headings (`## Foo`) from a markdown surface. Exported for
+ *  consumers building custom mutators that share the same invariant. */
+declare function extractH2Sections(text: string): string[];
+/** Sentence-level edit distance — count distinct add/remove ops between
+ *  two surfaces via a normalised line-by-line set diff. Treats trivial
+ *  whitespace as identical. Exported for tests + consumer-side validators. */
+declare function countSentenceEdits(baseline: string, candidate: string): number;
 
 /**
  * @experimental
@@ -414,4 +447,4 @@ interface RunImprovementLoopResult<TArtifact, TScenario extends Scenario> extend
 }
 declare function runImprovementLoop<TScenario extends Scenario, TArtifact>(opts: RunImprovementLoopOptions<TScenario, TArtifact>): Promise<RunImprovementLoopResult<TArtifact, TScenario>>;
 
-export { type CampaignStorage as C, type DefaultProductionGateOptions as D, type EvolutionaryDriverOptions as E, type GepaDriverOptions as G, type HeldOutGateOptions as H, type OpenAutoPrOptions as O, type RunCampaignOptions as R, type OpenAutoPrResult as a, type RunEvalOptions as b, type RunImprovementLoopOptions as c, type RunImprovementLoopResult as d, type RunOptimizationOptions as e, type RunOptimizationResult as f, composeGate as g, defaultProductionGate as h, evolutionaryDriver as i, fsCampaignStorage as j, gepaDriver as k, heldOutGate as l, inMemoryCampaignStorage as m, runEval as n, openAutoPr as o, runImprovementLoop as p, runOptimization as q, runCampaign as r, surfaceHash as s };
+export { type CampaignStorage as C, type DefaultProductionGateOptions as D, type EvolutionaryDriverOptions as E, type GepaDriverConstraints as G, type HeldOutGateOptions as H, type OpenAutoPrOptions as O, type RunCampaignOptions as R, type GepaDriverOptions as a, type OpenAutoPrResult as b, type RunEvalOptions as c, type RunImprovementLoopOptions as d, type RunImprovementLoopResult as e, type RunOptimizationOptions as f, type RunOptimizationResult as g, composeGate as h, countSentenceEdits as i, defaultProductionGate as j, evolutionaryDriver as k, extractH2Sections as l, fsCampaignStorage as m, gepaDriver as n, heldOutGate as o, inMemoryCampaignStorage as p, openAutoPr as q, runCampaign as r, runEval as s, runImprovementLoop as t, runOptimization as u, surfaceHash as v };

package/docs/design/self-improvement-protocol.md

@@ -0,0 +1,223 @@
+# Self-improvement protocol — the world-class architecture
+
+**Status:** Strategic design. The artifact that every roadmap entry maps to.
+**Date:** 2026-05-27.
+
+## Thesis
+
+**Self-improvement is a protocol, not a product.** We define the wire formats, surface abstractions, driver interface, gate interface, and insight format. We ship reference implementations. Customers plug in whatever framework, model, or runtime they already use — our infrastructure handles the rigorous middle (analysis, gating, version-safe deployment).
+
+No competitor ships this combination. LangSmith / Braintrust / Phoenix / LangFuse ship tracing. Hermes ships an agent. SkillOpt ships an academic optimizer. Anthropic's Claude Code ships skill-creation. **Nobody ships the protocol.**
+
+## The pipeline as a single abstract flow
+
+```
+┌──────────────────────────────────────────────────────────────────────┐
+│  WHATEVER YOU ALREADY USE                                            │
+│  LangChain · LlamaIndex · Anthropic SDK · OpenAI Assistants ·        │
+│  Hermes · Claude Code · Codex · agent-runtime · your own stack       │
+└─────────────────────────────────┬────────────────────────────────────┘
+                                  │ traces (any format)
+                                  ▼
+┌──────────────────────────────────────────────────────────────────────┐
+│  INGEST — universal trace adapters                                   │
+│  fromOtelSpans · fromFeedbackTable · fromLangChain · fromLlamaIndex ·│
+│  fromAnthropicSDK · fromOpenAISDK · fromHermesProfileLog · BYO       │
+│  → canonical RunRecord[]                                             │
+└─────────────────────────────────┬────────────────────────────────────┘
+                                  ▼
+┌──────────────────────────────────────────────────────────────────────┐
+│  ANALYZE — analyzeRuns({ runs, baselineRuns?, userFeedback? })       │
+│  paired-bootstrap CI · Pareto · failure clusters · prior-period      │
+│  delta · user-corrective-signal extraction · recommendations         │
+│  ← THE STATISTICAL EDGE NOBODY ELSE SHIPS                            │
+└─────────────────────────────────┬────────────────────────────────────┘
+                                  ▼
+┌──────────────────────────────────────────────────────────────────────┐
+│  IMPROVE — selfImprove() closed loop                                 │
+│  gepaDriver · evolutionaryDriver · BYO ImprovementDriver             │
+│  → ProfileDiff (versioned, hashed, content-addressable)              │
+└─────────────────────────────────┬────────────────────────────────────┘
+                                  ▼
+┌──────────────────────────────────────────────────────────────────────┐
+│  GATE — defaultProductionGate (paired-CI) · BYO gate                 │
+│  ship-substrate / ship-harness / merge / inconclusive                │
+│  ← STATISTICALLY STRICTER THAN ANY COMPETITOR                        │
+└─────────────────────────────────┬────────────────────────────────────┘
+                                  ▼
+┌──────────────────────────────────────────────────────────────────────┐
+│  DEPLOY — back into WHATEVER YOU ALREADY USE                         │
+│  agent-runtime · Hermes profile log · LangChain config · custom hook │
+└──────────────────────────────────────────────────────────────────────┘
+```
+
+## The integration promise
+
+Customers pick one of three integration shapes. All three work today (some are aspirational on adapter coverage). Every shape uses the same canonical types underneath.
+
+### Shape A — offline analysis only
+
+You have traces, you want a decision packet. Zero LLM cost. Zero closed loop.
+
+```typescript
+import { fromOtelSpans, analyzeRuns } from '@tangle-network/agent-eval'
+
+const runs = fromOtelSpans({ spans: mySpans })
+const report = await analyzeRuns({ runs })
+// → InsightReport with composite, recommendations, Pareto, ...
+```
+
+Use case: dashboards, weekly post-mortems, "did anything regress" checks. The intelligence-kernel ships this.
+
+### Shape B — closed loop, your runtime
+
+You have an agent, you want to improve it. We provide drivers + gate + insight. You decide when to deploy.
+
+```typescript
+import { selfImprove, gepaDriver } from '@tangle-network/agent-eval'
+
+const result = await selfImprove({
+  scenarios,
+  agent: yourAgent,           // any function (surface, scenario) → artifact
+  judge: yourJudge,           // any function (artifact) → JudgeScore
+  baselineSurface,
+  driver: gepaDriver({ llm, model, target }),
+  budget: { generations: 3, populationSize: 4, holdoutFraction: 0.3 },
+})
+// → SelfImproveResult { baselineHash, diff, winningHash, lift, gateDecision, insight }
+```
+
+Use case: every product agent we ship. Hermes-on-our-sandbox. Claude Code with skills. Anyone wanting "ship if statistically better, else hold."
+
+### Shape C — hosted, cross-language
+
+You stream traces from anywhere, get InsightReports + selfImprove orchestration. Bills usage-based.
+
+```sh
+# Stream traces
+curl https://api.tangle.tools/v1/ingest/otel \
+  -H "Authorization: Bearer ${TANGLE_KEY}" \
+  --data-binary @traces.jsonl
+
+# Get the decision packet
+curl https://api.tangle.tools/v1/insight/${runId}
+
+# Or run a closed-loop campaign
+curl https://api.tangle.tools/v1/improve \
+  -d '{"scenarios": ..., "baselineHash": "...", "budget": {...}}'
+```
+
+Use case: Python customers, Go customers, customers behind firewalls, customers who don't want to operate the substrate.
+
+## The five non-negotiables
+
+The protocol claim only holds if all five of these survive integration. Customers shouldn't have to compromise on any.
+
+1. **Universal ingest.** Any trace format → canonical RunRecord. Coverage: OTel ✓, multi-rater feedback ✓, LangChain ⏳, LlamaIndex ⏳, Anthropic SDK ⏳, OpenAI Assistants ⏳, Hermes profile log ⏳.
+2. **Statistical rigor.** Every claim falsifiable. Paired bootstrap CI on lift, Cohen's d on effect size, MDE-aware sample-size recommendations, p-values. **SkillOpt's gate is literal `cand > current`. Hermes has no gate. Ours has all of the above.** This is the moat.
+3. **Plug-in everything.** Driver, judge, gate, intake adapter, storage all swappable. Customer brings their LLM, their judge, their scenarios. We bring the rigor.
+4. **Version-safe deployment.** AgentProfile is content-addressable. Two writers (harness + substrate) can both mutate without lost-update. Gate verdicts are scoped to baseline hash, not absolute. Tracked as #98.
+5. **Cross-language wire format.** Python client at parity with TypeScript. Hosted ingest spec versioned. Customers in any language consume the same shape.
+
+## Where we are honest about gaps
+
+| Component | Status | Customer impact when missing |
+|---|---|---|
+| `fromOtelSpans` ingest adapter | ✓ shipped 0.50.0 | — |
+| `fromFeedbackTable` multi-rater intake | ✓ shipped 0.50.0 | — |
+| `analyzeRuns` decision packet | ✓ shipped 0.50.0 / 0.50.2 actionability | — |
+| `selfImprove` closed loop | ✓ shipped 0.50.0 | — |
+| Paired-bootstrap gate | ✓ shipped early; still our edge | — |
+| `gepaDriver` reflection (not full Pareto — task #101) | ⚠ partial | OK; customers don't need Pareto until plateau hit |
+| **Prior-period comparison** in `analyzeRuns` | ✗ MISSING | "Did my last change help?" — the #1 customer question — has no rigorous answer today |
+| **User-corrective-feedback signal extraction** | ✗ MISSING | Hermes' first-class skill signal. We have the trace data. We don't mine it. |
+| **`init` CLI** scaffolding canonical eval/ layout | ✗ MISSING | Every new consumer wires it by hand; the skill describes 80 lines they have to copy |
+| **Framework-specific intake adapters** (LangChain, LlamaIndex, Anthropic SDK, OpenAI Assistants) | ✗ MISSING | Customers using these frameworks can't ingest without writing custom adapter code |
+| **Profile versioning** (task #98) | ✗ MISSING | Offline/online drift; gate verdicts can be stale by the time they're applied |
+| **Composite driver** (optimize all surfaces against one gate) | ✗ MISSING | Customers can optimize prompts OR skills, not both jointly |
+| **Empirical proof drivers work** | ✗ MISSING | We've never published "we ran gepaDriver on real customer data, here's the lift CI" |
+| Hosted-tier production launch | ⚠ in scaffolding (intelligence-kernel) | Customers must self-host today |
+
+## The roadmap — what closes each gap
+
+Mapping every roadmap entry back to a concrete protocol gap.
+
+### 0.53.0 (this session-or-next) — answer "did my last change help?"
+
+- **`analyzeRuns({ runs, baselineRuns? })`** — when `baselineRuns` is provided, the report includes a `priorPeriodComparison?` block: per-metric delta with paired-bootstrap CI, MDE-aware significance judgment, "regressed metrics" surfaced in `recommendations`.
+- Built on top of existing `diffRuns()` primitive (already shipped 0.48.0).
+- 1 PR. Pure additive surface.
+- **Customer impact**: this is the conversion question for every prospect.
+
+### 0.54.0 — extract Hermes' missing signal
+
+- **`extractUserCorrections(runs)`** — new substrate primitive. Mines user messages in traces for corrective markers (regex pass + LLM classifier for nuance). Returns `UserCorrectionEvent[]` keyed by runId.
+- `analyzeRuns({ runs, userFeedback? })` includes a "common corrections" cluster in `recommendations`.
+- Bridge to Hermes-style signal without adopting Hermes' runtime.
+- **Customer impact**: distinctive — no competitor mines this signal.
+
+### 0.55.0 — framework-specific intake adapters
+
+- **`fromLangChain(traces)`**, **`fromLlamaIndex(traces)`**, **`fromAnthropicSDK(traces)`**, **`fromOpenAIAssistants(traces)`**.
+- Each maps the framework's native trace shape to RunRecord.
+- Top 4 frameworks = 80% of agent-builder market coverage.
+- **Customer impact**: removes "we don't support your framework" friction.
+
+### 0.56.0 — `init` CLI + worked examples
+
+- `pnpm dlx @tangle-network/agent-eval init` scaffolds the canonical `eval/scenarios.json` + 3 pnpm scripts + judges template + `.runs/` directory.
+- Adds 5+ end-to-end runnable examples covering Shapes A/B/C across the 4 framework adapters.
+- **Customer impact**: time-to-first-eval drops from 4 hours to 5 minutes.
+
+### 1.0.0 — profile versioning (#98) + composite driver
+
+- Content-addressable `AgentProfileVersion` + `ProfileDiff` + 3-way merge + 4-way `DriftGateDecision`.
+- `compositeDriver` — optimize all surfaces of one AgentProfile against one gate.
+- Hermes-on-sandbox forcing function validates the work before commit.
+- **Customer impact**: production-safe; the moat is locked.
+
+### 1.1.0 — empirical-proof publication
+
+- Pick one named customer or one synthetic-realistic corpus (legal-agent canonical).
+- Run gepaDriver end-to-end with real LLM cost.
+- Publish: "n=, lift=, CI=, p=, $cost=, vs no-driver baseline."
+- One blog post, one demo video, one runnable repro.
+- **Customer impact**: every other claim becomes credible because this one is verified.
+
+## Why this design is 100x
+
+Not a 10% improvement over LangSmith. A category change.
+
+| Capability | LangSmith / Braintrust / Phoenix | Hermes / Claude Code | Tangle (target) |
+|---|---|---|---|
+| Trace ingest | ✓ proprietary | ✓ own runtime | ✓ universal |
+| Decision packet | ⚠ scorecards (no CI) | ✗ | ✓ paired-bootstrap |
+| Closed loop | ✗ | ✓ heuristic | ✓ statistically rigorous |
+| Plug-in drivers | ✗ | ✗ | ✓ |
+| Profile versioning | ✗ | ✗ | ✓ (1.0.0) |
+| Composite multi-surface | ✗ | ✗ | ✓ (1.0.0) |
+| Cross-language | ✗ | ✗ | ✓ (Python at parity) |
+| Empirical-proof publication | ✗ | ✗ | ✓ (1.1.0) |
+
+Eight rows. Nobody else has eight. We can be the only one. The work is named, scoped, and queued.
+
+## What's NOT on the roadmap (and why)
+
+- **Building our own agent runtime.** Hermes / agent-runtime / Claude Code cover that. We are infrastructure, not a runtime.
+- **Single-vendor LLM.** Substrate stays model-agnostic.
+- **UI-first product.** API-first. UIs are downstream.
+- **LangChain replacement.** Wrong layer.
+- **"Self-improvement" without a held-out gate.** Hermes and SkillOpt both ship this; we explicitly refuse — every selfImprove() requires a holdout.
+
+## Decision log — what we committed to in 0.52.0 → 1.0.0
+
+1. **`skillOptDriver` removed; behavior in `gepaDriver({ constraints })`** — 0.52.0 ✓ shipped
+2. **Honest spec docs** — 0.52.0 ✓ shipped
+3. **Profile-versioning spec with symmetric-fork framing** — 0.52.0 ✓ shipped
+4. **No V2 names anywhere** — enforced
+5. **Forcing-function gate on profile-versioning work** — Hermes-on-sandbox experiment required before phases 1-5 commit
+6. **Single-PR-per-repo discipline** — enforced 0.52.0 onwards
+7. **Prior-period comparison as 0.53.0** — committed; the customer-conversion primitive
+8. **User-feedback extraction as 0.54.0** — committed; the Hermes-signal bridge
+9. **Framework intake adapters as 0.55.0** — committed; 80% market coverage
+10. **Empirical-proof publication as 1.1.0** — committed; the credibility lock

package/docs/specs/driver-honest-spec.md

@@ -0,0 +1,251 @@
+# Driver Honest Spec — what each driver IS, what each methodology actually is, where we deviate
+
+**Status:** Living document. Updated when we learn the truth from primary sources.
+**Date:** 2026-05-27
+
+This document exists because the project shipped two drivers with methodology names attached (`gepaDriver`, `skillOptDriver`) without the methodology specs being precisely encoded anywhere in the repo. That created an integrity gap. This doc closes it.
+
+Every claim in this doc is sourced from a primary reference (paper, code, or directly verifiable from our source). Marketing language is forbidden. If something is not implemented we say so.
+
+---
+
+## Part 1 — GEPA (the paper)
+
+**Source**: Agrawal et al., *"GEPA: Reflective Prompt Evolution Can Outperform Reinforcement Learning"*, arXiv:2507.19457, July 2025.
+
+### What GEPA actually does
+
+Outer loop (verbatim from abstract): "samples trajectories (e.g., reasoning, tool calls, and tool outputs) and reflects on them in natural language to diagnose problems, propose and test prompt updates, and combine complementary lessons from the **Pareto frontier of its own attempts**."
+
+Named primitives in the paper:
+- **GEPA** (Genetic-Pareto) — the overall optimizer
+- **Pareto frontier** — non-dominated candidate set retained across iterations
+- **Prompt updates** — mutations proposed by reflection
+- **Rollouts** — trajectory samples
+
+### What gepaDriver in our substrate ACTUALLY does
+
+Source: `src/campaign/drivers/gepa.ts` (132 lines)
+
+- Single LLM call per `propose()` invocation
+- Input: prior generation's **single best candidate by composite score** + that candidate's top/bottom scenarios + 3 weakest dimensions (`buildEvidence`)
+- Output: N proposals, each a full document rewrite
+- Dedup by exact text equality
+
+### Deviations from the GEPA paper
+
+| GEPA paper | Our `gepaDriver` |
+|---|---|
+| **Pareto frontier** of candidates | **Single "best by composite"** — no Pareto set, no non-dominated tracking |
+| **Combine complementary lessons** from frontier | Each generation reflects on ONE prior candidate; no combination |
+| Multi-objective optimization | Single-objective (composite score) |
+| Genetic operators (mutation, crossover) | Reflection only — no crossover |
+| Sample efficiency claim (35× fewer rollouts than GRPO) | Unmeasured against any baseline |
+
+**Honest assessment**: our `gepaDriver` is a **reflective full-rewrite driver**, not GEPA. It captures GEPA's *reflection* primitive but not its *Pareto* mechanism. The name oversells. A faithful renaming would be `reflectiveRewriteDriver`. A faithful implementation would add a Pareto candidate pool + combine step.
+
+---
+
+## Part 2 — SkillOpt (the paper + code)
+
+**Source**:
+- README: https://github.com/microsoft/SkillOpt
+- Source: `/tmp/SkillOpt/skillopt/` (cloned 2026-05-27)
+- Key files: `engine/trainer.py`, `optimizer/clip.py` (rank_and_select), `optimizer/update_modes.py`, `evaluation/gate.py`, `types.py`
+
+### What SkillOpt actually does
+
+**6-stage per-step pipeline** (verbatim from `trainer.py:516` and adjacent):
+
+1. **Rollout** — `adapter.rollout(train_env, current_skill, ...)` collects trajectories on a batch.
+2. **Reflect** — `adapter.reflect()` analyses trajectories and emits **structured patches** (NOT full rewrites in patch mode). Failure trials → failure patches; success trials → success patches.
+3. **Aggregate** — `merge_patches(current_skill, all_failure_patches, all_success_patches, batch_size=merge_bs)` — hierarchically merges patches across accumulated batches.
+4. **Select** — `rank_and_select(current_skill, merged_patch, max_edits=edit_budget)` — if edit pool > budget, calls an optimizer LLM to **rank edits by importance** and keep top-L. Budget is "analogous to gradient clipping" (their words).
+5. **Update** — apply patch in one of 3 modes:
+   - **`patch`** — deterministic diff apply via `apply_patch_with_report()`; ops are `append | insert_after | replace | delete`
+   - **`rewrite_from_suggestions`** — LLM regenerates full skill from suggestions
+   - **`full_rewrite_minibatch`** — reflection directly emits complete candidate skills; select picks the best
+6. **Evaluate & Gate** — runs candidate on selection set, calls `evaluate_gate(cand_hard, current_score, best_score)`. Returns `accept_new_best | accept | reject` from a **literal `cand_hard > current_score`** comparison (`evaluation/gate.py:38`). No statistical test.
+
+Plus epoch-level stages:
+- **Slow update** — `run_slow_update()` builds longitudinal pairs across epochs.
+- **Meta skill** — `run_meta_skill()` produces optimizer-side memory of patterns across adjacent epochs.
+
+### Canonical patch shape (from `types.py:22-45`)
+
+```python
+EditOp = Literal["append", "insert_after", "replace", "delete"]
+
+@dataclass
+class Edit:
+    op: EditOp
+    content: str
+    target: str  # for replace/delete/insert_after
+    support_count: int | None  # how many trials voted for this edit
+    source_type: Literal["failure", "success"] | None
+    merge_level: int | None
+
+@dataclass
+class Patch:
+    edits: list[Edit]
+    reasoning: str
+    ranking_details: dict | None
+```
+
+### What `skillOptDriver` v0.51.0 in our substrate ACTUALLY does
+
+Source: `src/campaign/drivers/skillopt.ts` (current as of 0.51.0)
+
+- Single LLM call per `propose()` returning N full document rewrites
+- Post-parse rejection on: (a) any H2 header dropped, (b) sentence-edit count > editBudget × 2
+- Substantively equivalent to `gepaDriver` + 2 validation constraints
+
+### Deviations from SkillOpt
+
+| SkillOpt actual | Our 0.51.0 `skillOptDriver` |
+|---|---|
+| 6-stage pipeline (rollout → reflect → aggregate → select → update → gate) | Single LLM call → N rewrites |
+| **Patch-based edits** (`{op, target, content, support_count, source_type}`) | Full document rewrites only |
+| `merge_patches()` hierarchical merge across batches | No aggregation; each `propose()` is independent |
+| `rank_and_select(max_edits=edit_budget)` LLM-ranking of edits | All candidates that pass validation are returned |
+| 3 update modes (`patch`, `rewrite_from_suggestions`, `full_rewrite_minibatch`) | Only `full_rewrite_minibatch`-equivalent |
+| `evaluate_gate()` with `accept_new_best/accept/reject` codes | Substrate's outer gate decides ship/hold/inspect; driver doesn't see fine-grained accept signal |
+| Longitudinal `slow_update` across epochs | Not implemented |
+| `meta_skill` optimizer-side memory | Not implemented |
+| Selection-set cache (`sel_cache`) for repeated candidate hashes | Not implemented |
+| Edit-budget LR scheduler (constant / linear / cosine / autonomous) | Single fixed `editBudget` |
+| Mini-batch accumulation (`steps_per_epoch`, `merge_batch_size`) | Not implemented |
+| `decide_autonomous_learning_rate()` | Not implemented |
+| `longitudinal_pair_policy` (mixed / changed / unchanged) | Not implemented |
+
+**Honest assessment**: 13 substantive deviations. `skillOptDriver` 0.51.0 is **not** SkillOpt. It is `gepaDriver` with two post-validation constraints (section preservation, sentence-edit count). The methodology name oversells the implementation.
+
+### One thing where we are STRICTER than SkillOpt
+
+**The gate.** SkillOpt: literal `cand_hard > current_score` (`evaluation/gate.py:38`). Our substrate: paired bootstrap + 95% CI + Cohen's d + MDE + p-value (`defaultProductionGate`). When the lift CI straddles zero, our gate returns `hold` / `inspect`. SkillOpt would accept any improvement at all, even single-sample noise.
+
+This is real differentiation we have not been crediting ourselves for.
+
+---
+
+## Part 3 — Hermes Agent's "self-improvement"
+
+**Source**: `/tmp/hermes-agent/` (cloned 2026-05-27)
+- `agent/curator.py` (the actual loop)
+- `agent/skill_commands.py`
+- `agent/skill_utils.py`
+
+### What Hermes actually does
+
+From `curator.py` line 1: "Curator — background skill maintenance orchestrator. The curator is an auxiliary-model task that periodically reviews agent-created skills and maintains the collection."
+
+Trigger: idle-driven, with default `DEFAULT_INTERVAL_HOURS = 24 * 7` (7 days). When the agent has been idle for `DEFAULT_MIN_IDLE_HOURS = 2` and the last curator run was > 7 days ago, `maybe_run_curator()` spawns a forked AIAgent.
+
+What the curator does:
+- "Auto-transition lifecycle states based on derived skill activity timestamps"
+- "Spawn a background review agent that can **pin / archive / consolidate / patch** agent-created skills via `skill_manage`"
+- "Persist curator state (last_run_at, paused, etc.) in `.curator_state`"
+
+Strict invariants:
+- Only touches agent-created skills
+- "Never auto-deletes — only archives"
+- Pinned skills bypass auto-transitions
+- Uses the auxiliary client (separate from main session)
+
+### Hermes' actual gate
+
+**There is none.** The curator is an LLM editor making editorial decisions. There is no:
+- Held-out validation
+- Performance comparison between old and new skill versions
+- Statistical test
+- Rejection-on-regression mechanism
+
+Skills are refined by an LLM looking at usage patterns; the refinement is accepted because the LLM proposed it.
+
+### Honest assessment
+
+Hermes has a **skill curation system**, not a self-improvement loop. The README's claim "the only agent with a built-in learning loop" is generous — it's a 7-day-cron LLM librarian. There's no measurable guarantee that today's curated skill collection performs better than yesterday's.
+
+Compare:
+| Component | Hermes | SkillOpt | Tangle |
+|---|---|---|---|
+| Validation gate | None | `>` | Paired bootstrap CI |
+| Patch-level edits | No (LLM rewrites whole skill) | Yes | No (full rewrite only) |
+| Skill ranking / selection | No | Yes | No |
+| Sample efficiency claim | None | 35× vs GRPO | None |
+| Frequency | 7-day cron | Per training step | Per `selfImprove()` call |
+
+Where Tangle WINS: the gate. Where SkillOpt WINS: the pipeline sophistication. Where Hermes WINS: the deployment story (multi-platform, multi-tool-backend).
+
+---
+
+## Part 4 — What we should actually do
+
+### Phase A — rename to honest names (0.51.1, this session)
+
+The current `skillOptDriver` and `gepaDriver` names overclaim. Options:
+
+1. **Rename both:**
+   - `gepaDriver` → `reflectiveRewriteDriver` (drops the "Pareto" implication)
+   - `skillOptDriver` → `constrainedReflectiveDriver` (drops the SkillOpt-methodology implication)
+   - Reserve `gepaDriver` + `skillOptDriver` for faithful implementations
+2. **Keep `gepaDriver` name** (it's our most-used driver; renaming is disruptive); rename `skillOptDriver`.
+3. **Keep both names; add `@experimental` + a "differs from paper" docstring section.** Cheapest. Truthful enough.
+
+Recommendation: **option 3 plus a frontmatter "deviations from paper" section** in each driver source file. Empirically test before renaming.
+
+### Phase B — build the honest empirical harness (0.51.1, this session)
+
+`tests/driver-empirical.bench.ts` — for each driver:
+- Same scenarios (5 synthetic + 5 real legal-agent scenarios)
+- Same judge
+- Same `baselineSurface`
+- Same `budget` (1 gen, 3 candidates, holdout 0.3)
+- Report: lift mean, lift CI95, p-value, rollouts spent, $$ spent
+
+Drivers in the matrix:
+- `gepaDriver` (current full-rewrite reflection)
+- `skillOptDriver` (current 0.51.0 full-rewrite + constraints)
+- Future: real `skillOptDriverV2` with patch mode
+
+This is the **falsifiable test** of whether our drivers' methodology claims are worth the names.
+
+### Phase C — implement SkillOpt patch mode for real (0.52.0)
+
+Build `skillOptDriverV2` with:
+1. **`Edit` type matching SkillOpt's**: `{op: 'append'|'insert_after'|'replace'|'delete', content, target?, support_count?, source_type?}`
+2. **Reflect step emits patches**, not full rewrites
+3. **`mergePatches()`** — LLM-driven hierarchical merge of failure + success patches
+4. **`rankAndSelect()`** — LLM-driven ranking when edit pool > budget
+5. **Deterministic `applyPatch()`** — string ops, no LLM
+6. **Keep our gate** (paired bootstrap CI). Don't downgrade to SkillOpt's `>` — that's our edge.
+
+Estimated scope: 400-600 lines + tests.
+
+### Phase D — implement GEPA's Pareto frontier (0.53.0)
+
+Build `gepaDriverV2` with:
+1. **Candidate pool** retained across generations (non-dominated)
+2. **Multi-objective evaluation** (composite + cost + length + diversity)
+3. **Combine step** — LLM combines lessons from non-dominated candidates
+4. Keep reflection.
+5. Sample-efficiency target: match the paper's ~35× claim on a benchmark we choose.
+
+Estimated scope: 500-800 lines + tests.
+
+---
+
+## Source pointers (audit trail)
+
+- GEPA paper: https://arxiv.org/abs/2507.19457
+- SkillOpt repo: https://github.com/microsoft/SkillOpt (cloned at `/tmp/SkillOpt/` 2026-05-27)
+- Hermes repo: https://github.com/NousResearch/hermes-agent (cloned at `/tmp/hermes-agent/` 2026-05-27)
+- Our gepaDriver: `src/campaign/drivers/gepa.ts`
+- Our skillOptDriver: `src/campaign/drivers/skillopt.ts`
+- Our gate: `src/campaign/gates/default-production-gate.ts`
+- Our reflection primitive: `src/reflective-mutation.ts`
+
+Update this doc when:
+- We discover new behavior in any of the upstream methods (via reading their code, not their READMEs)
+- We ship a driver that closes one of the named gaps
+- We run the empirical harness and have real numbers to add