cognitive-core 0.2.4 → 0.3.1

package/dist/learning/escalation.js

@@ -0,0 +1,168 @@
+/**
+ * runEscalation — a stall-triggered operator ladder over the GEPA-like proposer.
+ *
+ * No single operator dominates: bounded+gate+single is cheapest on *additive*
+ * deficiencies but stalls on *coupled-constraint* ones, where frontier+rewrite is
+ * more reliable (see docs/DESIGN-skillopt-headtohead.md §9d/§9e). Rather than
+ * predict the regime up front (only partly knowable), we **start cheap and
+ * escalate on stall**: run each tier with a sub-budget, carry the best skill
+ * forward, and step up the ladder whenever a tier fails to reach a perfect score.
+ *
+ * Memory is a *gated tier* — consulted only on late tiers (after cheaper ones
+ * stalled), so it never taxes easy tasks that exit at tier 1 (the "memory is a
+ * tax on easy/cold seeds" finding).
+ *
+ * Every run emits a `routingRecord` ({features, tierReached, ...}) — the training
+ * signal for a future meta-learned router (MoEGate/MetaLearner). `tierReached` is
+ * the supervised label: the cheapest tier that achieved the final best.
+ */
+import { runGepaProposer, } from './gepa-proposer.js';
+/** Default ladder: cheap additive fix → holistic rewrite → frontier → +memory. */
+export const DEFAULT_ESCALATION_LADDER = [
+    {
+        label: 'bounded',
+        operator: { editMode: 'bounded', gate: 'val', candidate: 'single' },
+        budgetFraction: 0.25,
+    },
+    {
+        label: 'rewrite',
+        operator: { editMode: 'rewrite', gate: 'val', candidate: 'single' },
+        budgetFraction: 0.25,
+    },
+    {
+        label: 'frontier',
+        operator: { editMode: 'rewrite', gate: 'none', candidate: 'frontier' },
+        budgetFraction: 0.3,
+    },
+    {
+        label: 'frontier+mem',
+        operator: { editMode: 'rewrite', gate: 'none', candidate: 'frontier' },
+        budgetFraction: 0.2,
+        useMemory: true,
+    },
+];
+/**
+ * Slow-update meta-reflection prompt: contrast seed vs current skill and the
+ * checks STILL failing → ask for ONE root-cause meta-rule (the tradeoff fix).
+ */
+function buildMetaReflectPrompt(seedSkill, currentSkill, fails) {
+    return [
+        'You have iteratively edited a SKILL document, but the checks below STILL fail',
+        'after multiple edits. This usually means a TRADEOFF: an earlier fix is undone by',
+        'another constraint (e.g. a required section gets dropped to satisfy a length cap).',
+        `\n# Original skill\n${seedSkill}`,
+        `\n# Current (edited) skill\n${currentSkill}`,
+        `\n# Checks STILL failing\n${fails.map((f) => `- ${f.task}: ${f.feedback}`).join('\n')}`,
+        '\nDiagnose the ROOT cause and state ONE concise, general meta-rule that resolves',
+        'the tradeoff so future answers satisfy these checks. Return only the rule (one line).',
+    ].join('\n');
+}
+/** Run the operator ladder, escalating on stall, until perfect or budget spent. */
+export async function runEscalation(cfg) {
+    const perfect = cfg.perfectScore ?? 1;
+    const total = cfg.maxMetricCalls ?? 60;
+    const tiers = cfg.tiers ?? DEFAULT_ESCALATION_LADDER;
+    const start = Math.max(0, Math.min(cfg.startTier ?? 0, tiers.length - 1));
+    // Establish the seed baseline so a tier that learns nothing reads improved=false
+    // (rather than spuriously "improving" over -Infinity). 'baseline' is the routing
+    // label when no operator beat the deficient seed within budget.
+    const seedEval = await cfg.evaluate(cfg.seedSkill, cfg.valset);
+    const seedScore = seedEval.scores.length
+        ? seedEval.scores.reduce((a, b) => a + b, 0) / seedEval.scores.length
+        : 0;
+    let currentSkill = cfg.seedSkill;
+    let bestScore = seedScore;
+    let reachedLabel = 'baseline';
+    let reachedIdx = -1;
+    let spent = cfg.valset.length;
+    let reachedPerfect = bestScore >= perfect;
+    let metaGuidance = ''; // accumulated slow-update meta-rules, fed to later tiers
+    const perTier = [];
+    for (let i = start; i < tiers.length; i++) {
+        if (reachedPerfect)
+            break; // seed (or a prior tier) already solved it
+        const remaining = total - spent;
+        if (remaining <= 0)
+            break;
+        const tier = tiers[i];
+        const tierBudget = Math.min(remaining, Math.max(1, Math.round(total * tier.budgetFraction)));
+        const sub = {
+            seedSkill: currentSkill,
+            trainset: cfg.trainset,
+            valset: cfg.valset,
+            evaluate: cfg.evaluate,
+            propose: cfg.propose,
+            describeTask: cfg.describeTask,
+            perfectScore: perfect,
+            maxMetricCalls: tierBudget,
+            reflectionMinibatchSize: cfg.reflectionMinibatchSize,
+            rng: cfg.rng,
+            operator: tier.operator,
+            // compose reflection context: retrieval (only on memory tiers) + accumulated
+            // slow-update meta-guidance (all tiers, once any exists).
+            memoryContext: async (tasks) => {
+                const base = tier.useMemory && cfg.memoryContext ? await cfg.memoryContext(tasks) : '';
+                return [base, metaGuidance].filter(Boolean).join('\n\n');
+            },
+        };
+        const res = await runGepaProposer(sub);
+        spent += res.metricCalls;
+        const improved = res.bestValScore > bestScore;
+        perTier.push({
+            label: tier.label,
+            valScore: res.bestValScore,
+            metricCalls: res.metricCalls,
+            improved,
+        });
+        if (improved) {
+            bestScore = res.bestValScore;
+            currentSkill = res.bestSkill; // carry the best forward (non-regressing)
+            reachedLabel = tier.label;
+            reachedIdx = i;
+        }
+        if (bestScore >= perfect) {
+            reachedPerfect = true;
+            break; // converged — no need to escalate further
+        }
+        // SLOW UPDATE (fix 2): this tier stalled → longitudinal meta-reflection on the
+        // checks STILL failing, producing a meta-rule fed to later tiers via metaGuidance.
+        if (cfg.slowUpdate && spent < total) {
+            const ev = await cfg.evaluate(currentSkill, cfg.valset);
+            spent += cfg.valset.length;
+            const fails = cfg.valset
+                .map((t, j) => ({
+                task: cfg.describeTask(t),
+                feedback: ev.feedback[j] ?? '',
+                score: ev.scores[j] ?? 0,
+            }))
+                .filter((x) => x.score < perfect);
+            if (fails.length > 0) {
+                const rule = (await cfg.propose(buildMetaReflectPrompt(cfg.seedSkill, currentSkill, fails))).trim();
+                if (rule)
+                    metaGuidance += (metaGuidance ? '\n' : '') + `- ${rule}`;
+            }
+        }
+        // escalate to the next tier.
+    }
+    const routingRecord = {
+        features: cfg.features,
+        tierReached: reachedLabel,
+        tierIndex: reachedIdx,
+        metricCalls: spent,
+        finalScore: bestScore < 0 ? 0 : bestScore,
+        reachedPerfect,
+    };
+    return {
+        bestSkill: currentSkill,
+        bestValScore: bestScore < 0 ? 0 : bestScore,
+        metricCalls: spent,
+        tierReached: reachedLabel,
+        tierIndex: reachedIdx,
+        perTier,
+        routingRecord,
+    };
+}
+export function createEscalation(cfg) {
+    return { run: () => runEscalation(cfg) };
+}
+//# sourceMappingURL=escalation.js.map

package/dist/learning/escalation.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"escalation.js","sourceRoot":"","sources":["../../src/learning/escalation.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AACH,OAAO,EACL,eAAe,GAKhB,MAAM,oBAAoB,CAAC;AAa5B,kFAAkF;AAClF,MAAM,CAAC,MAAM,yBAAyB,GAAqB;IACzD;QACE,KAAK,EAAE,SAAS;QAChB,QAAQ,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE;QACnE,cAAc,EAAE,IAAI;KACrB;IACD;QACE,KAAK,EAAE,SAAS;QAChB,QAAQ,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE;QACnE,cAAc,EAAE,IAAI;KACrB;IACD;QACE,KAAK,EAAE,UAAU;QACjB,QAAQ,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE;QACtE,cAAc,EAAE,GAAG;KACpB;IACD;QACE,KAAK,EAAE,cAAc;QACrB,QAAQ,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE;QACtE,cAAc,EAAE,GAAG;QACnB,SAAS,EAAE,IAAI;KAChB;CACF,CAAC;AAoDF;;;GAGG;AACH,SAAS,sBAAsB,CAC7B,SAAiB,EACjB,YAAoB,EACpB,KAAgD;IAEhD,OAAO;QACL,+EAA+E;QAC/E,kFAAkF;QAClF,oFAAoF;QACpF,uBAAuB,SAAS,EAAE;QAClC,+BAA+B,YAAY,EAAE;QAC7C,6BAA6B,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QACxF,kFAAkF;QAClF,uFAAuF;KACxF,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED,mFAAmF;AACnF,MAAM,CAAC,KAAK,UAAU,aAAa,CAAI,GAAwB;IAC7D,MAAM,OAAO,GAAG,GAAG,CAAC,YAAY,IAAI,CAAC,CAAC;IACtC,MAAM,KAAK,GAAG,GAAG,CAAC,cAAc,IAAI,EAAE,CAAC;IACvC,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,IAAI,yBAAyB,CAAC;IACrD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,SAAS,IAAI,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;IAE1E,iFAAiF;IACjF,iFAAiF;IACjF,gEAAgE;IAChE,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAC/D,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM;QACtC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM;QACrE,CAAC,CAAC,CAAC,CAAC;IAEN,IAAI,YAAY,GAAG,GAAG,CAAC,SAAS,CAAC;IACjC,IAAI,SAAS,GAAG,SAAS,CAAC;IAC1B,IAAI,YAAY,GAAG,UAAU,CAAC;IAC9B,IAAI,UAAU,GAAG,CAAC,CAAC,CAAC;IACpB,IAAI,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC;IAC9B,IAAI,cAAc,GAAG,SAAS,IAAI,OAAO,CAAC;IAC1C,IAAI,YAAY,GAAG,EAAE,CAAC,CAAC,yDAAyD;IAChF,MAAM,OAAO,GAAgC,EAAE,CAAC;IAEhD,KAAK,IAAI,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,IAAI,cAAc;YAAE,MAAM,CAAC,2CAA2C;QACtE,MAAM,SAAS,GAAG,KAAK,GAAG,KAAK,CAAC;QAChC,IAAI,SAAS,IAAI,CAAC;YAAE,MAAM;QAC1B,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACtB,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;QAE7F,MAAM,GAAG,GAA0B;YACjC,SAAS,EAAE,YAAY;YACvB,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,OAAO,EAAE,GAAG,CAAC,OAAO;YACpB,YAAY,EAAE,GAAG,CAAC,YAAY;YAC9B,YAAY,EAAE,OAAO;YACrB,cAAc,EAAE,UAAU;YAC1B,uBAAuB,EAAE,GAAG,CAAC,uBAAuB;YACpD,GAAG,EAAE,GAAG,CAAC,GAAG;YACZ,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,6EAA6E;YAC7E,0DAA0D;YAC1D,aAAa,EAAE,KAAK,EAAE,KAAU,EAAE,EAAE;gBAClC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,IAAI,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBACvF,OAAO,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC3D,CAAC;SACF,CAAC;QACF,MAAM,GAAG,GAAG,MAAM,eAAe,CAAC,GAAG,CAAC,CAAC;QACvC,KAAK,IAAI,GAAG,CAAC,WAAW,CAAC;QAEzB,MAAM,QAAQ,GAAG,GAAG,CAAC,YAAY,GAAG,SAAS,CAAC;QAC9C,OAAO,CAAC,IAAI,CAAC;YACX,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,QAAQ,EAAE,GAAG,CAAC,YAAY;YAC1B,WAAW,EAAE,GAAG,CAAC,WAAW;YAC5B,QAAQ;SACT,CAAC,CAAC;QACH,IAAI,QAAQ,EAAE,CAAC;YACb,SAAS,GAAG,GAAG,CAAC,YAAY,CAAC;YAC7B,YAAY,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC,0CAA0C;YACxE,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC;YAC1B,UAAU,GAAG,CAAC,CAAC;QACjB,CAAC;QACD,IAAI,SAAS,IAAI,OAAO,EAAE,CAAC;YACzB,cAAc,GAAG,IAAI,CAAC;YACtB,MAAM,CAAC,0CAA0C;QACnD,CAAC;QAED,+EAA+E;QAC/E,mFAAmF;QACnF,IAAI,GAAG,CAAC,UAAU,IAAI,KAAK,GAAG,KAAK,EAAE,CAAC;YACpC,MAAM,EAAE,GAAG,MAAM,GAAG,CAAC,QAAQ,CAAC,YAAY,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;YACxD,KAAK,IAAI,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC;YAC3B,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM;iBACrB,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;gBACd,IAAI,EAAE,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC;gBACzB,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,EAAE;gBAC9B,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;aACzB,CAAC,CAAC;iBACF,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,OAAO,CAAC,CAAC;YACpC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACrB,MAAM,IAAI,GAAG,CACX,MAAM,GAAG,CAAC,OAAO,CAAC,sBAAsB,CAAC,GAAG,CAAC,SAAS,EAAE,YAAY,EAAE,KAAK,CAAC,CAAC,CAC9E,CAAC,IAAI,EAAE,CAAC;gBACT,IAAI,IAAI;oBAAE,YAAY,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,KAAK,IAAI,EAAE,CAAC;YACrE,CAAC;QACH,CAAC;QACD,6BAA6B;IAC/B,CAAC;IAED,MAAM,aAAa,GAA4B;QAC7C,QAAQ,EAAE,GAAG,CAAC,QAAQ;QACtB,WAAW,EAAE,YAAY;QACzB,SAAS,EAAE,UAAU;QACrB,WAAW,EAAE,KAAK;QAClB,UAAU,EAAE,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS;QACzC,cAAc;KACf,CAAC;IAEF,OAAO;QACL,SAAS,EAAE,YAAY;QACvB,YAAY,EAAE,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS;QAC3C,WAAW,EAAE,KAAK;QAClB,WAAW,EAAE,YAAY;QACzB,SAAS,EAAE,UAAU;QACrB,OAAO;QACP,aAAa;KACd,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAI,GAAwB;IAG1D,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC;AAC3C,CAAC"}

package/dist/learning/external-exposure.d.ts

@@ -0,0 +1,57 @@
+/**
+ * External-trajectory playbook exposure detection.
+ *
+ * Trajectories that arrive via `processTrajectory()` (session-bank ingestion,
+ * skill-tree factory, external feeds) carry no routing snapshot — cognitive-core
+ * never routed them, so it doesn't know what knowledge was surfaced. But many
+ * of them WERE guided out-of-band: skill-tree serves published skills into
+ * Claude Code sessions, and the transcript shows the agent invoking them
+ * (Skill tool calls, SKILL.md reads). Without exposure detection those
+ * trajectories would be mislabeled "unguided" and contaminate the
+ * effectiveness baseline.
+ *
+ * Three evidence tiers, strong to weak:
+ *
+ * 1. **Declared application** — the serving layer stamped playbook IDs/names
+ *    with direct use evidence into
+ *    `trajectory.metadata[APPLIED_PLAYBOOKS_METADATA_KEY]` (e.g. SessionLog
+ *    `skillsUsed`: Skill tool invocations). Counts as surfaced AND applied.
+ * 2. **Declared exposure** — the serving layer stamped surfaced playbook IDs /
+ *    names into `trajectory.metadata[SURFACED_PLAYBOOKS_METADATA_KEY]` (e.g.
+ *    SessionLog `skillsSurfaced`: skills injected into context). Declared
+ *    playbooks count as surfaced; whether they were *applied* is decided by
+ *    usage inference downstream.
+ * 3. **Explicit reference** — a playbook's name appears verbatim in the
+ *    trajectory steps (skill invocation, SKILL.md path, etc.). Referenced
+ *    playbooks count as surfaced AND applied: the agent demonstrably touched
+ *    them. Matching is deliberately conservative (word-boundary, minimum name
+ *    length) — a false "guided" label is worse than a missed one, because the
+ *    unguided baseline is the lift denominator.
+ */
+import type { Playbook, Trajectory } from '../types/index.js';
+/**
+ * Trajectory-metadata key under which serving layers (sessionlog, skill-tree,
+ * macro-agent) declare the playbook/skill identifiers surfaced into the
+ * session. Value: array of playbook IDs or names (strings).
+ */
+export declare const SURFACED_PLAYBOOKS_METADATA_KEY = "surfacedPlaybookIds";
+/**
+ * Trajectory-metadata key under which serving layers declare playbooks/skills
+ * with direct evidence of use (e.g. SessionLog `skillsUsed` — Skill tool
+ * invocations). Declared-applied implies surfaced; entries need not be
+ * repeated under `SURFACED_PLAYBOOKS_METADATA_KEY`. Value: array of playbook
+ * IDs or names (strings).
+ */
+export declare const APPLIED_PLAYBOOKS_METADATA_KEY = "appliedPlaybookIds";
+export interface ExternalExposure {
+    /** Playbooks declared surfaced (metadata) or explicitly referenced in steps. */
+    surfaced: Playbook[];
+    /** Subset with direct in-trajectory evidence of use (explicit reference). */
+    applied: Playbook[];
+}
+/**
+ * Detect which library playbooks were exposed to an external trajectory.
+ * Pure string scanning — no LLM, cheap enough for the dream path.
+ */
+export declare function detectExternalPlaybookExposure(trajectory: Trajectory, libraryPlaybooks: Playbook[]): ExternalExposure;
+//# sourceMappingURL=external-exposure.d.ts.map

package/dist/learning/external-exposure.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"external-exposure.d.ts","sourceRoot":"","sources":["../../src/learning/external-exposure.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAE9D;;;;GAIG;AACH,eAAO,MAAM,+BAA+B,wBAAwB,CAAC;AAErE;;;;;;GAMG;AACH,eAAO,MAAM,8BAA8B,uBAAuB,CAAC;AAKnE,MAAM,WAAW,gBAAgB;IAC/B,gFAAgF;IAChF,QAAQ,EAAE,QAAQ,EAAE,CAAC;IACrB,6EAA6E;IAC7E,OAAO,EAAE,QAAQ,EAAE,CAAC;CACrB;AAED;;;GAGG;AACH,wBAAgB,8BAA8B,CAC5C,UAAU,EAAE,UAAU,EACtB,gBAAgB,EAAE,QAAQ,EAAE,GAC3B,gBAAgB,CA2BlB"}

package/dist/learning/external-exposure.js

@@ -0,0 +1,95 @@
+/**
+ * External-trajectory playbook exposure detection.
+ *
+ * Trajectories that arrive via `processTrajectory()` (session-bank ingestion,
+ * skill-tree factory, external feeds) carry no routing snapshot — cognitive-core
+ * never routed them, so it doesn't know what knowledge was surfaced. But many
+ * of them WERE guided out-of-band: skill-tree serves published skills into
+ * Claude Code sessions, and the transcript shows the agent invoking them
+ * (Skill tool calls, SKILL.md reads). Without exposure detection those
+ * trajectories would be mislabeled "unguided" and contaminate the
+ * effectiveness baseline.
+ *
+ * Three evidence tiers, strong to weak:
+ *
+ * 1. **Declared application** — the serving layer stamped playbook IDs/names
+ *    with direct use evidence into
+ *    `trajectory.metadata[APPLIED_PLAYBOOKS_METADATA_KEY]` (e.g. SessionLog
+ *    `skillsUsed`: Skill tool invocations). Counts as surfaced AND applied.
+ * 2. **Declared exposure** — the serving layer stamped surfaced playbook IDs /
+ *    names into `trajectory.metadata[SURFACED_PLAYBOOKS_METADATA_KEY]` (e.g.
+ *    SessionLog `skillsSurfaced`: skills injected into context). Declared
+ *    playbooks count as surfaced; whether they were *applied* is decided by
+ *    usage inference downstream.
+ * 3. **Explicit reference** — a playbook's name appears verbatim in the
+ *    trajectory steps (skill invocation, SKILL.md path, etc.). Referenced
+ *    playbooks count as surfaced AND applied: the agent demonstrably touched
+ *    them. Matching is deliberately conservative (word-boundary, minimum name
+ *    length) — a false "guided" label is worse than a missed one, because the
+ *    unguided baseline is the lift denominator.
+ */
+/**
+ * Trajectory-metadata key under which serving layers (sessionlog, skill-tree,
+ * macro-agent) declare the playbook/skill identifiers surfaced into the
+ * session. Value: array of playbook IDs or names (strings).
+ */
+export const SURFACED_PLAYBOOKS_METADATA_KEY = 'surfacedPlaybookIds';
+/**
+ * Trajectory-metadata key under which serving layers declare playbooks/skills
+ * with direct evidence of use (e.g. SessionLog `skillsUsed` — Skill tool
+ * invocations). Declared-applied implies surfaced; entries need not be
+ * repeated under `SURFACED_PLAYBOOKS_METADATA_KEY`. Value: array of playbook
+ * IDs or names (strings).
+ */
+export const APPLIED_PLAYBOOKS_METADATA_KEY = 'appliedPlaybookIds';
+/** Names shorter than this are too collision-prone to substring-match. */
+const MIN_NAME_MATCH_LENGTH = 6;
+/**
+ * Detect which library playbooks were exposed to an external trajectory.
+ * Pure string scanning — no LLM, cheap enough for the dream path.
+ */
+export function detectExternalPlaybookExposure(trajectory, libraryPlaybooks) {
+    const declaredSurfaced = readDeclaredIdentifiers(trajectory, SURFACED_PLAYBOOKS_METADATA_KEY);
+    const declaredApplied = readDeclaredIdentifiers(trajectory, APPLIED_PLAYBOOKS_METADATA_KEY);
+    const stepText = trajectory.steps
+        .map((s) => `${s.thought ?? ''} ${s.action} ${s.observation ?? ''}`)
+        .join('\n')
+        .toLowerCase();
+    const surfaced = new Map();
+    const applied = new Map();
+    for (const playbook of libraryPlaybooks) {
+        const identifiers = [playbook.id.toLowerCase(), playbook.name.toLowerCase()];
+        const isDeclaredSurfaced = identifiers.some((i) => declaredSurfaced.has(i));
+        const isDeclaredApplied = identifiers.some((i) => declaredApplied.has(i));
+        const isReferenced = isExplicitlyReferenced(playbook.name, stepText);
+        if (isDeclaredSurfaced || isDeclaredApplied || isReferenced) {
+            surfaced.set(playbook.id, playbook);
+        }
+        if (isDeclaredApplied || isReferenced)
+            applied.set(playbook.id, playbook);
+    }
+    return {
+        surfaced: [...surfaced.values()],
+        applied: [...applied.values()],
+    };
+}
+// === PRIVATE ===
+function readDeclaredIdentifiers(trajectory, key) {
+    const raw = trajectory.metadata?.[key];
+    if (!Array.isArray(raw))
+        return new Set();
+    return new Set(raw.filter((v) => typeof v === 'string').map((v) => v.toLowerCase()));
+}
+/**
+ * Conservative name match: slug-style playbook names (e.g.
+ * "verification-before-completion") matched on word boundaries against the
+ * concatenated step text. Short names are skipped entirely.
+ */
+function isExplicitlyReferenced(name, stepText) {
+    const needle = name.toLowerCase().trim();
+    if (needle.length < MIN_NAME_MATCH_LENGTH)
+        return false;
+    const escaped = needle.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+    return new RegExp(`(^|[^\\w-])${escaped}($|[^\\w-])`).test(stepText);
+}
+//# sourceMappingURL=external-exposure.js.map

package/dist/learning/external-exposure.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"external-exposure.js","sourceRoot":"","sources":["../../src/learning/external-exposure.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAIH;;;;GAIG;AACH,MAAM,CAAC,MAAM,+BAA+B,GAAG,qBAAqB,CAAC;AAErE;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,8BAA8B,GAAG,oBAAoB,CAAC;AAEnE,0EAA0E;AAC1E,MAAM,qBAAqB,GAAG,CAAC,CAAC;AAShC;;;GAGG;AACH,MAAM,UAAU,8BAA8B,CAC5C,UAAsB,EACtB,gBAA4B;IAE5B,MAAM,gBAAgB,GAAG,uBAAuB,CAAC,UAAU,EAAE,+BAA+B,CAAC,CAAC;IAC9F,MAAM,eAAe,GAAG,uBAAuB,CAAC,UAAU,EAAE,8BAA8B,CAAC,CAAC;IAC5F,MAAM,QAAQ,GAAG,UAAU,CAAC,KAAK;SAC9B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,OAAO,IAAI,EAAE,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,WAAW,IAAI,EAAE,EAAE,CAAC;SACnE,IAAI,CAAC,IAAI,CAAC;SACV,WAAW,EAAE,CAAC;IAEjB,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAoB,CAAC;IAC7C,MAAM,OAAO,GAAG,IAAI,GAAG,EAAoB,CAAC;IAE5C,KAAK,MAAM,QAAQ,IAAI,gBAAgB,EAAE,CAAC;QACxC,MAAM,WAAW,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,WAAW,EAAE,EAAE,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;QAC7E,MAAM,kBAAkB,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5E,MAAM,iBAAiB,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1E,MAAM,YAAY,GAAG,sBAAsB,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QAErE,IAAI,kBAAkB,IAAI,iBAAiB,IAAI,YAAY,EAAE,CAAC;YAC5D,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;QACtC,CAAC;QACD,IAAI,iBAAiB,IAAI,YAAY;YAAE,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;IAC5E,CAAC;IAED,OAAO;QACL,QAAQ,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC;QAChC,OAAO,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;KAC/B,CAAC;AACJ,CAAC;AAED,kBAAkB;AAElB,SAAS,uBAAuB,CAAC,UAAsB,EAAE,GAAW;IAClE,MAAM,GAAG,GAAG,UAAU,CAAC,QAAQ,EAAE,CAAC,GAAG,CAAC,CAAC;IACvC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,GAAG,EAAE,CAAC;IAC1C,OAAO,IAAI,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;AACpG,CAAC;AAED;;;;GAIG;AACH,SAAS,sBAAsB,CAAC,IAAY,EAAE,QAAgB;IAC5D,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC;IACzC,IAAI,MAAM,CAAC,MAAM,GAAG,qBAAqB;QAAE,OAAO,KAAK,CAAC;IACxD,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,qBAAqB,EAAE,MAAM,CAAC,CAAC;IAC9D,OAAO,IAAI,MAAM,CAAC,cAAc,OAAO,aAAa,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;AACvE,CAAC"}

package/dist/learning/gepa-proposer.d.ts

@@ -0,0 +1,116 @@
+/**
+ * GepaProposer — a GEPA-like LLM proposer for text artifacts (skill documents).
+ *
+ * This is ccore's net-new *LLM* learning operator, in contrast to the heuristic
+ * extractors (PlaybookExtractor, analyzer). It implements GEPA's core loop —
+ * reflective mutation + Pareto-frontier candidate selection — over a single
+ * optimizable text component (the skill doc), turning scored rollouts into a new
+ * candidate, accepting on minibatch improvement, and tracking the Pareto frontier
+ * by per-instance validation score.
+ *
+ * It is deliberately LLM-/benchmark-agnostic: the caller injects
+ *   - `evaluate(skill, tasks)`  → run the frozen target + judge, return per-task scores
+ *   - `propose(prompt)`         → the proposer/reflection LM (a single text call)
+ * so the same operator runs in unit tests (mock fns) and against the live
+ * `claude` CLI (see `src/eval/live/skillopt_v1.ts`).
+ *
+ * The `memoryContext` hook is the Phase-2 differentiator over *vanilla* GEPA:
+ * when supplied, retrieved exemplars/reflexion/causal context are folded into the
+ * reflection prompt. Left undefined (the default), the operator is a faithful
+ * vanilla-GEPA reflective-mutation proposer — the parity checkpoint.
+ *
+ * See docs/DESIGN-skillopt-headtohead.md.
+ */
+/** Per-set evaluation: parallel arrays over the input tasks. */
+export interface GepaEvalResult {
+    /** hard score in [0,1] per task (1 = all judge checks pass). */
+    scores: number[];
+    /** the target's raw output per task (for the reflective dataset). */
+    outputs: string[];
+    /** judge feedback per task (why it failed / "ok"). */
+    feedback: string[];
+}
+/** Run the frozen target + judge on a batch; the GEPA "rollout + metric". */
+export type GepaEvaluator<T> = (skill: string, tasks: T[]) => Promise<GepaEvalResult>;
+/** The proposer/reflection LM: one prompt in, new component text out. */
+export type GepaReflector = (prompt: string) => Promise<string>;
+/**
+ * Operator-design toggles (each independently switchable for ablation). Defaults
+ * reproduce vanilla GEPA; flipping all three gives a SkillOpt-grade core.
+ */
+export interface GepaOperatorConfig {
+    /** 'rewrite' (default, full doc) | 'bounded' (SkillOpt add/delete/replace edits). */
+    editMode?: 'rewrite' | 'bounded';
+    /** 'none' (default, minibatch accept) | 'val' (strict held-out gate: accept iff val improves). */
+    gate?: 'none' | 'val';
+    /** 'frontier' (default, GEPA Pareto) | 'single' (SkillOpt one current doc). */
+    candidate?: 'frontier' | 'single';
+    /** LR cap: max bounded edits applied per step (default unlimited). */
+    maxEditsPerStep?: number;
+}
+export interface GepaProposerConfig<T> {
+    /** The deficient starting skill document. */
+    seedSkill: string;
+    /** Reflection minibatch pool (GEPA trainset). */
+    trainset: T[];
+    /** Pareto/selection pool (GEPA valset). NEVER the held-out test set. */
+    valset: T[];
+    /** Frozen-target rollout + judge. */
+    evaluate: GepaEvaluator<T>;
+    /** Proposer LM call. */
+    propose: GepaReflector;
+    /** Human-readable task text for the reflective dataset. */
+    describeTask: (task: T) => string;
+    /** Stop once aggregate val score reaches this (default 1). */
+    perfectScore?: number;
+    /** Budget: max target evaluations (rollouts), GEPA's max_metric_calls. */
+    maxMetricCalls?: number;
+    /** Reflection minibatch size (default 3). */
+    reflectionMinibatchSize?: number;
+    /** Seeded RNG in [0,1) for reproducible sampling (default Math.random). */
+    rng?: () => number;
+    /**
+     * Phase-2 lever (memory-conditioned reflection). Returns extra context
+     * (retrieved exemplars, reflexion, causal edges) to fold into the reflection
+     * prompt for a given minibatch. Undefined = vanilla GEPA (parity checkpoint).
+     */
+    memoryContext?: (tasks: T[]) => Promise<string> | string;
+    /** Operator-design toggles (bounded edits / gate / single-candidate). */
+    operator?: GepaOperatorConfig;
+    /** Optional per-iteration callback (telemetry). */
+    onIteration?: (info: GepaIterationInfo) => void;
+}
+export interface GepaIterationInfo {
+    iteration: number;
+    metricCalls: number;
+    bestValScore: number;
+    candidateValScore: number;
+    accepted: boolean;
+}
+export interface GepaCandidate {
+    skill: string;
+    /** per-val-instance scores (the Pareto frontier key). */
+    valScores: number[];
+    valAgg: number;
+}
+export interface GepaProposerResult {
+    bestSkill: string;
+    bestValScore: number;
+    metricCalls: number;
+    iterations: number;
+    frontierSize: number;
+    history: GepaIterationInfo[];
+}
+/**
+ * Run the GEPA-like proposer to optimize a skill document.
+ *
+ * Loop: select Pareto parent → sample train minibatch → reflect (proposer LM) →
+ * accept new candidate iff it improves on the minibatch → full val eval onto the
+ * frontier → repeat until perfect val score or the metric-call budget is spent.
+ */
+export declare function runGepaProposer<T>(cfg: GepaProposerConfig<T>): Promise<GepaProposerResult>;
+/** Factory mirroring the codebase convention (createX alongside the impl). */
+export declare function createGepaProposer<T>(cfg: GepaProposerConfig<T>): {
+    run: () => Promise<GepaProposerResult>;
+};
+//# sourceMappingURL=gepa-proposer.d.ts.map

package/dist/learning/gepa-proposer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"gepa-proposer.d.ts","sourceRoot":"","sources":["../../src/learning/gepa-proposer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,gEAAgE;AAChE,MAAM,WAAW,cAAc;IAC7B,gEAAgE;IAChE,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,qEAAqE;IACrE,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,sDAAsD;IACtD,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,6EAA6E;AAC7E,MAAM,MAAM,aAAa,CAAC,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,OAAO,CAAC,cAAc,CAAC,CAAC;AAEtF,yEAAyE;AACzE,MAAM,MAAM,aAAa,GAAG,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;AAEhE;;;GAGG;AACH,MAAM,WAAW,kBAAkB;IACjC,qFAAqF;IACrF,QAAQ,CAAC,EAAE,SAAS,GAAG,SAAS,CAAC;IACjC,kGAAkG;IAClG,IAAI,CAAC,EAAE,MAAM,GAAG,KAAK,CAAC;IACtB,+EAA+E;IAC/E,SAAS,CAAC,EAAE,UAAU,GAAG,QAAQ,CAAC;IAClC,sEAAsE;IACtE,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,kBAAkB,CAAC,CAAC;IACnC,6CAA6C;IAC7C,SAAS,EAAE,MAAM,CAAC;IAClB,iDAAiD;IACjD,QAAQ,EAAE,CAAC,EAAE,CAAC;IACd,wEAAwE;IACxE,MAAM,EAAE,CAAC,EAAE,CAAC;IACZ,qCAAqC;IACrC,QAAQ,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC;IAC3B,wBAAwB;IACxB,OAAO,EAAE,aAAa,CAAC;IACvB,2DAA2D;IAC3D,YAAY,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,MAAM,CAAC;IAClC,8DAA8D;IAC9D,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,0EAA0E;IAC1E,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,6CAA6C;IAC7C,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,2EAA2E;IAC3E,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB;;;;OAIG;IACH,aAAa,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,KAAK,OAAO,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC;IACzD,yEAAyE;IACzE,QAAQ,CAAC,EAAE,kBAAkB,CAAC;IAC9B,mDAAmD;IACnD,WAAW,CAAC,EAAE,CAAC,IAAI,EAAE,iBAAiB,KAAK,IAAI,CAAC;CACjD;AAED,MAAM,WAAW,iBAAiB;IAChC,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,yDAAyD;IACzD,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,kBAAkB;IACjC,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,iBAAiB,EAAE,CAAC;CAC9B;AAiJD;;;;;;GAMG;AACH,wBAAsB,eAAe,CAAC,CAAC,EAAE,GAAG,EAAE,kBAAkB,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAmGhG;AAED,8EAA8E;AAC9E,wBAAgB,kBAAkB,CAAC,CAAC,EAAE,GAAG,EAAE,kBAAkB,CAAC,CAAC,CAAC,GAAG;IACjE,GAAG,EAAE,MAAM,OAAO,CAAC,kBAAkB,CAAC,CAAC;CACxC,CAEA"}