cognitive-core 0.2.4 → 0.3.1

package/dist/eval/live/skillopt_v1_memory.js

@@ -0,0 +1,147 @@
+/**
+ * Phase-2 memory provider for the skillopt-v1 head-to-head — the lever that
+ * turns ccore's GEPA-like proposer from "vanilla GEPA" into "GEPA + ccore memory".
+ *
+ * Lever 1 (retrieval-conditioned proposals), the dominant one: every optimization
+ * rollout is recorded into a real ccore `ExperienceMemory`; before each reflection
+ * the proposer is handed the top-k most-similar *successful* prior answers, so it
+ * can reuse a structure that already passed the checks instead of re-inventing it.
+ *
+ * Shared across seeds (lever 4, cross-seed accumulation) when the runner reuses one
+ * instance; within a single seed it already gives the two-timescale benefit (lever
+ * 3) — later iterations retrieve earlier iterations' successes. Held-out (`test`)
+ * rollouts are NEVER recorded (no leakage); the runner only records optimization
+ * rollouts.
+ *
+ * See docs/DESIGN-skillopt-headtohead.md §4.
+ */
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { createSqlitePersistence } from '../../persistence/index.js';
+import { ExperienceMemory } from '../../memory/experience.js';
+/**
+ * Extract the structural skeleton of an answer: markdown headings, bold labels,
+ * and `Label:` lines — the elements section_present / regex checks look for.
+ */
+export function structuralSignature(answer) {
+    const sigs = new Set();
+    for (const raw of (answer || '').split('\n')) {
+        const heading = raw.match(/^\s{0,3}(#{1,6})\s+(.+?)\s*$/);
+        if (heading)
+            sigs.add(`${heading[1]} ${heading[2]}`.slice(0, 60));
+        const bold = raw.match(/^\s*\*\*(.+?)\*\*/);
+        if (bold)
+            sigs.add(`**${bold[1]}**`.slice(0, 60));
+        const label = raw.match(/^\s*([A-Z][A-Za-z ]{1,28}):(?:\s|$)/);
+        if (label)
+            sigs.add(`${label[1]}:`.slice(0, 60));
+    }
+    return [...sigs];
+}
+export class SkillOptV1Memory {
+    memory;
+    k;
+    domain;
+    successOnly;
+    mode;
+    maxExemplars;
+    excerptChars;
+    counter = 0;
+    constructor(persistence, cfg) {
+        this.memory = new ExperienceMemory(persistence);
+        this.k = cfg.k ?? 4;
+        this.domain = cfg.domain ?? 'skillopt-v1';
+        this.successOnly = cfg.successOnly ?? true;
+        this.mode = cfg.mode ?? 'recipe';
+        this.maxExemplars = cfg.maxExemplars ?? 6;
+        this.excerptChars = cfg.excerptChars ?? 240;
+    }
+    /** Build an isolated, ephemeral memory (own temp SQLite dir). */
+    static async create(cfg = {}) {
+        const baseDir = join(tmpdir(), `skillopt_v1_mem_${process.pid}_${Date.now()}`);
+        const persistence = createSqlitePersistence({ baseDir });
+        await persistence.init();
+        const m = new SkillOptV1Memory(persistence, cfg);
+        await m.memory.init();
+        return m;
+    }
+    /** Record one optimization rollout as an experience (success keyed on hard score). */
+    async record(task, output, score, feedback) {
+        const id = `exp-${this.domain}-${this.counter++}`;
+        const exp = {
+            id,
+            taskInput: task.intent,
+            solutionOutput: output,
+            feedback,
+            success: score >= 1,
+            domain: this.domain,
+            trajectoryId: id,
+            usageCount: 0,
+            accessScore: 0,
+            retrievalCount: 0,
+            createdAt: new Date(),
+            tier: 'hot',
+            metadata: { seed: task.id.split(':')[0] ?? '', split: task.split },
+        };
+        await this.memory.add(exp);
+    }
+    /** Record a whole evaluated batch (parallel arrays). */
+    async recordBatch(tasks, ev) {
+        for (let i = 0; i < tasks.length; i++) {
+            await this.record(tasks[i], ev.outputs[i] ?? '', ev.scores[i] ?? 0, ev.feedback[i] ?? '');
+        }
+    }
+    /**
+     * Retrieve top-k similar successful exemplars for a minibatch and render them
+     * as reflection context. Deduplicates by experience id across the batch.
+     */
+    async context(tasks) {
+        // Gather distinct successful exemplars across the minibatch, capped.
+        const seen = new Set();
+        const exemplars = [];
+        for (const t of tasks) {
+            const hits = await this.memory.findSimilar(t.intent, {
+                k: this.k,
+                domain: this.domain,
+                successOnly: this.successOnly,
+            });
+            for (const { experience: e } of hits) {
+                if (seen.has(e.id))
+                    continue;
+                seen.add(e.id);
+                exemplars.push({ taskInput: e.taskInput, output: e.solutionOutput });
+                if (exemplars.length >= this.maxExemplars)
+                    break;
+            }
+            if (exemplars.length >= this.maxExemplars)
+                break;
+        }
+        if (exemplars.length === 0)
+            return '';
+        if (this.mode === 'recipe') {
+            // Compact: the union of structural elements the winners shared.
+            const sigs = new Set();
+            for (const e of exemplars)
+                for (const s of structuralSignature(e.output))
+                    sigs.add(s);
+            if (sigs.size > 0) {
+                return [
+                    'Prior answers that PASSED all checks on similar tasks consistently included',
+                    'these structural elements — reproduce every one of them:',
+                    ...[...sigs].map((s) => `- ${s}`),
+                ].join('\n');
+            }
+            // no extractable structure (e.g. max_chars/contains winners) → fall through
+        }
+        // excerpt mode (or recipe fallback): short raw excerpts.
+        return [
+            'These prior answers PASSED all automated checks on similar tasks. Reuse the',
+            'structure/sections/format that worked rather than re-inventing it:',
+            ...exemplars.map((e) => `- Task: ${e.taskInput}\n  Winning answer (excerpt): ${e.output.slice(0, this.excerptChars)}`),
+        ].join('\n');
+    }
+    async close() {
+        await this.memory.close();
+    }
+}
+//# sourceMappingURL=skillopt_v1_memory.js.map

package/dist/eval/live/skillopt_v1_memory.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"skillopt_v1_memory.js","sourceRoot":"","sources":["../../../src/eval/live/skillopt_v1_memory.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AACH,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,uBAAuB,EAA0B,MAAM,4BAA4B,CAAC;AAC7F,OAAO,EAAE,gBAAgB,EAAE,MAAM,4BAA4B,CAAC;AA4B9D;;;GAGG;AACH,MAAM,UAAU,mBAAmB,CAAC,MAAc;IAChD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QAC7C,MAAM,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC;QAC1D,IAAI,OAAO;YAAE,IAAI,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QAClE,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;QAC5C,IAAI,IAAI;YAAE,IAAI,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QAClD,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAC;QAC/D,IAAI,KAAK;YAAE,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;IACnD,CAAC;IACD,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC;AACnB,CAAC;AAED,MAAM,OAAO,gBAAgB;IACV,MAAM,CAAmB;IACzB,CAAC,CAAS;IACV,MAAM,CAAS;IACf,WAAW,CAAU;IACrB,IAAI,CAAuB;IAC3B,YAAY,CAAS;IACrB,YAAY,CAAS;IAC9B,OAAO,GAAG,CAAC,CAAC;IAEpB,YAAoB,WAA8B,EAAE,GAA2B;QAC7E,IAAI,CAAC,MAAM,GAAG,IAAI,gBAAgB,CAAC,WAAW,CAAC,CAAC;QAChD,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;QACpB,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC,MAAM,IAAI,aAAa,CAAC;QAC1C,IAAI,CAAC,WAAW,GAAG,GAAG,CAAC,WAAW,IAAI,IAAI,CAAC;QAC3C,IAAI,CAAC,IAAI,GAAG,GAAG,CAAC,IAAI,IAAI,QAAQ,CAAC;QACjC,IAAI,CAAC,YAAY,GAAG,GAAG,CAAC,YAAY,IAAI,CAAC,CAAC;QAC1C,IAAI,CAAC,YAAY,GAAG,GAAG,CAAC,YAAY,IAAI,GAAG,CAAC;IAC9C,CAAC;IAED,iEAAiE;IACjE,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAA8B,EAAE;QAClD,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,EAAE,EAAE,mBAAmB,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QAC/E,MAAM,WAAW,GAAG,uBAAuB,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC;QACzD,MAAM,WAAW,CAAC,IAAI,EAAE,CAAC;QACzB,MAAM,CAAC,GAAG,IAAI,gBAAgB,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;QACjD,MAAM,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QACtB,OAAO,CAAC,CAAC;IACX,CAAC;IAED,sFAAsF;IACtF,KAAK,CAAC,MAAM,CACV,IAAoB,EACpB,MAAc,EACd,KAAa,EACb,QAAgB;QAEhB,MAAM,EAAE,GAAG,OAAO,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;QAClD,MAAM,GAAG,GAAe;YACtB,EAAE;YACF,SAAS,EAAE,IAAI,CAAC,MAAM;YACtB,cAAc,EAAE,MAAM;YACtB,QAAQ;YACR,OAAO,EAAE,KAAK,IAAI,CAAC;YACnB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,YAAY,EAAE,EAAE;YAChB,UAAU,EAAE,CAAC;YACb,WAAW,EAAE,CAAC;YACd,cAAc,EAAE,CAAC;YACjB,SAAS,EAAE,IAAI,IAAI,EAAE;YACrB,IAAI,EAAE,KAAK;YACX,QAAQ,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE;SACnE,CAAC;QACF,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;IAED,wDAAwD;IACxD,KAAK,CAAC,WAAW,CAAC,KAAuB,EAAE,EAAkB;QAC3D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAC5F,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,OAAO,CAAC,KAAuB;QACnC,qEAAqE;QACrE,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,MAAM,SAAS,GAAiD,EAAE,CAAC;QACnE,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;YACtB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,EAAE;gBACnD,CAAC,EAAE,IAAI,CAAC,CAAC;gBACT,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,WAAW,EAAE,IAAI,CAAC,WAAW;aAC9B,CAAC,CAAC;YACH,KAAK,MAAM,EAAE,UAAU,EAAE,CAAC,EAAE,IAAI,IAAI,EAAE,CAAC;gBACrC,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;oBAAE,SAAS;gBAC7B,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBACf,SAAS,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC,cAAc,EAAE,CAAC,CAAC;gBACrE,IAAI,SAAS,CAAC,MAAM,IAAI,IAAI,CAAC,YAAY;oBAAE,MAAM;YACnD,CAAC;YACD,IAAI,SAAS,CAAC,MAAM,IAAI,IAAI,CAAC,YAAY;gBAAE,MAAM;QACnD,CAAC;QACD,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEtC,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC3B,gEAAgE;YAChE,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;YAC/B,KAAK,MAAM,CAAC,IAAI,SAAS;gBAAE,KAAK,MAAM,CAAC,IAAI,mBAAmB,CAAC,CAAC,CAAC,MAAM,CAAC;oBAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACtF,IAAI,IAAI,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;gBAClB,OAAO;oBACL,6EAA6E;oBAC7E,0DAA0D;oBAC1D,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC;iBAClC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACf,CAAC;YACD,4EAA4E;QAC9E,CAAC;QAED,yDAAyD;QACzD,OAAO;YACL,6EAA6E;YAC7E,oEAAoE;YACpE,GAAG,SAAS,CAAC,GAAG,CACd,CAAC,CAAC,EAAE,EAAE,CACJ,WAAW,CAAC,CAAC,SAAS,iCAAiC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,YAAY,CAAC,EAAE,CAChG;SACF,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACf,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;IAC5B,CAAC;CACF"}

package/dist/eval/live/skillopt_v1_rules.d.ts

@@ -0,0 +1,39 @@
+/**
+ * SkillOpt v1 rule-judge primitives + task shape.
+ *
+ * Extracted from skillopt_v1.ts so the memory/causal lever modules can import
+ * these without importing the main module (which type-imports the levers back
+ * — the old arrangement was a 3-way import cycle). skillopt_v1.ts re-exports
+ * everything here, so existing importers are unaffected.
+ */
+export interface RuleCheck {
+    op: string;
+    arg: unknown;
+}
+export interface RuleJudge {
+    kind?: string;
+    checks?: RuleCheck[];
+}
+/** Human-readable description of a single check (for causal/reflexion signals). */
+export declare function describeCheck(c: RuleCheck): string;
+export interface CheckResult {
+    op: string;
+    arg: unknown;
+    pass: boolean;
+    desc: string;
+}
+/** Per-check pass/fail breakdown for a response (used by the causal lever). */
+export declare function evaluateChecks(judge: RuleJudge, response: string, toolsCalled?: string[]): CheckResult[];
+/** (hard, soft, rationale) for a gbrain-style rule judge. hard=1 iff ALL pass. */
+export declare function scoreRuleJudge(judge: RuleJudge, response: string, toolsCalled?: string[]): {
+    hard: number;
+    soft: number;
+    rationale: string;
+};
+export interface SkillOptV1Task {
+    id: string;
+    intent: string;
+    judge: RuleJudge;
+    split: 'train' | 'val' | 'test';
+}
+//# sourceMappingURL=skillopt_v1_rules.d.ts.map

package/dist/eval/live/skillopt_v1_rules.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"skillopt_v1_rules.d.ts","sourceRoot":"","sources":["../../../src/eval/live/skillopt_v1_rules.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAIH,MAAM,WAAW,SAAS;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,GAAG,EAAE,OAAO,CAAC;CACd;AACD,MAAM,WAAW,SAAS;IACxB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC;CACtB;AA2CD,mFAAmF;AACnF,wBAAgB,aAAa,CAAC,CAAC,EAAE,SAAS,GAAG,MAAM,CAiBlD;AAED,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,GAAG,EAAE,OAAO,CAAC;IACb,IAAI,EAAE,OAAO,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;CACd;AAED,+EAA+E;AAC/E,wBAAgB,cAAc,CAC5B,KAAK,EAAE,SAAS,EAChB,QAAQ,EAAE,MAAM,EAChB,WAAW,GAAE,MAAM,EAAO,GACzB,WAAW,EAAE,CAOf;AAED,kFAAkF;AAClF,wBAAgB,cAAc,CAC5B,KAAK,EAAE,SAAS,EAChB,QAAQ,EAAE,MAAM,EAChB,WAAW,GAAE,MAAM,EAAO,GACzB;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,CAYnD;AAED,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,SAAS,CAAC;IACjB,KAAK,EAAE,OAAO,GAAG,KAAK,GAAG,MAAM,CAAC;CACjC"}

package/dist/eval/live/skillopt_v1_rules.js

@@ -0,0 +1,91 @@
+/**
+ * SkillOpt v1 rule-judge primitives + task shape.
+ *
+ * Extracted from skillopt_v1.ts so the memory/causal lever modules can import
+ * these without importing the main module (which type-imports the levers back
+ * — the old arrangement was a 3-way import cycle). skillopt_v1.ts re-exports
+ * everything here, so existing importers are unaffected.
+ */
+function sectionPresent(response, name) {
+    const esc = name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+    // markdown heading or bold line containing the name (port of judges.py)
+    const heading = new RegExp(`^\\s{0,3}(#{1,6}\\s*.*${esc}|\\*\\*.*${esc}.*\\*\\*\\s*:?)\\s*$`, 'im');
+    if (heading.test(response || ''))
+        return true;
+    // "Name:" style label at line start
+    const label = new RegExp(`^\\s*${esc}\\s*:`, 'im');
+    return label.test(response || '');
+}
+function checkOne(op, arg, response, toolsCalled) {
+    const r = response || '';
+    switch (op) {
+        case 'section_present':
+            return sectionPresent(r, String(arg));
+        case 'regex':
+            try {
+                return new RegExp(String(arg)).test(r);
+            }
+            catch {
+                return false;
+            }
+        case 'max_chars':
+            return r.length <= Number(arg);
+        case 'min_chars':
+            return r.length >= Number(arg);
+        case 'contains':
+            return r.toLowerCase().includes(String(arg).toLowerCase());
+        case 'tool_called': {
+            const name = String(arg).toLowerCase();
+            if (toolsCalled.some((t) => t.toLowerCase() === name))
+                return true;
+            // single-shot approximation: an explicit marker in the response
+            return new RegExp(`\\btool_call\\s*:\\s*${name}\\b`, 'i').test(r);
+        }
+        default:
+            return true; // unknown op: do not block
+    }
+}
+/** Human-readable description of a single check (for causal/reflexion signals). */
+export function describeCheck(c) {
+    switch (c.op) {
+        case 'section_present':
+            return `a "${String(c.arg)}" section`;
+        case 'regex':
+            return `text matching /${String(c.arg)}/`;
+        case 'max_chars':
+            return `length <= ${String(c.arg)} chars`;
+        case 'min_chars':
+            return `length >= ${String(c.arg)} chars`;
+        case 'contains':
+            return `the text "${String(c.arg)}"`;
+        case 'tool_called':
+            return `a call to the "${String(c.arg)}" tool`;
+        default:
+            return `${c.op}=${String(c.arg)}`;
+    }
+}
+/** Per-check pass/fail breakdown for a response (used by the causal lever). */
+export function evaluateChecks(judge, response, toolsCalled = []) {
+    return (judge?.checks ?? []).map((c) => ({
+        op: c.op ?? '',
+        arg: c.arg,
+        pass: checkOne(c.op ?? '', c.arg, response, toolsCalled),
+        desc: describeCheck(c),
+    }));
+}
+/** (hard, soft, rationale) for a gbrain-style rule judge. hard=1 iff ALL pass. */
+export function scoreRuleJudge(judge, response, toolsCalled = []) {
+    const results = evaluateChecks(judge, response, toolsCalled);
+    if (results.length === 0)
+        return { hard: 0, soft: 0, rationale: 'no checks' };
+    const passed = results.filter((r) => r.pass).length;
+    const failed = results.filter((r) => !r.pass).map((r) => `${r.op}=${String(r.arg)}`);
+    const soft = passed / results.length;
+    const hard = passed === results.length ? 1 : 0;
+    return {
+        hard,
+        soft,
+        rationale: hard ? 'all checks passed' : `failed: ${failed.join(', ')}`,
+    };
+}
+//# sourceMappingURL=skillopt_v1_rules.js.map

package/dist/eval/live/skillopt_v1_rules.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"skillopt_v1_rules.js","sourceRoot":"","sources":["../../../src/eval/live/skillopt_v1_rules.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAaH,SAAS,cAAc,CAAC,QAAgB,EAAE,IAAY;IACpD,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,qBAAqB,EAAE,MAAM,CAAC,CAAC;IACxD,wEAAwE;IACxE,MAAM,OAAO,GAAG,IAAI,MAAM,CACxB,yBAAyB,GAAG,YAAY,GAAG,sBAAsB,EACjE,IAAI,CACL,CAAC;IACF,IAAI,OAAO,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC;QAAE,OAAO,IAAI,CAAC;IAC9C,oCAAoC;IACpC,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,QAAQ,GAAG,OAAO,EAAE,IAAI,CAAC,CAAC;IACnD,OAAO,KAAK,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC;AACpC,CAAC;AAED,SAAS,QAAQ,CAAC,EAAU,EAAE,GAAY,EAAE,QAAgB,EAAE,WAAqB;IACjF,MAAM,CAAC,GAAG,QAAQ,IAAI,EAAE,CAAC;IACzB,QAAQ,EAAE,EAAE,CAAC;QACX,KAAK,iBAAiB;YACpB,OAAO,cAAc,CAAC,CAAC,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;QACxC,KAAK,OAAO;YACV,IAAI,CAAC;gBACH,OAAO,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACzC,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,KAAK,CAAC;YACf,CAAC;QACH,KAAK,WAAW;YACd,OAAO,CAAC,CAAC,MAAM,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC;QACjC,KAAK,WAAW;YACd,OAAO,CAAC,CAAC,MAAM,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC;QACjC,KAAK,UAAU;YACb,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;QAC7D,KAAK,aAAa,CAAC,CAAC,CAAC;YACnB,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;YACvC,IAAI,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,KAAK,IAAI,CAAC;gBAAE,OAAO,IAAI,CAAC;YACnE,gEAAgE;YAChE,OAAO,IAAI,MAAM,CAAC,wBAAwB,IAAI,KAAK,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACpE,CAAC;QACD;YACE,OAAO,IAAI,CAAC,CAAC,2BAA2B;IAC5C,CAAC;AACH,CAAC;AAED,mFAAmF;AACnF,MAAM,UAAU,aAAa,CAAC,CAAY;IACxC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC;QACb,KAAK,iBAAiB;YACpB,OAAO,MAAM,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC;QACxC,KAAK,OAAO;YACV,OAAO,kBAAkB,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC;QAC5C,KAAK,WAAW;YACd,OAAO,aAAa,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC;QAC5C,KAAK,WAAW;YACd,OAAO,aAAa,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC;QAC5C,KAAK,UAAU;YACb,OAAO,aAAa,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC;QACvC,KAAK,aAAa;YAChB,OAAO,kBAAkB,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC;QACjD;YACE,OAAO,GAAG,CAAC,CAAC,EAAE,IAAI,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC;IACtC,CAAC;AACH,CAAC;AASD,+EAA+E;AAC/E,MAAM,UAAU,cAAc,CAC5B,KAAgB,EAChB,QAAgB,EAChB,cAAwB,EAAE;IAE1B,OAAO,CAAC,KAAK,EAAE,MAAM,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACvC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE;QACd,GAAG,EAAE,CAAC,CAAC,GAAG;QACV,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,QAAQ,EAAE,WAAW,CAAC;QACxD,IAAI,EAAE,aAAa,CAAC,CAAC,CAAC;KACvB,CAAC,CAAC,CAAC;AACN,CAAC;AAED,kFAAkF;AAClF,MAAM,UAAU,cAAc,CAC5B,KAAgB,EAChB,QAAgB,EAChB,cAAwB,EAAE;IAE1B,MAAM,OAAO,GAAG,cAAc,CAAC,KAAK,EAAE,QAAQ,EAAE,WAAW,CAAC,CAAC;IAC7D,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,SAAS,EAAE,WAAW,EAAE,CAAC;IAC9E,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;IACpD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACrF,MAAM,IAAI,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IACrC,MAAM,IAAI,GAAG,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/C,OAAO;QACL,IAAI;QACJ,IAAI;QACJ,SAAS,EAAE,IAAI,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,WAAW,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;KACvE,CAAC;AACJ,CAAC"}

package/dist/eval/live/skillsbench.d.ts

@@ -0,0 +1,138 @@
+import { createStep, type Task } from '../../types/index.js';
+import type { EvalTask } from '../types.js';
+import type { ExtendedLiveBenchmark, BenchmarkVerdict } from './live-benchmark.js';
+import type { BenchmarkDelegate } from './live-runner.js';
+import { DockerOrchestrator } from './docker.js';
+/**
+ * SkillsBench — one-shot agentic tasks across multiple domains.
+ *
+ * The community standard for measuring skill quality: each task has a Docker
+ * container, a task instruction, and a ground-truth test suite. The agent
+ * gets one shot (with optional skill injection), and the test suite determines
+ * pass/fail. No LLM judge.
+ *
+ * This adapter loads tasks from a local SkillsBench repo (cloned by the user),
+ * builds Docker images per-task on setup, and runs agent + tests per task.
+ *
+ * Repo structure (Harbor format):
+ *   tasks/<task-id>/
+ *     task.toml             — metadata (category, difficulty, timeouts)
+ *     instruction.md        — task prompt
+ *     environment/
+ *       Dockerfile          — container definition
+ *     tests/                — test suite (run inside container)
+ *
+ * Environment variables:
+ *   SKILLSBENCH_REPO — path to the cloned SkillsBench repository
+ *   SKILLSBENCH_TIMEOUT — per-task timeout in ms (default 300_000)
+ */
+interface SkillsBenchTaskDef {
+    id: string;
+    /** Category from task.toml metadata (e.g. "industrial-physical-systems"). */
+    domain: string;
+    instruction: string;
+    dockerfilePath: string;
+    testCommand: string[];
+    workdir: string;
+    skillDir: string;
+    /** Difficulty from task.toml (e.g. "hard", "medium"). */
+    difficulty?: string;
+    /** Tags from task.toml for searchability. */
+    tags?: string[];
+    /** Absolute path to the task directory on host. */
+    taskDir: string;
+}
+export declare class SkillsBenchBenchmark implements ExtendedLiveBenchmark {
+    private readonly opts;
+    readonly name = "skillsbench";
+    readonly interactive = false;
+    readonly preserveOrder = false;
+    readonly playbookMinTrajectories = 3;
+    private docker;
+    private repoPath;
+    private taskDefs;
+    private builtDomainImages;
+    private traceDir;
+    constructor(opts?: {
+        repoPath?: string;
+        timeoutMs?: number;
+    });
+    setup(): Promise<void>;
+    teardown(): Promise<void>;
+    loadTasks(opts: {
+        count: number;
+        seed: number;
+    }): Promise<EvalTask[]>;
+    verify(_task: Task, _output: string): BenchmarkVerdict;
+    getDocker(): DockerOrchestrator;
+    getTaskDef(taskDefId: string): SkillsBenchTaskDef | undefined;
+    /**
+     * Get the Docker image for a task. Falls back to domain-level lookup
+     * for backwards compatibility with domain-nested layouts.
+     */
+    getTaskImage(taskDefId: string): string | undefined;
+    getDomainImage(domain: string): string | undefined;
+    /**
+     * Build the Docker image for a single task on demand.
+     * Returns the image tag, or undefined if no Dockerfile exists.
+     */
+    ensureImage(taskDefId: string): Promise<string | undefined>;
+    getTimeoutMs(): number;
+    getTraceDir(): string | undefined;
+    /**
+     * Load task definitions from the SkillsBench repo.
+     *
+     * Supports the actual Harbor format:
+     *   tasks/<task-id>/
+     *     task.toml             — metadata
+     *     instruction.md        — task prompt
+     *     environment/Dockerfile — container definition
+     *     tests/                 — test suite
+     *
+     * Also supports legacy nested structure for backwards compatibility:
+     *   tasks/<domain>/<task-id>/instruction.md
+     */
+    private loadManifest;
+    /**
+     * Recursively scan for task directories. A directory is a task if it contains
+     * instruction.md (and optionally task.toml for Harbor metadata).
+     */
+    private scanTaskDir;
+    private resolveDockerfile;
+    private resolveTestCommand;
+    /**
+     * Infer domain from task ID path (e.g. "web-scraping/task-1" → "web-scraping").
+     * For flat IDs, uses the task ID itself.
+     */
+    private inferDomain;
+    /**
+     * Minimal task.toml parser for the subset used by Harbor benchmarks.
+     * Handles strings, numbers, booleans, arrays, and [section] headers.
+     */
+    private parseTaskToml;
+    private parseTomlValue;
+    private shuffle;
+    delegateFactory: (_benchmark: ExtendedLiveBenchmark, opts?: {
+        model?: string;
+        timeoutMs?: number;
+        provider?: string;
+        awsRegion?: string;
+    }) => BenchmarkDelegate;
+}
+export declare function createSkillsBenchBenchmark(opts?: {
+    repoPath?: string;
+    timeoutMs?: number;
+}): SkillsBenchBenchmark;
+/**
+ * Map the per-step command/output records collected during the multi-turn loop
+ * into real Step objects so the delegate result carries a full trajectory.
+ *
+ * Factored as a pure function for unit-testing without Docker.
+ */
+export declare function mapSkillsBenchSteps(steps: Array<{
+    command: string;
+    exitCode: number;
+    output: string;
+}>, initialObs: string): ReturnType<typeof createStep>[];
+export {};
+//# sourceMappingURL=skillsbench.d.ts.map

package/dist/eval/live/skillsbench.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"skillsbench.d.ts","sourceRoot":"","sources":["../../../src/eval/live/skillsbench.ts"],"names":[],"mappings":"AAGA,OAAO,EAGL,UAAU,EAGV,KAAK,IAAI,EACV,MAAM,sBAAsB,CAAC;AAE9B,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,KAAK,EAAE,qBAAqB,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AACnF,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAC1D,OAAO,EAAwB,kBAAkB,EAA4B,MAAM,aAAa,CAAC;AAIjG;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,UAAU,kBAAkB;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,6EAA6E;IAC7E,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,yDAAyD;IACzD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,6CAA6C;IAC7C,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,mDAAmD;IACnD,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,qBAAa,oBAAqB,YAAW,qBAAqB;IAYpD,OAAO,CAAC,QAAQ,CAAC,IAAI;IAXjC,QAAQ,CAAC,IAAI,iBAAiB;IAC9B,QAAQ,CAAC,WAAW,SAAS;IAC7B,QAAQ,CAAC,aAAa,SAAS;IAC/B,QAAQ,CAAC,uBAAuB,KAAK;IAErC,OAAO,CAAC,MAAM,CAAqB;IACnC,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,QAAQ,CAA4B;IAC5C,OAAO,CAAC,iBAAiB,CAA6B;IACtD,OAAO,CAAC,QAAQ,CAAqB;gBAER,IAAI,GAAE;QAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAA;KAAO;IAK3E,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAatB,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAOzB,SAAS,CAAC,IAAI,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;IAyB3E,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,GAAG,gBAAgB;IAItD,SAAS,IAAI,kBAAkB;IAI/B,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,kBAAkB,GAAG,SAAS;IAI7D;;;OAGG;IACH,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS;IAInD,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS;IAIlD;;;OAGG;IACG,WAAW,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;IAejE,YAAY,IAAI,MAAM;IAItB,WAAW,IAAI,MAAM,GAAG,SAAS;IAIjC;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,YAAY;IAWpB;;;OAGG;IACH,OAAO,CAAC,WAAW;IAoDnB,OAAO,CAAC,iBAAiB;IAYzB,OAAO,CAAC,kBAAkB;IAyB1B;;;OAGG;IACH,OAAO,CAAC,WAAW;IAKnB;;;OAGG;IACH,OAAO,CAAC,aAAa;IAiCrB,OAAO,CAAC,cAAc;IAiBtB,OAAO,CAAC,OAAO;IAUf,eAAe,GACb,YAAY,qBAAqB,EACjC,OAAO;QAAE,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAA;KAAE,KACnF,iBAAiB,CAOlB;CACH;AAED,wBAAgB,0BAA0B,CAAC,IAAI,CAAC,EAAE;IAChD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,GAAG,oBAAoB,CAEvB;AAyYD;;;;;GAKG;AACH,wBAAgB,mBAAmB,CACjC,KAAK,EAAE,KAAK,CAAC;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC,EACnE,UAAU,EAAE,MAAM,GACjB,UAAU,CAAC,OAAO,UAAU,CAAC,EAAE,CAqBjC"}