cognitive-core 0.2.5 → 0.3.1

package/dist/eval/live/difficulty-registry.js

@@ -0,0 +1,169 @@
+/**
+ * Per-family difficulty registry + discriminative task sampler (Phase B(2) of
+ * the adaptive eval-economics roadmap — DESIGN-meta-learning-validation.md §10).
+ *
+ * Motivation: episodes spent on tasks the model always solves (or never
+ * solves) carry almost no information about a config delta — Fisher
+ * information for a binary outcome peaks at p(success) ≈ 0.5. The registry
+ * accumulates per-family success counts across runs; the sampler then weights
+ * task selection toward the discriminative band and away from
+ * saturated-either-way families.
+ *
+ * The family unit is `skillTag`: it is stamped on every TaskResult row (so
+ * historical artifacts are ingestible) and it is the unit that repeats across
+ * runs even when benchmark instances rotate. This is what makes the registry
+ * work "in the wild": a never-seen instance inherits its family's prior
+ * (hierarchical pooling on features), rather than requiring per-instance
+ * repetition.
+ *
+ * Difficulty is estimated from cold-arm counts when enough exist (learning
+ * shifts p over a run, contaminating the estimate), falling back to pooled
+ * counts when the cold sample is thin.
+ *
+ * Priors are ADVISORY: a family with fewer than `minRuns` ingested runs gets
+ * weight 1 (uniform), and known families keep a weight floor rather than
+ * being hard-dropped — per the design doc's small-n IRT risk note.
+ *
+ * Estimand caveat: discriminative selection changes what a success rate
+ * means. Numbers from a discriminative stream are comparable to each other,
+ * not to fixed-stream runs — always report which sampler produced a number.
+ */
+import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs';
+import { dirname } from 'node:path';
+import { seededRng } from '../rng.js';
+const COLD_ARM_LABEL = 'no-learning';
+/** Runs per family before the discriminative weight becomes non-uniform. */
+const DEFAULT_MIN_RUNS = 3;
+/** Cold-arm episodes needed before cold counts alone drive the estimate. */
+const MIN_COLD_ATTEMPTS = 4;
+/** Weight floor for known families — advisory down-weighting, never a drop. */
+const WEIGHT_FLOOR = 0.05;
+export function emptyDifficultyRegistry() {
+    return { version: 1, updatedAt: new Date(0).toISOString(), benchmarks: {} };
+}
+/** Load a registry file, returning an empty registry when the file is absent. */
+export function loadDifficultyRegistry(path) {
+    if (!existsSync(path))
+        return emptyDifficultyRegistry();
+    const raw = JSON.parse(readFileSync(path, 'utf8'));
+    if (raw.version !== 1) {
+        throw new Error(`Unsupported difficulty registry version ${String(raw.version)} in ${path}`);
+    }
+    return raw;
+}
+/** Atomic write (temp + rename), matching the checkpoint writer's discipline. */
+export function saveDifficultyRegistry(path, registry) {
+    registry.updatedAt = new Date().toISOString();
+    mkdirSync(dirname(path), { recursive: true });
+    const tmp = `${path}.tmp`;
+    writeFileSync(tmp, JSON.stringify(registry, null, 2));
+    renameSync(tmp, path);
+}
+/**
+ * Fold one run's episode rows into the registry. `runs` is incremented once
+ * per family per call, so a call corresponds to one run artifact.
+ */
+export function ingestRunRows(registry, benchmark, rows) {
+    const bench = (registry.benchmarks[benchmark] ??= { families: {} });
+    const touched = new Set();
+    for (const row of rows) {
+        if (row.status === 'env_error')
+            continue;
+        const stats = (bench.families[row.family] ??= {
+            attempts: 0,
+            successes: 0,
+            coldAttempts: 0,
+            coldSuccesses: 0,
+            runs: 0,
+        });
+        stats.attempts++;
+        if (row.success)
+            stats.successes++;
+        if (row.arm === COLD_ARM_LABEL) {
+            stats.coldAttempts++;
+            if (row.success)
+                stats.coldSuccesses++;
+        }
+        touched.add(row.family);
+    }
+    for (const family of touched) {
+        bench.families[family].runs++;
+    }
+}
+/**
+ * Extract ingestible rows from a run artifact: either a final result JSON
+ * (learning.perSeed + noLearning.perSeed) or a <out>.checkpoint.json partial.
+ * Returns the benchmark name found in the artifact alongside the rows.
+ */
+export function extractIngestRows(raw) {
+    const rows = [];
+    if (raw.partial === true) {
+        const file = raw;
+        for (const t of file.tasks ?? []) {
+            rows.push({ family: t.skillTag, arm: t.arm, success: t.success, status: t.status });
+        }
+        const meta = (raw.meta ?? {});
+        return { benchmark: typeof meta.benchmark === 'string' ? meta.benchmark : undefined, rows };
+    }
+    const pullArm = (arm, condition) => {
+        const perSeed = condition?.perSeed ?? [];
+        for (const seedRows of perSeed) {
+            for (const r of seedRows) {
+                rows.push({ family: r.skillTag, arm, success: r.success, status: r.status });
+            }
+        }
+    };
+    pullArm('learning', raw.learning);
+    pullArm(COLD_ARM_LABEL, raw.noLearning);
+    return {
+        benchmark: typeof raw.benchmarkName === 'string' ? raw.benchmarkName : undefined,
+        rows,
+    };
+}
+/**
+ * Posterior-mean success estimate for a family under a Beta(1,1) prior.
+ * Cold-arm counts drive the estimate once they are thick enough; otherwise
+ * pooled counts (learning contamination beats no data).
+ */
+export function estimateFamilySuccess(stats) {
+    const useCold = stats.coldAttempts >= MIN_COLD_ATTEMPTS;
+    const attempts = useCold ? stats.coldAttempts : stats.attempts;
+    const successes = useCold ? stats.coldSuccesses : stats.successes;
+    return (successes + 1) / (attempts + 2);
+}
+/**
+ * Sampling weight for a family: 4·p̂(1−p̂), which is 1 at p̂=0.5 and falls
+ * toward 0 as the family saturates in either direction. Unknown or
+ * under-observed families (< minRuns ingested runs) get weight 1 — the
+ * registry is advisory until it has seen enough evidence.
+ */
+export function discriminativeWeight(stats, minRuns = DEFAULT_MIN_RUNS) {
+    if (!stats || stats.runs < minRuns)
+        return 1;
+    const p = estimateFamilySuccess(stats);
+    return Math.max(WEIGHT_FLOOR, 4 * p * (1 - p));
+}
+/**
+ * Weighted sampling without replacement (Efraimidis–Spirakis: key = u^(1/w),
+ * keep the `count` largest keys), deterministic per (registry, benchmark,
+ * seed). Selected tasks keep their original relative order — ordered
+ * benchmarks stay ordered, just with gaps — and are re-indexed 0..n-1 so
+ * positional aggregation (curves, checkpoint rows) stays consistent.
+ */
+export function selectDiscriminativeTasks(pool, count, registry, benchmark, seed, minRuns = DEFAULT_MIN_RUNS) {
+    if (pool.length <= count)
+        return pool;
+    const families = registry.benchmarks[benchmark]?.families ?? {};
+    const rng = seededRng('difficulty-select', benchmark, seed);
+    const keyed = pool.map((task, poolIdx) => {
+        const w = discriminativeWeight(families[task.skillTag], minRuns);
+        const u = Math.max(rng(), Number.MIN_VALUE);
+        return { poolIdx, key: Math.pow(u, 1 / w) };
+    });
+    keyed.sort((a, b) => b.key - a.key);
+    const chosen = new Set(keyed.slice(0, count).map((k) => k.poolIdx));
+    return pool
+        .filter((_, poolIdx) => chosen.has(poolIdx))
+        .map((task, index) => ({ ...task, index }));
+}
+//# sourceMappingURL=difficulty-registry.js.map

package/dist/eval/live/difficulty-registry.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"difficulty-registry.js","sourceRoot":"","sources":["../../../src/eval/live/difficulty-registry.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACzF,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AA+BtC,MAAM,cAAc,GAAG,aAAa,CAAC;AACrC,4EAA4E;AAC5E,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAC3B,4EAA4E;AAC5E,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAC5B,+EAA+E;AAC/E,MAAM,YAAY,GAAG,IAAI,CAAC;AAE1B,MAAM,UAAU,uBAAuB;IACrC,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,SAAS,EAAE,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC;AAC9E,CAAC;AAED,iFAAiF;AACjF,MAAM,UAAU,sBAAsB,CAAC,IAAY;IACjD,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,uBAAuB,EAAE,CAAC;IACxD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAA2B,CAAC;IAC7E,IAAI,GAAG,CAAC,OAAO,KAAK,CAAC,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CAAC,2CAA2C,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC;IAC/F,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,iFAAiF;AACjF,MAAM,UAAU,sBAAsB,CAAC,IAAY,EAAE,QAAgC;IACnF,QAAQ,CAAC,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC9C,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9C,MAAM,GAAG,GAAG,GAAG,IAAI,MAAM,CAAC;IAC1B,aAAa,CAAC,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IACtD,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;AACxB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,aAAa,CAC3B,QAAgC,EAChC,SAAiB,EACjB,IAAiB;IAEjB,MAAM,KAAK,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;IACpE,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,GAAG,CAAC,MAAM,KAAK,WAAW;YAAE,SAAS;QACzC,MAAM,KAAK,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK;YAC5C,QAAQ,EAAE,CAAC;YACX,SAAS,EAAE,CAAC;YACZ,YAAY,EAAE,CAAC;YACf,aAAa,EAAE,CAAC;YAChB,IAAI,EAAE,CAAC;SACR,CAAC,CAAC;QACH,KAAK,CAAC,QAAQ,EAAE,CAAC;QACjB,IAAI,GAAG,CAAC,OAAO;YAAE,KAAK,CAAC,SAAS,EAAE,CAAC;QACnC,IAAI,GAAG,CAAC,GAAG,KAAK,cAAc,EAAE,CAAC;YAC/B,KAAK,CAAC,YAAY,EAAE,CAAC;YACrB,IAAI,GAAG,CAAC,OAAO;gBAAE,KAAK,CAAC,aAAa,EAAE,CAAC;QACzC,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAC1B,CAAC;IACD,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;IAChC,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,iBAAiB,CAAC,GAA4B;IAI5D,MAAM,IAAI,GAAgB,EAAE,CAAC;IAE7B,IAAI,GAAG,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;QACzB,MAAM,IAAI,GAAG,GAAgC,CAAC;QAC9C,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC;YACjC,IAAI,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;QACtF,CAAC;QACD,MAAM,IAAI,GAAG,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,CAA4B,CAAC;QACzD,OAAO,EAAE,SAAS,EAAE,OAAO,IAAI,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,EAAE,IAAI,EAAE,CAAC;IAC9F,CAAC;IAED,MAAM,OAAO,GAAG,CAAC,GAAW,EAAE,SAAkB,EAAQ,EAAE;QACxD,MAAM,OAAO,GAAI,SAAsD,EAAE,OAAO,IAAI,EAAE,CAAC;QACvF,KAAK,MAAM,QAAQ,IAAI,OAAO,EAAE,CAAC;YAC/B,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;gBACzB,IAAI,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;YAC/E,CAAC;QACH,CAAC;IACH,CAAC,CAAC;IACF,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC;IAClC,OAAO,CAAC,cAAc,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC;IACxC,OAAO;QACL,SAAS,EAAE,OAAO,GAAG,CAAC,aAAa,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS;QAChF,IAAI;KACL,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,qBAAqB,CAAC,KAA4B;IAChE,MAAM,OAAO,GAAG,KAAK,CAAC,YAAY,IAAI,iBAAiB,CAAC;IACxD,MAAM,QAAQ,GAAG,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC;IAC/D,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC;IAClE,OAAO,CAAC,SAAS,GAAG,CAAC,CAAC,GAAG,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC;AAC1C,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,oBAAoB,CAClC,KAAwC,EACxC,OAAO,GAAG,gBAAgB;IAE1B,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,IAAI,GAAG,OAAO;QAAE,OAAO,CAAC,CAAC;IAC7C,MAAM,CAAC,GAAG,qBAAqB,CAAC,KAAK,CAAC,CAAC;IACvC,OAAO,IAAI,CAAC,GAAG,CAAC,YAAY,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AACjD,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,yBAAyB,CACvC,IAAgB,EAChB,KAAa,EACb,QAAgC,EAChC,SAAiB,EACjB,IAAY,EACZ,OAAO,GAAG,gBAAgB;IAE1B,IAAI,IAAI,CAAC,MAAM,IAAI,KAAK;QAAE,OAAO,IAAI,CAAC;IACtC,MAAM,QAAQ,GAAG,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,QAAQ,IAAI,EAAE,CAAC;IAChE,MAAM,GAAG,GAAG,SAAS,CAAC,mBAAmB,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;IAE5D,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE;QACvC,MAAM,CAAC,GAAG,oBAAoB,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAC,CAAC;QACjE,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;QAC5C,OAAO,EAAE,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;IAC9C,CAAC,CAAC,CAAC;IACH,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;IACpC,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAEpE,OAAO,IAAI;SACR,MAAM,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;SAC3C,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;AAChD,CAAC"}

package/dist/eval/live/docker.d.ts

@@ -0,0 +1,52 @@
+/**
+ * Docker orchestration for benchmark evaluation.
+ *
+ * Shells out to the `docker` CLI (same subprocess pattern as claude-cli.ts).
+ * Each benchmark task gets its own container to prevent state leakage.
+ */
+export interface DockerTaskConfig {
+    image: string;
+    mounts?: Array<{
+        hostPath: string;
+        containerPath: string;
+    }>;
+    env?: Record<string, string>;
+    workdir?: string;
+    timeoutMs?: number;
+}
+export interface DockerRunResult {
+    exitCode: number;
+    stdout: string;
+    stderr: string;
+    durationMs: number;
+}
+export declare class DockerOrchestrator {
+    private readonly opts;
+    private readonly builtImages;
+    constructor(opts?: {
+        pullPolicy?: 'always' | 'if-not-present';
+    });
+    buildImage(dockerfilePath: string, tag: string, context?: string): Promise<string>;
+    run(config: DockerTaskConfig, command: string[]): Promise<DockerRunResult>;
+    copyToContainer(containerId: string, hostPath: string, containerPath: string): Promise<void>;
+    runTests(config: DockerTaskConfig, testCommand: string[]): Promise<{
+        passed: boolean;
+        output: string;
+    }>;
+    cleanup(tag: string): Promise<void>;
+    startPersistentContainer(config: DockerTaskConfig & {
+        name: string;
+    }): Promise<string>;
+    execInContainer(name: string, command: string, timeoutMs?: number): Promise<DockerRunResult>;
+    copyInto(name: string, hostPath: string, containerPath: string): Promise<void>;
+    stopAndRemove(name: string): Promise<void>;
+    private exec;
+    private execWithResult;
+    private execOnce;
+}
+export declare function createDockerOrchestrator(opts?: {
+    pullPolicy?: 'always' | 'if-not-present';
+}): DockerOrchestrator;
+/** Quick check whether the `docker` binary is available. */
+export declare function dockerAvailable(): Promise<boolean>;
+//# sourceMappingURL=docker.d.ts.map

package/dist/eval/live/docker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"docker.d.ts","sourceRoot":"","sources":["../../../src/eval/live/docker.ts"],"names":[],"mappings":"AAEA;;;;;GAKG;AAEH,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,KAAK,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,aAAa,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC5D,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,qBAAa,kBAAkB;IAGjB,OAAO,CAAC,QAAQ,CAAC,IAAI;IAFjC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAqB;gBAEpB,IAAI,GAAE;QAAE,UAAU,CAAC,EAAE,QAAQ,GAAG,gBAAgB,CAAA;KAAO;IAE9E,UAAU,CAAC,cAAc,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAUlF,GAAG,CAAC,MAAM,EAAE,gBAAgB,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,CAAC;IAiB1E,eAAe,CACnB,WAAW,EAAE,MAAM,EACnB,QAAQ,EAAE,MAAM,EAChB,aAAa,EAAE,MAAM,GACpB,OAAO,CAAC,IAAI,CAAC;IAIV,QAAQ,CACZ,MAAM,EAAE,gBAAgB,EACxB,WAAW,EAAE,MAAM,EAAE,GACpB,OAAO,CAAC;QAAE,MAAM,EAAE,OAAO,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAQzC,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IASnC,wBAAwB,CAAC,MAAM,EAAE,gBAAgB,GAAG;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,MAAM,CAAC;IAqBtF,eAAe,CACnB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,MAAM,EACf,SAAS,SAAU,GAClB,OAAO,CAAC,eAAe,CAAC;IAIrB,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAO9E,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAahD,OAAO,CAAC,IAAI;YAIE,cAAc;IAmB5B,OAAO,CAAC,QAAQ;CA0CjB;AAED,wBAAgB,wBAAwB,CAAC,IAAI,CAAC,EAAE;IAC9C,UAAU,CAAC,EAAE,QAAQ,GAAG,gBAAgB,CAAC;CAC1C,GAAG,kBAAkB,CAErB;AAED,4DAA4D;AAC5D,wBAAsB,eAAe,IAAI,OAAO,CAAC,OAAO,CAAC,CAMxD"}

package/dist/eval/live/docker.js

@@ -0,0 +1,164 @@
+import { spawn } from 'node:child_process';
+export class DockerOrchestrator {
+    opts;
+    builtImages = new Set();
+    constructor(opts = {}) {
+        this.opts = opts;
+    }
+    async buildImage(dockerfilePath, tag, context) {
+        if (this.opts.pullPolicy === 'if-not-present' && this.builtImages.has(tag)) {
+            return tag;
+        }
+        const args = ['build', '-t', tag, '-f', dockerfilePath, context ?? '.'];
+        await this.exec(args, 600_000);
+        this.builtImages.add(tag);
+        return tag;
+    }
+    async run(config, command) {
+        const args = ['run', '--rm'];
+        if (config.workdir)
+            args.push('-w', config.workdir);
+        if (config.mounts) {
+            for (const m of config.mounts) {
+                args.push('-v', `${m.hostPath}:${m.containerPath}`);
+            }
+        }
+        if (config.env) {
+            for (const [k, v] of Object.entries(config.env)) {
+                args.push('-e', `${k}=${v}`);
+            }
+        }
+        args.push(config.image, ...command);
+        return this.execWithResult(args, config.timeoutMs ?? 300_000);
+    }
+    async copyToContainer(containerId, hostPath, containerPath) {
+        await this.exec(['cp', hostPath, `${containerId}:${containerPath}`], 30_000);
+    }
+    async runTests(config, testCommand) {
+        const result = await this.run(config, testCommand);
+        return {
+            passed: result.exitCode === 0,
+            output: result.stdout + (result.stderr ? '\n' + result.stderr : ''),
+        };
+    }
+    async cleanup(tag) {
+        try {
+            await this.exec(['rmi', '-f', tag], 30_000);
+        }
+        catch {
+            // best-effort cleanup
+        }
+        this.builtImages.delete(tag);
+    }
+    async startPersistentContainer(config) {
+        const args = ['run', '-d', '--name', config.name];
+        if (config.workdir)
+            args.push('-w', config.workdir);
+        if (config.mounts) {
+            for (const m of config.mounts) {
+                args.push('-v', `${m.hostPath}:${m.containerPath}`);
+            }
+        }
+        if (config.env) {
+            for (const [k, v] of Object.entries(config.env)) {
+                args.push('-e', `${k}=${v}`);
+            }
+        }
+        args.push(config.image, 'tail', '-f', '/dev/null');
+        const result = await this.execOnce(args, config.timeoutMs ?? 60_000);
+        if (result.exitCode !== 0) {
+            throw new Error(`Failed to start container ${config.name}: ${result.stderr}`);
+        }
+        return config.name;
+    }
+    async execInContainer(name, command, timeoutMs = 120_000) {
+        return this.execOnce(['exec', name, 'bash', '-c', command], timeoutMs);
+    }
+    async copyInto(name, hostPath, containerPath) {
+        const result = await this.execOnce(['cp', hostPath, `${name}:${containerPath}`], 60_000);
+        if (result.exitCode !== 0) {
+            throw new Error(`docker cp failed: ${result.stderr}`);
+        }
+    }
+    async stopAndRemove(name) {
+        try {
+            await this.execOnce(['stop', '-t', '5', name], 15_000);
+        }
+        catch {
+            /* best effort */
+        }
+        try {
+            await this.execOnce(['rm', '-f', name], 15_000);
+        }
+        catch {
+            /* best effort */
+        }
+    }
+    exec(args, timeoutMs, retries = 2) {
+        return this.execWithResult(args, timeoutMs, retries).then(() => { });
+    }
+    async execWithResult(args, timeoutMs, retries = 2) {
+        let lastErr;
+        for (let attempt = 0; attempt <= retries; attempt++) {
+            try {
+                return await this.execOnce(args, timeoutMs);
+            }
+            catch (err) {
+                lastErr = err;
+                if (attempt < retries) {
+                    await new Promise((r) => setTimeout(r, 3000 * (attempt + 1)));
+                }
+            }
+        }
+        throw lastErr ?? new Error('docker command failed');
+    }
+    execOnce(args, timeoutMs) {
+        return new Promise((resolve, reject) => {
+            const t0 = Date.now();
+            const child = spawn('docker', args, { stdio: ['pipe', 'pipe', 'pipe'] });
+            let stdout = '';
+            let stderr = '';
+            let settled = false;
+            const timer = setTimeout(() => {
+                if (settled)
+                    return;
+                settled = true;
+                child.kill('SIGKILL');
+                reject(new Error(`docker command timed out after ${timeoutMs}ms: docker ${args.slice(0, 3).join(' ')}...`));
+            }, timeoutMs);
+            child.stdout.on('data', (d) => (stdout += d.toString()));
+            child.stderr.on('data', (d) => (stderr += d.toString()));
+            child.on('error', (err) => {
+                if (settled)
+                    return;
+                settled = true;
+                clearTimeout(timer);
+                reject(new Error(`Failed to spawn docker: ${err.message}`));
+            });
+            child.on('close', (code) => {
+                if (settled)
+                    return;
+                settled = true;
+                clearTimeout(timer);
+                resolve({
+                    exitCode: code ?? 1,
+                    stdout,
+                    stderr,
+                    durationMs: Date.now() - t0,
+                });
+            });
+        });
+    }
+}
+export function createDockerOrchestrator(opts) {
+    return new DockerOrchestrator(opts);
+}
+/** Quick check whether the `docker` binary is available. */
+export async function dockerAvailable() {
+    return new Promise((resolve) => {
+        const child = spawn('docker', ['version'], { stdio: 'ignore' });
+        child.on('error', () => resolve(false));
+        child.on('close', (code) => resolve(code === 0));
+    });
+}
+//# sourceMappingURL=docker.js.map

package/dist/eval/live/docker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"docker.js","sourceRoot":"","sources":["../../../src/eval/live/docker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAwB3C,MAAM,OAAO,kBAAkB;IAGA;IAFZ,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;IAEjD,YAA6B,OAAqD,EAAE;QAAvD,SAAI,GAAJ,IAAI,CAAmD;IAAG,CAAC;IAExF,KAAK,CAAC,UAAU,CAAC,cAAsB,EAAE,GAAW,EAAE,OAAgB;QACpE,IAAI,IAAI,CAAC,IAAI,CAAC,UAAU,KAAK,gBAAgB,IAAI,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAC3E,OAAO,GAAG,CAAC;QACb,CAAC;QACD,MAAM,IAAI,GAAG,CAAC,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,cAAc,EAAE,OAAO,IAAI,GAAG,CAAC,CAAC;QACxE,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAC/B,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC1B,OAAO,GAAG,CAAC;IACb,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,MAAwB,EAAE,OAAiB;QACnD,MAAM,IAAI,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QAC7B,IAAI,MAAM,CAAC,OAAO;YAAE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;QACpD,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YAClB,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;gBAC9B,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,aAAa,EAAE,CAAC,CAAC;YACtD,CAAC;QACH,CAAC;QACD,IAAI,MAAM,CAAC,GAAG,EAAE,CAAC;YACf,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;gBAChD,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,GAAG,OAAO,CAAC,CAAC;QACpC,OAAO,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,MAAM,CAAC,SAAS,IAAI,OAAO,CAAC,CAAC;IAChE,CAAC;IAED,KAAK,CAAC,eAAe,CACnB,WAAmB,EACnB,QAAgB,EAChB,aAAqB;QAErB,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,QAAQ,EAAE,GAAG,WAAW,IAAI,aAAa,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;IAC/E,CAAC;IAED,KAAK,CAAC,QAAQ,CACZ,MAAwB,EACxB,WAAqB;QAErB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;QACnD,OAAO;YACL,MAAM,EAAE,MAAM,CAAC,QAAQ,KAAK,CAAC;YAC7B,MAAM,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;SACpE,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,GAAW;QACvB,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,CAAC,EAAE,MAAM,CAAC,CAAC;QAC9C,CAAC;QAAC,MAAM,CAAC;YACP,sBAAsB;QACxB,CAAC;QACD,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAC/B,CAAC;IAED,KAAK,CAAC,wBAAwB,CAAC,MAA2C;QACxE,MAAM,IAAI,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;QAClD,IAAI,MAAM,CAAC,OAAO;YAAE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;QACpD,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YAClB,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;gBAC9B,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,aAAa,EAAE,CAAC,CAAC;YACtD,CAAC;QACH,CAAC;QACD,IAAI,MAAM,CAAC,GAAG,EAAE,CAAC;YACf,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;gBAChD,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,CAAC,CAAC;QACnD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,CAAC;QACrE,IAAI,MAAM,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CAAC,6BAA6B,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QAChF,CAAC;QACD,OAAO,MAAM,CAAC,IAAI,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,eAAe,CACnB,IAAY,EACZ,OAAe,EACf,SAAS,GAAG,OAAO;QAEnB,OAAO,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,CAAC,EAAE,SAAS,CAAC,CAAC;IACzE,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,IAAY,EAAE,QAAgB,EAAE,aAAqB;QAClE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,QAAQ,EAAE,GAAG,IAAI,IAAI,aAAa,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;QACzF,IAAI,MAAM,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CAAC,qBAAqB,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QACxD,CAAC;IACH,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,IAAY;QAC9B,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC;QACzD,CAAC;QAAC,MAAM,CAAC;YACP,iBAAiB;QACnB,CAAC;QACD,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC;QAClD,CAAC;QAAC,MAAM,CAAC;YACP,iBAAiB;QACnB,CAAC;IACH,CAAC;IAEO,IAAI,CAAC,IAAc,EAAE,SAAiB,EAAE,OAAO,GAAG,CAAC;QACzD,OAAO,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACtE,CAAC;IAEO,KAAK,CAAC,cAAc,CAC1B,IAAc,EACd,SAAiB,EACjB,OAAO,GAAG,CAAC;QAEX,IAAI,OAA0B,CAAC;QAC/B,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;YACpD,IAAI,CAAC;gBACH,OAAO,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;YAC9C,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,GAAG,GAAY,CAAC;gBACvB,IAAI,OAAO,GAAG,OAAO,EAAE,CAAC;oBACtB,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,IAAI,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;gBAChE,CAAC;YACH,CAAC;QACH,CAAC;QACD,MAAM,OAAO,IAAI,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;IACtD,CAAC;IAEO,QAAQ,CAAC,IAAc,EAAE,SAAiB;QAChD,OAAO,IAAI,OAAO,CAAkB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACtD,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACtB,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;YACzE,IAAI,MAAM,GAAG,EAAE,CAAC;YAChB,IAAI,MAAM,GAAG,EAAE,CAAC;YAChB,IAAI,OAAO,GAAG,KAAK,CAAC;YAEpB,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;gBAC5B,IAAI,OAAO;oBAAE,OAAO;gBACpB,OAAO,GAAG,IAAI,CAAC;gBACf,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBACtB,MAAM,CACJ,IAAI,KAAK,CACP,kCAAkC,SAAS,cAAc,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CACzF,CACF,CAAC;YACJ,CAAC,EAAE,SAAS,CAAC,CAAC;YAEd,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;YACzD,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;YAEzD,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;gBACxB,IAAI,OAAO;oBAAE,OAAO;gBACpB,OAAO,GAAG,IAAI,CAAC;gBACf,YAAY,CAAC,KAAK,CAAC,CAAC;gBACpB,MAAM,CAAC,IAAI,KAAK,CAAC,2BAA2B,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YAC9D,CAAC,CAAC,CAAC;YAEH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE;gBACzB,IAAI,OAAO;oBAAE,OAAO;gBACpB,OAAO,GAAG,IAAI,CAAC;gBACf,YAAY,CAAC,KAAK,CAAC,CAAC;gBACpB,OAAO,CAAC;oBACN,QAAQ,EAAE,IAAI,IAAI,CAAC;oBACnB,MAAM;oBACN,MAAM;oBACN,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE;iBAC5B,CAAC,CAAC;YACL,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;CACF;AAED,MAAM,UAAU,wBAAwB,CAAC,IAExC;IACC,OAAO,IAAI,kBAAkB,CAAC,IAAI,CAAC,CAAC;AACtC,CAAC;AAED,4DAA4D;AAC5D,MAAM,CAAC,KAAK,UAAU,eAAe;IACnC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC,SAAS,CAAC,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;QAChE,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;QACxC,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/eval/live/env-bridge.d.ts

@@ -0,0 +1,68 @@
+/**
+ * JSONL bridge to an external (Python) text-environment process, e.g. ALFWorld.
+ *
+ * Protocol (one JSON object per line, request → response):
+ *   → {"cmd":"list"}                       ← {"tasks":["...", ...]}
+ *   → {"cmd":"reset","task":"<id>"}        ← {"obs":"...", "admissible":[...], "done":false}
+ *   → {"cmd":"step","action":"go to ..."}  ← {"obs":"...", "reward":0|1, "done":bool, "admissible":[...]}
+ *
+ * The bridge process is supplied via the ALFWORLD_BRIDGE env var (a command
+ * line) or defaults to `python <repo>/src/eval/live/bridges/alfworld_bridge.py`.
+ * This keeps cognitive-core free of any Python/ALFWorld dependency — the bridge
+ * is opt-in and only needed for the ALFWorld live arm.
+ */
+export declare class EnvBridge {
+    private readonly command;
+    private readonly args;
+    private readonly requestTimeoutMs;
+    /** Extra env vars merged over process.env for the spawned bridge process.
+     * Used by the SkillFlow e2b runtime to inject a computed sandbox timeout. */
+    private readonly env?;
+    private proc;
+    private rl;
+    private queue;
+    private dead;
+    constructor(command: string, args: string[], requestTimeoutMs?: number, 
+    /** Extra env vars merged over process.env for the spawned bridge process.
+     * Used by the SkillFlow e2b runtime to inject a computed sandbox timeout. */
+    env?: Record<string, string> | undefined);
+    /** Start the bridge process. Waits for a ping handshake to confirm readiness. */
+    start(): Promise<void>;
+    protected request<T>(payload: Record<string, unknown>): Promise<T>;
+    listTasks(): Promise<{
+        tasks: string[];
+    }>;
+    reset(task: string): Promise<EnvObservation>;
+    step(action: string): Promise<EnvObservation>;
+    /**
+     * Send an arbitrary command to the bridge. Subclasses (e.g. SkillsBench,
+     * SkillFlow) use this to extend the JSONL protocol with benchmark-specific
+     * commands without duplicating the request plumbing.
+     */
+    send<T>(payload: Record<string, unknown>): Promise<T>;
+    close(): Promise<void>;
+}
+/**
+ * Thrown when the Python bridge reports an env-side crash via the `error` field.
+ * Distinguishes infra flakiness from genuine agent failures so the runner can
+ * mark the task as `env_error` instead of `failure`.
+ */
+export declare class EnvBridgeError extends Error {
+    readonly isEnvBridgeError = true;
+    constructor(message: string);
+}
+export interface EnvObservation {
+    obs: string;
+    reward?: number;
+    done?: boolean;
+    admissible?: string[];
+    /** Set by the bridge when the env raised — distinguishes infra error from a
+     * genuine task failure (the bridge still emits done:true so the loop ends). */
+    error?: string;
+}
+/** Resolve the bridge command from ALFWORLD_BRIDGE or a default python invocation. */
+export declare function resolveBridgeCommand(defaultScriptPath: string): {
+    command: string;
+    args: string[];
+};
+//# sourceMappingURL=env-bridge.d.ts.map

package/dist/eval/live/env-bridge.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"env-bridge.d.ts","sourceRoot":"","sources":["../../../src/eval/live/env-bridge.ts"],"names":[],"mappings":"AAIA;;;;;;;;;;;;GAYG;AACH,qBAAa,SAAS;IAOlB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,gBAAgB;IACjC;iFAC6E;IAC7E,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC;IAXvB,OAAO,CAAC,IAAI,CAA8D;IAC1E,OAAO,CAAC,EAAE,CAA0B;IACpC,OAAO,CAAC,KAAK,CAAgF;IAC7F,OAAO,CAAC,IAAI,CAAS;gBAGF,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,MAAM,EAAE,EACd,gBAAgB,SAAU;IAC3C;iFAC6E;IAC5D,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,YAAA;IAG/C,iFAAiF;IAC3E,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IA8B5B,SAAS,CAAC,OAAO,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IA+ClE,SAAS,IAAI,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;IAIzC,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC;IAI5C,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC;IAI7C;;;;OAIG;IACH,IAAI,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IAI/C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAoC7B;AAED;;;;GAIG;AACH,qBAAa,cAAe,SAAQ,KAAK;IACvC,QAAQ,CAAC,gBAAgB,QAAQ;gBACrB,OAAO,EAAE,MAAM;CAI5B;AAED,MAAM,WAAW,cAAc;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB;mFAC+E;IAC/E,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,sFAAsF;AACtF,wBAAgB,oBAAoB,CAAC,iBAAiB,EAAE,MAAM,GAAG;IAC/D,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,EAAE,CAAC;CAChB,CAOA"}