cognitive-core 0.2.4 → 0.3.1

package/dist/eval/live/claude-cli.js

@@ -0,0 +1,314 @@
+import { spawn } from 'node:child_process';
+import { existsSync } from 'node:fs';
+import { delimiter, dirname } from 'node:path';
+import { generateResponsesText, isResponsesProvider } from 'swarmkit-eval';
+/** Tools disabled when `disallowTools` is set — forces a single-inference step. */
+const DISALLOWED_TOOLS = [
+    'Bash',
+    'Read',
+    'Write',
+    'Edit',
+    'NotebookEdit',
+    'Glob',
+    'Grep',
+    'WebFetch',
+    'WebSearch',
+    'Task',
+    'TodoWrite',
+];
+const CLAUDE_COMMAND_FALLBACKS = [
+    '/usr/local/bin/claude',
+    '/opt/homebrew/bin/claude',
+    '/Applications/cmux.app/Contents/Resources/bin/claude',
+];
+export function resolveClaudeCommand() {
+    const configured = process.env.CLAUDE_CLI_PATH;
+    if (configured && existsSync(configured))
+        return configured;
+    for (const candidate of CLAUDE_COMMAND_FALLBACKS) {
+        if (existsSync(candidate))
+            return candidate;
+    }
+    return 'claude';
+}
+export function claudePathEntries(command = resolveClaudeCommand()) {
+    const entries = new Set();
+    if (command !== 'claude')
+        entries.add(dirname(command));
+    for (const candidate of CLAUDE_COMMAND_FALLBACKS) {
+        if (existsSync(candidate))
+            entries.add(dirname(candidate));
+    }
+    return [...entries];
+}
+export function buildClaudeChildEnv(base, command = resolveClaudeCommand()) {
+    const pathEntries = claudePathEntries(command);
+    const existingPath = base.PATH ? [base.PATH] : [];
+    return {
+        ...base,
+        CLAUDE_CLI_PATH: command !== 'claude' ? command : base.CLAUDE_CLI_PATH,
+        PATH: [...pathEntries, ...existingPath].filter(Boolean).join(delimiter),
+    };
+}
+/**
+ * Friendly model name → Bedrock cross-region inference-profile id. Used for
+ * ANTHROPIC_MODEL when `bedrock` is set. Unknown names pass through (so a full
+ * profile id can be supplied directly). Ids verified against the account's
+ * enabled profiles (us-east-1).
+ */
+export const BEDROCK_MODELS = {
+    haiku: 'us.anthropic.claude-haiku-4-5-20251001-v1:0',
+    sonnet: 'us.anthropic.claude-sonnet-4-6',
+    opus: 'us.anthropic.claude-opus-4-1-20250805-v1:0',
+};
+/**
+ * Cross-region inference profiles are geo-prefixed (`us.`, `eu.`, `apac.`) and
+ * a profile only resolves from a region in its own geo. Daily token quotas are
+ * per calling region, so being able to point runs at eu-* regions matters when
+ * the us-* quotas are exhausted.
+ */
+function bedrockGeoPrefix(region) {
+    if (region.startsWith('eu-'))
+        return 'eu';
+    if (region.startsWith('ap-'))
+        return 'apac';
+    return 'us';
+}
+export function bedrockModelId(name, region) {
+    const id = BEDROCK_MODELS[name] ?? name;
+    const prefix = bedrockGeoPrefix(region ?? 'us-east-1');
+    return prefix === 'us' ? id : id.replace(/^us\./, `${prefix}.`);
+}
+/**
+ * Run the Claude CLI with retry-on-failure. A single hung/throttled `claude -p`
+ * call should not abort an entire multi-hour eval; we retry a couple of times
+ * with backoff before giving up.
+ */
+export async function runClaude(prompt, opts = {}) {
+    const retries = opts.retries ?? 2;
+    let lastErr;
+    let nextCommandOverride = opts.commandOverride;
+    for (let attempt = 0; attempt <= retries; attempt++) {
+        try {
+            const attemptOpts = nextCommandOverride
+                ? { ...opts, commandOverride: nextCommandOverride }
+                : opts;
+            return await (isResponsesProvider(opts.provider)
+                ? runResponsesOnce(prompt, attemptOpts)
+                : runClaudeOnce(prompt, attemptOpts));
+        }
+        catch (err) {
+            lastErr = err;
+            // Surface the failure cause so Bedrock throttling/timeouts are visible
+            // instead of being silently retried away.
+            console.error(`[claude-cli] attempt ${attempt + 1}/${retries + 1} failed: ${lastErr.message.slice(0, 300)}`);
+            if (isClaudePathLookupError(lastErr)) {
+                const fallback = alternateClaudeCommand(nextCommandOverride ?? resolveClaudeCommand());
+                if (fallback) {
+                    nextCommandOverride = fallback;
+                    console.error(`[claude-cli] retrying with alternate Claude binary: ${fallback}`);
+                }
+            }
+            if (attempt < retries) {
+                // Linear backoff: 5s, 10s — gives a throttled CLI room to recover.
+                await new Promise((r) => setTimeout(r, 5000 * (attempt + 1)));
+            }
+        }
+    }
+    throw lastErr ?? new Error('claude CLI failed');
+}
+/**
+ * Bridge the Azure credential naming gap for the Responses solver/learning path.
+ *
+ * swarmkit-eval's `generateResponsesText` only reads the canonical
+ * `AZURE_OPENAI_*` env names, but this environment commonly exports the shorter
+ * `AZURE_API_*` aliases (endpoint/version/key). Resolve both, canonical names
+ * winning, and pass them explicitly so `--provider azure` works with whichever
+ * credential set is present. Deployment stays `AZURE_OPENAI_DEPLOYMENT` only;
+ * when unset the caller's `--model` becomes the deployment name downstream.
+ */
+export function resolveAzureResponsesEnv(env = process.env) {
+    const apiKey = env.AZURE_OPENAI_API_KEY ?? env.AZURE_OPENAI_KEY ?? env.AZURE_API_KEY;
+    const azureEndpoint = env.AZURE_OPENAI_ENDPOINT ?? env.AZURE_OPENAI_BASE_URL ?? env.AZURE_API_BASE;
+    const azureApiVersion = env.AZURE_OPENAI_API_VERSION ?? env.AZURE_API_VERSION;
+    const azureDeployment = env.AZURE_OPENAI_DEPLOYMENT;
+    const out = {};
+    if (apiKey)
+        out.apiKey = apiKey;
+    if (azureEndpoint)
+        out.azureEndpoint = azureEndpoint;
+    if (azureApiVersion)
+        out.azureApiVersion = azureApiVersion;
+    if (azureDeployment)
+        out.azureDeployment = azureDeployment;
+    return out;
+}
+async function runResponsesOnce(prompt, opts = {}) {
+    const azure = opts.provider === 'azure' || opts.provider === 'azure-openai' ? resolveAzureResponsesEnv() : {};
+    const result = await generateResponsesText({
+        provider: opts.provider,
+        model: opts.model ?? 'gpt-5.5',
+        input: prompt,
+        maxOutputTokens: Number(process.env.OPENAI_MAX_OUTPUT_TOKENS) || undefined,
+        reasoningEffort: parseReasoningEffort(process.env.OPENAI_REASONING_EFFORT),
+        timeoutMs: opts.timeoutMs,
+        ...azure,
+    });
+    return {
+        text: result.text,
+        ok: true,
+    };
+}
+function parseReasoningEffort(value) {
+    if (value === 'minimal' || value === 'low' || value === 'medium' || value === 'high') {
+        return value;
+    }
+    return undefined;
+}
+/**
+ * Invoke `claude -p --output-format json` with `prompt` on stdin and parse the
+ * single JSON result. Rejects on spawn error, non-zero exit, timeout, or
+ * unparseable output.
+ */
+function runClaudeOnce(prompt, opts = {}) {
+    // Always isolate from host plugins/settings. Otherwise every `claude -p` loads
+    // the user's global plugins (omc/swarm), each spawning ~5 MCP subprocesses that
+    // leak and hang at MCP-init — the dominant cause of multi-minute eval stalls.
+    // --strict-mcp-config (no host MCP) + empty --setting-sources keeps eval calls
+    // clean and fast (auth via ~/.claude / Bedrock env is unaffected).
+    const args = ['-p', '--output-format', 'json', '--strict-mcp-config', '--setting-sources', ''];
+    const command = opts.commandOverride ?? resolveClaudeCommand();
+    const env = buildClaudeChildEnv(process.env, command);
+    // `provider: 'bedrock'` must take the SAME env-var routing as `bedrock: true`.
+    // The old behavior passed `--settings '{"apiProvider":"bedrock"}'` — which is
+    // not a Claude Code setting, so the CLI silently routed to the user's
+    // Anthropic subscription instead of AWS. Found 2026-07-02 when "bedrock"
+    // SkillFlow runs died with a subscription 429 ("session limit") — every
+    // adapter that passed provider without bedrock (skillflow, clbench,
+    // skillsbench) had been burning the subscription.
+    const useBedrock = opts.bedrock || opts.provider === 'bedrock';
+    if (useBedrock) {
+        // Env-var Bedrock routing; DROP --model (CLI aliases are invalid Bedrock ids,
+        // → 400). Route the model via ANTHROPIC_MODEL = inference-profile id.
+        env.CLAUDE_CODE_USE_BEDROCK = '1';
+        env.AWS_REGION = opts.awsRegion ?? process.env.AWS_REGION ?? 'us-east-1';
+        if (opts.model)
+            env.ANTHROPIC_MODEL = bedrockModelId(opts.model, env.AWS_REGION);
+    }
+    else if (opts.model) {
+        args.push('--model', opts.model);
+    }
+    if (opts.resumeSessionId)
+        args.push('--resume', opts.resumeSessionId);
+    if (opts.permissionMode) {
+        args.push('--permission-mode', opts.permissionMode);
+    }
+    if (opts.allowedTools && opts.allowedTools.length > 0) {
+        args.push('--allowedTools', ...opts.allowedTools);
+    }
+    if (opts.provider && !useBedrock) {
+        // Legacy settings pass-through for non-bedrock providers. NOTE: this is
+        // dubious — `apiProvider` is not a documented Claude Code setting — but
+        // responses-providers (openai/azure) never reach runClaudeOnce, so this
+        // only fires for exotic values. Bedrock is env-routed above.
+        const settings = { apiProvider: opts.provider };
+        args.push('--settings', JSON.stringify(settings));
+    }
+    if (opts.disallowTools) {
+        args.push('--disallowedTools', ...DISALLOWED_TOOLS);
+    }
+    const timeoutMs = opts.timeoutMs ?? 90_000;
+    return new Promise((resolve, reject) => {
+        const child = spawn(command, args, {
+            stdio: ['pipe', 'pipe', 'pipe'],
+            env,
+            cwd: opts.cwd,
+        });
+        let stdout = '';
+        let stderr = '';
+        let settled = false;
+        const timer = setTimeout(() => {
+            if (settled)
+                return;
+            settled = true;
+            child.kill('SIGKILL');
+            reject(new Error(`claude CLI timed out after ${timeoutMs}ms`));
+        }, timeoutMs);
+        child.stdout.on('data', (d) => (stdout += d.toString()));
+        child.stderr.on('data', (d) => (stderr += d.toString()));
+        child.on('error', (err) => {
+            if (settled)
+                return;
+            settled = true;
+            clearTimeout(timer);
+            reject(new Error(`Failed to spawn claude CLI: ${err.message}`));
+        });
+        child.on('close', (code) => {
+            if (settled)
+                return;
+            settled = true;
+            clearTimeout(timer);
+            if (code !== 0) {
+                const errMsg = stderr.slice(0, 500);
+                // Nonzero exit with no stdout is an infra failure (CLI crash, auth
+                // error, rate-limit). Use a message pattern isEnvError() can match.
+                if (!stdout.trim()) {
+                    reject(new Error(`claude CLI exited ${code} with no output: ${errMsg}`));
+                    return;
+                }
+                reject(new Error(`claude CLI exited ${code}: ${errMsg}`));
+                return;
+            }
+            try {
+                const json = JSON.parse(stdout);
+                // Bedrock throttling/errors surface as api_error_status or is_error even
+                // on a clean exit. Log them so rate-limiting is diagnosable.
+                if (json.api_error_status || json.is_error === true) {
+                    const status = String(json.api_error_status ?? '');
+                    const subtype = String(json.subtype ?? '');
+                    console.error(`[claude-cli] API error (provider=${opts.provider ?? 'default'}): ` +
+                        `status=${status} subtype=${subtype} ` +
+                        `result=${String(json.result ?? '').slice(0, 200)}`);
+                    // 429 / 529 rate-limit or overload — surface as env_error-classifiable.
+                    if (status === '429' ||
+                        status === '529' ||
+                        /rate.?limit|too many requests|overloaded/i.test(subtype)) {
+                        reject(new Error(`claude CLI API ${status || subtype}: rate-limit/overloaded`));
+                        return;
+                    }
+                }
+                resolve({
+                    text: typeof json.result === 'string' ? json.result : '',
+                    sessionId: json.session_id,
+                    ok: json.is_error !== true && !json.api_error_status,
+                    costUsd: typeof json.total_cost_usd === 'number' ? json.total_cost_usd : undefined,
+                });
+            }
+            catch (err) {
+                reject(new Error(`Could not parse claude CLI JSON output: ${err.message}. ` +
+                    `First 300 chars: ${stdout.slice(0, 300)}`));
+            }
+        });
+        child.stdin.write(prompt);
+        child.stdin.end();
+    });
+}
+function isClaudePathLookupError(error) {
+    return /claude not found in PATH/i.test(error.message);
+}
+function alternateClaudeCommand(current) {
+    for (const candidate of CLAUDE_COMMAND_FALLBACKS) {
+        if (candidate !== current && existsSync(candidate))
+            return candidate;
+    }
+    return undefined;
+}
+/** Quick availability probe — true if the `claude` binary is on PATH. */
+export async function claudeAvailable() {
+    return new Promise((resolve) => {
+        const child = spawn(resolveClaudeCommand(), ['--version'], { stdio: 'ignore' });
+        child.on('error', () => resolve(false));
+        child.on('close', (code) => resolve(code === 0));
+    });
+}
+//# sourceMappingURL=claude-cli.js.map

package/dist/eval/live/claude-cli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"claude-cli.js","sourceRoot":"","sources":["../../../src/eval/live/claude-cli.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC/C,OAAO,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAC;AAoD3E,mFAAmF;AACnF,MAAM,gBAAgB,GAAG;IACvB,MAAM;IACN,MAAM;IACN,OAAO;IACP,MAAM;IACN,cAAc;IACd,MAAM;IACN,MAAM;IACN,UAAU;IACV,WAAW;IACX,MAAM;IACN,WAAW;CACZ,CAAC;AAEF,MAAM,wBAAwB,GAAG;IAC/B,uBAAuB;IACvB,0BAA0B;IAC1B,sDAAsD;CACvD,CAAC;AAEF,MAAM,UAAU,oBAAoB;IAClC,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC;IAC/C,IAAI,UAAU,IAAI,UAAU,CAAC,UAAU,CAAC;QAAE,OAAO,UAAU,CAAC;IAC5D,KAAK,MAAM,SAAS,IAAI,wBAAwB,EAAE,CAAC;QACjD,IAAI,UAAU,CAAC,SAAS,CAAC;YAAE,OAAO,SAAS,CAAC;IAC9C,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,OAAO,GAAG,oBAAoB,EAAE;IAChE,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,IAAI,OAAO,KAAK,QAAQ;QAAE,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;IACxD,KAAK,MAAM,SAAS,IAAI,wBAAwB,EAAE,CAAC;QACjD,IAAI,UAAU,CAAC,SAAS,CAAC;YAAE,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC;IAC7D,CAAC;IACD,OAAO,CAAC,GAAG,OAAO,CAAC,CAAC;AACtB,CAAC;AAED,MAAM,UAAU,mBAAmB,CACjC,IAAuB,EACvB,OAAO,GAAG,oBAAoB,EAAE;IAEhC,MAAM,WAAW,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAClD,OAAO;QACL,GAAG,IAAI;QACP,eAAe,EAAE,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe;QACtE,IAAI,EAAE,CAAC,GAAG,WAAW,EAAE,GAAG,YAAY,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC;KACxE,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,MAAM,cAAc,GAA2B;IACpD,KAAK,EAAE,6CAA6C;IACpD,MAAM,EAAE,gCAAgC;IACxC,IAAI,EAAE,4CAA4C;CACnD,CAAC;AAEF;;;;;GAKG;AACH,SAAS,gBAAgB,CAAC,MAAc;IACtC,IAAI,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAC1C,IAAI,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC;QAAE,OAAO,MAAM,CAAC;IAC5C,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,IAAY,EAAE,MAAe;IAC1D,MAAM,EAAE,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC;IACxC,MAAM,MAAM,GAAG,gBAAgB,CAAC,MAAM,IAAI,WAAW,CAAC,CAAC;IACvD,OAAO,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,MAAM,GAAG,CAAC,CAAC;AAClE,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,MAAc,EACd,OAAyB,EAAE;IAE3B,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC;IAClC,IAAI,OAA0B,CAAC;IAC/B,IAAI,mBAAmB,GAAG,IAAI,CAAC,eAAe,CAAC;IAC/C,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;QACpD,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,mBAAmB;gBACrC,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,eAAe,EAAE,mBAAmB,EAAE;gBACnD,CAAC,CAAC,IAAI,CAAC;YACT,OAAO,MAAM,CAAC,mBAAmB,CAAC,IAAI,CAAC,QAAQ,CAAC;gBAC9C,CAAC,CAAC,gBAAgB,CAAC,MAAM,EAAE,WAAW,CAAC;gBACvC,CAAC,CAAC,aAAa,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC,CAAC;QAC1C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,GAAG,GAAY,CAAC;YACvB,uEAAuE;YACvE,0CAA0C;YAC1C,OAAO,CAAC,KAAK,CACX,wBAAwB,OAAO,GAAG,CAAC,IAAI,OAAO,GAAG,CAAC,YAAY,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAC9F,CAAC;YACF,IAAI,uBAAuB,CAAC,OAAO,CAAC,EAAE,CAAC;gBACrC,MAAM,QAAQ,GAAG,sBAAsB,CAAC,mBAAmB,IAAI,oBAAoB,EAAE,CAAC,CAAC;gBACvF,IAAI,QAAQ,EAAE,CAAC;oBACb,mBAAmB,GAAG,QAAQ,CAAC;oBAC/B,OAAO,CAAC,KAAK,CAAC,uDAAuD,QAAQ,EAAE,CAAC,CAAC;gBACnF,CAAC;YACH,CAAC;YACD,IAAI,OAAO,GAAG,OAAO,EAAE,CAAC;gBACtB,mEAAmE;gBACnE,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,IAAI,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YAChE,CAAC;QACH,CAAC;IACH,CAAC;IACD,MAAM,OAAO,IAAI,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;AAClD,CAAC;AAUD;;;;;;;;;GASG;AACH,MAAM,UAAU,wBAAwB,CAAC,MAAyB,OAAO,CAAC,GAAG;IAC3E,MAAM,MAAM,GAAG,GAAG,CAAC,oBAAoB,IAAI,GAAG,CAAC,gBAAgB,IAAI,GAAG,CAAC,aAAa,CAAC;IACrF,MAAM,aAAa,GACjB,GAAG,CAAC,qBAAqB,IAAI,GAAG,CAAC,qBAAqB,IAAI,GAAG,CAAC,cAAc,CAAC;IAC/E,MAAM,eAAe,GAAG,GAAG,CAAC,wBAAwB,IAAI,GAAG,CAAC,iBAAiB,CAAC;IAC9E,MAAM,eAAe,GAAG,GAAG,CAAC,uBAAuB,CAAC;IAEpD,MAAM,GAAG,GAAsB,EAAE,CAAC;IAClC,IAAI,MAAM;QAAE,GAAG,CAAC,MAAM,GAAG,MAAM,CAAC;IAChC,IAAI,aAAa;QAAE,GAAG,CAAC,aAAa,GAAG,aAAa,CAAC;IACrD,IAAI,eAAe;QAAE,GAAG,CAAC,eAAe,GAAG,eAAe,CAAC;IAC3D,IAAI,eAAe;QAAE,GAAG,CAAC,eAAe,GAAG,eAAe,CAAC;IAC3D,OAAO,GAAG,CAAC;AACb,CAAC;AAED,KAAK,UAAU,gBAAgB,CAC7B,MAAc,EACd,OAAyB,EAAE;IAE3B,MAAM,KAAK,GACT,IAAI,CAAC,QAAQ,KAAK,OAAO,IAAI,IAAI,CAAC,QAAQ,KAAK,cAAc,CAAC,CAAC,CAAC,wBAAwB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAClG,MAAM,MAAM,GAAG,MAAM,qBAAqB,CAAC;QACzC,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,SAAS;QAC9B,KAAK,EAAE,MAAM;QACb,eAAe,EAAE,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,IAAI,SAAS;QAC1E,eAAe,EAAE,oBAAoB,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC;QAC1E,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,GAAG,KAAK;KACT,CAAC,CAAC;IACH,OAAO;QACL,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,EAAE,EAAE,IAAI;KACT,CAAC;AACJ,CAAC;AAaD,SAAS,oBAAoB,CAC3B,KAAyB;IAEzB,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,KAAK,IAAI,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,MAAM,EAAE,CAAC;QACrF,OAAO,KAAK,CAAC;IACf,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;;GAIG;AACH,SAAS,aAAa,CAAC,MAAc,EAAE,OAAyB,EAAE;IAChE,+EAA+E;IAC/E,gFAAgF;IAChF,8EAA8E;IAC9E,+EAA+E;IAC/E,mEAAmE;IACnE,MAAM,IAAI,GAAG,CAAC,IAAI,EAAE,iBAAiB,EAAE,MAAM,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,EAAE,CAAC,CAAC;IAC/F,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,IAAI,oBAAoB,EAAE,CAAC;IAC/D,MAAM,GAAG,GAAG,mBAAmB,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAEtD,+EAA+E;IAC/E,8EAA8E;IAC9E,sEAAsE;IACtE,yEAAyE;IACzE,wEAAwE;IACxE,oEAAoE;IACpE,kDAAkD;IAClD,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,QAAQ,KAAK,SAAS,CAAC;IAC/D,IAAI,UAAU,EAAE,CAAC;QACf,8EAA8E;QAC9E,sEAAsE;QACtE,GAAG,CAAC,uBAAuB,GAAG,GAAG,CAAC;QAClC,GAAG,CAAC,UAAU,GAAG,IAAI,CAAC,SAAS,IAAI,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,WAAW,CAAC;QACzE,IAAI,IAAI,CAAC,KAAK;YAAE,GAAG,CAAC,eAAe,GAAG,cAAc,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC;IACnF,CAAC;SAAM,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;QACtB,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;IACnC,CAAC;IACD,IAAI,IAAI,CAAC,eAAe;QAAE,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC;IACtE,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;QACxB,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE,IAAI,CAAC,cAAc,CAAC,CAAC;IACtD,CAAC;IACD,IAAI,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtD,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC;IACpD,CAAC;IACD,IAAI,IAAI,CAAC,QAAQ,IAAI,CAAC,UAAU,EAAE,CAAC;QACjC,wEAAwE;QACxE,wEAAwE;QACxE,wEAAwE;QACxE,6DAA6D;QAC7D,MAAM,QAAQ,GAA2B,EAAE,WAAW,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC;QACxE,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;IACpD,CAAC;IACD,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;QACvB,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE,GAAG,gBAAgB,CAAC,CAAC;IACtD,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,MAAM,CAAC;IAE3C,OAAO,IAAI,OAAO,CAAkB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACtD,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,EAAE,IAAI,EAAE;YACjC,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;YAC/B,GAAG;YACH,GAAG,EAAE,IAAI,CAAC,GAAG;SACd,CAAC,CAAC;QAEH,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,OAAO,GAAG,KAAK,CAAC;QAEpB,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,IAAI,OAAO;gBAAE,OAAO;YACpB,OAAO,GAAG,IAAI,CAAC;YACf,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACtB,MAAM,CAAC,IAAI,KAAK,CAAC,8BAA8B,SAAS,IAAI,CAAC,CAAC,CAAC;QACjE,CAAC,EAAE,SAAS,CAAC,CAAC;QAEd,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;QACzD,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;QAEzD,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YACxB,IAAI,OAAO;gBAAE,OAAO;YACpB,OAAO,GAAG,IAAI,CAAC;YACf,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,MAAM,CAAC,IAAI,KAAK,CAAC,+BAA+B,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QAClE,CAAC,CAAC,CAAC;QAEH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE;YACzB,IAAI,OAAO;gBAAE,OAAO;YACpB,OAAO,GAAG,IAAI,CAAC;YACf,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;gBACf,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;gBACpC,mEAAmE;gBACnE,oEAAoE;gBACpE,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC;oBACnB,MAAM,CAAC,IAAI,KAAK,CAAC,qBAAqB,IAAI,oBAAoB,MAAM,EAAE,CAAC,CAAC,CAAC;oBACzE,OAAO;gBACT,CAAC;gBACD,MAAM,CAAC,IAAI,KAAK,CAAC,qBAAqB,IAAI,KAAK,MAAM,EAAE,CAAC,CAAC,CAAC;gBAC1D,OAAO;YACT,CAAC;YACD,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBAChC,yEAAyE;gBACzE,6DAA6D;gBAC7D,IAAI,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC,QAAQ,KAAK,IAAI,EAAE,CAAC;oBACpD,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,gBAAgB,IAAI,EAAE,CAAC,CAAC;oBACnD,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC;oBAC3C,OAAO,CAAC,KAAK,CACX,oCAAoC,IAAI,CAAC,QAAQ,IAAI,SAAS,KAAK;wBACjE,UAAU,MAAM,YAAY,OAAO,GAAG;wBACtC,UAAU,MAAM,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CACtD,CAAC;oBACF,wEAAwE;oBACxE,IACE,MAAM,KAAK,KAAK;wBAChB,MAAM,KAAK,KAAK;wBAChB,2CAA2C,CAAC,IAAI,CAAC,OAAO,CAAC,EACzD,CAAC;wBACD,MAAM,CAAC,IAAI,KAAK,CAAC,kBAAkB,MAAM,IAAI,OAAO,yBAAyB,CAAC,CAAC,CAAC;wBAChF,OAAO;oBACT,CAAC;gBACH,CAAC;gBACD,OAAO,CAAC;oBACN,IAAI,EAAE,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE;oBACxD,SAAS,EAAE,IAAI,CAAC,UAAU;oBAC1B,EAAE,EAAE,IAAI,CAAC,QAAQ,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,gBAAgB;oBACpD,OAAO,EAAE,OAAO,IAAI,CAAC,cAAc,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,SAAS;iBACnF,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,CACJ,IAAI,KAAK,CACP,2CAA4C,GAAa,CAAC,OAAO,IAAI;oBACnE,oBAAoB,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAC7C,CACF,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC1B,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;IACpB,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,uBAAuB,CAAC,KAAY;IAC3C,OAAO,2BAA2B,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;AACzD,CAAC;AAED,SAAS,sBAAsB,CAAC,OAAe;IAC7C,KAAK,MAAM,SAAS,IAAI,wBAAwB,EAAE,CAAC;QACjD,IAAI,SAAS,KAAK,OAAO,IAAI,UAAU,CAAC,SAAS,CAAC;YAAE,OAAO,SAAS,CAAC;IACvE,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,yEAAyE;AACzE,MAAM,CAAC,KAAK,UAAU,eAAe;IACnC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,MAAM,KAAK,GAAG,KAAK,CAAC,oBAAoB,EAAE,EAAE,CAAC,WAAW,CAAC,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;QAChF,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;QACxC,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/eval/live/claude-delegate.d.ts

@@ -0,0 +1,41 @@
+import type { AgentDelegate, AgentDelegateOptions, AgentDelegateResult } from '../../runtime/types.js';
+import type { Task } from '../../types/index.js';
+import type { LiveBenchmark } from './live-benchmark.js';
+/**
+ * Single-turn agent delegate backed by the headless Claude CLI.
+ *
+ * Atlas injects its learned knowledge into `options.systemContext`; this
+ * delegate prepends it to the task prompt, runs `claude -p`, then **verifies the
+ * answer against ground truth** via the benchmark. The verified correctness
+ * becomes the returned `success`, so it flows into `trajectory.outcome.success`
+ * — meaning the learning pipeline learns from correctly-labeled outcomes, not
+ * the agent's self-report.
+ *
+ * The runner sets `currentTask` immediately before each `atlas.solve`, which is
+ * safe because the eval runs tasks strictly sequentially.
+ */
+export declare class ClaudeBenchmarkDelegate implements AgentDelegate {
+    private readonly benchmark;
+    private readonly opts;
+    private currentTask;
+    private lastCostUsd;
+    private totalCostUsd;
+    constructor(benchmark: LiveBenchmark, opts?: {
+        model?: string;
+        timeoutMs?: number;
+        provider?: string;
+        awsRegion?: string;
+        /** Per-call retry override (failFast → 0 = single attempt). */
+        cliRetries?: number;
+    });
+    /** Set the task being solved next (its gold answer is read during verify). */
+    setCurrentTask(task: Task): void;
+    /** Cumulative USD cost across all calls (best-effort, from the CLI). */
+    getTotalCostUsd(): number;
+    execute(prompt: string, options: AgentDelegateOptions): Promise<AgentDelegateResult>;
+}
+export declare function createClaudeBenchmarkDelegate(benchmark: LiveBenchmark, opts?: {
+    model?: string;
+    timeoutMs?: number;
+}): ClaudeBenchmarkDelegate;
+//# sourceMappingURL=claude-delegate.d.ts.map

package/dist/eval/live/claude-delegate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"claude-delegate.d.ts","sourceRoot":"","sources":["../../../src/eval/live/claude-delegate.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,aAAa,EACb,oBAAoB,EACpB,mBAAmB,EACpB,MAAM,wBAAwB,CAAC;AAChC,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,sBAAsB,CAAC;AACjD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAGzD;;;;;;;;;;;;GAYG;AACH,qBAAa,uBAAwB,YAAW,aAAa;IAMzD,OAAO,CAAC,QAAQ,CAAC,SAAS;IAC1B,OAAO,CAAC,QAAQ,CAAC,IAAI;IANvB,OAAO,CAAC,WAAW,CAAqB;IACxC,OAAO,CAAC,WAAW,CAAK;IACxB,OAAO,CAAC,YAAY,CAAK;gBAGN,SAAS,EAAE,aAAa,EACxB,IAAI,GAAE;QACrB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,+DAA+D;QAC/D,UAAU,CAAC,EAAE,MAAM,CAAC;KAChB;IAGR,8EAA8E;IAC9E,cAAc,CAAC,IAAI,EAAE,IAAI,GAAG,IAAI;IAIhC,wEAAwE;IACxE,eAAe,IAAI,MAAM;IAInB,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,oBAAoB,GAAG,OAAO,CAAC,mBAAmB,CAAC;CA0B3F;AAED,wBAAgB,6BAA6B,CAC3C,SAAS,EAAE,aAAa,EACxB,IAAI,CAAC,EAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAC5C,uBAAuB,CAEzB"}

package/dist/eval/live/claude-delegate.js

@@ -0,0 +1,59 @@
+import { runClaude } from './claude-cli.js';
+/**
+ * Single-turn agent delegate backed by the headless Claude CLI.
+ *
+ * Atlas injects its learned knowledge into `options.systemContext`; this
+ * delegate prepends it to the task prompt, runs `claude -p`, then **verifies the
+ * answer against ground truth** via the benchmark. The verified correctness
+ * becomes the returned `success`, so it flows into `trajectory.outcome.success`
+ * — meaning the learning pipeline learns from correctly-labeled outcomes, not
+ * the agent's self-report.
+ *
+ * The runner sets `currentTask` immediately before each `atlas.solve`, which is
+ * safe because the eval runs tasks strictly sequentially.
+ */
+export class ClaudeBenchmarkDelegate {
+    benchmark;
+    opts;
+    currentTask = null;
+    lastCostUsd = 0;
+    totalCostUsd = 0;
+    constructor(benchmark, opts = {}) {
+        this.benchmark = benchmark;
+        this.opts = opts;
+    }
+    /** Set the task being solved next (its gold answer is read during verify). */
+    setCurrentTask(task) {
+        this.currentTask = task;
+    }
+    /** Cumulative USD cost across all calls (best-effort, from the CLI). */
+    getTotalCostUsd() {
+        return this.totalCostUsd;
+    }
+    async execute(prompt, options) {
+        // systemContext carries Atlas's injected playbooks/experiences — keep it
+        // first so accumulated learning shapes the agent's approach.
+        const fullPrompt = [options.systemContext, prompt].filter(Boolean).join('\n\n');
+        const res = await runClaude(fullPrompt, {
+            model: this.opts.model,
+            timeoutMs: this.opts.timeoutMs ?? options.timeoutMs,
+            provider: this.opts.provider,
+            awsRegion: this.opts.awsRegion,
+            retries: this.opts.cliRetries,
+        });
+        this.lastCostUsd = res.costUsd ?? 0;
+        this.totalCostUsd += this.lastCostUsd;
+        const verdict = this.currentTask
+            ? this.benchmark.verify(this.currentTask, res.text)
+            : { correct: false, extracted: undefined };
+        return {
+            success: verdict.correct,
+            output: res.text,
+            structured: { extracted: verdict.extracted, costUsd: this.lastCostUsd },
+        };
+    }
+}
+export function createClaudeBenchmarkDelegate(benchmark, opts) {
+    return new ClaudeBenchmarkDelegate(benchmark, opts);
+}
+//# sourceMappingURL=claude-delegate.js.map

package/dist/eval/live/claude-delegate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"claude-delegate.js","sourceRoot":"","sources":["../../../src/eval/live/claude-delegate.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAE5C;;;;;;;;;;;;GAYG;AACH,MAAM,OAAO,uBAAuB;IAMf;IACA;IANX,WAAW,GAAgB,IAAI,CAAC;IAChC,WAAW,GAAG,CAAC,CAAC;IAChB,YAAY,GAAG,CAAC,CAAC;IAEzB,YACmB,SAAwB,EACxB,OAOb,EAAE;QARW,cAAS,GAAT,SAAS,CAAe;QACxB,SAAI,GAAJ,IAAI,CAOf;IACL,CAAC;IAEJ,8EAA8E;IAC9E,cAAc,CAAC,IAAU;QACvB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;IAC1B,CAAC;IAED,wEAAwE;IACxE,eAAe;QACb,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,MAAc,EAAE,OAA6B;QACzD,yEAAyE;QACzE,6DAA6D;QAC7D,MAAM,UAAU,GAAG,CAAC,OAAO,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEhF,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,UAAU,EAAE;YACtC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK;YACtB,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,OAAO,CAAC,SAAS;YACnD,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ;YAC5B,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS;YAC9B,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,UAAU;SAC9B,CAAC,CAAC;QAEH,IAAI,CAAC,WAAW,GAAG,GAAG,CAAC,OAAO,IAAI,CAAC,CAAC;QACpC,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,WAAW,CAAC;QAEtC,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW;YAC9B,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,CAAC,IAAI,CAAC;YACnD,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC;QAE7C,OAAO;YACL,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,MAAM,EAAE,GAAG,CAAC,IAAI;YAChB,UAAU,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,OAAO,EAAE,IAAI,CAAC,WAAW,EAAE;SACxE,CAAC;IACJ,CAAC;CACF;AAED,MAAM,UAAU,6BAA6B,CAC3C,SAAwB,EACxB,IAA6C;IAE7C,OAAO,IAAI,uBAAuB,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;AACtD,CAAC"}

package/dist/eval/live/claude-workspace-backend.d.ts

@@ -0,0 +1,24 @@
+import type { AgentBackend, AgentSession, AgentSpawnConfig } from '../../runtime/types.js';
+/**
+ * General Claude CLI backend for offline workspace-analysis agents.
+ *
+ * Unlike benchmark delegates, this backend does not verify task success against
+ * a benchmark. It runs Claude in the supplied workspace cwd and lets templates
+ * validate the files they asked the agent to write.
+ */
+export declare class ClaudeWorkspaceBackend implements AgentBackend {
+    private readonly opts;
+    readonly name = "claude-workspace";
+    readonly supportedTypes: string[];
+    private readonly sessions;
+    constructor(opts?: {
+        model?: string;
+        timeoutMs?: number;
+        provider?: string;
+        awsRegion?: string;
+    });
+    isAvailable(): Promise<boolean>;
+    spawn(config: AgentSpawnConfig): Promise<AgentSession>;
+    getSession(sessionId: string): Promise<AgentSession | undefined>;
+}
+//# sourceMappingURL=claude-workspace-backend.d.ts.map

package/dist/eval/live/claude-workspace-backend.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"claude-workspace-backend.d.ts","sourceRoot":"","sources":["../../../src/eval/live/claude-workspace-backend.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,YAAY,EAEZ,YAAY,EACZ,gBAAgB,EACjB,MAAM,wBAAwB,CAAC;AAGhC;;;;;;GAMG;AACH,qBAAa,sBAAuB,YAAW,YAAY;IAOvD,OAAO,CAAC,QAAQ,CAAC,IAAI;IANvB,QAAQ,CAAC,IAAI,sBAAsB;IACnC,QAAQ,CAAC,cAAc,WAAmB;IAE1C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAmC;gBAGzC,IAAI,GAAE;QACrB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,SAAS,CAAC,EAAE,MAAM,CAAC;KACf;IAGF,WAAW,IAAI,OAAO,CAAC,OAAO,CAAC;IAI/B,KAAK,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,YAAY,CAAC;IAuDtD,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,GAAG,SAAS,CAAC;CAGvE"}

package/dist/eval/live/claude-workspace-backend.js

@@ -0,0 +1,76 @@
+import { randomUUID } from 'node:crypto';
+import { runClaude } from './claude-cli.js';
+/**
+ * General Claude CLI backend for offline workspace-analysis agents.
+ *
+ * Unlike benchmark delegates, this backend does not verify task success against
+ * a benchmark. It runs Claude in the supplied workspace cwd and lets templates
+ * validate the files they asked the agent to write.
+ */
+export class ClaudeWorkspaceBackend {
+    opts;
+    name = 'claude-workspace';
+    supportedTypes = ['claude-code'];
+    sessions = new Map();
+    constructor(opts = {}) {
+        this.opts = opts;
+    }
+    async isAvailable() {
+        return true;
+    }
+    async spawn(config) {
+        const startTime = new Date();
+        const session = {
+            id: randomUUID(),
+            agentType: config.agentType,
+            task: config.task,
+            state: 'running',
+            messages: [],
+            toolCalls: [],
+            startTime,
+            metadata: { backend: this.name },
+        };
+        this.sessions.set(session.id, session);
+        try {
+            const prompt = [
+                config.systemPromptAdditions,
+                'You are running as a workspace automation agent. Required output files must be created on disk in this working directory. Use file tools to write the requested output files; do not only describe the result in your final answer. The caller validates files on disk and ignores prose unless the files exist.',
+                config.task.description,
+            ]
+                .filter(Boolean)
+                .join('\n\n');
+            const result = await runClaude(prompt, {
+                model: this.opts.model,
+                timeoutMs: config.timeout ?? this.opts.timeoutMs,
+                provider: this.opts.provider,
+                awsRegion: this.opts.awsRegion,
+                cwd: config.cwd,
+                permissionMode: 'acceptEdits',
+                allowedTools: ['Read', 'Write', 'Edit', 'Glob', 'Grep'],
+            });
+            const message = {
+                role: 'assistant',
+                content: result.text,
+                timestamp: new Date(),
+            };
+            session.messages.push(message);
+            session.result = result.text;
+            session.state = result.ok ? 'completed' : 'failed';
+            session.endTime = new Date();
+            if (!result.ok) {
+                session.error = 'Claude CLI returned an error result';
+            }
+        }
+        catch (error) {
+            session.state = 'failed';
+            session.error = error instanceof Error ? error.message : String(error);
+            session.endTime = new Date();
+        }
+        this.sessions.set(session.id, session);
+        return session;
+    }
+    async getSession(sessionId) {
+        return this.sessions.get(sessionId);
+    }
+}
+//# sourceMappingURL=claude-workspace-backend.js.map

package/dist/eval/live/claude-workspace-backend.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"claude-workspace-backend.js","sourceRoot":"","sources":["../../../src/eval/live/claude-workspace-backend.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAOzC,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAE5C;;;;;;GAMG;AACH,MAAM,OAAO,sBAAsB;IAOd;IANV,IAAI,GAAG,kBAAkB,CAAC;IAC1B,cAAc,GAAG,CAAC,aAAa,CAAC,CAAC;IAEzB,QAAQ,GAAG,IAAI,GAAG,EAAwB,CAAC;IAE5D,YACmB,OAKb,EAAE;QALW,SAAI,GAAJ,IAAI,CAKf;IACL,CAAC;IAEJ,KAAK,CAAC,WAAW;QACf,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,MAAwB;QAClC,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;QAC7B,MAAM,OAAO,GAAiB;YAC5B,EAAE,EAAE,UAAU,EAAE;YAChB,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,KAAK,EAAE,SAAS;YAChB,QAAQ,EAAE,EAAE;YACZ,SAAS,EAAE,EAAE;YACb,SAAS;YACT,QAAQ,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,IAAI,EAAE;SACjC,CAAC;QACF,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;QAEvC,IAAI,CAAC;YACH,MAAM,MAAM,GAAG;gBACb,MAAM,CAAC,qBAAqB;gBAC5B,kTAAkT;gBAClT,MAAM,CAAC,IAAI,CAAC,WAAW;aACxB;iBACE,MAAM,CAAC,OAAO,CAAC;iBACf,IAAI,CAAC,MAAM,CAAC,CAAC;YAEhB,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,MAAM,EAAE;gBACrC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK;gBACtB,SAAS,EAAE,MAAM,CAAC,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,SAAS;gBAChD,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ;gBAC5B,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS;gBAC9B,GAAG,EAAE,MAAM,CAAC,GAAG;gBACf,cAAc,EAAE,aAAa;gBAC7B,YAAY,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;aACxD,CAAC,CAAC;YAEH,MAAM,OAAO,GAAiB;gBAC5B,IAAI,EAAE,WAAW;gBACjB,OAAO,EAAE,MAAM,CAAC,IAAI;gBACpB,SAAS,EAAE,IAAI,IAAI,EAAE;aACtB,CAAC;YACF,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC/B,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC;YAC7B,OAAO,CAAC,KAAK,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ,CAAC;YACnD,OAAO,CAAC,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC;YAC7B,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;gBACf,OAAO,CAAC,KAAK,GAAG,qCAAqC,CAAC;YACxD,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,GAAG,QAAQ,CAAC;YACzB,OAAO,CAAC,KAAK,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACvE,OAAO,CAAC,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC;QAC/B,CAAC;QAED,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;QACvC,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,SAAiB;QAChC,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IACtC,CAAC;CACF"}

package/dist/eval/live/clbench.d.ts

@@ -0,0 +1,64 @@
+import { type Task } from '../../types/index.js';
+import type { EvalTask } from '../types.js';
+import type { ExtendedLiveBenchmark, BenchmarkVerdict } from './live-benchmark.js';
+import type { BenchmarkDelegate } from './live-runner.js';
+import { EnvBridge } from './env-bridge.js';
+export declare class ClBenchBenchmark implements ExtendedLiveBenchmark {
+    readonly name = "clbench";
+    readonly interactive = true;
+    readonly preserveOrder = true;
+    readonly playbookMinTrajectories = 3;
+    private bridge;
+    private readonly maxSteps;
+    private readonly reinject;
+    private readonly historyWindow;
+    constructor(opts?: {
+        maxSteps?: number;
+        reinject?: boolean;
+        historyWindow?: number;
+    });
+    setup(): Promise<void>;
+    teardown(): Promise<void>;
+    private getBridge;
+    freshBridge(): Promise<EnvBridge>;
+    /**
+     * Load conversations as tasks. Applies a seeded Fisher-Yates shuffle so
+     * different seeds produce different orderings (reproducible per seed).
+     */
+    loadTasks(opts: {
+        count: number;
+        seed: number;
+    }): Promise<EvalTask[]>;
+    verify(_task: Task, _output: string): BenchmarkVerdict;
+    getBridgeInstance(): EnvBridge | null;
+    delegateFactory: (_benchmark: ExtendedLiveBenchmark, opts?: {
+        model?: string;
+        timeoutMs?: number;
+        provider?: string;
+        awsRegion?: string;
+        cliRetries?: number;
+    }) => BenchmarkDelegate;
+}
+export declare function createClBenchBenchmark(opts?: {
+    maxSteps?: number;
+    reinject?: boolean;
+    historyWindow?: number;
+}): ClBenchBenchmark;
+/**
+ * Structured per-step prompt for CL-Bench CRM conversations.
+ * Surfaces tool catalog, conversation context, and injected knowledge.
+ */
+export declare function buildClBenchPrompt(p: {
+    knowledge?: string;
+    instruction: string;
+    category: string;
+    complexity: string;
+    history: Array<{
+        action: string;
+        obs: string;
+    }>;
+    obs: string;
+    availableTools?: string[];
+    context?: Record<string, unknown>;
+}): string;
+//# sourceMappingURL=clbench.d.ts.map

package/dist/eval/live/clbench.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"clbench.d.ts","sourceRoot":"","sources":["../../../src/eval/live/clbench.ts"],"names":[],"mappings":"AAEA,OAAO,EAML,KAAK,IAAI,EACV,MAAM,sBAAsB,CAAC;AAE9B,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,KAAK,EAAE,qBAAqB,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AACnF,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAC1D,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAuE5C,qBAAa,gBAAiB,YAAW,qBAAqB;IAC5D,QAAQ,CAAC,IAAI,aAAa;IAC1B,QAAQ,CAAC,WAAW,QAAQ;IAC5B,QAAQ,CAAC,aAAa,QAAQ;IAC9B,QAAQ,CAAC,uBAAuB,KAAK;IAErC,OAAO,CAAC,MAAM,CAA0B;IACxC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAU;IACnC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAS;gBAGrC,IAAI,GAAE;QACJ,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,QAAQ,CAAC,EAAE,OAAO,CAAC;QACnB,aAAa,CAAC,EAAE,MAAM,CAAC;KACnB;IAOF,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAItB,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;YAKjB,SAAS;IAiBjB,WAAW,IAAI,OAAO,CAAC,SAAS,CAAC;IAMvC;;;OAGG;IACG,SAAS,CAAC,IAAI,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;IA4C3E,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,GAAG,gBAAgB;IAItD,iBAAiB,IAAI,SAAS,GAAG,IAAI;IAIrC,eAAe,GACb,YAAY,qBAAqB,EACjC,OAAO;QACL,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,KACA,iBAAiB,CAWlB;CACH;AAED,wBAAgB,sBAAsB,CAAC,IAAI,CAAC,EAAE;IAC5C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB,GAAG,gBAAgB,CAEnB;AAwID;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,CAAC,EAAE;IACpC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,KAAK,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAChD,GAAG,EAAE,MAAM,CAAC;IACZ,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACnC,GAAG,MAAM,CAkCT"}