klyro 1.0.10 → 1.0.11

package/dist/cli/eval.d.ts

@@ -79,6 +79,8 @@ export interface EvalResult {
         notes: string;
         skipped: boolean;
     };
+    /** Isolated workdir the scenario ran in (tmp unless --cwd). Debugging aid. */
+    workDir?: string;
 }
 export interface RunEvalOptions {
     inputPath: string;
@@ -90,10 +92,16 @@ export interface RunEvalOptions {
     model?: string;
     /** Live model id for grading `judge.rubric` (env endpoint + key required). */
     judgeModel?: string;
+    /**
+     * Shared workdir for JSONL scenarios. When omitted each scenario runs in
+     * a fresh tmp dir (deleted afterwards) so scripted tool calls can never
+     * touch the caller's directory. Pass explicitly to inspect artifacts.
+     */
+    cwd?: string;
 }
 export declare function runEval(opts: RunEvalOptions): Promise<number>;
 export declare function scriptedAdapterFromSpec(spec: Array<Array<unknown[]>> | undefined): ProviderAdapter;
 export declare function runScenario(sc: EvalScenario, judgeOpts?: {
     adapter: ProviderAdapter;
     model: string;
-}): Promise<EvalResult>;
+}, workDir?: string): Promise<EvalResult>;

package/dist/cli/eval.js

@@ -40,6 +40,9 @@
  * otherwise.
  */
 import * as fs from 'node:fs';
+import * as fsp from 'node:fs/promises';
+import * as os from 'node:os';
+import * as path from 'node:path';
 import * as readline from 'node:readline/promises';
 import { stdin as input, stdout, stderr } from 'node:process';
 import { run } from '../agent/runtime.js';
@@ -153,7 +156,7 @@ export async function runEval(opts) {
     const results = [];
     for (const sc of scenarios) {
         const start = Date.now();
-        const r = await runScenario(sc, judgeAdapter && opts.judgeModel ? { adapter: judgeAdapter, model: opts.judgeModel } : undefined);
+        const r = await runScenario(sc, judgeAdapter && opts.judgeModel ? { adapter: judgeAdapter, model: opts.judgeModel } : undefined, opts.cwd);
         r.durationMs = Date.now() - start;
         results.push(r);
         if (opts.output === 'json') {
@@ -242,35 +245,53 @@ function tupleToEvent(tuple) {
             throw new Error(`scriptedAdapterFromSpec: unknown event kind: ${kind}`);
     }
 }
-export async function runScenario(sc, judgeOpts) {
+export async function runScenario(sc, judgeOpts, workDir) {
     const failures = [];
     const model = sc.model ?? 'mock';
     const adapter = scriptedAdapterFromSpec(sc.scripted_events);
     const registry = builtinRegistry();
     const policy = new PolicyEngine(builtinRules(), DEFAULT_POLICY_CONFIG);
-    const result = await run({
-        task: sc.task,
-        cwd: process.cwd(),
-        model,
-        maxSteps: sc.maxSteps,
-        maxTokens: sc.maxTokens,
-        nonInteractive: true,
-        ...(sc.verify
-            ? {
-                verify: {
-                    enabled: true,
-                    ...(sc.verify.command !== undefined ? { command: sc.verify.command } : {}),
-                    ...(sc.verify.mode !== undefined ? { mode: sc.verify.mode } : {}),
-                },
+    // Isolation (fix: scripted tool calls must never run in the caller's
+    // directory — a JSONL scenario writing a.txt/b.txt used to pollute it).
+    // Explicit workDir is shared as-is (inspect artifacts); otherwise each
+    // scenario gets a fresh tmp dir that is removed afterwards.
+    const owned = !workDir;
+    const cwd = workDir ?? await fsp.mkdtemp(path.join(os.tmpdir(), 'klyro-eval-jsonl-'));
+    await fsp.mkdir(cwd, { recursive: true });
+    let result;
+    try {
+        result = await run({
+            task: sc.task,
+            cwd,
+            model,
+            maxSteps: sc.maxSteps,
+            maxTokens: sc.maxTokens,
+            nonInteractive: true,
+            ...(sc.verify
+                ? {
+                    verify: {
+                        enabled: true,
+                        ...(sc.verify.command !== undefined ? { command: sc.verify.command } : {}),
+                        ...(sc.verify.mode !== undefined ? { mode: sc.verify.mode } : {}),
+                    },
+                }
+                : {}),
+        }, {
+            adapter,
+            registry,
+            policy,
+            approval: new DenyAllApprovalPrompt(),
+            systemPrompt: ({ cwd }) => `You are Klyro. cwd=${cwd}.`,
+        });
+    }
+    finally {
+        if (owned) {
+            try {
+                await fsp.rm(cwd, { recursive: true, force: true });
             }
-            : {}),
-    }, {
-        adapter,
-        registry,
-        policy,
-        approval: new DenyAllApprovalPrompt(),
-        systemPrompt: ({ cwd }) => `You are Klyro. cwd=${cwd}.`,
-    });
+            catch { /* ignore */ }
+        }
+    }
     const exp = sc.expect ?? {};
     if (exp.status !== undefined && result.status !== exp.status) {
         failures.push(`status: expected ${exp.status}, got ${result.status}`);
@@ -311,6 +332,7 @@ export async function runScenario(sc, judgeOpts) {
         toolCalls: result.toolCalls,
         text: result.finalText,
         durationMs: 0,
+        workDir: cwd,
         ...(judge ? { judge } : {}),
     };
 }

package/dist/index.js

@@ -390,6 +390,7 @@ async function main() {
         .option('--parallel <n>', 'Parallelism (default 1)', (v) => parsePositiveInt('--parallel', v))
         .option('--model <id>', 'Model for eval')
         .option('--judge-model <id>', 'Live model id for grading judge.rubric (needs endpoint + key)')
+        .option('--cwd <path>', 'Shared scenario workdir (default: isolated tmp per scenario)')
         .action(async (input, opts) => {
         const output = (opts.output ?? 'human');
         if (opts.suite) {
@@ -400,7 +401,7 @@ async function main() {
             process.stderr.write('klyro eval: missing input (provide <input> or --suite)\n');
             process.exit(2);
         }
-        const code = await runEval({ inputPath: input, output, suite: opts.suite, filter: opts.filter, runs: opts.runs, parallel: opts.parallel, model: opts.model, judgeModel: opts.judgeModel });
+        const code = await runEval({ inputPath: input, output, suite: opts.suite, filter: opts.filter, runs: opts.runs, parallel: opts.parallel, model: opts.model, judgeModel: opts.judgeModel, cwd: opts.cwd });
         process.exit(code);
     });
     program

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "klyro",
-  "version": "1.0.10",
+  "version": "1.0.11",
   "description": "Klyro — autonomous coding harness CLI that streams from any OpenAI-compatible or Anthropic LLM endpoint.",
   "type": "module",
   "main": "dist/index.js",