@qduc/term2 0.1.8 → 0.3.0

package/dist/scripts/eval-auto-approval/runner-utils.test.js

@@ -0,0 +1,135 @@
+import test from 'ava';
+import { mkdtempSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { createCacheKey, EVAL_CACHE_VERSION, getReportPath, loadModelRunsFromYaml, retryOnRateLimit, validateRunnerOptions, } from './runner-utils.js';
+test('getReportPath produces a distinct markdown path for json and extensionless outputs', (t) => {
+    t.is(getReportPath('eval/auto-approval/results.json'), 'eval/auto-approval/results.md');
+    t.is(getReportPath('eval/auto-approval/results'), 'eval/auto-approval/results.md');
+});
+test('validateRunnerOptions rejects non-positive concurrency and repeat values', (t) => {
+    t.notThrows(() => validateRunnerOptions({ concurrency: 1, repeat: 1 }));
+    t.throws(() => validateRunnerOptions({ concurrency: 0, repeat: 1 }), {
+        message: /--concurrency/,
+    });
+    t.throws(() => validateRunnerOptions({ concurrency: 1, repeat: 0 }), {
+        message: /--repeat/,
+    });
+});
+test('createCacheKey includes evaluator version to invalidate stale cached decisions', (t) => {
+    const key = createCacheKey({
+        model: 'gpt-4o-mini',
+        provider: 'openai',
+        promptVersion: 'auto-approval-prompt-v1',
+        command: 'ls',
+        history: [{ role: 'user', content: 'list files' }],
+    });
+    t.is(key.version, EVAL_CACHE_VERSION);
+    t.is(key.promptVersion, 'auto-approval-prompt-v1');
+});
+test('createCacheKey changes when prompt version changes', (t) => {
+    const base = {
+        model: 'gpt-4o-mini',
+        provider: 'openai',
+        command: 'ls',
+        history: [{ role: 'user', content: 'list files' }],
+    };
+    t.notDeepEqual(createCacheKey({ ...base, promptVersion: 'auto-approval-prompt-v1' }), createCacheKey({ ...base, promptVersion: 'auto-approval-prompt-v2' }));
+});
+test('loadModelRunsFromYaml expands provider keys into ordered provider/model pairs', (t) => {
+    const dir = mkdtempSync(join(tmpdir(), 'term2-eval-'));
+    const yamlPath = join(dir, 'models.yaml');
+    writeFileSync(yamlPath, `
+openai:
+  - gpt-4o
+  - gpt-4o-mini
+openrouter:
+  - anthropic/claude-3.5-sonnet
+`);
+    t.deepEqual(loadModelRunsFromYaml(yamlPath), [
+        { provider: 'openai', model: 'gpt-4o' },
+        { provider: 'openai', model: 'gpt-4o-mini' },
+        { provider: 'openrouter', model: 'anthropic/claude-3.5-sonnet' },
+    ]);
+});
+test('loadModelRunsFromYaml rejects providers without a model list', (t) => {
+    const dir = mkdtempSync(join(tmpdir(), 'term2-eval-'));
+    const yamlPath = join(dir, 'models.yaml');
+    writeFileSync(yamlPath, `
+openai: gpt-4o
+`);
+    t.throws(() => loadModelRunsFromYaml(yamlPath), {
+        message: /must map to a list of models/,
+    });
+});
+test('retryOnRateLimit retries twice and respects x-ratelimit-reset before succeeding', async (t) => {
+    let attempts = 0;
+    const sleepCalls = [];
+    const now = 1_741_305_599_000;
+    const result = await retryOnRateLimit({
+        operation: async () => {
+            attempts++;
+            if (attempts < 3) {
+                const err = new Error('Rate limit exceeded');
+                err.status = 429;
+                err.headers = {
+                    'X-RateLimit-Reset': String(now + 1_500),
+                };
+                throw err;
+            }
+            return 'ok';
+        },
+        maxRetries: 2,
+        sleep: async (ms) => {
+            sleepCalls.push(ms);
+        },
+        now: () => now,
+    });
+    t.is(result, 'ok');
+    t.is(attempts, 3);
+    t.deepEqual(sleepCalls, [1500, 1500]);
+});
+test('retryOnRateLimit rethrows after exhausting retries', async (t) => {
+    let attempts = 0;
+    const sleepCalls = [];
+    const error = await t.throwsAsync(() => retryOnRateLimit({
+        operation: async () => {
+            attempts++;
+            const err = Object.assign(new Error('Too Many Requests'), {
+                error: {
+                    code: 429,
+                    metadata: {
+                        headers: {
+                            'X-RateLimit-Reset': '1741305600000',
+                        },
+                    },
+                },
+            });
+            throw err;
+        },
+        maxRetries: 2,
+        sleep: async (ms) => {
+            sleepCalls.push(ms);
+        },
+        now: () => 1_741_305_599_000,
+    }));
+    t.truthy(error);
+    t.is(attempts, 3);
+    t.deepEqual(sleepCalls, [1000, 1000]);
+});
+test('retryOnRateLimit does not retry non-rate-limit errors', async (t) => {
+    let attempts = 0;
+    const error = await t.throwsAsync(() => retryOnRateLimit({
+        operation: async () => {
+            attempts++;
+            const err = new Error('boom');
+            err.status = 500;
+            throw err;
+        },
+        maxRetries: 2,
+        sleep: async () => { },
+    }));
+    t.truthy(error);
+    t.is(attempts, 1);
+});
+//# sourceMappingURL=runner-utils.test.js.map

package/dist/scripts/eval-auto-approval/runner-utils.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner-utils.test.js","sourceRoot":"","sources":["../../../source/scripts/eval-auto-approval/runner-utils.test.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,KAAK,CAAC;AACvB,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACrD,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EACL,cAAc,EACd,kBAAkB,EAClB,aAAa,EACb,qBAAqB,EACrB,gBAAgB,EAChB,qBAAqB,GACtB,MAAM,mBAAmB,CAAC;AAE3B,IAAI,CAAC,oFAAoF,EAAE,CAAC,CAAC,EAAE,EAAE;IAC/F,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,iCAAiC,CAAC,EAAE,+BAA+B,CAAC,CAAC;IACxF,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,4BAA4B,CAAC,EAAE,+BAA+B,CAAC,CAAC;AACrF,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,0EAA0E,EAAE,CAAC,CAAC,EAAE,EAAE;IACrF,CAAC,CAAC,SAAS,CAAC,GAAG,EAAE,CAAC,qBAAqB,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IACxE,CAAC,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,qBAAqB,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE;QACnE,OAAO,EAAE,eAAe;KACzB,CAAC,CAAC;IACH,CAAC,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,qBAAqB,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE;QACnE,OAAO,EAAE,UAAU;KACpB,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,gFAAgF,EAAE,CAAC,CAAC,EAAE,EAAE;IAC3F,MAAM,GAAG,GAAG,cAAc,CAAC;QACzB,KAAK,EAAE,aAAa;QACpB,QAAQ,EAAE,QAAQ;QAClB,aAAa,EAAE,yBAAyB;QACxC,OAAO,EAAE,IAAI;QACb,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC;KACnD,CAAC,CAAC;IAEH,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,OAAO,EAAE,kBAAkB,CAAC,CAAC;IACtC,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,aAAa,EAAE,yBAAyB,CAAC,CAAC;AACrD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,oDAAoD,EAAE,CAAC,CAAC,EAAE,EAAE;IAC/D,MAAM,IAAI,GAAG;QACX,KAAK,EAAE,aAAa;QACpB,QAAQ,EAAE,QAAQ;QAClB,OAAO,EAAE,IAAI;QACb,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC;KACnD,CAAC;IAEF,CAAC,CAAC,YAAY,CACZ,cAAc,CAAC,EAAE,GAAG,IAAI,EAAE,aAAa,EAAE,yBAAyB,EAAE,CAAC,EACrE,cAAc,CAAC,EAAE,GAAG,IAAI,EAAE,aAAa,EAAE,yBAAyB,EAAE,CAAC,CACtE,CAAC;AACJ,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,+EAA+E,EAAE,CAAC,CAAC,EAAE,EAAE;IAC1F,MAAM,GAAG,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,aAAa,CAAC,CAAC,CAAC;IACvD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;IAE1C,aAAa,CACX,QAAQ,EACR;;;;;;CAMH,CACE,CAAC;IAEF,CAAC,CAAC,SAAS,CAAC,qBAAqB,CAAC,QAAQ,CAAC,EAAE;QAC3C,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE;QACvC,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE,aAAa,EAAE;QAC5C,EAAE,QAAQ,EAAE,YAAY,EAAE,KAAK,EAAE,6BAA6B,EAAE;KACjE,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,8DAA8D,EAAE,CAAC,CAAC,EAAE,EAAE;IACzE,MAAM,GAAG,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,aAAa,CAAC,CAAC,CAAC;IACvD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;IAE1C,aAAa,CACX,QAAQ,EACR;;CAEH,CACE,CAAC;IAEF,CAAC,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,qBAAqB,CAAC,QAAQ,CAAC,EAAE;QAC9C,OAAO,EAAE,8BAA8B;KACxC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,iFAAiF,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IAClG,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,MAAM,UAAU,GAAa,EAAE,CAAC;IAChC,MAAM,GAAG,GAAG,iBAAiB,CAAC;IAE9B,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC;QACpC,SAAS,EAAE,KAAK,IAAI,EAAE;YACpB,QAAQ,EAAE,CAAC;YACX,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;gBACjB,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;gBAC5C,GAAmE,CAAC,MAAM,GAAG,GAAG,CAAC;gBACjF,GAAmE,CAAC,OAAO,GAAG;oBAC7E,mBAAmB,EAAE,MAAM,CAAC,GAAG,GAAG,KAAK,CAAC;iBACzC,CAAC;gBACF,MAAM,GAAG,CAAC;YACZ,CAAC;YAED,OAAO,IAAI,CAAC;QACd,CAAC;QACD,UAAU,EAAE,CAAC;QACb,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE;YAClB,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACtB,CAAC;QACD,GAAG,EAAE,GAAG,EAAE,CAAC,GAAG;KACf,CAAC,CAAC;IAEH,CAAC,CAAC,EAAE,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IACnB,CAAC,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,SAAS,CAAC,UAAU,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;AACxC,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,oDAAoD,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IACrE,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,MAAM,UAAU,GAAa,EAAE,CAAC;IAEhC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,WAAW,CAAC,GAAG,EAAE,CACrC,gBAAgB,CAAC;QACf,SAAS,EAAE,KAAK,IAAI,EAAE;YACpB,QAAQ,EAAE,CAAC;YACX,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,mBAAmB,CAAC,EAAE;gBACxD,KAAK,EAAE;oBACL,IAAI,EAAE,GAAG;oBACT,QAAQ,EAAE;wBACR,OAAO,EAAE;4BACP,mBAAmB,EAAE,eAAe;yBACrC;qBACF;iBACF;aACF,CAAC,CAAC;YACH,MAAM,GAAG,CAAC;QACZ,CAAC;QACD,UAAU,EAAE,CAAC;QACb,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE;YAClB,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACtB,CAAC;QACD,GAAG,EAAE,GAAG,EAAE,CAAC,iBAAiB;KAC7B,CAAC,CACH,CAAC;IAEF,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAChB,CAAC,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,SAAS,CAAC,UAAU,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;AACxC,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,uDAAuD,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IACxE,IAAI,QAAQ,GAAG,CAAC,CAAC;IAEjB,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,WAAW,CAAC,GAAG,EAAE,CACrC,gBAAgB,CAAC;QACf,SAAS,EAAE,KAAK,IAAI,EAAE;YACpB,QAAQ,EAAE,CAAC;YACX,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,MAAM,CAAC,CAAC;YAC7B,GAAkC,CAAC,MAAM,GAAG,GAAG,CAAC;YACjD,MAAM,GAAG,CAAC;QACZ,CAAC;QACD,UAAU,EAAE,CAAC;QACb,KAAK,EAAE,KAAK,IAAI,EAAE,GAAE,CAAC;KACtB,CAAC,CACH,CAAC;IAEF,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAChB,CAAC,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC"}

package/dist/scripts/eval-auto-approval/runner.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=runner.d.ts.map

package/dist/scripts/eval-auto-approval/runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../../source/scripts/eval-auto-approval/runner.ts"],"names":[],"mappings":""}

package/dist/scripts/eval-auto-approval/runner.js

@@ -0,0 +1,326 @@
+import meow from 'meow';
+import { loadDataset, filterDataset } from './dataset.js';
+import { createMockSettingsService } from '../../services/settings-service.mock.js';
+import { LoggingService } from '../../services/logging-service.js';
+import { OpenAIAgentClient } from '../../lib/openai-agent-client.js';
+import { evaluateShellAutoApprovalAdvisories, SHELL_AUTO_APPROVAL_PROMPT_VERSION, } from '../../services/shell-auto-approval-evaluator.js';
+import { ResponseCache } from './cache.js';
+import { buildModelLeaderboard, getModelLeaderboardPaths, loadPersistedModelResults, mergeModelResults, saveModelLeaderboardJson, saveModelLeaderboardMarkdown, saveModelResults, } from './leaderboard.js';
+import { computeMetrics } from './metrics.js';
+import { generateReport } from './report.js';
+import { createCacheKey, loadModelRunsFromYaml, retryOnRateLimit, validateRunnerOptions } from './runner-utils.js';
+import { appendFileSync, writeFileSync, mkdirSync, existsSync } from 'node:fs';
+import { join, dirname } from 'node:path';
+import { performance } from 'node:perf_hooks';
+const cli = meow(`
+	Usage
+	  $ node dist/scripts/eval-auto-approval/runner.js [options]
+
+	Options
+	  --model         Model to evaluate (e.g. gpt-4o)
+  --provider      Provider to use (openai, openrouter, etc.)
+  --dataset       Path to dataset JSON (default: eval/auto-approval/dataset.json)
+  --models-file   Path to YAML file mapping providers to model lists (default: eval/auto-approval/models.yaml)
+  --filter-cat    Filter by category
+  --filter-sev    Filter by severity
+  --concurrency   Max concurrent requests (default: 5)
+	  --repeat        Number of times to repeat each case (default: 1)
+	  --output        Path to output results (default: eval/auto-approval/reports/<date>/results-<timestamp>)
+	  --report-format Comma-separated list of formats (json,jsonl,md) (default: json)
+	  --dry-run       Print plan without executing
+	  --yes           Skip confirmation prompt
+	  --no-cache      Disable response caching
+	  --clear-cache   Clear the response cache before running
+
+	Examples
+	  $ node dist/scripts/eval-auto-approval/runner.js --model gpt-4o --concurrency 10
+`, {
+    importMeta: import.meta,
+    flags: {
+        model: { type: 'string' },
+        provider: { type: 'string', default: 'openai' },
+        dataset: { type: 'string', default: 'eval/auto-approval/dataset.json' },
+        modelsFile: { type: 'string', default: 'eval/auto-approval/models.yaml' },
+        filterCat: { type: 'string' },
+        filterSev: { type: 'string' },
+        concurrency: { type: 'number', default: 5 },
+        repeat: { type: 'number', default: 1 },
+        output: { type: 'string' },
+        reportFormat: { type: 'string', default: 'json' },
+        dryRun: { type: 'boolean' },
+        yes: { type: 'boolean' },
+        cache: { type: 'boolean', default: true },
+        clearCache: { type: 'boolean' },
+    },
+});
+async function run() {
+    const flags = cli.flags;
+    if (flags.clearCache) {
+        const cache = new ResponseCache('eval/auto-approval/.cache');
+        cache.clear();
+        console.log('Cache cleared.');
+        if (flags.dryRun)
+            return;
+    }
+    if (!process.env['RUN_LIVE_EVAL'] && !flags.dryRun) {
+        console.error('Error: RUN_LIVE_EVAL=1 environment variable must be set to run live evaluations.');
+        process.exit(1);
+    }
+    const datasetPath = flags.dataset;
+    validateRunnerOptions({
+        concurrency: flags.concurrency,
+        repeat: flags.repeat,
+    });
+    const cases = loadDataset(datasetPath);
+    const filteredCases = filterDataset(cases, {
+        category: flags.filterCat,
+        severity: flags.filterSev,
+    });
+    if (filteredCases.length === 0) {
+        console.log('No cases matched the filters.');
+        return;
+    }
+    const modelRuns = flags.model && flags.modelsFile === 'eval/auto-approval/models.yaml' && !existsSync(flags.modelsFile)
+        ? [{ provider: flags.provider, model: flags.model }]
+        : flags.modelsFile && existsSync(flags.modelsFile)
+            ? loadModelRunsFromYaml(flags.modelsFile)
+            : flags.model
+                ? [{ provider: flags.provider, model: flags.model }]
+                : [];
+    if (modelRuns.length === 0) {
+        if (!flags.model && (!flags.modelsFile || !existsSync(flags.modelsFile))) {
+            console.error('Error: Either --model must be specified, or a valid --models-file must exist.');
+            process.exit(1);
+        }
+        console.log('No model runs configured.');
+        return;
+    }
+    const totalRuns = filteredCases.length * flags.repeat * modelRuns.length;
+    console.log('--- Eval Plan ---');
+    if (flags.modelsFile) {
+        console.log(`Models file:  ${flags.modelsFile}`);
+        console.log('Model Runs:');
+        for (const run of modelRuns) {
+            console.log(`  - ${run.provider}/${run.model}`);
+        }
+    }
+    else {
+        console.log(`Model:       ${modelRuns[0].model}`);
+        console.log(`Provider:    ${modelRuns[0].provider}`);
+    }
+    console.log(`Dataset:     ${datasetPath}`);
+    console.log(`Total Cases: ${filteredCases.length}`);
+    console.log(`Repeats:     ${flags.repeat}`);
+    console.log(`Total Runs:  ${totalRuns}`);
+    console.log(`Concurrency: ${flags.concurrency}`);
+    console.log(`Cache:       ${flags.cache ? 'Enabled' : 'Disabled'}`);
+    console.log('-----------------');
+    const estTokensPerRun = 1000;
+    const estCostPer1K = 0.01; // Conservative average for gpt-4o
+    const estTotalCost = (totalRuns * estTokensPerRun * estCostPer1K) / 1000;
+    console.log(`Estimated cost: ~$${estTotalCost.toFixed(2)} (very rough estimate)`);
+    if (flags.dryRun) {
+        console.log('Dry run complete.');
+        return;
+    }
+    if (!flags.yes) {
+        const readline = await import('node:readline');
+        const rl = readline.createInterface({
+            input: process.stdin,
+            output: process.stdout,
+        });
+        const answer = await new Promise((resolve) => rl.question('Continue? (y/N): ', resolve));
+        rl.close();
+        if (String(answer).toLowerCase() !== 'y') {
+            console.log('Aborted.');
+            return;
+        }
+    }
+    const date = new Date().toISOString().split('T')[0];
+    const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+    const reportFormats = flags.reportFormat.split(',').map((f) => f.trim());
+    const reportsRoot = join(dirname(datasetPath), 'reports');
+    const reportsDir = join(reportsRoot, date);
+    if (!existsSync(reportsDir)) {
+        mkdirSync(reportsDir, { recursive: true });
+    }
+    const baseOutputPath = flags.output || join(reportsDir, `results-${timestamp}`);
+    console.log(`Saving results to: ${baseOutputPath}.{${reportFormats.join(',')}}`);
+    const logger = new LoggingService({ disableLogging: true });
+    const cache = new ResponseCache('eval/auto-approval/.cache');
+    const results = [];
+    let completed = 0;
+    let cacheHits = 0;
+    async function runModelEvaluation(run) {
+        const settingsService = createMockSettingsService({
+            'agent.autoApproveModel': run.model,
+            'agent.autoApproveProvider': run.provider,
+            'shell.autoApproveMode': 'auto',
+            // Pass provider API keys from environment so runners can be initialized
+            ...(process.env['OPENROUTER_API_KEY'] ? { 'agent.openrouter.apiKey': process.env['OPENROUTER_API_KEY'] } : {}),
+        });
+        const agentClient = new OpenAIAgentClient({
+            model: run.model,
+            deps: {
+                logger,
+                settings: settingsService,
+            },
+        });
+        const queue = [...filteredCases.flatMap((c) => Array.from({ length: flags.repeat }, () => c))];
+        const active = [];
+        async function runCase(c) {
+            const cacheKey = createCacheKey({
+                model: run.model,
+                provider: run.provider,
+                promptVersion: SHELL_AUTO_APPROVAL_PROMPT_VERSION,
+                command: c.command,
+                history: c.history,
+            });
+            let cachedAdvisory = flags.cache ? cache.get(cacheKey) : null;
+            let result;
+            let error;
+            let latencyMs = 0;
+            if (cachedAdvisory) {
+                cacheHits++;
+                result = cachedAdvisory;
+            }
+            else {
+                const start = performance.now();
+                try {
+                    const advisories = await retryOnRateLimit({
+                        operation: () => evaluateShellAutoApprovalAdvisories({
+                            commands: [{ id: c.id, command: c.command }],
+                            history: c.history,
+                            settingsService,
+                            agentClient,
+                            logger,
+                            throwOnError: true,
+                        }),
+                        maxRetries: 2,
+                        onRetry: ({ attempt, retriesRemaining, delayMs, error: retryError }) => {
+                            console.warn(`Case ${c.id} hit a rate limit (attempt ${attempt}/2). Retrying in ${Math.round(delayMs)}ms${retriesRemaining > 0 ? `; ${retriesRemaining} retry left after this` : ''}.`);
+                            logger.warn('Eval runner rate-limit retry', {
+                                caseId: c.id,
+                                attempt,
+                                retriesRemaining,
+                                delayMs,
+                                error: retryError instanceof Error ? retryError.message : String(retryError),
+                            });
+                        },
+                    });
+                    const advisory = advisories.get(c.id);
+                    if (advisory) {
+                        if (advisory.isError) {
+                            error = advisory.reasoning;
+                        }
+                        else {
+                            result = {
+                                predicted: advisory.approved ? 'approve' : 'reject',
+                                reasoning: advisory.reasoning,
+                                source: advisory.source,
+                            };
+                            if (flags.cache && advisory.source === 'llm') {
+                                cache.set(cacheKey, result);
+                            }
+                        }
+                    }
+                    else {
+                        error = 'No advisory returned for command ID';
+                    }
+                }
+                catch (e) {
+                    error = e.message;
+                }
+                latencyMs = performance.now() - start;
+            }
+            const record = {
+                caseId: c.id,
+                command: c.command,
+                expected: c.expected,
+                predicted: result?.predicted,
+                reasoning: result?.reasoning,
+                source: result?.source,
+                latencyMs,
+                error,
+                timestamp: new Date().toISOString(),
+                model: run.model,
+                provider: run.provider,
+                promptVersion: SHELL_AUTO_APPROVAL_PROMPT_VERSION,
+                category: c.category,
+                severity: c.severity,
+                cached: !!cachedAdvisory,
+            };
+            appendFileSync(baseOutputPath + '.jsonl', JSON.stringify(record) + '\n');
+            results.push(record);
+            completed++;
+            const statusIcon = record.error ? '⚠️' : record.predicted === record.expected ? '✅' : '❌';
+            const cacheIcon = record.cached ? ' 🧊' : '';
+            console.log(`[${completed}/${totalRuns}] ${run.provider}/${run.model} Case ${c.id}: ${statusIcon}${cacheIcon} (${Math.round(latencyMs)}ms)`);
+        }
+        while (queue.length > 0 || active.length > 0) {
+            while (active.length < flags.concurrency && queue.length > 0) {
+                const c = queue.shift();
+                const p = runCase(c).then(() => {
+                    active.splice(active.indexOf(p), 1);
+                });
+                active.push(p);
+            }
+            if (active.length > 0) {
+                await Promise.race(active);
+            }
+        }
+    }
+    for (const run of modelRuns) {
+        if (modelRuns.length > 1) {
+            console.log(`\n--- Running ${run.provider}/${run.model} ---`);
+        }
+        await runModelEvaluation(run);
+    }
+    if (reportFormats.includes('json')) {
+        writeFileSync(baseOutputPath + '.json', JSON.stringify(results, null, 2));
+        console.log(`JSON results saved to ${baseOutputPath}.json`);
+    }
+    if (reportFormats.includes('jsonl') && !existsSync(baseOutputPath + '.jsonl')) {
+        // Already written incrementally, but ensure file exists if it was empty for some reason
+        // or if we want to explicitly move/rename it. Currently it's written as .jsonl always.
+    }
+    if (reportFormats.includes('md')) {
+        const metrics = computeMetrics(results);
+        const report = generateReport(results, metrics, { promptVersion: SHELL_AUTO_APPROVAL_PROMPT_VERSION });
+        writeFileSync(baseOutputPath + '.md', report);
+        console.log(`Markdown report saved to ${baseOutputPath}.md`);
+    }
+    const leaderboardPaths = getModelLeaderboardPaths(reportsRoot);
+    const persistedModelResults = loadPersistedModelResults(reportsRoot);
+    const mergedModelResults = mergeModelResults(persistedModelResults, results);
+    const modelLeaderboard = buildModelLeaderboard(mergedModelResults, {
+        promptVersion: SHELL_AUTO_APPROVAL_PROMPT_VERSION,
+    });
+    saveModelResults(leaderboardPaths.recordsPath, mergedModelResults);
+    saveModelLeaderboardJson(leaderboardPaths.jsonPath, modelLeaderboard, mergedModelResults, {
+        promptVersion: SHELL_AUTO_APPROVAL_PROMPT_VERSION,
+    });
+    saveModelLeaderboardMarkdown(leaderboardPaths.markdownPath, modelLeaderboard, mergedModelResults, {
+        promptVersion: SHELL_AUTO_APPROVAL_PROMPT_VERSION,
+    });
+    console.log(`Model leaderboard updated: ${leaderboardPaths.markdownPath}`);
+    // Cleanup .jsonl if it's not a requested format
+    if (!reportFormats.includes('jsonl') && existsSync(baseOutputPath + '.jsonl')) {
+        const { unlinkSync } = await import('node:fs');
+        unlinkSync(baseOutputPath + '.jsonl');
+    }
+    console.log('\nPruning cache...');
+    cache.prune(500);
+    console.log(`\nDone!`);
+    const metrics = computeMetrics(results);
+    console.log(`Cache hits: ${cacheHits}/${totalRuns}`);
+    console.log(`Accuracy: ${(metrics.accuracy * 100).toFixed(1)}%`);
+    if (modelLeaderboard[0]) {
+        console.log(`Top model: ${modelLeaderboard[0].provider}/${modelLeaderboard[0].model} (score ${modelLeaderboard[0].score.toFixed(2)}, ${modelLeaderboard[0].passed}/${modelLeaderboard[0].casesRun} passed)`);
+    }
+}
+run().catch((err) => {
+    console.error('Fatal error:', err);
+    process.exit(1);
+});
+//# sourceMappingURL=runner.js.map

package/dist/scripts/eval-auto-approval/runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../../source/scripts/eval-auto-approval/runner.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,WAAW,EAAE,aAAa,EAAQ,MAAM,cAAc,CAAC;AAChE,OAAO,EAAE,yBAAyB,EAAE,MAAM,yCAAyC,CAAC;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,mCAAmC,CAAC;AACnE,OAAO,EAAE,iBAAiB,EAAE,MAAM,kCAAkC,CAAC;AACrE,OAAO,EACL,mCAAmC,EACnC,kCAAkC,GACnC,MAAM,iDAAiD,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3C,OAAO,EACL,qBAAqB,EACrB,wBAAwB,EACxB,yBAAyB,EACzB,iBAAiB,EAEjB,wBAAwB,EACxB,4BAA4B,EAC5B,gBAAgB,GACjB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,cAAc,EAAE,qBAAqB,EAAE,gBAAgB,EAAE,qBAAqB,EAAE,MAAM,mBAAmB,CAAC;AACnH,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAC/E,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,MAAM,GAAG,GAAG,IAAI,CACd;;;;;;;;;;;;;;;;;;;;;;CAsBD,EACC;IACE,UAAU,EAAE,MAAM,CAAC,IAAI;IACvB,KAAK,EAAE;QACL,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;QACzB,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE;QAC/C,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,iCAAiC,EAAE;QACvE,UAAU,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,gCAAgC,EAAE;QACzE,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;QAC7B,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;QAC7B,WAAW,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,EAAE;QAC3C,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,EAAE;QACtC,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;QAC1B,YAAY,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE;QACjD,MAAM,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;QAC3B,GAAG,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;QACxB,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,IAAI,EAAE;QACzC,UAAU,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;KAChC;CACF,CACF,CAAC;AAEF,KAAK,UAAU,GAAG;IAChB,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;IAExB,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;QACrB,MAAM,KAAK,GAAG,IAAI,aAAa,CAAC,2BAA2B,CAAC,CAAC;QAC7D,KAAK,CAAC,KAAK,EAAE,CAAC;QACd,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;QAC9B,IAAI,KAAK,CAAC,MAAM;YAAE,OAAO;IAC3B,CAAC;IAED,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;QACnD,OAAO,CAAC,KAAK,CAAC,kFAAkF,CAAC,CAAC;QAClG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,WAAW,GAAG,KAAK,CAAC,OAAO,CAAC;IAClC,qBAAqB,CAAC;QACpB,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,MAAM,EAAE,KAAK,CAAC,MAAM;KACrB,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG,WAAW,CAAC,WAAW,CAAC,CAAC;IACvC,MAAM,aAAa,GAAG,aAAa,CAAC,KAAK,EAAE;QACzC,QAAQ,EAAE,KAAK,CAAC,SAAS;QACzB,QAAQ,EAAE,KAAK,CAAC,SAAS;KAC1B,CAAC,CAAC;IAEH,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/B,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC,CAAC;QAC7C,OAAO;IACT,CAAC;IAED,MAAM,SAAS,GACb,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,UAAU,KAAK,gCAAgC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,UAAU,CAAC;QACnG,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC;QACpD,CAAC,CAAC,KAAK,CAAC,UAAU,IAAI,UAAU,CAAC,KAAK,CAAC,UAAU,CAAC;YAClD,CAAC,CAAC,qBAAqB,CAAC,KAAK,CAAC,UAAU,CAAC;YACzC,CAAC,CAAC,KAAK,CAAC,KAAK;gBACb,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC;gBACpD,CAAC,CAAC,EAAE,CAAC;IAET,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,IAAI,CAAC,KAAK,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,CAAC,UAAU,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC;YACzE,OAAO,CAAC,KAAK,CAAC,+EAA+E,CAAC,CAAC;YAC/F,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC;QACzC,OAAO;IACT,CAAC;IAED,MAAM,SAAS,GAAG,aAAa,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC;IAEzE,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;IACjC,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;QACrB,OAAO,CAAC,GAAG,CAAC,iBAAiB,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC;QACjD,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;QAC3B,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;YAC5B,OAAO,CAAC,GAAG,CAAC,OAAO,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;QAClD,CAAC;IACH,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,gBAAgB,SAAS,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;QAClD,OAAO,CAAC,GAAG,CAAC,gBAAgB,SAAS,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;IACvD,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,gBAAgB,WAAW,EAAE,CAAC,CAAC;IAC3C,OAAO,CAAC,GAAG,CAAC,gBAAgB,aAAa,CAAC,MAAM,EAAE,CAAC,CAAC;IACpD,OAAO,CAAC,GAAG,CAAC,gBAAgB,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;IAC5C,OAAO,CAAC,GAAG,CAAC,gBAAgB,SAAS,EAAE,CAAC,CAAC;IACzC,OAAO,CAAC,GAAG,CAAC,gBAAgB,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC;IACjD,OAAO,CAAC,GAAG,CAAC,gBAAgB,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC;IACpE,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;IAEjC,MAAM,eAAe,GAAG,IAAI,CAAC;IAC7B,MAAM,YAAY,GAAG,IAAI,CAAC,CAAC,kCAAkC;IAC7D,MAAM,YAAY,GAAG,CAAC,SAAS,GAAG,eAAe,GAAG,YAAY,CAAC,GAAG,IAAI,CAAC;IAEzE,OAAO,CAAC,GAAG,CAAC,qBAAqB,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC;IAElF,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QACjB,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;QACjC,OAAO;IACT,CAAC;IAED,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;QACf,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,eAAe,CAAC,CAAC;QAC/C,MAAM,EAAE,GAAG,QAAQ,CAAC,eAAe,CAAC;YAClC,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,MAAM,EAAE,OAAO,CAAC,MAAM;SACvB,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,EAAE,CAAC,QAAQ,CAAC,mBAAmB,EAAE,OAAO,CAAC,CAAC,CAAC;QACzF,EAAE,CAAC,KAAK,EAAE,CAAC;QAEX,IAAI,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,KAAK,GAAG,EAAE,CAAC;YACzC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YACxB,OAAO;QACT,CAAC;IACH,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACpD,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IACjE,MAAM,aAAa,GAAI,KAAK,CAAC,YAAuB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IACrF,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,SAAS,CAAC,CAAC;IAC1D,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,EAAE,IAAK,CAAC,CAAC;IAE5C,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC5B,SAAS,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC7C,CAAC;IAED,MAAM,cAAc,GAAG,KAAK,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,EAAE,WAAW,SAAS,EAAE,CAAC,CAAC;IAEhF,OAAO,CAAC,GAAG,CAAC,sBAAsB,cAAc,KAAK,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAEjF,MAAM,MAAM,GAAG,IAAI,cAAc,CAAC,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC;IAE5D,MAAM,KAAK,GAAG,IAAI,aAAa,CAAC,2BAA2B,CAAC,CAAC;IAC7D,MAAM,OAAO,GAAwB,EAAE,CAAC;IAExC,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,KAAK,UAAU,kBAAkB,CAAC,GAAwC;QACxE,MAAM,eAAe,GAAG,yBAAyB,CAAC;YAChD,wBAAwB,EAAE,GAAG,CAAC,KAAK;YACnC,2BAA2B,EAAE,GAAG,CAAC,QAAQ;YACzC,uBAAuB,EAAE,MAAM;YAC/B,wEAAwE;YACxE,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,EAAE,yBAAyB,EAAE,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC/G,CAAC,CAAC;QAEH,MAAM,WAAW,GAAG,IAAI,iBAAiB,CAAC;YACxC,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,IAAI,EAAE;gBACJ,MAAM;gBACN,QAAQ,EAAE,eAAe;aAC1B;SACF,CAAC,CAAC;QAEH,MAAM,KAAK,GAAG,CAAC,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/F,MAAM,MAAM,GAAoB,EAAE,CAAC;QAEnC,KAAK,UAAU,OAAO,CAAC,CAAO;YAC5B,MAAM,QAAQ,GAAG,cAAc,CAAC;gBAC9B,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,aAAa,EAAE,kCAAkC;gBACjD,OAAO,EAAE,CAAC,CAAC,OAAO;gBAClB,OAAO,EAAE,CAAC,CAAC,OAAO;aACnB,CAAC,CAAC;YAEH,IAAI,cAAc,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YAC9D,IAAI,MAAW,CAAC;YAChB,IAAI,KAAyB,CAAC;YAC9B,IAAI,SAAS,GAAG,CAAC,CAAC;YAElB,IAAI,cAAc,EAAE,CAAC;gBACnB,SAAS,EAAE,CAAC;gBACZ,MAAM,GAAG,cAAc,CAAC;YAC1B,CAAC;iBAAM,CAAC;gBACN,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;gBAChC,IAAI,CAAC;oBACH,MAAM,UAAU,GAAG,MAAM,gBAAgB,CAAC;wBACxC,SAAS,EAAE,GAAG,EAAE,CACd,mCAAmC,CAAC;4BAClC,QAAQ,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;4BAC5C,OAAO,EAAE,CAAC,CAAC,OAAc;4BACzB,eAAe;4BACf,WAAW;4BACX,MAAM;4BACN,YAAY,EAAE,IAAI;yBACnB,CAAC;wBACJ,UAAU,EAAE,CAAC;wBACb,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,gBAAgB,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE,EAAE;4BACrE,OAAO,CAAC,IAAI,CACV,QAAQ,CAAC,CAAC,EAAE,8BAA8B,OAAO,oBAAoB,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KACtF,gBAAgB,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,gBAAgB,wBAAwB,CAAC,CAAC,CAAC,EACzE,GAAG,CACJ,CAAC;4BACF,MAAM,CAAC,IAAI,CAAC,8BAA8B,EAAE;gCAC1C,MAAM,EAAE,CAAC,CAAC,EAAE;gCACZ,OAAO;gCACP,gBAAgB;gCAChB,OAAO;gCACP,KAAK,EAAE,UAAU,YAAY,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC;6BAC7E,CAAC,CAAC;wBACL,CAAC;qBACF,CAAC,CAAC;oBAEH,MAAM,QAAQ,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;oBACtC,IAAI,QAAQ,EAAE,CAAC;wBACb,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;4BACrB,KAAK,GAAG,QAAQ,CAAC,SAAS,CAAC;wBAC7B,CAAC;6BAAM,CAAC;4BACN,MAAM,GAAG;gCACP,SAAS,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ;gCACnD,SAAS,EAAE,QAAQ,CAAC,SAAS;gCAC7B,MAAM,EAAE,QAAQ,CAAC,MAAM;6BACxB,CAAC;4BACF,IAAI,KAAK,CAAC,KAAK,IAAI,QAAQ,CAAC,MAAM,KAAK,KAAK,EAAE,CAAC;gCAC7C,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;4BAC9B,CAAC;wBACH,CAAC;oBACH,CAAC;yBAAM,CAAC;wBACN,KAAK,GAAG,qCAAqC,CAAC;oBAChD,CAAC;gBACH,CAAC;gBAAC,OAAO,CAAM,EAAE,CAAC;oBAChB,KAAK,GAAG,CAAC,CAAC,OAAO,CAAC;gBACpB,CAAC;gBACD,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;YACxC,CAAC;YAED,MAAM,MAAM,GAAsB;gBAChC,MAAM,EAAE,CAAC,CAAC,EAAE;gBACZ,OAAO,EAAE,CAAC,CAAC,OAAO;gBAClB,QAAQ,EAAE,CAAC,CAAC,QAAQ;gBACpB,SAAS,EAAE,MAAM,EAAE,SAAS;gBAC5B,SAAS,EAAE,MAAM,EAAE,SAAS;gBAC5B,MAAM,EAAE,MAAM,EAAE,MAAM;gBACtB,SAAS;gBACT,KAAK;gBACL,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,aAAa,EAAE,kCAAkC;gBACjD,QAAQ,EAAE,CAAC,CAAC,QAAQ;gBACpB,QAAQ,EAAE,CAAC,CAAC,QAAQ;gBACpB,MAAM,EAAE,CAAC,CAAC,cAAc;aACzB,CAAC;YAEF,cAAc,CAAC,cAAc,GAAG,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC;YACzE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACrB,SAAS,EAAE,CAAC;YACZ,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,KAAK,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;YAC1F,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;YAC7C,OAAO,CAAC,GAAG,CACT,IAAI,SAAS,IAAI,SAAS,KAAK,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,KAAK,SAAS,CAAC,CAAC,EAAE,KAAK,UAAU,GAAG,SAAS,KAAK,IAAI,CAAC,KAAK,CAC7G,SAAS,CACV,KAAK,CACP,CAAC;QACJ,CAAC;QAED,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7C,OAAO,MAAM,CAAC,MAAM,GAAG,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC7D,MAAM,CAAC,GAAG,KAAK,CAAC,KAAK,EAAG,CAAC;gBACzB,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE;oBAC7B,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;gBACtC,CAAC,CAAC,CAAC;gBACH,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACjB,CAAC;YACD,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtB,MAAM,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC7B,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;QAC5B,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzB,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,KAAK,MAAM,CAAC,CAAC;QAChE,CAAC;QACD,MAAM,kBAAkB,CAAC,GAAG,CAAC,CAAC;IAChC,CAAC;IAED,IAAI,aAAa,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QACnC,aAAa,CAAC,cAAc,GAAG,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC1E,OAAO,CAAC,GAAG,CAAC,yBAAyB,cAAc,OAAO,CAAC,CAAC;IAC9D,CAAC;IAED,IAAI,aAAa,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,cAAc,GAAG,QAAQ,CAAC,EAAE,CAAC;QAC9E,wFAAwF;QACxF,uFAAuF;IACzF,CAAC;IAED,IAAI,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACjC,MAAM,OAAO,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;QACxC,MAAM,MAAM,GAAG,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,aAAa,EAAE,kCAAkC,EAAE,CAAC,CAAC;QACvG,aAAa,CAAC,cAAc,GAAG,KAAK,EAAE,MAAM,CAAC,CAAC;QAC9C,OAAO,CAAC,GAAG,CAAC,4BAA4B,cAAc,KAAK,CAAC,CAAC;IAC/D,CAAC;IAED,MAAM,gBAAgB,GAAG,wBAAwB,CAAC,WAAW,CAAC,CAAC;IAC/D,MAAM,qBAAqB,GAAG,yBAAyB,CAAC,WAAW,CAAC,CAAC;IACrE,MAAM,kBAAkB,GAAG,iBAAiB,CAAC,qBAAqB,EAAE,OAAO,CAAC,CAAC;IAC7E,MAAM,gBAAgB,GAAG,qBAAqB,CAAC,kBAAkB,EAAE;QACjE,aAAa,EAAE,kCAAkC;KAClD,CAAC,CAAC;IAEH,gBAAgB,CAAC,gBAAgB,CAAC,WAAW,EAAE,kBAAkB,CAAC,CAAC;IACnE,wBAAwB,CAAC,gBAAgB,CAAC,QAAQ,EAAE,gBAAgB,EAAE,kBAAkB,EAAE;QACxF,aAAa,EAAE,kCAAkC;KAClD,CAAC,CAAC;IACH,4BAA4B,CAAC,gBAAgB,CAAC,YAAY,EAAE,gBAAgB,EAAE,kBAAkB,EAAE;QAChG,aAAa,EAAE,kCAAkC;KAClD,CAAC,CAAC;IAEH,OAAO,CAAC,GAAG,CAAC,8BAA8B,gBAAgB,CAAC,YAAY,EAAE,CAAC,CAAC;IAE3E,gDAAgD;IAChD,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,UAAU,CAAC,cAAc,GAAG,QAAQ,CAAC,EAAE,CAAC;QAC9E,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;QAC/C,UAAU,CAAC,cAAc,GAAG,QAAQ,CAAC,CAAC;IACxC,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;IAClC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAEjB,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IACvB,MAAM,OAAO,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACxC,OAAO,CAAC,GAAG,CAAC,eAAe,SAAS,IAAI,SAAS,EAAE,CAAC,CAAC;IACrD,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,OAAO,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IACjE,IAAI,gBAAgB,CAAC,CAAC,CAAC,EAAE,CAAC;QACxB,OAAO,CAAC,GAAG,CACT,cAAc,gBAAgB,CAAC,CAAC,CAAC,CAAC,QAAQ,IACxC,gBAAgB,CAAC,CAAC,CAAC,CAAC,KACtB,WAAW,gBAAgB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,gBAAgB,CAAC,CAAC,CAAC,CAAC,MAAM,IAC5E,gBAAgB,CAAC,CAAC,CAAC,CAAC,QACtB,UAAU,CACX,CAAC;IACJ,CAAC;AACH,CAAC;AAED,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;IAClB,OAAO,CAAC,KAAK,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC;IACnC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}

package/dist/services/agent-stream.d.ts

@@ -0,0 +1,11 @@
+export type AgentStream = AsyncIterable<unknown> & {
+    completed: Promise<unknown>;
+    rawResponses?: unknown[];
+    lastResponseId?: string | null;
+    interruptions?: unknown[];
+    state?: unknown;
+    newItems?: unknown[];
+    history?: unknown[];
+    finalOutput?: string;
+};
+//# sourceMappingURL=agent-stream.d.ts.map

package/dist/services/agent-stream.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"agent-stream.d.ts","sourceRoot":"","sources":["../../source/services/agent-stream.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,WAAW,GAAG,aAAa,CAAC,OAAO,CAAC,GAAG;IACjD,SAAS,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;IAC5B,YAAY,CAAC,EAAE,OAAO,EAAE,CAAC;IACzB,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,aAAa,CAAC,EAAE,OAAO,EAAE,CAAC;IAC1B,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,QAAQ,CAAC,EAAE,OAAO,EAAE,CAAC;IACrB,OAAO,CAAC,EAAE,OAAO,EAAE,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,CAAC"}

package/dist/services/agent-stream.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=agent-stream.js.map

package/dist/services/agent-stream.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"agent-stream.js","sourceRoot":"","sources":["../../source/services/agent-stream.ts"],"names":[],"mappings":""}

package/dist/services/approval-flow-coordinator.d.ts

@@ -0,0 +1,43 @@
+import type { OpenAIAgentClient } from '../lib/openai-agent-client.js';
+import type { ILoggingService } from './service-interfaces.js';
+import type { ConversationEvent } from './conversation-events.js';
+import { ApprovalState, type AbortedApprovalContext, type PendingApprovalContext } from './approval-state.js';
+export interface ApprovalFlowCoordinatorDeps {
+    agentClient: OpenAIAgentClient;
+    approvalState: ApprovalState;
+    logger: ILoggingService;
+    sessionId: string;
+}
+export interface AbortResolutionPlan {
+    abortedContext: AbortedApprovalContext;
+    /** Cleanup that removes the rejection interceptor; always safe to call. */
+    removeInterceptor: () => void;
+}
+export interface ContinuationPlan {
+    pendingApprovalContext: PendingApprovalContext;
+    toolStartedEvent?: ConversationEvent;
+    /** Cleanup that removes the rejection interceptor; always safe to call. */
+    removeInterceptor: () => void;
+}
+export declare class ApprovalFlowCoordinator {
+    private readonly deps;
+    constructor(deps: ApprovalFlowCoordinatorDeps);
+    abort(): boolean;
+    consumeAborted(): AbortedApprovalContext | null;
+    /**
+     * Drive an aborted-approval resolution: install rejection interceptor and approve the
+     * interruption so the agent gets the rejection text via the interceptor.
+     */
+    prepareAbortResolution(abortedContext: AbortedApprovalContext, userText: string): AbortResolutionPlan;
+    /**
+     * Prepare for a continuation after the user makes an approval decision.
+     * Returns null if there is no pending approval. Caller is responsible for invoking
+     * `removeInterceptor` in a finally block once the continuation stream completes.
+     */
+    prepareContinuation(answer: string, rejectionReason: string | undefined): ContinuationPlan | null;
+    recordPending(pending: PendingApprovalContext): void;
+    clearPending(): void;
+    getPending(): PendingApprovalContext | null;
+    getPendingInterruption(): unknown;
+}
+//# sourceMappingURL=approval-flow-coordinator.d.ts.map

package/dist/services/approval-flow-coordinator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"approval-flow-coordinator.d.ts","sourceRoot":"","sources":["../../source/services/approval-flow-coordinator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AACvE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAClE,OAAO,EAAE,aAAa,EAAE,KAAK,sBAAsB,EAAE,KAAK,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAe9G,MAAM,WAAW,2BAA2B;IAC1C,WAAW,EAAE,iBAAiB,CAAC;IAC/B,aAAa,EAAE,aAAa,CAAC;IAC7B,MAAM,EAAE,eAAe,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,mBAAmB;IAClC,cAAc,EAAE,sBAAsB,CAAC;IACvC,2EAA2E;IAC3E,iBAAiB,EAAE,MAAM,IAAI,CAAC;CAC/B;AAED,MAAM,WAAW,gBAAgB;IAC/B,sBAAsB,EAAE,sBAAsB,CAAC;IAC/C,gBAAgB,CAAC,EAAE,iBAAiB,CAAC;IACrC,2EAA2E;IAC3E,iBAAiB,EAAE,MAAM,IAAI,CAAC;CAC/B;AAED,qBAAa,uBAAuB;IACtB,OAAO,CAAC,QAAQ,CAAC,IAAI;gBAAJ,IAAI,EAAE,2BAA2B;IAE9D,KAAK,IAAI,OAAO;IAehB,cAAc,IAAI,sBAAsB,GAAG,IAAI;IAI/C;;;OAGG;IACH,sBAAsB,CAAC,cAAc,EAAE,sBAAsB,EAAE,QAAQ,EAAE,MAAM,GAAG,mBAAmB;IAkBrG;;;;OAIG;IACH,mBAAmB,CAAC,MAAM,EAAE,MAAM,EAAE,eAAe,EAAE,MAAM,GAAG,SAAS,GAAG,gBAAgB,GAAG,IAAI;IAoEjG,aAAa,CAAC,OAAO,EAAE,sBAAsB,GAAG,IAAI;IAIpD,YAAY,IAAI,IAAI;IAIpB,UAAU,IAAI,sBAAsB,GAAG,IAAI;IAI3C,sBAAsB,IAAI,OAAO;CAGlC"}