@rulemetric/cli 0.1.1 → 0.2.0

package/dist/lib/handlers/cron-suggest-instructions.js

@@ -0,0 +1,159 @@
+import { readFileSync } from 'node:fs';
+import path from 'node:path';
+import os from 'node:os';
+import { apiGet, apiPost } from '../api-client.js';
+import { runLlmPrompt, detectLlmBackend } from '../llm-client.js';
+import { buildRecommendationPrompt } from '@rulemetric/core';
+import { normalizeProjectPath } from '@rulemetric/core/paths';
+import { detectLanguages } from '../detect-languages.js';
+const MAX_SUGGESTIONS_PER_PROJECT = 5;
+const MAX_PROJECTS_PER_RUN = Number(process.env.RULEMETRIC_MAX_SUGGESTION_PROJECTS ?? 10);
+const STALE_THRESHOLD_HOURS = 48;
+const MIN_SCORE = 0.5;
+const CONTENT_PREVIEW_CHARS = 400;
+async function checkSkillsRegistryStale() {
+    const registryPath = path.join(os.homedir(), '.config', 'rulemetric', 'skills-registry.json');
+    try {
+        const raw = readFileSync(registryPath, 'utf-8');
+        const registry = JSON.parse(raw);
+        const ageMs = Date.now() - new Date(registry.generatedAt).getTime();
+        return ageMs > STALE_THRESHOLD_HOURS * 60 * 60 * 1000;
+    }
+    catch {
+        return true;
+    }
+}
+export const cronSuggestInstructions = async (_payload, helpers) => {
+    // Scoring uses the user's locally-installed LLM tooling — Claude Code CLI
+    // when available, otherwise the Anthropic SDK with ANTHROPIC_API_KEY.
+    // No third-party LLM_PROVIDER / LLM_API_KEY required. If neither backend
+    // is reachable, skip cleanly.
+    const backend = detectLlmBackend();
+    if (backend === 'none') {
+        helpers.logger.warn('No LLM backend available (claude CLI missing and ANTHROPIC_API_KEY unset); skipping suggestion compute');
+        return;
+    }
+    helpers.logger.info(`Scoring backend: ${backend}`);
+    // 1. Fetch recent project paths
+    let projectPaths;
+    try {
+        const res = await apiGet('/api/sessions/project-paths?limit=' + MAX_PROJECTS_PER_RUN);
+        projectPaths = res.projectPaths;
+    }
+    catch (err) {
+        helpers.logger.warn(`Failed to fetch project paths: ${err instanceof Error ? err.message : String(err)}`);
+        return;
+    }
+    if (projectPaths.length === 0) {
+        helpers.logger.info('No recent project paths; nothing to score');
+        return;
+    }
+    // 2. Fetch candidate instructions: public+org+team from /catalog, plus
+    //    the user's own (personal+created) from /instructions. Dedup by id.
+    const candidates = new Map();
+    try {
+        const catalogRows = await apiGet('/api/catalog?limit=200');
+        for (const row of catalogRows) {
+            candidates.set(row.id, {
+                id: row.id,
+                name: row.name,
+                description: row.description ?? '',
+                tags: row.tags ?? [],
+                contentPreview: (row.content ?? '').slice(0, CONTENT_PREVIEW_CHARS),
+            });
+        }
+    }
+    catch (err) {
+        helpers.logger.warn(`Failed to fetch catalog: ${err instanceof Error ? err.message : String(err)}`);
+    }
+    try {
+        const personalRes = await apiGet('/api/instructions?limit=100');
+        for (const row of personalRes.instructions ?? []) {
+            if (!candidates.has(row.id)) {
+                candidates.set(row.id, {
+                    id: row.id,
+                    name: row.name,
+                    description: row.description ?? '',
+                    tags: row.tags ?? [],
+                    contentPreview: (row.content ?? '').slice(0, CONTENT_PREVIEW_CHARS),
+                });
+            }
+        }
+    }
+    catch (err) {
+        helpers.logger.warn(`Failed to fetch personal instructions: ${err instanceof Error ? err.message : String(err)}`);
+    }
+    const candidateList = [...candidates.values()];
+    if (candidateList.length === 0) {
+        helpers.logger.info('No candidate instructions; nothing to score');
+        return;
+    }
+    helpers.logger.info(`Scoring ${candidateList.length} candidates across ${projectPaths.length} project(s)`);
+    // 3. Score per project
+    let scored = 0;
+    for (const rawPath of projectPaths) {
+        try {
+            const projectPath = normalizeProjectPath(rawPath);
+            const languages = detectLanguages(projectPath);
+            if (languages.length === 0) {
+                helpers.logger.info(`Skipping ${projectPath}: insufficient language signals`);
+                continue;
+            }
+            const context = {
+                name: path.basename(projectPath),
+                description: null,
+                languages,
+                topics: [],
+                dependencies: [],
+            };
+            const prompt = buildRecommendationPrompt(context, candidateList);
+            // runLlmPrompt expects (systemPrompt, data, maxTokens). The recommendation
+            // prompt is fully self-contained; pass an empty data block.
+            const parsed = await runLlmPrompt(prompt, '', 4096);
+            if (!parsed) {
+                helpers.logger.warn(`LLM returned null for ${projectPath} (backend ${backend})`);
+                continue;
+            }
+            const raw = (parsed.recommendations ?? []);
+            const recommendations = raw
+                .filter((r) => typeof r.instructionId === 'string' && typeof r.score === 'number' && typeof r.reason === 'string')
+                .map((r) => ({
+                instructionId: r.instructionId,
+                score: r.score,
+                reason: r.reason,
+            }));
+            const top = recommendations
+                .filter((r) => r.score >= MIN_SCORE)
+                .sort((a, b) => b.score - a.score)
+                .slice(0, MAX_SUGGESTIONS_PER_PROJECT);
+            if (top.length === 0) {
+                helpers.logger.info(`No suggestions over threshold for ${projectPath}`);
+                continue;
+            }
+            await apiPost('/api/instruction-suggestions/batch-upsert', {
+                projectPath,
+                suggestions: top.map((r) => ({
+                    instructionId: r.instructionId,
+                    score: r.score,
+                    reason: r.reason,
+                })),
+            });
+            scored++;
+        }
+        catch (err) {
+            helpers.logger.warn(`Failed to score ${rawPath}: ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+    // 4. Mark stale if registry is old
+    if (await checkSkillsRegistryStale()) {
+        try {
+            await apiPost('/api/instruction-suggestions/mark-stale', {});
+            helpers.logger.warn('Skills registry is stale (>48h); marked all suggestions stale');
+        }
+        catch {
+            // Non-fatal
+        }
+    }
+    helpers.logger.info(`Done: scored ${scored}/${projectPaths.length} projects`);
+};
+//# sourceMappingURL=cron-suggest-instructions.js.map

package/dist/lib/handlers/cron-suggest-instructions.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cron-suggest-instructions.js","sourceRoot":"","sources":["../../../src/lib/handlers/cron-suggest-instructions.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAClE,OAAO,EAAE,yBAAyB,EAAE,MAAM,kBAAkB,CAAC;AAC7D,OAAO,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AAE9D,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAEzD,MAAM,2BAA2B,GAAG,CAAC,CAAC;AACtC,MAAM,oBAAoB,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,kCAAkC,IAAI,EAAE,CAAC,CAAC;AAC1F,MAAM,qBAAqB,GAAG,EAAE,CAAC;AACjC,MAAM,SAAS,GAAG,GAAG,CAAC;AACtB,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAwBlC,KAAK,UAAU,wBAAwB;IACrC,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,YAAY,EAAE,sBAAsB,CAAC,CAAC;IAC9F,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,YAAY,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;QAChD,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAuB,CAAC;QACvD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,OAAO,EAAE,CAAC;QACpE,OAAO,KAAK,GAAG,qBAAqB,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;IACxD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,CAAC,MAAM,uBAAuB,GAAS,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,EAAE;IACvE,0EAA0E;IAC1E,sEAAsE;IACtE,yEAAyE;IACzE,8BAA8B;IAC9B,MAAM,OAAO,GAAG,gBAAgB,EAAE,CAAC;IACnC,IAAI,OAAO,KAAK,MAAM,EAAE,CAAC;QACvB,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,wGAAwG,CAAC,CAAC;QAC9H,OAAO;IACT,CAAC;IACD,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,oBAAoB,OAAO,EAAE,CAAC,CAAC;IAEnD,gCAAgC;IAChC,IAAI,YAAsB,CAAC;IAC3B,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,MAAM,CAAuB,oCAAoC,GAAG,oBAAoB,CAAC,CAAC;QAC5G,YAAY,GAAG,GAAG,CAAC,YAAY,CAAC;IAClC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,kCAAkC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC1G,OAAO;IACT,CAAC;IAED,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC9B,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,2CAA2C,CAAC,CAAC;QACjE,OAAO;IACT,CAAC;IAED,uEAAuE;IACvE,wEAAwE;IACxE,MAAM,UAAU,GAAG,IAAI,GAAG,EAAgC,CAAC;IAC3D,IAAI,CAAC;QACH,MAAM,WAAW,GAAG,MAAM,MAAM,CAAe,wBAAwB,CAAC,CAAC;QACzE,KAAK,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;YAC9B,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE;gBACrB,EAAE,EAAE,GAAG,CAAC,EAAE;gBACV,IAAI,EAAE,GAAG,CAAC,IAAI;gBACd,WAAW,EAAE,GAAG,CAAC,WAAW,IAAI,EAAE;gBAClC,IAAI,EAAE,GAAG,CAAC,IAAI,IAAI,EAAE;gBACpB,cAAc,EAAE,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,qBAAqB,CAAC;aACpE,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,4BAA4B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACtG,CAAC;IACD,IAAI,CAAC;QACH,MAAM,WAAW,GAAG,MAAM,MAAM,CAAiC,6BAA6B,CAAC,CAAC;QAChG,KAAK,MAAM,GAAG,IAAI,WAAW,CAAC,YAAY,IAAI,EAAE,EAAE,CAAC;YACjD,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;gBAC5B,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE;oBACrB,EAAE,EAAE,GAAG,CAAC,EAAE;oBACV,IAAI,EAAE,GAAG,CAAC,IAAI;oBACd,WAAW,EAAE,GAAG,CAAC,WAAW,IAAI,EAAE;oBAClC,IAAI,EAAE,GAAG,CAAC,IAAI,IAAI,EAAE;oBACpB,cAAc,EAAE,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,qBAAqB,CAAC;iBACpE,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,0CAA0C,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACpH,CAAC;IAED,MAAM,aAAa,GAAG,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;IAC/C,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/B,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,6CAA6C,CAAC,CAAC;QACnE,OAAO;IACT,CAAC;IACD,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,aAAa,CAAC,MAAM,sBAAsB,YAAY,CAAC,MAAM,aAAa,CAAC,CAAC;IAE3G,uBAAuB;IACvB,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,MAAM,OAAO,IAAI,YAAY,EAAE,CAAC;QACnC,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,oBAAoB,CAAC,OAAO,CAAC,CAAC;YAClD,MAAM,SAAS,GAAG,eAAe,CAAC,WAAW,CAAC,CAAC;YAC/C,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC3B,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,YAAY,WAAW,iCAAiC,CAAC,CAAC;gBAC9E,SAAS;YACX,CAAC;YAED,MAAM,OAAO,GAAmB;gBAC9B,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC;gBAChC,WAAW,EAAE,IAAI;gBACjB,SAAS;gBACT,MAAM,EAAE,EAAE;gBACV,YAAY,EAAE,EAAE;aACjB,CAAC;YAEF,MAAM,MAAM,GAAG,yBAAyB,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;YACjE,2EAA2E;YAC3E,4DAA4D;YAC5D,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,MAAM,EAAE,EAAE,EAAE,IAAI,CAAC,CAAC;YACpD,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,yBAAyB,WAAW,aAAa,OAAO,GAAG,CAAC,CAAC;gBACjF,SAAS;YACX,CAAC;YACD,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,eAAe,IAAI,EAAE,CAAmC,CAAC;YAC7E,MAAM,eAAe,GAAqB,GAAG;iBAC1C,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,aAAa,KAAK,QAAQ,IAAI,OAAO,CAAC,CAAC,KAAK,KAAK,QAAQ,IAAI,OAAO,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC;iBACjH,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACX,aAAa,EAAE,CAAC,CAAC,aAAuB;gBACxC,KAAK,EAAE,CAAC,CAAC,KAAe;gBACxB,MAAM,EAAE,CAAC,CAAC,MAAgB;aAC3B,CAAC,CAAC,CAAC;YAEN,MAAM,GAAG,GAAG,eAAe;iBACxB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,SAAS,CAAC;iBACnC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;iBACjC,KAAK,CAAC,CAAC,EAAE,2BAA2B,CAAC,CAAC;YAEzC,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACrB,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,qCAAqC,WAAW,EAAE,CAAC,CAAC;gBACxE,SAAS;YACX,CAAC;YAED,MAAM,OAAO,CAAsB,2CAA2C,EAAE;gBAC9E,WAAW;gBACX,WAAW,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;oBAC3B,aAAa,EAAE,CAAC,CAAC,aAAa;oBAC9B,KAAK,EAAE,CAAC,CAAC,KAAK;oBACd,MAAM,EAAE,CAAC,CAAC,MAAM;iBACjB,CAAC,CAAC;aACJ,CAAC,CAAC;YACH,MAAM,EAAE,CAAC;QACX,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,mBAAmB,OAAO,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACzG,CAAC;IACH,CAAC;IAED,mCAAmC;IACnC,IAAI,MAAM,wBAAwB,EAAE,EAAE,CAAC;QACrC,IAAI,CAAC;YACH,MAAM,OAAO,CAAC,yCAAyC,EAAE,EAAE,CAAC,CAAC;YAC7D,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,+DAA+D,CAAC,CAAC;QACvF,CAAC;QAAC,MAAM,CAAC;YACP,YAAY;QACd,CAAC;IACH,CAAC;IAED,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,gBAAgB,MAAM,IAAI,YAAY,CAAC,MAAM,WAAW,CAAC,CAAC;AAChF,CAAC,CAAC"}

package/dist/lib/handlers/cron-suggest-instructions.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=cron-suggest-instructions.test.d.ts.map

package/dist/lib/handlers/cron-suggest-instructions.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cron-suggest-instructions.test.d.ts","sourceRoot":"","sources":["../../../src/lib/handlers/cron-suggest-instructions.test.ts"],"names":[],"mappings":""}

package/dist/lib/handlers/cron-suggest-instructions.test.js

@@ -0,0 +1,80 @@
+import { describe, it, expect, beforeAll, afterAll } from 'vitest';
+import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+import { detectLanguages } from '../detect-languages.js';
+describe('detectLanguages', () => {
+    let root;
+    beforeAll(() => {
+        root = mkdtempSync(path.join(tmpdir(), 'rulemetric-detect-'));
+    });
+    afterAll(() => {
+        rmSync(root, { recursive: true, force: true });
+    });
+    function fresh(name) {
+        const dir = path.join(root, name);
+        mkdirSync(dir, { recursive: true });
+        return dir;
+    }
+    it('returns [] for an empty directory', () => {
+        const dir = fresh('empty');
+        expect(detectLanguages(dir)).toEqual([]);
+    });
+    it('returns [] for a single signal (insufficient evidence)', () => {
+        const dir = fresh('one-signal');
+        writeFileSync(path.join(dir, 'package.json'), '{}');
+        // Just package.json with no .ts/.js source = 1 signal
+        expect(detectLanguages(dir)).toEqual([]);
+    });
+    it('detects TypeScript from package.json + tsconfig.json', () => {
+        const dir = fresh('ts-config');
+        writeFileSync(path.join(dir, 'package.json'), '{}');
+        writeFileSync(path.join(dir, 'tsconfig.json'), '{}');
+        expect(detectLanguages(dir)).toContain('TypeScript');
+    });
+    it('detects TypeScript from package.json + a .ts source file', () => {
+        const dir = fresh('ts-source');
+        writeFileSync(path.join(dir, 'package.json'), '{}');
+        const src = path.join(dir, 'src');
+        mkdirSync(src);
+        writeFileSync(path.join(src, 'index.ts'), 'export {};');
+        expect(detectLanguages(dir)).toContain('TypeScript');
+    });
+    it('detects Python from pyproject.toml + requirements.txt', () => {
+        const dir = fresh('py');
+        writeFileSync(path.join(dir, 'pyproject.toml'), '');
+        writeFileSync(path.join(dir, 'requirements.txt'), '');
+        expect(detectLanguages(dir)).toContain('Python');
+    });
+    it('detects multiple languages in a polyglot project', () => {
+        const dir = fresh('polyglot');
+        writeFileSync(path.join(dir, 'package.json'), '{}');
+        writeFileSync(path.join(dir, 'pyproject.toml'), '');
+        const langs = detectLanguages(dir);
+        expect(langs).toContain('JavaScript');
+        expect(langs).toContain('Python');
+    });
+    it('walks 3 levels deep (monorepo case)', () => {
+        const dir = fresh('mono');
+        // packages/foo/src/index.ts is 3 levels deep
+        const deep = path.join(dir, 'packages', 'foo', 'src');
+        mkdirSync(deep, { recursive: true });
+        writeFileSync(path.join(dir, 'package.json'), '{}');
+        writeFileSync(path.join(deep, 'index.ts'), 'export {};');
+        expect(detectLanguages(dir)).toContain('TypeScript');
+    });
+    it('skips excluded directories like node_modules', () => {
+        const dir = fresh('exclude');
+        // ONLY signals are inside node_modules — should not count
+        const nm = path.join(dir, 'node_modules', 'foo');
+        mkdirSync(nm, { recursive: true });
+        writeFileSync(path.join(nm, 'package.json'), '{}');
+        writeFileSync(path.join(nm, 'index.ts'), 'export {};');
+        // Root itself has nothing → 0 signals → []
+        expect(detectLanguages(dir)).toEqual([]);
+    });
+    it('returns [] for a non-existent path without throwing', () => {
+        expect(detectLanguages(path.join(root, 'nonexistent'))).toEqual([]);
+    });
+});
+//# sourceMappingURL=cron-suggest-instructions.test.js.map

package/dist/lib/handlers/cron-suggest-instructions.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cron-suggest-instructions.test.js","sourceRoot":"","sources":["../../../src/lib/handlers/cron-suggest-instructions.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AACnE,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACxE,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAEzD,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;IAC7B,IAAI,IAAY,CAAC;IAEjB,SAAS,CAAC,GAAG,EAAE;QACX,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,oBAAoB,CAAC,CAAC,CAAC;IAClE,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,GAAG,EAAE;QACV,MAAM,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,SAAS,KAAK,CAAC,IAAY;QACvB,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAClC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACpC,OAAO,GAAG,CAAC;IACf,CAAC;IAED,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;QACzC,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC;QAC3B,MAAM,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wDAAwD,EAAE,GAAG,EAAE;QAC9D,MAAM,GAAG,GAAG,KAAK,CAAC,YAAY,CAAC,CAAC;QAChC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,cAAc,CAAC,EAAE,IAAI,CAAC,CAAC;QACpD,sDAAsD;QACtD,MAAM,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sDAAsD,EAAE,GAAG,EAAE;QAC5D,MAAM,GAAG,GAAG,KAAK,CAAC,WAAW,CAAC,CAAC;QAC/B,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,cAAc,CAAC,EAAE,IAAI,CAAC,CAAC;QACpD,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,eAAe,CAAC,EAAE,IAAI,CAAC,CAAC;QACrD,MAAM,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;IACzD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0DAA0D,EAAE,GAAG,EAAE;QAChE,MAAM,GAAG,GAAG,KAAK,CAAC,WAAW,CAAC,CAAC;QAC/B,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,cAAc,CAAC,EAAE,IAAI,CAAC,CAAC;QACpD,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAClC,SAAS,CAAC,GAAG,CAAC,CAAC;QACf,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,CAAC,EAAE,YAAY,CAAC,CAAC;QACxD,MAAM,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;IACzD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uDAAuD,EAAE,GAAG,EAAE;QAC7D,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC;QACxB,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,gBAAgB,CAAC,EAAE,EAAE,CAAC,CAAC;QACpD,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,kBAAkB,CAAC,EAAE,EAAE,CAAC,CAAC;QACtD,MAAM,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kDAAkD,EAAE,GAAG,EAAE;QACxD,MAAM,GAAG,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC;QAC9B,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,cAAc,CAAC,EAAE,IAAI,CAAC,CAAC;QACpD,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,gBAAgB,CAAC,EAAE,EAAE,CAAC,CAAC;QACpD,MAAM,KAAK,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC;QACnC,MAAM,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;QACtC,MAAM,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;IACtC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;QAC3C,MAAM,GAAG,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;QAC1B,6CAA6C;QAC7C,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;QACtD,SAAS,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACrC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,cAAc,CAAC,EAAE,IAAI,CAAC,CAAC;QACpD,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC,EAAE,YAAY,CAAC,CAAC;QACzD,MAAM,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;IACzD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;QACpD,MAAM,GAAG,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC;QAC7B,0DAA0D;QAC1D,MAAM,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,cAAc,EAAE,KAAK,CAAC,CAAC;QACjD,SAAS,CAAC,EAAE,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACnC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,cAAc,CAAC,EAAE,IAAI,CAAC,CAAC;QACnD,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,UAAU,CAAC,EAAE,YAAY,CAAC,CAAC;QACvD,2CAA2C;QAC3C,MAAM,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qDAAqD,EAAE,GAAG,EAAE;QAC3D,MAAM,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IACxE,CAAC,CAAC,CAAC;AACP,CAAC,CAAC,CAAC"}

package/dist/lib/handlers/process-conversation.d.ts

@@ -0,0 +1,35 @@
+import type { JobHelpers } from 'graphile-worker';
+interface ConversationJobConfig {
+    type: 'conversation';
+    conversationId: string;
+    messageId: string;
+    messageContent: string;
+}
+interface ConversationPayload {
+    mirrorId: string;
+    evalTargetId: string;
+    config: ConversationJobConfig;
+}
+/**
+ * Graphile-shaped handler for the `process_conversation` task.
+ *
+ * Thin wrapper around `eval-conversation-handler.processConversationJob` —
+ * the existing function already does all the work (action decision,
+ * run_evals/generate_cases/analyze branches, idempotency guard via
+ * already-posted assistant messages). The handler just unpacks the payload
+ * and threads the helpers' logger through.
+ *
+ * Mechanical contract vs. the legacy worker:
+ *   - no claim — Graphile claimed the job before invoking us
+ *   - no /complete POST — we return a small marker on success (matches the
+ *     `{ type: 'conversation' }` blob the legacy code POSTed)
+ *   - no surrounding try/catch — errors propagate to Graphile, which flips
+ *     the mirror row to `failed` via the runner.events handler
+ *
+ * Conversation jobs run with `maxAttempts: 1` (set at enqueue time in
+ * `apps/api/src/lib/queue.ts`) — chat is interactive and silent retries
+ * would confuse users mid-flow.
+ */
+export declare function processConversation(payload: ConversationPayload, helpers: JobHelpers): Promise<Record<string, unknown>>;
+export {};
+//# sourceMappingURL=process-conversation.d.ts.map

package/dist/lib/handlers/process-conversation.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"process-conversation.d.ts","sourceRoot":"","sources":["../../../src/lib/handlers/process-conversation.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAIlD,UAAU,qBAAqB;IAC7B,IAAI,EAAE,cAAc,CAAC;IACrB,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,UAAU,mBAAmB;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,qBAAqB,CAAC;CAC/B;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAsB,mBAAmB,CACvC,OAAO,EAAE,mBAAmB,EAC5B,OAAO,EAAE,UAAU,GAClB,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAclC"}

package/dist/lib/handlers/process-conversation.js

@@ -0,0 +1,35 @@
+import { apiPatch } from '../api-client.js';
+import { processConversationJob } from '../eval-conversation-handler.js';
+/**
+ * Graphile-shaped handler for the `process_conversation` task.
+ *
+ * Thin wrapper around `eval-conversation-handler.processConversationJob` —
+ * the existing function already does all the work (action decision,
+ * run_evals/generate_cases/analyze branches, idempotency guard via
+ * already-posted assistant messages). The handler just unpacks the payload
+ * and threads the helpers' logger through.
+ *
+ * Mechanical contract vs. the legacy worker:
+ *   - no claim — Graphile claimed the job before invoking us
+ *   - no /complete POST — we return a small marker on success (matches the
+ *     `{ type: 'conversation' }` blob the legacy code POSTed)
+ *   - no surrounding try/catch — errors propagate to Graphile, which flips
+ *     the mirror row to `failed` via the runner.events handler
+ *
+ * Conversation jobs run with `maxAttempts: 1` (set at enqueue time in
+ * `apps/api/src/lib/queue.ts`) — chat is interactive and silent retries
+ * would confuse users mid-flow.
+ */
+export async function processConversation(payload, helpers) {
+    const { mirrorId, evalTargetId, config } = payload;
+    const log = (msg) => helpers.logger.info(msg);
+    await processConversationJob(config, evalTargetId, log);
+    const result = { type: 'conversation', conversationId: config.conversationId };
+    // Graphile discards task return values, so persist a small marker onto
+    // the mirror row so the dashboard can tell the job finished. Best-effort.
+    await apiPatch(`/api/eval-jobs/${mirrorId}/state`, { result }).catch((err) => {
+        log(`result persist failed: ${err instanceof Error ? err.message : String(err)}`);
+    });
+    return result;
+}
+//# sourceMappingURL=process-conversation.js.map

package/dist/lib/handlers/process-conversation.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"process-conversation.js","sourceRoot":"","sources":["../../../src/lib/handlers/process-conversation.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,sBAAsB,EAAE,MAAM,iCAAiC,CAAC;AAezE;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,OAA4B,EAC5B,OAAmB;IAEnB,MAAM,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IACnD,MAAM,GAAG,GAAG,CAAC,GAAW,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAEtD,MAAM,sBAAsB,CAAC,MAAM,EAAE,YAAY,EAAE,GAAG,CAAC,CAAC;IACxD,MAAM,MAAM,GAAG,EAAE,IAAI,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,CAAC,cAAc,EAAE,CAAC;IAE/E,uEAAuE;IACvE,0EAA0E;IAC1E,MAAM,QAAQ,CAAC,kBAAkB,QAAQ,QAAQ,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,GAAY,EAAE,EAAE;QACpF,GAAG,CAAC,0BAA0B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACpF,CAAC,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/lib/handlers/process-eval.d.ts

@@ -0,0 +1,32 @@
+import type { JobHelpers } from 'graphile-worker';
+interface EvalPayload {
+    mirrorId: string;
+    evalTargetId: string;
+    config: {
+        parallel?: number;
+        runsPerEval?: number;
+        iteration?: number;
+        skipBaseline?: boolean;
+        skipGrading?: boolean;
+        model?: string;
+        graderModel?: string;
+        timeout?: number;
+    };
+}
+/**
+ * Graphile-shaped handler for the `process_eval` task.
+ *
+ * Lifted verbatim from the legacy `eval-worker.ts:processJob` body, with the
+ * queue plumbing stripped out:
+ *   - no claim — Graphile already claimed the job before invoking us
+ *   - no /complete POST — we return the result object (or throw)
+ *   - no `processing` flag — that lived in the polling loop's closure
+ *   - no conversation branch — split into its own handler (Task 14)
+ *
+ * Mid-job progress patches still hit the existing
+ * `PATCH /api/eval-jobs/:id/progress` endpoint, just keyed by the mirror row's
+ * id instead of the legacy queue id.
+ */
+export declare function processEval(payload: EvalPayload, helpers: JobHelpers): Promise<Record<string, unknown>>;
+export {};
+//# sourceMappingURL=process-eval.d.ts.map

package/dist/lib/handlers/process-eval.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"process-eval.d.ts","sourceRoot":"","sources":["../../../src/lib/handlers/process-eval.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAwBlD,UAAU,WAAW;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE;QACN,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,YAAY,CAAC,EAAE,OAAO,CAAC;QACvB,WAAW,CAAC,EAAE,OAAO,CAAC;QACtB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;KAClB,CAAC;CACH;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAsB,WAAW,CAC/B,OAAO,EAAE,WAAW,EACpB,OAAO,EAAE,UAAU,GAClB,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAiMlC"}

package/dist/lib/handlers/process-eval.js

@@ -0,0 +1,207 @@
+import { apiGet, apiPost, apiPatch } from '../api-client.js';
+import { executeEval } from '../eval-executor.js';
+import { gradeEvalOutput } from '../eval-grader.js';
+import { judgeEvalQuality } from '../eval-meta-judge.js';
+import { createEvalWorktree } from '../eval-worktree.js';
+import { PermanentJobError } from '../permanent-job-error.js';
+/**
+ * Graphile-shaped handler for the `process_eval` task.
+ *
+ * Lifted verbatim from the legacy `eval-worker.ts:processJob` body, with the
+ * queue plumbing stripped out:
+ *   - no claim — Graphile already claimed the job before invoking us
+ *   - no /complete POST — we return the result object (or throw)
+ *   - no `processing` flag — that lived in the polling loop's closure
+ *   - no conversation branch — split into its own handler (Task 14)
+ *
+ * Mid-job progress patches still hit the existing
+ * `PATCH /api/eval-jobs/:id/progress` endpoint, just keyed by the mirror row's
+ * id instead of the legacy queue id.
+ */
+export async function processEval(payload, helpers) {
+    const { mirrorId, evalTargetId, config } = payload;
+    const log = (msg) => helpers.logger.info(msg);
+    // Fail-fast on retry. Each eval_run is INSERT'd as it completes, but
+    // there's no idempotency key tying a row to a specific job — re-running
+    // the whole batch would duplicate every previously-completed row in
+    // eval_runs. Until proper resume-from-progress-checkpoint lands, surface
+    // the failure to the user instead of silently doubling-up rows. See
+    // docs/audits/2026-05-07-worker-audit.md "Handlers not idempotent".
+    if (helpers.job.attempts > 1) {
+        throw new PermanentJobError(`process_eval is not safe to retry (attempt ${helpers.job.attempts}). Re-trigger the eval run from the UI.`);
+    }
+    const target = await apiGet(`/api/eval-targets/${evalTargetId}`);
+    const evalCases = await apiGet(`/api/eval-targets/${evalTargetId}/evals`);
+    if (evalCases.length === 0) {
+        // No retries — eval cases don't materialize on their own; the user has
+        // to add one via the UI/CLI. Skip Graphile's retry loop.
+        throw new PermanentJobError('No evals defined for target');
+    }
+    const runsPerEval = config.runsPerEval ?? 1;
+    const timeout = config.timeout ?? 120_000;
+    const iteration = config.iteration ?? 1;
+    const configurations = [
+        { name: 'with_target', content: target.content },
+    ];
+    if (!config.skipBaseline) {
+        configurations.push({ name: 'without_target' });
+    }
+    // Build work queue
+    const workQueue = [];
+    for (const eval_ of evalCases) {
+        for (const cfg of configurations) {
+            for (let run = 1; run <= runsPerEval; run++) {
+                workQueue.push({ eval_: eval_, configuration: cfg.name, targetContent: cfg.content, runNumber: run });
+            }
+        }
+    }
+    const total = workQueue.length;
+    let completed = 0;
+    const concurrency = Math.min(config.parallel ?? 1, 4); // cap at 4 parallel
+    await apiPatch(`/api/eval-jobs/${mirrorId}/progress`, {
+        status: 'running',
+        progress: { total, completed: 0, current: null, message: `Starting ${total} eval runs (concurrency: ${concurrency})...` },
+    });
+    const results = [];
+    // Process work items — sequential if concurrency=1, parallel otherwise
+    async function processItem(item) {
+        const label = `${item.configuration} · ${item.eval_.name} · run#${item.runNumber}`;
+        log(`[${completed + 1}/${total}] ${label}`);
+        // Create isolated worktree for parallel execution
+        const worktree = concurrency > 1 ? createEvalWorktree() : null;
+        try {
+            const execResult = await executeEval({
+                prompt: item.eval_.prompt,
+                targetContent: item.targetContent,
+                workDir: worktree?.workDir,
+                timeout,
+                model: config.model,
+            });
+            let grading = {};
+            if (!config.skipGrading && item.eval_.expectations.length > 0 && !execResult.error) {
+                const gradingResult = await gradeEvalOutput({
+                    prompt: item.eval_.prompt,
+                    output: execResult.output,
+                    expectations: item.eval_.expectations,
+                    model: config.graderModel ?? config.model,
+                    timeout,
+                });
+                grading = gradingResult;
+            }
+            return {
+                evalId: item.eval_.id,
+                evalTargetId: target.id,
+                evalTargetVersion: target.version,
+                configuration: item.configuration,
+                runNumber: item.runNumber,
+                iteration,
+                status: execResult.error ? 'failed' : 'completed',
+                outputs: execResult.error ? { error: execResult.error } : { transcript: execResult.output },
+                timing: { duration_ms: execResult.durationMs, total_tokens: execResult.totalTokens },
+                grading,
+                worktree: worktree?.isWorktree ? true : undefined,
+            };
+        }
+        finally {
+            worktree?.cleanup();
+        }
+    }
+    if (concurrency <= 1) {
+        // Sequential execution (existing behavior)
+        for (const item of workQueue) {
+            const label = `${item.configuration} · ${item.eval_.name} · run#${item.runNumber}`;
+            await apiPatch(`/api/eval-jobs/${mirrorId}/progress`, {
+                status: 'running',
+                progress: { total, completed, current: label, message: `Executing ${label}...` },
+            });
+            const runResult = await processItem(item);
+            await apiPost('/api/eval-runs', runResult);
+            results.push(runResult);
+            completed++;
+            await apiPatch(`/api/eval-jobs/${mirrorId}/progress`, {
+                status: 'running',
+                progress: { total, completed, current: label, message: `Completed ${label}` },
+            });
+        }
+    }
+    else {
+        // Parallel execution with worktree isolation
+        log(`Running ${total} evals with concurrency=${concurrency} (worktree isolation)`);
+        for (let i = 0; i < workQueue.length; i += concurrency) {
+            const batch = workQueue.slice(i, i + concurrency);
+            const batchLabel = `batch ${Math.floor(i / concurrency) + 1}/${Math.ceil(workQueue.length / concurrency)}`;
+            await apiPatch(`/api/eval-jobs/${mirrorId}/progress`, {
+                status: 'running',
+                progress: { total, completed, current: batchLabel, message: `Executing ${batchLabel} (${batch.length} parallel)...` },
+            });
+            const batchResults = await Promise.all(batch.map(processItem));
+            for (const runResult of batchResults) {
+                await apiPost('/api/eval-runs', runResult);
+                results.push(runResult);
+                completed++;
+            }
+            await apiPatch(`/api/eval-jobs/${mirrorId}/progress`, {
+                status: 'running',
+                progress: { total, completed, current: batchLabel, message: `Completed ${batchLabel}` },
+            });
+        }
+    }
+    const summary = buildSummary(results);
+    // Meta-judge: assess eval quality if we have grading results
+    let metaJudgement;
+    if (!config.skipGrading && results.some(r => r.grading && Object.keys(r.grading).length > 0)) {
+        try {
+            log(`Running meta-judge for job ${mirrorId}...`);
+            const evalInputs = evalCases.map(e => {
+                const matchingResults = results.filter(r => r.evalId === e.id && r.configuration === 'with_target');
+                const firstResult = matchingResults[0];
+                return {
+                    evalName: e.name,
+                    prompt: e.prompt,
+                    expectations: e.expectations,
+                    grading: firstResult?.grading ? firstResult.grading : null,
+                    output: firstResult?.outputs?.transcript?.slice(0, 500),
+                };
+            });
+            metaJudgement = await judgeEvalQuality({
+                targetName: target.name,
+                evalResults: evalInputs,
+                model: config.graderModel ?? config.model,
+                timeout: 60000,
+            });
+            log(`Meta-judge: quality=${metaJudgement.quality_score}/5, issues=${metaJudgement.issues.length}`);
+        }
+        catch (err) {
+            log(`Meta-judge failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+    const result = metaJudgement
+        ? { ...summary, meta_judgement: metaJudgement }
+        : summary;
+    log(`Job ${mirrorId} completed: ${completed}/${total} runs`);
+    // Graphile discards task return values, so persist the summary onto the
+    // mirror row here. Best-effort: a failure to write the result must not
+    // mark the job as failed (it succeeded; we just lose the summary blob).
+    await apiPatch(`/api/eval-jobs/${mirrorId}/state`, { result }).catch((err) => {
+        log(`result persist failed: ${err instanceof Error ? err.message : String(err)}`);
+    });
+    return result;
+}
+function buildSummary(results) {
+    const completed = results.filter((r) => r.status === 'completed');
+    const failed = results.filter((r) => r.status === 'failed');
+    const gradedRuns = completed.filter((r) => {
+        const grading = r.grading;
+        return grading && Object.keys(grading).length > 0;
+    });
+    const avgPassRate = gradedRuns.length > 0
+        ? gradedRuns.reduce((sum, r) => sum + (r.grading?.summary?.pass_rate ?? 0), 0) / gradedRuns.length
+        : null;
+    return {
+        total: results.length,
+        completed: completed.length,
+        failed: failed.length,
+        avgPassRate,
+    };
+}
+//# sourceMappingURL=process-eval.js.map

package/dist/lib/handlers/process-eval.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"process-eval.js","sourceRoot":"","sources":["../../../src/lib/handlers/process-eval.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC7D,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,EAAE,eAAe,EAAsB,MAAM,mBAAmB,CAAC;AACxE,OAAO,EAAE,gBAAgB,EAAsB,MAAM,uBAAuB,CAAC;AAC7E,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAiC9D;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,OAAoB,EACpB,OAAmB;IAEnB,MAAM,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IACnD,MAAM,GAAG,GAAG,CAAC,GAAW,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAEtD,qEAAqE;IACrE,wEAAwE;IACxE,oEAAoE;IACpE,yEAAyE;IACzE,oEAAoE;IACpE,oEAAoE;IACpE,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,GAAG,CAAC,EAAE,CAAC;QAC7B,MAAM,IAAI,iBAAiB,CACzB,8CAA8C,OAAO,CAAC,GAAG,CAAC,QAAQ,yCAAyC,CAC5G,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,MAAM,CAAa,qBAAqB,YAAY,EAAE,CAAC,CAAC;IAC7E,MAAM,SAAS,GAAG,MAAM,MAAM,CAAS,qBAAqB,YAAY,QAAQ,CAAC,CAAC;IAElF,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,uEAAuE;QACvE,yDAAyD;QACzD,MAAM,IAAI,iBAAiB,CAAC,6BAA6B,CAAC,CAAC;IAC7D,CAAC;IAED,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC;IAC5C,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,OAAO,CAAC;IAC1C,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,CAAC,CAAC;IAExC,MAAM,cAAc,GAA8C;QAChE,EAAE,IAAI,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE;KACjD,CAAC;IACF,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;QACzB,cAAc,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,CAAC,CAAC;IAClD,CAAC;IAED,mBAAmB;IACnB,MAAM,SAAS,GAA6F,EAAE,CAAC;IAC/G,KAAK,MAAM,KAAK,IAAI,SAAS,EAAE,CAAC;QAC9B,KAAK,MAAM,GAAG,IAAI,cAAc,EAAE,CAAC;YACjC,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,IAAI,WAAW,EAAE,GAAG,EAAE,EAAE,CAAC;gBAC5C,SAAS,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,aAAa,EAAE,GAAG,CAAC,IAAI,EAAE,aAAa,EAAE,GAAG,CAAC,OAAO,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC;YACxG,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC;IAC/B,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,oBAAoB;IAE3E,MAAM,QAAQ,CAAC,kBAAkB,QAAQ,WAAW,EAAE;QACpD,MAAM,EAAE,SAAS;QACjB,QAAQ,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,YAAY,KAAK,4BAA4B,WAAW,MAAM,EAAE;KAC1H,CAAC,CAAC;IAEH,MAAM,OAAO,GAA8B,EAAE,CAAC;IAE9C,uEAAuE;IACvE,KAAK,UAAU,WAAW,CAAC,IAAyB;QAClD,MAAM,KAAK,GAAG,GAAG,IAAI,CAAC,aAAa,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,UAAU,IAAI,CAAC,SAAS,EAAE,CAAC;QACnF,GAAG,CAAC,IAAI,SAAS,GAAG,CAAC,IAAI,KAAK,KAAK,KAAK,EAAE,CAAC,CAAC;QAE5C,kDAAkD;QAClD,MAAM,QAAQ,GAAG,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,kBAAkB,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QAE/D,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,MAAM,WAAW,CAAC;gBACnC,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM;gBACzB,aAAa,EAAE,IAAI,CAAC,aAAa;gBACjC,OAAO,EAAE,QAAQ,EAAE,OAAO;gBAC1B,OAAO;gBACP,KAAK,EAAE,MAAM,CAAC,KAAK;aACpB,CAAC,CAAC;YAEH,IAAI,OAAO,GAA4B,EAAE,CAAC;YAC1C,IAAI,CAAC,MAAM,CAAC,WAAW,IAAI,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;gBACnF,MAAM,aAAa,GAAG,MAAM,eAAe,CAAC;oBAC1C,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM;oBACzB,MAAM,EAAE,UAAU,CAAC,MAAM;oBACzB,YAAY,EAAE,IAAI,CAAC,KAAK,CAAC,YAAY;oBACrC,KAAK,EAAE,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,KAAK;oBACzC,OAAO;iBACR,CAAC,CAAC;gBACH,OAAO,GAAG,aAAmD,CAAC;YAChE,CAAC;YAED,OAAO;gBACL,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE;gBACrB,YAAY,EAAE,MAAM,CAAC,EAAE;gBACvB,iBAAiB,EAAE,MAAM,CAAC,OAAO;gBACjC,aAAa,EAAE,IAAI,CAAC,aAAa;gBACjC,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,SAAS;gBACT,MAAM,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW;gBACjD,OAAO,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,UAAU,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,UAAU,CAAC,MAAM,EAAE;gBAC3F,MAAM,EAAE,EAAE,WAAW,EAAE,UAAU,CAAC,UAAU,EAAE,YAAY,EAAE,UAAU,CAAC,WAAW,EAAE;gBACpF,OAAO;gBACP,QAAQ,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;aAClD,CAAC;QACJ,CAAC;gBAAS,CAAC;YACT,QAAQ,EAAE,OAAO,EAAE,CAAC;QACtB,CAAC;IACH,CAAC;IAED,IAAI,WAAW,IAAI,CAAC,EAAE,CAAC;QACrB,2CAA2C;QAC3C,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;YAC7B,MAAM,KAAK,GAAG,GAAG,IAAI,CAAC,aAAa,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,UAAU,IAAI,CAAC,SAAS,EAAE,CAAC;YACnF,MAAM,QAAQ,CAAC,kBAAkB,QAAQ,WAAW,EAAE;gBACpD,MAAM,EAAE,SAAS;gBACjB,QAAQ,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,aAAa,KAAK,KAAK,EAAE;aACjF,CAAC,CAAC;YAEH,MAAM,SAAS,GAAG,MAAM,WAAW,CAAC,IAAI,CAAC,CAAC;YAC1C,MAAM,OAAO,CAAC,gBAAgB,EAAE,SAAS,CAAC,CAAC;YAC3C,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACxB,SAAS,EAAE,CAAC;YAEZ,MAAM,QAAQ,CAAC,kBAAkB,QAAQ,WAAW,EAAE;gBACpD,MAAM,EAAE,SAAS;gBACjB,QAAQ,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,aAAa,KAAK,EAAE,EAAE;aAC9E,CAAC,CAAC;QACL,CAAC;IACH,CAAC;SAAM,CAAC;QACN,6CAA6C;QAC7C,GAAG,CAAC,WAAW,KAAK,2BAA2B,WAAW,uBAAuB,CAAC,CAAC;QACnF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,IAAI,WAAW,EAAE,CAAC;YACvD,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,CAAC;YAClD,MAAM,UAAU,GAAG,SAAS,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,WAAW,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,GAAG,WAAW,CAAC,EAAE,CAAC;YAE3G,MAAM,QAAQ,CAAC,kBAAkB,QAAQ,WAAW,EAAE;gBACpD,MAAM,EAAE,SAAS;gBACjB,QAAQ,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,aAAa,UAAU,KAAK,KAAK,CAAC,MAAM,eAAe,EAAE;aACtH,CAAC,CAAC;YAEH,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC;YAE/D,KAAK,MAAM,SAAS,IAAI,YAAY,EAAE,CAAC;gBACrC,MAAM,OAAO,CAAC,gBAAgB,EAAE,SAAS,CAAC,CAAC;gBAC3C,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBACxB,SAAS,EAAE,CAAC;YACd,CAAC;YAED,MAAM,QAAQ,CAAC,kBAAkB,QAAQ,WAAW,EAAE;gBACpD,MAAM,EAAE,SAAS;gBACjB,QAAQ,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,aAAa,UAAU,EAAE,EAAE;aACxF,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;IAEtC,6DAA6D;IAC7D,IAAI,aAAwC,CAAC;IAC7C,IAAI,CAAC,MAAM,CAAC,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,OAAiB,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;QACvG,IAAI,CAAC;YACH,GAAG,CAAC,8BAA8B,QAAQ,KAAK,CAAC,CAAC;YACjD,MAAM,UAAU,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE;gBACnC,MAAM,eAAe,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,aAAa,KAAK,aAAa,CAAC,CAAC;gBACpG,MAAM,WAAW,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC;gBACvC,OAAO;oBACL,QAAQ,EAAE,CAAC,CAAC,IAAI;oBAChB,MAAM,EAAE,CAAC,CAAC,MAAM;oBAChB,YAAY,EAAE,CAAC,CAAC,YAAY;oBAC5B,OAAO,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,OAAmC,CAAC,CAAC,CAAC,IAAI;oBACtF,MAAM,EAAG,WAAW,EAAE,OAAkC,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;iBACpF,CAAC;YACJ,CAAC,CAAC,CAAC;YACH,aAAa,GAAG,MAAM,gBAAgB,CAAC;gBACrC,UAAU,EAAE,MAAM,CAAC,IAAI;gBACvB,WAAW,EAAE,UAAU;gBACvB,KAAK,EAAE,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,KAAK;gBACzC,OAAO,EAAE,KAAK;aACf,CAAC,CAAC;YACH,GAAG,CAAC,uBAAuB,aAAa,CAAC,aAAa,cAAc,aAAa,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QACrG,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,kCAAkC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC5F,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,aAAa;QAC1B,CAAC,CAAC,EAAE,GAAG,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE;QAC/C,CAAC,CAAC,OAAO,CAAC;IACZ,GAAG,CAAC,OAAO,QAAQ,eAAe,SAAS,IAAI,KAAK,OAAO,CAAC,CAAC;IAE7D,wEAAwE;IACxE,uEAAuE;IACvE,wEAAwE;IACxE,MAAM,QAAQ,CAAC,kBAAkB,QAAQ,QAAQ,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,GAAY,EAAE,EAAE;QACpF,GAAG,CAAC,0BAA0B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACpF,CAAC,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,YAAY,CAAC,OAAkC;IACtD,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC;IAClE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC;IAE5D,MAAM,UAAU,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;QACxC,MAAM,OAAO,GAAG,CAAC,CAAC,OAA8B,CAAC;QACjD,OAAO,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACpD,CAAC,CAAC,CAAC;IAEH,MAAM,WAAW,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC;QACvC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAE,CAAC,CAAC,OAAe,EAAE,OAAO,EAAE,SAAS,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,UAAU,CAAC,MAAM;QAC3G,CAAC,CAAC,IAAI,CAAC;IAET,OAAO;QACL,KAAK,EAAE,OAAO,CAAC,MAAM;QACrB,SAAS,EAAE,SAAS,CAAC,MAAM;QAC3B,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,WAAW;KACZ,CAAC;AACJ,CAAC"}