@rulemetric/cli 0.1.1

package/dist/lib/eval-meta-judge.js

@@ -0,0 +1,116 @@
+import { executeEval } from './eval-executor.js';
+const META_JUDGE_SYSTEM_PROMPT = `You are a meta-judge evaluating the quality of an eval run — not the code output, but whether the test cases and grading themselves are rigorous and meaningful.
+
+You will receive a summary of eval cases, their expectations, and grading results.
+
+Analyze for these quality issues:
+
+1. **Weak expectations** — Expectations that are vague ("should follow conventions") or trivially satisfiable. Good expectations are specific and falsifiable.
+
+2. **Inconsistent grading** — Cases where similar expectations get different pass/fail results, or where evidence doesn't match the verdict.
+
+3. **Trivial tests** — Cases where the prompt doesn't actually challenge the rule. E.g., testing "NEVER use mocks" with a prompt that doesn't involve testing at all.
+
+4. **Untestable expectations** — Expectations that can't be verified from the output alone (e.g., "the developer should feel confident").
+
+5. **Suspicious 100% pass** — If every expectation passes, consider whether the tests were actually challenging enough.
+
+Respond in JSON:
+{
+  "quality_score": <1-5>,
+  "issues": [
+    {
+      "type": "weak_expectation | inconsistent_grading | trivial_test | untestable_expectation | all_pass_suspicious",
+      "severity": "low | medium | high",
+      "description": "<specific issue>",
+      "affected_eval": "<eval name if applicable>"
+    }
+  ],
+  "recommendations": ["<specific improvement suggestion>", ...],
+  "summary": "<1-2 sentence overall assessment>"
+}
+
+Be constructive but honest. A quality_score of 3 means acceptable, 4 means good, 5 means rigorous. Most real-world eval runs score 2-4.`;
+export async function judgeEvalQuality(input) {
+    // Build a summary of the eval run for the meta-judge
+    const evalSummary = input.evalResults.map((r) => {
+        const gradingInfo = r.grading
+            ? {
+                pass_rate: r.grading.summary.pass_rate,
+                expectations: r.grading.expectations.map((e) => ({
+                    text: e.text,
+                    passed: e.passed,
+                    evidence: e.evidence.slice(0, 200),
+                })),
+                feedback: r.grading.eval_feedback?.overall ?? '',
+            }
+            : null;
+        return {
+            eval_name: r.evalName,
+            prompt: r.prompt.slice(0, 500),
+            expectations: r.expectations,
+            grading: gradingInfo,
+            output_preview: r.output?.slice(0, 300) ?? '(not available)',
+        };
+    });
+    const prompt = `Evaluate the quality of this eval run for target "${input.targetName}":
+
+${JSON.stringify(evalSummary, null, 2)}`;
+    const result = await executeEval({
+        prompt,
+        systemPrompt: META_JUDGE_SYSTEM_PROMPT,
+        timeout: input.timeout ?? 60000,
+        model: input.model,
+    });
+    if (result.error || !result.output) {
+        return {
+            quality_score: 0,
+            issues: [{
+                    type: 'inconsistent_grading',
+                    severity: 'high',
+                    description: `Meta-judge failed: ${result.error ?? 'no output'}`,
+                }],
+            recommendations: ['Meta-judge could not complete — check model availability'],
+            summary: 'Meta-judge execution failed.',
+        };
+    }
+    try {
+        const parsed = parseJsonFromOutput(result.output);
+        const issues = Array.isArray(parsed.issues) ? parsed.issues : [];
+        const recs = Array.isArray(parsed.recommendations) ? parsed.recommendations : [];
+        return {
+            quality_score: Math.max(1, Math.min(5, Number(parsed.quality_score) || 3)),
+            issues: issues.map((i) => ({
+                type: (i.type ?? 'weak_expectation'),
+                severity: (i.severity ?? 'medium'),
+                description: String(i.description ?? ''),
+                affected_eval: i.affected_eval ? String(i.affected_eval) : undefined,
+            })),
+            recommendations: recs,
+            summary: String(parsed.summary ?? 'No summary provided.'),
+        };
+    }
+    catch {
+        return {
+            quality_score: 3,
+            issues: [],
+            recommendations: [],
+            summary: 'Meta-judge returned unparseable output — defaulting to neutral score.',
+        };
+    }
+}
+function parseJsonFromOutput(output) {
+    // Try direct JSON parse
+    try {
+        return JSON.parse(output);
+    }
+    catch {
+        // Try extracting from markdown code block
+        const match = output.match(/```(?:json)?\s*\n?([\s\S]*?)\n?```/);
+        if (match) {
+            return JSON.parse(match[1]);
+        }
+        throw new Error('No JSON found in output');
+    }
+}
+//# sourceMappingURL=eval-meta-judge.js.map

package/dist/lib/eval-meta-judge.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-meta-judge.js","sourceRoot":"","sources":["../../src/lib/eval-meta-judge.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAiBjD,MAAM,wBAAwB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;wIA+BuG,CAAC;AAezI,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,KAAqB;IAC1D,qDAAqD;IACrD,MAAM,WAAW,GAAG,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAC9C,MAAM,WAAW,GAAG,CAAC,CAAC,OAAO;YAC3B,CAAC,CAAC;gBACE,SAAS,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS;gBACtC,YAAY,EAAE,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;oBAC/C,IAAI,EAAE,CAAC,CAAC,IAAI;oBACZ,MAAM,EAAE,CAAC,CAAC,MAAM;oBAChB,QAAQ,EAAE,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;iBACnC,CAAC,CAAC;gBACH,QAAQ,EAAE,CAAC,CAAC,OAAO,CAAC,aAAa,EAAE,OAAO,IAAI,EAAE;aACjD;YACH,CAAC,CAAC,IAAI,CAAC;QAET,OAAO;YACL,SAAS,EAAE,CAAC,CAAC,QAAQ;YACrB,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;YAC9B,YAAY,EAAE,CAAC,CAAC,YAAY;YAC5B,OAAO,EAAE,WAAW;YACpB,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,iBAAiB;SAC7D,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,qDAAqD,KAAK,CAAC,UAAU;;EAEpF,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;IAEvC,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC;QAC/B,MAAM;QACN,YAAY,EAAE,wBAAwB;QACtC,OAAO,EAAE,KAAK,CAAC,OAAO,IAAI,KAAK;QAC/B,KAAK,EAAE,KAAK,CAAC,KAAK;KACnB,CAAC,CAAC;IAEH,IAAI,MAAM,CAAC,KAAK,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QACnC,OAAO;YACL,aAAa,EAAE,CAAC;YAChB,MAAM,EAAE,CAAC;oBACP,IAAI,EAAE,sBAAsB;oBAC5B,QAAQ,EAAE,MAAM;oBAChB,WAAW,EAAE,sBAAsB,MAAM,CAAC,KAAK,IAAI,WAAW,EAAE;iBACjE,CAAC;YACF,eAAe,EAAE,CAAC,0DAA0D,CAAC;YAC7E,OAAO,EAAE,8BAA8B;SACxC,CAAC;IACJ,CAAC;IAED,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,mBAAmB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAClD,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;QACjE,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,eAA2B,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7F,OAAO;YACL,aAAa,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC;YAC1E,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAyB,EAAE,EAAE,CAAC,CAAC;gBACjD,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,kBAAkB,CAAsB;gBACzD,QAAQ,EAAE,CAAC,CAAC,CAAC,QAAQ,IAAI,QAAQ,CAA0B;gBAC3D,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,WAAW,IAAI,EAAE,CAAC;gBACxC,aAAa,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,SAAS;aACrE,CAAC,CAAC;YACH,eAAe,EAAE,IAAI;YACrB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO,IAAI,sBAAsB,CAAC;SAC1D,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO;YACL,aAAa,EAAE,CAAC;YAChB,MAAM,EAAE,EAAE;YACV,eAAe,EAAE,EAAE;YACnB,OAAO,EAAE,uEAAuE;SACjF,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB,CAAC,MAAc;IACzC,wBAAwB;IACxB,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC5B,CAAC;IAAC,MAAM,CAAC;QACP,0CAA0C;QAC1C,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,oCAAoC,CAAC,CAAC;QACjE,IAAI,KAAK,EAAE,CAAC;YACV,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9B,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;IAC7C,CAAC;AACH,CAAC"}

package/dist/lib/eval-trigger-tester.d.ts

@@ -0,0 +1,19 @@
+export interface TriggerTestResult {
+    query: string;
+    shouldTrigger: boolean;
+    didTrigger: boolean;
+    correct: boolean;
+    confidence: number;
+    evidence: string;
+}
+export declare function testTrigger(options: {
+    query: string;
+    shouldTrigger: boolean;
+    skillName: string;
+    skillDescription: string;
+    skillContent: string;
+    model?: string;
+    timeout?: number;
+}): Promise<TriggerTestResult>;
+export declare function computeTriggerScore(results: TriggerTestResult[]): number;
+//# sourceMappingURL=eval-trigger-tester.d.ts.map

package/dist/lib/eval-trigger-tester.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-trigger-tester.d.ts","sourceRoot":"","sources":["../../src/lib/eval-trigger-tester.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,OAAO,CAAC;IACvB,UAAU,EAAE,OAAO,CAAC;IACpB,OAAO,EAAE,OAAO,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAuBD,wBAAsB,WAAW,CAAC,OAAO,EAAE;IACzC,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,OAAO,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAwE7B;AAED,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,iBAAiB,EAAE,GAAG,MAAM,CAIxE"}

package/dist/lib/eval-trigger-tester.js

@@ -0,0 +1,96 @@
+import { executeEval } from './eval-executor.js';
+const TRIGGER_DETECTOR_PROMPT = `You are determining whether an AI assistant's response shows evidence of using a specific skill/instruction.
+
+You will receive:
+1. The skill name and description
+2. Key content from the skill (to know what it teaches)
+3. The user's query
+4. The AI's response
+
+Determine if the response shows clear signs of the skill being used:
+- Does the response follow patterns/approaches described in the skill?
+- Does it use terminology or structures specific to the skill?
+- Would the response be noticeably different without the skill?
+
+Respond with ONLY a valid JSON object:
+
+{
+  "triggered": true,
+  "confidence": 0.85,
+  "evidence": "The response follows the X pattern described in the skill..."
+}`;
+export async function testTrigger(options) {
+    const { query, shouldTrigger, skillName, skillDescription, skillContent, model, timeout = 120_000 } = options;
+    // Run the query with the skill as system prompt
+    const execResult = await executeEval({
+        prompt: query,
+        targetContent: skillContent,
+        timeout,
+        model,
+    });
+    if (execResult.error) {
+        return {
+            query,
+            shouldTrigger,
+            didTrigger: false,
+            correct: !shouldTrigger,
+            confidence: 0,
+            evidence: `Execution failed: ${execResult.error}`,
+        };
+    }
+    // Use LLM to detect if the skill influenced the response
+    const detectorPrompt = `## Skill
+Name: ${skillName}
+Description: ${skillDescription}
+
+## Key Skill Content (first 500 chars)
+${skillContent.slice(0, 500)}
+
+## User Query
+${query}
+
+## AI Response
+${execResult.output}
+
+Did the AI's response show evidence of using this skill? Respond with ONLY valid JSON.`;
+    const detectorResult = await executeEval({
+        prompt: detectorPrompt,
+        targetContent: TRIGGER_DETECTOR_PROMPT,
+        timeout: 60_000,
+        model,
+    });
+    let didTrigger = false;
+    let confidence = 0;
+    let evidence = '';
+    try {
+        let jsonStr = detectorResult.output.trim();
+        const jsonMatch = jsonStr.match(/```(?:json)?\s*([\s\S]*?)```/);
+        if (jsonMatch)
+            jsonStr = jsonMatch[1].trim();
+        const objMatch = jsonStr.match(/\{[\s\S]*\}/);
+        if (objMatch)
+            jsonStr = objMatch[0];
+        const parsed = JSON.parse(jsonStr);
+        didTrigger = parsed.triggered === true;
+        confidence = parsed.confidence ?? 0;
+        evidence = parsed.evidence ?? '';
+    }
+    catch {
+        evidence = `Detector returned unparseable output: ${detectorResult.output.slice(0, 200)}`;
+    }
+    return {
+        query,
+        shouldTrigger,
+        didTrigger,
+        correct: didTrigger === shouldTrigger,
+        confidence,
+        evidence,
+    };
+}
+export function computeTriggerScore(results) {
+    if (results.length === 0)
+        return 0;
+    const correct = results.filter((r) => r.correct).length;
+    return correct / results.length;
+}
+//# sourceMappingURL=eval-trigger-tester.js.map

package/dist/lib/eval-trigger-tester.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-trigger-tester.js","sourceRoot":"","sources":["../../src/lib/eval-trigger-tester.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAWjD,MAAM,uBAAuB,GAAG;;;;;;;;;;;;;;;;;;;EAmB9B,CAAC;AAEH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,OAQjC;IACC,MAAM,EAAE,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,gBAAgB,EAAE,YAAY,EAAE,KAAK,EAAE,OAAO,GAAG,OAAO,EAAE,GAAG,OAAO,CAAC;IAE9G,gDAAgD;IAChD,MAAM,UAAU,GAAG,MAAM,WAAW,CAAC;QACnC,MAAM,EAAE,KAAK;QACb,aAAa,EAAE,YAAY;QAC3B,OAAO;QACP,KAAK;KACN,CAAC,CAAC;IAEH,IAAI,UAAU,CAAC,KAAK,EAAE,CAAC;QACrB,OAAO;YACL,KAAK;YACL,aAAa;YACb,UAAU,EAAE,KAAK;YACjB,OAAO,EAAE,CAAC,aAAa;YACvB,UAAU,EAAE,CAAC;YACb,QAAQ,EAAE,qBAAqB,UAAU,CAAC,KAAK,EAAE;SAClD,CAAC;IACJ,CAAC;IAED,yDAAyD;IACzD,MAAM,cAAc,GAAG;QACjB,SAAS;eACF,gBAAgB;;;EAG7B,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;;;EAG1B,KAAK;;;EAGL,UAAU,CAAC,MAAM;;uFAEoE,CAAC;IAEtF,MAAM,cAAc,GAAG,MAAM,WAAW,CAAC;QACvC,MAAM,EAAE,cAAc;QACtB,aAAa,EAAE,uBAAuB;QACtC,OAAO,EAAE,MAAM;QACf,KAAK;KACN,CAAC,CAAC;IAEH,IAAI,UAAU,GAAG,KAAK,CAAC;IACvB,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,QAAQ,GAAG,EAAE,CAAC;IAElB,IAAI,CAAC;QACH,IAAI,OAAO,GAAG,cAAc,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QAC3C,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC;QAChE,IAAI,SAAS;YAAE,OAAO,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC7C,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QAC9C,IAAI,QAAQ;YAAE,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QAEpC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACnC,UAAU,GAAG,MAAM,CAAC,SAAS,KAAK,IAAI,CAAC;QACvC,UAAU,GAAG,MAAM,CAAC,UAAU,IAAI,CAAC,CAAC;QACpC,QAAQ,GAAG,MAAM,CAAC,QAAQ,IAAI,EAAE,CAAC;IACnC,CAAC;IAAC,MAAM,CAAC;QACP,QAAQ,GAAG,yCAAyC,cAAc,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;IAC5F,CAAC;IAED,OAAO;QACL,KAAK;QACL,aAAa;QACb,UAAU;QACV,OAAO,EAAE,UAAU,KAAK,aAAa;QACrC,UAAU;QACV,QAAQ;KACT,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,OAA4B;IAC9D,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACnC,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;IACxD,OAAO,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC;AAClC,CAAC"}

package/dist/lib/eval-worker.d.ts

@@ -0,0 +1,12 @@
+export interface WorkerOptions {
+    supabaseUrl: string;
+    supabaseAnonKey: string;
+    accessToken: string;
+    userId: string;
+    log: (msg: string) => void;
+}
+export declare function createWorkerId(): string;
+/** Decode the `sub` claim from a JWT without a library. */
+export declare function getUserIdFromJwt(jwt: string): string | null;
+export declare function startWorker(options: WorkerOptions): Promise<() => void>;
+//# sourceMappingURL=eval-worker.d.ts.map

package/dist/lib/eval-worker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-worker.d.ts","sourceRoot":"","sources":["../../src/lib/eval-worker.ts"],"names":[],"mappings":"AA6CA,MAAM,WAAW,aAAa;IAC5B,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;IACxB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;CAC5B;AAED,wBAAgB,cAAc,IAAI,MAAM,CAEvC;AAED,2DAA2D;AAC3D,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAO3D;AAeD,wBAAsB,WAAW,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,MAAM,IAAI,CAAC,CA+X7E"}

package/dist/lib/eval-worker.js

@@ -0,0 +1,395 @@
+import { createClient } from '@supabase/supabase-js';
+import { apiGet, apiPost, apiPatch } from './api-client.js';
+import { executeEval } from './eval-executor.js';
+import { gradeEvalOutput } from './eval-grader.js';
+import { judgeEvalQuality } from './eval-meta-judge.js';
+import { createEvalWorktree } from './eval-worktree.js';
+import { processInsightsJob } from './insights-worker.js';
+import { processConversationJob } from './eval-conversation-handler.js';
+import os from 'node:os';
+import path from 'node:path';
+import { writeFile, mkdir } from 'node:fs/promises';
+export function createWorkerId() {
+    return `${os.hostname()}-${process.pid}`;
+}
+/** Decode the `sub` claim from a JWT without a library. */
+export function getUserIdFromJwt(jwt) {
+    try {
+        const payload = JSON.parse(Buffer.from(jwt.split('.')[1], 'base64url').toString());
+        return payload.sub ?? null;
+    }
+    catch {
+        return null;
+    }
+}
+/** Best-effort refresh of the local skills registry cache (~24h cycle). */
+async function refreshSkillsRegistry(log) {
+    const { fetchAllSources, TIER_1_SOURCES } = await import('@rulemetric/skills-registry');
+    const skills = await fetchAllSources(TIER_1_SOURCES, (source, count) => {
+        log(`Skills registry: fetched ${count} from ${source}`);
+    });
+    const configDir = path.join(os.homedir(), '.config', 'rulemetric');
+    await mkdir(configDir, { recursive: true });
+    const outPath = path.join(configDir, 'skills-registry.json');
+    await writeFile(outPath, JSON.stringify({ generatedAt: new Date().toISOString(), skills }, null, 2));
+    log(`Skills registry refreshed: ${skills.length} skills written to ${outPath}`);
+}
+export async function startWorker(options) {
+    const { supabaseUrl, supabaseAnonKey, accessToken, userId, log } = options;
+    const workerId = createWorkerId();
+    const supabase = createClient(supabaseUrl, supabaseAnonKey, {
+        global: { headers: { Authorization: `Bearer ${accessToken}` } },
+    });
+    let processing = false;
+    // Recover stale jobs (stuck in claimed/running for >5 min) on startup
+    async function recoverStaleJobs() {
+        try {
+            const evalRecovery = await apiPost('/api/eval-jobs/recover-stale', {});
+            const insightsRecovery = await apiPost('/api/insights-jobs/recover-stale', {});
+            const total = evalRecovery.recovered + insightsRecovery.recovered;
+            if (total > 0) {
+                log(`Recovered ${total} stale jobs (${evalRecovery.recovered} eval, ${insightsRecovery.recovered} insights)`);
+            }
+        }
+        catch {
+            // Non-fatal — older API versions may not have this endpoint
+        }
+    }
+    await recoverStaleJobs();
+    // Pick up any pending eval jobs created while we were offline
+    const pendingJobs = await apiGet('/api/eval-jobs/pending');
+    for (const job of pendingJobs) {
+        await processJob(job, workerId, log);
+    }
+    // Pick up any pending insights jobs
+    const pendingInsightsJobs = await apiGet('/api/insights-jobs/pending');
+    for (const job of pendingInsightsJobs) {
+        await processInsightsJob(job, workerId, log);
+    }
+    // Subscribe to new jobs via Realtime
+    const channel = supabase
+        .channel('eval-jobs-worker')
+        .on('postgres_changes', {
+        event: 'INSERT',
+        schema: 'public',
+        table: 'eval_jobs',
+        filter: `user_id=eq.${userId}`,
+    }, async (payload) => {
+        if (processing)
+            return;
+        const job = payload.new;
+        if (job.status === 'pending') {
+            await processJob(job, workerId, log);
+        }
+    })
+        .subscribe((status) => {
+        if (status === 'SUBSCRIBED') {
+            log(`Worker ${workerId} listening for eval jobs...`);
+        }
+    });
+    // Subscribe to insights jobs via Realtime
+    const insightsChannel = supabase
+        .channel('insights-jobs-worker')
+        .on('postgres_changes', {
+        event: 'INSERT',
+        schema: 'public',
+        table: 'insights_jobs',
+        filter: `user_id=eq.${userId}`,
+    }, async (payload) => {
+        if (processing)
+            return;
+        const job = payload.new;
+        if (job.status === 'pending') {
+            processing = true;
+            try {
+                await processInsightsJob(job, workerId, log);
+            }
+            finally {
+                processing = false;
+            }
+        }
+    })
+        .subscribe((status) => {
+        if (status === 'SUBSCRIBED') {
+            log(`Worker ${workerId} listening for insights jobs...`);
+        }
+    });
+    // Poll for pending jobs as a fallback — Realtime postgres_changes may not
+    // fire reliably depending on RLS/auth context. Polling every 5s is cheap.
+    let pollCount = 0;
+    const pollInterval = setInterval(async () => {
+        if (processing)
+            return;
+        try {
+            // Recover stale jobs every ~60s (every 12th poll)
+            if (++pollCount % 12 === 0) {
+                await recoverStaleJobs();
+            }
+            // Auto-generate insights for stale projects every ~6 hours (4320 polls)
+            if (pollCount % 4320 === 1) {
+                try {
+                    await apiPost('/api/insights-jobs/regenerate-all', {});
+                    log('Auto-triggered insights regeneration for stale projects');
+                }
+                catch {
+                    // Best-effort — silently ignore
+                }
+            }
+            // Refresh skills registry every ~24 hours (17280 polls at 5s each)
+            if (pollCount % 17280 === 100) {
+                try {
+                    await refreshSkillsRegistry(log);
+                }
+                catch {
+                    // Best-effort — don't fail the worker
+                }
+            }
+            const evalJobs = await apiGet('/api/eval-jobs/pending');
+            for (const job of evalJobs) {
+                if (!processing) {
+                    await processJob(job, workerId, log);
+                }
+            }
+            const insightsJobs = await apiGet('/api/insights-jobs/pending');
+            for (const job of insightsJobs) {
+                if (!processing) {
+                    processing = true;
+                    try {
+                        await processInsightsJob(job, workerId, log);
+                    }
+                    finally {
+                        processing = false;
+                    }
+                }
+            }
+        }
+        catch {
+            // Silently ignore poll errors (network blip, etc.)
+        }
+    }, 5_000);
+    // Heartbeat via broadcast (web UI listens for this)
+    const heartbeatChannel = supabase
+        .channel(`worker-heartbeat-${userId}`)
+        .subscribe((status) => {
+        if (status === 'SUBSCRIBED') {
+            heartbeatChannel.send({
+                type: 'broadcast',
+                event: 'heartbeat',
+                payload: { workerId, timestamp: Date.now() },
+            });
+        }
+    });
+    const heartbeatInterval = setInterval(() => {
+        heartbeatChannel.send({
+            type: 'broadcast',
+            event: 'heartbeat',
+            payload: { workerId, timestamp: Date.now() },
+        });
+    }, 5_000); // 5s heartbeat — fast enough for UI to detect worker within one interval
+    async function processJob(job, wId, logger) {
+        processing = true;
+        try {
+            // Claim the job (returns 409 if already claimed by another worker)
+            const claimed = await apiPost(`/api/eval-jobs/${job.id}/claim`, { workerId: wId }).catch((err) => {
+                if (err?.statusCode === 409)
+                    return null;
+                throw err;
+            });
+            if (!claimed) {
+                logger(`Job ${job.id} already claimed, skipping`);
+                processing = false;
+                return;
+            }
+            logger(`Claimed job ${job.id} for target ${job.evalTargetId}`);
+            // Check if this is a conversation job
+            if (job.config && job.config.type === 'conversation') {
+                await processConversationJob(job.config, job.evalTargetId, logger);
+                await apiPost(`/api/eval-jobs/${job.id}/complete`, { result: { type: 'conversation' } });
+                processing = false;
+                return;
+            }
+            const target = await apiGet(`/api/eval-targets/${job.evalTargetId}`);
+            const evalCases = await apiGet(`/api/eval-targets/${job.evalTargetId}/evals`);
+            if (evalCases.length === 0) {
+                await apiPost(`/api/eval-jobs/${job.id}/complete`, { error: 'No evals defined for target' });
+                processing = false;
+                return;
+            }
+            const config = job.config;
+            const runsPerEval = config.runsPerEval ?? 1;
+            const timeout = config.timeout ?? 120_000;
+            const iteration = config.iteration ?? 1;
+            const configurations = [
+                { name: 'with_target', content: target.content },
+            ];
+            if (!config.skipBaseline) {
+                configurations.push({ name: 'without_target' });
+            }
+            // Build work queue
+            const workQueue = [];
+            for (const eval_ of evalCases) {
+                for (const cfg of configurations) {
+                    for (let run = 1; run <= runsPerEval; run++) {
+                        workQueue.push({ eval_: eval_, configuration: cfg.name, targetContent: cfg.content, runNumber: run });
+                    }
+                }
+            }
+            const total = workQueue.length;
+            let completed = 0;
+            const concurrency = Math.min(config.parallel ?? 1, 4); // cap at 4 parallel
+            await apiPatch(`/api/eval-jobs/${job.id}/progress`, {
+                status: 'running',
+                progress: { total, completed: 0, current: null, message: `Starting ${total} eval runs (concurrency: ${concurrency})...` },
+            });
+            const results = [];
+            // Process work items — sequential if concurrency=1, parallel otherwise
+            async function processItem(item) {
+                const label = `${item.configuration} · ${item.eval_.name} · run#${item.runNumber}`;
+                logger(`[${completed + 1}/${total}] ${label}`);
+                // Create isolated worktree for parallel execution
+                const worktree = concurrency > 1 ? createEvalWorktree() : null;
+                try {
+                    const execResult = await executeEval({
+                        prompt: item.eval_.prompt,
+                        targetContent: item.targetContent,
+                        workDir: worktree?.workDir,
+                        timeout,
+                        model: config.model,
+                    });
+                    let grading = {};
+                    if (!config.skipGrading && item.eval_.expectations.length > 0 && !execResult.error) {
+                        const gradingResult = await gradeEvalOutput({
+                            prompt: item.eval_.prompt,
+                            output: execResult.output,
+                            expectations: item.eval_.expectations,
+                            model: config.graderModel ?? config.model,
+                            timeout,
+                        });
+                        grading = gradingResult;
+                    }
+                    return {
+                        evalId: item.eval_.id,
+                        evalTargetId: target.id,
+                        evalTargetVersion: target.version,
+                        configuration: item.configuration,
+                        runNumber: item.runNumber,
+                        iteration,
+                        status: execResult.error ? 'failed' : 'completed',
+                        outputs: execResult.error ? { error: execResult.error } : { transcript: execResult.output },
+                        timing: { duration_ms: execResult.durationMs, total_tokens: execResult.totalTokens },
+                        grading,
+                        worktree: worktree?.isWorktree ? true : undefined,
+                    };
+                }
+                finally {
+                    worktree?.cleanup();
+                }
+            }
+            if (concurrency <= 1) {
+                // Sequential execution (existing behavior)
+                for (const item of workQueue) {
+                    const label = `${item.configuration} · ${item.eval_.name} · run#${item.runNumber}`;
+                    await apiPatch(`/api/eval-jobs/${job.id}/progress`, {
+                        status: 'running',
+                        progress: { total, completed, current: label, message: `Executing ${label}...` },
+                    });
+                    const runResult = await processItem(item);
+                    await apiPost('/api/eval-runs', runResult);
+                    results.push(runResult);
+                    completed++;
+                    await apiPatch(`/api/eval-jobs/${job.id}/progress`, {
+                        status: 'running',
+                        progress: { total, completed, current: label, message: `Completed ${label}` },
+                    });
+                }
+            }
+            else {
+                // Parallel execution with worktree isolation
+                logger(`Running ${total} evals with concurrency=${concurrency} (worktree isolation)`);
+                for (let i = 0; i < workQueue.length; i += concurrency) {
+                    const batch = workQueue.slice(i, i + concurrency);
+                    const batchLabel = `batch ${Math.floor(i / concurrency) + 1}/${Math.ceil(workQueue.length / concurrency)}`;
+                    await apiPatch(`/api/eval-jobs/${job.id}/progress`, {
+                        status: 'running',
+                        progress: { total, completed, current: batchLabel, message: `Executing ${batchLabel} (${batch.length} parallel)...` },
+                    });
+                    const batchResults = await Promise.all(batch.map(processItem));
+                    for (const runResult of batchResults) {
+                        await apiPost('/api/eval-runs', runResult);
+                        results.push(runResult);
+                        completed++;
+                    }
+                    await apiPatch(`/api/eval-jobs/${job.id}/progress`, {
+                        status: 'running',
+                        progress: { total, completed, current: batchLabel, message: `Completed ${batchLabel}` },
+                    });
+                }
+            }
+            const summary = buildSummary(results);
+            // Meta-judge: assess eval quality if we have grading results
+            let metaJudgement;
+            if (!config.skipGrading && results.some(r => r.grading && Object.keys(r.grading).length > 0)) {
+                try {
+                    logger(`Running meta-judge for job ${job.id}...`);
+                    const evalInputs = evalCases.map(e => {
+                        const matchingResults = results.filter(r => r.evalId === e.id && r.configuration === 'with_target');
+                        const firstResult = matchingResults[0];
+                        return {
+                            evalName: e.name,
+                            prompt: e.prompt,
+                            expectations: e.expectations,
+                            grading: firstResult?.grading ? firstResult.grading : null,
+                            output: firstResult?.outputs?.transcript?.slice(0, 500),
+                        };
+                    });
+                    metaJudgement = await judgeEvalQuality({
+                        targetName: target.name,
+                        evalResults: evalInputs,
+                        model: config.graderModel ?? config.model,
+                        timeout: 60000,
+                    });
+                    logger(`Meta-judge: quality=${metaJudgement.quality_score}/5, issues=${metaJudgement.issues.length}`);
+                }
+                catch (err) {
+                    logger(`Meta-judge failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
+                }
+            }
+            const result = metaJudgement
+                ? { ...summary, meta_judgement: metaJudgement }
+                : summary;
+            await apiPost(`/api/eval-jobs/${job.id}/complete`, { result });
+            logger(`Job ${job.id} completed: ${completed}/${total} runs`);
+        }
+        catch (err) {
+            const errorMsg = err instanceof Error ? err.message : String(err);
+            logger(`Job ${job.id} failed: ${errorMsg}`);
+            await apiPost(`/api/eval-jobs/${job.id}/complete`, { error: errorMsg }).catch(() => { });
+        }
+        processing = false;
+    }
+    return () => {
+        clearInterval(pollInterval);
+        clearInterval(heartbeatInterval);
+        supabase.removeChannel(channel);
+        supabase.removeChannel(insightsChannel);
+        supabase.removeChannel(heartbeatChannel);
+    };
+}
+function buildSummary(results) {
+    const completed = results.filter((r) => r.status === 'completed');
+    const failed = results.filter((r) => r.status === 'failed');
+    const gradedRuns = completed.filter((r) => {
+        const grading = r.grading;
+        return grading && Object.keys(grading).length > 0;
+    });
+    const avgPassRate = gradedRuns.length > 0
+        ? gradedRuns.reduce((sum, r) => sum + (r.grading?.summary?.pass_rate ?? 0), 0) / gradedRuns.length
+        : null;
+    return {
+        total: results.length,
+        completed: completed.length,
+        failed: failed.length,
+        avgPassRate,
+    };
+}
+//# sourceMappingURL=eval-worker.js.map