@prowi/deskcheck 0.1.0 → 0.2.0

package/build/services/testing/JudgeService.js

@@ -0,0 +1,95 @@
+import { ExecutorService } from "../ExecutorService.js";
+import { buildJudgePrompt } from "../../prompts/JudgePrompt.js";
+// =============================================================================
+// JudgeService
+// =============================================================================
+/**
+ * Spawns a judge agent to evaluate executor findings against expected results.
+ *
+ * The judge is a pure reasoning agent (no tools, single turn) that compares
+ * findings to expectations and produces structured verdicts. Model is
+ * configurable via config.agents.judge.model (default: opus).
+ */
+export class JudgeService {
+    executorService;
+    judgeModel;
+    constructor(config, projectRoot) {
+        this.executorService = new ExecutorService(config, projectRoot);
+        this.judgeModel = config.agents.judge.model ?? "opus";
+    }
+    /**
+     * Evaluate executor findings against expected results for a criterion.
+     *
+     * @param criterion - The criterion that was tested.
+     * @param expectedContent - Content of expected.md describing what should be found.
+     * @param findings - Findings produced by the executor agent.
+     * @param fixtureContent - Content of the fixture file that was reviewed.
+     * @returns Judge result with finding and expectation reviews, plus usage data.
+     */
+    async evaluate(criterion, expectedContent, findings, fixtureContent) {
+        const prompt = buildJudgePrompt(criterion.prompt, expectedContent, findings, fixtureContent);
+        // Judge uses configured model (default opus), no tools, single turn
+        const executorResult = await this.executorService.execute(prompt, this.judgeModel, {
+            tools: [],
+            maxTurns: 1,
+        });
+        const result = parseJudgeOutput(executorResult.resultText);
+        return { result, usage: executorResult.usage };
+    }
+}
+// =============================================================================
+// Parsing
+// =============================================================================
+/**
+ * Parse the judge agent's output into a typed JudgeResult.
+ *
+ * Looks for a JSON object (not array) in the output, strips markdown fences,
+ * and validates the structure. Returns empty results rather than crashing on
+ * malformed output.
+ */
+function parseJudgeOutput(output) {
+    let jsonText = output.trim();
+    // Strip markdown code fences if present
+    const fencedMatch = jsonText.match(/```(?:json)?\s*\n?([\s\S]*?)\n?\s*```/);
+    if (fencedMatch) {
+        jsonText = fencedMatch[1].trim();
+    }
+    // Find the JSON object boundaries
+    const startIdx = jsonText.indexOf("{");
+    const endIdx = jsonText.lastIndexOf("}");
+    if (startIdx === -1 || endIdx === -1 || endIdx <= startIdx) {
+        console.error(`[deskcheck] Warning: judge output did not contain a JSON object. Output: ${output.slice(0, 200)}`);
+        return emptyJudgeResult();
+    }
+    jsonText = jsonText.slice(startIdx, endIdx + 1);
+    let parsed;
+    try {
+        parsed = JSON.parse(jsonText);
+    }
+    catch (error) {
+        console.error(`[deskcheck] Warning: failed to parse judge JSON output: ${error instanceof Error ? error.message : String(error)}. Output: ${jsonText.slice(0, 200)}`);
+        return emptyJudgeResult();
+    }
+    if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
+        console.error(`[deskcheck] Warning: judge output parsed but was not an object. Type: ${typeof parsed}`);
+        return emptyJudgeResult();
+    }
+    const record = parsed;
+    // Validate structure
+    if (!Array.isArray(record.findings_review) || !Array.isArray(record.expectations_review)) {
+        console.error(`[deskcheck] Warning: judge output missing findings_review or expectations_review arrays.`);
+        return emptyJudgeResult();
+    }
+    return {
+        findings_review: record.findings_review,
+        expectations_review: record.expectations_review,
+    };
+}
+/** Return an empty judge result for error cases. */
+function emptyJudgeResult() {
+    return {
+        findings_review: [],
+        expectations_review: [],
+    };
+}
+//# sourceMappingURL=JudgeService.js.map

package/build/services/testing/JudgeService.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"JudgeService.js","sourceRoot":"","sources":["../../../src/services/testing/JudgeService.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAC;AAOhE,gFAAgF;AAChF,eAAe;AACf,gFAAgF;AAEhF;;;;;;GAMG;AACH,MAAM,OAAO,YAAY;IACN,eAAe,CAAkB;IACjC,UAAU,CAAa;IAExC,YAAY,MAAoB,EAAE,WAAmB;QACnD,IAAI,CAAC,eAAe,GAAG,IAAI,eAAe,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;QAChE,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,IAAI,MAAM,CAAC;IACxD,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,QAAQ,CACZ,SAAuB,EACvB,eAAuB,EACvB,QAAmB,EACnB,cAAsB;QAEtB,MAAM,MAAM,GAAG,gBAAgB,CAC7B,SAAS,CAAC,MAAM,EAChB,eAAe,EACf,QAAQ,EACR,cAAc,CACf,CAAC;QAEF,oEAAoE;QACpE,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE;YACjF,KAAK,EAAE,EAAE;YACT,QAAQ,EAAE,CAAC;SACZ,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,gBAAgB,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC;QAE3D,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,cAAc,CAAC,KAAK,EAAE,CAAC;IACjD,CAAC;CACF;AAED,gFAAgF;AAChF,UAAU;AACV,gFAAgF;AAEhF;;;;;;GAMG;AACH,SAAS,gBAAgB,CAAC,MAAc;IACtC,IAAI,QAAQ,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC;IAE7B,wCAAwC;IACxC,MAAM,WAAW,GAAG,QAAQ,CAAC,KAAK,CAAC,uCAAuC,CAAC,CAAC;IAC5E,IAAI,WAAW,EAAE,CAAC;QAChB,QAAQ,GAAG,WAAW,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,CAAC;IACpC,CAAC;IAED,kCAAkC;IAClC,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACvC,MAAM,MAAM,GAAG,QAAQ,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IACzC,IAAI,QAAQ,KAAK,CAAC,CAAC,IAAI,MAAM,KAAK,CAAC,CAAC,IAAI,MAAM,IAAI,QAAQ,EAAE,CAAC;QAC3D,OAAO,CAAC,KAAK,CACX,4EAA4E,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CACnG,CAAC;QACF,OAAO,gBAAgB,EAAE,CAAC;IAC5B,CAAC;IAED,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC;IAEhD,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAChC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CACX,2DAA2D,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,aAAa,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CACvJ,CAAC;QACF,OAAO,gBAAgB,EAAE,CAAC;IAC5B,CAAC;IAED,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAC3E,OAAO,CAAC,KAAK,CACX,yEAAyE,OAAO,MAAM,EAAE,CACzF,CAAC;QACF,OAAO,gBAAgB,EAAE,CAAC;IAC5B,CAAC;IAED,MAAM,MAAM,GAAG,MAAiC,CAAC;IAEjD,qBAAqB;IACrB,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,mBAAmB,CAAC,EAAE,CAAC;QACzF,OAAO,CAAC,KAAK,CACX,0FAA0F,CAC3F,CAAC;QACF,OAAO,gBAAgB,EAAE,CAAC;IAC5B,CAAC;IAED,OAAO;QACL,eAAe,EAAE,MAAM,CAAC,eAAiD;QACzE,mBAAmB,EAAE,MAAM,CAAC,mBAAyD;KACtF,CAAC;AACJ,CAAC;AAED,oDAAoD;AACpD,SAAS,gBAAgB;IACvB,OAAO;QACL,eAAe,EAAE,EAAE;QACnB,mBAAmB,EAAE,EAAE;KACxB,CAAC;AACJ,CAAC"}

package/build/services/testing/TestDiscoveryService.d.ts

@@ -0,0 +1,16 @@
+import type { TestCase } from "../../types/testing.js";
+/**
+ * Discover test cases by scanning the tests directory.
+ *
+ * Test cases mirror the criteria directory structure:
+ *   tests/{criterionId}/{testName}/
+ *     - One fixture file (any extension except .md)
+ *     - One expected.md describing expected findings
+ *
+ * @param testsDir - Path to the tests directory.
+ * @param criteriaDir - Path to the criteria directory.
+ * @param criterionFilter - Optional list of criterion name patterns to include.
+ * @returns Sorted array of discovered test cases.
+ */
+export declare function discoverTests(testsDir: string, criteriaDir: string, criterionFilter?: string[]): TestCase[];
+//# sourceMappingURL=TestDiscoveryService.d.ts.map

package/build/services/testing/TestDiscoveryService.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"TestDiscoveryService.d.ts","sourceRoot":"","sources":["../../../src/services/testing/TestDiscoveryService.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAGvD;;;;;;;;;;;;GAYG;AACH,wBAAgB,aAAa,CAC3B,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,MAAM,EACnB,eAAe,CAAC,EAAE,MAAM,EAAE,GACzB,QAAQ,EAAE,CA4DZ"}

package/build/services/testing/TestDiscoveryService.js

@@ -0,0 +1,66 @@
+import fs from "node:fs";
+import path from "node:path";
+import { discoverCriteria, filterCriteria } from "../criteria/CriteriaService.js";
+/**
+ * Discover test cases by scanning the tests directory.
+ *
+ * Test cases mirror the criteria directory structure:
+ *   tests/{criterionId}/{testName}/
+ *     - One fixture file (any extension except .md)
+ *     - One expected.md describing expected findings
+ *
+ * @param testsDir - Path to the tests directory.
+ * @param criteriaDir - Path to the criteria directory.
+ * @param criterionFilter - Optional list of criterion name patterns to include.
+ * @returns Sorted array of discovered test cases.
+ */
+export function discoverTests(testsDir, criteriaDir, criterionFilter) {
+    const absoluteTestsDir = path.resolve(testsDir);
+    const absoluteCriteriaDir = path.resolve(criteriaDir);
+    // Discover all criteria, optionally filtered
+    let criteria = discoverCriteria(absoluteCriteriaDir);
+    if (criterionFilter && criterionFilter.length > 0) {
+        criteria = filterCriteria(criteria, criterionFilter);
+    }
+    const testCases = [];
+    for (const criterion of criteria) {
+        // Look for matching test directory: testsDir/{criterion.id}/
+        const criterionTestDir = path.join(absoluteTestsDir, criterion.id);
+        if (!fs.existsSync(criterionTestDir) || !fs.statSync(criterionTestDir).isDirectory()) {
+            continue;
+        }
+        // Each subdirectory is a test case
+        const entries = fs.readdirSync(criterionTestDir, { withFileTypes: true });
+        const testDirs = entries.filter((entry) => entry.isDirectory()).sort((a, b) => a.name.localeCompare(b.name));
+        for (const testDir of testDirs) {
+            const testPath = path.join(criterionTestDir, testDir.name);
+            const testFiles = fs.readdirSync(testPath);
+            // Find the fixture file (first non-.md file)
+            const fixtureFileName = testFiles
+                .filter((f) => !f.endsWith(".md"))
+                .sort()[0];
+            // Check for expected.md
+            const hasExpectedMd = testFiles.includes("expected.md");
+            if (!fixtureFileName || !hasExpectedMd) {
+                console.error(`[deskcheck] Warning: skipping test "${testDir.name}" for criterion "${criterion.id}": ` +
+                    `missing ${!fixtureFileName ? "fixture file" : "expected.md"}`);
+                continue;
+            }
+            testCases.push({
+                criterionId: criterion.id,
+                name: testDir.name,
+                criterionFile: criterion.file,
+                fixtureFile: path.join(testPath, fixtureFileName),
+                expectedFile: path.join(testPath, "expected.md"),
+            });
+        }
+    }
+    // Sort by criterionId, then by name for deterministic ordering
+    return testCases.sort((a, b) => {
+        const idCompare = a.criterionId.localeCompare(b.criterionId);
+        if (idCompare !== 0)
+            return idCompare;
+        return a.name.localeCompare(b.name);
+    });
+}
+//# sourceMappingURL=TestDiscoveryService.js.map

package/build/services/testing/TestDiscoveryService.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"TestDiscoveryService.js","sourceRoot":"","sources":["../../../src/services/testing/TestDiscoveryService.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,gCAAgC,CAAC;AAElF;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,aAAa,CAC3B,QAAgB,EAChB,WAAmB,EACnB,eAA0B;IAE1B,MAAM,gBAAgB,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAChD,MAAM,mBAAmB,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;IAEtD,6CAA6C;IAC7C,IAAI,QAAQ,GAAG,gBAAgB,CAAC,mBAAmB,CAAC,CAAC;IACrD,IAAI,eAAe,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClD,QAAQ,GAAG,cAAc,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAC;IACvD,CAAC;IAED,MAAM,SAAS,GAAe,EAAE,CAAC;IAEjC,KAAK,MAAM,SAAS,IAAI,QAAQ,EAAE,CAAC;QACjC,6DAA6D;QAC7D,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,SAAS,CAAC,EAAE,CAAC,CAAC;QAEnE,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,gBAAgB,CAAC,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC;YACrF,SAAS;QACX,CAAC;QAED,mCAAmC;QACnC,MAAM,OAAO,GAAG,EAAE,CAAC,WAAW,CAAC,gBAAgB,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1E,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAE7G,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;YAC3D,MAAM,SAAS,GAAG,EAAE,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;YAE3C,6CAA6C;YAC7C,MAAM,eAAe,GAAG,SAAS;iBAC9B,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;iBACjC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YAEb,wBAAwB;YACxB,MAAM,aAAa,GAAG,SAAS,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;YAExD,IAAI,CAAC,eAAe,IAAI,CAAC,aAAa,EAAE,CAAC;gBACvC,OAAO,CAAC,KAAK,CACX,uCAAuC,OAAO,CAAC,IAAI,oBAAoB,SAAS,CAAC,EAAE,KAAK;oBACxF,WAAW,CAAC,eAAe,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,aAAa,EAAE,CAC/D,CAAC;gBACF,SAAS;YACX,CAAC;YAED,SAAS,CAAC,IAAI,CAAC;gBACb,WAAW,EAAE,SAAS,CAAC,EAAE;gBACzB,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,aAAa,EAAE,SAAS,CAAC,IAAI;gBAC7B,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,eAAe,CAAC;gBACjD,YAAY,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC;aACjD,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,+DAA+D;IAC/D,OAAO,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QAC7B,MAAM,SAAS,GAAG,CAAC,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;QAC7D,IAAI,SAAS,KAAK,CAAC;YAAE,OAAO,SAAS,CAAC;QACtC,OAAO,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACtC,CAAC,CAAC,CAAC;AACL,CAAC"}

package/build/services/testing/TestRunnerService.d.ts

@@ -0,0 +1,35 @@
+import type { ReviewConfig } from "../../config/types.js";
+import type { TestCase, TestCaseResult, TestRun } from "../../types/testing.js";
+/** Options for controlling test run behavior. */
+export interface TestRunOptions {
+    /** Called after each test case completes (or errors), for live progress reporting. */
+    onTestComplete?: (criterionId: string, testName: string, result: TestCaseResult) => void;
+}
+/**
+ * Orchestrates the full test pipeline: executor -> judge -> scorer -> storage.
+ *
+ * Runs test cases sequentially (no concurrency). For each test case:
+ * 1. Execute the criterion against the fixture file
+ * 2. Judge the findings against expected results
+ * 3. Score the judge's verdicts
+ * 4. Persist everything to storage
+ */
+export declare class TestRunnerService {
+    private readonly config;
+    private readonly projectRoot;
+    constructor(config: ReviewConfig, projectRoot: string);
+    /**
+     * Run all test cases and return the completed test run.
+     *
+     * @param testCases - Discovered test cases to execute.
+     * @param storageDir - Directory where test run results are persisted.
+     * @param options - Optional callbacks for progress reporting.
+     * @returns The final TestRun with all results.
+     */
+    run(testCases: TestCase[], storageDir: string, options?: TestRunOptions): Promise<TestRun>;
+    /**
+     * Execute a single test case through the full pipeline.
+     */
+    private executeTestCase;
+}
+//# sourceMappingURL=TestRunnerService.d.ts.map

package/build/services/testing/TestRunnerService.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"TestRunnerService.d.ts","sourceRoot":"","sources":["../../../src/services/testing/TestRunnerService.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AAC1D,OAAO,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,wBAAwB,CAAC;AAchF,iDAAiD;AACjD,MAAM,WAAW,cAAc;IAC7B,sFAAsF;IACtF,cAAc,CAAC,EAAE,CAAC,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,cAAc,KAAK,IAAI,CAAC;CAC1F;AAMD;;;;;;;;GAQG;AACH,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAe;IACtC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;gBAEzB,MAAM,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM;IAKrD;;;;;;;OAOG;IACG,GAAG,CACP,SAAS,EAAE,QAAQ,EAAE,EACrB,UAAU,EAAE,MAAM,EAClB,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CAAC,OAAO,CAAC;IAyDnB;;OAEG;YACW,eAAe;CA6E9B"}

package/build/services/testing/TestRunnerService.js

@@ -0,0 +1,140 @@
+import fs from "node:fs";
+import path from "node:path";
+import { ExecutorService } from "../ExecutorService.js";
+import { buildExecutorPrompt } from "../../prompts/ExecutorPrompt.js";
+import { parseFindings } from "../FindingsParserService.js";
+import { parseCriterion } from "../criteria/CriteriaService.js";
+import { TestStorageService } from "./TestStorageService.js";
+import { JudgeService } from "./JudgeService.js";
+import { calculateScores } from "./TestScorerService.js";
+// =============================================================================
+// TestRunnerService
+// =============================================================================
+/**
+ * Orchestrates the full test pipeline: executor -> judge -> scorer -> storage.
+ *
+ * Runs test cases sequentially (no concurrency). For each test case:
+ * 1. Execute the criterion against the fixture file
+ * 2. Judge the findings against expected results
+ * 3. Score the judge's verdicts
+ * 4. Persist everything to storage
+ */
+export class TestRunnerService {
+    config;
+    projectRoot;
+    constructor(config, projectRoot) {
+        this.config = config;
+        this.projectRoot = projectRoot;
+    }
+    /**
+     * Run all test cases and return the completed test run.
+     *
+     * @param testCases - Discovered test cases to execute.
+     * @param storageDir - Directory where test run results are persisted.
+     * @param options - Optional callbacks for progress reporting.
+     * @returns The final TestRun with all results.
+     */
+    async run(testCases, storageDir, options) {
+        const storage = new TestStorageService(storageDir);
+        const executorService = new ExecutorService(this.config, this.projectRoot);
+        const judgeService = new JudgeService(this.config, this.projectRoot);
+        // Initialize the run with all tests as "pending"
+        const run = storage.createRun(testCases);
+        const runId = run.run_id;
+        // Track which suites have been started
+        const startedSuites = new Set();
+        for (const testCase of testCases) {
+            // Mark suite as running if not yet started
+            if (!startedSuites.has(testCase.criterionId)) {
+                storage.updateSuiteStatus(runId, testCase.criterionId, "running");
+                startedSuites.add(testCase.criterionId);
+            }
+            try {
+                await this.executeTestCase(testCase, runId, storage, executorService, judgeService);
+            }
+            catch (error) {
+                // Catch unexpected errors and record them
+                const errorMessage = error instanceof Error ? error.message : String(error);
+                storage.updateTestCase(runId, testCase.criterionId, testCase.name, {
+                    status: "error",
+                    error: errorMessage,
+                });
+            }
+            // Notify caller of test completion (whether success or error)
+            if (options?.onTestComplete) {
+                const currentRun = storage.getRun(runId);
+                const result = currentRun.suites[testCase.criterionId]?.tests[testCase.name];
+                if (result) {
+                    options.onTestComplete(testCase.criterionId, testCase.name, result);
+                }
+            }
+        }
+        // Update suite statuses to "complete"
+        for (const criterionId of startedSuites) {
+            storage.updateSuiteStatus(runId, criterionId, "complete");
+        }
+        // Complete the run
+        storage.completeRun(runId);
+        return storage.getRun(runId);
+    }
+    /**
+     * Execute a single test case through the full pipeline.
+     */
+    async executeTestCase(testCase, runId, storage, executorService, judgeService) {
+        // Step 1: Update status to "executing"
+        storage.updateTestCase(runId, testCase.criterionId, testCase.name, {
+            status: "executing",
+        });
+        // Step 2: Read the fixture file
+        const fixtureContent = fs.readFileSync(testCase.fixtureFile, "utf-8");
+        // Step 3: Parse the criterion
+        // criterionFile is relative to the parent of the criteria dir (e.g. "criteria/backend/controller-conventions.md")
+        // The criteria dir config value is e.g. "deskcheck/criteria", so the parent is "deskcheck/"
+        // parseCriterion needs the absolute file path and the absolute criteria dir as base
+        const criteriaDir = path.resolve(this.projectRoot, this.config.modules_dir);
+        const absoluteCriterionFile = path.resolve(path.dirname(criteriaDir), testCase.criterionFile);
+        const criterion = parseCriterion(absoluteCriterionFile, criteriaDir);
+        // Step 4: Build a synthetic ReviewTask for the executor
+        const task = {
+            task_id: `test-${testCase.criterionId}-${testCase.name}`,
+            review_id: testCase.criterionId,
+            review_file: testCase.criterionFile,
+            files: [testCase.fixtureFile],
+            hint: null,
+            model: criterion.model,
+            status: "pending",
+            created_at: new Date().toISOString(),
+            started_at: null,
+            completed_at: null,
+            context_type: "file",
+            context: fixtureContent,
+            symbol: null,
+            prompt: criterion.prompt,
+        };
+        // Step 5: Build executor prompt and execute
+        const prompt = buildExecutorPrompt(task);
+        const executorResult = await executorService.execute(prompt, criterion.model);
+        // Step 6: Parse findings
+        const findings = parseFindings(executorResult.resultText);
+        // Step 7: Update storage with findings and executor usage, move to "judging"
+        storage.updateTestCase(runId, testCase.criterionId, testCase.name, {
+            status: "judging",
+            findings,
+            executor_usage: executorResult.usage,
+        });
+        // Step 8: Read expected.md
+        const expectedContent = fs.readFileSync(testCase.expectedFile, "utf-8");
+        // Step 9: Judge findings against expectations
+        const judgeOutput = await judgeService.evaluate(criterion, expectedContent, findings, fixtureContent);
+        // Step 10: Score the judge's verdicts
+        const scores = calculateScores(judgeOutput.result);
+        // Step 11: Update storage with judge result, scores, and judge usage
+        storage.updateTestCase(runId, testCase.criterionId, testCase.name, {
+            status: "complete",
+            judge: judgeOutput.result,
+            scores,
+            judge_usage: judgeOutput.usage,
+        });
+    }
+}
+//# sourceMappingURL=TestRunnerService.js.map

package/build/services/testing/TestRunnerService.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"TestRunnerService.js","sourceRoot":"","sources":["../../../src/services/testing/TestRunnerService.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAI7B,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,mBAAmB,EAAE,MAAM,iCAAiC,CAAC;AACtE,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAC5D,OAAO,EAAE,cAAc,EAAE,MAAM,gCAAgC,CAAC;AAChE,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAC7D,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACjD,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAYzD,gFAAgF;AAChF,oBAAoB;AACpB,gFAAgF;AAEhF;;;;;;;;GAQG;AACH,MAAM,OAAO,iBAAiB;IACX,MAAM,CAAe;IACrB,WAAW,CAAS;IAErC,YAAY,MAAoB,EAAE,WAAmB;QACnD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;IACjC,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,GAAG,CACP,SAAqB,EACrB,UAAkB,EAClB,OAAwB;QAExB,MAAM,OAAO,GAAG,IAAI,kBAAkB,CAAC,UAAU,CAAC,CAAC;QACnD,MAAM,eAAe,GAAG,IAAI,eAAe,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QAC3E,MAAM,YAAY,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QAErE,iDAAiD;QACjD,MAAM,GAAG,GAAG,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;QACzC,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC;QAEzB,uCAAuC;QACvC,MAAM,aAAa,GAAG,IAAI,GAAG,EAAU,CAAC;QAExC,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,2CAA2C;YAC3C,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;gBAC7C,OAAO,CAAC,iBAAiB,CAAC,KAAK,EAAE,QAAQ,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;gBAClE,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;YAC1C,CAAC;YAED,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,eAAe,CACxB,QAAQ,EACR,KAAK,EACL,OAAO,EACP,eAAe,EACf,YAAY,CACb,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,0CAA0C;gBAC1C,MAAM,YAAY,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBAC5E,OAAO,CAAC,cAAc,CAAC,KAAK,EAAE,QAAQ,CAAC,WAAW,EAAE,QAAQ,CAAC,IAAI,EAAE;oBACjE,MAAM,EAAE,OAAO;oBACf,KAAK,EAAE,YAAY;iBACpB,CAAC,CAAC;YACL,CAAC;YAED,8DAA8D;YAC9D,IAAI,OAAO,EAAE,cAAc,EAAE,CAAC;gBAC5B,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACzC,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;gBAC7E,IAAI,MAAM,EAAE,CAAC;oBACX,OAAO,CAAC,cAAc,CAAC,QAAQ,CAAC,WAAW,EAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;gBACtE,CAAC;YACH,CAAC;QACH,CAAC;QAED,sCAAsC;QACtC,KAAK,MAAM,WAAW,IAAI,aAAa,EAAE,CAAC;YACxC,OAAO,CAAC,iBAAiB,CAAC,KAAK,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC;QAC5D,CAAC;QAED,mBAAmB;QACnB,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QAE3B,OAAO,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC/B,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,eAAe,CAC3B,QAAkB,EAClB,KAAa,EACb,OAA2B,EAC3B,eAAgC,EAChC,YAA0B;QAE1B,uCAAuC;QACvC,OAAO,CAAC,cAAc,CAAC,KAAK,EAAE,QAAQ,CAAC,WAAW,EAAE,QAAQ,CAAC,IAAI,EAAE;YACjE,MAAM,EAAE,WAAW;SACpB,CAAC,CAAC;QAEH,gCAAgC;QAChC,MAAM,cAAc,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;QAEtE,8BAA8B;QAC9B,kHAAkH;QAClH,4FAA4F;QAC5F,oFAAoF;QACpF,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;QAC5E,MAAM,qBAAqB,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,QAAQ,CAAC,aAAa,CAAC,CAAC;QAC9F,MAAM,SAAS,GAAG,cAAc,CAAC,qBAAqB,EAAE,WAAW,CAAC,CAAC;QAErE,wDAAwD;QACxD,MAAM,IAAI,GAAe;YACvB,OAAO,EAAE,QAAQ,QAAQ,CAAC,WAAW,IAAI,QAAQ,CAAC,IAAI,EAAE;YACxD,SAAS,EAAE,QAAQ,CAAC,WAAW;YAC/B,WAAW,EAAE,QAAQ,CAAC,aAAa;YACnC,KAAK,EAAE,CAAC,QAAQ,CAAC,WAAW,CAAC;YAC7B,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,SAAS,CAAC,KAAK;YACtB,MAAM,EAAE,SAAS;YACjB,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACpC,UAAU,EAAE,IAAI;YAChB,YAAY,EAAE,IAAI;YAClB,YAAY,EAAE,MAAM;YACpB,OAAO,EAAE,cAAc;YACvB,MAAM,EAAE,IAAI;YACZ,MAAM,EAAE,SAAS,CAAC,MAAM;SACzB,CAAC;QAEF,4CAA4C;QAC5C,MAAM,MAAM,GAAG,mBAAmB,CAAC,IAAI,CAAC,CAAC;QACzC,MAAM,cAAc,GAAG,MAAM,eAAe,CAAC,OAAO,CAAC,MAAM,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC;QAE9E,yBAAyB;QACzB,MAAM,QAAQ,GAAG,aAAa,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC;QAE1D,6EAA6E;QAC7E,OAAO,CAAC,cAAc,CAAC,KAAK,EAAE,QAAQ,CAAC,WAAW,EAAE,QAAQ,CAAC,IAAI,EAAE;YACjE,MAAM,EAAE,SAAS;YACjB,QAAQ;YACR,cAAc,EAAE,cAAc,CAAC,KAAK;SACrC,CAAC,CAAC;QAEH,2BAA2B;QAC3B,MAAM,eAAe,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;QAExE,8CAA8C;QAC9C,MAAM,WAAW,GAAG,MAAM,YAAY,CAAC,QAAQ,CAC7C,SAAS,EACT,eAAe,EACf,QAAQ,EACR,cAAc,CACf,CAAC;QAEF,sCAAsC;QACtC,MAAM,MAAM,GAAG,eAAe,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QAEnD,qEAAqE;QACrE,OAAO,CAAC,cAAc,CAAC,KAAK,EAAE,QAAQ,CAAC,WAAW,EAAE,QAAQ,CAAC,IAAI,EAAE;YACjE,MAAM,EAAE,UAAU;YAClB,KAAK,EAAE,WAAW,CAAC,MAAM;YACzB,MAAM;YACN,WAAW,EAAE,WAAW,CAAC,KAAK;SAC/B,CAAC,CAAC;IACL,CAAC;CACF"}

package/build/services/testing/TestScorerService.d.ts

@@ -0,0 +1,9 @@
+import type { JudgeResult, TestScores } from "../../types/testing.js";
+/**
+ * Calculate test scores from judge verdicts.
+ *
+ * Pure math — no LLM, no I/O. Computes recall, precision, and scope
+ * compliance from the judge's finding and expectation reviews.
+ */
+export declare function calculateScores(judgeResult: JudgeResult): TestScores;
+//# sourceMappingURL=TestScorerService.d.ts.map

package/build/services/testing/TestScorerService.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"TestScorerService.d.ts","sourceRoot":"","sources":["../../../src/services/testing/TestScorerService.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAEtE;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,WAAW,EAAE,WAAW,GAAG,UAAU,CA+BpE"}

package/build/services/testing/TestScorerService.js

@@ -0,0 +1,35 @@
+/**
+ * Calculate test scores from judge verdicts.
+ *
+ * Pure math — no LLM, no I/O. Computes recall, precision, and scope
+ * compliance from the judge's finding and expectation reviews.
+ */
+export function calculateScores(judgeResult) {
+    const { findings_review, expectations_review } = judgeResult;
+    // Finding counts by verdict
+    const total_findings = findings_review.length;
+    const correct = findings_review.filter((f) => f.verdict === "correct").length;
+    const out_of_scope = findings_review.filter((f) => f.verdict === "out_of_scope").length;
+    const incorrect_severity = findings_review.filter((f) => f.verdict === "incorrect_severity").length;
+    // Expectation counts by verdict
+    const total_expectations = expectations_review.length;
+    const expectations_found = expectations_review.filter((e) => e.verdict === "found").length;
+    const expectations_missed = expectations_review.filter((e) => e.verdict === "missed").length;
+    // Derived ratios (default to 1.0 when denominator is zero)
+    const recall = total_expectations > 0 ? expectations_found / total_expectations : 1.0;
+    const precision = total_findings > 0 ? correct / total_findings : 1.0;
+    const scope_compliance = total_findings > 0 ? (total_findings - out_of_scope) / total_findings : 1.0;
+    return {
+        total_findings,
+        correct,
+        out_of_scope,
+        incorrect_severity,
+        total_expectations,
+        expectations_found,
+        expectations_missed,
+        recall,
+        precision,
+        scope_compliance,
+    };
+}
+//# sourceMappingURL=TestScorerService.js.map

package/build/services/testing/TestScorerService.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"TestScorerService.js","sourceRoot":"","sources":["../../../src/services/testing/TestScorerService.ts"],"names":[],"mappings":"AAEA;;;;;GAKG;AACH,MAAM,UAAU,eAAe,CAAC,WAAwB;IACtD,MAAM,EAAE,eAAe,EAAE,mBAAmB,EAAE,GAAG,WAAW,CAAC;IAE7D,4BAA4B;IAC5B,MAAM,cAAc,GAAG,eAAe,CAAC,MAAM,CAAC;IAC9C,MAAM,OAAO,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,MAAM,CAAC;IAC9E,MAAM,YAAY,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,cAAc,CAAC,CAAC,MAAM,CAAC;IACxF,MAAM,kBAAkB,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,oBAAoB,CAAC,CAAC,MAAM,CAAC;IAEpG,gCAAgC;IAChC,MAAM,kBAAkB,GAAG,mBAAmB,CAAC,MAAM,CAAC;IACtD,MAAM,kBAAkB,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,OAAO,CAAC,CAAC,MAAM,CAAC;IAC3F,MAAM,mBAAmB,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,MAAM,CAAC;IAE7F,2DAA2D;IAC3D,MAAM,MAAM,GAAG,kBAAkB,GAAG,CAAC,CAAC,CAAC,CAAC,kBAAkB,GAAG,kBAAkB,CAAC,CAAC,CAAC,GAAG,CAAC;IACtF,MAAM,SAAS,GAAG,cAAc,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,cAAc,CAAC,CAAC,CAAC,GAAG,CAAC;IACtE,MAAM,gBAAgB,GAAG,cAAc,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,cAAc,GAAG,YAAY,CAAC,GAAG,cAAc,CAAC,CAAC,CAAC,GAAG,CAAC;IAErG,OAAO;QACL,cAAc;QACd,OAAO;QACP,YAAY;QACZ,kBAAkB;QAClB,kBAAkB;QAClB,kBAAkB;QAClB,mBAAmB;QACnB,MAAM;QACN,SAAS;QACT,gBAAgB;KACjB,CAAC;AACJ,CAAC"}

package/build/services/testing/TestStorageService.d.ts

@@ -0,0 +1,39 @@
+import type { TestCase, TestCaseResult, TestRun, TestSuiteStatus } from "../../types/testing.js";
+/**
+ * Persists test runs to .deskcheck/test-runs/ using atomic writes.
+ *
+ * Each test run lives in a timestamped directory containing a single
+ * results.json file. Uses write-to-tmp + rename for atomic updates.
+ * No file locking needed since test runs execute sequentially.
+ */
+export declare class TestStorageService {
+    private readonly storageDir;
+    constructor(storageDir: string);
+    /**
+     * Create a new test run with all test cases initialized as "pending".
+     *
+     * Creates a timestamped directory and writes an initial results.json.
+     */
+    createRun(testCases: TestCase[]): TestRun;
+    /** Read the results.json for a given run ID. */
+    getRun(runId: string): TestRun;
+    /**
+     * Return the most recent run directory name, or null if no runs exist.
+     */
+    getLatestRunId(): string | null;
+    /**
+     * Update a specific test case in results.json.
+     *
+     * Reads the current run, applies the partial update to the specified
+     * test case, and writes back atomically.
+     */
+    updateTestCase(runId: string, criterionId: string, testName: string, update: Partial<TestCaseResult>): void;
+    /** Update a suite's status. */
+    updateSuiteStatus(runId: string, criterionId: string, status: TestSuiteStatus): void;
+    /** Mark the run as complete with a completion timestamp. */
+    completeRun(runId: string): void;
+    private resultsPath;
+    /** Atomic write: write to .tmp then rename. */
+    private writeRun;
+}
+//# sourceMappingURL=TestStorageService.d.ts.map

package/build/services/testing/TestStorageService.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"TestStorageService.d.ts","sourceRoot":"","sources":["../../../src/services/testing/TestStorageService.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACV,QAAQ,EACR,cAAc,EACd,OAAO,EACP,eAAe,EAChB,MAAM,wBAAwB,CAAC;AAmBhC;;;;;;GAMG;AACH,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;gBAExB,UAAU,EAAE,MAAM;IAI9B;;;;OAIG;IACH,SAAS,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,OAAO;IA8CzC,gDAAgD;IAChD,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO;IAM9B;;OAEG;IACH,cAAc,IAAI,MAAM,GAAG,IAAI;IAsB/B;;;;;OAKG;IACH,cAAc,CACZ,KAAK,EAAE,MAAM,EACb,WAAW,EAAE,MAAM,EACnB,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,OAAO,CAAC,cAAc,CAAC,GAC9B,IAAI;IAiBP,+BAA+B;IAC/B,iBAAiB,CAAC,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,EAAE,eAAe,GAAG,IAAI;IAYpF,4DAA4D;IAC5D,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAWhC,OAAO,CAAC,WAAW;IAInB,+CAA+C;IAC/C,OAAO,CAAC,QAAQ;CAMjB"}

package/build/services/testing/TestStorageService.js

@@ -0,0 +1,144 @@
+import fs from "node:fs";
+import path from "node:path";
+// =============================================================================
+// Helpers
+// =============================================================================
+/** Format a Date as YYYY-MM-DD_HHmmss for use as a run ID / directory name. */
+function formatTimestamp(date) {
+    const pad2 = (n) => String(n).padStart(2, "0");
+    return (`${date.getFullYear()}-${pad2(date.getMonth() + 1)}-${pad2(date.getDate())}` +
+        `_${pad2(date.getHours())}${pad2(date.getMinutes())}${pad2(date.getSeconds())}`);
+}
+// =============================================================================
+// TestStorageService
+// =============================================================================
+/**
+ * Persists test runs to .deskcheck/test-runs/ using atomic writes.
+ *
+ * Each test run lives in a timestamped directory containing a single
+ * results.json file. Uses write-to-tmp + rename for atomic updates.
+ * No file locking needed since test runs execute sequentially.
+ */
+export class TestStorageService {
+    storageDir;
+    constructor(storageDir) {
+        this.storageDir = storageDir;
+    }
+    /**
+     * Create a new test run with all test cases initialized as "pending".
+     *
+     * Creates a timestamped directory and writes an initial results.json.
+     */
+    createRun(testCases) {
+        const now = new Date();
+        const runId = formatTimestamp(now);
+        const runDir = path.join(this.storageDir, runId);
+        fs.mkdirSync(runDir, { recursive: true });
+        // Group test cases by criterionId to build suites
+        const suites = {};
+        for (const testCase of testCases) {
+            if (!suites[testCase.criterionId]) {
+                suites[testCase.criterionId] = {
+                    status: "pending",
+                    criterion_file: testCase.criterionFile,
+                    tests: {},
+                };
+            }
+            const initialResult = {
+                status: "pending",
+                fixture_file: testCase.fixtureFile,
+                expected_file: testCase.expectedFile,
+                findings: null,
+                judge: null,
+                scores: null,
+                error: null,
+                executor_usage: null,
+                judge_usage: null,
+            };
+            suites[testCase.criterionId].tests[testCase.name] = initialResult;
+        }
+        const run = {
+            run_id: runId,
+            status: "running",
+            started_at: now.toISOString(),
+            completed_at: null,
+            suites,
+        };
+        this.writeRun(runId, run);
+        return run;
+    }
+    /** Read the results.json for a given run ID. */
+    getRun(runId) {
+        const resultsPath = this.resultsPath(runId);
+        const raw = fs.readFileSync(resultsPath, "utf-8");
+        return JSON.parse(raw);
+    }
+    /**
+     * Return the most recent run directory name, or null if no runs exist.
+     */
+    getLatestRunId() {
+        if (!fs.existsSync(this.storageDir)) {
+            return null;
+        }
+        const entries = fs.readdirSync(this.storageDir, { withFileTypes: true });
+        const runDirs = entries
+            .filter((entry) => entry.isDirectory() &&
+            fs.existsSync(path.join(this.storageDir, entry.name, "results.json")))
+            .map((entry) => entry.name)
+            .sort();
+        if (runDirs.length === 0) {
+            return null;
+        }
+        return runDirs[runDirs.length - 1];
+    }
+    /**
+     * Update a specific test case in results.json.
+     *
+     * Reads the current run, applies the partial update to the specified
+     * test case, and writes back atomically.
+     */
+    updateTestCase(runId, criterionId, testName, update) {
+        const run = this.getRun(runId);
+        const suite = run.suites[criterionId];
+        if (!suite) {
+            throw new Error(`Suite "${criterionId}" not found in run "${runId}"`);
+        }
+        const testCase = suite.tests[testName];
+        if (!testCase) {
+            throw new Error(`Test "${testName}" not found in suite "${criterionId}" of run "${runId}"`);
+        }
+        Object.assign(testCase, update);
+        this.writeRun(runId, run);
+    }
+    /** Update a suite's status. */
+    updateSuiteStatus(runId, criterionId, status) {
+        const run = this.getRun(runId);
+        const suite = run.suites[criterionId];
+        if (!suite) {
+            throw new Error(`Suite "${criterionId}" not found in run "${runId}"`);
+        }
+        suite.status = status;
+        this.writeRun(runId, run);
+    }
+    /** Mark the run as complete with a completion timestamp. */
+    completeRun(runId) {
+        const run = this.getRun(runId);
+        run.status = "complete";
+        run.completed_at = new Date().toISOString();
+        this.writeRun(runId, run);
+    }
+    // ---------------------------------------------------------------------------
+    // Private Helpers
+    // ---------------------------------------------------------------------------
+    resultsPath(runId) {
+        return path.join(this.storageDir, runId, "results.json");
+    }
+    /** Atomic write: write to .tmp then rename. */
+    writeRun(runId, run) {
+        const target = this.resultsPath(runId);
+        const tmp = target + ".tmp";
+        fs.writeFileSync(tmp, JSON.stringify(run, null, 2) + "\n");
+        fs.renameSync(tmp, target);
+    }
+}
+//# sourceMappingURL=TestStorageService.js.map

package/build/services/testing/TestStorageService.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"TestStorageService.js","sourceRoot":"","sources":["../../../src/services/testing/TestStorageService.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAQ7B,gFAAgF;AAChF,UAAU;AACV,gFAAgF;AAEhF,+EAA+E;AAC/E,SAAS,eAAe,CAAC,IAAU;IACjC,MAAM,IAAI,GAAG,CAAC,CAAS,EAAU,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IAC/D,OAAO,CACL,GAAG,IAAI,CAAC,WAAW,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE;QAC5E,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC,EAAE,CAChF,CAAC;AACJ,CAAC;AAED,gFAAgF;AAChF,qBAAqB;AACrB,gFAAgF;AAEhF;;;;;;GAMG;AACH,MAAM,OAAO,kBAAkB;IACZ,UAAU,CAAS;IAEpC,YAAY,UAAkB;QAC5B,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IAC/B,CAAC;IAED;;;;OAIG;IACH,SAAS,CAAC,SAAqB;QAC7B,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,KAAK,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC;QACnC,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;QAEjD,EAAE,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAE1C,kDAAkD;QAClD,MAAM,MAAM,GAAsB,EAAE,CAAC;QAErC,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;gBAClC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,GAAG;oBAC7B,MAAM,EAAE,SAAS;oBACjB,cAAc,EAAE,QAAQ,CAAC,aAAa;oBACtC,KAAK,EAAE,EAAE;iBACV,CAAC;YACJ,CAAC;YAED,MAAM,aAAa,GAAmB;gBACpC,MAAM,EAAE,SAAS;gBACjB,YAAY,EAAE,QAAQ,CAAC,WAAW;gBAClC,aAAa,EAAE,QAAQ,CAAC,YAAY;gBACpC,QAAQ,EAAE,IAAI;gBACd,KAAK,EAAE,IAAI;gBACX,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAE,IAAI;gBACX,cAAc,EAAE,IAAI;gBACpB,WAAW,EAAE,IAAI;aAClB,CAAC;YAEF,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,aAAa,CAAC;QACpE,CAAC;QAED,MAAM,GAAG,GAAY;YACnB,MAAM,EAAE,KAAK;YACb,MAAM,EAAE,SAAS;YACjB,UAAU,EAAE,GAAG,CAAC,WAAW,EAAE;YAC7B,YAAY,EAAE,IAAI;YAClB,MAAM;SACP,CAAC;QAEF,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAC1B,OAAO,GAAG,CAAC;IACb,CAAC;IAED,gDAAgD;IAChD,MAAM,CAAC,KAAa;QAClB,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;QAClD,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAY,CAAC;IACpC,CAAC;IAED;;OAEG;IACH,cAAc;QACZ,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;YACpC,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,OAAO,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,UAAU,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;QACzE,MAAM,OAAO,GAAG,OAAO;aACpB,MAAM,CACL,CAAC,KAAK,EAAE,EAAE,CACR,KAAK,CAAC,WAAW,EAAE;YACnB,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,KAAK,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC,CACxE;aACA,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC;aAC1B,IAAI,EAAE,CAAC;QAEV,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACrC,CAAC;IAED;;;;;OAKG;IACH,cAAc,CACZ,KAAa,EACb,WAAmB,EACnB,QAAgB,EAChB,MAA+B;QAE/B,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC/B,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;QAEtC,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,MAAM,IAAI,KAAK,CAAC,UAAU,WAAW,uBAAuB,KAAK,GAAG,CAAC,CAAC;QACxE,CAAC;QAED,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QACvC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CAAC,SAAS,QAAQ,yBAAyB,WAAW,aAAa,KAAK,GAAG,CAAC,CAAC;QAC9F,CAAC;QAED,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAChC,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IAC5B,CAAC;IAED,+BAA+B;IAC/B,iBAAiB,CAAC,KAAa,EAAE,WAAmB,EAAE,MAAuB;QAC3E,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC/B,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;QAEtC,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,MAAM,IAAI,KAAK,CAAC,UAAU,WAAW,uBAAuB,KAAK,GAAG,CAAC,CAAC;QACxE,CAAC;QAED,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC;QACtB,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IAC5B,CAAC;IAED,4DAA4D;IAC5D,WAAW,CAAC,KAAa;QACvB,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC/B,GAAG,CAAC,MAAM,GAAG,UAAU,CAAC;QACxB,GAAG,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC5C,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IAC5B,CAAC;IAED,8EAA8E;IAC9E,kBAAkB;IAClB,8EAA8E;IAEtE,WAAW,CAAC,KAAa;QAC/B,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,KAAK,EAAE,cAAc,CAAC,CAAC;IAC3D,CAAC;IAED,+CAA+C;IACvC,QAAQ,CAAC,KAAa,EAAE,GAAY;QAC1C,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QACvC,MAAM,GAAG,GAAG,MAAM,GAAG,MAAM,CAAC;QAC5B,EAAE,CAAC,aAAa,CAAC,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;QAC3D,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IAC7B,CAAC;CACF"}