claude-test-bench 1.0.0

package/dist/server/services/eval-parsers.d.ts

@@ -0,0 +1,24 @@
+import type { IndividualEvaluation, InstructionCompliance } from '../types/evaluation.js';
+export { parseSynthesisResponse, parseDebateResponse } from './eval-parsers-debate-impl.js';
+export type { Verdict, DebateParseResult } from './eval-parsers-debate-impl.js';
+export interface ScoreParseResult {
+    readonly scores: Readonly<Record<string, number>>;
+    readonly overallCloseness: number;
+    readonly missedCritical: readonly string[];
+    readonly strengths: readonly string[];
+    readonly weaknesses: readonly string[];
+    readonly summary: string;
+}
+export declare function parseScoreResponse(response: string): Partial<ScoreParseResult>;
+export interface ComplianceParseResult {
+    readonly followed: readonly string[];
+    readonly violated: readonly string[];
+    readonly notApplicable: readonly string[];
+    readonly overallCompliance: number;
+}
+export declare function parseComplianceResponse(response: string): Partial<ComplianceParseResult>;
+/** Convert parsed compliance into our InstructionCompliance type. */
+export declare function toInstructionCompliance(result: Partial<ComplianceParseResult>): InstructionCompliance;
+/** Convert score parse result into IndividualEvaluation entries. */
+export declare function toIndividualEvaluations(scores: Readonly<Record<string, number>>, role: string, reasoningMap: Readonly<Record<string, string>>): IndividualEvaluation[];
+//# sourceMappingURL=eval-parsers.d.ts.map

package/dist/server/services/eval-parsers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-parsers.d.ts","sourceRoot":"","sources":["../../../src/server/services/eval-parsers.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EACV,oBAAoB,EACpB,qBAAqB,EACtB,MAAM,wBAAwB,CAAC;AAGhC,OAAO,EAAE,sBAAsB,EAAE,mBAAmB,EAAE,MAAM,+BAA+B,CAAC;AAC5F,YAAY,EAAE,OAAO,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AAMhF,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAClD,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,cAAc,EAAE,SAAS,MAAM,EAAE,CAAC;IAC3C,QAAQ,CAAC,SAAS,EAAE,SAAS,MAAM,EAAE,CAAC;IACtC,QAAQ,CAAC,UAAU,EAAE,SAAS,MAAM,EAAE,CAAC;IACvC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B;AAED,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAa9E;AAMD,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;IACrC,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;IACrC,QAAQ,CAAC,aAAa,EAAE,SAAS,MAAM,EAAE,CAAC;IAC1C,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;CACpC;AAED,wBAAgB,uBAAuB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAMxF;AAED,qEAAqE;AACrE,wBAAgB,uBAAuB,CACrC,MAAM,EAAE,OAAO,CAAC,qBAAqB,CAAC,GACrC,qBAAqB,CAOvB;AAMD,oEAAoE;AACpE,wBAAgB,uBAAuB,CACrC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,EACxC,IAAI,EAAE,MAAM,EACZ,YAAY,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,GAC7C,oBAAoB,EAAE,CAOxB"}

package/dist/server/services/eval-parsers.js

@@ -0,0 +1,153 @@
+// ---------------------------------------------------------------------------
+// Evaluation Response Parsers — extract structured data from LLM responses
+// ---------------------------------------------------------------------------
+// Re-export debate/synthesis parsers so existing imports continue to work
+export { parseSynthesisResponse, parseDebateResponse } from './eval-parsers-debate-impl.js';
+export function parseScoreResponse(response) {
+    const parsed = tryParseJson(response);
+    if (parsed) {
+        return {
+            scores: validScores(parsed.scores),
+            overallCloseness: clamp01(parsed.overallCloseness),
+            missedCritical: toStringArray(parsed.missedCritical),
+            strengths: toStringArray(parsed.strengths),
+            weaknesses: toStringArray(parsed.weaknesses),
+            summary: typeof parsed.summary === 'string' ? parsed.summary : undefined,
+        };
+    }
+    return parseScoreFromText(response);
+}
+export function parseComplianceResponse(response) {
+    const parsed = tryParseJson(response);
+    if (parsed) {
+        return categorizeComplianceResults(parsed);
+    }
+    return parseComplianceFromText(response);
+}
+/** Convert parsed compliance into our InstructionCompliance type. */
+export function toInstructionCompliance(result) {
+    return {
+        followed: result.followed ?? [],
+        violated: result.violated ?? [],
+        notApplicable: result.notApplicable ?? [],
+        overallCompliance: result.overallCompliance ?? 0,
+    };
+}
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+/** Convert score parse result into IndividualEvaluation entries. */
+export function toIndividualEvaluations(scores, role, reasoningMap) {
+    return Object.entries(scores).map(([dimension, score]) => ({
+        evaluatorRole: role,
+        dimension,
+        score: clampScore(score),
+        reasoning: reasoningMap[dimension] ?? '',
+    }));
+}
+// ---------------------------------------------------------------------------
+// Internal: JSON parsing
+// ---------------------------------------------------------------------------
+function tryParseJson(text) {
+    try {
+        return JSON.parse(text);
+    }
+    catch {
+        // fall through
+    }
+    const jsonMatch = text.match(/```(?:json)?\s*([\s\S]*?)```/);
+    if (jsonMatch) {
+        try {
+            return JSON.parse(jsonMatch[1]);
+        }
+        catch {
+            // fall through
+        }
+    }
+    const braceMatch = text.match(/\{[\s\S]*\}/);
+    if (braceMatch) {
+        try {
+            return JSON.parse(braceMatch[0]);
+        }
+        catch {
+            // fall through
+        }
+    }
+    return undefined;
+}
+// ---------------------------------------------------------------------------
+// Internal: Text fallback parsers
+// ---------------------------------------------------------------------------
+function parseScoreFromText(text) {
+    const scores = {};
+    const scorePattern = /(\w[\w\s]*?):\s*(\d+(?:\.\d+)?)\s*(?:\/\s*10)?/g;
+    let match;
+    while ((match = scorePattern.exec(text)) !== null) {
+        const dim = match[1].trim();
+        const val = parseFloat(match[2]);
+        if (!isNaN(val) && val <= 10)
+            scores[dim] = val;
+    }
+    return { scores: Object.keys(scores).length > 0 ? scores : undefined };
+}
+function parseComplianceFromText(text) {
+    const followed = [];
+    const violated = [];
+    if (/followed|compliant/i.test(text))
+        followed.push('(extracted from text)');
+    if (/violated|non-compliant/i.test(text))
+        violated.push('(extracted from text)');
+    return { followed, violated, notApplicable: [], overallCompliance: undefined };
+}
+// ---------------------------------------------------------------------------
+// Internal: Utilities
+// ---------------------------------------------------------------------------
+function validScores(scores) {
+    if (!scores || typeof scores !== 'object')
+        return {};
+    const result = {};
+    for (const [k, v] of Object.entries(scores)) {
+        if (typeof v === 'number' && !isNaN(v))
+            result[k] = clampScore(v);
+    }
+    return result;
+}
+function clamp01(val) {
+    if (val === undefined || isNaN(val))
+        return 0;
+    return Math.max(0, Math.min(1, val));
+}
+function clampScore(val) {
+    if (val === undefined || isNaN(val))
+        return 0;
+    return Math.max(0, Math.min(10, val));
+}
+function toStringArray(arr) {
+    if (!Array.isArray(arr))
+        return [];
+    return arr.filter((x) => typeof x === 'string');
+}
+function categorizeComplianceResults(parsed) {
+    const followed = [];
+    const violated = [];
+    const notApplicable = [];
+    if (Array.isArray(parsed.results)) {
+        for (const r of parsed.results) {
+            const text = r.instruction ?? '(unknown)';
+            const status = (r.status ?? '').toLowerCase();
+            if (status === 'followed')
+                followed.push(text);
+            else if (status === 'violated')
+                violated.push(text);
+            else
+                notApplicable.push(text);
+        }
+    }
+    return {
+        followed,
+        violated,
+        notApplicable,
+        overallCompliance: clamp01(parsed.overallCompliance),
+    };
+}
+//# sourceMappingURL=eval-parsers.js.map

package/dist/server/services/eval-parsers.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-parsers.js","sourceRoot":"","sources":["../../../src/server/services/eval-parsers.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,2EAA2E;AAC3E,8EAA8E;AAO9E,0EAA0E;AAC1E,OAAO,EAAE,sBAAsB,EAAE,mBAAmB,EAAE,MAAM,+BAA+B,CAAC;AAgB5F,MAAM,UAAU,kBAAkB,CAAC,QAAgB;IACjD,MAAM,MAAM,GAAG,YAAY,CAAmB,QAAQ,CAAC,CAAC;IACxD,IAAI,MAAM,EAAE,CAAC;QACX,OAAO;YACL,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC;YAClC,gBAAgB,EAAE,OAAO,CAAC,MAAM,CAAC,gBAAgB,CAAC;YAClD,cAAc,EAAE,aAAa,CAAC,MAAM,CAAC,cAAc,CAAC;YACpD,SAAS,EAAE,aAAa,CAAC,MAAM,CAAC,SAAS,CAAC;YAC1C,UAAU,EAAE,aAAa,CAAC,MAAM,CAAC,UAAU,CAAC;YAC5C,OAAO,EAAE,OAAO,MAAM,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS;SACzE,CAAC;IACJ,CAAC;IACD,OAAO,kBAAkB,CAAC,QAAQ,CAAC,CAAC;AACtC,CAAC;AAaD,MAAM,UAAU,uBAAuB,CAAC,QAAgB;IACtD,MAAM,MAAM,GAAG,YAAY,CAAwB,QAAQ,CAAC,CAAC;IAC7D,IAAI,MAAM,EAAE,CAAC;QACX,OAAO,2BAA2B,CAAC,MAAM,CAAC,CAAC;IAC7C,CAAC;IACD,OAAO,uBAAuB,CAAC,QAAQ,CAAC,CAAC;AAC3C,CAAC;AAED,qEAAqE;AACrE,MAAM,UAAU,uBAAuB,CACrC,MAAsC;IAEtC,OAAO;QACL,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,EAAE;QAC/B,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,EAAE;QAC/B,aAAa,EAAE,MAAM,CAAC,aAAa,IAAI,EAAE;QACzC,iBAAiB,EAAE,MAAM,CAAC,iBAAiB,IAAI,CAAC;KACjD,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,UAAU;AACV,8EAA8E;AAE9E,oEAAoE;AACpE,MAAM,UAAU,uBAAuB,CACrC,MAAwC,EACxC,IAAY,EACZ,YAA8C;IAE9C,OAAO,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC;QACzD,aAAa,EAAE,IAAI;QACnB,SAAS;QACT,KAAK,EAAE,UAAU,CAAC,KAAK,CAAC;QACxB,SAAS,EAAE,YAAY,CAAC,SAAS,CAAC,IAAI,EAAE;KACzC,CAAC,CAAC,CAAC;AACN,CAAC;AAED,8EAA8E;AAC9E,yBAAyB;AACzB,8EAA8E;AAE9E,SAAS,YAAY,CAAI,IAAY;IACnC,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAM,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACP,eAAe;IACjB,CAAC;IACD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC;IAC7D,IAAI,SAAS,EAAE,CAAC;QACd,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAM,CAAC;QACvC,CAAC;QAAC,MAAM,CAAC;YACP,eAAe;QACjB,CAAC;IACH,CAAC;IACD,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IAC7C,IAAI,UAAU,EAAE,CAAC;QACf,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAM,CAAC;QACxC,CAAC;QAAC,MAAM,CAAC;YACP,eAAe;QACjB,CAAC;IACH,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,8EAA8E;AAC9E,kCAAkC;AAClC,8EAA8E;AAE9E,SAAS,kBAAkB,CAAC,IAAY;IACtC,MAAM,MAAM,GAA2B,EAAE,CAAC;IAC1C,MAAM,YAAY,GAAG,iDAAiD,CAAC;IACvE,IAAI,KAA6B,CAAC;IAClC,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QAClD,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC5B,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QACjC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,EAAE;YAAE,MAAM,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;IAClD,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC;AACzE,CAAC;AAED,SAAS,uBAAuB,CAAC,IAAY;IAC3C,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,IAAI,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,QAAQ,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;IAC7E,IAAI,yBAAyB,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,QAAQ,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;IACjF,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,aAAa,EAAE,EAAE,EAAE,iBAAiB,EAAE,SAAS,EAAE,CAAC;AACjF,CAAC;AAoBD,8EAA8E;AAC9E,sBAAsB;AACtB,8EAA8E;AAE9E,SAAS,WAAW,CAClB,MAA0C;IAE1C,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ;QAAE,OAAO,EAAE,CAAC;IACrD,MAAM,MAAM,GAA2B,EAAE,CAAC;IAC1C,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAC5C,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;YAAE,MAAM,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;IACpE,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,OAAO,CAAC,GAAuB;IACtC,IAAI,GAAG,KAAK,SAAS,IAAI,KAAK,CAAC,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC;IAC9C,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;AACvC,CAAC;AAED,SAAS,UAAU,CAAC,GAAuB;IACzC,IAAI,GAAG,KAAK,SAAS,IAAI,KAAK,CAAC,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC;IAC9C,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC;AACxC,CAAC;AAED,SAAS,aAAa,CAAC,GAAY;IACjC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC;QAAE,OAAO,EAAE,CAAC;IACnC,OAAO,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC;AAC/D,CAAC;AAED,SAAS,2BAA2B,CAClC,MAA6B;IAE7B,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,MAAM,aAAa,GAAa,EAAE,CAAC;IAEnC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;QAClC,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YAC/B,MAAM,IAAI,GAAG,CAAC,CAAC,WAAW,IAAI,WAAW,CAAC;YAC1C,MAAM,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;YAC9C,IAAI,MAAM,KAAK,UAAU;gBAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;iBAC1C,IAAI,MAAM,KAAK,UAAU;gBAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;;gBAC/C,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;IAED,OAAO;QACL,QAAQ;QACR,QAAQ;QACR,aAAa;QACb,iBAAiB,EAAE,OAAO,CAAC,MAAM,CAAC,iBAAiB,CAAC;KACrD,CAAC;AACJ,CAAC"}

package/dist/server/services/eval-prompts.d.ts

@@ -0,0 +1,9 @@
+import type { Scenario, Provider } from '../types/index.js';
+import type { IndividualEvaluation } from '../types/evaluation.js';
+import type { TranscriptSummary } from './transcript-formatter.js';
+import type { InstructionBlock } from './instruction-parser.js';
+export declare function buildScorePrompt(transcript: string, scenario: Scenario, summary: TranscriptSummary): string;
+export declare function buildCompliancePrompt(transcript: string, scenario: Scenario, instructions: readonly InstructionBlock[]): string;
+export declare function buildDebatePrompt(myPreviousAssessment: string, otherAssessments: readonly string[], roundNumber: number): string;
+export declare function buildSynthesisPrompt(allEvaluations: readonly IndividualEvaluation[], scenario: Scenario, provider: Provider): string;
+//# sourceMappingURL=eval-prompts.d.ts.map

package/dist/server/services/eval-prompts.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-prompts.d.ts","sourceRoot":"","sources":["../../../src/server/services/eval-prompts.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,QAAQ,EAAoB,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAC9E,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AACnE,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AACnE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAMhE,wBAAgB,gBAAgB,CAC9B,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,QAAQ,EAClB,OAAO,EAAE,iBAAiB,GACzB,MAAM,CA+CR;AAMD,wBAAgB,qBAAqB,CACnC,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,QAAQ,EAClB,YAAY,EAAE,SAAS,gBAAgB,EAAE,GACxC,MAAM,CAmCR;AAMD,wBAAgB,iBAAiB,CAC/B,oBAAoB,EAAE,MAAM,EAC5B,gBAAgB,EAAE,SAAS,MAAM,EAAE,EACnC,WAAW,EAAE,MAAM,GAClB,MAAM,CAyBR;AAMD,wBAAgB,oBAAoB,CAClC,cAAc,EAAE,SAAS,oBAAoB,EAAE,EAC/C,QAAQ,EAAE,QAAQ,EAClB,QAAQ,EAAE,QAAQ,GACjB,MAAM,CAkCR"}

package/dist/server/services/eval-prompts.js

@@ -0,0 +1,170 @@
+// ---------------------------------------------------------------------------
+// Evaluation Prompt Builders
+// ---------------------------------------------------------------------------
+// ---------------------------------------------------------------------------
+// Query 1: Score + answer comparison prompt
+// ---------------------------------------------------------------------------
+export function buildScorePrompt(transcript, scenario, summary) {
+    const dimensions = formatDimensions(scenario.scoringDimensions);
+    const toolSequence = summary.toolCallSequence.length > 0
+        ? `Tool call sequence: ${summary.toolCallSequence.join(' → ')}`
+        : 'No tool calls recorded.';
+    return `You are an expert evaluator assessing an AI agent's performance on a task.
+
+## Task Description
+${scenario.prompt}
+
+## Expected Answer
+${scenario.expectedAnswer}
+
+## Critical Requirements
+${formatCriticalRequirements(scenario.criticalRequirements)}
+
+## Grading Guidelines
+${scenario.gradingGuidelines || 'No specific grading guidelines provided.'}
+
+## Scoring Dimensions
+${dimensions}
+
+## Agent Behavior Summary
+${toolSequence}
+Files read: ${summary.filesRead.length > 0 ? summary.filesRead.join(', ') : 'none'}
+Files modified: ${summary.filesModified.length > 0 ? summary.filesModified.join(', ') : 'none'}
+Command failures: ${summary.commandFailures.length}
+Asked clarifying questions: ${summary.askedClarifyingQuestions ? 'yes' : 'no'}
+
+## Full Transcript
+${transcript}
+
+## Instructions
+Evaluate the agent's output. For each scoring dimension, provide a score from 0-10.
+Also assess how closely the agent's final answer matches the expected answer (0.0-1.0).
+Identify any critical requirements that were missed, as well as strengths and weaknesses.
+
+Respond with valid JSON matching this structure:
+{
+  "scores": { "<dimension_name>": <0-10>, ... },
+  "overallCloseness": <0.0-1.0>,
+  "missedCritical": ["<requirement that was not met>", ...],
+  "strengths": ["<strength>", ...],
+  "weaknesses": ["<weakness>", ...],
+  "summary": "<brief overall assessment>"
+}`;
+}
+// ---------------------------------------------------------------------------
+// Query 2: Instruction compliance prompt
+// ---------------------------------------------------------------------------
+export function buildCompliancePrompt(transcript, scenario, instructions) {
+    const instructionList = instructions
+        .map((b, i) => `${i + 1}. [${b.source}] ${b.text}`)
+        .join('\n');
+    const skillNames = scenario.skills.map((s) => s.name);
+    const subagentNames = scenario.subagents.map((s) => s.name);
+    return `You are an expert evaluator checking whether an AI agent followed its configured instructions.
+
+## Agent Configuration
+Skills: ${skillNames.length > 0 ? skillNames.join(', ') : 'none'}
+Subagents: ${subagentNames.length > 0 ? subagentNames.join(', ') : 'none'}
+
+## Instructions to Check
+${instructionList || 'No instructions configured.'}
+
+## Agent Transcript
+${transcript}
+
+## Instructions
+For each instruction listed above, determine if the agent:
+- "followed" it (clear evidence of compliance)
+- "violated" it (clear evidence of non-compliance)
+- "not_applicable" (instruction was not relevant to this task)
+
+Also rate overall compliance from 0.0 to 1.0.
+
+Respond with valid JSON:
+{
+  "results": [
+    { "instruction": "<instruction text>", "status": "followed|violated|not_applicable", "evidence": "<brief evidence>" }
+  ],
+  "overallCompliance": <0.0-1.0>
+}`;
+}
+// ---------------------------------------------------------------------------
+// Multi-round debate prompt
+// ---------------------------------------------------------------------------
+export function buildDebatePrompt(myPreviousAssessment, otherAssessments, roundNumber) {
+    const othersFormatted = otherAssessments
+        .map((a, i) => `### Other Evaluator ${i + 1}\n${a}`)
+        .join('\n\n');
+    return `You are participating in round ${roundNumber} of a multi-evaluator review.
+
+## Your Previous Assessment
+${myPreviousAssessment}
+
+## Other Evaluators' Assessments
+${othersFormatted}
+
+## Instructions
+Review the other evaluators' assessments and compare them with your own.
+Determine if you agree, partially agree, or disagree with the emerging consensus.
+Provide your updated scores if you have changed your mind, and explain why.
+
+Respond with valid JSON:
+{
+  "verdict": "AGREE|DISAGREE|PARTIAL",
+  "updatedScores": { "<dimension_name>": <0-10>, ... },
+  "critiques": ["<specific point of disagreement>", ...],
+  "reasoning": "<why you agree/disagree>"
+}`;
+}
+// ---------------------------------------------------------------------------
+// Synthesis prompt (final aggregation)
+// ---------------------------------------------------------------------------
+export function buildSynthesisPrompt(allEvaluations, scenario, provider) {
+    const evalSummaries = allEvaluations
+        .map((e) => `[${e.evaluatorRole}] ${e.dimension}: ${e.score}/10 — ${e.reasoning}`)
+        .join('\n');
+    const dimensions = formatDimensions(scenario.scoringDimensions);
+    return `You are the final synthesizer for a multi-evaluator assessment.
+
+## Scenario
+${scenario.name}: ${scenario.prompt}
+
+## Scoring Dimensions & Weights
+${dimensions}
+
+## Provider
+Name: ${provider.name}
+Model: ${provider.provider.model}
+
+## All Individual Evaluations
+${evalSummaries}
+
+## Instructions
+Synthesize all evaluations into final scores. Weight each dimension according to the scoring
+dimensions defined above. Identify areas of evaluator consensus and disagreement.
+Provide a confidence level (0.0-1.0) based on evaluator agreement.
+
+Respond with valid JSON:
+{
+  "dimensionScores": { "<dimension_name>": <0-10>, ... },
+  "weightedTotal": <weighted average 0-10>,
+  "confidence": <0.0-1.0>,
+  "dissenting": ["<areas where evaluators disagreed>", ...]
+}`;
+}
+// ---------------------------------------------------------------------------
+// Internal helpers
+// ---------------------------------------------------------------------------
+function formatDimensions(dims) {
+    if (dims.length === 0)
+        return 'No dimensions defined. Use general quality assessment.';
+    return dims
+        .map((d) => `- ${d.name} (weight: ${d.weight}): ${d.description}`)
+        .join('\n');
+}
+function formatCriticalRequirements(reqs) {
+    if (reqs.length === 0)
+        return 'None specified.';
+    return reqs.map((r, i) => `${i + 1}. ${r}`).join('\n');
+}
+//# sourceMappingURL=eval-prompts.js.map

package/dist/server/services/eval-prompts.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-prompts.js","sourceRoot":"","sources":["../../../src/server/services/eval-prompts.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,6BAA6B;AAC7B,8EAA8E;AAO9E,8EAA8E;AAC9E,4CAA4C;AAC5C,8EAA8E;AAE9E,MAAM,UAAU,gBAAgB,CAC9B,UAAkB,EAClB,QAAkB,EAClB,OAA0B;IAE1B,MAAM,UAAU,GAAG,gBAAgB,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC;IAChE,MAAM,YAAY,GAAG,OAAO,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC;QACtD,CAAC,CAAC,uBAAuB,OAAO,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;QAC/D,CAAC,CAAC,yBAAyB,CAAC;IAE9B,OAAO;;;EAGP,QAAQ,CAAC,MAAM;;;EAGf,QAAQ,CAAC,cAAc;;;EAGvB,0BAA0B,CAAC,QAAQ,CAAC,oBAAoB,CAAC;;;EAGzD,QAAQ,CAAC,iBAAiB,IAAI,0CAA0C;;;EAGxE,UAAU;;;EAGV,YAAY;cACA,OAAO,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM;kBAChE,OAAO,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM;oBAC1E,OAAO,CAAC,eAAe,CAAC,MAAM;8BACpB,OAAO,CAAC,wBAAwB,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI;;;EAG3E,UAAU;;;;;;;;;;;;;;;EAeV,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,yCAAyC;AACzC,8EAA8E;AAE9E,MAAM,UAAU,qBAAqB,CACnC,UAAkB,EAClB,QAAkB,EAClB,YAAyC;IAEzC,MAAM,eAAe,GAAG,YAAY;SACjC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;SAClD,IAAI,CAAC,IAAI,CAAC,CAAC;IAEd,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACtD,MAAM,aAAa,GAAG,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAE5D,OAAO;;;UAGC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM;aACnD,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM;;;EAGvE,eAAe,IAAI,6BAA6B;;;EAGhD,UAAU;;;;;;;;;;;;;;;;EAgBV,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,4BAA4B;AAC5B,8EAA8E;AAE9E,MAAM,UAAU,iBAAiB,CAC/B,oBAA4B,EAC5B,gBAAmC,EACnC,WAAmB;IAEnB,MAAM,eAAe,GAAG,gBAAgB;SACrC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,uBAAuB,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;SACnD,IAAI,CAAC,MAAM,CAAC,CAAC;IAEhB,OAAO,kCAAkC,WAAW;;;EAGpD,oBAAoB;;;EAGpB,eAAe;;;;;;;;;;;;;EAaf,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,uCAAuC;AACvC,8EAA8E;AAE9E,MAAM,UAAU,oBAAoB,CAClC,cAA+C,EAC/C,QAAkB,EAClB,QAAkB;IAElB,MAAM,aAAa,GAAG,cAAc;SACjC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,aAAa,KAAK,CAAC,CAAC,SAAS,KAAK,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC,SAAS,EAAE,CAAC;SACjF,IAAI,CAAC,IAAI,CAAC,CAAC;IAEd,MAAM,UAAU,GAAG,gBAAgB,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC;IAEhE,OAAO;;;EAGP,QAAQ,CAAC,IAAI,KAAK,QAAQ,CAAC,MAAM;;;EAGjC,UAAU;;;QAGJ,QAAQ,CAAC,IAAI;SACZ,QAAQ,CAAC,QAAQ,CAAC,KAAK;;;EAG9B,aAAa;;;;;;;;;;;;;EAab,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,mBAAmB;AACnB,8EAA8E;AAE9E,SAAS,gBAAgB,CAAC,IAAiC;IACzD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,wDAAwD,CAAC;IACvF,OAAO,IAAI;SACR,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,aAAa,CAAC,CAAC,MAAM,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;SACjE,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED,SAAS,0BAA0B,CAAC,IAAuB;IACzD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,iBAAiB,CAAC;IAChD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACzD,CAAC"}

package/dist/server/services/evaluator.d.ts

@@ -0,0 +1,10 @@
+import type { IEvaluator, EvaluationCallbacks } from '../interfaces/evaluator.js';
+import type { Run, Scenario, Provider, Evaluation, EvaluationRequest } from '../types/index.js';
+export declare class EvaluationOrchestrator implements IEvaluator {
+    evaluateRun(run: Run, scenario: Scenario, provider: Provider, request: EvaluationRequest, callbacks: EvaluationCallbacks): Promise<Evaluation>;
+    private runRound1;
+    private runDebateRound;
+    private runSynthesis;
+    private runQuery;
+}
+//# sourceMappingURL=evaluator.d.ts.map

package/dist/server/services/evaluator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluator.d.ts","sourceRoot":"","sources":["../../../src/server/services/evaluator.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,UAAU,EAAE,mBAAmB,EAAmB,MAAM,4BAA4B,CAAC;AACnG,OAAO,KAAK,EACV,GAAG,EACH,QAAQ,EACR,QAAQ,EACR,UAAU,EACV,iBAAiB,EAMlB,MAAM,mBAAmB,CAAC;AAyC3B,qBAAa,sBAAuB,YAAW,UAAU;IACjD,WAAW,CACf,GAAG,EAAE,GAAG,EACR,QAAQ,EAAE,QAAQ,EAClB,QAAQ,EAAE,QAAQ,EAClB,OAAO,EAAE,iBAAiB,EAC1B,SAAS,EAAE,mBAAmB,GAC7B,OAAO,CAAC,UAAU,CAAC;YAyER,SAAS;YA6BT,cAAc;YAyBd,YAAY;YAiBZ,QAAQ;CA4CvB"}

package/dist/server/services/evaluator.js

@@ -0,0 +1,156 @@
+// ---------------------------------------------------------------------------
+// Evaluator — orchestrates evaluation pipeline via SDK query()
+// ---------------------------------------------------------------------------
+import { query } from '@anthropic-ai/claude-agent-sdk';
+import { buildRunEnv } from './env-builder.js';
+import { formatTranscript } from './transcript-formatter.js';
+import { parseAllInstructions } from './instruction-parser.js';
+import { buildScorePrompt, buildCompliancePrompt, buildDebatePrompt, buildSynthesisPrompt, } from './eval-prompts.js';
+import { parseScoreResponse, parseComplianceResponse, parseSynthesisResponse, parseDebateResponse, toIndividualEvaluations, } from './eval-parsers.js';
+import { checkConsensus, buildAnswerComparison, buildCriticalResults, mergeCompliance, } from './eval-helpers.js';
+// ---------------------------------------------------------------------------
+// Implementation
+// ---------------------------------------------------------------------------
+export class EvaluationOrchestrator {
+    async evaluateRun(run, scenario, provider, request, callbacks) {
+        callbacks.onStatusChange('running');
+        callbacks.onProgress('preparing', 'Formatting transcript and parsing instructions...');
+        const { text: transcript, summary } = formatTranscript(run.messages);
+        const instructions = parseAllInstructions([
+            ...scenario.claudeMdFiles.map((c) => ({ content: c.content, source: `CLAUDE.md (${c.role})` })),
+            ...scenario.rules.map((r) => ({ content: r.content, source: `rule:${r.name}` })),
+        ]);
+        const accumulators = request.evaluators.map((e) => ({
+            role: e.role, costUsd: 0, tokensIn: 0, tokensOut: 0, rounds: 0,
+            scoreResult: {}, complianceResult: {}, assessmentText: '',
+        }));
+        // ── Round 1 ──────────────────────────────────────────────────────
+        callbacks.onProgress('scoring', `Running score and compliance queries (${request.evaluators.length} evaluator${request.evaluators.length > 1 ? 's' : ''})...`);
+        const round1Evals = await this.runRound1(request.evaluators, accumulators, transcript, scenario, provider, instructions, summary, callbacks);
+        const round1Consensus = request.maxRounds <= 1 || checkConsensus(accumulators);
+        const rounds = [{
+                roundNumber: 1, evaluations: round1Evals,
+                consensusReached: round1Consensus, timestamp: new Date().toISOString(),
+            }];
+        // ── Multi-round debate ───────────────────────────────────────────
+        if (request.maxRounds > 1 && !round1Consensus) {
+            for (let roundNum = 2; roundNum <= request.maxRounds; roundNum++) {
+                callbacks.onProgress('debate', `Debate round ${roundNum} of ${request.maxRounds}...`);
+                const debateEvals = await this.runDebateRound(request.evaluators, accumulators, roundNum, callbacks);
+                const consensus = checkConsensus(accumulators);
+                rounds.push({
+                    roundNumber: roundNum, evaluations: debateEvals,
+                    consensusReached: consensus, timestamp: new Date().toISOString(),
+                });
+                if (consensus)
+                    break;
+            }
+        }
+        // ── Synthesis ────────────────────────────────────────────────────
+        callbacks.onProgress('synthesis', 'Synthesizing final scores and confidence...');
+        const latestRound = rounds[rounds.length - 1];
+        const latestEvals = latestRound.evaluations;
+        const synthesizer = request.evaluators[request.evaluators.length - 1];
+        const synthInfo = { phase: 'synthesis', evaluatorRole: synthesizer.role, roundNumber: rounds.length };
+        const synthResult = await this.runSynthesis(synthesizer, latestEvals, scenario, provider, accumulators, callbacks, synthInfo);
+        // ── Assemble final evaluation ────────────────────────────────────
+        const ledger = accumulators.map((a) => ({
+            evaluatorRole: a.role, totalCostUsd: a.costUsd,
+            totalTokensIn: a.tokensIn, totalTokensOut: a.tokensOut, roundsParticipated: a.rounds,
+        }));
+        const now = new Date().toISOString();
+        callbacks.onProgress('complete', 'Evaluation finished.');
+        callbacks.onStatusChange('completed');
+        return {
+            id: '', runId: run.id, status: 'completed', evaluators: request.evaluators, rounds,
+            answerComparison: buildAnswerComparison(accumulators),
+            criticalResults: buildCriticalResults(accumulators, scenario),
+            setupCompliance: { instructionCompliance: mergeCompliance(accumulators), skillUsage: [], subagentUsage: [] },
+            synthesis: {
+                dimensionScores: synthResult.dimensionScores ?? {}, weightedTotal: synthResult.weightedTotal ?? 0,
+                confidence: synthResult.confidence ?? 0, dissenting: synthResult.dissenting ?? [],
+            },
+            ledger, totalCostUsd: ledger.reduce((sum, l) => sum + l.totalCostUsd, 0),
+            createdAt: now, updatedAt: now,
+        };
+    }
+    // ─── Round 1: Score + Compliance ─────────────────────────────────────
+    async runRound1(evaluators, accumulators, transcript, scenario, provider, instructions, summary, callbacks) {
+        const promises = evaluators.map(async (evaluator, idx) => {
+            const acc = accumulators[idx];
+            acc.rounds++;
+            const scoreInfo = { phase: 'score', evaluatorRole: evaluator.role, roundNumber: 1 };
+            const scoreResp = await this.runQuery(evaluator, buildScorePrompt(transcript, scenario, summary), callbacks, scoreInfo);
+            acc.costUsd += scoreResp.costUsd;
+            acc.scoreResult = parseScoreResponse(scoreResp.text);
+            acc.assessmentText = scoreResp.text;
+            const compInfo = { phase: 'compliance', evaluatorRole: evaluator.role, roundNumber: 1 };
+            const compResp = await this.runQuery(evaluator, buildCompliancePrompt(transcript, scenario, instructions), callbacks, compInfo);
+            acc.costUsd += compResp.costUsd;
+            acc.complianceResult = parseComplianceResponse(compResp.text);
+            return toIndividualEvaluations(acc.scoreResult.scores ?? {}, evaluator.role, {});
+        });
+        return (await Promise.all(promises)).flat();
+    }
+    // ─── Multi-round debate ──────────────────────────────────────────────
+    async runDebateRound(evaluators, accumulators, roundNumber, callbacks) {
+        const promises = evaluators.map(async (evaluator, idx) => {
+            const acc = accumulators[idx];
+            acc.rounds++;
+            const others = accumulators.filter((_, i) => i !== idx).map((a) => a.assessmentText);
+            const info = { phase: 'debate', evaluatorRole: evaluator.role, roundNumber };
+            const resp = await this.runQuery(evaluator, buildDebatePrompt(acc.assessmentText, others, roundNumber), callbacks, info);
+            acc.costUsd += resp.costUsd;
+            const result = parseDebateResponse(resp.text);
+            if (result.updatedScores && Object.keys(result.updatedScores).length > 0) {
+                acc.scoreResult = { ...acc.scoreResult, scores: result.updatedScores };
+            }
+            acc.assessmentText = resp.text;
+            return toIndividualEvaluations(acc.scoreResult.scores ?? {}, evaluator.role, {});
+        });
+        return (await Promise.all(promises)).flat();
+    }
+    // ─── Synthesis ───────────────────────────────────────────────────────
+    async runSynthesis(synthesizer, allEvals, scenario, provider, accumulators, callbacks, info) {
+        const resp = await this.runQuery(synthesizer, buildSynthesisPrompt(allEvals, scenario, provider), callbacks, info);
+        const acc = accumulators.find((a) => a.role === synthesizer.role);
+        if (acc)
+            acc.costUsd += resp.costUsd;
+        return parseSynthesisResponse(resp.text);
+    }
+    // ─── SDK query wrapper ───────────────────────────────────────────────
+    async runQuery(evaluator, prompt, callbacks, info) {
+        const q = query({
+            prompt,
+            options: {
+                env: buildRunEnv(evaluator.provider),
+                model: evaluator.provider.model,
+                tools: [],
+                maxTurns: 3,
+                permissionMode: 'dontAsk',
+                persistSession: false,
+            },
+        });
+        let resultText = '';
+        let costUsd = 0;
+        for await (const msg of q) {
+            const record = msg;
+            const sdkRecord = {
+                timestamp: new Date().toISOString(),
+                message: record,
+            };
+            callbacks.onMessage(info, sdkRecord);
+            if (record['type'] === 'result') {
+                const resultMsg = record;
+                costUsd = resultMsg.total_cost_usd ?? 0;
+                if (resultMsg.subtype !== 'success') {
+                    const errorDetail = resultMsg.result ?? 'unknown error';
+                    throw new Error(`SDK query failed (subtype: ${resultMsg.subtype}): ${errorDetail}`);
+                }
+                resultText = resultMsg.result ?? '';
+            }
+        }
+        return { text: resultText, costUsd };
+    }
+}
+//# sourceMappingURL=evaluator.js.map

package/dist/server/services/evaluator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluator.js","sourceRoot":"","sources":["../../../src/server/services/evaluator.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,+DAA+D;AAC/D,8EAA8E;AAE9E,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AAcvD,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAC;AAC7D,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EACL,gBAAgB,EAChB,qBAAqB,EACrB,iBAAiB,EACjB,oBAAoB,GACrB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EACL,kBAAkB,EAClB,uBAAuB,EACvB,sBAAsB,EACtB,mBAAmB,EACnB,uBAAuB,GACxB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EACL,cAAc,EACd,qBAAqB,EACrB,oBAAoB,EACpB,eAAe,GAChB,MAAM,mBAAmB,CAAC;AAe3B,8EAA8E;AAC9E,iBAAiB;AACjB,8EAA8E;AAE9E,MAAM,OAAO,sBAAsB;IACjC,KAAK,CAAC,WAAW,CACf,GAAQ,EACR,QAAkB,EAClB,QAAkB,EAClB,OAA0B,EAC1B,SAA8B;QAE9B,SAAS,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;QACpC,SAAS,CAAC,UAAU,CAAC,WAAW,EAAE,mDAAmD,CAAC,CAAC;QAEvF,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,gBAAgB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACrE,MAAM,YAAY,GAAG,oBAAoB,CAAC;YACxC,GAAG,QAAQ,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,cAAc,CAAC,CAAC,IAAI,GAAG,EAAE,CAAC,CAAC;YAC/F,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;SACjF,CAAC,CAAC;QAEH,MAAM,YAAY,GAA2B,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC1E,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,OAAO,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC;YAC9D,WAAW,EAAE,EAAE,EAAE,gBAAgB,EAAE,EAAE,EAAE,cAAc,EAAE,EAAE;SAC1D,CAAC,CAAC,CAAC;QAEJ,oEAAoE;QACpE,SAAS,CAAC,UAAU,CAAC,SAAS,EAAE,yCAAyC,OAAO,CAAC,UAAU,CAAC,MAAM,aAAa,OAAO,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;QAC/J,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,SAAS,CACtC,OAAO,CAAC,UAAU,EAAE,YAAY,EAAE,UAAU,EAAE,QAAQ,EAAE,QAAQ,EAAE,YAAY,EAAE,OAAO,EAAE,SAAS,CACnG,CAAC;QACF,MAAM,eAAe,GAAG,OAAO,CAAC,SAAS,IAAI,CAAC,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;QAC/E,MAAM,MAAM,GAAsB,CAAC;gBACjC,WAAW,EAAE,CAAC,EAAE,WAAW,EAAE,WAAW;gBACxC,gBAAgB,EAAE,eAAe,EAAE,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;aACvE,CAAC,CAAC;QAEH,oEAAoE;QACpE,IAAI,OAAO,CAAC,SAAS,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;YAC9C,KAAK,IAAI,QAAQ,GAAG,CAAC,EAAE,QAAQ,IAAI,OAAO,CAAC,SAAS,EAAE,QAAQ,EAAE,EAAE,CAAC;gBACjE,SAAS,CAAC,UAAU,CAAC,QAAQ,EAAE,gBAAgB,QAAQ,OAAO,OAAO,CAAC,SAAS,KAAK,CAAC,CAAC;gBACtF,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,UAAU,EAAE,YAAY,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC;gBACrG,MAAM,SAAS,GAAG,cAAc,CAAC,YAAY,CAAC,CAAC;gBAC/C,MAAM,CAAC,IAAI,CAAC;oBACV,WAAW,EAAE,QAAQ,EAAE,WAAW,EAAE,WAAW;oBAC/C,gBAAgB,EAAE,SAAS,EAAE,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;iBACjE,CAAC,CAAC;gBACH,IAAI,SAAS;oBAAE,MAAM;YACvB,CAAC;QACH,CAAC;QAED,oEAAoE;QACpE,SAAS,CAAC,UAAU,CAAC,WAAW,EAAE,6CAA6C,CAAC,CAAC;QACjF,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC9C,MAAM,WAAW,GAAG,WAAW,CAAC,WAAW,CAAC;QAC5C,MAAM,WAAW,GAAG,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACtE,MAAM,SAAS,GAAoB,EAAE,KAAK,EAAE,WAAW,EAAE,aAAa,EAAE,WAAW,CAAC,IAAI,EAAE,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;QACvH,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,WAAW,EAAE,WAAW,EAAE,QAAQ,EAAE,QAAQ,EAAE,YAAY,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;QAE9H,oEAAoE;QACpE,MAAM,MAAM,GAAsB,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACzD,aAAa,EAAE,CAAC,CAAC,IAAI,EAAE,YAAY,EAAE,CAAC,CAAC,OAAO;YAC9C,aAAa,EAAE,CAAC,CAAC,QAAQ,EAAE,cAAc,EAAE,CAAC,CAAC,SAAS,EAAE,kBAAkB,EAAE,CAAC,CAAC,MAAM;SACrF,CAAC,CAAC,CAAC;QACJ,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACrC,SAAS,CAAC,UAAU,CAAC,UAAU,EAAE,sBAAsB,CAAC,CAAC;QACzD,SAAS,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;QAEtC,OAAO;YACL,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,MAAM;YAClF,gBAAgB,EAAE,qBAAqB,CAAC,YAAY,CAAC;YACrD,eAAe,EAAE,oBAAoB,CAAC,YAAY,EAAE,QAAQ,CAAC;YAC7D,eAAe,EAAE,EAAE,qBAAqB,EAAE,eAAe,CAAC,YAAY,CAAC,EAAE,UAAU,EAAE,EAAE,EAAE,aAAa,EAAE,EAAE,EAAE;YAC5G,SAAS,EAAE;gBACT,eAAe,EAAE,WAAW,CAAC,eAAe,IAAI,EAAE,EAAE,aAAa,EAAE,WAAW,CAAC,aAAa,IAAI,CAAC;gBACjG,UAAU,EAAE,WAAW,CAAC,UAAU,IAAI,CAAC,EAAE,UAAU,EAAE,WAAW,CAAC,UAAU,IAAI,EAAE;aAClF;YACD,MAAM,EAAE,YAAY,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC;YACxE,SAAS,EAAE,GAAG,EAAE,SAAS,EAAE,GAAG;SAC/B,CAAC;IACJ,CAAC;IAED,wEAAwE;IAEhE,KAAK,CAAC,SAAS,CACrB,UAAsC,EACtC,YAAoC,EACpC,UAAkB,EAClB,QAAkB,EAClB,QAAkB,EAClB,YAAqD,EACrD,OAAuD,EACvD,SAA8B;QAE9B,MAAM,QAAQ,GAAG,UAAU,CAAC,GAAG,CAAC,KAAK,EAAE,SAAS,EAAE,GAAG,EAAE,EAAE;YACvD,MAAM,GAAG,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;YAC9B,GAAG,CAAC,MAAM,EAAE,CAAC;YACb,MAAM,SAAS,GAAoB,EAAE,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,SAAS,CAAC,IAAI,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC;YACrG,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,gBAAgB,CAAC,UAAU,EAAE,QAAQ,EAAE,OAAO,CAAC,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;YACxH,GAAG,CAAC,OAAO,IAAI,SAAS,CAAC,OAAO,CAAC;YACjC,GAAG,CAAC,WAAW,GAAG,kBAAkB,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YACrD,GAAG,CAAC,cAAc,GAAG,SAAS,CAAC,IAAI,CAAC;YACpC,MAAM,QAAQ,GAAoB,EAAE,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,SAAS,CAAC,IAAI,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC;YACzG,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,qBAAqB,CAAC,UAAU,EAAE,QAAQ,EAAE,YAAY,CAAC,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;YAChI,GAAG,CAAC,OAAO,IAAI,QAAQ,CAAC,OAAO,CAAC;YAChC,GAAG,CAAC,gBAAgB,GAAG,uBAAuB,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YAC9D,OAAO,uBAAuB,CAAC,GAAG,CAAC,WAAW,CAAC,MAAM,IAAI,EAAE,EAAE,SAAS,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QACnF,CAAC,CAAC,CAAC;QACH,OAAO,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC9C,CAAC;IAED,wEAAwE;IAEhE,KAAK,CAAC,cAAc,CAC1B,UAAsC,EACtC,YAAoC,EACpC,WAAmB,EACnB,SAA8B;QAE9B,MAAM,QAAQ,GAAG,UAAU,CAAC,GAAG,CAAC,KAAK,EAAE,SAAS,EAAE,GAAG,EAAE,EAAE;YACvD,MAAM,GAAG,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;YAC9B,GAAG,CAAC,MAAM,EAAE,CAAC;YACb,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC;YACrF,MAAM,IAAI,GAAoB,EAAE,KAAK,EAAE,QAAQ,EAAE,aAAa,EAAE,SAAS,CAAC,IAAI,EAAE,WAAW,EAAE,CAAC;YAC9F,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,iBAAiB,CAAC,GAAG,CAAC,cAAc,EAAE,MAAM,EAAE,WAAW,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;YACzH,GAAG,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,CAAC;YAC5B,MAAM,MAAM,GAAG,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC9C,IAAI,MAAM,CAAC,aAAa,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACzE,GAAG,CAAC,WAAW,GAAG,EAAE,GAAG,GAAG,CAAC,WAAW,EAAE,MAAM,EAAE,MAAM,CAAC,aAAa,EAAE,CAAC;YACzE,CAAC;YACD,GAAG,CAAC,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC;YAC/B,OAAO,uBAAuB,CAAC,GAAG,CAAC,WAAW,CAAC,MAAM,IAAI,EAAE,EAAE,SAAS,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QACnF,CAAC,CAAC,CAAC;QACH,OAAO,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC9C,CAAC;IAED,wEAAwE;IAEhE,KAAK,CAAC,YAAY,CACxB,WAA4B,EAC5B,QAAyC,EACzC,QAAkB,EAClB,QAAkB,EAClB,YAAoC,EACpC,SAA8B,EAC9B,IAAqB;QAErB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,oBAAoB,CAAC,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;QACnH,MAAM,GAAG,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC,IAAI,CAAC,CAAC;QAClE,IAAI,GAAG;YAAE,GAAG,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,CAAC;QACrC,OAAO,sBAAsB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC3C,CAAC;IAED,wEAAwE;IAEhE,KAAK,CAAC,QAAQ,CACpB,SAA0B,EAC1B,MAAc,EACd,SAA8B,EAC9B,IAAqB;QAErB,MAAM,CAAC,GAAG,KAAK,CAAC;YACd,MAAM;YACN,OAAO,EAAE;gBACP,GAAG,EAAE,WAAW,CAAC,SAAS,CAAC,QAAQ,CAAC;gBACpC,KAAK,EAAE,SAAS,CAAC,QAAQ,CAAC,KAAK;gBAC/B,KAAK,EAAE,EAAE;gBACT,QAAQ,EAAE,CAAC;gBACX,cAAc,EAAE,SAAS;gBACzB,cAAc,EAAE,KAAK;aACtB;SACF,CAAC,CAAC;QAEH,IAAI,UAAU,GAAG,EAAE,CAAC;QACpB,IAAI,OAAO,GAAG,CAAC,CAAC;QAEhB,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC;YAC1B,MAAM,MAAM,GAAG,GAAyC,CAAC;YACzD,MAAM,SAAS,GAAqB;gBAClC,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,OAAO,EAAE,MAAM;aAChB,CAAC;YACF,SAAS,CAAC,SAAS,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;YAErC,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,QAAQ,EAAE,CAAC;gBAChC,MAAM,SAAS,GAAG,MAAuC,CAAC;gBAC1D,OAAO,GAAG,SAAS,CAAC,cAAc,IAAI,CAAC,CAAC;gBACxC,IAAI,SAAS,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;oBACpC,MAAM,WAAW,GAAG,SAAS,CAAC,MAAM,IAAI,eAAe,CAAC;oBACxD,MAAM,IAAI,KAAK,CACb,8BAA8B,SAAS,CAAC,OAAO,MAAM,WAAW,EAAE,CACnE,CAAC;gBACJ,CAAC;gBACD,UAAU,GAAG,SAAS,CAAC,MAAM,IAAI,EAAE,CAAC;YACtC,CAAC;QACH,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,CAAC;IACvC,CAAC;CACF"}

package/dist/server/services/fs-adapter.d.ts

@@ -0,0 +1,20 @@
+/** File system abstraction for testing */
+export interface FsAdapter {
+    mkdir(dirPath: string, opts: {
+        recursive: boolean;
+    }): Promise<void>;
+    writeFile(filePath: string, data: string, opts?: {
+        mode?: number;
+    }): Promise<void>;
+    readFile(filePath: string, encoding: BufferEncoding): Promise<string>;
+    readdir(dirPath: string): Promise<string[]>;
+    unlink(filePath: string): Promise<void>;
+    rename(oldPath: string, newPath: string): Promise<void>;
+    stat(filePath: string): Promise<{
+        isFile(): boolean;
+    }>;
+    access(filePath: string): Promise<void>;
+}
+/** Default adapter that delegates to node:fs/promises */
+export declare const defaultFs: FsAdapter;
+//# sourceMappingURL=fs-adapter.d.ts.map

package/dist/server/services/fs-adapter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fs-adapter.d.ts","sourceRoot":"","sources":["../../../src/server/services/fs-adapter.ts"],"names":[],"mappings":"AAEA,0CAA0C;AAC1C,MAAM,WAAW,SAAS;IACxB,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE;QAAE,SAAS,EAAE,OAAO,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACpE,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACnF,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,cAAc,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACtE,OAAO,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAC5C,MAAM,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACxC,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACxD,IAAI,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,MAAM,IAAI,OAAO,CAAA;KAAE,CAAC,CAAC;IACvD,MAAM,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACzC;AAED,yDAAyD;AACzD,eAAO,MAAM,SAAS,EAAE,SAUvB,CAAC"}

package/dist/server/services/fs-adapter.js

@@ -0,0 +1,13 @@
+import fs from 'node:fs/promises';
+/** Default adapter that delegates to node:fs/promises */
+export const defaultFs = {
+    mkdir: (dirPath, opts) => fs.mkdir(dirPath, opts).then(() => undefined),
+    writeFile: (filePath, data, opts) => fs.writeFile(filePath, data, opts).then(() => undefined),
+    readFile: (filePath, encoding) => fs.readFile(filePath, encoding),
+    readdir: (dirPath) => fs.readdir(dirPath).then((entries) => entries.map(String)),
+    unlink: (filePath) => fs.unlink(filePath),
+    rename: (oldPath, newPath) => fs.rename(oldPath, newPath),
+    stat: (filePath) => fs.stat(filePath),
+    access: (filePath) => fs.access(filePath),
+};
+//# sourceMappingURL=fs-adapter.js.map

package/dist/server/services/fs-adapter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fs-adapter.js","sourceRoot":"","sources":["../../../src/server/services/fs-adapter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAclC,yDAAyD;AACzD,MAAM,CAAC,MAAM,SAAS,GAAc;IAClC,KAAK,EAAE,CAAC,OAAO,EAAE,IAAI,EAAE,EAAE,CAAC,EAAE,CAAC,KAAK,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC;IACvE,SAAS,EAAE,CAAC,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,CAClC,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC;IAC1D,QAAQ,EAAE,CAAC,QAAQ,EAAE,QAAQ,EAAE,EAAE,CAAC,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC;IACjE,OAAO,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAChF,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC;IACzC,MAAM,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC,EAAE,CAAC,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC;IACzD,IAAI,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC;IACrC,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC;CAC1C,CAAC"}