model-test-bench 1.0.2

package/dist/server/services/eval-helpers.js

@@ -0,0 +1,75 @@
+// ---------------------------------------------------------------------------
+// Evaluation Helpers — result aggregation extracted from the orchestrator
+// ---------------------------------------------------------------------------
+import { toInstructionCompliance } from './eval-parsers.js';
+/** Check whether evaluator scores have converged (within 1 point per dimension). */
+export function checkConsensus(accumulators) {
+    if (accumulators.length < 2)
+        return true;
+    // Require ALL evaluators to have parsed scores before declaring consensus.
+    // If any evaluator's score result is missing, consensus is false.
+    const allScores = accumulators.map((a) => a.scoreResult.scores ?? {});
+    if (allScores.some((s) => Object.keys(s).length === 0))
+        return false;
+    const dimensions = new Set(allScores.flatMap((s) => Object.keys(s)));
+    for (const dim of dimensions) {
+        const vals = allScores.map((s) => s[dim]).filter((v) => v !== undefined);
+        if (vals.length < 2)
+            continue;
+        const range = Math.max(...vals) - Math.min(...vals);
+        if (range > 1)
+            return false;
+    }
+    return true;
+}
+/** Aggregate answer closeness from all evaluators into a comparison. */
+export function buildAnswerComparison(accumulators) {
+    const closenessValues = accumulators
+        .map((a) => a.scoreResult.overallCloseness)
+        .filter((v) => v !== undefined && v > 0);
+    const avgCloseness = closenessValues.length > 0
+        ? closenessValues.reduce((a, b) => a + b, 0) / closenessValues.length
+        : 0;
+    const summaries = accumulators
+        .map((a) => a.scoreResult.summary)
+        .filter((s) => !!s);
+    return {
+        matches: avgCloseness >= 0.7,
+        explanation: summaries[0] ?? 'No explanation available',
+        similarity: avgCloseness,
+    };
+}
+/** Build critical requirement results by checking evaluator-flagged misses. */
+export function buildCriticalResults(accumulators, scenario) {
+    const allMissed = new Set(accumulators.flatMap((a) => a.scoreResult.missedCritical ?? []));
+    return scenario.criticalRequirements.map((req) => ({
+        requirement: req,
+        met: !allMissed.has(req),
+        evidence: allMissed.has(req) ? 'Flagged as missed by evaluator' : 'Not flagged',
+    }));
+}
+/** Merge compliance results from all evaluators into a single report. */
+export function mergeCompliance(accumulators) {
+    const followed = new Set();
+    const violated = new Set();
+    const notApplicable = new Set();
+    let complianceSum = 0;
+    let complianceCount = 0;
+    for (const acc of accumulators) {
+        const c = acc.complianceResult;
+        (c.followed ?? []).forEach((s) => followed.add(s));
+        (c.violated ?? []).forEach((s) => violated.add(s));
+        (c.notApplicable ?? []).forEach((s) => notApplicable.add(s));
+        if (c.overallCompliance !== undefined) {
+            complianceSum += c.overallCompliance;
+            complianceCount++;
+        }
+    }
+    return toInstructionCompliance({
+        followed: [...followed],
+        violated: [...violated],
+        notApplicable: [...notApplicable],
+        overallCompliance: complianceCount > 0 ? complianceSum / complianceCount : 0,
+    });
+}
+//# sourceMappingURL=eval-helpers.js.map

package/dist/server/services/eval-helpers.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-helpers.js","sourceRoot":"","sources":["../../../src/server/services/eval-helpers.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,0EAA0E;AAC1E,8EAA8E;AAQ9E,OAAO,EAAE,uBAAuB,EAAE,MAAM,mBAAmB,CAAC;AAe5D,oFAAoF;AACpF,MAAM,UAAU,cAAc,CAAC,YAA6C;IAC1E,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC;IACzC,2EAA2E;IAC3E,kEAAkE;IAClE,MAAM,SAAS,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;IACtE,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC;QAAE,OAAO,KAAK,CAAC;IAErE,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACrE,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;QAC7B,MAAM,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC;QACtF,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;YAAE,SAAS;QAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC;QACpD,IAAI,KAAK,GAAG,CAAC;YAAE,OAAO,KAAK,CAAC;IAC9B,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,wEAAwE;AACxE,MAAM,UAAU,qBAAqB,CACnC,YAA6C;IAE7C,MAAM,eAAe,GAAG,YAAY;SACjC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,gBAAgB,CAAC;SAC1C,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,CAAC,KAAK,SAAS,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IACxD,MAAM,YAAY,GAAG,eAAe,CAAC,MAAM,GAAG,CAAC;QAC7C,CAAC,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,eAAe,CAAC,MAAM;QACrE,CAAC,CAAC,CAAC,CAAC;IACN,MAAM,SAAS,GAAG,YAAY;SAC3B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,OAAO,CAAC;SACjC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAEnC,OAAO;QACL,OAAO,EAAE,YAAY,IAAI,GAAG;QAC5B,WAAW,EAAE,SAAS,CAAC,CAAC,CAAC,IAAI,0BAA0B;QACvD,UAAU,EAAE,YAAY;KACzB,CAAC;AACJ,CAAC;AAED,+EAA+E;AAC/E,MAAM,UAAU,oBAAoB,CAClC,YAA6C,EAC7C,QAAkB;IAElB,MAAM,SAAS,GAAG,IAAI,GAAG,CACvB,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,cAAc,IAAI,EAAE,CAAC,CAChE,CAAC;IACF,OAAO,QAAQ,CAAC,oBAAoB,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACjD,WAAW,EAAE,GAAG;QAChB,GAAG,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC;QACxB,QAAQ,EAAE,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,gCAAgC,CAAC,CAAC,CAAC,aAAa;KAChF,CAAC,CAAC,CAAC;AACN,CAAC;AAED,yEAAyE;AACzE,MAAM,UAAU,eAAe,CAC7B,YAA6C;IAE7C,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IACnC,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IACnC,MAAM,aAAa,GAAG,IAAI,GAAG,EAAU,CAAC;IACxC,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,eAAe,GAAG,CAAC,CAAC;IAExB,KAAK,MAAM,GAAG,IAAI,YAAY,EAAE,CAAC;QAC/B,MAAM,CAAC,GAAG,GAAG,CAAC,gBAAgB,CAAC;QAC/B,CAAC,CAAC,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACnD,CAAC,CAAC,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACnD,CAAC,CAAC,CAAC,aAAa,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7D,IAAI,CAAC,CAAC,iBAAiB,KAAK,SAAS,EAAE,CAAC;YACtC,aAAa,IAAI,CAAC,CAAC,iBAAiB,CAAC;YACrC,eAAe,EAAE,CAAC;QACpB,CAAC;IACH,CAAC;IAED,OAAO,uBAAuB,CAAC;QAC7B,QAAQ,EAAE,CAAC,GAAG,QAAQ,CAAC;QACvB,QAAQ,EAAE,CAAC,GAAG,QAAQ,CAAC;QACvB,aAAa,EAAE,CAAC,GAAG,aAAa,CAAC;QACjC,iBAAiB,EAAE,eAAe,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC;KAC7E,CAAC,CAAC;AACL,CAAC"}

package/dist/server/services/eval-parsers-debate-impl.d.ts

@@ -0,0 +1,11 @@
+import type { EvaluationSynthesis } from '../types/evaluation.js';
+export type Verdict = 'AGREE' | 'DISAGREE' | 'PARTIAL';
+export interface DebateParseResult {
+    readonly verdict: Verdict;
+    readonly updatedScores: Readonly<Record<string, number>>;
+    readonly critiques: readonly string[];
+    readonly reasoning: string;
+}
+export declare function parseSynthesisResponse(response: string): Partial<EvaluationSynthesis>;
+export declare function parseDebateResponse(response: string): Partial<DebateParseResult>;
+//# sourceMappingURL=eval-parsers-debate-impl.d.ts.map

package/dist/server/services/eval-parsers-debate-impl.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-parsers-debate-impl.d.ts","sourceRoot":"","sources":["../../../src/server/services/eval-parsers-debate-impl.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAMlE,MAAM,MAAM,OAAO,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAAC;AAEvD,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,aAAa,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACzD,QAAQ,CAAC,SAAS,EAAE,SAAS,MAAM,EAAE,CAAC;IACtC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AAMD,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAWrF;AAMD,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAahF"}

package/dist/server/services/eval-parsers-debate-impl.js

@@ -0,0 +1,133 @@
+// ---------------------------------------------------------------------------
+// Debate & Synthesis parsers — extracted from eval-parsers.ts
+// ---------------------------------------------------------------------------
+// ---------------------------------------------------------------------------
+// Synthesis response
+// ---------------------------------------------------------------------------
+export function parseSynthesisResponse(response) {
+    const parsed = tryParseJson(response);
+    if (parsed) {
+        return {
+            dimensionScores: validScores(parsed.dimensionScores),
+            weightedTotal: clampScore(parsed.weightedTotal),
+            confidence: clamp01(parsed.confidence),
+            dissenting: toStringArray(parsed.dissenting),
+        };
+    }
+    return parseSynthesisFromText(response);
+}
+// ---------------------------------------------------------------------------
+// Debate verdict parsing
+// ---------------------------------------------------------------------------
+export function parseDebateResponse(response) {
+    const parsed = tryParseJson(response);
+    if (parsed) {
+        return {
+            verdict: parseVerdict(parsed.verdict),
+            updatedScores: validScores(parsed.updatedScores),
+            critiques: toStringArray(parsed.critiques),
+            reasoning: typeof parsed.reasoning === 'string' ? parsed.reasoning : undefined,
+        };
+    }
+    // Fallback: extract verdict and scores from text patterns
+    return parseDebateFromText(response);
+}
+// ---------------------------------------------------------------------------
+// Internal: Text fallback parsers
+// ---------------------------------------------------------------------------
+function parseDebateFromText(text) {
+    // Try to extract verdict from text patterns like "VERDICT: AGREE"
+    const verdictMatch = text.match(/VERDICT\s*:\s*(AGREE|DISAGREE|PARTIAL)/i);
+    const verdict = verdictMatch ? parseVerdict(verdictMatch[1]) : 'PARTIAL';
+    // Try to extract scores using the same pattern as score text fallback
+    const scores = {};
+    const scorePattern = /(\w[\w\s]*?):\s*(\d+(?:\.\d+)?)\s*(?:\/\s*10)?/g;
+    let match;
+    while ((match = scorePattern.exec(text)) !== null) {
+        const dim = match[1].trim();
+        // Skip the VERDICT line we already parsed
+        if (/^verdict$/i.test(dim))
+            continue;
+        const val = parseFloat(match[2]);
+        if (!isNaN(val) && val <= 10)
+            scores[dim] = val;
+    }
+    return {
+        verdict,
+        updatedScores: Object.keys(scores).length > 0 ? scores : undefined,
+    };
+}
+function parseSynthesisFromText(text) {
+    const scoreMatch = text.match(/weighted\s*(?:total|average|score)\s*:?\s*(\d+(?:\.\d+)?)/i);
+    return {
+        weightedTotal: scoreMatch ? clampScore(parseFloat(scoreMatch[1])) : undefined,
+    };
+}
+// ---------------------------------------------------------------------------
+// Internal: JSON parsing
+// ---------------------------------------------------------------------------
+function tryParseJson(text) {
+    try {
+        return JSON.parse(text);
+    }
+    catch {
+        // fall through
+    }
+    const jsonMatch = text.match(/```(?:json)?\s*([\s\S]*?)```/);
+    if (jsonMatch) {
+        try {
+            return JSON.parse(jsonMatch[1]);
+        }
+        catch {
+            // fall through
+        }
+    }
+    const braceMatch = text.match(/\{[\s\S]*\}/);
+    if (braceMatch) {
+        try {
+            return JSON.parse(braceMatch[0]);
+        }
+        catch {
+            // fall through
+        }
+    }
+    return undefined;
+}
+// ---------------------------------------------------------------------------
+// Internal: Utilities
+// ---------------------------------------------------------------------------
+function validScores(scores) {
+    if (!scores || typeof scores !== 'object')
+        return {};
+    const result = {};
+    for (const [k, v] of Object.entries(scores)) {
+        if (typeof v === 'number' && !isNaN(v))
+            result[k] = clampScore(v);
+    }
+    return result;
+}
+function clamp01(val) {
+    if (val === undefined || isNaN(val))
+        return 0;
+    return Math.max(0, Math.min(1, val));
+}
+function clampScore(val) {
+    if (val === undefined || isNaN(val))
+        return 0;
+    return Math.max(0, Math.min(10, val));
+}
+function toStringArray(arr) {
+    if (!Array.isArray(arr))
+        return [];
+    return arr.filter((x) => typeof x === 'string');
+}
+function parseVerdict(val) {
+    if (typeof val === 'string') {
+        const upper = val.toUpperCase();
+        if (upper === 'AGREE' || upper === 'DISAGREE' || upper === 'PARTIAL') {
+            return upper;
+        }
+    }
+    return 'PARTIAL';
+}
+//# sourceMappingURL=eval-parsers-debate-impl.js.map

package/dist/server/services/eval-parsers-debate-impl.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-parsers-debate-impl.js","sourceRoot":"","sources":["../../../src/server/services/eval-parsers-debate-impl.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,8DAA8D;AAC9D,8EAA8E;AAiB9E,8EAA8E;AAC9E,qBAAqB;AACrB,8EAA8E;AAE9E,MAAM,UAAU,sBAAsB,CAAC,QAAgB;IACrD,MAAM,MAAM,GAAG,YAAY,CAAuB,QAAQ,CAAC,CAAC;IAC5D,IAAI,MAAM,EAAE,CAAC;QACX,OAAO;YACL,eAAe,EAAE,WAAW,CAAC,MAAM,CAAC,eAAe,CAAC;YACpD,aAAa,EAAE,UAAU,CAAC,MAAM,CAAC,aAAa,CAAC;YAC/C,UAAU,EAAE,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC;YACtC,UAAU,EAAE,aAAa,CAAC,MAAM,CAAC,UAAU,CAAC;SAC7C,CAAC;IACJ,CAAC;IACD,OAAO,sBAAsB,CAAC,QAAQ,CAAC,CAAC;AAC1C,CAAC;AAED,8EAA8E;AAC9E,yBAAyB;AACzB,8EAA8E;AAE9E,MAAM,UAAU,mBAAmB,CAAC,QAAgB;IAClD,MAAM,MAAM,GAAG,YAAY,CAAoB,QAAQ,CAAC,CAAC;IACzD,IAAI,MAAM,EAAE,CAAC;QACX,OAAO;YACL,OAAO,EAAE,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC;YACrC,aAAa,EAAE,WAAW,CAAC,MAAM,CAAC,aAAa,CAAC;YAChD,SAAS,EAAE,aAAa,CAAC,MAAM,CAAC,SAAS,CAAC;YAC1C,SAAS,EAAE,OAAO,MAAM,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;SAC/E,CAAC;IACJ,CAAC;IAED,0DAA0D;IAC1D,OAAO,mBAAmB,CAAC,QAAQ,CAAC,CAAC;AACvC,CAAC;AAED,8EAA8E;AAC9E,kCAAkC;AAClC,8EAA8E;AAE9E,SAAS,mBAAmB,CAAC,IAAY;IACvC,kEAAkE;IAClE,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,yCAAyC,CAAC,CAAC;IAC3E,MAAM,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAoB,CAAC;IAEpF,sEAAsE;IACtE,MAAM,MAAM,GAA2B,EAAE,CAAC;IAC1C,MAAM,YAAY,GAAG,iDAAiD,CAAC;IACvE,IAAI,KAA6B,CAAC;IAClC,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QAClD,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC5B,0CAA0C;QAC1C,IAAI,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC;YAAE,SAAS;QACrC,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QACjC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,EAAE;YAAE,MAAM,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;IAClD,CAAC;IAED,OAAO;QACL,OAAO;QACP,aAAa,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;KACnE,CAAC;AACJ,CAAC;AAED,SAAS,sBAAsB,CAAC,IAAY;IAC1C,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAC3B,4DAA4D,CAC7D,CAAC;IACF,OAAO;QACL,aAAa,EAAE,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS;KAC9E,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,yBAAyB;AACzB,8EAA8E;AAE9E,SAAS,YAAY,CAAI,IAAY;IACnC,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAM,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACP,eAAe;IACjB,CAAC;IACD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC;IAC7D,IAAI,SAAS,EAAE,CAAC;QACd,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAM,CAAC;QACvC,CAAC;QAAC,MAAM,CAAC;YACP,eAAe;QACjB,CAAC;IACH,CAAC;IACD,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IAC7C,IAAI,UAAU,EAAE,CAAC;QACf,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAM,CAAC;QACxC,CAAC;QAAC,MAAM,CAAC;YACP,eAAe;QACjB,CAAC;IACH,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAoBD,8EAA8E;AAC9E,sBAAsB;AACtB,8EAA8E;AAE9E,SAAS,WAAW,CAClB,MAA0C;IAE1C,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ;QAAE,OAAO,EAAE,CAAC;IACrD,MAAM,MAAM,GAA2B,EAAE,CAAC;IAC1C,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAC5C,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;YAAE,MAAM,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;IACpE,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,OAAO,CAAC,GAAuB;IACtC,IAAI,GAAG,KAAK,SAAS,IAAI,KAAK,CAAC,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC;IAC9C,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;AACvC,CAAC;AAED,SAAS,UAAU,CAAC,GAAuB;IACzC,IAAI,GAAG,KAAK,SAAS,IAAI,KAAK,CAAC,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC;IAC9C,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC;AACxC,CAAC;AAED,SAAS,aAAa,CAAC,GAAY;IACjC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC;QAAE,OAAO,EAAE,CAAC;IACnC,OAAO,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC;AAC/D,CAAC;AAED,SAAS,YAAY,CAAC,GAAY;IAChC,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QAC5B,MAAM,KAAK,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;QAChC,IAAI,KAAK,KAAK,OAAO,IAAI,KAAK,KAAK,UAAU,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACrE,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC"}

package/dist/server/services/eval-parsers.d.ts

@@ -0,0 +1,24 @@
+import type { IndividualEvaluation, InstructionCompliance } from '../types/evaluation.js';
+export { parseSynthesisResponse, parseDebateResponse } from './eval-parsers-debate-impl.js';
+export type { Verdict, DebateParseResult } from './eval-parsers-debate-impl.js';
+export interface ScoreParseResult {
+    readonly scores: Readonly<Record<string, number>>;
+    readonly overallCloseness: number;
+    readonly missedCritical: readonly string[];
+    readonly strengths: readonly string[];
+    readonly weaknesses: readonly string[];
+    readonly summary: string;
+}
+export declare function parseScoreResponse(response: string): Partial<ScoreParseResult>;
+export interface ComplianceParseResult {
+    readonly followed: readonly string[];
+    readonly violated: readonly string[];
+    readonly notApplicable: readonly string[];
+    readonly overallCompliance: number;
+}
+export declare function parseComplianceResponse(response: string): Partial<ComplianceParseResult>;
+/** Convert parsed compliance into our InstructionCompliance type. */
+export declare function toInstructionCompliance(result: Partial<ComplianceParseResult>): InstructionCompliance;
+/** Convert score parse result into IndividualEvaluation entries. */
+export declare function toIndividualEvaluations(scores: Readonly<Record<string, number>>, role: string, reasoningMap: Readonly<Record<string, string>>): IndividualEvaluation[];
+//# sourceMappingURL=eval-parsers.d.ts.map

package/dist/server/services/eval-parsers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-parsers.d.ts","sourceRoot":"","sources":["../../../src/server/services/eval-parsers.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EACV,oBAAoB,EACpB,qBAAqB,EACtB,MAAM,wBAAwB,CAAC;AAGhC,OAAO,EAAE,sBAAsB,EAAE,mBAAmB,EAAE,MAAM,+BAA+B,CAAC;AAC5F,YAAY,EAAE,OAAO,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AAMhF,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAClD,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,cAAc,EAAE,SAAS,MAAM,EAAE,CAAC;IAC3C,QAAQ,CAAC,SAAS,EAAE,SAAS,MAAM,EAAE,CAAC;IACtC,QAAQ,CAAC,UAAU,EAAE,SAAS,MAAM,EAAE,CAAC;IACvC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B;AAED,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAa9E;AAMD,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;IACrC,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;IACrC,QAAQ,CAAC,aAAa,EAAE,SAAS,MAAM,EAAE,CAAC;IAC1C,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;CACpC;AAED,wBAAgB,uBAAuB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAMxF;AAED,qEAAqE;AACrE,wBAAgB,uBAAuB,CACrC,MAAM,EAAE,OAAO,CAAC,qBAAqB,CAAC,GACrC,qBAAqB,CAOvB;AAMD,oEAAoE;AACpE,wBAAgB,uBAAuB,CACrC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,EACxC,IAAI,EAAE,MAAM,EACZ,YAAY,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,GAC7C,oBAAoB,EAAE,CAOxB"}

package/dist/server/services/eval-parsers.js

@@ -0,0 +1,153 @@
+// ---------------------------------------------------------------------------
+// Evaluation Response Parsers — extract structured data from LLM responses
+// ---------------------------------------------------------------------------
+// Re-export debate/synthesis parsers so existing imports continue to work
+export { parseSynthesisResponse, parseDebateResponse } from './eval-parsers-debate-impl.js';
+export function parseScoreResponse(response) {
+    const parsed = tryParseJson(response);
+    if (parsed) {
+        return {
+            scores: validScores(parsed.scores),
+            overallCloseness: clamp01(parsed.overallCloseness),
+            missedCritical: toStringArray(parsed.missedCritical),
+            strengths: toStringArray(parsed.strengths),
+            weaknesses: toStringArray(parsed.weaknesses),
+            summary: typeof parsed.summary === 'string' ? parsed.summary : undefined,
+        };
+    }
+    return parseScoreFromText(response);
+}
+export function parseComplianceResponse(response) {
+    const parsed = tryParseJson(response);
+    if (parsed) {
+        return categorizeComplianceResults(parsed);
+    }
+    return parseComplianceFromText(response);
+}
+/** Convert parsed compliance into our InstructionCompliance type. */
+export function toInstructionCompliance(result) {
+    return {
+        followed: result.followed ?? [],
+        violated: result.violated ?? [],
+        notApplicable: result.notApplicable ?? [],
+        overallCompliance: result.overallCompliance ?? 0,
+    };
+}
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+/** Convert score parse result into IndividualEvaluation entries. */
+export function toIndividualEvaluations(scores, role, reasoningMap) {
+    return Object.entries(scores).map(([dimension, score]) => ({
+        evaluatorRole: role,
+        dimension,
+        score: clampScore(score),
+        reasoning: reasoningMap[dimension] ?? '',
+    }));
+}
+// ---------------------------------------------------------------------------
+// Internal: JSON parsing
+// ---------------------------------------------------------------------------
+function tryParseJson(text) {
+    try {
+        return JSON.parse(text);
+    }
+    catch {
+        // fall through
+    }
+    const jsonMatch = text.match(/```(?:json)?\s*([\s\S]*?)```/);
+    if (jsonMatch) {
+        try {
+            return JSON.parse(jsonMatch[1]);
+        }
+        catch {
+            // fall through
+        }
+    }
+    const braceMatch = text.match(/\{[\s\S]*\}/);
+    if (braceMatch) {
+        try {
+            return JSON.parse(braceMatch[0]);
+        }
+        catch {
+            // fall through
+        }
+    }
+    return undefined;
+}
+// ---------------------------------------------------------------------------
+// Internal: Text fallback parsers
+// ---------------------------------------------------------------------------
+function parseScoreFromText(text) {
+    const scores = {};
+    const scorePattern = /(\w[\w\s]*?):\s*(\d+(?:\.\d+)?)\s*(?:\/\s*10)?/g;
+    let match;
+    while ((match = scorePattern.exec(text)) !== null) {
+        const dim = match[1].trim();
+        const val = parseFloat(match[2]);
+        if (!isNaN(val) && val <= 10)
+            scores[dim] = val;
+    }
+    return { scores: Object.keys(scores).length > 0 ? scores : undefined };
+}
+function parseComplianceFromText(text) {
+    const followed = [];
+    const violated = [];
+    if (/followed|compliant/i.test(text))
+        followed.push('(extracted from text)');
+    if (/violated|non-compliant/i.test(text))
+        violated.push('(extracted from text)');
+    return { followed, violated, notApplicable: [], overallCompliance: undefined };
+}
+// ---------------------------------------------------------------------------
+// Internal: Utilities
+// ---------------------------------------------------------------------------
+function validScores(scores) {
+    if (!scores || typeof scores !== 'object')
+        return {};
+    const result = {};
+    for (const [k, v] of Object.entries(scores)) {
+        if (typeof v === 'number' && !isNaN(v))
+            result[k] = clampScore(v);
+    }
+    return result;
+}
+function clamp01(val) {
+    if (val === undefined || isNaN(val))
+        return 0;
+    return Math.max(0, Math.min(1, val));
+}
+function clampScore(val) {
+    if (val === undefined || isNaN(val))
+        return 0;
+    return Math.max(0, Math.min(10, val));
+}
+function toStringArray(arr) {
+    if (!Array.isArray(arr))
+        return [];
+    return arr.filter((x) => typeof x === 'string');
+}
+function categorizeComplianceResults(parsed) {
+    const followed = [];
+    const violated = [];
+    const notApplicable = [];
+    if (Array.isArray(parsed.results)) {
+        for (const r of parsed.results) {
+            const text = r.instruction ?? '(unknown)';
+            const status = (r.status ?? '').toLowerCase();
+            if (status === 'followed')
+                followed.push(text);
+            else if (status === 'violated')
+                violated.push(text);
+            else
+                notApplicable.push(text);
+        }
+    }
+    return {
+        followed,
+        violated,
+        notApplicable,
+        overallCompliance: clamp01(parsed.overallCompliance),
+    };
+}
+//# sourceMappingURL=eval-parsers.js.map

package/dist/server/services/eval-parsers.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-parsers.js","sourceRoot":"","sources":["../../../src/server/services/eval-parsers.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,2EAA2E;AAC3E,8EAA8E;AAO9E,0EAA0E;AAC1E,OAAO,EAAE,sBAAsB,EAAE,mBAAmB,EAAE,MAAM,+BAA+B,CAAC;AAgB5F,MAAM,UAAU,kBAAkB,CAAC,QAAgB;IACjD,MAAM,MAAM,GAAG,YAAY,CAAmB,QAAQ,CAAC,CAAC;IACxD,IAAI,MAAM,EAAE,CAAC;QACX,OAAO;YACL,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC;YAClC,gBAAgB,EAAE,OAAO,CAAC,MAAM,CAAC,gBAAgB,CAAC;YAClD,cAAc,EAAE,aAAa,CAAC,MAAM,CAAC,cAAc,CAAC;YACpD,SAAS,EAAE,aAAa,CAAC,MAAM,CAAC,SAAS,CAAC;YAC1C,UAAU,EAAE,aAAa,CAAC,MAAM,CAAC,UAAU,CAAC;YAC5C,OAAO,EAAE,OAAO,MAAM,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS;SACzE,CAAC;IACJ,CAAC;IACD,OAAO,kBAAkB,CAAC,QAAQ,CAAC,CAAC;AACtC,CAAC;AAaD,MAAM,UAAU,uBAAuB,CAAC,QAAgB;IACtD,MAAM,MAAM,GAAG,YAAY,CAAwB,QAAQ,CAAC,CAAC;IAC7D,IAAI,MAAM,EAAE,CAAC;QACX,OAAO,2BAA2B,CAAC,MAAM,CAAC,CAAC;IAC7C,CAAC;IACD,OAAO,uBAAuB,CAAC,QAAQ,CAAC,CAAC;AAC3C,CAAC;AAED,qEAAqE;AACrE,MAAM,UAAU,uBAAuB,CACrC,MAAsC;IAEtC,OAAO;QACL,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,EAAE;QAC/B,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,EAAE;QAC/B,aAAa,EAAE,MAAM,CAAC,aAAa,IAAI,EAAE;QACzC,iBAAiB,EAAE,MAAM,CAAC,iBAAiB,IAAI,CAAC;KACjD,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,UAAU;AACV,8EAA8E;AAE9E,oEAAoE;AACpE,MAAM,UAAU,uBAAuB,CACrC,MAAwC,EACxC,IAAY,EACZ,YAA8C;IAE9C,OAAO,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC;QACzD,aAAa,EAAE,IAAI;QACnB,SAAS;QACT,KAAK,EAAE,UAAU,CAAC,KAAK,CAAC;QACxB,SAAS,EAAE,YAAY,CAAC,SAAS,CAAC,IAAI,EAAE;KACzC,CAAC,CAAC,CAAC;AACN,CAAC;AAED,8EAA8E;AAC9E,yBAAyB;AACzB,8EAA8E;AAE9E,SAAS,YAAY,CAAI,IAAY;IACnC,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAM,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACP,eAAe;IACjB,CAAC;IACD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC;IAC7D,IAAI,SAAS,EAAE,CAAC;QACd,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAM,CAAC;QACvC,CAAC;QAAC,MAAM,CAAC;YACP,eAAe;QACjB,CAAC;IACH,CAAC;IACD,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IAC7C,IAAI,UAAU,EAAE,CAAC;QACf,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAM,CAAC;QACxC,CAAC;QAAC,MAAM,CAAC;YACP,eAAe;QACjB,CAAC;IACH,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,8EAA8E;AAC9E,kCAAkC;AAClC,8EAA8E;AAE9E,SAAS,kBAAkB,CAAC,IAAY;IACtC,MAAM,MAAM,GAA2B,EAAE,CAAC;IAC1C,MAAM,YAAY,GAAG,iDAAiD,CAAC;IACvE,IAAI,KAA6B,CAAC;IAClC,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QAClD,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC5B,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QACjC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,EAAE;YAAE,MAAM,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;IAClD,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC;AACzE,CAAC;AAED,SAAS,uBAAuB,CAAC,IAAY;IAC3C,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,IAAI,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,QAAQ,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;IAC7E,IAAI,yBAAyB,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,QAAQ,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;IACjF,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,aAAa,EAAE,EAAE,EAAE,iBAAiB,EAAE,SAAS,EAAE,CAAC;AACjF,CAAC;AAoBD,8EAA8E;AAC9E,sBAAsB;AACtB,8EAA8E;AAE9E,SAAS,WAAW,CAClB,MAA0C;IAE1C,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ;QAAE,OAAO,EAAE,CAAC;IACrD,MAAM,MAAM,GAA2B,EAAE,CAAC;IAC1C,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAC5C,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;YAAE,MAAM,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;IACpE,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,OAAO,CAAC,GAAuB;IACtC,IAAI,GAAG,KAAK,SAAS,IAAI,KAAK,CAAC,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC;IAC9C,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;AACvC,CAAC;AAED,SAAS,UAAU,CAAC,GAAuB;IACzC,IAAI,GAAG,KAAK,SAAS,IAAI,KAAK,CAAC,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC;IAC9C,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC;AACxC,CAAC;AAED,SAAS,aAAa,CAAC,GAAY;IACjC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC;QAAE,OAAO,EAAE,CAAC;IACnC,OAAO,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC;AAC/D,CAAC;AAED,SAAS,2BAA2B,CAClC,MAA6B;IAE7B,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,MAAM,aAAa,GAAa,EAAE,CAAC;IAEnC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;QAClC,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YAC/B,MAAM,IAAI,GAAG,CAAC,CAAC,WAAW,IAAI,WAAW,CAAC;YAC1C,MAAM,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;YAC9C,IAAI,MAAM,KAAK,UAAU;gBAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;iBAC1C,IAAI,MAAM,KAAK,UAAU;gBAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;;gBAC/C,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;IAED,OAAO;QACL,QAAQ;QACR,QAAQ;QACR,aAAa;QACb,iBAAiB,EAAE,OAAO,CAAC,MAAM,CAAC,iBAAiB,CAAC;KACrD,CAAC;AACJ,CAAC"}

package/dist/server/services/eval-prompts.d.ts

@@ -0,0 +1,9 @@
+import type { Scenario, Provider } from '../types/index.js';
+import type { IndividualEvaluation } from '../types/evaluation.js';
+import type { TranscriptSummary } from './transcript-formatter.js';
+import type { InstructionBlock } from './instruction-parser.js';
+export declare function buildScorePrompt(transcript: string, scenario: Scenario, summary: TranscriptSummary): string;
+export declare function buildCompliancePrompt(transcript: string, scenario: Scenario, instructions: readonly InstructionBlock[]): string;
+export declare function buildDebatePrompt(myPreviousAssessment: string, otherAssessments: readonly string[], roundNumber: number): string;
+export declare function buildSynthesisPrompt(allEvaluations: readonly IndividualEvaluation[], scenario: Scenario, provider: Provider): string;
+//# sourceMappingURL=eval-prompts.d.ts.map

package/dist/server/services/eval-prompts.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-prompts.d.ts","sourceRoot":"","sources":["../../../src/server/services/eval-prompts.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,QAAQ,EAAoB,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAC9E,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AACnE,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AACnE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAMhE,wBAAgB,gBAAgB,CAC9B,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,QAAQ,EAClB,OAAO,EAAE,iBAAiB,GACzB,MAAM,CA+CR;AAMD,wBAAgB,qBAAqB,CACnC,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,QAAQ,EAClB,YAAY,EAAE,SAAS,gBAAgB,EAAE,GACxC,MAAM,CA4BR;AAMD,wBAAgB,iBAAiB,CAC/B,oBAAoB,EAAE,MAAM,EAC5B,gBAAgB,EAAE,SAAS,MAAM,EAAE,EACnC,WAAW,EAAE,MAAM,GAClB,MAAM,CAyBR;AAMD,wBAAgB,oBAAoB,CAClC,cAAc,EAAE,SAAS,oBAAoB,EAAE,EAC/C,QAAQ,EAAE,QAAQ,EAClB,QAAQ,EAAE,QAAQ,GACjB,MAAM,CAkCR"}

package/dist/server/services/eval-prompts.js

@@ -0,0 +1,164 @@
+// ---------------------------------------------------------------------------
+// Evaluation Prompt Builders
+// ---------------------------------------------------------------------------
+// ---------------------------------------------------------------------------
+// Query 1: Score + answer comparison prompt
+// ---------------------------------------------------------------------------
+export function buildScorePrompt(transcript, scenario, summary) {
+    const dimensions = formatDimensions(scenario.scoringDimensions);
+    const toolSequence = summary.toolCallSequence.length > 0
+        ? `Tool call sequence: ${summary.toolCallSequence.join(' → ')}`
+        : 'No tool calls recorded.';
+    return `You are an expert evaluator assessing an AI agent's performance on a task.
+
+## Task Description
+${scenario.prompt}
+
+## Expected Answer
+${scenario.expectedAnswer}
+
+## Critical Requirements
+${formatCriticalRequirements(scenario.criticalRequirements)}
+
+## Grading Guidelines
+${scenario.gradingGuidelines || 'No specific grading guidelines provided.'}
+
+## Scoring Dimensions
+${dimensions}
+
+## Agent Behavior Summary
+${toolSequence}
+Files read: ${summary.filesRead.length > 0 ? summary.filesRead.join(', ') : 'none'}
+Files modified: ${summary.filesModified.length > 0 ? summary.filesModified.join(', ') : 'none'}
+Command failures: ${summary.commandFailures.length}
+Asked clarifying questions: ${summary.askedClarifyingQuestions ? 'yes' : 'no'}
+
+## Full Transcript
+${transcript}
+
+## Instructions
+Evaluate the agent's output. For each scoring dimension, provide a score from 0-10.
+Also assess how closely the agent's final answer matches the expected answer (0.0-1.0).
+Identify any critical requirements that were missed, as well as strengths and weaknesses.
+
+Respond with valid JSON matching this structure:
+{
+  "scores": { "<dimension_name>": <0-10>, ... },
+  "overallCloseness": <0.0-1.0>,
+  "missedCritical": ["<requirement that was not met>", ...],
+  "strengths": ["<strength>", ...],
+  "weaknesses": ["<weakness>", ...],
+  "summary": "<brief overall assessment>"
+}`;
+}
+// ---------------------------------------------------------------------------
+// Query 2: Instruction compliance prompt
+// ---------------------------------------------------------------------------
+export function buildCompliancePrompt(transcript, scenario, instructions) {
+    const instructionList = instructions
+        .map((b, i) => `${i + 1}. [${b.source}] ${b.text}`)
+        .join('\n');
+    return `You are an expert evaluator checking whether an AI agent followed its configured instructions.
+
+## Instructions to Check
+${instructionList || 'No instructions configured.'}
+
+## Agent Transcript
+${transcript}
+
+## Instructions
+For each instruction listed above, determine if the agent:
+- "followed" it (clear evidence of compliance)
+- "violated" it (clear evidence of non-compliance)
+- "not_applicable" (instruction was not relevant to this task)
+
+Also rate overall compliance from 0.0 to 1.0.
+
+Respond with valid JSON:
+{
+  "results": [
+    { "instruction": "<instruction text>", "status": "followed|violated|not_applicable", "evidence": "<brief evidence>" }
+  ],
+  "overallCompliance": <0.0-1.0>
+}`;
+}
+// ---------------------------------------------------------------------------
+// Multi-round debate prompt
+// ---------------------------------------------------------------------------
+export function buildDebatePrompt(myPreviousAssessment, otherAssessments, roundNumber) {
+    const othersFormatted = otherAssessments
+        .map((a, i) => `### Other Evaluator ${i + 1}\n${a}`)
+        .join('\n\n');
+    return `You are participating in round ${roundNumber} of a multi-evaluator review.
+
+## Your Previous Assessment
+${myPreviousAssessment}
+
+## Other Evaluators' Assessments
+${othersFormatted}
+
+## Instructions
+Review the other evaluators' assessments and compare them with your own.
+Determine if you agree, partially agree, or disagree with the emerging consensus.
+Provide your updated scores if you have changed your mind, and explain why.
+
+Respond with valid JSON:
+{
+  "verdict": "AGREE|DISAGREE|PARTIAL",
+  "updatedScores": { "<dimension_name>": <0-10>, ... },
+  "critiques": ["<specific point of disagreement>", ...],
+  "reasoning": "<why you agree/disagree>"
+}`;
+}
+// ---------------------------------------------------------------------------
+// Synthesis prompt (final aggregation)
+// ---------------------------------------------------------------------------
+export function buildSynthesisPrompt(allEvaluations, scenario, provider) {
+    const evalSummaries = allEvaluations
+        .map((e) => `[${e.evaluatorRole}] ${e.dimension}: ${e.score}/10 — ${e.reasoning}`)
+        .join('\n');
+    const dimensions = formatDimensions(scenario.scoringDimensions);
+    return `You are the final synthesizer for a multi-evaluator assessment.
+
+## Scenario
+${scenario.name}: ${scenario.prompt}
+
+## Scoring Dimensions & Weights
+${dimensions}
+
+## Provider
+Name: ${provider.name}
+Model: ${provider.model}
+
+## All Individual Evaluations
+${evalSummaries}
+
+## Instructions
+Synthesize all evaluations into final scores. Weight each dimension according to the scoring
+dimensions defined above. Identify areas of evaluator consensus and disagreement.
+Provide a confidence level (0.0-1.0) based on evaluator agreement.
+
+Respond with valid JSON:
+{
+  "dimensionScores": { "<dimension_name>": <0-10>, ... },
+  "weightedTotal": <weighted average 0-10>,
+  "confidence": <0.0-1.0>,
+  "dissenting": ["<areas where evaluators disagreed>", ...]
+}`;
+}
+// ---------------------------------------------------------------------------
+// Internal helpers
+// ---------------------------------------------------------------------------
+function formatDimensions(dims) {
+    if (dims.length === 0)
+        return 'No dimensions defined. Use general quality assessment.';
+    return dims
+        .map((d) => `- ${d.name} (weight: ${d.weight}): ${d.description}`)
+        .join('\n');
+}
+function formatCriticalRequirements(reqs) {
+    if (reqs.length === 0)
+        return 'None specified.';
+    return reqs.map((r, i) => `${i + 1}. ${r}`).join('\n');
+}
+//# sourceMappingURL=eval-prompts.js.map

package/dist/server/services/eval-prompts.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-prompts.js","sourceRoot":"","sources":["../../../src/server/services/eval-prompts.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,6BAA6B;AAC7B,8EAA8E;AAO9E,8EAA8E;AAC9E,4CAA4C;AAC5C,8EAA8E;AAE9E,MAAM,UAAU,gBAAgB,CAC9B,UAAkB,EAClB,QAAkB,EAClB,OAA0B;IAE1B,MAAM,UAAU,GAAG,gBAAgB,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC;IAChE,MAAM,YAAY,GAAG,OAAO,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC;QACtD,CAAC,CAAC,uBAAuB,OAAO,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;QAC/D,CAAC,CAAC,yBAAyB,CAAC;IAE9B,OAAO;;;EAGP,QAAQ,CAAC,MAAM;;;EAGf,QAAQ,CAAC,cAAc;;;EAGvB,0BAA0B,CAAC,QAAQ,CAAC,oBAAoB,CAAC;;;EAGzD,QAAQ,CAAC,iBAAiB,IAAI,0CAA0C;;;EAGxE,UAAU;;;EAGV,YAAY;cACA,OAAO,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM;kBAChE,OAAO,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM;oBAC1E,OAAO,CAAC,eAAe,CAAC,MAAM;8BACpB,OAAO,CAAC,wBAAwB,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI;;;EAG3E,UAAU;;;;;;;;;;;;;;;EAeV,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,yCAAyC;AACzC,8EAA8E;AAE9E,MAAM,UAAU,qBAAqB,CACnC,UAAkB,EAClB,QAAkB,EAClB,YAAyC;IAEzC,MAAM,eAAe,GAAG,YAAY;SACjC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;SAClD,IAAI,CAAC,IAAI,CAAC,CAAC;IAEd,OAAO;;;EAGP,eAAe,IAAI,6BAA6B;;;EAGhD,UAAU;;;;;;;;;;;;;;;;EAgBV,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,4BAA4B;AAC5B,8EAA8E;AAE9E,MAAM,UAAU,iBAAiB,CAC/B,oBAA4B,EAC5B,gBAAmC,EACnC,WAAmB;IAEnB,MAAM,eAAe,GAAG,gBAAgB;SACrC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,uBAAuB,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;SACnD,IAAI,CAAC,MAAM,CAAC,CAAC;IAEhB,OAAO,kCAAkC,WAAW;;;EAGpD,oBAAoB;;;EAGpB,eAAe;;;;;;;;;;;;;EAaf,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,uCAAuC;AACvC,8EAA8E;AAE9E,MAAM,UAAU,oBAAoB,CAClC,cAA+C,EAC/C,QAAkB,EAClB,QAAkB;IAElB,MAAM,aAAa,GAAG,cAAc;SACjC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,aAAa,KAAK,CAAC,CAAC,SAAS,KAAK,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC,SAAS,EAAE,CAAC;SACjF,IAAI,CAAC,IAAI,CAAC,CAAC;IAEd,MAAM,UAAU,GAAG,gBAAgB,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC;IAEhE,OAAO;;;EAGP,QAAQ,CAAC,IAAI,KAAK,QAAQ,CAAC,MAAM;;;EAGjC,UAAU;;;QAGJ,QAAQ,CAAC,IAAI;SACZ,QAAQ,CAAC,KAAK;;;EAGrB,aAAa;;;;;;;;;;;;;EAab,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,mBAAmB;AACnB,8EAA8E;AAE9E,SAAS,gBAAgB,CAAC,IAAiC;IACzD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,wDAAwD,CAAC;IACvF,OAAO,IAAI;SACR,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,aAAa,CAAC,CAAC,MAAM,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;SACjE,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED,SAAS,0BAA0B,CAAC,IAAuB;IACzD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,iBAAiB,CAAC;IAChD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACzD,CAAC"}

package/dist/server/services/evaluator.d.ts

@@ -0,0 +1,10 @@
+import type { IEvaluator, EvaluationCallbacks } from '../interfaces/evaluator.js';
+import type { Run, Scenario, Provider, Evaluation, EvaluationRequest } from '../types/index.js';
+export declare class EvaluationOrchestrator implements IEvaluator {
+    evaluateRun(run: Run, scenario: Scenario, provider: Provider, request: EvaluationRequest, callbacks: EvaluationCallbacks): Promise<Evaluation>;
+    private runRound1;
+    private runDebateRound;
+    private runSynthesis;
+    private runQuery;
+}
+//# sourceMappingURL=evaluator.d.ts.map

package/dist/server/services/evaluator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluator.d.ts","sourceRoot":"","sources":["../../../src/server/services/evaluator.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,UAAU,EAAE,mBAAmB,EAAmB,MAAM,4BAA4B,CAAC;AACnG,OAAO,KAAK,EACV,GAAG,EACH,QAAQ,EACR,QAAQ,EACR,UAAU,EACV,iBAAiB,EAMlB,MAAM,mBAAmB,CAAC;AA6B3B,qBAAa,sBAAuB,YAAW,UAAU;IACjD,WAAW,CACf,GAAG,EAAE,GAAG,EACR,QAAQ,EAAE,QAAQ,EAClB,QAAQ,EAAE,QAAQ,EAClB,OAAO,EAAE,iBAAiB,EAC1B,SAAS,EAAE,mBAAmB,GAC7B,OAAO,CAAC,UAAU,CAAC;YA0ER,SAAS;YA6BT,cAAc;YAyBd,YAAY;YAiBZ,QAAQ;CA6BvB"}