oh-my-knowledge 0.0.1 → 0.19.0

package/dist/src/executors/shared.js

@@ -0,0 +1,49 @@
+import { execFile } from 'node:child_process';
+import { existsSync } from 'node:fs';
+import { delimiter, join } from 'node:path';
+import { promisify } from 'node:util';
+export const execFileAsync = promisify(execFile);
+export const DEFAULT_MODEL = 'sonnet';
+export const JUDGE_MODEL = 'haiku';
+export const DEFAULT_TIMEOUT_MS = 120_000;
+export const MAX_BUFFER = 10 * 1024 * 1024;
+export function asErrorLike(err) {
+    return typeof err === 'object' && err !== null ? err : {};
+}
+export function errorMessage(err, fallback = 'unknown error') {
+    const details = asErrorLike(err);
+    return details.message || fallback;
+}
+export function parseJson(content) {
+    return JSON.parse(content);
+}
+export function buildExecEnv(skillDir) {
+    const proxyUrl = process.env.CCV_PROXY_URL || undefined;
+    const env = proxyUrl
+        ? { ...process.env, ANTHROPIC_BASE_URL: proxyUrl }
+        : { ...process.env };
+    if (skillDir) {
+        const nodeBin = join(skillDir, 'node_modules', '.bin');
+        if (existsSync(nodeBin)) {
+            env.PATH = `${nodeBin}${env.PATH ? delimiter + env.PATH : ''}`;
+        }
+    }
+    return env;
+}
+export function timeoutExecResult(timeoutMs, durationMs) {
+    return {
+        ok: false,
+        error: `execution timed out after ${timeoutMs / 1000}s`,
+        durationMs,
+        durationApiMs: 0,
+        inputTokens: 0,
+        outputTokens: 0,
+        cacheReadTokens: 0,
+        cacheCreationTokens: 0,
+        costUSD: 0,
+        output: null,
+        stopReason: 'timeout',
+        numTurns: 0,
+    };
+}
+//# sourceMappingURL=shared.js.map

package/dist/src/executors/shared.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"shared.js","sourceRoot":"","sources":["../../../src/executors/shared.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAGtC,MAAM,CAAC,MAAM,aAAa,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC;AAEjD,MAAM,CAAC,MAAM,aAAa,GAAG,QAAQ,CAAC;AACtC,MAAM,CAAC,MAAM,WAAW,GAAG,OAAO,CAAC;AACnC,MAAM,CAAC,MAAM,kBAAkB,GAAG,OAAO,CAAC;AAC1C,MAAM,CAAC,MAAM,UAAU,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;AAmG3C,MAAM,UAAU,WAAW,CAAC,GAAY;IACtC,OAAO,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI,CAAC,CAAC,CAAC,GAAwB,CAAC,CAAC,CAAC,EAAE,CAAC;AACjF,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,GAAY,EAAE,WAAmB,eAAe;IAC3E,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;IACjC,OAAO,OAAO,CAAC,OAAO,IAAI,QAAQ,CAAC;AACrC,CAAC;AAED,MAAM,UAAU,SAAS,CAAI,OAAe;IAC1C,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAM,CAAC;AAClC,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,QAAwB;IACnD,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,aAAa,IAAI,SAAS,CAAC;IACxD,MAAM,GAAG,GAAsB,QAAQ;QACrC,CAAC,CAAC,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,kBAAkB,EAAE,QAAQ,EAAE;QAClD,CAAC,CAAC,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IAEvB,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,EAAE,cAAc,EAAE,MAAM,CAAC,CAAC;QACvD,IAAI,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YACxB,GAAG,CAAC,IAAI,GAAG,GAAG,OAAO,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QACjE,CAAC;IACH,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,SAAiB,EAAE,UAAkB;IACrE,OAAO;QACL,EAAE,EAAE,KAAK;QACT,KAAK,EAAE,6BAA6B,SAAS,GAAG,IAAI,GAAG;QACvD,UAAU;QACV,aAAa,EAAE,CAAC;QAChB,WAAW,EAAE,CAAC;QACd,YAAY,EAAE,CAAC;QACf,eAAe,EAAE,CAAC;QAClB,mBAAmB,EAAE,CAAC;QACtB,OAAO,EAAE,CAAC;QACV,MAAM,EAAE,IAAI;QACZ,UAAU,EAAE,SAAS;QACrB,QAAQ,EAAE,CAAC;KACZ,CAAC;AACJ,CAAC"}

package/dist/src/grading/assertions.d.ts

@@ -0,0 +1,18 @@
+import type { Assertion, AssertionResults, ExecutorFn, Sample, ToolCallInfo } from '../types.js';
+export declare const ASYNC_ASSERTION_TYPES: Set<string>;
+export interface AsyncAssertionContext {
+    executor: ExecutorFn;
+    judgeModel: string;
+    sample: Sample;
+    samplesDir: string;
+}
+export declare function ratioToScore(ratio: number): number;
+export declare function validateJsonSchema(data: unknown, schema: Record<string, unknown>): boolean;
+export declare function runAssertions(output: string, assertions: Assertion[], context?: {
+    costUSD?: number;
+    durationMs?: number;
+    numTurns?: number;
+    toolCalls?: ToolCallInfo[];
+}): AssertionResults;
+export declare function runAsyncAssertions(output: string, assertions: Assertion[], { executor, judgeModel, sample, samplesDir }: AsyncAssertionContext): Promise<AssertionResults>;
+//# sourceMappingURL=assertions.d.ts.map

package/dist/src/grading/assertions.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"assertions.d.ts","sourceRoot":"","sources":["../../../src/grading/assertions.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAmB,gBAAgB,EAAE,UAAU,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAMlH,eAAO,MAAM,qBAAqB,aAA6C,CAAC;AAEhF,MAAM,WAAW,qBAAqB;IACpC,QAAQ,EAAE,UAAU,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;CACpB;AAwBD,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAElD;AAED,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAQ1F;AAED,wBAAgB,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,EAAE,OAAO,GAAE;IAAE,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,YAAY,EAAE,CAAA;CAAO,GAAG,gBAAgB,CAqI/L;AAED,wBAAsB,kBAAkB,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,EAAE,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,EAAE,qBAAqB,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAkFhL"}

package/dist/src/grading/assertions.js

@@ -0,0 +1,239 @@
+import { resolve } from 'node:path';
+import _Ajv from 'ajv';
+const Ajv = _Ajv.default ?? _Ajv;
+const ajv = new Ajv();
+const CUSTOM_ASSERTION_TIMEOUT_MS = 30_000;
+export const ASYNC_ASSERTION_TYPES = new Set(['semantic_similarity', 'custom']);
+function getErrorMessage(err) {
+    return err instanceof Error ? err.message : String(err);
+}
+export function ratioToScore(ratio) {
+    return Number((1 + ratio * 4).toFixed(2));
+}
+export function validateJsonSchema(data, schema) {
+    if (!schema || typeof schema !== 'object')
+        return true;
+    try {
+        const validate = ajv.compile(schema);
+        return validate(data);
+    }
+    catch {
+        return false;
+    }
+}
+export function runAssertions(output, assertions, context = {}) {
+    const outputLower = output.toLowerCase();
+    const details = [];
+    const toolCalls = context.toolCalls || [];
+    const toolNames = toolCalls.map((tc) => tc.tool.toLowerCase());
+    for (const assertion of assertions) {
+        const weight = assertion.weight ?? 1;
+        let passed = false;
+        switch (assertion.type) {
+            case 'contains':
+                passed = outputLower.includes(String(assertion.value).toLowerCase());
+                break;
+            case 'not_contains':
+                passed = !outputLower.includes(String(assertion.value).toLowerCase());
+                break;
+            case 'regex': {
+                const flags = assertion.flags || 'i';
+                const re = new RegExp(assertion.pattern, flags);
+                passed = re.test(output);
+                break;
+            }
+            case 'min_length':
+                passed = output.length >= assertion.value;
+                break;
+            case 'max_length':
+                passed = output.length <= assertion.value;
+                break;
+            case 'json_valid':
+                try {
+                    JSON.parse(output);
+                    passed = true;
+                }
+                catch {
+                    passed = false;
+                }
+                break;
+            case 'json_schema':
+                try {
+                    const data = JSON.parse(output);
+                    passed = validateJsonSchema(data, assertion.schema);
+                }
+                catch {
+                    passed = false;
+                }
+                break;
+            case 'starts_with':
+                passed = outputLower.startsWith(String(assertion.value).toLowerCase());
+                break;
+            case 'ends_with':
+                passed = outputLower.endsWith(String(assertion.value).toLowerCase());
+                break;
+            case 'equals':
+                passed = output.trim() === String(assertion.value).trim();
+                break;
+            case 'not_equals':
+                passed = output.trim() !== String(assertion.value).trim();
+                break;
+            case 'word_count_min':
+                passed = output.split(/\s+/).filter(Boolean).length >= assertion.value;
+                break;
+            case 'word_count_max':
+                passed = output.split(/\s+/).filter(Boolean).length <= assertion.value;
+                break;
+            case 'contains_all':
+                passed = (assertion.values || []).every((v) => outputLower.includes(String(v).toLowerCase()));
+                break;
+            case 'contains_any':
+                passed = (assertion.values || []).some((v) => outputLower.includes(String(v).toLowerCase()));
+                break;
+            case 'cost_max':
+                passed = (context.costUSD ?? Infinity) <= assertion.value;
+                break;
+            case 'latency_max':
+                passed = (context.durationMs ?? Infinity) <= assertion.value;
+                break;
+            case 'turns_max':
+                passed = (context.numTurns ?? Infinity) <= assertion.value;
+                break;
+            case 'turns_min':
+                passed = (context.numTurns ?? 0) >= assertion.value;
+                break;
+            case 'tools_called':
+                passed = (assertion.values || []).every((v) => toolNames.includes(String(v).toLowerCase()));
+                break;
+            case 'tools_not_called':
+                passed = (assertion.values || []).every((v) => !toolNames.includes(String(v).toLowerCase()));
+                break;
+            case 'tools_count_max':
+                passed = toolCalls.length <= assertion.value;
+                break;
+            case 'tools_count_min':
+                passed = toolCalls.length >= assertion.value;
+                break;
+            case 'tool_output_contains': {
+                const sep = String(assertion.value).indexOf(':');
+                if (sep > 0) {
+                    const targetTool = String(assertion.value).slice(0, sep).toLowerCase();
+                    const expected = String(assertion.value).slice(sep + 1).toLowerCase();
+                    passed = toolCalls.some((tc) => tc.tool.toLowerCase() === targetTool &&
+                        String(tc.output || '').toLowerCase().includes(expected));
+                }
+                break;
+            }
+            case 'tool_input_contains': {
+                const sep = String(assertion.value).indexOf(':');
+                if (sep > 0) {
+                    const targetTool = String(assertion.value).slice(0, sep).toLowerCase();
+                    const expected = String(assertion.value).slice(sep + 1).toLowerCase();
+                    passed = toolCalls.some((tc) => tc.tool.toLowerCase() === targetTool &&
+                        JSON.stringify(tc.input || '').toLowerCase().includes(expected));
+                }
+                break;
+            }
+            default:
+                passed = false;
+        }
+        details.push({
+            type: assertion.type,
+            value: assertion.value ?? assertion.pattern ?? assertion.values?.join(', ') ?? '',
+            weight,
+            passed,
+        });
+    }
+    const totalWeight = details.reduce((s, d) => s + d.weight, 0);
+    const passedWeight = details.filter((d) => d.passed).reduce((s, d) => s + d.weight, 0);
+    const passedCount = details.filter((d) => d.passed).length;
+    const ratio = totalWeight > 0 ? passedWeight / totalWeight : 0;
+    return {
+        passed: passedCount,
+        total: details.length,
+        score: ratioToScore(ratio),
+        details,
+    };
+}
+export async function runAsyncAssertions(output, assertions, { executor, judgeModel, sample, samplesDir }) {
+    const details = [];
+    let asyncCostUSD = 0;
+    for (const assertion of assertions) {
+        const weight = assertion.weight ?? 1;
+        let passed = false;
+        let message = '';
+        if (assertion.type === 'semantic_similarity') {
+            const reference = assertion.reference || '';
+            const judgePrompt = [
+                '请判断以下两段文本的语义相似度。',
+                '',
+                '## 参考文本',
+                reference,
+                '',
+                '## 待评估文本',
+                output,
+                '',
+                '请返回 JSON（不要包含 markdown 代码块标记）：',
+                '{"score": <1-5的整数>, "reason": "<简短理由>"}',
+                '',
+                '评分：1=完全无关, 2=略有关联, 3=部分相似, 4=大致相同, 5=高度一致',
+            ].join('\n');
+            const result = await executor({
+                model: judgeModel,
+                system: '你是语义相似度评审员。只返回 JSON，不要其他内容。',
+                prompt: judgePrompt,
+            });
+            asyncCostUSD += result.costUSD || 0;
+            if (result.ok) {
+                try {
+                    const jsonMatch = result.output.trim().match(/\{[\s\S]*\}/);
+                    if (jsonMatch) {
+                        const parsed = JSON.parse(jsonMatch[0]);
+                        const score = Number(parsed.score) || 0;
+                        const threshold = assertion.threshold ?? 3;
+                        passed = score >= threshold;
+                        message = parsed.reason || '';
+                    }
+                    else {
+                        process.stderr.write(`[omk] semantic_similarity judge returned non-JSON: ${result.output.slice(0, 100)}\n`);
+                    }
+                }
+                catch (parseErr) {
+                    process.stderr.write(`[omk] semantic_similarity judge parse error: ${getErrorMessage(parseErr)}\n`);
+                }
+            }
+        }
+        else if (assertion.type === 'custom') {
+            try {
+                const fnPath = resolve(samplesDir, assertion.fn);
+                const mod = await import(fnPath);
+                const fn = mod.default || mod.check;
+                if (!fn)
+                    throw new Error('custom assertion module must export default or check');
+                const result = await Promise.race([
+                    fn(output, { sample, assertion }),
+                    new Promise((_, reject) => setTimeout(() => reject(new Error(`custom assertion timed out (${CUSTOM_ASSERTION_TIMEOUT_MS / 1000}s)`)), CUSTOM_ASSERTION_TIMEOUT_MS)),
+                ]);
+                passed = Boolean(result.pass);
+                message = result.message || '';
+            }
+            catch (err) {
+                passed = false;
+                message = `custom assertion error: ${getErrorMessage(err)}`;
+            }
+        }
+        details.push({
+            type: assertion.type,
+            value: assertion.reference || assertion.fn || '',
+            weight,
+            passed,
+            ...(message && { message }),
+        });
+    }
+    const totalWeight = details.reduce((s, d) => s + d.weight, 0);
+    const passedWeight = details.filter((d) => d.passed).reduce((s, d) => s + d.weight, 0);
+    const passedCount = details.filter((d) => d.passed).length;
+    const ratio = totalWeight > 0 ? passedWeight / totalWeight : 0;
+    return { passed: passedCount, total: details.length, score: ratioToScore(ratio), details, judgeCostUSD: asyncCostUSD };
+}
+//# sourceMappingURL=assertions.js.map

package/dist/src/grading/assertions.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"assertions.js","sourceRoot":"","sources":["../../../src/grading/assertions.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,IAAI,MAAM,KAAK,CAAC;AAGvB,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC;AACjC,MAAM,GAAG,GAAG,IAAI,GAAG,EAAE,CAAC;AACtB,MAAM,2BAA2B,GAAG,MAAM,CAAC;AAE3C,MAAM,CAAC,MAAM,qBAAqB,GAAG,IAAI,GAAG,CAAC,CAAC,qBAAqB,EAAE,QAAQ,CAAC,CAAC,CAAC;AA2BhF,SAAS,eAAe,CAAC,GAAY;IACnC,OAAO,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;AAC1D,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAAa;IACxC,OAAO,MAAM,CAAC,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;AAC5C,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,IAAa,EAAE,MAA+B;IAC/E,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IACvD,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACrC,OAAO,QAAQ,CAAC,IAAI,CAAY,CAAC;IACnC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,MAAc,EAAE,UAAuB,EAAE,UAAoG,EAAE;IAC3K,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC;IACzC,MAAM,OAAO,GAAsB,EAAE,CAAC;IACtC,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,EAAE,CAAC;IAC1C,MAAM,SAAS,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;IAE/D,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,IAAI,CAAC,CAAC;QACrC,IAAI,MAAM,GAAG,KAAK,CAAC;QAEnB,QAAQ,SAAS,CAAC,IAAI,EAAE,CAAC;YACvB,KAAK,UAAU;gBACb,MAAM,GAAG,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;gBACrE,MAAM;YACR,KAAK,cAAc;gBACjB,MAAM,GAAG,CAAC,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;gBACtE,MAAM;YACR,KAAK,OAAO,CAAC,CAAC,CAAC;gBACb,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,IAAI,GAAG,CAAC;gBACrC,MAAM,EAAE,GAAG,IAAI,MAAM,CAAC,SAAS,CAAC,OAAQ,EAAE,KAAK,CAAC,CAAC;gBACjD,MAAM,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACzB,MAAM;YACR,CAAC;YACD,KAAK,YAAY;gBACf,MAAM,GAAG,MAAM,CAAC,MAAM,IAAK,SAAS,CAAC,KAAgB,CAAC;gBACtD,MAAM;YACR,KAAK,YAAY;gBACf,MAAM,GAAG,MAAM,CAAC,MAAM,IAAK,SAAS,CAAC,KAAgB,CAAC;gBACtD,MAAM;YACR,KAAK,YAAY;gBACf,IAAI,CAAC;oBAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;oBAAC,MAAM,GAAG,IAAI,CAAC;gBAAC,CAAC;gBAAC,MAAM,CAAC;oBAAC,MAAM,GAAG,KAAK,CAAC;gBAAC,CAAC;gBACpE,MAAM;YACR,KAAK,aAAa;gBAChB,IAAI,CAAC;oBACH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;oBAChC,MAAM,GAAG,kBAAkB,CAAC,IAAI,EAAE,SAAS,CAAC,MAAO,CAAC,CAAC;gBACvD,CAAC;gBAAC,MAAM,CAAC;oBAAC,MAAM,GAAG,KAAK,CAAC;gBAAC,CAAC;gBAC3B,MAAM;YACR,KAAK,aAAa;gBAChB,MAAM,GAAG,WAAW,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;gBACvE,MAAM;YACR,KAAK,WAAW;gBACd,MAAM,GAAG,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;gBACrE,MAAM;YACR,KAAK,QAAQ;gBACX,MAAM,GAAG,MAAM,CAAC,IAAI,EAAE,KAAK,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;gBAC1D,MAAM;YACR,KAAK,YAAY;gBACf,MAAM,GAAG,MAAM,CAAC,IAAI,EAAE,KAAK,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;gBAC1D,MAAM;YACR,KAAK,gBAAgB;gBACnB,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,IAAK,SAAS,CAAC,KAAgB,CAAC;gBACnF,MAAM;YACR,KAAK,gBAAgB;gBACnB,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,IAAK,SAAS,CAAC,KAAgB,CAAC;gBACnF,MAAM;YACR,KAAK,cAAc;gBACjB,MAAM,GAAG,CAAC,SAAS,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;gBAC9F,MAAM;YACR,KAAK,cAAc;gBACjB,MAAM,GAAG,CAAC,SAAS,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;gBAC7F,MAAM;YACR,KAAK,UAAU;gBACb,MAAM,GAAG,CAAC,OAAO,CAAC,OAAO,IAAI,QAAQ,CAAC,IAAK,SAAS,CAAC,KAAgB,CAAC;gBACtE,MAAM;YACR,KAAK,aAAa;gBAChB,MAAM,GAAG,CAAC,OAAO,CAAC,UAAU,IAAI,QAAQ,CAAC,IAAK,SAAS,CAAC,KAAgB,CAAC;gBACzE,MAAM;YACR,KAAK,WAAW;gBACd,MAAM,GAAG,CAAC,OAAO,CAAC,QAAQ,IAAI,QAAQ,CAAC,IAAK,SAAS,CAAC,KAAgB,CAAC;gBACvE,MAAM;YACR,KAAK,WAAW;gBACd,MAAM,GAAG,CAAC,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAC,IAAK,SAAS,CAAC,KAAgB,CAAC;gBAChE,MAAM;YACR,KAAK,cAAc;gBACjB,MAAM,GAAG,CAAC,SAAS,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;gBAC5F,MAAM;YACR,KAAK,kBAAkB;gBACrB,MAAM,GAAG,CAAC,SAAS,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;gBAC7F,MAAM;YACR,KAAK,iBAAiB;gBACpB,MAAM,GAAG,SAAS,CAAC,MAAM,IAAK,SAAS,CAAC,KAAgB,CAAC;gBACzD,MAAM;YACR,KAAK,iBAAiB;gBACpB,MAAM,GAAG,SAAS,CAAC,MAAM,IAAK,SAAS,CAAC,KAAgB,CAAC;gBACzD,MAAM;YACR,KAAK,sBAAsB,CAAC,CAAC,CAAC;gBAC5B,MAAM,GAAG,GAAG,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBACjD,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC;oBACZ,MAAM,UAAU,GAAG,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;oBACvE,MAAM,QAAQ,GAAG,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;oBACtE,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAC7B,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,UAAU;wBACpC,MAAM,CAAC,EAAE,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CACzD,CAAC;gBACJ,CAAC;gBACD,MAAM;YACR,CAAC;YACD,KAAK,qBAAqB,CAAC,CAAC,CAAC;gBAC3B,MAAM,GAAG,GAAG,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBACjD,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC;oBACZ,MAAM,UAAU,GAAG,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;oBACvE,MAAM,QAAQ,GAAG,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;oBACtE,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAC7B,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,UAAU;wBACpC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAChE,CAAC;gBACJ,CAAC;gBACD,MAAM;YACR,CAAC;YACD;gBACE,MAAM,GAAG,KAAK,CAAC;QACnB,CAAC;QAED,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,SAAS,CAAC,IAAI;YACpB,KAAK,EAAE,SAAS,CAAC,KAAK,IAAI,SAAS,CAAC,OAAO,IAAI,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;YACjF,MAAM;YACN,MAAM;SACP,CAAC,CAAC;IACL,CAAC;IAED,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAC9D,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACvF,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAC3D,MAAM,KAAK,GAAG,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;IAE/D,OAAO;QACL,MAAM,EAAE,WAAW;QACnB,KAAK,EAAE,OAAO,CAAC,MAAM;QACrB,KAAK,EAAE,YAAY,CAAC,KAAK,CAAC;QAC1B,OAAO;KACR,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,MAAc,EAAE,UAAuB,EAAE,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAyB;IACnJ,MAAM,OAAO,GAAsB,EAAE,CAAC;IACtC,IAAI,YAAY,GAAG,CAAC,CAAC;IAErB,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,IAAI,CAAC,CAAC;QACrC,IAAI,MAAM,GAAG,KAAK,CAAC;QACnB,IAAI,OAAO,GAAG,EAAE,CAAC;QAEjB,IAAI,SAAS,CAAC,IAAI,KAAK,qBAAqB,EAAE,CAAC;YAC7C,MAAM,SAAS,GAAG,SAAS,CAAC,SAAS,IAAI,EAAE,CAAC;YAC5C,MAAM,WAAW,GAAG;gBAClB,kBAAkB;gBAClB,EAAE;gBACF,SAAS;gBACT,SAAS;gBACT,EAAE;gBACF,UAAU;gBACV,MAAM;gBACN,EAAE;gBACF,gCAAgC;gBAChC,yCAAyC;gBACzC,EAAE;gBACF,2CAA2C;aAC5C,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAEb,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC;gBAC5B,KAAK,EAAE,UAAU;gBACjB,MAAM,EAAE,6BAA6B;gBACrC,MAAM,EAAE,WAAW;aACpB,CAAC,CAAC;YAEH,YAAY,IAAI,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC;YACpC,IAAI,MAAM,CAAC,EAAE,EAAE,CAAC;gBACd,IAAI,CAAC;oBACH,MAAM,SAAS,GAAG,MAAM,CAAC,MAAO,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;oBAC7D,IAAI,SAAS,EAAE,CAAC;wBACd,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAkB,CAAC;wBACzD,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;wBACxC,MAAM,SAAS,GAAG,SAAS,CAAC,SAAS,IAAI,CAAC,CAAC;wBAC3C,MAAM,GAAG,KAAK,IAAI,SAAS,CAAC;wBAC5B,OAAO,GAAG,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC;oBAChC,CAAC;yBAAM,CAAC;wBACN,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,sDAAsD,MAAM,CAAC,MAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;oBAC/G,CAAC;gBACH,CAAC;gBAAC,OAAO,QAAiB,EAAE,CAAC;oBAC3B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,gDAAgD,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;gBACtG,CAAC;YACH,CAAC;QACH,CAAC;aAAM,IAAI,SAAS,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACvC,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,OAAO,CAAC,UAAU,EAAE,SAAS,CAAC,EAAG,CAAC,CAAC;gBAClD,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,MAAM,CAA0B,CAAC;gBAC1D,MAAM,EAAE,GAAG,GAAG,CAAC,OAAO,IAAI,GAAG,CAAC,KAAK,CAAC;gBACpC,IAAI,CAAC,EAAE;oBAAE,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;gBACjF,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,CAAwB;oBACvD,EAAE,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;oBACjC,IAAI,OAAO,CAAwB,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,+BAA+B,2BAA2B,GAAG,IAAI,IAAI,CAAC,CAAC,EAAE,2BAA2B,CAAC,CAAC;iBAC3L,CAAC,CAAC;gBACH,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBAC9B,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC;YACjC,CAAC;YAAC,OAAO,GAAY,EAAE,CAAC;gBACtB,MAAM,GAAG,KAAK,CAAC;gBACf,OAAO,GAAG,2BAA2B,eAAe,CAAC,GAAG,CAAC,EAAE,CAAC;YAC9D,CAAC;QACH,CAAC;QAED,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,SAAS,CAAC,IAAI;YACpB,KAAK,EAAE,SAAS,CAAC,SAAS,IAAI,SAAS,CAAC,EAAE,IAAI,EAAE;YAChD,MAAM;YACN,MAAM;YACN,GAAG,CAAC,OAAO,IAAI,EAAE,OAAO,EAAE,CAAC;SAC5B,CAAC,CAAC;IACL,CAAC;IAED,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAC9D,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACvF,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAC3D,MAAM,KAAK,GAAG,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;IAE/D,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE,YAAY,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,CAAC;AACzH,CAAC"}

package/dist/src/grading/index.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Mixed grading: deterministic assertions + LLM judge + multi-dimensional scoring.
+ */
+import type { GradeResult, ExecutorFn, Sample, ToolCallInfo, TurnInfo } from '../types.js';
+interface GradeOptions {
+    output: string;
+    sample: Sample;
+    executor: ExecutorFn;
+    judgeModel: string;
+    allowLlmJudge?: boolean;
+    execMetrics?: {
+        costUSD?: number;
+        durationMs?: number;
+        numTurns?: number;
+        toolCalls?: ToolCallInfo[];
+        turns?: TurnInfo[];
+    };
+    samplesDir?: string;
+}
+/**
+ * Grade a model output against a sample's criteria.
+ */
+export declare function grade({ output, sample, executor, judgeModel, allowLlmJudge, execMetrics, samplesDir }: GradeOptions): Promise<GradeResult>;
+export { runAssertions, validateJsonSchema } from './assertions.js';
+export { buildTraceSummary } from './judge.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/src/grading/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/grading/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAoB,WAAW,EAAkC,UAAU,EAAE,MAAM,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAK7I,UAAU,YAAY;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,UAAU,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,WAAW,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,YAAY,EAAE,CAAC;QAAC,KAAK,CAAC,EAAE,QAAQ,EAAE,CAAA;KAAE,CAAC;IAC3H,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,wBAAsB,KAAK,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,aAAoB,EAAE,WAAgB,EAAE,UAAgB,EAAE,EAAE,YAAY,GAAG,OAAO,CAAC,WAAW,CAAC,CA+FlK;AACD,OAAO,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AACpE,OAAO,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC"}

package/dist/src/grading/index.js

@@ -0,0 +1,98 @@
+/**
+ * Mixed grading: deterministic assertions + LLM judge + multi-dimensional scoring.
+ */
+import { ASYNC_ASSERTION_TYPES, ratioToScore, runAssertions, runAsyncAssertions } from './assertions.js';
+import { buildTraceSummary, llmJudge } from './judge.js';
+import { computeLayeredScores } from './layered-scores.js';
+/**
+ * Grade a model output against a sample's criteria.
+ */
+export async function grade({ output, sample, executor, judgeModel, allowLlmJudge = true, execMetrics = {}, samplesDir = '.' }) {
+    const results = { compositeScore: 0 };
+    // 1. Deterministic assertions (pure, no LLM)
+    const allAssertions = sample.assertions || [];
+    const syncAssertions = allAssertions.filter((a) => !ASYNC_ASSERTION_TYPES.has(a.type));
+    const asyncAssertions = allAssertions.filter((assertion) => {
+        if (!ASYNC_ASSERTION_TYPES.has(assertion.type))
+            return false;
+        if (allowLlmJudge)
+            return true;
+        return assertion.type === 'custom';
+    });
+    if (syncAssertions.length > 0) {
+        results.assertions = runAssertions(output, syncAssertions, { ...execMetrics });
+    }
+    // 1b. Async assertions (custom JS, semantic_similarity)
+    if (asyncAssertions.length > 0 && executor != null) {
+        const asyncResults = await runAsyncAssertions(output, asyncAssertions, {
+            executor, judgeModel, sample, samplesDir,
+        });
+        if (results.assertions) {
+            // Merge async results into sync results
+            results.assertions.details.push(...asyncResults.details);
+            results.assertions.total += asyncResults.total;
+            results.assertions.passed += asyncResults.passed;
+            // Recompute score from merged details
+            const allDetails = results.assertions.details;
+            const totalWeight = allDetails.reduce((s, d) => s + d.weight, 0);
+            const passedWeight = allDetails.filter((d) => d.passed).reduce((s, d) => s + d.weight, 0);
+            const ratio = totalWeight > 0 ? passedWeight / totalWeight : 0;
+            results.assertions.score = ratioToScore(ratio);
+        }
+        else {
+            results.assertions = asyncResults;
+        }
+        // Accumulate async assertion judge cost
+        if (asyncResults.judgeCostUSD && asyncResults.judgeCostUSD > 0) {
+            results.judgeCostUSD = (results.judgeCostUSD || 0) + asyncResults.judgeCostUSD;
+        }
+    }
+    // 2. LLM scoring
+    // Build execution trace summary for agent-aware judging
+    const traceSummary = buildTraceSummary(execMetrics.turns, execMetrics.toolCalls);
+    if (allowLlmJudge && sample.dimensions && Object.keys(sample.dimensions).length > 0) {
+        // Multi-dimensional scoring
+        results.dimensions = {};
+        for (const [dim, rubric] of Object.entries(sample.dimensions)) {
+            results.dimensions[dim] = await llmJudge({
+                output,
+                rubric,
+                prompt: sample.prompt,
+                executor,
+                model: judgeModel,
+                traceSummary,
+            });
+        }
+        const dimValues = Object.values(results.dimensions);
+        const dimScores = dimValues.map((d) => d.score).filter((s) => s > 0);
+        if (dimScores.length > 0) {
+            results.llmScore = Number((dimScores.reduce((a, b) => a + b, 0) / dimScores.length).toFixed(2));
+        }
+        // Accumulate judge cost from all dimensions (add to any existing async assertion cost)
+        const dimCost = dimValues.reduce((s, d) => s + (d.judgeCostUSD || 0), 0);
+        if (dimCost > 0)
+            results.judgeCostUSD = (results.judgeCostUSD || 0) + dimCost;
+    }
+    else if (allowLlmJudge && sample.rubric) {
+        // Single rubric scoring
+        const judge = await llmJudge({
+            output,
+            rubric: sample.rubric,
+            prompt: sample.prompt,
+            executor,
+            model: judgeModel,
+            traceSummary,
+        });
+        results.llmScore = judge.score;
+        results.llmReason = judge.reason;
+        results.judgeCostUSD = (results.judgeCostUSD || 0) + (judge.judgeCostUSD || 0);
+    }
+    // 3. Layered scores + composite
+    const { compositeScore, layeredScores } = computeLayeredScores(results);
+    results.compositeScore = compositeScore;
+    results.layeredScores = layeredScores;
+    return results;
+}
+export { runAssertions, validateJsonSchema } from './assertions.js';
+export { buildTraceSummary } from './judge.js';
+//# sourceMappingURL=index.js.map

package/dist/src/grading/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/grading/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,EAAE,qBAAqB,EAAE,YAAY,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AACzG,OAAO,EAAE,iBAAiB,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACzD,OAAO,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAY3D;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,KAAK,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,aAAa,GAAG,IAAI,EAAE,WAAW,GAAG,EAAE,EAAE,UAAU,GAAG,GAAG,EAAgB;IAC1I,MAAM,OAAO,GAQT,EAAE,cAAc,EAAE,CAAC,EAAE,CAAC;IAE1B,6CAA6C;IAC7C,MAAM,aAAa,GAAG,MAAM,CAAC,UAAU,IAAI,EAAE,CAAC;IAC9C,MAAM,cAAc,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IACvF,MAAM,eAAe,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,SAAS,EAAE,EAAE;QACzD,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC;YAAE,OAAO,KAAK,CAAC;QAC7D,IAAI,aAAa;YAAE,OAAO,IAAI,CAAC;QAC/B,OAAO,SAAS,CAAC,IAAI,KAAK,QAAQ,CAAC;IACrC,CAAC,CAAC,CAAC;IAEH,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9B,OAAO,CAAC,UAAU,GAAG,aAAa,CAAC,MAAM,EAAE,cAAc,EAAE,EAAE,GAAG,WAAW,EAAE,CAAC,CAAC;IACjF,CAAC;IAED,wDAAwD;IACxD,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;QACnD,MAAM,YAAY,GAAG,MAAM,kBAAkB,CAAC,MAAM,EAAE,eAAe,EAAE;YACrE,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU;SACzC,CAAC,CAAC;QACH,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;YACvB,wCAAwC;YACxC,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;YACzD,OAAO,CAAC,UAAU,CAAC,KAAK,IAAI,YAAY,CAAC,KAAK,CAAC;YAC/C,OAAO,CAAC,UAAU,CAAC,MAAM,IAAI,YAAY,CAAC,MAAM,CAAC;YACjD,sCAAsC;YACtC,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC;YAC9C,MAAM,WAAW,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YACjE,MAAM,YAAY,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YAC1F,MAAM,KAAK,GAAG,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;YAC/D,OAAO,CAAC,UAAU,CAAC,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;QACjD,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,UAAU,GAAG,YAAY,CAAC;QACpC,CAAC;QACD,wCAAwC;QACxC,IAAI,YAAY,CAAC,YAAY,IAAI,YAAY,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;YAC/D,OAAO,CAAC,YAAY,GAAG,CAAC,OAAO,CAAC,YAAY,IAAI,CAAC,CAAC,GAAG,YAAY,CAAC,YAAY,CAAC;QACjF,CAAC;IACH,CAAC;IAED,iBAAiB;IACjB,wDAAwD;IACxD,MAAM,YAAY,GAAG,iBAAiB,CAAC,WAAW,CAAC,KAAK,EAAE,WAAW,CAAC,SAAS,CAAC,CAAC;IAEjF,IAAI,aAAa,IAAI,MAAM,CAAC,UAAU,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpF,4BAA4B;QAC5B,OAAO,CAAC,UAAU,GAAG,EAAE,CAAC;QACxB,KAAK,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;YAC9D,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,MAAM,QAAQ,CAAC;gBACvC,MAAM;gBACN,MAAM;gBACN,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,QAAQ;gBACR,KAAK,EAAE,UAAU;gBACjB,YAAY;aACb,CAAC,CAAC;QACL,CAAC;QACD,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QACpD,MAAM,SAAS,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACrE,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzB,OAAO,CAAC,QAAQ,GAAG,MAAM,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QAClG,CAAC;QACD,uFAAuF;QACvF,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACzE,IAAI,OAAO,GAAG,CAAC;YAAE,OAAO,CAAC,YAAY,GAAG,CAAC,OAAO,CAAC,YAAY,IAAI,CAAC,CAAC,GAAG,OAAO,CAAC;IAChF,CAAC;SAAM,IAAI,aAAa,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAC1C,wBAAwB;QACxB,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC;YAC3B,MAAM;YACN,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,QAAQ;YACR,KAAK,EAAE,UAAU;YACjB,YAAY;SACb,CAAC,CAAC;QACH,OAAO,CAAC,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC;QAC/B,OAAO,CAAC,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC;QACjC,OAAO,CAAC,YAAY,GAAG,CAAC,OAAO,CAAC,YAAY,IAAI,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,YAAY,IAAI,CAAC,CAAC,CAAC;IACjF,CAAC;IAED,gCAAgC;IAChC,MAAM,EAAE,cAAc,EAAE,aAAa,EAAE,GAAG,oBAAoB,CAAC,OAAO,CAAC,CAAC;IACxE,OAAO,CAAC,cAAc,GAAG,cAAc,CAAC;IACxC,OAAO,CAAC,aAAa,GAAG,aAAa,CAAC;IAEtC,OAAO,OAAO,CAAC;AACjB,CAAC;AACD,OAAO,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AACpE,OAAO,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC"}

package/dist/src/grading/judge.d.ts

@@ -0,0 +1,13 @@
+import type { DimensionResult, ExecutorFn, ToolCallInfo, TurnInfo } from '../types.js';
+interface LlmJudgeOptions {
+    output: string;
+    rubric: string;
+    prompt: string;
+    executor: ExecutorFn;
+    model: string;
+    traceSummary?: string | null;
+}
+export declare function buildTraceSummary(turns?: TurnInfo[], toolCalls?: ToolCallInfo[]): string | null;
+export declare function llmJudge({ output, rubric, prompt, executor, model, traceSummary }: LlmJudgeOptions): Promise<DimensionResult>;
+export {};
+//# sourceMappingURL=judge.d.ts.map

package/dist/src/grading/judge.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"judge.d.ts","sourceRoot":"","sources":["../../../src/grading/judge.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,UAAU,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAOvF,UAAU,eAAe;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,UAAU,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC9B;AAMD,wBAAgB,iBAAiB,CAAC,KAAK,CAAC,EAAE,QAAQ,EAAE,EAAE,SAAS,CAAC,EAAE,YAAY,EAAE,GAAG,MAAM,GAAG,IAAI,CA6C/F;AAED,wBAAsB,QAAQ,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC,CAiDnI"}

package/dist/src/grading/judge.js

@@ -0,0 +1,98 @@
+function getErrorMessage(err) {
+    return err instanceof Error ? err.message : String(err);
+}
+export function buildTraceSummary(turns, toolCalls) {
+    if ((!turns || turns.length === 0) && (!toolCalls || toolCalls.length === 0))
+        return null;
+    const lines = [];
+    if (toolCalls && toolCalls.length > 0) {
+        lines.push(`共调用 ${toolCalls.length} 个工具：`);
+        const successCount = toolCalls.filter((tc) => tc.success).length;
+        const failureCount = toolCalls.length - successCount;
+        lines.push(`  成功 ${successCount}/${toolCalls.length}`);
+        if (failureCount > 0)
+            lines.push(`  失败 ${failureCount}/${toolCalls.length}`);
+        const dist = {};
+        for (const tc of toolCalls) {
+            dist[tc.tool] = (dist[tc.tool] || 0) + 1;
+        }
+        lines.push(`  工具分布：${Object.entries(dist).map(([k, v]) => `${k}(${v})`).join(', ')}`);
+        const failedTools = toolCalls.filter((tc) => !tc.success).map((tc) => tc.tool);
+        if (failedTools.length > 0) {
+            lines.push(`  失败工具：${[...new Set(failedTools)].join(', ')}`);
+        }
+    }
+    if (turns && turns.length > 0) {
+        lines.push('');
+        lines.push('执行轨迹摘要：');
+        const assistantTurns = turns.filter((turn) => turn.role === 'assistant').length;
+        const toolTurns = turns.filter((turn) => turn.role === 'tool').length;
+        lines.push(`  共 ${turns.length} 步（assistant ${assistantTurns} / tool ${toolTurns}）`);
+        const maxTurns = Math.min(turns.length, 10);
+        for (let i = 0; i < maxTurns; i++) {
+            const t = turns[i];
+            const preview = t.content.slice(0, 100) + (t.content.length > 100 ? '...' : '');
+            if (t.role === 'assistant' && t.toolCalls?.length) {
+                lines.push(`  [${i + 1}] assistant: 调用 ${t.toolCalls.map((tc) => tc.tool).join(', ')}`);
+            }
+            else if (t.role === 'tool') {
+                lines.push(`  [${i + 1}] tool: ${preview}`);
+            }
+            else {
+                lines.push(`  [${i + 1}] ${t.role}: ${preview}`);
+            }
+        }
+        if (turns.length > maxTurns)
+            lines.push(`  ... 还有 ${turns.length - maxTurns} 步`);
+    }
+    return lines.join('\n');
+}
+export async function llmJudge({ output, rubric, prompt, executor, model, traceSummary }) {
+    const traceSection = traceSummary
+        ? ['', '## Agent 执行过程', traceSummary, '', '请同时考虑执行过程的合理性（工具选择、步骤效率、错误恢复）。']
+        : [];
+    const judgePrompt = [
+        '请对以下 AI 输出进行质量评分。',
+        '',
+        '## 原始任务',
+        prompt,
+        '',
+        '## 评分标准',
+        rubric,
+        '',
+        '## AI 输出',
+        output,
+        ...traceSection,
+        '',
+        '请返回 JSON（不要包含 markdown 代码块标记）：',
+        '{"score": <1-5的整数>, "reason": "<简短理由>"}',
+        '',
+        '评分标准：1=完全不达标, 2=部分涉及, 3=基本达标, 4=较好, 5=优秀',
+    ].join('\n');
+    const result = await executor({
+        model,
+        system: '你是一个严格的 AI 输出质量评审员。只返回 JSON，不要其他内容。',
+        prompt: judgePrompt,
+    });
+    if (!result.ok)
+        return { score: 0, reason: `judge error: ${result.error}`, judgeCostUSD: result.costUSD };
+    try {
+        const text = result.output.trim();
+        const jsonMatch = text.match(/\{[\s\S]*\}/);
+        if (!jsonMatch) {
+            process.stderr.write(`[omk] LLM judge returned non-JSON: ${text.slice(0, 100)}\n`);
+            return { score: 0, reason: 'judge returned non-JSON', judgeCostUSD: result.costUSD };
+        }
+        const parsed = JSON.parse(jsonMatch[0]);
+        return {
+            score: Number(parsed.score) || 0,
+            reason: String(parsed.reason || ''),
+            judgeCostUSD: result.costUSD,
+        };
+    }
+    catch (parseErr) {
+        process.stderr.write(`[omk] LLM judge parse error: ${getErrorMessage(parseErr)}\n`);
+        return { score: 0, reason: 'failed to parse judge response', judgeCostUSD: result.costUSD };
+    }
+}
+//# sourceMappingURL=judge.js.map

package/dist/src/grading/judge.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"judge.js","sourceRoot":"","sources":["../../../src/grading/judge.ts"],"names":[],"mappings":"AAgBA,SAAS,eAAe,CAAC,GAAY;IACnC,OAAO,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;AAC1D,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAAkB,EAAE,SAA0B;IAC9E,IAAI,CAAC,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IAE1F,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,IAAI,SAAS,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtC,KAAK,CAAC,IAAI,CAAC,OAAO,SAAS,CAAC,MAAM,OAAO,CAAC,CAAC;QAC3C,MAAM,YAAY,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;QACjE,MAAM,YAAY,GAAG,SAAS,CAAC,MAAM,GAAG,YAAY,CAAC;QACrD,KAAK,CAAC,IAAI,CAAC,QAAQ,YAAY,IAAI,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC;QACvD,IAAI,YAAY,GAAG,CAAC;YAAE,KAAK,CAAC,IAAI,CAAC,QAAQ,YAAY,IAAI,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC;QAE7E,MAAM,IAAI,GAA2B,EAAE,CAAC;QACxC,KAAK,MAAM,EAAE,IAAI,SAAS,EAAE,CAAC;YAC3B,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QAC3C,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,UAAU,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACtF,MAAM,WAAW,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;QAC/E,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC;IAED,IAAI,KAAK,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACtB,MAAM,cAAc,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC,MAAM,CAAC;QAChF,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC;QACtE,KAAK,CAAC,IAAI,CAAC,OAAO,KAAK,CAAC,MAAM,gBAAgB,cAAc,WAAW,SAAS,GAAG,CAAC,CAAC;QACrF,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAC5C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;YAClC,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACnB,MAAM,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAChF,IAAI,CAAC,CAAC,IAAI,KAAK,WAAW,IAAI,CAAC,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC;gBAClD,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC1F,CAAC;iBAAM,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBAC7B,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,WAAW,OAAO,EAAE,CAAC,CAAC;YAC9C,CAAC;iBAAM,CAAC;gBACN,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC,CAAC;YACnD,CAAC;QACH,CAAC;QACD,IAAI,KAAK,CAAC,MAAM,GAAG,QAAQ;YAAE,KAAK,CAAC,IAAI,CAAC,YAAY,KAAK,CAAC,MAAM,GAAG,QAAQ,IAAI,CAAC,CAAC;IACnF,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,YAAY,EAAmB;IACvG,MAAM,YAAY,GAAG,YAAY;QAC/B,CAAC,CAAC,CAAC,EAAE,EAAE,eAAe,EAAE,YAAY,EAAE,EAAE,EAAE,gCAAgC,CAAC;QAC3E,CAAC,CAAC,EAAE,CAAC;IAEP,MAAM,WAAW,GAAG;QAClB,mBAAmB;QACnB,EAAE;QACF,SAAS;QACT,MAAM;QACN,EAAE;QACF,SAAS;QACT,MAAM;QACN,EAAE;QACF,UAAU;QACV,MAAM;QACN,GAAG,YAAY;QACf,EAAE;QACF,gCAAgC;QAChC,yCAAyC;QACzC,EAAE;QACF,0CAA0C;KAC3C,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAEb,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC;QAC5B,KAAK;QACL,MAAM,EAAE,qCAAqC;QAC7C,MAAM,EAAE,WAAW;KACpB,CAAC,CAAC;IAEH,IAAI,CAAC,MAAM,CAAC,EAAE;QAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,gBAAgB,MAAM,CAAC,KAAK,EAAE,EAAE,YAAY,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC;IAE1G,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,CAAC,MAAO,CAAC,IAAI,EAAE,CAAC;QACnC,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QAC5C,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,sCAAsC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;YACnF,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,yBAAyB,EAAE,YAAY,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC;QACvF,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAkB,CAAC;QACzD,OAAO;YACL,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;YAChC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC;YACnC,YAAY,EAAE,MAAM,CAAC,OAAO;SAC7B,CAAC;IACJ,CAAC;IAAC,OAAO,QAAiB,EAAE,CAAC;QAC3B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACpF,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,gCAAgC,EAAE,YAAY,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC;IAC9F,CAAC;AACH,CAAC"}

package/dist/src/grading/layered-scores.d.ts

@@ -0,0 +1,13 @@
+import type { AssertionDetail, LayeredScores } from '../types.js';
+interface CompositeInput {
+    assertions?: {
+        details?: AssertionDetail[];
+    };
+    llmScore?: number;
+}
+export declare function computeLayeredScores(results: CompositeInput): {
+    compositeScore: number;
+    layeredScores: LayeredScores;
+};
+export {};
+//# sourceMappingURL=layered-scores.d.ts.map

package/dist/src/grading/layered-scores.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"layered-scores.d.ts","sourceRoot":"","sources":["../../../src/grading/layered-scores.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AA+ClE,UAAU,cAAc;IACtB,UAAU,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,eAAe,EAAE,CAAA;KAAE,CAAC;IAC7C,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,cAAc,GAAG;IAAE,cAAc,EAAE,MAAM,CAAC;IAAC,aAAa,EAAE,aAAa,CAAA;CAAE,CAqBtH"}