@lesliechan721/agent-skill-eval 0.1.0-beta.7 → 0.1.0-beta.9

package/dist/eval-result.js

@@ -0,0 +1,144 @@
+import { stable } from './common.js';
+import { readEvalContractFile } from './eval-file.js';
+import { validateJsonSchema } from './json-schema.js';
+import { getSchema } from './schemas.js';
+import { assertEvalProviderSuiteKind } from './eval-suite-binding.js';
+export function evalResultVariantsForSpec(spec) {
+    if (spec.baseline.type === 'without-skill')
+        return ['current', 'without-skill'];
+    if (spec.baseline.type === 'previous-version')
+        return ['current', 'previous-version'];
+    return ['current'];
+}
+export function createEvalResultBinding(calculated, auditDigest, selectedCases) {
+    assertEvalProviderSuiteKind(calculated.spec.provider, selectedCases.suite.kind);
+    return {
+        specDigest: calculated.specDigest,
+        auditDigest,
+        targetDigest: calculated.targetDigest,
+        caseDigest: calculated.caseDigest,
+        provider: calculated.spec.provider,
+        selectedCaseIds: selectedCases.cases.map((entry) => entry.id),
+        allowedVariants: evalResultVariantsForSpec(calculated.spec),
+        runs: calculated.spec.runs,
+    };
+}
+export function computeEvalResultSummary(runs) {
+    const summary = { passed: 0, failed: 0, timeout: 0, error: 0 };
+    for (const run of runs) {
+        if (run.status === 'timeout')
+            summary.timeout += 1;
+        else if (run.status === 'failed')
+            summary.error += 1;
+        else if (run.grade?.passed)
+            summary.passed += 1;
+        else
+            summary.failed += 1;
+    }
+    return summary;
+}
+export async function validateEvalResult(value, binding) {
+    const errors = validateJsonSchema(await getSchema('eval-result'), value);
+    if (errors.length > 0)
+        return { valid: false, errors, calculated: null };
+    const result = value;
+    const leapSecond = /:60(?=(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})$)/iu;
+    const timestamp = (value) => {
+        const leap = leapSecond.test(value);
+        const parsed = Date.parse(leap ? value.replace(leapSecond, ':59') : value);
+        return leap ? parsed + 1_000 : parsed;
+    };
+    if (timestamp(result.finishedAt) < timestamp(result.startedAt))
+        errors.push('Result finishedAt must not be earlier than startedAt');
+    for (const [index, run] of result.runs.entries()) {
+        if (run.status === 'completed') {
+            if (run.grade === null)
+                errors.push(`run ${index} is completed but has no grade`);
+            if (run.error !== null)
+                errors.push(`run ${index} is completed but includes an error`);
+        }
+        else {
+            if (run.grade !== null)
+                errors.push(`run ${index} has status ${run.status} but includes a grade`);
+            if (run.error === null)
+                errors.push(`run ${index} has status ${run.status} but has no error`);
+        }
+        if (typeof result.provider === 'object'
+            && result.provider.type === 'codex-workflow'
+            && run.status === 'completed') {
+            if (run.usage === null)
+                errors.push(`run ${index} codex-workflow has no usage evidence`);
+            const expected = [
+                { path: 'workflow-evidence.json', role: 'evidence' },
+                { path: 'workflow-trace.json', role: 'trace' },
+                { path: 'model-output.txt', role: 'output' },
+                { path: 'provider-result.json', role: 'result' },
+            ];
+            if (!run.artifacts
+                || stable([...run.artifacts].sort((left, right) => left.path.localeCompare(right.path)))
+                    !== stable(expected.sort((left, right) => left.path.localeCompare(right.path))))
+                errors.push(`run ${index} codex-workflow artifacts are incomplete`);
+        }
+        else if (run.artifacts !== undefined) {
+            errors.push(`run ${index} must not include codex-workflow artifacts`);
+        }
+    }
+    const summary = computeEvalResultSummary(result.runs);
+    if (stable(summary) !== stable(result.summary))
+        errors.push('Result summary does not match runs');
+    if (binding) {
+        for (const key of ['specDigest', 'auditDigest', 'targetDigest', 'caseDigest']) {
+            if (result[key] !== binding[key])
+                errors.push(`Result ${key} does not match its binding`);
+        }
+        if (stable(result.provider) !== stable(binding.provider))
+            errors.push('Result provider does not match its binding');
+        const selectedCaseIds = new Set(binding.selectedCaseIds);
+        const allowedVariants = new Set(binding.allowedVariants);
+        const observedRuns = new Set();
+        for (const [index, run] of result.runs.entries()) {
+            if (!selectedCaseIds.has(run.caseId)) {
+                errors.push(`run ${index} caseId is not selected by its binding`);
+            }
+            if (!allowedVariants.has(run.variant)) {
+                errors.push(`run ${index} variant is not enabled by its binding`);
+            }
+            if (run.attempt > binding.runs) {
+                errors.push(`run ${index} attempt exceeds its binding`);
+            }
+            const identity = JSON.stringify([run.caseId, run.variant, run.attempt]);
+            if (observedRuns.has(identity)) {
+                errors.push(`run ${index} duplicates a case, variant, and attempt tuple`);
+            }
+            observedRuns.add(identity);
+        }
+    }
+    return { valid: errors.length === 0, errors, calculated: errors.length === 0 ? result : null };
+}
+export async function parseEvalResult(value) {
+    const validation = await validateEvalResult(value);
+    if (!validation.valid || !validation.calculated)
+        throw new Error(validation.errors.join('; '));
+    return validation.calculated;
+}
+export async function validateEvalResultFile(resultPath, binding) {
+    try {
+        const file = await readEvalContractFile(resultPath, 'EvalResult');
+        const validation = await validateEvalResult(file.value, binding);
+        if (!validation.valid || !validation.calculated)
+            return { valid: false, errors: validation.errors, calculated: null };
+        return {
+            valid: true,
+            errors: [],
+            calculated: { result: validation.calculated, resultDigest: file.digest },
+        };
+    }
+    catch (error) {
+        return {
+            valid: false,
+            errors: [`EvalResult is unavailable or invalid: ${error.message}`],
+            calculated: null,
+        };
+    }
+}
+//# sourceMappingURL=eval-result.js.map

package/dist/eval-result.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-result.js","sourceRoot":"","sources":["../src/eval-result.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAA;AACpC,OAAO,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAA;AAErD,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AACrD,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAA;AAExC,OAAO,EAAE,2BAA2B,EAAE,MAAM,yBAAyB,CAAA;AA8DrE,MAAM,UAAU,yBAAyB,CAAC,IAAc;IACtD,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,KAAK,eAAe;QAAE,OAAO,CAAC,SAAS,EAAE,eAAe,CAAC,CAAA;IAC/E,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,KAAK,kBAAkB;QAAE,OAAO,CAAC,SAAS,EAAE,kBAAkB,CAAC,CAAA;IACrF,OAAO,CAAC,SAAS,CAAC,CAAA;AACpB,CAAC;AAED,MAAM,UAAU,uBAAuB,CACrC,UAA8B,EAC9B,WAAmB,EACnB,aAA6D;IAE7D,2BAA2B,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IAC/E,OAAO;QACL,UAAU,EAAE,UAAU,CAAC,UAAU;QACjC,WAAW;QACX,YAAY,EAAE,UAAU,CAAC,YAAY;QACrC,UAAU,EAAE,UAAU,CAAC,UAAU;QACjC,QAAQ,EAAE,UAAU,CAAC,IAAI,CAAC,QAAQ;QAClC,eAAe,EAAE,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC;QAC7D,eAAe,EAAE,yBAAyB,CAAC,UAAU,CAAC,IAAI,CAAC;QAC3D,IAAI,EAAE,UAAU,CAAC,IAAI,CAAC,IAAI;KAC3B,CAAA;AACH,CAAC;AAED,MAAM,UAAU,wBAAwB,CAAC,IAA8B;IACrE,MAAM,OAAO,GAAsB,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAA;IACjF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS;YAAE,OAAO,CAAC,OAAO,IAAI,CAAC,CAAA;aAC7C,IAAI,GAAG,CAAC,MAAM,KAAK,QAAQ;YAAE,OAAO,CAAC,KAAK,IAAI,CAAC,CAAA;aAC/C,IAAI,GAAG,CAAC,KAAK,EAAE,MAAM;YAAE,OAAO,CAAC,MAAM,IAAI,CAAC,CAAA;;YAC1C,OAAO,CAAC,MAAM,IAAI,CAAC,CAAA;IAC1B,CAAC;IACD,OAAO,OAAO,CAAA;AAChB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,KAAc,EACd,OAA2B;IAE3B,MAAM,MAAM,GAAG,kBAAkB,CAAC,MAAM,SAAS,CAAC,aAAa,CAAC,EAAE,KAAK,CAAC,CAAA;IACxE,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,CAAA;IACxE,MAAM,MAAM,GAAG,KAAmB,CAAA;IAClC,MAAM,UAAU,GAAG,2CAA2C,CAAA;IAC9D,MAAM,SAAS,GAAG,CAAC,KAAa,EAAU,EAAE;QAC1C,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QACnC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAA;QAC1E,OAAO,IAAI,CAAC,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,MAAM,CAAA;IACvC,CAAC,CAAA;IACD,IAAI,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,SAAS,CAAC;QAC5D,MAAM,CAAC,IAAI,CAAC,sDAAsD,CAAC,CAAA;IACrE,KAAK,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;QACjD,IAAI,GAAG,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;YAC/B,IAAI,GAAG,CAAC,KAAK,KAAK,IAAI;gBAAE,MAAM,CAAC,IAAI,CAAC,OAAO,KAAK,gCAAgC,CAAC,CAAA;YACjF,IAAI,GAAG,CAAC,KAAK,KAAK,IAAI;gBAAE,MAAM,CAAC,IAAI,CAAC,OAAO,KAAK,qCAAqC,CAAC,CAAA;QACxF,CAAC;aAAM,CAAC;YACN,IAAI,GAAG,CAAC,KAAK,KAAK,IAAI;gBAAE,MAAM,CAAC,IAAI,CAAC,OAAO,KAAK,eAAe,GAAG,CAAC,MAAM,uBAAuB,CAAC,CAAA;YACjG,IAAI,GAAG,CAAC,KAAK,KAAK,IAAI;gBAAE,MAAM,CAAC,IAAI,CAAC,OAAO,KAAK,eAAe,GAAG,CAAC,MAAM,mBAAmB,CAAC,CAAA;QAC/F,CAAC;QACD,IACE,OAAO,MAAM,CAAC,QAAQ,KAAK,QAAQ;eAChC,MAAM,CAAC,QAAQ,CAAC,IAAI,KAAK,gBAAgB;eACzC,GAAG,CAAC,MAAM,KAAK,WAAW,EAC7B,CAAC;YACD,IAAI,GAAG,CAAC,KAAK,KAAK,IAAI;gBAAE,MAAM,CAAC,IAAI,CAAC,OAAO,KAAK,uCAAuC,CAAC,CAAA;YACxF,MAAM,QAAQ,GAAG;gBACf,EAAE,IAAI,EAAE,wBAAwB,EAAE,IAAI,EAAE,UAAU,EAAE;gBACpD,EAAE,IAAI,EAAE,qBAAqB,EAAE,IAAI,EAAE,OAAO,EAAE;gBAC9C,EAAE,IAAI,EAAE,kBAAkB,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAC5C,EAAE,IAAI,EAAE,sBAAsB,EAAE,IAAI,EAAE,QAAQ,EAAE;aACjD,CAAA;YACD,IACE,CAAC,GAAG,CAAC,SAAS;mBACX,MAAM,CAAC,CAAC,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;wBAClF,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;gBACjF,MAAM,CAAC,IAAI,CAAC,OAAO,KAAK,0CAA0C,CAAC,CAAA;QACvE,CAAC;aAAM,IAAI,GAAG,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;YACvC,MAAM,CAAC,IAAI,CAAC,OAAO,KAAK,4CAA4C,CAAC,CAAA;QACvE,CAAC;IACH,CAAC;IACD,MAAM,OAAO,GAAG,wBAAwB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;IACrD,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC;QAAE,MAAM,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAA;IACjG,IAAI,OAAO,EAAE,CAAC;QACZ,KAAK,MAAM,GAAG,IAAI,CAAC,YAAY,EAAE,aAAa,EAAE,cAAc,EAAE,YAAY,CAAU,EAAE,CAAC;YACvF,IAAI,MAAM,CAAC,GAAG,CAAC,KAAK,OAAO,CAAC,GAAG,CAAC;gBAAE,MAAM,CAAC,IAAI,CAAC,UAAU,GAAG,6BAA6B,CAAC,CAAA;QAC3F,CAAC;QACD,IAAI,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC;YACtD,MAAM,CAAC,IAAI,CAAC,4CAA4C,CAAC,CAAA;QAC3D,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,eAAe,CAAC,CAAA;QACxD,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,eAAe,CAAC,CAAA;QACxD,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,CAAA;QACtC,KAAK,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;YACjD,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;gBACrC,MAAM,CAAC,IAAI,CAAC,OAAO,KAAK,wCAAwC,CAAC,CAAA;YACnE,CAAC;YACD,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;gBACtC,MAAM,CAAC,IAAI,CAAC,OAAO,KAAK,wCAAwC,CAAC,CAAA;YACnE,CAAC;YACD,IAAI,GAAG,CAAC,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;gBAC/B,MAAM,CAAC,IAAI,CAAC,OAAO,KAAK,8BAA8B,CAAC,CAAA;YACzD,CAAC;YACD,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAA;YACvE,IAAI,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC/B,MAAM,CAAC,IAAI,CAAC,OAAO,KAAK,gDAAgD,CAAC,CAAA;YAC3E,CAAC;YACD,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;QAC5B,CAAC;IACH,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;AAChG,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,KAAc;IAClD,MAAM,UAAU,GAAG,MAAM,kBAAkB,CAAC,KAAK,CAAC,CAAA;IAClD,IAAI,CAAC,UAAU,CAAC,KAAK,IAAI,CAAC,UAAU,CAAC,UAAU;QAAE,MAAM,IAAI,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAA;IAC9F,OAAO,UAAU,CAAC,UAAU,CAAA;AAC9B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,UAAkB,EAClB,OAA2B;IAE3B,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,oBAAoB,CAAC,UAAU,EAAE,YAAY,CAAC,CAAA;QACjE,MAAM,UAAU,GAAG,MAAM,kBAAkB,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,CAAC,CAAA;QAChE,IAAI,CAAC,UAAU,CAAC,KAAK,IAAI,CAAC,UAAU,CAAC,UAAU;YAC7C,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,CAAA;QACtE,OAAO;YACL,KAAK,EAAE,IAAI;YACX,MAAM,EAAE,EAAE;YACV,UAAU,EAAE,EAAE,MAAM,EAAE,UAAU,CAAC,UAAU,EAAE,YAAY,EAAE,IAAI,CAAC,MAAM,EAAE;SACzE,CAAA;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO;YACL,KAAK,EAAE,KAAK;YACZ,MAAM,EAAE,CAAC,yCAA0C,KAAe,CAAC,OAAO,EAAE,CAAC;YAC7E,UAAU,EAAE,IAAI;SACjB,CAAA;IACH,CAAC;AACH,CAAC"}

package/dist/eval-run.d.ts

@@ -0,0 +1,23 @@
+import { type CustomProcessExecutorOptions, type EvalProviderRunResult } from './custom-process.js';
+import { type EvalResult } from './eval-result.js';
+import { type BuiltinEvalProviderDependencies, type BuiltinEvalProviderRunInput } from './providers.js';
+import { type CodexWorkflowExecutorOptions } from './codex-workflow.js';
+import { type SelectedCasesDocument, type SelectedEvalCase } from './selected-cases.js';
+import type { Digest } from './types.js';
+export interface RunEvalSpecOptions {
+    auditPath: string;
+    signal?: AbortSignal;
+    sourceEnv?: NodeJS.ProcessEnv;
+    builtinProviderDependencies?: BuiltinEvalProviderDependencies;
+    runBuiltinProvider?: (id: string, input: BuiltinEvalProviderRunInput, dependencies?: BuiltinEvalProviderDependencies) => Promise<EvalProviderRunResult>;
+    executeCustomProvider?: (options: CustomProcessExecutorOptions) => Promise<EvalProviderRunResult>;
+    executeCodexWorkflowProvider?: (options: CodexWorkflowExecutorOptions) => Promise<EvalProviderRunResult>;
+}
+export interface RunEvalSpecResult {
+    valid: true;
+    resultPath: string;
+    resultDigest: Digest;
+    result: EvalResult;
+}
+export declare function adaptConversationProviderCase(document: SelectedCasesDocument, evalCase: SelectedEvalCase): Record<string, unknown>;
+export declare function runEvalSpec(specPath: string, options: RunEvalSpecOptions): Promise<RunEvalSpecResult>;