oh-my-knowledge 0.48.0 → 0.49.0

package/dist/eval-core/evaluation-execution.js

@@ -4,9 +4,23 @@ import { safeSliceForJson } from '../util/safe-slice.js';
 import { grade } from '../grading/index.js';
 import { checkFacts } from './fact-checker.js';
 import { resolveExecutionStrategy } from './execution-strategy.js';
-import { DEFAULT_CACHE_DIR } from './default-dirs.js';
+import { DEFAULT_CACHE_DIR, DEFAULT_ISOLATED_CWD_DIR } from './default-dirs.js';
 import { getExecutorRuntimeFingerprint } from '../executors/runtime-fingerprint.js';
-import { resolve, dirname } from 'node:path';
+import { ownRecordValue, setOwnRecordValue, } from '../shared/record-count.js';
+import { executorResultValidationError, normalizeExecResultToolIdentities, } from '../shared/executor-result.js';
+import { hashSampleExecutionDependencies } from './sample-fingerprint.js';
+import { resolveDiagnosticTarget } from '../grading/diagnostic.js';
+import { dirname, join, resolve } from 'node:path';
+import { mkdir, mkdtemp, rm } from 'node:fs/promises';
+const PREFLIGHT_RUNTIME_LABEL_EXECUTORS = new Set([
+    'claude',
+    'claude-sdk',
+    'codex',
+    'codex-sdk',
+    'gemini',
+    'anthropic-api',
+    'openai-api',
+]);
 async function runWithConcurrency(tasks, concurrency, fn) {
     let index = 0;
     async function worker() {
@@ -28,7 +42,9 @@ function makeErrorResult(error) {
         outputTokens: 0,
         cacheReadTokens: 0,
         cacheCreationTokens: 0,
+        tokenUsageReportedByExecutor: false,
         costUSD: 0,
+        costReportedByExecutor: false,
         stopReason: 'error',
         numTurns: 0,
         error: message,
@@ -37,6 +53,24 @@ function makeErrorResult(error) {
 function sleep(ms) {
     return new Promise((resolve) => setTimeout(resolve, ms));
 }
+async function executeWithAttemptIsolation(executor, input, isolatedCwd) {
+    if (!isolatedCwd)
+        return executor(input);
+    await mkdir(DEFAULT_ISOLATED_CWD_DIR, { recursive: true });
+    const runtimeCwd = await mkdtemp(join(DEFAULT_ISOLATED_CWD_DIR, 'attempt-'));
+    try {
+        return await executor({ ...input, cwd: runtimeCwd });
+    }
+    finally {
+        try {
+            await rm(runtimeCwd, { recursive: true, force: true, maxRetries: 3, retryDelay: 50 });
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            process.stderr.write(`[omk] 无法清理 baseline 隔离目录 ${runtimeCwd}：${message}\n`);
+        }
+    }
+}
 export async function executeTasks({ tasks, executor, executorName, model, noJudge, samplesPath, samplesBaseDir, concurrency, timeoutMs, noCache, verbose, onProgress, retry = 0, existingResults, judgeRepeat = 1, judgeModels, judgeExecutors, lengthDebias = true, budget, effort, noDiagnostic = false, }) {
     const results = {};
     let started = 0;
@@ -44,17 +78,26 @@ export async function executeTasks({ tasks, executor, executorName, model, noJud
     let skipped = 0;
     let totalCostUSD = 0;
     let budgetExhausted = false;
+    if (!executorName && !noCache) {
+        throw new Error('executeTasks requires executorName when cache is enabled; '
+            + 'an anonymous executor cannot have a safe cross-run cache identity');
+    }
+    const effectiveExecutorName = executorName ?? 'custom-executor';
     // Seed results from previous run (--resume)
     if (existingResults) {
         for (const [sampleId, variants] of Object.entries(existingResults)) {
-            results[sampleId] = { ...variants };
+            setOwnRecordValue(results, sampleId, { ...variants });
+            for (const result of Object.values(variants)) {
+                if (result.ok)
+                    totalCostUSD += result.costUSD;
+            }
         }
     }
     const cacheDir = DEFAULT_CACHE_DIR;
     const cache = noCache ? null : createCache(cacheDir);
     async function executeTask(task) {
         // Skip if already have a successful result (--resume)
-        if (existingResults?.[task.sample_id]?.[task.variant]?.ok) {
+        if (ownRecordValue(ownRecordValue(existingResults ?? {}, task.sample_id) ?? {}, task.variant)?.ok) {
             skipped++;
             started++;
             completed++;
@@ -76,7 +119,6 @@ export async function executeTasks({ tasks, executor, executorName, model, noJud
         const total = tasks.length;
         onProgress?.({ phase: 'start', completed: idx, total, sample_id: task.sample_id, variant: task.variant });
         const executionPlan = resolveExecutionStrategy(task, model, timeoutMs, verbose, effort, samplesBaseDir);
-        const effectiveExecutorName = executorName || 'claude';
         const executorRuntime = getExecutorRuntimeFingerprint(effectiveExecutorName, model, {
             skillDir: executionPlan.input.skillDir,
         });
@@ -88,7 +130,7 @@ export async function executeTasks({ tasks, executor, executorName, model, noJud
         const key = cacheKey(model, executionPlan.cacheSystem, executionPlan.input.prompt, executionPlan.input.cwd, task.artifact.allowedSkills, effectiveExecutorName, executorRuntime.fingerprint, executionPlan.input.mocks, executionPlan.input.mocksStrict, effort, 
         // artifact 内容指纹进 key:本地 dir-skill 改 references/ 资产只动 contentHash,system 不变,
         // 不进 key 会命中旧输出贴到新 artifactHashes(静默污染)。
-        task.artifact.contentHash);
+        task.artifact.contentHash, hashSampleExecutionDependencies(task._sample, samplesBaseDir));
         const cached = cache?.get(key);
         const execStart = Date.now();
         if (cached) {
@@ -97,25 +139,60 @@ export async function executeTasks({ tasks, executor, executorName, model, noJud
         else {
             // Execute with retry on failure
             const maxAttempts = 1 + Math.max(0, retry);
+            let attemptCostUSD = 0;
+            let attemptCostReported = true;
+            let attemptCount = 0;
             for (let attempt = 1; attempt <= maxAttempts; attempt++) {
                 try {
-                    execResult = await executor(executionPlan.input);
+                    execResult = await executeWithAttemptIsolation(executor, executionPlan.input, executionPlan.isolatedCwd === true);
                 }
                 catch (err) {
                     execResult = makeErrorResult(err);
                 }
-                if (execResult.ok || attempt === maxAttempts)
+                const validationError = executorResultValidationError(execResult);
+                if (validationError) {
+                    execResult = makeErrorResult(`executor returned invalid result: ${validationError}`);
+                }
+                else {
+                    execResult = normalizeExecResultToolIdentities(execResult);
+                }
+                attemptCount += 1;
+                const nextAttemptCost = attemptCostUSD + execResult.costUSD;
+                if (Number.isFinite(nextAttemptCost)
+                    && nextAttemptCost >= 0
+                    && nextAttemptCost <= Number.MAX_SAFE_INTEGER) {
+                    attemptCostUSD = nextAttemptCost;
+                }
+                else {
+                    // Never turn overflow into a free execution. Saturation keeps persisted
+                    // numbers valid and remains a conservative lower bound; the completeness
+                    // flag tells reports that the exact amount is unavailable.
+                    attemptCostUSD = Number.MAX_SAFE_INTEGER;
+                    attemptCostReported = false;
+                }
+                if (execResult.costReportedByExecutor === false)
+                    attemptCostReported = false;
+                const retryWouldExceedBudget = (budget?.perSampleUSD != null
+                    && attemptCostUSD > budget.perSampleUSD) || (budget?.totalUSD != null
+                    && totalCostUSD + attemptCostUSD > budget.totalUSD);
+                if (execResult.ok || attempt === maxAttempts || retryWouldExceedBudget)
                     break;
                 // Exponential backoff before retry
                 const backoffMs = Math.min(2 ** (attempt - 1) * 1000, 30000);
                 onProgress?.({ phase: 'retry', completed: idx, total, sample_id: task.sample_id, variant: task.variant, attempt, maxAttempts });
                 await sleep(backoffMs);
             }
+            execResult = {
+                ...execResult,
+                costUSD: attemptCostUSD,
+                ...(attemptCostReported ? {} : { costReportedByExecutor: false }),
+                ...(attemptCount > 1 ? { attemptCount } : {}),
+            };
             if (cache && execResult.ok)
                 cache.set(key, execResult);
         }
         const execMs = Date.now() - execStart;
-        totalCostUSD += execResult.costUSD;
+        totalCostUSD = Math.min(Number.MAX_SAFE_INTEGER, totalCostUSD + execResult.costUSD);
         if (verbose && onProgress) {
             onProgress({
                 phase: 'exec_done',
@@ -184,35 +261,26 @@ export async function executeTasks({ tasks, executor, executorName, model, noJud
         if (execResult.ok && execResult.output && task.cwd) {
             factCheck = checkFacts(execResult.output, resolve(task.cwd));
         }
-        if (!results[task.sample_id])
-            results[task.sample_id] = {};
+        const sampleResults = ownRecordValue(results, task.sample_id)
+            ?? setOwnRecordValue(results, task.sample_id, {});
         const variantResult = buildVariantResult(execResult, gradeResult, { execMs, gradeMs, factCheck });
         // Diagnostic — 与 judge 完全独立的"哪错了 + skill 怎么改"诊断。
         // 触发条件:noDiagnostic=false + 至少 1 条 assertion fail + sample 跑成功(有 fullOutput)。
         //
-        // executor / model 选择:
-        //   - 优先用 judgeExecutors['claude'](已初始化好) + 'haiku' 模型 — 最便宜的标配。
-        //   - 用户没配 claude judge(只配了 openai / gemini 等)时,沿用第一个 judge 的
-        //     executor + 它对应的 model 名。硬写 'haiku' 会导致非 claude executor 拒绝
-        //     (model not found),诊断整段挂掉。
-        //   - 实在没有 judge executor 配置时回退主 executor(虽然不是 lean 也能跑)。
+        // executor / model 选择跟报告契约共用 resolveDiagnosticTarget:
+        // 跟随首位 judge；没有 judge 配置时才跟随主执行器。不得暗中偏爱某个 provider。
         const failedDetails = (gradeResult?.assertions?.details || []).filter((d) => !d.passed);
         const shouldDiagnose = !noDiagnostic && execResult.ok && failedDetails.length > 0;
         if (shouldDiagnose) {
+            const diagnosticStart = Date.now();
             try {
                 const { runDiagnostic } = await import('../grading/diagnostic.js');
-                // diagnostic 优先用 'claude' executor(claude 模型上跑 diagnostic 历史校准最好);
-                // 没有就用第一个可用的 judge executor 兜底。
-                const firstJudgeName = Object.keys(judgeExecutors)[0];
-                const diagExecutorName = ('claude' in judgeExecutors) ? 'claude' : firstJudgeName;
-                const diagExecutor = diagExecutorName ? judgeExecutors[diagExecutorName] : executor;
-                // 模型选择:claude executor 走 'haiku' 标配;非 claude executor 沿用它在
-                // judgeModels 里配的 model(用户已经验证可用)。
-                let diagModel = 'haiku';
-                if (diagExecutorName && diagExecutorName !== 'claude') {
-                    const judgeEntry = judgeModels.find((j) => j.executor === diagExecutorName);
-                    if (judgeEntry)
-                        diagModel = judgeEntry.model;
+                const diagnosticTarget = resolveDiagnosticTarget(judgeModels, effectiveExecutorName, model);
+                const diagExecutor = diagnosticTarget.executor === effectiveExecutorName
+                    ? executor
+                    : ownRecordValue(judgeExecutors, diagnosticTarget.executor);
+                if (!diagExecutor) {
+                    throw new Error(`diagnostic executor "${diagnosticTarget.executor}" is not registered`);
                 }
                 const diagnostic = await runDiagnostic({
                     sample: task._sample,
@@ -223,9 +291,12 @@ export async function executeTasks({ tasks, executor, executorName, model, noJud
                     fullOutput: execResult.output || undefined,
                     assertionDetails: gradeResult?.assertions?.details || [],
                     executor: diagExecutor,
-                    model: diagModel,
+                    model: diagnosticTarget.model,
                 });
                 variantResult.diagnostic = diagnostic;
+                if (diagnostic.costReportedByExecutor === false) {
+                    variantResult.judgeCostReportedByExecutor = false;
+                }
                 // diagnostic 成本三层对齐(reviewer PR#95 CR 2026-05-11 P2):
                 //   - meta.totalCostUSD 累加(下面 totalCostUSD += 这一行)
                 //   - variant summary 的 totalCostUSD / totalDiagnosticCostUSD 由 buildVariantSummary
@@ -240,6 +311,7 @@ export async function executeTasks({ tasks, executor, executorName, model, noJud
                 }
             }
             catch (err) {
+                variantResult.judgeCostReportedByExecutor = false;
                 // diagnostic 失败不影响主评测,降级成 minimal 错误对象
                 const msg = err instanceof Error ? err.message : String(err);
                 variantResult.diagnostic = {
@@ -252,6 +324,15 @@ export async function executeTasks({ tasks, executor, executorName, model, noJud
                     suggestion: { skill: '', sample: '', none: '' },
                 };
             }
+            finally {
+                const diagnosticMs = Date.now() - diagnosticStart;
+                variantResult.timing = {
+                    execMs,
+                    gradeMs,
+                    diagnosticMs,
+                    totalMs: execMs + gradeMs + diagnosticMs,
+                };
+            }
         }
         //  per-sample budget enforcement. If a sample's cost or latency
         // exceeds the per-sample cap, the result is kept (so the user can see
@@ -260,11 +341,12 @@ export async function executeTasks({ tasks, executor, executorName, model, noJud
             variantResult.ok = false;
             variantResult.error = `budget overrun: per-sample cost $${variantResult.costUSD.toFixed(4)} > cap $${budget.perSampleUSD.toFixed(4)}`;
         }
-        if (budget?.perSampleMs != null && (execMs + (gradeMs ?? 0)) > budget.perSampleMs) {
+        const sampleDurationMs = variantResult.timing?.totalMs ?? execMs + gradeMs;
+        if (budget?.perSampleMs != null && sampleDurationMs > budget.perSampleMs) {
             variantResult.ok = false;
-            variantResult.error = `budget overrun: per-sample latency ${execMs + (gradeMs ?? 0)}ms > cap ${budget.perSampleMs}ms`;
+            variantResult.error = `budget overrun: per-sample latency ${sampleDurationMs}ms > cap ${budget.perSampleMs}ms`;
         }
-        results[task.sample_id][task.variant] = variantResult;
+        setOwnRecordValue(sampleResults, task.variant, variantResult);
         completed++;
         onProgress?.({
             phase: 'done',
@@ -298,7 +380,10 @@ export async function executeTasks({ tasks, executor, executorName, model, noJud
     }
     return { results, totalCostUSD, skipped, budgetExhausted };
 }
-export async function preflight(executor, model, timeoutMs = 180000) {
+export function preflightRuntimeLabel(executorName, model) {
+    return PREFLIGHT_RUNTIME_LABEL_EXECUTORS.has(executorName) ? `${executorName}:${model}` : `custom:${model}`;
+}
+export async function preflight(executor, model, timeoutMs = 180000, label) {
     const result = await executor({
         model,
         system: '',
@@ -307,7 +392,7 @@ export async function preflight(executor, model, timeoutMs = 180000) {
         timeoutMs,
     });
     if (!result.ok) {
-        throw new Error(`preflight failed [${model}]: ${result.error}`);
+        throw new Error(`preflight failed [${label ?? model}]: ${result.error}`);
     }
 }
 /**
@@ -329,10 +414,10 @@ export async function preflightAllJudges(judgeModels, judgeExecutors, timeoutMs)
         if (seen.has(key))
             continue;
         seen.add(key);
-        const exec = judgeExecutors[jc.executor];
+        const exec = ownRecordValue(judgeExecutors, jc.executor);
         if (!exec) {
             throw new Error(`preflight: no executor registered for "${jc.executor}" (judge "${key}"); pipeline must populate judgeExecutors before preflight`);
         }
-        await preflight(exec, jc.model, timeoutMs);
+        await preflight(exec, jc.model, timeoutMs, preflightRuntimeLabel(jc.executor, jc.model));
     }
 }

package/dist/eval-core/evaluation-job.d.ts

@@ -1,5 +1,5 @@
 import type { Artifact, EvaluationErrorCategory, EvaluationJob, EvaluationRequest, EvaluationRun, JudgeConfig } from '../types/index.js';
-export declare function buildEvaluationRequest({ samplesPath, skillDir, artifacts, model, executor, noJudge, concurrency, timeoutMs, noCache, dryRun, project, owner, tags, repeat, holdoutRatio, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, effort, }: {
+export declare function buildEvaluationRequest({ samplesPath, skillDir, artifacts, model, executor, noJudge, concurrency, timeoutMs, noCache, dryRun, project, owner, tags, repeat, holdoutRatio, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, strictBaseline, effort, retry, noDiagnostic, }: {
     samplesPath: string;
     skillDir: string;
     artifacts: Artifact[];
@@ -22,7 +22,10 @@ export declare function buildEvaluationRequest({ samplesPath, skillDir, artifact
     bootstrapSamples?: number;
     lengthDebias?: boolean;
     budget?: import('../types/index.js').EvalBudget;
+    strictBaseline?: boolean;
     effort?: 'low' | 'medium' | 'high' | 'xhigh' | 'max';
+    retry?: number;
+    noDiagnostic?: boolean;
 }): EvaluationRequest;
 export declare function createEvaluationRun(runId: string, startedAt?: string): {
     run: EvaluationRun;

package/dist/eval-core/evaluation-job.js

@@ -1,7 +1,7 @@
 function nowIso() {
     return new Date().toISOString();
 }
-export function buildEvaluationRequest({ samplesPath, skillDir, artifacts, model, executor, noJudge, concurrency, timeoutMs, noCache, dryRun, project, owner, tags, repeat, holdoutRatio, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, effort, }) {
+export function buildEvaluationRequest({ samplesPath, skillDir, artifacts, model, executor, noJudge, concurrency, timeoutMs, noCache, dryRun, project, owner, tags, repeat, holdoutRatio, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, strictBaseline, effort, retry, noDiagnostic, }) {
     return {
         samplesPath,
         skillDir,
@@ -25,7 +25,10 @@ export function buildEvaluationRequest({ samplesPath, skillDir, artifacts, model
         bootstrapSamples,
         lengthDebias,
         budget,
+        strictBaseline,
         ...(effort ? { effort } : {}),
+        ...(retry && retry > 0 ? { retry } : {}),
+        ...(noDiagnostic ? { noDiagnostic: true } : {}),
     };
 }
 export function createEvaluationRun(runId, startedAt = nowIso()) {

package/dist/eval-core/evaluation-reporting.d.ts

@@ -1,15 +1,19 @@
-import type { Artifact, Report, Sample, Task, VariantResult, GitInfo, EvaluationJob, EvaluationRequest, EvaluationRun } from '../types/index.js';
+import { getExecutorRuntimeFingerprint } from '../executors/runtime-fingerprint.js';
+import type { Artifact, Report, Sample, Task, VariantResult, GitInfo, EvaluationJob, EvaluationRequest, EvaluationRun, ReportDocument } from '../types/index.js';
 export declare const DEFAULT_OUTPUT_DIR: string;
-export declare const EVALUATION_REPORT_SCHEMA_VERSION = 4;
+export declare const EVALUATION_REPORT_SCHEMA_VERSION = 5;
 export declare function hashString(str: string): string;
-/**
- * Stable content hash of a sample. Hashes the prompt + assertions + dimensions/rubric
- * (the parts that determine what's being measured). Two samples with the same hash
- * across runs measure the same thing; mismatched hashes mean the sample changed.
- */
-export declare function hashSample(sample: Sample): string;
+export { hashSample } from './sample-fingerprint.js';
 export declare function getCliVersion(): string;
 export declare function getGitInfo(): GitInfo | null;
+export declare function buildExecutorRuntimesByVariant({ variants, model, executorName, tasks, artifacts, request, }: {
+    variants: string[];
+    model: string;
+    executorName: string;
+    tasks: Task[];
+    artifacts: Artifact[];
+    request?: Pick<EvaluationRequest, 'skillDir' | 'timeoutMs'>;
+}): Record<string, ReturnType<typeof getExecutorRuntimeFingerprint>>;
 interface AggregateReportOptions {
     runId: string;
     variants: string[];
@@ -18,6 +22,7 @@ interface AggregateReportOptions {
     noJudge: boolean;
     executorName: string;
     samples: Sample[];
+    samplesBaseDir?: string;
     tasks: Task[];
     results: Record<string, Record<string, VariantResult>>;
     totalCostUSD: number;
@@ -27,10 +32,8 @@ interface AggregateReportOptions {
     job?: EvaluationJob;
     layeredStats?: boolean;
 }
-export declare function aggregateReport({ runId, variants, model, judgeModel, noJudge, executorName, samples, tasks, results, totalCostUSD, artifacts, request, run, job, layeredStats, }: AggregateReportOptions): Report;
-export interface PersistableReport {
-    id: string;
-}
+export declare function aggregateReport({ runId, variants, model, judgeModel, noJudge, executorName, samples, samplesBaseDir, tasks, results, totalCostUSD, artifacts, request, run, job, layeredStats, }: AggregateReportOptions): Report;
+export type PersistableReport = ReportDocument;
 export declare function persistReport(report: PersistableReport, outputDir: string | null): string | null;
 /**
  * run id 的时间戳后缀 `YYYYMMDDTHHmmss-rand4`。
@@ -40,4 +43,3 @@ export declare function persistReport(report: PersistableReport, outputDir: stri
  */
 export declare function runIdSuffix(): string;
 export declare function generateRunId(variants: string[]): string;
-export {};

package/dist/eval-core/evaluation-reporting.js

@@ -1,17 +1,22 @@
-import { readFileSync, writeFileSync, mkdirSync, existsSync } from 'node:fs';
+import { readFileSync, mkdirSync, existsSync } from 'node:fs';
 import { join, dirname } from 'node:path';
 import { execFileSync } from 'node:child_process';
 import { createHash } from 'node:crypto';
 import { fileURLToPath } from 'node:url';
 import { DEFAULT_REPORTS_DIR } from './default-dirs.js';
 import { indexReportWrite } from './artifact-index.js';
+import { parseReportDocument } from './report-document.js';
 import { randomRunToken, reportFilePath, runTimestamp } from './artifact-file-names.js';
 import { persistEvalGraphSidecar } from '../artifact-graph/eval.js';
 import { buildVariantSummary } from './schema.js';
 import { buildVariantConfig, resolveExecutionStrategy } from './execution-strategy.js';
 import { getJudgePromptHash } from '../grading/judge.js';
+import { getDiagnosticPromptHash, resolveDiagnosticTarget, } from '../grading/diagnostic.js';
 import { bootstrapMeanCI, bootstrapPairedDiffCI, DEFAULT_BOOTSTRAP_ALPHA, DEFAULT_BOOTSTRAP_SAMPLES, } from './bootstrap.js';
 import { getExecutorRuntimeFingerprint } from '../executors/runtime-fingerprint.js';
+import { ownRecordValue, setOwnRecordValue, } from '../shared/record-count.js';
+import { writeJsonFileAtomic } from '../shared/atomic-json.js';
+import { hashSample } from './sample-fingerprint.js';
 const __dirname = dirname(fileURLToPath(import.meta.url));
 function findPackageJson(startDir) {
     let dir = startDir;
@@ -26,39 +31,11 @@ function findPackageJson(startDir) {
 const PKG = JSON.parse(readFileSync(findPackageJson(__dirname), 'utf-8'));
 // 写报告的默认目录 = reports 单一来源(default-dirs)。保留 DEFAULT_OUTPUT_DIR 名给既有 16 处 import,值统一,杜绝写/读两端漂移。
 export const DEFAULT_OUTPUT_DIR = DEFAULT_REPORTS_DIR;
-export const EVALUATION_REPORT_SCHEMA_VERSION = 4;
+export const EVALUATION_REPORT_SCHEMA_VERSION = 5;
 export function hashString(str) {
     return createHash('sha256').update(str).digest('hex').slice(0, 12);
 }
-/**
- * Canonical (key-sorted, recursive) JSON serialization. Required for cross-run hash
- * stability — JS object key iteration order is implementation-defined for objects
- * built by spread / Object.assign / yaml.parse, so naive JSON.stringify can produce
- * different bytes for the "same" sample on different runs.
- */
-function canonicalStringify(value) {
-    if (value === null || typeof value !== 'object')
-        return JSON.stringify(value);
-    if (Array.isArray(value))
-        return '[' + value.map(canonicalStringify).join(',') + ']';
-    const entries = Object.keys(value).sort();
-    return '{' + entries.map((k) => JSON.stringify(k) + ':' + canonicalStringify(value[k])).join(',') + '}';
-}
-/**
- * Stable content hash of a sample. Hashes the prompt + assertions + dimensions/rubric
- * (the parts that determine what's being measured). Two samples with the same hash
- * across runs measure the same thing; mismatched hashes mean the sample changed.
- */
-export function hashSample(sample) {
-    const stableForm = canonicalStringify({
-        prompt: sample.prompt,
-        rubric: sample.rubric ?? null,
-        dimensions: sample.dimensions ?? null,
-        assertions: sample.assertions ?? null,
-        schema: sample.schema ?? null,
-    });
-    return hashString(stableForm);
-}
+export { hashSample } from './sample-fingerprint.js';
 export function getCliVersion() {
     return PKG.version;
 }
@@ -87,18 +64,18 @@ function commonRuntime(runtimes) {
 function representativeRuntime(runtimes) {
     return Object.values(runtimes)[0];
 }
-function buildExecutorRuntimesByVariant({ variants, model, executorName, tasks, artifacts, request, }) {
+export function buildExecutorRuntimesByVariant({ variants, model, executorName, tasks, artifacts, request, }) {
     const runtimes = {};
     for (const task of tasks) {
-        if (runtimes[task.variant])
+        if (ownRecordValue(runtimes, task.variant))
             continue;
         const executionPlan = resolveExecutionStrategy(task, model, request?.timeoutMs, false);
-        runtimes[task.variant] = getExecutorRuntimeFingerprint(executorName, model, {
+        setOwnRecordValue(runtimes, task.variant, getExecutorRuntimeFingerprint(executorName, model, {
             skillDir: executionPlan.input.skillDir,
-        });
+        }));
     }
     for (const variant of variants) {
-        if (runtimes[variant])
+        if (ownRecordValue(runtimes, variant))
             continue;
         const artifact = artifacts.find((a) => a.name === variant);
         // 与主路径 extractSkillDir 一致:dir-skill 优先隔离副本 execRoot(副本无 node_modules、PATH 不污染);
@@ -109,17 +86,19 @@ function buildExecutorRuntimesByVariant({ variants, model, executorName, tasks,
                 : artifact?.locator
                     ? dirname(artifact.locator)
                     : request?.skillDir);
-        runtimes[variant] = getExecutorRuntimeFingerprint(executorName, model, {
+        setOwnRecordValue(runtimes, variant, getExecutorRuntimeFingerprint(executorName, model, {
             skillDir: fallbackSkillDir,
-        });
+        }));
     }
     return runtimes;
 }
-export function aggregateReport({ runId, variants, model, judgeModel, noJudge, executorName, samples, tasks, results, totalCostUSD, artifacts, request, run, job, layeredStats, }) {
+export function aggregateReport({ runId, variants, model, judgeModel, noJudge, executorName, samples, samplesBaseDir, tasks, results, totalCostUSD, artifacts, request, run, job, layeredStats, }) {
     const summary = {};
     for (const variant of variants) {
-        const entries = Object.values(results).map((result) => result[variant]).filter(Boolean);
-        summary[variant] = buildVariantSummary(entries);
+        const entries = Object.values(results)
+            .map((result) => ownRecordValue(result, variant))
+            .filter((entry) => Boolean(entry));
+        setOwnRecordValue(summary, variant, buildVariantSummary(entries));
     }
     // Bootstrap CI (per-variant mean) when --bootstrap requested. Adds bootstrapCI to
     // each VariantSummary; legacy t-interval (in summary's other fields) is preserved.
@@ -131,7 +110,9 @@ export function aggregateReport({ runId, variants, model, judgeModel, noJudge, e
         // 当且仅当该样本**无任何可测层**(真·缺测,如纯评委样本且评委失败)。故 `> 0` 过滤精确剔除非测量、
         // 绝不丢"低分内容"(评委失败已在上游当缺测,不会以 0 进 composite)。下同(control / treatment)。
         for (const variant of variants) {
-            const entries = Object.values(results).map((r) => r[variant]).filter(Boolean);
+            const entries = Object.values(results)
+                .map((r) => ownRecordValue(r, variant))
+                .filter((entry) => Boolean(entry));
             const compositeScores = entries
                 .filter((e) => typeof e.compositeScore === 'number' && e.compositeScore > 0)
                 .map((e) => e.compositeScore);
@@ -155,8 +136,8 @@ export function aggregateReport({ runId, variants, model, judgeModel, noJudge, e
                 const treatmentName = variants[i];
                 const pairs = [];
                 for (const r of sampleRecords) {
-                    const c = r[controlName];
-                    const t = r[treatmentName];
+                    const c = ownRecordValue(r, controlName);
+                    const t = ownRecordValue(r, treatmentName);
                     const a = c && typeof c.compositeScore === 'number' && c.compositeScore > 0 ? c.compositeScore : undefined;
                     const b = t && typeof t.compositeScore === 'number' && t.compositeScore > 0 ? t.compositeScore : undefined;
                     if (a !== undefined && b !== undefined)
@@ -188,7 +169,7 @@ export function aggregateReport({ runId, variants, model, judgeModel, noJudge, e
     // contentHash 落在同一空间——证据可绑定的前提,也修掉「只哈 SKILL.md 正文、改资产指纹不变」的资产瞎。
     // baseline / 无 skill 记 'no-skill'。
     const artifactHashes = Object.fromEntries(artifacts.map((artifact) => [artifact.name, artifact.contentHash ?? 'no-skill']));
-    const sampleHashes = Object.fromEntries(samples.map((s) => [s.sample_id, hashSample(s)]));
+    const sampleHashes = Object.fromEntries(samples.map((sample) => [sample.sample_id, hashSample(sample, samplesBaseDir)]));
     const judgeRepeat = request?.judgeRepeat && request.judgeRepeat > 1 ? request.judgeRepeat : undefined;
     const runtimeOptions = { skillDir: request?.skillDir };
     const executorRuntimes = buildExecutorRuntimesByVariant({ variants, model, executorName, tasks, artifacts, request });
@@ -204,6 +185,17 @@ export function aggregateReport({ runId, variants, model, judgeModel, noJudge, e
         model: jc.model,
         ...(noJudge ? {} : { runtime: getExecutorRuntimeFingerprint(jc.executor, jc.model, runtimeOptions) }),
     }));
+    const diagnosticEnabled = request?.noDiagnostic !== true;
+    const diagnosticTarget = resolveDiagnosticTarget(requestJudges, executorName, model);
+    const diagnostic = diagnosticEnabled
+        ? {
+            enabled: true,
+            executor: diagnosticTarget.executor,
+            model: diagnosticTarget.model,
+            runtime: getExecutorRuntimeFingerprint(diagnosticTarget.executor, diagnosticTarget.model),
+            promptHash: getDiagnosticPromptHash(),
+        }
+        : { enabled: false };
     // length-debias is on by default; the request only sets it
     // false when the user passed --no-debias-length. The judgePromptHash differs between
     // the length-debias-on and -off prompt variants so readers can detect the divergence.
@@ -235,6 +227,7 @@ export function aggregateReport({ runId, variants, model, judgeModel, noJudge, e
             artifactHashes,
             sampleHashes,
             ...(noJudge ? {} : { judgePromptHash: getJudgePromptHash(lengthDebiasOn) }),
+            diagnostic,
             executorRuntime,
             executorRuntimes,
             judgeModels: judgeModelsMeta,
@@ -259,16 +252,22 @@ export function aggregateReport({ runId, variants, model, judgeModel, noJudge, e
             sample_id,
             variants: variantData,
         })),
-        // 用例设计快照,供单测视角渲染。只挑渲染需要的字段,跳过 cwd / allowedTools /
-        // expectedTools / dimensions / environment(对单测视图无附加价值)。Sample 字段
-        // 全选会让 report 体积接近翻倍,选子集 size 增长 ~10-20%。
+        // 用例设计快照,供单测视角与证据审计使用。执行/评分语义字段必须保留:
+        // cwd / environment / mocksStrict / allowedTools 等会改变真实构造，不能只留一个
+        // 不可解释的 sampleHash。纯扩展字段仍不盲目全选，控制报告体积。
         sampleSnapshots: Object.fromEntries(samples.map((s) => [s.sample_id, {
                 sample_id: s.sample_id,
                 prompt: s.prompt,
+                ...(s.cwd ? { cwd: s.cwd } : {}),
                 ...(s.rubric ? { rubric: s.rubric } : {}),
                 ...(s.context ? { context: s.context } : {}),
+                ...(s.dimensions && Object.keys(s.dimensions).length > 0 ? { dimensions: s.dimensions } : {}),
                 ...(s.assertions && s.assertions.length > 0 ? { assertions: s.assertions } : {}),
                 ...(s.mocks && s.mocks.length > 0 ? { mocks: s.mocks } : {}),
+                ...(s.mocksStrict !== undefined ? { mocksStrict: s.mocksStrict } : {}),
+                ...(s.environment ? { environment: s.environment } : {}),
+                ...(s.allowedTools && s.allowedTools.length > 0 ? { allowedTools: s.allowedTools } : {}),
+                ...(s.expectedTools && s.expectedTools.length > 0 ? { expectedTools: s.expectedTools } : {}),
                 ...(s.capability && s.capability.length > 0 ? { capability: s.capability } : {}),
                 ...(s.difficulty ? { difficulty: s.difficulty } : {}),
                 ...(s.construct ? { construct: s.construct } : {}),
@@ -279,7 +278,7 @@ export function aggregateReport({ runId, variants, model, judgeModel, noJudge, e
     };
 }
 function isEvaluationReport(report) {
-    return report['kind'] === 'evaluation';
+    return report.kind === 'evaluation';
 }
 function persistEvalGraphSidecarSafely(report, outputDir, sourcePath) {
     if (!isEvaluationReport(report))
@@ -298,11 +297,14 @@ export function persistReport(report, outputDir) {
     if (!existsSync(outputDir))
         mkdirSync(outputDir, { recursive: true });
     const filePath = reportFilePath(outputDir, report.id);
-    writeFileSync(filePath, JSON.stringify(report, null, 2));
-    persistEvalGraphSidecarSafely(report, outputDir, filePath);
+    const parsed = parseReportDocument(report, report.id, report.id);
+    if (!parsed)
+        throw new Error('invalid report');
+    writeJsonFileAtomic(filePath, parsed);
+    persistEvalGraphSidecarSafely(parsed, outputDir, filePath);
     // 产物发现索引:报告落项目本地后,best-effort 追加全局轻卡片,让 omk studio 跨项目聚合成机器级总览。
     // 永不抛、永不阻断报告落盘(正文是 source of truth)。
-    indexReportWrite(report, filePath, outputDir);
+    indexReportWrite(parsed, filePath, outputDir);
     return filePath;
 }
 /**

package/dist/eval-core/execution-strategy.d.ts

@@ -3,6 +3,8 @@ export interface ExecutionPlan {
     strategy: ExecutionStrategyKind;
     cacheSystem: string;
     input: ExecutorInput;
+    /** Replace the logical cwd with a fresh empty directory for each attempt. */
+    isolatedCwd?: boolean;
 }
 export declare function resolveArtifactExecutionStrategy(artifact: Artifact): ExecutionStrategyKind;
 export declare function resolveExperimentType(artifact: Artifact): ExperimentType;