@mingyuxu/check 0.1.0

package/dist/forge/evaluate-scenario.d.ts

@@ -0,0 +1,17 @@
+import type { LLMLike } from "../workflows/context.js";
+export declare function evaluateScenario(workspace: string, scenarioKey: string, opts?: {
+    llm?: LLMLike | null;
+    suite?: string;
+    projectRoot?: string;
+    gate?: boolean;
+    regressionTolerance?: number;
+    now?: number;
+}): Promise<Record<string, unknown>>;
+/** 用原 EvaluationSuite 重放；pack 版本不一致时拒绝伪装成可复现结果。 */
+export declare function replayEvaluation(workspace: string, scenarioKey: string, evaluationRunId: string, opts?: {
+    llm?: LLMLike | null;
+    gate?: boolean;
+    now?: number;
+    allowPackChange?: boolean;
+}): Promise<Record<string, unknown>>;
+//# sourceMappingURL=evaluate-scenario.d.ts.map

package/dist/forge/evaluate-scenario.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluate-scenario.d.ts","sourceRoot":"","sources":["../../src/forge/evaluate-scenario.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAKvD,wBAAsB,gBAAgB,CACpC,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,EACnB,IAAI,GAAE;IAAE,GAAG,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,OAAO,CAAC;IAAC,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAAC,GAAG,CAAC,EAAE,MAAM,CAAA;CAAO,GACpI,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CA0DlC;AAED,mDAAmD;AACnD,wBAAsB,gBAAgB,CACpC,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,EACnB,eAAe,EAAE,MAAM,EACvB,IAAI,GAAE;IAAE,GAAG,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC;IAAC,IAAI,CAAC,EAAE,OAAO,CAAC;IAAC,GAAG,CAAC,EAAE,MAAM,CAAC;IAAC,eAAe,CAAC,EAAE,OAAO,CAAA;CAAO,GAC3F,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAYlC"}

package/dist/forge/evaluate-scenario.js

@@ -0,0 +1,93 @@
+/**
+ * 完整 evaluateScenario —— 对冻结生产线跑 oracle 打分 + 回归门（load_pack 已就位）。
+ * 从 Python evaluate.py 组装（scoreAndGate + loadPack）。
+ */
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { createHash } from "node:crypto";
+import { loadPack } from "../knowledge/loader.js";
+import { scoreAndGate } from "./evaluate.js";
+import { loadBuiltinComponents } from "../nodes/base.js";
+import { buildEvaluationContracts } from "./contracts.js";
+import { recordEvaluationContracts } from "../ontology/runtime-graph.js";
+import { attributeEvaluation, latestAttemptDiff } from "./attribution.js";
+export async function evaluateScenario(workspace, scenarioKey, opts = {}) {
+    await loadBuiltinComponents();
+    const home = join(workspace, scenarioKey);
+    const packDir = join(home, "pack");
+    if (!existsSync(join(packDir, "pack.json")))
+        throw new Error(`场景 ${scenarioKey} 还没有冻结生产线，无可评测。`);
+    const pack = loadPack(packDir);
+    let oraclePath = join(packDir, "oracle", "gold.json");
+    let caseIds = null;
+    let suiteName = "full";
+    if (opts.suite) {
+        let suitePath = opts.suite;
+        if (opts.projectRoot && !suitePath.startsWith("/"))
+            suitePath = join(opts.projectRoot, suitePath);
+        const spec = JSON.parse(readFileSync(suitePath, "utf-8"));
+        suiteName = spec.name ?? suitePath.split("/").pop().replace(/\.json$/, "");
+        if (spec.oracle ?? spec.oracle_ref) {
+            oraclePath = spec.oracle ?? spec.oracle_ref;
+            if (opts.projectRoot && !oraclePath.startsWith("/"))
+                oraclePath = join(opts.projectRoot, oraclePath);
+        }
+        caseIds = spec.case_ids ?? null;
+    }
+    if (!existsSync(oraclePath))
+        throw new Error(`没有 oracle（${oraclePath}）——冻结时未带标注，或套件路径不对。`);
+    let oracle = JSON.parse(readFileSync(oraclePath, "utf-8"));
+    if (caseIds) {
+        oracle = { ...oracle, cases: oracle.cases.filter((c) => caseIds.includes(c.case_id)) };
+        if (!oracle.cases.length)
+            throw new Error(`套件 case_ids 与 oracle 无交集: ${caseIds}`);
+    }
+    const startedAt = Math.floor(opts.now ?? Date.now());
+    const result = await scoreAndGate(pack.executor("", { llm: opts.llm }), pack.targets, oracle, {
+        scenarioKey, suite: suiteName, historyPath: join(home, "evals", "history.jsonl"),
+        gate: opts.gate, regressionTolerance: opts.regressionTolerance,
+    });
+    const attributions = attributeEvaluation(result);
+    const attemptDiff = latestAttemptDiff(home);
+    const packVersion = fileDigest(join(packDir, "freeze_manifest.json"));
+    const contracts = buildEvaluationContracts({
+        scenarioKey, suiteName, oraclePath, oracle, caseIds: oracle.cases.map((row) => row.case_id), packRef: packDir, packVersion, attributions,
+        result, startedAt, completedAt: Math.floor(opts.now ?? Date.now()),
+    });
+    const evalDir = join(home, "evals", contracts.run.eval_run_id);
+    mkdirSync(evalDir, { recursive: true });
+    writeFileSync(join(evalDir, "suite.json"), JSON.stringify(contracts.suite, null, 2), "utf-8");
+    writeFileSync(join(evalDir, "run.json"), JSON.stringify(contracts.run, null, 2), "utf-8");
+    writeFileSync(join(evalDir, "report.json"), JSON.stringify(contracts.report, null, 2), "utf-8");
+    writeFileSync(join(evalDir, "failures.json"), JSON.stringify(contracts.failures, null, 2), "utf-8");
+    writeFileSync(join(evalDir, "attribution.json"), JSON.stringify(attributions, null, 2), "utf-8");
+    writeFileSync(join(evalDir, "attempt_diff.json"), JSON.stringify(attemptDiff, null, 2), "utf-8");
+    const ontology = recordEvaluationContracts(home, contracts);
+    writeFileSync(join(evalDir, "ontology_receipt.json"), JSON.stringify(ontology, null, 2), "utf-8");
+    return {
+        ...result,
+        evaluation_run_id: contracts.run.eval_run_id,
+        evaluation_contract_version: contracts.run.schema_version,
+        evaluation_dir: evalDir,
+        ontology,
+    };
+}
+/** 用原 EvaluationSuite 重放；pack 版本不一致时拒绝伪装成可复现结果。 */
+export async function replayEvaluation(workspace, scenarioKey, evaluationRunId, opts = {}) {
+    const home = join(workspace, scenarioKey);
+    const priorDir = join(home, "evals", evaluationRunId);
+    const priorRun = JSON.parse(readFileSync(join(priorDir, "run.json"), "utf-8"));
+    const priorReport = JSON.parse(readFileSync(join(priorDir, "report.json"), "utf-8"));
+    const currentPackVersion = fileDigest(join(home, "pack", "freeze_manifest.json"));
+    if (!opts.allowPackChange && priorRun.pack_version && priorRun.pack_version !== currentPackVersion) {
+        throw new Error(`无法严格重放：当前 pack=${currentPackVersion}，原评测 pack=${priorRun.pack_version}`);
+    }
+    const replay = await evaluateScenario(workspace, scenarioKey, { llm: opts.llm, suite: join(priorDir, "suite.json"), gate: opts.gate, now: opts.now });
+    const same = ["accuracy", "precision", "recall"].every((key) => normalize(replay[key]) === normalize(priorReport[key]));
+    return { ...replay, replay_of: evaluationRunId, replay_match: same };
+}
+function fileDigest(path) {
+    return createHash("sha256").update(readFileSync(path)).digest("hex").slice(0, 16);
+}
+function normalize(value) { return value == null ? "null" : JSON.stringify(value); }
+//# sourceMappingURL=evaluate-scenario.js.map

package/dist/forge/evaluate-scenario.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluate-scenario.js","sourceRoot":"","sources":["../../src/forge/evaluate-scenario.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAC7E,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AAGzD,OAAO,EAAE,wBAAwB,EAAE,MAAM,gBAAgB,CAAC;AAC1D,OAAO,EAAE,yBAAyB,EAAE,MAAM,8BAA8B,CAAC;AACzE,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAE1E,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,SAAiB,EACjB,WAAmB,EACnB,OAAmI,EAAE;IAErI,MAAM,qBAAqB,EAAE,CAAC;IAC9B,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;IAC1C,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACnC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,MAAM,WAAW,iBAAiB,CAAC,CAAC;IACjG,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC;IAE/B,IAAI,UAAU,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,WAAW,CAAC,CAAC;IACtD,IAAI,OAAO,GAAoB,IAAI,CAAC;IACpC,IAAI,SAAS,GAAG,MAAM,CAAC;IACvB,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;QACf,IAAI,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC;QAC3B,IAAI,IAAI,CAAC,WAAW,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;QAClG,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC;QAC1D,SAAS,GAAG,IAAI,CAAC,IAAI,IAAI,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAG,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QAC5E,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACnC,UAAU,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,CAAC;YAC5C,IAAI,IAAI,CAAC,WAAW,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,GAAG,CAAC;gBAAE,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,UAAU,CAAC,CAAC;QACvG,CAAC;QACD,OAAO,GAAG,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC;IAClC,CAAC;IACD,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,aAAa,UAAU,qBAAqB,CAAC,CAAC;IAE3F,IAAI,MAAM,GAAW,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC;IACnE,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,GAAG,EAAE,GAAG,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;QACxF,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,OAAO,EAAE,CAAC,CAAC;IACpF,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IACrD,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE;QAC5F,WAAW,EAAE,KAAK,EAAE,SAAS,EAAE,WAAW,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,eAAe,CAAC;QAChF,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,mBAAmB,EAAE,IAAI,CAAC,mBAAmB;KAC/D,CAAC,CAAC;IACH,MAAM,YAAY,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC;IACjD,MAAM,WAAW,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;IAC5C,MAAM,WAAW,GAAG,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,sBAAsB,CAAC,CAAC,CAAC;IACtE,MAAM,SAAS,GAAG,wBAAwB,CAAC;QACzC,WAAW,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,YAAY;QACxI,MAAM,EAAE,SAAS,EAAE,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;KACnE,CAAC,CAAC;IACH,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,SAAS,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;IAC/D,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACxC,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IAC9F,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,UAAU,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IAC1F,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,aAAa,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IAChG,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IACpG,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,kBAAkB,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IACjG,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,mBAAmB,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IACjG,MAAM,QAAQ,GAAG,yBAAyB,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;IAC5D,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,uBAAuB,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IAClG,OAAO;QACL,GAAG,MAAM;QACT,iBAAiB,EAAE,SAAS,CAAC,GAAG,CAAC,WAAW;QAC5C,2BAA2B,EAAE,SAAS,CAAC,GAAG,CAAC,cAAc;QACzD,cAAc,EAAE,OAAO;QACvB,QAAQ;KACT,CAAC;AACJ,CAAC;AAED,mDAAmD;AACnD,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,SAAiB,EACjB,WAAmB,EACnB,eAAuB,EACvB,OAA0F,EAAE;IAE5F,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;IAC1C,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,eAAe,CAAC,CAAC;IACtD,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;IAC/E,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;IACrF,MAAM,kBAAkB,GAAG,UAAU,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,sBAAsB,CAAC,CAAC,CAAC;IAClF,IAAI,CAAC,IAAI,CAAC,eAAe,IAAI,QAAQ,CAAC,YAAY,IAAI,QAAQ,CAAC,YAAY,KAAK,kBAAkB,EAAE,CAAC;QACnG,MAAM,IAAI,KAAK,CAAC,kBAAkB,kBAAkB,aAAa,QAAQ,CAAC,YAAY,EAAE,CAAC,CAAC;IAC5F,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,SAAS,EAAE,WAAW,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE,YAAY,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IACtJ,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,WAAW,EAAE,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,SAAS,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACxH,OAAO,EAAE,GAAG,MAAM,EAAE,SAAS,EAAE,eAAe,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;AACvE,CAAC;AAED,SAAS,UAAU,CAAC,IAAY;IAC9B,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AACpF,CAAC;AAED,SAAS,SAAS,CAAC,KAAc,IAAY,OAAO,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC"}

package/dist/forge/evaluate.d.ts

@@ -0,0 +1,25 @@
+import { type Oracle, type ScoreReport } from "./scorer.js";
+import type { Executor } from "../workflows/executor.js";
+import type { Target } from "../schemas/target.js";
+export interface RegressionInfo {
+    gate: true;
+    tolerance: number;
+    baseline_accuracy: number | null;
+    delta: number | null;
+    is_regression: boolean;
+    reason: string;
+    new_false_negatives: unknown[];
+}
+/** 同套件的上一条评测（回归门基线）。读在 append 之前，故最后一条即上次结果。 */
+export declare function baselineFor(historyPath: string, suiteName: string): Record<string, any> | null;
+/** 据基线计算回归判定（准确率跌破容差 或 漏报翻转 tp→fn 零容忍）。 */
+export declare function computeRegression(rep: ScoreReport, baseline: Record<string, any> | null, tolerance: number): RegressionInfo;
+/** 评分 + 可选回归门 + append 评测史。executorFactory 让 pack 加载与本文件解耦。 */
+export declare function scoreAndGate(executor: Executor, targets: Target[], oracle: Oracle, opts?: {
+    scenarioKey: string;
+    suite?: string;
+    historyPath?: string;
+    gate?: boolean;
+    regressionTolerance?: number;
+}): Promise<Record<string, unknown>>;
+//# sourceMappingURL=evaluate.d.ts.map

package/dist/forge/evaluate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluate.d.ts","sourceRoot":"","sources":["../../src/forge/evaluate.ts"],"names":[],"mappings":"AAMA,OAAO,EAAU,KAAK,MAAM,EAAE,KAAK,WAAW,EAAE,MAAM,aAAa,CAAC;AACpE,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AACzD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAEnD,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,IAAI,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,aAAa,EAAE,OAAO,CAAC;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,mBAAmB,EAAE,OAAO,EAAE,CAAC;CAChC;AAID,gDAAgD;AAChD,wBAAgB,WAAW,CAAC,WAAW,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,IAAI,CAU9F;AAED,2CAA2C;AAC3C,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,IAAI,EAAE,SAAS,EAAE,MAAM,GAAG,cAAc,CAqB3H;AAED,+DAA+D;AAC/D,wBAAsB,YAAY,CAChC,QAAQ,EAAE,QAAQ,EAClB,OAAO,EAAE,MAAM,EAAE,EACjB,MAAM,EAAE,MAAM,EACd,IAAI,GAAE;IAAE,WAAW,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,OAAO,CAAC;IAAC,mBAAmB,CAAC,EAAE,MAAM,CAAA;CAAwB,GACtI,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAkBlC"}

package/dist/forge/evaluate.js

@@ -0,0 +1,81 @@
+/**
+ * 评测 + 回归基线门 —— 从 Python evaluate.py 移植核心（load_pack 依赖 knowledge，完整
+ * evaluateScenario 在 knowledge 就绪后组装；此处提供可复用的评分+门逻辑）。
+ */
+import { existsSync, readFileSync, appendFileSync, mkdirSync } from "node:fs";
+import { dirname } from "node:path";
+import { Scorer } from "./scorer.js";
+const round3 = (x) => Math.round(x * 1000) / 1000;
+/** 同套件的上一条评测（回归门基线）。读在 append 之前，故最后一条即上次结果。 */
+export function baselineFor(historyPath, suiteName) {
+    if (!existsSync(historyPath))
+        return null;
+    let prior = null;
+    for (const line of readFileSync(historyPath, "utf-8").split("\n")) {
+        if (!line.trim())
+            continue;
+        let row;
+        try {
+            row = JSON.parse(line);
+        }
+        catch {
+            continue;
+        }
+        if (row.suite === suiteName && row.accuracy != null)
+            prior = row;
+    }
+    return prior;
+}
+/** 据基线计算回归判定（准确率跌破容差 或 漏报翻转 tp→fn 零容忍）。 */
+export function computeRegression(rep, baseline, tolerance) {
+    const reg = { gate: true, tolerance, baseline_accuracy: null, delta: null, is_regression: false, reason: "", new_false_negatives: [] };
+    if (baseline === null) {
+        reg.reason = "无同套件基线（首次评测），不判回归";
+        return reg;
+    }
+    const baseAcc = Number(baseline.accuracy);
+    reg.baseline_accuracy = baseAcc;
+    reg.delta = round3(rep.accuracy - baseAcc);
+    const reasons = [];
+    if (rep.accuracy < baseAcc - tolerance) {
+        reasons.push(`准确率 ${rep.accuracy} 跌破基线 ${baseAcc} - 容差 ${tolerance}（回退 ${Math.abs(reg.delta)}）`);
+    }
+    if (baseline.false_negatives != null && rep.false_negatives?.length) {
+        const baseFn = new Set(baseline.false_negatives.map((m) => `${m.case}|${m.target}`));
+        const flipped = rep.false_negatives.filter((m) => !baseFn.has(`${m.case}|${m.target}`));
+        if (flipped.length) {
+            reg.new_false_negatives = flipped;
+            reasons.push(`漏报翻转 ${flipped.length} 处：上次抓到的违规这次漏了 ${flipped.map((m) => `${m.case}·${m.target}`).join("、")}`);
+        }
+    }
+    if (reasons.length) {
+        reg.is_regression = true;
+        reg.reason = reasons.join("；");
+    }
+    else
+        reg.reason = `未跌破容差（Δ=${reg.delta}，基线 ${baseAcc}），无漏报翻转`;
+    return reg;
+}
+/** 评分 + 可选回归门 + append 评测史。executorFactory 让 pack 加载与本文件解耦。 */
+export async function scoreAndGate(executor, targets, oracle, opts = { scenarioKey: "" }) {
+    const suiteName = opts.suite ?? "full";
+    const baseline = opts.gate && opts.historyPath ? baselineFor(opts.historyPath, suiteName) : null;
+    const rep = await new Scorer(executor).score(targets, oracle);
+    const result = {
+        scenario_key: opts.scenarioKey, suite: suiteName,
+        accuracy: rep.accuracy, n: rep.n, llm_calls: rep.llm_calls,
+        per_target: rep.per_target, per_tier: rep.per_tier, mismatches: rep.mismatches,
+    };
+    for (const k of ["recall", "precision", "confusion", "false_negatives", "false_positives"]) {
+        if (rep[k] != null)
+            result[k] = rep[k];
+    }
+    if (opts.gate)
+        result.regression = computeRegression(rep, baseline, opts.regressionTolerance ?? 0.02);
+    if (opts.historyPath) {
+        mkdirSync(dirname(opts.historyPath), { recursive: true });
+        appendFileSync(opts.historyPath, JSON.stringify(result) + "\n", "utf-8");
+    }
+    return result;
+}
+//# sourceMappingURL=evaluate.js.map

package/dist/forge/evaluate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluate.js","sourceRoot":"","sources":["../../src/forge/evaluate.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAC9E,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,MAAM,EAAiC,MAAM,aAAa,CAAC;AAcpE,MAAM,MAAM,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC;AAE1D,gDAAgD;AAChD,MAAM,UAAU,WAAW,CAAC,WAAmB,EAAE,SAAiB;IAChE,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC;QAAE,OAAO,IAAI,CAAC;IAC1C,IAAI,KAAK,GAA+B,IAAI,CAAC;IAC7C,KAAK,MAAM,IAAI,IAAI,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QAClE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;YAAE,SAAS;QAC3B,IAAI,GAAQ,CAAC;QACb,IAAI,CAAC;YAAC,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC;YAAC,SAAS;QAAC,CAAC;QACnD,IAAI,GAAG,CAAC,KAAK,KAAK,SAAS,IAAI,GAAG,CAAC,QAAQ,IAAI,IAAI;YAAE,KAAK,GAAG,GAAG,CAAC;IACnE,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,2CAA2C;AAC3C,MAAM,UAAU,iBAAiB,CAAC,GAAgB,EAAE,QAAoC,EAAE,SAAiB;IACzG,MAAM,GAAG,GAAmB,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,iBAAiB,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,mBAAmB,EAAE,EAAE,EAAE,CAAC;IACvJ,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;QAAC,GAAG,CAAC,MAAM,GAAG,mBAAmB,CAAC;QAAC,OAAO,GAAG,CAAC;IAAC,CAAC;IACxE,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAC1C,GAAG,CAAC,iBAAiB,GAAG,OAAO,CAAC;IAChC,GAAG,CAAC,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,QAAQ,GAAG,OAAO,CAAC,CAAC;IAC3C,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,GAAG,CAAC,QAAQ,GAAG,OAAO,GAAG,SAAS,EAAE,CAAC;QACvC,OAAO,CAAC,IAAI,CAAC,OAAO,GAAG,CAAC,QAAQ,SAAS,OAAO,SAAS,SAAS,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACnG,CAAC;IACD,IAAI,QAAQ,CAAC,eAAe,IAAI,IAAI,IAAI,GAAG,CAAC,eAAe,EAAE,MAAM,EAAE,CAAC;QACpE,MAAM,MAAM,GAAG,IAAI,GAAG,CAAE,QAAQ,CAAC,eAAyB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAChG,MAAM,OAAO,GAAG,GAAG,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QACxF,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACnB,GAAG,CAAC,mBAAmB,GAAG,OAAO,CAAC;YAClC,OAAO,CAAC,IAAI,CAAC,QAAQ,OAAO,CAAC,MAAM,kBAAkB,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAChH,CAAC;IACH,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QAAC,GAAG,CAAC,aAAa,GAAG,IAAI,CAAC;QAAC,GAAG,CAAC,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAAC,CAAC;;QAC5E,GAAG,CAAC,MAAM,GAAG,WAAW,GAAG,CAAC,KAAK,OAAO,OAAO,SAAS,CAAC;IAC9D,OAAO,GAAG,CAAC;AACb,CAAC;AAED,+DAA+D;AAC/D,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,QAAkB,EAClB,OAAiB,EACjB,MAAc,EACd,OAAoH,EAAE,WAAW,EAAE,EAAE,EAAE;IAEvI,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,IAAI,MAAM,CAAC;IACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACjG,MAAM,GAAG,GAAG,MAAM,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAC9D,MAAM,MAAM,GAA4B;QACtC,YAAY,EAAE,IAAI,CAAC,WAAW,EAAE,KAAK,EAAE,SAAS;QAChD,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,SAAS,EAAE,GAAG,CAAC,SAAS;QAC1D,UAAU,EAAE,GAAG,CAAC,UAAU,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,UAAU,EAAE,GAAG,CAAC,UAAU;KAC/E,CAAC;IACF,KAAK,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,WAAW,EAAE,WAAW,EAAE,iBAAiB,EAAE,iBAAiB,CAAU,EAAE,CAAC;QACpG,IAAI,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI;YAAE,MAAM,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;IACzC,CAAC;IACD,IAAI,IAAI,CAAC,IAAI;QAAE,MAAM,CAAC,UAAU,GAAG,iBAAiB,CAAC,GAAG,EAAE,QAAQ,EAAE,IAAI,CAAC,mBAAmB,IAAI,IAAI,CAAC,CAAC;IACtG,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;QACrB,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1D,cAAc,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,IAAI,EAAE,OAAO,CAAC,CAAC;IAC3E,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/forge/feedback.d.ts

@@ -0,0 +1,21 @@
+export declare const DECISIONS: readonly ["accepted", "rejected", "triaged"];
+export interface Feedback {
+    id: string;
+    ts: number;
+    scenario: string;
+    text: string;
+    doc: string;
+    target: string;
+    expected: string;
+    status: string;
+    note?: string;
+}
+export declare function addFeedback(projectRoot: string, scenario: string, text: string, o?: {
+    doc?: string;
+    target?: string;
+    expected?: string;
+    now?: number;
+}): Feedback;
+export declare function listFeedback(projectRoot: string): Feedback[];
+export declare function triageFeedback(projectRoot: string, fbId: string, decision: string, note?: string): Feedback;
+//# sourceMappingURL=feedback.d.ts.map

package/dist/forge/feedback.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"feedback.d.ts","sourceRoot":"","sources":["../../src/forge/feedback.ts"],"names":[],"mappings":"AAQA,eAAO,MAAM,SAAS,8CAA+C,CAAC;AAGtE,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IAAC,EAAE,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IACvD,GAAG,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAC;CAC9E;AAED,wBAAgB,WAAW,CAAC,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EACnD,CAAC,GAAE;IAAE,GAAG,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAAC,GAAG,CAAC,EAAE,MAAM,CAAA;CAAO,GAAG,QAAQ,CAQhH;AAED,wBAAgB,YAAY,CAAC,WAAW,EAAE,MAAM,GAAG,QAAQ,EAAE,CAI5D;AAED,wBAAgB,cAAc,CAAC,WAAW,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,SAAK,GAAG,QAAQ,CAavG"}

package/dist/forge/feedback.js

@@ -0,0 +1,41 @@
+/**
+ * 反馈队列 —— 业务反馈只进队列，永不直改规则（I9 权限单向）。
+ * accepted 的反馈是重开五步的输入：变 oracle case → reset --keep-pack + 带新 oracle 重跑 build。
+ * 从 Python feedback.py 忠实移植。
+ */
+import { existsSync, readFileSync, writeFileSync, appendFileSync, mkdirSync } from "node:fs";
+import { join } from "node:path";
+export const DECISIONS = ["accepted", "rejected", "triaged"];
+const fbDir = (root) => join(root, "feedback");
+export function addFeedback(projectRoot, scenario, text, o = {}) {
+    const dir = fbDir(projectRoot);
+    mkdirSync(dir, { recursive: true });
+    const pending = join(dir, "pending.jsonl");
+    const count = existsSync(pending) ? readFileSync(pending, "utf-8").split("\n").filter((l) => l.trim()).length : 0;
+    const entry = { id: `fb-${String(count + 1).padStart(3, "0")}`, ts: o.now ?? Date.now() / 1000, scenario, text, doc: o.doc ?? "", target: o.target ?? "", expected: o.expected ?? "", status: "pending" };
+    appendFileSync(pending, JSON.stringify(entry) + "\n", "utf-8");
+    return entry;
+}
+export function listFeedback(projectRoot) {
+    const p = join(fbDir(projectRoot), "pending.jsonl");
+    if (!existsSync(p))
+        return [];
+    return readFileSync(p, "utf-8").split("\n").filter((l) => l.trim()).map((l) => JSON.parse(l));
+}
+export function triageFeedback(projectRoot, fbId, decision, note = "") {
+    if (!DECISIONS.includes(decision))
+        throw new Error(`decision 必须是 ${DECISIONS.join("/")}`);
+    const dir = fbDir(projectRoot);
+    const pending = join(dir, "pending.jsonl");
+    const entries = listFeedback(projectRoot);
+    const matched = entries.find((e) => e.id === fbId);
+    if (!matched)
+        throw new Error(`没有待处理反馈 ${fbId}（pending 里有: ${entries.map((e) => e.id).join(", ") || "空"}）`);
+    const entry = { ...matched, status: decision, note };
+    writeFileSync(pending, entries.filter((e) => e.id !== fbId).map((e) => JSON.stringify(e)).join("\n") + (entries.length > 1 ? "\n" : ""), "utf-8");
+    const dest = join(dir, decision);
+    mkdirSync(dest, { recursive: true });
+    writeFileSync(join(dest, `${fbId}.json`), JSON.stringify(entry, null, 2), "utf-8");
+    return entry;
+}
+//# sourceMappingURL=feedback.js.map

package/dist/forge/feedback.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"feedback.js","sourceRoot":"","sources":["../../src/forge/feedback.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,aAAa,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAC7F,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,UAAU,EAAE,UAAU,EAAE,SAAS,CAAU,CAAC;AACtE,MAAM,KAAK,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;AAOvD,MAAM,UAAU,WAAW,CAAC,WAAmB,EAAE,QAAgB,EAAE,IAAY,EACnD,IAAwE,EAAE;IACpG,MAAM,GAAG,GAAG,KAAK,CAAC,WAAW,CAAC,CAAC;IAC/B,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACpC,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,eAAe,CAAC,CAAC;IAC3C,MAAM,KAAK,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAClH,MAAM,KAAK,GAAa,EAAE,EAAE,EAAE,MAAM,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,IAAI,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,IAAI,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,IAAI,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;IACpN,cAAc,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/D,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,WAAmB;IAC9C,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,eAAe,CAAC,CAAC;IACpD,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;QAAE,OAAO,EAAE,CAAC;IAC9B,OAAO,YAAY,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AAChG,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,WAAmB,EAAE,IAAY,EAAE,QAAgB,EAAE,IAAI,GAAG,EAAE;IAC3F,IAAI,CAAE,SAA+B,CAAC,QAAQ,CAAC,QAAQ,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,gBAAgB,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACjH,MAAM,GAAG,GAAG,KAAK,CAAC,WAAW,CAAC,CAAC;IAC/B,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,eAAe,CAAC,CAAC;IAC3C,MAAM,OAAO,GAAG,YAAY,CAAC,WAAW,CAAC,CAAC;IAC1C,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC;IACnD,IAAI,CAAC,OAAO;QAAE,MAAM,IAAI,KAAK,CAAC,WAAW,IAAI,gBAAgB,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,GAAG,CAAC,CAAC;IAC5G,MAAM,KAAK,GAAG,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;IACrD,aAAa,CAAC,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,OAAO,CAAC,CAAC;IAClJ,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IACjC,SAAS,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACrC,aAAa,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,IAAI,OAAO,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IACnF,OAAO,KAAK,CAAC;AACf,CAAC"}

package/dist/forge/reflect.d.ts

@@ -0,0 +1,35 @@
+/**
+ * 反思引擎（P4 加深）—— 纯逻辑：读评测体系的 per-target FN/FP 明细，做定向修改。
+ * 路由：漏报/漏召（FN 为主）→ 放宽取证；误报（FP 为主）→ 收紧（加负向词 + 收窄窗口）。
+ * 只动判错的目标，判对的不碰。可测：给合成 report + targets 即可断言修改。
+ */
+import type { Target } from "../schemas/target.js";
+import type { Oracle } from "./scorer.js";
+export declare const tokens: (s: string) => string[];
+export declare const nameTokens: (name: string) => string[];
+export interface TargetDiagnosis {
+    target_id: string;
+    fn: number;
+    fp: number;
+    suspect: string;
+    action: string;
+}
+export interface WinnerReport {
+    mismatches?: Array<{
+        case: string;
+        target: string;
+        expected: unknown;
+        actual: unknown;
+    }>;
+    per_target?: Record<string, {
+        n: number;
+        ok: number;
+        accuracy: number;
+    }>;
+}
+/** 从 winner 明细做 per-target 归因并定向修改 targets（就地）。返回是否改动 + 每目标诊断。 */
+export declare function reviseFromReport(targets: Target[], report: WinnerReport, oracle?: Oracle): {
+    revised: boolean;
+    diagnoses: TargetDiagnosis[];
+};
+//# sourceMappingURL=reflect.d.ts.map

package/dist/forge/reflect.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"reflect.d.ts","sourceRoot":"","sources":["../../src/forge/reflect.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAI1C,eAAO,MAAM,MAAM,GAAI,GAAG,MAAM,KAAG,MAAM,EAA4B,CAAC;AACtE,eAAO,MAAM,UAAU,GAAI,MAAM,MAAM,KAAG,MAAM,EAA8B,CAAC;AAE/E,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,EAAE,EAAE,MAAM,CAAC;IACX,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,YAAY;IAC3B,UAAU,CAAC,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,OAAO,CAAC;QAAC,MAAM,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;IACzF,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC1E;AAED,kEAAkE;AAClE,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG;IAAE,OAAO,EAAE,OAAO,CAAC;IAAC,SAAS,EAAE,eAAe,EAAE,CAAA;CAAE,CAwC7I"}

package/dist/forge/reflect.js

@@ -0,0 +1,72 @@
+const canon = (v) => (v == null ? null : String(v).trim());
+const TOKEN = /[\w一-鿿]{2,}/g;
+export const tokens = (s) => (s.match(TOKEN) ?? []);
+export const nameTokens = (name) => tokens(name).slice(0, 6);
+/** 从 winner 明细做 per-target 归因并定向修改 targets（就地）。返回是否改动 + 每目标诊断。 */
+export function reviseFromReport(targets, report, oracle) {
+    const stat = new Map();
+    for (const m of report.mismatches ?? []) {
+        const e = canon(m.expected), a = canon(m.actual);
+        const rec = stat.get(m.target) ?? { fn: 0, fp: 0, fpCases: [] };
+        if (e === "fail" && a !== "fail")
+            rec.fn += 1; // 漏报
+        else if (e !== "fail" && a === "fail") {
+            rec.fp += 1;
+            rec.fpCases.push(m.case);
+        } // 误报
+        else
+            rec.fn += 1; // 非 predicate 的错判：按漏召兜底放宽
+        stat.set(m.target, rec);
+    }
+    const diagnoses = [];
+    let revised = false;
+    for (const t of targets) {
+        const rec = stat.get(t.target_id);
+        if (!rec)
+            continue; // 全对 → 不动
+        const r = (t.retrieval ??= {});
+        let action, suspect;
+        if (rec.fp > rec.fn) { // 误报为主 → 收紧
+            const negs = new Set(r.negative_keywords ?? []);
+            const harvested = harvestNegatives(t, rec.fpCases, oracle);
+            for (const tok of harvested)
+                negs.add(tok);
+            r.negative_keywords = [...negs];
+            r.window = Math.max(Number(r.window ?? 800) - 200, 300);
+            suspect = "误报 / 判据过严或触发过宽";
+            action = `加负向词 ${harvested.length} 个 · 收窄窗口→${r.window}`;
+        }
+        else { // 漏报 / 漏召为主 → 放宽取证
+            const kws = new Set(r.content_keywords ?? []);
+            for (const tok of nameTokens(t.meta.name ?? t.target_id))
+                kws.add(tok);
+            const rubric = String(t.judge?.rubric_text ?? "");
+            for (const tok of tokens(rubric).slice(0, 4))
+                kws.add(tok);
+            r.content_keywords = [...kws];
+            r.window = Math.min(Number(r.window ?? 800) + 400, 2000);
+            suspect = "漏报 / 漏召";
+            action = `加检索词→${r.content_keywords.length} · 放宽窗口→${r.window}`;
+        }
+        diagnoses.push({ target_id: t.target_id, fn: rec.fn, fp: rec.fp, suspect, action });
+        revised = true;
+    }
+    return { revised, diagnoses };
+}
+/** 从误报案例的文档里挖负向词：这些案例本应通过却被判 fail，取其高频词作为筛除线索。 */
+function harvestNegatives(t, fpCaseIds, oracle) {
+    if (!oracle || !fpCaseIds.length)
+        return [];
+    const already = new Set([...(t.retrieval?.content_keywords ?? []), ...nameTokens(t.meta.name ?? "")]);
+    const freq = new Map();
+    for (const c of oracle.cases) {
+        if (!fpCaseIds.includes(c.case_id))
+            continue;
+        const text = (c.docs ?? []).map((d) => d.text ?? "").join("\n");
+        for (const tok of new Set(tokens(text)))
+            if (!already.has(tok))
+                freq.set(tok, (freq.get(tok) ?? 0) + 1);
+    }
+    return [...freq.entries()].sort((a, b) => b[1] - a[1]).slice(0, 3).map(([tok]) => tok);
+}
+//# sourceMappingURL=reflect.js.map

package/dist/forge/reflect.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"reflect.js","sourceRoot":"","sources":["../../src/forge/reflect.ts"],"names":[],"mappings":"AAQA,MAAM,KAAK,GAAG,CAAC,CAAU,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;AACpE,MAAM,KAAK,GAAG,cAAc,CAAC;AAC7B,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,CAAS,EAAY,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;AACtE,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,IAAY,EAAY,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AAe/E,kEAAkE;AAClE,MAAM,UAAU,gBAAgB,CAAC,OAAiB,EAAE,MAAoB,EAAE,MAAe;IACvF,MAAM,IAAI,GAAG,IAAI,GAAG,EAAyD,CAAC;IAC9E,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,UAAU,IAAI,EAAE,EAAE,CAAC;QACxC,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QACjD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;QAChE,IAAI,CAAC,KAAK,MAAM,IAAI,CAAC,KAAK,MAAM;YAAE,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC,CAAiB,KAAK;aAC/D,IAAI,CAAC,KAAK,MAAM,IAAI,CAAC,KAAK,MAAM,EAAE,CAAC;YAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;YAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAAC,CAAC,CAAE,KAAK;;YACnF,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC,CAA6C,0BAA0B;QACxF,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC1B,CAAC;IAED,MAAM,SAAS,GAAsB,EAAE,CAAC;IACxC,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAClC,IAAI,CAAC,GAAG;YAAE,SAAS,CAA2C,UAAU;QACxE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,KAAK,EAAE,CAAQ,CAAC;QACtC,IAAI,MAAc,EAAE,OAAe,CAAC;QACpC,IAAI,GAAG,CAAC,EAAE,GAAG,GAAG,CAAC,EAAE,EAAE,CAAC,CAAwC,YAAY;YACxE,MAAM,IAAI,GAAG,IAAI,GAAG,CAAU,CAAC,CAAC,iBAA8B,IAAI,EAAE,CAAC,CAAC;YACtE,MAAM,SAAS,GAAG,gBAAgB,CAAC,CAAC,EAAE,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YAC3D,KAAK,MAAM,GAAG,IAAI,SAAS;gBAAE,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC3C,CAAC,CAAC,iBAAiB,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC;YAChC,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,IAAI,GAAG,CAAC,GAAG,GAAG,EAAE,GAAG,CAAC,CAAC;YACxD,OAAO,GAAG,gBAAgB,CAAC;YAC3B,MAAM,GAAG,QAAQ,SAAS,CAAC,MAAM,aAAa,CAAC,CAAC,MAAM,EAAE,CAAC;QAC3D,CAAC;aAAM,CAAC,CAAsD,mBAAmB;YAC/E,MAAM,GAAG,GAAG,IAAI,GAAG,CAAU,CAAC,CAAC,gBAA6B,IAAI,EAAE,CAAC,CAAC;YACpE,KAAK,MAAM,GAAG,IAAI,UAAU,CAAE,CAAC,CAAC,IAAI,CAAC,IAAe,IAAI,CAAC,CAAC,SAAS,CAAC;gBAAE,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACnF,MAAM,MAAM,GAAG,MAAM,CAAE,CAAC,CAAC,KAAa,EAAE,WAAW,IAAI,EAAE,CAAC,CAAC;YAC3D,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;gBAAE,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC3D,CAAC,CAAC,gBAAgB,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC;YAC9B,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,IAAI,GAAG,CAAC,GAAG,GAAG,EAAE,IAAI,CAAC,CAAC;YACzD,OAAO,GAAG,SAAS,CAAC;YACpB,MAAM,GAAG,QAAS,CAAC,CAAC,gBAA6B,CAAC,MAAM,WAAW,CAAC,CAAC,MAAM,EAAE,CAAC;QAChF,CAAC;QACD,SAAS,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QACpF,OAAO,GAAG,IAAI,CAAC;IACjB,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC;AAChC,CAAC;AAED,kDAAkD;AAClD,SAAS,gBAAgB,CAAC,CAAS,EAAE,SAAmB,EAAE,MAAe;IACvE,IAAI,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM;QAAE,OAAO,EAAE,CAAC;IAC5C,MAAM,OAAO,GAAG,IAAI,GAAG,CAAS,CAAC,GAAG,CAAE,CAAC,CAAC,SAAiB,EAAE,gBAAgB,IAAI,EAAE,CAAC,EAAE,GAAG,UAAU,CAAE,CAAC,CAAC,IAAI,CAAC,IAAe,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IACnI,MAAM,IAAI,GAAG,IAAI,GAAG,EAAkB,CAAC;IACvC,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QAC7B,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC;YAAE,SAAS;QAC7C,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAChE,KAAK,MAAM,GAAG,IAAI,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1G,CAAC;IACD,OAAO,CAAC,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC;AACzF,CAAC"}

package/dist/forge/scorer.d.ts

@@ -0,0 +1,67 @@
+import type { Target } from "../schemas/target.js";
+import type { Executor } from "../workflows/executor.js";
+export interface OracleCase {
+    case_id: string;
+    tier?: string;
+    docs: Array<{
+        document_id: string;
+        text: string;
+        received_at?: string;
+        doc_type?: string;
+    }>;
+    expected: Record<string, unknown>;
+}
+export interface Oracle {
+    task?: string;
+    spec?: string;
+    cases: OracleCase[];
+}
+export interface ScoreReport {
+    n: number;
+    matched: number;
+    accuracy: number;
+    llm_calls: number;
+    per_target: Record<string, {
+        n: number;
+        ok: number;
+        accuracy: number;
+    }>;
+    per_tier: Record<string, {
+        n: number;
+        ok: number;
+        accuracy: number;
+    }>;
+    mismatches: Array<{
+        case: string;
+        target: string;
+        expected: unknown;
+        actual: unknown;
+    }>;
+    confusion?: {
+        tp: number;
+        fn: number;
+        fp: number;
+        tn: number;
+    };
+    recall?: number | null;
+    precision?: number | null;
+    false_negatives?: Array<{
+        case: string;
+        target: string;
+        expected: unknown;
+        actual: unknown;
+    }>;
+    false_positives?: Array<{
+        case: string;
+        target: string;
+        expected: unknown;
+        actual: unknown;
+    }>;
+}
+export declare class Scorer {
+    private ex;
+    constructor(ex: Executor);
+    private run;
+    score(targets: Target[], oracle: Oracle): Promise<ScoreReport>;
+}
+//# sourceMappingURL=scorer.d.ts.map

package/dist/forge/scorer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scorer.d.ts","sourceRoot":"","sources":["../../src/forge/scorer.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AAGzD,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,KAAK,CAAC;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC5F,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACnC;AACD,MAAM,WAAW,MAAM;IACrB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,UAAU,EAAE,CAAC;CACrB;AAED,MAAM,WAAW,WAAW;IAC1B,CAAC,EAAE,MAAM,CAAC;IACV,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACxE,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACtE,UAAU,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,OAAO,CAAC;QAAC,MAAM,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;IACxF,SAAS,CAAC,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAA;KAAE,CAAC;IAC/D,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,eAAe,CAAC,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,OAAO,CAAC;QAAC,MAAM,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;IAC9F,eAAe,CAAC,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,OAAO,CAAC;QAAC,MAAM,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;CAC/F;AAKD,qBAAa,MAAM;IACL,OAAO,CAAC,EAAE;gBAAF,EAAE,EAAE,QAAQ;YAElB,GAAG;IAoBX,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC;CAqDrE"}

package/dist/forge/scorer.js

@@ -0,0 +1,99 @@
+/**
+ * Eval 台 —— 对着标注 oracle 给 spec 打分。predicate 比 verdict、extraction 比 value。
+ * predicate 额外出混淆矩阵（fail=阳性）/召回/精确/分层。从 Python scorer.py 忠实移植（异步）。
+ */
+import { DocumentIR } from "../schemas/payloads.js";
+const canon = (v) => (v == null ? null : String(v).trim());
+const round3 = (x) => Math.round(x * 1000) / 1000;
+export class Scorer {
+    ex;
+    constructor(ex) {
+        this.ex = ex;
+    }
+    async run(targets, c) {
+        const ledger = { calls: 0, record() { this.calls += 1; } };
+        const unit = this.ex.spec.unit();
+        let res;
+        if (unit === "per_doc_rule") {
+            const d = c.docs[0];
+            res = await this.ex.runDocument(new DocumentIR(d.document_id), targets, { rawText: d.text, ledger });
+        }
+        else {
+            res = await this.ex.runCase(targets, c.docs, { ledger });
+        }
+        this._lastCost = ledger.calls;
+        const out = {};
+        for (const r of res.results) {
+            if (r.value != null)
+                out[r.target_id] = r.value;
+            else if (r.comparison)
+                out[r.target_id] = r.comparison.actual;
+            else
+                out[r.target_id] = r.verdict;
+        }
+        return out;
+    }
+    async score(targets, oracle) {
+        let total = 0, matched = 0, llmCalls = 0;
+        const per = {};
+        const perTier = {};
+        const mismatches = [];
+        const predicate = oracle.task === "predicate";
+        const conf = { tp: 0, fn: 0, fp: 0, tn: 0 };
+        const falseNeg = [];
+        const falsePos = [];
+        for (const c of oracle.cases) {
+            const got = await this.run(targets, c);
+            llmCalls += this._lastCost ?? 0;
+            const tier = c.tier ?? "untiered";
+            for (const [tid, expected] of Object.entries(c.expected)) {
+                total += 1;
+                const actual = got[tid];
+                const ok = canon(actual) === canon(expected);
+                (per[tid] ??= { n: 0, ok: 0 }).n += 1;
+                per[tid].ok += ok ? 1 : 0;
+                (perTier[tier] ??= { n: 0, ok: 0 }).n += 1;
+                perTier[tier].ok += ok ? 1 : 0;
+                if (ok)
+                    matched += 1;
+                else
+                    mismatches.push({ case: c.case_id, target: tid, expected, actual });
+                if (predicate) {
+                    const expPos = canon(expected) === "fail";
+                    const actPos = canon(actual) === "fail";
+                    if (expPos && actPos)
+                        conf.tp += 1;
+                    else if (expPos && !actPos) {
+                        conf.fn += 1;
+                        falseNeg.push({ case: c.case_id, target: tid, expected, actual });
+                    }
+                    else if (!expPos && actPos) {
+                        conf.fp += 1;
+                        falsePos.push({ case: c.case_id, target: tid, expected, actual });
+                    }
+                    else
+                        conf.tn += 1;
+                }
+            }
+        }
+        const out = {
+            n: total, matched,
+            accuracy: total ? round3(matched / total) : 0.0,
+            llm_calls: llmCalls,
+            per_target: Object.fromEntries(Object.entries(per).map(([k, v]) => [k, { ...v, accuracy: round3(v.ok / v.n) }])),
+            per_tier: Object.fromEntries(Object.entries(perTier).map(([k, v]) => [k, { ...v, accuracy: round3(v.ok / v.n) }])),
+            mismatches,
+        };
+        if (predicate) {
+            const pos = conf.tp + conf.fn;
+            const rep = conf.tp + conf.fp;
+            out.confusion = conf;
+            out.recall = pos ? round3(conf.tp / pos) : null;
+            out.precision = rep ? round3(conf.tp / rep) : null;
+            out.false_negatives = falseNeg;
+            out.false_positives = falsePos;
+        }
+        return out;
+    }
+}
+//# sourceMappingURL=scorer.js.map

package/dist/forge/scorer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scorer.js","sourceRoot":"","sources":["../../src/forge/scorer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAgCpD,MAAM,KAAK,GAAG,CAAC,CAAU,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;AACpE,MAAM,MAAM,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC;AAE1D,MAAM,OAAO,MAAM;IACG;IAApB,YAAoB,EAAY;QAAZ,OAAE,GAAF,EAAE,CAAU;IAAG,CAAC;IAE5B,KAAK,CAAC,GAAG,CAAC,OAAiB,EAAE,CAAa;QAChD,MAAM,MAAM,GAAmC,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3F,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;QACjC,IAAI,GAA4B,CAAC;QACjC,IAAI,IAAI,KAAK,cAAc,EAAE,CAAC;YAC5B,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACpB,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,WAAW,CAAC,EAAE,OAAO,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;QACvG,CAAC;aAAM,CAAC;YACN,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;QAC3D,CAAC;QACA,IAAY,CAAC,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC;QACvC,MAAM,GAAG,GAA4B,EAAE,CAAC;QACxC,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,OAAgB,EAAE,CAAC;YACrC,IAAI,CAAC,CAAC,KAAK,IAAI,IAAI;gBAAE,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC;iBAC3C,IAAI,CAAC,CAAC,UAAU;gBAAE,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC;;gBACzD,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC;QACpC,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,OAAiB,EAAE,MAAc;QAC3C,IAAI,KAAK,GAAG,CAAC,EAAE,OAAO,GAAG,CAAC,EAAE,QAAQ,GAAG,CAAC,CAAC;QACzC,MAAM,GAAG,GAA8C,EAAE,CAAC;QAC1D,MAAM,OAAO,GAA8C,EAAE,CAAC;QAC9D,MAAM,UAAU,GAA8B,EAAE,CAAC;QACjD,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,KAAK,WAAW,CAAC;QAC9C,MAAM,IAAI,GAAG,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC;QAC5C,MAAM,QAAQ,GAAgD,EAAE,CAAC;QACjE,MAAM,QAAQ,GAAgD,EAAE,CAAC;QAEjE,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YAC7B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YACvC,QAAQ,IAAK,IAAY,CAAC,SAAS,IAAI,CAAC,CAAC;YACzC,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI,IAAI,UAAU,CAAC;YAClC,KAAK,MAAM,CAAC,GAAG,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACzD,KAAK,IAAI,CAAC,CAAC;gBACX,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;gBACxB,MAAM,EAAE,GAAG,KAAK,CAAC,MAAM,CAAC,KAAK,KAAK,CAAC,QAAQ,CAAC,CAAC;gBAC7C,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;gBACtC,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC1B,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;gBAC3C,OAAO,CAAC,IAAI,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC/B,IAAI,EAAE;oBAAE,OAAO,IAAI,CAAC,CAAC;;oBAChB,UAAU,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC;gBACzE,IAAI,SAAS,EAAE,CAAC;oBACd,MAAM,MAAM,GAAG,KAAK,CAAC,QAAQ,CAAC,KAAK,MAAM,CAAC;oBAC1C,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,KAAK,MAAM,CAAC;oBACxC,IAAI,MAAM,IAAI,MAAM;wBAAE,IAAI,CAAC,EAAE,IAAI,CAAC,CAAC;yBAC9B,IAAI,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;wBAAC,IAAI,CAAC,EAAE,IAAI,CAAC,CAAC;wBAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC;oBAAC,CAAC;yBAC3G,IAAI,CAAC,MAAM,IAAI,MAAM,EAAE,CAAC;wBAAC,IAAI,CAAC,EAAE,IAAI,CAAC,CAAC;wBAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC;oBAAC,CAAC;;wBAC3G,IAAI,CAAC,EAAE,IAAI,CAAC,CAAC;gBACpB,CAAC;YACH,CAAC;QACH,CAAC;QACD,MAAM,GAAG,GAAgB;YACvB,CAAC,EAAE,KAAK,EAAE,OAAO;YACjB,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG;YAC/C,SAAS,EAAE,QAAQ;YACnB,UAAU,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,GAAG,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YAChH,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,GAAG,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YAClH,UAAU;SACX,CAAC;QACF,IAAI,SAAS,EAAE,CAAC;YACd,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC;YAC9B,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC;YAC9B,GAAG,CAAC,SAAS,GAAG,IAAI,CAAC;YACrB,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YAChD,GAAG,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YACnD,GAAG,CAAC,eAAe,GAAG,QAAQ,CAAC;YAC/B,GAAG,CAAC,eAAe,GAAG,QAAQ,CAAC;QACjC,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;CACF"}