@hawkeyexl/manni 4.5.0 → 4.6.0

package/dist/{chunk-TA42STP7.js → chunk-UAKNZVCD.js}

@@ -15,7 +15,7 @@ import {
   paragraphAfter,
   parseSrc,
   ruleId
-} from "./chunk-UK2I3ACY.js";
+} from "./chunk-MHM4QEBV.js";
 import {
   findWindows,
   hashLines,
@@ -24,10 +24,10 @@ import {
   spellLines,
   toBodyLines,
   toFileLines
-} from "./chunk-HVFR66GQ.js";
+} from "./chunk-TUAAR3MM.js";
 import {
   isErrorSeverity
-} from "./chunk-527HHQPS.js";
+} from "./chunk-KCOFEWV5.js";
 
 // src/cite/core/spell.ts
 function shortPin(integrity) {
@@ -619,4 +619,4 @@ export {
   errorSite,
   toValidationResult
 };
-//# sourceMappingURL=chunk-TA42STP7.js.map
+//# sourceMappingURL=chunk-UAKNZVCD.js.map

package/dist/{chunk-C3KI45RF.js → chunk-UJSGS2Y5.js}

@@ -15,10 +15,10 @@ import {
 import {
   REPORT_FORMAT_LIST,
   renderJunit
-} from "./chunk-TCC737IS.js";
+} from "./chunk-5G53XATW.js";
 import {
   isErrorSeverity
-} from "./chunk-527HHQPS.js";
+} from "./chunk-KCOFEWV5.js";
 import {
   palette
 } from "./chunk-RNVYGWIC.js";
@@ -28,7 +28,7 @@ import {
 } from "./chunk-MNH7QO4Z.js";
 import {
   package_default
-} from "./chunk-6XTQDFGP.js";
+} from "./chunk-PCFG7E6B.js";
 
 // src/lint/core/rule-id.ts
 var RULE_ID_PREFIX = "manni:lint";
@@ -633,4 +633,4 @@ export {
   renderTemplates,
   renderTools
 };
-//# sourceMappingURL=chunk-C3KI45RF.js.map
+//# sourceMappingURL=chunk-UJSGS2Y5.js.map

package/dist/{chunk-ZX43SRZR.js → chunk-UWQ2UVHE.js}

@@ -1,16 +1,18 @@
 #!/usr/bin/env node
 import {
-  assertProviderSelection,
   renderCalibration,
   resolveBatchTraces,
-  runBatch,
-  selectProvider
-} from "./chunk-SJO2FTPG.js";
+  runBatch
+} from "./chunk-PL4UAF4S.js";
 import {
   discoverConfig,
   labelKey,
   loadLabels
-} from "./chunk-3R5Y3XOL.js";
+} from "./chunk-XPEWUK7J.js";
+import {
+  assertProviderSelection,
+  selectProvider
+} from "./chunk-BWCWTHYS.js";
 import {
   TracevalsError
 } from "./chunk-UKZNLVJW.js";
@@ -376,4 +378,4 @@ export {
   score,
   runCalibrate
 };
-//# sourceMappingURL=chunk-ZX43SRZR.js.map
+//# sourceMappingURL=chunk-UWQ2UVHE.js.map

package/dist/{chunk-ZX43SRZR.js.map → chunk-UWQ2UVHE.js.map}

@@ -1 +1 @@
-{"version":3,"sources":["../src/tracevals/commands/calibrate.ts","../src/tracevals/calibrate/score.ts"],"sourcesContent":["/**\n * `manni tracevals calibrate <traces...> --labels <file>` (ADR 01022).\n *\n * The tool could report what it decided; it could not report whether it was\n * right. This command runs the batch (ADR 01018), joins the results against a\n * human's answers, and reports the three numbers the calibration page used to\n * ask people to count by hand — false passes, false fails, and review volume —\n * plus which eval disagreed.\n *\n * `--sweep` costs no inference beyond the one run it already made: a judged\n * eval's consensus carries the whole `JudgeRun[]`, so a different zone\n * threshold or a smaller ensemble is arithmetic over verdicts already on disk.\n * Read-only throughout — labels, traces, and artifacts are never written.\n */\nimport { writeFile } from \"node:fs/promises\";\nimport { basename, resolve } from \"node:path\";\nimport { discoverConfig } from \"../core/config.js\";\nimport { assertProviderSelection, selectProvider } from \"../judge/provider.js\";\nimport { renderCalibration, type ReportFormat } from \"../reporters/index.js\";\nimport { TracevalsError, type EvalResult } from \"../types.js\";\nimport { loadLabels, type EvalLabel } from \"../calibrate/labels.js\";\nimport { joinLabels, score, type AmbiguousLabel } from \"../calibrate/score.js\";\nimport type {\n  CalibrationGate,\n  CalibrationReport,\n  JudgeSetting,\n  SweepCell,\n} from \"../calibrate/types.js\";\nimport {\n  resolveBatchTraces,\n  runBatch,\n  type BatchCommandOptions,\n} from \"./batch.js\";\n\nexport interface CalibrateCommandOptions extends BatchCommandOptions {\n  /** Overrides `calibrate.labels`; resolved against the working directory. */\n  labels?: string;\n  /** Re-score the corpus across the configured grid. */\n  sweep?: boolean;\n  /** Thresholds; undefined defers to the config, which defaults to no gate. */\n  maxFalsePass?: number;\n  maxFalseFail?: number;\n  maxReview?: number;\n}\n\nexport interface CalibrateCommandResult {\n  report: CalibrationReport;\n  rendered: string;\n}\n\n/** Largest ensemble the sweep needs cached, so every cell can be re-scored. */\nfunction sweepDepth(grid: number[], baseline: number): number {\n  return Math.max(baseline, ...grid);\n}\n\nexport async function runCalibrate(\n  options: CalibrateCommandOptions,\n): Promise<CalibrateCommandResult> {\n  const start = Date.now();\n  const { config, dir: configDir } = await discoverConfig(\n    options.configDir ?? process.cwd(),\n    {\n      ...(options.config === undefined ? {} : { configPath: options.config }),\n      ...(options.noConfig === undefined ? {} : { noConfig: options.noConfig }),\n    },\n  );\n\n  // Before the labels file is read: a contradictory `--local --provider`, an\n  // unknown provider name, or a model with no provider to own it is a usage\n  // error, and a usage error must not arrive behind an unrelated one about a\n  // file the run would never have needed.\n  assertProviderSelection(\n    selectProvider(config, {\n      ...(options.provider !== undefined ? { provider: options.provider } : {}),\n      ...(options.model !== undefined ? { model: options.model } : {}),\n      ...(options.local !== undefined ? { local: options.local } : {}),\n    }),\n  );\n\n  // A flag overrides the config rather than bypassing it (CLAUDE.md,\n  // \"Config <-> CLI flags\"). An explicit --labels is a working-directory path,\n  // the way every other path argument on the CLI is; the config default is a\n  // project path and resolves against the config file's directory.\n  const labelsFile =\n    options.labels !== undefined\n      ? resolve(options.labels)\n      : resolve(configDir, config.calibrate.labels);\n  const labels = await loadLabels(labelsFile);\n\n  // Resolve the corpus once and hand the list to the batch, so the membership\n  // check below cannot disagree with what actually ran.\n  const corpus = (await resolveBatchTraces(options)).map((f) => resolve(f));\n  assertLabelsInCorpus(labels, corpus, labelsFile);\n\n  const baseline: JudgeSetting = {\n    ensembleRuns: options.runs ?? config.judge.ensembleRuns,\n    zones: config.judge.zones,\n  };\n  const grid = config.calibrate.sweep;\n  // With --sweep the corpus is judged once at the deepest ensemble the grid\n  // asks for, and every cell — the baseline included — is re-scored from those\n  // runs. Judging at the configured depth instead would make the larger cells\n  // unscoreable, and judging per cell would multiply the bill by the grid.\n  const depth =\n    options.sweep === true\n      ? sweepDepth(grid.ensembleRuns, baseline.ensembleRuns)\n      : baseline.ensembleRuns;\n\n  const batch = await runBatch({\n    ...options,\n    traces: corpus,\n    allProjects: undefined,\n    newerThan: undefined,\n    limit: undefined,\n    runs: depth,\n    // The calibration report is the deliverable; rendering the batch as well\n    // would write the wrong thing to --output.\n    format: \"json\",\n    output: undefined,\n  });\n\n  const all: EvalResult[] = [];\n  for (const outcome of batch.outcomes) {\n    if (\"report\" in outcome) all.push(...outcome.report.evalResults);\n  }\n  const { joined, unmatched, ambiguous } = joinLabels(batch.outcomes, labels);\n  assertNoUnmatched(unmatched, batch, labelsFile);\n  assertNoAmbiguous(ambiguous, labelsFile);\n\n  // Without a sweep the reported outcomes are used verbatim, so the headline\n  // numbers are exactly the verdicts `run` produced. With one, the corpus was\n  // judged deeper than the configured ensemble, so the baseline row has to be\n  // re-scored back down to it.\n  const scored = score(\n    { joined, all },\n    options.sweep === true ? baseline : undefined,\n  );\n\n  const sweep =\n    options.sweep === true\n      ? buildSweep({ joined, all }, baseline, grid)\n      : undefined;\n\n  const gates = buildGates(scored.counts, {\n    falsePass: options.maxFalsePass ?? config.calibrate.maxFalsePass,\n    falseFail: options.maxFalseFail ?? config.calibrate.maxFalseFail,\n    review: options.maxReview ?? config.calibrate.maxReview,\n  });\n\n  const warnings = [...batch.report.warnings];\n  if (scored.counts.insufficient > 0) {\n    warnings.push(\n      `${scored.counts.insufficient} labelled eval(s) had fewer cached runs than the setting asks for and were left at their reported outcome`,\n    );\n  }\n\n  // A measurement that measured nothing must not report a pass.\n  //\n  // Every label joining to a `skipped` result — `--deterministic-only` over\n  // ai-graded labels, a budget exhausted on the first trace — leaves `scored`\n  // at 0, `agreement` at null, and each threshold reading \"0 of at most N, not\n  // exceeded\". The gates are not lying; the denominator behind them is empty.\n  // Saying so is the report's job, and exit 0 would say the opposite.\n  const measuredNothing = scored.counts.scored === 0;\n  if (measuredNothing) {\n    warnings.push(\n      `no labelled eval produced evidence: ${scored.counts.labels} label(s) all joined to a skipped result, ` +\n        `so agreement and every --max-* threshold are computed over an empty denominator`,\n    );\n  }\n\n  const report: CalibrationReport = {\n    labelsFile,\n    corpus,\n    setting: baseline,\n    counts: scored.counts,\n    disagreements: scored.disagreements,\n    unscored: scored.unscored,\n    ...(sweep !== undefined ? { sweep } : {}),\n    gates,\n    batch: batch.report,\n    warnings,\n    // A calibration run is a measurement, not a gate: disagreement is the\n    // finding, not a failure. Exit 1 only when a threshold was asked for and\n    // missed, or when the measurement itself is incomplete — a trace lost, the\n    // shared judge budget cut the corpus short, or nothing scored at all. An\n    // incomplete number presented as a clean one is worse than no number.\n    exitCode:\n      gates.some((g) => g.exceeded) ||\n      batch.report.summary.tracesErrored > 0 ||\n      batch.report.budget !== undefined ||\n      measuredNothing\n        ? 1\n        : 0,\n    turns: batch.report.turns,\n    durationMs: Date.now() - start,\n  };\n\n  const rendered = renderCalibration(report, options.format ?? \"pretty\", {\n    color: options.color === true,\n  });\n  if (options.output) await writeFile(options.output, rendered, \"utf-8\");\n  return { report, rendered };\n}\n\n/**\n * Every label must name a trace this run evaluates. Checked before the batch,\n * because the alternative is paying for a corpus and then being told the file\n * was misspelled.\n */\nfunction assertLabelsInCorpus(\n  labels: EvalLabel[],\n  corpus: string[],\n  labelsFile: string,\n): void {\n  const known = new Set(corpus);\n  const missing = [...new Set(labels.filter((l) => !known.has(l.traceFile)).map((l) => l.trace))];\n  if (missing.length === 0) return;\n  throw new TracevalsError(\n    `${labelsFile}: label(s) name trace(s) not in the corpus: ${missing.join(\", \")}. ` +\n      `The corpus is: ${corpus.map((f) => basename(f)).join(\", \")}`,\n  );\n}\n\n/**\n * A label that matched nothing is an error, not a quiet omission.\n *\n * It has to be: a typo'd eval id contributes to no count, so the report would\n * read as a cleaner corpus than it is — the same silent-nothing failure the\n * empty-selector rule exists to prevent (ADR 01018).\n */\nfunction assertNoUnmatched(\n  unmatched: EvalLabel[],\n  batch: { outcomes: { file: string; report?: { evalResults: EvalResult[] } }[] },\n  labelsFile: string,\n): void {\n  if (unmatched.length === 0) return;\n  const lines = unmatched.map((label) => {\n    const outcome = batch.outcomes.find((o) => o.file === label.traceFile);\n    const results = outcome?.report?.evalResults ?? [];\n    const sameArtifact = results.filter(\n      (r) => r.artifactName === label.artifact,\n    );\n    const hint =\n      sameArtifact.length > 0\n        ? `that trace evaluated ${sameArtifact.map((r) => r.evalName).sort().join(\", \")} for \"${label.artifact}\"`\n        : `that trace resolved no artifact named \"${label.artifact}\" — it evaluated ${\n            [...new Set(results.map((r) => r.artifactName))].sort().join(\", \") ||\n            \"nothing\"\n          }`;\n    return `  ${label.artifact} › ${label.eval} on ${basename(label.traceFile)} (${hint})`;\n  });\n  throw new TracevalsError(\n    `${labelsFile}: label(s) matched no eval result:\\n${lines.join(\"\\n\")}`,\n  );\n}\n\n/**\n * A label that omits `type:` over a name two artifact kinds both answer cannot\n * be resolved, and must not be guessed. Binding it to whichever result was\n * indexed last would score agreement against a verdict belonging to a\n * different artifact — the report would read as measured when it is wrong,\n * which is the one thing this command exists not to do.\n *\n * Operational, like `assertNoUnmatched`: the labels file is under-specified,\n * so this is exit 2 (bad input), not exit 1 (a check failed).\n */\nfunction assertNoAmbiguous(\n  ambiguous: AmbiguousLabel[],\n  labelsFile: string,\n): void {\n  if (ambiguous.length === 0) return;\n  const lines = ambiguous.map(({ label, candidates }) => {\n    const kinds = [...new Set(candidates.map((r) => r.artifactType))].sort();\n    return `  ${label.artifact} › ${label.eval} on ${basename(label.traceFile)} matches ${kinds.join(\" and \")} — add \\`type:\\` to say which`;\n  });\n  throw new TracevalsError(\n    `${labelsFile}: label(s) name an artifact two kinds answer:\\n${lines.join(\"\\n\")}`,\n  );\n}\n\nfunction buildGates(\n  counts: { falsePass: number; falseFail: number; reviewVolume: number },\n  limits: { falsePass?: number; falseFail?: number; review?: number },\n): CalibrationGate[] {\n  const gates: CalibrationGate[] = [];\n  const add = (\n    name: CalibrationGate[\"name\"],\n    limit: number | undefined,\n    actual: number,\n  ): void => {\n    if (limit === undefined) return;\n    gates.push({ name, limit, actual, exceeded: actual > limit });\n  };\n  add(\"falsePass\", limits.falsePass, counts.falsePass);\n  add(\"falseFail\", limits.falseFail, counts.falseFail);\n  add(\"review\", limits.review, counts.reviewVolume);\n  return gates;\n}\n\n/**\n * One axis at a time, every other knob held at its configured value.\n *\n * A full cross-product would be dozens of rows and would answer a question\n * nobody asked; calibration is \"move one knob, re-run, recount\", and the\n * per-axis shape is what makes a row readable as *the effect of that knob*.\n */\nfunction buildSweep(\n  input: Parameters<typeof score>[0],\n  baseline: JudgeSetting,\n  grid: { ensembleRuns: number[]; autoPass: number[]; autoFail: number[] },\n): SweepCell[] {\n  const cell = (\n    axis: SweepCell[\"axis\"],\n    value: number,\n    setting: JudgeSetting,\n  ): SweepCell => ({\n    axis,\n    value,\n    setting,\n    counts: score(input, setting).counts,\n  });\n\n  const cells: SweepCell[] = [\n    cell(\"baseline\", baseline.ensembleRuns, baseline),\n  ];\n  for (const runs of [...grid.ensembleRuns].sort((a, b) => a - b)) {\n    cells.push(\n      cell(\"ensembleRuns\", runs, { ensembleRuns: runs, zones: baseline.zones }),\n    );\n  }\n  for (const autoPass of [...grid.autoPass].sort((a, b) => a - b)) {\n    cells.push(\n      cell(\"zones.autoPass\", autoPass, {\n        ensembleRuns: baseline.ensembleRuns,\n        zones: { autoPass, autoFail: baseline.zones.autoFail },\n      }),\n    );\n  }\n  for (const autoFail of [...grid.autoFail].sort((a, b) => a - b)) {\n    cells.push(\n      cell(\"zones.autoFail\", autoFail, {\n        ensembleRuns: baseline.ensembleRuns,\n        zones: { autoPass: baseline.zones.autoPass, autoFail },\n      }),\n    );\n  }\n  return cells;\n}\n\nexport type { ReportFormat };\n","/**\n * Joining labels to results, and re-scoring cached verdicts (ADR 01022).\n *\n * Everything here is pure: results and labels in, counts out. That is what\n * makes a sweep free. A judged eval's `ConsensusResult` carries the whole\n * `JudgeRun[]` it was built from, so a different zone threshold — or a smaller\n * ensemble — is a fresh call to `computeConsensus`/`zoneFor` over runs the\n * corpus already produced, not a fresh call to a provider.\n */\nimport { computeConsensus, zoneFor } from \"@hawkeyexl/inference\";\nimport type { BatchOutcome } from \"../aggregate.js\";\nimport type { EvalResult, Outcome } from \"../types.js\";\nimport { labelKey, type EvalLabel, type LabelOutcome } from \"./labels.js\";\nimport type {\n  CalibrationCounts,\n  Disagreement,\n  JoinKind,\n  JudgeSetting,\n} from \"./types.js\";\n\n/**\n * How one label met one result, over the full (expected × actual) grid.\n *\n * The order matters. `skipped` and `error` are tested *first*, so a labelled\n * eval that produced no evidence can never be counted as agreement — the same\n * rule that keeps an errored judge run from becoming a silent pass.\n */\nexport function classify(expected: LabelOutcome, actual: Outcome): JoinKind {\n  if (actual === \"skipped\") return \"skipped\";\n  if (actual === \"error\") return \"error\";\n  if (actual === expected) return \"agree\";\n  if (expected === \"fail\" && actual === \"pass\") return \"false-pass\";\n  if (expected === \"pass\" && actual === \"fail\") return \"false-fail\";\n  if (actual === \"needs-review\") return \"review\";\n  return \"missed-review\";\n}\n\nexport interface JoinedLabel {\n  label: EvalLabel;\n  result: EvalResult;\n}\n\nexport interface JoinResult {\n  joined: JoinedLabel[];\n  /**\n   * Labels that matched no result. Never silently dropped: a typo'd eval id\n   * would otherwise deflate every count on the report by one.\n   */\n  unmatched: EvalLabel[];\n  /**\n   * Labels that omit `type:` over a name two artifact kinds both answer. Never\n   * bound to one of them: a skill and the slash command that types it can share\n   * a name in one trace, and picking whichever was indexed last would measure\n   * agreement against a verdict belonging to something else — a false pass or\n   * false fail reported with total confidence.\n   */\n  ambiguous: AmbiguousLabel[];\n}\n\n/** An untyped label and every result its name matched. */\nexport interface AmbiguousLabel {\n  label: EvalLabel;\n  candidates: EvalResult[];\n}\n\n/**\n * Match each label to the result its trace produced. The key is\n * `(trace file, artifact name, eval id)`, with the artifact *type* joining too\n * when the label states one — needed only for a skill and an agent sharing a\n * name, which is legal and does happen.\n */\nexport function joinLabels(\n  outcomes: BatchOutcome[],\n  labels: EvalLabel[],\n): JoinResult {\n  // Typed keys are unique by construction. Untyped ones are not: two artifact\n  // kinds may answer the same name, so keep every candidate rather than\n  // letting the last one indexed overwrite the rest.\n  const byType = new Map<string, EvalResult>();\n  const byName = new Map<string, EvalResult[]>();\n  for (const outcome of outcomes) {\n    if (!(\"report\" in outcome)) continue;\n    for (const result of outcome.report.evalResults) {\n      byType.set(\n        labelKey(\n          outcome.file,\n          result.artifactName,\n          result.evalName,\n          result.artifactType,\n        ),\n        result,\n      );\n      const key = labelKey(outcome.file, result.artifactName, result.evalName);\n      const seen = byName.get(key);\n      if (seen === undefined) byName.set(key, [result]);\n      else seen.push(result);\n    }\n  }\n\n  const joined: JoinedLabel[] = [];\n  const unmatched: EvalLabel[] = [];\n  const ambiguous: AmbiguousLabel[] = [];\n  for (const label of labels) {\n    if (label.type !== undefined) {\n      const result = byType.get(\n        labelKey(label.traceFile, label.artifact, label.eval, label.type),\n      );\n      if (result === undefined) unmatched.push(label);\n      else joined.push({ label, result });\n      continue;\n    }\n    const candidates =\n      byName.get(labelKey(label.traceFile, label.artifact, label.eval)) ?? [];\n    const kinds = new Set(candidates.map((r) => r.artifactType));\n    if (candidates.length === 0) unmatched.push(label);\n    else if (kinds.size > 1) ambiguous.push({ label, candidates });\n    else joined.push({ label, result: candidates[0] as EvalResult });\n  }\n  return { joined, unmatched, ambiguous };\n}\n\nexport interface Rescored {\n  outcome: Outcome;\n  /** The cache held fewer runs than the setting asks for. */\n  insufficient: boolean;\n  /**\n   * The arithmetic **at this setting**, not at the setting the corpus was\n   * judged with. A sweep judges deeper than most of its cells, so reporting\n   * the full cached ensemble beside a re-scored verdict would show a reader\n   * evidence that does not add up to the outcome next to it.\n   */\n  consensus?: Disagreement[\"consensus\"];\n}\n\n/** The numbers a reader needs from a consensus, without the run bodies. */\nconst summarise = (\n  consensus: NonNullable<EvalResult[\"consensus\"]>,\n): Disagreement[\"consensus\"] => ({\n  votes: consensus.votes,\n  agreement: consensus.agreement,\n  meanConfidence: consensus.meanConfidence,\n  runs: consensus.runs.length,\n});\n\n/**\n * What this result would have been at a different setting.\n *\n * A result with no `consensus` — deterministic, `human`, skipped, errored — is\n * returned untouched: zones and ensemble size do not reach it, and pretending\n * otherwise would make a sweep look like it moved things it cannot move.\n *\n * A smaller `ensembleRuns` re-scores the **first k** of the cached runs. That\n * is a sub-sample of the ensemble that was actually drawn, not a fresh k-run\n * ensemble; at temperature 0 the runs are independent samples of one prompt,\n * so it is an honest estimate, and the alternative is paying per grid cell.\n */\nexport function rescore(result: EvalResult, setting: JudgeSetting): Rescored {\n  const consensus = result.consensus;\n  if (consensus === undefined) {\n    return { outcome: result.outcome, insufficient: false };\n  }\n  const runs = consensus.runs;\n  if (runs.length < setting.ensembleRuns) {\n    // Never scale up by guessing. Report the setting as unreachable and leave\n    // the row at what it actually was.\n    return { outcome: result.outcome, insufficient: true };\n  }\n  const slice = runs.slice(0, setting.ensembleRuns);\n  const base = computeConsensus(slice);\n  const zone = zoneFor(base, setting.zones);\n  return {\n    outcome:\n      zone === \"auto-pass\" ? \"pass\" : zone === \"auto-fail\" ? \"fail\" : \"needs-review\",\n    insufficient: false,\n    consensus: {\n      votes: base.votes,\n      agreement: base.agreement,\n      meanConfidence: base.meanConfidence,\n      runs: slice.length,\n    },\n  };\n}\n\nconst emptyCounts = (): CalibrationCounts => ({\n  labels: 0,\n  scored: 0,\n  agree: 0,\n  falsePass: 0,\n  falseFail: 0,\n  review: 0,\n  missedReview: 0,\n  errored: 0,\n  skipped: 0,\n  reviewVolume: 0,\n  insufficient: 0,\n  agreement: null,\n});\n\nexport interface ScoreInput {\n  joined: JoinedLabel[];\n  /** Every eval result in the corpus, for the corpus-wide review volume. */\n  all: EvalResult[];\n}\n\nexport interface ScoreResult {\n  counts: CalibrationCounts;\n  disagreements: Disagreement[];\n  unscored: Disagreement[];\n}\n\n/**\n * Score the joined corpus, optionally at a setting other than the one it ran\n * at. Omit `setting` and the reported outcomes are used verbatim, so the\n * headline numbers of a plain `calibrate` are exactly the verdicts `run`\n * would have produced — no re-derivation standing between the two commands.\n */\nexport function score(input: ScoreInput, setting?: JudgeSetting): ScoreResult {\n  const counts = emptyCounts();\n  const disagreements: Disagreement[] = [];\n  const unscored: Disagreement[] = [];\n\n  for (const result of input.all) {\n    const outcome =\n      setting === undefined ? result.outcome : rescore(result, setting).outcome;\n    if (outcome === \"needs-review\") counts.reviewVolume += 1;\n  }\n\n  for (const { label, result } of input.joined) {\n    const scored: Rescored =\n      setting === undefined\n        ? { outcome: result.outcome, insufficient: false }\n        : rescore(result, setting);\n    if (scored.insufficient) counts.insufficient += 1;\n    const kind = classify(label.expected, scored.outcome);\n    counts.labels += 1;\n\n    const row: Disagreement = {\n      trace: label.traceFile,\n      artifactType: result.artifactType,\n      artifactName: result.artifactName,\n      evalName: result.evalName,\n      grader: result.grader,\n      expected: label.expected,\n      actual: scored.outcome,\n      kind,\n      ...(label.note !== undefined ? { note: label.note } : {}),\n      ...(result.skipReason !== undefined\n        ? { skipReason: result.skipReason }\n        : {}),\n      // At the setting being scored, falling back to the run's own numbers\n      // when nothing was re-scored.\n      ...(() => {\n        const c =\n          scored.consensus ??\n          (result.consensus !== undefined ? summarise(result.consensus) : undefined);\n        return c !== undefined ? { consensus: c } : {};\n      })(),\n    };\n\n    switch (kind) {\n      case \"skipped\":\n        counts.skipped += 1;\n        unscored.push(row);\n        break;\n      case \"agree\":\n        counts.agree += 1;\n        counts.scored += 1;\n        break;\n      case \"false-pass\":\n        counts.falsePass += 1;\n        counts.scored += 1;\n        disagreements.push(row);\n        break;\n      case \"false-fail\":\n        counts.falseFail += 1;\n        counts.scored += 1;\n        disagreements.push(row);\n        break;\n      case \"review\":\n        counts.review += 1;\n        counts.scored += 1;\n        disagreements.push(row);\n        break;\n      case \"missed-review\":\n        counts.missedReview += 1;\n        counts.scored += 1;\n        disagreements.push(row);\n        break;\n      case \"error\":\n        counts.errored += 1;\n        counts.scored += 1;\n        disagreements.push(row);\n        break;\n    }\n  }\n\n  counts.agreement = counts.scored > 0 ? counts.agree / counts.scored : null;\n  // Byte comparison, so the ubuntu and windows legs order a report the same\n  // way and two runs over one corpus stay diffable.\n  const byRow = (a: Disagreement, b: Disagreement): number => {\n    const ka = `${a.kind} ${a.trace} ${a.artifactName} ${a.evalName}`;\n    const kb = `${b.kind} ${b.trace} ${b.artifactName} ${b.evalName}`;\n    return ka < kb ? -1 : ka > kb ? 1 : 0;\n  };\n  disagreements.sort(byRow);\n  unscored.sort(byRow);\n  return { counts, disagreements, unscored };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAcA,SAAS,iBAAiB;AAC1B,SAAS,UAAU,eAAe;;;ACNlC,SAAS,kBAAkB,eAAe;AAkBnC,SAAS,SAAS,UAAwB,QAA2B;AAC1E,MAAI,WAAW,UAAW,QAAO;AACjC,MAAI,WAAW,QAAS,QAAO;AAC/B,MAAI,WAAW,SAAU,QAAO;AAChC,MAAI,aAAa,UAAU,WAAW,OAAQ,QAAO;AACrD,MAAI,aAAa,UAAU,WAAW,OAAQ,QAAO;AACrD,MAAI,WAAW,eAAgB,QAAO;AACtC,SAAO;AACT;AAoCO,SAAS,WACd,UACA,QACY;AAIZ,QAAM,SAAS,oBAAI,IAAwB;AAC3C,QAAM,SAAS,oBAAI,IAA0B;AAC7C,aAAW,WAAW,UAAU;AAC9B,QAAI,EAAE,YAAY,SAAU;AAC5B,eAAW,UAAU,QAAQ,OAAO,aAAa;AAC/C,aAAO;AAAA,QACL;AAAA,UACE,QAAQ;AAAA,UACR,OAAO;AAAA,UACP,OAAO;AAAA,UACP,OAAO;AAAA,QACT;AAAA,QACA;AAAA,MACF;AACA,YAAM,MAAM,SAAS,QAAQ,MAAM,OAAO,cAAc,OAAO,QAAQ;AACvE,YAAM,OAAO,OAAO,IAAI,GAAG;AAC3B,UAAI,SAAS,OAAW,QAAO,IAAI,KAAK,CAAC,MAAM,CAAC;AAAA,UAC3C,MAAK,KAAK,MAAM;AAAA,IACvB;AAAA,EACF;AAEA,QAAM,SAAwB,CAAC;AAC/B,QAAM,YAAyB,CAAC;AAChC,QAAM,YAA8B,CAAC;AACrC,aAAW,SAAS,QAAQ;AAC1B,QAAI,MAAM,SAAS,QAAW;AAC5B,YAAM,SAAS,OAAO;AAAA,QACpB,SAAS,MAAM,WAAW,MAAM,UAAU,MAAM,MAAM,MAAM,IAAI;AAAA,MAClE;AACA,UAAI,WAAW,OAAW,WAAU,KAAK,KAAK;AAAA,UACzC,QAAO,KAAK,EAAE,OAAO,OAAO,CAAC;AAClC;AAAA,IACF;AACA,UAAM,aACJ,OAAO,IAAI,SAAS,MAAM,WAAW,MAAM,UAAU,MAAM,IAAI,CAAC,KAAK,CAAC;AACxE,UAAM,QAAQ,IAAI,IAAI,WAAW,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC;AAC3D,QAAI,WAAW,WAAW,EAAG,WAAU,KAAK,KAAK;AAAA,aACxC,MAAM,OAAO,EAAG,WAAU,KAAK,EAAE,OAAO,WAAW,CAAC;AAAA,QACxD,QAAO,KAAK,EAAE,OAAO,QAAQ,WAAW,CAAC,EAAgB,CAAC;AAAA,EACjE;AACA,SAAO,EAAE,QAAQ,WAAW,UAAU;AACxC;AAgBA,IAAM,YAAY,CAChB,eAC+B;AAAA,EAC/B,OAAO,UAAU;AAAA,EACjB,WAAW,UAAU;AAAA,EACrB,gBAAgB,UAAU;AAAA,EAC1B,MAAM,UAAU,KAAK;AACvB;AAcO,SAAS,QAAQ,QAAoB,SAAiC;AAC3E,QAAM,YAAY,OAAO;AACzB,MAAI,cAAc,QAAW;AAC3B,WAAO,EAAE,SAAS,OAAO,SAAS,cAAc,MAAM;AAAA,EACxD;AACA,QAAM,OAAO,UAAU;AACvB,MAAI,KAAK,SAAS,QAAQ,cAAc;AAGtC,WAAO,EAAE,SAAS,OAAO,SAAS,cAAc,KAAK;AAAA,EACvD;AACA,QAAM,QAAQ,KAAK,MAAM,GAAG,QAAQ,YAAY;AAChD,QAAM,OAAO,iBAAiB,KAAK;AACnC,QAAM,OAAO,QAAQ,MAAM,QAAQ,KAAK;AACxC,SAAO;AAAA,IACL,SACE,SAAS,cAAc,SAAS,SAAS,cAAc,SAAS;AAAA,IAClE,cAAc;AAAA,IACd,WAAW;AAAA,MACT,OAAO,KAAK;AAAA,MACZ,WAAW,KAAK;AAAA,MAChB,gBAAgB,KAAK;AAAA,MACrB,MAAM,MAAM;AAAA,IACd;AAAA,EACF;AACF;AAEA,IAAM,cAAc,OAA0B;AAAA,EAC5C,QAAQ;AAAA,EACR,QAAQ;AAAA,EACR,OAAO;AAAA,EACP,WAAW;AAAA,EACX,WAAW;AAAA,EACX,QAAQ;AAAA,EACR,cAAc;AAAA,EACd,SAAS;AAAA,EACT,SAAS;AAAA,EACT,cAAc;AAAA,EACd,cAAc;AAAA,EACd,WAAW;AACb;AAoBO,SAAS,MAAM,OAAmB,SAAqC;AAC5E,QAAM,SAAS,YAAY;AAC3B,QAAM,gBAAgC,CAAC;AACvC,QAAM,WAA2B,CAAC;AAElC,aAAW,UAAU,MAAM,KAAK;AAC9B,UAAM,UACJ,YAAY,SAAY,OAAO,UAAU,QAAQ,QAAQ,OAAO,EAAE;AACpE,QAAI,YAAY,eAAgB,QAAO,gBAAgB;AAAA,EACzD;AAEA,aAAW,EAAE,OAAO,OAAO,KAAK,MAAM,QAAQ;AAC5C,UAAM,SACJ,YAAY,SACR,EAAE,SAAS,OAAO,SAAS,cAAc,MAAM,IAC/C,QAAQ,QAAQ,OAAO;AAC7B,QAAI,OAAO,aAAc,QAAO,gBAAgB;AAChD,UAAM,OAAO,SAAS,MAAM,UAAU,OAAO,OAAO;AACpD,WAAO,UAAU;AAEjB,UAAM,MAAoB;AAAA,MACxB,OAAO,MAAM;AAAA,MACb,cAAc,OAAO;AAAA,MACrB,cAAc,OAAO;AAAA,MACrB,UAAU,OAAO;AAAA,MACjB,QAAQ,OAAO;AAAA,MACf,UAAU,MAAM;AAAA,MAChB,QAAQ,OAAO;AAAA,MACf;AAAA,MACA,GAAI,MAAM,SAAS,SAAY,EAAE,MAAM,MAAM,KAAK,IAAI,CAAC;AAAA,MACvD,GAAI,OAAO,eAAe,SACtB,EAAE,YAAY,OAAO,WAAW,IAChC,CAAC;AAAA;AAAA;AAAA,MAGL,IAAI,MAAM;AACR,cAAM,IACJ,OAAO,cACN,OAAO,cAAc,SAAY,UAAU,OAAO,SAAS,IAAI;AAClE,eAAO,MAAM,SAAY,EAAE,WAAW,EAAE,IAAI,CAAC;AAAA,MAC/C,GAAG;AAAA,IACL;AAEA,YAAQ,MAAM;AAAA,MACZ,KAAK;AACH,eAAO,WAAW;AAClB,iBAAS,KAAK,GAAG;AACjB;AAAA,MACF,KAAK;AACH,eAAO,SAAS;AAChB,eAAO,UAAU;AACjB;AAAA,MACF,KAAK;AACH,eAAO,aAAa;AACpB,eAAO,UAAU;AACjB,sBAAc,KAAK,GAAG;AACtB;AAAA,MACF,KAAK;AACH,eAAO,aAAa;AACpB,eAAO,UAAU;AACjB,sBAAc,KAAK,GAAG;AACtB;AAAA,MACF,KAAK;AACH,eAAO,UAAU;AACjB,eAAO,UAAU;AACjB,sBAAc,KAAK,GAAG;AACtB;AAAA,MACF,KAAK;AACH,eAAO,gBAAgB;AACvB,eAAO,UAAU;AACjB,sBAAc,KAAK,GAAG;AACtB;AAAA,MACF,KAAK;AACH,eAAO,WAAW;AAClB,eAAO,UAAU;AACjB,sBAAc,KAAK,GAAG;AACtB;AAAA,IACJ;AAAA,EACF;AAEA,SAAO,YAAY,OAAO,SAAS,IAAI,OAAO,QAAQ,OAAO,SAAS;AAGtE,QAAM,QAAQ,CAAC,GAAiB,MAA4B;AAC1D,UAAM,KAAK,GAAG,EAAE,IAAI,IAAI,EAAE,KAAK,IAAI,EAAE,YAAY,IAAI,EAAE,QAAQ;AAC/D,UAAM,KAAK,GAAG,EAAE,IAAI,IAAI,EAAE,KAAK,IAAI,EAAE,YAAY,IAAI,EAAE,QAAQ;AAC/D,WAAO,KAAK,KAAK,KAAK,KAAK,KAAK,IAAI;AAAA,EACtC;AACA,gBAAc,KAAK,KAAK;AACxB,WAAS,KAAK,KAAK;AACnB,SAAO,EAAE,QAAQ,eAAe,SAAS;AAC3C;;;ADhQA,SAAS,WAAW,MAAgB,UAA0B;AAC5D,SAAO,KAAK,IAAI,UAAU,GAAG,IAAI;AACnC;AAEA,eAAsB,aACpB,SACiC;AACjC,QAAM,QAAQ,KAAK,IAAI;AACvB,QAAM,EAAE,QAAQ,KAAK,UAAU,IAAI,MAAM;AAAA,IACvC,QAAQ,aAAa,QAAQ,IAAI;AAAA,IACjC;AAAA,MACE,GAAI,QAAQ,WAAW,SAAY,CAAC,IAAI,EAAE,YAAY,QAAQ,OAAO;AAAA,MACrE,GAAI,QAAQ,aAAa,SAAY,CAAC,IAAI,EAAE,UAAU,QAAQ,SAAS;AAAA,IACzE;AAAA,EACF;AAMA;AAAA,IACE,eAAe,QAAQ;AAAA,MACrB,GAAI,QAAQ,aAAa,SAAY,EAAE,UAAU,QAAQ,SAAS,IAAI,CAAC;AAAA,MACvE,GAAI,QAAQ,UAAU,SAAY,EAAE,OAAO,QAAQ,MAAM,IAAI,CAAC;AAAA,MAC9D,GAAI,QAAQ,UAAU,SAAY,EAAE,OAAO,QAAQ,MAAM,IAAI,CAAC;AAAA,IAChE,CAAC;AAAA,EACH;AAMA,QAAM,aACJ,QAAQ,WAAW,SACf,QAAQ,QAAQ,MAAM,IACtB,QAAQ,WAAW,OAAO,UAAU,MAAM;AAChD,QAAM,SAAS,MAAM,WAAW,UAAU;AAI1C,QAAM,UAAU,MAAM,mBAAmB,OAAO,GAAG,IAAI,CAAC,MAAM,QAAQ,CAAC,CAAC;AACxE,uBAAqB,QAAQ,QAAQ,UAAU;AAE/C,QAAM,WAAyB;AAAA,IAC7B,cAAc,QAAQ,QAAQ,OAAO,MAAM;AAAA,IAC3C,OAAO,OAAO,MAAM;AAAA,EACtB;AACA,QAAM,OAAO,OAAO,UAAU;AAK9B,QAAM,QACJ,QAAQ,UAAU,OACd,WAAW,KAAK,cAAc,SAAS,YAAY,IACnD,SAAS;AAEf,QAAM,QAAQ,MAAM,SAAS;AAAA,IAC3B,GAAG;AAAA,IACH,QAAQ;AAAA,IACR,aAAa;AAAA,IACb,WAAW;AAAA,IACX,OAAO;AAAA,IACP,MAAM;AAAA;AAAA;AAAA,IAGN,QAAQ;AAAA,IACR,QAAQ;AAAA,EACV,CAAC;AAED,QAAM,MAAoB,CAAC;AAC3B,aAAW,WAAW,MAAM,UAAU;AACpC,QAAI,YAAY,QAAS,KAAI,KAAK,GAAG,QAAQ,OAAO,WAAW;AAAA,EACjE;AACA,QAAM,EAAE,QAAQ,WAAW,UAAU,IAAI,WAAW,MAAM,UAAU,MAAM;AAC1E,oBAAkB,WAAW,OAAO,UAAU;AAC9C,oBAAkB,WAAW,UAAU;AAMvC,QAAM,SAAS;AAAA,IACb,EAAE,QAAQ,IAAI;AAAA,IACd,QAAQ,UAAU,OAAO,WAAW;AAAA,EACtC;AAEA,QAAM,QACJ,QAAQ,UAAU,OACd,WAAW,EAAE,QAAQ,IAAI,GAAG,UAAU,IAAI,IAC1C;AAEN,QAAM,QAAQ,WAAW,OAAO,QAAQ;AAAA,IACtC,WAAW,QAAQ,gBAAgB,OAAO,UAAU;AAAA,IACpD,WAAW,QAAQ,gBAAgB,OAAO,UAAU;AAAA,IACpD,QAAQ,QAAQ,aAAa,OAAO,UAAU;AAAA,EAChD,CAAC;AAED,QAAM,WAAW,CAAC,GAAG,MAAM,OAAO,QAAQ;AAC1C,MAAI,OAAO,OAAO,eAAe,GAAG;AAClC,aAAS;AAAA,MACP,GAAG,OAAO,OAAO,YAAY;AAAA,IAC/B;AAAA,EACF;AASA,QAAM,kBAAkB,OAAO,OAAO,WAAW;AACjD,MAAI,iBAAiB;AACnB,aAAS;AAAA,MACP,uCAAuC,OAAO,OAAO,MAAM;AAAA,IAE7D;AAAA,EACF;AAEA,QAAM,SAA4B;AAAA,IAChC;AAAA,IACA;AAAA,IACA,SAAS;AAAA,IACT,QAAQ,OAAO;AAAA,IACf,eAAe,OAAO;AAAA,IACtB,UAAU,OAAO;AAAA,IACjB,GAAI,UAAU,SAAY,EAAE,MAAM,IAAI,CAAC;AAAA,IACvC;AAAA,IACA,OAAO,MAAM;AAAA,IACb;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAMA,UACE,MAAM,KAAK,CAAC,MAAM,EAAE,QAAQ,KAC5B,MAAM,OAAO,QAAQ,gBAAgB,KACrC,MAAM,OAAO,WAAW,UACxB,kBACI,IACA;AAAA,IACN,OAAO,MAAM,OAAO;AAAA,IACpB,YAAY,KAAK,IAAI,IAAI;AAAA,EAC3B;AAEA,QAAM,WAAW,kBAAkB,QAAQ,QAAQ,UAAU,UAAU;AAAA,IACrE,OAAO,QAAQ,UAAU;AAAA,EAC3B,CAAC;AACD,MAAI,QAAQ,OAAQ,OAAM,UAAU,QAAQ,QAAQ,UAAU,OAAO;AACrE,SAAO,EAAE,QAAQ,SAAS;AAC5B;AAOA,SAAS,qBACP,QACA,QACA,YACM;AACN,QAAM,QAAQ,IAAI,IAAI,MAAM;AAC5B,QAAM,UAAU,CAAC,GAAG,IAAI,IAAI,OAAO,OAAO,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,SAAS,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;AAC9F,MAAI,QAAQ,WAAW,EAAG;AAC1B,QAAM,IAAI;AAAA,IACR,GAAG,UAAU,+CAA+C,QAAQ,KAAK,IAAI,CAAC,oBAC1D,OAAO,IAAI,CAAC,MAAM,SAAS,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC;AAAA,EAC/D;AACF;AASA,SAAS,kBACP,WACA,OACA,YACM;AACN,MAAI,UAAU,WAAW,EAAG;AAC5B,QAAM,QAAQ,UAAU,IAAI,CAAC,UAAU;AACrC,UAAM,UAAU,MAAM,SAAS,KAAK,CAAC,MAAM,EAAE,SAAS,MAAM,SAAS;AACrE,UAAM,UAAU,SAAS,QAAQ,eAAe,CAAC;AACjD,UAAM,eAAe,QAAQ;AAAA,MAC3B,CAAC,MAAM,EAAE,iBAAiB,MAAM;AAAA,IAClC;AACA,UAAM,OACJ,aAAa,SAAS,IAClB,wBAAwB,aAAa,IAAI,CAAC,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,IAAI,CAAC,SAAS,MAAM,QAAQ,MACpG,0CAA0C,MAAM,QAAQ,yBACtD,CAAC,GAAG,IAAI,IAAI,QAAQ,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,IAAI,KACjE,SACF;AACN,WAAO,KAAK,MAAM,QAAQ,WAAM,MAAM,IAAI,OAAO,SAAS,MAAM,SAAS,CAAC,KAAK,IAAI;AAAA,EACrF,CAAC;AACD,QAAM,IAAI;AAAA,IACR,GAAG,UAAU;AAAA,EAAuC,MAAM,KAAK,IAAI,CAAC;AAAA,EACtE;AACF;AAYA,SAAS,kBACP,WACA,YACM;AACN,MAAI,UAAU,WAAW,EAAG;AAC5B,QAAM,QAAQ,UAAU,IAAI,CAAC,EAAE,OAAO,WAAW,MAAM;AACrD,UAAM,QAAQ,CAAC,GAAG,IAAI,IAAI,WAAW,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC,EAAE,KAAK;AACvE,WAAO,KAAK,MAAM,QAAQ,WAAM,MAAM,IAAI,OAAO,SAAS,MAAM,SAAS,CAAC,YAAY,MAAM,KAAK,OAAO,CAAC;AAAA,EAC3G,CAAC;AACD,QAAM,IAAI;AAAA,IACR,GAAG,UAAU;AAAA,EAAkD,MAAM,KAAK,IAAI,CAAC;AAAA,EACjF;AACF;AAEA,SAAS,WACP,QACA,QACmB;AACnB,QAAM,QAA2B,CAAC;AAClC,QAAM,MAAM,CACV,MACA,OACA,WACS;AACT,QAAI,UAAU,OAAW;AACzB,UAAM,KAAK,EAAE,MAAM,OAAO,QAAQ,UAAU,SAAS,MAAM,CAAC;AAAA,EAC9D;AACA,MAAI,aAAa,OAAO,WAAW,OAAO,SAAS;AACnD,MAAI,aAAa,OAAO,WAAW,OAAO,SAAS;AACnD,MAAI,UAAU,OAAO,QAAQ,OAAO,YAAY;AAChD,SAAO;AACT;AASA,SAAS,WACP,OACA,UACA,MACa;AACb,QAAM,OAAO,CACX,MACA,OACA,aACe;AAAA,IACf;AAAA,IACA;AAAA,IACA;AAAA,IACA,QAAQ,MAAM,OAAO,OAAO,EAAE;AAAA,EAChC;AAEA,QAAM,QAAqB;AAAA,IACzB,KAAK,YAAY,SAAS,cAAc,QAAQ;AAAA,EAClD;AACA,aAAW,QAAQ,CAAC,GAAG,KAAK,YAAY,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,GAAG;AAC/D,UAAM;AAAA,MACJ,KAAK,gBAAgB,MAAM,EAAE,cAAc,MAAM,OAAO,SAAS,MAAM,CAAC;AAAA,IAC1E;AAAA,EACF;AACA,aAAW,YAAY,CAAC,GAAG,KAAK,QAAQ,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,GAAG;AAC/D,UAAM;AAAA,MACJ,KAAK,kBAAkB,UAAU;AAAA,QAC/B,cAAc,SAAS;AAAA,QACvB,OAAO,EAAE,UAAU,UAAU,SAAS,MAAM,SAAS;AAAA,MACvD,CAAC;AAAA,IACH;AAAA,EACF;AACA,aAAW,YAAY,CAAC,GAAG,KAAK,QAAQ,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,GAAG;AAC/D,UAAM;AAAA,MACJ,KAAK,kBAAkB,UAAU;AAAA,QAC/B,cAAc,SAAS;AAAA,QACvB,OAAO,EAAE,UAAU,SAAS,MAAM,UAAU,SAAS;AAAA,MACvD,CAAC;AAAA,IACH;AAAA,EACF;AACA,SAAO;AACT;","names":[]}
+{"version":3,"sources":["../src/tracevals/commands/calibrate.ts","../src/tracevals/calibrate/score.ts"],"sourcesContent":["/**\n * `manni tracevals calibrate <traces...> --labels <file>` (ADR 01022).\n *\n * The tool could report what it decided; it could not report whether it was\n * right. This command runs the batch (ADR 01018), joins the results against a\n * human's answers, and reports the three numbers the calibration page used to\n * ask people to count by hand — false passes, false fails, and review volume —\n * plus which eval disagreed.\n *\n * `--sweep` costs no inference beyond the one run it already made: a judged\n * eval's consensus carries the whole `JudgeRun[]`, so a different zone\n * threshold or a smaller ensemble is arithmetic over verdicts already on disk.\n * Read-only throughout — labels, traces, and artifacts are never written.\n */\nimport { writeFile } from \"node:fs/promises\";\nimport { basename, resolve } from \"node:path\";\nimport { discoverConfig } from \"../core/config.js\";\nimport { assertProviderSelection, selectProvider } from \"../judge/provider.js\";\nimport { renderCalibration, type ReportFormat } from \"../reporters/index.js\";\nimport { TracevalsError, type EvalResult } from \"../types.js\";\nimport { loadLabels, type EvalLabel } from \"../calibrate/labels.js\";\nimport { joinLabels, score, type AmbiguousLabel } from \"../calibrate/score.js\";\nimport type {\n  CalibrationGate,\n  CalibrationReport,\n  JudgeSetting,\n  SweepCell,\n} from \"../calibrate/types.js\";\nimport {\n  resolveBatchTraces,\n  runBatch,\n  type BatchCommandOptions,\n} from \"./batch.js\";\n\nexport interface CalibrateCommandOptions extends BatchCommandOptions {\n  /** Overrides `calibrate.labels`; resolved against the working directory. */\n  labels?: string;\n  /** Re-score the corpus across the configured grid. */\n  sweep?: boolean;\n  /** Thresholds; undefined defers to the config, which defaults to no gate. */\n  maxFalsePass?: number;\n  maxFalseFail?: number;\n  maxReview?: number;\n}\n\nexport interface CalibrateCommandResult {\n  report: CalibrationReport;\n  rendered: string;\n}\n\n/** Largest ensemble the sweep needs cached, so every cell can be re-scored. */\nfunction sweepDepth(grid: number[], baseline: number): number {\n  return Math.max(baseline, ...grid);\n}\n\nexport async function runCalibrate(\n  options: CalibrateCommandOptions,\n): Promise<CalibrateCommandResult> {\n  const start = Date.now();\n  const { config, dir: configDir } = await discoverConfig(\n    options.configDir ?? process.cwd(),\n    {\n      ...(options.config === undefined ? {} : { configPath: options.config }),\n      ...(options.noConfig === undefined ? {} : { noConfig: options.noConfig }),\n    },\n  );\n\n  // Before the labels file is read: a contradictory `--local --provider`, an\n  // unknown provider name, or a model with no provider to own it is a usage\n  // error, and a usage error must not arrive behind an unrelated one about a\n  // file the run would never have needed.\n  assertProviderSelection(\n    selectProvider(config, {\n      ...(options.provider !== undefined ? { provider: options.provider } : {}),\n      ...(options.model !== undefined ? { model: options.model } : {}),\n      ...(options.local !== undefined ? { local: options.local } : {}),\n    }),\n  );\n\n  // A flag overrides the config rather than bypassing it (CLAUDE.md,\n  // \"Config <-> CLI flags\"). An explicit --labels is a working-directory path,\n  // the way every other path argument on the CLI is; the config default is a\n  // project path and resolves against the config file's directory.\n  const labelsFile =\n    options.labels !== undefined\n      ? resolve(options.labels)\n      : resolve(configDir, config.calibrate.labels);\n  const labels = await loadLabels(labelsFile);\n\n  // Resolve the corpus once and hand the list to the batch, so the membership\n  // check below cannot disagree with what actually ran.\n  const corpus = (await resolveBatchTraces(options)).map((f) => resolve(f));\n  assertLabelsInCorpus(labels, corpus, labelsFile);\n\n  const baseline: JudgeSetting = {\n    ensembleRuns: options.runs ?? config.judge.ensembleRuns,\n    zones: config.judge.zones,\n  };\n  const grid = config.calibrate.sweep;\n  // With --sweep the corpus is judged once at the deepest ensemble the grid\n  // asks for, and every cell — the baseline included — is re-scored from those\n  // runs. Judging at the configured depth instead would make the larger cells\n  // unscoreable, and judging per cell would multiply the bill by the grid.\n  const depth =\n    options.sweep === true\n      ? sweepDepth(grid.ensembleRuns, baseline.ensembleRuns)\n      : baseline.ensembleRuns;\n\n  const batch = await runBatch({\n    ...options,\n    traces: corpus,\n    allProjects: undefined,\n    newerThan: undefined,\n    limit: undefined,\n    runs: depth,\n    // The calibration report is the deliverable; rendering the batch as well\n    // would write the wrong thing to --output.\n    format: \"json\",\n    output: undefined,\n  });\n\n  const all: EvalResult[] = [];\n  for (const outcome of batch.outcomes) {\n    if (\"report\" in outcome) all.push(...outcome.report.evalResults);\n  }\n  const { joined, unmatched, ambiguous } = joinLabels(batch.outcomes, labels);\n  assertNoUnmatched(unmatched, batch, labelsFile);\n  assertNoAmbiguous(ambiguous, labelsFile);\n\n  // Without a sweep the reported outcomes are used verbatim, so the headline\n  // numbers are exactly the verdicts `run` produced. With one, the corpus was\n  // judged deeper than the configured ensemble, so the baseline row has to be\n  // re-scored back down to it.\n  const scored = score(\n    { joined, all },\n    options.sweep === true ? baseline : undefined,\n  );\n\n  const sweep =\n    options.sweep === true\n      ? buildSweep({ joined, all }, baseline, grid)\n      : undefined;\n\n  const gates = buildGates(scored.counts, {\n    falsePass: options.maxFalsePass ?? config.calibrate.maxFalsePass,\n    falseFail: options.maxFalseFail ?? config.calibrate.maxFalseFail,\n    review: options.maxReview ?? config.calibrate.maxReview,\n  });\n\n  const warnings = [...batch.report.warnings];\n  if (scored.counts.insufficient > 0) {\n    warnings.push(\n      `${scored.counts.insufficient} labelled eval(s) had fewer cached runs than the setting asks for and were left at their reported outcome`,\n    );\n  }\n\n  // A measurement that measured nothing must not report a pass.\n  //\n  // Every label joining to a `skipped` result — `--deterministic-only` over\n  // ai-graded labels, a budget exhausted on the first trace — leaves `scored`\n  // at 0, `agreement` at null, and each threshold reading \"0 of at most N, not\n  // exceeded\". The gates are not lying; the denominator behind them is empty.\n  // Saying so is the report's job, and exit 0 would say the opposite.\n  const measuredNothing = scored.counts.scored === 0;\n  if (measuredNothing) {\n    warnings.push(\n      `no labelled eval produced evidence: ${scored.counts.labels} label(s) all joined to a skipped result, ` +\n        `so agreement and every --max-* threshold are computed over an empty denominator`,\n    );\n  }\n\n  const report: CalibrationReport = {\n    labelsFile,\n    corpus,\n    setting: baseline,\n    counts: scored.counts,\n    disagreements: scored.disagreements,\n    unscored: scored.unscored,\n    ...(sweep !== undefined ? { sweep } : {}),\n    gates,\n    batch: batch.report,\n    warnings,\n    // A calibration run is a measurement, not a gate: disagreement is the\n    // finding, not a failure. Exit 1 only when a threshold was asked for and\n    // missed, or when the measurement itself is incomplete — a trace lost, the\n    // shared judge budget cut the corpus short, or nothing scored at all. An\n    // incomplete number presented as a clean one is worse than no number.\n    exitCode:\n      gates.some((g) => g.exceeded) ||\n      batch.report.summary.tracesErrored > 0 ||\n      batch.report.budget !== undefined ||\n      measuredNothing\n        ? 1\n        : 0,\n    turns: batch.report.turns,\n    durationMs: Date.now() - start,\n  };\n\n  const rendered = renderCalibration(report, options.format ?? \"pretty\", {\n    color: options.color === true,\n  });\n  if (options.output) await writeFile(options.output, rendered, \"utf-8\");\n  return { report, rendered };\n}\n\n/**\n * Every label must name a trace this run evaluates. Checked before the batch,\n * because the alternative is paying for a corpus and then being told the file\n * was misspelled.\n */\nfunction assertLabelsInCorpus(\n  labels: EvalLabel[],\n  corpus: string[],\n  labelsFile: string,\n): void {\n  const known = new Set(corpus);\n  const missing = [...new Set(labels.filter((l) => !known.has(l.traceFile)).map((l) => l.trace))];\n  if (missing.length === 0) return;\n  throw new TracevalsError(\n    `${labelsFile}: label(s) name trace(s) not in the corpus: ${missing.join(\", \")}. ` +\n      `The corpus is: ${corpus.map((f) => basename(f)).join(\", \")}`,\n  );\n}\n\n/**\n * A label that matched nothing is an error, not a quiet omission.\n *\n * It has to be: a typo'd eval id contributes to no count, so the report would\n * read as a cleaner corpus than it is — the same silent-nothing failure the\n * empty-selector rule exists to prevent (ADR 01018).\n */\nfunction assertNoUnmatched(\n  unmatched: EvalLabel[],\n  batch: { outcomes: { file: string; report?: { evalResults: EvalResult[] } }[] },\n  labelsFile: string,\n): void {\n  if (unmatched.length === 0) return;\n  const lines = unmatched.map((label) => {\n    const outcome = batch.outcomes.find((o) => o.file === label.traceFile);\n    const results = outcome?.report?.evalResults ?? [];\n    const sameArtifact = results.filter(\n      (r) => r.artifactName === label.artifact,\n    );\n    const hint =\n      sameArtifact.length > 0\n        ? `that trace evaluated ${sameArtifact.map((r) => r.evalName).sort().join(\", \")} for \"${label.artifact}\"`\n        : `that trace resolved no artifact named \"${label.artifact}\" — it evaluated ${\n            [...new Set(results.map((r) => r.artifactName))].sort().join(\", \") ||\n            \"nothing\"\n          }`;\n    return `  ${label.artifact} › ${label.eval} on ${basename(label.traceFile)} (${hint})`;\n  });\n  throw new TracevalsError(\n    `${labelsFile}: label(s) matched no eval result:\\n${lines.join(\"\\n\")}`,\n  );\n}\n\n/**\n * A label that omits `type:` over a name two artifact kinds both answer cannot\n * be resolved, and must not be guessed. Binding it to whichever result was\n * indexed last would score agreement against a verdict belonging to a\n * different artifact — the report would read as measured when it is wrong,\n * which is the one thing this command exists not to do.\n *\n * Operational, like `assertNoUnmatched`: the labels file is under-specified,\n * so this is exit 2 (bad input), not exit 1 (a check failed).\n */\nfunction assertNoAmbiguous(\n  ambiguous: AmbiguousLabel[],\n  labelsFile: string,\n): void {\n  if (ambiguous.length === 0) return;\n  const lines = ambiguous.map(({ label, candidates }) => {\n    const kinds = [...new Set(candidates.map((r) => r.artifactType))].sort();\n    return `  ${label.artifact} › ${label.eval} on ${basename(label.traceFile)} matches ${kinds.join(\" and \")} — add \\`type:\\` to say which`;\n  });\n  throw new TracevalsError(\n    `${labelsFile}: label(s) name an artifact two kinds answer:\\n${lines.join(\"\\n\")}`,\n  );\n}\n\nfunction buildGates(\n  counts: { falsePass: number; falseFail: number; reviewVolume: number },\n  limits: { falsePass?: number; falseFail?: number; review?: number },\n): CalibrationGate[] {\n  const gates: CalibrationGate[] = [];\n  const add = (\n    name: CalibrationGate[\"name\"],\n    limit: number | undefined,\n    actual: number,\n  ): void => {\n    if (limit === undefined) return;\n    gates.push({ name, limit, actual, exceeded: actual > limit });\n  };\n  add(\"falsePass\", limits.falsePass, counts.falsePass);\n  add(\"falseFail\", limits.falseFail, counts.falseFail);\n  add(\"review\", limits.review, counts.reviewVolume);\n  return gates;\n}\n\n/**\n * One axis at a time, every other knob held at its configured value.\n *\n * A full cross-product would be dozens of rows and would answer a question\n * nobody asked; calibration is \"move one knob, re-run, recount\", and the\n * per-axis shape is what makes a row readable as *the effect of that knob*.\n */\nfunction buildSweep(\n  input: Parameters<typeof score>[0],\n  baseline: JudgeSetting,\n  grid: { ensembleRuns: number[]; autoPass: number[]; autoFail: number[] },\n): SweepCell[] {\n  const cell = (\n    axis: SweepCell[\"axis\"],\n    value: number,\n    setting: JudgeSetting,\n  ): SweepCell => ({\n    axis,\n    value,\n    setting,\n    counts: score(input, setting).counts,\n  });\n\n  const cells: SweepCell[] = [\n    cell(\"baseline\", baseline.ensembleRuns, baseline),\n  ];\n  for (const runs of [...grid.ensembleRuns].sort((a, b) => a - b)) {\n    cells.push(\n      cell(\"ensembleRuns\", runs, { ensembleRuns: runs, zones: baseline.zones }),\n    );\n  }\n  for (const autoPass of [...grid.autoPass].sort((a, b) => a - b)) {\n    cells.push(\n      cell(\"zones.autoPass\", autoPass, {\n        ensembleRuns: baseline.ensembleRuns,\n        zones: { autoPass, autoFail: baseline.zones.autoFail },\n      }),\n    );\n  }\n  for (const autoFail of [...grid.autoFail].sort((a, b) => a - b)) {\n    cells.push(\n      cell(\"zones.autoFail\", autoFail, {\n        ensembleRuns: baseline.ensembleRuns,\n        zones: { autoPass: baseline.zones.autoPass, autoFail },\n      }),\n    );\n  }\n  return cells;\n}\n\nexport type { ReportFormat };\n","/**\n * Joining labels to results, and re-scoring cached verdicts (ADR 01022).\n *\n * Everything here is pure: results and labels in, counts out. That is what\n * makes a sweep free. A judged eval's `ConsensusResult` carries the whole\n * `JudgeRun[]` it was built from, so a different zone threshold — or a smaller\n * ensemble — is a fresh call to `computeConsensus`/`zoneFor` over runs the\n * corpus already produced, not a fresh call to a provider.\n */\nimport { computeConsensus, zoneFor } from \"@hawkeyexl/inference\";\nimport type { BatchOutcome } from \"../aggregate.js\";\nimport type { EvalResult, Outcome } from \"../types.js\";\nimport { labelKey, type EvalLabel, type LabelOutcome } from \"./labels.js\";\nimport type {\n  CalibrationCounts,\n  Disagreement,\n  JoinKind,\n  JudgeSetting,\n} from \"./types.js\";\n\n/**\n * How one label met one result, over the full (expected × actual) grid.\n *\n * The order matters. `skipped` and `error` are tested *first*, so a labelled\n * eval that produced no evidence can never be counted as agreement — the same\n * rule that keeps an errored judge run from becoming a silent pass.\n */\nexport function classify(expected: LabelOutcome, actual: Outcome): JoinKind {\n  if (actual === \"skipped\") return \"skipped\";\n  if (actual === \"error\") return \"error\";\n  if (actual === expected) return \"agree\";\n  if (expected === \"fail\" && actual === \"pass\") return \"false-pass\";\n  if (expected === \"pass\" && actual === \"fail\") return \"false-fail\";\n  if (actual === \"needs-review\") return \"review\";\n  return \"missed-review\";\n}\n\nexport interface JoinedLabel {\n  label: EvalLabel;\n  result: EvalResult;\n}\n\nexport interface JoinResult {\n  joined: JoinedLabel[];\n  /**\n   * Labels that matched no result. Never silently dropped: a typo'd eval id\n   * would otherwise deflate every count on the report by one.\n   */\n  unmatched: EvalLabel[];\n  /**\n   * Labels that omit `type:` over a name two artifact kinds both answer. Never\n   * bound to one of them: a skill and the slash command that types it can share\n   * a name in one trace, and picking whichever was indexed last would measure\n   * agreement against a verdict belonging to something else — a false pass or\n   * false fail reported with total confidence.\n   */\n  ambiguous: AmbiguousLabel[];\n}\n\n/** An untyped label and every result its name matched. */\nexport interface AmbiguousLabel {\n  label: EvalLabel;\n  candidates: EvalResult[];\n}\n\n/**\n * Match each label to the result its trace produced. The key is\n * `(trace file, artifact name, eval id)`, with the artifact *type* joining too\n * when the label states one — needed only for a skill and an agent sharing a\n * name, which is legal and does happen.\n */\nexport function joinLabels(\n  outcomes: BatchOutcome[],\n  labels: EvalLabel[],\n): JoinResult {\n  // Typed keys are unique by construction. Untyped ones are not: two artifact\n  // kinds may answer the same name, so keep every candidate rather than\n  // letting the last one indexed overwrite the rest.\n  const byType = new Map<string, EvalResult>();\n  const byName = new Map<string, EvalResult[]>();\n  for (const outcome of outcomes) {\n    if (!(\"report\" in outcome)) continue;\n    for (const result of outcome.report.evalResults) {\n      byType.set(\n        labelKey(\n          outcome.file,\n          result.artifactName,\n          result.evalName,\n          result.artifactType,\n        ),\n        result,\n      );\n      const key = labelKey(outcome.file, result.artifactName, result.evalName);\n      const seen = byName.get(key);\n      if (seen === undefined) byName.set(key, [result]);\n      else seen.push(result);\n    }\n  }\n\n  const joined: JoinedLabel[] = [];\n  const unmatched: EvalLabel[] = [];\n  const ambiguous: AmbiguousLabel[] = [];\n  for (const label of labels) {\n    if (label.type !== undefined) {\n      const result = byType.get(\n        labelKey(label.traceFile, label.artifact, label.eval, label.type),\n      );\n      if (result === undefined) unmatched.push(label);\n      else joined.push({ label, result });\n      continue;\n    }\n    const candidates =\n      byName.get(labelKey(label.traceFile, label.artifact, label.eval)) ?? [];\n    const kinds = new Set(candidates.map((r) => r.artifactType));\n    if (candidates.length === 0) unmatched.push(label);\n    else if (kinds.size > 1) ambiguous.push({ label, candidates });\n    else joined.push({ label, result: candidates[0] as EvalResult });\n  }\n  return { joined, unmatched, ambiguous };\n}\n\nexport interface Rescored {\n  outcome: Outcome;\n  /** The cache held fewer runs than the setting asks for. */\n  insufficient: boolean;\n  /**\n   * The arithmetic **at this setting**, not at the setting the corpus was\n   * judged with. A sweep judges deeper than most of its cells, so reporting\n   * the full cached ensemble beside a re-scored verdict would show a reader\n   * evidence that does not add up to the outcome next to it.\n   */\n  consensus?: Disagreement[\"consensus\"];\n}\n\n/** The numbers a reader needs from a consensus, without the run bodies. */\nconst summarise = (\n  consensus: NonNullable<EvalResult[\"consensus\"]>,\n): Disagreement[\"consensus\"] => ({\n  votes: consensus.votes,\n  agreement: consensus.agreement,\n  meanConfidence: consensus.meanConfidence,\n  runs: consensus.runs.length,\n});\n\n/**\n * What this result would have been at a different setting.\n *\n * A result with no `consensus` — deterministic, `human`, skipped, errored — is\n * returned untouched: zones and ensemble size do not reach it, and pretending\n * otherwise would make a sweep look like it moved things it cannot move.\n *\n * A smaller `ensembleRuns` re-scores the **first k** of the cached runs. That\n * is a sub-sample of the ensemble that was actually drawn, not a fresh k-run\n * ensemble; at temperature 0 the runs are independent samples of one prompt,\n * so it is an honest estimate, and the alternative is paying per grid cell.\n */\nexport function rescore(result: EvalResult, setting: JudgeSetting): Rescored {\n  const consensus = result.consensus;\n  if (consensus === undefined) {\n    return { outcome: result.outcome, insufficient: false };\n  }\n  const runs = consensus.runs;\n  if (runs.length < setting.ensembleRuns) {\n    // Never scale up by guessing. Report the setting as unreachable and leave\n    // the row at what it actually was.\n    return { outcome: result.outcome, insufficient: true };\n  }\n  const slice = runs.slice(0, setting.ensembleRuns);\n  const base = computeConsensus(slice);\n  const zone = zoneFor(base, setting.zones);\n  return {\n    outcome:\n      zone === \"auto-pass\" ? \"pass\" : zone === \"auto-fail\" ? \"fail\" : \"needs-review\",\n    insufficient: false,\n    consensus: {\n      votes: base.votes,\n      agreement: base.agreement,\n      meanConfidence: base.meanConfidence,\n      runs: slice.length,\n    },\n  };\n}\n\nconst emptyCounts = (): CalibrationCounts => ({\n  labels: 0,\n  scored: 0,\n  agree: 0,\n  falsePass: 0,\n  falseFail: 0,\n  review: 0,\n  missedReview: 0,\n  errored: 0,\n  skipped: 0,\n  reviewVolume: 0,\n  insufficient: 0,\n  agreement: null,\n});\n\nexport interface ScoreInput {\n  joined: JoinedLabel[];\n  /** Every eval result in the corpus, for the corpus-wide review volume. */\n  all: EvalResult[];\n}\n\nexport interface ScoreResult {\n  counts: CalibrationCounts;\n  disagreements: Disagreement[];\n  unscored: Disagreement[];\n}\n\n/**\n * Score the joined corpus, optionally at a setting other than the one it ran\n * at. Omit `setting` and the reported outcomes are used verbatim, so the\n * headline numbers of a plain `calibrate` are exactly the verdicts `run`\n * would have produced — no re-derivation standing between the two commands.\n */\nexport function score(input: ScoreInput, setting?: JudgeSetting): ScoreResult {\n  const counts = emptyCounts();\n  const disagreements: Disagreement[] = [];\n  const unscored: Disagreement[] = [];\n\n  for (const result of input.all) {\n    const outcome =\n      setting === undefined ? result.outcome : rescore(result, setting).outcome;\n    if (outcome === \"needs-review\") counts.reviewVolume += 1;\n  }\n\n  for (const { label, result } of input.joined) {\n    const scored: Rescored =\n      setting === undefined\n        ? { outcome: result.outcome, insufficient: false }\n        : rescore(result, setting);\n    if (scored.insufficient) counts.insufficient += 1;\n    const kind = classify(label.expected, scored.outcome);\n    counts.labels += 1;\n\n    const row: Disagreement = {\n      trace: label.traceFile,\n      artifactType: result.artifactType,\n      artifactName: result.artifactName,\n      evalName: result.evalName,\n      grader: result.grader,\n      expected: label.expected,\n      actual: scored.outcome,\n      kind,\n      ...(label.note !== undefined ? { note: label.note } : {}),\n      ...(result.skipReason !== undefined\n        ? { skipReason: result.skipReason }\n        : {}),\n      // At the setting being scored, falling back to the run's own numbers\n      // when nothing was re-scored.\n      ...(() => {\n        const c =\n          scored.consensus ??\n          (result.consensus !== undefined ? summarise(result.consensus) : undefined);\n        return c !== undefined ? { consensus: c } : {};\n      })(),\n    };\n\n    switch (kind) {\n      case \"skipped\":\n        counts.skipped += 1;\n        unscored.push(row);\n        break;\n      case \"agree\":\n        counts.agree += 1;\n        counts.scored += 1;\n        break;\n      case \"false-pass\":\n        counts.falsePass += 1;\n        counts.scored += 1;\n        disagreements.push(row);\n        break;\n      case \"false-fail\":\n        counts.falseFail += 1;\n        counts.scored += 1;\n        disagreements.push(row);\n        break;\n      case \"review\":\n        counts.review += 1;\n        counts.scored += 1;\n        disagreements.push(row);\n        break;\n      case \"missed-review\":\n        counts.missedReview += 1;\n        counts.scored += 1;\n        disagreements.push(row);\n        break;\n      case \"error\":\n        counts.errored += 1;\n        counts.scored += 1;\n        disagreements.push(row);\n        break;\n    }\n  }\n\n  counts.agreement = counts.scored > 0 ? counts.agree / counts.scored : null;\n  // Byte comparison, so the ubuntu and windows legs order a report the same\n  // way and two runs over one corpus stay diffable.\n  const byRow = (a: Disagreement, b: Disagreement): number => {\n    const ka = `${a.kind} ${a.trace} ${a.artifactName} ${a.evalName}`;\n    const kb = `${b.kind} ${b.trace} ${b.artifactName} ${b.evalName}`;\n    return ka < kb ? -1 : ka > kb ? 1 : 0;\n  };\n  disagreements.sort(byRow);\n  unscored.sort(byRow);\n  return { counts, disagreements, unscored };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAcA,SAAS,iBAAiB;AAC1B,SAAS,UAAU,eAAe;;;ACNlC,SAAS,kBAAkB,eAAe;AAkBnC,SAAS,SAAS,UAAwB,QAA2B;AAC1E,MAAI,WAAW,UAAW,QAAO;AACjC,MAAI,WAAW,QAAS,QAAO;AAC/B,MAAI,WAAW,SAAU,QAAO;AAChC,MAAI,aAAa,UAAU,WAAW,OAAQ,QAAO;AACrD,MAAI,aAAa,UAAU,WAAW,OAAQ,QAAO;AACrD,MAAI,WAAW,eAAgB,QAAO;AACtC,SAAO;AACT;AAoCO,SAAS,WACd,UACA,QACY;AAIZ,QAAM,SAAS,oBAAI,IAAwB;AAC3C,QAAM,SAAS,oBAAI,IAA0B;AAC7C,aAAW,WAAW,UAAU;AAC9B,QAAI,EAAE,YAAY,SAAU;AAC5B,eAAW,UAAU,QAAQ,OAAO,aAAa;AAC/C,aAAO;AAAA,QACL;AAAA,UACE,QAAQ;AAAA,UACR,OAAO;AAAA,UACP,OAAO;AAAA,UACP,OAAO;AAAA,QACT;AAAA,QACA;AAAA,MACF;AACA,YAAM,MAAM,SAAS,QAAQ,MAAM,OAAO,cAAc,OAAO,QAAQ;AACvE,YAAM,OAAO,OAAO,IAAI,GAAG;AAC3B,UAAI,SAAS,OAAW,QAAO,IAAI,KAAK,CAAC,MAAM,CAAC;AAAA,UAC3C,MAAK,KAAK,MAAM;AAAA,IACvB;AAAA,EACF;AAEA,QAAM,SAAwB,CAAC;AAC/B,QAAM,YAAyB,CAAC;AAChC,QAAM,YAA8B,CAAC;AACrC,aAAW,SAAS,QAAQ;AAC1B,QAAI,MAAM,SAAS,QAAW;AAC5B,YAAM,SAAS,OAAO;AAAA,QACpB,SAAS,MAAM,WAAW,MAAM,UAAU,MAAM,MAAM,MAAM,IAAI;AAAA,MAClE;AACA,UAAI,WAAW,OAAW,WAAU,KAAK,KAAK;AAAA,UACzC,QAAO,KAAK,EAAE,OAAO,OAAO,CAAC;AAClC;AAAA,IACF;AACA,UAAM,aACJ,OAAO,IAAI,SAAS,MAAM,WAAW,MAAM,UAAU,MAAM,IAAI,CAAC,KAAK,CAAC;AACxE,UAAM,QAAQ,IAAI,IAAI,WAAW,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC;AAC3D,QAAI,WAAW,WAAW,EAAG,WAAU,KAAK,KAAK;AAAA,aACxC,MAAM,OAAO,EAAG,WAAU,KAAK,EAAE,OAAO,WAAW,CAAC;AAAA,QACxD,QAAO,KAAK,EAAE,OAAO,QAAQ,WAAW,CAAC,EAAgB,CAAC;AAAA,EACjE;AACA,SAAO,EAAE,QAAQ,WAAW,UAAU;AACxC;AAgBA,IAAM,YAAY,CAChB,eAC+B;AAAA,EAC/B,OAAO,UAAU;AAAA,EACjB,WAAW,UAAU;AAAA,EACrB,gBAAgB,UAAU;AAAA,EAC1B,MAAM,UAAU,KAAK;AACvB;AAcO,SAAS,QAAQ,QAAoB,SAAiC;AAC3E,QAAM,YAAY,OAAO;AACzB,MAAI,cAAc,QAAW;AAC3B,WAAO,EAAE,SAAS,OAAO,SAAS,cAAc,MAAM;AAAA,EACxD;AACA,QAAM,OAAO,UAAU;AACvB,MAAI,KAAK,SAAS,QAAQ,cAAc;AAGtC,WAAO,EAAE,SAAS,OAAO,SAAS,cAAc,KAAK;AAAA,EACvD;AACA,QAAM,QAAQ,KAAK,MAAM,GAAG,QAAQ,YAAY;AAChD,QAAM,OAAO,iBAAiB,KAAK;AACnC,QAAM,OAAO,QAAQ,MAAM,QAAQ,KAAK;AACxC,SAAO;AAAA,IACL,SACE,SAAS,cAAc,SAAS,SAAS,cAAc,SAAS;AAAA,IAClE,cAAc;AAAA,IACd,WAAW;AAAA,MACT,OAAO,KAAK;AAAA,MACZ,WAAW,KAAK;AAAA,MAChB,gBAAgB,KAAK;AAAA,MACrB,MAAM,MAAM;AAAA,IACd;AAAA,EACF;AACF;AAEA,IAAM,cAAc,OAA0B;AAAA,EAC5C,QAAQ;AAAA,EACR,QAAQ;AAAA,EACR,OAAO;AAAA,EACP,WAAW;AAAA,EACX,WAAW;AAAA,EACX,QAAQ;AAAA,EACR,cAAc;AAAA,EACd,SAAS;AAAA,EACT,SAAS;AAAA,EACT,cAAc;AAAA,EACd,cAAc;AAAA,EACd,WAAW;AACb;AAoBO,SAAS,MAAM,OAAmB,SAAqC;AAC5E,QAAM,SAAS,YAAY;AAC3B,QAAM,gBAAgC,CAAC;AACvC,QAAM,WAA2B,CAAC;AAElC,aAAW,UAAU,MAAM,KAAK;AAC9B,UAAM,UACJ,YAAY,SAAY,OAAO,UAAU,QAAQ,QAAQ,OAAO,EAAE;AACpE,QAAI,YAAY,eAAgB,QAAO,gBAAgB;AAAA,EACzD;AAEA,aAAW,EAAE,OAAO,OAAO,KAAK,MAAM,QAAQ;AAC5C,UAAM,SACJ,YAAY,SACR,EAAE,SAAS,OAAO,SAAS,cAAc,MAAM,IAC/C,QAAQ,QAAQ,OAAO;AAC7B,QAAI,OAAO,aAAc,QAAO,gBAAgB;AAChD,UAAM,OAAO,SAAS,MAAM,UAAU,OAAO,OAAO;AACpD,WAAO,UAAU;AAEjB,UAAM,MAAoB;AAAA,MACxB,OAAO,MAAM;AAAA,MACb,cAAc,OAAO;AAAA,MACrB,cAAc,OAAO;AAAA,MACrB,UAAU,OAAO;AAAA,MACjB,QAAQ,OAAO;AAAA,MACf,UAAU,MAAM;AAAA,MAChB,QAAQ,OAAO;AAAA,MACf;AAAA,MACA,GAAI,MAAM,SAAS,SAAY,EAAE,MAAM,MAAM,KAAK,IAAI,CAAC;AAAA,MACvD,GAAI,OAAO,eAAe,SACtB,EAAE,YAAY,OAAO,WAAW,IAChC,CAAC;AAAA;AAAA;AAAA,MAGL,IAAI,MAAM;AACR,cAAM,IACJ,OAAO,cACN,OAAO,cAAc,SAAY,UAAU,OAAO,SAAS,IAAI;AAClE,eAAO,MAAM,SAAY,EAAE,WAAW,EAAE,IAAI,CAAC;AAAA,MAC/C,GAAG;AAAA,IACL;AAEA,YAAQ,MAAM;AAAA,MACZ,KAAK;AACH,eAAO,WAAW;AAClB,iBAAS,KAAK,GAAG;AACjB;AAAA,MACF,KAAK;AACH,eAAO,SAAS;AAChB,eAAO,UAAU;AACjB;AAAA,MACF,KAAK;AACH,eAAO,aAAa;AACpB,eAAO,UAAU;AACjB,sBAAc,KAAK,GAAG;AACtB;AAAA,MACF,KAAK;AACH,eAAO,aAAa;AACpB,eAAO,UAAU;AACjB,sBAAc,KAAK,GAAG;AACtB;AAAA,MACF,KAAK;AACH,eAAO,UAAU;AACjB,eAAO,UAAU;AACjB,sBAAc,KAAK,GAAG;AACtB;AAAA,MACF,KAAK;AACH,eAAO,gBAAgB;AACvB,eAAO,UAAU;AACjB,sBAAc,KAAK,GAAG;AACtB;AAAA,MACF,KAAK;AACH,eAAO,WAAW;AAClB,eAAO,UAAU;AACjB,sBAAc,KAAK,GAAG;AACtB;AAAA,IACJ;AAAA,EACF;AAEA,SAAO,YAAY,OAAO,SAAS,IAAI,OAAO,QAAQ,OAAO,SAAS;AAGtE,QAAM,QAAQ,CAAC,GAAiB,MAA4B;AAC1D,UAAM,KAAK,GAAG,EAAE,IAAI,IAAI,EAAE,KAAK,IAAI,EAAE,YAAY,IAAI,EAAE,QAAQ;AAC/D,UAAM,KAAK,GAAG,EAAE,IAAI,IAAI,EAAE,KAAK,IAAI,EAAE,YAAY,IAAI,EAAE,QAAQ;AAC/D,WAAO,KAAK,KAAK,KAAK,KAAK,KAAK,IAAI;AAAA,EACtC;AACA,gBAAc,KAAK,KAAK;AACxB,WAAS,KAAK,KAAK;AACnB,SAAO,EAAE,QAAQ,eAAe,SAAS;AAC3C;;;ADhQA,SAAS,WAAW,MAAgB,UAA0B;AAC5D,SAAO,KAAK,IAAI,UAAU,GAAG,IAAI;AACnC;AAEA,eAAsB,aACpB,SACiC;AACjC,QAAM,QAAQ,KAAK,IAAI;AACvB,QAAM,EAAE,QAAQ,KAAK,UAAU,IAAI,MAAM;AAAA,IACvC,QAAQ,aAAa,QAAQ,IAAI;AAAA,IACjC;AAAA,MACE,GAAI,QAAQ,WAAW,SAAY,CAAC,IAAI,EAAE,YAAY,QAAQ,OAAO;AAAA,MACrE,GAAI,QAAQ,aAAa,SAAY,CAAC,IAAI,EAAE,UAAU,QAAQ,SAAS;AAAA,IACzE;AAAA,EACF;AAMA;AAAA,IACE,eAAe,QAAQ;AAAA,MACrB,GAAI,QAAQ,aAAa,SAAY,EAAE,UAAU,QAAQ,SAAS,IAAI,CAAC;AAAA,MACvE,GAAI,QAAQ,UAAU,SAAY,EAAE,OAAO,QAAQ,MAAM,IAAI,CAAC;AAAA,MAC9D,GAAI,QAAQ,UAAU,SAAY,EAAE,OAAO,QAAQ,MAAM,IAAI,CAAC;AAAA,IAChE,CAAC;AAAA,EACH;AAMA,QAAM,aACJ,QAAQ,WAAW,SACf,QAAQ,QAAQ,MAAM,IACtB,QAAQ,WAAW,OAAO,UAAU,MAAM;AAChD,QAAM,SAAS,MAAM,WAAW,UAAU;AAI1C,QAAM,UAAU,MAAM,mBAAmB,OAAO,GAAG,IAAI,CAAC,MAAM,QAAQ,CAAC,CAAC;AACxE,uBAAqB,QAAQ,QAAQ,UAAU;AAE/C,QAAM,WAAyB;AAAA,IAC7B,cAAc,QAAQ,QAAQ,OAAO,MAAM;AAAA,IAC3C,OAAO,OAAO,MAAM;AAAA,EACtB;AACA,QAAM,OAAO,OAAO,UAAU;AAK9B,QAAM,QACJ,QAAQ,UAAU,OACd,WAAW,KAAK,cAAc,SAAS,YAAY,IACnD,SAAS;AAEf,QAAM,QAAQ,MAAM,SAAS;AAAA,IAC3B,GAAG;AAAA,IACH,QAAQ;AAAA,IACR,aAAa;AAAA,IACb,WAAW;AAAA,IACX,OAAO;AAAA,IACP,MAAM;AAAA;AAAA;AAAA,IAGN,QAAQ;AAAA,IACR,QAAQ;AAAA,EACV,CAAC;AAED,QAAM,MAAoB,CAAC;AAC3B,aAAW,WAAW,MAAM,UAAU;AACpC,QAAI,YAAY,QAAS,KAAI,KAAK,GAAG,QAAQ,OAAO,WAAW;AAAA,EACjE;AACA,QAAM,EAAE,QAAQ,WAAW,UAAU,IAAI,WAAW,MAAM,UAAU,MAAM;AAC1E,oBAAkB,WAAW,OAAO,UAAU;AAC9C,oBAAkB,WAAW,UAAU;AAMvC,QAAM,SAAS;AAAA,IACb,EAAE,QAAQ,IAAI;AAAA,IACd,QAAQ,UAAU,OAAO,WAAW;AAAA,EACtC;AAEA,QAAM,QACJ,QAAQ,UAAU,OACd,WAAW,EAAE,QAAQ,IAAI,GAAG,UAAU,IAAI,IAC1C;AAEN,QAAM,QAAQ,WAAW,OAAO,QAAQ;AAAA,IACtC,WAAW,QAAQ,gBAAgB,OAAO,UAAU;AAAA,IACpD,WAAW,QAAQ,gBAAgB,OAAO,UAAU;AAAA,IACpD,QAAQ,QAAQ,aAAa,OAAO,UAAU;AAAA,EAChD,CAAC;AAED,QAAM,WAAW,CAAC,GAAG,MAAM,OAAO,QAAQ;AAC1C,MAAI,OAAO,OAAO,eAAe,GAAG;AAClC,aAAS;AAAA,MACP,GAAG,OAAO,OAAO,YAAY;AAAA,IAC/B;AAAA,EACF;AASA,QAAM,kBAAkB,OAAO,OAAO,WAAW;AACjD,MAAI,iBAAiB;AACnB,aAAS;AAAA,MACP,uCAAuC,OAAO,OAAO,MAAM;AAAA,IAE7D;AAAA,EACF;AAEA,QAAM,SAA4B;AAAA,IAChC;AAAA,IACA;AAAA,IACA,SAAS;AAAA,IACT,QAAQ,OAAO;AAAA,IACf,eAAe,OAAO;AAAA,IACtB,UAAU,OAAO;AAAA,IACjB,GAAI,UAAU,SAAY,EAAE,MAAM,IAAI,CAAC;AAAA,IACvC;AAAA,IACA,OAAO,MAAM;AAAA,IACb;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAMA,UACE,MAAM,KAAK,CAAC,MAAM,EAAE,QAAQ,KAC5B,MAAM,OAAO,QAAQ,gBAAgB,KACrC,MAAM,OAAO,WAAW,UACxB,kBACI,IACA;AAAA,IACN,OAAO,MAAM,OAAO;AAAA,IACpB,YAAY,KAAK,IAAI,IAAI;AAAA,EAC3B;AAEA,QAAM,WAAW,kBAAkB,QAAQ,QAAQ,UAAU,UAAU;AAAA,IACrE,OAAO,QAAQ,UAAU;AAAA,EAC3B,CAAC;AACD,MAAI,QAAQ,OAAQ,OAAM,UAAU,QAAQ,QAAQ,UAAU,OAAO;AACrE,SAAO,EAAE,QAAQ,SAAS;AAC5B;AAOA,SAAS,qBACP,QACA,QACA,YACM;AACN,QAAM,QAAQ,IAAI,IAAI,MAAM;AAC5B,QAAM,UAAU,CAAC,GAAG,IAAI,IAAI,OAAO,OAAO,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,SAAS,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;AAC9F,MAAI,QAAQ,WAAW,EAAG;AAC1B,QAAM,IAAI;AAAA,IACR,GAAG,UAAU,+CAA+C,QAAQ,KAAK,IAAI,CAAC,oBAC1D,OAAO,IAAI,CAAC,MAAM,SAAS,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC;AAAA,EAC/D;AACF;AASA,SAAS,kBACP,WACA,OACA,YACM;AACN,MAAI,UAAU,WAAW,EAAG;AAC5B,QAAM,QAAQ,UAAU,IAAI,CAAC,UAAU;AACrC,UAAM,UAAU,MAAM,SAAS,KAAK,CAAC,MAAM,EAAE,SAAS,MAAM,SAAS;AACrE,UAAM,UAAU,SAAS,QAAQ,eAAe,CAAC;AACjD,UAAM,eAAe,QAAQ;AAAA,MAC3B,CAAC,MAAM,EAAE,iBAAiB,MAAM;AAAA,IAClC;AACA,UAAM,OACJ,aAAa,SAAS,IAClB,wBAAwB,aAAa,IAAI,CAAC,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,IAAI,CAAC,SAAS,MAAM,QAAQ,MACpG,0CAA0C,MAAM,QAAQ,yBACtD,CAAC,GAAG,IAAI,IAAI,QAAQ,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,IAAI,KACjE,SACF;AACN,WAAO,KAAK,MAAM,QAAQ,WAAM,MAAM,IAAI,OAAO,SAAS,MAAM,SAAS,CAAC,KAAK,IAAI;AAAA,EACrF,CAAC;AACD,QAAM,IAAI;AAAA,IACR,GAAG,UAAU;AAAA,EAAuC,MAAM,KAAK,IAAI,CAAC;AAAA,EACtE;AACF;AAYA,SAAS,kBACP,WACA,YACM;AACN,MAAI,UAAU,WAAW,EAAG;AAC5B,QAAM,QAAQ,UAAU,IAAI,CAAC,EAAE,OAAO,WAAW,MAAM;AACrD,UAAM,QAAQ,CAAC,GAAG,IAAI,IAAI,WAAW,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC,EAAE,KAAK;AACvE,WAAO,KAAK,MAAM,QAAQ,WAAM,MAAM,IAAI,OAAO,SAAS,MAAM,SAAS,CAAC,YAAY,MAAM,KAAK,OAAO,CAAC;AAAA,EAC3G,CAAC;AACD,QAAM,IAAI;AAAA,IACR,GAAG,UAAU;AAAA,EAAkD,MAAM,KAAK,IAAI,CAAC;AAAA,EACjF;AACF;AAEA,SAAS,WACP,QACA,QACmB;AACnB,QAAM,QAA2B,CAAC;AAClC,QAAM,MAAM,CACV,MACA,OACA,WACS;AACT,QAAI,UAAU,OAAW;AACzB,UAAM,KAAK,EAAE,MAAM,OAAO,QAAQ,UAAU,SAAS,MAAM,CAAC;AAAA,EAC9D;AACA,MAAI,aAAa,OAAO,WAAW,OAAO,SAAS;AACnD,MAAI,aAAa,OAAO,WAAW,OAAO,SAAS;AACnD,MAAI,UAAU,OAAO,QAAQ,OAAO,YAAY;AAChD,SAAO;AACT;AASA,SAAS,WACP,OACA,UACA,MACa;AACb,QAAM,OAAO,CACX,MACA,OACA,aACe;AAAA,IACf;AAAA,IACA;AAAA,IACA;AAAA,IACA,QAAQ,MAAM,OAAO,OAAO,EAAE;AAAA,EAChC;AAEA,QAAM,QAAqB;AAAA,IACzB,KAAK,YAAY,SAAS,cAAc,QAAQ;AAAA,EAClD;AACA,aAAW,QAAQ,CAAC,GAAG,KAAK,YAAY,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,GAAG;AAC/D,UAAM;AAAA,MACJ,KAAK,gBAAgB,MAAM,EAAE,cAAc,MAAM,OAAO,SAAS,MAAM,CAAC;AAAA,IAC1E;AAAA,EACF;AACA,aAAW,YAAY,CAAC,GAAG,KAAK,QAAQ,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,GAAG;AAC/D,UAAM;AAAA,MACJ,KAAK,kBAAkB,UAAU;AAAA,QAC/B,cAAc,SAAS;AAAA,QACvB,OAAO,EAAE,UAAU,UAAU,SAAS,MAAM,SAAS;AAAA,MACvD,CAAC;AAAA,IACH;AAAA,EACF;AACA,aAAW,YAAY,CAAC,GAAG,KAAK,QAAQ,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,GAAG;AAC/D,UAAM;AAAA,MACJ,KAAK,kBAAkB,UAAU;AAAA,QAC/B,cAAc,SAAS;AAAA,QACvB,OAAO,EAAE,UAAU,SAAS,MAAM,UAAU,SAAS;AAAA,MACvD,CAAC;AAAA,IACH;AAAA,EACF;AACA,SAAO;AACT;","names":[]}

package/dist/chunk-WQWKYAAK.js

@@ -0,0 +1,86 @@
+#!/usr/bin/env node
+import {
+  DEFAULT_KEEP_ALIVE_MS
+} from "./chunk-QCHVR4ZM.js";
+
+// src/tracevals/rules/host.ts
+import {
+  leaseModelHost,
+  ModelHostBusyError,
+  modelHostStatus,
+  releaseModelHost
+} from "@hawkeyexl/inference";
+var QUEUE_WAIT_MS = 12e4;
+var library = {
+  lease: leaseModelHost,
+  status: modelHostStatus,
+  release: releaseModelHost
+};
+function hostSetting(inLoop, sessionId, keepAliveMs) {
+  if (!inLoop || sessionId === void 0) return { host: "connect" };
+  return {
+    host: "spawn",
+    session: sessionId,
+    keepAlive: keepAliveMs ?? DEFAULT_KEEP_ALIVE_MS,
+    hostWaitMs: QUEUE_WAIT_MS
+  };
+}
+async function warmModel(lease, api = library) {
+  try {
+    const held = await api.lease({
+      model: lease.model,
+      session: lease.sessionId,
+      keepAlive: lease.keepAliveMs,
+      spawn: true,
+      hostWaitMs: QUEUE_WAIT_MS,
+      ...lease.modelsDirectory !== void 0 ? { modelsDirectory: lease.modelsDirectory } : {}
+    });
+    return held !== null;
+  } catch {
+    return false;
+  }
+}
+function releaseHost(target, api = library) {
+  return api.release("all" in target ? { all: true } : { session: target.sessionId });
+}
+async function hostStatus(api = library) {
+  const status = await api.status();
+  if (status === null) return null;
+  return {
+    pid: status.pid,
+    models: status.models.map(({ model, sessions, idleMs, queued: queued2 }) => ({
+      model,
+      sessions,
+      idleMs,
+      queued: queued2
+    }))
+  };
+}
+async function hostHolds(model, api = library) {
+  try {
+    return (await hostStatus(api))?.models.some((m) => m.model === model) ?? false;
+  } catch {
+    return false;
+  }
+}
+async function queued(_model, run) {
+  try {
+    return { busy: false, value: await run() };
+  } catch (err) {
+    if (err instanceof ModelHostBusyError || err instanceof Error && err.name === "ModelHostBusyError") {
+      return { busy: true };
+    }
+    throw err;
+  }
+}
+
+export {
+  QUEUE_WAIT_MS,
+  hostSetting,
+  warmModel,
+  releaseHost,
+  hostStatus,
+  hostHolds,
+  queued
+};
+//# sourceMappingURL=chunk-WQWKYAAK.js.map

package/dist/chunk-WQWKYAAK.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/tracevals/rules/host.ts"],"sourcesContent":["/**\n * The model host seam (proposal 0079, \"Concurrency and the model host\").\n *\n * The host is one background process per machine and user that keeps a local\n * model loaded between hooks, with a queue, leases and keep-alive. The\n * inference library owns it. Every host-dependent behavior goes through this\n * module and no other, and the library's three calls are injectable so a test\n * never starts a real host.\n */\nimport {\n  leaseModelHost,\n  ModelHostBusyError,\n  modelHostStatus,\n  releaseModelHost,\n  type LlamaCppProviderOptions,\n} from \"@hawkeyexl/inference\";\nimport { DEFAULT_KEEP_ALIVE_MS } from \"../../shared/providers.js\";\n\n/** A request that waits this long in the host's queue is withdrawn. */\nexport const QUEUE_WAIT_MS = 120_000;\n\n/** The library's host calls, as this module uses them. */\nexport interface HostApi {\n  lease: typeof leaseModelHost;\n  status: typeof modelHostStatus;\n  release: typeof releaseModelHost;\n}\n\nconst library: HostApi = {\n  lease: leaseModelHost,\n  status: modelHostStatus,\n  release: releaseModelHost,\n};\n\nexport interface ReleaseResult {\n  /** Sessions whose lease was returned. */\n  released: string[];\n  /** Models the host unloaded. */\n  unloaded: string[];\n  hostStopped: boolean;\n}\n\n/** What `manni status` shows of a running host. */\nexport interface HostStatus {\n  pid: number;\n  models: { model: string; sessions: number; idleMs: number; queued: number }[];\n}\n\n/** The host fields of a `llama-cpp` provider spec. */\nexport type HostSetting = Pick<\n  LlamaCppProviderOptions,\n  \"host\" | \"session\" | \"keepAlive\" | \"hostWaitMs\"\n>;\n\n/**\n * How a provider reaches the host. Inside a hook it starts one if none runs,\n * under the session's lease. By hand it uses a running host and never starts\n * one.\n */\nexport function hostSetting(\n  inLoop: boolean,\n  sessionId: string | undefined,\n  keepAliveMs: number | undefined,\n): HostSetting {\n  if (!inLoop || sessionId === undefined) return { host: \"connect\" };\n  return {\n    host: \"spawn\",\n    session: sessionId,\n    keepAlive: keepAliveMs ?? DEFAULT_KEEP_ALIVE_MS,\n    hostWaitMs: QUEUE_WAIT_MS,\n  };\n}\n\n/**\n * `prepare` under SessionStart: take the session's lease and load the model,\n * so the first Stop finds it warm. True when the model is now loaded. A host\n * that cannot start or load leaves the model cold, which the first Stop\n * handles, so it never fails `prepare`.\n */\nexport async function warmModel(\n  lease: {\n    sessionId: string;\n    model: string;\n    keepAliveMs: number;\n    modelsDirectory?: string;\n  },\n  api: HostApi = library,\n): Promise<boolean> {\n  try {\n    const held = await api.lease({\n      model: lease.model,\n      session: lease.sessionId,\n      keepAlive: lease.keepAliveMs,\n      spawn: true,\n      hostWaitMs: QUEUE_WAIT_MS,\n      ...(lease.modelsDirectory !== undefined ? { modelsDirectory: lease.modelsDirectory } : {}),\n    });\n    return held !== null;\n  } catch {\n    return false;\n  }\n}\n\n/** `release`: return one session's lease, or every lease with `all`. */\nexport function releaseHost(\n  target: { sessionId: string } | { all: true },\n  api: HostApi = library,\n): Promise<ReleaseResult> {\n  return api.release(\"all\" in target ? { all: true } : { session: target.sessionId });\n}\n\n/** The running host, or null when there is none. */\nexport async function hostStatus(api: HostApi = library): Promise<HostStatus | null> {\n  const status = await api.status();\n  if (status === null) return null;\n  return {\n    pid: status.pid,\n    models: status.models.map(({ model, sessions, idleMs, queued }) => ({\n      model,\n      sessions,\n      idleMs,\n      queued,\n    })),\n  };\n}\n\n/**\n * Whether the host already holds `model` loaded. A host that cannot be\n * reached, or that errors, holds nothing as far as a caller can tell.\n */\nexport async function hostHolds(model: string, api: HostApi = library): Promise<boolean> {\n  try {\n    return (await hostStatus(api))?.models.some((m) => m.model === model) ?? false;\n  } catch {\n    return false;\n  }\n}\n\n/**\n * Run a request for `model`. The host's queue lives inside the provider, so\n * `busy` is its `ModelHostBusyError`: the call waited `QUEUE_WAIT_MS` and\n * nothing ran.\n */\nexport async function queued<T>(\n  _model: string,\n  run: () => Promise<T>,\n): Promise<{ busy: false; value: T } | { busy: true }> {\n  try {\n    return { busy: false, value: await run() };\n  } catch (err) {\n    if (err instanceof ModelHostBusyError || (err instanceof Error && err.name === \"ModelHostBusyError\")) {\n      return { busy: true };\n    }\n    throw err;\n  }\n}\n"],"mappings":";;;;;;AASA;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OAEK;AAIA,IAAM,gBAAgB;AAS7B,IAAM,UAAmB;AAAA,EACvB,OAAO;AAAA,EACP,QAAQ;AAAA,EACR,SAAS;AACX;AA2BO,SAAS,YACd,QACA,WACA,aACa;AACb,MAAI,CAAC,UAAU,cAAc,OAAW,QAAO,EAAE,MAAM,UAAU;AACjE,SAAO;AAAA,IACL,MAAM;AAAA,IACN,SAAS;AAAA,IACT,WAAW,eAAe;AAAA,IAC1B,YAAY;AAAA,EACd;AACF;AAQA,eAAsB,UACpB,OAMA,MAAe,SACG;AAClB,MAAI;AACF,UAAM,OAAO,MAAM,IAAI,MAAM;AAAA,MAC3B,OAAO,MAAM;AAAA,MACb,SAAS,MAAM;AAAA,MACf,WAAW,MAAM;AAAA,MACjB,OAAO;AAAA,MACP,YAAY;AAAA,MACZ,GAAI,MAAM,oBAAoB,SAAY,EAAE,iBAAiB,MAAM,gBAAgB,IAAI,CAAC;AAAA,IAC1F,CAAC;AACD,WAAO,SAAS;AAAA,EAClB,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAGO,SAAS,YACd,QACA,MAAe,SACS;AACxB,SAAO,IAAI,QAAQ,SAAS,SAAS,EAAE,KAAK,KAAK,IAAI,EAAE,SAAS,OAAO,UAAU,CAAC;AACpF;AAGA,eAAsB,WAAW,MAAe,SAAqC;AACnF,QAAM,SAAS,MAAM,IAAI,OAAO;AAChC,MAAI,WAAW,KAAM,QAAO;AAC5B,SAAO;AAAA,IACL,KAAK,OAAO;AAAA,IACZ,QAAQ,OAAO,OAAO,IAAI,CAAC,EAAE,OAAO,UAAU,QAAQ,QAAAA,QAAO,OAAO;AAAA,MAClE;AAAA,MACA;AAAA,MACA;AAAA,MACA,QAAAA;AAAA,IACF,EAAE;AAAA,EACJ;AACF;AAMA,eAAsB,UAAU,OAAe,MAAe,SAA2B;AACvF,MAAI;AACF,YAAQ,MAAM,WAAW,GAAG,IAAI,OAAO,KAAK,CAAC,MAAM,EAAE,UAAU,KAAK,KAAK;AAAA,EAC3E,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAOA,eAAsB,OACpB,QACA,KACqD;AACrD,MAAI;AACF,WAAO,EAAE,MAAM,OAAO,OAAO,MAAM,IAAI,EAAE;AAAA,EAC3C,SAAS,KAAK;AACZ,QAAI,eAAe,sBAAuB,eAAe,SAAS,IAAI,SAAS,sBAAuB;AACpG,aAAO,EAAE,MAAM,KAAK;AAAA,IACtB;AACA,UAAM;AAAA,EACR;AACF;","names":["queued"]}

package/dist/{chunk-LJ2RDH3F.js → chunk-XG6PVB5S.js}

@@ -1,26 +1,5 @@
 #!/usr/bin/env node
 
-// src/shared/duration.ts
-var UNITS = {
-  m: 6e4,
-  h: 60 * 6e4,
-  d: 24 * 60 * 6e4,
-  w: 7 * 24 * 60 * 6e4
-};
-function durationMessage(value) {
-  return `--newer-than must be a duration such as 30m, 24h, 7d or 2w, got "${value}"`;
-}
-function parseDuration(text, makeError) {
-  const match = /^(\d+(?:\.\d+)?)([mhdw])$/.exec(text.trim());
-  const count = match?.[1];
-  const unit = match?.[2];
-  const ms = unit === void 0 ? void 0 : UNITS[unit];
-  if (count === void 0 || ms === void 0) {
-    throw makeError(durationMessage(text));
-  }
-  return Number(count) * ms;
-}
-
 // src/shared/exec.ts
 import { realExec } from "@hawkeyexl/inference";
 function outputTail(result, maxChars = 400) {
@@ -41,9 +20,8 @@ function isTurnBudgetSkip(reason) {
 export {
   outputTail,
   realExec,
-  parseDuration,
   TURN_BUDGET_SKIP,
   turnBudgetSkipReason,
   isTurnBudgetSkip
 };
-//# sourceMappingURL=chunk-LJ2RDH3F.js.map
+//# sourceMappingURL=chunk-XG6PVB5S.js.map

package/dist/chunk-XG6PVB5S.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/shared/exec.ts","../src/docevals/judge/budget.ts"],"sourcesContent":["/**\n * Process execution for every grader in the family that runs a program. The\n * implementation lives in `@hawkeyexl/inference`: the judge's subprocess\n * provider and the command graders were running copies of the same\n * cross-spawn wrapper, and the Windows-specific parts (npm `.cmd` shim\n * resolution without `shell: true`, stdin piping past the ~32K command-line\n * limit, and StringDecoder-backed output so multi-byte UTF-8 survives chunk\n * boundaries) are exactly the parts worth having in one place.\n *\n * Shared rather than per domain: `manni docevals` and `manni tracevals` both\n * run `command` evals, and both reach the OS through this one wrapper.\n */\nimport type { ExecResult } from \"@hawkeyexl/inference\";\n\nexport { realExec } from \"@hawkeyexl/inference\";\n\n/** Truncate command output for finding messages. */\nexport function outputTail(result: ExecResult, maxChars = 400): string {\n  const text = (result.stderr.trim() || result.stdout.trim()).trim();\n  if (text.length <= maxChars) return text;\n  return `…${text.slice(-maxChars)}`;\n}\n\n/**\n * How a finished process is described in a finding: \"exited 2\", or the truth\n * when there was no exit code.\n *\n * `code` is null when the process was killed by a signal rather than exiting.\n * \"exited null\" describes nothing a reader can act on, and it is the case a\n * timeout produces — so it is the one worth naming.\n */\nexport function exited(code: number | null): string {\n  return code === null ? \"was killed before exiting\" : `exited ${code}`;\n}\n","/**\n * The turn-budget skip marker (ADR 01019), in one place.\n *\n * `judge.ts` produces the skip reason; `core/engine.ts` and `commands/\n * calibrate.ts` both have to tell a budget truncation apart from every other\n * reason an eval went unjudged, because that distinction is what stops a\n * partial run from reporting as a complete one. They did it by matching the\n * bare substring `\"turn budget\"` against a message written in a third file,\n * so a reword in the producer would have left both detectors matching nothing\n * — silently, since the tests that exercise truncation supply the skip reason\n * themselves rather than getting it from the judge.\n *\n * Kept in its own module rather than exported from `judge.ts`: the detectors\n * need the marker, not the judge, and `engine.ts` deliberately takes the judge\n * by injection.\n */\n\n/** The substring every budget-truncation skip reason carries. */\nexport const TURN_BUDGET_SKIP = \"turn budget\";\n\n/** The skip reason recorded for an eval the budget stopped before dispatch. */\nexport function turnBudgetSkipReason(maxTurns: number): string {\n  return `judge ${TURN_BUDGET_SKIP} exhausted (${maxTurns})`;\n}\n\n/**\n * Whether a skip reason is the budget's doing rather than the judge's.\n *\n * A type predicate, not a plain boolean: a budget skip always carries a\n * reason, and callers interpolate it into the message they report.\n */\nexport function isTurnBudgetSkip(reason: string | undefined): reason is string {\n  return reason !== undefined && reason.includes(TURN_BUDGET_SKIP);\n}\n"],"mappings":";;;AAcA,SAAS,gBAAgB;AAGlB,SAAS,WAAW,QAAoB,WAAW,KAAa;AACrE,QAAM,QAAQ,OAAO,OAAO,KAAK,KAAK,OAAO,OAAO,KAAK,GAAG,KAAK;AACjE,MAAI,KAAK,UAAU,SAAU,QAAO;AACpC,SAAO,SAAI,KAAK,MAAM,CAAC,QAAQ,CAAC;AAClC;;;ACHO,IAAM,mBAAmB;AAGzB,SAAS,qBAAqB,UAA0B;AAC7D,SAAO,SAAS,gBAAgB,eAAe,QAAQ;AACzD;AAQO,SAAS,iBAAiB,QAA8C;AAC7E,SAAO,WAAW,UAAa,OAAO,SAAS,gBAAgB;AACjE;","names":[]}