@iowarp/clio-coder 0.3.9 → 0.4.0

package/dist/{eval-IJ5VEZDJ.js → eval-TFBYQH4H.js}

@@ -1,72 +1,79 @@
 import { createRequire as __clioCreateRequire } from "node:module"; const require = __clioCreateRequire(import.meta.url);
 import {
-  discoverAgentRecipes
-} from "./chunk-HCBCAYZU.js";
-import "./chunk-AD7Y7STJ.js";
-import "./chunk-AMKHQW3C.js";
+  EVAL_SUITE_V2_VERSION
+} from "./chunk-KMVISBZR.js";
 import {
   loadFragments,
   renderCodewikiDigest
-} from "./chunk-47CMYGET.js";
+} from "./chunk-VAZSBTKF.js";
 import {
-  EVAL_BEHAVIOR_METRIC_DEFINITIONS_V1,
-  EVAL_EXECUTION_ENVELOPE_SCHEMA_V1,
-  EVAL_TRACKED_METRIC_NAMES,
-  EVAL_VERDICT_SCHEMA_V1,
   EvalTaskFileError,
   TRUST_STATUS_AXES,
   adaptRunReceiptTrustStatus,
   assertComparableTrackedMetricSources,
-  assertEvalBehaviorReferencesVerdictV1,
-  buildEvalBehaviorMetricsV1,
-  createEvalId,
   evalClioProvenance,
   evalEnvironmentProvenance,
-  evalHarnessMetricsFromReceipt,
   evalServingConfiguration,
   evalServingObservationFrom,
   formatTrustSummary,
   inspectRunReceiptTrustStatus,
+  loadEvalTaskFile,
+  summarizeTrustStatus,
+  verifyReceiptIntegrity
+} from "./chunk-B7OBL7PK.js";
+import {
+  listSessionLedgerRefs,
+  parseSessionEntries
+} from "./chunk-QREDIESB.js";
+import "./chunk-3DPEIQKN.js";
+import {
+  EVAL_BEHAVIOR_METRIC_DEFINITIONS_V1,
+  EVAL_EXECUTION_ENVELOPE_SCHEMA_V1,
+  EVAL_TRACKED_METRIC_NAMES,
+  EVAL_VERDICT_SCHEMA_V1,
+  assertEvalBehaviorReferencesVerdictV1,
+  buildEvalBehaviorMetricsV1,
+  createEvalId,
+  evalHarnessMetricsFromReceipt,
+  evalServingConfigurationOf,
   judgeEvalBehaviorV1,
   loadEvalArtifactV4,
-  loadEvalTaskFile,
   parseEvalBehaviorScenarioV1,
   parseEvalExecutionMatrixDimensionsV1,
   parseEvalVerdictEnvelopeV1,
   renderEvalServingConfiguration,
   sameEvalServingConfiguration,
-  summarizeTrustStatus,
-  verifyReceiptIntegrity,
   writeEvalArtifactV4
-} from "./chunk-MLOK6ZOS.js";
-import {
-  listSessionLedgerRefs,
-  parseSessionEntries
-} from "./chunk-LXPJXFM5.js";
-import "./chunk-3DPEIQKN.js";
+} from "./chunk-NHLBIGRH.js";
 import "./chunk-W6GROXXM.js";
-import "./chunk-RKKLTLYB.js";
-import "./chunk-5QKCQQ3E.js";
 import {
   shellQuote
 } from "./chunk-TXOTCRLG.js";
 import "./chunk-HVDIIIQW.js";
-import "./chunk-VPTUJU4P.js";
+import {
+  discoverAgentRecipes
+} from "./chunk-Z4TXYIEG.js";
+import "./chunk-VYMXRQI6.js";
+import "./chunk-FEFIFZTL.js";
+import "./chunk-RKKLTLYB.js";
+import "./chunk-SUW5DORT.js";
+import "./chunk-SJ5ZKQ4S.js";
 import "./chunk-2JDWVJND.js";
-import "./chunk-H7IXIC72.js";
 import {
   createSafetyPolicyEngine
-} from "./chunk-N3PBVRTZ.js";
-import "./chunk-A2NJGIB3.js";
+} from "./chunk-32KWKNSF.js";
+import "./chunk-H7IXIC72.js";
+import "./chunk-5LXZXPKX.js";
+import "./chunk-RAPCMZL4.js";
+import "./chunk-HHV2GANA.js";
 import {
   agentSpecFingerprint,
   normalizeAgentSpec
-} from "./chunk-S4COXYBG.js";
-import "./chunk-MV3K5QF2.js";
-import "./chunk-RAPCMZL4.js";
+} from "./chunk-DYJP44XW.js";
+import "./chunk-GCSMB2KY.js";
 import "./chunk-UL3WSD3F.js";
 import "./chunk-ECH6PKUQ.js";
-import "./chunk-CGKSTWHD.js";
+import "./chunk-K6BSR66V.js";
 import {
   readCodewiki,
   structuralCodewikiHash
@@ -79,29 +86,30 @@ import {
 } from "./chunk-33YXPOE3.js";
 import "./chunk-7CR24IG7.js";
 import "./chunk-XPLRXC72.js";
-import "./chunk-IFBNV6H6.js";
+import "./chunk-2ANTL7MR.js";
 import {
   printError
-} from "./chunk-XK56QHLX.js";
+} from "./chunk-VPKWYKEY.js";
 import "./chunk-5TSRNF4G.js";
 import "./chunk-CFGTUFWB.js";
 import {
   extractReasoningTokens
-} from "./chunk-AEYBF3TB.js";
-import "./chunk-IHXBNWMM.js";
-import "./chunk-B5CSFE7B.js";
-import "./chunk-PNY46YEY.js";
-import "./chunk-FQ4SKYE4.js";
-import "./chunk-RKRLDWD3.js";
+} from "./chunk-UM7N4G5A.js";
+import "./chunk-HLW2MRKE.js";
+import "./chunk-76ONBSIA.js";
 import {
   InvalidIdError
-} from "./chunk-KV2AOLDF.js";
+} from "./chunk-R346GLFC.js";
+import "./chunk-IHXBNWMM.js";
+import "./chunk-VFA6GDY5.js";
+import "./chunk-BBVJUZHB.js";
+import "./chunk-FQ4SKYE4.js";
 import "./chunk-6EJMN2Y3.js";
 import "./chunk-IWHMRKLL.js";
-import "./chunk-XDOQXGFO.js";
+import "./chunk-GXNLGKAB.js";
 import "./chunk-LL4KHSZI.js";
 import "./chunk-4ZG3XFUR.js";
-import "./chunk-EQ63NRB7.js";
+import "./chunk-BBVYXMFO.js";
 import "./chunk-SST6Z5JA.js";
 import "./chunk-IKCO5N3L.js";
 import "./chunk-3I7MS7N2.js";
@@ -111,7 +119,7 @@ import {
 import {
   clioDataDir,
   clioStateDir
-} from "./chunk-BNAZZHFG.js";
+} from "./chunk-BYP5D4HI.js";
 import "./chunk-WEPFGWHJ.js";
 import "./chunk-YXLYO42X.js";
 import {
@@ -453,8 +461,8 @@ var EvalServingConfigurationDriftError = class extends Error {
 function compareEvalArtifactsV4(baseline, candidate, options = {}) {
   const baselineTokens = baseline.summary.tokens;
   const candidateTokens = candidate.summary.tokens;
-  const baselineServing = servingConfigurationOf(baseline);
-  const candidateServing = servingConfigurationOf(candidate);
+  const baselineServing = evalServingConfigurationOf(baseline);
+  const candidateServing = evalServingConfigurationOf(candidate);
   const configDrift = !sameEvalServingConfiguration(baselineServing, candidateServing);
   if (configDrift && options.allowConfigDrift !== true) {
     throw new EvalServingConfigurationDriftError(baselineServing, candidateServing);
@@ -654,17 +662,6 @@ function metricComparison(scenarioId, metric, baseline, candidate) {
     varianceChange: classifyChange(baseline.variance ?? null, candidate.variance ?? null, "lower")
   };
 }
-function servingConfigurationOf(artifact) {
-  return artifact.servingConfiguration ?? {
-    targetId: artifact.matrix.target,
-    runtimeId: null,
-    modelId: artifact.matrix.model,
-    serverBuild: null,
-    total_slots: null,
-    thinkingLevel: artifact.matrix.thinking,
-    compiledPromptHash: null
-  };
-}
 function normalizeMetricFilter(metric) {
   if (metric === void 0) return void 0;
   const trimmed = metric.trim();
@@ -1052,12 +1049,6 @@ import { dirname, resolve } from "node:path";
 
 // src/domains/eval/schema/validate.ts
 init_esm_shims();
-
-// src/domains/eval/schema/suite.ts
-init_esm_shims();
-var EVAL_SUITE_V2_VERSION = 2;
-
-// src/domains/eval/schema/validate.ts
 var RUNNER_KINDS = /* @__PURE__ */ new Set(["clio-run", "context-index", "context-init", "external-command"]);
 var WORKSPACE_KINDS = /* @__PURE__ */ new Set(["local", "git", "temp-copy"]);
 var OPS = /* @__PURE__ */ new Set(["lt", "lte", "gt", "gte", "eq", "neq"]);
@@ -4167,6 +4158,13 @@ Commands:
   clio-coder eval report <evalId> --format text|json|md|swe-jsonl|junit
 	  clio-coder eval compare <baselineEvalId> <candidateEvalId> [--metric <name>] [--format text|json|md|junit] [--allow-config-drift]
   clio-coder eval gate <candidateEvalId> --baseline <baselineEvalId> [--thresholds <file>]
+  clio-coder eval inventory --json
+
+inventory is the fixed machine-readable read a GUI host may run. Unlike report
+and compare it names no eval id, so the process it starts cannot be steered to a
+different report or a wider window. It carries each stored report's identity,
+provenance, serving facts, accounting, and per-scenario outcomes, and none of
+the runner attachments a report holds.
 `;
 function parseEvalArgs(args) {
   const parsed = {
@@ -4308,6 +4306,10 @@ function parseEvalArgs(args) {
   return parsed;
 }
 async function runEvalCommand(args) {
+  if (args[0] === "inventory") {
+    const { runEvalInventory } = await import("./eval-inventory-SXH7PDKX.js");
+    return runEvalInventory(args.slice(1));
+  }
   let parsed;
   try {
     parsed = parseEvalArgs(args);
@@ -4480,4 +4482,4 @@ function comparisonFormat(value) {
 export {
   runEvalCommand
 };
-//# sourceMappingURL=eval-IJ5VEZDJ.js.map
+//# sourceMappingURL=eval-TFBYQH4H.js.map

package/dist/eval-inventory-SXH7PDKX.js

@@ -0,0 +1,316 @@
+import { createRequire as __clioCreateRequire } from "node:module"; const require = __clioCreateRequire(import.meta.url);
+import {
+  CANONICAL_METRICS,
+  CANONICAL_METRIC_PREFIXES
+} from "./chunk-KMVISBZR.js";
+import {
+  EVAL_EXECUTION_MATRIX_DIMENSIONS_V1,
+  evalRoot,
+  evalServingConfigurationOf,
+  loadEvalArtifactV4,
+  sameEvalServingConfiguration
+} from "./chunk-NHLBIGRH.js";
+import "./chunk-R346GLFC.js";
+import {
+  clioDataDir
+} from "./chunk-BYP5D4HI.js";
+import "./chunk-WEPFGWHJ.js";
+import {
+  init_esm_shims
+} from "./chunk-3R73A4XB.js";
+
+// src/cli/eval-inventory.ts
+init_esm_shims();
+
+// src/domains/eval/inventory.ts
+init_esm_shims();
+import { readdir } from "node:fs/promises";
+var EVAL_ID_PATTERN = /^eval-(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})(\d{3})Z-([0-9a-f]{8})-([0-9a-f]{12})$/u;
+function idStamp(evalId) {
+  const match = EVAL_ID_PATTERN.exec(evalId);
+  if (match === null) return null;
+  const [, year, month, day, hour, minute, second, millisecond, suiteHashPrefix] = match;
+  const startedAt = `${year}-${month}-${day}T${hour}:${minute}:${second}.${millisecond}Z`;
+  const parsed = new Date(startedAt);
+  if (Number.isNaN(parsed.getTime()) || parsed.toISOString() !== startedAt) return null;
+  return { startedAt, suiteHashPrefix };
+}
+function byNewest(left, right) {
+  const leftStamp = idStamp(left)?.startedAt ?? "";
+  const rightStamp = idStamp(right)?.startedAt ?? "";
+  if (leftStamp !== rightStamp) return leftStamp < rightStamp ? 1 : -1;
+  return left < right ? 1 : -1;
+}
+async function listEvalReports(dataDir, limit) {
+  let entries;
+  try {
+    entries = await readdir(evalRoot(dataDir));
+  } catch (error) {
+    if (error.code === "ENOENT") {
+      return { available: false, stored: 0, reports: [], unreadable: 0 };
+    }
+    throw error;
+  }
+  const ids = entries.filter((entry) => entry.endsWith(".json")).map((entry) => entry.slice(0, -".json".length)).sort(byNewest);
+  const reports = [];
+  let unreadable = 0;
+  for (const evalId of ids.slice(0, limit)) {
+    let artifact;
+    try {
+      artifact = await loadEvalArtifactV4(dataDir, evalId);
+    } catch {
+      unreadable += 1;
+      continue;
+    }
+    if (artifact.evalId !== evalId) {
+      unreadable += 1;
+      continue;
+    }
+    const stamp = idStamp(evalId);
+    reports.push({
+      evalId,
+      startedAt: stamp !== null && artifact.suite.hash.startsWith(stamp.suiteHashPrefix) ? stamp.startedAt : null,
+      artifact
+    });
+  }
+  return { available: true, stored: ids.length, reports, unreadable };
+}
+
+// src/cli/eval-inventory.ts
+var EVAL_INVENTORY_MAX_REPORTS = 8;
+var EVAL_INVENTORY_MAX_SCENARIOS = 16;
+var FAILURE_CLASSES = [
+  "budget_exhausted",
+  "runner_failed",
+  "grader_failed",
+  "verifier_failed",
+  "forbidden_path",
+  "assertion_unresolved",
+  "assertion_failed",
+  "setup_failed",
+  "command_error"
+];
+var REPORTED_FAILURE_CLASSES = [...FAILURE_CLASSES, "other"];
+var BEHAVIOR_OUTCOMES = ["pass", "behavioral_failure", "unknown", "unmeasured", "infrastructure_failure"];
+var IDENTIFIER_PATTERN = /^[A-Za-z0-9._-]{1,128}$/u;
+var VERSION_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._+-]{0,63}$/u;
+var COMMIT_PATTERN = /^[0-9a-f]{7,64}$/u;
+var CANONICAL_METRIC_NAMES = new Set(CANONICAL_METRICS);
+function isCanonicalMetric(name) {
+  return CANONICAL_METRIC_NAMES.has(name) || CANONICAL_METRIC_PREFIXES.some((prefix) => name.startsWith(prefix));
+}
+function identifier(value) {
+  return value !== null && IDENTIFIER_PATTERN.test(value) ? value : null;
+}
+function tally(value) {
+  return typeof value === "number" && Number.isFinite(value) && value >= 0 ? Math.trunc(value) : 0;
+}
+function isRecord(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function projectScenarios(artifact) {
+  if (artifact.aggregates === void 0) return { scenarios: null, truncated: false, dropped: 0 };
+  const projected = [];
+  let dropped = 0;
+  for (const entry of artifact.aggregates) {
+    if (!isRecord(entry)) {
+      dropped += 1;
+      continue;
+    }
+    const scenarioId = identifier(typeof entry.scenarioId === "string" ? entry.scenarioId : null);
+    if (scenarioId === null) {
+      dropped += 1;
+      continue;
+    }
+    const trials = tally(entry.trials);
+    const passed = tally(entry.passed);
+    const failed = tally(entry.failed);
+    const unmeasured = tally(entry.unmeasured);
+    const machineryFailures = tally(entry.machineryFailures);
+    if (passed + failed + unmeasured !== trials || machineryFailures > failed) {
+      dropped += 1;
+      continue;
+    }
+    projected.push({
+      scenarioId,
+      trials,
+      passed,
+      failed,
+      unmeasured,
+      machineryFailures,
+      // Re-derived rather than read, because both are pure functions of the
+      // counts beside them and a stored disagreement is not a second opinion.
+      passAtK: trials > 0 && passed > 0 ? 1 : 0,
+      passPowK: trials > 0 && passed === trials ? 1 : 0
+    });
+  }
+  const scenarios = projected.slice(0, EVAL_INVENTORY_MAX_SCENARIOS);
+  return { scenarios, truncated: scenarios.length < projected.length, dropped };
+}
+function projectTokens(artifact) {
+  const tokens = artifact.summary.tokens;
+  if (!tokens.measured) {
+    return {
+      measured: false,
+      runs: tally(tokens.runs),
+      measuredRuns: 0,
+      input: null,
+      output: null,
+      total: null,
+      cacheRead: null,
+      cacheWrite: null
+    };
+  }
+  return {
+    measured: true,
+    runs: tally(tokens.runs),
+    measuredRuns: tally(tokens.measuredRuns),
+    input: tally(tokens.input),
+    output: tally(tokens.output),
+    total: tally(tokens.total),
+    cacheRead: tally(tokens.cacheRead),
+    cacheWrite: tally(tokens.cacheWrite)
+  };
+}
+function classifyFailure(failureClass) {
+  if (failureClass === null) return null;
+  return FAILURE_CLASSES.includes(failureClass) ? failureClass : "other";
+}
+function countResults(results) {
+  let withAssignment = 0;
+  let withTerminalReceipt = 0;
+  let withVerdict = 0;
+  let withBehavioral = 0;
+  let withExecutionEnvelope = 0;
+  let machineryFailures = 0;
+  let attachments = 0;
+  let canonicalMetrics = 0;
+  let otherMetrics = 0;
+  const failures = /* @__PURE__ */ new Map();
+  const outcomes = /* @__PURE__ */ new Map();
+  for (const result of results) {
+    if (result.assignmentId !== null) withAssignment += 1;
+    if (result.terminalReceiptDigest !== null) withTerminalReceipt += 1;
+    if (result.verdict !== void 0) withVerdict += 1;
+    if (result.verdict?.machinery === "infrastructure_failure") machineryFailures += 1;
+    if (result.behavioral !== void 0) withBehavioral += 1;
+    if (result.executionEnvelope !== void 0) withExecutionEnvelope += 1;
+    attachments += Object.keys(result.artifacts).length;
+    for (const name of Object.keys(result.metrics)) {
+      if (isCanonicalMetric(name)) canonicalMetrics += 1;
+      else otherMetrics += 1;
+    }
+    const failureClass = classifyFailure(result.failureClass);
+    if (failureClass !== null) failures.set(failureClass, (failures.get(failureClass) ?? 0) + 1);
+    const outcome = result.behavioral?.outcome;
+    if (outcome !== void 0 && BEHAVIOR_OUTCOMES.includes(outcome)) {
+      outcomes.set(outcome, (outcomes.get(outcome) ?? 0) + 1);
+    }
+  }
+  return {
+    counts: {
+      total: results.length,
+      withAssignment,
+      withTerminalReceipt,
+      withVerdict,
+      withBehavioral,
+      withExecutionEnvelope,
+      machineryFailures,
+      attachments,
+      canonicalMetrics,
+      otherMetrics
+    },
+    failureClasses: REPORTED_FAILURE_CLASSES.flatMap((failureClass) => {
+      const count = failures.get(failureClass) ?? 0;
+      return count === 0 ? [] : [{ failureClass, count }];
+    }),
+    behaviorOutcomes: BEHAVIOR_OUTCOMES.flatMap((outcome) => {
+      const count = outcomes.get(outcome) ?? 0;
+      return count === 0 ? [] : [{ outcome, count }];
+    })
+  };
+}
+function projectReport(evalId, startedAt, artifact, servingGroup) {
+  const serving = evalServingConfigurationOf(artifact);
+  const { counts, failureClasses, behaviorOutcomes } = countResults(artifact.results);
+  const { scenarios, truncated, dropped } = projectScenarios(artifact);
+  const version = artifact.clio.version;
+  const commit = artifact.clio.commit;
+  return {
+    evalId,
+    startedAt,
+    suiteId: identifier(artifact.suite.id) ?? "unavailable",
+    servingGroup,
+    clioVersion: VERSION_PATTERN.test(version) ? version : null,
+    clioCommit: commit !== null && COMMIT_PATTERN.test(commit) ? commit : null,
+    platform: identifier(artifact.environment.platform),
+    node: identifier(artifact.environment.node),
+    matrix: {
+      target: identifier(artifact.matrix.target),
+      model: identifier(artifact.matrix.model),
+      thinking: identifier(artifact.matrix.thinking),
+      dimensions: (artifact.matrix.dimensions ?? []).filter(
+        (dimension) => EVAL_EXECUTION_MATRIX_DIMENSIONS_V1.includes(dimension)
+      )
+    },
+    serving: {
+      observed: artifact.servingConfiguration !== void 0,
+      targetId: identifier(serving.targetId),
+      runtimeId: identifier(serving.runtimeId),
+      modelId: identifier(serving.modelId),
+      serverBuild: identifier(serving.serverBuild),
+      thinkingLevel: identifier(serving.thinkingLevel),
+      totalSlots: serving.total_slots === null ? null : tally(serving.total_slots),
+      compiledPromptPinned: serving.compiledPromptHash !== null
+    },
+    summary: {
+      runs: tally(artifact.summary.runs),
+      passed: tally(artifact.summary.passed),
+      failed: tally(artifact.summary.failed),
+      passRate: Number.isFinite(artifact.summary.passRate) ? Math.min(1, Math.max(0, artifact.summary.passRate)) : 0,
+      wallTimeMs: tally(artifact.summary.wallTimeMs),
+      tokens: projectTokens(artifact)
+    },
+    results: counts,
+    failureClasses,
+    behaviorOutcomes,
+    scenarios,
+    scenariosTruncated: truncated,
+    scenariosDropped: dropped
+  };
+}
+async function evalInventorySnapshot(now = Date.now, dataDir = clioDataDir()) {
+  const listing = await listEvalReports(dataDir, EVAL_INVENTORY_MAX_REPORTS);
+  const groups = [];
+  const reports = listing.reports.map((stored) => {
+    const serving = evalServingConfigurationOf(stored.artifact);
+    let index = groups.findIndex((candidate) => sameEvalServingConfiguration(candidate, serving));
+    if (index === -1) index = groups.push(serving) - 1;
+    return projectReport(stored.evalId, stored.startedAt, stored.artifact, index + 1);
+  });
+  return {
+    version: 1,
+    generatedAt: new Date(now()).toISOString(),
+    available: listing.available,
+    stored: listing.stored,
+    unreadable: listing.unreadable,
+    reports,
+    truncated: listing.stored > listing.reports.length + listing.unreadable
+  };
+}
+async function runEvalInventory(args) {
+  if (args.length !== 1 || args[0] !== "--json") {
+    process.stderr.write("clio-coder eval inventory: usage: clio-coder eval inventory --json\n");
+    return 2;
+  }
+  process.stdout.write(`${JSON.stringify(await evalInventorySnapshot(), null, 2)}
+`);
+  return 0;
+}
+export {
+  EVAL_INVENTORY_MAX_REPORTS,
+  EVAL_INVENTORY_MAX_SCENARIOS,
+  evalInventorySnapshot,
+  runEvalInventory
+};
+//# sourceMappingURL=eval-inventory-SXH7PDKX.js.map