@tangle-network/agent-eval 0.95.0 → 0.96.0

package/dist/contract/index.js

@@ -37,7 +37,7 @@ import {
 import "../chunk-VI2UW6B6.js";
 import {
   buildDefaultAnalystRegistry
-} from "../chunk-AQ5WQAIV.js";
+} from "../chunk-3NHEO6ZC.js";
 import "../chunk-S4SYLDFX.js";
 import "../chunk-5HRORJQY.js";
 import {
@@ -57,6 +57,9 @@ import {
   LLM_OUTPUT_TOKEN_ATTR_KEYS
 } from "../chunk-YBIGNSCZ.js";
 import "../chunk-PC4UYEBM.js";
+import {
+  parseRunRecordSafe
+} from "../chunk-LO6IOIJ2.js";
 import "../chunk-VSMTAMNK.js";
 import "../chunk-3BFEG2F6.js";
 import "../chunk-PZ5AY32C.js";
@@ -474,6 +477,145 @@ function requirePositiveInteger(value, field) {
   return value;
 }
 
+// src/contract/eval-reporting-suite.ts
+import { mkdir, writeFile } from "fs/promises";
+import { dirname, join as join2 } from "path";
+
+// src/contract/intake/run-record-dir.ts
+import { readdir, readFile, stat } from "fs/promises";
+import { join } from "path";
+var ANALYSIS_ARTIFACT = "analysis.json";
+function defaultInclude(fileName) {
+  if (fileName === ANALYSIS_ARTIFACT) return false;
+  return fileName.endsWith(".json") || fileName.endsWith(".jsonl");
+}
+async function fromRunRecordDir(path, options = {}) {
+  const onInvalid = options.onInvalid ?? "throw";
+  const include = options.include ?? defaultInclude;
+  const stats = await stat(path);
+  const filePaths = stats.isDirectory() ? await collectFiles(path, include, options.recursive ?? false) : [path];
+  const runs = [];
+  const rejected = [];
+  for (const file of filePaths) {
+    const raw = await parseRecordFile(file);
+    for (const { index, value } of raw) {
+      const parsed = parseRunRecordSafe(value);
+      if (parsed.ok) {
+        runs.push(parsed.value);
+        continue;
+      }
+      const rejection = { file, index, reason: parsed.error.message };
+      if (onInvalid === "throw") {
+        throw new Error(
+          `fromRunRecordDir: invalid RunRecord in '${file}' at index ${index}: ${parsed.error.message}`
+        );
+      }
+      rejected.push(rejection);
+    }
+  }
+  return { runs, rejected, files: filePaths };
+}
+async function parseRecordFile(file) {
+  const text = await readFile(file, "utf8");
+  const trimmed = text.trim();
+  if (trimmed.length === 0) return [];
+  if (trimmed.startsWith("[")) {
+    const parsed = JSON.parse(trimmed);
+    if (!Array.isArray(parsed)) {
+      throw new Error(`fromRunRecordDir: file '${file}' did not parse to an array`);
+    }
+    return parsed.map((value, index) => ({ index, value }));
+  }
+  const out = [];
+  const lines = trimmed.split("\n");
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i].trim();
+    if (line.length === 0) continue;
+    try {
+      out.push({ index: i, value: JSON.parse(line) });
+    } catch (err) {
+      throw new Error(
+        `fromRunRecordDir: file '${file}' line ${i + 1} is not valid JSON: ${err instanceof Error ? err.message : String(err)}`
+      );
+    }
+  }
+  return out;
+}
+async function collectFiles(dir, include, recursive) {
+  const entries = await readdir(dir, { withFileTypes: true });
+  const files = [];
+  const subdirs = [];
+  for (const entry of entries) {
+    if (entry.isDirectory()) {
+      if (recursive) subdirs.push(join(dir, entry.name));
+      continue;
+    }
+    if (include(entry.name)) files.push(join(dir, entry.name));
+  }
+  files.sort();
+  subdirs.sort();
+  for (const sub of subdirs) {
+    files.push(...await collectFiles(sub, include, recursive));
+  }
+  return files;
+}
+
+// src/contract/eval-reporting-suite.ts
+var ANALYSIS_ARTIFACT2 = "analysis.json";
+async function evalReportingSuite(input, options = {}) {
+  const fromPath = typeof input === "string";
+  let runs;
+  let files = [];
+  let rejected = [];
+  if (fromPath) {
+    const loaded = await fromRunRecordDir(input, options.load);
+    runs = loaded.runs;
+    files = loaded.files;
+    rejected = loaded.rejected;
+  } else {
+    runs = input;
+  }
+  if (runs.length === 0) {
+    throw new Error(
+      fromPath ? `evalReportingSuite: no RunRecords found at '${input}'` : "evalReportingSuite: no RunRecords to analyze"
+    );
+  }
+  const report = await analyzeRuns({ ...options.analyze, runs });
+  const result = {
+    report,
+    provenance: {
+      generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
+      runCount: runs.length,
+      sourcePath: fromPath ? input : null,
+      files,
+      rejected
+    },
+    writtenTo: null
+  };
+  const target = resolveWriteTarget(options.write, fromPath ? input : null);
+  if (target) {
+    await mkdir(dirname(target), { recursive: true });
+    await writeFile(target, `${JSON.stringify(result, null, 2)}
+`, "utf8");
+    result.writtenTo = target;
+  }
+  return result;
+}
+function resolveWriteTarget(write, sourcePath) {
+  if (!write) return null;
+  if (typeof write === "string") {
+    const looksLikeDir = write.endsWith("/") || !write.endsWith(".json") && !write.endsWith(".jsonl");
+    return looksLikeDir ? join2(write, ANALYSIS_ARTIFACT2) : write;
+  }
+  if (sourcePath === null) {
+    throw new Error(
+      "evalReportingSuite: write:true needs a source path to anchor analysis.json \u2014 pass an explicit output path when analyzing in-memory records"
+    );
+  }
+  const isFile = sourcePath.endsWith(".json") || sourcePath.endsWith(".jsonl");
+  return isFile ? join2(dirname(sourcePath), ANALYSIS_ARTIFACT2) : join2(sourcePath, ANALYSIS_ARTIFACT2);
+}
+
 // src/contract/diff.ts
 function keyForCell(cell) {
   return JSON.stringify([cell.scenarioId, cell.rep]);
@@ -833,6 +975,7 @@ export {
   diffGenerations,
   diffRunBaselineToWinner,
   diffRuns,
+  evalReportingSuite,
   evolutionaryProposer,
   fromClaudeCodeSession,
   fromCodexSession,
@@ -842,6 +985,7 @@ export {
   fromOtelSpans,
   fromPiSession,
   fromPigraphSession,
+  fromRunRecordDir,
   fsCampaignStorage,
   gepaProposer,
   heldOutGate,