@tangle-network/agent-eval 0.93.0 → 0.95.0

package/dist/{chunk-QG2OVF2D.js → chunk-77T4STFI.js}

@@ -1,18 +1,32 @@
 import {
   analyzeTraces
-} from "./chunk-UHMJT4T7.js";
+} from "./chunk-IZCEK2HR.js";
 import {
+  LLM_CACHED_TOKEN_ATTR_KEYS,
+  LLM_COST_ATTR_KEYS,
+  LLM_COST_USD,
+  LLM_INPUT_TOKENS,
+  LLM_INPUT_TOKEN_ATTR_KEYS,
+  LLM_MODEL_ATTR_KEYS,
+  LLM_MODEL_NAME,
+  LLM_OUTPUT_TOKENS,
+  LLM_OUTPUT_TOKEN_ATTR_KEYS,
+  OPENINFERENCE_SPAN_KIND,
+  TOOL_NAME,
+  compareSpanTime,
   firstNumberAttr,
   firstStringAttr,
-  projectOtlpFlatLine
-} from "./chunk-QAY5UIJO.js";
+  projectOtlpFlatLine,
+  spanEpochMillis,
+  traceSpanKindToOpenInferenceKind
+} from "./chunk-YBIGNSCZ.js";
 import {
   defaultProviderRedactor,
   providerFromBaseUrl
 } from "./chunk-PC4UYEBM.js";
 import {
   validateRunRecord
-} from "./chunk-KWRRMR3J.js";
+} from "./chunk-LO6IOIJ2.js";
 import {
   canonicalize,
   hashJson
@@ -348,14 +362,26 @@ function nanoToIso(nano) {
   return Number.isFinite(ms) ? new Date(ms).toISOString() : (/* @__PURE__ */ new Date(0)).toISOString();
 }
 function applyOpenInference(attrs) {
-  if ("llm.model" in attrs && !("llm.model_name" in attrs)) {
-    attrs["llm.model_name"] = attrs["llm.model"];
+  if ("llm.model" in attrs && !(LLM_MODEL_NAME in attrs)) {
+    attrs[LLM_MODEL_NAME] = attrs["llm.model"];
   }
-  if ("tool.name" in attrs && !("inference.tool.name" in attrs)) {
-    attrs["inference.tool.name"] = attrs["tool.name"];
+  if ("llm.input_tokens" in attrs && !(LLM_INPUT_TOKENS in attrs)) {
+    attrs[LLM_INPUT_TOKENS] = attrs["llm.input_tokens"];
   }
-  if ("span.kind" in attrs && !("openinference.span.kind" in attrs)) {
-    attrs["openinference.span.kind"] = String(attrs["span.kind"]).toUpperCase();
+  if ("inference.llm.input_tokens" in attrs && !(LLM_INPUT_TOKENS in attrs)) {
+    attrs[LLM_INPUT_TOKENS] = attrs["inference.llm.input_tokens"];
+  }
+  if ("llm.output_tokens" in attrs && !(LLM_OUTPUT_TOKENS in attrs)) {
+    attrs[LLM_OUTPUT_TOKENS] = attrs["llm.output_tokens"];
+  }
+  if ("inference.llm.output_tokens" in attrs && !(LLM_OUTPUT_TOKENS in attrs)) {
+    attrs[LLM_OUTPUT_TOKENS] = attrs["inference.llm.output_tokens"];
+  }
+  if (TOOL_NAME in attrs && !("inference.tool.name" in attrs)) {
+    attrs["inference.tool.name"] = attrs[TOOL_NAME];
+  }
+  if ("span.kind" in attrs && !(OPENINFERENCE_SPAN_KIND in attrs)) {
+    attrs[OPENINFERENCE_SPAN_KIND] = String(attrs["span.kind"]).toUpperCase();
   }
 }
 function flattenOtlpExportToNdjson(otlpExport, opts = {}) {
@@ -398,39 +424,6 @@ function flattenOtlpExportToNdjson(otlpExport, opts = {}) {
 }
 
 // src/trace-analyst/otlp-to-run-records.ts
-var INPUT_TOKEN_KEYS = [
-  "llm.input_tokens",
-  "inference.llm.input_tokens",
-  "llm.token_count.prompt",
-  "gen_ai.usage.input_tokens",
-  "gen_ai.usage.prompt_tokens"
-];
-var OUTPUT_TOKEN_KEYS = [
-  "llm.output_tokens",
-  "inference.llm.output_tokens",
-  "llm.token_count.completion",
-  "gen_ai.usage.output_tokens",
-  "gen_ai.usage.completion_tokens"
-];
-var CACHED_TOKEN_KEYS = [
-  "llm.cached_tokens",
-  "inference.llm.cached_tokens",
-  "llm.token_count.prompt_cache_hit",
-  "gen_ai.usage.cached_tokens"
-];
-var COST_KEYS = [
-  "llm.cost_usd",
-  "inference.llm.cost.total",
-  "llm.cost.total",
-  "gen_ai.usage.cost"
-];
-var MODEL_KEYS = [
-  "llm.model_name",
-  "inference.llm.model_name",
-  "llm.model",
-  "gen_ai.request.model",
-  "gen_ai.response.model"
-];
 function otlpToRunRecords(otlpJsonl, opts) {
   return otlpToTraceRunRecords(otlpJsonl, opts).map((r) => r.record);
 }
@@ -516,7 +509,7 @@ function groupSpansByTrace(otlpJsonl) {
 }
 function aggregateTrace(traceId, spans, fallbackModel) {
   const ordered = [...spans].sort(
-    (a, b) => a.start_time.localeCompare(b.start_time) || a.span_id.localeCompare(b.span_id)
+    (a2, b2) => compareSpanTime(a2.start_time, b2.start_time) || a2.span_id.localeCompare(b2.span_id)
   );
   let input = 0;
   let output = 0;
@@ -531,20 +524,21 @@ function aggregateTrace(traceId, spans, fallbackModel) {
   let earliest = ordered[0]?.start_time ?? "";
   let latest = ordered[0]?.end_time ?? "";
   for (const s of ordered) {
-    if (s.start_time && (!earliest || s.start_time < earliest)) earliest = s.start_time;
-    if (s.end_time && s.end_time > latest) latest = s.end_time;
+    if (s.start_time && (!earliest || compareSpanTime(s.start_time, earliest) < 0))
+      earliest = s.start_time;
+    if (s.end_time && (!latest || compareSpanTime(s.end_time, latest) > 0)) latest = s.end_time;
     if (s.kind === "LLM") {
       llmSpanCount += 1;
-      const i = firstNumberAttr(s.attributes, INPUT_TOKEN_KEYS);
-      const o = firstNumberAttr(s.attributes, OUTPUT_TOKEN_KEYS);
-      const c = firstNumberAttr(s.attributes, CACHED_TOKEN_KEYS);
+      const i = firstNumberAttr(s.attributes, LLM_INPUT_TOKEN_ATTR_KEYS);
+      const o = firstNumberAttr(s.attributes, LLM_OUTPUT_TOKEN_ATTR_KEYS);
+      const c = firstNumberAttr(s.attributes, LLM_CACHED_TOKEN_ATTR_KEYS);
       if (i !== null) input += i;
       if (o !== null) output += o;
       if (c !== null) {
         cached += c;
         sawCached = true;
       }
-      const m = firstStringAttr(s.attributes, MODEL_KEYS) ?? s.model_name;
+      const m = firstStringAttr(s.attributes, LLM_MODEL_ATTR_KEYS) ?? s.model_name;
       if (m) modelVotes.set(m, (modelVotes.get(m) ?? 0) + 1);
     } else if (s.kind === "TOOL") {
       toolSpanCount += 1;
@@ -560,11 +554,9 @@ function aggregateTrace(traceId, spans, fallbackModel) {
   }
   const model = topVote(modelVotes) ?? firstModelAttr(ordered) ?? fallbackModel;
   let wallMs = 0;
-  if (earliest && latest) {
-    const a = Date.parse(earliest);
-    const b = Date.parse(latest);
-    if (!Number.isNaN(a) && !Number.isNaN(b)) wallMs = Math.max(0, b - a);
-  }
+  const a = spanEpochMillis(earliest);
+  const b = spanEpochMillis(latest);
+  if (a !== null && b !== null) wallMs = Math.max(0, b - a);
   const tokenUsage = sawCached ? { input, output, cached } : { input, output };
   return {
     traceId,
@@ -598,7 +590,7 @@ function resolveCost(opts, agg, spans) {
   let sawCost = false;
   for (const s of spans) {
     if (s.kind !== "LLM") continue;
-    const c = firstNumberAttr(s.attributes, COST_KEYS);
+    const c = firstNumberAttr(s.attributes, LLM_COST_ATTR_KEYS);
     if (c !== null) {
       perSpanCost += c;
       sawCost = true;
@@ -612,7 +604,9 @@ function resolveCost(opts, agg, spans) {
   return { costUsd: 0, costUnpriced: true };
 }
 function extractPromptCompletion(spans) {
-  const llm = spans.filter((s) => s.kind === "LLM").sort((a, b) => a.start_time.localeCompare(b.start_time) || a.span_id.localeCompare(b.span_id));
+  const llm = spans.filter((s) => s.kind === "LLM").sort(
+    (a, b) => compareSpanTime(a.start_time, b.start_time) || a.span_id.localeCompare(b.span_id)
+  );
   if (llm.length === 0) return {};
   const promptText = firstStringAttr(llm[0].attributes, ["input.value", "llm.input_messages", "gen_ai.prompt"]) ?? void 0;
   const last = llm[llm.length - 1];
@@ -639,7 +633,7 @@ function topVote(votes) {
 }
 function firstModelAttr(spans) {
   for (const s of spans) {
-    const m = firstStringAttr(s.attributes, MODEL_KEYS) ?? s.model_name;
+    const m = firstStringAttr(s.attributes, LLM_MODEL_ATTR_KEYS) ?? s.model_name;
     if (m) return m;
   }
   return null;
@@ -747,7 +741,20 @@ var FileSystemTraceStore = class {
   maxBytes;
   /** Lazy in-memory index for queries — populated on first read. */
   index;
-  loaded = false;
+  /** Memoized index build — concurrent first reads share one build, and an
+   *  append racing an in-flight load awaits this so its row isn't lost. */
+  indexPromise;
+  /** Strictly-increasing rollover stamp. Date.now() alone collides when two
+   *  rollovers land in the same millisecond, overwriting a rolled file. */
+  lastRolloverStamp = 0;
+  /**
+   * Per-file append serialization. stat → conditional-rename → appendFile is a
+   * read-modify-write on the active file; without a lock two concurrent appends
+   * to the same `name` can both pass the size check (exceeding maxBytes) or one
+   * can append to a file the other just renamed away. Each entry chains the
+   * next append behind the prior one for that file.
+   */
+  appendLocks = /* @__PURE__ */ new Map();
   constructor(options) {
     this.dir = options.dir;
     this.maxBytes = options.maxBytes ?? 32 * 1024 * 1024;
@@ -757,6 +764,23 @@ var FileSystemTraceStore = class {
     await fs.mkdir(this.dir, { recursive: true });
   }
   async append(name, record) {
+    if (this.indexPromise) await this.indexPromise;
+    const prior = this.appendLocks.get(name) ?? Promise.resolve();
+    const result = prior.then(() => this.appendLocked(name, record));
+    const tail = result.then(
+      () => {
+      },
+      () => {
+      }
+    );
+    this.appendLocks.set(name, tail);
+    try {
+      await result;
+    } finally {
+      if (this.appendLocks.get(name) === tail) this.appendLocks.delete(name);
+    }
+  }
+  async appendLocked(name, record) {
     await this.ensureDir();
     const fs = await import("fs/promises");
     const path = await import("path");
@@ -764,7 +788,9 @@ var FileSystemTraceStore = class {
     try {
       const stat = await fs.stat(active);
       if (stat.size >= this.maxBytes) {
-        const rolled = path.join(this.dir, `${name}.${Date.now()}.ndjson`);
+        const stamp = Math.max(Date.now(), this.lastRolloverStamp + 1);
+        this.lastRolloverStamp = stamp;
+        const rolled = path.join(this.dir, `${name}.${stamp}.ndjson`);
         await fs.rename(active, rolled);
       }
     } catch {
@@ -772,7 +798,7 @@ var FileSystemTraceStore = class {
     await fs.appendFile(active, `${JSON.stringify(record)}
 `, "utf8");
     if (this.index && !record?._update) {
-      void this.insertInto(name, record);
+      await this.insertInto(name, record);
     }
   }
   async insertInto(name, record) {
@@ -795,22 +821,56 @@ var FileSystemTraceStore = class {
         break;
     }
   }
-  async load() {
-    if (this.loaded && this.index) return this.index;
+  load() {
+    this.indexPromise ??= this.buildIndex();
+    return this.indexPromise;
+  }
+  async buildIndex() {
     const fs = await import("fs/promises");
     const path = await import("path");
     const store = new InMemoryTraceStore();
+    let entries;
     try {
-      const entries = await fs.readdir(this.dir);
+      entries = await fs.readdir(this.dir);
+    } catch {
+      this.index = store;
+      return store;
+    }
+    entries.sort((a, b) => {
+      const baseA = a.split(".")[0];
+      const baseB = b.split(".")[0];
+      if (baseA !== baseB) return baseA < baseB ? -1 : 1;
+      const ta = a.match(/^[^.]+\.(\d+)\.ndjson$/);
+      const tb = b.match(/^[^.]+\.(\d+)\.ndjson$/);
+      const tsA = ta ? Number(ta[1]) : Number.POSITIVE_INFINITY;
+      const tsB = tb ? Number(tb[1]) : Number.POSITIVE_INFINITY;
+      return tsA - tsB;
+    });
+    {
+      const runUpdates = [];
+      const spanUpdates = [];
       for (const file of entries) {
         if (!file.endsWith(".ndjson")) continue;
         const full = path.join(this.dir, file);
         const content = await fs.readFile(full, "utf8");
         const base = file.split(".")[0];
-        for (const line of content.split("\n")) {
+        const lines = content.split("\n");
+        for (let ln = 0; ln < lines.length; ln++) {
+          const line = lines[ln];
           if (!line.trim()) continue;
-          const record = JSON.parse(line);
+          let record;
+          try {
+            record = JSON.parse(line);
+          } catch (err) {
+            throw new Error(
+              `FileSystemTraceStore: corrupt NDJSON in ${file} line ${ln + 1}: ${err instanceof Error ? err.message : String(err)} \u2014 ${line.slice(0, 120)}`
+            );
+          }
           if (base === "runs") {
+            if (record?._update) {
+              runUpdates.push(record);
+              continue;
+            }
             try {
               await store.appendRun(record);
             } catch {
@@ -818,14 +878,10 @@ var FileSystemTraceStore = class {
             }
           } else if (base === "spans") {
             if (record?._update) {
-              try {
-                await store.updateSpan(record.spanId, record);
-              } catch {
-                await store.appendSpan(record);
-              }
-            } else {
-              await store.appendSpan(record);
+              spanUpdates.push(record);
+              continue;
             }
+            await store.appendSpan(record);
           } else if (base === "events") {
             await store.appendEvent(record);
           } else if (base === "artifacts") {
@@ -835,10 +891,14 @@ var FileSystemTraceStore = class {
           }
         }
       }
-    } catch {
+      for (const record of runUpdates) {
+        await store.updateRun(record.runId, record);
+      }
+      for (const record of spanUpdates) {
+        await store.updateSpan(record.spanId, record);
+      }
     }
     this.index = store;
-    this.loaded = true;
     return store;
   }
   async appendRun(run) {
@@ -1153,16 +1213,16 @@ function spanToOtlp(span, traceId, events) {
 }
 function flattenSpanAttributes(span) {
   const base = {
-    "span.kind": span.kind
+    [OPENINFERENCE_SPAN_KIND]: traceSpanKindToOpenInferenceKind(span.kind)
   };
   if (span.kind === "llm") {
-    base["llm.model"] = span.model;
-    if (span.inputTokens !== void 0) base["llm.input_tokens"] = span.inputTokens;
-    if (span.outputTokens !== void 0) base["llm.output_tokens"] = span.outputTokens;
-    if (span.costUsd !== void 0) base["llm.cost_usd"] = span.costUsd;
+    base[LLM_MODEL_NAME] = span.model;
+    if (span.inputTokens !== void 0) base[LLM_INPUT_TOKENS] = span.inputTokens;
+    if (span.outputTokens !== void 0) base[LLM_OUTPUT_TOKENS] = span.outputTokens;
+    if (span.costUsd !== void 0) base[LLM_COST_USD] = span.costUsd;
     if (span.finishReason) base["llm.finish_reason"] = span.finishReason;
   } else if (span.kind === "tool") {
-    base["tool.name"] = span.toolName;
+    base[TOOL_NAME] = span.toolName;
     if (span.latencyMs !== void 0) base["tool.latency_ms"] = span.latencyMs;
   } else if (span.kind === "retrieval") {
     base["retrieval.query"] = span.query;
@@ -1372,12 +1432,12 @@ function parseHeadersFromEnv() {
 function toOtlpSpan(span) {
   const endedAt = span.endedAt ?? span.startedAt;
   const attrs = {
-    "span.kind": span.kind
+    [OPENINFERENCE_SPAN_KIND]: traceSpanKindToOpenInferenceKind(span.kind)
   };
-  if (span.model) attrs["llm.model"] = span.model;
-  if (span.inputTokens !== void 0) attrs["llm.input_tokens"] = span.inputTokens;
-  if (span.outputTokens !== void 0) attrs["llm.output_tokens"] = span.outputTokens;
-  if (span.costUsd !== void 0) attrs["llm.cost_usd"] = span.costUsd;
+  if (span.model) attrs[LLM_MODEL_NAME] = span.model;
+  if (span.inputTokens !== void 0) attrs[LLM_INPUT_TOKENS] = span.inputTokens;
+  if (span.outputTokens !== void 0) attrs[LLM_OUTPUT_TOKENS] = span.outputTokens;
+  if (span.costUsd !== void 0) attrs[LLM_COST_USD] = span.costUsd;
   if (span.attributes) {
     for (const [k, v] of Object.entries(span.attributes)) {
       if (typeof v === "string" || typeof v === "number" || typeof v === "boolean") attrs[k] = v;
@@ -1519,7 +1579,7 @@ function projectCell(args) {
     const runEnd = msToIso(run.endedAt ?? run.startedAt);
     const runStatus = run.outcome?.failureClass && run.outcome.failureClass !== "success" ? "STATUS_CODE_ERROR" : "STATUS_CODE_OK";
     const runAttrs = {
-      "openinference.span.kind": "AGENT",
+      [OPENINFERENCE_SPAN_KIND]: "AGENT",
       "agent.name": agentName,
       "agent.workflow.name": serviceName,
       ...runAttributes(run)
@@ -1626,13 +1686,13 @@ function readMergedShards(cellDir, name, idKey) {
 }
 function spanToAttributes(span, events) {
   const attrs = {
-    "openinference.span.kind": spanKindToOpenInferenceKind(span.kind)
+    [OPENINFERENCE_SPAN_KIND]: traceSpanKindToOpenInferenceKind(span.kind)
   };
   if (span.kind === "llm") {
-    attrs["llm.model_name"] = span.model;
-    if (span.inputTokens !== void 0) attrs["llm.token_count.prompt"] = span.inputTokens;
-    if (span.outputTokens !== void 0) attrs["llm.token_count.completion"] = span.outputTokens;
-    if (span.costUsd !== void 0) attrs["llm.cost_usd"] = span.costUsd;
+    attrs[LLM_MODEL_NAME] = span.model;
+    if (span.inputTokens !== void 0) attrs[LLM_INPUT_TOKENS] = span.inputTokens;
+    if (span.outputTokens !== void 0) attrs[LLM_OUTPUT_TOKENS] = span.outputTokens;
+    if (span.costUsd !== void 0) attrs[LLM_COST_USD] = span.costUsd;
     if (span.finishReason) attrs["llm.finish_reason"] = span.finishReason;
     if (Array.isArray(span.messages)) {
       attrs["llm.input_messages"] = JSON.stringify(span.messages.slice(-6));
@@ -1641,7 +1701,7 @@ function spanToAttributes(span, events) {
       attrs["llm.output_messages"] = JSON.stringify([{ role: "assistant", content: span.output }]);
     }
   } else if (span.kind === "tool") {
-    attrs["tool.name"] = span.toolName;
+    attrs[TOOL_NAME] = span.toolName;
     if (span.latencyMs !== void 0) attrs["tool.latency_ms"] = span.latencyMs;
     attrs["input.value"] = safeStringify(span.args);
     if (span.result !== void 0) attrs["output.value"] = safeStringify(span.result);
@@ -1672,24 +1732,6 @@ function spanKindToOtlpKind(kind) {
       return "SPAN_KIND_INTERNAL";
   }
 }
-function spanKindToOpenInferenceKind(kind) {
-  switch (kind) {
-    case "llm":
-      return "LLM";
-    case "tool":
-      return "TOOL";
-    case "retrieval":
-      return "CHAIN";
-    case "judge":
-      return "GUARDRAIL";
-    case "sandbox":
-      return "CHAIN";
-    case "agent":
-      return "AGENT";
-    default:
-      return "SPAN";
-  }
-}
 function toLine(args) {
   return {
     trace_id: args.traceId,
@@ -1912,4 +1954,4 @@ export {
   createReplayFetch,
   iterateRawCalls
 };
-//# sourceMappingURL=chunk-QG2OVF2D.js.map
+//# sourceMappingURL=chunk-77T4STFI.js.map