@tangle-network/agent-eval 0.121.0 → 0.122.0

package/dist/analyst/index.d.ts

@@ -1208,6 +1208,7 @@ interface SemanticConceptJudgeOptions {
     llm?: LlmClientOptions;
     costLedger?: CostLedgerHandle;
     costPhase?: string;
+    costTags?: Record<string, string>;
     signal?: AbortSignal;
     /**
      * Score aggregation strategy. Default `mean` — uniform average across

package/dist/analyst/index.js

@@ -9,7 +9,7 @@ import {
   diffFindings,
   emitSkillUsageFindings,
   runSemanticConceptJudge
-} from "../chunk-JM2SKQMS.js";
+} from "../chunk-P62I523A.js";
 import {
   behavioralAnalyst,
   buildDefaultAnalystRegistry,

package/dist/belief-state/index.d.ts

@@ -794,6 +794,37 @@ interface RunRecord {
 }
 
 type CodeAgentSessionSource = 'codex' | 'claude-code' | 'opencode' | 'kimi-code' | 'pi';
+type CodeAgentSessionTerminalStatus = 'completed' | 'failed' | 'unknown';
+type CodeAgentSessionActionKind = 'tool' | 'patch' | 'terminal' | 'graph-completion';
+type CodeAgentSessionActionSurface = 'tool' | 'mcp' | 'subagent' | 'skill' | 'hook' | 'web' | 'code';
+type CodeAgentSessionActionStatus = 'started' | 'completed' | 'failed' | 'unknown';
+interface CodeAgentSessionExecutionReceipt {
+    exitCode: number;
+    startedAtMs?: number;
+    completedAtMs?: number;
+}
+interface CodeAgentSessionAction {
+    id: string;
+    stepIndex: number;
+    kind: CodeAgentSessionActionKind;
+    surface: CodeAgentSessionActionSurface;
+    name: string;
+    status: CodeAgentSessionActionStatus;
+    timestampMs?: number;
+    costUsd?: number;
+    metadata: Record<string, unknown>;
+}
+interface CodeAgentSessionObservation {
+    source: CodeAgentSessionSource;
+    sessionId: string;
+    finalText: string | null;
+    terminal: {
+        status: CodeAgentSessionTerminalStatus;
+        explicit: boolean;
+    };
+    actions: CodeAgentSessionAction[];
+}
+
 interface CodeAgentSessionMetrics {
     entries: number;
     userMessages: number;
@@ -809,6 +840,12 @@ interface CodeAgentSessionMetrics {
     turnsCompleted: number;
     turnsAborted: number;
     contextCompactions: number;
+    mcpCalls: number;
+    subagentCalls: number;
+    skillCalls: number;
+    hookCalls: number;
+    webCalls: number;
+    codeActions: number;
     prLinks: number;
     fileSnapshots: number;
     graphNodes: number;
@@ -836,6 +873,7 @@ interface CodeAgentSessionDiagnostic {
     malformedLines: number;
     inferredScore: boolean;
     hasExplicitTerminalSignal: boolean;
+    hasFinalOutput: boolean;
     hasQualityLabel: boolean;
     hasTokenUsage: boolean;
     hasCost: boolean;
@@ -861,6 +899,9 @@ interface CodeAgentSessionIntakeOptions {
      *  sentinel as observed. When omitted, source-reported cost wins, then a
      *  token-priced estimate, then uncaptured. */
     costProvenance?: RunCostProvenance;
+    /** Exact executor-owned process result. This is required when a provider's
+     * JSON stream has no terminal event, as with `opencode run --format json`. */
+    execution?: CodeAgentSessionExecutionReceipt;
 }
 
 interface BeliefOpeOptions extends OffPolicyOptions {
@@ -909,6 +950,8 @@ type CodeAgentBeliefDecisionTargetId = 'failure-recovery' | 'tool-selection' | '
 interface ExtractCodeAgentBeliefDecisionPointsOptions {
     source: CodeAgentSessionSource;
     entries: unknown[];
+    /** Reuse intake's provider-neutral projection when available. */
+    observation?: CodeAgentSessionObservation;
     run: Pick<RunRecord, 'runId' | 'scenarioId' | 'outcome' | 'costUsd'>;
     sourcePath?: string;
 }

package/dist/belief-state/index.js

@@ -3,8 +3,9 @@ import {
   fromCodexSession,
   fromKimiCodeSession,
   fromOpenCodeSession,
-  fromPiSession
-} from "../chunk-HKUCJ437.js";
+  fromPiSession,
+  observeCodeAgentSession
+} from "../chunk-7VYEGHWF.js";
 import {
   calibrationFromPairs
 } from "../chunk-NPCTHQIO.js";
@@ -479,275 +480,24 @@ function analyzeBeliefDecisionCorpus(options) {
   return { inventory, target, policy, evaluation, diagnostics };
 }
 function observedActionsFor(source, entries, options) {
-  switch (source) {
-    case "codex":
-      return codexObservedActions(entries, options);
-    case "claude-code":
-      return claudeObservedActions(entries, options);
-    case "opencode":
-      return openCodeObservedActions(entries, options);
-    case "kimi-code":
-      return kimiObservedActions(entries, options);
-    case "pi":
-      return piObservedActions(entries, options);
+  const observation = options.observation ?? observeCodeAgentSession({ source, entries, sourcePath: options.sourcePath });
+  if (observation.source !== source) {
+    throw new Error("code-agent observation source does not match extraction source");
   }
+  return observation.actions.map((action) => observedActionFromSession(action, options));
 }
-function codexObservedActions(entries, options) {
-  const actions = [];
-  const calls = /* @__PURE__ */ new Map();
-  for (const entry of entries) {
-    const payload = record(entry.payload) ?? {};
-    const entryType = stringField(entry, "type");
-    const payloadType = stringField(payload, "type");
-    const timestamp = timestampMs(entry.timestamp);
-    if (entryType === "response_item") {
-      if (payloadType === "function_call" || payloadType === "custom_tool_call") {
-        const callId = stringField(payload, "call_id") ?? stringField(payload, "id") ?? `${actions.length}`;
-        const action = observedAction({
-          options,
-          localId: callId,
-          stepIndex: actions.length,
-          kind: "tool",
-          action: stringField(payload, "name") ?? payloadType,
-          timestamp,
-          metadata: { sourceEventType: entryType, payloadType }
-        });
-        calls.set(callId, action);
-        actions.push(action);
-      }
-      if (payloadType === "function_call_output" || payloadType === "custom_tool_call_output") {
-        const callId = stringField(payload, "call_id") ?? stringField(payload, "id");
-        const action = callId ? calls.get(callId) : void 0;
-        if (action) action.success = !looksLikeError(payload.output);
-      }
-    }
-    if (entryType === "event_msg") {
-      if (payloadType === "patch_apply_end") {
-        actions.push(
-          observedAction({
-            options,
-            localId: stringField(payload, "call_id") ?? `patch-${actions.length}`,
-            stepIndex: actions.length,
-            kind: "patch",
-            action: "patch",
-            timestamp,
-            success: typeof payload.success === "boolean" ? payload.success : void 0,
-            metadata: { sourceEventType: entryType, payloadType }
-          })
-        );
-      }
-      if (payloadType === "task_complete" || payloadType === "turn_aborted") {
-        actions.push(
-          observedAction({
-            options,
-            localId: `${payloadType}-${actions.length}`,
-            stepIndex: actions.length,
-            kind: "terminal",
-            action: payloadType === "task_complete" ? "stop" : "abort",
-            timestamp,
-            success: payloadType === "task_complete",
-            metadata: { sourceEventType: entryType, payloadType }
-          })
-        );
-      }
-    }
-  }
-  return actions;
-}
-function claudeObservedActions(entries, options) {
-  const actions = [];
-  const calls = /* @__PURE__ */ new Map();
-  for (const entry of entries) {
-    const timestamp = timestampMs(entry.timestamp);
-    const message = record(entry.message);
-    const content = Array.isArray(message?.content) ? message.content : [];
-    for (const item of content) {
-      const part = record(item);
-      if (!part) continue;
-      const partType = stringField(part, "type");
-      if (partType === "tool_use") {
-        const id = stringField(part, "id") ?? `tool-${actions.length}`;
-        const action = observedAction({
-          options,
-          localId: id,
-          stepIndex: actions.length,
-          kind: "tool",
-          action: stringField(part, "name") ?? "tool",
-          timestamp,
-          metadata: { sourceEventType: stringField(entry, "type"), partType }
-        });
-        calls.set(id, action);
-        actions.push(action);
-      }
-      if (partType === "tool_result") {
-        const id = stringField(part, "tool_use_id");
-        const action = id ? calls.get(id) : void 0;
-        if (action) action.success = part.is_error !== true;
-      }
-    }
-    if (stringField(entry, "type") === "pr-link") {
-      actions.push(
-        observedAction({
-          options,
-          localId: `pr-link-${actions.length}`,
-          stepIndex: actions.length,
-          kind: "terminal",
-          action: "stop",
-          timestamp,
-          success: true,
-          metadata: { sourceEventType: "pr-link" }
-        })
-      );
-    }
-  }
-  return actions;
-}
-function openCodeObservedActions(entries, options) {
-  const actions = [];
-  for (const entry of entries) {
-    const type = stringField(entry, "type");
-    const role = stringField(entry, "role");
-    const time = record(entry.time);
-    const timestamp = timestampMs(time?.created);
-    if (type === "tool") {
-      const state = record(entry.state);
-      const status = stringField(state ?? {}, "status");
-      actions.push(
-        observedAction({
-          options,
-          localId: stringField(entry, "id") ?? `tool-${actions.length}`,
-          stepIndex: actions.length,
-          kind: "tool",
-          action: stringField(entry, "tool") ?? "tool",
-          timestamp,
-          success: status === "completed" ? true : status === "error" ? false : void 0,
-          metadata: { sourceEventType: type, status }
-        })
-      );
-    }
-    if (type === "patch") {
-      actions.push(
-        observedAction({
-          options,
-          localId: stringField(entry, "id") ?? `patch-${actions.length}`,
-          stepIndex: actions.length,
-          kind: "patch",
-          action: "patch",
-          timestamp,
-          success: true,
-          metadata: { sourceEventType: type }
-        })
-      );
-    }
-    if (role === "assistant") {
-      const finish = stringField(entry, "finish");
-      if (finish === "stop" || finish === "error") {
-        actions.push(
-          observedAction({
-            options,
-            localId: stringField(entry, "id") ?? `terminal-${actions.length}`,
-            stepIndex: actions.length,
-            kind: "terminal",
-            action: finish === "stop" ? "stop" : "abort",
-            timestamp: timestampMs(time?.completed) ?? timestamp,
-            success: finish === "stop",
-            costUsd: numberField(entry, "cost"),
-            metadata: { sourceEventType: "assistant", finish }
-          })
-        );
-      }
-    }
-  }
-  return actions;
-}
-function kimiObservedActions(entries, options) {
-  const actions = [];
-  const calls = /* @__PURE__ */ new Map();
-  for (const entry of entries) {
-    const timestamp = timestampMs(entry.timestamp);
-    const message = record(entry.message);
-    const messageType = stringField(message ?? {}, "type");
-    const payload = record(message?.payload) ?? {};
-    if (messageType === "ToolCall") {
-      const call = record(payload.function);
-      const id = stringField(payload, "id") ?? `tool-${actions.length}`;
-      const action = observedAction({
-        options,
-        localId: id,
-        stepIndex: actions.length,
-        kind: "tool",
-        action: stringField(call ?? {}, "name") ?? "tool",
-        timestamp,
-        metadata: { sourceEventType: messageType }
-      });
-      calls.set(id, action);
-      actions.push(action);
-    }
-    if (messageType === "ToolResult") {
-      const id = stringField(payload, "tool_call_id");
-      const action = id ? calls.get(id) : void 0;
-      if (action) action.success = record(payload.return_value)?.is_error !== true;
-    }
-    if (messageType === "TurnEnd" || messageType === "StepInterrupted") {
-      actions.push(
-        observedAction({
-          options,
-          localId: `${messageType}-${actions.length}`,
-          stepIndex: actions.length,
-          kind: "terminal",
-          action: messageType === "TurnEnd" ? "stop" : "abort",
-          timestamp,
-          success: messageType === "TurnEnd",
-          metadata: { sourceEventType: messageType }
-        })
-      );
-    }
-  }
-  return actions;
-}
-function piObservedActions(entries, options) {
-  const actions = [];
-  for (const entry of entries) {
-    const nodes = Array.isArray(entry.nodes) ? entry.nodes : [];
-    for (const node of nodes) {
-      const obj = record(node);
-      const ir = record(obj?.ir) ?? obj;
-      const kind = stringField(ir ?? {}, "kind");
-      if (kind === "ToolInvocation") {
-        actions.push(
-          observedAction({
-            options,
-            localId: stringField(ir ?? {}, "id") ?? stringField(obj ?? {}, "id") ?? `tool-${actions.length}`,
-            stepIndex: actions.length,
-            kind: "tool",
-            action: "graph-tool",
-            timestamp: timestampMs(ir?.createdAt),
-            success: void 0,
-            metadata: { sourceEventType: "graph-node", graphKind: kind }
-          })
-        );
-      }
-      if (kind === "ToolResult") {
-        const prior = [...actions].reverse().find((action) => action.kind === "tool");
-        if (prior) prior.success = true;
-      }
-      if (kind === "CompletionDecision") {
-        actions.push(
-          observedAction({
-            options,
-            localId: stringField(ir ?? {}, "id") ?? stringField(obj ?? {}, "id") ?? `completion-${actions.length}`,
-            stepIndex: actions.length,
-            kind: "graph-completion",
-            action: "complete",
-            timestamp: timestampMs(ir?.createdAt),
-            success: true,
-            metadata: { sourceEventType: "graph-node", graphKind: kind }
-          })
-        );
-      }
-    }
-  }
-  return actions;
+function observedActionFromSession(action, options) {
+  return observedAction({
+    options,
+    localId: action.id,
+    stepIndex: action.stepIndex,
+    kind: action.kind,
+    action: action.name,
+    timestamp: action.timestampMs,
+    success: action.status === "completed" ? true : action.status === "failed" ? false : void 0,
+    costUsd: action.costUsd,
+    metadata: { surface: action.surface, status: action.status, ...action.metadata }
+  });
 }
 function toolSelectionDecision(action, options) {
   return {
@@ -946,35 +696,6 @@ function groupBy(values, keyOf) {
 function mean2(values) {
   return values.reduce((sum, value) => sum + value, 0) / values.length;
 }
-function looksLikeError(value) {
-  if (isRecord(value)) {
-    if (value.is_error === true || value.error === true) return true;
-    if ("Err" in value) return true;
-  }
-  if (typeof value !== "string") return false;
-  return /\b(error|failed|exception|traceback)\b/i.test(value);
-}
-function timestampMs(value) {
-  if (typeof value === "number" && Number.isFinite(value)) {
-    return value > 1e12 ? value : value * 1e3;
-  }
-  if (typeof value === "string" && value.length > 0) {
-    const parsed = Date.parse(value);
-    return Number.isFinite(parsed) ? parsed : void 0;
-  }
-  return void 0;
-}
-function stringField(obj, key) {
-  const value = obj[key];
-  return typeof value === "string" && value.length > 0 ? value : void 0;
-}
-function numberField(obj, key) {
-  const value = obj[key];
-  return typeof value === "number" && Number.isFinite(value) ? value : void 0;
-}
-function record(value) {
-  return isRecord(value) ? value : null;
-}
 function isRecord(value) {
   return value !== null && typeof value === "object" && !Array.isArray(value);
 }
@@ -1068,10 +789,11 @@ function buildCodeAgentBeliefEvidenceCorpus(options) {
     runs.push(...intake.runs);
     metrics.push(...intake.metrics);
     intakeDiagnostics.push(...intake.diagnostics);
-    for (const run of intake.runs) {
+    for (const [index, run] of intake.runs.entries()) {
       const extraction = extractCodeAgentBeliefDecisionPoints({
         source: session.source,
         entries: session.entries,
+        observation: intake.observations[index],
         run,
         sourcePath: session.sourcePath
       });
@@ -1265,9 +987,9 @@ async function extractBeliefDecisionPoints(store, options = {}) {
 }
 function parseDecisionEvent(event, context) {
   const payload = event.payload;
-  const marker = stringField2(payload, "kind") ?? stringField2(payload, "type");
+  const marker = stringField(payload, "kind") ?? stringField(payload, "type");
   if (!marker || !DECISION_MARKERS.has(marker)) return null;
-  const decisionKind = stringField2(payload, "decisionKind");
+  const decisionKind = stringField(payload, "decisionKind");
   if (!isBeliefDecisionKind(decisionKind)) {
     return {
       diagnostic: {
@@ -1278,7 +1000,7 @@ function parseDecisionEvent(event, context) {
       }
     };
   }
-  const chosenAction = stringField2(payload, "chosenAction");
+  const chosenAction = stringField(payload, "chosenAction");
   if (!chosenAction) {
     return {
       diagnostic: {
@@ -1309,16 +1031,16 @@ function parseDecisionEvent(event, context) {
   }
   return {
     decision: {
-      id: stringField2(payload, "id") ?? event.eventId,
+      id: stringField(payload, "id") ?? event.eventId,
       runId: event.runId,
-      scenarioId: stringField2(payload, "scenarioId") ?? context.scenarioId,
-      stepIndex: numberField2(payload, "stepIndex") ?? context.stepIndex,
+      scenarioId: stringField(payload, "scenarioId") ?? context.scenarioId,
+      stepIndex: numberField(payload, "stepIndex") ?? context.stepIndex,
       kind: decisionKind,
       chosenAction,
       candidateActions: stringArrayField(payload, "candidateActions"),
       confidence: finiteUnitField(payload, "confidence"),
-      behaviorProb: numberField2(payload, "behaviorProb"),
-      targetProb: numberField2(payload, "targetProb"),
+      behaviorProb: numberField(payload, "behaviorProb"),
+      targetProb: numberField(payload, "targetProb"),
       qHat: finiteUnitField(payload, "qHat"),
       costUsd: nonNegativeNumberField(payload, "costUsd"),
       evidence,
@@ -1335,24 +1057,24 @@ function parseOutcome(payload) {
     score: finiteUnitField(value, "score"),
     reward: finiteUnitField(value, "reward"),
     costUsd: nonNegativeNumberField(value, "costUsd"),
-    observedAt: stringField2(value, "observedAt"),
+    observedAt: stringField(value, "observedAt"),
     metadata: recordField(value, "metadata")
   };
 }
-function stringField2(obj, key) {
+function stringField(obj, key) {
   const value = obj[key];
   return typeof value === "string" && value.length > 0 ? value : void 0;
 }
-function numberField2(obj, key) {
+function numberField(obj, key) {
   const value = obj[key];
   return typeof value === "number" && Number.isFinite(value) ? value : void 0;
 }
 function finiteUnitField(obj, key) {
-  const value = numberField2(obj, key);
+  const value = numberField(obj, key);
   return value === void 0 ? void 0 : Math.max(0, Math.min(1, value));
 }
 function nonNegativeNumberField(obj, key) {
-  const value = numberField2(obj, key);
+  const value = numberField(obj, key);
   return value === void 0 ? void 0 : Math.max(0, value);
 }
 function stringArrayField(obj, key) {
@@ -1776,24 +1498,24 @@ function buildRuntimeBenchmarkBeliefPhase0Measurement(options) {
     diagnostics: [...trajectory.diagnostics, ...diagnostics, ...measurement.diagnostics]
   };
 }
-function runtimeBenchmarkRecordId(record2) {
+function runtimeBenchmarkRecordId(record) {
   const parts = [
-    nonEmptyString(record2.benchmark),
-    nonEmptyString(record2.instanceId),
-    nonEmptyString(record2.condition)
+    nonEmptyString(record.benchmark),
+    nonEmptyString(record.instanceId),
+    nonEmptyString(record.condition)
   ].filter((part) => part !== void 0);
   return parts.length > 0 ? parts.join(":") : void 0;
 }
-function runtimeBenchmarkScenarioId(record2) {
-  return nonEmptyString(record2.instanceId);
+function runtimeBenchmarkScenarioId(record) {
+  return nonEmptyString(record.instanceId);
 }
 function runtimeBenchmarkDecisionPoints(records, diagnostics) {
   const decisions = [];
   for (let recordIndex = 0; recordIndex < records.length; recordIndex += 1) {
-    const record2 = records[recordIndex];
-    const raw = record2.runtimeDecisionPoints;
+    const record = records[recordIndex];
+    const raw = record.runtimeDecisionPoints;
     if (raw === void 0) continue;
-    const recordId = runtimeBenchmarkRecordId(record2) ?? `record[${recordIndex}]`;
+    const recordId = runtimeBenchmarkRecordId(record) ?? `record[${recordIndex}]`;
     if (!Array.isArray(raw)) {
       diagnostics.push(`${recordId}: runtimeDecisionPoints is not an array`);
       continue;
@@ -1930,7 +1652,7 @@ async function runBeliefShadowProbe(options) {
     }
   }
   await Promise.all(Array.from({ length: Math.min(concurrency, options.points.length) }, worker));
-  const completed = records.filter((record2) => !!record2);
+  const completed = records.filter((record) => !!record);
   return {
     probeId: options.probeId,
     records: completed,
@@ -2081,14 +1803,14 @@ function normalizeProbeResponse(response, options) {
   };
 }
 function summarizeShadowProbe(attempted, records) {
-  const confidences = records.map((record2) => record2.confidence);
-  const agreements = records.filter((record2) => record2.agreesWithObservedAction).length;
+  const confidences = records.map((record) => record.confidence);
+  const agreements = records.filter((record) => record.agreesWithObservedAction).length;
   return {
     attempted,
     completed: records.length,
     dropped: attempted - records.length,
-    withOutcome: records.filter((record2) => record2.outcome !== void 0).length,
-    withTargetProb: records.filter((record2) => record2.targetProb !== void 0).length,
+    withOutcome: records.filter((record) => record.outcome !== void 0).length,
+    withTargetProb: records.filter((record) => record.targetProb !== void 0).length,
     meanConfidence: confidences.length > 0 ? mean3(confidences) : null,
     observedAgreementRate: records.length > 0 ? agreements / records.length : null
   };