@tangle-network/agent-eval 0.136.0 → 0.138.0

package/dist/{default-registry-Brxr728w.d.ts → default-registry-PUhIVRWz.d.ts}

@@ -1,9 +1,65 @@
-import { c as CostLedgerHandle } from "./cost-ledger-fGS_u_O1.js";
-import { A as ChatClient } from "./types-Cc3qbqzj.js";
-import { t as TraceAnalysisStore } from "./store-CxJry_cs.js";
-import { c as AnalystRunInputs, i as AnalystFinding, l as AnalystRunResult, n as AnalystContext, p as EvidenceRef, r as AnalystCost, s as AnalystRunEvent, t as Analyst, u as AnalystRunSummary } from "./types-DVjczBM9.js";
+import { N as TraceAnalystSpan, i as AnalystFinding, n as AnalystContext, p as EvidenceRef, r as AnalystCost, x as TraceAnalysisStore } from "./types-BtJhn8v6.js";
+import { l as ExactCapableAnalyst, u as ExactExecutionComponentIdentity } from "./exact-types-Dpw2LeHA.js";
+import { n as AnalystRegistry, r as AnalystRegistryOptions } from "./registry-C4yJTza7.js";
 import { AxAIService, AxFunction } from "@ax-llm/ax";
 import { z } from "zod";
+//#region src/trace-analyst/behavioral-metrics.d.ts
+type SuboptimalCode = 'monotonic-input-growth' | 'output-length-decay' | 'single-tool-dependency' | 'no-self-verification';
+interface SuboptimalSignal {
+  code: SuboptimalCode;
+  severity: 'high' | 'medium' | 'low';
+  /** Human-readable claim, with the backing numbers inlined. */
+  detail: string;
+  /** The exact figures the detector fired on — auditable, no model in the loop. */
+  evidence: Record<string, number | string | boolean>;
+}
+interface BehavioralMetrics {
+  /** The only trace represented by these metrics; null when spans are empty. */
+  traceId: string | null;
+  llmCallCount: number;
+  /** Causally serial LLM timelines. Parallel branches are never joined. */
+  tokenSequences: BehavioralTokenSequence[];
+  /** Token values from the longest serial timeline, retained for convenience. */
+  inputTokenTrajectory: number[];
+  outputTokenTrajectory: number[];
+  toolHistogram: Record<string, number>;
+  totalToolCalls: number;
+  distinctTools: number;
+  /** distinct/total tool calls; 1.0 when there are no tool calls. */
+  toolDiversityRatio: number;
+  hasSelfVerification: boolean;
+  signals: SuboptimalSignal[];
+}
+interface BehavioralTokenSequence {
+  scopeId: string;
+  spanIds: string[];
+  inputTokenTrajectory: Array<number | null>;
+  outputTokenTrajectory: Array<number | null>;
+}
+/**
+ * Reduce a span list to behavioral metrics + fired suboptimality signals.
+ * Pure + deterministic: same spans → same output, on any machine, no model.
+ */
+declare function computeTraceMetrics(spans: readonly TraceAnalystSpan[]): BehavioralMetrics;
+//#endregion
+//#region src/analyst/behavioral-analyst.d.ts
+interface BehavioralAnalystOptions {
+  /** Refuse larger unfiltered datasets instead of sampling them silently. Default 1,000. */
+  maxTraces?: number;
+  /** Evidence locations retained per finding. Counts still cover every analyzed trace. Default 20. */
+  maxEvidenceRefsPerFinding?: number;
+}
+/**
+ * Map computed signals → structured AnalystFindings. Pure: no LLM, no clock
+ * dependence beyond `produced_at` (overridable for deterministic tests).
+ */
+declare function deriveEfficiencyFindings(metrics: BehavioralMetrics, opts?: {
+  analystId?: string;
+  producedAt?: string;
+}): AnalystFinding[];
+/** The deterministic behavioral/efficiency analyst (no LLM, any-model). */
+declare function behavioralAnalyst(options?: BehavioralAnalystOptions): ExactCapableAnalyst<TraceAnalysisStore>;
+//#endregion
 //#region src/analyst/finding-signature.d.ts
 declare const ANALYST_SEVERITIES: readonly ['critical', 'high', 'medium', 'low', 'info'];
 declare const RawAnalystEvidenceSchema: z.ZodObject<{
@@ -58,6 +114,12 @@ interface TraceAnalystKindSpec {
   actorDescription: string;
   /** Tool functions the actor may call. Pick narrow subsets via `ANALYST_TOOL_GROUPS`. */
   buildTools: (store: TraceAnalysisStore) => AxFunction[];
+  /**
+   * Deterministically prepare one bounded context value before model work.
+   * Returning a string uses Ax's long-context path without tools. Returning
+   * undefined keeps the normal tool-driven path.
+   */
+  prepareContext?: (store: TraceAnalysisStore, ctx: AnalystContext) => string | undefined | Promise<string | undefined>;
   /** Bounded semantic subqueries. `maxCalls: 0` disables model fan-out. */
   subqueries?: {
     maxCalls: number;
@@ -75,22 +137,14 @@ interface TraceAnalystKindSpec {
   postProcess?: (row: RawAnalystFinding, ctx: AnalystContext) => RawAnalystFinding | null;
   /** Minimum citations per finding. Default 1; rows below it are rejected. */
   minimumEvidenceCitations?: number;
-  /** Optional optimizer hook — populated when a kind wants to fit its prompt against labeled examples. */
-  goldens?: TraceAnalystGolden[];
-}
-/**
- * One labeled example consumed by Ax optimizers (MIPRO / GEPA / Bootstrap).
- * Each input is the same `{question}` an analyst would receive; `expected`
- * is the ground-truth finding set a fitted prompt should produce on this
- * input. Metric: kind-specific (default: F1 on `finding_id` overlap).
- */
-interface TraceAnalystGolden {
-  question: string;
-  expected: ReadonlyArray<Omit<RawAnalystFinding, 'confidence'>>;
+  /** Reject a substantive report when no structured finding survives validation. */
+  requireStructuredFindings?: boolean;
 }
 interface CreateTraceAnalystKindOpts {
   /** AxAIService bound at registration time. */
   ai: AxAIService;
+  /** Required when this analyst is used by `runExact`; binds the effective AI service. */
+  aiIdentity?: ExactExecutionComponentIdentity;
   /** Required unless `ai` was created by {@link createAnalystAi}. */
   model?: string;
   /** Override the spec's `version` (e.g. when an optimizer has fitted a new prompt). */
@@ -119,7 +173,7 @@ interface CreateTraceAnalystKindOpts {
  * `analyze()` call (the agent carries chat-log + usage state we don't
  * want shared across analyst runs).
  */
-declare function createTraceAnalystKind(spec: TraceAnalystKindSpec, opts: CreateTraceAnalystKindOpts): Analyst<TraceAnalysisStore>;
+declare function createTraceAnalystKind(spec: TraceAnalystKindSpec, opts: CreateTraceAnalystKindOpts): ExactCapableAnalyst<TraceAnalysisStore>;
 /**
  * Render a compact prior-findings block the actor reads alongside its
  * brief. Each row is one line so the actor can scan dozens cheaply.
@@ -138,139 +192,24 @@ declare function renderPriorFindings(prior: AnalystContext['priorFindings']): st
 /** Render findings produced earlier in this same registry run. */
 declare function renderUpstreamFindings(upstream: AnalystContext['upstreamFindings']): string;
 //#endregion
-//#region src/analyst/registry.d.ts
-interface AnalystHooks {
-  /** Before analyze() — last chance to mutate ctx (e.g. inject tags, override budget). */
-  onBeforeAnalyze?(args: {
-    analyst: Analyst;
-    ctx: AnalystContext;
-    runId: string;
-  }): void | Promise<void>;
-  /** After every analyst (ok | failed | skipped). Use for telemetry, ingestion, rotation. */
-  onAfterAnalyze?(args: {
-    analyst: Analyst;
-    summary: AnalystRunSummary;
-    findings: AnalystFinding[];
-    runId: string;
-  }): void | Promise<void>;
-  /**
-   * On analyst exception. Hook MAY return findings to convert the
-   * error into structured findings; the summary still reports 'failed'.
-   * Return void to keep the default empty-findings behavior.
-   */
-  onError?(args: {
-    analyst: Analyst;
-    error: Error;
-    runId: string;
-  }): AnalystFinding[] | undefined | Promise<AnalystFinding[] | undefined>;
-  /** Once after registry.run() completes. Use for final aggregation, persistence. */
-  onComplete?(args: {
-    result: AnalystRunResult;
-  }): void | Promise<void>;
-}
-interface BudgetPolicy {
-  /** Overall USD cap across the registry.run(). */
-  totalUsd?: number;
-  /** Per-analyst weight for the default allocator. Missing ids get weight 1. */
-  weights?: Record<string, number>;
-  /**
-   * Custom allocator — receives the analyst, remaining/total budget, and
-   * the count of analysts that will run. Returns the per-analyst budget
-   * (or undefined only when the run has no overall cap). Overrides weights
-   * when set.
-   */
-  allocate?: (args: {
-    analyst: Analyst;
-    totalUsd: number | undefined;
-    remainingUsd: number | undefined;
-    runningCount: number;
-  }) => number | undefined;
-}
-interface AnalystRegistryOptions {
-  /** Shared chat client passed to every LLM analyst via AnalystContext. */
-  chat?: ChatClient;
-  /** Logger callback. Defaults to a no-op. */
-  log?: (msg: string, fields?: Record<string, unknown>) => void;
-  /** Hooks invoked around analyze() — observability + customization seam. */
-  hooks?: AnalystHooks;
-  /** Default budget when run() doesn't override. */
-  defaultBudget?: BudgetPolicy;
-}
-interface RegistryRunOpts {
-  /** Restrict to a subset of registered analysts by id. */
-  only?: string[];
-  /** Skip these analysts even if registered. Useful for cheap iteration. */
-  skip?: string[];
-  /** Budget policy — totalUsd + optional weights/allocator. Falls back to options.defaultBudget. */
-  budget?: BudgetPolicy;
-  /** Active-work cap for the complete registry run. Model receipt settlement may follow. */
-  timeoutMs?: number;
-  /** Abort signal — forwarded into every analyst's context. */
-  signal?: AbortSignal;
-  /** Shared paid-call account forwarded to every analyst. */
-  costLedger?: CostLedgerHandle;
-  /** Attribution phase for calls written to `costLedger`. */
-  costPhase?: string;
-  /** Tags echoed into AnalystContext.tags — useful for tracking environment/version in findings. */
-  tags?: Record<string, string>;
-  /**
-   * Prior-run findings made available as retrieval context to every
-   * analyst via `ctx.priorFindings`. The registry forwards the slice
-   * whose `analyst_id` matches each registered analyst so a kind sees
-   * only its own history. Pass `{ '*': findings }` to broadcast to
-   * every analyst (useful when several kinds share the same historical
-   * context). For findings from this run, use `chainFindings` instead.
-   */
-  priorFindings?: ReadonlyArray<AnalystFinding> | Record<string, ReadonlyArray<AnalystFinding>>;
-  /**
-   * Pass findings produced earlier in this registry run to each later analyst
-   * via `ctx.upstreamFindings`. Registration order is dependency order.
-   * Disabled by default because independent analyst suites must opt in.
-   */
-  chainFindings?: boolean;
-}
-declare class AnalystRegistry {
-  private readonly analysts;
-  private readonly options;
-  constructor(options?: AnalystRegistryOptions);
-  register(analyst: Analyst): void;
-  list(): ReadonlyArray<{
-    id: string;
-    description: string;
-    version: string;
-    cost: Analyst['cost'];
-  }>;
-  run(runId: string, inputs: AnalystRunInputs, runOpts?: RegistryRunOpts): Promise<AnalystRunResult>;
-  /**
-   * Streaming counterpart to `run()`. Emits `AnalystRunEvent` values
-   * in real time — `run-started`, then per-analyst `skipped` /
-   * `started` / `completed`, then a terminal `run-completed` whose
-   * payload is the full `AnalystRunResult`. UIs use this to render
-   * progress; persistence consumers use `run()` and read the result.
-   *
-   * Hooks (`onBeforeAnalyze` / `onAfterAnalyze` / `onError` /
-   * `onComplete`) fire as before — streaming is additive, not a hook
-   * replacement.
-   */
-  runStream(runId: string, inputs: AnalystRunInputs, runOpts?: RegistryRunOpts): AsyncGenerator<AnalystRunEvent, void, void>;
-  private selectAnalysts;
-  private routeInput;
-}
-//#endregion
 //#region src/analyst/default-registry.d.ts
 interface DefaultAnalystRegistryOptions {
   /** Ax service for the agentic RLM kinds. Omit → only the deterministic analyst. */
   ai?: AxAIService;
+  /** Required when agentic kinds will be selected by `runExact`. */
+  aiIdentity?: ExactExecutionComponentIdentity;
   /** Required unless `ai` was created by `createAnalystAi`. */
   model?: string;
   /** Which agentic kinds to register when `ai` is present. Default = the shipped suite. */
   kinds?: readonly TraceAnalystKindSpec[];
   /** Set false to omit the deterministic behavioral analyst (default: include). */
   includeBehavioral?: boolean;
+  /** Bound deterministic trace scanning without changing the store. */
+  behavioral?: BehavioralAnalystOptions;
   /** Forwarded to the AnalystRegistry constructor (signal, tags, priorFindings). */
   registry?: AnalystRegistryOptions;
 }
 declare function buildDefaultAnalystRegistry(opts?: DefaultAnalystRegistryOptions): AnalystRegistry;
 //#endregion
-export { RawAnalystEvidenceSchema as _, AnalystRegistryOptions as a, evidenceRefsFromRawFinding as b, CreateTraceAnalystKindOpts as c, createTraceAnalystKind as d, renderPriorFindings as f, RawAnalystEvidence as g, RAW_FINDING_SCHEMA_PROMPT as h, AnalystRegistry as i, TraceAnalystGolden as l, ANALYST_SEVERITIES as m, buildDefaultAnalystRegistry as n, BudgetPolicy as o, renderUpstreamFindings as p, AnalystHooks as r, RegistryRunOpts as s, DefaultAnalystRegistryOptions as t, TraceAnalystKindSpec as u, RawAnalystFinding as v, parseRawFinding as x, RawAnalystFindingSchema as y };
-//# sourceMappingURL=default-registry-Brxr728w.d.ts.map
+export { computeTraceMetrics as C, SuboptimalSignal as S, behavioralAnalyst as _, createTraceAnalystKind as a, BehavioralTokenSequence as b, ANALYST_SEVERITIES as c, RawAnalystEvidenceSchema as d, RawAnalystFinding as f, BehavioralAnalystOptions as g, parseRawFinding as h, TraceAnalystKindSpec as i, RAW_FINDING_SCHEMA_PROMPT as l, evidenceRefsFromRawFinding as m, buildDefaultAnalystRegistry as n, renderPriorFindings as o, RawAnalystFindingSchema as p, CreateTraceAnalystKindOpts as r, renderUpstreamFindings as s, DefaultAnalystRegistryOptions as t, RawAnalystEvidence as u, deriveEfficiencyFindings as v, SuboptimalCode as x, BehavioralMetrics as y };
+//# sourceMappingURL=default-registry-PUhIVRWz.d.ts.map

package/dist/default-registry-PUhIVRWz.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"default-registry-PUhIVRWz.d.ts","names":[],"sources":["../src/trace-analyst/behavioral-metrics.ts","../src/analyst/behavioral-analyst.ts","../src/analyst/finding-signature.ts","../src/analyst/kind-factory.ts","../src/analyst/default-registry.ts"],"mappings":";;;;;;KAoBY;UAMK;EACf,MAAM;EACN;;EAEA;;EAEA,UAAU;;UAGK;;EAEf;EACA;;EAEA,gBAAgB;;EAEhB;EACA;EACA,eAAe;EACf;EACA;;EAEA;EACA;EACA,SAAS;;UAGM;EACf;EACA;EACA,sBAAsB;EACtB,uBAAuB;;;;;;iBA+DT,oBAAoB,gBAAgB,qBAAqB;;;UCzFxD;;EAEf;;EAEA;;;;;;iBA+Dc,yBACd,SAAS,mBACT;EAAQ;EAAoB;IAC3B;;iBAkCa,kBACd,UAAS,2BACR,oBAAoB;;;cCtHV;cAEA,0BAAwB,EAAA;;;GAK1B,EAAA,KAAA;KAEC,qBAAqB,EAAE,aAAa;cAiBnC,yBAAuB,EAAA;;;;;;;;;;;;;;;;;GAKzB,EAAA,KAAA;KAEC,oBAAoB,EAAE,aAAa;;;;;;cAOlC;;iBAYG,2BAA2B,SAAS,oBAAoB;iBAQxD,gBACd,cACA,OAAO,aAAa,SAAS,mCAC5B;;;;;;;UC/Bc;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA,aAAa,OAAO,uBAAuB;;;;;;EAM3C,kBACE,OAAO,oBACP,KAAK,wCACmB;;EAE1B;IAAe;IAAkB;;;EAEjC;;EAEA;;EAEA;;EAEA,MAAM;;EAEN,eAAe,KAAK,mBAAmB,KAAK,mBAAmB;;EAE/D;;EAEA;;UAGe;;EAEf,IAAI;;EAEJ,aAAa;;EAEb;;EAEA;;;;;;;;EAQA;IAAa;IAAiB;IAAiB;IAAgB,mBAAmB;;;EAElF;;;;;;;;;;iBAWc,uBACd,MAAM,sBACN,MAAM,6BACL,oBAAoB;;;;;;;;;;;;;;;iBAyUP,oBAAoB,OAAO;;iBAyB3B,uBAAuB,UAAU;;;UC3chC;;EAEf,KAAK;;EAEL,aAAa;;EAEb;;EAEA,iBAAiB;;EAEjB;;EAEA,aAAa;;EAEb,WAAW;;iBAGG,4BACd,OAAM,gCACL"}