@tangle-network/agent-eval 0.129.0 → 0.130.0

package/dist/schema-BtVldJ3T.d.ts

@@ -0,0 +1,206 @@
+//#region src/trace/schema.d.ts
+/**
+ * TraceSchema v1 — the canonical data model for agent-eval.
+ *
+ * Every score, every failure class, every pipeline in the framework is
+ * a view over this data. Shape it once, live with it.
+ *
+ * Wire-compatible with OpenTelemetry span semantics (see trace/otel.ts)
+ * but extended with agent-specific span kinds (llm, tool, retrieval,
+ * judge, sandbox) and first-class BudgetLedger / Artifact / JudgeVerdict
+ * entities that OTEL leaves as free-form attributes.
+ */
+declare const TRACE_SCHEMA_VERSION = "1.0.0";
+type RunStatus = 'running' | 'completed' | 'failed' | 'aborted';
+interface BudgetSpec {
+  tokens?: number;
+  wallMs?: number;
+  calls?: number;
+  usd?: number;
+}
+interface RunOutcome {
+  score?: number;
+  pass?: boolean;
+  failureClass?: FailureClass;
+  notes?: string;
+}
+/**
+ * Layer — optional classification in a nested build workflow.
+ * `builder`: the meta-agent editing a project (e.g. agent-builder Forge chat).
+ * `app-build`: sandbox harness that compiled + tested the generated scaffold.
+ * `app-runtime`: a run of the generated agent against a domain scenario.
+ * `meta`: any meta-eval (judge replay, correlation analysis).
+ */
+type RunLayer = 'builder' | 'app-build' | 'app-runtime' | 'meta' | 'custom';
+interface Run {
+  runId: string;
+  /**
+   * Stable identifier of the scenario being executed.
+   *
+   * Always populated on the persisted Run — but `TraceEmitter.startRun` accepts
+   * input WITHOUT this field, substituting a sensible default
+   * (`run.layer ?? run.tags?.['kind'] ?? 'runtime'`) when the caller has no
+   * curated scenario to anchor to (runtime / operator / meta-eval runs). This
+   * keeps the persisted shape unambiguous for downstream filters + aggregations
+   * while removing the boilerplate of inventing placeholder ids at the call site.
+   */
+  scenarioId: string;
+  variantId?: string;
+  datasetVersion?: string;
+  /** Git SHA of agent code at run time. */
+  codeSha?: string;
+  /** Hash of the prompt template + any system prompt. */
+  promptSha?: string;
+  /** Model id + date + system-prompt hash, concatenated. */
+  modelFingerprint?: string;
+  seed?: number;
+  /** Arbitrary environment markers (shell, docker version, tz). */
+  envFingerprint?: Record<string, string>;
+  /** Version of the redaction rules applied to this run. */
+  redactionVersion?: string;
+  /** Parent run in a nested build workflow. A builder run's children are
+   *  app-build runs; those children are app-runtime runs. */
+  parentRunId?: string;
+  /** Stable project identifier — groups runs across chats + sessions. */
+  projectId?: string;
+  /** Chat/conversation identifier within a project. */
+  chatId?: string;
+  /** Layer classification — hint for aggregation; not enforced. */
+  layer?: RunLayer;
+  startedAt: number;
+  endedAt?: number;
+  status: RunStatus;
+  outcome?: RunOutcome;
+  budget?: BudgetSpec;
+  /** Free-form labels for downstream grouping. */
+  tags?: Record<string, string>;
+}
+type SpanKind = 'agent' | 'llm' | 'tool' | 'retrieval' | 'judge' | 'sandbox' | 'custom';
+type SpanStatus = 'ok' | 'error';
+interface SpanBase {
+  spanId: string;
+  parentSpanId?: string;
+  runId: string;
+  kind: SpanKind;
+  name: string;
+  startedAt: number;
+  endedAt?: number;
+  status?: SpanStatus;
+  error?: string;
+  /** Anything not covered by typed fields. Kept deliberately free-form. */
+  attributes?: Record<string, unknown>;
+}
+interface Message {
+  role: 'system' | 'user' | 'assistant' | 'tool';
+  content: string;
+  tokens?: number;
+  /** Multi-modal content descriptors; blobs themselves live in Artifacts. */
+  images?: Array<{
+    artifactId?: string;
+    url?: string;
+    mime?: string;
+  }>;
+}
+interface LlmSpan extends SpanBase {
+  kind: 'llm';
+  model: string;
+  messages: Message[];
+  output?: string;
+  inputTokens?: number;
+  /** All generated tokens, including the reasoning subset when present. */
+  outputTokens?: number;
+  cachedTokens?: number;
+  cacheWriteTokens?: number;
+  /** Reasoning-token subset of `outputTokens`. */
+  reasoningTokens?: number;
+  costUsd?: number;
+  finishReason?: string;
+}
+interface ToolSpan extends SpanBase {
+  kind: 'tool';
+  toolName: string;
+  args: unknown;
+  /** False when the source observed the call but did not capture its arguments. */
+  argsCaptured?: boolean;
+  result?: unknown;
+  latencyMs?: number;
+}
+interface RetrievalSpan extends SpanBase {
+  kind: 'retrieval';
+  query: string;
+  hits: Array<{
+    docId: string;
+    score: number;
+    content?: string;
+  }>;
+}
+interface JudgeSpan extends SpanBase {
+  kind: 'judge';
+  judgeId: string;
+  /** Span this judgment applies to. */
+  targetSpanId: string;
+  dimension: string;
+  /** Numeric score (free-range; interpretation up to the judge). */
+  score: number;
+  rationale?: string;
+  evidence?: string;
+}
+interface SandboxSpan extends SpanBase {
+  kind: 'sandbox';
+  image?: string;
+  command?: string;
+  exitCode?: number;
+  testsTotal?: number;
+  testsPassed?: number;
+  stdoutHash?: string;
+  stderrHash?: string;
+  /** Duration in ms; the harness fills this explicitly (endedAt - startedAt may miss setup). */
+  wallMs?: number;
+}
+interface GenericSpan extends SpanBase {
+  kind: 'agent' | 'custom';
+}
+type Span = LlmSpan | ToolSpan | RetrievalSpan | JudgeSpan | SandboxSpan | GenericSpan;
+type EventKind = 'log' | 'error' | 'budget_decrement' | 'budget_breach' | 'state_mutation' | 'policy_violation' | 'redaction_applied' | 'custom';
+interface TraceEvent {
+  eventId: string;
+  runId: string;
+  spanId?: string;
+  kind: EventKind;
+  timestamp: number;
+  payload: Record<string, unknown>;
+}
+interface BudgetLedgerEntry {
+  runId: string;
+  dimension: keyof BudgetSpec;
+  limit: number;
+  consumed: number;
+  remaining: number;
+  timestamp: number;
+  breached: boolean;
+  /** Span that triggered this entry, if any. */
+  spanId?: string;
+}
+interface Artifact {
+  artifactId: string;
+  runId: string;
+  spanId?: string;
+  contentType: string;
+  sizeBytes: number;
+  /** sha256 in hex. */
+  hash: string;
+  /** External storage URL (R2, S3, filesystem path). */
+  storageUrl?: string;
+  /** Inline content for small blobs — keep under ~64KB. */
+  inlineContent?: string;
+}
+type FailureClass = 'success' | 'reasoning_error' | 'tool_selection_error' | 'tool_argument_error' | 'tool_recovery_failure' | 'hallucination' | 'instruction_following' | 'safety_refusal_miss' | 'policy_violation' | 'budget_exceeded' | 'format_drift' | 'permission_escalation' | 'pii_leak' | 'cost_overrun' | 'timeout' | 'sandbox_failure' | 'missing_user_data' | 'missing_domain_data' | 'missing_codebase_context' | 'missing_runtime_context' | 'missing_credentials' | 'missing_integration_connection' | 'missing_integration_scope' | 'integration_approval_required' | 'integration_auth_expired' | 'integration_provider_failure' | 'bad_integration_manifest' | 'unsafe_integration_write_denied' | 'stale_external_data' | 'bad_retrieval' | 'insufficient_evidence' | 'contradictory_evidence' | 'ambiguous_user_intent' | 'knowledge_readiness_blocked' | 'unknown';
+declare const FAILURE_CLASSES: readonly FailureClass[];
+declare function isLlmSpan(s: Span): s is LlmSpan;
+declare function isToolSpan(s: Span): s is ToolSpan;
+declare function isRetrievalSpan(s: Span): s is RetrievalSpan;
+declare function isJudgeSpan(s: Span): s is JudgeSpan;
+declare function isSandboxSpan(s: Span): s is SandboxSpan;
+//#endregion
+export { TraceEvent as C, isSandboxSpan as D, isRetrievalSpan as E, isToolSpan as O, ToolSpan as S, isLlmSpan as T, Span as _, FAILURE_CLASSES as a, SpanStatus as b, JudgeSpan as c, RetrievalSpan as d, Run as f, SandboxSpan as g, RunStatus as h, EventKind as i, LlmSpan as l, RunOutcome as m, BudgetLedgerEntry as n, FailureClass as o, RunLayer as p, BudgetSpec as r, GenericSpan as s, Artifact as t, Message as u, SpanBase as v, isJudgeSpan as w, TRACE_SCHEMA_VERSION as x, SpanKind as y };
+//# sourceMappingURL=schema-BtVldJ3T.d.ts.map

package/dist/schema-BtVldJ3T.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema-BtVldJ3T.d.ts","names":[],"sources":["../src/trace/schema.ts"],"mappings":";;;;;;;;;;;;cAYa;KAID;UAEK;EACf;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA,eAAe;EACf;;;;;;;;;KAUU;UAEK;EACf;;;;;;;;;;;EAWA;EACA;EACA;;EAEA;;EAEA;;EAEA;EACA;;EAEA,iBAAiB;;EAEjB;;;EAGA;;EAEA;;EAEA;;EAEA,QAAQ;EACR;EACA;EACA,QAAQ;EACR,UAAU;EACV,SAAS;;EAET,OAAO;;KAKG;KAEA;UAEK;EACf;EACA;EACA;EACA,MAAM;EACN;EACA;EACA;EACA,SAAS;EACT;;EAEA,aAAa;;UAGE;EACf;EACA;EACA;;EAEA,SAAS;IAAQ;IAAqB;IAAc;;;UAGrC,gBAAgB;EAC/B;EACA;EACA,UAAU;EACV;EACA;;EAEA;EACA;EACA;;EAEA;EACA;EACA;;UAGe,iBAAiB;EAChC;EACA;EACA;;EAEA;EACA;EACA;;UAGe,sBAAsB;EACrC;EACA;EACA,MAAM;IAAQ;IAAe;IAAe;;;UAG7B,kBAAkB;EACjC;EACA;;EAEA;EACA;;EAEA;EACA;EACA;;UAGe,oBAAoB;EACnC;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;EAEA;;UAGe,oBAAoB;EACnC;;KAGU,OAAO,UAAU,WAAW,gBAAgB,YAAY,cAAc;KAItE;UAUK;EACf;EACA;EACA;EACA,MAAM;EACN;EACA,SAAS;;UAKM;EACf;EACA,iBAAiB;EACjB;EACA;EACA;EACA;EACA;;EAEA;;UAKe;EACf;EACA;EACA;EACA;EACA;;EAEA;;EAEA;;EAEA;;KAKU;cAqCC,0BAA0B;iBAwCvB,UAAU,GAAG,OAAO,KAAK;iBAGzB,WAAW,GAAG,OAAO,KAAK;iBAG1B,gBAAgB,GAAG,OAAO,KAAK;iBAG/B,YAAY,GAAG,OAAO,KAAK;iBAG3B,cAAc,GAAG,OAAO,KAAK"}