@tangle-network/agent-eval 1.0.2 → 1.0.4

package/dist/traces.d.ts

@@ -4,8 +4,8 @@ import { a as RunRecord, c as RunSplitTag, d as RunTokenUsage, u as RunTerminalO
 import { $ as QueryTracesPage, G as ReadSpanSourceResult, H as createBoundedTraceAnalysisStore, J as TraceAnalysisStore, K as SpanSourceReader, Q as ErrorCluster, U as BoundedTraceAnalysisStoreOptions, W as ReadSpanSourceInput, X as DEFAULT_TRACE_ANALYST_BUDGETS, Y as TraceAnalysisStoreContext, Z as DatasetOverview, at as TraceAnalystSpan, ct as TraceAnalystTraceSummary, dt as ViewTraceResult, et as SearchSpanResult, it as TraceAnalystFilters, lt as ViewSpansResult, nt as SpanMatchRecord, ot as TraceAnalystSpanKind, q as TRACE_ANALYSIS_LIMITS, rt as TraceAnalystByteBudgets, st as TraceAnalystSpanStatus, tt as SearchTraceResult, ut as ViewTraceOversized } from "./types-DzCO-S1m.js";
 import { G as InMemoryRawProviderSink, J as ProviderRedactor, K as InMemoryRawProviderSinkOptions, Q as RawProviderSinkFilter, U as FileSystemRawProviderSink, W as FileSystemRawProviderSinkOptions, X as RawProviderEvent, Y as RawProviderDirection, Z as RawProviderSink, q as NoopRawProviderSink } from "./types-0KglnQ9o.js";
 import { a as RunFilter, i as InMemoryTraceStore, n as FileSystemTraceStore, o as SpanFilter, r as FileSystemTraceStoreOptions, s as TraceStore, t as EventFilter } from "./store-CMs5wGXP.js";
-import { d as BuildTraceAnalysisToolsOptions, f as TRACE_ANALYST_TOOL_NAMESPACE, h as buildTraceAnalysisToolDescriptors, m as UNTRUSTED_TRACE_TEXT, p as TraceAnalysisToolDescriptor } from "./engine-CaHaCc3W.js";
 import { a as SpanHandle, i as RunCompleteHookContext, n as CaptureWrite, o as TraceEmitter, r as RunCompleteHook, s as TraceEmitterOptions, t as CaptureErrorHandler } from "./emitter-mAXC5PSN.js";
+import { d as BuildTraceAnalysisToolsOptions, f as TRACE_ANALYST_TOOL_NAMESPACE, h as buildTraceAnalysisToolDescriptors, m as UNTRUSTED_TRACE_TEXT, p as TraceAnalysisToolDescriptor } from "./engine-CaHaCc3W.js";
 import { C as classifyKey, E as normalizeKey, _ as redactText, a as RedactionFinding, c as SafetyCategory, d as ShareSafetyVerdict, f as assessShareSafety, g as redactForShare, h as redact, i as RedactOptions, l as ShareSafetyFinding, m as emptyRedactionReport, o as RedactionProfile, p as combineVerdicts, r as REDACTION_VERSION, s as RedactionReport, t as AssessOptions, u as ShareSafetyStatus, v as shareAllowed, w as detectCredential } from "./redact-jrT0APTU.js";
 import { A as TraceInsightReadiness, B as isTraceInsightEvidence, C as TraceInsightContext, D as TraceInsightPromptInput, E as TraceInsightPanelRole, F as describeTraceInsightScope, G as AnalyzeTracesResult, H as tokenizeDomainWords, I as domainEvidencePattern, K as analyzeTraces, L as inferDomainKeywords, M as TraceInsightTask, N as buildTraceInsightContext, O as TraceInsightQualityGate, P as buildTraceInsightPrompt, R as isTraceInsightArtifact, S as TraceInsightAttribution, T as TraceInsightFinding, U as AnalyzeTracesInput, V as scoreTraceInsightReadiness, W as AnalyzeTracesOptions, _ as ToolSpansToTraceAnalysisStoreOptions, a as captureFetchToRawSink, b as OtlpSpan, c as ExtractedUsage, d as extractUsageFromSse, f as OtlpFileTraceStore, g as spanRecordsToTraceAnalysisStore, h as otlpTextToTraceAnalysisStore, i as CaptureFetchOptions, j as TraceInsightSuite, k as TraceInsightQuestion, l as SseUsageMode, m as SpanRecordsToTraceAnalysisStoreOptions, n as toolSpansToTraceAnalysisStore, o as OtlpFlatLine, p as OtlpFileTraceStoreOptions, r as CaptureFetchContext, s as ExtractUsageFromSseOptions, t as ToolTraceMissingError, u as extractUsage, v as OtlpExport, w as TraceInsightEvidence, x as exportRunAsOtlp, y as OtlpResourceSpans, z as isTraceInsightAttribution } from "./store-tool-spans-lWf6INTH.js";
 import { a as RunIntegrityReport, i as RunIntegrityIssueCode, n as RunIntegrityExpectations, o as assertRunCaptured, r as RunIntegrityIssue, t as RunIntegrityError } from "./integrity-BAdVzT8u.js";

package/dist/wire/index.d.ts

@@ -1,7 +1,7 @@
 import { m as CustomTokenPricing, o as CostLedgerHandle } from "../cost-ledger-Czro9wmo.js";
 import { x as ChatClient } from "../types-0KglnQ9o.js";
 import { s as TraceStore } from "../store-CMs5wGXP.js";
-import { m as FeedbackTrajectoryStore } from "../feedback-trajectory-Ck7K7AhK.js";
+import { m as FeedbackTrajectoryStore } from "../feedback-trajectory-BgomnFOe.js";
 import { z } from "zod";
 import { ServerType } from "@hono/node-server";
 import { Hono } from "hono";
@@ -267,10 +267,10 @@ export declare const FeedbackTrajectorySchema: z.ZodObject<{
     metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
   }, z.core.$strip>>;
   split: z.ZodOptional<z.ZodEnum<{
-    dev: "dev";
-    holdout: "holdout";
     train: "train";
+    dev: "dev";
     test: "test";
+    holdout: "holdout";
   }>>;
   tags: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
   createdAt: z.ZodString;

package/docs/concepts.md

@@ -56,6 +56,28 @@ Paired lift, failure clusters, contamination checks, and outcome associations re
 Run records, scenarios, judge scores, statistics, and release decisions belong here because they work without an agent runtime.
 Agent sessions, worker coordination, sandbox execution, and runtime-specific profiles belong in `agent-runtime`.
 
+## Product integration scenarios
+
+`runSandboxScenario()` from `@tangle-network/agent-eval/campaign` runs a
+`HarnessScenario`: setup, named steps, expectations and a time budget.
+The product supplies transport callbacks and keeps its SDK, credentials and
+resource authorization. Steps receive one scope for measurement, assertions,
+condition polling and retirement registration. Register retirement immediately
+after allocation; callbacks use the scope signal for workload cancellation.
+Retirements run in reverse order even after setup or workload failure, and a
+failed retirement remains a failed assertion in the report.
+
+Use `scope.resource({ name, create, ready, retire, onCreate })` to measure acquisition,
+register retirement before readiness, and retain the allocated resource when readiness fails.
+A step marked `advisory` retains its failure timing while required expectations still gate
+the report. Unmarked steps and all setup/cleanup measurements remain required.
+
+Pass the product's existing `measure` adapter to retain its server timing
+ledgers. A poll returns `undefined` when its budget expires; the scenario must
+assert the condition when exhaustion is a failure. The scope's budget signal
+is cooperative: callbacks must pass it through to their transport. Campaigns
+remain the owner of scenario scheduling, retries, artifacts and run reports.
+
 ## Measuring a complete profile
 
 Use the profile improvement functions from `/contract` when a host owns immutable agent-profile snapshots.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-eval",
-  "version": "1.0.2",
+  "version": "1.0.4",
   "description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.",
   "homepage": "https://github.com/tangle-network/agent-eval#readme",
   "repository": {

package/dist/sandbox-harness-B3Kx1fOA.d.ts

@@ -1,68 +0,0 @@
-interface HarnessConfig {
-  /** Setup command (e.g. "pnpm install"). Non-zero exit fails the run. */
-  setupCommand?: string;
-  /** Run command (e.g. "pnpm build"). */
-  runCommand?: string;
-  /** Test command (e.g. "pnpm test --run"). Drives the test count + pass count. */
-  testCommand?: string;
-  /** Absolute cwd for the subprocess driver. Ignored by docker driver. */
-  cwd?: string;
-  /** Max wall-clock per phase in ms. Default 10 minutes. */
-  timeoutMs?: number;
-  /**
-   * Cap on captured stdout+stderr bytes per phase. A runaway process can
-   * otherwise grow the in-memory buffer without bound. Once hit, further
-   * output is dropped and `outputTruncated` is set. Default 16 MiB.
-   */
-  maxOutputBytes?: number;
-  /** Image for the docker driver. */
-  image?: string;
-  /** Extra env vars (validated; shell-escaped). */
-  env?: Record<string, string>;
-  /** Parser for the test output — maps stdout/stderr/exit code → pass count. */
-  testParser?: TestOutputParser;
-}
-interface TestOutputParser {
-  id: string;
-  parse(stdout: string, stderr: string, exitCode: number): {
-    testsTotal: number;
-    testsPassed: number;
-  } | undefined;
-}
-interface SandboxResult {
-  phase: 'setup' | 'run' | 'test';
-  exitCode: number;
-  stdout: string;
-  stderr: string;
-  wallMs: number;
-  testsTotal?: number;
-  testsPassed?: number;
-  /**
-   * True when the process was killed because it exceeded `timeoutMs`. A
-   * SIGKILLed child can still close with exit code 0; callers MUST treat
-   * a timed-out phase as a hard failure regardless of `exitCode`, never
-   * as a pass. `undefined`/`false` means the process completed on its own.
-   */
-  killedByTimeout?: boolean;
-  /**
-   * True when captured stdout/stderr hit `maxOutputBytes` and further
-   * output was dropped. The result is still returned (the process was
-   * not killed for this), but downstream parsers see truncated text.
-   */
-  outputTruncated?: boolean;
-}
-interface SandboxDriver {
-  id: string;
-  exec(phase: SandboxResult['phase'], command: string, config: HarnessConfig): Promise<SandboxResult>;
-}
-interface SandboxHarnessResult {
-  passed: boolean;
-  setup?: SandboxResult;
-  run?: SandboxResult;
-  test?: SandboxResult;
-  totalWallMs: number;
-  /** Final score — 0 when no tests; otherwise testsPassed/testsTotal. */
-  score: number;
-}
-export { SandboxDriver as n, SandboxHarnessResult as r, HarnessConfig as t };
-//# sourceMappingURL=sandbox-harness-B3Kx1fOA.d.ts.map

package/dist/sandbox-harness-B3Kx1fOA.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"sandbox-harness-B3Kx1fOA.d.ts","names":[],"sources":["../src/sandbox-harness.ts"],"mappings":"A,UAoBiB;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;;;;;EAMA;;EAEA;;EAEA,MAAM;;EAEN,aAAa;;UAGE;EACf;EACA,MACE,gBACA,gBACA;IACG;IAAoB;;;UAGV;EACf;EACA;EACA;EACA;EACA;EACA;EACA;;;;;;;EAOA;;;;;;EAMA;;UAGe;EACf;EACA,KACE,OAAO,wBACP,iBACA,QAAQ,gBACP,QAAQ;;UA0II;EACf;EACA,QAAQ;EACR,MAAM;EACN,OAAO;EACP;;EAEA"}