@tangle-network/agent-eval 0.53.0 → 0.55.0

package/dist/{store-Db2Bv8Cf.d.ts → schema-m0gsnbt3.d.ts}

@@ -196,102 +196,4 @@ declare function isRetrievalSpan(s: Span): s is RetrievalSpan;
 declare function isJudgeSpan(s: Span): s is JudgeSpan;
 declare function isSandboxSpan(s: Span): s is SandboxSpan;
 
-interface RunFilter {
-    scenarioId?: string;
-    variantId?: string;
-    status?: RunStatus;
-    since?: number;
-    until?: number;
-    tag?: {
-        key: string;
-        value: string;
-    };
-    parentRunId?: string;
-    projectId?: string;
-    chatId?: string;
-    layer?: RunLayer;
-}
-interface SpanFilter {
-    runId?: string;
-    parentSpanId?: string;
-    kind?: SpanKind;
-    name?: string;
-    toolName?: string;
-    judgeId?: string;
-    since?: number;
-    until?: number;
-}
-interface EventFilter {
-    runId?: string;
-    spanId?: string;
-    kind?: EventKind;
-    since?: number;
-    until?: number;
-}
-interface TraceStore {
-    appendRun(run: Run): Promise<void>;
-    updateRun(runId: string, patch: Partial<Run>): Promise<void>;
-    appendSpan(span: Span): Promise<void>;
-    updateSpan(spanId: string, patch: Partial<Span>): Promise<void>;
-    appendEvent(event: TraceEvent): Promise<void>;
-    appendArtifact(artifact: Artifact): Promise<void>;
-    appendBudgetEntry(entry: BudgetLedgerEntry): Promise<void>;
-    getRun(runId: string): Promise<Run | undefined>;
-    listRuns(filter?: RunFilter): Promise<Run[]>;
-    spans(filter?: SpanFilter): Promise<Span[]>;
-    events(filter?: EventFilter): Promise<TraceEvent[]>;
-    budget(runId: string): Promise<BudgetLedgerEntry[]>;
-    artifacts(runId: string): Promise<Artifact[]>;
-}
-declare class InMemoryTraceStore implements TraceStore {
-    private runs;
-    private allSpans;
-    private allEvents;
-    private allArtifacts;
-    private allBudget;
-    appendRun(run: Run): Promise<void>;
-    updateRun(runId: string, patch: Partial<Run>): Promise<void>;
-    appendSpan(span: Span): Promise<void>;
-    updateSpan(spanId: string, patch: Partial<Span>): Promise<void>;
-    appendEvent(event: TraceEvent): Promise<void>;
-    appendArtifact(artifact: Artifact): Promise<void>;
-    appendBudgetEntry(entry: BudgetLedgerEntry): Promise<void>;
-    getRun(runId: string): Promise<Run | undefined>;
-    listRuns(filter?: RunFilter): Promise<Run[]>;
-    spans(filter?: SpanFilter): Promise<Span[]>;
-    events(filter?: EventFilter): Promise<TraceEvent[]>;
-    budget(runId: string): Promise<BudgetLedgerEntry[]>;
-    artifacts(runId: string): Promise<Artifact[]>;
-}
-interface FileSystemTraceStoreOptions {
-    dir: string;
-    /** Roll over NDJSON files when they exceed this size in bytes. Default 32 MB. */
-    maxBytes?: number;
-}
-declare class FileSystemTraceStore implements TraceStore {
-    private dir;
-    private maxBytes;
-    /** Lazy in-memory index for queries — populated on first read. */
-    private index?;
-    private loaded;
-    constructor(options: FileSystemTraceStoreOptions);
-    private ensureDir;
-    private append;
-    private insertInto;
-    private load;
-    appendRun(run: Run): Promise<void>;
-    updateRun(runId: string, patch: Partial<Run>): Promise<void>;
-    appendSpan(span: Span): Promise<void>;
-    updateSpan(spanId: string, patch: Partial<Span>): Promise<void>;
-    appendEvent(event: TraceEvent): Promise<void>;
-    appendArtifact(artifact: Artifact): Promise<void>;
-    appendBudgetEntry(entry: BudgetLedgerEntry): Promise<void>;
-    getRun(runId: string): Promise<Run | undefined>;
-    listRuns(filter?: RunFilter): Promise<Run[]>;
-    spans(filter?: SpanFilter): Promise<Span[]>;
-    events(filter?: EventFilter): Promise<TraceEvent[]>;
-    budget(runId: string): Promise<BudgetLedgerEntry[]>;
-    artifacts(runId: string): Promise<Artifact[]>;
-}
-
-export { type Artifact as A, type BudgetLedgerEntry as B, type EventKind as E, type FailureClass as F, type GenericSpan as G, InMemoryTraceStore as I, type JudgeSpan as J, type LlmSpan as L, type Message as M, type Run as R, type Span as S, type TraceStore as T, type ToolSpan as a, type TraceEvent as b, type RunOutcome as c, type SpanKind as d, type RetrievalSpan as e, type SandboxSpan as f, type BudgetSpec as g, type RunFilter as h, type EventFilter as i, FAILURE_CLASSES as j, FileSystemTraceStore as k, type FileSystemTraceStoreOptions as l, type RunLayer as m, type RunStatus as n, type SpanBase as o, type SpanFilter as p, type SpanStatus as q, TRACE_SCHEMA_VERSION as r, isJudgeSpan as s, isLlmSpan as t, isRetrievalSpan as u, isSandboxSpan as v, isToolSpan as w };
+export { type Artifact as A, type BudgetLedgerEntry as B, type EventKind as E, type FailureClass as F, type GenericSpan as G, type JudgeSpan as J, type LlmSpan as L, type Message as M, type Run as R, type Span as S, type ToolSpan as T, type TraceEvent as a, type RunOutcome as b, type SpanKind as c, type RetrievalSpan as d, type SandboxSpan as e, type RunStatus as f, type RunLayer as g, type BudgetSpec as h, FAILURE_CLASSES as i, type SpanBase as j, type SpanStatus as k, TRACE_SCHEMA_VERSION as l, isJudgeSpan as m, isLlmSpan as n, isRetrievalSpan as o, isSandboxSpan as p, isToolSpan as q };

package/dist/store-CKUAgsJz.d.ts

@@ -0,0 +1,101 @@
+import { R as Run, S as Span, a as TraceEvent, A as Artifact, B as BudgetLedgerEntry, f as RunStatus, g as RunLayer, c as SpanKind, E as EventKind } from './schema-m0gsnbt3.js';
+
+interface RunFilter {
+    scenarioId?: string;
+    variantId?: string;
+    status?: RunStatus;
+    since?: number;
+    until?: number;
+    tag?: {
+        key: string;
+        value: string;
+    };
+    parentRunId?: string;
+    projectId?: string;
+    chatId?: string;
+    layer?: RunLayer;
+}
+interface SpanFilter {
+    runId?: string;
+    parentSpanId?: string;
+    kind?: SpanKind;
+    name?: string;
+    toolName?: string;
+    judgeId?: string;
+    since?: number;
+    until?: number;
+}
+interface EventFilter {
+    runId?: string;
+    spanId?: string;
+    kind?: EventKind;
+    since?: number;
+    until?: number;
+}
+interface TraceStore {
+    appendRun(run: Run): Promise<void>;
+    updateRun(runId: string, patch: Partial<Run>): Promise<void>;
+    appendSpan(span: Span): Promise<void>;
+    updateSpan(spanId: string, patch: Partial<Span>): Promise<void>;
+    appendEvent(event: TraceEvent): Promise<void>;
+    appendArtifact(artifact: Artifact): Promise<void>;
+    appendBudgetEntry(entry: BudgetLedgerEntry): Promise<void>;
+    getRun(runId: string): Promise<Run | undefined>;
+    listRuns(filter?: RunFilter): Promise<Run[]>;
+    spans(filter?: SpanFilter): Promise<Span[]>;
+    events(filter?: EventFilter): Promise<TraceEvent[]>;
+    budget(runId: string): Promise<BudgetLedgerEntry[]>;
+    artifacts(runId: string): Promise<Artifact[]>;
+}
+declare class InMemoryTraceStore implements TraceStore {
+    private runs;
+    private allSpans;
+    private allEvents;
+    private allArtifacts;
+    private allBudget;
+    appendRun(run: Run): Promise<void>;
+    updateRun(runId: string, patch: Partial<Run>): Promise<void>;
+    appendSpan(span: Span): Promise<void>;
+    updateSpan(spanId: string, patch: Partial<Span>): Promise<void>;
+    appendEvent(event: TraceEvent): Promise<void>;
+    appendArtifact(artifact: Artifact): Promise<void>;
+    appendBudgetEntry(entry: BudgetLedgerEntry): Promise<void>;
+    getRun(runId: string): Promise<Run | undefined>;
+    listRuns(filter?: RunFilter): Promise<Run[]>;
+    spans(filter?: SpanFilter): Promise<Span[]>;
+    events(filter?: EventFilter): Promise<TraceEvent[]>;
+    budget(runId: string): Promise<BudgetLedgerEntry[]>;
+    artifacts(runId: string): Promise<Artifact[]>;
+}
+interface FileSystemTraceStoreOptions {
+    dir: string;
+    /** Roll over NDJSON files when they exceed this size in bytes. Default 32 MB. */
+    maxBytes?: number;
+}
+declare class FileSystemTraceStore implements TraceStore {
+    private dir;
+    private maxBytes;
+    /** Lazy in-memory index for queries — populated on first read. */
+    private index?;
+    private loaded;
+    constructor(options: FileSystemTraceStoreOptions);
+    private ensureDir;
+    private append;
+    private insertInto;
+    private load;
+    appendRun(run: Run): Promise<void>;
+    updateRun(runId: string, patch: Partial<Run>): Promise<void>;
+    appendSpan(span: Span): Promise<void>;
+    updateSpan(spanId: string, patch: Partial<Span>): Promise<void>;
+    appendEvent(event: TraceEvent): Promise<void>;
+    appendArtifact(artifact: Artifact): Promise<void>;
+    appendBudgetEntry(entry: BudgetLedgerEntry): Promise<void>;
+    getRun(runId: string): Promise<Run | undefined>;
+    listRuns(filter?: RunFilter): Promise<Run[]>;
+    spans(filter?: SpanFilter): Promise<Span[]>;
+    events(filter?: EventFilter): Promise<TraceEvent[]>;
+    budget(runId: string): Promise<BudgetLedgerEntry[]>;
+    artifacts(runId: string): Promise<Artifact[]>;
+}
+
+export { type EventFilter as E, FileSystemTraceStore as F, InMemoryTraceStore as I, type RunFilter as R, type SpanFilter as S, type TraceStore as T, type FileSystemTraceStoreOptions as a };

package/dist/{summary-report-B7gNRX-r.d.ts → summary-report-DLxh4yWk.d.ts}

@@ -1,5 +1,5 @@
-import { R as RunRecord } from './run-record-BGY6bHRh.js';
-import { F as FailureClusterReport } from './failure-cluster-Cw65_5FY.js';
+import { R as RunRecord } from './run-record-etiCMsUq.js';
+import { F as FailureClusterReport } from './failure-cluster-CL7IVgkJ.js';
 
 /**
  * HeldOutGate — first-class held-out paired-delta promotion gate.

package/dist/{test-graded-scenario-B2kWEdh9.d.ts → test-graded-scenario-BdVaPyHT.d.ts}

@@ -1,5 +1,6 @@
-import { T as TraceEmitter } from './emitter-DP_cSSiw.js';
-import { R as Run, F as FailureClass, T as TraceStore } from './store-Db2Bv8Cf.js';
+import { T as TraceEmitter } from './emitter-DEZwY14K.js';
+import { R as Run, F as FailureClass } from './schema-m0gsnbt3.js';
+import { T as TraceStore } from './store-CKUAgsJz.js';
 
 /**
  * SandboxHarness — executes a scenario in an isolated environment and

package/dist/traces.d.ts

@@ -1,12 +1,13 @@
 import { N as NotFoundError, R as ReplayError } from './errors-mje_cKOs.js';
 import { R as RawProviderSink, d as RawProviderEvent } from './raw-provider-sink-C46HDghv.js';
 export { F as FileSystemRawProviderSink, a as FileSystemRawProviderSinkOptions, I as InMemoryRawProviderSink, b as InMemoryRawProviderSinkOptions, N as NoopRawProviderSink, P as ProviderRedactor, c as RawProviderDirection, e as RawProviderSinkFilter, f as defaultProviderRedactor, p as providerFromBaseUrl } from './raw-provider-sink-C46HDghv.js';
-import { R as RunCompleteHook, a as RunCompleteHookContext } from './emitter-DP_cSSiw.js';
-export { S as SpanHandle, T as TraceEmitter, b as TraceEmitterOptions, l as llmSpanFromProvider } from './emitter-DP_cSSiw.js';
-export { b as RunIntegrityError, R as RunIntegrityExpectations, c as RunIntegrityIssue, d as RunIntegrityIssueCode, a as RunIntegrityReport, e as assertRunCaptured, t as throwIfRunIncomplete } from './integrity-CTDhR1Sg.js';
-import { T as TraceStore } from './store-Db2Bv8Cf.js';
-export { A as Artifact, B as BudgetLedgerEntry, g as BudgetSpec, i as EventFilter, E as EventKind, j as FAILURE_CLASSES, F as FailureClass, k as FileSystemTraceStore, l as FileSystemTraceStoreOptions, G as GenericSpan, I as InMemoryTraceStore, J as JudgeSpan, L as LlmSpan, M as Message, e as RetrievalSpan, R as Run, h as RunFilter, m as RunLayer, c as RunOutcome, n as RunStatus, f as SandboxSpan, S as Span, o as SpanBase, p as SpanFilter, d as SpanKind, q as SpanStatus, r as TRACE_SCHEMA_VERSION, a as ToolSpan, b as TraceEvent, s as isJudgeSpan, t as isLlmSpan, u as isRetrievalSpan, v as isSandboxSpan, w as isToolSpan } from './store-Db2Bv8Cf.js';
-export { a as aggregateLlm, b as argHash, g as groupBy, j as judgeSpans, l as llmSpans, r as runFailureClass, c as runsForScenario, t as toolSpans } from './query-DODUYdPg.js';
+import { R as RunCompleteHook, a as RunCompleteHookContext } from './emitter-DEZwY14K.js';
+export { S as SpanHandle, T as TraceEmitter, b as TraceEmitterOptions, l as llmSpanFromProvider } from './emitter-DEZwY14K.js';
+export { b as RunIntegrityError, R as RunIntegrityExpectations, c as RunIntegrityIssue, d as RunIntegrityIssueCode, a as RunIntegrityReport, e as assertRunCaptured, t as throwIfRunIncomplete } from './integrity-CfXjSqEv.js';
+import { T as TraceStore } from './store-CKUAgsJz.js';
+export { E as EventFilter, F as FileSystemTraceStore, a as FileSystemTraceStoreOptions, I as InMemoryTraceStore, R as RunFilter, S as SpanFilter } from './store-CKUAgsJz.js';
+export { a as aggregateLlm, b as argHash, g as groupBy, j as judgeSpans, l as llmSpans, r as runFailureClass, c as runsForScenario, t as toolSpans } from './query-CqTxMwDw.js';
+export { A as Artifact, B as BudgetLedgerEntry, h as BudgetSpec, E as EventKind, i as FAILURE_CLASSES, F as FailureClass, G as GenericSpan, J as JudgeSpan, L as LlmSpan, M as Message, d as RetrievalSpan, R as Run, g as RunLayer, b as RunOutcome, f as RunStatus, e as SandboxSpan, S as Span, j as SpanBase, c as SpanKind, k as SpanStatus, l as TRACE_SCHEMA_VERSION, T as ToolSpan, a as TraceEvent, m as isJudgeSpan, n as isLlmSpan, o as isRetrievalSpan, p as isSandboxSpan, q as isToolSpan } from './schema-m0gsnbt3.js';
 import { AxAIService, AxFunction } from '@ax-llm/ax';
 import { T as TraceAnalysisStore, f as TraceAnalystFilters, a as DatasetOverview, Q as QueryTracesPage, l as ViewTraceResult, V as ViewSpansResult, b as SearchTraceResult, S as SearchSpanResult } from './store-CJbzDxZ2.js';
 export { D as DEFAULT_TRACE_ANALYST_BUDGETS, c as SpanMatchRecord, d as TRACE_ANALYST_TRUNCATION_MARKER_PREFIX, e as TraceAnalystByteBudgets, g as TraceAnalystSpan, h as TraceAnalystSpanKind, i as TraceAnalystSpanStatus, j as TraceAnalystTraceSummary, k as ViewTraceOversized } from './store-CJbzDxZ2.js';

package/dist/{trajectory-CnoBo-JY.d.ts → trajectory-GEdXJCL5.d.ts}

@@ -1,4 +1,5 @@
-import { S as Span, b as TraceEvent, T as TraceStore } from './store-Db2Bv8Cf.js';
+import { S as Span, a as TraceEvent } from './schema-m0gsnbt3.js';
+import { T as TraceStore } from './store-CKUAgsJz.js';
 
 /**
  * Trajectory — ordered, structured view over a run's spans.

package/dist/{types-Dbj5gu8n.d.ts → types-BgrxOJSf.d.ts}

@@ -248,6 +248,23 @@ interface CampaignCostMeter {
  *  training scenarios unless explicitly opted in). */
 type LabeledScenarioSource = 'production-trace' | 'eval-run' | 'manual' | 'red-team' | 'synthetic';
 type RedactionStatus = 'raw' | 'redacted-pii' | 'redacted-secrets' | 'fully-redacted';
+/** How much a label can be trusted to evaluate against — the gold-admission
+ *  gate. Strictly ordered: a record qualifies for a `minTrust` filter when its
+ *  trust rank is >= the requested rank.
+ *
+ *   - `unverified`      — label is a heuristic (e.g. raw outcome success/fail).
+ *                          Fine as corpus; MUST NOT enter a gold set that lift
+ *                          numbers are computed against.
+ *   - `verified-signal` — an external signal confirmed the outcome (PR merged,
+ *                          tests green, user did not retry, downstream check).
+ *   - `human-rated`     — a human explicitly rated or corrected the artifact.
+ *
+ *  Absent on a write ⇒ treated as `unverified` (fail-closed: a writer must
+ *  explicitly assert trust to make a record gold-eligible — it never happens
+ *  by accident). */
+type LabelTrust = 'unverified' | 'verified-signal' | 'human-rated';
+/** Ordinal rank for a label-trust tier; absent ⇒ `unverified` (rank 0). */
+declare function labelTrustRank(trust: LabelTrust | undefined): number;
 /** @experimental Required-provenance write. The store rejects writes that
  *  lack provenance — a default-on flywheel without provenance is the
  *  data-poisoning vector flagged in the alignment review. */
@@ -259,6 +276,11 @@ interface LabeledScenarioWrite<TScenario extends Scenario = Scenario, TArtifact
     sourceVersionHash: string;
     capturedAt: string;
     redactionStatus: RedactionStatus;
+    /** Gold-admission trust tier. Absent ⇒ `unverified` (fail-closed): the
+     *  record is corpus, never gold. A writer must explicitly assert
+     *  `verified-signal` or `human-rated` to make it eligible for a gold
+     *  sample. See {@link LabelTrust}. */
+    labelTrust?: LabelTrust;
     /** Optional per-source rate-limit bucket key (e.g., the tenant id). */
     rateLimitBucket?: string;
 }
@@ -282,6 +304,11 @@ interface LabeledScenarioSampleArgs {
         source?: LabeledScenarioSource | LabeledScenarioSource[];
         minComposite?: number;
         maxComposite?: number;
+        /** Gold gate: only records whose trust rank is >= this tier are
+         *  returned. `sample({ split: 'test', minTrust: 'verified-signal' })` is
+         *  the canonical "give me the gold set" call. Absent ⇒ no trust gate
+         *  (corpus-level read). */
+        minTrust?: LabelTrust;
     };
 }
 interface LabeledScenarioStore {
@@ -291,6 +318,9 @@ interface LabeledScenarioStore {
         train: number;
         test: number;
         bySource: Record<string, number>;
+        /** Count by trust tier — tells the flywheel how much gold it has
+         *  accumulated vs. raw corpus. */
+        byTrust: Record<LabelTrust, number>;
     }>;
 }
 interface CampaignCellResult<TArtifact> {
@@ -372,4 +402,4 @@ interface CampaignResult<TArtifact = unknown, TScenario extends Scenario = Scena
     scenarios: Array<Pick<TScenario, 'id' | 'kind'>>;
 }
 
-export type { CodeSurface as C, DispatchFn as D, Gate as G, ImprovementDriver as I, JudgeScore as J, LabeledScenarioStore as L, MutableSurface as M, OptimizerConfig as O, ProposeContext as P, RedactionStatus as R, Scenario as S, TraceSpan as T, JudgeConfig as a, DispatchContext as b, LabeledScenarioWrite as c, LabeledScenarioSampleArgs as d, LabeledScenarioRecord as e, CampaignAggregates as f, CampaignArtifactWriter as g, CampaignCellResult as h, CampaignCostMeter as i, CampaignResult as j, CampaignTraceWriter as k, GateContext as l, GateDecision as m, GateResult as n, GenerationCandidate as o, GenerationRecord as p, JudgeAggregate as q, JudgeDimension as r, LabeledScenarioSource as s, Mutator as t, ScenarioAggregate as u, SessionScript as v };
+export { type CodeSurface as C, type DispatchFn as D, type Gate as G, type ImprovementDriver as I, type JudgeScore as J, type LabeledScenarioStore as L, type MutableSurface as M, type OptimizerConfig as O, type ProposeContext as P, type RedactionStatus as R, type Scenario as S, type TraceSpan as T, type JudgeConfig as a, type DispatchContext as b, type LabeledScenarioWrite as c, type LabeledScenarioSampleArgs as d, type LabeledScenarioRecord as e, type LabelTrust as f, type CampaignAggregates as g, type CampaignArtifactWriter as h, type CampaignCellResult as i, type CampaignCostMeter as j, type CampaignResult as k, type CampaignTraceWriter as l, type GateContext as m, type GateDecision as n, type GateResult as o, type GenerationCandidate as p, type GenerationRecord as q, type JudgeAggregate as r, type JudgeDimension as s, type LabeledScenarioSource as t, type Mutator as u, type ScenarioAggregate as v, type SessionScript as w, labelTrustRank as x };

package/dist/wire/index.d.ts

@@ -1,12 +1,13 @@
-import { F as FeedbackTrajectoryStore } from '../feedback-trajectory-BSxqEpu7.js';
-import { T as TraceStore } from '../store-Db2Bv8Cf.js';
+import { F as FeedbackTrajectoryStore } from '../feedback-trajectory-DpUmE90J.js';
+import { T as TraceStore } from '../store-CKUAgsJz.js';
 import { z } from 'zod';
 import { OpenAPIObject } from 'openapi3-ts/oas31';
 import * as hono_types from 'hono/types';
 import { ServerType } from '@hono/node-server';
 import { Hono } from 'hono';
-import '../control-runtime-BZ_lVLYW.js';
-import '../emitter-DP_cSSiw.js';
+import '../control-runtime-DuFBYg7A.js';
+import '../emitter-DEZwY14K.js';
+import '../schema-m0gsnbt3.js';
 import '../dataset-BlwAtYYf.js';
 import '../errors-mje_cKOs.js';
 

package/docs/pilot/README.md

@@ -0,0 +1,62 @@
+# Pilot Kit — customer handoff materials
+
+What's here, in order of use:
+
+| File | For | When |
+|---|---|---|
+| [one-pager.md](./one-pager.md) | Customer's first read | Send as initial pitch — what they get, why it's different, what it looks like, what it costs. Now includes intake-paths matrix for non-Tangle customers (LangChain / LlamaIndex / Anthropic SDK / OpenAI Assistants / OpenRouter / vLLM / Ollama / custom). |
+| [integration-tangle-stack.md](./integration-tangle-stack.md) | Customer's engineer (Tangle-stack customers) | Send after one-pager when they want to see the code; full integration walkthrough for the canonical Tangle stack (sandbox + tcloud) |
+| [integration-foreign-stack.md](./integration-foreign-stack.md) | Customer's engineer (non-Tangle customers) | Send after one-pager when they're on OTel, LangChain, LlamaIndex, Anthropic SDK, OpenAI Assistants, OpenRouter, vLLM, Ollama, or custom. Covers every path. |
+| [sample-insight-report.json](./sample-insight-report.json) | Customer's team meeting | Concrete JSON they can show to demonstrate value pre-integration |
+| [customer-checklist.md](./customer-checklist.md) | Pre-onboarding-call | Send 48h before the call; ensures the 90min slot is productive. Provider-agnostic — works for any stack. |
+
+## How to use this kit
+
+**For a Tangle customer asking for it RIGHT NOW:**
+
+1. Reply with the one-pager (`one-pager.md`) inline + the sample InsightReport (`sample-insight-report.json`) attached. Their senior engineer reads this and decides if it's worth a call.
+2. If they say yes, send the integration guide (`integration-tangle-stack.md`) + the checklist (`customer-checklist.md`). Schedule a 90-minute onboarding call.
+3. On the call: walk through the integration, run a live `analyzeRuns()` against their existing sandbox sessions, render the deterministic packet, fire one small `selfImprove` cycle. By the end of the call they have a working pilot.
+
+**For Drew handling the conversation himself:**
+
+The whole kit is written in our voice (technical, direct, no marketing fluff). You can paste sections directly into Slack / email / a customer call. The one-pager is meant to read as YOUR pitch, not a generic SaaS handout.
+
+## What this kit assumes
+
+- Customer is on the Tangle stack (sandbox + tcloud) OR emits OTel traces
+- Customer has an agent with a clear system-prompt addendum we can optimize
+- Customer has at least 20 scenarios their agent handles
+- Customer is willing to set a `maxUsd` budget for closed-loop campaigns
+
+If any of those don't apply, the one-pager still works as a positioning piece. The integration doc gets adapted on the call.
+
+## Where this maps in the substrate
+
+- Substrate version: `@tangle-network/agent-eval@0.53.0` (npm), `agent-eval-rpc@0.53.0` (PyPI)
+- agent-runtime version: `@tangle-network/agent-runtime@0.29.0`
+- Key APIs: `fromTangleSandbox`, `fromOtelSpans`, `analyzeRuns`, `selfImprove`, `gepaDriver`, `defaultProductionGate`, `openAutoPr`
+- All ship today; no version-blocking dependencies
+
+## What this kit doesn't yet do
+
+- No `npx @tangle-network/intelligence demo` command shipped yet (queued #115 — extend existing `tangle-intel` CLI in ADC with customer-zero-touch subcommands `init` / `demo` / `report` / `improve`)
+- No `staging-intelligence.tangle.tools` live yet (queued #116 — matches existing `staging-{product}.tangle.tools` precedent like sandbox)
+- No live demo video (queued #117 — recorded against legal-agent canonical real data)
+- No screenshot dashboard (gated on Gate 2 task #109 — ADC intelligence frontend renders canonical InsightReport)
+- No published case study with named numbers (Gate 3 task #112 — after first pilot completes 4+ cycles)
+
+## Architectural decisions baked into this kit
+
+- **Customer-facing CLI is `@tangle-network/intelligence`** (binary `tangle-intel`), NOT `agent-eval`. `agent-eval` is the substrate package; `intelligence` is the customer product that wraps it. The CLI already exists at `services/intelligence/src/cli/` in agent-dev-container — we extend it with `init` / `demo` / `report` / `improve` subcommands per task #115.
+- **Hosted URL is `staging-intelligence.tangle.tools`** matching `staging-sandbox.tangle.tools` precedent. Production becomes `intelligence.tangle.tools` once Gate 2/3 close.
+- **`agent-eval` mentioned only when customer wants direct programmatic access** (not the default path). 90%+ of customers stay at the CLI + hosted dashboard layer.
+
+For the FIRST pilot conversation, the JSON sample is the dashboard substitute. After Gate 2 lands we replace it with live screenshots.
+
+## Update cadence
+
+This kit gets updated each time:
+- A substrate version ships that customers should know about
+- A real pilot completes and we have a case study to add
+- A customer gives feedback that re-shapes how we pitch

package/docs/pilot/customer-checklist.md

@@ -0,0 +1,90 @@
+# Pre-onboarding checklist — what to have ready
+
+Send this to the customer 48h before the onboarding call. If they show up to the call having done this, the 90-minute slot ends with a working pilot.
+
+## What we need from you before the call
+
+### Credentials
+
+- [ ] **LLM provider API key** — tcloud key, OpenRouter key, OpenAI key, Anthropic key, or any OpenAI-compat router endpoint
+- [ ] **GitHub token** with PR-write access to your agent repo (optional — required only if you want auto-PR promotion on green gate decisions)
+- [ ] **Sandbox session access** (Tangle stack customers only) — read access to the session IDs we'll analyze
+
+### Data
+
+- [ ] **Trace data** — ONE of:
+  - Tangle sandbox session IDs (we use `fromTangleSandbox`)
+  - OTel spans dumped as JSONL (we use `fromOtelSpans`)
+  - Multi-rater feedback table (CSV with runId / rater / score, we use `fromFeedbackTable`)
+  - LangChain / LlamaIndex / OpenAI Assistants trace export (we use the corresponding adapter)
+  - Custom trace format (we map it together on the call — usually 20 lines of glue)
+- [ ] **Scenarios** — 20-50 representative inputs your agent handles. Even YAML / JSON / TS array is fine; we'll convert to canonical `DatasetScenario[]` shape together
+- [ ] **The system prompt addendum** your agent uses today (or whichever text surface you want to optimize) — the closed loop edits this
+
+### Judge
+
+- [ ] **A judge function or rubric** — either:
+  - An existing function `(artifact) → { composite, dimensions }`
+  - A rubric describing what "good output" means (1-2 paragraphs is enough — we'll build the judge on the call)
+  - A set of "good" / "bad" labeled examples (we use these as anchors)
+
+### Constraints
+
+- [ ] **LLM cost budget for the closed loop** — default $25 per campaign. Tell us if you want a different ceiling
+- [ ] **Cadence** — how often should the loop run? Default: weekly. Some customers want daily; others want on-demand only
+- [ ] **Deployment gate preference** — do you want:
+  - Auto-PR on `ship-substrate` (we open the PR, your team reviews)
+  - Manual review only (we report; you decide)
+  - Auto-deploy on `ship-substrate` (only with explicit ack; not default)
+
+## Call agenda — 90 minutes
+
+| Time | Topic |
+|---|---|
+| 0:00 — 0:10 | Walk through your existing setup — what runs where, what scenarios exist, what success looks like for you |
+| 0:10 — 0:30 | Pick the right intake adapter; pull traces; run `analyzeRuns()` against last week's data — first decision packet rendered live |
+| 0:30 — 0:50 | Build the judge — either wrap your existing one or scaffold a new one from your rubric |
+| 0:50 — 1:10 | Fire one `selfImprove` cycle with a small budget ($5, single generation, 2 candidates) — watch the loop run end-to-end |
+| 1:10 — 1:25 | Wire the cron + auto-PR target; schedule first weekly run |
+| 1:25 — 1:30 | Confirm what we hand back to you between runs and what reaches you when |
+
+If something on the checklist isn't ready, we adapt — just send what you have. Worst case, we spend the first 30 minutes getting unblocked.
+
+## What you'll have at the end of the call
+
+- A working `analyzeRuns()` call against YOUR live trace data, returning a real `InsightReport`
+- A judge function (yours or scaffolded) wired to your agent's output shape
+- One completed `selfImprove` cycle with a real `gateDecision` + lift CI
+- A scheduled cron / GitHub Action that runs the loop weekly
+- Optional: an auto-PR target if you want green-gate proposals to land as draft PRs
+
+## After the call
+
+- Day 1-7: first weekly run fires; we monitor + jump in if anything breaks
+- Day 7: we send you a `selfImprove`-result summary + the corresponding `InsightReport`
+- Day 14-28: 3 more cycles complete; you have enough data to evaluate the pilot
+- Day 30: pilot review — what we found, what shipped, what's next
+
+## What we send back to you between runs
+
+- The full `InsightReport` JSON (you render it however you want, or use our hosted dashboard if it's available for your tier)
+- Slack / email digest of `regressedMetrics` + critical recommendations (opt-in)
+- Cost tally per campaign
+- Auto-PR links if green gate verdicts opened any
+
+## Common pre-call questions
+
+**Q: How small a corpus can we start with?**
+A: 15 scenarios works for the deterministic packet. 25+ is recommended for `selfImprove`'s held-out gate (the default `holdoutFraction: 0.3` reserves ~30% of scenarios for the gate).
+
+**Q: What if our judge isn't reliable yet?**
+A: Start with multi-rater intake — `fromFeedbackTable` produces inter-rater agreement (κ) so you can see exactly which scenarios humans disagree on. Iterate the judge until κ > 0.7, then go to closed loop.
+
+**Q: We don't use Tangle's sandbox — can we still pilot?**
+A: Yes. We have intake adapters for OTel, LangChain, LlamaIndex, Anthropic SDK, OpenAI Assistants, OpenRouter, multi-rater feedback tables, and custom trace formats. See `integration-foreign-stack.md`.
+
+**Q: We use OpenRouter — does the closed-loop driver work with our routing setup?**
+A: Yes. `gepaDriver` accepts any OpenAI-compatible endpoint via its `llm.baseUrl` option. Most customers run their selfImprove campaigns through OpenRouter or their existing provider — no migration required.
+
+**Q: What if the pilot fails — what do we get?**
+A: You get the deterministic `InsightReport` weekly regardless. Even if no `selfImprove` cycle ever ships a green gate verdict, you get the failure-cluster analysis, regressed-metric detection, and worst-runs surfacing. Those alone replace what most teams currently get from LangSmith / Braintrust / Phoenix scorecards.