@tangle-network/agent-eval 0.72.0 → 0.72.3

package/dist/{control-DxvZeV5X.d.ts → control-BgA6BYTm.d.ts}

@@ -1,4 +1,4 @@
-import { a as FeedbackLabel, p as ProposedSideEffect } from './feedback-trajectory-8hKC5EOb.js';
+import { b as FeedbackLabel, p as ProposedSideEffect } from './feedback-trajectory-B3rErRsh.js';
 import { C as ControlEvalResult, a as ControlRunResult, h as ControlRuntimeConfig } from './control-runtime-DuFBYg7A.js';
 import { T as TraceEmitter } from './emitter-DEZwY14K.js';
 import { F as FailureClass } from './schema-m0gsnbt3.js';

package/dist/control.d.ts

@@ -1,6 +1,6 @@
-export { A as ActionExecutionPolicy, a as ActionPolicyDecision, C as ControlRunToRunRecordOptions, e as ProposeReviewConfig, f as ProposeReviewControlAction, g as ProposeReviewControlConfig, h as ProposeReviewControlResult, i as ProposeReviewControlState, j as ProposeReviewReport, p as RunEvidenceMetadata, s as controlRunToRunRecord, u as evaluateActionPolicy, x as runProposeReview, y as runProposeReviewAsControlLoop, z as scoreFromEvals } from './control-DxvZeV5X.js';
+export { A as ActionExecutionPolicy, a as ActionPolicyDecision, C as ControlRunToRunRecordOptions, e as ProposeReviewConfig, f as ProposeReviewControlAction, g as ProposeReviewControlConfig, h as ProposeReviewControlResult, i as ProposeReviewControlState, j as ProposeReviewReport, p as RunEvidenceMetadata, s as controlRunToRunRecord, u as evaluateActionPolicy, x as runProposeReview, y as runProposeReviewAsControlLoop, z as scoreFromEvals } from './control-BgA6BYTm.js';
 export { c as ControlActionFailureMode, d as ControlActionOutcome, e as ControlBudget, f as ControlContext, g as ControlDecision, C as ControlEvalResult, a as ControlRunResult, h as ControlRuntimeConfig, i as ControlRuntimeError, j as ControlSeverity, b as ControlStep, k as ControlStopPolicies, S as StopDecision, l as allCriticalPassed, o as objectiveEval, r as runAgentControlLoop, s as stopOnNoProgress, m as stopOnRepeatedAction, n as subjectiveEval } from './control-runtime-DuFBYg7A.js';
-import './feedback-trajectory-8hKC5EOb.js';
+import './feedback-trajectory-B3rErRsh.js';
 import './dataset-B2kL-fSM.js';
 import './errors-Dwqw-T_m.js';
 import './emitter-DEZwY14K.js';

package/dist/{feedback-trajectory-8hKC5EOb.d.ts → feedback-trajectory-B3rErRsh.d.ts}

@@ -166,4 +166,4 @@ declare function controlRunToFeedbackTrajectory<TState, TAction, TActionResult>(
     createdAt?: string;
 }): FeedbackTrajectory;
 
-export { replayFeedbackTrajectory as A, serializeFeedbackTrajectoriesJsonl as B, summarizePreferenceMemory as C, withAssignedFeedbackSplit as D, type FeedbackTrajectoryStore as F, InMemoryFeedbackTrajectoryStore as I, type PreferenceMemoryEntry as P, type FeedbackLabel as a, type FeedbackTrajectory as b, type FeedbackArtifactType as c, type FeedbackAttempt as d, type FeedbackLabelKind as e, type FeedbackLabelSource as f, type FeedbackOptimizerRow as g, type FeedbackOutcome as h, type FeedbackReplayAdapter as i, type FeedbackReplayResult as j, type FeedbackSeverity as k, type FeedbackSplitPolicy as l, type FeedbackTask as m, type FeedbackTrajectoryFilter as n, FileSystemFeedbackTrajectoryStore as o, type ProposedSideEffect as p, assignFeedbackSplit as q, controlRunToFeedbackTrajectory as r, createFeedbackTrajectory as s, feedbackTrajectoriesToDatasetScenarios as t, feedbackTrajectoriesToOptimizerRows as u, feedbackTrajectoryToDatasetScenario as v, feedbackTrajectoryToOptimizerRow as w, parseFeedbackTrajectoriesJsonl as x, renderPreferenceMemoryMarkdown as y, replayFeedbackTrajectories as z };
+export { replayFeedbackTrajectory as A, serializeFeedbackTrajectoriesJsonl as B, summarizePreferenceMemory as C, withAssignedFeedbackSplit as D, type FeedbackTrajectoryStore as F, InMemoryFeedbackTrajectoryStore as I, type PreferenceMemoryEntry as P, type FeedbackTrajectory as a, type FeedbackLabel as b, type FeedbackArtifactType as c, type FeedbackAttempt as d, type FeedbackLabelKind as e, type FeedbackLabelSource as f, type FeedbackOptimizerRow as g, type FeedbackOutcome as h, type FeedbackReplayAdapter as i, type FeedbackReplayResult as j, type FeedbackSeverity as k, type FeedbackSplitPolicy as l, type FeedbackTask as m, type FeedbackTrajectoryFilter as n, FileSystemFeedbackTrajectoryStore as o, type ProposedSideEffect as p, assignFeedbackSplit as q, controlRunToFeedbackTrajectory as r, createFeedbackTrajectory as s, feedbackTrajectoriesToDatasetScenarios as t, feedbackTrajectoriesToOptimizerRows as u, feedbackTrajectoryToDatasetScenario as v, feedbackTrajectoryToOptimizerRow as w, parseFeedbackTrajectoriesJsonl as x, renderPreferenceMemoryMarkdown as y, replayFeedbackTrajectories as z };

package/dist/harness-optimizer-EnEnQPsr.d.ts

@@ -0,0 +1,106 @@
+import { O as Objective, P as ParetoResult } from './pareto-E-pembql.js';
+import { R as RunScore, a as RunTrace, b as RunScoreWeights } from './run-critic-BAIjX99r.js';
+
+interface SteeringRolePrompt {
+    system?: string;
+    append?: string;
+}
+interface SteeringBundle {
+    id: string;
+    coderPrompt?: string;
+    continuePrompt?: string;
+    reviewerPrompts?: Record<string, string>;
+    skills?: string[];
+    rolePrompts?: Record<string, SteeringRolePrompt>;
+    metadata?: Record<string, unknown>;
+}
+interface SteeringDelta {
+    coderPrompt?: string;
+    continuePrompt?: string;
+    reviewerPrompts?: Record<string, string>;
+    skills?: string[];
+    rolePrompts?: Record<string, SteeringRolePrompt>;
+    metadata?: Record<string, unknown>;
+}
+declare function mergeSteeringBundle(base: SteeringBundle, delta: SteeringDelta): SteeringBundle;
+declare function renderSteeringText(bundle: SteeringBundle): string;
+
+type HarnessIntervention = 'continue' | 'plan' | 'audit' | 'recover' | 'repair' | 'verify' | 'final_gate' | 'wait_for_measurement' | 'abort';
+interface WorkflowTopology {
+    id: string;
+    interventions: HarnessIntervention[];
+    maxParallelBranches?: number;
+    metadata?: Record<string, unknown>;
+}
+interface MeasurementPolicy {
+    required: string[];
+    optional?: string[];
+    promoteOn?: Array<keyof RunScore | 'aggregate'>;
+}
+interface HarnessVariant {
+    id: string;
+    steering?: SteeringBundle;
+    topology?: WorkflowTopology;
+    measurement?: MeasurementPolicy;
+    budgets?: Record<string, number>;
+    models?: Record<string, string>;
+    reviewers?: Record<string, string>;
+    metadata?: Record<string, unknown>;
+}
+interface HarnessScenario {
+    id: string;
+    task: string;
+    split?: 'train' | 'validation' | 'test' | string;
+    metadata?: Record<string, unknown>;
+}
+interface HarnessRunRequest {
+    variant: HarnessVariant;
+    scenario: HarnessScenario;
+    trialIndex: number;
+}
+interface HarnessAdapter {
+    run(request: HarnessRunRequest): Promise<RunTrace>;
+}
+interface HarnessRunResult {
+    variant: HarnessVariant;
+    scenario: HarnessScenario;
+    trialIndex: number;
+    trace: RunTrace;
+    score: RunScore;
+    aggregate: number;
+}
+interface HarnessVariantReport {
+    variant: HarnessVariant;
+    runs: HarnessRunResult[];
+    aggregateMean: number;
+    passRate: number;
+    costUsdMean: number;
+    wallSecondsMean: number;
+    scoreMean: RunScore;
+}
+interface HarnessSelection {
+    winner: HarnessVariantReport;
+    frontier: ParetoResult<HarnessVariantReport>;
+    reports: HarnessVariantReport[];
+}
+interface HarnessExperimentResult {
+    results: HarnessRunResult[];
+    selection: HarnessSelection;
+}
+interface HarnessExperimentConfig {
+    adapter: HarnessAdapter;
+    variants: HarnessVariant[];
+    scenarios: HarnessScenario[];
+    trialsPerScenario?: number;
+    parallelism?: number;
+    weights?: Partial<RunScoreWeights>;
+    objectives?: Array<Objective<HarnessVariantReport>>;
+    score?: (trace: RunTrace, request: HarnessRunRequest) => RunScore | Promise<RunScore>;
+    onResult?: (result: HarnessRunResult) => void | Promise<void>;
+}
+declare const DEFAULT_HARNESS_OBJECTIVES: Array<Objective<HarnessVariantReport>>;
+declare function runHarnessExperiment(config: HarnessExperimentConfig): Promise<HarnessExperimentResult>;
+declare function selectHarnessVariant(results: HarnessRunResult[], objectives?: Array<Objective<HarnessVariantReport>>): HarnessSelection;
+declare function summarizeHarnessResults(results: HarnessRunResult[]): HarnessVariantReport[];
+
+export { DEFAULT_HARNESS_OBJECTIVES as D, type HarnessAdapter as H, type MeasurementPolicy as M, type SteeringBundle as S, type WorkflowTopology as W, type HarnessExperimentConfig as a, type HarnessExperimentResult as b, type HarnessIntervention as c, type HarnessRunRequest as d, type HarnessRunResult as e, type HarnessScenario as f, type HarnessSelection as g, type HarnessVariant as h, type HarnessVariantReport as i, type SteeringDelta as j, type SteeringRolePrompt as k, runHarnessExperiment as l, mergeSteeringBundle as m, summarizeHarnessResults as n, renderSteeringText as r, selectHarnessVariant as s };

package/dist/hosted/index.d.ts

@@ -1,5 +1,5 @@
-export { E as EvalRunCellScore, d as EvalRunEvent, e as EvalRunGenerationSnapshot, f as EvalRunStatus, g as HOSTED_WIRE_VERSION, H as HostedClient, h as HostedIngestHeaders, a as HostedTenant, i as HostedWireVersion, j as IngestEvalRunsRequest, k as IngestResponse, l as IngestTracesRequest, T as TraceSpanEvent, m as createHostedClient, n as hostedClientFromEnv } from '../index-BGBrVS24.js';
-import '../types-CnmZ2bkP.js';
+import { M as MutableSurface, j as GateDecision } from '../types-Bba0vl1V.js';
+import { I as InsightReport } from '../insight-report-Df3lxYXM.js';
 import '../run-record-BgTFzO2r.js';
 import '../errors-Dwqw-T_m.js';
 import '../schema-m0gsnbt3.js';
@@ -7,3 +7,224 @@ import '../summary-report-ByiOUrHj.js';
 import '../failure-cluster-CL7IVgkJ.js';
 import '../store-CKUAgsJz.js';
 import '../judge-calibration-DilmB3Ml.js';
+
+/**
+ * # Hosted-tier wire format — the schema that EVERY orchestrator (ours,
+ * a partner's self-hosted one, a future open implementation) must accept.
+ *
+ * **Stability:** every type in this file is committed under semver. New
+ * minors only ADD optional fields. Breaking changes mean a major bump
+ * (`HostedWireVersion` literal increment).
+ *
+ * The wire format is two event streams in one transport:
+ *
+ *   1. **Eval-run events** (`POST /v1/ingest/eval-runs`). Posted when a
+ *      campaign / improvement-loop completes (or per-generation if
+ *      streaming). Carries the structured result + per-cell scores +
+ *      surface diffs the orchestrator stores for the dashboard.
+ *
+ *   2. **Trace spans** (`POST /v1/ingest/traces`). Standard OTLP-shaped
+ *      spans with a few additional attributes so the orchestrator can
+ *      pivot from eval-run → underlying execution. Compatible with any
+ *      OTel collector.
+ *
+ * Both endpoints are authenticated with a bearer token + a tenant id
+ * header. Tenants isolate everything downstream of ingest; no tenant
+ * ever sees another tenant's data.
+ */
+
+declare const HOSTED_WIRE_VERSION: "2026-05-26.v1";
+type HostedWireVersion = typeof HOSTED_WIRE_VERSION;
+/** Every ingest request carries these. */
+interface HostedIngestHeaders {
+    /** Bearer token. The orchestrator validates against the tenant key. */
+    authorization: `Bearer ${string}`;
+    /** Stable tenant id (the orchestrator-side primary key for the tenant). */
+    'x-tangle-tenant-id': string;
+    /** Wire-version pin so the server can reject incompatible payloads. */
+    'x-tangle-wire-version': HostedWireVersion;
+    /** Optional idempotency key for retry-safe ingest. */
+    'idempotency-key'?: string;
+}
+/** Lifecycle stages of an eval-run as the substrate reports them. */
+type EvalRunStatus = 'started' | 'baseline-complete' | 'generation-complete' | 'gate-decided' | 'finished' | 'errored';
+interface EvalRunCellScore {
+    /** Stable scenario id from the consumer's scenario set. */
+    scenarioId: string;
+    /** Repetition index when reps > 1; 0 for the default. */
+    rep: number;
+    /** Composite score across all judges + dimensions for this cell. */
+    compositeMean: number;
+    /** Per-judge → per-dimension scores; null where the judge did not run. */
+    dimensions: Record<string, Record<string, number>>;
+    /** Per-cell error message if the dispatch threw. Null on success. */
+    errorMessage?: string;
+}
+interface EvalRunGenerationSnapshot {
+    /** Generation index. 0 is baseline. */
+    index: number;
+    /** Candidate surface fingerprint (stable hash) — pivot key into the
+     *  trace stream to fetch the underlying execution. */
+    surfaceHash: string;
+    /** The candidate surface itself. May be omitted to avoid PII when the
+     *  consumer prefers not to ship verbatim prompts. */
+    surface?: MutableSurface;
+    /** Per-cell scores for this generation. */
+    cells: EvalRunCellScore[];
+    /** Aggregate composite mean across all cells in this generation. */
+    compositeMean: number;
+    /** Total $ spent across this generation. */
+    costUsd: number;
+    /** Wall-clock duration of this generation. */
+    durationMs: number;
+}
+/**
+ * The top-level eval-run event. One ingest call per logical eval-run;
+ * generations stream in incrementally via repeated calls with the same
+ * `runId`. The orchestrator deduplicates by `(runId, generation.index)`.
+ */
+interface EvalRunEvent {
+    /** Stable run id (the substrate's `runId`). UUID or substrate-generated. */
+    runId: string;
+    /** Where this run was happening — derived from `RunCampaignOptions.runDir`. */
+    runDir: string;
+    /** ISO-8601 timestamp the substrate recorded the event. */
+    timestamp: string;
+    /** Lifecycle stage this event represents. */
+    status: EvalRunStatus;
+    /** Free-form consumer tags (env, branch, model id, etc.). Searchable. */
+    labels: Record<string, string>;
+    /** Baseline campaign snapshot. Present when status >= baseline-complete. */
+    baseline?: EvalRunGenerationSnapshot;
+    /** Per-generation snapshots. Streams in; orchestrator appends. */
+    generations: EvalRunGenerationSnapshot[];
+    /** Final gate decision. Present when status >= gate-decided. */
+    gateDecision?: GateDecision;
+    /** Held-out lift = winner-on-holdout - baseline-on-holdout. */
+    holdoutLift?: number;
+    /** Total $ spent across baseline + every generation. */
+    totalCostUsd: number;
+    /** Total wall-clock duration. */
+    totalDurationMs: number;
+    /** Error message if status === 'errored'. */
+    errorMessage?: string;
+    /** Rigor packet emitted alongside the run — distributional summary,
+     *  paired-bootstrap lift CI, judge stats, inter-rater agreement,
+     *  contamination check, failure clusters (when an analyst is wired),
+     *  outcome correlation (when downstream signal is supplied), and the
+     *  recommendations the dashboard surfaces verbatim. Additive; older
+     *  clients that don't know about this field continue to work. */
+    insightReport?: InsightReport;
+}
+/**
+ * OTel-shape span with a few additional attributes for eval-run pivoting.
+ * Compatible with any OTLP collector — `name`, `traceId`, `spanId`,
+ * `startTimeUnixNano`, `endTimeUnixNano`, `attributes` are stock OTel.
+ */
+interface TraceSpanEvent {
+    traceId: string;
+    spanId: string;
+    parentSpanId?: string;
+    name: string;
+    startTimeUnixNano: number;
+    endTimeUnixNano: number;
+    attributes: Record<string, string | number | boolean>;
+    events?: Array<{
+        timeUnixNano: number;
+        name: string;
+        attributes?: Record<string, string | number | boolean>;
+    }>;
+    status?: {
+        code: 'OK' | 'ERROR' | 'UNSET';
+        message?: string;
+    };
+    /** Pivot back into the eval-run stream. */
+    'tangle.runId'?: string;
+    /** Pivot to the specific generation. */
+    'tangle.generation'?: number;
+    /** Pivot to the specific cell. */
+    'tangle.cellId'?: string;
+    /** Pivot to the specific scenario. */
+    'tangle.scenarioId'?: string;
+}
+interface IngestEvalRunsRequest {
+    wireVersion: HostedWireVersion;
+    events: EvalRunEvent[];
+}
+interface IngestTracesRequest {
+    wireVersion: HostedWireVersion;
+    spans: TraceSpanEvent[];
+}
+interface IngestResponse {
+    /** Accepted events / spans count. */
+    accepted: number;
+    /** Rejected events with reasons (validation failures, dup idempotency key, etc.). */
+    rejected: Array<{
+        index: number;
+        reason: string;
+    }>;
+}
+
+/**
+ * # Hosted-tier ingest client.
+ *
+ * Ships eval-run events + trace spans to any orchestrator (ours, a
+ * partner's self-hosted one, or a future open implementation) that
+ * speaks the wire format in `./types.ts`.
+ *
+ * Three modes:
+ *   - **Ours:** point at `https://orchestrator.tangle.tools` (the host root —
+ *     the client appends the versioned `/v1/ingest/...` path itself; a trailing
+ *     `/v1` on the endpoint is tolerated and normalized away). We handle ingest
+ *     + storage + dashboard.
+ *   - **Self-hosted:** point at whatever URL runs the reference receiver
+ *     from `examples/hosted-ingest-server/`.
+ *   - **Off (default):** when `hostedTenant` is unset, nothing is sent.
+ *     Everything stays local.
+ */
+
+interface HostedTenant {
+    /** Orchestrator endpoint base URL (no trailing slash). Required. */
+    endpoint: string;
+    /** Bearer token issued by the orchestrator. Required. */
+    apiKey: string;
+    /** Tenant id — the orchestrator's primary key for this consumer. Required. */
+    tenantId: string;
+    /** Optional `fetch` override (auth wrappers, custom agent, test mocks). */
+    fetchImpl?: typeof fetch;
+    /** Per-call timeout in ms. Default 30s. */
+    timeoutMs?: number;
+    /** Retries on 5xx / network errors. Default 2. */
+    retries?: number;
+}
+interface HostedClient {
+    ingestEvalRun(event: EvalRunEvent, idempotencyKey?: string): Promise<IngestResponse>;
+    ingestEvalRuns(events: EvalRunEvent[], idempotencyKey?: string): Promise<IngestResponse>;
+    ingestTraces(spans: TraceSpanEvent[], idempotencyKey?: string): Promise<IngestResponse>;
+    readonly tenant: HostedTenant;
+    readonly wireVersion: HostedWireVersion;
+}
+declare function createHostedClient(tenant: HostedTenant): HostedClient;
+/**
+ * Build a `HostedClient` from environment, or `undefined` when ingest is not
+ * configured — the canonical, fail-soft wiring every product uses so eval-run +
+ * trace provenance lands in the Intelligence dashboard with ONE call:
+ *
+ *   const hosted = hostedClientFromEnv()
+ *   // ...run the loop...
+ *   await emitLoopProvenance({ ..., hostedClient: hosted })  // no-op if undefined
+ *
+ * Returns `undefined` (NOT an error) when any of endpoint / apiKey / tenantId is
+ * missing — so a product wires the ship call unconditionally and it stays a
+ * no-op until the env is set. Env precedence:
+ *   - endpoint:  `TANGLE_INGEST_URL` → `TANGLE_ORCHESTRATOR_URL`
+ *   - apiKey:    `TANGLE_INGEST_API_KEY` → `TANGLE_API_KEY`
+ *   - tenantId:  `TANGLE_TENANT_ID`
+ * A trailing slash on the endpoint is stripped. Pass `overrides` to supply any
+ * field directly (e.g. a fixed `tenantId` per product) — overrides win over env.
+ */
+declare function hostedClientFromEnv(overrides?: Partial<HostedTenant> & {
+    env?: Record<string, string | undefined>;
+}): HostedClient | undefined;
+
+export { type EvalRunCellScore, type EvalRunEvent, type EvalRunGenerationSnapshot, type EvalRunStatus, HOSTED_WIRE_VERSION, type HostedClient, type HostedIngestHeaders, type HostedTenant, type HostedWireVersion, type IngestEvalRunsRequest, type IngestResponse, type IngestTracesRequest, type TraceSpanEvent, createHostedClient, hostedClientFromEnv };