@tangle-network/agent-eval 0.95.0 → 0.96.0

package/dist/failure-cluster-DH9Flgcf.d.ts

@@ -0,0 +1,76 @@
+import { R as Run, S as Span, a as TraceEvent, F as FailureClass } from './schema-m0gsnbt3.js';
+import { T as TraceStore } from './store-BcFXE6LG.js';
+
+/**
+ * Failure taxonomy — canonical classes + a default classifier.
+ *
+ * Every failed run should end up in a named class. The classifier here
+ * is rule-based (fast, deterministic); an LLM fallback can be added by
+ * the consumer for novel cases and trained into the rule base over time.
+ *
+ * Consumers call `classifyFailure(run, spans, events)` and persist the
+ * returned class as `Run.outcome.failureClass`.
+ */
+
+interface FailureContext {
+    run: Run;
+    spans: Span[];
+    events: TraceEvent[];
+}
+interface FailureClassification {
+    failureClass: FailureClass;
+    reason: string;
+    triggerSpanId?: string;
+    triggerEventId?: string;
+}
+/** Ordered rules — first match wins. */
+interface FailureRule {
+    id: string;
+    match: (ctx: FailureContext) => {
+        failureClass: FailureClass;
+        reason: string;
+        triggerSpanId?: string;
+        triggerEventId?: string;
+    } | null;
+}
+declare const DEFAULT_RULES: FailureRule[];
+/** Classify the failure mode of a run using an ordered rule list. */
+declare function classifyFailure(ctx: FailureContext, rules?: FailureRule[]): FailureClassification;
+
+/**
+ * FailureClusterView — groups failed runs by (failureClass, triggerTool,
+ * argHash-prefix) so weekly reviews can prioritize the top-N clusters.
+ *
+ * Each cluster includes: N runs, scenarios affected, representative
+ * error message, a proposed mitigation hint (rule → action table).
+ */
+
+interface FailureCluster {
+    failureClass: FailureClass;
+    /** Tool name when the trigger was a tool span, else undefined. */
+    toolName?: string;
+    /** First 16 chars of argHash — clusters similar args. */
+    argPrefix?: string;
+    /**
+     * Source dimension when the trigger was a judge span (e.g. `'format'`,
+     * `'safety'`, `'correctness'`). Lets cross-template aggregators
+     * group failures by the dimension that fired without overloading
+     * `argPrefix`. Optional — clusters without this field deserialize cleanly.
+     */
+    dimension?: string;
+    runCount: number;
+    scenarioIds: string[];
+    exampleError?: string;
+    exampleRunId: string;
+}
+interface FailureClusterReport {
+    clusters: FailureCluster[];
+    totalFailures: number;
+    totalRuns: number;
+}
+declare function failureClusterView(store: TraceStore, options?: {
+    rules?: FailureRule[];
+    minClusterSize?: number;
+}): Promise<FailureClusterReport>;
+
+export { DEFAULT_RULES as D, type FailureClusterReport as F, type FailureCluster as a, type FailureClassification as b, type FailureContext as c, type FailureRule as d, classifyFailure as e, failureClusterView as f };

package/dist/{feedback-trajectory.d.ts → feedback-trajectory-BxY0cKfs.d.ts}

@@ -1,14 +1,15 @@
-import type { ControlEvalResult, ControlRunResult, ControlStep } from './control-runtime';
-import type { DatasetScenario, DatasetSplit } from './dataset';
-export type FeedbackArtifactType = 'text' | 'code' | 'plan' | 'research' | 'action' | 'ui' | 'decision' | 'data' | 'other';
-export type FeedbackLabelSource = 'user' | 'judge' | 'environment' | 'metric' | 'policy' | 'system';
-export type FeedbackLabelKind = 'approve' | 'reject' | 'select' | 'edit' | 'rank' | 'rate' | 'comment' | 'metric_outcome' | 'policy_block' | 'revision_request';
-export type FeedbackSeverity = 'info' | 'warning' | 'error' | 'critical';
-export interface FeedbackTask {
+import { C as ControlEvalResult, a as ControlRunResult, b as ControlStep } from './control-runtime-Acf9CGhw.js';
+import { D as DatasetSplit, a as DatasetScenario } from './dataset-BbGkaN2I.js';
+
+type FeedbackArtifactType = 'text' | 'code' | 'plan' | 'research' | 'action' | 'ui' | 'decision' | 'data' | 'other';
+type FeedbackLabelSource = 'user' | 'judge' | 'environment' | 'metric' | 'policy' | 'system';
+type FeedbackLabelKind = 'approve' | 'reject' | 'select' | 'edit' | 'rank' | 'rate' | 'comment' | 'metric_outcome' | 'policy_block' | 'revision_request';
+type FeedbackSeverity = 'info' | 'warning' | 'error' | 'critical';
+interface FeedbackTask {
     intent: string;
     context?: unknown;
 }
-export interface ProposedSideEffect {
+interface ProposedSideEffect {
     type: string;
     risk?: 'low' | 'medium' | 'high';
     costUsd?: number;
@@ -16,7 +17,7 @@ export interface ProposedSideEffect {
     requiresApproval?: boolean;
     metadata?: Record<string, unknown>;
 }
-export interface FeedbackLabel {
+interface FeedbackLabel {
     id?: string;
     source: FeedbackLabelSource;
     kind: FeedbackLabelKind;
@@ -26,7 +27,7 @@ export interface FeedbackLabel {
     createdAt: string;
     metadata?: Record<string, unknown>;
 }
-export interface FeedbackAttempt {
+interface FeedbackAttempt {
     id: string;
     stepIndex: number;
     artifactType: FeedbackArtifactType;
@@ -38,7 +39,7 @@ export interface FeedbackAttempt {
     createdAt: string;
     metadata?: Record<string, unknown>;
 }
-export interface FeedbackOutcome {
+interface FeedbackOutcome {
     success?: boolean;
     score?: number;
     metrics?: Record<string, number>;
@@ -47,7 +48,7 @@ export interface FeedbackOutcome {
     observedAt?: string;
     metadata?: Record<string, unknown>;
 }
-export interface FeedbackTrajectory {
+interface FeedbackTrajectory {
     id: string;
     projectId?: string;
     scenarioId?: string;
@@ -61,26 +62,26 @@ export interface FeedbackTrajectory {
     updatedAt?: string;
     metadata?: Record<string, unknown>;
 }
-export interface FeedbackTrajectoryStore {
+interface FeedbackTrajectoryStore {
     save(trajectory: FeedbackTrajectory): Promise<void>;
     get(id: string): Promise<FeedbackTrajectory | null>;
     list(filter?: FeedbackTrajectoryFilter): Promise<FeedbackTrajectory[]>;
     appendAttempt(id: string, attempt: FeedbackAttempt): Promise<FeedbackTrajectory>;
     appendLabel(id: string, label: FeedbackLabel, attemptId?: string): Promise<FeedbackTrajectory>;
 }
-export interface FeedbackTrajectoryFilter {
+interface FeedbackTrajectoryFilter {
     projectId?: string;
     scenarioId?: string;
     split?: DatasetSplit;
     tag?: [string, string];
 }
-export interface FeedbackSplitPolicy {
+interface FeedbackSplitPolicy {
     trainPct?: number;
     devPct?: number;
     testPct?: number;
     holdoutPct?: number;
 }
-export interface PreferenceMemoryEntry {
+interface PreferenceMemoryEntry {
     instruction: string;
     rationale: string;
     weight: number;
@@ -88,14 +89,14 @@ export interface PreferenceMemoryEntry {
     sourceLabelId?: string;
     category?: string;
 }
-export interface FeedbackOptimizerRow {
+interface FeedbackOptimizerRow {
     scenarioId: string;
     trajectoryId: string;
     labelKinds: FeedbackLabelKind[];
     score?: number;
     metadata?: Record<string, unknown>;
 }
-export interface FeedbackReplayResult {
+interface FeedbackReplayResult {
     trajectoryId: string;
     pass: boolean;
     score?: number;
@@ -103,10 +104,10 @@ export interface FeedbackReplayResult {
     outcome?: FeedbackOutcome;
     metadata?: Record<string, unknown>;
 }
-export interface FeedbackReplayAdapter {
+interface FeedbackReplayAdapter {
     replay(trajectory: FeedbackTrajectory): Promise<Omit<FeedbackReplayResult, 'trajectoryId'>> | Omit<FeedbackReplayResult, 'trajectoryId'>;
 }
-export declare class InMemoryFeedbackTrajectoryStore implements FeedbackTrajectoryStore {
+declare class InMemoryFeedbackTrajectoryStore implements FeedbackTrajectoryStore {
     private readonly trajectories;
     save(trajectory: FeedbackTrajectory): Promise<void>;
     get(id: string): Promise<FeedbackTrajectory | null>;
@@ -114,7 +115,7 @@ export declare class InMemoryFeedbackTrajectoryStore implements FeedbackTrajecto
     appendAttempt(id: string, attempt: FeedbackAttempt): Promise<FeedbackTrajectory>;
     appendLabel(id: string, label: FeedbackLabel, attemptId?: string): Promise<FeedbackTrajectory>;
 }
-export declare class FileSystemFeedbackTrajectoryStore implements FeedbackTrajectoryStore {
+declare class FileSystemFeedbackTrajectoryStore implements FeedbackTrajectoryStore {
     private readonly dir;
     private readonly memory;
     private loaded;
@@ -129,7 +130,7 @@ export declare class FileSystemFeedbackTrajectoryStore implements FeedbackTrajec
     private append;
     private load;
 }
-export declare function createFeedbackTrajectory(input: {
+declare function createFeedbackTrajectory(input: {
     id?: string;
     projectId?: string;
     scenarioId?: string;
@@ -142,21 +143,21 @@ export declare function createFeedbackTrajectory(input: {
     createdAt?: string;
     metadata?: Record<string, unknown>;
 }): FeedbackTrajectory;
-export declare function assignFeedbackSplit(trajectory: Pick<FeedbackTrajectory, 'id' | 'projectId' | 'scenarioId' | 'task'>, policy?: FeedbackSplitPolicy): DatasetSplit;
-export declare function withAssignedFeedbackSplit(trajectory: FeedbackTrajectory, policy?: FeedbackSplitPolicy): FeedbackTrajectory;
-export declare function feedbackTrajectoryToDatasetScenario(trajectory: FeedbackTrajectory): DatasetScenario;
-export declare function feedbackTrajectoriesToDatasetScenarios(trajectories: FeedbackTrajectory[]): DatasetScenario[];
-export declare function feedbackTrajectoryToOptimizerRow(trajectory: FeedbackTrajectory): FeedbackOptimizerRow;
-export declare function feedbackTrajectoriesToOptimizerRows(trajectories: FeedbackTrajectory[]): FeedbackOptimizerRow[];
-export declare function replayFeedbackTrajectory(trajectory: FeedbackTrajectory, adapter: FeedbackReplayAdapter): Promise<FeedbackReplayResult>;
-export declare function replayFeedbackTrajectories(trajectories: FeedbackTrajectory[], adapter: FeedbackReplayAdapter): Promise<FeedbackReplayResult[]>;
-export declare function summarizePreferenceMemory(trajectories: FeedbackTrajectory[], options?: {
+declare function assignFeedbackSplit(trajectory: Pick<FeedbackTrajectory, 'id' | 'projectId' | 'scenarioId' | 'task'>, policy?: FeedbackSplitPolicy): DatasetSplit;
+declare function withAssignedFeedbackSplit(trajectory: FeedbackTrajectory, policy?: FeedbackSplitPolicy): FeedbackTrajectory;
+declare function feedbackTrajectoryToDatasetScenario(trajectory: FeedbackTrajectory): DatasetScenario;
+declare function feedbackTrajectoriesToDatasetScenarios(trajectories: FeedbackTrajectory[]): DatasetScenario[];
+declare function feedbackTrajectoryToOptimizerRow(trajectory: FeedbackTrajectory): FeedbackOptimizerRow;
+declare function feedbackTrajectoriesToOptimizerRows(trajectories: FeedbackTrajectory[]): FeedbackOptimizerRow[];
+declare function replayFeedbackTrajectory(trajectory: FeedbackTrajectory, adapter: FeedbackReplayAdapter): Promise<FeedbackReplayResult>;
+declare function replayFeedbackTrajectories(trajectories: FeedbackTrajectory[], adapter: FeedbackReplayAdapter): Promise<FeedbackReplayResult[]>;
+declare function summarizePreferenceMemory(trajectories: FeedbackTrajectory[], options?: {
     maxEntries?: number;
 }): PreferenceMemoryEntry[];
-export declare function renderPreferenceMemoryMarkdown(entries: PreferenceMemoryEntry[]): string;
-export declare function serializeFeedbackTrajectoriesJsonl(trajectories: FeedbackTrajectory[]): string;
-export declare function parseFeedbackTrajectoriesJsonl(jsonl: string): FeedbackTrajectory[];
-export declare function controlRunToFeedbackTrajectory<TState, TAction, TActionResult>(run: ControlRunResult<TState, TAction, TActionResult>, options?: {
+declare function renderPreferenceMemoryMarkdown(entries: PreferenceMemoryEntry[]): string;
+declare function serializeFeedbackTrajectoriesJsonl(trajectories: FeedbackTrajectory[]): string;
+declare function parseFeedbackTrajectoriesJsonl(jsonl: string): FeedbackTrajectory[];
+declare function controlRunToFeedbackTrajectory<TState, TAction, TActionResult>(run: ControlRunResult<TState, TAction, TActionResult>, options?: {
     projectId?: string;
     scenarioId?: string;
     artifactType?: FeedbackArtifactType;
@@ -164,4 +165,5 @@ export declare function controlRunToFeedbackTrajectory<TState, TAction, TActionR
     proposedActionFromStep?: (step: ControlStep<TState, TAction, TActionResult>) => ProposedSideEffect | undefined;
     createdAt?: string;
 }): FeedbackTrajectory;
-//# sourceMappingURL=feedback-trajectory.d.ts.map
+
+export { replayFeedbackTrajectory as A, serializeFeedbackTrajectoriesJsonl as B, summarizePreferenceMemory as C, withAssignedFeedbackSplit as D, type FeedbackTrajectoryStore as F, InMemoryFeedbackTrajectoryStore as I, type PreferenceMemoryEntry as P, type FeedbackTrajectory as a, type FeedbackLabel as b, type FeedbackArtifactType as c, type FeedbackAttempt as d, type FeedbackLabelKind as e, type FeedbackLabelSource as f, type FeedbackOptimizerRow as g, type FeedbackOutcome as h, type FeedbackReplayAdapter as i, type FeedbackReplayResult as j, type FeedbackSeverity as k, type FeedbackSplitPolicy as l, type FeedbackTask as m, type FeedbackTrajectoryFilter as n, FileSystemFeedbackTrajectoryStore as o, type ProposedSideEffect as p, assignFeedbackSplit as q, controlRunToFeedbackTrajectory as r, createFeedbackTrajectory as s, feedbackTrajectoriesToDatasetScenarios as t, feedbackTrajectoriesToOptimizerRows as u, feedbackTrajectoryToDatasetScenario as v, feedbackTrajectoryToOptimizerRow as w, parseFeedbackTrajectoriesJsonl as x, renderPreferenceMemoryMarkdown as y, replayFeedbackTrajectories as z };