agent-ablation 0.2.0 → 0.3.0

package/dist/index.d.ts

@@ -61,50 +61,384 @@ declare function fromRecords<T>(records: readonly T[] | T[], options: {
     confidenceOf?: (record: T, index: number) => number | undefined;
 }): Finding[];
 
+/**
+ * Structural representation of a task output or message produced by a CrewAI agent.
+ */
+interface CrewAITaskOutput {
+    /** The role or name of the agent that performed the task. */
+    agent?: string | {
+        role?: string;
+        name?: string;
+    } | null;
+    /** Raw string or structured output of the task. */
+    raw?: string;
+    /** Optional structured JSON / object output if configured with output_json/pydantic. */
+    json_dict?: Record<string, unknown> | null;
+    /** Optional numerical score or rating if present on the output. */
+    score?: number;
+    /** Optional tokens, cost or execution time telemetry. */
+    cost?: number;
+    tokens?: number;
+    latencyMs?: number;
+    /** Arbitrary metadata. */
+    [key: string]: unknown;
+}
+interface CrewAIAdapterOptions<TOutput extends CrewAITaskOutput = CrewAITaskOutput> {
+    /** Custom extractor for score if not directly in output.score */
+    scoreOf?: (output: TOutput) => number;
+    /** Custom extractor for agent name if not in output.agent */
+    agentIdOf?: (output: TOutput) => string;
+    /** Custom confidence extractor */
+    confidenceOf?: (output: TOutput) => number | undefined;
+    /** Cost extractor */
+    costOf?: (output: TOutput) => number | undefined;
+    /** Tokens extractor */
+    tokensOf?: (output: TOutput) => number | undefined;
+}
+/**
+ * Converts CrewAI task outputs or agent payloads into `Finding[]` objects.
+ */
+declare function fromCrewAITasks<TOutput extends CrewAITaskOutput = CrewAITaskOutput>(outputs: readonly TOutput[] | TOutput[], options?: CrewAIAdapterOptions<TOutput>): Finding[];
+
+/**
+ * Structural representation of a message from an AutoGen multi-agent chat session.
+ */
+interface AutoGenMessage {
+    /** The name/role of the agent that authored the message. */
+    name?: string | null;
+    /** Content of the message, either string or object payload. */
+    content?: unknown;
+    /** Role string (e.g. "user", "assistant"). */
+    role?: string | null;
+    /** Optional metadata / context dictionary. */
+    context?: Record<string, unknown>;
+    [key: string]: unknown;
+}
+interface AutoGenAdapterOptions<TMessage extends AutoGenMessage = AutoGenMessage> {
+    /** Extractor for numeric score / rating from the message. */
+    scoreOf: (message: TMessage) => number;
+    /** Optional extractor for confidence rating. */
+    confidenceOf?: (message: TMessage) => number | undefined;
+    /** Optional extractor for token usage. */
+    tokensOf?: (message: TMessage) => number | undefined;
+    /** Optional extractor for dollar cost. */
+    costOf?: (message: TMessage) => number | undefined;
+}
+/**
+ * Converts AutoGen conversation history into `Finding[]` objects.
+ */
+declare function fromAutoGenMessages<TMessage extends AutoGenMessage = AutoGenMessage>(messages: readonly TMessage[] | TMessage[], options: AutoGenAdapterOptions<TMessage>): Finding[];
+
+/**
+ * Structural representation of a step or tool execution result in Vercel AI SDK / Agent runs.
+ */
+interface AISDKStepResult {
+    /** Tool name or agent step identifier */
+    toolName?: string;
+    stepType?: string;
+    /** Result or args */
+    args?: unknown;
+    result?: unknown;
+    /** Token usage metadata if available */
+    usage?: {
+        promptTokens?: number;
+        completionTokens?: number;
+        totalTokens?: number;
+    };
+    /** Execution latency in ms */
+    latencyMs?: number;
+    [key: string]: unknown;
+}
+interface AISDKAdapterOptions<TStep extends AISDKStepResult = AISDKStepResult> {
+    /** Custom extractor for agent ID / tool name */
+    agentIdOf?: (step: TStep) => string;
+    /** Extractor for score */
+    scoreOf: (step: TStep) => number;
+    /** Optional confidence extractor */
+    confidenceOf?: (step: TStep) => number | undefined;
+    /** Optional cost extractor */
+    costOf?: (step: TStep) => number | undefined;
+}
+/**
+ * Converts Vercel AI SDK tool call steps or agent trace steps into `Finding[]` objects.
+ */
+declare function fromAISDKSteps<TStep extends AISDKStepResult = AISDKStepResult>(steps: readonly TStep[] | TStep[], options: AISDKAdapterOptions<TStep>): Finding[];
+
+interface ReportOptions {
+    /** Title for the report */
+    title?: string;
+    /** Whether to include ROI breakdown table */
+    includeRoi?: boolean;
+    /** Whether to include pruning recommendations */
+    includeRecommendations?: boolean;
+}
+/**
+ * Formats batch ablation results and ROI metrics into a GitHub / Dev.to-flavored Markdown report.
+ */
+declare function formatMarkdownReport(summary: BatchAblationSummary, options?: ReportOptions): string;
+/**
+ * Formats batch ablation results into an ASCII table string for CLI/terminal logs.
+ */
+declare function formatAsciiTable(summary: BatchAblationSummary): string;
+
+/**
+ * Represents a single finding or signal produced by an individual agent.
+ */
 interface Finding {
+    /** Unique identifier of the agent that produced this finding. */
     agentId: string;
+    /** Numerical score or risk rating produced by the agent. */
     score: number;
+    /** Optional confidence score (e.g. 0 to 1) associated with the finding. */
     confidence?: number;
+    /** Optional financial cost (in USD or custom currency unit) consumed to produce this finding. */
+    cost?: number;
+    /** Optional token count (prompt + completion) consumed to produce this finding. */
+    tokens?: number;
+    /** Optional execution time / latency in milliseconds. */
+    latencyMs?: number;
+    /** Arbitrary metadata associated with the finding (e.g. raw output or attributes). */
     metadata?: Record<string, unknown>;
 }
+/**
+ * Pure function that aggregates a list of agent findings into a verdict.
+ *
+ * @typeParam TVerdict Type of the verdict produced by the decision function.
+ */
 type DecisionFn<TVerdict> = (findings: Finding[]) => TVerdict;
+/**
+ * Async function that aggregates a list of agent findings into a verdict.
+ *
+ * @typeParam TVerdict Type of the verdict produced by the decision function.
+ */
+type AsyncDecisionFn<TVerdict> = (findings: Finding[]) => Promise<TVerdict> | TVerdict;
+/**
+ * Outcome of ablating a single agent from the panel.
+ *
+ * @typeParam TVerdict Type of the decision verdict.
+ */
 interface PerAgentAblation<TVerdict> {
+    /** Identifier of the agent that was removed in this ablation run. */
     removedAgentId: string;
+    /** Verdict produced by the decision function without this agent's finding. */
     verdictWithout: TVerdict;
+    /** Whether the verdict changed compared to the baseline verdict. */
     changed: boolean;
 }
+/**
+ * Result of a leave-one-out ablation across all agents for a single decision.
+ *
+ * @typeParam TVerdict Type of the decision verdict.
+ */
 interface AblationResult<TVerdict> {
+    /** Original decision verdict produced with all findings present. */
     baseline: TVerdict;
+    /** Detailed per-agent ablation outcomes. */
     perAgent: PerAgentAblation<TVerdict>[];
+    /** Count of agents whose removal flipped the decision outcome. */
     loadBearingCount: number;
+    /** Total number of agents evaluated in this decision. */
     totalAgents: number;
+    /** Fraction of agents that were load-bearing (`loadBearingCount / totalAgents`). */
     loadBearingRatio: number;
 }
+/**
+ * Detailed telemetry & ROI metrics for a single agent.
+ */
+interface AgentRoiMetrics {
+    /** Total dollar cost consumed across all appearances. */
+    totalCost?: number;
+    /** Total tokens consumed across all appearances. */
+    totalTokens?: number;
+    /** Mean latency in milliseconds. */
+    averageLatencyMs?: number;
+    /** Dollar cost per verdict flip (totalCost / verdictFlips). */
+    costPerVerdictFlip?: number;
+    /** Tokens per verdict flip (totalTokens / verdictFlips). */
+    tokensPerVerdictFlip?: number;
+    /** Percentage of overall multi-agent pipeline cost consumed by this agent. */
+    costShare?: number;
+    /** Ratio of influence share to cost share (>1 means high efficiency, <1 means expensive relative to impact). */
+    efficiencyRatio?: number;
+}
+/**
+ * Pruning or optimization recommendation for an agent.
+ */
+interface PruningRecommendation {
+    agentId: string;
+    recommendation: "prune" | "downgrade_model" | "keep";
+    reason: string;
+}
+/**
+ * Aggregated ROI metrics across all agents.
+ */
+interface RoiSummary {
+    totalCost: number;
+    totalTokens: number;
+    agents: Record<string, AgentRoiMetrics>;
+    recommendations: PruningRecommendation[];
+}
+/**
+ * Accuracy and ground-truth metrics.
+ */
+interface GroundTruthSummary {
+    baselineAccuracy: number;
+    correctBaselineCount: number;
+    totalEvaluated: number;
+}
+/**
+ * Statistics per agent including appearances, flips, and net accuracy impact.
+ */
+interface PerAgentStats {
+    appearances: number;
+    flips: number;
+    protectiveFlips?: number;
+    correctiveFlips?: number;
+    netAccuracyImpact?: number;
+    role?: "Protective" | "Harmful" | "Neutral";
+}
+/**
+ * Summary metrics aggregated across a batch of ablation cases.
+ */
 interface BatchAblationSummary {
+    /** Total number of cases evaluated in the batch. */
     cases: number;
+    /** Mean load-bearing ratio across all evaluated cases. */
     averageLoadBearingRatio: number;
+    /** Map of agent IDs to the fraction of cases where removing that agent changed the outcome. */
     perAgentInfluence: Record<string, number>;
+    /** Detailed statistics per agent. */
+    perAgentStats?: Record<string, PerAgentStats>;
+    /** Optional ROI breakdown (if cost/token telemetry was provided in findings). */
+    roi?: RoiSummary;
+    /** Optional ground-truth accuracy metrics (if groundTruth was supplied). */
+    accuracy?: GroundTruthSummary;
 }
+/**
+ * Options for batch ablation runs.
+ */
+interface BatchAblationOptions<TVerdict> {
+    /** Custom comparator for verdicts (defaults to ===). */
+    equals?: (a: TVerdict, b: TVerdict) => boolean;
+    /** Array of ground truth verdicts corresponding 1:1 with cases. */
+    groundTruth?: TVerdict[];
+}
+/**
+ * Options for async and stochastic ablation runs.
+ */
+interface AsyncAblationOptions<TVerdict> {
+    /** Custom comparator for verdicts (defaults to ===). */
+    equals?: (a: TVerdict, b: TVerdict) => boolean;
+    /** Number of stochastic samples to draw per decision (for non-deterministic LLM decide functions). Default: 1 */
+    samples?: number;
+    /** Custom aggregator for sample runs (defaults to majority voting). */
+    aggregateSamples?: (samples: TVerdict[]) => TVerdict;
+    /** Array of ground truth verdicts (for batch runs). */
+    groundTruth?: TVerdict[];
+}
+/**
+ * Resolves majority vote across multiple stochastic sample evaluations.
+ */
+declare function majorityVote<TVerdict>(samples: TVerdict[], equals?: (a: TVerdict, b: TVerdict) => boolean): TVerdict;
 /**
  * Runs a leave-one-out ablation over `findings`: computes the baseline verdict,
  * then re-runs `decide` once per finding with that finding removed, comparing each
  * result back to the baseline via `equals`.
  *
- * `equals` defaults to `===`. If TVerdict is an object or otherwise compared by
- * reference (not a primitive like a string or number), you MUST supply your own
- * `equals` — otherwise every ablation will spuriously read as "changed" because two
- * structurally identical objects are never `===` to one another, even when nothing
- * about the decision actually differed.
+ * @param findings Array of agent findings to ablate.
+ * @param decide Pure decision function mapping findings to a verdict.
+ * @param equals Optional comparator for verdicts (defaults to `===`).
+ * @returns Ablation results containing baseline verdict and per-agent outcomes.
  */
 declare function runAblation<TVerdict>(findings: Finding[], decide: DecisionFn<TVerdict>, equals?: (a: TVerdict, b: TVerdict) => boolean): AblationResult<TVerdict>;
+/**
+ * Runs an asynchronous leave-one-out ablation over `findings`.
+ * Supports stochastic decision functions with repeated K-sampling and majority voting.
+ *
+ * @param findings Array of agent findings to ablate.
+ * @param decide Async decision function mapping findings to a verdict.
+ * @param options Configuration for equals comparison, samples, and sampling aggregation.
+ */
+declare function runAblationAsync<TVerdict>(findings: Finding[], decide: AsyncDecisionFn<TVerdict>, options?: AsyncAblationOptions<TVerdict>): Promise<AblationResult<TVerdict>>;
+/**
+ * Aggregates batch ablation results into summary metrics, ROI breakdown, and ground-truth accuracy impact.
+ */
+declare function aggregateBatchResults<TVerdict>(cases: Finding[][], results: AblationResult<TVerdict>[], options?: BatchAblationOptions<TVerdict>): BatchAblationSummary;
 /**
  * Runs `runAblation` over a batch of independent cases and aggregates the results:
- * the mean load-bearing ratio across cases, and, per agent ID, the fraction of the
- * cases containing that agent in which removing it flipped the verdict.
+ * the mean load-bearing ratio across cases, per-agent influence, telemetry ROI, and ground-truth accuracy impact.
+ *
+ * @param cases Array of cases, where each case is an array of findings.
+ * @param decide Pure decision function mapping findings to a verdict.
+ * @param options Optional configuration object or comparator function.
+ * @returns Individual case results and aggregated batch summary.
  */
-declare function batchAblation<TVerdict>(cases: Finding[][], decide: DecisionFn<TVerdict>, equals?: (a: TVerdict, b: TVerdict) => boolean): {
+declare function batchAblation<TVerdict>(cases: Finding[][], decide: DecisionFn<TVerdict>, options?: ((a: TVerdict, b: TVerdict) => boolean) | BatchAblationOptions<TVerdict>): {
     results: AblationResult<TVerdict>[];
     summary: BatchAblationSummary;
 };
+/**
+ * Runs `runAblationAsync` over a batch of cases with async/stochastic decision functions.
+ */
+declare function batchAblationAsync<TVerdict>(cases: Finding[][], decide: AsyncDecisionFn<TVerdict>, options?: AsyncAblationOptions<TVerdict>): Promise<{
+    results: AblationResult<TVerdict>[];
+    summary: BatchAblationSummary;
+}>;
+/**
+ * Result of a greedy backward elimination pass to find the minimal agent panel.
+ */
+interface BackwardEliminationResult<TVerdict> {
+    /** Baseline verdict with all findings present. */
+    baseline: TVerdict;
+    /** Minimal subset of findings required to sustain the baseline verdict. */
+    minimalFindings: Finding[];
+    /** Agent IDs in the minimal subset. */
+    minimalAgentIds: string[];
+    /** Agents pruned during elimination in order of pruning. */
+    eliminatedAgentIds: string[];
+    /** Step-by-step trace of elimination rounds. */
+    steps: {
+        step: number;
+        eliminatedAgentId: string;
+        remainingAgentIds: string[];
+        verdict: TVerdict;
+    }[];
+    /** Whether elimination stopped because removing any further agent flips the verdict. */
+    stoppedDueToVerdictFlip: boolean;
+}
+/**
+ * Performs greedy backward elimination over an agent panel.
+ * Iteratively removes the least impactful agent while verifying that the verdict remains identical to the baseline.
+ * Stops when removing any remaining agent flips the verdict, revealing the minimal viable agent panel.
+ *
+ * @param findings Array of agent findings.
+ * @param decide Decision function.
+ * @param equals Optional verdict comparator.
+ */
+declare function runBackwardElimination<TVerdict>(findings: Finding[], decide: DecisionFn<TVerdict>, equals?: (a: TVerdict, b: TVerdict) => boolean): BackwardEliminationResult<TVerdict>;
+/**
+ * Async version of greedy backward elimination.
+ */
+declare function runBackwardEliminationAsync<TVerdict>(findings: Finding[], decide: AsyncDecisionFn<TVerdict>, options?: AsyncAblationOptions<TVerdict>): Promise<BackwardEliminationResult<TVerdict>>;
+/**
+ * Pairwise interaction ablation result.
+ */
+interface PairwiseAblationItem<TVerdict> {
+    pair: [string, string];
+    verdictWithout: TVerdict;
+    changed: boolean;
+    /** True if removing neither agent alone changed the outcome, but removing both together did! */
+    isInteraction: boolean;
+}
+interface PairwiseAblationResult<TVerdict> {
+    baseline: TVerdict;
+    pairs: PairwiseAblationItem<TVerdict>[];
+    interactionCount: number;
+}
+/**
+ * Evaluates pairwise combinations (pairs of agents) to detect joint dependencies where neither agent alone is load-bearing,
+ * but removing both together flips the outcome.
+ */
+declare function runPairwiseAblation<TVerdict>(findings: Finding[], decide: DecisionFn<TVerdict>, equals?: (a: TVerdict, b: TVerdict) => boolean): PairwiseAblationResult<TVerdict>;
 
-export { type AblationResult, type BatchAblationSummary, type DecisionFn, type Finding, type LangGraphAdapterOptions, type LangGraphAgentMessage, type PerAgentAblation, batchAblation, fromLangGraphMessages, fromRecords, runAblation };
+export { type AISDKAdapterOptions, type AISDKStepResult, type AblationResult, type AgentRoiMetrics, type AsyncAblationOptions, type AsyncDecisionFn, type AutoGenAdapterOptions, type AutoGenMessage, type BackwardEliminationResult, type BatchAblationOptions, type BatchAblationSummary, type CrewAIAdapterOptions, type CrewAITaskOutput, type DecisionFn, type Finding, type GroundTruthSummary, type LangGraphAdapterOptions, type LangGraphAgentMessage, type PairwiseAblationItem, type PairwiseAblationResult, type PerAgentAblation, type PerAgentStats, type PruningRecommendation, type ReportOptions, type RoiSummary, aggregateBatchResults, batchAblation, batchAblationAsync, formatAsciiTable, formatMarkdownReport, fromAISDKSteps, fromAutoGenMessages, fromCrewAITasks, fromLangGraphMessages, fromRecords, majorityVote, runAblation, runAblationAsync, runBackwardElimination, runBackwardEliminationAsync, runPairwiseAblation };