@crewai-ts/core 0.1.2 → 0.1.3

package/dist/evaluators.d.ts

@@ -0,0 +1,477 @@
+import { Agent } from "./agent.js";
+import { type StructuredModel } from "./converter.js";
+import { LiteAgentExecutionCompletedEvent, TaskCompletedEvent, type EventBus } from "./events.js";
+import { TaskOutput } from "./outputs.js";
+import { Task } from "./task.js";
+import type { LLM, TaskCallback } from "./types.js";
+export declare const MetricCategory: {
+    readonly GOAL_ALIGNMENT: "goal_alignment";
+    readonly SEMANTIC_QUALITY: "semantic_quality";
+    readonly REASONING_EFFICIENCY: "reasoning_efficiency";
+    readonly TOOL_SELECTION: "tool_selection";
+    readonly PARAMETER_EXTRACTION: "parameter_extraction";
+    readonly TOOL_INVOCATION: "tool_invocation";
+};
+export type MetricCategory = typeof MetricCategory[keyof typeof MetricCategory];
+export declare function metricCategoryTitle(category: MetricCategory): string;
+export declare class EvaluationScore {
+    readonly score: number | null;
+    readonly feedback: string;
+    readonly rawResponse: string | null;
+    readonly raw_response: string | null;
+    constructor(options?: {
+        score?: number | null;
+        feedback?: string;
+        rawResponse?: string | null;
+        raw_response?: string | null;
+    });
+    toString(): string;
+    __str__(): string;
+}
+export declare abstract class BaseEvaluator {
+    readonly llm: LLM | string | null;
+    constructor(llm?: LLM | string | null);
+    abstract get metricCategory(): MetricCategory;
+    get metric_category(): MetricCategory;
+    abstract evaluate(agent: unknown, executionTrace: Record<string, unknown>, finalOutput: unknown, task?: Task | null): EvaluationScore | Promise<EvaluationScore>;
+    evaluate_sync(agent: unknown, execution_trace: Record<string, unknown>, final_output: unknown, task?: Task | null): EvaluationScore | Promise<EvaluationScore>;
+}
+export declare class ConstantScoreEvaluator extends BaseEvaluator {
+    private readonly category;
+    private readonly defaultFeedback;
+    constructor(category: MetricCategory, defaultFeedback: string, llm?: LLM | string | null);
+    get metricCategory(): MetricCategory;
+    evaluate(_agent: unknown, executionTrace: Record<string, unknown>, _finalOutput: unknown, _task?: Task | null): EvaluationScore;
+}
+export declare class AgentEvaluationResult {
+    readonly agentId: string;
+    readonly agent_id: string;
+    readonly taskId: string;
+    readonly task_id: string;
+    readonly metrics: Map<MetricCategory, EvaluationScore>;
+    constructor(options: {
+        agentId?: string;
+        agent_id?: string;
+        taskId?: string;
+        task_id?: string;
+        metrics?: Map<MetricCategory, EvaluationScore> | Partial<Record<MetricCategory, EvaluationScore>>;
+    });
+}
+export declare const AggregationStrategy: {
+    readonly SIMPLE_AVERAGE: "simple_average";
+    readonly WEIGHTED_BY_COMPLEXITY: "weighted_by_complexity";
+    readonly BEST_PERFORMANCE: "best_performance";
+    readonly WORST_PERFORMANCE: "worst_performance";
+};
+export type AggregationStrategy = typeof AggregationStrategy[keyof typeof AggregationStrategy];
+export declare class AgentAggregatedEvaluationResult {
+    readonly agentId: string;
+    readonly agent_id: string;
+    readonly agentRole: string;
+    readonly agent_role: string;
+    readonly taskCount: number;
+    readonly task_count: number;
+    readonly aggregationStrategy: AggregationStrategy;
+    readonly aggregation_strategy: AggregationStrategy;
+    readonly metrics: Map<MetricCategory, EvaluationScore>;
+    readonly taskResults: string[];
+    readonly task_results: string[];
+    readonly overallScore: number | null;
+    readonly overall_score: number | null;
+    constructor(options?: {
+        agentId?: string;
+        agent_id?: string;
+        agentRole?: string;
+        agent_role?: string;
+        taskCount?: number;
+        task_count?: number;
+        aggregationStrategy?: AggregationStrategy;
+        aggregation_strategy?: AggregationStrategy;
+        metrics?: Map<MetricCategory, EvaluationScore> | Partial<Record<MetricCategory, EvaluationScore>>;
+        taskResults?: string[];
+        task_results?: string[];
+        overallScore?: number | null;
+        overall_score?: number | null;
+    });
+    toString(): string;
+    __str__(): string;
+}
+export declare class ExecutionState {
+    currentAgentId: string | null;
+    current_agent_id: string | null;
+    currentTaskId: string | null;
+    current_task_id: string | null;
+    traces: Record<string, unknown>;
+    iteration: number;
+    iterationsResults: Record<number, Record<string, AgentEvaluationResult[]>>;
+    iterations_results: Record<number, Record<string, AgentEvaluationResult[]>>;
+    agentEvaluators: Record<string, readonly BaseEvaluator[] | null>;
+    agent_evaluators: Record<string, readonly BaseEvaluator[] | null>;
+}
+export declare class AgentEvaluator {
+    readonly agents: readonly unknown[];
+    readonly evaluators: readonly BaseEvaluator[] | null;
+    readonly callback: EvaluationTraceCallback;
+    private readonly executionState;
+    readonly _execution_state: ExecutionState;
+    constructor(agents?: readonly unknown[], evaluators?: readonly BaseEvaluator[] | null);
+    _subscribe_to_events(): void;
+    _handle_task_completed(_source: unknown, event: TaskCompletedEvent | (TaskCompletedEvent & {
+        task?: unknown;
+    })): void;
+    _handle_lite_agent_completed(_source: unknown, event: LiteAgentExecutionCompletedEvent): void;
+    set_iteration(iteration: number): void;
+    reset_iterations_results(): void;
+    get_evaluation_results(): Record<string, AgentEvaluationResult[]>;
+    evaluate(options: {
+        agent: unknown;
+        task?: Task | null;
+        execution_trace?: Record<string, unknown>;
+        executionTrace?: Record<string, unknown>;
+        final_output?: unknown;
+        finalOutput?: unknown;
+        state?: ExecutionState;
+    }): AgentEvaluationResult;
+    get_agent_evaluation(strategy?: AggregationStrategy): Record<string, AgentAggregatedEvaluationResult>;
+    display_results_with_iterations(): string;
+    display_evaluation_with_feedback(): string;
+    emit_evaluation_started_event(agent_role: string, agent_id: string, task_id?: string | null): void;
+    emit_evaluation_completed_event(agent_role: string, agent_id: string, task_id?: string | null, metric_category?: MetricCategory | null, score?: EvaluationScore | null): void;
+    emit_evaluation_failed_event(agent_role: string, agent_id: string, error: unknown, task_id?: string | null): void;
+    private appendEvaluationResult;
+}
+export declare class ToolSelectionEvaluator extends BaseEvaluator {
+    constructor(llm?: LLM | string | null);
+    get metricCategory(): MetricCategory;
+    evaluate(agent: unknown, executionTrace: Record<string, unknown>, _finalOutput: unknown, task?: Task | null): EvaluationScore | Promise<EvaluationScore>;
+}
+export declare class ParameterExtractionEvaluator extends BaseEvaluator {
+    constructor(llm?: LLM | string | null);
+    get metricCategory(): MetricCategory;
+    evaluate(agent: unknown, executionTrace: Record<string, unknown>, _finalOutput: unknown, task?: Task | null): EvaluationScore | Promise<EvaluationScore>;
+}
+export declare class ToolInvocationEvaluator extends BaseEvaluator {
+    constructor(llm?: LLM | string | null);
+    get metricCategory(): MetricCategory;
+    evaluate(agent: unknown, executionTrace: Record<string, unknown>, _finalOutput: unknown, task?: Task | null): EvaluationScore | Promise<EvaluationScore>;
+}
+export declare const ReasoningPatternType: Readonly<{
+    readonly EFFICIENT: "efficient";
+    readonly LOOP: "loop";
+    readonly VERBOSE: "verbose";
+    readonly INDECISIVE: "indecisive";
+    readonly SCATTERED: "scattered";
+}>;
+export type ReasoningPatternType = typeof ReasoningPatternType[keyof typeof ReasoningPatternType];
+export declare class ReasoningEfficiencyEvaluator extends BaseEvaluator {
+    constructor(llm?: LLM | string | null);
+    get metricCategory(): MetricCategory;
+    evaluate(agent: unknown, executionTrace: Record<string, unknown>, finalOutput: unknown, task?: Task | null): EvaluationScore | Promise<EvaluationScore>;
+    _detect_loops(llmCalls: readonly unknown[]): [boolean, Array<Record<string, unknown>>];
+    _calculate_text_similarity(text1: string, text2: string): number;
+    _analyze_reasoning_patterns(llmCalls: readonly unknown[]): {
+        primary_pattern: ReasoningPatternType;
+        details: string;
+        metrics: {
+            avg_length: number;
+            std_length: number;
+            length_trend: number;
+            loop_score: number;
+        };
+    };
+    _calculate_trend(values: readonly number[]): number;
+    _calculate_loop_likelihood(callLengths: readonly number[], responseTimes?: readonly number[]): number;
+    _get_call_samples(llmCalls: readonly unknown[]): string;
+}
+export declare class GoalAlignmentEvaluator extends BaseEvaluator {
+    get metricCategory(): MetricCategory;
+    evaluate(agent: unknown, _executionTrace: Record<string, unknown>, finalOutput: unknown, task?: Task | null): EvaluationScore | Promise<EvaluationScore>;
+}
+export declare class SemanticQualityEvaluator extends BaseEvaluator {
+    get metricCategory(): MetricCategory;
+    evaluate(agent: unknown, _executionTrace: Record<string, unknown>, finalOutput: unknown, task?: Task | null): EvaluationScore | Promise<EvaluationScore>;
+}
+export declare function create_default_evaluator(llm?: LLM | string | null): BaseEvaluator[];
+export declare class EvaluationDisplayFormatter {
+    aggregateAgentResults(options: {
+        agentId?: string;
+        agent_id?: string;
+        agentRole?: string;
+        agent_role?: string;
+        results: readonly AgentEvaluationResult[];
+        strategy?: AggregationStrategy;
+    }): AgentAggregatedEvaluationResult;
+    _aggregate_agent_results(agentId: string, agentRole: string, results: readonly AgentEvaluationResult[], strategy?: AggregationStrategy): AgentAggregatedEvaluationResult;
+    display_evaluation_with_feedback(iterationsResults: Record<number, Record<string, AgentEvaluationResult[]>>): string;
+    display_summary_results(iterationsResults: Record<number, Record<string, AgentEvaluationResult[]>>): string;
+    private summarizeFeedbacks;
+    _summarize_feedbacks(_agentRole: string, _metric: string, feedbacks: readonly string[], _scores?: readonly (number | null)[], _strategy?: AggregationStrategy): string;
+    private formatIterations;
+}
+export declare class EvaluationTraceCallback {
+    readonly traces: Record<string, Record<string, unknown>>;
+    current_agent_id: string | null;
+    current_task_id: string | null;
+    current_llm_call: Record<string, unknown>;
+    private unsubscribeHandlers;
+    setupListeners(eventBus?: EventBus): void;
+    setup_listeners(eventBus?: EventBus): void;
+    disposeListeners(): void;
+    dispose_listeners(): void;
+    on_agent_started(_source: unknown, event: unknown): void;
+    on_lite_agent_started(_source: unknown, event: unknown): void;
+    on_agent_completed(_source: unknown, event: unknown): void;
+    on_lite_agent_completed(_source: unknown, event: unknown): void;
+    on_tool_completed(_source: unknown, event: unknown): void;
+    on_tool_usage_error(_source: unknown, event: unknown): void;
+    on_tool_execution_error(_source: unknown, event: unknown): void;
+    on_tool_selection_error(_source: unknown, event: unknown): void;
+    on_tool_validate_input_error(_source: unknown, event: unknown): void;
+    on_llm_call_started(_source: unknown, event: unknown): void;
+    on_llm_call_completed(_source: unknown, event: unknown): void;
+    on_agent_start(agent: unknown, task: unknown): void;
+    _init_trace(trace_key: string, trace?: Record<string, unknown>): void;
+    on_lite_agent_start(agentInfo: Record<string, unknown>): void;
+    on_agent_finish(agent: unknown, task: unknown, output: unknown): void;
+    _reset_current(): void;
+    on_lite_agent_finish(output: unknown): void;
+    get_trace(agent_id: string, task_id: string): Record<string, unknown>;
+    on_tool_use(tool: string, args: unknown, result: unknown, options?: {
+        success?: boolean;
+        error_type?: string | null;
+    }): void;
+    on_llm_call_start(messages: unknown, tools?: readonly Record<string, unknown>[] | null): void;
+    on_llm_call_end(messages: unknown, response: unknown, usage?: Record<string, unknown> | null): void;
+    private initTrace;
+    private resetCurrent;
+    private recordToolError;
+}
+export declare function create_evaluation_callbacks(): EvaluationTraceCallback;
+export declare class ExperimentResult {
+    readonly identifier: string;
+    readonly inputs: Record<string, unknown>;
+    readonly score: number | Record<string, number>;
+    readonly expected_score: number | Record<string, number>;
+    readonly passed: boolean;
+    readonly agent_evaluations: Record<string, unknown> | null;
+    constructor(options: {
+        identifier: string;
+        inputs?: Record<string, unknown>;
+        score: number | Record<string, number>;
+        expected_score?: number | Record<string, number>;
+        expectedScore?: number | Record<string, number>;
+        passed?: boolean;
+        agent_evaluations?: Record<string, unknown> | null;
+        agentEvaluations?: Record<string, unknown> | null;
+    });
+}
+export declare class ExperimentResults {
+    readonly results: ExperimentResult[];
+    readonly metadata: Record<string, unknown>;
+    readonly timestamp: Date;
+    readonly display: ExperimentResultsDisplay;
+    constructor(results: readonly ExperimentResult[], metadata?: Record<string, unknown>);
+    to_json(filepath?: string | null): Record<string, unknown>;
+    compare_with_baseline(baseline_filepath: string, save_current?: boolean, print_summary?: boolean): Record<string, unknown>;
+    compareWithBaseline(baselineFilepath: string, saveCurrent?: boolean, printSummary?: boolean): Record<string, unknown>;
+    _compare_with_run(baselineRun: Record<string, unknown>): Record<string, unknown>;
+    compareWithRun(baselineRun: Record<string, unknown>): Record<string, unknown>;
+}
+export type ExperimentScore = number | Record<string, number>;
+export declare class ExperimentRunner {
+    readonly dataset: Record<string, unknown>[];
+    evaluator: AgentEvaluator | null;
+    readonly display: ExperimentResultsDisplay;
+    constructor(dataset?: readonly Record<string, unknown>[]);
+    run(crewOrOptions?: {
+        crew?: {
+            agents?: readonly unknown[];
+            kickoff?: (options: {
+                inputs: Record<string, unknown>;
+            }) => unknown;
+        } | null;
+        agents?: readonly unknown[] | null;
+        print_summary?: boolean;
+        printSummary?: boolean;
+    } | {
+        agents?: readonly unknown[];
+        kickoff?: (options: {
+            inputs: Record<string, unknown>;
+        }) => unknown;
+    } | null, agentsArg?: readonly unknown[] | null, print_summary?: boolean): ExperimentResults;
+    _run_test_case(test_case: Record<string, unknown>, agents: readonly unknown[], crew?: {
+        kickoff?: (options: {
+            inputs: Record<string, unknown>;
+        }) => unknown;
+    } | null): ExperimentResult;
+    _extract_scores(agent_evaluations: Record<string, AgentAggregatedEvaluationResult>): ExperimentScore;
+    _assert_scores(expected: ExperimentScore, actual: ExperimentScore): boolean;
+}
+export type ExperimentResultsDisplayConsole = {
+    print?: (value: string) => void;
+    log?: (value: string) => void;
+};
+export declare class ExperimentResultsDisplay {
+    readonly console: ExperimentResultsDisplayConsole;
+    constructor(options?: {
+        console?: ExperimentResultsDisplayConsole;
+    });
+    summary(experiment_results: ExperimentResults): string;
+    comparison_summary(comparison: Record<string, unknown>, baseline_timestamp: string): string;
+    comparisonSummary(comparison: Record<string, unknown>, baselineTimestamp: string): string;
+    private formatComparisonRow;
+    private print;
+}
+export declare function assert_experiment_no_regression(comparison_result: Record<string, unknown>): void;
+export declare function _get_baseline_filepath_fallback(): string;
+export declare function assert_experiment_successfully(experiment_results: ExperimentResults, baseline_filepath?: string): void;
+export declare function run_experiment(dataset: readonly Record<string, unknown>[], crew?: {
+    agents?: readonly unknown[];
+    kickoff?: (options: {
+        inputs: Record<string, unknown>;
+    }) => unknown;
+} | null, agents?: readonly unknown[] | null, verbose?: boolean): ExperimentResults;
+export type Entity = {
+    name: string;
+    type: string;
+    description: string;
+    relationships: string[];
+};
+export type TaskEvaluation = {
+    suggestions: string[];
+    quality: number;
+    entities: Entity[];
+};
+export type TrainingTaskEvaluation = {
+    suggestions: string[];
+    quality: number;
+    final_summary?: string;
+    finalSummary?: string;
+};
+export type TaskEvaluationPydanticOutput = {
+    quality: number;
+};
+export declare const Entity: {
+    new (options: {
+        readonly name: string;
+        readonly type: string;
+        readonly description: string;
+        readonly relationships: string[];
+    }): {
+        readonly name: string;
+        readonly type: string;
+        readonly description: string;
+        readonly relationships: string[];
+    };
+};
+export declare const TaskEvaluation: {
+    new (options: {
+        readonly suggestions: string[];
+        readonly quality: number;
+        readonly entities: Entity[];
+    }): {
+        readonly suggestions: string[];
+        readonly quality: number;
+        readonly entities: Entity[];
+    };
+};
+export declare const TrainingTaskEvaluation: {
+    new (options: {
+        suggestions: string[];
+        quality: number;
+        final_summary?: string;
+        finalSummary?: string;
+    }): {
+        readonly suggestions: string[];
+        readonly quality: number;
+        readonly final_summary: string;
+        readonly finalSummary: string;
+    };
+};
+export declare const TaskEvaluationPydanticOutput: {
+    new (options: {
+        readonly quality: number;
+    }): {
+        readonly quality: number;
+    };
+};
+export type EvaluationAgentLike = {
+    llm: LLM | string | null;
+};
+export declare const taskEvaluationModel: StructuredModel<TaskEvaluation>;
+export declare const trainingTaskEvaluationModel: StructuredModel<TrainingTaskEvaluation>;
+export declare const taskEvaluationPydanticOutputModel: StructuredModel<TaskEvaluationPydanticOutput>;
+export declare class TaskEvaluator {
+    readonly llm: LLM | string | null;
+    readonly originalAgent: EvaluationAgentLike;
+    readonly original_agent: EvaluationAgentLike;
+    constructor(originalAgentOrOptions: EvaluationAgentLike | {
+        originalAgent?: EvaluationAgentLike;
+        original_agent?: EvaluationAgentLike;
+    });
+    evaluate(task: Task, output: string): Promise<TaskEvaluation>;
+    evaluate_training_data(trainingData: Record<string, unknown>, agentId: string): Promise<TrainingTaskEvaluation>;
+    evaluateTrainingData(trainingData: Record<string, unknown>, agentId: string): Promise<TrainingTaskEvaluation>;
+}
+export type CrewLikeForEvaluation = {
+    name?: string | null;
+    tasks: Task[];
+    taskCallback?: TaskCallback | null;
+};
+export declare class CrewEvaluator {
+    readonly crew: CrewLikeForEvaluation;
+    readonly llm: LLM | string | null;
+    tasksScores: Record<number, number[]>;
+    tasks_scores: Record<number, number[]>;
+    runExecutionTimes: Record<number, number[]>;
+    run_execution_times: Record<number, number[]>;
+    iteration: number;
+    private readonly previousTaskCallback;
+    constructor(crewOrOptions: CrewLikeForEvaluation | {
+        crew: CrewLikeForEvaluation;
+        evalLlm?: LLM | string | null;
+        eval_llm?: LLM | string | null;
+        openaiModelName?: string | null;
+        openai_model_name?: string | null;
+        llm?: LLM | string | null;
+    }, evalLlm?: LLM | string | null);
+    setIteration(iteration: number): void;
+    set_iteration(iteration: number): void;
+    evaluate(taskOutput: TaskOutput): Promise<void>;
+    printCrewEvaluationResult(): string;
+    print_crew_evaluation_result(): string;
+    private setupForEvaluating;
+    private evaluatorAgent;
+    static evaluationTask(evaluatorAgent: Agent, taskToEvaluate: Task, taskOutput: string): Task;
+    static _evaluation_task(evaluatorAgent: Agent, taskToEvaluate: Task, taskOutput: string): Task;
+}
+export declare const TaskEvaluationPydanticOutputModel: {
+    modelValidate?: (value: unknown) => TaskEvaluationPydanticOutput;
+    model_validate?: (value: unknown) => TaskEvaluationPydanticOutput;
+    modelValidateJson?: (value: string) => TaskEvaluationPydanticOutput;
+    model_validate_json?: (value: string) => TaskEvaluationPydanticOutput;
+    modelDump?: (value: TaskEvaluationPydanticOutput) => Record<string, unknown>;
+    model_dump?: (value: TaskEvaluationPydanticOutput) => Record<string, unknown>;
+    schema?: unknown;
+    name?: string;
+};
+export declare const TaskEvaluationModel: {
+    modelValidate?: (value: unknown) => TaskEvaluation;
+    model_validate?: (value: unknown) => TaskEvaluation;
+    modelValidateJson?: (value: string) => TaskEvaluation;
+    model_validate_json?: (value: string) => TaskEvaluation;
+    modelDump?: (value: TaskEvaluation) => Record<string, unknown>;
+    model_dump?: (value: TaskEvaluation) => Record<string, unknown>;
+    schema?: unknown;
+    name?: string;
+};
+export declare const TrainingTaskEvaluationModel: {
+    modelValidate?: (value: unknown) => TrainingTaskEvaluation;
+    model_validate?: (value: unknown) => TrainingTaskEvaluation;
+    modelValidateJson?: (value: string) => TrainingTaskEvaluation;
+    model_validate_json?: (value: string) => TrainingTaskEvaluation;
+    modelDump?: (value: TrainingTaskEvaluation) => Record<string, unknown>;
+    model_dump?: (value: TrainingTaskEvaluation) => Record<string, unknown>;
+    schema?: unknown;
+    name?: string;
+};