@lov3kaizen/agentsea-evaluate 0.5.1

package/dist/index.d.mts

@@ -0,0 +1,81 @@
+import { T as ThumbsCollector, F as FeedbackStoreInterface, a as ThumbsFeedback } from './index-BNTycFEA.mjs';
+export { s as AggregationMetric, A as AggregationOptions, t as AggregationResult, O as BaseCollector, B as BaseFeedbackEntry, o as CollectCorrectionInput, p as CollectMultiCriteriaInput, n as CollectPreferenceInput, D as CollectPreferenceInputSchema, m as CollectRatingInput, l as CollectThumbsInput, z as CollectThumbsInputSchema, Y as CorrectionCollector, C as CorrectionFeedback, d as CriterionDefinition, e as CriterionRating, E as ExportFormat, u as ExportOptions, J as FeedbackAggregator, g as FeedbackCollectorOptions, f as FeedbackEntry, L as FeedbackExporter, q as FeedbackQueryOptions, r as FeedbackQueryResult, v as FeedbackStoreConfig, G as MemoryFeedbackStore, _ as MultiCriteriaCollector, k as MultiCriteriaCollectorOptions, M as MultiCriteriaFeedback, P as PreferenceChoice, y as PreferenceChoiceSchema, W as PreferenceCollector, j as PreferenceCollectorOptions, c as PreferenceFeedback, U as RatingCollector, i as RatingCollectorOptions, R as RatingFeedback, H as SQLiteFeedbackStore, S as StarRating, x as StarRatingSchema, h as ThumbsCollectorOptions, b as ThumbsRating, w as ThumbsRatingSchema, Z as createCorrectionCollector, K as createFeedbackAggregator, N as createFeedbackExporter, I as createFeedbackStore, $ as createMultiCriteriaCollector, X as createPreferenceCollector, V as createRatingCollector, Q as createThumbsCollector } from './index-BNTycFEA.mjs';
+import { E as EvaluationPipeline, a as EvalDataset, P as PipelineEvaluationResult } from './index-CTYCfWfH.mjs';
+export { X as Accuracy, A as AccuracyMetricConfig, W as BaseMetric, B as BaseMetricConfig, aj as CodeQualityRubric, $ as Coherence, C as CoherenceMetricConfig, al as ComparativeJudge, m as ComparativeJudgeConfig, n as ComparisonInput, o as ComparisonResult, an as ConsensusJudge, p as ConsensusJudgeConfig, a5 as ContextRelevance, d as ContextRelevanceMetricConfig, a7 as CustomMetric, e as CustomMetricConfig, t as EvalDatasetConfig, K as EvalDatasetInterface, s as EvalDatasetItem, Q as EvalRunner, N as EvalRunnerConfig, x as EvaluationError, g as EvaluationInput, u as EvaluationPipelineConfig, w as EvaluationProgress, I as EvaluationSummary, D as FailureAnalysis, G as FailureFilterOptions, a3 as Faithfulness, F as FaithfulnessMetricConfig, H as HFDatasetConfig, ak as HelpfulnessRubric, h as JudgeCriterion, q as JudgeInterface, r as JudgeResult, J as JudgeType, ae as LLMJudge, L as LLMJudgeConfig, i as LLMProviderInterface, f as MetricInterface, b as MetricResult, M as MetricType, z as MetricsSummary, v as PipelineEvaluationOptions, ai as QualityRubric, Z as Relevance, R as RelevanceMetricConfig, k as RubricConfig, ag as RubricJudge, l as RubricJudgeConfig, j as RubricLevel, S as ScoreRange, y as SingleEvaluationResult, a1 as Toxicity, c as ToxicityCategory, T as ToxicityMetricConfig, Y as createAccuracyMetric, a0 as createCoherenceMetric, am as createComparativeJudge, ao as createConsensusJudge, ad as createContainsMetric, a6 as createContextRelevanceMetric, a8 as createCustomMetric, O as createEvalDataset, U as createEvalRunner, V as createEvaluationPipeline, a4 as createFaithfulnessMetric, ac as createJSONMetric, af as createLLMJudge, aa as createLengthMetric, ab as createRegexMetric, _ as createRelevanceMetric, ah as createRubricJudge, a9 as createSimpleMetric, a2 as createToxicityMetric } from './index-CTYCfWfH.mjs';
+export { d as Annotation, o as AnnotationEvent, n as AnnotationEventType, c as AnnotationItem, a as AnnotationItemStatus, e as AnnotationQuality, t as AnnotationQueue, h as AnnotationQueueConfig, m as AnnotationResults, p as AnnotationTask, b as AnnotationTaskConfig, A as AnnotationTaskStatus, f as Annotator, g as AnnotatorStats, i as AssignmentStrategy, B as BatchAssignment, r as BinaryClassificationSchema, j as ConsensusConfig, v as ConsensusManager, C as ConsensusMethod, k as ConsensusResult, D as Disagreement, G as GoldStandardItem, I as IAnnotationTask, P as PrioritizationType, Q as QualityControlConfig, s as QualityRatingSchema, l as QueueStats, T as TextSpanSchema, u as createAnnotationQueue, q as createAnnotationTask, w as createConsensusManager } from './index-bPghFsfP.mjs';
+export { A as AnthropicFormatItem, b as ConversationExample, C as ConversationTurn, n as DPOFormatItem, f as DatasetBuilderConfig, a as DatasetExportFormat, m as DatasetExportOptions, v as DatasetExporter, i as DatasetFilterConfig, c as DatasetItem, g as DatasetQueryOptions, h as DatasetQueryResult, d as DatasetStats, D as DatasetType, q as DatasetValidationError, p as DatasetValidationResult, r as DatasetValidationWarning, E as ExportResult, F as FeedbackStoreRef, H as HFExportOptions, k as InstructionBuildOptions, I as InstructionExample, O as OpenAIFormatItem, j as PreferenceBuildOptions, s as PreferenceDataset, t as PreferenceDatasetBuilder, l as PreferenceDatasetInterface, P as PreferencePair, Q as QAExample, R as RLHFFormatItem, o as SFTFormatItem, e as SamplingConfig, S as SamplingStrategyType, w as createDatasetExporter, u as createPreferenceDatasetBuilder } from './index-6Pbiq7ny.mjs';
+export { r as ABMetricResult, p as ABTest, m as ABTestConfig, q as ABTestResults, G as ABTestRunner, o as ABTestStatus, n as ABTestVariants, g as Alert, d as AlertChannelConfig, A as AlertChannelType, z as AlertManager, f as AlertManagerConfig, h as AlertNotification, e as AlertRule, B as BaselineMetrics, x as ContinuousEval, C as ContinuousEvalConfig, w as ContinuousEvalEvent, v as ContinuousEvalEventType, c as ContinuousEvalStats, D as DashboardUpdate, a as EvalInput, b as EvalOutput, E as EvaluationPipelineRef, H as HistoricalQueryOptions, i as MetricBaseline, l as MetricImprovement, k as MetricRegression, s as MetricSummary, M as MonitoringStatus, R as RegressionDetectorConfig, j as RegressionResult, S as SampleAssignment, u as ScheduleConfig, t as TimeSeries, T as TimeSeriesPoint, V as VariantConfig, I as createABTestRunner, F as createAlertManager, y as createContinuousEval } from './index-Cq5LwG_3.mjs';
+import 'zod';
+import 'eventemitter3';
+
+interface FeedbackMiddlewareOptions {
+    collector?: ThumbsCollector;
+    store?: FeedbackStoreInterface;
+    autoCapture?: boolean;
+    captureFields?: string[];
+}
+interface AgentMessage {
+    id: string;
+    role: 'user' | 'assistant';
+    content: string;
+    metadata?: Record<string, unknown>;
+}
+interface AgentContext {
+    conversationId: string;
+    messages: AgentMessage[];
+    metadata?: Record<string, unknown>;
+}
+declare class FeedbackMiddleware {
+    private collector;
+    private autoCapture;
+    private captureFields;
+    private pendingFeedback;
+    constructor(options: FeedbackMiddlewareOptions);
+    capture(context: AgentContext): void;
+    recordFeedback(responseId: string, rating: 'up' | 'down', comment?: string, userId?: string): Promise<ThumbsFeedback | null>;
+    getPendingIds(): string[];
+    clearPending(): void;
+    private cleanupPending;
+    getCollector(): ThumbsCollector;
+}
+declare function createFeedbackMiddleware(options: FeedbackMiddlewareOptions): FeedbackMiddleware;
+
+interface EvaluationScenario {
+    category: string;
+    dataset: EvalDataset;
+    weight?: number;
+}
+interface AgentEvaluatorOptions {
+    pipeline: EvaluationPipeline;
+    scenarios: EvaluationScenario[];
+}
+interface AgentInterface {
+    execute(input: string, context?: unknown): Promise<string>;
+}
+interface AgentEvaluationResult {
+    overallScore: number;
+    categoryScores: Record<string, number>;
+    categoryResults: Record<string, PipelineEvaluationResult>;
+    recommendations: string[];
+    summary: {
+        totalTests: number;
+        passed: number;
+        failed: number;
+        passRate: number;
+    };
+}
+declare class AgentEvaluator {
+    private pipeline;
+    private scenarios;
+    constructor(options: AgentEvaluatorOptions);
+    evaluate(agent: AgentInterface): Promise<AgentEvaluationResult>;
+    benchmark(agent: AgentInterface, sampleSize?: number): Promise<{
+        score: number;
+        latencyMs: number;
+    }>;
+    addScenario(scenario: EvaluationScenario): void;
+    getScenarios(): EvaluationScenario[];
+}
+declare function createAgentEvaluator(options: AgentEvaluatorOptions): AgentEvaluator;
+
+export { type AgentContext, type AgentEvaluationResult, AgentEvaluator, type AgentEvaluatorOptions, type AgentInterface, type AgentMessage, EvalDataset, EvaluationPipeline, type EvaluationScenario, FeedbackMiddleware, type FeedbackMiddlewareOptions, FeedbackStoreInterface, PipelineEvaluationResult, ThumbsCollector, ThumbsFeedback, createAgentEvaluator, createFeedbackMiddleware };

package/dist/index.d.ts

@@ -0,0 +1,81 @@
+import { T as ThumbsCollector, F as FeedbackStoreInterface, a as ThumbsFeedback } from './index-BNTycFEA.js';
+export { s as AggregationMetric, A as AggregationOptions, t as AggregationResult, O as BaseCollector, B as BaseFeedbackEntry, o as CollectCorrectionInput, p as CollectMultiCriteriaInput, n as CollectPreferenceInput, D as CollectPreferenceInputSchema, m as CollectRatingInput, l as CollectThumbsInput, z as CollectThumbsInputSchema, Y as CorrectionCollector, C as CorrectionFeedback, d as CriterionDefinition, e as CriterionRating, E as ExportFormat, u as ExportOptions, J as FeedbackAggregator, g as FeedbackCollectorOptions, f as FeedbackEntry, L as FeedbackExporter, q as FeedbackQueryOptions, r as FeedbackQueryResult, v as FeedbackStoreConfig, G as MemoryFeedbackStore, _ as MultiCriteriaCollector, k as MultiCriteriaCollectorOptions, M as MultiCriteriaFeedback, P as PreferenceChoice, y as PreferenceChoiceSchema, W as PreferenceCollector, j as PreferenceCollectorOptions, c as PreferenceFeedback, U as RatingCollector, i as RatingCollectorOptions, R as RatingFeedback, H as SQLiteFeedbackStore, S as StarRating, x as StarRatingSchema, h as ThumbsCollectorOptions, b as ThumbsRating, w as ThumbsRatingSchema, Z as createCorrectionCollector, K as createFeedbackAggregator, N as createFeedbackExporter, I as createFeedbackStore, $ as createMultiCriteriaCollector, X as createPreferenceCollector, V as createRatingCollector, Q as createThumbsCollector } from './index-BNTycFEA.js';
+import { E as EvaluationPipeline, a as EvalDataset, P as PipelineEvaluationResult } from './index-CTYCfWfH.js';
+export { X as Accuracy, A as AccuracyMetricConfig, W as BaseMetric, B as BaseMetricConfig, aj as CodeQualityRubric, $ as Coherence, C as CoherenceMetricConfig, al as ComparativeJudge, m as ComparativeJudgeConfig, n as ComparisonInput, o as ComparisonResult, an as ConsensusJudge, p as ConsensusJudgeConfig, a5 as ContextRelevance, d as ContextRelevanceMetricConfig, a7 as CustomMetric, e as CustomMetricConfig, t as EvalDatasetConfig, K as EvalDatasetInterface, s as EvalDatasetItem, Q as EvalRunner, N as EvalRunnerConfig, x as EvaluationError, g as EvaluationInput, u as EvaluationPipelineConfig, w as EvaluationProgress, I as EvaluationSummary, D as FailureAnalysis, G as FailureFilterOptions, a3 as Faithfulness, F as FaithfulnessMetricConfig, H as HFDatasetConfig, ak as HelpfulnessRubric, h as JudgeCriterion, q as JudgeInterface, r as JudgeResult, J as JudgeType, ae as LLMJudge, L as LLMJudgeConfig, i as LLMProviderInterface, f as MetricInterface, b as MetricResult, M as MetricType, z as MetricsSummary, v as PipelineEvaluationOptions, ai as QualityRubric, Z as Relevance, R as RelevanceMetricConfig, k as RubricConfig, ag as RubricJudge, l as RubricJudgeConfig, j as RubricLevel, S as ScoreRange, y as SingleEvaluationResult, a1 as Toxicity, c as ToxicityCategory, T as ToxicityMetricConfig, Y as createAccuracyMetric, a0 as createCoherenceMetric, am as createComparativeJudge, ao as createConsensusJudge, ad as createContainsMetric, a6 as createContextRelevanceMetric, a8 as createCustomMetric, O as createEvalDataset, U as createEvalRunner, V as createEvaluationPipeline, a4 as createFaithfulnessMetric, ac as createJSONMetric, af as createLLMJudge, aa as createLengthMetric, ab as createRegexMetric, _ as createRelevanceMetric, ah as createRubricJudge, a9 as createSimpleMetric, a2 as createToxicityMetric } from './index-CTYCfWfH.js';
+export { d as Annotation, o as AnnotationEvent, n as AnnotationEventType, c as AnnotationItem, a as AnnotationItemStatus, e as AnnotationQuality, t as AnnotationQueue, h as AnnotationQueueConfig, m as AnnotationResults, p as AnnotationTask, b as AnnotationTaskConfig, A as AnnotationTaskStatus, f as Annotator, g as AnnotatorStats, i as AssignmentStrategy, B as BatchAssignment, r as BinaryClassificationSchema, j as ConsensusConfig, v as ConsensusManager, C as ConsensusMethod, k as ConsensusResult, D as Disagreement, G as GoldStandardItem, I as IAnnotationTask, P as PrioritizationType, Q as QualityControlConfig, s as QualityRatingSchema, l as QueueStats, T as TextSpanSchema, u as createAnnotationQueue, q as createAnnotationTask, w as createConsensusManager } from './index-bPghFsfP.js';
+export { A as AnthropicFormatItem, b as ConversationExample, C as ConversationTurn, n as DPOFormatItem, f as DatasetBuilderConfig, a as DatasetExportFormat, m as DatasetExportOptions, v as DatasetExporter, i as DatasetFilterConfig, c as DatasetItem, g as DatasetQueryOptions, h as DatasetQueryResult, d as DatasetStats, D as DatasetType, q as DatasetValidationError, p as DatasetValidationResult, r as DatasetValidationWarning, E as ExportResult, F as FeedbackStoreRef, H as HFExportOptions, k as InstructionBuildOptions, I as InstructionExample, O as OpenAIFormatItem, j as PreferenceBuildOptions, s as PreferenceDataset, t as PreferenceDatasetBuilder, l as PreferenceDatasetInterface, P as PreferencePair, Q as QAExample, R as RLHFFormatItem, o as SFTFormatItem, e as SamplingConfig, S as SamplingStrategyType, w as createDatasetExporter, u as createPreferenceDatasetBuilder } from './index-6Pbiq7ny.js';
+export { r as ABMetricResult, p as ABTest, m as ABTestConfig, q as ABTestResults, G as ABTestRunner, o as ABTestStatus, n as ABTestVariants, g as Alert, d as AlertChannelConfig, A as AlertChannelType, z as AlertManager, f as AlertManagerConfig, h as AlertNotification, e as AlertRule, B as BaselineMetrics, x as ContinuousEval, C as ContinuousEvalConfig, w as ContinuousEvalEvent, v as ContinuousEvalEventType, c as ContinuousEvalStats, D as DashboardUpdate, a as EvalInput, b as EvalOutput, E as EvaluationPipelineRef, H as HistoricalQueryOptions, i as MetricBaseline, l as MetricImprovement, k as MetricRegression, s as MetricSummary, M as MonitoringStatus, R as RegressionDetectorConfig, j as RegressionResult, S as SampleAssignment, u as ScheduleConfig, t as TimeSeries, T as TimeSeriesPoint, V as VariantConfig, I as createABTestRunner, F as createAlertManager, y as createContinuousEval } from './index-Cq5LwG_3.js';
+import 'zod';
+import 'eventemitter3';
+
+interface FeedbackMiddlewareOptions {
+    collector?: ThumbsCollector;
+    store?: FeedbackStoreInterface;
+    autoCapture?: boolean;
+    captureFields?: string[];
+}
+interface AgentMessage {
+    id: string;
+    role: 'user' | 'assistant';
+    content: string;
+    metadata?: Record<string, unknown>;
+}
+interface AgentContext {
+    conversationId: string;
+    messages: AgentMessage[];
+    metadata?: Record<string, unknown>;
+}
+declare class FeedbackMiddleware {
+    private collector;
+    private autoCapture;
+    private captureFields;
+    private pendingFeedback;
+    constructor(options: FeedbackMiddlewareOptions);
+    capture(context: AgentContext): void;
+    recordFeedback(responseId: string, rating: 'up' | 'down', comment?: string, userId?: string): Promise<ThumbsFeedback | null>;
+    getPendingIds(): string[];
+    clearPending(): void;
+    private cleanupPending;
+    getCollector(): ThumbsCollector;
+}
+declare function createFeedbackMiddleware(options: FeedbackMiddlewareOptions): FeedbackMiddleware;
+
+interface EvaluationScenario {
+    category: string;
+    dataset: EvalDataset;
+    weight?: number;
+}
+interface AgentEvaluatorOptions {
+    pipeline: EvaluationPipeline;
+    scenarios: EvaluationScenario[];
+}
+interface AgentInterface {
+    execute(input: string, context?: unknown): Promise<string>;
+}
+interface AgentEvaluationResult {
+    overallScore: number;
+    categoryScores: Record<string, number>;
+    categoryResults: Record<string, PipelineEvaluationResult>;
+    recommendations: string[];
+    summary: {
+        totalTests: number;
+        passed: number;
+        failed: number;
+        passRate: number;
+    };
+}
+declare class AgentEvaluator {
+    private pipeline;
+    private scenarios;
+    constructor(options: AgentEvaluatorOptions);
+    evaluate(agent: AgentInterface): Promise<AgentEvaluationResult>;
+    benchmark(agent: AgentInterface, sampleSize?: number): Promise<{
+        score: number;
+        latencyMs: number;
+    }>;
+    addScenario(scenario: EvaluationScenario): void;
+    getScenarios(): EvaluationScenario[];
+}
+declare function createAgentEvaluator(options: AgentEvaluatorOptions): AgentEvaluator;
+
+export { type AgentContext, type AgentEvaluationResult, AgentEvaluator, type AgentEvaluatorOptions, type AgentInterface, type AgentMessage, EvalDataset, EvaluationPipeline, type EvaluationScenario, FeedbackMiddleware, type FeedbackMiddlewareOptions, FeedbackStoreInterface, PipelineEvaluationResult, ThumbsCollector, ThumbsFeedback, createAgentEvaluator, createFeedbackMiddleware };