@lov3kaizen/agentsea-evaluate 0.5.1

package/dist/index-BNTycFEA.d.ts

@@ -0,0 +1,479 @@
+import { z } from 'zod';
+
+type ThumbsRating = 'up' | 'down';
+type StarRating = 1 | 2 | 3 | 4 | 5;
+type PreferenceChoice = 'A' | 'B' | 'tie';
+interface BaseFeedbackEntry {
+    id: string;
+    responseId: string;
+    conversationId?: string;
+    input: string;
+    output: string;
+    userId?: string;
+    timestamp: number;
+    metadata?: Record<string, unknown>;
+}
+interface ThumbsFeedback extends BaseFeedbackEntry {
+    type: 'thumbs';
+    rating: ThumbsRating;
+    comment?: string;
+}
+interface RatingFeedback extends BaseFeedbackEntry {
+    type: 'rating';
+    rating: StarRating;
+    maxRating?: number;
+    comment?: string;
+}
+interface PreferenceFeedback extends BaseFeedbackEntry {
+    type: 'preference';
+    responseA: {
+        id: string;
+        content: string;
+        model?: string;
+    };
+    responseB: {
+        id: string;
+        content: string;
+        model?: string;
+    };
+    preference: PreferenceChoice;
+    reason?: string;
+    confidence?: number;
+}
+interface CorrectionFeedback extends BaseFeedbackEntry {
+    type: 'correction';
+    correctedOutput: string;
+    correctionType: 'factual' | 'grammar' | 'style' | 'completeness' | 'other';
+    explanation?: string;
+}
+interface MultiCriteriaFeedback extends BaseFeedbackEntry {
+    type: 'multi_criteria';
+    criteria: CriterionRating[];
+    overallRating?: StarRating;
+    comment?: string;
+}
+interface CriterionDefinition {
+    name: string;
+    description: string;
+    scale: [number, number];
+    weight?: number;
+}
+interface CriterionRating {
+    name: string;
+    rating: number;
+    correction?: string;
+}
+type FeedbackEntry = ThumbsFeedback | RatingFeedback | PreferenceFeedback | CorrectionFeedback | MultiCriteriaFeedback;
+interface FeedbackCollectorOptions {
+    store?: FeedbackStoreInterface;
+    autoTimestamp?: boolean;
+    generateId?: () => string;
+    validateInput?: boolean;
+}
+interface ThumbsCollectorOptions extends FeedbackCollectorOptions {
+    allowComment?: boolean;
+    requireComment?: 'always' | 'on_down' | 'never';
+}
+interface RatingCollectorOptions extends FeedbackCollectorOptions {
+    allowComment?: boolean;
+    minRating?: StarRating;
+    maxRating?: StarRating;
+    requireComment?: 'always' | 'on_low' | 'never';
+    lowRatingThreshold?: number;
+}
+interface PreferenceCollectorOptions extends FeedbackCollectorOptions {
+    allowTie?: boolean;
+    requireReason?: boolean;
+    requireConfidence?: boolean;
+    minConfidence?: number;
+}
+interface MultiCriteriaCollectorOptions extends FeedbackCollectorOptions {
+    criteria: CriterionDefinition[];
+    requireAllCriteria?: boolean;
+    allowCorrections?: boolean;
+}
+interface CollectThumbsInput {
+    responseId: string;
+    conversationId?: string;
+    input: string;
+    output: string;
+    feedback: {
+        rating: ThumbsRating;
+        comment?: string;
+    };
+    userId?: string;
+    metadata?: Record<string, unknown>;
+}
+interface CollectRatingInput {
+    responseId: string;
+    conversationId?: string;
+    input: string;
+    output: string;
+    feedback: {
+        rating: StarRating;
+        comment?: string;
+    };
+    userId?: string;
+    metadata?: Record<string, unknown>;
+}
+interface CollectPreferenceInput {
+    input: string;
+    responseA: {
+        id: string;
+        content: string;
+        model?: string;
+    };
+    responseB: {
+        id: string;
+        content: string;
+        model?: string;
+    };
+    preference: PreferenceChoice;
+    reason?: string;
+    confidence?: number;
+    userId?: string;
+    metadata?: Record<string, unknown>;
+}
+interface CollectCorrectionInput {
+    responseId: string;
+    conversationId?: string;
+    input: string;
+    output: string;
+    correctedOutput: string;
+    correctionType: 'factual' | 'grammar' | 'style' | 'completeness' | 'other';
+    explanation?: string;
+    userId?: string;
+    metadata?: Record<string, unknown>;
+}
+interface CollectMultiCriteriaInput {
+    responseId: string;
+    conversationId?: string;
+    input: string;
+    output: string;
+    ratings: Record<string, number>;
+    corrections?: Record<string, string>;
+    overallRating?: StarRating;
+    comment?: string;
+    userId?: string;
+    metadata?: Record<string, unknown>;
+}
+interface FeedbackStoreInterface {
+    save(entry: FeedbackEntry): Promise<string>;
+    saveBatch(entries: FeedbackEntry[]): Promise<string[]>;
+    get(id: string): Promise<FeedbackEntry | null>;
+    query(options: FeedbackQueryOptions): Promise<FeedbackQueryResult>;
+    delete(id: string): Promise<boolean>;
+    clear(): Promise<void>;
+    close(): Promise<void>;
+}
+interface FeedbackQueryOptions {
+    type?: FeedbackEntry['type'] | FeedbackEntry['type'][];
+    userId?: string;
+    conversationId?: string;
+    responseId?: string;
+    startTime?: number;
+    endTime?: number;
+    limit?: number;
+    offset?: number;
+    orderBy?: 'timestamp' | 'rating';
+    orderDir?: 'asc' | 'desc';
+    metadata?: Record<string, unknown>;
+}
+interface FeedbackQueryResult {
+    entries: FeedbackEntry[];
+    total: number;
+    hasMore: boolean;
+}
+interface AggregationOptions {
+    groupBy?: 'model' | 'userId' | 'hour' | 'day' | 'week' | 'month';
+    metrics: AggregationMetric[];
+    timeRange?: {
+        start: number;
+        end: number;
+    };
+    filters?: FeedbackQueryOptions;
+}
+type AggregationMetric = 'thumbsUpRate' | 'avgRating' | 'correctionRate' | 'preferenceWinRate' | 'count' | 'avgCriteriaRating';
+interface AggregationResult {
+    groupKey: string;
+    metrics: Record<AggregationMetric, number>;
+    count: number;
+}
+type ExportFormat = 'json' | 'csv' | 'jsonl';
+interface ExportOptions {
+    format: ExportFormat;
+    query?: FeedbackQueryOptions;
+    fields?: string[];
+    includeMetadata?: boolean;
+}
+interface FeedbackStoreConfig {
+    type: 'memory' | 'sqlite' | 'postgres';
+    path?: string;
+    connectionString?: string;
+    tableName?: string;
+}
+declare const ThumbsRatingSchema: z.ZodEnum<["up", "down"]>;
+declare const StarRatingSchema: z.ZodUnion<[z.ZodLiteral<1>, z.ZodLiteral<2>, z.ZodLiteral<3>, z.ZodLiteral<4>, z.ZodLiteral<5>]>;
+declare const PreferenceChoiceSchema: z.ZodEnum<["A", "B", "tie"]>;
+declare const CollectThumbsInputSchema: z.ZodObject<{
+    responseId: z.ZodString;
+    conversationId: z.ZodOptional<z.ZodString>;
+    input: z.ZodString;
+    output: z.ZodString;
+    feedback: z.ZodObject<{
+        rating: z.ZodEnum<["up", "down"]>;
+        comment: z.ZodOptional<z.ZodString>;
+    }, "strip", z.ZodTypeAny, {
+        rating: "up" | "down";
+        comment?: string | undefined;
+    }, {
+        rating: "up" | "down";
+        comment?: string | undefined;
+    }>;
+    userId: z.ZodOptional<z.ZodString>;
+    metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+}, "strip", z.ZodTypeAny, {
+    responseId: string;
+    input: string;
+    output: string;
+    feedback: {
+        rating: "up" | "down";
+        comment?: string | undefined;
+    };
+    userId?: string | undefined;
+    conversationId?: string | undefined;
+    metadata?: Record<string, unknown> | undefined;
+}, {
+    responseId: string;
+    input: string;
+    output: string;
+    feedback: {
+        rating: "up" | "down";
+        comment?: string | undefined;
+    };
+    userId?: string | undefined;
+    conversationId?: string | undefined;
+    metadata?: Record<string, unknown> | undefined;
+}>;
+declare const CollectPreferenceInputSchema: z.ZodObject<{
+    input: z.ZodString;
+    responseA: z.ZodObject<{
+        id: z.ZodString;
+        content: z.ZodString;
+        model: z.ZodOptional<z.ZodString>;
+    }, "strip", z.ZodTypeAny, {
+        id: string;
+        content: string;
+        model?: string | undefined;
+    }, {
+        id: string;
+        content: string;
+        model?: string | undefined;
+    }>;
+    responseB: z.ZodObject<{
+        id: z.ZodString;
+        content: z.ZodString;
+        model: z.ZodOptional<z.ZodString>;
+    }, "strip", z.ZodTypeAny, {
+        id: string;
+        content: string;
+        model?: string | undefined;
+    }, {
+        id: string;
+        content: string;
+        model?: string | undefined;
+    }>;
+    preference: z.ZodEnum<["A", "B", "tie"]>;
+    reason: z.ZodOptional<z.ZodString>;
+    confidence: z.ZodOptional<z.ZodNumber>;
+    userId: z.ZodOptional<z.ZodString>;
+    metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+}, "strip", z.ZodTypeAny, {
+    preference: "A" | "B" | "tie";
+    input: string;
+    responseA: {
+        id: string;
+        content: string;
+        model?: string | undefined;
+    };
+    responseB: {
+        id: string;
+        content: string;
+        model?: string | undefined;
+    };
+    userId?: string | undefined;
+    metadata?: Record<string, unknown> | undefined;
+    reason?: string | undefined;
+    confidence?: number | undefined;
+}, {
+    preference: "A" | "B" | "tie";
+    input: string;
+    responseA: {
+        id: string;
+        content: string;
+        model?: string | undefined;
+    };
+    responseB: {
+        id: string;
+        content: string;
+        model?: string | undefined;
+    };
+    userId?: string | undefined;
+    metadata?: Record<string, unknown> | undefined;
+    reason?: string | undefined;
+    confidence?: number | undefined;
+}>;
+
+declare abstract class BaseCollector<TInput, TEntry extends FeedbackEntry> {
+    protected store?: FeedbackStoreInterface;
+    protected autoTimestamp: boolean;
+    protected generateId: () => string;
+    protected validateInput: boolean;
+    constructor(options?: FeedbackCollectorOptions);
+    collect(input: TInput): Promise<TEntry>;
+    collectBatch(inputs: TInput[]): Promise<TEntry[]>;
+    protected abstract validate(input: TInput): void;
+    protected abstract transform(input: TInput): TEntry;
+    setStore(store: FeedbackStoreInterface): void;
+    getStore(): FeedbackStoreInterface | undefined;
+}
+
+declare class ThumbsCollector extends BaseCollector<CollectThumbsInput, ThumbsFeedback> {
+    private allowComment;
+    private requireComment;
+    constructor(options?: ThumbsCollectorOptions);
+    protected validate(input: CollectThumbsInput): void;
+    protected transform(input: CollectThumbsInput): ThumbsFeedback;
+}
+declare function createThumbsCollector(options?: ThumbsCollectorOptions): ThumbsCollector;
+
+declare class RatingCollector extends BaseCollector<CollectRatingInput, RatingFeedback> {
+    private allowComment;
+    private minRating;
+    private maxRating;
+    private requireComment;
+    private lowRatingThreshold;
+    constructor(options?: RatingCollectorOptions);
+    protected validate(input: CollectRatingInput): void;
+    protected transform(input: CollectRatingInput): RatingFeedback;
+}
+declare function createRatingCollector(options?: RatingCollectorOptions): RatingCollector;
+
+declare class PreferenceCollector extends BaseCollector<CollectPreferenceInput, PreferenceFeedback> {
+    private allowTie;
+    private requireReason;
+    private requireConfidence;
+    private minConfidence;
+    constructor(options?: PreferenceCollectorOptions);
+    protected validate(input: CollectPreferenceInput): void;
+    protected transform(input: CollectPreferenceInput): PreferenceFeedback;
+}
+declare function createPreferenceCollector(options?: PreferenceCollectorOptions): PreferenceCollector;
+
+declare class CorrectionCollector extends BaseCollector<CollectCorrectionInput, CorrectionFeedback> {
+    constructor(options?: FeedbackCollectorOptions);
+    protected validate(input: CollectCorrectionInput): void;
+    protected transform(input: CollectCorrectionInput): CorrectionFeedback;
+}
+declare function createCorrectionCollector(options?: FeedbackCollectorOptions): CorrectionCollector;
+
+declare class MultiCriteriaCollector extends BaseCollector<CollectMultiCriteriaInput, MultiCriteriaFeedback> {
+    private criteria;
+    private requireAllCriteria;
+    private allowCorrections;
+    constructor(options: MultiCriteriaCollectorOptions);
+    protected validate(input: CollectMultiCriteriaInput): void;
+    protected transform(input: CollectMultiCriteriaInput): MultiCriteriaFeedback;
+    getCriteria(): CriterionDefinition[];
+    addCriterion(criterion: CriterionDefinition): void;
+    removeCriterion(name: string): boolean;
+}
+declare function createMultiCriteriaCollector(options: MultiCriteriaCollectorOptions): MultiCriteriaCollector;
+
+declare class MemoryFeedbackStore implements FeedbackStoreInterface {
+    private entries;
+    save(entry: FeedbackEntry): Promise<string>;
+    saveBatch(entries: FeedbackEntry[]): Promise<string[]>;
+    get(id: string): Promise<FeedbackEntry | null>;
+    query(options: FeedbackQueryOptions): Promise<FeedbackQueryResult>;
+    delete(id: string): Promise<boolean>;
+    clear(): Promise<void>;
+    close(): Promise<void>;
+    private getRating;
+}
+declare class SQLiteFeedbackStore implements FeedbackStoreInterface {
+    private config;
+    private db;
+    private tableName;
+    private initialized;
+    constructor(config: {
+        path: string;
+        tableName?: string;
+    });
+    init(): Promise<void>;
+    save(entry: FeedbackEntry): Promise<string>;
+    saveBatch(entries: FeedbackEntry[]): Promise<string[]>;
+    get(id: string): Promise<FeedbackEntry | null>;
+    query(options: FeedbackQueryOptions): Promise<FeedbackQueryResult>;
+    delete(id: string): Promise<boolean>;
+    clear(): Promise<void>;
+    close(): Promise<void>;
+    private ensureInitialized;
+}
+declare function createFeedbackStore(config: FeedbackStoreConfig): FeedbackStoreInterface;
+
+declare class FeedbackAggregator {
+    private store;
+    constructor(store: FeedbackStoreInterface);
+    aggregate(options: AggregationOptions): Promise<AggregationResult[]>;
+    getSummary(options?: {
+        startTime?: number;
+        endTime?: number;
+    }): Promise<{
+        totalCount: number;
+        byType: Record<string, number>;
+        thumbsUpRate: number;
+        avgRating: number;
+        preferenceDistribution: {
+            A: number;
+            B: number;
+            tie: number;
+        };
+        correctionRate: number;
+    }>;
+    getTrends(options: {
+        metric: AggregationMetric;
+        interval: 'hour' | 'day' | 'week';
+        startTime: number;
+        endTime: number;
+    }): Promise<Array<{
+        timestamp: number;
+        value: number;
+        count: number;
+    }>>;
+    private groupEntries;
+    private calculateMetrics;
+    private calculateSingleMetric;
+    private getWeekKey;
+    private getWeekNumber;
+    private getIntervalMs;
+}
+declare function createFeedbackAggregator(store: FeedbackStoreInterface): FeedbackAggregator;
+
+declare class FeedbackExporter {
+    private store;
+    constructor(store: FeedbackStoreInterface);
+    exportToString(options: ExportOptions): Promise<string>;
+    exportToFile(path: string, options: ExportOptions): Promise<number>;
+    exportStream(options: ExportOptions, batchSize?: number): AsyncGenerator<string, void, unknown>;
+    private getEntries;
+    private filterFields;
+    private toCSV;
+    private toCSVRows;
+    private getCSVHeaders;
+    private formatCSVValue;
+    private escapeCSV;
+}
+declare function createFeedbackExporter(store: FeedbackStoreInterface): FeedbackExporter;
+
+export { createMultiCriteriaCollector as $, type AggregationOptions as A, type BaseFeedbackEntry as B, type CorrectionFeedback as C, CollectPreferenceInputSchema as D, type ExportFormat as E, type FeedbackStoreInterface as F, MemoryFeedbackStore as G, SQLiteFeedbackStore as H, createFeedbackStore as I, FeedbackAggregator as J, createFeedbackAggregator as K, FeedbackExporter as L, type MultiCriteriaFeedback as M, createFeedbackExporter as N, BaseCollector as O, type PreferenceChoice as P, createThumbsCollector as Q, type RatingFeedback as R, type StarRating as S, ThumbsCollector as T, RatingCollector as U, createRatingCollector as V, PreferenceCollector as W, createPreferenceCollector as X, CorrectionCollector as Y, createCorrectionCollector as Z, MultiCriteriaCollector as _, type ThumbsFeedback as a, type ThumbsRating as b, type PreferenceFeedback as c, type CriterionDefinition as d, type CriterionRating as e, type FeedbackEntry as f, type FeedbackCollectorOptions as g, type ThumbsCollectorOptions as h, type RatingCollectorOptions as i, type PreferenceCollectorOptions as j, type MultiCriteriaCollectorOptions as k, type CollectThumbsInput as l, type CollectRatingInput as m, type CollectPreferenceInput as n, type CollectCorrectionInput as o, type CollectMultiCriteriaInput as p, type FeedbackQueryOptions as q, type FeedbackQueryResult as r, type AggregationMetric as s, type AggregationResult as t, type ExportOptions as u, type FeedbackStoreConfig as v, ThumbsRatingSchema as w, StarRatingSchema as x, PreferenceChoiceSchema as y, CollectThumbsInputSchema as z };