@lov3kaizen/agentsea-evaluate 0.5.1

package/dist/feedback/index.mjs

@@ -0,0 +1,40 @@
+import {
+  BaseCollector,
+  CorrectionCollector,
+  FeedbackAggregator,
+  FeedbackExporter,
+  MemoryFeedbackStore,
+  MultiCriteriaCollector,
+  PreferenceCollector,
+  RatingCollector,
+  SQLiteFeedbackStore,
+  ThumbsCollector,
+  createCorrectionCollector,
+  createFeedbackAggregator,
+  createFeedbackExporter,
+  createFeedbackStore,
+  createMultiCriteriaCollector,
+  createPreferenceCollector,
+  createRatingCollector,
+  createThumbsCollector
+} from "../chunk-PAQ2TTJJ.mjs";
+export {
+  BaseCollector,
+  CorrectionCollector,
+  FeedbackAggregator,
+  FeedbackExporter,
+  MemoryFeedbackStore,
+  MultiCriteriaCollector,
+  PreferenceCollector,
+  RatingCollector,
+  SQLiteFeedbackStore,
+  ThumbsCollector,
+  createCorrectionCollector,
+  createFeedbackAggregator,
+  createFeedbackExporter,
+  createFeedbackStore,
+  createMultiCriteriaCollector,
+  createPreferenceCollector,
+  createRatingCollector,
+  createThumbsCollector
+};

package/dist/index-6Pbiq7ny.d.mts

@@ -0,0 +1,234 @@
+type DatasetType = 'preference' | 'instruction' | 'conversation' | 'qa';
+type DatasetExportFormat = 'jsonl' | 'json' | 'csv' | 'parquet' | 'huggingface' | 'anthropic' | 'openai';
+interface PreferencePair {
+    id: string;
+    prompt: string;
+    chosen: string;
+    rejected: string;
+    chosenModel?: string;
+    rejectedModel?: string;
+    chosenScore?: number;
+    rejectedScore?: number;
+    reason?: string;
+    confidence?: number;
+    metadata?: Record<string, unknown>;
+}
+interface InstructionExample {
+    id: string;
+    instruction: string;
+    input?: string;
+    output: string;
+    systemPrompt?: string;
+    metadata?: Record<string, unknown>;
+}
+interface ConversationTurn {
+    role: 'user' | 'assistant' | 'system';
+    content: string;
+    metadata?: Record<string, unknown>;
+}
+interface ConversationExample {
+    id: string;
+    turns: ConversationTurn[];
+    metadata?: Record<string, unknown>;
+}
+interface QAExample {
+    id: string;
+    question: string;
+    answer: string;
+    context?: string[];
+    metadata?: Record<string, unknown>;
+}
+type DatasetItem = PreferencePair | InstructionExample | ConversationExample | QAExample;
+interface DatasetStats {
+    size: number;
+    type: DatasetType;
+    avgPromptLength: number;
+    avgResponseLength: number;
+    uniquePrompts: number;
+    modelDistribution?: Record<string, number>;
+    winRateA?: number;
+    winRateB?: number;
+    tieRate?: number;
+}
+type SamplingStrategyType = 'random' | 'balanced' | 'stratified' | 'uncertainty' | 'diversity';
+interface SamplingConfig {
+    type: SamplingStrategyType;
+    seed?: number;
+    preferenceRatio?: number;
+    minConfidence?: number;
+    stratifyBy?: string;
+    stratifyRatios?: Record<string, number>;
+    diversityField?: string;
+    minDiversity?: number;
+}
+interface DatasetBuilderConfig {
+    feedbackStore: FeedbackStoreRef;
+    sampling?: SamplingConfig;
+    filters?: DatasetFilterConfig;
+}
+interface FeedbackStoreRef {
+    query(options: DatasetQueryOptions): Promise<DatasetQueryResult>;
+}
+interface DatasetQueryOptions {
+    type?: string | string[];
+    startTime?: number;
+    endTime?: number;
+    minConfidence?: number;
+    metadata?: Record<string, unknown>;
+    limit?: number;
+    offset?: number;
+}
+interface DatasetQueryResult {
+    items: unknown[];
+    total: number;
+}
+interface DatasetFilterConfig {
+    minLength?: number;
+    maxLength?: number;
+    excludePatterns?: RegExp[];
+    includePatterns?: RegExp[];
+    customFilter?: (item: unknown) => boolean;
+}
+interface PreferenceBuildOptions {
+    minPairs?: number;
+    maxPairs?: number;
+    includeRejected?: boolean;
+    filterFn?: (pair: PreferencePair) => boolean;
+    deduplication?: 'none' | 'prompt' | 'exact';
+}
+interface InstructionBuildOptions {
+    minExamples?: number;
+    maxExamples?: number;
+    includeSystemPrompt?: boolean;
+    filterFn?: (example: InstructionExample) => boolean;
+}
+interface PreferenceDatasetInterface {
+    readonly type: 'preference';
+    readonly size: number;
+    readonly stats: DatasetStats;
+    getPairs(): PreferencePair[];
+    filter(predicate: (pair: PreferencePair) => boolean): PreferenceDatasetInterface;
+    sample(count: number): PreferenceDatasetInterface;
+    split(ratio: number): [PreferenceDatasetInterface, PreferenceDatasetInterface];
+    shuffle(seed?: number): PreferenceDatasetInterface;
+}
+interface DatasetExportOptions {
+    format: DatasetExportFormat;
+    path?: string;
+    fields?: string[];
+    repoName?: string;
+    private?: boolean;
+    token?: string;
+    formatOptions?: Record<string, unknown>;
+}
+interface HFExportOptions {
+    name: string;
+    private?: boolean;
+    token?: string;
+    readme?: string;
+    license?: string;
+    tags?: string[];
+}
+interface ExportResult {
+    format: DatasetExportFormat;
+    path?: string;
+    url?: string;
+    itemCount: number;
+    bytesWritten?: number;
+    warnings?: string[];
+}
+interface DPOFormatItem {
+    prompt: string;
+    chosen: string;
+    rejected: string;
+}
+interface RLHFFormatItem {
+    prompt: string;
+    response: string;
+    reward: number;
+}
+interface SFTFormatItem {
+    instruction: string;
+    input?: string;
+    output: string;
+}
+interface AnthropicFormatItem {
+    prompt: string;
+    completion: string;
+}
+interface OpenAIFormatItem {
+    messages: Array<{
+        role: 'system' | 'user' | 'assistant';
+        content: string;
+    }>;
+}
+interface DatasetValidationResult {
+    valid: boolean;
+    errors: DatasetValidationError[];
+    warnings: DatasetValidationWarning[];
+    stats: DatasetStats;
+}
+interface DatasetValidationError {
+    itemId: string;
+    field: string;
+    message: string;
+    value?: unknown;
+}
+interface DatasetValidationWarning {
+    type: 'duplicate' | 'short' | 'long' | 'format' | 'quality';
+    message: string;
+    count: number;
+    examples?: string[];
+}
+
+declare class PreferenceDataset implements PreferenceDatasetInterface {
+    readonly type: "preference";
+    private pairs;
+    private _stats?;
+    constructor(pairs: PreferencePair[]);
+    get size(): number;
+    get stats(): DatasetStats;
+    getPairs(): PreferencePair[];
+    filter(predicate: (pair: PreferencePair) => boolean): PreferenceDataset;
+    sample(count: number): PreferenceDataset;
+    split(ratio: number): [PreferenceDataset, PreferenceDataset];
+    shuffle(seed?: number): PreferenceDataset;
+    private calculateStats;
+    private seededRandom;
+}
+declare class PreferenceDatasetBuilder {
+    private feedbackStore;
+    private sampling?;
+    constructor(config: {
+        feedbackStore: FeedbackStoreRef;
+        sampling?: SamplingConfig;
+    });
+    build(options?: PreferenceBuildOptions): Promise<PreferenceDataset>;
+    private deduplicate;
+    private applySampling;
+    private randomSample;
+    private balancedSample;
+    private stratifiedSample;
+    private seededRandom;
+}
+declare function createPreferenceDatasetBuilder(config: {
+    feedbackStore: FeedbackStoreRef;
+    sampling?: SamplingConfig;
+}): PreferenceDatasetBuilder;
+
+declare class DatasetExporter {
+    exportPreferences(dataset: PreferenceDataset, options: DatasetExportOptions): Promise<ExportResult>;
+    toJSONL(pairs: PreferencePair[], options?: {
+        formatOptions?: Record<string, unknown>;
+    }): string;
+    toCSV(pairs: PreferencePair[]): string;
+    toAnthropicFormat(pairs: PreferencePair[]): string;
+    toOpenAIFormat(pairs: PreferencePair[]): string;
+    exportToHuggingFace(pairs: PreferencePair[], options: DatasetExportOptions): Promise<ExportResult>;
+    exportMultiple(dataset: PreferenceDataset, formats: DatasetExportFormat[], basePath: string): Promise<Map<DatasetExportFormat, ExportResult>>;
+    private getExtension;
+    private escapeCSV;
+}
+declare function createDatasetExporter(): DatasetExporter;
+
+export { type AnthropicFormatItem as A, type ConversationTurn as C, type DatasetType as D, type ExportResult as E, type FeedbackStoreRef as F, type HFExportOptions as H, type InstructionExample as I, type OpenAIFormatItem as O, type PreferencePair as P, type QAExample as Q, type RLHFFormatItem as R, type SamplingStrategyType as S, type DatasetExportFormat as a, type ConversationExample as b, type DatasetItem as c, type DatasetStats as d, type SamplingConfig as e, type DatasetBuilderConfig as f, type DatasetQueryOptions as g, type DatasetQueryResult as h, type DatasetFilterConfig as i, type PreferenceBuildOptions as j, type InstructionBuildOptions as k, type PreferenceDatasetInterface as l, type DatasetExportOptions as m, type DPOFormatItem as n, type SFTFormatItem as o, type DatasetValidationResult as p, type DatasetValidationError as q, type DatasetValidationWarning as r, PreferenceDataset as s, PreferenceDatasetBuilder as t, createPreferenceDatasetBuilder as u, DatasetExporter as v, createDatasetExporter as w };

package/dist/index-6Pbiq7ny.d.ts

@@ -0,0 +1,234 @@
+type DatasetType = 'preference' | 'instruction' | 'conversation' | 'qa';
+type DatasetExportFormat = 'jsonl' | 'json' | 'csv' | 'parquet' | 'huggingface' | 'anthropic' | 'openai';
+interface PreferencePair {
+    id: string;
+    prompt: string;
+    chosen: string;
+    rejected: string;
+    chosenModel?: string;
+    rejectedModel?: string;
+    chosenScore?: number;
+    rejectedScore?: number;
+    reason?: string;
+    confidence?: number;
+    metadata?: Record<string, unknown>;
+}
+interface InstructionExample {
+    id: string;
+    instruction: string;
+    input?: string;
+    output: string;
+    systemPrompt?: string;
+    metadata?: Record<string, unknown>;
+}
+interface ConversationTurn {
+    role: 'user' | 'assistant' | 'system';
+    content: string;
+    metadata?: Record<string, unknown>;
+}
+interface ConversationExample {
+    id: string;
+    turns: ConversationTurn[];
+    metadata?: Record<string, unknown>;
+}
+interface QAExample {
+    id: string;
+    question: string;
+    answer: string;
+    context?: string[];
+    metadata?: Record<string, unknown>;
+}
+type DatasetItem = PreferencePair | InstructionExample | ConversationExample | QAExample;
+interface DatasetStats {
+    size: number;
+    type: DatasetType;
+    avgPromptLength: number;
+    avgResponseLength: number;
+    uniquePrompts: number;
+    modelDistribution?: Record<string, number>;
+    winRateA?: number;
+    winRateB?: number;
+    tieRate?: number;
+}
+type SamplingStrategyType = 'random' | 'balanced' | 'stratified' | 'uncertainty' | 'diversity';
+interface SamplingConfig {
+    type: SamplingStrategyType;
+    seed?: number;
+    preferenceRatio?: number;
+    minConfidence?: number;
+    stratifyBy?: string;
+    stratifyRatios?: Record<string, number>;
+    diversityField?: string;
+    minDiversity?: number;
+}
+interface DatasetBuilderConfig {
+    feedbackStore: FeedbackStoreRef;
+    sampling?: SamplingConfig;
+    filters?: DatasetFilterConfig;
+}
+interface FeedbackStoreRef {
+    query(options: DatasetQueryOptions): Promise<DatasetQueryResult>;
+}
+interface DatasetQueryOptions {
+    type?: string | string[];
+    startTime?: number;
+    endTime?: number;
+    minConfidence?: number;
+    metadata?: Record<string, unknown>;
+    limit?: number;
+    offset?: number;
+}
+interface DatasetQueryResult {
+    items: unknown[];
+    total: number;
+}
+interface DatasetFilterConfig {
+    minLength?: number;
+    maxLength?: number;
+    excludePatterns?: RegExp[];
+    includePatterns?: RegExp[];
+    customFilter?: (item: unknown) => boolean;
+}
+interface PreferenceBuildOptions {
+    minPairs?: number;
+    maxPairs?: number;
+    includeRejected?: boolean;
+    filterFn?: (pair: PreferencePair) => boolean;
+    deduplication?: 'none' | 'prompt' | 'exact';
+}
+interface InstructionBuildOptions {
+    minExamples?: number;
+    maxExamples?: number;
+    includeSystemPrompt?: boolean;
+    filterFn?: (example: InstructionExample) => boolean;
+}
+interface PreferenceDatasetInterface {
+    readonly type: 'preference';
+    readonly size: number;
+    readonly stats: DatasetStats;
+    getPairs(): PreferencePair[];
+    filter(predicate: (pair: PreferencePair) => boolean): PreferenceDatasetInterface;
+    sample(count: number): PreferenceDatasetInterface;
+    split(ratio: number): [PreferenceDatasetInterface, PreferenceDatasetInterface];
+    shuffle(seed?: number): PreferenceDatasetInterface;
+}
+interface DatasetExportOptions {
+    format: DatasetExportFormat;
+    path?: string;
+    fields?: string[];
+    repoName?: string;
+    private?: boolean;
+    token?: string;
+    formatOptions?: Record<string, unknown>;
+}
+interface HFExportOptions {
+    name: string;
+    private?: boolean;
+    token?: string;
+    readme?: string;
+    license?: string;
+    tags?: string[];
+}
+interface ExportResult {
+    format: DatasetExportFormat;
+    path?: string;
+    url?: string;
+    itemCount: number;
+    bytesWritten?: number;
+    warnings?: string[];
+}
+interface DPOFormatItem {
+    prompt: string;
+    chosen: string;
+    rejected: string;
+}
+interface RLHFFormatItem {
+    prompt: string;
+    response: string;
+    reward: number;
+}
+interface SFTFormatItem {
+    instruction: string;
+    input?: string;
+    output: string;
+}
+interface AnthropicFormatItem {
+    prompt: string;
+    completion: string;
+}
+interface OpenAIFormatItem {
+    messages: Array<{
+        role: 'system' | 'user' | 'assistant';
+        content: string;
+    }>;
+}
+interface DatasetValidationResult {
+    valid: boolean;
+    errors: DatasetValidationError[];
+    warnings: DatasetValidationWarning[];
+    stats: DatasetStats;
+}
+interface DatasetValidationError {
+    itemId: string;
+    field: string;
+    message: string;
+    value?: unknown;
+}
+interface DatasetValidationWarning {
+    type: 'duplicate' | 'short' | 'long' | 'format' | 'quality';
+    message: string;
+    count: number;
+    examples?: string[];
+}
+
+declare class PreferenceDataset implements PreferenceDatasetInterface {
+    readonly type: "preference";
+    private pairs;
+    private _stats?;
+    constructor(pairs: PreferencePair[]);
+    get size(): number;
+    get stats(): DatasetStats;
+    getPairs(): PreferencePair[];
+    filter(predicate: (pair: PreferencePair) => boolean): PreferenceDataset;
+    sample(count: number): PreferenceDataset;
+    split(ratio: number): [PreferenceDataset, PreferenceDataset];
+    shuffle(seed?: number): PreferenceDataset;
+    private calculateStats;
+    private seededRandom;
+}
+declare class PreferenceDatasetBuilder {
+    private feedbackStore;
+    private sampling?;
+    constructor(config: {
+        feedbackStore: FeedbackStoreRef;
+        sampling?: SamplingConfig;
+    });
+    build(options?: PreferenceBuildOptions): Promise<PreferenceDataset>;
+    private deduplicate;
+    private applySampling;
+    private randomSample;
+    private balancedSample;
+    private stratifiedSample;
+    private seededRandom;
+}
+declare function createPreferenceDatasetBuilder(config: {
+    feedbackStore: FeedbackStoreRef;
+    sampling?: SamplingConfig;
+}): PreferenceDatasetBuilder;
+
+declare class DatasetExporter {
+    exportPreferences(dataset: PreferenceDataset, options: DatasetExportOptions): Promise<ExportResult>;
+    toJSONL(pairs: PreferencePair[], options?: {
+        formatOptions?: Record<string, unknown>;
+    }): string;
+    toCSV(pairs: PreferencePair[]): string;
+    toAnthropicFormat(pairs: PreferencePair[]): string;
+    toOpenAIFormat(pairs: PreferencePair[]): string;
+    exportToHuggingFace(pairs: PreferencePair[], options: DatasetExportOptions): Promise<ExportResult>;
+    exportMultiple(dataset: PreferenceDataset, formats: DatasetExportFormat[], basePath: string): Promise<Map<DatasetExportFormat, ExportResult>>;
+    private getExtension;
+    private escapeCSV;
+}
+declare function createDatasetExporter(): DatasetExporter;
+
+export { type AnthropicFormatItem as A, type ConversationTurn as C, type DatasetType as D, type ExportResult as E, type FeedbackStoreRef as F, type HFExportOptions as H, type InstructionExample as I, type OpenAIFormatItem as O, type PreferencePair as P, type QAExample as Q, type RLHFFormatItem as R, type SamplingStrategyType as S, type DatasetExportFormat as a, type ConversationExample as b, type DatasetItem as c, type DatasetStats as d, type SamplingConfig as e, type DatasetBuilderConfig as f, type DatasetQueryOptions as g, type DatasetQueryResult as h, type DatasetFilterConfig as i, type PreferenceBuildOptions as j, type InstructionBuildOptions as k, type PreferenceDatasetInterface as l, type DatasetExportOptions as m, type DPOFormatItem as n, type SFTFormatItem as o, type DatasetValidationResult as p, type DatasetValidationError as q, type DatasetValidationWarning as r, PreferenceDataset as s, PreferenceDatasetBuilder as t, createPreferenceDatasetBuilder as u, DatasetExporter as v, createDatasetExporter as w };