@lov3kaizen/agentsea-evaluate 0.5.1

package/dist/datasets/index.d.ts

@@ -0,0 +1 @@
+export { v as DatasetExporter, s as PreferenceDataset, t as PreferenceDatasetBuilder, w as createDatasetExporter, u as createPreferenceDatasetBuilder } from '../index-6Pbiq7ny.js';

package/dist/datasets/index.js

@@ -0,0 +1,456 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/datasets/index.ts
+var datasets_exports = {};
+__export(datasets_exports, {
+  DatasetExporter: () => DatasetExporter,
+  PreferenceDataset: () => PreferenceDataset,
+  PreferenceDatasetBuilder: () => PreferenceDatasetBuilder,
+  createDatasetExporter: () => createDatasetExporter,
+  createPreferenceDatasetBuilder: () => createPreferenceDatasetBuilder
+});
+module.exports = __toCommonJS(datasets_exports);
+
+// src/datasets/PreferenceDatasetBuilder.ts
+var import_nanoid = require("nanoid");
+var PreferenceDataset = class _PreferenceDataset {
+  type = "preference";
+  pairs;
+  _stats;
+  constructor(pairs) {
+    this.pairs = pairs;
+  }
+  get size() {
+    return this.pairs.length;
+  }
+  get stats() {
+    if (!this._stats) {
+      this._stats = this.calculateStats();
+    }
+    return this._stats;
+  }
+  getPairs() {
+    return [...this.pairs];
+  }
+  filter(predicate) {
+    return new _PreferenceDataset(this.pairs.filter(predicate));
+  }
+  sample(count) {
+    if (count >= this.pairs.length) {
+      return new _PreferenceDataset([...this.pairs]);
+    }
+    const shuffled = [...this.pairs];
+    for (let i = shuffled.length - 1; i > 0; i--) {
+      const j = Math.floor(Math.random() * (i + 1));
+      [shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];
+    }
+    return new _PreferenceDataset(shuffled.slice(0, count));
+  }
+  split(ratio) {
+    const shuffled = [...this.pairs];
+    for (let i = shuffled.length - 1; i > 0; i--) {
+      const j = Math.floor(Math.random() * (i + 1));
+      [shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];
+    }
+    const splitIndex = Math.floor(shuffled.length * ratio);
+    return [
+      new _PreferenceDataset(shuffled.slice(0, splitIndex)),
+      new _PreferenceDataset(shuffled.slice(splitIndex))
+    ];
+  }
+  shuffle(seed) {
+    const shuffled = [...this.pairs];
+    const rng = seed !== void 0 ? this.seededRandom(seed) : Math.random;
+    for (let i = shuffled.length - 1; i > 0; i--) {
+      const j = Math.floor(rng() * (i + 1));
+      [shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];
+    }
+    return new _PreferenceDataset(shuffled);
+  }
+  calculateStats() {
+    const models = /* @__PURE__ */ new Map();
+    let totalPromptLen = 0;
+    let totalResponseLen = 0;
+    const uniquePrompts = /* @__PURE__ */ new Set();
+    for (const pair of this.pairs) {
+      uniquePrompts.add(pair.prompt);
+      totalPromptLen += pair.prompt.length;
+      totalResponseLen += (pair.chosen.length + pair.rejected.length) / 2;
+      if (pair.chosenModel) {
+        models.set(pair.chosenModel, (models.get(pair.chosenModel) ?? 0) + 1);
+      }
+      if (pair.rejectedModel) {
+        models.set(
+          pair.rejectedModel,
+          (models.get(pair.rejectedModel) ?? 0) + 1
+        );
+      }
+    }
+    return {
+      size: this.pairs.length,
+      type: "preference",
+      avgPromptLength: this.pairs.length > 0 ? totalPromptLen / this.pairs.length : 0,
+      avgResponseLength: this.pairs.length > 0 ? totalResponseLen / this.pairs.length : 0,
+      uniquePrompts: uniquePrompts.size,
+      modelDistribution: Object.fromEntries(models)
+    };
+  }
+  seededRandom(seed) {
+    return () => {
+      seed = (seed * 9301 + 49297) % 233280;
+      return seed / 233280;
+    };
+  }
+};
+var PreferenceDatasetBuilder = class {
+  feedbackStore;
+  sampling;
+  constructor(config) {
+    this.feedbackStore = config.feedbackStore;
+    this.sampling = config.sampling;
+  }
+  /**
+   * Build preference dataset from feedback
+   */
+  async build(options = {}) {
+    const minPairs = options.minPairs ?? 0;
+    const maxPairs = options.maxPairs ?? Infinity;
+    const result = await this.feedbackStore.query({
+      type: "preference",
+      minConfidence: this.sampling?.minConfidence,
+      limit: maxPairs * 2
+      // Fetch extra to account for filtering
+    });
+    let pairs = [];
+    for (const item of result.items) {
+      const feedback = item;
+      if (feedback.preference === "tie") {
+        continue;
+      }
+      const chosen = feedback.preference === "A" ? feedback.responseA : feedback.responseB;
+      const rejected = feedback.preference === "A" ? feedback.responseB : feedback.responseA;
+      const pair = {
+        id: (0, import_nanoid.nanoid)(),
+        prompt: feedback.input,
+        chosen: chosen.content,
+        rejected: rejected.content,
+        chosenModel: chosen.model,
+        rejectedModel: rejected.model,
+        reason: feedback.reason,
+        confidence: feedback.confidence,
+        metadata: feedback.metadata
+      };
+      pairs.push(pair);
+    }
+    if (options.filterFn) {
+      pairs = pairs.filter(options.filterFn);
+    }
+    if (options.deduplication && options.deduplication !== "none") {
+      pairs = this.deduplicate(pairs, options.deduplication);
+    }
+    if (this.sampling) {
+      pairs = this.applySampling(pairs);
+    }
+    if (pairs.length > maxPairs) {
+      pairs = pairs.slice(0, maxPairs);
+    }
+    if (pairs.length < minPairs) {
+      console.warn(
+        `Only ${pairs.length} pairs available, requested minimum ${minPairs}`
+      );
+    }
+    return new PreferenceDataset(pairs);
+  }
+  /**
+   * Deduplicate pairs
+   */
+  deduplicate(pairs, mode) {
+    const seen = /* @__PURE__ */ new Set();
+    const result = [];
+    for (const pair of pairs) {
+      const key = mode === "prompt" ? pair.prompt : `${pair.prompt}|${pair.chosen}|${pair.rejected}`;
+      if (!seen.has(key)) {
+        seen.add(key);
+        result.push(pair);
+      }
+    }
+    return result;
+  }
+  /**
+   * Apply sampling strategy
+   */
+  applySampling(pairs) {
+    if (!this.sampling) return pairs;
+    switch (this.sampling.type) {
+      case "random":
+        return this.randomSample(pairs);
+      case "balanced":
+        return this.balancedSample(pairs);
+      case "stratified":
+        return this.stratifiedSample(pairs);
+      default:
+        return pairs;
+    }
+  }
+  randomSample(pairs) {
+    const shuffled = [...pairs];
+    const seed = this.sampling?.seed;
+    const rng = seed !== void 0 ? this.seededRandom(seed) : Math.random;
+    for (let i = shuffled.length - 1; i > 0; i--) {
+      const j = Math.floor(rng() * (i + 1));
+      [shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];
+    }
+    return shuffled;
+  }
+  balancedSample(pairs) {
+    const minConf = this.sampling?.minConfidence ?? 0;
+    return pairs.filter((p) => (p.confidence ?? 1) >= minConf);
+  }
+  stratifiedSample(pairs) {
+    const field = this.sampling?.stratifyBy ?? "chosenModel";
+    const groups = /* @__PURE__ */ new Map();
+    for (const pair of pairs) {
+      const key = String(
+        pair[field] ?? "unknown"
+      );
+      if (!groups.has(key)) {
+        groups.set(key, []);
+      }
+      groups.get(key).push(pair);
+    }
+    const result = [];
+    const ratios = this.sampling?.stratifyRatios ?? {};
+    for (const [key, group] of groups) {
+      const ratio = ratios[key] ?? 1 / groups.size;
+      const count = Math.ceil(pairs.length * ratio);
+      result.push(...group.slice(0, count));
+    }
+    return result;
+  }
+  seededRandom(seed) {
+    return () => {
+      seed = (seed * 9301 + 49297) % 233280;
+      return seed / 233280;
+    };
+  }
+};
+function createPreferenceDatasetBuilder(config) {
+  return new PreferenceDatasetBuilder(config);
+}
+
+// src/datasets/DatasetExporter.ts
+var fs = __toESM(require("fs/promises"));
+var DatasetExporter = class {
+  /**
+   * Export preference dataset to file
+   */
+  async exportPreferences(dataset, options) {
+    const pairs = dataset.getPairs();
+    let content;
+    const warnings = [];
+    switch (options.format) {
+      case "jsonl":
+        content = this.toJSONL(pairs, options);
+        break;
+      case "json":
+        content = JSON.stringify(pairs, null, 2);
+        break;
+      case "csv":
+        content = this.toCSV(pairs);
+        break;
+      case "huggingface":
+        return this.exportToHuggingFace(pairs, options);
+      case "anthropic":
+        content = this.toAnthropicFormat(pairs);
+        break;
+      case "openai":
+        content = this.toOpenAIFormat(pairs);
+        break;
+      default:
+        throw new Error(`Unsupported export format: ${options.format}`);
+    }
+    if (options.path) {
+      await fs.writeFile(options.path, content, "utf-8");
+    }
+    return {
+      format: options.format,
+      path: options.path,
+      itemCount: pairs.length,
+      bytesWritten: Buffer.byteLength(content, "utf-8"),
+      warnings: warnings.length > 0 ? warnings : void 0
+    };
+  }
+  /**
+   * Convert to JSONL format
+   */
+  toJSONL(pairs, options) {
+    const format = options?.formatOptions?.format ?? "dpo";
+    return pairs.map((pair) => {
+      switch (format) {
+        case "dpo":
+          return JSON.stringify({
+            prompt: pair.prompt,
+            chosen: pair.chosen,
+            rejected: pair.rejected
+          });
+        case "sft":
+          return JSON.stringify({
+            instruction: pair.prompt,
+            output: pair.chosen
+          });
+        default:
+          return JSON.stringify(pair);
+      }
+    }).join("\n");
+  }
+  /**
+   * Convert to CSV format
+   */
+  toCSV(pairs) {
+    const headers = [
+      "prompt",
+      "chosen",
+      "rejected",
+      "chosen_model",
+      "rejected_model",
+      "confidence"
+    ];
+    const rows = pairs.map(
+      (pair) => [
+        this.escapeCSV(pair.prompt),
+        this.escapeCSV(pair.chosen),
+        this.escapeCSV(pair.rejected),
+        pair.chosenModel ?? "",
+        pair.rejectedModel ?? "",
+        pair.confidence?.toString() ?? ""
+      ].join(",")
+    );
+    return [headers.join(","), ...rows].join("\n");
+  }
+  /**
+   * Convert to Anthropic format
+   */
+  toAnthropicFormat(pairs) {
+    return pairs.map(
+      (pair) => JSON.stringify({
+        prompt: `
+
+Human: ${pair.prompt}
+
+Assistant:`,
+        completion: ` ${pair.chosen}`
+      })
+    ).join("\n");
+  }
+  /**
+   * Convert to OpenAI format
+   */
+  toOpenAIFormat(pairs) {
+    return pairs.map(
+      (pair) => JSON.stringify({
+        messages: [
+          { role: "user", content: pair.prompt },
+          { role: "assistant", content: pair.chosen }
+        ]
+      })
+    ).join("\n");
+  }
+  /**
+   * Export to HuggingFace Hub (stub)
+   */
+  async exportToHuggingFace(pairs, options) {
+    const hfOptions = options.formatOptions;
+    if (!hfOptions?.token) {
+      throw new Error("HuggingFace token is required for Hub export");
+    }
+    console.warn(
+      "HuggingFace Hub export not fully implemented. Saving locally instead."
+    );
+    const localPath = options.path ?? `./${hfOptions.name ?? "dataset"}.jsonl`;
+    const content = this.toJSONL(pairs, { formatOptions: { format: "dpo" } });
+    await fs.writeFile(localPath, content, "utf-8");
+    return {
+      format: "huggingface",
+      path: localPath,
+      itemCount: pairs.length,
+      warnings: ["Exported locally. Use @huggingface/hub to push to Hub."]
+    };
+  }
+  /**
+   * Export to multiple formats
+   */
+  async exportMultiple(dataset, formats, basePath) {
+    const results = /* @__PURE__ */ new Map();
+    for (const format of formats) {
+      const ext = this.getExtension(format);
+      const path = `${basePath}.${ext}`;
+      const result = await this.exportPreferences(dataset, { format, path });
+      results.set(format, result);
+    }
+    return results;
+  }
+  /**
+   * Get file extension for format
+   */
+  getExtension(format) {
+    switch (format) {
+      case "jsonl":
+        return "jsonl";
+      case "json":
+        return "json";
+      case "csv":
+        return "csv";
+      case "parquet":
+        return "parquet";
+      default:
+        return "jsonl";
+    }
+  }
+  /**
+   * Escape CSV value
+   */
+  escapeCSV(value) {
+    if (value.includes(",") || value.includes('"') || value.includes("\n")) {
+      return `"${value.replace(/"/g, '""')}"`;
+    }
+    return value;
+  }
+};
+function createDatasetExporter() {
+  return new DatasetExporter();
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  DatasetExporter,
+  PreferenceDataset,
+  PreferenceDatasetBuilder,
+  createDatasetExporter,
+  createPreferenceDatasetBuilder
+});

package/dist/datasets/index.mjs

@@ -0,0 +1,14 @@
+import {
+  DatasetExporter,
+  PreferenceDataset,
+  PreferenceDatasetBuilder,
+  createDatasetExporter,
+  createPreferenceDatasetBuilder
+} from "../chunk-TUMNJN2S.mjs";
+export {
+  DatasetExporter,
+  PreferenceDataset,
+  PreferenceDatasetBuilder,
+  createDatasetExporter,
+  createPreferenceDatasetBuilder
+};

package/dist/evaluation/index.d.mts

@@ -0,0 +1 @@
+export { X as Accuracy, W as BaseMetric, aj as CodeQualityRubric, $ as Coherence, al as ComparativeJudge, an as ConsensusJudge, a5 as ContextRelevance, a7 as CustomMetric, a as EvalDataset, Q as EvalRunner, E as EvaluationPipeline, a3 as Faithfulness, ak as HelpfulnessRubric, ae as LLMJudge, ai as QualityRubric, Z as Relevance, ag as RubricJudge, a1 as Toxicity, Y as createAccuracyMetric, a0 as createCoherenceMetric, am as createComparativeJudge, ao as createConsensusJudge, ad as createContainsMetric, a6 as createContextRelevanceMetric, a8 as createCustomMetric, O as createEvalDataset, U as createEvalRunner, V as createEvaluationPipeline, a4 as createFaithfulnessMetric, ac as createJSONMetric, af as createLLMJudge, aa as createLengthMetric, ab as createRegexMetric, _ as createRelevanceMetric, ah as createRubricJudge, a9 as createSimpleMetric, a2 as createToxicityMetric } from '../index-CTYCfWfH.mjs';

package/dist/evaluation/index.d.ts

@@ -0,0 +1 @@
+export { X as Accuracy, W as BaseMetric, aj as CodeQualityRubric, $ as Coherence, al as ComparativeJudge, an as ConsensusJudge, a5 as ContextRelevance, a7 as CustomMetric, a as EvalDataset, Q as EvalRunner, E as EvaluationPipeline, a3 as Faithfulness, ak as HelpfulnessRubric, ae as LLMJudge, ai as QualityRubric, Z as Relevance, ag as RubricJudge, a1 as Toxicity, Y as createAccuracyMetric, a0 as createCoherenceMetric, am as createComparativeJudge, ao as createConsensusJudge, ad as createContainsMetric, a6 as createContextRelevanceMetric, a8 as createCustomMetric, O as createEvalDataset, U as createEvalRunner, V as createEvaluationPipeline, a4 as createFaithfulnessMetric, ac as createJSONMetric, af as createLLMJudge, aa as createLengthMetric, ab as createRegexMetric, _ as createRelevanceMetric, ah as createRubricJudge, a9 as createSimpleMetric, a2 as createToxicityMetric } from '../index-CTYCfWfH.js';