@lov3kaizen/agentsea-evaluate 0.5.0

package/dist/chunk-TUMNJN2S.mjs

@@ -0,0 +1,416 @@
+// src/datasets/PreferenceDatasetBuilder.ts
+import { nanoid } from "nanoid";
+var PreferenceDataset = class _PreferenceDataset {
+  type = "preference";
+  pairs;
+  _stats;
+  constructor(pairs) {
+    this.pairs = pairs;
+  }
+  get size() {
+    return this.pairs.length;
+  }
+  get stats() {
+    if (!this._stats) {
+      this._stats = this.calculateStats();
+    }
+    return this._stats;
+  }
+  getPairs() {
+    return [...this.pairs];
+  }
+  filter(predicate) {
+    return new _PreferenceDataset(this.pairs.filter(predicate));
+  }
+  sample(count) {
+    if (count >= this.pairs.length) {
+      return new _PreferenceDataset([...this.pairs]);
+    }
+    const shuffled = [...this.pairs];
+    for (let i = shuffled.length - 1; i > 0; i--) {
+      const j = Math.floor(Math.random() * (i + 1));
+      [shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];
+    }
+    return new _PreferenceDataset(shuffled.slice(0, count));
+  }
+  split(ratio) {
+    const shuffled = [...this.pairs];
+    for (let i = shuffled.length - 1; i > 0; i--) {
+      const j = Math.floor(Math.random() * (i + 1));
+      [shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];
+    }
+    const splitIndex = Math.floor(shuffled.length * ratio);
+    return [
+      new _PreferenceDataset(shuffled.slice(0, splitIndex)),
+      new _PreferenceDataset(shuffled.slice(splitIndex))
+    ];
+  }
+  shuffle(seed) {
+    const shuffled = [...this.pairs];
+    const rng = seed !== void 0 ? this.seededRandom(seed) : Math.random;
+    for (let i = shuffled.length - 1; i > 0; i--) {
+      const j = Math.floor(rng() * (i + 1));
+      [shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];
+    }
+    return new _PreferenceDataset(shuffled);
+  }
+  calculateStats() {
+    const models = /* @__PURE__ */ new Map();
+    let totalPromptLen = 0;
+    let totalResponseLen = 0;
+    const uniquePrompts = /* @__PURE__ */ new Set();
+    for (const pair of this.pairs) {
+      uniquePrompts.add(pair.prompt);
+      totalPromptLen += pair.prompt.length;
+      totalResponseLen += (pair.chosen.length + pair.rejected.length) / 2;
+      if (pair.chosenModel) {
+        models.set(pair.chosenModel, (models.get(pair.chosenModel) ?? 0) + 1);
+      }
+      if (pair.rejectedModel) {
+        models.set(
+          pair.rejectedModel,
+          (models.get(pair.rejectedModel) ?? 0) + 1
+        );
+      }
+    }
+    return {
+      size: this.pairs.length,
+      type: "preference",
+      avgPromptLength: this.pairs.length > 0 ? totalPromptLen / this.pairs.length : 0,
+      avgResponseLength: this.pairs.length > 0 ? totalResponseLen / this.pairs.length : 0,
+      uniquePrompts: uniquePrompts.size,
+      modelDistribution: Object.fromEntries(models)
+    };
+  }
+  seededRandom(seed) {
+    return () => {
+      seed = (seed * 9301 + 49297) % 233280;
+      return seed / 233280;
+    };
+  }
+};
+var PreferenceDatasetBuilder = class {
+  feedbackStore;
+  sampling;
+  constructor(config) {
+    this.feedbackStore = config.feedbackStore;
+    this.sampling = config.sampling;
+  }
+  /**
+   * Build preference dataset from feedback
+   */
+  async build(options = {}) {
+    const minPairs = options.minPairs ?? 0;
+    const maxPairs = options.maxPairs ?? Infinity;
+    const result = await this.feedbackStore.query({
+      type: "preference",
+      minConfidence: this.sampling?.minConfidence,
+      limit: maxPairs * 2
+      // Fetch extra to account for filtering
+    });
+    let pairs = [];
+    for (const item of result.items) {
+      const feedback = item;
+      if (feedback.preference === "tie") {
+        continue;
+      }
+      const chosen = feedback.preference === "A" ? feedback.responseA : feedback.responseB;
+      const rejected = feedback.preference === "A" ? feedback.responseB : feedback.responseA;
+      const pair = {
+        id: nanoid(),
+        prompt: feedback.input,
+        chosen: chosen.content,
+        rejected: rejected.content,
+        chosenModel: chosen.model,
+        rejectedModel: rejected.model,
+        reason: feedback.reason,
+        confidence: feedback.confidence,
+        metadata: feedback.metadata
+      };
+      pairs.push(pair);
+    }
+    if (options.filterFn) {
+      pairs = pairs.filter(options.filterFn);
+    }
+    if (options.deduplication && options.deduplication !== "none") {
+      pairs = this.deduplicate(pairs, options.deduplication);
+    }
+    if (this.sampling) {
+      pairs = this.applySampling(pairs);
+    }
+    if (pairs.length > maxPairs) {
+      pairs = pairs.slice(0, maxPairs);
+    }
+    if (pairs.length < minPairs) {
+      console.warn(
+        `Only ${pairs.length} pairs available, requested minimum ${minPairs}`
+      );
+    }
+    return new PreferenceDataset(pairs);
+  }
+  /**
+   * Deduplicate pairs
+   */
+  deduplicate(pairs, mode) {
+    const seen = /* @__PURE__ */ new Set();
+    const result = [];
+    for (const pair of pairs) {
+      const key = mode === "prompt" ? pair.prompt : `${pair.prompt}|${pair.chosen}|${pair.rejected}`;
+      if (!seen.has(key)) {
+        seen.add(key);
+        result.push(pair);
+      }
+    }
+    return result;
+  }
+  /**
+   * Apply sampling strategy
+   */
+  applySampling(pairs) {
+    if (!this.sampling) return pairs;
+    switch (this.sampling.type) {
+      case "random":
+        return this.randomSample(pairs);
+      case "balanced":
+        return this.balancedSample(pairs);
+      case "stratified":
+        return this.stratifiedSample(pairs);
+      default:
+        return pairs;
+    }
+  }
+  randomSample(pairs) {
+    const shuffled = [...pairs];
+    const seed = this.sampling?.seed;
+    const rng = seed !== void 0 ? this.seededRandom(seed) : Math.random;
+    for (let i = shuffled.length - 1; i > 0; i--) {
+      const j = Math.floor(rng() * (i + 1));
+      [shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];
+    }
+    return shuffled;
+  }
+  balancedSample(pairs) {
+    const minConf = this.sampling?.minConfidence ?? 0;
+    return pairs.filter((p) => (p.confidence ?? 1) >= minConf);
+  }
+  stratifiedSample(pairs) {
+    const field = this.sampling?.stratifyBy ?? "chosenModel";
+    const groups = /* @__PURE__ */ new Map();
+    for (const pair of pairs) {
+      const key = String(
+        pair[field] ?? "unknown"
+      );
+      if (!groups.has(key)) {
+        groups.set(key, []);
+      }
+      groups.get(key).push(pair);
+    }
+    const result = [];
+    const ratios = this.sampling?.stratifyRatios ?? {};
+    for (const [key, group] of groups) {
+      const ratio = ratios[key] ?? 1 / groups.size;
+      const count = Math.ceil(pairs.length * ratio);
+      result.push(...group.slice(0, count));
+    }
+    return result;
+  }
+  seededRandom(seed) {
+    return () => {
+      seed = (seed * 9301 + 49297) % 233280;
+      return seed / 233280;
+    };
+  }
+};
+function createPreferenceDatasetBuilder(config) {
+  return new PreferenceDatasetBuilder(config);
+}
+
+// src/datasets/DatasetExporter.ts
+import * as fs from "fs/promises";
+var DatasetExporter = class {
+  /**
+   * Export preference dataset to file
+   */
+  async exportPreferences(dataset, options) {
+    const pairs = dataset.getPairs();
+    let content;
+    const warnings = [];
+    switch (options.format) {
+      case "jsonl":
+        content = this.toJSONL(pairs, options);
+        break;
+      case "json":
+        content = JSON.stringify(pairs, null, 2);
+        break;
+      case "csv":
+        content = this.toCSV(pairs);
+        break;
+      case "huggingface":
+        return this.exportToHuggingFace(pairs, options);
+      case "anthropic":
+        content = this.toAnthropicFormat(pairs);
+        break;
+      case "openai":
+        content = this.toOpenAIFormat(pairs);
+        break;
+      default:
+        throw new Error(`Unsupported export format: ${options.format}`);
+    }
+    if (options.path) {
+      await fs.writeFile(options.path, content, "utf-8");
+    }
+    return {
+      format: options.format,
+      path: options.path,
+      itemCount: pairs.length,
+      bytesWritten: Buffer.byteLength(content, "utf-8"),
+      warnings: warnings.length > 0 ? warnings : void 0
+    };
+  }
+  /**
+   * Convert to JSONL format
+   */
+  toJSONL(pairs, options) {
+    const format = options?.formatOptions?.format ?? "dpo";
+    return pairs.map((pair) => {
+      switch (format) {
+        case "dpo":
+          return JSON.stringify({
+            prompt: pair.prompt,
+            chosen: pair.chosen,
+            rejected: pair.rejected
+          });
+        case "sft":
+          return JSON.stringify({
+            instruction: pair.prompt,
+            output: pair.chosen
+          });
+        default:
+          return JSON.stringify(pair);
+      }
+    }).join("\n");
+  }
+  /**
+   * Convert to CSV format
+   */
+  toCSV(pairs) {
+    const headers = [
+      "prompt",
+      "chosen",
+      "rejected",
+      "chosen_model",
+      "rejected_model",
+      "confidence"
+    ];
+    const rows = pairs.map(
+      (pair) => [
+        this.escapeCSV(pair.prompt),
+        this.escapeCSV(pair.chosen),
+        this.escapeCSV(pair.rejected),
+        pair.chosenModel ?? "",
+        pair.rejectedModel ?? "",
+        pair.confidence?.toString() ?? ""
+      ].join(",")
+    );
+    return [headers.join(","), ...rows].join("\n");
+  }
+  /**
+   * Convert to Anthropic format
+   */
+  toAnthropicFormat(pairs) {
+    return pairs.map(
+      (pair) => JSON.stringify({
+        prompt: `
+
+Human: ${pair.prompt}
+
+Assistant:`,
+        completion: ` ${pair.chosen}`
+      })
+    ).join("\n");
+  }
+  /**
+   * Convert to OpenAI format
+   */
+  toOpenAIFormat(pairs) {
+    return pairs.map(
+      (pair) => JSON.stringify({
+        messages: [
+          { role: "user", content: pair.prompt },
+          { role: "assistant", content: pair.chosen }
+        ]
+      })
+    ).join("\n");
+  }
+  /**
+   * Export to HuggingFace Hub (stub)
+   */
+  async exportToHuggingFace(pairs, options) {
+    const hfOptions = options.formatOptions;
+    if (!hfOptions?.token) {
+      throw new Error("HuggingFace token is required for Hub export");
+    }
+    console.warn(
+      "HuggingFace Hub export not fully implemented. Saving locally instead."
+    );
+    const localPath = options.path ?? `./${hfOptions.name ?? "dataset"}.jsonl`;
+    const content = this.toJSONL(pairs, { formatOptions: { format: "dpo" } });
+    await fs.writeFile(localPath, content, "utf-8");
+    return {
+      format: "huggingface",
+      path: localPath,
+      itemCount: pairs.length,
+      warnings: ["Exported locally. Use @huggingface/hub to push to Hub."]
+    };
+  }
+  /**
+   * Export to multiple formats
+   */
+  async exportMultiple(dataset, formats, basePath) {
+    const results = /* @__PURE__ */ new Map();
+    for (const format of formats) {
+      const ext = this.getExtension(format);
+      const path = `${basePath}.${ext}`;
+      const result = await this.exportPreferences(dataset, { format, path });
+      results.set(format, result);
+    }
+    return results;
+  }
+  /**
+   * Get file extension for format
+   */
+  getExtension(format) {
+    switch (format) {
+      case "jsonl":
+        return "jsonl";
+      case "json":
+        return "json";
+      case "csv":
+        return "csv";
+      case "parquet":
+        return "parquet";
+      default:
+        return "jsonl";
+    }
+  }
+  /**
+   * Escape CSV value
+   */
+  escapeCSV(value) {
+    if (value.includes(",") || value.includes('"') || value.includes("\n")) {
+      return `"${value.replace(/"/g, '""')}"`;
+    }
+    return value;
+  }
+};
+function createDatasetExporter() {
+  return new DatasetExporter();
+}
+
+export {
+  PreferenceDataset,
+  PreferenceDatasetBuilder,
+  createPreferenceDatasetBuilder,
+  DatasetExporter,
+  createDatasetExporter
+};

package/dist/continuous/index.d.mts

@@ -0,0 +1,2 @@
+export { G as ABTestRunner, z as AlertManager, x as ContinuousEval, I as createABTestRunner, F as createAlertManager, y as createContinuousEval } from '../index-Cq5LwG_3.mjs';
+import 'eventemitter3';

package/dist/continuous/index.d.ts

@@ -0,0 +1,2 @@
+export { G as ABTestRunner, z as AlertManager, x as ContinuousEval, I as createABTestRunner, F as createAlertManager, y as createContinuousEval } from '../index-Cq5LwG_3.js';
+import 'eventemitter3';