@lov3kaizen/agentsea-evaluate 0.5.1

package/dist/continuous/index.js

@@ -0,0 +1,707 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/continuous/index.ts
+var continuous_exports = {};
+__export(continuous_exports, {
+  ABTestRunner: () => ABTestRunner,
+  AlertManager: () => AlertManager,
+  ContinuousEval: () => ContinuousEval,
+  createABTestRunner: () => createABTestRunner,
+  createAlertManager: () => createAlertManager,
+  createContinuousEval: () => createContinuousEval
+});
+module.exports = __toCommonJS(continuous_exports);
+
+// src/continuous/ContinuousEval.ts
+var import_eventemitter32 = require("eventemitter3");
+
+// src/continuous/ABTestRunner.ts
+var import_eventemitter3 = require("eventemitter3");
+var import_nanoid = require("nanoid");
+var ABTestRunner = class extends import_eventemitter3.EventEmitter {
+  id;
+  name;
+  config;
+  status = "draft";
+  startedAt;
+  completedAt;
+  controlSamples = /* @__PURE__ */ new Map();
+  treatmentSamples = /* @__PURE__ */ new Map();
+  sampleCount = 0;
+  constructor(config) {
+    super();
+    this.id = (0, import_nanoid.nanoid)();
+    this.name = config.name;
+    this.config = config;
+  }
+  /**
+   * Start the A/B test
+   */
+  start() {
+    if (this.status !== "draft") {
+      throw new Error(`Cannot start test in ${this.status} status`);
+    }
+    this.status = "running";
+    this.startedAt = Date.now();
+    this.emit("test:started");
+    return Promise.resolve();
+  }
+  /**
+   * Stop the A/B test
+   */
+  stop() {
+    this.status = "completed";
+    this.completedAt = Date.now();
+    const results = this.getResults();
+    this.emit("test:completed", results);
+    return results;
+  }
+  /**
+   * Pause the test
+   */
+  pause() {
+    if (this.status === "running") {
+      this.status = "paused";
+    }
+  }
+  /**
+   * Resume the test
+   */
+  resume() {
+    if (this.status === "paused") {
+      this.status = "running";
+    }
+  }
+  /**
+   * Assign a sample to a variant
+   */
+  assignVariant() {
+    if (this.status !== "running") {
+      throw new Error("Test is not running");
+    }
+    const isControl = Math.random() >= this.config.trafficSplit;
+    const variant = isControl ? "control" : "treatment";
+    const assignment = {
+      variant,
+      testId: this.id,
+      assignedAt: Date.now()
+    };
+    this.emit("sample:assigned", assignment);
+    return variant;
+  }
+  /**
+   * Record sample result
+   */
+  recordSample(variant, sampleId, scores) {
+    if (this.status !== "running") {
+      throw new Error("Test is not running");
+    }
+    const samples = variant === "control" ? this.controlSamples : this.treatmentSamples;
+    samples.set(sampleId, scores);
+    this.sampleCount++;
+    if (this.sampleCount >= this.config.minSamples) {
+      this.checkSignificance();
+    }
+    if (this.config.maxDuration && this.startedAt && Date.now() - this.startedAt > this.config.maxDuration) {
+      this.stop();
+    }
+  }
+  /**
+   * Get current results
+   */
+  getResults() {
+    const metrics = {};
+    for (const metric of this.config.metrics) {
+      metrics[metric] = this.calculateMetricResult(metric);
+    }
+    let controlWins = 0;
+    let treatmentWins = 0;
+    for (const result of Object.values(metrics)) {
+      if (result.isSignificant) {
+        if (result.winner === "control") controlWins++;
+        else if (result.winner === "treatment") treatmentWins++;
+      }
+    }
+    let winner = "none";
+    if (controlWins > treatmentWins) winner = "control";
+    else if (treatmentWins > controlWins) winner = "treatment";
+    const isSignificant = Object.values(metrics).some((m) => m.isSignificant);
+    const confidence = isSignificant ? Math.max(...Object.values(metrics).map((m) => 1 - m.pValue)) : 0;
+    return {
+      controlSamples: this.controlSamples.size,
+      treatmentSamples: this.treatmentSamples.size,
+      metrics,
+      winner,
+      isSignificant,
+      confidence,
+      recommendation: this.generateRecommendation(
+        winner,
+        isSignificant,
+        metrics
+      )
+    };
+  }
+  /**
+   * Calculate result for a single metric
+   */
+  calculateMetricResult(metric) {
+    const controlScores = this.getScoresForMetric(this.controlSamples, metric);
+    const treatmentScores = this.getScoresForMetric(
+      this.treatmentSamples,
+      metric
+    );
+    const controlSummary = this.calculateSummary(controlScores);
+    const treatmentSummary = this.calculateSummary(treatmentScores);
+    const difference = treatmentSummary.mean - controlSummary.mean;
+    const differencePercent = controlSummary.mean !== 0 ? difference / controlSummary.mean * 100 : 0;
+    const pValue = this.calculatePValue(controlScores, treatmentScores);
+    const isSignificant = pValue < (this.config.significanceLevel ?? 0.05);
+    let winner = "none";
+    if (isSignificant) {
+      winner = difference > 0 ? "treatment" : "control";
+    }
+    return {
+      control: controlSummary,
+      treatment: treatmentSummary,
+      difference,
+      differencePercent,
+      pValue,
+      isSignificant,
+      winner
+    };
+  }
+  /**
+   * Get scores for a metric from samples
+   */
+  getScoresForMetric(samples, metric) {
+    const scores = [];
+    for (const sampleScores of samples.values()) {
+      if (metric in sampleScores) {
+        scores.push(sampleScores[metric]);
+      }
+    }
+    return scores;
+  }
+  /**
+   * Calculate summary statistics
+   */
+  calculateSummary(scores) {
+    if (scores.length === 0) {
+      return {
+        mean: 0,
+        std: 0,
+        sampleCount: 0,
+        confidenceInterval: [0, 0]
+      };
+    }
+    const mean = scores.reduce((a, b) => a + b, 0) / scores.length;
+    const variance = scores.reduce((sum, s) => sum + Math.pow(s - mean, 2), 0) / scores.length;
+    const std = Math.sqrt(variance);
+    const se = std / Math.sqrt(scores.length);
+    const margin = 1.96 * se;
+    return {
+      mean,
+      std,
+      sampleCount: scores.length,
+      confidenceInterval: [mean - margin, mean + margin]
+    };
+  }
+  /**
+   * Calculate p-value using Welch's t-test approximation
+   */
+  calculatePValue(control, treatment) {
+    if (control.length < 2 || treatment.length < 2) return 1;
+    const n1 = control.length;
+    const n2 = treatment.length;
+    const mean1 = control.reduce((a, b) => a + b, 0) / n1;
+    const mean2 = treatment.reduce((a, b) => a + b, 0) / n2;
+    const var1 = control.reduce((sum, x) => sum + Math.pow(x - mean1, 2), 0) / (n1 - 1);
+    const var2 = treatment.reduce((sum, x) => sum + Math.pow(x - mean2, 2), 0) / (n2 - 1);
+    const se = Math.sqrt(var1 / n1 + var2 / n2);
+    if (se === 0) return 1;
+    const t = Math.abs(mean1 - mean2) / se;
+    const pValue = 2 * (1 - this.normalCDF(t));
+    return Math.max(0, Math.min(1, pValue));
+  }
+  /**
+   * Normal CDF approximation
+   */
+  normalCDF(x) {
+    const a1 = 0.254829592;
+    const a2 = -0.284496736;
+    const a3 = 1.421413741;
+    const a4 = -1.453152027;
+    const a5 = 1.061405429;
+    const p = 0.3275911;
+    const sign = x < 0 ? -1 : 1;
+    x = Math.abs(x) / Math.sqrt(2);
+    const t = 1 / (1 + p * x);
+    const y = 1 - ((((a5 * t + a4) * t + a3) * t + a2) * t + a1) * t * Math.exp(-x * x);
+    return 0.5 * (1 + sign * y);
+  }
+  /**
+   * Check for statistical significance
+   */
+  checkSignificance() {
+    for (const metric of this.config.metrics) {
+      const result = this.calculateMetricResult(metric);
+      if (result.isSignificant && result.winner !== "none") {
+        this.emit("test:significant", metric, result.winner);
+      }
+    }
+  }
+  /**
+   * Generate recommendation
+   */
+  generateRecommendation(winner, isSignificant, metrics) {
+    if (!isSignificant) {
+      const totalSamples = this.controlSamples.size + this.treatmentSamples.size;
+      if (totalSamples < this.config.minSamples) {
+        return `Not enough samples yet. Need ${this.config.minSamples - totalSamples} more.`;
+      }
+      return "No significant difference detected. Consider running longer or adjusting variants.";
+    }
+    if (winner === "treatment") {
+      const improvements = Object.entries(metrics).filter(([, r]) => r.winner === "treatment").map(([m, r]) => `${m}: +${r.differencePercent.toFixed(1)}%`);
+      return `Treatment variant wins. Improvements: ${improvements.join(", ")}`;
+    }
+    if (winner === "control") {
+      return "Control variant performs better. Consider keeping current configuration.";
+    }
+    return "Mixed results. Review individual metrics for details.";
+  }
+  /**
+   * Get test status
+   */
+  getStatus() {
+    return this.status;
+  }
+  /**
+   * Get test configuration
+   */
+  getConfig() {
+    return { ...this.config };
+  }
+  /**
+   * Get test timestamps
+   */
+  getTimestamps() {
+    return {
+      startedAt: this.startedAt,
+      completedAt: this.completedAt
+    };
+  }
+};
+function createABTestRunner(config) {
+  return new ABTestRunner(config);
+}
+
+// src/continuous/ContinuousEval.ts
+var ContinuousEval = class extends import_eventemitter32.EventEmitter {
+  pipeline;
+  sampleRate;
+  status = "stopped";
+  startedAt;
+  lastEvalAt;
+  totalEvaluations = 0;
+  passedCount = 0;
+  scoreHistory = {};
+  alertManager;
+  abTests = /* @__PURE__ */ new Map();
+  intervalId;
+  constructor(config) {
+    super();
+    this.pipeline = config.pipeline;
+    this.sampleRate = config.sampleRate;
+  }
+  /**
+   * Set alert manager
+   */
+  setAlerts(alertManager, rules) {
+    this.alertManager = alertManager;
+    for (const [metric, rule] of Object.entries(rules)) {
+      alertManager.addRule({
+        metric,
+        threshold: rule.threshold,
+        direction: rule.direction
+      });
+    }
+  }
+  /**
+   * Start monitoring
+   */
+  start() {
+    if (this.status === "running") return;
+    this.status = "running";
+    this.startedAt = Date.now();
+    this.emit("status:changed", this.status);
+    this.emit("eval:started");
+  }
+  /**
+   * Stop monitoring
+   */
+  stop() {
+    this.status = "stopped";
+    if (this.intervalId) {
+      clearInterval(this.intervalId);
+      this.intervalId = void 0;
+    }
+    this.emit("status:changed", this.status);
+  }
+  /**
+   * Pause monitoring
+   */
+  pause() {
+    if (this.status === "running") {
+      this.status = "paused";
+      this.emit("status:changed", this.status);
+    }
+  }
+  /**
+   * Resume monitoring
+   */
+  resume() {
+    if (this.status === "paused") {
+      this.status = "running";
+      this.emit("status:changed", this.status);
+    }
+  }
+  /**
+   * Evaluate a sample
+   */
+  async evaluate(input) {
+    if (Math.random() > this.sampleRate) {
+      return null;
+    }
+    if (this.status !== "running") {
+      return null;
+    }
+    try {
+      const result = await this.pipeline.evaluate(input);
+      this.totalEvaluations++;
+      this.lastEvalAt = Date.now();
+      if (result.passed) {
+        this.passedCount++;
+      }
+      for (const [metric, score] of Object.entries(result.scores)) {
+        if (!this.scoreHistory[metric]) {
+          this.scoreHistory[metric] = [];
+        }
+        this.scoreHistory[metric].push(score);
+        if (this.scoreHistory[metric].length > 1e3) {
+          this.scoreHistory[metric].shift();
+        }
+        if (this.alertManager) {
+          this.alertManager.check(metric, score);
+        }
+      }
+      this.emit("eval:completed", result);
+      return result;
+    } catch (error) {
+      this.status = "error";
+      this.emit("eval:error", error);
+      this.emit("status:changed", this.status);
+      return null;
+    }
+  }
+  /**
+   * Get statistics
+   */
+  getStats() {
+    const avgScores = {};
+    for (const [metric, scores] of Object.entries(this.scoreHistory)) {
+      if (scores.length > 0) {
+        avgScores[metric] = scores.reduce((a, b) => a + b, 0) / scores.length;
+      }
+    }
+    return {
+      status: this.status,
+      startedAt: this.startedAt,
+      lastEvalAt: this.lastEvalAt,
+      totalEvaluations: this.totalEvaluations,
+      passRate: this.totalEvaluations > 0 ? this.passedCount / this.totalEvaluations : 0,
+      avgScores,
+      alertsTriggered: this.alertManager?.getAlertCount() ?? 0
+    };
+  }
+  /**
+   * Create A/B test
+   */
+  createABTest(config) {
+    const test = new ABTestRunner(config);
+    this.abTests.set(test.id, test);
+    return test;
+  }
+  /**
+   * Get A/B test
+   */
+  getABTest(id) {
+    return this.abTests.get(id);
+  }
+  /**
+   * Get all A/B tests
+   */
+  getABTests() {
+    return Array.from(this.abTests.values());
+  }
+  /**
+   * Get score history for a metric
+   */
+  getScoreHistory(metric) {
+    return this.scoreHistory[metric] ?? [];
+  }
+  /**
+   * Reset statistics
+   */
+  reset() {
+    this.totalEvaluations = 0;
+    this.passedCount = 0;
+    this.scoreHistory = {};
+    this.lastEvalAt = void 0;
+  }
+};
+function createContinuousEval(config) {
+  return new ContinuousEval(config);
+}
+
+// src/continuous/AlertManager.ts
+var import_eventemitter33 = require("eventemitter3");
+var import_nanoid2 = require("nanoid");
+var AlertManager = class extends import_eventemitter33.EventEmitter {
+  channels;
+  rules = /* @__PURE__ */ new Map();
+  activeAlerts = /* @__PURE__ */ new Map();
+  cooldownMs;
+  lastAlertTime = /* @__PURE__ */ new Map();
+  alertCount = 0;
+  constructor(config) {
+    super();
+    this.channels = config.channels;
+    this.cooldownMs = config.cooldownMs ?? 3e5;
+    if (config.rules) {
+      for (const [metric, rule] of Object.entries(config.rules)) {
+        this.rules.set(metric, rule);
+      }
+    }
+  }
+  /**
+   * Add an alert rule
+   */
+  addRule(rule) {
+    this.rules.set(rule.metric, rule);
+  }
+  /**
+   * Remove an alert rule
+   */
+  removeRule(metric) {
+    return this.rules.delete(metric);
+  }
+  /**
+   * Check value against rules
+   */
+  check(metric, value) {
+    const rule = this.rules.get(metric);
+    if (!rule) return null;
+    const shouldAlert = rule.direction === "above" && value > rule.threshold || rule.direction === "below" && value < rule.threshold;
+    if (shouldAlert) {
+      return this.triggerAlert(rule, metric, value);
+    } else {
+      const existingAlert = this.activeAlerts.get(metric);
+      if (existingAlert) {
+        this.resolveAlert(metric);
+      }
+    }
+    return null;
+  }
+  /**
+   * Trigger an alert
+   */
+  triggerAlert(rule, metric, value) {
+    const lastTime = this.lastAlertTime.get(metric);
+    if (lastTime && Date.now() - lastTime < this.cooldownMs) {
+      return null;
+    }
+    const alert = {
+      id: (0, import_nanoid2.nanoid)(),
+      rule,
+      metric,
+      currentValue: value,
+      threshold: rule.threshold,
+      severity: rule.severity ?? "warning",
+      message: this.formatMessage(rule, metric, value),
+      triggeredAt: Date.now()
+    };
+    this.activeAlerts.set(metric, alert);
+    this.lastAlertTime.set(metric, Date.now());
+    this.alertCount++;
+    this.emit("alert:triggered", alert);
+    void this.sendNotifications(alert);
+    return alert;
+  }
+  /**
+   * Resolve an alert
+   */
+  resolveAlert(metric) {
+    const alert = this.activeAlerts.get(metric);
+    if (alert) {
+      alert.resolvedAt = Date.now();
+      this.activeAlerts.delete(metric);
+      this.emit("alert:resolved", alert);
+    }
+  }
+  /**
+   * Acknowledge an alert
+   */
+  acknowledgeAlert(alertId) {
+    for (const alert of this.activeAlerts.values()) {
+      if (alert.id === alertId) {
+        alert.acknowledged = true;
+        return true;
+      }
+    }
+    return false;
+  }
+  /**
+   * Send notifications
+   */
+  async sendNotifications(alert) {
+    for (const channel of this.channels) {
+      try {
+        await this.sendToChannel(channel, alert);
+        const notification = {
+          alertId: alert.id,
+          channel: channel.type,
+          sentAt: Date.now(),
+          success: true
+        };
+        this.emit("notification:sent", notification);
+      } catch (error) {
+        const notification = {
+          alertId: alert.id,
+          channel: channel.type,
+          sentAt: Date.now(),
+          success: false,
+          error: error.message
+        };
+        this.emit("notification:sent", notification);
+      }
+    }
+  }
+  /**
+   * Send to a specific channel
+   */
+  async sendToChannel(channel, alert) {
+    switch (channel.type) {
+      case "webhook":
+        if (channel.webhook) {
+          await fetch(channel.webhook, {
+            method: "POST",
+            headers: { "Content-Type": "application/json" },
+            body: JSON.stringify({
+              alert: {
+                id: alert.id,
+                metric: alert.metric,
+                severity: alert.severity,
+                message: alert.message,
+                value: alert.currentValue,
+                threshold: alert.threshold,
+                triggeredAt: new Date(alert.triggeredAt).toISOString()
+              }
+            })
+          });
+        }
+        break;
+      case "slack":
+        if (channel.webhook) {
+          await fetch(channel.webhook, {
+            method: "POST",
+            headers: { "Content-Type": "application/json" },
+            body: JSON.stringify({
+              text: `\u{1F6A8} *${alert.severity.toUpperCase()}*: ${alert.message}`,
+              attachments: [
+                {
+                  color: alert.severity === "critical" ? "danger" : "warning",
+                  fields: [
+                    { title: "Metric", value: alert.metric, short: true },
+                    {
+                      title: "Value",
+                      value: alert.currentValue.toFixed(4),
+                      short: true
+                    },
+                    {
+                      title: "Threshold",
+                      value: alert.threshold.toString(),
+                      short: true
+                    }
+                  ]
+                }
+              ]
+            })
+          });
+        }
+        break;
+      case "email":
+        console.log(`[Email Alert] To: ${channel.to?.join(", ")}`);
+        console.log(`Subject: Alert: ${alert.metric} - ${alert.severity}`);
+        console.log(`Body: ${alert.message}`);
+        break;
+      case "pagerduty":
+        console.log(`[PagerDuty Alert] ${alert.severity}: ${alert.message}`);
+        break;
+    }
+  }
+  /**
+   * Format alert message
+   */
+  formatMessage(rule, metric, value) {
+    const direction = rule.direction === "above" ? "exceeded" : "dropped below";
+    return `${metric} ${direction} threshold: ${value.toFixed(4)} (threshold: ${rule.threshold})`;
+  }
+  /**
+   * Get active alerts
+   */
+  getActiveAlerts() {
+    return Array.from(this.activeAlerts.values());
+  }
+  /**
+   * Get alert count
+   */
+  getAlertCount() {
+    return this.alertCount;
+  }
+  /**
+   * Get rules
+   */
+  getRules() {
+    return Array.from(this.rules.values());
+  }
+};
+function createAlertManager(config) {
+  return new AlertManager(config);
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  ABTestRunner,
+  AlertManager,
+  ContinuousEval,
+  createABTestRunner,
+  createAlertManager,
+  createContinuousEval
+});

package/dist/continuous/index.mjs

@@ -0,0 +1,16 @@
+import {
+  ABTestRunner,
+  AlertManager,
+  ContinuousEval,
+  createABTestRunner,
+  createAlertManager,
+  createContinuousEval
+} from "../chunk-EUXXIZK3.mjs";
+export {
+  ABTestRunner,
+  AlertManager,
+  ContinuousEval,
+  createABTestRunner,
+  createAlertManager,
+  createContinuousEval
+};

package/dist/datasets/index.d.mts

@@ -0,0 +1 @@
+export { v as DatasetExporter, s as PreferenceDataset, t as PreferenceDatasetBuilder, w as createDatasetExporter, u as createPreferenceDatasetBuilder } from '../index-6Pbiq7ny.mjs';