@lov3kaizen/agentsea-evaluate 0.5.1

package/dist/index.mjs

@@ -0,0 +1,429 @@
+import {
+  AnnotationQueue,
+  AnnotationTask,
+  BinaryClassificationSchema,
+  ConsensusManager,
+  QualityRatingSchema,
+  TextSpanSchema,
+  createAnnotationQueue,
+  createAnnotationTask,
+  createConsensusManager
+} from "./chunk-NBMUSATK.mjs";
+import {
+  ABTestRunner,
+  AlertManager,
+  ContinuousEval,
+  createABTestRunner,
+  createAlertManager,
+  createContinuousEval
+} from "./chunk-EUXXIZK3.mjs";
+import {
+  DatasetExporter,
+  PreferenceDataset,
+  PreferenceDatasetBuilder,
+  createDatasetExporter,
+  createPreferenceDatasetBuilder
+} from "./chunk-TUMNJN2S.mjs";
+import {
+  Accuracy,
+  BaseMetric,
+  CodeQualityRubric,
+  Coherence,
+  ComparativeJudge,
+  ConsensusJudge,
+  ContextRelevance,
+  CustomMetric,
+  EvalDataset,
+  EvalRunner,
+  EvaluationPipeline,
+  Faithfulness,
+  HelpfulnessRubric,
+  LLMJudge,
+  QualityRubric,
+  Relevance,
+  RubricJudge,
+  Toxicity,
+  createAccuracyMetric,
+  createCoherenceMetric,
+  createComparativeJudge,
+  createConsensusJudge,
+  createContainsMetric,
+  createContextRelevanceMetric,
+  createCustomMetric,
+  createEvalDataset,
+  createEvalRunner,
+  createEvaluationPipeline,
+  createFaithfulnessMetric,
+  createJSONMetric,
+  createLLMJudge,
+  createLengthMetric,
+  createRegexMetric,
+  createRelevanceMetric,
+  createRubricJudge,
+  createSimpleMetric,
+  createToxicityMetric
+} from "./chunk-5JRYKRSE.mjs";
+import {
+  BaseCollector,
+  CorrectionCollector,
+  FeedbackAggregator,
+  FeedbackExporter,
+  MemoryFeedbackStore,
+  MultiCriteriaCollector,
+  PreferenceCollector,
+  RatingCollector,
+  SQLiteFeedbackStore,
+  ThumbsCollector,
+  createCorrectionCollector,
+  createFeedbackAggregator,
+  createFeedbackExporter,
+  createFeedbackStore,
+  createMultiCriteriaCollector,
+  createPreferenceCollector,
+  createRatingCollector,
+  createThumbsCollector
+} from "./chunk-PAQ2TTJJ.mjs";
+
+// src/types/feedback.types.ts
+import { z } from "zod";
+var ThumbsRatingSchema = z.enum(["up", "down"]);
+var StarRatingSchema = z.union([
+  z.literal(1),
+  z.literal(2),
+  z.literal(3),
+  z.literal(4),
+  z.literal(5)
+]);
+var PreferenceChoiceSchema = z.enum(["A", "B", "tie"]);
+var CollectThumbsInputSchema = z.object({
+  responseId: z.string(),
+  conversationId: z.string().optional(),
+  input: z.string(),
+  output: z.string(),
+  feedback: z.object({
+    rating: ThumbsRatingSchema,
+    comment: z.string().optional()
+  }),
+  userId: z.string().optional(),
+  metadata: z.record(z.unknown()).optional()
+});
+var CollectPreferenceInputSchema = z.object({
+  input: z.string(),
+  responseA: z.object({
+    id: z.string(),
+    content: z.string(),
+    model: z.string().optional()
+  }),
+  responseB: z.object({
+    id: z.string(),
+    content: z.string(),
+    model: z.string().optional()
+  }),
+  preference: PreferenceChoiceSchema,
+  reason: z.string().optional(),
+  confidence: z.number().min(0).max(1).optional(),
+  userId: z.string().optional(),
+  metadata: z.record(z.unknown()).optional()
+});
+
+// src/integrations/agentsea/FeedbackMiddleware.ts
+var FeedbackMiddleware = class {
+  collector;
+  autoCapture;
+  captureFields;
+  pendingFeedback = /* @__PURE__ */ new Map();
+  constructor(options) {
+    this.collector = options.collector ?? new ThumbsCollector({ store: options.store });
+    this.autoCapture = options.autoCapture ?? true;
+    this.captureFields = options.captureFields ?? [
+      "input",
+      "output",
+      "toolCalls",
+      "latency"
+    ];
+  }
+  /**
+   * Process agent message and capture for potential feedback
+   */
+  capture(context) {
+    if (!this.autoCapture) return;
+    const messages = context.messages;
+    if (messages.length < 2) return;
+    let userMessage;
+    let assistantMessage;
+    for (let i = messages.length - 1; i >= 0; i--) {
+      if (!assistantMessage && messages[i].role === "assistant") {
+        assistantMessage = messages[i];
+      }
+      if (!userMessage && messages[i].role === "user") {
+        userMessage = messages[i];
+      }
+      if (userMessage && assistantMessage) break;
+    }
+    if (!userMessage || !assistantMessage) return;
+    const metadata = {};
+    if (this.captureFields.includes("toolCalls") && assistantMessage.metadata?.toolCalls) {
+      metadata.toolCalls = assistantMessage.metadata.toolCalls;
+    }
+    if (this.captureFields.includes("latency") && assistantMessage.metadata?.latencyMs) {
+      metadata.latencyMs = assistantMessage.metadata.latencyMs;
+    }
+    if (context.metadata?.model) {
+      metadata.model = context.metadata.model;
+    }
+    this.pendingFeedback.set(assistantMessage.id, {
+      input: userMessage.content,
+      output: assistantMessage.content,
+      conversationId: context.conversationId,
+      metadata,
+      timestamp: Date.now()
+    });
+    this.cleanupPending();
+  }
+  /**
+   * Record feedback for a response
+   */
+  async recordFeedback(responseId, rating, comment, userId) {
+    const pending = this.pendingFeedback.get(responseId);
+    if (!pending) {
+      console.warn(`No pending feedback found for response ${responseId}`);
+      return null;
+    }
+    const feedback = await this.collector.collect({
+      responseId,
+      conversationId: pending.conversationId,
+      input: pending.input,
+      output: pending.output,
+      feedback: { rating, comment },
+      userId,
+      metadata: pending.metadata
+    });
+    this.pendingFeedback.delete(responseId);
+    return feedback;
+  }
+  /**
+   * Get pending feedback IDs
+   */
+  getPendingIds() {
+    return Array.from(this.pendingFeedback.keys());
+  }
+  /**
+   * Clear pending feedback
+   */
+  clearPending() {
+    this.pendingFeedback.clear();
+  }
+  /**
+   * Clean up old pending feedback
+   */
+  cleanupPending() {
+    const oneHourAgo = Date.now() - 36e5;
+    for (const [id, data] of this.pendingFeedback.entries()) {
+      if (data.timestamp < oneHourAgo) {
+        this.pendingFeedback.delete(id);
+      }
+    }
+  }
+  /**
+   * Get collector
+   */
+  getCollector() {
+    return this.collector;
+  }
+};
+function createFeedbackMiddleware(options) {
+  return new FeedbackMiddleware(options);
+}
+
+// src/integrations/agentsea/AgentEvaluator.ts
+var AgentEvaluator = class {
+  pipeline;
+  scenarios;
+  constructor(options) {
+    this.pipeline = options.pipeline;
+    this.scenarios = options.scenarios;
+  }
+  /**
+   * Evaluate an agent
+   */
+  async evaluate(agent) {
+    const categoryScores = {};
+    const categoryResults = {};
+    const recommendations = [];
+    let totalTests = 0;
+    let totalPassed = 0;
+    let weightedScoreSum = 0;
+    let totalWeight = 0;
+    for (const scenario of this.scenarios) {
+      const result = await this.pipeline.evaluate({
+        dataset: scenario.dataset,
+        generateFn: async (input, context) => {
+          return agent.execute(input, context);
+        }
+      });
+      const avgScore = result.summary.avgScore;
+      const weight = scenario.weight ?? 1;
+      categoryScores[scenario.category] = avgScore;
+      categoryResults[scenario.category] = result;
+      totalTests += result.summary.totalItems;
+      totalPassed += result.summary.passedItems;
+      weightedScoreSum += avgScore * weight;
+      totalWeight += weight;
+      if (avgScore < 0.7) {
+        recommendations.push(
+          `Improve ${scenario.category}: current score ${(avgScore * 100).toFixed(1)}%`
+        );
+      }
+      if (result.summary.passRate < 0.8) {
+        const topFailures = result.failures.slice(0, 3).map((f) => f.failedMetrics.join(", "));
+        if (topFailures.length > 0) {
+          recommendations.push(
+            `${scenario.category} failures often in: ${[...new Set(topFailures)].join(", ")}`
+          );
+        }
+      }
+    }
+    const overallScore = totalWeight > 0 ? weightedScoreSum / totalWeight : 0;
+    return {
+      overallScore,
+      categoryScores,
+      categoryResults,
+      recommendations,
+      summary: {
+        totalTests,
+        passed: totalPassed,
+        failed: totalTests - totalPassed,
+        passRate: totalTests > 0 ? totalPassed / totalTests : 0
+      }
+    };
+  }
+  /**
+   * Run quick benchmark
+   */
+  async benchmark(agent, sampleSize = 10) {
+    const allItems = [];
+    for (const scenario of this.scenarios) {
+      allItems.push(...scenario.dataset.sample(sampleSize).getItems());
+    }
+    const startTime = performance.now();
+    let totalScore = 0;
+    let count = 0;
+    for (const item of allItems.slice(0, sampleSize)) {
+      try {
+        const output = await agent.execute(item.input);
+        if (output && output.length > 0) {
+          totalScore += 1;
+        }
+        count++;
+      } catch {
+        count++;
+      }
+    }
+    const latencyMs = (performance.now() - startTime) / count;
+    return {
+      score: count > 0 ? totalScore / count : 0,
+      latencyMs
+    };
+  }
+  /**
+   * Add a scenario
+   */
+  addScenario(scenario) {
+    this.scenarios.push(scenario);
+  }
+  /**
+   * Get scenarios
+   */
+  getScenarios() {
+    return [...this.scenarios];
+  }
+};
+function createAgentEvaluator(options) {
+  return new AgentEvaluator(options);
+}
+export {
+  ABTestRunner,
+  Accuracy,
+  AgentEvaluator,
+  AlertManager,
+  AnnotationQueue,
+  AnnotationTask,
+  BaseCollector,
+  BaseMetric,
+  BinaryClassificationSchema,
+  CodeQualityRubric,
+  Coherence,
+  CollectPreferenceInputSchema,
+  CollectThumbsInputSchema,
+  ComparativeJudge,
+  ConsensusJudge,
+  ConsensusManager,
+  ContextRelevance,
+  ContinuousEval,
+  CorrectionCollector,
+  CustomMetric,
+  DatasetExporter,
+  EvalDataset,
+  EvalRunner,
+  EvaluationPipeline,
+  Faithfulness,
+  FeedbackAggregator,
+  FeedbackExporter,
+  FeedbackMiddleware,
+  HelpfulnessRubric,
+  LLMJudge,
+  MemoryFeedbackStore,
+  MultiCriteriaCollector,
+  PreferenceChoiceSchema,
+  PreferenceCollector,
+  PreferenceDataset,
+  PreferenceDatasetBuilder,
+  QualityRatingSchema,
+  QualityRubric,
+  RatingCollector,
+  Relevance,
+  RubricJudge,
+  SQLiteFeedbackStore,
+  StarRatingSchema,
+  TextSpanSchema,
+  ThumbsCollector,
+  ThumbsRatingSchema,
+  Toxicity,
+  createABTestRunner,
+  createAccuracyMetric,
+  createAgentEvaluator,
+  createAlertManager,
+  createAnnotationQueue,
+  createAnnotationTask,
+  createCoherenceMetric,
+  createComparativeJudge,
+  createConsensusJudge,
+  createConsensusManager,
+  createContainsMetric,
+  createContextRelevanceMetric,
+  createContinuousEval,
+  createCorrectionCollector,
+  createCustomMetric,
+  createDatasetExporter,
+  createEvalDataset,
+  createEvalRunner,
+  createEvaluationPipeline,
+  createFaithfulnessMetric,
+  createFeedbackAggregator,
+  createFeedbackExporter,
+  createFeedbackMiddleware,
+  createFeedbackStore,
+  createJSONMetric,
+  createLLMJudge,
+  createLengthMetric,
+  createMultiCriteriaCollector,
+  createPreferenceCollector,
+  createPreferenceDatasetBuilder,
+  createRatingCollector,
+  createRegexMetric,
+  createRelevanceMetric,
+  createRubricJudge,
+  createSimpleMetric,
+  createThumbsCollector,
+  createToxicityMetric
+};

package/package.json

@@ -0,0 +1,102 @@
+{
+  "name": "@lov3kaizen/agentsea-evaluate",
+  "version": "0.5.1",
+  "description": "Comprehensive feedback collection and LLM evaluation platform for Node.js - human-in-the-loop annotation, automated evaluation pipelines, preference dataset generation",
+  "main": "dist/index.js",
+  "module": "dist/index.mjs",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.mjs",
+      "require": "./dist/index.js"
+    },
+    "./feedback": {
+      "types": "./dist/feedback/index.d.ts",
+      "import": "./dist/feedback/index.mjs",
+      "require": "./dist/feedback/index.js"
+    },
+    "./evaluation": {
+      "types": "./dist/evaluation/index.d.ts",
+      "import": "./dist/evaluation/index.mjs",
+      "require": "./dist/evaluation/index.js"
+    },
+    "./datasets": {
+      "types": "./dist/datasets/index.d.ts",
+      "import": "./dist/datasets/index.mjs",
+      "require": "./dist/datasets/index.js"
+    },
+    "./annotation": {
+      "types": "./dist/annotation/index.d.ts",
+      "import": "./dist/annotation/index.mjs",
+      "require": "./dist/annotation/index.js"
+    },
+    "./continuous": {
+      "types": "./dist/continuous/index.d.ts",
+      "import": "./dist/continuous/index.mjs",
+      "require": "./dist/continuous/index.js"
+    }
+  },
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "keywords": [
+    "llm",
+    "evaluation",
+    "feedback",
+    "annotation",
+    "rlhf",
+    "dpo",
+    "preference-learning",
+    "llm-as-judge",
+    "ai",
+    "machine-learning",
+    "nlp",
+    "rag",
+    "quality-assurance"
+  ],
+  "author": "lov3kaizen",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/lov3kaizen/agentsea.git",
+    "directory": "packages/evaluate"
+  },
+  "dependencies": {
+    "eventemitter3": "^5.0.0",
+    "nanoid": "^5.0.0",
+    "zod": "^3.22.0"
+  },
+  "devDependencies": {
+    "@types/better-sqlite3": "^7.6.0",
+    "@types/node": "^20.0.0",
+    "tsup": "^8.0.0",
+    "typescript": "^5.3.0",
+    "vitest": "^1.0.0"
+  },
+  "peerDependencies": {
+    "@lov3kaizen/agentsea-core": ">=0.5.0"
+  },
+  "peerDependenciesMeta": {
+    "@lov3kaizen/agentsea-core": {
+      "optional": true
+    }
+  },
+  "optionalDependencies": {
+    "better-sqlite3": "^9.2.0",
+    "@huggingface/hub": "^0.14.0"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "scripts": {
+    "build": "tsup src/index.ts src/feedback/index.ts src/evaluation/index.ts src/datasets/index.ts src/annotation/index.ts src/continuous/index.ts --format cjs,esm --dts --clean --external better-sqlite3 --external @huggingface/hub",
+    "dev": "tsup src/index.ts --format cjs,esm --dts --watch",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "test:coverage": "vitest run --coverage",
+    "lint": "eslint src --ext .ts",
+    "typecheck": "tsc --noEmit"
+  }
+}