@lov3kaizen/agentsea-evaluate 0.5.0

package/dist/chunk-NBMUSATK.mjs

@@ -0,0 +1,596 @@
+// src/annotation/AnnotationTask.ts
+import { nanoid } from "nanoid";
+import { z } from "zod";
+var AnnotationTask = class {
+  id;
+  name;
+  description;
+  instructions;
+  schema;
+  status;
+  itemsPerAnnotator;
+  annotatorsPerItem;
+  deadline;
+  createdAt;
+  updatedAt;
+  completedAt;
+  metadata;
+  constructor(config) {
+    this.id = nanoid();
+    this.name = config.name;
+    this.description = config.description;
+    this.instructions = config.instructions;
+    this.schema = config.schema;
+    this.status = "draft";
+    this.itemsPerAnnotator = config.itemsPerAnnotator ?? 100;
+    this.annotatorsPerItem = config.annotatorsPerItem ?? 1;
+    this.deadline = config.deadline;
+    this.createdAt = Date.now();
+    this.updatedAt = Date.now();
+    this.metadata = config.metadata;
+  }
+  /**
+   * Start the task
+   */
+  start() {
+    if (this.status !== "draft") {
+      throw new Error(`Cannot start task in ${this.status} status`);
+    }
+    this.status = "active";
+    this.updatedAt = Date.now();
+  }
+  /**
+   * Pause the task
+   */
+  pause() {
+    if (this.status !== "active") {
+      throw new Error(`Cannot pause task in ${this.status} status`);
+    }
+    this.status = "paused";
+    this.updatedAt = Date.now();
+  }
+  /**
+   * Resume the task
+   */
+  resume() {
+    if (this.status !== "paused") {
+      throw new Error(`Cannot resume task in ${this.status} status`);
+    }
+    this.status = "active";
+    this.updatedAt = Date.now();
+  }
+  /**
+   * Complete the task
+   */
+  complete() {
+    if (this.status !== "active") {
+      throw new Error(`Cannot complete task in ${this.status} status`);
+    }
+    this.status = "completed";
+    this.completedAt = Date.now();
+    this.updatedAt = Date.now();
+  }
+  /**
+   * Cancel the task
+   */
+  cancel() {
+    if (this.status === "completed") {
+      throw new Error("Cannot cancel completed task");
+    }
+    this.status = "cancelled";
+    this.updatedAt = Date.now();
+  }
+  /**
+   * Validate an annotation against the schema
+   */
+  validateAnnotation(value) {
+    try {
+      this.schema.parse(value);
+      return { valid: true };
+    } catch (error) {
+      if (error instanceof z.ZodError) {
+        return {
+          valid: false,
+          error: error.errors.map((e) => e.message).join(", ")
+        };
+      }
+      return { valid: false, error: "Unknown validation error" };
+    }
+  }
+  /**
+   * Check if task is past deadline
+   */
+  isPastDeadline() {
+    if (!this.deadline) return false;
+    return Date.now() > this.deadline.getTime();
+  }
+  /**
+   * Get task configuration for display
+   */
+  toConfig() {
+    return {
+      name: this.name,
+      description: this.description,
+      instructions: this.instructions,
+      schema: this.schema,
+      itemsPerAnnotator: this.itemsPerAnnotator,
+      annotatorsPerItem: this.annotatorsPerItem,
+      deadline: this.deadline,
+      metadata: this.metadata
+    };
+  }
+};
+function createAnnotationTask(config) {
+  return new AnnotationTask(config);
+}
+var BinaryClassificationSchema = z.object({
+  label: z.enum(["positive", "negative"]),
+  confidence: z.number().min(0).max(1).optional(),
+  notes: z.string().optional()
+});
+var QualityRatingSchema = z.object({
+  accuracy: z.number().min(1).max(5),
+  helpfulness: z.number().min(1).max(5),
+  safety: z.enum(["pass", "fail"]),
+  corrections: z.string().optional(),
+  notes: z.string().optional()
+});
+var TextSpanSchema = z.object({
+  spans: z.array(
+    z.object({
+      start: z.number(),
+      end: z.number(),
+      label: z.string(),
+      text: z.string().optional()
+    })
+  ),
+  notes: z.string().optional()
+});
+
+// src/annotation/AnnotationQueue.ts
+import { nanoid as nanoid2 } from "nanoid";
+import { EventEmitter } from "eventemitter3";
+var AnnotationQueue = class extends EventEmitter {
+  task;
+  items;
+  annotatorAssignments;
+  annotatorCounts;
+  constructor(config) {
+    super();
+    this.task = config.task;
+    this.items = /* @__PURE__ */ new Map();
+    this.annotatorAssignments = /* @__PURE__ */ new Map();
+    this.annotatorCounts = /* @__PURE__ */ new Map();
+    for (const item of config.items) {
+      this.items.set(item.id, item);
+    }
+  }
+  /**
+   * Get next item for annotator
+   */
+  getNextItem(annotatorId) {
+    const assigned = this.annotatorAssignments.get(annotatorId) ?? /* @__PURE__ */ new Set();
+    for (const item of this.items.values()) {
+      if (assigned.has(item.id)) continue;
+      if (item.status === "completed") continue;
+      const assignedCount = item.assignedTo?.length ?? 0;
+      if (assignedCount >= this.task.annotatorsPerItem) continue;
+      if (assignedCount === 0) {
+        this.assignItem(item.id, annotatorId);
+        return item;
+      }
+    }
+    for (const item of this.items.values()) {
+      if (assigned.has(item.id)) continue;
+      if (item.status === "completed") continue;
+      const assignedCount = item.assignedTo?.length ?? 0;
+      if (assignedCount >= this.task.annotatorsPerItem) continue;
+      this.assignItem(item.id, annotatorId);
+      return item;
+    }
+    return null;
+  }
+  /**
+   * Assign item to annotator
+   */
+  assignItem(itemId, annotatorId) {
+    const item = this.items.get(itemId);
+    if (!item) {
+      throw new Error(`Item ${itemId} not found`);
+    }
+    if (!this.annotatorAssignments.has(annotatorId)) {
+      this.annotatorAssignments.set(annotatorId, /* @__PURE__ */ new Set());
+    }
+    this.annotatorAssignments.get(annotatorId).add(itemId);
+    if (!item.assignedTo) {
+      item.assignedTo = [];
+    }
+    if (!item.assignedTo.includes(annotatorId)) {
+      item.assignedTo.push(annotatorId);
+    }
+    item.status = "assigned";
+    item.updatedAt = Date.now();
+    this.emit("item:assigned", itemId, annotatorId);
+  }
+  /**
+   * Submit annotation for item
+   */
+  submitAnnotation(itemId, annotatorId, value, duration) {
+    const item = this.items.get(itemId);
+    if (!item) {
+      throw new Error(`Item ${itemId} not found`);
+    }
+    const validation = this.task.validateAnnotation(value);
+    if (!validation.valid) {
+      throw new Error(`Invalid annotation: ${validation.error}`);
+    }
+    const annotation = {
+      id: nanoid2(),
+      itemId,
+      annotatorId,
+      value,
+      duration,
+      createdAt: Date.now()
+    };
+    item.annotations.push(annotation);
+    item.updatedAt = Date.now();
+    this.annotatorCounts.set(
+      annotatorId,
+      (this.annotatorCounts.get(annotatorId) ?? 0) + 1
+    );
+    if (item.annotations.length >= this.task.annotatorsPerItem) {
+      item.status = "completed";
+    } else {
+      item.status = "in_progress";
+    }
+    this.emit("item:annotated", itemId, annotation);
+    return annotation;
+  }
+  /**
+   * Flag item for review
+   */
+  flagItem(itemId, reason) {
+    const item = this.items.get(itemId);
+    if (!item) {
+      throw new Error(`Item ${itemId} not found`);
+    }
+    item.status = "flagged";
+    item.updatedAt = Date.now();
+    this.emit("item:flagged", itemId, reason);
+  }
+  /**
+   * Skip item
+   */
+  skipItem(itemId, annotatorId) {
+    const item = this.items.get(itemId);
+    if (!item) {
+      throw new Error(`Item ${itemId} not found`);
+    }
+    const assigned = this.annotatorAssignments.get(annotatorId);
+    if (assigned) {
+      assigned.delete(itemId);
+    }
+    if (item.assignedTo) {
+      const idx = item.assignedTo.indexOf(annotatorId);
+      if (idx >= 0) {
+        item.assignedTo.splice(idx, 1);
+      }
+    }
+    item.updatedAt = Date.now();
+  }
+  /**
+   * Get batch assignment for annotator
+   */
+  getBatchAssignment(annotatorId, count) {
+    const itemIds = [];
+    for (let i = 0; i < count; i++) {
+      const item = this.getNextItem(annotatorId);
+      if (!item) break;
+      itemIds.push(item.id);
+    }
+    return {
+      annotatorId,
+      itemIds
+    };
+  }
+  /**
+   * Get queue statistics
+   */
+  getStats() {
+    let pending = 0;
+    let assigned = 0;
+    let completed = 0;
+    let flagged = 0;
+    let totalAnnotations = 0;
+    for (const item of this.items.values()) {
+      switch (item.status) {
+        case "pending":
+          pending++;
+          break;
+        case "assigned":
+        case "in_progress":
+          assigned++;
+          break;
+        case "completed":
+          completed++;
+          break;
+        case "flagged":
+          flagged++;
+          break;
+      }
+      totalAnnotations += item.annotations.length;
+    }
+    const avgAnnotationsPerItem = this.items.size > 0 ? totalAnnotations / this.items.size : 0;
+    return {
+      taskId: this.task.id,
+      totalItems: this.items.size,
+      pendingItems: pending,
+      assignedItems: assigned,
+      completedItems: completed,
+      flaggedItems: flagged,
+      averageAnnotationsPerItem: avgAnnotationsPerItem,
+      averageAgreement: 0
+      // Would need consensus calculation
+    };
+  }
+  /**
+   * Get item by ID
+   */
+  getItem(itemId) {
+    return this.items.get(itemId);
+  }
+  /**
+   * Get all items
+   */
+  getItems() {
+    return Array.from(this.items.values());
+  }
+  /**
+   * Get items by status
+   */
+  getItemsByStatus(status) {
+    return Array.from(this.items.values()).filter(
+      (item) => item.status === status
+    );
+  }
+  /**
+   * Get annotator's completed count
+   */
+  getAnnotatorCount(annotatorId) {
+    return this.annotatorCounts.get(annotatorId) ?? 0;
+  }
+};
+function createAnnotationQueue(config) {
+  return new AnnotationQueue(config);
+}
+
+// src/annotation/ConsensusManager.ts
+var ConsensusManager = class {
+  method;
+  weights;
+  expertAnnotatorId;
+  constructor(config) {
+    this.method = config.method;
+    this.weights = config.weights;
+    this.expertAnnotatorId = config.expertAnnotatorId;
+  }
+  /**
+   * Calculate consensus from annotations
+   */
+  calculateConsensus(annotations) {
+    if (annotations.length === 0) {
+      return {
+        value: {},
+        method: this.method,
+        agreement: 0,
+        confidence: 0,
+        contributingAnnotations: []
+      };
+    }
+    if (annotations.length === 1) {
+      return {
+        value: annotations[0].value,
+        method: this.method,
+        agreement: 1,
+        confidence: annotations[0].confidence ?? 0.5,
+        contributingAnnotations: [annotations[0].id]
+      };
+    }
+    switch (this.method) {
+      case "majority":
+        return this.majorityConsensus(annotations);
+      case "unanimous":
+        return this.unanimousConsensus(annotations);
+      case "weighted":
+        return this.weightedConsensus(annotations);
+      case "expert":
+        return this.expertConsensus(annotations);
+      default:
+        return this.majorityConsensus(annotations);
+    }
+  }
+  /**
+   * Majority vote consensus
+   */
+  majorityConsensus(annotations) {
+    const allFields = /* @__PURE__ */ new Set();
+    for (const ann of annotations) {
+      for (const key of Object.keys(ann.value)) {
+        allFields.add(key);
+      }
+    }
+    const consensusValue = {};
+    const disagreements = [];
+    let totalAgreement = 0;
+    for (const field of allFields) {
+      const values = annotations.map((ann) => ann.value[field]);
+      const { value, agreement, disagrees } = this.findMajority(
+        values,
+        annotations
+      );
+      consensusValue[field] = value;
+      totalAgreement += agreement;
+      if (disagrees) {
+        disagreements.push({
+          field,
+          values: disagrees,
+          resolved: true,
+          resolution: value
+        });
+      }
+    }
+    const avgAgreement = allFields.size > 0 ? totalAgreement / allFields.size : 1;
+    return {
+      value: consensusValue,
+      method: "majority",
+      agreement: avgAgreement,
+      confidence: avgAgreement,
+      contributingAnnotations: annotations.map((a) => a.id),
+      disagreements: disagreements.length > 0 ? disagreements : void 0
+    };
+  }
+  /**
+   * Find majority value
+   */
+  findMajority(values, annotations) {
+    const counts = /* @__PURE__ */ new Map();
+    for (let i = 0; i < values.length; i++) {
+      const key = JSON.stringify(values[i]);
+      if (!counts.has(key)) {
+        counts.set(key, { value: values[i], count: 0, annotatorIds: [] });
+      }
+      counts.get(key).count++;
+      counts.get(key).annotatorIds.push(annotations[i].annotatorId);
+    }
+    let maxCount = 0;
+    let majorityValue = null;
+    for (const { value, count } of counts.values()) {
+      if (count > maxCount) {
+        maxCount = count;
+        majorityValue = value;
+      }
+    }
+    const agreement = maxCount / values.length;
+    if (counts.size > 1) {
+      const disagrees = Array.from(counts.entries()).map(([, v]) => ({
+        value: v.value,
+        annotatorIds: v.annotatorIds,
+        count: v.count
+      }));
+      return { value: majorityValue, agreement, disagrees };
+    }
+    return { value: majorityValue, agreement };
+  }
+  /**
+   * Unanimous consensus
+   */
+  unanimousConsensus(annotations) {
+    const firstValue = annotations[0].value;
+    const allMatch = annotations.every(
+      (ann) => JSON.stringify(ann.value) === JSON.stringify(firstValue)
+    );
+    return {
+      value: allMatch ? firstValue : {},
+      method: "unanimous",
+      agreement: allMatch ? 1 : 0,
+      confidence: allMatch ? 1 : 0,
+      contributingAnnotations: allMatch ? annotations.map((a) => a.id) : []
+    };
+  }
+  /**
+   * Weighted consensus
+   */
+  weightedConsensus(annotations) {
+    if (!this.weights) {
+      return this.majorityConsensus(annotations);
+    }
+    const allFields = /* @__PURE__ */ new Set();
+    for (const ann of annotations) {
+      for (const key of Object.keys(ann.value)) {
+        allFields.add(key);
+      }
+    }
+    const consensusValue = {};
+    let totalWeightedAgreement = 0;
+    let totalWeight = 0;
+    for (const field of allFields) {
+      const values = /* @__PURE__ */ new Map();
+      for (const ann of annotations) {
+        const key = JSON.stringify(ann.value[field]);
+        const weight = this.weights[ann.annotatorId] ?? 1;
+        if (!values.has(key)) {
+          values.set(key, { value: ann.value[field], weight: 0 });
+        }
+        values.get(key).weight += weight;
+        totalWeight += weight;
+      }
+      let maxWeight = 0;
+      let bestValue = null;
+      for (const { value, weight } of values.values()) {
+        if (weight > maxWeight) {
+          maxWeight = weight;
+          bestValue = value;
+        }
+      }
+      consensusValue[field] = bestValue;
+      totalWeightedAgreement += maxWeight;
+    }
+    const avgAgreement = totalWeight > 0 ? totalWeightedAgreement / totalWeight : 1;
+    return {
+      value: consensusValue,
+      method: "weighted",
+      agreement: avgAgreement,
+      confidence: avgAgreement,
+      contributingAnnotations: annotations.map((a) => a.id)
+    };
+  }
+  /**
+   * Expert consensus
+   */
+  expertConsensus(annotations) {
+    if (!this.expertAnnotatorId) {
+      return this.majorityConsensus(annotations);
+    }
+    const expertAnnotation = annotations.find(
+      (a) => a.annotatorId === this.expertAnnotatorId
+    );
+    if (!expertAnnotation) {
+      return this.majorityConsensus(annotations);
+    }
+    return {
+      value: expertAnnotation.value,
+      method: "expert",
+      agreement: 1,
+      confidence: expertAnnotation.confidence ?? 1,
+      contributingAnnotations: [expertAnnotation.id]
+    };
+  }
+  /**
+   * Calculate inter-annotator agreement (Fleiss' kappa approximation)
+   */
+  calculateAgreement(annotations) {
+    if (annotations.length < 2) return 1;
+    const values = annotations.map((a) => JSON.stringify(a.value));
+    const counts = /* @__PURE__ */ new Map();
+    for (const v of values) {
+      counts.set(v, (counts.get(v) ?? 0) + 1);
+    }
+    const maxCount = Math.max(...counts.values());
+    return maxCount / values.length;
+  }
+};
+function createConsensusManager(config) {
+  return new ConsensusManager(config);
+}
+
+export {
+  AnnotationTask,
+  createAnnotationTask,
+  BinaryClassificationSchema,
+  QualityRatingSchema,
+  TextSpanSchema,
+  AnnotationQueue,
+  createAnnotationQueue,
+  ConsensusManager,
+  createConsensusManager
+};