@od-oneapp/ai-platform 0.1.0

package/src/hitl/active-learning/index.ts

@@ -0,0 +1,332 @@
+/**
+ * Active Learning Selection
+ *
+ * Intelligently selects samples for human annotation to maximize
+ * model improvement with minimal labeling effort.
+ *
+ * @module @od-oneapp/ai-platform/hitl/active-learning
+ *
+ * @example
+ * ```typescript
+ * import { createActiveLearner } from '@od-oneapp/ai-platform/hitl/active-learning';
+ *
+ * const learner = createActiveLearner({
+ *   strategy: 'uncertainty',
+ *   batchSize: 10
+ * });
+ *
+ * // Get most informative samples to label
+ * const samples = await learner.selectSamples(unlabeledData);
+ * ```
+ */
+
+// -----------------------------------------------------------------------------
+// Types
+// -----------------------------------------------------------------------------
+
+/** Active learning strategy */
+export type SelectionStrategy =
+  | 'uncertainty' // Select samples with highest model uncertainty
+  | 'diversity' // Select diverse samples to cover feature space
+  | 'query-by-committee' // Select samples where multiple models disagree
+  | 'expected-improvement' // Select samples with highest expected improvement
+  | 'random'; // Random baseline
+
+/** Sample with prediction metadata */
+export interface Sample<T = unknown> {
+  readonly id: string;
+  readonly data: T;
+  readonly prediction?: {
+    label: string;
+    confidence: number;
+    probabilities?: Record<string, number>;
+  };
+  readonly embedding?: number[];
+  readonly metadata?: Record<string, unknown>;
+}
+
+/** Selection result */
+export interface SelectionResult<T = unknown> {
+  readonly selected: Sample<T>[];
+  readonly scores: Map<string, number>;
+  readonly strategy: SelectionStrategy;
+  readonly metadata?: Record<string, unknown>;
+}
+
+/** Active learner configuration */
+export interface ActiveLearnerConfig {
+  readonly strategy: SelectionStrategy;
+  readonly batchSize?: number;
+  readonly uncertaintyThreshold?: number;
+  readonly diversityWeight?: number;
+  readonly randomSeed?: number;
+}
+
+/** Active learner interface */
+export interface ActiveLearner<T = unknown> {
+  selectSamples(samples: Sample<T>[], count?: number): Promise<SelectionResult<T>>;
+  updateStrategy(strategy: SelectionStrategy): void;
+  getConfig(): ActiveLearnerConfig;
+}
+
+// -----------------------------------------------------------------------------
+// Selection Strategies
+// -----------------------------------------------------------------------------
+
+/**
+ * Uncertainty-based selection: picks samples where model is least confident.
+ */
+function uncertaintyScore(sample: Sample): number {
+  if (!sample.prediction) return 0.5; // No prediction = medium uncertainty
+
+  const { confidence } = sample.prediction;
+  // Higher score = more uncertain = better candidate
+  return 1 - confidence;
+}
+
+/**
+ * Entropy-based uncertainty for multi-class predictions.
+ */
+function entropyScore(sample: Sample): number {
+  const probs = sample.prediction?.probabilities;
+  if (!probs) return 0.5;
+
+  const values = Object.values(probs);
+  if (values.length === 0) return 0.5;
+
+  // Calculate entropy: -sum(p * log(p))
+  let entropy = 0;
+  for (const p of values) {
+    if (p > 0) {
+      entropy -= p * Math.log2(p);
+    }
+  }
+
+  // Normalize by max entropy (uniform distribution)
+  const maxEntropy = Math.log2(values.length);
+  return maxEntropy > 0 ? entropy / maxEntropy : 0;
+}
+
+/**
+ * Diversity-based selection using embedding distance.
+ */
+function selectDiverse<T>(
+  samples: Sample<T>[],
+  count: number,
+  existingSelected: Sample<T>[] = [],
+): Sample<T>[] {
+  const selected: Sample<T>[] = [...existingSelected];
+  const remaining = samples.filter(s => !selected.includes(s));
+
+  while (selected.length < count && remaining.length > 0) {
+    let bestSample: Sample<T> | null = null;
+    let bestMinDist = -1;
+
+    for (const candidate of remaining) {
+      if (!candidate.embedding) continue;
+
+      // Find minimum distance to any selected sample
+      let minDist = Infinity;
+      for (const sel of selected) {
+        if (!sel.embedding) continue;
+        const dist = euclideanDistance(candidate.embedding, sel.embedding);
+        minDist = Math.min(minDist, dist);
+      }
+
+      // Select sample with maximum minimum distance (most diverse)
+      if (minDist > bestMinDist) {
+        bestMinDist = minDist;
+        bestSample = candidate;
+      }
+    }
+
+    if (bestSample) {
+      selected.push(bestSample);
+      remaining.splice(remaining.indexOf(bestSample), 1);
+    } else {
+      // No more samples with embeddings, pick randomly
+      const randomIdx = Math.floor(Math.random() * remaining.length);
+      const randomSample = remaining[randomIdx];
+      if (randomSample) {
+        selected.push(randomSample);
+        remaining.splice(randomIdx, 1);
+      }
+    }
+  }
+
+  return selected.slice(existingSelected.length);
+}
+
+function euclideanDistance(a: number[], b: number[]): number {
+  if (a.length !== b.length) return Infinity;
+  let sum = 0;
+  for (let i = 0; i < a.length; i++) {
+    const diff = (a[i] ?? 0) - (b[i] ?? 0);
+    sum += diff * diff;
+  }
+  return Math.sqrt(sum);
+}
+
+// -----------------------------------------------------------------------------
+// Active Learner Implementation
+// -----------------------------------------------------------------------------
+
+/**
+ * Creates an active learner for intelligent sample selection.
+ *
+ * @example
+ * ```typescript
+ * const learner = createActiveLearner({
+ *   strategy: 'uncertainty',
+ *   batchSize: 10,
+ *   uncertaintyThreshold: 0.3
+ * });
+ *
+ * const samples = [
+ *   { id: '1', data: 'text1', prediction: { label: 'A', confidence: 0.9 } },
+ *   { id: '2', data: 'text2', prediction: { label: 'B', confidence: 0.5 } },
+ *   { id: '3', data: 'text3', prediction: { label: 'A', confidence: 0.3 } },
+ * ];
+ *
+ * const result = await learner.selectSamples(samples, 2);
+ * // Returns samples with confidence 0.3 and 0.5 (most uncertain)
+ * ```
+ */
+export function createActiveLearner<T = unknown>(config: ActiveLearnerConfig): ActiveLearner<T> {
+  let currentStrategy = config.strategy;
+  const batchSize = config.batchSize ?? 10;
+  const uncertaintyThreshold = config.uncertaintyThreshold ?? 0.5;
+  const diversityWeight = config.diversityWeight ?? 0.3;
+
+  async function selectSamples(samples: Sample<T>[], count?: number): Promise<SelectionResult<T>> {
+    const targetCount = count ?? batchSize;
+    const scores = new Map<string, number>();
+
+    // Score all samples based on strategy
+    for (const sample of samples) {
+      let score: number;
+
+      switch (currentStrategy) {
+        case 'uncertainty':
+          score = uncertaintyScore(sample);
+          break;
+        case 'diversity':
+          // For diversity, we use a different selection approach
+          score = Math.random(); // Initial random scores, diversity handled separately
+          break;
+        case 'query-by-committee':
+          // Use entropy as proxy for committee disagreement
+          score = entropyScore(sample);
+          break;
+        case 'expected-improvement':
+          // Combine uncertainty with potential information gain
+          score = uncertaintyScore(sample) * 0.7 + entropyScore(sample) * 0.3;
+          break;
+        case 'random':
+        default:
+          score = Math.random();
+          break;
+      }
+
+      scores.set(sample.id, score);
+    }
+
+    let selected: Sample<T>[];
+
+    if (currentStrategy === 'diversity') {
+      // Use diversity-based selection
+      selected = selectDiverse(samples, targetCount);
+    } else if (currentStrategy === 'uncertainty' && diversityWeight > 0) {
+      // Hybrid: uncertainty + diversity
+      // First get top candidates by uncertainty
+      const topK = Math.min(targetCount * 3, samples.length);
+      const sortedByScore = [...samples].sort(
+        (a, b) => (scores.get(b.id) ?? 0) - (scores.get(a.id) ?? 0),
+      );
+      const topCandidates = sortedByScore.slice(0, topK);
+
+      // Then select diverse subset
+      selected = selectDiverse(topCandidates, targetCount);
+    } else {
+      // Pure score-based selection
+      const sortedByScore = [...samples].sort(
+        (a, b) => (scores.get(b.id) ?? 0) - (scores.get(a.id) ?? 0),
+      );
+      selected = sortedByScore.slice(0, targetCount);
+    }
+
+    // Filter by uncertainty threshold if using uncertainty strategy
+    if (currentStrategy === 'uncertainty' && uncertaintyThreshold > 0) {
+      selected = selected.filter(s => (scores.get(s.id) ?? 0) >= uncertaintyThreshold);
+    }
+
+    return {
+      selected,
+      scores,
+      strategy: currentStrategy,
+      metadata: {
+        totalSamples: samples.length,
+        requestedCount: targetCount,
+        actualCount: selected.length,
+        avgScore:
+          selected.length > 0
+            ? selected.reduce((sum, s) => sum + (scores.get(s.id) ?? 0), 0) / selected.length
+            : 0,
+      },
+    };
+  }
+
+  function updateStrategy(strategy: SelectionStrategy): void {
+    currentStrategy = strategy;
+  }
+
+  function getConfig(): ActiveLearnerConfig {
+    return {
+      strategy: currentStrategy,
+      batchSize,
+      uncertaintyThreshold,
+      diversityWeight,
+    };
+  }
+
+  return {
+    selectSamples,
+    updateStrategy,
+    getConfig,
+  };
+}
+
+// -----------------------------------------------------------------------------
+// Utility Functions
+// -----------------------------------------------------------------------------
+
+/**
+ * Creates a sample from raw data with optional prediction.
+ */
+export function createSample<T>(
+  id: string,
+  data: T,
+  options?: {
+    prediction?: Sample['prediction'];
+    embedding?: number[];
+    metadata?: Record<string, unknown>;
+  },
+): Sample<T> {
+  return {
+    id,
+    data,
+    prediction: options?.prediction,
+    embedding: options?.embedding,
+    metadata: options?.metadata,
+  };
+}
+
+/**
+ * Batch creates samples from an array of data.
+ */
+export function createSamples<T>(items: T[], idPrefix: string = 'sample'): Sample<T>[] {
+  return items.map((data, i) => ({
+    id: `${idPrefix}-${i}`,
+    data,
+  }));
+}

package/src/hitl/annotation/index.ts

@@ -0,0 +1,362 @@
+/**
+ * Human Annotation Tools
+ *
+ * Enables human annotators to label, tag, and correct AI outputs
+ * for training data generation and quality assurance.
+ *
+ * @module @od-oneapp/ai-platform/hitl/annotation
+ *
+ * @example
+ * ```typescript
+ * import { createAnnotationStore } from '@od-oneapp/ai-platform/hitl/annotation';
+ *
+ * const store = createAnnotationStore();
+ *
+ * // Create annotation task
+ * const task = await store.createTask({
+ *   responseId: 'resp-123',
+ *   type: 'quality-check',
+ *   priority: 'high'
+ * });
+ *
+ * // Submit annotation
+ * await store.submitAnnotation(task.id, {
+ *   labels: ['accurate', 'helpful'],
+ *   corrections: [{ original: 'bad text', corrected: 'good text' }]
+ * });
+ * ```
+ */
+
+// -----------------------------------------------------------------------------
+// Types
+// -----------------------------------------------------------------------------
+
+/** Annotation task priority */
+export type TaskPriority = 'low' | 'medium' | 'high' | 'urgent';
+
+/** Annotation task status */
+export type TaskStatus = 'pending' | 'assigned' | 'in_progress' | 'completed' | 'skipped';
+
+/** Annotation task type */
+export type TaskType =
+  | 'quality-check'
+  | 'factual-accuracy'
+  | 'safety-review'
+  | 'preference-ranking'
+  | 'correction'
+  | 'labeling'
+  | 'custom';
+
+/** Annotation task */
+export interface AnnotationTask {
+  readonly id: string;
+  readonly responseId: string;
+  readonly type: TaskType;
+  readonly priority: TaskPriority;
+  readonly status: TaskStatus;
+  readonly assigneeId?: string;
+  readonly instructions?: string;
+  readonly metadata?: Record<string, unknown>;
+  readonly createdAt: Date;
+  readonly updatedAt: Date;
+  readonly completedAt?: Date;
+}
+
+/** Task input */
+export type TaskInput = Omit<
+  AnnotationTask,
+  'id' | 'status' | 'createdAt' | 'updatedAt' | 'completedAt'
+>;
+
+/** Text correction */
+export interface TextCorrection {
+  readonly original: string;
+  readonly corrected: string;
+  readonly reason?: string;
+  readonly startOffset?: number;
+  readonly endOffset?: number;
+}
+
+/** Annotation data */
+export interface Annotation {
+  readonly id: string;
+  readonly taskId: string;
+  readonly annotatorId: string;
+  readonly labels?: string[];
+  readonly score?: number;
+  readonly corrections?: TextCorrection[];
+  readonly notes?: string;
+  readonly timeSpentMs?: number;
+  readonly metadata?: Record<string, unknown>;
+  readonly createdAt: Date;
+}
+
+/** Annotation input */
+export type AnnotationInput = Omit<Annotation, 'id' | 'taskId' | 'createdAt'>;
+
+/** Task query options */
+export interface TaskQuery {
+  readonly status?: TaskStatus | TaskStatus[];
+  readonly type?: TaskType;
+  readonly priority?: TaskPriority;
+  readonly assigneeId?: string;
+  readonly limit?: number;
+  readonly offset?: number;
+}
+
+/** Annotation store interface */
+export interface AnnotationStore {
+  // Tasks
+  createTask(input: TaskInput): Promise<AnnotationTask>;
+  getTask(id: string): Promise<AnnotationTask | null>;
+  queryTasks(query: TaskQuery): Promise<AnnotationTask[]>;
+  assignTask(taskId: string, annotatorId: string): Promise<AnnotationTask>;
+  updateTaskStatus(taskId: string, status: TaskStatus): Promise<AnnotationTask>;
+
+  // Annotations
+  submitAnnotation(taskId: string, input: AnnotationInput): Promise<Annotation>;
+  getAnnotation(id: string): Promise<Annotation | null>;
+  getAnnotationsForTask(taskId: string): Promise<Annotation[]>;
+
+  // Stats
+  getStats(): Promise<AnnotationStats>;
+}
+
+/** Annotation statistics */
+export interface AnnotationStats {
+  readonly totalTasks: number;
+  readonly pendingTasks: number;
+  readonly completedTasks: number;
+  readonly avgCompletionTimeMs: number;
+  readonly annotatorStats: Map<string, AnnotatorStats>;
+}
+
+/** Per-annotator statistics */
+export interface AnnotatorStats {
+  readonly annotatorId: string;
+  readonly tasksCompleted: number;
+  readonly avgTimeMs: number;
+  readonly avgScore?: number;
+}
+
+// -----------------------------------------------------------------------------
+// In-Memory Implementation
+// -----------------------------------------------------------------------------
+
+/**
+ * Creates an in-memory annotation store.
+ *
+ * @example
+ * ```typescript
+ * const store = createAnnotationStore();
+ *
+ * // Create a task
+ * const task = await store.createTask({
+ *   responseId: 'resp-123',
+ *   type: 'quality-check',
+ *   priority: 'medium'
+ * });
+ *
+ * // Assign to annotator
+ * await store.assignTask(task.id, 'annotator-1');
+ *
+ * // Submit annotation
+ * await store.submitAnnotation(task.id, {
+ *   annotatorId: 'annotator-1',
+ *   labels: ['accurate', 'helpful'],
+ *   score: 4
+ * });
+ * ```
+ */
+export function createAnnotationStore(): AnnotationStore {
+  const tasks = new Map<string, AnnotationTask>();
+  const annotations = new Map<string, Annotation>();
+  const taskAnnotations = new Map<string, string[]>(); // taskId -> annotationIds
+
+  async function createTask(input: TaskInput): Promise<AnnotationTask> {
+    const now = new Date();
+    const task: AnnotationTask = {
+      ...input,
+      id: generateTaskId(),
+      status: 'pending',
+      createdAt: now,
+      updatedAt: now,
+    };
+    tasks.set(task.id, task);
+    taskAnnotations.set(task.id, []);
+    return task;
+  }
+
+  async function getTask(id: string): Promise<AnnotationTask | null> {
+    return tasks.get(id) ?? null;
+  }
+
+  async function queryTasks(query: TaskQuery): Promise<AnnotationTask[]> {
+    let results = Array.from(tasks.values());
+
+    if (query.status) {
+      const statuses = Array.isArray(query.status) ? query.status : [query.status];
+      results = results.filter(t => statuses.includes(t.status));
+    }
+    if (query.type) {
+      results = results.filter(t => t.type === query.type);
+    }
+    if (query.priority) {
+      results = results.filter(t => t.priority === query.priority);
+    }
+    if (query.assigneeId) {
+      results = results.filter(t => t.assigneeId === query.assigneeId);
+    }
+
+    // Sort by priority (urgent first) then by creation date
+    const priorityOrder: Record<TaskPriority, number> = {
+      urgent: 0,
+      high: 1,
+      medium: 2,
+      low: 3,
+    };
+    results.sort((a, b) => {
+      const pDiff = priorityOrder[a.priority] - priorityOrder[b.priority];
+      if (pDiff !== 0) return pDiff;
+      return a.createdAt.getTime() - b.createdAt.getTime();
+    });
+
+    const offset = query.offset ?? 0;
+    const limit = query.limit ?? 100;
+    return results.slice(offset, offset + limit);
+  }
+
+  async function assignTask(taskId: string, annotatorId: string): Promise<AnnotationTask> {
+    const task = tasks.get(taskId);
+    if (!task) throw new Error(`Task not found: ${taskId}`);
+
+    const updated: AnnotationTask = {
+      ...task,
+      assigneeId: annotatorId,
+      status: 'assigned',
+      updatedAt: new Date(),
+    };
+    tasks.set(taskId, updated);
+    return updated;
+  }
+
+  async function updateTaskStatus(taskId: string, status: TaskStatus): Promise<AnnotationTask> {
+    const task = tasks.get(taskId);
+    if (!task) throw new Error(`Task not found: ${taskId}`);
+
+    const updated: AnnotationTask = {
+      ...task,
+      status,
+      updatedAt: new Date(),
+      completedAt: status === 'completed' ? new Date() : task.completedAt,
+    };
+    tasks.set(taskId, updated);
+    return updated;
+  }
+
+  async function submitAnnotation(taskId: string, input: AnnotationInput): Promise<Annotation> {
+    const task = tasks.get(taskId);
+    if (!task) throw new Error(`Task not found: ${taskId}`);
+
+    const annotation: Annotation = {
+      ...input,
+      id: generateAnnotationId(),
+      taskId,
+      createdAt: new Date(),
+    };
+
+    annotations.set(annotation.id, annotation);
+
+    const taskAnns = taskAnnotations.get(taskId) ?? [];
+    taskAnns.push(annotation.id);
+    taskAnnotations.set(taskId, taskAnns);
+
+    // Auto-complete task
+    await updateTaskStatus(taskId, 'completed');
+
+    return annotation;
+  }
+
+  async function getAnnotation(id: string): Promise<Annotation | null> {
+    return annotations.get(id) ?? null;
+  }
+
+  async function getAnnotationsForTask(taskId: string): Promise<Annotation[]> {
+    const ids = taskAnnotations.get(taskId) ?? [];
+    return ids.map(id => annotations.get(id)).filter((a): a is Annotation => a !== undefined);
+  }
+
+  async function getStats(): Promise<AnnotationStats> {
+    const allTasks = Array.from(tasks.values());
+    const completedTasks = allTasks.filter(t => t.status === 'completed');
+    const pendingTasks = allTasks.filter(t => t.status === 'pending');
+
+    // Calculate avg completion time
+    let totalCompletionTime = 0;
+    for (const task of completedTasks) {
+      if (task.completedAt) {
+        totalCompletionTime += task.completedAt.getTime() - task.createdAt.getTime();
+      }
+    }
+    const avgCompletionTimeMs =
+      completedTasks.length > 0 ? totalCompletionTime / completedTasks.length : 0;
+
+    // Calculate annotator stats
+    const annotatorStats = new Map<string, AnnotatorStats>();
+    const allAnnotations = Array.from(annotations.values());
+
+    const annotatorData = new Map<string, { times: number[]; scores: number[] }>();
+
+    for (const ann of allAnnotations) {
+      const data = annotatorData.get(ann.annotatorId) ?? { times: [], scores: [] };
+      if (ann.timeSpentMs) data.times.push(ann.timeSpentMs);
+      if (ann.score !== undefined) data.scores.push(ann.score);
+      annotatorData.set(ann.annotatorId, data);
+    }
+
+    for (const [annotatorId, data] of annotatorData) {
+      annotatorStats.set(annotatorId, {
+        annotatorId,
+        tasksCompleted: data.times.length || data.scores.length,
+        avgTimeMs:
+          data.times.length > 0 ? data.times.reduce((a, b) => a + b, 0) / data.times.length : 0,
+        avgScore:
+          data.scores.length > 0
+            ? data.scores.reduce((a, b) => a + b, 0) / data.scores.length
+            : undefined,
+      });
+    }
+
+    return {
+      totalTasks: allTasks.length,
+      pendingTasks: pendingTasks.length,
+      completedTasks: completedTasks.length,
+      avgCompletionTimeMs,
+      annotatorStats,
+    };
+  }
+
+  return {
+    createTask,
+    getTask,
+    queryTasks,
+    assignTask,
+    updateTaskStatus,
+    submitAnnotation,
+    getAnnotation,
+    getAnnotationsForTask,
+    getStats,
+  };
+}
+
+// -----------------------------------------------------------------------------
+// Utilities
+// -----------------------------------------------------------------------------
+
+function generateTaskId(): string {
+  return `task-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
+}
+
+function generateAnnotationId(): string {
+  return `ann-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
+}