agentdb 3.0.0-alpha.2 → 3.0.0-alpha.4

package/src/controllers/MMRDiversityRanker.ts

@@ -0,0 +1,187 @@
+/**
+ * MMR (Maximal Marginal Relevance) Diversity Ranking
+ *
+ * Implements MMR algorithm to select diverse results that balance
+ * relevance to query with diversity from already-selected results.
+ *
+ * Formula: MMR = argmax [λ × Sim(Di, Q) - (1-λ) × max Sim(Di, Dj)]
+ *                Di∈R\S              Dj∈S
+ *
+ * Where:
+ * - Di = candidate document
+ * - Q = query
+ * - S = already selected documents
+ * - λ = balance parameter (0 = max diversity, 1 = max relevance)
+ */
+
+export interface MMROptions {
+  lambda?: number;  // Balance between relevance and diversity (default: 0.5)
+  k?: number;       // Number of results to return (default: 10)
+  metric?: 'cosine' | 'euclidean' | 'dot';  // Similarity metric
+}
+
+export interface MMRCandidate {
+  id: number;
+  embedding: number[];
+  similarity: number;  // Similarity to query
+  [key: string]: any;  // Additional data
+}
+
+export class MMRDiversityRanker {
+  /**
+   * Select diverse results using MMR algorithm
+   *
+   * @param candidates - All candidate results with embeddings
+   * @param queryEmbedding - Query vector
+   * @param options - MMR configuration
+   * @returns Diverse subset of candidates
+   */
+  static selectDiverse(
+    candidates: MMRCandidate[],
+    queryEmbedding: number[],
+    options: MMROptions = {}
+  ): MMRCandidate[] {
+    const lambda = options.lambda ?? 0.5;
+    const k = options.k ?? 10;
+    const metric = options.metric ?? 'cosine';
+
+    if (candidates.length === 0) {
+      return [];
+    }
+
+    if (candidates.length <= k) {
+      return candidates;
+    }
+
+    // Calculate initial similarities to query
+    const candidatesWithSim = candidates.map(c => ({
+      ...c,
+      similarity: c.similarity ?? this.calculateSimilarity(
+        queryEmbedding,
+        c.embedding,
+        metric
+      ),
+    }));
+
+    const selected: MMRCandidate[] = [];
+    const remaining = [...candidatesWithSim];
+
+    // Select first item (highest relevance)
+    remaining.sort((a, b) => b.similarity - a.similarity);
+    selected.push(remaining.shift()!);
+
+    // Iteratively select items with highest MMR score
+    while (selected.length < k && remaining.length > 0) {
+      let maxMMR = -Infinity;
+      let maxIdx = 0;
+
+      for (let i = 0; i < remaining.length; i++) {
+        const candidate = remaining[i];
+
+        // Calculate max similarity to already-selected items
+        let maxSimToSelected = -Infinity;
+        for (const selectedItem of selected) {
+          const sim = this.calculateSimilarity(
+            candidate.embedding,
+            selectedItem.embedding,
+            metric
+          );
+          maxSimToSelected = Math.max(maxSimToSelected, sim);
+        }
+
+        // Calculate MMR score
+        const mmrScore = lambda * candidate.similarity - (1 - lambda) * maxSimToSelected;
+
+        if (mmrScore > maxMMR) {
+          maxMMR = mmrScore;
+          maxIdx = i;
+        }
+      }
+
+      // Add item with highest MMR score
+      selected.push(remaining.splice(maxIdx, 1)[0]);
+    }
+
+    return selected;
+  }
+
+  /**
+   * Calculate similarity between two vectors
+   */
+  private static calculateSimilarity(
+    vec1: number[],
+    vec2: number[],
+    metric: 'cosine' | 'euclidean' | 'dot'
+  ): number {
+    if (vec1.length !== vec2.length) {
+      throw new Error(`Vector dimension mismatch: ${vec1.length} vs ${vec2.length}`);
+    }
+
+    switch (metric) {
+      case 'cosine': {
+        let dot = 0, mag1 = 0, mag2 = 0;
+        for (let i = 0; i < vec1.length; i++) {
+          dot += vec1[i] * vec2[i];
+          mag1 += vec1[i] * vec1[i];
+          mag2 += vec2[i] * vec2[i];
+        }
+        return dot / (Math.sqrt(mag1) * Math.sqrt(mag2));
+      }
+
+      case 'euclidean': {
+        let sum = 0;
+        for (let i = 0; i < vec1.length; i++) {
+          const diff = vec1[i] - vec2[i];
+          sum += diff * diff;
+        }
+        return 1 / (1 + Math.sqrt(sum));  // Normalized to 0-1
+      }
+
+      case 'dot': {
+        let dot = 0;
+        for (let i = 0; i < vec1.length; i++) {
+          dot += vec1[i] * vec2[i];
+        }
+        return dot;
+      }
+
+      default:
+        throw new Error(`Unknown metric: ${metric}`);
+    }
+  }
+
+  /**
+   * Calculate diversity score for a set of results
+   *
+   * @param results - Results to analyze
+   * @param metric - Similarity metric
+   * @returns Average pairwise distance (higher = more diverse)
+   */
+  static calculateDiversityScore(
+    results: MMRCandidate[],
+    metric: 'cosine' | 'euclidean' | 'dot' = 'cosine'
+  ): number {
+    if (results.length < 2) {
+      return 1.0;  // Single result is maximally diverse
+    }
+
+    let totalDistance = 0;
+    let comparisons = 0;
+
+    for (let i = 0; i < results.length; i++) {
+      for (let j = i + 1; j < results.length; j++) {
+        const similarity = this.calculateSimilarity(
+          results[i].embedding,
+          results[j].embedding,
+          metric
+        );
+        // Convert similarity to distance
+        const distance = metric === 'cosine' ? 1 - similarity : similarity;
+        totalDistance += distance;
+        comparisons++;
+      }
+    }
+
+    return comparisons > 0 ? totalDistance / comparisons : 0;
+  }
+}

package/src/controllers/MetadataFilter.ts

@@ -0,0 +1,280 @@
+/**
+ * Advanced Metadata Filtering
+ *
+ * Implements MongoDB-style query operators for filtering
+ * episodes and patterns based on metadata fields.
+ *
+ * Supported operators:
+ * - $eq: Equal to
+ * - $ne: Not equal to
+ * - $gt: Greater than
+ * - $gte: Greater than or equal to
+ * - $lt: Less than
+ * - $lte: Less than or equal to
+ * - $in: Value is in array
+ * - $nin: Value is not in array
+ * - $contains: String/array contains value
+ * - $exists: Field exists
+ */
+
+export type FilterOperator = '$eq' | '$ne' | '$gt' | '$gte' | '$lt' | '$lte' | '$in' | '$nin' | '$contains' | '$exists';
+
+export type FilterValue = string | number | boolean | string[] | number[] | { [op in FilterOperator]?: any };
+
+export interface MetadataFilters {
+  [field: string]: FilterValue;
+}
+
+export interface FilterableItem {
+  metadata?: any;
+  [key: string]: any;
+}
+
+export class MetadataFilter {
+  /**
+   * Apply filters to a collection of items
+   *
+   * @param items - Items to filter
+   * @param filters - MongoDB-style filter object
+   * @returns Filtered items
+   */
+  static apply<T extends FilterableItem>(items: T[], filters: MetadataFilters): T[] {
+    if (!filters || Object.keys(filters).length === 0) {
+      return items;
+    }
+
+    return items.filter(item => this.matchesFilters(item, filters));
+  }
+
+  /**
+   * Check if an item matches all filters
+   */
+  private static matchesFilters(item: FilterableItem, filters: MetadataFilters): boolean {
+    for (const [field, filter] of Object.entries(filters)) {
+      if (!this.matchesFilter(item, field, filter)) {
+        return false;
+      }
+    }
+    return true;
+  }
+
+  /**
+   * Check if an item matches a single filter
+   */
+  private static matchesFilter(item: FilterableItem, field: string, filter: FilterValue): boolean {
+    // Get field value (supports nested paths like "metadata.year")
+    const value = this.getFieldValue(item, field);
+
+    // Handle simple equality
+    if (typeof filter !== 'object' || Array.isArray(filter)) {
+      return value === filter;
+    }
+
+    // Handle operator-based filters
+    const operators = filter as { [op in FilterOperator]?: any };
+
+    for (const [operator, operand] of Object.entries(operators)) {
+      switch (operator as FilterOperator) {
+        case '$eq':
+          if (value !== operand) return false;
+          break;
+
+        case '$ne':
+          if (value === operand) return false;
+          break;
+
+        case '$gt':
+          if (!(value > operand)) return false;
+          break;
+
+        case '$gte':
+          if (!(value >= operand)) return false;
+          break;
+
+        case '$lt':
+          if (!(value < operand)) return false;
+          break;
+
+        case '$lte':
+          if (!(value <= operand)) return false;
+          break;
+
+        case '$in':
+          if (!Array.isArray(operand) || !operand.includes(value)) return false;
+          break;
+
+        case '$nin':
+          if (!Array.isArray(operand) || operand.includes(value)) return false;
+          break;
+
+        case '$contains':
+          if (typeof value === 'string') {
+            if (!value.includes(operand)) return false;
+          } else if (Array.isArray(value)) {
+            if (!value.includes(operand)) return false;
+          } else {
+            return false;
+          }
+          break;
+
+        case '$exists':
+          const exists = value !== undefined && value !== null;
+          if (exists !== operand) return false;
+          break;
+
+        default:
+          console.warn(`Unknown operator: ${operator}`);
+          return false;
+      }
+    }
+
+    return true;
+  }
+
+  /**
+   * Get field value from item (supports nested paths)
+   */
+  private static getFieldValue(item: FilterableItem, field: string): any {
+    const parts = field.split('.');
+    let value: any = item;
+
+    for (const part of parts) {
+      if (value === null || value === undefined) {
+        return undefined;
+      }
+
+      // Parse metadata JSON if needed
+      if (part === 'metadata' && typeof value.metadata === 'string') {
+        try {
+          value.metadata = JSON.parse(value.metadata);
+        } catch (e) {
+          return undefined;
+        }
+      }
+
+      value = value[part];
+    }
+
+    return value;
+  }
+
+  /**
+   * Build SQL WHERE clause from filters (for database queries)
+   *
+   * @param filters - Metadata filters
+   * @param tableName - Table name for column references
+   * @returns SQL WHERE clause and parameters
+   */
+  static toSQL(filters: MetadataFilters, tableName: string = ''): { where: string; params: any[] } {
+    const conditions: string[] = [];
+    const params: any[] = [];
+    const prefix = tableName ? `${tableName}.` : '';
+
+    for (const [field, filter] of Object.entries(filters)) {
+      // For metadata fields, use JSON extraction
+      const isMetadata = field.startsWith('metadata.');
+      const columnRef = isMetadata
+        ? `json_extract(${prefix}metadata, '$.${field.slice(9)}')`
+        : `${prefix}${field}`;
+
+      if (typeof filter !== 'object' || Array.isArray(filter)) {
+        // Simple equality
+        conditions.push(`${columnRef} = ?`);
+        params.push(filter);
+      } else {
+        // Operator-based filters
+        const operators = filter as { [op in FilterOperator]?: any };
+
+        for (const [operator, operand] of Object.entries(operators)) {
+          switch (operator as FilterOperator) {
+            case '$eq':
+              conditions.push(`${columnRef} = ?`);
+              params.push(operand);
+              break;
+
+            case '$ne':
+              conditions.push(`${columnRef} != ?`);
+              params.push(operand);
+              break;
+
+            case '$gt':
+              conditions.push(`${columnRef} > ?`);
+              params.push(operand);
+              break;
+
+            case '$gte':
+              conditions.push(`${columnRef} >= ?`);
+              params.push(operand);
+              break;
+
+            case '$lt':
+              conditions.push(`${columnRef} < ?`);
+              params.push(operand);
+              break;
+
+            case '$lte':
+              conditions.push(`${columnRef} <= ?`);
+              params.push(operand);
+              break;
+
+            case '$in':
+              if (Array.isArray(operand)) {
+                const placeholders = operand.map(() => '?').join(', ');
+                conditions.push(`${columnRef} IN (${placeholders})`);
+                params.push(...operand);
+              }
+              break;
+
+            case '$nin':
+              if (Array.isArray(operand)) {
+                const placeholders = operand.map(() => '?').join(', ');
+                conditions.push(`${columnRef} NOT IN (${placeholders})`);
+                params.push(...operand);
+              }
+              break;
+
+            case '$contains':
+              conditions.push(`${columnRef} LIKE ?`);
+              params.push(`%${operand}%`);
+              break;
+
+            case '$exists':
+              if (operand) {
+                conditions.push(`${columnRef} IS NOT NULL`);
+              } else {
+                conditions.push(`${columnRef} IS NULL`);
+              }
+              break;
+          }
+        }
+      }
+    }
+
+    const where = conditions.length > 0 ? conditions.join(' AND ') : '1=1';
+    return { where, params };
+  }
+
+  /**
+   * Validate filter object
+   */
+  static validate(filters: MetadataFilters): { valid: boolean; errors: string[] } {
+    const errors: string[] = [];
+
+    for (const [field, filter] of Object.entries(filters)) {
+      if (!field || field.trim() === '') {
+        errors.push('Filter field name cannot be empty');
+      }
+
+      if (typeof filter === 'object' && !Array.isArray(filter)) {
+        const operators = Object.keys(filter);
+        for (const op of operators) {
+          if (!op.startsWith('$')) {
+            errors.push(`Invalid operator: ${op} (must start with $)`);
+          }
+        }
+      }
+    }
+
+    return { valid: errors.length === 0, errors };
+  }
+}