@yamo/memory-mesh 2.0.1

package/lib/search/filter.js

@@ -0,0 +1,275 @@
+/**
+ * FilterBuilder - Fluent API for building LanceDB filter expressions
+ * Provides type-safe filter construction for metadata queries
+ */
+
+class FilterBuilder {
+  /**
+   * Create a new FilterBuilder
+   */
+  constructor() {
+    this.filters = [];
+    this.operator = 'AND';
+  }
+
+  /**
+   * Add equality filter
+   * @param {string} field - Field name
+   * @param {*} value - Value to compare
+   * @returns {FilterBuilder} this for chaining
+   */
+  equals(field, value) {
+    this.filters.push(`${field} = ${this._quote(value)}`);
+    return this;
+  }
+
+  /**
+   * Add inequality filter
+   * @param {string} field - Field name
+   * @param {*} value - Value to compare
+   * @returns {FilterBuilder} this for chaining
+   */
+  notEquals(field, value) {
+    this.filters.push(`${field} != ${this._quote(value)}`);
+    return this;
+  }
+
+  /**
+   * Add greater than filter
+   * @param {string} field - Field name
+   * @param {number} value - Value to compare
+   * @returns {FilterBuilder} this for chaining
+   */
+  gt(field, value) {
+    this.filters.push(`${field} > ${value}`);
+    return this;
+  }
+
+  /**
+   * Add greater than or equal filter
+   * @param {string} field - Field name
+   * @param {number} value - Value to compare
+   * @returns {FilterBuilder} this for chaining
+   */
+  gte(field, value) {
+    this.filters.push(`${field} >= ${value}`);
+    return this;
+  }
+
+  /**
+   * Add less than filter
+   * @param {string} field - Field name
+   * @param {number} value - Value to compare
+   * @returns {FilterBuilder} this for chaining
+   */
+  lt(field, value) {
+    this.filters.push(`${field} < ${value}`);
+    return this;
+  }
+
+  /**
+   * Add less than or equal filter
+   * @param {string} field - Field name
+   * @param {number} value - Value to compare
+   * @returns {FilterBuilder} this for chaining
+   */
+  lte(field, value) {
+    this.filters.push(`${field} <= ${value}`);
+    return this;
+  }
+
+  /**
+   * Add contains filter (LIKE)
+   * @param {string} field - Field name
+   * @param {string} value - Value to search for
+   * @returns {FilterBuilder} this for chaining
+   */
+  contains(field, value) {
+    this.filters.push(`${field} LIKE '%${this._escapeLike(value)}%'`);
+    return this;
+  }
+
+  /**
+   * Add starts with filter
+   * @param {string} field - Field name
+   * @param {string} value - Value to match
+   * @returns {FilterBuilder} this for chaining
+   */
+  startsWith(field, value) {
+    this.filters.push(`${field} LIKE '${this._escapeLike(value)}%'`);
+    return this;
+  }
+
+  /**
+   * Add ends with filter
+   * @param {string} field - Field name
+   * @param {string} value - Value to match
+   * @returns {FilterBuilder} this for chaining
+   */
+  endsWith(field, value) {
+    this.filters.push(`${field} LIKE '%${this._escapeLike(value)}'`);
+    return this;
+  }
+
+  /**
+   * Add IN array filter
+   * @param {string} field - Field name
+   * @param {Array} values - Array of values
+   * @returns {FilterBuilder} this for chaining
+   */
+  in(field, values) {
+    if (!Array.isArray(values) || values.length === 0) {
+      throw new Error('IN filter requires non-empty array');
+    }
+    const quoted = values.map(v => this._quote(v)).join(', ');
+    this.filters.push(`${field} IN [${quoted}]`);
+    return this;
+  }
+
+  /**
+   * Add NOT IN array filter
+   * @param {string} field - Field name
+   * @param {Array} values - Array of values
+   * @returns {FilterBuilder} this for chaining
+   */
+  notIn(field, values) {
+    if (!Array.isArray(values) || values.length === 0) {
+      throw new Error('NOT IN filter requires non-empty array');
+    }
+    const quoted = values.map(v => this._quote(v)).join(', ');
+    this.filters.push(`${field} NOT IN [${quoted}]`);
+    return this;
+  }
+
+  /**
+   * Add range filter (inclusive)
+   * @param {string} field - Field name
+   * @param {number} min - Minimum value
+   * @param {number} max - Maximum value
+   * @returns {FilterBuilder} this for chaining
+   */
+  range(field, min, max) {
+    this.filters.push(`${field} >= ${min} AND ${field} <= ${max}`);
+    return this;
+  }
+
+  /**
+   * Add date range filter
+   * @param {string} field - Field name
+   * @param {string|Date} startDate - Start date
+   * @param {string|Date} endDate - End date
+   * @returns {FilterBuilder} this for chaining
+   */
+  dateRange(field, startDate, endDate) {
+    const start = startDate instanceof Date ? startDate.toISOString() : startDate;
+    const end = endDate instanceof Date ? endDate.toISOString() : endDate;
+    this.filters.push(`${field} >= '${start}' AND ${field} <= '${end}'`);
+    return this;
+  }
+
+  /**
+   * Add nested metadata filter
+   * @param {string} field - Metadata field name
+   * @param {*} value - Value to compare
+   * @returns {FilterBuilder} this for chaining
+   */
+  metadata(field, value) {
+    this.filters.push(`metadata.${field} = ${this._quote(value)}`);
+    return this;
+  }
+
+  /**
+   * Combine filters with AND
+   * @returns {FilterBuilder} this for chaining
+   */
+  and() {
+    this.operator = 'AND';
+    return this;
+  }
+
+  /**
+   * Combine filters with OR
+   * @returns {FilterBuilder} this for chaining
+   */
+  or() {
+    this.operator = 'OR';
+    return this;
+  }
+
+  /**
+   * Build and return the filter string
+   * @returns {string} Filter expression
+   */
+  build() {
+    if (this.filters.length === 0) {
+      return '';
+    }
+
+    return this.filters.join(` ${this.operator} `);
+  }
+
+  /**
+   * Check if builder has any filters
+   * @returns {boolean} True if filters exist
+   */
+  hasFilters() {
+    return this.filters.length > 0;
+  }
+
+  /**
+   * Get filter count
+   * @returns {number} Number of filters
+   */
+  count() {
+    return this.filters.length;
+  }
+
+  /**
+   * Reset all filters
+   * @returns {FilterBuilder} this for chaining
+   */
+  reset() {
+    this.filters = [];
+    this.operator = 'AND';
+    return this;
+  }
+
+  /**
+   * Quote string values for SQL
+   * @private
+   * @param {*} value - Value to quote
+   * @returns {string} Quoted value
+   */
+  _quote(value) {
+    if (typeof value === 'string') {
+      return `'${value.replace(/'/g, "''")}'`;
+    }
+    if (value === null) {
+      return 'NULL';
+    }
+    if (typeof value === 'boolean') {
+      return value ? 'true' : 'false';
+    }
+    return String(value);
+  }
+
+  /**
+   * Escape special LIKE characters
+   * @private
+   * @param {string} value - Value to escape
+   * @returns {string} Escaped value
+   */
+  _escapeLike(value) {
+    return value.replace(/'/g, "''").replace(/%/g, '\\%').replace(/_/g, '\\_');
+  }
+
+  /**
+   * Create a new builder instance
+   * @returns {FilterBuilder} New builder
+   */
+  static create() {
+    return new FilterBuilder();
+  }
+}
+
+export default FilterBuilder;

package/lib/search/hybrid.js

@@ -0,0 +1,137 @@
+/**
+ * HybridSearch - Combines vector and keyword search
+ */
+
+import { QueryError } from "../lancedb/errors.js";
+
+class HybridSearch {
+  constructor(client, embeddingFactory, options = {}) {
+    this.client = client;
+    this.embeddingFactory = embeddingFactory;
+    this.alpha = options.alpha !== undefined
+      ? options.alpha
+      : parseFloat(process.env.HYBRID_SEARCH_ALPHA || '0.5');
+    this.rrfK = options.rrfK || 60;
+  }
+
+  async search(query, options = {}) {
+    // @ts-ignore
+    const limit = options.limit || 10;
+    // @ts-ignore
+    const alpha = options.alpha !== undefined ? options.alpha : this.alpha;
+
+    try {
+      const [vectorResults, keywordResults] = await Promise.all([
+        // @ts-ignore
+        this._vectorSearch(query, limit * 2, options.filter),
+        // @ts-ignore
+        this._keywordSearch(query, limit * 2, options.filter)
+      ]);
+
+      const mergedResults = this._reciprocalRankFusion(
+        vectorResults,
+        keywordResults,
+        alpha,
+        this.rrfK
+      );
+
+      return mergedResults.slice(0, limit);
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      throw new QueryError('Hybrid search failed', {
+        query,
+        alpha,
+        originalError: message
+      });
+    }
+  }
+
+  async _vectorSearch(query, limit, filter = null) {
+    const embedding = await this.embeddingFactory.embed(query);
+    const searchOptions = { limit, metric: 'cosine' };
+    if (filter) searchOptions.filter = filter;
+
+    const result = await this.client.search(embedding, searchOptions);
+    return result.map(r => ({
+      ...r,
+      score: 1 - (r.score || 0),
+      searchType: 'vector'
+    }));
+  }
+
+  async _keywordSearch(query, limit, filter = null) {
+    try {
+      const embedding = await this.embeddingFactory.embed(query);
+      const searchOptions = { limit: limit * 3, metric: 'cosine' };
+      if (filter) searchOptions.filter = filter;
+
+      const result = await this.client.search(embedding, searchOptions);
+      const queryTerms = query.toLowerCase().split(/\s+/);
+
+      return result
+        .filter(r => {
+          const content = r.content.toLowerCase();
+          return queryTerms.some(term => content.includes(term));
+        })
+        .slice(0, limit)
+        .map(r => ({
+          ...r,
+          score: r.score || 0,
+          searchType: 'keyword'
+        }));
+    } catch (error) {
+      return [];
+    }
+  }
+
+  _reciprocalRankFusion(vectorResults, keywordResults, alpha, k = 60) {
+    const scores = new Map();
+    vectorResults.forEach((result, index) => {
+      const rr = 1 / (k + index + 1);
+      scores.set(result.id, {
+        result,
+        vectorScore: rr * (1 - alpha),
+        keywordScore: 0,
+        vectorRank: index + 1,
+        keywordRank: null
+      });
+    });
+
+    keywordResults.forEach((result, index) => {
+      const rr = 1 / (k + index + 1);
+      if (scores.has(result.id)) {
+        const entry = scores.get(result.id);
+        entry.keywordScore = rr * alpha;
+        entry.keywordRank = index + 1;
+      } else {
+        scores.set(result.id, {
+          result,
+          vectorScore: 0,
+          keywordScore: rr * alpha,
+          vectorRank: null,
+          keywordRank: index + 1
+        });
+      }
+    });
+
+    return Array.from(scores.values())
+      .map(({ result, vectorScore, keywordScore, vectorRank, keywordRank }) => ({
+        ...result,
+        combinedScore: vectorScore + keywordScore,
+        vectorScore,
+        keywordScore,
+        vectorRank,
+        keywordRank,
+        searchType: vectorRank !== null && keywordRank !== null
+          ? 'hybrid'
+          : vectorRank !== null ? 'vector' : 'keyword'
+      }))
+      .sort((a, b) => b.combinedScore - a.combinedScore);
+  }
+
+  getStats() {
+    return { alpha: this.alpha, rrfK: this.rrfK, type: 'hybrid' };
+  }
+}
+
+export default HybridSearch;

package/lib/search/index.js

@@ -0,0 +1,3 @@
+export { default as HybridSearch } from './hybrid.js';
+export { default as FilterBuilder } from './filter.js';
+export { default as PatternMiner } from './pattern-miner.js';

package/lib/search/pattern-miner.js

@@ -0,0 +1,160 @@
+/**
+ * PatternMiner - Analyzes search results for patterns and insights
+ */
+
+import crypto from "crypto";
+
+class PatternMiner {
+  constructor(options = {}) {
+    // @ts-ignore
+    this.similarityThreshold = options.similarityThreshold || 0.8;
+    // @ts-ignore
+    this.minClusterSize = options.minClusterSize || 2;
+    // @ts-ignore
+    this.maxThemes = options.maxThemes || 10;
+    // @ts-ignore
+    this.minWordLength = options.minWordLength || 4;
+  }
+
+  async minePatterns(results, options = {}) {
+    // @ts-ignore
+    const { extractEntities = false, deduplicate = true } = options;
+    let workingResults = [...results];
+    if (deduplicate) workingResults = this.deduplicate(workingResults);
+
+    const patterns = {
+      originalCount: results.length,
+      uniqueCount: workingResults.length,
+      duplicatesRemoved: results.length - workingResults.length,
+      clusters: [],
+      themes: [],
+      entities: [],
+      summary: ''
+    };
+
+    // @ts-ignore
+    patterns.clusters = this._clusterResults(workingResults);
+    // @ts-ignore
+    patterns.themes = this._extractThemes(workingResults);
+
+    if (extractEntities) {
+      // @ts-ignore
+      patterns.entities = await this._extractEntities(workingResults);
+    }
+
+    patterns.summary = this._generateSummary(patterns);
+    return patterns;
+  }
+
+  _clusterResults(results) {
+    const clusters = [];
+    const used = new Set();
+    for (let i = 0; i < results.length; i++) {
+      if (used.has(i)) continue;
+      const cluster = { representative: results[i], members: [results[i]], similarityScore: 1.0 };
+      for (let j = i + 1; j < results.length; j++) {
+        if (used.has(j)) continue;
+        const similarity = this._calculateSimilarity(results[i], results[j]);
+        if (similarity >= this.similarityThreshold) {
+          cluster.members.push(results[j]);
+          used.add(j);
+          cluster.similarityScore = Math.min(cluster.similarityScore, similarity);
+        }
+      }
+      if (cluster.members.length >= this.minClusterSize || cluster.members.length === 1) {
+        used.add(i);
+        clusters.push(cluster);
+      }
+    }
+    return clusters;
+  }
+
+  _calculateSimilarity(result1, result2) {
+    let score = 0;
+    let factors = 0;
+    if (result1.score !== undefined && result2.score !== undefined) {
+      const scoreDiff = Math.abs(result1.score - result2.score);
+      const scoreSim = Math.max(0, 1 - scoreDiff);
+      score += scoreSim * 0.5;
+      factors += 0.5;
+    }
+    if (result1.metadata?.type && result2.metadata?.type) {
+      const typeMatch = result1.metadata.type === result2.metadata.type ? 1 : 0;
+      score += typeMatch * 0.25;
+      factors += 0.25;
+    }
+    const contentSim = this._contentOverlap(result1.content, result2.content);
+    score += contentSim * 0.1;
+    factors += 0.1;
+    return factors > 0 ? score / factors : 0;
+  }
+
+  _contentOverlap(content1, content2) {
+    const words1 = new Set(content1.toLowerCase().split(/\s+/).filter(w => w.length > this.minWordLength));
+    const words2 = new Set(content2.toLowerCase().split(/\s+/).filter(w => w.length > this.minWordLength));
+    if (words1.size === 0 || words2.size === 0) return 0;
+    const intersection = new Set([...words1].filter(x => words2.has(x)));
+    const union = new Set([...words1, ...words2]);
+    return intersection.size / union.size;
+  }
+
+  _extractThemes(results) {
+    const themeMap = new Map();
+    const stopWords = new Set(['the', 'this', 'that', 'with', 'from', 'have', 'been', 'were', 'they', 'their', 'what', 'when', 'where', 'which', 'will', 'your', 'about', 'into', 'through', 'during', 'before', 'after', 'above', 'below', 'could', 'would', 'should', 'each', 'only', 'being', 'other', 'some', 'such', 'them', 'these', 'those', 'over', 'also']);
+    results.forEach(result => {
+      const words = result.content.toLowerCase().split(/\s+/).map(w => w.replace(/[^a-z]/g, '')).filter(w => w.length >= this.minWordLength && !stopWords.has(w));
+      words.forEach(word => {
+        themeMap.set(word, (themeMap.get(word) || 0) + 1);
+      });
+    });
+    return Array.from(themeMap.entries()).sort((a, b) => b[1] - a[1]).slice(0, this.maxThemes).map(([word, count]) => ({
+      word, count, frequency: count / results.length
+    }));
+  }
+
+  async _extractEntities(results) {
+    const entities = [];
+    results.forEach(result => {
+      const emailMatches = result.content.match(/\b[\w.-]+@[\w.-]+\.\w+\b/g);
+      if (emailMatches) emailMatches.forEach(e => entities.push({ type: 'email', value: e }));
+    });
+    const uniqueEntities = [];
+    const seen = new Set();
+    entities.forEach(e => {
+      const key = `${e.type}:${e.value}`;
+      if (!seen.has(key)) {
+        seen.add(key);
+        uniqueEntities.push(e);
+      }
+    });
+    return uniqueEntities;
+  }
+
+  _generateSummary(patterns) {
+    return `Original results: ${patterns.originalCount}, Unique results: ${patterns.uniqueCount}`;
+  }
+
+  deduplicate(results) {
+    const seen = new Set();
+    const unique = [];
+    results.forEach(result => {
+      const hash = crypto.createHash('md5').update(result.content).digest('hex');
+      if (!seen.has(hash)) {
+        seen.add(hash);
+        unique.push(result);
+      }
+    });
+    return unique;
+  }
+
+  getStats() {
+    return {
+      similarityThreshold: this.similarityThreshold,
+      minClusterSize: this.minClusterSize,
+      maxThemes: this.maxThemes,
+      minWordLength: this.minWordLength
+    };
+  }
+}
+
+export default PatternMiner;

package/lib/utils/error-sanitizer.js

@@ -0,0 +1,84 @@
+/**
+ * Secure Error Handling Utilities
+ * Sanitizes error messages to prevent API key leakage
+ */
+
+/**
+ * Sanitize error messages by redacting sensitive information
+ * @param {string} message - Error message to sanitize
+ * @returns {string} Sanitized error message
+ */
+export function sanitizeErrorMessage(message) {
+  if (typeof message !== 'string') {
+    return '[Non-string error message]';
+  }
+
+  // Redact common sensitive patterns
+  return message
+    // Redact Bearer tokens
+    .replace(/Bearer\s+[A-Za-z0-9\-._~+/]+=*/gi, 'Bearer [REDACTED]')
+    // Redact OpenAI API keys (sk- followed by 32+ chars)
+    .replace(/sk-[A-Za-z0-9]{32,}/g, 'sk-[REDACTED]')
+    // Redact generic API keys (20+ alphanumeric chars after api_key)
+    .replace(/api_key["\s:]+[A-Za-z0-9]{20,}/gi, 'api_key: [REDACTED]')
+    // Redact environment variable patterns that might contain secrets
+    .replace(/(OPENAI_API_KEY|ANTHROPIC_API_KEY|GOOGLE_API_KEY)[="'\s]+[A-Za-z0-9\-_]+/gi, '$1=[REDACTED]')
+    // Redact Authorization headers
+    .replace(/Authorization:\s*[^"\r\n]+/gi, 'Authorization: [REDACTED]')
+    // Redact potential JWT tokens (header.payload.signature pattern)
+    .replace(/eyJ[A-Za-z0-9_-]+\.eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]*/g, '[JWT_REDACTED]');
+}
+
+/**
+ * Sanitize error object for logging
+ * @param {Error|Object} error - Error object to sanitize
+ * @returns {Object} Sanitized error object safe for logging
+ */
+export function sanitizeErrorForLogging(error) {
+  if (!error || typeof error !== 'object') {
+    return { message: '[Invalid error object]' };
+  }
+
+  const sanitized = {
+    name: error.name || 'Error',
+    message: sanitizeErrorMessage(error.message || 'Unknown error')
+  };
+
+  // Only include stack in development
+  if (process.env.NODE_ENV === 'development' && error.stack) {
+    sanitized.stack = error.stack;
+  }
+
+  // Include code if present (non-sensitive)
+  if (error.code) {
+    sanitized.code = error.code;
+  }
+
+  // Include timestamp
+  sanitized.timestamp = new Date().toISOString();
+
+  return sanitized;
+}
+
+/**
+ * Wrap a function to catch and sanitize errors
+ * @param {Function} fn - Function to wrap
+ * @param {string} context - Context description for error logging
+ * @returns {Function} Wrapped function with error sanitization
+ */
+export function withSanitizedErrors(fn, context = 'operation') {
+  return async (...args) => {
+    try {
+      return await fn(...args);
+    } catch (error) {
+      const sanitizedError = sanitizeErrorForLogging(error);
+      throw {
+        success: false,
+        error: {
+          ...sanitizedError,
+          context
+        }
+      };
+    }
+  };
+}