@twelvehart/supermemory-runtime 1.0.0-next.0

package/src/services/vectorstore/memory.ts

@@ -0,0 +1,286 @@
+/**
+ * In-Memory Vector Store
+ *
+ * A fast, ephemeral vector store implementation that stores all vectors in memory.
+ * Suitable for development, testing, and small-scale production use.
+ *
+ * Features:
+ * - O(n) linear search with optimized similarity calculation
+ * - Metadata filtering support
+ * - No external dependencies
+ * - Thread-safe operations
+ */
+
+import {
+  VectorEntry,
+  VectorSearchResult,
+  SearchOptions,
+  AddOptions,
+  DeleteOptions,
+  VectorStoreConfig,
+  VectorStoreStats,
+  BatchResult,
+} from './types.js'
+import { BaseVectorStore, validateVector } from './base.js'
+import { ConflictError } from '../../utils/errors.js'
+
+/**
+ * Internal entry with additional tracking
+ */
+interface InternalEntry extends VectorEntry {
+  namespace: string
+}
+
+/**
+ * In-Memory Vector Store implementation
+ */
+export class InMemoryVectorStore extends BaseVectorStore {
+  private entries: Map<string, InternalEntry> = new Map()
+  private initialized = false
+
+  constructor(config: VectorStoreConfig) {
+    super({
+      ...config,
+      provider: 'memory',
+    })
+  }
+
+  /**
+   * Initialize the in-memory store
+   */
+  async initialize(): Promise<void> {
+    if (this.initialized) return
+    this.entries.clear()
+    this.initialized = true
+  }
+
+  /**
+   * Add a single vector entry
+   */
+  async add(entry: VectorEntry, options?: AddOptions): Promise<void> {
+    this.validateEntry(entry)
+    const namespace = options?.namespace ?? this.config.defaultNamespace ?? 'default'
+
+    if (this.entries.has(entry.id) && !options?.overwrite) {
+      throw new ConflictError(`Entry with ID ${entry.id} already exists`, 'duplicate', { entryId: entry.id })
+    }
+
+    const internalEntry: InternalEntry = {
+      ...entry,
+      namespace,
+      createdAt: entry.createdAt ?? new Date(),
+      updatedAt: new Date(),
+    }
+
+    this.entries.set(entry.id, internalEntry)
+    this.emit('add', { id: entry.id })
+  }
+
+  /**
+   * Add multiple vector entries
+   */
+  async addBatch(entries: VectorEntry[], options?: AddOptions): Promise<BatchResult> {
+    const result: BatchResult = {
+      successful: 0,
+      failed: 0,
+      errors: [],
+    }
+
+    for (const entry of entries) {
+      try {
+        await this.add(entry, options)
+        result.successful++
+      } catch (error) {
+        result.failed++
+        result.errors?.push({
+          id: entry.id,
+          error: error instanceof Error ? error.message : String(error),
+        })
+      }
+    }
+
+    return result
+  }
+
+  /**
+   * Update an existing vector entry
+   */
+  async update(id: string, updates: Partial<VectorEntry>): Promise<boolean> {
+    const existing = this.entries.get(id)
+    if (!existing) {
+      return false
+    }
+
+    // Validate embedding if provided
+    if (updates.embedding) {
+      validateVector(updates.embedding, this.config.dimensions)
+    }
+
+    const updated: InternalEntry = {
+      ...existing,
+      ...updates,
+      id, // Ensure ID cannot be changed
+      namespace: existing.namespace, // Preserve namespace
+      updatedAt: new Date(),
+    }
+
+    this.entries.set(id, updated)
+    this.emit('update', { id })
+    return true
+  }
+
+  /**
+   * Delete vector entries
+   */
+  async delete(options: DeleteOptions): Promise<number> {
+    let deleted = 0
+
+    if (options.deleteAll) {
+      const namespace = options.namespace ?? this.config.defaultNamespace ?? 'default'
+      for (const [id, entry] of this.entries) {
+        if (entry.namespace === namespace) {
+          this.entries.delete(id)
+          deleted++
+        }
+      }
+    } else if (options.ids && options.ids.length > 0) {
+      for (const id of options.ids) {
+        if (this.entries.delete(id)) {
+          deleted++
+        }
+      }
+    } else if (options.filter) {
+      for (const [id, entry] of this.entries) {
+        if (this.matchesFilter(entry.metadata, options.filter)) {
+          this.entries.delete(id)
+          deleted++
+        }
+      }
+    }
+
+    if (deleted > 0) {
+      this.emit('delete', { count: deleted })
+    }
+
+    return deleted
+  }
+
+  /**
+   * Get a vector entry by ID
+   */
+  async get(id: string): Promise<VectorEntry | null> {
+    const entry = this.entries.get(id)
+    if (!entry) return null
+
+    // Return copy without internal fields
+    const { namespace: _namespace, ...publicEntry } = entry
+    return publicEntry
+  }
+
+  /**
+   * Check if a vector entry exists
+   */
+  async exists(id: string): Promise<boolean> {
+    return this.entries.has(id)
+  }
+
+  /**
+   * Search for similar vectors using cosine similarity
+   */
+  async search(query: number[], options?: SearchOptions): Promise<VectorSearchResult[]> {
+    validateVector(query, this.config.dimensions)
+    const opts = this.mergeOptions(options)
+
+    // Get all entries and apply filters
+    const allEntries = Array.from(this.entries.values())
+    const candidates = this.applyFilters(allEntries as VectorEntry[], opts.filters) as InternalEntry[]
+
+    // Calculate similarities
+    const results: VectorSearchResult[] = []
+    for (const entry of candidates) {
+      const score = this.calculateSimilarity(query, entry.embedding)
+
+      if (score >= opts.threshold) {
+        results.push({
+          id: entry.id,
+          score,
+          embedding: opts.includeVectors ? entry.embedding : undefined,
+          metadata: opts.includeMetadata ? entry.metadata : {},
+        })
+      }
+    }
+
+    // Sort by score descending and apply limit
+    results.sort((a, b) => b.score - a.score)
+
+    this.emit('search', {
+      resultsCount: Math.min(results.length, opts.limit),
+      totalCandidates: candidates.length,
+    })
+
+    return results.slice(0, opts.limit)
+  }
+
+  /**
+   * Get statistics about the vector store
+   */
+  async getStats(): Promise<VectorStoreStats> {
+    const namespaces = new Set<string>()
+    for (const entry of this.entries.values()) {
+      namespaces.add(entry.namespace)
+    }
+
+    return {
+      totalVectors: this.entries.size,
+      dimensions: this.config.dimensions,
+      indexType: 'flat',
+      metric: this.config.metric ?? 'cosine',
+      indexBuilt: true, // Always true for in-memory
+      namespaces: Array.from(namespaces),
+    }
+  }
+
+  /**
+   * Clear all vectors from the store
+   */
+  async clear(): Promise<void> {
+    this.entries.clear()
+    this.emit('delete', { deleteAll: true })
+  }
+
+  /**
+   * Close the vector store and release resources
+   */
+  async close(): Promise<void> {
+    this.entries.clear()
+    this.initialized = false
+  }
+
+  /**
+   * Get all entries (for migration/export)
+   */
+  async getAllEntries(): Promise<VectorEntry[]> {
+    return Array.from(this.entries.values()).map(({ namespace: _namespace, ...entry }) => entry)
+  }
+
+  /**
+   * Get the number of entries
+   */
+  size(): number {
+    return this.entries.size
+  }
+}
+
+/**
+ * Create an in-memory vector store
+ */
+export function createInMemoryVectorStore(
+  dimensions: number,
+  options?: Partial<Omit<VectorStoreConfig, 'provider' | 'dimensions'>>
+): InMemoryVectorStore {
+  return new InMemoryVectorStore({
+    provider: 'memory',
+    dimensions,
+    ...options,
+  })
+}

package/src/services/vectorstore/migration.ts

@@ -0,0 +1,295 @@
+/**
+ * Vector Store Migration Utilities
+ *
+ * Utilities for migrating vector data between different vector store implementations.
+ * Supports batch processing with progress tracking and error handling.
+ */
+
+import { MigrationProgress, BatchResult } from './types.js'
+import { InMemoryVectorStore } from './memory.js'
+import { PgVectorStore } from './pgvector.js'
+
+/**
+ * Migrate vectors from InMemoryVectorStore to PgVectorStore
+ */
+export async function migrateMemoryToPgVector(
+  source: InMemoryVectorStore,
+  target: PgVectorStore,
+  options?: {
+    batchSize?: number
+    onProgress?: (progress: MigrationProgress) => void
+  }
+): Promise<BatchResult> {
+  const batchSize = options?.batchSize ?? 100
+
+  // Get all entries from source
+  const entries = await source.getAllEntries()
+  const total = entries.length
+
+  if (total === 0) {
+    return {
+      successful: 0,
+      failed: 0,
+      errors: [],
+    }
+  }
+
+  // Initialize result
+  const result: BatchResult = {
+    successful: 0,
+    failed: 0,
+    errors: [],
+  }
+
+  // Calculate batches
+  const totalBatches = Math.ceil(total / batchSize)
+  let currentBatch = 0
+  const startTime = Date.now()
+
+  // Process in batches
+  for (let i = 0; i < entries.length; i += batchSize) {
+    const batch = entries.slice(i, i + batchSize)
+    currentBatch++
+
+    // Add batch to target
+    const batchResult = await target.addBatch(batch, { overwrite: true })
+
+    // Update result
+    result.successful += batchResult.successful
+    result.failed += batchResult.failed
+    if (batchResult.errors && batchResult.errors.length > 0) {
+      result.errors?.push(...batchResult.errors)
+    }
+
+    // Calculate progress
+    const migrated = Math.min(i + batchSize, total)
+    const percentage = (migrated / total) * 100
+    const elapsed = Date.now() - startTime
+    const estimatedTotal = (elapsed / migrated) * total
+    const estimatedTimeRemaining = (estimatedTotal - elapsed) / 1000
+
+    // Report progress
+    if (options?.onProgress) {
+      const progress: MigrationProgress = {
+        total,
+        migrated,
+        percentage,
+        currentBatch,
+        totalBatches,
+        estimatedTimeRemaining,
+      }
+      options.onProgress(progress)
+    }
+  }
+
+  return result
+}
+
+/**
+ * Migrate all vectors from source to target with automatic type detection
+ */
+export async function migrateVectorStore(
+  source: InMemoryVectorStore | PgVectorStore,
+  target: InMemoryVectorStore | PgVectorStore,
+  options?: {
+    batchSize?: number
+    onProgress?: (progress: MigrationProgress) => void
+  }
+): Promise<BatchResult> {
+  // Detect migration type
+  const isMemoryToMemory = source instanceof InMemoryVectorStore && target instanceof InMemoryVectorStore
+  const isMemoryToPg = source instanceof InMemoryVectorStore && target instanceof PgVectorStore
+  const isPgToPg = source instanceof PgVectorStore && target instanceof PgVectorStore
+
+  if (isMemoryToMemory || isPgToPg) {
+    console.warn('Migrating between same store types. Consider using copy instead.')
+  }
+
+  // Perform migration
+  if (isMemoryToPg) {
+    return migrateMemoryToPgVector(source, target, options)
+  }
+
+  // Generic migration for other types
+  return genericMigration(source, target, options)
+}
+
+/**
+ * Generic migration implementation
+ */
+async function genericMigration(
+  source: InMemoryVectorStore | PgVectorStore,
+  target: InMemoryVectorStore | PgVectorStore,
+  options?: {
+    batchSize?: number
+    onProgress?: (progress: MigrationProgress) => void
+  }
+): Promise<BatchResult> {
+  const batchSize = options?.batchSize ?? 100
+
+  // Get all entries
+  const entries = await source.getAllEntries()
+  const total = entries.length
+
+  if (total === 0) {
+    return {
+      successful: 0,
+      failed: 0,
+      errors: [],
+    }
+  }
+
+  // Initialize result
+  const result: BatchResult = {
+    successful: 0,
+    failed: 0,
+    errors: [],
+  }
+
+  // Calculate batches
+  const totalBatches = Math.ceil(total / batchSize)
+  let currentBatch = 0
+  const startTime = Date.now()
+
+  // Process in batches
+  for (let i = 0; i < entries.length; i += batchSize) {
+    const batch = entries.slice(i, i + batchSize)
+    currentBatch++
+
+    // Add batch to target
+    const batchResult = await target.addBatch(batch, { overwrite: true })
+
+    // Update result
+    result.successful += batchResult.successful
+    result.failed += batchResult.failed
+    if (batchResult.errors && batchResult.errors.length > 0) {
+      result.errors?.push(...batchResult.errors)
+    }
+
+    // Calculate progress
+    const migrated = Math.min(i + batchSize, total)
+    const percentage = (migrated / total) * 100
+    const elapsed = Date.now() - startTime
+    const estimatedTotal = (elapsed / migrated) * total
+    const estimatedTimeRemaining = (estimatedTotal - elapsed) / 1000
+
+    // Report progress
+    if (options?.onProgress) {
+      const progress: MigrationProgress = {
+        total,
+        migrated,
+        percentage,
+        currentBatch,
+        totalBatches,
+        estimatedTimeRemaining,
+      }
+      options.onProgress(progress)
+    }
+  }
+
+  return result
+}
+
+/**
+ * Verify migration by comparing vector counts and sample entries
+ */
+export async function verifyMigration(
+  source: InMemoryVectorStore | PgVectorStore,
+  target: InMemoryVectorStore | PgVectorStore,
+  sampleSize = 10
+): Promise<{
+  success: boolean
+  issues: string[]
+  sourceCount: number
+  targetCount: number
+  samplesMatch: number
+  samplesMismatch: number
+}> {
+  const issues: string[] = []
+  let samplesMatch = 0
+  let samplesMismatch = 0
+
+  // Compare counts
+  const sourceStats = await source.getStats()
+  const targetStats = await target.getStats()
+  const sourceCount = sourceStats.totalVectors
+  const targetCount = targetStats.totalVectors
+
+  if (sourceCount !== targetCount) {
+    issues.push(`Vector count mismatch: source has ${sourceCount}, target has ${targetCount}`)
+  }
+
+  // Compare dimensions
+  if (sourceStats.dimensions !== targetStats.dimensions) {
+    issues.push(`Dimension mismatch: source has ${sourceStats.dimensions}, target has ${targetStats.dimensions}`)
+  }
+
+  // Sample verification
+  const sourceEntries = await source.getAllEntries()
+  const sampleIndices = new Set<number>()
+
+  // Generate random sample indices
+  while (sampleIndices.size < Math.min(sampleSize, sourceEntries.length)) {
+    sampleIndices.add(Math.floor(Math.random() * sourceEntries.length))
+  }
+
+  // Compare samples
+  for (const index of sampleIndices) {
+    const sourceEntry = sourceEntries[index]
+    if (!sourceEntry) continue
+
+    const targetEntry = await target.get(sourceEntry.id)
+
+    if (!targetEntry) {
+      issues.push(`Entry ${sourceEntry.id} not found in target`)
+      samplesMismatch++
+      continue
+    }
+
+    // Compare embeddings
+    const embeddingsMatch = sourceEntry.embedding.every(
+      (val, i) => Math.abs(val - (targetEntry.embedding[i] ?? 0)) < 0.0001
+    )
+
+    if (!embeddingsMatch) {
+      issues.push(`Embedding mismatch for entry ${sourceEntry.id}`)
+      samplesMismatch++
+      continue
+    }
+
+    // Compare metadata
+    const metadataMatch = JSON.stringify(sourceEntry.metadata) === JSON.stringify(targetEntry.metadata)
+
+    if (!metadataMatch) {
+      issues.push(`Metadata mismatch for entry ${sourceEntry.id}`)
+      samplesMismatch++
+      continue
+    }
+
+    samplesMatch++
+  }
+
+  return {
+    success: issues.length === 0,
+    issues,
+    sourceCount,
+    targetCount,
+    samplesMatch,
+    samplesMismatch,
+  }
+}
+
+/**
+ * Create a progress reporter function
+ */
+export function createProgressReporter(onUpdate?: (message: string) => void): (progress: MigrationProgress) => void {
+  return (progress: MigrationProgress) => {
+    const message = `Migration progress: ${progress.migrated}/${progress.total} (${progress.percentage.toFixed(1)}%) - Batch ${progress.currentBatch}/${progress.totalBatches}${progress.estimatedTimeRemaining ? ` - ETA: ${Math.round(progress.estimatedTimeRemaining)}s` : ''}`
+
+    if (onUpdate) {
+      onUpdate(message)
+    } else {
+      console.log(message)
+    }
+  }
+}