@longarc/mdash 3.1.2 → 3.1.3

package/src/context/store.ts

@@ -0,0 +1,503 @@
+/**
+ * Caret — Fragment Store Implementation
+ * @module @longarcstudios/caret/store
+ * 
+ * Storage interface for context fragments.
+ * Provides in-memory implementation and interface for custom backends.
+ */
+
+import type {
+  ContextFragment,
+  FragmentStore,
+  FragmentId,
+  Hash,
+  SourceURI,
+  Timestamp,
+} from './types.js';
+import { verifyFragment } from './fragment.js';
+
+// ============================================================================
+// IN-MEMORY STORE
+// ============================================================================
+
+/**
+ * In-memory fragment store.
+ * Suitable for development, testing, and single-session use.
+ */
+export class MemoryStore implements FragmentStore {
+  private fragments: Map<FragmentId, ContextFragment> = new Map();
+  private hashIndex: Map<Hash, FragmentId> = new Map();
+  private sourceIndex: Map<SourceURI, Set<FragmentId>> = new Map();
+  private deletedIds: Set<FragmentId> = new Set();
+  private seal_key: string;
+  
+  constructor(seal_key: string) {
+    this.seal_key = seal_key;
+  }
+  
+  async put(fragment: ContextFragment): Promise<void> {
+    this.fragments.set(fragment.id, fragment);
+    this.hashIndex.set(fragment.hash, fragment.id);
+    
+    const source = fragment.provenance.head.source;
+    if (!this.sourceIndex.has(source)) {
+      this.sourceIndex.set(source, new Set());
+    }
+    this.sourceIndex.get(source)!.add(fragment.id);
+    this.deletedIds.delete(fragment.id);
+  }
+  
+  async get(id: FragmentId): Promise<ContextFragment | null> {
+    if (this.deletedIds.has(id)) return null;
+    return this.fragments.get(id) ?? null;
+  }
+  
+  async getByHash(hash: Hash): Promise<ContextFragment | null> {
+    const id = this.hashIndex.get(hash);
+    if (!id || this.deletedIds.has(id)) return null;
+    return this.fragments.get(id) ?? null;
+  }
+  
+  async queryBySource(source: SourceURI): Promise<readonly ContextFragment[]> {
+    const ids = this.sourceIndex.get(source);
+    if (!ids) return [];
+    
+    const results: ContextFragment[] = [];
+    for (const id of ids) {
+      if (!this.deletedIds.has(id)) {
+        const fragment = this.fragments.get(id);
+        if (fragment) results.push(fragment);
+      }
+    }
+    return results;
+  }
+  
+  async queryByTime(from: Timestamp, to: Timestamp): Promise<readonly ContextFragment[]> {
+    const fromTime = new Date(from).getTime();
+    const toTime = new Date(to).getTime();
+    const results: ContextFragment[] = [];
+    
+    for (const [id, fragment] of this.fragments) {
+      if (this.deletedIds.has(id)) continue;
+      const sealedTime = new Date(fragment.sealed_at).getTime();
+      if (sealedTime >= fromTime && sealedTime <= toTime) {
+        results.push(fragment);
+      }
+    }
+    
+    return results.sort((a, b) => 
+      new Date(a.sealed_at).getTime() - new Date(b.sealed_at).getTime()
+    );
+  }
+  
+  async delete(id: FragmentId): Promise<boolean> {
+    if (!this.fragments.has(id) || this.deletedIds.has(id)) return false;
+    this.deletedIds.add(id);
+    return true;
+  }
+  
+  async verify(id: FragmentId): Promise<boolean> {
+    const fragment = await this.get(id);
+    if (!fragment) return false;
+    const result = await verifyFragment(fragment, this.seal_key);
+    return result.valid;
+  }
+  
+  size(): number {
+    return this.fragments.size - this.deletedIds.size;
+  }
+  
+  async all(): Promise<readonly ContextFragment[]> {
+    const results: ContextFragment[] = [];
+    for (const [id, fragment] of this.fragments) {
+      if (!this.deletedIds.has(id)) results.push(fragment);
+    }
+    return results;
+  }
+  
+  clear(): void {
+    this.fragments.clear();
+    this.hashIndex.clear();
+    this.sourceIndex.clear();
+    this.deletedIds.clear();
+  }
+  
+  compact(): void {
+    for (const id of this.deletedIds) {
+      const fragment = this.fragments.get(id);
+      if (fragment) {
+        this.hashIndex.delete(fragment.hash);
+        this.sourceIndex.get(fragment.provenance.head.source)?.delete(id);
+      }
+      this.fragments.delete(id);
+    }
+    this.deletedIds.clear();
+  }
+  
+  /**
+   * Layer 2: Keyword search (lexical).
+   * Searches fragment content for matching keywords.
+   * 
+   * @param keywords - Keywords to match (case-insensitive)
+   * @param matchMode - 'any' matches if any keyword found, 'all' requires all keywords
+   * @returns Matching fragments
+   */
+  async queryByKeywords(
+    keywords: string[],
+    matchMode: 'any' | 'all' = 'any'
+  ): Promise<readonly ContextFragment[]> {
+    if (keywords.length === 0) return [];
+    
+    const normalizedKeywords = keywords.map(k => k.toLowerCase());
+    const results: ContextFragment[] = [];
+    
+    for (const [id, fragment] of this.fragments) {
+      if (this.deletedIds.has(id)) continue;
+      
+      // Convert fragment content to searchable text
+      const text = this.fragmentToSearchableText(fragment).toLowerCase();
+      
+      const matches = normalizedKeywords.map(keyword => text.includes(keyword));
+      
+      const isMatch = matchMode === 'any'
+        ? matches.some(Boolean)
+        : matches.every(Boolean);
+      
+      if (isMatch) {
+        results.push(fragment);
+      }
+    }
+    
+    return results;
+  }
+  
+  /**
+   * Convert fragment to searchable text.
+   * Extracts text from content and source.
+   */
+  private fragmentToSearchableText(fragment: ContextFragment): string {
+    const parts: string[] = [];
+    
+    // Content
+    if (typeof fragment.content.data === 'string') {
+      parts.push(fragment.content.data);
+    } else if (fragment.content.data !== null && typeof fragment.content.data === 'object') {
+      parts.push(JSON.stringify(fragment.content.data));
+    }
+    
+    // Source URI
+    parts.push(fragment.provenance.head.source);
+    
+    return parts.join(' ');
+  }
+}
+
+// ============================================================================
+// CACHED STORE
+// ============================================================================
+
+export interface CacheConfig {
+  maxSize: number;
+  ttlMs?: number;
+}
+
+export class CachedStore implements FragmentStore {
+  private backend: FragmentStore;
+  private cache: Map<FragmentId, { fragment: ContextFragment; timestamp: number }>;
+  private maxSize: number;
+  private ttlMs: number;
+  
+  constructor(backend: FragmentStore, config: CacheConfig) {
+    this.backend = backend;
+    this.cache = new Map();
+    this.maxSize = config.maxSize;
+    this.ttlMs = config.ttlMs ?? 300000;
+  }
+  
+  async put(fragment: ContextFragment): Promise<void> {
+    await this.backend.put(fragment);
+    this.cacheFragment(fragment);
+  }
+  
+  async get(id: FragmentId): Promise<ContextFragment | null> {
+    const cached = this.cache.get(id);
+    if (cached && Date.now() - cached.timestamp < this.ttlMs) {
+      this.cache.delete(id);
+      this.cache.set(id, cached);
+      return cached.fragment;
+    }
+    if (cached) this.cache.delete(id);
+    
+    const fragment = await this.backend.get(id);
+    if (fragment) this.cacheFragment(fragment);
+    return fragment;
+  }
+  
+  async getByHash(hash: Hash): Promise<ContextFragment | null> {
+    const fragment = await this.backend.getByHash(hash);
+    if (fragment) this.cacheFragment(fragment);
+    return fragment;
+  }
+  
+  async queryBySource(source: SourceURI): Promise<readonly ContextFragment[]> {
+    return this.backend.queryBySource(source);
+  }
+  
+  async queryByTime(from: Timestamp, to: Timestamp): Promise<readonly ContextFragment[]> {
+    return this.backend.queryByTime(from, to);
+  }
+  
+  async delete(id: FragmentId): Promise<boolean> {
+    this.cache.delete(id);
+    return this.backend.delete(id);
+  }
+  
+  async verify(id: FragmentId): Promise<boolean> {
+    return this.backend.verify(id);
+  }
+  
+  private cacheFragment(fragment: ContextFragment): void {
+    if (this.cache.size >= this.maxSize) {
+      const oldest = this.cache.keys().next().value;
+      if (oldest) this.cache.delete(oldest);
+    }
+    this.cache.set(fragment.id, { fragment, timestamp: Date.now() });
+  }
+  
+  clearCache(): void {
+    this.cache.clear();
+  }
+  
+  cacheStats(): { size: number; maxSize: number } {
+    return { size: this.cache.size, maxSize: this.maxSize };
+  }
+}
+
+// ============================================================================
+// COMPOSITE STORE
+// ============================================================================
+
+export class CompositeStore implements FragmentStore {
+  private primary: FragmentStore;
+  private replicas: FragmentStore[];
+  
+  constructor(primary: FragmentStore, replicas: FragmentStore[] = []) {
+    this.primary = primary;
+    this.replicas = replicas;
+  }
+  
+  async put(fragment: ContextFragment): Promise<void> {
+    await Promise.all([
+      this.primary.put(fragment),
+      ...this.replicas.map(r => r.put(fragment)),
+    ]);
+  }
+  
+  async get(id: FragmentId): Promise<ContextFragment | null> {
+    return this.primary.get(id);
+  }
+  
+  async getByHash(hash: Hash): Promise<ContextFragment | null> {
+    return this.primary.getByHash(hash);
+  }
+  
+  async queryBySource(source: SourceURI): Promise<readonly ContextFragment[]> {
+    return this.primary.queryBySource(source);
+  }
+  
+  async queryByTime(from: Timestamp, to: Timestamp): Promise<readonly ContextFragment[]> {
+    return this.primary.queryByTime(from, to);
+  }
+  
+  async delete(id: FragmentId): Promise<boolean> {
+    const results = await Promise.all([
+      this.primary.delete(id),
+      ...this.replicas.map(r => r.delete(id)),
+    ]);
+    return results[0];
+  }
+  
+  async verify(id: FragmentId): Promise<boolean> {
+    return this.primary.verify(id);
+  }
+}
+
+// ============================================================================
+// STORE FACTORY
+// ============================================================================
+
+export type StoreType = 'memory' | 'cached-memory';
+
+export interface StoreOptions {
+  type: StoreType;
+  seal_key: string;
+  cache?: CacheConfig;
+}
+
+export function createStore(options: StoreOptions): FragmentStore {
+  switch (options.type) {
+    case 'memory':
+      return new MemoryStore(options.seal_key);
+    case 'cached-memory':
+      const memory = new MemoryStore(options.seal_key);
+      return new CachedStore(memory, options.cache ?? { maxSize: 1000 });
+    default:
+      throw new Error(`Unknown store type: ${options.type}`);
+  }
+}
+
+// ============================================================================
+// EXTENDED MEMORY STORE (Three-Layer Index)
+// ============================================================================
+
+/**
+ * Embedding function type.
+ * Takes text and returns embedding vector.
+ */
+export type EmbeddingFunction = (text: string) => Promise<Float32Array>;
+
+/**
+ * Extended memory store with Three-Layer Index support.
+ * 
+ * Layer 1: Semantic (embedding-based similarity)
+ * Layer 2: Lexical (keyword search)
+ * Layer 3: Symbolic (structured metadata — inherited from MemoryStore)
+ */
+export class ExtendedMemoryStore extends MemoryStore {
+  private embeddings: Map<FragmentId, Float32Array> = new Map();
+  private embeddingFn: EmbeddingFunction | null = null;
+  
+  /**
+   * Set the embedding function for semantic search.
+   * 
+   * @param fn - Function that converts text to embedding vector
+   * 
+   * @example
+   * ```ts
+   * store.setEmbeddingFunction(async (text) => {
+   *   // Use OpenAI, local model, etc.
+   *   return await embed(text);
+   * });
+   * ```
+   */
+  setEmbeddingFunction(fn: EmbeddingFunction): void {
+    this.embeddingFn = fn;
+  }
+  
+  /**
+   * Store fragment and compute embedding if embedding function is set.
+   */
+  async put(fragment: ContextFragment): Promise<void> {
+    await super.put(fragment);
+    
+    // Compute and store embedding if function available
+    if (this.embeddingFn) {
+      const text = this.fragmentToText(fragment);
+      const embedding = await this.embeddingFn(text);
+      this.embeddings.set(fragment.id, embedding);
+    }
+  }
+  
+  /**
+   * Manually set embedding for a fragment.
+   * Useful when embeddings are pre-computed.
+   */
+  setEmbedding(id: FragmentId, embedding: Float32Array): void {
+    this.embeddings.set(id, embedding);
+  }
+  
+  /**
+   * Layer 1: Semantic similarity search.
+   * Requires embedding function to be set.
+   * 
+   * @param embedding - Query embedding vector
+   * @param threshold - Minimum cosine similarity (0.0-1.0)
+   * @param limit - Maximum results
+   * @returns Matching fragments with similarity scores
+   */
+  async queryBySimilarity(
+    embedding: Float32Array,
+    threshold: number = 0.7,
+    limit: number = 10
+  ): Promise<readonly { fragment: ContextFragment; similarity: number }[]> {
+    const results: { fragment: ContextFragment; similarity: number }[] = [];
+    
+    const allFragments = await this.all();
+    
+    for (const fragment of allFragments) {
+      const storedEmbedding = this.embeddings.get(fragment.id);
+      if (!storedEmbedding) continue;
+      
+      const similarity = cosineSimilarity(embedding, storedEmbedding);
+      
+      if (similarity >= threshold) {
+        results.push({ fragment, similarity });
+      }
+    }
+    
+    // Sort by similarity descending
+    results.sort((a, b) => b.similarity - a.similarity);
+    
+    // Apply limit
+    return results.slice(0, limit);
+  }
+  
+  /**
+   * Convert fragment to text for embedding.
+   */
+  private fragmentToText(fragment: ContextFragment): string {
+    const parts: string[] = [];
+    
+    if (typeof fragment.content.data === 'string') {
+      parts.push(fragment.content.data);
+    } else if (fragment.content.data !== null && typeof fragment.content.data === 'object') {
+      parts.push(JSON.stringify(fragment.content.data));
+    }
+    
+    return parts.join(' ');
+  }
+  
+  /**
+   * Clear embeddings cache.
+   */
+  clearEmbeddings(): void {
+    this.embeddings.clear();
+  }
+  
+  /**
+   * Get embedding statistics.
+   */
+  embeddingStats(): { count: number; fragmentCount: number } {
+    return {
+      count: this.embeddings.size,
+      fragmentCount: this.size(),
+    };
+  }
+}
+
+/**
+ * Compute cosine similarity between two vectors.
+ */
+function cosineSimilarity(a: Float32Array, b: Float32Array): number {
+  if (a.length !== b.length) {
+    throw new Error(`Embedding dimension mismatch: ${a.length} vs ${b.length}`);
+  }
+  
+  let dotProduct = 0;
+  let normA = 0;
+  let normB = 0;
+  
+  for (let i = 0; i < a.length; i++) {
+    const aVal = a[i] ?? 0;
+    const bVal = b[i] ?? 0;
+    dotProduct += aVal * bVal;
+    normA += aVal * aVal;
+    normB += bVal * bVal;
+  }
+  
+  const denominator = Math.sqrt(normA) * Math.sqrt(normB);
+  
+  if (denominator === 0) return 0;
+  
+  return dotProduct / denominator;
+}