claude-brain 0.30.2 → 0.30.3

package/src/memory/consolidation/merger.ts

@@ -1,186 +1,182 @@
-/**
- * Memory Merger
- * Finds and merges highly similar memories
- */
-
-import type { Logger } from 'pino'
-import type { CollectionManager } from '../chroma/collection-manager'
-import type { EmbeddingProvider } from '../chroma/embeddings'
-
-export interface MergeCandidate {
-  idA: string
-  idB: string
-  similarity: number
-  contentA: string
-  contentB: string
-}
-
-export interface MergeResult {
-  mergedId: string
-  sourceIds: string[]
-  mergedContent: string
-}
-
-export class MemoryMerger {
-  private logger: Logger
-  private collections: CollectionManager
-  private embeddings?: EmbeddingProvider
-
-  constructor(logger: Logger, collections: CollectionManager, embeddings?: EmbeddingProvider) {
-    this.logger = logger.child({ component: 'memory-merger' })
-    this.collections = collections
-    this.embeddings = embeddings
-  }
-
-  async findMergeCandidates(project: string, threshold: number = 0.85): Promise<MergeCandidate[]> {
-    const candidates: MergeCandidate[] = []
-
-    try {
-      const collection = await this.collections.getDecisions()
-      const results = await collection.get({
-        where: { project: { $eq: project } },
-        include: ['documents', 'metadatas', 'embeddings']
-      })
-
-      if (results.ids.length < 2) return candidates
-
-      // Pairwise comparison using embeddings
-      for (let i = 0; i < results.ids.length; i++) {
-        const docI = results.documents?.[i] as string
-        if (!docI) continue
-
-        // Query for similar items to this document
-        let similar: any
-
-        if (this.embeddings && results.embeddings?.[i]) {
-          similar = await collection.query({
-            queryEmbeddings: [results.embeddings[i] as number[]],
-            nResults: 5,
-            where: { project: { $eq: project } },
-            include: ['documents', 'distances']
-          })
-        } else {
-          similar = await collection.query({
-            queryTexts: [docI],
-            nResults: 5,
-            where: { project: { $eq: project } },
-            include: ['documents', 'distances']
-          })
-        }
-
-        if (!similar.ids?.[0]) continue
-
-        for (let j = 0; j < similar.ids[0].length; j++) {
-          const otherId = similar.ids[0][j]
-          if (otherId === results.ids[i]) continue
-
-          const distance = similar.distances?.[0]?.[j] || 1
-          const similarity = 1 - distance
-
-          if (similarity >= threshold) {
-            // Avoid duplicate pairs
-            const exists = candidates.some(c =>
-              (c.idA === results.ids[i] && c.idB === otherId) ||
-              (c.idA === otherId && c.idB === results.ids[i])
-            )
-
-            if (!exists) {
-              candidates.push({
-                idA: results.ids[i]!,
-                idB: otherId,
-                similarity,
-                contentA: docI,
-                contentB: similar.documents?.[0]?.[j] || ''
-              })
-            }
-          }
-        }
-      }
-
-      this.logger.debug({ project, candidateCount: candidates.length }, 'Found merge candidates')
-    } catch (error) {
-      this.logger.error({ error, project }, 'Failed to find merge candidates')
-    }
-
-    return candidates.sort((a, b) => b.similarity - a.similarity)
-  }
-
-  async merge(memoryIds: string[]): Promise<MergeResult | null> {
-    if (memoryIds.length < 2) return null
-
-    try {
-      const collection = await this.collections.getDecisions()
-      const results = await collection.get({
-        ids: memoryIds,
-        include: ['documents', 'metadatas']
-      })
-
-      if (results.ids.length < 2) return null
-
-      // Keep the newest document as the base
-      const documents = results.documents as string[]
-      const metadatas = results.metadatas as Record<string, any>[]
-
-      // Sort by created_at descending (newest first)
-      const indexed = results.ids.map((id, i) => ({
-        id,
-        document: documents[i],
-        metadata: metadatas[i]
-      }))
-
-      indexed.sort((a, b) => {
-        const dateA = a.metadata?.created_at || ''
-        const dateB = b.metadata?.created_at || ''
-        return dateB.localeCompare(dateA)
-      })
-
-      const newest = indexed[0]!
-      const mergedContent = newest.document ?? ''
-
-      // Update the newest with merged metadata
-      const mergedMetadata: Record<string, any> = {
-        ...newest.metadata,
-        updated_at: new Date().toISOString(),
-        merged_from: memoryIds.filter(id => id !== newest.id).join(',')
-      }
-
-      // Combine tags from all sources
-      const allTags = new Set<string>()
-      for (const item of indexed) {
-        const tags = item.metadata?.tags?.split(',') || []
-        for (const tag of tags) {
-          if (tag.trim()) allTags.add(tag.trim())
-        }
-      }
-      mergedMetadata.tags = Array.from(allTags).join(',')
-
-      // Update the surviving memory
-      await collection.update({
-        ids: [newest.id],
-        documents: [mergedContent],
-        metadatas: [mergedMetadata]
-      })
-
-      // Delete the other memories
-      const toDelete = memoryIds.filter(id => id !== newest.id)
-      if (toDelete.length > 0) {
-        await collection.delete({ ids: toDelete })
-      }
-
-      this.logger.info({
-        mergedId: newest.id,
-        sourceIds: memoryIds,
-        deletedCount: toDelete.length
-      }, 'Memories merged')
-
-      return {
-        mergedId: newest.id,
-        sourceIds: memoryIds,
-        mergedContent
-      }
-    } catch (error) {
-      this.logger.error({ error, memoryIds }, 'Failed to merge memories')
-      return null
-    }
-  }
-}
+/**
+ * Memory Merger
+ * Finds and merges highly similar memories
+ */
+
+import type { Logger } from 'pino'
+import type { CollectionManager } from '../chroma/collection-manager'
+import type { EmbeddingProvider } from '../chroma/embeddings'
+
+export interface MergeCandidate {
+  idA: string
+  idB: string
+  similarity: number
+  contentA: string
+  contentB: string
+}
+
+export interface MergeResult {
+  mergedId: string
+  sourceIds: string[]
+  mergedContent: string
+}
+
+export class MemoryMerger {
+  private logger: Logger
+  private collections: CollectionManager
+  private embeddings?: EmbeddingProvider
+
+  constructor(logger: Logger, collections: CollectionManager, embeddings?: EmbeddingProvider) {
+    this.logger = logger.child({ component: 'memory-merger' })
+    this.collections = collections
+    this.embeddings = embeddings
+  }
+
+  async findMergeCandidates(project: string, threshold: number = 0.85): Promise<MergeCandidate[]> {
+    const candidates: MergeCandidate[] = []
+
+    try {
+      const collection = await this.collections.getDecisions()
+      const results = await collection.get({
+        where: { project: { $eq: project } },
+        include: ['documents', 'metadatas', 'embeddings']
+      })
+
+      if (results.ids.length < 2) return candidates
+
+      // Pairwise comparison using embeddings
+      for (let i = 0; i < results.ids.length; i++) {
+        const docI = results.documents?.[i] as string
+        if (!docI) continue
+
+        // Query for similar items to this document
+        const similar = (this.embeddings && results.embeddings?.[i])
+          ? await collection.query({
+              queryEmbeddings: [results.embeddings[i] as number[]],
+              nResults: 5,
+              where: { project: { $eq: project } },
+              include: ['documents', 'distances']
+            })
+          : await collection.query({
+              queryTexts: [docI],
+              nResults: 5,
+              where: { project: { $eq: project } },
+              include: ['documents', 'distances']
+            })
+
+        if (!similar.ids?.[0]) continue
+
+        for (let j = 0; j < similar.ids[0].length; j++) {
+          const otherId = similar.ids[0][j]
+          if (otherId === results.ids[i]) continue
+
+          const distance = similar.distances?.[0]?.[j] || 1
+          const similarity = 1 - distance
+
+          if (similarity >= threshold) {
+            // Avoid duplicate pairs
+            const exists = candidates.some(c =>
+              (c.idA === results.ids[i] && c.idB === otherId) ||
+              (c.idA === otherId && c.idB === results.ids[i])
+            )
+
+            if (!exists) {
+              candidates.push({
+                idA: results.ids[i]!,
+                idB: otherId!,
+                similarity,
+                contentA: docI,
+                contentB: similar.documents?.[0]?.[j] || ''
+              })
+            }
+          }
+        }
+      }
+
+      this.logger.debug({ project, candidateCount: candidates.length }, 'Found merge candidates')
+    } catch (error) {
+      this.logger.error({ error, project }, 'Failed to find merge candidates')
+    }
+
+    return candidates.sort((a, b) => b.similarity - a.similarity)
+  }
+
+  async merge(memoryIds: string[]): Promise<MergeResult | null> {
+    if (memoryIds.length < 2) return null
+
+    try {
+      const collection = await this.collections.getDecisions()
+      const results = await collection.get({
+        ids: memoryIds,
+        include: ['documents', 'metadatas']
+      })
+
+      if (results.ids.length < 2) return null
+
+      // Keep the newest document as the base
+      const documents = results.documents as string[]
+      const metadatas = results.metadatas as Record<string, any>[]
+
+      // Sort by created_at descending (newest first)
+      const indexed = results.ids.map((id, i) => ({
+        id,
+        document: documents[i],
+        metadata: metadatas[i]
+      }))
+
+      indexed.sort((a, b) => {
+        const dateA = a.metadata?.created_at || ''
+        const dateB = b.metadata?.created_at || ''
+        return dateB.localeCompare(dateA)
+      })
+
+      const newest = indexed[0]!
+      const mergedContent = newest.document ?? ''
+
+      // Update the newest with merged metadata
+      const mergedMetadata: Record<string, any> = {
+        ...newest.metadata,
+        updated_at: new Date().toISOString(),
+        merged_from: memoryIds.filter(id => id !== newest.id).join(',')
+      }
+
+      // Combine tags from all sources
+      const allTags = new Set<string>()
+      for (const item of indexed) {
+        const tags = item.metadata?.tags?.split(',') || []
+        for (const tag of tags) {
+          if (tag.trim()) allTags.add(tag.trim())
+        }
+      }
+      mergedMetadata.tags = Array.from(allTags).join(',')
+
+      // Update the surviving memory
+      await collection.update({
+        ids: [newest.id],
+        documents: [mergedContent],
+        metadatas: [mergedMetadata]
+      })
+
+      // Delete the other memories
+      const toDelete = memoryIds.filter(id => id !== newest.id)
+      if (toDelete.length > 0) {
+        await collection.delete({ ids: toDelete })
+      }
+
+      this.logger.info({
+        mergedId: newest.id,
+        sourceIds: memoryIds,
+        deletedCount: toDelete.length
+      }, 'Memories merged')
+
+      return {
+        mergedId: newest.id,
+        sourceIds: memoryIds,
+        mergedContent
+      }
+    } catch (error) {
+      this.logger.error({ error, memoryIds }, 'Failed to merge memories')
+      return null
+    }
+  }
+}

package/src/memory/consolidation/scorer.ts

@@ -1,136 +1,136 @@
-/**
- * Importance Scorer
- * Multi-factor importance scoring for memories
- */
-
-import type { Logger } from 'pino'
-import type { InMemoryKnowledgeGraph } from '../../knowledge/graph/memory-graph'
-
-export interface ImportanceWeights {
-  recency: number
-  frequency: number
-  references: number
-  feedback: number
-  uniqueness: number
-}
-
-export interface ImportanceScore {
-  total: number
-  components: {
-    recency: number
-    frequency: number
-    references: number
-    feedback: number
-    uniqueness: number
-  }
-  memoryId: string
-}
-
-const DEFAULT_WEIGHTS: ImportanceWeights = {
-  recency: 0.2,
-  frequency: 0.2,
-  references: 0.2,
-  feedback: 0.2,
-  uniqueness: 0.2
-}
-
-export class ImportanceScorer {
-  private weights: ImportanceWeights
-  private graph?: InMemoryKnowledgeGraph
-
-  constructor(_logger: Logger, weights?: Partial<ImportanceWeights>, graph?: InMemoryKnowledgeGraph) {
-    this.weights = { ...DEFAULT_WEIGHTS, ...weights }
-    this.graph = graph
-  }
-
-  score(memory: {
-    id: string
-    created_at: string
-    recall_count?: number
-    average_rating?: number
-    max_similarity?: number
-    project?: string
-  }): ImportanceScore {
-    const recencyScore = this.calculateRecency(memory.created_at)
-    const frequencyScore = this.calculateFrequency(memory.recall_count || 0)
-    const referencesScore = this.calculateReferences(memory.id)
-    const feedbackScore = this.calculateFeedback(memory.average_rating)
-    const uniquenessScore = this.calculateUniqueness(memory.max_similarity)
-
-    const total =
-      this.weights.recency * recencyScore +
-      this.weights.frequency * frequencyScore +
-      this.weights.references * referencesScore +
-      this.weights.feedback * feedbackScore +
-      this.weights.uniqueness * uniquenessScore
-
-    return {
-      total,
-      components: {
-        recency: recencyScore,
-        frequency: frequencyScore,
-        references: referencesScore,
-        feedback: feedbackScore,
-        uniqueness: uniquenessScore
-      },
-      memoryId: memory.id
-    }
-  }
-
-  scoreBatch(memories: Array<{
-    id: string
-    created_at: string
-    recall_count?: number
-    average_rating?: number
-    max_similarity?: number
-    project?: string
-  }>): ImportanceScore[] {
-    return memories
-      .map(m => this.score(m))
-      .sort((a, b) => b.total - a.total)
-  }
-
-  private calculateRecency(createdAt: string): number {
-    const now = Date.now()
-    const created = new Date(createdAt).getTime()
-    const ageMs = now - created
-
-    // Exponential decay: half-life of 30 days
-    const halfLifeMs = 30 * 24 * 60 * 60 * 1000
-    return Math.exp(-0.693 * (ageMs / halfLifeMs))
-  }
-
-  private calculateFrequency(recallCount: number): number {
-    // Logarithmic scaling: diminishing returns for very high recall
-    if (recallCount === 0) return 0
-    return Math.min(1.0, Math.log10(recallCount + 1) / 2)
-  }
-
-  private calculateReferences(memoryId: string): number {
-    if (!this.graph) return 0.5 // neutral if no graph
-
-    // Count edges pointing to nodes related to this memory
-    const nodes = this.graph.findNodes({ properties: { decision_id: memoryId } })
-    if (nodes.length === 0) return 0.3
-
-    let edgeCount = 0
-    for (const node of nodes) {
-      edgeCount += this.graph.getEdges(node.id).length
-    }
-
-    // Normalize: 10+ references → score of 1.0
-    return Math.min(1.0, edgeCount / 10)
-  }
-
-  private calculateFeedback(averageRating?: number): number {
-    if (averageRating === undefined) return 0.5 // neutral if no feedback
-    // Scale 1-5 rating to 0-1
-    return (averageRating - 1) / 4
-  }
-
-  private calculateUniqueness(maxSimilarity?: number): number {
-    if (maxSimilarity === undefined) return 0.5 // neutral if unknown
-    // Inverse of max similarity: unique memories score higher
-    return 1 - maxSimilarity
-  }
-}
+/**
+ * Importance Scorer
+ * Multi-factor importance scoring for memories
+ */
+
+import type { Logger } from 'pino'
+import type { InMemoryKnowledgeGraph } from '../../knowledge/graph/memory-graph'
+
+export interface ImportanceWeights {
+  recency: number
+  frequency: number
+  references: number
+  feedback: number
+  uniqueness: number
+}
+
+export interface ImportanceScore {
+  total: number
+  components: {
+    recency: number
+    frequency: number
+    references: number
+    feedback: number
+    uniqueness: number
+  }
+  memoryId: string
+}
+
+const DEFAULT_WEIGHTS: ImportanceWeights = {
+  recency: 0.2,
+  frequency: 0.2,
+  references: 0.2,
+  feedback: 0.2,
+  uniqueness: 0.2
+}
+
+export class ImportanceScorer {
+  private weights: ImportanceWeights
+  private graph?: InMemoryKnowledgeGraph
+
+  constructor(_logger: Logger, weights?: Partial<ImportanceWeights>, graph?: InMemoryKnowledgeGraph) {
+    this.weights = { ...DEFAULT_WEIGHTS, ...weights }
+    this.graph = graph
+  }
+
+  score(memory: {
+    id: string
+    created_at: string
+    recall_count?: number
+    average_rating?: number
+    max_similarity?: number
+    project?: string
+  }): ImportanceScore {
+    const recencyScore = this.calculateRecency(memory.created_at)
+    const frequencyScore = this.calculateFrequency(memory.recall_count || 0)
+    const referencesScore = this.calculateReferences(memory.id)
+    const feedbackScore = this.calculateFeedback(memory.average_rating)
+    const uniquenessScore = this.calculateUniqueness(memory.max_similarity)
+
+    const total =
+      this.weights.recency * recencyScore +
+      this.weights.frequency * frequencyScore +
+      this.weights.references * referencesScore +
+      this.weights.feedback * feedbackScore +
+      this.weights.uniqueness * uniquenessScore
+
+    return {
+      total,
+      components: {
+        recency: recencyScore,
+        frequency: frequencyScore,
+        references: referencesScore,
+        feedback: feedbackScore,
+        uniqueness: uniquenessScore
+      },
+      memoryId: memory.id
+    }
+  }
+
+  scoreBatch(memories: Array<{
+    id: string
+    created_at: string
+    recall_count?: number
+    average_rating?: number
+    max_similarity?: number
+    project?: string
+  }>): ImportanceScore[] {
+    return memories
+      .map(m => this.score(m))
+      .sort((a, b) => b.total - a.total)
+  }
+
+  private calculateRecency(createdAt: string): number {
+    const now = Date.now()
+    const created = new Date(createdAt).getTime()
+    const ageMs = now - created
+
+    // Exponential decay: half-life of 30 days
+    const halfLifeMs = 30 * 24 * 60 * 60 * 1000
+    return Math.exp(-0.693 * (ageMs / halfLifeMs))
+  }
+
+  private calculateFrequency(recallCount: number): number {
+    // Logarithmic scaling: diminishing returns for very high recall
+    if (recallCount === 0) return 0
+    return Math.min(1.0, Math.log10(recallCount + 1) / 2)
+  }
+
+  private calculateReferences(memoryId: string): number {
+    if (!this.graph) return 0.5 // neutral if no graph
+
+    // Count edges pointing to nodes related to this memory
+    const nodes = this.graph.findNodes({ properties: { decision_id: memoryId } })
+    if (nodes.length === 0) return 0.3
+
+    let edgeCount = 0
+    for (const node of nodes) {
+      edgeCount += this.graph.getEdges(node.id).length
+    }
+
+    // Normalize: 10+ references → score of 1.0
+    return Math.min(1.0, edgeCount / 10)
+  }
+
+  private calculateFeedback(averageRating?: number): number {
+    if (averageRating === undefined) return 0.5 // neutral if no feedback
+    // Scale 1-5 rating to 0-1
+    return (averageRating - 1) / 4
+  }
+
+  private calculateUniqueness(maxSimilarity?: number): number {
+    if (maxSimilarity === undefined) return 0.5 // neutral if unknown
+    // Inverse of max similarity: unique memories score higher
+    return 1 - maxSimilarity
+  }
+}

package/src/memory/database.ts

@@ -39,6 +39,15 @@ export class MemoryDatabase {
       // Enable foreign keys
       this.db.run('PRAGMA foreign_keys = ON')
 
+      // Performance optimizations
+      this.db.run('PRAGMA synchronous = NORMAL')
+      this.db.run('PRAGMA cache_size = -64000')
+      this.db.run('PRAGMA mmap_size = 268435456')
+      this.db.run('PRAGMA temp_store = MEMORY')
+      this.db.run('PRAGMA journal_size_limit = 67108864')
+      this.db.run('PRAGMA wal_autocheckpoint = 1000')
+      this.db.run('PRAGMA busy_timeout = 5000')
+
       // Create schema
       this.db.exec(MEMORY_SCHEMA)