swarm_sdk 2.7.14 → 3.0.0.alpha2

data/lib/swarm_sdk/v3/memory/consolidator.rb

@@ -0,0 +1,427 @@
+# frozen_string_literal: true
+
+module SwarmSDK
+  module V3
+    module Memory
+      # Periodic maintenance for the memory system
+      #
+      # Performs:
+      # - Deduplication and canonicalization of similar cards
+      # - Cluster summary updates
+      # - Conflict detection between contradicting cards
+      # - Merging of redundant cards
+      #
+      # @example
+      #   consolidator = Consolidator.new(adapter: adapter, embedder: embedder)
+      #   consolidator.run
+      class Consolidator
+        # Edge types that indicate a supporting relationship between cards.
+        # Pairs connected by these edges should never be flagged as contradictions.
+        # Derived from Edge::TYPES so new edge types are non-conflicting by default.
+        SUPPORTING_EDGE_TYPES = (Edge::TYPES - [:contradicts]).freeze
+
+        # @param adapter [Adapters::Base] Storage adapter
+        # @param embedder [Embedder] Text embedder
+        def initialize(adapter:, embedder:)
+          @adapter = adapter
+          @embedder = embedder
+          @config = Configuration.instance
+        end
+
+        # Run full consolidation
+        #
+        # @return [Hash] Summary of actions taken
+        def run
+          deduped = deduplicate
+          conflicts = detect_conflicts
+          clusters_updated = update_cluster_summaries
+
+          {
+            duplicates_merged: deduped,
+            conflicts_detected: conflicts,
+            clusters_updated: clusters_updated,
+          }
+        end
+
+        # Run full consolidation with progress events
+        #
+        # @param base_completed [Integer] Items completed before this phase
+        # @param total_items [Integer] Total items across all phases
+        # @return [Hash] Summary of actions taken
+        def run_with_progress(base_completed, total_items)
+          deduped = deduplicate_with_progress(base_completed, total_items)
+
+          # Update base for next sub-phase
+          dedup_count = @adapter.list_cards.select(&:embedding).size
+          conflicts = detect_conflicts_with_progress(base_completed + dedup_count, total_items)
+
+          # Update base for cluster phase
+          conflict_types = [:constraint, :decision, :fact]
+          conflict_count = @adapter.list_cards.count { |c| conflict_types.include?(c.type) && c.embedding }
+          clusters_updated = update_cluster_summaries_with_progress(
+            base_completed + dedup_count + conflict_count,
+            total_items,
+          )
+
+          {
+            duplicates_merged: deduped,
+            conflicts_detected: conflicts,
+            clusters_updated: clusters_updated,
+          }
+        end
+
+        # Find and merge duplicate cards
+        #
+        # @return [Integer] Number of duplicates merged
+        def deduplicate
+          cards = @adapter.list_cards
+          merged_count = 0
+          seen = Set.new
+
+          cards.each do |card|
+            next if seen.include?(card.id)
+            next unless card.embedding
+
+            # Find similar cards via vector search
+            similar = @adapter.vector_search(card.embedding, top_k: 5, threshold: @config.consolidator_dedup_threshold)
+
+            similar.each do |result|
+              next if result[:id] == card.id
+              next if seen.include?(result[:id])
+
+              duplicate = @adapter.read_card(result[:id])
+              next unless duplicate
+
+              merge_cards(canonical: card, duplicate: duplicate)
+              seen.add(duplicate.id)
+              merged_count += 1
+            end
+
+            seen.add(card.id)
+          end
+
+          merged_count
+        end
+
+        # Detect contradicting cards and create conflict edges
+        #
+        # Finds cards with high text similarity but different content,
+        # particularly when both are constraint or decision type cards.
+        # Creates `contradicts` edges between them.
+        #
+        # @return [Integer] Number of conflicts detected
+        def detect_conflicts
+          cards = @adapter.list_cards
+          conflict_types = [:constraint, :decision, :fact]
+          candidates = cards.select { |c| conflict_types.include?(c.type) && c.embedding }
+          conflict_count = 0
+          seen_pairs = Set.new
+
+          candidates.each do |card|
+            # Find similar cards that might contradict
+            similar = @adapter.vector_search(card.embedding, top_k: 5, threshold: @config.consolidator_conflict_threshold)
+
+            similar.each do |result|
+              next if result[:id] == card.id
+              next if result[:similarity] > @config.consolidator_dedup_threshold # Too similar = duplicate, not conflict
+
+              pair_key = [card.id, result[:id]].sort.join(":")
+              next if seen_pairs.include?(pair_key)
+
+              other = @adapter.read_card(result[:id])
+              next unless other
+              next unless conflict_types.include?(other.type)
+
+              # Check if an edge already exists between them
+              existing_edges = @adapter.edges_for(card.id)
+              other_id = result[:id]
+
+              already_contradicts = existing_edges.any? do |e|
+                (e.from_id == other_id || e.to_id == other_id) && e.type == :contradicts
+              end
+              next if already_contradicts
+
+              # Skip pairs connected by supporting edges — these reinforce
+              # the same fact in different phrasings, not contradictions
+              has_supporting_edge = existing_edges.any? do |e|
+                (e.from_id == other_id || e.to_id == other_id) && SUPPORTING_EDGE_TYPES.include?(e.type)
+              end
+              next if has_supporting_edge
+
+              # Cards are similar enough to be about the same topic but different
+              # enough to potentially contradict — flag them
+              edge = Edge.new(
+                from_id: card.id,
+                to_id: result[:id],
+                type: :contradicts,
+                weight: result[:similarity],
+              )
+              @adapter.write_edge(edge)
+              seen_pairs.add(pair_key)
+              conflict_count += 1
+            end
+          end
+
+          conflict_count
+        end
+
+        # Update rolling summaries for all clusters
+        #
+        # @return [Integer] Number of clusters updated
+        def update_cluster_summaries
+          clusters = @adapter.list_clusters
+          updated = 0
+
+          clusters.each do |cluster|
+            next if cluster.card_ids.empty?
+
+            # Load cluster cards
+            cards = cluster.card_ids.filter_map { |id| @adapter.read_card(id) }
+            next if cards.empty?
+
+            # Update key entities from member cards
+            all_entities = cards.flat_map(&:entities).tally
+            cluster.key_entities = all_entities.sort_by { |_, count| -count }.take(10).map(&:first)
+
+            # Update rolling summary from card texts
+            cluster.rolling_summary = cards.map(&:text).join(" | ").slice(0, 500)
+
+            # Update cluster embedding (average of member embeddings)
+            embeddings = cards.filter_map(&:embedding)
+            cluster.embedding = average_embedding(embeddings) unless embeddings.empty?
+
+            cluster.updated_at = Time.now
+            @adapter.write_cluster(cluster)
+            updated += 1
+          end
+
+          updated
+        end
+
+        # Find and merge duplicate cards with progress events
+        #
+        # @param base_completed [Integer] Items completed before this phase
+        # @param total_items [Integer] Total items across all phases
+        # @return [Integer] Number of duplicates merged
+        def deduplicate_with_progress(base_completed, total_items)
+          cards = @adapter.list_cards
+          cards_with_embedding = cards.select(&:embedding)
+          merged_count = 0
+          seen = Set.new
+
+          cards_with_embedding.each_with_index do |card, index|
+            unless seen.include?(card.id)
+              # Find similar cards via vector search
+              similar = @adapter.vector_search(card.embedding, top_k: 5, threshold: @config.consolidator_dedup_threshold)
+
+              similar.each do |result|
+                next if result[:id] == card.id
+                next if seen.include?(result[:id])
+
+                duplicate = @adapter.read_card(result[:id])
+                next unless duplicate
+
+                merge_cards(canonical: card, duplicate: duplicate)
+                seen.add(duplicate.id)
+                merged_count += 1
+              end
+
+              seen.add(card.id)
+            end
+
+            EventStream.emit(
+              type: "memory_defrag_progress",
+              phase: "consolidate_dedup",
+              description: "Finding and merging duplicate memory cards",
+              phase_current: index + 1,
+              phase_total: cards_with_embedding.size,
+              overall_current: base_completed + index + 1,
+              overall_total: total_items,
+            )
+          end
+
+          merged_count
+        end
+
+        # Detect contradicting cards with progress events
+        #
+        # @param base_completed [Integer] Items completed before this phase
+        # @param total_items [Integer] Total items across all phases
+        # @return [Integer] Number of conflicts detected
+        def detect_conflicts_with_progress(base_completed, total_items)
+          cards = @adapter.list_cards
+          conflict_types = [:constraint, :decision, :fact]
+          candidates = cards.select { |c| conflict_types.include?(c.type) && c.embedding }
+          conflict_count = 0
+          seen_pairs = Set.new
+
+          candidates.each_with_index do |card, index|
+            # Find similar cards that might contradict
+            similar = @adapter.vector_search(card.embedding, top_k: 5, threshold: @config.consolidator_conflict_threshold)
+
+            similar.each do |result|
+              next if result[:id] == card.id
+              next if result[:similarity] > @config.consolidator_dedup_threshold
+
+              pair_key = [card.id, result[:id]].sort.join(":")
+              next if seen_pairs.include?(pair_key)
+
+              other = @adapter.read_card(result[:id])
+              next unless other
+              next unless conflict_types.include?(other.type)
+
+              existing_edges = @adapter.edges_for(card.id)
+              other_id = result[:id]
+
+              already_contradicts = existing_edges.any? do |e|
+                (e.from_id == other_id || e.to_id == other_id) && e.type == :contradicts
+              end
+              next if already_contradicts
+
+              has_supporting_edge = existing_edges.any? do |e|
+                (e.from_id == other_id || e.to_id == other_id) && SUPPORTING_EDGE_TYPES.include?(e.type)
+              end
+              next if has_supporting_edge
+
+              edge = Edge.new(
+                from_id: card.id,
+                to_id: result[:id],
+                type: :contradicts,
+                weight: result[:similarity],
+              )
+              @adapter.write_edge(edge)
+              seen_pairs.add(pair_key)
+              conflict_count += 1
+            end
+
+            EventStream.emit(
+              type: "memory_defrag_progress",
+              phase: "consolidate_conflicts",
+              description: "Detecting contradicting information in memory",
+              phase_current: index + 1,
+              phase_total: candidates.size,
+              overall_current: base_completed + index + 1,
+              overall_total: total_items,
+            )
+          end
+
+          conflict_count
+        end
+
+        # Update cluster summaries with progress events
+        #
+        # @param base_completed [Integer] Items completed before this phase
+        # @param total_items [Integer] Total items across all phases
+        # @return [Integer] Number of clusters updated
+        def update_cluster_summaries_with_progress(base_completed, total_items)
+          clusters = @adapter.list_clusters
+          updated = 0
+
+          clusters.each_with_index do |cluster, index|
+            unless cluster.card_ids.empty?
+              cards = cluster.card_ids.filter_map { |id| @adapter.read_card(id) }
+              unless cards.empty?
+                all_entities = cards.flat_map(&:entities).tally
+                cluster.key_entities = all_entities.sort_by { |_, count| -count }.take(10).map(&:first)
+                cluster.rolling_summary = cards.map(&:text).join(" | ").slice(0, 500)
+                embeddings = cards.filter_map(&:embedding)
+                cluster.embedding = average_embedding(embeddings) unless embeddings.empty?
+                cluster.updated_at = Time.now
+                @adapter.write_cluster(cluster)
+                updated += 1
+              end
+            end
+
+            EventStream.emit(
+              type: "memory_defrag_progress",
+              phase: "consolidate_clusters",
+              description: "Updating cluster summaries and embeddings",
+              phase_current: index + 1,
+              phase_total: clusters.size,
+              overall_current: base_completed + index + 1,
+              overall_total: total_items,
+            )
+          end
+
+          updated
+        end
+
+        private
+
+        # Merge a duplicate card into a canonical card
+        #
+        # @param canonical [Card] Card to keep
+        # @param duplicate [Card] Card to merge and mark
+        # @return [void]
+        def merge_cards(canonical:, duplicate:)
+          # Transfer source turn IDs
+          combined_turns = (canonical.source_turn_ids + duplicate.source_turn_ids).uniq
+
+          # Transfer entities
+          combined_entities = (canonical.entities + duplicate.entities).uniq
+
+          # Keep the longer/more detailed text
+          if duplicate.text.length > canonical.text.length && duplicate.compression_level <= canonical.compression_level
+            canonical.text = duplicate.text
+          end
+
+          canonical.source_turn_ids = combined_turns
+          canonical.entities = combined_entities
+          canonical.access_count += duplicate.access_count
+          canonical.updated_at = Time.now
+
+          # Mark duplicate as merged
+          duplicate.canonical_id = canonical.id
+          duplicate.updated_at = Time.now
+
+          # Transfer edges from duplicate to canonical
+          edges = @adapter.edges_for(duplicate.id)
+          edges.each do |edge|
+            new_from = edge.from_id == duplicate.id ? canonical.id : edge.from_id
+            new_to = edge.to_id == duplicate.id ? canonical.id : edge.to_id
+            next if new_from == new_to # Skip self-edges
+
+            new_edge = Edge.new(
+              from_id: new_from,
+              to_id: new_to,
+              type: edge.type,
+              weight: edge.weight,
+            )
+            @adapter.write_edge(new_edge)
+          end
+
+          # Persist changes
+          @adapter.write_card(canonical)
+          @adapter.delete_card(duplicate.id)
+          @adapter.delete_edges_for(duplicate.id)
+
+          # Remove from clusters
+          @adapter.list_clusters.each do |cluster|
+            next unless cluster.card_ids.include?(duplicate.id)
+
+            cluster.remove_card(duplicate.id)
+            cluster.add_card(canonical.id) unless cluster.card_ids.include?(canonical.id)
+            @adapter.write_cluster(cluster)
+          end
+        end
+
+        # Average multiple embedding vectors
+        #
+        # @param embeddings [Array<Array<Float>>] Vectors to average
+        # @return [Array<Float>] Averaged vector
+        def average_embedding(embeddings)
+          return embeddings.first if embeddings.size == 1
+
+          dims = embeddings.first.size
+          avg = Array.new(dims, 0.0)
+
+          embeddings.each do |emb|
+            emb.each_with_index { |v, i| avg[i] += v }
+          end
+
+          avg.map { |v| v / embeddings.size }
+        end
+      end
+    end
+  end
+end