swarm_sdk 2.7.13 → 3.0.0.alpha1

data/lib/swarm_sdk/v3/memory/exposure_tracker.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+module SwarmSDK
+  module V3
+    module Memory
+      # Calculates exposure scores for memory cards
+      #
+      # Exposure score = α·frequency + β·recency + γ·dwell  (weighted sum)
+      #
+      # Uses an additive formula so each signal contributes independently.
+      # A multiplicative formula would zero out the score whenever any single
+      # factor is zero (e.g., a fresh card with access_count=0), which would
+      # make every new card an immediate compression candidate.
+      #
+      # Used by the compressor to decide which cards get lossy compression first.
+      # Low-exposure cards are candidates for compression or eviction.
+      #
+      # @example
+      #   tracker = ExposureTracker.new(adapter)
+      #   score = tracker.exposure_score(card)
+      class ExposureTracker
+        # Default weights for the additive exposure formula
+        DEFAULT_FREQUENCY_WEIGHT = 0.4
+        DEFAULT_RECENCY_WEIGHT = 0.4
+        DEFAULT_DWELL_WEIGHT = 0.2
+
+        # Default half-life for recency decay in seconds (7 days)
+        DEFAULT_RECENCY_HALF_LIFE = 7 * 24 * 3600
+
+        # @param adapter [Adapters::Base] Storage adapter
+        # @param frequency_weight [Float] Weight for frequency component (α)
+        # @param recency_weight [Float] Weight for recency component (β)
+        # @param dwell_weight [Float] Weight for dwell component (γ)
+        # @param recency_half_life [Integer] Half-life for recency decay in seconds
+        def initialize(adapter, frequency_weight: nil, recency_weight: nil, dwell_weight: nil, recency_half_life: nil)
+          @adapter = adapter
+          config = Configuration.instance
+          @frequency_weight = frequency_weight || config.exposure_frequency_weight
+          @recency_weight = recency_weight || config.exposure_recency_weight
+          @dwell_weight = dwell_weight || config.exposure_dwell_weight
+          @recency_half_life = recency_half_life || config.exposure_recency_half_life
+        end
+
+        # Calculate exposure score for a card
+        #
+        # Combines three signals additively:
+        # - Frequency: log(1 + access_count) — dampens high-access cards
+        # - Recency: exponential decay from last access time
+        # - Dwell: accumulated time in working context
+        #
+        # E = α·log(1 + access_count) + β·recency(last_accessed) + γ·dwell
+        #
+        # @param card [Card] Card to score
+        # @return [Float] Exposure score (higher = more exposed)
+        #
+        # @example
+        #   score = tracker.exposure_score(card)
+        #   # High score: frequently accessed, recently used, high dwell
+        #   # Low score: rarely accessed, long ago, low dwell
+        def exposure_score(card)
+          frequency = Math.log(1 + card.access_count)
+          recency = recency_factor(card.last_accessed)
+          dwell = card.dwell
+
+          @frequency_weight * frequency +
+            @recency_weight * recency +
+            @dwell_weight * dwell
+        end
+
+        # Rank all cards by exposure score (ascending = least exposed first)
+        #
+        # @return [Array<Hash>] Array of { card:, score: } sorted ascending
+        def rank_by_exposure
+          @adapter.list_cards.map do |card|
+            { card: card, score: exposure_score(card) }
+          end.sort_by { |entry| entry[:score] }
+        end
+
+        # Find cards with low exposure (candidates for compression)
+        #
+        # @param threshold [Float] Maximum exposure score
+        # @return [Array<Card>] Low-exposure cards
+        def low_exposure_cards(threshold: 1.0)
+          rank_by_exposure
+            .select { |entry| entry[:score] < threshold }
+            .map { |entry| entry[:card] }
+        end
+
+        private
+
+        # Calculate recency factor using exponential decay
+        #
+        # @param last_accessed [Time, nil] Last access time
+        # @return [Float] Recency factor (0.0-1.0)
+        def recency_factor(last_accessed)
+          return 0.0 if last_accessed.nil?
+
+          age_seconds = Time.now - last_accessed
+          Math.exp(-age_seconds * Math.log(2) / @recency_half_life)
+        end
+      end
+    end
+  end
+end

data/lib/swarm_sdk/v3/memory/ingestion_pipeline.rb

@@ -0,0 +1,394 @@
+# frozen_string_literal: true
+
+module SwarmSDK
+  module V3
+    module Memory
+      # Async post-turn processing pipeline
+      #
+      # After each conversation turn, this pipeline:
+      # 1. Segments the turn into atomic ideas (LLM when available, heuristic fallback)
+      # 2. Creates memory cards (<=250 words each)
+      # 3. Extracts entities
+      # 4. Creates graph edges between related cards (within turn and cross-turn)
+      # 5. Assigns cards to clusters (updating decision logs for decision cards)
+      # 6. Generates embeddings
+      #
+      # @example
+      #   pipeline = IngestionPipeline.new(adapter: adapter, embedder: embedder)
+      #   pipeline.ingest(turn_text: "The API uses JWT...", turn_id: "turn_001")
+      class IngestionPipeline
+        # @param adapter [Adapters::Base] Storage adapter
+        # @param embedder [Embedder] Text embedder
+        # @param chat [RubyLLM::Chat, nil] LLM for segmentation (nil = simple splitting)
+        def initialize(adapter:, embedder:, chat: nil)
+          @adapter = adapter
+          @embedder = embedder
+          @chat = chat
+          @config = Configuration.instance
+        end
+
+        # Ingest a conversation turn into memory cards
+        #
+        # @param turn_text [String] Full turn text (user + assistant + tool calls)
+        # @param turn_id [String] Unique turn identifier
+        # @return [Array<Card>] Created cards
+        def ingest(turn_text:, turn_id:)
+          DebugLog.log("ingestion", "ingest: turn_id=#{turn_id}, text_len=#{turn_text.size}")
+
+          segments = DebugLog.time("ingestion", "segment_turn") do
+            segment_turn(turn_text)
+          end
+          DebugLog.log("ingestion", "segments=#{segments.size}")
+
+          cards = DebugLog.time("ingestion", "create_cards(#{segments.size})") do
+            segments.map { |segment| create_card(text: segment, turn_id: turn_id) }
+          end
+
+          DebugLog.time("ingestion", "create_turn_edges") { create_turn_edges(cards) }
+          DebugLog.time("ingestion", "cross_turn_entity_edges") { create_cross_turn_entity_edges(cards) }
+          DebugLog.time("ingestion", "assign_to_clusters") { assign_to_clusters(cards) }
+
+          DebugLog.log("ingestion", "ingest complete: #{cards.size} cards created")
+          cards
+        end
+
+        private
+
+        # Segment a turn into atomic ideas
+        #
+        # Uses LLM for semantic segmentation when available. The LLM produces
+        # better idea-boundary splits and self-contained cards without dangling
+        # pronouns. Falls back to paragraph-based splitting when no LLM is
+        # configured.
+        #
+        # @param text [String] Turn text
+        # @return [Array<String>] Segments
+        def segment_turn(text)
+          return [] if text.nil? || text.strip.empty?
+
+          if @chat
+            llm_segment(text)
+          else
+            heuristic_segment(text)
+          end
+        end
+
+        # Use LLM to segment text into atomic, self-contained ideas
+        #
+        # Resets the background chat before each call to prevent message
+        # accumulation across multiple ingestion operations.
+        #
+        # @param text [String] Turn text
+        # @return [Array<String>] Self-contained segments
+        def llm_segment(text)
+          @chat.reset_messages!
+          response = @chat.ask(<<~PROMPT)
+            Segment the following conversation turn into atomic, self-contained memory cards.
+            Each card should:
+            - Capture exactly ONE idea, fact, decision, or concept
+            - Be <=250 words
+            - Be fully self-contained (no dangling "this/that/it" without referent)
+            - Include the subject/entity and claim/decision explicitly
+
+            Separate each card with "---" on its own line.
+            Output ONLY the cards, no explanations.
+
+            Text to segment:
+            #{text}
+          PROMPT
+
+          segments = response.content.strip.split(/^---+\s*$/).map(&:strip).reject(&:empty?)
+          # Validate segments are reasonable
+          return heuristic_segment(text) if segments.empty?
+
+          # Enforce word limit on LLM output
+          segments.flat_map { |s| word_count(s) > 250 ? split_long_segment(s) : s }
+        rescue StandardError => e
+          EventStream.emit(
+            type: "memory_segmentation_llm_error",
+            error: "#{e.class}: #{e.message}",
+          )
+          # Fall back to heuristic on any LLM error
+          heuristic_segment(text)
+        end
+
+        # Heuristic segmentation by paragraphs with merging/splitting
+        #
+        # @param text [String] Turn text
+        # @return [Array<String>] Segments
+        def heuristic_segment(text)
+          paragraphs = text.split(/\n\n+/).map(&:strip).reject(&:empty?)
+
+          # Merge very short paragraphs, split very long ones
+          segments = []
+          buffer = +""
+
+          paragraphs.each do |para|
+            if buffer.empty?
+              buffer << para
+            elsif word_count(buffer) + word_count(para) <= 250
+              buffer << "\n\n#{para}"
+            else
+              segments << buffer.dup
+              buffer.replace(para)
+            end
+          end
+
+          segments << buffer unless buffer.empty?
+
+          # Split any segments that are still too long
+          segments.flat_map { |s| word_count(s) > 250 ? split_long_segment(s) : s }
+        end
+
+        # Create a memory card from a text segment
+        #
+        # Sets importance based on inferred card type:
+        # - Constraints and decisions get high importance (0.8)
+        # - Preferences and incidents get medium-high importance (0.7)
+        # - Concepts get medium importance (0.6)
+        # - Facts get default importance (0.5)
+        #
+        # @param text [String] Card text
+        # @param turn_id [String] Source turn ID
+        # @return [Card] Created and persisted card
+        def create_card(text:, turn_id:)
+          entities = extract_entities(text)
+          type = infer_type(text)
+          embedding = @embedder.embed(text)
+          importance = importance_for_type(type)
+
+          card = Card.new(
+            text: text,
+            type: type,
+            entities: entities,
+            source_turn_ids: [turn_id],
+            embedding: embedding,
+            importance: importance,
+            dwell: 0.0,
+          )
+
+          @adapter.write_card(card)
+          card
+        end
+
+        # Determine importance score based on card type
+        #
+        # Constraints and decisions are high-importance because violating
+        # or forgetting them has serious consequences. Preferences and
+        # incidents are medium-high because they inform future decisions.
+        #
+        # @param type [Symbol] Card type
+        # @return [Float] Importance score (0.0-1.0)
+        def importance_for_type(type)
+          case type
+          when :constraint then 0.8
+          when :decision   then 0.8
+          when :preference then 0.7
+          when :incident   then 0.7
+          when :concept    then 0.6
+          else 0.5 # :fact
+          end
+        end
+
+        # Extract named entities from text
+        #
+        # Simple heuristic: capitalized multi-word phrases, technical terms.
+        #
+        # @param text [String] Text to extract from
+        # @return [Array<String>] Extracted entities
+        def extract_entities(text)
+          # Find capitalized words that aren't at sentence starts
+          words = text.scan(/(?<=[.!?]\s)\w+|\b[A-Z][a-z]*(?:\s+[A-Z][a-z]*)*\b/)
+          words.uniq.reject { |w| w.length < 2 }.take(10)
+        end
+
+        # Infer card type from text content
+        #
+        # @param text [String] Card text
+        # @return [Symbol] Card type
+        def infer_type(text)
+          lower = text.downcase
+          if lower.include?("decided") || lower.include?("decision") || lower.include?("chose")
+            :decision
+          elsif lower.include?("must") || lower.include?("always") || lower.include?("never") || lower.include?("constraint")
+            :constraint
+          elsif lower.include?("prefer") || lower.include?("like") || lower.include?("want")
+            :preference
+          elsif lower.include?("error") || lower.include?("bug") || lower.include?("crash") || lower.include?("incident")
+            :incident
+          elsif lower.include?("concept") || lower.include?("pattern") || lower.include?("architecture")
+            :concept
+          else
+            :fact
+          end
+        end
+
+        # Create edges between cards from the same turn
+        #
+        # @param cards [Array<Card>] Cards from the same turn
+        # @return [void]
+        def create_turn_edges(cards)
+          cards.each_cons(2) do |a, b|
+            edge = Edge.new(
+              from_id: a.id,
+              to_id: b.id,
+              type: :same_episode,
+              weight: 0.8,
+            )
+            @adapter.write_edge(edge)
+
+            # Also create same_entity edges for shared entities
+            shared = a.entities & b.entities
+            next if shared.empty?
+
+            entity_edge = Edge.new(
+              from_id: a.id,
+              to_id: b.id,
+              type: :same_entity,
+              weight: [0.5 + (shared.size * 0.1), 1.0].min,
+            )
+            @adapter.write_edge(entity_edge)
+          end
+        end
+
+        # Create edges between new cards and existing cards that share entities
+        #
+        # Scans recent existing cards for shared entity names and creates
+        # same_entity edges when matches are found. This connects new knowledge
+        # to the existing memory graph across turns.
+        #
+        # @param new_cards [Array<Card>] Newly created cards
+        # @return [void]
+        def create_cross_turn_entity_edges(new_cards)
+          return if new_cards.empty?
+
+          new_card_ids = Set.new(new_cards.map(&:id))
+          existing_cards = @adapter.list_cards.reject { |c| new_card_ids.include?(c.id) }
+          return if existing_cards.empty?
+
+          # Limit scan to avoid O(n*m) explosion on large memory stores
+          candidates = existing_cards.last(@config.cross_turn_edge_scan_limit)
+
+          new_cards.each do |new_card|
+            next if new_card.entities.empty?
+
+            new_entities_down = new_card.entities.map(&:downcase).to_set
+
+            candidates.each do |existing_card|
+              next if existing_card.entities.empty?
+
+              shared = existing_card.entities.select { |e| new_entities_down.include?(e.downcase) }
+              next if shared.empty?
+
+              edge = Edge.new(
+                from_id: new_card.id,
+                to_id: existing_card.id,
+                type: :same_entity,
+                weight: [0.4 + (shared.size * 0.1), 1.0].min,
+              )
+              @adapter.write_edge(edge)
+            end
+          end
+        end
+
+        # Assign cards to existing clusters or create new ones
+        #
+        # When a decision-type card is added to a cluster, the cluster's
+        # decision_log is updated with the decision text.
+        #
+        # @param cards [Array<Card>] Cards to assign
+        # @return [void]
+        def assign_to_clusters(cards)
+          @adapter.transaction do
+            clusters = @adapter.list_clusters
+
+            cards.each do |card|
+              best_cluster = find_best_cluster(card, clusters)
+
+              if best_cluster
+                best_cluster.add_card(card.id)
+                update_cluster_for_card(best_cluster, card)
+                @adapter.write_cluster(best_cluster)
+              else
+                # Create a new cluster from this card
+                cluster = Cluster.new(
+                  title: card.entities.first || card.type.to_s.capitalize,
+                  card_ids: [card.id],
+                  key_entities: card.entities.take(5),
+                  embedding: card.embedding,
+                )
+                update_cluster_for_card(cluster, card)
+                @adapter.write_cluster(cluster)
+                clusters << cluster
+              end
+            end
+          end
+        end
+
+        # Update a cluster when a new card is added
+        #
+        # Appends to the cluster's decision_log if the card is a decision type.
+        # Updates key_entities with any new entities from the card.
+        #
+        # @param cluster [Cluster] Cluster to update
+        # @param card [Card] Card being added
+        # @return [void]
+        def update_cluster_for_card(cluster, card)
+          # Update decision log for decision-type cards
+          if card.type == :decision
+            decision_entry = card.text.slice(0, 200)
+            cluster.decision_log << decision_entry unless cluster.decision_log.include?(decision_entry)
+          end
+
+          # Merge new entities into cluster
+          new_entities = card.entities - cluster.key_entities
+          cluster.key_entities.concat(new_entities.take(5)) unless new_entities.empty?
+        end
+
+        # Find the best matching cluster for a card
+        #
+        # Uses the adapter's similarity method so that storage backends
+        # like pgvector can compute similarity server-side.
+        #
+        # @param card [Card] Card to match
+        # @param clusters [Array<Cluster>] Available clusters
+        # @return [Cluster, nil] Best cluster or nil
+        def find_best_cluster(card, clusters)
+          return if clusters.empty? || card.embedding.nil?
+
+          best = nil
+          best_score = 0.3 # Minimum similarity threshold
+
+          clusters.each do |cluster|
+            next unless cluster.embedding
+
+            score = @adapter.similarity(card.embedding, cluster.embedding)
+            if score > best_score
+              best_score = score
+              best = cluster
+            end
+          end
+
+          best
+        end
+
+        # @param text [String]
+        # @return [Integer]
+        def word_count(text)
+          text.split(/\s+/).size
+        end
+
+        # Split a long segment into chunks of ~250 words
+        #
+        # @param text [String] Long text
+        # @return [Array<String>] Chunks
+        def split_long_segment(text)
+          words = text.split(/\s+/)
+          chunks = []
+          words.each_slice(250) { |slice| chunks << slice.join(" ") }
+          chunks
+        end
+      end
+    end
+  end
+end