swarm_sdk 2.7.14 → 3.0.0.alpha1

data/lib/swarm_sdk/v3/memory/store.rb

@@ -0,0 +1,489 @@
+# frozen_string_literal: true
+
+module SwarmSDK
+  module V3
+    module Memory
+      # Orchestrator for the memory system
+      #
+      # Coordinates all memory components: adapter, embedder, retriever,
+      # context builder, ingestion pipeline, compressor, and consolidator.
+      # This is the main entry point for Agent to interact with memory.
+      #
+      # @example
+      #   store = Store.new(adapter: adapter, embedder: embedder)
+      #   store.load
+      #
+      #   # Retrieve relevant context for a query
+      #   messages = store.build_context(query: "auth", recent_turns: turns)
+      #
+      #   # Ingest a new turn
+      #   store.ingest_turn(text: "We use JWT...", turn_id: "turn_001")
+      #
+      #   store.save
+      class Store
+        # @return [Adapters::Base] Storage adapter
+        attr_reader :adapter
+
+        # @return [Embedder] Text embedder
+        attr_reader :embedder
+
+        # @return [Retriever] Hybrid search retriever
+        attr_reader :retriever
+
+        # @param adapter [Adapters::Base] Storage adapter
+        # @param embedder [Embedder] Text embedder
+        # @param retrieval_top_k [Integer] Cards to retrieve per query
+        # @param semantic_weight [Float] Semantic search weight
+        # @param keyword_weight [Float] Keyword search weight
+        # @param chat [RubyLLM::Chat, nil] LLM chat for compression/ingestion
+        # @param associative_memory [Boolean] Whether to enable associative memory
+        def initialize(adapter:, embedder:, retrieval_top_k: 15, semantic_weight: 0.5, keyword_weight: 0.5, chat: nil,
+          associative_memory: false)
+          @adapter = adapter
+          @embedder = embedder
+          @chat = chat
+
+          @retriever = Retriever.new(
+            adapter: adapter,
+            embedder: embedder,
+            semantic_weight: semantic_weight,
+            keyword_weight: keyword_weight,
+          )
+
+          @context_builder = ContextBuilder.new(
+            retriever: @retriever,
+            adapter: adapter,
+            retrieval_top_k: retrieval_top_k,
+            embedder: embedder,
+            associative_memory: associative_memory,
+          )
+
+          @ingestion_pipeline = IngestionPipeline.new(
+            adapter: adapter,
+            embedder: embedder,
+            chat: chat,
+          )
+
+          @exposure_tracker = ExposureTracker.new(adapter)
+
+          @compressor = Compressor.new(
+            adapter: adapter,
+            chat: chat,
+          )
+
+          @consolidator = Consolidator.new(
+            adapter: adapter,
+            embedder: embedder,
+          )
+        end
+
+        # Load memory state from durable storage
+        #
+        # @return [void]
+        def load
+          DebugLog.time("store", "adapter.load") { @adapter.load }
+        end
+
+        # Save memory state to durable storage
+        #
+        # @return [void]
+        def save
+          DebugLog.time("store", "adapter.save") { @adapter.save }
+        end
+
+        # Build working context for a query
+        #
+        # @param query [String] Current user query
+        # @param recent_turns [Array<Hash>] Recent conversation turns
+        # @param system_prompt [String, nil] Agent's system prompt
+        # @param read_only [Boolean] When true, skips recording access on retrieved cards
+        # @return [Array<Hash>] Messages array for LLM
+        def build_context(query:, recent_turns: [], system_prompt: nil, read_only: false)
+          @context_builder.build(
+            query: query,
+            recent_turns: recent_turns,
+            system_prompt: system_prompt,
+            read_only: read_only,
+          )
+        end
+
+        # Ingest a conversation turn into memory
+        #
+        # @param text [String] Full turn text
+        # @param turn_id [String] Unique turn identifier
+        # @return [Array<Card>] Created memory cards
+        def ingest_turn(text:, turn_id:)
+          @ingestion_pipeline.ingest(turn_text: text, turn_id: turn_id)
+        end
+
+        # Search memory for relevant cards
+        #
+        # @param query [String] Search query
+        # @param top_k [Integer] Number of results
+        # @return [Array<Card>] Matching cards
+        def search(query, top_k: 15)
+          @retriever.search(query, top_k: top_k)
+        end
+
+        # Run compression on low-exposure cards
+        #
+        # @param threshold [Float] Exposure score threshold
+        # @return [Integer] Number of cards compressed
+        def compress(threshold: 1.0)
+          @compressor.compress_low_exposure(
+            exposure_tracker: @exposure_tracker,
+            threshold: threshold,
+          )
+        end
+
+        # Run consolidation (dedup, cluster updates)
+        #
+        # @return [Hash] Summary of consolidation actions
+        def consolidate
+          @consolidator.run
+        end
+
+        # Promote compressed cards that have been accessed frequently
+        #
+        # Cards at L1+ with access_count above the threshold get promoted
+        # one level toward L0, rebuilding richer text from graph neighbors.
+        #
+        # @param access_threshold [Integer] Minimum access count for promotion
+        # @return [Integer] Number of cards promoted
+        #
+        # @example
+        #   promoted = store.promote(access_threshold: 5)
+        def promote(access_threshold: 5)
+          @compressor.promote_eligible(access_threshold: access_threshold)
+        end
+
+        # Prune L4 cards with low retention priority
+        #
+        # Retention priority: R = w1*importance + w2*exposure_score + w3*reuse_potential - w4*size
+        # Cards at compression level 4 with R below the threshold are deleted,
+        # along with their edges and cluster references.
+        #
+        # @param threshold [Float] Minimum retention priority to keep (default: 0.3)
+        # @param weights [Hash] Weight parameters for retention formula
+        # @option weights [Float] :importance Weight for card importance (default: 0.4)
+        # @option weights [Float] :exposure Weight for exposure score (default: 0.3)
+        # @option weights [Float] :reuse Weight for reuse potential (default: 0.2)
+        # @option weights [Float] :size Weight for size penalty (default: 0.1)
+        # @return [Integer] Number of cards pruned
+        #
+        # @example Prune with default settings
+        #   pruned = store.prune
+        #
+        # @example Prune with custom threshold
+        #   pruned = store.prune(threshold: 0.5)
+        def prune(threshold: 0.3, weights: {})
+          w = {
+            importance: 0.4,
+            exposure: 0.3,
+            reuse: 0.2,
+            size: 0.1,
+          }.merge(weights)
+
+          l4_cards = @adapter.list_cards.select { |c| c.compression_level == 4 }
+          return 0 if l4_cards.empty?
+
+          pruned = 0
+
+          l4_cards.each do |card|
+            priority = retention_priority(card, w)
+            next if priority >= threshold
+
+            delete_card_with_cleanup(card.id)
+            pruned += 1
+          end
+
+          pruned
+        end
+
+        # Run all memory defragmentation operations
+        #
+        # Consolidates duplicates, compresses low-exposure cards,
+        # promotes frequently-accessed cards, and prunes L4 cards.
+        # Saves state after all operations complete.
+        #
+        # @return [Hash] Summary of defragmentation actions
+        #
+        # @example
+        #   result = store.defrag!
+        #   #=> { duplicates_merged: 0, conflicts_detected: 0,
+        #   #     cards_compressed: 3, cards_promoted: 1, cards_pruned: 0 }
+        def defrag!
+          result = {}
+          start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+          DebugLog.time("store", "defrag!") do
+            # Calculate work items upfront for accurate progress tracking
+            all_cards = @adapter.list_cards
+            all_clusters = @adapter.list_clusters
+
+            # Consolidate phase items
+            dedup_cards = all_cards.select(&:embedding)
+            conflict_types = [:constraint, :decision, :fact]
+            conflict_candidates = all_cards.select { |c| conflict_types.include?(c.type) && c.embedding }
+
+            # Compress phase items (estimate - actual may differ after consolidate)
+            compress_candidates = @exposure_tracker.low_exposure_cards(threshold: 1.0)
+            compress_eligible = compress_candidates.reject { |c| c.compression_level >= 4 }
+
+            # Promote phase items
+            promote_eligible = all_cards.select do |card|
+              card.compression_level >= 1 &&
+                card.access_count >= Configuration.instance.promotion_access_threshold
+            end
+
+            # Prune phase items
+            prune_candidates = all_cards.select { |c| c.compression_level == 4 }
+
+            # Build phase breakdown for SDK user
+            phases = [
+              { name: "consolidate_dedup", items: dedup_cards.size },
+              { name: "consolidate_conflicts", items: conflict_candidates.size },
+              { name: "consolidate_clusters", items: all_clusters.size },
+              { name: "compress", items: compress_eligible.size },
+              { name: "promote", items: promote_eligible.size },
+              { name: "prune", items: prune_candidates.size },
+            ]
+            total_items = phases.sum { |p| p[:items] }
+
+            # Emit start event with full breakdown
+            EventStream.emit(
+              type: "memory_defrag_start",
+              total_items: total_items,
+              total_cards: all_cards.size,
+              total_clusters: all_clusters.size,
+              phases: phases,
+            )
+
+            # Track cumulative progress across phases
+            items_completed = 0
+
+            # Phase 1: Consolidate
+            consolidation = DebugLog.time("store", "consolidate") do
+              @consolidator.run_with_progress(items_completed, total_items)
+            end
+            result[:duplicates_merged] = consolidation[:duplicates_merged]
+            result[:conflicts_detected] = consolidation[:conflicts_detected]
+            result[:clusters_updated] = consolidation[:clusters_updated]
+            items_completed += dedup_cards.size + conflict_candidates.size + all_clusters.size
+
+            # Phase 2: Compress
+            result[:cards_compressed] = DebugLog.time("store", "compress") do
+              compress_with_progress(threshold: 1.0, base_completed: items_completed, total_items: total_items)
+            end
+            items_completed += compress_eligible.size
+
+            # Phase 3: Promote
+            result[:cards_promoted] = DebugLog.time("store", "promote") do
+              promote_with_progress(
+                access_threshold: Configuration.instance.promotion_access_threshold,
+                base_completed: items_completed,
+                total_items: total_items,
+              )
+            end
+            items_completed += promote_eligible.size
+
+            # Phase 4: Prune
+            result[:cards_pruned] = DebugLog.time("store", "prune") do
+              prune_with_progress(base_completed: items_completed, total_items: total_items)
+            end
+
+            # Phase 5: Save
+            save
+          end
+
+          # Emit completion event with summary
+          elapsed_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time) * 1000).round
+          EventStream.emit(
+            type: "memory_defrag_complete",
+            elapsed_ms: elapsed_ms,
+            duplicates_merged: result[:duplicates_merged],
+            conflicts_detected: result[:conflicts_detected],
+            clusters_updated: result[:clusters_updated],
+            cards_compressed: result[:cards_compressed],
+            cards_promoted: result[:cards_promoted],
+            cards_pruned: result[:cards_pruned],
+          )
+
+          result
+        end
+
+        # Get memory statistics
+        #
+        # @return [Hash] Memory stats
+        def stats
+          cards = @adapter.list_cards
+          clusters = @adapter.list_clusters
+
+          {
+            total_cards: cards.size,
+            total_clusters: clusters.size,
+            compression_levels: cards.group_by(&:compression_level).transform_values(&:size),
+            card_types: cards.group_by(&:type).transform_values(&:size),
+          }
+        end
+
+        private
+
+        # Calculate retention priority for a card
+        #
+        # @param card [Card] Card to evaluate
+        # @param weights [Hash] Weight parameters
+        # @return [Float] Retention priority score
+        def retention_priority(card, weights)
+          exposure = @exposure_tracker.exposure_score(card)
+          reuse = reuse_potential(card)
+          size = card.text.split(/\s+/).size / 250.0 # Normalized by max card size
+
+          weights[:importance] * card.importance +
+            weights[:exposure] * [exposure, 1.0].min +
+            weights[:reuse] * reuse -
+            weights[:size] * size
+        end
+
+        # Estimate reuse potential for a card
+        #
+        # Based on how "general" the card is (entity connectivity)
+        # and whether it's a type that tends to be reused (preferences, constraints).
+        #
+        # @param card [Card] Card to evaluate
+        # @return [Float] Reuse potential (0.0-1.0)
+        def reuse_potential(card)
+          # Preferences and constraints tend to be reused
+          type_boost = [:preference, :constraint].include?(card.type) ? 0.3 : 0.0
+
+          # More connected cards (via edges) have higher reuse potential
+          edge_count = @adapter.edges_for(card.id).size
+          connectivity = [edge_count / 5.0, 1.0].min * 0.4
+
+          # More entities = more general/connected
+          entity_score = [card.entities.size / 5.0, 1.0].min * 0.3
+
+          type_boost + connectivity + entity_score
+        end
+
+        # Delete a card and clean up its edges and cluster references
+        #
+        # @param card_id [String] Card ID to delete
+        # @return [void]
+        def delete_card_with_cleanup(card_id)
+          @adapter.delete_edges_for(card_id)
+
+          @adapter.list_clusters.each do |cluster|
+            next unless cluster.card_ids.include?(card_id)
+
+            cluster.remove_card(card_id)
+            @adapter.write_cluster(cluster)
+          end
+
+          @adapter.delete_card(card_id)
+        end
+
+        # Compress low-exposure cards with progress events
+        #
+        # @param threshold [Float] Exposure score threshold
+        # @param base_completed [Integer] Items completed before this phase
+        # @param total_items [Integer] Total items across all phases
+        # @return [Integer] Number of cards compressed
+        def compress_with_progress(threshold:, base_completed:, total_items:)
+          candidates = @exposure_tracker.low_exposure_cards(threshold: threshold)
+          eligible = candidates.reject { |c| c.compression_level >= 4 }
+          return 0 if eligible.empty?
+
+          compressed = 0
+          eligible.each_with_index do |card, index|
+            @compressor.compress_card(card)
+            compressed += 1
+
+            EventStream.emit(
+              type: "memory_defrag_progress",
+              phase: "compress",
+              description: "Compressing low-exposure cards to save space",
+              phase_current: index + 1,
+              phase_total: eligible.size,
+              overall_current: base_completed + index + 1,
+              overall_total: total_items,
+            )
+          end
+
+          compressed
+        end
+
+        # Promote eligible cards with progress events
+        #
+        # @param access_threshold [Integer] Minimum access count for promotion
+        # @param base_completed [Integer] Items completed before this phase
+        # @param total_items [Integer] Total items across all phases
+        # @return [Integer] Number of cards promoted
+        def promote_with_progress(access_threshold:, base_completed:, total_items:)
+          eligible = @adapter.list_cards.select do |card|
+            card.compression_level >= 1 && card.access_count >= access_threshold
+          end
+          return 0 if eligible.empty?
+
+          promoted = 0
+          eligible.each_with_index do |card, index|
+            @compressor.promote_card(card)
+            promoted += 1
+
+            EventStream.emit(
+              type: "memory_defrag_progress",
+              phase: "promote",
+              description: "Promoting frequently-accessed cards to restore detail",
+              phase_current: index + 1,
+              phase_total: eligible.size,
+              overall_current: base_completed + index + 1,
+              overall_total: total_items,
+            )
+          end
+
+          promoted
+        end
+
+        # Prune L4 cards with progress events
+        #
+        # @param threshold [Float] Minimum retention priority to keep
+        # @param weights [Hash] Weight parameters for retention formula
+        # @param base_completed [Integer] Items completed before this phase
+        # @param total_items [Integer] Total items across all phases
+        # @return [Integer] Number of cards pruned
+        def prune_with_progress(threshold: 0.3, weights: {}, base_completed:, total_items:)
+          w = {
+            importance: 0.4,
+            exposure: 0.3,
+            reuse: 0.2,
+            size: 0.1,
+          }.merge(weights)
+
+          l4_cards = @adapter.list_cards.select { |c| c.compression_level == 4 }
+          return 0 if l4_cards.empty?
+
+          pruned = 0
+          l4_cards.each_with_index do |card, index|
+            priority = retention_priority(card, w)
+            if priority < threshold
+              delete_card_with_cleanup(card.id)
+              pruned += 1
+            end
+
+            EventStream.emit(
+              type: "memory_defrag_progress",
+              phase: "prune",
+              description: "Removing low-value compressed cards",
+              phase_current: index + 1,
+              phase_total: l4_cards.size,
+              overall_current: base_completed + index + 1,
+              overall_total: total_items,
+            )
+          end
+
+          pruned
+        end
+      end
+    end
+  end
+end

data/lib/swarm_sdk/v3/skills/loader.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+module SwarmSDK
+  module V3
+    module Skills
+      # Stateless utility for discovering skills and generating XML metadata
+      #
+      # Skills are directories containing a SKILL.md file with YAML frontmatter.
+      # The loader scans configured directories, parses frontmatter, and produces
+      # XML metadata for injection into agent system prompts.
+      #
+      # @example Scan and format skills
+      #   manifests = Loader.scan(["/path/to/skills"])
+      #   xml = Loader.format_xml(manifests)
+      #   # => "<available_skills>\n  <skill>..."
+      module Loader
+        SKILL_FILENAME = "SKILL.md"
+
+        class << self
+          # Scan directories recursively for SKILL.md files and return parsed manifests
+          #
+          # SKILL.md is the definitive marker for a skill directory per the
+          # agentskills.io spec. Uses Dir.glob with `**/SKILL.md` for fast
+          # recursive discovery. Skips missing directories and invalid YAML
+          # silently via DebugLog. Deduplicates by absolute SKILL.md path.
+          #
+          # @param directories [Array<String>] Root paths to scan
+          # @return [Array<Manifest>] Discovered skill manifests
+          #
+          # @example
+          #   manifests = Loader.scan(["/skills", "/more-skills"])
+          #   manifests.each { |m| puts "#{m.name}: #{m.description}" }
+          def scan(directories)
+            seen = {}
+
+            Array(directories).each do |dir|
+              next unless File.directory?(dir)
+
+              Dir.glob(File.join(dir, "**", SKILL_FILENAME)).sort.each do |skill_path|
+                parse_and_register(skill_path, seen)
+              end
+            end
+
+            seen.values
+          end
+
+          # Format manifests as XML for system prompt injection
+          #
+          # Produces an `<available_skills>` XML block following the agentskills.io
+          # standard. Values are HTML-escaped for safety.
+          #
+          # @param manifests [Array<Manifest>] Skill manifests to format
+          # @return [String] XML string, or empty string if no manifests
+          #
+          # @example
+          #   xml = Loader.format_xml(manifests)
+          #   # => "<available_skills>\n  <skill>\n    <name>..."
+          def format_xml(manifests)
+            return "" if manifests.empty?
+
+            entries = manifests.map { |m| format_skill_entry(m) }
+            skills_block = "<available_skills>\n#{entries.join("\n")}\n</available_skills>"
+
+            <<~PROMPT.strip
+              # Skills
+
+              You have skills available. Each skill has a name, description, and a SKILL.md file location.
+              When a user's task matches a skill description, activate the skill by reading its SKILL.md file to get full instructions. Do NOT read SKILL.md files unless you are actually activating a skill to perform a task. The name and description below are sufficient for listing or discussing available skills.
+
+              #{skills_block}
+            PROMPT
+          end
+
+          private
+
+          # Parse a SKILL.md and register the manifest if valid
+          #
+          # @param path [String] Path to SKILL.md
+          # @param seen [Hash<String, Manifest>] Dedup registry
+          # @return [void]
+          def parse_and_register(path, seen)
+            absolute = File.expand_path(path)
+            return if seen.key?(absolute)
+
+            manifest = parse_skill_md(absolute)
+            seen[absolute] = manifest if manifest
+          end
+
+          # Parse a SKILL.md file into a Manifest
+          #
+          # Extracts YAML frontmatter between --- delimiters. Requires
+          # `name` and `description` fields in the frontmatter.
+          #
+          # @param path [String] Absolute path to SKILL.md
+          # @return [Manifest, nil] Parsed manifest or nil on failure
+          def parse_skill_md(path)
+            content = File.read(path)
+            frontmatter = extract_frontmatter(content)
+            return unless frontmatter
+
+            data = YAML.safe_load(frontmatter)
+            return unless data.is_a?(Hash) && data["name"] && data["description"]
+
+            Manifest.new(
+              name: data["name"].to_s,
+              description: data["description"].to_s,
+              location: path,
+            )
+          rescue StandardError => e
+            DebugLog.log("skills", "Failed to parse #{path}: #{e.message}")
+            nil
+          end
+
+          # Extract YAML frontmatter from markdown content
+          #
+          # @param content [String] File content
+          # @return [String, nil] Frontmatter text or nil if not found
+          def extract_frontmatter(content)
+            return unless content.start_with?("---")
+
+            parts = content.split("---", 3)
+            return if parts.length < 3
+
+            frontmatter = parts[1].strip
+            return if frontmatter.empty?
+
+            frontmatter
+          end
+
+          # Format a single skill manifest as an XML entry
+          #
+          # @param manifest [Manifest] Skill manifest
+          # @return [String] Indented XML entry
+          def format_skill_entry(manifest)
+            <<~XML.chomp
+              #{" " * 2}<skill>
+              #{" " * 4}<name>#{CGI.escapeHTML(manifest.name)}</name>
+              #{" " * 4}<description>#{CGI.escapeHTML(manifest.description)}</description>
+              #{" " * 4}<location>#{CGI.escapeHTML(manifest.location)}</location>
+              #{" " * 2}</skill>
+            XML
+          end
+        end
+      end
+    end
+  end
+end

data/lib/swarm_sdk/v3/skills/manifest.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module SwarmSDK
+  module V3
+    module Skills
+      # Immutable value object for parsed SKILL.md frontmatter
+      #
+      # Represents a single discovered skill with its metadata.
+      # Created by {Loader.scan} from SKILL.md files found on disk.
+      #
+      # @example
+      #   manifest = Manifest.new(
+      #     name: "pdf-processing",
+      #     description: "Extract text and tables from PDF files",
+      #     location: "/path/to/pdf-processing/SKILL.md",
+      #   )
+      #   manifest.name        #=> "pdf-processing"
+      #   manifest.description #=> "Extract text and tables from PDF files"
+      #   manifest.location    #=> "/path/to/pdf-processing/SKILL.md"
+      #   manifest.frozen?     #=> true
+      class Manifest
+        # @return [String] Skill name from frontmatter
+        attr_reader :name
+
+        # @return [String] Skill description from frontmatter
+        attr_reader :description
+
+        # @return [String] Absolute path to the SKILL.md file
+        attr_reader :location
+
+        # Create a new skill manifest
+        #
+        # @param name [String] Skill name
+        # @param description [String] Skill description
+        # @param location [String] Absolute path to SKILL.md
+        def initialize(name:, description:, location:)
+          @name = name
+          @description = description
+          @location = location
+          freeze
+        end
+      end
+    end
+  end
+end