claude_swarm 1.0.0 → 1.0.2

data/lib/swarm_memory/chat_extension.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module SwarmMemory
+  # Extension module for SwarmSDK::Agent::Chat
+  #
+  # Adds individual tool removal capability needed for:
+  # 1. Mode-based tool filtering (retrieval/interactive/researcher)
+  # 2. LoadSkill's fine-grained tool swapping
+  #
+  # This is injected into SwarmSDK::Agent::Chat when SwarmMemory is loaded.
+  module ChatExtension
+    # Remove a specific tool by name
+    #
+    # Used by SwarmMemory to filter tools based on memory mode.
+    # Unlike remove_mutable_tools (which removes ALL mutable tools),
+    # this removes a single tool by name.
+    #
+    # @param tool_name [String, Symbol] Tool name to remove
+    # @return [void]
+    def remove_tool(tool_name)
+      tool_sym = tool_name.to_sym
+      tool_str = tool_name.to_s
+
+      # Remove from @tools hash (tools are keyed by symbol)
+      @tools.delete(tool_sym)
+      @tools.delete(tool_str)
+    end
+  end
+end
+
+# Inject into SwarmSDK when both gems are loaded
+if defined?(SwarmSDK::Agent::Chat)
+  SwarmSDK::Agent::Chat.include(SwarmMemory::ChatExtension)
+end

data/lib/swarm_memory/cli/commands.rb

@@ -0,0 +1,306 @@
+# frozen_string_literal: true
+
+module SwarmMemory
+  module CLI
+    # CLI commands for managing SwarmMemory embeddings
+    #
+    # Registers with SwarmCLI to provide:
+    #   swarm memory setup      - Download embedding model
+    #   swarm memory status     - Check model cache status
+    #   swarm memory model-path - Show cache location
+    #   swarm memory defrag     - Optimize memory storage
+    #   swarm memory rebuild    - Rebuild all embeddings
+    class Commands
+      class << self
+        # Execute memory command (called by SwarmCLI)
+        #
+        # @param args [Array<String>] Command arguments (e.g., ["defrag", ".swarm/memory"])
+        # @return [void]
+        def execute(args)
+          subcommand = args.first
+          subcommand_args = args[1..] # Remaining args after subcommand
+
+          case subcommand
+          when "setup"
+            setup_embeddings
+          when "status"
+            show_status
+          when "model-path"
+            show_model_path
+          when "defrag"
+            defrag_memory(subcommand_args)
+          when "rebuild"
+            rebuild_embeddings(subcommand_args)
+          else
+            show_help
+            exit(1)
+          end
+        rescue StandardError => e
+          $stderr.puts "Error: #{e.message}"
+          exit(1)
+        end
+
+        private
+
+        def setup_embeddings
+          puts "Setting up SwarmMemory embeddings..."
+          puts
+
+          begin
+            embedder = SwarmMemory::Embeddings::InformersEmbedder.new
+          rescue StandardError => e
+            $stderr.puts "Error: #{e.message}"
+            $stderr.puts
+            $stderr.puts "Make sure the 'informers' gem is installed:"
+            $stderr.puts "  gem install informers"
+            exit(1)
+          end
+
+          model_name = ENV["SWARM_MEMORY_EMBEDDING_MODEL"] || "sentence-transformers/multi-qa-MiniLM-L6-cos-v1"
+
+          if embedder.cached?
+            puts "✓ Model already cached!"
+            puts "  Model: #{model_name}"
+            puts "  Location: #{Informers.cache_dir}/#{model_name}/"
+            puts
+            puts "No download needed. Embeddings ready to use."
+          else
+            puts "Model not cached. Downloading..."
+            puts "  Model: #{model_name}"
+            puts "  Size: ~90MB (unquantized ONNX)"
+            puts "  Location: #{Informers.cache_dir}"
+            puts
+            puts "This is a one-time download. Please wait..."
+            puts
+
+            embedder.preload!
+
+            puts
+            puts "✓ Setup complete!"
+            puts "  Model: #{model_name}"
+            puts "  Model cached and ready to use."
+            puts "  Semantic search is now available."
+          end
+
+          exit(0)
+        end
+
+        def show_status
+          begin
+            embedder = SwarmMemory::Embeddings::InformersEmbedder.new
+          rescue StandardError => e
+            $stderr.puts "Error: #{e.message}"
+            $stderr.puts
+            $stderr.puts "Make sure the 'informers' gem is installed:"
+            $stderr.puts "  gem install informers"
+            exit(1)
+          end
+
+          model_name = ENV["SWARM_MEMORY_EMBEDDING_MODEL"] || "sentence-transformers/multi-qa-MiniLM-L6-cos-v1"
+
+          puts "SwarmMemory Embedding Status"
+          puts "=" * 50
+          puts
+
+          if embedder.cached?
+            puts "Status: ✓ Model cached"
+            puts "Model: #{model_name}"
+            puts "Dimensions: #{embedder.dimensions}"
+            puts "Cache: #{Informers.cache_dir}"
+            puts
+            puts "Semantic search is available for memory defragmentation."
+          else
+            puts "Status: ✗ Model not cached"
+            puts "Model: #{model_name}"
+            puts "Dimensions: #{embedder.dimensions}"
+            puts
+            puts "Run 'swarm memory setup' to download the model."
+            puts "Or it will download automatically on first use."
+          end
+
+          exit(0)
+        end
+
+        def show_model_path
+          puts Informers.cache_dir
+          exit(0)
+        end
+
+        def defrag_memory(args)
+          # Expect directory path as argument
+          directory = args&.first
+
+          unless directory && !directory.empty?
+            $stderr.puts "Error: Memory directory path required"
+            $stderr.puts
+            $stderr.puts "Usage: swarm memory defrag DIRECTORY"
+            $stderr.puts
+            $stderr.puts "Example:"
+            $stderr.puts "  swarm memory defrag .swarm/assistant-memory"
+            exit(1)
+          end
+
+          unless Dir.exist?(directory)
+            $stderr.puts "Error: Directory not found: #{directory}"
+            exit(1)
+          end
+
+          puts "Defragmenting memory at: #{directory}"
+          puts "=" * 70
+          puts
+
+          # Create storage
+          adapter = SwarmMemory::Adapters::FilesystemAdapter.new(directory: directory)
+          storage = SwarmMemory::Core::Storage.new(adapter: adapter)
+
+          # Create defrag tool
+          defrag = SwarmMemory::Tools::MemoryDefrag.new(storage: storage)
+
+          # Run full analysis
+          puts "Running full defrag analysis..."
+          puts
+          result = defrag.execute(action: "full", dry_run: false)
+          puts result
+
+          exit(0)
+        end
+
+        def rebuild_embeddings(args)
+          # Expect directory path as argument
+          directory = args&.first
+
+          unless directory && !directory.empty?
+            $stderr.puts "Error: Memory directory path required"
+            $stderr.puts
+            $stderr.puts "Usage: swarm memory rebuild DIRECTORY"
+            $stderr.puts
+            $stderr.puts "Example:"
+            $stderr.puts "  swarm memory rebuild .swarm/assistant-memory"
+            exit(1)
+          end
+
+          unless Dir.exist?(directory)
+            $stderr.puts "Error: Directory not found: #{directory}"
+            exit(1)
+          end
+
+          puts "Rebuilding embeddings for memory at: #{directory}"
+          puts "=" * 70
+          puts
+
+          # Initialize embedder
+          begin
+            embedder = SwarmMemory::Embeddings::InformersEmbedder.new
+          rescue StandardError => e
+            $stderr.puts "Error: #{e.message}"
+            $stderr.puts
+            $stderr.puts "Make sure the 'informers' gem is installed:"
+            $stderr.puts "  gem install informers"
+            exit(1)
+          end
+
+          # Ensure model is cached
+          unless embedder.cached?
+            puts "Model not cached. Downloading..."
+            puts "  Model: sentence-transformers/all-MiniLM-L6-v2"
+            puts "  Size: ~90MB (unquantized ONNX)"
+            puts
+            embedder.preload!
+            puts
+          end
+
+          # Create storage with embedder
+          adapter = SwarmMemory::Adapters::FilesystemAdapter.new(directory: directory)
+          storage = SwarmMemory::Core::Storage.new(adapter: adapter, embedder: embedder)
+
+          # Get all entries
+          all_entries = adapter.all_entries
+          total_count = all_entries.size
+
+          if total_count.zero?
+            puts "No entries found in #{directory}"
+            exit(0)
+          end
+
+          puts "Found #{total_count} entries to rebuild"
+          puts
+
+          # Rebuild each entry
+          processed = 0
+          errors = 0
+
+          all_entries.each do |path, entry|
+            # Re-write the entry to regenerate embedding
+            # The storage.write() method will automatically generate the embedding
+            storage.write(
+              file_path: path,
+              content: entry.content,
+              title: entry.title,
+              metadata: entry.metadata,
+              generate_embedding: true,
+            )
+
+            processed += 1
+            print("\rProcessed: #{processed}/#{total_count} (#{errors} errors)")
+          rescue StandardError => e
+            errors += 1
+            print("\rProcessed: #{processed}/#{total_count} (#{errors} errors)")
+            warn("\nError rebuilding #{path}: #{e.message}")
+          end
+
+          puts
+          puts
+          puts "Rebuild complete!"
+          puts "  Total entries: #{total_count}"
+          puts "  Successfully rebuilt: #{processed}"
+          puts "  Errors: #{errors}" if errors.positive?
+          puts
+
+          exit(0)
+        end
+
+        def show_help
+          puts
+          puts "Usage: swarm memory SUBCOMMAND"
+          puts
+          puts "Subcommands:"
+          puts "  setup              Setup embeddings (download model ~90MB, one-time)"
+          puts "  status             Check if embeddings are ready"
+          puts "  model-path         Show embedding model cache path"
+          puts "  defrag DIRECTORY   Defrag memory at given directory"
+          puts "  rebuild DIRECTORY  Rebuild all embeddings for memory at directory"
+          puts
+          puts "Environment Variables:"
+          puts "  SWARM_MEMORY_EMBEDDING_MODEL              Model to use (default: all-MiniLM-L6-v2)"
+          puts "                                            Options: all-MiniLM-L6-v2, multi-qa-MiniLM-L6-cos-v1"
+          puts "  SWARM_MEMORY_EMBEDDING_MAX_CHARS          Max chars to embed (default: 300, -1: unlimited)"
+          puts
+          puts "  Adaptive Thresholds (short queries use lower threshold):"
+          puts "  SWARM_MEMORY_DISCOVERY_THRESHOLD          Normal query threshold (default: 0.35)"
+          puts "  SWARM_MEMORY_DISCOVERY_THRESHOLD_SHORT    Short query threshold (default: 0.25)"
+          puts "  SWARM_MEMORY_ADAPTIVE_WORD_CUTOFF         Word count cutoff (default: 10)"
+          puts "                                            Queries < 10 words use short threshold"
+          puts
+          puts "  SWARM_MEMORY_SEMANTIC_WEIGHT              Semantic weight (default: 0.5)"
+          puts "  SWARM_MEMORY_KEYWORD_WEIGHT               Keyword weight (default: 0.5)"
+          puts
+          puts "Examples:"
+          puts "  swarm memory setup                            # Download model"
+          puts "  swarm memory status                           # Check if ready"
+          puts "  swarm memory model-path                       # Show model location"
+          puts "  swarm memory defrag .swarm/assistant-memory   # Optimize memory"
+          puts "  swarm memory rebuild .swarm/assistant-memory  # Rebuild embeddings"
+          puts
+          puts "  # Use Q&A-optimized model"
+          puts "  SWARM_MEMORY_EMBEDDING_MODEL=sentence-transformers/multi-qa-MiniLM-L6-cos-v1 \\"
+          puts "    swarm memory setup"
+          puts
+          puts "  # Rebuild with more content (850 chars)"
+          puts "  SWARM_MEMORY_EMBEDDING_MAX_CHARS=850 \\"
+          puts "    swarm memory rebuild .swarm/assistant-memory"
+          puts
+        end
+      end
+    end
+  end
+end

data/lib/swarm_memory/core/entry.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module SwarmMemory
+  module Core
+    # Represents a single memory entry with metadata and optional embedding
+    #
+    # @attr content [String] The actual content stored
+    # @attr title [String] Brief description of the content
+    # @attr updated_at [Time] Last modification timestamp
+    # @attr size [Integer] Content size in bytes
+    # @attr embedding [Array<Float>, nil] Optional 384-dim embedding vector
+    # @attr metadata [Hash, nil] Optional parsed frontmatter metadata
+    Entry = Struct.new(
+      :content,
+      :title,
+      :updated_at,
+      :size,
+      :embedding,
+      :metadata,
+      keyword_init: true,
+    ) do
+      # Check if entry has an embedding
+      #
+      # @return [Boolean]
+      def embedded?
+        !embedding.nil? && !embedding.empty?
+      end
+
+      # Check if entry has metadata
+      #
+      # @return [Boolean]
+      def has_metadata?
+        !metadata.nil? && !metadata.empty?
+      end
+    end
+  end
+end

data/lib/swarm_memory/core/frontmatter_parser.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+module SwarmMemory
+  module Core
+    # Parser for YAML frontmatter in memory entries
+    #
+    # Parses markdown content with YAML frontmatter:
+    #   ---
+    #   type: concept
+    #   confidence: high
+    #   tags: [ruby, testing]
+    #   ---
+    #
+    #   # Title
+    #   Content here...
+    class FrontmatterParser
+      # Regex pattern to match frontmatter (same as MarkdownParser)
+      FRONTMATTER_PATTERN = /\A---\s*\n(.*?)\n---\s*\n(.*)\z/m
+
+      class << self
+        # Parse content and extract frontmatter
+        #
+        # @param content [String] Full entry content
+        # @return [Hash] { frontmatter: Hash, body: String, error: nil|String }
+        #
+        # @example
+        #   parsed = FrontmatterParser.parse("---\ntype: fact\n---\nContent")
+        #   parsed[:frontmatter] # => { type: "fact" }
+        #   parsed[:body] # => "Content"
+        def parse(content)
+          return { frontmatter: {}, body: content, error: nil } if content.nil? || content.empty?
+
+          if content =~ FRONTMATTER_PATTERN
+            frontmatter_yaml = Regexp.last_match(1)
+            body = Regexp.last_match(2)
+
+            begin
+              frontmatter = YAML.safe_load(frontmatter_yaml, permitted_classes: [Symbol, Date, Time], aliases: true)
+              frontmatter = symbolize_keys(frontmatter) if frontmatter.is_a?(Hash)
+              { frontmatter: frontmatter || {}, body: body, error: nil }
+            rescue StandardError => e
+              # If YAML parsing fails, treat as body without frontmatter
+              { frontmatter: {}, body: content, error: e.message }
+            end
+          else
+            # No frontmatter
+            { frontmatter: {}, body: content, error: nil }
+          end
+        end
+
+        # Extract specific metadata fields from frontmatter
+        #
+        # @param content [String] Full entry content
+        # @return [Hash] Extracted metadata fields
+        #
+        # @example
+        #   metadata = FrontmatterParser.extract_metadata(content)
+        #   metadata[:confidence] # => "high"
+        #   metadata[:type] # => "concept"
+        def extract_metadata(content)
+          parsed = parse(content)
+          fm = parsed[:frontmatter]
+
+          {
+            confidence: fm[:confidence]&.to_s&.downcase, # "high", "medium", "low"
+            type: fm[:type]&.to_s&.downcase, # "concept", "fact", "skill", "experience"
+            tags: Array(fm[:tags] || []),
+            last_verified: parse_date(fm[:last_verified]),
+            related: Array(fm[:related] || []),
+            domain: fm[:domain]&.to_s,
+            source: fm[:source]&.to_s,
+          }
+        end
+
+        private
+
+        # Parse date from various formats
+        #
+        # @param value [String, Date, Time, nil] Date value
+        # @return [Date, nil]
+        def parse_date(value)
+          return if value.nil?
+          return value.to_date if value.is_a?(Time)
+          return value if value.is_a?(Date)
+
+          Date.parse(value.to_s)
+        rescue ArgumentError
+          nil
+        end
+
+        # Recursively symbolize hash keys
+        #
+        # @param obj [Object] Object to symbolize
+        # @return [Object] Object with symbolized keys
+        def symbolize_keys(obj)
+          case obj
+          when Hash
+            obj.transform_keys(&:to_sym).transform_values { |v| symbolize_keys(v) }
+          when Array
+            obj.map { |item| symbolize_keys(item) }
+          else
+            obj
+          end
+        end
+      end
+    end
+  end
+end

data/lib/swarm_memory/core/metadata_extractor.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module SwarmMemory
+  module Core
+    # Extracts structured metadata from memory entries
+    #
+    # This class wraps FrontmatterParser and provides additional
+    # metadata extraction logic specific to the memory system.
+    class MetadataExtractor
+      class << self
+        # Extract all metadata from a memory entry
+        #
+        # @param content [String] Full entry content
+        # @return [Hash] Extracted metadata
+        #
+        # @example
+        #   metadata = MetadataExtractor.extract(content)
+        #   metadata[:confidence] # => "high"
+        #   metadata[:type] # => "concept"
+        #   metadata[:tags] # => ["ruby", "testing"]
+        def extract(content)
+          FrontmatterParser.extract_metadata(content)
+        end
+
+        # Check if entry has required frontmatter for quality
+        #
+        # @param content [String] Full entry content
+        # @return [Boolean] True if entry has basic required fields
+        def has_required_frontmatter?(content)
+          metadata = extract(content)
+          !metadata[:type].nil? && !metadata[:confidence].nil?
+        end
+
+        # Calculate entry quality score (0-100)
+        #
+        # @param content [String] Full entry content
+        # @return [Integer] Quality score
+        def quality_score(content)
+          metadata = extract(content)
+          score = 0
+
+          # Has type (20 points)
+          score += 20 if metadata[:type]
+
+          # Has confidence (20 points)
+          score += 20 if metadata[:confidence]
+
+          # Has tags (15 points)
+          score += 15 unless metadata[:tags].empty?
+
+          # Has related links (15 points)
+          score += 15 unless metadata[:related].empty?
+
+          # Has domain (10 points)
+          score += 10 if metadata[:domain]
+
+          # Has last_verified (10 points)
+          score += 10 if metadata[:last_verified]
+
+          # High confidence bonus (10 points)
+          score += 10 if metadata[:confidence] == "high"
+
+          score
+        end
+      end
+    end
+  end
+end

data/lib/swarm_memory/core/path_normalizer.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module SwarmMemory
+  module Core
+    # Validates and normalizes memory paths
+    #
+    # Ensures paths are safe, hierarchical, and follow conventions.
+    class PathNormalizer
+      # Invalid path patterns
+      INVALID_PATTERNS = [
+        %r{\A/}, # Absolute paths
+        /\.\./,          # Parent directory references
+        %r{//},          # Double slashes
+        /\A\s/,          # Leading whitespace
+        /\s\z/,          # Trailing whitespace
+        /[<>:"|?*]/,     # Invalid filesystem characters
+      ].freeze
+
+      class << self
+        # Normalize and validate a memory path
+        #
+        # @param path [String] Path to normalize
+        # @return [String] Normalized path
+        # @raise [ArgumentError] If path is invalid
+        #
+        # @example
+        #   PathNormalizer.normalize("concepts/ruby/classes.md")
+        #   # => "concepts/ruby/classes.md"
+        #
+        #   PathNormalizer.normalize("../secrets")
+        #   # => ArgumentError: Path cannot contain '..'
+        def normalize(path)
+          raise ArgumentError, "path is required" if path.nil? || path.to_s.strip.empty?
+
+          original_path = path.to_s.strip
+
+          # Check for absolute paths and parent references FIRST (before normalization)
+          if original_path.start_with?("/")
+            raise ArgumentError, "Invalid path: #{original_path}. Paths must be relative, hierarchical, and safe."
+          end
+
+          if original_path.include?("..")
+            raise ArgumentError, "Invalid path: #{original_path}. Paths must be relative, hierarchical, and safe."
+          end
+
+          # Normalize (remove leading/trailing slashes, collapse doubles)
+          path = original_path
+          path = path.sub(%r{\A/+}, "")  # Remove leading slashes
+          path = path.sub(%r{/+\z}, "")  # Remove trailing slashes
+          path = path.gsub(%r{/+}, "/")  # Collapse multiple slashes
+
+          # Check for other invalid characters
+          if path.match?(/[<>:"|?*]/)
+            raise ArgumentError, "Invalid path: #{original_path}. Paths must be relative, hierarchical, and safe."
+          end
+
+          raise ArgumentError, "Normalized path is empty" if path.empty?
+
+          path
+        end
+
+        # Check if a path is valid without raising an exception
+        #
+        # @param path [String] Path to validate
+        # @return [Boolean] True if path is valid
+        def valid?(path)
+          normalize(path)
+          true
+        rescue ArgumentError
+          false
+        end
+      end
+    end
+  end
+end