claude_swarm 1.0.0 → 1.0.2

data/lib/swarm_memory/core/semantic_index.rb

@@ -0,0 +1,244 @@
+# frozen_string_literal: true
+
+module SwarmMemory
+  module Core
+    # Semantic search abstraction layer
+    #
+    # Provides embedding computation and semantic search operations
+    # that work with any storage adapter. Easily replaceable with
+    # vector database implementations (Qdrant, Milvus, Chroma, etc.)
+    #
+    # Uses hybrid search: combines semantic similarity with keyword matching
+    # for better recall accuracy.
+    #
+    # @example
+    #   index = SemanticIndex.new(adapter: adapter, embedder: embedder)
+    #   results = index.search(query: "how to debug", top_k: 5, threshold: 0.7)
+    class SemanticIndex
+      # Default weights for hybrid scoring (optimal: 50/50 discovered via systematic evaluation)
+      # Configurable via ENV vars: SWARM_MEMORY_SEMANTIC_WEIGHT, SWARM_MEMORY_KEYWORD_WEIGHT
+      DEFAULT_SEMANTIC_WEIGHT = (ENV["SWARM_MEMORY_SEMANTIC_WEIGHT"] || "0.5").to_f
+      DEFAULT_KEYWORD_WEIGHT = (ENV["SWARM_MEMORY_KEYWORD_WEIGHT"] || "0.5").to_f
+
+      # @param adapter [Adapters::Base] Storage adapter
+      # @param embedder [Embeddings::Embedder] Embedding model
+      # @param semantic_weight [Float] Weight for semantic similarity (0.0-1.0)
+      # @param keyword_weight [Float] Weight for keyword matching (0.0-1.0)
+      def initialize(adapter:, embedder:, semantic_weight: DEFAULT_SEMANTIC_WEIGHT, keyword_weight: DEFAULT_KEYWORD_WEIGHT)
+        @adapter = adapter
+        @embedder = embedder
+        @semantic_weight = semantic_weight
+        @keyword_weight = keyword_weight
+      end
+
+      # Compute embedding for text
+      #
+      # @param text [String] Text to embed
+      # @return [Array<Float>] Embedding vector
+      def compute_embedding(text)
+        @embedder.embed(text)
+      end
+
+      # Semantic search by text query
+      #
+      # @param query [String] Search query
+      # @param top_k [Integer] Number of results to return
+      # @param threshold [Float] Minimum similarity score (0.0-1.0)
+      # @param filter [Hash, nil] Optional metadata filters (e.g., { "type" => "skill" })
+      # @return [Array<Hash>] Results with similarity scores, sorted by similarity descending
+      #
+      # @example
+      #   results = index.search(
+      #     query: "how to create a swarm",
+      #     top_k: 3,
+      #     threshold: 0.65,
+      #     filter: { "type" => "skill" }
+      #   )
+      #
+      #   results.each do |result|
+      #     puts "#{result[:path]} (#{result[:similarity]})"
+      #     puts result[:title]
+      #   end
+      def search(query:, top_k: 10, threshold: 0.0, filter: nil)
+        # Extract keywords from query for keyword matching
+        query_keywords = extract_keywords(query)
+
+        # Compute query embedding
+        query_embedding = compute_embedding(query)
+
+        # Delegate to adapter-specific search (gets semantic similarity only)
+        # Use threshold of 0.0 to get all results, we'll filter after hybrid scoring
+        results = @adapter.semantic_search(
+          embedding: query_embedding,
+          top_k: top_k * 3, # Get extra for reranking
+          threshold: 0.0,   # No threshold yet - will apply after hybrid scoring
+        )
+
+        # Calculate hybrid scores (semantic + keyword)
+        results = calculate_hybrid_scores(results, query_keywords)
+
+        # Apply metadata filters if provided
+        results = apply_filters(results, filter) if filter
+
+        # Filter by threshold on hybrid score
+        results = results.select { |r| r[:similarity] >= threshold }
+
+        # Return top K after filtering and reranking
+        results.take(top_k)
+      end
+
+      # Find similar entries by embedding vector
+      #
+      # @param embedding [Array<Float>] Embedding vector
+      # @param top_k [Integer] Number of results to return
+      # @param threshold [Float] Minimum similarity score (0.0-1.0)
+      # @param filter [Hash, nil] Optional metadata filters
+      # @return [Array<Hash>] Similar entries sorted by similarity descending
+      def find_similar(embedding:, top_k: 10, threshold: 0.0, filter: nil)
+        results = @adapter.semantic_search(
+          embedding: embedding,
+          top_k: top_k * 2,
+          threshold: threshold,
+        )
+
+        # Apply metadata filters if provided
+        results = apply_filters(results, filter) if filter
+
+        # Return top K after filtering
+        results.take(top_k)
+      end
+
+      private
+
+      # Extract keywords from query text
+      #
+      # Removes common words and punctuation, lowercases everything.
+      #
+      # @param text [String] Query text
+      # @return [Array<String>] Extracted keywords
+      def extract_keywords(text)
+        # Common stop words to ignore
+        stop_words = [
+          "a",
+          "an",
+          "and",
+          "are",
+          "as",
+          "at",
+          "be",
+          "by",
+          "for",
+          "from",
+          "has",
+          "have",
+          "in",
+          "is",
+          "it",
+          "of",
+          "on",
+          "that",
+          "the",
+          "this",
+          "to",
+          "was",
+          "will",
+          "with",
+          "how",
+          "what",
+          "when",
+          "where",
+          "why",
+          "who",
+          "which",
+          "do",
+          "does",
+          "did",
+          "can",
+          "could",
+          "should",
+          "would",
+          "may",
+          "might",
+          "must",
+          "me",
+          "my",
+          "you",
+          "your",
+          "we",
+          "us",
+          "our",
+        ]
+
+        # Extract words (lowercase, alphanumeric only)
+        words = text.downcase
+          .gsub(/[^a-z0-9\s\-]/, " ") # Remove punctuation except hyphens
+          .split(/\s+/)
+          .reject { |w| w.length < 2 } # Skip single chars
+          .reject { |w| stop_words.include?(w) } # Skip stop words
+
+        words.uniq
+      end
+
+      # Calculate hybrid scores combining semantic similarity and keyword matching
+      #
+      # @param results [Array<Hash>] Results with semantic :similarity scores
+      # @param query_keywords [Array<String>] Keywords from query
+      # @return [Array<Hash>] Results with updated :similarity (hybrid score) and debug info
+      def calculate_hybrid_scores(results, query_keywords)
+        results.map do |result|
+          semantic_score = result[:similarity]
+          keyword_score = calculate_keyword_score(result, query_keywords)
+
+          # Hybrid score: weighted combination
+          hybrid_score = (@semantic_weight * semantic_score) + (@keyword_weight * keyword_score)
+
+          # Update result with hybrid score and debug info
+          result.merge(
+            similarity: hybrid_score,
+            semantic_score: semantic_score,
+            keyword_score: keyword_score,
+          )
+        end.sort_by { |r| -r[:similarity] }
+      end
+
+      # Calculate keyword matching score based on tag overlap
+      #
+      # @param result [Hash] Search result with :metadata containing tags
+      # @param query_keywords [Array<String>] Keywords from query
+      # @return [Float] Keyword score (0.0-1.0)
+      def calculate_keyword_score(result, query_keywords)
+        return 0.0 if query_keywords.empty?
+
+        # Get tags from metadata
+        tags = result.dig(:metadata, "tags") || result.dig(:metadata, :tags) || []
+        return 0.0 if tags.empty?
+
+        # Normalize tags to lowercase
+        normalized_tags = tags.map(&:downcase)
+
+        # Count keyword matches (fuzzy matching - substring or contains)
+        matches = query_keywords.count do |keyword|
+          normalized_tags.any? { |tag| tag.include?(keyword) || keyword.include?(tag) }
+        end
+
+        # Normalize to 0-1 scale
+        # Use min(query_keywords.size, 5) as denominator to avoid penalizing long queries
+        denominator = [query_keywords.size, 5].min
+        matches.to_f / denominator
+      end
+
+      # Apply metadata filters to results
+      #
+      # @param results [Array<Hash>] Search results
+      # @param filter [Hash] Metadata filters
+      # @return [Array<Hash>] Filtered results
+      def apply_filters(results, filter)
+        results.select do |result|
+          filter.all? do |key, value|
+            result.dig(:metadata, key) == value || result.dig(:metadata, key.to_s) == value
+          end
+        end
+      end
+    end
+  end
+end

data/lib/swarm_memory/core/storage.rb

@@ -0,0 +1,288 @@
+# frozen_string_literal: true
+
+module SwarmMemory
+  module Core
+    # High-level storage orchestration
+    #
+    # Coordinates adapter operations, path normalization, embedding generation,
+    # and metadata extraction.
+    #
+    # @example
+    #   adapter = Adapters::FilesystemAdapter.new(persist_to: ".swarm/memory.json")
+    #   storage = Storage.new(adapter: adapter)
+    #   storage.write(file_path: "concepts/ruby", content: "...", title: "Ruby Classes")
+    class Storage
+      attr_reader :adapter
+
+      # Initialize storage with an adapter
+      #
+      # @param adapter [Adapters::Base] Storage adapter
+      # @param embedder [Embeddings::Embedder, nil] Optional embedder for semantic search
+      def initialize(adapter:, embedder: nil)
+        raise ArgumentError, "adapter is required" unless adapter.is_a?(Adapters::Base)
+
+        @adapter = adapter
+        @embedder = embedder
+
+        # Create semantic index if embedder is provided
+        @semantic_index = if embedder
+          SemanticIndex.new(adapter: adapter, embedder: embedder)
+        end
+      end
+
+      # Get semantic index for semantic search operations
+      #
+      # @return [SemanticIndex, nil] Semantic index instance or nil if no embedder
+      attr_reader :semantic_index
+
+      # Write content to storage
+      #
+      # @param file_path [String] Path to store content (with .md extension)
+      # @param content [String] Content to store (pure markdown)
+      # @param title [String] Brief title
+      # @param metadata [Hash, nil] Optional metadata
+      # @param generate_embedding [Boolean] Whether to generate embedding (default: true if embedder present)
+      # @return [Entry] The created entry
+      def write(file_path:, content:, title:, metadata: nil, generate_embedding: nil)
+        # Normalize path
+        normalized_path = PathNormalizer.normalize(file_path)
+
+        # Generate embedding if requested and embedder available
+        embedding = nil
+        should_embed = generate_embedding.nil? ? !@embedder.nil? : generate_embedding
+
+        if should_embed && @embedder
+          begin
+            # Build searchable text for better semantic matching
+            # Uses title + tags + first paragraph instead of full content
+            searchable_text = build_searchable_text(content, title, metadata)
+
+            # ALWAYS emit to LogStream (create if needed for debugging)
+            # This ensures we can see what's being embedded
+            begin
+              if defined?(SwarmSDK::LogStream)
+                SwarmSDK::LogStream.emit(
+                  type: "memory_embedding_generated",
+                  file_path: normalized_path,
+                  title: title,
+                  searchable_text_length: searchable_text.length,
+                  searchable_text_preview: searchable_text.slice(0, 300),
+                  full_searchable_text: searchable_text,
+                  metadata_tags: metadata&.dig("tags"),
+                  metadata_domain: metadata&.dig("domain"),
+                )
+              end
+            rescue StandardError => e
+              # Don't fail if logging fails
+              warn("Failed to log embedding: #{e.message}")
+            end
+
+            embedding = @embedder.embed(searchable_text)
+          rescue StandardError => e
+            # Don't fail write if embedding generation fails
+            warn("Warning: Failed to generate embedding for #{normalized_path}: #{e.message}")
+            embedding = nil
+          end
+        end
+
+        # Write to adapter (metadata passed from tool parameters)
+        @adapter.write(
+          file_path: normalized_path,
+          content: content,
+          title: title,
+          embedding: embedding,
+          metadata: metadata,
+        )
+      end
+
+      # Read content from storage
+      #
+      # @param file_path [String] Path to read from
+      # @return [String] Content at the path
+      def read(file_path:)
+        normalized_path = PathNormalizer.normalize(file_path)
+        @adapter.read(file_path: normalized_path)
+      end
+
+      # Read full entry with metadata
+      #
+      # @param file_path [String] Path to read from
+      # @return [Entry] Full entry object
+      def read_entry(file_path:)
+        normalized_path = PathNormalizer.normalize(file_path)
+        @adapter.read_entry(file_path: normalized_path)
+      end
+
+      # Delete an entry
+      #
+      # @param file_path [String] Path to delete
+      # @return [void]
+      def delete(file_path:)
+        normalized_path = PathNormalizer.normalize(file_path)
+        @adapter.delete(file_path: normalized_path)
+      end
+
+      # List all entries
+      #
+      # @param prefix [String, nil] Optional prefix filter
+      # @return [Array<Hash>] Entry metadata
+      def list(prefix: nil)
+        @adapter.list(prefix: prefix)
+      end
+
+      # Search by glob pattern
+      #
+      # @param pattern [String] Glob pattern
+      # @return [Array<Hash>] Matching entries
+      def glob(pattern:)
+        @adapter.glob(pattern: pattern)
+      end
+
+      # Search by content pattern
+      #
+      # @param pattern [String] Regex pattern
+      # @param case_insensitive [Boolean] Case-insensitive search
+      # @param output_mode [String] Output mode
+      # @param path [String, nil] Optional path prefix filter
+      # @return [Array<Hash>] Search results
+      def grep(pattern:, case_insensitive: false, output_mode: "files_with_matches", path: nil)
+        @adapter.grep(
+          pattern: pattern,
+          case_insensitive: case_insensitive,
+          output_mode: output_mode,
+          path: path,
+        )
+      end
+
+      # Clear all entries
+      #
+      # @return [void]
+      def clear
+        @adapter.clear
+      end
+
+      # Get total storage size
+      #
+      # @return [Integer] Size in bytes
+      def total_size
+        @adapter.total_size
+      end
+
+      # Get number of entries
+      #
+      # @return [Integer] Entry count
+      def size
+        @adapter.size
+      end
+
+      # Get all entries (for optimization/analysis)
+      #
+      # @return [Hash<String, Entry>] All entries
+      def all_entries
+        @adapter.all_entries
+      end
+
+      private
+
+      # Build searchable text for embedding
+      #
+      # Creates a condensed representation optimized for semantic search.
+      # Uses title + tags + first paragraph instead of full content.
+      #
+      # @param content [String] Full entry content
+      # @param title [String] Entry title
+      # @param metadata [Hash, nil] Entry metadata
+      # @return [String] Searchable text for embedding
+      def build_searchable_text(content, title, metadata)
+        parts = []
+
+        # 1. Title (most important for matching)
+        parts << "Title: #{title}"
+
+        # 2. Tags (critical keywords that users would search for)
+        if metadata && metadata["tags"]&.any?
+          parts << "Tags: #{metadata["tags"].join(", ")}"
+        end
+
+        # 3. Domain (additional context)
+        if metadata && metadata["domain"]
+          parts << "Domain: #{metadata["domain"]}"
+        end
+
+        # 4. First paragraph (summary/description)
+        first_para = extract_first_paragraph(content)
+        parts << "Summary: #{first_para}" if first_para
+
+        parts.join("\n\n")
+      end
+
+      # Extract first meaningful paragraph from content
+      #
+      # Skips YAML frontmatter, headings, and empty lines to find
+      # the first substantive paragraph.
+      #
+      # Character limit can be controlled via SWARM_MEMORY_EMBEDDING_MAX_CHARS:
+      # - Default: 300
+      # - -1: Unlimited (use full content)
+      # - Any positive number: Custom limit
+      #
+      # @param content [String] Full content
+      # @return [String, nil] First paragraph (max chars based on env var) or nil
+      def extract_first_paragraph(content)
+        return if content.nil? || content.strip.empty?
+
+        # Get character limit from environment variable
+        # Default: 300, -1 = unlimited
+        max_chars = (ENV["SWARM_MEMORY_EMBEDDING_MAX_CHARS"] || "1200").to_i
+        unlimited = max_chars == -1
+
+        lines = content.lines
+
+        # Skip YAML frontmatter (--- ... ---)
+        in_frontmatter = false
+        lines = lines.drop_while do |line|
+          if line.strip == "---"
+            in_frontmatter = !in_frontmatter
+            true
+          else
+            in_frontmatter
+          end
+        end
+
+        # Find first non-heading, non-empty paragraph(s)
+        paragraph = []
+        lines.each do |line|
+          stripped = line.strip
+
+          # Skip empty lines
+          next if stripped.empty?
+
+          # Stop if we hit a heading after collecting some text (unless unlimited)
+          if stripped.start_with?("#") && paragraph.any? && !unlimited
+            break
+          end
+
+          # Skip headings
+          next if stripped.start_with?("#")
+
+          # Skip code blocks
+          next if stripped.start_with?("```")
+
+          # Add line to paragraph
+          paragraph << stripped
+
+          # Stop if we have enough text (unless unlimited)
+          unless unlimited
+            break if paragraph.join(" ").length > (max_chars - 100)
+          end
+        end
+
+        return if paragraph.empty?
+
+        # Join and cap at max_chars (or don't cap if unlimited)
+        text = paragraph.join(" ")
+        unlimited ? text : text.slice(0, max_chars)
+      end
+    end
+  end
+end

data/lib/swarm_memory/core/storage_read_tracker.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module SwarmMemory
+  module Core
+    # StorageReadTracker manages read-entry tracking for all agents
+    #
+    # This module maintains a global registry of which memory entries each agent
+    # has read during their conversation. This enables enforcement of the
+    # "read-before-edit" rule that ensures agents have context before modifying entries.
+    #
+    # Each agent maintains an independent set of read entries, keyed by agent identifier.
+    module StorageReadTracker
+      @read_entries = {}
+      @mutex = Mutex.new
+
+      class << self
+        # Register that an agent has read a storage entry
+        #
+        # @param agent_id [Symbol] The agent identifier
+        # @param entry_path [String] The storage entry path
+        # @return [void]
+        def register_read(agent_id, entry_path)
+          @mutex.synchronize do
+            @read_entries[agent_id] ||= Set.new
+            @read_entries[agent_id] << entry_path
+          end
+        end
+
+        # Check if an agent has read a storage entry
+        #
+        # @param agent_id [Symbol] The agent identifier
+        # @param entry_path [String] The storage entry path
+        # @return [Boolean] true if the agent has read this entry
+        def entry_read?(agent_id, entry_path)
+          @mutex.synchronize do
+            return false unless @read_entries[agent_id]
+
+            @read_entries[agent_id].include?(entry_path)
+          end
+        end
+
+        # Clear read history for an agent (useful for testing)
+        #
+        # @param agent_id [Symbol] The agent identifier
+        # @return [void]
+        def clear(agent_id)
+          @mutex.synchronize do
+            @read_entries.delete(agent_id)
+          end
+        end
+
+        # Clear all read history (useful for testing)
+        #
+        # @return [void]
+        def clear_all
+          @mutex.synchronize do
+            @read_entries.clear
+          end
+        end
+      end
+    end
+  end
+end

data/lib/swarm_memory/dsl/builder_extension.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module SwarmMemory
+  module DSL
+    # Extension module that injects memory DSL into SwarmSDK::Agent::Builder
+    #
+    # This module is included into Agent::Builder when swarm_memory is required,
+    # adding the `memory` configuration method.
+    module BuilderExtension
+      # Configure persistent memory for this agent
+      #
+      # @example Interactive mode (default) - Learn and retrieve
+      #   memory do
+      #     directory ".swarm/agent-memory"
+      #   end
+      #
+      # @example Retrieval mode - Read-only Q&A
+      #   memory do
+      #     directory "team-knowledge/"
+      #     mode :retrieval
+      #   end
+      #
+      # @example Researcher mode - Knowledge extraction
+      #   memory do
+      #     directory "team-knowledge/"
+      #     mode :researcher
+      #   end
+      def memory(&block)
+        @memory_config = SwarmMemory::DSL::MemoryConfig.new
+        @memory_config.instance_eval(&block) if block_given?
+        @memory_config
+      end
+    end
+  end
+end
+
+# Inject memory DSL into Agent::Builder when this file is loaded
+if defined?(SwarmSDK::Agent::Builder)
+  SwarmSDK::Agent::Builder.include(SwarmMemory::DSL::BuilderExtension)
+end