swarm_sdk 2.0.6 → 2.0.7

data/lib/swarm_sdk/agent/context_manager.rb

@@ -0,0 +1,309 @@
+# frozen_string_literal: true
+
+module SwarmSDK
+  module Agent
+    # Manages conversation context and message optimization
+    #
+    # Responsibilities:
+    # - Handle ephemeral messages (sent to LLM but not persisted)
+    # - Extract and strip system reminders
+    # - Prepare messages for LLM API calls
+    # - Future: Context window management, summarization, truncation
+    #
+    # @example
+    #   manager = ContextManager.new
+    #   manager.add_ephemeral_reminder("<system-reminder>Use caution</system-reminder>")
+    #   messages_for_llm = manager.prepare_for_llm(persistent_messages)
+    #   manager.clear_ephemeral  # After LLM call
+    class ContextManager
+      SYSTEM_REMINDER_REGEX = %r{<system-reminder>.*?</system-reminder>}m
+
+      def initialize
+        # Ephemeral content to append to messages for this turn only
+        # Format: { message_index => [array of reminder strings] }
+        @ephemeral_content = {}
+      end
+
+      # Track ephemeral content to append to a specific message
+      #
+      # Reminders will be embedded in the message content when sent to LLM,
+      # but are NOT persisted in the message history.
+      #
+      # @param message_index [Integer] Index of message to append to
+      # @param content [String] Reminder content to append
+      # @return [void]
+      def add_ephemeral_content_for_message(message_index, content)
+        @ephemeral_content[message_index] ||= []
+        @ephemeral_content[message_index] << content
+      end
+
+      # Add ephemeral reminder to the most recent message
+      #
+      # This will append the reminder to the last message in the array when
+      # preparing for LLM, but won't modify the stored message.
+      #
+      # @param content [String] Reminder content
+      # @param messages_array [Array<RubyLLM::Message>] Message array to get index from
+      # @return [void]
+      def add_ephemeral_reminder(content, messages_array:)
+        message_index = messages_array.size - 1
+        return if message_index < 0
+
+        add_ephemeral_content_for_message(message_index, content)
+      end
+
+      # Prepare messages for LLM API call
+      #
+      # Embeds ephemeral content into messages for this turn only.
+      # Does NOT modify the persistent messages array.
+      #
+      # @param persistent_messages [Array<RubyLLM::Message>] Messages from @messages
+      # @return [Array<RubyLLM::Message>] Messages with ephemeral content embedded
+      def prepare_for_llm(persistent_messages)
+        return persistent_messages.dup if @ephemeral_content.empty?
+
+        # Clone messages and embed ephemeral content
+        messages_for_llm = persistent_messages.map.with_index do |msg, index|
+          ephemeral_for_this_msg = @ephemeral_content[index]
+
+          # No ephemeral content for this message - use as-is
+          next msg unless ephemeral_for_this_msg&.any?
+
+          # Embed ephemeral content in this message
+          original_content = msg.content.is_a?(RubyLLM::Content) ? msg.content.text : msg.content.to_s
+          embedded_content = [original_content, *ephemeral_for_this_msg].join("\n\n")
+
+          # Create new message with embedded content
+          if msg.content.is_a?(RubyLLM::Content)
+            RubyLLM::Message.new(
+              role: msg.role,
+              content: RubyLLM::Content.new(embedded_content, msg.content.attachments),
+              tool_call_id: msg.tool_call_id,
+            )
+          else
+            RubyLLM::Message.new(
+              role: msg.role,
+              content: embedded_content,
+              tool_call_id: msg.tool_call_id,
+            )
+          end
+        end
+
+        messages_for_llm
+      end
+
+      # Clear all ephemeral content
+      #
+      # Should be called after LLM response is received.
+      #
+      # @return [void]
+      def clear_ephemeral
+        @ephemeral_content.clear
+      end
+
+      # Check if there is pending ephemeral content
+      #
+      # @return [Boolean] True if ephemeral content exists
+      def has_ephemeral?
+        @ephemeral_content.any?
+      end
+
+      # Get count of messages with ephemeral content
+      #
+      # @return [Integer] Number of messages with ephemeral content attached
+      def ephemeral_count
+        @ephemeral_content.size
+      end
+
+      # Extract all <system-reminder> blocks from content
+      #
+      # @param content [String] Content to extract from
+      # @return [Array<String>] Array of system reminder blocks
+      def extract_system_reminders(content)
+        return [] if content.nil? || content.empty?
+
+        content.scan(SYSTEM_REMINDER_REGEX)
+      end
+
+      # Strip all <system-reminder> blocks from content
+      #
+      # Returns clean content without system reminders.
+      #
+      # @param content [String] Content to strip from
+      # @return [String] Clean content
+      def strip_system_reminders(content)
+        return content if content.nil? || content.empty?
+
+        content.gsub(SYSTEM_REMINDER_REGEX, "").strip
+      end
+
+      # Check if content contains system reminders
+      #
+      # @param content [String] Content to check
+      # @return [Boolean] True if reminders found
+      def has_system_reminders?(content)
+        return false if content.nil? || content.empty?
+
+        SYSTEM_REMINDER_REGEX.match?(content)
+      end
+
+      # ============================================================================
+      # FUTURE: Context Optimization Methods (Hooks for Later Implementation)
+      # ============================================================================
+
+      # Future: Summarize old messages to save context window space
+      #
+      # @param messages [Array<RubyLLM::Message>] Messages to potentially summarize
+      # @param before_index [Integer] Summarize messages before this index
+      # @param strategy [Symbol] Summarization strategy (:llm, :truncate, :remove)
+      # @return [Array<RubyLLM::Message>] Optimized message array
+      def summarize_old_messages(messages, before_index:, strategy: :truncate)
+        # TODO: Implement when needed
+        messages
+      end
+
+      # Future: Truncate messages to fit within context window
+      #
+      # @param messages [Array<RubyLLM::Message>] Messages to fit
+      # @param max_tokens [Integer] Maximum token budget
+      # @param keep_recent [Integer] Number of recent messages to always keep
+      # @return [Array<RubyLLM::Message>] Truncated messages
+      def truncate_to_fit(messages, max_tokens:, keep_recent: 10)
+        # TODO: Implement when needed
+        messages
+      end
+
+      # Compress verbose tool results for older messages
+      #
+      # Uses progressive compression: older messages are compressed more aggressively.
+      # Preserves user/assistant messages at full detail (conversational context).
+      #
+      # @param messages [Array<RubyLLM::Message>] Messages to compress
+      # @param keep_recent [Integer] Number of recent messages to keep at full detail
+      # @return [Array<RubyLLM::Message>] Compressed messages
+      def compress_tool_results(messages, keep_recent: 10)
+        messages.map.with_index do |msg, i|
+          # Keep recent messages at full detail
+          next msg if i >= messages.size - keep_recent
+
+          # Keep user/assistant messages (conversational flow is important)
+          next msg if [:user, :assistant].include?(msg.role)
+
+          # Compress old tool results
+          if msg.role == :tool
+            compress_tool_message(msg, age: messages.size - i)
+          else
+            msg
+          end
+        end
+      end
+
+      # Compress a single tool message based on age
+      #
+      # Progressive compression: older messages get compressed more.
+      # For re-runnable tools (Read, Grep, Glob, etc.), adds instruction to re-run if needed.
+      #
+      # @param msg [RubyLLM::Message] Tool message to compress
+      # @param age [Integer] How many messages ago (higher = older)
+      # @return [RubyLLM::Message] Compressed message
+      def compress_tool_message(msg, age:)
+        content = msg.content.to_s
+
+        # Progressive compression based on age
+        max_length = case age
+        when 0..10   then return msg           # Recent: keep full detail
+        when 11..20  then 1000                 # Medium age: light compression
+        when 21..40  then 500                  # Old: moderate compression
+        when 41..60  then 200                  # Very old: heavy compression
+        else              100                  # Ancient: minimal summary
+        end
+
+        return msg if content.length <= max_length
+
+        # Compress while preserving structure
+        compressed = content.slice(0, max_length)
+        truncated_chars = content.length - max_length
+        compressed += "\n...[#{truncated_chars} chars truncated for context management]"
+
+        # Detect if this is a re-runnable tool and add helpful instruction
+        tool_name = detect_tool_name(content)
+        if rerunnable_tool?(tool_name)
+          compressed += "\n\n💡 If you need the full output, re-run the #{tool_name} tool with the same parameters."
+        end
+
+        RubyLLM::Message.new(
+          role: :tool,
+          content: compressed,
+          tool_call_id: msg.tool_call_id,
+        )
+      end
+
+      # Detect tool name from content
+      #
+      # @param content [String] Tool result content
+      # @return [String, nil] Tool name or nil
+      def detect_tool_name(content)
+        # Many tool results start with patterns we can detect
+        case content
+        when /^\s*\d+→/ # Line numbers (Read, MemoryRead)
+          content.include?("memory://") ? "MemoryRead" : "Read"
+        when /^Memory entries matching/ # MemoryGlob
+          "MemoryGlob"
+        when /^Found \d+ files? matching/ # Glob
+          "Glob"
+        when /matches in \d+ files?|No matches found/ # Grep, MemoryGrep
+          content.include?("memory://") ? "MemoryGrep" : "Grep"
+        when %r{^Stored at memory://} # MemoryWrite (not re-runnable but identifiable)
+          "MemoryWrite"
+        when %r{^Deleted memory://} # MemoryDelete
+          "MemoryDelete"
+        end
+      end
+
+      # Check if a tool is re-runnable (idempotent, can get same data again)
+      #
+      # @param tool_name [String, nil] Tool name
+      # @return [Boolean] True if tool can be re-run safely
+      def rerunnable_tool?(tool_name)
+        return false if tool_name.nil?
+
+        # These tools are idempotent - re-running gives same/current data
+        ["Read", "MemoryRead", "Grep", "MemoryGrep", "Glob", "MemoryGlob"].include?(tool_name)
+      end
+
+      # Automatically compress messages when context threshold is hit
+      #
+      # This is called automatically when context usage crosses 60% threshold.
+      # Returns compressed messages array for immediate use.
+      #
+      # @param messages [Array<RubyLLM::Message>] Current message array
+      # @param keep_recent [Integer] Number of recent messages to keep full
+      # @return [Array<RubyLLM::Message>] Compressed messages
+      def auto_compress_on_threshold(messages, keep_recent: 10)
+        return messages if @compression_applied
+
+        # Mark as applied to avoid compressing multiple times
+        @compression_applied = true
+
+        compress_tool_results(messages, keep_recent: keep_recent)
+      end
+
+      # Reset compression flag (when conversation is reset)
+      #
+      # @return [void]
+      def reset_compression
+        @compression_applied = false
+      end
+
+      # Future: Detect if context is becoming bloated
+      #
+      # @param messages [Array<RubyLLM::Message>] Messages to analyze
+      # @param threshold [Float] Bloat threshold (0.0-1.0)
+      # @return [Hash] Bloat analysis with recommendations
+      def analyze_context_bloat(messages, threshold: 0.7)
+        # TODO: Implement when needed
+        { bloated: false, recommendations: [] }
+      end
+    end
+  end
+end

data/lib/swarm_sdk/agent/definition.rb

@@ -125,24 +125,36 @@ module SwarmSDK
       #
       # @return [Boolean]
       def memory_enabled?
-        @memory&.respond_to?(:enabled?) && @memory.enabled?
+        return false if @memory.nil?
+
+        # MemoryConfig object (from DSL)
+        return @memory.enabled? if @memory.respond_to?(:enabled?)
+
+        # Hash (from YAML) - check for directory key
+        if @memory.is_a?(Hash)
+          directory = @memory[:directory] || @memory["directory"]
+          return !directory.nil? && !directory.to_s.strip.empty?
+        end
+
+        false
       end
 
       # Parse memory configuration from Hash or MemoryConfig object
       #
-      # @param memory_config [Hash, Agent::MemoryConfig, nil] Memory configuration
-      # @return [Agent::MemoryConfig, nil]
+      # @param memory_config [Hash, Object, nil] Memory configuration
+      # @return [Object, Hash, nil] Memory config (could be MemoryConfig from swarm_memory or Hash)
       def parse_memory_config(memory_config)
         return if memory_config.nil?
-        return memory_config if memory_config.is_a?(Agent::MemoryConfig)
-
-        # Convert hash (from YAML) to MemoryConfig object
-        config = Agent::MemoryConfig.new
-        adapter_value = memory_config[:adapter] || memory_config["adapter"] || :filesystem
-        config.adapter(adapter_value.to_sym)
-        directory_value = memory_config[:directory] || memory_config["directory"]
-        config.directory(directory_value) if directory_value
-        config
+
+        # If it's a MemoryConfig object (duck typing - has directory, adapter, mode methods)
+        # return as-is. This could be SwarmMemory::DSL::MemoryConfig or any compatible object.
+        return memory_config if memory_config.respond_to?(:directory) &&
+          memory_config.respond_to?(:adapter) &&
+          memory_config.respond_to?(:enabled?)
+
+        # If it's a hash (from YAML), keep it as a hash
+        # Plugin will create storage adapter based on the hash values
+        memory_config
       end
 
       def to_h
@@ -271,13 +283,14 @@ module SwarmSDK
           (custom_prompt || "").to_s
         end
 
-        # Append memory instructions if memory is enabled
-        if memory_enabled?
-          memory_prompt = render_memory_prompt
+        # Append plugin contributions to system prompt
+        plugin_contributions = collect_plugin_prompt_contributions
+        if plugin_contributions.any?
+          combined_contributions = plugin_contributions.join("\n\n")
           prompt = if prompt && !prompt.strip.empty?
-            "#{prompt}\n\n#{memory_prompt}"
+            "#{prompt}\n\n#{combined_contributions}"
           else
-            memory_prompt
+            combined_contributions
           end
         end
 
@@ -305,11 +318,26 @@ module SwarmSDK
         ERB.new(template_content).result(binding)
       end
 
-      def render_memory_prompt
-        # Load and render the memory system prompt
-        memory_prompt_path = File.expand_path("../prompts/memory.md.erb", __dir__)
-        template_content = File.read(memory_prompt_path)
-        ERB.new(template_content).result(binding)
+      # Collect system prompt contributions from all plugins
+      #
+      # Asks each registered plugin if it wants to contribute to the system prompt.
+      # Plugins can return custom instructions based on their configuration.
+      #
+      # @return [Array<String>] Array of prompt contributions from plugins
+      def collect_plugin_prompt_contributions
+        contributions = []
+
+        PluginRegistry.all.each do |plugin|
+          # Check if plugin has storage enabled for this agent
+          next unless plugin.storage_enabled?(self)
+
+          # Ask plugin for prompt contribution
+          # Note: storage is not available yet at this point, so we pass nil
+          contribution = plugin.system_prompt_contribution(agent_definition: self, storage: nil)
+          contributions << contribution if contribution && !contribution.strip.empty?
+        end
+
+        contributions
       end
 
       def render_non_coding_base_prompt
@@ -326,13 +354,18 @@ module SwarmSDK
         date = Time.now.strftime("%Y-%m-%d")
 
         <<~PROMPT.strip
-          # Environment
+          # Today's date
+
+          <today-date>
+          #{date}
+          #</today-date>
+
+          # Current Environment
 
           <env>
           Working directory: #{cwd}
           Platform: #{platform}
           OS Version: #{os_version}
-          Today's date: #{date}
           </env>
 
           # Task Management

data/lib/swarm_sdk/plugin.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+module SwarmSDK
+  # Base class for SwarmSDK plugins
+  #
+  # Plugins provide tools, storage, configuration parsing, and lifecycle hooks.
+  # Plugins are self-registering - they call SwarmSDK::PluginRegistry.register
+  # when the gem is loaded.
+  #
+  # @example Implementing a plugin
+  #   class MyPlugin < SwarmSDK::Plugin
+  #     def name
+  #       :my_plugin
+  #     end
+  #
+  #     def tools
+  #       [:MyTool, :OtherTool]
+  #     end
+  #
+  #     def create_tool(tool_name, context)
+  #       # Create and return tool instance
+  #     end
+  #   end
+  #
+  #   SwarmSDK::PluginRegistry.register(MyPlugin.new)
+  class Plugin
+    # Plugin name (must be unique)
+    #
+    # @return [Symbol] Plugin identifier
+    def name
+      raise NotImplementedError, "#{self.class} must implement #name"
+    end
+
+    # List of tools provided by this plugin
+    #
+    # @return [Array<Symbol>] Tool names (e.g., [:MemoryWrite, :MemoryRead])
+    def tools
+      []
+    end
+
+    # Create a tool instance
+    #
+    # @param tool_name [Symbol] Tool name (e.g., :MemoryWrite)
+    # @param context [Hash] Creation context
+    #   - :agent_name [Symbol] Agent identifier
+    #   - :storage [Object] Plugin storage instance (if created)
+    #   - :agent_definition [Agent::Definition] Full agent definition
+    #   - :chat [Agent::Chat] Chat instance (for tools that need it)
+    #   - :tool_configurator [Swarm::ToolConfigurator] For tools that register other tools
+    # @return [RubyLLM::Tool] Tool instance
+    def create_tool(tool_name, context)
+      raise NotImplementedError, "#{self.class} must implement #create_tool"
+    end
+
+    # Create plugin storage for an agent (optional)
+    #
+    # Called during agent initialization. Return nil if plugin doesn't need storage.
+    #
+    # @param agent_name [Symbol] Agent identifier
+    # @param config [Object] Plugin configuration from agent definition
+    # @return [Object, nil] Storage instance or nil
+    def create_storage(agent_name:, config:)
+      nil
+    end
+
+    # Parse plugin configuration from agent definition
+    #
+    # @param raw_config [Object] Raw config (DSL object or Hash from YAML)
+    # @return [Object] Parsed configuration
+    def parse_config(raw_config)
+      raw_config
+    end
+
+    # Contribute to agent system prompt (optional)
+    #
+    # @param agent_definition [Agent::Definition] Agent definition
+    # @param storage [Object, nil] Plugin storage instance (if created)
+    # @return [String, nil] Prompt contribution or nil
+    def system_prompt_contribution(agent_definition:, storage:)
+      nil
+    end
+
+    # Tools that should be marked immutable (optional)
+    #
+    # Immutable tools cannot be removed by other tools (e.g., LoadSkill).
+    #
+    # @return [Array<Symbol>] Tool names
+    def immutable_tools
+      []
+    end
+
+    # Agent storage enabled for this agent? (optional)
+    #
+    # @param agent_definition [Agent::Definition] Agent definition
+    # @return [Boolean] True if storage should be created
+    def storage_enabled?(agent_definition)
+      false
+    end
+
+    # Lifecycle: Called when agent is initialized
+    #
+    # @param agent_name [Symbol] Agent identifier
+    # @param agent [Agent::Chat] Chat instance
+    # @param context [Hash] Initialization context
+    #   - :storage [Object, nil] Plugin storage
+    #   - :agent_definition [Agent::Definition] Definition
+    #   - :tool_configurator [Swarm::ToolConfigurator] Configurator
+    def on_agent_initialized(agent_name:, agent:, context:)
+      # Override if needed
+    end
+
+    # Lifecycle: Called when swarm starts
+    #
+    # @param swarm [Swarm] Swarm instance
+    def on_swarm_started(swarm:)
+      # Override if needed
+    end
+
+    # Lifecycle: Called when swarm stops
+    #
+    # @param swarm [Swarm] Swarm instance
+    def on_swarm_stopped(swarm:)
+      # Override if needed
+    end
+
+    # Lifecycle: Called on every user message
+    #
+    # Plugins can return system reminders to inject based on the user's prompt.
+    # This enables features like semantic skill discovery, context injection, etc.
+    #
+    # @param agent_name [Symbol] Agent identifier
+    # @param prompt [String] The user's message
+    # @param is_first_message [Boolean] True if this is the first message in the conversation
+    # @return [Array<String>] System reminders to inject (empty array if none)
+    #
+    # @example Semantic skill discovery
+    #   def on_user_message(agent_name:, prompt:, is_first_message:)
+    #     skills = semantic_search(prompt, threshold: 0.65)
+    #     return [] if skills.empty?
+    #
+    #     [build_skill_reminder(skills)]
+    #   end
+    def on_user_message(agent_name:, prompt:, is_first_message:)
+      []
+    end
+  end
+end

data/lib/swarm_sdk/plugin_registry.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module SwarmSDK
+  # Plugin registry for managing SwarmSDK extensions
+  #
+  # Plugins register themselves when loaded, providing tools, storage,
+  # and lifecycle hooks without SwarmSDK needing to know about them.
+  module PluginRegistry
+    @plugins = {}
+    @tool_map = {}
+
+    class << self
+      # Register a plugin
+      #
+      # @param plugin [Plugin] Plugin instance
+      # @raise [ArgumentError] If plugin with same name already registered
+      def register(plugin)
+        raise ArgumentError, "Plugin must inherit from SwarmSDK::Plugin" unless plugin.is_a?(Plugin)
+
+        name = plugin.name
+        raise ArgumentError, "Plugin name required" unless name
+        raise ArgumentError, "Plugin #{name} already registered" if @plugins.key?(name)
+
+        @plugins[name] = plugin
+
+        # Build tool → plugin mapping
+        plugin.tools.each do |tool_name|
+          if @tool_map.key?(tool_name)
+            raise ArgumentError, "Tool #{tool_name} already registered by #{@tool_map[tool_name].name}"
+          end
+
+          @tool_map[tool_name] = plugin
+        end
+      end
+
+      # Get plugin by name
+      #
+      # @param name [Symbol] Plugin name
+      # @return [Plugin, nil] Plugin instance or nil
+      def get(name)
+        @plugins[name]
+      end
+
+      # Get all registered plugins
+      #
+      # @return [Array<Plugin>] All plugins
+      def all
+        @plugins.values
+      end
+
+      # Check if plugin is registered
+      #
+      # @param name [Symbol] Plugin name
+      # @return [Boolean] True if registered
+      def registered?(name)
+        @plugins.key?(name)
+      end
+
+      # Get plugin that provides a tool
+      #
+      # @param tool_name [Symbol] Tool name
+      # @return [Plugin, nil] Plugin that provides tool or nil
+      def plugin_for_tool(tool_name)
+        @tool_map[tool_name]
+      end
+
+      # Check if tool is provided by a plugin
+      #
+      # @param tool_name [Symbol] Tool name
+      # @return [Boolean] True if tool is plugin-provided
+      def plugin_tool?(tool_name)
+        @tool_map.key?(tool_name)
+      end
+
+      # Get all tools provided by plugins
+      #
+      # @return [Hash<Symbol, Plugin>] Tool name → Plugin mapping
+      def tools
+        @tool_map.dup
+      end
+
+      # Clear all plugins (for testing)
+      #
+      # @return [void]
+      def clear
+        @plugins.clear
+        @tool_map.clear
+      end
+
+      # Emit lifecycle event to all plugins
+      #
+      # @param event [Symbol] Event name
+      # @param args [Hash] Event arguments
+      def emit_event(event, **args)
+        @plugins.each_value do |plugin|
+          plugin.public_send(event, **args) if plugin.respond_to?(event)
+        end
+      end
+    end
+  end
+end

data/lib/swarm_sdk/prompts/base_system_prompt.md.erb

@@ -215,6 +215,13 @@ IMPORTANT: Always use the TodoWrite tool to plan and track tasks throughout the
 
 You MUST answer concisely with fewer than 4 lines of text (not including tool use or code generation), unless user asks for detail.
 
+
+# Today's date
+
+<today-date>
+#{date}
+#</today-date>
+
 # Environment information
 
 Here is useful information about the environment you are running in:
@@ -222,7 +229,6 @@ Here is useful information about the environment you are running in:
 Working directory: <%= cwd %>
 Platform: <%= platform %>
 OS Version: <%= os_version %>
-Today's date: <%= date %>
 </env>
 
 IMPORTANT: Assist with defensive security tasks only. Refuse to create, modify, or improve code that may be used maliciously. Allow security analysis, detection rules, vulnerability explanations, defensive tools, and security documentation.