ruby-mana 0.5.1 → 0.5.7

data/lib/mana/introspect.rb

@@ -4,18 +4,17 @@ require "prism"
 
 module Mana
   # Introspects the caller's source file to discover user-defined methods.
-  # Uses Prism AST to extract `def` nodes with their parameter signatures.
+  # Uses Prism AST to extract `def` nodes with their parameter signatures,
+  # descriptions (from comments above the def), and parameter types (from YARD @param tags).
   module Introspect
     class << self
       # Extract method definitions from a Ruby source file.
-      # Returns an array of { name:, params: } hashes.
-      #
-      # @param path [String] path to the Ruby source file
-      # @return [Array<Hash>] method definitions found
+      # Returns an array of { name:, params:, description:, param_types: } hashes.
       def methods_from_file(path)
         return [] unless path && File.exist?(path)
 
         source = File.read(path)
+        source_lines = source.lines
         result = Prism.parse(source)
         methods = []
 
@@ -23,22 +22,30 @@ module Mana
           next unless node.is_a?(Prism::DefNode)
 
           params = extract_params(node)
-          methods << { name: node.name.to_s, params: params }
+          description, param_types = extract_comments(node, source_lines)
+          methods << {
+            name: node.name.to_s,
+            params: params,
+            description: description,
+            param_types: param_types
+          }
         end
 
         methods
       end
 
       # Format discovered methods as a string for the system prompt.
-      #
-      # @param methods [Array<Hash>] from methods_from_file
-      # @return [String] formatted method list
+      # Includes descriptions when available.
       def format_for_prompt(methods)
         return "" if methods.empty?
 
         lines = methods.map do |m|
           sig = m[:params].empty? ? m[:name] : "#{m[:name]}(#{m[:params].join(', ')})"
-          "  #{sig}"
+          if m[:description]
+            "  #{sig} — #{m[:description]}"
+          else
+            "  #{sig}"
+          end
         end
 
         "Available Ruby functions:\n#{lines.join("\n")}"
@@ -46,6 +53,7 @@ module Mana
 
       private
 
+      # Breadth-first walk over the AST, yielding each node to the block
       def walk(node, &block)
         queue = [node]
         while (current = queue.shift)
@@ -56,29 +64,64 @@ module Mana
         end
       end
 
+      # Extract description and @param types from comments above a def node.
+      # Supports YARD-style comments:
+      #   # Description text here
+      #   # @param name [Type] description
+      def extract_comments(def_node, source_lines)
+        def_line = def_node.location.start_line - 1  # 0-indexed
+        description_parts = []
+        param_types = {}
+
+        # Walk backwards from the line above def, collecting # comments
+        line_idx = def_line - 1
+        comment_lines = []
+        while line_idx >= 0
+          line = source_lines[line_idx]&.strip
+          break unless line&.start_with?("#")
+          comment_lines.unshift(line)
+          line_idx -= 1
+        end
+
+        comment_lines.each do |line|
+          text = line.sub(/^#\s?/, "")
+          if text.match?(/^@param\s/)
+            # @param name [Type] description
+            match = text.match(/^@param\s+(\w+)\s+\[(\w+)\]/)
+            if match
+              param_types[match[1]] = match[2].downcase
+            end
+          elsif text.match?(/^@return\s/)
+            # Skip @return tags
+          else
+            description_parts << text unless text.empty?
+          end
+        end
+
+        description = description_parts.empty? ? nil : description_parts.join(" ")
+        [description, param_types]
+      end
+
+      # Extract all parameter signatures from a DefNode's parameter list.
       def extract_params(def_node)
         params_node = def_node.parameters
         return [] unless params_node
 
         result = []
 
-        # Required parameters
         (params_node.requireds || []).each do |p|
           result << param_name(p)
         end
 
-        # Optional parameters
         (params_node.optionals || []).each do |p|
           result << "#{param_name(p)}=..."
         end
 
-        # Rest parameter
         if params_node.rest && !params_node.rest.is_a?(Prism::ImplicitRestNode)
           name = params_node.rest.name
           result << "*#{name || ''}"
         end
 
-        # Keyword parameters
         (params_node.keywords || []).each do |p|
           case p
           when Prism::RequiredKeywordParameterNode
@@ -88,13 +131,11 @@ module Mana
           end
         end
 
-        # Keyword rest
         if params_node.keyword_rest.is_a?(Prism::KeywordRestParameterNode)
           name = params_node.keyword_rest.name
           result << "**#{name || ''}"
         end
 
-        # Block parameter
         if params_node.block
           result << "&#{params_node.block.name || ''}"
         end
@@ -104,9 +145,7 @@ module Mana
 
       def param_name(node)
         case node
-        when Prism::RequiredParameterNode
-          node.name.to_s
-        when Prism::OptionalParameterNode
+        when Prism::RequiredParameterNode, Prism::OptionalParameterNode
           node.name.to_s
         else
           node.respond_to?(:name) ? node.name.to_s : "_"

data/lib/mana/logger.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+module Mana
+  # Verbose logging utilities for tracing LLM interactions.
+  # Included by Engine — provides vlog, vlog_value, vlog_code, etc.
+  # All methods are no-ops unless @config.verbose is true.
+  module Logger
+    private
+
+    # Log a debug message to stderr
+    def vlog(msg)
+      return unless @config.verbose
+
+      $stderr.puts "\e[2m[mana] #{msg}\e[0m"
+    end
+
+    # Log a value with smart formatting:
+    #   - Multi-line strings → highlighted code block
+    #   - Long strings (>200 chars) → truncated with char count
+    #   - Long arrays/hashes (>5 items) → truncated with item count
+    #   - Everything else → inline inspect
+    def vlog_value(prefix, value)
+      return unless @config.verbose
+
+      case value
+      when String
+        if value.include?("\n")
+          vlog(prefix)
+          vlog_code(value)
+        elsif value.length > 200
+          vlog("#{prefix} #{value[0, 80].inspect}... (#{value.length} chars)")
+        else
+          vlog("#{prefix} #{value.inspect}")
+        end
+      when Array
+        if value.length > 5
+          preview = value.first(3).map(&:inspect).join(", ")
+          vlog("#{prefix} [#{preview}, ...] (#{value.length} items)")
+        else
+          vlog("#{prefix} #{value.inspect}")
+        end
+      when Hash
+        if value.length > 5
+          preview = value.first(3).map { |k, v| "#{k.inspect}=>#{v.inspect}" }.join(", ")
+          vlog("#{prefix} {#{preview}, ...} (#{value.length} keys)")
+        else
+          vlog("#{prefix} #{value.inspect}")
+        end
+      else
+        str = value.inspect
+        if str.length > 200
+          vlog("#{prefix} #{str[0, 80]}... (#{str.length} chars)")
+        else
+          vlog("#{prefix} #{str}")
+        end
+      end
+    end
+
+    # Log a code block with Ruby syntax highlighting to stderr
+    def vlog_code(code)
+      return unless @config.verbose
+
+      highlighted = highlight_ruby(code)
+      highlighted.each_line do |line|
+        $stderr.puts "\e[2m[mana]\e[0m   #{line.rstrip}"
+      end
+    end
+
+    # Summarize tool input for compact logging.
+    # Multi-line string values are replaced with a brief summary.
+    def summarize_input(input)
+      return input.inspect unless input.is_a?(Hash)
+
+      summarized = input.map do |k, v|
+        if v.is_a?(String) && v.include?("\n")
+          lines = v.lines.size
+          words = v.split.size
+          first = v.lines.first&.strip&.slice(0, 30)
+          "#{k}: \"#{first}...\" (#{lines} lines, #{words} words)"
+        elsif v.is_a?(String) && v.length > 100
+          "#{k}: \"#{v[0, 50]}...\" (#{v.length} chars)"
+        else
+          "#{k}: #{v.inspect}"
+        end
+      end
+      "{#{summarized.join(', ')}}"
+    end
+
+    # Minimal Ruby syntax highlighter using ANSI escape codes
+    def highlight_ruby(code)
+      code
+        .gsub(/\b(def|end|do|if|elsif|else|unless|case|when|class|module|return|require|include|raise|begin|rescue|ensure|yield|while|until|for|break|next|nil|true|false|self)\b/) { "\e[35m#{$1}\e[0m" }
+        .gsub(/(["'])(?:(?=(\\?))\2.)*?\1/) { "\e[32m#{$&}\e[0m" }
+        .gsub(/(#[^\n]*)/) { "\e[2m#{$1}\e[0m" }
+        .gsub(/\b(\d+(?:\.\d+)?)\b/) { "\e[33m#{$1}\e[0m" }
+        .gsub(/(:[\w]+)/) { "\e[36m#{$1}\e[0m" }
+    end
+  end
+end

data/lib/mana/memory.rb

@@ -4,6 +4,7 @@ module Mana
   class Memory
     attr_reader :short_term, :long_term, :summaries
 
+    # Initialize with empty short-term and load persisted long-term memories from disk
     def initialize
       @short_term = []
       @long_term = []
@@ -17,22 +18,27 @@ module Mana
     # --- Class methods ---
 
     class << self
+      # Return the current thread's memory instance (lazy-initialized).
+      # Returns nil in incognito mode.
       def current
         return nil if incognito?
 
         Thread.current[:mana_memory] ||= new
       end
 
+      # Check if the current thread is in incognito mode (no memory)
       def incognito?
         Thread.current[:mana_incognito] == true
       end
 
+      # Run a block with memory disabled. Saves and restores previous state.
       def incognito(&block)
         previous_memory = Thread.current[:mana_memory]
         previous_incognito = Thread.current[:mana_incognito]
         Thread.current[:mana_incognito] = true
         Thread.current[:mana_memory] = nil
         block.call
+      # Always restore previous state, even if the block raises
       ensure
         Thread.current[:mana_incognito] = previous_incognito
         Thread.current[:mana_memory] = previous_memory
@@ -41,14 +47,18 @@ module Mana
 
     # --- Token estimation ---
 
+    # Estimate total token count across short-term messages, long-term facts, and summaries.
+    # Used to determine when memory compaction is needed.
     def token_count
       count = 0
       @short_term.each do |msg|
         content = msg[:content]
         case content
         when String
+          # Plain text message
           count += estimate_tokens(content)
         when Array
+          # Array of content blocks (tool_use, tool_result, text)
           content.each do |block|
             count += estimate_tokens(block[:text] || block[:content] || "")
           end
@@ -61,27 +71,37 @@ module Mana
 
     # --- Memory management ---
 
+    # Clear both short-term and long-term memory
     def clear!
       clear_short_term!
       clear_long_term!
     end
 
+    # Clear conversation history and compaction summaries
     def clear_short_term!
       @short_term.clear
       @summaries.clear
     end
 
+    # Clear persistent memories from both in-memory array and disk
     def clear_long_term!
       @long_term.clear
       store.clear(namespace)
     end
 
+    # Remove a specific long-term memory by ID and persist the change
     def forget(id:)
       @long_term.reject! { |m| m[:id] == id }
       store.write(namespace, @long_term)
     end
 
+    # Store a fact in long-term memory. Deduplicates by content.
+    # Persists to disk immediately after adding.
     def remember(content)
+      # Deduplicate: skip if identical content already exists
+      existing = @long_term.find { |e| e[:content] == content }
+      return existing if existing
+
       entry = { id: @next_id, content: content, created_at: Time.now.iso8601 }
       @next_id += 1
       @long_term << entry
@@ -91,20 +111,25 @@ module Mana
 
     # --- Compaction ---
 
+    # Synchronous compaction: wait for any background run, then compact immediately
     def compact!
       wait_for_compaction
       perform_compaction
     end
 
+    # Check if token usage exceeds the configured memory pressure threshold
     def needs_compaction?
       cw = context_window
       token_count > (cw * Mana.config.memory_pressure)
     end
 
+    # Launch background compaction if token pressure exceeds the threshold.
+    # Only one compaction thread runs at a time (guarded by mutex).
     def schedule_compaction
       return unless needs_compaction?
 
       @compact_mutex.synchronize do
+        # Skip if a compaction is already in progress
         return if @compact_thread&.alive?
 
         @compact_thread = Thread.new do
@@ -116,6 +141,7 @@ module Mana
       end
     end
 
+    # Block until the background compaction thread finishes (if running)
     def wait_for_compaction
       thread = @compact_mutex.synchronize { @compact_thread }
       thread&.join
@@ -123,65 +149,94 @@ module Mana
 
     # --- Display ---
 
+    # Human-readable summary: counts and token usage
     def inspect
       "#<Mana::Memory long_term=#{@long_term.size}, short_term=#{short_term_rounds} rounds, tokens=#{token_count}/#{context_window}>"
     end
 
     private
 
+    # Count conversation rounds (user-prompt messages only, not tool results)
     def short_term_rounds
       @short_term.count { |m| m[:role] == "user" && m[:content].is_a?(String) }
     end
 
+    # Rough token estimate: ~4 characters per token
     def estimate_tokens(text)
       return 0 unless text.is_a?(String)
 
-      # Rough estimate: ~4 chars per token
       (text.length / 4.0).ceil
     end
 
     def context_window
-      Mana.config.context_window || ContextWindow.detect(Mana.config.model)
+      Mana.config.context_window
     end
 
+    # Resolve memory store: user config > default file-based store
     def store
       Mana.config.memory_store || default_store
     end
 
+    # Lazy-initialized default FileStore singleton
     def default_store
       @default_store ||= FileStore.new
     end
 
     def namespace
-      Namespace.detect
+      ns = Mana.config.namespace
+      return ns if ns && !ns.to_s.empty?
+
+      dir = `git rev-parse --show-toplevel 2>/dev/null`.strip
+      return File.basename(dir) unless dir.empty?
+
+      d = Dir.pwd
+      loop do
+        return File.basename(d) if File.exist?(File.join(d, "Gemfile"))
+        parent = File.dirname(d)
+        break if parent == d
+        d = parent
+      end
+
+      File.basename(Dir.pwd)
     end
 
+    # Load long-term memories from the persistent store on initialization.
+    # Skips loading in incognito mode.
     def load_long_term
       return if self.class.incognito?
 
       @long_term = store.read(namespace)
+      # Set next ID to one past the highest existing ID
       @next_id = (@long_term.map { |m| m[:id] }.max || 0) + 1
     end
 
+    # Compact short-term memory: summarize old messages and keep only recent rounds.
+    # Merges existing summaries + old messages into a single new summary, so
+    # summaries don't accumulate unboundedly.
     def perform_compaction
       keep_recent = Mana.config.memory_keep_recent
-      # Count user-prompt messages (rounds)
+      # Find indices of user-prompt messages (each marks a conversation round)
       user_indices = @short_term.each_with_index
         .select { |msg, _| msg[:role] == "user" && msg[:content].is_a?(String) }
         .map(&:last)
 
+      # Not enough rounds to compact — nothing to do
       return if user_indices.size <= keep_recent
 
-      # Find the cutoff point: keep the last N rounds
-      cutoff_user_idx = user_indices[-(keep_recent)]
+      # Find the cutoff point: everything before the last N rounds gets summarized
+      # Clamp keep_recent to avoid negative index beyond array bounds
+      keep = [keep_recent, user_indices.size].min
+      cutoff_user_idx = user_indices[-keep]
       old_messages = @short_term[0...cutoff_user_idx]
       return if old_messages.empty?
 
       # Build text from old messages for summarization
       text_parts = old_messages.map do |msg|
         content = msg[:content]
+        # Format each message as "role: content" for the summarizer
         case content
         when String then "#{msg[:role]}: #{content}"
+        # Array blocks: extract text parts and join
         when Array
           texts = content.map { |b| b[:text] || b[:content] }.compact
           "#{msg[:role]}: #{texts.join(' ')}" unless texts.empty?
@@ -190,46 +245,84 @@ module Mana
 
       return if text_parts.empty?
 
-      summary = summarize(text_parts.join("\n"))
+      # Merge existing summaries into the input so we produce ONE rolling summary
+      # instead of accumulating separate summaries that never get cleaned up
+      prior_context = ""
+      unless @summaries.empty?
+        prior_context = "Previous summary:\n#{@summaries.join("\n")}\n\nNew conversation:\n"
+      end
 
-      # Replace old messages with summary
-      @short_term = @short_term[cutoff_user_idx..]
-      @summaries << summary
+      # Calculate how many tokens the kept messages will use after compaction
+      kept_messages = @short_term[cutoff_user_idx..]
+      keep_tokens = kept_messages.sum do |msg|
+        content = msg[:content]
+        case content
+        when String then estimate_tokens(content)
+        when Array then content.sum { |b| estimate_tokens(b[:text] || b[:content] || "") }
+        else 0
+        end
+      end
+      @long_term.each { |m| keep_tokens += estimate_tokens(m[:content]) }
+
+      # Call the LLM to produce a single merged summary
+      summary = summarize(prior_context + text_parts.join("\n"), keep_tokens: keep_tokens)
 
+      # Replace old messages with the summary, keeping only recent rounds.
+      # Clear all previous summaries — they are now merged into the new one.
+      @short_term = kept_messages
+      @summaries = [summary]
+
+      # Notify the on_compact callback if configured
       Mana.config.on_compact&.call(summary)
     end
 
-    def summarize(text)
+    # Call the LLM to produce a concise summary of the given conversation text.
+    # Uses the configured backend (Anthropic/OpenAI), respects timeout settings.
+    # Falls back to "Summary unavailable" on any error.
+    #
+    # @param keep_tokens [Integer] tokens already committed to keep_recent + long_term
+    # Retry up to 3 times on failure or refusal before giving up
+    SUMMARIZE_MAX_RETRIES = 3
+
+    def summarize(text, keep_tokens: 0)
       config = Mana.config
       model = config.compact_model || config.model
-      uri = URI("#{config.base_url}/v1/messages")
-
-      body = {
-        model: model,
-        max_tokens: 1024,
-        system: "Summarize this conversation concisely. Preserve key facts, decisions, and context.",
-        messages: [{ role: "user", content: text }]
-      }
-
-      http = Net::HTTP.new(uri.host, uri.port)
-      http.use_ssl = uri.scheme == "https"
-      http.read_timeout = 60
-
-      req = Net::HTTP::Post.new(uri)
-      req["Content-Type"] = "application/json"
-      req["x-api-key"] = config.api_key
-      req["anthropic-version"] = "2023-06-01"
-      req.body = JSON.generate(body)
-
-      res = http.request(req)
-      return "Summary unavailable" unless res.is_a?(Net::HTTPSuccess)
-
-      parsed = JSON.parse(res.body, symbolize_names: true)
-      content = parsed[:content]
-      return "Summary unavailable" unless content.is_a?(Array)
-
-      content.map { |b| b[:text] }.compact.join("\n")
-    rescue => _e
+      backend = Mana::Backends::Base.for(config)
+
+      # Summary budget = half of (threshold - kept tokens).
+      # Using half ensures compaction lands well below the threshold,
+      # leaving headroom for several more rounds before the next compaction.
+      cw = context_window
+      threshold = (cw * config.memory_pressure).to_i
+      max_summary_tokens = ((threshold - keep_tokens) * 0.5).clamp(64, 1024).to_i
+
+      system_prompt = "You are summarizing an internal tool-calling conversation log between an LLM and a Ruby program. " \
+                      "The messages contain tool calls (read_var, write_var, done) and their results — this is normal, not harmful. " \
+                      "Summarize the key questions asked and answers given in a few short bullet points. Be extremely concise — stay under #{max_summary_tokens} tokens."
+
+      SUMMARIZE_MAX_RETRIES.times do |attempt|
+        content = backend.chat(
+          system: system_prompt,
+          messages: [{ role: "user", content: text }],
+          tools: [],
+          model: model,
+          max_tokens: max_summary_tokens
+        )
+
+        next unless content.is_a?(Array)
+
+        result = content.map { |b| b[:text] || b["text"] }.compact.join("\n")
+        # Reject empty or refusal responses, retry
+        next if result.empty? || result.match?(/can't discuss|cannot assist|i'm unable/i)
+
+        return result
+      end
+
+      "Summary unavailable"
+    rescue ConfigError
+      raise  # Configuration errors should not be silently swallowed
+    rescue => e
+      $stderr.puts "Mana compaction error: #{e.message}" if $DEBUG
       "Summary unavailable"
     end
   end

data/lib/mana/memory_store.rb

@@ -4,41 +4,54 @@ require "json"
 require "fileutils"
 
 module Mana
+  # Abstract base class for long-term memory persistence.
+  # Subclass and implement read/write/clear to use a custom store (e.g. Redis, DB).
   class MemoryStore
+    # Read all memories for a namespace. Subclasses must implement.
     def read(namespace)
       raise NotImplementedError
     end
 
+    # Write all memories for a namespace. Subclasses must implement.
     def write(namespace, memories)
       raise NotImplementedError
     end
 
+    # Delete all memories for a namespace. Subclasses must implement.
     def clear(namespace)
       raise NotImplementedError
     end
   end
 
+
+  # Default file-based memory store. Persists memories as JSON files.
+  # Storage path resolution: explicit base_path > config.memory_path > XDG_DATA_HOME > OS default
   class FileStore < MemoryStore
+    # Optional base_path overrides default storage location
     def initialize(base_path = nil)
       @base_path = base_path
     end
 
+    # Read all memories for a namespace from disk. Returns [] on missing file or parse error.
     def read(namespace)
       path = file_path(namespace)
       return [] unless File.exist?(path)
 
       data = JSON.parse(File.read(path), symbolize_names: true)
       data.is_a?(Array) ? data : []
+    # Corrupted JSON file — return empty array rather than crashing
     rescue JSON::ParserError
       []
     end
 
+    # Write all memories for a namespace to disk (overwrites existing file)
     def write(namespace, memories)
       path = file_path(namespace)
       FileUtils.mkdir_p(File.dirname(path))
       File.write(path, JSON.pretty_generate(memories))
     end
 
+    # Delete the memory file for a namespace
     def clear(namespace)
       path = file_path(namespace)
       File.delete(path) if File.exist?(path)
@@ -46,24 +59,21 @@ module Mana
 
     private
 
+    # Build the full path for a namespace's JSON file
     def file_path(namespace)
       File.join(base_dir, "#{namespace}.json")
     end
 
+    # Resolve the base directory for memory storage.
+    # Priority: explicit base_path > config.memory_path > ~/.mana/memory
     def base_dir
       return File.join(@base_path, "memory") if @base_path
 
       custom_path = Mana.config.memory_path
       return File.join(custom_path, "memory") if custom_path
 
-      xdg = ENV["XDG_DATA_HOME"]
-      if xdg && !xdg.empty?
-        File.join(xdg, "mana", "memory")
-      elsif RUBY_PLATFORM.include?("darwin")
-        File.join(Dir.home, "Library", "Application Support", "mana", "memory")
-      else
-        File.join(Dir.home, ".local", "share", "mana", "memory")
-      end
+      # Default fallback
+      File.join(Dir.home, ".mana", "memory")
     end
   end
 end