llmemory 0.2.0 → 0.2.2

data/lib/llmemory/mcp/tools/memory_episode_record.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Llmemory
+  module MCP
+    module Tools
+      class MemoryEpisodeRecord < ::MCP::Tool
+        description "Record an experience (episodic memory): a trajectory of steps with an optional summary, outcome and importance. Use to remember what just happened so it can be retrieved or distilled into knowledge later."
+
+        input_schema(
+          properties: {
+            user_id: { type: "string", description: "User identifier" },
+            steps: {
+              type: "array",
+              description: "Ordered list of steps (objects with observation/action/result)",
+              items: {
+                type: "object",
+                properties: {
+                  observation: { type: "string" },
+                  action: { type: "string" },
+                  result: { type: "string" }
+                }
+              }
+            },
+            summary: { type: "string", description: "Optional summary (derived from steps if omitted)" },
+            outcome: { type: "string", description: "Outcome label, e.g. 'success', 'failure', 'recovered'" },
+            importance: { type: "number", description: "Importance 0-1 (default 0.5)" }
+          },
+          required: ["user_id", "steps"]
+        )
+
+        class << self
+          def call(user_id:, steps:, summary: nil, outcome: nil, importance: nil, server_context: nil)
+            memory = Llmemory::LongTerm::Episodic::Memory.new(user_id: user_id)
+            id = memory.record_episode(
+              steps: Array(steps),
+              summary: summary,
+              outcome: outcome,
+              importance: importance.nil? ? 0.5 : importance.to_f
+            )
+            ::MCP::Tool::Response.new([{ type: "text", text: "Episode recorded: #{id}" }])
+          rescue => e
+            ::MCP::Tool::Response.new([{ type: "text", text: "Error recording episode: #{e.message}" }], error: true)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/llmemory/mcp/tools/memory_episodes.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Llmemory
+  module MCP
+    module Tools
+      class MemoryEpisodes < ::MCP::Tool
+        description "List recent episodes (episodic memory) for a user. Optionally filter by a keyword query."
+
+        input_schema(
+          properties: {
+            user_id: { type: "string", description: "User identifier" },
+            query: { type: "string", description: "Optional keyword filter" },
+            limit: { type: "integer", description: "Max episodes to return (default 10)" }
+          },
+          required: ["user_id"]
+        )
+
+        class << self
+          def call(user_id:, query: nil, limit: nil, server_context: nil)
+            memory = Llmemory::LongTerm::Episodic::Memory.new(user_id: user_id)
+            cap = (limit || 10).to_i
+            episodes = if query.to_s.strip.empty?
+              memory.recent_episodes(limit: cap)
+            else
+              memory.search_candidates(query, top_k: cap).filter_map { |c| memory.find_episode(c[:id]) }
+            end
+
+            if episodes.empty?
+              return ::MCP::Tool::Response.new([{ type: "text", text: "No episodes for user #{user_id}." }])
+            end
+
+            lines = episodes.map do |ep|
+              "[#{ep.id}] (importance: #{ep.importance}; outcome: #{ep.outcome || 'n/a'}) #{ep.summary || ep.searchable_text[0, 120]}"
+            end
+            ::MCP::Tool::Response.new([{ type: "text", text: lines.join("\n") }])
+          rescue => e
+            ::MCP::Tool::Response.new([{ type: "text", text: "Error listing episodes: #{e.message}" }], error: true)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/llmemory/mcp/tools/memory_forget.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Llmemory
+  module MCP
+    module Tools
+      class MemoryForget < ::MCP::Tool
+        description "Remove entries from a memory by their ids (the ids returned by retrieval / listing tools), recording the removal in the audit log. memory_type: file_based | graph_based | episodic | procedural."
+
+        input_schema(
+          properties: {
+            user_id: { type: "string", description: "User identifier" },
+            memory_type: { type: "string", description: "file_based | graph_based | episodic | procedural" },
+            ids: { type: "array", items: { type: "string" }, description: "Entry ids to forget" },
+            reason: { type: "string", description: "Optional reason (recorded in audit)" }
+          },
+          required: ["user_id", "memory_type", "ids"]
+        )
+
+        class << self
+          def call(user_id:, memory_type:, ids:, reason: nil, server_context: nil)
+            memory = build_memory(user_id, memory_type)
+            removed = memory.forget(ids: Array(ids), reason: reason)
+            ::MCP::Tool::Response.new([{
+              type: "text",
+              text: "Forgot #{removed} entries from #{memory_type} memory for user #{user_id}."
+            }])
+          rescue NotImplementedError => e
+            ::MCP::Tool::Response.new([{ type: "text", text: "Not supported: #{e.message}" }], error: true)
+          rescue => e
+            ::MCP::Tool::Response.new([{ type: "text", text: "Error forgetting: #{e.message}" }], error: true)
+          end
+
+          private
+
+          def build_memory(user_id, memory_type)
+            case memory_type.to_s
+            when "file_based"
+              Llmemory::LongTerm::FileBased::Memory.new(user_id: user_id)
+            when "graph_based"
+              Llmemory::LongTerm::GraphBased::Memory.new(user_id: user_id)
+            when "episodic"
+              Llmemory::LongTerm::Episodic::Memory.new(user_id: user_id)
+            when "procedural"
+              Llmemory::LongTerm::Procedural::Memory.new(user_id: user_id)
+            else
+              raise ArgumentError, "Unknown memory_type: #{memory_type} (expected file_based|graph_based|episodic|procedural)"
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/llmemory/mcp/tools/memory_maintain.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Llmemory
+  module MCP
+    module Tools
+      class MemoryMaintain < ::MCP::Tool
+        description "Run the cognitive maintenance pass for a user: reflect (episodes -> insights), mine skills (episodes -> procedural), and expire entries past their TTL. Each step is isolated; a failure in one is reported and does not abort the others. Returns a summary report."
+
+        input_schema(
+          properties: {
+            user_id: { type: "string", description: "User identifier" },
+            reflect: { type: "boolean", description: "Distill insights from recent episodes (default true)" },
+            mine_skills: { type: "boolean", description: "Mine reusable skills from episodes and register them (default: config.skill_mining_enabled)" },
+            expire: { type: "boolean", description: "Soft-archive entries past their TTL (default true)" },
+            reflection_window: { type: "integer", description: "Episodes to reflect over (default 10)" },
+            mining_window: { type: "integer", description: "Episodes to mine for skills (default 20)" }
+          },
+          required: ["user_id"]
+        )
+
+        class << self
+          def call(user_id:, reflect: true, mine_skills: nil, expire: true,
+                   reflection_window: nil, mining_window: nil, server_context: nil)
+            opts = { reflect: reflect, expire: expire }
+            opts[:mine_skills] = mine_skills unless mine_skills.nil?
+            opts[:reflection_window] = reflection_window.to_i unless reflection_window.nil?
+            opts[:mining_window] = mining_window.to_i unless mining_window.nil?
+
+            report = Llmemory::Maintenance::CognitivePass.run!(user_id, **opts)
+            ::MCP::Tool::Response.new([{ type: "text", text: format_report(user_id, report) }])
+          rescue => e
+            ::MCP::Tool::Response.new([{ type: "text", text: "Error running maintenance pass: #{e.message}" }], error: true)
+          end
+
+          private
+
+          def format_report(user_id, report)
+            expired = report[:expired] || {}
+            lines = [
+              "Cognitive pass for #{user_id}:",
+              "  insights: #{Array(report[:insights]).size}",
+              "  skills mined: #{Array(report[:mined]).size}",
+              "  expired: episodic=#{expired[:episodic] || 0} procedural=#{expired[:procedural] || 0}"
+            ]
+            errors = report[:errors] || {}
+            lines << "  errors: #{errors.map { |k, v| "#{k}: #{v}" }.join('; ')}" unless errors.empty?
+            lines.join("\n")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/llmemory/mcp/tools/memory_mine_skills.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Llmemory
+  module MCP
+    module Tools
+      class MemoryMineSkills < ::MCP::Tool
+        description "Mine reusable skills from a user's successful episode trajectories (procedural learning). Human-in-the-loop by default: returns skill *proposals* and writes nothing. Set auto_register=true to register them in procedural memory (with provenance back to the source episodes)."
+
+        input_schema(
+          properties: {
+            user_id: { type: "string", description: "User identifier" },
+            window: { type: "integer", description: "Episodes to mine (default 20)" },
+            outcomes: { type: "array", items: { type: "string" }, description: "Optional allowlist of outcome labels to pre-filter episodes (e.g. ['success'])" },
+            auto_register: { type: "boolean", description: "Register the proposals instead of only returning them (default false)" }
+          },
+          required: ["user_id"]
+        )
+
+        class << self
+          def call(user_id:, window: nil, outcomes: nil, auto_register: false, server_context: nil)
+            episodic = Llmemory::LongTerm::Episodic::Memory.new(user_id: user_id)
+            procedural = Llmemory::LongTerm::Procedural::Memory.new(user_id: user_id)
+            result = Llmemory::SkillMining::Miner.new(episodic: episodic, procedural: procedural).mine(
+              window: (window || Llmemory::SkillMining::Miner::DEFAULT_WINDOW).to_i,
+              outcomes: outcomes,
+              auto_register: auto_register
+            )
+
+            ::MCP::Tool::Response.new([{ type: "text", text: format_result(user_id, result, auto_register) }])
+          rescue => e
+            ::MCP::Tool::Response.new([{ type: "text", text: "Error mining skills: #{e.message}" }], error: true)
+          end
+
+          private
+
+          def format_result(user_id, result, auto_register)
+            return "No skills could be mined for user #{user_id}." if result.empty?
+
+            if auto_register
+              "Registered #{result.size} mined skill(s): #{result.join(', ')}"
+            else
+              lines = ["#{result.size} skill proposal(s) for user #{user_id} (not registered):"]
+              result.each do |p|
+                lines << "  - #{p[:name]} (#{p[:kind]}, confidence: #{p[:confidence]}): #{p[:description] || p[:body]}"
+              end
+              lines.join("\n")
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/llmemory/mcp/tools/memory_retrieve.rb

@@ -59,8 +59,10 @@ module Llmemory
             items = storage.search_items(user_id, query)
             return "" if items.empty?
 
-            # Get timeline context around the first match
-            top_item = items.first
+            # Anchor on the most precise match: keyword search is recall-oriented
+            # (tokenized OR), so prefer the item whose content contains the full
+            # query verbatim, falling back to the first match.
+            top_item = best_match(items, query)
             item_id = top_item[:id] || top_item["id"]
             return "" unless item_id
 
@@ -70,6 +72,12 @@ module Llmemory
             ""
           end
 
+          def best_match(items, query)
+            q = query.to_s.downcase.strip
+            return items.first if q.empty?
+            items.find { |i| (i[:content] || i["content"]).to_s.downcase.include?(q) } || items.first
+          end
+
           def build_storage
             if Llmemory.configuration.long_term_type.to_s == "graph_based"
               LongTerm::GraphBased::Storages.build

data/lib/llmemory/mcp/tools/memory_skill_register.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Llmemory
+  module MCP
+    module Tools
+      class MemorySkillRegister < ::MCP::Tool
+        description "Register a reusable skill (procedural memory): a prompt, template or code snippet the agent can retrieve later. Re-registering the same name auto-increments the version."
+
+        input_schema(
+          properties: {
+            user_id: { type: "string", description: "User identifier" },
+            name: { type: "string", description: "Short identifier (skills with the same name get auto-versioned)" },
+            body: { type: "string", description: "The skill content (prompt / template / code)" },
+            description: { type: "string", description: "Optional human-readable description" },
+            kind: { type: "string", description: "prompt | template | code (default: prompt)" }
+          },
+          required: ["user_id", "name", "body"]
+        )
+
+        class << self
+          def call(user_id:, name:, body:, description: nil, kind: nil, server_context: nil)
+            memory = Llmemory::LongTerm::Procedural::Memory.new(user_id: user_id)
+            id = memory.register_skill(
+              name: name, body: body, description: description,
+              kind: kind || Llmemory::LongTerm::Procedural::Skill::DEFAULT_KIND
+            )
+            ::MCP::Tool::Response.new([{ type: "text", text: "Skill registered: #{id} (#{name})" }])
+          rescue => e
+            ::MCP::Tool::Response.new([{ type: "text", text: "Error registering skill: #{e.message}" }], error: true)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/llmemory/mcp/tools/memory_skill_report.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Llmemory
+  module MCP
+    module Tools
+      class MemorySkillReport < ::MCP::Tool
+        description "Report the outcome of applying a skill (success or failure). Feeds retrieval ranking: proven skills surface higher next time."
+
+        input_schema(
+          properties: {
+            user_id: { type: "string", description: "User identifier" },
+            skill_id: { type: "string", description: "Skill id (from MemorySkillRegister / MemorySkills)" },
+            success: { type: "boolean", description: "True if the skill worked; false otherwise" }
+          },
+          required: ["user_id", "skill_id", "success"]
+        )
+
+        class << self
+          def call(user_id:, skill_id:, success:, server_context: nil)
+            memory = Llmemory::LongTerm::Procedural::Memory.new(user_id: user_id)
+            skill = memory.report_outcome(skill_id, success: success == true)
+            if skill.nil?
+              return ::MCP::Tool::Response.new([{ type: "text", text: "Skill not found: #{skill_id}" }], error: true)
+            end
+
+            text = "Outcome recorded for #{skill.name} (#{skill_id}): success #{skill.success_count} / failure #{skill.failure_count} (rate #{format('%.2f', skill.success_rate)})"
+            ::MCP::Tool::Response.new([{ type: "text", text: text }])
+          rescue => e
+            ::MCP::Tool::Response.new([{ type: "text", text: "Error reporting outcome: #{e.message}" }], error: true)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/llmemory/mcp/tools/memory_skills.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Llmemory
+  module MCP
+    module Tools
+      class MemorySkills < ::MCP::Tool
+        description "List registered skills (procedural memory) for a user, ranked by proven utility when a query is given."
+
+        input_schema(
+          properties: {
+            user_id: { type: "string", description: "User identifier" },
+            query: { type: "string", description: "Optional keyword to filter skills" },
+            limit: { type: "integer", description: "Max skills to return (default 10)" }
+          },
+          required: ["user_id"]
+        )
+
+        class << self
+          def call(user_id:, query: nil, limit: nil, server_context: nil)
+            memory = Llmemory::LongTerm::Procedural::Memory.new(user_id: user_id)
+            cap = (limit || 10).to_i
+            skills = if query.to_s.strip.empty?
+              memory.skills(limit: cap)
+            else
+              memory.search_candidates(query, top_k: cap).filter_map { |c| memory.get_skill(c[:id]) }
+            end
+
+            if skills.empty?
+              return ::MCP::Tool::Response.new([{ type: "text", text: "No skills for user #{user_id}." }])
+            end
+
+            lines = skills.map do |s|
+              "[#{s.id}] #{s.name} v#{s.version} (#{s.kind}) — success rate #{format('%.2f', s.success_rate)} (#{s.success_count}/#{s.success_count + s.failure_count})"
+            end
+            ::MCP::Tool::Response.new([{ type: "text", text: lines.join("\n") }])
+          rescue => e
+            ::MCP::Tool::Response.new([{ type: "text", text: "Error listing skills: #{e.message}" }], error: true)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/llmemory/memory.rb

@@ -10,11 +10,13 @@ module Llmemory
     DEFAULT_SESSION_ID = "default"
     STATE_KEY_MESSAGES = :messages
 
-    def initialize(user_id:, session_id: DEFAULT_SESSION_ID, checkpoint: nil, long_term: nil, long_term_type: nil, retrieval_engine: nil, working_memory: nil, api_key: nil)
+    def initialize(user_id:, session_id: DEFAULT_SESSION_ID, checkpoint: nil, long_term: nil, long_term_type: nil, retrieval_engine: nil, working_memory: nil, episodic: nil, procedural: nil, api_key: nil)
       @user_id = user_id
       @session_id = session_id
       @checkpoint = checkpoint || ShortTerm::Checkpoint.new(user_id: user_id, session_id: session_id)
       @working_memory = working_memory
+      @episodic = episodic
+      @procedural = procedural
       @llm = api_key.to_s.empty? ? nil : Llmemory::LLM.client(api_key: api_key)
       type = long_term_type || Llmemory.configuration.long_term_type || :file_based
       @long_term = long_term || build_long_term(type)
@@ -22,12 +24,55 @@ module Llmemory
     end
 
     # Structured working memory for this session (CoALA working memory),
-    # parallel to the message checkpoint. Lazily built so it costs nothing
-    # unless an agent uses it.
+    # parallel to the message checkpoint. Lazily built.
     def working_memory
       @working_memory ||= WorkingMemory.new(user_id: @user_id, session_id: @session_id)
     end
 
+    # Episodic long-term memory (CoALA): records and retrieves agent trajectories.
+    # Additive — coexists with the semantic store (file/graph). Lazily built.
+    def episodic
+      @episodic ||= LongTerm::Episodic::Memory.new(user_id: @user_id)
+    end
+
+    # Procedural long-term memory (Voyager-style skill library). Lazily built.
+    def procedural
+      @procedural ||= LongTerm::Procedural::Memory.new(user_id: @user_id)
+    end
+
+    # Reflects over recent episodes and writes distilled insights to the
+    # semantic store (file/graph) with provenance back to source episodes.
+    def reflect!(window: 10, category: "insights")
+      Reflection::Reflector.new(episodic: episodic, semantic: @long_term, llm: @llm)
+        .reflect(window: window, category: category)
+    end
+
+    # Reasoning action: render a prompt from working memory, call the LLM, write
+    # the result back. Composable; does not touch long-term memory.
+    def reason(template:, into: Actions::Reason::DEFAULT_SLOT, parse: nil)
+      Actions::Reason.call(working_memory: working_memory, template: template, into: into, parse: parse, llm: @llm)
+    end
+
+    # Mines recent episodes for reusable skills (Voyager-style). Human-in-the-loop
+    # by default: returns skill proposals and writes nothing. With
+    # `auto_register: true`, registers them in procedural memory (with provenance
+    # back to the source episodes) and returns the new skill ids.
+    def mine_skills!(window: SkillMining::Miner::DEFAULT_WINDOW, outcomes: nil, auto_register: false)
+      SkillMining::Miner.new(episodic: episodic, procedural: procedural, llm: @llm)
+        .mine(window: window, outcomes: outcomes, auto_register: auto_register)
+    end
+
+    # Cognitive maintenance pass: consolidate -> reflect -> mine skills -> expire,
+    # in one step, closing the CoALA learning loop. Each step is isolated; a
+    # failure in one is captured in the report and never aborts the others.
+    def maintain!(**opts)
+      Maintenance::CognitivePass.run!(
+        @user_id,
+        memory: self, episodic: episodic, procedural: procedural, semantic: @long_term, llm: @llm,
+        **opts
+      )
+    end
+
     def add_message(role:, content:)
       msgs = messages
       msgs << { role: role.to_sym, content: content.to_s }

data/lib/llmemory/memory_module.rb

@@ -6,10 +6,10 @@ module Llmemory
   # abstractions; this mixin gives any memory store the same agent-facing
   # surface so frameworks can treat them polymorphically:
   #
-  #   read(query, user_id:, limit:)  -> relevant entries (retrieval)
-  #   write(payload, ...)            -> ingest into the store (learning)
-  #   list(user_id:, limit:)         -> enumerate stored entries
-  #   stats(user_id:)                -> counts and metadata
+  #   read(query, user_id:, limit:)          -> relevant entries (retrieval)
+  #   write(payload, ...)                    -> ingest into the store (learning)
+  #   list(user_id:, limit:, offset:)        -> enumerate stored entries (paginated)
+  #   stats(user_id:)                        -> counts and metadata
   #
   # `read` defaults to the de-facto `search_candidates` interface the retrieval
   # Engine already relies on. `write`, `list` and `stats` are implemented by each
@@ -30,7 +30,7 @@ module Llmemory
       raise NotImplementedError, "#{self.class} must implement #write"
     end
 
-    def list(user_id: nil, limit: nil)
+    def list(user_id: nil, limit: nil, offset: nil)
       raise NotImplementedError, "#{self.class} must implement #list"
     end
 
@@ -40,9 +40,16 @@ module Llmemory
 
     # Removes entries by id (the same ids returned by #read) and records the
     # removal in a ForgetLog audit. Returns the number of entries removed.
+    #
+    # mode:
+    #   :soft (default) — soft-archive: entries are excluded from list/search/
+    #     retrieval but remain accessible by id (think "trash"). Reversible if
+    #     the store supports it.
+    #   :hard           — physical deletion. Irreversible.
+    #
     # Implemented by stores with a clear deletion model; others may not support
     # it (CoALA-style "unlearning" is understudied; deletion semantics differ).
-    def forget(ids:, reason: nil)
+    def forget(ids:, reason: nil, mode: :soft)
       raise NotImplementedError, "#{self.class} does not support #forget"
     end
 

data/lib/llmemory/reflection/reflector.rb

@@ -30,27 +30,31 @@ module Llmemory
       # Reflects over the most recent `window` episodes and writes the resulting
       # insights to semantic memory. Returns the ids of the stored insights.
       def reflect(window: 10, category: DEFAULT_CATEGORY)
-        episodes = @episodic.recent_episodes(limit: window)
-        return [] if episodes.empty?
-
-        insights = distill(episodes)
-        return [] if insights.empty?
-
-        sources = episodes.map(&:id).compact.map { |id| { type: "episode", id: id } }
-
-        insights.filter_map do |insight|
-          provenance = Llmemory::Provenance.build(
-            method: "reflection",
-            sources: sources,
-            confidence: insight[:confidence]
-          )
-          @semantic.remember_fact(
-            content: insight[:content],
-            category: category,
-            importance: insight[:confidence] || DEFAULT_IMPORTANCE,
-            provenance: provenance
-          )
+        result = []
+        Llmemory::Instrumentation.instrument(:reflect, window: window, category: category) do
+          episodes = @episodic.recent_episodes(limit: window)
+          next if episodes.empty?
+
+          insights = distill(episodes)
+          next if insights.empty?
+
+          sources = episodes.map(&:id).compact.map { |id| { type: "episode", id: id } }
+
+          result = insights.filter_map do |insight|
+            provenance = Llmemory::Provenance.build(
+              method: "reflection",
+              sources: sources,
+              confidence: insight[:confidence]
+            )
+            @semantic.remember_fact(
+              content: insight[:content],
+              category: category,
+              importance: insight[:confidence] || DEFAULT_IMPORTANCE,
+              provenance: provenance
+            )
+          end
         end
+        result
       end
 
       private

data/lib/llmemory/retrieval/bm25_scorer.rb

@@ -53,7 +53,7 @@ module Llmemory
       private
 
       def tokenize(text)
-        text.to_s.downcase.scan(/\b[a-z0-9]{2,}\b/)
+        Llmemory::Tokenizer.tokenize(text)
       end
     end
   end

data/lib/llmemory/retrieval/engine.rb

@@ -24,9 +24,13 @@ module Llmemory
 
       def retrieve_for_inference(user_message, user_id: nil, max_tokens: nil)
         user_id ||= @memory.respond_to?(:user_id) ? @memory.user_id : nil
-        search_query = generate_query(user_message)
-        ranked = ranked_candidates(search_query, user_id, user_message)
-        @assembler.assemble(ranked, max_tokens: max_tokens)
+        result = nil
+        Llmemory::Instrumentation.instrument(:retrieve, user_id: user_id, query_chars: user_message.to_s.length) do
+          search_query = generate_query(user_message)
+          ranked = ranked_candidates(search_query, user_id, user_message)
+          result = @assembler.assemble(ranked, max_tokens: max_tokens)
+        end
+        result
       end
 
       # Multi-hop retrieval (CoALA: integrating retrieval and reasoning). After
@@ -42,21 +46,26 @@ module Llmemory
         user_id ||= @memory.respond_to?(:user_id) ? @memory.user_id : nil
         reasoner ||= method(:default_followup_query)
 
-        query = generate_query(user_message)
-        seen = []
-        accumulated = []
-        hop = 0
-
-        while hop < max_hops && live_query?(query) && !seen.include?(query)
-          seen << query
-          accumulated = merge_candidates(accumulated, ranked_candidates(query, user_id, query))
-          hop += 1
-          break if hop >= max_hops
+        final = nil
+        hops_done = 0
+        Llmemory::Instrumentation.instrument(:iterative_retrieve, user_id: user_id, query_chars: user_message.to_s.length, max_hops: max_hops) do
+          query = generate_query(user_message)
+          seen = []
+          accumulated = []
+          hop = 0
+
+          while hop < max_hops && live_query?(query) && !seen.include?(query)
+            seen << query
+            accumulated = merge_candidates(accumulated, ranked_candidates(query, user_id, query))
+            hop += 1
+            break if hop >= max_hops
+
+            query = reasoner.call(user_message, accumulated, hop).to_s.strip
+          end
+          hops_done = hop
 
-          query = reasoner.call(user_message, accumulated, hop).to_s.strip
+          final = accumulated.sort_by { |c| -(c[:temporal_score] || c[:score] || 0) }
         end
-
-        final = accumulated.sort_by { |c| -(c[:temporal_score] || c[:score] || 0) }
         @assembler.assemble(final, max_tokens: max_tokens)
       end
 

data/lib/llmemory/retrieval/mmr_reranker.rb

@@ -49,7 +49,7 @@ module Llmemory
       end
 
       def tokenize(text)
-        text.downcase.scan(/\b[a-z0-9]{2,}\b/).uniq
+        Llmemory::Tokenizer.tokenize(text).uniq
       end
     end
   end