llmemory 0.2.11 → 0.3.0

data/lib/llmemory/mcp/tools/memory_maintain.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative "../store_helpers"
+
 module Llmemory
   module MCP
     module Tools
@@ -26,7 +28,8 @@ module Llmemory
             opts[:reflection_window] = reflection_window.to_i unless reflection_window.nil?
             opts[:mining_window] = mining_window.to_i unless mining_window.nil?
 
-            report = Llmemory::Maintenance::CognitivePass.run!(user_id, **opts)
+            memory = StoreHelpers.build_memory(user_id: user_id)
+            report = Llmemory::Maintenance::CognitivePass.run!(user_id, memory: memory, **opts)
             ::MCP::Tool::Response.new([{ type: "text", text: format_report(user_id, report) }])
           rescue => e
             ::MCP::Tool::Response.new([{ type: "text", text: "Error running maintenance pass: #{e.message}" }], error: true)
@@ -42,6 +45,8 @@ module Llmemory
               "  skills mined: #{Array(report[:mined]).size}",
               "  expired: episodic=#{expired[:episodic] || 0} procedural=#{expired[:procedural] || 0}"
             ]
+            disabled = Array(report[:disabled])
+            lines << "  disabled: #{disabled.join(', ')}" if disabled.any?
             errors = report[:errors] || {}
             lines << "  errors: #{errors.map { |k, v| "#{k}: #{v}" }.join('; ')}" unless errors.empty?
             lines.join("\n")

data/lib/llmemory/mcp/tools/memory_retrieve.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative "../store_helpers"
+
 module Llmemory
   module MCP
     module Tools
@@ -25,7 +27,7 @@ module Llmemory
             include_timeline = include_timeline_context == true
             window = timeline_window || 3
 
-            memory = Llmemory::Memory.new(user_id: user_id, session_id: session)
+            memory = StoreHelpers.build_memory(user_id: user_id, session_id: session)
             context = memory.retrieve(query, max_tokens: tokens)
 
             # Add timeline context if requested

data/lib/llmemory/mcp/tools/memory_retrieve_evidence.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require "json"
+require_relative "../store_helpers"
+
+module Llmemory
+  module MCP
+    module Tools
+      class MemoryRetrieveEvidence < ::MCP::Tool
+        description "Retrieve structured Zero-Mem evidence (traces) for a query without generative LLM calls. Returns JSON plus a text context block."
+
+        input_schema(
+          properties: {
+            query: { type: "string", description: "Evidence query" },
+            user_id: { type: "string", description: "User identifier" },
+            session_id: { type: "string", description: "Session identifier (default: 'default')" },
+            top_k: { type: "integer", description: "Maximum seed traces (default from config)" },
+            max_tokens: { type: "integer", description: "Token budget for assembled evidence" },
+            boundary: { type: "object", description: "Optional session/boundary filter" },
+            explain: { type: "boolean", description: "Include routing/fusion explain payload (default false)" }
+          },
+          required: ["query", "user_id"]
+        )
+
+        class << self
+          def call(query:, user_id:, session_id: nil, top_k: nil, max_tokens: nil, boundary: nil,
+                   explain: false, server_context: nil)
+            unless ZeroMem::Mode.zero_mem_enabled?(Llmemory.configuration.memory_mode)
+              return error_response("memory_retrieve_evidence requires memory_mode :zero_mem or :hybrid")
+            end
+
+            session = session_id || "default"
+            memory = StoreHelpers.build_memory(user_id: user_id, session_id: session)
+            result = memory.retrieve_evidence(
+              query,
+              top_k: top_k,
+              max_tokens: max_tokens,
+              boundary: boundary,
+              explain: explain == true
+            )
+
+            payload = {
+              route: result.route,
+              profile: result.profile.to_h,
+              evidence: result.evidence.map(&:to_h),
+              seed_evidence: result.seed_evidence.map(&:to_h),
+              closure_evidence: result.closure_evidence.map(&:to_h),
+              metrics: result.metrics,
+              degraded: result.degraded,
+              warnings: result.warnings,
+              explain: result.explain,
+              context: result.to_context
+            }
+
+            ::MCP::Tool::Response.new([{
+              type: "text",
+              text: JSON.pretty_generate(payload)
+            }])
+          rescue => e
+            error_response(e.message)
+          end
+
+          def error_response(message)
+            ::MCP::Tool::Response.new([{ type: "text", text: "Error retrieving evidence: #{message}" }], error: true)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/llmemory/mcp/tools/memory_stats.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative "../store_helpers"
+
 module Llmemory
   module MCP
     module Tools
@@ -53,6 +55,16 @@ module Llmemory
             end
 
             stats[:llm_usage] = Llmemory::LLM::UsageLedger.new(store: store).totals(user_id)
+            stats[:memory_mode] = Llmemory.configuration.memory_mode
+            if ZeroMem::Mode.zero_mem_enabled?(Llmemory.configuration.memory_mode)
+              memory = StoreHelpers.build_memory(user_id: user_id)
+              stats[:zero_mem] = memory.zero_mem_status
+              invoke_calls = stats[:llm_usage].dig(:invoke, :calls).to_i
+              stats[:zero_mem_compliant] = ZeroMem::Mode.zero_mem_strict?(Llmemory.configuration.memory_mode) &&
+                                           invoke_calls.zero?
+            else
+              stats[:zero_mem_compliant] = false
+            end
 
             ::MCP::Tool::Response.new([{
               type: "text",
@@ -106,6 +118,16 @@ module Llmemory
 
             output << ""
             output << Llmemory::LLM::UsageLedger.format_text(stats[:llm_usage] || default_llm_usage)
+            output << ""
+            output << "MEMORY MODE: #{stats[:memory_mode]}"
+            if stats[:zero_mem]
+              zm = stats[:zero_mem]
+              output << "ZERO-MEM:"
+              output << "  traces: #{zm[:trace_count]}"
+              output << "  index_lag: #{zm[:index_lag]}"
+              output << "  index_version: #{zm[:index_version]}"
+            end
+            output << "  zero_mem_compliant: #{stats[:zero_mem_compliant]}"
 
             output.join("\n")
           end

data/lib/llmemory/memory.rb

@@ -10,9 +10,20 @@ module Llmemory
     DEFAULT_SESSION_ID = "default"
     STATE_KEY_MESSAGES = :messages
 
-    def initialize(user_id:, session_id: DEFAULT_SESSION_ID, checkpoint: nil, long_term: nil, long_term_type: nil, retrieval_engine: nil, working_memory: nil, episodic: nil, procedural: nil, api_key: nil, encryption_key: :inherit)
+    def initialize(user_id:, session_id: DEFAULT_SESSION_ID, checkpoint: nil, long_term: nil, long_term_type: nil,
+                   retrieval_engine: nil, working_memory: nil, episodic: nil, procedural: nil, api_key: nil,
+                   encryption_key: :inherit, trace_store: nil, compact_strategy: nil, forget_log: nil,
+                   memory_mode: nil)
       @user_id = user_id
       @session_id = session_id
+      @memory_mode = ZeroMem::Mode.normalize(memory_mode || Llmemory.configuration.memory_mode)
+      @trace_store = trace_store
+      if @trace_store.nil? && (zero_mem_enabled? || shadow_write_enabled?)
+        @trace_store = ZeroMem::Storages.build
+      end
+      @compact_strategy = compact_strategy
+      @forget_log = forget_log
+      @trace_indexer = @trace_store ? ZeroMem::Indexer.new(@trace_store) : nil
       resolved_key = encryption_key == :inherit ? nil : encryption_key
       @cipher = Llmemory.build_cipher(resolved_key)
       if checkpoint
@@ -74,6 +85,7 @@ module Llmemory
     # Reflects over recent episodes and writes distilled insights to the
     # semantic store (file/graph) with provenance back to source episodes.
     def reflect!(window: 10, category: "insights")
+      deny_generative!(:reflect!) if zero_mem_strict?
       Reflection::Reflector.new(episodic: episodic, semantic: @long_term, llm: tracked_llm_client)
         .reflect(window: window, category: category)
     end
@@ -89,6 +101,7 @@ module Llmemory
     # `auto_register: true`, registers them in procedural memory (with provenance
     # back to the source episodes) and returns the new skill ids.
     def mine_skills!(window: SkillMining::Miner::DEFAULT_WINDOW, outcomes: nil, auto_register: false)
+      deny_generative!(:mine_skills!) if zero_mem_strict?
       SkillMining::Miner.new(episodic: episodic, procedural: procedural, llm: tracked_llm_client)
         .mine(window: window, outcomes: outcomes, auto_register: auto_register)
     end
@@ -104,18 +117,135 @@ module Llmemory
       )
     end
 
-    def add_message(role:, content:)
-      @short_term_store.update(@user_id, @session_id) do |state|
-        state = normalize_state_hash(state)
-        list = state[STATE_KEY_MESSAGES]
-        list = list.is_a?(Array) ? list.dup : []
-        list << { role: role.to_sym, content: content.to_s }
-        list = sanitize_messages(list) if Llmemory.configuration.message_sanitizer_enabled
-        state.merge(STATE_KEY_MESSAGES => list, last_activity_at: Time.now, **preserved_flush_state_from(state))
+    def add_message(role:, content:, occurred_at: nil, boundary_id: nil, metadata: nil, idempotency_key: nil)
+      if @trace_store
+        record_trace(
+          role: role,
+          content: content,
+          occurred_at: occurred_at,
+          boundary_id: boundary_id,
+          metadata: metadata,
+          idempotency_key: idempotency_key,
+          add_to_checkpoint: true
+        )
+      else
+        append_message_to_checkpoint(role: role, content: content)
+      end
+      true
+    end
+
+    # Persists an immutable trace before updating the checkpoint (Zero-Mem source of truth).
+    def record_trace(role:, content:, occurred_at: nil, boundary_id: nil, metadata: nil, idempotency_key: nil,
+                     state_key: nil, valid_from: nil, valid_to: nil, supersedes_trace_id: nil, source: :explicit,
+                     add_to_checkpoint: true)
+      raise ConfigurationError, "trace_store is not configured" unless @trace_store
+
+      if idempotency_key && (existing = @trace_store.find_by_idempotency_key(@user_id, idempotency_key))
+        return existing.id
+      end
+
+      sequence = @trace_store.next_sequence(@user_id, @session_id)
+      trace = ZeroMem::Trace.build(
+        user_id: @user_id,
+        session_id: @session_id,
+        role: role,
+        content: content,
+        sequence: sequence,
+        boundary_id: boundary_id,
+        occurred_at: occurred_at,
+        metadata: metadata,
+        idempotency_key: idempotency_key
+      )
+
+      index_stats = {}
+      Llmemory::Instrumentation.instrument(
+        :trace_write,
+        user_id: @user_id,
+        session_id: @session_id,
+        trace_id: trace.id,
+        sequence: sequence,
+        role: trace.role,
+        maintenance_fanout: state_key ? 2 : 1
+      ) do
+        @trace_store.write_trace(trace)
+        if state_key
+          effective_from = valid_from || trace.occurred_at
+          if supersedes_trace_id
+            @trace_store.close_open_state_links(
+              @user_id,
+              state_key,
+              valid_to: effective_from,
+              except_trace_id: trace.id
+            )
+          end
+          link = ZeroMem::TraceStateLink.build(
+            user_id: @user_id,
+            state_key: state_key,
+            trace_id: trace.id,
+            valid_from: effective_from,
+            valid_to: valid_to,
+            supersedes_trace_id: supersedes_trace_id,
+            source: source
+          )
+          @trace_store.write_state_link(link)
+          Llmemory::Instrumentation.instrument(
+            :zero_mem_state_update,
+            user_id: @user_id,
+            state_key: state_key,
+            trace_id: trace.id,
+            supersedes_trace_id: supersedes_trace_id
+          )
+        end
+        index_stats = @trace_indexer.after_trace_write(
+          trace: trace,
+          state_link: !state_key.nil?
+        )
       end
+
+      append_message_to_checkpoint(role: trace.role, content: trace.content) if add_to_checkpoint
+      trace.id
+    end
+
+    def retrieve_evidence(query, top_k: nil, max_tokens: nil, boundary: nil, explain: false, current_trace_id: nil,
+                          **opts)
+      raise ConfigurationError, "retrieve_evidence requires memory_mode :zero_mem or :hybrid" unless zero_mem_enabled?
+      raise ConfigurationError, "trace_store is not configured" unless @trace_store
+
+      invoke_before = generative_invoke_calls
+      result = zero_mem_engine.retrieve_evidence(
+        query,
+        top_k: top_k,
+        max_tokens: max_tokens,
+        boundary: boundary,
+        explain: explain,
+        current_trace_id: current_trace_id,
+        **opts
+      )
+      compliant = generative_invoke_delta(invoke_before).zero?
+      result.metrics[:zero_mem_compliant] = compliant if zero_mem_strict?
+      result
+    end
+
+    def calibrate_answer(answer, evidence_result:)
+      ZeroMem::AnswerCalibrator.new.calibrate(answer, evidence_result: evidence_result)
+    end
+
+    def forget_traces!(trace_ids, reason: nil)
+      return false unless @trace_store
+
+      ids = Array(trace_ids).map(&:to_s)
+      ids.each { |id| @trace_store.archive_trace(@user_id, id) }
+      forget_log.record(
+        @user_id,
+        memory_type: ZeroMem::MEMORY_TYPE,
+        ids: ids,
+        reason: reason
+      )
       true
     end
 
+    attr_reader :trace_store
+
     def messages
       state = @checkpoint.restore_state
       return [] unless state.is_a?(Hash)
@@ -125,6 +255,21 @@ module Llmemory
     end
 
     def retrieve(query, max_tokens: nil)
+      if zero_mem_enabled? && @trace_store
+        invoke_before = generative_invoke_calls
+        msgs = pruned_messages
+        short_context = format_short_term_context(msgs)
+        evidence_context = zero_mem_engine.to_context(query, max_tokens: max_tokens)
+        combined = combine_contexts(short_context, evidence_context)
+        compliant = generative_invoke_delta(invoke_before).zero?
+        Llmemory::Instrumentation.instrument(
+          :retrieve,
+          query_chars: query.to_s.length,
+          zero_mem_compliant: zero_mem_strict? ? compliant : false
+        )
+        return combined
+      end
+
       msgs = pruned_messages
       short_context = format_short_term_context(msgs)
       long_context = @retrieval_engine.retrieve_for_inference(query, user_id: @user_id, max_tokens: max_tokens)
@@ -162,6 +307,7 @@ module Llmemory
     end
 
     def consolidate!
+      deny_generative!(:consolidate!) if zero_mem_strict?
       msgs = messages
       return true if msgs.empty?
       conversation_text = msgs.map { |m| format_message(m) }.join("\n")
@@ -177,6 +323,8 @@ module Llmemory
 
     def compact!(max_bytes: nil)
       max = max_bytes || Llmemory.configuration.compact_max_bytes
+      return compact_trace_deterministic!(max) if trace_backed? || zero_mem_strict?
+
       msgs = messages
       current_bytes = messages_byte_size(msgs)
       return false if current_bytes <= max
@@ -195,6 +343,7 @@ module Llmemory
     end
 
     def maybe_flush_memory!
+      return false if zero_mem_strict?
       return false unless Llmemory.configuration.memory_flush_enabled
       msgs = messages
       return false if msgs.empty?
@@ -241,6 +390,11 @@ module Llmemory
     def check_context_window!
       return false if messages.empty?
 
+      if zero_mem_strict?
+        return compact! if should_compact?
+        return false
+      end
+
       flushed = false
       if should_auto_consolidate? && Llmemory.configuration.memory_flush_enabled
         consolidate!
@@ -255,6 +409,50 @@ module Llmemory
       flushed || compacted
     end
 
+    def zero_mem_status(session_id: @session_id)
+      raise ConfigurationError, "trace_store is not configured" unless @trace_store
+
+      wm = @trace_store.get_watermark(@user_id, session_id)
+      traces = @trace_store.list_traces(@user_id, session_id: session_id)
+      last_seq = traces.map(&:sequence).max.to_i
+      lag = [last_seq - wm[:last_sequence].to_i, 0].max
+      {
+        memory_mode: @memory_mode,
+        session_id: session_id.to_s,
+        trace_count: traces.size,
+        last_sequence: last_seq,
+        watermark_sequence: wm[:last_sequence].to_i,
+        index_version: wm[:index_version].to_i,
+        index_lag: lag
+      }
+    end
+
+    def reindex_traces!(session_id: @session_id)
+      raise ConfigurationError, "trace_store is not configured" unless @trace_store
+
+      traces = @trace_store.list_traces(@user_id, session_id: session_id)
+      traces.each do |trace|
+        @trace_indexer.after_trace_write(trace: trace)
+      end
+      zero_mem_status(session_id: session_id)
+    end
+
+    def memory_mode
+      @memory_mode
+    end
+
+    def zero_mem_enabled?
+      ZeroMem::Mode.zero_mem_enabled?(@memory_mode)
+    end
+
+    def zero_mem_strict?
+      ZeroMem::Mode.zero_mem_strict?(@memory_mode)
+    end
+
+    def shadow_write_enabled?
+      @memory_mode == :classic && Llmemory.configuration.zero_mem_shadow_write
+    end
+
     def user_id
       @user_id
     end
@@ -265,6 +463,43 @@ module Llmemory
 
     private
 
+    def forget_log
+      @forget_log ||= ForgetLog.new(store: @short_term_store)
+    end
+
+    def zero_mem_engine
+      @zero_mem_engine ||= ZeroMem::Engine.new(trace_store: @trace_store, user_id: @user_id)
+    end
+
+    def trace_backed?
+      !@trace_store.nil?
+    end
+
+    def compact_trace_deterministic!(max_bytes)
+      msgs = messages
+      current_bytes = messages_byte_size(msgs)
+      return false if current_bytes <= max_bytes
+
+      old_msgs, recent_msgs = split_messages_by_bytes(msgs, max_bytes)
+      return false if old_msgs.empty?
+
+      state = restore_state_for_save
+      flush_ts = state[:last_flush_at] || state["last_flush_at"]
+      save_state(messages: recent_msgs, last_compact_at: Time.now, last_flush_at: flush_ts)
+      true
+    end
+
+    def append_message_to_checkpoint(role:, content:)
+      @short_term_store.update(@user_id, @session_id) do |state|
+        state = normalize_state_hash(state)
+        list = state[STATE_KEY_MESSAGES]
+        list = list.is_a?(Array) ? list.dup : []
+        list << { role: role.to_sym, content: content.to_s }
+        list = sanitize_messages(list) if Llmemory.configuration.message_sanitizer_enabled
+        state.merge(STATE_KEY_MESSAGES => list, last_activity_at: Time.now, **preserved_flush_state_from(state))
+      end
+    end
+
     def summarize_messages(msgs)
       conversation = msgs.map { |m| format_message(m) }.join("\n")
       prompt = <<~PROMPT
@@ -293,7 +528,20 @@ module Llmemory
       )
     end
 
+    def deny_generative!(operation)
+      raise GenerativeOperationDisabled, "#{operation} is disabled when memory_mode is :zero_mem"
+    end
+
+    def generative_invoke_calls
+      llm_usage.dig(:invoke, :calls).to_i
+    end
+
+    def generative_invoke_delta(before)
+      generative_invoke_calls - before.to_i
+    end
+
     def flush_memory_before_compaction!(msgs)
+      return false if zero_mem_strict?
       return false unless Llmemory.configuration.memory_flush_enabled
       return false if msgs.empty?
       return false if estimated_tokens(msgs) < Llmemory.configuration.memory_flush_threshold_tokens

data/lib/llmemory/retrieval/bm25_scorer.rb

@@ -11,6 +11,11 @@ module Llmemory
         @b = b
       end
 
+      # Score arbitrary documents `{ id:, text: }` (or candidates with :text).
+      def score_documents(query, documents)
+        score_candidates(query, documents)
+      end
+
       def score_candidates(query, candidates)
         return [] if candidates.empty?
 

data/lib/llmemory/tokenizer.rb

@@ -2,25 +2,35 @@
 
 module Llmemory
   # Shared word tokenizer for keyword search and lexical scoring (BM25, MMR).
-  # Centralizes the tokenization regex that was duplicated across the codebase.
+  # Unicode-aware (NFKC, accented letters, CJK, digits).
   module Tokenizer
     module_function
 
-    WORD = /\b[a-z0-9]{2,}\b/
+    # Latin letters (2+), single Han ideographs, digit runs.
+    TOKEN_PATTERN = /\d+|\p{Han}+|[\p{L}&&[^\p{Han}]]{2,}/u
+
+    def normalize(text)
+      str = text.to_s
+      unless str.encoding == Encoding::UTF_8 && str.valid_encoding?
+        str = str.encode(Encoding::UTF_8, invalid: :replace, undef: :replace)
+      end
+      str.unicode_normalize(:nfkc).downcase
+    end
 
     def tokenize(text)
-      text.to_s.downcase.scan(WORD)
+      normalized = normalize(text)
+      return [] if normalized.strip.empty?
+
+      normalized.scan(TOKEN_PATTERN)
     end
 
     # Lexical match used by storage-level keyword search. A query is split into
-    # tokens and matched as an OR of per-token substrings, so multi-word queries
-    # work (a single contiguous substring of the whole query is no longer
-    # required) while single-term/partial matches are preserved. An empty query
-    # (no tokens) matches everything, keeping prior "return all" behavior.
+    # tokens and matched as an OR of per-token substrings.
     def matches?(text, query)
       tokens = tokenize(query)
       return true if tokens.empty?
-      haystack = text.to_s.downcase
+
+      haystack = normalize(text)
       tokens.any? { |t| haystack.include?(t) }
     end
   end

data/lib/llmemory/vector_store/memory_store.rb

@@ -12,10 +12,14 @@ module Llmemory
       end
 
       def embed(text)
-        return Array.new(1536, 0.0) unless @embedding_provider&.respond_to?(:embed)
+        return Array.new(default_dimensions, 0.0) unless @embedding_provider&.respond_to?(:embed)
         @embedding_provider.embed(text)
       end
 
+      def embedding_dimensions
+        default_dimensions
+      end
+
       def last_usage
         return @embedding_provider.last_usage if @embedding_provider&.respond_to?(:last_usage)
 
@@ -23,6 +27,7 @@ module Llmemory
       end
 
       def store(id:, embedding:, metadata: {}, user_id: nil)
+        validate_dimensions!(embedding)
         key = user_id ? "#{user_id}:#{id}" : id.to_s
         meta = (metadata || {}).dup
         if meta["text"] && @cipher.enabled?
@@ -40,6 +45,7 @@ module Llmemory
 
       def search(query_embedding, top_k: 10, user_id: nil)
         query = query_embedding.to_a.map(&:to_f)
+        validate_dimensions!(query)
         return [] if query.empty?
         entries = user_id ? @entries.select { |k, _| k.to_s.start_with?("#{user_id}:") } : @entries
         scores = entries.map do |_key, data|
@@ -79,6 +85,22 @@ module Llmemory
         out
       end
 
+      def default_dimensions
+        if @embedding_provider.respond_to?(:dimensions)
+          @embedding_provider.dimensions
+        else
+          1536
+        end
+      end
+
+      def validate_dimensions!(vector)
+        dims = default_dimensions
+        size = vector.to_a.size
+        return if size == dims
+
+        raise ConfigurationError, "embedding dimension mismatch: expected #{dims}, got #{size}"
+      end
+
       def cosine_similarity(a, b)
         return 0.0 if a.size != b.size || a.empty?
         dot = a.zip(b).sum { |x, y| x * y }