llmemory 0.2.10 → 0.3.0

data/lib/llmemory/memory.rb

@@ -10,9 +10,20 @@ module Llmemory
     DEFAULT_SESSION_ID = "default"
     STATE_KEY_MESSAGES = :messages
 
-    def initialize(user_id:, session_id: DEFAULT_SESSION_ID, checkpoint: nil, long_term: nil, long_term_type: nil, retrieval_engine: nil, working_memory: nil, episodic: nil, procedural: nil, api_key: nil, encryption_key: :inherit)
+    def initialize(user_id:, session_id: DEFAULT_SESSION_ID, checkpoint: nil, long_term: nil, long_term_type: nil,
+                   retrieval_engine: nil, working_memory: nil, episodic: nil, procedural: nil, api_key: nil,
+                   encryption_key: :inherit, trace_store: nil, compact_strategy: nil, forget_log: nil,
+                   memory_mode: nil)
       @user_id = user_id
       @session_id = session_id
+      @memory_mode = ZeroMem::Mode.normalize(memory_mode || Llmemory.configuration.memory_mode)
+      @trace_store = trace_store
+      if @trace_store.nil? && (zero_mem_enabled? || shadow_write_enabled?)
+        @trace_store = ZeroMem::Storages.build
+      end
+      @compact_strategy = compact_strategy
+      @forget_log = forget_log
+      @trace_indexer = @trace_store ? ZeroMem::Indexer.new(@trace_store) : nil
       resolved_key = encryption_key == :inherit ? nil : encryption_key
       @cipher = Llmemory.build_cipher(resolved_key)
       if checkpoint
@@ -74,6 +85,7 @@ module Llmemory
     # Reflects over recent episodes and writes distilled insights to the
     # semantic store (file/graph) with provenance back to source episodes.
     def reflect!(window: 10, category: "insights")
+      deny_generative!(:reflect!) if zero_mem_strict?
       Reflection::Reflector.new(episodic: episodic, semantic: @long_term, llm: tracked_llm_client)
         .reflect(window: window, category: category)
     end
@@ -89,6 +101,7 @@ module Llmemory
     # `auto_register: true`, registers them in procedural memory (with provenance
     # back to the source episodes) and returns the new skill ids.
     def mine_skills!(window: SkillMining::Miner::DEFAULT_WINDOW, outcomes: nil, auto_register: false)
+      deny_generative!(:mine_skills!) if zero_mem_strict?
       SkillMining::Miner.new(episodic: episodic, procedural: procedural, llm: tracked_llm_client)
         .mine(window: window, outcomes: outcomes, auto_register: auto_register)
     end
@@ -104,18 +117,135 @@ module Llmemory
       )
     end
 
-    def add_message(role:, content:)
-      @short_term_store.update(@user_id, @session_id) do |state|
-        state = normalize_state_hash(state)
-        list = state[STATE_KEY_MESSAGES]
-        list = list.is_a?(Array) ? list.dup : []
-        list << { role: role.to_sym, content: content.to_s }
-        list = sanitize_messages(list) if Llmemory.configuration.message_sanitizer_enabled
-        state.merge(STATE_KEY_MESSAGES => list, last_activity_at: Time.now, **preserved_flush_state_from(state))
+    def add_message(role:, content:, occurred_at: nil, boundary_id: nil, metadata: nil, idempotency_key: nil)
+      if @trace_store
+        record_trace(
+          role: role,
+          content: content,
+          occurred_at: occurred_at,
+          boundary_id: boundary_id,
+          metadata: metadata,
+          idempotency_key: idempotency_key,
+          add_to_checkpoint: true
+        )
+      else
+        append_message_to_checkpoint(role: role, content: content)
+      end
+      true
+    end
+
+    # Persists an immutable trace before updating the checkpoint (Zero-Mem source of truth).
+    def record_trace(role:, content:, occurred_at: nil, boundary_id: nil, metadata: nil, idempotency_key: nil,
+                     state_key: nil, valid_from: nil, valid_to: nil, supersedes_trace_id: nil, source: :explicit,
+                     add_to_checkpoint: true)
+      raise ConfigurationError, "trace_store is not configured" unless @trace_store
+
+      if idempotency_key && (existing = @trace_store.find_by_idempotency_key(@user_id, idempotency_key))
+        return existing.id
+      end
+
+      sequence = @trace_store.next_sequence(@user_id, @session_id)
+      trace = ZeroMem::Trace.build(
+        user_id: @user_id,
+        session_id: @session_id,
+        role: role,
+        content: content,
+        sequence: sequence,
+        boundary_id: boundary_id,
+        occurred_at: occurred_at,
+        metadata: metadata,
+        idempotency_key: idempotency_key
+      )
+
+      index_stats = {}
+      Llmemory::Instrumentation.instrument(
+        :trace_write,
+        user_id: @user_id,
+        session_id: @session_id,
+        trace_id: trace.id,
+        sequence: sequence,
+        role: trace.role,
+        maintenance_fanout: state_key ? 2 : 1
+      ) do
+        @trace_store.write_trace(trace)
+        if state_key
+          effective_from = valid_from || trace.occurred_at
+          if supersedes_trace_id
+            @trace_store.close_open_state_links(
+              @user_id,
+              state_key,
+              valid_to: effective_from,
+              except_trace_id: trace.id
+            )
+          end
+          link = ZeroMem::TraceStateLink.build(
+            user_id: @user_id,
+            state_key: state_key,
+            trace_id: trace.id,
+            valid_from: effective_from,
+            valid_to: valid_to,
+            supersedes_trace_id: supersedes_trace_id,
+            source: source
+          )
+          @trace_store.write_state_link(link)
+          Llmemory::Instrumentation.instrument(
+            :zero_mem_state_update,
+            user_id: @user_id,
+            state_key: state_key,
+            trace_id: trace.id,
+            supersedes_trace_id: supersedes_trace_id
+          )
+        end
+        index_stats = @trace_indexer.after_trace_write(
+          trace: trace,
+          state_link: !state_key.nil?
+        )
       end
+
+      append_message_to_checkpoint(role: trace.role, content: trace.content) if add_to_checkpoint
+      trace.id
+    end
+
+    def retrieve_evidence(query, top_k: nil, max_tokens: nil, boundary: nil, explain: false, current_trace_id: nil,
+                          **opts)
+      raise ConfigurationError, "retrieve_evidence requires memory_mode :zero_mem or :hybrid" unless zero_mem_enabled?
+      raise ConfigurationError, "trace_store is not configured" unless @trace_store
+
+      invoke_before = generative_invoke_calls
+      result = zero_mem_engine.retrieve_evidence(
+        query,
+        top_k: top_k,
+        max_tokens: max_tokens,
+        boundary: boundary,
+        explain: explain,
+        current_trace_id: current_trace_id,
+        **opts
+      )
+      compliant = generative_invoke_delta(invoke_before).zero?
+      result.metrics[:zero_mem_compliant] = compliant if zero_mem_strict?
+      result
+    end
+
+    def calibrate_answer(answer, evidence_result:)
+      ZeroMem::AnswerCalibrator.new.calibrate(answer, evidence_result: evidence_result)
+    end
+
+    def forget_traces!(trace_ids, reason: nil)
+      return false unless @trace_store
+
+      ids = Array(trace_ids).map(&:to_s)
+      ids.each { |id| @trace_store.archive_trace(@user_id, id) }
+      forget_log.record(
+        @user_id,
+        memory_type: ZeroMem::MEMORY_TYPE,
+        ids: ids,
+        reason: reason
+      )
       true
     end
 
+    attr_reader :trace_store
+
     def messages
       state = @checkpoint.restore_state
       return [] unless state.is_a?(Hash)
@@ -125,6 +255,21 @@ module Llmemory
     end
 
     def retrieve(query, max_tokens: nil)
+      if zero_mem_enabled? && @trace_store
+        invoke_before = generative_invoke_calls
+        msgs = pruned_messages
+        short_context = format_short_term_context(msgs)
+        evidence_context = zero_mem_engine.to_context(query, max_tokens: max_tokens)
+        combined = combine_contexts(short_context, evidence_context)
+        compliant = generative_invoke_delta(invoke_before).zero?
+        Llmemory::Instrumentation.instrument(
+          :retrieve,
+          query_chars: query.to_s.length,
+          zero_mem_compliant: zero_mem_strict? ? compliant : false
+        )
+        return combined
+      end
+
       msgs = pruned_messages
       short_context = format_short_term_context(msgs)
       long_context = @retrieval_engine.retrieve_for_inference(query, user_id: @user_id, max_tokens: max_tokens)
@@ -162,6 +307,7 @@ module Llmemory
     end
 
     def consolidate!
+      deny_generative!(:consolidate!) if zero_mem_strict?
       msgs = messages
       return true if msgs.empty?
       conversation_text = msgs.map { |m| format_message(m) }.join("\n")
@@ -177,6 +323,8 @@ module Llmemory
 
     def compact!(max_bytes: nil)
       max = max_bytes || Llmemory.configuration.compact_max_bytes
+      return compact_trace_deterministic!(max) if trace_backed? || zero_mem_strict?
+
       msgs = messages
       current_bytes = messages_byte_size(msgs)
       return false if current_bytes <= max
@@ -195,6 +343,7 @@ module Llmemory
     end
 
     def maybe_flush_memory!
+      return false if zero_mem_strict?
       return false unless Llmemory.configuration.memory_flush_enabled
       msgs = messages
       return false if msgs.empty?
@@ -241,6 +390,11 @@ module Llmemory
     def check_context_window!
       return false if messages.empty?
 
+      if zero_mem_strict?
+        return compact! if should_compact?
+        return false
+      end
+
       flushed = false
       if should_auto_consolidate? && Llmemory.configuration.memory_flush_enabled
         consolidate!
@@ -255,6 +409,50 @@ module Llmemory
       flushed || compacted
     end
 
+    def zero_mem_status(session_id: @session_id)
+      raise ConfigurationError, "trace_store is not configured" unless @trace_store
+
+      wm = @trace_store.get_watermark(@user_id, session_id)
+      traces = @trace_store.list_traces(@user_id, session_id: session_id)
+      last_seq = traces.map(&:sequence).max.to_i
+      lag = [last_seq - wm[:last_sequence].to_i, 0].max
+      {
+        memory_mode: @memory_mode,
+        session_id: session_id.to_s,
+        trace_count: traces.size,
+        last_sequence: last_seq,
+        watermark_sequence: wm[:last_sequence].to_i,
+        index_version: wm[:index_version].to_i,
+        index_lag: lag
+      }
+    end
+
+    def reindex_traces!(session_id: @session_id)
+      raise ConfigurationError, "trace_store is not configured" unless @trace_store
+
+      traces = @trace_store.list_traces(@user_id, session_id: session_id)
+      traces.each do |trace|
+        @trace_indexer.after_trace_write(trace: trace)
+      end
+      zero_mem_status(session_id: session_id)
+    end
+
+    def memory_mode
+      @memory_mode
+    end
+
+    def zero_mem_enabled?
+      ZeroMem::Mode.zero_mem_enabled?(@memory_mode)
+    end
+
+    def zero_mem_strict?
+      ZeroMem::Mode.zero_mem_strict?(@memory_mode)
+    end
+
+    def shadow_write_enabled?
+      @memory_mode == :classic && Llmemory.configuration.zero_mem_shadow_write
+    end
+
     def user_id
       @user_id
     end
@@ -265,6 +463,43 @@ module Llmemory
 
     private
 
+    def forget_log
+      @forget_log ||= ForgetLog.new(store: @short_term_store)
+    end
+
+    def zero_mem_engine
+      @zero_mem_engine ||= ZeroMem::Engine.new(trace_store: @trace_store, user_id: @user_id)
+    end
+
+    def trace_backed?
+      !@trace_store.nil?
+    end
+
+    def compact_trace_deterministic!(max_bytes)
+      msgs = messages
+      current_bytes = messages_byte_size(msgs)
+      return false if current_bytes <= max_bytes
+
+      old_msgs, recent_msgs = split_messages_by_bytes(msgs, max_bytes)
+      return false if old_msgs.empty?
+
+      state = restore_state_for_save
+      flush_ts = state[:last_flush_at] || state["last_flush_at"]
+      save_state(messages: recent_msgs, last_compact_at: Time.now, last_flush_at: flush_ts)
+      true
+    end
+
+    def append_message_to_checkpoint(role:, content:)
+      @short_term_store.update(@user_id, @session_id) do |state|
+        state = normalize_state_hash(state)
+        list = state[STATE_KEY_MESSAGES]
+        list = list.is_a?(Array) ? list.dup : []
+        list << { role: role.to_sym, content: content.to_s }
+        list = sanitize_messages(list) if Llmemory.configuration.message_sanitizer_enabled
+        state.merge(STATE_KEY_MESSAGES => list, last_activity_at: Time.now, **preserved_flush_state_from(state))
+      end
+    end
+
     def summarize_messages(msgs)
       conversation = msgs.map { |m| format_message(m) }.join("\n")
       prompt = <<~PROMPT
@@ -293,7 +528,20 @@ module Llmemory
       )
     end
 
+    def deny_generative!(operation)
+      raise GenerativeOperationDisabled, "#{operation} is disabled when memory_mode is :zero_mem"
+    end
+
+    def generative_invoke_calls
+      llm_usage.dig(:invoke, :calls).to_i
+    end
+
+    def generative_invoke_delta(before)
+      generative_invoke_calls - before.to_i
+    end
+
     def flush_memory_before_compaction!(msgs)
+      return false if zero_mem_strict?
       return false unless Llmemory.configuration.memory_flush_enabled
       return false if msgs.empty?
       return false if estimated_tokens(msgs) < Llmemory.configuration.memory_flush_threshold_tokens

data/lib/llmemory/retrieval/bm25_scorer.rb

@@ -11,6 +11,11 @@ module Llmemory
         @b = b
       end
 
+      # Score arbitrary documents `{ id:, text: }` (or candidates with :text).
+      def score_documents(query, documents)
+        score_candidates(query, documents)
+      end
+
       def score_candidates(query, candidates)
         return [] if candidates.empty?
 

data/lib/llmemory/retrieval/context_assembler.rb

@@ -14,13 +14,15 @@ module Llmemory
 
         ranked_memories.each do |memory|
           text = memory[:text] || memory["text"] || ""
+          text = label_volatile(text) if volatile_memory?(memory)
           memory_tokens = count_tokens(text)
           break if token_count + memory_tokens > max_tokens
 
           selected << {
             text: text,
             timestamp: memory[:timestamp] || memory["timestamp"],
-            confidence: memory[:temporal_score] || memory[:score] || memory["score"]
+            confidence: memory[:temporal_score] || memory[:score] || memory["score"],
+            volatile: volatile_memory?(memory)
           }
           token_count += memory_tokens
         end
@@ -48,6 +50,20 @@ module Llmemory
         lines << "=== END MEMORIES ==="
         lines.join("\n")
       end
+
+      def volatile_memory?(memory)
+        memory[:volatile] == true || memory["volatile"] == true
+      end
+
+      def label_volatile_enabled?
+        Llmemory.configuration.retrieval_label_volatile
+      end
+
+      def label_volatile(text)
+        return text unless label_volatile_enabled?
+
+        "[verify live] #{text}"
+      end
     end
   end
 end

data/lib/llmemory/tokenizer.rb

@@ -2,25 +2,35 @@
 
 module Llmemory
   # Shared word tokenizer for keyword search and lexical scoring (BM25, MMR).
-  # Centralizes the tokenization regex that was duplicated across the codebase.
+  # Unicode-aware (NFKC, accented letters, CJK, digits).
   module Tokenizer
     module_function
 
-    WORD = /\b[a-z0-9]{2,}\b/
+    # Latin letters (2+), single Han ideographs, digit runs.
+    TOKEN_PATTERN = /\d+|\p{Han}+|[\p{L}&&[^\p{Han}]]{2,}/u
+
+    def normalize(text)
+      str = text.to_s
+      unless str.encoding == Encoding::UTF_8 && str.valid_encoding?
+        str = str.encode(Encoding::UTF_8, invalid: :replace, undef: :replace)
+      end
+      str.unicode_normalize(:nfkc).downcase
+    end
 
     def tokenize(text)
-      text.to_s.downcase.scan(WORD)
+      normalized = normalize(text)
+      return [] if normalized.strip.empty?
+
+      normalized.scan(TOKEN_PATTERN)
     end
 
     # Lexical match used by storage-level keyword search. A query is split into
-    # tokens and matched as an OR of per-token substrings, so multi-word queries
-    # work (a single contiguous substring of the whole query is no longer
-    # required) while single-term/partial matches are preserved. An empty query
-    # (no tokens) matches everything, keeping prior "return all" behavior.
+    # tokens and matched as an OR of per-token substrings.
     def matches?(text, query)
       tokens = tokenize(query)
       return true if tokens.empty?
-      haystack = text.to_s.downcase
+
+      haystack = normalize(text)
       tokens.any? { |t| haystack.include?(t) }
     end
   end

data/lib/llmemory/vector_store/memory_store.rb

@@ -12,10 +12,14 @@ module Llmemory
       end
 
       def embed(text)
-        return Array.new(1536, 0.0) unless @embedding_provider&.respond_to?(:embed)
+        return Array.new(default_dimensions, 0.0) unless @embedding_provider&.respond_to?(:embed)
         @embedding_provider.embed(text)
       end
 
+      def embedding_dimensions
+        default_dimensions
+      end
+
       def last_usage
         return @embedding_provider.last_usage if @embedding_provider&.respond_to?(:last_usage)
 
@@ -23,6 +27,7 @@ module Llmemory
       end
 
       def store(id:, embedding:, metadata: {}, user_id: nil)
+        validate_dimensions!(embedding)
         key = user_id ? "#{user_id}:#{id}" : id.to_s
         meta = (metadata || {}).dup
         if meta["text"] && @cipher.enabled?
@@ -40,6 +45,7 @@ module Llmemory
 
       def search(query_embedding, top_k: 10, user_id: nil)
         query = query_embedding.to_a.map(&:to_f)
+        validate_dimensions!(query)
         return [] if query.empty?
         entries = user_id ? @entries.select { |k, _| k.to_s.start_with?("#{user_id}:") } : @entries
         scores = entries.map do |_key, data|
@@ -79,6 +85,22 @@ module Llmemory
         out
       end
 
+      def default_dimensions
+        if @embedding_provider.respond_to?(:dimensions)
+          @embedding_provider.dimensions
+        else
+          1536
+        end
+      end
+
+      def validate_dimensions!(vector)
+        dims = default_dimensions
+        size = vector.to_a.size
+        return if size == dims
+
+        raise ConfigurationError, "embedding dimension mismatch: expected #{dims}, got #{size}"
+      end
+
       def cosine_similarity(a, b)
         return 0.0 if a.size != b.size || a.empty?
         dot = a.zip(b).sum { |x, y| x * y }

data/lib/llmemory/vector_store/openai_embeddings.rb

@@ -14,37 +14,59 @@ module Llmemory
       DEFAULT_MODEL = "text-embedding-3-small"
       DEFAULT_DIMS = 1536
 
-      attr_reader :last_usage
+      attr_reader :last_usage, :model
 
-      def initialize(api_key: nil, model: nil, base_url: nil)
+      def initialize(api_key: nil, model: nil, base_url: nil, dimensions: nil)
         @api_key = api_key || Llmemory.configuration.llm_api_key
         @model = model || DEFAULT_MODEL
+        @dimensions = dimensions || DEFAULT_DIMS
         @base_url = base_url || Llmemory.configuration.llm_base_url || "https://api.openai.com/v1"
         @cache = {}
         @cache_order = []
         @last_usage = Llmemory::LLM::Usage.zero
       end
 
+      def dimensions
+        @dimensions
+      end
+
+      def local?
+        false
+      end
+
       def embed(text)
-        return Array.new(DEFAULT_DIMS, 0.0) if text.to_s.strip.empty?
+        embed_batch([text]).first
+      end
 
-        if Llmemory.configuration.embedding_cache_enabled
-          key = cache_key(text)
-          if @cache.key?(key)
+      def embed_batch(texts)
+        list = Array(texts).map(&:to_s)
+        return [] if list.empty?
+
+        results = list.map do |text|
+          if text.strip.empty?
+            Array.new(dimensions, 0.0)
+          elsif Llmemory.configuration.embedding_cache_enabled && @cache.key?(cache_key(text))
             @last_usage = Llmemory::LLM::Usage.zero
-            return @cache[key].dup
+            @cache[cache_key(text)].dup
+          else
+            :miss
           end
         end
 
-        result = fetch_embedding(text)
-
-        if Llmemory.configuration.embedding_cache_enabled
-          evict_if_needed
-          @cache[cache_key(text)] = result.dup
-          @cache_order << cache_key(text)
+        miss_indices = results.each_index.select { |i| results[i] == :miss }
+        if miss_indices.any?
+          fetched = fetch_embeddings(miss_indices.map { |i| list[i] })
+          miss_indices.each_with_index do |idx, j|
+            vec = fetched[j]
+            results[idx] = vec
+            if Llmemory.configuration.embedding_cache_enabled
+              evict_if_needed
+              @cache[cache_key(list[idx])] = vec.dup
+              @cache_order << cache_key(list[idx])
+            end
+          end
         end
-
-        result
+        results
       end
 
       private
@@ -63,14 +85,16 @@ module Llmemory
         end
       end
 
-      def fetch_embedding(text)
-        result = nil
-        payload = { provider: :openai, model: @model, text_chars: text.to_s.length }
+      def fetch_embeddings(texts)
+        payload = { provider: :openai, model: @model, text_chars: texts.sum(&:length) }
+        vectors = nil
         Llmemory::Instrumentation.instrument(:llm_embed, payload) do
+          body_input = { input: texts.map { |t| t.to_s.strip }, model: @model }
+          body_input[:dimensions] = @dimensions if @dimensions != DEFAULT_DIMS
           response = post_with_resilience(connection, "embeddings") do |req|
             req.headers["Authorization"] = "Bearer #{@api_key}"
             req.headers["Content-Type"] = "application/json"
-            req.body = { input: text.to_s.strip, model: @model }.to_json
+            req.body = body_input.to_json
           end
           raise Llmemory::LLMError, "OpenAI Embeddings API error: #{response.body}" unless response.success?
           body = response.body.is_a?(Hash) ? response.body : JSON.parse(response.body.to_s)
@@ -80,9 +104,11 @@ module Llmemory
             output_tokens: @last_usage.output_tokens,
             total_tokens: @last_usage.total_tokens
           )
-          result = body.dig("data", 0, "embedding")&.map(&:to_f) || Array.new(DEFAULT_DIMS, 0.0)
+          vectors = Array(body["data"]).sort_by { |row| row["index"] }.map do |row|
+            row["embedding"]&.map(&:to_f) || Array.new(dimensions, 0.0)
+          end
         end
-        result
+        vectors
       end
 
       def parse_embed_usage(raw)

data/lib/llmemory/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Llmemory
-  VERSION = "0.2.10"
+  VERSION = "0.3.0"
 end

data/lib/llmemory/zero_mem/answer_calibrator.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module Llmemory
+  module ZeroMem
+    class AnswerCalibrator
+      def calibrate(answer, evidence_result:)
+        answer = answer.to_s
+        evidence = Array(evidence_result&.evidence)
+        checks = []
+        unsupported = []
+
+        if list_like?(answer)
+          checks << "list_answer"
+          return unchanged(answer, checks, unsupported)
+        end
+
+        scalar = extract_scalar_candidate(answer)
+        if scalar
+          matches = evidence.select { |ev| ev.content.to_s.include?(scalar) }
+          checks << "scalar_support"
+          if matches.size == 1
+            return {
+              answer: scalar,
+              changed: answer.strip != scalar,
+              checks: checks,
+              unsupported_fragments: unsupported
+            }
+          end
+
+          checks << "ambiguous_scalar"
+          return unchanged(answer, checks, unsupported)
+        end
+
+        checks << "free_text"
+        unchanged(answer, checks, unsupported)
+      end
+
+      private
+
+      def unchanged(answer, checks, unsupported)
+        {
+          answer: answer,
+          changed: false,
+          checks: checks,
+          unsupported_fragments: unsupported
+        }
+      end
+
+      def extract_scalar_candidate(answer)
+        stripped = answer.strip
+        return stripped if stripped.match?(/\A[\d.]+\z/)
+        return stripped if stripped.length <= 32 && stripped !~ /\s/
+
+        m = stripped.match(/\b([A-Z]{1,3})\b/)
+        m&.[](1)
+      end
+
+      def list_like?(answer)
+        answer.include?(",") || answer.match?(/\b(and|y)\b/i)
+      end
+    end
+  end
+end