llmemory 0.1.13 → 0.1.15

data/lib/llmemory/memory.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative "short_term/checkpoint"
+require_relative "short_term/pruner"
 require_relative "long_term/file_based"
 require_relative "retrieval/engine"
 
@@ -22,7 +23,7 @@ module Llmemory
     def add_message(role:, content:)
       msgs = messages
       msgs << { role: role.to_sym, content: content.to_s }
-      save_state(messages: msgs)
+      save_state(messages: msgs, **preserved_flush_state)
       true
     end
 
@@ -30,15 +31,47 @@ module Llmemory
       state = @checkpoint.restore_state
       return [] unless state.is_a?(Hash)
       list = state[STATE_KEY_MESSAGES] || state[STATE_KEY_MESSAGES.to_s]
-      list.is_a?(Array) ? list.dup : []
+      list = list.is_a?(Array) ? list.dup : []
+      sanitize_messages(list)
     end
 
     def retrieve(query, max_tokens: nil)
-      short_context = format_short_term_context(messages)
+      msgs = pruned_messages
+      short_context = format_short_term_context(msgs)
       long_context = @retrieval_engine.retrieve_for_inference(query, user_id: @user_id, max_tokens: max_tokens)
       combine_contexts(short_context, long_context)
     end
 
+    def recall_for(query: nil, max_tokens: nil)
+      return "" unless Llmemory.configuration.auto_recall_enabled
+
+      effective_query = query || last_user_message
+      return "" if effective_query.to_s.strip.empty?
+
+      retrieve(effective_query, max_tokens: max_tokens)
+    end
+
+    def last_user_message
+      msgs = messages
+      idx = msgs.rindex { |m| (m[:role] || m["role"]).to_s == "user" }
+      idx ? (msgs[idx][:content] || msgs[idx]["content"]).to_s : ""
+    end
+
+    def prune!(mode: nil)
+      return false unless Llmemory.configuration.prune_tool_results_enabled
+
+      msgs = messages
+      return false if msgs.empty?
+
+      mode ||= Llmemory.configuration.prune_tool_results_mode
+      pruner = ShortTerm::Pruner.new(
+        soft_trim_max_bytes: Llmemory.configuration.prune_tool_results_max_bytes
+      )
+      pruned = pruner.prune!(msgs, mode: mode)
+      save_state(messages: pruned, **preserved_flush_state)
+      true
+    end
+
     def consolidate!
       msgs = messages
       return true if msgs.empty?
@@ -58,15 +91,80 @@ module Llmemory
       current_bytes = messages_byte_size(msgs)
       return false if current_bytes <= max
 
+      flushed = flush_memory_before_compaction!(msgs)
+
       old_msgs, recent_msgs = split_messages_by_bytes(msgs, max)
       return false if old_msgs.empty?
 
       summary = summarize_messages(old_msgs)
       compacted = [{ role: :system, content: summary }] + recent_msgs
-      save_state(messages: compacted)
+      state = restore_state_for_save
+      flush_ts = flushed ? Time.now : (state[:last_flush_at] || state["last_flush_at"])
+      save_state(messages: compacted, last_compact_at: Time.now, last_flush_at: flush_ts)
       true
     end
 
+    def maybe_flush_memory!
+      return false unless Llmemory.configuration.memory_flush_enabled
+      msgs = messages
+      return false if msgs.empty?
+      return false if estimated_tokens(msgs) < Llmemory.configuration.memory_flush_threshold_tokens
+
+      consolidate!
+    end
+
+    def context_tokens
+      estimated_tokens(messages)
+    end
+
+    def should_auto_consolidate?
+      ctx = context_tokens
+      threshold = Llmemory.configuration.context_window_tokens - Llmemory.configuration.reserve_tokens
+      ctx >= threshold
+    end
+
+    def should_compact?
+      ctx = context_tokens
+      threshold = Llmemory.configuration.context_window_tokens - Llmemory.configuration.reserve_tokens
+      ctx >= threshold
+    end
+
+    def with_overflow_recovery(max_retries: 2, &block)
+      return yield unless Llmemory.configuration.overflow_recovery_enabled
+      return yield unless block_given?
+
+      retries = 0
+      begin
+        yield
+      rescue Llmemory::LLMError => e
+        msg = e.message.to_s.downcase
+        overflow = msg.include?("context") || msg.include?("token") || msg.include?("overflow") || msg.include?("limit")
+        raise unless overflow && retries < max_retries
+
+        prune! if Llmemory.configuration.prune_tool_results_enabled
+        compact!
+        retries += 1
+        retry
+      end
+    end
+
+    def check_context_window!
+      return false if messages.empty?
+
+      flushed = false
+      if should_auto_consolidate? && Llmemory.configuration.memory_flush_enabled
+        consolidate!
+        flushed = true
+      end
+
+      compacted = false
+      if should_compact?
+        compacted = compact!
+      end
+
+      flushed || compacted
+    end
+
     def user_id
       @user_id
     end
@@ -92,6 +190,47 @@ module Llmemory
       @llm ||= Llmemory::LLM.client
     end
 
+    def flush_memory_before_compaction!(msgs)
+      return false unless Llmemory.configuration.memory_flush_enabled
+      return false if msgs.empty?
+      return false if estimated_tokens(msgs) < Llmemory.configuration.memory_flush_threshold_tokens
+
+      state = restore_state_for_save
+      last_compact = state[:last_compact_at] || state["last_compact_at"]
+      window = Llmemory.configuration.flush_once_per_cycle_seconds.to_i
+
+      if last_compact
+        t = last_compact.is_a?(Time) ? last_compact : Time.parse(last_compact.to_s)
+        return false if (Time.now - t).to_i < window
+      end
+
+      consolidate!
+      true
+    end
+
+    def sanitize_messages(msgs)
+      return msgs unless Llmemory.configuration.message_sanitizer_enabled
+
+      sanitizer = ShortTerm::MessageSanitizer.new
+      sanitizer.sanitize!(msgs)
+    end
+
+    def restore_state_for_save
+      @checkpoint.restore_state || {}
+    end
+
+    def preserved_flush_state
+      state = restore_state_for_save
+      {}.tap do |h|
+        h[:last_flush_at] = state[:last_flush_at] || state["last_flush_at"] if state[:last_flush_at] || state["last_flush_at"]
+        h[:last_compact_at] = state[:last_compact_at] || state["last_compact_at"] if state[:last_compact_at] || state["last_compact_at"]
+      end
+    end
+
+    def estimated_tokens(msgs)
+      (messages_byte_size(msgs) / 4.0).ceil
+    end
+
     def messages_byte_size(msgs)
       msgs.sum { |m| message_byte_size(m) }
     end
@@ -137,8 +276,20 @@ module Llmemory
       end
     end
 
-    def save_state(messages:)
-      @checkpoint.save_state(STATE_KEY_MESSAGES => messages)
+    def save_state(messages:, last_flush_at: nil, last_compact_at: nil)
+      state = { STATE_KEY_MESSAGES => messages, last_activity_at: Time.now }
+      state[:last_flush_at] = last_flush_at if last_flush_at
+      state[:last_compact_at] = last_compact_at if last_compact_at
+      @checkpoint.save_state(state)
+    end
+
+    def pruned_messages
+      return messages unless Llmemory.configuration.prune_tool_results_enabled
+
+      pruner = ShortTerm::Pruner.new(
+        soft_trim_max_bytes: Llmemory.configuration.prune_tool_results_max_bytes
+      )
+      pruner.prune!(messages, mode: Llmemory.configuration.prune_tool_results_mode)
     end
 
     def format_short_term_context(msgs)

data/lib/llmemory/noise_filter.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Llmemory
+  class NoiseFilter
+    NO_REPLY_MARKER = "NO_REPLY"
+    DEFAULT_MIN_CHARS = 10
+
+    def initialize(min_chars: nil, enabled: true)
+      @min_chars = min_chars || Llmemory.configuration.noise_filter_min_chars
+      @enabled = enabled
+    end
+
+    def filter(conversation_text)
+      return conversation_text.to_s unless @enabled
+
+      lines = conversation_text.to_s.split("\n")
+      seen = {}
+      filtered = lines.select do |line|
+        next false if line.strip.length < @min_chars
+        next false if line.include?(NO_REPLY_MARKER)
+        next false if seen[line.strip]
+
+        seen[line.strip] = true
+        true
+      end
+
+      filtered.join("\n").strip
+    end
+
+    def self.filter?(conversation_text)
+      return conversation_text.to_s unless Llmemory.configuration.noise_filter_enabled
+
+      new.filter(conversation_text)
+    end
+  end
+end

data/lib/llmemory/retrieval/bm25_scorer.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Llmemory
+  module Retrieval
+    class Bm25Scorer
+      K1 = 1.5
+      B = 0.75
+
+      def initialize(k1: K1, b: B)
+        @k1 = k1
+        @b = b
+      end
+
+      def score_candidates(query, candidates)
+        return [] if candidates.empty?
+
+        query_tokens = tokenize(query)
+        return candidates.map { |c| c.merge(bm25_score: 0.0, normalized_bm25: 0.0) } if query_tokens.empty?
+
+        doc_tokens_list = candidates.map { |c| tokenize((c[:text] || c["text"]).to_s) }
+        avg_doc_len = doc_tokens_list.map(&:size).sum.to_f / [doc_tokens_list.size, 1].max
+        n_docs = candidates.size
+
+        doc_freq = Hash.new(0)
+        doc_tokens_list.each do |tokens|
+          tokens.uniq.each { |t| doc_freq[t] += 1 }
+        end
+
+        candidates.each_with_index.map do |c, i|
+          doc_tokens = doc_tokens_list[i]
+          doc_len = doc_tokens.size
+          bm25 = 0.0
+
+          query_tokens.uniq.each do |term|
+            tf = doc_tokens.count(term)
+            next if tf.zero?
+
+            n_qi = doc_freq[term]
+            idf = Math.log((n_docs - n_qi + 0.5) / (n_qi + 0.5) + 1.0)
+            numerator = tf * (@k1 + 1)
+            denom = tf + @k1 * (1 - @b + @b * doc_len.to_f / [avg_doc_len, 1].max)
+            bm25 += idf * numerator / denom
+          end
+
+          c.merge(bm25_score: bm25)
+        end.tap do |scored|
+          max_bm25 = scored.map { |s| s[:bm25_score] }.max.to_f
+          max_bm25 = 1.0 if max_bm25.zero?
+          scored.each { |s| s[:normalized_bm25] = s[:bm25_score] / max_bm25 }
+        end
+      end
+
+      private
+
+      def tokenize(text)
+        text.to_s.downcase.scan(/\b[a-z0-9]{2,}\b/)
+      end
+    end
+  end
+end

data/lib/llmemory/retrieval/engine.rb

@@ -2,6 +2,8 @@
 
 require_relative "temporal_ranker"
 require_relative "context_assembler"
+require_relative "bm25_scorer"
+require_relative "mmr_reranker"
 
 module Llmemory
   module Retrieval
@@ -13,15 +15,19 @@ module Llmemory
         @llm = llm || Llmemory::LLM.client
         @ranker = TemporalRanker.new
         @assembler = ContextAssembler.new
+        @bm25_scorer = Bm25Scorer.new
+        @mmr_reranker = MmrReranker.new(lambda: Llmemory.configuration.mmr_lambda)
       end
 
       def retrieve_for_inference(user_message, user_id: nil, max_tokens: nil)
         user_id ||= @memory.respond_to?(:user_id) ? @memory.user_id : nil
         search_query = generate_query(user_message)
         candidates = fetch_candidates(search_query, user_id)
+        candidates = apply_hybrid_scoring(candidates, search_query) if Llmemory.configuration.hybrid_search_enabled
 
         relevant = filter_by_relevance(candidates, user_message)
         ranked = @ranker.rank(relevant)
+        ranked = @mmr_reranker.rerank(ranked) if Llmemory.configuration.mmr_enabled
         @assembler.assemble(ranked, max_tokens: max_tokens)
       end
 
@@ -47,7 +53,8 @@ module Llmemory
           {
             text: c[:text] || c["text"],
             timestamp: parse_timestamp(c[:timestamp] || c["timestamp"] || c[:created_at] || c["created_at"]),
-            score: (c[:score] || c["score"] || 1.0).to_f
+            score: (c[:score] || c["score"] || 1.0).to_f,
+            evergreen: c[:evergreen] || c["evergreen"]
           }
         end
       end
@@ -58,6 +65,21 @@ module Llmemory
         Time.now
       end
 
+      def apply_hybrid_scoring(candidates, query)
+        return candidates if candidates.empty?
+
+        scored = @bm25_scorer.score_candidates(query, candidates)
+        weight = Llmemory.configuration.bm25_weight.to_f
+        weight = 0.3 if weight < 0 || weight > 1
+
+        scored.map do |c|
+          vector_score = (c[:score] || c["score"] || 1.0).to_f
+          bm25_norm = (c[:normalized_bm25] || 0).to_f
+          hybrid = weight * bm25_norm + (1 - weight) * vector_score
+          c.merge(score: hybrid)
+        end
+      end
+
       def filter_by_relevance(candidates, user_message)
         return candidates if candidates.size <= 5
         user_lower = user_message.to_s.downcase

data/lib/llmemory/retrieval/mmr_reranker.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Llmemory
+  module Retrieval
+    class MmrReranker
+      def initialize(lambda: 0.7)
+        @lambda = lambda
+      end
+
+      def rerank(candidates, score_key: :temporal_score)
+        return candidates if candidates.size <= 1
+
+        selected = []
+        remaining = candidates.dup
+
+        while remaining.any?
+          best_idx = nil
+          best_mmr = -Float::INFINITY
+
+          remaining.each_with_index do |cand, i|
+            rel = (cand[score_key] || cand[score_key.to_s] || cand[:score] || cand["score"] || 0).to_f
+            max_sim = selected.map { |s| similarity(cand, s) }.max || 0
+            mmr = @lambda * rel - (1 - @lambda) * max_sim
+
+            if mmr > best_mmr
+              best_mmr = mmr
+              best_idx = i
+            end
+          end
+
+          break unless best_idx
+
+          selected << remaining.delete_at(best_idx)
+        end
+
+        selected
+      end
+
+      private
+
+      def similarity(a, b)
+        text_a = tokenize((a[:text] || a["text"]).to_s)
+        text_b = tokenize((b[:text] || b["text"]).to_s)
+        return 0.0 if text_a.empty? || text_b.empty?
+
+        intersection = (text_a & text_b).size
+        union = (text_a | text_b).size
+        union.zero? ? 0.0 : intersection.to_f / union
+      end
+
+      def tokenize(text)
+        text.downcase.scan(/\b[a-z0-9]{2,}\b/).uniq
+      end
+    end
+  end
+end

data/lib/llmemory/retrieval/temporal_ranker.rb

@@ -8,12 +8,20 @@ module Llmemory
       end
 
       def rank(candidates, now: Time.now)
+        lambda_val = Math.log(2) / @half_life_days.to_f
+
         candidates.map do |c|
           score = (c[:score] || c["score"] || 1.0).to_f
           timestamp = c[:timestamp] || c["timestamp"]
           timestamp = Time.parse(timestamp.to_s) if timestamp.is_a?(String)
           age_days = timestamp ? (now - timestamp).to_i / 86400 : 0
-          time_decay = 1.0 / (1.0 + (age_days.to_f / @half_life_days))
+
+          time_decay = if c[:evergreen] || c["evergreen"]
+            1.0
+          else
+            Math.exp(-lambda_val * age_days.to_f)
+          end
+
           final_score = score * time_decay
           c.merge(score: score, temporal_score: final_score, timestamp: timestamp)
         end.sort_by { |c| -(c[:temporal_score] || 0) }

data/lib/llmemory/short_term/message_sanitizer.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Llmemory
+  module ShortTerm
+    class MessageSanitizer
+      def initialize(max_message_chars: nil)
+        @max_chars = max_message_chars || Llmemory.configuration.max_message_chars
+      end
+
+      def sanitize!(messages)
+        return [] if messages.nil? || !messages.is_a?(Array)
+
+        out = []
+        expect_tool_result = false
+
+        messages.each do |msg|
+          msg = msg.dup
+          content = (msg[:content] || msg["content"]).to_s
+          role = (msg[:role] || msg["role"]).to_s
+
+          next if content.strip.empty?
+
+          content = content[0, @max_chars] if @max_chars && content.length > @max_chars
+
+          if role == "tool"
+            expect_tool_result = true
+          elsif role == "tool_result"
+            next unless expect_tool_result
+            expect_tool_result = false
+          else
+            expect_tool_result = false
+          end
+
+          msg[:content] = content if msg.key?(:content)
+          msg["content"] = content if msg.key?("content")
+          out << msg
+        end
+
+        out
+      end
+    end
+  end
+end

data/lib/llmemory/short_term/pruner.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Llmemory
+  module ShortTerm
+    class Pruner
+      DEFAULT_PRUNABLE_ROLES = %i[tool tool_result].freeze
+      PLACEHOLDER = "[Tool result pruned]"
+
+      def initialize(prunable_roles: nil, soft_trim_max_bytes: 2048, soft_trim_head_ratio: 0.4, soft_trim_tail_ratio: 0.2)
+        @prunable_roles = prunable_roles || DEFAULT_PRUNABLE_ROLES.map(&:to_s)
+        @soft_trim_max_bytes = soft_trim_max_bytes
+        @head_ratio = soft_trim_head_ratio
+        @tail_ratio = soft_trim_tail_ratio
+      end
+
+      def prune!(messages, mode: :soft_trim)
+        return messages.dup if messages.empty?
+
+        messages.map do |msg|
+          if prunable?(msg)
+            apply_prune(msg, mode)
+          else
+            msg.dup
+          end
+        end
+      end
+
+      private
+
+      def prunable?(msg)
+        role = (msg[:role] || msg["role"]).to_s
+        @prunable_roles.include?(role)
+      end
+
+      def apply_prune(msg, mode)
+        content = (msg[:content] || msg["content"]).to_s
+        new_content = case mode.to_s.to_sym
+        when :hard_clear
+          PLACEHOLDER
+        when :soft_trim
+          soft_trim(content)
+        else
+          content
+        end
+
+        result = msg.dup
+        result[:content] = new_content if result.key?(:content)
+        result["content"] = new_content if result.key?("content")
+        result
+      end
+
+      def soft_trim(content)
+        return content if content.bytesize <= @soft_trim_max_bytes
+
+        head_chars = (@soft_trim_max_bytes * @head_ratio).to_i
+        tail_chars = (@soft_trim_max_bytes * @tail_ratio).to_i
+
+        head = content.byteslice(0, head_chars)
+        tail = content.bytesize > (head_chars + tail_chars) ? content.byteslice(-tail_chars, tail_chars) : ""
+
+        "#{head}\n...\n#{tail}"
+      end
+    end
+  end
+end

data/lib/llmemory/short_term/session_lifecycle.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module Llmemory
+  module ShortTerm
+    class SessionLifecycle
+      def initialize(store: nil)
+        @store = store || build_store
+      end
+
+      def cleanup_idle_sessions!(user_id:, idle_minutes: nil)
+        idle_minutes ||= Llmemory.configuration.session_idle_minutes
+        cutoff = Time.now - (idle_minutes * 60)
+        deleted = 0
+
+        @store.list_sessions(user_id: user_id).each do |session_id|
+          state = @store.load(user_id, session_id)
+          next unless state.is_a?(Hash)
+
+          last_activity = state[:last_activity_at] || state["last_activity_at"]
+          next if last_activity.nil?
+
+          last_time = last_activity.is_a?(Time) ? last_activity : Time.parse(last_activity.to_s)
+          if last_time < cutoff
+            @store.delete(user_id, session_id)
+            deleted += 1
+          end
+        end
+
+        deleted
+      end
+
+      def cleanup_stale_sessions!(user_id:, prune_after_days: nil)
+        prune_after_days ||= Llmemory.configuration.session_prune_after_days
+        cutoff = Time.now - (prune_after_days * 86400)
+        deleted = 0
+
+        @store.list_sessions(user_id: user_id).each do |session_id|
+          state = @store.load(user_id, session_id)
+          next unless state.is_a?(Hash)
+
+          last_activity = state[:last_activity_at] || state["last_activity_at"]
+          next if last_activity.nil?
+
+          last_time = last_activity.is_a?(Time) ? last_activity : Time.parse(last_activity.to_s)
+          if last_time < cutoff
+            @store.delete(user_id, session_id)
+            deleted += 1
+          end
+        end
+
+        deleted
+      end
+
+      def enforce_max_entries!(user_id:, max_entries: nil)
+        max_entries ||= Llmemory.configuration.session_max_entries_per_user
+        sessions = @store.list_sessions(user_id: user_id)
+        return 0 if sessions.size <= max_entries
+
+        session_ages = sessions.map do |session_id|
+          state = @store.load(user_id, session_id)
+          last_activity = state&.dig(:last_activity_at) || state&.dig("last_activity_at")
+          last_time = last_activity.is_a?(Time) ? last_activity : (last_activity ? Time.parse(last_activity.to_s) : Time.at(0))
+          [session_id, last_time]
+        end
+
+        session_ages.sort_by! { |_, t| t }
+        to_delete = session_ages.first(session_ages.size - max_entries).map(&:first)
+        to_delete.each { |sid| @store.delete(user_id, sid) }
+        to_delete.size
+      end
+
+      private
+
+      def build_store
+        case Llmemory.configuration.short_term_store.to_sym
+        when :memory then Stores::MemoryStore.new
+        when :redis then Stores::RedisStore.new
+        when :postgres then Stores::PostgresStore.new
+        when :active_record, :activerecord
+          require_relative "stores/active_record_store"
+          Stores::ActiveRecordStore.new
+        else
+          Stores::MemoryStore.new
+        end
+      end
+    end
+  end
+end

data/lib/llmemory/short_term.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 require_relative "short_term/checkpoint"
+require_relative "short_term/session_lifecycle"
+require_relative "short_term/message_sanitizer"
 
 module Llmemory
   module ShortTerm