llmemory 0.1.14 → 0.1.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3bdfe8a5f7301af319a99e853c36646491cdc64c581b566811605fb4e63415dd
-  data.tar.gz: 27d827d262c35f3e42757416895fe91ad84290032f0091dbfd7a24806da5cf73
+  metadata.gz: 723fae20d0310ccaeaf9ba600148061d17b2a0b29f933d455d1cf656dee85636
+  data.tar.gz: a135ea1661af46e96843bf52744e8004d0ebe7e8d94b0c46a097c36df53d5bc4
 SHA512:
-  metadata.gz: 72337abbc9bd02e9289a9b4dad399741f066fc9dbaf01cd2fa8de80813f4042c8939313afb7aa248addf02ade1cdae379a93585618200f96c3e2d0dbafef9138
-  data.tar.gz: 263872146fb6654ecfef977b7a7075798b6439e06405eeb512ff48b67c7294387eb963421cbffed1fecf0e4863f7bfc82b0d6f88f3e92eb8cc06898d587c50b2
+  metadata.gz: 256caaee94233d5e57b8d9e6007fe1ced57d35e21d40260ce34b2803ba0ef3593b66668aa06334e647edd103aa431113e38b639776163d71153c4b9bac68c1a1
+  data.tar.gz: 33cd1726e9f7bb3328610bafabca5ebfe51f080e7d34c523fc0b363eb290b353c9109937f2782ed7e60906965236e79229838e32c698d2f0e2f73aa2d421970b

data/README.md

@@ -2,6 +2,8 @@
 
 Persistent memory system for LLM agents. Implements short-term checkpointing, long-term memory (file-based or **graph-based**), retrieval with time decay, and maintenance jobs. You can inspect memory from the **CLI** or, in Rails apps, from an optional **dashboard**.
 
+Includes advanced memory management features inspired by [OpenClaw](https://github.com/openclaw/openclaw): pre-compaction memory flush, hybrid search (BM25 + vector), tool result pruning, context window tracking, session lifecycle management, daily memory logs, and auto-recall.
+
 ## Installation
 
 Add to your Gemfile:
@@ -40,11 +42,14 @@ memory.compact!(max_bytes: 8192)  # or use config default
 memory.clear_session!
 ```
 
-- **`add_message(role:, content:)`** — Persists messages in short-term.
+- **`add_message(role:, content:)`** — Persists messages in short-term. Supports `user`, `assistant`, `system`, `tool`, and `tool_result` roles.
 - **`messages`** — Returns the current conversation history.
 - **`retrieve(query, max_tokens: nil)`** — Returns combined context: recent conversation + relevant long-term memories.
+- **`recall_for(query: nil)`** — Auto-recall: returns context for the given query (or last user message if `query` is nil). Only active when `auto_recall_enabled` is true.
 - **`consolidate!`** — Extracts facts from the current conversation and stores them in long-term.
-- **`compact!(max_bytes: nil)`** — Compacts short-term memory by summarizing old messages when byte size exceeds limit. Uses LLM to create a summary, keeping recent messages intact.
+- **`compact!(max_bytes: nil)`** — Compacts short-term memory by summarizing old messages when byte size exceeds limit. Automatically flushes to long-term before compacting when over `memory_flush_threshold_tokens`.
+- **`prune!(mode: nil)`** — Prunes oversized tool results (soft-trim or hard-clear). Only when `prune_tool_results_enabled` is true.
+- **`check_context_window!`** — Triggers consolidate and compact when context exceeds configured thresholds.
 - **`clear_session!`** — Clears short-term only.
 
 ## Configuration
@@ -64,6 +69,37 @@ Llmemory.configure do |config|
   config.max_retrieval_tokens = 2000
   config.prune_after_days = 90
   config.compact_max_bytes = 8192  # max bytes before compact! triggers
+
+  # Pre-compaction memory flush (prevents knowledge loss when compacting)
+  config.memory_flush_enabled = true
+  config.memory_flush_threshold_tokens = 4000
+
+  # Hybrid search (BM25 + vector) and MMR re-ranking
+  config.hybrid_search_enabled = true
+  config.bm25_weight = 0.3
+  config.mmr_enabled = false
+  config.mmr_lambda = 0.7
+
+  # Tool result pruning (soft-trim or hard-clear for tool/tool_result messages)
+  config.prune_tool_results_enabled = false
+  config.prune_tool_results_mode = :soft_trim
+  config.prune_tool_results_max_bytes = 2048
+
+  # Context window tracking and auto-consolidation
+  config.context_window_tokens = 128_000
+  config.reserve_tokens = 16_384
+  config.keep_recent_tokens = 20_000
+
+  # Session lifecycle management
+  config.session_idle_minutes = 60
+  config.session_prune_after_days = 30
+  config.session_max_entries_per_user = 500
+
+  # Daily memory logs (file-based, FileStorage only)
+  config.daily_logs_enabled = false
+
+  # Auto-recall (inject relevant memories before each LLM turn)
+  config.auto_recall_enabled = false
 end
 ```
 
@@ -159,6 +195,71 @@ candidates = memory.search_candidates("job", top_k: 20)
 
 **Graph storage:** `:memory` (in-memory) or `:active_record` (Rails). For ActiveRecord, run `rails g llmemory:install` and migrate; the migration creates `llmemory_nodes`, `llmemory_edges`, and `llmemory_embeddings` (pgvector). Enable the `vector` extension in PostgreSQL for embeddings.
 
+## Advanced Memory Management
+
+These features improve robustness and efficiency, inspired by OpenClaw's memory system.
+
+### Pre-Compaction Memory Flush
+
+Before compacting short-term memory, llmemory can automatically consolidate the conversation into long-term storage. This prevents knowledge loss when the context is summarized.
+
+- **`memory_flush_enabled`** — When true, `compact!` calls `consolidate!` first when messages exceed `memory_flush_threshold_tokens`.
+- **`maybe_flush_memory!`** — Call explicitly to flush when approaching context limits.
+
+### Hybrid Search (BM25 + Vector)
+
+Retrieval combines keyword matching (BM25) with vector similarity for more robust search. Optional MMR (Maximal Marginal Relevance) re-ranking improves result diversity.
+
+- **`hybrid_search_enabled`** — Combines BM25 and vector scores.
+- **`bm25_weight`** — Weight for BM25 (0–1); remainder is vector score.
+- **`mmr_enabled`** — Re-ranks results for diversity.
+- **`mmr_lambda`** — Balance between relevance and diversity (0–1).
+
+### Tool Result Pruning
+
+Large tool outputs can consume most of the context window. Pruning selectively trims `tool` and `tool_result` messages while keeping user/assistant intact.
+
+- **`prune_tool_results_enabled`** — When true, `retrieve` uses pruned messages and `prune!` is available.
+- **`prune_tool_results_mode`** — `:soft_trim` (keep head+tail) or `:hard_clear` (replace with placeholder).
+- **`prune_tool_results_max_bytes`** — Max bytes before soft-trim applies.
+
+### Context Window Tracking
+
+Track estimated tokens and trigger consolidation/compaction automatically.
+
+- **`context_tokens`** — Returns estimated token count for current messages.
+- **`should_auto_consolidate?`** — True when over `context_window_tokens - reserve_tokens`.
+- **`check_context_window!`** — Runs consolidate and compact when thresholds are exceeded.
+
+### Session Lifecycle Management
+
+Clean up stale or idle sessions to control storage usage.
+
+```ruby
+lifecycle = Llmemory::ShortTerm::SessionLifecycle.new
+lifecycle.cleanup_idle_sessions!(user_id: "user_123", idle_minutes: 60)
+lifecycle.cleanup_stale_sessions!(user_id: "user_123", prune_after_days: 30)
+lifecycle.enforce_max_entries!(user_id: "user_123", max_entries: 500)
+```
+
+Sessions store `last_activity_at` automatically on each save.
+
+### Daily Memory Logs
+
+With `daily_logs_enabled` and FileStorage, file-based memory writes to `memory/YYYY-MM-DD.md` per user. Today's and yesterday's logs are included in retrieval. Useful for temporal organization and human-readable logs.
+
+### Auto-Recall
+
+When `auto_recall_enabled` is true, call `recall_for(query: nil)` before each LLM turn. If `query` is nil, the last user message is used as the search query. Returns combined context without explicit `retrieve` calls.
+
+```ruby
+Llmemory.configure { |c| c.auto_recall_enabled = true }
+# Before each LLM call:
+context = memory.recall_for(query: user_message)
+# Or use last user message automatically:
+context = memory.recall_for
+```
+
 ## Lower-Level APIs
 
 ### Short-Term Memory (Checkpointing)
@@ -333,7 +434,7 @@ MCP_TOKEN=your-secret-token llmemory mcp serve --http --port 443 \
 | `memory_retrieve` | Get context optimized for LLM inference (supports timeline context) |
 | `memory_timeline` | Get chronological timeline of recent memories |
 | `memory_timeline_context` | Get N items before/after a specific memory |
-| `memory_add_message` | Add message to short-term conversation |
+| `memory_add_message` | Add message to short-term conversation (roles: user, assistant, system, tool, tool_result) |
 | `memory_consolidate` | Extract facts from conversation to long-term |
 | `memory_stats` | Get memory statistics for a user |
 | `memory_info` | Documentation on how to use the tools |

data/lib/generators/llmemory/install/templates/create_llmemory_tables.rb

@@ -16,6 +16,7 @@ class CreateLlmemoryTables < ActiveRecord::Migration[7.0]
       t.string :category, null: false
       t.text :content, null: false
       t.string :source_resource_id
+      t.float :importance, default: 0.7
       t.timestamps
     end
     add_index :llmemory_items, :user_id

data/lib/llmemory/configuration.rb

@@ -33,7 +33,15 @@ module Llmemory
                   :session_prune_after_days,
                   :session_max_entries_per_user,
                   :daily_logs_enabled,
-                  :auto_recall_enabled
+                  :auto_recall_enabled,
+                  :noise_filter_enabled,
+                  :noise_filter_min_chars,
+                  :flush_once_per_cycle_seconds,
+                  :overflow_recovery_enabled,
+                  :embedding_cache_enabled,
+                  :embedding_cache_max_entries,
+                  :max_message_chars,
+                  :message_sanitizer_enabled
 
     def initialize
       @llm_provider = :openai
@@ -68,6 +76,14 @@ module Llmemory
       @session_max_entries_per_user = 500
       @daily_logs_enabled = false
       @auto_recall_enabled = false
+      @noise_filter_enabled = false
+      @noise_filter_min_chars = 10
+      @flush_once_per_cycle_seconds = 60
+      @overflow_recovery_enabled = false
+      @embedding_cache_enabled = true
+      @embedding_cache_max_entries = 10_000
+      @max_message_chars = 32_000
+      @message_sanitizer_enabled = false
     end
   end
 

data/lib/llmemory/extractors/fact_extractor.rb

@@ -14,7 +14,9 @@ module Llmemory
           Extract discrete facts from this conversation.
           Focus on preferences, behaviors, and important details.
           Conversation: #{conversation_text}
-          Return as JSON array of objects with "content" key. Example: [{"content": "User prefers Ruby"}, {"content": "User is vegan"}]
+          Return as JSON array of objects with "content" and "importance" (0-1) keys.
+          Importance: 0.8-0.95 for preferences/corrections/decisions, 0.5-0.8 for factual context, 0.3-0.5 for ephemeral.
+          Example: [{"content": "User prefers Ruby", "importance": 0.9}, {"content": "User mentioned the weather", "importance": 0.4}]
         PROMPT
         response = @llm.invoke(prompt.strip)
         parse_items_response(response)
@@ -56,7 +58,12 @@ module Llmemory
       def parse_items_response(response)
         json = extract_json_array(response)
         return [] unless json
-        json.map { |item| item.is_a?(Hash) ? item : { "content" => item.to_s } }
+        json.map do |item|
+          h = item.is_a?(Hash) ? item : { "content" => item.to_s }
+          imp = h["importance"] || h[:importance]
+          h["importance"] = imp.nil? ? 0.7 : (imp.to_f.between?(0, 1) ? imp.to_f : 0.7)
+          h
+        end
       end
 
       def extract_json_array(response)

data/lib/llmemory/long_term/file_based/memory.rb

@@ -4,6 +4,7 @@ require_relative "resource"
 require_relative "item"
 require_relative "category"
 require_relative "storage"
+require_relative "../../noise_filter"
 
 module Llmemory
   module LongTerm
@@ -17,17 +18,21 @@ module Llmemory
         end
 
         def memorize(conversation_text)
-          resource_id = save_resource(conversation_text)
-          append_to_daily_log(conversation_text) if Llmemory.configuration.daily_logs_enabled && @storage.respond_to?(:save_daily_log_entry)
-          items = @extractor.extract_items(conversation_text)
+          text = Llmemory.configuration.noise_filter_enabled ? NoiseFilter.filter?(conversation_text) : conversation_text.to_s
+          return true if text.strip.empty?
+
+          resource_id = save_resource(text)
+          append_to_daily_log(text) if Llmemory.configuration.daily_logs_enabled && @storage.respond_to?(:save_daily_log_entry)
+          items = @extractor.extract_items(text)
           updates_by_category = {}
 
           items.each do |item|
             content = item.is_a?(Hash) ? (item["content"] || item[:content]) : item.to_s
+            importance = (item["importance"] || item[:importance] || 0.7).to_f
             cat = @extractor.classify_item(content)
             updates_by_category[cat] ||= []
             updates_by_category[cat] << content.to_s
-            save_item(category: cat, item: item, source_resource_id: resource_id)
+            save_item(category: cat, item: item, source_resource_id: resource_id, importance: importance)
           end
 
           updates_by_category.each do |category, new_memories|
@@ -49,12 +54,19 @@ module Llmemory
           items = @storage.search_items(uid, query)
           resources = @storage.search_resources(uid, query)
           daily_logs = load_daily_logs_for_retrieval(uid) if Llmemory.configuration.daily_logs_enabled && @storage.respond_to?(:load_daily_logs)
+          category_summaries = load_category_summaries_as_candidates(uid, query)
           out = []
+
+          category_summaries.each do |c|
+            out << c.merge(evergreen: true)
+          end
+
           items.first(top_k).each do |i|
             out << {
               text: i[:content] || i["content"],
               timestamp: i[:created_at] || i["created_at"],
-              score: 1.0
+              score: (i[:importance] || i["importance"] || 1.0).to_f,
+              evergreen: i[:evergreen] || i["evergreen"]
             }
           end
           resources.first([top_k - out.size, 0].max).each do |r|
@@ -80,9 +92,9 @@ module Llmemory
           @storage.save_resource(@user_id, text)
         end
 
-        def save_item(category:, item:, source_resource_id:)
+        def save_item(category:, item:, source_resource_id:, importance: 0.7)
           content = item.is_a?(Hash) ? item["content"] || item[:content] : item.to_s
-          @storage.save_item(@user_id, category: category, content: content, source_resource_id: source_resource_id)
+          @storage.save_item(@user_id, category: category, content: content, source_resource_id: source_resource_id, importance: importance)
         end
 
         def append_to_daily_log(conversation_text)
@@ -96,6 +108,22 @@ module Llmemory
           logs = @storage.load_daily_logs(user_id, from_date: yesterday, to_date: today)
           logs.map { |l| { date: l[:date], content: "[#{l[:date]}] #{l[:content]}" } }
         end
+
+        def load_category_summaries_as_candidates(user_id, query)
+          return [] unless @storage.respond_to?(:list_categories)
+
+          categories = @storage.list_categories(user_id)
+          return [] if categories.empty?
+
+          query_lower = query.to_s.downcase
+          categories.filter_map do |cat|
+            summary = @storage.load_category(user_id, cat)
+            next if summary.to_s.strip.empty?
+            next unless summary.to_s.downcase.include?(query_lower)
+
+            { text: "[#{cat}] #{summary}", timestamp: Time.now, score: 0.95 }
+          end
+        end
       end
     end
   end

data/lib/llmemory/long_term/file_based/storages/active_record_storage.rb

@@ -30,16 +30,18 @@ module Llmemory
             id
           end
 
-          def save_item(user_id, category:, content:, source_resource_id:)
+          def save_item(user_id, category:, content:, source_resource_id:, importance: 0.7)
             id = "item_#{SecureRandom.hex(8)}"
-            LlmemoryItem.create!(
+            attrs = {
               id: id,
               user_id: user_id,
               category: category,
               content: content,
               source_resource_id: source_resource_id,
               created_at: Time.current
-            )
+            }
+            attrs[:importance] = importance if LlmemoryItem.column_names.include?("importance")
+            LlmemoryItem.create!(attrs)
             id
           end
 
@@ -96,14 +98,16 @@ module Llmemory
           def replace_items(user_id, ids_to_remove, merged_item)
             LlmemoryItem.where(user_id: user_id, id: ids_to_remove).destroy_all
             created_at = merged_item[:created_at] || Time.current
-            LlmemoryItem.create!(
+            attrs = {
               id: "item_#{SecureRandom.hex(8)}",
               user_id: user_id,
               category: merged_item[:category],
               content: merged_item[:content],
               source_resource_id: merged_item[:source_resource_id],
               created_at: created_at
-            )
+            }
+            attrs[:importance] = merged_item[:importance] if LlmemoryItem.column_names.include?("importance") && merged_item[:importance]
+            LlmemoryItem.create!(attrs)
           end
 
           def archive_items(user_id, item_ids)
@@ -177,13 +181,15 @@ module Llmemory
           end
 
           def row_to_item(r)
-            {
+            h = {
               id: r.id,
               category: r.category,
               content: r.content,
               source_resource_id: r.source_resource_id,
               created_at: r.created_at
             }
+            h[:importance] = r.respond_to?(:importance) ? (r.importance || 0.7).to_f : 0.7
+            h
           end
 
           def row_to_resource(r)

data/lib/llmemory/long_term/file_based/storages/base.rb

@@ -9,7 +9,7 @@ module Llmemory
             raise NotImplementedError, "#{self.class}#save_resource must be implemented"
           end
 
-          def save_item(user_id, category:, content:, source_resource_id:)
+          def save_item(user_id, category:, content:, source_resource_id:, importance: 0.7)
             raise NotImplementedError, "#{self.class}#save_item must be implemented"
           end
 

data/lib/llmemory/long_term/file_based/storages/database_storage.rb

@@ -24,12 +24,12 @@ module Llmemory
             id
           end
 
-          def save_item(user_id, category:, content:, source_resource_id:)
+          def save_item(user_id, category:, content:, source_resource_id:, importance: 0.7)
             ensure_tables!
             id = "item_#{SecureRandom.hex(8)}"
             conn.exec_params(
-              "INSERT INTO llmemory_items (id, user_id, category, content, source_resource_id, created_at) VALUES ($1, $2, $3, $4, $5, $6)",
-              [id, user_id, category, content, source_resource_id, Time.now.utc.iso8601]
+              "INSERT INTO llmemory_items (id, user_id, category, content, source_resource_id, importance, created_at) VALUES ($1, $2, $3, $4, $5, $6, $7)",
+              [id, user_id, category, content, source_resource_id, importance.to_f, Time.now.utc.iso8601]
             )
             id
           end
@@ -67,7 +67,7 @@ module Llmemory
             ensure_tables!
             pattern = "%#{conn.escape_string(query.to_s.downcase)}%"
             rows = conn.exec_params(
-              "SELECT id, category, content, source_resource_id, created_at FROM llmemory_items WHERE user_id = $1 AND LOWER(content) LIKE $2",
+              "SELECT id, category, content, source_resource_id, importance, created_at FROM llmemory_items WHERE user_id = $1 AND LOWER(content) LIKE $2",
               [user_id, pattern]
             )
             rows_to_items(rows)
@@ -97,7 +97,7 @@ module Llmemory
             ensure_tables!
             cutoff = (Time.now - (days * 86400)).utc.iso8601
             rows = conn.exec_params(
-              "SELECT id, category, content, source_resource_id, created_at FROM llmemory_items WHERE user_id = $1 AND created_at < $2 ORDER BY created_at",
+              "SELECT id, category, content, source_resource_id, importance, created_at FROM llmemory_items WHERE user_id = $1 AND created_at < $2 ORDER BY created_at",
               [user_id, cutoff]
             )
             rows_to_items(rows)
@@ -106,7 +106,7 @@ module Llmemory
           def get_all_items(user_id)
             ensure_tables!
             rows = conn.exec_params(
-              "SELECT id, category, content, source_resource_id, created_at FROM llmemory_items WHERE user_id = $1 ORDER BY created_at",
+              "SELECT id, category, content, source_resource_id, importance, created_at FROM llmemory_items WHERE user_id = $1 ORDER BY created_at",
               [user_id]
             )
             rows_to_items(rows)
@@ -125,7 +125,7 @@ module Llmemory
             ensure_tables!
             cutoff = (Time.now - (hours * 3600)).utc.iso8601
             rows = conn.exec_params(
-              "SELECT id, category, content, source_resource_id, created_at FROM llmemory_items WHERE user_id = $1 AND created_at >= $2 ORDER BY created_at",
+              "SELECT id, category, content, source_resource_id, importance, created_at FROM llmemory_items WHERE user_id = $1 AND created_at >= $2 ORDER BY created_at",
               [user_id, cutoff]
             )
             rows_to_items(rows)
@@ -179,7 +179,7 @@ module Llmemory
 
           def list_items(user_id:, category: nil, limit: nil)
             ensure_tables!
-            sql = "SELECT id, category, content, source_resource_id, created_at FROM llmemory_items WHERE user_id = $1"
+            sql = "SELECT id, category, content, source_resource_id, importance, created_at FROM llmemory_items WHERE user_id = $1"
             params = [user_id]
             if category
               sql += " AND category = $2"
@@ -257,10 +257,12 @@ module Llmemory
                 category TEXT NOT NULL,
                 content TEXT NOT NULL,
                 source_resource_id TEXT,
+                importance REAL DEFAULT 0.7,
                 created_at TIMESTAMPTZ NOT NULL
               );
               CREATE INDEX IF NOT EXISTS idx_llmemory_items_user_id ON llmemory_items(user_id);
             SQL
+            conn.exec("ALTER TABLE llmemory_items ADD COLUMN IF NOT EXISTS importance REAL DEFAULT 0.7") rescue nil
             conn.exec(<<~SQL)
               CREATE TABLE IF NOT EXISTS llmemory_categories (
                 user_id TEXT NOT NULL,
@@ -279,6 +281,7 @@ module Llmemory
                 category: r["category"],
                 content: r["content"],
                 source_resource_id: r["source_resource_id"],
+                importance: (r["importance"] || 0.7).to_f,
                 created_at: Time.parse(r["created_at"])
               }
             end

data/lib/llmemory/long_term/file_based/storages/file_storage.rb

@@ -24,7 +24,7 @@ module Llmemory
             id
           end
 
-          def save_item(user_id, category:, content:, source_resource_id:)
+          def save_item(user_id, category:, content:, source_resource_id:, importance: 0.7)
             ensure_user_dir(user_id)
             seq = next_seq(user_id, "item_id_seq")
             id = "item_#{seq}"
@@ -34,6 +34,7 @@ module Llmemory
               category: category,
               content: content,
               source_resource_id: source_resource_id,
+              importance: importance,
               created_at: Time.now.iso8601
             }
             File.write(path, JSON.generate(data))

data/lib/llmemory/long_term/file_based/storages/memory_storage.rb

@@ -22,7 +22,7 @@ module Llmemory
             id
           end
 
-          def save_item(user_id, category:, content:, source_resource_id:)
+          def save_item(user_id, category:, content:, source_resource_id:, importance: 0.7)
             @item_id_seq += 1
             id = "item_#{@item_id_seq}"
             @items[user_id] << {
@@ -30,6 +30,7 @@ module Llmemory
               category: category,
               content: content,
               source_resource_id: source_resource_id,
+              importance: importance,
               created_at: Time.now
             }
             id

data/lib/llmemory/long_term/graph_based/memory.rb

@@ -5,6 +5,7 @@ require_relative "edge"
 require_relative "knowledge_graph"
 require_relative "conflict_resolver"
 require_relative "storage"
+require_relative "../../noise_filter"
 
 module Llmemory
   module LongTerm
@@ -21,7 +22,10 @@ module Llmemory
         end
 
         def memorize(conversation_text)
-          data = @extractor.extract(conversation_text) rescue { entities: [], relations: [] }
+          text = Llmemory.configuration.noise_filter_enabled ? NoiseFilter.filter?(conversation_text) : conversation_text.to_s
+          return true if text.strip.empty?
+
+          data = @extractor.extract(text) rescue { entities: [], relations: [] }
           data = { entities: [], relations: [] } unless data.is_a?(Hash)
           entities = Array(data[:entities] || data["entities"])
           relations = Array(data[:relations] || data["relations"])

data/lib/llmemory/memory.rb

@@ -23,7 +23,7 @@ module Llmemory
     def add_message(role:, content:)
       msgs = messages
       msgs << { role: role.to_sym, content: content.to_s }
-      save_state(messages: msgs)
+      save_state(messages: msgs, **preserved_flush_state)
       true
     end
 
@@ -31,7 +31,8 @@ module Llmemory
       state = @checkpoint.restore_state
       return [] unless state.is_a?(Hash)
       list = state[STATE_KEY_MESSAGES] || state[STATE_KEY_MESSAGES.to_s]
-      list.is_a?(Array) ? list.dup : []
+      list = list.is_a?(Array) ? list.dup : []
+      sanitize_messages(list)
     end
 
     def retrieve(query, max_tokens: nil)
@@ -67,7 +68,7 @@ module Llmemory
         soft_trim_max_bytes: Llmemory.configuration.prune_tool_results_max_bytes
       )
       pruned = pruner.prune!(msgs, mode: mode)
-      save_state(messages: pruned)
+      save_state(messages: pruned, **preserved_flush_state)
       true
     end
 
@@ -90,14 +91,16 @@ module Llmemory
       current_bytes = messages_byte_size(msgs)
       return false if current_bytes <= max
 
-      flush_memory_before_compaction!(msgs)
+      flushed = flush_memory_before_compaction!(msgs)
 
       old_msgs, recent_msgs = split_messages_by_bytes(msgs, max)
       return false if old_msgs.empty?
 
       summary = summarize_messages(old_msgs)
       compacted = [{ role: :system, content: summary }] + recent_msgs
-      save_state(messages: compacted)
+      state = restore_state_for_save
+      flush_ts = flushed ? Time.now : (state[:last_flush_at] || state["last_flush_at"])
+      save_state(messages: compacted, last_compact_at: Time.now, last_flush_at: flush_ts)
       true
     end
 
@@ -126,6 +129,25 @@ module Llmemory
       ctx >= threshold
     end
 
+    def with_overflow_recovery(max_retries: 2, &block)
+      return yield unless Llmemory.configuration.overflow_recovery_enabled
+      return yield unless block_given?
+
+      retries = 0
+      begin
+        yield
+      rescue Llmemory::LLMError => e
+        msg = e.message.to_s.downcase
+        overflow = msg.include?("context") || msg.include?("token") || msg.include?("overflow") || msg.include?("limit")
+        raise unless overflow && retries < max_retries
+
+        prune! if Llmemory.configuration.prune_tool_results_enabled
+        compact!
+        retries += 1
+        retry
+      end
+    end
+
     def check_context_window!
       return false if messages.empty?
 
@@ -169,11 +191,40 @@ module Llmemory
     end
 
     def flush_memory_before_compaction!(msgs)
-      return unless Llmemory.configuration.memory_flush_enabled
-      return if msgs.empty?
-      return if estimated_tokens(msgs) < Llmemory.configuration.memory_flush_threshold_tokens
+      return false unless Llmemory.configuration.memory_flush_enabled
+      return false if msgs.empty?
+      return false if estimated_tokens(msgs) < Llmemory.configuration.memory_flush_threshold_tokens
+
+      state = restore_state_for_save
+      last_compact = state[:last_compact_at] || state["last_compact_at"]
+      window = Llmemory.configuration.flush_once_per_cycle_seconds.to_i
+
+      if last_compact
+        t = last_compact.is_a?(Time) ? last_compact : Time.parse(last_compact.to_s)
+        return false if (Time.now - t).to_i < window
+      end
 
       consolidate!
+      true
+    end
+
+    def sanitize_messages(msgs)
+      return msgs unless Llmemory.configuration.message_sanitizer_enabled
+
+      sanitizer = ShortTerm::MessageSanitizer.new
+      sanitizer.sanitize!(msgs)
+    end
+
+    def restore_state_for_save
+      @checkpoint.restore_state || {}
+    end
+
+    def preserved_flush_state
+      state = restore_state_for_save
+      {}.tap do |h|
+        h[:last_flush_at] = state[:last_flush_at] || state["last_flush_at"] if state[:last_flush_at] || state["last_flush_at"]
+        h[:last_compact_at] = state[:last_compact_at] || state["last_compact_at"] if state[:last_compact_at] || state["last_compact_at"]
+      end
     end
 
     def estimated_tokens(msgs)
@@ -225,8 +276,10 @@ module Llmemory
       end
     end
 
-    def save_state(messages:)
+    def save_state(messages:, last_flush_at: nil, last_compact_at: nil)
       state = { STATE_KEY_MESSAGES => messages, last_activity_at: Time.now }
+      state[:last_flush_at] = last_flush_at if last_flush_at
+      state[:last_compact_at] = last_compact_at if last_compact_at
       @checkpoint.save_state(state)
     end
 

data/lib/llmemory/noise_filter.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Llmemory
+  class NoiseFilter
+    NO_REPLY_MARKER = "NO_REPLY"
+    DEFAULT_MIN_CHARS = 10
+
+    def initialize(min_chars: nil, enabled: true)
+      @min_chars = min_chars || Llmemory.configuration.noise_filter_min_chars
+      @enabled = enabled
+    end
+
+    def filter(conversation_text)
+      return conversation_text.to_s unless @enabled
+
+      lines = conversation_text.to_s.split("\n")
+      seen = {}
+      filtered = lines.select do |line|
+        next false if line.strip.length < @min_chars
+        next false if line.include?(NO_REPLY_MARKER)
+        next false if seen[line.strip]
+
+        seen[line.strip] = true
+        true
+      end
+
+      filtered.join("\n").strip
+    end
+
+    def self.filter?(conversation_text)
+      return conversation_text.to_s unless Llmemory.configuration.noise_filter_enabled
+
+      new.filter(conversation_text)
+    end
+  end
+end

data/lib/llmemory/retrieval/engine.rb

@@ -53,7 +53,8 @@ module Llmemory
           {
             text: c[:text] || c["text"],
             timestamp: parse_timestamp(c[:timestamp] || c["timestamp"] || c[:created_at] || c["created_at"]),
-            score: (c[:score] || c["score"] || 1.0).to_f
+            score: (c[:score] || c["score"] || 1.0).to_f,
+            evergreen: c[:evergreen] || c["evergreen"]
           }
         end
       end

data/lib/llmemory/retrieval/temporal_ranker.rb

@@ -8,12 +8,20 @@ module Llmemory
       end
 
       def rank(candidates, now: Time.now)
+        lambda_val = Math.log(2) / @half_life_days.to_f
+
         candidates.map do |c|
           score = (c[:score] || c["score"] || 1.0).to_f
           timestamp = c[:timestamp] || c["timestamp"]
           timestamp = Time.parse(timestamp.to_s) if timestamp.is_a?(String)
           age_days = timestamp ? (now - timestamp).to_i / 86400 : 0
-          time_decay = 1.0 / (1.0 + (age_days.to_f / @half_life_days))
+
+          time_decay = if c[:evergreen] || c["evergreen"]
+            1.0
+          else
+            Math.exp(-lambda_val * age_days.to_f)
+          end
+
           final_score = score * time_decay
           c.merge(score: score, temporal_score: final_score, timestamp: timestamp)
         end.sort_by { |c| -(c[:temporal_score] || 0) }

data/lib/llmemory/short_term/message_sanitizer.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Llmemory
+  module ShortTerm
+    class MessageSanitizer
+      def initialize(max_message_chars: nil)
+        @max_chars = max_message_chars || Llmemory.configuration.max_message_chars
+      end
+
+      def sanitize!(messages)
+        return [] if messages.nil? || !messages.is_a?(Array)
+
+        out = []
+        expect_tool_result = false
+
+        messages.each do |msg|
+          msg = msg.dup
+          content = (msg[:content] || msg["content"]).to_s
+          role = (msg[:role] || msg["role"]).to_s
+
+          next if content.strip.empty?
+
+          content = content[0, @max_chars] if @max_chars && content.length > @max_chars
+
+          if role == "tool"
+            expect_tool_result = true
+          elsif role == "tool_result"
+            next unless expect_tool_result
+            expect_tool_result = false
+          else
+            expect_tool_result = false
+          end
+
+          msg[:content] = content if msg.key?(:content)
+          msg["content"] = content if msg.key?("content")
+          out << msg
+        end
+
+        out
+      end
+    end
+  end
+end

data/lib/llmemory/short_term.rb

@@ -2,6 +2,7 @@
 
 require_relative "short_term/checkpoint"
 require_relative "short_term/session_lifecycle"
+require_relative "short_term/message_sanitizer"
 
 module Llmemory
   module ShortTerm

data/lib/llmemory/vector_store/openai_embeddings.rb

@@ -2,6 +2,7 @@
 
 require "faraday"
 require "json"
+require "digest"
 require_relative "base"
 
 module Llmemory
@@ -13,10 +14,46 @@ module Llmemory
       def initialize(api_key: nil, model: nil)
         @api_key = api_key || Llmemory.configuration.llm_api_key
         @model = model || DEFAULT_MODEL
+        @cache = {}
+        @cache_order = []
       end
 
       def embed(text)
         return Array.new(DEFAULT_DIMS, 0.0) if text.to_s.strip.empty?
+
+        if Llmemory.configuration.embedding_cache_enabled
+          key = cache_key(text)
+          return @cache[key].dup if @cache.key?(key)
+        end
+
+        result = fetch_embedding(text)
+
+        if Llmemory.configuration.embedding_cache_enabled
+          evict_if_needed
+          @cache[cache_key(text)] = result.dup
+          @cache_order << cache_key(text)
+        end
+
+        result
+      end
+
+      private
+
+      def cache_key(text)
+        Digest::SHA256.hexdigest("#{@model}:#{text.to_s.strip}")
+      end
+
+      def evict_if_needed
+        max = Llmemory.configuration.embedding_cache_max_entries.to_i
+        return if max <= 0 || @cache.size < max
+
+        while @cache_order.any? && @cache.size >= max
+          k = @cache_order.shift
+          @cache.delete(k)
+        end
+      end
+
+      def fetch_embedding(text)
         response = connection.post("embeddings") do |req|
           req.headers["Authorization"] = "Bearer #{@api_key}"
           req.headers["Content-Type"] = "application/json"
@@ -27,16 +64,6 @@ module Llmemory
         body.dig("data", 0, "embedding")&.map(&:to_f) || Array.new(DEFAULT_DIMS, 0.0)
       end
 
-      def store(id:, embedding:, metadata: {})
-        raise NotImplementedError, "OpenAIEmbeddings does not store; use a VectorStore backend (e.g. MemoryStore)"
-      end
-
-      def search(query_embedding, top_k: 10)
-        raise NotImplementedError, "OpenAIEmbeddings does not search; use a VectorStore backend"
-      end
-
-      private
-
       def connection
         @connection ||= Faraday.new(url: "https://api.openai.com/v1") do |f|
           f.request :json
@@ -44,6 +71,14 @@ module Llmemory
           f.adapter Faraday.default_adapter
         end
       end
+
+      def store(id:, embedding:, metadata: {})
+        raise NotImplementedError, "OpenAIEmbeddings does not store; use a VectorStore backend (e.g. MemoryStore)"
+      end
+
+      def search(query_embedding, top_k: 10)
+        raise NotImplementedError, "OpenAIEmbeddings does not search; use a VectorStore backend"
+      end
     end
   end
 end

data/lib/llmemory/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Llmemory
-  VERSION = "0.1.14"
+  VERSION = "0.1.15"
 end

data/lib/tasks/release.rake

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+namespace :release do
+  desc "Bump version (patch|minor|major). Checks: branch=main, no uncommitted changes, tests pass. Then: Gemfile.lock, CHANGELOG, commit, push, tag"
+  task :bump, [:bump_type] => [] do |_t, args|
+    require_relative "../llmemory/version"
+
+    # Pre-flight checks
+    current_branch = `git rev-parse --abbrev-ref HEAD`.strip
+    abort "Current branch must be main (got: #{current_branch})" unless current_branch == "main"
+
+    # Allow only release-related files to be modified (we'll commit them)
+    release_files = %w[lib/llmemory/version.rb Gemfile.lock CHANGELOG.txt]
+    status_lines = `git status --porcelain`.strip.lines
+    other_changes = status_lines.reject do |line|
+      path = line.sub(/\A..\s+/, "").strip
+      release_files.include?(path)
+    end
+    abort "Working tree has uncommitted changes outside release files. Commit or stash them first." unless other_changes.empty?
+
+    puts "Running tests..."
+    sh "bundle exec rspec"
+    puts "Tests passed.\n\n"
+
+    bump_type = (args[:bump_type] || "patch").to_s.downcase
+    unless %w[patch minor major].include?(bump_type)
+      abort "Bump type must be: patch, minor, or major"
+    end
+
+    seg = Gem::Version.new(Llmemory::VERSION).segments
+    new_version = case bump_type
+    when "patch" then Gem::Version.new("#{seg[0]}.#{seg[1] || 0}.#{(seg[2] || 0) + 1}")
+    when "minor" then Gem::Version.new("#{seg[0]}.#{(seg[1] || 0) + 1}.0")
+    when "major" then Gem::Version.new("#{(seg[0] || 0) + 1}.0.0")
+    end
+    new_version_s = new_version.to_s
+
+    puts "Bumping #{Llmemory::VERSION} -> #{new_version_s} (#{bump_type})"
+
+    # 1. Update version.rb
+    version_file = File.expand_path("../llmemory/version.rb", __dir__)
+    content = File.read(version_file)
+    content = content.sub(/VERSION = "[^"]+"/, %(VERSION = "#{new_version_s}"))
+    File.write(version_file, content)
+    puts "  Updated lib/llmemory/version.rb"
+
+    # 2. bundle install
+    sh "bundle install"
+    puts "  Updated Gemfile.lock"
+
+    # 3. Update CHANGELOG.txt
+    changelog_path = File.expand_path("../../CHANGELOG.txt", __dir__)
+    changelog_content = if File.exist?(changelog_path)
+      File.read(changelog_path)
+    else
+      ""
+    end
+
+    today = Time.now.strftime("%Y-%m-%d")
+    last_tag = `git describe --tags --abbrev=0 2>/dev/null`.strip
+    commits = if last_tag.empty?
+      `git log --oneline`.strip
+    else
+      `git log #{last_tag}..HEAD --oneline`.strip
+    end
+
+    new_entry = <<~CHANGELOG
+
+      ## [#{new_version_s}] - #{today}
+
+      ### Changes
+      #{commits.lines.map { |l| "- #{l.strip}" }.join("\n")}
+    CHANGELOG
+
+    header = "# Changelog\n\n"
+    if changelog_content.empty?
+      changelog_content = header + new_entry
+    else
+      changelog_content = header + changelog_content unless changelog_content.start_with?(header)
+      changelog_content = changelog_content.sub(/(# Changelog\n\n)/m, "\\1#{new_entry.lstrip}")
+    end
+    File.write(changelog_path, changelog_content)
+    puts "  Updated CHANGELOG.txt"
+
+    # 4. Commit
+    sh "git add lib/llmemory/version.rb Gemfile.lock CHANGELOG.txt"
+    sh "git commit -m 'Release v#{new_version_s}'"
+
+    # 5. Push
+    sh "git push"
+
+    # 6. Tag
+    sh "git tag v#{new_version_s}"
+
+    # 7. Push tag
+    sh "git push origin v#{new_version_s}"
+
+    puts "\nDone. Released v#{new_version_s}"
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: llmemory
 version: !ruby/object:Gem::Version
-  version: 0.1.14
+  version: 0.1.15
 platform: ruby
 authors:
 - llmemory
@@ -37,6 +37,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.6'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.22'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.22'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -180,6 +194,7 @@ files:
 - lib/llmemory/mcp/tools/memory_timeline.rb
 - lib/llmemory/mcp/tools/memory_timeline_context.rb
 - lib/llmemory/memory.rb
+- lib/llmemory/noise_filter.rb
 - lib/llmemory/retrieval.rb
 - lib/llmemory/retrieval/bm25_scorer.rb
 - lib/llmemory/retrieval/context_assembler.rb
@@ -188,6 +203,7 @@ files:
 - lib/llmemory/retrieval/temporal_ranker.rb
 - lib/llmemory/short_term.rb
 - lib/llmemory/short_term/checkpoint.rb
+- lib/llmemory/short_term/message_sanitizer.rb
 - lib/llmemory/short_term/pruner.rb
 - lib/llmemory/short_term/session_lifecycle.rb
 - lib/llmemory/short_term/stores/active_record_checkpoint.rb
@@ -203,6 +219,7 @@ files:
 - lib/llmemory/vector_store/memory_store.rb
 - lib/llmemory/vector_store/openai_embeddings.rb
 - lib/llmemory/version.rb
+- lib/tasks/release.rake
 homepage: https://github.com/entaina/llmemory
 licenses:
 - MIT