llmemory 0.1.17 → 0.2.1

data/lib/llmemory/long_term/episodic/memory.rb

@@ -2,6 +2,8 @@
 
 require_relative "episode"
 require_relative "storage"
+require_relative "../../memory_module"
+require_relative "../../vector_store"
 
 module Llmemory
   module LongTerm
@@ -11,14 +13,19 @@ module Llmemory
       # memory (file/graph), not replace it, and to feed reflection (P2), which
       # distills episodes into semantic knowledge.
       #
-      # Deliberately LLM-free: recording and retrieval are deterministic. Higher
-      # order summarization belongs to reflection.
+      # Recording/retrieval are deterministic and LLM-free by default. Semantic
+      # (embedding) retrieval is opt-in via `config.episodic_vector_enabled` or by
+      # injecting a `vector_store:`; when off, search is keyword-only (unchanged).
       class Memory
+        include Llmemory::MemoryModule
+
         attr_reader :user_id, :storage
 
-        def initialize(user_id:, storage: nil)
+        def initialize(user_id:, storage: nil, vector_store: nil)
           @user_id = user_id
           @storage = storage || Storages.build
+          @vector_store = vector_store
+          @vector_explicit = !vector_store.nil?
         end
 
         # Records a trajectory. `steps` is an array of hashes with any of
@@ -36,7 +43,9 @@ module Llmemory
             episode.searchable_text, method: "episode_recording", confidence: episode.importance
           )
           record = episode.to_h.merge(provenance: provenance)
-          @storage.save_episode(@user_id, record)
+          id = @storage.save_episode(@user_id, record)
+          index_vector(id, episode.searchable_text)
+          id
         end
 
         def recent_episodes(limit: 10)
@@ -59,25 +68,98 @@ module Llmemory
         # Retrieval Engine integration. Returns candidates shaped like the other
         # long-term memories so the Engine can rank episodes by relevance,
         # recency (temporal decay) and importance (P3), with provenance (P10).
+        # Hybrid (vector + keyword) when a vector store is active; otherwise
+        # keyword-only.
         def search_candidates(query, user_id: nil, top_k: 20)
           uid = user_id || @user_id
           return [] unless uid == @user_id
 
-          @storage.search_episodes(uid, query).first(top_k).map do |e|
-            episode = Episode.from_h(e)
-            {
-              text: episode.summary.to_s.empty? ? episode.searchable_text : episode.summary,
-              timestamp: episode.created_at,
-              score: 1.0,
-              importance: episode.importance,
-              evergreen: false,
-              provenance: e[:provenance] || e["provenance"]
-            }
-          end
+          keyword = @storage.search_episodes(uid, query).first(top_k).map { |e| candidate_for(e, 1.0) }
+          vs = vector_store
+          return keyword unless vs
+
+          merge_candidates(vector_candidates(query, top_k, vs), keyword, top_k)
+        end
+
+        # --- MemoryModule uniform interface ---
+
+        def write(steps:, summary: nil, outcome: nil, importance: 0.5, **_meta)
+          record_episode(steps: steps, summary: summary, outcome: outcome, importance: importance)
+        end
+
+        def list(user_id: nil, limit: nil)
+          episodes(limit: limit)
+        end
+
+        def stats(user_id: nil)
+          { episodes: count }
+        end
+
+        def forget(ids:, reason: nil)
+          requested = Array(ids).map(&:to_s)
+          existing = @storage.list_episodes(@user_id).map { |e| (e[:id] || e["id"]).to_s }
+          removed = requested & existing
+          @storage.delete_episodes(@user_id, removed)
+          forget_log.record(@user_id, memory_type: "episodic", ids: removed, reason: reason)
+          removed.size
         end
 
         private
 
+        # Active vector store: the injected one, or a config-gated lazy build.
+        # Returns nil when semantic search is disabled (default).
+        def vector_store
+          if @vector_explicit
+            @vector_store
+          elsif Llmemory.configuration.episodic_vector_enabled
+            @vector_store ||= Llmemory::VectorStore.build(source_type: "episode")
+          end
+        end
+
+        # Best-effort embedding indexing; a failure must never break recording.
+        def index_vector(id, text)
+          vs = vector_store
+          return if vs.nil? || text.to_s.strip.empty?
+          embedding = vs.embed(text)
+          return unless embedding
+          vs.store(id: id, embedding: embedding, metadata: { text: text, created_at: Time.now }, user_id: @user_id)
+        rescue StandardError
+          nil
+        end
+
+        def vector_candidates(query, top_k, vs)
+          vs.search_by_text(query.to_s, top_k: top_k, user_id: @user_id).filter_map do |r|
+            raw = @storage.get_episode(@user_id, r[:id] || r["id"])
+            raw && candidate_for(raw, (r[:score] || r["score"] || 1.0).to_f)
+          end
+        rescue StandardError
+          []
+        end
+
+        def candidate_for(raw, score)
+          episode = Episode.from_h(raw)
+          {
+            id: episode.id,
+            text: episode.summary.to_s.empty? ? episode.searchable_text : episode.summary,
+            timestamp: episode.created_at,
+            score: score,
+            importance: episode.importance,
+            evergreen: false,
+            provenance: raw[:provenance] || raw["provenance"]
+          }
+        end
+
+        # Dedup by id keeping the higher score; highest score first, capped.
+        def merge_candidates(primary, secondary, top_k)
+          by_id = {}
+          (primary + secondary).each do |c|
+            key = c[:id] || c[:text]
+            existing = by_id[key]
+            by_id[key] = c if existing.nil? || c[:score].to_f > existing[:score].to_f
+          end
+          by_id.values.sort_by { |c| -c[:score].to_f }.first(top_k)
+        end
+
         # Cheap, deterministic summary when the caller does not provide one.
         # LLM-based summarization is reflection's job (P2).
         def derive_summary(steps)

data/lib/llmemory/long_term/episodic/storage.rb

@@ -3,6 +3,7 @@
 require_relative "storages/base"
 require_relative "storages/memory_storage"
 require_relative "storages/file_storage"
+require_relative "storages/database_storage"
 
 module Llmemory
   module LongTerm
@@ -11,16 +12,17 @@ module Llmemory
       Storage = Storages::MemoryStorage
 
       module Storages
-        def self.build(store: nil, base_path: nil)
+        def self.build(store: nil, base_path: nil, database_url: nil)
           case (store || Llmemory.configuration.long_term_store).to_s.to_sym
           when :memory
             MemoryStorage.new
           when :file
             FileStorage.new(base_path: base_path || Llmemory.configuration.long_term_storage_path)
-          when :postgres, :database, :active_record, :activerecord
-            raise NotImplementedError,
-              "Episodic SQL/ActiveRecord storage is not implemented yet; use :memory or :file " \
-              "(or pass an explicit storage instance)."
+          when :postgres, :database
+            DatabaseStorage.new(database_url: database_url || Llmemory.configuration.database_url)
+          when :active_record, :activerecord
+            require_relative "storages/active_record_storage"
+            ActiveRecordStorage.new
           else
             MemoryStorage.new
           end

data/lib/llmemory/long_term/episodic/storages/active_record_models.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+# Model for Episodic ActiveRecordStorage. Loaded only when using
+# store: :active_record. JSONB `data` auto-deserializes to a Hash in Rails 5+.
+
+module Llmemory
+  module LongTerm
+    module Episodic
+      module Storages
+        class LlmemoryEpisode < ::ActiveRecord::Base
+          self.table_name = "llmemory_episodes"
+          self.primary_key = "id"
+        end
+      end
+    end
+  end
+end

data/lib/llmemory/long_term/episodic/storages/active_record_storage.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require "json"
+require "securerandom"
+require "time"
+require_relative "base"
+
+module Llmemory
+  module LongTerm
+    module Episodic
+      module Storages
+        # ActiveRecord backend. Stores each episode as a JSONB `data` document;
+        # AR auto-deserializes jsonb to a Hash (string keys), which Episode.from_h
+        # handles. Mirrors the file-based ActiveRecordStorage pattern.
+        class ActiveRecordStorage < Base
+          def initialize
+            self.class.load_models!
+          end
+
+          def self.load_models!
+            return if @models_loaded
+            require "active_record"
+            require_relative "active_record_models"
+            @models_loaded = true
+          end
+
+          def save_episode(user_id, episode)
+            id = episode[:id] || episode["id"] || "ep_#{SecureRandom.hex(8)}"
+            data = stringify(episode).merge("id" => id, "user_id" => user_id)
+            data["created_at"] ||= Time.now.utc.iso8601
+            rec = LlmemoryEpisode.find_or_initialize_by(id: id)
+            rec.user_id = user_id
+            rec.data = data
+            rec.search_text = searchable_text(data)
+            rec.created_at ||= Time.current
+            rec.save!
+            id
+          end
+
+          def get_episode(user_id, id)
+            rec = LlmemoryEpisode.find_by(user_id: user_id, id: id)
+            rec&.data
+          end
+
+          def list_episodes(user_id, limit: nil)
+            scope = LlmemoryEpisode.where(user_id: user_id).order(created_at: :desc)
+            scope = scope.limit(limit) if limit && limit.to_i.positive?
+            scope.map(&:data)
+          end
+
+          def search_episodes(user_id, query)
+            token_scope(LlmemoryEpisode.where(user_id: user_id), "search_text", query)
+              .order(created_at: :desc).map(&:data)
+          end
+
+          def count_episodes(user_id)
+            LlmemoryEpisode.where(user_id: user_id).count
+          end
+
+          def delete_episodes(user_id, ids)
+            LlmemoryEpisode.where(user_id: user_id, id: Array(ids).map(&:to_s)).delete_all
+          end
+
+          def list_users
+            LlmemoryEpisode.distinct.pluck(:user_id)
+          end
+
+          private
+
+          def token_scope(scope, column, query)
+            tokens = Llmemory::Tokenizer.tokenize(query)
+            return scope if tokens.empty?
+            clause = tokens.map { "LOWER(#{column}) LIKE LOWER(?)" }.join(" OR ")
+            scope.where(clause, *tokens.map { |t| "%#{t}%" })
+          end
+
+          def stringify(hash)
+            JSON.parse(JSON.generate(hash))
+          end
+
+          def searchable_text(data)
+            parts = [data["summary"], data["outcome"]]
+            Array(data["steps"]).each do |s|
+              next unless s.is_a?(Hash)
+              parts << s["observation"] << s["action"] << s["result"]
+            end
+            parts.compact.join("\n")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/llmemory/long_term/episodic/storages/base.rb

@@ -29,6 +29,11 @@ module Llmemory
             raise NotImplementedError, "#{self.class}#count_episodes must be implemented"
           end
 
+          # Deletes episodes by id. Returns the number actually removed.
+          def delete_episodes(user_id, ids)
+            raise NotImplementedError, "#{self.class}#delete_episodes must be implemented"
+          end
+
           def list_users
             raise NotImplementedError, "#{self.class}#list_users must be implemented"
           end

data/lib/llmemory/long_term/episodic/storages/database_storage.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+require "json"
+require "securerandom"
+require "time"
+require_relative "base"
+
+module Llmemory
+  module LongTerm
+    module Episodic
+      module Storages
+        # PostgreSQL backend. Each episode is stored as a JSONB `data` document
+        # (plus id/user_id/created_at and a denormalized search_text for keyword
+        # search), mirroring the file-based DatabaseStorage pattern.
+        class DatabaseStorage < Base
+          def initialize(database_url: nil)
+            @database_url = database_url || Llmemory.configuration.database_url
+            @connection = nil
+          end
+
+          def save_episode(user_id, episode)
+            ensure_tables!
+            id = episode[:id] || episode["id"] || "ep_#{SecureRandom.hex(8)}"
+            data = symbolize(episode).merge(id: id, user_id: user_id)
+            data[:created_at] ||= Time.now.utc.iso8601
+            conn.exec_params(
+              "INSERT INTO llmemory_episodes (id, user_id, data, search_text, created_at) " \
+              "VALUES ($1, $2, $3::jsonb, $4, $5) " \
+              "ON CONFLICT (id) DO UPDATE SET data = $3::jsonb, search_text = $4",
+              [id, user_id, JSON.generate(data), searchable_text(data), created_at_value(data)]
+            )
+            id
+          end
+
+          def get_episode(user_id, id)
+            ensure_tables!
+            rows = conn.exec_params("SELECT data FROM llmemory_episodes WHERE user_id = $1 AND id = $2", [user_id, id])
+            rows.any? ? parse_data(rows.first["data"]) : nil
+          end
+
+          def list_episodes(user_id, limit: nil)
+            ensure_tables!
+            sql = "SELECT data FROM llmemory_episodes WHERE user_id = $1 ORDER BY created_at DESC"
+            sql += " LIMIT #{limit.to_i}" if limit && limit.to_i.positive?
+            conn.exec_params(sql, [user_id]).map { |r| parse_data(r["data"]) }
+          end
+
+          def search_episodes(user_id, query)
+            ensure_tables!
+            suffix, params = token_filter("search_text", query, 2)
+            conn.exec_params(
+              "SELECT data FROM llmemory_episodes WHERE user_id = $1#{suffix} ORDER BY created_at DESC",
+              [user_id, *params]
+            ).map { |r| parse_data(r["data"]) }
+          end
+
+          def count_episodes(user_id)
+            ensure_tables!
+            conn.exec_params("SELECT COUNT(*) AS c FROM llmemory_episodes WHERE user_id = $1", [user_id]).first["c"].to_i
+          end
+
+          def delete_episodes(user_id, ids)
+            ensure_tables!
+            Array(ids).sum do |id|
+              conn.exec_params("DELETE FROM llmemory_episodes WHERE user_id = $1 AND id = $2", [user_id, id]).cmd_tuples
+            end
+          end
+
+          def list_users
+            ensure_tables!
+            conn.exec("SELECT DISTINCT user_id FROM llmemory_episodes").map { |r| r["user_id"] }
+          end
+
+          private
+
+          def conn
+            @connection ||= begin
+              require "pg"
+              PG.connect(@database_url)
+            end
+          end
+
+          def ensure_tables!
+            conn.exec(<<~SQL)
+              CREATE TABLE IF NOT EXISTS llmemory_episodes (
+                id TEXT NOT NULL PRIMARY KEY,
+                user_id TEXT NOT NULL,
+                data JSONB NOT NULL DEFAULT '{}'::jsonb,
+                search_text TEXT,
+                created_at TIMESTAMPTZ NOT NULL
+              );
+              CREATE INDEX IF NOT EXISTS idx_llmemory_episodes_user_id ON llmemory_episodes(user_id);
+            SQL
+          end
+
+          # OR-of-token LIKE filter (see file-based DatabaseStorage). [""] for an
+          # empty query => match all.
+          def token_filter(column, query, start_index)
+            tokens = Llmemory::Tokenizer.tokenize(query)
+            return ["", []] if tokens.empty?
+            likes = tokens.each_index.map { |i| "LOWER(#{column}) LIKE $#{start_index + i}" }
+            [" AND (#{likes.join(' OR ')})", tokens.map { |t| "%#{t}%" }]
+          end
+
+          def parse_data(value)
+            JSON.parse(value.to_s, symbolize_names: true)
+          rescue JSON::ParserError
+            {}
+          end
+
+          def symbolize(hash)
+            hash.each_with_object({}) { |(k, v), acc| acc[k.to_sym] = v }
+          end
+
+          def searchable_text(data)
+            parts = [data[:summary], data[:outcome]]
+            Array(data[:steps]).each do |s|
+              next unless s.is_a?(Hash)
+              parts << (s[:observation] || s["observation"])
+              parts << (s[:action] || s["action"])
+              parts << (s[:result] || s["result"])
+            end
+            parts.compact.join("\n")
+          end
+
+          def created_at_value(data)
+            ca = data[:created_at]
+            return Time.now.utc.iso8601 if ca.nil?
+            ca.respond_to?(:iso8601) ? ca.iso8601 : ca.to_s
+          end
+        end
+      end
+    end
+  end
+end

data/lib/llmemory/long_term/episodic/storages/file_storage.rb

@@ -35,9 +35,8 @@ module Llmemory
           end
 
           def search_episodes(user_id, query)
-            q = query.to_s.downcase
-            return list_episodes(user_id) if q.strip.empty?
-            all_episodes(user_id).select { |e| episode_text(e).downcase.include?(q) }
+            return list_episodes(user_id) if query.to_s.strip.empty?
+            all_episodes(user_id).select { |e| Llmemory::Tokenizer.matches?(episode_text(e), query) }
           end
 
           def count_episodes(user_id)
@@ -46,6 +45,15 @@ module Llmemory
             Dir.children(dir).count { |f| f.end_with?(".json") }
           end
 
+          def delete_episodes(user_id, ids)
+            Array(ids).map(&:to_s).count do |id|
+              path = episode_path(user_id, id)
+              next false unless File.file?(path)
+              File.delete(path)
+              true
+            end
+          end
+
           def list_users
             return [] unless Dir.exist?(@base_path)
             Dir.children(@base_path).select { |d| Dir.exist?(File.join(@base_path, d, "episodes")) }

data/lib/llmemory/long_term/episodic/storages/memory_storage.rb

@@ -31,15 +31,21 @@ module Llmemory
           end
 
           def search_episodes(user_id, query)
-            q = query.to_s.downcase
-            return list_episodes(user_id) if q.strip.empty?
-            @episodes[user_id].select { |e| episode_text(e).downcase.include?(q) }
+            return list_episodes(user_id) if query.to_s.strip.empty?
+            @episodes[user_id].select { |e| Llmemory::Tokenizer.matches?(episode_text(e), query) }
           end
 
           def count_episodes(user_id)
             @episodes[user_id].size
           end
 
+          def delete_episodes(user_id, ids)
+            ids = Array(ids).map(&:to_s)
+            before = @episodes[user_id].size
+            @episodes[user_id].reject! { |e| ids.include?(e[:id].to_s) }
+            before - @episodes[user_id].size
+          end
+
           def list_users
             @episodes.keys
           end

data/lib/llmemory/long_term/file_based/memory.rb

@@ -5,11 +5,14 @@ require_relative "item"
 require_relative "category"
 require_relative "storage"
 require_relative "../../noise_filter"
+require_relative "../../memory_module"
 
 module Llmemory
   module LongTerm
     module FileBased
       class Memory
+        include Llmemory::MemoryModule
+
         def initialize(user_id:, storage: nil, llm: nil, extractor: nil)
           @user_id = user_id
           @storage = storage || Storages.build
@@ -63,6 +66,7 @@ module Llmemory
 
           items.first(top_k).each do |i|
             out << {
+              id: i[:id] || i["id"],
               text: i[:content] || i["content"],
               timestamp: i[:created_at] || i["created_at"],
               score: 1.0,
@@ -72,6 +76,7 @@ module Llmemory
           end
           resources.first([top_k - out.size, 0].max).each do |r|
             out << {
+              id: r[:id] || r["id"],
               text: r[:text] || r["text"],
               timestamp: r[:created_at] || r["created_at"],
               score: 0.9
@@ -100,6 +105,32 @@ module Llmemory
           )
         end
 
+        # --- MemoryModule uniform interface ---
+
+        def write(payload, **_meta)
+          memorize(payload)
+        end
+
+        def list(user_id: nil, limit: nil)
+          @storage.list_items(user_id: user_id || @user_id, limit: limit)
+        end
+
+        def stats(user_id: nil)
+          { items: @storage.count_items(user_id: user_id || @user_id) }
+        end
+
+        # Removes items/resources by id and records the removal in the audit log.
+        def forget(ids:, reason: nil)
+          requested = Array(ids).map(&:to_s)
+          existing = (@storage.get_all_items(@user_id) + @storage.get_all_resources(@user_id))
+            .map { |r| (r[:id] || r["id"]).to_s }
+          removed = requested & existing
+          @storage.archive_items(@user_id, removed)
+          @storage.archive_resources(@user_id, removed)
+          forget_log.record(@user_id, memory_type: "file_based", ids: removed, reason: reason)
+          removed.size
+        end
+
         attr_reader :storage, :user_id
 
         private

data/lib/llmemory/long_term/file_based/storages/active_record_storage.rb

@@ -64,13 +64,11 @@ module Llmemory
           end
 
           def search_items(user_id, query)
-            q = "%#{sanitize_like(query)}%"
-            LlmemoryItem.where(user_id: user_id).where("LOWER(content) LIKE LOWER(?)", q).map { |r| row_to_item(r) }
+            token_scope(LlmemoryItem.where(user_id: user_id), "content", query).map { |r| row_to_item(r) }
           end
 
           def search_resources(user_id, query)
-            q = "%#{sanitize_like(query)}%"
-            LlmemoryResource.where(user_id: user_id).where("LOWER(text) LIKE LOWER(?)", q).map { |r| row_to_resource(r) }
+            token_scope(LlmemoryResource.where(user_id: user_id), "text", query).map { |r| row_to_resource(r) }
           end
 
           def get_resources_since(user_id, hours:)
@@ -181,6 +179,15 @@ module Llmemory
             (str || "").to_s.gsub(/[%_\\]/) { |c| "\\#{c}" }
           end
 
+          # OR-of-token LIKE scope for keyword search; unchanged scope (match all)
+          # when the query has no tokens.
+          def token_scope(scope, column, query)
+            tokens = Llmemory::Tokenizer.tokenize(query)
+            return scope if tokens.empty?
+            clause = tokens.map { "LOWER(#{column}) LIKE LOWER(?)" }.join(" OR ")
+            scope.where(clause, *tokens.map { |t| "%#{sanitize_like(t)}%" })
+          end
+
           def row_to_item(r)
             h = {
               id: r.id,

data/lib/llmemory/long_term/file_based/storages/database_storage.rb

@@ -65,20 +65,20 @@ module Llmemory
 
           def search_items(user_id, query)
             ensure_tables!
-            pattern = "%#{conn.escape_string(query.to_s.downcase)}%"
+            suffix, params = token_filter("content", query, 2)
             rows = conn.exec_params(
-              "SELECT id, category, content, source_resource_id, importance, provenance, created_at FROM llmemory_items WHERE user_id = $1 AND LOWER(content) LIKE $2",
-              [user_id, pattern]
+              "SELECT id, category, content, source_resource_id, importance, provenance, created_at FROM llmemory_items WHERE user_id = $1#{suffix}",
+              [user_id, *params]
             )
             rows_to_items(rows)
           end
 
           def search_resources(user_id, query)
             ensure_tables!
-            pattern = "%#{conn.escape_string(query.to_s.downcase)}%"
+            suffix, params = token_filter("text", query, 2)
             rows = conn.exec_params(
-              "SELECT id, text, created_at FROM llmemory_resources WHERE user_id = $1 AND LOWER(text) LIKE $2",
-              [user_id, pattern]
+              "SELECT id, text, created_at FROM llmemory_resources WHERE user_id = $1#{suffix}",
+              [user_id, *params]
             )
             rows_to_resources(rows)
           end
@@ -290,6 +290,16 @@ module Llmemory
             end
           end
 
+          # Builds an OR-of-token LIKE filter for keyword search. Returns
+          # ["" , []] for an empty query (match all). Tokens are [a-z0-9]{2,} so
+          # they carry no LIKE wildcards.
+          def token_filter(column, query, start_index)
+            tokens = Llmemory::Tokenizer.tokenize(query)
+            return ["", []] if tokens.empty?
+            likes = tokens.each_index.map { |i| "LOWER(#{column}) LIKE $#{start_index + i}" }
+            [" AND (#{likes.join(' OR ')})", tokens.map { |t| "%#{t}%" }]
+          end
+
           def parse_provenance(value)
             return nil if value.nil? || value.to_s.strip.empty?
             JSON.parse(value, symbolize_names: true)

data/lib/llmemory/long_term/file_based/storages/file_storage.rb

@@ -62,13 +62,11 @@ module Llmemory
           end
 
           def search_items(user_id, query)
-            query_lower = query.downcase
-            get_all_items(user_id).select { |i| (i[:content] || i["content"]).to_s.downcase.include?(query_lower) }
+            get_all_items(user_id).select { |i| Llmemory::Tokenizer.matches?(i[:content] || i["content"], query) }
           end
 
           def search_resources(user_id, query)
-            query_lower = query.downcase
-            get_all_resources(user_id).select { |r| (r[:text] || r["text"]).to_s.downcase.include?(query_lower) }
+            get_all_resources(user_id).select { |r| Llmemory::Tokenizer.matches?(r[:text] || r["text"], query) }
           end
 
           def get_resources_since(user_id, hours:)

data/lib/llmemory/long_term/file_based/storages/memory_storage.rb

@@ -51,13 +51,11 @@ module Llmemory
           end
 
           def search_items(user_id, query)
-            query_lower = query.downcase
-            @items[user_id].select { |i| i[:content].to_s.downcase.include?(query_lower) }
+            @items[user_id].select { |i| Llmemory::Tokenizer.matches?(i[:content], query) }
           end
 
           def search_resources(user_id, query)
-            query_lower = query.downcase
-            @resources[user_id].select { |r| r[:text].to_s.downcase.include?(query_lower) }
+            @resources[user_id].select { |r| Llmemory::Tokenizer.matches?(r[:text], query) }
           end
 
           def get_resources_since(user_id, hours:)