llmemory 0.2.0 → 0.2.2

data/lib/llmemory/long_term/episodic/storages/active_record_models.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+# Model for Episodic ActiveRecordStorage. Loaded only when using
+# store: :active_record. JSONB `data` auto-deserializes to a Hash in Rails 5+.
+
+module Llmemory
+  module LongTerm
+    module Episodic
+      module Storages
+        class LlmemoryEpisode < ::ActiveRecord::Base
+          self.table_name = "llmemory_episodes"
+          self.primary_key = "id"
+        end
+      end
+    end
+  end
+end

data/lib/llmemory/long_term/episodic/storages/active_record_storage.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+require "json"
+require "securerandom"
+require "time"
+require_relative "base"
+
+module Llmemory
+  module LongTerm
+    module Episodic
+      module Storages
+        # ActiveRecord backend. Stores each episode as a JSONB `data` document;
+        # AR auto-deserializes jsonb to a Hash (string keys), which Episode.from_h
+        # handles. Mirrors the file-based ActiveRecordStorage pattern.
+        class ActiveRecordStorage < Base
+          def initialize
+            self.class.load_models!
+          end
+
+          def self.load_models!
+            return if @models_loaded
+            require "active_record"
+            require_relative "active_record_models"
+            @models_loaded = true
+          end
+
+          def save_episode(user_id, episode)
+            id = episode[:id] || episode["id"] || "ep_#{SecureRandom.hex(8)}"
+            data = stringify(episode).merge("id" => id, "user_id" => user_id)
+            data["created_at"] ||= Time.now.utc.iso8601
+            rec = LlmemoryEpisode.find_or_initialize_by(id: id)
+            rec.user_id = user_id
+            rec.data = data
+            rec.search_text = searchable_text(data)
+            rec.created_at ||= Time.current
+            rec.save!
+            id
+          end
+
+          def get_episode(user_id, id)
+            rec = LlmemoryEpisode.find_by(user_id: user_id, id: id)
+            rec&.data
+          end
+
+          def list_episodes(user_id, limit: nil, offset: nil)
+            scope = LlmemoryEpisode.where(user_id: user_id, archived_at: nil).order(created_at: :desc)
+            scope = scope.limit(limit) if limit && limit.to_i.positive?
+            scope = scope.offset(offset) if offset && offset.to_i.positive?
+            scope.map(&:data)
+          end
+
+          def search_episodes(user_id, query)
+            token_scope(LlmemoryEpisode.where(user_id: user_id, archived_at: nil), "search_text", query)
+              .order(created_at: :desc).map(&:data)
+          end
+
+          def count_episodes(user_id)
+            LlmemoryEpisode.where(user_id: user_id, archived_at: nil).count
+          end
+
+          def delete_episodes(user_id, ids)
+            LlmemoryEpisode.where(user_id: user_id, id: Array(ids).map(&:to_s)).delete_all
+          end
+
+          def archive_episodes(user_id, ids)
+            LlmemoryEpisode.where(user_id: user_id, id: Array(ids).map(&:to_s), archived_at: nil)
+              .update_all(archived_at: Time.current)
+          end
+
+          def expired_episode_ids(user_id, cutoff:)
+            LlmemoryEpisode.where(user_id: user_id, archived_at: nil).where("created_at < ?", cutoff).pluck(:id)
+          end
+
+          def list_users
+            LlmemoryEpisode.distinct.pluck(:user_id)
+          end
+
+          private
+
+          def token_scope(scope, column, query)
+            tokens = Llmemory::Tokenizer.tokenize(query)
+            return scope if tokens.empty?
+            clause = tokens.map { "LOWER(#{column}) LIKE LOWER(?)" }.join(" OR ")
+            scope.where(clause, *tokens.map { |t| "%#{t}%" })
+          end
+
+          def stringify(hash)
+            JSON.parse(JSON.generate(hash))
+          end
+
+          def searchable_text(data)
+            parts = [data["summary"], data["outcome"]]
+            Array(data["steps"]).each do |s|
+              next unless s.is_a?(Hash)
+              parts << s["observation"] << s["action"] << s["result"]
+            end
+            parts.compact.join("\n")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/llmemory/long_term/episodic/storages/base.rb

@@ -16,8 +16,8 @@ module Llmemory
             raise NotImplementedError, "#{self.class}#get_episode must be implemented"
           end
 
-          # Newest first. Optionally capped by limit.
-          def list_episodes(user_id, limit: nil)
+          # Newest first. Optionally paginated with offset/limit.
+          def list_episodes(user_id, limit: nil, offset: nil)
             raise NotImplementedError, "#{self.class}#list_episodes must be implemented"
           end
 
@@ -34,6 +34,19 @@ module Llmemory
             raise NotImplementedError, "#{self.class}#delete_episodes must be implemented"
           end
 
+          # Soft-archives episodes by id (sets archived_at on the record). Archived
+          # episodes are excluded from list_episodes / search_episodes / count_episodes
+          # but remain accessible via get_episode. Returns the number archived.
+          def archive_episodes(user_id, ids)
+            raise NotImplementedError, "#{self.class}#archive_episodes must be implemented"
+          end
+
+          # Returns episodes whose created_at is older than the cutoff and that are
+          # not already archived. Used by the TTL maintenance job.
+          def expired_episode_ids(user_id, cutoff:)
+            raise NotImplementedError, "#{self.class}#expired_episode_ids must be implemented"
+          end
+
           def list_users
             raise NotImplementedError, "#{self.class}#list_users must be implemented"
           end

data/lib/llmemory/long_term/episodic/storages/database_storage.rb

@@ -0,0 +1,156 @@
+# frozen_string_literal: true
+
+require "json"
+require "securerandom"
+require "time"
+require_relative "base"
+
+module Llmemory
+  module LongTerm
+    module Episodic
+      module Storages
+        # PostgreSQL backend. Each episode is stored as a JSONB `data` document
+        # (plus id/user_id/created_at and a denormalized search_text for keyword
+        # search), mirroring the file-based DatabaseStorage pattern.
+        class DatabaseStorage < Base
+          def initialize(database_url: nil)
+            @database_url = database_url || Llmemory.configuration.database_url
+            @connection = nil
+          end
+
+          def save_episode(user_id, episode)
+            ensure_tables!
+            id = episode[:id] || episode["id"] || "ep_#{SecureRandom.hex(8)}"
+            data = symbolize(episode).merge(id: id, user_id: user_id)
+            data[:created_at] ||= Time.now.utc.iso8601
+            conn.exec_params(
+              "INSERT INTO llmemory_episodes (id, user_id, data, search_text, created_at) " \
+              "VALUES ($1, $2, $3::jsonb, $4, $5) " \
+              "ON CONFLICT (id) DO UPDATE SET data = $3::jsonb, search_text = $4",
+              [id, user_id, JSON.generate(data), searchable_text(data), created_at_value(data)]
+            )
+            id
+          end
+
+          def get_episode(user_id, id)
+            ensure_tables!
+            rows = conn.exec_params("SELECT data FROM llmemory_episodes WHERE user_id = $1 AND id = $2", [user_id, id])
+            rows.any? ? parse_data(rows.first["data"]) : nil
+          end
+
+          def list_episodes(user_id, limit: nil, offset: nil)
+            ensure_tables!
+            sql = "SELECT data FROM llmemory_episodes WHERE user_id = $1 AND archived_at IS NULL ORDER BY created_at DESC"
+            sql += " LIMIT #{limit.to_i}" if limit && limit.to_i.positive?
+            sql += " OFFSET #{offset.to_i}" if offset && offset.to_i.positive?
+            conn.exec_params(sql, [user_id]).map { |r| parse_data(r["data"]) }
+          end
+
+          def search_episodes(user_id, query)
+            ensure_tables!
+            suffix, params = token_filter("search_text", query, 2)
+            conn.exec_params(
+              "SELECT data FROM llmemory_episodes WHERE user_id = $1 AND archived_at IS NULL#{suffix} ORDER BY created_at DESC",
+              [user_id, *params]
+            ).map { |r| parse_data(r["data"]) }
+          end
+
+          def count_episodes(user_id)
+            ensure_tables!
+            conn.exec_params("SELECT COUNT(*) AS c FROM llmemory_episodes WHERE user_id = $1 AND archived_at IS NULL", [user_id]).first["c"].to_i
+          end
+
+          def delete_episodes(user_id, ids)
+            ensure_tables!
+            Array(ids).sum do |id|
+              conn.exec_params("DELETE FROM llmemory_episodes WHERE user_id = $1 AND id = $2", [user_id, id]).cmd_tuples
+            end
+          end
+
+          def archive_episodes(user_id, ids)
+            ensure_tables!
+            Array(ids).sum do |id|
+              conn.exec_params(
+                "UPDATE llmemory_episodes SET archived_at = NOW() WHERE user_id = $1 AND id = $2 AND archived_at IS NULL",
+                [user_id, id]
+              ).cmd_tuples
+            end
+          end
+
+          def expired_episode_ids(user_id, cutoff:)
+            ensure_tables!
+            conn.exec_params(
+              "SELECT id FROM llmemory_episodes WHERE user_id = $1 AND archived_at IS NULL AND created_at < $2",
+              [user_id, cutoff.iso8601]
+            ).map { |r| r["id"] }
+          end
+
+          def list_users
+            ensure_tables!
+            conn.exec("SELECT DISTINCT user_id FROM llmemory_episodes").map { |r| r["user_id"] }
+          end
+
+          private
+
+          def conn
+            @connection ||= begin
+              require "pg"
+              PG.connect(@database_url)
+            end
+          end
+
+          def ensure_tables!
+            conn.exec(<<~SQL)
+              CREATE TABLE IF NOT EXISTS llmemory_episodes (
+                id TEXT NOT NULL PRIMARY KEY,
+                user_id TEXT NOT NULL,
+                data JSONB NOT NULL DEFAULT '{}'::jsonb,
+                search_text TEXT,
+                created_at TIMESTAMPTZ NOT NULL,
+                archived_at TIMESTAMPTZ
+              );
+              CREATE INDEX IF NOT EXISTS idx_llmemory_episodes_user_id ON llmemory_episodes(user_id);
+              ALTER TABLE llmemory_episodes ADD COLUMN IF NOT EXISTS archived_at TIMESTAMPTZ;
+            SQL
+          end
+
+          # OR-of-token LIKE filter (see file-based DatabaseStorage). [""] for an
+          # empty query => match all.
+          def token_filter(column, query, start_index)
+            tokens = Llmemory::Tokenizer.tokenize(query)
+            return ["", []] if tokens.empty?
+            likes = tokens.each_index.map { |i| "LOWER(#{column}) LIKE $#{start_index + i}" }
+            [" AND (#{likes.join(' OR ')})", tokens.map { |t| "%#{t}%" }]
+          end
+
+          def parse_data(value)
+            JSON.parse(value.to_s, symbolize_names: true)
+          rescue JSON::ParserError
+            {}
+          end
+
+          def symbolize(hash)
+            hash.each_with_object({}) { |(k, v), acc| acc[k.to_sym] = v }
+          end
+
+          def searchable_text(data)
+            parts = [data[:summary], data[:outcome]]
+            Array(data[:steps]).each do |s|
+              next unless s.is_a?(Hash)
+              parts << (s[:observation] || s["observation"])
+              parts << (s[:action] || s["action"])
+              parts << (s[:result] || s["result"])
+            end
+            parts.compact.join("\n")
+          end
+
+          def created_at_value(data)
+            ca = data[:created_at]
+            return Time.now.utc.iso8601 if ca.nil?
+            ca.respond_to?(:iso8601) ? ca.iso8601 : ca.to_s
+          end
+        end
+      end
+    end
+  end
+end

data/lib/llmemory/long_term/episodic/storages/file_storage.rb

@@ -29,21 +29,19 @@ module Llmemory
             load_episode(path)
           end
 
-          def list_episodes(user_id, limit: nil)
-            sorted = all_episodes(user_id).sort_by { |e| e[:created_at] }.reverse
+          def list_episodes(user_id, limit: nil, offset: nil)
+            sorted = active_episodes(user_id).sort_by { |e| e[:created_at] }.reverse
+            sorted = sorted.drop(offset.to_i) if offset && offset.to_i.positive?
             limit && limit.to_i.positive? ? sorted.first(limit.to_i) : sorted
           end
 
           def search_episodes(user_id, query)
-            q = query.to_s.downcase
-            return list_episodes(user_id) if q.strip.empty?
-            all_episodes(user_id).select { |e| episode_text(e).downcase.include?(q) }
+            return list_episodes(user_id) if query.to_s.strip.empty?
+            active_episodes(user_id).select { |e| Llmemory::Tokenizer.matches?(episode_text(e), query) }
           end
 
           def count_episodes(user_id)
-            dir = user_path(user_id, "episodes")
-            return 0 unless Dir.exist?(dir)
-            Dir.children(dir).count { |f| f.end_with?(".json") }
+            active_episodes(user_id).size
           end
 
           def delete_episodes(user_id, ids)
@@ -55,6 +53,24 @@ module Llmemory
             end
           end
 
+          def archive_episodes(user_id, ids)
+            Array(ids).map(&:to_s).count do |id|
+              path = episode_path(user_id, id)
+              next false unless File.file?(path)
+              data = JSON.parse(File.read(path))
+              next false if data["archived_at"]
+              data["archived_at"] = Time.now.iso8601
+              File.write(path, JSON.generate(data))
+              true
+            end
+          end
+
+          def expired_episode_ids(user_id, cutoff:)
+            active_episodes(user_id)
+              .select { |e| (e[:created_at] || Time.now) < cutoff }
+              .map { |e| e[:id].to_s }
+          end
+
           def list_users
             return [] unless Dir.exist?(@base_path)
             Dir.children(@base_path).select { |d| Dir.exist?(File.join(@base_path, d, "episodes")) }
@@ -62,6 +78,10 @@ module Llmemory
 
           private
 
+          def active_episodes(user_id)
+            all_episodes(user_id).reject { |e| e[:archived_at] }
+          end
+
           def all_episodes(user_id)
             dir = user_path(user_id, "episodes")
             return [] unless Dir.exist?(dir)

data/lib/llmemory/long_term/episodic/storages/memory_storage.rb

@@ -25,19 +25,19 @@ module Llmemory
             @episodes[user_id].find { |e| e[:id] == id }
           end
 
-          def list_episodes(user_id, limit: nil)
-            sorted = @episodes[user_id].sort_by { |e| e[:created_at] }.reverse
+          def list_episodes(user_id, limit: nil, offset: nil)
+            sorted = active_episodes(user_id).sort_by { |e| as_time(e[:created_at]) }.reverse
+            sorted = sorted.drop(offset.to_i) if offset && offset.to_i.positive?
             limit && limit.to_i.positive? ? sorted.first(limit.to_i) : sorted
           end
 
           def search_episodes(user_id, query)
-            q = query.to_s.downcase
-            return list_episodes(user_id) if q.strip.empty?
-            @episodes[user_id].select { |e| episode_text(e).downcase.include?(q) }
+            return list_episodes(user_id) if query.to_s.strip.empty?
+            active_episodes(user_id).select { |e| Llmemory::Tokenizer.matches?(episode_text(e), query) }
           end
 
           def count_episodes(user_id)
-            @episodes[user_id].size
+            active_episodes(user_id).size
           end
 
           def delete_episodes(user_id, ids)
@@ -47,12 +47,42 @@ module Llmemory
             before - @episodes[user_id].size
           end
 
+          def archive_episodes(user_id, ids)
+            ids = Array(ids).map(&:to_s)
+            count = 0
+            @episodes[user_id].each do |e|
+              next unless ids.include?(e[:id].to_s)
+              next if e[:archived_at]
+              e[:archived_at] = Time.now
+              count += 1
+            end
+            count
+          end
+
+          def expired_episode_ids(user_id, cutoff:)
+            active_episodes(user_id)
+              .select { |e| as_time(e[:created_at]) < cutoff }
+              .map { |e| e[:id].to_s }
+          end
+
           def list_users
             @episodes.keys
           end
 
           private
 
+          def active_episodes(user_id)
+            @episodes[user_id].reject { |e| e[:archived_at] }
+          end
+
+          def as_time(value)
+            return Time.now if value.nil?
+            return value if value.is_a?(Time)
+            Time.parse(value.to_s)
+          rescue ArgumentError
+            Time.now
+          end
+
           def symbolize(hash)
             hash.each_with_object({}) { |(k, v), acc| acc[k.to_sym] = v }
           end

data/lib/llmemory/long_term/file_based/memory.rb

@@ -108,11 +108,15 @@ module Llmemory
         # --- MemoryModule uniform interface ---
 
         def write(payload, **_meta)
-          memorize(payload)
+          result = nil
+          Llmemory::Instrumentation.instrument(:memory_write, memory_type: "file_based", user_id: @user_id) do
+            result = memorize(payload)
+          end
+          result
         end
 
-        def list(user_id: nil, limit: nil)
-          @storage.list_items(user_id: user_id || @user_id, limit: limit)
+        def list(user_id: nil, limit: nil, offset: nil)
+          @storage.list_items(user_id: user_id || @user_id, limit: limit, offset: offset)
         end
 
         def stats(user_id: nil)
@@ -120,7 +124,10 @@ module Llmemory
         end
 
         # Removes items/resources by id and records the removal in the audit log.
-        def forget(ids:, reason: nil)
+        # Note: file-based storages currently implement `archive_*` as physical
+        # removal — `mode: :soft` and `mode: :hard` are functionally equivalent
+        # here. Kept for API uniformity.
+        def forget(ids:, reason: nil, mode: :soft)
           requested = Array(ids).map(&:to_s)
           existing = (@storage.get_all_items(@user_id) + @storage.get_all_resources(@user_id))
             .map { |r| (r[:id] || r["id"]).to_s }
@@ -128,6 +135,7 @@ module Llmemory
           @storage.archive_items(@user_id, removed)
           @storage.archive_resources(@user_id, removed)
           forget_log.record(@user_id, memory_type: "file_based", ids: removed, reason: reason)
+          Llmemory::Instrumentation.instrument(:memory_forget, memory_type: "file_based", user_id: @user_id, count: removed.size, mode: mode)
           removed.size
         end
 

data/lib/llmemory/long_term/file_based/storages/active_record_storage.rb

@@ -64,13 +64,11 @@ module Llmemory
           end
 
           def search_items(user_id, query)
-            q = "%#{sanitize_like(query)}%"
-            LlmemoryItem.where(user_id: user_id).where("LOWER(content) LIKE LOWER(?)", q).map { |r| row_to_item(r) }
+            token_scope(LlmemoryItem.where(user_id: user_id), "content", query).map { |r| row_to_item(r) }
           end
 
           def search_resources(user_id, query)
-            q = "%#{sanitize_like(query)}%"
-            LlmemoryResource.where(user_id: user_id).where("LOWER(text) LIKE LOWER(?)", q).map { |r| row_to_resource(r) }
+            token_scope(LlmemoryResource.where(user_id: user_id), "text", query).map { |r| row_to_resource(r) }
           end
 
           def get_resources_since(user_id, hours:)
@@ -125,17 +123,19 @@ module Llmemory
              LlmemoryCategory.distinct.pluck(:user_id)).uniq
           end
 
-          def list_resources(user_id:, limit: nil)
+          def list_resources(user_id:, limit: nil, offset: nil)
             scope = LlmemoryResource.where(user_id: user_id).order(:created_at)
             scope = scope.limit(limit) if limit && limit.to_i.positive?
+            scope = scope.offset(offset) if offset && offset.to_i.positive?
             scope.map { |r| row_to_resource(r) }
           end
 
-          def list_items(user_id:, category: nil, limit: nil)
+          def list_items(user_id:, category: nil, limit: nil, offset: nil)
             scope = LlmemoryItem.where(user_id: user_id)
             scope = scope.where(category: category) if category
             scope = scope.order(:created_at)
             scope = scope.limit(limit) if limit && limit.to_i.positive?
+            scope = scope.offset(offset) if offset && offset.to_i.positive?
             scope.map { |r| row_to_item(r) }
           end
 
@@ -181,6 +181,15 @@ module Llmemory
             (str || "").to_s.gsub(/[%_\\]/) { |c| "\\#{c}" }
           end
 
+          # OR-of-token LIKE scope for keyword search; unchanged scope (match all)
+          # when the query has no tokens.
+          def token_scope(scope, column, query)
+            tokens = Llmemory::Tokenizer.tokenize(query)
+            return scope if tokens.empty?
+            clause = tokens.map { "LOWER(#{column}) LIKE LOWER(?)" }.join(" OR ")
+            scope.where(clause, *tokens.map { |t| "%#{sanitize_like(t)}%" })
+          end
+
           def row_to_item(r)
             h = {
               id: r.id,

data/lib/llmemory/long_term/file_based/storages/base.rb

@@ -69,11 +69,11 @@ module Llmemory
             raise NotImplementedError, "#{self.class}#list_users must be implemented"
           end
 
-          def list_resources(user_id:, limit: nil)
+          def list_resources(user_id:, limit: nil, offset: nil)
             raise NotImplementedError, "#{self.class}#list_resources must be implemented"
           end
 
-          def list_items(user_id:, category: nil, limit: nil)
+          def list_items(user_id:, category: nil, limit: nil, offset: nil)
             raise NotImplementedError, "#{self.class}#list_items must be implemented"
           end
 

data/lib/llmemory/long_term/file_based/storages/database_storage.rb

@@ -65,20 +65,20 @@ module Llmemory
 
           def search_items(user_id, query)
             ensure_tables!
-            pattern = "%#{conn.escape_string(query.to_s.downcase)}%"
+            suffix, params = token_filter("content", query, 2)
             rows = conn.exec_params(
-              "SELECT id, category, content, source_resource_id, importance, provenance, created_at FROM llmemory_items WHERE user_id = $1 AND LOWER(content) LIKE $2",
-              [user_id, pattern]
+              "SELECT id, category, content, source_resource_id, importance, provenance, created_at FROM llmemory_items WHERE user_id = $1#{suffix}",
+              [user_id, *params]
             )
             rows_to_items(rows)
           end
 
           def search_resources(user_id, query)
             ensure_tables!
-            pattern = "%#{conn.escape_string(query.to_s.downcase)}%"
+            suffix, params = token_filter("text", query, 2)
             rows = conn.exec_params(
-              "SELECT id, text, created_at FROM llmemory_resources WHERE user_id = $1 AND LOWER(text) LIKE $2",
-              [user_id, pattern]
+              "SELECT id, text, created_at FROM llmemory_resources WHERE user_id = $1#{suffix}",
+              [user_id, *params]
             )
             rows_to_resources(rows)
           end
@@ -169,15 +169,16 @@ module Llmemory
              conn.exec("SELECT DISTINCT user_id FROM llmemory_categories").map { |r| r["user_id"] }).uniq
           end
 
-          def list_resources(user_id:, limit: nil)
+          def list_resources(user_id:, limit: nil, offset: nil)
             ensure_tables!
             sql = "SELECT id, text, created_at FROM llmemory_resources WHERE user_id = $1 ORDER BY created_at"
             sql += " LIMIT #{limit.to_i}" if limit && limit.to_i.positive?
+            sql += " OFFSET #{offset.to_i}" if offset && offset.to_i.positive?
             rows = conn.exec_params(sql, [user_id])
             rows_to_resources(rows)
           end
 
-          def list_items(user_id:, category: nil, limit: nil)
+          def list_items(user_id:, category: nil, limit: nil, offset: nil)
             ensure_tables!
             sql = "SELECT id, category, content, source_resource_id, importance, provenance, created_at FROM llmemory_items WHERE user_id = $1"
             params = [user_id]
@@ -187,6 +188,7 @@ module Llmemory
             end
             sql += " ORDER BY created_at"
             sql += " LIMIT #{limit.to_i}" if limit && limit.to_i.positive?
+            sql += " OFFSET #{offset.to_i}" if offset && offset.to_i.positive?
             rows = params.size == 1 ? conn.exec_params(sql, params) : conn.exec_params(sql, params)
             rows_to_items(rows)
           end
@@ -290,6 +292,16 @@ module Llmemory
             end
           end
 
+          # Builds an OR-of-token LIKE filter for keyword search. Returns
+          # ["" , []] for an empty query (match all). Tokens are [a-z0-9]{2,} so
+          # they carry no LIKE wildcards.
+          def token_filter(column, query, start_index)
+            tokens = Llmemory::Tokenizer.tokenize(query)
+            return ["", []] if tokens.empty?
+            likes = tokens.each_index.map { |i| "LOWER(#{column}) LIKE $#{start_index + i}" }
+            [" AND (#{likes.join(' OR ')})", tokens.map { |t| "%#{t}%" }]
+          end
+
           def parse_provenance(value)
             return nil if value.nil? || value.to_s.strip.empty?
             JSON.parse(value, symbolize_names: true)

data/lib/llmemory/long_term/file_based/storages/file_storage.rb

@@ -62,13 +62,11 @@ module Llmemory
           end
 
           def search_items(user_id, query)
-            query_lower = query.downcase
-            get_all_items(user_id).select { |i| (i[:content] || i["content"]).to_s.downcase.include?(query_lower) }
+            get_all_items(user_id).select { |i| Llmemory::Tokenizer.matches?(i[:content] || i["content"], query) }
           end
 
           def search_resources(user_id, query)
-            query_lower = query.downcase
-            get_all_resources(user_id).select { |r| (r[:text] || r["text"]).to_s.downcase.include?(query_lower) }
+            get_all_resources(user_id).select { |r| Llmemory::Tokenizer.matches?(r[:text] || r["text"], query) }
           end
 
           def get_resources_since(user_id, hours:)
@@ -154,14 +152,16 @@ module Llmemory
             Dir.children(@base_path).select { |e| File.directory?(File.join(@base_path, e)) && !e.start_with?(".") }
           end
 
-          def list_resources(user_id:, limit: nil)
+          def list_resources(user_id:, limit: nil, offset: nil)
             list = get_all_resources(user_id)
+            list = list.drop(offset.to_i) if offset && offset.to_i.positive?
             limit ? list.take(limit) : list
           end
 
-          def list_items(user_id:, category: nil, limit: nil)
+          def list_items(user_id:, category: nil, limit: nil, offset: nil)
             list = get_all_items(user_id)
             list = list.select { |i| (i[:category] || i["category"]).to_s == category.to_s } if category
+            list = list.drop(offset.to_i) if offset && offset.to_i.positive?
             list = list.take(limit) if limit
             list
           end