llmemory 0.2.4 → 0.2.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 296b9d61d6c474145ecaa607653b37438b2491c846aac602f65d5fd850dae9ef
-  data.tar.gz: 521fd05b577c6c17a7dbc5d3771ff9fb3f7cddeaeef31938efabdcfd40db74a5
+  metadata.gz: eb36583aecef9a4a0d4a89e703d1edf7326112df260ceccd1b181cf1411a45d1
+  data.tar.gz: b91c1b67b6c88c12d0515a03304aa04b0f61bc310acbb6e1b02391fbe9944d08
 SHA512:
-  metadata.gz: edf6ee6b41cb366f2ddef5ac2deec9c3c03090d4920d2386f9578e31999d135b34ad7f995ee9a9e803cf43627f969a3fab9ee191c68e2362a4d62fa8bade0729
-  data.tar.gz: 83266918faeb2bb4f7b57e89007b642283d7d64600bf83513948d5378a625fa911aefa6823c54115961bafe34dfdd147f4fb297a8226caad49aaceb6a01372e0
+  metadata.gz: 597d6cd707b07ebc3c754e2ac67269143c6bff221a22529e692b8a4cc4fc4415180b83f97d7336f95e6eef025c75a18f64819daa740588f032052fccca1a3632
+  data.tar.gz: 7b9ec61f4edb2cc053e86f1f478e7a23c82784732fbc918ca6bf35756583e568c92039c7a1cdf2939c6066df76e85ee99176f03b44c82ae708f2705093cb67e8

data/README.md

@@ -14,6 +14,8 @@ gem "llmemory"
 
 Then run `bundle install`.
 
+**Upgrading?** See the [Migration guide](docs/MIGRATION.md) for database columns, encryption backfill, and graph store configuration changes.
+
 ## Quick Start (Unified API)
 
 The recommended way to use llmemory in a chat is the unified `Llmemory::Memory` API. It abstracts short-term (conversation history) and long-term (extracted facts) and combines retrieval from both:
@@ -171,7 +173,7 @@ memory = Llmemory::Memory.new(user_id: "agent-1", encryption_key: "tenant-specif
 **What is encrypted:** conversation checkpoints (redis/postgres/active_record), file-based facts/resources/categories, episodic/procedural documents, graph node names/types/predicates (deterministic) and properties (random IV). **Vector embeddings are not encrypted** (required for pgvector search); associated `text_content` metadata is encrypted.
 
 **Trade-offs:**
-- Database keyword search (`LIKE`, BM25 on encrypted columns) no longer works on ciphertext; file backends still search in memory after decrypt.
+- Database keyword search on encrypted columns requires the `search_tokens` blind-index column (see [Migration guide](docs/MIGRATION.md)); file backends search in memory after decrypt without schema changes.
 - `:memory` backends are in-process only and are **not** encrypted at rest.
 - Existing plaintext data remains readable (markers `enc:v1:` / `encd:v1:`); new writes are encrypted when enabled.
 - Deterministic encryption on graph identifiers leaks equality (same name ⇒ same ciphertext) but keeps graph traversal working.

data/lib/generators/llmemory/install/templates/create_llmemory_tables.rb

@@ -6,6 +6,7 @@ class CreateLlmemoryTables < ActiveRecord::Migration[7.0]
       t.string :id, null: false, primary_key: true
       t.string :user_id, null: false
       t.text :text, null: false
+      t.text :search_tokens
       t.timestamps
     end
     add_index :llmemory_resources, :user_id
@@ -18,6 +19,7 @@ class CreateLlmemoryTables < ActiveRecord::Migration[7.0]
       t.string :source_resource_id
       t.float :importance, default: 0.7
       t.jsonb :provenance
+      t.text :search_tokens
       t.timestamps
     end
     add_index :llmemory_items, :user_id
@@ -36,6 +38,7 @@ class CreateLlmemoryTables < ActiveRecord::Migration[7.0]
       t.string :user_id, null: false
       t.jsonb :data, null: false, default: {}
       t.text :search_text
+      t.text :search_tokens
       t.datetime :archived_at
       t.timestamps
     end
@@ -47,6 +50,8 @@ class CreateLlmemoryTables < ActiveRecord::Migration[7.0]
       t.string :user_id, null: false
       t.jsonb :data, null: false, default: {}
       t.text :search_text
+      t.text :search_tokens
+      t.text :name_det
       t.datetime :archived_at
       t.timestamps
     end

data/lib/llmemory/crypto/cipher.rb

@@ -36,6 +36,10 @@ module Llmemory
       def encrypted?(str)
         false
       end
+
+      def blind_index(token)
+        token.to_s.downcase
+      end
     end
 
     # AES-256-GCM encryption with separate content (random IV) and index
@@ -111,6 +115,14 @@ module Llmemory
         s.start_with?(MARKER) || s.start_with?(DETERMINISTIC_MARKER)
       end
 
+      # Deterministic HMAC digest for blind-index keyword search over encrypted fields.
+      def blind_index(token)
+        str = token.to_s.downcase
+        return str if str.empty?
+
+        OpenSSL::HMAC.hexdigest("SHA256", @index_key, str)[0, 32]
+      end
+
       private
 
       def encrypt_with_key(plaintext, key, iv:, marker:)

data/lib/llmemory/crypto/field_helpers.rb

@@ -105,6 +105,31 @@ module Llmemory
       rescue JSON::ParserError
         nil
       end
+
+      # Space-delimited blind-index digests for encrypted keyword search.
+      def search_tokens_for(text)
+        tokens = Llmemory::Tokenizer.tokenize(text).uniq
+        return nil if tokens.empty?
+
+        digests = tokens.map { |t| cipher.blind_index(t) }
+        " #{digests.join(' ')} "
+      end
+
+      # OR-of-token SQL filter. Uses blind index on `search_tokens_column` when
+      # encryption is enabled; otherwise LIKE on the plaintext `column`.
+      def blind_token_filter(column, query, start_index, search_tokens_column: nil)
+        tokens = Llmemory::Tokenizer.tokenize(query)
+        return ["", []] if tokens.empty?
+
+        if cipher.enabled? && search_tokens_column
+          digests = tokens.map { |t| cipher.blind_index(t) }
+          likes = digests.each_index.map { |i| "#{search_tokens_column} LIKE $#{start_index + i}" }
+          [" AND (#{likes.join(' OR ')})", digests.map { |d| "% #{d} %" }]
+        else
+          likes = tokens.each_index.map { |i| "LOWER(#{column}) LIKE $#{start_index + i}" }
+          [" AND (#{likes.join(' OR ')})", tokens.map { |t| "%#{t}%" }]
+        end
+      end
     end
   end
 end

data/lib/llmemory/long_term/episodic/storages/active_record_storage.rb

@@ -35,7 +35,9 @@ module Llmemory
             rec = LlmemoryEpisode.find_or_initialize_by(id: id)
             rec.user_id = user_id
             rec.data = cipher.enabled? ? enc_json(data) : data
-            rec.search_text = enc(searchable_text(data))
+            text = searchable_text(data)
+            rec.search_text = enc(text)
+            rec.search_tokens = search_tokens_for(text) if LlmemoryEpisode.column_names.include?("search_tokens")
             rec.created_at ||= Time.current
             rec.save!
             id
@@ -56,8 +58,10 @@ module Llmemory
           end
 
           def search_episodes(user_id, query)
-            token_scope(LlmemoryEpisode.where(user_id: user_id, archived_at: nil), "search_text", query)
-              .order(created_at: :desc).map { |r| decode_data(r.data) }
+            scope = LlmemoryEpisode.where(user_id: user_id, archived_at: nil)
+            token_scope(scope, "search_text", query, model: LlmemoryEpisode)
+              .sort_by { |r| -r.created_at.to_i }
+              .map { |r| decode_data(r.data) }
           end
 
           def count_episodes(user_id)
@@ -83,11 +87,25 @@ module Llmemory
 
           private
 
-          def token_scope(scope, column, query)
+          def token_scope(scope, column, query, model:)
             tokens = Llmemory::Tokenizer.tokenize(query)
-            return scope if tokens.empty?
-            clause = tokens.map { "LOWER(#{column}) LIKE LOWER(?)" }.join(" OR ")
-            scope.where(clause, *tokens.map { |t| "%#{t}%" })
+            return scope.to_a if tokens.empty?
+
+            if cipher.enabled? && model.column_names.include?("search_tokens")
+              digests = tokens.map { |t| cipher.blind_index(t) }
+              clause = digests.map { "search_tokens LIKE ?" }.join(" OR ")
+              indexed = scope.where(clause, *digests.map { |d| "% #{d} %" })
+              legacy_scope = scope.where(search_tokens: nil)
+              indexed_records = indexed.to_a
+              legacy_records = legacy_scope.to_a.select do |record|
+                plaintext = dec(record.public_send(column))
+                Llmemory::Tokenizer.matches?(plaintext, query)
+              end
+              (indexed_records + legacy_records).uniq { |r| r.id }
+            else
+              clause = tokens.map { "LOWER(#{column}) LIKE LOWER(?)" }.join(" OR ")
+              scope.where(clause, *tokens.map { |t| "%#{t}%" }).to_a
+            end
           end
 
           def stringify(hash)

data/lib/llmemory/long_term/episodic/storages/database_storage.rb

@@ -29,10 +29,10 @@ module Llmemory
             data[:created_at] ||= Time.now.utc.iso8601
             search = searchable_text(data)
             conn.exec_params(
-              "INSERT INTO llmemory_episodes (id, user_id, data, search_text, created_at) " \
-              "VALUES ($1, $2, $3::jsonb, $4, $5) " \
-              "ON CONFLICT (id) DO UPDATE SET data = $3::jsonb, search_text = $4",
-              [id, user_id, store_data(data), enc(search), created_at_value(data)]
+              "INSERT INTO llmemory_episodes (id, user_id, data, search_text, search_tokens, created_at) " \
+              "VALUES ($1, $2, $3::jsonb, $4, $5, $6) " \
+              "ON CONFLICT (id) DO UPDATE SET data = $3::jsonb, search_text = $4, search_tokens = $5",
+              [id, user_id, store_data(data), enc(search), search_tokens_for(search), created_at_value(data)]
             )
             id
           end
@@ -53,11 +53,30 @@ module Llmemory
 
           def search_episodes(user_id, query)
             ensure_tables!
-            suffix, params = token_filter("search_text", query, 2)
-            conn.exec_params(
+            tokens = Llmemory::Tokenizer.tokenize(query)
+            return conn.exec_params(
+              "SELECT data FROM llmemory_episodes WHERE user_id = $1 AND archived_at IS NULL ORDER BY created_at DESC",
+              [user_id]
+            ).map { |r| parse_data(r["data"]) } if tokens.empty?
+
+            suffix, params = blind_token_filter("search_text", query, 2, search_tokens_column: cipher.enabled? ? "search_tokens" : nil)
+            rows = conn.exec_params(
               "SELECT data FROM llmemory_episodes WHERE user_id = $1 AND archived_at IS NULL#{suffix} ORDER BY created_at DESC",
               [user_id, *params]
-            ).map { |r| parse_data(r["data"]) }
+            )
+            episodes = rows.map { |r| parse_data(r["data"]) }
+            return episodes unless cipher.enabled?
+
+            legacy_rows = conn.exec_params(
+              "SELECT data FROM llmemory_episodes WHERE user_id = $1 AND archived_at IS NULL AND search_tokens IS NULL",
+              [user_id]
+            )
+            legacy = legacy_rows.map { |r| parse_data(r["data"]) }.select do |ep|
+              Llmemory::Tokenizer.matches?(searchable_text(ep), query)
+            end
+            by_id = {}
+            (episodes + legacy).each { |ep| by_id[ep[:id] || ep["id"]] = ep }
+            by_id.values
           end
 
           def count_episodes(user_id)
@@ -116,16 +135,14 @@ module Llmemory
               );
               CREATE INDEX IF NOT EXISTS idx_llmemory_episodes_user_id ON llmemory_episodes(user_id);
               ALTER TABLE llmemory_episodes ADD COLUMN IF NOT EXISTS archived_at TIMESTAMPTZ;
+              ALTER TABLE llmemory_episodes ADD COLUMN IF NOT EXISTS search_tokens TEXT;
             SQL
           end
 
           # OR-of-token LIKE filter (see file-based DatabaseStorage). [""] for an
           # empty query => match all.
           def token_filter(column, query, start_index)
-            tokens = Llmemory::Tokenizer.tokenize(query)
-            return ["", []] if tokens.empty?
-            likes = tokens.each_index.map { |i| "LOWER(#{column}) LIKE $#{start_index + i}" }
-            [" AND (#{likes.join(' OR ')})", tokens.map { |t| "%#{t}%" }]
+            blind_token_filter(column, query, start_index, search_tokens_column: cipher.enabled? ? "search_tokens" : nil)
           end
 
           def parse_data(value)

data/lib/llmemory/long_term/file_based/storages/active_record_storage.rb

@@ -25,12 +25,14 @@ module Llmemory
 
           def save_resource(user_id, text)
             id = "res_#{SecureRandom.hex(8)}"
-            LlmemoryResource.create!(
+            attrs = {
               id: id,
               user_id: user_id,
               text: enc(text),
               created_at: Time.current
-            )
+            }
+            attrs[:search_tokens] = search_tokens_for(text) if LlmemoryResource.column_names.include?("search_tokens")
+            LlmemoryResource.create!(attrs)
             id
           end
 
@@ -48,6 +50,7 @@ module Llmemory
             if provenance && LlmemoryItem.column_names.include?("provenance")
               attrs[:provenance] = cipher.enabled? ? enc_json(provenance) : provenance
             end
+            attrs[:search_tokens] = search_tokens_for(content) if LlmemoryItem.column_names.include?("search_tokens")
             LlmemoryItem.create!(attrs)
             id
           end
@@ -70,11 +73,13 @@ module Llmemory
           end
 
           def search_items(user_id, query)
-            token_scope(LlmemoryItem.where(user_id: user_id), "content", query).map { |r| row_to_item(r) }
+            scope = LlmemoryItem.where(user_id: user_id)
+            token_scope(scope, "content", query, model: LlmemoryItem).map { |r| row_to_item(r) }
           end
 
           def search_resources(user_id, query)
-            token_scope(LlmemoryResource.where(user_id: user_id), "text", query).map { |r| row_to_resource(r) }
+            scope = LlmemoryResource.where(user_id: user_id)
+            token_scope(scope, "text", query, model: LlmemoryResource).map { |r| row_to_resource(r) }
           end
 
           def get_resources_since(user_id, hours:)
@@ -112,6 +117,7 @@ module Llmemory
               created_at: created_at
             }
             attrs[:importance] = merged_item[:importance] if LlmemoryItem.column_names.include?("importance") && merged_item[:importance]
+            attrs[:search_tokens] = search_tokens_for(merged_item[:content]) if LlmemoryItem.column_names.include?("search_tokens")
             LlmemoryItem.create!(attrs)
           end
 
@@ -150,52 +156,40 @@ module Llmemory
           end
 
           def get_items_around(user_id, reference, before: 5, after: 5)
-            items = get_all_items(user_id)
-            find_around(items, reference, before, after)
+            find_around(get_all_items(user_id), reference, before, after)
           end
 
           def get_resources_around(user_id, reference, before: 5, after: 5)
-            resources = get_all_resources(user_id)
-            find_around(resources, reference, before, after)
+            find_around(get_all_resources(user_id), reference, before, after)
           end
 
           private
 
-          def find_around(items, reference, before, after)
-            return { before: [], target: nil, after: [] } if items.empty?
+          def token_scope(scope, column, query, model:)
+            tokens = Llmemory::Tokenizer.tokenize(query)
+            return scope if tokens.empty?
 
-            idx = if reference.is_a?(String) && reference.match?(/^\d{4}-/)
-              target_time = Time.parse(reference)
-              items.index { |i| i[:created_at] >= target_time } || items.size
+            if cipher.enabled? && model.column_names.include?("search_tokens")
+              digests = tokens.map { |t| cipher.blind_index(t) }
+              clause = digests.map { "search_tokens LIKE ?" }.join(" OR ")
+              indexed = scope.where(clause, *digests.map { |d| "% #{d} %" })
+              legacy_scope = scope.where(search_tokens: nil)
+              indexed_records = indexed.to_a
+              legacy_records = legacy_scope.to_a.select do |record|
+                plaintext = dec(record.public_send(column))
+                Llmemory::Tokenizer.matches?(plaintext, query)
+              end
+              (indexed_records + legacy_records).uniq { |r| r.id }
             else
-              items.index { |i| i[:id] == reference }
+              clause = tokens.map { "LOWER(#{column}) LIKE LOWER(?)" }.join(" OR ")
+              scope.where(clause, *tokens.map { |t| "%#{sanitize_like(t)}%" }).to_a
             end
-
-            return { before: [], target: nil, after: [] } unless idx
-
-            start_idx = [idx - before, 0].max
-            end_idx = [idx + after, items.size - 1].min
-
-            {
-              before: items[start_idx...idx] || [],
-              target: items[idx],
-              after: items[(idx + 1)..end_idx] || []
-            }
           end
 
           def sanitize_like(str)
             (str || "").to_s.gsub(/[%_\\]/) { |c| "\\#{c}" }
           end
 
-          # OR-of-token LIKE scope for keyword search; unchanged scope (match all)
-          # when the query has no tokens.
-          def token_scope(scope, column, query)
-            tokens = Llmemory::Tokenizer.tokenize(query)
-            return scope if tokens.empty?
-            clause = tokens.map { "LOWER(#{column}) LIKE LOWER(?)" }.join(" OR ")
-            scope.where(clause, *tokens.map { |t| "%#{sanitize_like(t)}%" })
-          end
-
           def row_to_item(r)
             h = {
               id: r.id,

data/lib/llmemory/long_term/file_based/storages/base.rb

@@ -88,6 +88,30 @@ module Llmemory
           def get_resources_around(user_id, reference, before: 5, after: 5)
             raise NotImplementedError, "#{self.class}#get_resources_around must be implemented"
           end
+
+          protected
+
+          def find_around(items, reference, before, after, id_key: :id)
+            return { before: [], target: nil, after: [] } if items.empty?
+
+            idx = if reference.is_a?(String) && reference.match?(/^\d{4}-/)
+              target_time = Time.parse(reference)
+              items.index { |i| (i[:created_at] || i["created_at"]) >= target_time } || items.size
+            else
+              items.index { |i| (i[id_key] || i[id_key.to_s]) == reference }
+            end
+
+            return { before: [], target: nil, after: [] } unless idx
+
+            start_idx = [idx - before, 0].max
+            end_idx = [idx + after, items.size - 1].min
+
+            {
+              before: items[start_idx...idx] || [],
+              target: items[idx],
+              after: items[(idx + 1)..end_idx] || []
+            }
+          end
         end
       end
     end

data/lib/llmemory/long_term/file_based/storages/database_storage.rb

@@ -22,8 +22,8 @@ module Llmemory
             ensure_tables!
             id = "res_#{SecureRandom.hex(8)}"
             conn.exec_params(
-              "INSERT INTO llmemory_resources (id, user_id, text, created_at) VALUES ($1, $2, $3, $4)",
-              [id, user_id, enc(text), Time.now.utc.iso8601]
+              "INSERT INTO llmemory_resources (id, user_id, text, search_tokens, created_at) VALUES ($1, $2, $3, $4, $5)",
+              [id, user_id, enc(text), search_tokens_for(text), Time.now.utc.iso8601]
             )
             id
           end
@@ -32,8 +32,8 @@ module Llmemory
             ensure_tables!
             id = "item_#{SecureRandom.hex(8)}"
             conn.exec_params(
-              "INSERT INTO llmemory_items (id, user_id, category, content, source_resource_id, importance, provenance, created_at) VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb, $8)",
-              [id, user_id, category, enc(content), source_resource_id, importance.to_f, provenance_json(provenance), Time.now.utc.iso8601]
+              "INSERT INTO llmemory_items (id, user_id, category, content, source_resource_id, importance, provenance, search_tokens, created_at) VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb, $8, $9)",
+              [id, user_id, category, enc(content), source_resource_id, importance.to_f, provenance_json(provenance), search_tokens_for(content), Time.now.utc.iso8601]
             )
             id
           end
@@ -69,22 +69,34 @@ module Llmemory
 
           def search_items(user_id, query)
             ensure_tables!
-            suffix, params = token_filter("content", query, 2)
+            tokens = Llmemory::Tokenizer.tokenize(query)
+            return get_all_items(user_id) if tokens.empty?
+
+            suffix, params = blind_token_filter("content", query, 2, search_tokens_column: cipher.enabled? ? "search_tokens" : nil)
             rows = conn.exec_params(
               "SELECT id, category, content, source_resource_id, importance, provenance, created_at FROM llmemory_items WHERE user_id = $1#{suffix}",
               [user_id, *params]
             )
-            rows_to_items(rows)
+            items = rows_to_items(rows)
+            return items unless cipher.enabled?
+
+            merge_legacy_search(items, legacy_item_rows(user_id), query, text_key: :content)
           end
 
           def search_resources(user_id, query)
             ensure_tables!
-            suffix, params = token_filter("text", query, 2)
+            tokens = Llmemory::Tokenizer.tokenize(query)
+            return get_all_resources(user_id) if tokens.empty?
+
+            suffix, params = blind_token_filter("text", query, 2, search_tokens_column: cipher.enabled? ? "search_tokens" : nil)
             rows = conn.exec_params(
               "SELECT id, text, created_at FROM llmemory_resources WHERE user_id = $1#{suffix}",
               [user_id, *params]
             )
-            rows_to_resources(rows)
+            resources = rows_to_resources(rows)
+            return resources unless cipher.enabled?
+
+            merge_legacy_search(resources, legacy_resource_rows(user_id), query, text_key: :text)
           end
 
           def get_resources_since(user_id, hours:)
@@ -144,13 +156,14 @@ module Llmemory
             created_at = created_at.utc.iso8601 if created_at.respond_to?(:utc)
             id = "item_#{SecureRandom.hex(8)}"
             conn.exec_params(
-              "INSERT INTO llmemory_items (id, user_id, category, content, source_resource_id, created_at) VALUES ($1, $2, $3, $4, $5, $6)",
+              "INSERT INTO llmemory_items (id, user_id, category, content, source_resource_id, search_tokens, created_at) VALUES ($1, $2, $3, $4, $5, $6, $7)",
               [
                 id,
                 user_id,
                 merged_item[:category],
                 enc(merged_item[:content]),
                 merged_item[:source_resource_id],
+                search_tokens_for(merged_item[:content]),
                 created_at
               ]
             )
@@ -205,38 +218,42 @@ module Llmemory
 
           def get_items_around(user_id, reference, before: 5, after: 5)
             ensure_tables!
-            items = get_all_items(user_id)
-            find_around(items, reference, before, after)
+            find_around(get_all_items(user_id), reference, before, after)
           end
 
           def get_resources_around(user_id, reference, before: 5, after: 5)
             ensure_tables!
-            resources = get_all_resources(user_id)
-            find_around(resources, reference, before, after)
+            find_around(get_all_resources(user_id), reference, before, after)
           end
 
           private
 
-          def find_around(items, reference, before, after)
-            return { before: [], target: nil, after: [] } if items.empty?
-
-            idx = if reference.is_a?(String) && reference.match?(/^\d{4}-/)
-              target_time = Time.parse(reference)
-              items.index { |i| i[:created_at] >= target_time } || items.size
-            else
-              items.index { |i| i[:id] == reference }
-            end
+          def legacy_item_rows(user_id)
+            conn.exec_params(
+              "SELECT id, category, content, source_resource_id, importance, provenance, created_at FROM llmemory_items WHERE user_id = $1 AND search_tokens IS NULL",
+              [user_id]
+            )
+          end
 
-            return { before: [], target: nil, after: [] } unless idx
+          def legacy_resource_rows(user_id)
+            conn.exec_params(
+              "SELECT id, text, created_at FROM llmemory_resources WHERE user_id = $1 AND search_tokens IS NULL",
+              [user_id]
+            )
+          end
 
-            start_idx = [idx - before, 0].max
-            end_idx = [idx + after, items.size - 1].min
+          def merge_legacy_search(indexed, legacy_rows, query, text_key:)
+            return indexed if legacy_rows.none?
 
-            {
-              before: items[start_idx...idx] || [],
-              target: items[idx],
-              after: items[(idx + 1)..end_idx] || []
-            }
+            legacy = if text_key == :content
+              rows_to_items(legacy_rows)
+            else
+              rows_to_resources(legacy_rows)
+            end
+            legacy_matches = legacy.select { |row| Llmemory::Tokenizer.matches?(row[text_key], query) }
+            by_id = {}
+            (indexed + legacy_matches).each { |row| by_id[row[:id]] = row }
+            by_id.values
           end
 
           def conn
@@ -271,6 +288,8 @@ module Llmemory
             SQL
             conn.exec("ALTER TABLE llmemory_items ADD COLUMN IF NOT EXISTS importance REAL DEFAULT 0.7") rescue nil
             conn.exec("ALTER TABLE llmemory_items ADD COLUMN IF NOT EXISTS provenance JSONB") rescue nil
+            conn.exec("ALTER TABLE llmemory_items ADD COLUMN IF NOT EXISTS search_tokens TEXT") rescue nil
+            conn.exec("ALTER TABLE llmemory_resources ADD COLUMN IF NOT EXISTS search_tokens TEXT") rescue nil
             conn.exec(<<~SQL)
               CREATE TABLE IF NOT EXISTS llmemory_categories (
                 user_id TEXT NOT NULL,
@@ -300,10 +319,7 @@ module Llmemory
           # ["" , []] for an empty query (match all). Tokens are [a-z0-9]{2,} so
           # they carry no LIKE wildcards.
           def token_filter(column, query, start_index)
-            tokens = Llmemory::Tokenizer.tokenize(query)
-            return ["", []] if tokens.empty?
-            likes = tokens.each_index.map { |i| "LOWER(#{column}) LIKE $#{start_index + i}" }
-            [" AND (#{likes.join(' OR ')})", tokens.map { |t| "%#{t}%" }]
+            blind_token_filter(column, query, start_index)
           end
 
           def parse_provenance(value)

data/lib/llmemory/long_term/file_based/storages/file_storage.rb

@@ -91,7 +91,7 @@ module Llmemory
               data = decrypt_item(data)
               data[:created_at] = parse_time(data[:created_at])
               data
-            end.sort_by { |i| i[:created_at] }
+            end.sort_by { |i| [i[:created_at], i[:id].to_s] }
           end
 
           def get_all_resources(user_id)
@@ -104,7 +104,7 @@ module Llmemory
               data[:text] = dec(data[:text] || data["text"])
               data[:created_at] = parse_time(data[:created_at])
               data
-            end.sort_by { |r| r[:created_at] }
+            end.sort_by { |r| [r[:created_at], r[:id].to_s] }
           end
 
           def get_items_since(user_id, hours:)
@@ -177,6 +177,14 @@ module Llmemory
             get_all_items(user_id).size
           end
 
+          def get_items_around(user_id, reference, before: 5, after: 5)
+            find_around(get_all_items(user_id), reference, before, after)
+          end
+
+          def get_resources_around(user_id, reference, before: 5, after: 5)
+            find_around(get_all_resources(user_id), reference, before, after)
+          end
+
           private
 
           def user_path(user_id, *parts)

data/lib/llmemory/long_term/file_based/storages/memory_storage.rb

@@ -121,38 +121,12 @@ module Llmemory
 
           def get_items_around(user_id, reference, before: 5, after: 5)
             items = @items[user_id].sort_by { |i| i[:created_at] }
-            find_around(items, reference, :id, before, after)
+            find_around(items, reference, before, after)
           end
 
           def get_resources_around(user_id, reference, before: 5, after: 5)
             resources = @resources[user_id].sort_by { |r| r[:created_at] }
-            find_around(resources, reference, :id, before, after)
-          end
-
-          private
-
-          def find_around(items, reference, id_key, before, after)
-            return { before: [], target: nil, after: [] } if items.empty?
-
-            idx = if reference.is_a?(String) && reference.match?(/^\d{4}-/)
-              # ISO timestamp - find closest item at or after this time
-              target_time = Time.parse(reference)
-              items.index { |i| i[:created_at] >= target_time } || items.size
-            else
-              # Item ID
-              items.index { |i| i[id_key] == reference }
-            end
-
-            return { before: [], target: nil, after: [] } unless idx
-
-            start_idx = [idx - before, 0].max
-            end_idx = [idx + after, items.size - 1].min
-
-            {
-              before: items[start_idx...idx] || [],
-              target: items[idx],
-              after: items[(idx + 1)..end_idx] || []
-            }
+            find_around(resources, reference, before, after)
           end
         end
       end

data/lib/llmemory/long_term/graph_based/storage.rb

@@ -16,7 +16,9 @@ module Llmemory
             require_relative "storages/active_record_storage"
             ActiveRecordStorage.new(cipher: resolved_cipher)
           else
-            MemoryStorage.new
+            store_name = (store || Llmemory.configuration.long_term_store).to_s
+            raise Llmemory::ConfigurationError,
+                  "graph_based long-term memory supports long_term_store :memory or :active_record; got #{store_name.inspect}"
           end
         end
       end