documentrix 0.3.0 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 68c409476101e4632597c139494f3fc1fe67bc0af7a655e5e5b3e4ebbd58f58c
-  data.tar.gz: fcd07ca7694b3fbed81c3a25f3fa9d9a675d120e5b82b43a13f5f71970012f3a
+  metadata.gz: 332c67275b90bcc797cdd8e8df75a751bcfa7a52c759c9bcbe4a55970e905401
+  data.tar.gz: c546f89c61613b11d18c2cd724cadae8c303a988f925ad1624c5121f511e88fd
 SHA512:
-  metadata.gz: cf1e95f2d994bb130b89bff9d2e43062621b849b5d0f7fb4543092b1f491bab4b9dd9e51047c3dfb216090a4f9bc853c08ba35719dea7409c34a15758070fcba
-  data.tar.gz: 59aa3347c07c2521f5661326a55d73733470ef30c62e86fc7d1ff53a4acb6cbc2da06e4c7814896b59ca16bdfab8933a7018f3f41ab30ce29593c5fac480f342
+  metadata.gz: c80a4c253b4fe367fae1dbb7e6aea50e274a495a0026a7b676e1e962b80c33806fe89976338c12931f11a5059e384fdfddcfe4665f68bde406b118edd8c04d15
+  data.tar.gz: '09292fd5d6aed6c939c0244467c743d515e68e5e253d7f443ef96a9c295a61689f1be5b9aec53e402287b6b6c7d305757eb26f79f5ade0161f7e82d534a7b3bf'

data/CHANGES.md

@@ -1,5 +1,39 @@
 # Changes
 
+## 2026-05-20 v0.3.2
+
+### Performance Improvements
+
+- Optimized collections lookup by memoizing
+  `Documentrix::Documents#collections` using `@collections_cache`.
+- Added `Documentrix::Documents#invalidate_collections_cache!` to reset the
+  memoized list.
+- Integrated `invalidate_collections_cache!` into `add`, `delete`, `clear`,
+  `source_remove`, and `rename_collection` to maintain cache consistency.
+
+### Database & Cache
+
+- Standardized SQL keyword casing to uppercase for data types (e.g., `FLOAT`,
+  `TEXT`, `INTEGER`, `JSON`) within `SQLiteCache`.
+
+### Documentation & Testing
+
+- Refined documentation and return type descriptions for
+  `Documentrix::Documents#find` and `Documentrix::Documents#prefix`.
+- Updated `spec/documents_spec.rb` to ensure `invalidate_collections_cache!` is
+  correctly triggered during mutations.
+
+## 2026-05-18 v0.3.1
+
+- Fixed scoping bugs in `clear_by_source` and `source_exist?` by implementing a
+  `key LIKE ?` constraint.
+- Added regression tests for prefix isolation in
+  `spec/documentrix/documents/cache/sqlite_cache_spec.rb`.
+- Introduced the `start_with_prefix` method in
+  `lib/documentrix/documents/cache/sqlite_cache.rb` to unify prefix patterns.
+- Updated `tags`, `size`, `clear_all_with_prefix`, and `each` to utilize the
+  new `start_with_prefix` method.
+
 ## 2026-05-17 v0.3.0
 
 ### New Features

data/documentrix.gemspec

@@ -1,9 +1,9 @@
 # -*- encoding: utf-8 -*-
-# stub: documentrix 0.3.0 ruby lib
+# stub: documentrix 0.3.2 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "documentrix".freeze
-  s.version = "0.3.0".freeze
+  s.version = "0.3.2".freeze
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]

data/lib/documentrix/documents/cache/sqlite_cache.rb

@@ -114,7 +114,7 @@ class Documentrix::Documents::Cache::SQLiteCache
     result = Documentrix::Utils::Tags.new
     execute(%{
         SELECT DISTINCT(tags) FROM records WHERE key LIKE ?
-      }, [ "#@prefix%" ]
+      }, [ start_with_prefix ]
     ).flatten.each do
       JSON(_1).each { |t| result.add(t) }
     end
@@ -126,7 +126,10 @@ class Documentrix::Documents::Cache::SQLiteCache
   #
   # @return [ Integer ] the count of records
   def size
-    execute(%{SELECT COUNT(*) FROM records WHERE key LIKE ?}, [ "#@prefix%" ]).flatten.first
+    execute(
+      %{SELECT COUNT(*) FROM records WHERE key LIKE ?},
+      [ start_with_prefix ]
+    ).flatten.first
   end
 
   # The clear_for_tags method clears the cache for specific tags by deleting
@@ -154,7 +157,7 @@ class Documentrix::Documents::Cache::SQLiteCache
   #
   # @return [ Documentrix::Documents::RedisBackedMemoryCache ] self
   def clear_all_with_prefix
-    execute(%{DELETE FROM records WHERE key LIKE ?}, [ "#@prefix%" ])
+    execute(%{DELETE FROM records WHERE key LIKE ?}, [ start_with_prefix ])
     self
   end
 
@@ -172,9 +175,21 @@ class Documentrix::Documents::Cache::SQLiteCache
   def clear_by_source(source, digest: nil, operator: ?=)
     operator = '!=' if operator != ?=
     if digest
-      execute(%{DELETE FROM records WHERE source = ? AND digest #{operator} ? }, [ source, digest ])
+      execute(
+        %{
+          DELETE FROM records
+          WHERE key LIKE ? AND source = ? AND digest #{operator} ?
+        },
+        [ start_with_prefix, source, digest ]
+      )
     else
-      execute(%{DELETE FROM records WHERE source = ?}, [ source ])
+      execute(
+        %{
+          DELETE FROM records
+          WHERE key LIKE ? AND source = ?
+        },
+        [ start_with_prefix, source ]
+      )
     end
     self
   end
@@ -194,9 +209,19 @@ class Documentrix::Documents::Cache::SQLiteCache
   def source_exist?(source, digest: nil, operator: ?=)
     operator = '!=' if operator != ?=
     if digest
-      !!execute(%{SELECT 1 FROM records WHERE source = ? AND digest #{operator} ? }, [ source, digest ]).first
+      !!execute(
+        %{
+          SELECT 1 FROM records WHERE key LIKE ? AND source = ? AND digest #{operator} ?
+        },
+        [ start_with_prefix, source, digest ]
+      ).first
     else
-      !!execute(%{SELECT 1 FROM records WHERE source = ?}, [ source ]).first
+      !!execute(
+        %{
+          SELECT 1 FROM records WHERE key LIKE ? AND source = ?
+        },
+        [ start_with_prefix, source ]
+      ).first
     end
   end
 
@@ -211,10 +236,9 @@ class Documentrix::Documents::Cache::SQLiteCache
     block or return enum_for(__method__)
 
     execute(%{
-      SELECT DISTINCT source
-      FROM records
+      SELECT DISTINCT source FROM records
       WHERE key LIKE ? AND source IS NOT NULL
-    }, [ "#@prefix%" ]).each do |source,|
+    }, [ start_with_prefix ]).each do |source,|
       source = source.full? or next
 
       block.(source)
@@ -257,7 +281,7 @@ class Documentrix::Documents::Cache::SQLiteCache
   #   cache.each do |key, value|
   #     puts "#{key}: #{value}"
   #   end
-  def each(prefix: "#@prefix%", &block)
+  def each(prefix: start_with_prefix, &block)
     block or return enum_for(__method__, prefix:)
 
     execute(%{
@@ -315,7 +339,7 @@ class Documentrix::Documents::Cache::SQLiteCache
       SELECT key, tags, embedding_id
       FROM records
       WHERE key LIKE ?#{tags_where}
-    }, [ "#@prefix%" ])
+    }, [ start_with_prefix ])
     if tags_filter
       records = records.select { |key, tags, embedding_id|
         (tags_filter & JSON(tags.to_s).to_a).size >= 1
@@ -368,6 +392,13 @@ class Documentrix::Documents::Cache::SQLiteCache
 
   private
 
+  # Returns the SQL LIKE pattern for records starting with the current prefix.
+  #
+  # @return [ String ] the prefix pattern used in SQL WHERE clauses
+  def start_with_prefix
+    "#@prefix%"
+  end
+
   # The execute method executes an SQL query on the database by calling the
   # \@database.execute method.
   #
@@ -412,18 +443,18 @@ class Documentrix::Documents::Cache::SQLiteCache
     @database.enable_load_extension(false)
     execute %{
       CREATE VIRTUAL TABLE IF NOT EXISTS embeddings USING vec0(
-        embedding float[#@embedding_length]
+        embedding FLOAT[#@embedding_length]
       )
     }
     execute %{
       CREATE TABLE IF NOT EXISTS records (
-        key          text NOT NULL PRIMARY KEY ON CONFLICT REPLACE,
-        text         text NOT NULL DEFAULT '',
-        embedding_id integer,
-        norm         float NOT NULL DEFAULT 0.0,
-        source       text,
-        digest       text,
-        tags         json NOT NULL DEFAULT [],
+        key          TEXT NOT NULL PRIMARY KEY ON CONFLICT REPLACE,
+        text         TEXT NOT NULL DEFAULT '',
+        embedding_id INTEGER,
+        norm         FLOAT NOT NULL DEFAULT 0.0,
+        source       TEXT,
+        digest       TEXT,
+        tags         JSON NOT NULL DEFAULT [],
         FOREIGN KEY(embedding_id) REFERENCES embeddings(id) ON DELETE CASCADE
       )
     }

data/lib/documentrix/documents.rb

@@ -162,7 +162,7 @@ class Documentrix::Documents
       infobar.progress by: batch.size
     end
     infobar.newline
-    self
+    invalidate_collections_cache!
   end
   alias << add
 
@@ -201,7 +201,9 @@ class Documentrix::Documents
   # @return [ FalseClass, TrueClass ] true if the text was removed, false
   #         otherwise.
   def delete(text)
-    @cache.delete(key(text))
+    res = @cache.delete(key(text))
+    invalidate_collections_cache! if res
+    res
   end
 
   # The size method returns the number of texts stored in the cache of this
@@ -220,7 +222,7 @@ class Documentrix::Documents
   # @return [ Documentrix::Documents ] self
   def clear(tags: nil)
     @cache.clear(tags:)
-    self
+    invalidate_collections_cache!
   end
 
   # Normalizes the source identifier to a canonical form.
@@ -318,7 +320,7 @@ class Documentrix::Documents
   def source_remove(source, digest: nil)
     source = normalize_source(source)
     @cache.clear_by_source(source, digest:, operator: '!=')
-    self
+    invalidate_collections_cache!
   end
 
   # The find method searches for strings within the cache by computing their
@@ -359,7 +361,7 @@ class Documentrix::Documents
   # @param text_count [Integer] the maximum number of records to return
   # @param opts [Hash] additional options passed to #find, such as:
   #   * :tags [Array<String>] filter results by tags
-  #   * :prompt [String] a prompt to use for the search
+  #   * :prompt [String] use for the search
   #   * :min_similarity [Numeric] minimum similarity score
   #
   # @example
@@ -379,7 +381,9 @@ class Documentrix::Documents
   #
   # @return [Array] An array of unique collection names
   def collections
-    ([ default_collection ] + @cache.collections('%s-' % class_prefix)).uniq
+    @collections_cache ||= (
+      [ default_collection ] + @cache.collections('%s-' % class_prefix)
+    ).uniq
   end
 
   # Rename the current collection, moving all keys from the old prefix to a new
@@ -395,6 +399,7 @@ class Documentrix::Documents
     new_prefix = '%s-%s-' % [ class_prefix, new_collection ]
     @cache.move_prefix(prefix, new_prefix)
     self.collection = new_collection
+    invalidate_collections_cache!
   end
 
   # The tags method returns an array of unique tags from the cache.
@@ -406,6 +411,18 @@ class Documentrix::Documents
 
   private
 
+  # Resets the memoized list of collections.
+  #
+  # This is called whenever a mutation occurs that could change the set of
+  # existing collections, ensuring that the #collections method returns a
+  # fresh, accurate list on the next call.
+  #
+  # @return [ Documentrix::Documents ] self
+  def invalidate_collections_cache!
+    @collections_cache = nil
+    self
+  end
+
   # The connect_cache method initializes and returns an instance of the
   # specified cache class.
   #
@@ -482,7 +499,7 @@ class Documentrix::Documents
   # The prefix method returns a string that is used as the prefix for keys in
   # the cache of the currently configured collection.
   #
-  # @return [ String ] The prefix string
+  # @return [ String ] the prefix string
   def prefix
     '%s-%s-' % [ class_prefix, @collection ]
   end

data/lib/documentrix/version.rb

@@ -1,6 +1,6 @@
 module Documentrix
   # Documentrix version
-  VERSION         = '0.3.0'
+  VERSION         = '0.3.2'
   VERSION_ARRAY   = VERSION.split('.').map(&:to_i) # :nodoc:
   VERSION_MAJOR   = VERSION_ARRAY[0] # :nodoc:
   VERSION_MINOR   = VERSION_ARRAY[1] # :nodoc:

data/spec/documentrix/documents/cache/sqlite_cache_spec.rb

@@ -250,6 +250,47 @@ describe Documentrix::Documents::SQLiteCache do
     ]
   end
 
+  describe 'Prefix Isolation' do
+    let(:cache2) { cache.dup }
+
+    before do
+      cache2.prefix = 'other-'
+
+      # Setup shared sources and tags across prefixes
+      cache['foo'] = test_value.merge(source: 'shared.txt', tags: %w[ a ])
+      cache2['bar'] = test_value.merge(source: 'shared.txt', tags: %w[ a ])
+    end
+
+    it 'does not leak clear_by_source' do
+      expect {
+        cache.clear_by_source('shared.txt')
+      }.to change { cache.size }.from(1).to(0)
+
+      expect(cache2.size).to eq 1
+      expect(cache2.key?('bar')).to be true
+    end
+
+    it 'does not leak source_exist?' do
+      # Ensure we are checking a source that ONLY exists in the other prefix
+      cache.clear_all_with_prefix
+      cache2['baz'] = test_value.merge(source: 'only-in-2.txt')
+
+      expect(cache.source_exist?('only-in-2.txt')).to be false
+      expect(cache2.source_exist?('only-in-2.txt')).to be true
+    end
+
+    it 'does not leak tags' do
+      cache.clear_all_with_prefix
+      cache2.clear_all_with_prefix
+
+      cache['foo'] = test_value.merge(tags: %w[ prefix1 ])
+      cache2['bar'] = test_value.merge(tags: %w[ prefix2 ])
+
+      expect(cache.tags.to_a).to match_array(['prefix1'])
+      expect(cache2.tags.to_a).to match_array(['prefix2'])
+    end
+  end
+
   describe '#find_records' do
     let(:needle) { [ 0.5 ] * 1_024 }
 

data/spec/documents_spec.rb

@@ -23,6 +23,8 @@ describe Documentrix::Documents do
     expect(ollama).to receive(:embed).
       with(model:, input: %w[ foo bar ], options: nil).
       and_return(double(embeddings: [ [ 0.1 ], [ 0.2 ] ]))
+    expect(documents).to receive(:invalidate_collections_cache!).
+      and_call_original
     expect(documents.add(%w[ foo bar ])).to eq documents
     expect(documents.exist?('foo')).to eq true
     expect(documents.exist?('bar')).to eq true
@@ -33,6 +35,8 @@ describe Documentrix::Documents do
     expect(ollama).to receive(:embed).
       with(model:, input: %w[ foo ], options: nil).
       and_return(double(embeddings: [ [ 0.1 ] ]))
+    expect(documents).to receive(:invalidate_collections_cache!).
+      and_call_original
     expect(documents << 'foo').to eq documents
     expect(documents.exist?('foo')).to eq true
     expect(documents.exist?('bar')).to eq false
@@ -123,6 +127,8 @@ describe Documentrix::Documents do
 
     it 'can delete texts' do
       expect(documents << 'foo').to eq documents
+      expect(documents).to receive(:invalidate_collections_cache!).
+        and_call_original
       expect {
         documents.delete('foo')
       }.to change { documents.exist?('foo') }.from(true).to(false)
@@ -136,6 +142,8 @@ describe Documentrix::Documents do
 
     it 'can clear texts' do
       expect(documents << 'foo').to eq documents
+      expect(documents).to receive(:invalidate_collections_cache!).
+        and_call_original
       expect {
         documents.clear
       }.to change { documents.size }.from(1).to(0)
@@ -148,9 +156,13 @@ describe Documentrix::Documents do
       expect(documents.add('foo', tags: %w[ test ])).to eq documents
       expect(documents.add('bar', tags: %w[ test2 ])).to eq documents
       expect(documents.tags.to_a).to eq %w[ test test2 ]
+      expect(documents).to receive(:invalidate_collections_cache!).
+        and_call_original
       expect {
         documents.clear tags: 'test'
       }.to change { documents.size }.from(2).to(1)
+      expect(documents).to receive(:invalidate_collections_cache!).
+        and_call_original
       expect {
         documents.clear tags: :test2
       }.to change { documents.size }.from(1).to(0)
@@ -166,6 +178,8 @@ describe Documentrix::Documents do
 
       expect(documents.size).to eq 3
 
+      expect(documents).to receive(:invalidate_collections_cache!).
+        and_call_original
       documents.source_remove('source1')
 
       expect(documents.size).to eq 1
@@ -182,6 +196,8 @@ describe Documentrix::Documents do
       documents.collection = :foo
       documents << 'foo'
       expect(documents.collections).to eq %i[ default foo ]
+      expect(documents).to receive(:invalidate_collections_cache!).
+        and_call_original
       documents.rename_collection(:bar)
       expect(documents.collection).to eq :bar
       expect(documents.collections).to eq %i[ default bar ]
@@ -195,6 +211,8 @@ describe Documentrix::Documents do
       documents.collection = :bar
       documents << 'foo'
       expect(documents.collections).to eq %i[ default foo bar ]
+      expect(documents).not_to receive(:invalidate_collections_cache!).
+        and_call_original
       expect {
         documents.rename_collection(:foo)
       }.to raise_error(ArgumentError, 'new collection foo already exists!')
@@ -243,6 +261,8 @@ describe Documentrix::Documents do
       documents.add('foo', source: 's1')
 
       expect(ollama).not_to receive(:embed)
+      expect(documents).to receive(:invalidate_collections_cache!).
+        and_call_original
       documents.source_update(['foo'], source: 's1')
       expect(documents.exist?('foo')).to be true
     end
@@ -255,6 +275,8 @@ describe Documentrix::Documents do
       allow(documents.cache).to receive(:compute_file_digest).with('s1').and_return('d2')
 
       expect(ollama).to receive(:embed).once
+      expect(documents).to receive(:invalidate_collections_cache!).
+        at_least(1).and_call_original
       documents.source_update(['bar'], source: 's1')
 
       expect(documents.exist?('bar')).to be true
@@ -263,6 +285,8 @@ describe Documentrix::Documents do
 
     it 'updates the source if it is an URL' do
       expect(ollama).to receive(:embed).once
+      expect(documents).to receive(:invalidate_collections_cache!).
+        and_call_original
       documents.source_update('foo', source: 'https://www.example.com/s1')
       expect(documents.exist?('foo')).to be true
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: documentrix
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.2
 platform: ruby
 authors:
 - Florian Frank