documentrix 0.4.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e32a72c0a1f93a96f7c3cecd185f13a0f17a2629b2dc509ec3a29fd2d7b51a41
-  data.tar.gz: 3f2c21125adf7061dcba94f1843456064245222839de0df6ddbcd8743a1aea13
+  metadata.gz: ced4bf69e3ae1b4251844dd7915f8d75e72d1c40410f34c14311f70dd55c91f4
+  data.tar.gz: 6fc0799e4559f50a22d211e630ea5f7deaac671f24805f5501c85a11c7a9b853
 SHA512:
-  metadata.gz: 91dbf3ddfdeb124661ff78a4cbc1635dbb8bdf6606078aeeef77b4f9d14688d073261ad08b59f0333895c7391d83eff8a6c76467f19af4f04ca55172c1d934a5
-  data.tar.gz: d40e5a53ceeda71c7a2be37d3bfb718ffc2ddb0fddc0c6eb2346c295d06713c895493854fab5f768af075bec2595e2f23c40e893d7f7266c6f61cd95c534d733
+  metadata.gz: 8023a4b4d7cad8948e6cbd2a65cefd8b3993ae3de38ae5520a2ebf178989a78efb4c1296b51c614c4066c547372f9ad8067b6c8e196caaa098876a3adf44f523
+  data.tar.gz: 0c4544a80ecde5c98c1d3da64ffeb97155324fdc063993caddf6d2d0780c490b0c8ec24dbc2ca392b3ceb6023d964f58bad099ca4c509a6e43c33c4f7406beb0

data/CHANGES.md

@@ -1,5 +1,43 @@
 # Changes
 
+## 2026-06-17 v0.6.0
+
+### Changed
+
+- Refactored collection discovery to ensure strong consistency across multiple
+  client instances by delegating `Documentrix::Documents#collections` directly
+  to the cache backend.
+- Removed `@collections_cache` and the `invalidate_collections_cache!` method
+  from `lib/documentrix/documents.rb`.
+- Implemented a high-performance `#collections` method in
+  `lib/documentrix/documents/cache/redis_cache.rb` utilizing `scan_each`.
+- Implemented a high-performance `#collections` method in
+  `lib/documentrix/documents/cache/sqlite_cache.rb` using SQL `DISTINCT`.
+- Added specialized unit tests for collection extraction and regex patterns
+  within `spec/documentrix/documents/cache/redis_cache_spec.rb` and
+  `spec/documentrix/documents/cache/sqlite_cache_spec.rb`.
+- Updated `spec/documents_spec.rb` and
+  `spec/documentrix/documents/cache/interface_spec.rb` to remove dependencies
+  on the deleted invalidation method.
+
+## 2026-06-16 v0.5.0
+
+### Improvements
+
+- Enhanced SQLite concurrency and prevented database locks:
+    - Added `database_busy_timeout` parameter to
+      `Documentrix::Documents#initialize`, defaulting to **5000**ms.
+    - Updated `Documentrix::Documents#connect_cache` to pass the timeout value
+      to the cache backend.
+    - Implemented `busy_timeout` support in
+      `Documentrix::Documents::Cache::SQLiteCache#initialize`.
+    - Configured `@database.busy_handler_timeout` in
+      `Documentrix::Documents::Cache::SQLiteCache#setup_database` to ensure
+      GVL-friendly waiting during lock contention.
+- Prevented immediate `SQLITE_BUSY` errors on writes by updating
+  `Documentrix::Documents::Cache::SQLiteCache#[]=` to use `BEGIN IMMEDIATE`
+  instead of `BEGIN`, avoiding transaction upgrade failures.
+
 ## 2026-05-22 v0.4.0
 
 ### Added

data/documentrix.gemspec

@@ -1,9 +1,9 @@
 # -*- encoding: utf-8 -*-
-# stub: documentrix 0.4.0 ruby lib
+# stub: documentrix 0.6.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "documentrix".freeze
-  s.version = "0.4.0".freeze
+  s.version = "0.6.0".freeze
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]

data/lib/documentrix/documents/cache/redis_cache.rb

@@ -101,8 +101,23 @@ class Documentrix::Documents::RedisCache
     s
   end
 
-  # The clear_all_with_prefix method removes all key-value pairs associated
-  # with the given prefix from this cache instance.
+  # Returns an array of collection names that match the given prefix.
+  # This is a high-performance override for Redis that only queries keys.
+  #
+  # @param prefix [String] the prefix to search for in collection names
+  # @return [Array<Symbol>] an array of matching collection names
+  def collections(prefix)
+    unique = Set.new
+    redis.scan_each(match: "#{prefix}*") do |key|
+      if key =~ /\A#{prefix}(.+)-/
+        unique << $1.to_sym
+      end
+    end
+    unique.to_a
+  end
+
+  # The clear_all_with_prefix method removes all key-value pairs associated with
+  # the given prefix from this cache instance.
   #
   # @return [Documentrix::Documents::RedisCache] self
   def clear_all_with_prefix

data/lib/documentrix/documents/cache/sqlite_cache.rb

@@ -21,13 +21,15 @@ class Documentrix::Documents::Cache::SQLiteCache
   # @param embedding_length [ Integer ] the length of the embeddings vector
   # @param filename [ String ] the name of the SQLite database file or ':memory:' for in-memory.
   # @param debug [ FalseClass, TrueClass ] whether to enable debugging
+  # @param busy_timeout [ Integer ] the SQLite busy timeout in milliseconds (defaults to 5000)
   #
   # @return [ void ]
-  def initialize(prefix:, embedding_length: 1_024, filename: ':memory:', debug: false)
+  def initialize(prefix:, embedding_length: 1_024, filename: ':memory:', debug: false, busy_timeout: 5000)
     super(prefix:)
     @embedding_length = embedding_length
     @filename         = filename
     @debug            = debug
+    @busy_timeout     = busy_timeout
     setup_database(filename)
   end
 
@@ -69,7 +71,7 @@ class Documentrix::Documents::Cache::SQLiteCache
     value     = convert_value_to_record(value)
     digest    = compute_file_digest(value.source)
     embedding = value.embedding.pack("f*")
-    execute(%{BEGIN})
+    execute(%{BEGIN IMMEDIATE})
     execute(%{INSERT INTO embeddings(embedding) VALUES(?)}, [ embedding ])
     embedding_id, = execute(%{ SELECT last_insert_rowid() }).flatten
     execute(%{
@@ -106,6 +108,22 @@ class Documentrix::Documents::Cache::SQLiteCache
     result
   end
 
+  # Returns an array of collection names that match the given prefix.
+  # This is a high-performance override for SQLite that only queries keys.
+  #
+  # @param prefix [String] the prefix to search for in collection names
+  # @return [Array<Symbol>] an array of matching collection names
+  def collections(prefix)
+    execute(
+      %{ SELECT DISTINCT key FROM records WHERE key LIKE ? },
+      [ "#{prefix}%" ]
+    ).flatten.each_with_object(Set.new) do |key, set|
+      if key =~ /\A#{prefix}(.+)-/
+        set << $1.to_sym
+      end
+    end.to_a
+  end
+
   # The tags method returns an array of unique tags from the database.
   #
   # @return [Documentrix::Utils::Tags] An instance of Documentrix::Utils::Tags
@@ -170,6 +188,8 @@ class Documentrix::Documents::Cache::SQLiteCache
   # @param source [String] the source identifier used to filter records
   # @param digest [String, nil] the SHA256 hexadecimal digest of the source.
   #   Records matching this digest will be preserved.
+  # @param operator [String] the operator to use for comparison ('=' or '!=').
+  #   Defaults to '='.
   #
   # @return [self] the cache instance for method chaining
   def clear_by_source(source, digest: nil, operator: ?=)
@@ -438,6 +458,7 @@ class Documentrix::Documents::Cache::SQLiteCache
   # @return [ nil ]
   def setup_database(filename)
     @database = SQLite3::Database.new(filename)
+    @database.busy_handler_timeout = @busy_timeout
     @database.enable_load_extension(true)
     SqliteVec.load(@database)
     @database.enable_load_extension(false)

data/lib/documentrix/documents.rb

@@ -76,12 +76,13 @@ class Documentrix::Documents
   # @param database_filename [ String ] the filename of the SQLite database to use (defaults to ':memory:')
   # @param redis_url [ String ] the URL of the Redis server to use (defaults to nil)
   # @param debug [ FalseClass, TrueClass ] whether to enable debugging mode (defaults to false)
-  def initialize(ollama:, model:, model_options: nil, collection: nil, embedding_length: 1_024, cache: MemoryCache, database_filename: nil, redis_url: nil, debug: false)
+  # @param database_busy_timeout [ Integer ] the SQLite busy timeout in milliseconds (defaults to 5000)
+  def initialize(ollama:, model:, model_options: nil, collection: nil, embedding_length: 1_024, cache: MemoryCache, database_filename: nil, redis_url: nil, debug: false, database_busy_timeout: 5000)
     collection ||= default_collection
     @ollama, @model, @model_options, @collection, @debug =
       ollama, model, model_options, collection.to_sym, debug
     database_filename ||= ':memory:'
-    @cache = connect_cache(cache, redis_url, embedding_length, database_filename)
+    @cache = connect_cache(cache, redis_url, embedding_length, database_filename, database_busy_timeout)
   end
 
   # The default_collection method returns the default collection name.
@@ -104,7 +105,7 @@ class Documentrix::Documents
   # The prepare_texts method filters out existing texts from the input array
   # and returns the filtered array.
   #
-  # @param texts [ Array ] an array of text strings or #read objects.
+  # @param texts [ Array ] an array of text strings
   #
   # @return [ Array ] the filtered array of text strings
   private def prepare_texts(texts)
@@ -162,7 +163,7 @@ class Documentrix::Documents
       infobar.progress by: batch.size
     end
     infobar.newline
-    invalidate_collections_cache!
+    self
   end
   alias << add
 
@@ -201,9 +202,7 @@ class Documentrix::Documents
   # @return [ FalseClass, TrueClass ] true if the text was removed, false
   #         otherwise.
   def delete(text)
-    res = @cache.delete(key(text))
-    invalidate_collections_cache! if res
-    res
+    @cache.delete(key(text))
   end
 
   # The size method returns the number of texts stored in the cache of this
@@ -222,7 +221,7 @@ class Documentrix::Documents
   # @return [ Documentrix::Documents ] self
   def clear(tags: nil)
     @cache.clear(tags:)
-    invalidate_collections_cache!
+    self
   end
 
   # Normalizes the source identifier to a canonical form.
@@ -320,7 +319,7 @@ class Documentrix::Documents
   def source_remove(source, digest: nil)
     source = normalize_source(source)
     @cache.clear_by_source(source, digest:, operator: '!=')
-    invalidate_collections_cache!
+    self
   end
 
   # The find method searches for strings within the cache by computing their
@@ -381,9 +380,7 @@ class Documentrix::Documents
   #
   # @return [Array] An array of unique collection names
   def collections
-    @collections_cache ||= (
-      [ default_collection ] + @cache.collections('%s-' % class_prefix)
-    ).uniq
+    [ default_collection ].concat(@cache.collections('%s-' % class_prefix)).uniq
   end
 
   # Rename the current collection, moving all keys from the old prefix to a new
@@ -399,7 +396,7 @@ class Documentrix::Documents
     new_prefix = '%s-%s-' % [ class_prefix, new_collection ]
     @cache.move_prefix(prefix, new_prefix)
     self.collection = new_collection
-    invalidate_collections_cache!
+    self
   end
 
   # The tags method returns an array of unique tags from the cache.
@@ -427,18 +424,6 @@ class Documentrix::Documents
 
   private
 
-  # Resets the memoized list of collections.
-  #
-  # This is called whenever a mutation occurs that could change the set of
-  # existing collections, ensuring that the #collections method returns a
-  # fresh, accurate list on the next call.
-  #
-  # @return [ Documentrix::Documents ] self
-  def invalidate_collections_cache!
-    @collections_cache = nil
-    self
-  end
-
   # The connect_cache method initializes and returns an instance of the
   # specified cache class.
   #
@@ -446,9 +431,10 @@ class Documentrix::Documents
   # @param redis_url [String] the URL of the Redis server
   # @param embedding_length [Integer] the length of the embeddings used in the cache
   # @param database_filename [String] the filename of the SQLite database file
+  # @param database_busy_timeout [Integer] the SQLite busy timeout in milliseconds
   #
   # @return [CacheInstance] an instance of the specified cache class
-  def connect_cache(cache_class, redis_url, embedding_length, database_filename)
+  def connect_cache(cache_class, redis_url, embedding_length, database_filename, database_busy_timeout)
     cache = nil
     if (cache_class.instance_method(:redis) rescue nil)
       begin
@@ -465,6 +451,7 @@ class Documentrix::Documents
         prefix:,
         embedding_length:,
         filename: database_filename,
+        busy_timeout: database_busy_timeout,
         debug: @debug
       )
     end

data/lib/documentrix/version.rb

@@ -1,6 +1,6 @@
 module Documentrix
   # Documentrix version
-  VERSION         = '0.4.0'
+  VERSION         = '0.6.0'
   VERSION_ARRAY   = VERSION.split('.').map(&:to_i) # :nodoc:
   VERSION_MAJOR   = VERSION_ARRAY[0] # :nodoc:
   VERSION_MINOR   = VERSION_ARRAY[1] # :nodoc:

data/spec/documentrix/documents/cache/interface_spec.rb

@@ -101,7 +101,7 @@ describe 'Documentrix::Documents::Cache Interface' do
 
       # Common methods from Cache::Common
       expect(cache).to respond_to(:collections)
-      expect(cache.method(:collections).owner).to eq Documentrix::Documents::Cache::Common
+      expect(cache.method(:collections).owner).to eq Documentrix::Documents::RedisCache
 
       expect(cache).to respond_to(:pre)
       expect(cache.method(:pre).owner).to eq Documentrix::Documents::Cache::Common
@@ -167,7 +167,7 @@ describe 'Documentrix::Documents::Cache Interface' do
 
       # Common methods from Cache::Common
       expect(cache).to respond_to(:collections)
-      expect(cache.method(:collections).owner).to eq Documentrix::Documents::Cache::Common
+      expect(cache.method(:collections).owner).to eq Documentrix::Documents::Cache::SQLiteCache
 
       expect(cache).to respond_to(:pre)
       expect(cache.method(:pre).owner).to eq Documentrix::Documents::Cache::Common

data/spec/documentrix/documents/cache/redis_cache_spec.rb

@@ -184,5 +184,29 @@ describe Documentrix::Documents::RedisCache do
       expect(redis).to receive(:get).with("#{prefix}foo").and_return(JSON(source: 's1', digest: 'd1'))
       expect(cache.source_exist?('s1', digest: 'd2')).to be false
     end
+
+    describe '#collections' do
+      it 'extracts unique collection names from keys' do
+        expect(redis).to receive(:scan_each).with(match: "#{prefix}*").and_yield(
+          "#{prefix}col1-foo"
+        ).and_yield(
+          "#{prefix}col1-bar"
+        ).and_yield(
+          "#{prefix}col2-baz"
+        )
+
+        expect(cache.collections(prefix)).to match_array([:col1, :col2])
+      end
+
+      it 'ignores keys that do not follow the collection pattern' do
+        expect(redis).to receive(:scan_each).with(match: "#{prefix}*").and_yield(
+          "#{prefix}valid-foo"
+        ).and_yield(
+          "#{prefix}invalid" # No trailing dash after the name
+        )
+
+        expect(cache.collections(prefix)).to eq [:valid]
+      end
+    end
   end
 end

data/spec/documentrix/documents/cache/sqlite_cache_spec.rb

@@ -353,4 +353,29 @@ describe Documentrix::Documents::SQLiteCache do
       expect(cache.find_records(needle)).to eq []
     end
   end
+
+  describe '#collections' do
+    it 'extracts unique collection names matching the prefix' do
+      # Since cache['key'] = val stores as "#{prefix}#{key}",
+      # we can create keys like "col1-foo" to get "test-col1-foo"
+      cache['col1-foo'] = test_value
+      cache['col1-bar'] = test_value
+      cache['col2-baz'] = test_value
+      cache['justprefix'] = test_value # Matches prefix, but not the pattern "prefix(name)-"
+
+      expect(cache.collections('test-')).to match_array([:col1, :col2])
+    end
+
+    it 'returns empty array when no keys match the prefix' do
+      cache['foo'] = test_value
+      expect(cache.collections('nonexistent-')).to eq []
+    end
+
+    it 'returns empty array when keys start with prefix but lack a following hyphen' do
+      # We need a key that starts with "test-" but doesn't have another "-" later.
+      # Because cache['foo'] = val results in "test-foo", this is exactly what happens.
+      cache['foo'] = test_value
+      expect(cache.collections('test-')).to eq []
+    end
+  end
 end

data/spec/documents_spec.rb

@@ -23,8 +23,6 @@ describe Documentrix::Documents do
     expect(ollama).to receive(:embed).
       with(model:, input: %w[ foo bar ], options: nil).
       and_return(double(embeddings: [ [ 0.1 ], [ 0.2 ] ]))
-    expect(documents).to receive(:invalidate_collections_cache!).
-      and_call_original
     expect(documents.add(%w[ foo bar ])).to eq documents
     expect(documents.exist?('foo')).to eq true
     expect(documents.exist?('bar')).to eq true
@@ -35,8 +33,6 @@ describe Documentrix::Documents do
     expect(ollama).to receive(:embed).
       with(model:, input: %w[ foo ], options: nil).
       and_return(double(embeddings: [ [ 0.1 ] ]))
-    expect(documents).to receive(:invalidate_collections_cache!).
-      and_call_original
     expect(documents << 'foo').to eq documents
     expect(documents.exist?('foo')).to eq true
     expect(documents.exist?('bar')).to eq false
@@ -127,8 +123,6 @@ describe Documentrix::Documents do
 
     it 'can delete texts' do
       expect(documents << 'foo').to eq documents
-      expect(documents).to receive(:invalidate_collections_cache!).
-        and_call_original
       expect {
         documents.delete('foo')
       }.to change { documents.exist?('foo') }.from(true).to(false)
@@ -142,8 +136,6 @@ describe Documentrix::Documents do
 
     it 'can clear texts' do
       expect(documents << 'foo').to eq documents
-      expect(documents).to receive(:invalidate_collections_cache!).
-        and_call_original
       expect {
         documents.clear
       }.to change { documents.size }.from(1).to(0)
@@ -156,13 +148,9 @@ describe Documentrix::Documents do
       expect(documents.add('foo', tags: %w[ test ])).to eq documents
       expect(documents.add('bar', tags: %w[ test2 ])).to eq documents
       expect(documents.tags.to_a).to eq %w[ test test2 ]
-      expect(documents).to receive(:invalidate_collections_cache!).
-        and_call_original
       expect {
         documents.clear tags: 'test'
       }.to change { documents.size }.from(2).to(1)
-      expect(documents).to receive(:invalidate_collections_cache!).
-        and_call_original
       expect {
         documents.clear tags: :test2
       }.to change { documents.size }.from(1).to(0)
@@ -178,8 +166,6 @@ describe Documentrix::Documents do
 
       expect(documents.size).to eq 3
 
-      expect(documents).to receive(:invalidate_collections_cache!).
-        and_call_original
       documents.source_remove('source1')
 
       expect(documents.size).to eq 1
@@ -196,8 +182,6 @@ describe Documentrix::Documents do
       documents.collection = :foo
       documents << 'foo'
       expect(documents.collections).to eq %i[ default foo ]
-      expect(documents).to receive(:invalidate_collections_cache!).
-        and_call_original
       documents.rename_collection(:bar)
       expect(documents.collection).to eq :bar
       expect(documents.collections).to eq %i[ default bar ]
@@ -211,8 +195,6 @@ describe Documentrix::Documents do
       documents.collection = :bar
       documents << 'foo'
       expect(documents.collections).to eq %i[ default foo bar ]
-      expect(documents).not_to receive(:invalidate_collections_cache!).
-        and_call_original
       expect {
         documents.rename_collection(:foo)
       }.to raise_error(ArgumentError, 'new collection foo already exists!')
@@ -287,8 +269,6 @@ describe Documentrix::Documents do
       documents.add('foo', source: 's1')
 
       expect(ollama).not_to receive(:embed)
-      expect(documents).to receive(:invalidate_collections_cache!).
-        and_call_original
       documents.source_update(['foo'], source: 's1')
       expect(documents.exist?('foo')).to be true
     end
@@ -301,8 +281,6 @@ describe Documentrix::Documents do
       allow(documents.cache).to receive(:compute_file_digest).with('s1').and_return('d2')
 
       expect(ollama).to receive(:embed).once
-      expect(documents).to receive(:invalidate_collections_cache!).
-        at_least(1).and_call_original
       documents.source_update(['bar'], source: 's1')
 
       expect(documents.exist?('bar')).to be true
@@ -311,8 +289,6 @@ describe Documentrix::Documents do
 
     it 'updates the source if it is an URL' do
       expect(ollama).to receive(:embed).once
-      expect(documents).to receive(:invalidate_collections_cache!).
-        and_call_original
       documents.source_update('foo', source: 'https://www.example.com/s1')
       expect(documents.exist?('foo')).to be true
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: documentrix
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Florian Frank