documentrix 0.3.2 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 332c67275b90bcc797cdd8e8df75a751bcfa7a52c759c9bcbe4a55970e905401
-  data.tar.gz: c546f89c61613b11d18c2cd724cadae8c303a988f925ad1624c5121f511e88fd
+  metadata.gz: 11ea07d3133f8de898211353a4bcff7f926b9f75114696af333e928862ee5aeb
+  data.tar.gz: b60d8606b1974bcb43e6584f690ecb21a7e54af6915b09d9adb904bc9269b417
 SHA512:
-  metadata.gz: c80a4c253b4fe367fae1dbb7e6aea50e274a495a0026a7b676e1e962b80c33806fe89976338c12931f11a5059e384fdfddcfe4665f68bde406b118edd8c04d15
-  data.tar.gz: '09292fd5d6aed6c939c0244467c743d515e68e5e253d7f443ef96a9c295a61689f1be5b9aec53e402287b6b6c7d305757eb26f79f5ade0161f7e82d534a7b3bf'
+  metadata.gz: ae907af900fc6932de6b2d022a4ca97367d6e1901442ed30e838eb7bf128b5e5d7dd572862f29cfe6196f58553d394e390062287b21cac6185e2c6845244ee5a
+  data.tar.gz: 75b95e852cbd6b412ae2651668db5cbffcb9f1c5845dd3bdfb2ea252380b9ab37d15f4b8381398f941c4c1dabeb1e7a174e9e6ef7dc4e0dc153455c4c0413b1e

data/CHANGES.md

@@ -1,5 +1,41 @@
 # Changes
 
+## 2026-06-16 v0.5.0
+
+### Improvements
+
+- Enhanced SQLite concurrency and prevented database locks:
+    - Added `database_busy_timeout` parameter to
+      `Documentrix::Documents#initialize`, defaulting to **5000**ms.
+    - Updated `Documentrix::Documents#connect_cache` to pass the timeout value
+      to the cache backend.
+    - Implemented `busy_timeout` support in
+      `Documentrix::Documents::Cache::SQLiteCache#initialize`.
+    - Configured `@database.busy_handler_timeout` in
+      `Documentrix::Documents::Cache::SQLiteCache#setup_database` to ensure
+      GVL-friendly waiting during lock contention.
+- Prevented immediate `SQLITE_BUSY` errors on writes by updating
+  `Documentrix::Documents::Cache::SQLiteCache#[]=` to use `BEGIN IMMEDIATE`
+  instead of `BEGIN`, avoiding transaction upgrade failures.
+
+## 2026-05-22 v0.4.0
+
+### Added
+
+- Added introspection methods to `Documents`:
+    - Implemented `sources` to return an array of unique source identifiers.
+    - Implemented `each_record` to iterate over records, returning an
+      `Enumerator` if no block is provided.
+
+### Changed
+
+- Updated CI images from Alpine to Debian:
+    - Switched Ruby **4.0**, **3.4**, **3.3**, and **3.2** images to the
+      `trixie` distribution.
+    - Switched Ruby **3.1** image to the `bookworm` distribution.
+    - Updated the `dockerfile` in `.all_images.yml` to use `apt-get` instead of
+      `apk` for installing build dependencies.
+
 ## 2026-05-20 v0.3.2
 
 ### Performance Improvements

data/documentrix.gemspec

@@ -1,9 +1,9 @@
 # -*- encoding: utf-8 -*-
-# stub: documentrix 0.3.2 ruby lib
+# stub: documentrix 0.5.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "documentrix".freeze
-  s.version = "0.3.2".freeze
+  s.version = "0.5.0".freeze
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]

data/lib/documentrix/documents/cache/sqlite_cache.rb

@@ -21,13 +21,15 @@ class Documentrix::Documents::Cache::SQLiteCache
   # @param embedding_length [ Integer ] the length of the embeddings vector
   # @param filename [ String ] the name of the SQLite database file or ':memory:' for in-memory.
   # @param debug [ FalseClass, TrueClass ] whether to enable debugging
+  # @param busy_timeout [ Integer ] the SQLite busy timeout in milliseconds (defaults to 5000)
   #
   # @return [ void ]
-  def initialize(prefix:, embedding_length: 1_024, filename: ':memory:', debug: false)
+  def initialize(prefix:, embedding_length: 1_024, filename: ':memory:', debug: false, busy_timeout: 5000)
     super(prefix:)
     @embedding_length = embedding_length
     @filename         = filename
     @debug            = debug
+    @busy_timeout     = busy_timeout
     setup_database(filename)
   end
 
@@ -69,7 +71,7 @@ class Documentrix::Documents::Cache::SQLiteCache
     value     = convert_value_to_record(value)
     digest    = compute_file_digest(value.source)
     embedding = value.embedding.pack("f*")
-    execute(%{BEGIN})
+    execute(%{BEGIN IMMEDIATE})
     execute(%{INSERT INTO embeddings(embedding) VALUES(?)}, [ embedding ])
     embedding_id, = execute(%{ SELECT last_insert_rowid() }).flatten
     execute(%{
@@ -170,6 +172,8 @@ class Documentrix::Documents::Cache::SQLiteCache
   # @param source [String] the source identifier used to filter records
   # @param digest [String, nil] the SHA256 hexadecimal digest of the source.
   #   Records matching this digest will be preserved.
+  # @param operator [String] the operator to use for comparison ('=' or '!=').
+  #   Defaults to '='.
   #
   # @return [self] the cache instance for method chaining
   def clear_by_source(source, digest: nil, operator: ?=)
@@ -438,6 +442,7 @@ class Documentrix::Documents::Cache::SQLiteCache
   # @return [ nil ]
   def setup_database(filename)
     @database = SQLite3::Database.new(filename)
+    @database.busy_handler_timeout = @busy_timeout
     @database.enable_load_extension(true)
     SqliteVec.load(@database)
     @database.enable_load_extension(false)

data/lib/documentrix/documents.rb

@@ -76,12 +76,13 @@ class Documentrix::Documents
   # @param database_filename [ String ] the filename of the SQLite database to use (defaults to ':memory:')
   # @param redis_url [ String ] the URL of the Redis server to use (defaults to nil)
   # @param debug [ FalseClass, TrueClass ] whether to enable debugging mode (defaults to false)
-  def initialize(ollama:, model:, model_options: nil, collection: nil, embedding_length: 1_024, cache: MemoryCache, database_filename: nil, redis_url: nil, debug: false)
+  # @param database_busy_timeout [ Integer ] the SQLite busy timeout in milliseconds (defaults to 5000)
+  def initialize(ollama:, model:, model_options: nil, collection: nil, embedding_length: 1_024, cache: MemoryCache, database_filename: nil, redis_url: nil, debug: false, database_busy_timeout: 5000)
     collection ||= default_collection
     @ollama, @model, @model_options, @collection, @debug =
       ollama, model, model_options, collection.to_sym, debug
     database_filename ||= ':memory:'
-    @cache = connect_cache(cache, redis_url, embedding_length, database_filename)
+    @cache = connect_cache(cache, redis_url, embedding_length, database_filename, database_busy_timeout)
   end
 
   # The default_collection method returns the default collection name.
@@ -104,7 +105,7 @@ class Documentrix::Documents
   # The prepare_texts method filters out existing texts from the input array
   # and returns the filtered array.
   #
-  # @param texts [ Array ] an array of text strings or #read objects.
+  # @param texts [ Array ] an array of text strings
   #
   # @return [ Array ] the filtered array of text strings
   private def prepare_texts(texts)
@@ -409,6 +410,22 @@ class Documentrix::Documents
     @cache.tags
   end
 
+  # Returns an array of all unique sources stored in the cache.
+  #
+  # @return [Array<String>] An array of unique source identifiers.
+  def sources
+    @cache.each_source.to_a
+  end
+
+ # The each_record method iterates over all records stored in the cache.
+ #
+ # @yield [record] The record being iterated over.
+ # @return [ Enumerator ] an enumerator if no block is provided.
+  def each_record(&block)
+    block or return enum_for(__method__)
+    @cache.each { |_key, record| block.(record) }
+  end
+
   private
 
   # Resets the memoized list of collections.
@@ -430,9 +447,10 @@ class Documentrix::Documents
   # @param redis_url [String] the URL of the Redis server
   # @param embedding_length [Integer] the length of the embeddings used in the cache
   # @param database_filename [String] the filename of the SQLite database file
+  # @param database_busy_timeout [Integer] the SQLite busy timeout in milliseconds
   #
   # @return [CacheInstance] an instance of the specified cache class
-  def connect_cache(cache_class, redis_url, embedding_length, database_filename)
+  def connect_cache(cache_class, redis_url, embedding_length, database_filename, database_busy_timeout)
     cache = nil
     if (cache_class.instance_method(:redis) rescue nil)
       begin
@@ -449,6 +467,7 @@ class Documentrix::Documents
         prefix:,
         embedding_length:,
         filename: database_filename,
+        busy_timeout: database_busy_timeout,
         debug: @debug
       )
     end

data/lib/documentrix/version.rb

@@ -1,6 +1,6 @@
 module Documentrix
   # Documentrix version
-  VERSION         = '0.3.2'
+  VERSION         = '0.5.0'
   VERSION_ARRAY   = VERSION.split('.').map(&:to_i) # :nodoc:
   VERSION_MAJOR   = VERSION_ARRAY[0] # :nodoc:
   VERSION_MINOR   = VERSION_ARRAY[1] # :nodoc:

data/spec/documents_spec.rb

@@ -226,6 +226,32 @@ describe Documentrix::Documents do
         from(:default).
         to(:new_collection)
     end
+
+    it 'returns unique sources' do
+      allow(ollama).to receive(:embed).and_return(double(embeddings: [ [ 0.1 ] ]))
+      documents.add('foo', source: 's1')
+      documents.add('bar', source: 's1')
+      documents.add('baz', source: 's2')
+      expect(documents.sources).to match_array %w[ s1 s2 ]
+    end
+
+    it 'can iterate over records' do
+      allow(ollama).to receive(:embed).and_return(double(embeddings: [ [ 0.1 ] ]))
+      documents.add('foo')
+      documents.add('bar')
+      records = []
+      documents.each_record { |r| records << r }
+      expect(records.size).to eq 2
+      expect(records.map(&:text)).to match_array %w[ foo bar ]
+    end
+
+    it 'returns an enumerator for each_record' do
+      allow(ollama).to receive(:embed).and_return(double(embeddings: [ [ 0.1 ] ]))
+      documents.add('foo')
+      documents.add('bar')
+      expect(documents.each_record).to be_a Enumerator
+      expect(documents.each_record.map(&:text)).to match_array %w[ foo bar ]
+    end
   end
 
   context 'source management' do
@@ -233,7 +259,7 @@ describe Documentrix::Documents do
       allow(documents).to receive(:compute_file_digest).and_return('d1')
       allow(documents.cache).to receive(:compute_file_digest).and_return('d1')
 
-      allow(ollama).to receive(:embed).and_return(double(embeddings: [[0.1]]))
+      allow(ollama).to receive(:embed).and_return(double(embeddings: [ [ 0.1 ] ]))
     end
 
     it 'can check if a source exists' do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: documentrix
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.5.0
 platform: ruby
 authors:
 - Florian Frank