documentrix 0.2.0 → 0.3.0

data/spec/documentrix/documents/cache/redis_cache_spec.rb

@@ -1,28 +1,20 @@
-require 'spec_helper'
-
 describe Documentrix::Documents::RedisCache do
+  let :object_class do
+    Documentrix::Documents::Cache::Records::Record
+  end
+
   let :prefix do
     'test-'
   end
 
   let :cache do
-    described_class.new prefix:, url: 'something'
+    described_class.new prefix:, url: 'something', object_class:
   end
 
   it 'can be instantiated' do
     expect(cache).to be_a described_class
   end
 
-  it 'defaults to nil object_class' do
-    expect(cache.object_class).to be_nil
-  end
-
-  it 'can be configured with object_class' do
-    object_class = Class.new(JSON::GenericObject)
-    cache = described_class.new(prefix:, url: 'something', object_class:)
-    expect(cache.object_class).to eq object_class
-  end
-
   it 'raises ArgumentError if url is missing' do
     expect {
       described_class.new prefix:, url: nil
@@ -38,6 +30,10 @@ describe Documentrix::Documents::RedisCache do
       allow_any_instance_of(described_class).to receive(:redis).and_return(redis)
     end
 
+    it 'can be configured with object_class' do
+      expect(cache.object_class).to eq object_class
+    end
+
     it 'has Redis client' do
       expect(cache.redis).to eq redis
     end
@@ -62,9 +58,9 @@ describe Documentrix::Documents::RedisCache do
     end
 
     it 'can move prefixes' do
-      expect(redis).to receive(:get).with(prefix + 'foo').and_return(JSON(foo: true))
-      expect(redis).to receive(:get).with('test2-bar').and_return(JSON(foo: true))
-      expect(redis).to receive(:set).with('test3-foo', '{"foo":true}')
+      expect(redis).to receive(:get).with(prefix + 'foo').and_return(object_class[foo: true].to_json)
+      expect(redis).to receive(:get).with('test2-bar').and_return(object_class[foo: true].to_json)
+      expect(redis).to receive(:set).with('test3-foo', /"foo":true/)
       expect(redis).to receive(:del).with('test-foo')
       expect(redis).to receive(:scan_each).with(match: ?*).
         and_yield("#{prefix}foo").
@@ -81,12 +77,12 @@ describe Documentrix::Documents::RedisCache do
     end
 
     it 'can iterate over keys, values' do
-      key, value = 'foo', { 'test' => true }
-      expect(redis).to receive(:set).with(prefix + key, JSON(value))
+      key, value = 'foo', object_class[test: true]
+      expect(redis).to receive(:set).with(prefix + key, object_class[value].to_json)
       cache[key] = value
       expect(redis).to receive(:scan_each).with(match: "#{prefix}*").
         and_yield("#{prefix}foo")
-      expect(redis).to receive(:get).with(prefix + key).and_return(JSON(test: true))
+      expect(redis).to receive(:get).with(prefix + key).and_return(object_class[test: true].to_json)
       cache.each do |k, v|
         expect(k).to eq prefix + key
         expect(v).to eq value
@@ -135,5 +131,58 @@ describe Documentrix::Documents::RedisCache do
     it 'can remove prefix with unpre' do
       expect(cache.unpre('test-foo')).to eq 'foo'
     end
+
+    it 'can iterate over unique sources' do
+      expect(redis).to receive(:scan_each).with(match: "#{prefix}*").and_yield(
+        "#{prefix}foo"
+      ).and_yield(
+        "#{prefix}bar"
+      )
+      expect(redis).to receive(:get).with("#{prefix}foo").and_return(JSON(source: 's1'))
+      expect(redis).to receive(:get).with("#{prefix}bar").and_return(JSON(source: 's2'))
+
+      expect(cache.each_source.to_a).to match_array(['s1', 's2'])
+    end
+
+    it 'can retrieve all unique tags' do
+      expect(redis).to receive(:scan_each).with(match: "#{prefix}*").and_yield(
+        "#{prefix}foo"
+      ).and_yield(
+        "#{prefix}bar"
+      )
+      expect(redis).to receive(:get).with("#{prefix}foo").and_return(JSON(source: 's1', tags: ['a', 'b']))
+      expect(redis).to receive(:get).with("#{prefix}bar").and_return(JSON(source: 's2', tags: ['b', 'c']))
+
+      expect(cache.tags.to_a).to match_array(['a', 'b', 'c'])
+    end
+
+    it 'can clear records by tags' do
+      expect(redis).to receive(:scan_each).with(match: "#{prefix}*").and_yield(
+        "#{prefix}foo"
+      ).and_yield(
+        "#{prefix}bar"
+      )
+      expect(redis).to receive(:get).with("#{prefix}foo").and_return(JSON(source: 's1', tags: ['trash']))
+      expect(redis).to receive(:get).with("#{prefix}bar").and_return(JSON(source: 's2', tags: ['keep']))
+      expect(redis).to receive(:del).with("#{prefix}foo")
+
+      expect(cache.clear_for_tags(['trash'])).to eq cache
+    end
+
+    it 'can check if a source exists with a specific digest' do
+      expect(redis).to receive(:scan_each).with(match: "#{prefix}*").and_yield(
+        "#{prefix}foo"
+      )
+      expect(redis).to receive(:get).with("#{prefix}foo").and_return(JSON(source: 's1', digest: 'd1'))
+
+      expect(cache.source_exist?('s1', digest: 'd1')).to be true
+
+      # Reset for the negative case
+      expect(redis).to receive(:scan_each).with(match: "#{prefix}*").and_yield(
+        "#{prefix}foo"
+      )
+      expect(redis).to receive(:get).with("#{prefix}foo").and_return(JSON(source: 's1', digest: 'd1'))
+      expect(cache.source_exist?('s1', digest: 'd2')).to be false
+    end
   end
 end

data/spec/documentrix/documents/cache/sqlite_cache_spec.rb

@@ -1,5 +1,3 @@
-require 'spec_helper'
-
 describe Documentrix::Documents::SQLiteCache do
   let :prefix do
     'test-'
@@ -145,6 +143,20 @@ describe Documentrix::Documents::SQLiteCache do
     expect(cache).to be_key 'bar'
   end
 
+  it 'can clear all without tags' do
+    key, value = 'foo', { tags: %w[ foo ], embedding: [ 0.5 ] * 1_024 }
+    cache[key] = value
+    key, value = 'bar', { embedding: [ 0.5 ] * 1_024 }
+    cache[key] = value
+    expect {
+      expect(cache.clear_for_tags).to eq cache
+    }.to change {
+      cache.size
+    }.from(2).to(0)
+    expect(cache).not_to be_key 'foo'
+    expect(cache).not_to be_key 'bar'
+  end
+
   it 'can clear by source' do
     val1 = test_value.merge(source: 's1')
     val2 = test_value.merge(source: 's1')
@@ -159,6 +171,46 @@ describe Documentrix::Documents::SQLiteCache do
     expect(cache.key?('foo')).to be false
   end
 
+  it 'can clear by source and digest' do
+    allow(cache).to receive(:compute_file_digest).and_return('d1', 'd2', 'd3')
+    cache['foo'] = test_value.merge(source: 's1') # d1
+    cache['bar'] = test_value.merge(source: 's1') # d2
+    cache['baz'] = test_value.merge(source: 's1') # d3
+
+    # Clear those that match d1
+    expect {
+      cache.clear_by_source('s1', digest: 'd1')
+    }.to change { cache.size }.from(3).to(2)
+    expect(cache.key?('foo')).to be false
+    expect(cache.key?('bar')).to be true
+
+    # Clear those that do NOT match d2 (should clear baz)
+    expect {
+      cache.clear_by_source('s1', digest: 'd2', operator: '!=')
+    }.to change { cache.size }.from(2).to(1)
+    expect(cache.key?('baz')).to be false
+    expect(cache.key?('bar')).to be true
+  end
+
+  describe '#source_exist?' do
+    it 'returns true if source exists' do
+      cache['foo'] = test_value
+      expect(cache.source_exist?('for-test.txt')).to be true
+      expect(cache.source_exist?('non-existent')).to be false
+    end
+
+    it 'filters by digest' do
+      allow(cache).to receive(:compute_file_digest).and_return('d1', 'd2')
+      cache['foo'] = test_value.merge(source: 's1') # d1
+      cache['bar'] = test_value.merge(source: 's1') # d2
+
+      expect(cache.source_exist?('s1', digest: 'd1')).to be true
+      expect(cache.source_exist?('s1', digest: 'd3')).to be false
+      expect(cache.source_exist?('s1', digest: 'd1', operator: '!=')).to be true # bar exists
+      expect(cache.source_exist?('s1', digest: 'd2', operator: '!=')).to be true # foo exists
+    end
+  end
+
   it 'can return tags' do
     key, value = 'foo', { tags: %w[ foo ], embedding: [ 0.5 ] * 1_024 }
     cache[key] = value
@@ -169,6 +221,17 @@ describe Documentrix::Documents::SQLiteCache do
     expect(tags.to_a).to eq %w[ bar baz foo ]
   end
 
+  it 'can iterate over unique sources' do
+    val1 = test_value.merge(source: 's1')
+    val2 = test_value.merge(source: 's1')
+    val3 = test_value.merge(source: 's2')
+    cache['foo'] = val1
+    cache['bar'] = val2
+    cache['baz'] = val3
+
+    expect(cache.each_source.to_a).to match_array(['s1', 's2'])
+  end
+
   it 'can iterate over keys under a prefix' do
     cache['foo'] = test_value
     expect(cache.each.to_a).to eq [ [ 'test-foo', Documentrix::Documents::Record[test_value] ] ]
@@ -186,4 +249,67 @@ describe Documentrix::Documents::SQLiteCache do
       ["test2-bar", Documentrix::Documents::Record[test_value] ],
     ]
   end
+
+  describe '#find_records' do
+    let(:needle) { [ 0.5 ] * 1_024 }
+
+    it 'raises ArgumentError if needle length is incorrect' do
+      expect {
+        cache.find_records([ 0.1 ])
+      }.to raise_error(ArgumentError, /needle embedding length/)
+    end
+
+    it 'returns the most similar record' do
+      # Record 1: Exact match
+      val1 = test_value.merge(text: 'match', embedding: needle)
+      # Record 2: Different vector
+      val2 = test_value.merge(text: 'diff', embedding: [ 0.1 ] * 1_024)
+
+      cache['r1'] = val1
+      cache['r2'] = val2
+
+      results = cache.find_records(needle)
+
+      expect(results.size).to eq 2
+      expect(results.first.text).to eq 'match'
+      expect(results.first.similarity).to be_within(0.001).of(1.0)
+    end
+
+    it 'filters results by tags' do
+      val1 = test_value.merge(text: 'tagged', tags: %w[ a ], embedding: needle)
+      val2 = test_value.merge(text: 'untagged', tags: %w[ b ], embedding: needle)
+
+      cache['r1'] = val1
+      cache['r2'] = val2
+
+      expect(cache.find_records(needle, tags: %w[ a ]).map(&:text)).to eq %w[ tagged ]
+      expect(cache.find_records(needle, tags: %w[ b ]).map(&:text)).to eq %w[ untagged ]
+      expect(cache.find_records(needle, tags: %w[ c ]).size).to eq 0
+    end
+
+    it 'filters results by min_similarity' do
+      # Exact match
+      cache['r1'] = test_value.merge(text: 'match', embedding: needle)
+      # Very different vector
+      cache['r2'] = test_value.merge(text: 'diff', embedding: [ -0.5 ] * 1_024)
+
+      # Low threshold: both should appear
+      expect(cache.find_records(needle, min_similarity: -1).size).to eq 2
+
+      # High threshold: only match should appear
+      expect(cache.find_records(needle, min_similarity: 0.9).map(&:text)).to eq %w[ match ]
+    end
+
+    it 'limits results via max_records' do
+      3.times do |i|
+        cache["r#{i}"] = test_value.merge(text: "t#{i}", embedding: needle)
+      end
+
+      expect(cache.find_records(needle, max_records: 2).size).to eq 2
+    end
+
+    it 'returns empty array when no records match' do
+      expect(cache.find_records(needle)).to eq []
+    end
+  end
 end

data/spec/documentrix/documents/splitters/character_spec.rb

@@ -1,5 +1,3 @@
-require 'spec_helper'
-
 describe Documentrix::Documents::Splitters::Character do
   let :splitter do
     described_class.new chunk_size: 23, combining_string: ''
@@ -50,6 +48,26 @@ describe Documentrix::Documents::Splitters::Character do
     expect(result.to_a.join('')).to eq ?A * 25
   end
 
+  context 'with force' do
+    let :splitter do
+      described_class.new chunk_size: 23, combining_string: '', force: true
+    end
+
+    it 'can split with force' do
+      text = [ ?A * 10 ] * 10 * "\n"
+      result = splitter.split(text)
+      expect(result.count).to eq 10
+      expect(result.to_a.join('').count(?A)).to eq text.count(?A)
+    end
+
+    it 'can with force split2' do
+      text = ?A * 25
+      result = splitter.split(text)
+      expect(result.count).to eq 2
+      expect(result.to_a.join('')).to eq ?A * 25
+    end
+  end
+
   it 'can split sentences' do
     text     = "foo.foo. bar!bar! baz?baz? quux.\nquux."
     splitter = described_class.new(separator: /[.!?]\s*(?:\b|\z)/, chunk_size: 2)

data/spec/documentrix/documents/splitters/semantic_spec.rb

@@ -1,5 +1,3 @@
-require 'spec_helper'
-
 describe Documentrix::Documents::Splitters::Semantic do
   let :ollama do
     double('Ollama::Client')
@@ -29,11 +27,11 @@ describe Documentrix::Documents::Splitters::Semantic do
     expect(result.to_a.join('').count(?B)).to eq text.count(?B)
   end
 
-  it 'can split with breakpoint :percentile' do
-    described_class.new ollama:, model: 'mxbai-embed-large', chunk_size: 50
+  it 'can split with breakpoint :percentile, chunk_size 23' do
+    splitter = described_class.new ollama:, model: 'mxbai-embed-large', chunk_size: 23
     text = ([ "A" * 10 ] * 6 + [ "B" * 10 ] * 3 + [ "A" * 10 ] * 3) * ". "
     result = splitter.split(text, breakpoint: :percentile, percentile: 75)
-    expect(result.count).to eq 4
+    expect(result.count).to eq 11
     expect(result.to_a.join('').count(?A)).to eq text.count(?A)
     expect(result.to_a.join('').count(?B)).to eq text.count(?B)
   end
@@ -53,4 +51,18 @@ describe Documentrix::Documents::Splitters::Semantic do
     expect(result.to_a.join('').count(?A)).to eq text.count(?A)
     expect(result.to_a.join('').count(?B)).to eq text.count(?B)
   end
+
+  context 'with force' do
+    let :splitter do
+      described_class.new ollama:, model: 'mxbai-embed-large', chunk_size: 7, force: true
+    end
+
+    it 'can split with force' do
+      text = ([ "A" * 10 ] * 3 + [ "B" * 10 ] * 3 + [ "A" * 10 ] * 3) * ". "
+      result = splitter.split(text, breakpoint: :percentile, percentile: 75)
+      expect(result.count).to eq 18
+      expect(result.to_a.join('').count(?A)).to eq text.count(?A)
+      expect(result.to_a.join('').count(?B)).to eq text.count(?B)
+    end
+  end
 end

data/spec/documents_spec.rb

@@ -1,5 +1,3 @@
-require 'spec_helper'
-
 describe Documentrix::Documents do
   let :ollama do
     double('Ollama::Client')
@@ -168,7 +166,7 @@ describe Documentrix::Documents do
 
       expect(documents.size).to eq 3
 
-      documents.remove('source1')
+      documents.source_remove('source1')
 
       expect(documents.size).to eq 1
       expect(documents.exist?('baz')).to be true
@@ -211,4 +209,62 @@ describe Documentrix::Documents do
         to(:new_collection)
     end
   end
+
+  context 'source management' do
+    before do
+      allow(documents).to receive(:compute_file_digest).and_return('d1')
+      allow(documents.cache).to receive(:compute_file_digest).and_return('d1')
+
+      allow(ollama).to receive(:embed).and_return(double(embeddings: [[0.1]]))
+    end
+
+    it 'can check if a source exists' do
+      documents.add('foo', source: 's1')
+      expect(documents.source_exist?('s1')).to be true
+      expect(documents.source_exist?('s2')).to be false
+    end
+
+    it 'can determine if a source is modified' do
+      documents.add('foo', source: 's1')
+
+      # Case 1: Source is up to date
+      expect(documents.source_modified?('s1')).to be false
+
+      # Case 2: Source is missing
+      expect(documents.source_modified?('s2')).to be true
+
+      # Case 3: Digest changed - mock both to return the new digest
+      allow(documents).to receive(:compute_file_digest).with('s1').and_return('d2')
+      allow(documents.cache).to receive(:compute_file_digest).with('s1').and_return('d2')
+      expect(documents.source_modified?('s1')).to be true
+    end
+
+    it 'does not update the source if the digest matches' do
+      documents.add('foo', source: 's1')
+
+      expect(ollama).not_to receive(:embed)
+      documents.source_update(['foo'], source: 's1')
+      expect(documents.exist?('foo')).to be true
+    end
+
+    it 'updates the source if the digest has changed' do
+      documents.add('foo', source: 's1')
+
+      # Simulate a file change by updating the digest
+      allow(documents).to receive(:compute_file_digest).with('s1').and_return('d2')
+      allow(documents.cache).to receive(:compute_file_digest).with('s1').and_return('d2')
+
+      expect(ollama).to receive(:embed).once
+      documents.source_update(['bar'], source: 's1')
+
+      expect(documents.exist?('bar')).to be true
+      expect(documents.exist?('foo')).to be false
+    end
+
+    it 'updates the source if it is an URL' do
+      expect(ollama).to receive(:embed).once
+      documents.source_update('foo', source: 'https://www.example.com/s1')
+      expect(documents.exist?('foo')).to be true
+    end
+  end
 end

data/spec/utils/colorize_texts_spec.rb

@@ -1,5 +1,3 @@
-require 'spec_helper'
-
 describe Documentrix::Utils::ColorizeTexts do
   it 'colorizes texts' do
     ct = described_class.new(%w[ foo bar ])

data/spec/utils/digests_spec.rb

@@ -0,0 +1,97 @@
+require 'tempfile'
+
+describe Documentrix::Utils::Digests do
+  let(:test_class) do
+    Class.new do
+      include Documentrix::Utils::Digests
+    end
+  end
+
+  let(:subject) { test_class.new.expose }
+
+  describe '#compute_digest' do
+    it 'computes a valid SHA256 digest of a string' do
+      text = 'hello world'
+      expected = Digest::SHA256.hexdigest(text)
+      expect(subject.compute_digest(text)).to eq expected
+    end
+  end
+
+  describe '#compute_file_digest' do
+    it 'returns nil for an empty filename' do
+      expect(subject.compute_file_digest(nil)).to be_nil
+      expect(subject.compute_file_digest('')).to be_nil
+    end
+
+    it 'returns nil for an absolute URL' do
+      expect(subject.compute_file_digest('https://example.com/file.txt')).to be_nil
+    end
+
+    it 'returns nil for a non-existent file' do
+      expect(subject.compute_file_digest('/tmp/non_existent_file_12345')).to be_nil
+    end
+
+    it 'computes the digest of a local file' do
+      file = Tempfile.create('documentrix_test')
+      content = 'file content'
+      file.write(content)
+      file.close
+
+      expected = Digest::SHA256.hexdigest(content)
+      expect(subject.compute_file_digest(file.path)).to eq expected
+    end
+
+    context 'with caching' do
+      let(:file) { Tempfile.create('documentrix_cache_test') }
+      let(:content) { 'initial content' }
+
+      before do
+        file.write(content)
+        file.close
+        subject.file_digest_cache_clear
+      end
+
+      it 'returns the same digest on subsequent calls' do
+        digest1 = subject.compute_file_digest(file.path)
+        digest2 = subject.compute_file_digest(file.path)
+        expect(digest1).to eq digest2
+      end
+
+      it 'recomputes the digest when the file is modified' do
+        digest1 = subject.compute_file_digest(file.path)
+
+        # Update file content and force mtime change
+        File.write(file.path, 'updated content')
+        # Ensure mtime is actually different (some FS have low precision)
+        File.utime(Time.now + 1, Time.now + 1, file.path)
+
+        digest2 = subject.compute_file_digest(file.path)
+        expect(digest1).not_to eq digest2
+      end
+
+      it 'recomputes the digest after cache clear' do
+        digest1 = subject.compute_file_digest(file.path)
+        subject.file_digest_cache_clear
+
+        # Even though file hasn't changed, it should re-read and return same value
+        digest2 = subject.compute_file_digest(file.path)
+        expect(digest1).to eq digest2
+      end
+    end
+  end
+
+  describe '#file_digest_cache_clear' do
+    it 'clears the internal cache' do
+      file = Tempfile.create('documentrix_clear_test')
+      file.write('test')
+      file.close
+
+      subject.compute_file_digest(file.path)
+      subject.file_digest_cache_clear
+
+      # We can verify this indirectly by checking if the cache is empty
+      # or by the fact that it will re-compute in tests.
+      expect(subject).to respond_to(:file_digest_cache_clear)
+    end
+  end
+end

data/spec/utils/tags_spec.rb

@@ -1,5 +1,3 @@
-require 'spec_helper'
-
 describe Documentrix::Utils::Tags do
   it 'can be instantiated' do
     expect(described_class.new).to be_a described_class

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: documentrix
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Florian Frank
@@ -249,9 +249,11 @@ extra_rdoc_files:
 - lib/documentrix/documents/cache/redis_cache.rb
 - lib/documentrix/documents/cache/sqlite_cache.rb
 - lib/documentrix/documents/splitters/character.rb
+- lib/documentrix/documents/splitters/common.rb
 - lib/documentrix/documents/splitters/semantic.rb
 - lib/documentrix/utils.rb
 - lib/documentrix/utils/colorize_texts.rb
+- lib/documentrix/utils/digests.rb
 - lib/documentrix/utils/math.rb
 - lib/documentrix/utils/tags.rb
 - lib/documentrix/version.rb
@@ -274,9 +276,11 @@ files:
 - lib/documentrix/documents/cache/redis_cache.rb
 - lib/documentrix/documents/cache/sqlite_cache.rb
 - lib/documentrix/documents/splitters/character.rb
+- lib/documentrix/documents/splitters/common.rb
 - lib/documentrix/documents/splitters/semantic.rb
 - lib/documentrix/utils.rb
 - lib/documentrix/utils/colorize_texts.rb
+- lib/documentrix/utils/digests.rb
 - lib/documentrix/utils/math.rb
 - lib/documentrix/utils/tags.rb
 - lib/documentrix/version.rb
@@ -291,6 +295,7 @@ files:
 - spec/documents_spec.rb
 - spec/spec_helper.rb
 - spec/utils/colorize_texts_spec.rb
+- spec/utils/digests_spec.rb
 - spec/utils/tags_spec.rb
 homepage: https://github.com/flori/documentrix
 licenses:
@@ -327,4 +332,5 @@ test_files:
 - spec/documents_spec.rb
 - spec/spec_helper.rb
 - spec/utils/colorize_texts_spec.rb
+- spec/utils/digests_spec.rb
 - spec/utils/tags_spec.rb