documentrix 0.0.0

data/spec/documentrix/documents/cache/memory_cache_spec.rb

@@ -0,0 +1,98 @@
+require 'spec_helper'
+
+RSpec.describe Documentrix::Documents::MemoryCache do
+  let :prefix do
+    'test-'
+  end
+
+  let :cache do
+    described_class.new prefix:
+  end
+
+  it 'can be instantiated' do
+    expect(cache).to be_a described_class
+  end
+
+  it 'can get/set a key' do
+    key, value = 'foo', { test: true }
+    expect {
+      cache[key] = value
+    }.to change {
+      cache[key]
+    }.from(nil).to(value)
+  end
+
+  it 'can determine if key exists' do
+    key, value = 'foo', { test: true }
+    expect {
+      cache[key] = value
+    }.to change {
+      cache.key?(key)
+    }.from(false).to(true)
+  end
+
+  it 'can set key with different prefixes' do
+    key, value = 'foo', { test: true }
+    expect {
+      cache[key] = value
+    }.to change {
+      cache.size
+    }.from(0).to(1)
+    cache2 = cache.dup
+    cache2.prefix = 'test2-'
+    expect {
+      cache2[key] = value
+    }.to change {
+      cache2.size
+    }.from(0).to(1)
+    expect(cache.size).to eq 1
+    s = 0
+    cache.full_each { s += 1 }
+    expect(s).to eq 2
+  end
+
+  it 'can delete' do
+    key, value = 'foo', { test: true }
+    expect(cache.delete(key)).to be_falsy
+    cache[key] = value
+    expect {
+      expect(cache.delete(key)).to eq true
+    }.to change {
+      cache.key?(key)
+    }.from(true).to(false)
+      expect(cache.delete(key)).to eq false
+  end
+
+  it 'can iterate over keys, values' do
+    key, value = 'foo', { test: true }
+    cache[key] = value
+    cache.each do |k, v|
+      expect(k).to eq prefix + key
+      expect(v).to eq value
+    end
+  end
+
+  it 'returns size' do
+    key, value = 'foo', { test: true }
+    expect {
+      cache[key] = value
+    }.to change {
+      cache.size
+    }.from(0).to(1)
+  end
+
+  it 'can clear' do
+    key, value = 'foo', { test: true }
+    cache[key] = value
+    expect {
+      expect(cache.clear).to eq cache
+    }.to change {
+      cache.size
+    }.from(1).to(0)
+  end
+
+  it 'can iterate over keys under a prefix' do
+    cache['foo'] = 'bar'
+    expect(cache.to_a).to eq [ %W[ #{prefix}foo bar ] ]
+  end
+end

data/spec/documentrix/documents/cache/redis_backed_memory_cache_spec.rb

@@ -0,0 +1,121 @@
+require 'spec_helper'
+
+RSpec.describe Documentrix::Documents::RedisBackedMemoryCache do
+  let :prefix do
+    'test-'
+  end
+
+  let :cache do
+    described_class.new prefix: 'test-', url: 'something'
+  end
+
+  it 'raises ArgumentError if url is missing' do
+    expect {
+      described_class.new prefix:, url: nil
+    }.to raise_error ArgumentError
+  end
+
+  context 'test redis interactions' do
+    let :data do
+      cache.instance_eval { @data }
+    end
+
+    let :redis_cache do
+      cache.instance_eval { @redis_cache }
+    end
+
+    let :redis do
+      double('Redis')
+    end
+
+    before do
+      allow_any_instance_of(Documentrix::Documents::RedisCache).to\
+        receive(:redis).and_return(redis)
+      allow(redis).to receive(:scan_each)
+    end
+
+    it 'can be instantiated and initialized' do
+      expect(cache).to be_a described_class
+    end
+
+    it 'defaults to nil object_class' do
+      expect(cache.object_class).to be_nil
+    end
+
+    it 'can be configured with object_class' do
+      object_class = Class.new(JSON::GenericObject)
+      cache = described_class.new(prefix: 'test-', url: 'something', object_class:)
+      expect(cache.object_class).to eq object_class
+    end
+
+    it 'has Redis client' do
+      expect(cache.redis).to eq redis
+    end
+
+    it 'can get a key' do
+      key = 'foo'
+      expect(data).to receive(:[]).with('test-' + key).and_return 666
+      expect(cache[key]).to eq 666
+    end
+
+    it 'can set a value for a key' do
+      key, value = 'foo', { test: true }
+      expect(data).to receive(:[]=).with('test-' + key, { test: true }).and_call_original
+      expect(redis).to receive(:set).with('test-' + key, JSON(value))
+      cache[key] = value
+    end
+
+    it 'can determine if key exists' do
+      key = 'foo'
+      expect(data).to receive(:key?).with('test-' + key).and_return(false, true)
+      expect(cache.key?('foo')).to eq false
+      expect(cache.key?('foo')).to eq true
+    end
+
+    it 'can delete' do
+      key = 'foo'
+      expect(data).to receive(:delete).with('test-' + key).and_return 'bar'
+      expect(redis).to receive(:del).with('test-' + key).and_return 1
+      expect(cache.delete(key)).to eq true
+      expect(data).to receive(:delete).with('test-' + key).and_return nil
+      expect(redis).to receive(:del).with(prefix + key).and_return 0
+      expect(cache.delete(key)).to eq false
+    end
+
+    it 'can iterate over keys, values' do
+      key, value = 'foo', { 'test' => true }
+      expect(redis).to receive(:set).with('test-' + key, JSON(value))
+      cache[key] = value
+      cache.each do |k, v|
+        expect(k).to eq prefix + key
+        expect(v).to eq value
+      end
+    end
+
+    it 'returns size' do
+      expect(cache).to receive(:count).and_return 3
+      expect(cache.size).to eq 3
+    end
+
+    it 'can clear' do
+      expect(redis).to receive(:scan_each).with(match: 'test-*').and_yield(
+        'test-foo'
+      )
+      expect(redis).to receive(:del).with('test-foo')
+      expect(cache.clear).to eq cache
+    end
+
+    it 'can iterate over keys under a prefix' do
+      data['test-foo'] = 'bar'
+      expect(cache.to_a).to eq [ %w[ test-foo bar ] ]
+    end
+
+    it 'can compute prefix with pre' do
+      expect(cache.pre('foo')).to eq 'test-foo'
+    end
+
+    it 'can remove prefix with unpre' do
+      expect(cache.unpre('test-foo')).to eq 'foo'
+    end
+  end
+end

data/spec/documentrix/documents/cache/redis_cache_spec.rb

@@ -0,0 +1,123 @@
+require 'spec_helper'
+
+RSpec.describe Documentrix::Documents::RedisCache do
+  let :prefix do
+    'test-'
+  end
+
+  let :cache do
+    described_class.new prefix:, url: 'something'
+  end
+
+  it 'can be instantiated' do
+    expect(cache).to be_a described_class
+  end
+
+  it 'defaults to nil object_class' do
+    expect(cache.object_class).to be_nil
+  end
+
+  it 'can be configured with object_class' do
+    object_class = Class.new(JSON::GenericObject)
+    cache = described_class.new(prefix:, url: 'something', object_class:)
+    expect(cache.object_class).to eq object_class
+  end
+
+  it 'raises ArgumentError if url is missing' do
+    expect {
+      described_class.new prefix:, url: nil
+    }.to raise_error ArgumentError
+  end
+
+  context 'test redis interactions' do
+    let :redis do
+      double('Redis')
+    end
+
+    before do
+      allow_any_instance_of(described_class).to receive(:redis).and_return(redis)
+    end
+
+    it 'has Redis client' do
+      expect(cache.redis).to eq redis
+    end
+
+    it 'can get a key' do
+      key = 'foo'
+      expect(redis).to receive(:get).with(prefix + key).and_return '"some_json"'
+      expect(cache[key]).to eq 'some_json'
+    end
+
+    it 'can set a value for a key' do
+      key, value = 'foo', { test: true }
+      expect(redis).to receive(:set).with(prefix + key, JSON(value), ex: nil)
+      cache[key] = value
+    end
+
+    it 'can set a value for a key with ttl' do
+      cache = described_class.new prefix:, url: 'something', ex: 3_600
+      key, value = 'foo', { test: true }
+      expect(redis).to receive(:set).with(prefix + key, JSON(value), ex: 3_600)
+      cache[key] = value
+      expect(redis).to receive(:ttl).with(prefix + key).and_return 3_600
+      expect(cache.ttl(key)).to eq 3_600
+    end
+
+    it 'can determine if key exists' do
+      key = 'foo'
+      expect(redis).to receive(:exists?).with(prefix + key).and_return(false, true)
+      expect(cache.key?('foo')).to eq false
+      expect(cache.key?('foo')).to eq true
+    end
+
+    it 'can delete' do
+      key = 'foo'
+      expect(redis).to receive(:del).with(prefix + key).and_return 1
+      expect(cache.delete(key)).to eq true
+      expect(redis).to receive(:del).with(prefix + key).and_return 0
+      expect(cache.delete(key)).to eq false
+    end
+
+    it 'can iterate over keys, values' do
+      key, value = 'foo', { 'test' => true }
+      expect(redis).to receive(:set).with(prefix + key, JSON(value), ex: nil)
+      cache[key] = value
+      expect(redis).to receive(:scan_each).with(match: "#{prefix}*").
+        and_yield("#{prefix}foo")
+      expect(redis).to receive(:get).with(prefix + key).and_return(JSON(test: true))
+      cache.each do |k, v|
+        expect(k).to eq prefix + key
+        expect(v).to eq value
+      end
+    end
+
+    it 'returns size' do
+      expect(redis).to receive(:scan_each).with(match: "#{prefix}*").
+        and_yield("#{prefix}foo").
+        and_yield("#{prefix}bar").
+        and_yield("#{prefix}baz")
+      expect(cache.size).to eq 3
+    end
+
+    it 'can clear' do
+      expect(redis).to receive(:scan_each).with(match: 'test-*').and_yield(
+        'test-foo'
+      )
+      expect(redis).to receive(:del).with('test-foo')
+      expect(cache.clear).to eq cache
+    end
+
+    it 'can iterate over keys under a prefix' do
+      expect(redis).to receive(:scan_each).with(match: 'test-*')
+      cache.to_a
+    end
+
+    it 'can compute prefix with pre' do
+      expect(cache.pre('foo')).to eq 'test-foo'
+    end
+
+    it 'can remove prefix with unpre' do
+      expect(cache.unpre('test-foo')).to eq 'foo'
+    end
+  end
+end

data/spec/documentrix/documents/cache/sqlite_cache_spec.rb

@@ -0,0 +1,141 @@
+require 'spec_helper'
+
+RSpec.describe Documentrix::Documents::SQLiteCache do
+  let :prefix do
+    'test-'
+  end
+
+  let :test_value do
+    {
+      key:       'test',
+      text:      'test text',
+      norm:      0.5,
+      source:    'for-test.txt',
+      tags:      %w[ test ],
+      embedding: [ 0.5 ] * 1_024,
+    }
+  end
+
+  let :cache do
+    described_class.new prefix:
+  end
+
+  it 'can be instantiated' do
+    expect(cache).to be_a described_class
+  end
+
+  it 'defaults to :memory: mode' do
+    expect(cache.filename).to eq ':memory:'
+  end
+
+  it 'can be switchted to file mode' do
+    expect(SQLite3::Database).to receive(:new).with('foo.sqlite').
+      and_return(double.as_null_object)
+    cache = described_class.new prefix:, filename: 'foo.sqlite'
+    expect(cache.filename).to eq 'foo.sqlite'
+  end
+
+  it 'can get/set a key' do
+    key, value = 'foo', test_value
+    queried_value = nil
+    expect {
+      cache[key] = value
+    }.to change {
+      queried_value = cache[key]
+    }.from(nil).to(Documentrix::Documents::Record[value])
+    expect(queried_value.embedding).to eq [ 0.5 ] * 1_024
+  end
+
+  it 'can determine if key exists' do
+    key, value = 'foo', test_value
+    expect {
+      cache[key] = value
+    }.to change {
+      cache.key?(key)
+    }.from(false).to(true)
+  end
+
+  it 'can set key with different prefixes' do
+    key, value = 'foo', test_value
+    expect {
+      cache[key] = value
+    }.to change {
+      cache.size
+    }.from(0).to(1)
+    cache2 = cache.dup
+    cache2.prefix = 'test2-'
+    expect {
+      cache2[key] = value
+    }.to change {
+      cache2.size
+    }.from(0).to(1)
+    expect(cache.size).to eq 1
+    s = 0
+    cache.full_each { s += 1 }
+    expect(s).to eq 2
+  end
+
+  it 'can delete' do
+    key, value = 'foo', test_value
+    expect(cache.delete(key)).to be_falsy
+    cache[key] = value
+    expect {
+      expect(cache.delete(key)).to eq true
+    }.to change {
+      cache.key?(key)
+    }.from(true).to(false)
+  end
+
+  it 'returns size' do
+    key, value = 'foo', test_value
+    expect {
+      cache[key] = value
+    }.to change {
+      cache.size
+    }.from(0).to(1)
+  end
+
+  it 'can convert_to_vector' do
+    vector = [ 23.0, 666.0 ]
+    expect(cache.convert_to_vector(vector)).to eq vector
+  end
+
+  it 'can clear' do
+    key, value = 'foo', { embedding: [ 0.5 ] * 1_024 }
+    cache[key] = value
+    expect {
+      expect(cache.clear).to eq cache
+    }.to change {
+      cache.size
+    }.from(1).to(0)
+  end
+
+  it 'can clear for tags' do
+    key, value = 'foo', { tags: %w[ foo ], embedding: [ 0.5 ] * 1_024 }
+    cache[key] = value
+    key, value = 'bar', { embedding: [ 0.5 ] * 1_024 }
+    cache[key] = value
+    expect {
+      expect(cache.clear_for_tags(%w[ #foo ])).to eq cache
+    }.to change {
+      cache.size
+    }.from(2).to(1)
+    expect(cache).not_to be_key 'foo'
+    expect(cache).to be_key 'bar'
+  end
+
+  it 'can return tags' do
+    key, value = 'foo', { tags: %w[ foo ], embedding: [ 0.5 ] * 1_024 }
+    cache[key] = value
+    key, value = 'bar', { tags: %w[ bar baz ], embedding: [ 0.5 ] * 1_024 }
+    cache[key] = value
+    tags = cache.tags
+    expect(tags).to be_a Documentrix::Utils::Tags
+    expect(tags.to_a).to eq %w[ bar baz foo ]
+  end
+
+  it 'can iterate over keys under a prefix' do
+    cache['foo'] = test_value
+    expect(cache.to_a).to eq [ [ 'test-foo', Documentrix::Documents::Record[test_value] ] ]
+  end
+end

data/spec/documentrix/documents/splitters/character_spec.rb

@@ -0,0 +1,110 @@
+require 'spec_helper'
+
+RSpec.describe Documentrix::Documents::Splitters::Character do
+  let :splitter do
+    described_class.new chunk_size: 23, combining_string: ''
+  end
+
+  it 'can be instantiated' do
+    expect(splitter).to be_a described_class
+  end
+
+  it 'can split' do
+    text = [ ?A * 10 ] * 10 * "\n\n"
+    result = splitter.split(text)
+    expect(result.count).to eq 5
+    expect(result.to_a.join('')).to eq ?A * 100
+  end
+
+  it 'can split combining with separation' do
+    splitter = described_class.new chunk_size: 25, include_separator: false,
+      combining_string: ?X
+    text = [ ?A * 10 ] * 10 * "\n\n"
+    result = splitter.split(text)
+    expect(result.count).to eq 5
+    expect(result.to_a.join(?B)).to eq\
+      "AAAAAAAAAAXAAAAAAAAAAXBAAAAAAAAAAAAAAAAAAAAXBAAAAAAAAAAAAAAAAAAAAXB"\
+      "AAAAAAAAAAAAAAAAAAAAXBAAAAAAAAAAAAAAAAAAAAX"
+  end
+
+  it 'can split including separator' do
+    splitter = described_class.new chunk_size: 25, include_separator: true,
+      combining_string: ''
+    text = [ ?A * 10 ] * 10 * "\n\n"
+    result = splitter.split(text)
+    expect(result.count).to eq 5
+    expect(result.to_a.join('')).to eq text
+  end
+
+  it 'cannot split' do
+    text = [ ?A * 10 ] * 10 * "\n"
+    result = splitter.split(text)
+    expect(result.count).to eq 1
+    expect(result.to_a.join('').count(?A)).to eq text.count(?A)
+  end
+
+  it 'cannot split2' do
+    text = ?A * 25
+    result = splitter.split(text)
+    expect(result.count).to eq 1
+    expect(result.to_a.join('')).to eq ?A * 25
+  end
+
+  it 'can split sentences' do
+    text     = "foo.foo. bar!bar! baz?baz? quux.\nquux."
+    splitter = described_class.new(separator: /[.!?]\s*(?:\b|\z)/, chunk_size: 2)
+    result   = splitter.split(text)
+    expect(result.to_a).to eq %w[ foo foo bar bar baz baz quux quux ]
+  end
+end
+
+RSpec.describe Documentrix::Documents::Splitters::RecursiveCharacter do
+  let :splitter do
+    described_class.new chunk_size: 23, combining_string: ''
+  end
+
+  it 'can be instantiated' do
+    expect(splitter).to be_a described_class
+  end
+
+  it 'can split' do
+    text = [ ?A * 10 ] * 10 * "\n\n"
+    result = splitter.split(text)
+    expect(result.count).to eq 5
+    expect(result.to_a.join('')).to eq ?A * 100
+  end
+
+  it 'cannot split' do
+    splitter = described_class.new chunk_size: 23, include_separator: true,
+      separators: described_class::DEFAULT_SEPARATORS[0..-2]
+    text = ?A * 25
+    result = splitter.split(text)
+    expect(result.count).to eq 1
+    expect(result.to_a.join('')).to eq ?A * 25
+  end
+
+  it 'can split including separator' do
+    splitter = described_class.new chunk_size: 25, include_separator: true,
+      combining_string: ''
+    text = [ ?A * 10 ] * 10 * "\n\n"
+    result = splitter.split(text)
+    expect(result.count).to eq 5
+    expect(result.to_a.join('')).to eq text
+  end
+
+  it 'can split single newline as well' do
+    text = [ ?A * 10 ] * 10 * "\n"
+    result = splitter.split(text)
+    expect(result.count).to eq 5
+    expect(result.to_a.join('')).to eq ?A * 100
+  end
+
+  it 'can split single newline as well including separator' do
+    splitter = described_class.new chunk_size: 25, include_separator: true,
+      combining_string: ''
+    text = [ ?A * 10 ] * 10 * "\n"
+    result = splitter.split(text)
+    expect(result.count).to eq 5
+    expect(result.to_a.join('')).to eq text
+  end
+end

data/spec/documentrix/documents/splitters/semantic_spec.rb

@@ -0,0 +1,56 @@
+require 'spec_helper'
+
+RSpec.describe Documentrix::Documents::Splitters::Semantic do
+  let :ollama do
+    double('Ollama::Client')
+  end
+
+  let :splitter do
+    described_class.new ollama:, model: 'mxbai-embed-large'
+  end
+
+  let :embeddings do
+    JSON(File.read(asset('embeddings.json')))
+  end
+
+  it 'can be instantiated' do
+    expect(splitter).to be_a described_class
+  end
+
+  before do
+    allow(ollama).to receive(:embed).and_return(double(embeddings:))
+  end
+
+  it 'can split with breakpoint :percentile' do
+    text = ([ "A" * 10 ] * 3 + [ "B" * 10 ] * 3 + [ "A" * 10 ] * 3) * ". "
+    result = splitter.split(text, breakpoint: :percentile, percentile: 75)
+    expect(result.count).to eq 3
+    expect(result.to_a.join('').count(?A)).to eq text.count(?A)
+    expect(result.to_a.join('').count(?B)).to eq text.count(?B)
+  end
+
+  it 'can split with breakpoint :percentile' do
+    described_class.new ollama:, model: 'mxbai-embed-large', chunk_size: 50
+    text = ([ "A" * 10 ] * 6 + [ "B" * 10 ] * 3 + [ "A" * 10 ] * 3) * ". "
+    result = splitter.split(text, breakpoint: :percentile, percentile: 75)
+    expect(result.count).to eq 4
+    expect(result.to_a.join('').count(?A)).to eq text.count(?A)
+    expect(result.to_a.join('').count(?B)).to eq text.count(?B)
+  end
+
+  it 'can split with breakpoint :standard_deviation' do
+    text = ([ "A" * 10 ] * 3 + [ "B" * 10 ] * 3 + [ "A" * 10 ] * 3) * ". "
+    result = splitter.split(text, breakpoint: :standard_deviation, percentage: 100)
+    expect(result.count).to eq 3
+    expect(result.to_a.join('').count(?A)).to eq text.count(?A)
+    expect(result.to_a.join('').count(?B)).to eq text.count(?B)
+  end
+
+  it 'can split with breakpoint :interquartile' do
+    text = ([ "A" * 10 ] * 3 + [ "B" * 10 ] * 3 + [ "A" * 10 ] * 3) * ". "
+    result = splitter.split(text, breakpoint: :interquartile, percentage: 75)
+    expect(result.count).to eq 3
+    expect(result.to_a.join('').count(?A)).to eq text.count(?A)
+    expect(result.to_a.join('').count(?B)).to eq text.count(?B)
+  end
+end