documentrix 0.0.0

data/spec/documents_spec.rb

@@ -0,0 +1,174 @@
+require 'spec_helper'
+
+RSpec.describe Documentrix::Documents do
+  let :ollama do
+    double('Ollama::Client')
+  end
+
+  let :model do
+    'mxbai-embed-large'
+  end
+
+  let :documents do
+    described_class.new ollama:, model:
+  end
+
+  it 'can be instantiated' do
+    expect(documents).to be_a described_class
+  end
+
+  it 'no texts can be added to it' do
+    expect(documents.add([])).to eq documents
+  end
+
+  it 'texts can be added to it' do
+    expect(ollama).to receive(:embed).
+      with(model:, input: %w[ foo bar ], options: nil).
+      and_return(double(embeddings: [ [ 0.1 ], [ 0.2 ] ]))
+    expect(documents.add(%w[ foo bar ])).to eq documents
+    expect(documents.exist?('foo')).to eq true
+    expect(documents.exist?('bar')).to eq true
+    expect(documents['foo']).to be_a Documentrix::Documents::Record
+  end
+
+  it 'a text can be added to it' do
+    expect(ollama).to receive(:embed).
+      with(model:, input: %w[ foo ], options: nil).
+      and_return(double(embeddings: [ [ 0.1 ] ]))
+    expect(documents << 'foo').to eq documents
+    expect(documents.exist?('foo')).to eq true
+    expect(documents.exist?('bar')).to eq false
+    expect(documents['foo']).to be_a Documentrix::Documents::Record
+  end
+
+  it 'a text with can be added to it' do
+    expect(ollama).to receive(:embed).
+      with(model:, input: %w[ foo ], options: nil).
+      and_return(double(embeddings: [ [ 0.1 ] ]))
+    expect_any_instance_of(Documentrix::Utils::Tags).to\
+      receive(:add).with('bar.rb', source: '/foo/bar.rb')
+    expect(documents.add('foo', source: '/foo/bar.rb')).to eq documents
+    expect(documents.exist?('foo')).to eq true
+    expect(documents.exist?('bar')).to eq false
+    expect(documents['foo']).to be_a Documentrix::Documents::Record
+  end
+
+  it 'can find strings' do
+    expect(ollama).to receive(:embed).
+      with(model:, input: [ 'foo' ], options: nil).
+      and_return(double(embeddings: [ [ 0.1 ] ]))
+    expect(documents << 'foo').to eq documents
+    expect(ollama).to receive(:embed).
+      with(model:, input: 'foo', options: nil).
+      and_return(double(embeddings: [ [ 0.1 ] ]))
+    records = documents.find('foo')
+    expect(records).to eq [
+      Documentrix::Documents::Record[text: 'foo', embedding: [ 0.1 ], similarity: 1.0 ]
+    ]
+    expect(records[0].to_s).to eq '#<Documentrix::Documents::Record "foo" 1.0>'
+  end
+
+  it 'can find only tagged strings' do
+    expect(ollama).to receive(:embed).
+      with(model:, input: [ 'foo' ], options: nil).
+      and_return(double(embeddings: [ [ 0.1 ] ]))
+    expect(documents.add('foo', tags: %w[ test ])).to eq documents
+    expect(ollama).to receive(:embed).
+      with(model:, input: 'foo', options: nil).
+      and_return(double(embeddings: [ [ 0.1 ] ]))
+    records = documents.find('foo', tags: %w[ nix ])
+    expect(records).to eq []
+    expect(ollama).to receive(:embed).
+      with(model:, input: 'foo', options: nil).
+      and_return(double(embeddings: [ [ 0.1 ] ]))
+    records = documents.find('foo', tags: %w[ test ])
+    expect(records).to eq [
+      Documentrix::Documents::Record[text: 'foo', embedding: [ 0.1 ], similarity: 1.0 ]
+    ]
+    expect(records[0].to_s).to eq '#<Documentrix::Documents::Record "foo" #test 1.0>'
+  end
+
+  it 'can find strings conditionally' do
+    expect(ollama).to receive(:embed).
+      with(model:, input: [ 'foobar' ], options: nil).
+      and_return(double(embeddings: [ [ 0.01 ] ]))
+    expect(ollama).to receive(:embed).
+      with(model:, input: [ 'foo' ], options: nil).
+      and_return(double(embeddings: [ [ 0.1 ] ]))
+    expect(documents << 'foobar').to eq documents
+    expect(documents << 'foo').to eq documents
+    expect(ollama).to receive(:embed).at_least(:once).
+      with(model:, input: 'foo', options: nil).
+      and_return(double(embeddings: [ [ 0.1 ] ]))
+    records = documents.find_where('foo', text_count: 1)
+    expect(records).to eq [
+      Documentrix::Documents::Record[text: 'foo', embedding: [ 0.1 ], similarity: 1.0 ],
+    ]
+    records = documents.find_where('foo', text_size: 3)
+    expect(records).to eq [
+      Documentrix::Documents::Record[text: 'foo', embedding: [ 0.1 ], similarity: 1.0 ],
+    ]
+    records = documents.find_where('foo')
+    expect(records).to eq [
+      Documentrix::Documents::Record[text: 'foo', embedding: [ 0.1 ], similarity: 1.0 ],
+      Documentrix::Documents::Record[text: 'foobar', embedding: [ 0.1 ], similarity: 1.0 ],
+    ]
+  end
+
+
+  context 'it uses cache' do
+    before do
+      allow(ollama).to receive(:embed).
+        with(model:, input: %w[ foo ], options: nil).
+        and_return(double(embeddings: [ [ 0.1 ] ]))
+    end
+
+    it 'can delete texts' do
+      expect(documents << 'foo').to eq documents
+      expect {
+        documents.delete('foo')
+      }.to change { documents.exist?('foo') }.from(true).to(false)
+    end
+
+    it 'tracks size' do
+      expect {
+        expect(documents << 'foo').to eq documents
+      }.to change { documents.size }.from(0).to(1)
+    end
+
+    it 'can clear texts' do
+      expect(documents << 'foo').to eq documents
+      expect {
+        documents.clear
+      }.to change { documents.size }.from(1).to(0)
+    end
+
+    it 'can clear texts with tags' do
+      expect(ollama).to receive(:embed).
+        with(model:, input: %w[ bar ], options: nil).
+        and_return(double(embeddings: [ [ 0.1 ] ]))
+      expect(documents.add('foo', tags: %w[ test ])).to eq documents
+      expect(documents.add('bar', tags: %w[ test2 ])).to eq documents
+      expect(documents.tags.to_a).to eq %w[ test test2 ]
+      expect {
+        documents.clear tags: 'test'
+      }.to change { documents.size }.from(2).to(1)
+      expect {
+        documents.clear tags: :test2
+      }.to change { documents.size }.from(1).to(0)
+    end
+
+    it 'returns collections' do
+      expect(documents.collections).to eq [ :default ]
+    end
+
+    it 'can change collection' do
+      expect(documents.instance_eval { @cache }).to receive(:prefix=).
+        with(/#@collection/).and_call_original
+      expect { documents.collection = :new_collection }.
+        to change { documents.collection }.
+        from(:default).
+        to(:new_collection)
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,23 @@
+if ENV['START_SIMPLECOV'].to_i == 1
+  require 'simplecov'
+  SimpleCov.start do
+    add_filter "#{File.basename(File.dirname(__FILE__))}/"
+  end
+end
+require 'rspec'
+require 'tins/xt/expose'
+begin
+  require 'debug'
+rescue LoadError
+end
+require 'documentrix'
+
+def asset(name)
+  File.join(__dir__, 'assets', name)
+end
+
+RSpec.configure do |config|
+  config.before(:suite) do
+    infobar.show = nil
+  end
+end

data/spec/utils/colorize_texts_spec.rb

@@ -0,0 +1,13 @@
+require 'spec_helper'
+
+RSpec.describe Documentrix::Utils::ColorizeTexts do
+  it 'colorizes texts' do
+    ct = described_class.new(%w[ foo bar ])
+    colored   = ct.to_s
+    uncolored = Term::ANSIColor.uncolor(ct.to_s)
+    expect(colored.size).to be > uncolored.size
+    expect(uncolored).to eq(
+      "foo\n#3 \n\nbar\n#3 \n\n"
+    )
+  end
+end

data/spec/utils/tags_spec.rb

@@ -0,0 +1,53 @@
+require 'spec_helper'
+
+RSpec.describe Documentrix::Utils::Tags do
+  it 'can be instantiated' do
+    expect(described_class.new).to be_a described_class
+  end
+
+  it 'can contain unique tags and is sorted' do
+    tags = described_class.new(%w[ bar foo ])
+    expect(tags.to_a).to eq %w[ bar foo ]
+  end
+
+  it 'can contain unique tags with leading # characters and is sorted' do
+    tags = described_class.new(%w[ #bar ##foo ])
+    expect(tags.to_a).to eq %w[ bar foo ]
+  end
+
+  it 'tags can be added to it' do
+    tags = described_class.new([ 'foo' ])
+    tags.add 'bar'
+    expect(tags.to_a).to eq %w[ bar foo ]
+  end
+
+  it 'can increase in size' do
+    tags = described_class.new
+    expect { tags.add 'foo' }.to change { tags.size }.from(0).to(1)
+    expect { tags.add 'bar' }.to change { tags.size }.from(1).to(2)
+  end
+
+  it 'can be cleared' do
+    tags = described_class.new([ 'foo', 'bar' ])
+    expect { tags.clear }.to change { tags.size }.from(2).to(0)
+  end
+
+  it 'tags can be empty' do
+    tags = described_class.new([ 'foo' ])
+    expect { tags.clear }.to change { tags.empty? }.from(false).to(true)
+  end
+
+  it 'can be output nicely' do
+    expect(described_class.new(%w[ #foo bar ]).to_s).to eq '#bar #foo'
+  end
+
+  it 'can be output nicely with links to source' do
+    tags = described_class.new([ 'foo' ], source: 'https://foo.example.com')
+    tags.add 'bar', source: '/path/to/bar.html'
+    expect(tags.to_a).to eq %w[ bar foo ]
+    tags.all? { expect(_1).to be_a(Documentrix::Utils::Tags::Tag) }
+    expect(tags.to_s).to eq(
+      "\e]8;;file:///path/to/bar.html\e\\#bar\e]8;;\e\\ \e]8;;https://foo.example.com\e\\#foo\e]8;;\e\\"
+    )
+  end
+end

metadata

@@ -0,0 +1,329 @@
+--- !ruby/object:Gem::Specification
+name: documentrix
+version: !ruby/object:Gem::Version
+  version: 0.0.0
+platform: ruby
+authors:
+- Florian Frank
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2024-12-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: gem_hadar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.19'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.19'
+- !ruby/object:Gem::Dependency
+  name: all_images
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+- !ruby/object:Gem::Dependency
+  name: kramdown
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: debug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: infobar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: tins
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.34'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.34'
+- !ruby/object:Gem::Dependency
+  name: sqlite-vec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.1
+- !ruby/object:Gem::Dependency
+  name: kramdown-ansi
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.1
+- !ruby/object:Gem::Dependency
+  name: numo-narray
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: more_math
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+description: |
+  The Ruby library, Documentrix, is designed to provide a way to build and
+  query vector databases for applications in natural language processing
+  (NLP) and large language models (LLMs). It allows users to store and
+  retrieve dense vector embeddings for text strings.
+email: flori@ping.de
+executables: []
+extensions: []
+extra_rdoc_files:
+- README.md
+- lib/documentrix.rb
+- lib/documentrix/documents.rb
+- lib/documentrix/documents/cache/common.rb
+- lib/documentrix/documents/cache/memory_cache.rb
+- lib/documentrix/documents/cache/records.rb
+- lib/documentrix/documents/cache/redis_backed_memory_cache.rb
+- lib/documentrix/documents/cache/redis_cache.rb
+- lib/documentrix/documents/cache/sqlite_cache.rb
+- lib/documentrix/documents/splitters/character.rb
+- lib/documentrix/documents/splitters/semantic.rb
+- lib/documentrix/utils.rb
+- lib/documentrix/utils/colorize_texts.rb
+- lib/documentrix/utils/math.rb
+- lib/documentrix/utils/tags.rb
+- lib/documentrix/version.rb
+files:
+- ".yardopts"
+- Gemfile
+- README.md
+- Rakefile
+- documentrix.gemspec
+- lib/documentrix.rb
+- lib/documentrix/documents.rb
+- lib/documentrix/documents/cache/common.rb
+- lib/documentrix/documents/cache/memory_cache.rb
+- lib/documentrix/documents/cache/records.rb
+- lib/documentrix/documents/cache/redis_backed_memory_cache.rb
+- lib/documentrix/documents/cache/redis_cache.rb
+- lib/documentrix/documents/cache/sqlite_cache.rb
+- lib/documentrix/documents/splitters/character.rb
+- lib/documentrix/documents/splitters/semantic.rb
+- lib/documentrix/utils.rb
+- lib/documentrix/utils/colorize_texts.rb
+- lib/documentrix/utils/math.rb
+- lib/documentrix/utils/tags.rb
+- lib/documentrix/version.rb
+- spec/assets/embeddings.json
+- spec/documentrix/documents/cache/memory_cache_spec.rb
+- spec/documentrix/documents/cache/redis_backed_memory_cache_spec.rb
+- spec/documentrix/documents/cache/redis_cache_spec.rb
+- spec/documentrix/documents/cache/sqlite_cache_spec.rb
+- spec/documentrix/documents/splitters/character_spec.rb
+- spec/documentrix/documents/splitters/semantic_spec.rb
+- spec/documents_spec.rb
+- spec/spec_helper.rb
+- spec/utils/colorize_texts_spec.rb
+- spec/utils/tags_spec.rb
+homepage: https://github.com/flori/documentrix
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options:
+- "--title"
+- Documentrix - "Ruby library for embedding vector database
+- "--main"
+- README.md
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - "~>"
+    - !ruby/object:Gem::Version
+      version: '3.1'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.23
+signing_key:
+specification_version: 4
+summary: "\"Ruby library for embedding vector database"
+test_files:
+- spec/documentrix/documents/cache/memory_cache_spec.rb
+- spec/documentrix/documents/cache/redis_backed_memory_cache_spec.rb
+- spec/documentrix/documents/cache/redis_cache_spec.rb
+- spec/documentrix/documents/cache/sqlite_cache_spec.rb
+- spec/documentrix/documents/splitters/character_spec.rb
+- spec/documentrix/documents/splitters/semantic_spec.rb
+- spec/documents_spec.rb
+- spec/spec_helper.rb
+- spec/utils/colorize_texts_spec.rb
+- spec/utils/tags_spec.rb