ollama-ruby 0.12.1 → 0.13.0

data/spec/ollama/documents/cache/redis_cache_spec.rb

@@ -1,121 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe Ollama::Documents::RedisCache do
-  let :prefix do
-    'test-'
-  end
-
-  let :cache do
-    described_class.new prefix:, url: 'something'
-  end
-
-  it 'can be instantiated' do
-    expect(cache).to be_a described_class
-  end
-
-  it 'defaults to nil object_class' do
-    expect(cache.object_class).to be_nil
-  end
-
-  it 'can be configured with object_class' do
-    object_class = Class.new(JSON::GenericObject)
-    cache = described_class.new(prefix:, url: 'something', object_class:)
-    expect(cache.object_class).to eq object_class
-  end
-
-  it 'raises ArgumentError if url is missing' do
-    expect {
-      described_class.new prefix:, url: nil
-    }.to raise_error ArgumentError
-  end
-
-  context 'test redis interactions' do
-    let :redis do
-      double('Redis')
-    end
-
-    before do
-      allow_any_instance_of(described_class).to receive(:redis).and_return(redis)
-    end
-
-    it 'has Redis client' do
-      expect(cache.redis).to eq redis
-    end
-
-    it 'can get a key' do
-      key = 'foo'
-      expect(redis).to receive(:get).with(prefix + key).and_return '"some_json"'
-      expect(cache[key]).to eq 'some_json'
-    end
-
-    it 'can set a value for a key' do
-      key, value = 'foo', { test: true }
-      expect(redis).to receive(:set).with(prefix + key, JSON(value), ex: nil)
-      cache[key] = value
-    end
-
-    it 'can set a value for a key with ttl' do
-      cache = described_class.new prefix:, url: 'something', ex: 3_600
-      key, value = 'foo', { test: true }
-      expect(redis).to receive(:set).with(prefix + key, JSON(value), ex: 3_600)
-      cache[key] = value
-      expect(redis).to receive(:ttl).with(prefix + key).and_return 3_600
-      expect(cache.ttl(key)).to eq 3_600
-    end
-
-    it 'can determine if key exists' do
-      key = 'foo'
-      expect(redis).to receive(:exists?).with(prefix + key).and_return(false, true)
-      expect(cache.key?('foo')).to eq false
-      expect(cache.key?('foo')).to eq true
-    end
-
-    it 'can delete' do
-      key = 'foo'
-      expect(redis).to receive(:del).with(prefix + key)
-      cache.delete(key)
-    end
-
-    it 'can iterate over keys, values' do
-      key, value = 'foo', { 'test' => true }
-      expect(redis).to receive(:set).with(prefix + key, JSON(value), ex: nil)
-      cache[key] = value
-      expect(redis).to receive(:scan_each).with(match: "#{prefix}*").
-        and_yield("#{prefix}foo")
-      expect(redis).to receive(:get).with(prefix + key).and_return(JSON(test: true))
-      cache.each do |k, v|
-        expect(k).to eq prefix + key
-        expect(v).to eq value
-      end
-    end
-
-    it 'returns size' do
-      expect(redis).to receive(:scan_each).with(match: "#{prefix}*").
-        and_yield("#{prefix}foo").
-        and_yield("#{prefix}bar").
-        and_yield("#{prefix}baz")
-      expect(cache.size).to eq 3
-    end
-
-    it 'can clear' do
-      expect(redis).to receive(:scan_each).with(match: 'test-*').and_yield(
-        'test-foo'
-      )
-      expect(redis).to receive(:del).with('test-foo')
-      expect(cache.clear).to eq cache
-    end
-
-    it 'can iterate over keys under a prefix' do
-      expect(redis).to receive(:scan_each).with(match: 'test-*')
-      cache.to_a
-    end
-
-    it 'can compute prefix with pre' do
-      expect(cache.pre('foo')).to eq 'test-foo'
-    end
-
-    it 'can remove prefix with unpre' do
-      expect(cache.unpre('test-foo')).to eq 'foo'
-    end
-  end
-end

data/spec/ollama/documents/cache/sqlite_cache_spec.rb

@@ -1,141 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe Ollama::Documents::SQLiteCache do
-  let :prefix do
-    'test-'
-  end
-
-  let :test_value do
-    {
-      key:       'test',
-      text:      'test text',
-      norm:      0.5,
-      source:    'for-test.txt',
-      tags:      %w[ test ],
-      embedding: [ 0.5 ] * 1_024,
-    }
-  end
-
-  let :cache do
-    described_class.new prefix:
-  end
-
-  it 'can be instantiated' do
-    expect(cache).to be_a described_class
-  end
-
-  it 'defaults to :memory: mode' do
-    expect(cache.filename).to eq ':memory:'
-  end
-
-  it 'can be switchted to file mode' do
-    expect(SQLite3::Database).to receive(:new).with('foo.sqlite').
-      and_return(double.as_null_object)
-    cache = described_class.new prefix:, filename: 'foo.sqlite'
-    expect(cache.filename).to eq 'foo.sqlite'
-  end
-
-  it 'can get/set a key' do
-    key, value = 'foo', test_value
-    queried_value = nil
-    expect {
-      cache[key] = value
-    }.to change {
-      queried_value = cache[key]
-    }.from(nil).to(Ollama::Documents::Record[value])
-    expect(queried_value.embedding).to eq [ 0.5 ] * 1_024
-  end
-
-  it 'can determine if key exists' do
-    key, value = 'foo', test_value
-    expect {
-      cache[key] = value
-    }.to change {
-      cache.key?(key)
-    }.from(false).to(true)
-  end
-
-  it 'can set key with different prefixes' do
-    key, value = 'foo', test_value
-    expect {
-      cache[key] = value
-    }.to change {
-      cache.size
-    }.from(0).to(1)
-    cache2 = cache.dup
-    cache2.prefix = 'test2-'
-    expect {
-      cache2[key] = value
-    }.to change {
-      cache2.size
-    }.from(0).to(1)
-    expect(cache.size).to eq 1
-    s = 0
-    cache.full_each { s += 1 }
-    expect(s).to eq 2
-  end
-
-  it 'can delete' do
-    key, value = 'foo', test_value
-    expect(cache.delete(key)).to be_falsy
-    cache[key] = value
-    expect {
-      expect(cache.delete(key)).to be_truthy
-    }.to change {
-      cache.key?(key)
-    }.from(true).to(false)
-  end
-
-  it 'returns size' do
-    key, value = 'foo', test_value
-    expect {
-      cache[key] = value
-    }.to change {
-      cache.size
-    }.from(0).to(1)
-  end
-
-  it 'can convert_to_vector' do
-    vector = [ 23.0, 666.0 ]
-    expect(cache.convert_to_vector(vector)).to eq vector
-  end
-
-  it 'can clear' do
-    key, value = 'foo', { embedding: [ 0.5 ] * 1_024 }
-    cache[key] = value
-    expect {
-      expect(cache.clear).to eq cache
-    }.to change {
-      cache.size
-    }.from(1).to(0)
-  end
-
-  it 'can clear for tags' do
-    key, value = 'foo', { tags: %w[ foo ], embedding: [ 0.5 ] * 1_024 }
-    cache[key] = value
-    key, value = 'bar', { embedding: [ 0.5 ] * 1_024 }
-    cache[key] = value
-    expect {
-      expect(cache.clear_for_tags(%w[ #foo ])).to eq cache
-    }.to change {
-      cache.size
-    }.from(2).to(1)
-    expect(cache).not_to be_key 'foo'
-    expect(cache).to be_key 'bar'
-  end
-
-  it 'can return tags' do
-    key, value = 'foo', { tags: %w[ foo ], embedding: [ 0.5 ] * 1_024 }
-    cache[key] = value
-    key, value = 'bar', { tags: %w[ bar baz ], embedding: [ 0.5 ] * 1_024 }
-    cache[key] = value
-    tags = cache.tags
-    expect(tags).to be_a Ollama::Utils::Tags
-    expect(tags.to_a).to eq %w[ bar baz foo ]
-  end
-
-  it 'can iterate over keys under a prefix' do
-    cache['foo'] = test_value
-    expect(cache.to_a).to eq [ [ 'test-foo', Ollama::Documents::Record[test_value] ] ]
-  end
-end

data/spec/ollama/documents/splitters/character_spec.rb

@@ -1,110 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe Ollama::Documents::Splitters::Character do
-  let :splitter do
-    described_class.new chunk_size: 23, combining_string: ''
-  end
-
-  it 'can be instantiated' do
-    expect(splitter).to be_a described_class
-  end
-
-  it 'can split' do
-    text = [ ?A * 10 ] * 10 * "\n\n"
-    result = splitter.split(text)
-    expect(result.count).to eq 5
-    expect(result.to_a.join('')).to eq ?A * 100
-  end
-
-  it 'can split combining with separation' do
-    splitter = described_class.new chunk_size: 25, include_separator: false,
-      combining_string: ?X
-    text = [ ?A * 10 ] * 10 * "\n\n"
-    result = splitter.split(text)
-    expect(result.count).to eq 5
-    expect(result.to_a.join(?B)).to eq\
-      "AAAAAAAAAAXAAAAAAAAAAXBAAAAAAAAAAAAAAAAAAAAXBAAAAAAAAAAAAAAAAAAAAXB"\
-      "AAAAAAAAAAAAAAAAAAAAXBAAAAAAAAAAAAAAAAAAAAX"
-  end
-
-  it 'can split including separator' do
-    splitter = described_class.new chunk_size: 25, include_separator: true,
-      combining_string: ''
-    text = [ ?A * 10 ] * 10 * "\n\n"
-    result = splitter.split(text)
-    expect(result.count).to eq 5
-    expect(result.to_a.join('')).to eq text
-  end
-
-  it 'cannot split' do
-    text = [ ?A * 10 ] * 10 * "\n"
-    result = splitter.split(text)
-    expect(result.count).to eq 1
-    expect(result.to_a.join('').count(?A)).to eq text.count(?A)
-  end
-
-  it 'cannot split2' do
-    text = ?A * 25
-    result = splitter.split(text)
-    expect(result.count).to eq 1
-    expect(result.to_a.join('')).to eq ?A * 25
-  end
-
-  it 'can split sentences' do
-    text     = "foo.foo. bar!bar! baz?baz? quux.\nquux."
-    splitter = described_class.new(separator: /[.!?]\s*(?:\b|\z)/, chunk_size: 2)
-    result   = splitter.split(text)
-    expect(result.to_a).to eq %w[ foo foo bar bar baz baz quux quux ]
-  end
-end
-
-RSpec.describe Ollama::Documents::Splitters::RecursiveCharacter do
-  let :splitter do
-    described_class.new chunk_size: 23, combining_string: ''
-  end
-
-  it 'can be instantiated' do
-    expect(splitter).to be_a described_class
-  end
-
-  it 'can split' do
-    text = [ ?A * 10 ] * 10 * "\n\n"
-    result = splitter.split(text)
-    expect(result.count).to eq 5
-    expect(result.to_a.join('')).to eq ?A * 100
-  end
-
-  it 'cannot split' do
-    splitter = described_class.new chunk_size: 23, include_separator: true,
-      separators: described_class::DEFAULT_SEPARATORS[0..-2]
-    text = ?A * 25
-    result = splitter.split(text)
-    expect(result.count).to eq 1
-    expect(result.to_a.join('')).to eq ?A * 25
-  end
-
-  it 'can split including separator' do
-    splitter = described_class.new chunk_size: 25, include_separator: true,
-      combining_string: ''
-    text = [ ?A * 10 ] * 10 * "\n\n"
-    result = splitter.split(text)
-    expect(result.count).to eq 5
-    expect(result.to_a.join('')).to eq text
-  end
-
-  it 'can split single newline as well' do
-    text = [ ?A * 10 ] * 10 * "\n"
-    result = splitter.split(text)
-    expect(result.count).to eq 5
-    expect(result.to_a.join('')).to eq ?A * 100
-  end
-
-  it 'can split single newline as well including separator' do
-    splitter = described_class.new chunk_size: 25, include_separator: true,
-      combining_string: ''
-    text = [ ?A * 10 ] * 10 * "\n"
-    result = splitter.split(text)
-    expect(result.count).to eq 5
-    expect(result.to_a.join('')).to eq text
-  end
-end

data/spec/ollama/documents/splitters/semantic_spec.rb

@@ -1,56 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe Ollama::Documents::Splitters::Semantic do
-  let :ollama do
-    double('Ollama::Client')
-  end
-
-  let :splitter do
-    described_class.new ollama:, model: 'mxbai-embed-large'
-  end
-
-  let :embeddings do
-    JSON(File.read(asset('embeddings.json')))
-  end
-
-  it 'can be instantiated' do
-    expect(splitter).to be_a described_class
-  end
-
-  before do
-    allow(ollama).to receive(:embed).and_return(double(embeddings:))
-  end
-
-  it 'can split with breakpoint :percentile' do
-    text = ([ "A" * 10 ] * 3 + [ "B" * 10 ] * 3 + [ "A" * 10 ] * 3) * ". "
-    result = splitter.split(text, breakpoint: :percentile, percentile: 75)
-    expect(result.count).to eq 3
-    expect(result.to_a.join('').count(?A)).to eq text.count(?A)
-    expect(result.to_a.join('').count(?B)).to eq text.count(?B)
-  end
-
-  it 'can split with breakpoint :percentile' do
-    described_class.new ollama:, model: 'mxbai-embed-large', chunk_size: 50
-    text = ([ "A" * 10 ] * 6 + [ "B" * 10 ] * 3 + [ "A" * 10 ] * 3) * ". "
-    result = splitter.split(text, breakpoint: :percentile, percentile: 75)
-    expect(result.count).to eq 4
-    expect(result.to_a.join('').count(?A)).to eq text.count(?A)
-    expect(result.to_a.join('').count(?B)).to eq text.count(?B)
-  end
-
-  it 'can split with breakpoint :standard_deviation' do
-    text = ([ "A" * 10 ] * 3 + [ "B" * 10 ] * 3 + [ "A" * 10 ] * 3) * ". "
-    result = splitter.split(text, breakpoint: :standard_deviation, percentage: 100)
-    expect(result.count).to eq 3
-    expect(result.to_a.join('').count(?A)).to eq text.count(?A)
-    expect(result.to_a.join('').count(?B)).to eq text.count(?B)
-  end
-
-  it 'can split with breakpoint :interquartile' do
-    text = ([ "A" * 10 ] * 3 + [ "B" * 10 ] * 3 + [ "A" * 10 ] * 3) * ". "
-    result = splitter.split(text, breakpoint: :interquartile, percentage: 75)
-    expect(result.count).to eq 3
-    expect(result.to_a.join('').count(?A)).to eq text.count(?A)
-    expect(result.to_a.join('').count(?B)).to eq text.count(?B)
-  end
-end

data/spec/ollama/documents_spec.rb

@@ -1,162 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe Ollama::Documents do
-  let :ollama do
-    double('Ollama::Client')
-  end
-
-  let :model do
-    'mxbai-embed-large'
-  end
-
-  let :documents do
-    described_class.new ollama:, model:
-  end
-
-  it 'can be instantiated' do
-    expect(documents).to be_a described_class
-  end
-
-  it 'no texts can be added to it' do
-    expect(documents.add([])).to eq documents
-  end
-
-  it 'texts can be added to it' do
-    expect(ollama).to receive(:embed).
-      with(model:, input: %w[ foo bar ], options: nil).
-      and_return(double(embeddings: [ [ 0.1 ], [ 0.2 ] ]))
-    expect(documents.add(%w[ foo bar ])).to eq documents
-    expect(documents.exist?('foo')).to eq true
-    expect(documents.exist?('bar')).to eq true
-    expect(documents['foo']).to be_a Ollama::Documents::Record
-  end
-
-  it 'a text can be added to it' do
-    expect(ollama).to receive(:embed).
-      with(model:, input: %w[ foo ], options: nil).
-      and_return(double(embeddings: [ [ 0.1 ] ]))
-    expect(documents << 'foo').to eq documents
-    expect(documents.exist?('foo')).to eq true
-    expect(documents.exist?('bar')).to eq false
-    expect(documents['foo']).to be_a Ollama::Documents::Record
-  end
-
-  it 'can find strings' do
-    expect(ollama).to receive(:embed).
-      with(model:, input: [ 'foo' ], options: nil).
-      and_return(double(embeddings: [ [ 0.1 ] ]))
-    expect(documents << 'foo').to eq documents
-    expect(ollama).to receive(:embed).
-      with(model:, input: 'foo', options: nil).
-      and_return(double(embeddings: [ [ 0.1 ] ]))
-    records = documents.find('foo')
-    expect(records).to eq [
-      Ollama::Documents::Record[text: 'foo', embedding: [ 0.1 ], similarity: 1.0 ]
-    ]
-    expect(records[0].to_s).to eq '#<Ollama::Documents::Record "foo" 1.0>'
-  end
-
-  it 'can find only tagged strings' do
-    expect(ollama).to receive(:embed).
-      with(model:, input: [ 'foo' ], options: nil).
-      and_return(double(embeddings: [ [ 0.1 ] ]))
-    expect(documents.add('foo', tags: %w[ test ])).to eq documents
-    expect(ollama).to receive(:embed).
-      with(model:, input: 'foo', options: nil).
-      and_return(double(embeddings: [ [ 0.1 ] ]))
-    records = documents.find('foo', tags: %w[ nix ])
-    expect(records).to eq []
-    expect(ollama).to receive(:embed).
-      with(model:, input: 'foo', options: nil).
-      and_return(double(embeddings: [ [ 0.1 ] ]))
-    records = documents.find('foo', tags: %w[ test ])
-    expect(records).to eq [
-      Ollama::Documents::Record[text: 'foo', embedding: [ 0.1 ], similarity: 1.0 ]
-    ]
-    expect(records[0].to_s).to eq '#<Ollama::Documents::Record "foo" #test 1.0>'
-  end
-
-  it 'can find strings conditionally' do
-    expect(ollama).to receive(:embed).
-      with(model:, input: [ 'foobar' ], options: nil).
-      and_return(double(embeddings: [ [ 0.01 ] ]))
-    expect(ollama).to receive(:embed).
-      with(model:, input: [ 'foo' ], options: nil).
-      and_return(double(embeddings: [ [ 0.1 ] ]))
-    expect(documents << 'foobar').to eq documents
-    expect(documents << 'foo').to eq documents
-    expect(ollama).to receive(:embed).at_least(:once).
-      with(model:, input: 'foo', options: nil).
-      and_return(double(embeddings: [ [ 0.1 ] ]))
-    records = documents.find_where('foo', text_count: 1)
-    expect(records).to eq [
-      Ollama::Documents::Record[text: 'foo', embedding: [ 0.1 ], similarity: 1.0 ],
-    ]
-    records = documents.find_where('foo', text_size: 3)
-    expect(records).to eq [
-      Ollama::Documents::Record[text: 'foo', embedding: [ 0.1 ], similarity: 1.0 ],
-    ]
-    records = documents.find_where('foo')
-    expect(records).to eq [
-      Ollama::Documents::Record[text: 'foo', embedding: [ 0.1 ], similarity: 1.0 ],
-      Ollama::Documents::Record[text: 'foobar', embedding: [ 0.1 ], similarity: 1.0 ],
-    ]
-  end
-
-
-  context 'it uses cache' do
-    before do
-      allow(ollama).to receive(:embed).
-        with(model:, input: %w[ foo ], options: nil).
-        and_return(double(embeddings: [ [ 0.1 ] ]))
-    end
-
-    it 'can delete texts' do
-      expect(documents << 'foo').to eq documents
-      expect {
-        documents.delete('foo')
-      }.to change { documents.exist?('foo') }.from(true).to(false)
-    end
-
-    it 'tracks size' do
-      expect {
-        expect(documents << 'foo').to eq documents
-      }.to change { documents.size }.from(0).to(1)
-    end
-
-    it 'can clear texts' do
-      expect(documents << 'foo').to eq documents
-      expect {
-        documents.clear
-      }.to change { documents.size }.from(1).to(0)
-    end
-
-    it 'can clear texts with tags' do
-      expect(ollama).to receive(:embed).
-        with(model:, input: %w[ bar ], options: nil).
-        and_return(double(embeddings: [ [ 0.1 ] ]))
-      expect(documents.add('foo', tags: %w[ test ])).to eq documents
-      expect(documents.add('bar', tags: %w[ test2 ])).to eq documents
-      expect(documents.tags.to_a).to eq %w[ test test2 ]
-      expect {
-        documents.clear tags: 'test'
-      }.to change { documents.size }.from(2).to(1)
-      expect {
-        documents.clear tags: :test2
-      }.to change { documents.size }.from(1).to(0)
-    end
-
-    it 'returns collections' do
-      expect(documents.collections).to eq [ :default ]
-    end
-
-    it 'can change collection' do
-      expect(documents.instance_eval { @cache }).to receive(:prefix=).
-        with(/#@collection/).and_call_original
-      expect { documents.collection = :new_collection }.
-        to change { documents.collection }.
-        from(:default).
-        to(:new_collection)
-    end
-  end
-end

data/spec/ollama/utils/colorize_texts_spec.rb

@@ -1,13 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe Ollama::Utils::ColorizeTexts do
-  it 'colorizes texts' do
-    ct = described_class.new(%w[ foo bar ])
-    colored   = ct.to_s
-    uncolored = Term::ANSIColor.uncolor(ct.to_s)
-    expect(colored.size).to be > uncolored.size
-    expect(uncolored).to eq(
-      "foo\n#3 \n\nbar\n#3 \n\n"
-    )
-  end
-end

data/spec/ollama/utils/tags_spec.rb

@@ -1,53 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe Ollama::Utils::Tags do
-  it 'can be instantiated' do
-    expect(described_class.new).to be_a described_class
-  end
-
-  it 'can contain unique tags and is sorted' do
-    tags = described_class.new(%w[ bar foo ])
-    expect(tags.to_a).to eq %w[ bar foo ]
-  end
-
-  it 'can contain unique tags with leading # characters and is sorted' do
-    tags = described_class.new(%w[ #bar ##foo ])
-    expect(tags.to_a).to eq %w[ bar foo ]
-  end
-
-  it 'tags can be added to it' do
-    tags = described_class.new([ 'foo' ])
-    tags.add 'bar'
-    expect(tags.to_a).to eq %w[ bar foo ]
-  end
-
-  it 'can increase in size' do
-    tags = described_class.new
-    expect { tags.add 'foo' }.to change { tags.size }.from(0).to(1)
-    expect { tags.add 'bar' }.to change { tags.size }.from(1).to(2)
-  end
-
-  it 'can be cleared' do
-    tags = described_class.new([ 'foo', 'bar' ])
-    expect { tags.clear }.to change { tags.size }.from(2).to(0)
-  end
-
-  it 'tags can be empty' do
-    tags = described_class.new([ 'foo' ])
-    expect { tags.clear }.to change { tags.empty? }.from(false).to(true)
-  end
-
-  it 'can be output nicely' do
-    expect(described_class.new(%w[ #foo bar ]).to_s).to eq '#bar #foo'
-  end
-
-  it 'can be output nicely with links to source' do
-    tags = described_class.new([ 'foo' ], source: 'https://foo.example.com')
-    tags.add 'bar', source: '/path/to/bar.html'
-    expect(tags.to_a).to eq %w[ bar foo ]
-    tags.all? { expect(_1).to be_a(Ollama::Utils::Tags::Tag) }
-    expect(tags.to_s).to eq(
-      "\e]8;;file:///path/to/bar.html\e\\#bar\e]8;;\e\\ \e]8;;https://foo.example.com\e\\#foo\e]8;;\e\\"
-    )
-  end
-end