ollama-ruby 0.12.1 → 0.13.0

data/lib/ollama/documents/cache/common.rb

@@ -1,36 +0,0 @@
-module Ollama::Documents::Cache::Common
-  include Ollama::Utils::Math
-
-  def initialize(prefix:)
-    self.prefix = prefix
-  end
-
-  attr_accessor :prefix # current prefix defined for the cache
-
-  # Returns an array of collection names that match the given prefix.
-  #
-  # @param prefix [String] a string to search for in collection names
-  # @return [Array<Symbol>] an array of matching collection names
-  def collections(prefix)
-    unique = Set.new
-    full_each { |key, _| unique << key[/\A#{prefix}(.*)-/, 1] }
-    unique.map(&:to_sym)
-  end
-
-  # Returns a string representing the given `key` prefixed with the defined
-  # prefix.
-  #
-  # @param key [String] the key to join with the prefix
-  # @return [String] the joined string of prefix and key
-  def pre(key)
-    [ @prefix, key ].join
-  end
-
-  # Returns a string with the prefix removed from the given `key`.
-  #
-  # @param key [String] the input string containing the prefix.
-  # @return [String] the input string without the prefix.
-  def unpre(key)
-    key.sub(/\A#@prefix/, '')
-  end
-end

data/lib/ollama/documents/cache/memory_cache.rb

@@ -1,44 +0,0 @@
-require 'ollama/documents/cache/common'
-
-class Ollama::Documents::MemoryCache
-  include Ollama::Documents::Cache::Common
-
-  def initialize(prefix:)
-    super(prefix:)
-    @data   = {}
-  end
-
-  def [](key)
-    @data[pre(key)]
-  end
-
-  def []=(key, value)
-    @data[pre(key)] = value
-  end
-
-  def key?(key)
-    @data.key?(pre(key))
-  end
-
-  def delete(key)
-    @data.delete(pre(key))
-  end
-
-  def size
-    count
-  end
-
-  def clear
-    @data.delete_if { |key, _| key.start_with?(@prefix) }
-    self
-  end
-
-  def each(&block)
-    @data.select { |key,| key.start_with?(@prefix) }.each(&block)
-  end
-  include Enumerable
-
-  def full_each(&block)
-    @data.each(&block)
-  end
-end

data/lib/ollama/documents/cache/records.rb

@@ -1,87 +0,0 @@
-module Ollama::Documents::Cache::Records
-  class Record < JSON::GenericObject
-    def initialize(*a)
-      super
-      self.text ||= ''
-      self.norm ||= 0.0
-    end
-
-    def to_s
-      my_tags = tags_set
-      my_tags.empty? or my_tags = " #{my_tags}"
-      "#<#{self.class} #{text.inspect}#{my_tags} #{similarity || 'n/a'}>"
-    end
-
-    def tags_set
-      Ollama::Utils::Tags.new(tags, source:)
-    end
-
-    def ==(other)
-      text == other.text
-    end
-
-    alias inspect to_s
-  end
-
-  module RedisFullEach
-    def full_each(&block)
-      redis.scan_each(match: [ Ollama::Documents, ?* ] * ?-) do |key|
-        value = redis.get(key) or next
-        value = JSON(value, object_class: Ollama::Documents::Record)
-        block.(key, value)
-      end
-    end
-  end
-
-  module FindRecords
-    def find_records(needle, tags: nil, max_records: nil)
-      tags = Ollama::Utils::Tags.new(Array(tags)).to_a
-      records = self
-      if tags.present?
-        records = records.select { |_key, record| (tags & record.tags).size >= 1 }
-      end
-      needle_norm = norm(needle)
-      records     = records.sort_by { |key, record|
-        record.key        = key
-        record.similarity = cosine_similarity(
-          a: needle,
-          b: record.embedding,
-          a_norm: needle_norm,
-          b_norm: record.norm,
-        )
-      }
-      records.transpose.last&.reverse.to_a
-    end
-  end
-
-  module Tags
-    def clear(tags: nil)
-      tags = Ollama::Utils::Tags.new(tags).to_a
-      if tags.present?
-        if respond_to?(:clear_for_tags)
-          clear_for_tags(tags)
-        else
-          each do |key, record|
-            if (tags & record.tags.to_a).size >= 1
-              delete(unpre(key))
-            end
-          end
-        end
-      else
-        super()
-      end
-    end
-
-    def tags
-      if defined? super
-        super
-      else
-        each_with_object(Ollama::Utils::Tags.new) do |(_, record), t|
-          record.tags.each do |tag|
-            t.add(tag, source: record.source)
-          end
-        end
-      end
-    end
-  end
-end

data/lib/ollama/documents/cache/redis_backed_memory_cache.rb

@@ -1,39 +0,0 @@
-require 'redis'
-
-class Ollama::Documents
-  class RedisBackedMemoryCache < MemoryCache
-    def initialize(prefix:, url: ENV['REDIS_URL'], object_class: nil)
-      super(prefix:)
-      url or raise ArgumentError, 'require redis url'
-      @url, @object_class = url, object_class
-      @redis_cache  = Ollama::Documents::RedisCache.new(prefix:, url:, object_class:)
-      @redis_cache.extend(Ollama::Documents::Cache::Records::RedisFullEach)
-      @redis_cache.full_each do |key, value|
-        @data[key] = value
-      end
-    end
-
-    attr_reader :object_class
-
-    def redis
-      @redis_cache.redis
-    end
-
-    def []=(key, value)
-      super
-      redis.set(pre(key), JSON(value))
-    end
-
-    def delete(key)
-      result = redis.del(pre(key))
-      super
-      result
-    end
-
-    def clear
-      redis.scan_each(match: "#@prefix*") { |key| redis.del(key) }
-      super
-      self
-    end
-  end
-end

data/lib/ollama/documents/cache/redis_cache.rb

@@ -1,68 +0,0 @@
-require 'ollama/documents/cache/common'
-require 'redis'
-
-class Ollama::Documents::RedisCache
-  include Ollama::Documents::Cache::Common
-
-  def initialize(prefix:, url: ENV['REDIS_URL'], object_class: nil, ex: nil)
-    super(prefix:)
-    url or raise ArgumentError, 'require redis url'
-    @url, @object_class, @ex = url, object_class, ex
-  end
-
-  attr_reader :object_class
-
-  def redis
-    @redis ||= Redis.new(url: @url)
-  end
-
-  def [](key)
-    value = redis.get(pre(key))
-    unless value.nil?
-      object_class ? JSON(value, object_class:) : JSON(value)
-    end
-  end
-
-  def []=(key, value)
-    set(key, value)
-  end
-
-  def set(key, value, ex: nil)
-    ex ||= @ex
-    if !ex.nil? && ex < 1
-      redis.del(pre(key))
-    else
-      redis.set(pre(key), JSON.generate(value), ex:)
-    end
-    value
-  end
-
-  def ttl(key)
-    redis.ttl(pre(key))
-  end
-
-  def key?(key)
-    !!redis.exists?(pre(key))
-  end
-
-  def delete(key)
-    redis.del(pre(key)) == 1
-  end
-
-  def size
-    s = 0
-    redis.scan_each(match: "#@prefix*") { |key| s += 1 }
-    s
-  end
-
-  def clear
-    redis.scan_each(match: "#@prefix*") { |key| redis.del(key) }
-    self
-  end
-
-  def each(&block)
-    redis.scan_each(match: "#@prefix*") { |key| block.(key, self[unpre(key)]) }
-    self
-  end
-  include Enumerable
-end

data/lib/ollama/documents/cache/sqlite_cache.rb

@@ -1,215 +0,0 @@
-require 'ollama/documents/cache/common'
-require 'sqlite3'
-require 'sqlite_vec'
-require 'digest/md5'
-
-class Ollama::Documents::Cache::SQLiteCache
-  include Ollama::Documents::Cache::Common
-
-  def initialize(prefix:, embedding_length: 1_024, filename: ':memory:', debug: false)
-    super(prefix:)
-    @embedding_length = embedding_length
-    @filename         = filename
-    @debug            = debug
-    setup_database(filename)
-  end
-
-  attr_reader :filename # filename for the database, `:memory:` is in memory
-
-  attr_reader :embedding_length # length of the embeddings vector
-
-  def [](key)
-    result = execute(
-      %{
-        SELECT records.key, records.text, records.norm, records.source,
-          records.tags, embeddings.embedding
-        FROM records
-        INNER JOIN embeddings ON records.embedding_id = embeddings.rowid
-        WHERE records.key = ?
-      },
-      pre(key)
-    )&.first or return
-    key, text, norm, source, tags, embedding = *result
-    embedding = embedding.unpack("f*")
-    tags      = Ollama::Utils::Tags.new(JSON(tags.to_s).to_a, source:)
-    convert_value_to_record(key:, text:, norm:, source:, tags:, embedding:)
-  end
-
-  def []=(key, value)
-    value = convert_value_to_record(value)
-    embedding = value.embedding.pack("f*")
-    execute(%{BEGIN})
-    execute(%{INSERT INTO embeddings(embedding) VALUES(?)}, [ embedding ])
-    embedding_id, = execute(%{ SELECT last_insert_rowid() }).flatten
-    execute(%{
-      INSERT INTO records(key,text,embedding_id,norm,source,tags)
-      VALUES(?,?,?,?,?,?)
-    }, [ pre(key), value.text, embedding_id, value.norm, value.source, JSON(value.tags) ])
-    execute(%{COMMIT})
-  end
-
-  def key?(key)
-    execute(
-      %{ SELECT count(records.key) FROM records WHERE records.key = ? },
-      pre(key)
-    ).flatten.first == 1
-  end
-
-  def delete(key)
-    result = key?(key) ? pre(key) : nil
-    execute(
-      %{ DELETE FROM records WHERE records.key = ? },
-      pre(key)
-    )
-    result
-  end
-
-  def tags
-    result = Ollama::Utils::Tags.new
-    execute(%{
-        SELECT DISTINCT(tags) FROM records WHERE key LIKE ?
-      }, [ "#@prefix%" ]
-    ).flatten.each do
-      JSON(_1).each { |t| result.add(t) }
-    end
-    result
-  end
-
-  def size
-    execute(%{SELECT COUNT(*) FROM records WHERE key LIKE ?}, [ "#@prefix%" ]).flatten.first
-  end
-
-  def clear_for_tags(tags = nil)
-    tags = Ollama::Utils::Tags.new(tags).to_a
-    if tags.present?
-      records = find_records_for_tags(tags)
-      keys = '(%s)' % records.transpose.first.map { "'%s'" % quote(_1) }.join(?,)
-      execute(%{DELETE FROM records WHERE key IN #{keys}})
-    else
-      clear
-    end
-    self
-  end
-
-  def clear
-    execute(%{DELETE FROM records WHERE key LIKE ?}, [ "#@prefix%" ])
-    self
-  end
-
-  def each(prefix: "#@prefix%", &block)
-    execute(%{
-      SELECT records.key, records.text, records.norm, records.source,
-        records.tags, embeddings.embedding
-      FROM records
-      INNER JOIN embeddings ON records.embedding_id = embeddings.rowid
-      WHERE records.key LIKE ?
-    }, [ prefix ]).each do |key, text, norm, source, tags, embedding|
-      embedding = embedding.unpack("f*")
-      tags      = Ollama::Utils::Tags.new(JSON(tags.to_s).to_a, source:)
-      value     = convert_value_to_record(key:, text:, norm:, source:, tags:, embedding:)
-      block.(key, value)
-    end
-  end
-  include Enumerable
-
-  def full_each(&block)
-    each(prefix: ?%, &block)
-  end
-
-  def convert_to_vector(vector)
-    vector
-  end
-
-  def find_records_for_tags(tags)
-    if tags.present?
-      tags_filter = Ollama::Utils::Tags.new(tags).to_a
-      unless  tags_filter.empty?
-        tags_where  = ' AND (%s)' % tags_filter.map {
-          'tags LIKE "%%%s%%"' % quote(_1)
-        }.join(' OR ')
-      end
-    end
-    records = execute(%{
-      SELECT key, tags, embedding_id
-      FROM records
-      WHERE key LIKE ?#{tags_where}
-    }, [ "#@prefix%" ])
-    if tags_filter
-      records = records.select { |key, tags, embedding_id|
-        (tags_filter & JSON(tags.to_s).to_a).size >= 1
-      }
-    end
-    records
-  end
-
-  def find_records(needle, tags: nil, max_records: nil)
-    needle.size != @embedding_length and
-      raise ArgumentError, "needle embedding length != %s" % @embedding_length
-    needle_binary = needle.pack("f*")
-    max_records   = [ max_records, size, 4_096 ].compact.min
-    records = find_records_for_tags(tags)
-    rowids_where = '(%s)' % records.transpose.last&.join(?,)
-    execute(%{
-      SELECT records.key, records.text, records.norm, records.source,
-        records.tags, embeddings.embedding
-      FROM records
-      INNER JOIN embeddings ON records.embedding_id = embeddings.rowid
-      WHERE embeddings.rowid IN #{rowids_where}
-        AND embeddings.embedding MATCH ? AND embeddings.k = ?
-    }, [ needle_binary, max_records ]).map do |key, text, norm, source, tags, embedding|
-      key       = unpre(key)
-      embedding = embedding.unpack("f*")
-      tags      = Ollama::Utils::Tags.new(JSON(tags.to_s).to_a, source:)
-      convert_value_to_record(key:, text:, norm:, source:, tags:, embedding:)
-    end
-  end
-
-  private
-
-  def execute(*a)
-    if @debug
-      e = a[0].gsub(/^\s*\n/, '')
-      e = e.gsub(/\A\s+/, '')
-      n = $&.to_s.size
-      e = e.gsub(/^\s{0,#{n}}/, '')
-      e = e.chomp
-      STDERR.puts("EXPLANATION:\n%s\n%s" % [
-        e,
-        @database.execute("EXPLAIN #{e}", *a[1..-1]).pretty_inspect
-      ])
-    end
-    @database.execute(*a)
-  end
-
-  def quote(string)
-    SQLite3::Database.quote(string)
-  end
-
-  def setup_database(filename)
-    @database = SQLite3::Database.new(filename)
-    @database.enable_load_extension(true)
-    SqliteVec.load(@database)
-    @database.enable_load_extension(false)
-    execute %{
-      CREATE VIRTUAL TABLE IF NOT EXISTS embeddings USING vec0(
-        embedding float[#@embedding_length]
-      )
-    }
-    execute %{
-      CREATE TABLE IF NOT EXISTS records (
-        key          text NOT NULL PRIMARY KEY ON CONFLICT REPLACE,
-        text         text NOT NULL DEFAULT '',
-        embedding_id integer,
-        norm         float NOT NULL DEFAULT 0.0,
-        source       text,
-        tags         json NOT NULL DEFAULT [],
-        FOREIGN KEY(embedding_id) REFERENCES embeddings(id) ON DELETE CASCADE
-      )
-    }
-  end
-
-  def convert_value_to_record(value)
-    value.is_a?(Ollama::Documents::Record) and return value
-    Ollama::Documents::Record[value.to_hash]
-  end
-end

data/lib/ollama/documents/splitters/character.rb

@@ -1,72 +0,0 @@
-module Ollama::Documents::Splitters
-  class Character
-    DEFAULT_SEPARATOR = /(?:\r?\n){2,}/
-
-    def initialize(separator: DEFAULT_SEPARATOR, include_separator: false, combining_string: "\n\n", chunk_size: 4096)
-      @separator, @include_separator, @combining_string, @chunk_size =
-        separator, include_separator, combining_string, chunk_size
-      if include_separator
-        @separator = Regexp.new("(#@separator)")
-      end
-    end
-
-    def split(text)
-      texts = []
-      text.split(@separator) do |t|
-        if @include_separator && t =~ @separator
-          texts.last&.concat t
-        else
-          texts.push(t)
-        end
-      end
-      result = []
-      current_text = +''
-      texts.each do |t|
-        if current_text.size + t.size < @chunk_size
-          current_text << t << @combining_string
-        else
-          current_text.empty? or result << current_text
-          current_text = t
-        end
-      end
-      current_text.empty? or result << current_text
-      result
-    end
-  end
-
-  class RecursiveCharacter
-    DEFAULT_SEPARATORS = [
-      /(?:\r?\n){2,}/,
-      /\r?\n/,
-      /\b/,
-      //,
-    ].freeze
-
-    def initialize(separators: DEFAULT_SEPARATORS, include_separator: false, combining_string: "\n\n", chunk_size: 4096)
-      separators.empty? and
-        raise ArgumentError, "non-empty array of separators required"
-      @separators, @include_separator, @combining_string, @chunk_size =
-        separators, include_separator, combining_string, chunk_size
-    end
-
-    def split(text, separators: @separators)
-      separators.empty? and return [ text ]
-      separators = separators.dup
-      separator = separators.shift
-      texts = Character.new(
-        separator:,
-        include_separator: @include_separator,
-        combining_string: @combining_string,
-        chunk_size: @chunk_size
-      ).split(text)
-      texts.count == 0 and return [ text ]
-      texts.inject([]) do |r, t|
-        if t.size > @chunk_size
-          r.concat(split(t, separators:))
-        else
-          r.concat([ t ])
-        end
-      end
-    end
-  end
-end

data/lib/ollama/documents/splitters/semantic.rb

@@ -1,91 +0,0 @@
-module Ollama::Documents::Splitters
-  class Semantic
-    include Ollama::Utils::Math
-
-    DEFAULT_SEPARATOR = /[.!?]\s*(?:\b|\z)/
-
-    def initialize(ollama:, model:, model_options: nil, separator: DEFAULT_SEPARATOR, chunk_size: 4096)
-      @ollama, @model, @model_options, @separator, @chunk_size =
-        ollama, model, model_options, separator, chunk_size
-    end
-
-    def split(text, batch_size: 100, breakpoint: :percentile, **opts)
-      sentences  = Ollama::Documents::Splitters::Character.new(
-        separator: @separator,
-        include_separator: opts.fetch(:include_separator, true),
-        chunk_size: 1,
-      ).split(text)
-      embeddings = sentences.with_infobar(label: 'Split').each_slice(batch_size).inject([]) do |e, batch|
-        e.concat sentence_embeddings(batch)
-        infobar.progress by: batch.size
-        e
-      end
-      infobar.newline
-      embeddings.size < 2 and return sentences
-      distances = embeddings.each_cons(2).map do |a, b|
-        1.0 - cosine_similarity(a:, b:)
-      end
-      max_distance = calculate_breakpoint_threshold(breakpoint, distances, **opts)
-      gaps = distances.each_with_index.select do |d, i|
-        d > max_distance
-      end.transpose.last
-      gaps or return sentences
-      if gaps.last < distances.size
-        gaps << distances.size
-      end
-      if gaps.last < sentences.size - 1
-        gaps << sentences.size - 1
-      end
-      result = []
-      sg = 0
-      current_text = +''
-      gaps.each do |g|
-        sg.upto(g) do |i|
-          sentence = sentences[i]
-          if current_text.size + sentence.size < @chunk_size
-            current_text += sentence
-          else
-            current_text.empty? or result << current_text
-            current_text = sentence
-          end
-        end
-        unless current_text.empty?
-          result << current_text
-          current_text = +''
-        end
-        sg = g.succ
-      end
-      current_text.empty? or result << current_text
-      result
-    end
-
-    private
-
-    def calculate_breakpoint_threshold(breakpoint_method, distances, **opts)
-      sequence = MoreMath::Sequence.new(distances)
-      case breakpoint_method
-      when :percentile
-        percentile = opts.fetch(:percentile, 95)
-        sequence.percentile(percentile)
-      when :standard_deviation
-        percentage = opts.fetch(:percentage, 100)
-        (
-          sequence.mean + sequence.standard_deviation * (percentage / 100.0)
-        ).clamp(0, sequence.max)
-      when :interquartile
-        percentage = opts.fetch(:percentage, 100)
-        iqr = sequence.interquartile_range
-        max = sequence.max
-        (sequence.mean + iqr * (percentage / 100.0)).clamp(0, max)
-      else
-        raise ArgumentError, "invalid breakpoint method #{breakpoint_method}"
-      end
-    end
-
-    def sentence_embeddings(input)
-      @ollama.embed(model: @model, input:, options: @model_options).embeddings.map! {
-        Numo::NArray[*_1]
-      }
-    end
-  end
-end