ollama-ruby 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1f80ae8ee6e8acbbedfff8b56923b25b583fc60b96c733985e32908874d542bb
-  data.tar.gz: 80834beb676929f08f4216e373e56cadd12d16c0b50e95b1a599bdd48bf29c86
+  metadata.gz: 510c31683b2251118a7c3b469620400016b29fd1ea6f17cc86e4f99f62ecea2f
+  data.tar.gz: a00c4275e42002a01a99f874a855386c83e502551360c1f4146eee0b74f2fd08
 SHA512:
-  metadata.gz: 94823ec618f940056bcee6ac7d706c633087404efcb49122cdccc89a4596b4a5f14a0d1c1d60fd7bb13a28e10f3ba4dc2027cd96cfe384169b25acaf75f532b7
-  data.tar.gz: ef8d4a7001c5502bc787f074f10dfaf16692b8fa90c4a6e32ae3c0c75b94da45b4691094229106cec72438837eef57b933cd5e5c9b28e63d0d4a598c152a9c32
+  metadata.gz: 17e50d40e4c24b56b5c2923f0b73f3e4294587b61a302fbaf4cd8f891e879eb435541e1f7e195a4f88352fd99aa80ff44c8577b8c195d7ef31af1c33229018b7
+  data.tar.gz: a18bcef82e9481b75fee4a4c88598a7a374c48f841e9870b39174c0d1f5f235f1173bdbff939182481d431e4e9c8c85501be4f5ef917ca02865692cfc39fde5a

data/CHANGES.md

@@ -1,5 +1,32 @@
 # Changes
 
+## 2024-09-05 v0.3.0
+
+* **New Features**
+  * Created new file `ollama_cli` with Ollama CLI functionality.
+  * Added executable `ollama_cli` to s.executables in ollama-ruby.gemspec.
+  * Added `find_where` method in `documents.rb` to filter records by text size
+    and count.
+  * Added test for `find_where` method in `documents_spec.rb`.
+  * Features for `ollama_chat`
+      * Added `found_texts_count` option to `OllamaChatConfig`.
+      * Implemented `parse_rss` method for RSS feeds and `parse_atom` method
+        for Atom feeds.
+      * Added links to titles in RSS feed item summaries and Atom feed item
+        summaries.
+      * Updated `parse_source` method to handle different content types,
+        including HTML, XML, and RSS/Atom feeds.
+      * Added `/web [n] query` command to search web and return n or 1 results
+        in chat interface.
+* **Improvements**
+  * Improved validation for system prompts
+  * Extracted file argument handling into a separate module and method
+  * Added default value for config or model system prompt
+  * Improved input validation for `system_prompt` path
+  * Updated collection clearing logic to accept optional tags parameter
+  * Updated `Tags` class to overload `to_a` method for converting to array of
+    strings
+
 ## 2024-09-03 v0.2.0
 
 ### Changes

data/README.md

@@ -152,19 +152,20 @@ subject - the young, blue-eyed cat.
 The following commands can be given inside the chat, if prefixed by a `/`:
 
 ```
-/paste                             to paste content
-/markdown                          toggle markdown output
-/list                              list the messages of the conversation
-/clear                             clear the conversation messages
-/pop [n]                           pop the last n exchanges, defaults to 1
-/model                             change the model
-/regenerate                        the last answer message
-/collection clear|stats|change|new clear or show stats of current collection
-/summarize source                  summarize the URL/file source's content
-/save filename                     store conversation messages
-/load filename                     load conversation messages
-/quit                              to quit
-/help                              to view this help
+/paste                                   to paste content
+/markdown                                toggle markdown output
+/list                                    list the messages of the conversation
+/clear                                   clear the conversation messages
+/pop [n]                                 pop the last n exchanges, defaults to 1
+/model                                   change the model
+/regenerate                              the last answer message
+/collection clear [tag]|stats|change|new clear or show stats of current collection
+/summarize source                        summarize the URL/file source's content
+/web [n] query                           query web search & return n or 1 results
+/save filename                           store conversation messages
+/load filename                           load conversation messages
+/quit                                    to quit
+/help                                    to view this help
 ```
 
 ### ollama\_console

data/Rakefile

@@ -18,7 +18,8 @@ GemHadar do
      '.utilsrc', '.rspec', *Dir.glob('.github/**/*', File::FNM_DOTMATCH)
   readme      'README.md'
 
-  executables << 'ollama_console' << 'ollama_chat' << 'ollama_update'
+  executables << 'ollama_console' << 'ollama_chat' <<
+    'ollama_update' << 'ollama_cli'
 
   required_ruby_version  '~> 3.1'
 
@@ -36,6 +37,7 @@ GemHadar do
   dependency             'complex_config',        '~> 0.20'
   dependency             'search_ui',             '~> 0.0'
   dependency             'amatch',                '~> 0.4.1'
+  dependency             'pdf-reader',            '~> 2.0'
   development_dependency 'all_images',            '~> 0.4'
   development_dependency 'rspec',                 '~> 3.2'
   development_dependency 'utils'

data/bin/ollama_chat

@@ -12,6 +12,8 @@ require 'complex_config'
 require 'fileutils'
 require 'uri'
 require 'nokogiri'
+require 'rss'
+require 'pdf/reader'
 
 class OllamaChatConfig
   include ComplexConfig
@@ -36,6 +38,7 @@ class OllamaChatConfig
         prompt: 'Represent this sentence for searching relevant passages: %s'
       collection: <%= ENV.fetch('OLLAMA_CHAT_COLLECTION', 'ollama_chat') %>
       found_texts_size: 4096
+      found_texts_count: null
       splitter:
         name: RecursiveCharacter
         chunk_size: 1024
@@ -153,7 +156,8 @@ end
 
 def pull_model_unless_present(model, options, retried = false)
   ollama.show(name: model) { |response|
-    puts "Model #{bold{model}} with architecture #{response.model_info['general.architecture']} found."
+    puts "Model #{bold{model}} with architecture "\
+      "#{response.model_info['general.architecture']} found."
     if system = response.system
       puts "Configured model system prompt is:\n#{italic { system }}"
       return system
@@ -226,17 +230,77 @@ def list_conversation(messages, markdown)
   end
 end
 
+def reverse_markdown(html)
+  ReverseMarkdown.convert(
+    html,
+    unknown_tags: :bypass,
+    github_flavored: true,
+    tag_border: ''
+  )
+end
+
+def parse_rss(source_io)
+  feed = RSS::Parser.parse(source_io, false, false)
+  title = <<~end
+    # #{feed&.channel&.title}
+
+  end
+  feed.items.inject(title) do |text, item|
+    text << <<~end
+      ## [#{item&.title}](#{item&.link})
+
+      updated on #{item&.pubDate}
+
+      #{reverse_markdown(item&.description)}
+
+    end
+  end
+end
+
+def parse_atom(source_io)
+  feed = RSS::Parser.parse(source_io, false, false)
+  title = <<~end
+    # #{feed.title.content}
+
+  end
+  feed.items.inject(title) do |text, item|
+    text << <<~end
+      ## [#{item&.title&.content}](#{item&.link&.href})
+
+      updated on #{item&.updated&.content}
+
+      #{reverse_markdown(item&.content&.content)}
+
+    end
+  end
+end
+
 def parse_source(source_io)
-  case source_io&.content_type&.sub_type
-  when 'html'
-    ReverseMarkdown.convert(
-      source_io.read,
-      unknown_tags: :bypass,
-      github_flavored: true,
-      tag_border: ''
-    )
-  when 'plain', 'csv', 'xml'
+  case source_io&.content_type
+  when 'text/html'
+    reverse_markdown(source_io.read)
+  when 'text/xml'
+    if source_io.readline =~ %r(^\s*<rss\s)
+      source_io.rewind
+      return parse_rss(source_io)
+    end
+    source_io.rewind
+    source_io.read
+  when %r(\Atext/)
     source_io.read
+  when 'application/rss+xml'
+    parse_rss(source_io)
+  when 'application/atom+xml'
+    parse_atom(source_io)
+  when 'application/json'
+    source_io.read
+  when 'application/pdf'
+    reader = PDF::Reader.new(source_io)
+    result = +''
+    reader.pages.each do |page|
+      result << page.text
+    end
+    result
   else
     STDERR.puts "Cannot import #{source_io&.content_type} document."
     return
@@ -327,7 +391,7 @@ def parse_content(content, images)
         case source_io&.content_type&.media_type
         when 'image'
           add_image(images, source_io, source)
-        when 'text'
+        when 'text', 'application'
           import_document(source_io, source)
         else
           STDERR.puts(
@@ -391,19 +455,20 @@ end
 
 def display_chat_help
   puts <<~end
-    /paste                             to paste content
-    /markdown                          toggle markdown output
-    /list                              list the messages of the conversation
-    /clear                             clear the conversation messages
-    /pop [n]                           pop the last n exchanges, defaults to 1
-    /model                             change the model
-    /regenerate                        the last answer message
-    /collection clear|stats|change|new clear or show stats of current collection
-    /summarize source                  summarize the URL/file source's content
-    /save filename                     store conversation messages
-    /load filename                     load conversation messages
-    /quit                              to quit
-    /help                              to view this help
+    /paste                                   to paste content
+    /markdown                                toggle markdown output
+    /list                                    list the messages of the conversation
+    /clear                                   clear the conversation messages
+    /pop [n]                                 pop the last n exchanges, defaults to 1
+    /model                                   change the model
+    /regenerate                              the last answer message
+    /collection clear [tag]|stats|change|new clear or show stats of current collection
+    /summarize source                        summarize the URL/file source's content
+    /web [n] query                           query web search & return n or 1 results
+    /save filename                           store conversation messages
+    /load filename                           load conversation messages
+    /quit                                    to quit
+    /help                                    to view this help
   end
 end
 
@@ -497,23 +562,16 @@ markdown = set_markdown($config.markdown)
 if opts[?c]
   messages.concat load_conversation(opts[?c])
 else
-  system = nil
-  if system_prompt_file = opts[?s]
-    system = File.read(system_prompt_file)
-  end
-  system ||= $config.system
-
-  if system
+  if system = Ollama::Utils::FileArgument.
+      get_file_argument(opts[?s], default: $config.system? || model_system)
     messages << Message.new(role: 'system', content: system)
     puts "Configured system prompt is:\n#{italic { system }}"
-  elsif model_system.present?
-    puts "Using model system prompt."
   end
 end
 
 puts "\nType /help to display the chat help."
 
-images           = []
+images = []
 loop do
   parse_content = true
   input_prompt = bold { color(172) { message_type(images) + " user" } } + bold { "> " }
@@ -536,11 +594,18 @@ loop do
     messages.clear
     puts "Cleared messages."
     next
-  when %r(^/collection (clear|stats|change|new)$)
-    case $1
+  when %r(^/collection\s+(clear|stats|change|new)(?:\s+(.+))?$)
+    command, arg = $1, $2
+    case command
     when 'clear'
-      $documents.clear
-      puts "Cleared collection #{bold{collection}}."
+      tags = arg.present? ? arg.sub(/\A#*/, '') : nil
+      if tags
+        $documents.clear(tags:)
+        puts "Cleared tag ##{tags} from collection #{bold{collection}}."
+      else
+        $documents.clear
+        puts "Cleared collection #{bold{collection}}."
+      end
     when 'stats'
       collection_stats
     when 'change'
@@ -571,7 +636,7 @@ loop do
   when %r(^/summarize\s+(.+))
     parse_content = false
     content       = summarize($1) or next
-  when %r(^/web\s+(?:(\d+)\s+)(.+)$)
+  when %r(^/web\s+(?:(\d+)\s+)?(.+)$)
     parse_content = true
     urls = search_web($2, $1.to_i)
     content = <<~end
@@ -602,15 +667,13 @@ loop do
                   end
 
   if $config.embedding.enabled && content
-    records = $documents.find(
+    records = $documents.find_where(
       content.downcase,
       tags:,
-      prompt: $config.embedding.model.prompt?
+      prompt:     $config.embedding.model.prompt?,
+      text_size:  $config.embedding.found_texts_size?,
+      text_count: $config.embedding.found_texts_count?,
     )
-    s, found_texts_size = 0, $config.embedding.found_texts_size
-    records = records.take_while {
-      (s += _1.text.size) <= found_texts_size
-    }
     found_texts = records.map(&:text)
     unless found_texts.empty?
       content += "\nConsider these chunks for your answer:\n"\

data/bin/ollama_cli

@@ -0,0 +1,68 @@
+#!/usr/bin/env ruby
+
+require 'ollama'
+include Ollama
+include Ollama::Utils::FileArgument
+require 'tins'
+include Tins::GO
+require 'json'
+
+def usage
+  puts <<~end
+    #{File.basename($0)} [OPTIONS]
+
+      -u URL         the ollama base url, OLLAMA_URL
+      -m MODEL       the ollama model to chat with, OLLAMA_MODEL
+      -M OPTIONS     the ollama model options to use, OLLAMA_MODEL_OPTIONS
+      -s SYSTEM      the system prompt to use as a file, OLLAMA_SYSTEM
+      -p PROMPT      the user prompt to use as a file, OLLAMA_PROMPT
+      -H HANDLER     the handler to use for the response, defaults to Print
+      -S             use streaming for generation
+      -h             this help
+
+  end
+  exit 0
+end
+
+opts = go 'u:m:M:s:p:H:Sh', defaults: { ?H => 'Print', ?M => '{}' }
+
+opts[?h] and usage
+
+base_url = opts[?u] || ENV['OLLAMA_URL'] || 'http://%s' % ENV.fetch('OLLAMA_HOST')
+model    = opts[?m] || ENV.fetch('OLLAMA_MODEL', 'llama3.1')
+options  = Ollama::Options.from_hash(JSON(
+  get_file_argument(opts[?M], default: ENV['OLLAMA_MODEL_OPTIONS'])
+))
+system   = get_file_argument(opts[?s], default: ENV['OLLAMA_SYSTEM'])
+prompt   = get_file_argument(opts[?p], default: ENV['OLLAMA_PROMPT'])
+
+if prompt.nil?
+  prompt = STDIN.read
+elsif c = prompt.scan('%s').size
+  case c
+  when 0
+  when 1
+    prompt = prompt % STDIN.read
+  else
+    STDERR.puts "Found more than one plaeceholder %s. => Ignoring."
+  end
+end
+
+if ENV['DEBUG'].to_i == 1
+  puts <<~EOT
+    base_url = #{base_url.inspect}
+    model    = #{model.inspect}
+    system   = #{system.inspect}
+    prompt   = #{prompt.inspect}
+    options  = #{options.to_json}
+  EOT
+end
+
+Client.new(base_url:, read_timeout: 120).generate(
+  model:,
+  system:,
+  prompt:,
+  options:,
+  stream: !!opts[?S],
+  &Object.const_get(opts[?H])
+)

data/bin/ollama_console

@@ -5,8 +5,13 @@ include Ollama
 require 'irb'
 require 'irb/history'
 
-base_url = ENV['OLLAMA_URL'] || 'http://%s' % ENV.fetch('OLLAMA_HOST')
-ollama = Client.new(base_url:)
+def base_url
+  ENV['OLLAMA_URL'] || 'http://%s' % ENV.fetch('OLLAMA_HOST')
+end
+
+def ollama
+  $ollama ||= Client.new(base_url:)
+end
 IRB.setup nil
 IRB.conf[:MAIN_CONTEXT] = IRB::Irb.new.context
 IRB.conf[:HISTORY_FILE] = File.join(ENV.fetch('HOME'), '.ollama_console-history')

data/lib/ollama/documents/memory_cache.rb

@@ -36,9 +36,11 @@ class Ollama::Documents::MemoryCache
   end
   include Enumerable
 
-  private
-
   def pre(key)
     [ @prefix, key ].join
   end
+
+  def unpre(key)
+    key.sub(/\A#@prefix/, '')
+  end
 end

data/lib/ollama/documents/redis_cache.rb

@@ -13,7 +13,10 @@ class Ollama::Documents::RedisCache
   end
 
   def [](key)
-    JSON(redis.get(pre(key)), object_class: Ollama::Documents::Record)
+    value = redis.get(pre(key))
+    unless value.nil?
+      JSON(value, object_class: Ollama::Documents::Record)
+    end
   end
 
   def []=(key, value)
@@ -45,8 +48,6 @@ class Ollama::Documents::RedisCache
   end
   include Enumerable
 
-  private
-
   def pre(key)
     [ @prefix, key ].join
   end

data/lib/ollama/documents.rb

@@ -12,6 +12,7 @@ require 'ollama/documents/splitters/semantic'
 
 class Ollama::Documents
   include Ollama::Utils::Math
+  include Ollama::Utils::Width
 
   class Record < JSON::GenericObject
     def to_s
@@ -42,7 +43,7 @@ class Ollama::Documents
   def add(inputs, batch_size: 10, source: nil, tags: [])
     inputs = Array(inputs)
     tags   = Ollama::Utils::Tags.new(tags)
-    source and tags.add File.basename(source)
+    source and tags.add File.basename(source).gsub(/\?.*/, '')
     inputs.map! { |i|
       text = i.respond_to?(:read) ? i.read : i.to_s
       text
@@ -51,7 +52,7 @@ class Ollama::Documents
     inputs.empty? and return self
     batches = inputs.each_slice(batch_size).
       with_infobar(
-        label: "Add #{tags}",
+        label: "Add #{truncate(tags.to_s, percentage: 25)}",
         total: inputs.size
       )
     batches.each do |batch|
@@ -87,8 +88,18 @@ class Ollama::Documents
     @cache.size
   end
 
-  def clear
-    @cache.clear
+  def clear(tags: nil)
+    if tags
+      tags = Ollama::Utils::Tags.new(Array(tags)).to_a
+      @cache.each do |key, record|
+        if (tags & record.tags).size >= 1
+          @cache.delete(@cache.unpre(key))
+        end
+      end
+    else
+      @cache.clear
+    end
+    self
   end
 
   def find(string, tags: nil, prompt: nil)
@@ -96,7 +107,7 @@ class Ollama::Documents
     needle_norm = norm(needle)
     records = @cache
     if tags
-      tags = Ollama::Utils::Tags.new(tags)
+      tags = Ollama::Utils::Tags.new(tags).to_a
       records = records.select { |_key, record| (tags & record.tags).size >= 1 }
     end
     records = records.sort_by { |key, record|
@@ -111,6 +122,20 @@ class Ollama::Documents
     records.transpose.last&.reverse.to_a
   end
 
+  def find_where(string, text_size: nil, text_count: nil, **opts)
+    records = find(string, **opts)
+    size, count = 0, 0
+    records.take_while do |record|
+      if text_size and (size += record.text.size) > text_size
+        next false
+      end
+      if text_count and (count += 1) > text_count
+        next false
+      end
+      true
+    end
+  end
+
   def collections
     case @cache
     when MemoryCache

data/lib/ollama/utils/file_argument.rb

@@ -0,0 +1,16 @@
+module Ollama::Utils::FileArgument
+  module_function
+
+  def get_file_argument(prompt, default: nil)
+    if prompt.present? && prompt.size < 2 ** 15 &&
+        File.basename(prompt).size < 2 ** 8 &&
+        File.exist?(prompt)
+      then
+      File.read(prompt)
+    elsif prompt.present?
+      prompt
+    else
+      default
+    end
+  end
+end

data/lib/ollama/utils/tags.rb

@@ -1,6 +1,10 @@
 require 'sorted_set'
 
 class Ollama::Utils::Tags < SortedSet
+  def to_a
+    super.map(&:to_s)
+  end
+
   def to_s
     map { |t| '#%s' % t } * ' '
   end

data/lib/ollama/utils/width.rb

@@ -19,4 +19,17 @@ module Ollama::Utils::Width
       end
     end * ?\n
   end
+
+  def truncate(text, percentage: nil, length: nil, ellipsis: ?…)
+    percentage.nil? ^ length.nil? or
+      raise ArgumentError, "either pass percentage or length argument"
+    percentage and length ||= width(percentage:)
+    if length < 1
+      +''
+    elsif text.size > length
+      text[0, length - 1] + ?…
+    else
+      text
+    end
+  end
 end

data/lib/ollama/version.rb

@@ -1,6 +1,6 @@
 module Ollama
   # Ollama version
-  VERSION         = '0.2.0'
+  VERSION         = '0.3.0'
   VERSION_ARRAY   = VERSION.split('.').map(&:to_i) # :nodoc:
   VERSION_MAJOR   = VERSION_ARRAY[0] # :nodoc:
   VERSION_MINOR   = VERSION_ARRAY[1] # :nodoc:

data/lib/ollama.rb

@@ -18,6 +18,7 @@ require 'ollama/utils/math'
 require 'ollama/utils/colorize_texts'
 require 'ollama/utils/fetcher'
 require 'ollama/utils/chooser'
+require 'ollama/utils/file_argument'
 
 require 'ollama/version'
 require 'ollama/errors'

data/ollama-ruby.gemspec

@@ -1,24 +1,24 @@
 # -*- encoding: utf-8 -*-
-# stub: ollama-ruby 0.2.0 ruby lib
+# stub: ollama-ruby 0.3.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "ollama-ruby".freeze
-  s.version = "0.2.0".freeze
+  s.version = "0.3.0".freeze
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
   s.authors = ["Florian Frank".freeze]
-  s.date = "2024-09-03"
+  s.date = "2024-09-06"
   s.description = "Library that allows interacting with the Ollama API".freeze
   s.email = "flori@ping.de".freeze
-  s.executables = ["ollama_console".freeze, "ollama_chat".freeze, "ollama_update".freeze]
-  s.extra_rdoc_files = ["README.md".freeze, "lib/ollama.rb".freeze, "lib/ollama/client.rb".freeze, "lib/ollama/client/command.rb".freeze, "lib/ollama/client/doc.rb".freeze, "lib/ollama/commands/chat.rb".freeze, "lib/ollama/commands/copy.rb".freeze, "lib/ollama/commands/create.rb".freeze, "lib/ollama/commands/delete.rb".freeze, "lib/ollama/commands/embed.rb".freeze, "lib/ollama/commands/embeddings.rb".freeze, "lib/ollama/commands/generate.rb".freeze, "lib/ollama/commands/ps.rb".freeze, "lib/ollama/commands/pull.rb".freeze, "lib/ollama/commands/push.rb".freeze, "lib/ollama/commands/show.rb".freeze, "lib/ollama/commands/tags.rb".freeze, "lib/ollama/documents.rb".freeze, "lib/ollama/documents/memory_cache.rb".freeze, "lib/ollama/documents/redis_cache.rb".freeze, "lib/ollama/documents/splitters/character.rb".freeze, "lib/ollama/documents/splitters/semantic.rb".freeze, "lib/ollama/dto.rb".freeze, "lib/ollama/errors.rb".freeze, "lib/ollama/handlers.rb".freeze, "lib/ollama/handlers/collector.rb".freeze, "lib/ollama/handlers/concern.rb".freeze, "lib/ollama/handlers/dump_json.rb".freeze, "lib/ollama/handlers/dump_yaml.rb".freeze, "lib/ollama/handlers/markdown.rb".freeze, "lib/ollama/handlers/nop.rb".freeze, "lib/ollama/handlers/print.rb".freeze, "lib/ollama/handlers/progress.rb".freeze, "lib/ollama/handlers/say.rb".freeze, "lib/ollama/handlers/single.rb".freeze, "lib/ollama/image.rb".freeze, "lib/ollama/message.rb".freeze, "lib/ollama/options.rb".freeze, "lib/ollama/response.rb".freeze, "lib/ollama/tool.rb".freeze, "lib/ollama/tool/function.rb".freeze, "lib/ollama/tool/function/parameters.rb".freeze, "lib/ollama/tool/function/parameters/property.rb".freeze, "lib/ollama/utils/ansi_markdown.rb".freeze, "lib/ollama/utils/chooser.rb".freeze, "lib/ollama/utils/colorize_texts.rb".freeze, "lib/ollama/utils/fetcher.rb".freeze, "lib/ollama/utils/math.rb".freeze, "lib/ollama/utils/tags.rb".freeze, "lib/ollama/utils/width.rb".freeze, "lib/ollama/version.rb".freeze]
-  s.files = [".envrc".freeze, "CHANGES.md".freeze, "Gemfile".freeze, "LICENSE".freeze, "README.md".freeze, "Rakefile".freeze, "bin/ollama_chat".freeze, "bin/ollama_console".freeze, "bin/ollama_update".freeze, "config/redis.conf".freeze, "docker-compose.yml".freeze, "lib/ollama.rb".freeze, "lib/ollama/client.rb".freeze, "lib/ollama/client/command.rb".freeze, "lib/ollama/client/doc.rb".freeze, "lib/ollama/commands/chat.rb".freeze, "lib/ollama/commands/copy.rb".freeze, "lib/ollama/commands/create.rb".freeze, "lib/ollama/commands/delete.rb".freeze, "lib/ollama/commands/embed.rb".freeze, "lib/ollama/commands/embeddings.rb".freeze, "lib/ollama/commands/generate.rb".freeze, "lib/ollama/commands/ps.rb".freeze, "lib/ollama/commands/pull.rb".freeze, "lib/ollama/commands/push.rb".freeze, "lib/ollama/commands/show.rb".freeze, "lib/ollama/commands/tags.rb".freeze, "lib/ollama/documents.rb".freeze, "lib/ollama/documents/memory_cache.rb".freeze, "lib/ollama/documents/redis_cache.rb".freeze, "lib/ollama/documents/splitters/character.rb".freeze, "lib/ollama/documents/splitters/semantic.rb".freeze, "lib/ollama/dto.rb".freeze, "lib/ollama/errors.rb".freeze, "lib/ollama/handlers.rb".freeze, "lib/ollama/handlers/collector.rb".freeze, "lib/ollama/handlers/concern.rb".freeze, "lib/ollama/handlers/dump_json.rb".freeze, "lib/ollama/handlers/dump_yaml.rb".freeze, "lib/ollama/handlers/markdown.rb".freeze, "lib/ollama/handlers/nop.rb".freeze, "lib/ollama/handlers/print.rb".freeze, "lib/ollama/handlers/progress.rb".freeze, "lib/ollama/handlers/say.rb".freeze, "lib/ollama/handlers/single.rb".freeze, "lib/ollama/image.rb".freeze, "lib/ollama/message.rb".freeze, "lib/ollama/options.rb".freeze, "lib/ollama/response.rb".freeze, "lib/ollama/tool.rb".freeze, "lib/ollama/tool/function.rb".freeze, "lib/ollama/tool/function/parameters.rb".freeze, "lib/ollama/tool/function/parameters/property.rb".freeze, "lib/ollama/utils/ansi_markdown.rb".freeze, "lib/ollama/utils/chooser.rb".freeze, "lib/ollama/utils/colorize_texts.rb".freeze, "lib/ollama/utils/fetcher.rb".freeze, "lib/ollama/utils/math.rb".freeze, "lib/ollama/utils/tags.rb".freeze, "lib/ollama/utils/width.rb".freeze, "lib/ollama/version.rb".freeze, "ollama-ruby.gemspec".freeze, "spec/assets/embeddings.json".freeze, "spec/assets/kitten.jpg".freeze, "spec/ollama/client/doc_spec.rb".freeze, "spec/ollama/client_spec.rb".freeze, "spec/ollama/commands/chat_spec.rb".freeze, "spec/ollama/commands/copy_spec.rb".freeze, "spec/ollama/commands/create_spec.rb".freeze, "spec/ollama/commands/delete_spec.rb".freeze, "spec/ollama/commands/embed_spec.rb".freeze, "spec/ollama/commands/embeddings_spec.rb".freeze, "spec/ollama/commands/generate_spec.rb".freeze, "spec/ollama/commands/ps_spec.rb".freeze, "spec/ollama/commands/pull_spec.rb".freeze, "spec/ollama/commands/push_spec.rb".freeze, "spec/ollama/commands/show_spec.rb".freeze, "spec/ollama/commands/tags_spec.rb".freeze, "spec/ollama/documents/memory_cache_spec.rb".freeze, "spec/ollama/documents/redis_cache_spec.rb".freeze, "spec/ollama/documents/splitters/character_spec.rb".freeze, "spec/ollama/documents/splitters/semantic_spec.rb".freeze, "spec/ollama/documents_spec.rb".freeze, "spec/ollama/handlers/collector_spec.rb".freeze, "spec/ollama/handlers/dump_json_spec.rb".freeze, "spec/ollama/handlers/dump_yaml_spec.rb".freeze, "spec/ollama/handlers/markdown_spec.rb".freeze, "spec/ollama/handlers/nop_spec.rb".freeze, "spec/ollama/handlers/print_spec.rb".freeze, "spec/ollama/handlers/progress_spec.rb".freeze, "spec/ollama/handlers/say_spec.rb".freeze, "spec/ollama/handlers/single_spec.rb".freeze, "spec/ollama/image_spec.rb".freeze, "spec/ollama/message_spec.rb".freeze, "spec/ollama/options_spec.rb".freeze, "spec/ollama/tool_spec.rb".freeze, "spec/ollama/utils/ansi_markdown_spec.rb".freeze, "spec/ollama/utils/fetcher_spec.rb".freeze, "spec/ollama/utils/tags_spec.rb".freeze, "spec/spec_helper.rb".freeze, "tmp/.keep".freeze]
+  s.executables = ["ollama_console".freeze, "ollama_chat".freeze, "ollama_update".freeze, "ollama_cli".freeze]
+  s.extra_rdoc_files = ["README.md".freeze, "lib/ollama.rb".freeze, "lib/ollama/client.rb".freeze, "lib/ollama/client/command.rb".freeze, "lib/ollama/client/doc.rb".freeze, "lib/ollama/commands/chat.rb".freeze, "lib/ollama/commands/copy.rb".freeze, "lib/ollama/commands/create.rb".freeze, "lib/ollama/commands/delete.rb".freeze, "lib/ollama/commands/embed.rb".freeze, "lib/ollama/commands/embeddings.rb".freeze, "lib/ollama/commands/generate.rb".freeze, "lib/ollama/commands/ps.rb".freeze, "lib/ollama/commands/pull.rb".freeze, "lib/ollama/commands/push.rb".freeze, "lib/ollama/commands/show.rb".freeze, "lib/ollama/commands/tags.rb".freeze, "lib/ollama/documents.rb".freeze, "lib/ollama/documents/memory_cache.rb".freeze, "lib/ollama/documents/redis_cache.rb".freeze, "lib/ollama/documents/splitters/character.rb".freeze, "lib/ollama/documents/splitters/semantic.rb".freeze, "lib/ollama/dto.rb".freeze, "lib/ollama/errors.rb".freeze, "lib/ollama/handlers.rb".freeze, "lib/ollama/handlers/collector.rb".freeze, "lib/ollama/handlers/concern.rb".freeze, "lib/ollama/handlers/dump_json.rb".freeze, "lib/ollama/handlers/dump_yaml.rb".freeze, "lib/ollama/handlers/markdown.rb".freeze, "lib/ollama/handlers/nop.rb".freeze, "lib/ollama/handlers/print.rb".freeze, "lib/ollama/handlers/progress.rb".freeze, "lib/ollama/handlers/say.rb".freeze, "lib/ollama/handlers/single.rb".freeze, "lib/ollama/image.rb".freeze, "lib/ollama/message.rb".freeze, "lib/ollama/options.rb".freeze, "lib/ollama/response.rb".freeze, "lib/ollama/tool.rb".freeze, "lib/ollama/tool/function.rb".freeze, "lib/ollama/tool/function/parameters.rb".freeze, "lib/ollama/tool/function/parameters/property.rb".freeze, "lib/ollama/utils/ansi_markdown.rb".freeze, "lib/ollama/utils/chooser.rb".freeze, "lib/ollama/utils/colorize_texts.rb".freeze, "lib/ollama/utils/fetcher.rb".freeze, "lib/ollama/utils/file_argument.rb".freeze, "lib/ollama/utils/math.rb".freeze, "lib/ollama/utils/tags.rb".freeze, "lib/ollama/utils/width.rb".freeze, "lib/ollama/version.rb".freeze]
+  s.files = [".envrc".freeze, "CHANGES.md".freeze, "Gemfile".freeze, "LICENSE".freeze, "README.md".freeze, "Rakefile".freeze, "bin/ollama_chat".freeze, "bin/ollama_cli".freeze, "bin/ollama_console".freeze, "bin/ollama_update".freeze, "config/redis.conf".freeze, "docker-compose.yml".freeze, "lib/ollama.rb".freeze, "lib/ollama/client.rb".freeze, "lib/ollama/client/command.rb".freeze, "lib/ollama/client/doc.rb".freeze, "lib/ollama/commands/chat.rb".freeze, "lib/ollama/commands/copy.rb".freeze, "lib/ollama/commands/create.rb".freeze, "lib/ollama/commands/delete.rb".freeze, "lib/ollama/commands/embed.rb".freeze, "lib/ollama/commands/embeddings.rb".freeze, "lib/ollama/commands/generate.rb".freeze, "lib/ollama/commands/ps.rb".freeze, "lib/ollama/commands/pull.rb".freeze, "lib/ollama/commands/push.rb".freeze, "lib/ollama/commands/show.rb".freeze, "lib/ollama/commands/tags.rb".freeze, "lib/ollama/documents.rb".freeze, "lib/ollama/documents/memory_cache.rb".freeze, "lib/ollama/documents/redis_cache.rb".freeze, "lib/ollama/documents/splitters/character.rb".freeze, "lib/ollama/documents/splitters/semantic.rb".freeze, "lib/ollama/dto.rb".freeze, "lib/ollama/errors.rb".freeze, "lib/ollama/handlers.rb".freeze, "lib/ollama/handlers/collector.rb".freeze, "lib/ollama/handlers/concern.rb".freeze, "lib/ollama/handlers/dump_json.rb".freeze, "lib/ollama/handlers/dump_yaml.rb".freeze, "lib/ollama/handlers/markdown.rb".freeze, "lib/ollama/handlers/nop.rb".freeze, "lib/ollama/handlers/print.rb".freeze, "lib/ollama/handlers/progress.rb".freeze, "lib/ollama/handlers/say.rb".freeze, "lib/ollama/handlers/single.rb".freeze, "lib/ollama/image.rb".freeze, "lib/ollama/message.rb".freeze, "lib/ollama/options.rb".freeze, "lib/ollama/response.rb".freeze, "lib/ollama/tool.rb".freeze, "lib/ollama/tool/function.rb".freeze, "lib/ollama/tool/function/parameters.rb".freeze, "lib/ollama/tool/function/parameters/property.rb".freeze, "lib/ollama/utils/ansi_markdown.rb".freeze, "lib/ollama/utils/chooser.rb".freeze, "lib/ollama/utils/colorize_texts.rb".freeze, "lib/ollama/utils/fetcher.rb".freeze, "lib/ollama/utils/file_argument.rb".freeze, "lib/ollama/utils/math.rb".freeze, "lib/ollama/utils/tags.rb".freeze, "lib/ollama/utils/width.rb".freeze, "lib/ollama/version.rb".freeze, "ollama-ruby.gemspec".freeze, "spec/assets/embeddings.json".freeze, "spec/assets/kitten.jpg".freeze, "spec/ollama/client/doc_spec.rb".freeze, "spec/ollama/client_spec.rb".freeze, "spec/ollama/commands/chat_spec.rb".freeze, "spec/ollama/commands/copy_spec.rb".freeze, "spec/ollama/commands/create_spec.rb".freeze, "spec/ollama/commands/delete_spec.rb".freeze, "spec/ollama/commands/embed_spec.rb".freeze, "spec/ollama/commands/embeddings_spec.rb".freeze, "spec/ollama/commands/generate_spec.rb".freeze, "spec/ollama/commands/ps_spec.rb".freeze, "spec/ollama/commands/pull_spec.rb".freeze, "spec/ollama/commands/push_spec.rb".freeze, "spec/ollama/commands/show_spec.rb".freeze, "spec/ollama/commands/tags_spec.rb".freeze, "spec/ollama/documents/memory_cache_spec.rb".freeze, "spec/ollama/documents/redis_cache_spec.rb".freeze, "spec/ollama/documents/splitters/character_spec.rb".freeze, "spec/ollama/documents/splitters/semantic_spec.rb".freeze, "spec/ollama/documents_spec.rb".freeze, "spec/ollama/handlers/collector_spec.rb".freeze, "spec/ollama/handlers/dump_json_spec.rb".freeze, "spec/ollama/handlers/dump_yaml_spec.rb".freeze, "spec/ollama/handlers/markdown_spec.rb".freeze, "spec/ollama/handlers/nop_spec.rb".freeze, "spec/ollama/handlers/print_spec.rb".freeze, "spec/ollama/handlers/progress_spec.rb".freeze, "spec/ollama/handlers/say_spec.rb".freeze, "spec/ollama/handlers/single_spec.rb".freeze, "spec/ollama/image_spec.rb".freeze, "spec/ollama/message_spec.rb".freeze, "spec/ollama/options_spec.rb".freeze, "spec/ollama/tool_spec.rb".freeze, "spec/ollama/utils/ansi_markdown_spec.rb".freeze, "spec/ollama/utils/fetcher_spec.rb".freeze, "spec/ollama/utils/tags_spec.rb".freeze, "spec/spec_helper.rb".freeze, "tmp/.keep".freeze]
   s.homepage = "https://github.com/flori/ollama-ruby".freeze
   s.licenses = ["MIT".freeze]
   s.rdoc_options = ["--title".freeze, "Ollama-ruby - Interacting with the Ollama API".freeze, "--main".freeze, "README.md".freeze]
   s.required_ruby_version = Gem::Requirement.new("~> 3.1".freeze)
-  s.rubygems_version = "3.5.11".freeze
+  s.rubygems_version = "3.5.18".freeze
   s.summary = "Interacting with the Ollama API".freeze
   s.test_files = ["spec/ollama/client/doc_spec.rb".freeze, "spec/ollama/client_spec.rb".freeze, "spec/ollama/commands/chat_spec.rb".freeze, "spec/ollama/commands/copy_spec.rb".freeze, "spec/ollama/commands/create_spec.rb".freeze, "spec/ollama/commands/delete_spec.rb".freeze, "spec/ollama/commands/embed_spec.rb".freeze, "spec/ollama/commands/embeddings_spec.rb".freeze, "spec/ollama/commands/generate_spec.rb".freeze, "spec/ollama/commands/ps_spec.rb".freeze, "spec/ollama/commands/pull_spec.rb".freeze, "spec/ollama/commands/push_spec.rb".freeze, "spec/ollama/commands/show_spec.rb".freeze, "spec/ollama/commands/tags_spec.rb".freeze, "spec/ollama/documents/memory_cache_spec.rb".freeze, "spec/ollama/documents/redis_cache_spec.rb".freeze, "spec/ollama/documents/splitters/character_spec.rb".freeze, "spec/ollama/documents/splitters/semantic_spec.rb".freeze, "spec/ollama/documents_spec.rb".freeze, "spec/ollama/handlers/collector_spec.rb".freeze, "spec/ollama/handlers/dump_json_spec.rb".freeze, "spec/ollama/handlers/dump_yaml_spec.rb".freeze, "spec/ollama/handlers/markdown_spec.rb".freeze, "spec/ollama/handlers/nop_spec.rb".freeze, "spec/ollama/handlers/print_spec.rb".freeze, "spec/ollama/handlers/progress_spec.rb".freeze, "spec/ollama/handlers/say_spec.rb".freeze, "spec/ollama/handlers/single_spec.rb".freeze, "spec/ollama/image_spec.rb".freeze, "spec/ollama/message_spec.rb".freeze, "spec/ollama/options_spec.rb".freeze, "spec/ollama/tool_spec.rb".freeze, "spec/ollama/utils/ansi_markdown_spec.rb".freeze, "spec/ollama/utils/fetcher_spec.rb".freeze, "spec/ollama/utils/tags_spec.rb".freeze, "spec/spec_helper.rb".freeze]
 
@@ -43,4 +43,5 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency(%q<complex_config>.freeze, ["~> 0.20".freeze])
   s.add_runtime_dependency(%q<search_ui>.freeze, ["~> 0.0".freeze])
   s.add_runtime_dependency(%q<amatch>.freeze, ["~> 0.4.1".freeze])
+  s.add_runtime_dependency(%q<pdf-reader>.freeze, ["~> 2.0".freeze])
 end

data/spec/ollama/documents/redis_cache_spec.rb

@@ -74,5 +74,13 @@ RSpec.describe Ollama::Documents::RedisCache do
       expect(redis).to receive(:scan_each).with(match: 'test-*')
       redis_cache.to_a
     end
+
+    it 'can compute prefix with pre' do
+      expect(redis_cache.pre('foo')).to eq 'test-foo'
+    end
+
+    it 'can remove prefix with unpre' do
+      expect(redis_cache.unpre('test-foo')).to eq 'foo'
+    end
   end
 end

data/spec/ollama/documents_spec.rb

@@ -76,6 +76,34 @@ RSpec.describe Ollama::Documents do
     expect(records[0].to_s).to eq '#<Ollama::Documents::Record "foo" #test 1.0>'
   end
 
+  it 'can find strings conditionally' do
+    allow(ollama).to receive(:embed).
+      with(model:, input: [ 'foobar' ], options: nil).
+      and_return(double(embeddings: [ [ 0.01 ] ]))
+    allow(ollama).to receive(:embed).
+      with(model:, input: [ 'foo' ], options: nil).
+      and_return(double(embeddings: [ [ 0.1 ] ]))
+    expect(documents << 'foobar').to eq documents
+    expect(documents << 'foo').to eq documents
+    expect(ollama).to receive(:embed).at_least(:once).
+      with(model:, input: 'foo', options: nil).
+      and_return(double(embeddings: [ [ 0.1 ] ]))
+    records = documents.find_where('foo', text_count: 1)
+    expect(records).to eq [
+      Ollama::Documents::Record[text: 'foo', embedding: [ 0.1 ], similarity: 1.0 ],
+    ]
+    records = documents.find_where('foo', text_size: 3)
+    expect(records).to eq [
+      Ollama::Documents::Record[text: 'foo', embedding: [ 0.1 ], similarity: 1.0 ],
+    ]
+    records = documents.find_where('foo')
+    expect(records).to eq [
+      Ollama::Documents::Record[text: 'foo', embedding: [ 0.1 ], similarity: 1.0 ],
+      Ollama::Documents::Record[text: 'foobar', embedding: [ 0.1 ], similarity: 1.0 ],
+    ]
+  end
+
+
   context 'it uses cache' do
     before do
       allow(ollama).to receive(:embed).
@@ -103,6 +131,20 @@ RSpec.describe Ollama::Documents do
       }.to change { documents.size }.from(1).to(0)
     end
 
+    it 'can clear texts with tags' do
+      allow(ollama).to receive(:embed).
+        with(model:, input: %w[ bar ], options: nil).
+        and_return(double(embeddings: [ [ 0.1 ] ]))
+      expect(documents.add('foo', tags: %i[ test ])).to eq documents
+      expect(documents.add('bar', tags: %i[ test2 ])).to eq documents
+      expect {
+        documents.clear tags: 'test'
+      }.to change { documents.size }.from(2).to(1)
+      expect {
+        documents.clear tags: :test2
+      }.to change { documents.size }.from(1).to(0)
+    end
+
     it 'returns collections' do
       expect(documents.collections).to eq [ :default ]
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ollama-ruby
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Florian Frank
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-09-03 00:00:00.000000000 Z
+date: 2024-09-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: gem_hadar
@@ -276,12 +276,27 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.4.1
+- !ruby/object:Gem::Dependency
+  name: pdf-reader
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
 description: Library that allows interacting with the Ollama API
 email: flori@ping.de
 executables:
 - ollama_console
 - ollama_chat
 - ollama_update
+- ollama_cli
 extensions: []
 extra_rdoc_files:
 - README.md
@@ -331,6 +346,7 @@ extra_rdoc_files:
 - lib/ollama/utils/chooser.rb
 - lib/ollama/utils/colorize_texts.rb
 - lib/ollama/utils/fetcher.rb
+- lib/ollama/utils/file_argument.rb
 - lib/ollama/utils/math.rb
 - lib/ollama/utils/tags.rb
 - lib/ollama/utils/width.rb
@@ -343,6 +359,7 @@ files:
 - README.md
 - Rakefile
 - bin/ollama_chat
+- bin/ollama_cli
 - bin/ollama_console
 - bin/ollama_update
 - config/redis.conf
@@ -393,6 +410,7 @@ files:
 - lib/ollama/utils/chooser.rb
 - lib/ollama/utils/colorize_texts.rb
 - lib/ollama/utils/fetcher.rb
+- lib/ollama/utils/file_argument.rb
 - lib/ollama/utils/math.rb
 - lib/ollama/utils/tags.rb
 - lib/ollama/utils/width.rb
@@ -460,7 +478,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.11
+rubygems_version: 3.5.18
 signing_key:
 specification_version: 4
 summary: Interacting with the Ollama API