ollama-ruby 0.2.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1f80ae8ee6e8acbbedfff8b56923b25b583fc60b96c733985e32908874d542bb
-  data.tar.gz: 80834beb676929f08f4216e373e56cadd12d16c0b50e95b1a599bdd48bf29c86
+  metadata.gz: c0e3428486bf4d60d1e4a64574a9c04bb41da5a5fe76b5f4bc37c5f089221cf3
+  data.tar.gz: 0a2f6f5269c97b22493400448204116aebef642273c024aee474a38741dfdce5
 SHA512:
-  metadata.gz: 94823ec618f940056bcee6ac7d706c633087404efcb49122cdccc89a4596b4a5f14a0d1c1d60fd7bb13a28e10f3ba4dc2027cd96cfe384169b25acaf75f532b7
-  data.tar.gz: ef8d4a7001c5502bc787f074f10dfaf16692b8fa90c4a6e32ae3c0c75b94da45b4691094229106cec72438837eef57b933cd5e5c9b28e63d0d4a598c152a9c32
+  metadata.gz: 3fd8db76a1cad30ba41af287e1abb77a102913f09f1648c9fbaf89f5e3bf4f3dace4a39cc6b72da456b633eb42d331b356134aecc196b112c7f94215177b4023
+  data.tar.gz: 4541a1e93c2714b786cb21b5e401ffc48a08e05c39bfac860e8cdd275a447bbc7db0b6cf80fee5f459f86734320a90362418a50f4174d53ece4e1a9d2f1dc647

data/CHANGES.md

@@ -1,5 +1,71 @@
 # Changes
 
+## 2024-09-12 v0.3.1
+
+* Update dependencies and date in gemspec files:
+  - Updated `complex_config` dependency to '~> 0.22'
+* Refactor FollowChat#eval_stats to add bold eval rates
+  * Improve formatting in eval_stats using bold and color for better
+    readability.
+  * Update import_document and add_image methods to handle nil values
+    correctly.
+  * Update width method in utils/width.rb to use uncolor when checking line
+    length.
+* Refactor eval stats output in FollowChat class
+  - Add indentation to eval stats output for better readability
+* FollowChat evaluation stats refactored
+  - Removed hardcoded eval_stats hash and replaced with method call
+    `eval_stats(response)`
+  - Added new method `eval_stats(response)` to calculate evaluation statistics
+    - Calculates eval duration, prompt eval duration, total duration, and load
+      duration
+    - Adds eval count, prompt eval count, eval rate, and prompt eval rate
+* Use default to_s tree representation of config.
+  * Update complex_config dependency to ~> 0.21, >= 0.21.1 in Rakefile
+  * Update complex_config dependency to ~> 0.21, >= 0.21.1 in
+    ollama-ruby.gemspec
+* Update dependencies and configuration display
+  * Update 'complex_config' dependency to '~> 0.21'
+  * Change OllamaChatConfig to display configuration as a tree instead of yaml
+* Improve /web search command
+  * Update infobar dependency to ~> 0.8
+  * Update /web command to summarize web sources as well as importing them
+    directly
+  * Add /clobber command to clear conversation messages and collection
+* Refactor Ollama chat configuration and summary generation.
+  * Update `OllamaChatConfig` to use `prompts.system` instead of `system`.
+  * Introduce `prompts.summarize` config as template for generating abstract
+    summaries.
+  * Replace hardcoded summary generation with call to `prompts.summarize`.
+  * Display /help for all unknown chat commands starting wit `/`
+
+## 2024-09-05 v0.3.0
+
+* **New Features**
+  * Created new file `ollama_cli` with Ollama CLI functionality.
+  * Added executable `ollama_cli` to s.executables in ollama-ruby.gemspec.
+  * Added `find_where` method in `documents.rb` to filter records by text size
+    and count.
+  * Added test for `find_where` method in `documents_spec.rb`.
+  * Features for `ollama_chat`
+      * Added `found_texts_count` option to `OllamaChatConfig`.
+      * Implemented `parse_rss` method for RSS feeds and `parse_atom` method
+        for Atom feeds.
+      * Added links to titles in RSS feed item summaries and Atom feed item
+        summaries.
+      * Updated `parse_source` method to handle different content types,
+        including HTML, XML, and RSS/Atom feeds.
+      * Added `/web [n] query` command to search web and return n or 1 results
+        in chat interface.
+* **Improvements**
+  * Improved validation for system prompts
+  * Extracted file argument handling into a separate module and method
+  * Added default value for config or model system prompt
+  * Improved input validation for `system_prompt` path
+  * Updated collection clearing logic to accept optional tags parameter
+  * Updated `Tags` class to overload `to_a` method for converting to array of
+    strings
+
 ## 2024-09-03 v0.2.0
 
 ### Changes

data/README.md

@@ -152,19 +152,21 @@ subject - the young, blue-eyed cat.
 The following commands can be given inside the chat, if prefixed by a `/`:
 
 ```
-/paste                             to paste content
-/markdown                          toggle markdown output
-/list                              list the messages of the conversation
-/clear                             clear the conversation messages
-/pop [n]                           pop the last n exchanges, defaults to 1
-/model                             change the model
-/regenerate                        the last answer message
-/collection clear|stats|change|new clear or show stats of current collection
-/summarize source                  summarize the URL/file source's content
-/save filename                     store conversation messages
-/load filename                     load conversation messages
-/quit                              to quit
-/help                              to view this help
+/paste                                   to paste content
+/markdown                                toggle markdown output
+/list                                    list the messages of the conversation
+/clear                                   clear the conversation messages
+/clobber                                 clear conversation messages and collection
+/pop [n]                                 pop the last n exchanges, defaults to 1
+/model                                   change the model
+/regenerate                              the last answer message
+/collection clear [tag]|stats|change|new clear or show stats of current collection
+/summarize source                        summarize the URL/file source's content
+/web [n] query                           query web search & return n or 1 results
+/save filename                           store conversation messages
+/load filename                           load conversation messages
+/quit                                    to quit
+/help                                    to view this help
 ```
 
 ### ollama\_console

data/Rakefile

@@ -18,12 +18,13 @@ GemHadar do
      '.utilsrc', '.rspec', *Dir.glob('.github/**/*', File::FNM_DOTMATCH)
   readme      'README.md'
 
-  executables << 'ollama_console' << 'ollama_chat' << 'ollama_update'
+  executables << 'ollama_console' << 'ollama_chat' <<
+    'ollama_update' << 'ollama_cli'
 
   required_ruby_version  '~> 3.1'
 
   dependency             'excon',                 '~> 0.111'
-  dependency             'infobar',               '~> 0.7'
+  dependency             'infobar',               '~> 0.8'
   dependency             'term-ansicolor',        '~> 1.11'
   dependency             'kramdown-parser-gfm',   '~> 1.1'
   dependency             'terminal-table',        '~> 3.0'
@@ -33,9 +34,10 @@ GemHadar do
   dependency             'sorted_set',            '~> 1.0'
   dependency             'mime-types',            '~> 3.0'
   dependency             'reverse_markdown',      '~> 2.0'
-  dependency             'complex_config',        '~> 0.20'
+  dependency             'complex_config',        '~> 0.22'
   dependency             'search_ui',             '~> 0.0'
   dependency             'amatch',                '~> 0.4.1'
+  dependency             'pdf-reader',            '~> 2.0'
   development_dependency 'all_images',            '~> 0.4'
   development_dependency 'rspec',                 '~> 3.2'
   development_dependency 'utils'

data/bin/ollama_chat

@@ -12,6 +12,8 @@ require 'complex_config'
 require 'fileutils'
 require 'uri'
 require 'nokogiri'
+require 'rss'
+require 'pdf/reader'
 
 class OllamaChatConfig
   include ComplexConfig
@@ -24,7 +26,12 @@ class OllamaChatConfig
       name: <%= ENV.fetch('OLLAMA_CHAT_MODEL', 'llama3.1') %>
       options:
         num_ctx: 8192
-    system: <%= ENV.fetch('OLLAMA_CHAT_SYSTEM', 'null') %>
+    prompts:
+      system: <%= ENV.fetch('OLLAMA_CHAT_SYSTEM', 'null') %>
+      summarize: |
+        Generate an abstract summary of the content in this document:
+
+        %s
     voice: Samantha
     markdown: true
     embedding:
@@ -36,6 +43,7 @@ class OllamaChatConfig
         prompt: 'Represent this sentence for searching relevant passages: %s'
       collection: <%= ENV.fetch('OLLAMA_CHAT_COLLECTION', 'ollama_chat') %>
       found_texts_size: 4096
+      found_texts_count: null
       splitter:
         name: RecursiveCharacter
         chunk_size: 1024
@@ -47,7 +55,7 @@ class OllamaChatConfig
 
   def initialize(filename = nil)
     @filename = filename || default_path
-    @config = Provider.config(@filename)
+    @config = Provider.config(@filename, '⚙️')
     retried = false
   rescue ConfigurationFileMissing
     if @filename == default_path && !retried
@@ -112,19 +120,28 @@ class FollowChat
       @say.call(response)
     end
     if response.done
-      @output.puts
-      eval_stats = {
-        eval_duration:        Tins::Duration.new(response.eval_duration / 1e9),
-        eval_count:           response.eval_count,
-        prompt_eval_duration: Tins::Duration.new(response.prompt_eval_duration / 1e9),
-        prompt_eval_count:    response.prompt_eval_count,
-        total_duration:       Tins::Duration.new(response.total_duration / 1e9),
-        load_duration:        Tins::Duration.new(response.load_duration / 1e9),
-      }.map { _1 * '=' } * ' '
-      @output.puts '📊 ' + color(111) { Utils::Width.wrap(eval_stats, percentage: 90) }
+      @output.puts "", eval_stats(response)
     end
     self
   end
+
+  def eval_stats(response)
+    eval_duration = response.eval_duration / 1e9
+    prompt_eval_duration = response.prompt_eval_duration / 1e9
+    stats_text = {
+      eval_duration:        Tins::Duration.new(eval_duration),
+      eval_count:           response.eval_count,
+      eval_rate:            bold { "%.2f c/s" % (response.eval_count / eval_duration) } + color(111),
+      prompt_eval_duration: Tins::Duration.new(prompt_eval_duration),
+      prompt_eval_count:    response.prompt_eval_count,
+      prompt_eval_rate:     bold { "%.2f c/s" % (response.prompt_eval_count / prompt_eval_duration) } + color(111),
+      total_duration:       Tins::Duration.new(response.total_duration / 1e9),
+      load_duration:        Tins::Duration.new(response.load_duration / 1e9),
+    }.map { _1 * '=' } * ' '
+    '📊 ' + color(111) {
+      Utils::Width.wrap(stats_text, percentage: 90).gsub(/(?<!\A)^/, '   ')
+    }
+  end
 end
 
 def search_web(query, n = 5)
@@ -153,7 +170,8 @@ end
 
 def pull_model_unless_present(model, options, retried = false)
   ollama.show(name: model) { |response|
-    puts "Model #{bold{model}} with architecture #{response.model_info['general.architecture']} found."
+    puts "Model #{bold{model}} with architecture "\
+      "#{response.model_info['general.architecture']} found."
     if system = response.system
       puts "Configured model system prompt is:\n#{italic { system }}"
       return system
@@ -226,17 +244,77 @@ def list_conversation(messages, markdown)
   end
 end
 
+def reverse_markdown(html)
+  ReverseMarkdown.convert(
+    html,
+    unknown_tags: :bypass,
+    github_flavored: true,
+    tag_border: ''
+  )
+end
+
+def parse_rss(source_io)
+  feed = RSS::Parser.parse(source_io, false, false)
+  title = <<~end
+    # #{feed&.channel&.title}
+
+  end
+  feed.items.inject(title) do |text, item|
+    text << <<~end
+      ## [#{item&.title}](#{item&.link})
+
+      updated on #{item&.pubDate}
+
+      #{reverse_markdown(item&.description)}
+
+    end
+  end
+end
+
+def parse_atom(source_io)
+  feed = RSS::Parser.parse(source_io, false, false)
+  title = <<~end
+    # #{feed.title.content}
+
+  end
+  feed.items.inject(title) do |text, item|
+    text << <<~end
+      ## [#{item&.title&.content}](#{item&.link&.href})
+
+      updated on #{item&.updated&.content}
+
+      #{reverse_markdown(item&.content&.content)}
+
+    end
+  end
+end
+
 def parse_source(source_io)
-  case source_io&.content_type&.sub_type
-  when 'html'
-    ReverseMarkdown.convert(
-      source_io.read,
-      unknown_tags: :bypass,
-      github_flavored: true,
-      tag_border: ''
-    )
-  when 'plain', 'csv', 'xml'
+  case source_io&.content_type
+  when 'text/html'
+    reverse_markdown(source_io.read)
+  when 'text/xml'
+    if source_io.readline =~ %r(^\s*<rss\s)
+      source_io.rewind
+      return parse_rss(source_io)
+    end
+    source_io.rewind
+    source_io.read
+  when %r(\Atext/)
     source_io.read
+  when 'application/rss+xml'
+    parse_rss(source_io)
+  when 'application/atom+xml'
+    parse_atom(source_io)
+  when 'application/json'
+    source_io.read
+  when 'application/pdf'
+    reader = PDF::Reader.new(source_io)
+    result = +''
+    reader.pages.each do |page|
+      result << page.text
+    end
+    result
   else
     STDERR.puts "Cannot import #{source_io&.content_type} document."
     return
@@ -248,7 +326,7 @@ def import_document(source_io, source)
     STDOUT.puts "Embedding disabled, I won't import any documents, try: /summarize"
     return
   end
-  infobar.puts "Importing #{italic { source_io.content_type }} document #{source.to_s.inspect}."
+  puts "Importing #{italic { source_io&.content_type }} document #{source.to_s.inspect}."
   text = parse_source(source_io) or return
   text.downcase!
   splitter_config = $config.embedding.splitter
@@ -276,7 +354,7 @@ def import_document(source_io, source)
 end
 
 def add_image(images, source_io, source)
-  STDERR.puts "Adding #{source_io.content_type} image #{source.to_s.inspect}."
+  STDERR.puts "Adding #{source_io&.content_type} image #{source.to_s.inspect}."
   image = Image.for_io(source_io, path: source.to_s)
   (images << image).uniq!
 end
@@ -301,16 +379,12 @@ rescue => e
 end
 
 def summarize(source)
-  puts  "Now summarizing #{source.inspect}."
+  puts "Now summarizing #{source.to_s.inspect}."
   source_content =
     fetch_source(source) do |source_io|
       parse_source(source_io) or return
     end
-  <<~end
-    # Generate an abstract summary of the content in this document:
-
-    #{source_content}
-  end
+  $config.prompts.summarize % source_content
 end
 
 def parse_content(content, images)
@@ -327,7 +401,7 @@ def parse_content(content, images)
         case source_io&.content_type&.media_type
         when 'image'
           add_image(images, source_io, source)
-        when 'text'
+        when 'text', 'application'
           import_document(source_io, source)
         else
           STDERR.puts(
@@ -391,19 +465,21 @@ end
 
 def display_chat_help
   puts <<~end
-    /paste                             to paste content
-    /markdown                          toggle markdown output
-    /list                              list the messages of the conversation
-    /clear                             clear the conversation messages
-    /pop [n]                           pop the last n exchanges, defaults to 1
-    /model                             change the model
-    /regenerate                        the last answer message
-    /collection clear|stats|change|new clear or show stats of current collection
-    /summarize source                  summarize the URL/file source's content
-    /save filename                     store conversation messages
-    /load filename                     load conversation messages
-    /quit                              to quit
-    /help                              to view this help
+    /paste                                   to paste content
+    /markdown                                toggle markdown output
+    /list                                    list the messages of the conversation
+    /clear                                   clear the conversation messages
+    /clobber                                 clear conversation messages and collection
+    /pop [n]                                 pop the last n exchanges, defaults to 1
+    /model                                   change the model
+    /regenerate                              the last answer message
+    /collection clear [tag]|stats|change|new clear or show stats of current collection
+    /summarize source                        summarize the URL/file source's content
+    /web [n] query                           query web search & return n or 1 results
+    /save filename                           store conversation messages
+    /load filename                           load conversation messages
+    /quit                                    to quit
+    /help                                    to view this help
   end
 end
 
@@ -436,8 +512,7 @@ $config = config.config
 
 opts[?h] and usage
 
-puts "Configuration read from #{config.filename.inspect} is:"
-y $config.to_h
+puts "Configuration read from #{config.filename.inspect} is:", $config
 
 base_url = opts[?u] || $config.url
 $ollama      = Client.new(base_url:, debug: $config.debug)
@@ -475,7 +550,7 @@ if $config.embedding.enabled
         File.expand_path(doc)
       end
     end
-    infobar.puts "Collection #{bold{collection}}: Adding #{document_list.size} documents…"
+    puts "Collection #{bold{collection}}: Adding #{document_list.size} documents…"
     document_list.each_slice(25) do |docs|
       docs.each do |doc|
         fetch_source(doc) do |doc_io|
@@ -497,23 +572,16 @@ markdown = set_markdown($config.markdown)
 if opts[?c]
   messages.concat load_conversation(opts[?c])
 else
-  system = nil
-  if system_prompt_file = opts[?s]
-    system = File.read(system_prompt_file)
-  end
-  system ||= $config.system
-
-  if system
+  if system = Ollama::Utils::FileArgument.
+      get_file_argument(opts[?s], default: $config.prompts.system? || model_system)
     messages << Message.new(role: 'system', content: system)
     puts "Configured system prompt is:\n#{italic { system }}"
-  elsif model_system.present?
-    puts "Using model system prompt."
   end
 end
 
 puts "\nType /help to display the chat help."
 
-images           = []
+images = []
 loop do
   parse_content = true
   input_prompt = bold { color(172) { message_type(images) + " user" } } + bold { "> " }
@@ -526,7 +594,7 @@ loop do
   when %r(^/quit$)
     puts "Goodbye."
     exit 0
-  when %r(^/markdown)
+  when %r(^/markdown$)
     markdown = set_markdown(!markdown)
     next
   when %r(^/list$)
@@ -536,11 +604,23 @@ loop do
     messages.clear
     puts "Cleared messages."
     next
-  when %r(^/collection (clear|stats|change|new)$)
-    case $1
+  when %r(^/clobber$)
+    messages.clear
+    $documents.clear
+    puts "Cleared messages and collection."
+    next
+  when %r(^/collection\s+(clear|stats|change|new)(?:\s+(.+))?$)
+    command, arg = $1, $2
+    case command
     when 'clear'
-      $documents.clear
-      puts "Cleared collection #{bold{collection}}."
+      tags = arg.present? ? arg.sub(/\A#*/, '') : nil
+      if tags
+        $documents.clear(tags:)
+        puts "Cleared tag ##{tags} from collection #{bold{collection}}."
+      else
+        $documents.clear
+        puts "Cleared collection #{bold{collection}}."
+      end
     when 'stats'
       collection_stats
     when 'change'
@@ -571,13 +651,19 @@ loop do
   when %r(^/summarize\s+(.+))
     parse_content = false
     content       = summarize($1) or next
-  when %r(^/web\s+(?:(\d+)\s+)(.+)$)
-    parse_content = true
-    urls = search_web($2, $1.to_i)
+  when %r(^/web\s+(?:(\d+)\s+)?(.+))
+    parse_content   = false
+    urls            = search_web($2, $1.to_i)
+    urls.each do |url|
+      fetch_source(url) do |url_io|
+        import_document(url_io, url)
+      end
+    end
+    urls_summarized = urls.map { summarize(_1) }
     content = <<~end
-      Answer the the query #{$2.inspect} using these sources:
+      Answer the the query #{$2.inspect} using these sources and summaries:
 
-      #{urls * ?\n}
+      #{urls.zip(urls_summarized).map { |u, s| "%s as \n:%s" % [ u, s ] } * "\n\n"}
     end
   when %r(^/save\s+(.+)$)
     save_conversation($1, messages)
@@ -587,7 +673,7 @@ loop do
     messages = load_conversation($1)
     puts "Loaded conversation from #$1."
     next
-  when %r(^/help$)
+  when %r(^/)
     display_chat_help
     next
   when nil, ''
@@ -602,15 +688,13 @@ loop do
                   end
 
   if $config.embedding.enabled && content
-    records = $documents.find(
+    records = $documents.find_where(
       content.downcase,
       tags:,
-      prompt: $config.embedding.model.prompt?
+      prompt:     $config.embedding.model.prompt?,
+      text_size:  $config.embedding.found_texts_size?,
+      text_count: $config.embedding.found_texts_count?,
     )
-    s, found_texts_size = 0, $config.embedding.found_texts_size
-    records = records.take_while {
-      (s += _1.text.size) <= found_texts_size
-    }
     found_texts = records.map(&:text)
     unless found_texts.empty?
       content += "\nConsider these chunks for your answer:\n"\

data/bin/ollama_cli

@@ -0,0 +1,68 @@
+#!/usr/bin/env ruby
+
+require 'ollama'
+include Ollama
+include Ollama::Utils::FileArgument
+require 'tins'
+include Tins::GO
+require 'json'
+
+def usage
+  puts <<~end
+    #{File.basename($0)} [OPTIONS]
+
+      -u URL         the ollama base url, OLLAMA_URL
+      -m MODEL       the ollama model to chat with, OLLAMA_MODEL
+      -M OPTIONS     the ollama model options to use, OLLAMA_MODEL_OPTIONS
+      -s SYSTEM      the system prompt to use as a file, OLLAMA_SYSTEM
+      -p PROMPT      the user prompt to use as a file, OLLAMA_PROMPT
+      -H HANDLER     the handler to use for the response, defaults to Print
+      -S             use streaming for generation
+      -h             this help
+
+  end
+  exit 0
+end
+
+opts = go 'u:m:M:s:p:H:Sh', defaults: { ?H => 'Print', ?M => '{}' }
+
+opts[?h] and usage
+
+base_url = opts[?u] || ENV['OLLAMA_URL'] || 'http://%s' % ENV.fetch('OLLAMA_HOST')
+model    = opts[?m] || ENV.fetch('OLLAMA_MODEL', 'llama3.1')
+options  = Ollama::Options.from_hash(JSON(
+  get_file_argument(opts[?M], default: ENV['OLLAMA_MODEL_OPTIONS'])
+))
+system   = get_file_argument(opts[?s], default: ENV['OLLAMA_SYSTEM'])
+prompt   = get_file_argument(opts[?p], default: ENV['OLLAMA_PROMPT'])
+
+if prompt.nil?
+  prompt = STDIN.read
+elsif c = prompt.scan('%s').size
+  case c
+  when 0
+  when 1
+    prompt = prompt % STDIN.read
+  else
+    STDERR.puts "Found more than one plaeceholder %s. => Ignoring."
+  end
+end
+
+if ENV['DEBUG'].to_i == 1
+  puts <<~EOT
+    base_url = #{base_url.inspect}
+    model    = #{model.inspect}
+    system   = #{system.inspect}
+    prompt   = #{prompt.inspect}
+    options  = #{options.to_json}
+  EOT
+end
+
+Client.new(base_url:, read_timeout: 120).generate(
+  model:,
+  system:,
+  prompt:,
+  options:,
+  stream: !!opts[?S],
+  &Object.const_get(opts[?H])
+)

data/bin/ollama_console

@@ -5,8 +5,13 @@ include Ollama
 require 'irb'
 require 'irb/history'
 
-base_url = ENV['OLLAMA_URL'] || 'http://%s' % ENV.fetch('OLLAMA_HOST')
-ollama = Client.new(base_url:)
+def base_url
+  ENV['OLLAMA_URL'] || 'http://%s' % ENV.fetch('OLLAMA_HOST')
+end
+
+def ollama
+  $ollama ||= Client.new(base_url:)
+end
 IRB.setup nil
 IRB.conf[:MAIN_CONTEXT] = IRB::Irb.new.context
 IRB.conf[:HISTORY_FILE] = File.join(ENV.fetch('HOME'), '.ollama_console-history')

data/lib/ollama/documents/memory_cache.rb

@@ -36,9 +36,11 @@ class Ollama::Documents::MemoryCache
   end
   include Enumerable
 
-  private
-
   def pre(key)
     [ @prefix, key ].join
   end
+
+  def unpre(key)
+    key.sub(/\A#@prefix/, '')
+  end
 end

data/lib/ollama/documents/redis_cache.rb

@@ -13,7 +13,10 @@ class Ollama::Documents::RedisCache
   end
 
   def [](key)
-    JSON(redis.get(pre(key)), object_class: Ollama::Documents::Record)
+    value = redis.get(pre(key))
+    unless value.nil?
+      JSON(value, object_class: Ollama::Documents::Record)
+    end
   end
 
   def []=(key, value)
@@ -45,8 +48,6 @@ class Ollama::Documents::RedisCache
   end
   include Enumerable
 
-  private
-
   def pre(key)
     [ @prefix, key ].join
   end

data/lib/ollama/documents.rb

@@ -12,6 +12,7 @@ require 'ollama/documents/splitters/semantic'
 
 class Ollama::Documents
   include Ollama::Utils::Math
+  include Ollama::Utils::Width
 
   class Record < JSON::GenericObject
     def to_s
@@ -42,7 +43,7 @@ class Ollama::Documents
   def add(inputs, batch_size: 10, source: nil, tags: [])
     inputs = Array(inputs)
     tags   = Ollama::Utils::Tags.new(tags)
-    source and tags.add File.basename(source)
+    source and tags.add File.basename(source).gsub(/\?.*/, '')
     inputs.map! { |i|
       text = i.respond_to?(:read) ? i.read : i.to_s
       text
@@ -51,7 +52,7 @@ class Ollama::Documents
     inputs.empty? and return self
     batches = inputs.each_slice(batch_size).
       with_infobar(
-        label: "Add #{tags}",
+        label: "Add #{truncate(tags.to_s, percentage: 25)}",
         total: inputs.size
       )
     batches.each do |batch|
@@ -87,8 +88,18 @@ class Ollama::Documents
     @cache.size
   end
 
-  def clear
-    @cache.clear
+  def clear(tags: nil)
+    if tags
+      tags = Ollama::Utils::Tags.new(Array(tags)).to_a
+      @cache.each do |key, record|
+        if (tags & record.tags).size >= 1
+          @cache.delete(@cache.unpre(key))
+        end
+      end
+    else
+      @cache.clear
+    end
+    self
   end
 
   def find(string, tags: nil, prompt: nil)
@@ -96,7 +107,7 @@ class Ollama::Documents
     needle_norm = norm(needle)
     records = @cache
     if tags
-      tags = Ollama::Utils::Tags.new(tags)
+      tags = Ollama::Utils::Tags.new(tags).to_a
       records = records.select { |_key, record| (tags & record.tags).size >= 1 }
     end
     records = records.sort_by { |key, record|
@@ -111,6 +122,20 @@ class Ollama::Documents
     records.transpose.last&.reverse.to_a
   end
 
+  def find_where(string, text_size: nil, text_count: nil, **opts)
+    records = find(string, **opts)
+    size, count = 0, 0
+    records.take_while do |record|
+      if text_size and (size += record.text.size) > text_size
+        next false
+      end
+      if text_count and (count += 1) > text_count
+        next false
+      end
+      true
+    end
+  end
+
   def collections
     case @cache
     when MemoryCache

data/lib/ollama/utils/file_argument.rb

@@ -0,0 +1,16 @@
+module Ollama::Utils::FileArgument
+  module_function
+
+  def get_file_argument(prompt, default: nil)
+    if prompt.present? && prompt.size < 2 ** 15 &&
+        File.basename(prompt).size < 2 ** 8 &&
+        File.exist?(prompt)
+      then
+      File.read(prompt)
+    elsif prompt.present?
+      prompt
+    else
+      default
+    end
+  end
+end

data/lib/ollama/utils/tags.rb

@@ -1,6 +1,10 @@
 require 'sorted_set'
 
 class Ollama::Utils::Tags < SortedSet
+  def to_a
+    super.map(&:to_s)
+  end
+
   def to_s
     map { |t| '#%s' % t } * ' '
   end

data/lib/ollama/utils/width.rb

@@ -1,6 +1,8 @@
 require 'tins/terminal'
 
 module Ollama::Utils::Width
+  include Term::ANSIColor
+
   module_function
 
   def width(percentage: 100.0)
@@ -12,11 +14,24 @@ module Ollama::Utils::Width
       raise ArgumentError, "either pass percentage or length argument"
     percentage and length ||= width(percentage:)
     text.gsub(/(?<!\n)\n(?!\n)/, ' ').lines.map do |line|
-      if length >= 1 && line.length > length
+      if length >= 1 && uncolor { line }.length > length
         line.gsub(/(.{1,#{length}})(\s+|$)/, "\\1\n").strip
       else
         line.strip
       end
     end * ?\n
   end
+
+  def truncate(text, percentage: nil, length: nil, ellipsis: ?…)
+    percentage.nil? ^ length.nil? or
+      raise ArgumentError, "either pass percentage or length argument"
+    percentage and length ||= width(percentage:)
+    if length < 1
+      +''
+    elsif text.size > length
+      text[0, length - 1] + ?…
+    else
+      text
+    end
+  end
 end

data/lib/ollama/version.rb

@@ -1,6 +1,6 @@
 module Ollama
   # Ollama version
-  VERSION         = '0.2.0'
+  VERSION         = '0.3.1'
   VERSION_ARRAY   = VERSION.split('.').map(&:to_i) # :nodoc:
   VERSION_MAJOR   = VERSION_ARRAY[0] # :nodoc:
   VERSION_MINOR   = VERSION_ARRAY[1] # :nodoc:

data/lib/ollama.rb

@@ -18,6 +18,7 @@ require 'ollama/utils/math'
 require 'ollama/utils/colorize_texts'
 require 'ollama/utils/fetcher'
 require 'ollama/utils/chooser'
+require 'ollama/utils/file_argument'
 
 require 'ollama/version'
 require 'ollama/errors'

data/ollama-ruby.gemspec

@@ -1,36 +1,36 @@
 # -*- encoding: utf-8 -*-
-# stub: ollama-ruby 0.2.0 ruby lib
+# stub: ollama-ruby 0.3.1 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "ollama-ruby".freeze
-  s.version = "0.2.0".freeze
+  s.version = "0.3.1".freeze
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
   s.authors = ["Florian Frank".freeze]
-  s.date = "2024-09-03"
+  s.date = "2024-09-12"
   s.description = "Library that allows interacting with the Ollama API".freeze
   s.email = "flori@ping.de".freeze
-  s.executables = ["ollama_console".freeze, "ollama_chat".freeze, "ollama_update".freeze]
-  s.extra_rdoc_files = ["README.md".freeze, "lib/ollama.rb".freeze, "lib/ollama/client.rb".freeze, "lib/ollama/client/command.rb".freeze, "lib/ollama/client/doc.rb".freeze, "lib/ollama/commands/chat.rb".freeze, "lib/ollama/commands/copy.rb".freeze, "lib/ollama/commands/create.rb".freeze, "lib/ollama/commands/delete.rb".freeze, "lib/ollama/commands/embed.rb".freeze, "lib/ollama/commands/embeddings.rb".freeze, "lib/ollama/commands/generate.rb".freeze, "lib/ollama/commands/ps.rb".freeze, "lib/ollama/commands/pull.rb".freeze, "lib/ollama/commands/push.rb".freeze, "lib/ollama/commands/show.rb".freeze, "lib/ollama/commands/tags.rb".freeze, "lib/ollama/documents.rb".freeze, "lib/ollama/documents/memory_cache.rb".freeze, "lib/ollama/documents/redis_cache.rb".freeze, "lib/ollama/documents/splitters/character.rb".freeze, "lib/ollama/documents/splitters/semantic.rb".freeze, "lib/ollama/dto.rb".freeze, "lib/ollama/errors.rb".freeze, "lib/ollama/handlers.rb".freeze, "lib/ollama/handlers/collector.rb".freeze, "lib/ollama/handlers/concern.rb".freeze, "lib/ollama/handlers/dump_json.rb".freeze, "lib/ollama/handlers/dump_yaml.rb".freeze, "lib/ollama/handlers/markdown.rb".freeze, "lib/ollama/handlers/nop.rb".freeze, "lib/ollama/handlers/print.rb".freeze, "lib/ollama/handlers/progress.rb".freeze, "lib/ollama/handlers/say.rb".freeze, "lib/ollama/handlers/single.rb".freeze, "lib/ollama/image.rb".freeze, "lib/ollama/message.rb".freeze, "lib/ollama/options.rb".freeze, "lib/ollama/response.rb".freeze, "lib/ollama/tool.rb".freeze, "lib/ollama/tool/function.rb".freeze, "lib/ollama/tool/function/parameters.rb".freeze, "lib/ollama/tool/function/parameters/property.rb".freeze, "lib/ollama/utils/ansi_markdown.rb".freeze, "lib/ollama/utils/chooser.rb".freeze, "lib/ollama/utils/colorize_texts.rb".freeze, "lib/ollama/utils/fetcher.rb".freeze, "lib/ollama/utils/math.rb".freeze, "lib/ollama/utils/tags.rb".freeze, "lib/ollama/utils/width.rb".freeze, "lib/ollama/version.rb".freeze]
-  s.files = [".envrc".freeze, "CHANGES.md".freeze, "Gemfile".freeze, "LICENSE".freeze, "README.md".freeze, "Rakefile".freeze, "bin/ollama_chat".freeze, "bin/ollama_console".freeze, "bin/ollama_update".freeze, "config/redis.conf".freeze, "docker-compose.yml".freeze, "lib/ollama.rb".freeze, "lib/ollama/client.rb".freeze, "lib/ollama/client/command.rb".freeze, "lib/ollama/client/doc.rb".freeze, "lib/ollama/commands/chat.rb".freeze, "lib/ollama/commands/copy.rb".freeze, "lib/ollama/commands/create.rb".freeze, "lib/ollama/commands/delete.rb".freeze, "lib/ollama/commands/embed.rb".freeze, "lib/ollama/commands/embeddings.rb".freeze, "lib/ollama/commands/generate.rb".freeze, "lib/ollama/commands/ps.rb".freeze, "lib/ollama/commands/pull.rb".freeze, "lib/ollama/commands/push.rb".freeze, "lib/ollama/commands/show.rb".freeze, "lib/ollama/commands/tags.rb".freeze, "lib/ollama/documents.rb".freeze, "lib/ollama/documents/memory_cache.rb".freeze, "lib/ollama/documents/redis_cache.rb".freeze, "lib/ollama/documents/splitters/character.rb".freeze, "lib/ollama/documents/splitters/semantic.rb".freeze, "lib/ollama/dto.rb".freeze, "lib/ollama/errors.rb".freeze, "lib/ollama/handlers.rb".freeze, "lib/ollama/handlers/collector.rb".freeze, "lib/ollama/handlers/concern.rb".freeze, "lib/ollama/handlers/dump_json.rb".freeze, "lib/ollama/handlers/dump_yaml.rb".freeze, "lib/ollama/handlers/markdown.rb".freeze, "lib/ollama/handlers/nop.rb".freeze, "lib/ollama/handlers/print.rb".freeze, "lib/ollama/handlers/progress.rb".freeze, "lib/ollama/handlers/say.rb".freeze, "lib/ollama/handlers/single.rb".freeze, "lib/ollama/image.rb".freeze, "lib/ollama/message.rb".freeze, "lib/ollama/options.rb".freeze, "lib/ollama/response.rb".freeze, "lib/ollama/tool.rb".freeze, "lib/ollama/tool/function.rb".freeze, "lib/ollama/tool/function/parameters.rb".freeze, "lib/ollama/tool/function/parameters/property.rb".freeze, "lib/ollama/utils/ansi_markdown.rb".freeze, "lib/ollama/utils/chooser.rb".freeze, "lib/ollama/utils/colorize_texts.rb".freeze, "lib/ollama/utils/fetcher.rb".freeze, "lib/ollama/utils/math.rb".freeze, "lib/ollama/utils/tags.rb".freeze, "lib/ollama/utils/width.rb".freeze, "lib/ollama/version.rb".freeze, "ollama-ruby.gemspec".freeze, "spec/assets/embeddings.json".freeze, "spec/assets/kitten.jpg".freeze, "spec/ollama/client/doc_spec.rb".freeze, "spec/ollama/client_spec.rb".freeze, "spec/ollama/commands/chat_spec.rb".freeze, "spec/ollama/commands/copy_spec.rb".freeze, "spec/ollama/commands/create_spec.rb".freeze, "spec/ollama/commands/delete_spec.rb".freeze, "spec/ollama/commands/embed_spec.rb".freeze, "spec/ollama/commands/embeddings_spec.rb".freeze, "spec/ollama/commands/generate_spec.rb".freeze, "spec/ollama/commands/ps_spec.rb".freeze, "spec/ollama/commands/pull_spec.rb".freeze, "spec/ollama/commands/push_spec.rb".freeze, "spec/ollama/commands/show_spec.rb".freeze, "spec/ollama/commands/tags_spec.rb".freeze, "spec/ollama/documents/memory_cache_spec.rb".freeze, "spec/ollama/documents/redis_cache_spec.rb".freeze, "spec/ollama/documents/splitters/character_spec.rb".freeze, "spec/ollama/documents/splitters/semantic_spec.rb".freeze, "spec/ollama/documents_spec.rb".freeze, "spec/ollama/handlers/collector_spec.rb".freeze, "spec/ollama/handlers/dump_json_spec.rb".freeze, "spec/ollama/handlers/dump_yaml_spec.rb".freeze, "spec/ollama/handlers/markdown_spec.rb".freeze, "spec/ollama/handlers/nop_spec.rb".freeze, "spec/ollama/handlers/print_spec.rb".freeze, "spec/ollama/handlers/progress_spec.rb".freeze, "spec/ollama/handlers/say_spec.rb".freeze, "spec/ollama/handlers/single_spec.rb".freeze, "spec/ollama/image_spec.rb".freeze, "spec/ollama/message_spec.rb".freeze, "spec/ollama/options_spec.rb".freeze, "spec/ollama/tool_spec.rb".freeze, "spec/ollama/utils/ansi_markdown_spec.rb".freeze, "spec/ollama/utils/fetcher_spec.rb".freeze, "spec/ollama/utils/tags_spec.rb".freeze, "spec/spec_helper.rb".freeze, "tmp/.keep".freeze]
+  s.executables = ["ollama_console".freeze, "ollama_chat".freeze, "ollama_update".freeze, "ollama_cli".freeze]
+  s.extra_rdoc_files = ["README.md".freeze, "lib/ollama.rb".freeze, "lib/ollama/client.rb".freeze, "lib/ollama/client/command.rb".freeze, "lib/ollama/client/doc.rb".freeze, "lib/ollama/commands/chat.rb".freeze, "lib/ollama/commands/copy.rb".freeze, "lib/ollama/commands/create.rb".freeze, "lib/ollama/commands/delete.rb".freeze, "lib/ollama/commands/embed.rb".freeze, "lib/ollama/commands/embeddings.rb".freeze, "lib/ollama/commands/generate.rb".freeze, "lib/ollama/commands/ps.rb".freeze, "lib/ollama/commands/pull.rb".freeze, "lib/ollama/commands/push.rb".freeze, "lib/ollama/commands/show.rb".freeze, "lib/ollama/commands/tags.rb".freeze, "lib/ollama/documents.rb".freeze, "lib/ollama/documents/memory_cache.rb".freeze, "lib/ollama/documents/redis_cache.rb".freeze, "lib/ollama/documents/splitters/character.rb".freeze, "lib/ollama/documents/splitters/semantic.rb".freeze, "lib/ollama/dto.rb".freeze, "lib/ollama/errors.rb".freeze, "lib/ollama/handlers.rb".freeze, "lib/ollama/handlers/collector.rb".freeze, "lib/ollama/handlers/concern.rb".freeze, "lib/ollama/handlers/dump_json.rb".freeze, "lib/ollama/handlers/dump_yaml.rb".freeze, "lib/ollama/handlers/markdown.rb".freeze, "lib/ollama/handlers/nop.rb".freeze, "lib/ollama/handlers/print.rb".freeze, "lib/ollama/handlers/progress.rb".freeze, "lib/ollama/handlers/say.rb".freeze, "lib/ollama/handlers/single.rb".freeze, "lib/ollama/image.rb".freeze, "lib/ollama/message.rb".freeze, "lib/ollama/options.rb".freeze, "lib/ollama/response.rb".freeze, "lib/ollama/tool.rb".freeze, "lib/ollama/tool/function.rb".freeze, "lib/ollama/tool/function/parameters.rb".freeze, "lib/ollama/tool/function/parameters/property.rb".freeze, "lib/ollama/utils/ansi_markdown.rb".freeze, "lib/ollama/utils/chooser.rb".freeze, "lib/ollama/utils/colorize_texts.rb".freeze, "lib/ollama/utils/fetcher.rb".freeze, "lib/ollama/utils/file_argument.rb".freeze, "lib/ollama/utils/math.rb".freeze, "lib/ollama/utils/tags.rb".freeze, "lib/ollama/utils/width.rb".freeze, "lib/ollama/version.rb".freeze]
+  s.files = [".envrc".freeze, "CHANGES.md".freeze, "Gemfile".freeze, "LICENSE".freeze, "README.md".freeze, "Rakefile".freeze, "bin/ollama_chat".freeze, "bin/ollama_cli".freeze, "bin/ollama_console".freeze, "bin/ollama_update".freeze, "config/redis.conf".freeze, "docker-compose.yml".freeze, "lib/ollama.rb".freeze, "lib/ollama/client.rb".freeze, "lib/ollama/client/command.rb".freeze, "lib/ollama/client/doc.rb".freeze, "lib/ollama/commands/chat.rb".freeze, "lib/ollama/commands/copy.rb".freeze, "lib/ollama/commands/create.rb".freeze, "lib/ollama/commands/delete.rb".freeze, "lib/ollama/commands/embed.rb".freeze, "lib/ollama/commands/embeddings.rb".freeze, "lib/ollama/commands/generate.rb".freeze, "lib/ollama/commands/ps.rb".freeze, "lib/ollama/commands/pull.rb".freeze, "lib/ollama/commands/push.rb".freeze, "lib/ollama/commands/show.rb".freeze, "lib/ollama/commands/tags.rb".freeze, "lib/ollama/documents.rb".freeze, "lib/ollama/documents/memory_cache.rb".freeze, "lib/ollama/documents/redis_cache.rb".freeze, "lib/ollama/documents/splitters/character.rb".freeze, "lib/ollama/documents/splitters/semantic.rb".freeze, "lib/ollama/dto.rb".freeze, "lib/ollama/errors.rb".freeze, "lib/ollama/handlers.rb".freeze, "lib/ollama/handlers/collector.rb".freeze, "lib/ollama/handlers/concern.rb".freeze, "lib/ollama/handlers/dump_json.rb".freeze, "lib/ollama/handlers/dump_yaml.rb".freeze, "lib/ollama/handlers/markdown.rb".freeze, "lib/ollama/handlers/nop.rb".freeze, "lib/ollama/handlers/print.rb".freeze, "lib/ollama/handlers/progress.rb".freeze, "lib/ollama/handlers/say.rb".freeze, "lib/ollama/handlers/single.rb".freeze, "lib/ollama/image.rb".freeze, "lib/ollama/message.rb".freeze, "lib/ollama/options.rb".freeze, "lib/ollama/response.rb".freeze, "lib/ollama/tool.rb".freeze, "lib/ollama/tool/function.rb".freeze, "lib/ollama/tool/function/parameters.rb".freeze, "lib/ollama/tool/function/parameters/property.rb".freeze, "lib/ollama/utils/ansi_markdown.rb".freeze, "lib/ollama/utils/chooser.rb".freeze, "lib/ollama/utils/colorize_texts.rb".freeze, "lib/ollama/utils/fetcher.rb".freeze, "lib/ollama/utils/file_argument.rb".freeze, "lib/ollama/utils/math.rb".freeze, "lib/ollama/utils/tags.rb".freeze, "lib/ollama/utils/width.rb".freeze, "lib/ollama/version.rb".freeze, "ollama-ruby.gemspec".freeze, "spec/assets/embeddings.json".freeze, "spec/assets/kitten.jpg".freeze, "spec/ollama/client/doc_spec.rb".freeze, "spec/ollama/client_spec.rb".freeze, "spec/ollama/commands/chat_spec.rb".freeze, "spec/ollama/commands/copy_spec.rb".freeze, "spec/ollama/commands/create_spec.rb".freeze, "spec/ollama/commands/delete_spec.rb".freeze, "spec/ollama/commands/embed_spec.rb".freeze, "spec/ollama/commands/embeddings_spec.rb".freeze, "spec/ollama/commands/generate_spec.rb".freeze, "spec/ollama/commands/ps_spec.rb".freeze, "spec/ollama/commands/pull_spec.rb".freeze, "spec/ollama/commands/push_spec.rb".freeze, "spec/ollama/commands/show_spec.rb".freeze, "spec/ollama/commands/tags_spec.rb".freeze, "spec/ollama/documents/memory_cache_spec.rb".freeze, "spec/ollama/documents/redis_cache_spec.rb".freeze, "spec/ollama/documents/splitters/character_spec.rb".freeze, "spec/ollama/documents/splitters/semantic_spec.rb".freeze, "spec/ollama/documents_spec.rb".freeze, "spec/ollama/handlers/collector_spec.rb".freeze, "spec/ollama/handlers/dump_json_spec.rb".freeze, "spec/ollama/handlers/dump_yaml_spec.rb".freeze, "spec/ollama/handlers/markdown_spec.rb".freeze, "spec/ollama/handlers/nop_spec.rb".freeze, "spec/ollama/handlers/print_spec.rb".freeze, "spec/ollama/handlers/progress_spec.rb".freeze, "spec/ollama/handlers/say_spec.rb".freeze, "spec/ollama/handlers/single_spec.rb".freeze, "spec/ollama/image_spec.rb".freeze, "spec/ollama/message_spec.rb".freeze, "spec/ollama/options_spec.rb".freeze, "spec/ollama/tool_spec.rb".freeze, "spec/ollama/utils/ansi_markdown_spec.rb".freeze, "spec/ollama/utils/fetcher_spec.rb".freeze, "spec/ollama/utils/tags_spec.rb".freeze, "spec/spec_helper.rb".freeze, "tmp/.keep".freeze]
   s.homepage = "https://github.com/flori/ollama-ruby".freeze
   s.licenses = ["MIT".freeze]
   s.rdoc_options = ["--title".freeze, "Ollama-ruby - Interacting with the Ollama API".freeze, "--main".freeze, "README.md".freeze]
   s.required_ruby_version = Gem::Requirement.new("~> 3.1".freeze)
-  s.rubygems_version = "3.5.11".freeze
+  s.rubygems_version = "3.5.18".freeze
   s.summary = "Interacting with the Ollama API".freeze
   s.test_files = ["spec/ollama/client/doc_spec.rb".freeze, "spec/ollama/client_spec.rb".freeze, "spec/ollama/commands/chat_spec.rb".freeze, "spec/ollama/commands/copy_spec.rb".freeze, "spec/ollama/commands/create_spec.rb".freeze, "spec/ollama/commands/delete_spec.rb".freeze, "spec/ollama/commands/embed_spec.rb".freeze, "spec/ollama/commands/embeddings_spec.rb".freeze, "spec/ollama/commands/generate_spec.rb".freeze, "spec/ollama/commands/ps_spec.rb".freeze, "spec/ollama/commands/pull_spec.rb".freeze, "spec/ollama/commands/push_spec.rb".freeze, "spec/ollama/commands/show_spec.rb".freeze, "spec/ollama/commands/tags_spec.rb".freeze, "spec/ollama/documents/memory_cache_spec.rb".freeze, "spec/ollama/documents/redis_cache_spec.rb".freeze, "spec/ollama/documents/splitters/character_spec.rb".freeze, "spec/ollama/documents/splitters/semantic_spec.rb".freeze, "spec/ollama/documents_spec.rb".freeze, "spec/ollama/handlers/collector_spec.rb".freeze, "spec/ollama/handlers/dump_json_spec.rb".freeze, "spec/ollama/handlers/dump_yaml_spec.rb".freeze, "spec/ollama/handlers/markdown_spec.rb".freeze, "spec/ollama/handlers/nop_spec.rb".freeze, "spec/ollama/handlers/print_spec.rb".freeze, "spec/ollama/handlers/progress_spec.rb".freeze, "spec/ollama/handlers/say_spec.rb".freeze, "spec/ollama/handlers/single_spec.rb".freeze, "spec/ollama/image_spec.rb".freeze, "spec/ollama/message_spec.rb".freeze, "spec/ollama/options_spec.rb".freeze, "spec/ollama/tool_spec.rb".freeze, "spec/ollama/utils/ansi_markdown_spec.rb".freeze, "spec/ollama/utils/fetcher_spec.rb".freeze, "spec/ollama/utils/tags_spec.rb".freeze, "spec/spec_helper.rb".freeze]
 
   s.specification_version = 4
 
-  s.add_development_dependency(%q<gem_hadar>.freeze, ["~> 1.17.0".freeze])
+  s.add_development_dependency(%q<gem_hadar>.freeze, ["~> 1.17.1".freeze])
   s.add_development_dependency(%q<all_images>.freeze, ["~> 0.4".freeze])
   s.add_development_dependency(%q<rspec>.freeze, ["~> 3.2".freeze])
   s.add_development_dependency(%q<utils>.freeze, [">= 0".freeze])
   s.add_development_dependency(%q<webmock>.freeze, [">= 0".freeze])
   s.add_runtime_dependency(%q<excon>.freeze, ["~> 0.111".freeze])
-  s.add_runtime_dependency(%q<infobar>.freeze, ["~> 0.7".freeze])
+  s.add_runtime_dependency(%q<infobar>.freeze, ["~> 0.8".freeze])
   s.add_runtime_dependency(%q<term-ansicolor>.freeze, ["~> 1.11".freeze])
   s.add_runtime_dependency(%q<kramdown-parser-gfm>.freeze, ["~> 1.1".freeze])
   s.add_runtime_dependency(%q<terminal-table>.freeze, ["~> 3.0".freeze])
@@ -40,7 +40,8 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency(%q<sorted_set>.freeze, ["~> 1.0".freeze])
   s.add_runtime_dependency(%q<mime-types>.freeze, ["~> 3.0".freeze])
   s.add_runtime_dependency(%q<reverse_markdown>.freeze, ["~> 2.0".freeze])
-  s.add_runtime_dependency(%q<complex_config>.freeze, ["~> 0.20".freeze])
+  s.add_runtime_dependency(%q<complex_config>.freeze, ["~> 0.22".freeze])
   s.add_runtime_dependency(%q<search_ui>.freeze, ["~> 0.0".freeze])
   s.add_runtime_dependency(%q<amatch>.freeze, ["~> 0.4.1".freeze])
+  s.add_runtime_dependency(%q<pdf-reader>.freeze, ["~> 2.0".freeze])
 end

data/spec/ollama/documents/redis_cache_spec.rb

@@ -74,5 +74,13 @@ RSpec.describe Ollama::Documents::RedisCache do
       expect(redis).to receive(:scan_each).with(match: 'test-*')
       redis_cache.to_a
     end
+
+    it 'can compute prefix with pre' do
+      expect(redis_cache.pre('foo')).to eq 'test-foo'
+    end
+
+    it 'can remove prefix with unpre' do
+      expect(redis_cache.unpre('test-foo')).to eq 'foo'
+    end
   end
 end

data/spec/ollama/documents_spec.rb

@@ -76,6 +76,34 @@ RSpec.describe Ollama::Documents do
     expect(records[0].to_s).to eq '#<Ollama::Documents::Record "foo" #test 1.0>'
   end
 
+  it 'can find strings conditionally' do
+    allow(ollama).to receive(:embed).
+      with(model:, input: [ 'foobar' ], options: nil).
+      and_return(double(embeddings: [ [ 0.01 ] ]))
+    allow(ollama).to receive(:embed).
+      with(model:, input: [ 'foo' ], options: nil).
+      and_return(double(embeddings: [ [ 0.1 ] ]))
+    expect(documents << 'foobar').to eq documents
+    expect(documents << 'foo').to eq documents
+    expect(ollama).to receive(:embed).at_least(:once).
+      with(model:, input: 'foo', options: nil).
+      and_return(double(embeddings: [ [ 0.1 ] ]))
+    records = documents.find_where('foo', text_count: 1)
+    expect(records).to eq [
+      Ollama::Documents::Record[text: 'foo', embedding: [ 0.1 ], similarity: 1.0 ],
+    ]
+    records = documents.find_where('foo', text_size: 3)
+    expect(records).to eq [
+      Ollama::Documents::Record[text: 'foo', embedding: [ 0.1 ], similarity: 1.0 ],
+    ]
+    records = documents.find_where('foo')
+    expect(records).to eq [
+      Ollama::Documents::Record[text: 'foo', embedding: [ 0.1 ], similarity: 1.0 ],
+      Ollama::Documents::Record[text: 'foobar', embedding: [ 0.1 ], similarity: 1.0 ],
+    ]
+  end
+
+
   context 'it uses cache' do
     before do
       allow(ollama).to receive(:embed).
@@ -103,6 +131,20 @@ RSpec.describe Ollama::Documents do
       }.to change { documents.size }.from(1).to(0)
     end
 
+    it 'can clear texts with tags' do
+      allow(ollama).to receive(:embed).
+        with(model:, input: %w[ bar ], options: nil).
+        and_return(double(embeddings: [ [ 0.1 ] ]))
+      expect(documents.add('foo', tags: %i[ test ])).to eq documents
+      expect(documents.add('bar', tags: %i[ test2 ])).to eq documents
+      expect {
+        documents.clear tags: 'test'
+      }.to change { documents.size }.from(2).to(1)
+      expect {
+        documents.clear tags: :test2
+      }.to change { documents.size }.from(1).to(0)
+    end
+
     it 'returns collections' do
       expect(documents.collections).to eq [ :default ]
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ollama-ruby
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.1
 platform: ruby
 authors:
 - Florian Frank
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-09-03 00:00:00.000000000 Z
+date: 2024-09-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: gem_hadar
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.17.0
+        version: 1.17.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.17.0
+        version: 1.17.1
 - !ruby/object:Gem::Dependency
   name: all_images
   requirement: !ruby/object:Gem::Requirement
@@ -100,14 +100,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.7'
+        version: '0.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.7'
+        version: '0.8'
 - !ruby/object:Gem::Dependency
   name: term-ansicolor
   requirement: !ruby/object:Gem::Requirement
@@ -240,14 +240,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.20'
+        version: '0.22'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.20'
+        version: '0.22'
 - !ruby/object:Gem::Dependency
   name: search_ui
   requirement: !ruby/object:Gem::Requirement
@@ -276,12 +276,27 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.4.1
+- !ruby/object:Gem::Dependency
+  name: pdf-reader
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
 description: Library that allows interacting with the Ollama API
 email: flori@ping.de
 executables:
 - ollama_console
 - ollama_chat
 - ollama_update
+- ollama_cli
 extensions: []
 extra_rdoc_files:
 - README.md
@@ -331,6 +346,7 @@ extra_rdoc_files:
 - lib/ollama/utils/chooser.rb
 - lib/ollama/utils/colorize_texts.rb
 - lib/ollama/utils/fetcher.rb
+- lib/ollama/utils/file_argument.rb
 - lib/ollama/utils/math.rb
 - lib/ollama/utils/tags.rb
 - lib/ollama/utils/width.rb
@@ -343,6 +359,7 @@ files:
 - README.md
 - Rakefile
 - bin/ollama_chat
+- bin/ollama_cli
 - bin/ollama_console
 - bin/ollama_update
 - config/redis.conf
@@ -393,6 +410,7 @@ files:
 - lib/ollama/utils/chooser.rb
 - lib/ollama/utils/colorize_texts.rb
 - lib/ollama/utils/fetcher.rb
+- lib/ollama/utils/file_argument.rb
 - lib/ollama/utils/math.rb
 - lib/ollama/utils/tags.rb
 - lib/ollama/utils/width.rb
@@ -460,7 +478,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.11
+rubygems_version: 3.5.18
 signing_key:
 specification_version: 4
 summary: Interacting with the Ollama API