ollama_chat 0.0.0

data/lib/ollama_chat/parsing.rb

@@ -0,0 +1,159 @@
+module OllamaChat::Parsing
+  def parse_source(source_io)
+    case source_io&.content_type
+    when 'text/html'
+      reverse_markdown(source_io.read)
+    when 'text/xml'
+      if source_io.read(8192) =~ %r(^\s*<rss\s)
+        source_io.rewind
+        return parse_rss(source_io)
+      end
+      source_io.rewind
+      source_io.read
+    when 'text/csv'
+      parse_csv(source_io)
+    when 'application/rss+xml'
+      parse_rss(source_io)
+    when 'application/atom+xml'
+      parse_atom(source_io)
+    when 'application/postscript'
+      ps_read(source_io)
+    when 'application/pdf'
+      pdf_read(source_io)
+    when %r(\Aapplication/(json|ld\+json|x-ruby|x-perl|x-gawk|x-python|x-javascript|x-c?sh|x-dosexec|x-shellscript|x-tex|x-latex|x-lyx|x-bibtex)), %r(\Atext/), nil
+      source_io.read
+    else
+      STDERR.puts "Cannot embed #{source_io&.content_type} document."
+      return
+    end
+  end
+
+  def parse_csv(source_io)
+    result = +''
+    CSV.table(File.new(source_io), col_sep: ?,).each do |row|
+      next if row.fields.select(&:present?).none?
+      result << row.map { |pair|
+        pair.compact.map { _1.to_s.strip } * ': ' if pair.last.present?
+      }.select(&:present?).map { _1.prepend('  ') } * ?\n
+      result << "\n\n"
+    end
+    result
+  end
+
+  def parse_rss(source_io)
+    feed = RSS::Parser.parse(source_io, false, false)
+    title = <<~EOT
+      # #{feed&.channel&.title}
+
+    EOT
+    feed.items.inject(title) do |text, item|
+      text << <<~EOT
+        ## [#{item&.title}](#{item&.link})
+
+        updated on #{item&.pubDate}
+
+        #{reverse_markdown(item&.description)}
+
+      EOT
+    end
+  end
+
+  def parse_atom(source_io)
+    feed = RSS::Parser.parse(source_io, false, false)
+    title = <<~EOT
+      # #{feed.title.content}
+
+    EOT
+    feed.items.inject(title) do |text, item|
+      text << <<~EOT
+        ## [#{item&.title&.content}](#{item&.link&.href})
+
+        updated on #{item&.updated&.content}
+
+        #{reverse_markdown(item&.content&.content)}
+
+      EOT
+    end
+  end
+
+  def pdf_read(io)
+    reader = PDF::Reader.new(io)
+    reader.pages.inject(+'') { |result, page| result << page.text }
+  end
+
+  def ps_read(io)
+    gs = `which gs`.chomp
+    if gs.present?
+      Tempfile.create do |tmp|
+        IO.popen("#{gs} -q -sDEVICE=pdfwrite -sOutputFile=#{tmp.path} -", 'wb') do |gs_io|
+          until io.eof?
+            buffer = io.read(1 << 17)
+            IO.select(nil, [ gs_io ], nil)
+            gs_io.write buffer
+          end
+          gs_io.close
+          File.open(tmp.path, 'rb') do |pdf|
+            pdf_read(pdf)
+          end
+        end
+      end
+    else
+      STDERR.puts "Cannot convert #{io&.content_type} whith ghostscript, gs not in path."
+    end
+  end
+
+  def reverse_markdown(html)
+    ReverseMarkdown.convert(
+      html,
+      unknown_tags: :bypass,
+      github_flavored: true,
+      tag_border: ''
+    )
+  end
+
+  def parse_content(content, images)
+    images.clear
+    tags = Documentrix::Utils::Tags.new
+
+    contents = [ content ]
+    content.scan(%r((https?://\S+)|(#\S+)|(?:file://)?(\S*\/\S+))).each do |url, tag, file|
+      case
+      when tag
+        tags.add(tag)
+        next
+      when file
+        file = file.sub(/#.*/, '')
+        file =~ %r(\A[~./]) or file.prepend('./')
+        File.exist?(file) or next
+        source = file
+      when url
+        links.add(url.to_s)
+        source = url
+      end
+      fetch_source(source) do |source_io|
+        case source_io&.content_type&.media_type
+        when 'image'
+          add_image(images, source_io, source)
+        when 'text', 'application', nil
+          case @document_policy
+          when 'ignoring'
+            nil
+          when 'importing'
+            contents << import_source(source_io, source)
+          when 'embedding'
+            embed_source(source_io, source)
+          when 'summarizing'
+            contents << summarize_source(source_io, source)
+          end
+        else
+          STDERR.puts(
+            "Cannot fetch #{source.to_s.inspect} with content type "\
+            "#{source_io&.content_type.inspect}"
+          )
+        end
+      end
+    end
+    new_content = contents.select { _1.present? rescue nil }.compact * "\n\n"
+    return new_content, (tags unless tags.empty?)
+  end
+end

data/lib/ollama_chat/source_fetching.rb

@@ -0,0 +1,173 @@
+module OllamaChat::SourceFetching
+  def http_options(url)
+    options = {}
+    if ssl_no_verify = config.ssl_no_verify?
+      hostname = URI.parse(url).hostname
+      options |= { ssl_verify_peer: !ssl_no_verify.include?(hostname) }
+    end
+    if proxy = config.proxy?
+      options |= { proxy: }
+    end
+    options
+  end
+
+  def fetch_source(source, &block)
+    case source
+    when %r(\A!(.*))
+      command = $1
+      OllamaChat::Utils::Fetcher.execute(command) do |tmp|
+        block.(tmp)
+      end
+    when %r(\Ahttps?://\S+)
+      links.add(source.to_s)
+      OllamaChat::Utils::Fetcher.get(
+        source,
+        cache:        @cache,
+        debug:        config.debug,
+        http_options: http_options(OllamaChat::Utils::Fetcher.normalize_url(source))
+      ) do |tmp|
+        block.(tmp)
+      end
+    when %r(\Afile://(/\S*?)#|\A((?:\.\.|[~.]?)/\S*))
+      filename = $~.captures.compact.first
+      filename = File.expand_path(filename)
+      OllamaChat::Utils::Fetcher.read(filename) do |tmp|
+        block.(tmp)
+      end
+    else
+      raise "invalid source"
+    end
+  rescue => e
+    STDERR.puts "Cannot fetch source #{source.to_s.inspect}: #{e.class} #{e}\n#{e.backtrace * ?\n}"
+  end
+
+  def add_image(images, source_io, source)
+    STDERR.puts "Adding #{source_io&.content_type} image #{source.to_s.inspect}."
+    image = Ollama::Image.for_io(source_io, path: source.to_s)
+    (images << image).uniq!
+  end
+
+  def import_source(source_io, source)
+    source = source.to_s
+    STDOUT.puts "Importing #{italic { source_io&.content_type }} document #{source.to_s.inspect} now."
+    source_content = parse_source(source_io)
+    "Imported #{source.inspect}:\n\n#{source_content}\n\n"
+  end
+
+  def import(source)
+    fetch_source(source) do |source_io|
+      content = import_source(source_io, source) or return
+      source_io.rewind
+      content
+    end
+  end
+
+  def summarize_source(source_io, source, words: nil)
+    STDOUT.puts "Summarizing #{italic { source_io&.content_type }} document #{source.to_s.inspect} now."
+    words = words.to_i
+    words < 1 and words = 100
+    source_content = parse_source(source_io)
+    source_content.present? or return
+    config.prompts.summarize % { source_content:, words: }
+  end
+
+  def summarize(source, words: nil)
+    fetch_source(source) do |source_io|
+      content = summarize_source(source_io, source, words:) or return
+      source_io.rewind
+      content
+    end
+  end
+
+  def embed_source(source_io, source, count: nil)
+    @embedding.on? or return parse_source(source_io)
+    m = "Embedding #{italic { source_io&.content_type }} document #{source.to_s.inspect}."
+    if count
+      STDOUT.puts '%u. %s' % [ count, m ]
+    else
+      STDOUT.puts m
+    end
+    text = parse_source(source_io) or return
+    text.downcase!
+    splitter_config = config.embedding.splitter
+    inputs = nil
+    case splitter_config.name
+    when 'Character'
+      splitter = Documentrix::Documents::Splitters::Character.new(
+        chunk_size: splitter_config.chunk_size,
+      )
+      inputs = splitter.split(text)
+    when 'RecursiveCharacter'
+      splitter = Documentrix::Documents::Splitters::RecursiveCharacter.new(
+        chunk_size: splitter_config.chunk_size,
+      )
+      inputs = splitter.split(text)
+    when 'Semantic'
+      splitter = Documentrix::Documents::Splitters::Semantic.new(
+        ollama:, model: config.embedding.model.name,
+        chunk_size: splitter_config.chunk_size,
+      )
+      inputs = splitter.split(
+        text,
+        breakpoint: splitter_config.breakpoint.to_sym,
+        percentage: splitter_config.percentage?,
+        percentile: splitter_config.percentile?,
+      )
+    end
+    inputs or return
+    source = source.to_s
+    if source.start_with?(?!)
+      source = Kramdown::ANSI::Width.truncate(
+        source[1..-1].gsub(/\W+/, ?_),
+        length: 10
+      )
+    end
+    @documents.add(inputs, source:, batch_size: config.embedding.batch_size?)
+  end
+
+  def embed(source)
+    if @embedding.on?
+      STDOUT.puts "Now embedding #{source.to_s.inspect}."
+      fetch_source(source) do |source_io|
+        content = parse_source(source_io)
+        content.present? or return
+        source_io.rewind
+        embed_source(source_io, source)
+      end
+      config.prompts.embed % { source: }
+    else
+      STDOUT.puts "Embedding is off, so I will just give a small summary of this source."
+      summarize(source)
+    end
+  end
+
+  def search_web(query, n = nil)
+    if l = @messages.at_location.full?
+      query += " #{l}"
+    end
+    n = n.to_i
+    n < 1 and n = 1
+    query = URI.encode_uri_component(query)
+    url = "https://www.duckduckgo.com/html/?q=#{query}"
+    OllamaChat::Utils::Fetcher.get(url, debug: config.debug) do |tmp|
+      result = []
+      doc = Nokogiri::HTML(tmp)
+      doc.css('.results_links').each do |link|
+        if n > 0
+          url = link.css('.result__a').first&.[]('href')
+          url.sub!(%r(\A(//duckduckgo\.com)?/l/\?uddg=), '')
+          url.sub!(%r(&rut=.*), '')
+          url = URI.decode_uri_component(url)
+          url = URI.parse(url)
+          url.host =~ /duckduckgo\.com/ and next
+          links.add(url.to_s)
+          result << url
+          n -= 1
+        else
+          break
+        end
+      end
+      result
+    end
+  end
+end

data/lib/ollama_chat/switches.rb

@@ -0,0 +1,119 @@
+module OllamaChat::Switches
+  module CheckSwitch
+    extend Tins::Concern
+
+    included do
+      alias_method :on?, :value
+    end
+
+    def off?
+      !on?
+    end
+
+    def show
+      STDOUT.puts @msg[value]
+    end
+  end
+
+  class Switch
+    def initialize(name, msg:, config:)
+      @value = [ false, true ].include?(config) ? config : !!config.send("#{name}?")
+      @msg   = msg
+    end
+
+    attr_reader :value
+
+    def set(value, show: false)
+      @value = !!value
+      show && self.show
+    end
+
+    def toggle(show: true)
+      @value = !@value
+      show && self.show
+    end
+
+    include CheckSwitch
+  end
+
+  class CombinedSwitch
+    def initialize(value:, msg:)
+      @value = value
+      @msg   = msg
+    end
+
+    def value
+      @value.()
+    end
+
+    include CheckSwitch
+  end
+
+  attr_reader :markdown
+
+  attr_reader :location
+
+  def setup_switches(config)
+    @markdown = Switch.new(
+      :markdown,
+      config:,
+      msg: {
+        true  => "Using #{italic{'ANSI'}} markdown to output content.",
+        false => "Using plaintext for outputting content.",
+      }
+    )
+
+    @stream = Switch.new(
+      :stream,
+      config:,
+      msg: {
+        true  => "Streaming enabled.",
+        false => "Streaming disabled.",
+      }
+    )
+
+    @voice = Switch.new(
+      :stream,
+      config: config.voice,
+      msg: {
+        true  => "Voice output enabled.",
+        false => "Voice output disabled.",
+      }
+    )
+
+    @embedding_enabled = Switch.new(
+      :embedding_enabled,
+      config:,
+      msg: {
+        true  => "Embedding enabled.",
+        false => "Embedding disabled.",
+      }
+    )
+
+    @embedding_paused = Switch.new(
+      :embedding_paused,
+      config:,
+      msg: {
+        true  => "Embedding paused.",
+        false => "Embedding resumed.",
+      }
+    )
+
+    @embedding = CombinedSwitch.new(
+      value: -> { @embedding_enabled.on? && @embedding_paused.off? },
+      msg: {
+        true  => "Embedding is currently performed.",
+        false => "Embedding is currently not performed.",
+      }
+    )
+
+    @location = Switch.new(
+      :location,
+      config: config.location.enabled,
+      msg: {
+        true  => "Location and localtime enabled.",
+        false => "Location and localtime disabled.",
+      }
+    )
+  end
+end

data/lib/ollama_chat/utils/cache_fetcher.rb

@@ -0,0 +1,38 @@
+require 'digest/md5'
+
+class OllamaChat::Utils::CacheFetcher
+  def initialize(cache)
+    @cache = cache
+  end
+
+  def get(url, &block)
+    block or raise ArgumentError, 'require block argument'
+    body         = @cache[key(:body, url)]
+    content_type = @cache[key(:content_type, url)]
+    content_type = MIME::Types[content_type].first
+    if body && content_type
+      io = StringIO.new(body)
+      io.rewind
+      io.extend(OllamaChat::Utils::Fetcher::HeaderExtension)
+      io.content_type = content_type
+      block.(io)
+    end
+  end
+
+  def put(url, io)
+    io.rewind
+    body = io.read
+    body.empty? and return
+    content_type = io.content_type
+    content_type.nil? and return
+    @cache.set(key(:body, url), body, ex: io.ex)
+    @cache.set(key(:content_type,  url), content_type.to_s, ex: io.ex)
+    self
+  end
+
+  private
+
+  def key(type, url)
+    [ type, Digest::MD5.hexdigest(url) ] * ?-
+  end
+end

data/lib/ollama_chat/utils/chooser.rb

@@ -0,0 +1,53 @@
+require 'amatch'
+require 'search_ui'
+require 'term/ansicolor'
+
+module OllamaChat::Utils::Chooser
+  class << self
+    include SearchUI
+    include Term::ANSIColor
+
+    # The choose method presents a list of entries and prompts the user
+    # for input, allowing them to select one entry based on their input.
+    #
+    # @param entries [Array] the list of entries to present to the user
+    # @param prompt [String] the prompt message to display when asking for input (default: 'Search? %s')
+    # @param return_immediately [Boolean] whether to immediately return the
+    #        first entry if there is only one or nil when there is none (default: false)
+    #
+    # @return [Object] the selected entry, or nil if no entry was chosen
+    #
+    # @example
+    #   choose(['entry1', 'entry2'], prompt: 'Choose an option:')
+    def choose(entries, prompt: 'Search? %s', return_immediately: false)
+      if return_immediately && entries.size <= 1
+        return entries.first
+      end
+      entry = Search.new(
+        prompt:,
+        match: -> answer {
+          matcher = Amatch::PairDistance.new(answer.downcase)
+          matches = entries.map { |n| [ n, -matcher.similar(n.to_s.downcase) ] }.
+            select { |_, s| s < 0 }.sort_by(&:last).map(&:first)
+          matches.empty? and matches = entries
+          matches.first(Tins::Terminal.lines - 1)
+        },
+        query: -> _answer, matches, selector {
+          matches.each_with_index.map { |m, i|
+            i == selector ? "#{blue{?⮕}} #{on_blue{m}}" : "  #{m.to_s}"
+          } * ?\n
+        },
+        found: -> _answer, matches, selector {
+          matches[selector]
+        },
+        output: STDOUT
+      ).start
+      if entry
+        entry
+      else
+        print clear_screen, move_home
+        nil
+      end
+    end
+  end
+end

data/lib/ollama_chat/utils/fetcher.rb

@@ -0,0 +1,175 @@
+require 'tempfile'
+require 'tins/unit'
+require 'infobar'
+require 'mime-types'
+require 'stringio'
+require 'ollama_chat/utils/cache_fetcher'
+
+class OllamaChat::Utils::Fetcher
+  module HeaderExtension
+    attr_accessor :content_type
+
+    attr_accessor :ex
+
+    def self.failed
+      object = StringIO.new.extend(self)
+      object.content_type = MIME::Types['text/plain'].first
+      object
+    end
+  end
+
+  class RetryWithoutStreaming < StandardError; end
+
+  def self.get(url, **options, &block)
+    cache = options.delete(:cache) and
+      cache = OllamaChat::Utils::CacheFetcher.new(cache)
+    if result = cache&.get(url, &block)
+      infobar.puts "Getting #{url.to_s.inspect} from cache."
+      return result
+    else
+      new(**options).send(:get, url) do |tmp|
+        result = block.(tmp)
+        if cache && !tmp.is_a?(StringIO)
+          tmp.rewind
+          cache.put(url, tmp)
+        end
+        result
+      end
+    end
+  end
+
+  def self.normalize_url(url)
+    url = url.to_s
+    url = URI.decode_uri_component(url)
+    url = url.sub(/#.*/, '')
+    URI::Parser.new.escape(url).to_s
+  end
+
+  def self.read(filename, &block)
+    if File.exist?(filename)
+      File.open(filename) do |file|
+        file.extend(OllamaChat::Utils::Fetcher::HeaderExtension)
+        file.content_type = MIME::Types.type_for(filename).first
+        block.(file)
+      end
+    else
+      STDERR.puts "File #{filename.to_s.inspect} doesn't exist."
+    end
+  end
+
+  def self.execute(command, &block)
+    Tempfile.open do |tmp|
+      IO.popen(command) do |command|
+        until command.eof?
+          tmp.write command.read(1 << 14)
+        end
+        tmp.rewind
+        tmp.extend(OllamaChat::Utils::Fetcher::HeaderExtension)
+        tmp.content_type = MIME::Types['text/plain'].first
+        block.(tmp)
+      end
+    end
+  rescue => e
+    STDERR.puts "Cannot execute #{command.inspect} (#{e})"
+    if @debug && !e.is_a?(RuntimeError)
+      STDERR.puts "#{e.backtrace * ?\n}"
+    end
+    yield HeaderExtension.failed
+  end
+
+  def initialize(debug: false, http_options: {})
+    @debug        = debug
+    @started      = false
+    @streaming    = true
+    @http_options = http_options
+  end
+
+  private
+
+  def excon(url, **options)
+    url = self.class.normalize_url(url)
+    Excon.new(url, options.merge(@http_options))
+  end
+
+  def get(url, &block)
+    response = nil
+    Tempfile.open do |tmp|
+      infobar.label = 'Getting'
+      if @streaming
+        response = excon(url, headers:, response_block: callback(tmp)).request(method: :get)
+        response.status != 200 || !@started and raise RetryWithoutStreaming
+        decorate_io(tmp, response)
+        infobar.finish
+        block.(tmp)
+      else
+        response = excon(url, headers:, middlewares:).request(method: :get)
+        if response.status != 200
+          raise "invalid response status code"
+        end
+        body = response.body
+        tmp.print body
+        infobar.update(message: message(body.size, body.size), force: true)
+        decorate_io(tmp, response)
+        infobar.finish
+        block.(tmp)
+      end
+    end
+  rescue RetryWithoutStreaming
+    @streaming = false
+    retry
+  rescue => e
+    STDERR.puts "Cannot get #{url.to_s.inspect} (#{e}): #{response&.status_line || 'n/a'}"
+    if @debug && !e.is_a?(RuntimeError)
+      STDERR.puts "#{e.backtrace * ?\n}"
+    end
+    yield HeaderExtension.failed
+  end
+
+  def headers
+    {
+      'User-Agent' => OllamaChat::Chat.user_agent,
+    }
+  end
+
+  def middlewares
+    (Excon.defaults[:middlewares] + [ Excon::Middleware::RedirectFollower ]).uniq
+  end
+
+  private
+
+  def decorate_io(tmp, response)
+    tmp.rewind
+    tmp.extend(HeaderExtension)
+    if content_type = MIME::Types[response.headers['content-type']].first
+      tmp.content_type = content_type
+    end
+    if cache_control = response.headers['cache-control'] and
+        cache_control !~ /no-store|no-cache/ and
+        ex = cache_control[/s-maxage\s*=\s*(\d+)/, 1] || cache_control[/max-age\s*=\s*(\d+)/, 1]
+    then
+      tmp.ex = ex.to_i
+    end
+  end
+
+  def callback(tmp)
+    -> chunk, remaining_bytes, total_bytes do
+      total   = total_bytes or next
+      current = total_bytes - remaining_bytes
+      if @started
+        infobar.counter.progress(by: total - current)
+      else
+        @started = true
+        infobar.counter.reset(total:, current:)
+      end
+      infobar.update(message: message(current, total), force: true)
+      tmp.print(chunk)
+    end
+  end
+
+  def message(current, total)
+    progress = '%s/%s' % [ current, total ].map {
+      Tins::Unit.format(_1, format: '%.2f %U')
+    }
+    '%l ' + progress + ' in %te, ETA %e @%E'
+  end
+end