ruby_llm 2.1.1 → 2.2.0

data/lib/ruby_llm/file_store/tools/edit.rb

@@ -0,0 +1,155 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  class FileStore
+    module Tools
+      # The +edit_file+ tool: replaces exact spans of text files, one or
+      # several in a call, all or none.
+      class Edit < Base # :nodoc:
+        Change = Struct.new(:label, :path, :old_text, :new_text, :replace_all) # :nodoc:
+
+        def self.tool_name
+          'edit_file'
+        end
+
+        def description
+          'Changes text files you have read by replacing old_text, which must appear exactly once, with ' \
+            'new_text. A text file attached to the question counts as read, at its file name. Include ' \
+            'surrounding text to make old_text unique, or set replace_all. Pass edits for several changes in ' \
+            'one call, to one file or several: they apply in order, all or none.'
+        end
+
+        def parameters_schema
+          {
+            'type' => 'object',
+            'properties' => {
+              'path' => { 'type' => %w[string null], 'description' => 'The file to change' },
+              'old_text' => { 'type' => %w[string null], 'description' => 'The exact text to replace' },
+              'new_text' => { 'type' => %w[string null], 'description' => 'The text to put in its place' },
+              'replace_all' => { 'type' => %w[boolean null], 'description' => 'Replace every occurrence' },
+              'edits' => { 'type' => %w[array null], 'description' => 'Several changes; each may name a path',
+                           'items' => edit_schema }
+            },
+            'additionalProperties' => false
+          }
+        end
+
+        def execute(path: nil, old_text: nil, new_text: nil, replace_all: false, edits: nil)
+          guarded_write do |write|
+            raise ToolError, 'Pass old_text and new_text, or edits, not both.' if edits && (old_text || new_text)
+
+            changes = changes_for(path, old_text, new_text, replace_all, edits)
+            store.with_lock do
+              files = apply(changes)
+              write_all(files, write) || report(files)
+            end
+          end
+        end
+
+        private
+
+        def edit_schema
+          {
+            'type' => 'object',
+            'properties' => {
+              'path' => { 'type' => %w[string null] },
+              'old_text' => { 'type' => 'string' },
+              'new_text' => { 'type' => 'string' },
+              'replace_all' => { 'type' => %w[boolean null] }
+            },
+            'required' => %w[old_text new_text],
+            'additionalProperties' => false
+          }
+        end
+
+        def changes_for(path, old_text, new_text, replace_all, edits)
+          return batch(path, edits) unless edits.nil?
+          raise ToolError, 'Pass path.' if path.to_s.strip.empty?
+
+          [Change.new('', path, old_text, new_text, replace_all)]
+        end
+
+        def batch(path, edits)
+          raise ToolError, 'edits is empty.' if Array(edits).empty?
+
+          Array(edits).each_with_index.map do |edit, index|
+            edit = edit.to_h.transform_keys(&:to_s)
+            target = edit['path'] || path
+            if target.to_s.strip.empty?
+              raise ToolError,
+                    "Edit #{index + 1} has no path. Pass path, or a path in each edit."
+            end
+
+            Change.new("Edit #{index + 1}: ", target, edit['old_text'], edit['new_text'], edit['replace_all'])
+          end
+        end
+
+        def apply(changes)
+          changes.each_with_object({}) do |change, files|
+            path = labelled(change) { path_for(change.path) }
+            files[path] ||= labelled(change) { [contents(path), 0] }
+            content, count = labelled(change) { replace(files[path].first, path, change) }
+            files[path] = [content, files[path].last + count]
+          end
+        end
+
+        # Returns the error once a write fails after others landed, instead
+        # of raising, so a store's transaction keeps what the answer reports.
+        def write_all(files, write)
+          changed = []
+          files.each do |path, (content, _count)|
+            write.call(path, content)
+            changed << path
+          end
+          nil
+        rescue FileStoreFullError, SystemCallError => e
+          raise if changed.empty?
+
+          reason = e.is_a?(SystemCallError) ? SystemCallError.new(e.errno).message : e.message
+          { error: "#{reason} These files already changed: #{changed.join(', ')}." }
+        end
+
+        def labelled(change)
+          yield
+        rescue ToolError => e
+          raise ToolError, "#{change.label}#{e.message}"
+        end
+
+        def contents(path)
+          file = store[path] || raise(ToolError, "There is no file at #{path}. #{missing_hint(path)}")
+          fresh!(file)
+          text_of(file)
+        end
+
+        def replace(content, path, change)
+          raise ToolError, 'Pass new_text, or an empty string to delete old_text.' if change.new_text.nil?
+
+          old_text = change.old_text.to_s
+          count = old_text.empty? ? 0 : content.scan(old_text).size
+          raise ToolError, "old_text does not appear in #{path}." if count.zero?
+          if count > 1 && !change.replace_all
+            raise ToolError, "old_text appears #{count} times in #{path}. Add the text around it to make " \
+                             'it unique, or set replace_all.'
+          end
+
+          [content.public_send(change.replace_all ? :gsub : :sub, old_text) { change.new_text.to_s }, count]
+        end
+
+        def report(files)
+          count = files.values.sum(&:last)
+          replaced = "Replaced #{count} #{count == 1 ? 'occurrence' : 'occurrences'}"
+          return "#{replaced} in #{files.keys.first}." if files.one?
+
+          "#{replaced} in #{files.size} files: #{files.keys.join(', ')}."
+        end
+
+        def text_of(file)
+          content = store.read(file.path).dup.force_encoding(Encoding::UTF_8)
+          return content if content.valid_encoding? && !content.include?("\0")
+
+          raise ToolError, "#{file.path} is not a text file, so edit_file cannot change it."
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/file_store/tools/list.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  class FileStore
+    module Tools
+      # The +list_files+ tool: the chat's files, those in one folder, or
+      # those matching a glob, with their sizes and types.
+      class List < Base # :nodoc:
+        MAX_FILES = 200
+
+        def self.tool_name
+          'list_files'
+        end
+
+        def description
+          'Lists the files, with sizes and types. Pass path for one folder, and pattern for a glob: ' \
+            '*.csv matches names in any folder, reports/**/*.csv matches paths.'
+        end
+
+        def parameters_schema
+          {
+            'type' => 'object',
+            'properties' => {
+              'path' => { 'type' => %w[string null], 'description' => 'A folder to list' },
+              'pattern' => { 'type' => %w[string null], 'description' => 'A glob, like *.md or reports/**/*.csv' }
+            },
+            'additionalProperties' => false
+          }
+        end
+
+        def execute(path: nil, pattern: nil)
+          guarded do
+            folder = folder_for(path)
+            files = store.list(folder)
+            pattern = pattern.to_s.strip.then { |glob| glob unless glob.empty? }
+            files = files.select { |file| FileStore.match?(file.path, pattern, inside: folder) } if pattern
+            next empty(folder, pattern) if files.empty?
+
+            lines = files.first(MAX_FILES).map { |file| "#{file.path}\t#{file.human_size}\t#{file.content_type}" }
+            if files.size > MAX_FILES
+              lines << "[#{files.size - MAX_FILES} more files. Narrow the list with path or pattern.]"
+            end
+            lines.join("\n")
+          end
+        end
+
+        private
+
+        def empty(folder, pattern)
+          return "No files#{" in #{folder}" if folder} match #{pattern}." if pattern
+
+          folder ? "There are no files in #{folder}." : 'There are no files yet.'
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/file_store/tools/read.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  class FileStore
+    module Tools
+      # The +read_file+ tool: a range of a file's text, by lines or by
+      # characters, with the text a converter makes of other files. Images,
+      # and PDFs no converter reads, come back as an attachment the model
+      # sees.
+      class Read < Base # :nodoc:
+        MAX_LINES = 1_000
+        MAX_CHARACTERS = 20_000
+        MAX_LINE = 2_000
+        UNITS = %w[lines characters].freeze
+
+        def self.tool_name
+          'read_file'
+        end
+
+        def description
+          "Reads a file. Text comes back with line numbers, #{MAX_LINES} lines or #{MAX_CHARACTERS} characters " \
+            'at a time: pass offset and limit to read further, and unit "characters" for very long lines. ' \
+            'After search_files, read just the lines around a match with offset and limit. Documents a ' \
+            'converter reads come back as text, and other images and PDFs as the file itself.'
+        end
+
+        def parameters_schema
+          {
+            'type' => 'object',
+            'properties' => {
+              'path' => { 'type' => 'string', 'description' => 'The file to read' },
+              'offset' => { 'type' => %w[integer null],
+                            'description' => 'The line to start at, from 1, or the character, from 0' },
+              'limit' => { 'type' => %w[integer null], 'description' => 'How many lines or characters to read' },
+              'unit' => { 'type' => %w[string null], 'enum' => [*UNITS, nil],
+                          'description' => 'lines (the default) or characters' }
+            },
+            'required' => %w[path],
+            'additionalProperties' => false
+          }
+        end
+
+        def execute(path:, offset: nil, limit: nil, unit: nil)
+          seen = nil
+          result = guarded do
+            file = file_at(path)
+            seen = [file.path, digest_of(file)]
+            unit = (unit || 'lines').to_s
+            raise ToolError, "unit is lines or characters, not #{unit}." unless UNITS.include?(unit)
+            next "#{file.path} is empty." if file.byte_size.zero?
+
+            text = store.text(file.path, config:)
+            next unreadable(file) if text.nil?
+
+            unit == 'characters' ? characters(file, text, offset, limit) : lines(file, text, offset, limit)
+          end
+          store.record_read(*seen) if seen && !result.is_a?(Hash)
+          result
+        end
+
+        private
+
+        def lines(file, text, offset, limit)
+          all = text.lines
+          start = [offset.to_i, 1].max
+          raise ToolError, "#{file.path} has #{all.size} lines; offset #{start} is past the end." if
+            start > all.size && !all.empty?
+
+          count = limit.to_i.positive? ? [limit.to_i, MAX_LINES].min : MAX_LINES
+          shown = []
+          size = 0
+          all.drop(start - 1).first(count).each_with_index do |line, index|
+            line = shorten(line.chomp)
+            break if size + line.size > MAX_CHARACTERS && shown.any?
+
+            shown << format('%<number>6d	%<line>s', number: start + index, line:)
+            size += line.size
+          end
+          last = start + shown.size - 1
+          [header(file, "lines #{start}-#{last} of #{all.size}"), *shown, more_lines(all.size, last)].compact.join("\n")
+        end
+
+        def characters(file, text, offset, limit)
+          start = [offset.to_i, 0].max
+          count = limit.to_i.positive? ? [limit.to_i, MAX_CHARACTERS].min : MAX_CHARACTERS
+          part = text[start, count].to_s
+          finish = start + part.size
+          more = "[#{text.size - finish} more characters. Continue with offset #{finish}.]" if finish < text.size
+          [header(file, "characters #{start}-#{finish} of #{text.size}"), part, more].compact.join("\n")
+        end
+
+        def header(file, range)
+          source = " (text read from #{file.content_type})" unless file.text?
+          "#{file.path}#{source}, #{range}:"
+        end
+
+        def more_lines(total, last)
+          "[#{total - last} more lines. Continue with offset #{last + 1}.]" if last < total
+        end
+
+        def shorten(line)
+          return line if line.size <= MAX_LINE
+
+          "#{line[0, MAX_LINE]} [line cut: #{line.size - MAX_LINE} more characters, read them with unit characters]"
+        end
+
+        def unreadable(file)
+          type = Files::MimeType
+          if type.image?(file.content_type) || type.pdf?(file.content_type)
+            return ["#{file.path} (#{file.content_type}, #{file.human_size}) is attached to this result.",
+                    store.attachment(file.path, file:)]
+          end
+
+          { error: "#{file.path} is #{file.content_type} and has no text to read. Work with it in code if you " \
+                   'can run code, or describe it from its name.' }
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/file_store/tools/search.rb

@@ -0,0 +1,208 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  class FileStore
+    module Tools
+      # The +search_files+ tool: lines matching a regular expression across
+      # the text of the chat's files, or the files that have one, like grep.
+      class Search < Base # :nodoc:
+        MAX_MATCHES = 100
+        MAX_CHARACTERS = 20_000
+        MAX_CONTEXT = 10
+        MAX_LINE = 300
+        MAX_FILE_SIZE = 50 * 1024 * 1024
+        MAX_SECONDS = 10
+
+        def self.tool_name
+          'search_files'
+        end
+
+        def description
+          'Searches the text of the files for lines matching a regular expression, like grep, and returns ' \
+            'path:line: text. Start with files_only to see which files match, then read them with read_file. ' \
+            'Files whose text read_file has not extracted yet are skipped and named.'
+        end
+
+        def parameters_schema
+          {
+            'type' => 'object',
+            'properties' => {
+              'query' => { 'type' => 'string', 'description' => 'A regular expression, or plain words' },
+              'path' => { 'type' => %w[string null], 'description' => 'A file or folder to search' },
+              'glob' => { 'type' => %w[string null], 'description' => 'Only files matching this, like *.md' },
+              'context' => { 'type' => %w[integer null], 'description' => 'Lines to show around each match' },
+              'case_sensitive' => { 'type' => %w[boolean null], 'description' => 'Match case. Default: false' },
+              'files_only' => { 'type' => %w[boolean null], 'description' => 'Return only the matching paths' }
+            },
+            'required' => %w[query],
+            'additionalProperties' => false
+          }
+        end
+
+        def execute(query:, path: nil, glob: nil, context: nil, case_sensitive: false, files_only: false)
+          guarded do
+            pattern = pattern_for(query.to_s, case_sensitive)
+            glob = nil if glob.to_s.strip.empty?
+            files = scope(path, glob)
+            next "No files match #{glob}." if files.empty? && glob
+
+            Run.new(pattern, context: context.to_i.clamp(0, MAX_CONTEXT), files_only:).call(files) do |file|
+              searchable_text(file)
+            end.report(query)
+          end
+        end
+
+        private
+
+        def scope(path, glob)
+          folder = folder_for(path)
+          files = folder ? files_at(folder) : store.list
+          return files unless glob
+
+          files.select { |file| FileStore.match?(file.path, glob, inside: folder) }
+        end
+
+        def files_at(folder)
+          file = store[folder]
+          files = file ? [file] : store.list(folder)
+          raise ToolError, "There is no file or folder at #{folder}." if files.empty?
+
+          files
+        end
+
+        def pattern_for(query, case_sensitive)
+          raise ToolError, 'query is empty.' if query.strip.empty?
+
+          options = case_sensitive ? 0 : Regexp::IGNORECASE
+          begin
+            Regexp.new(query, options, timeout: 1.0)
+          rescue RegexpError
+            Regexp.new(Regexp.escape(query), options, timeout: 1.0)
+          end
+        end
+
+        # Converters can cost money, like OCR, so search reads only text
+        # files and conversions already made.
+        def searchable_text(file)
+          return '' if file.byte_size.zero?
+          return if file.byte_size > MAX_FILE_SIZE
+
+          cached = store.cached_text(file)
+          return cached if cached
+          return Converters::Text.call(store.attachment(file.path, file:)) if file.text?
+
+          :unextracted if Converters.for_type(file.content_type, config:).any?
+        end
+
+        # One search: walks the files, collects what it shows, and stops at
+        # its limits.
+        class Run # :nodoc:
+          def initialize(pattern, context:, files_only:)
+            @pattern = pattern
+            @context = context
+            @files_only = files_only
+            @lines = []
+            @size = 0
+            @matches = 0
+            @skipped = { none: [], unextracted: [] }
+            @stopped = nil
+            @deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + MAX_SECONDS
+          end
+
+          def call(files)
+            files.each do |file|
+              break if @stopped
+
+              text = yield(file)
+              next @skipped[text.is_a?(Symbol) ? text : :none] << file.path unless text.is_a?(String)
+
+              search(file.path, text.lines)
+            end
+            self
+          rescue Regexp::TimeoutError
+            raise Base::ToolError, 'The regular expression took too long. Use a simpler one.'
+          end
+
+          def report(query)
+            lines = @lines.empty? ? ["No lines match #{query}."] : @lines.dup
+            lines << notice if @stopped
+            if @skipped[:unextracted].any?
+              unextracted = @skipped[:unextracted]
+              lines << "[Not searched, text not extracted yet: #{unextracted.join(', ')}. " \
+                       "read_file extracts #{unextracted.one? ? 'it' : 'them'}.]"
+            end
+            lines << "[Not searched, no text: #{@skipped[:none].join(', ')}]" if @skipped[:none].any?
+            lines.join("\n")
+          end
+
+          private
+
+          def search(path, lines)
+            matched = []
+            lines.each_with_index do |line, index|
+              return stop(:time) if late?
+              next unless @pattern.match?(line)
+              return show([path]) if @files_only
+
+              matched << index
+            end
+            show_matches(path, lines, matched)
+          end
+
+          def show_matches(path, lines, matched)
+            hits = matched.to_set
+            windows(matched, lines.size).each_with_index.all? do |(first, last), index|
+              (index.zero? || @context.zero? || show(['--'], match: false)) &&
+                (first..last).all? do |number|
+                  mark = hits.include?(number) ? ':' : '-'
+                  show(["#{path}#{mark}#{number + 1}#{mark} #{clean(lines[number])}"], match: mark == ':')
+                end
+            end
+          end
+
+          def windows(matched, size)
+            matched.each_with_object([]) do |index, windows|
+              first = [index - @context, 0].max
+              last = [index + @context, size - 1].min
+              next windows.last[1] = last if windows.any? && first <= windows.last[1] + 1
+
+              windows << [first, last]
+            end
+          end
+
+          def show(lines, match: true)
+            text = lines.join("\n")
+            return stop(:size) if @size + text.size > MAX_CHARACTERS && @lines.any?
+            return stop(:size) if match && @matches >= MAX_MATCHES
+
+            @lines.concat(lines)
+            @size += text.size + 1
+            @matches += 1 if match
+            true
+          end
+
+          def clean(line)
+            line.chomp.strip[0, MAX_LINE]
+          end
+
+          def late?
+            Process.clock_gettime(Process::CLOCK_MONOTONIC) > @deadline
+          end
+
+          def stop(reason)
+            @stopped ||= reason
+            nil
+          end
+
+          def notice
+            if @stopped == :time
+              '[The search took too long and stopped. Narrow it with path or glob.]'
+            else
+              '[More matches. Narrow the query, path, or glob.]'
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/file_store/tools/write.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  class FileStore
+    module Tools
+      # The +write_file+ tool: creates a text file in the chat's files, or
+      # replaces one.
+      class Write < Base # :nodoc:
+        def self.tool_name
+          'write_file'
+        end
+
+        def description
+          'Writes a text file, replacing any file at path. Read an existing file before you replace it. ' \
+            'To change part of a file, use edit_file.'
+        end
+
+        def parameters_schema
+          {
+            'type' => 'object',
+            'properties' => {
+              'path' => { 'type' => 'string', 'description' => 'Where to write, such as notes/summary.md' },
+              'content' => { 'type' => 'string', 'description' => 'The complete content of the file' }
+            },
+            'required' => %w[path content],
+            'additionalProperties' => false
+          }
+        end
+
+        def execute(path:, content:)
+          guarded_write do |write|
+            path = path_for(path)
+            file = changing(path) { write.call(path, content.to_s) }
+            "Wrote #{file.path} (#{file.human_size})."
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/file_store/tools.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  class FileStore
+    # The tools Chat#with_file_store gives the model, by the names its
+    # +tools:+ option takes: +:read+ for +list_files+, +read_file+, and
+    # +search_files+, +:write+ for +write_file+, +:edit+ for +edit_file+,
+    # and +:delete+ for +delete_file+.
+    module Tools # :nodoc:
+      GROUPS = { read: %i[List Read Search], write: %i[Write], edit: %i[Edit], delete: %i[Delete] }.freeze
+      DEFAULT = %i[read write edit].freeze
+
+      class << self
+        # Returns +tools+, a name or an Array of them, as Symbols in the
+        # order of GROUPS. Raises ArgumentError for a name it does not know,
+        # and without +:read+.
+        def names(tools)
+          names = Array(tools).map { |name| name.to_s.to_sym }
+          unknown = names - GROUPS.keys
+          if unknown.any?
+            raise ArgumentError,
+                  "tools: takes :read, :write, :edit, and :delete, got #{unknown.map(&:inspect).join(', ')}"
+          end
+          unless names.include?(:read)
+            raise ArgumentError,
+                  'tools: needs :read, since the model reads its files with it'
+          end
+
+          GROUPS.keys & names
+        end
+
+        # Returns the tool classes for +names+.
+        def classes(names)
+          names.flat_map { |name| GROUPS.fetch(name) }.map { |name| const_get(name) }
+        end
+      end
+    end
+  end
+end