ruby_llm 2.1.1 → 2.2.0

data/lib/ruby_llm/file_store/converters/office.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+require 'zlib'
+
+module RubyLLM
+  class FileStore
+    module Converters
+      # Reads the text of Word documents, Excel workbooks, and PowerPoint
+      # decks (.docx, .xlsx, .pptx) on this machine, with no service and no
+      # cost: a paragraph per line, a table or sheet row per line with tabs
+      # between cells, and a heading for each sheet and slide, with a slide's
+      # speaker notes under it. Excel dates read as dates, like 2026-03-31.
+      # Text comes back in document order, without layout or images.
+      module Office
+        TYPES = {
+          'application/vnd.openxmlformats-officedocument.wordprocessingml.document' => :document,
+          'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' => :workbook,
+          'application/vnd.openxmlformats-officedocument.presentationml.presentation' => :deck
+        }.freeze # :nodoc:
+        MAX_SIZE = 50 * 1024 * 1024 # :nodoc:
+
+        class << self
+          # Returns the text of +attachment+, or +nil+ when it is not a
+          # document, workbook, or deck it can read. A file it can't read
+          # never raises.
+          def call(attachment)
+            kind = TYPES[attachment.mime_type]
+            return unless kind
+
+            send(kind, Zip.new(attachment.content.to_s.b))
+          rescue Zip::Error, Zlib::Error
+            nil
+          rescue StandardError => e
+            RubyLLM.logger.warn { "Could not read an Office file: #{e.class}" }
+            nil
+          end
+
+          # Returns +true+: it reads files on this machine, for free.
+          def local?
+            true
+          end
+
+          private
+
+          def document(zip)
+            xml = zip['word/document.xml']
+            Prose.lines(xml, outline: Outline.read(zip)).join("\n")
+          end
+
+          def workbook(zip)
+            sheets = parts(zip, 'xl/workbook.xml', 'sheet', %r{\Axl/worksheets/[^/]+\.xml\z})
+            Workbook.new(zip, sheets).text
+          end
+
+          def deck(zip)
+            slides = parts(zip, 'ppt/presentation.xml', 'sldId', %r{\Appt/slides/[^/]+\.xml\z})
+            slides.each_with_index.map do |(part, _tag), index|
+              notes = notes_of(zip, part)
+              sections = ["## Slide #{index + 1}", Prose.lines(zip[part]).join("\n")]
+              sections << "Speaker notes:\n#{notes.join("\n")}" if notes.any?
+              sections.reject(&:empty?).join("\n\n")
+            end.join("\n\n")
+          end
+
+          def notes_of(zip, slide)
+            notes = relationships(zip, slide).find { |relationship| relationship[:type]&.end_with?('/notesSlide') }
+            notes ? Prose.lines(zip[notes[:target]]) : []
+          end
+
+          # Returns each part the +owner+ lists with a +tag+, in its order,
+          # with that tag, or the parts matching +fallback+ in name order
+          # when the listing doesn't resolve.
+          def parts(zip, owner, tag, fallback)
+            targets = relationships(zip, owner).to_h { |relationship| relationship.values_at(:id, :target) }
+            listed = Markup.tags(zip[owner], tag).filter_map do |element|
+              part = targets[Markup.relationship_id(element)]
+              [part, element] if part && zip.include?(part)
+            end
+            return listed if listed.any?
+
+            zip.names.grep(fallback).sort_by { |name| name[/\d+/].to_i }.map { |name| [name, nil] }
+          end
+
+          def relationships(zip, part)
+            folder, _, file = part.rpartition('/')
+            Markup.tags(zip["#{folder}/_rels/#{file}.rels"], 'Relationship').map do |element|
+              { id: Markup.attribute(element, 'Id'), type: Markup.attribute(element, 'Type'),
+                target: resolve(folder, Markup.attribute(element, 'Target').to_s) }
+            end
+          end
+
+          def resolve(folder, target)
+            target = target.tr('\\', '/')
+            base = target.start_with?('/') ? [] : folder.split('/')
+            segments = target.split('/').each_with_object(base) do |segment, path|
+              next if segment.empty? || segment == '.'
+
+              segment == '..' ? path.pop : path.push(segment)
+            end
+            segments.join('/').force_encoding(Encoding::UTF_8)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/file_store/converters/text.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  class FileStore
+    module Converters
+      # Reads a file as UTF-8 text, as it is. Text files are read without
+      # registering anything; register it for other types that hold plain
+      # text:
+      #
+      #   config.file_converters["application/x-subrip"] = RubyLLM::FileStore::Converters::Text
+      #
+      module Text
+        module_function
+
+        # Returns the content of +attachment+ as UTF-8 text.
+        def call(attachment)
+          attachment.content.to_s.dup.force_encoding(Encoding::UTF_8).scrub
+        end
+
+        # Returns true: it reads files here, for free.
+        def local?
+          true
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/file_store/converters.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  class FileStore
+    # Converters turn files into text the model can read and search. A
+    # converter is any object that answers +call+ with an Attachment and
+    # returns the file's text, or +nil+ when it cannot read that file, so
+    # the next converter gets a turn. FileStore#text tries the converters
+    # in Configuration#file_converters first, then reads text files,
+    # including source code, CSV, JSON, and Markdown, as they are.
+    #
+    # Converters::Office reads Word, Excel, and PowerPoint files without
+    # any setup. The OCR converter reads PDFs, scans, and photos with
+    # RubyLLM.ocr. It costs a request per file, so it only runs where you
+    # register it. Converters::Text reads a file as it is, for other types
+    # that hold plain text.
+    #
+    # A converter that answers +local?+ with +true+ reads files on this
+    # machine for free, so a chat runs it as soon as it keeps an
+    # attachment, while it prepares the request, so keep it fast, and
+    # search_files finds the text without a read_file first. The others wait
+    # for read_file, or for a request that must send as text a file the
+    # provider cannot take.
+    #
+    #   RubyLLM.configure do |config|
+    #     config.file_converters["image/*"] = RubyLLM::FileStore::Converters::OCR.new
+    #     config.file_converters["application/pdf"] = RubyLLM::FileStore::Converters::OCR.new
+    #     config.file_converters["application/x-subrip"] = RubyLLM::FileStore::Converters::Text
+    #     config.file_converters["application/vnd.oasis.opendocument.text"] = ->(file) { OdtText.call(file.content) }
+    #   end
+    #
+    # Converters for other formats can ship as gems named
+    # <tt>ruby_llm-converters-NAME</tt>, like provider gems.
+    module Converters
+      class << self
+        # Returns the text the first converter for +attachment+ makes of it,
+        # or +nil+ when none can read it.
+        # With <tt>local: true</tt>, stops at the first converter that does
+        # not run on this machine for free, so one that may cost money, or
+        # that was registered to go first, is never passed over.
+        def convert(attachment, config: RubyLLM.config, local: false) # :nodoc:
+          converters = for_type(attachment.mime_type, config:)
+          converters = converters.take_while { |converter| local?(converter) } if local
+          converters.each do |converter|
+            text = converter.call(attachment)
+            next unless text.is_a?(String)
+
+            text = text.dup.force_encoding(Encoding::UTF_8).scrub
+            return text unless text.strip.empty?
+          end
+          nil
+        end
+
+        # Returns the converters for files of +type+, in the order they are
+        # tried.
+        def for_type(type, config: RubyLLM.config) # :nodoc:
+          configured = config.file_converters.to_h.select { |pattern, _| matches?(pattern.to_s, type) }
+          configured.sort_by { |pattern, _| pattern.to_s.end_with?('*') ? 1 : 0 }
+                    .flat_map { |_, converters| Array(converters) } + built_in(type)
+        end
+
+        # Returns whether +converter+ reads files on this machine for free.
+        def local?(converter) # :nodoc:
+          converter.respond_to?(:local?) && converter.local? == true
+        end
+
+        private
+
+        def matches?(pattern, type)
+          return true if pattern == '*/*'
+          return type.start_with?(pattern.delete_suffix('*')) if pattern.end_with?('/*')
+
+          pattern == type
+        end
+
+        def built_in(type)
+          return [Text] if Files::MimeType.text?(type)
+
+          Office::TYPES.key?(type) ? [Office] : []
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/file_store/directory.rb

@@ -0,0 +1,190 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'find'
+
+module RubyLLM
+  class FileStore
+    # Keeps files in a folder on disk, created when missing. Files someone
+    # puts in the folder are the chat's files too. Hidden files and
+    # folders, whose names start with a dot, are left out, and symbolic
+    # links are never followed. Code execution runs in the folder itself
+    # and can change anything in it, so give each store a folder of its
+    # own.
+    #
+    #   chat.with_file_store("tmp/chats/42")
+    #
+    class Directory < FileStore
+      OPEN_FLAGS = File::RDONLY | File::BINARY | (defined?(File::NOFOLLOW) ? File::NOFOLLOW : 0)
+      SNIFF_FLAGS = OPEN_FLAGS | (defined?(File::NONBLOCK) ? File::NONBLOCK : 0)
+      WRITE_FLAGS = File::WRONLY | File::CREAT | File::EXCL | File::BINARY
+      private_constant :OPEN_FLAGS, :SNIFF_FLAGS, :WRITE_FLAGS
+
+      # The folder holding the files, as an absolute path.
+      attr_reader :root
+
+      # Creates a store for the folder +root+.
+      def initialize(root)
+        super()
+        FileUtils.mkdir_p(root)
+        @root = File.realpath(root)
+      end
+
+      # Returns #root: code execution runs in this folder.
+      def local_root # :nodoc:
+        root
+      end
+
+      # Returns the files inside the folder +prefix+, or every file.
+      def list(prefix = nil)
+        start = prefix.to_s.empty? ? root : full_path(FileStore.path(prefix))
+        return [self[prefix]].compact unless File.lstat(start).directory?
+
+        walk(start).sort_by(&:path)
+      rescue SystemCallError
+        []
+      end
+
+      # Returns the StoredFile at +path+, or +nil+.
+      def [](path)
+        full = full_path(FileStore.path(path))
+        stat = File.lstat(full)
+        stored_file(full, stat) if stat.file?
+      rescue SystemCallError
+        nil
+      end
+
+      # Returns the content of the file at +path+.
+      def read(path)
+        path = FileStore.path(path)
+        File.open(full_path(path), OPEN_FLAGS, &:read).b
+      rescue SystemCallError
+        raise FileNotFoundError, "No file at #{path}"
+      end
+
+      # Writes +content+ to +path+ and returns its StoredFile. Raises
+      # ArgumentError when +path+ is a folder.
+      def write(path, content, content_type: nil)
+        path = FileStore.path(path)
+        file = with_lock { write_file(path, content) }
+        content_type ? StoredFile.new(**file.to_h, content_type:) : file
+      end
+
+      # Deletes the file at +path+ and returns its StoredFile, or +nil+.
+      def delete(path)
+        path = FileStore.path(path)
+        with_lock do
+          refuse_folder(path) if folder?(path)
+
+          file = self[path]
+          File.delete(full_path(path)) if file
+          file
+        end
+      end
+
+      # Deletes the folder +path+ with everything in it, hidden files
+      # included, and returns the deleted StoredFile objects. Links are
+      # removed, never followed.
+      def delete_folder(path)
+        path = FileStore.path(path)
+        with_lock do
+          refuse_file(path) if self[path]
+          next [] unless folder?(path)
+
+          full = full_path(path)
+          walk(full).sort_by(&:path).tap { FileUtils.rm_r(full) }
+        end
+      end
+
+      # Runs the block with the store locked, against other threads and,
+      # through a lock on #root, other processes, and returns its value.
+      def with_lock
+        super do
+          next yield if @locked
+
+          File.open(root, File::RDONLY) do |folder|
+            folder.flock(File::LOCK_EX)
+            @locked = true
+            yield
+          ensure
+            @locked = false
+          end
+        end
+      end
+
+      private
+
+      def write_file(path, content)
+        raise ArgumentError, "#{path} is a folder, not a file" if folder?(path)
+
+        full = full_path(path)
+        make_room(full)
+        temporary = File.join(File.dirname(full), ".#{SecureRandom.hex(16)}.tmp")
+        File.open(temporary, WRITE_FLAGS, binmode: true) do |io|
+          content.respond_to?(:read) ? IO.copy_stream(content, io) : io.write(content)
+        end
+        File.rename(temporary, full)
+        stored_file(full, File.lstat(full))
+      ensure
+        FileUtils.rm_f(temporary) if temporary
+      end
+
+      def full_path(path)
+        parent = root
+        path.split('/').each do |part|
+          raise ArgumentError, "#{path} is not inside the chat's files" if part.start_with?('.')
+
+          parent = File.join(parent, part)
+        end
+        folder = File.dirname(parent)
+        if File.directory?(folder) && !"#{File.realpath(folder)}/".start_with?("#{root}/")
+          raise ArgumentError, "#{path} is not inside the chat's files"
+        end
+
+        parent
+      end
+
+      def walk(start)
+        files = []
+        Find.find(start) do |path|
+          next if path == root
+
+          stat = File.lstat(path)
+          Find.prune if File.basename(path).start_with?('.')
+          files << stored_file(path, stat) if stat.file?
+        end
+        files
+      end
+
+      def make_room(full)
+        parent = root
+        File.dirname(full).delete_prefix(root).split('/').reject(&:empty?).each do |part|
+          parent = File.join(parent, part)
+          File.delete(parent) if File.symlink?(parent) || File.file?(parent)
+        end
+        FileUtils.mkdir_p(File.dirname(full))
+      end
+
+      def folder?(path)
+        File.lstat(full_path(path)).directory?
+      rescue SystemCallError
+        false
+      end
+
+      def file_content_type(full, path)
+        type = FileStore.content_type_for(path, '')
+        return type unless type == 'application/octet-stream'
+
+        File.open(full, SNIFF_FLAGS) { |io| io.stat.file? ? Files::MimeType.for(io, name: path) : type }
+      rescue SystemCallError
+        type
+      end
+
+      def stored_file(full, stat)
+        path = full.delete_prefix("#{root}/")
+        StoredFile.new(path:, byte_size: stat.size, content_type: file_content_type(full, path),
+                       updated_at: stat.mtime)
+      end
+    end
+  end
+end

data/lib/ruby_llm/file_store/memory.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  class FileStore
+    # Keeps files in memory, for as long as the chat lives. It is the
+    # store Chat#with_file_store uses when given none.
+    #
+    #   chat.with_file_store(:memory)
+    #
+    class Memory < FileStore
+      def initialize # :nodoc:
+        super
+        @files = {}
+        @lock = Mutex.new
+      end
+
+      # Returns the files inside the folder +prefix+, or every file.
+      def list(prefix = nil)
+        @lock.synchronize { @files.values.map(&:first) }
+             .select { |file| FileStore.inside?(file.path, prefix) }
+             .sort_by(&:path)
+      end
+
+      # Returns the StoredFile at +path+, or +nil+.
+      def [](path)
+        @lock.synchronize { @files[FileStore.path(path)]&.first }
+      end
+
+      # Returns the content of the file at +path+.
+      def read(path)
+        path = FileStore.path(path)
+        content = @lock.synchronize { @files[path]&.last }
+        raise FileNotFoundError, "No file at #{path}" unless content
+
+        content.dup
+      end
+
+      # Writes +content+ to +path+ and returns its StoredFile.
+      def write(path, content, content_type: nil)
+        path = FileStore.path(path)
+        content = FileStore.bytes(content).freeze
+        file = StoredFile.new(path:, byte_size: content.bytesize,
+                              content_type: content_type || FileStore.content_type_for(path, content),
+                              digest: FileStore.digest(content), updated_at: Time.now)
+        with_lock { @lock.synchronize { @files[path] = [file, content] } }
+        file
+      end
+
+      # Deletes the file at +path+ and returns its StoredFile, or +nil+.
+      def delete(path)
+        path = FileStore.path(path)
+        with_lock do
+          @lock.synchronize do
+            refuse_folder(path) if !@files.key?(path) && @files.keys.any? { |key| FileStore.inside?(key, path) }
+            @files.delete(path)&.first
+          end
+        end
+      end
+
+      # Deletes everything inside the folder +path+ and returns the deleted
+      # StoredFile objects.
+      def delete_folder(path)
+        path = FileStore.path(path)
+        with_lock do
+          @lock.synchronize do
+            refuse_file(path) if @files.key?(path)
+
+            paths = @files.keys.select { |key| FileStore.inside?(key, path) }
+            paths.map { |key| @files.delete(key).first }.sort_by(&:path)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/file_store/tools/base.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  class FileStore
+    module Tools
+      # What the file tools share: the chat's store, its configuration,
+      # the callback for each file a tool writes, and the check that the
+      # model read a file before it changes it.
+      class Base < Tool # :nodoc:
+        class ToolError < StandardError; end # :nodoc:
+
+        inline_limit false
+
+        def initialize(store:, config: -> { RubyLLM.config }, written: ->(_file) {})
+          super()
+          @store = store
+          @config = config
+          @written = written
+        end
+
+        private
+
+        def store
+          @store.call
+        end
+
+        def config
+          @config.call
+        end
+
+        def path_for(name)
+          FileStore.path(name)
+        rescue ArgumentError => e
+          raise ToolError, e.message
+        end
+
+        def folder_for(name)
+          path_for(name) unless name.to_s.strip.empty? || %w[. /].include?(name.to_s.strip)
+        end
+
+        def file_at(name)
+          path = path_for(name)
+          store[path] || raise(ToolError, "There is no file at #{path}. #{missing_hint(path)}")
+        end
+
+        def missing_hint(path)
+          folder = store.list(path).first
+          return "#{path} is a folder: list it with list_files." if folder
+
+          'List the files with list_files.'
+        end
+
+        def digest_of(file)
+          file.digest || FileStore.digest(store.read(file.path))
+        end
+
+        # Yields the file at +path+, or +nil+ for a new one, with the store
+        # locked, once the model has read the file as it is now.
+        def changing(path)
+          store.with_lock do
+            file = store[path]
+            fresh!(file) if file
+            yield file
+          end
+        end
+
+        def fresh!(file)
+          seen = store.read_digest(file.path)
+          raise ToolError, "Read #{file.path} before you change it." unless seen
+          return if seen == digest_of(file)
+
+          raise ToolError, "#{file.path} changed since you read it. Read it again."
+        end
+
+        def guarded_write
+          written = []
+          result = guarded do
+            yield(lambda do |path, content|
+              file = store.write(path, content)
+              store.record_read(file.path, file.digest || FileStore.digest(content))
+              written << file
+              file
+            end)
+          end
+          written.each { |file| @written.call(file) }
+          result
+        end
+
+        def guarded
+          yield
+        rescue ToolError, FileStoreFullError, ArgumentError => e
+          { error: e.message }
+        rescue SystemCallError => e
+          { error: SystemCallError.new(e.errno).message }
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/file_store/tools/delete.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  class FileStore
+    module Tools
+      # The +delete_file+ tool: deletes one file. Chats get it only when
+      # they ask for it.
+      class Delete < Base # :nodoc:
+        def self.tool_name
+          'delete_file'
+        end
+
+        def description
+          'Deletes one file.'
+        end
+
+        def parameters_schema
+          {
+            'type' => 'object',
+            'properties' => {
+              'path' => { 'type' => 'string', 'description' => 'The file to delete' }
+            },
+            'required' => %w[path],
+            'additionalProperties' => false
+          }
+        end
+
+        def execute(path:)
+          guarded do
+            path = path_for(path)
+            store.with_lock { delete(path) }
+          end
+        end
+
+        private
+
+        def delete(path)
+          file = store.delete(path)
+          raise ToolError, "There is no file at #{path}. List the files with list_files." unless file
+
+          "Deleted #{file.path}."
+        rescue FolderError
+          raise ToolError, "#{path} is a folder. Delete its files one by one."
+        end
+      end
+    end
+  end
+end