ruby_llm 2.1.1 → 2.2.0

data/lib/ruby_llm/file_store/converters/office/outline.rb

@@ -0,0 +1,193 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  class FileStore
+    module Converters
+      module Office
+        # What a Word paragraph's style and numbering make of it: a Markdown
+        # heading, or a list marker with the count Word would show. A style
+        # is a heading by its outline level, the nearest one along +basedOn+,
+        # which also covers localized names, or else by being named
+        # <tt>heading 1</tt> to <tt>heading 6</tt>. Numbers are always
+        # decimal, without a level's own format or text. Missing or broken
+        # parts mark nothing.
+        class Outline # :nodoc:
+          HEADING_NAME = /\Aheading ([1-6])\z/i
+          HEADING_LEVELS = (0..5)
+          MAX_CHAIN = 32
+          MAX_LEVEL = 8
+          NO_LIST = 0
+          private_constant :HEADING_NAME, :HEADING_LEVELS, :MAX_CHAIN, :MAX_LEVEL, :NO_LIST
+
+          def self.read(zip)
+            new(part(zip, 'word/styles.xml'), part(zip, 'word/numbering.xml'))
+          end
+
+          def self.part(zip, name)
+            zip[name]
+          rescue Zip::Error, Zlib::Error
+            nil
+          end
+          private_class_method :part
+
+          def initialize(styles, numbering)
+            @styles = {}
+            @resolved = {}
+            @abstracts = {}
+            @lists = {}
+            @counts = Hash.new { |counts, abstract| counts[abstract] = {} }
+            @overridden = {}
+            read_styles(styles)
+            read_numbering(numbering)
+          end
+
+          # Returns the marker for a paragraph with these properties of its
+          # own, counting it if it is a list item.
+          def marker(style:, list:, level:, outline:)
+            inherited = resolve(style)
+            heading = outline ? heading_of(outline) : inherited[:heading]
+            return "#{'#' * heading} " if heading
+
+            item(list || inherited[:list], level || inherited[:level] || 0)
+          end
+
+          private
+
+          def heading_of(outline)
+            outline + 1 if HEADING_LEVELS.cover?(outline)
+          end
+
+          def resolve(id)
+            @resolved[id] ||= resolve_chain(id)
+          end
+
+          # Walks at most MAX_CHAIN styles, which also ends a basedOn cycle.
+          def resolve_chain(id)
+            found = {}
+            MAX_CHAIN.times do
+              style = @styles[id]
+              break unless style
+
+              found = style.merge(found)
+              id = style[:based_on]
+            end
+            heading = found.key?(:outline) ? heading_of(found[:outline]) : found[:named]
+            { heading:, list: found[:list], level: found[:level] }
+          end
+
+          def item(list_id, level)
+            return '' if list_id.nil? || list_id == NO_LIST
+
+            list = @lists[list_id]
+            definition = list && @abstracts.dig(list[:abstract], level)
+            return '' unless definition
+
+            count = next_count(list_id, list, level, definition)
+            render(definition[:format], count, level)
+          end
+
+          def render(format, count, level)
+            case format
+            when 'none' then ''
+            when 'bullet' then "#{'  ' * level}- "
+            else "#{'  ' * level}#{count}. "
+            end
+          end
+
+          def next_count(list_id, list, level, definition)
+            counts = @counts[list[:abstract]]
+            override = list[:overrides][level]
+            if override && !@overridden[[list_id, level]]
+              @overridden[[list_id, level]] = true
+              counts[level] = override - 1
+            end
+            counts[level] = (counts[level] || (definition[:start] - 1)) + 1
+            counts.delete_if { |deeper, _count| deeper > level }
+            counts[level]
+          end
+
+          def read_styles(xml)
+            style = nil
+            Markup.scan(xml) do |kind, name, tag|
+              next if kind == :text || name.nil?
+              next style = nil if kind == :close && name == 'style'
+              next if kind == :close
+
+              name == 'style' ? style = open_style(tag) : (style_property(style, name, tag) if style)
+            end
+          end
+
+          def open_style(tag)
+            return unless Markup.prefixed(tag, 'type') == 'paragraph'
+
+            id = Markup.prefixed(tag, 'styleId')
+            @styles[id] = {} if id
+          end
+
+          def style_property(style, name, tag)
+            case name
+            when 'name' then assign(style, :named, Markup.prefixed(tag, 'val').to_s[HEADING_NAME, 1]&.to_i)
+            when 'basedOn' then style[:based_on] = Markup.prefixed(tag, 'val')
+            when 'outlineLvl' then assign(style, :outline, integer(tag))
+            when 'numId' then assign(style, :list, integer(tag))
+            when 'ilvl' then assign(style, :level, level_of(tag))
+            end
+          end
+
+          def assign(style, key, value)
+            style[key] = value unless value.nil?
+          end
+
+          def read_numbering(xml)
+            state = {}
+            Markup.scan(xml) do |kind, name, tag|
+              next if kind == :text
+
+              kind == :close ? close_numbering(state, name) : open_numbering(state, name, tag)
+            end
+          end
+
+          def open_numbering(state, name, tag)
+            case name
+            when 'abstractNum' then state[:abstract] = @abstracts[integer(tag, 'abstractNumId')] = {}
+            when 'lvl' then open_level(state, tag)
+            when 'start' then state[:level][:start] = integer(tag) || 0 if state[:level]
+            when 'numFmt' then state[:level][:format] = Markup.prefixed(tag, 'val') if state[:level]
+            when 'num' then state[:list] = @lists[integer(tag, 'numId')] = { overrides: {} }
+            when 'abstractNumId' then state[:list][:abstract] = integer(tag) if state[:list]
+            when 'lvlOverride' then state[:override] = level_of(tag, 'ilvl')
+            when 'startOverride' then start_override(state, integer(tag))
+            end
+          end
+
+          def open_level(state, tag)
+            level = level_of(tag, 'ilvl')
+            state[:level] = (state[:abstract][level] = { start: 0 } if state[:abstract] && level)
+          end
+
+          def start_override(state, value)
+            state[:list][:overrides][state[:override]] = value if state[:list] && state[:override] && value
+          end
+
+          def close_numbering(state, name)
+            case name
+            when 'abstractNum' then state[:abstract] = nil
+            when 'lvl' then state[:level] = nil
+            when 'num' then state[:list] = nil
+            when 'lvlOverride' then state[:override] = nil
+            end
+          end
+
+          def integer(tag, attribute = 'val')
+            Markup.prefixed(tag, attribute).to_s[/\A\d{1,9}\z/]&.to_i
+          end
+
+          def level_of(tag, attribute = 'val')
+            level = integer(tag, attribute)
+            level if level && level <= MAX_LEVEL
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/file_store/converters/office/prose.rb

@@ -0,0 +1,161 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  class FileStore
+    module Converters
+      module Office
+        # Reads the paragraphs and tables of a Word part or a slide: a line
+        # for each paragraph, and a line with tabs between cells for each
+        # table row. Word and PowerPoint share these element names.
+        class Prose # :nodoc:
+          SKIPPED = %w[tabs tabLst pPrChange rPr tblPr tcPr trPr sectPr Fallback].freeze
+          BREAKS = { 'tab' => "\t", 'br' => "\n", 'cr' => "\n" }.freeze
+          BLANK = /\A[[:space:]]*\z/
+          UNREAD_PLACEHOLDERS = %w[sldNum hdr ftr dt sldImg].freeze
+          Paragraph = Struct.new(:text, :style, :list, :level, :outline, :marked, :marker)
+          private_constant :SKIPPED, :BREAKS, :BLANK, :UNREAD_PLACEHOLDERS, :Paragraph
+
+          def self.lines(xml, outline: nil)
+            new(outline).read(xml)
+          end
+
+          def initialize(outline = nil)
+            @outline = outline
+            @sinks = [[]]
+            @paragraphs = []
+            @rows = []
+            @shapes = []
+            @capturing = false
+          end
+
+          def read(xml)
+            Markup.scan(xml, skip: SKIPPED) do |kind, name, content|
+              case kind
+              when :text then text(content)
+              when :open then opened(name, content)
+              when :empty then empty(name, content)
+              when :close then close(name)
+              end
+            end
+            @sinks.first
+          end
+
+          private
+
+          def text(content)
+            @paragraphs.last.text << Markup.unescape(content) if @capturing && @paragraphs.any?
+          end
+
+          def opened(name, tag)
+            case name
+            when 'p' then open_paragraph
+            when 't' then @capturing = true
+            when 'tr' then @rows << []
+            when 'tc' then @sinks << []
+            when 'sp' then open_shape
+            when 'ph' then placeholder(tag)
+            else leaf(name, tag)
+            end
+          end
+
+          def empty(name, tag)
+            case name
+            when 'tc' then @rows.last&.push('')
+            when 'ph' then placeholder(tag)
+            else leaf(name, tag)
+            end
+          end
+
+          def leaf(name, tag)
+            return unless @paragraphs.any?
+
+            paragraph = @paragraphs.last
+            case name
+            when 'tab', 'br', 'cr' then paragraph.text << BREAKS[name]
+            when 'pStyle' then paragraph.style = Markup.prefixed(tag, 'val')
+            when 'numId' then paragraph.list = number(tag)
+            when 'ilvl' then paragraph.level = number(tag)
+            when 'outlineLvl' then paragraph.outline = number(tag)
+            end
+          end
+
+          def number(tag)
+            Markup.prefixed(tag, 'val').to_s[/\A\d{1,9}\z/]&.to_i
+          end
+
+          def close(name)
+            case name
+            when 't' then @capturing = false
+            when 'p' then close_paragraph
+            when 'tc' then close_cell
+            when 'tr' then close_row
+            when 'sp' then close_shape
+            end
+          end
+
+          # A paragraph inside another, as in a text box, comes between the
+          # text around it, so what the outer one holds so far goes first.
+          def open_paragraph
+            outer = @paragraphs.last
+            emit(outer) if outer && !outer.text.empty?
+            @paragraphs << Paragraph.new(+'')
+          end
+
+          def close_paragraph
+            paragraph = @paragraphs.pop
+            emit(paragraph) if paragraph
+          end
+
+          def emit(paragraph)
+            marker = marker_of(paragraph)
+            text = paragraph.text
+            paragraph.text = +''
+            return if text.empty? || (!marker.to_s.empty? && text.match?(BLANK))
+
+            paragraph.marker = nil
+            @sinks.last << "#{marker}#{text}"
+          end
+
+          # A list item counts wherever it is, but a table row keeps to one
+          # line of plain cells. The marker waits for text that isn't blank.
+          def marker_of(paragraph)
+            return paragraph.marker if paragraph.marked || @outline.nil?
+
+            paragraph.marked = true
+            marker = @outline.marker(style: paragraph.style, list: paragraph.list, level: paragraph.level,
+                                     outline: paragraph.outline)
+            paragraph.marker = marker if @sinks.size == 1
+          end
+
+          def close_cell
+            return unless @sinks.size > 1
+
+            cell = @sinks.pop.join(' ').tr("\t\n", '  ')
+            @rows.last&.push(cell)
+          end
+
+          def close_row
+            row = @rows.pop
+            @sinks.last << row.join("\t") if row&.any? { |cell| !cell.empty? }
+          end
+
+          def open_shape
+            @sinks << []
+            @shapes << false
+          end
+
+          def placeholder(tag)
+            @shapes[-1] = true if !@shapes.empty? && UNREAD_PLACEHOLDERS.include?(Markup.attribute(tag, 'type'))
+          end
+
+          def close_shape
+            return if @sinks.size <= 1 || @shapes.empty?
+
+            lines = @sinks.pop
+            @sinks.last.concat(lines) unless @shapes.pop
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/file_store/converters/office/workbook.rb

@@ -0,0 +1,187 @@
+# frozen_string_literal: true
+
+require 'date'
+
+module RubyLLM
+  class FileStore
+    module Converters
+      module Office
+        # Reads the sheets of an Excel workbook: a heading for each, and a
+        # line with tabs between cells for each row that holds anything.
+        class Workbook # :nodoc:
+          MAX_COLUMN = 16_384
+          MAX_SERIAL = 2_958_465
+          SECONDS_A_DAY = 86_400
+          BUILT_IN_DATE_FORMATS = [*14..22, *27..36, *45..47, *50..58].freeze
+          TRUTHY = %w[1 true on].freeze
+          private_constant :MAX_COLUMN, :MAX_SERIAL, :SECONDS_A_DAY, :BUILT_IN_DATE_FORMATS, :TRUTHY
+
+          def initialize(zip, sheets)
+            @zip = zip
+            @sheets = sheets
+            @size = 0
+          end
+
+          def text
+            @strings = shared_strings
+            @date_styles = date_styles
+            @date1904 = date1904?
+            @sheets.each_with_index.map do |(part, tag), index|
+              ["## #{Markup.attribute(tag, 'name') || "Sheet #{index + 1}"}", *rows(@zip[part])].join("\n")
+            end.join("\n\n")
+          end
+
+          private
+
+          def shared_strings
+            strings = []
+            capturing = false
+            Markup.scan(@zip['xl/sharedStrings.xml'], skip: %w[rPh]) do |kind, name, content|
+              case kind
+              when :open
+                strings << +'' if name == 'si'
+                capturing = true if name == 't'
+              when :close then capturing = false if name == 't'
+              when :text then strings.last << Markup.unescape(content) if capturing && strings.any?
+              end
+            end
+            strings
+          end
+
+          def rows(xml)
+            @lines = []
+            @row = @cell = @value = nil
+            @capturing = false
+            Markup.scan(xml, skip: %w[rPh f]) do |kind, name, content|
+              case kind
+              when :open then opened(name, content)
+              when :empty then place(@row, content, '') if name == 'c' && @row
+              when :text then @value << content if @capturing && @value
+              when :close then closed(name)
+              end
+            end
+            @lines
+          end
+
+          def opened(name, tag)
+            case name
+            when 'row' then @row = []
+            when 'c'
+              @cell = tag
+              @value = +''.b
+            when 'v', 't' then @capturing = true
+            end
+          end
+
+          def closed(name)
+            case name
+            when 'v', 't' then @capturing = false
+            when 'c' then place(@row, @cell, render(@cell, @value)) if @row && @cell
+            when 'row'
+              add_row(@lines, @row) if @row
+              @row = nil
+            end
+          end
+
+          def add_row(lines, row)
+            return if row.all?(&:empty?)
+
+            line = row.join("\t")
+            @size += line.bytesize
+            raise Zip::Error, 'workbook too large' if @size > MAX_SIZE
+
+            lines << line
+          end
+
+          def place(row, tag, value)
+            column = column_of(tag)
+            row.fill('', row.size...column) if column
+            row << value
+          end
+
+          def column_of(tag)
+            letters = Markup.attribute(tag, 'r').to_s[/\A([A-Z]{1,3})\d+\z/, 1]
+            column = letters&.each_char&.inject(0) { |sum, letter| (sum * 26) + letter.ord - 64 }
+            column.pred if column && column <= MAX_COLUMN
+          end
+
+          def render(tag, value)
+            case Markup.attribute(tag, 't')
+            when 's' then @strings[value.to_i].to_s
+            when 'b' then value == '1' ? 'TRUE' : 'FALSE'
+            when nil, 'n' then number(tag, Markup.unescape(value))
+            else Markup.unescape(value)
+            end
+          end
+
+          def number(tag, value)
+            style = Markup.attribute(tag, 's')
+            return value unless style && @date_styles[style.to_i]
+
+            serial = Float(value, exception: false)
+            serial && serial >= 0 && serial <= MAX_SERIAL ? date(serial) || value : value
+          end
+
+          def date(serial)
+            day = serial.floor
+            seconds = ((serial - day) * SECONDS_A_DAY).round
+            if seconds == SECONDS_A_DAY
+              day += 1
+              seconds = 0
+            end
+            time = format('%<hours>02d:%<minutes>02d:%<seconds>02d', hours: seconds / 3600,
+                                                                     minutes: seconds % 3600 / 60,
+                                                                     seconds: seconds % 60)
+            return time if day.zero? && seconds.positive?
+
+            calendar = calendar_day(day)
+            return unless calendar
+
+            seconds.zero? ? calendar : "#{calendar} #{time}"
+          end
+
+          # Excel counts 1900 as a leap year, so serial 60 is a day that never
+          # was, and the serials after it are one day ahead of the calendar.
+          def calendar_day(day)
+            return (Date.new(1904, 1, 1) + day).iso8601 if @date1904
+            return if day.zero?
+            return '1900-02-29' if day == 60
+
+            (Date.new(1899, 12, day < 60 ? 31 : 30) + day).iso8601
+          end
+
+          def date_styles
+            formats = {}
+            styles = []
+            in_cell_formats = false
+            Markup.scan(@zip['xl/styles.xml']) do |kind, name, tag|
+              case name
+              when 'numFmt' then formats[Markup.attribute(tag, 'numFmtId').to_i] = Markup.attribute(tag, 'formatCode')
+              when 'cellXfs' then in_cell_formats = kind == :open
+              when 'xf' then if in_cell_formats && tag
+                               styles << date_format?(Markup.attribute(tag, 'numFmtId').to_i,
+                                                      formats)
+                             end
+              end
+            end
+            styles
+          end
+
+          def date_format?(id, formats)
+            code = formats[id]
+            return BUILT_IN_DATE_FORMATS.include?(id) unless code
+            return false if code.match?(/\[(?:h+|m+|s+)\]/i)
+
+            code.gsub(/"[^"]*"|\\.|_.|\*.|\[[^\]]*\]/, '').match?(/[ymdhs]/i)
+          end
+
+          def date1904?
+            Markup.tags(@zip['xl/workbook.xml'], 'workbookPr').any? do |tag|
+              TRUTHY.include?(Markup.attribute(tag, 'date1904'))
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/file_store/converters/office/zip.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+require 'zlib'
+
+module RubyLLM
+  class FileStore
+    module Converters
+      module Office
+        # The entries of a zip archive, read from its central directory,
+        # with Zip64 sizes when an entry or the directory needs them. It
+        # inflates at most MAX_SIZE in all, counting what comes out rather
+        # than what an entry declares, so a small archive cannot fill the
+        # memory.
+        class Zip # :nodoc:
+          class Error < StandardError; end # :nodoc:
+
+          END_OF_DIRECTORY = "PK\x05\x06".b
+          ZIP64_LOCATOR = "PK\x06\x07".b
+          ZIP64_END_OF_DIRECTORY = "PK\x06\x06".b
+          DIRECTORY_ENTRY = "PK\x01\x02".b
+          LOCAL_ENTRY = "PK\x03\x04".b
+          MASKED = 0xFFFFFFFF
+          MASKED_COUNT = 0xFFFF
+          ZIP64_EXTRA = 1
+          INFLATE_STEP = 4096
+          private_constant :END_OF_DIRECTORY, :ZIP64_LOCATOR, :ZIP64_END_OF_DIRECTORY, :DIRECTORY_ENTRY,
+                           :LOCAL_ENTRY, :MASKED, :MASKED_COUNT, :ZIP64_EXTRA, :INFLATE_STEP
+
+          def initialize(bytes)
+            @bytes = bytes
+            @inflated = 0
+            @entries = read_directory
+          end
+
+          def names
+            @entries.keys
+          end
+
+          def include?(name)
+            @entries.key?(name)
+          end
+
+          def [](name)
+            entry = @entries[name]
+            entry && extract(**entry)
+          end
+
+          private
+
+          def slice(offset, length)
+            part = @bytes.byteslice(offset, length)
+            raise Error, 'truncated zip archive' unless part&.bytesize == length
+
+            part
+          end
+
+          def read_directory
+            finish = @bytes.rindex(END_OF_DIRECTORY) || raise(Error, 'not a zip archive')
+            count, _size, offset = slice(finish + 10, 10).unpack('vVV')
+            count, offset = zip64_directory(finish) if count == MASKED_COUNT || offset == MASKED
+            count.times.with_object({}) { |_index, entries| offset = read_entry(offset, entries) }
+          end
+
+          def zip64_directory(finish)
+            raise Error, 'bad Zip64 locator' unless finish >= 20 && slice(finish - 20, 4) == ZIP64_LOCATOR
+
+            record = slice(finish - 12, 8).unpack1('Q<')
+            raise Error, 'bad Zip64 directory' unless slice(record, 4) == ZIP64_END_OF_DIRECTORY
+
+            slice(record + 32, 24).unpack('Q<x8Q<')
+          end
+
+          def read_entry(offset, entries)
+            raise Error, 'bad central directory' unless slice(offset, 4) == DIRECTORY_ENTRY
+
+            method, compressed, size, name_length, extra_length, comment_length =
+              slice(offset + 10, 24).unpack('vx8VVvvv')
+            local = slice(offset + 42, 4).unpack1('V')
+            raw_name = slice(offset + 46, name_length)
+            name = raw_name.tr('\\', '/').force_encoding(Encoding::UTF_8)
+            extra = slice(offset + 46 + name_length, extra_length)
+            alias_of_entry = raw_name.include?('\\') && entries.key?(name)
+            entries[name] = { method:, **sizes(extra, size:, compressed:, local:) } unless alias_of_entry
+            offset + 46 + name_length + extra_length + comment_length
+          end
+
+          # Returns the three values, taking each masked one from the Zip64
+          # extra field, in the order the format gives them, or +nil+ when
+          # the field does not have it.
+          def sizes(extra, values)
+            return values unless values.value?(MASKED)
+
+            field = zip64_field(extra).unpack('Q<*')
+            values.transform_values { |value| value == MASKED ? field.shift : value }
+          end
+
+          def zip64_field(extra)
+            position = 0
+            while position + 4 <= extra.bytesize
+              id, length = extra.byteslice(position, 4).unpack('vv')
+              return extra.byteslice(position + 4, length).to_s if id == ZIP64_EXTRA
+
+              position += 4 + length
+            end
+            ''.b
+          end
+
+          def extract(method:, size:, compressed:, local:)
+            raise Error, 'bad local entry' unless local && slice(local, 4) == LOCAL_ENTRY
+
+            name_length, extra_length = slice(local + 26, 4).unpack('vv')
+            start = local + 30 + name_length + extra_length
+            data = compressed ? slice(start, compressed) : @bytes.byteslice(start..)
+            limit = [size, MAX_SIZE - @inflated].compact.min
+            output = contents(method, data, limit)
+            @inflated += output.bytesize
+            output
+          end
+
+          def contents(method, data, limit)
+            case method
+            when 0
+              raise Error, 'entry too large' if data.bytesize > limit
+
+              data
+            when 8 then inflate(data, limit)
+            else raise Error, "unsupported compression #{method}"
+            end
+          end
+
+          def inflate(data, limit)
+            inflater = Zlib::Inflate.new(-Zlib::MAX_WBITS)
+            output = +''.b
+            (0...data.bytesize).step(INFLATE_STEP) do |offset|
+              output << inflater.inflate(data.byteslice(offset, INFLATE_STEP))
+              raise Error, 'entry too large' if output.bytesize > limit
+              break if inflater.finished?
+            end
+            raise Error, 'truncated entry' unless inflater.finished?
+
+            output
+          ensure
+            inflater&.close
+          end
+        end
+      end
+    end
+  end
+end