readword 0.1.1

data/lib/readword/docx.rb

@@ -0,0 +1,294 @@
+# frozen_string_literal: true
+
+require 'zip'
+
+module Readword
+  # Reads the text of a Word 2007 or later .docx (ECMA-376 WordprocessingML) from its XML parts.
+  class Docx
+    MAX_XML_BYTES = 10 * 1024 * 1024
+    MAX_XML_NODES = 400_000
+    MAX_GRID_SPAN = 63
+    MAX_STYLE_DEPTH = 16
+    UTF16_STARTS = ["\xFF\xFE".b, "\xFE\xFF".b, "<\0".b, "\0<".b].freeze
+    RELATIONSHIP = %r{/relationships/(officeDocument|header|footer|footnotes|endnotes|styles|numbering)\z}
+    SKIPPED = %w[del moveFrom instrText delText rPr pPr sectPr tblPr trPr tcPr sdtPr].freeze
+    CHARACTERS = %w[t tab ptab br cr noBreakHyphen sym].freeze
+    SEPARATOR_NOTES = %w[separator continuationSeparator continuationNotice].freeze
+    HEADING_STYLE = /\Aheading ([1-9])\z/i
+    TITLE_STYLE = /\Atitle\z/i
+    LEVEL_REFERENCE = /\A%[1-9]\z/
+    FORMATS = { 'upperRoman' => :upper_roman, 'lowerRoman' => :lower_roman, 'upperLetter' => :upper_letter,
+                'lowerLetter' => :lower_letter, 'decimalZero' => :decimal_zero, 'ordinal' => :ordinal,
+                'bullet' => :bullet, 'none' => :none }.freeze
+
+    def initialize(bytes)
+      @bytes = bytes
+    end
+
+    def blocks
+      @entries = Zip::File.open_buffer(@bytes).entries.to_h { |entry| [entry.name.downcase, entry] }
+      @budgets = { content: { bytes: 0, nodes: 0 }, formatting: { bytes: 0, nodes: 0 } }
+      @tables = 0
+      @counter = Numbering.new
+      @chains = {}
+      main = main_part
+      parts = related_parts(main)
+      directory = File.dirname(main)
+      @styles = read_styles(xml(parts['styles'].first || File.join(directory, 'styles.xml'), :formatting))
+      @lists = read_lists(xml(parts['numbering'].first || File.join(directory, 'numbering.xml'), :formatting))
+      [*unique(parts['header']), *part(main), *notes(parts['footnotes'] + parts['endnotes']), *unique(parts['footer'])]
+    end
+
+    private
+
+    def main_part
+      target = relationships('_rels/.rels').find { |type, _| type == 'officeDocument' }&.last
+      target ? resolve('.', target) : 'word/document.xml'
+    end
+
+    def related_parts(main)
+      directory = File.dirname(main)
+      parts = Hash.new { |hash, type| hash[type] = [] }
+      seen = Set[main.downcase]
+      relationships(File.join(directory, '_rels', "#{File.basename(main)}.rels")).each do |type, target|
+        path = resolve(directory, target)
+        parts[type] << path if seen.add?(path.downcase)
+      end
+      parts
+    end
+
+    def relationships(path)
+      document = xml(path)
+      return [] unless document
+
+      document.xpath('//Relationship').filter_map do |relationship|
+        type = relationship['Type'].to_s[RELATIONSHIP, 1]
+        [type, relationship['Target'].to_s] if type && relationship['TargetMode'] != 'External'
+      end
+    end
+
+    def resolve(directory, target)
+      Pathname.new('/').join(directory, target).cleanpath.to_s.delete_prefix('/')
+    end
+
+    def xml(path, budget = :content)
+      entry = @entries[path.downcase] or return
+      remaining = MAX_XML_BYTES - @budgets[budget][:bytes]
+      raise Error, TOO_LONG if entry.size > remaining
+
+      data = entry.get_input_stream { |stream| stream.read(remaining + 1) }.to_s
+      raise Error, TOO_LONG if data.bytesize > remaining
+
+      @budgets[budget][:bytes] += data.bytesize
+      count_nodes(data.count('<&='), budget)
+
+      utf16 = data.start_with?(*UTF16_STARTS)
+      data = data.b.force_encoding(Encoding::UTF_8).scrub unless utf16
+      Nokogiri::XML(data, nil, utf16 ? nil : 'UTF-8').tap do |document|
+        raise Error, UNREADABLE if document.internal_subset
+
+        document.remove_namespaces!
+      end
+    end
+
+    def part(path)
+      @counter.reset
+      content(xml(path))
+    end
+
+    def unique(paths)
+      paths.flat_map { |path| part(path) }.reject { |block| block.is_a?(Paragraph) && Readword.blank?(block.text) }.uniq
+    end
+
+    def notes(paths)
+      paths.flat_map do |path|
+        @counter.reset
+        found = Array(xml(path)&.xpath('//*[self::footnote or self::endnote]'))
+        found.reject { |note| SEPARATOR_NOTES.include?(note['type']) }.flat_map { |note| content(note) }
+      end
+    end
+
+    def content(node)
+      return [] unless node
+
+      node.element_children.flat_map do |child|
+        case child.name
+        when 'p' then paragraph_blocks(child)
+        when 'tbl' then table_blocks(child)
+        when 'AlternateContent' then content(alternative(child))
+        when *SKIPPED then []
+        else content(child)
+        end
+      end
+    end
+
+    def paragraph_blocks(paragraph)
+      boxes = []
+      text = inline(paragraph, boxes, [])
+      properties = child_named(paragraph, 'pPr')
+      style = value(properties, 'pStyle')
+      block = Paragraph.new(text: text, heading: heading_level(properties, style), list: list_item(properties, style))
+      [block, *boxes.flat_map { |box| content(box) }]
+    end
+
+    def heading_level(properties, style)
+      chain = style_chain(style)
+      outline = value(properties, 'outlineLvl') || chain.filter_map { |entry| entry[:outline] }.first
+      level = if outline then outline.to_i
+              elsif TITLE_STYLE.match?(chain.first&.fetch(:name).to_s) then 0
+              else chain.filter_map { |entry| HEADING_STYLE.match(entry[:name].to_s)&.then { |match| match[1].to_i - 1 } }.first
+              end
+      [level + 1, 6].min if level&.between?(0, 8)
+    end
+
+    def list_item(properties, style)
+      own = child_named(properties, 'numPr')
+      num_id = value(own, 'numId')
+      level = value(own, 'ilvl')
+      unless num_id
+        inherited = style_chain(style).find { |entry| entry[:num_id] }
+        num_id = inherited&.fetch(:num_id)
+        level ||= inherited&.fetch(:level) || style_level(num_id, style)
+      end
+      list = @lists[num_id] if num_id && num_id != '0' && level.to_i.between?(0, 8)
+      list && @counter.next_item(list[:key], list[:levels], level.to_i, list[:starts])
+    end
+
+    def style_level(num_id, style)
+      @lists[num_id]&.fetch(:levels)&.find { |_, definition| definition.style == style }&.first
+    end
+
+    def style_chain(id)
+      @chains[id] ||= begin
+        chain = []
+        while id && (entry = @styles[id]) && chain.size < MAX_STYLE_DEPTH && !chain.include?(entry)
+          chain << entry
+          id = entry[:based_on]
+        end
+        chain
+      end
+    end
+
+    def read_styles(document)
+      return {} unless document
+
+      document.xpath('//style[@type="paragraph"]').to_h do |style|
+        properties = child_named(style, 'pPr')
+        numbering = child_named(properties, 'numPr')
+        [style['styleId'], { name: value(style, 'name'), based_on: value(style, 'basedOn'), outline: value(properties, 'outlineLvl'),
+                             num_id: value(numbering, 'numId'), level: value(numbering, 'ilvl') }]
+      end
+    end
+
+    def read_lists(document)
+      return {} unless document
+
+      abstracts = {}
+      linked = {}
+      document.xpath('//abstractNum').each do |abstract|
+        levels = abstract.element_children.select { |child| child.name == 'lvl' }.to_h { |lvl| [lvl['ilvl'].to_i, list_level(lvl)] }
+        abstracts[abstract['abstractNumId']] = { levels: levels, link: value(abstract, 'numStyleLink') }
+        link = value(abstract, 'styleLink')
+        linked[link] = levels if link
+      end
+      document.xpath('//num').to_h do |num|
+        abstract_id = value(num, 'abstractNumId')
+        abstract = abstracts.fetch(abstract_id, { levels: {} })
+        levels = abstract[:levels].empty? && abstract[:link] ? linked.fetch(abstract[:link], {}) : abstract[:levels]
+        overrides = num.element_children.select { |child| child.name == 'lvlOverride' }
+        starts = overrides.to_h { |override| [override['ilvl'].to_i, value(override, 'startOverride')] }.compact.transform_values(&:to_i)
+        redefined = overrides.filter_map { |override| child_named(override, 'lvl') }
+        levels = levels.merge(redefined.to_h { |lvl| [lvl['ilvl'].to_i, list_level(lvl)] })
+        [num['numId'], { key: starts.empty? ? "abstract #{abstract_id}" : "num #{num['numId']}", levels: levels, starts: starts }]
+      end
+    end
+
+    def list_level(lvl)
+      Numbering::Level.new(start: value(lvl, 'start')&.to_i, format: FORMATS.fetch(value(lvl, 'numFmt'), :decimal),
+                           template: template(value(lvl, 'lvlText').to_s), style: value(lvl, 'pStyle'))
+    end
+
+    def template(text)
+      text.split(/(%[1-9])/).reject(&:empty?).map { |part| LEVEL_REFERENCE.match?(part) ? part[1].to_i - 1 : part }
+    end
+
+    def inline(node, boxes, fields)
+      node.element_children.map do |child|
+        case child.name
+        when *CHARACTERS then fields.include?(:code) ? '' : character(child)
+        when 'fldChar' then field_char(child['fldCharType'], fields)
+        when 'txbxContent'
+          boxes << child
+          ''
+        when 'AlternateContent' then inline(alternative(child), boxes, fields)
+        when *SKIPPED then ''
+        else inline(child, boxes, fields)
+        end
+      end.join
+    end
+
+    def character(node)
+      case node.name
+      when 't' then node.text.scrub('')
+      when 'tab', 'ptab' then "\t"
+      when 'br', 'cr' then "\n"
+      when 'noBreakHyphen' then '-'
+      when 'sym' then Readword.symbol(node['font'], node['char'].to_s.hex)
+      end
+    end
+
+    def field_char(type, fields)
+      case type
+      when 'begin' then fields.push(:code)
+      when 'separate' then fields[-1] = :result if fields.last == :code
+      when 'end' then fields.pop
+      end
+      ''
+    end
+
+    def alternative(node)
+      child_named(node, 'Choice') || child_named(node, 'Fallback') || node
+    end
+
+    def table_blocks(table)
+      @tables += 1
+      children_named(table, 'tr').flat_map do |row|
+        cells = children_named(row, 'tc')
+        tables_before = @tables
+        rendered = cells.map { |cell| content(cell) }
+        next rendered.flatten if @tables > tables_before
+
+        values = cells.zip(rendered).flat_map { |cell, blocks| cell_values(cell, blocks) }
+        values.all? { |text| Readword.blank?(text) } ? [] : [Row.new(values)]
+      end
+    end
+
+    def cell_values(cell, blocks)
+      span = value(child_named(cell, 'tcPr'), 'gridSpan').to_i.clamp(1, MAX_GRID_SPAN)
+      count_nodes(span - 1)
+      [Readword.squish(blocks.map { |block| Markdown.plain(block) }.join(' ')), *Array.new(span - 1, '')]
+    end
+
+    def count_nodes(count, budget = :content)
+      @budgets[budget][:nodes] += count
+      raise Error, TOO_LONG if @budgets[budget][:nodes] > MAX_XML_NODES
+    end
+
+    def children_named(node, name)
+      node.element_children.flat_map do |child|
+        next [child] if child.name == name
+        next [] if child.name == 'tbl' || SKIPPED.include?(child.name)
+
+        children_named(child, name)
+      end
+    end
+
+    def child_named(node, name)
+      node&.element_children&.find { |child| child.name == name }
+    end
+
+    def value(node, name)
+      child_named(node, name)&.[]('val')&.scrub('')
+    end
+  end
+end

data/lib/readword/markdown.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module Readword
+  Paragraph = Struct.new(:text, :heading, :list, keyword_init: true)
+  ListItem = Struct.new(:level, :label, :bullet, :list, keyword_init: true)
+  Row = Struct.new(:cells)
+
+  # Writes the readers' paragraphs and table rows as Markdown: headings, nested lists and pipe tables.
+  module Markdown
+    ORDERED_MARKER = /\A\d{1,9}[.)]\z/
+    LITERAL_START = /\A[#>|]/
+
+    module_function
+
+    def render(blocks)
+      parts = []
+      rows = []
+      offsets = []
+      blocks.each do |block|
+        next rows << block.cells if block.is_a?(Row)
+
+        parts << [:block, table(rows)] if rows.any?
+        rows = []
+        next if Readword.blank?(block.text)
+
+        if block.list && !block.heading
+          offsets.clear unless parts.last && parts.last[2] == block.list.list
+          parts << [:item, item(block, offsets), block.list.list]
+        else
+          offsets.clear
+          parts << [:block, block.heading ? heading(block) : escape(block.text.lstrip)]
+        end
+      end
+      parts << [:block, table(rows)] if rows.any?
+      join(parts)
+    end
+
+    def plain(block)
+      return block.cells.join(' | ') if block.is_a?(Row)
+
+      list = block.list
+      return block.text unless list
+
+      "#{list.bullet ? '•' : list.label} #{block.text}"
+    end
+
+    def join(parts)
+      text = +''
+      parts.each_with_index do |(kind, chunk, list), index|
+        previous_kind, _, previous_list = parts[index - 1]
+        text << (kind == :item && previous_kind == :item && list == previous_list ? "\n" : "\n\n") if index.positive?
+        text << chunk
+      end
+      text
+    end
+
+    def heading(block)
+      label = block.list && !block.list.bullet ? "#{block.list.label} " : ''
+      "#{'#' * block.heading} #{label}#{Readword.squish(block.text)}"
+    end
+
+    def item(block, offsets)
+      list = block.list
+      level = list.level
+      marker = list.bullet || !ORDERED_MARKER.match?(list.label) ? '-' : list.label
+      text = block.text.strip
+      text = "#{list.label} #{text}" if marker == '-' && !list.bullet
+      indent = (0...level).sum { |parent| offsets[parent] || 2 }
+      offsets[level] = marker.size + 1
+      offsets.slice!((level + 1)..)
+      "#{' ' * indent}#{marker} #{text.gsub("\n", "\n#{' ' * (indent + marker.size + 1)}")}"
+    end
+
+    def table(rows)
+      width = rows.map(&:size).max
+      lines = rows.map do |cells|
+        padded = cells.map { |cell| cell.gsub('|', '\|') } + Array.new(width - cells.size, '')
+        "| #{padded.join(' | ')} |"
+      end
+      lines.insert(1, "| #{Array.new(width, '---').join(' | ')} |").join("\n")
+    end
+
+    def escape(text)
+      LITERAL_START.match?(text) ? "\\#{text}" : text
+    end
+  end
+end

data/lib/readword/numbering.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module Readword
+  # List numbers the way Word counts them: one counter per list and level, and deeper levels restart
+  # whenever a higher level moves on.
+  class Numbering
+    Level = Struct.new(:start, :format, :template, :style, keyword_init: true)
+
+    ROMAN = { 1000 => 'M', 900 => 'CM', 500 => 'D', 400 => 'CD', 100 => 'C', 90 => 'XC',
+              50 => 'L', 40 => 'XL', 10 => 'X', 9 => 'IX', 5 => 'V', 4 => 'IV', 1 => 'I' }.freeze
+
+    def initialize
+      reset
+    end
+
+    def reset
+      @counts = {}
+    end
+
+    def next_item(key, levels, level, starts = {})
+      definition = levels[level] or return
+      counts = (@counts[key] ||= {})
+      counts.delete_if { |deeper, _| deeper > level }
+      (0...level).each { |higher| counts[higher] ||= start(levels, higher, starts) }
+      counts[level] = counts.key?(level) ? counts[level] + 1 : start(levels, level, starts)
+      return ListItem.new(level: level, label: nil, bullet: true, list: key) if definition.format == :bullet
+
+      label = definition.template.map do |part|
+        part.is_a?(Integer) ? number(counts[part] || start(levels, part, starts), levels[part]&.format) : part
+      end.join.strip
+      ListItem.new(level: level, label: label, bullet: false, list: key) unless label.empty?
+    end
+
+    private
+
+    def start(levels, level, starts)
+      starts[level] || levels[level]&.start || 1
+    end
+
+    def number(value, format)
+      case format
+      when :none then ''
+      when :upper_roman then roman(value)
+      when :lower_roman then roman(value).downcase
+      when :upper_letter then letter(value)
+      when :lower_letter then letter(value).downcase
+      when :decimal_zero then value.between?(0, 9) ? "0#{value}" : value.to_s
+      when :ordinal then ordinal(value)
+      else value.to_s
+      end
+    end
+
+    def roman(value)
+      return value.to_s unless value.between?(1, 3999)
+
+      ROMAN.reduce(+'') do |text, (amount, letters)|
+        count, value = value.divmod(amount)
+        text << (letters * count)
+      end
+    end
+
+    def letter(value)
+      return value.to_s unless value.positive?
+
+      (('A'.ord + ((value - 1) % 26)).chr * (((value - 1) / 26) + 1))
+    end
+
+    def ordinal(value)
+      suffix = if (11..13).cover?(value % 100) then 'th'
+               else { 1 => 'st', 2 => 'nd', 3 => 'rd' }.fetch(value % 10, 'th')
+               end
+      "#{value}#{suffix}"
+    end
+  end
+end

data/lib/readword/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Readword
+  VERSION = '0.1.1'
+end

data/lib/readword.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+require 'nokogiri'
+require 'pathname'
+require 'set'
+require 'zip'
+
+require_relative 'readword/version'
+
+# Turns Word documents (.doc and .docx) into Markdown text.
+module Readword
+  SYMBOL_FONT = {
+    0x44 => 'Δ', 0x61 => 'α', 0x62 => 'β', 0x63 => 'χ', 0x64 => 'δ', 0x65 => 'ε', 0x67 => 'γ', 0x6B => 'κ',
+    0x6C => 'λ', 0x6D => 'µ', 0x70 => 'π', 0x72 => 'ρ', 0x73 => 'σ', 0x74 => 'τ', 0x77 => 'ω', 0xA3 => '≤',
+    0xB0 => '°', 0xB1 => '±', 0xB3 => '≥', 0xB4 => '×', 0xB7 => '•', 0xB9 => '≠', 0xBB => '≈'
+  }.freeze
+  ZIP_SIGNATURE = "PK\x03\x04".b.freeze
+
+  UNREADABLE = :unreadable
+  NO_TEXT = :no_text
+  TOO_LONG = :too_long
+  PASSWORD_PROTECTED = :password_protected
+  OLD_FORMAT = :old_format
+
+  MESSAGES = {
+    UNREADABLE => 'The file could not be read as a Word document (.doc or .docx).',
+    NO_TEXT => 'The Word document has no text.',
+    TOO_LONG => 'The Word document is longer than the reader takes.',
+    PASSWORD_PROTECTED => 'The Word document is password-protected.',
+    OLD_FORMAT => 'The Word document is older than Word 97 (Word 6 or 95), which is not supported.'
+  }.freeze
+
+  # Raised when a file cannot be turned into text; #reason is one of the symbols above.
+  class Error < StandardError
+    attr_reader :reason
+
+    def initialize(reason = UNREADABLE)
+      @reason = reason
+      super(MESSAGES.fetch(reason))
+    end
+  end
+
+  def self.convert_file(path, **options)
+    convert(File.binread(path), **options)
+  end
+
+  def self.convert(bytes, max_characters: nil)
+    text = Markdown.render(reader(bytes.b).blocks).split("\n").map(&:rstrip).join("\n").gsub(/\n{3,}/, "\n\n").strip
+    raise Error, NO_TEXT if text.empty?
+    raise Error, TOO_LONG if max_characters && text.size > max_characters
+
+    text
+  rescue Error
+    raise
+  rescue StandardError
+    raise Error, UNREADABLE
+  end
+
+  def self.reader(bytes)
+    if bytes.start_with?(CompoundFile::SIGNATURE) then Doc.new(bytes)
+    elsif bytes.start_with?(ZIP_SIGNATURE) then Docx.new(bytes)
+    else raise Error, UNREADABLE
+    end
+  end
+  private_class_method :reader
+
+  def self.symbol(font, code)
+    return SYMBOL_FONT.fetch(code & 0xFF, '') if font.to_s.casecmp?('Symbol')
+
+    code.between?(0x20, 0xD7FF) ? code.chr(Encoding::UTF_8) : ''
+  end
+
+  def self.blank?(text)
+    text.nil? || text.match?(/\A[[:space:]]*\z/)
+  end
+
+  def self.squish(text)
+    text.gsub(/[[:space:]]+/, ' ').strip
+  end
+end
+
+require_relative 'readword/markdown'
+require_relative 'readword/numbering'
+require_relative 'readword/binary'
+require_relative 'readword/compound_file'
+require_relative 'readword/doc'
+require_relative 'readword/docx'

metadata

@@ -0,0 +1,96 @@
+--- !ruby/object:Gem::Specification
+name: readword
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- urbullet
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2026-10-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.13'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.13'
+- !ruby/object:Gem::Dependency
+  name: rubyzip
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.3'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.3'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4'
+description: Reads Word 97-2003 (.doc) and Word 2007+ (.docx) files without LibreOffice,
+  Java or antiword, and prints their text, tables, headers, footers and footnotes
+  as Markdown. Built for feeding Word documents to LLMs and coding agents, and safe
+  on untrusted files.
+email:
+executables:
+- readword
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- LICENSE.txt
+- README.md
+- exe/readword
+- lib/readword.rb
+- lib/readword/binary.rb
+- lib/readword/compound_file.rb
+- lib/readword/doc.rb
+- lib/readword/docx.rb
+- lib/readword/markdown.rb
+- lib/readword/numbering.rb
+- lib/readword/version.rb
+homepage: https://github.com/urbullet/readword
+licenses:
+- MIT
+metadata:
+  source_code_uri: https://github.com/urbullet/readword
+  changelog_uri: https://github.com/urbullet/readword/blob/main/CHANGELOG.md
+  bug_tracker_uri: https://github.com/urbullet/readword/issues
+  rubygems_mfa_required: 'true'
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.1'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.11
+signing_key:
+specification_version: 4
+summary: Turn Word .doc and .docx files into Markdown, in plain Ruby.
+test_files: []