rubino-agent 0.3.0

data/lib/rubino/documents/converters/csv.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+module Rubino
+  module Documents
+    module Converters
+      # CSV -> a GFM Markdown table (first row = header), via the shared Table
+      # emitter. `csv` was removed from Ruby's default gems in 3.4, so we require
+      # it defensively: if it isn't present we still parse with a tiny built-in
+      # splitter (handles the common quoted-field case) rather than going
+      # unavailable -- CSV is too central to drop on a missing stdlib gem.
+      class Csv
+        def available?
+          true
+        end
+
+        def accepts?(mime, path)
+          m = mime.to_s
+          return true if ["text/csv", "application/csv"].include?(m)
+
+          File.extname(path.to_s).downcase == ".csv"
+        end
+
+        def convert(path)
+          rows = parse(path)
+          Table.emit(rows)
+        end
+
+        private
+
+        def parse(path)
+          raw = File.read(path, encoding: "bom|utf-8")
+          require "csv"
+          ::CSV.parse(raw)
+        rescue LoadError, ::CSV::MalformedCSVError
+          fallback_parse(raw)
+        end
+
+        # Minimal RFC-4180-ish parser for the no-stdlib-csv case: splits on
+        # newlines and commas, honouring double-quoted fields with embedded
+        # commas/quotes. Good enough for the common spreadsheet export.
+        def fallback_parse(raw)
+          raw.to_s.each_line.map do |line|
+            split_line(line.chomp)
+          end
+        end
+
+        def split_line(line)
+          fields = []
+          field = +""
+          in_quotes = false
+          i = 0
+          while i < line.length
+            ch = line[i]
+            if in_quotes
+              if ch == '"' && line[i + 1] == '"'
+                field << '"'
+                i += 1
+              elsif ch == '"'
+                in_quotes = false
+              else
+                field << ch
+              end
+            elsif ch == '"'
+              in_quotes = true
+            elsif ch == ","
+              fields << field
+              field = +""
+            else
+              field << ch
+            end
+            i += 1
+          end
+          fields << field
+          fields
+        end
+      end
+    end
+  end
+end

data/lib/rubino/documents/converters/docx.rb

@@ -0,0 +1,129 @@
+# frozen_string_literal: true
+
+module Rubino
+  module Documents
+    module Converters
+      # DOCX -> Markdown via the `docx` gem (MIT, OPTIONAL). markitdown gets this
+      # "for free" by going docx->HTML (mammoth) then through its HTML core; the
+      # Ruby `docx` gem instead hands us paragraphs (with a style name) and
+      # tables, so we map the structure directly:
+      #   "Heading 1".."Heading 6"  -> "#".."######"
+      #   "Title"                   -> "#"
+      #   list paragraphs           -> "- " / "1. "
+      #   bold/italic runs          -> "**"/"*"
+      #   tables                    -> GFM table via the shared Table emitter
+      # Known limitations (documented in specs): embedded images are dropped,
+      # nested tables are flattened, and run-level formatting beyond bold/italic
+      # is not preserved.
+      class Docx
+        MIMES = %w[
+          application/vnd.openxmlformats-officedocument.wordprocessingml.document
+        ].freeze
+
+        def available?
+          require "docx"
+          true
+        rescue LoadError
+          false
+        end
+
+        def accepts?(mime, path)
+          return true if MIMES.include?(mime.to_s)
+
+          File.extname(path.to_s).downcase == ".docx"
+        end
+
+        def convert(path)
+          require "docx"
+          doc = ::Docx::Document.open(path)
+          blocks = []
+          # Iterate document order when the gem exposes it; otherwise paragraphs
+          # then tables (best-effort -- the gem version dictates what's available).
+          if doc.respond_to?(:each_paragraph)
+            doc.each_paragraph { |p| blocks << paragraph_markdown(p) }
+          else
+            doc.paragraphs.each { |p| blocks << paragraph_markdown(p) }
+          end
+          doc.tables.each { |t| blocks << table_markdown(t) } if doc.respond_to?(:tables)
+          blocks.compact.reject(&:empty?).join("\n\n")
+        end
+
+        private
+
+        def paragraph_markdown(para)
+          text = inline_text(para)
+          return "" if text.strip.empty?
+
+          case paragraph_style(para)
+          when /\AHeading\s*([1-6])\z/i
+            "#{"#" * Regexp.last_match(1).to_i} #{text.strip}"
+          when /\ATitle\z/i
+            "# #{text.strip}"
+          when /\ASubtitle\z/i
+            "## #{text.strip}"
+          when /List|Bullet/i
+            "- #{text.strip}"
+          else
+            text.strip
+          end
+        end
+
+        # The gem maps the style id to a human name via styles.xml; both id and
+        # name vary by authoring tool ("Heading1" / "heading 1"), and the gem
+        # raises on a malformed paragraph, so guard and normalise to a single
+        # spaced form the case/when above matches case-insensitively.
+        def paragraph_style(para)
+          return "" unless para.respond_to?(:style)
+
+          para.style.to_s
+        rescue StandardError
+          ""
+        end
+
+        # Joins a paragraph's text runs, wrapping bold/italic runs in Markdown
+        # emphasis when the gem exposes run-level formatting.
+        def inline_text(para)
+          unless para.respond_to?(:each_text_run)
+            return para.respond_to?(:text) ? para.text.to_s : para.to_s
+          end
+
+          out = +""
+          para.each_text_run do |run|
+            t = run.respond_to?(:text) ? run.text.to_s : run.to_s
+            next if t.empty?
+
+            t = "**#{t}**" if bold?(run)
+            t = "*#{t}*" if italic?(run) && !bold?(run)
+            out << t
+          end
+          out
+        end
+
+        # The gem names these `bolded?`/`italicized?` (older forks use
+        # `bold?`/`italic?`); probe both so run emphasis survives a version bump.
+        def bold?(run)
+          (run.respond_to?(:bolded?) && run.bolded?) ||
+            (run.respond_to?(:bold?) && run.bold?)
+        rescue StandardError
+          false
+        end
+
+        def italic?(run)
+          (run.respond_to?(:italicized?) && run.italicized?) ||
+            (run.respond_to?(:italic?) && run.italic?)
+        rescue StandardError
+          false
+        end
+
+        def table_markdown(table)
+          rows = table.rows.map do |row|
+            row.cells.map { |cell| cell.respond_to?(:text) ? cell.text.to_s : cell.to_s }
+          end
+          Table.emit(rows)
+        rescue StandardError
+          ""
+        end
+      end
+    end
+  end
+end

data/lib/rubino/documents/converters/html.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Rubino
+  module Documents
+    module Converters
+      # HTML / XHTML -> Markdown via the shared HTML core (Documents::Html).
+      # Thin by design: read the file, hand the bytes to the core. This is the
+      # engine the other shaped-as-HTML converters reuse.
+      class Html
+        def available?
+          true
+        end
+
+        def accepts?(mime, path)
+          m = mime.to_s
+          return true if ["text/html", "application/xhtml+xml"].include?(m)
+
+          %w[.html .htm .xhtml].include?(File.extname(path.to_s).downcase)
+        end
+
+        def convert(path)
+          raw = File.read(path, encoding: "bom|utf-8")
+          Documents::Html.to_markdown(raw)
+        end
+      end
+    end
+  end
+end

data/lib/rubino/documents/converters/json.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Rubino
+  module Documents
+    module Converters
+      # JSON -> Markdown: pretty-printed inside a ```json fence (same as
+      # markitdown). On a parse error we fence the raw bytes verbatim rather than
+      # failing -- the model still gets the content, just unprettified.
+      class Json
+        def available?
+          true
+        end
+
+        def accepts?(mime, path)
+          m = mime.to_s
+          return true if m == "application/json" || m.end_with?("+json")
+
+          File.extname(path.to_s).downcase == ".json"
+        end
+
+        def convert(path)
+          raw = File.read(path, encoding: "bom|utf-8")
+          pretty = begin
+            ::JSON.pretty_generate(::JSON.parse(raw))
+          rescue ::JSON::ParserError
+            raw.strip
+          end
+          "```json\n#{pretty}\n```\n"
+        end
+      end
+    end
+  end
+end

data/lib/rubino/documents/converters/pdf.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Rubino
+  module Documents
+    module Converters
+      # PDF -> Markdown via `pdf-reader` (pure Ruby, MIT, OPTIONAL). Text-first:
+      # each page's text is extracted and pages are joined with a blank line.
+      # Honest limits (documented in specs):
+      #   - No OCR. A scanned / image-only PDF yields no extractable text; we
+      #     return a clear "no extractable text (scanned?)" note, not a crash.
+      #   - Multi-column / complex layout: pdf-reader gives reading order by
+      #     token position, which is imperfect for multi-column pages -- word
+      #     order may differ from the visual layout. Best-effort, not exact.
+      #   - The token-position table heuristic markitdown does with pdfplumber is
+      #     intentionally deferred; it is the hard, low-ceiling part.
+      class Pdf
+        MIMES = %w[application/pdf].freeze
+
+        def available?
+          require "pdf/reader"
+          true
+        rescue LoadError
+          false
+        end
+
+        def accepts?(mime, path)
+          return true if MIMES.include?(mime.to_s)
+
+          File.extname(path.to_s).downcase == ".pdf"
+        end
+
+        def convert(path)
+          require "pdf/reader"
+          reader = PDF::Reader.new(path)
+          pages = reader.pages.map { |page| page_text(page) }
+          text = pages.reject(&:empty?).join("\n\n")
+          return scanned_note if text.strip.empty?
+
+          text
+        rescue PDF::Reader::MalformedPDFError, PDF::Reader::UnsupportedFeatureError
+          scanned_note
+        end
+
+        private
+
+        def page_text(page)
+          page.text.to_s.gsub(/[ \t]+\n/, "\n").strip
+        rescue StandardError
+          ""
+        end
+
+        def scanned_note
+          "_(No extractable text found in this PDF -- it may be scanned or " \
+            "image-only. No OCR is performed in-process.)_"
+        end
+      end
+    end
+  end
+end

data/lib/rubino/documents/converters/plain.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Rubino
+  module Documents
+    module Converters
+      # Plain text / source code -> Markdown. The last-resort converter: it
+      # accepts any text/* MIME (and is the registry's final fallback). Markdown
+      # passes through unchanged; other source files are wrapped in a fenced code
+      # block tagged with the language inferred from the extension, so the model
+      # sees the code as code, not prose. Encoding is normalised to UTF-8.
+      class Plain
+        # Extension -> fenced-code language hint. Markdown/plain text are NOT
+        # fenced (they pass through). Anything else with a known mapping fences.
+        LANGS = {
+          ".rb" => "ruby", ".py" => "python", ".js" => "javascript",
+          ".ts" => "typescript", ".jsx" => "jsx", ".tsx" => "tsx",
+          ".go" => "go", ".rs" => "rust", ".java" => "java", ".c" => "c",
+          ".h" => "c", ".cpp" => "cpp", ".cc" => "cpp", ".hpp" => "cpp",
+          ".cs" => "csharp", ".php" => "php", ".rb_" => "ruby",
+          ".sh" => "bash", ".bash" => "bash", ".zsh" => "bash",
+          ".sql" => "sql", ".yml" => "yaml", ".yaml" => "yaml",
+          ".toml" => "toml", ".ini" => "ini", ".css" => "css",
+          ".scss" => "scss", ".swift" => "swift", ".kt" => "kotlin",
+          ".lua" => "lua", ".pl" => "perl", ".r" => "r"
+        }.freeze
+
+        MARKDOWN_EXTS = %w[.md .markdown .mdown .mkd].freeze
+
+        def available?
+          true
+        end
+
+        # Accepts anything textual: text/* MIME, the textual application/* types,
+        # or -- as the final fallback -- a file with a known code/markdown
+        # extension even when MIME is unknown.
+        def accepts?(mime, path)
+          m = mime.to_s
+          return true if m.start_with?("text/")
+          return true if textual_application?(m)
+
+          ext = File.extname(path.to_s).downcase
+          MARKDOWN_EXTS.include?(ext) || LANGS.key?(ext)
+        end
+
+        def convert(path)
+          raw = File.binread(path).to_s.dup.force_encoding("UTF-8")
+          raw = raw.scrub("�") unless raw.valid_encoding?
+          ext = File.extname(path.to_s).downcase
+
+          return raw if MARKDOWN_EXTS.include?(ext)
+
+          lang = LANGS[ext]
+          return raw if lang.nil? # unknown text: pass through as-is
+
+          "```#{lang}\n#{raw.chomp}\n```\n"
+        end
+
+        private
+
+        def textual_application?(mime)
+          mime == "application/json" || mime == "application/xml" ||
+            mime == "application/javascript" || mime == "application/x-yaml" ||
+            mime.end_with?("+json") || mime.end_with?("+xml")
+        end
+      end
+    end
+  end
+end

data/lib/rubino/documents/converters/pptx.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module Rubino
+  module Documents
+    module Converters
+      # PPTX -> Markdown via the `ruby_powerpoint` gem (MIT, OPTIONAL). Each
+      # slide becomes a `## Slide N` heading; the slide's text frames become
+      # paragraphs/bullets and speaker notes go under a `>` block quote. The gem
+      # gives us text per slide (and notes); it does not preserve shape geometry,
+      # so we emit text in document order -- good enough for an LLM to read.
+      class Pptx
+        MIMES = %w[
+          application/vnd.openxmlformats-officedocument.presentationml.presentation
+        ].freeze
+
+        def available?
+          require "ruby_powerpoint"
+          true
+        rescue LoadError
+          false
+        end
+
+        def accepts?(mime, path)
+          return true if MIMES.include?(mime.to_s)
+
+          File.extname(path.to_s).downcase == ".pptx"
+        end
+
+        def convert(path)
+          require "ruby_powerpoint"
+          ppt = RubyPowerpoint::Presentation.new(path)
+          parts = ppt.slides.each_with_index.map { |slide, i| slide_markdown(slide, i + 1) }
+          parts.compact.join("\n\n")
+        end
+
+        private
+
+        def slide_markdown(slide, number)
+          lines = ["## Slide #{number}"]
+
+          title = slide.respond_to?(:title) ? slide.title.to_s.strip : ""
+          lines << "### #{title}" unless title.empty?
+
+          texts = Array(slide.respond_to?(:text) ? slide.text : nil)
+                  .flatten
+                  .map { |t| t.to_s.strip }
+                  .reject { |t| t.empty? || t == title }
+          texts.each { |t| lines << "- #{t}" }
+
+          if slide.respond_to?(:notes)
+            notes = slide.notes.to_s.strip
+            lines << "\n> Notes: #{notes}" unless notes.empty?
+          end
+
+          return nil if lines.length == 1 # only the "## Slide N" header
+
+          lines.join("\n")
+        rescue StandardError
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/rubino/documents/converters/xlsx.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Rubino
+  module Documents
+    module Converters
+      # XLSX (and ODS/legacy XLS where roo supports them) -> Markdown. Each
+      # sheet becomes a `## SheetName` heading followed by a GFM table emitted by
+      # the shared Table emitter. The `roo` gem (MIT) is OPTIONAL: #available?
+      # reports false when it can't be required, so the registry never offers
+      # this converter on an install without roo -- the caller then falls back to
+      # the shell-extraction hint.
+      class Xlsx
+        MIMES = %w[
+          application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
+          application/vnd.oasis.opendocument.spreadsheet
+          application/vnd.ms-excel
+        ].freeze
+        EXTS = %w[.xlsx .ods .xls].freeze
+
+        def available?
+          require "roo"
+          true
+        rescue LoadError
+          false
+        end
+
+        def accepts?(mime, path)
+          return true if MIMES.include?(mime.to_s)
+
+          EXTS.include?(File.extname(path.to_s).downcase)
+        end
+
+        def convert(path)
+          require "roo"
+          book = Roo::Spreadsheet.open(path)
+          parts = book.sheets.map { |name| sheet_markdown(book, name) }.compact
+          parts.join("\n\n")
+        ensure
+          book&.close if defined?(book) && book.respond_to?(:close)
+        end
+
+        private
+
+        def sheet_markdown(book, name)
+          sheet = book.sheet(name)
+          rows = []
+          if sheet.first_row && sheet.last_row
+            (sheet.first_row..sheet.last_row).each do |r|
+              rows << (sheet.first_column..sheet.last_column).map { |c| sheet.cell(r, c) }
+            end
+          end
+          table = Table.emit(rows)
+          return nil if table.empty?
+
+          "## #{name}\n\n#{table}"
+        rescue StandardError
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/rubino/documents/converters/xml.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Rubino
+  module Documents
+    module Converters
+      # XML -> Markdown: pretty-printed inside a ```xml fence (markitdown does
+      # the same for generic XML). Uses stdlib REXML, which ships with Ruby; if
+      # pretty-printing fails we fence the raw bytes. SVG is deliberately NOT
+      # handled here -- Classify routes SVG to :text, never :document.
+      class Xml
+        def available?
+          true
+        end
+
+        def accepts?(mime, path)
+          m = mime.to_s
+          return false if m == "image/svg+xml"
+          return true if m == "application/xml" || m == "text/xml" || m.end_with?("+xml")
+
+          File.extname(path.to_s).downcase == ".xml"
+        end
+
+        def convert(path)
+          raw = File.read(path, encoding: "bom|utf-8")
+          pretty = pretty_print(raw) || raw.strip
+          "```xml\n#{pretty}\n```\n"
+        end
+
+        private
+
+        def pretty_print(raw)
+          require "rexml/document"
+          doc = REXML::Document.new(raw)
+          out = +""
+          formatter = REXML::Formatters::Pretty.new(2)
+          formatter.compact = true
+          formatter.write(doc, out)
+          out.strip.empty? ? nil : out.strip
+        rescue LoadError, StandardError
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/rubino/documents/html.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require "kramdown"
+
+module Rubino
+  module Documents
+    # The ONE HTML->Markdown core (markitdown's `HtmlConverter` / `markdownify`
+    # equivalent). Every converter that can shape its content as HTML (the html
+    # converter itself, and docx via a paragraphs->HTML step) feeds this. Built
+    # on kramdown, which is ALREADY a rubino dependency, so no new lib is added.
+    #
+    # kramdown parses HTML and emits Markdown but defaults to reference-style
+    # links ([text][1] + a [1]: url footer). LLMs read inline links more
+    # naturally, so we post-process the reference definitions back inline. We
+    # also strip non-content elements (script/style) before conversion.
+    module Html
+      module_function
+
+      # Converts an HTML string to Markdown. Returns "" on failure rather than
+      # raising -- the caller (to_markdown) treats empty as nil.
+      def to_markdown(html)
+        return "" if html.nil? || html.to_s.strip.empty?
+
+        cleaned = strip_noise(html.to_s)
+        md = Kramdown::Document.new(
+          cleaned,
+          input: "html",
+          html_to_native: true
+        ).to_kramdown
+        inline_reference_links(md).strip
+      rescue StandardError
+        ""
+      end
+
+      # Removes script/style/head blocks (their text is not document content)
+      # and the html/body document-wrapper tags, which kramdown otherwise leaves
+      # as literal `<html>...</html>` lines around the converted body. What's
+      # left is the inner content kramdown shapes into Markdown.
+      def strip_noise(html)
+        html
+          .gsub(%r{<script\b[^>]*>.*?</script>}mi, "")
+          .gsub(%r{<style\b[^>]*>.*?</style>}mi, "")
+          .gsub(%r{<head\b[^>]*>.*?</head>}mi, "")
+          .gsub(/<!--.*?-->/m, "")
+          .gsub(%r{</?(?:html|body|!doctype)\b[^>]*>}mi, "")
+      end
+
+      # Rewrites kramdown's reference-style links/images back to inline form:
+      #   [text][1] ... [1]: http://x  ->  [text](http://x)
+      # Leaves the body untouched when there are no reference definitions.
+      def inline_reference_links(markdown)
+        defs = {}
+        markdown.each_line do |line|
+          m = line.match(/^\s*\[([^\]]+)\]:\s+(\S+)(?:\s+"[^"]*")?\s*$/)
+          defs[m[1]] = m[2] if m
+        end
+        return markdown if defs.empty?
+
+        body = markdown.gsub(/(!?)\[([^\]]*)\]\[([^\]]+)\]/) do
+          bang = Regexp.last_match(1)
+          text = Regexp.last_match(2)
+          ref  = Regexp.last_match(3)
+          url  = defs[ref.empty? ? text : ref]
+          url ? "#{bang}[#{text}](#{url})" : Regexp.last_match(0)
+        end
+        # Drop the now-inlined reference-definition lines.
+        body.each_line.grep_v(/^\s*\[[^\]]+\]:\s+\S+/).join
+      end
+    end
+  end
+end