markdownator 0.1.0

data/lib/markdownator/converters/pdf.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Markdownator
+  module Converters
+    # Extracts text from a PDF (one block per page) using the `pdf-reader` gem.
+    class Pdf < Base
+      def accepts?(io, stream_info)
+        return true if matches?(stream_info, extensions: %w[pdf], mimetypes: %w[application/pdf])
+
+        magic_pdf?(io)
+      end
+
+      def convert(io, _stream_info, **_options)
+        Markdownator.require_optional("pdf-reader", feature: "PDF conversion")
+
+        reader = PDF::Reader.new(io)
+        pages = reader.pages.map { |page| page.text.strip }
+        pages.reject!(&:empty?)
+        Result.new(
+          markdown: pages.join("\n\n---\n\n"),
+          metadata: { page_count: reader.page_count }
+        )
+      rescue PDF::Reader::MalformedPDFError, PDF::Reader::UnsupportedFeatureError => e
+        raise FileConversionError, "Could not read PDF: #{e.message}"
+      end
+
+      private
+
+      def magic_pdf?(io)
+        io.rewind if io.respond_to?(:rewind)
+        io.read(5) == "%PDF-"
+      ensure
+        io.rewind if io.respond_to?(:rewind)
+      end
+    end
+  end
+end

data/lib/markdownator/converters/plain_text.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Markdownator
+  module Converters
+    # Passes plain text (and Markdown) through unchanged.
+    class PlainText < Base
+      EXTENSIONS = %w[txt text md markdown].freeze
+      MIMETYPES = %w[text/plain text/markdown].freeze
+
+      def accepts?(_io, stream_info)
+        return true if matches?(stream_info, extensions: EXTENSIONS, mimetypes: MIMETYPES)
+
+        mime = stream_info.guessed_mimetype
+        !mime.nil? && mime.start_with?("text/")
+      end
+
+      def convert(io, stream_info, **_options)
+        Result.new(markdown: read_all(io, stream_info).strip)
+      end
+    end
+  end
+end

data/lib/markdownator/converters/pptx.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Markdownator
+  module Converters
+    # Converts a PowerPoint .pptx deck into Markdown: a `## Slide N` heading per
+    # slide followed by its text (one line per paragraph).
+    class Pptx < Base
+      SLIDE_PATTERN = %r{\Appt/slides/slide(\d+)\.xml\z}.freeze
+
+      def accepts?(_io, stream_info)
+        matches?(
+          stream_info,
+          extensions: %w[pptx],
+          mimetypes: %w[application/vnd.openxmlformats-officedocument.presentationml.presentation]
+        )
+      end
+
+      def convert(io, _stream_info, **_options)
+        Markdownator.require_optional("zip", feature: "PPTX conversion")
+        Markdownator.require_optional("nokogiri", feature: "PPTX conversion")
+
+        sections = []
+        ::Zip::File.open_buffer(io) do |zip|
+          slides = zip.entries.select { |e| e.name.match?(SLIDE_PATTERN) }
+          slides.sort_by! { |e| e.name[SLIDE_PATTERN, 1].to_i }
+          slides.each_with_index do |entry, index|
+            sections << render_slide(entry.get_input_stream.read, index + 1)
+          end
+        end
+        Result.new(markdown: sections.join("\n\n"))
+      end
+
+      private
+
+      def render_slide(xml, number)
+        doc = Nokogiri::XML(xml)
+        doc.remove_namespaces!
+        lines = doc.xpath("//p").map do |para|
+          para.xpath(".//t").map(&:text).join.strip
+        end
+        lines.reject!(&:empty?)
+        body = lines.empty? ? "" : "\n\n#{lines.join("\n")}"
+        "## Slide #{number}#{body}"
+      end
+    end
+  end
+end

data/lib/markdownator/converters/xlsx.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require "tempfile"
+
+module Markdownator
+  module Converters
+    # Converts an Excel .xlsx workbook into Markdown: one `## SheetName` heading
+    # and a Markdown table per sheet, using the `roo` gem.
+    class Xlsx < Base
+      def accepts?(_io, stream_info)
+        matches?(
+          stream_info,
+          extensions: %w[xlsx],
+          mimetypes: %w[application/vnd.openxmlformats-officedocument.spreadsheetml.sheet]
+        )
+      end
+
+      def convert(io, _stream_info, **_options)
+        Markdownator.require_optional("roo", feature: "XLSX conversion")
+
+        with_tempfile(io) do |path|
+          workbook = Roo::Excelx.new(path)
+          sections = workbook.sheets.map { |name| render_sheet(workbook, name) }
+          Result.new(markdown: sections.compact.join("\n\n"))
+        end
+      rescue StandardError => e
+        raise FileConversionError, "Could not read XLSX: #{e.message}"
+      end
+
+      private
+
+      def render_sheet(workbook, name)
+        sheet = workbook.sheet(name)
+        rows = (1..sheet.last_row.to_i).map do |r|
+          (1..sheet.last_column.to_i).map { |c| sheet.cell(r, c) }
+        end
+        table = markdown_table(rows)
+        table.empty? ? nil : "## #{name}\n\n#{table}"
+      end
+
+      def with_tempfile(io)
+        Tempfile.create(["markdownator", ".xlsx"]) do |file|
+          file.binmode
+          file.write(io.read)
+          file.flush
+          yield file.path
+        end
+      end
+    end
+  end
+end

data/lib/markdownator/converters/xml.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Markdownator
+  module Converters
+    # Converts XML into an indented Markdown outline of elements and their text.
+    class Xml < Base
+      def accepts?(_io, stream_info)
+        matches?(stream_info, extensions: %w[xml], mimetypes: %w[application/xml text/xml])
+      end
+
+      def convert(io, stream_info, **_options)
+        Markdownator.require_optional("nokogiri", feature: "XML conversion")
+        doc = Nokogiri::XML(read_all(io, stream_info))
+        raise FileConversionError, "Could not parse XML" if doc.root.nil?
+
+        lines = []
+        walk(doc.root, 0, lines)
+        Result.new(markdown: lines.join("\n"))
+      end
+
+      private
+
+      def walk(node, depth, lines)
+        indent = "  " * depth
+        children = node.element_children
+        own_text = node.xpath("./text()").map(&:text).join(" ").gsub(/\s+/, " ").strip
+
+        label = node.name
+        label = "#{label}: #{own_text}" unless own_text.empty?
+        lines << "#{indent}- #{label}"
+
+        children.each { |child| walk(child, depth + 1, lines) }
+      end
+    end
+  end
+end

data/lib/markdownator/converters/zip.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require "stringio"
+
+module Markdownator
+  module Converters
+    # Converts a ZIP archive by recursing each contained file back through the
+    # engine and concatenating the results under per-file headings.
+    class Zip < Base
+      def accepts?(io, stream_info)
+        return true if matches?(stream_info, extensions: %w[zip], mimetypes: %w[application/zip])
+
+        magic_zip?(io)
+      end
+
+      def convert(io, _stream_info, **options)
+        Markdownator.require_optional("zip", feature: "ZIP conversion")
+        engine = options[:engine] || Engine.new
+
+        sections = []
+        ::Zip::File.open_buffer(io) do |zip|
+          zip.entries.sort_by(&:name).each do |entry|
+            next if entry.directory?
+
+            section = convert_entry(engine, entry, options)
+            sections << section unless section.nil?
+          end
+        end
+        Result.new(markdown: sections.join("\n\n"))
+      end
+
+      private
+
+      def convert_entry(engine, entry, options)
+        stream_info = StreamInfo.new(
+          extension: File.extname(entry.name),
+          filename: File.basename(entry.name)
+        )
+        result = engine.convert_stream(
+          StringIO.new(entry.get_input_stream.read),
+          stream_info,
+          **options.reject { |k, _| k == :engine }
+        )
+        body = result.markdown.strip
+        body.empty? ? nil : "## #{entry.name}\n\n#{body}"
+      rescue UnsupportedFormatError, FileConversionError
+        nil
+      end
+
+      def magic_zip?(io)
+        io.rewind if io.respond_to?(:rewind)
+        io.read(4) == "PK\x03\x04"
+      ensure
+        io.rewind if io.respond_to?(:rewind)
+      end
+    end
+  end
+end

data/lib/markdownator/engine.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+require "stringio"
+require "uri"
+require "net/http"
+
+module Markdownator
+  # Orchestrator that holds an ordered list of converters and dispatches an
+  # input (local path, URL, or IO stream) to the first converter that accepts it.
+  class Engine
+    # Default converters, in priority order. More specific formats (the
+    # ZIP-based Office/EPUB containers) must come before the generic ZIP
+    # converter, and the plain-text fallback comes last.
+    DEFAULT_CONVERTER_ORDER = %i[
+      docx xlsx pptx epub zip pdf image html csv json xml plain_text
+    ].freeze
+
+    attr_reader :converters
+
+    # @param converters [Array<Converters::Base>] custom converter chain.
+    # @param options [Hash] default options threaded into every conversion
+    #   (e.g. `captioner:`).
+    def initialize(converters: nil, **options)
+      @converters = converters || self.class.default_converters
+      @default_options = options
+    end
+
+    def self.default_converters
+      DEFAULT_CONVERTER_ORDER.map do |name|
+        Converters.const_get(camelize(name)).new
+      end
+    end
+
+    def self.camelize(name)
+      name.to_s.split("_").map(&:capitalize).join
+    end
+
+    # Permissive entry point: dispatches based on what `source` looks like.
+    def convert(source, **options)
+      if source.respond_to?(:read)
+        convert_stream(source, options.delete(:stream_info), **options)
+      elsif url?(source)
+        convert_url(source, **options)
+      else
+        convert_local(source, **options)
+      end
+    end
+
+    # Converts a local file path.
+    def convert_local(path, **options)
+      raise FileConversionError, "No such file: #{path}" unless File.file?(path)
+
+      stream_info = StreamInfo.new(
+        extension: File.extname(path),
+        filename: File.basename(path),
+        local_path: path
+      )
+      File.open(path, "rb") do |io|
+        convert_stream(io, stream_info, **options)
+      end
+    end
+
+    # Fetches an http(s) URL and converts the response body.
+    def convert_url(url, **options)
+      uri = URI.parse(url)
+      response = Net::HTTP.get_response(uri)
+      raise FileConversionError, "HTTP #{response.code} fetching #{url}" unless response.is_a?(Net::HTTPSuccess)
+
+      stream_info = StreamInfo.new(
+        mimetype: response.content_type,
+        extension: File.extname(uri.path.to_s),
+        charset: response.type_params["charset"],
+        filename: File.basename(uri.path.to_s),
+        url: url
+      )
+      convert_stream(StringIO.new(response.body), stream_info, **options)
+    end
+
+    # Converts an open IO stream. `stream_info` provides format hints.
+    def convert_stream(io, stream_info = nil, **options)
+      stream_info ||= StreamInfo.new
+      opts = @default_options.merge(options)
+      opts[:engine] = self
+
+      converter = pick_converter(io, stream_info)
+      raise UnsupportedFormatError, describe_unsupported(stream_info) if converter.nil?
+
+      io.rewind if io.respond_to?(:rewind)
+      converter.convert(io, stream_info, **opts)
+    end
+
+    private
+
+    def pick_converter(io, stream_info)
+      converters.find do |converter|
+        io.rewind if io.respond_to?(:rewind)
+        converter.accepts?(io, stream_info)
+      end
+    end
+
+    def url?(source)
+      source.is_a?(String) && source.match?(%r{\Ahttps?://}i)
+    end
+
+    def describe_unsupported(stream_info)
+      hint = stream_info.filename || stream_info.url || stream_info.guessed_mimetype || "the given stream"
+      "No converter accepted #{hint}"
+    end
+  end
+end

data/lib/markdownator/errors.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Markdownator
+  # Base error for all Markdownator failures.
+  class Error < StandardError; end
+
+  # Raised when no registered converter accepts the given input.
+  class UnsupportedFormatError < Error; end
+
+  # Raised when a converter needs an optional gem that is not installed.
+  class MissingDependencyError < Error; end
+
+  # Raised when a converter accepts the input but fails to convert it.
+  class FileConversionError < Error; end
+end

data/lib/markdownator/result.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Markdownator
+  # The result of a conversion: the produced Markdown plus optional metadata.
+  #
+  # `#to_s` and `#text_content` both return the Markdown so the result is
+  # convenient to print or interpolate.
+  class Result
+    attr_reader :markdown, :title, :metadata
+
+    def initialize(markdown:, title: nil, metadata: {})
+      @markdown = markdown.to_s
+      @title = title
+      @metadata = metadata || {}
+    end
+
+    def to_s
+      markdown
+    end
+
+    alias text_content markdown
+  end
+end

data/lib/markdownator/stream_info.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module Markdownator
+  # Immutable value object describing a stream of bytes to be converted.
+  #
+  # It carries the hints (extension, mimetype, charset, filename, url,
+  # local_path) that converters use to decide whether they can handle a given
+  # input.
+  class StreamInfo
+    # Maps a lower-case file extension (without the dot) to a mimetype.
+    EXTENSION_TO_MIMETYPE = {
+      "txt" => "text/plain",
+      "text" => "text/plain",
+      "md" => "text/markdown",
+      "markdown" => "text/markdown",
+      "html" => "text/html",
+      "htm" => "text/html",
+      "csv" => "text/csv",
+      "json" => "application/json",
+      "xml" => "application/xml",
+      "pdf" => "application/pdf",
+      "docx" => "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+      "xlsx" => "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+      "pptx" => "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+      "epub" => "application/epub+zip",
+      "zip" => "application/zip",
+      "jpg" => "image/jpeg",
+      "jpeg" => "image/jpeg",
+      "png" => "image/png",
+      "gif" => "image/gif",
+      "tif" => "image/tiff",
+      "tiff" => "image/tiff"
+    }.freeze
+
+    attr_reader :mimetype, :extension, :charset, :filename, :url, :local_path
+
+    def initialize(mimetype: nil, extension: nil, charset: nil, filename: nil, url: nil, local_path: nil)
+      @mimetype = mimetype
+      @extension = normalize_extension(extension)
+      @charset = charset
+      @filename = filename
+      @url = url
+      @local_path = local_path
+      freeze
+    end
+
+    # Returns a new StreamInfo with the given attributes overridden, filling in
+    # any attribute not provided from the current instance.
+    def copy_with(**overrides)
+      self.class.new(
+        mimetype: overrides.fetch(:mimetype, mimetype),
+        extension: overrides.fetch(:extension, extension),
+        charset: overrides.fetch(:charset, charset),
+        filename: overrides.fetch(:filename, filename),
+        url: overrides.fetch(:url, url),
+        local_path: overrides.fetch(:local_path, local_path)
+      )
+    end
+
+    # Best-effort mimetype: the explicit one, otherwise derived from extension.
+    def guessed_mimetype
+      mimetype || EXTENSION_TO_MIMETYPE[extension]
+    end
+
+    private
+
+    def normalize_extension(ext)
+      return nil if ext.nil? || ext.empty?
+
+      ext.to_s.downcase.delete_prefix(".")
+    end
+  end
+end

data/lib/markdownator/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Markdownator
+  VERSION = "0.1.0"
+end

data/lib/markdownator.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require_relative "markdownator/version"
+require_relative "markdownator/errors"
+require_relative "markdownator/stream_info"
+require_relative "markdownator/result"
+require_relative "markdownator/converters/base"
+require_relative "markdownator/converters/plain_text"
+require_relative "markdownator/converters/html"
+require_relative "markdownator/converters/csv"
+require_relative "markdownator/converters/json"
+require_relative "markdownator/converters/xml"
+require_relative "markdownator/converters/docx"
+require_relative "markdownator/converters/xlsx"
+require_relative "markdownator/converters/pptx"
+require_relative "markdownator/converters/pdf"
+require_relative "markdownator/converters/epub"
+require_relative "markdownator/converters/zip"
+require_relative "markdownator/converters/image"
+require_relative "markdownator/engine"
+
+# Convert assorted file formats (Office docs, PDF, HTML, structured data,
+# archives, images) into LLM-friendly Markdown.
+module Markdownator
+  class << self
+    # Convert a local path, http(s) URL, or open IO stream to Markdown.
+    # @return [Markdownator::Result]
+    def convert(source, **options)
+      default_engine.convert(source, **options)
+    end
+
+    # Convert a local file path to Markdown.
+    def convert_local(path, **options)
+      default_engine.convert_local(path, **options)
+    end
+
+    # Convert an open IO stream to Markdown. `stream_info` supplies format hints.
+    def convert_stream(io, stream_info = nil, **options)
+      default_engine.convert_stream(io, stream_info, **options)
+    end
+
+    # Lazily require an optional gem, raising a helpful error if it is missing.
+    def require_optional(gem_name, feature:)
+      require gem_name
+    rescue LoadError
+      raise MissingDependencyError,
+            "#{feature} requires the '#{gem_name}' gem. Add it to your Gemfile: gem \"#{gem_name}\""
+    end
+
+    private
+
+    def default_engine
+      @default_engine ||= Engine.new
+    end
+  end
+end

data/markdownator.gemspec

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require_relative "lib/markdownator/version"
+
+Gem::Specification.new do |spec|
+  spec.name = "markdownator"
+  spec.version = Markdownator::VERSION
+  spec.authors = ["alexrupom"]
+  spec.email = ["alexrupom@hotmail.com"]
+
+  spec.summary = "Convert files (Office docs, PDF, HTML, data, archives, images) to LLM-friendly Markdown."
+  spec.description = "Markdownator converts PDF, Word, Excel, PowerPoint, EPUB, HTML, CSV, JSON, XML, " \
+                     "ZIP archives and images into clean Markdown suitable for large language models, " \
+                     "using a pluggable converter architecture."
+  spec.homepage = "https://github.com/alexrupom/markdownator"
+  spec.license = "MIT"
+  spec.required_ruby_version = ">= 2.7.0"
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = spec.homepage
+  spec.metadata["changelog_uri"] = "#{spec.homepage}/blob/main/CHANGELOG.md"
+  spec.metadata["rubygems_mfa_required"] = "true"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(__dir__) do
+    `git ls-files -z`.split("\x0").reject do |f|
+      (f == __FILE__) || f.match(%r{\A(?:(?:bin|test|spec|features)/|\.(?:git|travis|circleci)|appveyor)})
+    end
+  end
+  spec.bindir = "exe"
+  spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  # Heavy format gems are intentionally NOT runtime dependencies. Each converter
+  # requires its gem lazily and raises a helpful error if it is missing, so apps
+  # install only what they need. The gems used to exercise every format in the
+  # test suite are declared as development dependencies in the Gemfile.
+end

data/sig/markdownator.rbs

@@ -0,0 +1,50 @@
+module Markdownator
+  VERSION: String
+
+  def self.convert: (untyped source, **untyped options) -> Result
+  def self.convert_local: (String path, **untyped options) -> Result
+  def self.convert_stream: (untyped io, ?StreamInfo? stream_info, **untyped options) -> Result
+  def self.require_optional: (String gem_name, feature: String) -> void
+
+  class Error < StandardError
+  end
+
+  class UnsupportedFormatError < Error
+  end
+
+  class MissingDependencyError < Error
+  end
+
+  class FileConversionError < Error
+  end
+
+  class StreamInfo
+    attr_reader mimetype: String?
+    attr_reader extension: String?
+    attr_reader charset: String?
+    attr_reader filename: String?
+    attr_reader url: String?
+    attr_reader local_path: String?
+
+    def guessed_mimetype: () -> String?
+    def copy_with: (**untyped overrides) -> StreamInfo
+  end
+
+  class Result
+    attr_reader markdown: String
+    attr_reader title: String?
+    attr_reader metadata: Hash[untyped, untyped]
+
+    def to_s: () -> String
+    def text_content: () -> String
+  end
+
+  class Engine
+    attr_reader converters: Array[untyped]
+
+    def convert: (untyped source, **untyped options) -> Result
+    def convert_local: (String path, **untyped options) -> Result
+    def convert_url: (String url, **untyped options) -> Result
+    def convert_stream: (untyped io, ?StreamInfo? stream_info, **untyped options) -> Result
+  end
+end