html-to-markdown 2.29.0-arm64-darwin

data/sig/html_to_markdown/cli.rbs

@@ -0,0 +1,24 @@
+module HtmlToMarkdown
+  module CLI
+    # Module method (module_function creates both module and instance methods)
+    #
+    # Run the CLI with the given arguments
+    #
+    # @param argv Command-line arguments (defaults to ARGV)
+    # @param stdout Output stream for standard output
+    # @param stderr Output stream for standard error
+    # @return Exit code (0 for success, non-zero for failure)
+    def self.run: (
+      ?Array[String] argv,
+      ?stdout: IO,
+      ?stderr: IO
+    ) -> Integer
+
+    # Instance method version (created by module_function)
+    def run: (
+      ?Array[String] argv,
+      ?stdout: IO,
+      ?stderr: IO
+    ) -> Integer
+  end
+end

data/sig/html_to_markdown/cli_proxy.rbs

@@ -0,0 +1,48 @@
+module HtmlToMarkdown
+  module CLIProxy
+    # Base error class
+    class Error < StandardError
+    end
+
+    # Error when CLI binary is not found
+    class MissingBinaryError < Error
+    end
+
+    # Error when CLI execution fails
+    class CLIExecutionError < Error
+      attr_reader stderr: String
+      attr_reader status: Integer?
+
+      def initialize: (String message, stderr: String, status: Integer?) -> void
+    end
+
+    # Module methods (module_function creates both module and instance methods)
+
+    # Execute CLI with given arguments
+    def self.call: (Array[String] argv) -> String
+
+    # Find the CLI binary in search paths
+    def self.find_cli_binary: () -> Pathname
+
+    # Get root path of the gem
+    def self.root_path: () -> Pathname
+
+    # Get lib path of the gem
+    def self.lib_path: () -> Pathname
+
+    # Get search paths for CLI binary
+    def self.search_paths: (String binary_name) -> Array[Pathname]
+
+    # Get error message for missing binary
+    def self.missing_binary_message: () -> String
+
+    # Instance method versions (created by module_function)
+
+    def call: (Array[String] argv) -> String
+    def find_cli_binary: () -> Pathname
+    def root_path: () -> Pathname
+    def lib_path: () -> Pathname
+    def search_paths: (String binary_name) -> Array[Pathname]
+    def missing_binary_message: () -> String
+  end
+end

data/sig/html_to_markdown.rbs

@@ -0,0 +1,498 @@
+# Type definitions for HtmlToMarkdown Ruby gem
+module HtmlToMarkdown
+  VERSION: String
+
+  # Opaque handle for reusable conversion options
+  class Options
+  end
+
+  # Visitor context information passed to visitor callbacks
+  class NodeContext
+    attr_reader node_type: Symbol
+    attr_reader tag_name: String
+    attr_reader attributes: Hash[String, String]
+    attr_reader depth: Integer
+    attr_reader index_in_parent: Integer
+    attr_reader parent_tag: String | nil
+    attr_reader is_inline: bool
+  end
+
+  # Result of a visitor callback
+  type visitor_result = {
+    type: :continue,
+  } | {
+    type: :custom,
+    output: String,
+  } | {
+    type: :skip,
+  } | {
+    type: :preserve_html,
+  } | {
+    type: :error,
+    message: String,
+  }
+
+  type heading_style = :underlined | :atx | :atx_closed
+  type list_indent_type = :spaces | :tabs
+  type highlight_style = :double_equal | :html | :bold | :none
+  type whitespace_mode = :normalized | :strict
+  type newline_style = :spaces | :backslash
+  type code_block_style = :indented | :backticks | :tildes
+  type output_format = :markdown | :djot
+  type preprocessing_preset = :minimal | :standard | :aggressive
+
+  type preprocessing_options = {
+    enabled?: bool,
+    preset?: preprocessing_preset,
+    remove_navigation?: bool,
+    remove_forms?: bool
+  }
+
+  type conversion_options = {
+    heading_style?: heading_style,
+    list_indent_type?: list_indent_type,
+    list_indent_width?: Integer,
+    bullets?: String,
+    strong_em_symbol?: String,
+    escape_asterisks?: bool,
+    escape_underscores?: bool,
+    escape_misc?: bool,
+    escape_ascii?: bool,
+    code_language?: String,
+    autolinks?: bool,
+    default_title?: bool,
+    br_in_tables?: bool,
+    hocr_spatial_tables?: bool,
+    highlight_style?: highlight_style,
+    extract_metadata?: bool,
+    whitespace_mode?: whitespace_mode,
+    strip_newlines?: bool,
+    wrap?: bool,
+    wrap_width?: Integer,
+    convert_as_inline?: bool,
+    sub_symbol?: String,
+    sup_symbol?: String,
+    newline_style?: newline_style,
+    code_block_style?: code_block_style,
+    keep_inline_images_in?: Array[String],
+    preprocessing?: preprocessing_options,
+    encoding?: String,
+    debug?: bool,
+    strip_tags?: Array[String],
+    preserve_tags?: Array[String],
+    output_format?: output_format,
+    skip_images?: bool
+  }
+
+  type inline_image_config = {
+    max_decoded_size_bytes?: Integer,
+    filename_prefix?: String?,
+    capture_svg?: bool,
+    infer_dimensions?: bool
+  }
+
+  type inline_image_format = "png" | "jpeg" | "gif" | "bmp" | "webp" | "svg" | String
+
+  type inline_image_source = "img_data_uri" | "svg_element"
+
+  type inline_image = {
+    data: String,
+    format: inline_image_format,
+    filename: String?,
+    description: String?,
+    dimensions: [Integer, Integer]?,
+    source: inline_image_source,
+    attributes: Hash[String, String]
+  }
+
+  type inline_image_warning = {
+    index: Integer,
+    message: String
+  }
+
+  type html_extraction = {
+    markdown: String,
+    inline_images: Array[inline_image],
+    warnings: Array[inline_image_warning]
+  }
+
+  type metadata_config = {
+    extract_document?: bool,
+    extract_headers?: bool,
+    extract_links?: bool,
+    extract_images?: bool,
+    extract_structured_data?: bool,
+    max_structured_data_size?: Integer
+  }
+
+  type text_direction = "ltr" | "rtl" | "auto" | nil
+
+  type document_metadata = {
+    title: String?,
+    description: String?,
+    keywords: Array[String],
+    author: String?,
+    canonical_url: String?,
+    base_href: String?,
+    language: String?,
+    text_direction: text_direction,
+    open_graph: Hash[String, String],
+    twitter_card: Hash[String, String],
+    meta_tags: Hash[String, String]
+  }
+
+  type header_metadata = {
+    level: Integer,
+    text: String,
+    id: String?,
+    depth: Integer,
+    html_offset: Integer
+  }
+
+  type link_type = "anchor" | "internal" | "external" | "email" | "phone" | "other"
+
+  type link_metadata = {
+    href: String,
+    text: String,
+    title: String?,
+    link_type: link_type,
+    rel: Array[String],
+    attributes: Hash[String, String]
+  }
+
+  type image_type = "data_uri" | "inline_svg" | "external" | "relative"
+
+  type image_metadata = {
+    src: String,
+    alt: String?,
+    title: String?,
+    dimensions: [Integer, Integer]?,
+    image_type: image_type,
+    attributes: Hash[String, String]
+  }
+
+  type structured_data = {
+    data_type: "json_ld" | "microdata" | "rdfa",
+    raw_json: String,
+    schema_type: String?
+  }
+
+  type extended_metadata = {
+    document: document_metadata,
+    headers: Array[header_metadata],
+    links: Array[link_metadata],
+    images: Array[image_metadata],
+    structured_data: Array[structured_data]
+  }
+
+  type table_data = {
+    cells: Array[Array[String]],
+    markdown: String,
+    is_header_row: Array[bool]
+  }
+
+  type table_extraction_result = {
+    content: String,
+    metadata: extended_metadata?,
+    tables: Array[table_data]
+  }
+
+  # Native methods (implemented in Rust via Magnus/rb-sys)
+  # These are aliased from the Rust extension and available as both module and instance methods
+  private
+
+  def self.native_convert: (String html, conversion_options? options) -> String
+  def self.native_options: (conversion_options? options_hash) -> Options
+  def self.native_convert_with_options: (String html, Options options_handle) -> String
+  def self.native_convert_with_inline_images_handle: (
+    String html,
+    Options options_handle,
+    inline_image_config? image_config
+  ) -> html_extraction
+  def self.native_convert_with_inline_images: (
+    String html,
+    conversion_options? options,
+    inline_image_config? image_config
+  ) -> html_extraction
+  def self.native_convert_with_metadata_handle: (
+    String html,
+    Options options_handle,
+    metadata_config? metadata_config
+  ) -> [String, extended_metadata]
+  def self.native_convert_with_metadata: (
+    String html,
+    conversion_options? options,
+    metadata_config? metadata_config
+  ) -> [String, extended_metadata]
+  def self.native_convert_with_visitor: (
+    String html,
+    conversion_options? options,
+    visitor? visitor
+  ) -> String
+  def self.native_convert_with_tables: (
+    String html,
+    conversion_options? options,
+    metadata_config? metadata_config
+  ) -> table_extraction_result
+
+  def native_convert: (String html, conversion_options? options) -> String
+  def native_options: (conversion_options? options_hash) -> Options
+  def native_convert_with_options: (String html, Options options_handle) -> String
+  def native_convert_with_inline_images_handle: (
+    String html,
+    Options options_handle,
+    inline_image_config? image_config
+  ) -> html_extraction
+  def native_convert_with_inline_images: (
+    String html,
+    conversion_options? options,
+    inline_image_config? image_config
+  ) -> html_extraction
+  def native_convert_with_metadata_handle: (
+    String html,
+    Options options_handle,
+    metadata_config? metadata_config
+  ) -> [String, extended_metadata]
+  def native_convert_with_metadata: (
+    String html,
+    conversion_options? options,
+    metadata_config? metadata_config
+  ) -> [String, extended_metadata]
+  def native_convert_with_visitor: (
+    String html,
+    conversion_options? options,
+    visitor? visitor
+  ) -> String
+  def native_convert_with_tables: (
+    String html,
+    conversion_options? options,
+    metadata_config? metadata_config
+  ) -> table_extraction_result
+
+  # Visitor interface for customizing conversion behavior
+  type visitor = Object
+
+  public
+
+  # Convert HTML to Markdown with optional configuration and visitor
+  #
+  # The optional visitor parameter allows customization of conversion behavior for specific elements.
+  # When both options and visitor are provided, the visitor can override default conversions.
+  #
+  # Args:
+  #   html: HTML string to convert
+  #   options: Optional conversion configuration
+  #   visitor: Optional visitor object for customizing conversion
+  #
+  # Returns:
+  #   markdown: String - Converted markdown output
+  #
+  # Example:
+  #   markdown = HtmlToMarkdown.convert(html, { wrap: true }, my_visitor)
+  def self.convert: (String html, ?conversion_options options, ?visitor visitor) -> String
+
+  # Create a reusable options handle for performance
+  def self.options: (?conversion_options options_hash) -> Options
+
+  # Convert HTML using a pre-built options handle
+  def self.convert_with_options: (String html, Options options_handle) -> String
+  def self.convert_with_inline_images_handle: (
+    String html,
+    Options options_handle,
+    ?inline_image_config image_config
+  ) -> html_extraction
+
+  # Convert HTML with inline image extraction
+  #
+  # Optionally accepts a visitor for customizing conversion behavior.
+  #
+  # Args:
+  #   html: HTML string to convert
+  #   options: Optional conversion configuration
+  #   image_config: Optional inline image extraction configuration
+  #   visitor: Optional visitor object for customizing conversion
+  #
+  # Returns:
+  #   html_extraction: Hash containing markdown, inline_images array, and warnings array
+  #
+  # Example:
+  #   result = HtmlToMarkdown.convert_with_inline_images(html, { wrap: true }, image_config, my_visitor)
+  def self.convert_with_inline_images: (
+    String html,
+    ?conversion_options options,
+    ?inline_image_config image_config,
+    ?visitor visitor
+  ) -> html_extraction
+
+  # Convert HTML to Markdown with a custom visitor (deprecated)
+  #
+  # DEPRECATED: Use convert() with the optional visitor parameter instead.
+  # This method is maintained for backward compatibility.
+  #
+  # All convert functions now accept optional visitors:
+  #   - convert(html, options, visitor)
+  #   - convert_with_inline_images(html, options, image_config, visitor)
+  #   - convert_with_metadata(html, options, metadata_config, visitor)
+  #
+  # The visitor object can implement any of the following methods:
+  #   - visit_element_start(ctx) -> visitor_result
+  #   - visit_element_end(ctx, output) -> visitor_result
+  #   - visit_text(ctx, text) -> visitor_result
+  #   - visit_link(ctx, href, text, title) -> visitor_result
+  #   - visit_image(ctx, src, alt, title) -> visitor_result
+  #   - visit_heading(ctx, level, text, id) -> visitor_result
+  #   - visit_code_block(ctx, lang, code) -> visitor_result
+  #   - visit_code_inline(ctx, code) -> visitor_result
+  #   - visit_list_item(ctx, ordered, marker, text) -> visitor_result
+  #   - visit_list_start(ctx, ordered) -> visitor_result
+  #   - visit_list_end(ctx, ordered, output) -> visitor_result
+  #   - visit_table_start(ctx) -> visitor_result
+  #   - visit_table_row(ctx, cells, is_header) -> visitor_result
+  #   - visit_table_end(ctx, output) -> visitor_result
+  #   - visit_blockquote(ctx, content, depth) -> visitor_result
+  #   - visit_strong(ctx, text) -> visitor_result
+  #   - visit_emphasis(ctx, text) -> visitor_result
+  #   - visit_strikethrough(ctx, text) -> visitor_result
+  #   - visit_underline(ctx, text) -> visitor_result
+  #   - visit_subscript(ctx, text) -> visitor_result
+  #   - visit_superscript(ctx, text) -> visitor_result
+  #   - visit_mark(ctx, text) -> visitor_result
+  #   - visit_line_break(ctx) -> visitor_result
+  #   - visit_horizontal_rule(ctx) -> visitor_result
+  #   - visit_custom_element(ctx, tag_name, html) -> visitor_result
+  #   - visit_definition_list_start(ctx) -> visitor_result
+  #   - visit_definition_term(ctx, text) -> visitor_result
+  #   - visit_definition_description(ctx, text) -> visitor_result
+  #   - visit_definition_list_end(ctx, output) -> visitor_result
+  #   - visit_form(ctx, action, method) -> visitor_result
+  #   - visit_input(ctx, input_type, name, value) -> visitor_result
+  #   - visit_button(ctx, text) -> visitor_result
+  #   - visit_audio(ctx, src) -> visitor_result
+  #   - visit_video(ctx, src) -> visitor_result
+  #   - visit_iframe(ctx, src) -> visitor_result
+  #   - visit_details(ctx, open) -> visitor_result
+  #   - visit_summary(ctx, text) -> visitor_result
+  #   - visit_figure_start(ctx) -> visitor_result
+  #   - visit_figcaption(ctx, text) -> visitor_result
+  #   - visit_figure_end(ctx, output) -> visitor_result
+  #
+  # Each method should return a Hash with at least :type key:
+  #   { type: :continue } - Continue with default behavior
+  #   { type: :custom, output: "..." } - Replace with custom markdown
+  #   { type: :skip } - Skip this element entirely
+  #   { type: :preserve_html } - Keep original HTML
+  #   { type: :error, message: "..." } - Stop conversion with error
+  #
+  # Args:
+  #   html: HTML string to convert
+  #   options: Optional conversion configuration
+  #   visitor: Visitor object that responds to visitor callback methods
+  #
+  # Returns:
+  #   markdown: String - Converted markdown output
+  #
+  # Example:
+  #   class MyVisitor
+  #     def visit_link(ctx, href, text, title = nil)
+  #       { type: :custom, output: "[#{text}](#{href})" }
+  #     end
+  #   end
+  #
+  #   HtmlToMarkdown.convert_with_visitor(html, visitor: MyVisitor.new)
+  def self.convert_with_visitor: (String html, ?conversion_options options, visitor: visitor) -> String
+
+  # Convert HTML to Markdown with metadata extraction
+  #
+  # Extracts comprehensive metadata (headers, links, images, structured data) during conversion.
+  # Optionally accepts a visitor for customizing conversion behavior.
+  #
+  # Args:
+  #   html: HTML string to convert
+  #   options: Optional conversion configuration
+  #   metadata_config: Optional metadata extraction configuration
+  #   visitor: Optional visitor object for customizing conversion
+  #
+  # Returns:
+  #   Array containing:
+  #   - [0] markdown: String - Converted markdown output
+  #   - [1] metadata: Hash - Extracted metadata with document, headers, links, images, structured_data
+  #
+  # The metadata hash contains:
+  #   - document: Document-level metadata (title, description, lang, etc.)
+  #   - headers: List of header elements with hierarchy
+  #   - links: List of extracted hyperlinks with classification
+  #   - images: List of extracted images with metadata
+  #   - structured_data: List of JSON-LD, Microdata, or RDFa blocks
+  #
+  # Example:
+  #   html = '<html lang="en"><head><title>Test</title></head><body><h1>Hello</h1></body></html>'
+  #   markdown, metadata = HtmlToMarkdown.convert_with_metadata(html)
+  #   puts "Title: #{metadata['document']['title']}"
+  #   puts "Headers: #{metadata['headers'].length}"
+  #
+  # Example with visitor:
+  #   markdown, metadata = HtmlToMarkdown.convert_with_metadata(html, options, metadata_config, my_visitor)
+  def self.convert_with_metadata: (
+    String html,
+    ?conversion_options options,
+    ?metadata_config metadata_config,
+    ?visitor visitor
+  ) -> [String, extended_metadata]
+  def self.convert_with_metadata_handle: (
+    String html,
+    Options options_handle,
+    ?metadata_config metadata_config
+  ) -> [String, extended_metadata]
+
+  # Convert HTML and extract tables as structured data
+  #
+  # Args:
+  #   html: HTML string to convert
+  #   options: Optional conversion configuration
+  #   metadata_config: Optional metadata extraction configuration
+  #
+  # Returns:
+  #   table_extraction_result: Hash containing content, metadata, and tables array
+  #
+  # Example:
+  #   result = HtmlToMarkdown.convert_with_tables(html)
+  #   puts result[:tables].length
+  def self.convert_with_tables: (
+    String html,
+    ?conversion_options options,
+    ?metadata_config metadata_config
+  ) -> table_extraction_result
+
+  # Instance method versions (created by module_function)
+  def convert: (String html, ?conversion_options options, ?visitor visitor) -> String
+  def options: (?conversion_options options_hash) -> Options
+  def convert_with_options: (String html, Options options_handle) -> String
+  def convert_with_inline_images_handle: (
+    String html,
+    Options options_handle,
+    ?inline_image_config image_config
+  ) -> html_extraction
+  def convert_with_inline_images: (
+    String html,
+    ?conversion_options options,
+    ?inline_image_config image_config,
+    ?visitor visitor
+  ) -> html_extraction
+  def convert_with_visitor: (String html, ?conversion_options options, visitor: visitor) -> String
+  def convert_with_metadata: (
+    String html,
+    ?conversion_options options,
+    ?metadata_config metadata_config,
+    ?visitor visitor
+  ) -> [String, extended_metadata]
+  def convert_with_metadata_handle: (
+    String html,
+    Options options_handle,
+    ?metadata_config metadata_config
+  ) -> [String, extended_metadata]
+  def convert_with_tables: (
+    String html,
+    ?conversion_options options,
+    ?metadata_config metadata_config
+  ) -> table_extraction_result
+end

data/sig/open3.rbs

@@ -0,0 +1,12 @@
+# Type signature for Open3 standard library
+module Open3
+  # Execute command and capture stdout, stderr, and status
+  #
+  # @param cmd Command to execute
+  # @param args Command arguments
+  # @return Array containing stdout (String), stderr (String), and status (Process::Status)
+  def self.capture3: (
+    String cmd,
+    *String args
+  ) -> [String, String, Process::Status]
+end

data/spec/cli_proxy_spec.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require 'html_to_markdown/cli_proxy'
+require 'html_to_markdown/cli'
+require 'stringio'
+
+RSpec.describe HtmlToMarkdown::CLIProxy do
+  describe '.call' do
+    it 'executes the CLI binary' do
+      begin
+        binary = described_class.find_cli_binary
+      rescue HtmlToMarkdown::CLIProxy::MissingBinaryError
+        skip 'CLI binary not built'
+      end
+
+      expect(binary).to be_file
+
+      output = described_class.call(['--version'])
+      expect(output).to include(HtmlToMarkdown::VERSION)
+    end
+  end
+
+  describe HtmlToMarkdown::CLI do
+    it 'writes CLI output to stdout' do
+      begin
+        HtmlToMarkdown::CLIProxy.find_cli_binary
+      rescue HtmlToMarkdown::CLIProxy::MissingBinaryError
+        skip 'CLI binary not built'
+      end
+
+      stdout = StringIO.new
+      stderr = StringIO.new
+
+      exit_code = described_class.run(['--version'], stdout: stdout, stderr: stderr)
+
+      expect(exit_code).to eq(0)
+      expect(stdout.string).to include(HtmlToMarkdown::VERSION)
+      expect(stderr.string).to be_empty
+    end
+  end
+end

data/spec/convert_spec.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe HtmlToMarkdown do
+  describe '.convert' do
+    it 'converts simple headings' do
+      expect(described_class.convert('<h1>Hello</h1>')).to eq("# Hello\n")
+    end
+
+    it 'accepts options hash' do
+      result = described_class.convert(
+        '<h1>Hello</h1>',
+        heading_style: :atx_closed,
+        default_title: true
+      )
+      expect(result).to include('Hello')
+    end
+  end
+
+  describe '.convert_with_inline_images' do
+    it 'returns inline images metadata' do
+      html = '<p><img src="data:image/png;base64,ZmFrZQ==" alt="fake"></p>'
+      extraction = described_class.convert_with_inline_images(html)
+      expect(extraction).to include(:markdown, :inline_images, :warnings)
+      expect(extraction[:inline_images].first[:description]).to eq('fake')
+    end
+  end
+
+  describe '.options' do
+    it 'returns a reusable options handle' do
+      handle = described_class.options(heading_style: :atx_closed)
+      expect(handle).to be_a(HtmlToMarkdown::Options)
+      result = described_class.convert_with_options('<h1>Hello</h1>', handle)
+      expect(result).to include('# Hello #')
+    end
+  end
+
+  describe 'panic handling' do
+    context 'when a Rust panic would occur' do
+      it 'catches panics in convert method' do
+        malformed_html = "#{'<' * 100_000}div#{'>' * 100_000}"
+
+        begin
+          result = described_class.convert(malformed_html)
+          expect(result).to be_a(String)
+        rescue RuntimeError => e
+          expect(e.message).to match(/html-to-markdown panic during conversion/)
+        end
+      end
+
+      it 'catches panics in convert_with_options method' do
+        malformed_html = "#{'<' * 100_000}div#{'>' * 100_000}"
+        handle = described_class.options(heading_style: :atx)
+
+        begin
+          result = described_class.convert_with_options(malformed_html, handle)
+          expect(result).to be_a(String)
+        rescue RuntimeError => e
+          expect(e.message).to match(/html-to-markdown panic during conversion/)
+        end
+      end
+
+      it 'catches panics in convert_with_inline_images method' do
+        malformed_html = "#{'<' * 100_000}div#{'>' * 100_000}"
+
+        begin
+          result = described_class.convert_with_inline_images(malformed_html)
+          expect(result).to be_a(Hash)
+          expect(result).to include(:markdown, :inline_images, :warnings)
+        rescue RuntimeError => e
+          expect(e.message).to match(/html-to-markdown panic during conversion/)
+        end
+      end
+    end
+  end
+end