html-to-markdown 2.29.0-arm64-darwin

data/spec/convert_with_tables_spec.rb

@@ -0,0 +1,194 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe HtmlToMarkdown do
+  describe '.convert_with_tables' do
+    it 'returns a hash with content, metadata, and tables keys' do
+      html = '<table><tr><td>Cell</td></tr></table>'
+      result = described_class.convert_with_tables(html)
+
+      expect(result).to be_a(Hash)
+      expect(result).to include(:content, :metadata, :tables)
+    end
+
+    context 'with a basic table with header' do
+      let(:html) do
+        <<~HTML
+          <table>
+            <thead>
+              <tr><th>Name</th><th>Age</th></tr>
+            </thead>
+            <tbody>
+              <tr><td>Alice</td><td>30</td></tr>
+            </tbody>
+          </table>
+        HTML
+      end
+
+      it 'extracts exactly one table' do
+        result = described_class.convert_with_tables(html)
+
+        expect(result[:tables].length).to eq(1)
+      end
+
+      it 'extracts cells as rows of columns' do
+        result = described_class.convert_with_tables(html)
+        table = result[:tables][0]
+
+        expect(table[:cells]).to be_an(Array)
+        expect(table[:cells].length).to eq(2)
+        expect(table[:cells][0]).to eq(%w[Name Age])
+        expect(table[:cells][1]).to eq(%w[Alice 30])
+      end
+
+      it 'provides markdown representation' do
+        result = described_class.convert_with_tables(html)
+        table = result[:tables][0]
+
+        expect(table[:markdown]).to be_a(String)
+        expect(table[:markdown]).to include('Name')
+        expect(table[:markdown]).to include('Alice')
+      end
+
+      it 'marks header rows correctly' do
+        result = described_class.convert_with_tables(html)
+        table = result[:tables][0]
+
+        expect(table[:is_header_row]).to be_an(Array)
+        expect(table[:is_header_row].length).to eq(2)
+        expect(table[:is_header_row][0]).to be true
+        expect(table[:is_header_row][1]).to be false
+      end
+
+      it 'includes converted markdown content' do
+        result = described_class.convert_with_tables(html)
+
+        expect(result[:content]).to be_a(String)
+        expect(result[:content]).not_to be_empty
+      end
+    end
+
+    context 'with empty HTML' do
+      it 'returns empty tables array' do
+        result = described_class.convert_with_tables('')
+
+        expect(result[:tables]).to eq([])
+        expect(result[:content]).to be_a(String)
+      end
+    end
+
+    context 'with HTML containing no tables' do
+      it 'returns empty tables array' do
+        html = '<p>No tables here</p>'
+        result = described_class.convert_with_tables(html)
+
+        expect(result[:tables]).to eq([])
+        expect(result[:content]).to include('No tables here')
+      end
+    end
+
+    context 'with multiple tables' do
+      let(:html) do
+        <<~HTML
+          <table>
+            <tr><th>A</th></tr>
+            <tr><td>1</td></tr>
+          </table>
+          <p>Some text between tables</p>
+          <table>
+            <tr><th>B</th><th>C</th></tr>
+            <tr><td>2</td><td>3</td></tr>
+            <tr><td>4</td><td>5</td></tr>
+          </table>
+        HTML
+      end
+
+      it 'extracts all tables' do
+        result = described_class.convert_with_tables(html)
+
+        expect(result[:tables].length).to eq(2)
+      end
+
+      it 'preserves table order' do
+        result = described_class.convert_with_tables(html)
+
+        first_table = result[:tables][0]
+        second_table = result[:tables][1]
+
+        expect(first_table[:cells][0]).to eq(['A'])
+        expect(second_table[:cells][0]).to eq(%w[B C])
+      end
+
+      it 'extracts correct row counts per table' do
+        result = described_class.convert_with_tables(html)
+
+        expect(result[:tables][0][:cells].length).to eq(2)
+        expect(result[:tables][1][:cells].length).to eq(3)
+      end
+    end
+
+    context 'with special characters in cells' do
+      let(:html) do
+        <<~HTML
+          <table>
+            <tr><th>Key</th><th>Value</th></tr>
+            <tr><td>Brackets &lt;&gt;</td><td>Ampersand &amp;</td></tr>
+            <tr><td>Quotes "double"</td><td>Quotes 'single'</td></tr>
+            <tr><td>Unicode: cafe\u0301</td><td>Emoji: test</td></tr>
+          </table>
+        HTML
+      end
+
+      it 'handles HTML entities in cells' do
+        result = described_class.convert_with_tables(html)
+        table = result[:tables][0]
+
+        expect(table[:cells][1][0]).to include('<>')
+        expect(table[:cells][1][1]).to include('&')
+      end
+
+      it 'handles quotes in cells' do
+        result = described_class.convert_with_tables(html)
+        table = result[:tables][0]
+
+        expect(table[:cells][2][0]).to include('"double"')
+        expect(table[:cells][2][1]).to include("'single'")
+      end
+
+      it 'handles unicode in cells' do
+        result = described_class.convert_with_tables(html)
+        table = result[:tables][0]
+
+        expect(table[:cells][3][0]).to be_a(String)
+      end
+    end
+
+    context 'with conversion options' do
+      it 'accepts options hash' do
+        html = '<table><tr><th>Header</th></tr><tr><td>Data</td></tr></table>'
+        result = described_class.convert_with_tables(html, { heading_style: :atx })
+
+        expect(result).to be_a(Hash)
+        expect(result[:tables].length).to eq(1)
+      end
+
+      it 'accepts nil options' do
+        html = '<table><tr><td>Data</td></tr></table>'
+        result = described_class.convert_with_tables(html, nil, nil)
+
+        expect(result).to be_a(Hash)
+        expect(result[:tables].length).to eq(1)
+      end
+    end
+
+    context 'with metadata config' do
+      it 'includes metadata when configured' do
+        html = '<html><head><title>Test</title></head><body><table><tr><td>Data</td></tr></table></body></html>'
+        result = described_class.convert_with_tables(html, nil, { extract_headers: true })
+
+        expect(result[:metadata]).to be_a(Hash).or(be_nil)
+      end
+    end
+  end
+end

data/spec/metadata_extraction_spec.rb

@@ -0,0 +1,437 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe HtmlToMarkdown do
+  describe '.convert_with_metadata' do
+    it 'returns array with markdown and metadata' do
+      html = '<html><head><title>Test</title></head><body><p>Content</p></body></html>'
+      result = described_class.convert_with_metadata(html)
+
+      expect(result).to be_an(Array)
+      expect(result.length).to eq(2)
+      expect(result[0]).to be_a(String)
+      expect(result[1]).to be_a(Hash)
+    end
+
+    context 'when extracting document metadata' do
+      it 'extracts title' do
+        html = '<html><head><title>My Page Title</title></head><body><p>Content</p></body></html>'
+        _, metadata = described_class.convert_with_metadata(html)
+
+        expect(metadata[:document][:title]).to eq('My Page Title')
+      end
+
+      it 'extracts description' do
+        html = <<~HTML
+          <html>
+            <head><meta name="description" content="Page description"></head>
+            <body><p>Content</p></body>
+          </html>
+        HTML
+        _, metadata = described_class.convert_with_metadata(html)
+
+        expect(metadata[:document][:description]).to eq('Page description')
+      end
+
+      it 'extracts keywords' do
+        html = <<~HTML
+          <html>
+            <head><meta name="keywords" content="keyword1, keyword2, keyword3"></head>
+            <body><p>Content</p></body>
+          </html>
+        HTML
+        _, metadata = described_class.convert_with_metadata(html)
+
+        expect(metadata[:document][:keywords]).to include('keyword1', 'keyword2', 'keyword3')
+      end
+
+      it 'extracts author' do
+        html = '<html><head><meta name="author" content="John Doe"></head><body><p>Content</p></body></html>'
+        _, metadata = described_class.convert_with_metadata(html)
+
+        expect(metadata[:document][:author]).to eq('John Doe')
+      end
+
+      it 'extracts base href' do
+        html = '<html><head><base href="https://example.com/"></head><body><p>Content</p></body></html>'
+        _, metadata = described_class.convert_with_metadata(html)
+
+        expect(metadata[:document][:base_href]).to eq('https://example.com/')
+      end
+
+      it 'extracts canonical URL' do
+        html = '<html><head><link rel="canonical" href="https://example.com/page"></head><body><p>Content</p></body></html>'
+        _, metadata = described_class.convert_with_metadata(html)
+
+        expect(metadata[:document][:canonical_url]).to eq('https://example.com/page')
+      end
+
+      it 'extracts language' do
+        html = '<html lang="en"><head></head><body><p>Content</p></body></html>'
+        _, metadata = described_class.convert_with_metadata(html)
+
+        expect(metadata[:document][:language]).to eq('en')
+      end
+
+      it 'extracts text direction' do
+        html = '<html dir="ltr"><head></head><body><p>Content</p></body></html>'
+        _, metadata = described_class.convert_with_metadata(html)
+
+        expect(metadata[:document][:text_direction]).to eq('ltr')
+      end
+
+      it 'extracts open graph metadata' do
+        html = <<~HTML
+          <html>
+            <head>
+              <meta property="og:title" content="OG Title">
+              <meta property="og:description" content="OG Description">
+              <meta property="og:image" content="https://example.com/image.jpg">
+            </head>
+            <body><p>Content</p></body>
+          </html>
+        HTML
+        _, metadata = described_class.convert_with_metadata(html)
+
+        expect(metadata[:document][:open_graph]).to include(
+          'title' => 'OG Title',
+          'description' => 'OG Description',
+          'image' => 'https://example.com/image.jpg'
+        )
+      end
+
+      it 'extracts twitter card metadata' do
+        html = <<~HTML
+          <html>
+            <head>
+              <meta name="twitter:card" content="summary_large_image">
+              <meta name="twitter:title" content="Twitter Title">
+            </head>
+            <body><p>Content</p></body>
+          </html>
+        HTML
+        _, metadata = described_class.convert_with_metadata(html)
+
+        expect(metadata[:document][:twitter_card]).to include(
+          'card' => 'summary_large_image',
+          'title' => 'Twitter Title'
+        )
+      end
+
+      it 'returns empty arrays and hashes for missing metadata' do
+        html = '<p>Content</p>'
+        _, metadata = described_class.convert_with_metadata(html)
+
+        expect(metadata[:document][:title]).to be_nil
+        expect(metadata[:document][:description]).to be_nil
+        expect(metadata[:document][:keywords]).to eq([])
+        expect(metadata[:document][:open_graph]).to eq({})
+        expect(metadata[:document][:twitter_card]).to eq({})
+        expect(metadata[:document][:meta_tags]).to eq({})
+      end
+    end
+
+    context 'when extracting header metadata' do
+      it 'extracts headers with hierarchy' do
+        html = <<~HTML
+          <html>
+            <body>
+              <h1>Main Title</h1>
+              <h2>Section</h2>
+              <h3>Subsection</h3>
+            </body>
+          </html>
+        HTML
+        _, metadata = described_class.convert_with_metadata(html)
+
+        expect(metadata[:headers].length).to eq(3)
+        expect(metadata[:headers][0][:level]).to eq(1)
+        expect(metadata[:headers][0][:text]).to eq('Main Title')
+        expect(metadata[:headers][1][:level]).to eq(2)
+        expect(metadata[:headers][1][:text]).to eq('Section')
+        expect(metadata[:headers][2][:level]).to eq(3)
+        expect(metadata[:headers][2][:text]).to eq('Subsection')
+      end
+
+      it 'includes header id' do
+        html = '<html><body><h1 id="main-title">Title</h1></body></html>'
+        _, metadata = described_class.convert_with_metadata(html)
+
+        expect(metadata[:headers][0][:id]).to eq('main-title')
+      end
+
+      it 'includes depth and html_offset' do
+        html = '<html><body><h1>Title</h1></body></html>'
+        _, metadata = described_class.convert_with_metadata(html)
+
+        header = metadata[:headers][0]
+        expect(header).to include(:depth, :html_offset)
+        expect(header[:depth]).to be_a(Integer)
+        expect(header[:html_offset]).to be_a(Integer)
+      end
+    end
+
+    context 'when extracting link metadata' do
+      it 'extracts links with classification' do
+        html = <<~HTML
+          <html>
+            <body>
+              <a href="#section">Anchor</a>
+              <a href="https://example.com">External</a>
+              <a href="/page">Internal</a>
+              <a href="mailto:test@example.com">Email</a>
+              <a href="tel:+1234567890">Phone</a>
+            </body>
+          </html>
+        HTML
+        _, metadata = described_class.convert_with_metadata(html)
+
+        links = metadata[:links]
+        expect(links.length).to eq(5)
+
+        expect(links[0][:link_type]).to eq('anchor')
+        expect(links[1][:link_type]).to eq('external')
+        expect(links[2][:link_type]).to eq('internal')
+        expect(links[3][:link_type]).to eq('email')
+        expect(links[4][:link_type]).to eq('phone')
+      end
+
+      it 'includes link text and href' do
+        html = '<html><body><a href="https://example.com">Click here</a></body></html>'
+        _, metadata = described_class.convert_with_metadata(html)
+
+        link = metadata[:links][0]
+        expect(link[:href]).to eq('https://example.com')
+        expect(link[:text]).to eq('Click here')
+      end
+
+      it 'includes link title attribute' do
+        html = '<html><body><a href="https://example.com" title="Example Site">Link</a></body></html>'
+        _, metadata = described_class.convert_with_metadata(html)
+
+        link = metadata[:links][0]
+        expect(link[:title]).to eq('Example Site')
+      end
+
+      it 'includes link rel attributes' do
+        html = '<html><body><a href="https://example.com" rel="nofollow external">Link</a></body></html>'
+        _, metadata = described_class.convert_with_metadata(html)
+
+        link = metadata[:links][0]
+        expect(link[:rel]).to include('nofollow', 'external')
+      end
+
+      it 'includes link attributes' do
+        html = '<html><body><a href="https://example.com" data-custom="value">Link</a></body></html>'
+        _, metadata = described_class.convert_with_metadata(html)
+
+        link = metadata[:links][0]
+        expect(link[:attributes]).to include('data-custom' => 'value')
+      end
+    end
+
+    context 'when extracting image metadata' do
+      it 'extracts images with source type' do
+        html = <<~HTML
+          <html>
+            <body>
+              <img src="https://example.com/image.jpg" alt="External">
+              <img src="/images/local.jpg" alt="Relative">
+              <img src="data:image/png;base64,..." alt="Data URI">
+            </body>
+          </html>
+        HTML
+        _, metadata = described_class.convert_with_metadata(html)
+
+        images = metadata[:images]
+        expect(images.length).to eq(3)
+
+        expect(images[0][:image_type]).to eq('external')
+        expect(images[1][:image_type]).to eq('relative')
+        expect(images[2][:image_type]).to eq('data_uri')
+      end
+
+      it 'includes image alt and title' do
+        html = '<html><body><img src="image.jpg" alt="Alt text" title="Image title"></body></html>'
+        _, metadata = described_class.convert_with_metadata(html)
+
+        image = metadata[:images][0]
+        expect(image[:alt]).to eq('Alt text')
+        expect(image[:title]).to eq('Image title')
+      end
+
+      it 'includes image dimensions' do
+        html = '<html><body><img src="image.jpg" width="800" height="600"></body></html>'
+        _, metadata = described_class.convert_with_metadata(html)
+
+        image = metadata[:images][0]
+        expect(image[:dimensions]).to be_an(Array)
+        expect(image[:dimensions].length).to eq(2)
+      end
+
+      it 'handles missing image attributes' do
+        html = '<html><body><img src="image.jpg"></body></html>'
+        _, metadata = described_class.convert_with_metadata(html)
+
+        image = metadata[:images][0]
+        expect(image[:alt]).to be_nil
+        expect(image[:title]).to be_nil
+      end
+    end
+
+    context 'with metadata configuration flags' do
+      it 'respects extract_headers flag' do
+        html = '<html><body><h1>Title</h1><p>Content</p></body></html>'
+        config = { extract_headers: false }
+        _, metadata = described_class.convert_with_metadata(html, nil, config)
+
+        expect(metadata[:headers]).to eq([])
+      end
+
+      it 'respects extract_links flag' do
+        html = '<html><body><a href="https://example.com">Link</a></body></html>'
+        config = { extract_links: false }
+        _, metadata = described_class.convert_with_metadata(html, nil, config)
+
+        expect(metadata[:links]).to eq([])
+      end
+
+      it 'respects extract_images flag' do
+        html = '<html><body><img src="image.jpg" alt="test"></body></html>'
+        config = { extract_images: false }
+        _, metadata = described_class.convert_with_metadata(html, nil, config)
+
+        expect(metadata[:images]).to eq([])
+      end
+
+      it 'respects extract_structured_data flag' do
+        html = '<html><body><script type="application/ld+json">{"@type":"Article"}</script></body></html>'
+        config = { extract_structured_data: false }
+        _, metadata = described_class.convert_with_metadata(html, nil, config)
+
+        expect(metadata[:structured_data]).to eq([])
+      end
+    end
+
+    context 'with conversion options and metadata config' do
+      it 'accepts both conversion options and metadata config' do
+        html = '<html><head><title>Test</title></head><body><h1>Heading</h1></body></html>'
+        conv_opts = { heading_style: :atx_closed }
+        meta_opts = { extract_headers: true }
+
+        markdown, metadata = described_class.convert_with_metadata(html, conv_opts, meta_opts)
+
+        expect(markdown).to include('# Heading #')
+        expect(metadata[:headers].length).to eq(1)
+      end
+
+      it 'works with nil options' do
+        html = '<html><head><title>Test</title></head><body><p>Content</p></body></html>'
+        result = described_class.convert_with_metadata(html, nil, nil)
+
+        expect(result).to be_an(Array)
+        expect(result.length).to eq(2)
+      end
+    end
+
+    context 'when extracting structured data' do
+      it 'extracts JSON-LD blocks' do
+        html = <<~HTML
+          <html>
+            <head>
+              <script type="application/ld+json">
+                {"@context":"https://schema.org","@type":"Article","headline":"Test"}
+              </script>
+            </head>
+            <body><p>Content</p></body>
+          </html>
+        HTML
+        _, metadata = described_class.convert_with_metadata(html)
+
+        expect(metadata[:structured_data]).to be_an(Array)
+      end
+    end
+
+    context 'with edge cases' do
+      it 'handles empty HTML' do
+        html = ''
+        markdown, metadata = described_class.convert_with_metadata(html)
+
+        expect(markdown).to be_a(String)
+        expect(metadata).to be_a(Hash)
+      end
+
+      it 'handles malformed HTML' do
+        html = '<html><head><title>Unclosed'
+        markdown, metadata = described_class.convert_with_metadata(html)
+
+        expect(markdown).to be_a(String)
+        expect(metadata).to be_a(Hash)
+      end
+
+      it 'handles special characters in metadata' do
+        html = '<html><head><title>Title with "quotes" & <brackets></title></head><body><p>Content</p></body></html>'
+        _, metadata = described_class.convert_with_metadata(html)
+
+        expect(metadata[:document][:title]).to be_a(String)
+      end
+
+      it 'handles whitespace in metadata' do
+        html = '<html><head><title>  Title with   spaces  </title></head><body><p>Content</p></body></html>'
+        _, metadata = described_class.convert_with_metadata(html)
+
+        expect(metadata[:document][:title]).to match(/Title.*spaces/)
+      end
+
+      it 'handles multiple values for same metadata key' do
+        html = <<~HTML
+          <html>
+            <head>
+              <meta name="author" content="Author 1">
+              <meta name="author" content="Author 2">
+            </head>
+            <body><p>Content</p></body>
+          </html>
+        HTML
+        _, metadata = described_class.convert_with_metadata(html)
+
+        expect(metadata[:document][:author]).to be_a(String)
+      end
+    end
+
+    context 'when returning value structure' do
+      it 'returns proper metadata hash structure' do
+        html = <<~HTML
+          <html>
+            <head><title>Test</title><base href="https://example.com"></head>
+            <body><h1>H1</h1><a href="link">Link</a><img src="img.jpg"></body>
+          </html>
+        HTML
+        _, metadata = described_class.convert_with_metadata(html)
+
+        expect(metadata).to include(
+          :document,
+          :headers,
+          :links,
+          :images,
+          :structured_data
+        )
+
+        expect(metadata[:document]).to include(
+          :title,
+          :description,
+          :keywords,
+          :author,
+          :canonical_url,
+          :base_href,
+          :language,
+          :text_direction,
+          :open_graph,
+          :twitter_card,
+          :meta_tags
+        )
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'html_to_markdown'
+
+RSpec.configure do |config|
+  config.expect_with :rspec do |c|
+    c.syntax = :expect
+  end
+end