html2text 0.3.1 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7d1902161f7964cd95630662cfe326001842de6ae9cfc791216b2a5c2d6fc763
-  data.tar.gz: 4940f60ec3ea46df4a3117aa7c053d1b30b935c3114bddb81e8d6e81e29fccbb
+  metadata.gz: 1a1e6fcee8973815b391ca28bd156a878e36594e0bfbece01f7ff8c67f6cad6f
+  data.tar.gz: f515c6edb112deeae87f59b717aa4a2aed5d37b4b42db151fc49359f2b98d0b9
 SHA512:
-  metadata.gz: cd7354466697fc737c336a6abf38e6c70a9480e7d609de135348d4f8b6ab765832929ccd5687fc88209a75d2f82932421a8a59fe8c0754121680d60a0a5f3496
-  data.tar.gz: 39337ef32bc46adf101c06fc33cc98d8960bf31ce1816fde93dfb1a8a6aa75381b28114a8ff0ad363c5335f2bd61df9766ece0ef8c2b325c28d261e9a3552f7b
+  metadata.gz: 3972abc5f287146a5f43969fe3a37b83333559618b198ccc6ca3b105178c7cb37a03c9b44317428c8323c9b86e7e7a22a78276d9ddb7cccaab1819105e469a18
+  data.tar.gz: dcc2f39ebc4d772b47762cacb5de22ab16f386fbc4952c7fcf72ae66938fffcacad1fad7062714fdebc21a6b74c802224d02b796c099d2a00d1b3d13729f5d6d

data/CHANGELOG.md

@@ -4,7 +4,30 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [Unreleased]
+## [Unreleased](https://github.com/soundasleep/html2text_ruby/compare/v0.5.0...master)
+
+## [0.5.0](https://github.com/soundasleep/html2text_ruby/compare/v0.4.0...v0.5.0) - 2026-07-08
+### Added
+- Add a `wrap_links:` option to `Html2Text.convert` and `Html2Text.new` (default `true`); when `false`, hyperlinks render as `text: href` instead of markdown-style `[text](href)`. Only affects `<a href>` links — images and named anchors keep their `[bracketed]` format ([#40](https://github.com/soundasleep/html2text_ruby/pull/40))
+- Specs covering subclass overrides of `prefix_whitespace`, `suffix_whitespace`, `iterate_over`, `wrap_link` and `image_text`, plus unit specs for `fix_newlines` and `replace_entities` ([#41](https://github.com/soundasleep/html2text_ruby/pull/41))
+
+### Changed
+- Require Ruby 3.3+, since all earlier Rubies are EOL; CI now tests Ruby 3.3, 3.4 and 4.0 ([#42](https://github.com/soundasleep/html2text_ruby/pull/42))
+- Reduce object allocations in `iterate_over` for ~1.6x faster conversion of large documents ([#41](https://github.com/soundasleep/html2text_ruby/pull/41))
+- Update development dependencies (rubocop 1.88, nokogiri 1.19 in the lockfile), removing the transitive `rexml` dependency ([#42](https://github.com/soundasleep/html2text_ruby/pull/42))
+
+## [0.4.0](https://github.com/soundasleep/html2text_ruby/compare/0.3.1...v0.4.0) - 2024-06-08
+### Added
+- Switch from Travis to Github Actions for Build and Test
+- Add rubocop for linting and cleanup existing violations ([#36](https://github.com/soundasleep/html2text_ruby/pull/36))
+
+### Changed
+- Add support for Ruby 3.x, removed support for Ruby < 3.0 since it is EOL
+- Allow subclassing of `Html2Text` to override the default behaviour ([#30](https://github.com/soundasleep/html2text_ruby/pull/30))
+
+### Fixed
+- Loosen nokogiri dependency to allow for nokogiri < 2.0 ([#17](https://github.com/soundasleep/html2text_ruby/pull/17))
+- Fix `NoMethodError` when parsing nodes with no name ([#15](https://github.com/soundasleep/html2text_ruby/pull/15))
 
 ## [0.3.1] - 2019-06-12
 ### Security

data/README.md

@@ -1,5 +1,5 @@
-html2text [![Build Status](https://travis-ci.org/soundasleep/html2text_ruby.svg?branch=master)](https://travis-ci.org/soundasleep/html2text_ruby) [![Total Downloads](https://ruby-gem-downloads-badge.herokuapp.com/html2text?type=total&metric=true)](https://rubygems.org/gems/html2text/)
-==============
+html2text ![Build](https://github.com/soundasleep/html2text_ruby/actions/workflows/build.yml/badge.svg) [![Gem Version](https://badge.fury.io/rb/html2text.svg)](https://badge.fury.io/rb/html2text)
+---
 
 `html2text` is a very simple gem that uses DOM methods to convert HTML into a format similar to what would be
 rendered by a browser - perfect for places where you need a quick text representation. For example:
@@ -20,7 +20,7 @@ rendered by a browser - perfect for places where you need a quick text represent
   <div>Another div</div>
   <div>A div<div>within a div</div></div>
 
-  <a href="http://foo.com">A link</a>
+  <a href="https://foo.com">A link</a>
 
 </body>
 </html>
@@ -40,7 +40,7 @@ Another div
 A div
 within a div
 
-[A link](http://foo.com)
+[A link](https://foo.com)
 ```
 
 See the [original blog post](http://journals.jevon.org/users/jevon-phd/entry/19818) or the related [StackOverflow answer](http://stackoverflow.com/a/2564472/39531).
@@ -63,7 +63,7 @@ text = Html2Text.convert(html)
 
 ## Tests
 
-See all of the test cases defined in [spec/examples/](spec/examples/). These can be run with `bundle && rspec`.
+See all of the test cases defined in [spec/examples/](spec/examples/). These can be run with `bundle exec rake`.
 
 ## License
 

data/lib/html2text/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class Html2Text
-  VERSION = "0.3.1"
+  VERSION = '0.5.0'
 end

data/lib/html2text.rb

@@ -1,23 +1,26 @@
+# frozen_string_literal: true
+
 require 'nokogiri'
 
 class Html2Text
   attr_reader :doc
 
-  def initialize(doc)
+  def initialize(doc, wrap_links: true)
     @doc = doc
+    @wrap_links = wrap_links
   end
 
-  def self.convert(html)
+  def self.convert(html, wrap_links: true)
     html = html.to_s
 
-    if is_office_document?(html)
+    if office_document?(html)
       # Emulate the CSS rendering of Office documents
-      html = html.gsub("<p class=MsoNormal>", "<br>")
-        .gsub("<o:p>&nbsp;</o:p>", "<br>")
-        .gsub("<o:p></o:p>", "")
+      html = html.gsub('<p class=MsoNormal>', '<br>')
+                 .gsub('<o:p>&nbsp;</o:p>', '<br>')
+                 .gsub('<o:p></o:p>', '')
     end
 
-    if !html.include?("<html")
+    unless html.include?('<html')
       # Stop Nokogiri from inserting in <p> tags
       html = "<div>#{html}</div>"
     end
@@ -25,25 +28,30 @@ class Html2Text
     html = fix_newlines(replace_entities(html))
     doc = Nokogiri::HTML(html)
 
-    Html2Text.new(doc).convert
+    new(doc, wrap_links: wrap_links).convert
   end
 
   def self.fix_newlines(text)
+    # rubocop:disable Performance/StringReplacement
     text.gsub("\r\n", "\n").gsub("\r", "\n")
+    # rubocop:enable Performance/StringReplacement
   end
 
   def self.replace_entities(text)
-    text.gsub("&nbsp;", " ").gsub("\u00a0", " ").gsub("&zwnj;", "")
+    # rubocop:disable Performance/StringReplacement
+    text.gsub('&nbsp;', ' ').gsub("\u00a0", ' ').gsub('&zwnj;', '')
+    # rubocop:enable Performance/StringReplacement
   end
 
   def convert
     output = iterate_over(doc)
     output = remove_leading_and_trailing_whitespace(output)
     output = remove_unnecessary_empty_lines(output)
-    return output.strip
+    output.strip
   end
 
-  DO_NOT_TOUCH_WHITESPACE = "<do-not-touch-whitespace>"
+  DO_NOT_TOUCH_WHITESPACE = '<do-not-touch-whitespace>'
+  IGNORED_TAGS = Set.new(%w[style head title meta script]).freeze
 
   def remove_leading_and_trailing_whitespace(text)
     # ignore any <pre> blocks, which we don't want to interact with
@@ -51,211 +59,195 @@ class Html2Text
 
     output = []
     pre_blocks.each.with_index do |block, index|
-      if index % 2 == 0
-        output << block.gsub(/[ \t]*\n[ \t]*/im, "\n").gsub(/ *\t */im, "\t")
-      else
-        output << block
-      end
+      output << if index.even?
+                  block.gsub(/[ \t]*\n[ \t]*/im, "\n").gsub(/ *\t */im, "\t")
+                else
+                  block
+                end
     end
 
-    output.join("")
+    output.join
   end
 
-  private
-
-  def self.is_office_document?(text)
-    text.include?("urn:schemas-microsoft-com:office")
+  private_class_method def self.office_document?(text)
+    text.include?('urn:schemas-microsoft-com:office')
   end
 
+  private
+
   def remove_unnecessary_empty_lines(text)
-    text.gsub(/\n\n\n*/im, "\n\n")
+    text.gsub(/\n{3,}/, "\n\n")
   end
 
   def trimmed_whitespace(text)
     # Replace whitespace characters with a space (equivalent to \s)
     # and force any text encoding into UTF-8
     if text.valid_encoding?
-      text.gsub(/[\t\n\f\r ]+/im, " ")
+      text.gsub(/[\t\n\f\r ]+/, ' ')
     else
-      text.force_encoding("WINDOWS-1252")
-      return trimmed_whitespace(text.encode("UTF-16be", invalid: :replace, replace: "?").encode('UTF-8'))
+      text.force_encoding('WINDOWS-1252')
+      trimmed_whitespace(text.encode('UTF-16be', invalid: :replace, replace: '?').encode('UTF-8'))
     end
   end
 
   def iterate_over(node)
-    return "\n" if node.name.downcase == "br" && next_node_is_text?(node)
-
     return trimmed_whitespace(node.text) if node.text?
 
-    if ["style", "head", "title", "meta", "script"].include?(node.name.downcase)
-      return ""
-    end
+    name = node.name&.downcase
 
-    if node.name.downcase == "pre"
-      return "\n#{DO_NOT_TOUCH_WHITESPACE}#{node.text}#{DO_NOT_TOUCH_WHITESPACE}"
-    end
+    return '' if name.nil? || IGNORED_TAGS.include?(name)
 
-    output = []
+    return "\n" if name == 'br' && next_node_is_text?(node)
 
-    output << prefix_whitespace(node)
-    output += node.children.map do |child|
-      iterate_over(child)
-    end
-    output << suffix_whitespace(node)
+    return "\n#{DO_NOT_TOUCH_WHITESPACE}#{node.text}#{DO_NOT_TOUCH_WHITESPACE}" if name == 'pre'
 
-    output = output.compact.join("") || ""
+    return image_text(node) if name == 'img'
 
-    if node.name.downcase == "a"
-      output = wrap_link(node, output)
-    elsif node.name.downcase == "img"
-      output = image_text(node)
+    output = +''
+    output << (prefix_whitespace(node) || '')
+    node.children.each do |child|
+      output << iterate_over(child)
     end
+    output << (suffix_whitespace(node) || '')
 
-    return output
+    output = wrap_link(node, output) if name == 'a'
+
+    output
   end
 
   def prefix_whitespace(node)
     case node.name.downcase
-      when "hr"
-        "\n---------------------------------------------------------------\n"
+    when 'hr'
+      "\n---------------------------------------------------------------\n"
 
-      when "h1", "h2", "h3", "h4", "h5", "h6", "ol", "ul"
-        "\n\n"
+    when 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'ol', 'ul', 'p'
+      "\n\n"
 
-      when "p"
-        "\n\n"
+    when 'tr'
+      "\n"
 
-      when "tr"
+    when 'div'
+      if node.parent.name == 'div' && (node.parent.text.strip == node.text.strip)
+        ''
+      else
         "\n"
+      end
 
-      when "div"
-        if node.parent.name == "div" && (node.parent.text.strip == node.text.strip)
-          ""
-        else
-          "\n"
-        end
-
-      when "td", "th"
-        "\t"
+    when 'td', 'th'
+      "\t"
 
-      when "li"
-        "- "
+    when 'li'
+      '- '
     end
   end
 
   def suffix_whitespace(node)
     case node.name.downcase
-      when "h1", "h2", "h3", "h4", "h5", "h6"
-        # add another line
-        "\n\n"
+    when 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p'
+      "\n\n"
 
-      when "p"
-        "\n\n"
+    when 'br'
+      next_name = next_node_name(node)
+      "\n" if !next_name.nil? && next_name != 'div'
 
-      when "br"
-        if next_node_name(node) != "div" && next_node_name(node) != nil
-          "\n"
-        end
+    when 'li'
+      "\n"
 
-      when "li"
+    when 'div'
+      if next_node_is_text?(node)
         "\n"
-
-      when "div"
-        if next_node_is_text?(node)
-          "\n"
-        elsif next_node_name(node) != "div" && next_node_name(node) != nil
-          "\n"
-        end
+      else
+        next_name = next_node_name(node)
+        "\n" if !next_name.nil? && next_name != 'div'
+      end
     end
   end
 
-  # links are returned in [text](link) format
+  # links are returned in [text](link) format, or "text: link" with wrap_links: false
   def wrap_link(node, output)
-    href = node.attribute("href")
-    name = node.attribute("name")
+    href = node.attribute('href')
+    name = node.attribute('name')
 
     output = output.strip
 
     # remove double [[ ]]s from linking images
-    if output[0] == "[" && output[-1] == "]"
+    if output[0] == '[' && output[-1] == ']'
       output = output[1, output.length - 2]
 
       # for linking images, the title of the <a> overrides the title of the <img>
-      if node.attribute("title")
-        output = node.attribute("title").to_s
-      end
+      output = node.attribute('title').to_s if node.attribute('title')
     end
 
     # if there is no link text, but a title attr
-    if output.empty? && node.attribute("title")
-      output = node.attribute("title").to_s
-    end
+    output = node.attribute('title').to_s if output.empty? && node.attribute('title')
 
     if href.nil?
-      if !name.nil?
-        output = "[#{output}]"
-      end
+      output = "[#{output}]" unless name.nil?
     else
       href = href.to_s
 
       if href != output && href != "mailto:#{output}" &&
-          href != "http://#{output}" && href != "https://#{output}"
-        if output.empty?
-          output = href
-        else
-          output = "[#{output}](#{href})"
-        end
+         href != "http://#{output}" && href != "https://#{output}"
+        output = if output.empty?
+                   href
+                 elsif !@wrap_links
+                   "#{output}: #{href}"
+                 else
+                   "[#{output}](#{href})"
+                 end
       end
     end
 
     case next_node_name(node)
-      when "h1", "h2", "h3", "h4", "h5", "h6"
-        output += "\n"
+    when 'h1', 'h2', 'h3', 'h4', 'h5', 'h6'
+      output += "\n"
     end
 
     output
   end
 
   def image_text(node)
-    if node.attribute("title")
-      "[" + node.attribute("title").to_s + "]"
-    elsif node.attribute("alt")
-      "[" + node.attribute("alt").to_s + "]"
+    if node.attribute('title')
+      "[#{node.attribute('title')}]"
+    elsif node.attribute('alt')
+      "[#{node.attribute('alt')}]"
     else
-      ""
+      ''
     end
   end
 
   def next_node_name(node)
     next_node = node.next_sibling
-    while next_node != nil
+    until next_node.nil?
       break if next_node.element?
+
       next_node = next_node.next_sibling
     end
 
-    if next_node && next_node.element?
-      next_node.name.downcase
-    end
+    return unless next_node&.element?
+
+    next_node.name.downcase
   end
 
   def next_node_is_text?(node)
-    return !node.next_sibling.nil? && node.next_sibling.text? && !node.next_sibling.text.strip.empty?
+    !node.next_sibling.nil? && node.next_sibling.text? && !node.next_sibling.text.strip.empty?
   end
 
   def previous_node_name(node)
     previous_node = node.previous_sibling
-    while previous_node != nil
+    until previous_node.nil?
       break if previous_node.element?
+
       previous_node = previous_node.previous_sibling
     end
 
-    if previous_node && previous_node.element?
-      previous_node.name.downcase
-    end
+    return unless previous_node&.element?
+
+    previous_node.name.downcase
   end
 
   def previous_node_is_text?(node)
-    return !node.previous_sibling.nil? && node.previous_sibling.text? && !node.previous_sibling.text.strip.empty?
+    !node.previous_sibling.nil? && node.previous_sibling.text? && !node.previous_sibling.text.strip.empty?
   end
 
   # def previous_node_is_not_text?(node)