detergent 2.0.0 → 2.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 492c13c265f4d55f83ea2d4e32352ebf0209abf4bf15524ad291d352a7ca56ff
-  data.tar.gz: cd5fcadafd69fb5d43e9db36e7b3f2aff3516d792098a982584fb69eab91b2ba
+  metadata.gz: 1e00871e3ab04f35424e5c563328b1e036bf14a10ec9738da79c81462135e963
+  data.tar.gz: b600be23e126d45cdfcc8e058398a1096274693be3d985f3051511e718f7c248
 SHA512:
-  metadata.gz: 1135a0396b72e5f1c3892befd5f9930d5a8193ca9ffd36d42b5d431e7824d0e2a62b2a2a8afbd1084942780257d6651dc066a3efd18995fed81b53b6d3b3cf8c
-  data.tar.gz: e9809a3df3521cb9cd78c8c9b54f590de968ca84693454feb0e64e62137b0f1aaecc6df7877fed74f19251ece1ccf415328e68f60e92e8569ce52167d920c881
+  metadata.gz: 5263b607e0e15a25b15ab1df4c846f5228a7c188daf6a78811a67e91bf06fdc46b2b9e188fc4158657223e8665df5de894e3b1a6611be5886ab2489c7d63210c
+  data.tar.gz: '0884b42ce893b34ff696234b0d094d9a0a8613c5d9eee3b979308c9214eb4a0b67b5247a9b7cb33e0af9e9320922995dd234c5844e0e3169e899da4e6ab687fe'

data/README.md

@@ -21,6 +21,10 @@ clean_html = Detergent.clean(dirty_html)
 # Or the title and the extracted content node (a Nokogiri node):
 title, content = Detergent.extract(dirty_html)
 
+# Or the extracted main content as Markdown or plain text:
+markdown = Detergent.markdown(dirty_html)
+text = Detergent.text(dirty_html)
+
 # A Cleaner instance is reusable if you're processing many pages:
 cleaner = Detergent::Cleaner.new
 clean_html = cleaner.clean(dirty_html)

data/lib/detergent/cleaner.rb

@@ -62,6 +62,18 @@ module Detergent
       [title, content]
     end
 
+    # Returns the extracted main content as Markdown.
+    def markdown(html)
+      _, content = extract(html)
+      content ? MarkdownRenderer.new.render(content) : ""
+    end
+
+    # Returns the extracted main content as plain text.
+    def text(html)
+      _, content = extract(html)
+      content ? TextRenderer.new.render(content) : ""
+    end
+
     def title(html)
       extract_title(Nokogiri::HTML5(html))
     end

data/lib/detergent/content_locator.rb

@@ -11,13 +11,23 @@ module Detergent
     # promoted — not just score marginally higher from including the child.
     PARENT_PROMOTION_RATIO = 1.3
 
+    # If even the best-scoring node falls below this, the page has no main
+    # content to extract. Pages made entirely of links (index pages, link
+    # aggregators like the Hacker News front page) score at or near zero
+    # everywhere, and without a floor a degenerate winner gets elected —
+    # e.g. a logo cell that picked up a few points for containing an image.
+    MINIMUM_CONTENT_SCORE = 25
+
     def initialize(scorer)
       @scorer = scorer
     end
 
+    # Returns the main content root node, or nil if the page doesn't
+    # appear to have main content at all.
     def locate(body)
       highest_scoring = find_highest_scoring_node(body)
       return nil unless highest_scoring
+      return nil if score(highest_scoring) < MINIMUM_CONTENT_SCORE
 
       find_content_root(highest_scoring)
     end

data/lib/detergent/markdown_renderer.rb

@@ -0,0 +1,164 @@
+# frozen_string_literal: true
+
+module Detergent
+  # Renders a cleaned content node as Markdown. Expects the article-shaped
+  # markup that comes out of extraction; it is not a general-purpose
+  # HTML-to-Markdown converter.
+  #
+  # The formatting decisions live in small hook methods (heading, link,
+  # image, ...) so TextRenderer can subclass this and strip the syntax.
+  class MarkdownRenderer
+    def render(node)
+      render_blocks(node).gsub(/\n{3,}/, "\n\n").strip
+    end
+
+    private
+
+    INLINE_TAGS = %w[a strong b em i code span img br sub sup small u s
+                     strike mark abbr time].freeze
+
+    def inline?(node)
+      node.text? || (node.element? && INLINE_TAGS.include?(node.name.downcase))
+    end
+
+    # Walk a container's children, gathering runs of inline content into
+    # paragraphs and rendering block elements between them.
+    def render_blocks(node)
+      out = []
+      buffer = +""
+
+      node.children.each do |child|
+        if inline?(child)
+          buffer << inline(child)
+        elsif child.element?
+          out << buffer.strip unless buffer.strip.empty?
+          buffer = +""
+
+          block = render_block(child)
+          out << block unless block.empty?
+        end
+      end
+
+      out << buffer.strip unless buffer.strip.empty?
+      out.join("\n\n")
+    end
+
+    def render_block(node)
+      tag = node.name.downcase
+
+      case tag
+      when /\Ah([1-6])\z/
+        heading(Regexp.last_match(1).to_i, inline_content(node).strip)
+      when 'p'
+        inline_content(node).strip
+      when 'blockquote'
+        blockquote(render_blocks(node))
+      when 'ul'
+        list(node, ordered: false)
+      when 'ol'
+        list(node, ordered: true)
+      when 'pre'
+        preformatted(node.text.rstrip)
+      when 'hr'
+        rule
+      when 'figcaption'
+        caption(inline_content(node).strip)
+      when 'table'
+        table(node)
+      else
+        # Generic container (div, section, article, figure, ...)
+        render_blocks(node)
+      end
+    end
+
+    def inline(node)
+      return node.text.gsub(/\s+/, " ") if node.text?
+
+      case node.name.downcase
+      when 'br' then "\n"
+      when 'img' then image(node)
+      when 'a' then link(node)
+      when 'strong', 'b' then wrap(node, "**")
+      when 'em', 'i' then wrap(node, "*")
+      when 'code' then code_span(node)
+      else inline_content(node)
+      end
+    end
+
+    def inline_content(node)
+      node.children.map { |child| inline(child) }.join
+    end
+
+    def list(node, ordered:)
+      items = node.children.select { |c| c.element? && c.name.downcase == 'li' }
+
+      items.each_with_index.map do |li, index|
+        marker = ordered ? "#{index + 1}. " : "- "
+        indent = " " * marker.length
+        marker + render_blocks(li).gsub("\n", "\n#{indent}")
+      end.join("\n")
+    end
+
+    def table(node)
+      rows = node.xpath('.//tr')
+      return "" if rows.empty?
+
+      lines = rows.map do |row|
+        cells = row.xpath('./th | ./td').map { |cell| inline_content(cell).strip }
+        "| #{cells.join(' | ')} |"
+      end
+
+      column_count = rows.first.xpath('./th | ./td').length
+      lines.insert(1, "|#{' --- |' * column_count}")
+      lines.join("\n")
+    end
+
+    # -- Formatting hooks, overridden by TextRenderer --
+
+    def heading(level, text)
+      "#{'#' * level} #{text}"
+    end
+
+    def blockquote(content)
+      content.split("\n").map { |line| line.empty? ? ">" : "> #{line}" }.join("\n")
+    end
+
+    def preformatted(text)
+      "```\n#{text}\n```"
+    end
+
+    def rule
+      "---"
+    end
+
+    def caption(text)
+      text.empty? ? "" : "*#{text}*"
+    end
+
+    def link(node)
+      text = inline_content(node).strip
+      href = node['href'].to_s
+      return text if href.empty? || text.empty?
+
+      "[#{text}](#{href})"
+    end
+
+    def image(node)
+      src = node['src'].to_s
+      return "" if src.empty?
+
+      alt = node['alt'].to_s.gsub(/\s+/, " ").strip
+      "![#{alt}](#{src})"
+    end
+
+    def wrap(node, marker)
+      content = inline_content(node).strip
+      content.empty? ? "" : "#{marker}#{content}#{marker}"
+    end
+
+    def code_span(node)
+      content = node.text.strip
+      content.empty? ? "" : "`#{content}`"
+    end
+  end
+end

data/lib/detergent/text_renderer.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Detergent
+  # Renders a cleaned content node as plain text: the same document
+  # structure as MarkdownRenderer (blank-line paragraphs, list bullets)
+  # with all Markdown syntax stripped. Images are dropped entirely.
+  class TextRenderer < MarkdownRenderer
+    private
+
+    def heading(_level, text)
+      text
+    end
+
+    def blockquote(content)
+      content
+    end
+
+    def preformatted(text)
+      text
+    end
+
+    def rule
+      ""
+    end
+
+    def caption(text)
+      text
+    end
+
+    def link(node)
+      inline_content(node).strip
+    end
+
+    def image(_node)
+      ""
+    end
+
+    def wrap(node, _marker)
+      inline_content(node).strip
+    end
+
+    def code_span(node)
+      node.text.strip
+    end
+  end
+end

data/lib/detergent/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Detergent
-  VERSION = "2.0.0"
+  VERSION = "2.2.0"
 end

data/lib/detergent.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require "cgi"
+require "cgi/escape"
 require "nokogiri"
 
 module Detergent
@@ -31,11 +31,23 @@ module Detergent
   def self.extract(html)
     Cleaner.new.extract(html)
   end
+
+  # Returns the extracted main content as Markdown.
+  def self.markdown(html)
+    Cleaner.new.markdown(html)
+  end
+
+  # Returns the extracted main content as plain text.
+  def self.text(html)
+    Cleaner.new.text(html)
+  end
 end
 
 require_relative "detergent/version"
 require_relative "detergent/node_scorer"
 require_relative "detergent/content_locator"
+require_relative "detergent/markdown_renderer"
+require_relative "detergent/text_renderer"
 require_relative "detergent/matchers/obvious_junk_matcher"
 require_relative "detergent/matchers/removable_node_matcher"
 require_relative "detergent/cleaner"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: detergent
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.2.0
 platform: ruby
 authors:
 - Jeff McFadden
@@ -64,9 +64,11 @@ files:
 - lib/detergent.rb
 - lib/detergent/cleaner.rb
 - lib/detergent/content_locator.rb
+- lib/detergent/markdown_renderer.rb
 - lib/detergent/matchers/obvious_junk_matcher.rb
 - lib/detergent/matchers/removable_node_matcher.rb
 - lib/detergent/node_scorer.rb
+- lib/detergent/text_renderer.rb
 - lib/detergent/version.rb
 homepage: https://github.com/jeffmcfadden/detergent
 licenses: