nokodown 0.1.0

data/lib/nokodown/extractor.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+module Nokodown
+  class Extractor
+    TITLE_SIMILARITY = 0.75
+    MIN_ARTICLE_LENGTH = 500
+    CONDITIONAL_SHARE = 0.5
+    REMOVED_LINK_SHARE = 0.5
+
+    def initialize(html, url: nil, main_content: true, title: true, images: true, links: true)
+      @html = html
+      @url = url
+      @main_content = main_content
+      @title = title
+      @images = images
+      @links = links
+    end
+
+    def call
+      doc = Nokolexbor::HTML(Charset.utf8(@html.to_s))
+      metadata = Metadata.new(doc).to_h
+      urls = UrlResolver.new(@url, doc.at_css("base[href]")&.[]("href"))
+      body = doc.at_css("body")
+      return result([], metadata, :none) if body.nil?
+
+      product_page = @main_content && ProductReviews.product_page?(doc, metadata[:page_type])
+      Preprocessor.new(body).call
+      nodes, mode = select_content(body, metadata[:page_type], product_page)
+      blocks = render(nodes, mode, urls)
+      blocks = with_title(blocks, metadata[:title]) if @title
+      result(blocks, metadata, mode)
+    end
+
+    private
+
+    def render(nodes, mode, urls)
+      redundant = redundant_images(nodes, mode, urls)
+      renderer = Markdown::Renderer.new(urls, images: @images, links: @links, redundant_images: redundant)
+      Markdown::Cleanup.call(renderer.render(nodes), renderer.rendered_images)
+    end
+
+    # Converting the whole page keeps every visible image.
+    def redundant_images(nodes, mode, urls)
+      return {} if mode == :full || !@images
+
+      Markdown::RedundantImages.new(nodes, mode, urls).call
+    end
+
+    def select_content(body, page_type, product_page)
+      return [[body], :full] unless @main_content
+
+      Boilerplate.new(body).call
+      ProductReviews.new(body).call if product_page
+      selection = ContentLocator.new(body, page_type: page_type).call
+      return [cleaned_article(selection.nodes, page_type), :article] if selection.mode == :article
+
+      Pruner.new(selection.nodes, :page).call
+      [selection.nodes, :page]
+    end
+
+    # Like Readability, fall back to lighter cleaning when conditional cleaning
+    # throws away most of what it sees; markup made of many small linky blocks
+    # (React apps, card grids) trips it, and a short intro would otherwise pass
+    # for the article. When almost nothing is left at all, the content is
+    # pruned like a page. Cleaning runs on a copy so it can be undone.
+    def cleaned_article(nodes, page_type)
+      copies = nodes.map(&:dup)
+      pruner = Pruner.new(copies, :article).call
+      if almost_nothing_kept?(copies, nodes)
+        Pruner.new(nodes, :page).call
+        return nodes
+      end
+      return copies if keep_conditional_cleaning?(pruner, page_type)
+
+      Pruner.new(nodes, :article, conditional: false).call
+      nodes
+    end
+
+    # When what conditional cleaning removed from a declared article is mostly
+    # links, it was a grid of teasers for other stories, not the article. On
+    # listings and hubs the same teasers are the content.
+    def keep_conditional_cleaning?(pruner, page_type)
+      return true if pruner.conditional_share >= CONDITIONAL_SHARE
+
+      page_type == :article && pruner.removed_link_share > REMOVED_LINK_SHARE
+    end
+
+    def almost_nothing_kept?(copies, nodes)
+      kept = copies.sum { |node| Dom.text(node).length }
+      kept < MIN_ARTICLE_LENGTH && kept < nodes.sum { |node| Dom.text(node).length } * 0.25
+    end
+
+    # One H1 at the top: the page's own if the content has one, otherwise the
+    # metadata title, replacing a near-duplicate heading right below it.
+    # Nothing found stays empty, so a script-rendered shell or a bot wall does
+    # not pass for a page; Result#title still has the title.
+    def with_title(blocks, title)
+      return blocks if blocks.empty?
+
+      first_h1 = blocks.find { |block| block.kind == :heading && block.level == 1 }
+      return demote_h1s(blocks, first_h1) if first_h1
+      return blocks if title.nil?
+
+      duplicate = blocks.first(3).find do |block|
+        block.kind == :heading && Dom.similarity(title, Markdown::Cleanup.heading_text(block)) > TITLE_SIMILARITY
+      end
+      [Markdown::Block.new(:heading, "# #{Markdown::Text.escape(title)}", 1)] + blocks.reject { |block| block.equal?(duplicate) }
+    end
+
+    def demote_h1s(blocks, keep)
+      blocks.each do |block|
+        next unless block.kind == :heading && block.level == 1 && !block.equal?(keep)
+
+        block.text = "##{block.text}"
+        block.level = 2
+      end
+    end
+
+    def result(blocks, metadata, mode)
+      Result.new(
+        markdown: blocks.map(&:text).join("\n\n"),
+        title: metadata[:title],
+        byline: metadata[:byline],
+        excerpt: metadata[:excerpt] || first_paragraph(blocks),
+        site_name: metadata[:site_name],
+        published_time: metadata[:published_time],
+        lang: metadata[:lang],
+        mode: mode
+      )
+    end
+
+    def first_paragraph(blocks)
+      paragraphs = blocks.select { |block| block.kind == :paragraph }.map { |block| Markdown::Text.plain(block.text) }
+      paragraphs.find { |text| text.length >= 80 } || paragraphs.find { |text| !text.empty? }
+    end
+  end
+end

data/lib/nokodown/markdown/cleanup.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+module Nokodown
+  module Markdown
+    # Block-level tidying after rendering: repeated blocks, stray rules,
+    # punctuation-only lines, section headings left without a section, and
+    # redundant or repeated images.
+    module Cleanup
+      DEDUPE_WINDOW = 4
+      IMAGE_ONLY = /\A!\[([^\]]*)\]\([^)]*\)\z/.freeze
+      LINKED_HEADING = /\A#+ \[.*\]\(.*\)\z/.freeze
+      TITLE_LIST_MIN = 3
+
+      module_function
+
+      # images: the Renderer's rendered_images.
+      def call(blocks, images = {})
+        blocks = blocks.reject { |block| noise?(block) }
+        blocks = dedupe(blocks)
+        blocks = trim_rules(blocks)
+        blocks = drop_empty_sections(blocks)
+        images.empty? ? blocks : ImageCleanup.new(images).call(blocks)
+      end
+
+      def noise?(block)
+        return false unless block.kind == :paragraph
+
+        text = block.text
+        return true unless text.match?(/[\p{L}\p{N}]/) || text.include?("](")
+
+        plain = text.gsub(/[*_`~]/, "").strip
+        plain.length <= 80 && plain.match?(Patterns::BOILERPLATE_TEXT)
+      end
+
+      # Responsive and "enlarge" variants repeat an image (and its caption) a
+      # block or two later, within the same section. Dropping the image makes
+      # the captions adjacent.
+      def dedupe(blocks)
+        blocks.each_with_object([]) do |block, kept|
+          window = block.text.match?(IMAGE_ONLY) ? same_section(kept.last(DEDUPE_WINDOW)) : kept.last(1)
+          kept << block unless window.any? { |previous| identity(previous) == identity(block) }
+        end
+      end
+
+      def same_section(blocks)
+        heading = blocks.rindex { |block| block.kind == :heading }
+        heading ? blocks.drop(heading + 1) : blocks
+      end
+
+      # Variants of one image differ in URL (size, crop) but share alt text.
+      def identity(block)
+        alt = block.text[IMAGE_ONLY, 1]
+        alt && alt.length >= 10 ? "image:#{alt}" : block.text
+      end
+
+      def trim_rules(blocks)
+        blocks = blocks.each_with_object([]) do |block, kept|
+          next if block.kind == :rule && (kept.empty? || kept.last.kind == :rule || kept.last.kind == :heading)
+
+          kept << block
+        end
+        blocks.pop while blocks.last&.kind == :rule
+        blocks
+      end
+
+      # A heading followed by nothing, or by a heading at the same or a higher
+      # level, lost its section during extraction. Title lists are content in
+      # their own right, and a page that is only its headline keeps it.
+      def drop_empty_sections(blocks)
+        titles = title_list_indexes(blocks)
+        kept = blocks.each_index.reject { |index| !titles.include?(index) && empty_section?(blocks, index) }
+        kept.pop while kept.any? && blocks[kept.last].kind == :heading && !titles.include?(kept.last)
+        return blocks.values_at(*kept) if kept.any?
+
+        [blocks.find { |block| block.kind == :heading && block.level == 1 }].compact
+      end
+
+      def empty_section?(blocks, index)
+        block = blocks[index]
+        return false unless block.kind == :heading
+
+        following = blocks[(index + 1)..].find { |b| b.kind != :rule }
+        return true if following.nil?
+        return false unless following.kind == :heading && following.level <= block.level
+
+        heading_text(block).match?(Patterns::NOISE_HEADING)
+      end
+
+      # Same-level headings in a row, mostly links, are the titles of cards
+      # whose details were removed (products, related guides), not sections.
+      # A "Trending" or "Related" heading among them is still a section label.
+      def title_list_indexes(blocks)
+        runs = blocks.each_index.reject { |index| blocks[index].kind == :rule }.slice_when do |a, b|
+          blocks[a].kind != :heading || blocks[b].kind != :heading || blocks[a].level != blocks[b].level
+        end
+        indexes = runs.select { |run| title_list?(blocks.values_at(*run)) }.flatten
+        indexes.reject { |index| heading_text(blocks[index]).match?(Patterns::NOISE_HEADING) }.to_set
+      end
+
+      def title_list?(headings)
+        return false if headings.size < TITLE_LIST_MIN
+
+        headings.count { |heading| heading.text.match?(LINKED_HEADING) } * 2 > headings.size
+      end
+
+      def heading_text(block)
+        block.text.sub(/\A#+\s*/, "").gsub(/\[([^\]]*)\]\([^)]*\)/, '\1').gsub(/[*_`]/, "")
+      end
+    end
+  end
+end

data/lib/nokodown/markdown/code_block.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module Nokodown
+  module Markdown
+    module CodeBlock
+      LANGUAGE_CLASS = /\A(?:language|lang|highlight-source|highlight)-([\w+#.-]+)\z/i.freeze
+      NO_LANGUAGE = %w[none plain plaintext text nohighlight].freeze
+
+      module_function
+
+      def render(node, raw)
+        text = raw.tr(Dom::NBSP, " ").sub(/\A\n/, "").split("\n").map(&:rstrip).join("\n")
+        text = text.sub(/\A(?:[ \t]*\n)+/, "").rstrip
+        return [] if text.strip.empty?
+
+        fence = Text.code_fence(text)
+        [Block.new(:code, "#{fence}#{language(node)}\n#{text}\n#{fence}")]
+      end
+
+      # <code> used as a block: <div class="code"><code>multi-line</code></div>.
+      def standalone?(node)
+        parent = node.parent
+        return false if parent.nil? || parent.name == "pre"
+        return false unless node.content.strip.include?("\n")
+
+        parent.children.all? { |child| child == node || (child.text? && child.content.strip.empty?) }
+      end
+
+      # Highlighters often put each line in its own block element instead of
+      # separating lines with "\n".
+      def preformatted(node)
+        node.children.each_with_object(+"") do |child, out|
+          if child.text? || child.cdata?
+            out << child.content
+          elsif child.name == "br"
+            out << "\n"
+          elsif Dom.block?(child)
+            out << "\n" unless out.empty? || out.end_with?("\n")
+            out << preformatted(child)
+            out << "\n" unless out.end_with?("\n")
+          elsif child.element?
+            out << preformatted(child)
+          end
+        end
+      end
+
+      # The language sits on the <pre>, its <code>, or a wrapper up to two
+      # levels out (GitHub, Sphinx).
+      def language(node)
+        [node, node.at_css("code"), node.parent, node.parent&.parent].each do |element|
+          next unless element&.element?
+
+          name = explicit_language(element)
+          return clean(name) if name
+        end
+        ""
+      end
+
+      def explicit_language(element)
+        classes = element["class"].to_s
+        element["data-lang"] || element["data-language"] || classes[/brush:\s*([\w+#.-]+)/, 1] ||
+          classes.split.map { |name| name[LANGUAGE_CLASS, 1] }.compact.first
+      end
+
+      def clean(name)
+        name = name.downcase.sub(/\Asource-/, "")
+        name.match?(/\A[\w+#.-]{1,24}\z/) && !NO_LANGUAGE.include?(name) ? name : ""
+      end
+    end
+  end
+end

data/lib/nokodown/markdown/image.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module Nokodown
+  module Markdown
+    module Image
+      TRACKING = %r{(?:\A|/)(?:pixel|spacer|blank|transparent|clear|tracking|beacon|1x1|[\w-]*placeholder)\.(?:gif|png|svg|jpe?g|webp)(?:\?|\z)|/(?:pixel|beacon)(?:/|\?)}i.freeze
+      # README status badges: build, version and coverage shields.
+      BADGE = %r{//(?:img\.shields\.io|badge\.fury\.io|badgen\.net)/|badge\.svg(?:[?#]|\z)|/badges?/[^?#]*\.svg}i.freeze
+      # Emoji, including skin-tone modifiers, variation selectors and joiners.
+      EMOJI = /\A[\p{So}\p{Sk}\p{Mn}\p{Cf}]+\z/.freeze
+
+      module_function
+
+      def source(node, urls)
+        src = node["src"].to_s.strip
+        src = largest_srcset(node["srcset"].to_s) if src.empty? || src.start_with?("data:")
+        return if src.nil? || src.empty? || src.start_with?("data:")
+
+        urls.resolve_image(src)
+      end
+
+      def largest_srcset(srcset)
+        candidates = srcset.split(/,\s+/).map do |candidate|
+          url, descriptor = candidate.strip.split(/\s+/, 2)
+          [url, descriptor.to_f]
+        end
+        candidates.reject { |url, _| url.nil? || url.empty? }.max_by { |_, size| size }&.first
+      end
+
+      # Icons, spacers, tracking pixels, placeholders and badges.
+      def decorative?(node, src, alt)
+        return true if alt.empty? && src.match?(/\.svg(?:[?#]|\z)/i)
+
+        width = pixels(node["width"])
+        height = pixels(node["height"])
+        return true if (width && width <= 2) || (height && height <= 2)
+        return true if width && height && width <= 32 && height <= 32
+
+        src.match?(TRACKING) || src.match?(BADGE)
+      end
+
+      def pixels(value)
+        value.to_s.strip.match?(/\A\d+(?:px)?\z/) ? value.to_i : nil
+      end
+
+      # WordPress and Twitter render emoji as images; the alt is the emoji.
+      def emoji?(node, alt)
+        node["class"].to_s.match?(/emoji|wp-smiley/) || (alt.length.between?(1, 8) && alt.match?(EMOJI))
+      end
+
+      # Wikipedia's math fallback images carry the TeX source as alt text.
+      def tex(alt)
+        tex = alt.sub(/\A\{\\displaystyle\s*(.*)\}\z/m, '\1').strip
+        tex.empty? ? "" : "$#{tex}$"
+      end
+
+      def video_page(src)
+        id = src[%r{youtube(?:-nocookie)?\.com/embed/([\w-]{6,})}, 1]
+        id ? "https://www.youtube.com/watch?v=#{id}" : src
+      end
+    end
+  end
+end

data/lib/nokodown/markdown/image_cleanup.rb

@@ -0,0 +1,166 @@
+# frozen_string_literal: true
+
+module Nokodown
+  module Markdown
+    # The last step of Cleanup: removes redundant images and repeats of an
+    # image anywhere on the page. Run earlier, it would leave a heading whose
+    # section held only an image for drop_empty_sections to take.
+    class ImageCleanup
+      # Markup as Inline writes it: "](" in the alt is escaped, and the
+      # destination's parentheses are balanced or percent-encoded. One pattern
+      # for every image keeps the cost linear on pages with thousands; the
+      # rare URL with nested parentheses is not matched, so its image stays.
+      IMAGE = /!\[(?:\\.|[^\\\[\]\n]|\[(?:\\.|[^\\\[\]\n])*\])*\]\((?:[^()\s]|\([^()\s]*\))*\)/.freeze
+      DESTINATION = /\]\(((?:[^()\s]|\([^()\s]*\))*)\)/.freeze
+      # A line with nothing but list or quote markers, nested lists included.
+      BARE_LINE = /\A(?:\s|>|[-*+]|\d+\.)*\z/.freeze
+      LIST_MARKER = /\A *(?:[-*+]|\d+\.) /.freeze
+      LIST_MARKERS = /\A *(?:(?:[-*+]|\d+\.) )+/.freeze
+      # The indentation and markers before a line's text.
+      LINE_PREFIX = /\A(?: |>|(?:[-*+]|\d+\.) )*/.freeze
+      # Text never starts a line with ``` (Text.escape_line_starts), so this
+      # is the fence of a code block nested in a list or a quote.
+      FENCE = /#{LINE_PREFIX.source}(`{3,})/.freeze
+
+      def initialize(images)
+        @images = images
+        # Image URL => @text_lines when its last copy was written.
+        @seen = {}
+        @text_lines = 0
+      end
+
+      def call(blocks)
+        @linked = linked_pages(blocks)
+        blocks.select do |block|
+          if block.kind == :code || !block.text.include?("![")
+            @text_lines += 1
+            next true
+          end
+
+          text = without_images(block.text)
+          next true if text == block.text
+
+          block.text = text
+          text.match?(/[\p{L}\p{N}]/) && !Cleanup.noise?(block)
+        end
+      end
+
+      private
+
+      # Pages the final Markdown still links to.
+      def linked_pages(blocks)
+        destinations = blocks.reject { |block| block.kind == :code }.flat_map { |block| block.text.scan(DESTINATION).flatten }
+        Set.new(destinations.map { |destination| RedundantImages.page(destination) })
+      end
+
+      def without_images(text)
+        lines = with_code_flags(text.split("\n")).map do |line, code|
+          next [line, line_without_images(line), false] unless code
+
+          @text_lines += 1
+          [line, line, true]
+        end
+        squeeze(drop_bare_lines(lines)).strip
+      end
+
+      # Each line, and whether it belongs to a nested code block, whose lines
+      # stay as they are.
+      def with_code_flags(lines)
+        fence = nil
+        lines.map do |line|
+          run = line[FENCE, 1]
+          if fence
+            fence = nil if run && run.length >= fence.length
+            next [line, true]
+          end
+          fence = run
+          [line, !run.nil?]
+        end
+      end
+
+      # Text that followed a removed image can now start the line, where "#"
+      # or ">" would turn it into a heading or a quote.
+      def line_without_images(line)
+        removed = removed_images(line)
+        return line if removed.empty?
+
+        kept = line.gsub(IMAGE) { |markup| removed.include?(markup) ? " " : markup }
+        prefix = line[LINE_PREFIX]
+        prefix + Text.escape_line_starts(kept[prefix.length..].strip.squeeze(" "))
+      end
+
+      # Redundant images, and later copies of an image standing alone on its
+      # line. Beside text, as in a table of tick marks, each copy counts.
+      def removed_images(line)
+        markups = line.scan(IMAGE).select { |markup| @images.key?(markup) }
+        content = markups.reject { |markup| redundant?(@images[markup]) }
+        alone = line.gsub(IMAGE) { |markup| @images.key?(markup) ? "" : markup }.match?(BARE_LINE)
+        removed = Set.new(markups - content)
+        removed.merge(repeats(content)) if alone
+        content.each { |markup| @seen[@images[markup].url] = @text_lines }
+        @text_lines += 1 unless alone
+        removed
+      end
+
+      # A line with several copies of one image is a count, such as a row of
+      # rating stars, and keeps them all. Otherwise a copy without alt text
+      # adds nothing. One with alt text can mark its own item (an award, a
+      # verified tick), so it goes only when nothing but images came since
+      # the last copy, as in a logo strip shown twice.
+      def repeats(content)
+        urls = content.map { |markup| @images[markup].url }
+        return [] if urls.uniq.size < urls.size
+
+        content.select do |markup|
+          seen = @seen[@images[markup].url]
+          seen && (markup.start_with?("![]") || seen == @text_lines)
+        end
+      end
+
+      # A thumbnail goes only while the Markdown still links to its page; a
+      # card whose title link was dropped keeps its image.
+      def redundant?(image)
+        image.redundant && image.targets.all? { |target| @linked.include?(target) }
+      end
+
+      # Lines left with only markers go. A list item whose first line held
+      # only images starts at its next line instead.
+      def drop_bare_lines(lines)
+        marker = nil
+        lines.filter_map do |line, kept, code|
+          if kept != line && kept.match?(BARE_LINE)
+            marker = pending_marker(line, marker)
+            next
+          end
+          next [kept, code] if kept.empty?
+
+          kept = with_marker(kept, marker) if marker
+          marker = nil
+          [kept, code]
+        end
+      end
+
+      # One blank line where dropped lines left several, except in code.
+      def squeeze(lines)
+        lines.each_with_object([]) do |(line, code), kept|
+          kept << line unless !code && line.empty? && kept.last == ""
+        end.join("\n")
+      end
+
+      # The outermost item still waiting for its text: a bare line nested in
+      # it, such as a list of thumbnails, does not take its place.
+      def pending_marker(line, marker)
+        return marker if marker && line.start_with?(" " * marker[LIST_MARKER].length)
+
+        line[LIST_MARKERS] || marker
+      end
+
+      # The markers of the items the line is indented into, not deeper.
+      def with_marker(line, marker)
+        prefixes = marker.scan(/ *(?:[-*+]|\d+\.) /).each_with_object([]) { |piece, all| all << "#{all.last}#{piece}" }
+        prefix = prefixes.reverse.find { |candidate| line.start_with?(" " * candidate.length) && !line[candidate.length..].match?(/\A\s/) }
+        prefix ? prefix + line[prefix.length..] : line
+      end
+    end
+  end
+end