nokodown 0.1.0

data/lib/nokodown/preprocessor.rb

@@ -0,0 +1,226 @@
+# frozen_string_literal: true
+
+module Nokodown
+  # Removes everything that can never be content: scripts, invisible elements,
+  # form controls, embeds. Lazy-loaded images are restored first so their
+  # placeholders are not mistaken for decoration.
+  class Preprocessor
+    JUNK = %w[
+      script style noscript template link meta base object embed applet param canvas map area svg
+      source track audio video dialog select option optgroup datalist textarea output progress meter
+      keygen frame frameset portal slot
+    ].join(",").freeze
+
+    HIDDEN_CLASSES = Set.new(%w[
+      hidden hide d-none is-hidden u-hidden visually-hidden visuallyhidden sr-only screen-reader-text
+      screen-reader-only screenreader-only a11y-hidden offscreen invisible element-invisible
+      print-only visible-print d-print-block show-for-sr
+    ]).freeze
+
+    # Found by Lexbor's selector engine, so the checks below only run on
+    # elements that could be hidden.
+    HIDDEN_CANDIDATES = (["[hidden]", "[aria-hidden=true]", "[style]"] + HIDDEN_CLASSES.map { |name| ".#{name}" }).join(", ").freeze
+
+    # Utility classes that make a "hidden" element visible again at some breakpoint.
+    RESPONSIVE_SHOW = /\A(?:(?:sm|md|lg|xl|2xl):(?:block|flex|grid|inline|inline-block|inline-flex|table|contents)|d-(?:sm|md|lg|xl|xxl)-(?:block|flex|grid|inline|inline-block|inline-flex|table))\z/.freeze
+
+    HIDDEN_STYLE = /(?:\A|;)\s*(?:display\s*:\s*none|visibility\s*:\s*hidden)/i.freeze
+
+    LINE_NUMBERS = /(?<![a-z])(?:line-?numbers?(?:-rows)?|linenos?|lineno|gutter|hljs-ln-numbers|ln-num)(?![a-z])/.freeze
+
+    VIDEO_HOSTS = %r{//(?:www\.)?(?:(?:dailymotion|youtube|youtube-nocookie|player\.vimeo|v\.qq|bilibili|live\.bilibili)\.com|(?:archive|upload\.wikimedia)\.org|player\.twitch\.tv|fast\.wistia\.(?:net|com)|play\.vidyard\.com|loom\.com/embed)}i.freeze
+
+    IMAGE_URL = /\.(?:jpe?g|png|webp|gif|avif)(?:[?#]\S*)?\s*\z/i.freeze
+    IMAGE_SRCSET = /\.(?:jpe?g|png|webp|gif|avif)(?:\?\S*)?\s+\d/i.freeze
+    LAZY_SRC = %w[
+      data-src data-lazy-src data-original data-lazy data-url data-hi-res-src data-full-src
+      data-large-src data-orig-src data-actualsrc data-echo data-img-src data-image
+    ].freeze
+    LAZY_SRCSET = %w[data-srcset data-lazy-srcset data-original-set].freeze
+
+    def initialize(body)
+      @body = body
+    end
+
+    def call
+      @body.xpath(".//comment()").each(&:remove)
+      unwrap_noscript
+      @body.css("img").each { |img| restore_lazy_image(img) }
+      restore_picture_sources
+      @body.css(JUNK).each(&:remove)
+      @body.css("iframe").each { |iframe| iframe.remove unless iframe["src"].to_s.match?(VIDEO_HOSTS) }
+      @body.css("pre [class], table[class] td[class]").each { |node| node.remove if node["class"].downcase.match?(LINE_NUMBERS) }
+      remove_hidden
+      remove_form_controls
+      Dom.assign_keys(@body)
+    end
+
+    private
+
+    # Lexbor keeps <noscript> content as text. Lazy loaders put the real <img>
+    # there, and some apps (Discourse) put the whole page there for crawlers.
+    def unwrap_noscript
+      @body.css("noscript").each do |noscript|
+        fragment = @body.document.fragment(noscript.content)
+        images = fragment.css("img")
+        if images.size == 1 && Dom.text(fragment).empty?
+          (lazy_copy(noscript, images.first) || noscript).replace(images.first)
+        elsif Dom.char_count(fragment.content) >= 300 && empty_without_noscript?
+          noscript.replace(fragment.children)
+        end
+      end
+    end
+
+    def empty_without_noscript?
+      @empty_without_noscript = visible_char_count < 200 if @empty_without_noscript.nil?
+      @empty_without_noscript
+    end
+
+    def visible_char_count
+      invisible = @body.css("script, style, noscript, template").sum { |node| Dom.char_count(node.content) }
+      Dom.char_count(@body.content) - invisible
+    end
+
+    # The lazy copy sits just before the <noscript>, or just after it when it
+    # loads the same file. Otherwise both copies would be rendered.
+    def lazy_copy(noscript, image)
+      following = lone_image(noscript.next_element)
+      return following if following && same_source?(following, image)
+
+      lone_image(noscript.previous_element)
+    end
+
+    def lone_image(node)
+      return unless node
+      return node if node.name == "img"
+
+      images = node.css("img")
+      images.first if images.size == 1 && Dom.text(node).empty?
+    end
+
+    def same_source?(img, image)
+      src = image["src"].to_s.strip
+      !src.empty? && ["src", *LAZY_SRC].any? { |name| img[name].to_s.strip == src }
+    end
+
+    def restore_lazy_image(img)
+      img.remove_attribute("src") if placeholder_src?(img)
+      return if img["src"] && !img["class"].to_s.downcase.include?("lazy")
+
+      copy_first_attribute(img, LAZY_SRC, "src")
+      copy_first_attribute(img, LAZY_SRCSET, "srcset")
+      restore_from_any_attribute(img) unless img["src"] || img["srcset"]
+    end
+
+    def copy_first_attribute(img, names, target)
+      value = names.map { |name| img[name].to_s.strip }.find { |v| !v.empty? && !v.start_with?("data:") }
+      img[target] = value if value
+    end
+
+    def restore_from_any_attribute(img)
+      img.attribute_nodes.each do |attribute|
+        next if %w[src srcset alt].include?(attribute.name)
+
+        value = attribute.value.to_s
+        if value.match?(IMAGE_SRCSET)
+          img["srcset"] = value
+        elsif value.match?(IMAGE_URL) && !value.include?(" ")
+          img["src"] = value
+        end
+      end
+    end
+
+    # Tiny base64 GIF/PNG stand-ins that a lazy loader swaps out.
+    def placeholder_src?(img)
+      src = img["src"].to_s.strip
+      return false unless src.start_with?("data:")
+      return false if src.start_with?("data:image/svg+xml") && src.length > 200
+
+      src.length < 300 || img.attribute_nodes.any? { |a| a.name != "src" && a.value.to_s.match?(IMAGE_URL) }
+    end
+
+    def restore_picture_sources
+      @body.css("picture").each do |picture|
+        img = picture.at_css("img")
+        next if img.nil? || img["src"] || img["srcset"]
+
+        source = picture.css("source").find { |s| s["srcset"] || s["data-srcset"] }
+        img["srcset"] = source["srcset"] || source["data-srcset"] if source
+      end
+    end
+
+    def remove_hidden
+      total = Dom.char_count(@body.content)
+      @panel_ids = @body.css("[aria-controls]").flat_map { |control| control["aria-controls"].split }.to_set
+      @body.css(HIDDEN_CANDIDATES).each do |node|
+        next unless hidden?(node)
+        # A hidden wrapper around most of the page is an anti-flicker snippet
+        # or an open modal in the snapshot, not something readers cannot see.
+        next if total > 500 && Dom.char_count(node.content) > total * 0.3
+
+        node.remove
+      end
+    end
+
+    def hidden?(node)
+      return !disclosure?(node) if node.key?("hidden") && node["hidden"] != "until-found"
+      return !node["class"].to_s.include?("fallback-image") && !closed_panel?(node) if node["aria-hidden"] == "true"
+      return !disclosure?(node) if node["style"].to_s.match?(HIDDEN_STYLE)
+
+      hidden_by_class?(node) && !disclosure?(node)
+    end
+
+    def hidden_by_class?(node)
+      classes = node["class"].to_s.split
+      classes.any? { |c| HIDDEN_CLASSES.include?(c) } && classes.none? { |c| c.match?(RESPONSIVE_SHOW) }
+    end
+
+    def disclosure?(node)
+      node["role"] == "tabpanel" || Dom.signature(node).match?(Patterns::DISCLOSURE)
+    end
+
+    # Accordions and tabs mark closed panels aria-hidden, but icons and the
+    # full text behind "read more" are aria-hidden too, so a panel only counts
+    # when a control on the page opens it or a tab labels it.
+    def closed_panel?(node)
+      disclosure?(node) && (node.key?("aria-labelledby") || @panel_ids.include?(node["id"]))
+    end
+
+    def remove_form_controls
+      @body.css("form").each { |form| form.remove if utility_form?(form) }
+      @body.css("input").each { |input| input.remove unless task_checkbox?(input) }
+      @body.css("button").each { |button| disclosure_button?(button) ? button.replace(button.children) : button.remove }
+      @body.css("label").each { |label| label.remove if label["for"] || Dom.ancestors_named(label, "form").any? }
+      @body.css("form").each { |form| form.remove if leftover_form?(form) }
+    end
+
+    # Only a prompt such as "Search:" is left once the controls are gone.
+    def leftover_form?(form)
+      text = Dom.text(form)
+      text.length < 30 && !text.match?(/\d/) && form.at_css("img").nil?
+    end
+
+    def task_checkbox?(input)
+      input["type"].to_s.casecmp?("checkbox") && input.parent&.name == "li"
+    end
+
+    # Search boxes, sign-ups and logins. Forms that wrap real content (ASP.NET
+    # pages, product option pickers) are left alone.
+    def utility_form?(form)
+      return false if Dom.char_count(form.content) >= 300
+
+      form["role"] == "search" ||
+        form.css("input").any? { |i| %w[search email password].include?(i["type"].to_s.downcase) } ||
+        form["action"].to_s.match?(/search|subscribe|newsletter|login|sign-?in|sign-?up/i) ||
+        Dom.signature(form).match?(Patterns::NOISE)
+    end
+
+    # Accordion triggers usually carry the question text of an FAQ.
+    def disclosure_button?(button)
+      parent = button.parent
+      return false unless button.key?("aria-expanded") || button.key?("aria-controls") || (parent && Dom.heading?(parent))
+
+      Dom.text(button).split.size >= 3
+    end
+  end
+end

data/lib/nokodown/product_reviews.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module Nokodown
+  # Removes customer reviews, ratings and Q&A widgets from product pages.
+  # Shoppers' prose easily outweighs a product description, so it would win
+  # the content scoring or, in page mode, bury the description. On other
+  # pages a reviews section is usually the content itself (a review article,
+  # a service's testimonials), so only product pages are cleaned.
+  class ProductReviews
+    # Shops that declare no page type in JSON-LD or Open Graph. Form controls
+    # are gone once the body is cleaned, so this is checked on the raw
+    # document.
+    PRODUCT_MARKUP = "[itemtype*='schema.org/Product'], form[action*='/cart/add']"
+
+    # The product's title and description stay even when a wrapper around
+    # them is named after its reviews.
+    PROTECTED = "h1, [itemprop~=description]"
+
+    # Testimonials and a book's editorial reviews are the seller's own words.
+    SELLER_CONTENT = /testimonial|editorial/.freeze
+
+    ITEMPROP = /\Areviews?\z/i.freeze
+
+    # With less text than this left, the reviews are what the page is about.
+    MIN_REMAINING_CHARS = 200
+
+    # Review articles embed affiliate product boxes and quick-add forms too,
+    # so the markup only counts on a page that declares no type.
+    def self.product_page?(doc, page_type)
+      return page_type == :product unless page_type.nil?
+
+      !doc.at_css(PRODUCT_MARKUP).nil?
+    end
+
+    def initialize(body)
+      @body = body
+    end
+
+    def call
+      @protected = protected_keys
+      # Card grids repeat the same class list hundreds of times.
+      @review_names = Hash.new { |cache, signature| cache[signature] = signature.match?(Patterns::REVIEWS) }
+      widgets = outermost_widgets(@body)
+      return if widgets.empty?
+
+      remaining = Dom.char_count(@body.content) - widgets.sum { |node| Dom.char_count(node.content) }
+      widgets.each(&:remove) if remaining >= MIN_REMAINING_CHARS
+    end
+
+    private
+
+    # A review's own description (schema.org Review) must not shield the
+    # list it sits in.
+    def protected_keys
+      kept = @body.css(PROTECTED).reject { |node| inside_review_item?(node) }
+      kept.flat_map { |node| [node] + node.ancestors.to_a }.map { |node| Dom.key(node) }.compact.to_set
+    end
+
+    def inside_review_item?(node)
+      node.ancestors.any? { |ancestor| ancestor.element? && ancestor["itemprop"].to_s.match?(ITEMPROP) }
+    end
+
+    def outermost_widgets(node, found = [])
+      node.element_children.each do |child|
+        signature = Dom.signature(child)
+        next if signature.match?(SELLER_CONTENT)
+
+        if widget?(child, signature)
+          found << child
+        else
+          outermost_widgets(child, found)
+        end
+      end
+      found
+    end
+
+    def widget?(node, signature)
+      return false if kept?(node, signature)
+
+      @review_names[signature] || node["itemprop"].to_s.match?(ITEMPROP)
+    end
+
+    # A rating badge in a line of text ("4.7 stars, 212 reviews") sums the
+    # reviews up like a price does. A section named after the description too
+    # holds the seller's words, such as critics' quotes on a game page.
+    def kept?(node, signature)
+      @protected.include?(Dom.key(node)) || Boilerplate::PHRASING.include?(node.name) || signature.include?("description")
+    end
+  end
+end

data/lib/nokodown/pruner.rb

@@ -0,0 +1,337 @@
+# frozen_string_literal: true
+
+module Nokodown
+  # Final cleanup inside the selected content.
+  class Pruner
+    CLUSTER_CONTAINERS = "ul, ol, menu, dl, div, section, p, td, li, span, table, header, footer"
+    TEXT_BLOCKS = "p, div, span, li, h1, h2, h3, h4, h5, h6, small, section, aside, figcaption, b, strong, em, i"
+    BYLINE = /byline|author|dateline|writtenby|p-author/.freeze
+    SHARE = /(?<![a-z])(?:share|sharedaddy)(?![a-z])/.freeze
+    # A class names a button when that is its last word, before a modifier,
+    # a size or a rank: "btn", "c-button", "hero-cta", "btn--ghost",
+    # "btn_medium", "button-primary". "button-text" and
+    # "btn-view-author-options" name something else, and Bootstrap's
+    # "btn-link" is a button that looks like a link.
+    BUTTON_CLASS = /
+      (?:\A|[-_])(?:btn|btns|button|buttons|cta|ctas)\d*
+      (?:--[\w-]+|[-_](?:primary|secondary|tertiary|default|small|medium|large|sm|md|lg))?\z
+    /x.freeze
+    BUTTON_WORD = /btn|button|cta/.freeze
+    LINK_STYLED_BUTTON = /\A(?:btn|button)-link\z/.freeze
+    CALL_TO_ACTION_PARTS = "img, picture, h1, h2, h3, h4, h5, h6, table, ul, ol, p"
+    # Inside these a button is a title, a term or a footnote marker.
+    LABEL_HOLDERS = %w[h1 h2 h3 h4 h5 h6 dt th sup label summary].freeze
+    MIN_INDEX_LENGTH = 80
+    DOWNLOAD = /\.(?:pdf|zip|docx?|xlsx?|pptx?|csv|epub|mp3|mp4|dmg|exe|msi|apk|tar|gz)\z/i.freeze
+
+    def initialize(nodes, mode, conditional: true)
+      @nodes = nodes
+      @mode = mode
+      @conditional = conditional
+      @conditional_seen = 0
+      @conditional_kept = 0
+      @conditional_removed_links = 0
+    end
+
+    def call
+      @nodes.each do |root|
+        @mode == :article ? prune_article(root) : remove_link_clusters(root)
+        remove_boilerplate_text(root)
+        remove_calls_to_action(root)
+      end
+      self
+    end
+
+    # Share of the text conditional cleaning was given that it kept.
+    def conditional_share
+      @conditional_seen.zero? ? 1.0 : @conditional_kept.fdiv(@conditional_seen)
+    end
+
+    # Share of the text conditional cleaning removed that was link text.
+    def removed_link_share
+      removed = @conditional_seen - @conditional_kept
+      removed.zero? ? 0.0 : @conditional_removed_links.fdiv(removed)
+    end
+
+    private
+
+    def prune_article(root)
+      content_length = Dom.text(root).length
+      remove_secondary(root, content_length)
+      remove_bylines(root)
+      remove_article_furniture(root, content_length)
+      remove_related_links(root)
+      remove_stray_lines(root)
+      clean_conditionally(root, content_length)
+      root.css("h1, h2").each { |heading| remove(heading) if Scorer.cleaning_weight(heading).negative? }
+    end
+
+    # Boxes of links to other stories placed inside the body: "READ MORE:
+    # <link>", a "More:" label over a list of headlines, a paragraph of links.
+    def remove_related_links(root)
+      root.css("p, li, h2, h3, h4, h5, h6, strong, b").each do |node|
+        text = Dom.text(node)
+        if related_label?(text)
+          remove_with_link_box(node)
+        elsif related_box?(node, text)
+          remove(node)
+        end
+      end
+    end
+
+    # "More:" or "DON'T MISS" on its own.
+    def related_label?(text)
+      text.length <= 30 && text.match?(Patterns::RELATED_LABEL) && text.sub(Patterns::RELATED_LABEL, "").empty?
+    end
+
+    # The label may sit in a wrapper of its own (Wikipedia's div.mw-heading),
+    # so the box of links follows the wrapper.
+    def remove_with_link_box(label)
+      text = Dom.text(label)
+      holder = label
+      holder = holder.parent while holder.next_element.nil? && holder.parent&.element? && Dom.text(holder.parent) == text
+      following = holder.next_element
+      remove(following) if following && link_density(following) >= 0.6
+      remove(holder)
+    end
+
+    # "READ MORE: <headline>", or a paragraph made of several headlines.
+    def related_box?(node, text)
+      return link_density(node) >= 0.5 if text.match?(Patterns::RELATED_LABEL)
+
+      node.name == "p" && node.css("a[href]").size >= 2 && link_density(node) >= 0.8
+    end
+
+    # Datelines and "follow us" lines.
+    def remove_stray_lines(root)
+      root.css("p, div, span, li, small, time").each do |node|
+        next if Dom.ancestors_named(node, "table").any?
+
+        text = Dom.text(node)
+        remove(node) if text.length <= 120 && text.match?(Patterns::STRAY_LINE)
+      end
+    end
+
+    def link_density(node)
+      text_length = Dom.text(node).length
+      return 0.0 if text_length.zero?
+
+      link_text_length(node).fdiv(text_length)
+    end
+
+    def link_text_length(node)
+      node.css("a[href]").sum { |link| Dom.text(link).length }
+    end
+
+    # Readability drops footers, asides and share widgets inside the article;
+    # asides holding real prose (side notes) stay.
+    def remove_article_furniture(root, content_length)
+      root.css("footer").each { |node| remove(node) if Dom.text(node).length <= content_length * 0.5 }
+      root.css("aside").each do |node|
+        remove(node) if Dom.text(node).length < 200 || ContainerCheck.new(node, content_length, Set.new).remove?
+      end
+      root.css("[class], [id]").each do |node|
+        remove(node) if Dom.signature(node).match?(SHARE) && Dom.text(node).length < 500
+      end
+    end
+
+    # Turned off, it still drops unlabeled navigation, as page pruning does.
+    def clean_conditionally(root, content_length)
+      return remove_link_clusters(root) unless @conditional
+
+      text_before = Dom.text(root).length
+      links_before = link_text_length(root)
+      remove_suspicious_containers(root, content_length)
+      @conditional_seen += text_before
+      @conditional_kept += Dom.text(root).length
+      @conditional_removed_links += links_before - link_text_length(root)
+    end
+
+    # Innermost containers first, so an outer one is judged on what is left.
+    def remove_suspicious_containers(root, content_length)
+      paragraph_keys = Dom.paragraph_keys(root)
+      memo = { root: root }
+      ["form, fieldset", "table", "ul", "div"].each do |selector|
+        root.css(selector).reverse_each do |node|
+          next if paragraph_keys.include?(Dom.key(node)) || !ContainerCheck.new(node, content_length, paragraph_keys, memo).remove?
+
+          ContainerCheck.forget(memo, node)
+          remove(node)
+        end
+      end
+    end
+
+    # A forum's thread is content.
+    def remove_secondary(root, total)
+      root.element_children.each do |node|
+        next if Dom.thread?(node)
+
+        if Dom.signature(node).match?(Patterns::SECONDARY) && Dom.text(node).length <= total * 0.5
+          remove(node)
+        else
+          remove_secondary(node, total)
+        end
+      end
+    end
+
+    # The author line is returned as metadata, like Readability does. Many of
+    # them mean a thread or a list of posts, where the authors are content.
+    def remove_bylines(root)
+      bylines = root.css("*").select do |node|
+        byline = node["rel"] == "author" || node["itemprop"].to_s.include?("author") || Dom.signature(node).match?(BYLINE)
+        byline && Dom.text(node).length.between?(1, 99)
+      end
+      bylines.each { |node| remove(node) } if bylines.size <= 2
+    end
+
+    # Navigation that survived as unlabeled markup: several links, almost no
+    # other text, and no link long enough to be a headline.
+    def remove_link_clusters(root)
+      clusters = root.css(CLUSTER_CONTAINERS).select { |node| link_cluster?(node) && !in_data_table?(node) }
+      # Removal can only start at one of these, so none means nothing to remove.
+      return if clusters.empty? || clusters_dominate?(root, clusters)
+
+      root.css(CLUSTER_CONTAINERS).reverse_each do |node|
+        remove(node) if link_cluster?(node) && !in_data_table?(node)
+      end
+    end
+
+    # On an index page (an A-Z list, a category grid) the clusters are the
+    # content, not navigation around it. A footer sitemap, or a few skip and
+    # account links left in a root that missed the content, is no index.
+    def clusters_dominate?(root, clusters)
+      index = outermost(clusters).reject do |node|
+        %w[header footer].include?(node.name) || Dom.ancestors_named(node, "header", "footer").any?
+      end
+      length = index.sum { |node| Dom.text(node).length }
+      length >= MIN_INDEX_LENGTH && length >= Dom.text(root).length * 0.5
+    end
+
+    def outermost(nodes)
+      keys = nodes.map { |node| Dom.key(node) }.to_set
+      nodes.reject do |node|
+        node.ancestors.any? { |ancestor| ancestor.element? && keys.include?(Dom.key(ancestor)) }
+      end
+    end
+
+    def link_cluster?(node)
+      links = node.css("a[href]")
+      return false if links.size < 3
+
+      text_length = Dom.text(node).length
+      lengths = links.map { |link| Dom.text(link).length }
+      text_length.positive? && lengths.max < 30 && lengths.sum.fdiv(text_length) >= 0.7
+    end
+
+    def in_data_table?(node)
+      tables = node.name == "table" ? [node] : []
+      (tables + Dom.ancestors_named(node, "table")).any? { |table| Tables.data_table?(table) }
+    end
+
+    def remove_boilerplate_text(root)
+      root.css(TEXT_BLOCKS).each do |node|
+        text = Dom.text(node)
+        next if text.empty? || text.length > 200
+
+        boilerplate = text.length <= 80 && text.match?(Patterns::BOILERPLATE_TEXT)
+        copyright = text.match?(Patterns::COPYRIGHT_TEXT) && node.at_css("img").nil?
+        remove(node) if boilerplate || copyright
+      end
+    end
+
+    # Links and boxes styled as buttons ("Add to cart", "Book a demo") are
+    # controls, not content. Cards (an image, a heading, a list) are content.
+    # All are judged before any is removed, so document order does not matter.
+    def remove_calls_to_action(root)
+      @running_text = {}
+      root.css("a, span, div, p").select { |node| call_to_action?(node) }.each { |node| remove(node) }
+    end
+
+    def destination(link)
+      href = link["href"].to_s.strip
+      href unless href.empty? || href.start_with?("#") || href.match?(/\Ajavascript:/i)
+    end
+
+    def links_of(node)
+      (node.name == "a" ? [node] : node.css("a[href]").to_a).select { |link| destination(link) }
+    end
+
+    def download?(node)
+      links_of(node).any? { |link| destination(link).sub(/[?#].*/m, "").match?(DOWNLOAD) }
+    end
+
+    def only_link_in_card?(node, card)
+      own = links_of(node).map { |link| Dom.key(link) }
+      return false if own.empty? || card.nil?
+
+      card.css("a[href]").none? { |link| destination(link) && !own.include?(Dom.key(link)) }
+    end
+
+    def call_to_action?(node)
+      return false unless button?(node)
+
+      text = Dom.text(node)
+      return false if text.length > 40 || text.split.size > 6 || text.match?(/\d/) || node.at_css(CALL_TO_ACTION_PARTS)
+      return false if Dom.ancestors_named(node, *LABEL_HOLDERS).any?
+
+      # FAQ questions and section titles that open a disclosure; the toggle
+      # may be a wrapper around the styled label.
+      wrappers = wrappers(node, text)
+      return false if text.end_with?("?") || wrappers.any? { |wrapper| toggle?(wrapper) }
+      return false if download?(node) || only_link_in_card?(node, wrappers.last.parent)
+
+      # In a data table the button is a cell value, like a plan's "Order" column.
+      !inside_sentence?(wrappers) && !in_data_table?(node)
+    end
+
+    def button?(node)
+      return true if node["role"] == "button"
+      return false unless "#{node["class"]} #{node["id"]}".downcase.match?(BUTTON_WORD)
+
+      tokens = Dom.signature(node).split
+      tokens.any? { |token| token.match?(BUTTON_CLASS) } && tokens.none? { |token| token.match?(LINK_STYLED_BUTTON) }
+    end
+
+    # The node and the elements around it that hold nothing else, such as the
+    # link of an accordion title around a styled label.
+    def wrappers(node, text)
+      chain = [node]
+      parent = node.parent
+      while parent&.element? && Dom.text(parent) == text
+        chain << parent
+        parent = parent.parent
+      end
+      chain
+    end
+
+    def toggle?(node)
+      node.key?("aria-expanded") || node.key?("aria-controls") || Dom.signature(node).match?(Patterns::DISCLOSURE)
+    end
+
+    # A button-styled link in running text ("<a class=btn>Download</a> the
+    # installer") is part of the sentence. Blocks and other buttons beside it
+    # are not running text, so a button after a heading or in a group is not.
+    def inside_sentence?(wrappers)
+      control = wrappers.last
+      return false if control.parent.nil? || wrappers.any? { |wrapper| Dom.block?(wrapper) }
+
+      running_text(control.parent) - running_text_length(control) >= 20
+    end
+
+    def running_text(parent)
+      @running_text[Dom.key(parent)] ||= parent.children.sum { |child| running_text_length(child) }
+    end
+
+    def running_text_length(child)
+      return Dom.text(child).length if child.text?
+      return 0 if !child.element? || Dom.block?(child) || button?(child)
+
+      Dom.text(child).length
+    end
+
+    # Every rule removes through here: nothing inside code is furniture.
+    def remove(node)
+      node.remove unless Dom.in_code?(node)
+    end
+  end
+end

data/lib/nokodown/result.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Nokodown
+  # mode is :article when the page was narrowed to one body of prose, :page
+  # when the whole main region was kept, :full when main_content was off and
+  # :none when the document had no body.
+  Result = Struct.new(:markdown, :title, :byline, :excerpt, :site_name, :published_time, :lang, :mode, keyword_init: true)
+end