nokodown 0.1.0

data/lib/nokodown/boilerplate.rb

@@ -0,0 +1,236 @@
+# frozen_string_literal: true
+
+module Nokodown
+  # Removes site furniture (navigation, headers, footers, sidebars) and noise
+  # widgets (ads, cookie banners, share bars) from the whole page.
+  #
+  # No rule may remove a content landmark, an ancestor of one, or an element
+  # holding most of the page's prose: class names such as "has-sidebar" and
+  # ad-targeting attributes routinely sit on the wrappers around the content.
+  class Boilerplate
+    LANDMARKS = "main, [role=main], [itemprop~=articleBody]"
+    CONTENT_ANCESTORS = %w[article main].freeze
+    SECTIONING = %w[article aside main nav section].freeze
+    PHRASING = Set.new(%w[a abbr b bdi bdo cite code data dfn em i kbd mark q s samp small span strong sub sup time u var]).freeze
+    AD_ATTRIBUTES = %w[data-ad-slot data-ad-unit data-adunit data-ad-client data-google-query-id data-ad-format].freeze
+    DRAWER = /(?<![a-z])drawer(?![a-z])/.freeze
+    MIN_CARD_RUN = 3
+    CARD_RUN_SHARE = 0.2
+
+    def initialize(body)
+      @body = body
+    end
+
+    def call
+      @stats = Stats.new(@body)
+      @total = @stats.text_chars(@body)
+      @shielded = shielded_keys
+      # Card grids repeat the same class list hundreds of times.
+      @furniture = Hash.new { |cache, signature| cache[signature] = signature.match?(Patterns::PAGE_FURNITURE) }
+      @prose_blocks = Set.new
+      @card_classes = card_classes
+      @card_bases = @card_classes.map { |classes| base_classes(classes) }
+      walk(@body)
+    end
+
+    private
+
+    def shielded_keys
+      landmarks = @body.css(LANDMARKS).to_a
+      articles = @body.css("article")
+      landmarks << articles.first if articles.size == 1
+      landmarks.flat_map { |node| [node] + node.ancestors.to_a }.map { |node| Dom.key(node) }.compact.to_set
+    end
+
+    # Code is never furniture, whatever classes its highlighter gives tokens.
+    def walk(node)
+      node.element_children.each do |child|
+        next if Dom::CODE.include?(child.name)
+
+        if remove?(child)
+          child.remove
+        else
+          walk(child)
+        end
+      end
+    end
+
+    # Semantic chrome (nav, a page-level footer) goes even when it outweighs
+    # the content, as legal footnotes often do; class-name guesses do not.
+    def remove?(node)
+      return false if @shielded.include?(Dom.key(node))
+      # Side notes inside an article are content whatever their class says.
+      return false if node.name == "aside" && !aside_noise?(node)
+      return true if chrome_element?(node) || chrome_role?(node)
+      return false if dominant?(node)
+      return false unless advertising?(node) || chrome_label?(node) || class_noise?(node)
+
+      !inside_sentence?(node)
+    end
+
+    # A link or span in the middle of running text is part of that text,
+    # whatever its class says ("recommendation-inline-link", "tooltip-term").
+    def inside_sentence?(node)
+      return false unless PHRASING.include?(node.name) && node.parent
+
+      @stats.chars(node.parent) - @stats.chars(node) >= 20
+    end
+
+    def chrome_element?(node)
+      case node.name
+      when "nav" then true
+      when "footer" then !inside_sectioning?(node)
+      when "header" then site_header?(node)
+      when "aside" then aside_noise?(node)
+      else false
+      end
+    end
+
+    def chrome_role?(node)
+      role = node["role"]
+      return false unless role && Patterns::CHROME_ROLES.include?(role)
+
+      case role
+      when "complementary" then aside_noise?(node)
+      when "banner" then site_header?(node)
+      when "alert", "status" then Dom.char_count(node.content) < 300
+      when "tablist" then tab_strip?(node)
+      else true
+      end
+    end
+
+    # Sites also put role=tablist on the wrapper around the tabs and their
+    # panels, or on an FAQ accordion; only a strip of short labels or links
+    # is chrome.
+    def tab_strip?(node)
+      return false if node.at_css("[role=tabpanel]")
+
+      @stats.text_chars(node) < 200 || @stats.link_density(node) > 0.5
+    end
+
+    def advertising?(node)
+      return true if node.name == "ins" && node["class"].to_s.include?("adsbygoogle")
+      return true if node["id"].to_s.start_with?("div-gpt-ad", "google_ads")
+
+      AD_ATTRIBUTES.any? { |name| node.key?(name) }
+    end
+
+    def chrome_label?(node)
+      return true if node["itemtype"].to_s.match?(Patterns::CHROME_ITEMTYPES)
+
+      node["aria-label"].to_s.match?(Patterns::ARIA_LABEL) && !%w[main article].include?(node.name)
+    end
+
+    def class_noise?(node)
+      return false if !@prose_blocks.empty? && prose_part?(node)
+
+      furniture_class?(node) && (@card_bases.empty? || !card?(node))
+    end
+
+    def card?(node)
+      classes = base_classes(node["class"])
+      @card_bases.any? { |base| (base - classes).empty? } && card_candidate?(node)
+    end
+
+    def base_classes(classes)
+      classes.to_s.split.reject { |token| token.include?("--") }
+    end
+
+    def prose_part?(node)
+      node["class"].to_s.split.any? { |token| @prose_blocks.any? { |block| token.start_with?("#{block}_", "#{block}-") } }
+    end
+
+    def furniture_class?(node)
+      signature = Dom.signature(node)
+      return false if signature.empty?
+
+      chrome = @furniture[signature]
+      return site_header?(node) if !chrome && signature.match?(Patterns::HEADER)
+      return @stats.link_density(node) > 0.5 if !chrome && signature.match?(Patterns::MENU)
+      return false unless chrome
+      return false if prose_section?(node, signature)
+
+      !(signature.match?(Patterns::POSITIVE) && @stats.text_chars(node) >= 200)
+    end
+
+    # Sections named after the TOC entry or skip link that points to them
+    # ("toc-para", "skip-item") and accordions named "drawer" like slide-out
+    # panels are content when they hold prose. Furniture named for its place
+    # on the page or for an overlay, ad or share bar is not.
+    def prose_section?(node, signature)
+      return false if @stats.text_chars(node) < 400 || @stats.link_density(node) >= 0.2
+      return false if signature.match?(Patterns::PLACE) || signature.gsub(DRAWER, " ").match?(Patterns::NOISE)
+      return false if node.css("p").size < 2
+
+      @prose_blocks.merge(node["class"].to_s.split.select { |token| token.downcase.match?(Patterns::PAGE_FURNITURE) })
+      true
+    end
+
+    # A hub page often splits its content into modules named after their role
+    # elsewhere ("recirc-list", "hide-read-more-ad"). None holds enough of the
+    # page to be dominant, so a run of them is judged as a whole. Card text
+    # is link text, so all characters count.
+    def card_classes
+      min_chars = @stats.chars(@body) * CARD_RUN_SHARE
+      runs = @body.css("[class]").group_by { |node| node["class"] }
+      runs.each_with_object(Set.new) do |(classes, nodes), cards|
+        cards << classes if card_run?(classes, nodes, min_chars)
+      end
+    end
+
+    def card_run?(classes, nodes, min_chars)
+      # Most class lists fail on size or name alone, before the slower checks below.
+      return false if nodes.size < MIN_CARD_RUN || !(furniture_name?(classes) || nodes.any? { |node| node.key?("id") })
+      return false if nodes.sum { |node| @stats.chars(node) } < min_chars
+
+      cards = nodes.select { |node| card_candidate?(node) }
+      # Wrappers repeating their child's class would count its text twice.
+      cards.reject! { |node| node.ancestors.any? { |ancestor| ancestor.element? && ancestor["class"] == classes } }
+      cards.size >= MIN_CARD_RUN && cards.sum { |node| @stats.chars(node) } >= min_chars
+    end
+
+    def furniture_name?(classes)
+      signature = Dom.signature_of("#{classes} ")
+      @furniture[signature] || signature.match?(Patterns::HEADER) || signature.match?(Patterns::MENU)
+    end
+
+    # Menus repeat their items too, so a run named after a place is never
+    # content, and neither is a run of cookie categories or dialogs.
+    def card_candidate?(node)
+      return false if @shielded.include?(Dom.key(node)) || !furniture_class?(node)
+      return false if Dom.signature(node).match?(Patterns::PLACE) || Dom.signature(node).match?(Patterns::OVERLAY)
+
+      !chrome_element?(node) && !chrome_role?(node)
+    end
+
+    # Per the HTML spec a header inside sectioning content belongs to that
+    # section; outside, one holding the page title (h1, few links, no
+    # navigation) is still content.
+    def site_header?(node)
+      return false if inside_sectioning?(node) || dominant?(node)
+
+      title_block = node.at_css("h1") && @stats.link_density(node) < 0.5 && !node.at_css("nav, [role=navigation], [role=search]")
+      !title_block
+    end
+
+    def aside_noise?(node)
+      !(inside_content?(node) && @stats.link_density(node) < 0.5)
+    end
+
+    def inside_sectioning?(node)
+      node.ancestors.any? { |ancestor| ancestor.element? && (SECTIONING.include?(ancestor.name) || ancestor["role"] == "main") }
+    end
+
+    def inside_content?(node)
+      node.ancestors.any? do |ancestor|
+        ancestor.element? && (CONTENT_ANCESTORS.include?(ancestor.name) || ancestor["role"] == "main" ||
+          ancestor["itemprop"].to_s.include?("articleBody"))
+      end
+    end
+
+    def dominant?(node)
+      text = @stats.text_chars(node)
+      text >= 200 && text >= @total * 0.5
+    end
+  end
+end

data/lib/nokodown/charset.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Nokodown
+  # Lexbor reads bytes as UTF-8 and ignores <meta charset>, so HTML is
+  # transcoded first. Invalid bytes are dropped: they would make every regex
+  # downstream raise.
+  module Charset
+    META_CHARSET = /<meta[^>]+charset\s*=\s*["']?\s*([\w-]+)/i.freeze
+
+    module_function
+
+    def utf8(html)
+      html = html.dup.force_encoding(declared(html) || Encoding::UTF_8) if html.encoding == Encoding::BINARY
+      html = html.encode(Encoding::UTF_8, invalid: :replace, undef: :replace, replace: "") if html.encoding != Encoding::UTF_8
+      html.valid_encoding? ? html : html.scrub("")
+    end
+
+    def declared(html)
+      charset = html.byteslice(0, 4096)[META_CHARSET, 1]
+      return unless charset
+
+      Encoding.find(charset.sub(/\Autf8\z/i, "UTF-8"))
+    rescue ArgumentError
+      nil
+    end
+  end
+end

data/lib/nokodown/container_check.rb

@@ -0,0 +1,174 @@
+# frozen_string_literal: true
+
+module Nokodown
+  # Readability's conditional cleaning for one container: what it holds (link
+  # density, images per paragraph, inputs, embeds) decides whether it is prose
+  # or page furniture.
+  class ContainerCheck
+    HEADINGS = "h1, h2, h3, h4, h5, h6"
+    TEXTISH = "span, li, td, blockquote, dl, div, img, ol, p, pre, table, ul"
+    AD_WORDS = /\A(?:ad(?:vertising|vertisement)?|pub(?:licité)?|werb(?:ung)?|广告|Реклама|Anuncio)\z/i.freeze
+    LOADING_WORDS = /\A(?:(?:loading|正在加载|Загрузка|chargement|cargando)(?:…|\.\.\.)?)\z/i.freeze
+    POST_BODY_SHARE = 0.8
+
+    def initialize(node, content_length, paragraph_keys, memo = {})
+      @node = node
+      @text = Dom.text(node)
+      @content_length = content_length
+      @paragraph_keys = paragraph_keys
+      @memo = memo
+    end
+
+    def self.forget(memo, node)
+      memo.delete(:tables) if node.name == "table" || node.at_css("table") || Dom.ancestors_named(node, "table").any?
+      memo.delete(:posts) if Dom.thread?(node)
+    end
+
+    def remove?
+      return false if majority? || forum_post?
+
+      tables, code = Dom.ancestors_named(@node, "table", "code", "pre").partition { |ancestor| ancestor.name == "table" }
+      return false if code.any? || data_table_involved?(tables)
+      return true if weight.negative?
+      return false if @text.count(",") >= 10 || video_embed?
+      return true if @text.match?(AD_WORDS) || @text.match?(LOADING_WORDS)
+
+      suspicious? && !image_gallery?
+    end
+
+    private
+
+    # Page builders (Elementor, Divi) wrap whole articles in "widget" and
+    # "module" containers; class weight alone must not delete the article.
+    def majority?
+      @text.length > @content_length * 0.5
+    end
+
+    def weight
+      @weight ||= Scorer.cleaning_weight(@node)
+    end
+
+    def data_table_involved?(outer_tables)
+      return true if (@node.name == "table" && data_table?(@node)) || holders_of_data_tables.include?(Dom.key(@node))
+
+      outer_tables.any? { |table| data_table?(table) }
+    end
+
+    def holders_of_data_tables
+      tables = (@memo[:tables] ||= {})
+      tables[:holders] ||= @memo.fetch(:root, @node).css("table").each_with_object(Set.new) do |table, keys|
+        next unless data_table?(table)
+
+        keys << Dom.key(table)
+        table.ancestors.each { |ancestor| keys << Dom.key(ancestor) if ancestor.element? }
+      end
+    end
+
+    def data_table?(table)
+      verdicts = ((@memo[:tables] ||= {})[:verdicts] ||= {})
+      key = Dom.key(table)
+      verdicts.key?(key) ? verdicts[key] : (verdicts[key] = Tables.data_table?(table))
+    end
+
+    def post_length(post)
+      lengths = (@memo[:posts] ||= {})
+      lengths[Dom.key(post)] ||= Dom.text(post).length
+    end
+
+    def forum_post?
+      return false unless Dom.thread?(@node)
+      return true if Dom.thread_post?(@node)
+
+      post = @node.ancestors.find { |ancestor| ancestor.element? && Dom.thread_post?(ancestor) }
+      !post.nil? && @text.length >= post_length(post) * POST_BODY_SHARE
+    end
+
+    def video_embed?
+      @node.css("object, embed, iframe").any? do |embed|
+        embed.attribute_nodes.any? { |attribute| attribute.value.to_s.match?(Preprocessor::VIDEO_HOSTS) }
+      end
+    end
+
+    def suspicious?
+      too_many_images? || too_many_list_items? || too_many_inputs? || short_with_links? ||
+        too_linky? || suspicious_embeds? || no_text_containers?
+    end
+
+    def too_many_images?
+      !figure_child? && images > 1 && paragraphs.fdiv(images) < 0.5
+    end
+
+    def too_many_list_items?
+      !list? && @node.css("li").size - 100 > paragraphs
+    end
+
+    def too_many_inputs?
+      @node.css("input").size > paragraphs / 3
+    end
+
+    def short_with_links?
+      !list? && !figure_child? && density(HEADINGS) < 0.9 && @text.length < 25 &&
+        (images.zero? || images > 2) && link_density.positive?
+    end
+
+    def too_linky?
+      (!list? && weight < 25 && link_density > 0.2) || (weight >= 25 && link_density > 0.5)
+    end
+
+    def suspicious_embeds?
+      embeds = @node.css("object, embed, iframe").size
+      (embeds == 1 && @text.length < 75) || embeds > 1
+    end
+
+    def no_text_containers?
+      images.zero? && density(TEXTISH, own_text_length).zero?
+    end
+
+    # Galleries survive as long as every list item is a single image.
+    def image_gallery?
+      list? && @node.element_children.none? { |item| item.element_children.size > 1 } && images == @node.css("li").size
+    end
+
+    def paragraphs
+      @paragraphs ||= @node.css("p").size + @node.css("div").count { |div| @paragraph_keys.include?(Dom.key(div)) }
+    end
+
+    def images
+      @images ||= @node.css("img").size
+    end
+
+    def list?
+      return @list if defined?(@list)
+
+      @list = %w[ul ol].include?(@node.name) || density("ul, ol") > 0.9
+    end
+
+    def figure_child?
+      return @figure_child if defined?(@figure_child)
+
+      @figure_child = Dom.ancestors_named(@node, "figure").any?
+    end
+
+    def link_density
+      @link_density ||= begin
+        linked = @node.css("a").sum do |anchor|
+          weight = anchor["href"].to_s.start_with?("#") ? Stats::FRAGMENT_LINK_WEIGHT : 1.0
+          Dom.text(anchor).length * weight
+        end
+        @text.empty? ? 0.0 : linked / @text.length
+      end
+    end
+
+    def density(selector, extra = 0)
+      return 0.0 if @text.empty?
+
+      (@node.css(selector).sum { |child| Dom.text(child).length } + extra).fdiv(@text.length)
+    end
+
+    # Readability wraps a container's loose text in <p>, which then counts as text.
+    def own_text_length
+      loose = @node.children.reject { |child| child.element? && Dom::PARAGRAPH_BREAKERS.include?(child.name) }
+      Dom.squish(loose.map(&:content).join).length
+    end
+  end
+end

data/lib/nokodown/content_locator.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+module Nokodown
+  # Decides what part of the cleaned page is the content.
+  #
+  # Pages with one dominant body of prose (articles, posts, documentation)
+  # are narrowed to that body using Readability's scoring. Everything else
+  # (home pages, listings, product pages, pricing tables) keeps the whole main
+  # region, because there the content is spread across many small blocks and
+  # picking a single winner would throw most of it away.
+  class ContentLocator
+    Selection = Struct.new(:nodes, :mode)
+
+    MIN_ARTICLE_CHARS = 250
+
+    # Share of the page's prose the article candidate must hold, by the page
+    # type its metadata declares. A declared article only needs a real body.
+    REQUIRED_SHARE = { article: 0.15, product: 0.9 }.freeze
+    DEFAULT_SHARE = 0.7
+
+    # Page types whose replies are content: forum threads and Q&A pages.
+    DISCUSSIONS = %i[forum qa].freeze
+
+    # Share of a forum page that one comment-named container, or a run of
+    # same-named ones, must hold to be the thread itself.
+    THREAD_SHARE = 0.25
+    THREAD_RUN_SHARE = 0.5
+
+    def initialize(body, page_type: nil)
+      @body = body
+      @page_type = page_type
+    end
+
+    def call
+      stats = Stats.new(@body)
+      root = main_region(stats) || @body
+      mark_thread(root, stats) if @page_type == :forum
+      result = Scorer.new(root, stats).call
+      return Selection.new(result.nodes, :article) if result && article?(result, root, stats)
+
+      # A thread's posts score alike, so no candidate holds it, and the replies'
+      # container alone would leave out the question. Cleaning the whole region
+      # as an article still drops per-answer comments, badges and chrome.
+      return Selection.new([root], :article) if DISCUSSIONS.include?(@page_type)
+
+      Selection.new([root], :page)
+    end
+
+    private
+
+    # The outermost <main> holding a fair share of the page, if any.
+    def main_region(stats)
+      regions = @body.css("main, [role=main]").reject do |node|
+        node.ancestors.any? { |ancestor| ancestor.element? && (ancestor.name == "main" || ancestor["role"] == "main") }
+      end
+      best = regions.max_by { |node| stats.text_chars(node) }
+      best if best && stats.text_chars(best) >= stats.text_chars(@body) * 0.4
+    end
+
+    def article?(result, root, stats)
+      article_text = result.nodes.sum { |node| stats.text_chars(node) }
+      return false if article_text < MIN_ARTICLE_CHARS
+
+      page_text = stats.text_chars(root) - secondary_text(root, result.nodes, stats)
+      return true if page_text <= 0
+
+      article_text / page_text >= REQUIRED_SHARE.fetch(@page_type, DEFAULT_SHARE)
+    end
+
+    # Comment threads and "most popular" rails beside an article do not count
+    # against it. A forum's own thread is not beside it.
+    def secondary_text(root, article_nodes, stats)
+      article_keys = article_nodes.map { |node| Dom.key(node) }
+      article_ancestor_keys = article_nodes.flat_map { |node| node.ancestors.map { |ancestor| Dom.key(ancestor) } }
+      total = 0.0
+      stack = root.element_children.to_a
+      until stack.empty?
+        node = stack.pop
+        key = Dom.key(node)
+        next if article_keys.include?(key) || Dom.thread?(node)
+
+        if Dom.signature(node).match?(Patterns::SECONDARY) && !article_ancestor_keys.include?(key)
+          total += stats.text_chars(node)
+        else
+          stack.concat(node.element_children.to_a)
+        end
+      end
+      total
+    end
+
+    # A forum's replies sit in containers named like a comment section: one
+    # holding a fair share of the page, or a run of them, one per post. Not on
+    # Q&A pages, where the comments under a short question pass the share.
+    def mark_thread(root, stats)
+      total = stats.text_chars(root)
+      regions = outermost_secondary(root)
+      thread = regions.select { |node| stats.text_chars(node) >= total * THREAD_SHARE }
+      runs = regions.group_by { |node| "#{node.name} #{Dom.signature(node).delete("0-9")}" }.values +
+             regions.group_by { |node| "#{node.name} #{Dom.key(node.parent)}" }.values
+      runs.each { |run| thread.concat(run) if run.sum { |node| stats.text_chars(node) } >= total * THREAD_RUN_SHARE }
+      thread.each { |node| Dom.mark_as_thread(node) }
+    end
+
+    def outermost_secondary(root)
+      found = []
+      stack = root.element_children.to_a
+      until stack.empty?
+        node = stack.pop
+        if Dom.signature(node).match?(Patterns::SECONDARY)
+          found << node
+        else
+          stack.concat(node.element_children.to_a)
+        end
+      end
+      found
+    end
+  end
+end

data/lib/nokodown/dom.rb

@@ -0,0 +1,140 @@
+# frozen_string_literal: true
+
+require "set"
+
+module Nokodown
+  module Dom
+    BLOCK = Set.new(%w[
+      address article aside blockquote body caption center dd details dialog dir div dl dt
+      fieldset figcaption figure footer form h1 h2 h3 h4 h5 h6 header hgroup hr html iframe
+      legend li main menu nav ol p pre section summary table tbody td tfoot th thead tr ul
+    ]).freeze
+
+    HEADINGS = Set.new(%w[h1 h2 h3 h4 h5 h6]).freeze
+
+    # Elements that stop Readability from treating a div as a paragraph.
+    PARAGRAPH_BREAKERS = Set.new(%w[blockquote dl div img ol p pre table ul]).freeze
+
+    CODE = %w[pre code].freeze
+
+    NBSP = "\u00A0"
+    WHITESPACE = " \t\n\r\f\u00A0"
+
+    # Every character [[:space:]] matches. tr/squeeze is several times faster
+    # than a regex gsub on large subtrees.
+    UNICODE_SPACES = "\t\n\v\f\r\u0085\u00A0\u1680\u2000-\u200A\u2028\u2029\u202F\u205F\u3000"
+
+    # Nokolexbor returns a new Ruby object every time a node is read, so nodes
+    # cannot be hash keys. Elements get a numbered attribute to key on instead.
+    KEY = "data-nokodown-key"
+
+    UUID = /\h{8}-\h{4}-\h{4}-\h{4}-\h{12}/.freeze
+    CSS_MODULE_HASH = /__(?=[\w-]{0,4}[A-Z\d])[\w-]{5}(?![\w-])/.freeze
+
+    # On a forum, the replies sit in containers named like a comment section.
+    # ContentLocator marks them and everything inside them, so the mark
+    # survives the copy that article cleaning works on.
+    THREAD = "data-nokodown-thread"
+
+    module_function
+
+    def assign_keys(root)
+      root[KEY] = "0"
+      root.css("*").each_with_index { |node, index| node[KEY] = (index + 1).to_s }
+    end
+
+    def key(node)
+      node[KEY]
+    end
+
+    def mark_as_thread(node)
+      node[THREAD] = "1"
+      node.css("*").each { |descendant| descendant[THREAD] = "1" }
+    end
+
+    def thread?(node)
+      !node[THREAD].nil?
+    end
+
+    def thread_post?(node)
+      thread?(node) && signature(node).match?(Patterns::SECONDARY)
+    end
+
+    # Keys of the <p>s and of the divs Readability would rewrite into <p>
+    # (no block-level descendants), found in one bottom-up pass.
+    def paragraph_keys(root)
+      keys = Set.new
+      breaks = {}
+      root.traverse do |node|
+        next unless node.element?
+
+        key = key(node)
+        breaks[key] = node.element_children.any? { |child| PARAGRAPH_BREAKERS.include?(child.name) || breaks[key(child)] }
+        keys << key if node.name == "p" || (node.name == "div" && !breaks[key])
+      end
+      keys
+    end
+
+    # Nokolexbor answers ancestors(selector) by running CSS matching per
+    # ancestor, which dominated extraction time; compare tag names instead.
+    def ancestors_named(node, *names)
+      node.ancestors.select { |ancestor| ancestor.element? && names.include?(ancestor.name) }
+    end
+
+    # Syntax highlighters wrap tokens in spans whose class ("token comment",
+    # "tag") or whole text ("print", "email") reads like page furniture.
+    def in_code?(node)
+      CODE.include?(node.name) || ancestors_named(node, *CODE).any?
+    end
+
+    def block?(node)
+      node.element? && BLOCK.include?(node.name)
+    end
+
+    def heading?(node)
+      node.element? && HEADINGS.include?(node.name)
+    end
+
+    def char_count(string)
+      string.length - string.count(WHITESPACE)
+    end
+
+    def text(node)
+      squish(node.content)
+    end
+
+    def squish(string)
+      string.tr(UNICODE_SPACES, " ").squeeze(" ").strip
+    end
+
+    def signature(node)
+      classes = node["class"]
+      id = node["id"]
+      return "" if classes.nil? && id.nil?
+
+      signature_of("#{classes} #{id}")
+    end
+
+    def signature_of(value)
+      # Generated ids (Webflow "w-node-<uuid>") would otherwise read as words.
+      value = value.gsub(UUID, " ") if value.match?(UUID)
+      value = value.gsub(CSS_MODULE_HASH, "") if value.include?("__")
+      value = value.gsub(/([a-z\d])([A-Z])/, '\1-\2') if value.match?(/[a-z\d][A-Z]/)
+      value.downcase
+    end
+
+    # Readability's text similarity: share of B's distinct tokens that also occur in A.
+    def similarity(a, b)
+      tokens_a = tokenize(a)
+      tokens_b = tokenize(b)
+      return 0.0 if tokens_a.empty? || tokens_b.empty?
+
+      unique_b = tokens_b - tokens_a
+      1.0 - (unique_b.join(" ").length.to_f / tokens_b.join(" ").length)
+    end
+
+    def tokenize(string)
+      string.to_s.downcase.split(/[^\p{L}\p{N}]+/).reject(&:empty?)
+    end
+  end
+end