nokodown 0.1.0

data/lib/nokodown/scorer.rb

@@ -0,0 +1,230 @@
+# frozen_string_literal: true
+
+module Nokodown
+  # Readability's paragraph scoring, run without rewriting the DOM: paragraphs
+  # push score up to their ancestors, the best ancestor wins, and siblings that
+  # look like part of the same article are merged back in.
+  class Scorer
+    Result = Struct.new(:top, :nodes, :score)
+
+    SCORED = Set.new(%w[p td pre section h2 h3 h4 h5 h6]).freeze
+    PARAGRAPH_MIN = 25
+    TOP_CANDIDATES = 5
+    COMMAS = /[,\u060C\uFE50\uFE10\uFE11\u2E41\u2E34\u2E32\uFF0C]/.freeze
+
+    def self.class_weight(node)
+      [node["class"], node["id"]].sum { |value| words_weight(value.to_s.downcase) }
+    end
+
+    # Conditional cleaning deletes whatever weighs negative, so it ignores CSS
+    # utility tokens. Candidate scoring keeps Readability's weight, which some
+    # pages need to pick the right candidate.
+    def self.cleaning_weight(node)
+      [node["class"], node["id"]].sum do |value|
+        tokens = value.to_s.downcase.split.reject { |token| utility_class?(token) }
+        words_weight(tokens.join(" "))
+      end
+    end
+
+    # "lg:overflow-hidden" is judged without its variant prefix.
+    def self.utility_class?(token)
+      token.rpartition(":").last.match?(Patterns::UTILITY_CLASS)
+    end
+
+    def self.words_weight(value)
+      return 0 if value.empty?
+
+      (value.match?(Patterns::POSITIVE) ? 25 : 0) - (value.match?(Patterns::NEGATIVE) ? 25 : 0)
+    end
+    private_class_method :utility_class?, :words_weight
+
+    def initialize(root, stats)
+      @root = root
+      @root_key = Dom.key(root)
+      @stats = stats
+      @scores = {}
+      @nodes = {}
+    end
+
+    def call
+      @paragraph_keys = Dom.paragraph_keys(@root)
+      score_paragraphs
+      return if @scores.empty?
+
+      @scores.each_key { |key| @scores[key] *= 1 - @stats.link_density(@nodes[key]) }
+      ranked = @scores.keys.max_by(TOP_CANDIDATES) { |key| @scores[key] }.map { |key| @nodes[key] }
+      top = climb(shared_ancestor(ranked) || ranked.first)
+      Result.new(top, merge_siblings(top), score(top) || 0.0)
+    end
+
+    private
+
+    def paragraph?(node)
+      @paragraph_keys.include?(Dom.key(node))
+    end
+
+    def score(node)
+      @scores[Dom.key(node)]
+    end
+
+    def score_paragraphs
+      ignored = secondary_keys
+      @root.traverse do |node|
+        next unless node.element?
+        next if ignored.include?(Dom.key(node))
+
+        if SCORED.include?(node.name) || paragraph?(node)
+          add_score(Dom.text(node), scoring_parent(node))
+        elsif node.name == "div"
+          phrasing_runs(node).each { |text| add_score(text, node) }
+        end
+      end
+    end
+
+    # Comment threads are prose too, and on news sites they often outscore the
+    # article. A thread holding most of the page (a forum) still counts.
+    def secondary_keys
+      limit = @stats.text_chars(@root) * 0.5
+      regions = @root.css("*").select do |node|
+        Dom.signature(node).match?(Patterns::SECONDARY) && @stats.text_chars(node) < limit
+      end
+      regions.flat_map { |region| [region] + region.css("*").to_a }.map { |node| Dom.key(node) }.to_set
+    end
+
+    # A div wrapping a single paragraph is that paragraph, as far as scoring goes.
+    def scoring_parent(node)
+      parent = node.parent
+      return parent unless node.name == "p" && parent&.name == "div" && parent.element_children.size == 1
+      return parent unless parent.children.all? { |child| child.element? || child.content.strip.empty? }
+
+      parent.parent
+    end
+
+    # Text sitting directly in a div between block children counts as paragraphs.
+    def phrasing_runs(div)
+      runs = [+""]
+      div.children.each do |child|
+        if child.element? && (Dom::PARAGRAPH_BREAKERS.include?(child.name) || Dom.block?(child))
+          runs << +""
+        else
+          runs.last << child.content
+        end
+      end
+      runs.map { |run| Dom.squish(run) }.reject(&:empty?)
+    end
+
+    def add_score(text, parent)
+      return if text.length < PARAGRAPH_MIN || parent.nil? || !within_root?(parent)
+
+      points = 1 + text.split(COMMAS, -1).size + [text.length / 100, 3].min
+      ancestors_from(parent).each_with_index do |ancestor, level|
+        key = Dom.key(ancestor)
+        @nodes[key] ||= ancestor
+        @scores[key] ||= initial_score(ancestor)
+        @scores[key] += points.to_f / divider(level)
+      end
+    end
+
+    def divider(level)
+      case level
+      when 0 then 1
+      when 1 then 2
+      else level * 3
+      end
+    end
+
+    def ancestors_from(parent)
+      list = []
+      current = parent
+      while current&.element? && list.size < 5
+        list << current
+        break if Dom.key(current) == @root_key
+
+        current = current.parent
+      end
+      list
+    end
+
+    def within_root?(node)
+      Dom.key(node) == @root_key || node.ancestors.any? { |ancestor| ancestor.element? && Dom.key(ancestor) == @root_key }
+    end
+
+    def initial_score(node)
+      self.class.class_weight(node) +
+        case node.name
+        when "div" then 5
+        when "pre", "td", "blockquote" then 3
+        when "address", "ol", "ul", "dl", "dd", "dt", "li", "form" then -3
+        when "h1", "h2", "h3", "h4", "h5", "h6", "th" then -5
+        else 0
+        end
+    end
+
+    # When several near-equal candidates share an ancestor, the content is
+    # split across them: a list of posts, or an article broken up by ads.
+    def shared_ancestor(ranked)
+      top_score = score(ranked.first)
+      return if top_score <= 0
+
+      peers = ranked.drop(1).select { |node| score(node) / top_score >= 0.75 }
+      return if peers.size < 3
+
+      peer_ancestor_keys = peers.map { |node| node.ancestors.map { |ancestor| Dom.key(ancestor) } }
+      climbable_ancestors(ranked.first).find do |ancestor|
+        key = Dom.key(ancestor)
+        peer_ancestor_keys.count { |keys| keys.include?(key) } >= 3
+      end
+    end
+
+    def climbable_ancestors(node)
+      node.ancestors.take_while { |ancestor| ancestor.element? && ancestor.name != "body" && within_root?(ancestor) }
+    end
+
+    # Readability walks up while the parent scores higher than its child: the
+    # content probably continues in the parent's other children.
+    def climb(top)
+      last = score(top) || 0.0
+      threshold = last / 3
+      climbable_ancestors(top).each do |parent|
+        parent_score = score(parent)
+        next unless parent_score
+        break if parent_score < threshold
+
+        return single_child_ancestor(parent) if parent_score > last
+
+        last = parent_score
+      end
+      single_child_ancestor(top)
+    end
+
+    def single_child_ancestor(node)
+      climbable_ancestors(node).each do |parent|
+        break unless parent.element_children.size == 1
+
+        node = parent
+      end
+      node
+    end
+
+    def merge_siblings(top)
+      return [top] if Dom.key(top) == @root_key || top.parent.nil?
+
+      top_score = score(top) || 0.0
+      threshold = [10, top_score * 0.2].max
+      top.parent.element_children.select do |sibling|
+        sibling == top || related_sibling?(sibling, top, top_score, threshold)
+      end
+    end
+
+    def related_sibling?(sibling, top, top_score, threshold)
+      same_class = !top["class"].to_s.empty? && sibling["class"] == top["class"]
+      sibling_score = score(sibling)
+      return true if sibling_score && sibling_score + (same_class ? top_score * 0.2 : 0) >= threshold
+      return false unless paragraph?(sibling)
+
+      text = Dom.text(sibling)
+      density = @stats.link_density(sibling)
+      (text.length > 80 && density < 0.25) || (text.length.between?(1, 80) && density.zero? && text.match?(/\.( |\z)/))
+    end
+  end
+end

data/lib/nokodown/stats.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+module Nokodown
+  # Character and link counts for every element under a root, computed in one
+  # pass. Counts are non-whitespace characters, so indentation and line breaks
+  # in the source do not skew them.
+  class Stats
+    FRAGMENT_LINK_WEIGHT = 0.3
+
+    def initialize(root)
+      @chars = {}
+      @link_chars = {}
+      measure(root, 0.0)
+    end
+
+    def chars(node)
+      lookup(@chars, node)
+    end
+
+    def link_chars(node)
+      lookup(@link_chars, node)
+    end
+
+    def text_chars(node)
+      chars(node) - link_chars(node)
+    end
+
+    def link_density(node)
+      total = chars(node)
+      total.zero? ? 0.0 : link_chars(node) / total
+    end
+
+    private
+
+    def lookup(table, node)
+      key = Dom.key(node)
+      measure(node, inherited_link_weight(node)) unless table.key?(key)
+      table[key]
+    end
+
+    def measure(node, link_weight)
+      chars = 0.0
+      link_chars = 0.0
+      node.children.each do |child|
+        if child.text? || child.cdata?
+          count = Dom.char_count(child.content)
+          chars += count
+          link_chars += count * link_weight
+        elsif child.element?
+          child_chars, child_link_chars = measure(child, link_weight_for(child, link_weight))
+          chars += child_chars
+          link_chars += child_link_chars
+        end
+      end
+      store(node, chars, link_chars)
+    end
+
+    def store(node, chars, link_chars)
+      key = Dom.key(node)
+      if key
+        @chars[key] = chars
+        @link_chars[key] = link_chars
+      end
+      [chars, link_chars]
+    end
+
+    def link_weight_for(node, inherited)
+      href = node.name == "a" && node["href"]
+      return inherited unless href
+
+      href.start_with?("#") ? FRAGMENT_LINK_WEIGHT : 1.0
+    end
+
+    def inherited_link_weight(node)
+      anchor = node.ancestors.find { |ancestor| ancestor.element? && ancestor.name == "a" && ancestor["href"] }
+      anchor ? link_weight_for(anchor, 0.0) : 0.0
+    end
+  end
+end

data/lib/nokodown/tables.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Nokodown
+  module Tables
+    module_function
+
+    # Data vs. layout tables, following the checks Firefox uses for
+    # accessibility (and Readability copies).
+    def data_table?(table)
+      return false if table["role"] == "presentation" || table["datatable"] == "0"
+      return true if table["summary"].to_s.strip != ""
+
+      caption = table.at_css("caption")
+      return true if caption && !caption.content.strip.empty?
+      return true if table.at_css("col, colgroup, tfoot, thead, th")
+      return false if table.at_css("table")
+
+      rows, columns = size(table)
+      return false if rows == 1 || columns == 1 || sparse?(table)
+      return true if rows >= 10 || columns > 4
+
+      rows * columns > 10
+    end
+
+    # Headerless grids with many blank cells (spacers, indentation) are layout,
+    # however many rows they have. Hacker News is the canonical example.
+    def sparse?(table)
+      cells = table.css("td")
+      blank = cells.count { |cell| cell.content.strip.empty? && cell.at_css("img").nil? }
+      blank > cells.size * 0.3
+    end
+
+    def size(table)
+      rows = 0
+      columns = 0
+      table.css("tr").each do |tr|
+        rowspan = tr["rowspan"].to_i
+        rows += rowspan.positive? ? rowspan : 1
+        cells = tr.css("td").sum do |td|
+          colspan = td["colspan"].to_i
+          colspan.positive? ? colspan : 1
+        end
+        columns = cells if cells > columns
+      end
+      [rows, columns]
+    end
+
+    def rows(table)
+      table.element_children.flat_map do |child|
+        case child.name
+        when "tr" then [child]
+        when "thead", "tbody", "tfoot" then child.element_children.select { |row| row.name == "tr" }
+        else []
+        end
+      end
+    end
+
+    def cells(row)
+      row.element_children.select { |cell| cell.name == "td" || cell.name == "th" }
+    end
+  end
+end

data/lib/nokodown/url_resolver.rb

@@ -0,0 +1,249 @@
+# frozen_string_literal: true
+
+require "uri"
+
+module Nokodown
+  # Resolves hrefs against the page URL (and <base>), tolerating the malformed
+  # URLs real pages contain, and shortens them to the address they point at:
+  # tracking parameters, click redirectors and image resize options cost
+  # tokens and tell a reader nothing.
+  class UrlResolver
+    # From Ruby 3.4 URI::DEFAULT_PARSER is the RFC 3986 parser, whose unescape is obsolete.
+    URI_PARSER = defined?(URI::RFC2396_PARSER) ? URI::RFC2396_PARSER : URI::DEFAULT_PARSER
+    TRACKING_PARAM = /\A(?:utm_[a-z_]+|fbclid|gclid|gclsrc|dclid|gbraid|wbraid|msclkid|yclid|igshid|mc_cid|mc_eid|_hsenc|_hsmi|mkt_tok|__s|
+                       _gl|_ga|srsltid|trk|icid|cmpid|hsCtaTracking|ref_|pd_rd_[a-z]+|pf_rd_[a-z]+)\z/ix.freeze
+    # Amazon session, search-rank and affiliate parameters; on other sites
+    # these names can matter (Microsoft fwlinks need linkid).
+    AMAZON_PARAM = /\A(?:ref|qid|sr|th|smid|psc|crid|dib|dib_tag|spLa|sprefix|linkCode|linkId|ascsubtag|camp|creative)\z/.freeze
+    AMAZON = /\A(?:www\.|smile\.)?amazon\.(?:com?\.)?[a-z]{2,3}\z/i.freeze
+    AMAZON_PRODUCT = %r{\A.*?/(?:dp|gp/product|gp/aw/d)/[A-Z0-9]{10}(?![A-Za-z0-9])}.freeze
+    # Bestseller links put the session id after the ref segment.
+    AMAZON_REF = %r{/ref=[^/]*(?:/\d{3}-\d{7}-\d{7})?\z}.freeze
+    REDIRECTOR = %r{
+      \A(?:www\.)?google\.(?:com?\.)?[a-z]{2,3}/url\z |
+      \A(?:www\.)?youtube\.com/redirect\z |
+      \Alm?\.facebook\.com/l\.php\z |
+      \.safelinks\.protection\.outlook\.com/?\z |
+      \Ago\.(?:skimresources|redirectingat)\.com/?\z |
+      \Aredirect\.viglink\.com/?\z |
+      \Aclick\.linksynergy\.com/ |
+      \A(?:www\.)?avantlink\.com/click\.php\z |
+      \A[\w-]+\.(?:pxf|sjv)\.io/c/ |
+      \A(?:www\.)?(?:anrdoezrs\.net|dpbolvw\.net|jdoqocy\.com|kqzyfj\.com|tkqlhce\.com)/click- |
+      \Asteamcommunity\.com/linkfilter/?\z
+    }xi.freeze
+    DESTINATION_PARAM = /\A(?:url|u|murl|q)\z/.freeze
+    MAX_REDIRECTS = 3
+    # Image CDN size, crop and format options; the default rendition shows the
+    # same picture.
+    PRESENTATION_PARAM = /\A(?:fit|crop|auto|dpr|fm|fmt|resize|resMode|op_usm|ixlib|impolicy)\z/.freeze
+    # Only a number is a size: Google and Bing thumbnails name the image in q.
+    SIZE_PARAM = /\A(?:w|h|width|height|q|quality|wid|hei|qlt|imwidth|imheight)\z/.freeze
+    NUMBER = /\A\d+(?:[.,]\d+)?\z/.freeze
+    EXTENSION = /\.[A-Za-z][A-Za-z0-9]{1,4}\z/.freeze
+    # Signed URLs stop working once any parameter changes.
+    SIGNATURE_PARAM = /\A(?:s|sig|sign|signature|token|__token__|hdnts|expires|x-amz-[a-z-]+|x-goog-[a-z-]+|key-pair-id|
+                        policy|hmac|hash|jwt|itok)\z/ix.freeze
+    NEXT_IMAGE = %r{/_(?:next|vercel)/image/?\z}.freeze
+    # Cloudinary (on any domain), Forem, Cloudflare and ShortPixel append the
+    # original to their path. Screenshot services and web archives do too, but
+    # they do not serve the image at that URL.
+    FETCH_PROXY = %r{
+      \A[^/]*(?:/[\w-]+)?/image/fetch/ |
+      \A[^/]*/(?:dynamic|cdn-cgi)/image/ |
+      \Asp-ao\.shortpixel\.ai/client/
+    }ix.freeze
+    EMBEDDED_URL = %r{/(https?(?::|%3A)(?:/|%2F){2}.+)\z}i.freeze
+    CDN_CGI_IMAGE = %r{\A/cdn-cgi/image/[^/]+(/.+)\z}.freeze
+    PRIVATE_BUCKET = /\Abucketeer-/i.freeze
+    WEB_URL = %r{\Ahttps?://}i.freeze
+    UNSAFE = %r{[^A-Za-z0-9\-._~:/?#@!$&'()*+,;=%]}.freeze
+
+    def initialize(url, base_href = nil)
+      @base = parse(url)
+      href = base_href.to_s.strip
+      @base = join(@base, href) || @base if !href.empty? && !href.start_with?("javascript:")
+    end
+
+    def resolve(href)
+      absolute(href) { |uri| clean_link(uri) }
+    end
+
+    def resolve_image(src)
+      absolute(src) { |uri| clean_image(uri) }
+    end
+
+    def same_page?(url)
+      return false unless @base
+
+      uri = parse(url)
+      return false unless uri
+
+      @page ||= strip_fragment(clean_link(@base))
+      uri.fragment && strip_fragment(uri) == @page
+    end
+
+    private
+
+    def absolute(href)
+      href = href.to_s.strip.delete("\t\n\r")
+      return if href.empty?
+      return href if href.match?(/\A(?:mailto|tel|data):/i)
+
+      uri = @base ? join(@base, href) : parse(href)
+      uri ? yield(uri).to_s : href
+    end
+
+    def parse(url)
+      return if url.nil? || url.to_s.strip.empty?
+
+      URI.parse(encode(url.to_s.strip))
+    rescue URI::Error, ArgumentError
+      nil
+    end
+
+    def join(base, href)
+      return parse(href) unless base
+
+      base.merge(encode(href))
+    rescue URI::Error, ArgumentError
+      nil
+    end
+
+    def encode(string)
+      string.gsub(UNSAFE) { |char| char.bytes.map { |byte| format("%%%02X", byte) }.join }
+    end
+
+    def clean_link(uri, redirects = 0)
+      target = redirect_target(uri) if redirects < MAX_REDIRECTS
+      return clean_link(target, redirects + 1) if target
+      return clean_amazon(uri) if uri.host.to_s.match?(AMAZON)
+
+      strip_tracking(uri)
+    end
+
+    def strip_tracking(uri)
+      reject_params(uri) { |key, _| key.match?(TRACKING_PARAM) }
+    end
+
+    def redirect_target(uri)
+      return unless "#{uri.host}#{uri.path}".match?(REDIRECTOR)
+
+      param_values(uri, DESTINATION_PARAM).map { |value| parse(value) }.find { |target| web?(target) }
+    end
+
+    def web?(uri)
+      uri.is_a?(URI::HTTP) && !uri.host.to_s.empty?
+    end
+
+    def clean_amazon(uri)
+      product = uri.path[AMAZON_PRODUCT]
+      uri = uri.dup
+      uri.path = product || uri.path.sub(AMAZON_REF, "")
+      return reject_params(uri) { |key, _| key != "tag" } if product
+
+      reject_params(uri) { |key, _| key.match?(TRACKING_PARAM) || key.match?(AMAZON_PARAM) }
+    end
+
+    def clean_image(uri)
+      return uri if signed_image?(uri)
+
+      embedded = embedded_image(uri)
+      return original_image(uri, embedded) if embedded
+
+      proxy?(uri) ? strip_tracking(uri) : strip_presentation(uri)
+    end
+
+    def proxy?(uri)
+      uri.query.to_s.split("&").any? { |pair| decode(param(pair).last).to_s.match?(WEB_URL) }
+    end
+
+    # A proxy needs its options, so it stays whole when its original is unusable
+    # or is the proxy itself. The original is not unwrapped in turn, because
+    # proxies can point at each other.
+    def original_image(proxy, embedded)
+      original = join(proxy, embedded) unless embedded.empty?
+      return strip_tracking(proxy) unless web?(original) && !same_resource?(original, proxy) && !original.host.match?(PRIVATE_BUCKET)
+
+      signed_image?(original) ? original : strip_presentation(original)
+    end
+
+    def same_resource?(uri, other)
+      uri.host == other.host && uri.path == other.path
+    end
+
+    def strip_presentation(uri)
+      extension = uri.path.to_s.match?(EXTENSION)
+      reject_params(uri) { |key, value| key.match?(TRACKING_PARAM) || presentation?(key, value, extension) }
+    end
+
+    # Twitter media has no file extension and is a 404 without its format.
+    def presentation?(key, value, extension)
+      return value.match?(NUMBER) if key.match?(SIZE_PARAM)
+      return extension if key == "format"
+
+      key.match?(PRESENTATION_PARAM) || (key == "v" && value.match?(/\A\d+\z/))
+    end
+
+    def signed_image?(uri)
+      uri.query.to_s.split("&").any? { |pair| param(pair).first.match?(SIGNATURE_PARAM) }
+    end
+
+    def embedded_image(uri)
+      path = uri.path.to_s
+      return param_values(uri, /\Aurl\z/).first.to_s if path.match?(NEXT_IMAGE)
+      return fetched_image(uri) if "#{uri.host}#{path}".match?(FETCH_PROXY) && path.match?(EMBEDDED_URL)
+
+      original = path[CDN_CGI_IMAGE, 1]
+      original && uri.query ? "#{original}?#{uri.query}" : original
+    end
+
+    # An encoded original ends where the proxy's query starts. A plain one was
+    # split in two by the parser, so that query is the original's own.
+    def fetched_image(uri)
+      embedded = uri.path[EMBEDDED_URL, 1]
+      return unescape(embedded).to_s if embedded.match?(/\Ahttps?%3A/i)
+
+      uri.query ? "#{embedded}?#{uri.query}" : embedded
+    end
+
+    def param_values(uri, key)
+      params = uri.query.to_s.split("&").map { |pair| param(pair) }
+      params.select { |name, _| name.match?(key) }.map { |_, value| decode(value) }.compact
+    end
+
+    def param(pair)
+      key, value = pair.split("=", 2)
+      [key.to_s, value.to_s]
+    end
+
+    def decode(value)
+      decoded = URI.decode_www_form_component(value)
+      decoded if decoded.valid_encoding?
+    rescue ArgumentError
+      nil
+    end
+
+    def unescape(value)
+      decoded = URI_PARSER.unescape(value)
+      decoded if decoded.valid_encoding?
+    end
+
+    def reject_params(uri)
+      return uri unless uri.query
+
+      pairs = uri.query.split("&")
+      kept = pairs.reject { |pair| yield(*param(pair)) }
+      return uri if kept.size == pairs.size
+
+      kept -= [""]
+      uri = uri.dup
+      uri.query = kept.empty? ? nil : kept.join("&")
+      uri
+    end
+
+    def strip_fragment(uri)
+      uri.to_s.sub(/#.*\z/, "")
+    end
+  end
+end

data/lib/nokodown/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Nokodown
+  VERSION = "0.1.0"
+end

data/lib/nokodown.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require "nokolexbor"
+
+require_relative "nokodown/version"
+require_relative "nokodown/charset"
+require_relative "nokodown/dom"
+require_relative "nokodown/stats"
+require_relative "nokodown/patterns"
+require_relative "nokodown/tables"
+require_relative "nokodown/url_resolver"
+require_relative "nokodown/metadata"
+require_relative "nokodown/preprocessor"
+require_relative "nokodown/boilerplate"
+require_relative "nokodown/product_reviews"
+require_relative "nokodown/scorer"
+require_relative "nokodown/content_locator"
+require_relative "nokodown/container_check"
+require_relative "nokodown/pruner"
+require_relative "nokodown/markdown/text"
+require_relative "nokodown/markdown/image"
+require_relative "nokodown/markdown/code_block"
+require_relative "nokodown/markdown/inline"
+require_relative "nokodown/markdown/redundant_images"
+require_relative "nokodown/markdown/table"
+require_relative "nokodown/markdown/renderer"
+require_relative "nokodown/markdown/cleanup"
+require_relative "nokodown/markdown/image_cleanup"
+require_relative "nokodown/result"
+require_relative "nokodown/extractor"
+
+module Nokodown
+  # Extracts the content of an HTML page as Markdown, plus page metadata.
+  #
+  #   result = Nokodown.extract(html, url: "https://example.com/post")
+  #   result.markdown
+  #   result.title
+  #
+  # Options:
+  #   url:          page URL, used to resolve relative links and images
+  #   main_content: true (default) keeps only the main content; false converts
+  #                 the whole page, removing only invisible elements and scripts
+  #   title:        prepend the page title as an H1 when the content has none;
+  #                 a page with no content gives empty Markdown, not the title
+  #   images:       include images
+  #   links:        keep links; false renders link text only
+  def self.extract(html, **options)
+    Extractor.new(html, **options).call
+  end
+
+  def self.to_markdown(html, **options)
+    extract(html, **options).markdown
+  end
+end