coelacanth 0.3.9 → 0.4.0

data/lib/coelacanth/extractor/listing_collector.rb

@@ -0,0 +1,270 @@
+# frozen_string_literal: true
+
+require_relative "utilities"
+
+module Coelacanth
+  class Extractor
+    # Identifies sidebar or inline news listings and returns link arrays.
+    class ListingCollector
+      CANDIDATE_SELECTOR = "aside, section, div, ul, ol, dl".freeze
+      MIN_ITEMS = 3
+      MIN_TITLE_LENGTH = 2
+
+      def call(document:, base_url: nil, primary_node: nil)
+        candidates = collect_candidates(document, base_url, primary_node)
+
+        candidates
+          .sort_by { |candidate| -candidate[:score] }
+          .reject { |candidate| candidate[:score] < minimum_score }
+          .first(3)
+          .map { |candidate| format_candidate(candidate) }
+      end
+
+      private
+
+      def collect_candidates(document, base_url, primary_node)
+        document.css(CANDIDATE_SELECTOR).filter_map do |node|
+          next if skip_node?(node, primary_node)
+
+          items = extract_items(node, base_url)
+          next if items.length < MIN_ITEMS
+
+          heading = heading_for(node)
+          score = score_node(node, items, heading)
+          next if score < minimum_score
+
+          { node: node, items: items, heading: heading, score: score }
+        end
+      end
+
+      def skip_node?(node, primary_node)
+        return true if nested_listing_container?(node)
+
+        return false unless primary_node
+        return false unless primary_node.respond_to?(:name)
+        return false if %w[body html].include?(primary_node.name)
+
+        node == primary_node ||
+          ancestor?(node, primary_node) ||
+          ancestor?(primary_node, node)
+      end
+
+      def nested_listing_container?(node)
+        Utilities.ancestors(node).any? do |ancestor|
+          Utilities.element?(ancestor) && LISTING_CONTAINER_TAGS.include?(ancestor.name)
+        end
+      end
+
+      LISTING_CONTAINER_TAGS = %w[aside section div ul ol dl].freeze
+
+      def ancestor?(node, candidate)
+        Utilities.ancestors(node).any? { |ancestor| ancestor == candidate }
+      end
+
+      def extract_items(node, base_url)
+        item_nodes = candidate_children(node)
+        return [] if item_nodes.empty?
+
+        item_nodes.filter_map do |child|
+          next unless contains_link?(child)
+
+          anchor = primary_anchor(child)
+          next unless anchor
+
+          title = normalize_text(anchor.text)
+          next if title.length < MIN_TITLE_LENGTH
+
+          href = anchor["href"].to_s.strip
+          next if href.empty?
+
+          url = base_url ? Utilities.absolute_url(base_url, href) : href
+          url ||= href
+
+          snippet = build_snippet(child, title)
+
+          item = { title: title, url: url }
+          item[:snippet] = snippet unless snippet.nil? || snippet.empty?
+          item
+        end.uniq { |item| [item[:title], item[:url]] }
+      end
+
+      def candidate_children(node)
+        direct_children = Utilities.element_children(node)
+        return [] if direct_children.empty?
+
+        anchor_children = direct_children.select { |child| contains_link?(child) }
+        return anchor_children if anchor_children.length >= MIN_ITEMS
+
+        groups = %w[li article div section p dd]
+
+        groups.each do |tag|
+          grouped = direct_children.select { |child| child.name == tag }
+          return grouped if grouped.length >= MIN_ITEMS
+        end
+
+        list_container = direct_children.find { |child| %w[ul ol dl].include?(child.name) }
+        return Utilities.element_children(list_container) if list_container
+
+        []
+      end
+
+      def contains_link?(node)
+        node.css("a[href]").any?
+      end
+
+      def primary_anchor(node)
+        anchors = node.css("a[href]")
+        anchors.max_by { |anchor| normalize_text(anchor.text).length }
+      end
+
+      def normalize_text(text)
+        text.to_s.gsub(/[\r\n\t]/, " ").squeeze(" ").strip
+      end
+
+      def build_snippet(node, title)
+        snippet_from_node_text(node, title) || metadata_context(node, title)
+      end
+
+      def snippet_from_node_text(node, title)
+        text = normalize_text(node.text)
+        snippet = text.sub(title, "").strip
+        snippet.empty? ? nil : truncate(snippet)
+      end
+
+      def metadata_context(node, title)
+        candidate = time_text(node) || preceding_metadata(node)
+        return nil if candidate.nil?
+
+        candidate = candidate.sub(title, "").strip
+        candidate.empty? ? nil : truncate(candidate)
+      end
+
+      def time_text(node)
+        node.css("time").filter_map { |time| normalize_text(time.text) }.find { |text| !text.empty? }
+      end
+
+      def preceding_metadata(node)
+        previous = Utilities.previous_element(node)
+        3.times do
+          break unless previous
+
+          text = normalize_text(previous.text)
+          return text unless text.empty?
+
+          previous = Utilities.previous_element(previous)
+        end
+
+        nil
+      end
+
+      def truncate(text)
+        return text if text.length <= 120
+
+        text[0...117] + "..."
+      end
+
+      def heading_for(node)
+        if (heading = node.at_css("h1, h2, h3, h4"))
+          return normalize_text(heading.text)
+        end
+
+        previous = Utilities.previous_element(node)
+        3.times do
+          break unless previous
+
+          return normalize_text(previous.text) if previous.name =~ /h[1-6]/
+          previous = Utilities.previous_element(previous)
+        end
+
+        nil
+      end
+
+      def score_node(node, items, heading)
+        structure_score = structural_score(node)
+        heading_score = heading ? 45 : 0
+        item_score = items.length * 40
+        density_score = Utilities.link_density(node) * 90
+        adjacency_score = sibling_sequence_bonus(node)
+        depth_penalty = Utilities.depth(node) * 5
+        length_penalty = long_text_penalty(node)
+
+        structure_score + heading_score + item_score + density_score + adjacency_score - depth_penalty - length_penalty
+      end
+
+      def structural_score(node)
+        children = candidate_children(node)
+        return 0 if children.empty?
+
+        dominant_tag, dominant_children = children.group_by(&:name).max_by { |_, nodes| nodes.length }
+        dominant_count = dominant_children.length
+
+        uniform_bonus = dominant_count == children.length ? 60 : 20
+        list_bonus = %w[ul ol dl].include?(node.name) ? 90 : 0
+        list_bonus += 45 if dominant_tag && %w[li dd].include?(dominant_tag)
+
+        distribution_bonus = distribution_consistency_bonus(children)
+
+        dominant_count * 12 + uniform_bonus + list_bonus + distribution_bonus
+      end
+
+      def distribution_consistency_bonus(children)
+        return 0 if children.length < MIN_ITEMS
+
+        lengths = children.map { |child| Utilities.text_length(child) }
+        average = lengths.sum.to_f / lengths.length
+        variance = lengths.map { |len| (len - average).abs }
+
+        variance.max <= 120 ? 40 : 10
+      end
+
+      def sibling_sequence_bonus(node)
+        siblings = Utilities.sibling_elements(node)
+        return 0 if siblings.empty?
+
+        index = siblings.index(node)
+        return 0 unless index
+
+        forward = 0
+        while (candidate = siblings[index + forward + 1]) && similar_structure?(node, candidate)
+          forward += 1
+        end
+
+        backward = 0
+        while index - backward - 1 >= 0 && (candidate = siblings[index - backward - 1]) && similar_structure?(node, candidate)
+          backward += 1
+        end
+
+        (forward + backward) * 15
+      end
+
+      def similar_structure?(node, other)
+        return false unless other
+
+        node_children = candidate_children(node)
+        other_children = candidate_children(other)
+        return false if node_children.empty? || other_children.empty?
+
+        node_children.first.name == other_children.first.name && node_children.length == other_children.length
+      end
+
+      def long_text_penalty(node)
+        children = candidate_children(node)
+        return 0 if children.empty?
+
+        overlong = children.count { |child| Utilities.text_length(child) > 280 }
+        overlong * 30
+      end
+
+      def minimum_score
+        180
+      end
+
+      def format_candidate(candidate)
+        {
+          heading: candidate[:heading],
+          items: candidate[:items]
+        }
+      end
+    end
+  end
+end

data/lib/coelacanth/extractor/markdown_renderer.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+module Coelacanth
+  class Extractor
+    # Converts a DOM node into a lightweight Markdown representation.
+    class MarkdownRenderer
+      def self.render(node)
+        new(node).render
+      end
+
+      def initialize(node)
+        @node = node
+      end
+
+      def render
+        return "" unless @node
+
+        lines = traverse(@node)
+        lines.compact.join("\n").gsub(/\n{3,}/, "\n\n")
+      end
+
+      private
+
+      def traverse(node, depth = 0)
+        return if node.nil?
+
+        if document_node?(node)
+          node.children.flat_map { |child| traverse(child, depth) }
+        elsif element_node?(node)
+          render_element(node, depth)
+        elsif text_node?(node)
+          text = node.text.to_s.strip
+          text.empty? ? nil : text
+        end
+      end
+
+      def render_element(node, depth)
+        case node.name
+        when "p"
+          [node.children.flat_map { |child| traverse(child, depth) }.join(" "), ""]
+        when "br"
+          "\n"
+        when "h1", "h2", "h3", "h4", "h5", "h6"
+          level = node.name.delete_prefix("h").to_i
+          heading = "#" * level + " " + inline_children(node, depth)
+          [heading, ""]
+        when "ul"
+          element_children(node).flat_map { |child| render_list_item(child, depth, "-") } + [""]
+        when "ol"
+          element_children(node).each_with_index.flat_map do |child, index|
+            render_list_item(child, depth, "#{index + 1}.")
+          end + [""]
+        when "li"
+          ["- #{inline_children(node, depth)}"]
+        when "strong", "b"
+          "**#{inline_children(node, depth)}**"
+        when "em", "i"
+          "*#{inline_children(node, depth)}*"
+        when "blockquote"
+          quote = node.children.flat_map { |child| traverse(child, depth + 1) }.compact
+          quote.map { |line| "> #{line}" } + [""]
+        when "pre", "code"
+          content = node.text
+          ["```", content.rstrip, "```", ""]
+        when "img"
+          alt = node["alt"].to_s.strip
+          src = node["src"].to_s.strip
+          ["![#{alt}](#{src})", ""]
+        else
+          node.children.flat_map { |child| traverse(child, depth) }
+        end
+      end
+
+      def inline_children(node, depth)
+        node.children.flat_map { |child| traverse(child, depth) }.join(" ").squeeze(" ").strip
+      end
+
+      def render_list_item(node, depth, marker)
+        text = inline_children(node, depth)
+        return [] if text.empty?
+
+        ["#{marker} #{text}"]
+      end
+
+      def document_node?(node)
+        return false unless node
+
+        (defined?(::Oga::XML::Document) && node.is_a?(::Oga::XML::Document)) ||
+          (node.respond_to?(:document?) && node.document?) ||
+          (node.respond_to?(:type) && node.type == :document)
+      end
+
+      def element_node?(node)
+        return false unless node
+
+        if node.respond_to?(:element?)
+          node.element?
+        elsif defined?(::Oga::XML::Element) && node.is_a?(::Oga::XML::Element)
+          true
+        elsif node.respond_to?(:type)
+          node.type == :element
+        else
+          false
+        end
+      end
+
+      def text_node?(node)
+        return false unless node
+
+        if node.respond_to?(:text?)
+          node.text?
+        elsif defined?(::Oga::XML::Text) && node.is_a?(::Oga::XML::Text)
+          true
+        elsif node.respond_to?(:type)
+          node.type == :text
+        else
+          false
+        end
+      end
+
+      def element_children(node)
+        return [] unless node.respond_to?(:children)
+
+        node.children.select { |child| element_node?(child) }
+      end
+    end
+  end
+end

data/lib/coelacanth/extractor/metadata_probe.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+require "json"
+require "oga"
+
+require_relative "utilities"
+
+module Coelacanth
+  class Extractor
+    # Attempts to pull article metadata such as JSON-LD and OpenGraph tags.
+    class MetadataProbe
+      ARTICLE_TYPES = %w[Article NewsArticle BlogPosting ReportageNewsArticle LiveBlogPosting].freeze
+
+      Result = Struct.new(
+        :title,
+        :node,
+        :published_at,
+        :byline,
+        :source_tag,
+        :confidence,
+        keyword_init: true
+      )
+
+      def call(doc:, url: nil)
+        from_jsonld(doc, url) || from_semantic_nodes(doc)
+      end
+
+      private
+
+      def from_jsonld(doc, url)
+        doc.css("script[type='application/ld+json']").each do |script|
+          next if script.text.strip.empty?
+
+          begin
+            payload = JSON.parse(script.text)
+          rescue JSON::ParserError
+            next
+          end
+
+          candidates = payload.is_a?(Array) ? payload : [payload]
+          candidates.each do |candidate|
+            next unless article_type?(candidate)
+
+            body = candidate["articleBody"].to_s.strip
+            next if body.empty?
+
+            node = Oga.parse_html("<article>#{body}</article>").at_css("article")
+            return Result.new(
+              title: candidate["headline"] || candidate["name"],
+              node: node,
+              published_at: Utilities.parse_time(candidate["datePublished"] || candidate["dateCreated"]),
+              byline: extract_author(candidate["author"]),
+              source_tag: :jsonld,
+              confidence: 0.9
+            )
+          end
+        end
+        nil
+      end
+
+      def from_semantic_nodes(doc)
+        node = doc.at_css("main, article, [role='main'], [itemprop='articleBody']")
+        return if node.nil?
+
+        Result.new(
+          title: title_from_meta(doc),
+          node: node,
+          published_at: published_at_from_meta(doc),
+          byline: byline_from_meta(doc),
+          source_tag: :semantic,
+          confidence: 0.82
+        )
+      end
+
+      def article_type?(candidate)
+        type = candidate["@type"]
+        Array(type).any? { |value| ARTICLE_TYPES.include?(value) }
+      end
+
+      def extract_author(author)
+        case author
+        when String
+          author
+        when Hash
+          author["name"]
+        when Array
+          author.map { |item| extract_author(item) }.compact.join(", ")
+        end
+      end
+
+      def title_from_meta(doc)
+        Utilities.meta_content(
+          doc,
+          "meta[property='og:title']",
+          "meta[name='twitter:title']",
+          "meta[name='title']"
+        ) || doc.at_css("title")&.text&.strip
+      end
+
+      def published_at_from_meta(doc)
+        Utilities.parse_time(
+          Utilities.meta_content(
+            doc,
+            "meta[property='article:published_time']",
+            "meta[name='pubdate']",
+            "meta[name='publish_date']",
+            "meta[name='date']"
+          )
+        )
+      end
+
+      def byline_from_meta(doc)
+        Utilities.meta_content(
+          doc,
+          "meta[name='author']",
+          "meta[property='article:author']"
+        )
+      end
+    end
+  end
+end

data/lib/coelacanth/extractor/normalizer.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require "oga"
+
+require_relative "utilities"
+
+module Coelacanth
+  class Extractor
+    # Sanitizes HTML and prepares an Oga document.
+    class Normalizer
+      REMOVABLE_SELECTORS = %w[style noscript iframe form nav].freeze
+
+      def call(html:, base_url: nil)
+        document = Oga.parse_html(html)
+        remove_noise(document)
+        normalize_images(document, base_url)
+        document
+      end
+
+      private
+
+      def remove_noise(document)
+        REMOVABLE_SELECTORS.each do |selector|
+          document.css(selector).each(&:remove)
+        end
+
+        document.css("script").each do |node|
+          next if node["type"].to_s.strip.casecmp("application/ld+json").zero?
+
+          node.remove
+        end
+      end
+
+      def normalize_images(document, base_url)
+        return unless base_url
+
+        document.css("img").each do |image|
+          src = image["src"].to_s.strip
+          next if src.empty?
+
+          absolute = Utilities.absolute_url(base_url, src)
+          image.set("src", absolute) if absolute
+        end
+      end
+    end
+  end
+end

data/lib/coelacanth/extractor/utilities.rb

@@ -0,0 +1,145 @@
+# frozen_string_literal: true
+
+require "time"
+require "uri"
+
+module Coelacanth
+  class Extractor
+    # Shared helpers for the extractor pipeline.
+    module Utilities
+      PUNCTUATION = %w[。 、 ． ・ . , ! ? ： ； ; :]
+
+      module_function
+
+      def text_length(node)
+        node&.text&.strip&.length.to_i
+      end
+
+      def link_text_length(node)
+        return 0 unless node
+
+        node.css("a").sum { |anchor| anchor.text.strip.length }
+      end
+
+      def punctuation_density(node)
+        length = text_length(node)
+        return 0.0 if length.zero?
+
+        count = node.text.chars.count { |char| PUNCTUATION.include?(char) }
+        count.to_f / length
+      end
+
+      def link_density(node)
+        length = text_length(node)
+        return 0.0 if length.zero?
+
+        link_text_length(node).to_f / length
+      end
+
+      def depth(node)
+        ancestors(node).length
+      end
+
+      def ancestors(node)
+        return [] unless node
+
+        if node.respond_to?(:ancestors)
+          Array(node.ancestors)
+        else
+          collect_ancestors(node)
+        end
+      end
+
+      def collect_ancestors(node)
+        ancestors = []
+        current = node
+
+        while current.respond_to?(:parent) && (current = current.parent)
+          ancestors << current
+        end
+
+        ancestors
+      end
+
+      def element?(node)
+        return false unless node
+
+        if node.respond_to?(:element?)
+          node.element?
+        elsif defined?(::Oga::XML::Element) && node.is_a?(::Oga::XML::Element)
+          true
+        elsif node.respond_to?(:type)
+          node.type == :element
+        else
+          false
+        end
+      end
+
+      def element_children(node)
+        return [] unless node.respond_to?(:children)
+
+        node.children.select { |child| element?(child) }
+      end
+
+      def sibling_elements(node)
+        parent = node.respond_to?(:parent) ? node.parent : nil
+        return [] unless parent
+
+        element_children(parent)
+      end
+
+      def previous_element(node)
+        siblings = sibling_elements(node)
+        index = siblings.index(node)
+        return unless index && index.positive?
+
+        siblings[index - 1]
+      end
+
+      def next_element(node)
+        siblings = sibling_elements(node)
+        index = siblings.index(node)
+        return unless index
+
+        siblings[index + 1]
+      end
+
+      def class_id_tokens(node)
+        tokens = []
+        tokens.concat(split_tokens(node[:class])) if node[:class]
+        tokens.concat(split_tokens(node[:id])) if node[:id]
+        tokens
+      end
+
+      def split_tokens(value)
+        value.to_s.split(/[\s_-]+/).map(&:downcase)
+      end
+
+      def meta_content(doc, *selectors)
+        selectors.each do |selector|
+          if (node = doc.at_css(selector))
+            return node["content"].to_s.strip unless node["content"].to_s.strip.empty?
+          end
+        end
+        nil
+      end
+
+      def parse_time(value)
+        return if value.nil? || value.empty?
+
+        Time.parse(value)
+      rescue ArgumentError
+        nil
+      end
+
+      def absolute_url(base_url, path)
+        return if path.nil? || path.empty?
+        return path if path =~ /^https?:/i
+
+        URI.join(base_url, path).to_s
+      rescue URI::Error
+        path
+      end
+    end
+  end
+end