coelacanth 0.3.10 → 0.4.1

data/lib/coelacanth/extractor/image_collector.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Coelacanth
+  class Extractor
+    # Collects image metadata from the extracted DOM node.
+    class ImageCollector
+      def call(node)
+        return [] unless node
+
+        node.css("img").map do |image|
+          {
+            src: image["src"].to_s.strip,
+            alt: image["alt"].to_s.strip
+          }
+        end.reject { |entry| entry[:src].empty? }
+      end
+    end
+  end
+end

data/lib/coelacanth/extractor/markdown_listing_collector.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+require_relative "utilities"
+
+module Coelacanth
+  class Extractor
+    # Extracts structured listings from Markdown content.
+    class MarkdownListingCollector
+      LIST_ITEM_PATTERN = /\A(?:[-*+]|\d+\.)\s+/.freeze
+      HEADING_PATTERN = /\A#+\s*/.freeze
+      MIN_ITEMS = 3
+      MIN_TITLE_LENGTH = 2
+
+      def call(markdown:, base_url: nil)
+        return [] if markdown.to_s.strip.empty?
+
+        listings = []
+        current = nil
+        pending_heading = nil
+
+        finalize_current = lambda do
+          next unless current
+
+          if current[:items].length >= MIN_ITEMS
+            listings << { heading: current[:heading], items: current[:items] }
+          end
+
+          current = nil
+        end
+
+        markdown.each_line do |line|
+          stripped = line.strip
+
+          if stripped.empty?
+            finalize_current.call
+            next
+          end
+
+          if heading_line?(stripped)
+            finalize_current.call
+            pending_heading = normalize_heading(stripped)
+            next
+          end
+
+          if list_item_line?(stripped)
+            current ||= { heading: pending_heading, items: [] }
+            pending_heading = nil
+
+            if (item = build_item(stripped, base_url))
+              current[:items] << item
+            end
+          else
+            finalize_current.call
+            pending_heading = nil
+          end
+        end
+
+        finalize_current.call
+
+        listings
+      end
+
+      private
+
+      def heading_line?(line)
+        line.start_with?("#") && line.match?(HEADING_PATTERN)
+      end
+
+      def list_item_line?(line)
+        line.match?(LIST_ITEM_PATTERN)
+      end
+
+      def normalize_heading(line)
+        line.sub(HEADING_PATTERN, "").strip
+      end
+
+      def build_item(line, base_url)
+        content = line.sub(LIST_ITEM_PATTERN, "").strip
+        return if content.empty?
+
+        if (match = content.match(/\A\[([^\]]+)\]\(([^\)]+)\)(.*)\z/))
+          title = match[1].to_s.strip
+          href = match[2].to_s.strip
+          trailing = match[3].to_s.strip
+
+          return if title.length < MIN_TITLE_LENGTH
+
+          url = Utilities.absolute_url(base_url, href) || href
+          item = { title: title, url: url }
+
+          snippet = normalize_snippet(trailing)
+          item[:snippet] = snippet unless snippet.nil? || snippet.empty?
+          item
+        else
+          title = content
+          return if title.length < MIN_TITLE_LENGTH
+
+          { title: title }
+        end
+      end
+
+      def normalize_snippet(text)
+        stripped = text.to_s.sub(/\A[-–—:]\s*/, "").strip
+        stripped.empty? ? nil : stripped
+      end
+    end
+  end
+end

data/lib/coelacanth/extractor/markdown_renderer.rb

@@ -0,0 +1,132 @@
+# frozen_string_literal: true
+
+module Coelacanth
+  class Extractor
+    # Converts a DOM node into a lightweight Markdown representation.
+    class MarkdownRenderer
+      def self.render(node)
+        new(node).render
+      end
+
+      def initialize(node)
+        @node = node
+      end
+
+      def render
+        return "" unless @node
+
+        lines = traverse(@node)
+        lines.compact.join("\n").gsub(/\n{3,}/, "\n\n")
+      end
+
+      private
+
+      def traverse(node, depth = 0)
+        return if node.nil?
+
+        if document_node?(node)
+          node.children.flat_map { |child| traverse(child, depth) }
+        elsif element_node?(node)
+          render_element(node, depth)
+        elsif text_node?(node)
+          text = node.text.to_s.strip
+          text.empty? ? nil : text
+        end
+      end
+
+      def render_element(node, depth)
+        case node.name
+        when "p"
+          [node.children.flat_map { |child| traverse(child, depth) }.join(" "), ""]
+        when "br"
+          "\n"
+        when "h1", "h2", "h3", "h4", "h5", "h6"
+          level = node.name.delete_prefix("h").to_i
+          heading = "#" * level + " " + inline_children(node, depth)
+          [heading, ""]
+        when "ul"
+          element_children(node).flat_map { |child| render_list_item(child, depth, "-") } + [""]
+        when "ol"
+          element_children(node).each_with_index.flat_map do |child, index|
+            render_list_item(child, depth, "#{index + 1}.")
+          end + [""]
+        when "li"
+          ["- #{inline_children(node, depth)}"]
+        when "a"
+          href = node["href"].to_s.strip
+          text = inline_children(node, depth)
+          href.empty? ? text : "[#{text}](#{href})"
+        when "strong", "b"
+          "**#{inline_children(node, depth)}**"
+        when "em", "i"
+          "*#{inline_children(node, depth)}*"
+        when "blockquote"
+          quote = node.children.flat_map { |child| traverse(child, depth + 1) }.compact
+          quote.map { |line| "> #{line}" } + [""]
+        when "pre", "code"
+          content = node.text
+          ["```", content.rstrip, "```", ""]
+        when "img"
+          alt = node["alt"].to_s.strip
+          src = node["src"].to_s.strip
+          ["![#{alt}](#{src})", ""]
+        else
+          node.children.flat_map { |child| traverse(child, depth) }
+        end
+      end
+
+      def inline_children(node, depth)
+        node.children.flat_map { |child| traverse(child, depth) }.join(" ").squeeze(" ").strip
+      end
+
+      def render_list_item(node, depth, marker)
+        text = inline_children(node, depth)
+        return [] if text.empty?
+
+        ["#{marker} #{text}"]
+      end
+
+      def document_node?(node)
+        return false unless node
+
+        (defined?(::Oga::XML::Document) && node.is_a?(::Oga::XML::Document)) ||
+          (node.respond_to?(:document?) && node.document?) ||
+          (node.respond_to?(:type) && node.type == :document)
+      end
+
+      def element_node?(node)
+        return false unless node
+
+        if node.respond_to?(:element?)
+          node.element?
+        elsif defined?(::Oga::XML::Element) && node.is_a?(::Oga::XML::Element)
+          true
+        elsif node.respond_to?(:type)
+          node.type == :element
+        else
+          false
+        end
+      end
+
+      def text_node?(node)
+        return false unless node
+
+        if node.respond_to?(:text?)
+          node.text?
+        elsif defined?(::Oga::XML::Text) && node.is_a?(::Oga::XML::Text)
+          true
+        elsif node.respond_to?(:type)
+          node.type == :text
+        else
+          false
+        end
+      end
+
+      def element_children(node)
+        return [] unless node.respond_to?(:children)
+
+        node.children.select { |child| element_node?(child) }
+      end
+    end
+  end
+end

data/lib/coelacanth/extractor/metadata_probe.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+require "json"
+require "oga"
+
+require_relative "utilities"
+
+module Coelacanth
+  class Extractor
+    # Attempts to pull article metadata such as JSON-LD and OpenGraph tags.
+    class MetadataProbe
+      ARTICLE_TYPES = %w[Article NewsArticle BlogPosting ReportageNewsArticle LiveBlogPosting].freeze
+
+      Result = Struct.new(
+        :title,
+        :node,
+        :published_at,
+        :byline,
+        :source_tag,
+        :confidence,
+        keyword_init: true
+      )
+
+      def call(doc:, url: nil)
+        from_jsonld(doc, url) || from_semantic_nodes(doc)
+      end
+
+      private
+
+      def from_jsonld(doc, url)
+        doc.css("script[type='application/ld+json']").each do |script|
+          next if script.text.strip.empty?
+
+          begin
+            payload = JSON.parse(script.text)
+          rescue JSON::ParserError
+            next
+          end
+
+          candidates = payload.is_a?(Array) ? payload : [payload]
+          candidates.each do |candidate|
+            next unless article_type?(candidate)
+
+            body = candidate["articleBody"].to_s.strip
+            next if body.empty?
+
+            node = Oga.parse_html("<article>#{body}</article>").at_css("article")
+            return Result.new(
+              title: candidate["headline"] || candidate["name"],
+              node: node,
+              published_at: Utilities.parse_time(candidate["datePublished"] || candidate["dateCreated"]),
+              byline: extract_author(candidate["author"]),
+              source_tag: :jsonld,
+              confidence: 0.9
+            )
+          end
+        end
+        nil
+      end
+
+      def from_semantic_nodes(doc)
+        node = doc.at_css("main, article, [role='main'], [itemprop='articleBody']")
+        return if node.nil?
+
+        Result.new(
+          title: title_from_meta(doc),
+          node: node,
+          published_at: published_at_from_meta(doc),
+          byline: byline_from_meta(doc),
+          source_tag: :semantic,
+          confidence: 0.82
+        )
+      end
+
+      def article_type?(candidate)
+        type = candidate["@type"]
+        Array(type).any? { |value| ARTICLE_TYPES.include?(value) }
+      end
+
+      def extract_author(author)
+        case author
+        when String
+          author
+        when Hash
+          author["name"]
+        when Array
+          author.map { |item| extract_author(item) }.compact.join(", ")
+        end
+      end
+
+      def title_from_meta(doc)
+        Utilities.meta_content(
+          doc,
+          "meta[property='og:title']",
+          "meta[name='twitter:title']",
+          "meta[name='title']"
+        ) || doc.at_css("title")&.text&.strip
+      end
+
+      def published_at_from_meta(doc)
+        Utilities.parse_time(
+          Utilities.meta_content(
+            doc,
+            "meta[property='article:published_time']",
+            "meta[name='pubdate']",
+            "meta[name='publish_date']",
+            "meta[name='date']"
+          )
+        )
+      end
+
+      def byline_from_meta(doc)
+        Utilities.meta_content(
+          doc,
+          "meta[name='author']",
+          "meta[property='article:author']"
+        )
+      end
+    end
+  end
+end

data/lib/coelacanth/extractor/normalizer.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require "oga"
+
+require_relative "utilities"
+
+module Coelacanth
+  class Extractor
+    # Sanitizes HTML and prepares an Oga document.
+    class Normalizer
+      REMOVABLE_SELECTORS = %w[style noscript iframe form nav].freeze
+
+      def call(html:, base_url: nil)
+        document = Oga.parse_html(html)
+        remove_noise(document)
+        normalize_images(document, base_url)
+        document
+      end
+
+      private
+
+      def remove_noise(document)
+        REMOVABLE_SELECTORS.each do |selector|
+          document.css(selector).each(&:remove)
+        end
+
+        document.css("script").each do |node|
+          next if node["type"].to_s.strip.casecmp("application/ld+json").zero?
+
+          node.remove
+        end
+      end
+
+      def normalize_images(document, base_url)
+        return unless base_url
+
+        document.css("img").each do |image|
+          src = image["src"].to_s.strip
+          next if src.empty?
+
+          absolute = Utilities.absolute_url(base_url, src)
+          image.set("src", absolute) if absolute
+        end
+      end
+    end
+  end
+end

data/lib/coelacanth/extractor/utilities.rb

@@ -0,0 +1,145 @@
+# frozen_string_literal: true
+
+require "time"
+require "uri"
+
+module Coelacanth
+  class Extractor
+    # Shared helpers for the extractor pipeline.
+    module Utilities
+      PUNCTUATION = %w[。 、 ． ・ . , ! ? ： ； ; :]
+
+      module_function
+
+      def text_length(node)
+        node&.text&.strip&.length.to_i
+      end
+
+      def link_text_length(node)
+        return 0 unless node
+
+        node.css("a").sum { |anchor| anchor.text.strip.length }
+      end
+
+      def punctuation_density(node)
+        length = text_length(node)
+        return 0.0 if length.zero?
+
+        count = node.text.chars.count { |char| PUNCTUATION.include?(char) }
+        count.to_f / length
+      end
+
+      def link_density(node)
+        length = text_length(node)
+        return 0.0 if length.zero?
+
+        link_text_length(node).to_f / length
+      end
+
+      def depth(node)
+        ancestors(node).length
+      end
+
+      def ancestors(node)
+        return [] unless node
+
+        if node.respond_to?(:ancestors)
+          Array(node.ancestors)
+        else
+          collect_ancestors(node)
+        end
+      end
+
+      def collect_ancestors(node)
+        ancestors = []
+        current = node
+
+        while current.respond_to?(:parent) && (current = current.parent)
+          ancestors << current
+        end
+
+        ancestors
+      end
+
+      def element?(node)
+        return false unless node
+
+        if node.respond_to?(:element?)
+          node.element?
+        elsif defined?(::Oga::XML::Element) && node.is_a?(::Oga::XML::Element)
+          true
+        elsif node.respond_to?(:type)
+          node.type == :element
+        else
+          false
+        end
+      end
+
+      def element_children(node)
+        return [] unless node.respond_to?(:children)
+
+        node.children.select { |child| element?(child) }
+      end
+
+      def sibling_elements(node)
+        parent = node.respond_to?(:parent) ? node.parent : nil
+        return [] unless parent
+
+        element_children(parent)
+      end
+
+      def previous_element(node)
+        siblings = sibling_elements(node)
+        index = siblings.index(node)
+        return unless index && index.positive?
+
+        siblings[index - 1]
+      end
+
+      def next_element(node)
+        siblings = sibling_elements(node)
+        index = siblings.index(node)
+        return unless index
+
+        siblings[index + 1]
+      end
+
+      def class_id_tokens(node)
+        tokens = []
+        tokens.concat(split_tokens(node[:class])) if node[:class]
+        tokens.concat(split_tokens(node[:id])) if node[:id]
+        tokens
+      end
+
+      def split_tokens(value)
+        value.to_s.split(/[\s_-]+/).map(&:downcase)
+      end
+
+      def meta_content(doc, *selectors)
+        selectors.each do |selector|
+          if (node = doc.at_css(selector))
+            return node["content"].to_s.strip unless node["content"].to_s.strip.empty?
+          end
+        end
+        nil
+      end
+
+      def parse_time(value)
+        return if value.nil? || value.empty?
+
+        Time.parse(value)
+      rescue ArgumentError
+        nil
+      end
+
+      def absolute_url(base_url, path)
+        return if path.nil? || path.empty?
+        return path if path =~ /^https?:/i
+
+        URI.join(base_url, path).to_s
+      rescue URI::Error
+        path
+      end
+    end
+  end
+end

data/lib/coelacanth/extractor/weak_ml_probe.rb

@@ -0,0 +1,136 @@
+# frozen_string_literal: true
+
+require "oga"
+
+require_relative "utilities"
+
+module Coelacanth
+  class Extractor
+    # Lightweight probabilistic scorer that emulates a learned classifier using heuristics.
+    class WeakMlProbe
+      Result = Struct.new(
+        :title,
+        :node,
+        :published_at,
+        :byline,
+        :source_tag,
+        :confidence,
+        keyword_init: true
+      )
+
+      BLOCK_SELECTOR = "article, main, section, div".freeze
+      TOKEN_WEIGHTS = {
+        "content" => 1.1,
+        "article" => 1.0,
+        "body" => 0.9,
+        "post" => 0.8,
+        "entry" => 0.75,
+        "text" => 0.6,
+        "story" => 0.6,
+        "blog" => 0.5,
+        "share" => -1.0,
+        "nav" => -1.3,
+        "footer" => -1.2,
+        "header" => -1.1,
+        "related" => -0.8
+      }.freeze
+
+      FEATURE_WEIGHTS = {
+        bias: -1.2,
+        text_length: 0.002,
+        link_density: -2.6,
+        punctuation_density: 1.8,
+        depth: -0.12,
+        token_score: 1.6
+      }.freeze
+
+      def call(doc:, url: nil)
+        candidates = doc.css(BLOCK_SELECTOR).map do |node|
+          evaluate(node)
+        end.compact
+
+        return if candidates.empty?
+
+        best = candidates.max_by { |candidate| candidate[:probability] }
+        return if best[:probability] < 0.45
+
+        Result.new(
+          title: title_from_meta(doc),
+          node: best[:node],
+          published_at: published_at_from_meta(doc),
+          byline: byline_from_meta(doc),
+          source_tag: :ml,
+          confidence: best[:probability].clamp(0.0, 0.9)
+        )
+      end
+
+      private
+
+      def evaluate(node)
+        text_length = Utilities.text_length(node)
+        return if text_length < 60
+
+        features = {
+          text_length: text_length,
+          link_density: Utilities.link_density(node),
+          punctuation_density: Utilities.punctuation_density(node),
+          depth: Utilities.depth(node),
+          token_score: token_score(node)
+        }
+
+        score = linear_combination(features)
+        probability = logistic(score)
+
+        { node: node, probability: probability }
+      end
+
+      def token_score(node)
+        Utilities.class_id_tokens(node).sum do |token|
+          TOKEN_WEIGHTS.fetch(token, 0.0)
+        end
+      end
+
+      def linear_combination(features)
+        FEATURE_WEIGHTS[:bias] +
+          FEATURE_WEIGHTS[:text_length] * features[:text_length] +
+          FEATURE_WEIGHTS[:link_density] * features[:link_density] +
+          FEATURE_WEIGHTS[:punctuation_density] * features[:punctuation_density] +
+          FEATURE_WEIGHTS[:depth] * features[:depth] +
+          FEATURE_WEIGHTS[:token_score] * features[:token_score]
+      end
+
+      def logistic(score)
+        1.0 / (1.0 + Math.exp(-score))
+      end
+
+      def title_from_meta(doc)
+        Utilities.meta_content(
+          doc,
+          "meta[property='og:title']",
+          "meta[name='twitter:title']",
+          "meta[name='title']"
+        ) || doc.at_css("title")&.text&.strip
+      end
+
+      def published_at_from_meta(doc)
+        Utilities.parse_time(
+          Utilities.meta_content(
+            doc,
+            "meta[property='article:published_time']",
+            "meta[name='pubdate']",
+            "meta[name='publish_date']",
+            "meta[name='date']"
+          )
+        )
+      end
+
+      def byline_from_meta(doc)
+        Utilities.meta_content(
+          doc,
+          "meta[name='author']",
+          "meta[property='article:author']"
+        )
+      end
+    end
+  end
+end