coelacanth 0.3.9 → 0.4.0

data/lib/coelacanth/extractor/weak_ml_probe.rb

@@ -0,0 +1,136 @@
+# frozen_string_literal: true
+
+require "oga"
+
+require_relative "utilities"
+
+module Coelacanth
+  class Extractor
+    # Lightweight probabilistic scorer that emulates a learned classifier using heuristics.
+    class WeakMlProbe
+      Result = Struct.new(
+        :title,
+        :node,
+        :published_at,
+        :byline,
+        :source_tag,
+        :confidence,
+        keyword_init: true
+      )
+
+      BLOCK_SELECTOR = "article, main, section, div".freeze
+      TOKEN_WEIGHTS = {
+        "content" => 1.1,
+        "article" => 1.0,
+        "body" => 0.9,
+        "post" => 0.8,
+        "entry" => 0.75,
+        "text" => 0.6,
+        "story" => 0.6,
+        "blog" => 0.5,
+        "share" => -1.0,
+        "nav" => -1.3,
+        "footer" => -1.2,
+        "header" => -1.1,
+        "related" => -0.8
+      }.freeze
+
+      FEATURE_WEIGHTS = {
+        bias: -1.2,
+        text_length: 0.002,
+        link_density: -2.6,
+        punctuation_density: 1.8,
+        depth: -0.12,
+        token_score: 1.6
+      }.freeze
+
+      def call(doc:, url: nil)
+        candidates = doc.css(BLOCK_SELECTOR).map do |node|
+          evaluate(node)
+        end.compact
+
+        return if candidates.empty?
+
+        best = candidates.max_by { |candidate| candidate[:probability] }
+        return if best[:probability] < 0.45
+
+        Result.new(
+          title: title_from_meta(doc),
+          node: best[:node],
+          published_at: published_at_from_meta(doc),
+          byline: byline_from_meta(doc),
+          source_tag: :ml,
+          confidence: best[:probability].clamp(0.0, 0.9)
+        )
+      end
+
+      private
+
+      def evaluate(node)
+        text_length = Utilities.text_length(node)
+        return if text_length < 60
+
+        features = {
+          text_length: text_length,
+          link_density: Utilities.link_density(node),
+          punctuation_density: Utilities.punctuation_density(node),
+          depth: Utilities.depth(node),
+          token_score: token_score(node)
+        }
+
+        score = linear_combination(features)
+        probability = logistic(score)
+
+        { node: node, probability: probability }
+      end
+
+      def token_score(node)
+        Utilities.class_id_tokens(node).sum do |token|
+          TOKEN_WEIGHTS.fetch(token, 0.0)
+        end
+      end
+
+      def linear_combination(features)
+        FEATURE_WEIGHTS[:bias] +
+          FEATURE_WEIGHTS[:text_length] * features[:text_length] +
+          FEATURE_WEIGHTS[:link_density] * features[:link_density] +
+          FEATURE_WEIGHTS[:punctuation_density] * features[:punctuation_density] +
+          FEATURE_WEIGHTS[:depth] * features[:depth] +
+          FEATURE_WEIGHTS[:token_score] * features[:token_score]
+      end
+
+      def logistic(score)
+        1.0 / (1.0 + Math.exp(-score))
+      end
+
+      def title_from_meta(doc)
+        Utilities.meta_content(
+          doc,
+          "meta[property='og:title']",
+          "meta[name='twitter:title']",
+          "meta[name='title']"
+        ) || doc.at_css("title")&.text&.strip
+      end
+
+      def published_at_from_meta(doc)
+        Utilities.parse_time(
+          Utilities.meta_content(
+            doc,
+            "meta[property='article:published_time']",
+            "meta[name='pubdate']",
+            "meta[name='publish_date']",
+            "meta[name='date']"
+          )
+        )
+      end
+
+      def byline_from_meta(doc)
+        Utilities.meta_content(
+          doc,
+          "meta[name='author']",
+          "meta[property='article:author']"
+        )
+      end
+    end
+  end
+end

data/lib/coelacanth/extractor.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require_relative "extractor/normalizer"
+require_relative "extractor/metadata_probe"
+require_relative "extractor/heuristic_probe"
+require_relative "extractor/weak_ml_probe"
+require_relative "extractor/fallback_probe"
+require_relative "extractor/markdown_renderer"
+require_relative "extractor/image_collector"
+require_relative "extractor/listing_collector"
+
+module Coelacanth
+  # High-level API for extracting articles without site-specific selectors.
+  class Extractor
+    PipelineResult = Struct.new(
+      :title,
+      :node,
+      :published_at,
+      :byline,
+      :source_tag,
+      :confidence,
+      keyword_init: true
+    )
+
+    def call(html:, url: nil)
+      document = Normalizer.new.call(html: html, base_url: url)
+
+      [
+        [MetadataProbe.new, 0.85],
+        [HeuristicProbe.new, 0.75],
+        [WeakMlProbe.new, 0.70],
+        [FallbackProbe.new, 0.0]
+      ].each do |probe, threshold|
+        result = probe.call(doc: document, url: url)
+        next unless result
+
+        return build_response(result, document:, url:) if result.confidence.to_f >= threshold
+      end
+
+      build_response(
+        PipelineResult.new(node: document, source_tag: :none, confidence: 0.0),
+        document: document,
+        url: url
+      )
+    end
+
+    private
+
+    def build_response(result, document:, url:)
+      node = result.node
+      body_markdown = MarkdownRenderer.render(node)
+      body_markdown_list = body_markdown.to_s.split(/\n{2,}/).map { |segment| segment.strip }.reject(&:empty?)
+
+      {
+        title: result.title,
+        body_markdown: body_markdown,
+        body_markdown_list: body_markdown_list,
+        images: ImageCollector.new.call(node),
+        published_at: result.published_at,
+        byline: result.byline,
+        source: result.source_tag,
+        confidence: result.confidence,
+        listings: ListingCollector.new.call(document: document, base_url: url, primary_node: node)
+      }
+    end
+  end
+end

data/lib/coelacanth/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Coelacanth
-  VERSION = "0.3.9"
+  VERSION = "0.4.0"
 end

data/lib/coelacanth.rb

@@ -6,6 +6,7 @@ require_relative "coelacanth/client/base"
 require_relative "coelacanth/client/ferrum"
 require_relative "coelacanth/client/screenshot_one"
 require_relative "coelacanth/dom"
+require_relative "coelacanth/extractor"
 require_relative "coelacanth/redirect"
 require_relative "coelacanth/validator"
 require_relative "coelacanth/version"
@@ -20,9 +21,15 @@ module Coelacanth
     client_class = config.read("client") == "screenshot_one" ? Client::ScreenshotOne : Client::Ferrum
     @client = client_class.new(url)
     regular_url = Redirect.new.resolve_redirect(url)
+    response = Net::HTTP.get_response(URI.parse(regular_url))
+    html = response.body.to_s
+    html = html.force_encoding(Encoding::UTF_8)
+    html = html.encode(Encoding::UTF_8, invalid: :replace, undef: :replace)
+    extractor_result = Extractor.new.call(html: html, url: regular_url)
     {
-      dom: Dom.new.oga(regular_url),
+      dom: Dom.new.oga(regular_url, html: html),
       screenshot: @client.get_screenshot,
+      extraction: extractor_result,
     }
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: coelacanth
 version: !ruby/object:Gem::Version
-  version: 0.3.9
+  version: 0.4.0
 platform: ruby
 authors:
 - Yusuke
@@ -24,7 +24,6 @@ files:
 - CODE_OF_CONDUCT.md
 - Dockerfile
 - Gemfile
-- Gemfile.lock
 - LICENSE.txt
 - README.md
 - Rakefile
@@ -36,6 +35,16 @@ files:
 - lib/coelacanth/client/screenshot_one.rb
 - lib/coelacanth/configure.rb
 - lib/coelacanth/dom.rb
+- lib/coelacanth/extractor.rb
+- lib/coelacanth/extractor/fallback_probe.rb
+- lib/coelacanth/extractor/heuristic_probe.rb
+- lib/coelacanth/extractor/image_collector.rb
+- lib/coelacanth/extractor/listing_collector.rb
+- lib/coelacanth/extractor/markdown_renderer.rb
+- lib/coelacanth/extractor/metadata_probe.rb
+- lib/coelacanth/extractor/normalizer.rb
+- lib/coelacanth/extractor/utilities.rb
+- lib/coelacanth/extractor/weak_ml_probe.rb
 - lib/coelacanth/redirect.rb
 - lib/coelacanth/validator.rb
 - lib/coelacanth/version.rb

data/Gemfile.lock

@@ -1,103 +0,0 @@
-PATH
-  remote: .
-  specs:
-    coelacanth (0.3.5)
-
-GEM
-  remote: https://rubygems.org/
-  specs:
-    addressable (2.8.7)
-      public_suffix (>= 2.0.2, < 7.0)
-    ansi (1.5.0)
-    ast (2.4.3)
-    base64 (0.2.0)
-    bigdecimal (3.1.9)
-    concurrent-ruby (1.3.5)
-    crack (1.0.0)
-      bigdecimal
-      rexml
-    diff-lcs (1.6.1)
-    ferrum (0.17.1)
-      addressable (~> 2.5)
-      base64 (~> 0.2)
-      concurrent-ruby (~> 1.1)
-      webrick (~> 1.7)
-      websocket-driver (~> 0.7)
-    hashdiff (1.1.2)
-    json (2.12.2)
-    language_server-protocol (3.17.0.5)
-    lint_roller (1.1.0)
-    oga (3.4)
-      ast
-      ruby-ll (~> 2.1)
-    parallel (1.27.0)
-    parser (3.3.8.0)
-      ast (~> 2.4.1)
-      racc
-    prism (1.4.0)
-    public_suffix (6.0.2)
-    racc (1.8.1)
-    rainbow (3.1.1)
-    rake (13.2.1)
-    regexp_parser (2.10.0)
-    rexml (3.4.1)
-    rspec (3.13.0)
-      rspec-core (~> 3.13.0)
-      rspec-expectations (~> 3.13.0)
-      rspec-mocks (~> 3.13.0)
-    rspec-core (3.13.3)
-      rspec-support (~> 3.13.0)
-    rspec-expectations (3.13.3)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.13.0)
-    rspec-mocks (3.13.2)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.13.0)
-    rspec-support (3.13.2)
-    rubocop (1.75.7)
-      json (~> 2.3)
-      language_server-protocol (~> 3.17.0.2)
-      lint_roller (~> 1.1.0)
-      parallel (~> 1.10)
-      parser (>= 3.3.0.2)
-      rainbow (>= 2.2.2, < 4.0)
-      regexp_parser (>= 2.9.3, < 3.0)
-      rubocop-ast (>= 1.44.0, < 2.0)
-      ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 2.4.0, < 4.0)
-    rubocop-ast (1.44.1)
-      parser (>= 3.3.7.2)
-      prism (~> 1.4)
-    ruby-ll (2.1.3)
-      ansi
-      ast
-    ruby-progressbar (1.13.0)
-    unicode-display_width (3.1.4)
-      unicode-emoji (~> 4.0, >= 4.0.4)
-    unicode-emoji (4.0.4)
-    webmock (3.25.1)
-      addressable (>= 2.8.0)
-      crack (>= 0.3.2)
-      hashdiff (>= 0.4.0, < 2.0.0)
-    webrick (1.9.1)
-    websocket-driver (0.7.7)
-      base64
-      websocket-extensions (>= 0.1.0)
-    websocket-extensions (0.1.5)
-
-PLATFORMS
-  ruby
-  x86_64-linux
-
-DEPENDENCIES
-  base64 (~> 0.2.0)
-  coelacanth!
-  ferrum (~> 0.16)
-  oga (~> 3.4)
-  rake (~> 13.2)
-  rspec (~> 3.0)
-  rubocop (~> 1.75)
-  webmock (~> 3.25)
-
-BUNDLED WITH
-   2.6.7