coelacanth 0.3.10 → 0.4.1

data/lib/coelacanth/extractor.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require_relative "extractor/normalizer"
+require_relative "extractor/metadata_probe"
+require_relative "extractor/heuristic_probe"
+require_relative "extractor/weak_ml_probe"
+require_relative "extractor/fallback_probe"
+require_relative "extractor/markdown_renderer"
+require_relative "extractor/image_collector"
+require_relative "extractor/markdown_listing_collector"
+
+module Coelacanth
+  # High-level API for extracting articles without site-specific selectors.
+  class Extractor
+    PipelineResult = Struct.new(
+      :title,
+      :node,
+      :published_at,
+      :byline,
+      :source_tag,
+      :confidence,
+      keyword_init: true
+    )
+
+    def call(html:, url: nil)
+      document = Normalizer.new.call(html: html, base_url: url)
+
+      [
+        [MetadataProbe.new, 0.85],
+        [HeuristicProbe.new, 0.75],
+        [WeakMlProbe.new, 0.70],
+        [FallbackProbe.new, 0.0]
+      ].each do |probe, threshold|
+        result = probe.call(doc: document, url: url)
+        next unless result
+
+        return build_response(result, document:, url:) if result.confidence.to_f >= threshold
+      end
+
+      build_response(
+        PipelineResult.new(node: document, source_tag: :none, confidence: 0.0),
+        document: document,
+        url: url
+      )
+    end
+
+    private
+
+    def build_response(result, document:, url:)
+      node = result.node
+      body_markdown = MarkdownRenderer.render(node)
+      body_markdown_list = body_markdown.to_s.split(/\n{2,}/).map { |segment| segment.strip }.reject(&:empty?)
+
+      {
+        title: result.title,
+        body_markdown: body_markdown,
+        body_markdown_list: body_markdown_list,
+        images: ImageCollector.new.call(node),
+        published_at: result.published_at,
+        byline: result.byline,
+        source: result.source_tag,
+        confidence: result.confidence,
+        listings: MarkdownListingCollector.new.call(markdown: body_markdown, base_url: url)
+      }
+    end
+  end
+end

data/lib/coelacanth/http.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "open-uri"
+require "timeout"
+
+require_relative "robots"
+
+module Coelacanth
+  class TimeoutError < StandardError; end unless const_defined?(:TimeoutError)
+
+  module HTTP
+    DEFAULT_OPEN_TIMEOUT = 5
+    DEFAULT_READ_TIMEOUT = 10
+    MAX_RETRIES = 2
+
+    ErrorResponse = Struct.new(:status, :meta, :base_uri, :body, keyword_init: true) do
+      def string
+        body.to_s
+      end
+
+      alias to_s string
+    end
+
+    module_function
+
+    def get_response(uri, open_timeout: DEFAULT_OPEN_TIMEOUT, read_timeout: DEFAULT_READ_TIMEOUT, retries: MAX_RETRIES)
+      ensure_allowed!(uri)
+      raw_get_response(uri, open_timeout: open_timeout, read_timeout: read_timeout, retries: retries)
+    end
+
+    def raw_get_response(uri, open_timeout: DEFAULT_OPEN_TIMEOUT, read_timeout: DEFAULT_READ_TIMEOUT, retries: MAX_RETRIES)
+      attempts = 0
+      begin
+        attempts += 1
+        request = Net::HTTP::Get.new(uri)
+        Net::HTTP.start(
+          uri.host,
+          uri.port,
+          use_ssl: uri.scheme == "https",
+          open_timeout: open_timeout,
+          read_timeout: read_timeout
+        ) do |http|
+          return http.request(request)
+        end
+      rescue Net::OpenTimeout, Net::ReadTimeout, Timeout::Error => e
+        retry if attempts <= retries
+
+        raise Coelacanth::TimeoutError, "GET #{uri} timed out after #{attempts} attempts: #{e.message}"
+      end
+    end
+
+    def ensure_allowed!(uri)
+      return if Coelacanth::Robots.allowed?(uri)
+
+      raise Coelacanth::RobotsDisallowedError,
+            "Access to #{uri} is disallowed by robots.txt for user-agent '#{Coelacanth::Robots.user_agent}'"
+    end
+
+    def raise_http_error(uri, response)
+      message = format("%s %s for GET %s", response.code, response.message, uri)
+      io = ErrorResponse.new(
+        status: [response.code, response.message],
+        meta: response.each_header.to_h,
+        base_uri: uri,
+        body: response.body
+      )
+
+      raise OpenURI::HTTPError.new(message, io)
+    end
+  end
+end

data/lib/coelacanth/redirect.rb

@@ -2,6 +2,7 @@
 
 require "ferrum"
 require "oga"
+require_relative "http"
 
 module Coelacanth
   # Coelacanth::Redirect
@@ -11,11 +12,15 @@ module Coelacanth
       raise Coelacanth::DeepRedirectError, "Too many redirect" if limit.zero?
       raise Coelacanth::RedirectError, "Url or location is nil" if @url.nil?
 
-      response = Net::HTTP.get_response(URI.parse(@url))
+      response = Coelacanth::HTTP.get_response(URI.parse(@url))
       @status_code = response.code
       @origin_response = response
 
       handle_response(@origin_response, limit)
+    rescue Coelacanth::TimeoutError
+      @status_code = nil
+      @origin_response = nil
+      @url
     end
 
     private

data/lib/coelacanth/robots.rb

@@ -0,0 +1,150 @@
+# frozen_string_literal: true
+
+require "uri"
+
+module Coelacanth
+  module Robots
+    DEFAULT_USER_AGENT = "CoelacanthBot"
+    RULE_STRUCT = Struct.new(:type, :pattern, :regex, :length, keyword_init: true)
+
+    module_function
+
+    def allowed?(uri, user_agent: user_agent())
+      rules = rules_for(uri)
+      return true if rules.empty?
+
+      agent_key = normalize_agent(user_agent)
+      agent_rules = rules[agent_key]
+      agent_rules = rules["*"] if agent_rules.nil? || agent_rules.empty?
+
+      return true if agent_rules.nil? || agent_rules.empty?
+
+      evaluate(agent_rules, normalize_path(uri))
+    end
+
+    def user_agent
+      ENV.fetch("COELACANTH_HTTP_USER_AGENT", DEFAULT_USER_AGENT)
+    end
+
+    def rules_for(uri)
+      robots_cache[cache_key(uri)] ||= fetch_rules(uri)
+    end
+
+    def clear_cache!
+      robots_cache.clear
+    end
+
+    def robots_cache
+      @robots_cache ||= {}
+    end
+
+    def fetch_rules(uri)
+      response = Coelacanth::HTTP.raw_get_response(robots_uri_for(uri))
+      return {} unless response.is_a?(Net::HTTPSuccess)
+
+      parse_robots(response.body.to_s)
+    rescue Coelacanth::TimeoutError, StandardError
+      {}
+    end
+
+    def robots_uri_for(uri)
+      klass = uri.scheme == "https" ? URI::HTTPS : URI::HTTP
+      port = uri.port
+      port = nil if port == default_port_for(uri.scheme)
+
+      klass.build(host: uri.host, path: "/robots.txt", port: port)
+    end
+
+    def parse_robots(body)
+      rules = Hash.new { |hash, key| hash[key] = [] }
+      current_agents = []
+      last_directive = nil
+
+      body.each_line do |line|
+        sanitized = sanitize_line(line)
+        if sanitized.empty?
+          current_agents = []
+          last_directive = nil
+          next
+        end
+
+        field, value = sanitized.split(":", 2)
+        next if value.nil?
+
+        field = field.strip.downcase
+        value = value.strip
+
+        case field
+        when "user-agent"
+          current_agents = [] unless last_directive == :user_agent
+          agent = normalize_agent(value)
+          current_agents << agent unless current_agents.include?(agent)
+          last_directive = :user_agent
+        when "allow", "disallow"
+          last_directive = field.to_sym
+          next if value.empty?
+
+          current_agents = ["*"] if current_agents.empty?
+          rule = build_rule(type: last_directive, value: value)
+          current_agents.each do |agent|
+            rules[agent] << rule
+          end
+        else
+          last_directive = field.to_sym
+        end
+      end
+
+      rules
+    end
+
+    def sanitize_line(line)
+      line.split("#", 2).first.to_s.strip
+    end
+
+    def build_rule(type:, value:)
+      pattern = value.start_with?("/") ? value : "/#{value}"
+      escaped = Regexp.escape(pattern)
+      escaped = escaped.gsub("\\*", ".*")
+      escaped = escaped.gsub("\\$", "\\z")
+      regex = Regexp.new("\\A" + escaped)
+      RULE_STRUCT.new(type: type, pattern: pattern, regex: regex, length: pattern.length)
+    end
+
+    def evaluate(rules, path)
+      matches = rules.select { |rule| rule.regex.match?(path) }
+      return true if matches.empty?
+
+      longest_allow = matches.select { |rule| rule.type == :allow }.max_by(&:length)
+      longest_disallow = matches.select { |rule| rule.type == :disallow }.max_by(&:length)
+
+      return true if longest_disallow.nil?
+      return true if longest_allow && longest_allow.length >= longest_disallow.length
+
+      false
+    end
+
+    def normalize_path(uri)
+      path = uri.path
+      path = "/" if path.nil? || path.empty?
+      query = uri.query
+      return path if query.nil? || query.empty?
+
+      "#{path}?#{query}"
+    end
+
+    def normalize_agent(agent)
+      agent.to_s.strip.downcase
+    end
+
+    def cache_key(uri)
+      port = uri.port
+      default_port = default_port_for(uri.scheme)
+      port_part = port && port != default_port ? ":#{port}" : ""
+      "#{uri.scheme}://#{uri.host}#{port_part}"
+    end
+
+    def default_port_for(scheme)
+      scheme == "https" ? URI::HTTPS.default_port : URI::HTTP.default_port
+    end
+  end
+end

data/lib/coelacanth/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Coelacanth
-  VERSION = "0.3.10"
+  VERSION = "0.4.1"
 end

data/lib/coelacanth.rb

@@ -6,6 +6,8 @@ require_relative "coelacanth/client/base"
 require_relative "coelacanth/client/ferrum"
 require_relative "coelacanth/client/screenshot_one"
 require_relative "coelacanth/dom"
+require_relative "coelacanth/extractor"
+require_relative "coelacanth/http"
 require_relative "coelacanth/redirect"
 require_relative "coelacanth/validator"
 require_relative "coelacanth/version"
@@ -15,14 +17,27 @@ module Coelacanth
   class Error < StandardError; end
   class RedirectError < StandardError; end
   class DeepRedirectError < StandardError; end
+  class TimeoutError < StandardError; end
+  class RobotsDisallowedError < StandardError; end
 
   def self.analyze(url)
     client_class = config.read("client") == "screenshot_one" ? Client::ScreenshotOne : Client::Ferrum
     @client = client_class.new(url)
     regular_url = Redirect.new.resolve_redirect(url)
+    response = begin
+      Coelacanth::HTTP.get_response(URI.parse(regular_url))
+    rescue Coelacanth::TimeoutError
+      nil
+    end
+    html = response&.body.to_s
+    html = html.dup
+    html = html.force_encoding(Encoding::UTF_8)
+    html = html.encode(Encoding::UTF_8, invalid: :replace, undef: :replace)
+    extractor_result = Extractor.new.call(html: html, url: regular_url)
     {
-      dom: Dom.new.oga(regular_url),
+      dom: Dom.new.oga(regular_url, html: html),
       screenshot: @client.get_screenshot,
+      extraction: extractor_result,
     }
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: coelacanth
 version: !ruby/object:Gem::Version
-  version: 0.3.10
+  version: 0.4.1
 platform: ruby
 authors:
 - Yusuke
@@ -18,13 +18,13 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".env.example"
 - ".rspec"
 - ".rubocop.yml"
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Dockerfile
 - Gemfile
-- Gemfile.lock
 - LICENSE.txt
 - README.md
 - Rakefile
@@ -36,7 +36,19 @@ files:
 - lib/coelacanth/client/screenshot_one.rb
 - lib/coelacanth/configure.rb
 - lib/coelacanth/dom.rb
+- lib/coelacanth/extractor.rb
+- lib/coelacanth/extractor/fallback_probe.rb
+- lib/coelacanth/extractor/heuristic_probe.rb
+- lib/coelacanth/extractor/image_collector.rb
+- lib/coelacanth/extractor/markdown_listing_collector.rb
+- lib/coelacanth/extractor/markdown_renderer.rb
+- lib/coelacanth/extractor/metadata_probe.rb
+- lib/coelacanth/extractor/normalizer.rb
+- lib/coelacanth/extractor/utilities.rb
+- lib/coelacanth/extractor/weak_ml_probe.rb
+- lib/coelacanth/http.rb
 - lib/coelacanth/redirect.rb
+- lib/coelacanth/robots.rb
 - lib/coelacanth/validator.rb
 - lib/coelacanth/version.rb
 homepage: https://github.com/slidict/coelacanth

data/Gemfile.lock

@@ -1,103 +0,0 @@
-PATH
-  remote: .
-  specs:
-    coelacanth (0.3.5)
-
-GEM
-  remote: https://rubygems.org/
-  specs:
-    addressable (2.8.7)
-      public_suffix (>= 2.0.2, < 7.0)
-    ansi (1.5.0)
-    ast (2.4.3)
-    base64 (0.3.0)
-    bigdecimal (3.1.9)
-    concurrent-ruby (1.3.5)
-    crack (1.0.0)
-      bigdecimal
-      rexml
-    diff-lcs (1.6.1)
-    ferrum (0.17.1)
-      addressable (~> 2.5)
-      base64 (~> 0.2)
-      concurrent-ruby (~> 1.1)
-      webrick (~> 1.7)
-      websocket-driver (~> 0.7)
-    hashdiff (1.1.2)
-    json (2.12.2)
-    language_server-protocol (3.17.0.5)
-    lint_roller (1.1.0)
-    oga (3.4)
-      ast
-      ruby-ll (~> 2.1)
-    parallel (1.27.0)
-    parser (3.3.8.0)
-      ast (~> 2.4.1)
-      racc
-    prism (1.4.0)
-    public_suffix (6.0.2)
-    racc (1.8.1)
-    rainbow (3.1.1)
-    rake (13.3.0)
-    regexp_parser (2.10.0)
-    rexml (3.4.1)
-    rspec (3.13.0)
-      rspec-core (~> 3.13.0)
-      rspec-expectations (~> 3.13.0)
-      rspec-mocks (~> 3.13.0)
-    rspec-core (3.13.3)
-      rspec-support (~> 3.13.0)
-    rspec-expectations (3.13.3)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.13.0)
-    rspec-mocks (3.13.2)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.13.0)
-    rspec-support (3.13.2)
-    rubocop (1.76.1)
-      json (~> 2.3)
-      language_server-protocol (~> 3.17.0.2)
-      lint_roller (~> 1.1.0)
-      parallel (~> 1.10)
-      parser (>= 3.3.0.2)
-      rainbow (>= 2.2.2, < 4.0)
-      regexp_parser (>= 2.9.3, < 3.0)
-      rubocop-ast (>= 1.45.0, < 2.0)
-      ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 2.4.0, < 4.0)
-    rubocop-ast (1.45.1)
-      parser (>= 3.3.7.2)
-      prism (~> 1.4)
-    ruby-ll (2.1.3)
-      ansi
-      ast
-    ruby-progressbar (1.13.0)
-    unicode-display_width (3.1.4)
-      unicode-emoji (~> 4.0, >= 4.0.4)
-    unicode-emoji (4.0.4)
-    webmock (3.25.1)
-      addressable (>= 2.8.0)
-      crack (>= 0.3.2)
-      hashdiff (>= 0.4.0, < 2.0.0)
-    webrick (1.9.1)
-    websocket-driver (0.7.7)
-      base64
-      websocket-extensions (>= 0.1.0)
-    websocket-extensions (0.1.5)
-
-PLATFORMS
-  ruby
-  x86_64-linux
-
-DEPENDENCIES
-  base64 (~> 0.3.0)
-  coelacanth!
-  ferrum (~> 0.16)
-  oga (~> 3.4)
-  rake (~> 13.3)
-  rspec (~> 3.0)
-  rubocop (~> 1.76)
-  webmock (~> 3.25)
-
-BUNDLED WITH
-   2.6.7