coelacanth 0.4.0 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3f8a15ca435355182b77131ffa50b2c93b08693c3ccf0d26b5010e1f7f9df4fd
-  data.tar.gz: 34f198f28014ed558f14ef5599f9275579054a1ec9ebeb6b503c60555329bcf2
+  metadata.gz: 9097f36247caad8f0764313b306398e3707290eca25d5cfffc05f61e97784884
+  data.tar.gz: 19e093800bcb9ae663e0f36a1ad18d15472ff2e76f34f6ddf65c396440aea2e0
 SHA512:
-  metadata.gz: aa7cdd1ac14d7104604114885facaffc6dc5e4d8b91911cb1ae26b34aa2550010bd1658f3c832a6e1dc4a0a059113317c7242bc187e02a396cc17398570407e0
-  data.tar.gz: 8f1c4c988ec19a862409e06fbe297529d058a78dc77f328959ab5274fdd372ef65d8e92b38ba76ef08b55ad7d5abeb75d585685f411e223ce9fc60515430ccc7
+  metadata.gz: 32c30afcf5316814e42f2ed2f5b94efe49daecdd78b7acdf1d10d64ed2c8253a86a0a0be3b66ed1824cfbefb32c5654f3abf0fb08c41411d736281f480375400
+  data.tar.gz: 4e6683d596d50535dd13df26d2c6c4339fe543013d37250c8e55f7605dc441aa3ed888fcf46f9550540583e3234bd52667cd28d1b9e098a18e3eb5faae354bed

data/.env.example

@@ -0,0 +1,5 @@
+# Copy this file to .env and fill in the values to configure Coelacanth.
+# Optional: only set when the remote browser requires authentication.
+COELACANTH_REMOTE_CLIENT_AUTHORIZATION=
+COELACANTH_REMOTE_CLIENT_USER_AGENT="Coelacanth Chrome Extension"
+COELACANTH_SCREENSHOT_ONE_API_KEY="your_screenshot_one_api_key_here"

data/CHANGELOG.md

@@ -4,19 +4,8 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [v0.4.0] - 2025-11-02
-### :sparkles: New Features
-- [`d13c21e`](https://github.com/slidict/coelacanth/commit/d13c21e106e7155ba7d545570341da37caa0b4b3) - support structural listings for digital.go.jp *(commit by [@yubele](https://github.com/yubele))*
-- [`19fb4d7`](https://github.com/slidict/coelacanth/commit/19fb4d7a29350bb063c6bb76a89c46d7d001fc3e) - expose body markdown blocks *(commit by [@yubele](https://github.com/yubele))*
-
+## [v0.4.1] - 2025-11-03
 ### :wrench: Chores
-- [`513d522`](https://github.com/slidict/coelacanth/commit/513d5226147b7be72c43817d0ca417a2ea355d31) - **deps**: Bump rubocop from 1.76.1 to 1.77.0 *(commit by [@dependabot[bot]](https://github.com/apps/dependabot))*
-- [`388658f`](https://github.com/slidict/coelacanth/commit/388658fcaa94f09123c67ab62dd69d207d03ee7a) - **deps**: Bump rubocop from 1.77.0 to 1.81.6 *(commit by [@dependabot[bot]](https://github.com/apps/dependabot))*
-- [`be62dbb`](https://github.com/slidict/coelacanth/commit/be62dbbf77fc9a324f29d684df167cbedcea7b8f) - translate listing samples to English *(commit by [@yubele](https://github.com/yubele))*
-- [`41bf358`](https://github.com/slidict/coelacanth/commit/41bf358eb955fb75b467719750a19f9d07a2cba1) - require ruby 3.4 or newer *(commit by [@yubele](https://github.com/yubele))*
-- [`cff304d`](https://github.com/slidict/coelacanth/commit/cff304d255952ad6be18e796c6649b72272bc23e) - **deps-dev**: Bump rexml in the bundler group across 1 directory *(commit by [@dependabot[bot]](https://github.com/apps/dependabot))*
-- [`962c267`](https://github.com/slidict/coelacanth/commit/962c267ab8f301e1375b67fc32a3cbc54effc232) - Bump version to 0.4.0 *(commit by [@yubele](https://github.com/yubele))*
-- [`bf3633c`](https://github.com/slidict/coelacanth/commit/bf3633c82a61684238485606300bfa0891d247eb) - Delete Gemfile.lock *(commit by [@yubele](https://github.com/yubele))*
-- [`62f2a0e`](https://github.com/slidict/coelacanth/commit/62f2a0ecb80fb5d77f704e6c48d1d8f4c7670818) - Add Gemfile.lock to .gitignore *(commit by [@yubele](https://github.com/yubele))*
+- [`41e89b7`](https://github.com/slidict/coelacanth/commit/41e89b799573f6cfaf0a12e7abc5c260f1905aec) - Bump version from 0.4.0 to 0.4.1 *(commit by [@yubele](https://github.com/yubele))*
 
-[v0.4.0]: https://github.com/slidict/coelacanth/compare/v0.3.10...v0.4.0
+[v0.4.1]: https://github.com/slidict/coelacanth/compare/v0.4.0...v0.4.1

data/README.md

@@ -102,8 +102,8 @@ Coelacanth ships with a multi-stage extractor that tries increasingly involved p
 4. **FallbackProbe** acts as a safety net by following AMP/print links or summarizing the whole document when the previous
    probes fail.
 
-A `ListingCollector` scans nearby layout regions to surface structured listings (e.g., sidebar "latest news" blocks) so you can
-store related links alongside the article.
+Markdown-based listings are generated from the extracted body so lists such as "Latest news" blocks can be stored alongside the
+article without scanning the rest of the page layout.
 
 ## Configuration
 Runtime configuration is stored in `config/coelacanth.yml`. Environments inherit from the `development` section by default.
@@ -115,10 +115,12 @@ development:
     ws_url: "ws://chrome:3000/chrome"
     timeout: 10
     headers:
-      Authorization: "Bearer 1234567890"
-      User-Agent: "Coelacanth Chrome Extension"
+<% if (auth = ENV["COELACANTH_REMOTE_CLIENT_AUTHORIZATION"]).to_s.strip != "" %>
+      Authorization: "<%= auth %>"
+<% end %>
+      User-Agent: "<%= ENV.fetch("COELACANTH_REMOTE_CLIENT_USER_AGENT", "Coelacanth Chrome Extension") %>"
   screenshot_one:
-    key: "your_screenshot_one_api_key_here"
+    key: "<%= ENV.fetch("COELACANTH_SCREENSHOT_ONE_API_KEY", "your_screenshot_one_api_key_here") %>"
 ```
 
 - **Ferrum client** – Requires a running Chrome instance that exposes the DevTools protocol via WebSocket. Configure the URL,
@@ -127,6 +129,25 @@ development:
 - Configuration is environment-aware: set `RAILS_ENV`/`RACK_ENV` or use Rails' built-in environment handling when the gem is
   used inside a Rails project.
 
+### Environment variables
+
+Configuration values that would otherwise contain credentials are loaded from environment variables. Set the following
+variables in your shell (or `dotenv` file) before running the gem:
+
+```bash
+# Optional: only set when the remote browser requires authentication.
+export COELACANTH_REMOTE_CLIENT_AUTHORIZATION="Bearer <token>"
+
+export COELACANTH_REMOTE_CLIENT_USER_AGENT="Coelacanth Chrome Extension"
+export COELACANTH_SCREENSHOT_ONE_API_KEY="your_screenshot_one_api_key_here"
+```
+
+If `COELACANTH_REMOTE_CLIENT_AUTHORIZATION` is omitted or left blank, the `Authorization` header is not injected into the
+remote browser session.
+
+When using Docker Compose, you can create a `.env` file or export the variables in your environment so the `app` service picks
+them up automatically.
+
 If you are working inside Docker, make sure the `UID` environment variable matches your host user by exporting it in your shell
 startup file:
 

data/compose.yml

@@ -3,8 +3,11 @@ networks:
     driver: bridge
 services:
   app:
-    environment:
-      - UID=${UID}
+    environment:
+      - UID=${UID}
+      - COELACANTH_REMOTE_CLIENT_AUTHORIZATION=${COELACANTH_REMOTE_CLIENT_AUTHORIZATION:-}
+      - COELACANTH_REMOTE_CLIENT_USER_AGENT=${COELACANTH_REMOTE_CLIENT_USER_AGENT:-}
+      - COELACANTH_SCREENSHOT_ONE_API_KEY=${COELACANTH_SCREENSHOT_ONE_API_KEY:-}
     tty: true
     stdin_open: true
     build:

data/config/coelacanth.yml

@@ -4,10 +4,12 @@ development: &development
     ws_url: "ws://chrome:3000/chrome"
     timeout: 10 # seconds
     headers:
-      Authorization: "Bearer 1234567890"
-      User-Agent: "Coelacanth Chrome Extension"
+<% if (auth = ENV["COELACANTH_REMOTE_CLIENT_AUTHORIZATION"]).to_s.strip != "" %>
+      Authorization: "<%= auth %>"
+<% end %>
+      User-Agent: "<%= ENV.fetch("COELACANTH_REMOTE_CLIENT_USER_AGENT", "Coelacanth Chrome Extension") %>"
   screenshot_one:
-    key: "your_screenshot_one_api_key_here"
+    key: "<%= ENV.fetch("COELACANTH_SCREENSHOT_ONE_API_KEY", "your_screenshot_one_api_key_here") %>"
 test:
   <<: *development
 production:

data/lib/coelacanth/client/ferrum.rb

@@ -16,7 +16,7 @@ module Coelacanth::Client
       body = remote_client.body
       body
     rescue => e
-      raise "#{e.class}: #{e.message} RemoteClient: #{@remote_client.inspect}"
+      raise sanitized_remote_client_error(e)
     end
 
     def get_screenshot
@@ -26,11 +26,21 @@ module Coelacanth::Client
       File.read(tempfile.path)
     rescue => e
       tempfile.close
-      raise "#{e.class}: #{e.message} RemoteClient: #{@remote_client.inspect}"
+      raise sanitized_remote_client_error(e)
     end
 
     private
 
+    def sanitized_remote_client_error(error)
+      "#{error.class}: #{error.message} RemoteClient: #{sanitized_remote_client_identifier}"
+    end
+
+    def sanitized_remote_client_identifier
+      return "nil" unless @remote_client
+
+      "#{@remote_client.class.name}(object_id=#{@remote_client.object_id})"
+    end
+
     def remote_client
       return @remote_client if @remote_client
 

data/lib/coelacanth/client/screenshot_one.rb

@@ -1,19 +1,29 @@
 # frozen_string_literal: true
 
-require "open-uri"
 require "ferrum"
+require_relative "ferrum"
+require_relative "../http"
 
 module Coelacanth::Client
   # Coelacanth::Client
   class ScreenshotOne < Coelacanth::Client::Base
     def get_response
-      @origin_response = URI(@url).open
-      @status_code = @origin_response.status[0].to_i
-      body = @origin_response.read
-      body
-    rescue OpenURI::HTTPError => e
-      @status_code = e.io.status[0].to_i
-      raise e
+      uri = URI.parse(@url)
+      response = Coelacanth::HTTP.get_response(
+        uri,
+        open_timeout: Coelacanth::HTTP::DEFAULT_OPEN_TIMEOUT,
+        read_timeout: Coelacanth::HTTP::DEFAULT_READ_TIMEOUT
+      )
+      @origin_response = response
+      @status_code = response.code.to_i
+
+      return response.body if response.is_a?(Net::HTTPSuccess)
+
+      Coelacanth::HTTP.raise_http_error(uri, response)
+    rescue Coelacanth::TimeoutError
+      fallback_response = fallback_client.get_response
+      @status_code = fallback_client.instance_variable_get(:@status_code)
+      fallback_response
     end
 
     def get_screenshot
@@ -34,10 +44,22 @@ module Coelacanth::Client
       }
       uri.query = URI.encode_www_form(params)
 
-      response = Net::HTTP.get_response(uri)
+      response = Coelacanth::HTTP.get_response(
+        uri,
+        open_timeout: Coelacanth::HTTP::DEFAULT_OPEN_TIMEOUT,
+        read_timeout: 30
+      )
       raise "Failed to fetch screenshot: #{response.code}" unless response.is_a?(Net::HTTPSuccess)
 
       response.body
+    rescue Coelacanth::TimeoutError
+      fallback_client.get_screenshot
+    end
+
+    private
+
+    def fallback_client
+      @fallback_client ||= Coelacanth::Client::Ferrum.new(@url, @config)
     end
   end
 end

data/lib/coelacanth/configure.rb

@@ -15,7 +15,12 @@ module Coelacanth
     end
 
     def yaml
-      @yaml ||= YAML.unsafe_load(ERB.new(File.read(file)).result)[env]
+      @yaml ||= YAML.safe_load(
+        ERB.new(File.read(file)).result,
+        permitted_classes: [],
+        permitted_symbols: [],
+        aliases: true
+      )[env]
     end
 
     private

data/lib/coelacanth/dom.rb

@@ -1,13 +1,18 @@
 # frozen_string_literal: true
 
 require "oga"
+require_relative "http"
 
 module Coelacanth
   # Coelacanth::Dom
   class Dom
     def oga(url, html: nil)
-      html ||= Net::HTTP.get_response(URI.parse(url)).body
-      Oga.parse_xml(html)
+      html ||= begin
+        Coelacanth::HTTP.get_response(URI.parse(url)).body
+      rescue Coelacanth::TimeoutError
+        ""
+      end
+      Oga.parse_xml(html.to_s)
     end
   end
 end

data/lib/coelacanth/extractor/markdown_listing_collector.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+require_relative "utilities"
+
+module Coelacanth
+  class Extractor
+    # Extracts structured listings from Markdown content.
+    class MarkdownListingCollector
+      LIST_ITEM_PATTERN = /\A(?:[-*+]|\d+\.)\s+/.freeze
+      HEADING_PATTERN = /\A#+\s*/.freeze
+      MIN_ITEMS = 3
+      MIN_TITLE_LENGTH = 2
+
+      def call(markdown:, base_url: nil)
+        return [] if markdown.to_s.strip.empty?
+
+        listings = []
+        current = nil
+        pending_heading = nil
+
+        finalize_current = lambda do
+          next unless current
+
+          if current[:items].length >= MIN_ITEMS
+            listings << { heading: current[:heading], items: current[:items] }
+          end
+
+          current = nil
+        end
+
+        markdown.each_line do |line|
+          stripped = line.strip
+
+          if stripped.empty?
+            finalize_current.call
+            next
+          end
+
+          if heading_line?(stripped)
+            finalize_current.call
+            pending_heading = normalize_heading(stripped)
+            next
+          end
+
+          if list_item_line?(stripped)
+            current ||= { heading: pending_heading, items: [] }
+            pending_heading = nil
+
+            if (item = build_item(stripped, base_url))
+              current[:items] << item
+            end
+          else
+            finalize_current.call
+            pending_heading = nil
+          end
+        end
+
+        finalize_current.call
+
+        listings
+      end
+
+      private
+
+      def heading_line?(line)
+        line.start_with?("#") && line.match?(HEADING_PATTERN)
+      end
+
+      def list_item_line?(line)
+        line.match?(LIST_ITEM_PATTERN)
+      end
+
+      def normalize_heading(line)
+        line.sub(HEADING_PATTERN, "").strip
+      end
+
+      def build_item(line, base_url)
+        content = line.sub(LIST_ITEM_PATTERN, "").strip
+        return if content.empty?
+
+        if (match = content.match(/\A\[([^\]]+)\]\(([^\)]+)\)(.*)\z/))
+          title = match[1].to_s.strip
+          href = match[2].to_s.strip
+          trailing = match[3].to_s.strip
+
+          return if title.length < MIN_TITLE_LENGTH
+
+          url = Utilities.absolute_url(base_url, href) || href
+          item = { title: title, url: url }
+
+          snippet = normalize_snippet(trailing)
+          item[:snippet] = snippet unless snippet.nil? || snippet.empty?
+          item
+        else
+          title = content
+          return if title.length < MIN_TITLE_LENGTH
+
+          { title: title }
+        end
+      end
+
+      def normalize_snippet(text)
+        stripped = text.to_s.sub(/\A[-–—:]\s*/, "").strip
+        stripped.empty? ? nil : stripped
+      end
+    end
+  end
+end

data/lib/coelacanth/extractor/markdown_renderer.rb

@@ -52,6 +52,10 @@ module Coelacanth
           end + [""]
         when "li"
           ["- #{inline_children(node, depth)}"]
+        when "a"
+          href = node["href"].to_s.strip
+          text = inline_children(node, depth)
+          href.empty? ? text : "[#{text}](#{href})"
         when "strong", "b"
           "**#{inline_children(node, depth)}**"
         when "em", "i"

data/lib/coelacanth/extractor.rb

@@ -7,7 +7,7 @@ require_relative "extractor/weak_ml_probe"
 require_relative "extractor/fallback_probe"
 require_relative "extractor/markdown_renderer"
 require_relative "extractor/image_collector"
-require_relative "extractor/listing_collector"
+require_relative "extractor/markdown_listing_collector"
 
 module Coelacanth
   # High-level API for extracting articles without site-specific selectors.
@@ -60,7 +60,7 @@ module Coelacanth
         byline: result.byline,
         source: result.source_tag,
         confidence: result.confidence,
-        listings: ListingCollector.new.call(document: document, base_url: url, primary_node: node)
+        listings: MarkdownListingCollector.new.call(markdown: body_markdown, base_url: url)
       }
     end
   end

data/lib/coelacanth/http.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "open-uri"
+require "timeout"
+
+require_relative "robots"
+
+module Coelacanth
+  class TimeoutError < StandardError; end unless const_defined?(:TimeoutError)
+
+  module HTTP
+    DEFAULT_OPEN_TIMEOUT = 5
+    DEFAULT_READ_TIMEOUT = 10
+    MAX_RETRIES = 2
+
+    ErrorResponse = Struct.new(:status, :meta, :base_uri, :body, keyword_init: true) do
+      def string
+        body.to_s
+      end
+
+      alias to_s string
+    end
+
+    module_function
+
+    def get_response(uri, open_timeout: DEFAULT_OPEN_TIMEOUT, read_timeout: DEFAULT_READ_TIMEOUT, retries: MAX_RETRIES)
+      ensure_allowed!(uri)
+      raw_get_response(uri, open_timeout: open_timeout, read_timeout: read_timeout, retries: retries)
+    end
+
+    def raw_get_response(uri, open_timeout: DEFAULT_OPEN_TIMEOUT, read_timeout: DEFAULT_READ_TIMEOUT, retries: MAX_RETRIES)
+      attempts = 0
+      begin
+        attempts += 1
+        request = Net::HTTP::Get.new(uri)
+        Net::HTTP.start(
+          uri.host,
+          uri.port,
+          use_ssl: uri.scheme == "https",
+          open_timeout: open_timeout,
+          read_timeout: read_timeout
+        ) do |http|
+          return http.request(request)
+        end
+      rescue Net::OpenTimeout, Net::ReadTimeout, Timeout::Error => e
+        retry if attempts <= retries
+
+        raise Coelacanth::TimeoutError, "GET #{uri} timed out after #{attempts} attempts: #{e.message}"
+      end
+    end
+
+    def ensure_allowed!(uri)
+      return if Coelacanth::Robots.allowed?(uri)
+
+      raise Coelacanth::RobotsDisallowedError,
+            "Access to #{uri} is disallowed by robots.txt for user-agent '#{Coelacanth::Robots.user_agent}'"
+    end
+
+    def raise_http_error(uri, response)
+      message = format("%s %s for GET %s", response.code, response.message, uri)
+      io = ErrorResponse.new(
+        status: [response.code, response.message],
+        meta: response.each_header.to_h,
+        base_uri: uri,
+        body: response.body
+      )
+
+      raise OpenURI::HTTPError.new(message, io)
+    end
+  end
+end

data/lib/coelacanth/redirect.rb

@@ -2,6 +2,7 @@
 
 require "ferrum"
 require "oga"
+require_relative "http"
 
 module Coelacanth
   # Coelacanth::Redirect
@@ -11,11 +12,15 @@ module Coelacanth
       raise Coelacanth::DeepRedirectError, "Too many redirect" if limit.zero?
       raise Coelacanth::RedirectError, "Url or location is nil" if @url.nil?
 
-      response = Net::HTTP.get_response(URI.parse(@url))
+      response = Coelacanth::HTTP.get_response(URI.parse(@url))
       @status_code = response.code
       @origin_response = response
 
       handle_response(@origin_response, limit)
+    rescue Coelacanth::TimeoutError
+      @status_code = nil
+      @origin_response = nil
+      @url
     end
 
     private

data/lib/coelacanth/robots.rb

@@ -0,0 +1,150 @@
+# frozen_string_literal: true
+
+require "uri"
+
+module Coelacanth
+  module Robots
+    DEFAULT_USER_AGENT = "CoelacanthBot"
+    RULE_STRUCT = Struct.new(:type, :pattern, :regex, :length, keyword_init: true)
+
+    module_function
+
+    def allowed?(uri, user_agent: user_agent())
+      rules = rules_for(uri)
+      return true if rules.empty?
+
+      agent_key = normalize_agent(user_agent)
+      agent_rules = rules[agent_key]
+      agent_rules = rules["*"] if agent_rules.nil? || agent_rules.empty?
+
+      return true if agent_rules.nil? || agent_rules.empty?
+
+      evaluate(agent_rules, normalize_path(uri))
+    end
+
+    def user_agent
+      ENV.fetch("COELACANTH_HTTP_USER_AGENT", DEFAULT_USER_AGENT)
+    end
+
+    def rules_for(uri)
+      robots_cache[cache_key(uri)] ||= fetch_rules(uri)
+    end
+
+    def clear_cache!
+      robots_cache.clear
+    end
+
+    def robots_cache
+      @robots_cache ||= {}
+    end
+
+    def fetch_rules(uri)
+      response = Coelacanth::HTTP.raw_get_response(robots_uri_for(uri))
+      return {} unless response.is_a?(Net::HTTPSuccess)
+
+      parse_robots(response.body.to_s)
+    rescue Coelacanth::TimeoutError, StandardError
+      {}
+    end
+
+    def robots_uri_for(uri)
+      klass = uri.scheme == "https" ? URI::HTTPS : URI::HTTP
+      port = uri.port
+      port = nil if port == default_port_for(uri.scheme)
+
+      klass.build(host: uri.host, path: "/robots.txt", port: port)
+    end
+
+    def parse_robots(body)
+      rules = Hash.new { |hash, key| hash[key] = [] }
+      current_agents = []
+      last_directive = nil
+
+      body.each_line do |line|
+        sanitized = sanitize_line(line)
+        if sanitized.empty?
+          current_agents = []
+          last_directive = nil
+          next
+        end
+
+        field, value = sanitized.split(":", 2)
+        next if value.nil?
+
+        field = field.strip.downcase
+        value = value.strip
+
+        case field
+        when "user-agent"
+          current_agents = [] unless last_directive == :user_agent
+          agent = normalize_agent(value)
+          current_agents << agent unless current_agents.include?(agent)
+          last_directive = :user_agent
+        when "allow", "disallow"
+          last_directive = field.to_sym
+          next if value.empty?
+
+          current_agents = ["*"] if current_agents.empty?
+          rule = build_rule(type: last_directive, value: value)
+          current_agents.each do |agent|
+            rules[agent] << rule
+          end
+        else
+          last_directive = field.to_sym
+        end
+      end
+
+      rules
+    end
+
+    def sanitize_line(line)
+      line.split("#", 2).first.to_s.strip
+    end
+
+    def build_rule(type:, value:)
+      pattern = value.start_with?("/") ? value : "/#{value}"
+      escaped = Regexp.escape(pattern)
+      escaped = escaped.gsub("\\*", ".*")
+      escaped = escaped.gsub("\\$", "\\z")
+      regex = Regexp.new("\\A" + escaped)
+      RULE_STRUCT.new(type: type, pattern: pattern, regex: regex, length: pattern.length)
+    end
+
+    def evaluate(rules, path)
+      matches = rules.select { |rule| rule.regex.match?(path) }
+      return true if matches.empty?
+
+      longest_allow = matches.select { |rule| rule.type == :allow }.max_by(&:length)
+      longest_disallow = matches.select { |rule| rule.type == :disallow }.max_by(&:length)
+
+      return true if longest_disallow.nil?
+      return true if longest_allow && longest_allow.length >= longest_disallow.length
+
+      false
+    end
+
+    def normalize_path(uri)
+      path = uri.path
+      path = "/" if path.nil? || path.empty?
+      query = uri.query
+      return path if query.nil? || query.empty?
+
+      "#{path}?#{query}"
+    end
+
+    def normalize_agent(agent)
+      agent.to_s.strip.downcase
+    end
+
+    def cache_key(uri)
+      port = uri.port
+      default_port = default_port_for(uri.scheme)
+      port_part = port && port != default_port ? ":#{port}" : ""
+      "#{uri.scheme}://#{uri.host}#{port_part}"
+    end
+
+    def default_port_for(scheme)
+      scheme == "https" ? URI::HTTPS.default_port : URI::HTTP.default_port
+    end
+  end
+end

data/lib/coelacanth/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Coelacanth
-  VERSION = "0.4.0"
+  VERSION = "0.4.1"
 end

data/lib/coelacanth.rb

@@ -7,6 +7,7 @@ require_relative "coelacanth/client/ferrum"
 require_relative "coelacanth/client/screenshot_one"
 require_relative "coelacanth/dom"
 require_relative "coelacanth/extractor"
+require_relative "coelacanth/http"
 require_relative "coelacanth/redirect"
 require_relative "coelacanth/validator"
 require_relative "coelacanth/version"
@@ -16,13 +17,20 @@ module Coelacanth
   class Error < StandardError; end
   class RedirectError < StandardError; end
   class DeepRedirectError < StandardError; end
+  class TimeoutError < StandardError; end
+  class RobotsDisallowedError < StandardError; end
 
   def self.analyze(url)
     client_class = config.read("client") == "screenshot_one" ? Client::ScreenshotOne : Client::Ferrum
     @client = client_class.new(url)
     regular_url = Redirect.new.resolve_redirect(url)
-    response = Net::HTTP.get_response(URI.parse(regular_url))
-    html = response.body.to_s
+    response = begin
+      Coelacanth::HTTP.get_response(URI.parse(regular_url))
+    rescue Coelacanth::TimeoutError
+      nil
+    end
+    html = response&.body.to_s
+    html = html.dup
     html = html.force_encoding(Encoding::UTF_8)
     html = html.encode(Encoding::UTF_8, invalid: :replace, undef: :replace)
     extractor_result = Extractor.new.call(html: html, url: regular_url)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: coelacanth
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.1
 platform: ruby
 authors:
 - Yusuke
@@ -18,6 +18,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".env.example"
 - ".rspec"
 - ".rubocop.yml"
 - CHANGELOG.md
@@ -39,13 +40,15 @@ files:
 - lib/coelacanth/extractor/fallback_probe.rb
 - lib/coelacanth/extractor/heuristic_probe.rb
 - lib/coelacanth/extractor/image_collector.rb
-- lib/coelacanth/extractor/listing_collector.rb
+- lib/coelacanth/extractor/markdown_listing_collector.rb
 - lib/coelacanth/extractor/markdown_renderer.rb
 - lib/coelacanth/extractor/metadata_probe.rb
 - lib/coelacanth/extractor/normalizer.rb
 - lib/coelacanth/extractor/utilities.rb
 - lib/coelacanth/extractor/weak_ml_probe.rb
+- lib/coelacanth/http.rb
 - lib/coelacanth/redirect.rb
+- lib/coelacanth/robots.rb
 - lib/coelacanth/validator.rb
 - lib/coelacanth/version.rb
 homepage: https://github.com/slidict/coelacanth

data/lib/coelacanth/extractor/listing_collector.rb

@@ -1,270 +0,0 @@
-# frozen_string_literal: true
-
-require_relative "utilities"
-
-module Coelacanth
-  class Extractor
-    # Identifies sidebar or inline news listings and returns link arrays.
-    class ListingCollector
-      CANDIDATE_SELECTOR = "aside, section, div, ul, ol, dl".freeze
-      MIN_ITEMS = 3
-      MIN_TITLE_LENGTH = 2
-
-      def call(document:, base_url: nil, primary_node: nil)
-        candidates = collect_candidates(document, base_url, primary_node)
-
-        candidates
-          .sort_by { |candidate| -candidate[:score] }
-          .reject { |candidate| candidate[:score] < minimum_score }
-          .first(3)
-          .map { |candidate| format_candidate(candidate) }
-      end
-
-      private
-
-      def collect_candidates(document, base_url, primary_node)
-        document.css(CANDIDATE_SELECTOR).filter_map do |node|
-          next if skip_node?(node, primary_node)
-
-          items = extract_items(node, base_url)
-          next if items.length < MIN_ITEMS
-
-          heading = heading_for(node)
-          score = score_node(node, items, heading)
-          next if score < minimum_score
-
-          { node: node, items: items, heading: heading, score: score }
-        end
-      end
-
-      def skip_node?(node, primary_node)
-        return true if nested_listing_container?(node)
-
-        return false unless primary_node
-        return false unless primary_node.respond_to?(:name)
-        return false if %w[body html].include?(primary_node.name)
-
-        node == primary_node ||
-          ancestor?(node, primary_node) ||
-          ancestor?(primary_node, node)
-      end
-
-      def nested_listing_container?(node)
-        Utilities.ancestors(node).any? do |ancestor|
-          Utilities.element?(ancestor) && LISTING_CONTAINER_TAGS.include?(ancestor.name)
-        end
-      end
-
-      LISTING_CONTAINER_TAGS = %w[aside section div ul ol dl].freeze
-
-      def ancestor?(node, candidate)
-        Utilities.ancestors(node).any? { |ancestor| ancestor == candidate }
-      end
-
-      def extract_items(node, base_url)
-        item_nodes = candidate_children(node)
-        return [] if item_nodes.empty?
-
-        item_nodes.filter_map do |child|
-          next unless contains_link?(child)
-
-          anchor = primary_anchor(child)
-          next unless anchor
-
-          title = normalize_text(anchor.text)
-          next if title.length < MIN_TITLE_LENGTH
-
-          href = anchor["href"].to_s.strip
-          next if href.empty?
-
-          url = base_url ? Utilities.absolute_url(base_url, href) : href
-          url ||= href
-
-          snippet = build_snippet(child, title)
-
-          item = { title: title, url: url }
-          item[:snippet] = snippet unless snippet.nil? || snippet.empty?
-          item
-        end.uniq { |item| [item[:title], item[:url]] }
-      end
-
-      def candidate_children(node)
-        direct_children = Utilities.element_children(node)
-        return [] if direct_children.empty?
-
-        anchor_children = direct_children.select { |child| contains_link?(child) }
-        return anchor_children if anchor_children.length >= MIN_ITEMS
-
-        groups = %w[li article div section p dd]
-
-        groups.each do |tag|
-          grouped = direct_children.select { |child| child.name == tag }
-          return grouped if grouped.length >= MIN_ITEMS
-        end
-
-        list_container = direct_children.find { |child| %w[ul ol dl].include?(child.name) }
-        return Utilities.element_children(list_container) if list_container
-
-        []
-      end
-
-      def contains_link?(node)
-        node.css("a[href]").any?
-      end
-
-      def primary_anchor(node)
-        anchors = node.css("a[href]")
-        anchors.max_by { |anchor| normalize_text(anchor.text).length }
-      end
-
-      def normalize_text(text)
-        text.to_s.gsub(/[\r\n\t]/, " ").squeeze(" ").strip
-      end
-
-      def build_snippet(node, title)
-        snippet_from_node_text(node, title) || metadata_context(node, title)
-      end
-
-      def snippet_from_node_text(node, title)
-        text = normalize_text(node.text)
-        snippet = text.sub(title, "").strip
-        snippet.empty? ? nil : truncate(snippet)
-      end
-
-      def metadata_context(node, title)
-        candidate = time_text(node) || preceding_metadata(node)
-        return nil if candidate.nil?
-
-        candidate = candidate.sub(title, "").strip
-        candidate.empty? ? nil : truncate(candidate)
-      end
-
-      def time_text(node)
-        node.css("time").filter_map { |time| normalize_text(time.text) }.find { |text| !text.empty? }
-      end
-
-      def preceding_metadata(node)
-        previous = Utilities.previous_element(node)
-        3.times do
-          break unless previous
-
-          text = normalize_text(previous.text)
-          return text unless text.empty?
-
-          previous = Utilities.previous_element(previous)
-        end
-
-        nil
-      end
-
-      def truncate(text)
-        return text if text.length <= 120
-
-        text[0...117] + "..."
-      end
-
-      def heading_for(node)
-        if (heading = node.at_css("h1, h2, h3, h4"))
-          return normalize_text(heading.text)
-        end
-
-        previous = Utilities.previous_element(node)
-        3.times do
-          break unless previous
-
-          return normalize_text(previous.text) if previous.name =~ /h[1-6]/
-          previous = Utilities.previous_element(previous)
-        end
-
-        nil
-      end
-
-      def score_node(node, items, heading)
-        structure_score = structural_score(node)
-        heading_score = heading ? 45 : 0
-        item_score = items.length * 40
-        density_score = Utilities.link_density(node) * 90
-        adjacency_score = sibling_sequence_bonus(node)
-        depth_penalty = Utilities.depth(node) * 5
-        length_penalty = long_text_penalty(node)
-
-        structure_score + heading_score + item_score + density_score + adjacency_score - depth_penalty - length_penalty
-      end
-
-      def structural_score(node)
-        children = candidate_children(node)
-        return 0 if children.empty?
-
-        dominant_tag, dominant_children = children.group_by(&:name).max_by { |_, nodes| nodes.length }
-        dominant_count = dominant_children.length
-
-        uniform_bonus = dominant_count == children.length ? 60 : 20
-        list_bonus = %w[ul ol dl].include?(node.name) ? 90 : 0
-        list_bonus += 45 if dominant_tag && %w[li dd].include?(dominant_tag)
-
-        distribution_bonus = distribution_consistency_bonus(children)
-
-        dominant_count * 12 + uniform_bonus + list_bonus + distribution_bonus
-      end
-
-      def distribution_consistency_bonus(children)
-        return 0 if children.length < MIN_ITEMS
-
-        lengths = children.map { |child| Utilities.text_length(child) }
-        average = lengths.sum.to_f / lengths.length
-        variance = lengths.map { |len| (len - average).abs }
-
-        variance.max <= 120 ? 40 : 10
-      end
-
-      def sibling_sequence_bonus(node)
-        siblings = Utilities.sibling_elements(node)
-        return 0 if siblings.empty?
-
-        index = siblings.index(node)
-        return 0 unless index
-
-        forward = 0
-        while (candidate = siblings[index + forward + 1]) && similar_structure?(node, candidate)
-          forward += 1
-        end
-
-        backward = 0
-        while index - backward - 1 >= 0 && (candidate = siblings[index - backward - 1]) && similar_structure?(node, candidate)
-          backward += 1
-        end
-
-        (forward + backward) * 15
-      end
-
-      def similar_structure?(node, other)
-        return false unless other
-
-        node_children = candidate_children(node)
-        other_children = candidate_children(other)
-        return false if node_children.empty? || other_children.empty?
-
-        node_children.first.name == other_children.first.name && node_children.length == other_children.length
-      end
-
-      def long_text_penalty(node)
-        children = candidate_children(node)
-        return 0 if children.empty?
-
-        overlong = children.count { |child| Utilities.text_length(child) > 280 }
-        overlong * 30
-      end
-
-      def minimum_score
-        180
-      end
-
-      def format_candidate(candidate)
-        {
-          heading: candidate[:heading],
-          items: candidate[:items]
-        }
-      end
-    end
-  end
-end