coelacanth 0.5.1 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f6e9c7c1d9049b351076d0238e2b5747144b73244e3f0c37d2ce69fb625b823a
-  data.tar.gz: d5ef6a678e876b1fabfffd68426785ba6bb4174178e323adbe9593bf9c6ff57a
+  metadata.gz: f1bc84d97135234e0e80586a789381ea2e10785c66fac0bebe21375205489ef3
+  data.tar.gz: fdbea80d5c55c1059aedc568ec593121517cf0cbcf55d145fbe123c121b66c2a
 SHA512:
-  metadata.gz: 5f9c343b0ddb3a58a78af386715c8c99d905e2161905684438972fc177c8943d2c9a51ebadeffb9393162d83e914436ea134e5c78a3affe8db379f3a65fc3a65
-  data.tar.gz: cc6fda69757f79b5a53de36f0ead0315cf1598a4cc372f4e8d2c71cd021c1cf7370aacb583111cdaf0e5bcd711eb445c2d7c673705578217599bd94e7b1fc2ec
+  metadata.gz: 37165c8087b6119d7cd9ce3517bfd12b1c5f52c8c27b7c4eab12351f47034d46b2e86c7fe72820e6b425e02e7f643494e74766995805d1052469fdfaec7f47f2
+  data.tar.gz: d45c1d607c7e35f938ca6fc3658ec082020fcdcfb6888c2a60a7e080e508c7430eedcf9f7eddcf78dad2320f3dfae2803de975d8de9f7cbfb66282944547c5f1

data/CHANGELOG.md

@@ -4,8 +4,8 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [v0.5.1] - 2025-11-08
-### :sparkles: New Features
-- [`726419e`](https://github.com/slidict/coelacanth/commit/726419e6b5fcb6b887d3881532328fdab25e3f0d) - expose standalone morphological analysis *(commit by [@yubele](https://github.com/yubele))*
+## [v0.7.0] - 2026-07-08
+### :wrench: Chores
+- [`ab6d634`](https://github.com/slidict/coelacanth/commit/ab6d634abbc5f954b4b9861e19e3420ee034fd77) - bump version to 0.7.0 *(commit by [@abechan1](https://github.com/abechan1))*
 
-[v0.5.1]: https://github.com/slidict/coelacanth/compare/v0.5.0...v0.5.1
+[v0.7.0]: https://github.com/slidict/coelacanth/compare/v0.6.1...v0.7.0

data/README.md

@@ -32,6 +32,7 @@ it into contemporary ingestion stacks without bespoke glue code.
   multi-byte sources.
 - **Screenshot capture** – Fetches full-page PNGs via a configurable browser client so you can archive visual context alongside
   the extracted text.
+- **PDF generation** – Renders any URL to PDF through Ferrum so you can persist printer-friendly snapshots of source pages.
 - **Redirect resolution** – Follows HTTP redirects and long redirect chains to guarantee the extractor works on the final
   landing page.
 - **Configurable HTTP headers** – Inject custom headers (user agent, authorization, etc.) into the remote browser session for
@@ -83,6 +84,10 @@ result[:dom]        # => Oga DOM representation for downstream processing
 result[:screenshot] # => PNG screenshot as a binary string
 result[:response]   # => HTTP status, headers, and final URL
 
+pdf = Coelacanth.pdf("https://example.com/article")
+File.binwrite("article.pdf", pdf)
+```
+
 # Plain-text morphology
 
 You can run the morphological analyzer without fetching a page by passing plain
@@ -96,7 +101,6 @@ Coelacanth.morphological_analysis("これはテストです。 Testing morpholog
 #      ...
 #    ]
 ```
-```
 
 The returned hash includes:
 
@@ -143,7 +147,7 @@ Runtime configuration is stored in `config/coelacanth.yml`. Environments inherit
 
 ```yaml
 development:
-  client: "ferrum" # Options: "ferrum", "screenshot_one"
+  client: "ferrum" # Options: "ferrum", "screenshot_one", "gotenberg"
   remote_client:
     ws_url: "ws://chrome:3000/chrome"
     timeout: 10
@@ -155,6 +159,13 @@ development:
       User-Agent: "<%= ENV.fetch("COELACANTH_REMOTE_CLIENT_USER_AGENT", "Coelacanth Chrome Extension") %>"
   screenshot_one:
     key: "<%= ENV.fetch("COELACANTH_SCREENSHOT_ONE_API_KEY", "your_screenshot_one_api_key_here") %>"
+  gotenberg:
+    url: "<%= ENV.fetch("COELACANTH_GOTENBERG_URL", "http://gotenberg:3000") %>"
+    open_timeout: 5
+    read_timeout: 30
+    wait_delay: "<%= ENV.fetch("COELACANTH_GOTENBERG_WAIT_DELAY", "") %>"
+    user_agent: "<%= ENV.fetch("COELACANTH_GOTENBERG_USER_AGENT", "") %>"
+    extra_http_headers:
   youtube:
     api_key: "<%= ENV.fetch("COELACANTH_YOUTUBE_API_KEY", "") %>"
   morphology:
@@ -171,6 +182,11 @@ development:
 - **Ferrum client** – Requires a running Chrome instance that exposes the DevTools protocol via WebSocket. Configure the URL,
   timeout, the network idle timeout, and any headers to inject.
 - **ScreenshotOne client** – Supply an API key to offload screenshot capture to [ScreenshotOne](https://screenshotone.com/).
+- **Gotenberg client** – Set `client: "gotenberg"` to capture screenshots through Gotenberg's Chromium URL screenshot
+  endpoint. Configure `gotenberg.url`, request timeouts, an optional `wait_delay`, an optional screenshot `user_agent`,
+  and optional `extra_http_headers`.
+- **PDF generation** – `Coelacanth.pdf(url)` renders the URL with the Ferrum client and returns the generated PDF as a binary
+  string. Configure `remote_client` with a Chrome-compatible WebSocket endpoint before using this helper.
 - **Eyecatch image extraction** – Representative images are discovered automatically by checking Open Graph/Twitter metadata,
   Schema.org JSON-LD payloads, and high-signal `<img>` elements (hero/cover images, large dimensions, etc.). No manual XPath
   maintenance is required.
@@ -207,6 +223,9 @@ export COELACANTH_REMOTE_CLIENT_AUTHORIZATION="Bearer <token>"
 
 export COELACANTH_REMOTE_CLIENT_USER_AGENT="Coelacanth Chrome Extension"
 export COELACANTH_SCREENSHOT_ONE_API_KEY="your_screenshot_one_api_key_here"
+export COELACANTH_GOTENBERG_URL="http://gotenberg:3000"
+export COELACANTH_GOTENBERG_WAIT_DELAY="2s"
+export COELACANTH_GOTENBERG_USER_AGENT="Coelacanth Chrome Extension"
 export COELACANTH_YOUTUBE_API_KEY="your_youtube_data_api_key"
 ```
 
@@ -230,6 +249,10 @@ YouTube, so non-video pages continue to behave as before.
 When using Docker Compose, you can create a `.env` file or export the variables in your environment so the `app` service picks
 them up automatically.
 
+Docker Compose also starts a `gotenberg` service and passes `COELACANTH_GOTENBERG_URL` to the `app` service by default.
+To try that path locally, set `client: "gotenberg"` in `config/coelacanth.yml` or export the equivalent environment-specific
+configuration before running the app container.
+
 If you are working inside Docker, make sure the `UID` environment variable matches your host user by exporting it in your shell
 startup file:
 

data/compose.yml

@@ -1,25 +1,34 @@
-networks:
-  app-tier:
-    driver: bridge
-services:
-  app:
+networks:
+  app-tier:
+    driver: bridge
+services:
+  app:
     environment:
       - UID=${UID}
       - COELACANTH_REMOTE_CLIENT_AUTHORIZATION=${COELACANTH_REMOTE_CLIENT_AUTHORIZATION:-}
       - COELACANTH_REMOTE_CLIENT_USER_AGENT=${COELACANTH_REMOTE_CLIENT_USER_AGENT:-}
       - COELACANTH_SCREENSHOT_ONE_API_KEY=${COELACANTH_SCREENSHOT_ONE_API_KEY:-}
-    tty: true
-    stdin_open: true
-    build:
-      context: .
-      dockerfile: Dockerfile
-    volumes:
-      - ./:/app:cached
-    working_dir: /app
-    command: bash
-    networks:
-      - app-tier
-  chrome:
-    image: browserless/chrome:latest
-    networks:
-      - app-tier
+      - COELACANTH_GOTENBERG_URL=${COELACANTH_GOTENBERG_URL:-http://gotenberg:3000}
+      - COELACANTH_GOTENBERG_WAIT_DELAY=${COELACANTH_GOTENBERG_WAIT_DELAY:-}
+      - COELACANTH_GOTENBERG_USER_AGENT=${COELACANTH_GOTENBERG_USER_AGENT:-}
+    tty: true
+    stdin_open: true
+    build:
+      context: .
+      dockerfile: Dockerfile
+    volumes:
+      - ./:/app:cached
+    working_dir: /app
+    command: bash
+    depends_on:
+      - gotenberg
+    networks:
+      - app-tier
+  chrome:
+    image: browserless/chrome:latest
+    networks:
+      - app-tier
+  gotenberg:
+    image: gotenberg/gotenberg:8
+    networks:
+      - app-tier

data/config/coelacanth.yml

@@ -1,5 +1,5 @@
 development: &development
-  client: "ferrum" # Options: "ferrum", "screenshot_one"
+  client: "ferrum" # Options: "ferrum", "screenshot_one", "gotenberg"
   remote_client:
     ws_url: "ws://chrome:3000/chrome"
     timeout: 10 # seconds
@@ -11,6 +11,13 @@ development: &development
       User-Agent: "<%= ENV.fetch("COELACANTH_REMOTE_CLIENT_USER_AGENT", "Coelacanth Chrome Extension") %>"
   screenshot_one:
     key: "<%= ENV.fetch("COELACANTH_SCREENSHOT_ONE_API_KEY", "your_screenshot_one_api_key_here") %>"
+  gotenberg:
+    url: "<%= ENV.fetch("COELACANTH_GOTENBERG_URL", "http://gotenberg:3000") %>"
+    open_timeout: 5
+    read_timeout: 30
+    wait_delay: "<%= ENV.fetch("COELACANTH_GOTENBERG_WAIT_DELAY", "") %>"
+    user_agent: "<%= ENV.fetch("COELACANTH_GOTENBERG_USER_AGENT", "") %>"
+    extra_http_headers:
   youtube:
     api_key: "<%= ENV.fetch("COELACANTH_YOUTUBE_API_KEY", "") %>"
   morphology:

data/lib/coelacanth/client/base.rb

@@ -23,5 +23,9 @@ module Coelacanth::Client
     def get_screenshot
       raise "Must be implemented in subclass"
     end
+
+    def get_pdf
+      raise "Must be implemented in subclass"
+    end
   end
 end

data/lib/coelacanth/client/ferrum.rb

@@ -5,8 +5,8 @@ require "ferrum"
 module Coelacanth::Client
   # Coelacanth::Client
   class Ferrum < Coelacanth::Client::Base
-    def initialize(url)
-      super(url)
+    def initialize(url, config = Coelacanth.config)
+      super(url, config)
       remote_client.goto(url)
     end
 
@@ -29,6 +29,13 @@ module Coelacanth::Client
       raise sanitized_remote_client_error(e)
     end
 
+    def get_pdf
+      wait_for_network_idle
+      remote_client.pdf
+    rescue => e
+      raise sanitized_remote_client_error(e)
+    end
+
     private
 
     def sanitized_remote_client_error(error)

data/lib/coelacanth/client/gotenberg.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+require "json"
+require "net/http"
+require "uri"
+
+require_relative "../http"
+
+module Coelacanth::Client
+  # Client for capturing screenshots through a Gotenberg Chromium service.
+  class Gotenberg < Coelacanth::Client::Base
+    SCREENSHOT_URL_PATH = "/forms/chromium/screenshot/url"
+
+    def get_response
+      uri = URI.parse(@url)
+      response = Coelacanth::HTTP.get_response(
+        uri,
+        open_timeout: Coelacanth::HTTP::DEFAULT_OPEN_TIMEOUT,
+        read_timeout: Coelacanth::HTTP::DEFAULT_READ_TIMEOUT
+      )
+      @origin_response = response
+      @status_code = response.code.to_i
+
+      return response.body if response.is_a?(Net::HTTPSuccess)
+
+      Coelacanth::HTTP.raise_http_error(uri, response)
+    end
+
+    def get_screenshot
+      response = Net::HTTP.start(
+        endpoint_uri.host,
+        endpoint_uri.port,
+        use_ssl: endpoint_uri.scheme == "https",
+        open_timeout: open_timeout,
+        read_timeout: read_timeout
+      ) do |http|
+        http.request(screenshot_request)
+      end
+
+      return response.body if response.is_a?(Net::HTTPSuccess)
+
+      raise "Failed to fetch screenshot from Gotenberg: #{response.code} #{response.message}"
+    rescue Net::OpenTimeout, Net::ReadTimeout, Timeout::Error => e
+      raise Coelacanth::TimeoutError, "Gotenberg screenshot request timed out: #{e.message}"
+    end
+
+    private
+
+    def screenshot_request
+      request = Net::HTTP::Post.new(endpoint_uri)
+      request.set_form(gotenberg_form_fields, "multipart/form-data")
+      request
+    end
+
+    def gotenberg_form_fields
+      fields = [["url", @url]]
+      wait_delay = @config.read("gotenberg.wait_delay")
+      user_agent = @config.read("gotenberg.user_agent")
+      extra_http_headers = @config.read("gotenberg.extra_http_headers")
+
+      fields << ["waitDelay", wait_delay] if present?(wait_delay)
+      fields << ["userAgent", user_agent] if present?(user_agent)
+      if extra_http_headers && extra_http_headers.any?
+        fields << ["extraHttpHeaders", extra_http_headers.to_json]
+      end
+      fields
+    end
+
+    def endpoint_uri
+      @endpoint_uri ||= URI.join(base_url, SCREENSHOT_URL_PATH)
+    end
+
+    def base_url
+      @config.read("gotenberg.url") || "http://localhost:3000"
+    end
+
+    def open_timeout
+      @config.read("gotenberg.open_timeout") || Coelacanth::HTTP::DEFAULT_OPEN_TIMEOUT
+    end
+
+    def read_timeout
+      @config.read("gotenberg.read_timeout") || 30
+    end
+
+    def present?(value)
+      !value.nil? && value.to_s.strip != ""
+    end
+  end
+end

data/lib/coelacanth/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Coelacanth
-  VERSION = "0.5.1"
+  VERSION = "0.7.0"
 end

data/lib/coelacanth.rb

@@ -4,6 +4,7 @@ require "net/http"
 require_relative "coelacanth/configure"
 require_relative "coelacanth/client/base"
 require_relative "coelacanth/client/ferrum"
+require_relative "coelacanth/client/gotenberg"
 require_relative "coelacanth/client/screenshot_one"
 require_relative "coelacanth/dom"
 require_relative "coelacanth/extractor"
@@ -21,7 +22,7 @@ module Coelacanth
   class RobotsDisallowedError < StandardError; end
 
   def self.analyze(url)
-    client_class = config.read("client") == "screenshot_one" ? Client::ScreenshotOne : Client::Ferrum
+    client_class = client_class_for(config.read("client"))
     @client = client_class.new(url)
     regular_url = Redirect.new.resolve_redirect(url)
     response = begin
@@ -47,6 +48,21 @@ module Coelacanth
     }
   end
 
+  def self.pdf(url)
+    Client::Ferrum.new(url).get_pdf
+  end
+
+  def self.client_class_for(client_name)
+    case client_name
+    when "screenshot_one"
+      Client::ScreenshotOne
+    when "gotenberg"
+      Client::Gotenberg
+    else
+      Client::Ferrum
+    end
+  end
+
   def self.config
     @config ||= Configure.new
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: coelacanth
 version: !ruby/object:Gem::Version
-  version: 0.5.1
+  version: 0.7.0
 platform: ruby
 authors:
 - Yusuke
@@ -33,6 +33,7 @@ files:
 - lib/coelacanth.rb
 - lib/coelacanth/client/base.rb
 - lib/coelacanth/client/ferrum.rb
+- lib/coelacanth/client/gotenberg.rb
 - lib/coelacanth/client/screenshot_one.rb
 - lib/coelacanth/configure.rb
 - lib/coelacanth/dom.rb