coelacanth 0.2.3 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7d92c04c82f120570d29b58787a36897bb769d789dfd5ecbf5f7da725c08710a
-  data.tar.gz: d7e25b8f1eb0e269dec2d6cf8c0c9209693b2f068ebf389ee61f3450fe7db3b2
+  metadata.gz: dc0f98072fb81b09973a8acbee60d8f9b34c31cdb75e161f7d2f1e8f2bd9ee3c
+  data.tar.gz: fba2700643da179439dc2588717bb78b06fb6d9ef056c262ae792084e81ab4eb
 SHA512:
-  metadata.gz: 432f908bf812f69f6398e0346d6f7a104391f56c1057adaa14de06457f2512ddd695b58f2b96ad90ca9063ea533bc54429f4b199c5dea541901dc2839c645143
-  data.tar.gz: 1fdbe4cbe2d354d15d04fb2ba4fd3fd04f2f6c966d91123f443b75831d28aa977c691864bb3d84e1a6922878dbc0120d88d1cdc54ae0390f3052e6331f7dd12f
+  metadata.gz: 0a7c6bd212ba78c390798d9b522f5f3f5927941a95eec56ec20e78c0d4744756cd8b3a5cc8eada99f8e5f4539309785d7b8dec1c6ab0454201450e700063016f
+  data.tar.gz: 7144aaf89b5ad33ebb8858e866c639c70da8aaffa9d028b7fe898d4fdf0c4d2cb65c4793d96af8bfa4cc2c595b1cb4559927dceda1368323b8d98cfb215363fb

data/Gemfile

@@ -5,8 +5,14 @@ source "https://rubygems.org"
 # Specify your gem's dependencies in coelacanth.gemspec
 gemspec
 
-gem "ferrum", "~> 0.15"
+gem "ferrum", "~> 0.16"
 gem "rake", "~> 13.2"
 gem "rspec", "~> 3.0"
 gem "rubocop", "~> 1.21"
 gem "oga", "~> 3.4"
+gem "base64", "~> 0.2.0"
+
+group :development, :test do
+  gem "webmock", "~> 3.25"
+end
+

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    coelacanth (0.2.3)
+    coelacanth (0.3.1)
 
 GEM
   remote: https://rubygems.org/
@@ -9,33 +9,43 @@ GEM
     addressable (2.8.7)
       public_suffix (>= 2.0.2, < 7.0)
     ansi (1.5.0)
-    ast (2.4.2)
-    concurrent-ruby (1.3.4)
-    diff-lcs (1.5.1)
-    ferrum (0.15)
+    ast (2.4.3)
+    base64 (0.2.0)
+    bigdecimal (3.1.9)
+    concurrent-ruby (1.3.5)
+    crack (1.0.0)
+      bigdecimal
+      rexml
+    diff-lcs (1.6.1)
+    ferrum (0.16)
       addressable (~> 2.5)
+      base64 (~> 0.2)
       concurrent-ruby (~> 1.1)
       webrick (~> 1.7)
       websocket-driver (~> 0.7)
-    json (2.7.2)
-    language_server-protocol (3.17.0.3)
+    hashdiff (1.1.2)
+    json (2.10.2)
+    language_server-protocol (3.17.0.4)
+    lint_roller (1.1.0)
     oga (3.4)
       ast
       ruby-ll (~> 2.1)
-    parallel (1.26.3)
-    parser (3.3.5.0)
+    parallel (1.27.0)
+    parser (3.3.8.0)
       ast (~> 2.4.1)
       racc
+    prism (1.4.0)
     public_suffix (6.0.1)
     racc (1.8.1)
     rainbow (3.1.1)
     rake (13.2.1)
-    regexp_parser (2.9.2)
+    regexp_parser (2.10.0)
+    rexml (3.4.1)
     rspec (3.13.0)
       rspec-core (~> 3.13.0)
       rspec-expectations (~> 3.13.0)
       rspec-mocks (~> 3.13.0)
-    rspec-core (3.13.2)
+    rspec-core (3.13.3)
       rspec-support (~> 3.13.0)
     rspec-expectations (3.13.3)
       diff-lcs (>= 1.2.0, < 2.0)
@@ -43,26 +53,35 @@ GEM
     rspec-mocks (3.13.2)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
-    rspec-support (3.13.1)
-    rubocop (1.67.0)
+    rspec-support (3.13.2)
+    rubocop (1.75.3)
       json (~> 2.3)
-      language_server-protocol (>= 3.17.0)
+      language_server-protocol (~> 3.17.0.2)
+      lint_roller (~> 1.1.0)
       parallel (~> 1.10)
       parser (>= 3.3.0.2)
       rainbow (>= 2.2.2, < 4.0)
-      regexp_parser (>= 2.4, < 3.0)
-      rubocop-ast (>= 1.32.2, < 2.0)
+      regexp_parser (>= 2.9.3, < 3.0)
+      rubocop-ast (>= 1.44.0, < 2.0)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 2.4.0, < 3.0)
-    rubocop-ast (1.32.3)
-      parser (>= 3.3.1.0)
+      unicode-display_width (>= 2.4.0, < 4.0)
+    rubocop-ast (1.44.1)
+      parser (>= 3.3.7.2)
+      prism (~> 1.4)
     ruby-ll (2.1.3)
       ansi
       ast
     ruby-progressbar (1.13.0)
-    unicode-display_width (2.6.0)
-    webrick (1.8.2)
-    websocket-driver (0.7.6)
+    unicode-display_width (3.1.4)
+      unicode-emoji (~> 4.0, >= 4.0.4)
+    unicode-emoji (4.0.4)
+    webmock (3.25.1)
+      addressable (>= 2.8.0)
+      crack (>= 0.3.2)
+      hashdiff (>= 0.4.0, < 2.0.0)
+    webrick (1.9.1)
+    websocket-driver (0.7.7)
+      base64
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)
 
@@ -71,12 +90,14 @@ PLATFORMS
   x86_64-linux
 
 DEPENDENCIES
+  base64 (~> 0.2.0)
   coelacanth!
-  ferrum (~> 0.15)
+  ferrum (~> 0.16)
   oga (~> 3.4)
   rake (~> 13.2)
   rspec (~> 3.0)
   rubocop (~> 1.21)
+  webmock (~> 3.25)
 
 BUNDLED WITH
-   2.5.17
+   2.6.7

data/README.md

@@ -53,6 +53,12 @@ url = "https://example.com"
 stats = Coelacanth.analyze(url)
 ```
 
+- rspec
+
+```
+$ bundle exec rspec
+```
+
 ## Features
 - Get dom by oga
 - Get screenshot

data/config/coelacanth.yml

@@ -1,10 +1,13 @@
 development: &development
+  client: "ferrum" # Options: "ferrum", "screenshot_one"
   remote_client:
     ws_url: "ws://chrome:3000/chrome"
     timeout: 10 # seconds
     headers:
       Authorization: "Bearer 1234567890"
       User-Agent: "Coelacanth Chrome Extension"
+  screenshot_one:
+    key: "your_screenshot_one_api_key_here"
 test:
   <<: *development
 production:

data/lib/coelacanth/client/base.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require "ferrum"
+
+module Coelacanth::Client
+  # Coelacanth::Client
+  class Base
+    def initialize(url, config = Coelacanth.config)
+      @validator = Coelacanth::Validator.new
+      raise URI::InvalidURIError unless @validator.valid_url?(url)
+      @config = config
+      @url = url
+    end
+
+    def client
+      @config.read("client")
+    end
+
+    def get_response(url = nil)
+      raise "Must be implemented in subclass"
+    end
+
+    def get_screenshot
+      raise "Must be implemented in subclass"
+    end
+  end
+end

data/lib/coelacanth/{client.rb → client/ferrum.rb}

@@ -2,13 +2,11 @@
 
 require "ferrum"
 
-module Coelacanth
+module Coelacanth::Client
   # Coelacanth::Client
-  class Client
+  class Ferrum < Coelacanth::Client::Base
     def initialize(url)
-      @validator = Validator.new
-      raise URI::InvalidURIError unless @validator.valid_url?(url)
-      @config = Coelacanth.config
+      super(url)
       remote_client.goto(url)
     end
 
@@ -32,7 +30,7 @@ module Coelacanth
     def remote_client
       if @remote_client.nil?
         headers = @config.read("remote_client.headers")
-        @remote_client = Ferrum::Browser.new(
+        @remote_client = ::Ferrum::Browser.new(
           ws_url: @config.read("remote_client.ws_url"),
           timeout: @config.read("remote_client.timeout")
         ).create_page

data/lib/coelacanth/client/screenshot_one.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require "open-uri"
+require "ferrum"
+
+module Coelacanth::Client
+  # Coelacanth::Client
+  class ScreenshotOne < Coelacanth::Client::Base
+    def get_response
+      @origin_response = URI(@url).open
+      @status_code = @origin_response.status[0].to_i
+      body = @origin_response.read
+      body
+    rescue OpenURI::HTTPError => e
+      @status_code = e.io.status[0].to_i
+      raise e
+    end
+
+    def get_screenshot
+      api_key = @config.read("screenshot_one.key")
+      uri = URI("https://api.screenshotone.com/take")
+      params = {
+        access_key: api_key,
+        url: @url,
+        format: "jpg",
+        block_ads: true,
+        block_cookie_banners: true,
+        block_banners_by_heuristics: false,
+        block_trackers: true,
+        delay: 0,
+        timeout: 60,
+        response_type: "by_format",
+        image_quality: 80
+      }
+      uri.query = URI.encode_www_form(params)
+
+      response = Net::HTTP.get_response(uri)
+      raise "Failed to fetch screenshot: #{response.code}" unless response.is_a?(Net::HTTPSuccess)
+
+      response.body
+    end
+  end
+end

data/lib/coelacanth/dom.rb

@@ -6,7 +6,7 @@ module Coelacanth
   # Coelacanth::Dom
   class Dom
     def oga(url)
-      Oga.parse_xml(Client.new(url).get_response)
+      Oga.parse_xml(Client::Ferrum.new(url).get_response)
     end
   end
 end

data/lib/coelacanth/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Coelacanth
-  VERSION = "0.2.3"
+  VERSION = "0.3.2"
 end

data/lib/coelacanth.rb

@@ -2,7 +2,9 @@
 
 require "net/http"
 require_relative "coelacanth/configure"
-require_relative "coelacanth/client"
+require_relative "coelacanth/client/base"
+require_relative "coelacanth/client/ferrum"
+require_relative "coelacanth/client/screenshot_one"
 require_relative "coelacanth/dom"
 require_relative "coelacanth/redirect"
 require_relative "coelacanth/validator"
@@ -15,7 +17,8 @@ module Coelacanth
   class DeepRedirectError < StandardError; end
 
   def self.analyze(url)
-    @client = Client.new(url)
+    client_class = config.read("client") == "screenshot_one" ? Client::ScreenshotOne : Client::Ferrum
+    @client = client_class.new(url)
     regular_url = Redirect.new.resolve_redirect(url)
     {
       dom: Dom.new.oga(regular_url),

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: coelacanth
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.3.2
 platform: ruby
 authors:
 - Yusuke
-autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-10-23 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies: []
 description: |
   coelacanth is a gem that allows you to easily parse and analyze web pages,
@@ -31,7 +30,9 @@ files:
 - compose.yml
 - config/coelacanth.yml
 - lib/coelacanth.rb
-- lib/coelacanth/client.rb
+- lib/coelacanth/client/base.rb
+- lib/coelacanth/client/ferrum.rb
+- lib/coelacanth/client/screenshot_one.rb
 - lib/coelacanth/configure.rb
 - lib/coelacanth/dom.rb
 - lib/coelacanth/redirect.rb
@@ -44,7 +45,6 @@ metadata:
   homepage_uri: https://github.com/slidict/coelacanth
   source_code_uri: https://github.com/slidict/coelacanth
   changelog_uri: https://github.com/slidict/coelacanth/releases
-post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -59,8 +59,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.1
-signing_key: 
+rubygems_version: 3.6.7
 specification_version: 4
 summary: A gem for analyzing and extracting statistics from web pages.
 test_files: []