coelacanth 0.1.6 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '0172679388ce0d8718122988de86589b623e0ab432f9e08b589dd22e2009f74a'
-  data.tar.gz: baa0edff8a94564e33cc8596e44cfe49a040dd920798a10bbca824c2ce4d1f89
+  metadata.gz: 506450022b367ef2795c9d67d66548b1893757dd740512286fb90f9b952f21d1
+  data.tar.gz: df860c4216846549ac335b15dbef4cd9f99aa068a464a47b58ba973ba9d3b771
 SHA512:
-  metadata.gz: 16564c8664acf81dddb9188f828ab7e59da5969c610302e64325e6233f928a908e404332b244c48f3b9e0a7e984d88d3713ca885600cd1cd9a6e3057e273630a
-  data.tar.gz: 543fff73d944bf4b328e0bd3cebf975b9aaa093e2553d36c61ef09cefd1cc76ac997387cebba43580f059d497899006107a6bb0473afc2524f10423fa3cfad54
+  metadata.gz: f4f483791381bf5672c74c382c73129703983c936720184cdd870db211cb740bb5ff6685460c6152213eea8dfcce85c6666ffa189d23e3b58e92c183658b486e
+  data.tar.gz: 577737cc29a81134a288bb3bee66a220d46f80749a9824ff5b487b62dfe97a8f1209124266715c58d9021e7e88d47e634f675462e9d27e62bd8e3e93c532e31d

data/Gemfile

@@ -6,7 +6,7 @@ source "https://rubygems.org"
 gemspec
 
 gem "ferrum", "~> 0.15"
-gem "rake", "~> 13.0"
+gem "rake", "~> 13.2"
 gem "rspec", "~> 3.0"
 gem "rubocop", "~> 1.21"
 gem "oga", "~> 3.4"

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    coelacanth (0.1.6)
+    coelacanth (0.2.2)
 
 GEM
   remote: https://rubygems.org/
@@ -11,59 +11,57 @@ GEM
     ansi (1.5.0)
     ast (2.4.2)
     concurrent-ruby (1.3.4)
-    diff-lcs (1.4.4)
+    diff-lcs (1.5.1)
     ferrum (0.15)
       addressable (~> 2.5)
       concurrent-ruby (~> 1.1)
       webrick (~> 1.7)
       websocket-driver (~> 0.7)
-    json (2.7.1)
+    json (2.7.2)
     language_server-protocol (3.17.0.3)
     oga (3.4)
       ast
       ruby-ll (~> 2.1)
-    parallel (1.24.0)
-    parser (3.3.0.5)
+    parallel (1.26.3)
+    parser (3.3.5.0)
       ast (~> 2.4.1)
       racc
     public_suffix (6.0.1)
-    racc (1.7.3)
+    racc (1.8.1)
     rainbow (3.1.1)
-    rake (13.1.0)
-    regexp_parser (2.9.0)
-    rexml (3.2.6)
-    rspec (3.10.0)
-      rspec-core (~> 3.10.0)
-      rspec-expectations (~> 3.10.0)
-      rspec-mocks (~> 3.10.0)
-    rspec-core (3.10.1)
-      rspec-support (~> 3.10.0)
-    rspec-expectations (3.10.1)
+    rake (13.2.1)
+    regexp_parser (2.9.2)
+    rspec (3.13.0)
+      rspec-core (~> 3.13.0)
+      rspec-expectations (~> 3.13.0)
+      rspec-mocks (~> 3.13.0)
+    rspec-core (3.13.2)
+      rspec-support (~> 3.13.0)
+    rspec-expectations (3.13.3)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-mocks (3.10.2)
+      rspec-support (~> 3.13.0)
+    rspec-mocks (3.13.2)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-support (3.10.2)
-    rubocop (1.62.1)
+      rspec-support (~> 3.13.0)
+    rspec-support (3.13.1)
+    rubocop (1.67.0)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)
       parser (>= 3.3.0.2)
       rainbow (>= 2.2.2, < 4.0)
-      regexp_parser (>= 1.8, < 3.0)
-      rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.31.1, < 2.0)
+      regexp_parser (>= 2.4, < 3.0)
+      rubocop-ast (>= 1.32.2, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 3.0)
-    rubocop-ast (1.31.2)
-      parser (>= 3.3.0.4)
+    rubocop-ast (1.32.3)
+      parser (>= 3.3.1.0)
     ruby-ll (2.1.3)
       ansi
       ast
     ruby-progressbar (1.13.0)
-    unicode-display_width (2.5.0)
-    webrick (1.8.1)
+    unicode-display_width (2.6.0)
+    webrick (1.8.2)
     websocket-driver (0.7.6)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)
@@ -76,9 +74,9 @@ DEPENDENCIES
   coelacanth!
   ferrum (~> 0.15)
   oga (~> 3.4)
-  rake (~> 13.0)
+  rake (~> 13.2)
   rspec (~> 3.0)
   rubocop (~> 1.21)
 
 BUNDLED WITH
-   2.5.7
+   2.5.17

data/README.md

@@ -50,11 +50,12 @@ Then, you can easily parse and extract information from a web page like this:
 
 ```ruby
 url = "https://example.com"
-stats = Coelacanth.analyze(url) # Not yet implement
+stats = Coelacanth.analyze(url)
 ```
 
 ## Features
-- More features coming soon!
+- Get dom by oga
+- Get screenshot
 
 ## Commit Message Guidelines
 

data/compose.yml

@@ -18,6 +18,5 @@ services:
       - app-tier
   chrome:
     image: browserless/chrome:latest
-    container_name: chrome
     networks:
       - app-tier

data/config/coelacanth.yml

@@ -1,5 +1,4 @@
 development: &development
-  use_remote_client: false
   remote_client:
     ws_url: "ws://chrome:3000/chrome"
     timeout: 10 # seconds

data/lib/coelacanth/client.rb

@@ -1,76 +1,34 @@
 # frozen_string_literal: true
 
 require "ferrum"
-require "oga"
 
 module Coelacanth
   # Coelacanth::Client
   class Client
-    def initialize(url = nil)
+    def initialize(url)
+      @validator = Validator.new
+      raise URI::InvalidURIError unless @validator.valid_url?(url)
       @config = Coelacanth.config
-      @url = url if url && valid_url?(url)
-    end
-
-    def valid_url?(url = nil)
-      @url = url if url
-      uri = URI.parse(@url)
-      uri.is_a?(URI::HTTP) || uri.is_a?(URI::HTTPS)
-    rescue URI::InvalidURIError
-      false
-    end
-
-    def resolve_redirect(url = nil, limit = 10)
-      @url = url if url && valid_url?(url)
-      raise Coelacanth::DeepRedirectError, "Too many redirect" if limit.zero?
-      raise Coelacanth::RedirectError, "Url or location is nil" if @url.nil?
-
-      get_response(@url)
-      handle_response(@origin_response, limit)
-    end
-
-    def oga(url = nil)
-      @url = url if url && valid_url?(url)
-      Oga.parse_xml(get_response(@url))
+      remote_client.goto(url)
     end
 
     def get_response(url = nil)
-      @url = url if url && valid_url?(url)
-      if @config.read("use_remote_client")
-        response_by_remote_client
-      else
-        response_by_net_http
-      end
-    end
-
-    private
-
-    def handle_response(response, limit)
-      codes = Net::HTTPResponse::CODE_CLASS_TO_OBJ.invert
-      case @status_code.to_s
-      when /^#{codes[Net::HTTPSuccess]}\d\d$/
-        @url
-      when /^#{codes[Net::HTTPRedirection]}\d\d$/
-        @url = response["location"]
-        resolve_redirect(response["location"], limit - 1)
-      else
-        raise Coelacanth::RedirectError
-      end
-    end
-
-    def response_by_remote_client
-      remote_client.goto(@url)
       @status_code = remote_client.network.status
       @origin_response = remote_client
-      remote_client.body
+      body = remote_client.body
+      page.network.wait_for_idle! # might raise an error
+      body
     end
 
-    def response_by_net_http
-      response = Net::HTTP.get_response(URI.parse(@url))
-      @status_code = response.code
-      @origin_response = response
-      response.body
+    def get_screenshot
+      tempfile = Tempfile.new
+      remote_client.screenshot(path: tempfile.path, format: "png")
+      page.network.wait_for_idle! # might raise an error
+      File.read(tempfile.path)
     end
 
+    private
+
     def remote_client
       if @remote_client.nil?
         headers = @config.read("remote_client.headers")

data/lib/coelacanth/dom.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require "oga"
+
+module Coelacanth
+  # Coelacanth::Dom
+  class Dom
+    def oga(url)
+      Oga.parse_xml(Client.new(url).get_response)
+    end
+  end
+end

data/lib/coelacanth/redirect.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require "ferrum"
+require "oga"
+
+module Coelacanth
+  # Coelacanth::Redirect
+  class Redirect
+    def resolve_redirect(url, limit = 10)
+      @url = url if url && Validator.new.valid_url?(url)
+      raise Coelacanth::DeepRedirectError, "Too many redirect" if limit.zero?
+      raise Coelacanth::RedirectError, "Url or location is nil" if @url.nil?
+
+      response = Net::HTTP.get_response(URI.parse(@url))
+      @status_code = response.code
+      @origin_response = response
+
+      handle_response(@origin_response, limit)
+    end
+
+    private
+
+    def handle_response(response, limit)
+      codes = Net::HTTPResponse::CODE_CLASS_TO_OBJ.invert
+      case @status_code.to_s
+      when /^#{codes[Net::HTTPSuccess]}\d\d$/
+        @url
+      when /^#{codes[Net::HTTPRedirection]}\d\d$/
+        @url = response["location"]
+        resolve_redirect(response["location"], limit - 1)
+      else
+        raise Coelacanth::RedirectError
+      end
+    end
+  end
+end

data/lib/coelacanth/validator.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require "ferrum"
+
+module Coelacanth
+  # Coelacanth::Validator
+  class Validator
+    def valid_url?(url)
+      uri = URI.parse(url)
+      uri.is_a?(URI::HTTP) || uri.is_a?(URI::HTTPS)
+    rescue URI::InvalidURIError
+      false
+    end
+  end
+end

data/lib/coelacanth/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Coelacanth
-  VERSION = "0.1.6"
+  VERSION = "0.2.2"
 end

data/lib/coelacanth.rb

@@ -1,9 +1,12 @@
 # frozen_string_literal: true
 
 require "net/http"
-require_relative "coelacanth/version"
 require_relative "coelacanth/configure"
 require_relative "coelacanth/client"
+require_relative "coelacanth/dom"
+require_relative "coelacanth/redirect"
+require_relative "coelacanth/validator"
+require_relative "coelacanth/version"
 
 # Coelacanth
 module Coelacanth
@@ -13,10 +16,10 @@ module Coelacanth
 
   def self.analyze(url)
     @client = Client.new(url)
-    @client.resolve_redirect
+    regular_url = Redirect.new.resolve_redirect(url)
     {
-      remote_client: @config.read("use_remote_client"),
-      oga: @client.oga
+      dom: Dom.new.oga(regular_url),
+      screenshot: @client.get_screenshot,
     }
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: coelacanth
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.2.2
 platform: ruby
 authors:
 - Yusuke
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-08-29 00:00:00.000000000 Z
+date: 2024-10-23 00:00:00.000000000 Z
 dependencies: []
 description: |
   coelacanth is a gem that allows you to easily parse and analyze web pages,
@@ -21,7 +21,6 @@ extra_rdoc_files: []
 files:
 - ".rspec"
 - ".rubocop.yml"
-- CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Dockerfile
 - Gemfile
@@ -34,6 +33,9 @@ files:
 - lib/coelacanth.rb
 - lib/coelacanth/client.rb
 - lib/coelacanth/configure.rb
+- lib/coelacanth/dom.rb
+- lib/coelacanth/redirect.rb
+- lib/coelacanth/validator.rb
 - lib/coelacanth/version.rb
 homepage: https://github.com/slidict/coelacanth
 licenses:

data/CHANGELOG.md

@@ -1,5 +0,0 @@
-## [Unreleased]
-
-## [0.1.0] - 2024-08-24
-
-- Initial release