makuri 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: efda242d320f75d0c7d483bd71f65f638a62624d0319543386b085914610010e
-  data.tar.gz: 12a830ef4e4a72926ee91e50f378fc0170acfd491bbc6c6a16dee08c0116288e
+  metadata.gz: '08d39eda46b02f5f6894d581eb0d837ac4680f4c2f6239b6f585a4c187ed44d7'
+  data.tar.gz: 433093581b01eb5327e1bb8e6dd6653f0b51fcfdae54276a914339b0bff61b2f
 SHA512:
-  metadata.gz: aaa1e7fb93b2cd25892dea2d34ea0e9edce13a6ac31aed7dc44b62b49b8b5080d4515eb903cba916d2a9634101f37980a872035f41a5b3632f2a608dd6e936b7
-  data.tar.gz: e32c252ed9940070600ce4f08ff9faa1ff93f5e225c674a5ad501ae5d4f9a6c3c02cdfd935d49284e981651fa86126298dcd1b06783ced0e85ef699f6d5dddad
+  metadata.gz: daaf82155b49cc1efe7da7d092c617fb47e8948758108afbea79a5d484a08757781a2f156d066946d821c0c25b5984693b69dbdde764bf2dd14c07a9997d3112
+  data.tar.gz: e8655c684cfa1087d1046a7cf0447f8a4b52ee8d17c6b25028c78d8f50bfa8df0a076ecd38f0cb504311dc1b55bae53b199fea2bc5982b65bf5b9b9d10221e33

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.4.0 (2024-06-13)
+- Add Ferrum support
+- Remove Capybara/chrome support
+- Bug fixes
+
 ## 0.3.0 (2021-02-07)
 - Allow scrape from multiple start_urls
 - Add :engine support to Spider

data/lib/makuri/browser.rb

@@ -24,23 +24,26 @@ module Makuri
       @url = url
       validate_url
 
-      return follow if engine_chrome?
+      return follow if ['Ferrum', 'Chrome'].include? browser_engine
 
       @request_method = options.fetch(:method, request_method)
       @request_body   = options.fetch(:body, request_body)
       browser.visit(@url)
     end
 
+    def quit
+      browser.respond_to?(:quit) ? browser.quit : nil
+    end
+
+    private
+
     # Only allow for capybara get request
     def follow
       browser.visit(@url)
       browser
     end
 
-    private
-
     def create_browser
-      browser_engine = engine_chrome? ? 'Chrome' : 'NetHttp'
       builder        = Object.const_get "Makuri::BrowserBuilder::#{browser_engine}"
       builder.new(browser_params).build
     end
@@ -59,8 +62,13 @@ module Makuri
       raise 'Invalid URL supplied' unless uri.is_a?(URI::HTTP) && !uri.host.nil?
     end
 
-    def engine_chrome?
-      engine == :chrome
+    def browser_engine
+      case engine
+      when :ferrum
+        'Ferrum'
+      else
+        'NetHttp'
+      end
     end
   end
 end

data/lib/makuri/browser_builder/ferrum.rb

@@ -0,0 +1,44 @@
+require 'ferrum'
+
+module Makuri::BrowserBuilder
+  class Ferrum < Base
+    attr_accessor :headless, :html
+
+    def initialize(options = {})
+      super
+      @headless      = options.fetch(:headless, true)
+    end
+
+    def build
+      headers = { 'User-Agent': user_agent }
+      @ferrum = ::Ferrum::Browser.new(headless: @headless, headers: headers)
+      self
+    end
+
+    def visit(url)
+      @ferrum.go_to url
+      @html = @ferrum.body
+      self
+    end
+
+    def quit
+      @ferrum.quit
+    end
+
+    private
+
+    # TODO:
+    # def browser_options
+    #   args = %w[
+    #     --disable-gpu
+    #     --no-sandbox
+    #     --disable-translate
+    #     --ignore-certificate-errors
+    #   ]
+    #   args << '--headless' if headless
+    #   args << "--user-agent=#{user_agent}"
+    #   args << "--blink-settings=imagesEnabled=#{enable_images}"
+    #   Selenium::WebDriver::Chrome::Options.new(args: args)
+    # end
+  end
+end

data/lib/makuri/spider.rb

@@ -20,28 +20,37 @@ module Makuri
 
       def spider_options(**options)
         @engine = options.fetch(:engine, :net_http)
+        @headless = options.fetch(:headless, true)
       end
 
       def run
         raise "Start URLs not found. Define start_urls for #{self}." unless defined? @start_urls
 
         @engine ||= :net_http
+        @headless = defined?(@headless) ? @headless : true
 
-        @start_urls.each { |start_url| new(start_url: start_url, engine: @engine).parse }
+        @start_urls.each { |start_url| new(start_url: start_url, engine: @engine, headless: @headless).parse }
       end
     end
 
-    attr_accessor :engine, :response
+    attr_accessor :engine, :headless, :response
 
     def initialize(**config)
       @start_url = config.fetch(:start_url, nil)
       @engine    = config.fetch(:engine, :net_http)
+      @headless  = config.fetch(:headless, true)
 
       update_response(@start_url)
+      browser_quit_wrapper { parse }
     end
 
     def browser
-      @browser ||= Makuri::Browser.new(engine: engine)
+      @browser ||= Makuri::Browser.new(engine: engine, headless: headless)
+    end
+
+    def browser_quit_wrapper
+      yield
+      browser.quit
     end
 
     def parse
@@ -77,7 +86,6 @@ module Makuri
 
     def update_response(url)
       res = browser.request(absolute_url(url)).html
-      # res = res.html if @engine == :chrome
       @response = Nokogiri::HTML(res)
     end
   end

data/lib/makuri/version.rb

@@ -1,3 +1,3 @@
 module Makuri
-  VERSION = '0.3.0'.freeze
+  VERSION = '0.4.0'.freeze
 end

data/lib/makuri.rb

@@ -5,7 +5,7 @@ require 'makuri/spider'
 require 'makuri/version'
 
 require 'makuri/browser_builder/base'
-require 'makuri/browser_builder/chrome'
+require 'makuri/browser_builder/ferrum'
 require 'makuri/browser_builder/net_http'
 
 module Makuri

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: makuri
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Lal Saud
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-02-07 00:00:00.000000000 Z
+date: 2024-06-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -16,98 +16,70 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.7'
+        version: '2.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.7'
-- !ruby/object:Gem::Dependency
-  name: capybara
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.34'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.34'
+        version: '2.8'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.10.10
+        version: 1.15.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.10.10
+        version: 1.15.5
 - !ruby/object:Gem::Dependency
-  name: selenium-webdriver
+  name: ferrum
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.5'
+        version: '0.15'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.5'
+        version: '0.15'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.0'
-- !ruby/object:Gem::Dependency
-  name: webdrivers
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '4.0'
+        version: 2.5.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.0'
+        version: 2.5.0
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.11'
+        version: '3.20'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.11'
+        version: '3.20'
 description:
 email: lalusaud@gmail.com
 executables: []
@@ -120,7 +92,7 @@ files:
 - lib/makuri.rb
 - lib/makuri/browser.rb
 - lib/makuri/browser_builder/base.rb
-- lib/makuri/browser_builder/chrome.rb
+- lib/makuri/browser_builder/ferrum.rb
 - lib/makuri/browser_builder/net_http.rb
 - lib/makuri/spider.rb
 - lib/makuri/version.rb
@@ -136,14 +108,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.5'
+      version: '2.7'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.5.11
 signing_key:
 specification_version: 4
 summary: Web-crawling framework for Ruby

data/lib/makuri/browser_builder/chrome.rb

@@ -1,37 +0,0 @@
-require 'capybara'
-require 'selenium-webdriver'
-
-module Makuri::BrowserBuilder
-  class Chrome < Base
-    attr_accessor :headless, :enable_images
-
-    def initialize(options = {})
-      super
-      @headless      = options.fetch(:headless, true)
-      @enable_images = options.fetch(:enable_images, true)
-    end
-
-    def build
-      Capybara.register_driver :selenium_chrome do |app|
-        Capybara::Selenium::Driver.new app, browser: :chrome, options: browser_options
-      end
-      Capybara.threadsafe = true
-      Capybara::Session.new :selenium_chrome
-    end
-
-    private
-
-    def browser_options
-      args = %w[
-        --disable-gpu
-        --no-sandbox
-        --disable-translate
-        --ignore-certificate-errors
-      ]
-      args << '--headless' if headless
-      args << "--user-agent=#{user_agent}"
-      args << "--blink-settings=imagesEnabled=#{enable_images}"
-      Selenium::WebDriver::Chrome::Options.new(args: args)
-    end
-  end
-end