makuri 0.2.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f1106d01d72194894872b6fa5409c9f6951f27ff462a1ad81c793dd10c043974
-  data.tar.gz: 2d8ae6ef66759cdcdcf980e9e3b44f840f954db9ea3536b1a9ad3d656fec5e17
+  metadata.gz: '08d39eda46b02f5f6894d581eb0d837ac4680f4c2f6239b6f585a4c187ed44d7'
+  data.tar.gz: 433093581b01eb5327e1bb8e6dd6653f0b51fcfdae54276a914339b0bff61b2f
 SHA512:
-  metadata.gz: ddf5f382fccbcbfff68fa26f3bed25c07accaee86d2d3379b1471ea4690ff431b1f70330e9a29be1d65af64bc03652e06938dc5b31b3d5f506c96a375438c473
-  data.tar.gz: '0046904279c80dd1c9e6db0621dddcbcf8796ed8977940fc79dd9f500287128591cc2206f289ae271bbb0114c38d03ebe4d9a2ad894f4ceb34f99a6768493b6b'
+  metadata.gz: daaf82155b49cc1efe7da7d092c617fb47e8948758108afbea79a5d484a08757781a2f156d066946d821c0c25b5984693b69dbdde764bf2dd14c07a9997d3112
+  data.tar.gz: e8655c684cfa1087d1046a7cf0447f8a4b52ee8d17c6b25028c78d8f50bfa8df0a076ecd38f0cb504311dc1b55bae53b199fea2bc5982b65bf5b9b9d10221e33

data/CHANGELOG.md

@@ -1,10 +1,20 @@
+## 0.4.0 (2024-06-13)
+- Add Ferrum support
+- Remove Capybara/chrome support
+- Bug fixes
+
+## 0.3.0 (2021-02-07)
+- Allow scrape from multiple start_urls
+- Add :engine support to Spider
+- Add :enable_images support to :chrome engine
+- Bug fixes and restructure #request_method & #request_body
+
 ## 0.2.0 (2021-01-25)
 - Add Capybara Support
 - Add :headless option to Browser object
 - Bug fixes & restructure Browser #follow, #request methods
 
 ## 0.1.0 (2021-01-05)
-
 - First Release
 - Add Basic Browser Builders for crawling web pages
 - Add Spider Framework to support scraping

data/lib/makuri/browser.rb

@@ -1,50 +1,50 @@
 module Makuri
   class Browser
-    attr_accessor :url, :js, :headless, :user_agent, :request_method, :request_body
+    attr_accessor :url, :engine, :headless, :user_agent, :request_method, :request_body
 
     def initialize(options = {})
-      @url            = options.fetch(:url, '')
-      @request_method = options.fetch(:request_method, :get)
-      @request_body   = options.fetch(:request_body, {})
-
-      @js             = options.fetch(:js, false)
+      @engine         = options.fetch(:engine, :net_http)
       @headless       = options.fetch(:headless, true)
       @user_agent     = options.fetch(
         :user_agent,
         'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36'
       )
+
+      # Defaults
+      @request_method = :get
+      @request_body   = {}
     end
 
     def browser
       @browser ||= create_browser
     end
 
-    # Only allow for capybara get request
-    def follow(current_url = '')
-      @url = current_url unless current_url.empty?
-      raise 'Invalid URL supplied' if url.empty?
+    # Allow for NetHttp get request
+    def request(url, options = {})
+      @url = url
+      validate_url
 
-      raise invalid_request_without_get if request_method != :get
+      return follow if ['Ferrum', 'Chrome'].include? browser_engine
 
-      browser.visit(url)
-      browser
+      @request_method = options.fetch(:method, request_method)
+      @request_body   = options.fetch(:body, request_body)
+      browser.visit(@url)
     end
 
-    # Allow for NetHttp get request
-    def request(current_url = '')
-      @url = current_url unless current_url.empty?
-      raise 'Invalid URL supplied' if url.empty?
-
-      raise 'Invalid request type for js=true. Use #follow in place of #get' if js
-
-      browser.visit(url).body
+    def quit
+      browser.respond_to?(:quit) ? browser.quit : nil
     end
 
     private
 
+    # Only allow for capybara get request
+    def follow
+      browser.visit(@url)
+      browser
+    end
+
     def create_browser
-      engine = js ? 'Chrome' : 'NetHttp'
-      builder = Object.const_get "Makuri::BrowserBuilder::#{engine}"
+      builder        = Object.const_get "Makuri::BrowserBuilder::#{browser_engine}"
       builder.new(browser_params).build
     end
 
@@ -57,8 +57,18 @@ module Makuri
       }
     end
 
-    def invalid_request_without_get
-      "#{request_method.to_s.upcase} request not allowed for JS Engine. Try without 'js=true' argument!"
+    def validate_url
+      uri = URI.parse(@url)
+      raise 'Invalid URL supplied' unless uri.is_a?(URI::HTTP) && !uri.host.nil?
+    end
+
+    def browser_engine
+      case engine
+      when :ferrum
+        'Ferrum'
+      else
+        'NetHttp'
+      end
     end
   end
 end

data/lib/makuri/browser_builder/ferrum.rb

@@ -0,0 +1,44 @@
+require 'ferrum'
+
+module Makuri::BrowserBuilder
+  class Ferrum < Base
+    attr_accessor :headless, :html
+
+    def initialize(options = {})
+      super
+      @headless      = options.fetch(:headless, true)
+    end
+
+    def build
+      headers = { 'User-Agent': user_agent }
+      @ferrum = ::Ferrum::Browser.new(headless: @headless, headers: headers)
+      self
+    end
+
+    def visit(url)
+      @ferrum.go_to url
+      @html = @ferrum.body
+      self
+    end
+
+    def quit
+      @ferrum.quit
+    end
+
+    private
+
+    # TODO:
+    # def browser_options
+    #   args = %w[
+    #     --disable-gpu
+    #     --no-sandbox
+    #     --disable-translate
+    #     --ignore-certificate-errors
+    #   ]
+    #   args << '--headless' if headless
+    #   args << "--user-agent=#{user_agent}"
+    #   args << "--blink-settings=imagesEnabled=#{enable_images}"
+    #   Selenium::WebDriver::Chrome::Options.new(args: args)
+    # end
+  end
+end

data/lib/makuri/browser_builder/net_http.rb

@@ -2,6 +2,8 @@ require 'net/http'
 
 module Makuri::BrowserBuilder
   class NetHttp < Base
+    attr_accessor :response
+
     def build
       self
     end
@@ -12,9 +14,14 @@ module Makuri::BrowserBuilder
 
       request = send("#{request_method}_request", uri.request_uri, headers)
 
-      Net::HTTP.start(uri.host, uri.port, use_ssl: (uri.scheme == 'https')) do |http|
+      @response = Net::HTTP.start(uri.host, uri.port, use_ssl: (uri.scheme == 'https')) do |http|
         http.request(request)
       end
+      self
+    end
+
+    def html
+      @response.body
     end
 
     private

data/lib/makuri/spider.rb

@@ -18,23 +18,39 @@ module Makuri
         @start_urls = urls
       end
 
+      def spider_options(**options)
+        @engine = options.fetch(:engine, :net_http)
+        @headless = options.fetch(:headless, true)
+      end
+
       def run
         raise "Start URLs not found. Define start_urls for #{self}." unless defined? @start_urls
 
-        new(@start_urls).parse
+        @engine ||= :net_http
+        @headless = defined?(@headless) ? @headless : true
+
+        @start_urls.each { |start_url| new(start_url: start_url, engine: @engine, headless: @headless).parse }
       end
     end
 
-    attr_accessor :start_urls, :response
+    attr_accessor :engine, :headless, :response
 
-    def initialize(start_urls)
-      @start_urls = start_urls
+    def initialize(**config)
+      @start_url = config.fetch(:start_url, nil)
+      @engine    = config.fetch(:engine, :net_http)
+      @headless  = config.fetch(:headless, true)
 
-      update_response(@start_urls[0])
+      update_response(@start_url)
+      browser_quit_wrapper { parse }
     end
 
     def browser
-      @browser ||= Makuri::Browser.new
+      @browser ||= Makuri::Browser.new(engine: engine, headless: headless)
+    end
+
+    def browser_quit_wrapper
+      yield
+      browser.quit
     end
 
     def parse
@@ -55,18 +71,22 @@ module Makuri
     end
 
     def absolute_url(relative_url)
-      Addressable::URI.join(browser.url, relative_url).to_s
+      Addressable::URI.join(base_url, relative_url).to_s
     end
 
     private
 
+    def base_url
+      @start_url || browser.url.to_s
+    end
+
     def valid_url?(url)
       defined?(url) && !url.to_s.empty?
     end
 
     def update_response(url)
-      html = browser.request absolute_url(url)
-      @response = Nokogiri::HTML(html)
+      res = browser.request(absolute_url(url)).html
+      @response = Nokogiri::HTML(res)
     end
   end
 end

data/lib/makuri/version.rb

@@ -1,3 +1,3 @@
 module Makuri
-  VERSION = '0.2.0'.freeze
+  VERSION = '0.4.0'.freeze
 end

data/lib/makuri.rb

@@ -5,7 +5,7 @@ require 'makuri/spider'
 require 'makuri/version'
 
 require 'makuri/browser_builder/base'
-require 'makuri/browser_builder/chrome'
+require 'makuri/browser_builder/ferrum'
 require 'makuri/browser_builder/net_http'
 
 module Makuri

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: makuri
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Lal Saud
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-01-25 00:00:00.000000000 Z
+date: 2024-06-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -16,84 +16,70 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.7'
+        version: '2.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.7'
-- !ruby/object:Gem::Dependency
-  name: capybara
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.34'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.34'
+        version: '2.8'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.10.10
+        version: 1.15.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.10.10
+        version: 1.15.5
 - !ruby/object:Gem::Dependency
-  name: selenium-webdriver
+  name: ferrum
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.5'
+        version: '0.15'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.5'
+        version: '0.15'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: 2.5.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: 2.5.0
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.11'
+        version: '3.20'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.11'
+        version: '3.20'
 description:
 email: lalusaud@gmail.com
 executables: []
@@ -106,7 +92,7 @@ files:
 - lib/makuri.rb
 - lib/makuri/browser.rb
 - lib/makuri/browser_builder/base.rb
-- lib/makuri/browser_builder/chrome.rb
+- lib/makuri/browser_builder/ferrum.rb
 - lib/makuri/browser_builder/net_http.rb
 - lib/makuri/spider.rb
 - lib/makuri/version.rb
@@ -122,14 +108,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.5'
+      version: '2.7'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.5.11
 signing_key:
 specification_version: 4
 summary: Web-crawling framework for Ruby

data/lib/makuri/browser_builder/chrome.rb

@@ -1,34 +0,0 @@
-require 'capybara'
-require 'selenium-webdriver'
-
-module Makuri::BrowserBuilder
-  class Chrome < Base
-    def initialize(options = {})
-      super
-      @headless = options.fetch(:headless, true)
-    end
-
-    def build
-      Capybara.register_driver :selenium_chrome do |app|
-        Capybara::Selenium::Driver.new app, browser: :chrome, options: browser_options
-      end
-      Capybara.threadsafe = true
-      Capybara::Session.new :selenium_chrome
-    end
-
-    private
-
-    def browser_options
-      args = %w[
-        --disable-gpu
-        --no-sandbox
-        --disable-translate
-        --blink-settings=imagesEnabled=false
-        --ignore-certificate-errors
-      ]
-      args << '--headless' if @headless
-      args << "--user-agent=#{@user_agent}"
-      Selenium::WebDriver::Chrome::Options.new(args: args)
-    end
-  end
-end