makuri 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f1106d01d72194894872b6fa5409c9f6951f27ff462a1ad81c793dd10c043974
-  data.tar.gz: 2d8ae6ef66759cdcdcf980e9e3b44f840f954db9ea3536b1a9ad3d656fec5e17
+  metadata.gz: efda242d320f75d0c7d483bd71f65f638a62624d0319543386b085914610010e
+  data.tar.gz: 12a830ef4e4a72926ee91e50f378fc0170acfd491bbc6c6a16dee08c0116288e
 SHA512:
-  metadata.gz: ddf5f382fccbcbfff68fa26f3bed25c07accaee86d2d3379b1471ea4690ff431b1f70330e9a29be1d65af64bc03652e06938dc5b31b3d5f506c96a375438c473
-  data.tar.gz: '0046904279c80dd1c9e6db0621dddcbcf8796ed8977940fc79dd9f500287128591cc2206f289ae271bbb0114c38d03ebe4d9a2ad894f4ceb34f99a6768493b6b'
+  metadata.gz: aaa1e7fb93b2cd25892dea2d34ea0e9edce13a6ac31aed7dc44b62b49b8b5080d4515eb903cba916d2a9634101f37980a872035f41a5b3632f2a608dd6e936b7
+  data.tar.gz: e32c252ed9940070600ce4f08ff9faa1ff93f5e225c674a5ad501ae5d4f9a6c3c02cdfd935d49284e981651fa86126298dcd1b06783ced0e85ef699f6d5dddad

data/CHANGELOG.md

@@ -1,10 +1,15 @@
+## 0.3.0 (2021-02-07)
+- Allow scrape from multiple start_urls
+- Add :engine support to Spider
+- Add :enable_images support to :chrome engine
+- Bug fixes and restructure #request_method & #request_body
+
 ## 0.2.0 (2021-01-25)
 - Add Capybara Support
 - Add :headless option to Browser object
 - Bug fixes & restructure Browser #follow, #request methods
 
 ## 0.1.0 (2021-01-05)
-
 - First Release
 - Add Basic Browser Builders for crawling web pages
 - Add Spider Framework to support scraping

data/lib/makuri/browser.rb

@@ -1,50 +1,47 @@
 module Makuri
   class Browser
-    attr_accessor :url, :js, :headless, :user_agent, :request_method, :request_body
+    attr_accessor :url, :engine, :headless, :user_agent, :request_method, :request_body
 
     def initialize(options = {})
-      @url            = options.fetch(:url, '')
-      @request_method = options.fetch(:request_method, :get)
-      @request_body   = options.fetch(:request_body, {})
-
-      @js             = options.fetch(:js, false)
+      @engine         = options.fetch(:engine, :net_http)
       @headless       = options.fetch(:headless, true)
       @user_agent     = options.fetch(
         :user_agent,
         'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36'
       )
+
+      # Defaults
+      @request_method = :get
+      @request_body   = {}
     end
 
     def browser
       @browser ||= create_browser
     end
 
-    # Only allow for capybara get request
-    def follow(current_url = '')
-      @url = current_url unless current_url.empty?
-      raise 'Invalid URL supplied' if url.empty?
+    # Allow for NetHttp get request
+    def request(url, options = {})
+      @url = url
+      validate_url
 
-      raise invalid_request_without_get if request_method != :get
+      return follow if engine_chrome?
 
-      browser.visit(url)
-      browser
+      @request_method = options.fetch(:method, request_method)
+      @request_body   = options.fetch(:body, request_body)
+      browser.visit(@url)
     end
 
-    # Allow for NetHttp get request
-    def request(current_url = '')
-      @url = current_url unless current_url.empty?
-      raise 'Invalid URL supplied' if url.empty?
-
-      raise 'Invalid request type for js=true. Use #follow in place of #get' if js
-
-      browser.visit(url).body
+    # Only allow for capybara get request
+    def follow
+      browser.visit(@url)
+      browser
     end
 
     private
 
     def create_browser
-      engine = js ? 'Chrome' : 'NetHttp'
-      builder = Object.const_get "Makuri::BrowserBuilder::#{engine}"
+      browser_engine = engine_chrome? ? 'Chrome' : 'NetHttp'
+      builder        = Object.const_get "Makuri::BrowserBuilder::#{browser_engine}"
       builder.new(browser_params).build
     end
 
@@ -57,8 +54,13 @@ module Makuri
       }
     end
 
-    def invalid_request_without_get
-      "#{request_method.to_s.upcase} request not allowed for JS Engine. Try without 'js=true' argument!"
+    def validate_url
+      uri = URI.parse(@url)
+      raise 'Invalid URL supplied' unless uri.is_a?(URI::HTTP) && !uri.host.nil?
+    end
+
+    def engine_chrome?
+      engine == :chrome
     end
   end
 end

data/lib/makuri/browser_builder/chrome.rb

@@ -3,9 +3,12 @@ require 'selenium-webdriver'
 
 module Makuri::BrowserBuilder
   class Chrome < Base
+    attr_accessor :headless, :enable_images
+
     def initialize(options = {})
       super
-      @headless = options.fetch(:headless, true)
+      @headless      = options.fetch(:headless, true)
+      @enable_images = options.fetch(:enable_images, true)
     end
 
     def build
@@ -23,11 +26,11 @@ module Makuri::BrowserBuilder
         --disable-gpu
         --no-sandbox
         --disable-translate
-        --blink-settings=imagesEnabled=false
         --ignore-certificate-errors
       ]
-      args << '--headless' if @headless
-      args << "--user-agent=#{@user_agent}"
+      args << '--headless' if headless
+      args << "--user-agent=#{user_agent}"
+      args << "--blink-settings=imagesEnabled=#{enable_images}"
       Selenium::WebDriver::Chrome::Options.new(args: args)
     end
   end

data/lib/makuri/browser_builder/net_http.rb

@@ -2,6 +2,8 @@ require 'net/http'
 
 module Makuri::BrowserBuilder
   class NetHttp < Base
+    attr_accessor :response
+
     def build
       self
     end
@@ -12,9 +14,14 @@ module Makuri::BrowserBuilder
 
       request = send("#{request_method}_request", uri.request_uri, headers)
 
-      Net::HTTP.start(uri.host, uri.port, use_ssl: (uri.scheme == 'https')) do |http|
+      @response = Net::HTTP.start(uri.host, uri.port, use_ssl: (uri.scheme == 'https')) do |http|
         http.request(request)
       end
+      self
+    end
+
+    def html
+      @response.body
     end
 
     private

data/lib/makuri/spider.rb

@@ -18,23 +18,30 @@ module Makuri
         @start_urls = urls
       end
 
+      def spider_options(**options)
+        @engine = options.fetch(:engine, :net_http)
+      end
+
       def run
         raise "Start URLs not found. Define start_urls for #{self}." unless defined? @start_urls
 
-        new(@start_urls).parse
+        @engine ||= :net_http
+
+        @start_urls.each { |start_url| new(start_url: start_url, engine: @engine).parse }
       end
     end
 
-    attr_accessor :start_urls, :response
+    attr_accessor :engine, :response
 
-    def initialize(start_urls)
-      @start_urls = start_urls
+    def initialize(**config)
+      @start_url = config.fetch(:start_url, nil)
+      @engine    = config.fetch(:engine, :net_http)
 
-      update_response(@start_urls[0])
+      update_response(@start_url)
     end
 
     def browser
-      @browser ||= Makuri::Browser.new
+      @browser ||= Makuri::Browser.new(engine: engine)
     end
 
     def parse
@@ -55,18 +62,23 @@ module Makuri
     end
 
     def absolute_url(relative_url)
-      Addressable::URI.join(browser.url, relative_url).to_s
+      Addressable::URI.join(base_url, relative_url).to_s
     end
 
     private
 
+    def base_url
+      @start_url || browser.url.to_s
+    end
+
     def valid_url?(url)
       defined?(url) && !url.to_s.empty?
     end
 
     def update_response(url)
-      html = browser.request absolute_url(url)
-      @response = Nokogiri::HTML(html)
+      res = browser.request(absolute_url(url)).html
+      # res = res.html if @engine == :chrome
+      @response = Nokogiri::HTML(res)
     end
   end
 end

data/lib/makuri/version.rb

@@ -1,3 +1,3 @@
 module Makuri
-  VERSION = '0.2.0'.freeze
+  VERSION = '0.3.0'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: makuri
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Lal Saud
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-01-25 00:00:00.000000000 Z
+date: 2021-02-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -80,6 +80,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: webdrivers
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement