makuri 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3a5ecdfca240407bdcc2554114a82c0f7e66790fed4097d4b5b9cb654c603bbc
-  data.tar.gz: bfc53dea621f432bd764bdb371e48a8da6aceeb2a3b1c364fa12634f87e5a982
+  metadata.gz: f1106d01d72194894872b6fa5409c9f6951f27ff462a1ad81c793dd10c043974
+  data.tar.gz: 2d8ae6ef66759cdcdcf980e9e3b44f840f954db9ea3536b1a9ad3d656fec5e17
 SHA512:
-  metadata.gz: 98763e884eeb48faf3febfb6f2b2872cb59ec4dbe4be31e5dd45c5b1b8a228b317ad5e15b911ccb119d18494eb1e6be7e76864fe490e483ab9d4aba1b12704bb
-  data.tar.gz: cd8dc00f1ec717983f8ee715ba891a86af53eaebec85333c1b0248096860cf899df1c4ce9b371ca56394c5873a0c00b485e950f6f29ec782b1cc59b741d2c95f
+  metadata.gz: ddf5f382fccbcbfff68fa26f3bed25c07accaee86d2d3379b1471ea4690ff431b1f70330e9a29be1d65af64bc03652e06938dc5b31b3d5f506c96a375438c473
+  data.tar.gz: '0046904279c80dd1c9e6db0621dddcbcf8796ed8977940fc79dd9f500287128591cc2206f289ae271bbb0114c38d03ebe4d9a2ad894f4ceb34f99a6768493b6b'

data/CHANGELOG.md

@@ -1,3 +1,10 @@
-## 0.1.0 (2020-12-31)
+## 0.2.0 (2021-01-25)
+- Add Capybara Support
+- Add :headless option to Browser object
+- Bug fixes & restructure Browser #follow, #request methods
 
-- Unreleased - Work in Progress
+## 0.1.0 (2021-01-05)
+
+- First Release
+- Add Basic Browser Builders for crawling web pages
+- Add Spider Framework to support scraping

data/README.md

@@ -2,6 +2,21 @@
 
 Makuri is a Web-crawling framework for Ruby.
 
+# Install
+
+Add this to your application's Gemfile
+```ruby
+gem 'makuri'
+```
+And execute
+```sh
+$ bundle
+```
+Or install it as:
+```sh
+$ gem install makuri
+```
+
 # Usage
 In this example, we are going to crawl the [quotes website](https://quotes.toscrape.com) and scrape data as:
 ```ruby
@@ -32,8 +47,8 @@ end
 QuotesSpider.run
 ```
 Now save the file to ```quotes_spider.rb``` file and run it as:
-```
-$> ruby quotes_spider.rb > quotes.json
+```sh
+$ ruby quotes_spider.rb > quotes.json
 ```
 When it's done, you will find all the quotes saved to ```quotes.json``` file. It's that easy.
 
@@ -53,8 +68,8 @@ Everyone is encouraged to help improve this project. Here are a few ways you can
 To get started with development:
 
 ```sh
-git clone https://github.com/lalusaud/makuri.git
-cd rover
-bundle install
-bundle exec rake test
+$ git clone https://github.com/lalusaud/makuri.git
+$ cd makuri
+$ bundle install
+$ bundle exec rake test
 ```

data/lib/makuri/browser.rb

@@ -1,41 +1,64 @@
 module Makuri
   class Browser
-    attr_accessor :current_url, :render_js, :user_agent, :request_method, :request_body
+    attr_accessor :url, :js, :headless, :user_agent, :request_method, :request_body
 
     def initialize(options = {})
-      @current_url    = options.fetch(:url, '')
+      @url            = options.fetch(:url, '')
       @request_method = options.fetch(:request_method, :get)
       @request_body   = options.fetch(:request_body, {})
 
-      @render_js      = options.fetch(:render_js, false)
+      @js             = options.fetch(:js, false)
+      @headless       = options.fetch(:headless, true)
       @user_agent     = options.fetch(
         :user_agent,
         'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36'
       )
     end
 
-    def follow(url = '')
-      @current_url = url unless url.empty?
-      raise 'Invalid URL supplied' if @current_url.empty?
+    def browser
+      @browser ||= create_browser
+    end
+
+    # Only allow for capybara get request
+    def follow(current_url = '')
+      @url = current_url unless current_url.empty?
+      raise 'Invalid URL supplied' if url.empty?
+
+      raise invalid_request_without_get if request_method != :get
 
-      @browser = create_browser
-      @browser.visit(@current_url)
+      browser.visit(url)
+      browser
+    end
+
+    # Allow for NetHttp get request
+    def request(current_url = '')
+      @url = current_url unless current_url.empty?
+      raise 'Invalid URL supplied' if url.empty?
+
+      raise 'Invalid request type for js=true. Use #follow in place of #get' if js
+
+      browser.visit(url).body
     end
 
     private
 
     def create_browser
-      engine = render_js ? 'Chrome' : 'NetHttp'
+      engine = js ? 'Chrome' : 'NetHttp'
       builder = Object.const_get "Makuri::BrowserBuilder::#{engine}"
-      builder.new browser_params
+      builder.new(browser_params).build
     end
 
     def browser_params
       {
         user_agent: user_agent,
+        headless: headless,
         request_method: request_method,
         request_body: request_body
       }
     end
+
+    def invalid_request_without_get
+      "#{request_method.to_s.upcase} request not allowed for JS Engine. Try without 'js=true' argument!"
+    end
   end
 end

data/lib/makuri/browser_builder/base.rb

@@ -2,10 +2,14 @@ module Makuri::BrowserBuilder
   class Base
     attr_accessor :user_agent, :request_method, :request_body
 
-    def initialize(options)
-      @user_agent     = options.fetch(:user_agent)
-      @request_method = options.fetch(:request_method)
-      @request_body   = options.fetch(:request_body)
+    def initialize(options = {})
+      @user_agent     = options.fetch(:user_agent, '')
+      @request_method = options.fetch(:request_method, :get)
+      @request_body   = options.fetch(:request_body, '')
+    end
+
+    def build
+      raise NotImplementedError
     end
   end
 end

data/lib/makuri/browser_builder/chrome.rb

@@ -1,28 +1,34 @@
+require 'capybara'
 require 'selenium-webdriver'
 
 module Makuri::BrowserBuilder
   class Chrome < Base
-    def visit(url)
-      raise invalid_request_message if request_method != :get
+    def initialize(options = {})
+      super
+      @headless = options.fetch(:headless, true)
+    end
 
-      @browser = Selenium::WebDriver.for :chrome, options: browser_options
-      @browser.get url
-      @browser.page_source
+    def build
+      Capybara.register_driver :selenium_chrome do |app|
+        Capybara::Selenium::Driver.new app, browser: :chrome, options: browser_options
+      end
+      Capybara.threadsafe = true
+      Capybara::Session.new :selenium_chrome
     end
 
     private
 
     def browser_options
       args = %w[
-        --headless --disable-gpu --no-sandbox --disable-translate
-        --blink-settings=imagesEnabled=false --ignore-certificate-errors
+        --disable-gpu
+        --no-sandbox
+        --disable-translate
+        --blink-settings=imagesEnabled=false
+        --ignore-certificate-errors
       ]
-      args << "--user-agent=#{user_agent}"
+      args << '--headless' if @headless
+      args << "--user-agent=#{@user_agent}"
       Selenium::WebDriver::Chrome::Options.new(args: args)
     end
-
-    def invalid_request_message
-      "#{request_method.to_s.upcase} request not allowed for JS Engine. Try without 'render_js=true' argument!"
-    end
   end
 end

data/lib/makuri/browser_builder/net_http.rb

@@ -2,6 +2,10 @@ require 'net/http'
 
 module Makuri::BrowserBuilder
   class NetHttp < Base
+    def build
+      self
+    end
+
     def visit(url)
       uri = URI(url)
       headers = { 'User-Agent': user_agent }
@@ -10,7 +14,7 @@ module Makuri::BrowserBuilder
 
       Net::HTTP.start(uri.host, uri.port, use_ssl: (uri.scheme == 'https')) do |http|
         http.request(request)
-      end.body
+      end
     end
 
     private

data/lib/makuri/spider.rb

@@ -55,7 +55,7 @@ module Makuri
     end
 
     def absolute_url(relative_url)
-      Addressable::URI.join(browser.current_url, relative_url).to_s
+      Addressable::URI.join(browser.url, relative_url).to_s
     end
 
     private
@@ -65,7 +65,7 @@ module Makuri
     end
 
     def update_response(url)
-      html = browser.follow absolute_url(url)
+      html = browser.request absolute_url(url)
       @response = Nokogiri::HTML(html)
     end
   end

data/lib/makuri/version.rb

@@ -1,3 +1,3 @@
 module Makuri
-  VERSION = '0.1.0'.freeze
+  VERSION = '0.2.0'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: makuri
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Lal Saud
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-01-06 00:00:00.000000000 Z
+date: 2021-01-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -24,6 +24,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.7'
+- !ruby/object:Gem::Dependency
+  name: capybara
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.34'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.34'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
@@ -44,14 +58,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.142'
+        version: '3.5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.142'
+        version: '3.5'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement