scrap_kit 0.1.8 → 0.1.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9125aec53d6a517aa7679bc67df8c6ffd366d77afea643fae757ab7862767c0d
-  data.tar.gz: bbfa2cae0560461e7fb1f5af1cb672a0b5c11686e75696b24578861e0d14aa1c
+  metadata.gz: b3c0ad11b6a5a21a15c49f4203e2256ebac9d83fa9ed8de4060edd7b49ed6513
+  data.tar.gz: 4a9aa63a96b9bfda89f578827a3f7df5f03458d9a8191911d5e2ffa723ac8913
 SHA512:
-  metadata.gz: f2f8e6b7cb709ec2db86696adf27ec64e9c5faf2c339248ef34cb31764155d11de7904e1e881b7a21ba933469ebbccee998db35748030544fd699cb41a2cd7e1
-  data.tar.gz: 79fcdcf90aae48eefdf83ed6e285114221047f007a15e7e5d3f5b357091b37f7f51e0778c6ad03f03caa0fa291666831f02a1459d426e522aef1f6c02b4b509d
+  metadata.gz: c21f2fc8782ca5552f92020cbf766f982cbbc1621256a0885340507e74b73d895086343111f65b3a7c6aba23771e345b8e57f962bcb3d1ab77a56392006469e4
+  data.tar.gz: bece24176f75eb7113fd5a8b0d1aaf151fa4d61ddd3bd697e9b4e53d2390888ef38fee6f9761ac3c02b8671710b8816b44a8236ef9f02cf547a95eff28796f88

data/CHANGELOG.md

@@ -1,5 +1,30 @@
 # Changelog
 
+## [0.1.13] 2020-09-04
+
+### Changed/Added
+- Return nil if element doesn't exist for `extract_value_from_element`
+
+## [0.1.12] 2020-09-03
+
+### Changed/Added
+- Return nil if `extract_attribute` fails
+
+## [0.1.11] 2020-09-03
+
+### Changed/Added
+- Add `user_agent` accessor for browser
+
+## [0.1.10] 2020-09-03
+
+### Changed/Added
+- Map attributes to JavaScript calls
+
+## [0.1.9] 2020-08-31
+
+### Changed/Added
+- Set arguments for Chrome driver
+
 ## [0.1.8] 2020-08-29
 
 ### Changed/Added

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    scrap_kit (0.1.8)
+    scrap_kit (0.1.13)
       activesupport (~> 6.0)
       watir (~> 6.16.5)
       webdrivers (~> 4.0)
@@ -21,7 +21,7 @@ GEM
     i18n (1.8.5)
       concurrent-ruby (~> 1.0)
     mini_portile2 (2.4.0)
-    minitest (5.14.1)
+    minitest (5.14.2)
     nokogiri (1.10.10)
       mini_portile2 (~> 2.4.0)
     rake (13.0.1)

data/lib/scrap_kit/recipe.rb

@@ -4,6 +4,8 @@ require "watir"
 
 module ScrapKit
   class Recipe
+    attr_accessor :user_agent
+
     class << self
       def load(source)
         input = if source.is_a?(Hash)
@@ -27,7 +29,7 @@ module ScrapKit
     def run
       output = {}
 
-      @browser = Watir::Browser.new(:chrome, headless: true)
+      @browser = create_browser
       @browser.goto @url
 
       @steps.each do |step|
@@ -89,6 +91,8 @@ module ScrapKit
     end
 
     def extract_value_from_element(element)
+      return nil unless element.exists?
+
       if element&.respond_to?(:tag_name)
         if element.tag_name.downcase == "input"
           return element.attribute_value(:value)
@@ -125,6 +129,8 @@ module ScrapKit
 
             output
           end
+        elsif selector_or_object[:javascript]
+          @browser.execute_script(selector_or_object[:javascript])
         else
           found_elements = elements_from_selector(browser_or_element, selector_or_object)
 
@@ -137,6 +143,8 @@ module ScrapKit
           end
         end
       end
+    rescue
+      nil
     end
 
     private
@@ -187,10 +195,26 @@ module ScrapKit
         end
       end
 
-      sleep 0.25
+      sleep 1
       @browser.wait_until do
         @browser.ready_state == "complete"
       end
     end
+
+    def create_browser
+      options = Selenium::WebDriver::Chrome::Options.new
+
+      options.add_argument "--headless"
+      options.add_argument "--window-size=1080x720"
+      options.add_argument "--hide-scrollbars"
+      options.add_argument "--user-agent=#{@user_agent}" if @user_agent
+
+      if chrome_bin = ENV["GOOGLE_CHROME_SHIM"]
+        options.add_argument "--no-sandbox"
+        options.binary = chrome_bin
+      end
+
+      Watir::Browser.new(:chrome, options: options)
+    end
   end
 end

data/lib/scrap_kit/version.rb

@@ -1,3 +1,3 @@
 module ScrapKit
-  VERSION = "0.1.8"
+  VERSION = "0.1.13"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: scrap_kit
 version: !ruby/object:Gem::Version
-  version: 0.1.8
+  version: 0.1.13
 platform: ruby
 authors:
 - Gustavo Leon
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-08-30 00:00:00.000000000 Z
+date: 2020-09-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler