scrap_kit 0.1.9 → 0.1.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ab218fee7bbf85145afd031bbe59a0820b93eb1e57ee27d50163aa25dbcb8e46
-  data.tar.gz: 4d304049b137f2750b3843c80ff925d872576459627bebac443e395d5e897749
+  metadata.gz: d387636088d4461f9f26bcc312f9f634d20fb24ce8d0e7eff9672dfbd87664ba
+  data.tar.gz: fe5e85497ba66a7370a4c9b4017058ad5f9d938ff2a4add96aba2888e6f6f6ca
 SHA512:
-  metadata.gz: 5e2dd47d6b77ac3983efeb99e351a5e26b2737c1aa0c77972213a744ce44c6033dc2ae3498aca40f68471010b33b6fc3aeb1bf03f922ffa6e95aa0d715493722
-  data.tar.gz: bc92e505d8ffca95d621ad8756350ae174bc316efbd015724d87478364c9d500682a4da765595c9158bc56ed3fd855e9827dc4f357fbea64901fdb53665c32b1
+  metadata.gz: fe464311773e564ee88ce8dab45478445a6291cc6a3c2151b2d45a819e8205fea168b15127fa61fab261042a30ddb8f605f3763629f58df528afa40c5853a324
+  data.tar.gz: 43d4c05f94cbff4daa91c22a0d0010dbd857abcd95ee73f1f501288733d33d3c5813b4b84c6c2e09b2a62a5efadd6808d5b7a2de7c012629899ac8b966af1a71

data/CHANGELOG.md

@@ -1,5 +1,30 @@
 # Changelog
 
+## [0.1.14] 2020-10-10
+
+### Changed/Added
+- Increase timeout for Chrome driver
+
+## [0.1.13] 2020-09-04
+
+### Changed/Added
+- Return nil if element doesn't exist for `extract_value_from_element`
+
+## [0.1.12] 2020-09-03
+
+### Changed/Added
+- Return nil if `extract_attribute` fails
+
+## [0.1.11] 2020-09-03
+
+### Changed/Added
+- Add `user_agent` accessor for browser
+
+## [0.1.10] 2020-09-03
+
+### Changed/Added
+- Map attributes to JavaScript calls
+
 ## [0.1.9] 2020-08-31
 
 ### Changed/Added

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    scrap_kit (0.1.9)
+    scrap_kit (0.1.14)
       activesupport (~> 6.0)
       watir (~> 6.16.5)
       webdrivers (~> 4.0)
@@ -9,7 +9,7 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (6.0.3.2)
+    activesupport (6.0.3.4)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
@@ -21,11 +21,11 @@ GEM
     i18n (1.8.5)
       concurrent-ruby (~> 1.0)
     mini_portile2 (2.4.0)
-    minitest (5.14.1)
+    minitest (5.14.2)
     nokogiri (1.10.10)
       mini_portile2 (~> 2.4.0)
     rake (13.0.1)
-    regexp_parser (1.7.1)
+    regexp_parser (1.8.1)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
       rspec-expectations (~> 3.9.0)

data/lib/scrap_kit/recipe.rb

@@ -4,6 +4,8 @@ require "watir"
 
 module ScrapKit
   class Recipe
+    attr_accessor :user_agent, :browser
+
     class << self
       def load(source)
         input = if source.is_a?(Hash)
@@ -89,6 +91,8 @@ module ScrapKit
     end
 
     def extract_value_from_element(element)
+      return nil unless element.exists?
+
       if element&.respond_to?(:tag_name)
         if element.tag_name.downcase == "input"
           return element.attribute_value(:value)
@@ -125,6 +129,8 @@ module ScrapKit
 
             output
           end
+        elsif selector_or_object[:javascript]
+          @browser.execute_script(selector_or_object[:javascript])
         else
           found_elements = elements_from_selector(browser_or_element, selector_or_object)
 
@@ -137,6 +143,8 @@ module ScrapKit
           end
         end
       end
+    rescue
+      nil
     end
 
     private
@@ -187,7 +195,7 @@ module ScrapKit
         end
       end
 
-      sleep 0.25
+      sleep 1
       @browser.wait_until do
         @browser.ready_state == "complete"
       end
@@ -199,13 +207,17 @@ module ScrapKit
       options.add_argument "--headless"
       options.add_argument "--window-size=1080x720"
       options.add_argument "--hide-scrollbars"
+      options.add_argument "--user-agent=#{@user_agent}" if @user_agent
 
       if chrome_bin = ENV["GOOGLE_CHROME_SHIM"]
         options.add_argument "--no-sandbox"
         options.binary = chrome_bin
       end
 
-      Watir::Browser.new(:chrome, options: options)
+      new_browser = Watir::Browser.new(:chrome, options: options)
+      new_browser.driver.manage.timeouts.page_load = 120
+
+      new_browser
     end
   end
 end

data/lib/scrap_kit/version.rb

@@ -1,3 +1,3 @@
 module ScrapKit
-  VERSION = "0.1.9"
+  VERSION = "0.1.14"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: scrap_kit
 version: !ruby/object:Gem::Version
-  version: 0.1.9
+  version: 0.1.14
 platform: ruby
 authors:
 - Gustavo Leon
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-08-31 00:00:00.000000000 Z
+date: 2020-10-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler