scrap_kit 0.1.10 → 0.1.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 28049015b73f5b5508d952f54fd9d5a326e3377043be78776ad925e963161ffa
-  data.tar.gz: ebf4112f8a71bc4fe6ba98d6bc02a417f3cd1269a493f59657164945747363d2
+  metadata.gz: 96b093d466ebf16f066524ee5ffe6d462fa8b6f5fb0cf4839e58ccb4ea0a64e7
+  data.tar.gz: b720216ab63263c2d28842285d2fd1a7aab6e0634e51098e9de9a2e8d471e1bb
 SHA512:
-  metadata.gz: 50bbf6756482a1d3a94ea5994efa9921049405bfd3d5486ab854d0478adaa0d4c7e891e645ee9c9d774398cacacb26a6814b34efdf858be6ab14a7765da2c9b2
-  data.tar.gz: 8457382de09844d4fc469c4045e1e916335a1b3152bfd3b5bd0b0c464b590bc306d1fa7f8d66e0c3c40abe490d0b35aed2659223a6fcb0d81ec9c2efa94498d9
+  metadata.gz: 354f88e8f875eca5a45ef557501005752e5ca8e1abbb4bd6933f7640ddf9f64a31f33d766eaa9e4b154da59cf6a53cb5a065795e597b3b213ea6c7610260da6d
+  data.tar.gz: cefdf23aa7b1c104992c125e40deb9637269f1c8f29e56c2e9bfbee98e9f59000a8a4c3f7c6a5c710a6e2dc3cc0a8208de5ab863bfaf59533a4095cf9721a4a3

data/CHANGELOG.md

@@ -1,5 +1,30 @@
 # Changelog
 
+## [0.1.15] 2020-10-25
+
+### Changed/Added
+- Add `--disable-dev-shm-usage` arguments to Chrome driver
+
+## [0.1.14] 2020-10-10
+
+### Changed/Added
+- Increase timeout for Chrome driver
+
+## [0.1.13] 2020-09-04
+
+### Changed/Added
+- Return nil if element doesn't exist for `extract_value_from_element`
+
+## [0.1.12] 2020-09-03
+
+### Changed/Added
+- Return nil if `extract_attribute` fails
+
+## [0.1.11] 2020-09-03
+
+### Changed/Added
+- Add `user_agent` accessor for browser
+
 ## [0.1.10] 2020-09-03
 
 ### Changed/Added

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    scrap_kit (0.1.10)
+    scrap_kit (0.1.15)
       activesupport (~> 6.0)
       watir (~> 6.16.5)
       webdrivers (~> 4.0)
@@ -9,7 +9,7 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (6.0.3.2)
+    activesupport (6.0.3.4)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
@@ -25,7 +25,7 @@ GEM
     nokogiri (1.10.10)
       mini_portile2 (~> 2.4.0)
     rake (13.0.1)
-    regexp_parser (1.7.1)
+    regexp_parser (1.8.2)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
       rspec-expectations (~> 3.9.0)

data/lib/scrap_kit/recipe.rb

@@ -4,6 +4,8 @@ require "watir"
 
 module ScrapKit
   class Recipe
+    attr_accessor :user_agent, :browser
+
     class << self
       def load(source)
         input = if source.is_a?(Hash)
@@ -89,6 +91,8 @@ module ScrapKit
     end
 
     def extract_value_from_element(element)
+      return nil unless element.exists?
+
       if element&.respond_to?(:tag_name)
         if element.tag_name.downcase == "input"
           return element.attribute_value(:value)
@@ -139,6 +143,8 @@ module ScrapKit
           end
         end
       end
+    rescue
+      nil
     end
 
     private
@@ -189,7 +195,7 @@ module ScrapKit
         end
       end
 
-      sleep 0.25
+      sleep 1
       @browser.wait_until do
         @browser.ready_state == "complete"
       end
@@ -201,13 +207,18 @@ module ScrapKit
       options.add_argument "--headless"
       options.add_argument "--window-size=1080x720"
       options.add_argument "--hide-scrollbars"
+      options.add_argument "--user-agent=#{@user_agent}" if @user_agent
 
       if chrome_bin = ENV["GOOGLE_CHROME_SHIM"]
         options.add_argument "--no-sandbox"
+        options.add_argument "--disable-dev-shm-usage"
         options.binary = chrome_bin
       end
 
-      Watir::Browser.new(:chrome, options: options)
+      new_browser = Watir::Browser.new(:chrome, options: options)
+      new_browser.driver.manage.timeouts.page_load = 120
+
+      new_browser
     end
   end
 end

data/lib/scrap_kit/version.rb

@@ -1,3 +1,3 @@
 module ScrapKit
-  VERSION = "0.1.10"
+  VERSION = "0.1.15"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: scrap_kit
 version: !ruby/object:Gem::Version
-  version: 0.1.10
+  version: 0.1.15
 platform: ruby
 authors:
 - Gustavo Leon
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-09-04 00:00:00.000000000 Z
+date: 2020-10-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -142,7 +142,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: Scrap web sites using recipes.