scrap_kit 0.1.11 → 0.1.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 36f370f43360cdc2725cbb83591738dc46c5949cb995039f45cf1ddaccd091a4
-  data.tar.gz: b904bef830b3e5cd88892677eed3f0cf97635e2d46277cc69280ca5ee7b6a0a3
+  metadata.gz: a3db092bd833a6c977d64468c38c6361922498683606f3ca543a3c7fdb5a91a1
+  data.tar.gz: 865ca828a23e0c2ce43f9070ce85e199979774964d10018881befd504925608f
 SHA512:
-  metadata.gz: e0c1321ef88bf5be53603e41e0826676d9bc50bb741b4d45d8cc79ff42bc6e0360d989c3deb097d8c784b9c795495898a39cf7ff4a41f36c62f57d0f030a8915
-  data.tar.gz: b668a193cf260b94239d1d1e016e9eb4b2d8e3de43aaf604d6d9fb255b5147295e2079e5da7b3b3dbb6177ea45ac413ce73467f11ba73d0f2495fa91b3196af4
+  metadata.gz: 0e25dc4878540dd64d4d436a07d70f250ef4c6451f2586758324ad819d256af190c7e8f72a3b113722911649485cdf2a9793186e607fa3ff159ee992b424d6f5
+  data.tar.gz: 2de4cd58ddc9f46f1a9d198d95163900bf1394e87ed9733a444693a1775f0d59a72793212fd70d2a3656005d5739f730109e33375733e28146f2ee3bedeecaa2

data/CHANGELOG.md

@@ -1,5 +1,30 @@
 # Changelog
 
+## [0.1.16] 2020-11-20
+
+### Changed/Added
+- Return `src` value for `img` elements
+
+## [0.1.15] 2020-10-25
+
+### Changed/Added
+- Add `--disable-dev-shm-usage` arguments to Chrome driver
+
+## [0.1.14] 2020-10-10
+
+### Changed/Added
+- Increase timeout for Chrome driver
+
+## [0.1.13] 2020-09-04
+
+### Changed/Added
+- Return nil if element doesn't exist for `extract_value_from_element`
+
+## [0.1.12] 2020-09-03
+
+### Changed/Added
+- Return nil if `extract_attribute` fails
+
 ## [0.1.11] 2020-09-03
 
 ### Changed/Added

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    scrap_kit (0.1.11)
+    scrap_kit (0.1.16)
       activesupport (~> 6.0)
       watir (~> 6.16.5)
       webdrivers (~> 4.0)
@@ -9,7 +9,7 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (6.0.3.2)
+    activesupport (6.0.3.4)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
@@ -25,7 +25,7 @@ GEM
     nokogiri (1.10.10)
       mini_portile2 (~> 2.4.0)
     rake (13.0.1)
-    regexp_parser (1.7.1)
+    regexp_parser (1.8.2)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
       rspec-expectations (~> 3.9.0)
@@ -44,7 +44,7 @@ GEM
       childprocess (>= 0.5, < 4.0)
       rubyzip (>= 1.2.2)
     thread_safe (0.3.6)
-    tzinfo (1.2.7)
+    tzinfo (1.2.8)
       thread_safe (~> 0.1)
     watir (6.16.5)
       regexp_parser (~> 1.2)
@@ -53,7 +53,7 @@ GEM
       nokogiri (~> 1.6)
       rubyzip (>= 1.3.0)
       selenium-webdriver (>= 3.0, < 4.0)
-    zeitwerk (2.4.0)
+    zeitwerk (2.4.1)
 
 PLATFORMS
   ruby

data/lib/scrap_kit/recipe.rb

@@ -4,7 +4,7 @@ require "watir"
 
 module ScrapKit
   class Recipe
-    attr_accessor :user_agent
+    attr_accessor :user_agent, :browser
 
     class << self
       def load(source)
@@ -91,9 +91,13 @@ module ScrapKit
     end
 
     def extract_value_from_element(element)
+      return nil unless element.exists?
+
       if element&.respond_to?(:tag_name)
         if element.tag_name.downcase == "input"
           return element.attribute_value(:value)
+        elsif element.tag_name.downcase == "img"
+          return element.attribute_value(:src)
         end
       end
 
@@ -141,6 +145,8 @@ module ScrapKit
           end
         end
       end
+    rescue
+      nil
     end
 
     private
@@ -191,7 +197,7 @@ module ScrapKit
         end
       end
 
-      sleep 0.25
+      sleep 1
       @browser.wait_until do
         @browser.ready_state == "complete"
       end
@@ -207,10 +213,14 @@ module ScrapKit
 
       if chrome_bin = ENV["GOOGLE_CHROME_SHIM"]
         options.add_argument "--no-sandbox"
+        options.add_argument "--disable-dev-shm-usage"
         options.binary = chrome_bin
       end
 
-      Watir::Browser.new(:chrome, options: options)
+      new_browser = Watir::Browser.new(:chrome, options: options)
+      new_browser.driver.manage.timeouts.page_load = 120
+
+      new_browser
     end
   end
 end

data/lib/scrap_kit/version.rb

@@ -1,3 +1,3 @@
 module ScrapKit
-  VERSION = "0.1.11"
+  VERSION = "0.1.16"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: scrap_kit
 version: !ruby/object:Gem::Version
-  version: 0.1.11
+  version: 0.1.16
 platform: ruby
 authors:
 - Gustavo Leon
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-09-04 00:00:00.000000000 Z
+date: 2020-11-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -142,7 +142,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: Scrap web sites using recipes.