snapcrawl 0.5.2 → 0.5.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 62a293da259afce5690315f27f2bbcd881e495a3d1b5344eb9ed9e2c46bd4a4d
-  data.tar.gz: d600fdbcd2344e5a19f853cbea67a0d8ad0c365a38d00aa4de8d02dd6e52e5b0
+  metadata.gz: 1238ab663146a888fc002379efaae0abaa72f02fc9ef7954bffb79e7dad4b07d
+  data.tar.gz: c340cf0b5d1675158077257007f0d58883e9d3e0418f4c0551039fa2e09c6df8
 SHA512:
-  metadata.gz: 3ebdb2355480bacd7f7a6faba264a31086e68c1864c692607fdb6fbc11df210eee17af936ab63305484ee46ac473d50b4033be11e995b51b9050b359c81dd906
-  data.tar.gz: 42a0a9f048fe9b5b1b04426d444710a256ccc8e9a914e3277f062c4ebf760d50a018c1f189e7b0cebced1c236f5d13ca56ab4abbf808a5ec4812bf9a754a9343
+  metadata.gz: 5e1c2ca21bbfa5471d58fce0e20d61237a87e14157b156fec0e0ee0b07d8c2a169dd38bffda1c99292bc737fc5cf25c64811e2650de958756c29fd630ca87721
+  data.tar.gz: fd7d8e18393e00274f3c145c103e03c0572f7113fab98b7cf3572e99f77d16b88628be0b3050bf98fba311fb05db974453377cb96e8bbd567794426b2a83c099

data/README.md

@@ -112,6 +112,14 @@ url_blacklist:
 
 # take a screenshot of this CSS selector only
 css_selector: 
+
+# when true, ignore SSL related errors
+skip_ssl_verification: false
+
+# set to any number of seconds to wait for the page to load before taking
+# a screenshot, leave empty to not wait at all (only needed for pages with
+# animations or other post-load events).
+screenshot_delay: 
 ```
 
 ## Contributing / Support

data/lib/snapcrawl/config.rb

@@ -40,6 +40,8 @@ module Snapcrawl
           css_selector: nil,
           log_level: 1,
           log_color: 'auto',
+          skip_ssl_verification: false,
+          screenshot_delay: nil
         }
       end
 

data/lib/snapcrawl/page.rb

@@ -52,7 +52,7 @@ module Snapcrawl
     end
 
     def http_response!
-      response = cache.get(url) { HTTParty.get url }
+      response = cache.get(url) { HTTParty.get url, httparty_options }
 
       if !response.success?
         $logger.warn "http error on !undpur!#{url}!txtrst!, code: !txtylw!#{response.code}!txtrst!, message: #{response.message.strip}"
@@ -66,6 +66,10 @@ module Snapcrawl
 
     end
 
+    def httparty_options
+      Config.skip_ssl_verification ? { verify: false } : {}
+    end
+
     def normalize_links(links)
       result = []
 

data/lib/snapcrawl/screenshot.rb

@@ -12,27 +12,20 @@ module Snapcrawl
 
     def save(outfile = nil)
       outfile ||= "#{url.to_slug}.png"
-
-      fetch_opts = { allowed_status_codes: [404, 401, 403] }
-      if Config.selector
-        fetch_opts[:selector] = Config.selector
-        fetch_opts[:full] = false
-      end
-
-      webshot_capture url, outfile, fetch_opts
+      webshot_capture url, outfile
     end
 
   private
 
-    def webshot_capture(url, image_path, fetch_opts)
-      webshot_capture! url, image_path, fetch_opts
+    def webshot_capture(url, image_path)
+      webshot_capture! url, image_path
     rescue => e
       raise ScreenshotError, "#{e.class} #{e.message}"
     end
 
-    def webshot_capture!(url, image_path, fetch_opts)
+    def webshot_capture!(url, image_path)
       hide_output do
-        webshot.capture url, image_path, fetch_opts do |magick|
+        webshot.capture url, image_path, webshot_options do |magick|
           magick.combine_options do |c|
             c.background "white"
             c.gravity 'north'
@@ -43,6 +36,21 @@ module Snapcrawl
       end
     end
 
+    def webshot_options
+      result = { allowed_status_codes: [404, 401, 403] }
+      
+      if Config.selector
+        result[:selector] = Config.selector
+        result[:full] = false
+      end
+
+      if Config.screenshot_delay
+        result[:timeout] = Config.screenshot_delay
+      end
+
+      result
+    end
+
     def webshot
       @webshot ||= Webshot::Screenshot.instance
     end

data/lib/snapcrawl/templates/config.yml

@@ -39,3 +39,11 @@ url_blacklist:
 
 # take a screenshot of this CSS selector only
 css_selector: 
+
+# when true, ignore SSL related errors
+skip_ssl_verification: false
+
+# set to any number of seconds to wait for the page to load before taking
+# a screenshot, leave empty to not wait at all (only needed for pages with
+# animations or other post-load events).
+screenshot_delay: 

data/lib/snapcrawl/version.rb

@@ -1,3 +1,3 @@
 module Snapcrawl
-  VERSION = "0.5.2"
+  VERSION = "0.5.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: snapcrawl
 version: !ruby/object:Gem::Version
-  version: 0.5.2
+  version: 0.5.3
 platform: ruby
 authors:
 - Danny Ben Shitrit
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-02-25 00:00:00.000000000 Z
+date: 2021-03-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: colsole