snapcrawl 0.3.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e0dba7d3febfccf88e22194125387c6219cf500f0cb0e00fe4ed343a57fe9034
-  data.tar.gz: d05440bccd6ddde9773b24c425f3420b1970c3985805f6eb685171237e382cda
+  metadata.gz: d5c46ba3f4a171eb59bbc9afa6eca7327c8ef1964268bb63f407354201bb553b
+  data.tar.gz: beff7935a00a34edf97207f443a0a8a1b7e4734a6149f3b60fc6c625030a1be2
 SHA512:
-  metadata.gz: 71f65748af4339bf606765e2f1ea0aab90f4099ff986ce59318c6c966363ca6386a92d08ce175373cfa8e83c096af844983da2cde22ccbbebd2ab151514b6e72
-  data.tar.gz: a641b038a18114117375249d260895d73d08cd2cf47c7f668641c0891cd4b4723db80d4313a6a64cd89de02be0ea72aa9fa7b15f74d1ba6e0724017486c2d9c4
+  metadata.gz: 6e72274c6fa25d0b397bfd20fd676f0f8c83abb1d493610d58d091af4bdab700614ed8530aeaa75bea89c4c754a9fbea25feec669a62cdfb79d759c296d530ef
+  data.tar.gz: 349670f9155d5dae148c77af37f9aab2c6b334c560d73b48ca09a8fe10c15b485daee015082ac76c1698f0aa2d313bccbd747560d731f64738f8c94a801e2b4d

data/lib/snapcrawl/crawler.rb

@@ -46,6 +46,7 @@ module Snapcrawl
     private
 
     def crawl(url, opts={})
+      url = protocolize url
       defaults = {
         width: 1280,
         height: 0,
@@ -55,13 +56,13 @@ module Snapcrawl
         name: '%{url}',
         base: url,
       }
-      urls = [protocolize(url)]
+      urls = [url]
 
       @opts = OpenStruct.new defaults.merge(opts)
 
       make_screenshot_dir @opts.folder
 
-      @opts.depth.times do 
+      @opts.depth.times do
         urls = crawl_and_snap urls
       end
     end
@@ -136,9 +137,13 @@ module Snapcrawl
         if response.success?
           doc = Nokogiri::HTML response.body
           links = doc.css('a')
-          links = normalize_links links
+          links, warnings = normalize_links links
           @store.transaction { @store[url] = links }
           say "done"
+          warnings.each do |warning|
+            say "!txtylw!       Warn:  #{warning[:link]}"
+            say word_wrap "              #{warning[:message]}"
+          end
         else
           links = []
           say "!txtred!FAILED"
@@ -184,6 +189,7 @@ module Snapcrawl
       beginnings = "mailto|tel"
 
       links_array = []
+      warnings = []
 
       links.each do |link|
         link = link.attribute('href').to_s
@@ -200,7 +206,16 @@ module Snapcrawl
         link.strip!
 
         # Convert relative links to absolute
-        link = URI.join( @opts.base, link ).to_s
+        begin
+          link = URI.join( @opts.base, link ).to_s
+        rescue URI::InvalidURIError
+          escaped_link = URI.escape link
+          warnings << { link: link, message: "Using escaped link: #{escaped_link}" }
+          link = URI.join( @opts.base, escaped_link ).to_s
+        rescue => e
+          warnings << { link: link, message: "#{e.class} #{e.message}" }
+          next
+        end
 
         # Keep only links in our base domain
         next unless link.include? @opts.base
@@ -208,7 +223,7 @@ module Snapcrawl
         links_array << link
       end
 
-      links_array.uniq
+      [links_array.uniq, warnings]
     end
 
     def doc
@@ -233,7 +248,7 @@ module Snapcrawl
     end
 
     def webshot
-      Webshot::Screenshot.instance
+      @webshot ||= Webshot::Screenshot.instance
     end
 
     # The webshot gem messes with stdout/stderr streams so we keep it in 

data/lib/snapcrawl/version.rb

@@ -1,3 +1,3 @@
 module Snapcrawl
-  VERSION = "0.3.0"
+  VERSION = "0.3.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: snapcrawl
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
 platform: ruby
 authors:
 - Danny Ben Shitrit
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-09-10 00:00:00.000000000 Z
+date: 2019-09-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: colsole