snapcrawl 0.4.1 → 0.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 64c0d3c62258c70417520749bdd71b7d33c8d920ec94f2dfb0aaed961310563b
-  data.tar.gz: 72ece0ddb0f5f9f2b578d4c369814507bb1b2343170d799eb27497a3c937475a
+  metadata.gz: 327da92cb63a1a8e6f58e58a4806d4e10b2cfa491960306544165be6423c9b3e
+  data.tar.gz: c6f2056f9ca5614a76bce68bdf2f001668ab626764bce89cf1b1bc4a8f68f833
 SHA512:
-  metadata.gz: b60088e4dab7944269f76344094a6f80e6b304c0591fff2c3e9aa0e18ffac89b2a9927bf2759e7a17f9944dd925b01b342f46ec6587754ac206bf614741be0c3
-  data.tar.gz: ae1b3dda9b0339440b9f37727503ad87f1f5a71d2eadc3283250f481b9391f1d530213a05a0f5ce0adc7a1f9bd183c049c5cfc7a7fcd1b06d5286c312d8d0747
+  metadata.gz: 91b00e39fbf5943501cc7f67eb1c684811d10ae3f5acc0263a3a6259ae64ad51d01d89aba75576990d7517a07a53660ec1f63d13adbbaf5bdd6380b7d9dd8050
+  data.tar.gz: 318d7c11aa087a20a8f5c0dd922e9f11f2eeca1be7c165bfa04d54d775d0688e3d3532434a4987cca039c362bb3817542331ce135f8ec686f0178d6e5fa343e5

data/lib/snapcrawl.rb

@@ -1,6 +1,6 @@
 require 'snapcrawl/version'
+require 'snapcrawl/exceptions'
 require 'snapcrawl/crawler'
 
 require 'byebug' if ENV['BYEBUG']
 
-self.extend Snapcrawl

data/lib/snapcrawl/crawler.rb

@@ -5,16 +5,12 @@ require 'httparty'
 require 'nokogiri'
 require 'ostruct'
 require 'pstore'
-require 'uri'
 require 'addressable/uri'
 require 'webshot'
 
 module Snapcrawl
   include Colsole
 
-  class MissingPhantomJS < StandardError; end
-  class MissingImageMagick < StandardError; end
-
   class Crawler
     include Singleton
     
@@ -206,11 +202,7 @@ module Snapcrawl
 
         # Convert relative links to absolute
         begin
-          link = URI.join( @opts.base, link ).to_s.dup
-        rescue URI::InvalidURIError
-          escaped_link = Addressable::URI.encode link
-          warnings << { link: link, message: "Using escaped link: #{escaped_link}" }
-          link = URI.join( @opts.base, escaped_link ).to_s.dup
+          link = Addressable::URI.join( @opts.base, link ).to_s.dup
         rescue => e
           warnings << { link: link, message: "#{e.class} #{e.message}" }
           next
@@ -226,11 +218,11 @@ module Snapcrawl
     end
 
     def doc
-      @doc ||= File.read template 'docopt.txt'
+      @doc ||= File.read docopt
     end
 
-    def template(file)
-      File.expand_path("../templates/#{file}", __FILE__)
+    def docopt
+      File.expand_path "docopt.txt", __dir__
     end
 
     def opts_from_args(args)

data/lib/snapcrawl/exceptions.rb

@@ -0,0 +1,4 @@
+module Snapcrawl
+  class MissingPhantomJS < StandardError; end
+  class MissingImageMagick < StandardError; end
+end

data/lib/snapcrawl/version.rb

@@ -1,3 +1,3 @@
 module Snapcrawl
-  VERSION = "0.4.1"
+  VERSION = "0.4.2"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: snapcrawl
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.4.2
 platform: ruby
 authors:
 - Danny Ben Shitrit
@@ -112,7 +112,7 @@ files:
 - bin/snapcrawl
 - lib/snapcrawl.rb
 - lib/snapcrawl/crawler.rb
-- lib/snapcrawl/templates/docopt.txt
+- lib/snapcrawl/exceptions.rb
 - lib/snapcrawl/version.rb
 homepage: https://github.com/DannyBen/snapcrawl
 licenses:

data/lib/snapcrawl/templates/docopt.txt

@@ -1,48 +0,0 @@
-Snapcrawl
-
-Usage:
-  snapcrawl URL [options]
-  snapcrawl -h | --help 
-  snapcrawl -v | --version
-
-Options:
-  -f, --folder PATH
-    Where to save screenshots [default: snaps]
-
-  -n, --name TEMPLATE
-    Filename template. Include the string '%{url}' anywhere in the name to 
-    use the captured URL in the filename [default: %{url}]
-
-  -a, --age SECONDS
-    Number of seconds to consider screenshots fresh [default: 86400]
-
-  -d, --depth LEVELS
-    Number of levels to crawl [default: 1]
-
-  -W, --width PIXELS
-    Screen width in pixels [default: 1280]
-
-  -H, --height PIXELS
-    Screen height in pixels. Use 0 to capture the full page [default: 0]
-
-  -s, --selector SELECTOR
-    CSS selector to capture
-
-  -o, --only REGEX
-    Include only URLs that match REGEX
-
-  -h, --help
-    Show this screen
-
-  -v, --version
-    Show version number
-
-Examples:
-  snapcrawl example.com
-  snapcrawl example.com -d2 -fscreens
-  snapcrawl example.com -d2 > out.txt 2> err.txt &
-  snapcrawl example.com -W360 -H480
-  snapcrawl example.com --selector "#main-content"
-  snapcrawl example.com --only "products|collections"
-  snapcrawl example.com --name "screenshot-%{url}"
-  snapcrawl example.com --name "`date +%Y%m%d`_%{url}"