snapcrawl 0.2.1 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7f8bfcb13d6d049104a97fe95b4f20527c9e93f9
-  data.tar.gz: 5d39a2e40270cbe8ddfd5e0863016a665059e747
+  metadata.gz: e2794d3fe40619ef7de870f738767419cdb893fc
+  data.tar.gz: 6a866a6ac4808e7a522e67db0415e31d1d463b62
 SHA512:
-  metadata.gz: c0ad7d74dff9e73d5892870cf162c2b61c96a3f316f6ffedf2b9bd84c09c080bf0e330d32a9f567d4f3e8bec14964afe2773e010177236f438046e3f06b87624
-  data.tar.gz: 015a6dd81b525bcd59cc52360c5d4989542646af72e84e48a350719cb4b88751e6def5ab201b81de9c256c7036e9894dc5ca03ae30829e7a8d4ab2de0af47aaa
+  metadata.gz: 6195bff2403b3c4e7900b6c6e33e962eafa37bbbfed78940da5fce4152318bb6715073ce78bc8d7865451eb287a9e57aa77806064634e9e388c20fef8c5f929a
+  data.tar.gz: f2b0207a6b976a34e1554adaa305f415191903cdd51b7dfd7bf70735fd225f4ef7de58589ed3aa179cde10e621be170b93897c3523f9e323cbfddf35dfca854f

data/README.md

@@ -1,5 +1,10 @@
 # SnapCrawl - crawl a website and take screenshots
 
+[![Gem Version](https://badge.fury.io/rb/snapcrawl.svg)](http://badge.fury.io/rb/snapcrawl)
+[![Code Climate](https://codeclimate.com/github/DannyBen/snapcrawl/badges/gpa.svg)](https://codeclimate.com/github/DannyBen/snapcrawl)
+[![Dependency Status](https://gemnasium.com/DannyBen/snapcrawl.svg)](https://gemnasium.com/DannyBen/snapcrawl)
+
+
 SnapCrawl is a command line utility for crawling a website and saving
 screenshots. 
 
@@ -18,8 +23,8 @@ screenshots.
 
 ## Usage
 
-	$ snapcrawl --help
-
+	$ snapcrawl --help
+
     Snapcrawl
     
     Usage:
@@ -46,11 +51,18 @@ screenshots.
       snapcrawl go example.com -d2 > out.txt 2> err.txt &
       snapcrawl go example.com -W360 -H480
       snapcrawl go example.com --selector "#main-content"
-      snapcrawl go example.com --only "products|collections"
-
+      snapcrawl go example.com --only "products|collections"
+
 ---
 
 ## Notes
 
 1. If a URL cannot be found, SnapCrawl will report to stderr. 
-   You can create a report by running `snapcrawl go example.com 2> err.txt`
+   You can create a report by running `snapcrawl go example.com 2> err.txt`
+
+## Todo
+
+- [x] Tests (probably against some ad hoc sinatra)
+- [ ] Make ths test server start/stop automatically when testing
+- [ ] Move ignored file extensions and mailto/tel links to config
+- [ ] Add screen size presets (also to user-overridable config)

data/lib/snapcrawl/crawler.rb

@@ -18,10 +18,10 @@ module Snapcrawl
     def initialize
       @storefile  = "snapcrawl.pstore"
       @store      = PStore.new(@storefile)
-      @done       = []
     end
 
     def handle(args)
+      @done = []
       begin
         execute Docopt::docopt(doc, argv: args)
       rescue Docopt::Exit => e
@@ -34,6 +34,12 @@ module Snapcrawl
       crawl args['<url>'].dup, opts_from_args(args)
     end
 
+    def clear_cache
+      FileUtils.rm @storefile if File.exist? @storefile
+    end
+
+    private
+
     def crawl(url, opts={})
       defaults = {
         width: 1280,
@@ -54,8 +60,6 @@ module Snapcrawl
       end
     end
 
-    private    
-
     def crawl_and_snap(urls)
       new_urls = []
       urls.each do |url|
@@ -94,7 +98,7 @@ module Snapcrawl
       fetch_opts[:div]    = @opts.selector if @opts.selector
       # :top => 0, :left => 0, :width => 100, :height => 100 # dimensions for a specific area
 
-      screenshot = f.fetch fetch_opts 
+      f.fetch fetch_opts 
       say "done"
     end
 
@@ -128,12 +132,12 @@ module Snapcrawl
 
     # mkdir the screenshots folder, if needed
     def make_screenshot_dir(dir)
-      Dir.exists? dir or FileUtils.mkdir_p dir
+      Dir.exist? dir or FileUtils.mkdir_p dir
     end
 
     # Convert any string to a proper handle
     def handelize(str)
-      str.downcase.gsub /[^a-z0-9]+/, '-'
+      str.downcase.gsub(/[^a-z0-9]+/, '-')
     end
 
     # Return proper image path for a UR
@@ -148,7 +152,7 @@ module Snapcrawl
 
     # Return true if the file exists and is not too old
     def file_fresh?(file)
-      File.exist?(file) and file_age(file) < @opts.age
+      @opts.age > 0 and File.exist?(file) and file_age(file) < @opts.age
     end
 
     # Return file age in seconds
@@ -158,27 +162,33 @@ module Snapcrawl
 
     # Process an array of links and return a better one
     def normalize_links(links)
-      # Remove the #hash part from all links
-      links = links.map {|link| link.attribute('href').to_s.gsub(/#.+$/, '')}
+      extensions = "png|gif|jpg|pdf|zip"
+      beginnings = "mailto|tel"
 
-      # Make unique and remove empties
-      links = links.uniq.reject {|link| link.empty?}
+      links_array = []
 
-      # Remove links to images and other files
-      extensions = "png|gif|jpg|pdf|zip"
-      links = links.reject {|link| link =~ /\.(#{extensions})(\?.*)?$/}
+      links.each_with_index do |link|
+        link = link.attribute('href').to_s
 
-      # Remove mailto, tel links
-      beginnings = "mailto|tel"
-      links = links.reject {|link| link =~ /^(#{beginnings})/}
+        # remove #hash
+        link.gsub!(/#.+$/, '')
+        next if link.empty?
 
-      # Add the base domain to relative URLs
-      links = links.map {|link| link =~ /^http/ ? link : "http://#{@base}#{link}"}
+        # Remove links to images and other files then to mailto/tel 
+        next if link =~ /\.(#{extensions})(\?.*)?$/
+        next if link =~ /^(#{beginnings})/
+        
+        # Add the base domain to relative URLs
+        link = link =~ /^http/ ? link : "#{@opts.base}#{link}" 
+        link = "http://#{link}" unless link =~ /^http/
 
-      # Keep only links in our base domain
-      links = links.select {|link| link =~ /https?:\/\/#{@base}.*/}
+        # Keep only links in our base domain
+        next unless link.include? @opts.base
 
-      links
+        links_array << link
+      end
+
+      links_array.uniq
     end
 
     def show_version

data/lib/snapcrawl/version.rb

@@ -1,3 +1,3 @@
 module Snapcrawl
-  VERSION = "0.2.1"
+  VERSION = "0.2.2"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: snapcrawl
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.2
 platform: ruby
 authors:
 - Danny Ben Shitrit
@@ -24,20 +24,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.3'
-- !ruby/object:Gem::Dependency
-  name: net-ssh
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: docopt
   requirement: !ruby/object:Gem::Requirement
@@ -108,6 +94,48 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.2'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.8'
+- !ruby/object:Gem::Dependency
+  name: minitest-reporters
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
 description: Snapcrawl is a command line utility for crawling a website and saving
   screenshots.
 email: db@dannyben.com