snapcrawl 0.2.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 8de37399949606ebc180cc7a80bf3ad91869e064
+  data.tar.gz: 0b56bd7207f4b0f0e44ecdfcffe890d50bf1d34c
+SHA512:
+  metadata.gz: 1ef25f70c86ce8f8d626b43d080d4f4048cae9a421111a3fbd232ebc6f26d9d4b67d69a90bbe306b3ff995c136ea8a4252b693dac6e1f2f82c1ff0d8a7e379e9
+  data.tar.gz: 9fb48c4490dc64c14284cbfd2bd4809ec1c6de67d535a6e4175aed4140ecc7366ebe92c791551668c08ef5b30a0b9d4f9ff6eeebee83af715d6910b9ab9f9df4

data/README.md

@@ -0,0 +1,23 @@
+# SnapCrawl - crawl a website and take screenshots
+
+`snapcrawl` is a command line utility for crawling a website and saving
+screenshots. It is using [Runfile](https://github.com/DannyBen/runfile).
+
+## Features
+
+- Crawls a website to any given depth and save screenshots
+- Can capture the full length of the page
+- Can use a specific resolution for screenshots
+- Skips capturing if the screenshot was already saved recently
+- Uses local caching to avoid expensive crawl operations if not needed
+- Reports broken links
+
+## Install
+
+	$ gem install snapcrawl
+
+## Usage
+
+	$ snapcrawl --help
+
+

data/bin/snapcrawl

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+require 'snapcrawl'
+trap(:INT) { abort "\r\nGoodbye" }
+include Snapcrawl
+Crawler.instance.handle ARGV
+

data/lib/snapcrawl.rb

@@ -0,0 +1,4 @@
+require 'snapcrawl/version'
+require 'snapcrawl/crawler'
+
+self.extend Snapcrawl

data/lib/snapcrawl/crawler.rb

@@ -0,0 +1,200 @@
+require 'colsole'
+require 'docopt'
+require 'fileutils'
+require 'nokogiri'
+require 'open-uri'
+require 'ostruct'
+require 'pstore'
+require 'screencap'
+
+module Snapcrawl
+  include Colsole
+
+  class Crawler
+    def self.instance
+      @@instance ||= self.new
+    end
+
+    def initialize
+      @storefile  = "snapcrawl.pstore"
+      @store      = PStore.new(@storefile)
+      @done       = []
+    end
+
+    def handle(args)
+      begin
+        execute Docopt::docopt(doc, argv: args)
+      rescue Docopt::Exit => e
+        puts e.message
+      end
+    end
+
+    def execute(args)
+      return show_version if args['--version']
+      crawl args['<url>'].dup, opts_from_args(args)
+    end
+
+    def crawl(url, opts={})
+      defaults = {
+        depth: 1,
+        age: 86400,
+        dir: 'snaps',
+        base: url, 
+      }
+      urls = [protocolize(url)]
+
+      @opts = OpenStruct.new defaults.merge(opts)
+
+      make_screenshot_dir @opts.dir
+
+      @opts.depth.times do 
+        urls = crawl_and_snap urls
+      end
+    end
+
+    private    
+
+    def crawl_and_snap(urls)
+      new_urls = []
+      urls.each do |url|
+        next if @done.include? url
+        @done << url
+        say "\n!txtgrn!-----> Visit: #{url}"
+        snap url
+        new_urls += extract_urls_from url
+      end
+      new_urls
+    end
+
+    # Take a screenshot of a URL, unless we already did so recently
+    def snap(url)
+      file = image_path_for(url)
+      if file_fresh? file
+        say "       Snap:  Skipping. File exists and seems fresh"
+      else
+        snap!(url)
+      end
+    end
+
+    # Take a screenshot of the URL, even if file exists
+    def snap!(url)
+      say "       !txtblu!Snap!!txtrst!  Snapping picture... "
+
+      f = Screencap::Fetcher.new url
+      fetch_opts = {}
+      fetch_opts[:output] = image_path_for(url)
+      fetch_opts[:width]  = @opts.width
+      fetch_opts[:height] = @opts.height if @opts.height
+      # :height => 768,
+      # :div => '.header', # selector for a specific element to take screenshot of
+      # :top => 0, :left => 0, :width => 100, :height => 100 # dimensions for a specific area
+
+      screenshot = f.fetch fetch_opts 
+      say "done"
+    end
+
+    def extract_urls_from(url)
+      cached = nil
+      @store.transaction { cached = @store[url] }
+      if cached
+        say "       Crawl: Page was cached. Reading subsequent URLs from cache"
+        return cached
+      else
+        return extract_urls_from! url
+      end
+    end
+
+    def extract_urls_from!(url)
+      say "       !txtblu!Crawl!!txtrst! Extracting links... "
+
+      begin
+        doc = Nokogiri::HTML open url
+        links = doc.css('a')
+        links = normalize_links links
+        @store.transaction { @store[url] = links }
+        say "done"
+      rescue OpenURI::HTTPError => e
+        links = []
+        say "!txtred!FAILED"
+        say! "!txtred!  !    HTTP Error: #{e.message} at #{url}"
+      end
+      links
+    end
+
+    # mkdir the screenshots folder, if needed
+    def make_screenshot_dir(dir)
+      Dir.exists? dir or FileUtils.mkdir_p dir
+    end
+
+    # Convert any string to a proper handle
+    def handelize(str)
+      str.downcase.gsub /[^a-z0-9]+/, '-'
+    end
+
+    # Return proper image path for a UR
+    def image_path_for(url)
+      "#{@opts.dir}/#{handelize(url)}.png"
+    end
+
+    # Add protocol to a URL if neeed
+    def protocolize(url)
+      url =~ /^http/ ? url : "http://#{url}"
+    end
+
+    # Return true if the file exists and is not too old
+    def file_fresh?(file)
+      File.exist?(file) and file_age(file) < @opts.age
+    end
+
+    # Return file age in seconds
+    def file_age(file)
+      (Time.now - File.stat(file).mtime).to_i
+    end
+
+    # Process an array of links and return a better one
+    def normalize_links(links)
+      # Remove the #hash part from all links
+      links = links.map {|link| link.attribute('href').to_s.gsub(/#.+$/, '')}
+
+      # Make unique and remove empties
+      links = links.uniq.reject {|link| link.empty?}
+
+      # Remove links to images and other files
+      extensions = "png|gif|jpg|pdf|zip"
+      links = links.reject {|link| link =~ /\.(#{extensions})(\?.*)?$/}
+
+      # Remove mailto, tel links
+      beginnings = "mailto|tel"
+      links = links.reject {|link| link =~ /^(#{beginnings})/}
+
+      # Add the base domain to relative URLs
+      links = links.map {|link| link =~ /^http/ ? link : "http://#{@base}#{link}"}
+
+      # Keep only links in our base domain
+      links = links.select {|link| link =~ /https?:\/\/#{@base}.*/}
+
+      links
+    end
+
+    def show_version
+      puts VERSION
+    end
+
+    def doc
+      return @doc if @doc 
+      @doc = File.read template 'docopt.txt'
+    end
+
+    def template(file)
+      File.expand_path("../templates/#{file}", __FILE__)
+    end
+
+    def opts_from_args(args)
+      opts = {}
+      opts[:folder] = args['--folder'] if args['--folder']
+      opts[:age] = args['--age'].to_i if args['--age']
+      opts[:depth] = args['--depth'].to_i if args['--depth']
+      opts
+    end
+  end
+end

data/lib/snapcrawl/templates/docopt.txt

@@ -0,0 +1,15 @@
+Snapcrawl
+
+Usage:
+  snapcrawl go <url> [options]
+  snapcrawl -h | --help 
+  snapcrawl -v | --version
+
+Options:
+  -f --folder <path>     Where to save screenshots [default: snaps]
+  -a --age <n>           Number of seconds to consider screenshots fresh
+                         [default: 86400]
+  -d --depth <n>         Number of levels to crawl [default: 1]
+  -h --help              Show this screen
+  -v --version           Show version
+

data/lib/snapcrawl/version.rb

@@ -0,0 +1,3 @@
+module Snapcrawl
+  VERSION = "0.2.0"
+end

metadata

@@ -0,0 +1,149 @@
+--- !ruby/object:Gem::Specification
+name: snapcrawl
+version: !ruby/object:Gem::Version
+  version: 0.2.0
+platform: ruby
+authors:
+- Danny Ben Shitrit
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-12-05 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: colsole
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+- !ruby/object:Gem::Dependency
+  name: net-ssh
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: docopt
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: screencap
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+- !ruby/object:Gem::Dependency
+  name: runfile
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
+- !ruby/object:Gem::Dependency
+  name: run-gem-dev
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+description: Snapcrawl is a command line utility for crawling a website and saving
+  screenshots.
+email: db@dannyben.com
+executables:
+- snapcrawl
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- bin/snapcrawl
+- lib/snapcrawl.rb
+- lib/snapcrawl/crawler.rb
+- lib/snapcrawl/templates/docopt.txt
+- lib/snapcrawl/version.rb
+homepage: https://github.com/DannyBen/snapcrawl
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.6
+signing_key: 
+specification_version: 4
+summary: Crawl a website and take screenshots (CLI + Library)
+test_files: []