crawl 1.0.3 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4825e68f15b25c5f26947ea149b6c835ced7bdf8
-  data.tar.gz: fdebd71a13a40f51cb20a635c5e02ea6dd9a4f6f
+  metadata.gz: 70e1653730fc26973b11c738e31b30655948fb79
+  data.tar.gz: 43506b21e7ce399c35bef3e7a48f6acff72632ae
 SHA512:
-  metadata.gz: 25a485be0c41e3f23b60f5974806757c6a6ab6888bbadbf0df317b3c7bce1825616026a5d1cd0c2feaf223a7af844b3630de0d6e547108d87d77c00b89dd27c0
-  data.tar.gz: abe1b5261102f0cd498d75caedef412e4bee3a7995401fe3c88ef0619b01c4d18a46904db4d5b778916fbf76f1ae7faf0935758463ab45e389ef98915ff633e7
+  metadata.gz: 1b96daad9bdfb00388beffec236f13f9463f5df139160fe504779a46881eabc56ee9d357beff2609e933549f8638fbb391d29caa06902cae0aa1b10c3e581e0d
+  data.tar.gz: e4c7521800d88c5984177a9b1801a9dcb507081a3f4d0a309de896422a1ddcc7e4ce8a038f7e0906a83cccd22a3c5d374355592fb6915a52ecfae094fff20e40

data/Gemfile

@@ -1,3 +1,3 @@
-source 'http://rubygems.org'
+source 'https://rubygems.org'
 
 gemspec

data/README.md

@@ -1,25 +1,36 @@
 # Crawl
 
-Crawl pages witin a domain, reporting any page that returns a bad response code
+Crawl pages within a domain, reporting any page that returns a bad response code
 
 Usage:
 
     > crawl [options] domain
 
+    Usage: crawl [options] domain
     -s, --start /home,/about         Starting path(s), defaults to /
     -u, --username username          Basic auth username
     -p, --password password          Basic auth password
+    -c, --connections count          Max mumber of parallel connections to use. The default is 20.
     -v, --verbose                    Give details when crawling
-    -m, --markup                     Validate markup
     -h, --help                       Show this message
+        --version                    Print version
+
+
 
 Example:
 
-    > crawl http://alphasights.com --start=/no-such-page --verbose
+    > crawl https://engineering.alphasights.com --connections=5 --start=/ --verbose
 
-      Adding /no-such-page
-    Fetching /no-such-page ...
+      Adding /
+    Fetching / ...
+      Adding /positions/ruby-developer
+      Adding /positions/js-ember-developer
+      Adding /positions/ux-ui-designer
+      Adding /positions/support-specialist
+    Fetching /positions/ruby-developer
+    Fetching /positions/js-ember-developer ...
+    Fetching /positions/ux-ui-designer ...
+    Fetching /positions/support-specialist ...
 
-    Pages with errors:
-    /no-such-page found on the command line - Status code: 404
+    5 pages crawled without errors.
 

data/bin/crawl

@@ -2,12 +2,15 @@
 require 'optparse'
 require_relative '../lib/crawl.rb'
 
+EM.threadpool_size = 5
+
 options = {}
 optparse = OptionParser.new do |opts|
   opts.banner = "Crawl pages witin a domain, reporting any page that returns a bad response code\nUsage: crawl [options] domain"
   opts.on('-s', '--start /home,/about', Array, 'Starting path(s), defaults to /') { |o| options[:start] = o }
   opts.on('-u', '--username username', String, 'Basic auth username') { |o| options[:username] = o }
   opts.on('-p', '--password password', String, 'Basic auth password') { |o| options[:password] = o }
+  opts.on('-c', '--connections count', Integer, "Max mumber of parallel connections to use. The default is #{EM.threadpool_size}.") { |o| EM.threadpool_size = o }
   opts.on('-v', '--verbose', 'Give details when crawling') { |o| $verbose = o }
   opts.on_tail("-h", "--help", "Show this message") { |o| puts opts; exit }
   opts.on_tail("-v", "--version", "Print version") { |o| puts Crawl::VERSION; exit }

data/crawl.gemspec

@@ -14,8 +14,9 @@ Gem::Specification.new do |gem|
   gem.name          = "crawl"
   gem.require_paths = ["lib"]
   gem.version       = Crawl::VERSION
-  gem.add_dependency('nokogiri')
-  gem.add_dependency('rest-client')
-  gem.add_dependency('eventmachine', '1.0.1')
-  gem.add_dependency('em-http-request', '1.0.3')
+  gem.licenses      = ['MIT']
+  gem.add_dependency('nokogiri', '~> 1.6')
+  gem.add_dependency('rest-client', '~> 1.7')
+  gem.add_dependency('eventmachine', '~> 1.0')
+  gem.add_dependency('em-http-request', '~> 1.1')
 end

data/lib/crawl/version.rb

@@ -1,4 +1,4 @@
 # encoding: utf-8
 module Crawl
-  VERSION = "1.0.3"
+  VERSION = "1.1.0"
 end

metadata

@@ -1,71 +1,71 @@
 --- !ruby/object:Gem::Specification
 name: crawl
 version: !ruby/object:Gem::Version
-  version: 1.0.3
+  version: 1.1.0
 platform: ruby
 authors:
 - Tor Erik Linnerud
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-15 00:00:00.000000000 Z
+date: 2015-02-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.6'
 - !ruby/object:Gem::Dependency
   name: rest-client
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.7'
 - !ruby/object:Gem::Dependency
   name: eventmachine
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.1
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.1
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: em-http-request
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.3
+        version: '1.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.3
+        version: '1.1'
 description: Crawl all pages on a domain, checking for errors
 email:
 - tor@alphasights.com
@@ -74,7 +74,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
+- ".gitignore"
 - Gemfile
 - README.md
 - Rakefile
@@ -88,7 +88,8 @@ files:
 - lib/crawl/string.rb
 - lib/crawl/version.rb
 homepage: http://github.com/alphasights/crawl
-licenses: []
+licenses:
+- MIT
 metadata: {}
 post_install_message: 
 rdoc_options: []
@@ -96,19 +97,20 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.0
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Crawl pages witin a domain, reporting any page that returns a bad response
   code
 test_files: []
+has_rdoc: