crawl 1.1.1 → 1.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d31f6ad78385f65e06abb7837358453330394baf
-  data.tar.gz: 7a8f0781ae652f1500b9a5fde7a08380627ce1de
+  metadata.gz: fe74cc42a5729f8b4a6773e6c7e2326abddefc7a
+  data.tar.gz: 22677615aa464e1549a34b99273631337bb11cc1
 SHA512:
-  metadata.gz: 071573a7d5bd59416b2d6a582a51a0a56a30000184391ae05688ed894b40b50e410661dfef2ac4554fd29f2dae117504bccdda00ef9144888c8ca46210c9f980
-  data.tar.gz: 7954d33220e05baf7667590b6fcb082800fbbda753016d2014331a4afec02ef8f8951f68256a90ea4baaee4024abadde53541e256d59b2171b9f61f629323d71
+  metadata.gz: 4fe1ac4cd0bf951d62a2b0c6851818b5317f8dfd469ce4133478a1a179a08e9566d627a5b2ba02b4301fa48bc511686ba02e71edeaf7f41f4e6125643b563105
+  data.tar.gz: 1f69f772212400ea07ad8cb27d2c6daefb5549805d3cb1e2aa632e7ac63b8fa5b801a926d8341721a4e35d94d55e0c54daa71e3393da96b593f17954416298f6

data/LICENSE.txt

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 AlphaSights
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -10,7 +10,7 @@ Usage:
     -s, --start /home,/about         Starting path(s), defaults to /
     -u, --username username          Basic auth username
     -p, --password password          Basic auth password
-    -c, --connections count          Max mumber of parallel connections to use. The default is 20.
+    -c, --connections count          Max mumber of parallel connections to use. The default is 5.
     -v, --verbose                    Give details when crawling
     -h, --help                       Show this message
         --version                    Print version
@@ -34,3 +34,8 @@ Example:
 
     5 pages crawled without errors.
 
+### Copyright and License
+
+Copyright AlphaSights and Contributors, 2015
+
+[MIT Licence](LICENSE.txt)

data/bin/crawl

@@ -23,6 +23,8 @@ unless options[:domain]
   exit -1
 end
 
+options[:domain] = "http://#{options[:domain]}" unless options[:domain].include?('://')
+
 crawler = Crawl::Engine.new(options)
 
 trap("SIGINT") do

data/lib/crawl.rb

@@ -1,7 +1,6 @@
 # encoding: utf-8
 require('nokogiri')
 require('rest_client')
-require 'ci/reporter/core'
 require 'eventmachine'
 require 'em-http-request'
 require 'base64'

data/lib/crawl/engine.rb

@@ -59,21 +59,21 @@ private
   def retrieve(page)
     puts "Fetching #{page.url} ..." if $verbose
 
-    full_url = options[:domain] + page.url
+    absolute_url = options[:domain] + page.relative_url
 
-    http = EventMachine::HttpRequest.new(full_url)
+    http = EventMachine::HttpRequest.new(absolute_url)
     req = http.get :redirects => MAX_REDIRECTS, :head => {'authorization' => [options[:username], options[:password]]}
     req.timeout(15)
 
     req.errback do
       if req.nil?
-         page.intermittent("Req is nil. WAT?")
+        page.intermittent("Req is nil. WAT?")
       elsif msg = req.error
-       page.intermittent(msg)
+        page.intermittent(msg)
       elsif req.response.nil? || req.response.empty?
-       page.intermittent('Timeout?')
+        page.intermittent('Timeout?')
       else
-       page.intermittent('Partial response: Server Broke Connection?')
+        page.intermittent('Partial response: Server Broke Connection?')
       end
       process_next
     end

data/lib/crawl/page.rb

@@ -13,6 +13,14 @@ class Page
     @errors = nil
   end
 
+  def relative_url
+    if url.start_with?('/')
+      url
+    else
+      "#{source}/#{url}"
+    end
+  end
+
   def <=>(other)
     url <=> other.url
   end

data/lib/crawl/version.rb

@@ -1,4 +1,4 @@
 # encoding: utf-8
 module Crawl
-  VERSION = "1.1.1"
+  VERSION = "1.1.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: crawl
 version: !ruby/object:Gem::Version
-  version: 1.1.1
+  version: 1.1.2
 platform: ruby
 authors:
 - Tor Erik Linnerud
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-02-12 00:00:00.000000000 Z
+date: 2015-10-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -76,6 +76,7 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - Gemfile
+- LICENSE.txt
 - README.md
 - Rakefile
 - bin/crawl
@@ -107,7 +108,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.5.1
 signing_key: 
 specification_version: 4
 summary: Crawl pages witin a domain, reporting any page that returns a bad response