driller 0.1.1 → 0.1.2

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    ZWQzNGUxOWVjMmE4ZThmMjVmMTYwMTA2OTk4ZTYxZjk0ZGIzNDUzOQ==
+    YzMzYWYyZGM4MzA2OTk1NDZlNjQ4ZTQyOWFkN2UzNmNiMjYxNDQyMQ==
   data.tar.gz: !binary |-
-    ZjFkNmFlOWE5Njg0YzhlODlmODcxMWI5NGI4MTZkYzM4NzQyZmFlZg==
+    NGM4YWIwZTU3YTY4MDIxMzhiM2UyZWM0ZGQ2M2Q5ZDlmMDBhNDAyNA==
 SHA512:
   metadata.gz: !binary |-
-    NGM5ZjcyMDk1MjU2NzZmYzZhZWVkNmFjODk0M2Y5Nzg3Y2VjOWFiYmU0ZWEz
-    NDM5MDFlMDhjZDhjZjQ0NjBiNTg4NzAyNjVlNzYzY2ZmNzY5MDUyMTMwYWUx
-    NzY4MDkxNDU2ZTk2NjRlYTI0NWVhMjI3YjlmNGViOGNlMDQ4YjA=
+    Mzk4NDU2ZDlkNTEyMDNlZDgwMjQxNDMwNjM0MDJiYzA4ZTA5Y2JkMDRkODlk
+    ZmFlOTI0ZDU4MDRkNTBmNjk1OWY4YTllNTAyNDEwOTA1ODRjMDVlODcxY2Rj
+    NDE2ODcyYTcwODhlYjY0NWUwMDJlNzE5MDA2MDdhNzc1NDQ3ZjI=
   data.tar.gz: !binary |-
-    NzAxNmE4Y2VmZDkyZTM0MjVkNTM3Y2QxOGM1NmRlODhmZDQyYTViOWQ3Mjg0
-    NjQyMzUyYTNjMDViODg0Y2M3YzU1OWYzZjE0MjU2YTQ3YzZhMzZlOWUyZGFk
-    MzEyODVkODQyN2I0OTAyYjk1ZGJhNmM2OTg4OWJhZmJhZDE4ZjM=
+    NDNhZWQ5YWYwNWY1MDE5NzBhYzJjZDgwMTJmMWU1ZmE4NDliMzE5NzdhYTM1
+    OWViOTFlYWZjOTg2M2JlY2E0MzQ2NGI3ZDlhZWYxYWI1ZmZhM2ZhNjA4NWQ1
+    ZDA2NTdiYmU2NDlmZjE2NzdlMjU0Y2IwYzgxYWFkNzA4NDMxNjE=

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2015 Shashikant86
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,54 @@
+# Driller
+
+Driller is a command line Ruby based web crawler based on Anemone. Driller can  
+
+* Crawl website and reports error pages which are not 200 or 301. This will report all other HTTP codes.
+* Driller will report slow pages which are returned response time > 5000
+* This will create three HTML files valid_urls.html which are 200 response. broken.html wich are not 200. slow_pages.html which are retuned reaponse time > 5000
+
+
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'driller'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install driller
+
+## Usage
+
+  Driller takes two arguments
+
+  * URL of the page to be crawled
+  * Depth of the crawling
+  * Proxy Hostname (optional)   [default: nil]
+  * Proxy Port (optional)       [default: 80]
+
+          $ driller <webpage> <depth> [<proxy_host>] [<proxy_port>]
+          $ driller http://www.example.com 2
+          $ driller http://www.example.com 2 'www-proxy.domain.co.uk' 80
+
+  If you have installed it from bundle the
+
+           $ bundle exec driller  http://www.example.com 2
+
+ This will crawl website upto level 2. You can increase depth as per your need. This will create three HTML files valid_urls.html which are 200 response. broken.html wich are not 200. slow_pages.html which are retuned reaponse time > 5000
+
+ You an display these html files to CI server.
+
+## Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/driller/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/bin/driller

@@ -0,0 +1,21 @@
+##!/usr/bin/env ruby
+require_relative '../lib/driller/main'
+
+webpage = ARGV[0]
+depth   = ARGV[1]
+
+proxy_host = ARGV[2]
+proxy_port = ARGV[3]
+
+if proxy_port.nil?
+    proxy_port = 80
+end
+
+if webpage.nil? || depth.nil?
+    puts "Error: Invalid number of arguments supplied."
+    abort("Usage: driller <webpage> <depth>")
+end
+
+
+m = Main.new(webpage, depth, proxy_host, proxy_port)
+m.execute

data/driller.gemspec

@@ -0,0 +1,22 @@
+# coding: utf-8
+lib = File.expand_path('./lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+# require_relative './lib/driller/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "driller"
+  spec.version       = "0.1.2"
+  spec.authors       = ["Shashikant86", "Qambar"]
+  spec.email         = ["shashikant.jagtap@aol.co.uk"]
+  spec.summary       = %q{Drill your website for error and slow pages}
+  spec.description   = %q{Driller is a command line Ruby based web crawler based on Anemone. Driller can crawl website and reports error pages which are not 200 or 301.}
+  spec.homepage      = "https://github.com/Shashikant86/driller"
+  spec.license       = "MIT"
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+  spec.add_runtime_dependency "bundler", "~> 1.0"
+  spec.add_runtime_dependency "rake"
+  spec.add_runtime_dependency "anemone"
+end

data/lib/driller/file_handler.rb

@@ -0,0 +1,15 @@
+class File_Handler
+    def initialize(filename, data)
+        write(filename, data)
+    end 
+
+    def write(filename, data)
+        begin 
+            file = File.new(filename + '.html', 'w')
+            file.puts data
+            file.close
+        rescue
+            puts "Unable to write to file, please check your permissions" 
+        end
+    end
+end

data/lib/driller/html_report.rb

@@ -0,0 +1,57 @@
+class Html_Report
+    def initialize(reportName, rows)   
+        @reportName = reportName     
+        @html = ""
+        createTable(rows)
+    end 
+
+    def createTable(rows)
+
+        if rows.length == 0
+            @html = "#{@html}<tr><td>There are no #{@reportName}.</td></tr>"
+            return 
+        end
+
+        addHeading(rows[0])
+
+        if rows.kind_of?(Array)
+            rows.each do |row|
+                unless row.nil?
+                    addRow(row)
+                end
+            end
+        else 
+            abort("Rows should be an array in Html_Report");
+        end
+    end
+
+    def addHeading(row)
+        @html = "#{@html}<tr>"
+        row.each do |key, value|
+            @html = "#{@html}<th>"
+            @html = "#{@html} #{(key.to_s).capitalize}"
+            @html = "#{@html}</th>"
+        end
+        @html = @html + "</tr>"
+    end
+
+    def addRow(row)
+        @html = @html + "<tr>"
+        row.each do |key, value|
+            @html = "#{@html}<td>"
+            @html = "#{@html}#{value}"
+            @html = "#{@html}</td>"
+        end
+        @html = @html + "</tr>"
+    end
+
+    def getReport
+      
+        @htmlWithWrapper = "<!DOCTYPE html><html><head><title>Report</title></head><body>"
+        @htmlWithWrapper = @htmlWithWrapper + "<h1>#{@reportName.capitalize}</h1>"
+        @htmlWithWrapper = @htmlWithWrapper + "<table class='table'>#{@html}</table>"
+        @htmlWithWrapper = @htmlWithWrapper + "</body></html>"
+        
+        return @htmlWithWrapper
+    end
+end

data/lib/driller/main.rb

@@ -0,0 +1,117 @@
+#!/usr/bin/env ruby
+require_relative "version"
+require_relative "uri_helper"
+require_relative "file_handler"
+require_relative "html_report"
+
+require "fileutils"
+require "rubygems"
+require "anemone"
+
+class Main
+    attr_accessor :valid, :invalid, :slow, :extremely_slow
+    
+    def initialize(webpage, depth, proxy_host, proxy_port)
+        @webpage    = webpage
+        @depth      = depth
+        @proxy_port = proxy_port
+        @proxy_host = proxy_host
+
+        @pageCount  = 0
+
+        # initializing symbols
+    end
+
+    def execute
+        puts "Webpage   : "    + @webpage
+        puts "Depth     : "    + @depth
+
+        result = Hash.new
+        Anemone.crawl(@webpage) do |anemone|
+
+            unless @proxy_host.nil?
+                anemone.proxy_host = @proxy_host
+                anemone.proxy_port = @proxy_port
+            end
+            
+            anemone.depth_limit = @depth.to_i
+            puts "============= Driller is now checking your website links. If any of the link returned non 200, it will be displayed here========="
+            anemone.focus_crawl do |page|
+                page.links.select { |url| url.starts_with? @webpage }
+            end
+
+
+            result[:valid]              = Array.new
+            result[:invalid]            = Array.new
+            result[:slow]               = Array.new
+            result[:extremely_slow]     = Array.new
+
+            anemone.on_every_page do |page|
+
+                pageObject = getPageObject(page)
+
+                if page.code == 200
+                    result[:valid].push(pageObject)
+
+                    # Check Response Time
+                    # -------------------
+                    # We only check response time
+                    # for pages which are 200
+                    # because there is no point of 
+                    # optimizing error pages.
+                    
+                    if page.response_time > 10000
+                        result[:slow].push(pageObject)
+
+                        puts "=======Slow Page======\n"
+                        puts "Time: #{page.response_time} - #{page.url}"
+                    elsif page.response_time > 5000
+                        result[:extremely_slow].push(pageObject)
+
+                        puts "=======Very Slow Page======\n"
+                        puts "Time: #{page.response_time} - #{page.url}"
+                    end
+                    #End checking response time.
+
+                else #404, 301, 500
+                    result[:invalid].push(pageObject)
+
+                    puts "======= NON-200 Page ======\n"
+                    puts "#{page.code} Response from : #{page.url}"
+                    puts 
+
+                end
+
+            end
+
+            @pageCount = @pageCount + 1
+            puts "Checked #{@pageCount} pages" if @pageCount % 100 == 0
+        end
+
+        generateReport('valid_pages', result[:valid])
+        generateReport('broken', result[:invalid])
+        generateReport('slow_pages', result[:slow])
+        generateReport('extremely_slow_pages', result[:extremely_slow])
+
+    end
+
+    def getPageObject(page)
+        # This means we never got an access to internet
+        if page.code.nil?
+            abort("Error: Either you are offline or behind proxy.")
+        end
+
+        obj = Hash.new
+
+        obj['code']             = page.code
+        obj['url']              = (page.url).to_s
+        obj['response_time']    = page.response_time
+
+        return obj
+    end
+
+    def generateReport(reportName, data)
+        report = Html_Report.new(reportName, data)
+        File_Handler.new(reportName, report.getReport)
+    end
+end

data/lib/driller/uri_helper.rb

@@ -0,0 +1,8 @@
+module URI
+  class Generic
+    def starts_with?(prefix)
+      prefix = prefix.to_s
+      self.to_s[0, prefix.length] == prefix
+    end
+  end
+end

data/lib/driller/version.rb

@@ -0,0 +1,3 @@
+module Driller
+  VERSION = "0.1.2"
+end

data/lib/driller.rb

@@ -0,0 +1,8 @@
+require "driller/version"
+require "driller/uri_helper"
+require "driller/crawler"
+require "anemone"
+
+module Driller
+  
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: driller
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Shashikant86
@@ -57,10 +57,24 @@ description: Driller is a command line Ruby based web crawler based on Anemone.
   can crawl website and reports error pages which are not 200 or 301.
 email:
 - shashikant.jagtap@aol.co.uk
-executables: []
+executables:
+- driller
 extensions: []
 extra_rdoc_files: []
-files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/driller
+- driller.gemspec
+- lib/driller.rb
+- lib/driller/file_handler.rb
+- lib/driller/html_report.rb
+- lib/driller/main.rb
+- lib/driller/uri_helper.rb
+- lib/driller/version.rb
 homepage: https://github.com/Shashikant86/driller
 licenses:
 - MIT