ssoroka-spider_bot 0.9.1

data/README.rdoc

@@ -0,0 +1,50 @@
+= Spider bot
+ 	 
+a non-threaded spider bot that spiders a site with response time stats. easily extendable
+
+== Installation
+
+  sudo gem install ssoroka-spider_bot
+
+== Usage 	 
+ 	 
+Usable in code or as executable!
+
+  spider_bot http://www.example.com
+  spider_bot http://0.0.0.0:3000
+
+Example as script/spider:
+
+  #!/usr/bin/env ruby
+  require 'rubygems'
+  require 'spider_bot'
+
+  class MySpider < SpiderBot
+    # override these for handling events
+    def on_page(page)
+    end
+
+    def on_404(link)
+    end
+
+    def on_500(link)
+    end
+
+    # override these for changing how urls are classified as links
+    def off_site?(url)
+      url !~ /^\// # urls not starting with a /
+    end
+
+    def ignorable?(url)
+      url =~ /\/.*\..+/ && # files with extensions
+        url !~ /\.html$/ # but not html files
+    end
+  end
+
+  spider = MySpider.new(:quiet => false)
+  spider.start(ARGV[1])
+
+== Gem Requirements
+
+* ssoroka-ansi
+* mechanize

data/Rakefile

@@ -0,0 +1,14 @@
+require 'rubygems'
+require 'rake'
+require 'echoe'
+
+Echoe.new('spider_bot', '0.9.1') do |p|
+  p.description = 'A non-threaded spider bot that spiders a site with response time stats. easily extendable'
+  p.url = 'http://github.com/ssoroka/spider_bot'
+  p.author = 'Steven Soroka'
+  p.email = 'ssoroka78@gmail.com'
+  p.ignore_pattern = ["tmp/*"]
+  p.development_dependencies = ['mechanize', 'ssoroka-ansi']
+end
+
+Dir["#{File.dirname(__FILE__)}/tasks/*.rake"].sort.each{|f| load f }

data/bin/spider_bot

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'lib/spider_bot'
+
+spider = SpiderBot.new(:quiet => false)
+spider.start(ARGV[1])

data/lib/spider_bot.rb

@@ -0,0 +1,118 @@
+require 'mechanize'
+require 'ansi'
+require 'benchmark'
+
+class SpiderBot
+  # override these for handling events
+  def on_page(page)
+  end
+
+  def on_404(link)
+  end
+  
+  def on_500(link)
+  end
+
+  # override these for changing how urls are classified as links
+  def off_site?(url)
+    url !~ /^\// # urls not starting with a /
+  end
+
+  def ignorable?(url)
+    url =~ /\/.*\..+/ && # files with extensions
+      url !~ /\.html$/ # but not html files
+  end
+
+  ## implementation! :)
+  attr_accessor :fourohfour, :fivehundred, :response_times, :been_to, :go_to
+  def initialize(options = {})
+    @agent = WWW::Mechanize.new
+    @agent.user_agent = options[:agent] || 'spider bot'
+    @been_to = ['/']
+    @go_to = []
+    @fourohfour = []
+    @fivehundred = []
+    @response_times = []
+    @key = "\n\nKey:\n{#{ANSI.color(:yellow){'queued'}}}/{#{ANSI.color(:green){'hit'}}}/{#{ANSI.color(:red){'404s'}}}/{#{ANSI.color(:red){'500s'}}} {current action}"
+    @quiet = options[:quiet]
+  end
+  
+  def start(url = nil)
+    @starting_url = url || 'http://0.0.0.0:3000/'
+    fetch @starting_url
+  end
+  
+  def fetch(link)
+    status "fetching #{link}"
+    @been_to << link
+    begin
+      page = nil
+      real = Benchmark.measure {
+        page = @agent.get link
+      }.real
+      @response_times << [link, real]
+      (page/'a').each{|el|
+        if el['href']
+          url = el['href'].gsub(/\#.*/, '')
+          @go_to.push url unless @go_to.include?(url) || @been_to.include?(url) || off_site?(url) || ignorable?(url)
+        end
+      }
+      on_page(page) # on_page event
+    rescue Net::HTTPNotFound => e
+      @fourohfour << link
+      status "#{link} not found, 404"
+      on_404(link)
+    rescue Net::HTTPInternalServerError => e
+      @fivehundred << link
+      status ANSI.color(:red) {"#{link} dead! 500 error!"}
+      on_500(link)
+    rescue Interrupt => e
+      status "Interrupt caught, shutting down."
+      close_up_shop
+      exit
+    end
+
+    # fetch next link in list.
+    next_link = @go_to.shift
+    if next_link
+      fetch(next_link)
+    else
+      close_up_shop
+    end
+  end
+  
+  def status(s)
+    return if @quiet
+    to_go = ANSI.color(:yellow) { @go_to.size.to_s }
+    done = ANSI.color(:green) { @been_to.size.to_s }
+    fourohfour = ANSI.color(:red) { @fourohfour.size.to_s }
+    fivehundred = ANSI.color(:red) { @fivehundred.size.to_s }
+    STDOUT.print(ANSI.clear_screen + ANSI.up(100) + ANSI.left(100))
+    STDOUT.print("spidering #{@starting_url}..\n")
+    STDOUT.print("#{to_go}/#{done}/#{fourohfour}/#{fivehundred}: #{s}")
+    STDOUT.print("\n\nNext 15 links:\n#{@go_to[0..14].join("\n")}")
+    STDOUT.print(@key)
+    STDOUT.flush
+  end
+  
+  def close_up_shop
+    return if @quiet
+    STDOUT.puts "\n\nDone!"
+    if @fourohfour.any?
+      STDOUT.puts "Here are all your broken links:\n#{@fourohfour.join("\n")}"
+    else
+      STDOUT.puts "You have no broken links that I could find"
+    end
+    if @fivehundred.any?
+      STDOUT.puts "Here are all your dead 500 links:\n#{@fivehundred.join("\n")}"
+    else
+      STDOUT.puts "You have no dead (500) pages that I could find"
+    end
+    STDOUT.puts "\n5 slowest pages: "
+    @response_times.sort_by{|link, time|
+      -time
+    }.first(5).each{|link, time|
+      STDOUT.printf "%0.3fs %s\n", time, link
+    }
+  end
+end

data/spider_bot.gemspec

@@ -0,0 +1,37 @@
+Gem::Specification.new do |s|
+  s.name = %q{spider_bot}
+  s.version = "0.9.1"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Steven Soroka"]
+  s.date = %q{2008-12-04}
+  s.default_executable = %q{spider_bot}
+  s.description = %q{A non-threaded spider bot that spiders a site with response time stats. easily extendable}
+  s.email = %q{ssoroka78@gmail.com}
+  s.executables = ["spider_bot"]
+  s.extra_rdoc_files = ["bin/spider_bot", "lib/spider_bot.rb", "README.rdoc"]
+  s.files = ["bin/spider_bot", "lib/spider_bot.rb", "Rakefile", "README.rdoc", "spider_bot.gemspec", "Manifest"]
+  s.has_rdoc = true
+  s.homepage = %q{http://github.com/ssoroka/spider_bot}
+  s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Spider_bot", "--main", "README.rdoc"]
+  s.require_paths = ["lib"]
+  s.rubyforge_project = %q{spider_bot}
+  s.rubygems_version = %q{1.2.0}
+  s.summary = %q{A non-threaded spider bot that spiders a site with response time stats. easily extendable}
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 2
+
+    if current_version >= 3 then
+      s.add_development_dependency(%q<mechanize>, [">= 0"])
+      s.add_development_dependency(%q<ssoroka-ansi>, [">= 0"])
+    else
+      s.add_dependency(%q<mechanize>, [">= 0"])
+      s.add_dependency(%q<ssoroka-ansi>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<mechanize>, [">= 0"])
+    s.add_dependency(%q<ssoroka-ansi>, [">= 0"])
+  end
+end

metadata

@@ -0,0 +1,82 @@
+--- !ruby/object:Gem::Specification 
+name: ssoroka-spider_bot
+version: !ruby/object:Gem::Version 
+  version: 0.9.1
+platform: ruby
+authors: 
+- Steven Soroka
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2008-12-04 00:00:00 -08:00
+default_executable: spider_bot
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: mechanize
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: ssoroka-ansi
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+description: A non-threaded spider bot that spiders a site with response time stats. easily extendable
+email: ssoroka78@gmail.com
+executables: 
+- spider_bot
+extensions: []
+
+extra_rdoc_files: 
+- bin/spider_bot
+- lib/spider_bot.rb
+- README.rdoc
+files: 
+- bin/spider_bot
+- lib/spider_bot.rb
+- Rakefile
+- README.rdoc
+- spider_bot.gemspec
+- Manifest
+has_rdoc: true
+homepage: http://github.com/ssoroka/spider_bot
+post_install_message: 
+rdoc_options: 
+- --line-numbers
+- --inline-source
+- --title
+- Spider_bot
+- --main
+- README.rdoc
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "1.2"
+  version: 
+requirements: []
+
+rubyforge_project: spider_bot
+rubygems_version: 1.2.0
+signing_key: 
+specification_version: 2
+summary: A non-threaded spider bot that spiders a site with response time stats. easily extendable
+test_files: []
+