killsite 0.1.0 → 0.2.0

data/README.rdoc

@@ -1,16 +1,17 @@
 = killsite
 
-Description goes here.
-
-== Contributing to killsite
- 
-* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet
-* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it
-* Fork the project
-* Start a feature/bugfix branch
-* Commit and push until you are happy with your contribution
-* Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
-* Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
+Recursively GET all the pages of a given site through hyperlinks.
+The tool is for the usage of testing and profiling.
+
+== Installation
+
+Simply `gem install killsite`.
+
+== Usage
+
+  killsite [website] [# of GET for each pages]
+
+The second argument can be omitted.
 
 == Copyright
 

data/VERSION

@@ -1 +1 @@
-0.1.0
+0.2.0

data/bin/killsite

@@ -10,6 +10,6 @@ end
 prefix = ARGV[0]
 limit = (ARGV[1] || 1).to_i
 
-ks = KillSite.new(prefix, limit, true)
+killer = SiteKiller.new(prefix, limit, true)
 
-EM.run { ks.run }
+EM.run { limit.times { killer.run } }

data/killsite.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{killsite}
-  s.version = "0.1.0"
+  s.version = "0.2.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Andrew Liu"]
-  s.date = %q{2011-07-15}
+  s.date = %q{2011-07-16}
   s.default_executable = %q{killsite}
   s.description = %q{Recursively GET the site by links, try to kill the site}
   s.email = %q{andrewliu33@gmail.com}
@@ -29,8 +29,7 @@ Gem::Specification.new do |s|
     "bin/killsite",
     "killsite.gemspec",
     "lib/killsite.rb",
-    "test/helper.rb",
-    "test/test_killsite.rb"
+    "test/helper.rb"
   ]
   s.homepage = %q{http://github.com/eggegg/killsite}
   s.licenses = ["MIT"]

data/lib/killsite.rb

@@ -2,46 +2,48 @@ require 'bundler'
 require 'uri'
 Bundler.require
 
-class KillSite
+class SiteKiller
   def initialize(prefix, limit = 1, verbose = false)
     @count = 1
-    @prefix = prefix
-    @visited = [prefix]
-    @queue = [prefix]
+    @prefix = URI.parse(prefix)
+    @visited = { @prefix => 0 }
     @verbose = verbose
     @limit = limit
   end
 
-  def run
-    EM.stop if @count == 0
-    return if @queue.empty?
-
-    url = @queue.shift
+  def run url = nil
+    url ||= @prefix
 
     http = EventMachine::HttpRequest.new(url).get
     http.callback do
-      puts "Processing '#{url}'" if @verbose
-
-      Nokogiri::HTML.parse(http.response).xpath("//a[@href]").each do |link|
-	next_url = process_url link['href']
-	if next_url and !@visited.include? next_url
-	  puts "  Queueing '#{next_url}'" if @verbose
+      if @visited[url] == 0
+        puts "Processing '#{url}'" if @verbose
+        Nokogiri::HTML.parse(http.response).xpath("//a[@href]").each do |link|
+          next_url = process_url link['href']
+          if next_url and !@visited.include?(next_url)
+            puts "  Queueing '#{next_url}'" if @verbose
 
-	  @limit.times { @queue << next_url }
-	  @visited << next_url
-	  @count += @limit
-	end
+            @visited[next_url] = 0
+            @count += @limit
+            @limit.times { run next_url }
+          end
+        end
+        print "  Progress " if @verbose
+      else
+        print '*' if @verbose
       end
+
+      @visited[url] += 1
+      puts if @verbose and @visited[url] == @limit
       @count -= 1
-      EM.next_tick { run }
+      EM.stop if @count == 0
     end
   end
 
-private
+  private
   def process_url url
-    return nil if url =~ /^http/ and URI.parse(@prefix).host != URI.parse(url).host
-    return nil if url =~ /^javascript/
-    return nil if url =~ /^#/
-    URI.join @prefix, url
+    return nil if url =~ /^http/ and URI.parse(@prefix.to_s).host != URI.parse(url).host
+    return nil if url =~ /^javascript/ or url =~ /^#/
+    URI.join @prefix.to_s, url
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: killsite
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,12 +9,12 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-07-15 00:00:00.000000000 +08:00
+date: 2011-07-16 00:00:00.000000000 +08:00
 default_executable: killsite
 dependencies:
 - !ruby/object:Gem::Dependency
   name: em-http-request
-  requirement: &2154293760 !ruby/object:Gem::Requirement
+  requirement: &2157593620 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -22,10 +22,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2154293760
+  version_requirements: *2157593620
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &2154293280 !ruby/object:Gem::Requirement
+  requirement: &2157593140 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -33,10 +33,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2154293280
+  version_requirements: *2157593140
 - !ruby/object:Gem::Dependency
   name: shoulda
-  requirement: &2154292800 !ruby/object:Gem::Requirement
+  requirement: &2157592660 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -44,10 +44,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2154292800
+  version_requirements: *2157592660
 - !ruby/object:Gem::Dependency
   name: bundler
-  requirement: &2154292320 !ruby/object:Gem::Requirement
+  requirement: &2157592180 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -55,10 +55,10 @@ dependencies:
         version: 1.0.0
   type: :development
   prerelease: false
-  version_requirements: *2154292320
+  version_requirements: *2157592180
 - !ruby/object:Gem::Dependency
   name: jeweler
-  requirement: &2154291840 !ruby/object:Gem::Requirement
+  requirement: &2157591700 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -66,10 +66,10 @@ dependencies:
         version: 1.6.4
   type: :development
   prerelease: false
-  version_requirements: *2154291840
+  version_requirements: *2157591700
 - !ruby/object:Gem::Dependency
   name: rcov
-  requirement: &2154291360 !ruby/object:Gem::Requirement
+  requirement: &2157591220 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -77,7 +77,7 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2154291360
+  version_requirements: *2157591220
 description: Recursively GET the site by links, try to kill the site
 email: andrewliu33@gmail.com
 executables:
@@ -98,7 +98,6 @@ files:
 - killsite.gemspec
 - lib/killsite.rb
 - test/helper.rb
-- test/test_killsite.rb
 has_rdoc: true
 homepage: http://github.com/eggegg/killsite
 licenses:
@@ -115,7 +114,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 4297911674037475226
+      hash: -2909793090681244207
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:

data/test/test_killsite.rb

@@ -1,7 +0,0 @@
-require 'helper'
-
-class TestKillsite < Test::Unit::TestCase
-  should "probably rename this file and start testing for real" do
-    flunk "hey buddy, you should probably rename this file and start testing for real"
-  end
-end