killsite 0.2.0 → 0.3.0

data/Gemfile

@@ -1,6 +1,5 @@
 source :rubygems
 
-gem "em-http-request"
 gem "nokogiri"
 
 group :development do

data/Gemfile.lock

@@ -1,13 +1,6 @@
 GEM
   remote: http://rubygems.org/
   specs:
-    addressable (2.2.6)
-    em-http-request (0.3.0)
-      addressable (>= 2.0.0)
-      escape_utils
-      eventmachine (>= 0.12.9)
-    escape_utils (0.2.3)
-    eventmachine (0.12.10)
     git (1.2.5)
     jeweler (1.6.4)
       bundler (~> 1.0)
@@ -23,7 +16,6 @@ PLATFORMS
 
 DEPENDENCIES
   bundler (~> 1.0.0)
-  em-http-request
   jeweler (~> 1.6.4)
   nokogiri
   rcov

data/README.md

@@ -0,0 +1,25 @@
+= killsite
+
+Recursively GET all the pages of a given site through hyperlinks.
+The tool is for the usage of testing and profiling.
+
+Using memory monitor to find out the memory leaks within different URLs.
+
+== Installation
+
+Simply `gem install killsite`.
+`ab` (ApacheBench) is require as an benchmark tool.
+
+== Usage
+
+Usage: killsite [options]
+    -h, --help                       Show help message
+    -l, --limit NUM                  Setting the limit of a single test
+    -c, --concurrency NUM            Setting the number of concurrent connection
+    -p, --pid PID                    The PID is the monitored server process
+
+== Copyright
+
+Copyright (c) 2011 Andrew Liu. See LICENSE.txt for
+further details.
+

data/VERSION

@@ -1 +1 @@
-0.2.0
+0.3.0

data/bin/killsite

@@ -1,15 +1,59 @@
 #!/usr/bin/env ruby
 
+require 'optparse'
 require 'killsite'
+require 'memory_monitor'
 
-if ARGV.size == 0
-  puts "USAGE: killsite [site] [limit]"
-  exit
+if `which ab`.empty?
+  puts "ab (ApacheBench) must be installed to proceed"
+  exit -1
 end
 
-prefix = ARGV[0]
-limit = (ARGV[1] || 1).to_i
+options = {}
 
-killer = SiteKiller.new(prefix, limit, true)
+optparse = OptionParser.new do |opts|
+  opts.on('-h', '--help', 'Show help message') do
+    puts opts
+    exit 0
+  end
+  
+  opts.on('-l', '--limit NUM', 'Setting the limit of a single test') do |limit|
+    if limit.to_i > 0
+      options[:limit] = limit.to_i
+    else
+      puts "Limit must greater than 1"
+      exit 1
+    end
+  end
+    
+  opts.on('-c', '--concurrency NUM', 'Setting the number of concurrent connection') do |num|
+    if num.to_i > 0
+      options[:concurrency] = num.to_i
+    else
+      puts "The number of concurrency must greater than 1"
+      exit 2
+    end
+  end
+    
+  opts.on('-p', '--pid PID', 'The PID is the monitored server process') do |pid|
+    if pid.to_i > 0 and `ps -o pid #{pid}`.split.size > 1
+      options[:pid] = pid.to_i
+    else
+      puts "PID not exists or invalid"
+      exit 3
+    end
+  end
+end
+
+optparse.parse!
+
+options[:prefix] = ARGV.shift
+
+unless options[:prefix]
+  puts "Please specify the URL"
+  exit 4
+end
 
-EM.run { limit.times { killer.run } }
+monitors = (options[:pid]) ? [MemoryMonitor.new(options[:pid])] : []
+SiteKiller.new(options[:prefix], options[:limit], options[:concurrency], true, monitors).run
+monitors.each(&:report) unless monitors.empty?

data/killsite.gemspec

@@ -5,30 +5,31 @@
 
 Gem::Specification.new do |s|
   s.name = %q{killsite}
-  s.version = "0.2.0"
+  s.version = "0.3.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Andrew Liu"]
-  s.date = %q{2011-07-16}
+  s.date = %q{2011-07-18}
   s.default_executable = %q{killsite}
   s.description = %q{Recursively GET the site by links, try to kill the site}
   s.email = %q{andrewliu33@gmail.com}
   s.executables = ["killsite"]
   s.extra_rdoc_files = [
     "LICENSE.txt",
-    "README.rdoc"
+    "README.md"
   ]
   s.files = [
     ".document",
     "Gemfile",
     "Gemfile.lock",
     "LICENSE.txt",
-    "README.rdoc",
+    "README.md",
     "Rakefile",
     "VERSION",
     "bin/killsite",
     "killsite.gemspec",
     "lib/killsite.rb",
+    "lib/memory_monitor.rb",
     "test/helper.rb"
   ]
   s.homepage = %q{http://github.com/eggegg/killsite}
@@ -41,14 +42,12 @@ Gem::Specification.new do |s|
     s.specification_version = 3
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<em-http-request>, [">= 0"])
       s.add_runtime_dependency(%q<nokogiri>, [">= 0"])
       s.add_development_dependency(%q<shoulda>, [">= 0"])
       s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
       s.add_development_dependency(%q<jeweler>, ["~> 1.6.4"])
       s.add_development_dependency(%q<rcov>, [">= 0"])
     else
-      s.add_dependency(%q<em-http-request>, [">= 0"])
       s.add_dependency(%q<nokogiri>, [">= 0"])
       s.add_dependency(%q<shoulda>, [">= 0"])
       s.add_dependency(%q<bundler>, ["~> 1.0.0"])
@@ -56,7 +55,6 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<rcov>, [">= 0"])
     end
   else
-    s.add_dependency(%q<em-http-request>, [">= 0"])
     s.add_dependency(%q<nokogiri>, [">= 0"])
     s.add_dependency(%q<shoulda>, [">= 0"])
     s.add_dependency(%q<bundler>, ["~> 1.0.0"])

data/lib/killsite.rb

@@ -1,49 +1,48 @@
-require 'bundler'
-require 'uri'
-Bundler.require
+require 'open-uri'
+require 'nokogiri'
 
 class SiteKiller
-  def initialize(prefix, limit = 1, verbose = false)
-    @count = 1
+  def initialize(prefix, limit = nil, concurrency = nil, verbose = nil, observers = nil)
     @prefix = URI.parse(prefix)
-    @visited = { @prefix => 0 }
-    @verbose = verbose
-    @limit = limit
+    @visited = [@prefix]
+    @queue = [@prefix]
+    @verbose = verbose || false
+    @limit = limit || 1
+    @concurrency = concurrency || 10
+    @concurrency = @limit if @concurrency > @limit
+    @observers = observers || []
   end
 
-  def run url = nil
-    url ||= @prefix
-
-    http = EventMachine::HttpRequest.new(url).get
-    http.callback do
-      if @visited[url] == 0
-        puts "Processing '#{url}'" if @verbose
-        Nokogiri::HTML.parse(http.response).xpath("//a[@href]").each do |link|
+  def run
+    while @queue.size > 0
+      url = @queue.shift
+    
+      @observers.each { |o| o.before_request(url) if o.respond_to? :before_request }
+      `ab -n #{@limit} -c #{@concurrency} #{url}`
+      @observers.each { |o| o.after_request(url) if o.respond_to? :after_request }
+      
+      puts "Processing '#{url}'" if @verbose
+      begin
+        response = open(url).read
+        Nokogiri::HTML.parse(response).xpath("//a[@href]").each do |link|
           next_url = process_url link['href']
           if next_url and !@visited.include?(next_url)
             puts "  Queueing '#{next_url}'" if @verbose
 
-            @visited[next_url] = 0
-            @count += @limit
-            @limit.times { run next_url }
+            @visited << next_url
+            @queue << next_url
           end
         end
-        print "  Progress " if @verbose
-      else
-        print '*' if @verbose
+      rescue => e
+        puts "  No valid response: #{e}"
       end
-
-      @visited[url] += 1
-      puts if @verbose and @visited[url] == @limit
-      @count -= 1
-      EM.stop if @count == 0
     end
   end
 
   private
   def process_url url
     return nil if url =~ /^http/ and URI.parse(@prefix.to_s).host != URI.parse(url).host
-    return nil if url =~ /^javascript/ or url =~ /^#/
+    return nil if url =~ /^javascript|^#|^mailto/
     URI.join @prefix.to_s, url
   end
 end

data/lib/memory_monitor.rb

@@ -0,0 +1,32 @@
+
+class MemoryMonitor
+  def initialize pid
+    @pid = pid
+    @data = {}
+  end
+  
+  def before_request url
+    @data[url] = memory
+  end
+  
+  def after_request url
+    @data[url] = memory - @data[url]
+  end
+  
+  def report
+    sorted = @data.sort_by { |url, memory| -memory }
+    
+    
+    puts "\nMost memory used actions:"
+    sorted.each_with_index do |(url, memory), index|
+      puts "##{index + 1}\t#{memory/1024} KB\t=> #{url.to_s}"
+    end
+  end
+  
+  private
+  def memory
+    mem = `ps -o rss #{@pid}`[/\d+/].to_i
+    raise 'invalid PID' unless mem > 0
+    mem
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: killsite
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,23 +9,12 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-07-16 00:00:00.000000000 +08:00
+date: 2011-07-18 00:00:00.000000000 +08:00
 default_executable: killsite
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: em-http-request
-  requirement: &2157593620 !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: *2157593620
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &2157593140 !ruby/object:Gem::Requirement
+  requirement: &2158843600 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -33,10 +22,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2157593140
+  version_requirements: *2158843600
 - !ruby/object:Gem::Dependency
   name: shoulda
-  requirement: &2157592660 !ruby/object:Gem::Requirement
+  requirement: &2158843120 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -44,10 +33,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2157592660
+  version_requirements: *2158843120
 - !ruby/object:Gem::Dependency
   name: bundler
-  requirement: &2157592180 !ruby/object:Gem::Requirement
+  requirement: &2158842640 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -55,10 +44,10 @@ dependencies:
         version: 1.0.0
   type: :development
   prerelease: false
-  version_requirements: *2157592180
+  version_requirements: *2158842640
 - !ruby/object:Gem::Dependency
   name: jeweler
-  requirement: &2157591700 !ruby/object:Gem::Requirement
+  requirement: &2158842160 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -66,10 +55,10 @@ dependencies:
         version: 1.6.4
   type: :development
   prerelease: false
-  version_requirements: *2157591700
+  version_requirements: *2158842160
 - !ruby/object:Gem::Dependency
   name: rcov
-  requirement: &2157591220 !ruby/object:Gem::Requirement
+  requirement: &2158841680 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -77,7 +66,7 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2157591220
+  version_requirements: *2158841680
 description: Recursively GET the site by links, try to kill the site
 email: andrewliu33@gmail.com
 executables:
@@ -85,18 +74,19 @@ executables:
 extensions: []
 extra_rdoc_files:
 - LICENSE.txt
-- README.rdoc
+- README.md
 files:
 - .document
 - Gemfile
 - Gemfile.lock
 - LICENSE.txt
-- README.rdoc
+- README.md
 - Rakefile
 - VERSION
 - bin/killsite
 - killsite.gemspec
 - lib/killsite.rb
+- lib/memory_monitor.rb
 - test/helper.rb
 has_rdoc: true
 homepage: http://github.com/eggegg/killsite
@@ -114,7 +104,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -2909793090681244207
+      hash: -1937579483118733362
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:

data/README.rdoc

@@ -1,20 +0,0 @@
-= killsite
-
-Recursively GET all the pages of a given site through hyperlinks.
-The tool is for the usage of testing and profiling.
-
-== Installation
-
-Simply `gem install killsite`.
-
-== Usage
-
-  killsite [website] [# of GET for each pages]
-
-The second argument can be omitted.
-
-== Copyright
-
-Copyright (c) 2011 Andrew Liu. See LICENSE.txt for
-further details.
-