killsite 0.2.0 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- data/Gemfile +0 -1
- data/Gemfile.lock +0 -8
- data/README.md +25 -0
- data/VERSION +1 -1
- data/bin/killsite +51 -7
- data/killsite.gemspec +5 -7
- data/lib/killsite.rb +27 -28
- data/lib/memory_monitor.rb +32 -0
- metadata +16 -26
- data/README.rdoc +0 -20
data/Gemfile
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,13 +1,6 @@
|
|
1
1
|
GEM
|
2
2
|
remote: http://rubygems.org/
|
3
3
|
specs:
|
4
|
-
addressable (2.2.6)
|
5
|
-
em-http-request (0.3.0)
|
6
|
-
addressable (>= 2.0.0)
|
7
|
-
escape_utils
|
8
|
-
eventmachine (>= 0.12.9)
|
9
|
-
escape_utils (0.2.3)
|
10
|
-
eventmachine (0.12.10)
|
11
4
|
git (1.2.5)
|
12
5
|
jeweler (1.6.4)
|
13
6
|
bundler (~> 1.0)
|
@@ -23,7 +16,6 @@ PLATFORMS
|
|
23
16
|
|
24
17
|
DEPENDENCIES
|
25
18
|
bundler (~> 1.0.0)
|
26
|
-
em-http-request
|
27
19
|
jeweler (~> 1.6.4)
|
28
20
|
nokogiri
|
29
21
|
rcov
|
data/README.md
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
= killsite
|
2
|
+
|
3
|
+
Recursively GET all the pages of a given site through hyperlinks.
|
4
|
+
The tool is for the usage of testing and profiling.
|
5
|
+
|
6
|
+
Using memory monitor to find out the memory leaks within different URLs.
|
7
|
+
|
8
|
+
== Installation
|
9
|
+
|
10
|
+
Simply `gem install killsite`.
|
11
|
+
`ab` (ApacheBench) is require as an benchmark tool.
|
12
|
+
|
13
|
+
== Usage
|
14
|
+
|
15
|
+
Usage: killsite [options]
|
16
|
+
-h, --help Show help message
|
17
|
+
-l, --limit NUM Setting the limit of a single test
|
18
|
+
-c, --concurrency NUM Setting the number of concurrent connection
|
19
|
+
-p, --pid PID The PID is the monitored server process
|
20
|
+
|
21
|
+
== Copyright
|
22
|
+
|
23
|
+
Copyright (c) 2011 Andrew Liu. See LICENSE.txt for
|
24
|
+
further details.
|
25
|
+
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.
|
1
|
+
0.3.0
|
data/bin/killsite
CHANGED
@@ -1,15 +1,59 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
|
3
|
+
require 'optparse'
|
3
4
|
require 'killsite'
|
5
|
+
require 'memory_monitor'
|
4
6
|
|
5
|
-
if
|
6
|
-
puts "
|
7
|
-
exit
|
7
|
+
if `which ab`.empty?
|
8
|
+
puts "ab (ApacheBench) must be installed to proceed"
|
9
|
+
exit -1
|
8
10
|
end
|
9
11
|
|
10
|
-
|
11
|
-
limit = (ARGV[1] || 1).to_i
|
12
|
+
options = {}
|
12
13
|
|
13
|
-
|
14
|
+
optparse = OptionParser.new do |opts|
|
15
|
+
opts.on('-h', '--help', 'Show help message') do
|
16
|
+
puts opts
|
17
|
+
exit 0
|
18
|
+
end
|
19
|
+
|
20
|
+
opts.on('-l', '--limit NUM', 'Setting the limit of a single test') do |limit|
|
21
|
+
if limit.to_i > 0
|
22
|
+
options[:limit] = limit.to_i
|
23
|
+
else
|
24
|
+
puts "Limit must greater than 1"
|
25
|
+
exit 1
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
opts.on('-c', '--concurrency NUM', 'Setting the number of concurrent connection') do |num|
|
30
|
+
if num.to_i > 0
|
31
|
+
options[:concurrency] = num.to_i
|
32
|
+
else
|
33
|
+
puts "The number of concurrency must greater than 1"
|
34
|
+
exit 2
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
opts.on('-p', '--pid PID', 'The PID is the monitored server process') do |pid|
|
39
|
+
if pid.to_i > 0 and `ps -o pid #{pid}`.split.size > 1
|
40
|
+
options[:pid] = pid.to_i
|
41
|
+
else
|
42
|
+
puts "PID not exists or invalid"
|
43
|
+
exit 3
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
optparse.parse!
|
49
|
+
|
50
|
+
options[:prefix] = ARGV.shift
|
51
|
+
|
52
|
+
unless options[:prefix]
|
53
|
+
puts "Please specify the URL"
|
54
|
+
exit 4
|
55
|
+
end
|
14
56
|
|
15
|
-
|
57
|
+
monitors = (options[:pid]) ? [MemoryMonitor.new(options[:pid])] : []
|
58
|
+
SiteKiller.new(options[:prefix], options[:limit], options[:concurrency], true, monitors).run
|
59
|
+
monitors.each(&:report) unless monitors.empty?
|
data/killsite.gemspec
CHANGED
@@ -5,30 +5,31 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = %q{killsite}
|
8
|
-
s.version = "0.
|
8
|
+
s.version = "0.3.0"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["Andrew Liu"]
|
12
|
-
s.date = %q{2011-07-
|
12
|
+
s.date = %q{2011-07-18}
|
13
13
|
s.default_executable = %q{killsite}
|
14
14
|
s.description = %q{Recursively GET the site by links, try to kill the site}
|
15
15
|
s.email = %q{andrewliu33@gmail.com}
|
16
16
|
s.executables = ["killsite"]
|
17
17
|
s.extra_rdoc_files = [
|
18
18
|
"LICENSE.txt",
|
19
|
-
"README.
|
19
|
+
"README.md"
|
20
20
|
]
|
21
21
|
s.files = [
|
22
22
|
".document",
|
23
23
|
"Gemfile",
|
24
24
|
"Gemfile.lock",
|
25
25
|
"LICENSE.txt",
|
26
|
-
"README.
|
26
|
+
"README.md",
|
27
27
|
"Rakefile",
|
28
28
|
"VERSION",
|
29
29
|
"bin/killsite",
|
30
30
|
"killsite.gemspec",
|
31
31
|
"lib/killsite.rb",
|
32
|
+
"lib/memory_monitor.rb",
|
32
33
|
"test/helper.rb"
|
33
34
|
]
|
34
35
|
s.homepage = %q{http://github.com/eggegg/killsite}
|
@@ -41,14 +42,12 @@ Gem::Specification.new do |s|
|
|
41
42
|
s.specification_version = 3
|
42
43
|
|
43
44
|
if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
|
44
|
-
s.add_runtime_dependency(%q<em-http-request>, [">= 0"])
|
45
45
|
s.add_runtime_dependency(%q<nokogiri>, [">= 0"])
|
46
46
|
s.add_development_dependency(%q<shoulda>, [">= 0"])
|
47
47
|
s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
|
48
48
|
s.add_development_dependency(%q<jeweler>, ["~> 1.6.4"])
|
49
49
|
s.add_development_dependency(%q<rcov>, [">= 0"])
|
50
50
|
else
|
51
|
-
s.add_dependency(%q<em-http-request>, [">= 0"])
|
52
51
|
s.add_dependency(%q<nokogiri>, [">= 0"])
|
53
52
|
s.add_dependency(%q<shoulda>, [">= 0"])
|
54
53
|
s.add_dependency(%q<bundler>, ["~> 1.0.0"])
|
@@ -56,7 +55,6 @@ Gem::Specification.new do |s|
|
|
56
55
|
s.add_dependency(%q<rcov>, [">= 0"])
|
57
56
|
end
|
58
57
|
else
|
59
|
-
s.add_dependency(%q<em-http-request>, [">= 0"])
|
60
58
|
s.add_dependency(%q<nokogiri>, [">= 0"])
|
61
59
|
s.add_dependency(%q<shoulda>, [">= 0"])
|
62
60
|
s.add_dependency(%q<bundler>, ["~> 1.0.0"])
|
data/lib/killsite.rb
CHANGED
@@ -1,49 +1,48 @@
|
|
1
|
-
require '
|
2
|
-
require '
|
3
|
-
Bundler.require
|
1
|
+
require 'open-uri'
|
2
|
+
require 'nokogiri'
|
4
3
|
|
5
4
|
class SiteKiller
|
6
|
-
def initialize(prefix, limit =
|
7
|
-
@count = 1
|
5
|
+
def initialize(prefix, limit = nil, concurrency = nil, verbose = nil, observers = nil)
|
8
6
|
@prefix = URI.parse(prefix)
|
9
|
-
@visited =
|
10
|
-
@
|
11
|
-
@
|
7
|
+
@visited = [@prefix]
|
8
|
+
@queue = [@prefix]
|
9
|
+
@verbose = verbose || false
|
10
|
+
@limit = limit || 1
|
11
|
+
@concurrency = concurrency || 10
|
12
|
+
@concurrency = @limit if @concurrency > @limit
|
13
|
+
@observers = observers || []
|
12
14
|
end
|
13
15
|
|
14
|
-
def run
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
16
|
+
def run
|
17
|
+
while @queue.size > 0
|
18
|
+
url = @queue.shift
|
19
|
+
|
20
|
+
@observers.each { |o| o.before_request(url) if o.respond_to? :before_request }
|
21
|
+
`ab -n #{@limit} -c #{@concurrency} #{url}`
|
22
|
+
@observers.each { |o| o.after_request(url) if o.respond_to? :after_request }
|
23
|
+
|
24
|
+
puts "Processing '#{url}'" if @verbose
|
25
|
+
begin
|
26
|
+
response = open(url).read
|
27
|
+
Nokogiri::HTML.parse(response).xpath("//a[@href]").each do |link|
|
22
28
|
next_url = process_url link['href']
|
23
29
|
if next_url and !@visited.include?(next_url)
|
24
30
|
puts " Queueing '#{next_url}'" if @verbose
|
25
31
|
|
26
|
-
@visited
|
27
|
-
@
|
28
|
-
@limit.times { run next_url }
|
32
|
+
@visited << next_url
|
33
|
+
@queue << next_url
|
29
34
|
end
|
30
35
|
end
|
31
|
-
|
32
|
-
|
33
|
-
print '*' if @verbose
|
36
|
+
rescue => e
|
37
|
+
puts " No valid response: #{e}"
|
34
38
|
end
|
35
|
-
|
36
|
-
@visited[url] += 1
|
37
|
-
puts if @verbose and @visited[url] == @limit
|
38
|
-
@count -= 1
|
39
|
-
EM.stop if @count == 0
|
40
39
|
end
|
41
40
|
end
|
42
41
|
|
43
42
|
private
|
44
43
|
def process_url url
|
45
44
|
return nil if url =~ /^http/ and URI.parse(@prefix.to_s).host != URI.parse(url).host
|
46
|
-
return nil if url =~ /^javascript/
|
45
|
+
return nil if url =~ /^javascript|^#|^mailto/
|
47
46
|
URI.join @prefix.to_s, url
|
48
47
|
end
|
49
48
|
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
|
2
|
+
class MemoryMonitor
|
3
|
+
def initialize pid
|
4
|
+
@pid = pid
|
5
|
+
@data = {}
|
6
|
+
end
|
7
|
+
|
8
|
+
def before_request url
|
9
|
+
@data[url] = memory
|
10
|
+
end
|
11
|
+
|
12
|
+
def after_request url
|
13
|
+
@data[url] = memory - @data[url]
|
14
|
+
end
|
15
|
+
|
16
|
+
def report
|
17
|
+
sorted = @data.sort_by { |url, memory| -memory }
|
18
|
+
|
19
|
+
|
20
|
+
puts "\nMost memory used actions:"
|
21
|
+
sorted.each_with_index do |(url, memory), index|
|
22
|
+
puts "##{index + 1}\t#{memory/1024} KB\t=> #{url.to_s}"
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
private
|
27
|
+
def memory
|
28
|
+
mem = `ps -o rss #{@pid}`[/\d+/].to_i
|
29
|
+
raise 'invalid PID' unless mem > 0
|
30
|
+
mem
|
31
|
+
end
|
32
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: killsite
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,23 +9,12 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2011-07-
|
12
|
+
date: 2011-07-18 00:00:00.000000000 +08:00
|
13
13
|
default_executable: killsite
|
14
14
|
dependencies:
|
15
|
-
- !ruby/object:Gem::Dependency
|
16
|
-
name: em-http-request
|
17
|
-
requirement: &2157593620 !ruby/object:Gem::Requirement
|
18
|
-
none: false
|
19
|
-
requirements:
|
20
|
-
- - ! '>='
|
21
|
-
- !ruby/object:Gem::Version
|
22
|
-
version: '0'
|
23
|
-
type: :runtime
|
24
|
-
prerelease: false
|
25
|
-
version_requirements: *2157593620
|
26
15
|
- !ruby/object:Gem::Dependency
|
27
16
|
name: nokogiri
|
28
|
-
requirement: &
|
17
|
+
requirement: &2158843600 !ruby/object:Gem::Requirement
|
29
18
|
none: false
|
30
19
|
requirements:
|
31
20
|
- - ! '>='
|
@@ -33,10 +22,10 @@ dependencies:
|
|
33
22
|
version: '0'
|
34
23
|
type: :runtime
|
35
24
|
prerelease: false
|
36
|
-
version_requirements: *
|
25
|
+
version_requirements: *2158843600
|
37
26
|
- !ruby/object:Gem::Dependency
|
38
27
|
name: shoulda
|
39
|
-
requirement: &
|
28
|
+
requirement: &2158843120 !ruby/object:Gem::Requirement
|
40
29
|
none: false
|
41
30
|
requirements:
|
42
31
|
- - ! '>='
|
@@ -44,10 +33,10 @@ dependencies:
|
|
44
33
|
version: '0'
|
45
34
|
type: :development
|
46
35
|
prerelease: false
|
47
|
-
version_requirements: *
|
36
|
+
version_requirements: *2158843120
|
48
37
|
- !ruby/object:Gem::Dependency
|
49
38
|
name: bundler
|
50
|
-
requirement: &
|
39
|
+
requirement: &2158842640 !ruby/object:Gem::Requirement
|
51
40
|
none: false
|
52
41
|
requirements:
|
53
42
|
- - ~>
|
@@ -55,10 +44,10 @@ dependencies:
|
|
55
44
|
version: 1.0.0
|
56
45
|
type: :development
|
57
46
|
prerelease: false
|
58
|
-
version_requirements: *
|
47
|
+
version_requirements: *2158842640
|
59
48
|
- !ruby/object:Gem::Dependency
|
60
49
|
name: jeweler
|
61
|
-
requirement: &
|
50
|
+
requirement: &2158842160 !ruby/object:Gem::Requirement
|
62
51
|
none: false
|
63
52
|
requirements:
|
64
53
|
- - ~>
|
@@ -66,10 +55,10 @@ dependencies:
|
|
66
55
|
version: 1.6.4
|
67
56
|
type: :development
|
68
57
|
prerelease: false
|
69
|
-
version_requirements: *
|
58
|
+
version_requirements: *2158842160
|
70
59
|
- !ruby/object:Gem::Dependency
|
71
60
|
name: rcov
|
72
|
-
requirement: &
|
61
|
+
requirement: &2158841680 !ruby/object:Gem::Requirement
|
73
62
|
none: false
|
74
63
|
requirements:
|
75
64
|
- - ! '>='
|
@@ -77,7 +66,7 @@ dependencies:
|
|
77
66
|
version: '0'
|
78
67
|
type: :development
|
79
68
|
prerelease: false
|
80
|
-
version_requirements: *
|
69
|
+
version_requirements: *2158841680
|
81
70
|
description: Recursively GET the site by links, try to kill the site
|
82
71
|
email: andrewliu33@gmail.com
|
83
72
|
executables:
|
@@ -85,18 +74,19 @@ executables:
|
|
85
74
|
extensions: []
|
86
75
|
extra_rdoc_files:
|
87
76
|
- LICENSE.txt
|
88
|
-
- README.
|
77
|
+
- README.md
|
89
78
|
files:
|
90
79
|
- .document
|
91
80
|
- Gemfile
|
92
81
|
- Gemfile.lock
|
93
82
|
- LICENSE.txt
|
94
|
-
- README.
|
83
|
+
- README.md
|
95
84
|
- Rakefile
|
96
85
|
- VERSION
|
97
86
|
- bin/killsite
|
98
87
|
- killsite.gemspec
|
99
88
|
- lib/killsite.rb
|
89
|
+
- lib/memory_monitor.rb
|
100
90
|
- test/helper.rb
|
101
91
|
has_rdoc: true
|
102
92
|
homepage: http://github.com/eggegg/killsite
|
@@ -114,7 +104,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
114
104
|
version: '0'
|
115
105
|
segments:
|
116
106
|
- 0
|
117
|
-
hash: -
|
107
|
+
hash: -1937579483118733362
|
118
108
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
119
109
|
none: false
|
120
110
|
requirements:
|
data/README.rdoc
DELETED
@@ -1,20 +0,0 @@
|
|
1
|
-
= killsite
|
2
|
-
|
3
|
-
Recursively GET all the pages of a given site through hyperlinks.
|
4
|
-
The tool is for the usage of testing and profiling.
|
5
|
-
|
6
|
-
== Installation
|
7
|
-
|
8
|
-
Simply `gem install killsite`.
|
9
|
-
|
10
|
-
== Usage
|
11
|
-
|
12
|
-
killsite [website] [# of GET for each pages]
|
13
|
-
|
14
|
-
The second argument can be omitted.
|
15
|
-
|
16
|
-
== Copyright
|
17
|
-
|
18
|
-
Copyright (c) 2011 Andrew Liu. See LICENSE.txt for
|
19
|
-
further details.
|
20
|
-
|