cobweb 0.0.43 → 0.0.44
Sign up to get free protection for your applications and to get access to all the features.
- data/README.textile +1 -1
- data/lib/cobweb.rb +1 -1
- data/lib/cobweb_crawler.rb +4 -3
- data/lib/stats.rb +42 -42
- metadata +21 -21
data/README.textile
CHANGED
data/lib/cobweb.rb
CHANGED
data/lib/cobweb_crawler.rb
CHANGED
|
@@ -76,12 +76,13 @@ class CobwebCrawler
|
|
|
76
76
|
internal_links.each do |link|
|
|
77
77
|
puts "Added #{link.to_s} to queue" if @debug
|
|
78
78
|
@redis.sadd "queued", link
|
|
79
|
+
queue_counter += 1
|
|
79
80
|
end
|
|
80
81
|
|
|
81
82
|
crawl_counter = crawl_counter + 1 #@redis.scard("crawled").to_i
|
|
82
83
|
queue_counter = queue_counter - 1 #@redis.scard("queued").to_i
|
|
83
84
|
|
|
84
|
-
@stats.update_statistics(content)
|
|
85
|
+
@stats.update_statistics(content, crawl_counter, queue_counter)
|
|
85
86
|
@stats.update_status("Completed #{url}.")
|
|
86
87
|
puts "Crawled: #{crawl_counter.to_i} Limit: #{@options[:crawl_limit].to_i} Queued: #{queue_counter.to_i}" if @debug
|
|
87
88
|
|
|
@@ -125,7 +126,7 @@ class CobwebCrawler
|
|
|
125
126
|
|
|
126
127
|
def all_links_from_content(content)
|
|
127
128
|
links = content[:links].keys.map{|key| content[:links][key]}.flatten
|
|
128
|
-
links.reject!{|link| link.
|
|
129
|
+
links.reject!{|link| link.cobweb_starts_with?("javascript:")}
|
|
129
130
|
links = links.map{|link| UriHelper.join_no_fragment(content[:url], link) }
|
|
130
131
|
links.select!{|link| link.scheme == "http" || link.scheme == "https"}
|
|
131
132
|
links.uniq
|
|
@@ -134,7 +135,7 @@ class CobwebCrawler
|
|
|
134
135
|
end
|
|
135
136
|
|
|
136
137
|
class String
|
|
137
|
-
def
|
|
138
|
+
def cobweb_starts_with?(val)
|
|
138
139
|
if self.length >= val.length
|
|
139
140
|
self[0..val.length-1] == val
|
|
140
141
|
else
|
data/lib/stats.rb
CHANGED
|
@@ -21,50 +21,48 @@ class Stats
|
|
|
21
21
|
@redis.del "crawl_details"
|
|
22
22
|
end
|
|
23
23
|
|
|
24
|
-
def update_statistics(content)
|
|
24
|
+
def update_statistics(content, crawl_counter=@redis.scard("crawled").to_i, queue_counter=@redis.scard("queued").to_i)
|
|
25
25
|
|
|
26
|
-
|
|
27
|
-
queue_counter = @redis.scard("queued").to_i
|
|
26
|
+
@statistics = get_statistics
|
|
28
27
|
|
|
29
|
-
if @
|
|
30
|
-
@
|
|
28
|
+
if @statistics.has_key? :average_response_time
|
|
29
|
+
@statistics[:average_response_time] = (((@redis.hget("statistics", "average_response_time").to_f*crawl_counter) + content[:response_time].to_f) / (crawl_counter + 1))
|
|
31
30
|
else
|
|
32
|
-
@
|
|
31
|
+
@statistics[:average_response_time] = content[:response_time].to_f
|
|
33
32
|
end
|
|
34
|
-
@
|
|
35
|
-
@
|
|
36
|
-
if @
|
|
37
|
-
@
|
|
33
|
+
@statistics[:maximum_response_time] = content[:response_time].to_f if @statistics[:maximum_response_time].nil? or content[:response_time].to_f > @statistics[:maximum_response_time].to_f
|
|
34
|
+
@statistics[:minimum_response_time] = content[:response_time].to_f if @statistics[:minimum_response_time].nil? or content[:response_time].to_f < @statistics[:minimum_response_time].to_f
|
|
35
|
+
if @statistics.has_key? :average_length
|
|
36
|
+
@statistics[:average_length] = (((@redis.hget("statistics", "average_length").to_i*crawl_counter) + content[:length].to_i) / (crawl_counter + 1))
|
|
38
37
|
else
|
|
39
|
-
@
|
|
38
|
+
@statistics[:average_length] = content[:length].to_i
|
|
40
39
|
end
|
|
41
|
-
@
|
|
42
|
-
@
|
|
43
|
-
|
|
44
|
-
|
|
40
|
+
@statistics[:maximum_length] = content[:length].to_i if @redis.hget("statistics", "maximum_length").nil? or content[:length].to_i > @statistics[:maximum_length].to_i
|
|
41
|
+
@statistics[:minimum_length] = content[:length].to_i if @redis.hget("statistics", "minimum_length").nil? or content[:length].to_i < @statistics[:minimum_length].to_i
|
|
42
|
+
|
|
45
43
|
if content[:mime_type].include?("text/html") or content[:mime_type].include?("application/xhtml+xml")
|
|
46
|
-
@
|
|
47
|
-
@
|
|
44
|
+
@statistics[:page_count] = @statistics[:page_count].to_i + 1
|
|
45
|
+
@statistics[:page_size] = @statistics[:page_size].to_i + content[:length].to_i
|
|
48
46
|
increment_time_stat("pages_count")
|
|
49
47
|
else
|
|
50
|
-
@
|
|
51
|
-
@
|
|
48
|
+
@statistics[:asset_count] = @statistics[:asset_count].to_i + 1
|
|
49
|
+
@statistics[:asset_size] = @statistics[:asset_size].to_i + content[:length].to_i
|
|
52
50
|
increment_time_stat("assets_count")
|
|
53
51
|
end
|
|
54
52
|
|
|
55
|
-
total_redirects = @
|
|
56
|
-
@
|
|
57
|
-
@
|
|
53
|
+
total_redirects = @statistics[:total_redirects].to_i
|
|
54
|
+
@statistics[:total_redirects] = 0 if total_redirects.nil?
|
|
55
|
+
@statistics[:total_redirects] = total_redirects += content[:redirect_through].count unless content[:redirect_through].nil?
|
|
58
56
|
|
|
59
|
-
@
|
|
60
|
-
@
|
|
57
|
+
@statistics[:crawl_counter] = crawl_counter
|
|
58
|
+
@statistics[:queue_counter] = queue_counter
|
|
61
59
|
|
|
62
|
-
total_length = @
|
|
63
|
-
@
|
|
60
|
+
total_length = @statistics[:total_length].to_i
|
|
61
|
+
@statistics[:total_length] = total_length + content[:length].to_i
|
|
64
62
|
|
|
65
63
|
mime_counts = {}
|
|
66
|
-
if @
|
|
67
|
-
mime_counts =
|
|
64
|
+
if @statistics.has_key? :mime_counts
|
|
65
|
+
mime_counts = @statistics[:mime_counts]
|
|
68
66
|
if mime_counts.has_key? content[:mime_type]
|
|
69
67
|
mime_counts[content[:mime_type]] += 1
|
|
70
68
|
else
|
|
@@ -73,30 +71,30 @@ class Stats
|
|
|
73
71
|
else
|
|
74
72
|
mime_counts = {content[:mime_type] => 1}
|
|
75
73
|
end
|
|
76
|
-
@
|
|
74
|
+
@statistics[:mime_counts] = mime_counts.to_json
|
|
77
75
|
|
|
78
76
|
# record mime categories stats
|
|
79
|
-
if content[:mime_type].
|
|
77
|
+
if content[:mime_type].cobweb_starts_with? "text"
|
|
80
78
|
increment_time_stat("mime_text_count")
|
|
81
|
-
elsif content[:mime_type].
|
|
79
|
+
elsif content[:mime_type].cobweb_starts_with? "application"
|
|
82
80
|
increment_time_stat("mime_application_count")
|
|
83
|
-
elsif content[:mime_type].
|
|
81
|
+
elsif content[:mime_type].cobweb_starts_with? "audio"
|
|
84
82
|
increment_time_stat("mime_audio_count")
|
|
85
|
-
elsif content[:mime_type].
|
|
83
|
+
elsif content[:mime_type].cobweb_starts_with? "image"
|
|
86
84
|
increment_time_stat("mime_image_count")
|
|
87
|
-
elsif content[:mime_type].
|
|
85
|
+
elsif content[:mime_type].cobweb_starts_with? "message"
|
|
88
86
|
increment_time_stat("mime_message_count")
|
|
89
|
-
elsif content[:mime_type].
|
|
87
|
+
elsif content[:mime_type].cobweb_starts_with? "model"
|
|
90
88
|
increment_time_stat("mime_model_count")
|
|
91
|
-
elsif content[:mime_type].
|
|
89
|
+
elsif content[:mime_type].cobweb_starts_with? "multipart"
|
|
92
90
|
increment_time_stat("mime_multipart_count")
|
|
93
|
-
elsif content[:mime_type].
|
|
91
|
+
elsif content[:mime_type].cobweb_starts_with? "video"
|
|
94
92
|
increment_time_stat("mime_video_count")
|
|
95
93
|
end
|
|
96
94
|
|
|
97
95
|
status_counts = {}
|
|
98
|
-
if @
|
|
99
|
-
status_counts =
|
|
96
|
+
if @statistics.has_key? :status_counts
|
|
97
|
+
status_counts = @statistics[:status_counts]
|
|
100
98
|
status_code = content[:status_code].to_i.to_s.to_sym
|
|
101
99
|
if status_counts.has_key? status_code
|
|
102
100
|
status_counts[status_code] += 1
|
|
@@ -116,13 +114,15 @@ class Stats
|
|
|
116
114
|
increment_time_stat("status|_500_count")
|
|
117
115
|
end
|
|
118
116
|
|
|
119
|
-
@
|
|
120
|
-
|
|
117
|
+
@statistics[:status_counts] = status_counts.to_json
|
|
121
118
|
|
|
122
119
|
## time based statistics
|
|
123
120
|
increment_time_stat("minute_totals", "minute", 60)
|
|
124
121
|
|
|
125
|
-
|
|
122
|
+
redis_command = "@redis.hmset 'statistics', #{@statistics.keys.map{|key| "'#{key}', '#{@statistics[key]}'"}.join(", ")}"
|
|
123
|
+
instance_eval redis_command
|
|
124
|
+
|
|
125
|
+
@statistics
|
|
126
126
|
end
|
|
127
127
|
|
|
128
128
|
def record_time_stat(stat_name, value, type="minute", duration=60)
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: cobweb
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.0.
|
|
4
|
+
version: 0.0.44
|
|
5
5
|
prerelease:
|
|
6
6
|
platform: ruby
|
|
7
7
|
authors:
|
|
@@ -13,7 +13,7 @@ date: 2012-05-03 00:00:00.000000000 Z
|
|
|
13
13
|
dependencies:
|
|
14
14
|
- !ruby/object:Gem::Dependency
|
|
15
15
|
name: resque
|
|
16
|
-
requirement: &
|
|
16
|
+
requirement: &70139600146360 !ruby/object:Gem::Requirement
|
|
17
17
|
none: false
|
|
18
18
|
requirements:
|
|
19
19
|
- - ! '>='
|
|
@@ -21,10 +21,10 @@ dependencies:
|
|
|
21
21
|
version: '0'
|
|
22
22
|
type: :runtime
|
|
23
23
|
prerelease: false
|
|
24
|
-
version_requirements: *
|
|
24
|
+
version_requirements: *70139600146360
|
|
25
25
|
- !ruby/object:Gem::Dependency
|
|
26
26
|
name: redis
|
|
27
|
-
requirement: &
|
|
27
|
+
requirement: &70139600145940 !ruby/object:Gem::Requirement
|
|
28
28
|
none: false
|
|
29
29
|
requirements:
|
|
30
30
|
- - ! '>='
|
|
@@ -32,10 +32,10 @@ dependencies:
|
|
|
32
32
|
version: '0'
|
|
33
33
|
type: :runtime
|
|
34
34
|
prerelease: false
|
|
35
|
-
version_requirements: *
|
|
35
|
+
version_requirements: *70139600145940
|
|
36
36
|
- !ruby/object:Gem::Dependency
|
|
37
37
|
name: nokogiri
|
|
38
|
-
requirement: &
|
|
38
|
+
requirement: &70139600145520 !ruby/object:Gem::Requirement
|
|
39
39
|
none: false
|
|
40
40
|
requirements:
|
|
41
41
|
- - ! '>='
|
|
@@ -43,10 +43,10 @@ dependencies:
|
|
|
43
43
|
version: '0'
|
|
44
44
|
type: :runtime
|
|
45
45
|
prerelease: false
|
|
46
|
-
version_requirements: *
|
|
46
|
+
version_requirements: *70139600145520
|
|
47
47
|
- !ruby/object:Gem::Dependency
|
|
48
48
|
name: addressable
|
|
49
|
-
requirement: &
|
|
49
|
+
requirement: &70139600145100 !ruby/object:Gem::Requirement
|
|
50
50
|
none: false
|
|
51
51
|
requirements:
|
|
52
52
|
- - ! '>='
|
|
@@ -54,10 +54,10 @@ dependencies:
|
|
|
54
54
|
version: '0'
|
|
55
55
|
type: :runtime
|
|
56
56
|
prerelease: false
|
|
57
|
-
version_requirements: *
|
|
57
|
+
version_requirements: *70139600145100
|
|
58
58
|
- !ruby/object:Gem::Dependency
|
|
59
59
|
name: rspec
|
|
60
|
-
requirement: &
|
|
60
|
+
requirement: &70139600144680 !ruby/object:Gem::Requirement
|
|
61
61
|
none: false
|
|
62
62
|
requirements:
|
|
63
63
|
- - ! '>='
|
|
@@ -65,10 +65,10 @@ dependencies:
|
|
|
65
65
|
version: '0'
|
|
66
66
|
type: :runtime
|
|
67
67
|
prerelease: false
|
|
68
|
-
version_requirements: *
|
|
68
|
+
version_requirements: *70139600144680
|
|
69
69
|
- !ruby/object:Gem::Dependency
|
|
70
70
|
name: awesome_print
|
|
71
|
-
requirement: &
|
|
71
|
+
requirement: &70139600144260 !ruby/object:Gem::Requirement
|
|
72
72
|
none: false
|
|
73
73
|
requirements:
|
|
74
74
|
- - ! '>='
|
|
@@ -76,10 +76,10 @@ dependencies:
|
|
|
76
76
|
version: '0'
|
|
77
77
|
type: :runtime
|
|
78
78
|
prerelease: false
|
|
79
|
-
version_requirements: *
|
|
79
|
+
version_requirements: *70139600144260
|
|
80
80
|
- !ruby/object:Gem::Dependency
|
|
81
81
|
name: sinatra
|
|
82
|
-
requirement: &
|
|
82
|
+
requirement: &70139600143840 !ruby/object:Gem::Requirement
|
|
83
83
|
none: false
|
|
84
84
|
requirements:
|
|
85
85
|
- - ! '>='
|
|
@@ -87,10 +87,10 @@ dependencies:
|
|
|
87
87
|
version: '0'
|
|
88
88
|
type: :runtime
|
|
89
89
|
prerelease: false
|
|
90
|
-
version_requirements: *
|
|
90
|
+
version_requirements: *70139600143840
|
|
91
91
|
- !ruby/object:Gem::Dependency
|
|
92
92
|
name: thin
|
|
93
|
-
requirement: &
|
|
93
|
+
requirement: &70139600143420 !ruby/object:Gem::Requirement
|
|
94
94
|
none: false
|
|
95
95
|
requirements:
|
|
96
96
|
- - ! '>='
|
|
@@ -98,10 +98,10 @@ dependencies:
|
|
|
98
98
|
version: '0'
|
|
99
99
|
type: :runtime
|
|
100
100
|
prerelease: false
|
|
101
|
-
version_requirements: *
|
|
101
|
+
version_requirements: *70139600143420
|
|
102
102
|
- !ruby/object:Gem::Dependency
|
|
103
103
|
name: haml
|
|
104
|
-
requirement: &
|
|
104
|
+
requirement: &70139600143000 !ruby/object:Gem::Requirement
|
|
105
105
|
none: false
|
|
106
106
|
requirements:
|
|
107
107
|
- - ! '>='
|
|
@@ -109,10 +109,10 @@ dependencies:
|
|
|
109
109
|
version: '0'
|
|
110
110
|
type: :runtime
|
|
111
111
|
prerelease: false
|
|
112
|
-
version_requirements: *
|
|
112
|
+
version_requirements: *70139600143000
|
|
113
113
|
- !ruby/object:Gem::Dependency
|
|
114
114
|
name: namespaced_redis
|
|
115
|
-
requirement: &
|
|
115
|
+
requirement: &70139600142580 !ruby/object:Gem::Requirement
|
|
116
116
|
none: false
|
|
117
117
|
requirements:
|
|
118
118
|
- - ! '>='
|
|
@@ -120,7 +120,7 @@ dependencies:
|
|
|
120
120
|
version: '0'
|
|
121
121
|
type: :runtime
|
|
122
122
|
prerelease: false
|
|
123
|
-
version_requirements: *
|
|
123
|
+
version_requirements: *70139600142580
|
|
124
124
|
description: Web Crawler that uses resque background job engine to allow you to cluster
|
|
125
125
|
your crawl.
|
|
126
126
|
email: stewart@rockwellcottage.com
|