cobweb 1.0.29 → 1.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (9) hide show
  1. checksums.yaml +4 -4
  2. data/README.textile +2 -1
  3. data/lib/cobweb_version.rb +1 -1
  4. data/lib/content_link_parser.rb +5 -4
  5. data/lib/sidekiq/cobweb_helper.rb +2 -1
  6. data/spec/cobweb/cobweb_crawl_helper_spec.rb +1 -1
  7. data/spec/cobweb/crawl_worker_spec.rb +1 -1
  8. data/spec/spec_helper.rb +3 -3
  9. metadata +133 -35
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: ba10134e0b3da7418f0a1a5772ca196cf5525066
4
- data.tar.gz: ae2f27f0036172b001968e277913d0038549220b
3
+ metadata.gz: 854165929dc7a5e3e16d138515723cfd2d3f95b5
4
+ data.tar.gz: 8444c545e80547e41dcaeae817a81183ed4a8d54
5
5
  SHA512:
6
- metadata.gz: 175e8dedf0592c1cc8e9abb50cc0efa835ffb8e1d3c0df3ecb3c3a9900e24a1a2aebc479c5188ea44d57de210b559e0637efb8beac4f90c49f3f6e54bc0492d7
7
- data.tar.gz: d4fc557a20e3b4d54daaecaa151b5365e4a483067b6e2cc4429f846e47066d1326a868383f2d1a2031159dc21cfbd053854e2c12ea57cd1eeb2de999f8aec2ab
6
+ metadata.gz: 181edda4c8fc822f52729a645046c6e3a8d11ad0ff3d5540cfcc055528563605c82eec5a5615074fe86f97c8655694dc75c29d61b8e324fed02291372e5f107a
7
+ data.tar.gz: 830a4aceeb58d6d5b43d284622a9054aea66a5a46aadabec319bf9eebeb3e20444e476a3a0db42afcc2344a6a5f7dae210aaed024055d6d7d25042f3cbd38f79
data/README.textile CHANGED
@@ -1,7 +1,8 @@
1
- h1. Cobweb v1.0.29
1
+ h1. Cobweb v1.1.0
2
2
 
3
3
  "@cobweb_gem":https://twitter.com/cobweb_gem
4
4
  !https://badge.fury.io/rb/cobweb.png!:http://badge.fury.io/rb/cobweb
5
+ !https://circleci.com/gh/stewartmckee/cobweb.svg?style=shield&circle-token=07357f0bd17ac67e21ea161fb9abdb35ecac4c2e!
5
6
  !https://gemnasium.com/stewartmckee/cobweb.png!
6
7
  !https://coveralls.io/repos/stewartmckee/cobweb/badge.png?branch=master(Coverage Status)!:https://coveralls.io/r/stewartmckee/cobweb
7
8
 
data/lib/cobweb_version.rb CHANGED
@@ -3,7 +3,7 @@ class CobwebVersion
3
3
 
4
4
  # Returns a string of the current version
5
5
  def self.version
6
- "1.0.29"
6
+ "1.1.0"
7
7
  end
8
8
 
9
9
  end
data/lib/content_link_parser.rb CHANGED
@@ -8,12 +8,11 @@ class ContentLinkParser
8
8
  def initialize(url, content, options = {})
9
9
  @options = {}.merge(options)
10
10
  @url = url
11
+ @base_url = ''
11
12
  @doc = Nokogiri::HTML(content)
12
13
 
13
- base_url = @url.to_s
14
14
  if @doc.at("base[href]")
15
- base_url = @doc.at("base[href]").attr("href").to_s
16
- @url = base_url if base_url
15
+ @base_url = @doc.at("base[href]").attr("href").to_s if @doc.at("base[href]").attr("href").to_s.present?
17
16
  end
18
17
 
19
18
  @options[:tags] = {}
@@ -46,7 +45,9 @@ class ContentLinkParser
46
45
  options[:valid_schemes] = [:http, :https] unless options.has_key? :valid_schemes
47
46
  data = link_data
48
47
  links = data.keys.map{|key| data[key]}.flatten.uniq
49
- links = links.map{|link| UriHelper.join_no_fragment(@url, link).to_s }
48
+ links = links.map{|link| UriHelper.join_no_fragment(@url, UriHelper.join_no_fragment(@base_url, link))}
49
+ .reject(&:nil?)
50
+ .map(&:to_s)
50
51
  links = links.reject{|link| link =~ /\/([^\/]+?)\/\1\// }
51
52
  links = links.reject{|link| link =~ /([^\/]+?)\/([^\/]+?)\/.*?\1\/\2/ }
52
53
  links = links.select{|link| options[:valid_schemes].include? link.split(':')[0].to_sym}
data/lib/sidekiq/cobweb_helper.rb CHANGED
@@ -2,6 +2,7 @@
2
2
  if Gem::Specification.find_all_by_name("sidekiq", ">=1.0.0").count >= 1
3
3
  SIDEKIQ_INSTALLED = true
4
4
  require 'sidekiq'
5
+ require 'sidekiq/api'
5
6
  else
6
7
  SIDEKIQ_INSTALLED = false
7
8
  puts "sidekiq gem not installed, skipping crawl_worker specs" if defined?(ENVIRONMENT) && ENVIRONMENT=="test"
@@ -29,4 +30,4 @@ module Sidekiq
29
30
  end
30
31
  end
31
32
  end
32
- end
33
+ end
data/spec/cobweb/cobweb_crawl_helper_spec.rb CHANGED
@@ -111,4 +111,4 @@ describe CobwebCrawlHelper do
111
111
  end
112
112
 
113
113
 
114
- end
114
+ end
data/spec/cobweb/crawl_worker_spec.rb CHANGED
@@ -212,7 +212,7 @@ def wait_for_crawl_finished(crawl_id, timeout=20)
212
212
  @counter = 0
213
213
  start_time = Time.now
214
214
  while(running?(crawl_id) && Time.now < start_time + timeout) do
215
- puts Sidekiq::Stats.new.queues
215
+ # puts Sidekiq::Stats.new.queues
216
216
  sleep 1
217
217
  end
218
218
  if Time.now > start_time + timeout
data/spec/spec_helper.rb CHANGED
@@ -35,13 +35,13 @@ RSpec.configure do |config|
35
35
  }
36
36
 
37
37
  config.before(:each) {
38
-
38
+
39
39
  @redis_mock_object = MockRedis.new
40
40
  Redis.stub(:new).and_return(@redis_mock_object)
41
41
  Redis::Namespace.stub(:new).and_return(@redis_mock_object)
42
-
42
+
43
43
  @redis_mock_object.flushdb
44
-
44
+
45
45
  }
46
46
 
47
47
  end
metadata CHANGED
@@ -1,127 +1,225 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: cobweb
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.29
4
+ version: 1.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Stewart McKee
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-11-05 00:00:00.000000000 Z
11
+ date: 2015-11-06 00:00:00.000000000 Z
12
12
  dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: rake
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ">="
18
+ - !ruby/object:Gem::Version
19
+ version: '0'
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ">="
25
+ - !ruby/object:Gem::Version
26
+ version: '0'
13
27
  - !ruby/object:Gem::Dependency
14
28
  name: redis
15
29
  requirement: !ruby/object:Gem::Requirement
16
30
  requirements:
17
- - - "~>"
31
+ - - ">="
18
32
  - !ruby/object:Gem::Version
19
- version: '3.0'
33
+ version: 3.2.1
20
34
  type: :runtime
21
35
  prerelease: false
22
36
  version_requirements: !ruby/object:Gem::Requirement
23
37
  requirements:
24
- - - "~>"
38
+ - - ">="
25
39
  - !ruby/object:Gem::Version
26
- version: '3.0'
40
+ version: 3.2.1
27
41
  - !ruby/object:Gem::Dependency
28
42
  name: nokogiri
29
43
  requirement: !ruby/object:Gem::Requirement
30
44
  requirements:
31
- - - "~>"
45
+ - - ">="
32
46
  - !ruby/object:Gem::Version
33
- version: '1.6'
47
+ version: 1.6.6.2
34
48
  type: :runtime
35
49
  prerelease: false
36
50
  version_requirements: !ruby/object:Gem::Requirement
37
51
  requirements:
38
- - - "~>"
52
+ - - ">="
39
53
  - !ruby/object:Gem::Version
40
- version: '1.6'
54
+ version: 1.6.6.2
41
55
  - !ruby/object:Gem::Dependency
42
56
  name: addressable
43
57
  requirement: !ruby/object:Gem::Requirement
44
58
  requirements:
45
- - - "~>"
59
+ - - ">="
46
60
  - !ruby/object:Gem::Version
47
- version: '2.3'
61
+ version: 2.3.8
48
62
  type: :runtime
49
63
  prerelease: false
50
64
  version_requirements: !ruby/object:Gem::Requirement
51
65
  requirements:
52
- - - "~>"
66
+ - - ">="
53
67
  - !ruby/object:Gem::Version
54
- version: '2.3'
68
+ version: 2.3.8
55
69
  - !ruby/object:Gem::Dependency
56
70
  name: sinatra
57
71
  requirement: !ruby/object:Gem::Requirement
58
72
  requirements:
59
- - - "~>"
73
+ - - ">="
60
74
  - !ruby/object:Gem::Version
61
- version: '1.4'
75
+ version: 1.4.6
62
76
  type: :runtime
63
77
  prerelease: false
64
78
  version_requirements: !ruby/object:Gem::Requirement
65
79
  requirements:
66
- - - "~>"
80
+ - - ">="
67
81
  - !ruby/object:Gem::Version
68
- version: '1.4'
82
+ version: 1.4.6
69
83
  - !ruby/object:Gem::Dependency
70
84
  name: haml
71
85
  requirement: !ruby/object:Gem::Requirement
72
86
  requirements:
73
- - - "~>"
87
+ - - ">="
74
88
  - !ruby/object:Gem::Version
75
- version: '4.0'
89
+ version: 4.0.7
76
90
  type: :runtime
77
91
  prerelease: false
78
92
  version_requirements: !ruby/object:Gem::Requirement
79
93
  requirements:
80
- - - "~>"
94
+ - - ">="
81
95
  - !ruby/object:Gem::Version
82
- version: '4.0'
96
+ version: 4.0.7
83
97
  - !ruby/object:Gem::Dependency
84
98
  name: redis-namespace
85
99
  requirement: !ruby/object:Gem::Requirement
86
100
  requirements:
87
- - - "~>"
101
+ - - ">="
88
102
  - !ruby/object:Gem::Version
89
- version: '1.3'
103
+ version: 1.5.2
90
104
  type: :runtime
91
105
  prerelease: false
92
106
  version_requirements: !ruby/object:Gem::Requirement
93
107
  requirements:
94
- - - "~>"
108
+ - - ">="
95
109
  - !ruby/object:Gem::Version
96
- version: '1.3'
110
+ version: 1.5.2
97
111
  - !ruby/object:Gem::Dependency
98
112
  name: json
99
113
  requirement: !ruby/object:Gem::Requirement
100
114
  requirements:
101
- - - "~>"
115
+ - - ">="
102
116
  - !ruby/object:Gem::Version
103
- version: '1.8'
117
+ version: 1.8.3
104
118
  type: :runtime
105
119
  prerelease: false
106
120
  version_requirements: !ruby/object:Gem::Requirement
107
121
  requirements:
108
- - - "~>"
122
+ - - ">="
109
123
  - !ruby/object:Gem::Version
110
- version: '1.8'
124
+ version: 1.8.3
111
125
  - !ruby/object:Gem::Dependency
112
126
  name: slop
113
127
  requirement: !ruby/object:Gem::Requirement
114
128
  requirements:
115
- - - "~>"
129
+ - - ">="
116
130
  - !ruby/object:Gem::Version
117
- version: '3.4'
131
+ version: 4.2.0
118
132
  type: :runtime
119
133
  prerelease: false
120
134
  version_requirements: !ruby/object:Gem::Requirement
121
135
  requirements:
122
- - - "~>"
136
+ - - ">="
137
+ - !ruby/object:Gem::Version
138
+ version: 4.2.0
139
+ - !ruby/object:Gem::Dependency
140
+ name: rspec
141
+ requirement: !ruby/object:Gem::Requirement
142
+ requirements:
143
+ - - ">="
144
+ - !ruby/object:Gem::Version
145
+ version: '0'
146
+ type: :development
147
+ prerelease: false
148
+ version_requirements: !ruby/object:Gem::Requirement
149
+ requirements:
150
+ - - ">="
151
+ - !ruby/object:Gem::Version
152
+ version: '0'
153
+ - !ruby/object:Gem::Dependency
154
+ name: rspec-core
155
+ requirement: !ruby/object:Gem::Requirement
156
+ requirements:
157
+ - - ">="
158
+ - !ruby/object:Gem::Version
159
+ version: '0'
160
+ type: :development
161
+ prerelease: false
162
+ version_requirements: !ruby/object:Gem::Requirement
163
+ requirements:
164
+ - - ">="
165
+ - !ruby/object:Gem::Version
166
+ version: '0'
167
+ - !ruby/object:Gem::Dependency
168
+ name: mock_redis
169
+ requirement: !ruby/object:Gem::Requirement
170
+ requirements:
171
+ - - ">="
172
+ - !ruby/object:Gem::Version
173
+ version: '0'
174
+ type: :development
175
+ prerelease: false
176
+ version_requirements: !ruby/object:Gem::Requirement
177
+ requirements:
178
+ - - ">="
179
+ - !ruby/object:Gem::Version
180
+ version: '0'
181
+ - !ruby/object:Gem::Dependency
182
+ name: thin
183
+ requirement: !ruby/object:Gem::Requirement
184
+ requirements:
185
+ - - ">="
186
+ - !ruby/object:Gem::Version
187
+ version: '0'
188
+ type: :development
189
+ prerelease: false
190
+ version_requirements: !ruby/object:Gem::Requirement
191
+ requirements:
192
+ - - ">="
193
+ - !ruby/object:Gem::Version
194
+ version: '0'
195
+ - !ruby/object:Gem::Dependency
196
+ name: coveralls
197
+ requirement: !ruby/object:Gem::Requirement
198
+ requirements:
199
+ - - ">="
200
+ - !ruby/object:Gem::Version
201
+ version: '0'
202
+ type: :development
203
+ prerelease: false
204
+ version_requirements: !ruby/object:Gem::Requirement
205
+ requirements:
206
+ - - ">="
207
+ - !ruby/object:Gem::Version
208
+ version: '0'
209
+ - !ruby/object:Gem::Dependency
210
+ name: sidekiq
211
+ requirement: !ruby/object:Gem::Requirement
212
+ requirements:
213
+ - - ">="
214
+ - !ruby/object:Gem::Version
215
+ version: '0'
216
+ type: :development
217
+ prerelease: false
218
+ version_requirements: !ruby/object:Gem::Requirement
219
+ requirements:
220
+ - - ">="
123
221
  - !ruby/object:Gem::Version
124
- version: '3.4'
222
+ version: '0'
125
223
  description: Cobweb is a web crawler that can use resque to cluster crawls to quickly
126
224
  crawl extremely large sites which is much more performant than multi-threaded crawlers. It
127
225
  is also a standalone crawler that has a sophisticated statistics monitoring interface
@@ -518,7 +616,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
518
616
  version: '0'
519
617
  requirements: []
520
618
  rubyforge_project:
521
- rubygems_version: 2.4.8
619
+ rubygems_version: 2.4.5.1
522
620
  signing_key:
523
621
  specification_version: 4
524
622
  summary: Cobweb is a web crawler that can use resque to cluster crawls to quickly