polipus 0.0.1 → 0.0.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (5) hide show
  1. checksums.yaml +6 -14
  2. data/examples/basic.rb +3 -3
  3. data/lib/polipus/version.rb +1 -1
  4. data/polipus.gemspec +7 -4
  5. metadata +39 -53
checksums.yaml CHANGED
@@ -1,15 +1,7 @@
1
1
  ---
2
- !binary "U0hBMQ==":
3
- metadata.gz: !binary |-
4
- MjAwM2Q2OWZlZDc3ODkxMzE0YzZjY2UzNzcwODBmZDQ0NDdkYWY4Mg==
5
- data.tar.gz: !binary |-
6
- ZjUxNGNhY2RmODI3ZWIzNGQzODkwZjgwZjU3Njk1Njk2OTIwYWRjMw==
7
- !binary "U0hBNTEy":
8
- metadata.gz: !binary |-
9
- YTU3N2U4YmJjZjZjNzhiZTE2ZjIwY2YwOTc1MjNkMTVmMzdjZmY1NTlhZWQ4
10
- MTFjZGIyYjNmY2IwMmM1ZjFiZjFjZGE3NjA4ZWJjNDEyOTA4MmM5MDU4NDYx
11
- NTI1NGYwNzgwNzExNWI5NzBkMTY5ZDJiMmYzYmE2ZWNmMjIxNjU=
12
- data.tar.gz: !binary |-
13
- NWQyYjAyYzZiNDE4ZDM5ZmJlNWI1YjVlNTU3Mzk5ZGI3MDRhYTc2YTFlOWE1
14
- ZTk3YTc4ZTZjYWJkZDg1OTgzYTA0MjYwYzBlOTEwOWQyMDY3YTg0YTAxZjg5
15
- ZDYyN2QwNzU0MDk0NWMwZDQ4NjczZDFhMzk0MTVkNGY2ODk3YWE=
2
+ SHA1:
3
+ metadata.gz: 16d138f0f4b5f27b4e34851e577947641b2617eb
4
+ data.tar.gz: 4fdf80f57823ac96080641d3546dd67d13f36a64
5
+ SHA512:
6
+ metadata.gz: a3f21d158167db5030c3587dcdd5bfb83b2d167bef634af887e2563437988dc6b2908bcac44a0386459c0f02b2597cc515d9a383e0d2cb483e857828bd213349
7
+ data.tar.gz: d268b9485a71474da10b91fded3e673d832a27690e16b149abf83c2125a9bbbdf369b4efeb6d3913a83da6a3b9483f239f8dd607748e930c7ef75d4a55d4b74d
data/examples/basic.rb CHANGED
@@ -1,6 +1,6 @@
1
1
  require "polipus"
2
2
  require "mongo"
3
-
3
+ require "polipus/plugins/cleaner"
4
4
  # Define a Mongo connection
5
5
  mongo = Mongo::Connection.new(:pool_size => 15, :pool_timeout => 5).db('crawler')
6
6
 
@@ -20,9 +20,9 @@ options = {
20
20
  # Use 5 threads
21
21
  :workers => 5,
22
22
  # Logs goes to the crawler.log file
23
- :logger => Logger.new("/tmp/crawler.log")
23
+ :logger => Logger.new(STDOUT)
24
24
  }
25
-
25
+ Polipus::Plugin.register Polipus::Plugin::Cleaner, reset:true
26
26
  starting_urls = ["http://rubygems.org/gems"]
27
27
 
28
28
  # Crawl the entire rubygems's site
data/lib/polipus/version.rb CHANGED
@@ -1,4 +1,4 @@
1
1
  module Polipus
2
- VERSION = "0.0.1"
2
+ VERSION = "0.0.2"
3
3
  HOMEPAGE = "https://github.com/taganaka/polipus"
4
4
  end
data/polipus.gemspec CHANGED
@@ -7,7 +7,7 @@ Gem::Specification.new do |s|
7
7
  s.version = Polipus::VERSION
8
8
  s.authors = ["Francesco Laurita"]
9
9
  s.email = ["francesco.laurita@gmail.com"]
10
- s.homepage = "https://github.com/taganaka/polipus"
10
+ s.homepage = Polipus::HOMEPAGE
11
11
  s.summary = %q{Polipus distributed web-crawler framework}
12
12
  s.description = %q{
13
13
  An easy to use distributed web-crawler framework based on Redis
@@ -25,9 +25,12 @@ Gem::Specification.new do |s|
25
25
  s.add_dependency "nokogiri", "~> 1.6.0"
26
26
  s.add_dependency "hiredis", "~> 0.4.5"
27
27
  s.add_dependency "redis", "~> 3.0.4"
28
- s.add_dependency "mongo", "~> 1.8.6"
29
- s.add_dependency "bson_ext", "~> 1.8.6"
30
- s.add_dependency "json", "~> 1.8.0"
28
+ s.add_dependency "mongo", "~> 1.9.2"
29
+ if defined?(JRUBY_VERSION)
30
+ s.add_dependency "bson", "~> 1.9.2"
31
+ else
32
+ s.add_dependency "bson_ext", "~> 1.9.2"
33
+ end
31
34
  s.add_dependency "aws-s3", "~> 0.6.3"
32
35
  s.add_dependency "http-cookie", "~> 1.0.1"
33
36
 
metadata CHANGED
@@ -1,212 +1,198 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: polipus
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.1
4
+ version: 0.0.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Francesco Laurita
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2014-01-05 00:00:00.000000000 Z
11
+ date: 2014-02-01 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: redis-bloomfilter
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
- - - ~>
17
+ - - "~>"
18
18
  - !ruby/object:Gem::Version
19
19
  version: 0.0.1
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
- - - ~>
24
+ - - "~>"
25
25
  - !ruby/object:Gem::Version
26
26
  version: 0.0.1
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: redis-queue
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
- - - ~>
31
+ - - "~>"
32
32
  - !ruby/object:Gem::Version
33
33
  version: 0.0.3
34
34
  type: :runtime
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
- - - ~>
38
+ - - "~>"
39
39
  - !ruby/object:Gem::Version
40
40
  version: 0.0.3
41
41
  - !ruby/object:Gem::Dependency
42
42
  name: nokogiri
43
43
  requirement: !ruby/object:Gem::Requirement
44
44
  requirements:
45
- - - ~>
45
+ - - "~>"
46
46
  - !ruby/object:Gem::Version
47
47
  version: 1.6.0
48
48
  type: :runtime
49
49
  prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
- - - ~>
52
+ - - "~>"
53
53
  - !ruby/object:Gem::Version
54
54
  version: 1.6.0
55
55
  - !ruby/object:Gem::Dependency
56
56
  name: hiredis
57
57
  requirement: !ruby/object:Gem::Requirement
58
58
  requirements:
59
- - - ~>
59
+ - - "~>"
60
60
  - !ruby/object:Gem::Version
61
61
  version: 0.4.5
62
62
  type: :runtime
63
63
  prerelease: false
64
64
  version_requirements: !ruby/object:Gem::Requirement
65
65
  requirements:
66
- - - ~>
66
+ - - "~>"
67
67
  - !ruby/object:Gem::Version
68
68
  version: 0.4.5
69
69
  - !ruby/object:Gem::Dependency
70
70
  name: redis
71
71
  requirement: !ruby/object:Gem::Requirement
72
72
  requirements:
73
- - - ~>
73
+ - - "~>"
74
74
  - !ruby/object:Gem::Version
75
75
  version: 3.0.4
76
76
  type: :runtime
77
77
  prerelease: false
78
78
  version_requirements: !ruby/object:Gem::Requirement
79
79
  requirements:
80
- - - ~>
80
+ - - "~>"
81
81
  - !ruby/object:Gem::Version
82
82
  version: 3.0.4
83
83
  - !ruby/object:Gem::Dependency
84
84
  name: mongo
85
85
  requirement: !ruby/object:Gem::Requirement
86
86
  requirements:
87
- - - ~>
87
+ - - "~>"
88
88
  - !ruby/object:Gem::Version
89
- version: 1.8.6
89
+ version: 1.9.2
90
90
  type: :runtime
91
91
  prerelease: false
92
92
  version_requirements: !ruby/object:Gem::Requirement
93
93
  requirements:
94
- - - ~>
94
+ - - "~>"
95
95
  - !ruby/object:Gem::Version
96
- version: 1.8.6
96
+ version: 1.9.2
97
97
  - !ruby/object:Gem::Dependency
98
98
  name: bson_ext
99
99
  requirement: !ruby/object:Gem::Requirement
100
100
  requirements:
101
- - - ~>
101
+ - - "~>"
102
102
  - !ruby/object:Gem::Version
103
- version: 1.8.6
103
+ version: 1.9.2
104
104
  type: :runtime
105
105
  prerelease: false
106
106
  version_requirements: !ruby/object:Gem::Requirement
107
107
  requirements:
108
- - - ~>
108
+ - - "~>"
109
109
  - !ruby/object:Gem::Version
110
- version: 1.8.6
111
- - !ruby/object:Gem::Dependency
112
- name: json
113
- requirement: !ruby/object:Gem::Requirement
114
- requirements:
115
- - - ~>
116
- - !ruby/object:Gem::Version
117
- version: 1.8.0
118
- type: :runtime
119
- prerelease: false
120
- version_requirements: !ruby/object:Gem::Requirement
121
- requirements:
122
- - - ~>
123
- - !ruby/object:Gem::Version
124
- version: 1.8.0
110
+ version: 1.9.2
125
111
  - !ruby/object:Gem::Dependency
126
112
  name: aws-s3
127
113
  requirement: !ruby/object:Gem::Requirement
128
114
  requirements:
129
- - - ~>
115
+ - - "~>"
130
116
  - !ruby/object:Gem::Version
131
117
  version: 0.6.3
132
118
  type: :runtime
133
119
  prerelease: false
134
120
  version_requirements: !ruby/object:Gem::Requirement
135
121
  requirements:
136
- - - ~>
122
+ - - "~>"
137
123
  - !ruby/object:Gem::Version
138
124
  version: 0.6.3
139
125
  - !ruby/object:Gem::Dependency
140
126
  name: http-cookie
141
127
  requirement: !ruby/object:Gem::Requirement
142
128
  requirements:
143
- - - ~>
129
+ - - "~>"
144
130
  - !ruby/object:Gem::Version
145
131
  version: 1.0.1
146
132
  type: :runtime
147
133
  prerelease: false
148
134
  version_requirements: !ruby/object:Gem::Requirement
149
135
  requirements:
150
- - - ~>
136
+ - - "~>"
151
137
  - !ruby/object:Gem::Version
152
138
  version: 1.0.1
153
139
  - !ruby/object:Gem::Dependency
154
140
  name: rspec
155
141
  requirement: !ruby/object:Gem::Requirement
156
142
  requirements:
157
- - - ! '>='
143
+ - - ">="
158
144
  - !ruby/object:Gem::Version
159
145
  version: '0'
160
146
  type: :development
161
147
  prerelease: false
162
148
  version_requirements: !ruby/object:Gem::Requirement
163
149
  requirements:
164
- - - ! '>='
150
+ - - ">="
165
151
  - !ruby/object:Gem::Version
166
152
  version: '0'
167
153
  - !ruby/object:Gem::Dependency
168
154
  name: vcr
169
155
  requirement: !ruby/object:Gem::Requirement
170
156
  requirements:
171
- - - ~>
157
+ - - "~>"
172
158
  - !ruby/object:Gem::Version
173
159
  version: 2.5.0
174
160
  type: :development
175
161
  prerelease: false
176
162
  version_requirements: !ruby/object:Gem::Requirement
177
163
  requirements:
178
- - - ~>
164
+ - - "~>"
179
165
  - !ruby/object:Gem::Version
180
166
  version: 2.5.0
181
167
  - !ruby/object:Gem::Dependency
182
168
  name: webmock
183
169
  requirement: !ruby/object:Gem::Requirement
184
170
  requirements:
185
- - - ! '>='
171
+ - - ">="
186
172
  - !ruby/object:Gem::Version
187
173
  version: '0'
188
174
  type: :development
189
175
  prerelease: false
190
176
  version_requirements: !ruby/object:Gem::Requirement
191
177
  requirements:
192
- - - ! '>='
178
+ - - ">="
193
179
  - !ruby/object:Gem::Version
194
180
  version: '0'
195
181
  - !ruby/object:Gem::Dependency
196
182
  name: flexmock
197
183
  requirement: !ruby/object:Gem::Requirement
198
184
  requirements:
199
- - - ~>
185
+ - - "~>"
200
186
  - !ruby/object:Gem::Version
201
187
  version: 1.3.2
202
188
  type: :development
203
189
  prerelease: false
204
190
  version_requirements: !ruby/object:Gem::Requirement
205
191
  requirements:
206
- - - ~>
192
+ - - "~>"
207
193
  - !ruby/object:Gem::Version
208
194
  version: 1.3.2
209
- description: ! "\n An easy to use distributed web-crawler framework based on Redis\n
195
+ description: "\n An easy to use distributed web-crawler framework based on Redis\n
210
196
  \ "
211
197
  email:
212
198
  - francesco.laurita@gmail.com
@@ -214,9 +200,9 @@ executables: []
214
200
  extensions: []
215
201
  extra_rdoc_files: []
216
202
  files:
217
- - .document
218
- - .gitignore
219
- - .rspec
203
+ - ".document"
204
+ - ".gitignore"
205
+ - ".rspec"
220
206
  - Gemfile
221
207
  - LICENSE.txt
222
208
  - README.md
@@ -276,17 +262,17 @@ require_paths:
276
262
  - lib
277
263
  required_ruby_version: !ruby/object:Gem::Requirement
278
264
  requirements:
279
- - - ! '>='
265
+ - - ">="
280
266
  - !ruby/object:Gem::Version
281
267
  version: '0'
282
268
  required_rubygems_version: !ruby/object:Gem::Requirement
283
269
  requirements:
284
- - - ! '>='
270
+ - - ">="
285
271
  - !ruby/object:Gem::Version
286
272
  version: '0'
287
273
  requirements: []
288
274
  rubyforge_project: polipus
289
- rubygems_version: 2.0.7
275
+ rubygems_version: 2.2.1
290
276
  signing_key:
291
277
  specification_version: 4
292
278
  summary: Polipus distributed web-crawler framework