polipus 0.0.1 → 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (5) hide show
  1. checksums.yaml +6 -14
  2. data/examples/basic.rb +3 -3
  3. data/lib/polipus/version.rb +1 -1
  4. data/polipus.gemspec +7 -4
  5. metadata +39 -53
checksums.yaml CHANGED
@@ -1,15 +1,7 @@
1
1
  ---
2
- !binary "U0hBMQ==":
3
- metadata.gz: !binary |-
4
- MjAwM2Q2OWZlZDc3ODkxMzE0YzZjY2UzNzcwODBmZDQ0NDdkYWY4Mg==
5
- data.tar.gz: !binary |-
6
- ZjUxNGNhY2RmODI3ZWIzNGQzODkwZjgwZjU3Njk1Njk2OTIwYWRjMw==
7
- !binary "U0hBNTEy":
8
- metadata.gz: !binary |-
9
- YTU3N2U4YmJjZjZjNzhiZTE2ZjIwY2YwOTc1MjNkMTVmMzdjZmY1NTlhZWQ4
10
- MTFjZGIyYjNmY2IwMmM1ZjFiZjFjZGE3NjA4ZWJjNDEyOTA4MmM5MDU4NDYx
11
- NTI1NGYwNzgwNzExNWI5NzBkMTY5ZDJiMmYzYmE2ZWNmMjIxNjU=
12
- data.tar.gz: !binary |-
13
- NWQyYjAyYzZiNDE4ZDM5ZmJlNWI1YjVlNTU3Mzk5ZGI3MDRhYTc2YTFlOWE1
14
- ZTk3YTc4ZTZjYWJkZDg1OTgzYTA0MjYwYzBlOTEwOWQyMDY3YTg0YTAxZjg5
15
- ZDYyN2QwNzU0MDk0NWMwZDQ4NjczZDFhMzk0MTVkNGY2ODk3YWE=
2
+ SHA1:
3
+ metadata.gz: 16d138f0f4b5f27b4e34851e577947641b2617eb
4
+ data.tar.gz: 4fdf80f57823ac96080641d3546dd67d13f36a64
5
+ SHA512:
6
+ metadata.gz: a3f21d158167db5030c3587dcdd5bfb83b2d167bef634af887e2563437988dc6b2908bcac44a0386459c0f02b2597cc515d9a383e0d2cb483e857828bd213349
7
+ data.tar.gz: d268b9485a71474da10b91fded3e673d832a27690e16b149abf83c2125a9bbbdf369b4efeb6d3913a83da6a3b9483f239f8dd607748e930c7ef75d4a55d4b74d
data/examples/basic.rb CHANGED
@@ -1,6 +1,6 @@
1
1
  require "polipus"
2
2
  require "mongo"
3
-
3
+ require "polipus/plugins/cleaner"
4
4
  # Define a Mongo connection
5
5
  mongo = Mongo::Connection.new(:pool_size => 15, :pool_timeout => 5).db('crawler')
6
6
 
@@ -20,9 +20,9 @@ options = {
20
20
  # Use 5 threads
21
21
  :workers => 5,
22
22
  # Logs goes to the crawler.log file
23
- :logger => Logger.new("/tmp/crawler.log")
23
+ :logger => Logger.new(STDOUT)
24
24
  }
25
-
25
+ Polipus::Plugin.register Polipus::Plugin::Cleaner, reset:true
26
26
  starting_urls = ["http://rubygems.org/gems"]
27
27
 
28
28
  # Crawl the entire rubygems's site
data/lib/polipus/version.rb CHANGED
@@ -1,4 +1,4 @@
1
1
  module Polipus
2
- VERSION = "0.0.1"
2
+ VERSION = "0.0.2"
3
3
  HOMEPAGE = "https://github.com/taganaka/polipus"
4
4
  end
data/polipus.gemspec CHANGED
@@ -7,7 +7,7 @@ Gem::Specification.new do |s|
7
7
  s.version = Polipus::VERSION
8
8
  s.authors = ["Francesco Laurita"]
9
9
  s.email = ["francesco.laurita@gmail.com"]
10
- s.homepage = "https://github.com/taganaka/polipus"
10
+ s.homepage = Polipus::HOMEPAGE
11
11
  s.summary = %q{Polipus distributed web-crawler framework}
12
12
  s.description = %q{
13
13
  An easy to use distributed web-crawler framework based on Redis
@@ -25,9 +25,12 @@ Gem::Specification.new do |s|
25
25
  s.add_dependency "nokogiri", "~> 1.6.0"
26
26
  s.add_dependency "hiredis", "~> 0.4.5"
27
27
  s.add_dependency "redis", "~> 3.0.4"
28
- s.add_dependency "mongo", "~> 1.8.6"
29
- s.add_dependency "bson_ext", "~> 1.8.6"
30
- s.add_dependency "json", "~> 1.8.0"
28
+ s.add_dependency "mongo", "~> 1.9.2"
29
+ if defined?(JRUBY_VERSION)
30
+ s.add_dependency "bson", "~> 1.9.2"
31
+ else
32
+ s.add_dependency "bson_ext", "~> 1.9.2"
33
+ end
31
34
  s.add_dependency "aws-s3", "~> 0.6.3"
32
35
  s.add_dependency "http-cookie", "~> 1.0.1"
33
36
 
metadata CHANGED
@@ -1,212 +1,198 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: polipus
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.1
4
+ version: 0.0.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Francesco Laurita
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2014-01-05 00:00:00.000000000 Z
11
+ date: 2014-02-01 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: redis-bloomfilter
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
- - - ~>
17
+ - - "~>"
18
18
  - !ruby/object:Gem::Version
19
19
  version: 0.0.1
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
- - - ~>
24
+ - - "~>"
25
25
  - !ruby/object:Gem::Version
26
26
  version: 0.0.1
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: redis-queue
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
- - - ~>
31
+ - - "~>"
32
32
  - !ruby/object:Gem::Version
33
33
  version: 0.0.3
34
34
  type: :runtime
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
- - - ~>
38
+ - - "~>"
39
39
  - !ruby/object:Gem::Version
40
40
  version: 0.0.3
41
41
  - !ruby/object:Gem::Dependency
42
42
  name: nokogiri
43
43
  requirement: !ruby/object:Gem::Requirement
44
44
  requirements:
45
- - - ~>
45
+ - - "~>"
46
46
  - !ruby/object:Gem::Version
47
47
  version: 1.6.0
48
48
  type: :runtime
49
49
  prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
- - - ~>
52
+ - - "~>"
53
53
  - !ruby/object:Gem::Version
54
54
  version: 1.6.0
55
55
  - !ruby/object:Gem::Dependency
56
56
  name: hiredis
57
57
  requirement: !ruby/object:Gem::Requirement
58
58
  requirements:
59
- - - ~>
59
+ - - "~>"
60
60
  - !ruby/object:Gem::Version
61
61
  version: 0.4.5
62
62
  type: :runtime
63
63
  prerelease: false
64
64
  version_requirements: !ruby/object:Gem::Requirement
65
65
  requirements:
66
- - - ~>
66
+ - - "~>"
67
67
  - !ruby/object:Gem::Version
68
68
  version: 0.4.5
69
69
  - !ruby/object:Gem::Dependency
70
70
  name: redis
71
71
  requirement: !ruby/object:Gem::Requirement
72
72
  requirements:
73
- - - ~>
73
+ - - "~>"
74
74
  - !ruby/object:Gem::Version
75
75
  version: 3.0.4
76
76
  type: :runtime
77
77
  prerelease: false
78
78
  version_requirements: !ruby/object:Gem::Requirement
79
79
  requirements:
80
- - - ~>
80
+ - - "~>"
81
81
  - !ruby/object:Gem::Version
82
82
  version: 3.0.4
83
83
  - !ruby/object:Gem::Dependency
84
84
  name: mongo
85
85
  requirement: !ruby/object:Gem::Requirement
86
86
  requirements:
87
- - - ~>
87
+ - - "~>"
88
88
  - !ruby/object:Gem::Version
89
- version: 1.8.6
89
+ version: 1.9.2
90
90
  type: :runtime
91
91
  prerelease: false
92
92
  version_requirements: !ruby/object:Gem::Requirement
93
93
  requirements:
94
- - - ~>
94
+ - - "~>"
95
95
  - !ruby/object:Gem::Version
96
- version: 1.8.6
96
+ version: 1.9.2
97
97
  - !ruby/object:Gem::Dependency
98
98
  name: bson_ext
99
99
  requirement: !ruby/object:Gem::Requirement
100
100
  requirements:
101
- - - ~>
101
+ - - "~>"
102
102
  - !ruby/object:Gem::Version
103
- version: 1.8.6
103
+ version: 1.9.2
104
104
  type: :runtime
105
105
  prerelease: false
106
106
  version_requirements: !ruby/object:Gem::Requirement
107
107
  requirements:
108
- - - ~>
108
+ - - "~>"
109
109
  - !ruby/object:Gem::Version
110
- version: 1.8.6
111
- - !ruby/object:Gem::Dependency
112
- name: json
113
- requirement: !ruby/object:Gem::Requirement
114
- requirements:
115
- - - ~>
116
- - !ruby/object:Gem::Version
117
- version: 1.8.0
118
- type: :runtime
119
- prerelease: false
120
- version_requirements: !ruby/object:Gem::Requirement
121
- requirements:
122
- - - ~>
123
- - !ruby/object:Gem::Version
124
- version: 1.8.0
110
+ version: 1.9.2
125
111
  - !ruby/object:Gem::Dependency
126
112
  name: aws-s3
127
113
  requirement: !ruby/object:Gem::Requirement
128
114
  requirements:
129
- - - ~>
115
+ - - "~>"
130
116
  - !ruby/object:Gem::Version
131
117
  version: 0.6.3
132
118
  type: :runtime
133
119
  prerelease: false
134
120
  version_requirements: !ruby/object:Gem::Requirement
135
121
  requirements:
136
- - - ~>
122
+ - - "~>"
137
123
  - !ruby/object:Gem::Version
138
124
  version: 0.6.3
139
125
  - !ruby/object:Gem::Dependency
140
126
  name: http-cookie
141
127
  requirement: !ruby/object:Gem::Requirement
142
128
  requirements:
143
- - - ~>
129
+ - - "~>"
144
130
  - !ruby/object:Gem::Version
145
131
  version: 1.0.1
146
132
  type: :runtime
147
133
  prerelease: false
148
134
  version_requirements: !ruby/object:Gem::Requirement
149
135
  requirements:
150
- - - ~>
136
+ - - "~>"
151
137
  - !ruby/object:Gem::Version
152
138
  version: 1.0.1
153
139
  - !ruby/object:Gem::Dependency
154
140
  name: rspec
155
141
  requirement: !ruby/object:Gem::Requirement
156
142
  requirements:
157
- - - ! '>='
143
+ - - ">="
158
144
  - !ruby/object:Gem::Version
159
145
  version: '0'
160
146
  type: :development
161
147
  prerelease: false
162
148
  version_requirements: !ruby/object:Gem::Requirement
163
149
  requirements:
164
- - - ! '>='
150
+ - - ">="
165
151
  - !ruby/object:Gem::Version
166
152
  version: '0'
167
153
  - !ruby/object:Gem::Dependency
168
154
  name: vcr
169
155
  requirement: !ruby/object:Gem::Requirement
170
156
  requirements:
171
- - - ~>
157
+ - - "~>"
172
158
  - !ruby/object:Gem::Version
173
159
  version: 2.5.0
174
160
  type: :development
175
161
  prerelease: false
176
162
  version_requirements: !ruby/object:Gem::Requirement
177
163
  requirements:
178
- - - ~>
164
+ - - "~>"
179
165
  - !ruby/object:Gem::Version
180
166
  version: 2.5.0
181
167
  - !ruby/object:Gem::Dependency
182
168
  name: webmock
183
169
  requirement: !ruby/object:Gem::Requirement
184
170
  requirements:
185
- - - ! '>='
171
+ - - ">="
186
172
  - !ruby/object:Gem::Version
187
173
  version: '0'
188
174
  type: :development
189
175
  prerelease: false
190
176
  version_requirements: !ruby/object:Gem::Requirement
191
177
  requirements:
192
- - - ! '>='
178
+ - - ">="
193
179
  - !ruby/object:Gem::Version
194
180
  version: '0'
195
181
  - !ruby/object:Gem::Dependency
196
182
  name: flexmock
197
183
  requirement: !ruby/object:Gem::Requirement
198
184
  requirements:
199
- - - ~>
185
+ - - "~>"
200
186
  - !ruby/object:Gem::Version
201
187
  version: 1.3.2
202
188
  type: :development
203
189
  prerelease: false
204
190
  version_requirements: !ruby/object:Gem::Requirement
205
191
  requirements:
206
- - - ~>
192
+ - - "~>"
207
193
  - !ruby/object:Gem::Version
208
194
  version: 1.3.2
209
- description: ! "\n An easy to use distributed web-crawler framework based on Redis\n
195
+ description: "\n An easy to use distributed web-crawler framework based on Redis\n
210
196
  \ "
211
197
  email:
212
198
  - francesco.laurita@gmail.com
@@ -214,9 +200,9 @@ executables: []
214
200
  extensions: []
215
201
  extra_rdoc_files: []
216
202
  files:
217
- - .document
218
- - .gitignore
219
- - .rspec
203
+ - ".document"
204
+ - ".gitignore"
205
+ - ".rspec"
220
206
  - Gemfile
221
207
  - LICENSE.txt
222
208
  - README.md
@@ -276,17 +262,17 @@ require_paths:
276
262
  - lib
277
263
  required_ruby_version: !ruby/object:Gem::Requirement
278
264
  requirements:
279
- - - ! '>='
265
+ - - ">="
280
266
  - !ruby/object:Gem::Version
281
267
  version: '0'
282
268
  required_rubygems_version: !ruby/object:Gem::Requirement
283
269
  requirements:
284
- - - ! '>='
270
+ - - ">="
285
271
  - !ruby/object:Gem::Version
286
272
  version: '0'
287
273
  requirements: []
288
274
  rubyforge_project: polipus
289
- rubygems_version: 2.0.7
275
+ rubygems_version: 2.2.1
290
276
  signing_key:
291
277
  specification_version: 4
292
278
  summary: Polipus distributed web-crawler framework