apollo-crawler 0.1.18 → 0.1.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (11) hide show
  1. checksums.yaml +15 -0
  2. data/lib/apollo_crawler.rb +3 -0
  3. data/lib/apollo_crawler/cache/mongo_cache.rb +2 -0
  4. data/lib/apollo_crawler/cli/cli.rb +19 -0
  5. data/lib/apollo_crawler/config.rb +3 -3
  6. data/lib/apollo_crawler/crawler/base_crawler.rb +2 -0
  7. data/lib/apollo_crawler/fetcher/base_fetcher.rb +28 -4
  8. data/lib/apollo_crawler/lib.rb +4 -0
  9. data/lib/apollo_crawler/program/crawler_program.rb +1 -2
  10. data/lib/apollo_crawler/version.rb +1 -1
  11. metadata +45 -80
checksums.yaml ADDED
@@ -0,0 +1,15 @@
1
+ ---
2
+ !binary "U0hBMQ==":
3
+ metadata.gz: !binary |-
4
+ OGFlNjhjOWJhMjI2YjdkMmU5YzgzMDUxZWEyNGI5MWZhMGIzYmRlZg==
5
+ data.tar.gz: !binary |-
6
+ ODc3YTMzOGIzMWU0Zjg2MDc4Njg5NWFlZWE5MDY0OTc5NWY0YmMyNg==
7
+ !binary "U0hBNTEy":
8
+ metadata.gz: !binary |-
9
+ MDQxM2U1YzYyNjRjOGYzN2RhZjY0ZmFjYTA5YWQxNWE3MDE3NTM3Mjg3OWE4
10
+ YWRkNmUzZWE3OGY5YjFjOTBiZTQ4YzY5ZGI4NmRmNjEyODVjMDQyZWE5MDk0
11
+ ZThiMGJmZjRjYWUyZWU1YTgwZGI0NDA1OGNmM2MyZDRiNGQzN2Y=
12
+ data.tar.gz: !binary |-
13
+ Y2M4ZWRlZjFiOTE5OWE5NzA1YzY5YzU4NmYxMjdlMjJjZmM4ZTc5ZGZkOWUy
14
+ MWU1OTMzNTZhOWJkMzVhMzIxMmQ5MWViMWM0ZmQ5YmI5ZDVkMDc5MDUwYmFh
15
+ Mzg5MzhlZmUwZmQxZjhmZWFhNTUyNzlkNTZlMDNkNTc2NTFjNTE=
data/lib/apollo_crawler.rb CHANGED
@@ -24,6 +24,9 @@
24
24
  # Caches
25
25
  require File.join(File.dirname(__FILE__), 'apollo_crawler/cache/caches')
26
26
 
27
+ # CLI - Command Line Interface
28
+ require File.join(File.dirname(__FILE__), 'apollo_crawler/cli/cli')
29
+
27
30
  # Crawlers
28
31
  require File.join(File.dirname(__FILE__), 'apollo_crawler/crawler/crawlers')
29
32
 
data/lib/apollo_crawler/cache/mongo_cache.rb CHANGED
@@ -50,6 +50,8 @@ module Apollo
50
50
 
51
51
  # Get value associated with key from cache
52
52
  def try_get(key, *args)
53
+ key = key.to_s
54
+
53
55
  res = get(key)
54
56
 
55
57
  # Not found, Create, cache and return
data/lib/apollo_crawler/cli/cli.rb ADDED
@@ -0,0 +1,19 @@
1
+ # Copyright, 2013, by Tomas Korcak. <korczis@gmail.com>
2
+ #
3
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
4
+ # of this software and associated documentation files (the "Software"), to deal
5
+ # in the Software without restriction, including without limitation the rights
6
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7
+ # copies of the Software, and to permit persons to whom the Software is
8
+ # furnished to do so, subject to the following conditions:
9
+ #
10
+ # The above copyright notice and this permission notice shall be included in
11
+ # all copies or substantial portions of the Software.
12
+ #
13
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19
+ # THE SOFTWARE.
data/lib/apollo_crawler/config.rb CHANGED
@@ -68,15 +68,15 @@ module RbConfig
68
68
  # CACHE_CLASS = Apollo::Cache::NullCache
69
69
 
70
70
  # Used caching mechanism by default
71
- CACHE_CLASS = Apollo::Cache::MemoryCache
71
+ CACHE_CLASS = Apollo::Cache::MongoCache
72
72
 
73
73
  CACHE_CLASS_OPTIONS = {
74
- :host => 'localhost',
74
+ :host => 'apollo-crawler.no-ip.org',
75
75
  :port => 27017,
76
76
  :pool_size => 5,
77
77
  :pool_timeout => 5,
78
78
  :db => 'apollo-crawler',
79
- :collection => 'cached_docs'
79
+ :collection => 'fetched_docs'
80
80
  }
81
81
 
82
82
 
data/lib/apollo_crawler/crawler/base_crawler.rb CHANGED
@@ -191,6 +191,8 @@ module Apollo
191
191
  return nil
192
192
  end
193
193
 
194
+ url = url.to_s
195
+
194
196
  # TODO: Use some (custom-made) low-level HTTTP Protocol cache - just for sure
195
197
  cache = Apollo::Cache::Factory.instance.construct
196
198
  metadoc = cache.try_get(url) do
data/lib/apollo_crawler/fetcher/base_fetcher.rb CHANGED
@@ -18,16 +18,40 @@
18
18
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19
19
  # THE SOFTWARE.
20
20
 
21
+ require 'cgi'
22
+ require "net/http"
21
23
  require "open-uri"
22
- require "nokogiri"
23
-
24
- require "em-http-request"
24
+ require 'faraday'
25
+ require 'ipaddr'
25
26
 
26
27
  module Apollo
27
28
  module Fetcher
28
29
  class BaseFetcher
30
+ def self.get_fake_headers(url)
31
+ ip = IPAddr.new(rand(2**32), Socket::AF_INET).to_s
32
+
33
+ return {
34
+ "X-Forwarded-For" => ip
35
+ }
36
+ end
37
+
29
38
  def self.fetch(url)
30
- return open(url).read
39
+ uri = URI.parse(url.to_s)
40
+
41
+ # See https://github.com/lostisland/faraday
42
+ conn = Faraday.new(:url => url) do |faraday|
43
+ # faraday.request :url_encoded # form-encode POST params
44
+ # faraday.response :logger # log requests to STDOUT
45
+ faraday.adapter Faraday.default_adapter # make requests with Net::HTTP
46
+ end
47
+
48
+ # Make request
49
+ res = conn.get(uri) do |request|
50
+ request.headers = BaseFetcher.get_fake_headers(uri)
51
+ end
52
+
53
+ # Return result
54
+ return res.body
31
55
  end
32
56
  end # class BaseFetcher
33
57
  end # module Fetcher
data/lib/apollo_crawler/lib.rb CHANGED
@@ -21,11 +21,15 @@
21
21
  # Caches
22
22
  require File.join(File.dirname(__FILE__), 'cache/caches')
23
23
 
24
+ # CLI - Command Line Interface
25
+ require File.join(File.dirname(__FILE__), 'cli/cli')
26
+
24
27
  # Crawlers
25
28
  require File.join(File.dirname(__FILE__), 'crawler/crawlers')
26
29
 
27
30
  # Fetchers
28
31
  require File.join(File.dirname(__FILE__), 'fetcher/fetchers')
32
+
29
33
  # Formatters
30
34
  require File.join(File.dirname(__FILE__), 'formatter/formatters')
31
35
 
data/lib/apollo_crawler/program/crawler_program.rb CHANGED
@@ -65,6 +65,7 @@ module Apollo
65
65
  @crawlers = {}
66
66
  @formatter = RbConfig::DEFAULT_FORMATTER
67
67
  @formatters = {}
68
+ @options = {}
68
69
 
69
70
  at_exit {
70
71
  at_exit_handler
@@ -73,8 +74,6 @@ module Apollo
73
74
 
74
75
  # Initialize command-line options
75
76
  def init_options()
76
- @options = {}
77
-
78
77
  @options[:doc_limit] = nil
79
78
  @options[:verbose] = false
80
79
  @options[:version] = false
data/lib/apollo_crawler/version.rb CHANGED
@@ -19,5 +19,5 @@
19
19
  # THE SOFTWARE.
20
20
 
21
21
  module Apollo
22
- VERSION = '0.1.18'
22
+ VERSION = '0.1.19'
23
23
  end # Apollo
metadata CHANGED
@@ -1,8 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: apollo-crawler
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.18
5
- prerelease:
4
+ version: 0.1.19
6
5
  platform: ruby
7
6
  authors:
8
7
  - Tomas Korcak
@@ -14,7 +13,6 @@ dependencies:
14
13
  - !ruby/object:Gem::Dependency
15
14
  name: awesome_print
16
15
  requirement: !ruby/object:Gem::Requirement
17
- none: false
18
16
  requirements:
19
17
  - - ~>
20
18
  - !ruby/object:Gem::Version
@@ -22,7 +20,6 @@ dependencies:
22
20
  type: :runtime
23
21
  prerelease: false
24
22
  version_requirements: !ruby/object:Gem::Requirement
25
- none: false
26
23
  requirements:
27
24
  - - ~>
28
25
  - !ruby/object:Gem::Version
@@ -30,7 +27,6 @@ dependencies:
30
27
  - !ruby/object:Gem::Dependency
31
28
  name: activesupport
32
29
  requirement: !ruby/object:Gem::Requirement
33
- none: false
34
30
  requirements:
35
31
  - - ! '>='
36
32
  - !ruby/object:Gem::Version
@@ -38,7 +34,6 @@ dependencies:
38
34
  type: :runtime
39
35
  prerelease: false
40
36
  version_requirements: !ruby/object:Gem::Requirement
41
- none: false
42
37
  requirements:
43
38
  - - ! '>='
44
39
  - !ruby/object:Gem::Version
@@ -46,7 +41,6 @@ dependencies:
46
41
  - !ruby/object:Gem::Dependency
47
42
  name: dalli
48
43
  requirement: !ruby/object:Gem::Requirement
49
- none: false
50
44
  requirements:
51
45
  - - ! '>='
52
46
  - !ruby/object:Gem::Version
@@ -54,7 +48,6 @@ dependencies:
54
48
  type: :runtime
55
49
  prerelease: false
56
50
  version_requirements: !ruby/object:Gem::Requirement
57
- none: false
58
51
  requirements:
59
52
  - - ! '>='
60
53
  - !ruby/object:Gem::Version
@@ -62,7 +55,6 @@ dependencies:
62
55
  - !ruby/object:Gem::Dependency
63
56
  name: fastercsv
64
57
  requirement: !ruby/object:Gem::Requirement
65
- none: false
66
58
  requirements:
67
59
  - - ~>
68
60
  - !ruby/object:Gem::Version
@@ -70,7 +62,6 @@ dependencies:
70
62
  type: :runtime
71
63
  prerelease: false
72
64
  version_requirements: !ruby/object:Gem::Requirement
73
- none: false
74
65
  requirements:
75
66
  - - ~>
76
67
  - !ruby/object:Gem::Version
@@ -78,7 +69,6 @@ dependencies:
78
69
  - !ruby/object:Gem::Dependency
79
70
  name: eventmachine
80
71
  requirement: !ruby/object:Gem::Requirement
81
- none: false
82
72
  requirements:
83
73
  - - ! '>='
84
74
  - !ruby/object:Gem::Version
@@ -86,7 +76,6 @@ dependencies:
86
76
  type: :runtime
87
77
  prerelease: false
88
78
  version_requirements: !ruby/object:Gem::Requirement
89
- none: false
90
79
  requirements:
91
80
  - - ! '>='
92
81
  - !ruby/object:Gem::Version
@@ -94,7 +83,6 @@ dependencies:
94
83
  - !ruby/object:Gem::Dependency
95
84
  name: em-http-request
96
85
  requirement: !ruby/object:Gem::Requirement
97
- none: false
98
86
  requirements:
99
87
  - - ! '>='
100
88
  - !ruby/object:Gem::Version
@@ -102,7 +90,6 @@ dependencies:
102
90
  type: :runtime
103
91
  prerelease: false
104
92
  version_requirements: !ruby/object:Gem::Requirement
105
- none: false
106
93
  requirements:
107
94
  - - ! '>='
108
95
  - !ruby/object:Gem::Version
@@ -110,7 +97,6 @@ dependencies:
110
97
  - !ruby/object:Gem::Dependency
111
98
  name: em-synchrony
112
99
  requirement: !ruby/object:Gem::Requirement
113
- none: false
114
100
  requirements:
115
101
  - - ! '>='
116
102
  - !ruby/object:Gem::Version
@@ -118,15 +104,27 @@ dependencies:
118
104
  type: :runtime
119
105
  prerelease: false
120
106
  version_requirements: !ruby/object:Gem::Requirement
121
- none: false
122
107
  requirements:
123
108
  - - ! '>='
124
109
  - !ruby/object:Gem::Version
125
110
  version: '0'
111
+ - !ruby/object:Gem::Dependency
112
+ name: faraday
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - ~>
116
+ - !ruby/object:Gem::Version
117
+ version: 0.8.6
118
+ type: :runtime
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - ~>
123
+ - !ruby/object:Gem::Version
124
+ version: 0.8.6
126
125
  - !ruby/object:Gem::Dependency
127
126
  name: amqp
128
127
  requirement: !ruby/object:Gem::Requirement
129
- none: false
130
128
  requirements:
131
129
  - - ~>
132
130
  - !ruby/object:Gem::Version
@@ -134,7 +132,6 @@ dependencies:
134
132
  type: :runtime
135
133
  prerelease: false
136
134
  version_requirements: !ruby/object:Gem::Requirement
137
- none: false
138
135
  requirements:
139
136
  - - ~>
140
137
  - !ruby/object:Gem::Version
@@ -142,7 +139,6 @@ dependencies:
142
139
  - !ruby/object:Gem::Dependency
143
140
  name: json
144
141
  requirement: !ruby/object:Gem::Requirement
145
- none: false
146
142
  requirements:
147
143
  - - ~>
148
144
  - !ruby/object:Gem::Version
@@ -150,7 +146,6 @@ dependencies:
150
146
  type: :runtime
151
147
  prerelease: false
152
148
  version_requirements: !ruby/object:Gem::Requirement
153
- none: false
154
149
  requirements:
155
150
  - - ~>
156
151
  - !ruby/object:Gem::Version
@@ -158,7 +153,6 @@ dependencies:
158
153
  - !ruby/object:Gem::Dependency
159
154
  name: memcache-client
160
155
  requirement: !ruby/object:Gem::Requirement
161
- none: false
162
156
  requirements:
163
157
  - - ! '>='
164
158
  - !ruby/object:Gem::Version
@@ -166,7 +160,6 @@ dependencies:
166
160
  type: :runtime
167
161
  prerelease: false
168
162
  version_requirements: !ruby/object:Gem::Requirement
169
- none: false
170
163
  requirements:
171
164
  - - ! '>='
172
165
  - !ruby/object:Gem::Version
@@ -174,7 +167,6 @@ dependencies:
174
167
  - !ruby/object:Gem::Dependency
175
168
  name: mongo
176
169
  requirement: !ruby/object:Gem::Requirement
177
- none: false
178
170
  requirements:
179
171
  - - ! '>='
180
172
  - !ruby/object:Gem::Version
@@ -182,7 +174,6 @@ dependencies:
182
174
  type: :runtime
183
175
  prerelease: false
184
176
  version_requirements: !ruby/object:Gem::Requirement
185
- none: false
186
177
  requirements:
187
178
  - - ! '>='
188
179
  - !ruby/object:Gem::Version
@@ -190,7 +181,6 @@ dependencies:
190
181
  - !ruby/object:Gem::Dependency
191
182
  name: mongoid
192
183
  requirement: !ruby/object:Gem::Requirement
193
- none: false
194
184
  requirements:
195
185
  - - ! '>='
196
186
  - !ruby/object:Gem::Version
@@ -198,7 +188,6 @@ dependencies:
198
188
  type: :runtime
199
189
  prerelease: false
200
190
  version_requirements: !ruby/object:Gem::Requirement
201
- none: false
202
191
  requirements:
203
192
  - - ! '>='
204
193
  - !ruby/object:Gem::Version
@@ -206,7 +195,6 @@ dependencies:
206
195
  - !ruby/object:Gem::Dependency
207
196
  name: mime-types
208
197
  requirement: !ruby/object:Gem::Requirement
209
- none: false
210
198
  requirements:
211
199
  - - ! '>='
212
200
  - !ruby/object:Gem::Version
@@ -214,7 +202,6 @@ dependencies:
214
202
  type: :runtime
215
203
  prerelease: false
216
204
  version_requirements: !ruby/object:Gem::Requirement
217
- none: false
218
205
  requirements:
219
206
  - - ! '>='
220
207
  - !ruby/object:Gem::Version
@@ -222,7 +209,6 @@ dependencies:
222
209
  - !ruby/object:Gem::Dependency
223
210
  name: nokogiri
224
211
  requirement: !ruby/object:Gem::Requirement
225
- none: false
226
212
  requirements:
227
213
  - - ~>
228
214
  - !ruby/object:Gem::Version
@@ -230,7 +216,6 @@ dependencies:
230
216
  type: :runtime
231
217
  prerelease: false
232
218
  version_requirements: !ruby/object:Gem::Requirement
233
- none: false
234
219
  requirements:
235
220
  - - ~>
236
221
  - !ruby/object:Gem::Version
@@ -238,7 +223,6 @@ dependencies:
238
223
  - !ruby/object:Gem::Dependency
239
224
  name: openurl
240
225
  requirement: !ruby/object:Gem::Requirement
241
- none: false
242
226
  requirements:
243
227
  - - ! '>='
244
228
  - !ruby/object:Gem::Version
@@ -246,7 +230,6 @@ dependencies:
246
230
  type: :runtime
247
231
  prerelease: false
248
232
  version_requirements: !ruby/object:Gem::Requirement
249
- none: false
250
233
  requirements:
251
234
  - - ! '>='
252
235
  - !ruby/object:Gem::Version
@@ -254,7 +237,6 @@ dependencies:
254
237
  - !ruby/object:Gem::Dependency
255
238
  name: parallel
256
239
  requirement: !ruby/object:Gem::Requirement
257
- none: false
258
240
  requirements:
259
241
  - - ! '>='
260
242
  - !ruby/object:Gem::Version
@@ -262,7 +244,6 @@ dependencies:
262
244
  type: :runtime
263
245
  prerelease: false
264
246
  version_requirements: !ruby/object:Gem::Requirement
265
- none: false
266
247
  requirements:
267
248
  - - ! '>='
268
249
  - !ruby/object:Gem::Version
@@ -270,7 +251,6 @@ dependencies:
270
251
  - !ruby/object:Gem::Dependency
271
252
  name: rack
272
253
  requirement: !ruby/object:Gem::Requirement
273
- none: false
274
254
  requirements:
275
255
  - - ! '>='
276
256
  - !ruby/object:Gem::Version
@@ -278,7 +258,6 @@ dependencies:
278
258
  type: :runtime
279
259
  prerelease: false
280
260
  version_requirements: !ruby/object:Gem::Requirement
281
- none: false
282
261
  requirements:
283
262
  - - ! '>='
284
263
  - !ruby/object:Gem::Version
@@ -286,7 +265,6 @@ dependencies:
286
265
  - !ruby/object:Gem::Dependency
287
266
  name: terminal-table
288
267
  requirement: !ruby/object:Gem::Requirement
289
- none: false
290
268
  requirements:
291
269
  - - ~>
292
270
  - !ruby/object:Gem::Version
@@ -294,7 +272,6 @@ dependencies:
294
272
  type: :runtime
295
273
  prerelease: false
296
274
  version_requirements: !ruby/object:Gem::Requirement
297
- none: false
298
275
  requirements:
299
276
  - - ~>
300
277
  - !ruby/object:Gem::Version
@@ -302,7 +279,6 @@ dependencies:
302
279
  - !ruby/object:Gem::Dependency
303
280
  name: thor
304
281
  requirement: !ruby/object:Gem::Requirement
305
- none: false
306
282
  requirements:
307
283
  - - ~>
308
284
  - !ruby/object:Gem::Version
@@ -310,7 +286,6 @@ dependencies:
310
286
  type: :runtime
311
287
  prerelease: false
312
288
  version_requirements: !ruby/object:Gem::Requirement
313
- none: false
314
289
  requirements:
315
290
  - - ~>
316
291
  - !ruby/object:Gem::Version
@@ -318,7 +293,6 @@ dependencies:
318
293
  - !ruby/object:Gem::Dependency
319
294
  name: writeexcel
320
295
  requirement: !ruby/object:Gem::Requirement
321
- none: false
322
296
  requirements:
323
297
  - - ~>
324
298
  - !ruby/object:Gem::Version
@@ -326,7 +300,6 @@ dependencies:
326
300
  type: :runtime
327
301
  prerelease: false
328
302
  version_requirements: !ruby/object:Gem::Requirement
329
- none: false
330
303
  requirements:
331
304
  - - ~>
332
305
  - !ruby/object:Gem::Version
@@ -334,7 +307,6 @@ dependencies:
334
307
  - !ruby/object:Gem::Dependency
335
308
  name: ffi
336
309
  requirement: !ruby/object:Gem::Requirement
337
- none: false
338
310
  requirements:
339
311
  - - ! '>='
340
312
  - !ruby/object:Gem::Version
@@ -342,7 +314,6 @@ dependencies:
342
314
  type: :runtime
343
315
  prerelease: false
344
316
  version_requirements: !ruby/object:Gem::Requirement
345
- none: false
346
317
  requirements:
347
318
  - - ! '>='
348
319
  - !ruby/object:Gem::Version
@@ -350,7 +321,6 @@ dependencies:
350
321
  - !ruby/object:Gem::Dependency
351
322
  name: guard
352
323
  requirement: !ruby/object:Gem::Requirement
353
- none: false
354
324
  requirements:
355
325
  - - ! '>='
356
326
  - !ruby/object:Gem::Version
@@ -358,7 +328,6 @@ dependencies:
358
328
  type: :development
359
329
  prerelease: false
360
330
  version_requirements: !ruby/object:Gem::Requirement
361
- none: false
362
331
  requirements:
363
332
  - - ! '>='
364
333
  - !ruby/object:Gem::Version
@@ -366,7 +335,6 @@ dependencies:
366
335
  - !ruby/object:Gem::Dependency
367
336
  name: guard-rake
368
337
  requirement: !ruby/object:Gem::Requirement
369
- none: false
370
338
  requirements:
371
339
  - - ! '>='
372
340
  - !ruby/object:Gem::Version
@@ -374,7 +342,6 @@ dependencies:
374
342
  type: :development
375
343
  prerelease: false
376
344
  version_requirements: !ruby/object:Gem::Requirement
377
- none: false
378
345
  requirements:
379
346
  - - ! '>='
380
347
  - !ruby/object:Gem::Version
@@ -382,7 +349,6 @@ dependencies:
382
349
  - !ruby/object:Gem::Dependency
383
350
  name: guard-rspec
384
351
  requirement: !ruby/object:Gem::Requirement
385
- none: false
386
352
  requirements:
387
353
  - - ! '>='
388
354
  - !ruby/object:Gem::Version
@@ -390,7 +356,6 @@ dependencies:
390
356
  type: :development
391
357
  prerelease: false
392
358
  version_requirements: !ruby/object:Gem::Requirement
393
- none: false
394
359
  requirements:
395
360
  - - ! '>='
396
361
  - !ruby/object:Gem::Version
@@ -402,69 +367,69 @@ executables:
402
367
  extensions: []
403
368
  extra_rdoc_files: []
404
369
  files:
405
- - ./lib/apollo_crawler/lib.rb
406
- - ./lib/apollo_crawler/program/base_program.rb
407
- - ./lib/apollo_crawler/program/programs.rb
408
- - ./lib/apollo_crawler/program/crawler_program.rb
409
370
  - ./lib/apollo_crawler/fetcher/smart_fetcher.rb
371
+ - ./lib/apollo_crawler/fetcher/fetchers.rb
410
372
  - ./lib/apollo_crawler/fetcher/simple_fetcher.rb
411
373
  - ./lib/apollo_crawler/fetcher/base_fetcher.rb
412
- - ./lib/apollo_crawler/fetcher/fetchers.rb
374
+ - ./lib/apollo_crawler/lib.rb
413
375
  - ./lib/apollo_crawler/version.rb
376
+ - ./lib/apollo_crawler/program/crawler_program.rb
377
+ - ./lib/apollo_crawler/program/base_program.rb
378
+ - ./lib/apollo_crawler/program/programs.rb
379
+ - ./lib/apollo_crawler/logger/console_logger.rb
380
+ - ./lib/apollo_crawler/logger/base_logger.rb
381
+ - ./lib/apollo_crawler/logger/loggers.rb
382
+ - ./lib/apollo_crawler/helper/core_helper.rb
383
+ - ./lib/apollo_crawler/helper/helpers.rb
384
+ - ./lib/apollo_crawler/cli/cli.rb
385
+ - ./lib/apollo_crawler/config.rb
386
+ - ./lib/apollo_crawler/cache/sqlite_cache.rb
387
+ - ./lib/apollo_crawler/cache/factory.rb
388
+ - ./lib/apollo_crawler/cache/null_cache.rb
389
+ - ./lib/apollo_crawler/cache/memory_cache.rb
390
+ - ./lib/apollo_crawler/cache/base_cache.rb
391
+ - ./lib/apollo_crawler/cache/mongo_cache.rb
392
+ - ./lib/apollo_crawler/cache/memcached_cache.rb
393
+ - ./lib/apollo_crawler/cache/caches.rb
394
+ - ./lib/apollo_crawler/crawler/xkcd_crawler.rb
395
+ - ./lib/apollo_crawler/crawler/google_crawler.rb
396
+ - ./lib/apollo_crawler/crawler/youjizz_crawler.rb
414
397
  - ./lib/apollo_crawler/crawler/slashdot_crawler.rb
415
- - ./lib/apollo_crawler/crawler/stackoverflow_crawler.rb
416
398
  - ./lib/apollo_crawler/crawler/hacker_news_crawler.rb
417
- - ./lib/apollo_crawler/crawler/youjizz_crawler.rb
418
- - ./lib/apollo_crawler/crawler/google_crawler.rb
419
399
  - ./lib/apollo_crawler/crawler/base_crawler.rb
420
- - ./lib/apollo_crawler/crawler/xkcd_crawler.rb
421
400
  - ./lib/apollo_crawler/crawler/crawlers.rb
422
- - ./lib/apollo_crawler/logger/loggers.rb
423
- - ./lib/apollo_crawler/logger/console_logger.rb
424
- - ./lib/apollo_crawler/logger/base_logger.rb
425
- - ./lib/apollo_crawler/config.rb
401
+ - ./lib/apollo_crawler/crawler/stackoverflow_crawler.rb
426
402
  - ./lib/apollo_crawler/formatter/table_formatter.rb
427
403
  - ./lib/apollo_crawler/formatter/base_formatter.rb
428
404
  - ./lib/apollo_crawler/formatter/json_formatter.rb
429
405
  - ./lib/apollo_crawler/formatter/plain_formatter.rb
430
406
  - ./lib/apollo_crawler/formatter/formatters.rb
431
- - ./lib/apollo_crawler/cache/memcached_cache.rb
432
- - ./lib/apollo_crawler/cache/memory_cache.rb
433
- - ./lib/apollo_crawler/cache/mongo_cache.rb
434
- - ./lib/apollo_crawler/cache/null_cache.rb
435
- - ./lib/apollo_crawler/cache/factory.rb
436
- - ./lib/apollo_crawler/cache/caches.rb
437
- - ./lib/apollo_crawler/cache/sqlite_cache.rb
438
- - ./lib/apollo_crawler/cache/base_cache.rb
439
- - ./lib/apollo_crawler/store/base_store.rb
440
407
  - ./lib/apollo_crawler/store/stores.rb
441
- - ./lib/apollo_crawler/helper/core_helper.rb
442
- - ./lib/apollo_crawler/helper/helpers.rb
408
+ - ./lib/apollo_crawler/store/base_store.rb
443
409
  - ./lib/apollo_crawler.rb
444
410
  - bin/apollo-crawler
445
411
  homepage: https://github.com/korczis/apollo-crawler
446
412
  licenses:
447
413
  - MIT
414
+ metadata: {}
448
415
  post_install_message: Thanks for installing Apollo Crawler!
449
416
  rdoc_options: []
450
417
  require_paths:
451
418
  - lib
452
419
  required_ruby_version: !ruby/object:Gem::Requirement
453
- none: false
454
420
  requirements:
455
421
  - - ! '>='
456
422
  - !ruby/object:Gem::Version
457
- version: '0'
423
+ version: 1.9.3
458
424
  required_rubygems_version: !ruby/object:Gem::Requirement
459
- none: false
460
425
  requirements:
461
426
  - - ! '>='
462
427
  - !ruby/object:Gem::Version
463
- version: '0'
428
+ version: 1.8.11
464
429
  requirements: []
465
430
  rubyforge_project:
466
- rubygems_version: 1.8.23
431
+ rubygems_version: 2.0.0
467
432
  signing_key:
468
- specification_version: 3
433
+ specification_version: 4
469
434
  summary: Apollo Platform Crawler
470
435
  test_files: []