apollo-crawler 0.1.18 → 0.1.19

Sign up to get free protection for your applications and to get access to all the features.
Files changed (11) hide show
  1. checksums.yaml +15 -0
  2. data/lib/apollo_crawler.rb +3 -0
  3. data/lib/apollo_crawler/cache/mongo_cache.rb +2 -0
  4. data/lib/apollo_crawler/cli/cli.rb +19 -0
  5. data/lib/apollo_crawler/config.rb +3 -3
  6. data/lib/apollo_crawler/crawler/base_crawler.rb +2 -0
  7. data/lib/apollo_crawler/fetcher/base_fetcher.rb +28 -4
  8. data/lib/apollo_crawler/lib.rb +4 -0
  9. data/lib/apollo_crawler/program/crawler_program.rb +1 -2
  10. data/lib/apollo_crawler/version.rb +1 -1
  11. metadata +45 -80
checksums.yaml ADDED
@@ -0,0 +1,15 @@
1
+ ---
2
+ !binary "U0hBMQ==":
3
+ metadata.gz: !binary |-
4
+ OGFlNjhjOWJhMjI2YjdkMmU5YzgzMDUxZWEyNGI5MWZhMGIzYmRlZg==
5
+ data.tar.gz: !binary |-
6
+ ODc3YTMzOGIzMWU0Zjg2MDc4Njg5NWFlZWE5MDY0OTc5NWY0YmMyNg==
7
+ !binary "U0hBNTEy":
8
+ metadata.gz: !binary |-
9
+ MDQxM2U1YzYyNjRjOGYzN2RhZjY0ZmFjYTA5YWQxNWE3MDE3NTM3Mjg3OWE4
10
+ YWRkNmUzZWE3OGY5YjFjOTBiZTQ4YzY5ZGI4NmRmNjEyODVjMDQyZWE5MDk0
11
+ ZThiMGJmZjRjYWUyZWU1YTgwZGI0NDA1OGNmM2MyZDRiNGQzN2Y=
12
+ data.tar.gz: !binary |-
13
+ Y2M4ZWRlZjFiOTE5OWE5NzA1YzY5YzU4NmYxMjdlMjJjZmM4ZTc5ZGZkOWUy
14
+ MWU1OTMzNTZhOWJkMzVhMzIxMmQ5MWViMWM0ZmQ5YmI5ZDVkMDc5MDUwYmFh
15
+ Mzg5MzhlZmUwZmQxZjhmZWFhNTUyNzlkNTZlMDNkNTc2NTFjNTE=
data/lib/apollo_crawler.rb CHANGED
@@ -24,6 +24,9 @@
24
24
  # Caches
25
25
  require File.join(File.dirname(__FILE__), 'apollo_crawler/cache/caches')
26
26
 
27
+ # CLI - Command Line Interface
28
+ require File.join(File.dirname(__FILE__), 'apollo_crawler/cli/cli')
29
+
27
30
  # Crawlers
28
31
  require File.join(File.dirname(__FILE__), 'apollo_crawler/crawler/crawlers')
29
32
 
data/lib/apollo_crawler/cache/mongo_cache.rb CHANGED
@@ -50,6 +50,8 @@ module Apollo
50
50
 
51
51
  # Get value associated with key from cache
52
52
  def try_get(key, *args)
53
+ key = key.to_s
54
+
53
55
  res = get(key)
54
56
 
55
57
  # Not found, Create, cache and return
data/lib/apollo_crawler/cli/cli.rb ADDED
@@ -0,0 +1,19 @@
1
+ # Copyright, 2013, by Tomas Korcak. <korczis@gmail.com>
2
+ #
3
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
4
+ # of this software and associated documentation files (the "Software"), to deal
5
+ # in the Software without restriction, including without limitation the rights
6
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7
+ # copies of the Software, and to permit persons to whom the Software is
8
+ # furnished to do so, subject to the following conditions:
9
+ #
10
+ # The above copyright notice and this permission notice shall be included in
11
+ # all copies or substantial portions of the Software.
12
+ #
13
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19
+ # THE SOFTWARE.
data/lib/apollo_crawler/config.rb CHANGED
@@ -68,15 +68,15 @@ module RbConfig
68
68
  # CACHE_CLASS = Apollo::Cache::NullCache
69
69
 
70
70
  # Used caching mechanism by default
71
- CACHE_CLASS = Apollo::Cache::MemoryCache
71
+ CACHE_CLASS = Apollo::Cache::MongoCache
72
72
 
73
73
  CACHE_CLASS_OPTIONS = {
74
- :host => 'localhost',
74
+ :host => 'apollo-crawler.no-ip.org',
75
75
  :port => 27017,
76
76
  :pool_size => 5,
77
77
  :pool_timeout => 5,
78
78
  :db => 'apollo-crawler',
79
- :collection => 'cached_docs'
79
+ :collection => 'fetched_docs'
80
80
  }
81
81
 
82
82
 
data/lib/apollo_crawler/crawler/base_crawler.rb CHANGED
@@ -191,6 +191,8 @@ module Apollo
191
191
  return nil
192
192
  end
193
193
 
194
+ url = url.to_s
195
+
194
196
  # TODO: Use some (custom-made) low-level HTTTP Protocol cache - just for sure
195
197
  cache = Apollo::Cache::Factory.instance.construct
196
198
  metadoc = cache.try_get(url) do
data/lib/apollo_crawler/fetcher/base_fetcher.rb CHANGED
@@ -18,16 +18,40 @@
18
18
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19
19
  # THE SOFTWARE.
20
20
 
21
+ require 'cgi'
22
+ require "net/http"
21
23
  require "open-uri"
22
- require "nokogiri"
23
-
24
- require "em-http-request"
24
+ require 'faraday'
25
+ require 'ipaddr'
25
26
 
26
27
  module Apollo
27
28
  module Fetcher
28
29
  class BaseFetcher
30
+ def self.get_fake_headers(url)
31
+ ip = IPAddr.new(rand(2**32), Socket::AF_INET).to_s
32
+
33
+ return {
34
+ "X-Forwarded-For" => ip
35
+ }
36
+ end
37
+
29
38
  def self.fetch(url)
30
- return open(url).read
39
+ uri = URI.parse(url.to_s)
40
+
41
+ # See https://github.com/lostisland/faraday
42
+ conn = Faraday.new(:url => url) do |faraday|
43
+ # faraday.request :url_encoded # form-encode POST params
44
+ # faraday.response :logger # log requests to STDOUT
45
+ faraday.adapter Faraday.default_adapter # make requests with Net::HTTP
46
+ end
47
+
48
+ # Make request
49
+ res = conn.get(uri) do |request|
50
+ request.headers = BaseFetcher.get_fake_headers(uri)
51
+ end
52
+
53
+ # Return result
54
+ return res.body
31
55
  end
32
56
  end # class BaseFetcher
33
57
  end # module Fetcher
data/lib/apollo_crawler/lib.rb CHANGED
@@ -21,11 +21,15 @@
21
21
  # Caches
22
22
  require File.join(File.dirname(__FILE__), 'cache/caches')
23
23
 
24
+ # CLI - Command Line Interface
25
+ require File.join(File.dirname(__FILE__), 'cli/cli')
26
+
24
27
  # Crawlers
25
28
  require File.join(File.dirname(__FILE__), 'crawler/crawlers')
26
29
 
27
30
  # Fetchers
28
31
  require File.join(File.dirname(__FILE__), 'fetcher/fetchers')
32
+
29
33
  # Formatters
30
34
  require File.join(File.dirname(__FILE__), 'formatter/formatters')
31
35
 
data/lib/apollo_crawler/program/crawler_program.rb CHANGED
@@ -65,6 +65,7 @@ module Apollo
65
65
  @crawlers = {}
66
66
  @formatter = RbConfig::DEFAULT_FORMATTER
67
67
  @formatters = {}
68
+ @options = {}
68
69
 
69
70
  at_exit {
70
71
  at_exit_handler
@@ -73,8 +74,6 @@ module Apollo
73
74
 
74
75
  # Initialize command-line options
75
76
  def init_options()
76
- @options = {}
77
-
78
77
  @options[:doc_limit] = nil
79
78
  @options[:verbose] = false
80
79
  @options[:version] = false
data/lib/apollo_crawler/version.rb CHANGED
@@ -19,5 +19,5 @@
19
19
  # THE SOFTWARE.
20
20
 
21
21
  module Apollo
22
- VERSION = '0.1.18'
22
+ VERSION = '0.1.19'
23
23
  end # Apollo
metadata CHANGED
@@ -1,8 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: apollo-crawler
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.18
5
- prerelease:
4
+ version: 0.1.19
6
5
  platform: ruby
7
6
  authors:
8
7
  - Tomas Korcak
@@ -14,7 +13,6 @@ dependencies:
14
13
  - !ruby/object:Gem::Dependency
15
14
  name: awesome_print
16
15
  requirement: !ruby/object:Gem::Requirement
17
- none: false
18
16
  requirements:
19
17
  - - ~>
20
18
  - !ruby/object:Gem::Version
@@ -22,7 +20,6 @@ dependencies:
22
20
  type: :runtime
23
21
  prerelease: false
24
22
  version_requirements: !ruby/object:Gem::Requirement
25
- none: false
26
23
  requirements:
27
24
  - - ~>
28
25
  - !ruby/object:Gem::Version
@@ -30,7 +27,6 @@ dependencies:
30
27
  - !ruby/object:Gem::Dependency
31
28
  name: activesupport
32
29
  requirement: !ruby/object:Gem::Requirement
33
- none: false
34
30
  requirements:
35
31
  - - ! '>='
36
32
  - !ruby/object:Gem::Version
@@ -38,7 +34,6 @@ dependencies:
38
34
  type: :runtime
39
35
  prerelease: false
40
36
  version_requirements: !ruby/object:Gem::Requirement
41
- none: false
42
37
  requirements:
43
38
  - - ! '>='
44
39
  - !ruby/object:Gem::Version
@@ -46,7 +41,6 @@ dependencies:
46
41
  - !ruby/object:Gem::Dependency
47
42
  name: dalli
48
43
  requirement: !ruby/object:Gem::Requirement
49
- none: false
50
44
  requirements:
51
45
  - - ! '>='
52
46
  - !ruby/object:Gem::Version
@@ -54,7 +48,6 @@ dependencies:
54
48
  type: :runtime
55
49
  prerelease: false
56
50
  version_requirements: !ruby/object:Gem::Requirement
57
- none: false
58
51
  requirements:
59
52
  - - ! '>='
60
53
  - !ruby/object:Gem::Version
@@ -62,7 +55,6 @@ dependencies:
62
55
  - !ruby/object:Gem::Dependency
63
56
  name: fastercsv
64
57
  requirement: !ruby/object:Gem::Requirement
65
- none: false
66
58
  requirements:
67
59
  - - ~>
68
60
  - !ruby/object:Gem::Version
@@ -70,7 +62,6 @@ dependencies:
70
62
  type: :runtime
71
63
  prerelease: false
72
64
  version_requirements: !ruby/object:Gem::Requirement
73
- none: false
74
65
  requirements:
75
66
  - - ~>
76
67
  - !ruby/object:Gem::Version
@@ -78,7 +69,6 @@ dependencies:
78
69
  - !ruby/object:Gem::Dependency
79
70
  name: eventmachine
80
71
  requirement: !ruby/object:Gem::Requirement
81
- none: false
82
72
  requirements:
83
73
  - - ! '>='
84
74
  - !ruby/object:Gem::Version
@@ -86,7 +76,6 @@ dependencies:
86
76
  type: :runtime
87
77
  prerelease: false
88
78
  version_requirements: !ruby/object:Gem::Requirement
89
- none: false
90
79
  requirements:
91
80
  - - ! '>='
92
81
  - !ruby/object:Gem::Version
@@ -94,7 +83,6 @@ dependencies:
94
83
  - !ruby/object:Gem::Dependency
95
84
  name: em-http-request
96
85
  requirement: !ruby/object:Gem::Requirement
97
- none: false
98
86
  requirements:
99
87
  - - ! '>='
100
88
  - !ruby/object:Gem::Version
@@ -102,7 +90,6 @@ dependencies:
102
90
  type: :runtime
103
91
  prerelease: false
104
92
  version_requirements: !ruby/object:Gem::Requirement
105
- none: false
106
93
  requirements:
107
94
  - - ! '>='
108
95
  - !ruby/object:Gem::Version
@@ -110,7 +97,6 @@ dependencies:
110
97
  - !ruby/object:Gem::Dependency
111
98
  name: em-synchrony
112
99
  requirement: !ruby/object:Gem::Requirement
113
- none: false
114
100
  requirements:
115
101
  - - ! '>='
116
102
  - !ruby/object:Gem::Version
@@ -118,15 +104,27 @@ dependencies:
118
104
  type: :runtime
119
105
  prerelease: false
120
106
  version_requirements: !ruby/object:Gem::Requirement
121
- none: false
122
107
  requirements:
123
108
  - - ! '>='
124
109
  - !ruby/object:Gem::Version
125
110
  version: '0'
111
+ - !ruby/object:Gem::Dependency
112
+ name: faraday
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - ~>
116
+ - !ruby/object:Gem::Version
117
+ version: 0.8.6
118
+ type: :runtime
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - ~>
123
+ - !ruby/object:Gem::Version
124
+ version: 0.8.6
126
125
  - !ruby/object:Gem::Dependency
127
126
  name: amqp
128
127
  requirement: !ruby/object:Gem::Requirement
129
- none: false
130
128
  requirements:
131
129
  - - ~>
132
130
  - !ruby/object:Gem::Version
@@ -134,7 +132,6 @@ dependencies:
134
132
  type: :runtime
135
133
  prerelease: false
136
134
  version_requirements: !ruby/object:Gem::Requirement
137
- none: false
138
135
  requirements:
139
136
  - - ~>
140
137
  - !ruby/object:Gem::Version
@@ -142,7 +139,6 @@ dependencies:
142
139
  - !ruby/object:Gem::Dependency
143
140
  name: json
144
141
  requirement: !ruby/object:Gem::Requirement
145
- none: false
146
142
  requirements:
147
143
  - - ~>
148
144
  - !ruby/object:Gem::Version
@@ -150,7 +146,6 @@ dependencies:
150
146
  type: :runtime
151
147
  prerelease: false
152
148
  version_requirements: !ruby/object:Gem::Requirement
153
- none: false
154
149
  requirements:
155
150
  - - ~>
156
151
  - !ruby/object:Gem::Version
@@ -158,7 +153,6 @@ dependencies:
158
153
  - !ruby/object:Gem::Dependency
159
154
  name: memcache-client
160
155
  requirement: !ruby/object:Gem::Requirement
161
- none: false
162
156
  requirements:
163
157
  - - ! '>='
164
158
  - !ruby/object:Gem::Version
@@ -166,7 +160,6 @@ dependencies:
166
160
  type: :runtime
167
161
  prerelease: false
168
162
  version_requirements: !ruby/object:Gem::Requirement
169
- none: false
170
163
  requirements:
171
164
  - - ! '>='
172
165
  - !ruby/object:Gem::Version
@@ -174,7 +167,6 @@ dependencies:
174
167
  - !ruby/object:Gem::Dependency
175
168
  name: mongo
176
169
  requirement: !ruby/object:Gem::Requirement
177
- none: false
178
170
  requirements:
179
171
  - - ! '>='
180
172
  - !ruby/object:Gem::Version
@@ -182,7 +174,6 @@ dependencies:
182
174
  type: :runtime
183
175
  prerelease: false
184
176
  version_requirements: !ruby/object:Gem::Requirement
185
- none: false
186
177
  requirements:
187
178
  - - ! '>='
188
179
  - !ruby/object:Gem::Version
@@ -190,7 +181,6 @@ dependencies:
190
181
  - !ruby/object:Gem::Dependency
191
182
  name: mongoid
192
183
  requirement: !ruby/object:Gem::Requirement
193
- none: false
194
184
  requirements:
195
185
  - - ! '>='
196
186
  - !ruby/object:Gem::Version
@@ -198,7 +188,6 @@ dependencies:
198
188
  type: :runtime
199
189
  prerelease: false
200
190
  version_requirements: !ruby/object:Gem::Requirement
201
- none: false
202
191
  requirements:
203
192
  - - ! '>='
204
193
  - !ruby/object:Gem::Version
@@ -206,7 +195,6 @@ dependencies:
206
195
  - !ruby/object:Gem::Dependency
207
196
  name: mime-types
208
197
  requirement: !ruby/object:Gem::Requirement
209
- none: false
210
198
  requirements:
211
199
  - - ! '>='
212
200
  - !ruby/object:Gem::Version
@@ -214,7 +202,6 @@ dependencies:
214
202
  type: :runtime
215
203
  prerelease: false
216
204
  version_requirements: !ruby/object:Gem::Requirement
217
- none: false
218
205
  requirements:
219
206
  - - ! '>='
220
207
  - !ruby/object:Gem::Version
@@ -222,7 +209,6 @@ dependencies:
222
209
  - !ruby/object:Gem::Dependency
223
210
  name: nokogiri
224
211
  requirement: !ruby/object:Gem::Requirement
225
- none: false
226
212
  requirements:
227
213
  - - ~>
228
214
  - !ruby/object:Gem::Version
@@ -230,7 +216,6 @@ dependencies:
230
216
  type: :runtime
231
217
  prerelease: false
232
218
  version_requirements: !ruby/object:Gem::Requirement
233
- none: false
234
219
  requirements:
235
220
  - - ~>
236
221
  - !ruby/object:Gem::Version
@@ -238,7 +223,6 @@ dependencies:
238
223
  - !ruby/object:Gem::Dependency
239
224
  name: openurl
240
225
  requirement: !ruby/object:Gem::Requirement
241
- none: false
242
226
  requirements:
243
227
  - - ! '>='
244
228
  - !ruby/object:Gem::Version
@@ -246,7 +230,6 @@ dependencies:
246
230
  type: :runtime
247
231
  prerelease: false
248
232
  version_requirements: !ruby/object:Gem::Requirement
249
- none: false
250
233
  requirements:
251
234
  - - ! '>='
252
235
  - !ruby/object:Gem::Version
@@ -254,7 +237,6 @@ dependencies:
254
237
  - !ruby/object:Gem::Dependency
255
238
  name: parallel
256
239
  requirement: !ruby/object:Gem::Requirement
257
- none: false
258
240
  requirements:
259
241
  - - ! '>='
260
242
  - !ruby/object:Gem::Version
@@ -262,7 +244,6 @@ dependencies:
262
244
  type: :runtime
263
245
  prerelease: false
264
246
  version_requirements: !ruby/object:Gem::Requirement
265
- none: false
266
247
  requirements:
267
248
  - - ! '>='
268
249
  - !ruby/object:Gem::Version
@@ -270,7 +251,6 @@ dependencies:
270
251
  - !ruby/object:Gem::Dependency
271
252
  name: rack
272
253
  requirement: !ruby/object:Gem::Requirement
273
- none: false
274
254
  requirements:
275
255
  - - ! '>='
276
256
  - !ruby/object:Gem::Version
@@ -278,7 +258,6 @@ dependencies:
278
258
  type: :runtime
279
259
  prerelease: false
280
260
  version_requirements: !ruby/object:Gem::Requirement
281
- none: false
282
261
  requirements:
283
262
  - - ! '>='
284
263
  - !ruby/object:Gem::Version
@@ -286,7 +265,6 @@ dependencies:
286
265
  - !ruby/object:Gem::Dependency
287
266
  name: terminal-table
288
267
  requirement: !ruby/object:Gem::Requirement
289
- none: false
290
268
  requirements:
291
269
  - - ~>
292
270
  - !ruby/object:Gem::Version
@@ -294,7 +272,6 @@ dependencies:
294
272
  type: :runtime
295
273
  prerelease: false
296
274
  version_requirements: !ruby/object:Gem::Requirement
297
- none: false
298
275
  requirements:
299
276
  - - ~>
300
277
  - !ruby/object:Gem::Version
@@ -302,7 +279,6 @@ dependencies:
302
279
  - !ruby/object:Gem::Dependency
303
280
  name: thor
304
281
  requirement: !ruby/object:Gem::Requirement
305
- none: false
306
282
  requirements:
307
283
  - - ~>
308
284
  - !ruby/object:Gem::Version
@@ -310,7 +286,6 @@ dependencies:
310
286
  type: :runtime
311
287
  prerelease: false
312
288
  version_requirements: !ruby/object:Gem::Requirement
313
- none: false
314
289
  requirements:
315
290
  - - ~>
316
291
  - !ruby/object:Gem::Version
@@ -318,7 +293,6 @@ dependencies:
318
293
  - !ruby/object:Gem::Dependency
319
294
  name: writeexcel
320
295
  requirement: !ruby/object:Gem::Requirement
321
- none: false
322
296
  requirements:
323
297
  - - ~>
324
298
  - !ruby/object:Gem::Version
@@ -326,7 +300,6 @@ dependencies:
326
300
  type: :runtime
327
301
  prerelease: false
328
302
  version_requirements: !ruby/object:Gem::Requirement
329
- none: false
330
303
  requirements:
331
304
  - - ~>
332
305
  - !ruby/object:Gem::Version
@@ -334,7 +307,6 @@ dependencies:
334
307
  - !ruby/object:Gem::Dependency
335
308
  name: ffi
336
309
  requirement: !ruby/object:Gem::Requirement
337
- none: false
338
310
  requirements:
339
311
  - - ! '>='
340
312
  - !ruby/object:Gem::Version
@@ -342,7 +314,6 @@ dependencies:
342
314
  type: :runtime
343
315
  prerelease: false
344
316
  version_requirements: !ruby/object:Gem::Requirement
345
- none: false
346
317
  requirements:
347
318
  - - ! '>='
348
319
  - !ruby/object:Gem::Version
@@ -350,7 +321,6 @@ dependencies:
350
321
  - !ruby/object:Gem::Dependency
351
322
  name: guard
352
323
  requirement: !ruby/object:Gem::Requirement
353
- none: false
354
324
  requirements:
355
325
  - - ! '>='
356
326
  - !ruby/object:Gem::Version
@@ -358,7 +328,6 @@ dependencies:
358
328
  type: :development
359
329
  prerelease: false
360
330
  version_requirements: !ruby/object:Gem::Requirement
361
- none: false
362
331
  requirements:
363
332
  - - ! '>='
364
333
  - !ruby/object:Gem::Version
@@ -366,7 +335,6 @@ dependencies:
366
335
  - !ruby/object:Gem::Dependency
367
336
  name: guard-rake
368
337
  requirement: !ruby/object:Gem::Requirement
369
- none: false
370
338
  requirements:
371
339
  - - ! '>='
372
340
  - !ruby/object:Gem::Version
@@ -374,7 +342,6 @@ dependencies:
374
342
  type: :development
375
343
  prerelease: false
376
344
  version_requirements: !ruby/object:Gem::Requirement
377
- none: false
378
345
  requirements:
379
346
  - - ! '>='
380
347
  - !ruby/object:Gem::Version
@@ -382,7 +349,6 @@ dependencies:
382
349
  - !ruby/object:Gem::Dependency
383
350
  name: guard-rspec
384
351
  requirement: !ruby/object:Gem::Requirement
385
- none: false
386
352
  requirements:
387
353
  - - ! '>='
388
354
  - !ruby/object:Gem::Version
@@ -390,7 +356,6 @@ dependencies:
390
356
  type: :development
391
357
  prerelease: false
392
358
  version_requirements: !ruby/object:Gem::Requirement
393
- none: false
394
359
  requirements:
395
360
  - - ! '>='
396
361
  - !ruby/object:Gem::Version
@@ -402,69 +367,69 @@ executables:
402
367
  extensions: []
403
368
  extra_rdoc_files: []
404
369
  files:
405
- - ./lib/apollo_crawler/lib.rb
406
- - ./lib/apollo_crawler/program/base_program.rb
407
- - ./lib/apollo_crawler/program/programs.rb
408
- - ./lib/apollo_crawler/program/crawler_program.rb
409
370
  - ./lib/apollo_crawler/fetcher/smart_fetcher.rb
371
+ - ./lib/apollo_crawler/fetcher/fetchers.rb
410
372
  - ./lib/apollo_crawler/fetcher/simple_fetcher.rb
411
373
  - ./lib/apollo_crawler/fetcher/base_fetcher.rb
412
- - ./lib/apollo_crawler/fetcher/fetchers.rb
374
+ - ./lib/apollo_crawler/lib.rb
413
375
  - ./lib/apollo_crawler/version.rb
376
+ - ./lib/apollo_crawler/program/crawler_program.rb
377
+ - ./lib/apollo_crawler/program/base_program.rb
378
+ - ./lib/apollo_crawler/program/programs.rb
379
+ - ./lib/apollo_crawler/logger/console_logger.rb
380
+ - ./lib/apollo_crawler/logger/base_logger.rb
381
+ - ./lib/apollo_crawler/logger/loggers.rb
382
+ - ./lib/apollo_crawler/helper/core_helper.rb
383
+ - ./lib/apollo_crawler/helper/helpers.rb
384
+ - ./lib/apollo_crawler/cli/cli.rb
385
+ - ./lib/apollo_crawler/config.rb
386
+ - ./lib/apollo_crawler/cache/sqlite_cache.rb
387
+ - ./lib/apollo_crawler/cache/factory.rb
388
+ - ./lib/apollo_crawler/cache/null_cache.rb
389
+ - ./lib/apollo_crawler/cache/memory_cache.rb
390
+ - ./lib/apollo_crawler/cache/base_cache.rb
391
+ - ./lib/apollo_crawler/cache/mongo_cache.rb
392
+ - ./lib/apollo_crawler/cache/memcached_cache.rb
393
+ - ./lib/apollo_crawler/cache/caches.rb
394
+ - ./lib/apollo_crawler/crawler/xkcd_crawler.rb
395
+ - ./lib/apollo_crawler/crawler/google_crawler.rb
396
+ - ./lib/apollo_crawler/crawler/youjizz_crawler.rb
414
397
  - ./lib/apollo_crawler/crawler/slashdot_crawler.rb
415
- - ./lib/apollo_crawler/crawler/stackoverflow_crawler.rb
416
398
  - ./lib/apollo_crawler/crawler/hacker_news_crawler.rb
417
- - ./lib/apollo_crawler/crawler/youjizz_crawler.rb
418
- - ./lib/apollo_crawler/crawler/google_crawler.rb
419
399
  - ./lib/apollo_crawler/crawler/base_crawler.rb
420
- - ./lib/apollo_crawler/crawler/xkcd_crawler.rb
421
400
  - ./lib/apollo_crawler/crawler/crawlers.rb
422
- - ./lib/apollo_crawler/logger/loggers.rb
423
- - ./lib/apollo_crawler/logger/console_logger.rb
424
- - ./lib/apollo_crawler/logger/base_logger.rb
425
- - ./lib/apollo_crawler/config.rb
401
+ - ./lib/apollo_crawler/crawler/stackoverflow_crawler.rb
426
402
  - ./lib/apollo_crawler/formatter/table_formatter.rb
427
403
  - ./lib/apollo_crawler/formatter/base_formatter.rb
428
404
  - ./lib/apollo_crawler/formatter/json_formatter.rb
429
405
  - ./lib/apollo_crawler/formatter/plain_formatter.rb
430
406
  - ./lib/apollo_crawler/formatter/formatters.rb
431
- - ./lib/apollo_crawler/cache/memcached_cache.rb
432
- - ./lib/apollo_crawler/cache/memory_cache.rb
433
- - ./lib/apollo_crawler/cache/mongo_cache.rb
434
- - ./lib/apollo_crawler/cache/null_cache.rb
435
- - ./lib/apollo_crawler/cache/factory.rb
436
- - ./lib/apollo_crawler/cache/caches.rb
437
- - ./lib/apollo_crawler/cache/sqlite_cache.rb
438
- - ./lib/apollo_crawler/cache/base_cache.rb
439
- - ./lib/apollo_crawler/store/base_store.rb
440
407
  - ./lib/apollo_crawler/store/stores.rb
441
- - ./lib/apollo_crawler/helper/core_helper.rb
442
- - ./lib/apollo_crawler/helper/helpers.rb
408
+ - ./lib/apollo_crawler/store/base_store.rb
443
409
  - ./lib/apollo_crawler.rb
444
410
  - bin/apollo-crawler
445
411
  homepage: https://github.com/korczis/apollo-crawler
446
412
  licenses:
447
413
  - MIT
414
+ metadata: {}
448
415
  post_install_message: Thanks for installing Apollo Crawler!
449
416
  rdoc_options: []
450
417
  require_paths:
451
418
  - lib
452
419
  required_ruby_version: !ruby/object:Gem::Requirement
453
- none: false
454
420
  requirements:
455
421
  - - ! '>='
456
422
  - !ruby/object:Gem::Version
457
- version: '0'
423
+ version: 1.9.3
458
424
  required_rubygems_version: !ruby/object:Gem::Requirement
459
- none: false
460
425
  requirements:
461
426
  - - ! '>='
462
427
  - !ruby/object:Gem::Version
463
- version: '0'
428
+ version: 1.8.11
464
429
  requirements: []
465
430
  rubyforge_project:
466
- rubygems_version: 1.8.23
431
+ rubygems_version: 2.0.0
467
432
  signing_key:
468
- specification_version: 3
433
+ specification_version: 4
469
434
  summary: Apollo Platform Crawler
470
435
  test_files: []