apollo-crawler 0.1.5 → 0.1.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +8 -8
- data/bin/apollo-crawler +12 -410
- data/lib/apollo_crawler.rb +31 -20
- data/lib/apollo_crawler/{cache.rb → cache/cache_base.rb} +37 -34
- data/lib/apollo_crawler/cache/factory.rb +35 -0
- data/lib/apollo_crawler/{caches → cache}/filesystem_cache.rb +37 -34
- data/lib/apollo_crawler/cache/memcached_cache.rb +51 -0
- data/lib/apollo_crawler/{caches → cache}/memory_cache.rb +46 -43
- data/lib/apollo_crawler/{caches → cache}/null_cache.rb +33 -30
- data/lib/apollo_crawler/config.rb +53 -0
- data/lib/apollo_crawler/{crawler.rb → crawler/crawler_base.rb} +157 -155
- data/lib/apollo_crawler/{crawler_template.rb → crawler/crawler_template.rb} +24 -24
- data/lib/apollo_crawler/{crawlers → crawler}/google_com/google.rb +40 -40
- data/lib/apollo_crawler/{crawlers → crawler}/slashdot_org/slashdot.rb +40 -40
- data/lib/apollo_crawler/{crawlers → crawler}/stackoverflow_com/stackoverflow.rb +44 -44
- data/lib/apollo_crawler/{crawlers → crawler}/xkcd_com/xkcd.rb +35 -35
- data/lib/apollo_crawler/{crawlers → crawler}/ycombinator_com/hacker_news.rb +44 -44
- data/lib/apollo_crawler/fetcher/fetcher_base.rb +6 -0
- data/lib/apollo_crawler/fetcher/simple_fetcher.rb +8 -0
- data/lib/apollo_crawler/formatter/formatter_base.rb +6 -0
- data/lib/apollo_crawler/{formatters → formatter}/formatter_json.rb +17 -17
- data/lib/apollo_crawler/{formatters → formatter}/formatter_plain.rb +17 -17
- data/lib/apollo_crawler/{formatters → formatter}/formatter_table.rb +35 -35
- data/lib/apollo_crawler/lib.rb +28 -0
- data/lib/apollo_crawler/program.rb +406 -0
- data/lib/apollo_crawler/store/store_base.rb +6 -0
- data/lib/apollo_crawler/version.rb +2 -2
- metadata +52 -17
- data/lib/apollo_crawler/caches/factory.rb +0 -30
- data/lib/apollo_crawler/formatter.rb +0 -6
@@ -1,3 +1,3 @@
|
|
1
|
-
module Apollo
|
2
|
-
VERSION = '0.1.
|
1
|
+
module Apollo
|
2
|
+
VERSION = '0.1.6'
|
3
3
|
end # Apollo
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: apollo-crawler
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Tomas Korcak
|
@@ -52,6 +52,20 @@ dependencies:
|
|
52
52
|
- - ! '>='
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: dalli
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ! '>='
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ! '>='
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
55
69
|
- !ruby/object:Gem::Dependency
|
56
70
|
name: fastercsv
|
57
71
|
requirement: !ruby/object:Gem::Requirement
|
@@ -108,6 +122,20 @@ dependencies:
|
|
108
122
|
- - ! '>='
|
109
123
|
- !ruby/object:Gem::Version
|
110
124
|
version: '0'
|
125
|
+
- !ruby/object:Gem::Dependency
|
126
|
+
name: memcached
|
127
|
+
requirement: !ruby/object:Gem::Requirement
|
128
|
+
requirements:
|
129
|
+
- - ! '>='
|
130
|
+
- !ruby/object:Gem::Version
|
131
|
+
version: '0'
|
132
|
+
type: :runtime
|
133
|
+
prerelease: false
|
134
|
+
version_requirements: !ruby/object:Gem::Requirement
|
135
|
+
requirements:
|
136
|
+
- - ! '>='
|
137
|
+
- !ruby/object:Gem::Version
|
138
|
+
version: '0'
|
111
139
|
- !ruby/object:Gem::Dependency
|
112
140
|
name: mime-types
|
113
141
|
requirement: !ruby/object:Gem::Requirement
|
@@ -199,23 +227,30 @@ executables:
|
|
199
227
|
extensions: []
|
200
228
|
extra_rdoc_files: []
|
201
229
|
files:
|
202
|
-
- ./lib/apollo_crawler/
|
203
|
-
- ./lib/apollo_crawler/
|
204
|
-
- ./lib/apollo_crawler/
|
205
|
-
- ./lib/apollo_crawler/caches/memory_cache.rb
|
206
|
-
- ./lib/apollo_crawler/caches/null_cache.rb
|
207
|
-
- ./lib/apollo_crawler/crawler.rb
|
208
|
-
- ./lib/apollo_crawler/crawlers/google_com/google.rb
|
209
|
-
- ./lib/apollo_crawler/crawlers/slashdot_org/slashdot.rb
|
210
|
-
- ./lib/apollo_crawler/crawlers/stackoverflow_com/stackoverflow.rb
|
211
|
-
- ./lib/apollo_crawler/crawlers/xkcd_com/xkcd.rb
|
212
|
-
- ./lib/apollo_crawler/crawlers/ycombinator_com/hacker_news.rb
|
213
|
-
- ./lib/apollo_crawler/crawler_template.rb
|
214
|
-
- ./lib/apollo_crawler/formatter.rb
|
215
|
-
- ./lib/apollo_crawler/formatters/formatter_json.rb
|
216
|
-
- ./lib/apollo_crawler/formatters/formatter_plain.rb
|
217
|
-
- ./lib/apollo_crawler/formatters/formatter_table.rb
|
230
|
+
- ./lib/apollo_crawler/fetcher/simple_fetcher.rb
|
231
|
+
- ./lib/apollo_crawler/fetcher/fetcher_base.rb
|
232
|
+
- ./lib/apollo_crawler/lib.rb
|
218
233
|
- ./lib/apollo_crawler/version.rb
|
234
|
+
- ./lib/apollo_crawler/program.rb
|
235
|
+
- ./lib/apollo_crawler/config.rb
|
236
|
+
- ./lib/apollo_crawler/cache/factory.rb
|
237
|
+
- ./lib/apollo_crawler/cache/null_cache.rb
|
238
|
+
- ./lib/apollo_crawler/cache/cache_base.rb
|
239
|
+
- ./lib/apollo_crawler/cache/memory_cache.rb
|
240
|
+
- ./lib/apollo_crawler/cache/filesystem_cache.rb
|
241
|
+
- ./lib/apollo_crawler/cache/memcached_cache.rb
|
242
|
+
- ./lib/apollo_crawler/crawler/crawler_template.rb
|
243
|
+
- ./lib/apollo_crawler/crawler/stackoverflow_com/stackoverflow.rb
|
244
|
+
- ./lib/apollo_crawler/crawler/xkcd_com/xkcd.rb
|
245
|
+
- ./lib/apollo_crawler/crawler/google_com/google.rb
|
246
|
+
- ./lib/apollo_crawler/crawler/crawler_base.rb
|
247
|
+
- ./lib/apollo_crawler/crawler/slashdot_org/slashdot.rb
|
248
|
+
- ./lib/apollo_crawler/crawler/ycombinator_com/hacker_news.rb
|
249
|
+
- ./lib/apollo_crawler/formatter/formatter_base.rb
|
250
|
+
- ./lib/apollo_crawler/formatter/formatter_plain.rb
|
251
|
+
- ./lib/apollo_crawler/formatter/formatter_json.rb
|
252
|
+
- ./lib/apollo_crawler/formatter/formatter_table.rb
|
253
|
+
- ./lib/apollo_crawler/store/store_base.rb
|
219
254
|
- ./lib/apollo_crawler.rb
|
220
255
|
- bin/apollo-crawler
|
221
256
|
homepage: https://github.com/korczis/apollo-crawler
|
@@ -1,30 +0,0 @@
|
|
1
|
-
require File.join(File.dirname(__FILE__), '..', 'cache')
|
2
|
-
require 'singleton'
|
3
|
-
|
4
|
-
module Apollo
|
5
|
-
module Caches
|
6
|
-
class Factory
|
7
|
-
include Singleton
|
8
|
-
|
9
|
-
def self.construct()
|
10
|
-
self.singleton.construct()
|
11
|
-
end
|
12
|
-
|
13
|
-
def construct()
|
14
|
-
# Basic implementation
|
15
|
-
# res = Cache.new()
|
16
|
-
|
17
|
-
# Filesystem implementation
|
18
|
-
res = Filesystem.new()
|
19
|
-
|
20
|
-
# In-memory implementation
|
21
|
-
# res = Memory.new()
|
22
|
-
|
23
|
-
# Null (Dummy) implementation
|
24
|
-
# res = Null.new()
|
25
|
-
|
26
|
-
return res
|
27
|
-
end
|
28
|
-
end # Factory
|
29
|
-
end # Caches
|
30
|
-
end # Apollo
|