apollo-crawler 0.1.5 → 0.1.6

Sign up to get free protection for your applications and to get access to all the features.
Files changed (30) hide show
  1. checksums.yaml +8 -8
  2. data/bin/apollo-crawler +12 -410
  3. data/lib/apollo_crawler.rb +31 -20
  4. data/lib/apollo_crawler/{cache.rb → cache/cache_base.rb} +37 -34
  5. data/lib/apollo_crawler/cache/factory.rb +35 -0
  6. data/lib/apollo_crawler/{caches → cache}/filesystem_cache.rb +37 -34
  7. data/lib/apollo_crawler/cache/memcached_cache.rb +51 -0
  8. data/lib/apollo_crawler/{caches → cache}/memory_cache.rb +46 -43
  9. data/lib/apollo_crawler/{caches → cache}/null_cache.rb +33 -30
  10. data/lib/apollo_crawler/config.rb +53 -0
  11. data/lib/apollo_crawler/{crawler.rb → crawler/crawler_base.rb} +157 -155
  12. data/lib/apollo_crawler/{crawler_template.rb → crawler/crawler_template.rb} +24 -24
  13. data/lib/apollo_crawler/{crawlers → crawler}/google_com/google.rb +40 -40
  14. data/lib/apollo_crawler/{crawlers → crawler}/slashdot_org/slashdot.rb +40 -40
  15. data/lib/apollo_crawler/{crawlers → crawler}/stackoverflow_com/stackoverflow.rb +44 -44
  16. data/lib/apollo_crawler/{crawlers → crawler}/xkcd_com/xkcd.rb +35 -35
  17. data/lib/apollo_crawler/{crawlers → crawler}/ycombinator_com/hacker_news.rb +44 -44
  18. data/lib/apollo_crawler/fetcher/fetcher_base.rb +6 -0
  19. data/lib/apollo_crawler/fetcher/simple_fetcher.rb +8 -0
  20. data/lib/apollo_crawler/formatter/formatter_base.rb +6 -0
  21. data/lib/apollo_crawler/{formatters → formatter}/formatter_json.rb +17 -17
  22. data/lib/apollo_crawler/{formatters → formatter}/formatter_plain.rb +17 -17
  23. data/lib/apollo_crawler/{formatters → formatter}/formatter_table.rb +35 -35
  24. data/lib/apollo_crawler/lib.rb +28 -0
  25. data/lib/apollo_crawler/program.rb +406 -0
  26. data/lib/apollo_crawler/store/store_base.rb +6 -0
  27. data/lib/apollo_crawler/version.rb +2 -2
  28. metadata +52 -17
  29. data/lib/apollo_crawler/caches/factory.rb +0 -30
  30. data/lib/apollo_crawler/formatter.rb +0 -6
@@ -0,0 +1,6 @@
1
+ module Apollo
2
+ module Store
3
+ class StoreBase
4
+ end # StoreBase
5
+ end # Store
6
+ end # Apollo
@@ -1,3 +1,3 @@
1
- module Apollo
2
- VERSION = '0.1.5'
1
+ module Apollo
2
+ VERSION = '0.1.6'
3
3
  end # Apollo
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: apollo-crawler
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.5
4
+ version: 0.1.6
5
5
  platform: ruby
6
6
  authors:
7
7
  - Tomas Korcak
@@ -52,6 +52,20 @@ dependencies:
52
52
  - - ! '>='
53
53
  - !ruby/object:Gem::Version
54
54
  version: '0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: dalli
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - ! '>='
60
+ - !ruby/object:Gem::Version
61
+ version: '0'
62
+ type: :runtime
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - ! '>='
67
+ - !ruby/object:Gem::Version
68
+ version: '0'
55
69
  - !ruby/object:Gem::Dependency
56
70
  name: fastercsv
57
71
  requirement: !ruby/object:Gem::Requirement
@@ -108,6 +122,20 @@ dependencies:
108
122
  - - ! '>='
109
123
  - !ruby/object:Gem::Version
110
124
  version: '0'
125
+ - !ruby/object:Gem::Dependency
126
+ name: memcached
127
+ requirement: !ruby/object:Gem::Requirement
128
+ requirements:
129
+ - - ! '>='
130
+ - !ruby/object:Gem::Version
131
+ version: '0'
132
+ type: :runtime
133
+ prerelease: false
134
+ version_requirements: !ruby/object:Gem::Requirement
135
+ requirements:
136
+ - - ! '>='
137
+ - !ruby/object:Gem::Version
138
+ version: '0'
111
139
  - !ruby/object:Gem::Dependency
112
140
  name: mime-types
113
141
  requirement: !ruby/object:Gem::Requirement
@@ -199,23 +227,30 @@ executables:
199
227
  extensions: []
200
228
  extra_rdoc_files: []
201
229
  files:
202
- - ./lib/apollo_crawler/cache.rb
203
- - ./lib/apollo_crawler/caches/factory.rb
204
- - ./lib/apollo_crawler/caches/filesystem_cache.rb
205
- - ./lib/apollo_crawler/caches/memory_cache.rb
206
- - ./lib/apollo_crawler/caches/null_cache.rb
207
- - ./lib/apollo_crawler/crawler.rb
208
- - ./lib/apollo_crawler/crawlers/google_com/google.rb
209
- - ./lib/apollo_crawler/crawlers/slashdot_org/slashdot.rb
210
- - ./lib/apollo_crawler/crawlers/stackoverflow_com/stackoverflow.rb
211
- - ./lib/apollo_crawler/crawlers/xkcd_com/xkcd.rb
212
- - ./lib/apollo_crawler/crawlers/ycombinator_com/hacker_news.rb
213
- - ./lib/apollo_crawler/crawler_template.rb
214
- - ./lib/apollo_crawler/formatter.rb
215
- - ./lib/apollo_crawler/formatters/formatter_json.rb
216
- - ./lib/apollo_crawler/formatters/formatter_plain.rb
217
- - ./lib/apollo_crawler/formatters/formatter_table.rb
230
+ - ./lib/apollo_crawler/fetcher/simple_fetcher.rb
231
+ - ./lib/apollo_crawler/fetcher/fetcher_base.rb
232
+ - ./lib/apollo_crawler/lib.rb
218
233
  - ./lib/apollo_crawler/version.rb
234
+ - ./lib/apollo_crawler/program.rb
235
+ - ./lib/apollo_crawler/config.rb
236
+ - ./lib/apollo_crawler/cache/factory.rb
237
+ - ./lib/apollo_crawler/cache/null_cache.rb
238
+ - ./lib/apollo_crawler/cache/cache_base.rb
239
+ - ./lib/apollo_crawler/cache/memory_cache.rb
240
+ - ./lib/apollo_crawler/cache/filesystem_cache.rb
241
+ - ./lib/apollo_crawler/cache/memcached_cache.rb
242
+ - ./lib/apollo_crawler/crawler/crawler_template.rb
243
+ - ./lib/apollo_crawler/crawler/stackoverflow_com/stackoverflow.rb
244
+ - ./lib/apollo_crawler/crawler/xkcd_com/xkcd.rb
245
+ - ./lib/apollo_crawler/crawler/google_com/google.rb
246
+ - ./lib/apollo_crawler/crawler/crawler_base.rb
247
+ - ./lib/apollo_crawler/crawler/slashdot_org/slashdot.rb
248
+ - ./lib/apollo_crawler/crawler/ycombinator_com/hacker_news.rb
249
+ - ./lib/apollo_crawler/formatter/formatter_base.rb
250
+ - ./lib/apollo_crawler/formatter/formatter_plain.rb
251
+ - ./lib/apollo_crawler/formatter/formatter_json.rb
252
+ - ./lib/apollo_crawler/formatter/formatter_table.rb
253
+ - ./lib/apollo_crawler/store/store_base.rb
219
254
  - ./lib/apollo_crawler.rb
220
255
  - bin/apollo-crawler
221
256
  homepage: https://github.com/korczis/apollo-crawler
@@ -1,30 +0,0 @@
1
- require File.join(File.dirname(__FILE__), '..', 'cache')
2
- require 'singleton'
3
-
4
- module Apollo
5
- module Caches
6
- class Factory
7
- include Singleton
8
-
9
- def self.construct()
10
- self.singleton.construct()
11
- end
12
-
13
- def construct()
14
- # Basic implementation
15
- # res = Cache.new()
16
-
17
- # Filesystem implementation
18
- res = Filesystem.new()
19
-
20
- # In-memory implementation
21
- # res = Memory.new()
22
-
23
- # Null (Dummy) implementation
24
- # res = Null.new()
25
-
26
- return res
27
- end
28
- end # Factory
29
- end # Caches
30
- end # Apollo
@@ -1,6 +0,0 @@
1
- module Apollo
2
- module Formatters
3
- class Formatter
4
- end # Formatter
5
- end # Formatters
6
- end # Apollo