apollo-crawler 0.1.5 → 0.1.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. checksums.yaml +8 -8
  2. data/bin/apollo-crawler +12 -410
  3. data/lib/apollo_crawler.rb +31 -20
  4. data/lib/apollo_crawler/{cache.rb → cache/cache_base.rb} +37 -34
  5. data/lib/apollo_crawler/cache/factory.rb +35 -0
  6. data/lib/apollo_crawler/{caches → cache}/filesystem_cache.rb +37 -34
  7. data/lib/apollo_crawler/cache/memcached_cache.rb +51 -0
  8. data/lib/apollo_crawler/{caches → cache}/memory_cache.rb +46 -43
  9. data/lib/apollo_crawler/{caches → cache}/null_cache.rb +33 -30
  10. data/lib/apollo_crawler/config.rb +53 -0
  11. data/lib/apollo_crawler/{crawler.rb → crawler/crawler_base.rb} +157 -155
  12. data/lib/apollo_crawler/{crawler_template.rb → crawler/crawler_template.rb} +24 -24
  13. data/lib/apollo_crawler/{crawlers → crawler}/google_com/google.rb +40 -40
  14. data/lib/apollo_crawler/{crawlers → crawler}/slashdot_org/slashdot.rb +40 -40
  15. data/lib/apollo_crawler/{crawlers → crawler}/stackoverflow_com/stackoverflow.rb +44 -44
  16. data/lib/apollo_crawler/{crawlers → crawler}/xkcd_com/xkcd.rb +35 -35
  17. data/lib/apollo_crawler/{crawlers → crawler}/ycombinator_com/hacker_news.rb +44 -44
  18. data/lib/apollo_crawler/fetcher/fetcher_base.rb +6 -0
  19. data/lib/apollo_crawler/fetcher/simple_fetcher.rb +8 -0
  20. data/lib/apollo_crawler/formatter/formatter_base.rb +6 -0
  21. data/lib/apollo_crawler/{formatters → formatter}/formatter_json.rb +17 -17
  22. data/lib/apollo_crawler/{formatters → formatter}/formatter_plain.rb +17 -17
  23. data/lib/apollo_crawler/{formatters → formatter}/formatter_table.rb +35 -35
  24. data/lib/apollo_crawler/lib.rb +28 -0
  25. data/lib/apollo_crawler/program.rb +406 -0
  26. data/lib/apollo_crawler/store/store_base.rb +6 -0
  27. data/lib/apollo_crawler/version.rb +2 -2
  28. metadata +52 -17
  29. data/lib/apollo_crawler/caches/factory.rb +0 -30
  30. data/lib/apollo_crawler/formatter.rb +0 -6
@@ -0,0 +1,6 @@
1
+ module Apollo
2
+ module Store
3
+ class StoreBase
4
+ end # StoreBase
5
+ end # Store
6
+ end # Apollo
@@ -1,3 +1,3 @@
1
- module Apollo
2
- VERSION = '0.1.5'
1
+ module Apollo
2
+ VERSION = '0.1.6'
3
3
  end # Apollo
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: apollo-crawler
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.5
4
+ version: 0.1.6
5
5
  platform: ruby
6
6
  authors:
7
7
  - Tomas Korcak
@@ -52,6 +52,20 @@ dependencies:
52
52
  - - ! '>='
53
53
  - !ruby/object:Gem::Version
54
54
  version: '0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: dalli
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - ! '>='
60
+ - !ruby/object:Gem::Version
61
+ version: '0'
62
+ type: :runtime
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - ! '>='
67
+ - !ruby/object:Gem::Version
68
+ version: '0'
55
69
  - !ruby/object:Gem::Dependency
56
70
  name: fastercsv
57
71
  requirement: !ruby/object:Gem::Requirement
@@ -108,6 +122,20 @@ dependencies:
108
122
  - - ! '>='
109
123
  - !ruby/object:Gem::Version
110
124
  version: '0'
125
+ - !ruby/object:Gem::Dependency
126
+ name: memcached
127
+ requirement: !ruby/object:Gem::Requirement
128
+ requirements:
129
+ - - ! '>='
130
+ - !ruby/object:Gem::Version
131
+ version: '0'
132
+ type: :runtime
133
+ prerelease: false
134
+ version_requirements: !ruby/object:Gem::Requirement
135
+ requirements:
136
+ - - ! '>='
137
+ - !ruby/object:Gem::Version
138
+ version: '0'
111
139
  - !ruby/object:Gem::Dependency
112
140
  name: mime-types
113
141
  requirement: !ruby/object:Gem::Requirement
@@ -199,23 +227,30 @@ executables:
199
227
  extensions: []
200
228
  extra_rdoc_files: []
201
229
  files:
202
- - ./lib/apollo_crawler/cache.rb
203
- - ./lib/apollo_crawler/caches/factory.rb
204
- - ./lib/apollo_crawler/caches/filesystem_cache.rb
205
- - ./lib/apollo_crawler/caches/memory_cache.rb
206
- - ./lib/apollo_crawler/caches/null_cache.rb
207
- - ./lib/apollo_crawler/crawler.rb
208
- - ./lib/apollo_crawler/crawlers/google_com/google.rb
209
- - ./lib/apollo_crawler/crawlers/slashdot_org/slashdot.rb
210
- - ./lib/apollo_crawler/crawlers/stackoverflow_com/stackoverflow.rb
211
- - ./lib/apollo_crawler/crawlers/xkcd_com/xkcd.rb
212
- - ./lib/apollo_crawler/crawlers/ycombinator_com/hacker_news.rb
213
- - ./lib/apollo_crawler/crawler_template.rb
214
- - ./lib/apollo_crawler/formatter.rb
215
- - ./lib/apollo_crawler/formatters/formatter_json.rb
216
- - ./lib/apollo_crawler/formatters/formatter_plain.rb
217
- - ./lib/apollo_crawler/formatters/formatter_table.rb
230
+ - ./lib/apollo_crawler/fetcher/simple_fetcher.rb
231
+ - ./lib/apollo_crawler/fetcher/fetcher_base.rb
232
+ - ./lib/apollo_crawler/lib.rb
218
233
  - ./lib/apollo_crawler/version.rb
234
+ - ./lib/apollo_crawler/program.rb
235
+ - ./lib/apollo_crawler/config.rb
236
+ - ./lib/apollo_crawler/cache/factory.rb
237
+ - ./lib/apollo_crawler/cache/null_cache.rb
238
+ - ./lib/apollo_crawler/cache/cache_base.rb
239
+ - ./lib/apollo_crawler/cache/memory_cache.rb
240
+ - ./lib/apollo_crawler/cache/filesystem_cache.rb
241
+ - ./lib/apollo_crawler/cache/memcached_cache.rb
242
+ - ./lib/apollo_crawler/crawler/crawler_template.rb
243
+ - ./lib/apollo_crawler/crawler/stackoverflow_com/stackoverflow.rb
244
+ - ./lib/apollo_crawler/crawler/xkcd_com/xkcd.rb
245
+ - ./lib/apollo_crawler/crawler/google_com/google.rb
246
+ - ./lib/apollo_crawler/crawler/crawler_base.rb
247
+ - ./lib/apollo_crawler/crawler/slashdot_org/slashdot.rb
248
+ - ./lib/apollo_crawler/crawler/ycombinator_com/hacker_news.rb
249
+ - ./lib/apollo_crawler/formatter/formatter_base.rb
250
+ - ./lib/apollo_crawler/formatter/formatter_plain.rb
251
+ - ./lib/apollo_crawler/formatter/formatter_json.rb
252
+ - ./lib/apollo_crawler/formatter/formatter_table.rb
253
+ - ./lib/apollo_crawler/store/store_base.rb
219
254
  - ./lib/apollo_crawler.rb
220
255
  - bin/apollo-crawler
221
256
  homepage: https://github.com/korczis/apollo-crawler
@@ -1,30 +0,0 @@
1
- require File.join(File.dirname(__FILE__), '..', 'cache')
2
- require 'singleton'
3
-
4
- module Apollo
5
- module Caches
6
- class Factory
7
- include Singleton
8
-
9
- def self.construct()
10
- self.singleton.construct()
11
- end
12
-
13
- def construct()
14
- # Basic implementation
15
- # res = Cache.new()
16
-
17
- # Filesystem implementation
18
- res = Filesystem.new()
19
-
20
- # In-memory implementation
21
- # res = Memory.new()
22
-
23
- # Null (Dummy) implementation
24
- # res = Null.new()
25
-
26
- return res
27
- end
28
- end # Factory
29
- end # Caches
30
- end # Apollo
@@ -1,6 +0,0 @@
1
- module Apollo
2
- module Formatters
3
- class Formatter
4
- end # Formatter
5
- end # Formatters
6
- end # Apollo