apollo-crawler 0.1.2 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +15 -0
- data/bin/apollo-crawler +404 -404
- data/lib/apollo_crawler.rb +20 -20
- data/lib/apollo_crawler/cache.rb +34 -34
- data/lib/apollo_crawler/caches/factory.rb +18 -18
- data/lib/apollo_crawler/caches/filesystem_cache.rb +30 -30
- data/lib/apollo_crawler/caches/memory_cache.rb +43 -43
- data/lib/apollo_crawler/caches/null_cache.rb +30 -30
- data/lib/apollo_crawler/crawler.rb +128 -128
- data/lib/apollo_crawler/crawler_template.rb +24 -24
- data/lib/apollo_crawler/crawlers/alexa_com/alexa.rb +26 -26
- data/lib/apollo_crawler/crawlers/firmy_cz/firmy.rb +26 -26
- data/lib/apollo_crawler/crawlers/google_com/google.rb +26 -26
- data/lib/apollo_crawler/crawlers/slashdot_org/slashdot.rb +26 -26
- data/lib/apollo_crawler/crawlers/stackoverflow_com/stackoverflow.rb +26 -26
- data/lib/apollo_crawler/crawlers/xkcd_com/xkcd.rb +35 -35
- data/lib/apollo_crawler/crawlers/ycombinator_com/hacker_news.rb +26 -26
- data/lib/apollo_crawler/formatter.rb +6 -6
- data/lib/apollo_crawler/formatters/formatter_json.rb +17 -17
- data/lib/apollo_crawler/formatters/formatter_plain.rb +17 -17
- data/lib/apollo_crawler/formatters/formatter_table.rb +33 -33
- data/lib/apollo_crawler/version.rb +2 -2
- metadata +16 -44
@@ -1,17 +1,17 @@
|
|
1
|
-
require 'json'
|
2
|
-
|
3
|
-
require File.join(File.dirname(__FILE__), '..', 'formatter')
|
4
|
-
|
5
|
-
module Apollo
|
6
|
-
module Formatters
|
7
|
-
class Json < Formatter
|
8
|
-
def format(obj)
|
9
|
-
return Json.format(obj)
|
10
|
-
end
|
11
|
-
|
12
|
-
def self.format(obj)
|
13
|
-
return JSON.pretty_generate(obj)
|
14
|
-
end
|
15
|
-
end
|
16
|
-
end # Formatters
|
17
|
-
end # Apollo
|
1
|
+
require 'json'
|
2
|
+
|
3
|
+
require File.join(File.dirname(__FILE__), '..', 'formatter')
|
4
|
+
|
5
|
+
module Apollo
|
6
|
+
module Formatters
|
7
|
+
class Json < Formatter
|
8
|
+
def format(obj)
|
9
|
+
return Json.format(obj)
|
10
|
+
end
|
11
|
+
|
12
|
+
def self.format(obj)
|
13
|
+
return JSON.pretty_generate(obj)
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end # Formatters
|
17
|
+
end # Apollo
|
@@ -1,17 +1,17 @@
|
|
1
|
-
require 'awesome_print'
|
2
|
-
|
3
|
-
require File.join(File.dirname(__FILE__), '..', 'formatter')
|
4
|
-
|
5
|
-
module Apollo
|
6
|
-
module Formatters
|
7
|
-
class Plain < Formatter
|
8
|
-
def format(obj)
|
9
|
-
return Plain.format(obj)
|
10
|
-
end
|
11
|
-
|
12
|
-
def self.format(obj)
|
13
|
-
return obj.inspect
|
14
|
-
end
|
15
|
-
end
|
16
|
-
end # Formatters
|
17
|
-
end # Apollo
|
1
|
+
require 'awesome_print'
|
2
|
+
|
3
|
+
require File.join(File.dirname(__FILE__), '..', 'formatter')
|
4
|
+
|
5
|
+
module Apollo
|
6
|
+
module Formatters
|
7
|
+
class Plain < Formatter
|
8
|
+
def format(obj)
|
9
|
+
return Plain.format(obj)
|
10
|
+
end
|
11
|
+
|
12
|
+
def self.format(obj)
|
13
|
+
return obj.inspect
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end # Formatters
|
17
|
+
end # Apollo
|
@@ -1,33 +1,33 @@
|
|
1
|
-
require 'terminal-table'
|
2
|
-
|
3
|
-
require File.join(File.dirname(__FILE__), '..', 'formatter')
|
4
|
-
|
5
|
-
module Apollo
|
6
|
-
module Formatters
|
7
|
-
class Table < Formatter
|
8
|
-
def format(obj)
|
9
|
-
return Table.format(obj)
|
10
|
-
end
|
11
|
-
|
12
|
-
def self.format(obj)
|
13
|
-
headings = []
|
14
|
-
if(obj[:data].length > 0)
|
15
|
-
headings = obj[:data][0].keys
|
16
|
-
end
|
17
|
-
|
18
|
-
rows = []
|
19
|
-
obj[:data].each do |line|
|
20
|
-
data = []
|
21
|
-
headings.each do |column|
|
22
|
-
data << line[column]
|
23
|
-
end
|
24
|
-
|
25
|
-
rows << data
|
26
|
-
end
|
27
|
-
|
28
|
-
table = Terminal::Table.new :headings => headings, :rows => rows
|
29
|
-
return table
|
30
|
-
end
|
31
|
-
end
|
32
|
-
end # Formatters
|
33
|
-
end # Apollo
|
1
|
+
require 'terminal-table'
|
2
|
+
|
3
|
+
require File.join(File.dirname(__FILE__), '..', 'formatter')
|
4
|
+
|
5
|
+
module Apollo
|
6
|
+
module Formatters
|
7
|
+
class Table < Formatter
|
8
|
+
def format(obj)
|
9
|
+
return Table.format(obj)
|
10
|
+
end
|
11
|
+
|
12
|
+
def self.format(obj)
|
13
|
+
headings = []
|
14
|
+
if(obj[:data].length > 0)
|
15
|
+
headings = obj[:data][0].keys
|
16
|
+
end
|
17
|
+
|
18
|
+
rows = []
|
19
|
+
obj[:data].each do |line|
|
20
|
+
data = []
|
21
|
+
headings.each do |column|
|
22
|
+
data << line[column]
|
23
|
+
end
|
24
|
+
|
25
|
+
rows << data
|
26
|
+
end
|
27
|
+
|
28
|
+
table = Terminal::Table.new :headings => headings, :rows => rows
|
29
|
+
return table
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end # Formatters
|
33
|
+
end # Apollo
|
@@ -1,3 +1,3 @@
|
|
1
|
-
module Apollo
|
2
|
-
VERSION = '0.1.
|
1
|
+
module Apollo
|
2
|
+
VERSION = '0.1.3'
|
3
3
|
end # Apollo
|
metadata
CHANGED
@@ -1,8 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: apollo-crawler
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
5
|
-
prerelease:
|
4
|
+
version: 0.1.3
|
6
5
|
platform: ruby
|
7
6
|
authors:
|
8
7
|
- Tomas Korcak
|
@@ -14,7 +13,6 @@ dependencies:
|
|
14
13
|
- !ruby/object:Gem::Dependency
|
15
14
|
name: amqp
|
16
15
|
requirement: !ruby/object:Gem::Requirement
|
17
|
-
none: false
|
18
16
|
requirements:
|
19
17
|
- - ! '>='
|
20
18
|
- !ruby/object:Gem::Version
|
@@ -22,7 +20,6 @@ dependencies:
|
|
22
20
|
type: :runtime
|
23
21
|
prerelease: false
|
24
22
|
version_requirements: !ruby/object:Gem::Requirement
|
25
|
-
none: false
|
26
23
|
requirements:
|
27
24
|
- - ! '>='
|
28
25
|
- !ruby/object:Gem::Version
|
@@ -30,7 +27,6 @@ dependencies:
|
|
30
27
|
- !ruby/object:Gem::Dependency
|
31
28
|
name: awesome_print
|
32
29
|
requirement: !ruby/object:Gem::Requirement
|
33
|
-
none: false
|
34
30
|
requirements:
|
35
31
|
- - ! '>='
|
36
32
|
- !ruby/object:Gem::Version
|
@@ -38,7 +34,6 @@ dependencies:
|
|
38
34
|
type: :runtime
|
39
35
|
prerelease: false
|
40
36
|
version_requirements: !ruby/object:Gem::Requirement
|
41
|
-
none: false
|
42
37
|
requirements:
|
43
38
|
- - ! '>='
|
44
39
|
- !ruby/object:Gem::Version
|
@@ -46,7 +41,6 @@ dependencies:
|
|
46
41
|
- !ruby/object:Gem::Dependency
|
47
42
|
name: active_support
|
48
43
|
requirement: !ruby/object:Gem::Requirement
|
49
|
-
none: false
|
50
44
|
requirements:
|
51
45
|
- - ! '>='
|
52
46
|
- !ruby/object:Gem::Version
|
@@ -54,7 +48,6 @@ dependencies:
|
|
54
48
|
type: :runtime
|
55
49
|
prerelease: false
|
56
50
|
version_requirements: !ruby/object:Gem::Requirement
|
57
|
-
none: false
|
58
51
|
requirements:
|
59
52
|
- - ! '>='
|
60
53
|
- !ruby/object:Gem::Version
|
@@ -62,7 +55,6 @@ dependencies:
|
|
62
55
|
- !ruby/object:Gem::Dependency
|
63
56
|
name: fastercsv
|
64
57
|
requirement: !ruby/object:Gem::Requirement
|
65
|
-
none: false
|
66
58
|
requirements:
|
67
59
|
- - ~>
|
68
60
|
- !ruby/object:Gem::Version
|
@@ -70,7 +62,6 @@ dependencies:
|
|
70
62
|
type: :runtime
|
71
63
|
prerelease: false
|
72
64
|
version_requirements: !ruby/object:Gem::Requirement
|
73
|
-
none: false
|
74
65
|
requirements:
|
75
66
|
- - ~>
|
76
67
|
- !ruby/object:Gem::Version
|
@@ -78,7 +69,6 @@ dependencies:
|
|
78
69
|
- !ruby/object:Gem::Dependency
|
79
70
|
name: eventmachine
|
80
71
|
requirement: !ruby/object:Gem::Requirement
|
81
|
-
none: false
|
82
72
|
requirements:
|
83
73
|
- - ! '>='
|
84
74
|
- !ruby/object:Gem::Version
|
@@ -86,7 +76,6 @@ dependencies:
|
|
86
76
|
type: :runtime
|
87
77
|
prerelease: false
|
88
78
|
version_requirements: !ruby/object:Gem::Requirement
|
89
|
-
none: false
|
90
79
|
requirements:
|
91
80
|
- - ! '>='
|
92
81
|
- !ruby/object:Gem::Version
|
@@ -94,7 +83,6 @@ dependencies:
|
|
94
83
|
- !ruby/object:Gem::Dependency
|
95
84
|
name: writeexcel
|
96
85
|
requirement: !ruby/object:Gem::Requirement
|
97
|
-
none: false
|
98
86
|
requirements:
|
99
87
|
- - ! '>='
|
100
88
|
- !ruby/object:Gem::Version
|
@@ -102,7 +90,6 @@ dependencies:
|
|
102
90
|
type: :runtime
|
103
91
|
prerelease: false
|
104
92
|
version_requirements: !ruby/object:Gem::Requirement
|
105
|
-
none: false
|
106
93
|
requirements:
|
107
94
|
- - ! '>='
|
108
95
|
- !ruby/object:Gem::Version
|
@@ -110,7 +97,6 @@ dependencies:
|
|
110
97
|
- !ruby/object:Gem::Dependency
|
111
98
|
name: json
|
112
99
|
requirement: !ruby/object:Gem::Requirement
|
113
|
-
none: false
|
114
100
|
requirements:
|
115
101
|
- - ! '>='
|
116
102
|
- !ruby/object:Gem::Version
|
@@ -118,7 +104,6 @@ dependencies:
|
|
118
104
|
type: :runtime
|
119
105
|
prerelease: false
|
120
106
|
version_requirements: !ruby/object:Gem::Requirement
|
121
|
-
none: false
|
122
107
|
requirements:
|
123
108
|
- - ! '>='
|
124
109
|
- !ruby/object:Gem::Version
|
@@ -126,7 +111,6 @@ dependencies:
|
|
126
111
|
- !ruby/object:Gem::Dependency
|
127
112
|
name: mime-types
|
128
113
|
requirement: !ruby/object:Gem::Requirement
|
129
|
-
none: false
|
130
114
|
requirements:
|
131
115
|
- - ! '>='
|
132
116
|
- !ruby/object:Gem::Version
|
@@ -134,7 +118,6 @@ dependencies:
|
|
134
118
|
type: :runtime
|
135
119
|
prerelease: false
|
136
120
|
version_requirements: !ruby/object:Gem::Requirement
|
137
|
-
none: false
|
138
121
|
requirements:
|
139
122
|
- - ! '>='
|
140
123
|
- !ruby/object:Gem::Version
|
@@ -142,7 +125,6 @@ dependencies:
|
|
142
125
|
- !ruby/object:Gem::Dependency
|
143
126
|
name: nokogiri
|
144
127
|
requirement: !ruby/object:Gem::Requirement
|
145
|
-
none: false
|
146
128
|
requirements:
|
147
129
|
- - ! '>='
|
148
130
|
- !ruby/object:Gem::Version
|
@@ -150,7 +132,6 @@ dependencies:
|
|
150
132
|
type: :runtime
|
151
133
|
prerelease: false
|
152
134
|
version_requirements: !ruby/object:Gem::Requirement
|
153
|
-
none: false
|
154
135
|
requirements:
|
155
136
|
- - ! '>='
|
156
137
|
- !ruby/object:Gem::Version
|
@@ -158,7 +139,6 @@ dependencies:
|
|
158
139
|
- !ruby/object:Gem::Dependency
|
159
140
|
name: openurl
|
160
141
|
requirement: !ruby/object:Gem::Requirement
|
161
|
-
none: false
|
162
142
|
requirements:
|
163
143
|
- - ! '>='
|
164
144
|
- !ruby/object:Gem::Version
|
@@ -166,7 +146,6 @@ dependencies:
|
|
166
146
|
type: :runtime
|
167
147
|
prerelease: false
|
168
148
|
version_requirements: !ruby/object:Gem::Requirement
|
169
|
-
none: false
|
170
149
|
requirements:
|
171
150
|
- - ! '>='
|
172
151
|
- !ruby/object:Gem::Version
|
@@ -174,7 +153,6 @@ dependencies:
|
|
174
153
|
- !ruby/object:Gem::Dependency
|
175
154
|
name: parallel
|
176
155
|
requirement: !ruby/object:Gem::Requirement
|
177
|
-
none: false
|
178
156
|
requirements:
|
179
157
|
- - ! '>='
|
180
158
|
- !ruby/object:Gem::Version
|
@@ -182,7 +160,6 @@ dependencies:
|
|
182
160
|
type: :runtime
|
183
161
|
prerelease: false
|
184
162
|
version_requirements: !ruby/object:Gem::Requirement
|
185
|
-
none: false
|
186
163
|
requirements:
|
187
164
|
- - ! '>='
|
188
165
|
- !ruby/object:Gem::Version
|
@@ -190,7 +167,6 @@ dependencies:
|
|
190
167
|
- !ruby/object:Gem::Dependency
|
191
168
|
name: terminal-table
|
192
169
|
requirement: !ruby/object:Gem::Requirement
|
193
|
-
none: false
|
194
170
|
requirements:
|
195
171
|
- - ! '>='
|
196
172
|
- !ruby/object:Gem::Version
|
@@ -198,7 +174,6 @@ dependencies:
|
|
198
174
|
type: :runtime
|
199
175
|
prerelease: false
|
200
176
|
version_requirements: !ruby/object:Gem::Requirement
|
201
|
-
none: false
|
202
177
|
requirements:
|
203
178
|
- - ! '>='
|
204
179
|
- !ruby/object:Gem::Version
|
@@ -206,7 +181,6 @@ dependencies:
|
|
206
181
|
- !ruby/object:Gem::Dependency
|
207
182
|
name: thor
|
208
183
|
requirement: !ruby/object:Gem::Requirement
|
209
|
-
none: false
|
210
184
|
requirements:
|
211
185
|
- - ! '>='
|
212
186
|
- !ruby/object:Gem::Version
|
@@ -214,7 +188,6 @@ dependencies:
|
|
214
188
|
type: :runtime
|
215
189
|
prerelease: false
|
216
190
|
version_requirements: !ruby/object:Gem::Requirement
|
217
|
-
none: false
|
218
191
|
requirements:
|
219
192
|
- - ! '>='
|
220
193
|
- !ruby/object:Gem::Version
|
@@ -226,50 +199,49 @@ executables:
|
|
226
199
|
extensions: []
|
227
200
|
extra_rdoc_files: []
|
228
201
|
files:
|
229
|
-
- ./lib/apollo_crawler/
|
230
|
-
- ./lib/apollo_crawler/formatters/formatter_json.rb
|
231
|
-
- ./lib/apollo_crawler/formatters/formatter_table.rb
|
232
|
-
- ./lib/apollo_crawler/version.rb
|
202
|
+
- ./lib/apollo_crawler/cache.rb
|
233
203
|
- ./lib/apollo_crawler/caches/factory.rb
|
234
|
-
- ./lib/apollo_crawler/caches/null_cache.rb
|
235
|
-
- ./lib/apollo_crawler/caches/memory_cache.rb
|
236
204
|
- ./lib/apollo_crawler/caches/filesystem_cache.rb
|
237
|
-
- ./lib/apollo_crawler/
|
205
|
+
- ./lib/apollo_crawler/caches/memory_cache.rb
|
206
|
+
- ./lib/apollo_crawler/caches/null_cache.rb
|
238
207
|
- ./lib/apollo_crawler/crawler.rb
|
239
|
-
- ./lib/apollo_crawler/crawlers/
|
240
|
-
- ./lib/apollo_crawler/crawlers/
|
208
|
+
- ./lib/apollo_crawler/crawlers/alexa_com/alexa.rb
|
209
|
+
- ./lib/apollo_crawler/crawlers/firmy_cz/firmy.rb
|
241
210
|
- ./lib/apollo_crawler/crawlers/google_com/google.rb
|
242
211
|
- ./lib/apollo_crawler/crawlers/slashdot_org/slashdot.rb
|
243
|
-
- ./lib/apollo_crawler/crawlers/
|
244
|
-
- ./lib/apollo_crawler/crawlers/
|
212
|
+
- ./lib/apollo_crawler/crawlers/stackoverflow_com/stackoverflow.rb
|
213
|
+
- ./lib/apollo_crawler/crawlers/xkcd_com/xkcd.rb
|
245
214
|
- ./lib/apollo_crawler/crawlers/ycombinator_com/hacker_news.rb
|
215
|
+
- ./lib/apollo_crawler/crawler_template.rb
|
246
216
|
- ./lib/apollo_crawler/formatter.rb
|
247
|
-
- ./lib/apollo_crawler/
|
217
|
+
- ./lib/apollo_crawler/formatters/formatter_json.rb
|
218
|
+
- ./lib/apollo_crawler/formatters/formatter_plain.rb
|
219
|
+
- ./lib/apollo_crawler/formatters/formatter_table.rb
|
220
|
+
- ./lib/apollo_crawler/version.rb
|
248
221
|
- ./lib/apollo_crawler.rb
|
249
222
|
- bin/apollo-crawler
|
250
223
|
homepage: https://github.com/korczis/apollo-crawler
|
251
224
|
licenses:
|
252
225
|
- MIT
|
226
|
+
metadata: {}
|
253
227
|
post_install_message: Thanks for installing Apollo Crawler!
|
254
228
|
rdoc_options: []
|
255
229
|
require_paths:
|
256
230
|
- lib
|
257
231
|
required_ruby_version: !ruby/object:Gem::Requirement
|
258
|
-
none: false
|
259
232
|
requirements:
|
260
233
|
- - ! '>='
|
261
234
|
- !ruby/object:Gem::Version
|
262
235
|
version: '0'
|
263
236
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
264
|
-
none: false
|
265
237
|
requirements:
|
266
238
|
- - ! '>='
|
267
239
|
- !ruby/object:Gem::Version
|
268
240
|
version: '0'
|
269
241
|
requirements: []
|
270
242
|
rubyforge_project:
|
271
|
-
rubygems_version:
|
243
|
+
rubygems_version: 2.0.0
|
272
244
|
signing_key:
|
273
|
-
specification_version:
|
245
|
+
specification_version: 4
|
274
246
|
summary: Apollo Platform Crawler
|
275
247
|
test_files: []
|