wukong 1.4.12 → 1.5.0
Sign up to get free protection for your applications and to get access to all the features.
- data/CHANGELOG.textile +20 -1
- data/bin/bootstrap.sh +32 -0
- data/bin/hdp-sort +3 -0
- data/bin/hdp-stream +3 -0
- data/docpages/README-elastic_map_reduce.textile +377 -0
- data/docpages/avro/avro_notes.textile +56 -0
- data/docpages/avro/tethering.textile +19 -0
- data/docpages/pig/commandline_params.txt +26 -0
- data/examples/emr/elastic_mapreduce_example.rb +35 -0
- data/lib/wukong/logger.rb +8 -1
- data/lib/wukong/script/avro_command.rb +5 -0
- data/lib/wukong/script/emr_command.rb +119 -0
- data/lib/wukong/script/hadoop_command.rb +72 -90
- data/lib/wukong/script/local_command.rb +18 -8
- data/lib/wukong/script.rb +87 -92
- data/wukong.gemspec +27 -18
- metadata +30 -21
data/wukong.gemspec
CHANGED
@@ -5,11 +5,11 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = %q{wukong}
|
8
|
-
s.version = "1.
|
8
|
+
s.version = "1.5.0"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["Philip (flip) Kromer"]
|
12
|
-
s.date = %q{2010-08-
|
12
|
+
s.date = %q{2010-08-10}
|
13
13
|
s.description = %q{ Treat your dataset like a:
|
14
14
|
|
15
15
|
* stream of lines when it's efficient to process by lines
|
@@ -30,6 +30,7 @@ Gem::Specification.new do |s|
|
|
30
30
|
"LICENSE.textile",
|
31
31
|
"README.textile",
|
32
32
|
"TODO.textile",
|
33
|
+
"bin/bootstrap.sh",
|
33
34
|
"bin/cutc",
|
34
35
|
"bin/cuttab",
|
35
36
|
"bin/greptrue",
|
@@ -65,12 +66,15 @@ Gem::Specification.new do |s|
|
|
65
66
|
"bin/wu-sum",
|
66
67
|
"docpages/INSTALL.textile",
|
67
68
|
"docpages/LICENSE.textile",
|
69
|
+
"docpages/README-elastic_map_reduce.textile",
|
68
70
|
"docpages/README-performance.textile",
|
69
71
|
"docpages/README-wulign.textile",
|
70
72
|
"docpages/UsingWukong-part1-get_ready.textile",
|
71
73
|
"docpages/UsingWukong-part2-ThinkingBigData.textile",
|
72
74
|
"docpages/UsingWukong-part3-parsing.textile",
|
73
75
|
"docpages/_config.yml",
|
76
|
+
"docpages/avro/avro_notes.textile",
|
77
|
+
"docpages/avro/tethering.textile",
|
74
78
|
"docpages/bigdata-tips.textile",
|
75
79
|
"docpages/code/api_response_example.txt",
|
76
80
|
"docpages/code/parser_skeleton.rb",
|
@@ -84,6 +88,7 @@ Gem::Specification.new do |s|
|
|
84
88
|
"docpages/news.html",
|
85
89
|
"docpages/pig/PigLatinExpressionsList.txt",
|
86
90
|
"docpages/pig/PigLatinReferenceManual.txt",
|
91
|
+
"docpages/pig/commandline_params.txt",
|
87
92
|
"docpages/pig/cookbook.html",
|
88
93
|
"docpages/pig/images/hadoop-logo.jpg",
|
89
94
|
"docpages/pig/images/instruction_arrow.png",
|
@@ -132,6 +137,7 @@ Gem::Specification.new do |s|
|
|
132
137
|
"examples/corpus/words_to_bigrams.rb",
|
133
138
|
"examples/count_keys.rb",
|
134
139
|
"examples/count_keys_at_mapper.rb",
|
140
|
+
"examples/emr/elastic_mapreduce_example.rb",
|
135
141
|
"examples/keystore/cassandra_batch_test.rb",
|
136
142
|
"examples/keystore/conditional_outputter_example.rb",
|
137
143
|
"examples/network_graph/adjacency_list.rb",
|
@@ -193,6 +199,8 @@ Gem::Specification.new do |s|
|
|
193
199
|
"lib/wukong/rdf.rb",
|
194
200
|
"lib/wukong/schema.rb",
|
195
201
|
"lib/wukong/script.rb",
|
202
|
+
"lib/wukong/script/avro_command.rb",
|
203
|
+
"lib/wukong/script/emr_command.rb",
|
196
204
|
"lib/wukong/script/hadoop_command.rb",
|
197
205
|
"lib/wukong/script/local_command.rb",
|
198
206
|
"lib/wukong/store.rb",
|
@@ -245,31 +253,32 @@ Gem::Specification.new do |s|
|
|
245
253
|
"spec/spec_helper.rb",
|
246
254
|
"spec/wukong/encoding_spec.rb",
|
247
255
|
"spec/wukong/script_spec.rb",
|
256
|
+
"examples/binning_percentile_estimator.rb",
|
257
|
+
"examples/contrib/jeans/normalize.rb",
|
258
|
+
"examples/contrib/jeans/sizes.rb",
|
259
|
+
"examples/corpus/words_to_bigrams.rb",
|
260
|
+
"examples/count_keys.rb",
|
261
|
+
"examples/count_keys_at_mapper.rb",
|
262
|
+
"examples/emr/elastic_mapreduce_example.rb",
|
263
|
+
"examples/keystore/cassandra_batch_test.rb",
|
264
|
+
"examples/keystore/conditional_outputter_example.rb",
|
265
|
+
"examples/network_graph/adjacency_list.rb",
|
266
|
+
"examples/network_graph/breadth_first_search.rb",
|
267
|
+
"examples/network_graph/gen_2paths.rb",
|
268
|
+
"examples/network_graph/gen_multi_edge.rb",
|
269
|
+
"examples/network_graph/gen_symmetric_links.rb",
|
248
270
|
"examples/pagerank/pagerank.rb",
|
249
271
|
"examples/pagerank/pagerank_initialize.rb",
|
272
|
+
"examples/rank_and_bin.rb",
|
250
273
|
"examples/sample_records.rb",
|
251
274
|
"examples/server_logs/apache_log_parser.rb",
|
252
275
|
"examples/server_logs/breadcrumbs.rb",
|
253
276
|
"examples/server_logs/user_agent.rb",
|
254
|
-
"examples/corpus/words_to_bigrams.rb",
|
255
|
-
"examples/count_keys.rb",
|
256
|
-
"examples/rank_and_bin.rb",
|
257
|
-
"examples/binning_percentile_estimator.rb",
|
258
277
|
"examples/size.rb",
|
259
|
-
"examples/store/chunked_store_example.rb",
|
260
|
-
"examples/network_graph/breadth_first_search.rb",
|
261
|
-
"examples/network_graph/gen_symmetric_links.rb",
|
262
|
-
"examples/network_graph/gen_multi_edge.rb",
|
263
|
-
"examples/network_graph/adjacency_list.rb",
|
264
|
-
"examples/network_graph/gen_2paths.rb",
|
265
|
-
"examples/keystore/cassandra_batch_test.rb",
|
266
|
-
"examples/keystore/conditional_outputter_example.rb",
|
267
278
|
"examples/stats/avg_value_frequency.rb",
|
268
|
-
"examples/
|
269
|
-
"examples/contrib/jeans/normalize.rb",
|
270
|
-
"examples/word_count.rb",
|
279
|
+
"examples/store/chunked_store_example.rb",
|
271
280
|
"examples/stupidly_simple_filter.rb",
|
272
|
-
"examples/
|
281
|
+
"examples/word_count.rb"
|
273
282
|
]
|
274
283
|
|
275
284
|
if s.respond_to? :specification_version then
|
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: wukong
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 3
|
5
5
|
prerelease: false
|
6
6
|
segments:
|
7
7
|
- 1
|
8
|
-
-
|
9
|
-
-
|
10
|
-
version: 1.
|
8
|
+
- 5
|
9
|
+
- 0
|
10
|
+
version: 1.5.0
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Philip (flip) Kromer
|
@@ -15,7 +15,7 @@ autorequire:
|
|
15
15
|
bindir: bin
|
16
16
|
cert_chain: []
|
17
17
|
|
18
|
-
date: 2010-08-
|
18
|
+
date: 2010-08-10 00:00:00 -05:00
|
19
19
|
default_executable:
|
20
20
|
dependencies:
|
21
21
|
- !ruby/object:Gem::Dependency
|
@@ -124,6 +124,7 @@ files:
|
|
124
124
|
- LICENSE.textile
|
125
125
|
- README.textile
|
126
126
|
- TODO.textile
|
127
|
+
- bin/bootstrap.sh
|
127
128
|
- bin/cutc
|
128
129
|
- bin/cuttab
|
129
130
|
- bin/greptrue
|
@@ -159,12 +160,15 @@ files:
|
|
159
160
|
- bin/wu-sum
|
160
161
|
- docpages/INSTALL.textile
|
161
162
|
- docpages/LICENSE.textile
|
163
|
+
- docpages/README-elastic_map_reduce.textile
|
162
164
|
- docpages/README-performance.textile
|
163
165
|
- docpages/README-wulign.textile
|
164
166
|
- docpages/UsingWukong-part1-get_ready.textile
|
165
167
|
- docpages/UsingWukong-part2-ThinkingBigData.textile
|
166
168
|
- docpages/UsingWukong-part3-parsing.textile
|
167
169
|
- docpages/_config.yml
|
170
|
+
- docpages/avro/avro_notes.textile
|
171
|
+
- docpages/avro/tethering.textile
|
168
172
|
- docpages/bigdata-tips.textile
|
169
173
|
- docpages/code/api_response_example.txt
|
170
174
|
- docpages/code/parser_skeleton.rb
|
@@ -178,6 +182,7 @@ files:
|
|
178
182
|
- docpages/news.html
|
179
183
|
- docpages/pig/PigLatinExpressionsList.txt
|
180
184
|
- docpages/pig/PigLatinReferenceManual.txt
|
185
|
+
- docpages/pig/commandline_params.txt
|
181
186
|
- docpages/pig/cookbook.html
|
182
187
|
- docpages/pig/images/hadoop-logo.jpg
|
183
188
|
- docpages/pig/images/instruction_arrow.png
|
@@ -226,6 +231,7 @@ files:
|
|
226
231
|
- examples/corpus/words_to_bigrams.rb
|
227
232
|
- examples/count_keys.rb
|
228
233
|
- examples/count_keys_at_mapper.rb
|
234
|
+
- examples/emr/elastic_mapreduce_example.rb
|
229
235
|
- examples/keystore/cassandra_batch_test.rb
|
230
236
|
- examples/keystore/conditional_outputter_example.rb
|
231
237
|
- examples/network_graph/adjacency_list.rb
|
@@ -287,6 +293,8 @@ files:
|
|
287
293
|
- lib/wukong/rdf.rb
|
288
294
|
- lib/wukong/schema.rb
|
289
295
|
- lib/wukong/script.rb
|
296
|
+
- lib/wukong/script/avro_command.rb
|
297
|
+
- lib/wukong/script/emr_command.rb
|
290
298
|
- lib/wukong/script/hadoop_command.rb
|
291
299
|
- lib/wukong/script/local_command.rb
|
292
300
|
- lib/wukong/store.rb
|
@@ -367,28 +375,29 @@ test_files:
|
|
367
375
|
- spec/spec_helper.rb
|
368
376
|
- spec/wukong/encoding_spec.rb
|
369
377
|
- spec/wukong/script_spec.rb
|
378
|
+
- examples/binning_percentile_estimator.rb
|
379
|
+
- examples/contrib/jeans/normalize.rb
|
380
|
+
- examples/contrib/jeans/sizes.rb
|
381
|
+
- examples/corpus/words_to_bigrams.rb
|
382
|
+
- examples/count_keys.rb
|
383
|
+
- examples/count_keys_at_mapper.rb
|
384
|
+
- examples/emr/elastic_mapreduce_example.rb
|
385
|
+
- examples/keystore/cassandra_batch_test.rb
|
386
|
+
- examples/keystore/conditional_outputter_example.rb
|
387
|
+
- examples/network_graph/adjacency_list.rb
|
388
|
+
- examples/network_graph/breadth_first_search.rb
|
389
|
+
- examples/network_graph/gen_2paths.rb
|
390
|
+
- examples/network_graph/gen_multi_edge.rb
|
391
|
+
- examples/network_graph/gen_symmetric_links.rb
|
370
392
|
- examples/pagerank/pagerank.rb
|
371
393
|
- examples/pagerank/pagerank_initialize.rb
|
394
|
+
- examples/rank_and_bin.rb
|
372
395
|
- examples/sample_records.rb
|
373
396
|
- examples/server_logs/apache_log_parser.rb
|
374
397
|
- examples/server_logs/breadcrumbs.rb
|
375
398
|
- examples/server_logs/user_agent.rb
|
376
|
-
- examples/corpus/words_to_bigrams.rb
|
377
|
-
- examples/count_keys.rb
|
378
|
-
- examples/rank_and_bin.rb
|
379
|
-
- examples/binning_percentile_estimator.rb
|
380
399
|
- examples/size.rb
|
381
|
-
- examples/store/chunked_store_example.rb
|
382
|
-
- examples/network_graph/breadth_first_search.rb
|
383
|
-
- examples/network_graph/gen_symmetric_links.rb
|
384
|
-
- examples/network_graph/gen_multi_edge.rb
|
385
|
-
- examples/network_graph/adjacency_list.rb
|
386
|
-
- examples/network_graph/gen_2paths.rb
|
387
|
-
- examples/keystore/cassandra_batch_test.rb
|
388
|
-
- examples/keystore/conditional_outputter_example.rb
|
389
400
|
- examples/stats/avg_value_frequency.rb
|
390
|
-
- examples/
|
391
|
-
- examples/contrib/jeans/normalize.rb
|
392
|
-
- examples/word_count.rb
|
401
|
+
- examples/store/chunked_store_example.rb
|
393
402
|
- examples/stupidly_simple_filter.rb
|
394
|
-
- examples/
|
403
|
+
- examples/word_count.rb
|