wukong 1.4.12 → 1.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (17) hide show
  1. data/CHANGELOG.textile +20 -1
  2. data/bin/bootstrap.sh +32 -0
  3. data/bin/hdp-sort +3 -0
  4. data/bin/hdp-stream +3 -0
  5. data/docpages/README-elastic_map_reduce.textile +377 -0
  6. data/docpages/avro/avro_notes.textile +56 -0
  7. data/docpages/avro/tethering.textile +19 -0
  8. data/docpages/pig/commandline_params.txt +26 -0
  9. data/examples/emr/elastic_mapreduce_example.rb +35 -0
  10. data/lib/wukong/logger.rb +8 -1
  11. data/lib/wukong/script/avro_command.rb +5 -0
  12. data/lib/wukong/script/emr_command.rb +119 -0
  13. data/lib/wukong/script/hadoop_command.rb +72 -90
  14. data/lib/wukong/script/local_command.rb +18 -8
  15. data/lib/wukong/script.rb +87 -92
  16. data/wukong.gemspec +27 -18
  17. metadata +30 -21
data/wukong.gemspec CHANGED
@@ -5,11 +5,11 @@
5
5
 
6
6
  Gem::Specification.new do |s|
7
7
  s.name = %q{wukong}
8
- s.version = "1.4.12"
8
+ s.version = "1.5.0"
9
9
 
10
10
  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
11
11
  s.authors = ["Philip (flip) Kromer"]
12
- s.date = %q{2010-08-03}
12
+ s.date = %q{2010-08-10}
13
13
  s.description = %q{ Treat your dataset like a:
14
14
 
15
15
  * stream of lines when it's efficient to process by lines
@@ -30,6 +30,7 @@ Gem::Specification.new do |s|
30
30
  "LICENSE.textile",
31
31
  "README.textile",
32
32
  "TODO.textile",
33
+ "bin/bootstrap.sh",
33
34
  "bin/cutc",
34
35
  "bin/cuttab",
35
36
  "bin/greptrue",
@@ -65,12 +66,15 @@ Gem::Specification.new do |s|
65
66
  "bin/wu-sum",
66
67
  "docpages/INSTALL.textile",
67
68
  "docpages/LICENSE.textile",
69
+ "docpages/README-elastic_map_reduce.textile",
68
70
  "docpages/README-performance.textile",
69
71
  "docpages/README-wulign.textile",
70
72
  "docpages/UsingWukong-part1-get_ready.textile",
71
73
  "docpages/UsingWukong-part2-ThinkingBigData.textile",
72
74
  "docpages/UsingWukong-part3-parsing.textile",
73
75
  "docpages/_config.yml",
76
+ "docpages/avro/avro_notes.textile",
77
+ "docpages/avro/tethering.textile",
74
78
  "docpages/bigdata-tips.textile",
75
79
  "docpages/code/api_response_example.txt",
76
80
  "docpages/code/parser_skeleton.rb",
@@ -84,6 +88,7 @@ Gem::Specification.new do |s|
84
88
  "docpages/news.html",
85
89
  "docpages/pig/PigLatinExpressionsList.txt",
86
90
  "docpages/pig/PigLatinReferenceManual.txt",
91
+ "docpages/pig/commandline_params.txt",
87
92
  "docpages/pig/cookbook.html",
88
93
  "docpages/pig/images/hadoop-logo.jpg",
89
94
  "docpages/pig/images/instruction_arrow.png",
@@ -132,6 +137,7 @@ Gem::Specification.new do |s|
132
137
  "examples/corpus/words_to_bigrams.rb",
133
138
  "examples/count_keys.rb",
134
139
  "examples/count_keys_at_mapper.rb",
140
+ "examples/emr/elastic_mapreduce_example.rb",
135
141
  "examples/keystore/cassandra_batch_test.rb",
136
142
  "examples/keystore/conditional_outputter_example.rb",
137
143
  "examples/network_graph/adjacency_list.rb",
@@ -193,6 +199,8 @@ Gem::Specification.new do |s|
193
199
  "lib/wukong/rdf.rb",
194
200
  "lib/wukong/schema.rb",
195
201
  "lib/wukong/script.rb",
202
+ "lib/wukong/script/avro_command.rb",
203
+ "lib/wukong/script/emr_command.rb",
196
204
  "lib/wukong/script/hadoop_command.rb",
197
205
  "lib/wukong/script/local_command.rb",
198
206
  "lib/wukong/store.rb",
@@ -245,31 +253,32 @@ Gem::Specification.new do |s|
245
253
  "spec/spec_helper.rb",
246
254
  "spec/wukong/encoding_spec.rb",
247
255
  "spec/wukong/script_spec.rb",
256
+ "examples/binning_percentile_estimator.rb",
257
+ "examples/contrib/jeans/normalize.rb",
258
+ "examples/contrib/jeans/sizes.rb",
259
+ "examples/corpus/words_to_bigrams.rb",
260
+ "examples/count_keys.rb",
261
+ "examples/count_keys_at_mapper.rb",
262
+ "examples/emr/elastic_mapreduce_example.rb",
263
+ "examples/keystore/cassandra_batch_test.rb",
264
+ "examples/keystore/conditional_outputter_example.rb",
265
+ "examples/network_graph/adjacency_list.rb",
266
+ "examples/network_graph/breadth_first_search.rb",
267
+ "examples/network_graph/gen_2paths.rb",
268
+ "examples/network_graph/gen_multi_edge.rb",
269
+ "examples/network_graph/gen_symmetric_links.rb",
248
270
  "examples/pagerank/pagerank.rb",
249
271
  "examples/pagerank/pagerank_initialize.rb",
272
+ "examples/rank_and_bin.rb",
250
273
  "examples/sample_records.rb",
251
274
  "examples/server_logs/apache_log_parser.rb",
252
275
  "examples/server_logs/breadcrumbs.rb",
253
276
  "examples/server_logs/user_agent.rb",
254
- "examples/corpus/words_to_bigrams.rb",
255
- "examples/count_keys.rb",
256
- "examples/rank_and_bin.rb",
257
- "examples/binning_percentile_estimator.rb",
258
277
  "examples/size.rb",
259
- "examples/store/chunked_store_example.rb",
260
- "examples/network_graph/breadth_first_search.rb",
261
- "examples/network_graph/gen_symmetric_links.rb",
262
- "examples/network_graph/gen_multi_edge.rb",
263
- "examples/network_graph/adjacency_list.rb",
264
- "examples/network_graph/gen_2paths.rb",
265
- "examples/keystore/cassandra_batch_test.rb",
266
- "examples/keystore/conditional_outputter_example.rb",
267
278
  "examples/stats/avg_value_frequency.rb",
268
- "examples/contrib/jeans/sizes.rb",
269
- "examples/contrib/jeans/normalize.rb",
270
- "examples/word_count.rb",
279
+ "examples/store/chunked_store_example.rb",
271
280
  "examples/stupidly_simple_filter.rb",
272
- "examples/count_keys_at_mapper.rb"
281
+ "examples/word_count.rb"
273
282
  ]
274
283
 
275
284
  if s.respond_to? :specification_version then
metadata CHANGED
@@ -1,13 +1,13 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: wukong
3
3
  version: !ruby/object:Gem::Version
4
- hash: 31
4
+ hash: 3
5
5
  prerelease: false
6
6
  segments:
7
7
  - 1
8
- - 4
9
- - 12
10
- version: 1.4.12
8
+ - 5
9
+ - 0
10
+ version: 1.5.0
11
11
  platform: ruby
12
12
  authors:
13
13
  - Philip (flip) Kromer
@@ -15,7 +15,7 @@ autorequire:
15
15
  bindir: bin
16
16
  cert_chain: []
17
17
 
18
- date: 2010-08-03 00:00:00 +00:00
18
+ date: 2010-08-10 00:00:00 -05:00
19
19
  default_executable:
20
20
  dependencies:
21
21
  - !ruby/object:Gem::Dependency
@@ -124,6 +124,7 @@ files:
124
124
  - LICENSE.textile
125
125
  - README.textile
126
126
  - TODO.textile
127
+ - bin/bootstrap.sh
127
128
  - bin/cutc
128
129
  - bin/cuttab
129
130
  - bin/greptrue
@@ -159,12 +160,15 @@ files:
159
160
  - bin/wu-sum
160
161
  - docpages/INSTALL.textile
161
162
  - docpages/LICENSE.textile
163
+ - docpages/README-elastic_map_reduce.textile
162
164
  - docpages/README-performance.textile
163
165
  - docpages/README-wulign.textile
164
166
  - docpages/UsingWukong-part1-get_ready.textile
165
167
  - docpages/UsingWukong-part2-ThinkingBigData.textile
166
168
  - docpages/UsingWukong-part3-parsing.textile
167
169
  - docpages/_config.yml
170
+ - docpages/avro/avro_notes.textile
171
+ - docpages/avro/tethering.textile
168
172
  - docpages/bigdata-tips.textile
169
173
  - docpages/code/api_response_example.txt
170
174
  - docpages/code/parser_skeleton.rb
@@ -178,6 +182,7 @@ files:
178
182
  - docpages/news.html
179
183
  - docpages/pig/PigLatinExpressionsList.txt
180
184
  - docpages/pig/PigLatinReferenceManual.txt
185
+ - docpages/pig/commandline_params.txt
181
186
  - docpages/pig/cookbook.html
182
187
  - docpages/pig/images/hadoop-logo.jpg
183
188
  - docpages/pig/images/instruction_arrow.png
@@ -226,6 +231,7 @@ files:
226
231
  - examples/corpus/words_to_bigrams.rb
227
232
  - examples/count_keys.rb
228
233
  - examples/count_keys_at_mapper.rb
234
+ - examples/emr/elastic_mapreduce_example.rb
229
235
  - examples/keystore/cassandra_batch_test.rb
230
236
  - examples/keystore/conditional_outputter_example.rb
231
237
  - examples/network_graph/adjacency_list.rb
@@ -287,6 +293,8 @@ files:
287
293
  - lib/wukong/rdf.rb
288
294
  - lib/wukong/schema.rb
289
295
  - lib/wukong/script.rb
296
+ - lib/wukong/script/avro_command.rb
297
+ - lib/wukong/script/emr_command.rb
290
298
  - lib/wukong/script/hadoop_command.rb
291
299
  - lib/wukong/script/local_command.rb
292
300
  - lib/wukong/store.rb
@@ -367,28 +375,29 @@ test_files:
367
375
  - spec/spec_helper.rb
368
376
  - spec/wukong/encoding_spec.rb
369
377
  - spec/wukong/script_spec.rb
378
+ - examples/binning_percentile_estimator.rb
379
+ - examples/contrib/jeans/normalize.rb
380
+ - examples/contrib/jeans/sizes.rb
381
+ - examples/corpus/words_to_bigrams.rb
382
+ - examples/count_keys.rb
383
+ - examples/count_keys_at_mapper.rb
384
+ - examples/emr/elastic_mapreduce_example.rb
385
+ - examples/keystore/cassandra_batch_test.rb
386
+ - examples/keystore/conditional_outputter_example.rb
387
+ - examples/network_graph/adjacency_list.rb
388
+ - examples/network_graph/breadth_first_search.rb
389
+ - examples/network_graph/gen_2paths.rb
390
+ - examples/network_graph/gen_multi_edge.rb
391
+ - examples/network_graph/gen_symmetric_links.rb
370
392
  - examples/pagerank/pagerank.rb
371
393
  - examples/pagerank/pagerank_initialize.rb
394
+ - examples/rank_and_bin.rb
372
395
  - examples/sample_records.rb
373
396
  - examples/server_logs/apache_log_parser.rb
374
397
  - examples/server_logs/breadcrumbs.rb
375
398
  - examples/server_logs/user_agent.rb
376
- - examples/corpus/words_to_bigrams.rb
377
- - examples/count_keys.rb
378
- - examples/rank_and_bin.rb
379
- - examples/binning_percentile_estimator.rb
380
399
  - examples/size.rb
381
- - examples/store/chunked_store_example.rb
382
- - examples/network_graph/breadth_first_search.rb
383
- - examples/network_graph/gen_symmetric_links.rb
384
- - examples/network_graph/gen_multi_edge.rb
385
- - examples/network_graph/adjacency_list.rb
386
- - examples/network_graph/gen_2paths.rb
387
- - examples/keystore/cassandra_batch_test.rb
388
- - examples/keystore/conditional_outputter_example.rb
389
400
  - examples/stats/avg_value_frequency.rb
390
- - examples/contrib/jeans/sizes.rb
391
- - examples/contrib/jeans/normalize.rb
392
- - examples/word_count.rb
401
+ - examples/store/chunked_store_example.rb
393
402
  - examples/stupidly_simple_filter.rb
394
- - examples/count_keys_at_mapper.rb
403
+ - examples/word_count.rb