wukong 1.4.9 → 1.4.10
Sign up to get free protection for your applications and to get access to all the features.
- data/TODO.textile +13 -0
- data/bin/hdp-bzip +3 -3
- data/bin/hdp-kill-task +3 -0
- data/bin/hdp-mkdir +0 -1
- data/bin/hdp-put +1 -1
- data/bin/hdp-sort +5 -17
- data/bin/hdp-stream +5 -17
- data/bin/hdp-stream-flat +5 -5
- data/bin/wu-sum +1 -0
- data/docpages/README-performance.textile +90 -0
- data/examples/binning_percentile_estimator.rb +142 -0
- data/examples/corpus/words_to_bigrams.rb +52 -0
- data/examples/keystore/cassandra_batch_test.rb +41 -0
- data/examples/network_graph/gen_multi_edge.rb +3 -2
- data/examples/sample_records.rb +1 -0
- data/lib/wukong/extensions/date_time.rb +4 -3
- data/lib/wukong/extensions/enumerable.rb +79 -0
- data/lib/wukong/extensions.rb +1 -0
- data/lib/wukong/keystore/redis_db.rb +24 -0
- data/lib/wukong/keystore/tyrant_db.rb +124 -0
- data/lib/wukong/keystore/tyrant_notes.textile +145 -0
- data/lib/wukong/periodic_monitor.rb +57 -0
- data/lib/wukong/script/hadoop_command.rb +3 -1
- data/lib/wukong/streamer/accumulating_reducer.rb +1 -0
- data/lib/wukong/streamer/cassandra_streamer.rb +61 -0
- data/lib/wukong/streamer.rb +12 -10
- data/wukong.gemspec +34 -16
- metadata +60 -16
metadata
CHANGED
@@ -1,12 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: wukong
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
+
hash: 19
|
4
5
|
prerelease: false
|
5
6
|
segments:
|
6
7
|
- 1
|
7
8
|
- 4
|
8
|
-
-
|
9
|
-
version: 1.4.
|
9
|
+
- 10
|
10
|
+
version: 1.4.10
|
10
11
|
platform: ruby
|
11
12
|
authors:
|
12
13
|
- Philip (flip) Kromer
|
@@ -14,16 +15,18 @@ autorequire:
|
|
14
15
|
bindir: bin
|
15
16
|
cert_chain: []
|
16
17
|
|
17
|
-
date: 2010-
|
18
|
+
date: 2010-07-19 00:00:00 +00:00
|
18
19
|
default_executable:
|
19
20
|
dependencies:
|
20
21
|
- !ruby/object:Gem::Dependency
|
21
22
|
name: rspec
|
22
23
|
prerelease: false
|
23
24
|
requirement: &id001 !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
24
26
|
requirements:
|
25
27
|
- - ">="
|
26
28
|
- !ruby/object:Gem::Version
|
29
|
+
hash: 13
|
27
30
|
segments:
|
28
31
|
- 1
|
29
32
|
- 2
|
@@ -35,9 +38,11 @@ dependencies:
|
|
35
38
|
name: yard
|
36
39
|
prerelease: false
|
37
40
|
requirement: &id002 !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
38
42
|
requirements:
|
39
43
|
- - ">="
|
40
44
|
- !ruby/object:Gem::Version
|
45
|
+
hash: 3
|
41
46
|
segments:
|
42
47
|
- 0
|
43
48
|
version: "0"
|
@@ -47,9 +52,11 @@ dependencies:
|
|
47
52
|
name: addressable
|
48
53
|
prerelease: false
|
49
54
|
requirement: &id003 !ruby/object:Gem::Requirement
|
55
|
+
none: false
|
50
56
|
requirements:
|
51
57
|
- - ">="
|
52
58
|
- !ruby/object:Gem::Version
|
59
|
+
hash: 3
|
53
60
|
segments:
|
54
61
|
- 0
|
55
62
|
version: "0"
|
@@ -59,9 +66,11 @@ dependencies:
|
|
59
66
|
name: extlib
|
60
67
|
prerelease: false
|
61
68
|
requirement: &id004 !ruby/object:Gem::Requirement
|
69
|
+
none: false
|
62
70
|
requirements:
|
63
71
|
- - ">="
|
64
72
|
- !ruby/object:Gem::Version
|
73
|
+
hash: 3
|
65
74
|
segments:
|
66
75
|
- 0
|
67
76
|
version: "0"
|
@@ -71,14 +80,30 @@ dependencies:
|
|
71
80
|
name: htmlentities
|
72
81
|
prerelease: false
|
73
82
|
requirement: &id005 !ruby/object:Gem::Requirement
|
83
|
+
none: false
|
74
84
|
requirements:
|
75
85
|
- - ">="
|
76
86
|
- !ruby/object:Gem::Version
|
87
|
+
hash: 3
|
77
88
|
segments:
|
78
89
|
- 0
|
79
90
|
version: "0"
|
80
91
|
type: :runtime
|
81
92
|
version_requirements: *id005
|
93
|
+
- !ruby/object:Gem::Dependency
|
94
|
+
name: configliere
|
95
|
+
prerelease: false
|
96
|
+
requirement: &id006 !ruby/object:Gem::Requirement
|
97
|
+
none: false
|
98
|
+
requirements:
|
99
|
+
- - ">="
|
100
|
+
- !ruby/object:Gem::Version
|
101
|
+
hash: 3
|
102
|
+
segments:
|
103
|
+
- 0
|
104
|
+
version: "0"
|
105
|
+
type: :runtime
|
106
|
+
version_requirements: *id006
|
82
107
|
description: " Treat your dataset like a:\n\n * stream of lines when it's efficient to process by lines\n * stream of field arrays when it's efficient to deal directly with fields\n * stream of lightweight objects when it's efficient to deal with objects\n\n Wukong is friends with Hadoop the elephant, Pig the query language, and the cat on your command line.\n"
|
83
108
|
email: flip@infochimps.org
|
84
109
|
executables:
|
@@ -98,6 +123,7 @@ files:
|
|
98
123
|
- INSTALL.textile
|
99
124
|
- LICENSE.textile
|
100
125
|
- README.textile
|
126
|
+
- TODO.textile
|
101
127
|
- bin/cutc
|
102
128
|
- bin/cuttab
|
103
129
|
- bin/greptrue
|
@@ -107,6 +133,7 @@ files:
|
|
107
133
|
- bin/hdp-du
|
108
134
|
- bin/hdp-get
|
109
135
|
- bin/hdp-kill
|
136
|
+
- bin/hdp-kill-task
|
110
137
|
- bin/hdp-ls
|
111
138
|
- bin/hdp-mkdir
|
112
139
|
- bin/hdp-mv
|
@@ -131,6 +158,7 @@ files:
|
|
131
158
|
- bin/wu-sum
|
132
159
|
- docpages/INSTALL.textile
|
133
160
|
- docpages/LICENSE.textile
|
161
|
+
- docpages/README-performance.textile
|
134
162
|
- docpages/README-wulign.textile
|
135
163
|
- docpages/UsingWukong-part1-get_ready.textile
|
136
164
|
- docpages/UsingWukong-part2-ThinkingBigData.textile
|
@@ -187,14 +215,17 @@ files:
|
|
187
215
|
- docpages/usage.textile
|
188
216
|
- docpages/wutils.textile
|
189
217
|
- examples/README.txt
|
218
|
+
- examples/binning_percentile_estimator.rb
|
190
219
|
- examples/contrib/jeans/README.markdown
|
191
220
|
- examples/contrib/jeans/data/normalized_sizes
|
192
221
|
- examples/contrib/jeans/data/orders.tsv
|
193
222
|
- examples/contrib/jeans/data/sizes
|
194
223
|
- examples/contrib/jeans/normalize.rb
|
195
224
|
- examples/contrib/jeans/sizes.rb
|
225
|
+
- examples/corpus/words_to_bigrams.rb
|
196
226
|
- examples/count_keys.rb
|
197
227
|
- examples/count_keys_at_mapper.rb
|
228
|
+
- examples/keystore/cassandra_batch_test.rb
|
198
229
|
- examples/keystore/conditional_outputter_example.rb
|
199
230
|
- examples/network_graph/adjacency_list.rb
|
200
231
|
- examples/network_graph/breadth_first_search.rb
|
@@ -229,6 +260,7 @@ files:
|
|
229
260
|
- lib/wukong/extensions/class.rb
|
230
261
|
- lib/wukong/extensions/date_time.rb
|
231
262
|
- lib/wukong/extensions/emittable.rb
|
263
|
+
- lib/wukong/extensions/enumerable.rb
|
232
264
|
- lib/wukong/extensions/hash.rb
|
233
265
|
- lib/wukong/extensions/hash_keys.rb
|
234
266
|
- lib/wukong/extensions/hash_like.rb
|
@@ -239,8 +271,12 @@ files:
|
|
239
271
|
- lib/wukong/extensions/struct.rb
|
240
272
|
- lib/wukong/extensions/symbol.rb
|
241
273
|
- lib/wukong/keystore/cassandra_conditional_outputter.rb
|
274
|
+
- lib/wukong/keystore/redis_db.rb
|
275
|
+
- lib/wukong/keystore/tyrant_db.rb
|
276
|
+
- lib/wukong/keystore/tyrant_notes.textile
|
242
277
|
- lib/wukong/logger.rb
|
243
278
|
- lib/wukong/models/graph.rb
|
279
|
+
- lib/wukong/periodic_monitor.rb
|
244
280
|
- lib/wukong/rdf.rb
|
245
281
|
- lib/wukong/schema.rb
|
246
282
|
- lib/wukong/script.rb
|
@@ -249,6 +285,7 @@ files:
|
|
249
285
|
- lib/wukong/streamer.rb
|
250
286
|
- lib/wukong/streamer/accumulating_reducer.rb
|
251
287
|
- lib/wukong/streamer/base.rb
|
288
|
+
- lib/wukong/streamer/cassandra_streamer.rb
|
252
289
|
- lib/wukong/streamer/count_keys.rb
|
253
290
|
- lib/wukong/streamer/count_lines.rb
|
254
291
|
- lib/wukong/streamer/counting_reducer.rb
|
@@ -283,23 +320,27 @@ rdoc_options:
|
|
283
320
|
require_paths:
|
284
321
|
- lib
|
285
322
|
required_ruby_version: !ruby/object:Gem::Requirement
|
323
|
+
none: false
|
286
324
|
requirements:
|
287
325
|
- - ">="
|
288
326
|
- !ruby/object:Gem::Version
|
327
|
+
hash: 3
|
289
328
|
segments:
|
290
329
|
- 0
|
291
330
|
version: "0"
|
292
331
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
332
|
+
none: false
|
293
333
|
requirements:
|
294
334
|
- - ">="
|
295
335
|
- !ruby/object:Gem::Version
|
336
|
+
hash: 3
|
296
337
|
segments:
|
297
338
|
- 0
|
298
339
|
version: "0"
|
299
340
|
requirements: []
|
300
341
|
|
301
342
|
rubyforge_project:
|
302
|
-
rubygems_version: 1.3.
|
343
|
+
rubygems_version: 1.3.7
|
303
344
|
signing_key:
|
304
345
|
specification_version: 3
|
305
346
|
summary: Hadoop Streaming for Ruby. Wukong makes Hadoop so easy a chimpanzee can use it, yet handles terabyte-scale computation with ease.
|
@@ -307,24 +348,27 @@ test_files:
|
|
307
348
|
- spec/spec_helper.rb
|
308
349
|
- spec/wukong/encoding_spec.rb
|
309
350
|
- spec/wukong/script_spec.rb
|
310
|
-
- examples/contrib/jeans/normalize.rb
|
311
|
-
- examples/contrib/jeans/sizes.rb
|
312
|
-
- examples/count_keys.rb
|
313
|
-
- examples/count_keys_at_mapper.rb
|
314
|
-
- examples/keystore/conditional_outputter_example.rb
|
315
|
-
- examples/network_graph/adjacency_list.rb
|
316
|
-
- examples/network_graph/breadth_first_search.rb
|
317
|
-
- examples/network_graph/gen_2paths.rb
|
318
|
-
- examples/network_graph/gen_multi_edge.rb
|
319
|
-
- examples/network_graph/gen_symmetric_links.rb
|
320
351
|
- examples/pagerank/pagerank.rb
|
321
352
|
- examples/pagerank/pagerank_initialize.rb
|
322
|
-
- examples/rank_and_bin.rb
|
323
353
|
- examples/sample_records.rb
|
324
354
|
- examples/server_logs/apache_log_parser.rb
|
325
355
|
- examples/server_logs/breadcrumbs.rb
|
326
356
|
- examples/server_logs/user_agent.rb
|
357
|
+
- examples/corpus/words_to_bigrams.rb
|
358
|
+
- examples/count_keys.rb
|
359
|
+
- examples/rank_and_bin.rb
|
360
|
+
- examples/binning_percentile_estimator.rb
|
327
361
|
- examples/size.rb
|
362
|
+
- examples/network_graph/breadth_first_search.rb
|
363
|
+
- examples/network_graph/gen_symmetric_links.rb
|
364
|
+
- examples/network_graph/gen_multi_edge.rb
|
365
|
+
- examples/network_graph/adjacency_list.rb
|
366
|
+
- examples/network_graph/gen_2paths.rb
|
367
|
+
- examples/keystore/cassandra_batch_test.rb
|
368
|
+
- examples/keystore/conditional_outputter_example.rb
|
328
369
|
- examples/stats/avg_value_frequency.rb
|
329
|
-
- examples/
|
370
|
+
- examples/contrib/jeans/sizes.rb
|
371
|
+
- examples/contrib/jeans/normalize.rb
|
330
372
|
- examples/word_count.rb
|
373
|
+
- examples/stupidly_simple_filter.rb
|
374
|
+
- examples/count_keys_at_mapper.rb
|