wukong 1.4.2 → 1.4.5
Sign up to get free protection for your applications and to get access to all the features.
- data/CHANGELOG.textile +9 -0
- data/bin/hdp-stream2 +2 -2
- data/examples/contrib/jeans/README.markdown +165 -0
- data/examples/contrib/jeans/data/normalized_sizes +3 -0
- data/examples/contrib/jeans/data/orders.tsv +1302 -0
- data/examples/contrib/jeans/data/sizes +3 -0
- data/examples/contrib/jeans/normalize.rb +20 -0
- data/examples/contrib/jeans/sizes.rb +55 -0
- data/examples/foo.rb +9 -0
- data/examples/word_count.rb +1 -1
- data/lib/wukong.rb +1 -1
- data/lib/wukong/extensions/hash.rb +52 -18
- data/lib/wukong/schema.rb +10 -10
- data/lib/wukong/script.rb +77 -97
- data/lib/wukong/script/hadoop_command.rb +21 -19
- data/lib/wukong/script/local_command.rb +9 -1
- data/lib/wukong/streamer/base.rb +1 -1
- data/spec/spec.opts +1 -0
- data/spec/spec_helper.rb +11 -0
- data/spec/wukong/encoding_spec.rb +36 -0
- data/spec/wukong/script_spec.rb +80 -0
- data/wukong.gemspec +23 -24
- metadata +18 -5
- data/lib/wukong/boot.rb +0 -47
- data/spec/bin/hdp-wc_spec.rb +0 -4
data/lib/wukong/boot.rb
DELETED
|
@@ -1,47 +0,0 @@
|
|
|
1
|
-
module Wukong
|
|
2
|
-
|
|
3
|
-
# ---------------------------------------------------------------------------
|
|
4
|
-
#
|
|
5
|
-
# Default options for Wukong
|
|
6
|
-
# http://github.com/infochimps/wukong
|
|
7
|
-
#
|
|
8
|
-
# If you set an environment variable WUKONG_CONFIG, *or* if the file
|
|
9
|
-
# $HOME/.wukong.rb exists, that file will be +require+'d as well.
|
|
10
|
-
#
|
|
11
|
-
# Important values to set:
|
|
12
|
-
#
|
|
13
|
-
# * Wukong::CONFIG[:hadoop_home] --
|
|
14
|
-
# Path to root of hadoop install. If your hadoop runner is
|
|
15
|
-
# /usr/local/share/hadoop/bin/hadoop
|
|
16
|
-
# then your hadoop_home is
|
|
17
|
-
# /usr/local/share/hadoop.
|
|
18
|
-
# You can also set a
|
|
19
|
-
#
|
|
20
|
-
# * Wukong::CONFIG[:default_run_mode] -- Whether to run using hadoop (and
|
|
21
|
-
# thus, requiring a working hadoop install), or to run in local mode
|
|
22
|
-
# (script --map | sort | script --reduce)
|
|
23
|
-
#
|
|
24
|
-
CONFIG = {
|
|
25
|
-
# Run as local or as hadoop?
|
|
26
|
-
:default_run_mode => 'hadoop',
|
|
27
|
-
|
|
28
|
-
# The command to run when a nil mapper or reducer is given.
|
|
29
|
-
:default_mapper => '/bin/cat',
|
|
30
|
-
:default_reducer => '/bin/cat',
|
|
31
|
-
|
|
32
|
-
# Anything in HADOOP_OPTIONS_MAP (see lib/wukong/script/hadoop_command.rb)
|
|
33
|
-
:runner_defaults => {
|
|
34
|
-
},
|
|
35
|
-
}
|
|
36
|
-
|
|
37
|
-
def self.config_options
|
|
38
|
-
# # override with site-specific options
|
|
39
|
-
site_config_filename = ENV['WUKONG_CONFIG'] || (ENV['HOME'].to_s+'/.wukong.rb')
|
|
40
|
-
require site_config_filename.gsub(/\.rb$/,'') if File.exists?(site_config_filename)
|
|
41
|
-
|
|
42
|
-
# try to guess a hadoop_home if none given
|
|
43
|
-
Wukong::CONFIG[:hadoop_home] ||= ENV['HADOOP_HOME'] || '/usr/lib/hadoop'
|
|
44
|
-
end
|
|
45
|
-
self.config_options
|
|
46
|
-
end
|
|
47
|
-
|
data/spec/bin/hdp-wc_spec.rb
DELETED