mortar 0.11.1 → 0.12.0
Sign up to get free protection for your applications and to get access to all the features.
- data/lib/mortar/command.rb +2 -0
- data/lib/mortar/command/base.rb +46 -1
- data/lib/mortar/command/describe.rb +8 -4
- data/lib/mortar/command/illustrate.rb +2 -1
- data/lib/mortar/command/jobs.rb +3 -0
- data/lib/mortar/command/local.rb +12 -6
- data/lib/mortar/command/validate.rb +2 -1
- data/lib/mortar/local/controller.rb +16 -11
- data/lib/mortar/local/pig.rb +44 -41
- data/lib/mortar/pigversion.rb +71 -0
- data/lib/mortar/templates/script/runpig.sh +1 -1
- data/lib/mortar/version.rb +1 -1
- data/spec/mortar/command/base_spec.rb +116 -0
- data/spec/mortar/command/describe_spec.rb +4 -4
- data/spec/mortar/command/illustrate_spec.rb +7 -7
- data/spec/mortar/command/jobs_spec.rb +14 -3
- data/spec/mortar/command/local_spec.rb +10 -10
- data/spec/mortar/command/validate_spec.rb +4 -4
- data/spec/mortar/local/controller_spec.rb +29 -6
- data/spec/mortar/local/pig_spec.rb +42 -18
- data/spec/spec_helper.rb +6 -1
- metadata +180 -132
- checksums.yaml +0 -7
data/lib/mortar/command.rb
CHANGED
@@ -127,6 +127,7 @@ module Mortar
|
|
127
127
|
global_option :remote, "--remote REMOTE"
|
128
128
|
global_option :polling_interval, "--polling_interval SECONDS", "-p"
|
129
129
|
global_option :no_browser, "--no_browser"
|
130
|
+
global_option :pigversion, "--pigversion PIG_VERSION", "-g"
|
130
131
|
|
131
132
|
def self.prepare_run(cmd, args=[])
|
132
133
|
command = parse(cmd)
|
@@ -176,6 +177,7 @@ module Mortar
|
|
176
177
|
opts[global_option[:name]] = value
|
177
178
|
end
|
178
179
|
end
|
180
|
+
|
179
181
|
command[:options].each do |name, option|
|
180
182
|
parser.on("-#{option[:short]}", "--#{option[:long]}", option[:desc]) do |value|
|
181
183
|
opt_name_sym = name.gsub("-", "_").to_sym
|
data/lib/mortar/command/base.rb
CHANGED
@@ -18,8 +18,10 @@
|
|
18
18
|
#
|
19
19
|
|
20
20
|
require "fileutils"
|
21
|
+
require "parseconfig"
|
21
22
|
require "mortar/auth"
|
22
23
|
require "mortar/command"
|
24
|
+
require "mortar/pigversion"
|
23
25
|
require "mortar/project"
|
24
26
|
require "mortar/git"
|
25
27
|
|
@@ -36,6 +38,11 @@ class Mortar::Command::Base
|
|
36
38
|
def initialize(args=[], options={})
|
37
39
|
@args = args
|
38
40
|
@options = options
|
41
|
+
#We never want to override the command line options so we store them.
|
42
|
+
@original_options = options.dup
|
43
|
+
|
44
|
+
#Initialize defaults from .mortar-defaults
|
45
|
+
load_defaults('DEFAULTS')
|
39
46
|
end
|
40
47
|
|
41
48
|
def project
|
@@ -181,15 +188,28 @@ protected
|
|
181
188
|
end
|
182
189
|
end
|
183
190
|
|
191
|
+
def self.replace_templates(help)
|
192
|
+
#Leave --pigversion undocumented for now.
|
193
|
+
#help.each do |line|
|
194
|
+
# #line.gsub!("<PIG_VERSION_OPTIONS>", "0.9 (default) and 0.12 (beta)")
|
195
|
+
#
|
196
|
+
#end
|
197
|
+
help.reject! do |line|
|
198
|
+
line.include?("<PIG_VERSION_OPTIONS>")
|
199
|
+
end
|
200
|
+
end
|
201
|
+
|
184
202
|
def self.method_added(method)
|
185
203
|
return if self == Mortar::Command::Base
|
186
204
|
return if private_method_defined?(method)
|
187
205
|
return if protected_method_defined?(method)
|
188
206
|
|
189
207
|
help = extract_help_from_caller(caller.first)
|
208
|
+
replace_templates(help)
|
209
|
+
|
190
210
|
resolved_method = (method.to_s == "index") ? nil : method.to_s
|
191
211
|
command = [ self.namespace, resolved_method ].compact.join(":")
|
192
|
-
banner = extract_banner(help) || command
|
212
|
+
banner = extract_banner(help) || command
|
193
213
|
|
194
214
|
Mortar::Command.register_command(
|
195
215
|
:klass => self,
|
@@ -324,6 +344,9 @@ protected
|
|
324
344
|
error("Naming conflict. #{script_name} refers to both a pigscript and a controlscript. Please rename scripts to avoid conflicts.")
|
325
345
|
end
|
326
346
|
|
347
|
+
#While validating we can load the defaults that are relevant to this script.
|
348
|
+
load_defaults(shortened_script_name)
|
349
|
+
|
327
350
|
pigscript or controlscript
|
328
351
|
end
|
329
352
|
|
@@ -333,6 +356,10 @@ protected
|
|
333
356
|
available_scripts = project.pigscripts.none? ? "No pigscripts found" : "Available scripts:\n#{project.pigscripts.collect{|k,v| v.executable_path}.sort.join("\n")}"
|
334
357
|
error("Unable to find pigscript #{pigscript_name}\n#{available_scripts}")
|
335
358
|
end
|
359
|
+
|
360
|
+
#While validating we can load the defaults that are relevant to this script.
|
361
|
+
load_defaults(shortened_pigscript_name)
|
362
|
+
|
336
363
|
pigscript
|
337
364
|
end
|
338
365
|
|
@@ -345,6 +372,19 @@ protected
|
|
345
372
|
return missing_dir ? nil : [File.basename(Dir.getwd), nil]
|
346
373
|
end
|
347
374
|
|
375
|
+
def load_defaults(section_name)
|
376
|
+
if File.exists?('.mortar-defaults')
|
377
|
+
default_options = ParseConfig.new('.mortar-defaults')
|
378
|
+
if default_options.groups.include?(section_name)
|
379
|
+
default_options[section_name].each do |k, v|
|
380
|
+
unless @original_options.include? k.to_sym
|
381
|
+
@options[k.to_sym] = v
|
382
|
+
end
|
383
|
+
end
|
384
|
+
end
|
385
|
+
end
|
386
|
+
end
|
387
|
+
|
348
388
|
def extract_project_in_dir(project_name=nil)
|
349
389
|
# returns [project_name, remote_name]
|
350
390
|
# TODO refactor this very messy method
|
@@ -398,6 +438,11 @@ protected
|
|
398
438
|
(options[:no_browser])
|
399
439
|
end
|
400
440
|
|
441
|
+
def pig_version
|
442
|
+
pig_version_str = options[:pigversion] || '0.9'
|
443
|
+
pig_version = Mortar::PigVersion.from_string(pig_version_str)
|
444
|
+
end
|
445
|
+
|
401
446
|
def sync_code_with_cloud
|
402
447
|
# returns git_ref
|
403
448
|
if project.embedded_project?
|
@@ -28,6 +28,7 @@ class Mortar::Command::Describe < Mortar::Command::Base
|
|
28
28
|
#
|
29
29
|
# -p, --parameter NAME=VALUE # Set a pig parameter value in your script.
|
30
30
|
# -f, --param-file PARAMFILE # Load pig parameter values from a file.
|
31
|
+
# -g, --pigversion PIG_VERSION # Set pig version. Options are <PIG_VERSION_OPTIONS>.
|
31
32
|
#
|
32
33
|
# Examples:
|
33
34
|
#
|
@@ -51,7 +52,7 @@ class Mortar::Command::Describe < Mortar::Command::Base
|
|
51
52
|
|
52
53
|
describe_id = nil
|
53
54
|
action("Starting describe") do
|
54
|
-
describe_id = api.post_describe(project.name, pigscript.name, alias_name, git_ref, :parameters => pig_parameters).body["describe_id"]
|
55
|
+
describe_id = api.post_describe(project.name, pigscript.name, alias_name, git_ref, :pig_version => pig_version.version, :parameters => pig_parameters).body["describe_id"]
|
55
56
|
end
|
56
57
|
|
57
58
|
describe_result = nil
|
@@ -86,9 +87,12 @@ class Mortar::Command::Describe < Mortar::Command::Base
|
|
86
87
|
when Mortar::API::Describe::STATUS_SUCCESS
|
87
88
|
web_result_url = describe_result['web_result_url']
|
88
89
|
display("Results available at #{web_result_url}")
|
89
|
-
|
90
|
-
|
91
|
-
|
90
|
+
|
91
|
+
unless no_browser?
|
92
|
+
action("Opening web browser to show results") do
|
93
|
+
require "launchy"
|
94
|
+
Launchy.open(web_result_url).join
|
95
|
+
end
|
92
96
|
end
|
93
97
|
else
|
94
98
|
raise RuntimeError, "Unknown describe status: #{describe_result['status']} for describe_id: #{describe_id}"
|
@@ -29,6 +29,7 @@ class Mortar::Command::Illustrate < Mortar::Command::Base
|
|
29
29
|
# -s, --skippruning # Don't try to reduce the illustrate results to the smallest size possible.
|
30
30
|
# -p, --parameter NAME=VALUE # Set a pig parameter value in your script.
|
31
31
|
# -f, --param-file PARAMFILE # Load pig parameter values from a file.
|
32
|
+
# -g, --pigversion PIG_VERSION # Set pig version. Options are <PIG_VERSION_OPTIONS>.
|
32
33
|
# --no_browser # Don't open the illustrate results automatically in the browser.
|
33
34
|
#
|
34
35
|
# Examples:
|
@@ -56,7 +57,7 @@ class Mortar::Command::Illustrate < Mortar::Command::Base
|
|
56
57
|
|
57
58
|
illustrate_id = nil
|
58
59
|
action("Starting illustrate") do
|
59
|
-
illustrate_id = api.post_illustrate(project.name, pigscript.name, alias_name, skip_pruning, git_ref, :parameters => pig_parameters).body["illustrate_id"]
|
60
|
+
illustrate_id = api.post_illustrate(project.name, pigscript.name, alias_name, skip_pruning, git_ref, :pig_version => pig_version.version, :parameters => pig_parameters).body["illustrate_id"]
|
60
61
|
end
|
61
62
|
|
62
63
|
illustrate_result = nil
|
data/lib/mortar/command/jobs.rb
CHANGED
@@ -67,6 +67,7 @@ class Mortar::Command::Jobs < Mortar::Command::Base
|
|
67
67
|
# -d, --donotnotify # Don't send an email on job completion. (Default: false--an email will be sent to you once the job completes)
|
68
68
|
# -P, --project PROJECTNAME # Use a project that is not checked out in the current directory. Runs code from project's master branch in github rather than snapshotting local code.
|
69
69
|
# -B, --branch BRANCHNAME # Used with --project to specify a non-master branch
|
70
|
+
# -g, --pigversion PIG_VERSION # Set pig version. Options are <PIG_VERSION_OPTIONS>.
|
70
71
|
#
|
71
72
|
#Examples:
|
72
73
|
#
|
@@ -162,6 +163,7 @@ class Mortar::Command::Jobs < Mortar::Command::Base
|
|
162
163
|
cluster_type = CLUSTER_TYPE__PERMANENT
|
163
164
|
end
|
164
165
|
api.post_job_new_cluster(project_name, script_name, git_ref, cluster_size,
|
166
|
+
:pig_version => pig_version.version,
|
165
167
|
:parameters => pig_parameters,
|
166
168
|
:cluster_type => cluster_type,
|
167
169
|
:notify_on_job_finish => notify_on_job_finish,
|
@@ -169,6 +171,7 @@ class Mortar::Command::Jobs < Mortar::Command::Base
|
|
169
171
|
else
|
170
172
|
cluster_id = options[:clusterid]
|
171
173
|
api.post_job_existing_cluster(project_name, script_name, git_ref, cluster_id,
|
174
|
+
:pig_version => pig_version.version,
|
172
175
|
:parameters => pig_parameters,
|
173
176
|
:notify_on_job_finish => notify_on_job_finish,
|
174
177
|
:is_control_script => is_control_script).body
|
data/lib/mortar/command/local.rb
CHANGED
@@ -27,9 +27,11 @@ class Mortar::Command::Local < Mortar::Command::Base
|
|
27
27
|
#
|
28
28
|
# Install dependencies for running this mortar project locally - other mortar:local commands will also perform this step automatically.
|
29
29
|
#
|
30
|
-
#
|
30
|
+
# -g, --pigversion PIG_VERSION # Set pig version. Options are <PIG_VERSION_OPTIONS>.
|
31
|
+
# --project-root PROJECTDIR # The root directory of the project if not the CWD
|
31
32
|
#
|
32
33
|
def configure
|
34
|
+
validate_arguments!
|
33
35
|
|
34
36
|
# cd into the project root
|
35
37
|
project_root = options[:project_root] ||= Dir.getwd
|
@@ -39,7 +41,7 @@ class Mortar::Command::Local < Mortar::Command::Base
|
|
39
41
|
Dir.chdir(project_root)
|
40
42
|
|
41
43
|
ctrl = Mortar::Local::Controller.new
|
42
|
-
ctrl.install_and_configure
|
44
|
+
ctrl.install_and_configure(pig_version)
|
43
45
|
end
|
44
46
|
|
45
47
|
# local:run SCRIPT
|
@@ -48,6 +50,7 @@ class Mortar::Command::Local < Mortar::Command::Base
|
|
48
50
|
#
|
49
51
|
# -p, --parameter NAME=VALUE # Set a pig parameter value in your script.
|
50
52
|
# -f, --param-file PARAMFILE # Load pig parameter values from a file.
|
53
|
+
# -g, --pigversion PIG_VERSION # Set pig version. Options are <PIG_VERSION_OPTIONS>.
|
51
54
|
# --project-root PROJECTDIR # The root directory of the project if not the CWD
|
52
55
|
#
|
53
56
|
#Examples:
|
@@ -69,7 +72,7 @@ class Mortar::Command::Local < Mortar::Command::Base
|
|
69
72
|
Dir.chdir(project_root)
|
70
73
|
script = validate_script!(script_name)
|
71
74
|
ctrl = Mortar::Local::Controller.new
|
72
|
-
ctrl.run(script, pig_parameters)
|
75
|
+
ctrl.run(script, pig_version, pig_parameters)
|
73
76
|
end
|
74
77
|
|
75
78
|
# local:characterize -f PARAMFILE
|
@@ -79,6 +82,7 @@ class Mortar::Command::Local < Mortar::Command::Base
|
|
79
82
|
# statistics about your data (most common values, percent null, etc.)
|
80
83
|
#
|
81
84
|
# -f, --param-file PARAMFILE # Load pig parameter values from a file
|
85
|
+
# -g, --pigversion PIG_VERSION # Set pig version. Options are <PIG_VERSION_OPTIONS>.
|
82
86
|
#
|
83
87
|
# Load some data and emit statistics.
|
84
88
|
# PARAMFILE (Required):
|
@@ -116,7 +120,7 @@ class Mortar::Command::Local < Mortar::Command::Base
|
|
116
120
|
gen.generate_characterize
|
117
121
|
script = validate_script!(controlscript_name)
|
118
122
|
ctrl = Mortar::Local::Controller.new
|
119
|
-
ctrl.run(script, pig_parameters)
|
123
|
+
ctrl.run(script, pig_version, pig_parameters)
|
120
124
|
gen.cleanup_characterize(project_root)
|
121
125
|
end
|
122
126
|
|
@@ -129,6 +133,7 @@ class Mortar::Command::Local < Mortar::Command::Base
|
|
129
133
|
# -s, --skippruning # Don't try to reduce the illustrate results to the smallest size possible.
|
130
134
|
# -p, --parameter NAME=VALUE # Set a pig parameter value in your script.
|
131
135
|
# -f, --param-file PARAMFILE # Load pig parameter values from a file.
|
136
|
+
# -g, --pigversion PIG_VERSION # Set pig version. Options are <PIG_VERSION_OPTIONS>.
|
132
137
|
# --no_browser # Don't open the illustrate results automatically in the browser.
|
133
138
|
# --project-root PROJECTDIR # The root directory of the project if not the CWD
|
134
139
|
#
|
@@ -156,7 +161,7 @@ class Mortar::Command::Local < Mortar::Command::Base
|
|
156
161
|
pigscript = validate_pigscript!(pigscript_name)
|
157
162
|
|
158
163
|
ctrl = Mortar::Local::Controller.new
|
159
|
-
ctrl.illustrate(pigscript, alias_name, pig_parameters, skip_pruning)
|
164
|
+
ctrl.illustrate(pigscript, alias_name, pig_version, pig_parameters, skip_pruning)
|
160
165
|
end
|
161
166
|
|
162
167
|
|
@@ -166,6 +171,7 @@ class Mortar::Command::Local < Mortar::Command::Base
|
|
166
171
|
#
|
167
172
|
# -p, --parameter NAME=VALUE # Set a pig parameter value in your script.
|
168
173
|
# -f, --param-file PARAMFILE # Load pig parameter values from a file.
|
174
|
+
# -g, --pigversion PIG_VERSION # Set pig version. Options are <PIG_VERSION_OPTIONS>.
|
169
175
|
# --project-root PROJECTDIR # The root directory of the project if not the CWD
|
170
176
|
#
|
171
177
|
#Examples:
|
@@ -188,7 +194,7 @@ class Mortar::Command::Local < Mortar::Command::Base
|
|
188
194
|
|
189
195
|
script = validate_script!(script_name)
|
190
196
|
ctrl = Mortar::Local::Controller.new
|
191
|
-
ctrl.validate(script, pig_parameters)
|
197
|
+
ctrl.validate(script, pig_version, pig_parameters)
|
192
198
|
end
|
193
199
|
|
194
200
|
end
|
@@ -29,6 +29,7 @@ class Mortar::Command::Validate < Mortar::Command::Base
|
|
29
29
|
#
|
30
30
|
# -p, --parameter NAME=VALUE # Set a pig parameter value in your script.
|
31
31
|
# -f, --param-file PARAMFILE # Load pig parameter values from a file.
|
32
|
+
# -g, --pigversion PIG_VERSION # Set pig version. Options are <PIG_VERSION_OPTIONS>.
|
32
33
|
#
|
33
34
|
def index
|
34
35
|
pigscript_name = shift_argument
|
@@ -46,7 +47,7 @@ class Mortar::Command::Validate < Mortar::Command::Base
|
|
46
47
|
|
47
48
|
validate_id = nil
|
48
49
|
action("Starting validate") do
|
49
|
-
validate_id = api.post_validate(project.name, pigscript.name, git_ref, :parameters => pig_parameters).body["validate_id"]
|
50
|
+
validate_id = api.post_validate(project.name, pigscript.name, git_ref, :pig_version => pig_version, :pig_version => pig_version.version, :parameters => pig_parameters).body["validate_id"]
|
50
51
|
end
|
51
52
|
|
52
53
|
validate_result = nil
|
@@ -13,6 +13,7 @@
|
|
13
13
|
#
|
14
14
|
|
15
15
|
require "mortar/helpers"
|
16
|
+
require "mortar/pigversion"
|
16
17
|
require "mortar/local/pig"
|
17
18
|
require "mortar/local/java"
|
18
19
|
require "mortar/local/python"
|
@@ -76,14 +77,18 @@ EOF
|
|
76
77
|
|
77
78
|
# Main entry point to perform installation and configuration necessary
|
78
79
|
# to run pig on the users local machine
|
79
|
-
def install_and_configure
|
80
|
+
def install_and_configure(pig_version=nil)
|
81
|
+
#To support old watchtower plugins we'll accept nil pig_version
|
82
|
+
if pig_version.nil?
|
83
|
+
pig_version = Mortar::PigVersion::Pig09.new
|
84
|
+
end
|
80
85
|
java = Mortar::Local::Java.new()
|
81
86
|
unless java.check_install
|
82
87
|
error(NO_JAVA_ERROR_MESSAGE)
|
83
88
|
end
|
84
89
|
|
85
90
|
pig = Mortar::Local::Pig.new()
|
86
|
-
pig.install_or_update()
|
91
|
+
pig.install_or_update(pig_version)
|
87
92
|
|
88
93
|
py = Mortar::Local::Python.new()
|
89
94
|
unless py.check_or_install
|
@@ -132,25 +137,25 @@ EOF
|
|
132
137
|
end
|
133
138
|
|
134
139
|
# Main entry point for user running a pig script
|
135
|
-
def run(pig_script, pig_parameters)
|
140
|
+
def run(pig_script, pig_version, pig_parameters)
|
136
141
|
require_aws_keys
|
137
|
-
install_and_configure
|
142
|
+
install_and_configure(pig_version)
|
138
143
|
pig = Mortar::Local::Pig.new()
|
139
|
-
pig.run_script(pig_script, pig_parameters)
|
144
|
+
pig.run_script(pig_script, pig_version, pig_parameters)
|
140
145
|
end
|
141
146
|
|
142
147
|
# Main entry point for illustrating a pig alias
|
143
|
-
def illustrate(pig_script, pig_alias, pig_parameters, skip_pruning)
|
148
|
+
def illustrate(pig_script, pig_alias, pig_version, pig_parameters, skip_pruning)
|
144
149
|
require_aws_keys
|
145
|
-
install_and_configure
|
150
|
+
install_and_configure(pig_version)
|
146
151
|
pig = Mortar::Local::Pig.new()
|
147
|
-
pig.illustrate_alias(pig_script, pig_alias, skip_pruning, pig_parameters)
|
152
|
+
pig.illustrate_alias(pig_script, pig_alias, skip_pruning, pig_version, pig_parameters)
|
148
153
|
end
|
149
154
|
|
150
|
-
def validate(pig_script, pig_parameters)
|
151
|
-
install_and_configure
|
155
|
+
def validate(pig_script, pig_version, pig_parameters)
|
156
|
+
install_and_configure(pig_version)
|
152
157
|
pig = Mortar::Local::Pig.new()
|
153
|
-
pig.validate_script(pig_script, pig_parameters)
|
158
|
+
pig.validate_script(pig_script, pig_version, pig_parameters)
|
154
159
|
end
|
155
160
|
|
156
161
|
end
|
data/lib/mortar/local/pig.rb
CHANGED
@@ -23,8 +23,6 @@ class Mortar::Local::Pig
|
|
23
23
|
include Mortar::Local::InstallUtil
|
24
24
|
|
25
25
|
PIG_LOG_FORMAT = "humanreadable"
|
26
|
-
PIG_TGZ_NAME = "pig-0.9.tar.gz"
|
27
|
-
PIG_TGZ_DEFAULT_URL_PATH = "resource/pig_0_9"
|
28
26
|
LIB_TGZ_NAME = "lib-common.tar.gz"
|
29
27
|
PIG_COMMON_LIB_URL_PATH = "resource/lib_common"
|
30
28
|
|
@@ -78,21 +76,21 @@ class Mortar::Local::Pig
|
|
78
76
|
}
|
79
77
|
end
|
80
78
|
|
81
|
-
def command
|
82
|
-
return File.join(pig_directory, "bin", "pig")
|
79
|
+
def command(pig_version)
|
80
|
+
return File.join(pig_directory(pig_version), "bin", "pig")
|
83
81
|
end
|
84
82
|
|
85
|
-
def pig_directory
|
86
|
-
return File.join(local_install_directory,
|
83
|
+
def pig_directory(pig_version)
|
84
|
+
return File.join(local_install_directory, pig_version.name)
|
87
85
|
end
|
88
86
|
|
89
87
|
def lib_directory
|
90
88
|
return File.join(local_install_directory, "lib-common")
|
91
89
|
end
|
92
90
|
|
93
|
-
def pig_archive_url
|
91
|
+
def pig_archive_url(pig_version)
|
94
92
|
full_host = (host =~ /^http/) ? host : "https://api.#{host}"
|
95
|
-
default_url = full_host + "/" +
|
93
|
+
default_url = full_host + "/" + pig_version.tgz_default_url_path
|
96
94
|
ENV.fetch('PIG_DISTRO_URL', default_url)
|
97
95
|
end
|
98
96
|
|
@@ -103,8 +101,8 @@ class Mortar::Local::Pig
|
|
103
101
|
end
|
104
102
|
|
105
103
|
# Determines if a pig install needs to occur, true if no pig install present
|
106
|
-
def should_do_pig_install?
|
107
|
-
not (File.exists?(pig_directory))
|
104
|
+
def should_do_pig_install?(pig_version)
|
105
|
+
not (File.exists?(pig_directory(pig_version)))
|
108
106
|
end
|
109
107
|
|
110
108
|
def should_do_lib_install?
|
@@ -113,22 +111,22 @@ class Mortar::Local::Pig
|
|
113
111
|
|
114
112
|
# Determines if a pig install needs to occur, true if server side
|
115
113
|
# pig tgz is newer than date of the existing install
|
116
|
-
def should_do_pig_update?
|
117
|
-
return is_newer_version(
|
114
|
+
def should_do_pig_update?(pig_version)
|
115
|
+
return is_newer_version(pig_version.name, pig_archive_url(pig_version))
|
118
116
|
end
|
119
117
|
|
120
118
|
def should_do_lib_update?
|
121
119
|
return is_newer_version('lib-common', lib_archive_url)
|
122
120
|
end
|
123
121
|
|
124
|
-
def install_or_update()
|
125
|
-
if should_do_pig_install?
|
126
|
-
action "Installing
|
127
|
-
install_pig()
|
122
|
+
def install_or_update(pig_version)
|
123
|
+
if should_do_pig_install?(pig_version)
|
124
|
+
action "Installing #{pig_version.name} to #{local_install_directory_name}" do
|
125
|
+
install_pig(pig_version)
|
128
126
|
end
|
129
|
-
elsif should_do_pig_update?
|
130
|
-
action "Updating to latest
|
131
|
-
install_pig()
|
127
|
+
elsif should_do_pig_update?(pig_version)
|
128
|
+
action "Updating to latest #{pig_version.name} in #{local_install_directory_name}" do
|
129
|
+
install_pig(pig_version)
|
132
130
|
end
|
133
131
|
end
|
134
132
|
|
@@ -144,23 +142,23 @@ class Mortar::Local::Pig
|
|
144
142
|
end
|
145
143
|
|
146
144
|
# Installs pig for this project if it is not already present
|
147
|
-
def install_pig
|
145
|
+
def install_pig(pig_version)
|
148
146
|
#Delete the directory if it already exists to ensure cruft isn't left around.
|
149
|
-
if File.directory? pig_directory
|
150
|
-
FileUtils.rm_rf pig_directory
|
147
|
+
if File.directory? pig_directory(pig_version)
|
148
|
+
FileUtils.rm_rf pig_directory(pig_version)
|
151
149
|
end
|
152
150
|
|
153
151
|
FileUtils.mkdir_p(local_install_directory)
|
154
|
-
local_tgz = File.join(local_install_directory,
|
155
|
-
download_file(pig_archive_url, local_tgz)
|
152
|
+
local_tgz = File.join(local_install_directory, pig_version.tgz_name)
|
153
|
+
download_file(pig_archive_url(pig_version), local_tgz)
|
156
154
|
extract_tgz(local_tgz, local_install_directory)
|
157
155
|
|
158
156
|
# This has been seening coming out of the tgz w/o +x so we do
|
159
157
|
# here to be sure it has the necessary permissions
|
160
|
-
FileUtils.chmod(0755, command)
|
158
|
+
FileUtils.chmod(0755, command(pig_version))
|
161
159
|
|
162
160
|
File.delete(local_tgz)
|
163
|
-
note_install(
|
161
|
+
note_install(pig_version.name)
|
164
162
|
end
|
165
163
|
|
166
164
|
def install_lib
|
@@ -178,14 +176,14 @@ class Mortar::Local::Pig
|
|
178
176
|
note_install("lib-common")
|
179
177
|
end
|
180
178
|
|
181
|
-
def validate_script(pig_script, pig_parameters)
|
182
|
-
run_pig_command(" -check #{pig_script.path}", pig_parameters)
|
179
|
+
def validate_script(pig_script, pig_version, pig_parameters)
|
180
|
+
run_pig_command(" -check #{pig_script.path}", pig_version, pig_parameters)
|
183
181
|
end
|
184
182
|
|
185
183
|
|
186
184
|
# run the pig script with user supplied pig parameters
|
187
|
-
def run_script(pig_script, pig_parameters)
|
188
|
-
run_pig_command(" -f #{pig_script.path}", pig_parameters, true)
|
185
|
+
def run_script(pig_script, pig_version, pig_parameters)
|
186
|
+
run_pig_command(" -f #{pig_script.path}", pig_version, pig_parameters, true)
|
189
187
|
end
|
190
188
|
|
191
189
|
# Create a temp file to be used for writing the illustrate
|
@@ -256,7 +254,7 @@ class Mortar::Local::Pig
|
|
256
254
|
return params
|
257
255
|
end
|
258
256
|
|
259
|
-
def illustrate_alias(pig_script, pig_alias, skip_pruning, pig_parameters)
|
257
|
+
def illustrate_alias(pig_script, pig_alias, skip_pruning, pig_version, pig_parameters)
|
260
258
|
cmd = "-e 'illustrate "
|
261
259
|
|
262
260
|
# Parameters have to be entered with the illustrate command (as
|
@@ -279,7 +277,7 @@ class Mortar::Local::Pig
|
|
279
277
|
cmd += " #{pig_alias} "
|
280
278
|
end
|
281
279
|
|
282
|
-
result = run_pig_command(cmd, [], false)
|
280
|
+
result = run_pig_command(cmd, pig_version, [], false)
|
283
281
|
if result
|
284
282
|
show_illustrate_output(illustrate_outpath)
|
285
283
|
end
|
@@ -288,12 +286,12 @@ class Mortar::Local::Pig
|
|
288
286
|
# Run pig with the specified command ('command' is anything that
|
289
287
|
# can be appended to the command line invocation of Pig that will
|
290
288
|
# get it to do something interesting, such as '-f some-file.pig'
|
291
|
-
def run_pig_command(cmd, parameters = nil, jython_output = true)
|
289
|
+
def run_pig_command(cmd, pig_version, parameters = nil, jython_output = true)
|
292
290
|
unset_hadoop_env_vars
|
293
291
|
reset_local_logs
|
294
292
|
# Generate the script for running the command, then
|
295
293
|
# write it to a temp script which will be exectued
|
296
|
-
script_text = script_for_command(cmd, parameters)
|
294
|
+
script_text = script_for_command(cmd, pig_version, parameters)
|
297
295
|
script = Tempfile.new("mortar-")
|
298
296
|
script.write(script_text)
|
299
297
|
script.close(false)
|
@@ -320,8 +318,8 @@ class Mortar::Local::Pig
|
|
320
318
|
|
321
319
|
# Generates a bash script which sets up the necessary environment and
|
322
320
|
# then runs the pig command
|
323
|
-
def script_for_command(cmd, parameters, jython_output = true)
|
324
|
-
template_params = pig_command_script_template_parameters(cmd, parameters)
|
321
|
+
def script_for_command(cmd, pig_version, parameters, jython_output = true)
|
322
|
+
template_params = pig_command_script_template_parameters(cmd, pig_version, parameters)
|
325
323
|
template_params['pig_opts']['jython.output'] = jython_output
|
326
324
|
erb = ERB.new(File.read(pig_command_script_template_path), 0, "%<>")
|
327
325
|
erb.result(BindingClazz.new(template_params).get_binding)
|
@@ -332,8 +330,12 @@ class Mortar::Local::Pig
|
|
332
330
|
File.expand_path("../../templates/script/runpig.sh", __FILE__)
|
333
331
|
end
|
334
332
|
|
335
|
-
def template_params_classpath
|
336
|
-
|
333
|
+
def template_params_classpath(pig_version=nil)
|
334
|
+
# Need to support old watchtower plugins that don't set pig_version
|
335
|
+
if pig_version.nil?
|
336
|
+
pig_version = Mortar::PigVersion::Pig09.new
|
337
|
+
end
|
338
|
+
"#{pig_directory(pig_version)}/*:#{pig_directory(pig_version)}/lib-local/*:#{lib_directory}/lib-local/*:#{pig_directory(pig_version)}/lib-pig/*:#{pig_directory(pig_version)}/lib-cluster/*:#{lib_directory}/lib-pig/*:#{lib_directory}/lib-cluster/*:#{jython_directory}/jython.jar:#{lib_directory}/conf/jets3t.properties"
|
337
339
|
end
|
338
340
|
|
339
341
|
def log4j_conf
|
@@ -341,11 +343,12 @@ class Mortar::Local::Pig
|
|
341
343
|
end
|
342
344
|
|
343
345
|
# Parameters necessary for rendering the bash script template
|
344
|
-
def pig_command_script_template_parameters(cmd, pig_parameters)
|
346
|
+
def pig_command_script_template_parameters(cmd, pig_version, pig_parameters)
|
345
347
|
template_params = {}
|
346
348
|
template_params['pig_params_file'] = make_pig_param_file(pig_parameters)
|
347
|
-
template_params['
|
348
|
-
template_params['
|
349
|
+
template_params['pig_dir'] = pig_version.name
|
350
|
+
template_params['pig_home'] = pig_directory(pig_version)
|
351
|
+
template_params['pig_classpath'] = "#{pig_directory(pig_version)}/lib-local/*:#{lib_directory}/lib-local/*:#{pig_directory(pig_version)}/lib-pig/*:#{pig_directory(pig_version)}/lib-cluster/*:#{lib_directory}/lib-pig/*:#{lib_directory}/lib-cluster/*:#{jython_directory}/jython.jar"
|
349
352
|
template_params['classpath'] = template_params_classpath
|
350
353
|
template_params['log4j_conf'] = log4j_conf
|
351
354
|
template_params['project_home'] = File.expand_path("..", local_install_directory)
|