mortar 0.11.1 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/mortar/command.rb +2 -0
- data/lib/mortar/command/base.rb +46 -1
- data/lib/mortar/command/describe.rb +8 -4
- data/lib/mortar/command/illustrate.rb +2 -1
- data/lib/mortar/command/jobs.rb +3 -0
- data/lib/mortar/command/local.rb +12 -6
- data/lib/mortar/command/validate.rb +2 -1
- data/lib/mortar/local/controller.rb +16 -11
- data/lib/mortar/local/pig.rb +44 -41
- data/lib/mortar/pigversion.rb +71 -0
- data/lib/mortar/templates/script/runpig.sh +1 -1
- data/lib/mortar/version.rb +1 -1
- data/spec/mortar/command/base_spec.rb +116 -0
- data/spec/mortar/command/describe_spec.rb +4 -4
- data/spec/mortar/command/illustrate_spec.rb +7 -7
- data/spec/mortar/command/jobs_spec.rb +14 -3
- data/spec/mortar/command/local_spec.rb +10 -10
- data/spec/mortar/command/validate_spec.rb +4 -4
- data/spec/mortar/local/controller_spec.rb +29 -6
- data/spec/mortar/local/pig_spec.rb +42 -18
- data/spec/spec_helper.rb +6 -1
- metadata +180 -132
- checksums.yaml +0 -7
data/lib/mortar/command.rb
CHANGED
@@ -127,6 +127,7 @@ module Mortar
|
|
127
127
|
global_option :remote, "--remote REMOTE"
|
128
128
|
global_option :polling_interval, "--polling_interval SECONDS", "-p"
|
129
129
|
global_option :no_browser, "--no_browser"
|
130
|
+
global_option :pigversion, "--pigversion PIG_VERSION", "-g"
|
130
131
|
|
131
132
|
def self.prepare_run(cmd, args=[])
|
132
133
|
command = parse(cmd)
|
@@ -176,6 +177,7 @@ module Mortar
|
|
176
177
|
opts[global_option[:name]] = value
|
177
178
|
end
|
178
179
|
end
|
180
|
+
|
179
181
|
command[:options].each do |name, option|
|
180
182
|
parser.on("-#{option[:short]}", "--#{option[:long]}", option[:desc]) do |value|
|
181
183
|
opt_name_sym = name.gsub("-", "_").to_sym
|
data/lib/mortar/command/base.rb
CHANGED
@@ -18,8 +18,10 @@
|
|
18
18
|
#
|
19
19
|
|
20
20
|
require "fileutils"
|
21
|
+
require "parseconfig"
|
21
22
|
require "mortar/auth"
|
22
23
|
require "mortar/command"
|
24
|
+
require "mortar/pigversion"
|
23
25
|
require "mortar/project"
|
24
26
|
require "mortar/git"
|
25
27
|
|
@@ -36,6 +38,11 @@ class Mortar::Command::Base
|
|
36
38
|
def initialize(args=[], options={})
|
37
39
|
@args = args
|
38
40
|
@options = options
|
41
|
+
#We never want to override the command line options so we store them.
|
42
|
+
@original_options = options.dup
|
43
|
+
|
44
|
+
#Initialize defaults from .mortar-defaults
|
45
|
+
load_defaults('DEFAULTS')
|
39
46
|
end
|
40
47
|
|
41
48
|
def project
|
@@ -181,15 +188,28 @@ protected
|
|
181
188
|
end
|
182
189
|
end
|
183
190
|
|
191
|
+
def self.replace_templates(help)
|
192
|
+
#Leave --pigversion undocumented for now.
|
193
|
+
#help.each do |line|
|
194
|
+
# #line.gsub!("<PIG_VERSION_OPTIONS>", "0.9 (default) and 0.12 (beta)")
|
195
|
+
#
|
196
|
+
#end
|
197
|
+
help.reject! do |line|
|
198
|
+
line.include?("<PIG_VERSION_OPTIONS>")
|
199
|
+
end
|
200
|
+
end
|
201
|
+
|
184
202
|
def self.method_added(method)
|
185
203
|
return if self == Mortar::Command::Base
|
186
204
|
return if private_method_defined?(method)
|
187
205
|
return if protected_method_defined?(method)
|
188
206
|
|
189
207
|
help = extract_help_from_caller(caller.first)
|
208
|
+
replace_templates(help)
|
209
|
+
|
190
210
|
resolved_method = (method.to_s == "index") ? nil : method.to_s
|
191
211
|
command = [ self.namespace, resolved_method ].compact.join(":")
|
192
|
-
banner = extract_banner(help) || command
|
212
|
+
banner = extract_banner(help) || command
|
193
213
|
|
194
214
|
Mortar::Command.register_command(
|
195
215
|
:klass => self,
|
@@ -324,6 +344,9 @@ protected
|
|
324
344
|
error("Naming conflict. #{script_name} refers to both a pigscript and a controlscript. Please rename scripts to avoid conflicts.")
|
325
345
|
end
|
326
346
|
|
347
|
+
#While validating we can load the defaults that are relevant to this script.
|
348
|
+
load_defaults(shortened_script_name)
|
349
|
+
|
327
350
|
pigscript or controlscript
|
328
351
|
end
|
329
352
|
|
@@ -333,6 +356,10 @@ protected
|
|
333
356
|
available_scripts = project.pigscripts.none? ? "No pigscripts found" : "Available scripts:\n#{project.pigscripts.collect{|k,v| v.executable_path}.sort.join("\n")}"
|
334
357
|
error("Unable to find pigscript #{pigscript_name}\n#{available_scripts}")
|
335
358
|
end
|
359
|
+
|
360
|
+
#While validating we can load the defaults that are relevant to this script.
|
361
|
+
load_defaults(shortened_pigscript_name)
|
362
|
+
|
336
363
|
pigscript
|
337
364
|
end
|
338
365
|
|
@@ -345,6 +372,19 @@ protected
|
|
345
372
|
return missing_dir ? nil : [File.basename(Dir.getwd), nil]
|
346
373
|
end
|
347
374
|
|
375
|
+
def load_defaults(section_name)
|
376
|
+
if File.exists?('.mortar-defaults')
|
377
|
+
default_options = ParseConfig.new('.mortar-defaults')
|
378
|
+
if default_options.groups.include?(section_name)
|
379
|
+
default_options[section_name].each do |k, v|
|
380
|
+
unless @original_options.include? k.to_sym
|
381
|
+
@options[k.to_sym] = v
|
382
|
+
end
|
383
|
+
end
|
384
|
+
end
|
385
|
+
end
|
386
|
+
end
|
387
|
+
|
348
388
|
def extract_project_in_dir(project_name=nil)
|
349
389
|
# returns [project_name, remote_name]
|
350
390
|
# TODO refactor this very messy method
|
@@ -398,6 +438,11 @@ protected
|
|
398
438
|
(options[:no_browser])
|
399
439
|
end
|
400
440
|
|
441
|
+
def pig_version
|
442
|
+
pig_version_str = options[:pigversion] || '0.9'
|
443
|
+
pig_version = Mortar::PigVersion.from_string(pig_version_str)
|
444
|
+
end
|
445
|
+
|
401
446
|
def sync_code_with_cloud
|
402
447
|
# returns git_ref
|
403
448
|
if project.embedded_project?
|
@@ -28,6 +28,7 @@ class Mortar::Command::Describe < Mortar::Command::Base
|
|
28
28
|
#
|
29
29
|
# -p, --parameter NAME=VALUE # Set a pig parameter value in your script.
|
30
30
|
# -f, --param-file PARAMFILE # Load pig parameter values from a file.
|
31
|
+
# -g, --pigversion PIG_VERSION # Set pig version. Options are <PIG_VERSION_OPTIONS>.
|
31
32
|
#
|
32
33
|
# Examples:
|
33
34
|
#
|
@@ -51,7 +52,7 @@ class Mortar::Command::Describe < Mortar::Command::Base
|
|
51
52
|
|
52
53
|
describe_id = nil
|
53
54
|
action("Starting describe") do
|
54
|
-
describe_id = api.post_describe(project.name, pigscript.name, alias_name, git_ref, :parameters => pig_parameters).body["describe_id"]
|
55
|
+
describe_id = api.post_describe(project.name, pigscript.name, alias_name, git_ref, :pig_version => pig_version.version, :parameters => pig_parameters).body["describe_id"]
|
55
56
|
end
|
56
57
|
|
57
58
|
describe_result = nil
|
@@ -86,9 +87,12 @@ class Mortar::Command::Describe < Mortar::Command::Base
|
|
86
87
|
when Mortar::API::Describe::STATUS_SUCCESS
|
87
88
|
web_result_url = describe_result['web_result_url']
|
88
89
|
display("Results available at #{web_result_url}")
|
89
|
-
|
90
|
-
|
91
|
-
|
90
|
+
|
91
|
+
unless no_browser?
|
92
|
+
action("Opening web browser to show results") do
|
93
|
+
require "launchy"
|
94
|
+
Launchy.open(web_result_url).join
|
95
|
+
end
|
92
96
|
end
|
93
97
|
else
|
94
98
|
raise RuntimeError, "Unknown describe status: #{describe_result['status']} for describe_id: #{describe_id}"
|
@@ -29,6 +29,7 @@ class Mortar::Command::Illustrate < Mortar::Command::Base
|
|
29
29
|
# -s, --skippruning # Don't try to reduce the illustrate results to the smallest size possible.
|
30
30
|
# -p, --parameter NAME=VALUE # Set a pig parameter value in your script.
|
31
31
|
# -f, --param-file PARAMFILE # Load pig parameter values from a file.
|
32
|
+
# -g, --pigversion PIG_VERSION # Set pig version. Options are <PIG_VERSION_OPTIONS>.
|
32
33
|
# --no_browser # Don't open the illustrate results automatically in the browser.
|
33
34
|
#
|
34
35
|
# Examples:
|
@@ -56,7 +57,7 @@ class Mortar::Command::Illustrate < Mortar::Command::Base
|
|
56
57
|
|
57
58
|
illustrate_id = nil
|
58
59
|
action("Starting illustrate") do
|
59
|
-
illustrate_id = api.post_illustrate(project.name, pigscript.name, alias_name, skip_pruning, git_ref, :parameters => pig_parameters).body["illustrate_id"]
|
60
|
+
illustrate_id = api.post_illustrate(project.name, pigscript.name, alias_name, skip_pruning, git_ref, :pig_version => pig_version.version, :parameters => pig_parameters).body["illustrate_id"]
|
60
61
|
end
|
61
62
|
|
62
63
|
illustrate_result = nil
|
data/lib/mortar/command/jobs.rb
CHANGED
@@ -67,6 +67,7 @@ class Mortar::Command::Jobs < Mortar::Command::Base
|
|
67
67
|
# -d, --donotnotify # Don't send an email on job completion. (Default: false--an email will be sent to you once the job completes)
|
68
68
|
# -P, --project PROJECTNAME # Use a project that is not checked out in the current directory. Runs code from project's master branch in github rather than snapshotting local code.
|
69
69
|
# -B, --branch BRANCHNAME # Used with --project to specify a non-master branch
|
70
|
+
# -g, --pigversion PIG_VERSION # Set pig version. Options are <PIG_VERSION_OPTIONS>.
|
70
71
|
#
|
71
72
|
#Examples:
|
72
73
|
#
|
@@ -162,6 +163,7 @@ class Mortar::Command::Jobs < Mortar::Command::Base
|
|
162
163
|
cluster_type = CLUSTER_TYPE__PERMANENT
|
163
164
|
end
|
164
165
|
api.post_job_new_cluster(project_name, script_name, git_ref, cluster_size,
|
166
|
+
:pig_version => pig_version.version,
|
165
167
|
:parameters => pig_parameters,
|
166
168
|
:cluster_type => cluster_type,
|
167
169
|
:notify_on_job_finish => notify_on_job_finish,
|
@@ -169,6 +171,7 @@ class Mortar::Command::Jobs < Mortar::Command::Base
|
|
169
171
|
else
|
170
172
|
cluster_id = options[:clusterid]
|
171
173
|
api.post_job_existing_cluster(project_name, script_name, git_ref, cluster_id,
|
174
|
+
:pig_version => pig_version.version,
|
172
175
|
:parameters => pig_parameters,
|
173
176
|
:notify_on_job_finish => notify_on_job_finish,
|
174
177
|
:is_control_script => is_control_script).body
|
data/lib/mortar/command/local.rb
CHANGED
@@ -27,9 +27,11 @@ class Mortar::Command::Local < Mortar::Command::Base
|
|
27
27
|
#
|
28
28
|
# Install dependencies for running this mortar project locally - other mortar:local commands will also perform this step automatically.
|
29
29
|
#
|
30
|
-
#
|
30
|
+
# -g, --pigversion PIG_VERSION # Set pig version. Options are <PIG_VERSION_OPTIONS>.
|
31
|
+
# --project-root PROJECTDIR # The root directory of the project if not the CWD
|
31
32
|
#
|
32
33
|
def configure
|
34
|
+
validate_arguments!
|
33
35
|
|
34
36
|
# cd into the project root
|
35
37
|
project_root = options[:project_root] ||= Dir.getwd
|
@@ -39,7 +41,7 @@ class Mortar::Command::Local < Mortar::Command::Base
|
|
39
41
|
Dir.chdir(project_root)
|
40
42
|
|
41
43
|
ctrl = Mortar::Local::Controller.new
|
42
|
-
ctrl.install_and_configure
|
44
|
+
ctrl.install_and_configure(pig_version)
|
43
45
|
end
|
44
46
|
|
45
47
|
# local:run SCRIPT
|
@@ -48,6 +50,7 @@ class Mortar::Command::Local < Mortar::Command::Base
|
|
48
50
|
#
|
49
51
|
# -p, --parameter NAME=VALUE # Set a pig parameter value in your script.
|
50
52
|
# -f, --param-file PARAMFILE # Load pig parameter values from a file.
|
53
|
+
# -g, --pigversion PIG_VERSION # Set pig version. Options are <PIG_VERSION_OPTIONS>.
|
51
54
|
# --project-root PROJECTDIR # The root directory of the project if not the CWD
|
52
55
|
#
|
53
56
|
#Examples:
|
@@ -69,7 +72,7 @@ class Mortar::Command::Local < Mortar::Command::Base
|
|
69
72
|
Dir.chdir(project_root)
|
70
73
|
script = validate_script!(script_name)
|
71
74
|
ctrl = Mortar::Local::Controller.new
|
72
|
-
ctrl.run(script, pig_parameters)
|
75
|
+
ctrl.run(script, pig_version, pig_parameters)
|
73
76
|
end
|
74
77
|
|
75
78
|
# local:characterize -f PARAMFILE
|
@@ -79,6 +82,7 @@ class Mortar::Command::Local < Mortar::Command::Base
|
|
79
82
|
# statistics about your data (most common values, percent null, etc.)
|
80
83
|
#
|
81
84
|
# -f, --param-file PARAMFILE # Load pig parameter values from a file
|
85
|
+
# -g, --pigversion PIG_VERSION # Set pig version. Options are <PIG_VERSION_OPTIONS>.
|
82
86
|
#
|
83
87
|
# Load some data and emit statistics.
|
84
88
|
# PARAMFILE (Required):
|
@@ -116,7 +120,7 @@ class Mortar::Command::Local < Mortar::Command::Base
|
|
116
120
|
gen.generate_characterize
|
117
121
|
script = validate_script!(controlscript_name)
|
118
122
|
ctrl = Mortar::Local::Controller.new
|
119
|
-
ctrl.run(script, pig_parameters)
|
123
|
+
ctrl.run(script, pig_version, pig_parameters)
|
120
124
|
gen.cleanup_characterize(project_root)
|
121
125
|
end
|
122
126
|
|
@@ -129,6 +133,7 @@ class Mortar::Command::Local < Mortar::Command::Base
|
|
129
133
|
# -s, --skippruning # Don't try to reduce the illustrate results to the smallest size possible.
|
130
134
|
# -p, --parameter NAME=VALUE # Set a pig parameter value in your script.
|
131
135
|
# -f, --param-file PARAMFILE # Load pig parameter values from a file.
|
136
|
+
# -g, --pigversion PIG_VERSION # Set pig version. Options are <PIG_VERSION_OPTIONS>.
|
132
137
|
# --no_browser # Don't open the illustrate results automatically in the browser.
|
133
138
|
# --project-root PROJECTDIR # The root directory of the project if not the CWD
|
134
139
|
#
|
@@ -156,7 +161,7 @@ class Mortar::Command::Local < Mortar::Command::Base
|
|
156
161
|
pigscript = validate_pigscript!(pigscript_name)
|
157
162
|
|
158
163
|
ctrl = Mortar::Local::Controller.new
|
159
|
-
ctrl.illustrate(pigscript, alias_name, pig_parameters, skip_pruning)
|
164
|
+
ctrl.illustrate(pigscript, alias_name, pig_version, pig_parameters, skip_pruning)
|
160
165
|
end
|
161
166
|
|
162
167
|
|
@@ -166,6 +171,7 @@ class Mortar::Command::Local < Mortar::Command::Base
|
|
166
171
|
#
|
167
172
|
# -p, --parameter NAME=VALUE # Set a pig parameter value in your script.
|
168
173
|
# -f, --param-file PARAMFILE # Load pig parameter values from a file.
|
174
|
+
# -g, --pigversion PIG_VERSION # Set pig version. Options are <PIG_VERSION_OPTIONS>.
|
169
175
|
# --project-root PROJECTDIR # The root directory of the project if not the CWD
|
170
176
|
#
|
171
177
|
#Examples:
|
@@ -188,7 +194,7 @@ class Mortar::Command::Local < Mortar::Command::Base
|
|
188
194
|
|
189
195
|
script = validate_script!(script_name)
|
190
196
|
ctrl = Mortar::Local::Controller.new
|
191
|
-
ctrl.validate(script, pig_parameters)
|
197
|
+
ctrl.validate(script, pig_version, pig_parameters)
|
192
198
|
end
|
193
199
|
|
194
200
|
end
|
@@ -29,6 +29,7 @@ class Mortar::Command::Validate < Mortar::Command::Base
|
|
29
29
|
#
|
30
30
|
# -p, --parameter NAME=VALUE # Set a pig parameter value in your script.
|
31
31
|
# -f, --param-file PARAMFILE # Load pig parameter values from a file.
|
32
|
+
# -g, --pigversion PIG_VERSION # Set pig version. Options are <PIG_VERSION_OPTIONS>.
|
32
33
|
#
|
33
34
|
def index
|
34
35
|
pigscript_name = shift_argument
|
@@ -46,7 +47,7 @@ class Mortar::Command::Validate < Mortar::Command::Base
|
|
46
47
|
|
47
48
|
validate_id = nil
|
48
49
|
action("Starting validate") do
|
49
|
-
validate_id = api.post_validate(project.name, pigscript.name, git_ref, :parameters => pig_parameters).body["validate_id"]
|
50
|
+
validate_id = api.post_validate(project.name, pigscript.name, git_ref, :pig_version => pig_version, :pig_version => pig_version.version, :parameters => pig_parameters).body["validate_id"]
|
50
51
|
end
|
51
52
|
|
52
53
|
validate_result = nil
|
@@ -13,6 +13,7 @@
|
|
13
13
|
#
|
14
14
|
|
15
15
|
require "mortar/helpers"
|
16
|
+
require "mortar/pigversion"
|
16
17
|
require "mortar/local/pig"
|
17
18
|
require "mortar/local/java"
|
18
19
|
require "mortar/local/python"
|
@@ -76,14 +77,18 @@ EOF
|
|
76
77
|
|
77
78
|
# Main entry point to perform installation and configuration necessary
|
78
79
|
# to run pig on the users local machine
|
79
|
-
def install_and_configure
|
80
|
+
def install_and_configure(pig_version=nil)
|
81
|
+
#To support old watchtower plugins we'll accept nil pig_version
|
82
|
+
if pig_version.nil?
|
83
|
+
pig_version = Mortar::PigVersion::Pig09.new
|
84
|
+
end
|
80
85
|
java = Mortar::Local::Java.new()
|
81
86
|
unless java.check_install
|
82
87
|
error(NO_JAVA_ERROR_MESSAGE)
|
83
88
|
end
|
84
89
|
|
85
90
|
pig = Mortar::Local::Pig.new()
|
86
|
-
pig.install_or_update()
|
91
|
+
pig.install_or_update(pig_version)
|
87
92
|
|
88
93
|
py = Mortar::Local::Python.new()
|
89
94
|
unless py.check_or_install
|
@@ -132,25 +137,25 @@ EOF
|
|
132
137
|
end
|
133
138
|
|
134
139
|
# Main entry point for user running a pig script
|
135
|
-
def run(pig_script, pig_parameters)
|
140
|
+
def run(pig_script, pig_version, pig_parameters)
|
136
141
|
require_aws_keys
|
137
|
-
install_and_configure
|
142
|
+
install_and_configure(pig_version)
|
138
143
|
pig = Mortar::Local::Pig.new()
|
139
|
-
pig.run_script(pig_script, pig_parameters)
|
144
|
+
pig.run_script(pig_script, pig_version, pig_parameters)
|
140
145
|
end
|
141
146
|
|
142
147
|
# Main entry point for illustrating a pig alias
|
143
|
-
def illustrate(pig_script, pig_alias, pig_parameters, skip_pruning)
|
148
|
+
def illustrate(pig_script, pig_alias, pig_version, pig_parameters, skip_pruning)
|
144
149
|
require_aws_keys
|
145
|
-
install_and_configure
|
150
|
+
install_and_configure(pig_version)
|
146
151
|
pig = Mortar::Local::Pig.new()
|
147
|
-
pig.illustrate_alias(pig_script, pig_alias, skip_pruning, pig_parameters)
|
152
|
+
pig.illustrate_alias(pig_script, pig_alias, skip_pruning, pig_version, pig_parameters)
|
148
153
|
end
|
149
154
|
|
150
|
-
def validate(pig_script, pig_parameters)
|
151
|
-
install_and_configure
|
155
|
+
def validate(pig_script, pig_version, pig_parameters)
|
156
|
+
install_and_configure(pig_version)
|
152
157
|
pig = Mortar::Local::Pig.new()
|
153
|
-
pig.validate_script(pig_script, pig_parameters)
|
158
|
+
pig.validate_script(pig_script, pig_version, pig_parameters)
|
154
159
|
end
|
155
160
|
|
156
161
|
end
|
data/lib/mortar/local/pig.rb
CHANGED
@@ -23,8 +23,6 @@ class Mortar::Local::Pig
|
|
23
23
|
include Mortar::Local::InstallUtil
|
24
24
|
|
25
25
|
PIG_LOG_FORMAT = "humanreadable"
|
26
|
-
PIG_TGZ_NAME = "pig-0.9.tar.gz"
|
27
|
-
PIG_TGZ_DEFAULT_URL_PATH = "resource/pig_0_9"
|
28
26
|
LIB_TGZ_NAME = "lib-common.tar.gz"
|
29
27
|
PIG_COMMON_LIB_URL_PATH = "resource/lib_common"
|
30
28
|
|
@@ -78,21 +76,21 @@ class Mortar::Local::Pig
|
|
78
76
|
}
|
79
77
|
end
|
80
78
|
|
81
|
-
def command
|
82
|
-
return File.join(pig_directory, "bin", "pig")
|
79
|
+
def command(pig_version)
|
80
|
+
return File.join(pig_directory(pig_version), "bin", "pig")
|
83
81
|
end
|
84
82
|
|
85
|
-
def pig_directory
|
86
|
-
return File.join(local_install_directory,
|
83
|
+
def pig_directory(pig_version)
|
84
|
+
return File.join(local_install_directory, pig_version.name)
|
87
85
|
end
|
88
86
|
|
89
87
|
def lib_directory
|
90
88
|
return File.join(local_install_directory, "lib-common")
|
91
89
|
end
|
92
90
|
|
93
|
-
def pig_archive_url
|
91
|
+
def pig_archive_url(pig_version)
|
94
92
|
full_host = (host =~ /^http/) ? host : "https://api.#{host}"
|
95
|
-
default_url = full_host + "/" +
|
93
|
+
default_url = full_host + "/" + pig_version.tgz_default_url_path
|
96
94
|
ENV.fetch('PIG_DISTRO_URL', default_url)
|
97
95
|
end
|
98
96
|
|
@@ -103,8 +101,8 @@ class Mortar::Local::Pig
|
|
103
101
|
end
|
104
102
|
|
105
103
|
# Determines if a pig install needs to occur, true if no pig install present
|
106
|
-
def should_do_pig_install?
|
107
|
-
not (File.exists?(pig_directory))
|
104
|
+
def should_do_pig_install?(pig_version)
|
105
|
+
not (File.exists?(pig_directory(pig_version)))
|
108
106
|
end
|
109
107
|
|
110
108
|
def should_do_lib_install?
|
@@ -113,22 +111,22 @@ class Mortar::Local::Pig
|
|
113
111
|
|
114
112
|
# Determines if a pig install needs to occur, true if server side
|
115
113
|
# pig tgz is newer than date of the existing install
|
116
|
-
def should_do_pig_update?
|
117
|
-
return is_newer_version(
|
114
|
+
def should_do_pig_update?(pig_version)
|
115
|
+
return is_newer_version(pig_version.name, pig_archive_url(pig_version))
|
118
116
|
end
|
119
117
|
|
120
118
|
def should_do_lib_update?
|
121
119
|
return is_newer_version('lib-common', lib_archive_url)
|
122
120
|
end
|
123
121
|
|
124
|
-
def install_or_update()
|
125
|
-
if should_do_pig_install?
|
126
|
-
action "Installing
|
127
|
-
install_pig()
|
122
|
+
def install_or_update(pig_version)
|
123
|
+
if should_do_pig_install?(pig_version)
|
124
|
+
action "Installing #{pig_version.name} to #{local_install_directory_name}" do
|
125
|
+
install_pig(pig_version)
|
128
126
|
end
|
129
|
-
elsif should_do_pig_update?
|
130
|
-
action "Updating to latest
|
131
|
-
install_pig()
|
127
|
+
elsif should_do_pig_update?(pig_version)
|
128
|
+
action "Updating to latest #{pig_version.name} in #{local_install_directory_name}" do
|
129
|
+
install_pig(pig_version)
|
132
130
|
end
|
133
131
|
end
|
134
132
|
|
@@ -144,23 +142,23 @@ class Mortar::Local::Pig
|
|
144
142
|
end
|
145
143
|
|
146
144
|
# Installs pig for this project if it is not already present
|
147
|
-
def install_pig
|
145
|
+
def install_pig(pig_version)
|
148
146
|
#Delete the directory if it already exists to ensure cruft isn't left around.
|
149
|
-
if File.directory? pig_directory
|
150
|
-
FileUtils.rm_rf pig_directory
|
147
|
+
if File.directory? pig_directory(pig_version)
|
148
|
+
FileUtils.rm_rf pig_directory(pig_version)
|
151
149
|
end
|
152
150
|
|
153
151
|
FileUtils.mkdir_p(local_install_directory)
|
154
|
-
local_tgz = File.join(local_install_directory,
|
155
|
-
download_file(pig_archive_url, local_tgz)
|
152
|
+
local_tgz = File.join(local_install_directory, pig_version.tgz_name)
|
153
|
+
download_file(pig_archive_url(pig_version), local_tgz)
|
156
154
|
extract_tgz(local_tgz, local_install_directory)
|
157
155
|
|
158
156
|
# This has been seening coming out of the tgz w/o +x so we do
|
159
157
|
# here to be sure it has the necessary permissions
|
160
|
-
FileUtils.chmod(0755, command)
|
158
|
+
FileUtils.chmod(0755, command(pig_version))
|
161
159
|
|
162
160
|
File.delete(local_tgz)
|
163
|
-
note_install(
|
161
|
+
note_install(pig_version.name)
|
164
162
|
end
|
165
163
|
|
166
164
|
def install_lib
|
@@ -178,14 +176,14 @@ class Mortar::Local::Pig
|
|
178
176
|
note_install("lib-common")
|
179
177
|
end
|
180
178
|
|
181
|
-
def validate_script(pig_script, pig_parameters)
|
182
|
-
run_pig_command(" -check #{pig_script.path}", pig_parameters)
|
179
|
+
def validate_script(pig_script, pig_version, pig_parameters)
|
180
|
+
run_pig_command(" -check #{pig_script.path}", pig_version, pig_parameters)
|
183
181
|
end
|
184
182
|
|
185
183
|
|
186
184
|
# run the pig script with user supplied pig parameters
|
187
|
-
def run_script(pig_script, pig_parameters)
|
188
|
-
run_pig_command(" -f #{pig_script.path}", pig_parameters, true)
|
185
|
+
def run_script(pig_script, pig_version, pig_parameters)
|
186
|
+
run_pig_command(" -f #{pig_script.path}", pig_version, pig_parameters, true)
|
189
187
|
end
|
190
188
|
|
191
189
|
# Create a temp file to be used for writing the illustrate
|
@@ -256,7 +254,7 @@ class Mortar::Local::Pig
|
|
256
254
|
return params
|
257
255
|
end
|
258
256
|
|
259
|
-
def illustrate_alias(pig_script, pig_alias, skip_pruning, pig_parameters)
|
257
|
+
def illustrate_alias(pig_script, pig_alias, skip_pruning, pig_version, pig_parameters)
|
260
258
|
cmd = "-e 'illustrate "
|
261
259
|
|
262
260
|
# Parameters have to be entered with the illustrate command (as
|
@@ -279,7 +277,7 @@ class Mortar::Local::Pig
|
|
279
277
|
cmd += " #{pig_alias} "
|
280
278
|
end
|
281
279
|
|
282
|
-
result = run_pig_command(cmd, [], false)
|
280
|
+
result = run_pig_command(cmd, pig_version, [], false)
|
283
281
|
if result
|
284
282
|
show_illustrate_output(illustrate_outpath)
|
285
283
|
end
|
@@ -288,12 +286,12 @@ class Mortar::Local::Pig
|
|
288
286
|
# Run pig with the specified command ('command' is anything that
|
289
287
|
# can be appended to the command line invocation of Pig that will
|
290
288
|
# get it to do something interesting, such as '-f some-file.pig'
|
291
|
-
def run_pig_command(cmd, parameters = nil, jython_output = true)
|
289
|
+
def run_pig_command(cmd, pig_version, parameters = nil, jython_output = true)
|
292
290
|
unset_hadoop_env_vars
|
293
291
|
reset_local_logs
|
294
292
|
# Generate the script for running the command, then
|
295
293
|
# write it to a temp script which will be exectued
|
296
|
-
script_text = script_for_command(cmd, parameters)
|
294
|
+
script_text = script_for_command(cmd, pig_version, parameters)
|
297
295
|
script = Tempfile.new("mortar-")
|
298
296
|
script.write(script_text)
|
299
297
|
script.close(false)
|
@@ -320,8 +318,8 @@ class Mortar::Local::Pig
|
|
320
318
|
|
321
319
|
# Generates a bash script which sets up the necessary environment and
|
322
320
|
# then runs the pig command
|
323
|
-
def script_for_command(cmd, parameters, jython_output = true)
|
324
|
-
template_params = pig_command_script_template_parameters(cmd, parameters)
|
321
|
+
def script_for_command(cmd, pig_version, parameters, jython_output = true)
|
322
|
+
template_params = pig_command_script_template_parameters(cmd, pig_version, parameters)
|
325
323
|
template_params['pig_opts']['jython.output'] = jython_output
|
326
324
|
erb = ERB.new(File.read(pig_command_script_template_path), 0, "%<>")
|
327
325
|
erb.result(BindingClazz.new(template_params).get_binding)
|
@@ -332,8 +330,12 @@ class Mortar::Local::Pig
|
|
332
330
|
File.expand_path("../../templates/script/runpig.sh", __FILE__)
|
333
331
|
end
|
334
332
|
|
335
|
-
def template_params_classpath
|
336
|
-
|
333
|
+
def template_params_classpath(pig_version=nil)
|
334
|
+
# Need to support old watchtower plugins that don't set pig_version
|
335
|
+
if pig_version.nil?
|
336
|
+
pig_version = Mortar::PigVersion::Pig09.new
|
337
|
+
end
|
338
|
+
"#{pig_directory(pig_version)}/*:#{pig_directory(pig_version)}/lib-local/*:#{lib_directory}/lib-local/*:#{pig_directory(pig_version)}/lib-pig/*:#{pig_directory(pig_version)}/lib-cluster/*:#{lib_directory}/lib-pig/*:#{lib_directory}/lib-cluster/*:#{jython_directory}/jython.jar:#{lib_directory}/conf/jets3t.properties"
|
337
339
|
end
|
338
340
|
|
339
341
|
def log4j_conf
|
@@ -341,11 +343,12 @@ class Mortar::Local::Pig
|
|
341
343
|
end
|
342
344
|
|
343
345
|
# Parameters necessary for rendering the bash script template
|
344
|
-
def pig_command_script_template_parameters(cmd, pig_parameters)
|
346
|
+
def pig_command_script_template_parameters(cmd, pig_version, pig_parameters)
|
345
347
|
template_params = {}
|
346
348
|
template_params['pig_params_file'] = make_pig_param_file(pig_parameters)
|
347
|
-
template_params['
|
348
|
-
template_params['
|
349
|
+
template_params['pig_dir'] = pig_version.name
|
350
|
+
template_params['pig_home'] = pig_directory(pig_version)
|
351
|
+
template_params['pig_classpath'] = "#{pig_directory(pig_version)}/lib-local/*:#{lib_directory}/lib-local/*:#{pig_directory(pig_version)}/lib-pig/*:#{pig_directory(pig_version)}/lib-cluster/*:#{lib_directory}/lib-pig/*:#{lib_directory}/lib-cluster/*:#{jython_directory}/jython.jar"
|
349
352
|
template_params['classpath'] = template_params_classpath
|
350
353
|
template_params['log4j_conf'] = log4j_conf
|
351
354
|
template_params['project_home'] = File.expand_path("..", local_install_directory)
|