mortar 0.6.2 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/mortar/auth.rb +4 -0
- data/lib/mortar/command/base.rb +16 -0
- data/lib/mortar/command/describe.rb +7 -1
- data/lib/mortar/command/help.rb +1 -22
- data/lib/mortar/command/illustrate.rb +8 -3
- data/lib/mortar/command/jobs.rb +41 -14
- data/lib/mortar/command/local.rb +87 -0
- data/lib/mortar/command/projects.rb +12 -11
- data/lib/mortar/command/validate.rb +6 -1
- data/lib/mortar/helpers.rb +12 -0
- data/lib/mortar/local/controller.rb +108 -0
- data/lib/mortar/local/installutil.rb +94 -0
- data/lib/mortar/local/java.rb +48 -0
- data/lib/mortar/local/pig.rb +309 -0
- data/lib/mortar/local/python.rb +184 -0
- data/lib/mortar/project.rb +35 -9
- data/lib/mortar/templates/project/gitignore +3 -1
- data/lib/mortar/templates/report/illustrate-report.html +96 -0
- data/lib/mortar/templates/script/runpig.sh +23 -0
- data/lib/mortar/version.rb +1 -1
- data/spec/mortar/auth_spec.rb +8 -0
- data/spec/mortar/command/describe_spec.rb +14 -1
- data/spec/mortar/command/illustrate_spec.rb +14 -1
- data/spec/mortar/command/jobs_spec.rb +125 -7
- data/spec/mortar/command/local_spec.rb +144 -0
- data/spec/mortar/command/validate_spec.rb +14 -1
- data/spec/mortar/local/controller_spec.rb +102 -0
- data/spec/mortar/local/installutil_spec.rb +70 -0
- data/spec/mortar/local/java_spec.rb +62 -0
- data/spec/mortar/local/pig_spec.rb +157 -0
- data/spec/mortar/project_spec.rb +11 -0
- data/spec/spec_helper.rb +1 -0
- metadata +152 -130
data/lib/mortar/auth.rb
CHANGED
@@ -88,6 +88,10 @@ class Mortar::Auth
|
|
88
88
|
get_credentials[1]
|
89
89
|
end
|
90
90
|
|
91
|
+
def user_s3_safe
|
92
|
+
return user.gsub(/[^0-9a-zA-Z]/i, '-')
|
93
|
+
end
|
94
|
+
|
91
95
|
def api_key(user = get_credentials[0], password = get_credentials[1])
|
92
96
|
require("mortar-api-ruby")
|
93
97
|
api = Mortar::API.new(default_params)
|
data/lib/mortar/command/base.rb
CHANGED
@@ -243,6 +243,22 @@ protected
|
|
243
243
|
error("Unable to find git remote for project #{project.name}")
|
244
244
|
end
|
245
245
|
end
|
246
|
+
|
247
|
+
def validate_script!(script_name)
|
248
|
+
pigscript = project.pigscripts[script_name]
|
249
|
+
controlscript = project.controlscripts[script_name]
|
250
|
+
unless pigscript || controlscript
|
251
|
+
available_pigscripts = project.pigscripts.none? ? "No pigscripts found" : "Available pigscripts:\n#{project.pigscripts.keys.sort.join("\n")}"
|
252
|
+
available_controlscripts = project.controlscripts.none? ? "No controlscripts found" : "Available controlscripts:\n#{project.controlscripts.keys.sort.join("\n")}"
|
253
|
+
error("Unable to find a pigscript or controlscript for #{script_name}\n\n#{available_pigscripts}\n\n#{available_controlscripts}")
|
254
|
+
end
|
255
|
+
|
256
|
+
if pigscript && controlscript
|
257
|
+
error("Naming conflict. #{script_name} refers to both a pigscript and a controlscript. Please rename scripts to avoid conflicts.")
|
258
|
+
end
|
259
|
+
|
260
|
+
pigscript or controlscript
|
261
|
+
end
|
246
262
|
|
247
263
|
def validate_pigscript!(pigscript_name)
|
248
264
|
unless pigscript = project.pigscripts[pigscript_name]
|
@@ -39,9 +39,15 @@ class Mortar::Command::Describe < Mortar::Command::Base
|
|
39
39
|
unless pigscript_name && alias_name
|
40
40
|
error("Usage: mortar describe PIGSCRIPT ALIAS\nMust specify PIGSCRIPT and ALIAS.")
|
41
41
|
end
|
42
|
+
|
42
43
|
validate_arguments!
|
44
|
+
pigscript = validate_script!(pigscript_name)
|
45
|
+
|
46
|
+
if pigscript.is_a? Mortar::Project::ControlScript
|
47
|
+
error "Currently Mortar does not support describing control scripts"
|
48
|
+
end
|
49
|
+
|
43
50
|
validate_git_based_project!
|
44
|
-
pigscript = validate_pigscript!(pigscript_name)
|
45
51
|
git_ref = git.create_and_push_snapshot_branch(project)
|
46
52
|
|
47
53
|
describe_id = nil
|
data/lib/mortar/command/help.rb
CHANGED
@@ -67,24 +67,6 @@ private
|
|
67
67
|
Mortar::Command.commands
|
68
68
|
end
|
69
69
|
|
70
|
-
def legacy_help_for_namespace(namespace)
|
71
|
-
instance = Mortar::Command::Help.groups.map do |group|
|
72
|
-
[ group.title, group.select { |c| c.first =~ /^#{namespace}/ }.length ]
|
73
|
-
end.sort_by { |l| l.last }.last
|
74
|
-
return nil unless instance
|
75
|
-
return nil if instance.last.zero?
|
76
|
-
instance.first
|
77
|
-
end
|
78
|
-
|
79
|
-
def legacy_help_for_command(command)
|
80
|
-
Mortar::Command::Help.groups.each do |group|
|
81
|
-
group.each do |cmd, description|
|
82
|
-
return description if cmd.split(" ").first == command
|
83
|
-
end
|
84
|
-
end
|
85
|
-
nil
|
86
|
-
end
|
87
|
-
|
88
70
|
def primary_namespaces
|
89
71
|
PRIMARY_NAMESPACES.map { |name| namespaces[name] }.compact
|
90
72
|
end
|
@@ -97,7 +79,6 @@ private
|
|
97
79
|
size = longest(namespaces.map { |n| n[:name] })
|
98
80
|
namespaces.sort_by {|namespace| namespace[:name]}.each do |namespace|
|
99
81
|
name = namespace[:name]
|
100
|
-
namespace[:description] ||= legacy_help_for_namespace(name)
|
101
82
|
puts " %-#{size}s # %s" % [ name, namespace[:description] ]
|
102
83
|
end
|
103
84
|
end
|
@@ -121,8 +102,6 @@ private
|
|
121
102
|
unless namespace_commands.empty?
|
122
103
|
size = longest(namespace_commands.map { |c| c[:banner] })
|
123
104
|
namespace_commands.sort_by { |c| c[:banner].to_s }.each do |command|
|
124
|
-
next if command[:help] =~ /DEPRECATED/
|
125
|
-
command[:summary] ||= legacy_help_for_command(command[:command])
|
126
105
|
puts " %-#{size}s # %s" % [ command[:banner], command[:summary] ]
|
127
106
|
end
|
128
107
|
end
|
@@ -140,7 +119,7 @@ private
|
|
140
119
|
puts command[:help].split("\n")[1..-1].join("\n")
|
141
120
|
else
|
142
121
|
puts
|
143
|
-
|
122
|
+
error "No help available for #{command[:command]}. Please contact us at support@mortardata.com for assistance."
|
144
123
|
end
|
145
124
|
puts
|
146
125
|
end
|
@@ -40,16 +40,21 @@ class Mortar::Command::Illustrate < Mortar::Command::Base
|
|
40
40
|
alias_name = shift_argument
|
41
41
|
skip_pruning = options[:skippruning] ||= false
|
42
42
|
|
43
|
+
validate_arguments!
|
44
|
+
pigscript = validate_script!(pigscript_name)
|
45
|
+
|
43
46
|
# TODO: When illustrating without alias works, remove the `&& alias_name` to re-enable the feature on CLI
|
44
47
|
unless pigscript_name && alias_name
|
45
48
|
error("Usage: mortar illustrate PIGSCRIPT ALIAS\nMust specify PIGSCRIPT and ALIAS.")
|
46
49
|
end
|
47
50
|
|
48
|
-
|
51
|
+
if pigscript.is_a? Mortar::Project::ControlScript
|
52
|
+
error "Currently Mortar does not support illustrating control scripts"
|
53
|
+
end
|
54
|
+
|
49
55
|
validate_git_based_project!
|
50
|
-
pigscript = validate_pigscript!(pigscript_name)
|
51
56
|
git_ref = git.create_and_push_snapshot_branch(project)
|
52
|
-
|
57
|
+
|
53
58
|
illustrate_id = nil
|
54
59
|
action("Starting illustrate") do
|
55
60
|
illustrate_id = api.post_illustrate(project.name, pigscript.name, alias_name, skip_pruning, git_ref, :parameters => pig_parameters).body["illustrate_id"]
|
data/lib/mortar/command/jobs.rb
CHANGED
@@ -50,7 +50,7 @@ class Mortar::Command::Jobs < Mortar::Command::Base
|
|
50
50
|
display_table(jobs, columns, headers)
|
51
51
|
end
|
52
52
|
|
53
|
-
# jobs:run
|
53
|
+
# jobs:run SCRIPT
|
54
54
|
#
|
55
55
|
# Run a job on a Mortar Hadoop cluster.
|
56
56
|
#
|
@@ -66,11 +66,23 @@ class Mortar::Command::Jobs < Mortar::Command::Base
|
|
66
66
|
# Run the generate_regression_model_coefficients script on a 3 node cluster.
|
67
67
|
# $ mortar jobs:run generate_regression_model_coefficients --clustersize 3
|
68
68
|
def run
|
69
|
-
|
70
|
-
unless
|
71
|
-
error("Usage: mortar jobs:run
|
69
|
+
script_name = shift_argument
|
70
|
+
unless script_name
|
71
|
+
error("Usage: mortar jobs:run SCRIPT\nMust specify SCRIPT.")
|
72
72
|
end
|
73
|
+
|
73
74
|
validate_arguments!
|
75
|
+
script = validate_script!(script_name)
|
76
|
+
|
77
|
+
case script
|
78
|
+
when Mortar::Project::PigScript
|
79
|
+
is_control_script = false
|
80
|
+
when Mortar::Project::ControlScript
|
81
|
+
is_control_script = true
|
82
|
+
else
|
83
|
+
error "Unknown Script Type"
|
84
|
+
end
|
85
|
+
|
74
86
|
|
75
87
|
unless options[:clusterid] || options[:clustersize]
|
76
88
|
clusters = api.get_clusters().body['clusters']
|
@@ -98,27 +110,28 @@ class Mortar::Command::Jobs < Mortar::Command::Base
|
|
98
110
|
end
|
99
111
|
|
100
112
|
validate_git_based_project!
|
101
|
-
pigscript = validate_pigscript!(pigscript_name)
|
102
113
|
git_ref = git.create_and_push_snapshot_branch(project)
|
103
114
|
notify_on_job_finish = ! options[:donotnotify]
|
104
115
|
|
105
|
-
# post job to API
|
116
|
+
# post job to API
|
106
117
|
response = action("Requesting job execution") do
|
107
118
|
if options[:clustersize]
|
108
119
|
cluster_size = options[:clustersize].to_i
|
109
120
|
keepalive = ! options[:singlejobcluster]
|
110
|
-
api.post_job_new_cluster(project.name,
|
121
|
+
api.post_job_new_cluster(project.name, script.name, git_ref, cluster_size,
|
111
122
|
:parameters => pig_parameters,
|
112
123
|
:keepalive => keepalive,
|
113
|
-
:notify_on_job_finish => notify_on_job_finish
|
124
|
+
:notify_on_job_finish => notify_on_job_finish,
|
125
|
+
:is_control_script => is_control_script).body
|
114
126
|
else
|
115
127
|
cluster_id = options[:clusterid]
|
116
|
-
api.post_job_existing_cluster(project.name,
|
128
|
+
api.post_job_existing_cluster(project.name, script.name, git_ref, cluster_id,
|
117
129
|
:parameters => pig_parameters,
|
118
|
-
:notify_on_job_finish => notify_on_job_finish
|
130
|
+
:notify_on_job_finish => notify_on_job_finish,
|
131
|
+
:is_control_script => is_control_script).body
|
119
132
|
end
|
120
133
|
end
|
121
|
-
|
134
|
+
|
122
135
|
display("job_id: #{response['job_id']}")
|
123
136
|
display
|
124
137
|
display("Job status can be viewed on the web at:\n\n #{response['web_job_url']}")
|
@@ -146,7 +159,6 @@ class Mortar::Command::Jobs < Mortar::Command::Base
|
|
146
159
|
def display_job_status(job_status)
|
147
160
|
job_display_entries = {
|
148
161
|
"status" => job_status["status_description"],
|
149
|
-
"progress" => "#{job_status["progress"]}%",
|
150
162
|
"cluster_id" => job_status["cluster_id"],
|
151
163
|
"job submitted at" => job_status["start_timestamp"],
|
152
164
|
"job began running at" => job_status["running_timestamp"],
|
@@ -167,8 +179,13 @@ class Mortar::Command::Jobs < Mortar::Command::Base
|
|
167
179
|
end
|
168
180
|
|
169
181
|
if job_status["num_hadoop_jobs"] && job_status["num_hadoop_jobs_succeeded"]
|
182
|
+
job_display_entries["progress"] = "#{job_status["progress"]}%"
|
170
183
|
job_display_entries["hadoop jobs complete"] =
|
171
184
|
'%0.2f / %0.2f' % [job_status["num_hadoop_jobs_succeeded"], job_status["num_hadoop_jobs"]]
|
185
|
+
elsif job_status["num_hadoop_jobs_succeeded"]
|
186
|
+
job_display_entries["progress"] = '%0.2f MapReduce Jobs complete.' % job_status["num_hadoop_jobs_succeeded"]
|
187
|
+
else
|
188
|
+
job_display_entries["progress"] = "#{job_status["progress"]}%"
|
172
189
|
end
|
173
190
|
|
174
191
|
if job_status["outputs"] && job_status["outputs"].length > 0
|
@@ -180,7 +197,13 @@ class Mortar::Command::Jobs < Mortar::Command::Base
|
|
180
197
|
end]
|
181
198
|
end
|
182
199
|
|
183
|
-
|
200
|
+
if job_status["controlscript_name"]
|
201
|
+
script_name = job_status["controlscript_name"]
|
202
|
+
else
|
203
|
+
script_name = job_status["pigscript_name"]
|
204
|
+
end
|
205
|
+
|
206
|
+
styled_header("#{job_status["project_name"]}: #{script_name} (job_id: #{job_status["job_id"]})")
|
184
207
|
styled_hash(job_display_entries)
|
185
208
|
end
|
186
209
|
|
@@ -196,7 +219,7 @@ class Mortar::Command::Jobs < Mortar::Command::Base
|
|
196
219
|
end
|
197
220
|
|
198
221
|
# If the job is running show the progress bar
|
199
|
-
if job_status["status_code"] == Mortar::API::Jobs::STATUS_RUNNING
|
222
|
+
if job_status["status_code"] == Mortar::API::Jobs::STATUS_RUNNING && job_status["num_hadoop_jobs"]
|
200
223
|
progressbar = "=" + ("=" * (job_status["progress"].to_i / 5)) + ">"
|
201
224
|
|
202
225
|
if job_status["num_hadoop_jobs"] && job_status["num_hadoop_jobs_succeeded"]
|
@@ -206,6 +229,10 @@ class Mortar::Command::Jobs < Mortar::Command::Base
|
|
206
229
|
|
207
230
|
printf("\r[#{spinner(ticks)}] Status: [%-22s] %s%% Complete (%s MapReduce jobs finished)", progressbar, job_status["progress"], hadoop_jobs_ratio_complete)
|
208
231
|
|
232
|
+
elsif job_status["status_code"] == Mortar::API::Jobs::STATUS_RUNNING
|
233
|
+
jobs_complete = '%0.2f' % job_status["num_hadoop_jobs_succeeded"]
|
234
|
+
printf("\r[#{spinner(ticks)}] #{jobs_complete} MapReduce Jobs complete.")
|
235
|
+
|
209
236
|
# If the job is not complete, but not in the running state, just display its status
|
210
237
|
else
|
211
238
|
redisplay("[#{spinner(ticks)}] Status: #{job_status['status_description']}")
|
@@ -0,0 +1,87 @@
|
|
1
|
+
#
|
2
|
+
# Copyright 2012 Mortar Data Inc.
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require "mortar/local/controller"
|
18
|
+
require "mortar/command/base"
|
19
|
+
|
20
|
+
# run select pig commands on your local machine
|
21
|
+
#
|
22
|
+
class Mortar::Command::Local < Mortar::Command::Base
|
23
|
+
|
24
|
+
|
25
|
+
# configure
|
26
|
+
#
|
27
|
+
# Install dependencies for running this pig project locally, other
|
28
|
+
# commands will also perform this step automatically.
|
29
|
+
#
|
30
|
+
def configure
|
31
|
+
ctrl = Mortar::Local::Controller.new
|
32
|
+
ctrl.install_and_configure
|
33
|
+
end
|
34
|
+
|
35
|
+
# local:run PIGSCRIPT
|
36
|
+
#
|
37
|
+
# Run a job on your local machine
|
38
|
+
#
|
39
|
+
# -p, --parameter NAME=VALUE # Set a pig parameter value in your script.
|
40
|
+
# -f, --param-file PARAMFILE # Load pig parameter values from a file.
|
41
|
+
#
|
42
|
+
#Examples:
|
43
|
+
#
|
44
|
+
# Run the generate_regression_model_coefficients script locally.
|
45
|
+
# $ mortar local:run generate_regression_model_coefficients
|
46
|
+
def run
|
47
|
+
pigscript_name = shift_argument
|
48
|
+
unless pigscript_name
|
49
|
+
error("Usage: mortar local:run PIGSCRIPT\nMust specify PIGSCRIPT.")
|
50
|
+
end
|
51
|
+
validate_arguments!
|
52
|
+
pigscript = validate_pigscript!(pigscript_name)
|
53
|
+
ctrl = Mortar::Local::Controller.new
|
54
|
+
ctrl.run(pigscript, pig_parameters)
|
55
|
+
end
|
56
|
+
|
57
|
+
# illustrate [PIGSCRIPT] [ALIAS]
|
58
|
+
#
|
59
|
+
# Locallay illustrate the effects and output of a pigscript.
|
60
|
+
#
|
61
|
+
# -s, --skippruning # Don't try to reduce the illustrate results to the smallest size possible.
|
62
|
+
# -p, --parameter NAME=VALUE # Set a pig parameter value in your script.
|
63
|
+
# -f, --param-file PARAMFILE # Load pig parameter values from a file.
|
64
|
+
# --no_browser # Don't open the illustrate results automatically in the browser.
|
65
|
+
#
|
66
|
+
# Examples:
|
67
|
+
#
|
68
|
+
# Illustrate the songs_sample relation in the generate_regression_model_coefficients script.
|
69
|
+
# $ mortar illustrate generate_regression_model_coefficients songs_sample
|
70
|
+
def illustrate
|
71
|
+
pigscript_name = shift_argument
|
72
|
+
alias_name = shift_argument
|
73
|
+
skip_pruning = options[:skippruning] ||= false
|
74
|
+
|
75
|
+
unless pigscript_name && alias_name
|
76
|
+
error("Usage: mortar local:illustrate PIGSCRIPT ALIAS\nMust specify PIGSCRIPT and ALIAS.")
|
77
|
+
end
|
78
|
+
|
79
|
+
validate_arguments!
|
80
|
+
pigscript = validate_pigscript!(pigscript_name)
|
81
|
+
|
82
|
+
ctrl = Mortar::Local::Controller.new
|
83
|
+
ctrl.illustrate(pigscript, alias_name, pig_parameters, skip_pruning)
|
84
|
+
end
|
85
|
+
|
86
|
+
|
87
|
+
end
|
@@ -26,7 +26,7 @@ class Mortar::Command::Projects < Mortar::Command::Base
|
|
26
26
|
|
27
27
|
# projects
|
28
28
|
#
|
29
|
-
# Display the available set of projects
|
29
|
+
# Display the available set of Mortar projects.
|
30
30
|
def index
|
31
31
|
validate_arguments!
|
32
32
|
projects = api.get_projects().body["projects"]
|
@@ -40,7 +40,7 @@ class Mortar::Command::Projects < Mortar::Command::Base
|
|
40
40
|
|
41
41
|
# projects:delete PROJECTNAME
|
42
42
|
#
|
43
|
-
# Delete
|
43
|
+
# Delete the Mortar project PROJECTNAME.
|
44
44
|
def delete
|
45
45
|
name = shift_argument
|
46
46
|
unless name
|
@@ -58,7 +58,7 @@ class Mortar::Command::Projects < Mortar::Command::Base
|
|
58
58
|
|
59
59
|
# projects:create PROJECTNAME
|
60
60
|
#
|
61
|
-
#
|
61
|
+
# Used when you want to start a new Mortar project using Mortar generated code.
|
62
62
|
def create
|
63
63
|
name = shift_argument
|
64
64
|
unless name
|
@@ -75,9 +75,9 @@ class Mortar::Command::Projects < Mortar::Command::Base
|
|
75
75
|
end
|
76
76
|
alias_command "new", "projects:create"
|
77
77
|
|
78
|
-
# projects:register
|
78
|
+
# projects:register PROJECTNAME
|
79
79
|
#
|
80
|
-
#
|
80
|
+
# Used when you want to start a new Mortar project using your existing code in the current directory.
|
81
81
|
def register
|
82
82
|
name = shift_argument
|
83
83
|
unless name
|
@@ -145,10 +145,12 @@ class Mortar::Command::Projects < Mortar::Command::Base
|
|
145
145
|
end
|
146
146
|
alias_command "register", "projects:register"
|
147
147
|
|
148
|
-
# projects:set_remote
|
149
|
-
#
|
150
|
-
# Adds the Mortar remote to the local git project. This is necessary for successfully executing many of the Mortar commands.
|
148
|
+
# projects:set_remote PROJECTNAME
|
151
149
|
#
|
150
|
+
# Used after you checkout code for an existing Mortar project from a non-Mortar git repository.
|
151
|
+
# Adds a remote to your local git repository to the Mortar git repository. For example if a
|
152
|
+
# co-worker creates a Mortar project from an internal repository you would clone the internal
|
153
|
+
# repository and then after cloning call mortar projects:set_remote.
|
152
154
|
def set_remote
|
153
155
|
project_name = shift_argument
|
154
156
|
|
@@ -176,10 +178,9 @@ class Mortar::Command::Projects < Mortar::Command::Base
|
|
176
178
|
|
177
179
|
end
|
178
180
|
|
179
|
-
# projects:clone
|
180
|
-
#
|
181
|
-
# clone the mortar project PROJECT into the current directory.
|
181
|
+
# projects:clone PROJECTNAME
|
182
182
|
#
|
183
|
+
# Used when you want to clone an existing Mortar project into the current directory.
|
183
184
|
def clone
|
184
185
|
name = shift_argument
|
185
186
|
unless name
|
@@ -36,8 +36,13 @@ class Mortar::Command::Validate < Mortar::Command::Base
|
|
36
36
|
error("Usage: mortar validate PIGSCRIPT\nMust specify PIGSCRIPT.")
|
37
37
|
end
|
38
38
|
validate_arguments!
|
39
|
+
pigscript = validate_script!(pigscript_name)
|
40
|
+
|
41
|
+
if pigscript.is_a? Mortar::Project::ControlScript
|
42
|
+
error "Currently Mortar does not support validating control scripts"
|
43
|
+
end
|
44
|
+
|
39
45
|
validate_git_based_project!
|
40
|
-
pigscript = validate_pigscript!(pigscript_name)
|
41
46
|
git_ref = git.create_and_push_snapshot_branch(project)
|
42
47
|
|
43
48
|
validate_id = nil
|
data/lib/mortar/helpers.rb
CHANGED
@@ -494,6 +494,18 @@ module Mortar
|
|
494
494
|
end
|
495
495
|
end
|
496
496
|
|
497
|
+
def ensure_dir_exists(dir)
|
498
|
+
unless Dir.exists? dir
|
499
|
+
Dir.mkdir(dir)
|
500
|
+
end
|
501
|
+
end
|
502
|
+
|
503
|
+
def copy_if_not_present_at_dest(res_src, res_dest)
|
504
|
+
unless File.exists?(res_dest)
|
505
|
+
FileUtils.cp(res_src, res_dest)
|
506
|
+
end
|
507
|
+
end
|
508
|
+
|
497
509
|
private
|
498
510
|
|
499
511
|
def create_display_method(name, colour_code, new_line=true)
|
@@ -0,0 +1,108 @@
|
|
1
|
+
#
|
2
|
+
# Copyright 2012 Mortar Data Inc.
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require "mortar/helpers"
|
18
|
+
require "mortar/local/pig"
|
19
|
+
require "mortar/local/java"
|
20
|
+
require "mortar/local/python"
|
21
|
+
|
22
|
+
|
23
|
+
class Mortar::Local::Controller
|
24
|
+
include Mortar::Helpers
|
25
|
+
|
26
|
+
NO_JAVA_ERROR_MESSAGE = <<EOF
|
27
|
+
A suitable java installation could not be found. If you already have java installed
|
28
|
+
please set your JAVA_HOME environment variable before continuing. Otherwise, a
|
29
|
+
suitable java installation will need to be added to your local system.
|
30
|
+
|
31
|
+
Installing Java
|
32
|
+
On OSX run `javac` from the command line. This will intiate the installation. For
|
33
|
+
Linux systems please consult the documentation on your relevant package manager.
|
34
|
+
EOF
|
35
|
+
|
36
|
+
NO_PYTHON_ERROR_MESSAGE = <<EOF
|
37
|
+
pA suitable python installation with virtualenv could not be located. Please ensure
|
38
|
+
you have python 2.6+ installed on your local system. If you need to obtain a copy
|
39
|
+
of virtualenv it can be located here:
|
40
|
+
https://pypi.python.org/pypi/virtualenv
|
41
|
+
EOF
|
42
|
+
|
43
|
+
NO_AWS_KEYS_ERROR_MESSAGE = <<EOF
|
44
|
+
Please specify your aws access key via enviroment variable AWS_ACCESS_KEY
|
45
|
+
and your aws secret key via enviroment variable AWS_SECRET_KEY"
|
46
|
+
EOF
|
47
|
+
|
48
|
+
|
49
|
+
# Checks if the user has properly specified their AWS keys
|
50
|
+
def verify_aws_keys()
|
51
|
+
if (not (ENV['AWS_ACCESS_KEY'] and ENV['AWS_SECRET_KEY'])) then
|
52
|
+
if not ENV['MORTAR_IGNORE_AWS_KEYS']
|
53
|
+
return false
|
54
|
+
else
|
55
|
+
return true
|
56
|
+
end
|
57
|
+
else
|
58
|
+
return true
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
# Exits with a helpful message if the user has not setup their aws keys
|
63
|
+
def require_aws_keys()
|
64
|
+
unless verify_aws_keys()
|
65
|
+
error(NO_AWS_KEYS_ERROR_MESSAGE)
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
# Main entry point to perform installation and configuration necessary
|
70
|
+
# to run pig on the users local machine
|
71
|
+
def install_and_configure
|
72
|
+
java = Mortar::Local::Java.new()
|
73
|
+
unless java.check_install
|
74
|
+
error(NO_JAVA_ERROR_MESSAGE)
|
75
|
+
end
|
76
|
+
|
77
|
+
pig = Mortar::Local::Pig.new()
|
78
|
+
pig.install()
|
79
|
+
|
80
|
+
py = Mortar::Local::Python.new()
|
81
|
+
unless py.check_or_install
|
82
|
+
error(NO_PYTHON_ERROR_MESSAGE)
|
83
|
+
end
|
84
|
+
|
85
|
+
unless py.setup_project_python_environment
|
86
|
+
msg = "\nUnable to setup a python environment with your dependencies, "
|
87
|
+
msg += "see #{py.pip_error_log_path} for more details"
|
88
|
+
error(msg)
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
# Main entry point for user running a pig script
|
93
|
+
def run(pig_script, pig_parameters)
|
94
|
+
require_aws_keys
|
95
|
+
install_and_configure
|
96
|
+
pig = Mortar::Local::Pig.new()
|
97
|
+
pig.run_script(pig_script, pig_parameters)
|
98
|
+
end
|
99
|
+
|
100
|
+
# Main entry point for illustrating a pig alias
|
101
|
+
def illustrate(pig_script, pig_alias, pig_parameters, skip_pruning)
|
102
|
+
require_aws_keys
|
103
|
+
install_and_configure
|
104
|
+
pig = Mortar::Local::Pig.new()
|
105
|
+
pig.illustrate_alias(pig_script, pig_alias, skip_pruning, pig_parameters)
|
106
|
+
end
|
107
|
+
|
108
|
+
end
|