mortar 0.6.2 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/lib/mortar/auth.rb CHANGED
@@ -88,6 +88,10 @@ class Mortar::Auth
88
88
  get_credentials[1]
89
89
  end
90
90
 
91
+ def user_s3_safe
92
+ return user.gsub(/[^0-9a-zA-Z]/i, '-')
93
+ end
94
+
91
95
  def api_key(user = get_credentials[0], password = get_credentials[1])
92
96
  require("mortar-api-ruby")
93
97
  api = Mortar::API.new(default_params)
@@ -243,6 +243,22 @@ protected
243
243
  error("Unable to find git remote for project #{project.name}")
244
244
  end
245
245
  end
246
+
247
+ def validate_script!(script_name)
248
+ pigscript = project.pigscripts[script_name]
249
+ controlscript = project.controlscripts[script_name]
250
+ unless pigscript || controlscript
251
+ available_pigscripts = project.pigscripts.none? ? "No pigscripts found" : "Available pigscripts:\n#{project.pigscripts.keys.sort.join("\n")}"
252
+ available_controlscripts = project.controlscripts.none? ? "No controlscripts found" : "Available controlscripts:\n#{project.controlscripts.keys.sort.join("\n")}"
253
+ error("Unable to find a pigscript or controlscript for #{script_name}\n\n#{available_pigscripts}\n\n#{available_controlscripts}")
254
+ end
255
+
256
+ if pigscript && controlscript
257
+ error("Naming conflict. #{script_name} refers to both a pigscript and a controlscript. Please rename scripts to avoid conflicts.")
258
+ end
259
+
260
+ pigscript or controlscript
261
+ end
246
262
 
247
263
  def validate_pigscript!(pigscript_name)
248
264
  unless pigscript = project.pigscripts[pigscript_name]
@@ -39,9 +39,15 @@ class Mortar::Command::Describe < Mortar::Command::Base
39
39
  unless pigscript_name && alias_name
40
40
  error("Usage: mortar describe PIGSCRIPT ALIAS\nMust specify PIGSCRIPT and ALIAS.")
41
41
  end
42
+
42
43
  validate_arguments!
44
+ pigscript = validate_script!(pigscript_name)
45
+
46
+ if pigscript.is_a? Mortar::Project::ControlScript
47
+ error "Currently Mortar does not support describing control scripts"
48
+ end
49
+
43
50
  validate_git_based_project!
44
- pigscript = validate_pigscript!(pigscript_name)
45
51
  git_ref = git.create_and_push_snapshot_branch(project)
46
52
 
47
53
  describe_id = nil
@@ -67,24 +67,6 @@ private
67
67
  Mortar::Command.commands
68
68
  end
69
69
 
70
- def legacy_help_for_namespace(namespace)
71
- instance = Mortar::Command::Help.groups.map do |group|
72
- [ group.title, group.select { |c| c.first =~ /^#{namespace}/ }.length ]
73
- end.sort_by { |l| l.last }.last
74
- return nil unless instance
75
- return nil if instance.last.zero?
76
- instance.first
77
- end
78
-
79
- def legacy_help_for_command(command)
80
- Mortar::Command::Help.groups.each do |group|
81
- group.each do |cmd, description|
82
- return description if cmd.split(" ").first == command
83
- end
84
- end
85
- nil
86
- end
87
-
88
70
  def primary_namespaces
89
71
  PRIMARY_NAMESPACES.map { |name| namespaces[name] }.compact
90
72
  end
@@ -97,7 +79,6 @@ private
97
79
  size = longest(namespaces.map { |n| n[:name] })
98
80
  namespaces.sort_by {|namespace| namespace[:name]}.each do |namespace|
99
81
  name = namespace[:name]
100
- namespace[:description] ||= legacy_help_for_namespace(name)
101
82
  puts " %-#{size}s # %s" % [ name, namespace[:description] ]
102
83
  end
103
84
  end
@@ -121,8 +102,6 @@ private
121
102
  unless namespace_commands.empty?
122
103
  size = longest(namespace_commands.map { |c| c[:banner] })
123
104
  namespace_commands.sort_by { |c| c[:banner].to_s }.each do |command|
124
- next if command[:help] =~ /DEPRECATED/
125
- command[:summary] ||= legacy_help_for_command(command[:command])
126
105
  puts " %-#{size}s # %s" % [ command[:banner], command[:summary] ]
127
106
  end
128
107
  end
@@ -140,7 +119,7 @@ private
140
119
  puts command[:help].split("\n")[1..-1].join("\n")
141
120
  else
142
121
  puts
143
- puts " " + legacy_help_for_command(name).to_s
122
+ error "No help available for #{command[:command]}. Please contact us at support@mortardata.com for assistance."
144
123
  end
145
124
  puts
146
125
  end
@@ -40,16 +40,21 @@ class Mortar::Command::Illustrate < Mortar::Command::Base
40
40
  alias_name = shift_argument
41
41
  skip_pruning = options[:skippruning] ||= false
42
42
 
43
+ validate_arguments!
44
+ pigscript = validate_script!(pigscript_name)
45
+
43
46
  # TODO: When illustrating without alias works, remove the `&& alias_name` to re-enable the feature on CLI
44
47
  unless pigscript_name && alias_name
45
48
  error("Usage: mortar illustrate PIGSCRIPT ALIAS\nMust specify PIGSCRIPT and ALIAS.")
46
49
  end
47
50
 
48
- validate_arguments!
51
+ if pigscript.is_a? Mortar::Project::ControlScript
52
+ error "Currently Mortar does not support illustrating control scripts"
53
+ end
54
+
49
55
  validate_git_based_project!
50
- pigscript = validate_pigscript!(pigscript_name)
51
56
  git_ref = git.create_and_push_snapshot_branch(project)
52
-
57
+
53
58
  illustrate_id = nil
54
59
  action("Starting illustrate") do
55
60
  illustrate_id = api.post_illustrate(project.name, pigscript.name, alias_name, skip_pruning, git_ref, :parameters => pig_parameters).body["illustrate_id"]
@@ -50,7 +50,7 @@ class Mortar::Command::Jobs < Mortar::Command::Base
50
50
  display_table(jobs, columns, headers)
51
51
  end
52
52
 
53
- # jobs:run PIGSCRIPT
53
+ # jobs:run SCRIPT
54
54
  #
55
55
  # Run a job on a Mortar Hadoop cluster.
56
56
  #
@@ -66,11 +66,23 @@ class Mortar::Command::Jobs < Mortar::Command::Base
66
66
  # Run the generate_regression_model_coefficients script on a 3 node cluster.
67
67
  # $ mortar jobs:run generate_regression_model_coefficients --clustersize 3
68
68
  def run
69
- pigscript_name = shift_argument
70
- unless pigscript_name
71
- error("Usage: mortar jobs:run PIGSCRIPT\nMust specify PIGSCRIPT.")
69
+ script_name = shift_argument
70
+ unless script_name
71
+ error("Usage: mortar jobs:run SCRIPT\nMust specify SCRIPT.")
72
72
  end
73
+
73
74
  validate_arguments!
75
+ script = validate_script!(script_name)
76
+
77
+ case script
78
+ when Mortar::Project::PigScript
79
+ is_control_script = false
80
+ when Mortar::Project::ControlScript
81
+ is_control_script = true
82
+ else
83
+ error "Unknown Script Type"
84
+ end
85
+
74
86
 
75
87
  unless options[:clusterid] || options[:clustersize]
76
88
  clusters = api.get_clusters().body['clusters']
@@ -98,27 +110,28 @@ class Mortar::Command::Jobs < Mortar::Command::Base
98
110
  end
99
111
 
100
112
  validate_git_based_project!
101
- pigscript = validate_pigscript!(pigscript_name)
102
113
  git_ref = git.create_and_push_snapshot_branch(project)
103
114
  notify_on_job_finish = ! options[:donotnotify]
104
115
 
105
- # post job to API
116
+ # post job to API
106
117
  response = action("Requesting job execution") do
107
118
  if options[:clustersize]
108
119
  cluster_size = options[:clustersize].to_i
109
120
  keepalive = ! options[:singlejobcluster]
110
- api.post_job_new_cluster(project.name, pigscript.name, git_ref, cluster_size,
121
+ api.post_job_new_cluster(project.name, script.name, git_ref, cluster_size,
111
122
  :parameters => pig_parameters,
112
123
  :keepalive => keepalive,
113
- :notify_on_job_finish => notify_on_job_finish).body
124
+ :notify_on_job_finish => notify_on_job_finish,
125
+ :is_control_script => is_control_script).body
114
126
  else
115
127
  cluster_id = options[:clusterid]
116
- api.post_job_existing_cluster(project.name, pigscript.name, git_ref, cluster_id,
128
+ api.post_job_existing_cluster(project.name, script.name, git_ref, cluster_id,
117
129
  :parameters => pig_parameters,
118
- :notify_on_job_finish => notify_on_job_finish).body
130
+ :notify_on_job_finish => notify_on_job_finish,
131
+ :is_control_script => is_control_script).body
119
132
  end
120
133
  end
121
-
134
+
122
135
  display("job_id: #{response['job_id']}")
123
136
  display
124
137
  display("Job status can be viewed on the web at:\n\n #{response['web_job_url']}")
@@ -146,7 +159,6 @@ class Mortar::Command::Jobs < Mortar::Command::Base
146
159
  def display_job_status(job_status)
147
160
  job_display_entries = {
148
161
  "status" => job_status["status_description"],
149
- "progress" => "#{job_status["progress"]}%",
150
162
  "cluster_id" => job_status["cluster_id"],
151
163
  "job submitted at" => job_status["start_timestamp"],
152
164
  "job began running at" => job_status["running_timestamp"],
@@ -167,8 +179,13 @@ class Mortar::Command::Jobs < Mortar::Command::Base
167
179
  end
168
180
 
169
181
  if job_status["num_hadoop_jobs"] && job_status["num_hadoop_jobs_succeeded"]
182
+ job_display_entries["progress"] = "#{job_status["progress"]}%"
170
183
  job_display_entries["hadoop jobs complete"] =
171
184
  '%0.2f / %0.2f' % [job_status["num_hadoop_jobs_succeeded"], job_status["num_hadoop_jobs"]]
185
+ elsif job_status["num_hadoop_jobs_succeeded"]
186
+ job_display_entries["progress"] = '%0.2f MapReduce Jobs complete.' % job_status["num_hadoop_jobs_succeeded"]
187
+ else
188
+ job_display_entries["progress"] = "#{job_status["progress"]}%"
172
189
  end
173
190
 
174
191
  if job_status["outputs"] && job_status["outputs"].length > 0
@@ -180,7 +197,13 @@ class Mortar::Command::Jobs < Mortar::Command::Base
180
197
  end]
181
198
  end
182
199
 
183
- styled_header("#{job_status["project_name"]}: #{job_status["pigscript_name"]} (job_id: #{job_status["job_id"]})")
200
+ if job_status["controlscript_name"]
201
+ script_name = job_status["controlscript_name"]
202
+ else
203
+ script_name = job_status["pigscript_name"]
204
+ end
205
+
206
+ styled_header("#{job_status["project_name"]}: #{script_name} (job_id: #{job_status["job_id"]})")
184
207
  styled_hash(job_display_entries)
185
208
  end
186
209
 
@@ -196,7 +219,7 @@ class Mortar::Command::Jobs < Mortar::Command::Base
196
219
  end
197
220
 
198
221
  # If the job is running show the progress bar
199
- if job_status["status_code"] == Mortar::API::Jobs::STATUS_RUNNING
222
+ if job_status["status_code"] == Mortar::API::Jobs::STATUS_RUNNING && job_status["num_hadoop_jobs"]
200
223
  progressbar = "=" + ("=" * (job_status["progress"].to_i / 5)) + ">"
201
224
 
202
225
  if job_status["num_hadoop_jobs"] && job_status["num_hadoop_jobs_succeeded"]
@@ -206,6 +229,10 @@ class Mortar::Command::Jobs < Mortar::Command::Base
206
229
 
207
230
  printf("\r[#{spinner(ticks)}] Status: [%-22s] %s%% Complete (%s MapReduce jobs finished)", progressbar, job_status["progress"], hadoop_jobs_ratio_complete)
208
231
 
232
+ elsif job_status["status_code"] == Mortar::API::Jobs::STATUS_RUNNING
233
+ jobs_complete = '%0.2f' % job_status["num_hadoop_jobs_succeeded"]
234
+ printf("\r[#{spinner(ticks)}] #{jobs_complete} MapReduce Jobs complete.")
235
+
209
236
  # If the job is not complete, but not in the running state, just display its status
210
237
  else
211
238
  redisplay("[#{spinner(ticks)}] Status: #{job_status['status_description']}")
@@ -0,0 +1,87 @@
1
+ #
2
+ # Copyright 2012 Mortar Data Inc.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require "mortar/local/controller"
18
+ require "mortar/command/base"
19
+
20
+ # run select pig commands on your local machine
21
+ #
22
+ class Mortar::Command::Local < Mortar::Command::Base
23
+
24
+
25
+ # configure
26
+ #
27
+ # Install dependencies for running this pig project locally, other
28
+ # commands will also perform this step automatically.
29
+ #
30
+ def configure
31
+ ctrl = Mortar::Local::Controller.new
32
+ ctrl.install_and_configure
33
+ end
34
+
35
+ # local:run PIGSCRIPT
36
+ #
37
+ # Run a job on your local machine
38
+ #
39
+ # -p, --parameter NAME=VALUE # Set a pig parameter value in your script.
40
+ # -f, --param-file PARAMFILE # Load pig parameter values from a file.
41
+ #
42
+ #Examples:
43
+ #
44
+ # Run the generate_regression_model_coefficients script locally.
45
+ # $ mortar local:run generate_regression_model_coefficients
46
+ def run
47
+ pigscript_name = shift_argument
48
+ unless pigscript_name
49
+ error("Usage: mortar local:run PIGSCRIPT\nMust specify PIGSCRIPT.")
50
+ end
51
+ validate_arguments!
52
+ pigscript = validate_pigscript!(pigscript_name)
53
+ ctrl = Mortar::Local::Controller.new
54
+ ctrl.run(pigscript, pig_parameters)
55
+ end
56
+
57
+ # illustrate [PIGSCRIPT] [ALIAS]
58
+ #
59
+ # Locallay illustrate the effects and output of a pigscript.
60
+ #
61
+ # -s, --skippruning # Don't try to reduce the illustrate results to the smallest size possible.
62
+ # -p, --parameter NAME=VALUE # Set a pig parameter value in your script.
63
+ # -f, --param-file PARAMFILE # Load pig parameter values from a file.
64
+ # --no_browser # Don't open the illustrate results automatically in the browser.
65
+ #
66
+ # Examples:
67
+ #
68
+ # Illustrate the songs_sample relation in the generate_regression_model_coefficients script.
69
+ # $ mortar illustrate generate_regression_model_coefficients songs_sample
70
+ def illustrate
71
+ pigscript_name = shift_argument
72
+ alias_name = shift_argument
73
+ skip_pruning = options[:skippruning] ||= false
74
+
75
+ unless pigscript_name && alias_name
76
+ error("Usage: mortar local:illustrate PIGSCRIPT ALIAS\nMust specify PIGSCRIPT and ALIAS.")
77
+ end
78
+
79
+ validate_arguments!
80
+ pigscript = validate_pigscript!(pigscript_name)
81
+
82
+ ctrl = Mortar::Local::Controller.new
83
+ ctrl.illustrate(pigscript, alias_name, pig_parameters, skip_pruning)
84
+ end
85
+
86
+
87
+ end
@@ -26,7 +26,7 @@ class Mortar::Command::Projects < Mortar::Command::Base
26
26
 
27
27
  # projects
28
28
  #
29
- # Display the available set of projects
29
+ # Display the available set of Mortar projects.
30
30
  def index
31
31
  validate_arguments!
32
32
  projects = api.get_projects().body["projects"]
@@ -40,7 +40,7 @@ class Mortar::Command::Projects < Mortar::Command::Base
40
40
 
41
41
  # projects:delete PROJECTNAME
42
42
  #
43
- # Delete a mortar project
43
+ # Delete the Mortar project PROJECTNAME.
44
44
  def delete
45
45
  name = shift_argument
46
46
  unless name
@@ -58,7 +58,7 @@ class Mortar::Command::Projects < Mortar::Command::Base
58
58
 
59
59
  # projects:create PROJECTNAME
60
60
  #
61
- # Generate and register a new Mortar project for code in the current directory, with the name PROJECTNAME.
61
+ # Used when you want to start a new Mortar project using Mortar generated code.
62
62
  def create
63
63
  name = shift_argument
64
64
  unless name
@@ -75,9 +75,9 @@ class Mortar::Command::Projects < Mortar::Command::Base
75
75
  end
76
76
  alias_command "new", "projects:create"
77
77
 
78
- # projects:register PROJECT
78
+ # projects:register PROJECTNAME
79
79
  #
80
- # register a mortar project for the current directory with the name PROJECT
80
+ # Used when you want to start a new Mortar project using your existing code in the current directory.
81
81
  def register
82
82
  name = shift_argument
83
83
  unless name
@@ -145,10 +145,12 @@ class Mortar::Command::Projects < Mortar::Command::Base
145
145
  end
146
146
  alias_command "register", "projects:register"
147
147
 
148
- # projects:set_remote PROJECT
149
- #
150
- # Adds the Mortar remote to the local git project. This is necessary for successfully executing many of the Mortar commands.
148
+ # projects:set_remote PROJECTNAME
151
149
  #
150
+ # Used after you checkout code for an existing Mortar project from a non-Mortar git repository.
151
+ # Adds a remote to your local git repository to the Mortar git repository. For example if a
152
+ # co-worker creates a Mortar project from an internal repository you would clone the internal
153
+ # repository and then after cloning call mortar projects:set_remote.
152
154
  def set_remote
153
155
  project_name = shift_argument
154
156
 
@@ -176,10 +178,9 @@ class Mortar::Command::Projects < Mortar::Command::Base
176
178
 
177
179
  end
178
180
 
179
- # projects:clone PROJECT
180
- #
181
- # clone the mortar project PROJECT into the current directory.
181
+ # projects:clone PROJECTNAME
182
182
  #
183
+ # Used when you want to clone an existing Mortar project into the current directory.
183
184
  def clone
184
185
  name = shift_argument
185
186
  unless name
@@ -36,8 +36,13 @@ class Mortar::Command::Validate < Mortar::Command::Base
36
36
  error("Usage: mortar validate PIGSCRIPT\nMust specify PIGSCRIPT.")
37
37
  end
38
38
  validate_arguments!
39
+ pigscript = validate_script!(pigscript_name)
40
+
41
+ if pigscript.is_a? Mortar::Project::ControlScript
42
+ error "Currently Mortar does not support validating control scripts"
43
+ end
44
+
39
45
  validate_git_based_project!
40
- pigscript = validate_pigscript!(pigscript_name)
41
46
  git_ref = git.create_and_push_snapshot_branch(project)
42
47
 
43
48
  validate_id = nil
@@ -494,6 +494,18 @@ module Mortar
494
494
  end
495
495
  end
496
496
 
497
+ def ensure_dir_exists(dir)
498
+ unless Dir.exists? dir
499
+ Dir.mkdir(dir)
500
+ end
501
+ end
502
+
503
+ def copy_if_not_present_at_dest(res_src, res_dest)
504
+ unless File.exists?(res_dest)
505
+ FileUtils.cp(res_src, res_dest)
506
+ end
507
+ end
508
+
497
509
  private
498
510
 
499
511
  def create_display_method(name, colour_code, new_line=true)
@@ -0,0 +1,108 @@
1
+ #
2
+ # Copyright 2012 Mortar Data Inc.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require "mortar/helpers"
18
+ require "mortar/local/pig"
19
+ require "mortar/local/java"
20
+ require "mortar/local/python"
21
+
22
+
23
+ class Mortar::Local::Controller
24
+ include Mortar::Helpers
25
+
26
+ NO_JAVA_ERROR_MESSAGE = <<EOF
27
+ A suitable java installation could not be found. If you already have java installed
28
+ please set your JAVA_HOME environment variable before continuing. Otherwise, a
29
+ suitable java installation will need to be added to your local system.
30
+
31
+ Installing Java
32
+ On OSX run `javac` from the command line. This will intiate the installation. For
33
+ Linux systems please consult the documentation on your relevant package manager.
34
+ EOF
35
+
36
+ NO_PYTHON_ERROR_MESSAGE = <<EOF
37
+ pA suitable python installation with virtualenv could not be located. Please ensure
38
+ you have python 2.6+ installed on your local system. If you need to obtain a copy
39
+ of virtualenv it can be located here:
40
+ https://pypi.python.org/pypi/virtualenv
41
+ EOF
42
+
43
+ NO_AWS_KEYS_ERROR_MESSAGE = <<EOF
44
+ Please specify your aws access key via enviroment variable AWS_ACCESS_KEY
45
+ and your aws secret key via enviroment variable AWS_SECRET_KEY"
46
+ EOF
47
+
48
+
49
+ # Checks if the user has properly specified their AWS keys
50
+ def verify_aws_keys()
51
+ if (not (ENV['AWS_ACCESS_KEY'] and ENV['AWS_SECRET_KEY'])) then
52
+ if not ENV['MORTAR_IGNORE_AWS_KEYS']
53
+ return false
54
+ else
55
+ return true
56
+ end
57
+ else
58
+ return true
59
+ end
60
+ end
61
+
62
+ # Exits with a helpful message if the user has not setup their aws keys
63
+ def require_aws_keys()
64
+ unless verify_aws_keys()
65
+ error(NO_AWS_KEYS_ERROR_MESSAGE)
66
+ end
67
+ end
68
+
69
+ # Main entry point to perform installation and configuration necessary
70
+ # to run pig on the users local machine
71
+ def install_and_configure
72
+ java = Mortar::Local::Java.new()
73
+ unless java.check_install
74
+ error(NO_JAVA_ERROR_MESSAGE)
75
+ end
76
+
77
+ pig = Mortar::Local::Pig.new()
78
+ pig.install()
79
+
80
+ py = Mortar::Local::Python.new()
81
+ unless py.check_or_install
82
+ error(NO_PYTHON_ERROR_MESSAGE)
83
+ end
84
+
85
+ unless py.setup_project_python_environment
86
+ msg = "\nUnable to setup a python environment with your dependencies, "
87
+ msg += "see #{py.pip_error_log_path} for more details"
88
+ error(msg)
89
+ end
90
+ end
91
+
92
+ # Main entry point for user running a pig script
93
+ def run(pig_script, pig_parameters)
94
+ require_aws_keys
95
+ install_and_configure
96
+ pig = Mortar::Local::Pig.new()
97
+ pig.run_script(pig_script, pig_parameters)
98
+ end
99
+
100
+ # Main entry point for illustrating a pig alias
101
+ def illustrate(pig_script, pig_alias, pig_parameters, skip_pruning)
102
+ require_aws_keys
103
+ install_and_configure
104
+ pig = Mortar::Local::Pig.new()
105
+ pig.illustrate_alias(pig_script, pig_alias, skip_pruning, pig_parameters)
106
+ end
107
+
108
+ end