mortar 0.14.1 → 0.15.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: a48ef7a19f19d16af145ed38847a5c316c509361
4
+ data.tar.gz: 4d554dd57193f50f23b0f98b273695d6e7d1af2c
5
+ SHA512:
6
+ metadata.gz: d8a722fdcc23c6312b8faac36dba45d9f2399d6bbb64c2187576abd600fead99d0c20dfd7073002d1f4903c9026409b57b219ba49ace184d5334f0b575ab7ba1
7
+ data.tar.gz: 868f456e447061aaeb90a941a00f2cdf0b0706fa7355fe5753a7554ec99af13d43501bcb553aa3246ff0aa9605382f6cc5f620733ae57327cbe521b1970a5b1c
@@ -399,7 +399,18 @@ protected
399
399
 
400
400
  pigscript or controlscript
401
401
  end
402
-
402
+
403
+ def validate_luigiscript!(luigiscript_name)
404
+ shortened_script_name = File.basename(luigiscript_name, ".*")
405
+ unless luigiscript = project.luigiscripts[shortened_script_name]
406
+ available_scripts = project.luigiscripts.none? ? "No luigiscripts found" : "Available luigiscripts:\n#{project.luigiscripts.collect{|k,v| v.executable_path}.sort.join("\n")}"
407
+ error("Unable to find luigiscript #{shortened_script_name}\n#{available_scripts}")
408
+ end
409
+ #While validating we can load the defaults that are relevant to this script.
410
+ load_defaults(shortened_script_name)
411
+ luigiscript
412
+ end
413
+
403
414
  def validate_pigscript!(pigscript_name)
404
415
  shortened_pigscript_name = File.basename(pigscript_name, ".*")
405
416
  unless pigscript = project.pigscripts[shortened_pigscript_name]
@@ -467,6 +478,11 @@ protected
467
478
  elsif remotes.values.uniq.size == 1
468
479
  # take the only project in the remotes
469
480
  [remotes.first[1], remotes.first[0]]
481
+ elsif remotes.has_key? 'mortar'
482
+ # In some cases (like forking a public project in mortar-code)
483
+ # we'll have more than one possible remote. We'll default to the
484
+ # one called mortar.
485
+ [remotes['mortar'], 'mortar']
470
486
  else
471
487
  raise(Mortar::Command::CommandFailed, "Multiple projects in folder and no project specified.\nSpecify which project to use with --project <project name>")
472
488
  end
@@ -22,7 +22,7 @@ class Mortar::Command::Describe < Mortar::Command::Base
22
22
 
23
23
  include Mortar::Git
24
24
 
25
- # describe [PIGSCRIPT] [ALIAS]
25
+ # describe PIGSCRIPT ALIAS
26
26
  #
27
27
  # Describe the schema of an alias and all of its ancestors.
28
28
  #
@@ -40,14 +40,14 @@ class Mortar::Command::Illustrate < Mortar::Command::Base
40
40
  def index
41
41
  pigscript_name = shift_argument
42
42
  alias_name = shift_argument
43
- skip_pruning = options[:skippruning] ||= false
44
-
45
43
  validate_arguments!
46
- pigscript = validate_script!(pigscript_name)
44
+
45
+ skip_pruning = options[:skippruning] ||= false
47
46
 
48
47
  unless pigscript_name
49
48
  error("Usage: mortar illustrate PIGSCRIPT [ALIAS]\nMust specify PIGSCRIPT.")
50
49
  end
50
+ pigscript = validate_script!(pigscript_name)
51
51
 
52
52
  if pigscript.is_a? Mortar::Project::ControlScript
53
53
  error "Currently Mortar does not support illustrating control scripts"
@@ -205,6 +205,7 @@ class Mortar::Command::Jobs < Mortar::Command::Base
205
205
  unless job_id
206
206
  error("Usage: mortar jobs:status JOB_ID\nMust specify JOB_ID.")
207
207
  end
208
+ validate_arguments!
208
209
 
209
210
  # Inner function to display the hash table when the job is complte
210
211
  def display_job_status(job_status)
@@ -313,6 +314,7 @@ class Mortar::Command::Jobs < Mortar::Command::Base
313
314
  unless job_id
314
315
  error("Usage: mortar jobs:stop JOB_ID\nMust specify JOB_ID.")
315
316
  end
317
+ validate_arguments!
316
318
 
317
319
  response = api.stop_job(job_id).body
318
320
 
@@ -144,7 +144,10 @@ class Mortar::Command::Local < Mortar::Command::Base
144
144
  def illustrate
145
145
  pigscript_name = shift_argument
146
146
  alias_name = shift_argument
147
+ validate_arguments!
148
+
147
149
  skip_pruning = options[:skippruning] ||= false
150
+ no_browser = options[:no_browser] ||= false
148
151
 
149
152
  unless pigscript_name
150
153
  error("Usage: mortar local:illustrate PIGSCRIPT [ALIAS]\nMust specify PIGSCRIPT.")
@@ -157,11 +160,10 @@ class Mortar::Command::Local < Mortar::Command::Base
157
160
  end
158
161
  Dir.chdir(project_root)
159
162
 
160
- validate_arguments!
161
163
  pigscript = validate_pigscript!(pigscript_name)
162
164
 
163
165
  ctrl = Mortar::Local::Controller.new
164
- ctrl.illustrate(pigscript, alias_name, pig_version, pig_parameters, skip_pruning)
166
+ ctrl.illustrate(pigscript, alias_name, pig_version, pig_parameters, skip_pruning, no_browser)
165
167
  end
166
168
 
167
169
 
@@ -211,4 +213,39 @@ class Mortar::Command::Local < Mortar::Command::Base
211
213
  ctrl.repl(pig_version, pig_parameters)
212
214
  end
213
215
 
216
+
217
+ # local:luigi SCRIPT
218
+ #
219
+ # Run a luigi workflow on your local machine in local scheduler mode.
220
+ # Any additional command line arguments will be passed directly to the luigi script.
221
+ #
222
+ # -p, --parameter NAME=VALUE # Set a pig parameter value in your script.
223
+ # -f, --param-file PARAMFILE # Load pig parameter values from a file.
224
+ # --project-root PROJECTDIR # The root directory of the project if not the CWD
225
+ #
226
+ #Examples:
227
+ #
228
+ # Run the recsys luigi script with a parameter named date-interval
229
+ # $ mortar local:luigi luigiscripts/recsys.py --date-interval 2012-04
230
+ def luigi
231
+ script_name = shift_argument
232
+ unless script_name
233
+ error("Usage: mortar local:luigi SCRIPT\nMust specify SCRIPT.")
234
+ end
235
+ validate_arguments!
236
+
237
+ # cd into the project root
238
+ project_root = options[:project_root] ||= Dir.getwd
239
+ unless File.directory?(project_root)
240
+ error("No such directory #{project_root}")
241
+ end
242
+ Dir.chdir(project_root)
243
+ script = validate_luigiscript!(script_name)
244
+ ctrl = Mortar::Local::Controller.new
245
+ luigi_params = pig_parameters.sort_by { |p| p['name'] }
246
+ luigi_params = luigi_params.map { |arg| ["--#{arg['name']}", "#{arg['value']}"] }.flatten
247
+ ctrl.run_luigi(script, luigi_params)
248
+ end
249
+
250
+
214
251
  end
@@ -0,0 +1,30 @@
1
+
2
+ [loggers]
3
+ keys=root,luigi-interface
4
+
5
+ [handlers]
6
+ keys=console
7
+
8
+ [formatters]
9
+ keys=standard,plain
10
+
11
+ [logger_root]
12
+ level=DEBUG
13
+ handlers=console
14
+
15
+ [logger_luigi-interface]
16
+ level=DEBUG
17
+ handlers=
18
+ qualname=luigi-interface
19
+
20
+ [handler_console]
21
+ level=DEBUG
22
+ class=StreamHandler
23
+ args=(sys.stderr,)
24
+ formatter=standard
25
+
26
+ [formatter_standard]
27
+ format=%(asctime)s [%(process)d:%(threadName)s] (%(filename)s:%(lineno)d) %(levelname)-5.5s - %(message)s
28
+
29
+ [formatter_plain]
30
+ format=%(message)s
@@ -181,11 +181,11 @@ EOF
181
181
  end
182
182
 
183
183
  # Main entry point for illustrating a pig alias
184
- def illustrate(pig_script, pig_alias, pig_version, pig_parameters, skip_pruning)
184
+ def illustrate(pig_script, pig_alias, pig_version, pig_parameters, skip_pruning, no_browser)
185
185
  require_aws_keys
186
186
  install_and_configure(pig_version)
187
187
  pig = Mortar::Local::Pig.new()
188
- pig.illustrate_alias(pig_script, pig_alias, skip_pruning, pig_version, pig_parameters)
188
+ pig.illustrate_alias(pig_script, pig_alias, skip_pruning, no_browser, pig_version, pig_parameters)
189
189
  end
190
190
 
191
191
  def validate(pig_script, pig_version, pig_parameters)
@@ -200,4 +200,10 @@ EOF
200
200
  pig.launch_repl(pig_version, pig_parameters)
201
201
  end
202
202
 
203
+ def run_luigi(luigi_script, user_script_args)
204
+ install_and_configure()
205
+ py = Mortar::Local::Python.new()
206
+ py.run_luigi_script(luigi_script, user_script_args)
207
+ end
208
+
203
209
  end
@@ -165,6 +165,7 @@ module Mortar
165
165
  else
166
166
  raise RuntimeError, "Unknown call type: #{call_func}"
167
167
  end
168
+
168
169
  case response.status
169
170
  when 300..303 then
170
171
  make_call(response.headers['Location'], call_func, redirect_times+1, errors)
@@ -207,6 +208,59 @@ module Mortar
207
208
  return existing_install_date < remote_archive_date
208
209
  end
209
210
 
211
+ def run_templated_script(template, template_params)
212
+ # Insert standard template variables
213
+ template_params['project_home'] = File.expand_path("..", local_install_directory)
214
+ template_params['local_install_dir'] = local_install_directory
215
+
216
+ unset_hadoop_env_vars
217
+ reset_local_logs
218
+ # Generate the script for running the command, then
219
+ # write it to a temp script which will be exectued
220
+ script_text = render_script_template(template, template_params)
221
+ script = Tempfile.new("mortar-")
222
+ script.write(script_text)
223
+ script.close(false)
224
+ FileUtils.chmod(0755, script.path)
225
+ system(script.path)
226
+ script.unlink
227
+ return (0 == $?.to_i)
228
+ end
229
+
230
+ def render_script_template(template, template_params)
231
+ erb = ERB.new(File.read(template), 0, "%<>")
232
+ erb.result(BindingClazz.new(template_params).get_binding)
233
+ end
234
+
235
+ # so Pig doesn't try to load the wrong hadoop jar/configuration
236
+ # this doesn't mess up the env vars in the terminal, just this process (ruby)
237
+ def unset_hadoop_env_vars
238
+ ENV['HADOOP_HOME'] = ''
239
+ ENV['HADOOP_CONF_DIR'] = ''
240
+ end
241
+
242
+ def reset_local_logs
243
+ if File.directory? local_log_dir
244
+ FileUtils.rm_rf local_log_dir
245
+ end
246
+ Dir.mkdir local_log_dir
247
+ Dir.mkdir local_udf_log_dir
248
+ end
249
+
250
+
251
+ # Allows us to use a hash for template variables
252
+ class BindingClazz
253
+ def initialize(attrs)
254
+ attrs.each{ |k, v|
255
+ # set an instance variable with the key name so the binding will find it in scope
256
+ self.instance_variable_set("@#{k}".to_sym, v)
257
+ }
258
+ end
259
+ def get_binding()
260
+ binding
261
+ end
262
+ end
263
+
210
264
  end
211
265
  end
212
266
  end
@@ -202,7 +202,7 @@ class Mortar::Local::Pig
202
202
  outfile.path
203
203
  end
204
204
 
205
- # Given a file path, open it and decode the containing json
205
+ # Given a file path, open it and decode the containing text
206
206
  def decode_illustrate_input_file(illustrate_outpath)
207
207
  data_raw = File.read(illustrate_outpath)
208
208
  begin
@@ -212,10 +212,9 @@ class Mortar::Local::Pig
212
212
  ic = Iconv.new('UTF-8//IGNORE', 'UTF-8')
213
213
  data_encoded = ic.iconv(data_raw)
214
214
  end
215
- json_decode(data_encoded)
216
215
  end
217
216
 
218
- def show_illustrate_output(illustrate_outpath)
217
+ def show_illustrate_output_browser(illustrate_outpath)
219
218
  ensure_dir_exists("illustrate-output")
220
219
  ensure_dir_exists("illustrate-output/resources")
221
220
  ensure_dir_exists("illustrate-output/resources/css")
@@ -229,7 +228,8 @@ class Mortar::Local::Pig
229
228
  }
230
229
 
231
230
  # Pull in the dumped json file
232
- illustrate_data = decode_illustrate_input_file(illustrate_outpath)
231
+ illustrate_data_json_text = decode_illustrate_input_file(illustrate_outpath)
232
+ illustrate_data = json_decode(illustrate_data_json_text)
233
233
 
234
234
  # Render a template using it's values
235
235
  template_params = create_illustrate_template_parameters(illustrate_data)
@@ -248,7 +248,6 @@ class Mortar::Local::Pig
248
248
  require "launchy"
249
249
  Launchy.open(File.expand_path(@resource_destinations["illustrate_html"]))
250
250
  end
251
-
252
251
  end
253
252
 
254
253
  def create_illustrate_template_parameters(illustrate_data)
@@ -258,7 +257,7 @@ class Mortar::Local::Pig
258
257
  return params
259
258
  end
260
259
 
261
- def illustrate_alias(pig_script, pig_alias, skip_pruning, pig_version, pig_parameters)
260
+ def illustrate_alias(pig_script, pig_alias, skip_pruning, no_browser, pig_version, pig_parameters)
262
261
  cmd = "-e 'illustrate "
263
262
 
264
263
  # Parameters have to be entered with the illustrate command (as
@@ -275,7 +274,11 @@ class Mortar::Local::Pig
275
274
  cmd += " -skipPruning "
276
275
  end
277
276
 
278
- cmd += " -json '"
277
+ if no_browser
278
+ cmd += " -str '"
279
+ else
280
+ cmd += " -json '"
281
+ end
279
282
 
280
283
  if pig_alias
281
284
  cmd += " #{pig_alias} "
@@ -283,7 +286,11 @@ class Mortar::Local::Pig
283
286
 
284
287
  result = run_pig_command(cmd, pig_version, [], false)
285
288
  if result
286
- show_illustrate_output(illustrate_outpath)
289
+ if no_browser
290
+ display decode_illustrate_input_file(illustrate_outpath)
291
+ else
292
+ show_illustrate_output_browser(illustrate_outpath)
293
+ end
287
294
  end
288
295
  end
289
296
 
@@ -291,42 +298,9 @@ class Mortar::Local::Pig
291
298
  # can be appended to the command line invocation of Pig that will
292
299
  # get it to do something interesting, such as '-f some-file.pig'
293
300
  def run_pig_command(cmd, pig_version, parameters = nil, jython_output = true)
294
- unset_hadoop_env_vars
295
- reset_local_logs
296
- # Generate the script for running the command, then
297
- # write it to a temp script which will be exectued
298
- script_text = script_for_command(cmd, pig_version, parameters)
299
- script = Tempfile.new("mortar-")
300
- script.write(script_text)
301
- script.close(false)
302
- FileUtils.chmod(0755, script.path)
303
- system(script.path)
304
- script.unlink
305
- return (0 == $?.to_i)
306
- end
307
-
308
- # so Pig doesn't try to load the wrong hadoop jar/configuration
309
- # this doesn't mess up the env vars in the terminal, just this process (ruby)
310
- def unset_hadoop_env_vars
311
- ENV['HADOOP_HOME'] = ''
312
- ENV['HADOOP_CONF_DIR'] = ''
313
- end
314
-
315
- def reset_local_logs
316
- if File.directory? local_log_dir
317
- FileUtils.rm_rf local_log_dir
318
- end
319
- Dir.mkdir local_log_dir
320
- Dir.mkdir local_udf_log_dir
321
- end
322
-
323
- # Generates a bash script which sets up the necessary environment and
324
- # then runs the pig command
325
- def script_for_command(cmd, pig_version, parameters, jython_output = true)
326
301
  template_params = pig_command_script_template_parameters(cmd, pig_version, parameters)
327
302
  template_params['pig_opts']['jython.output'] = jython_output
328
- erb = ERB.new(File.read(pig_command_script_template_path), 0, "%<>")
329
- erb.result(BindingClazz.new(template_params).get_binding)
303
+ return run_templated_script(pig_command_script_template_path, template_params)
330
304
  end
331
305
 
332
306
  # Path to the template which generates the bash script for running pig
@@ -355,8 +329,6 @@ class Mortar::Local::Pig
355
329
  template_params['pig_classpath'] = "#{pig_directory(pig_version)}/lib-local/*:#{lib_directory}/lib-local/*:#{pig_directory(pig_version)}/lib-pig/*:#{pig_directory(pig_version)}/lib-cluster/*:#{lib_directory}/lib-pig/*:#{lib_directory}/lib-cluster/*:#{jython_directory}/jython.jar"
356
330
  template_params['classpath'] = template_params_classpath
357
331
  template_params['log4j_conf'] = log4j_conf
358
- template_params['project_home'] = File.expand_path("..", local_install_directory)
359
- template_params['local_install_dir'] = local_install_directory
360
332
  template_params['pig_sub_command'] = cmd
361
333
  template_params['pig_opts'] = pig_options
362
334
  template_params
@@ -423,17 +395,4 @@ class Mortar::Local::Pig
423
395
  param_file.path
424
396
  end
425
397
 
426
- # Allows us to use a hash for template variables
427
- class BindingClazz
428
- def initialize(attrs)
429
- attrs.each{ |k, v|
430
- # set an instance variable with the key name so the binding will find it in scope
431
- self.instance_variable_set("@#{k}".to_sym, v)
432
- }
433
- end
434
- def get_binding()
435
- binding
436
- end
437
- end
438
-
439
398
  end
@@ -21,8 +21,9 @@ class Mortar::Local::Python
21
21
 
22
22
  PYTHON_OSX_TGZ_NAME = "mortar-python-osx.tgz"
23
23
  PYTHON_OSX_TGZ_DEFAULT_URL_PATH = "resource/python_osx"
24
+ PYPI_URL_PATH = "resource/mortar_pypi"
24
25
 
25
- MORTAR_PYTHON_PACKAGES = ["luigi"]
26
+ MORTAR_PYTHON_PACKAGES = ["luigi", "mortar-luigi"]
26
27
 
27
28
  # Path to the python binary that should be used
28
29
  # for running UDFs
@@ -167,8 +168,11 @@ class Mortar::Local::Python
167
168
  end
168
169
 
169
170
  def has_valid_virtualenv?
170
- `#{@command} -m virtualenv #{python_env_dir}`
171
+ output = `#{@command} -m virtualenv #{python_env_dir} 2>&1`
171
172
  if 0 != $?.to_i
173
+ File.open(virtualenv_error_log_path, 'w') { |f|
174
+ f.write(output)
175
+ }
172
176
  return false
173
177
  end
174
178
  return true
@@ -205,6 +209,10 @@ class Mortar::Local::Python
205
209
  return ENV.fetch('PIP_ERROR_LOG', "dependency_install.log")
206
210
  end
207
211
 
212
+ def virtualenv_error_log_path
213
+ return ENV.fetch('VE_ERROR_LOG', "virtualenv.log")
214
+ end
215
+
208
216
  # Whether or not we need to do a `pip install -r requirements.txt` because
209
217
  # we've never done one before or the dependencies have changed
210
218
  def should_do_requirements_install
@@ -230,7 +238,9 @@ class Mortar::Local::Python
230
238
  end
231
239
 
232
240
  def mortar_package_url(package)
233
- return "http://s3.amazonaws.com/mortar-pypi/#{package}/#{package}.tar.gz";
241
+ full_host = (host =~ /^http/) ? host : "https://api.#{host}"
242
+ default_url = full_host + "/" + PYPI_URL_PATH
243
+ "#{ENV.fetch('MORTAR_PACKAGE_URL', default_url)}/#{package}"
234
244
  end
235
245
 
236
246
  def update_mortar_package?(package)
@@ -266,9 +276,27 @@ class Mortar::Local::Python
266
276
  return true
267
277
  end
268
278
 
279
+ def local_activate_path
280
+ return "#{python_env_dir}/bin/activate"
281
+ end
282
+
283
+ def local_python_bin
284
+ return "#{python_env_dir}/bin/python"
285
+ end
286
+
287
+ def local_pip_bin
288
+ return "#{python_env_dir}/bin/pip"
289
+ end
290
+
269
291
  def pip_install package_url
270
- pip_output = `. #{python_env_dir}/bin/activate &&
271
- #{python_env_dir}/bin/pip install #{package_url} --use-mirrors`
292
+ # Note that we're executing pip by passing it as a script for python to execute, this is
293
+ # explicitly done to deal with this command breaking due to the maximum size of the path
294
+ # to the interpreter in a shebang. Since the containing virtualenv is already buried
295
+ # several layers deep in the .mortar-local directory we're very likely to (read: have) hit
296
+ # this limit. This unfortunately leads to very vague errors about pip not existing when
297
+ # in fact it is the truncated path to the interpreter that does not exist. I would now
298
+ # like the last day of my life back.
299
+ pip_output = `. #{local_activate_path} && #{local_python_bin} #{local_pip_bin} install #{package_url} --use-mirrors;`
272
300
  if 0 != $?.to_i
273
301
  File.open(pip_error_log_path, 'w') { |f|
274
302
  f.write(pip_output)
@@ -287,4 +315,31 @@ class Mortar::Local::Python
287
315
  note_install mortar_package_dir(package_name)
288
316
  end
289
317
 
318
+ def run_luigi_script(luigi_script, user_script_args)
319
+ template_params = luigi_command_template_parameters(luigi_script, user_script_args)
320
+ return run_templated_script(python_command_script_template_path, template_params)
321
+ end
322
+
323
+ # Path to the template which generates the bash script for running python
324
+ def python_command_script_template_path
325
+ File.expand_path("../../templates/script/runpython.sh", __FILE__)
326
+ end
327
+
328
+ def luigi_logging_config_file_path
329
+ File.expand_path("../../conf/luigi/logging.ini", __FILE__)
330
+ end
331
+
332
+ def luigi_command_template_parameters(luigi_script, user_script_args)
333
+ script_args = [
334
+ "--local-scheduler",
335
+ "--logging-conf-file #{luigi_logging_config_file_path}",
336
+ user_script_args.join(" "),
337
+ ]
338
+ return {
339
+ :python_arugments => "",
340
+ :python_script => luigi_script.executable_path(),
341
+ :script_arguments => script_args.join(" ")
342
+ }
343
+ end
344
+
290
345
  end
@@ -70,7 +70,20 @@ module Mortar
70
70
  :optional => true)
71
71
  @controlscripts
72
72
  end
73
-
73
+
74
+ def luigiscripts_path
75
+ File.join(@root_path, "luigiscripts")
76
+ end
77
+
78
+ def luigiscripts
79
+ @luigiscripts ||= LuigiScripts.new(
80
+ luigiscripts_path,
81
+ "luigiscripts",
82
+ ".py",
83
+ :optional => true)
84
+ @luigiscripts
85
+ end
86
+
74
87
  def tmp_path
75
88
  path = File.join(@root_path, "tmp")
76
89
  unless File.directory? path
@@ -153,6 +166,12 @@ module Mortar
153
166
  end
154
167
  end
155
168
 
169
+ class LuigiScripts < ProjectEntity
170
+ def element(name, path)
171
+ LuigiScript.new(name, path)
172
+ end
173
+ end
174
+
156
175
  class PythonUDFs < ProjectEntity
157
176
  def element(name, path)
158
177
  Script.new(name, path)
@@ -180,6 +199,14 @@ module Mortar
180
199
  code
181
200
  end
182
201
  end
202
+
203
+ class LuigiScript < Script
204
+
205
+ def executable_path
206
+ "luigiscripts/#{self.name}.py"
207
+ end
208
+
209
+ end
183
210
 
184
211
  class ControlScript < Script
185
212
 
@@ -0,0 +1,12 @@
1
+ #!/bin/bash
2
+
3
+ set -e
4
+
5
+ # Setup python environment
6
+ source <%= @local_install_dir %>/pythonenv/bin/activate
7
+
8
+ # Run Python
9
+ <%= @local_install_dir %>/pythonenv/bin/python \
10
+ <%= @python_arugments %> \
11
+ <%= @python_script %> \
12
+ <%= @script_arguments %>
@@ -16,5 +16,5 @@
16
16
 
17
17
  module Mortar
18
18
  # see http://semver.org/
19
- VERSION = "0.14.1"
19
+ VERSION = "0.15.0"
20
20
  end
@@ -69,6 +69,13 @@ other\tgit@github.com:other.git (push)
69
69
  @base.project.name.should == 'myproject'
70
70
  end
71
71
 
72
+ it "gets the project from remotes when there's two projects but one mortar remote" do
73
+ stub(@base.git).has_dot_git? {true}
74
+ stub(@base.git).remotes {{ 'mortar' => 'myproject', 'base' => 'my-base-project'}}
75
+ mock(@base.git).git("config mortar.remote", false).returns("")
76
+ @base.project.name.should == 'myproject'
77
+ end
78
+
72
79
  it "accepts a --remote argument to choose the project from the remote name" do
73
80
  stub(@base.git).has_dot_git?.returns(true)
74
81
  stub(@base.git).remotes.returns({ 'staging' => 'myproject-staging', 'production' => 'myproject' })
@@ -44,7 +44,7 @@ STDERR
44
44
  pigscript = Mortar::Project::PigScript.new(script_name, script_path)
45
45
  mock(Mortar::Project::PigScript).new(script_name, script_path).returns(pigscript)
46
46
  any_instance_of(Mortar::Local::Controller) do |u|
47
- mock(u).illustrate(pigscript, "some_alias", is_a(Mortar::PigVersion::Pig09), [], false).returns(nil)
47
+ mock(u).illustrate(pigscript, "some_alias", is_a(Mortar::PigVersion::Pig09), [], false, false).returns(nil)
48
48
  end
49
49
  stderr, stdout = execute("local:illustrate #{script_name} some_alias", p)
50
50
  stderr.should == ""
@@ -59,7 +59,7 @@ STDERR
59
59
  pigscript = Mortar::Project::PigScript.new(script_name, script_path)
60
60
  mock(Mortar::Project::PigScript).new(script_name, script_path).returns(pigscript)
61
61
  any_instance_of(Mortar::Local::Controller) do |u|
62
- mock(u).illustrate(pigscript, nil, is_a(Mortar::PigVersion::Pig012), [], false).returns(nil)
62
+ mock(u).illustrate(pigscript, nil, is_a(Mortar::PigVersion::Pig012), [], false, false).returns(nil)
63
63
  end
64
64
  stderr, stdout = execute("local:illustrate #{script_name} -g 0.12", p)
65
65
  stderr.should == ""
@@ -228,6 +228,50 @@ PARAMS
228
228
 
229
229
  end
230
230
 
231
+ context "local:luigi" do
232
+
233
+ it "Exits with error if no script specified" do
234
+ with_git_initialized_project do |p|
235
+ stderr, stdout = execute("local:luigi", p)
236
+ stderr.should == <<-STDERR
237
+ ! Usage: mortar local:luigi SCRIPT
238
+ ! Must specify SCRIPT.
239
+ STDERR
240
+ end
241
+ end
242
+
243
+ it "Exits with error if script doesn't exist" do
244
+ with_git_initialized_project do |p|
245
+ stderr, stdout = execute("local:luigi foobarbaz", p)
246
+ stderr.should == <<-STDERR
247
+ ! Unable to find luigiscript foobarbaz
248
+ ! No luigiscripts found
249
+ STDERR
250
+ end
251
+ end
252
+
253
+ it "Runs script forwarding options to luigi script" do
254
+ with_git_initialized_project do |p|
255
+ script_name = "some_luigi_script"
256
+ script_path = File.join(p.luigiscripts_path, "#{script_name}.py")
257
+ write_file(script_path)
258
+ luigi_script = Mortar::Project::LuigiScript.new(script_name, script_path)
259
+ mock(Mortar::Project::LuigiScript).new(script_name, script_path).returns(luigi_script)
260
+ any_instance_of(Mortar::Local::Python) do |u|
261
+ mock(u).run_luigi_script(luigi_script, %W{--myoption 2 --myotheroption 3})
262
+ end
263
+ any_instance_of(Mortar::Local::Controller) do |u|
264
+ mock(u).install_and_configure
265
+ end
266
+ stderr, stdout = execute("local:luigi some_luigi_script -p myoption=2 -p myotheroption=3", p)
267
+ stderr.should == ""
268
+ end
269
+ end
270
+
271
+
272
+ end
273
+
274
+
231
275
  end
232
276
  end
233
277
 
@@ -175,11 +175,12 @@ module Mortar::Local
175
175
  test_script = "foobar-script"
176
176
  script_alias = "some_alias"
177
177
  prune = false
178
+ no_browser = false
178
179
  the_parameters = []
179
180
  any_instance_of(Mortar::Local::Pig) do |p|
180
- mock(p).illustrate_alias(test_script, script_alias, prune, "0.9", the_parameters)
181
+ mock(p).illustrate_alias(test_script, script_alias, prune, no_browser, "0.9", the_parameters)
181
182
  end
182
- c.illustrate(test_script, script_alias, "0.9", the_parameters, prune)
183
+ c.illustrate(test_script, script_alias, "0.9", the_parameters, prune, no_browser)
183
184
  end
184
185
  end
185
186
 
@@ -19,6 +19,7 @@ require 'fakefs/spec_helpers'
19
19
  require 'mortar/local/pig'
20
20
  require 'mortar/auth'
21
21
  require 'launchy'
22
+ require 'mortar/pigversion'
22
23
 
23
24
  class Mortar::Local::Pig
24
25
  attr_reader :command
@@ -131,7 +132,9 @@ module Mortar::Local
131
132
  template_location = pig.resource_locations["illustrate_template"]
132
133
  template_contents = File.read(template_location)
133
134
  output_path = pig.resource_destinations["illustrate_html"]
134
- mock(pig).decode_illustrate_input_file("foo/bar/file.json").returns(fake_illustrate_data)
135
+ temp_text = "This is temporary text"
136
+ mock(pig).decode_illustrate_input_file("foo/bar/file.json").returns(temp_text)
137
+ mock(pig).json_decode(temp_text).returns(fake_illustrate_data)
135
138
 
136
139
  # TODO: test that these files are copied
137
140
  ["illustrate_css",
@@ -149,7 +152,7 @@ module Mortar::Local
149
152
  File.open(template_location, 'w') { |f| f.write(template_contents) }
150
153
  begin
151
154
  previous_stdout, $stdout = $stdout, StringIO.new
152
- pig.show_illustrate_output("foo/bar/file.json")
155
+ pig.show_illustrate_output_browser("foo/bar/file.json")
153
156
  ensure
154
157
  $stdout = previous_stdout
155
158
  end
@@ -172,7 +175,8 @@ module Mortar::Local
172
175
  illustrate_output_file = 'illustrate-output.json'
173
176
  FakeFS do
174
177
  File.open(illustrate_output_file, 'w') { |f| f.write(json) }
175
- actual_data = pig.decode_illustrate_input_file(illustrate_output_file)
178
+ actual_data_raw = pig.decode_illustrate_input_file(illustrate_output_file)
179
+ actual_data = pig.json_decode(actual_data_raw)
176
180
  expect(actual_data).to eq(expected_data)
177
181
  end
178
182
  end
@@ -187,7 +191,8 @@ module Mortar::Local
187
191
  illustrate_output_file = 'illustrate-output.json'
188
192
  FakeFS do
189
193
  File.open(illustrate_output_file, 'w') { |f| f.write(json) }
190
- actual_data = pig.decode_illustrate_input_file(illustrate_output_file)
194
+ actual_data_raw = pig.decode_illustrate_input_file(illustrate_output_file)
195
+ actual_data = pig.json_decode(actual_data_raw)
191
196
  expect(actual_data).to eq(expected_data)
192
197
  end
193
198
  end
@@ -201,9 +206,19 @@ module Mortar::Local
201
206
  script = Mortar::Project::PigScripts.new('/foo/bar/baz.pig', 'baz', 'pig')
202
207
  pig = Mortar::Local::Pig.new
203
208
  mock(pig).run_pig_command.with_any_args.returns(true)
204
- mock(pig).show_illustrate_output.with_any_args
209
+ mock(pig).show_illustrate_output_browser.with_any_args
205
210
  stub(pig).make_pig_param_file.returns('param.file')
206
- pig.illustrate_alias(script, 'my_alias', false, "0.9", [])
211
+ pig.illustrate_alias(script, 'my_alias', false, false, "0.9", [])
212
+ end
213
+
214
+ it "displays text results if illustrate was successful with no_browser" do
215
+ any_instance_of(Mortar::Project::PigScripts, :elements => nil, :path => "/foo/bar/baz.pig")
216
+ script = Mortar::Project::PigScripts.new('/foo/bar/baz.pig', 'baz', 'pig')
217
+ pig = Mortar::Local::Pig.new
218
+ stub(pig).run_pig_command.with_any_args.returns(true)
219
+ mock(pig).display.with_any_args
220
+ stub(pig).make_pig_param_file.returns('param.file')
221
+ pig.illustrate_alias(script, 'my_alias', false, true, "0.9", [])
207
222
  end
208
223
 
209
224
  it "skips results if illustrate was unsuccessful" do
@@ -211,9 +226,9 @@ module Mortar::Local
211
226
  script = Mortar::Project::PigScripts.new('/foo/bar/baz.pig', 'baz', 'pig')
212
227
  pig = Mortar::Local::Pig.new
213
228
  mock(pig).run_pig_command.with_any_args.returns(false)
214
- mock(pig).show_illustrate_output.with_any_args.never
229
+ mock(pig).show_illustrate_output_browser.with_any_args.never
215
230
  stub(pig).make_pig_param_file.returns('param.file')
216
- pig.illustrate_alias(script, 'my_alias', false, "0.9", [])
231
+ pig.illustrate_alias(script, 'my_alias', false, false, "0.9", [])
217
232
  end
218
233
 
219
234
  it "does not require login credentials for illustration" do
@@ -222,8 +237,8 @@ module Mortar::Local
222
237
  pig = Mortar::Local::Pig.new
223
238
  mock(Mortar::Auth).user_s3_safe(true).returns('notloggedin-user-org')
224
239
  mock(pig).run_pig_command.with_any_args.returns(true)
225
- mock(pig).show_illustrate_output.with_any_args
226
- pig.illustrate_alias(script, 'my_alias', false, "0.9", [])
240
+ mock(pig).show_illustrate_output_browser.with_any_args
241
+ pig.illustrate_alias(script, 'my_alias', false, false, "0.9", [])
227
242
  end
228
243
 
229
244
  end
@@ -128,5 +128,26 @@ module Mortar::Local
128
128
 
129
129
  end
130
130
 
131
+ context "running python commands" do
132
+
133
+ it "Generates the appropriate tempate variables" do
134
+ with_git_initialized_project do |p|
135
+ script_name = "some_luigi_script"
136
+ script_path = File.join(p.luigiscripts_path, "#{script_name}.py")
137
+ write_file(script_path)
138
+ luigi_script = Mortar::Project::LuigiScript.new(script_name, script_path)
139
+ args = %W{--myoption 2 --myotheroption 3}
140
+ py = Mortar::Local::Python.new
141
+ expected_hash = {
142
+ :python_arugments => "",
143
+ :python_script => "luigiscripts/#{script_name}.py",
144
+ :script_arguments => "--local-scheduler --logging-conf-file #{py.luigi_logging_config_file_path} --myoption 2 --myotheroption 3",
145
+ }
146
+ expect(py.luigi_command_template_parameters(luigi_script, args)).to eq(expected_hash)
147
+ end
148
+ end
149
+
150
+ end
151
+
131
152
  end
132
153
  end
metadata CHANGED
@@ -1,36 +1,32 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: mortar
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.14.1
5
- prerelease:
4
+ version: 0.15.0
6
5
  platform: ruby
7
6
  authors:
8
7
  - Mortar Data
9
8
  autorequire:
10
9
  bindir: bin
11
10
  cert_chain: []
12
- date: 2014-02-18 00:00:00.000000000 Z
11
+ date: 2014-02-25 00:00:00.000000000 Z
13
12
  dependencies:
14
13
  - !ruby/object:Gem::Dependency
15
14
  name: rdoc
16
15
  requirement: !ruby/object:Gem::Requirement
17
- none: false
18
16
  requirements:
19
- - - ! '>='
17
+ - - '>='
20
18
  - !ruby/object:Gem::Version
21
19
  version: 4.0.0
22
20
  type: :runtime
23
21
  prerelease: false
24
22
  version_requirements: !ruby/object:Gem::Requirement
25
- none: false
26
23
  requirements:
27
- - - ! '>='
24
+ - - '>='
28
25
  - !ruby/object:Gem::Version
29
26
  version: 4.0.0
30
27
  - !ruby/object:Gem::Dependency
31
28
  name: mortar-api-ruby
32
29
  requirement: !ruby/object:Gem::Requirement
33
- none: false
34
30
  requirements:
35
31
  - - ~>
36
32
  - !ruby/object:Gem::Version
@@ -38,7 +34,6 @@ dependencies:
38
34
  type: :runtime
39
35
  prerelease: false
40
36
  version_requirements: !ruby/object:Gem::Requirement
41
- none: false
42
37
  requirements:
43
38
  - - ~>
44
39
  - !ruby/object:Gem::Version
@@ -46,7 +41,6 @@ dependencies:
46
41
  - !ruby/object:Gem::Dependency
47
42
  name: netrc
48
43
  requirement: !ruby/object:Gem::Requirement
49
- none: false
50
44
  requirements:
51
45
  - - ~>
52
46
  - !ruby/object:Gem::Version
@@ -54,7 +48,6 @@ dependencies:
54
48
  type: :runtime
55
49
  prerelease: false
56
50
  version_requirements: !ruby/object:Gem::Requirement
57
- none: false
58
51
  requirements:
59
52
  - - ~>
60
53
  - !ruby/object:Gem::Version
@@ -62,7 +55,6 @@ dependencies:
62
55
  - !ruby/object:Gem::Dependency
63
56
  name: launchy
64
57
  requirement: !ruby/object:Gem::Requirement
65
- none: false
66
58
  requirements:
67
59
  - - ~>
68
60
  - !ruby/object:Gem::Version
@@ -70,7 +62,6 @@ dependencies:
70
62
  type: :runtime
71
63
  prerelease: false
72
64
  version_requirements: !ruby/object:Gem::Requirement
73
- none: false
74
65
  requirements:
75
66
  - - ~>
76
67
  - !ruby/object:Gem::Version
@@ -78,7 +69,6 @@ dependencies:
78
69
  - !ruby/object:Gem::Dependency
79
70
  name: parseconfig
80
71
  requirement: !ruby/object:Gem::Requirement
81
- none: false
82
72
  requirements:
83
73
  - - ~>
84
74
  - !ruby/object:Gem::Version
@@ -86,7 +76,6 @@ dependencies:
86
76
  type: :runtime
87
77
  prerelease: false
88
78
  version_requirements: !ruby/object:Gem::Requirement
89
- none: false
90
79
  requirements:
91
80
  - - ~>
92
81
  - !ruby/object:Gem::Version
@@ -94,7 +83,6 @@ dependencies:
94
83
  - !ruby/object:Gem::Dependency
95
84
  name: excon
96
85
  requirement: !ruby/object:Gem::Requirement
97
- none: false
98
86
  requirements:
99
87
  - - ~>
100
88
  - !ruby/object:Gem::Version
@@ -102,7 +90,6 @@ dependencies:
102
90
  type: :development
103
91
  prerelease: false
104
92
  version_requirements: !ruby/object:Gem::Requirement
105
- none: false
106
93
  requirements:
107
94
  - - ~>
108
95
  - !ruby/object:Gem::Version
@@ -110,7 +97,6 @@ dependencies:
110
97
  - !ruby/object:Gem::Dependency
111
98
  name: fakefs
112
99
  requirement: !ruby/object:Gem::Requirement
113
- none: false
114
100
  requirements:
115
101
  - - ~>
116
102
  - !ruby/object:Gem::Version
@@ -118,7 +104,6 @@ dependencies:
118
104
  type: :development
119
105
  prerelease: false
120
106
  version_requirements: !ruby/object:Gem::Requirement
121
- none: false
122
107
  requirements:
123
108
  - - ~>
124
109
  - !ruby/object:Gem::Version
@@ -126,65 +111,57 @@ dependencies:
126
111
  - !ruby/object:Gem::Dependency
127
112
  name: gem-release
128
113
  requirement: !ruby/object:Gem::Requirement
129
- none: false
130
114
  requirements:
131
- - - ! '>='
115
+ - - '>='
132
116
  - !ruby/object:Gem::Version
133
117
  version: '0'
134
118
  type: :development
135
119
  prerelease: false
136
120
  version_requirements: !ruby/object:Gem::Requirement
137
- none: false
138
121
  requirements:
139
- - - ! '>='
122
+ - - '>='
140
123
  - !ruby/object:Gem::Version
141
124
  version: '0'
142
125
  - !ruby/object:Gem::Dependency
143
126
  name: rake
144
127
  requirement: !ruby/object:Gem::Requirement
145
- none: false
146
128
  requirements:
147
- - - ! '>='
129
+ - - '>='
148
130
  - !ruby/object:Gem::Version
149
131
  version: '0'
150
132
  type: :development
151
133
  prerelease: false
152
134
  version_requirements: !ruby/object:Gem::Requirement
153
- none: false
154
135
  requirements:
155
- - - ! '>='
136
+ - - '>='
156
137
  - !ruby/object:Gem::Version
157
138
  version: '0'
158
139
  - !ruby/object:Gem::Dependency
159
140
  name: rr
160
141
  requirement: !ruby/object:Gem::Requirement
161
- none: false
162
142
  requirements:
163
- - - ! '>='
143
+ - - '>='
164
144
  - !ruby/object:Gem::Version
165
145
  version: '0'
166
146
  type: :development
167
147
  prerelease: false
168
148
  version_requirements: !ruby/object:Gem::Requirement
169
- none: false
170
149
  requirements:
171
- - - ! '>='
150
+ - - '>='
172
151
  - !ruby/object:Gem::Version
173
152
  version: '0'
174
153
  - !ruby/object:Gem::Dependency
175
154
  name: rspec
176
155
  requirement: !ruby/object:Gem::Requirement
177
- none: false
178
156
  requirements:
179
- - - ! '>='
157
+ - - '>='
180
158
  - !ruby/object:Gem::Version
181
159
  version: '0'
182
160
  type: :development
183
161
  prerelease: false
184
162
  version_requirements: !ruby/object:Gem::Requirement
185
- none: false
186
163
  requirements:
187
- - - ! '>='
164
+ - - '>='
188
165
  - !ruby/object:Gem::Version
189
166
  version: '0'
190
167
  description: Client library and command-line tool to interact with the Mortar service.
@@ -224,6 +201,7 @@ files:
224
201
  - lib/mortar/command/projects.rb
225
202
  - lib/mortar/command/validate.rb
226
203
  - lib/mortar/command/version.rb
204
+ - lib/mortar/conf/luigi/logging.ini
227
205
  - lib/mortar/errors.rb
228
206
  - lib/mortar/generators/characterize_generator.rb
229
207
  - lib/mortar/generators/controlscript_generator.rb
@@ -283,6 +261,7 @@ files:
283
261
  - lib/mortar/templates/project/vendor/udfs/python/gitkeep
284
262
  - lib/mortar/templates/report/illustrate-report.html
285
263
  - lib/mortar/templates/script/runpig.sh
264
+ - lib/mortar/templates/script/runpython.sh
286
265
  - lib/mortar/templates/udf/python_udf.py
287
266
  - lib/mortar/updater.rb
288
267
  - lib/mortar/version.rb
@@ -320,26 +299,25 @@ files:
320
299
  - spec/support/display_message_matcher.rb
321
300
  homepage: http://mortardata.com/
322
301
  licenses: []
302
+ metadata: {}
323
303
  post_install_message:
324
304
  rdoc_options: []
325
305
  require_paths:
326
306
  - lib
327
307
  required_ruby_version: !ruby/object:Gem::Requirement
328
- none: false
329
308
  requirements:
330
- - - ! '>='
309
+ - - '>='
331
310
  - !ruby/object:Gem::Version
332
311
  version: 1.8.7
333
312
  required_rubygems_version: !ruby/object:Gem::Requirement
334
- none: false
335
313
  requirements:
336
- - - ! '>='
314
+ - - '>='
337
315
  - !ruby/object:Gem::Version
338
316
  version: '0'
339
317
  requirements: []
340
318
  rubyforge_project:
341
- rubygems_version: 1.8.23
319
+ rubygems_version: 2.0.6
342
320
  signing_key:
343
- specification_version: 3
321
+ specification_version: 4
344
322
  summary: Client library and CLI to interact with the Mortar service.
345
323
  test_files: []