mortar 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. data/README.md +36 -0
  2. data/bin/mortar +13 -0
  3. data/lib/mortar.rb +23 -0
  4. data/lib/mortar/auth.rb +312 -0
  5. data/lib/mortar/cli.rb +54 -0
  6. data/lib/mortar/command.rb +267 -0
  7. data/lib/mortar/command/auth.rb +96 -0
  8. data/lib/mortar/command/base.rb +319 -0
  9. data/lib/mortar/command/clusters.rb +41 -0
  10. data/lib/mortar/command/describe.rb +97 -0
  11. data/lib/mortar/command/generate.rb +121 -0
  12. data/lib/mortar/command/help.rb +166 -0
  13. data/lib/mortar/command/illustrate.rb +97 -0
  14. data/lib/mortar/command/jobs.rb +174 -0
  15. data/lib/mortar/command/pigscripts.rb +45 -0
  16. data/lib/mortar/command/projects.rb +128 -0
  17. data/lib/mortar/command/validate.rb +94 -0
  18. data/lib/mortar/command/version.rb +42 -0
  19. data/lib/mortar/errors.rb +24 -0
  20. data/lib/mortar/generators/generator_base.rb +107 -0
  21. data/lib/mortar/generators/macro_generator.rb +37 -0
  22. data/lib/mortar/generators/pigscript_generator.rb +40 -0
  23. data/lib/mortar/generators/project_generator.rb +67 -0
  24. data/lib/mortar/generators/udf_generator.rb +28 -0
  25. data/lib/mortar/git.rb +233 -0
  26. data/lib/mortar/helpers.rb +488 -0
  27. data/lib/mortar/project.rb +156 -0
  28. data/lib/mortar/snapshot.rb +39 -0
  29. data/lib/mortar/templates/macro/macro.pig +14 -0
  30. data/lib/mortar/templates/pigscript/pigscript.pig +38 -0
  31. data/lib/mortar/templates/pigscript/python_udf.py +13 -0
  32. data/lib/mortar/templates/project/Gemfile +3 -0
  33. data/lib/mortar/templates/project/README.md +8 -0
  34. data/lib/mortar/templates/project/gitignore +4 -0
  35. data/lib/mortar/templates/project/macros/gitkeep +0 -0
  36. data/lib/mortar/templates/project/pigscripts/pigscript.pig +35 -0
  37. data/lib/mortar/templates/project/udfs/python/python_udf.py +13 -0
  38. data/lib/mortar/templates/udf/python_udf.py +13 -0
  39. data/lib/mortar/version.rb +20 -0
  40. data/lib/vendor/mortar/okjson.rb +598 -0
  41. data/lib/vendor/mortar/uuid.rb +312 -0
  42. data/spec/mortar/auth_spec.rb +156 -0
  43. data/spec/mortar/command/auth_spec.rb +46 -0
  44. data/spec/mortar/command/base_spec.rb +82 -0
  45. data/spec/mortar/command/clusters_spec.rb +61 -0
  46. data/spec/mortar/command/describe_spec.rb +135 -0
  47. data/spec/mortar/command/generate_spec.rb +139 -0
  48. data/spec/mortar/command/illustrate_spec.rb +140 -0
  49. data/spec/mortar/command/jobs_spec.rb +364 -0
  50. data/spec/mortar/command/pigscripts_spec.rb +70 -0
  51. data/spec/mortar/command/projects_spec.rb +165 -0
  52. data/spec/mortar/command/validate_spec.rb +119 -0
  53. data/spec/mortar/command_spec.rb +122 -0
  54. data/spec/mortar/git_spec.rb +278 -0
  55. data/spec/mortar/helpers_spec.rb +82 -0
  56. data/spec/mortar/project_spec.rb +76 -0
  57. data/spec/mortar/snapshot_spec.rb +46 -0
  58. data/spec/spec.opts +1 -0
  59. data/spec/spec_helper.rb +278 -0
  60. data/spec/support/display_message_matcher.rb +68 -0
  61. metadata +259 -0
@@ -0,0 +1,488 @@
1
+ #
2
+ # Copyright 2012 Mortar Data Inc.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+ # Portions of this code from heroku (https://github.com/heroku/heroku/) Copyright Heroku 2008 - 2012,
17
+ # used under an MIT license (https://github.com/heroku/heroku/blob/master/LICENSE).
18
+ #
19
+
20
+ require 'fileutils'
21
+ require "vendor/mortar/okjson"
22
+
23
+ module Mortar
24
+ module Helpers
25
+
26
+ extend self
27
+
28
+ def home_directory
29
+ running_on_windows? ? ENV['USERPROFILE'].gsub("\\","/") : ENV['HOME']
30
+ end
31
+
32
+ def running_on_windows?
33
+ RUBY_PLATFORM =~ /mswin32|mingw32/
34
+ end
35
+
36
+ def running_on_a_mac?
37
+ RUBY_PLATFORM =~ /-darwin\d/
38
+ end
39
+
40
+ def write_to_file(str_data, path, mkdir_p=true)
41
+ if mkdir_p
42
+ FileUtils.mkdir_p File.dirname(path)
43
+ end
44
+ File.open(path, "w"){|f| f.write(str_data)}
45
+ end
46
+
47
+ def display(msg="", new_line=true)
48
+ if new_line
49
+ puts(msg)
50
+ else
51
+ print(msg)
52
+ $stdout.flush
53
+ end
54
+ end
55
+
56
+ def redisplay(line, line_break = false)
57
+ display("\r\e[0K#{line}", line_break)
58
+ end
59
+
60
+ def deprecate(message)
61
+ display "WARNING: #{message}"
62
+ end
63
+
64
+ def confirm(message="Are you sure you wish to continue? (y/n)?")
65
+ display("#{message} ", false)
66
+ ['y', 'yes'].include?(ask.downcase)
67
+ end
68
+
69
+ def format_date(date)
70
+ date = Time.parse(date) if date.is_a?(String)
71
+ date.strftime("%Y-%m-%d %H:%M %Z")
72
+ end
73
+
74
+ def ask
75
+ $stdin.gets.to_s.strip
76
+ end
77
+
78
+ def shell(cmd)
79
+ FileUtils.cd(Dir.pwd) {|d| return `#{cmd}`}
80
+ end
81
+
82
+ def retry_on_exception(*exceptions)
83
+ retry_count = 0
84
+ begin
85
+ yield
86
+ rescue *exceptions => ex
87
+ raise ex if retry_count >= 3
88
+ sleep 3
89
+ retry_count += 1
90
+ retry
91
+ end
92
+ end
93
+
94
+ def time_ago(elapsed)
95
+ if elapsed <= 60
96
+ "#{elapsed.floor}s ago"
97
+ elsif elapsed <= (60 * 60)
98
+ "#{(elapsed / 60).floor}m ago"
99
+ elsif elapsed <= (60 * 60 * 25)
100
+ "#{(elapsed / 60 / 60).floor}h ago"
101
+ else
102
+ (Time.now - elapsed).strftime("%Y/%m/%d %H:%M:%S")
103
+ end
104
+ end
105
+
106
+ def truncate(text, length)
107
+ if text.size > length
108
+ text[0, length - 2] + '..'
109
+ else
110
+ text
111
+ end
112
+ end
113
+
114
+ @@kb = 1024
115
+ @@mb = 1024 * @@kb
116
+ @@gb = 1024 * @@mb
117
+ def format_bytes(amount)
118
+ amount = amount.to_i
119
+ return '(empty)' if amount == 0
120
+ return amount if amount < @@kb
121
+ return "#{(amount / @@kb).round}k" if amount < @@mb
122
+ return "#{(amount / @@mb).round}M" if amount < @@gb
123
+ return "#{(amount / @@gb).round}G"
124
+ end
125
+
126
+ def quantify(string, num)
127
+ "%d %s" % [ num, num.to_i == 1 ? string : "#{string}s" ]
128
+ end
129
+
130
+ def longest(items)
131
+ items.map { |i| i.to_s.length }.sort.last
132
+ end
133
+
134
+ def display_table(objects, columns, headers)
135
+ lengths = []
136
+ columns.each_with_index do |column, index|
137
+ header = headers[index]
138
+ lengths << longest([header].concat(objects.map { |o| o[column].to_s }))
139
+ end
140
+ lines = lengths.map {|length| "-" * length}
141
+ lengths[-1] = 0 # remove padding from last column
142
+ display_row headers, lengths
143
+ display_row lines, lengths
144
+ objects.each do |row|
145
+ display_row columns.map { |column| row[column] }, lengths
146
+ end
147
+ end
148
+
149
+ def display_row(row, lengths)
150
+ row_data = []
151
+ row.zip(lengths).each do |column, length|
152
+ format = column.is_a?(Fixnum) ? "%#{length}s" : "%-#{length}s"
153
+ row_data << format % column
154
+ end
155
+ display(row_data.join(" "))
156
+ end
157
+
158
+ def json_encode(object)
159
+ Mortar::OkJson.encode(object)
160
+ rescue Mortar::OkJson::Error
161
+ nil
162
+ end
163
+
164
+ def json_decode(json)
165
+ Mortar::OkJson.decode(json)
166
+ rescue Mortar::OkJson::Error
167
+ nil
168
+ end
169
+
170
+ def set_buffer(enable)
171
+ with_tty do
172
+ if enable
173
+ `stty icanon echo`
174
+ else
175
+ `stty -icanon -echo`
176
+ end
177
+ end
178
+ end
179
+
180
+ def with_tty(&block)
181
+ return unless $stdin.isatty
182
+ begin
183
+ yield
184
+ rescue
185
+ # fails on windows
186
+ end
187
+ end
188
+
189
+ def get_terminal_environment
190
+ { "TERM" => ENV["TERM"], "COLUMNS" => `tput cols`.strip, "LINES" => `tput lines`.strip }
191
+ rescue
192
+ { "TERM" => ENV["TERM"] }
193
+ end
194
+
195
+ ## DISPLAY HELPERS
196
+
197
+ def action(message, options={})
198
+ display("#{message}... ", false)
199
+ Mortar::Helpers.error_with_failure = true
200
+ ret = yield
201
+ Mortar::Helpers.error_with_failure = false
202
+ display((options[:success] || "done"), false)
203
+ if @status
204
+ display(", #{@status}", false)
205
+ @status = nil
206
+ end
207
+ display
208
+ ret
209
+ end
210
+
211
+ def status(message)
212
+ @status = message
213
+ end
214
+
215
+ def format_with_bang(message)
216
+ return '' if message.to_s.strip == ""
217
+ " ! " + message.split("\n").join("\n ! ")
218
+ end
219
+
220
+ def output_with_bang(message="", new_line=true)
221
+ return if message.to_s.strip == ""
222
+ display(format_with_bang(message), new_line)
223
+ end
224
+
225
+ def error(message)
226
+ if Mortar::Helpers.error_with_failure
227
+ display("failed")
228
+ Mortar::Helpers.error_with_failure = false
229
+ end
230
+ $stderr.puts(format_with_bang(message))
231
+ exit(1)
232
+ end
233
+
234
+ def self.error_with_failure
235
+ @@error_with_failure ||= false
236
+ end
237
+
238
+ def self.error_with_failure=(new_error_with_failure)
239
+ @@error_with_failure = new_error_with_failure
240
+ end
241
+
242
+ def self.included_into
243
+ @@included_into ||= []
244
+ end
245
+
246
+ def self.extended_into
247
+ @@extended_into ||= []
248
+ end
249
+
250
+ def self.included(base)
251
+ included_into << base
252
+ end
253
+
254
+ def self.extended(base)
255
+ extended_into << base
256
+ end
257
+
258
+ def display_header(message="", new_line=true)
259
+ return if message.to_s.strip == ""
260
+ display("=== " + message.to_s.split("\n").join("\n=== "), new_line)
261
+ end
262
+
263
+ def display_object(object)
264
+ case object
265
+ when Array
266
+ # list of objects
267
+ object.each do |item|
268
+ display_object(item)
269
+ end
270
+ when Hash
271
+ # if all values are arrays, it is a list with headers
272
+ # otherwise it is a single header with pairs of data
273
+ if object.values.all? {|value| value.is_a?(Array)}
274
+ object.keys.sort_by {|key| key.to_s}.each do |key|
275
+ display_header(key)
276
+ display_object(object[key])
277
+ hputs
278
+ end
279
+ end
280
+ else
281
+ hputs(object.to_s)
282
+ end
283
+ end
284
+
285
+ def hputs(string='')
286
+ Kernel.puts(string)
287
+ end
288
+
289
+ def hprint(string='')
290
+ Kernel.print(string)
291
+ $stdout.flush
292
+ end
293
+
294
+ def ticking(sleep_time)
295
+ ticks = 0
296
+ loop do
297
+ yield(ticks)
298
+ ticks +=1
299
+ sleep sleep_time
300
+ end
301
+ end
302
+
303
+ def spinner(ticks)
304
+ %w(/ - \\ |)[ticks % 4]
305
+ end
306
+
307
+ # produces a printf formatter line for an array of items
308
+ # if an individual line item is an array, it will create columns
309
+ # that are lined-up
310
+ #
311
+ # line_formatter(["foo", "barbaz"]) # => "%-6s"
312
+ # line_formatter(["foo", "barbaz"], ["bar", "qux"]) # => "%-3s %-6s"
313
+ #
314
+ def line_formatter(array)
315
+ if array.any? {|item| item.is_a?(Array)}
316
+ cols = []
317
+ array.each do |item|
318
+ if item.is_a?(Array)
319
+ item.each_with_index { |val,idx| cols[idx] = [cols[idx]||0, (val || '').length].max }
320
+ end
321
+ end
322
+ cols.map { |col| "%-#{col}s" }.join(" ")
323
+ else
324
+ "%s"
325
+ end
326
+ end
327
+
328
+ def styled_array(array, options={})
329
+ fmt = line_formatter(array)
330
+ array = array.sort unless options[:sort] == false
331
+ array.each do |element|
332
+ display((fmt % element).rstrip)
333
+ end
334
+ display
335
+ end
336
+
337
+ def styled_error(error, message='Mortar client internal error.')
338
+ if Mortar::Helpers.error_with_failure
339
+ display("failed")
340
+ Mortar::Helpers.error_with_failure = false
341
+ end
342
+ $stderr.puts(" ! #{message}.")
343
+ $stderr.puts(" ! Search for help at: https://help.mortardata.com")
344
+ $stderr.puts(" ! Or report a bug at: https://github.com/mortardata/mortar/issues/new")
345
+ $stderr.puts
346
+ $stderr.puts(" Error: #{error.message} (#{error.class})")
347
+ $stderr.puts(" Backtrace: #{error.backtrace.first}")
348
+ error.backtrace[1..-1].each do |line|
349
+ $stderr.puts(" #{line}")
350
+ end
351
+ if error.backtrace.length > 1
352
+ $stderr.puts
353
+ end
354
+ command = ARGV.map do |arg|
355
+ if arg.include?(' ')
356
+ arg = %{"#{arg}"}
357
+ else
358
+ arg
359
+ end
360
+ end.join(' ')
361
+ $stderr.puts(" Command: mortar #{command}")
362
+ unless Mortar::Auth.host == Mortar::Auth.default_host
363
+ $stderr.puts(" Host: #{Mortar::Auth.host}")
364
+ end
365
+ if http_proxy = ENV['http_proxy'] || ENV['HTTP_PROXY']
366
+ $stderr.puts(" HTTP Proxy: #{http_proxy}")
367
+ end
368
+ if https_proxy = ENV['https_proxy'] || ENV['HTTPS_PROXY']
369
+ $stderr.puts(" HTTPS Proxy: #{https_proxy}")
370
+ end
371
+ $stderr.puts(" Version: #{Mortar::USER_AGENT}")
372
+ $stderr.puts
373
+ end
374
+
375
+ def styled_header(header)
376
+ display("=== #{header}")
377
+ end
378
+
379
+ def styled_hash(hash, keys=nil, indent=0)
380
+ def display_with_indent(indent, msg="", new_line=true)
381
+ display("#{"".ljust(indent)}#{msg.to_s}", new_line)
382
+ end
383
+
384
+ max_key_length = hash.keys.map {|key| key.to_s.length}.max + 2
385
+ keys ||= hash.keys.sort {|x,y| x.to_s <=> y.to_s}
386
+ keys.each do |key|
387
+ case value = hash[key]
388
+ when Hash
389
+ display_with_indent(indent, "#{key}: ".ljust(max_key_length))
390
+ styled_hash(hash[key], nil, indent + 2)
391
+ when Array
392
+ if value.empty?
393
+ next
394
+ else
395
+ elements = value.sort {|x,y| x.to_s <=> y.to_s}
396
+ display_with_indent(indent, "#{key}: ".ljust(max_key_length), false)
397
+ display_with_indent(indent, elements[0])
398
+ elements[1..-1].each do |element|
399
+ display_with_indent(indent, "#{' ' * max_key_length}#{element}")
400
+ end
401
+ if elements.length > 1
402
+ display
403
+ end
404
+ end
405
+ when nil
406
+ next
407
+ else
408
+ display_with_indent(indent, "#{key}: ".ljust(max_key_length), false)
409
+ display_with_indent(indent, value)
410
+ end
411
+ end
412
+ end
413
+
414
+ def string_distance(first, last)
415
+ distances = [] # 0x0s
416
+ 0.upto(first.length) do |index|
417
+ distances << [index] + [0] * last.length
418
+ end
419
+ distances[0] = 0.upto(last.length).to_a
420
+ 1.upto(last.length) do |last_index|
421
+ 1.upto(first.length) do |first_index|
422
+ first_char = first[first_index - 1, 1]
423
+ last_char = last[last_index - 1, 1]
424
+ if first_char == last_char
425
+ distances[first_index][last_index] = distances[first_index - 1][last_index - 1] # noop
426
+ else
427
+ distances[first_index][last_index] = [
428
+ distances[first_index - 1][last_index], # deletion
429
+ distances[first_index][last_index - 1], # insertion
430
+ distances[first_index - 1][last_index - 1] # substitution
431
+ ].min + 1 # cost
432
+ if first_index > 1 && last_index > 1
433
+ first_previous_char = first[first_index - 2, 1]
434
+ last_previous_char = last[last_index - 2, 1]
435
+ if first_char == last_previous_char && first_previous_char == last_char
436
+ distances[first_index][last_index] = [
437
+ distances[first_index][last_index],
438
+ distances[first_index - 2][last_index - 2] + 1 # transposition
439
+ ].min
440
+ end
441
+ end
442
+ end
443
+ end
444
+ end
445
+ distances[first.length][last.length]
446
+ end
447
+
448
+ def suggestion(actual, possibilities)
449
+ distances = Hash.new {|hash,key| hash[key] = []}
450
+
451
+ possibilities.each do |suggestion|
452
+ distances[string_distance(actual, suggestion)] << suggestion
453
+ end
454
+
455
+ minimum_distance = distances.keys.min
456
+ if minimum_distance < 4
457
+ suggestions = distances[minimum_distance].sort
458
+ if suggestions.length == 1
459
+ "Perhaps you meant `#{suggestions.first}`."
460
+ else
461
+ "Perhaps you meant #{suggestions[0...-1].map {|suggestion| "`#{suggestion}`"}.join(', ')} or `#{suggestions.last}`."
462
+ end
463
+ else
464
+ nil
465
+ end
466
+ end
467
+
468
+ private
469
+
470
+ def create_display_method(name, colour_code, new_line=true)
471
+ define_method("display_#{name}") do |msg|
472
+ if new_line
473
+ printf("\e[#{colour_code}m%12s\e[0m #{msg}\n", name)
474
+ else
475
+ printf("\e[#{colour_code}m#{name}\e[0m\t#{msg}")
476
+ $stdout.flush
477
+ end
478
+ end
479
+ end
480
+
481
+ create_display_method("create", "1;32")
482
+ create_display_method("run", "1;32")
483
+ create_display_method("exists", "1;34")
484
+ create_display_method("identical", "1;34")
485
+ create_display_method("conflict", "1;31")
486
+
487
+ end
488
+ end