mortar 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (61) hide show
  1. data/README.md +36 -0
  2. data/bin/mortar +13 -0
  3. data/lib/mortar.rb +23 -0
  4. data/lib/mortar/auth.rb +312 -0
  5. data/lib/mortar/cli.rb +54 -0
  6. data/lib/mortar/command.rb +267 -0
  7. data/lib/mortar/command/auth.rb +96 -0
  8. data/lib/mortar/command/base.rb +319 -0
  9. data/lib/mortar/command/clusters.rb +41 -0
  10. data/lib/mortar/command/describe.rb +97 -0
  11. data/lib/mortar/command/generate.rb +121 -0
  12. data/lib/mortar/command/help.rb +166 -0
  13. data/lib/mortar/command/illustrate.rb +97 -0
  14. data/lib/mortar/command/jobs.rb +174 -0
  15. data/lib/mortar/command/pigscripts.rb +45 -0
  16. data/lib/mortar/command/projects.rb +128 -0
  17. data/lib/mortar/command/validate.rb +94 -0
  18. data/lib/mortar/command/version.rb +42 -0
  19. data/lib/mortar/errors.rb +24 -0
  20. data/lib/mortar/generators/generator_base.rb +107 -0
  21. data/lib/mortar/generators/macro_generator.rb +37 -0
  22. data/lib/mortar/generators/pigscript_generator.rb +40 -0
  23. data/lib/mortar/generators/project_generator.rb +67 -0
  24. data/lib/mortar/generators/udf_generator.rb +28 -0
  25. data/lib/mortar/git.rb +233 -0
  26. data/lib/mortar/helpers.rb +488 -0
  27. data/lib/mortar/project.rb +156 -0
  28. data/lib/mortar/snapshot.rb +39 -0
  29. data/lib/mortar/templates/macro/macro.pig +14 -0
  30. data/lib/mortar/templates/pigscript/pigscript.pig +38 -0
  31. data/lib/mortar/templates/pigscript/python_udf.py +13 -0
  32. data/lib/mortar/templates/project/Gemfile +3 -0
  33. data/lib/mortar/templates/project/README.md +8 -0
  34. data/lib/mortar/templates/project/gitignore +4 -0
  35. data/lib/mortar/templates/project/macros/gitkeep +0 -0
  36. data/lib/mortar/templates/project/pigscripts/pigscript.pig +35 -0
  37. data/lib/mortar/templates/project/udfs/python/python_udf.py +13 -0
  38. data/lib/mortar/templates/udf/python_udf.py +13 -0
  39. data/lib/mortar/version.rb +20 -0
  40. data/lib/vendor/mortar/okjson.rb +598 -0
  41. data/lib/vendor/mortar/uuid.rb +312 -0
  42. data/spec/mortar/auth_spec.rb +156 -0
  43. data/spec/mortar/command/auth_spec.rb +46 -0
  44. data/spec/mortar/command/base_spec.rb +82 -0
  45. data/spec/mortar/command/clusters_spec.rb +61 -0
  46. data/spec/mortar/command/describe_spec.rb +135 -0
  47. data/spec/mortar/command/generate_spec.rb +139 -0
  48. data/spec/mortar/command/illustrate_spec.rb +140 -0
  49. data/spec/mortar/command/jobs_spec.rb +364 -0
  50. data/spec/mortar/command/pigscripts_spec.rb +70 -0
  51. data/spec/mortar/command/projects_spec.rb +165 -0
  52. data/spec/mortar/command/validate_spec.rb +119 -0
  53. data/spec/mortar/command_spec.rb +122 -0
  54. data/spec/mortar/git_spec.rb +278 -0
  55. data/spec/mortar/helpers_spec.rb +82 -0
  56. data/spec/mortar/project_spec.rb +76 -0
  57. data/spec/mortar/snapshot_spec.rb +46 -0
  58. data/spec/spec.opts +1 -0
  59. data/spec/spec_helper.rb +278 -0
  60. data/spec/support/display_message_matcher.rb +68 -0
  61. metadata +259 -0
@@ -0,0 +1,488 @@
1
+ #
2
+ # Copyright 2012 Mortar Data Inc.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+ # Portions of this code from heroku (https://github.com/heroku/heroku/) Copyright Heroku 2008 - 2012,
17
+ # used under an MIT license (https://github.com/heroku/heroku/blob/master/LICENSE).
18
+ #
19
+
20
+ require 'fileutils'
21
+ require "vendor/mortar/okjson"
22
+
23
+ module Mortar
24
+ module Helpers
25
+
26
+ extend self
27
+
28
+ def home_directory
29
+ running_on_windows? ? ENV['USERPROFILE'].gsub("\\","/") : ENV['HOME']
30
+ end
31
+
32
+ def running_on_windows?
33
+ RUBY_PLATFORM =~ /mswin32|mingw32/
34
+ end
35
+
36
+ def running_on_a_mac?
37
+ RUBY_PLATFORM =~ /-darwin\d/
38
+ end
39
+
40
+ def write_to_file(str_data, path, mkdir_p=true)
41
+ if mkdir_p
42
+ FileUtils.mkdir_p File.dirname(path)
43
+ end
44
+ File.open(path, "w"){|f| f.write(str_data)}
45
+ end
46
+
47
+ def display(msg="", new_line=true)
48
+ if new_line
49
+ puts(msg)
50
+ else
51
+ print(msg)
52
+ $stdout.flush
53
+ end
54
+ end
55
+
56
+ def redisplay(line, line_break = false)
57
+ display("\r\e[0K#{line}", line_break)
58
+ end
59
+
60
+ def deprecate(message)
61
+ display "WARNING: #{message}"
62
+ end
63
+
64
+ def confirm(message="Are you sure you wish to continue? (y/n)?")
65
+ display("#{message} ", false)
66
+ ['y', 'yes'].include?(ask.downcase)
67
+ end
68
+
69
+ def format_date(date)
70
+ date = Time.parse(date) if date.is_a?(String)
71
+ date.strftime("%Y-%m-%d %H:%M %Z")
72
+ end
73
+
74
+ def ask
75
+ $stdin.gets.to_s.strip
76
+ end
77
+
78
+ def shell(cmd)
79
+ FileUtils.cd(Dir.pwd) {|d| return `#{cmd}`}
80
+ end
81
+
82
+ def retry_on_exception(*exceptions)
83
+ retry_count = 0
84
+ begin
85
+ yield
86
+ rescue *exceptions => ex
87
+ raise ex if retry_count >= 3
88
+ sleep 3
89
+ retry_count += 1
90
+ retry
91
+ end
92
+ end
93
+
94
+ def time_ago(elapsed)
95
+ if elapsed <= 60
96
+ "#{elapsed.floor}s ago"
97
+ elsif elapsed <= (60 * 60)
98
+ "#{(elapsed / 60).floor}m ago"
99
+ elsif elapsed <= (60 * 60 * 25)
100
+ "#{(elapsed / 60 / 60).floor}h ago"
101
+ else
102
+ (Time.now - elapsed).strftime("%Y/%m/%d %H:%M:%S")
103
+ end
104
+ end
105
+
106
+ def truncate(text, length)
107
+ if text.size > length
108
+ text[0, length - 2] + '..'
109
+ else
110
+ text
111
+ end
112
+ end
113
+
114
+ @@kb = 1024
115
+ @@mb = 1024 * @@kb
116
+ @@gb = 1024 * @@mb
117
+ def format_bytes(amount)
118
+ amount = amount.to_i
119
+ return '(empty)' if amount == 0
120
+ return amount if amount < @@kb
121
+ return "#{(amount / @@kb).round}k" if amount < @@mb
122
+ return "#{(amount / @@mb).round}M" if amount < @@gb
123
+ return "#{(amount / @@gb).round}G"
124
+ end
125
+
126
+ def quantify(string, num)
127
+ "%d %s" % [ num, num.to_i == 1 ? string : "#{string}s" ]
128
+ end
129
+
130
+ def longest(items)
131
+ items.map { |i| i.to_s.length }.sort.last
132
+ end
133
+
134
+ def display_table(objects, columns, headers)
135
+ lengths = []
136
+ columns.each_with_index do |column, index|
137
+ header = headers[index]
138
+ lengths << longest([header].concat(objects.map { |o| o[column].to_s }))
139
+ end
140
+ lines = lengths.map {|length| "-" * length}
141
+ lengths[-1] = 0 # remove padding from last column
142
+ display_row headers, lengths
143
+ display_row lines, lengths
144
+ objects.each do |row|
145
+ display_row columns.map { |column| row[column] }, lengths
146
+ end
147
+ end
148
+
149
+ def display_row(row, lengths)
150
+ row_data = []
151
+ row.zip(lengths).each do |column, length|
152
+ format = column.is_a?(Fixnum) ? "%#{length}s" : "%-#{length}s"
153
+ row_data << format % column
154
+ end
155
+ display(row_data.join(" "))
156
+ end
157
+
158
+ def json_encode(object)
159
+ Mortar::OkJson.encode(object)
160
+ rescue Mortar::OkJson::Error
161
+ nil
162
+ end
163
+
164
+ def json_decode(json)
165
+ Mortar::OkJson.decode(json)
166
+ rescue Mortar::OkJson::Error
167
+ nil
168
+ end
169
+
170
+ def set_buffer(enable)
171
+ with_tty do
172
+ if enable
173
+ `stty icanon echo`
174
+ else
175
+ `stty -icanon -echo`
176
+ end
177
+ end
178
+ end
179
+
180
+ def with_tty(&block)
181
+ return unless $stdin.isatty
182
+ begin
183
+ yield
184
+ rescue
185
+ # fails on windows
186
+ end
187
+ end
188
+
189
+ def get_terminal_environment
190
+ { "TERM" => ENV["TERM"], "COLUMNS" => `tput cols`.strip, "LINES" => `tput lines`.strip }
191
+ rescue
192
+ { "TERM" => ENV["TERM"] }
193
+ end
194
+
195
+ ## DISPLAY HELPERS
196
+
197
+ def action(message, options={})
198
+ display("#{message}... ", false)
199
+ Mortar::Helpers.error_with_failure = true
200
+ ret = yield
201
+ Mortar::Helpers.error_with_failure = false
202
+ display((options[:success] || "done"), false)
203
+ if @status
204
+ display(", #{@status}", false)
205
+ @status = nil
206
+ end
207
+ display
208
+ ret
209
+ end
210
+
211
+ def status(message)
212
+ @status = message
213
+ end
214
+
215
+ def format_with_bang(message)
216
+ return '' if message.to_s.strip == ""
217
+ " ! " + message.split("\n").join("\n ! ")
218
+ end
219
+
220
+ def output_with_bang(message="", new_line=true)
221
+ return if message.to_s.strip == ""
222
+ display(format_with_bang(message), new_line)
223
+ end
224
+
225
+ def error(message)
226
+ if Mortar::Helpers.error_with_failure
227
+ display("failed")
228
+ Mortar::Helpers.error_with_failure = false
229
+ end
230
+ $stderr.puts(format_with_bang(message))
231
+ exit(1)
232
+ end
233
+
234
+ def self.error_with_failure
235
+ @@error_with_failure ||= false
236
+ end
237
+
238
+ def self.error_with_failure=(new_error_with_failure)
239
+ @@error_with_failure = new_error_with_failure
240
+ end
241
+
242
+ def self.included_into
243
+ @@included_into ||= []
244
+ end
245
+
246
+ def self.extended_into
247
+ @@extended_into ||= []
248
+ end
249
+
250
+ def self.included(base)
251
+ included_into << base
252
+ end
253
+
254
+ def self.extended(base)
255
+ extended_into << base
256
+ end
257
+
258
+ def display_header(message="", new_line=true)
259
+ return if message.to_s.strip == ""
260
+ display("=== " + message.to_s.split("\n").join("\n=== "), new_line)
261
+ end
262
+
263
+ def display_object(object)
264
+ case object
265
+ when Array
266
+ # list of objects
267
+ object.each do |item|
268
+ display_object(item)
269
+ end
270
+ when Hash
271
+ # if all values are arrays, it is a list with headers
272
+ # otherwise it is a single header with pairs of data
273
+ if object.values.all? {|value| value.is_a?(Array)}
274
+ object.keys.sort_by {|key| key.to_s}.each do |key|
275
+ display_header(key)
276
+ display_object(object[key])
277
+ hputs
278
+ end
279
+ end
280
+ else
281
+ hputs(object.to_s)
282
+ end
283
+ end
284
+
285
+ def hputs(string='')
286
+ Kernel.puts(string)
287
+ end
288
+
289
+ def hprint(string='')
290
+ Kernel.print(string)
291
+ $stdout.flush
292
+ end
293
+
294
+ def ticking(sleep_time)
295
+ ticks = 0
296
+ loop do
297
+ yield(ticks)
298
+ ticks +=1
299
+ sleep sleep_time
300
+ end
301
+ end
302
+
303
+ def spinner(ticks)
304
+ %w(/ - \\ |)[ticks % 4]
305
+ end
306
+
307
+ # produces a printf formatter line for an array of items
308
+ # if an individual line item is an array, it will create columns
309
+ # that are lined-up
310
+ #
311
+ # line_formatter(["foo", "barbaz"]) # => "%-6s"
312
+ # line_formatter(["foo", "barbaz"], ["bar", "qux"]) # => "%-3s %-6s"
313
+ #
314
+ def line_formatter(array)
315
+ if array.any? {|item| item.is_a?(Array)}
316
+ cols = []
317
+ array.each do |item|
318
+ if item.is_a?(Array)
319
+ item.each_with_index { |val,idx| cols[idx] = [cols[idx]||0, (val || '').length].max }
320
+ end
321
+ end
322
+ cols.map { |col| "%-#{col}s" }.join(" ")
323
+ else
324
+ "%s"
325
+ end
326
+ end
327
+
328
+ def styled_array(array, options={})
329
+ fmt = line_formatter(array)
330
+ array = array.sort unless options[:sort] == false
331
+ array.each do |element|
332
+ display((fmt % element).rstrip)
333
+ end
334
+ display
335
+ end
336
+
337
+ def styled_error(error, message='Mortar client internal error.')
338
+ if Mortar::Helpers.error_with_failure
339
+ display("failed")
340
+ Mortar::Helpers.error_with_failure = false
341
+ end
342
+ $stderr.puts(" ! #{message}.")
343
+ $stderr.puts(" ! Search for help at: https://help.mortardata.com")
344
+ $stderr.puts(" ! Or report a bug at: https://github.com/mortardata/mortar/issues/new")
345
+ $stderr.puts
346
+ $stderr.puts(" Error: #{error.message} (#{error.class})")
347
+ $stderr.puts(" Backtrace: #{error.backtrace.first}")
348
+ error.backtrace[1..-1].each do |line|
349
+ $stderr.puts(" #{line}")
350
+ end
351
+ if error.backtrace.length > 1
352
+ $stderr.puts
353
+ end
354
+ command = ARGV.map do |arg|
355
+ if arg.include?(' ')
356
+ arg = %{"#{arg}"}
357
+ else
358
+ arg
359
+ end
360
+ end.join(' ')
361
+ $stderr.puts(" Command: mortar #{command}")
362
+ unless Mortar::Auth.host == Mortar::Auth.default_host
363
+ $stderr.puts(" Host: #{Mortar::Auth.host}")
364
+ end
365
+ if http_proxy = ENV['http_proxy'] || ENV['HTTP_PROXY']
366
+ $stderr.puts(" HTTP Proxy: #{http_proxy}")
367
+ end
368
+ if https_proxy = ENV['https_proxy'] || ENV['HTTPS_PROXY']
369
+ $stderr.puts(" HTTPS Proxy: #{https_proxy}")
370
+ end
371
+ $stderr.puts(" Version: #{Mortar::USER_AGENT}")
372
+ $stderr.puts
373
+ end
374
+
375
+ def styled_header(header)
376
+ display("=== #{header}")
377
+ end
378
+
379
+ def styled_hash(hash, keys=nil, indent=0)
380
+ def display_with_indent(indent, msg="", new_line=true)
381
+ display("#{"".ljust(indent)}#{msg.to_s}", new_line)
382
+ end
383
+
384
+ max_key_length = hash.keys.map {|key| key.to_s.length}.max + 2
385
+ keys ||= hash.keys.sort {|x,y| x.to_s <=> y.to_s}
386
+ keys.each do |key|
387
+ case value = hash[key]
388
+ when Hash
389
+ display_with_indent(indent, "#{key}: ".ljust(max_key_length))
390
+ styled_hash(hash[key], nil, indent + 2)
391
+ when Array
392
+ if value.empty?
393
+ next
394
+ else
395
+ elements = value.sort {|x,y| x.to_s <=> y.to_s}
396
+ display_with_indent(indent, "#{key}: ".ljust(max_key_length), false)
397
+ display_with_indent(indent, elements[0])
398
+ elements[1..-1].each do |element|
399
+ display_with_indent(indent, "#{' ' * max_key_length}#{element}")
400
+ end
401
+ if elements.length > 1
402
+ display
403
+ end
404
+ end
405
+ when nil
406
+ next
407
+ else
408
+ display_with_indent(indent, "#{key}: ".ljust(max_key_length), false)
409
+ display_with_indent(indent, value)
410
+ end
411
+ end
412
+ end
413
+
414
+ def string_distance(first, last)
415
+ distances = [] # 0x0s
416
+ 0.upto(first.length) do |index|
417
+ distances << [index] + [0] * last.length
418
+ end
419
+ distances[0] = 0.upto(last.length).to_a
420
+ 1.upto(last.length) do |last_index|
421
+ 1.upto(first.length) do |first_index|
422
+ first_char = first[first_index - 1, 1]
423
+ last_char = last[last_index - 1, 1]
424
+ if first_char == last_char
425
+ distances[first_index][last_index] = distances[first_index - 1][last_index - 1] # noop
426
+ else
427
+ distances[first_index][last_index] = [
428
+ distances[first_index - 1][last_index], # deletion
429
+ distances[first_index][last_index - 1], # insertion
430
+ distances[first_index - 1][last_index - 1] # substitution
431
+ ].min + 1 # cost
432
+ if first_index > 1 && last_index > 1
433
+ first_previous_char = first[first_index - 2, 1]
434
+ last_previous_char = last[last_index - 2, 1]
435
+ if first_char == last_previous_char && first_previous_char == last_char
436
+ distances[first_index][last_index] = [
437
+ distances[first_index][last_index],
438
+ distances[first_index - 2][last_index - 2] + 1 # transposition
439
+ ].min
440
+ end
441
+ end
442
+ end
443
+ end
444
+ end
445
+ distances[first.length][last.length]
446
+ end
447
+
448
+ def suggestion(actual, possibilities)
449
+ distances = Hash.new {|hash,key| hash[key] = []}
450
+
451
+ possibilities.each do |suggestion|
452
+ distances[string_distance(actual, suggestion)] << suggestion
453
+ end
454
+
455
+ minimum_distance = distances.keys.min
456
+ if minimum_distance < 4
457
+ suggestions = distances[minimum_distance].sort
458
+ if suggestions.length == 1
459
+ "Perhaps you meant `#{suggestions.first}`."
460
+ else
461
+ "Perhaps you meant #{suggestions[0...-1].map {|suggestion| "`#{suggestion}`"}.join(', ')} or `#{suggestions.last}`."
462
+ end
463
+ else
464
+ nil
465
+ end
466
+ end
467
+
468
+ private
469
+
470
+ def create_display_method(name, colour_code, new_line=true)
471
+ define_method("display_#{name}") do |msg|
472
+ if new_line
473
+ printf("\e[#{colour_code}m%12s\e[0m #{msg}\n", name)
474
+ else
475
+ printf("\e[#{colour_code}m#{name}\e[0m\t#{msg}")
476
+ $stdout.flush
477
+ end
478
+ end
479
+ end
480
+
481
+ create_display_method("create", "1;32")
482
+ create_display_method("run", "1;32")
483
+ create_display_method("exists", "1;34")
484
+ create_display_method("identical", "1;34")
485
+ create_display_method("conflict", "1;31")
486
+
487
+ end
488
+ end