mortar 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. data/README.md +36 -0
  2. data/bin/mortar +13 -0
  3. data/lib/mortar.rb +23 -0
  4. data/lib/mortar/auth.rb +312 -0
  5. data/lib/mortar/cli.rb +54 -0
  6. data/lib/mortar/command.rb +267 -0
  7. data/lib/mortar/command/auth.rb +96 -0
  8. data/lib/mortar/command/base.rb +319 -0
  9. data/lib/mortar/command/clusters.rb +41 -0
  10. data/lib/mortar/command/describe.rb +97 -0
  11. data/lib/mortar/command/generate.rb +121 -0
  12. data/lib/mortar/command/help.rb +166 -0
  13. data/lib/mortar/command/illustrate.rb +97 -0
  14. data/lib/mortar/command/jobs.rb +174 -0
  15. data/lib/mortar/command/pigscripts.rb +45 -0
  16. data/lib/mortar/command/projects.rb +128 -0
  17. data/lib/mortar/command/validate.rb +94 -0
  18. data/lib/mortar/command/version.rb +42 -0
  19. data/lib/mortar/errors.rb +24 -0
  20. data/lib/mortar/generators/generator_base.rb +107 -0
  21. data/lib/mortar/generators/macro_generator.rb +37 -0
  22. data/lib/mortar/generators/pigscript_generator.rb +40 -0
  23. data/lib/mortar/generators/project_generator.rb +67 -0
  24. data/lib/mortar/generators/udf_generator.rb +28 -0
  25. data/lib/mortar/git.rb +233 -0
  26. data/lib/mortar/helpers.rb +488 -0
  27. data/lib/mortar/project.rb +156 -0
  28. data/lib/mortar/snapshot.rb +39 -0
  29. data/lib/mortar/templates/macro/macro.pig +14 -0
  30. data/lib/mortar/templates/pigscript/pigscript.pig +38 -0
  31. data/lib/mortar/templates/pigscript/python_udf.py +13 -0
  32. data/lib/mortar/templates/project/Gemfile +3 -0
  33. data/lib/mortar/templates/project/README.md +8 -0
  34. data/lib/mortar/templates/project/gitignore +4 -0
  35. data/lib/mortar/templates/project/macros/gitkeep +0 -0
  36. data/lib/mortar/templates/project/pigscripts/pigscript.pig +35 -0
  37. data/lib/mortar/templates/project/udfs/python/python_udf.py +13 -0
  38. data/lib/mortar/templates/udf/python_udf.py +13 -0
  39. data/lib/mortar/version.rb +20 -0
  40. data/lib/vendor/mortar/okjson.rb +598 -0
  41. data/lib/vendor/mortar/uuid.rb +312 -0
  42. data/spec/mortar/auth_spec.rb +156 -0
  43. data/spec/mortar/command/auth_spec.rb +46 -0
  44. data/spec/mortar/command/base_spec.rb +82 -0
  45. data/spec/mortar/command/clusters_spec.rb +61 -0
  46. data/spec/mortar/command/describe_spec.rb +135 -0
  47. data/spec/mortar/command/generate_spec.rb +139 -0
  48. data/spec/mortar/command/illustrate_spec.rb +140 -0
  49. data/spec/mortar/command/jobs_spec.rb +364 -0
  50. data/spec/mortar/command/pigscripts_spec.rb +70 -0
  51. data/spec/mortar/command/projects_spec.rb +165 -0
  52. data/spec/mortar/command/validate_spec.rb +119 -0
  53. data/spec/mortar/command_spec.rb +122 -0
  54. data/spec/mortar/git_spec.rb +278 -0
  55. data/spec/mortar/helpers_spec.rb +82 -0
  56. data/spec/mortar/project_spec.rb +76 -0
  57. data/spec/mortar/snapshot_spec.rb +46 -0
  58. data/spec/spec.opts +1 -0
  59. data/spec/spec_helper.rb +278 -0
  60. data/spec/support/display_message_matcher.rb +68 -0
  61. metadata +259 -0
@@ -0,0 +1,96 @@
1
+ #
2
+ # Copyright 2012 Mortar Data Inc.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+ # Portions of this code from heroku (https://github.com/heroku/heroku/) Copyright Heroku 2008 - 2012,
17
+ # used under an MIT license (https://github.com/heroku/heroku/blob/master/LICENSE).
18
+ #
19
+
20
+ require "mortar/command/base"
21
+
22
+ # authentication (login, logout)
23
+ #
24
+ class Mortar::Command::Auth < Mortar::Command::Base
25
+
26
+ # auth:login
27
+ #
28
+ # log in with your mortar credentials
29
+ #
30
+ #Example:
31
+ #
32
+ # $ mortar auth:login
33
+ # Enter your Mortar credentials:
34
+ # Email: email@example.com
35
+ # Password (typing will be hidden):
36
+ # Authentication successful.
37
+ #
38
+ def login
39
+ validate_arguments!
40
+
41
+ Mortar::Auth.login
42
+ display "Authentication successful."
43
+ end
44
+
45
+ alias_command "login", "auth:login"
46
+
47
+ # auth:logout
48
+ #
49
+ # clear local authentication credentials
50
+ #
51
+ #Example:
52
+ #
53
+ # $ mortar auth:logout
54
+ # Local credentials cleared.
55
+ #
56
+ def logout
57
+ validate_arguments!
58
+
59
+ Mortar::Auth.logout
60
+ display "Local credentials cleared."
61
+ end
62
+
63
+ alias_command "logout", "auth:logout"
64
+
65
+ # auth:key
66
+ #
67
+ # display your api key
68
+ #
69
+ #Example:
70
+ #
71
+ # $ mortar auth:key
72
+ # ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789ABCD
73
+ #
74
+ def key
75
+ validate_arguments!
76
+
77
+ display Mortar::Auth.password
78
+ end
79
+
80
+ # auth:whoami
81
+ #
82
+ # display your mortar email address
83
+ #
84
+ #Example:
85
+ #
86
+ # $ mortar auth:whoami
87
+ # email@example.com
88
+ #
89
+ def whoami
90
+ validate_arguments!
91
+
92
+ display Mortar::Auth.user
93
+ end
94
+
95
+ end
96
+
@@ -0,0 +1,319 @@
1
+ #
2
+ # Copyright 2012 Mortar Data Inc.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+ # Portions of this code from heroku (https://github.com/heroku/heroku/) Copyright Heroku 2008 - 2012,
17
+ # used under an MIT license (https://github.com/heroku/heroku/blob/master/LICENSE).
18
+ #
19
+
20
+ require "fileutils"
21
+ require "mortar/auth"
22
+ require "mortar/command"
23
+ require "mortar/project"
24
+ require "mortar/git"
25
+
26
+ class Mortar::Command::Base
27
+ include Mortar::Helpers
28
+
29
+ def self.namespace
30
+ self.to_s.split("::").last.downcase
31
+ end
32
+
33
+ attr_reader :args
34
+ attr_reader :options
35
+
36
+ def initialize(args=[], options={})
37
+ @args = args
38
+ @options = options
39
+ end
40
+
41
+ def project
42
+ unless @project
43
+ project_name, project_dir, remote =
44
+ if project_from_dir = extract_project_in_dir()
45
+ [project_from_dir[0], Dir.pwd, project_from_dir[1]]
46
+ elsif project_from_dir = extract_project_in_dir_no_git()
47
+ [project_from_dir[0], Dir.pwd, project_from_dir[1]]
48
+ else
49
+ raise Mortar::Command::CommandFailed, "No project found.\nThis command must be run from within a project folder."
50
+ end
51
+
52
+ # if we only have a project name, look for the remote in the current dir
53
+ unless remote
54
+ if project_from_dir = extract_project_in_dir(project_name)
55
+ project_dir = Dir.pwd
56
+ remote = project_from_dir[1]
57
+ end
58
+ end
59
+
60
+ @project = Mortar::Project::Project.new(project_name, project_dir, remote)
61
+ end
62
+ @project
63
+ end
64
+
65
+ def api
66
+ Mortar::Auth.api
67
+ end
68
+
69
+ def git
70
+ @git ||= Mortar::Git::Git.new
71
+ end
72
+
73
+ def pig_parameters
74
+ paramfile_params = {}
75
+ if options[:param_file]
76
+ File.open(options[:param_file], "r").each do |line|
77
+ # If the line isn't empty
78
+ if not line.chomp.empty? and not line.chomp.match(/^;/)
79
+ name, value = line.split('=', 2)
80
+ if not name or not value
81
+ error("Parameter file is malformed")
82
+ end
83
+ paramfile_params[name] = value
84
+ end
85
+ end
86
+ end
87
+
88
+
89
+ paramoption_params = {}
90
+ input_parameters = options[:parameter] ? Array(options[:parameter]) : []
91
+ input_parameters.each do |name_equals_value|
92
+ name, value = name_equals_value.split('=', 2)
93
+ paramoption_params[name] = value
94
+ end
95
+
96
+ parameters = []
97
+ paramfile_params.merge(paramoption_params).each do |name, value|
98
+ parameters << {"name" => name, "value" => value}
99
+ end
100
+
101
+ return parameters
102
+ end
103
+
104
+ def get_error_message_context(message)
105
+ if message.start_with? "Undefined parameter"
106
+ return "Use -p, --parameter NAME=VALUE to set parameter NAME to value VALUE."
107
+ end
108
+ return ""
109
+ end
110
+
111
+ protected
112
+
113
+ def self.inherited(klass)
114
+ unless klass == Mortar::Command::Base
115
+ help = extract_help_from_caller(caller.first)
116
+
117
+ Mortar::Command.register_namespace(
118
+ :name => klass.namespace,
119
+ :description => help.first
120
+ )
121
+ end
122
+ end
123
+
124
+ def self.method_added(method)
125
+ return if self == Mortar::Command::Base
126
+ return if private_method_defined?(method)
127
+ return if protected_method_defined?(method)
128
+
129
+ help = extract_help_from_caller(caller.first)
130
+ resolved_method = (method.to_s == "index") ? nil : method.to_s
131
+ command = [ self.namespace, resolved_method ].compact.join(":")
132
+ banner = extract_banner(help) || command
133
+
134
+ Mortar::Command.register_command(
135
+ :klass => self,
136
+ :method => method,
137
+ :namespace => self.namespace,
138
+ :command => command,
139
+ :banner => banner.strip,
140
+ :help => help.join("\n"),
141
+ :summary => extract_summary(help),
142
+ :description => extract_description(help),
143
+ :options => extract_options(help)
144
+ )
145
+ end
146
+
147
+ def self.alias_command(new, old)
148
+ raise "no such command: #{old}" unless Mortar::Command.commands[old]
149
+ Mortar::Command.command_aliases[new] = old
150
+ end
151
+
152
+ #
153
+ # Parse the caller format and identify the file and line number as identified
154
+ # in : http://www.ruby-doc.org/core/classes/Kernel.html#M001397. This will
155
+ # look for a colon followed by a digit as the delimiter. The biggest
156
+ # complication is windows paths, which have a color after the drive letter.
157
+ # This regex will match paths as anything from the beginning to a colon
158
+ # directly followed by a number (the line number).
159
+ #
160
+ # Examples of the caller format :
161
+ # * c:/Ruby192/lib/.../lib/mortar/command/addons.rb:8:in `<module:Command>'
162
+ # * c:/Ruby192/lib/.../mortar-2.0.1/lib/mortar/command/pg.rb:96:in `<class:Pg>'
163
+ # * /Users/ph7/...../xray-1.1/lib/xray/thread_dump_signal_handler.rb:9
164
+ #
165
+ def self.extract_help_from_caller(line)
166
+ # pull out of the caller the information for the file path and line number
167
+ if line =~ /^(.+?):(\d+)/
168
+ extract_help($1, $2)
169
+ else
170
+ raise("unable to extract help from caller: #{line}")
171
+ end
172
+ end
173
+
174
+ def self.extract_help(file, line_number)
175
+ buffer = []
176
+ lines = Mortar::Command.files[file]
177
+
178
+ (line_number.to_i-2).downto(0) do |i|
179
+ line = lines[i]
180
+ case line[0..0]
181
+ when ""
182
+ when "#"
183
+ buffer.unshift(line[1..-1])
184
+ else
185
+ break
186
+ end
187
+ end
188
+
189
+ buffer
190
+ end
191
+
192
+ def self.extract_banner(help)
193
+ help.first
194
+ end
195
+
196
+ def self.extract_summary(help)
197
+ extract_description(help).split("\n")[2].to_s.split("\n").first
198
+ end
199
+
200
+ def self.extract_description(help)
201
+ help.reject do |line|
202
+ line =~ /^\s+-(.+)#(.+)/
203
+ end.join("\n")
204
+ end
205
+
206
+ def self.extract_options(help)
207
+ help.select do |line|
208
+ line =~ /^\s+-(.+)#(.+)/
209
+ end.inject({}) do |hash, line|
210
+ description = line.split("#", 2).last
211
+ long = line.match(/--([0-9A-Za-z\- ]+)/)[1].strip
212
+ short = line.match(/-([0-9A-Za-z ])[ ,]/) && $1 && $1.strip
213
+ hash.update(long.split(" ").first => { :desc => description, :short => short, :long => long })
214
+ end
215
+ end
216
+
217
+ def current_command
218
+ Mortar::Command.current_command
219
+ end
220
+
221
+ def extract_option(key)
222
+ options[key.dup.gsub('-','').to_sym]
223
+ end
224
+
225
+ def invalid_arguments
226
+ Mortar::Command.invalid_arguments
227
+ end
228
+
229
+ def shift_argument
230
+ Mortar::Command.shift_argument
231
+ end
232
+
233
+ def validate_arguments!
234
+ Mortar::Command.validate_arguments!
235
+ end
236
+
237
+ def validate_git_based_project!
238
+ unless project.root_path
239
+ error("#{current_command[:command]} must be run from the checked-out project directory")
240
+ end
241
+
242
+ unless project.remote
243
+ error("Unable to find git remote for project #{project.name}")
244
+ end
245
+ end
246
+
247
+ def validate_pigscript!(pigscript_name)
248
+ unless pigscript = project.pigscripts[pigscript_name]
249
+ available_scripts = project.pigscripts.none? ? "No pigscripts found" : "Available scripts:\n#{project.pigscripts.keys.sort.join("\n")}"
250
+ error("Unable to find pigscript #{pigscript_name}\n#{available_scripts}")
251
+ end
252
+ pigscript
253
+ end
254
+
255
+ def extract_project_in_dir_no_git()
256
+ current_dirs = Dir.glob("*/")
257
+ missing_dir = Mortar::Project::Project.required_directories.find do |required_dir|
258
+ ! current_dirs.include?("#{required_dir}/")
259
+ end
260
+
261
+ return missing_dir ? nil : [File.basename(Dir.getwd), nil]
262
+ end
263
+
264
+ def extract_project_in_dir(project_name=nil)
265
+ # returns [project_name, remote_name]
266
+ # TODO refactor this very messy method
267
+ # when we have a more full sense of which options are supported when
268
+ return unless git.has_dot_git?
269
+
270
+ remotes = git.remotes(git_organization)
271
+ return if remotes.empty?
272
+
273
+ if remote = options[:remote]
274
+ # extract the project whose remote was provided
275
+ [remotes[remote], remote]
276
+ elsif remote = extract_project_from_git_config
277
+ # extract the project setup in git config
278
+ [remotes[remote], remote]
279
+ else
280
+ if project_name
281
+ # search for project by name
282
+ if project_remote = remotes.find {|r_name, p_name| p_name == project_name}
283
+ [project_name, project_remote.first[0]]
284
+ else
285
+ [project_name, nil]
286
+ end
287
+ elsif remotes.values.uniq.size == 1
288
+ # take the only project in the remotes
289
+ [remotes.first[1], remotes.first[0]]
290
+ else
291
+ raise(Mortar::Command::CommandFailed, "Multiple projects in folder and no project specified.\nSpecify which project to use with --project <project name>")
292
+ end
293
+ end
294
+ end
295
+
296
+ def extract_project_from_git_config
297
+ remote = git.git("config mortar.remote", false)
298
+ remote == "" ? nil : remote
299
+ end
300
+
301
+ def git_organization
302
+ ENV['MORTAR_ORGANIZATION'] || default_git_organization
303
+ end
304
+
305
+ def default_git_organization
306
+ "mortarcode"
307
+ end
308
+
309
+ def polling_interval
310
+ (options[:polling_interval] || 2.0).to_f
311
+ end
312
+
313
+ end
314
+
315
+ module Mortar::Command
316
+ unless const_defined?(:BaseWithApp)
317
+ BaseWithApp = Base
318
+ end
319
+ end
@@ -0,0 +1,41 @@
1
+ #
2
+ # Copyright 2012 Mortar Data Inc.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require "mortar/command/base"
18
+
19
+ ## manage clusters
20
+ #
21
+ class Mortar::Command::Clusters < Mortar::Command::Base
22
+
23
+ # clusters
24
+ #
25
+ # Display running and recently terminated clusters.
26
+ #
27
+ #Examples:
28
+ #
29
+ # $ mortar clusters
30
+ #
31
+ #TBD
32
+ #
33
+ def index
34
+ validate_arguments!
35
+
36
+ clusters = api.get_clusters().body['clusters']
37
+ display_table(clusters,
38
+ %w( cluster_id size status_description cluster_type_description start_timestamp duration),
39
+ ['cluster_id', 'Size (# of Nodes)', 'Status', 'Type', 'Start Timestamp', 'Elapsed Time'])
40
+ end
41
+ end