mortar 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.md +36 -0
- data/bin/mortar +13 -0
- data/lib/mortar.rb +23 -0
- data/lib/mortar/auth.rb +312 -0
- data/lib/mortar/cli.rb +54 -0
- data/lib/mortar/command.rb +267 -0
- data/lib/mortar/command/auth.rb +96 -0
- data/lib/mortar/command/base.rb +319 -0
- data/lib/mortar/command/clusters.rb +41 -0
- data/lib/mortar/command/describe.rb +97 -0
- data/lib/mortar/command/generate.rb +121 -0
- data/lib/mortar/command/help.rb +166 -0
- data/lib/mortar/command/illustrate.rb +97 -0
- data/lib/mortar/command/jobs.rb +174 -0
- data/lib/mortar/command/pigscripts.rb +45 -0
- data/lib/mortar/command/projects.rb +128 -0
- data/lib/mortar/command/validate.rb +94 -0
- data/lib/mortar/command/version.rb +42 -0
- data/lib/mortar/errors.rb +24 -0
- data/lib/mortar/generators/generator_base.rb +107 -0
- data/lib/mortar/generators/macro_generator.rb +37 -0
- data/lib/mortar/generators/pigscript_generator.rb +40 -0
- data/lib/mortar/generators/project_generator.rb +67 -0
- data/lib/mortar/generators/udf_generator.rb +28 -0
- data/lib/mortar/git.rb +233 -0
- data/lib/mortar/helpers.rb +488 -0
- data/lib/mortar/project.rb +156 -0
- data/lib/mortar/snapshot.rb +39 -0
- data/lib/mortar/templates/macro/macro.pig +14 -0
- data/lib/mortar/templates/pigscript/pigscript.pig +38 -0
- data/lib/mortar/templates/pigscript/python_udf.py +13 -0
- data/lib/mortar/templates/project/Gemfile +3 -0
- data/lib/mortar/templates/project/README.md +8 -0
- data/lib/mortar/templates/project/gitignore +4 -0
- data/lib/mortar/templates/project/macros/gitkeep +0 -0
- data/lib/mortar/templates/project/pigscripts/pigscript.pig +35 -0
- data/lib/mortar/templates/project/udfs/python/python_udf.py +13 -0
- data/lib/mortar/templates/udf/python_udf.py +13 -0
- data/lib/mortar/version.rb +20 -0
- data/lib/vendor/mortar/okjson.rb +598 -0
- data/lib/vendor/mortar/uuid.rb +312 -0
- data/spec/mortar/auth_spec.rb +156 -0
- data/spec/mortar/command/auth_spec.rb +46 -0
- data/spec/mortar/command/base_spec.rb +82 -0
- data/spec/mortar/command/clusters_spec.rb +61 -0
- data/spec/mortar/command/describe_spec.rb +135 -0
- data/spec/mortar/command/generate_spec.rb +139 -0
- data/spec/mortar/command/illustrate_spec.rb +140 -0
- data/spec/mortar/command/jobs_spec.rb +364 -0
- data/spec/mortar/command/pigscripts_spec.rb +70 -0
- data/spec/mortar/command/projects_spec.rb +165 -0
- data/spec/mortar/command/validate_spec.rb +119 -0
- data/spec/mortar/command_spec.rb +122 -0
- data/spec/mortar/git_spec.rb +278 -0
- data/spec/mortar/helpers_spec.rb +82 -0
- data/spec/mortar/project_spec.rb +76 -0
- data/spec/mortar/snapshot_spec.rb +46 -0
- data/spec/spec.opts +1 -0
- data/spec/spec_helper.rb +278 -0
- data/spec/support/display_message_matcher.rb +68 -0
- metadata +259 -0
@@ -0,0 +1,96 @@
|
|
1
|
+
#
|
2
|
+
# Copyright 2012 Mortar Data Inc.
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
# Portions of this code from heroku (https://github.com/heroku/heroku/) Copyright Heroku 2008 - 2012,
|
17
|
+
# used under an MIT license (https://github.com/heroku/heroku/blob/master/LICENSE).
|
18
|
+
#
|
19
|
+
|
20
|
+
require "mortar/command/base"
|
21
|
+
|
22
|
+
# authentication (login, logout)
|
23
|
+
#
|
24
|
+
class Mortar::Command::Auth < Mortar::Command::Base
|
25
|
+
|
26
|
+
# auth:login
|
27
|
+
#
|
28
|
+
# log in with your mortar credentials
|
29
|
+
#
|
30
|
+
#Example:
|
31
|
+
#
|
32
|
+
# $ mortar auth:login
|
33
|
+
# Enter your Mortar credentials:
|
34
|
+
# Email: email@example.com
|
35
|
+
# Password (typing will be hidden):
|
36
|
+
# Authentication successful.
|
37
|
+
#
|
38
|
+
def login
|
39
|
+
validate_arguments!
|
40
|
+
|
41
|
+
Mortar::Auth.login
|
42
|
+
display "Authentication successful."
|
43
|
+
end
|
44
|
+
|
45
|
+
alias_command "login", "auth:login"
|
46
|
+
|
47
|
+
# auth:logout
|
48
|
+
#
|
49
|
+
# clear local authentication credentials
|
50
|
+
#
|
51
|
+
#Example:
|
52
|
+
#
|
53
|
+
# $ mortar auth:logout
|
54
|
+
# Local credentials cleared.
|
55
|
+
#
|
56
|
+
def logout
|
57
|
+
validate_arguments!
|
58
|
+
|
59
|
+
Mortar::Auth.logout
|
60
|
+
display "Local credentials cleared."
|
61
|
+
end
|
62
|
+
|
63
|
+
alias_command "logout", "auth:logout"
|
64
|
+
|
65
|
+
# auth:key
|
66
|
+
#
|
67
|
+
# display your api key
|
68
|
+
#
|
69
|
+
#Example:
|
70
|
+
#
|
71
|
+
# $ mortar auth:key
|
72
|
+
# ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789ABCD
|
73
|
+
#
|
74
|
+
def key
|
75
|
+
validate_arguments!
|
76
|
+
|
77
|
+
display Mortar::Auth.password
|
78
|
+
end
|
79
|
+
|
80
|
+
# auth:whoami
|
81
|
+
#
|
82
|
+
# display your mortar email address
|
83
|
+
#
|
84
|
+
#Example:
|
85
|
+
#
|
86
|
+
# $ mortar auth:whoami
|
87
|
+
# email@example.com
|
88
|
+
#
|
89
|
+
def whoami
|
90
|
+
validate_arguments!
|
91
|
+
|
92
|
+
display Mortar::Auth.user
|
93
|
+
end
|
94
|
+
|
95
|
+
end
|
96
|
+
|
@@ -0,0 +1,319 @@
|
|
1
|
+
#
|
2
|
+
# Copyright 2012 Mortar Data Inc.
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
# Portions of this code from heroku (https://github.com/heroku/heroku/) Copyright Heroku 2008 - 2012,
|
17
|
+
# used under an MIT license (https://github.com/heroku/heroku/blob/master/LICENSE).
|
18
|
+
#
|
19
|
+
|
20
|
+
require "fileutils"
|
21
|
+
require "mortar/auth"
|
22
|
+
require "mortar/command"
|
23
|
+
require "mortar/project"
|
24
|
+
require "mortar/git"
|
25
|
+
|
26
|
+
class Mortar::Command::Base
|
27
|
+
include Mortar::Helpers
|
28
|
+
|
29
|
+
def self.namespace
|
30
|
+
self.to_s.split("::").last.downcase
|
31
|
+
end
|
32
|
+
|
33
|
+
attr_reader :args
|
34
|
+
attr_reader :options
|
35
|
+
|
36
|
+
def initialize(args=[], options={})
|
37
|
+
@args = args
|
38
|
+
@options = options
|
39
|
+
end
|
40
|
+
|
41
|
+
def project
|
42
|
+
unless @project
|
43
|
+
project_name, project_dir, remote =
|
44
|
+
if project_from_dir = extract_project_in_dir()
|
45
|
+
[project_from_dir[0], Dir.pwd, project_from_dir[1]]
|
46
|
+
elsif project_from_dir = extract_project_in_dir_no_git()
|
47
|
+
[project_from_dir[0], Dir.pwd, project_from_dir[1]]
|
48
|
+
else
|
49
|
+
raise Mortar::Command::CommandFailed, "No project found.\nThis command must be run from within a project folder."
|
50
|
+
end
|
51
|
+
|
52
|
+
# if we only have a project name, look for the remote in the current dir
|
53
|
+
unless remote
|
54
|
+
if project_from_dir = extract_project_in_dir(project_name)
|
55
|
+
project_dir = Dir.pwd
|
56
|
+
remote = project_from_dir[1]
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
@project = Mortar::Project::Project.new(project_name, project_dir, remote)
|
61
|
+
end
|
62
|
+
@project
|
63
|
+
end
|
64
|
+
|
65
|
+
def api
|
66
|
+
Mortar::Auth.api
|
67
|
+
end
|
68
|
+
|
69
|
+
def git
|
70
|
+
@git ||= Mortar::Git::Git.new
|
71
|
+
end
|
72
|
+
|
73
|
+
def pig_parameters
|
74
|
+
paramfile_params = {}
|
75
|
+
if options[:param_file]
|
76
|
+
File.open(options[:param_file], "r").each do |line|
|
77
|
+
# If the line isn't empty
|
78
|
+
if not line.chomp.empty? and not line.chomp.match(/^;/)
|
79
|
+
name, value = line.split('=', 2)
|
80
|
+
if not name or not value
|
81
|
+
error("Parameter file is malformed")
|
82
|
+
end
|
83
|
+
paramfile_params[name] = value
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
|
89
|
+
paramoption_params = {}
|
90
|
+
input_parameters = options[:parameter] ? Array(options[:parameter]) : []
|
91
|
+
input_parameters.each do |name_equals_value|
|
92
|
+
name, value = name_equals_value.split('=', 2)
|
93
|
+
paramoption_params[name] = value
|
94
|
+
end
|
95
|
+
|
96
|
+
parameters = []
|
97
|
+
paramfile_params.merge(paramoption_params).each do |name, value|
|
98
|
+
parameters << {"name" => name, "value" => value}
|
99
|
+
end
|
100
|
+
|
101
|
+
return parameters
|
102
|
+
end
|
103
|
+
|
104
|
+
def get_error_message_context(message)
|
105
|
+
if message.start_with? "Undefined parameter"
|
106
|
+
return "Use -p, --parameter NAME=VALUE to set parameter NAME to value VALUE."
|
107
|
+
end
|
108
|
+
return ""
|
109
|
+
end
|
110
|
+
|
111
|
+
protected
|
112
|
+
|
113
|
+
def self.inherited(klass)
|
114
|
+
unless klass == Mortar::Command::Base
|
115
|
+
help = extract_help_from_caller(caller.first)
|
116
|
+
|
117
|
+
Mortar::Command.register_namespace(
|
118
|
+
:name => klass.namespace,
|
119
|
+
:description => help.first
|
120
|
+
)
|
121
|
+
end
|
122
|
+
end
|
123
|
+
|
124
|
+
def self.method_added(method)
|
125
|
+
return if self == Mortar::Command::Base
|
126
|
+
return if private_method_defined?(method)
|
127
|
+
return if protected_method_defined?(method)
|
128
|
+
|
129
|
+
help = extract_help_from_caller(caller.first)
|
130
|
+
resolved_method = (method.to_s == "index") ? nil : method.to_s
|
131
|
+
command = [ self.namespace, resolved_method ].compact.join(":")
|
132
|
+
banner = extract_banner(help) || command
|
133
|
+
|
134
|
+
Mortar::Command.register_command(
|
135
|
+
:klass => self,
|
136
|
+
:method => method,
|
137
|
+
:namespace => self.namespace,
|
138
|
+
:command => command,
|
139
|
+
:banner => banner.strip,
|
140
|
+
:help => help.join("\n"),
|
141
|
+
:summary => extract_summary(help),
|
142
|
+
:description => extract_description(help),
|
143
|
+
:options => extract_options(help)
|
144
|
+
)
|
145
|
+
end
|
146
|
+
|
147
|
+
def self.alias_command(new, old)
|
148
|
+
raise "no such command: #{old}" unless Mortar::Command.commands[old]
|
149
|
+
Mortar::Command.command_aliases[new] = old
|
150
|
+
end
|
151
|
+
|
152
|
+
#
|
153
|
+
# Parse the caller format and identify the file and line number as identified
|
154
|
+
# in : http://www.ruby-doc.org/core/classes/Kernel.html#M001397. This will
|
155
|
+
# look for a colon followed by a digit as the delimiter. The biggest
|
156
|
+
# complication is windows paths, which have a color after the drive letter.
|
157
|
+
# This regex will match paths as anything from the beginning to a colon
|
158
|
+
# directly followed by a number (the line number).
|
159
|
+
#
|
160
|
+
# Examples of the caller format :
|
161
|
+
# * c:/Ruby192/lib/.../lib/mortar/command/addons.rb:8:in `<module:Command>'
|
162
|
+
# * c:/Ruby192/lib/.../mortar-2.0.1/lib/mortar/command/pg.rb:96:in `<class:Pg>'
|
163
|
+
# * /Users/ph7/...../xray-1.1/lib/xray/thread_dump_signal_handler.rb:9
|
164
|
+
#
|
165
|
+
def self.extract_help_from_caller(line)
|
166
|
+
# pull out of the caller the information for the file path and line number
|
167
|
+
if line =~ /^(.+?):(\d+)/
|
168
|
+
extract_help($1, $2)
|
169
|
+
else
|
170
|
+
raise("unable to extract help from caller: #{line}")
|
171
|
+
end
|
172
|
+
end
|
173
|
+
|
174
|
+
def self.extract_help(file, line_number)
|
175
|
+
buffer = []
|
176
|
+
lines = Mortar::Command.files[file]
|
177
|
+
|
178
|
+
(line_number.to_i-2).downto(0) do |i|
|
179
|
+
line = lines[i]
|
180
|
+
case line[0..0]
|
181
|
+
when ""
|
182
|
+
when "#"
|
183
|
+
buffer.unshift(line[1..-1])
|
184
|
+
else
|
185
|
+
break
|
186
|
+
end
|
187
|
+
end
|
188
|
+
|
189
|
+
buffer
|
190
|
+
end
|
191
|
+
|
192
|
+
def self.extract_banner(help)
|
193
|
+
help.first
|
194
|
+
end
|
195
|
+
|
196
|
+
def self.extract_summary(help)
|
197
|
+
extract_description(help).split("\n")[2].to_s.split("\n").first
|
198
|
+
end
|
199
|
+
|
200
|
+
def self.extract_description(help)
|
201
|
+
help.reject do |line|
|
202
|
+
line =~ /^\s+-(.+)#(.+)/
|
203
|
+
end.join("\n")
|
204
|
+
end
|
205
|
+
|
206
|
+
def self.extract_options(help)
|
207
|
+
help.select do |line|
|
208
|
+
line =~ /^\s+-(.+)#(.+)/
|
209
|
+
end.inject({}) do |hash, line|
|
210
|
+
description = line.split("#", 2).last
|
211
|
+
long = line.match(/--([0-9A-Za-z\- ]+)/)[1].strip
|
212
|
+
short = line.match(/-([0-9A-Za-z ])[ ,]/) && $1 && $1.strip
|
213
|
+
hash.update(long.split(" ").first => { :desc => description, :short => short, :long => long })
|
214
|
+
end
|
215
|
+
end
|
216
|
+
|
217
|
+
def current_command
|
218
|
+
Mortar::Command.current_command
|
219
|
+
end
|
220
|
+
|
221
|
+
def extract_option(key)
|
222
|
+
options[key.dup.gsub('-','').to_sym]
|
223
|
+
end
|
224
|
+
|
225
|
+
def invalid_arguments
|
226
|
+
Mortar::Command.invalid_arguments
|
227
|
+
end
|
228
|
+
|
229
|
+
def shift_argument
|
230
|
+
Mortar::Command.shift_argument
|
231
|
+
end
|
232
|
+
|
233
|
+
def validate_arguments!
|
234
|
+
Mortar::Command.validate_arguments!
|
235
|
+
end
|
236
|
+
|
237
|
+
def validate_git_based_project!
|
238
|
+
unless project.root_path
|
239
|
+
error("#{current_command[:command]} must be run from the checked-out project directory")
|
240
|
+
end
|
241
|
+
|
242
|
+
unless project.remote
|
243
|
+
error("Unable to find git remote for project #{project.name}")
|
244
|
+
end
|
245
|
+
end
|
246
|
+
|
247
|
+
def validate_pigscript!(pigscript_name)
|
248
|
+
unless pigscript = project.pigscripts[pigscript_name]
|
249
|
+
available_scripts = project.pigscripts.none? ? "No pigscripts found" : "Available scripts:\n#{project.pigscripts.keys.sort.join("\n")}"
|
250
|
+
error("Unable to find pigscript #{pigscript_name}\n#{available_scripts}")
|
251
|
+
end
|
252
|
+
pigscript
|
253
|
+
end
|
254
|
+
|
255
|
+
def extract_project_in_dir_no_git()
|
256
|
+
current_dirs = Dir.glob("*/")
|
257
|
+
missing_dir = Mortar::Project::Project.required_directories.find do |required_dir|
|
258
|
+
! current_dirs.include?("#{required_dir}/")
|
259
|
+
end
|
260
|
+
|
261
|
+
return missing_dir ? nil : [File.basename(Dir.getwd), nil]
|
262
|
+
end
|
263
|
+
|
264
|
+
def extract_project_in_dir(project_name=nil)
|
265
|
+
# returns [project_name, remote_name]
|
266
|
+
# TODO refactor this very messy method
|
267
|
+
# when we have a more full sense of which options are supported when
|
268
|
+
return unless git.has_dot_git?
|
269
|
+
|
270
|
+
remotes = git.remotes(git_organization)
|
271
|
+
return if remotes.empty?
|
272
|
+
|
273
|
+
if remote = options[:remote]
|
274
|
+
# extract the project whose remote was provided
|
275
|
+
[remotes[remote], remote]
|
276
|
+
elsif remote = extract_project_from_git_config
|
277
|
+
# extract the project setup in git config
|
278
|
+
[remotes[remote], remote]
|
279
|
+
else
|
280
|
+
if project_name
|
281
|
+
# search for project by name
|
282
|
+
if project_remote = remotes.find {|r_name, p_name| p_name == project_name}
|
283
|
+
[project_name, project_remote.first[0]]
|
284
|
+
else
|
285
|
+
[project_name, nil]
|
286
|
+
end
|
287
|
+
elsif remotes.values.uniq.size == 1
|
288
|
+
# take the only project in the remotes
|
289
|
+
[remotes.first[1], remotes.first[0]]
|
290
|
+
else
|
291
|
+
raise(Mortar::Command::CommandFailed, "Multiple projects in folder and no project specified.\nSpecify which project to use with --project <project name>")
|
292
|
+
end
|
293
|
+
end
|
294
|
+
end
|
295
|
+
|
296
|
+
def extract_project_from_git_config
|
297
|
+
remote = git.git("config mortar.remote", false)
|
298
|
+
remote == "" ? nil : remote
|
299
|
+
end
|
300
|
+
|
301
|
+
def git_organization
|
302
|
+
ENV['MORTAR_ORGANIZATION'] || default_git_organization
|
303
|
+
end
|
304
|
+
|
305
|
+
def default_git_organization
|
306
|
+
"mortarcode"
|
307
|
+
end
|
308
|
+
|
309
|
+
def polling_interval
|
310
|
+
(options[:polling_interval] || 2.0).to_f
|
311
|
+
end
|
312
|
+
|
313
|
+
end
|
314
|
+
|
315
|
+
module Mortar::Command
|
316
|
+
unless const_defined?(:BaseWithApp)
|
317
|
+
BaseWithApp = Base
|
318
|
+
end
|
319
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
#
|
2
|
+
# Copyright 2012 Mortar Data Inc.
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require "mortar/command/base"
|
18
|
+
|
19
|
+
## manage clusters
|
20
|
+
#
|
21
|
+
class Mortar::Command::Clusters < Mortar::Command::Base
|
22
|
+
|
23
|
+
# clusters
|
24
|
+
#
|
25
|
+
# Display running and recently terminated clusters.
|
26
|
+
#
|
27
|
+
#Examples:
|
28
|
+
#
|
29
|
+
# $ mortar clusters
|
30
|
+
#
|
31
|
+
#TBD
|
32
|
+
#
|
33
|
+
def index
|
34
|
+
validate_arguments!
|
35
|
+
|
36
|
+
clusters = api.get_clusters().body['clusters']
|
37
|
+
display_table(clusters,
|
38
|
+
%w( cluster_id size status_description cluster_type_description start_timestamp duration),
|
39
|
+
['cluster_id', 'Size (# of Nodes)', 'Status', 'Type', 'Start Timestamp', 'Elapsed Time'])
|
40
|
+
end
|
41
|
+
end
|