mortar 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (61) hide show
  1. data/README.md +36 -0
  2. data/bin/mortar +13 -0
  3. data/lib/mortar.rb +23 -0
  4. data/lib/mortar/auth.rb +312 -0
  5. data/lib/mortar/cli.rb +54 -0
  6. data/lib/mortar/command.rb +267 -0
  7. data/lib/mortar/command/auth.rb +96 -0
  8. data/lib/mortar/command/base.rb +319 -0
  9. data/lib/mortar/command/clusters.rb +41 -0
  10. data/lib/mortar/command/describe.rb +97 -0
  11. data/lib/mortar/command/generate.rb +121 -0
  12. data/lib/mortar/command/help.rb +166 -0
  13. data/lib/mortar/command/illustrate.rb +97 -0
  14. data/lib/mortar/command/jobs.rb +174 -0
  15. data/lib/mortar/command/pigscripts.rb +45 -0
  16. data/lib/mortar/command/projects.rb +128 -0
  17. data/lib/mortar/command/validate.rb +94 -0
  18. data/lib/mortar/command/version.rb +42 -0
  19. data/lib/mortar/errors.rb +24 -0
  20. data/lib/mortar/generators/generator_base.rb +107 -0
  21. data/lib/mortar/generators/macro_generator.rb +37 -0
  22. data/lib/mortar/generators/pigscript_generator.rb +40 -0
  23. data/lib/mortar/generators/project_generator.rb +67 -0
  24. data/lib/mortar/generators/udf_generator.rb +28 -0
  25. data/lib/mortar/git.rb +233 -0
  26. data/lib/mortar/helpers.rb +488 -0
  27. data/lib/mortar/project.rb +156 -0
  28. data/lib/mortar/snapshot.rb +39 -0
  29. data/lib/mortar/templates/macro/macro.pig +14 -0
  30. data/lib/mortar/templates/pigscript/pigscript.pig +38 -0
  31. data/lib/mortar/templates/pigscript/python_udf.py +13 -0
  32. data/lib/mortar/templates/project/Gemfile +3 -0
  33. data/lib/mortar/templates/project/README.md +8 -0
  34. data/lib/mortar/templates/project/gitignore +4 -0
  35. data/lib/mortar/templates/project/macros/gitkeep +0 -0
  36. data/lib/mortar/templates/project/pigscripts/pigscript.pig +35 -0
  37. data/lib/mortar/templates/project/udfs/python/python_udf.py +13 -0
  38. data/lib/mortar/templates/udf/python_udf.py +13 -0
  39. data/lib/mortar/version.rb +20 -0
  40. data/lib/vendor/mortar/okjson.rb +598 -0
  41. data/lib/vendor/mortar/uuid.rb +312 -0
  42. data/spec/mortar/auth_spec.rb +156 -0
  43. data/spec/mortar/command/auth_spec.rb +46 -0
  44. data/spec/mortar/command/base_spec.rb +82 -0
  45. data/spec/mortar/command/clusters_spec.rb +61 -0
  46. data/spec/mortar/command/describe_spec.rb +135 -0
  47. data/spec/mortar/command/generate_spec.rb +139 -0
  48. data/spec/mortar/command/illustrate_spec.rb +140 -0
  49. data/spec/mortar/command/jobs_spec.rb +364 -0
  50. data/spec/mortar/command/pigscripts_spec.rb +70 -0
  51. data/spec/mortar/command/projects_spec.rb +165 -0
  52. data/spec/mortar/command/validate_spec.rb +119 -0
  53. data/spec/mortar/command_spec.rb +122 -0
  54. data/spec/mortar/git_spec.rb +278 -0
  55. data/spec/mortar/helpers_spec.rb +82 -0
  56. data/spec/mortar/project_spec.rb +76 -0
  57. data/spec/mortar/snapshot_spec.rb +46 -0
  58. data/spec/spec.opts +1 -0
  59. data/spec/spec_helper.rb +278 -0
  60. data/spec/support/display_message_matcher.rb +68 -0
  61. metadata +259 -0
@@ -0,0 +1,312 @@
1
+ # Copyright(c) 2005 URABE, Shyouhei.
2
+ #
3
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
4
+ # of this code, to deal in the code without restriction, including without
5
+ # limitation the rights to use, copy, modify, merge, publish, distribute,
6
+ # sublicense, and/or sell copies of the code, and to permit persons to whom the
7
+ # code is furnished to do so, subject to the following conditions:
8
+ #
9
+ # The above copyright notice and this permission notice shall be
10
+ # included in all copies or substantial portions of the code.
11
+ #
12
+ # THE CODE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
13
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
14
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
15
+ # AUTHOR OR COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
16
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
17
+ # OUT OF OR IN CONNECTION WITH THE CODE OR THE USE OR OTHER DEALINGS IN THE
18
+ # CODE.
19
+
20
+ %w[
21
+ digest/md5
22
+ digest/sha1
23
+ tmpdir
24
+ ].each do |f|
25
+ require f
26
+ end
27
+
28
+ # Pure ruby UUID generator, which is compatible with RFC4122
29
+ module Mortar
30
+ class UUID
31
+ # UUID epoch is 15th Oct. 1582
32
+ UNIXEpoch = 0x01B21DD213814000 # in 100-nanoseconds resolution
33
+
34
+ private_class_method :new
35
+
36
+ private
37
+ def initialize str
38
+ tmp = str.unpack "C*"
39
+ @num = tmp.inject do |r, i|
40
+ r * 256 | i
41
+ end
42
+ @num.freeze
43
+ self.freeze
44
+ end
45
+
46
+ public
47
+
48
+ def raw_bytes
49
+ ret = String.new
50
+ tmp = @num
51
+ 16.times do |i|
52
+ x, y = tmp.divmod 256
53
+ ret << y
54
+ tmp = x
55
+ end
56
+ ret.reverse!
57
+ ret
58
+ end
59
+
60
+ class << self
61
+ def mask ver, str # :nodoc:
62
+ ver = ver & 15
63
+ v = str[6].ord
64
+ v &= 0b0000_1111
65
+ v |= ver << 4
66
+ str[6] = v.chr
67
+ r = str[8].ord
68
+ r &= 0b0011_1111
69
+ r |= 0b1000_0000
70
+ str[8] = r.chr
71
+ str
72
+ end
73
+
74
+ def prand # :nodoc:
75
+ rand 0x100000000
76
+ end
77
+
78
+ private :mask, :prand
79
+
80
+ # UUID generation using SHA1. Recommended over create_md5.
81
+ # Namespace object is another UUID, some of them are pre-defined below.
82
+ def create_sha1 str, namespace
83
+ sha1 = Digest::SHA1.new
84
+ sha1.update namespace.raw_bytes
85
+ sha1.update str
86
+ sum = sha1.digest
87
+ raw = mask 5, sum[0..15]
88
+ new raw
89
+ end
90
+
91
+ # UUID generation using MD5 (for backward compat.)
92
+ def create_md5 str, namespace
93
+ md5 = Digest::MD5.new
94
+ md5.update namespace.raw_bytes
95
+ md5.update str
96
+ sum = md5.digest
97
+ raw = mask 3, sum[0..16]
98
+ new raw
99
+ end
100
+
101
+ # UUID generation using random-number generator. From it's random
102
+ # nature, there's no warranty that the created ID is really universaly
103
+ # unique.
104
+ def create_random
105
+ rnd = [prand, prand, prand, prand].pack "N4"
106
+ raw = mask 4, rnd
107
+ new raw
108
+ end
109
+
110
+ def read_state fp # :nodoc:
111
+ fp.rewind
112
+ Marshal.load fp.read
113
+ end
114
+
115
+ def write_state fp, c, m # :nodoc:
116
+ fp.rewind
117
+ str = Marshal.dump [c, m]
118
+ fp.write str
119
+ end
120
+
121
+ private :read_state, :write_state
122
+ STATE_FILE = 'ruby-uuid'
123
+
124
+ # create the "version 1" UUID with current system clock, current UTC
125
+ # timestamp, and the IEEE 802 address (so-called MAC address).
126
+ #
127
+ # Speed notice: it's slow. It writes some data into hard drive on every
128
+ # invokation. If you want to speed this up, try remounting tmpdir with a
129
+ # memory based filesystem (such as tmpfs). STILL slow? then no way but
130
+ # rewrite it with c :)
131
+ def create clock=nil, time=Time.now, mac_addr=nil
132
+ c = t = m = nil
133
+ Dir.chdir Dir.tmpdir do
134
+ unless FileTest.exist? STATE_FILE then
135
+ # Generate a pseudo MAC address because we have no pure-ruby way
136
+ # to know the MAC address of the NIC this system uses. Note
137
+ # that cheating with pseudo arresses here is completely legal:
138
+ # see Section 4.5 of RFC4122 for details.
139
+ sha1 = Digest::SHA1.new
140
+ 256.times do
141
+ r = [prand].pack "N"
142
+ sha1.update r
143
+ end
144
+ ary = sha1.digest.bytes.to_a
145
+ node = ary.last 6
146
+ node[0] |= 0x01 # multicast bit
147
+ node = node.pack "C*"
148
+ k = rand 0x40000
149
+ open STATE_FILE, 'w' do |fp|
150
+ fp.flock IO::LOCK_EX
151
+ write_state fp, k, node
152
+ fp.chmod 0o777 # must be world writable
153
+ end
154
+ end
155
+ open STATE_FILE, 'r+' do |fp|
156
+ fp.flock IO::LOCK_EX
157
+ c, m = read_state fp
158
+ c += 1 # important; increment here
159
+ write_state fp, c, m
160
+ end
161
+ end
162
+ c = clock & 0b11_1111_1111_1111 if clock
163
+ m = mac_addr if mac_addr
164
+ time = Time.at time if time.is_a? Float
165
+ case time
166
+ when Time
167
+ t = time.to_i * 10_000_000 + time.tv_usec * 10 + UNIXEpoch
168
+ when Integer
169
+ t = time + UNIXEpoch
170
+ else
171
+ raise TypeError, "cannot convert ``#{time}'' into Time."
172
+ end
173
+
174
+ tl = t & 0xFFFF_FFFF
175
+ tm = t >> 32
176
+ tm = tm & 0xFFFF
177
+ th = t >> 48
178
+ th = th & 0b0000_1111_1111_1111
179
+ th = th | 0b0001_0000_0000_0000
180
+ cl = c & 0b0000_0000_1111_1111
181
+ ch = c & 0b0011_1111_0000_0000
182
+ ch = ch >> 8
183
+ ch = ch | 0b1000_0000
184
+ pack tl, tm, th, ch, cl, m
185
+ end
186
+
187
+ # A simple GUID parser: just ignores unknown characters and convert
188
+ # hexadecimal dump into 16-octet object.
189
+ def parse obj
190
+ str = obj.to_s.sub %r/\Aurn:uuid:/, ''
191
+ str.gsub! %r/[^0-9A-Fa-f]/, ''
192
+ raw = [str[0..31]].pack 'H*'
193
+ new raw
194
+ end
195
+
196
+ # The 'primitive constructor' of this class
197
+ # Note UUID.pack(uuid.unpack) == uuid
198
+ def pack tl, tm, th, ch, cl, n
199
+ raw = [tl, tm, th, ch, cl, n].pack "NnnCCa6"
200
+ new raw
201
+ end
202
+ end
203
+
204
+ # The 'primitive deconstructor', or the dual to pack.
205
+ # Note UUID.pack(uuid.unpack) == uuid
206
+ def unpack
207
+ raw_bytes.unpack "NnnCCa6"
208
+ end
209
+
210
+ # The timestamp of this UUID.
211
+ # Throws RageError if that time exceeds UNIX time range
212
+ def time
213
+ a = unpack
214
+ tl = a[0]
215
+ tm = a[1]
216
+ th = a[2] & 0x0FFF
217
+ t = tl
218
+ t += tm << 32
219
+ t += th << 48
220
+ t -= UNIXEpoch
221
+ tv_sec = t / 10_000_000
222
+ t -= tv_sec * 10_000_000
223
+ tv_usec = t / 10
224
+ Time.at tv_sec, tv_usec
225
+ end
226
+
227
+ # The version of this UUID
228
+ def version
229
+ v = unpack[2] & 0b1111_0000_0000_0000
230
+ v >> 12
231
+ end
232
+
233
+ # The clock sequence of this UUID
234
+ def clock
235
+ a = unpack
236
+ ch = a[3] & 0b0001_1111
237
+ cl = a[4]
238
+ c = cl
239
+ c += ch << 8
240
+ c
241
+ end
242
+
243
+ # The IEEE 802 address in a hexadecimal format
244
+ def node
245
+ m = unpack[5].unpack 'C*'
246
+ '%02x%02x%02x%02x%02x%02x' % m
247
+ end
248
+ alias mac_address node
249
+ alias ieee802 node
250
+
251
+ # Generate the string representation (a.k.a GUID) of this UUID
252
+ def to_s
253
+ a = unpack
254
+ a[-1] = mac_address
255
+ "%08x-%04x-%04x-%02x%02x-%s" % a
256
+ end
257
+ alias guid to_s
258
+
259
+ # Convert into a RFC4122-comforming URN representation
260
+ def to_uri
261
+ "urn:uuid:" + self.to_s
262
+ end
263
+ alias urn to_uri
264
+ alias inspect to_uri
265
+
266
+ # Convert into 128-bit unsigned integer
267
+ # Typically a Bignum instance, but can be a Fixnum.
268
+ def to_int
269
+ @num
270
+ end
271
+ alias to_i to_int
272
+
273
+ # Two UUIDs are said to be equal if and only if their (byte-order
274
+ # canonicalized) integer representations are equivallent. Refer RFC4122 for
275
+ # details.
276
+ def == other
277
+ to_i == other.to_i
278
+ end
279
+ alias eql? ==
280
+
281
+ # Two identical UUIDs should have same hash
282
+ def hash
283
+ to_i
284
+ end
285
+
286
+ include Comparable
287
+ # UUIDs are comparable (don't know what benefits are there, though).
288
+ def <=> other
289
+ to_s <=> other.to_s
290
+ end
291
+
292
+ # Pre-defined UUID Namespaces described in RFC4122 Appendix C.
293
+ NameSpace_DNS = parse "6ba7b810-9dad-11d1-80b4-00c04fd430c8"
294
+ NameSpace_URL = parse "6ba7b811-9dad-11d1-80b4-00c04fd430c8"
295
+ NameSpace_OID = parse "6ba7b812-9dad-11d1-80b4-00c04fd430c8"
296
+ NameSpace_X500 = parse "6ba7b814-9dad-11d1-80b4-00c04fd430c8"
297
+
298
+ # The Nil UUID in RFC4122 Section 4.1.7
299
+ Nil = parse "00000000-0000-0000-0000-000000000000"
300
+ end
301
+ end
302
+
303
+ # Local Variables:
304
+ # mode: ruby
305
+ # coding: utf-8
306
+ # indent-tabs-mode: t
307
+ # tab-width: 3
308
+ # ruby-indent-level: 3
309
+ # fill-column: 79
310
+ # default-justification: full
311
+ # End:
312
+ # vi: ts=3 sw=3
@@ -0,0 +1,156 @@
1
+ #
2
+ # Copyright 2012 Mortar Data Inc.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+ # Portions of this code from heroku (https://github.com/heroku/heroku/) Copyright Heroku 2008 - 2012,
17
+ # used under an MIT license (https://github.com/heroku/heroku/blob/master/LICENSE).
18
+ #
19
+
20
+ require "spec_helper"
21
+ require "mortar/auth"
22
+ require "mortar/helpers"
23
+
24
+ module Mortar
25
+ describe Auth do
26
+ include Mortar::Helpers
27
+
28
+ before do
29
+ ENV['MORTAR_API_KEY'] = nil
30
+
31
+ @cli = Mortar::Auth
32
+ stub(@cli).display
33
+ stub(@cli).running_on_a_mac? {false}
34
+ @cli.credentials = nil
35
+
36
+ FakeFS.activate!
37
+
38
+ stub(my_mode_output = Object.new).mode {"0600".to_i(8)}
39
+ stub(FakeFS::File).stat {my_mode_output}
40
+ stub(FakeFS::FileUtils).chmod
41
+ stub(FakeFS::File).readlines do |path|
42
+ File.read(path).split("\n").map {|line| "#{line}\n"}
43
+ end
44
+
45
+ FileUtils.mkdir_p(@cli.netrc_path.split("/")[0..-2].join("/"))
46
+
47
+ File.open(@cli.netrc_path, "w") do |file|
48
+ file.puts("machine api.mortardata.com\n login user\n password pass\n")
49
+ end
50
+ end
51
+
52
+ after do
53
+ FileUtils.rm_rf(@cli.netrc_path)
54
+ FakeFS.deactivate!
55
+ end
56
+
57
+ it "asks for credentials when the file doesn't exist" do
58
+ stub(@cli).check
59
+ @cli.delete_credentials
60
+ mock(@cli).ask_for_credentials {["u", "p"]}
61
+ #@cli.should_receive(:check_for_associated_ssh_key)
62
+ @cli.user.should == 'u'
63
+ @cli.password.should == 'p'
64
+ end
65
+
66
+ it "writes credentials and uploads authkey when credentials are saved" do
67
+ stub(@cli).credentials
68
+ stub(@cli).check
69
+ stub(@cli).ask_for_credentials.returns("username", "apikey")
70
+ mock(@cli).write_credentials
71
+ #@cli.should_receive(:check_for_associated_ssh_key)
72
+ @cli.ask_for_and_save_credentials
73
+ end
74
+
75
+ it "save_credentials deletes the credentials when the upload authkey is unauthorized" do
76
+ stub(@cli).write_credentials
77
+ stub(@cli).retry_login? { false }
78
+ stub(@cli).ask_for_credentials.returns("username", "apikey")
79
+ stub(@cli).check { raise Mortar::API::Errors::Unauthorized.new("Login Failed", Excon::Response.new) }
80
+ mock.proxy(@cli).delete_credentials
81
+ lambda { @cli.ask_for_and_save_credentials }.should raise_error(SystemExit)
82
+ end
83
+
84
+ it "asks for login again when not authorized, for three times" do
85
+ stub(@cli).read_credentials
86
+ stub(@cli).write_credentials
87
+ stub(@cli).delete_credentials
88
+ #stub(@cli).ask_for_credentials.returns("username", "apikey")
89
+ stub(@cli).check { raise Mortar::API::Errors::Unauthorized.new("Login Failed", Excon::Response.new) }
90
+ mock(@cli).ask_for_credentials.times(3).returns("username", "apikey")
91
+ lambda { @cli.ask_for_and_save_credentials }.should raise_error(SystemExit)
92
+ end
93
+
94
+ it "writes the login information to the credentials file for the 'mortar login' command" do
95
+ stub(@cli).ask_for_credentials.returns(['one', 'two'])
96
+ stub(@cli).check
97
+ #@cli.should_receive(:check_for_associated_ssh_key)
98
+ @cli.reauthorize
99
+ Netrc.read(@cli.netrc_path)["api.#{@cli.host}"].should == (['one', 'two'])
100
+ end
101
+
102
+ it "prompts for github_username when user doesn't have one." do
103
+ user_id = "123456789"
104
+ new_github_username = "some_new_github_username"
105
+ task_id = "1a2b3c4d"
106
+ stub(@cli).polling_interval.returns(0.05)
107
+
108
+ mock(@cli).ask_for_credentials.returns("username", "apikey")
109
+ stub(@cli).write_credentials
110
+ mock(@cli.api).get_user() {Excon::Response.new(:body => {"user_id" => user_id, "user_email" => "foo@foo.com"})}
111
+ mock(@cli).ask_for_github_username.returns(new_github_username)
112
+
113
+ mock(@cli.api).update_user(user_id,{"user_github_username" => new_github_username}) {Excon::Response.new(:body => {"task_id" => task_id})}
114
+
115
+ mock(@cli.api).get_task(task_id).returns(Excon::Response.new(:body => {"task_id" => task_id, "status_code" => "QUEUED"})).ordered
116
+ mock(@cli.api).get_task(task_id).returns(Excon::Response.new(:body => {"task_id" => task_id, "status_code" => "PROGRESS"})).ordered
117
+ mock(@cli.api).get_task(task_id).returns(Excon::Response.new(:body => {"task_id" => task_id, "status_code" => "SUCCESS"})).ordered
118
+
119
+ @cli.ask_for_and_save_credentials
120
+ end
121
+
122
+ it "remove credentials when call to set github_username fails" do
123
+ user_id = "abcdef"
124
+ new_github_username = "some_new_github_username"
125
+ task_id = "1a2b3c4d5e"
126
+ stub(@cli).polling_interval.returns(0.05)
127
+ mock(@cli).retry_set_github_username?.returns(false)
128
+
129
+
130
+ mock(@cli).ask_for_credentials.returns("username", "apikey")
131
+ stub(@cli).write_credentials
132
+ mock(@cli.api).get_user() {Excon::Response.new(:body => {"user_id" => user_id, "user_email" => "foo@foo.com"})}
133
+ mock(@cli).ask_for_github_username.returns(new_github_username)
134
+
135
+ mock(@cli.api).update_user(user_id,{"user_github_username" => new_github_username}) {Excon::Response.new(:body => {"task_id" => task_id})}
136
+
137
+ mock(@cli.api).get_task(task_id).returns(Excon::Response.new(:body => {"task_id" => task_id, "status_code" => "QUEUED"})).ordered
138
+ mock(@cli.api).get_task(task_id).returns(Excon::Response.new(:body => {"task_id" => task_id, "status_code" => "PROGRESS"})).ordered
139
+ mock(@cli.api).get_task(task_id).returns(Excon::Response.new(:body => {"task_id" => task_id, "status_code" => "FAILURE"})).ordered
140
+
141
+ mock(@cli).delete_credentials
142
+
143
+ lambda { @cli.ask_for_and_save_credentials }.should raise_error(SystemExit)
144
+ end
145
+
146
+ it "try 3 times to get github username" do
147
+ user_id = "abcdefghijkl"
148
+
149
+ mock(@cli.api).get_user() {Excon::Response.new(:body => {"user_id" => user_id, "user_email" => "foo@foo.com"})}
150
+
151
+ stub(@cli).ask_for_and_save_github_username.times(3).returns { raise Mortar::CLI::Errors::InvalidGithubUsername.new }
152
+
153
+ lambda { @cli.check }.should raise_error(Mortar::CLI::Errors::InvalidGithubUsername)
154
+ end
155
+ end
156
+ end