mortar 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. data/README.md +36 -0
  2. data/bin/mortar +13 -0
  3. data/lib/mortar.rb +23 -0
  4. data/lib/mortar/auth.rb +312 -0
  5. data/lib/mortar/cli.rb +54 -0
  6. data/lib/mortar/command.rb +267 -0
  7. data/lib/mortar/command/auth.rb +96 -0
  8. data/lib/mortar/command/base.rb +319 -0
  9. data/lib/mortar/command/clusters.rb +41 -0
  10. data/lib/mortar/command/describe.rb +97 -0
  11. data/lib/mortar/command/generate.rb +121 -0
  12. data/lib/mortar/command/help.rb +166 -0
  13. data/lib/mortar/command/illustrate.rb +97 -0
  14. data/lib/mortar/command/jobs.rb +174 -0
  15. data/lib/mortar/command/pigscripts.rb +45 -0
  16. data/lib/mortar/command/projects.rb +128 -0
  17. data/lib/mortar/command/validate.rb +94 -0
  18. data/lib/mortar/command/version.rb +42 -0
  19. data/lib/mortar/errors.rb +24 -0
  20. data/lib/mortar/generators/generator_base.rb +107 -0
  21. data/lib/mortar/generators/macro_generator.rb +37 -0
  22. data/lib/mortar/generators/pigscript_generator.rb +40 -0
  23. data/lib/mortar/generators/project_generator.rb +67 -0
  24. data/lib/mortar/generators/udf_generator.rb +28 -0
  25. data/lib/mortar/git.rb +233 -0
  26. data/lib/mortar/helpers.rb +488 -0
  27. data/lib/mortar/project.rb +156 -0
  28. data/lib/mortar/snapshot.rb +39 -0
  29. data/lib/mortar/templates/macro/macro.pig +14 -0
  30. data/lib/mortar/templates/pigscript/pigscript.pig +38 -0
  31. data/lib/mortar/templates/pigscript/python_udf.py +13 -0
  32. data/lib/mortar/templates/project/Gemfile +3 -0
  33. data/lib/mortar/templates/project/README.md +8 -0
  34. data/lib/mortar/templates/project/gitignore +4 -0
  35. data/lib/mortar/templates/project/macros/gitkeep +0 -0
  36. data/lib/mortar/templates/project/pigscripts/pigscript.pig +35 -0
  37. data/lib/mortar/templates/project/udfs/python/python_udf.py +13 -0
  38. data/lib/mortar/templates/udf/python_udf.py +13 -0
  39. data/lib/mortar/version.rb +20 -0
  40. data/lib/vendor/mortar/okjson.rb +598 -0
  41. data/lib/vendor/mortar/uuid.rb +312 -0
  42. data/spec/mortar/auth_spec.rb +156 -0
  43. data/spec/mortar/command/auth_spec.rb +46 -0
  44. data/spec/mortar/command/base_spec.rb +82 -0
  45. data/spec/mortar/command/clusters_spec.rb +61 -0
  46. data/spec/mortar/command/describe_spec.rb +135 -0
  47. data/spec/mortar/command/generate_spec.rb +139 -0
  48. data/spec/mortar/command/illustrate_spec.rb +140 -0
  49. data/spec/mortar/command/jobs_spec.rb +364 -0
  50. data/spec/mortar/command/pigscripts_spec.rb +70 -0
  51. data/spec/mortar/command/projects_spec.rb +165 -0
  52. data/spec/mortar/command/validate_spec.rb +119 -0
  53. data/spec/mortar/command_spec.rb +122 -0
  54. data/spec/mortar/git_spec.rb +278 -0
  55. data/spec/mortar/helpers_spec.rb +82 -0
  56. data/spec/mortar/project_spec.rb +76 -0
  57. data/spec/mortar/snapshot_spec.rb +46 -0
  58. data/spec/spec.opts +1 -0
  59. data/spec/spec_helper.rb +278 -0
  60. data/spec/support/display_message_matcher.rb +68 -0
  61. metadata +259 -0
@@ -0,0 +1,312 @@
1
+ # Copyright(c) 2005 URABE, Shyouhei.
2
+ #
3
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
4
+ # of this code, to deal in the code without restriction, including without
5
+ # limitation the rights to use, copy, modify, merge, publish, distribute,
6
+ # sublicense, and/or sell copies of the code, and to permit persons to whom the
7
+ # code is furnished to do so, subject to the following conditions:
8
+ #
9
+ # The above copyright notice and this permission notice shall be
10
+ # included in all copies or substantial portions of the code.
11
+ #
12
+ # THE CODE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
13
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
14
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
15
+ # AUTHOR OR COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
16
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
17
+ # OUT OF OR IN CONNECTION WITH THE CODE OR THE USE OR OTHER DEALINGS IN THE
18
+ # CODE.
19
+
20
+ %w[
21
+ digest/md5
22
+ digest/sha1
23
+ tmpdir
24
+ ].each do |f|
25
+ require f
26
+ end
27
+
28
+ # Pure ruby UUID generator, which is compatible with RFC4122
29
+ module Mortar
30
+ class UUID
31
+ # UUID epoch is 15th Oct. 1582
32
+ UNIXEpoch = 0x01B21DD213814000 # in 100-nanoseconds resolution
33
+
34
+ private_class_method :new
35
+
36
+ private
37
+ def initialize str
38
+ tmp = str.unpack "C*"
39
+ @num = tmp.inject do |r, i|
40
+ r * 256 | i
41
+ end
42
+ @num.freeze
43
+ self.freeze
44
+ end
45
+
46
+ public
47
+
48
+ def raw_bytes
49
+ ret = String.new
50
+ tmp = @num
51
+ 16.times do |i|
52
+ x, y = tmp.divmod 256
53
+ ret << y
54
+ tmp = x
55
+ end
56
+ ret.reverse!
57
+ ret
58
+ end
59
+
60
+ class << self
61
+ def mask ver, str # :nodoc:
62
+ ver = ver & 15
63
+ v = str[6].ord
64
+ v &= 0b0000_1111
65
+ v |= ver << 4
66
+ str[6] = v.chr
67
+ r = str[8].ord
68
+ r &= 0b0011_1111
69
+ r |= 0b1000_0000
70
+ str[8] = r.chr
71
+ str
72
+ end
73
+
74
+ def prand # :nodoc:
75
+ rand 0x100000000
76
+ end
77
+
78
+ private :mask, :prand
79
+
80
+ # UUID generation using SHA1. Recommended over create_md5.
81
+ # Namespace object is another UUID, some of them are pre-defined below.
82
+ def create_sha1 str, namespace
83
+ sha1 = Digest::SHA1.new
84
+ sha1.update namespace.raw_bytes
85
+ sha1.update str
86
+ sum = sha1.digest
87
+ raw = mask 5, sum[0..15]
88
+ new raw
89
+ end
90
+
91
+ # UUID generation using MD5 (for backward compat.)
92
+ def create_md5 str, namespace
93
+ md5 = Digest::MD5.new
94
+ md5.update namespace.raw_bytes
95
+ md5.update str
96
+ sum = md5.digest
97
+ raw = mask 3, sum[0..16]
98
+ new raw
99
+ end
100
+
101
+ # UUID generation using random-number generator. From it's random
102
+ # nature, there's no warranty that the created ID is really universaly
103
+ # unique.
104
+ def create_random
105
+ rnd = [prand, prand, prand, prand].pack "N4"
106
+ raw = mask 4, rnd
107
+ new raw
108
+ end
109
+
110
+ def read_state fp # :nodoc:
111
+ fp.rewind
112
+ Marshal.load fp.read
113
+ end
114
+
115
+ def write_state fp, c, m # :nodoc:
116
+ fp.rewind
117
+ str = Marshal.dump [c, m]
118
+ fp.write str
119
+ end
120
+
121
+ private :read_state, :write_state
122
+ STATE_FILE = 'ruby-uuid'
123
+
124
+ # create the "version 1" UUID with current system clock, current UTC
125
+ # timestamp, and the IEEE 802 address (so-called MAC address).
126
+ #
127
+ # Speed notice: it's slow. It writes some data into hard drive on every
128
+ # invokation. If you want to speed this up, try remounting tmpdir with a
129
+ # memory based filesystem (such as tmpfs). STILL slow? then no way but
130
+ # rewrite it with c :)
131
+ def create clock=nil, time=Time.now, mac_addr=nil
132
+ c = t = m = nil
133
+ Dir.chdir Dir.tmpdir do
134
+ unless FileTest.exist? STATE_FILE then
135
+ # Generate a pseudo MAC address because we have no pure-ruby way
136
+ # to know the MAC address of the NIC this system uses. Note
137
+ # that cheating with pseudo arresses here is completely legal:
138
+ # see Section 4.5 of RFC4122 for details.
139
+ sha1 = Digest::SHA1.new
140
+ 256.times do
141
+ r = [prand].pack "N"
142
+ sha1.update r
143
+ end
144
+ ary = sha1.digest.bytes.to_a
145
+ node = ary.last 6
146
+ node[0] |= 0x01 # multicast bit
147
+ node = node.pack "C*"
148
+ k = rand 0x40000
149
+ open STATE_FILE, 'w' do |fp|
150
+ fp.flock IO::LOCK_EX
151
+ write_state fp, k, node
152
+ fp.chmod 0o777 # must be world writable
153
+ end
154
+ end
155
+ open STATE_FILE, 'r+' do |fp|
156
+ fp.flock IO::LOCK_EX
157
+ c, m = read_state fp
158
+ c += 1 # important; increment here
159
+ write_state fp, c, m
160
+ end
161
+ end
162
+ c = clock & 0b11_1111_1111_1111 if clock
163
+ m = mac_addr if mac_addr
164
+ time = Time.at time if time.is_a? Float
165
+ case time
166
+ when Time
167
+ t = time.to_i * 10_000_000 + time.tv_usec * 10 + UNIXEpoch
168
+ when Integer
169
+ t = time + UNIXEpoch
170
+ else
171
+ raise TypeError, "cannot convert ``#{time}'' into Time."
172
+ end
173
+
174
+ tl = t & 0xFFFF_FFFF
175
+ tm = t >> 32
176
+ tm = tm & 0xFFFF
177
+ th = t >> 48
178
+ th = th & 0b0000_1111_1111_1111
179
+ th = th | 0b0001_0000_0000_0000
180
+ cl = c & 0b0000_0000_1111_1111
181
+ ch = c & 0b0011_1111_0000_0000
182
+ ch = ch >> 8
183
+ ch = ch | 0b1000_0000
184
+ pack tl, tm, th, ch, cl, m
185
+ end
186
+
187
+ # A simple GUID parser: just ignores unknown characters and convert
188
+ # hexadecimal dump into 16-octet object.
189
+ def parse obj
190
+ str = obj.to_s.sub %r/\Aurn:uuid:/, ''
191
+ str.gsub! %r/[^0-9A-Fa-f]/, ''
192
+ raw = [str[0..31]].pack 'H*'
193
+ new raw
194
+ end
195
+
196
+ # The 'primitive constructor' of this class
197
+ # Note UUID.pack(uuid.unpack) == uuid
198
+ def pack tl, tm, th, ch, cl, n
199
+ raw = [tl, tm, th, ch, cl, n].pack "NnnCCa6"
200
+ new raw
201
+ end
202
+ end
203
+
204
+ # The 'primitive deconstructor', or the dual to pack.
205
+ # Note UUID.pack(uuid.unpack) == uuid
206
+ def unpack
207
+ raw_bytes.unpack "NnnCCa6"
208
+ end
209
+
210
+ # The timestamp of this UUID.
211
+ # Throws RageError if that time exceeds UNIX time range
212
+ def time
213
+ a = unpack
214
+ tl = a[0]
215
+ tm = a[1]
216
+ th = a[2] & 0x0FFF
217
+ t = tl
218
+ t += tm << 32
219
+ t += th << 48
220
+ t -= UNIXEpoch
221
+ tv_sec = t / 10_000_000
222
+ t -= tv_sec * 10_000_000
223
+ tv_usec = t / 10
224
+ Time.at tv_sec, tv_usec
225
+ end
226
+
227
+ # The version of this UUID
228
+ def version
229
+ v = unpack[2] & 0b1111_0000_0000_0000
230
+ v >> 12
231
+ end
232
+
233
+ # The clock sequence of this UUID
234
+ def clock
235
+ a = unpack
236
+ ch = a[3] & 0b0001_1111
237
+ cl = a[4]
238
+ c = cl
239
+ c += ch << 8
240
+ c
241
+ end
242
+
243
+ # The IEEE 802 address in a hexadecimal format
244
+ def node
245
+ m = unpack[5].unpack 'C*'
246
+ '%02x%02x%02x%02x%02x%02x' % m
247
+ end
248
+ alias mac_address node
249
+ alias ieee802 node
250
+
251
+ # Generate the string representation (a.k.a GUID) of this UUID
252
+ def to_s
253
+ a = unpack
254
+ a[-1] = mac_address
255
+ "%08x-%04x-%04x-%02x%02x-%s" % a
256
+ end
257
+ alias guid to_s
258
+
259
+ # Convert into a RFC4122-comforming URN representation
260
+ def to_uri
261
+ "urn:uuid:" + self.to_s
262
+ end
263
+ alias urn to_uri
264
+ alias inspect to_uri
265
+
266
+ # Convert into 128-bit unsigned integer
267
+ # Typically a Bignum instance, but can be a Fixnum.
268
+ def to_int
269
+ @num
270
+ end
271
+ alias to_i to_int
272
+
273
+ # Two UUIDs are said to be equal if and only if their (byte-order
274
+ # canonicalized) integer representations are equivallent. Refer RFC4122 for
275
+ # details.
276
+ def == other
277
+ to_i == other.to_i
278
+ end
279
+ alias eql? ==
280
+
281
+ # Two identical UUIDs should have same hash
282
+ def hash
283
+ to_i
284
+ end
285
+
286
+ include Comparable
287
+ # UUIDs are comparable (don't know what benefits are there, though).
288
+ def <=> other
289
+ to_s <=> other.to_s
290
+ end
291
+
292
+ # Pre-defined UUID Namespaces described in RFC4122 Appendix C.
293
+ NameSpace_DNS = parse "6ba7b810-9dad-11d1-80b4-00c04fd430c8"
294
+ NameSpace_URL = parse "6ba7b811-9dad-11d1-80b4-00c04fd430c8"
295
+ NameSpace_OID = parse "6ba7b812-9dad-11d1-80b4-00c04fd430c8"
296
+ NameSpace_X500 = parse "6ba7b814-9dad-11d1-80b4-00c04fd430c8"
297
+
298
+ # The Nil UUID in RFC4122 Section 4.1.7
299
+ Nil = parse "00000000-0000-0000-0000-000000000000"
300
+ end
301
+ end
302
+
303
+ # Local Variables:
304
+ # mode: ruby
305
+ # coding: utf-8
306
+ # indent-tabs-mode: t
307
+ # tab-width: 3
308
+ # ruby-indent-level: 3
309
+ # fill-column: 79
310
+ # default-justification: full
311
+ # End:
312
+ # vi: ts=3 sw=3
@@ -0,0 +1,156 @@
1
+ #
2
+ # Copyright 2012 Mortar Data Inc.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+ # Portions of this code from heroku (https://github.com/heroku/heroku/) Copyright Heroku 2008 - 2012,
17
+ # used under an MIT license (https://github.com/heroku/heroku/blob/master/LICENSE).
18
+ #
19
+
20
+ require "spec_helper"
21
+ require "mortar/auth"
22
+ require "mortar/helpers"
23
+
24
+ module Mortar
25
+ describe Auth do
26
+ include Mortar::Helpers
27
+
28
+ before do
29
+ ENV['MORTAR_API_KEY'] = nil
30
+
31
+ @cli = Mortar::Auth
32
+ stub(@cli).display
33
+ stub(@cli).running_on_a_mac? {false}
34
+ @cli.credentials = nil
35
+
36
+ FakeFS.activate!
37
+
38
+ stub(my_mode_output = Object.new).mode {"0600".to_i(8)}
39
+ stub(FakeFS::File).stat {my_mode_output}
40
+ stub(FakeFS::FileUtils).chmod
41
+ stub(FakeFS::File).readlines do |path|
42
+ File.read(path).split("\n").map {|line| "#{line}\n"}
43
+ end
44
+
45
+ FileUtils.mkdir_p(@cli.netrc_path.split("/")[0..-2].join("/"))
46
+
47
+ File.open(@cli.netrc_path, "w") do |file|
48
+ file.puts("machine api.mortardata.com\n login user\n password pass\n")
49
+ end
50
+ end
51
+
52
+ after do
53
+ FileUtils.rm_rf(@cli.netrc_path)
54
+ FakeFS.deactivate!
55
+ end
56
+
57
+ it "asks for credentials when the file doesn't exist" do
58
+ stub(@cli).check
59
+ @cli.delete_credentials
60
+ mock(@cli).ask_for_credentials {["u", "p"]}
61
+ #@cli.should_receive(:check_for_associated_ssh_key)
62
+ @cli.user.should == 'u'
63
+ @cli.password.should == 'p'
64
+ end
65
+
66
+ it "writes credentials and uploads authkey when credentials are saved" do
67
+ stub(@cli).credentials
68
+ stub(@cli).check
69
+ stub(@cli).ask_for_credentials.returns("username", "apikey")
70
+ mock(@cli).write_credentials
71
+ #@cli.should_receive(:check_for_associated_ssh_key)
72
+ @cli.ask_for_and_save_credentials
73
+ end
74
+
75
+ it "save_credentials deletes the credentials when the upload authkey is unauthorized" do
76
+ stub(@cli).write_credentials
77
+ stub(@cli).retry_login? { false }
78
+ stub(@cli).ask_for_credentials.returns("username", "apikey")
79
+ stub(@cli).check { raise Mortar::API::Errors::Unauthorized.new("Login Failed", Excon::Response.new) }
80
+ mock.proxy(@cli).delete_credentials
81
+ lambda { @cli.ask_for_and_save_credentials }.should raise_error(SystemExit)
82
+ end
83
+
84
+ it "asks for login again when not authorized, for three times" do
85
+ stub(@cli).read_credentials
86
+ stub(@cli).write_credentials
87
+ stub(@cli).delete_credentials
88
+ #stub(@cli).ask_for_credentials.returns("username", "apikey")
89
+ stub(@cli).check { raise Mortar::API::Errors::Unauthorized.new("Login Failed", Excon::Response.new) }
90
+ mock(@cli).ask_for_credentials.times(3).returns("username", "apikey")
91
+ lambda { @cli.ask_for_and_save_credentials }.should raise_error(SystemExit)
92
+ end
93
+
94
+ it "writes the login information to the credentials file for the 'mortar login' command" do
95
+ stub(@cli).ask_for_credentials.returns(['one', 'two'])
96
+ stub(@cli).check
97
+ #@cli.should_receive(:check_for_associated_ssh_key)
98
+ @cli.reauthorize
99
+ Netrc.read(@cli.netrc_path)["api.#{@cli.host}"].should == (['one', 'two'])
100
+ end
101
+
102
+ it "prompts for github_username when user doesn't have one." do
103
+ user_id = "123456789"
104
+ new_github_username = "some_new_github_username"
105
+ task_id = "1a2b3c4d"
106
+ stub(@cli).polling_interval.returns(0.05)
107
+
108
+ mock(@cli).ask_for_credentials.returns("username", "apikey")
109
+ stub(@cli).write_credentials
110
+ mock(@cli.api).get_user() {Excon::Response.new(:body => {"user_id" => user_id, "user_email" => "foo@foo.com"})}
111
+ mock(@cli).ask_for_github_username.returns(new_github_username)
112
+
113
+ mock(@cli.api).update_user(user_id,{"user_github_username" => new_github_username}) {Excon::Response.new(:body => {"task_id" => task_id})}
114
+
115
+ mock(@cli.api).get_task(task_id).returns(Excon::Response.new(:body => {"task_id" => task_id, "status_code" => "QUEUED"})).ordered
116
+ mock(@cli.api).get_task(task_id).returns(Excon::Response.new(:body => {"task_id" => task_id, "status_code" => "PROGRESS"})).ordered
117
+ mock(@cli.api).get_task(task_id).returns(Excon::Response.new(:body => {"task_id" => task_id, "status_code" => "SUCCESS"})).ordered
118
+
119
+ @cli.ask_for_and_save_credentials
120
+ end
121
+
122
+ it "remove credentials when call to set github_username fails" do
123
+ user_id = "abcdef"
124
+ new_github_username = "some_new_github_username"
125
+ task_id = "1a2b3c4d5e"
126
+ stub(@cli).polling_interval.returns(0.05)
127
+ mock(@cli).retry_set_github_username?.returns(false)
128
+
129
+
130
+ mock(@cli).ask_for_credentials.returns("username", "apikey")
131
+ stub(@cli).write_credentials
132
+ mock(@cli.api).get_user() {Excon::Response.new(:body => {"user_id" => user_id, "user_email" => "foo@foo.com"})}
133
+ mock(@cli).ask_for_github_username.returns(new_github_username)
134
+
135
+ mock(@cli.api).update_user(user_id,{"user_github_username" => new_github_username}) {Excon::Response.new(:body => {"task_id" => task_id})}
136
+
137
+ mock(@cli.api).get_task(task_id).returns(Excon::Response.new(:body => {"task_id" => task_id, "status_code" => "QUEUED"})).ordered
138
+ mock(@cli.api).get_task(task_id).returns(Excon::Response.new(:body => {"task_id" => task_id, "status_code" => "PROGRESS"})).ordered
139
+ mock(@cli.api).get_task(task_id).returns(Excon::Response.new(:body => {"task_id" => task_id, "status_code" => "FAILURE"})).ordered
140
+
141
+ mock(@cli).delete_credentials
142
+
143
+ lambda { @cli.ask_for_and_save_credentials }.should raise_error(SystemExit)
144
+ end
145
+
146
+ it "try 3 times to get github username" do
147
+ user_id = "abcdefghijkl"
148
+
149
+ mock(@cli.api).get_user() {Excon::Response.new(:body => {"user_id" => user_id, "user_email" => "foo@foo.com"})}
150
+
151
+ stub(@cli).ask_for_and_save_github_username.times(3).returns { raise Mortar::CLI::Errors::InvalidGithubUsername.new }
152
+
153
+ lambda { @cli.check }.should raise_error(Mortar::CLI::Errors::InvalidGithubUsername)
154
+ end
155
+ end
156
+ end