mamiya 0.0.1.alpha2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +17 -0
  3. data/.rspec +2 -0
  4. data/.travis.yml +16 -0
  5. data/Gemfile +8 -0
  6. data/LICENSE.txt +22 -0
  7. data/README.md +43 -0
  8. data/Rakefile +6 -0
  9. data/bin/mamiya +17 -0
  10. data/config.example.yml +11 -0
  11. data/docs/sequences/deploy.png +0 -0
  12. data/docs/sequences/deploy.uml +58 -0
  13. data/example.rb +74 -0
  14. data/lib/mamiya.rb +5 -0
  15. data/lib/mamiya/agent.rb +181 -0
  16. data/lib/mamiya/agent/actions.rb +12 -0
  17. data/lib/mamiya/agent/fetcher.rb +137 -0
  18. data/lib/mamiya/agent/handlers/abstract.rb +20 -0
  19. data/lib/mamiya/agent/handlers/fetch.rb +68 -0
  20. data/lib/mamiya/cli.rb +322 -0
  21. data/lib/mamiya/cli/client.rb +172 -0
  22. data/lib/mamiya/config.rb +57 -0
  23. data/lib/mamiya/dsl.rb +192 -0
  24. data/lib/mamiya/helpers/git.rb +75 -0
  25. data/lib/mamiya/logger.rb +190 -0
  26. data/lib/mamiya/master.rb +118 -0
  27. data/lib/mamiya/master/agent_monitor.rb +146 -0
  28. data/lib/mamiya/master/agent_monitor_handlers.rb +44 -0
  29. data/lib/mamiya/master/web.rb +148 -0
  30. data/lib/mamiya/package.rb +122 -0
  31. data/lib/mamiya/script.rb +117 -0
  32. data/lib/mamiya/steps/abstract.rb +19 -0
  33. data/lib/mamiya/steps/build.rb +72 -0
  34. data/lib/mamiya/steps/extract.rb +26 -0
  35. data/lib/mamiya/steps/fetch.rb +24 -0
  36. data/lib/mamiya/steps/push.rb +34 -0
  37. data/lib/mamiya/storages.rb +17 -0
  38. data/lib/mamiya/storages/abstract.rb +48 -0
  39. data/lib/mamiya/storages/mock.rb +61 -0
  40. data/lib/mamiya/storages/s3.rb +127 -0
  41. data/lib/mamiya/util/label_matcher.rb +38 -0
  42. data/lib/mamiya/version.rb +3 -0
  43. data/mamiya.gemspec +35 -0
  44. data/misc/logger_test.rb +12 -0
  45. data/spec/agent/actions_spec.rb +37 -0
  46. data/spec/agent/fetcher_spec.rb +199 -0
  47. data/spec/agent/handlers/fetch_spec.rb +121 -0
  48. data/spec/agent_spec.rb +255 -0
  49. data/spec/config_spec.rb +50 -0
  50. data/spec/dsl_spec.rb +291 -0
  51. data/spec/fixtures/dsl_test_load.rb +1 -0
  52. data/spec/fixtures/dsl_test_use.rb +1 -0
  53. data/spec/fixtures/helpers/foo.rb +1 -0
  54. data/spec/fixtures/test-package-source/.mamiya.meta.json +1 -0
  55. data/spec/fixtures/test-package-source/greeting +1 -0
  56. data/spec/fixtures/test-package.tar.gz +0 -0
  57. data/spec/fixtures/test.yml +4 -0
  58. data/spec/logger_spec.rb +68 -0
  59. data/spec/master/agent_monitor_spec.rb +269 -0
  60. data/spec/master/web_spec.rb +121 -0
  61. data/spec/master_spec.rb +94 -0
  62. data/spec/package_spec.rb +394 -0
  63. data/spec/script_spec.rb +78 -0
  64. data/spec/spec_helper.rb +38 -0
  65. data/spec/steps/build_spec.rb +261 -0
  66. data/spec/steps/extract_spec.rb +68 -0
  67. data/spec/steps/fetch_spec.rb +96 -0
  68. data/spec/steps/push_spec.rb +73 -0
  69. data/spec/storages/abstract_spec.rb +22 -0
  70. data/spec/storages/s3_spec.rb +342 -0
  71. data/spec/storages_spec.rb +33 -0
  72. data/spec/support/dummy_serf.rb +70 -0
  73. data/spec/util/label_matcher_spec.rb +85 -0
  74. metadata +272 -0
@@ -0,0 +1,121 @@
1
+ require 'spec_helper'
2
+ require 'tmpdir'
3
+ require 'fileutils'
4
+
5
+ require 'mamiya/storages/mock'
6
+ require 'mamiya/package'
7
+
8
+ require 'mamiya/master/web'
9
+
10
+ describe Mamiya::Master::Web do
11
+ include Rack::Test::Methods
12
+
13
+ let!(:tmpdir) { Dir.mktmpdir('maimya-master-web-spec') }
14
+ after { FileUtils.remove_entry_secure(tmpdir) }
15
+
16
+ let(:app) { described_class }
17
+
18
+ let(:config_source) do
19
+ {}
20
+ end
21
+
22
+ let(:config) do
23
+ double('config').tap do |c|
24
+ allow(c).to receive(:[]) do |k|
25
+ config_source[k]
26
+ end
27
+ end
28
+ end
29
+
30
+ let(:master) do
31
+ double('master', config: config).tap do |m|
32
+ allow(m).to receive(:storage) do |app|
33
+ Mamiya::Storages::Mock.new(application: app)
34
+ end
35
+ end
36
+ end
37
+
38
+ let(:package) do
39
+ File.write File.join(tmpdir, 'mypackage.tar.gz'), "\n"
40
+ File.write File.join(tmpdir, 'mypackage.json'), "#{{meta: 'data'}.to_json}\n"
41
+ Mamiya::Package.new(File.join(tmpdir, 'mypackage'))
42
+ end
43
+
44
+ before do
45
+ described_class.set :environment, :test
46
+
47
+ current_session.envs["rack.logger"] = Mamiya::Logger.new
48
+ current_session.envs["mamiya.master"] = master
49
+
50
+ Mamiya::Storages::Mock.new(application: 'myapp').push(package)
51
+ end
52
+
53
+ describe "GET /" do
54
+ it "returns text" do
55
+ get '/'
56
+
57
+ expect(last_response.status).to eq 200
58
+ expect(last_response.body).to match(/^mamiya/)
59
+ end
60
+ end
61
+
62
+ describe "GET /packages/:application" do
63
+ it "returns package list" do
64
+ get '/packages/myapp'
65
+
66
+ expect(last_response.status).to eq 200
67
+ expect(last_response.content_type).to eq 'application/json'
68
+ json = JSON.parse(last_response.body)
69
+ expect(json['packages']).to eq ['mypackage']
70
+ end
71
+
72
+ it "returns empty list with 204 for inexistence app" do
73
+ get '/packages/noapp'
74
+
75
+ expect(last_response.status).to eq 404
76
+ expect(last_response.content_type).to eq 'application/json'
77
+ json = JSON.parse(last_response.body)
78
+ expect(json['packages']).to eq []
79
+ end
80
+ end
81
+
82
+ describe "GET /packages/:application/:package" do
83
+ it "returns package detail" do
84
+ get '/packages/myapp/mypackage'
85
+
86
+ expect(last_response.status).to eq 200
87
+ expect(last_response.content_type).to eq 'application/json'
88
+ json = JSON.parse(last_response.body)
89
+ expect(json['application']).to eq 'myapp'
90
+ expect(json['name']).to eq 'mypackage'
91
+ expect(json['meta']).to eq('meta' => 'data')
92
+ end
93
+
94
+ context "when not exists" do
95
+ it "returns 404" do
96
+ get '/packages/myapp/mypkg'
97
+
98
+ expect(last_response.status).to eq 404
99
+ expect(last_response.content_type).to eq 'application/json'
100
+ expect(JSON.parse(last_response.body)).to eq({})
101
+ end
102
+ end
103
+ end
104
+
105
+ describe "POST /packages/:application/:package/distribute" do
106
+ it "dispatchs distribute request" do
107
+ expect(master).to receive(:distribute).with('myapp', 'mypackage')
108
+
109
+ post '/packages/myapp/mypackage/distribute'
110
+
111
+ expect(last_response.status).to eq 204 # no content
112
+ end
113
+
114
+ context "when package not found" do
115
+ it "returns 404" do
116
+ post '/packages/myapp/noexist/distribute'
117
+ expect(last_response.status).to eq 404
118
+ end
119
+ end
120
+ end
121
+ end
@@ -0,0 +1,94 @@
1
+ require 'spec_helper'
2
+ require 'tmpdir'
3
+ require 'fileutils'
4
+ require 'json'
5
+ require 'villein/event'
6
+
7
+ require 'mamiya/agent'
8
+ require 'mamiya/storages/mock'
9
+
10
+ require 'mamiya/master/agent_monitor'
11
+ require 'mamiya/master'
12
+
13
+ require_relative './support/dummy_serf.rb'
14
+
15
+ describe Mamiya::Master do
16
+ let(:serf) { DummySerf.new }
17
+ let(:agent_monitor) { double('agent_monitor', start!: nil) }
18
+
19
+ let(:config) do
20
+ {
21
+ serf: {agent: {rpc_addr: '127.0.0.1:17373', bind: '127.0.0.1:17946'}},
22
+ web: {port: 0, bind: 'localhost'},
23
+ storage: {type: :mock},
24
+ }
25
+ end
26
+
27
+ before do
28
+ allow(Villein::Agent).to receive(:new).and_return(serf)
29
+ allow(Mamiya::Master::AgentMonitor).to receive(:new).and_return(agent_monitor)
30
+ end
31
+
32
+ subject(:master) { described_class.new(config) }
33
+
34
+ it "inherits Mamiya::Agent" do
35
+ expect(described_class.superclass).to eq Mamiya::Agent
36
+ end
37
+
38
+ describe "#distribute" do
39
+ # let!(:tmpdir) { Dir.mktmpdir('maimya-master-spec') }
40
+ # after { FileUtils.remove_entry_secure(tmpdir) }
41
+
42
+ # let(:package) do
43
+ # File.write File.join(tmpdir, 'mypackage.tar.gz'), "\n"
44
+ # File.write File.join(tmpdir, 'mypackage.json'), "#{{meta: 'data'}.to_json}\n"
45
+ # Mamiya::Package.new(File.join(tmpdir, 'mypackage'))
46
+ # end
47
+
48
+ # before do
49
+ # Mamiya::Storages::Mock.new(application: 'myapp').push(package)
50
+ # end
51
+
52
+ it "triggers fetch event" do
53
+ expect(master).to receive(:trigger).with(:fetch, application: 'myapp', package: 'mypackage')
54
+
55
+ master.distribute('myapp', 'mypackage')
56
+ end
57
+ end
58
+
59
+ it "starts agent monitor"
60
+
61
+ describe "(member join event)" do
62
+ it "initiates refresh" do
63
+ master # initiate
64
+
65
+ expect(agent_monitor).to receive(:refresh).with(node: ['the-node', 'another-node'])
66
+
67
+ serf.trigger("member_join", Villein::Event.new(
68
+ {"SERF_EVENT" => "member-join"},
69
+ payload: "the-node\tX.X.X.X\t\tkey=val,a=b\nanother-node\tY.Y.Y.Y\t\tkey=val,a=b\n"
70
+ ))
71
+ end
72
+ end
73
+
74
+ describe "#run!" do
75
+ it "starts serf and web" do
76
+ begin
77
+ flag = false
78
+
79
+ expect_any_instance_of(Rack::Server).to receive(:start)
80
+ expect(serf).to receive(:start!)
81
+ expect(serf).to receive(:auto_stop) do
82
+ flag = true
83
+ end
84
+
85
+ th = Thread.new { master.run! }
86
+ th.abort_on_exception = true
87
+
88
+ 10.times { break if flag; sleep 0.1 }
89
+ ensure
90
+ th.kill if th && th.alive?
91
+ end
92
+ end
93
+ end
94
+ end
@@ -0,0 +1,394 @@
1
+ require 'spec_helper'
2
+ require 'tmpdir'
3
+ require 'pathname'
4
+ require 'digest/sha2'
5
+ require 'json'
6
+
7
+ require 'mamiya/package'
8
+
9
+ describe Mamiya::Package do
10
+ let!(:tmpdir) { Dir.mktmpdir("mamiya-package-spec") }
11
+ after { FileUtils.remove_entry_secure tmpdir }
12
+
13
+ let(:build_dir) { Pathname.new(tmpdir).join('build') }
14
+ let(:extract_dir) { Pathname.new(tmpdir).join('extract') }
15
+ before do
16
+ build_dir.mkdir
17
+ extract_dir.mkdir
18
+ end
19
+
20
+ let(:package_path) { File.join(tmpdir, 'test.tar.gz') }
21
+ let(:meta_path) { package_path.sub(/tar\.gz$/, "json") }
22
+
23
+ let(:arg) { package_path }
24
+
25
+ subject(:package) {
26
+ Mamiya::Package.new(arg)
27
+ }
28
+
29
+ describe "#initialize" do
30
+ context "without file extension" do
31
+ it "accepts" do
32
+ expect { described_class.new('foo') }.not_to raise_error
33
+ end
34
+ end
35
+
36
+ context "with .tar.gz" do
37
+ it "accepts" do
38
+ expect { described_class.new('foo.tar.gz') }.not_to raise_error
39
+ end
40
+ end
41
+
42
+ context "with .json" do
43
+ it "accepts" do
44
+ expect { described_class.new('foo.json') }.not_to raise_error
45
+ end
46
+ end
47
+ end
48
+
49
+ describe "#path" do
50
+ subject { package.path }
51
+ it { should eq Pathname.new(package_path) }
52
+
53
+ context "without file extension" do
54
+ let(:arg) { 'test' }
55
+ it { should eq Pathname.new('test.tar.gz') }
56
+ end
57
+
58
+ context "with .tar.gz" do
59
+ let(:arg) { 'test.tar.gz' }
60
+ it { should eq Pathname.new('test.tar.gz') }
61
+ end
62
+
63
+ context "with .json" do
64
+ let(:arg) { 'test.json' }
65
+ it { should eq Pathname.new('test.tar.gz') }
66
+ end
67
+
68
+ context "with filename containing dot" do
69
+ let(:arg) { 'a.b' }
70
+ it { should eq Pathname.new('a.b.tar.gz') }
71
+ end
72
+ end
73
+
74
+ describe "#meta_path" do
75
+ subject { package.meta_path }
76
+
77
+ context "without file extension" do
78
+ let(:arg) { 'test' }
79
+ it { should eq Pathname.new('test.json') }
80
+ end
81
+
82
+ context "with .tar.gz" do
83
+ let(:arg) { 'test.tar.gz' }
84
+ it { should eq Pathname.new('test.json') }
85
+ end
86
+
87
+ context "with .json" do
88
+ let(:arg) { 'test.json' }
89
+ it { should eq Pathname.new('test.json') }
90
+ end
91
+
92
+ context "with filename containing dot" do
93
+ let(:arg) { 'a.b' }
94
+ it { should eq Pathname.new('a.b.json') }
95
+ end
96
+ end
97
+
98
+ describe "#extract_onto!(path)" do
99
+ subject { package.extract_onto!(extract_dir) }
100
+
101
+ context "with package" do
102
+ before do
103
+ File.write build_dir.join("greeting"), "hello\n"
104
+ Dir.chdir(build_dir) {
105
+ system "tar", "cjf", package_path, '.'
106
+ }
107
+ end
108
+
109
+ context "when directory exists" do
110
+ it "extracts onto specified directory" do
111
+ expect {
112
+ package.extract_onto!(extract_dir)
113
+ }.to change {
114
+ extract_dir.join('greeting').exist?
115
+ }.from(false).to(true)
116
+
117
+ expect(extract_dir.join('greeting').read).to eq "hello\n"
118
+ end
119
+ end
120
+
121
+ context "when directory not exists" do
122
+ before do
123
+ FileUtils.remove_entry_secure extract_dir
124
+ end
125
+
126
+ it "creates directory" do
127
+ expect {
128
+ package.extract_onto!(extract_dir)
129
+ }.to change {
130
+ extract_dir.exist?
131
+ }.from(false).to(true)
132
+
133
+ expect(extract_dir.join('greeting').read).to eq "hello\n"
134
+ end
135
+ end
136
+ end
137
+
138
+ context "without package" do
139
+ it "raises error" do
140
+ expect {
141
+ package.extract_onto!(extract_dir)
142
+ }.to raise_error(Mamiya::Package::NotExists)
143
+ end
144
+ end
145
+ end
146
+
147
+ describe "#valid?" do
148
+ subject { package.valid? }
149
+
150
+ it "verifies signature"
151
+
152
+ context "when checksum is correct" do
153
+ before do
154
+ File.write(package_path, "test\n")
155
+ File.write(meta_path, {"checksum" => Digest::SHA2.hexdigest("test\n")}.to_json + "\n")
156
+ end
157
+
158
+ it { should be_true }
159
+ end
160
+
161
+ context "when checksum is incorrect" do
162
+ before do
163
+ File.write(package_path, "wrong\n")
164
+ File.write(meta_path, {"checksum" => Digest::SHA2.hexdigest("text\n")}.to_json + "\n")
165
+ end
166
+
167
+ it { should be_false }
168
+ end
169
+
170
+ context "when package not exists" do
171
+ it "raises error" do
172
+ expect { subject }.to raise_error(Mamiya::Package::NotExists)
173
+ end
174
+ end
175
+
176
+ context "when package meta json not exists" do
177
+ before do
178
+ File.write package_path, "\n"
179
+ end
180
+
181
+ it "raises error" do
182
+ expect { subject }.to raise_error(Mamiya::Package::NotExists)
183
+ end
184
+ end
185
+ end
186
+
187
+ describe "#name" do
188
+ subject { package.name }
189
+
190
+ it { should eq 'test' }
191
+
192
+ context "when meta['name'] exists" do
193
+ before do
194
+ package.meta['name'] = 'pack'
195
+ end
196
+
197
+ it { should eq 'pack' }
198
+ end
199
+ end
200
+
201
+ describe "#application" do
202
+ subject { package.application }
203
+
204
+ it { should eq nil }
205
+
206
+ context "when meta['application'] exists" do
207
+ before do
208
+ package.meta['application'] = 'app'
209
+ end
210
+
211
+ it { should eq 'app' }
212
+ end
213
+ end
214
+
215
+ describe "#checksum" do
216
+ subject { package.checksum }
217
+
218
+ it { should be_nil }
219
+
220
+ context "when package exists" do
221
+ before do
222
+ File.write package_path, "text\n"
223
+ end
224
+
225
+ it { should eq Digest::SHA2.hexdigest("text\n") }
226
+ end
227
+ end
228
+
229
+ describe "#exists?" do
230
+ subject { package.exists? }
231
+
232
+ context "when package exists" do
233
+ before do
234
+ File.write package_path, ''
235
+ end
236
+
237
+ it { should be_true }
238
+ end
239
+
240
+ context "when package not exists" do
241
+ before do
242
+ File.unlink(package_path) if File.exists?(package_path)
243
+ end
244
+
245
+ it { should be_false }
246
+ end
247
+ end
248
+
249
+ describe "#sign!" do
250
+ it "signs package"
251
+ end
252
+
253
+ describe "#build!(build_dir)" do
254
+ let(:dereference_symlinks) { nil }
255
+ let(:exclude_from_package) { nil }
256
+ let(:package_under) { nil }
257
+
258
+ let(:build) {
259
+ kwargs = {}
260
+ kwargs[:dereference_symlinks] = dereference_symlinks unless dereference_symlinks.nil?
261
+ kwargs[:exclude_from_package] = exclude_from_package unless exclude_from_package.nil?
262
+ kwargs[:package_under] = package_under unless package_under.nil?
263
+ package.build!(
264
+ build_dir,
265
+ **kwargs
266
+ )
267
+ }
268
+
269
+ before do
270
+ File.write(build_dir.join('greeting'), 'hello')
271
+ end
272
+
273
+ def build_then_extract!
274
+ build
275
+
276
+ expect(Pathname.new(package_path)).not_to be_nil
277
+
278
+ system "tar", "xf", package_path, "-C", extract_dir.to_s
279
+ end
280
+
281
+ it "creates a package to path" do
282
+ expect {
283
+ build
284
+ }.to change {
285
+ File.exist? package_path
286
+ }.from(false).to(true)
287
+ end
288
+
289
+ it "includes file in build_dir" do
290
+ build_then_extract!
291
+ expect(extract_dir.join('greeting').read).to eq 'hello'
292
+ end
293
+
294
+ it "excludes SCM directories" do
295
+ build_dir.join('.git').mkdir
296
+ File.write build_dir.join('.git', 'test'), "test\n"
297
+ build_then_extract!
298
+
299
+ expect(extract_dir.join('.git')).not_to be_exist
300
+ end
301
+
302
+ it "includes deploy script itself"
303
+
304
+ it "saves meta file" do
305
+ package.meta = {"a" => 1, "b" => {"c" => ["d", "e"]}}
306
+ packed_meta = package.meta.dup
307
+
308
+ build_then_extract!
309
+
310
+ expect(package.meta["checksum"]).to eq Digest::SHA2.file(package_path).hexdigest
311
+
312
+ meta_path_in_build = extract_dir.join('.mamiya.meta.json')
313
+ packed_meta['name'] = package.meta['name']
314
+ json = JSON.parse(File.read(meta_path_in_build))
315
+ expect(json).to eq JSON.parse(packed_meta.to_json)
316
+
317
+ json = JSON.parse(File.read(meta_path))
318
+ expect(json).to eq JSON.parse(package.meta.to_json)
319
+ end
320
+
321
+ it "doesn't leave meta file in build_dir" do
322
+ package.meta = {"a" => 1, "b" => {"c" => ["d", "e"]}}
323
+ build
324
+
325
+ expect(build_dir.join('.mamiya.meta.json')).not_to be_exist
326
+ end
327
+
328
+ context "with exclude_from_package option" do
329
+ let(:exclude_from_package) { ['foo', 'hoge*'] }
330
+
331
+ before do
332
+ File.write build_dir.join('foo'), "test\n"
333
+ File.write build_dir.join('hogefuga'), "test\n"
334
+
335
+ build_then_extract!
336
+ end
337
+
338
+ it "excludes matched files from package" do
339
+ expect(extract_dir.join('foo')).not_to be_exist
340
+ expect(extract_dir.join('hogefuga')).not_to be_exist
341
+
342
+ expect(extract_dir.join('greeting').read).to eq 'hello'
343
+ end
344
+ end
345
+
346
+ context "with package_under option" do
347
+ let(:package_under) { 'dir' }
348
+
349
+ before do
350
+ build_dir.join('dir').mkdir
351
+
352
+ File.write build_dir.join('root'), "shouldnt-be-included\n"
353
+ File.write build_dir.join('dir', 'greeting'), "hola\n"
354
+
355
+ build_then_extract!
356
+ end
357
+
358
+ it "packages under specified directory" do
359
+ expect(extract_dir.join('root')).not_to be_exist
360
+ expect(extract_dir.join('greeting').read).to eq "hola\n"
361
+ end
362
+ end
363
+
364
+ context "with dereference_symlinks option" do
365
+ before do
366
+ File.write build_dir.join('target'), "I am target\n"
367
+ build_dir.join('alias').make_symlink('target')
368
+
369
+ build_then_extract!
370
+ end
371
+
372
+ context "when the option is true" do
373
+ let(:dereference_symlinks) { true }
374
+
375
+ it "dereferences symlinks for package" do
376
+ expect(extract_dir.join('alias')).to be_exist
377
+ expect(extract_dir.join('alias')).not_to be_symlink
378
+ expect(extract_dir.join('alias').read).to eq "I am target\n"
379
+ end
380
+ end
381
+
382
+ context "when the option is false" do
383
+ let(:dereference_symlinks) { false }
384
+
385
+ it "doesn't dereference symlinks" do
386
+ expect(extract_dir.join('alias')).to be_exist
387
+ expect(extract_dir.join('alias')).to be_symlink
388
+ realpath = extract_dir.join('alias').realpath.relative_path_from(extract_dir.realpath).to_s
389
+ expect(realpath).to eq 'target'
390
+ end
391
+ end
392
+ end
393
+ end
394
+ end