mamiya 0.0.1.alpha2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +17 -0
  3. data/.rspec +2 -0
  4. data/.travis.yml +16 -0
  5. data/Gemfile +8 -0
  6. data/LICENSE.txt +22 -0
  7. data/README.md +43 -0
  8. data/Rakefile +6 -0
  9. data/bin/mamiya +17 -0
  10. data/config.example.yml +11 -0
  11. data/docs/sequences/deploy.png +0 -0
  12. data/docs/sequences/deploy.uml +58 -0
  13. data/example.rb +74 -0
  14. data/lib/mamiya.rb +5 -0
  15. data/lib/mamiya/agent.rb +181 -0
  16. data/lib/mamiya/agent/actions.rb +12 -0
  17. data/lib/mamiya/agent/fetcher.rb +137 -0
  18. data/lib/mamiya/agent/handlers/abstract.rb +20 -0
  19. data/lib/mamiya/agent/handlers/fetch.rb +68 -0
  20. data/lib/mamiya/cli.rb +322 -0
  21. data/lib/mamiya/cli/client.rb +172 -0
  22. data/lib/mamiya/config.rb +57 -0
  23. data/lib/mamiya/dsl.rb +192 -0
  24. data/lib/mamiya/helpers/git.rb +75 -0
  25. data/lib/mamiya/logger.rb +190 -0
  26. data/lib/mamiya/master.rb +118 -0
  27. data/lib/mamiya/master/agent_monitor.rb +146 -0
  28. data/lib/mamiya/master/agent_monitor_handlers.rb +44 -0
  29. data/lib/mamiya/master/web.rb +148 -0
  30. data/lib/mamiya/package.rb +122 -0
  31. data/lib/mamiya/script.rb +117 -0
  32. data/lib/mamiya/steps/abstract.rb +19 -0
  33. data/lib/mamiya/steps/build.rb +72 -0
  34. data/lib/mamiya/steps/extract.rb +26 -0
  35. data/lib/mamiya/steps/fetch.rb +24 -0
  36. data/lib/mamiya/steps/push.rb +34 -0
  37. data/lib/mamiya/storages.rb +17 -0
  38. data/lib/mamiya/storages/abstract.rb +48 -0
  39. data/lib/mamiya/storages/mock.rb +61 -0
  40. data/lib/mamiya/storages/s3.rb +127 -0
  41. data/lib/mamiya/util/label_matcher.rb +38 -0
  42. data/lib/mamiya/version.rb +3 -0
  43. data/mamiya.gemspec +35 -0
  44. data/misc/logger_test.rb +12 -0
  45. data/spec/agent/actions_spec.rb +37 -0
  46. data/spec/agent/fetcher_spec.rb +199 -0
  47. data/spec/agent/handlers/fetch_spec.rb +121 -0
  48. data/spec/agent_spec.rb +255 -0
  49. data/spec/config_spec.rb +50 -0
  50. data/spec/dsl_spec.rb +291 -0
  51. data/spec/fixtures/dsl_test_load.rb +1 -0
  52. data/spec/fixtures/dsl_test_use.rb +1 -0
  53. data/spec/fixtures/helpers/foo.rb +1 -0
  54. data/spec/fixtures/test-package-source/.mamiya.meta.json +1 -0
  55. data/spec/fixtures/test-package-source/greeting +1 -0
  56. data/spec/fixtures/test-package.tar.gz +0 -0
  57. data/spec/fixtures/test.yml +4 -0
  58. data/spec/logger_spec.rb +68 -0
  59. data/spec/master/agent_monitor_spec.rb +269 -0
  60. data/spec/master/web_spec.rb +121 -0
  61. data/spec/master_spec.rb +94 -0
  62. data/spec/package_spec.rb +394 -0
  63. data/spec/script_spec.rb +78 -0
  64. data/spec/spec_helper.rb +38 -0
  65. data/spec/steps/build_spec.rb +261 -0
  66. data/spec/steps/extract_spec.rb +68 -0
  67. data/spec/steps/fetch_spec.rb +96 -0
  68. data/spec/steps/push_spec.rb +73 -0
  69. data/spec/storages/abstract_spec.rb +22 -0
  70. data/spec/storages/s3_spec.rb +342 -0
  71. data/spec/storages_spec.rb +33 -0
  72. data/spec/support/dummy_serf.rb +70 -0
  73. data/spec/util/label_matcher_spec.rb +85 -0
  74. metadata +272 -0
@@ -0,0 +1,96 @@
1
+ require 'spec_helper'
2
+ require 'tmpdir'
3
+ require 'pathname'
4
+ require 'fileutils'
5
+ require 'json'
6
+
7
+ require 'mamiya/package'
8
+ require 'mamiya/storages/mock'
9
+
10
+ require 'mamiya/steps/fetch'
11
+
12
+ describe Mamiya::Steps::Fetch do
13
+ let!(:tmpdir) { Dir.mktmpdir("mamiya-steps-fetch-spec") }
14
+ after { FileUtils.remove_entry_secure tmpdir }
15
+
16
+ let(:package_dir) { Pathname.new(tmpdir).join('pkg').tap(&:mkdir) }
17
+ let(:destination_dir) { Pathname.new(tmpdir).join('dst').tap(&:mkdir) }
18
+
19
+ let(:package_name) { 'test' }
20
+ let(:package_path) { package_dir.join("#{package_name}.tar.gz") }
21
+
22
+ let(:package2_name) { 'test2' }
23
+ let(:package2_path) { package_dir.join("#{package2_name}.tar.gz") }
24
+
25
+
26
+ let(:script) do
27
+ double('script',
28
+ application: 'another',
29
+ )
30
+ end
31
+
32
+ let(:config_source) do
33
+ { storage: {} }
34
+ end
35
+
36
+ let(:config) do
37
+ double('config',
38
+ storage_class: Mamiya::Storages::Mock
39
+ ).tap do |_|
40
+ allow(_).to receive(:[]) do |k|
41
+ config_source[k]
42
+ end
43
+ end
44
+ end
45
+
46
+ let(:options) do
47
+ {package: package_name, application: 'app', destination: destination_dir.to_s}
48
+ end
49
+
50
+ subject(:fetch_step) { described_class.new(script: script, config: config, **options) }
51
+
52
+ describe "#run!" do
53
+ before do
54
+ File.write package_path, "\n"
55
+ File.write(package_path.to_s.gsub(/\.tar\.gz$/,'.json'),
56
+ {'name' => 'test', 'application' => 'app'}.to_json)
57
+
58
+ File.write package2_path, "\n"
59
+ File.write(package2_path.to_s.gsub(/\.tar\.gz$/,'.json'),
60
+ {'name' => 'test2', 'application' => 'another'}.to_json)
61
+
62
+
63
+ Mamiya::Storages::Mock.new(application: 'app').push(
64
+ Mamiya::Package.new(package_path))
65
+ Mamiya::Storages::Mock.new(application: 'another').push(
66
+ Mamiya::Package.new(package2_path))
67
+ end
68
+
69
+ it "fetches package from storage" do
70
+ fetch_step.run!
71
+ expect(destination_dir.join("#{package_name}.tar.gz")).to be_exist
72
+ end
73
+
74
+ context "when options[:application] is nil" do
75
+ let(:options) do
76
+ {package: package2_name, destination: destination_dir.to_s}
77
+ end
78
+
79
+ it "takes application name from script if available" do
80
+ fetch_step.run!
81
+ expect(destination_dir.join("#{package2_name}.tar.gz")).to be_exist
82
+ expect(destination_dir.join("#{package_name}.tar.gz")).not_to be_exist
83
+
84
+ expect(JSON.parse(destination_dir.join("#{package2_name}.json").read)['application']).to eq 'another'
85
+ end
86
+ end
87
+
88
+ context "with verify option" do
89
+ it "verifies "
90
+ end
91
+
92
+ context "when package not exists" do
93
+ it "-"
94
+ end
95
+ end
96
+ end
@@ -0,0 +1,73 @@
1
+ require 'spec_helper'
2
+ require 'tmpdir'
3
+ require 'pathname'
4
+ require 'fileutils'
5
+
6
+ require 'mamiya/package'
7
+ require 'mamiya/storages/mock'
8
+
9
+ require 'mamiya/steps/push'
10
+
11
+ describe Mamiya::Steps::Push do
12
+ let!(:tmpdir) { Dir.mktmpdir("mamiya-steps-build-spec") }
13
+ after { FileUtils.remove_entry_secure tmpdir }
14
+
15
+ let(:package_dir) { Pathname.new(tmpdir).join('pkg').tap(&:mkdir) }
16
+
17
+ let(:target_package) { package_dir.join('test.tar.gz').to_s }
18
+ let(:script) do
19
+ double('script',
20
+ application: 'myapp',
21
+ )
22
+ end
23
+
24
+ let(:config_source) do
25
+ {
26
+ storage: {
27
+ }
28
+ }
29
+ end
30
+ let(:config) do
31
+ double('config',
32
+ storage_class: Mamiya::Storages::Mock
33
+ ).tap do |_|
34
+ allow(_).to receive(:[]) do |k|
35
+ config_source[k]
36
+ end
37
+ end
38
+ end
39
+
40
+ let(:options) do
41
+ {package: target_package}
42
+ end
43
+
44
+ subject(:push_step) { described_class.new(script: script, config: config, **options) }
45
+
46
+ describe "#run!" do
47
+ let(:package_application) do
48
+ # to confirm Push step uses package's application name
49
+ 'app'
50
+ end
51
+
52
+ before do
53
+ File.write target_package, "\n"
54
+ File.write target_package.gsub(/\.tar\.gz$/,'.json'), "{}\n"
55
+
56
+ allow_any_instance_of(Mamiya::Package).to receive(:application).and_return(package_application)
57
+ end
58
+
59
+ it "pushes package to storage" do
60
+ push_step.run!
61
+ expect(Mamiya::Storages::Mock.storage['app']['test']).not_to be_nil
62
+ end
63
+
64
+ context "with application option" do
65
+ let(:options) { {package: target_package, application: 'newapp'} }
66
+
67
+ it "pushes package to storage for given application name" do
68
+ push_step.run!
69
+ expect(Mamiya::Storages::Mock.storage['newapp']['test']).not_to be_nil
70
+ end
71
+ end
72
+ end
73
+ end
@@ -0,0 +1,22 @@
1
+ require 'spec_helper'
2
+
3
+ require 'mamiya/storages/abstract'
4
+
5
+ describe Mamiya::Storages::Abstract do
6
+ subject(:storage) { described_class.new }
7
+
8
+ describe "#prune(nums_to_keep)" do
9
+ before do
10
+ allow(storage).to receive(:packages).and_return(%w(1 2 3 4 5 6 7))
11
+ end
12
+
13
+ it "discards old releases" do
14
+ expect(storage).to receive(:remove).with('1')
15
+ expect(storage).to receive(:remove).with('2')
16
+ expect(storage).to receive(:remove).with('3')
17
+ expect(storage).to receive(:remove).with('4')
18
+
19
+ storage.prune(3)
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,342 @@
1
+ require 'spec_helper'
2
+ require 'aws-sdk-core'
3
+ require 'mamiya/package'
4
+ require 'mamiya/storages/abstract'
5
+ require 'mamiya/storages/s3'
6
+ require 'tmpdir'
7
+ require 'fileutils'
8
+ require 'stringio'
9
+
10
+ describe Mamiya::Storages::S3 do
11
+ before(:suite) do
12
+ %w(AWS_ACCESS_KEY AWS_ACCESS_KEY_ID AMAZON_ACCESS_KEY_ID
13
+ AWS_SECRET_KEY AWS_SECRET_ACCESS_KEY AMAZON_SECRET_ACCESS_KEY
14
+ AWS_SESSION_TOKEN AMAZON_SESSION_TOKEN).each do |key|
15
+ ENV.delete key
16
+ end
17
+ end
18
+
19
+ let(:bucket) { 'testbucket' }
20
+ let(:config) do
21
+ {
22
+ application: 'myapp',
23
+ bucket: bucket,
24
+ foo: :bar,
25
+ access_key_id: 'AKI',
26
+ secret_access_key: 'secret',
27
+ region: 'ap-northeast-1'
28
+ }
29
+ end
30
+ subject(:storage) { described_class.new(config) }
31
+
32
+ let(:s3) do
33
+ double('s3',
34
+ put_object: nil,
35
+ head_object: nil,
36
+ list_objects: nil,
37
+ delete_objects: nil
38
+ )
39
+ end
40
+
41
+ before do
42
+ allow(Aws::S3).to receive(:new).with(foo: :bar, access_key_id: 'AKI', secret_access_key: 'secret', region: 'ap-northeast-1').and_return(s3)
43
+ end
44
+
45
+ describe "#push(package)" do
46
+ let!(:tmpdir) { Dir.mktmpdir("mamiya-package-spec") }
47
+ let(:tarball) { File.join(tmpdir, 'test.tar.gz') }
48
+ let(:metafile) { File.join(tmpdir, 'test.json') }
49
+ before do
50
+ File.write tarball, "aaaaa\n"
51
+ File.write File.join(tmpdir, 'test.json'), "{}\n"
52
+ end
53
+ after { FileUtils.remove_entry_secure tmpdir }
54
+
55
+ let(:package) { double('package', :kind_of? => true, :exists? => true, path: tarball, meta_path: metafile, name: 'test') }
56
+
57
+ it "uploads package to S3" do
58
+ allow(s3).to receive(:head_object).and_raise(Aws::S3::Errors::NotFound.new(nil, ''))
59
+
60
+ expect(s3).to receive(:put_object) do |options|
61
+ expect(options[:bucket]).to eq 'testbucket'
62
+ expect(options[:key]).to eq "myapp/test.tar.gz"
63
+ expect(options[:body]).to be_a_kind_of(File)
64
+ expect(options[:body].path).to eq tarball
65
+ end
66
+
67
+ expect(s3).to receive(:put_object) do |options|
68
+ expect(options[:bucket]).to eq 'testbucket'
69
+ expect(options[:key]).to eq "myapp/test.json"
70
+ expect(options[:body]).to be_a_kind_of(File)
71
+ expect(options[:body].path).to eq metafile
72
+ end
73
+
74
+ storage.push package
75
+ end
76
+
77
+ context "when not built" do
78
+ before do
79
+ package.stub(:exists? => false)
80
+ end
81
+
82
+ it "raises error" do
83
+ expect {
84
+ storage.push(package)
85
+ }.to raise_error(Mamiya::Storages::Abstract::NotBuilt)
86
+ end
87
+ end
88
+
89
+ context "when already uploaded" do
90
+ it "raises error" do
91
+ allow(s3).to receive(:head_object).with(bucket: 'testbucket', key: 'myapp/test.tar.gz').and_return(double('response'))
92
+
93
+ expect {
94
+ storage.push(package)
95
+ }.to raise_error(Mamiya::Storages::Abstract::AlreadyExists)
96
+ end
97
+ end
98
+ end
99
+
100
+ describe "#fetch(package_name, dir)" do
101
+ let!(:tmpdir) { Dir.mktmpdir("mamiya-package-spec") }
102
+ after { FileUtils.remove_entry_secure tmpdir }
103
+
104
+ let(:tarball) { File.join(tmpdir, 'test.tar.gz') }
105
+ let(:metafile) { File.join(tmpdir, 'test.json') }
106
+
107
+ let(:package_name) { 'test' }
108
+ subject(:fetch) { storage.fetch(package_name, tmpdir) }
109
+
110
+ it "retrieves package from S3" do
111
+ requests = []
112
+ expect(s3).to receive(:get_object).twice do |options, send_options|
113
+ requests << [options, send_options]
114
+ if options[:key] && send_options[:target] &&
115
+ options[:key].end_with?('.json') && send_options[:target].kind_of?(IO)
116
+ expect(send_options[:target]).to be_binmode
117
+ send_options[:target].puts "{}"
118
+ end
119
+ end
120
+
121
+ fetch
122
+
123
+ options, send_options = requests.shift
124
+ expect(options[:bucket]).to eq 'testbucket'
125
+ expect(options[:key]).to eq "myapp/test.tar.gz"
126
+ expect(send_options[:target]).to be_a_kind_of(File)
127
+ expect(send_options[:target].path).to eq tarball
128
+
129
+ options, send_options = requests.shift
130
+ expect(options[:bucket]).to eq 'testbucket'
131
+ expect(options[:key]).to eq "myapp/test.json"
132
+ expect(send_options[:target]).to be_a_kind_of(File)
133
+ expect(send_options[:target].path).to eq metafile
134
+ end
135
+
136
+ it "returns Mamiya::Package" do
137
+ allow(s3).to receive(:get_object) do
138
+ File.write metafile, "{}\n"
139
+ end
140
+
141
+ expect(fetch).to be_a_kind_of(Mamiya::Package)
142
+ expect(File.realpath(fetch.path)).to eq File.realpath(tarball)
143
+ end
144
+
145
+ context "when not found" do
146
+ before do
147
+ allow(s3).to receive(:get_object).and_raise(Aws::S3::Errors::NoSuchKey.new(nil, ''))
148
+ end
149
+
150
+ it "raises error" do
151
+ expect {
152
+ fetch
153
+ }.to raise_error(Mamiya::Storages::Abstract::NotFound)
154
+ end
155
+ end
156
+
157
+ context "when meta already exists" do
158
+ before do
159
+ File.write metafile, "\n"
160
+ end
161
+
162
+ it "raises error" do
163
+ expect {
164
+ fetch
165
+ }.to raise_error(Mamiya::Storages::Abstract::AlreadyFetched)
166
+ end
167
+ end
168
+
169
+ context "when tarball already exists" do
170
+ before do
171
+ File.write tarball, "\n"
172
+ end
173
+
174
+ it "raises error" do
175
+ expect {
176
+ fetch
177
+ }.to raise_error(Mamiya::Storages::Abstract::AlreadyFetched)
178
+ end
179
+ end
180
+
181
+ context "when name has .json" do
182
+ let(:package_name) { 'test.json' }
183
+
184
+ it "retrieves package" do
185
+ expect(s3).to receive(:get_object).with(
186
+ hash_including(bucket: 'testbucket', key: 'myapp/test.tar.gz'), hash_including(target: an_instance_of(File)))
187
+ expect(s3).to receive(:get_object).with(
188
+ hash_including(bucket: 'testbucket', key: 'myapp/test.json'), hash_including(target: an_instance_of(File))) do
189
+ File.write metafile, "{}\n"
190
+ end
191
+
192
+ fetch
193
+ end
194
+ end
195
+
196
+ context "when name has .tar.gz" do
197
+ let(:package_name) { 'test.tar.gz' }
198
+
199
+ it "retrieves package" do
200
+ expect(s3).to receive(:get_object).with(
201
+ hash_including(bucket: 'testbucket', key: 'myapp/test.tar.gz'), hash_including(target: an_instance_of(File)))
202
+ expect(s3).to receive(:get_object).with(
203
+ hash_including(bucket: 'testbucket', key: 'myapp/test.json'), hash_including(target: an_instance_of(File))) do
204
+ File.write metafile, "{}\n"
205
+ end
206
+
207
+ fetch
208
+ end
209
+ end
210
+ end
211
+
212
+ describe "#meta(package_name)" do
213
+ let(:package_name) { 'test' }
214
+ subject(:meta) { storage.meta(package_name) }
215
+
216
+ before do
217
+ allow(s3).to receive(:get_object).with(bucket: 'testbucket', key: 'myapp/test.json').and_return(
218
+ double("response", body: StringIO.new({"foo" => "bar"}.to_json, "r").tap(&:read))
219
+ )
220
+ end
221
+
222
+ it "retrieves meta JSON from S3" do
223
+ expect(meta).to eq("foo" => "bar")
224
+ end
225
+
226
+ context "when not found" do
227
+ before do
228
+ allow(s3).to receive(:get_object).and_raise(Aws::S3::Errors::NoSuchKey.new(nil, ''))
229
+ end
230
+
231
+ it "returns nil" do
232
+ expect(meta).to be_nil
233
+ end
234
+ end
235
+
236
+ context "when name has .json" do
237
+ let(:package_name) { 'test.json' }
238
+
239
+ it "retrieves meta JSON from S3" do
240
+ expect(meta).to eq("foo" => "bar")
241
+ end
242
+ end
243
+
244
+ context "when name has .tar.gz" do
245
+ let(:package_name) { 'test.tar.gz' }
246
+
247
+ it "retrieves meta JSON from S3" do
248
+ expect(meta).to eq("foo" => "bar")
249
+ end
250
+ end
251
+ end
252
+
253
+ describe "#remove(package_name)" do
254
+ let(:package_name) { 'test' }
255
+ subject(:remove) { storage.remove(package_name) }
256
+
257
+ before do
258
+ allow(s3).to receive(:head_object).and_return(double('response'))
259
+ end
260
+
261
+ it "removes specified package from S3" do
262
+ expect(s3).to receive(:delete_objects).with(bucket: 'testbucket', objects: [{key: 'myapp/test.tar.gz'}, {key: 'myapp/test.json'}])
263
+ remove
264
+ end
265
+
266
+ context "with name has .tar.gz" do
267
+ let(:package_name) { 'test.tar.gz' }
268
+
269
+ it "removes specified package from S3" do
270
+ expect(s3).to receive(:delete_objects).with(bucket: 'testbucket', objects: [{key: 'myapp/test.tar.gz'}, {key: 'myapp/test.json'}])
271
+ remove
272
+ end
273
+ end
274
+
275
+ context "with name has .json" do
276
+ let(:package_name) { 'test.json' }
277
+
278
+ it "removes specified package from S3" do
279
+ expect(s3).to receive(:delete_objects).with(bucket: 'testbucket', objects: [{key: 'myapp/test.tar.gz'}, {key: 'myapp/test.json'}])
280
+ remove
281
+ end
282
+ end
283
+
284
+ context "when not found" do
285
+ before do
286
+ allow(s3).to receive(:head_object).and_raise(Aws::S3::Errors::NotFound.new(nil, ''))
287
+ end
288
+
289
+ it "raises error" do
290
+ expect { storage.remove('test') }.to raise_error(Mamiya::Storages::Abstract::NotFound)
291
+ end
292
+ end
293
+ end
294
+
295
+ describe ".find" do
296
+ before do
297
+ allow(s3).to receive(:list_objects).with(bucket: 'testbucket', delimiter: '/') \
298
+ .and_return(
299
+ double("object list",
300
+ common_prefixes: [
301
+ double("prefix1", prefix: 'myapp'),
302
+ double("prefix2", prefix: 'testapp')
303
+ ]
304
+ )
305
+ )
306
+ end
307
+
308
+ subject(:applications) { described_class.find(config.dup.tap{|_| _.delete(:application) }) }
309
+
310
+ it "lists applications in S3" do
311
+ expect(applications).to be_a_kind_of(Hash)
312
+ expect(applications['myapp']).to be_a_kind_of(described_class)
313
+ expect(applications['myapp'].application).to eq 'myapp'
314
+ expect(applications['testapp']).to be_a_kind_of(described_class)
315
+ expect(applications['testapp'].application).to eq 'testapp'
316
+ end
317
+ end
318
+
319
+ describe "#packages" do
320
+ before do
321
+ allow(s3).to receive(:list_objects).with(bucket: 'testbucket', delimiter: '/', prefix: 'myapp/') \
322
+ .and_return(
323
+ double("object list",
324
+ contents: [
325
+ double("obj1.1", key: 'myapp/1.tar.gz'),
326
+ double("obj1.2", key: 'myapp/1.json'),
327
+ double("obj2.1", key: 'myapp/2.tar.gz'),
328
+ double("obj2.2", key: 'myapp/2.json'),
329
+ double("obj3.2", key: 'myapp/3.json'),
330
+ double("obj4.1", key: 'myapp/4.tar.gz'),
331
+ ]
332
+ )
333
+ )
334
+ end
335
+
336
+ subject(:packages) { storage.packages }
337
+
338
+ it "lists packages in S3" do
339
+ expect(packages).to eq ['1', '2']
340
+ end
341
+ end
342
+ end