mamiya 0.0.1.alpha2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +17 -0
- data/.rspec +2 -0
- data/.travis.yml +16 -0
- data/Gemfile +8 -0
- data/LICENSE.txt +22 -0
- data/README.md +43 -0
- data/Rakefile +6 -0
- data/bin/mamiya +17 -0
- data/config.example.yml +11 -0
- data/docs/sequences/deploy.png +0 -0
- data/docs/sequences/deploy.uml +58 -0
- data/example.rb +74 -0
- data/lib/mamiya.rb +5 -0
- data/lib/mamiya/agent.rb +181 -0
- data/lib/mamiya/agent/actions.rb +12 -0
- data/lib/mamiya/agent/fetcher.rb +137 -0
- data/lib/mamiya/agent/handlers/abstract.rb +20 -0
- data/lib/mamiya/agent/handlers/fetch.rb +68 -0
- data/lib/mamiya/cli.rb +322 -0
- data/lib/mamiya/cli/client.rb +172 -0
- data/lib/mamiya/config.rb +57 -0
- data/lib/mamiya/dsl.rb +192 -0
- data/lib/mamiya/helpers/git.rb +75 -0
- data/lib/mamiya/logger.rb +190 -0
- data/lib/mamiya/master.rb +118 -0
- data/lib/mamiya/master/agent_monitor.rb +146 -0
- data/lib/mamiya/master/agent_monitor_handlers.rb +44 -0
- data/lib/mamiya/master/web.rb +148 -0
- data/lib/mamiya/package.rb +122 -0
- data/lib/mamiya/script.rb +117 -0
- data/lib/mamiya/steps/abstract.rb +19 -0
- data/lib/mamiya/steps/build.rb +72 -0
- data/lib/mamiya/steps/extract.rb +26 -0
- data/lib/mamiya/steps/fetch.rb +24 -0
- data/lib/mamiya/steps/push.rb +34 -0
- data/lib/mamiya/storages.rb +17 -0
- data/lib/mamiya/storages/abstract.rb +48 -0
- data/lib/mamiya/storages/mock.rb +61 -0
- data/lib/mamiya/storages/s3.rb +127 -0
- data/lib/mamiya/util/label_matcher.rb +38 -0
- data/lib/mamiya/version.rb +3 -0
- data/mamiya.gemspec +35 -0
- data/misc/logger_test.rb +12 -0
- data/spec/agent/actions_spec.rb +37 -0
- data/spec/agent/fetcher_spec.rb +199 -0
- data/spec/agent/handlers/fetch_spec.rb +121 -0
- data/spec/agent_spec.rb +255 -0
- data/spec/config_spec.rb +50 -0
- data/spec/dsl_spec.rb +291 -0
- data/spec/fixtures/dsl_test_load.rb +1 -0
- data/spec/fixtures/dsl_test_use.rb +1 -0
- data/spec/fixtures/helpers/foo.rb +1 -0
- data/spec/fixtures/test-package-source/.mamiya.meta.json +1 -0
- data/spec/fixtures/test-package-source/greeting +1 -0
- data/spec/fixtures/test-package.tar.gz +0 -0
- data/spec/fixtures/test.yml +4 -0
- data/spec/logger_spec.rb +68 -0
- data/spec/master/agent_monitor_spec.rb +269 -0
- data/spec/master/web_spec.rb +121 -0
- data/spec/master_spec.rb +94 -0
- data/spec/package_spec.rb +394 -0
- data/spec/script_spec.rb +78 -0
- data/spec/spec_helper.rb +38 -0
- data/spec/steps/build_spec.rb +261 -0
- data/spec/steps/extract_spec.rb +68 -0
- data/spec/steps/fetch_spec.rb +96 -0
- data/spec/steps/push_spec.rb +73 -0
- data/spec/storages/abstract_spec.rb +22 -0
- data/spec/storages/s3_spec.rb +342 -0
- data/spec/storages_spec.rb +33 -0
- data/spec/support/dummy_serf.rb +70 -0
- data/spec/util/label_matcher_spec.rb +85 -0
- metadata +272 -0
@@ -0,0 +1,72 @@
|
|
1
|
+
require 'mamiya/steps/abstract'
|
2
|
+
require 'mamiya/package'
|
3
|
+
|
4
|
+
module Mamiya
|
5
|
+
module Steps
|
6
|
+
class Build < Abstract
|
7
|
+
def run!
|
8
|
+
@exception = nil
|
9
|
+
|
10
|
+
logger.info "Initiating package build"
|
11
|
+
|
12
|
+
logger.info "Running script.before_build"
|
13
|
+
script.before_build[]
|
14
|
+
|
15
|
+
unless script.skip_prepare_build
|
16
|
+
logger.info "Running script.prepare_build"
|
17
|
+
script.prepare_build[File.exists?(script.build_from)]
|
18
|
+
else
|
19
|
+
logger.debug "prepare_build skipped due to script.skip_prepare_build"
|
20
|
+
end
|
21
|
+
|
22
|
+
old_pwd = Dir.pwd
|
23
|
+
begin
|
24
|
+
# Using without block because chdir in block shows warning
|
25
|
+
Dir.chdir(script.build_from)
|
26
|
+
logger.info "Running script.build ..."
|
27
|
+
logger.debug "pwd=#{Dir.pwd}"
|
28
|
+
script.build[]
|
29
|
+
ensure
|
30
|
+
Dir.chdir old_pwd
|
31
|
+
end
|
32
|
+
|
33
|
+
|
34
|
+
logger.debug "Determining package name..."
|
35
|
+
package_name = Dir.chdir(script.build_from) {
|
36
|
+
script.package_name[
|
37
|
+
[Time.now.strftime("%Y-%m-%d_%H.%M.%S"), script.application]
|
38
|
+
].join('-')
|
39
|
+
}
|
40
|
+
logger.info "Package name determined: #{package_name}"
|
41
|
+
|
42
|
+
package_path = File.join(script.build_to, package_name)
|
43
|
+
package = Mamiya::Package.new(package_path)
|
44
|
+
package.meta[:application] = script.application
|
45
|
+
|
46
|
+
Dir.chdir(script.build_from) do
|
47
|
+
package.meta.replace script.package_meta[package.meta]
|
48
|
+
end
|
49
|
+
|
50
|
+
logger.info "Packaging to: #{package.path}"
|
51
|
+
logger.debug "meta=#{package.meta.inspect}"
|
52
|
+
package.build!(script.build_from,
|
53
|
+
exclude_from_package: script.exclude_from_package || [],
|
54
|
+
dereference_symlinks: script.dereference_symlinks || false,
|
55
|
+
package_under: script.package_under || nil,
|
56
|
+
logger: logger)
|
57
|
+
logger.info "Packed."
|
58
|
+
|
59
|
+
rescue Exception => e
|
60
|
+
@exception = e
|
61
|
+
raise
|
62
|
+
ensure
|
63
|
+
logger.warn "Exception occured, cleaning up..." if @exception
|
64
|
+
|
65
|
+
logger.info "Running script.after_build"
|
66
|
+
script.after_build[@exception]
|
67
|
+
|
68
|
+
logger.info "DONE!" unless @exception
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'mamiya/steps/abstract'
|
2
|
+
|
3
|
+
|
4
|
+
module Mamiya
|
5
|
+
module Steps
|
6
|
+
class Extract < Abstract
|
7
|
+
def run!
|
8
|
+
package = case options[:package]
|
9
|
+
when Mamiya::Package
|
10
|
+
options[:package]
|
11
|
+
else
|
12
|
+
Mamiya::Package.new(options[:package])
|
13
|
+
end
|
14
|
+
|
15
|
+
if File.exists?(options[:destination])
|
16
|
+
destination = File.join(options[:destination], package.name)
|
17
|
+
else
|
18
|
+
destination = options[:destination]
|
19
|
+
end
|
20
|
+
|
21
|
+
logger.info "Extracting #{package.path} onto #{destination}"
|
22
|
+
package.extract_onto!(destination)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
require 'mamiya/steps/abstract'
|
2
|
+
require 'mamiya/package'
|
3
|
+
|
4
|
+
module Mamiya
|
5
|
+
module Steps
|
6
|
+
class Fetch < Abstract
|
7
|
+
def run!
|
8
|
+
application = options[:application] || (script && script.application)
|
9
|
+
|
10
|
+
raise 'no application name given' unless application
|
11
|
+
|
12
|
+
storage = config.storage_class.new(
|
13
|
+
config[:storage].merge(
|
14
|
+
application: application
|
15
|
+
)
|
16
|
+
)
|
17
|
+
|
18
|
+
logger.info("Fetching package #{options[:package]} from storage(app=#{storage.application}) to #{options[:destination]}...")
|
19
|
+
storage.fetch(options[:package], options[:destination])
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
@@ -0,0 +1,34 @@
|
|
1
|
+
require 'mamiya/steps/abstract'
|
2
|
+
require 'mamiya/package'
|
3
|
+
|
4
|
+
module Mamiya
|
5
|
+
module Steps
|
6
|
+
class Push < Abstract
|
7
|
+
def run!
|
8
|
+
package = case options[:package]
|
9
|
+
when Mamiya::Package
|
10
|
+
options[:package]
|
11
|
+
else
|
12
|
+
Mamiya::Package.new(options[:package])
|
13
|
+
end
|
14
|
+
|
15
|
+
application = options[:application] || package.application
|
16
|
+
|
17
|
+
raise 'no application name given' unless application
|
18
|
+
|
19
|
+
storage = config.storage_class.new(
|
20
|
+
config[:storage].merge(
|
21
|
+
application: application
|
22
|
+
)
|
23
|
+
)
|
24
|
+
|
25
|
+
logger.info "Pushing #{package.path} to storage(app=#{storage.application})..."
|
26
|
+
|
27
|
+
storage.push(package)
|
28
|
+
|
29
|
+
logger.info "DONE!"
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
@@ -0,0 +1,17 @@
|
|
1
|
+
module Mamiya
|
2
|
+
module Storages
|
3
|
+
def self.find(name)
|
4
|
+
name = name.to_s
|
5
|
+
classish_name = name.capitalize.gsub(/_./) { |s| s[1].upcase }
|
6
|
+
|
7
|
+
begin
|
8
|
+
return const_get(classish_name)
|
9
|
+
rescue NameError; end
|
10
|
+
|
11
|
+
require "mamiya/storages/#{File.basename(name)}"
|
12
|
+
const_get(classish_name)
|
13
|
+
rescue NameError, LoadError
|
14
|
+
return nil
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,48 @@
|
|
1
|
+
module Mamiya
|
2
|
+
module Storages
|
3
|
+
class Abstract
|
4
|
+
class NotBuilt < Exception; end
|
5
|
+
class NotFound < Exception; end
|
6
|
+
class AlreadyExists < Exception; end
|
7
|
+
class AlreadyFetched < Exception; end
|
8
|
+
|
9
|
+
def initialize(config = {})
|
10
|
+
@config = config.dup
|
11
|
+
@application = config.delete(:application)
|
12
|
+
end
|
13
|
+
|
14
|
+
attr_reader :config, :application
|
15
|
+
|
16
|
+
def self.find(config={})
|
17
|
+
{}
|
18
|
+
end
|
19
|
+
|
20
|
+
def packages
|
21
|
+
[]
|
22
|
+
end
|
23
|
+
|
24
|
+
def push(package)
|
25
|
+
raise NotImplementedError
|
26
|
+
end
|
27
|
+
|
28
|
+
def fetch(package_name, dir)
|
29
|
+
raise NotImplementedError
|
30
|
+
end
|
31
|
+
|
32
|
+
def meta(package_name)
|
33
|
+
raise NotImplementedError
|
34
|
+
end
|
35
|
+
|
36
|
+
def remove(package)
|
37
|
+
raise NotImplementedError
|
38
|
+
end
|
39
|
+
|
40
|
+
def prune(nums_to_keep)
|
41
|
+
packages = self.packages()
|
42
|
+
(packages - packages.last(nums_to_keep)).each do |package|
|
43
|
+
self.remove(package)
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
@@ -0,0 +1,61 @@
|
|
1
|
+
require 'mamiya/package'
|
2
|
+
require 'mamiya/storages/abstract'
|
3
|
+
require 'fileutils'
|
4
|
+
|
5
|
+
module Mamiya
|
6
|
+
module Storages
|
7
|
+
class Mock < Mamiya::Storages::Abstract
|
8
|
+
def self.storage
|
9
|
+
@storage ||= {}
|
10
|
+
end
|
11
|
+
|
12
|
+
def self.clear
|
13
|
+
@storage = {}
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.find(config={})
|
17
|
+
storage.keys
|
18
|
+
end
|
19
|
+
|
20
|
+
def initialize(*)
|
21
|
+
super
|
22
|
+
self.class.storage[application] ||= {}
|
23
|
+
end
|
24
|
+
|
25
|
+
def packages
|
26
|
+
self.class.storage[application].keys
|
27
|
+
end
|
28
|
+
|
29
|
+
def push(package)
|
30
|
+
raise TypeError, "package should be a kind of Mamiya::Package" unless package.kind_of?(Mamiya::Package)
|
31
|
+
raise NotBuilt, "package not built" unless package.exists?
|
32
|
+
self.class.storage[application] ||= {}
|
33
|
+
raise AlreadyExists if self.class.storage[application][package.name]
|
34
|
+
self.class.storage[application][package.name] = {package: package, config: config}
|
35
|
+
end
|
36
|
+
|
37
|
+
def fetch(package_name, destination)
|
38
|
+
self.class.storage[application] ||= {}
|
39
|
+
raise NotFound unless self.class.storage[application][package_name]
|
40
|
+
package_path = File.join(destination, "#{package_name}.tar.gz")
|
41
|
+
meta_path = File.join(destination, "#{package_name}.json")
|
42
|
+
|
43
|
+
if File.exists?(package_path) || File.exists?(meta_path)
|
44
|
+
raise AlreadyFetched
|
45
|
+
end
|
46
|
+
|
47
|
+
package = self.class.storage[application][package_name][:package]
|
48
|
+
FileUtils.cp package.path, package_path
|
49
|
+
FileUtils.cp package.meta_path, meta_path
|
50
|
+
return package
|
51
|
+
end
|
52
|
+
|
53
|
+
def meta(package_name)
|
54
|
+
package = self.class.storage[application][package_name]
|
55
|
+
return unless package
|
56
|
+
|
57
|
+
package[:package].meta
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
@@ -0,0 +1,127 @@
|
|
1
|
+
require 'mamiya/package'
|
2
|
+
require 'mamiya/storages/abstract'
|
3
|
+
require 'aws-sdk-core'
|
4
|
+
require 'json'
|
5
|
+
|
6
|
+
module Mamiya
|
7
|
+
module Storages
|
8
|
+
class S3 < Mamiya::Storages::Abstract
|
9
|
+
def self.find(config={})
|
10
|
+
s3 = initiate_s3_with_config(config)
|
11
|
+
Hash[s3.list_objects(bucket: config[:bucket], delimiter: '/').common_prefixes.map { |prefix|
|
12
|
+
app = prefix.prefix.gsub(%r{/$},'')
|
13
|
+
[app, self.new(config.merge(application: app))]
|
14
|
+
}]
|
15
|
+
end
|
16
|
+
|
17
|
+
def packages
|
18
|
+
s3.list_objects(bucket: @config[:bucket], delimiter: '/', prefix: "#{self.application}/").contents.map { |content|
|
19
|
+
content.key.sub(/\A#{Regexp.escape(self.application)}\//, '')
|
20
|
+
}.group_by { |key|
|
21
|
+
key.sub(Package::PATH_SUFFIXES,'')
|
22
|
+
}.select { |key, files|
|
23
|
+
files.find { |file| file.end_with?('.tar.gz') } && files.find { |file| file.end_with?('.json') }
|
24
|
+
}.keys
|
25
|
+
end
|
26
|
+
|
27
|
+
def push(package)
|
28
|
+
raise TypeError, "package should be a kind of Mamiya::Package" unless package.kind_of?(Mamiya::Package)
|
29
|
+
raise NotBuilt, "package not built" unless package.exists?
|
30
|
+
|
31
|
+
package_key, meta_key = package_and_meta_key_for(package.name)
|
32
|
+
|
33
|
+
[package_key, meta_key].each do |key|
|
34
|
+
raise AlreadyExists if key_exists_in_s3?(key)
|
35
|
+
end
|
36
|
+
|
37
|
+
open(package.path, 'rb') do |io|
|
38
|
+
s3.put_object(bucket: @config[:bucket], key: package_key, body: io)
|
39
|
+
end
|
40
|
+
open(package.meta_path, 'rb') do |io|
|
41
|
+
s3.put_object(bucket: @config[:bucket], key: meta_key, body: io)
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def fetch(package_name, dir)
|
46
|
+
package_key, meta_key = package_and_meta_key_for(package_name)
|
47
|
+
|
48
|
+
package_path = File.join(dir, File.basename(package_key))
|
49
|
+
meta_path = File.join(dir, File.basename(meta_key))
|
50
|
+
|
51
|
+
if File.exists?(package_path) || File.exists?(meta_path)
|
52
|
+
raise AlreadyFetched
|
53
|
+
end
|
54
|
+
|
55
|
+
open(package_path, 'wb+') do |io|
|
56
|
+
s3.get_object({bucket: @config[:bucket], key: package_key}, target: io)
|
57
|
+
end
|
58
|
+
open(meta_path, 'wb+') do |io|
|
59
|
+
s3.get_object({bucket: @config[:bucket], key: meta_key}, target: io)
|
60
|
+
end
|
61
|
+
|
62
|
+
return Mamiya::Package.new(package_path)
|
63
|
+
rescue Aws::S3::Errors::NoSuchKey
|
64
|
+
File.unlink package_path if package_path && File.exists?(package_path)
|
65
|
+
File.unlink meta_path if meta_path && File.exists?(meta_path)
|
66
|
+
|
67
|
+
raise NotFound
|
68
|
+
rescue Exception => e
|
69
|
+
File.unlink package_path if package_path && File.exists?(package_path)
|
70
|
+
File.unlink meta_path if meta_path && File.exists?(meta_path)
|
71
|
+
|
72
|
+
raise e
|
73
|
+
end
|
74
|
+
|
75
|
+
def meta(package_name)
|
76
|
+
_, meta_key = package_and_meta_key_for(package_name)
|
77
|
+
JSON.parse(s3.get_object(bucket: @config[:bucket], key: meta_key).body.string)
|
78
|
+
rescue Aws::S3::Errors::NoSuchKey
|
79
|
+
return nil
|
80
|
+
end
|
81
|
+
|
82
|
+
def remove(package_name)
|
83
|
+
package_key, meta_key = package_and_meta_key_for(package_name)
|
84
|
+
|
85
|
+
objs_to_delete = [package_key, meta_key].map { |key|
|
86
|
+
if key_exists_in_s3?(key)
|
87
|
+
{key: key}
|
88
|
+
else
|
89
|
+
nil
|
90
|
+
end
|
91
|
+
}.compact
|
92
|
+
raise NotFound if objs_to_delete.empty?
|
93
|
+
|
94
|
+
s3.delete_objects(bucket: @config[:bucket], objects: objs_to_delete)
|
95
|
+
end
|
96
|
+
|
97
|
+
def self.initiate_s3_with_config(config) # :nodoc:
|
98
|
+
s3_config = config.dup
|
99
|
+
s3_config.delete(:bucket)
|
100
|
+
s3_config.delete(:application)
|
101
|
+
s3_config.delete(:type)
|
102
|
+
Aws::S3.new(s3_config)
|
103
|
+
end
|
104
|
+
|
105
|
+
private
|
106
|
+
|
107
|
+
def s3
|
108
|
+
@s3 ||= self.class.initiate_s3_with_config(@config)
|
109
|
+
end
|
110
|
+
|
111
|
+
def package_and_meta_key_for(package_name)
|
112
|
+
package_name = package_name.sub(/\.(?:tar\.gz|json)\z/, '')
|
113
|
+
["#{self.application}/#{package_name}.tar.gz", "#{self.application}/#{package_name}.json"]
|
114
|
+
end
|
115
|
+
|
116
|
+
def key_exists_in_s3?(key)
|
117
|
+
begin
|
118
|
+
if s3.head_object(bucket: @config[:bucket], key: key)
|
119
|
+
true
|
120
|
+
end
|
121
|
+
rescue Aws::S3::Errors::NotFound
|
122
|
+
false
|
123
|
+
end
|
124
|
+
end
|
125
|
+
end
|
126
|
+
end
|
127
|
+
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
module Mamiya
|
2
|
+
module Util
|
3
|
+
module LabelMatcher
|
4
|
+
def match?(*expressions)
|
5
|
+
labels = self.labels()
|
6
|
+
|
7
|
+
if expressions.all? { |_| _.kind_of?(Symbol) || _.kind_of?(String) }
|
8
|
+
return self.match?(expressions)
|
9
|
+
end
|
10
|
+
|
11
|
+
expressions.any? do |expression|
|
12
|
+
case expression
|
13
|
+
when Symbol, String
|
14
|
+
labels.include?(expression)
|
15
|
+
when Array
|
16
|
+
if expression.any? { |_| _.kind_of?(Array) }
|
17
|
+
self.match?(*expression)
|
18
|
+
else
|
19
|
+
expression.all? { |_| labels.include?(_) }
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
alias matches? match?
|
26
|
+
|
27
|
+
class Simple
|
28
|
+
def initialize(*labels)
|
29
|
+
@labels = labels.flatten
|
30
|
+
end
|
31
|
+
|
32
|
+
attr_reader :labels
|
33
|
+
|
34
|
+
include LabelMatcher
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|