mamiya 0.0.1.alpha2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +17 -0
- data/.rspec +2 -0
- data/.travis.yml +16 -0
- data/Gemfile +8 -0
- data/LICENSE.txt +22 -0
- data/README.md +43 -0
- data/Rakefile +6 -0
- data/bin/mamiya +17 -0
- data/config.example.yml +11 -0
- data/docs/sequences/deploy.png +0 -0
- data/docs/sequences/deploy.uml +58 -0
- data/example.rb +74 -0
- data/lib/mamiya.rb +5 -0
- data/lib/mamiya/agent.rb +181 -0
- data/lib/mamiya/agent/actions.rb +12 -0
- data/lib/mamiya/agent/fetcher.rb +137 -0
- data/lib/mamiya/agent/handlers/abstract.rb +20 -0
- data/lib/mamiya/agent/handlers/fetch.rb +68 -0
- data/lib/mamiya/cli.rb +322 -0
- data/lib/mamiya/cli/client.rb +172 -0
- data/lib/mamiya/config.rb +57 -0
- data/lib/mamiya/dsl.rb +192 -0
- data/lib/mamiya/helpers/git.rb +75 -0
- data/lib/mamiya/logger.rb +190 -0
- data/lib/mamiya/master.rb +118 -0
- data/lib/mamiya/master/agent_monitor.rb +146 -0
- data/lib/mamiya/master/agent_monitor_handlers.rb +44 -0
- data/lib/mamiya/master/web.rb +148 -0
- data/lib/mamiya/package.rb +122 -0
- data/lib/mamiya/script.rb +117 -0
- data/lib/mamiya/steps/abstract.rb +19 -0
- data/lib/mamiya/steps/build.rb +72 -0
- data/lib/mamiya/steps/extract.rb +26 -0
- data/lib/mamiya/steps/fetch.rb +24 -0
- data/lib/mamiya/steps/push.rb +34 -0
- data/lib/mamiya/storages.rb +17 -0
- data/lib/mamiya/storages/abstract.rb +48 -0
- data/lib/mamiya/storages/mock.rb +61 -0
- data/lib/mamiya/storages/s3.rb +127 -0
- data/lib/mamiya/util/label_matcher.rb +38 -0
- data/lib/mamiya/version.rb +3 -0
- data/mamiya.gemspec +35 -0
- data/misc/logger_test.rb +12 -0
- data/spec/agent/actions_spec.rb +37 -0
- data/spec/agent/fetcher_spec.rb +199 -0
- data/spec/agent/handlers/fetch_spec.rb +121 -0
- data/spec/agent_spec.rb +255 -0
- data/spec/config_spec.rb +50 -0
- data/spec/dsl_spec.rb +291 -0
- data/spec/fixtures/dsl_test_load.rb +1 -0
- data/spec/fixtures/dsl_test_use.rb +1 -0
- data/spec/fixtures/helpers/foo.rb +1 -0
- data/spec/fixtures/test-package-source/.mamiya.meta.json +1 -0
- data/spec/fixtures/test-package-source/greeting +1 -0
- data/spec/fixtures/test-package.tar.gz +0 -0
- data/spec/fixtures/test.yml +4 -0
- data/spec/logger_spec.rb +68 -0
- data/spec/master/agent_monitor_spec.rb +269 -0
- data/spec/master/web_spec.rb +121 -0
- data/spec/master_spec.rb +94 -0
- data/spec/package_spec.rb +394 -0
- data/spec/script_spec.rb +78 -0
- data/spec/spec_helper.rb +38 -0
- data/spec/steps/build_spec.rb +261 -0
- data/spec/steps/extract_spec.rb +68 -0
- data/spec/steps/fetch_spec.rb +96 -0
- data/spec/steps/push_spec.rb +73 -0
- data/spec/storages/abstract_spec.rb +22 -0
- data/spec/storages/s3_spec.rb +342 -0
- data/spec/storages_spec.rb +33 -0
- data/spec/support/dummy_serf.rb +70 -0
- data/spec/util/label_matcher_spec.rb +85 -0
- metadata +272 -0
@@ -0,0 +1,44 @@
|
|
1
|
+
require 'mamiya/master'
|
2
|
+
|
3
|
+
module Mamiya
|
4
|
+
class Master
|
5
|
+
module AgentMonitorHandlers
|
6
|
+
def fetch_result__ack(status, payload, event)
|
7
|
+
status['fetcher'] ||= {}
|
8
|
+
status['fetcher']['pending'] = payload['pending']
|
9
|
+
end
|
10
|
+
|
11
|
+
def fetch_result__start(status, payload, event)
|
12
|
+
status['fetcher'] ||= {}
|
13
|
+
status['fetcher']['fetching'] = [payload['application'], payload['package']]
|
14
|
+
end
|
15
|
+
|
16
|
+
def fetch_result__error(status, payload, event)
|
17
|
+
status['fetcher'] ||= {}
|
18
|
+
|
19
|
+
if status['fetcher']['fetching'] == [payload['application'], payload['package']]
|
20
|
+
status['fetcher']['fetching'] = nil
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
def fetch_result__success(status, payload, event)
|
25
|
+
status['fetcher'] ||= {}
|
26
|
+
|
27
|
+
if status['fetcher']['fetching'] == [payload['application'], payload['package']]
|
28
|
+
status['fetcher']['fetching'] = nil
|
29
|
+
end
|
30
|
+
|
31
|
+
status['packages'] ||= {}
|
32
|
+
status['packages'][payload['application']] ||= []
|
33
|
+
status['packages'][payload['application']] << payload['package']
|
34
|
+
end
|
35
|
+
|
36
|
+
def fetch_result__remove(status, payload, event)
|
37
|
+
status['packages'] ||= {}
|
38
|
+
packages = status['packages'][payload['application']]
|
39
|
+
packages.delete(payload['package']) if packages
|
40
|
+
end
|
41
|
+
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
@@ -0,0 +1,148 @@
|
|
1
|
+
require 'mamiya/version'
|
2
|
+
require 'mamiya/agent'
|
3
|
+
require 'sinatra/base'
|
4
|
+
require 'json'
|
5
|
+
|
6
|
+
module Mamiya
|
7
|
+
class Master < Agent
|
8
|
+
class Web < Sinatra::Base
|
9
|
+
helpers do
|
10
|
+
def master
|
11
|
+
env['mamiya.master']
|
12
|
+
end
|
13
|
+
|
14
|
+
def agent_monitor
|
15
|
+
master.agent_monitor
|
16
|
+
end
|
17
|
+
|
18
|
+
def storage(app)
|
19
|
+
master.storage(app)
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
get '/' do
|
24
|
+
"mamiya v#{Mamiya::VERSION}"
|
25
|
+
end
|
26
|
+
|
27
|
+
get '/packages/:application' do
|
28
|
+
content_type :json
|
29
|
+
packages = storage(params[:application]).packages
|
30
|
+
|
31
|
+
status 404 if packages.empty?
|
32
|
+
{packages: packages}.to_json
|
33
|
+
end
|
34
|
+
|
35
|
+
get '/packages/:application/:package' do
|
36
|
+
content_type :json
|
37
|
+
|
38
|
+
meta = storage(params[:application]).meta(params[:package])
|
39
|
+
|
40
|
+
if meta
|
41
|
+
{application: params[:application], name: params[:package],
|
42
|
+
meta: meta}.to_json
|
43
|
+
else
|
44
|
+
status 404
|
45
|
+
{}.to_json
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
get '/packages' do
|
50
|
+
content_type :json
|
51
|
+
applications = master.applications
|
52
|
+
|
53
|
+
{applications: applications}.to_json
|
54
|
+
end
|
55
|
+
|
56
|
+
post '/packages/:application/:package/distribute' do
|
57
|
+
# TODO: filter with label
|
58
|
+
if storage(params[:application]).meta(params[:package])
|
59
|
+
status 204
|
60
|
+
master.distribute(params[:application], params[:package])
|
61
|
+
else
|
62
|
+
status 404
|
63
|
+
content_type :json
|
64
|
+
{error: 'not found'}.to_json
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
get '/packages/:application/:package/distribution' do
|
69
|
+
# TODO: filter with label
|
70
|
+
content_type :json
|
71
|
+
meta = storage(params[:application]).meta(params[:package])
|
72
|
+
unless meta
|
73
|
+
status 404
|
74
|
+
next {error: 'not found'}.to_json
|
75
|
+
end
|
76
|
+
|
77
|
+
result = {application: params[:application], package: params[:package], distributed: [], not_distributed: []}
|
78
|
+
statuses = agent_monitor.statuses
|
79
|
+
|
80
|
+
statuses.each do |name, status|
|
81
|
+
next if status["master"]
|
82
|
+
if status["packages"] && status["packages"][params[:application]] &&
|
83
|
+
status["packages"][params[:application]].include?(params[:package])
|
84
|
+
|
85
|
+
result[:distributed] << name
|
86
|
+
else
|
87
|
+
result[:not_distributed] << name
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
result[:distributed_count] = result[:distributed].size
|
92
|
+
result[:not_distributed_count] = result[:not_distributed].size
|
93
|
+
|
94
|
+
result.to_json
|
95
|
+
end
|
96
|
+
|
97
|
+
|
98
|
+
get '/agents' do
|
99
|
+
statuses = agent_monitor.statuses
|
100
|
+
members = agent_monitor.agents
|
101
|
+
failed_agents = agent_monitor.failed_agents
|
102
|
+
|
103
|
+
agents = {}
|
104
|
+
members.each do |name, status|
|
105
|
+
agents[name] ||= {}
|
106
|
+
agents[name]["membership"] = status
|
107
|
+
end
|
108
|
+
|
109
|
+
statuses.each do |name, status|
|
110
|
+
if status["master"]
|
111
|
+
agents.delete name
|
112
|
+
next
|
113
|
+
end
|
114
|
+
|
115
|
+
agents[name] ||= {}
|
116
|
+
agents[name]["status"] = status
|
117
|
+
end
|
118
|
+
|
119
|
+
|
120
|
+
content_type :json
|
121
|
+
|
122
|
+
{
|
123
|
+
agents: agents,
|
124
|
+
failed_agents: failed_agents,
|
125
|
+
}.to_json
|
126
|
+
end
|
127
|
+
|
128
|
+
get '/agents/:name' do
|
129
|
+
content_type :json
|
130
|
+
|
131
|
+
status = agent_monitor.statuses[params[:name]]
|
132
|
+
membership = agent_monitor.agents[params[:name]]
|
133
|
+
|
134
|
+
if status || membership
|
135
|
+
{name: params[:name], status: status, membership: membership}.to_json
|
136
|
+
else
|
137
|
+
status 404
|
138
|
+
{error: 'not found'}.to_json
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
post '/agents/refresh' do
|
143
|
+
agent_monitor.refresh
|
144
|
+
status 204
|
145
|
+
end
|
146
|
+
end
|
147
|
+
end
|
148
|
+
end
|
@@ -0,0 +1,122 @@
|
|
1
|
+
require 'mamiya/logger'
|
2
|
+
|
3
|
+
require 'pathname'
|
4
|
+
require 'digest/sha2'
|
5
|
+
require 'json'
|
6
|
+
|
7
|
+
module Mamiya
|
8
|
+
class Package
|
9
|
+
class NotExists < Exception; end
|
10
|
+
class InternalError < Exception; end
|
11
|
+
PATH_SUFFIXES = /\.(?:tar\.gz|json)\z/
|
12
|
+
|
13
|
+
def initialize(path)
|
14
|
+
@path_without_ext = Pathname.new(path.sub(PATH_SUFFIXES, ''))
|
15
|
+
@meta_loaded_from_file = false
|
16
|
+
@loaded_meta = nil
|
17
|
+
meta # load
|
18
|
+
end
|
19
|
+
|
20
|
+
attr_reader :path
|
21
|
+
attr_writer :meta
|
22
|
+
|
23
|
+
def name
|
24
|
+
meta['name'] || @path_without_ext.basename.to_s
|
25
|
+
end
|
26
|
+
|
27
|
+
def path
|
28
|
+
Pathname.new(@path_without_ext.to_s + '.tar.gz')
|
29
|
+
end
|
30
|
+
|
31
|
+
def meta_path
|
32
|
+
Pathname.new(@path_without_ext.to_s + '.json')
|
33
|
+
end
|
34
|
+
|
35
|
+
def meta
|
36
|
+
if !@meta_loaded_from_file && meta_path.exist?
|
37
|
+
@meta_loaded_from_file = true
|
38
|
+
loaded_meta = load_meta()
|
39
|
+
if @loaded_meta == @meta
|
40
|
+
@loaded_meta = loaded_meta
|
41
|
+
@meta = load_meta()
|
42
|
+
end
|
43
|
+
end
|
44
|
+
@meta ||= {}
|
45
|
+
end
|
46
|
+
|
47
|
+
def application
|
48
|
+
meta['application'] || meta[:application]
|
49
|
+
end
|
50
|
+
|
51
|
+
def checksum
|
52
|
+
return nil unless exist?
|
53
|
+
Digest::SHA2.file(path).hexdigest
|
54
|
+
end
|
55
|
+
|
56
|
+
def valid?
|
57
|
+
raise NotExists, 'package not exist' unless exist?
|
58
|
+
raise NotExists, 'meta not exist' unless meta_path.exist?
|
59
|
+
!meta['checksum'] || checksum == meta['checksum']
|
60
|
+
end
|
61
|
+
|
62
|
+
def exists?
|
63
|
+
path.exist?
|
64
|
+
end
|
65
|
+
alias exist? exists?
|
66
|
+
|
67
|
+
def build!(build_dir, exclude_from_package: [], dereference_symlinks: false, package_under: nil, logger: Mamiya::Logger.new)
|
68
|
+
logger = logger['Package']
|
69
|
+
|
70
|
+
exclude_from_package.push('.svn', '.git').uniq!
|
71
|
+
|
72
|
+
build_dir = Pathname.new(build_dir)
|
73
|
+
build_dir += package_under if package_under
|
74
|
+
meta_in_build = build_dir.join('.mamiya.meta.json')
|
75
|
+
|
76
|
+
meta['name'] = self.name
|
77
|
+
File.write meta_in_build, self.meta.to_json
|
78
|
+
|
79
|
+
Dir.chdir(build_dir) do
|
80
|
+
excludes = exclude_from_package.flat_map { |exclude| ['--exclude', exclude] }
|
81
|
+
dereference = dereference_symlinks ? ['-h'] : []
|
82
|
+
|
83
|
+
cmd = ["tar", "czf", self.path.to_s,
|
84
|
+
*dereference,
|
85
|
+
*excludes,
|
86
|
+
"."]
|
87
|
+
|
88
|
+
logger.debug "$ #{cmd.join(' ')}"
|
89
|
+
result = system(*cmd)
|
90
|
+
raise InternalError, "failed to run: #{cmd.inspect}" unless result
|
91
|
+
end
|
92
|
+
|
93
|
+
checksum = self.checksum()
|
94
|
+
raise InternalError, 'checksum should not be nil after package built' unless checksum
|
95
|
+
meta['checksum'] = checksum
|
96
|
+
|
97
|
+
File.write meta_path, self.meta.to_json
|
98
|
+
nil
|
99
|
+
ensure
|
100
|
+
if meta_in_build && meta_in_build.exist?
|
101
|
+
meta_in_build.delete()
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
def extract_onto!(destination)
|
106
|
+
raise NotExists unless exist?
|
107
|
+
Dir.mkdir(destination) unless File.directory?(destination)
|
108
|
+
|
109
|
+
cmd = ["tar", "xf", path.to_s, "-C", destination.to_s]
|
110
|
+
result = system(*cmd)
|
111
|
+
raise InternalError, "Failed to run: #{cmd.inspect}" unless result
|
112
|
+
|
113
|
+
nil
|
114
|
+
end
|
115
|
+
|
116
|
+
private
|
117
|
+
|
118
|
+
def load_meta
|
119
|
+
meta_path.exist? && JSON.parse(meta_path.read)
|
120
|
+
end
|
121
|
+
end
|
122
|
+
end
|
@@ -0,0 +1,117 @@
|
|
1
|
+
require 'mamiya/dsl'
|
2
|
+
require 'mamiya/logger'
|
3
|
+
|
4
|
+
require 'shellwords'
|
5
|
+
|
6
|
+
module Mamiya
|
7
|
+
class Script < DSL
|
8
|
+
class CommandFailed < Exception; end
|
9
|
+
|
10
|
+
add_hook :before_build
|
11
|
+
add_hook :prepare_build
|
12
|
+
add_hook :build
|
13
|
+
add_hook :after_build
|
14
|
+
|
15
|
+
add_hook :before_distribute
|
16
|
+
add_hook :after_distribute
|
17
|
+
|
18
|
+
add_hook :before_prepare
|
19
|
+
add_hook :prepare
|
20
|
+
add_hook :after_prepare
|
21
|
+
|
22
|
+
add_hook :before_finalize
|
23
|
+
add_hook :finalize
|
24
|
+
add_hook :after_finalize
|
25
|
+
|
26
|
+
add_hook :before_rollback
|
27
|
+
add_hook :rollback
|
28
|
+
add_hook :after_rollback
|
29
|
+
|
30
|
+
add_hook :package_name, chain: true
|
31
|
+
add_hook :package_meta, chain: true
|
32
|
+
|
33
|
+
set_default :application, nil
|
34
|
+
set_default :repository, nil
|
35
|
+
set_default :ref, nil
|
36
|
+
|
37
|
+
set_default :discover_servers, true
|
38
|
+
set_default :on_client_failure, :error # error, warn, ignore
|
39
|
+
|
40
|
+
set_default :build_from, nil
|
41
|
+
set_default :build_to, nil
|
42
|
+
set_default :package_under, nil
|
43
|
+
set_default :exclude_from_package, []
|
44
|
+
set_default :dereference_symlinks, true
|
45
|
+
|
46
|
+
set_default :package_to, nil
|
47
|
+
|
48
|
+
# TODO: use variable in config.yml
|
49
|
+
set_default :deploy_to, nil
|
50
|
+
set_default :prepare_to, nil
|
51
|
+
|
52
|
+
set_default :logger, Mamiya::Logger.new(outputs: [])
|
53
|
+
|
54
|
+
set_default :skip_prepare_build, false
|
55
|
+
|
56
|
+
def run(*args, allow_failure: false)
|
57
|
+
# TODO: Stop when fail
|
58
|
+
actual = -> do
|
59
|
+
logger = self.logger['RUN']
|
60
|
+
|
61
|
+
logger.info("$ #{args.shelljoin}")
|
62
|
+
|
63
|
+
err_r, err_w = IO.pipe
|
64
|
+
out_r, out_w = IO.pipe
|
65
|
+
|
66
|
+
pid = spawn(*args, out: out_w, err: err_w)
|
67
|
+
|
68
|
+
[out_w, err_w].each(&:close)
|
69
|
+
|
70
|
+
buf = ""
|
71
|
+
|
72
|
+
ths = {:debug => out_r, :warn => err_r}.map do |severity, io|
|
73
|
+
Thread.new {
|
74
|
+
until io.eof?
|
75
|
+
str = io.gets
|
76
|
+
logger.__send__(severity, str.chomp)
|
77
|
+
buf << str
|
78
|
+
end
|
79
|
+
}.tap { |_| _.abort_on_exception = true }
|
80
|
+
end
|
81
|
+
|
82
|
+
pid, status = Process.waitpid2(pid)
|
83
|
+
|
84
|
+
begin
|
85
|
+
timeout(3) { ths.each(&:join) }
|
86
|
+
rescue Timeout::Error
|
87
|
+
end
|
88
|
+
ths.each { |_| _.alive? && _.kill }
|
89
|
+
|
90
|
+
[out_r, err_r].each(&:close)
|
91
|
+
|
92
|
+
unless allow_failure || status.success?
|
93
|
+
raise CommandFailed,
|
94
|
+
"Excecution failed (" \
|
95
|
+
"status=#{status.exitstatus}" \
|
96
|
+
" pid=#{status.pid}" \
|
97
|
+
"#{status.signaled? ? "termsig=#{status.termsig.inspect} stopsig=#{status.stopsig.inspect}" : nil}" \
|
98
|
+
"#{status.stopped? ? " stopped" : nil}" \
|
99
|
+
"): #{args.inspect}"
|
100
|
+
end
|
101
|
+
|
102
|
+
buf
|
103
|
+
end
|
104
|
+
|
105
|
+
if defined? Bundler
|
106
|
+
Bundler.with_clean_env(&actual)
|
107
|
+
else
|
108
|
+
actual.call
|
109
|
+
end
|
110
|
+
end
|
111
|
+
|
112
|
+
def cd(*args)
|
113
|
+
logger.info "$ cd #{args[0]}"
|
114
|
+
Dir.chdir *args
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
require 'mamiya/config'
|
2
|
+
require 'mamiya/script'
|
3
|
+
require 'mamiya/logger'
|
4
|
+
|
5
|
+
module Mamiya
|
6
|
+
module Steps
|
7
|
+
class Abstract
|
8
|
+
def initialize(script: Mamiya::Script.new, config: Mamiya::Config.new, logger: Mamiya::Logger.new, **options)
|
9
|
+
@script, @config, @options = script, config, options
|
10
|
+
@logger = logger[self.class.name.sub(/^Mamiya::Steps::/,'')]
|
11
|
+
end
|
12
|
+
|
13
|
+
attr_reader :script, :config, :options, :logger
|
14
|
+
|
15
|
+
def run!
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|