mamiya 0.0.1.alpha2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +17 -0
  3. data/.rspec +2 -0
  4. data/.travis.yml +16 -0
  5. data/Gemfile +8 -0
  6. data/LICENSE.txt +22 -0
  7. data/README.md +43 -0
  8. data/Rakefile +6 -0
  9. data/bin/mamiya +17 -0
  10. data/config.example.yml +11 -0
  11. data/docs/sequences/deploy.png +0 -0
  12. data/docs/sequences/deploy.uml +58 -0
  13. data/example.rb +74 -0
  14. data/lib/mamiya.rb +5 -0
  15. data/lib/mamiya/agent.rb +181 -0
  16. data/lib/mamiya/agent/actions.rb +12 -0
  17. data/lib/mamiya/agent/fetcher.rb +137 -0
  18. data/lib/mamiya/agent/handlers/abstract.rb +20 -0
  19. data/lib/mamiya/agent/handlers/fetch.rb +68 -0
  20. data/lib/mamiya/cli.rb +322 -0
  21. data/lib/mamiya/cli/client.rb +172 -0
  22. data/lib/mamiya/config.rb +57 -0
  23. data/lib/mamiya/dsl.rb +192 -0
  24. data/lib/mamiya/helpers/git.rb +75 -0
  25. data/lib/mamiya/logger.rb +190 -0
  26. data/lib/mamiya/master.rb +118 -0
  27. data/lib/mamiya/master/agent_monitor.rb +146 -0
  28. data/lib/mamiya/master/agent_monitor_handlers.rb +44 -0
  29. data/lib/mamiya/master/web.rb +148 -0
  30. data/lib/mamiya/package.rb +122 -0
  31. data/lib/mamiya/script.rb +117 -0
  32. data/lib/mamiya/steps/abstract.rb +19 -0
  33. data/lib/mamiya/steps/build.rb +72 -0
  34. data/lib/mamiya/steps/extract.rb +26 -0
  35. data/lib/mamiya/steps/fetch.rb +24 -0
  36. data/lib/mamiya/steps/push.rb +34 -0
  37. data/lib/mamiya/storages.rb +17 -0
  38. data/lib/mamiya/storages/abstract.rb +48 -0
  39. data/lib/mamiya/storages/mock.rb +61 -0
  40. data/lib/mamiya/storages/s3.rb +127 -0
  41. data/lib/mamiya/util/label_matcher.rb +38 -0
  42. data/lib/mamiya/version.rb +3 -0
  43. data/mamiya.gemspec +35 -0
  44. data/misc/logger_test.rb +12 -0
  45. data/spec/agent/actions_spec.rb +37 -0
  46. data/spec/agent/fetcher_spec.rb +199 -0
  47. data/spec/agent/handlers/fetch_spec.rb +121 -0
  48. data/spec/agent_spec.rb +255 -0
  49. data/spec/config_spec.rb +50 -0
  50. data/spec/dsl_spec.rb +291 -0
  51. data/spec/fixtures/dsl_test_load.rb +1 -0
  52. data/spec/fixtures/dsl_test_use.rb +1 -0
  53. data/spec/fixtures/helpers/foo.rb +1 -0
  54. data/spec/fixtures/test-package-source/.mamiya.meta.json +1 -0
  55. data/spec/fixtures/test-package-source/greeting +1 -0
  56. data/spec/fixtures/test-package.tar.gz +0 -0
  57. data/spec/fixtures/test.yml +4 -0
  58. data/spec/logger_spec.rb +68 -0
  59. data/spec/master/agent_monitor_spec.rb +269 -0
  60. data/spec/master/web_spec.rb +121 -0
  61. data/spec/master_spec.rb +94 -0
  62. data/spec/package_spec.rb +394 -0
  63. data/spec/script_spec.rb +78 -0
  64. data/spec/spec_helper.rb +38 -0
  65. data/spec/steps/build_spec.rb +261 -0
  66. data/spec/steps/extract_spec.rb +68 -0
  67. data/spec/steps/fetch_spec.rb +96 -0
  68. data/spec/steps/push_spec.rb +73 -0
  69. data/spec/storages/abstract_spec.rb +22 -0
  70. data/spec/storages/s3_spec.rb +342 -0
  71. data/spec/storages_spec.rb +33 -0
  72. data/spec/support/dummy_serf.rb +70 -0
  73. data/spec/util/label_matcher_spec.rb +85 -0
  74. metadata +272 -0
@@ -0,0 +1,44 @@
1
+ require 'mamiya/master'
2
+
3
+ module Mamiya
4
+ class Master
5
+ module AgentMonitorHandlers
6
+ def fetch_result__ack(status, payload, event)
7
+ status['fetcher'] ||= {}
8
+ status['fetcher']['pending'] = payload['pending']
9
+ end
10
+
11
+ def fetch_result__start(status, payload, event)
12
+ status['fetcher'] ||= {}
13
+ status['fetcher']['fetching'] = [payload['application'], payload['package']]
14
+ end
15
+
16
+ def fetch_result__error(status, payload, event)
17
+ status['fetcher'] ||= {}
18
+
19
+ if status['fetcher']['fetching'] == [payload['application'], payload['package']]
20
+ status['fetcher']['fetching'] = nil
21
+ end
22
+ end
23
+
24
+ def fetch_result__success(status, payload, event)
25
+ status['fetcher'] ||= {}
26
+
27
+ if status['fetcher']['fetching'] == [payload['application'], payload['package']]
28
+ status['fetcher']['fetching'] = nil
29
+ end
30
+
31
+ status['packages'] ||= {}
32
+ status['packages'][payload['application']] ||= []
33
+ status['packages'][payload['application']] << payload['package']
34
+ end
35
+
36
+ def fetch_result__remove(status, payload, event)
37
+ status['packages'] ||= {}
38
+ packages = status['packages'][payload['application']]
39
+ packages.delete(payload['package']) if packages
40
+ end
41
+
42
+ end
43
+ end
44
+ end
@@ -0,0 +1,148 @@
1
+ require 'mamiya/version'
2
+ require 'mamiya/agent'
3
+ require 'sinatra/base'
4
+ require 'json'
5
+
6
+ module Mamiya
7
+ class Master < Agent
8
+ class Web < Sinatra::Base
9
+ helpers do
10
+ def master
11
+ env['mamiya.master']
12
+ end
13
+
14
+ def agent_monitor
15
+ master.agent_monitor
16
+ end
17
+
18
+ def storage(app)
19
+ master.storage(app)
20
+ end
21
+ end
22
+
23
+ get '/' do
24
+ "mamiya v#{Mamiya::VERSION}"
25
+ end
26
+
27
+ get '/packages/:application' do
28
+ content_type :json
29
+ packages = storage(params[:application]).packages
30
+
31
+ status 404 if packages.empty?
32
+ {packages: packages}.to_json
33
+ end
34
+
35
+ get '/packages/:application/:package' do
36
+ content_type :json
37
+
38
+ meta = storage(params[:application]).meta(params[:package])
39
+
40
+ if meta
41
+ {application: params[:application], name: params[:package],
42
+ meta: meta}.to_json
43
+ else
44
+ status 404
45
+ {}.to_json
46
+ end
47
+ end
48
+
49
+ get '/packages' do
50
+ content_type :json
51
+ applications = master.applications
52
+
53
+ {applications: applications}.to_json
54
+ end
55
+
56
+ post '/packages/:application/:package/distribute' do
57
+ # TODO: filter with label
58
+ if storage(params[:application]).meta(params[:package])
59
+ status 204
60
+ master.distribute(params[:application], params[:package])
61
+ else
62
+ status 404
63
+ content_type :json
64
+ {error: 'not found'}.to_json
65
+ end
66
+ end
67
+
68
+ get '/packages/:application/:package/distribution' do
69
+ # TODO: filter with label
70
+ content_type :json
71
+ meta = storage(params[:application]).meta(params[:package])
72
+ unless meta
73
+ status 404
74
+ next {error: 'not found'}.to_json
75
+ end
76
+
77
+ result = {application: params[:application], package: params[:package], distributed: [], not_distributed: []}
78
+ statuses = agent_monitor.statuses
79
+
80
+ statuses.each do |name, status|
81
+ next if status["master"]
82
+ if status["packages"] && status["packages"][params[:application]] &&
83
+ status["packages"][params[:application]].include?(params[:package])
84
+
85
+ result[:distributed] << name
86
+ else
87
+ result[:not_distributed] << name
88
+ end
89
+ end
90
+
91
+ result[:distributed_count] = result[:distributed].size
92
+ result[:not_distributed_count] = result[:not_distributed].size
93
+
94
+ result.to_json
95
+ end
96
+
97
+
98
+ get '/agents' do
99
+ statuses = agent_monitor.statuses
100
+ members = agent_monitor.agents
101
+ failed_agents = agent_monitor.failed_agents
102
+
103
+ agents = {}
104
+ members.each do |name, status|
105
+ agents[name] ||= {}
106
+ agents[name]["membership"] = status
107
+ end
108
+
109
+ statuses.each do |name, status|
110
+ if status["master"]
111
+ agents.delete name
112
+ next
113
+ end
114
+
115
+ agents[name] ||= {}
116
+ agents[name]["status"] = status
117
+ end
118
+
119
+
120
+ content_type :json
121
+
122
+ {
123
+ agents: agents,
124
+ failed_agents: failed_agents,
125
+ }.to_json
126
+ end
127
+
128
+ get '/agents/:name' do
129
+ content_type :json
130
+
131
+ status = agent_monitor.statuses[params[:name]]
132
+ membership = agent_monitor.agents[params[:name]]
133
+
134
+ if status || membership
135
+ {name: params[:name], status: status, membership: membership}.to_json
136
+ else
137
+ status 404
138
+ {error: 'not found'}.to_json
139
+ end
140
+ end
141
+
142
+ post '/agents/refresh' do
143
+ agent_monitor.refresh
144
+ status 204
145
+ end
146
+ end
147
+ end
148
+ end
@@ -0,0 +1,122 @@
1
+ require 'mamiya/logger'
2
+
3
+ require 'pathname'
4
+ require 'digest/sha2'
5
+ require 'json'
6
+
7
+ module Mamiya
8
+ class Package
9
+ class NotExists < Exception; end
10
+ class InternalError < Exception; end
11
+ PATH_SUFFIXES = /\.(?:tar\.gz|json)\z/
12
+
13
+ def initialize(path)
14
+ @path_without_ext = Pathname.new(path.sub(PATH_SUFFIXES, ''))
15
+ @meta_loaded_from_file = false
16
+ @loaded_meta = nil
17
+ meta # load
18
+ end
19
+
20
+ attr_reader :path
21
+ attr_writer :meta
22
+
23
+ def name
24
+ meta['name'] || @path_without_ext.basename.to_s
25
+ end
26
+
27
+ def path
28
+ Pathname.new(@path_without_ext.to_s + '.tar.gz')
29
+ end
30
+
31
+ def meta_path
32
+ Pathname.new(@path_without_ext.to_s + '.json')
33
+ end
34
+
35
+ def meta
36
+ if !@meta_loaded_from_file && meta_path.exist?
37
+ @meta_loaded_from_file = true
38
+ loaded_meta = load_meta()
39
+ if @loaded_meta == @meta
40
+ @loaded_meta = loaded_meta
41
+ @meta = load_meta()
42
+ end
43
+ end
44
+ @meta ||= {}
45
+ end
46
+
47
+ def application
48
+ meta['application'] || meta[:application]
49
+ end
50
+
51
+ def checksum
52
+ return nil unless exist?
53
+ Digest::SHA2.file(path).hexdigest
54
+ end
55
+
56
+ def valid?
57
+ raise NotExists, 'package not exist' unless exist?
58
+ raise NotExists, 'meta not exist' unless meta_path.exist?
59
+ !meta['checksum'] || checksum == meta['checksum']
60
+ end
61
+
62
+ def exists?
63
+ path.exist?
64
+ end
65
+ alias exist? exists?
66
+
67
+ def build!(build_dir, exclude_from_package: [], dereference_symlinks: false, package_under: nil, logger: Mamiya::Logger.new)
68
+ logger = logger['Package']
69
+
70
+ exclude_from_package.push('.svn', '.git').uniq!
71
+
72
+ build_dir = Pathname.new(build_dir)
73
+ build_dir += package_under if package_under
74
+ meta_in_build = build_dir.join('.mamiya.meta.json')
75
+
76
+ meta['name'] = self.name
77
+ File.write meta_in_build, self.meta.to_json
78
+
79
+ Dir.chdir(build_dir) do
80
+ excludes = exclude_from_package.flat_map { |exclude| ['--exclude', exclude] }
81
+ dereference = dereference_symlinks ? ['-h'] : []
82
+
83
+ cmd = ["tar", "czf", self.path.to_s,
84
+ *dereference,
85
+ *excludes,
86
+ "."]
87
+
88
+ logger.debug "$ #{cmd.join(' ')}"
89
+ result = system(*cmd)
90
+ raise InternalError, "failed to run: #{cmd.inspect}" unless result
91
+ end
92
+
93
+ checksum = self.checksum()
94
+ raise InternalError, 'checksum should not be nil after package built' unless checksum
95
+ meta['checksum'] = checksum
96
+
97
+ File.write meta_path, self.meta.to_json
98
+ nil
99
+ ensure
100
+ if meta_in_build && meta_in_build.exist?
101
+ meta_in_build.delete()
102
+ end
103
+ end
104
+
105
+ def extract_onto!(destination)
106
+ raise NotExists unless exist?
107
+ Dir.mkdir(destination) unless File.directory?(destination)
108
+
109
+ cmd = ["tar", "xf", path.to_s, "-C", destination.to_s]
110
+ result = system(*cmd)
111
+ raise InternalError, "Failed to run: #{cmd.inspect}" unless result
112
+
113
+ nil
114
+ end
115
+
116
+ private
117
+
118
+ def load_meta
119
+ meta_path.exist? && JSON.parse(meta_path.read)
120
+ end
121
+ end
122
+ end
@@ -0,0 +1,117 @@
1
+ require 'mamiya/dsl'
2
+ require 'mamiya/logger'
3
+
4
+ require 'shellwords'
5
+
6
+ module Mamiya
7
+ class Script < DSL
8
+ class CommandFailed < Exception; end
9
+
10
+ add_hook :before_build
11
+ add_hook :prepare_build
12
+ add_hook :build
13
+ add_hook :after_build
14
+
15
+ add_hook :before_distribute
16
+ add_hook :after_distribute
17
+
18
+ add_hook :before_prepare
19
+ add_hook :prepare
20
+ add_hook :after_prepare
21
+
22
+ add_hook :before_finalize
23
+ add_hook :finalize
24
+ add_hook :after_finalize
25
+
26
+ add_hook :before_rollback
27
+ add_hook :rollback
28
+ add_hook :after_rollback
29
+
30
+ add_hook :package_name, chain: true
31
+ add_hook :package_meta, chain: true
32
+
33
+ set_default :application, nil
34
+ set_default :repository, nil
35
+ set_default :ref, nil
36
+
37
+ set_default :discover_servers, true
38
+ set_default :on_client_failure, :error # error, warn, ignore
39
+
40
+ set_default :build_from, nil
41
+ set_default :build_to, nil
42
+ set_default :package_under, nil
43
+ set_default :exclude_from_package, []
44
+ set_default :dereference_symlinks, true
45
+
46
+ set_default :package_to, nil
47
+
48
+ # TODO: use variable in config.yml
49
+ set_default :deploy_to, nil
50
+ set_default :prepare_to, nil
51
+
52
+ set_default :logger, Mamiya::Logger.new(outputs: [])
53
+
54
+ set_default :skip_prepare_build, false
55
+
56
+ def run(*args, allow_failure: false)
57
+ # TODO: Stop when fail
58
+ actual = -> do
59
+ logger = self.logger['RUN']
60
+
61
+ logger.info("$ #{args.shelljoin}")
62
+
63
+ err_r, err_w = IO.pipe
64
+ out_r, out_w = IO.pipe
65
+
66
+ pid = spawn(*args, out: out_w, err: err_w)
67
+
68
+ [out_w, err_w].each(&:close)
69
+
70
+ buf = ""
71
+
72
+ ths = {:debug => out_r, :warn => err_r}.map do |severity, io|
73
+ Thread.new {
74
+ until io.eof?
75
+ str = io.gets
76
+ logger.__send__(severity, str.chomp)
77
+ buf << str
78
+ end
79
+ }.tap { |_| _.abort_on_exception = true }
80
+ end
81
+
82
+ pid, status = Process.waitpid2(pid)
83
+
84
+ begin
85
+ timeout(3) { ths.each(&:join) }
86
+ rescue Timeout::Error
87
+ end
88
+ ths.each { |_| _.alive? && _.kill }
89
+
90
+ [out_r, err_r].each(&:close)
91
+
92
+ unless allow_failure || status.success?
93
+ raise CommandFailed,
94
+ "Excecution failed (" \
95
+ "status=#{status.exitstatus}" \
96
+ " pid=#{status.pid}" \
97
+ "#{status.signaled? ? "termsig=#{status.termsig.inspect} stopsig=#{status.stopsig.inspect}" : nil}" \
98
+ "#{status.stopped? ? " stopped" : nil}" \
99
+ "): #{args.inspect}"
100
+ end
101
+
102
+ buf
103
+ end
104
+
105
+ if defined? Bundler
106
+ Bundler.with_clean_env(&actual)
107
+ else
108
+ actual.call
109
+ end
110
+ end
111
+
112
+ def cd(*args)
113
+ logger.info "$ cd #{args[0]}"
114
+ Dir.chdir *args
115
+ end
116
+ end
117
+ end
@@ -0,0 +1,19 @@
1
+ require 'mamiya/config'
2
+ require 'mamiya/script'
3
+ require 'mamiya/logger'
4
+
5
+ module Mamiya
6
+ module Steps
7
+ class Abstract
8
+ def initialize(script: Mamiya::Script.new, config: Mamiya::Config.new, logger: Mamiya::Logger.new, **options)
9
+ @script, @config, @options = script, config, options
10
+ @logger = logger[self.class.name.sub(/^Mamiya::Steps::/,'')]
11
+ end
12
+
13
+ attr_reader :script, :config, :options, :logger
14
+
15
+ def run!
16
+ end
17
+ end
18
+ end
19
+ end