mamiya 0.0.1.alpha2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +17 -0
- data/.rspec +2 -0
- data/.travis.yml +16 -0
- data/Gemfile +8 -0
- data/LICENSE.txt +22 -0
- data/README.md +43 -0
- data/Rakefile +6 -0
- data/bin/mamiya +17 -0
- data/config.example.yml +11 -0
- data/docs/sequences/deploy.png +0 -0
- data/docs/sequences/deploy.uml +58 -0
- data/example.rb +74 -0
- data/lib/mamiya.rb +5 -0
- data/lib/mamiya/agent.rb +181 -0
- data/lib/mamiya/agent/actions.rb +12 -0
- data/lib/mamiya/agent/fetcher.rb +137 -0
- data/lib/mamiya/agent/handlers/abstract.rb +20 -0
- data/lib/mamiya/agent/handlers/fetch.rb +68 -0
- data/lib/mamiya/cli.rb +322 -0
- data/lib/mamiya/cli/client.rb +172 -0
- data/lib/mamiya/config.rb +57 -0
- data/lib/mamiya/dsl.rb +192 -0
- data/lib/mamiya/helpers/git.rb +75 -0
- data/lib/mamiya/logger.rb +190 -0
- data/lib/mamiya/master.rb +118 -0
- data/lib/mamiya/master/agent_monitor.rb +146 -0
- data/lib/mamiya/master/agent_monitor_handlers.rb +44 -0
- data/lib/mamiya/master/web.rb +148 -0
- data/lib/mamiya/package.rb +122 -0
- data/lib/mamiya/script.rb +117 -0
- data/lib/mamiya/steps/abstract.rb +19 -0
- data/lib/mamiya/steps/build.rb +72 -0
- data/lib/mamiya/steps/extract.rb +26 -0
- data/lib/mamiya/steps/fetch.rb +24 -0
- data/lib/mamiya/steps/push.rb +34 -0
- data/lib/mamiya/storages.rb +17 -0
- data/lib/mamiya/storages/abstract.rb +48 -0
- data/lib/mamiya/storages/mock.rb +61 -0
- data/lib/mamiya/storages/s3.rb +127 -0
- data/lib/mamiya/util/label_matcher.rb +38 -0
- data/lib/mamiya/version.rb +3 -0
- data/mamiya.gemspec +35 -0
- data/misc/logger_test.rb +12 -0
- data/spec/agent/actions_spec.rb +37 -0
- data/spec/agent/fetcher_spec.rb +199 -0
- data/spec/agent/handlers/fetch_spec.rb +121 -0
- data/spec/agent_spec.rb +255 -0
- data/spec/config_spec.rb +50 -0
- data/spec/dsl_spec.rb +291 -0
- data/spec/fixtures/dsl_test_load.rb +1 -0
- data/spec/fixtures/dsl_test_use.rb +1 -0
- data/spec/fixtures/helpers/foo.rb +1 -0
- data/spec/fixtures/test-package-source/.mamiya.meta.json +1 -0
- data/spec/fixtures/test-package-source/greeting +1 -0
- data/spec/fixtures/test-package.tar.gz +0 -0
- data/spec/fixtures/test.yml +4 -0
- data/spec/logger_spec.rb +68 -0
- data/spec/master/agent_monitor_spec.rb +269 -0
- data/spec/master/web_spec.rb +121 -0
- data/spec/master_spec.rb +94 -0
- data/spec/package_spec.rb +394 -0
- data/spec/script_spec.rb +78 -0
- data/spec/spec_helper.rb +38 -0
- data/spec/steps/build_spec.rb +261 -0
- data/spec/steps/extract_spec.rb +68 -0
- data/spec/steps/fetch_spec.rb +96 -0
- data/spec/steps/push_spec.rb +73 -0
- data/spec/storages/abstract_spec.rb +22 -0
- data/spec/storages/s3_spec.rb +342 -0
- data/spec/storages_spec.rb +33 -0
- data/spec/support/dummy_serf.rb +70 -0
- data/spec/util/label_matcher_spec.rb +85 -0
- metadata +272 -0
@@ -0,0 +1,190 @@
|
|
1
|
+
require 'logger'
|
2
|
+
require 'forwardable'
|
3
|
+
require 'thread'
|
4
|
+
require 'term/ansicolor'
|
5
|
+
require 'time'
|
6
|
+
|
7
|
+
module Mamiya
|
8
|
+
class Logger
|
9
|
+
include ::Logger::Severity
|
10
|
+
extend Forwardable
|
11
|
+
|
12
|
+
def self.defaults
|
13
|
+
return @defaults if @defaults
|
14
|
+
if ENV["MAMIYA_LOG_LEVEL"]
|
15
|
+
level = ::Logger::Severity.const_get(ENV["MAMIYA_LOG_LEVEL"].upcase) rescue INFO
|
16
|
+
else
|
17
|
+
level = INFO
|
18
|
+
end
|
19
|
+
@defaults = {color: nil, outputs: [STDOUT], level: level}
|
20
|
+
end
|
21
|
+
|
22
|
+
def initialize(color: self.class.defaults[:color], outputs: self.class.defaults[:outputs], level: self.class.defaults[:level])
|
23
|
+
@logdev = LogDev.new(outputs)
|
24
|
+
@logger = ::Logger.new(@logdev)
|
25
|
+
@logger.level = level
|
26
|
+
@logger.formatter = method(:format)
|
27
|
+
|
28
|
+
@color = color.nil? ? @logdev.tty? : color
|
29
|
+
end
|
30
|
+
|
31
|
+
attr_accessor :color
|
32
|
+
def_delegators :@logger,
|
33
|
+
:<<, :add, :log,
|
34
|
+
:fatal, :error, :warn, :info, :debug,
|
35
|
+
:fatal?, :error?, :warn?, :info?, :debug?,
|
36
|
+
:level, :level=, :progname, :progname=,
|
37
|
+
:close
|
38
|
+
|
39
|
+
def_delegators :@logdev, :reopen
|
40
|
+
|
41
|
+
def add_output(*outputs)
|
42
|
+
@logdev.add(*outputs)
|
43
|
+
end
|
44
|
+
|
45
|
+
def remove_output(*outputs)
|
46
|
+
@logdev.remove(*outputs)
|
47
|
+
end
|
48
|
+
|
49
|
+
def with_additional_file(*outputs)
|
50
|
+
@logdev.add_output(*outputs)
|
51
|
+
|
52
|
+
yield
|
53
|
+
|
54
|
+
ensure
|
55
|
+
@logdev.remove_output(*outputs)
|
56
|
+
end
|
57
|
+
|
58
|
+
def [](progname)
|
59
|
+
self.dup.tap do |new_logger|
|
60
|
+
new_logger.instance_eval do
|
61
|
+
@logger = @logger.dup
|
62
|
+
@logger.progname = progname
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
private
|
68
|
+
|
69
|
+
def format(severity, time, progname, msg)
|
70
|
+
rseverity = " #{severity.rjust(5)} "
|
71
|
+
if @color
|
72
|
+
colored_severity = case severity
|
73
|
+
when 'ANY'.freeze
|
74
|
+
rseverity
|
75
|
+
when 'DEBUG'.freeze
|
76
|
+
Term::ANSIColor.on_black(rseverity)
|
77
|
+
when 'INFO'.freeze
|
78
|
+
Term::ANSIColor.on_blue(rseverity)
|
79
|
+
when 'WARN'.freeze
|
80
|
+
Term::ANSIColor.on_yellow(Term::ANSIColor.black(rseverity))
|
81
|
+
when 'ERROR'.freeze
|
82
|
+
Term::ANSIColor.on_magenta(rseverity)
|
83
|
+
when 'FATAL'.freeze
|
84
|
+
Term::ANSIColor.on_red(Term::ANSIColor.white(Term::ANSIColor.bold(rseverity)))
|
85
|
+
else
|
86
|
+
rseverity
|
87
|
+
end
|
88
|
+
else
|
89
|
+
colored_severity = "#{rseverity}|"
|
90
|
+
end
|
91
|
+
|
92
|
+
msg = "#{(progname && "[#{progname}] ")}#{msg}"
|
93
|
+
if @color
|
94
|
+
colored_msg = case severity
|
95
|
+
when 'DEBUG'.freeze
|
96
|
+
Term::ANSIColor.bright_black(msg)
|
97
|
+
when 'FATAL'.freeze
|
98
|
+
Term::ANSIColor.bold(msg)
|
99
|
+
else
|
100
|
+
msg
|
101
|
+
end
|
102
|
+
else
|
103
|
+
colored_msg = msg
|
104
|
+
end
|
105
|
+
|
106
|
+
formatted_time = time.strftime('%m/%d %H:%M:%S')
|
107
|
+
colored_time = @color ? Term::ANSIColor.bright_black(formatted_time) : formatted_time
|
108
|
+
|
109
|
+
"#{colored_time} " \
|
110
|
+
"#{colored_severity} " \
|
111
|
+
"#{colored_msg}" \
|
112
|
+
"\n"
|
113
|
+
end
|
114
|
+
|
115
|
+
class LogDev
|
116
|
+
def initialize(outputs)
|
117
|
+
@outputs = normalize_outputs(outputs)
|
118
|
+
@mutex = Mutex.new
|
119
|
+
end
|
120
|
+
|
121
|
+
def tty?
|
122
|
+
@outputs.all?(&:tty?)
|
123
|
+
end
|
124
|
+
|
125
|
+
def write(*args)
|
126
|
+
@outputs.each do |output|
|
127
|
+
output.write(*args) unless output.respond_to?(:closed?) && output.closed?
|
128
|
+
end
|
129
|
+
self
|
130
|
+
end
|
131
|
+
|
132
|
+
def close
|
133
|
+
@outputs.each do |output|
|
134
|
+
output.close unless output.respond_to?(:closed?) && output.closed?
|
135
|
+
end
|
136
|
+
self
|
137
|
+
end
|
138
|
+
|
139
|
+
def reopen
|
140
|
+
@outputs.select { |io| io.respond_to?(:path) }.each do |io|
|
141
|
+
sync = io.sync
|
142
|
+
io.reopen(io.path, 'a')
|
143
|
+
io.sync = sync
|
144
|
+
end
|
145
|
+
end
|
146
|
+
|
147
|
+
def add(*outputs)
|
148
|
+
@mutex.synchronize do
|
149
|
+
@outputs.push(*normalize_outputs(outputs))
|
150
|
+
end
|
151
|
+
self
|
152
|
+
end
|
153
|
+
|
154
|
+
def remove(*removing_outputs)
|
155
|
+
@mutex.synchronize do
|
156
|
+
removing_outputs.each do |removing|
|
157
|
+
case removing
|
158
|
+
when File
|
159
|
+
@outputs.reject! { |out| out.kind_of?(File) && out.path == removing.path }
|
160
|
+
when IO
|
161
|
+
@outputs.reject! { |out| out.kind_of?(IO) && out.fileno == removing.fileno }
|
162
|
+
when String
|
163
|
+
@outputs.reject! { |out| out.kind_of?(File) && out.path == removing }
|
164
|
+
when Integer
|
165
|
+
@outputs.reject! { |out| out.kind_of?(IO) && out.fileno == removing }
|
166
|
+
else
|
167
|
+
@outputs.reject! { |out| out == removing }
|
168
|
+
end
|
169
|
+
end
|
170
|
+
end
|
171
|
+
self
|
172
|
+
end
|
173
|
+
|
174
|
+
private
|
175
|
+
|
176
|
+
def normalize_outputs(ary)
|
177
|
+
ary.map do |out|
|
178
|
+
case
|
179
|
+
when out.respond_to?(:write)
|
180
|
+
out
|
181
|
+
when out.kind_of?(String)
|
182
|
+
File.open(out, 'a').tap{ |_| _.sync = true }
|
183
|
+
else
|
184
|
+
raise ArgumentError, 'output should able to respond to :write or be String'
|
185
|
+
end
|
186
|
+
end
|
187
|
+
end
|
188
|
+
end
|
189
|
+
end
|
190
|
+
end
|
@@ -0,0 +1,118 @@
|
|
1
|
+
require 'mamiya/agent'
|
2
|
+
require 'mamiya/master/web'
|
3
|
+
require 'mamiya/master/agent_monitor'
|
4
|
+
|
5
|
+
module Mamiya
|
6
|
+
class Master < Agent
|
7
|
+
MASTER_EVENTS = []
|
8
|
+
|
9
|
+
def initialize(*)
|
10
|
+
super
|
11
|
+
|
12
|
+
@agent_monitor = AgentMonitor.new(self)
|
13
|
+
@events_only ||= []
|
14
|
+
@events_only << MASTER_EVENTS
|
15
|
+
end
|
16
|
+
|
17
|
+
attr_reader :agent_monitor
|
18
|
+
|
19
|
+
def web
|
20
|
+
logger = self.logger
|
21
|
+
this = self
|
22
|
+
|
23
|
+
@web ||= Rack::Builder.new do
|
24
|
+
use AppBridge, logger, this
|
25
|
+
run Web
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def start
|
30
|
+
# Override and stop starting fetcher
|
31
|
+
web_start
|
32
|
+
serf_start
|
33
|
+
monitor_start
|
34
|
+
end
|
35
|
+
|
36
|
+
def distribute(application, package)
|
37
|
+
trigger(:fetch, application: application, package: package)
|
38
|
+
end
|
39
|
+
|
40
|
+
def storage(app)
|
41
|
+
config.storage_class.new(
|
42
|
+
config[:storage].merge(
|
43
|
+
application: app
|
44
|
+
)
|
45
|
+
)
|
46
|
+
end
|
47
|
+
|
48
|
+
def applications
|
49
|
+
config.storage_class.find(
|
50
|
+
config[:storage]
|
51
|
+
).keys
|
52
|
+
end
|
53
|
+
|
54
|
+
def status
|
55
|
+
{name: serf.name, master: true}
|
56
|
+
end
|
57
|
+
|
58
|
+
private
|
59
|
+
|
60
|
+
def init_serf
|
61
|
+
super.tap do |serf|
|
62
|
+
serf.on_user_event do |event|
|
63
|
+
monitor_commit_event(event)
|
64
|
+
end
|
65
|
+
|
66
|
+
serf.on_member_join do |event|
|
67
|
+
@agent_monitor.refresh(node: event.members.map { |_| _[:name] })
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
def monitor_commit_event(event)
|
73
|
+
@agent_monitor.commit_event(event)
|
74
|
+
rescue Exception => e
|
75
|
+
logger.fatal("Error during commiting event: #{e.inspect}")
|
76
|
+
e.backtrace.each do |line|
|
77
|
+
logger.fatal "\t#{line}"
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
def web_start
|
82
|
+
@web_thread = Thread.new do
|
83
|
+
options = config[:web] || {}
|
84
|
+
rack_options = {
|
85
|
+
app: self.web,
|
86
|
+
Port: options[:port].to_i,
|
87
|
+
Host: options[:bind],
|
88
|
+
environment: options[:environment],
|
89
|
+
server: options[:server],
|
90
|
+
Logger: logger['web']
|
91
|
+
}
|
92
|
+
server = Rack::Server.new(rack_options)
|
93
|
+
# To disable trap(:INT) and trap(:TERM)
|
94
|
+
server.define_singleton_method(:trap) { |*args| }
|
95
|
+
server.start
|
96
|
+
end
|
97
|
+
@web_thread.abort_on_exception = true
|
98
|
+
end
|
99
|
+
|
100
|
+
def monitor_start
|
101
|
+
logger.debug "Starting agent_monitor..."
|
102
|
+
@agent_monitor.start!
|
103
|
+
logger.debug "agent_monitor became ready"
|
104
|
+
end
|
105
|
+
|
106
|
+
class AppBridge
|
107
|
+
def initialize(app, log, this)
|
108
|
+
@app, @logger, @this = app, log['web'], this
|
109
|
+
end
|
110
|
+
|
111
|
+
def call(env)
|
112
|
+
env['rack.logger'] = @logger
|
113
|
+
env['mamiya.master'] = @this
|
114
|
+
@app.call(env)
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end
|
118
|
+
end
|
@@ -0,0 +1,146 @@
|
|
1
|
+
require 'json'
|
2
|
+
require 'set'
|
3
|
+
require 'thread'
|
4
|
+
|
5
|
+
require 'mamiya/master'
|
6
|
+
require 'mamiya/master/agent_monitor_handlers'
|
7
|
+
|
8
|
+
module Mamiya
|
9
|
+
class Master
|
10
|
+
##
|
11
|
+
# Class to monitor agent's status. This collects all agents' status.
|
12
|
+
# Statuses are updated by event from agent, and running serf query `mamiya:status` periodically.
|
13
|
+
class AgentMonitor
|
14
|
+
include AgentMonitorHandlers
|
15
|
+
|
16
|
+
STATUS_QUERY = 'mamiya:status'.freeze
|
17
|
+
DEFAULT_INTERVAL = 60
|
18
|
+
|
19
|
+
def initialize(master, raise_exception: false)
|
20
|
+
@master = master
|
21
|
+
@interval = (master.config[:master] &&
|
22
|
+
master.config[:master][:monitor] &&
|
23
|
+
master.config[:master][:monitor][:refresh_interval]) ||
|
24
|
+
DEFAULT_INTERVAL
|
25
|
+
|
26
|
+
@raise_exception = raise_exception
|
27
|
+
|
28
|
+
@agents = {}.freeze
|
29
|
+
@failed_agents = [].freeze
|
30
|
+
@statuses = {}
|
31
|
+
@commit_lock = Mutex.new
|
32
|
+
end
|
33
|
+
|
34
|
+
attr_reader :statuses, :agents, :failed_agents
|
35
|
+
|
36
|
+
def start!
|
37
|
+
@thread ||= Thread.new do
|
38
|
+
loop do
|
39
|
+
self.work_loop
|
40
|
+
sleep @interval
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def stop!
|
46
|
+
@thread.kill if running?
|
47
|
+
@thread = nil
|
48
|
+
end
|
49
|
+
|
50
|
+
def running?
|
51
|
+
@thread && @thread.alive?
|
52
|
+
end
|
53
|
+
|
54
|
+
def work_loop
|
55
|
+
self.refresh
|
56
|
+
rescue Exception => e
|
57
|
+
raise e if @raise_exception
|
58
|
+
|
59
|
+
logger.fatal "Periodical refreshing failed: #{e.class}: #{e.message}"
|
60
|
+
e.backtrace.each do |line|
|
61
|
+
logger.fatal "\t#{line}"
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
def commit_event(event)
|
66
|
+
@commit_lock.synchronize { commit_event_without_lock(event) }
|
67
|
+
end
|
68
|
+
|
69
|
+
def commit_event_without_lock(event)
|
70
|
+
return unless /\Amamiya:/ === event.user_event
|
71
|
+
|
72
|
+
method_name = event.user_event[7..-1].gsub(/:/, '__').gsub(/-/,'_')
|
73
|
+
return unless self.respond_to?(method_name, true)
|
74
|
+
|
75
|
+
payload = JSON.parse(event.payload)
|
76
|
+
agent = @statuses[payload["name"]]
|
77
|
+
return unless agent
|
78
|
+
|
79
|
+
logger.debug "Commiting #{event.user_event}"
|
80
|
+
logger.debug "- #{agent.inspect}"
|
81
|
+
__send__ method_name, agent, payload, event
|
82
|
+
logger.debug "+ #{agent.inspect}"
|
83
|
+
|
84
|
+
rescue JSON::ParserError => e
|
85
|
+
logger.warn "Failed to parse payload in event #{event.user_event}: #{e.message}"
|
86
|
+
end
|
87
|
+
|
88
|
+
def refresh(**kwargs)
|
89
|
+
# TODO: lock
|
90
|
+
logger.debug "Refreshing..."
|
91
|
+
|
92
|
+
new_agents = {}
|
93
|
+
new_failed_agents = Set.new
|
94
|
+
new_statuses = {}
|
95
|
+
|
96
|
+
@master.serf.members.each do |member|
|
97
|
+
new_agents[member["name"]] = member
|
98
|
+
new_failed_agents.add(member["name"]) unless member["status"] == 'alive'
|
99
|
+
end
|
100
|
+
|
101
|
+
@commit_lock.synchronize {
|
102
|
+
response = @master.serf.query(STATUS_QUERY, '', **kwargs)
|
103
|
+
response["Responses"].each do |name, json|
|
104
|
+
begin
|
105
|
+
new_statuses[name] = JSON.parse(json)
|
106
|
+
rescue JSON::ParserError => e
|
107
|
+
logger.warn "Failed to parse status from #{name}: #{e.message}"
|
108
|
+
new_failed_agents << name
|
109
|
+
next
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
113
|
+
new_failed_agents = new_failed_agents.to_a
|
114
|
+
|
115
|
+
(new_agents.keys - @agents.keys).join(", ").tap do |agents|
|
116
|
+
logger.info "Added agents: #{agents}" unless agents.empty?
|
117
|
+
end
|
118
|
+
|
119
|
+
(@agents.keys - new_agents.keys).join(", ").tap do |agents|
|
120
|
+
logger.info "Removed agents: #{agents}" unless agents.empty?
|
121
|
+
end
|
122
|
+
|
123
|
+
(failed_agents - new_failed_agents).join(", ").tap do |agents|
|
124
|
+
logger.info "Recovered agents: #{agents}" unless agents.empty?
|
125
|
+
end
|
126
|
+
|
127
|
+
(new_failed_agents - failed_agents).join(", ").tap do |agents|
|
128
|
+
logger.info "Newly failed agents: #{agents}" unless agents.empty?
|
129
|
+
end
|
130
|
+
|
131
|
+
@agents = new_agents.freeze
|
132
|
+
@failed_agents = new_failed_agents.freeze
|
133
|
+
@statuses = new_statuses
|
134
|
+
}
|
135
|
+
|
136
|
+
self
|
137
|
+
end
|
138
|
+
|
139
|
+
private
|
140
|
+
|
141
|
+
def logger
|
142
|
+
@logger ||= @master.logger['agent-monitor']
|
143
|
+
end
|
144
|
+
end
|
145
|
+
end
|
146
|
+
end
|