mamiya 0.0.1.alpha2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +17 -0
  3. data/.rspec +2 -0
  4. data/.travis.yml +16 -0
  5. data/Gemfile +8 -0
  6. data/LICENSE.txt +22 -0
  7. data/README.md +43 -0
  8. data/Rakefile +6 -0
  9. data/bin/mamiya +17 -0
  10. data/config.example.yml +11 -0
  11. data/docs/sequences/deploy.png +0 -0
  12. data/docs/sequences/deploy.uml +58 -0
  13. data/example.rb +74 -0
  14. data/lib/mamiya.rb +5 -0
  15. data/lib/mamiya/agent.rb +181 -0
  16. data/lib/mamiya/agent/actions.rb +12 -0
  17. data/lib/mamiya/agent/fetcher.rb +137 -0
  18. data/lib/mamiya/agent/handlers/abstract.rb +20 -0
  19. data/lib/mamiya/agent/handlers/fetch.rb +68 -0
  20. data/lib/mamiya/cli.rb +322 -0
  21. data/lib/mamiya/cli/client.rb +172 -0
  22. data/lib/mamiya/config.rb +57 -0
  23. data/lib/mamiya/dsl.rb +192 -0
  24. data/lib/mamiya/helpers/git.rb +75 -0
  25. data/lib/mamiya/logger.rb +190 -0
  26. data/lib/mamiya/master.rb +118 -0
  27. data/lib/mamiya/master/agent_monitor.rb +146 -0
  28. data/lib/mamiya/master/agent_monitor_handlers.rb +44 -0
  29. data/lib/mamiya/master/web.rb +148 -0
  30. data/lib/mamiya/package.rb +122 -0
  31. data/lib/mamiya/script.rb +117 -0
  32. data/lib/mamiya/steps/abstract.rb +19 -0
  33. data/lib/mamiya/steps/build.rb +72 -0
  34. data/lib/mamiya/steps/extract.rb +26 -0
  35. data/lib/mamiya/steps/fetch.rb +24 -0
  36. data/lib/mamiya/steps/push.rb +34 -0
  37. data/lib/mamiya/storages.rb +17 -0
  38. data/lib/mamiya/storages/abstract.rb +48 -0
  39. data/lib/mamiya/storages/mock.rb +61 -0
  40. data/lib/mamiya/storages/s3.rb +127 -0
  41. data/lib/mamiya/util/label_matcher.rb +38 -0
  42. data/lib/mamiya/version.rb +3 -0
  43. data/mamiya.gemspec +35 -0
  44. data/misc/logger_test.rb +12 -0
  45. data/spec/agent/actions_spec.rb +37 -0
  46. data/spec/agent/fetcher_spec.rb +199 -0
  47. data/spec/agent/handlers/fetch_spec.rb +121 -0
  48. data/spec/agent_spec.rb +255 -0
  49. data/spec/config_spec.rb +50 -0
  50. data/spec/dsl_spec.rb +291 -0
  51. data/spec/fixtures/dsl_test_load.rb +1 -0
  52. data/spec/fixtures/dsl_test_use.rb +1 -0
  53. data/spec/fixtures/helpers/foo.rb +1 -0
  54. data/spec/fixtures/test-package-source/.mamiya.meta.json +1 -0
  55. data/spec/fixtures/test-package-source/greeting +1 -0
  56. data/spec/fixtures/test-package.tar.gz +0 -0
  57. data/spec/fixtures/test.yml +4 -0
  58. data/spec/logger_spec.rb +68 -0
  59. data/spec/master/agent_monitor_spec.rb +269 -0
  60. data/spec/master/web_spec.rb +121 -0
  61. data/spec/master_spec.rb +94 -0
  62. data/spec/package_spec.rb +394 -0
  63. data/spec/script_spec.rb +78 -0
  64. data/spec/spec_helper.rb +38 -0
  65. data/spec/steps/build_spec.rb +261 -0
  66. data/spec/steps/extract_spec.rb +68 -0
  67. data/spec/steps/fetch_spec.rb +96 -0
  68. data/spec/steps/push_spec.rb +73 -0
  69. data/spec/storages/abstract_spec.rb +22 -0
  70. data/spec/storages/s3_spec.rb +342 -0
  71. data/spec/storages_spec.rb +33 -0
  72. data/spec/support/dummy_serf.rb +70 -0
  73. data/spec/util/label_matcher_spec.rb +85 -0
  74. metadata +272 -0
@@ -0,0 +1,190 @@
1
+ require 'logger'
2
+ require 'forwardable'
3
+ require 'thread'
4
+ require 'term/ansicolor'
5
+ require 'time'
6
+
7
+ module Mamiya
8
+ class Logger
9
+ include ::Logger::Severity
10
+ extend Forwardable
11
+
12
+ def self.defaults
13
+ return @defaults if @defaults
14
+ if ENV["MAMIYA_LOG_LEVEL"]
15
+ level = ::Logger::Severity.const_get(ENV["MAMIYA_LOG_LEVEL"].upcase) rescue INFO
16
+ else
17
+ level = INFO
18
+ end
19
+ @defaults = {color: nil, outputs: [STDOUT], level: level}
20
+ end
21
+
22
+ def initialize(color: self.class.defaults[:color], outputs: self.class.defaults[:outputs], level: self.class.defaults[:level])
23
+ @logdev = LogDev.new(outputs)
24
+ @logger = ::Logger.new(@logdev)
25
+ @logger.level = level
26
+ @logger.formatter = method(:format)
27
+
28
+ @color = color.nil? ? @logdev.tty? : color
29
+ end
30
+
31
+ attr_accessor :color
32
+ def_delegators :@logger,
33
+ :<<, :add, :log,
34
+ :fatal, :error, :warn, :info, :debug,
35
+ :fatal?, :error?, :warn?, :info?, :debug?,
36
+ :level, :level=, :progname, :progname=,
37
+ :close
38
+
39
+ def_delegators :@logdev, :reopen
40
+
41
+ def add_output(*outputs)
42
+ @logdev.add(*outputs)
43
+ end
44
+
45
+ def remove_output(*outputs)
46
+ @logdev.remove(*outputs)
47
+ end
48
+
49
+ def with_additional_file(*outputs)
50
+ @logdev.add_output(*outputs)
51
+
52
+ yield
53
+
54
+ ensure
55
+ @logdev.remove_output(*outputs)
56
+ end
57
+
58
+ def [](progname)
59
+ self.dup.tap do |new_logger|
60
+ new_logger.instance_eval do
61
+ @logger = @logger.dup
62
+ @logger.progname = progname
63
+ end
64
+ end
65
+ end
66
+
67
+ private
68
+
69
+ def format(severity, time, progname, msg)
70
+ rseverity = " #{severity.rjust(5)} "
71
+ if @color
72
+ colored_severity = case severity
73
+ when 'ANY'.freeze
74
+ rseverity
75
+ when 'DEBUG'.freeze
76
+ Term::ANSIColor.on_black(rseverity)
77
+ when 'INFO'.freeze
78
+ Term::ANSIColor.on_blue(rseverity)
79
+ when 'WARN'.freeze
80
+ Term::ANSIColor.on_yellow(Term::ANSIColor.black(rseverity))
81
+ when 'ERROR'.freeze
82
+ Term::ANSIColor.on_magenta(rseverity)
83
+ when 'FATAL'.freeze
84
+ Term::ANSIColor.on_red(Term::ANSIColor.white(Term::ANSIColor.bold(rseverity)))
85
+ else
86
+ rseverity
87
+ end
88
+ else
89
+ colored_severity = "#{rseverity}|"
90
+ end
91
+
92
+ msg = "#{(progname && "[#{progname}] ")}#{msg}"
93
+ if @color
94
+ colored_msg = case severity
95
+ when 'DEBUG'.freeze
96
+ Term::ANSIColor.bright_black(msg)
97
+ when 'FATAL'.freeze
98
+ Term::ANSIColor.bold(msg)
99
+ else
100
+ msg
101
+ end
102
+ else
103
+ colored_msg = msg
104
+ end
105
+
106
+ formatted_time = time.strftime('%m/%d %H:%M:%S')
107
+ colored_time = @color ? Term::ANSIColor.bright_black(formatted_time) : formatted_time
108
+
109
+ "#{colored_time} " \
110
+ "#{colored_severity} " \
111
+ "#{colored_msg}" \
112
+ "\n"
113
+ end
114
+
115
+ class LogDev
116
+ def initialize(outputs)
117
+ @outputs = normalize_outputs(outputs)
118
+ @mutex = Mutex.new
119
+ end
120
+
121
+ def tty?
122
+ @outputs.all?(&:tty?)
123
+ end
124
+
125
+ def write(*args)
126
+ @outputs.each do |output|
127
+ output.write(*args) unless output.respond_to?(:closed?) && output.closed?
128
+ end
129
+ self
130
+ end
131
+
132
+ def close
133
+ @outputs.each do |output|
134
+ output.close unless output.respond_to?(:closed?) && output.closed?
135
+ end
136
+ self
137
+ end
138
+
139
+ def reopen
140
+ @outputs.select { |io| io.respond_to?(:path) }.each do |io|
141
+ sync = io.sync
142
+ io.reopen(io.path, 'a')
143
+ io.sync = sync
144
+ end
145
+ end
146
+
147
+ def add(*outputs)
148
+ @mutex.synchronize do
149
+ @outputs.push(*normalize_outputs(outputs))
150
+ end
151
+ self
152
+ end
153
+
154
+ def remove(*removing_outputs)
155
+ @mutex.synchronize do
156
+ removing_outputs.each do |removing|
157
+ case removing
158
+ when File
159
+ @outputs.reject! { |out| out.kind_of?(File) && out.path == removing.path }
160
+ when IO
161
+ @outputs.reject! { |out| out.kind_of?(IO) && out.fileno == removing.fileno }
162
+ when String
163
+ @outputs.reject! { |out| out.kind_of?(File) && out.path == removing }
164
+ when Integer
165
+ @outputs.reject! { |out| out.kind_of?(IO) && out.fileno == removing }
166
+ else
167
+ @outputs.reject! { |out| out == removing }
168
+ end
169
+ end
170
+ end
171
+ self
172
+ end
173
+
174
+ private
175
+
176
+ def normalize_outputs(ary)
177
+ ary.map do |out|
178
+ case
179
+ when out.respond_to?(:write)
180
+ out
181
+ when out.kind_of?(String)
182
+ File.open(out, 'a').tap{ |_| _.sync = true }
183
+ else
184
+ raise ArgumentError, 'output should able to respond to :write or be String'
185
+ end
186
+ end
187
+ end
188
+ end
189
+ end
190
+ end
@@ -0,0 +1,118 @@
1
+ require 'mamiya/agent'
2
+ require 'mamiya/master/web'
3
+ require 'mamiya/master/agent_monitor'
4
+
5
+ module Mamiya
6
+ class Master < Agent
7
+ MASTER_EVENTS = []
8
+
9
+ def initialize(*)
10
+ super
11
+
12
+ @agent_monitor = AgentMonitor.new(self)
13
+ @events_only ||= []
14
+ @events_only << MASTER_EVENTS
15
+ end
16
+
17
+ attr_reader :agent_monitor
18
+
19
+ def web
20
+ logger = self.logger
21
+ this = self
22
+
23
+ @web ||= Rack::Builder.new do
24
+ use AppBridge, logger, this
25
+ run Web
26
+ end
27
+ end
28
+
29
+ def start
30
+ # Override and stop starting fetcher
31
+ web_start
32
+ serf_start
33
+ monitor_start
34
+ end
35
+
36
+ def distribute(application, package)
37
+ trigger(:fetch, application: application, package: package)
38
+ end
39
+
40
+ def storage(app)
41
+ config.storage_class.new(
42
+ config[:storage].merge(
43
+ application: app
44
+ )
45
+ )
46
+ end
47
+
48
+ def applications
49
+ config.storage_class.find(
50
+ config[:storage]
51
+ ).keys
52
+ end
53
+
54
+ def status
55
+ {name: serf.name, master: true}
56
+ end
57
+
58
+ private
59
+
60
+ def init_serf
61
+ super.tap do |serf|
62
+ serf.on_user_event do |event|
63
+ monitor_commit_event(event)
64
+ end
65
+
66
+ serf.on_member_join do |event|
67
+ @agent_monitor.refresh(node: event.members.map { |_| _[:name] })
68
+ end
69
+ end
70
+ end
71
+
72
+ def monitor_commit_event(event)
73
+ @agent_monitor.commit_event(event)
74
+ rescue Exception => e
75
+ logger.fatal("Error during commiting event: #{e.inspect}")
76
+ e.backtrace.each do |line|
77
+ logger.fatal "\t#{line}"
78
+ end
79
+ end
80
+
81
+ def web_start
82
+ @web_thread = Thread.new do
83
+ options = config[:web] || {}
84
+ rack_options = {
85
+ app: self.web,
86
+ Port: options[:port].to_i,
87
+ Host: options[:bind],
88
+ environment: options[:environment],
89
+ server: options[:server],
90
+ Logger: logger['web']
91
+ }
92
+ server = Rack::Server.new(rack_options)
93
+ # To disable trap(:INT) and trap(:TERM)
94
+ server.define_singleton_method(:trap) { |*args| }
95
+ server.start
96
+ end
97
+ @web_thread.abort_on_exception = true
98
+ end
99
+
100
+ def monitor_start
101
+ logger.debug "Starting agent_monitor..."
102
+ @agent_monitor.start!
103
+ logger.debug "agent_monitor became ready"
104
+ end
105
+
106
+ class AppBridge
107
+ def initialize(app, log, this)
108
+ @app, @logger, @this = app, log['web'], this
109
+ end
110
+
111
+ def call(env)
112
+ env['rack.logger'] = @logger
113
+ env['mamiya.master'] = @this
114
+ @app.call(env)
115
+ end
116
+ end
117
+ end
118
+ end
@@ -0,0 +1,146 @@
1
+ require 'json'
2
+ require 'set'
3
+ require 'thread'
4
+
5
+ require 'mamiya/master'
6
+ require 'mamiya/master/agent_monitor_handlers'
7
+
8
+ module Mamiya
9
+ class Master
10
+ ##
11
+ # Class to monitor agent's status. This collects all agents' status.
12
+ # Statuses are updated by event from agent, and running serf query `mamiya:status` periodically.
13
+ class AgentMonitor
14
+ include AgentMonitorHandlers
15
+
16
+ STATUS_QUERY = 'mamiya:status'.freeze
17
+ DEFAULT_INTERVAL = 60
18
+
19
+ def initialize(master, raise_exception: false)
20
+ @master = master
21
+ @interval = (master.config[:master] &&
22
+ master.config[:master][:monitor] &&
23
+ master.config[:master][:monitor][:refresh_interval]) ||
24
+ DEFAULT_INTERVAL
25
+
26
+ @raise_exception = raise_exception
27
+
28
+ @agents = {}.freeze
29
+ @failed_agents = [].freeze
30
+ @statuses = {}
31
+ @commit_lock = Mutex.new
32
+ end
33
+
34
+ attr_reader :statuses, :agents, :failed_agents
35
+
36
+ def start!
37
+ @thread ||= Thread.new do
38
+ loop do
39
+ self.work_loop
40
+ sleep @interval
41
+ end
42
+ end
43
+ end
44
+
45
+ def stop!
46
+ @thread.kill if running?
47
+ @thread = nil
48
+ end
49
+
50
+ def running?
51
+ @thread && @thread.alive?
52
+ end
53
+
54
+ def work_loop
55
+ self.refresh
56
+ rescue Exception => e
57
+ raise e if @raise_exception
58
+
59
+ logger.fatal "Periodical refreshing failed: #{e.class}: #{e.message}"
60
+ e.backtrace.each do |line|
61
+ logger.fatal "\t#{line}"
62
+ end
63
+ end
64
+
65
+ def commit_event(event)
66
+ @commit_lock.synchronize { commit_event_without_lock(event) }
67
+ end
68
+
69
+ def commit_event_without_lock(event)
70
+ return unless /\Amamiya:/ === event.user_event
71
+
72
+ method_name = event.user_event[7..-1].gsub(/:/, '__').gsub(/-/,'_')
73
+ return unless self.respond_to?(method_name, true)
74
+
75
+ payload = JSON.parse(event.payload)
76
+ agent = @statuses[payload["name"]]
77
+ return unless agent
78
+
79
+ logger.debug "Commiting #{event.user_event}"
80
+ logger.debug "- #{agent.inspect}"
81
+ __send__ method_name, agent, payload, event
82
+ logger.debug "+ #{agent.inspect}"
83
+
84
+ rescue JSON::ParserError => e
85
+ logger.warn "Failed to parse payload in event #{event.user_event}: #{e.message}"
86
+ end
87
+
88
+ def refresh(**kwargs)
89
+ # TODO: lock
90
+ logger.debug "Refreshing..."
91
+
92
+ new_agents = {}
93
+ new_failed_agents = Set.new
94
+ new_statuses = {}
95
+
96
+ @master.serf.members.each do |member|
97
+ new_agents[member["name"]] = member
98
+ new_failed_agents.add(member["name"]) unless member["status"] == 'alive'
99
+ end
100
+
101
+ @commit_lock.synchronize {
102
+ response = @master.serf.query(STATUS_QUERY, '', **kwargs)
103
+ response["Responses"].each do |name, json|
104
+ begin
105
+ new_statuses[name] = JSON.parse(json)
106
+ rescue JSON::ParserError => e
107
+ logger.warn "Failed to parse status from #{name}: #{e.message}"
108
+ new_failed_agents << name
109
+ next
110
+ end
111
+ end
112
+
113
+ new_failed_agents = new_failed_agents.to_a
114
+
115
+ (new_agents.keys - @agents.keys).join(", ").tap do |agents|
116
+ logger.info "Added agents: #{agents}" unless agents.empty?
117
+ end
118
+
119
+ (@agents.keys - new_agents.keys).join(", ").tap do |agents|
120
+ logger.info "Removed agents: #{agents}" unless agents.empty?
121
+ end
122
+
123
+ (failed_agents - new_failed_agents).join(", ").tap do |agents|
124
+ logger.info "Recovered agents: #{agents}" unless agents.empty?
125
+ end
126
+
127
+ (new_failed_agents - failed_agents).join(", ").tap do |agents|
128
+ logger.info "Newly failed agents: #{agents}" unless agents.empty?
129
+ end
130
+
131
+ @agents = new_agents.freeze
132
+ @failed_agents = new_failed_agents.freeze
133
+ @statuses = new_statuses
134
+ }
135
+
136
+ self
137
+ end
138
+
139
+ private
140
+
141
+ def logger
142
+ @logger ||= @master.logger['agent-monitor']
143
+ end
144
+ end
145
+ end
146
+ end