mkit 0.10.5 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Makefile +42 -0
- data/README.md +4 -1
- data/config/mkit_config.yml +3 -0
- data/docs/architecture.md +102 -0
- data/lib/mkit/app/controllers/pods_controller.rb +2 -2
- data/lib/mkit/app/controllers/services_controller.rb +4 -4
- data/lib/mkit/app/model/ingress.rb +1 -1
- data/lib/mkit/app/model/mkit_job.rb +0 -1
- data/lib/mkit/app/model/pod.rb +51 -36
- data/lib/mkit/app/model/resource.rb +7 -4
- data/lib/mkit/app/model/service.rb +51 -23
- data/lib/mkit/app/model/volume.rb +3 -3
- data/lib/mkit/app/templates/haproxy/app_haproxy.cfg.erb +1 -1
- data/lib/mkit/container/abstract.rb +16 -0
- data/lib/mkit/container/api.rb +22 -0
- data/lib/mkit/container/containers.rb +22 -0
- data/lib/mkit/container/docker/api.rb +25 -0
- data/lib/mkit/container/docker/containers.rb +178 -0
- data/lib/mkit/container/docker/events.rb +22 -0
- data/lib/mkit/container/docker/mapper.rb +54 -0
- data/lib/mkit/container/docker/networks.rb +37 -0
- data/lib/mkit/container/docker/payload.rb +41 -0
- data/lib/mkit/container/docker/stream_demux.rb +51 -0
- data/lib/mkit/container/docker/volumes.rb +29 -0
- data/lib/mkit/container/errors.rb +21 -0
- data/lib/mkit/container/event.rb +29 -0
- data/lib/mkit/container/events.rb +15 -0
- data/lib/mkit/container/http/unix_client.rb +139 -0
- data/lib/mkit/container/info.rb +41 -0
- data/lib/mkit/container/networks.rb +19 -0
- data/lib/mkit/container/registry.rb +37 -0
- data/lib/mkit/container/spec.rb +67 -0
- data/lib/mkit/container/utf8_buffer.rb +62 -0
- data/lib/mkit/container/volumes.rb +15 -0
- data/lib/mkit/ctypes.rb +0 -1
- data/lib/mkit/exceptions.rb +0 -2
- data/lib/mkit/pods/container_exec_command.rb +50 -0
- data/lib/mkit/pods/container_listener.rb +187 -0
- data/lib/mkit/pods/container_log_listener.rb +68 -0
- data/lib/mkit/pods/pod_reconciler.rb +48 -0
- data/lib/mkit/sagas/create_pod_saga.rb +22 -5
- data/lib/mkit/sagas/destroy_pod_saga.rb +6 -3
- data/lib/mkit/version.rb +1 -1
- data/lib/mkit/workers/haproxy_worker.rb +16 -10
- data/lib/mkit.rb +6 -4
- data/mkit.gemspec +11 -6
- metadata +121 -58
- data/lib/mkit/app/helpers/docker_helper.rb +0 -106
- data/lib/mkit/app/templates/docker/docker_run.sh.erb +0 -10
- data/lib/mkit/cmd/shell_client.rb +0 -57
- data/lib/mkit/cmd_runner.rb +0 -27
- data/lib/mkit/pods/docker_exec_command.rb +0 -38
- data/lib/mkit/pods/docker_listener.rb +0 -123
- data/lib/mkit/pods/docker_log_listener.rb +0 -40
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'mkit/container/abstract'
|
|
4
|
+
|
|
5
|
+
module MKIt
|
|
6
|
+
module Container
|
|
7
|
+
# Contract for container networks.
|
|
8
|
+
class Networks
|
|
9
|
+
extend Abstract
|
|
10
|
+
|
|
11
|
+
# find(name): Hash, nil when it does not exist
|
|
12
|
+
abstract_methods :create, :find, :attach, :detach, :remove
|
|
13
|
+
|
|
14
|
+
def exists?(name)
|
|
15
|
+
!find(name).nil?
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
end
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'mkit/container/docker/api'
|
|
4
|
+
|
|
5
|
+
module MKIt
|
|
6
|
+
module Container
|
|
7
|
+
# Holds the Api of the configured runtime.
|
|
8
|
+
module Registry
|
|
9
|
+
BUILDERS = { 'docker' => ->(socket) { Docker::Api.build(socket: socket) } }.freeze
|
|
10
|
+
|
|
11
|
+
class << self
|
|
12
|
+
def api
|
|
13
|
+
@api ||= build(nil, nil)
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
# Selects the runtime (and its socket) used by .api.
|
|
17
|
+
def configure(runtime: nil, socket: nil)
|
|
18
|
+
@api = build(runtime, socket)
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# Reads mkit.container.{runtime,socket}; mkit.docker.socket is the legacy key for the socket.
|
|
22
|
+
def configure_from(config)
|
|
23
|
+
container = config.container
|
|
24
|
+
configure(runtime: container&.runtime, socket: container&.socket || config.docker&.socket)
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
private
|
|
28
|
+
|
|
29
|
+
def build(runtime, socket)
|
|
30
|
+
name = (runtime || 'docker').to_s
|
|
31
|
+
builder = BUILDERS[name] or raise Error, "unsupported container runtime '#{name}'"
|
|
32
|
+
builder.call(socket)
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'shellwords'
|
|
4
|
+
require 'mkit/ctypes'
|
|
5
|
+
|
|
6
|
+
module MKIt
|
|
7
|
+
module Container
|
|
8
|
+
# Runtime neutral description of the container a pod needs.
|
|
9
|
+
class Spec
|
|
10
|
+
attr_reader :name
|
|
11
|
+
|
|
12
|
+
def initialize(name:, service:)
|
|
13
|
+
@name = name
|
|
14
|
+
@service = service
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def image
|
|
18
|
+
@service.image
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def env
|
|
22
|
+
(@service.service_config || [])
|
|
23
|
+
.select { |c| c.ctype == MKIt::CType::ENVIRONMENT.to_s }
|
|
24
|
+
.map { |c| "#{c.key}=#{c.value}" }
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def binds
|
|
28
|
+
(@service.volume || []).map { |v| "#{v.name}:#{v.path}" }
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def command
|
|
32
|
+
Shellwords.split(@service.command.to_s)
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def network
|
|
36
|
+
@service.pods_network
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def dns
|
|
40
|
+
@service.my_dns
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# Limits are expressed as '\d+m' (millicores / MiB), see Resource.validate
|
|
44
|
+
def cpu_millicores
|
|
45
|
+
amount(resource&.cpu_limits)
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def memory_mib
|
|
49
|
+
amount(resource&.memory_limits)
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def memory_swap_mib
|
|
53
|
+
amount(resource&.memory_swap_limits)
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
private
|
|
57
|
+
|
|
58
|
+
def resource
|
|
59
|
+
@service.resource
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def amount(value)
|
|
63
|
+
value.nil? ? nil : value.to_s.delete_suffix('m').to_i
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
end
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module MKIt
|
|
4
|
+
module Container
|
|
5
|
+
# Turns a stream of byte chunks into UTF-8 text without breaking characters.
|
|
6
|
+
#
|
|
7
|
+
# A multibyte character split between two chunks (socket reads, log frames) would otherwise
|
|
8
|
+
# be invalid in both halves. The incomplete tail of a chunk is held back until the rest arrives.
|
|
9
|
+
class Utf8Buffer
|
|
10
|
+
MAX_SEQUENCE = 4
|
|
11
|
+
|
|
12
|
+
def self.decode(data)
|
|
13
|
+
buffer = new
|
|
14
|
+
buffer.feed(data) + buffer.flush
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def initialize
|
|
18
|
+
@pending = String.new(encoding: Encoding::BINARY)
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# text of the chunk (plus what was held back), without a trailing incomplete character
|
|
22
|
+
def feed(chunk)
|
|
23
|
+
data = @pending + chunk.b
|
|
24
|
+
held = incomplete_tail(data)
|
|
25
|
+
@pending = data.byteslice(data.bytesize - held, held)
|
|
26
|
+
text(data.byteslice(0, data.bytesize - held))
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# what is left when the stream ends (an incomplete character becomes the replacement character)
|
|
30
|
+
def flush
|
|
31
|
+
rest = @pending
|
|
32
|
+
@pending = String.new(encoding: Encoding::BINARY)
|
|
33
|
+
text(rest)
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
private
|
|
37
|
+
|
|
38
|
+
def text(bytes)
|
|
39
|
+
bytes.force_encoding(Encoding::UTF_8).scrub
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# number of bytes at the end that start a character that is not complete yet
|
|
43
|
+
def incomplete_tail(data)
|
|
44
|
+
(1..[MAX_SEQUENCE - 1, data.bytesize].min).each do |back|
|
|
45
|
+
byte = data.getbyte(-back)
|
|
46
|
+
next if byte & 0xC0 == 0x80 # continuation byte
|
|
47
|
+
|
|
48
|
+
return back < sequence_length(byte) ? back : 0
|
|
49
|
+
end
|
|
50
|
+
0
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def sequence_length(lead_byte)
|
|
54
|
+
if lead_byte >= 0xF0 then 4
|
|
55
|
+
elsif lead_byte >= 0xE0 then 3
|
|
56
|
+
elsif lead_byte >= 0xC0 then 2
|
|
57
|
+
else 1
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
end
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'mkit/container/abstract'
|
|
4
|
+
|
|
5
|
+
module MKIt
|
|
6
|
+
module Container
|
|
7
|
+
# Contract for named volumes.
|
|
8
|
+
class Volumes
|
|
9
|
+
extend Abstract
|
|
10
|
+
|
|
11
|
+
# find(name): Hash, nil when it does not exist
|
|
12
|
+
abstract_methods :create, :find, :remove
|
|
13
|
+
end
|
|
14
|
+
end
|
|
15
|
+
end
|
data/lib/mkit/ctypes.rb
CHANGED
data/lib/mkit/exceptions.rb
CHANGED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
require 'shellwords'
|
|
2
|
+
require 'mkit/container/registry'
|
|
3
|
+
require 'mkit/container/utf8_buffer'
|
|
4
|
+
|
|
5
|
+
module MKIt
|
|
6
|
+
# Runs a command in a pod through the container runtime and bridges its
|
|
7
|
+
# (TTY) stream to a websocket.
|
|
8
|
+
class ContainerExecCommand
|
|
9
|
+
def initialize(pod, ws, options: {}, containers: Container::Registry.api.containers)
|
|
10
|
+
@containers = containers
|
|
11
|
+
@pod = pod
|
|
12
|
+
@ws = ws
|
|
13
|
+
@detached = options[:detached] == 'true'
|
|
14
|
+
@cmd = Shellwords.split(Array(options[:varargs]).join(' '))
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def register
|
|
18
|
+
@socket = @containers.attach_exec(@pod.name, @cmd, detached: @detached)
|
|
19
|
+
@stdout_thread = Thread.new { pump_output }
|
|
20
|
+
@ws.onmessage { |msg| @socket.write(msg) rescue nil } unless @detached
|
|
21
|
+
@stdout_thread
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def unregister
|
|
25
|
+
@stdout_thread&.kill
|
|
26
|
+
@socket&.close unless @socket&.closed?
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def close
|
|
30
|
+
@ws.close_websocket unless @ws.nil?
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
private
|
|
34
|
+
|
|
35
|
+
def pump_output
|
|
36
|
+
text = Container::Utf8Buffer.new
|
|
37
|
+
while (data = @socket.readpartial(4096))
|
|
38
|
+
out = text.feed(data)
|
|
39
|
+
@ws.send(out) unless out.empty?
|
|
40
|
+
end
|
|
41
|
+
rescue EOFError, IOError, Errno::ECONNRESET
|
|
42
|
+
# exec finished: send what was held back
|
|
43
|
+
rest = text.flush
|
|
44
|
+
@ws.send(rest) unless rest.empty?
|
|
45
|
+
ensure
|
|
46
|
+
@socket.close unless @socket.closed?
|
|
47
|
+
close
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
end
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
require 'mkit/status'
|
|
2
|
+
require 'mkit/container/registry'
|
|
3
|
+
require 'mkit/pods/pod_reconciler'
|
|
4
|
+
|
|
5
|
+
module MKIt
|
|
6
|
+
class StopThread < RuntimeError; end
|
|
7
|
+
|
|
8
|
+
# Consumes the container runtime events and keeps pods and services in sync.
|
|
9
|
+
class ContainerListener
|
|
10
|
+
RECONNECT_DELAY = 2
|
|
11
|
+
# container actions that only require the service status to be recomputed
|
|
12
|
+
STATUS_ACTIONS = %i[kill stop].freeze
|
|
13
|
+
# lines of the container output reported when it fails
|
|
14
|
+
FAILURE_LOG_LINES = 20
|
|
15
|
+
# exit codes of a process ended by SIGKILL / SIGTERM (e.g. stop, kill, oom)
|
|
16
|
+
SIGNAL_EXIT_CODES = [137, 143].freeze
|
|
17
|
+
|
|
18
|
+
def initialize(api: Container::Registry.api, reconciler: PodReconciler.new(containers: api.containers))
|
|
19
|
+
@api = api
|
|
20
|
+
@reconciler = reconciler
|
|
21
|
+
@queue = Queue.new
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def enqueue(event)
|
|
25
|
+
@queue << event
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def start
|
|
29
|
+
@consumer.run if register_consumer
|
|
30
|
+
register_listener
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def stop
|
|
34
|
+
@listener&.kill
|
|
35
|
+
@consumer&.raise StopThread.new
|
|
36
|
+
MKItLogger.info("container listener stopped")
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
private
|
|
40
|
+
|
|
41
|
+
def parse_message(event)
|
|
42
|
+
MKItLogger.info("container event received: #{event}")
|
|
43
|
+
case event.kind
|
|
44
|
+
when :container then on_container_event(event)
|
|
45
|
+
when :network then on_network_event(event)
|
|
46
|
+
else MKItLogger.info("\t#{event.kind} #{event.action} <<unknown>>")
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def on_container_event(event)
|
|
51
|
+
pod = Pod.find_by(name: event.container_name)
|
|
52
|
+
return unknown_pod(event, event.container_name) if pod.nil?
|
|
53
|
+
|
|
54
|
+
case event.action
|
|
55
|
+
when :create then pod_created(pod, event.container_id)
|
|
56
|
+
when :start then pod_started(pod, event.container_id)
|
|
57
|
+
when :die then pod_died(pod, event)
|
|
58
|
+
when :destroy then pod_gone(pod)
|
|
59
|
+
when :oom then MKItLogger.error("pod #{pod.name} ran out of memory (see its memory limit in resources)")
|
|
60
|
+
when *STATUS_ACTIONS then pod.service.update_status!
|
|
61
|
+
else MKItLogger.debug("\t#{event.kind} #{event.action} <<TODO>>")
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def pod_created(pod, container_id)
|
|
66
|
+
pod.pod_id = container_id
|
|
67
|
+
pod.status = MKIt::Status::CREATED
|
|
68
|
+
pod.save
|
|
69
|
+
pod.service.update_status!
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# the container runs: its network is up and the ip is known
|
|
73
|
+
def pod_started(pod, container_id)
|
|
74
|
+
pod.pod_id = container_id
|
|
75
|
+
pod.sync_ip(@api.containers.find(container_id))
|
|
76
|
+
pod.save
|
|
77
|
+
pod.service.update_status!
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# the container was removed: the pod no longer has an address nor takes part in the proxy config
|
|
81
|
+
def pod_gone(pod)
|
|
82
|
+
pod.sync_ip(nil)
|
|
83
|
+
pod.save
|
|
84
|
+
pod.service.update_status!
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def pod_died(pod, event)
|
|
88
|
+
report_exit(pod, event)
|
|
89
|
+
pod.service.update_status!
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
# Explains why a container ended: exit code, oom, runtime error and its last output.
|
|
93
|
+
def report_exit(pod, event)
|
|
94
|
+
info = @api.containers.find(event.container_id)
|
|
95
|
+
code = event.exit_code || info&.exit_code
|
|
96
|
+
return MKItLogger.info("pod #{pod.name} exited with code 0") if code.to_i.zero? && !info&.oom_killed?
|
|
97
|
+
|
|
98
|
+
message = exit_message(pod, code, info)
|
|
99
|
+
SIGNAL_EXIT_CODES.include?(code) && !info&.oom_killed? ? MKItLogger.warn(message) : MKItLogger.error(message)
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
def exit_message(pod, code, info)
|
|
103
|
+
lines = ["pod #{pod.name} died with exit code #{code}"]
|
|
104
|
+
lines << 'killed by the runtime: out of memory' if info&.oom_killed?
|
|
105
|
+
lines << "runtime error: #{info.error}" if info&.error
|
|
106
|
+
output = last_output(pod.name)
|
|
107
|
+
lines << "last output:\n#{output}" unless output.empty?
|
|
108
|
+
lines.join("\n\t")
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
def last_output(name)
|
|
112
|
+
text = @api.containers.logs(name, tail: FAILURE_LOG_LINES).to_s.strip
|
|
113
|
+
text.lines.map { |line| "\t\t#{line.chomp}" }.join("\n")
|
|
114
|
+
rescue Container::Error => e
|
|
115
|
+
MKItLogger.debug("could not read the output of #{name}: #{e.message}")
|
|
116
|
+
''
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
# only connect events of a container matter; network create/destroy carry no container
|
|
120
|
+
def on_network_event(event)
|
|
121
|
+
return unless event.action == :connect && event.container_id
|
|
122
|
+
|
|
123
|
+
info = @api.containers.find(event.container_id)
|
|
124
|
+
return unknown_pod(event, event.container_id) if info.nil?
|
|
125
|
+
|
|
126
|
+
pod = Pod.find_by(name: info.name)
|
|
127
|
+
return unknown_pod(event, info.name) if pod.nil?
|
|
128
|
+
|
|
129
|
+
pod_connected(pod, event, info)
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
# connecting before the container starts yields no ip yet, the start event handles it
|
|
133
|
+
def pod_connected(pod, event, info)
|
|
134
|
+
MKItLogger.info("network #{event.action} received for pod #{pod.name}")
|
|
135
|
+
pod.sync_ip(info)
|
|
136
|
+
pod.save
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
def unknown_pod(event, reference)
|
|
140
|
+
MKItLogger.warn("container event #{event} received, but I don't know anything about pod #{reference}")
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
def register_consumer
|
|
144
|
+
return false unless @consumer.nil?
|
|
145
|
+
|
|
146
|
+
@consumer = Thread.new do
|
|
147
|
+
running = true
|
|
148
|
+
while running
|
|
149
|
+
begin
|
|
150
|
+
parse_message(@queue.pop)
|
|
151
|
+
rescue StopThread
|
|
152
|
+
running = false
|
|
153
|
+
MKItLogger.info("container consumer ended")
|
|
154
|
+
rescue => e
|
|
155
|
+
MKItLogger.error("error while consuming container event: #{e}", e.message, e.backtrace.join("\n"))
|
|
156
|
+
end
|
|
157
|
+
end
|
|
158
|
+
end
|
|
159
|
+
true
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
def reconcile
|
|
163
|
+
@reconciler.call
|
|
164
|
+
rescue StandardError => e
|
|
165
|
+
MKItLogger.warn("could not reconcile the pods: #{e.message}")
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
def register_listener
|
|
169
|
+
@listener = Thread.new { listen }
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
# Streams the runtime events; reconnects when the stream drops. The events lost while it was
|
|
173
|
+
# down (or before the daemon started) are recovered by reconciling the pods before each subscription.
|
|
174
|
+
def listen
|
|
175
|
+
loop do
|
|
176
|
+
begin
|
|
177
|
+
reconcile
|
|
178
|
+
@api.events.each { |event| enqueue event }
|
|
179
|
+
MKItLogger.warn('container event stream closed, reconnecting')
|
|
180
|
+
rescue StandardError => e
|
|
181
|
+
MKItLogger.warn("container event stream error: #{e.message}, reconnecting")
|
|
182
|
+
end
|
|
183
|
+
sleep(RECONNECT_DELAY)
|
|
184
|
+
end
|
|
185
|
+
end
|
|
186
|
+
end
|
|
187
|
+
end
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
require 'mkit/container/registry'
|
|
2
|
+
|
|
3
|
+
module MKIt
|
|
4
|
+
# Streams a pod's logs to a websocket through the container runtime.
|
|
5
|
+
class ContainerLogListener
|
|
6
|
+
DEFAULT_LINES = 10
|
|
7
|
+
|
|
8
|
+
def initialize(pod, ws, options: {}, containers: Container::Registry.api.containers)
|
|
9
|
+
@containers = containers
|
|
10
|
+
clear_logs(pod.name) if options[:clear]
|
|
11
|
+
@pod = pod
|
|
12
|
+
@ws = ws
|
|
13
|
+
@options = options
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def register
|
|
17
|
+
@client = Thread.new do
|
|
18
|
+
begin
|
|
19
|
+
stream_logs
|
|
20
|
+
rescue StandardError => e
|
|
21
|
+
MKItLogger.warn("log stream for #{@pod.name} ended: #{e.message}")
|
|
22
|
+
ensure
|
|
23
|
+
close
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def unregister
|
|
29
|
+
@client&.kill
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def close
|
|
33
|
+
@ws.close_websocket unless @ws.nil?
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
private
|
|
37
|
+
|
|
38
|
+
def query
|
|
39
|
+
{
|
|
40
|
+
tail: @options[:nr_lines] || DEFAULT_LINES,
|
|
41
|
+
follow: @options[:follow] == 'true',
|
|
42
|
+
timestamps: @options[:timestamps] == 'true',
|
|
43
|
+
details: @options[:details] == 'true',
|
|
44
|
+
since: @options[:since],
|
|
45
|
+
until_: @options[:until]
|
|
46
|
+
}
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def clear_logs(name)
|
|
50
|
+
path = @containers.log_path(name)
|
|
51
|
+
File.write(path, '') unless path.nil? || path.empty?
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def stream_logs
|
|
55
|
+
pending = +''
|
|
56
|
+
@containers.follow_logs(@pod.name, **query) { |text| pending = send_lines(pending + text) }
|
|
57
|
+
@ws.send(pending) unless pending.empty?
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
# sends complete lines, returns the unfinished remainder
|
|
61
|
+
def send_lines(text)
|
|
62
|
+
lines = text.split("\n", -1)
|
|
63
|
+
rest = lines.pop
|
|
64
|
+
lines.each { |line| @ws.send(line) }
|
|
65
|
+
rest
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
end
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
require 'mkit/container/registry'
|
|
2
|
+
require 'mkit/container/errors'
|
|
3
|
+
|
|
4
|
+
module MKIt
|
|
5
|
+
# Brings the pods (container id, ip, dns) and the proxy config back in sync with the containers.
|
|
6
|
+
#
|
|
7
|
+
# Needed when events may have been lost: on startup and every time the event stream is
|
|
8
|
+
# reestablished (the runtime restarted, the socket dropped).
|
|
9
|
+
# A service whose pod ips changed gets a single update_proxy_config job, however many of its pods changed.
|
|
10
|
+
class PodReconciler
|
|
11
|
+
def initialize(containers: Container::Registry.api.containers)
|
|
12
|
+
@containers = containers
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
# returns the services whose proxy config was refreshed
|
|
16
|
+
def call
|
|
17
|
+
changed = {}
|
|
18
|
+
Pod.all.each do |pod|
|
|
19
|
+
service = pod.service
|
|
20
|
+
next if service.nil?
|
|
21
|
+
|
|
22
|
+
changed[service.id] = service if sync(pod)
|
|
23
|
+
end
|
|
24
|
+
changed.each_value { |service| refresh(service) }
|
|
25
|
+
MKItLogger.info("pods reconciled, #{changed.size} service(s) with a new proxy config")
|
|
26
|
+
changed.values
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
private
|
|
30
|
+
|
|
31
|
+
# true when the ip of the pod changed
|
|
32
|
+
def sync(pod)
|
|
33
|
+
info = @containers.find(pod.name)
|
|
34
|
+
pod.sync_ip(info, notify: false)
|
|
35
|
+
ip_changed = pod.saved_change_to_ip?
|
|
36
|
+
pod.update!(pod_id: info.id) if info && pod.pod_id != info.id
|
|
37
|
+
ip_changed
|
|
38
|
+
rescue Container::Error => e
|
|
39
|
+
MKItLogger.warn("could not reconcile pod #{pod.name}: #{e.message}")
|
|
40
|
+
false
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def refresh(service)
|
|
44
|
+
MkitJob.publish(topic: :update_proxy_config, service_id: service.id)
|
|
45
|
+
service.update_status!
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
end
|
|
@@ -1,8 +1,11 @@
|
|
|
1
|
-
require 'mkit/
|
|
1
|
+
require 'mkit/container/registry'
|
|
2
2
|
|
|
3
3
|
module MKIt
|
|
4
4
|
class CreatePodSaga < ASaga
|
|
5
|
-
|
|
5
|
+
def initialize(containers: Container::Registry.api.containers)
|
|
6
|
+
@containers = containers
|
|
7
|
+
super()
|
|
8
|
+
end
|
|
6
9
|
|
|
7
10
|
def topics
|
|
8
11
|
%w{create_pod_saga}
|
|
@@ -22,9 +25,23 @@ module MKIt
|
|
|
22
25
|
|
|
23
26
|
pod_name = job.data['pod_name']
|
|
24
27
|
pod = Pod.find_by_name(pod_name)
|
|
25
|
-
|
|
26
|
-
MKItLogger.info("deploying
|
|
27
|
-
|
|
28
|
+
spec = pod.container_spec
|
|
29
|
+
MKItLogger.info("deploying pod #{pod_name} (#{spec.image})")
|
|
30
|
+
run(pod_name, spec)
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
private
|
|
34
|
+
|
|
35
|
+
# logs why the container could not be created or started, then lets the job fail
|
|
36
|
+
def run(pod_name, spec)
|
|
37
|
+
@containers.run(spec)
|
|
38
|
+
rescue Container::Error => e
|
|
39
|
+
MKItLogger.error("pod #{pod_name} (#{spec.image}) could not be started: #{e.message}#{status_of(e)}")
|
|
40
|
+
raise
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def status_of(error)
|
|
44
|
+
error.status.nil? ? '' : " (status #{error.status})"
|
|
28
45
|
end
|
|
29
46
|
end
|
|
30
47
|
end
|
|
@@ -1,8 +1,11 @@
|
|
|
1
|
-
require 'mkit/
|
|
1
|
+
require 'mkit/container/registry'
|
|
2
2
|
|
|
3
3
|
module MKIt
|
|
4
4
|
class DestroyPodSaga < ASaga
|
|
5
|
-
|
|
5
|
+
def initialize(containers: Container::Registry.api.containers)
|
|
6
|
+
@containers = containers
|
|
7
|
+
super()
|
|
8
|
+
end
|
|
6
9
|
|
|
7
10
|
def topics
|
|
8
11
|
%w{destroy_pod_saga}
|
|
@@ -22,7 +25,7 @@ module MKIt
|
|
|
22
25
|
MKItLogger.info("#{self.class} <#{job.topic}> #{job.inspect}....")
|
|
23
26
|
|
|
24
27
|
begin
|
|
25
|
-
|
|
28
|
+
@containers.remove(job.data['pod_name'])
|
|
26
29
|
rescue => e
|
|
27
30
|
MKItLogger.warn(e)
|
|
28
31
|
end
|
data/lib/mkit/version.rb
CHANGED
|
@@ -7,21 +7,13 @@ module MKIt
|
|
|
7
7
|
|
|
8
8
|
def do_the(job)
|
|
9
9
|
MKItLogger.info("#{self.class} working on the job #{job.inspect}....")
|
|
10
|
-
unless job.service_id.nil?
|
|
11
|
-
srv = Service.find(job.service_id)
|
|
12
|
-
config = srv.proxy_config
|
|
13
|
-
end
|
|
14
10
|
case job.topic.to_sym
|
|
15
|
-
when :update_proxy_config
|
|
16
|
-
|
|
17
|
-
MKIt::HAProxy.create_config_file(filename: config[:filename], data: config[:data])
|
|
11
|
+
when :update_proxy_config, :create_proxy_config
|
|
12
|
+
write_config(Service.find(job.service_id))
|
|
18
13
|
MKIt::HAProxy.reload
|
|
19
14
|
when :destroy_proxy_config
|
|
20
15
|
MKIt::HAProxy.delete_config_file(filename: job.data['filename'])
|
|
21
16
|
MKIt::HAProxy.reload
|
|
22
|
-
when :create_proxy_config
|
|
23
|
-
MKIt::HAProxy.create_config_file(filename: config[:filename], data: config[:data])
|
|
24
|
-
MKIt::HAProxy.reload
|
|
25
17
|
when :reload_proxy
|
|
26
18
|
MKIt::HAProxy.reload
|
|
27
19
|
when :restart_proxy
|
|
@@ -30,6 +22,20 @@ module MKIt
|
|
|
30
22
|
MKItLogger.warn("#{self.class} <<TODO>> job #{job.inspect}....")
|
|
31
23
|
end
|
|
32
24
|
end
|
|
25
|
+
|
|
26
|
+
private
|
|
27
|
+
|
|
28
|
+
# A service without a pod to route to has no valid config: its file must leave the config dir.
|
|
29
|
+
def write_config(srv)
|
|
30
|
+
if srv.routable?
|
|
31
|
+
config = srv.proxy_config
|
|
32
|
+
MKItLogger.debug config.inspect
|
|
33
|
+
MKIt::HAProxy.create_config_file(filename: config[:filename], data: config[:data])
|
|
34
|
+
else
|
|
35
|
+
MKItLogger.warn("service #{srv.name} has no pod with an ip, removing its proxy config")
|
|
36
|
+
MKIt::HAProxy.delete_config_file(filename: srv.proxy_filename)
|
|
37
|
+
end
|
|
38
|
+
end
|
|
33
39
|
end
|
|
34
40
|
end
|
|
35
41
|
|