mkit 0.2.0

data/lib/mkit/docker_listener.rb

@@ -0,0 +1,97 @@
+require 'pty'
+require 'mkit/status'
+
+#
+# https://docs.docker.com/engine/reference/commandline/events
+require 'mkit/app/helpers/docker_helper'
+module MKIt
+  class DockerListener
+    include MKIt::DockerHelper
+
+    def initialize
+      @consumers = []
+    end
+
+    def register_consumer(consumer:)
+    end
+
+    def parse_message(msg)
+      action = msg['Action'].to_sym
+      type = msg['Type'].to_sym
+      MKItLogger.info("docker <#{type}> <#{action}> received: \n\t#{msg}")
+      case type
+      when :container
+        pod_id = msg.id
+        pod_name = msg.Actor.Attributes.name
+        pod = Pod.find_by(name: pod_name)
+        unless pod.nil?
+          case action
+          when :create
+            pod.pod_id = pod_id
+            pod.status = MKIt::Status::CREATED
+            pod.save
+            pod.service.update_status!
+          when :start
+            pod.pod_id = pod_id
+            pod.save
+            pod.service.update_status!
+          when :kill
+            MKItLogger.debug("	#{type} #{action} <<NOOP / TODO>>")
+          when :die
+            MKItLogger.debug("	#{type} #{action} <<NOOP / TODO>>")
+          when :stop
+            pod.service.update_status!
+          else
+            MKItLogger.debug("	#{type} #{action} <<TODO>>")
+          end
+        else
+          MKItLogger.warn("docker <<#{type}>> <#{action}> received: #{msg}. But I don't know anything about pod #{pod_id}")
+        end
+      when :network
+        pod_id = msg.Actor.Attributes.container
+        pod = Pod.find_by(pod_id: pod_id)
+        unless pod.nil?
+          case action
+          when :connect
+            MKItLogger.info("docker network #{action} received: #{msg}")
+            pod.update_ip
+            pod.save
+          when :disconnect
+            MKItLogger.debug("  #{type} #{action} <<NOOP / TODO>>")
+          else
+            MKItLogger.debug("  #{type} #{action} <<TODO>>")
+          end
+        else
+          MKItLogger.warn("docker <<#{type}>> <#{action}> received: #{msg}. But I don't know anything about pod #{pod_id}")
+        end
+      else
+        MKItLogger.info("\t#{type} #{action} <<unknown>>")
+      end
+    end
+
+    def start
+      @thread ||= Thread.new {
+        cmd = "docker events --format '{{json .}}'"
+        begin
+          PTY.spawn( cmd ) do |stdout, stdin, pid|
+            begin
+              stdout.each { |line| parse_message JSON.parse(line).to_o }
+            rescue Errno::EIO
+              MKItLogger.warn("Errno:EIO error, but this probably just means " +
+                "that the process has finished giving output")
+            end
+          end
+        rescue PTY::ChildExited
+          MKItLogger.warn("docker event listener process exited!")
+        end
+      }
+      @thread.run
+      MKItLogger.info("docker listener started")
+    end
+    def stop
+      @thread.exit if @thread
+      MKItLogger.info("docker listener stopped")
+    end
+  end
+end
+

data/lib/mkit/exceptions.rb

@@ -0,0 +1,30 @@
+module MKIt
+  class BaseException < Exception
+    attr_reader :error_code
+    def initialize(error_code, message = nil)
+      super(message)
+      @error_code = error_code
+    end
+  end
+
+  class ServiceAlreadyExists < BaseException
+    def initialize(message = nil)
+      super(409, message)
+    end
+  end
+  class ServiceNameMismatch < BaseException
+    def initialize(message = nil)
+      super(400, message)
+    end
+  end
+  class ServiceNotFoundException < StandardError; end
+  class PodNotFoundException     < StandardError; end
+  class AppAlreadyDeployedException < StandardError; end
+  class InvalidPortMappingTypeException < StandardError; end
+
+  class PoolExaustedException < StandardError; end
+
+  class CmdRunnerException < StandardError; end
+
+end
+

data/lib/mkit/haproxy.rb

@@ -0,0 +1,48 @@
+require 'pty'
+#
+# 
+#
+module MKIt
+  class HAProxy
+
+    def initialize
+      # configs
+      # run standalone | daemon
+      @running = false
+    end
+
+    def start
+      @thread ||= Thread.new {
+        while (@running) do
+        cmd = "/usr/sbin/haproxy -f /etc/haproxy/haproxy.d"
+        %x{#{cmd}}
+        sleep 1
+        end
+      }
+      @thread.run
+      puts "haproxy started"
+    end
+
+    def start
+      @running = true
+      @thread ||= Thread.new {
+        while (@running) do
+          %{/usr/sbin/haproxy -f /etc/haproxy/haproxy.d/}
+          sleep(1)
+        end
+      }
+      puts "proxy started"
+    end
+
+    def stop
+      puts "proxy stopped"
+    end
+
+    def status
+    end
+
+    def reload
+    end
+  end
+end
+

data/lib/mkit/job_manager.rb

@@ -0,0 +1,53 @@
+require 'mkit/app/model/mkit_job'
+require 'mkit/utils'
+
+module MKIt
+  class JobManager
+    def initialize
+      @workers = {}
+    end
+
+    def register_worker(worker, topics)
+      topics.each { | topic |
+        @workers[topic] ||= []
+        MKItLogger.info("register #{worker.class} for topic #{topic}")
+        @workers[topic] << worker
+      }
+    end
+
+    def start
+      MKItLogger.info('starting job manager')
+      @thread = Thread.new do
+        loop do
+          job = MkitJob.take
+          begin
+            if job.nil?
+              sleep(10)
+            else
+              topic = job.topic
+              job.processing!
+              if @workers[topic].nil?
+                MKItLogger.warn("no workers found for topic '#{topic}'")
+              else
+                workers = @workers[topic]
+                workers.each { | worker |
+                  worker.do_the(job)
+                }
+              end 
+            end
+            job.done! unless job.nil?
+          rescue Exception => e
+            job.error! unless job.nil?
+            MKItLogger.error e, e.message, e.backtrace.join("\n")
+          end
+        end
+      end
+    end
+
+    def stop
+      @thread.exit if @thread
+    end
+  end
+end
+
+

data/lib/mkit/mkit_dns.rb

@@ -0,0 +1,54 @@
+require 'async/dns'
+require 'async/dns/system'
+require 'mkit/mkit_interface'
+require 'ipaddr'
+
+# INTERFACES = [
+#   [:udp, "127.0.0.20", 53],
+#   [:tcp, "127.0.0.20", 53],
+# ]
+# @resolver = RubyDNS::Resolver.new(
+#   [:udp, "192.168.4.254", 53],
+#   [:tcp, "192.168.4.254", 53]
+# )
+# 
+# # Use upstream DNS for name resolution.
+UPSTREAM = RubyDNS::Resolver.new([
+   [:udp, "8.8.8.8", 53],
+   [:tcp, "8.8.8.8", 53]
+ ])
+
+Name = Resolv::DNS::Name
+IN = Resolv::DNS::Resource::IN
+
+module MKIt
+  class DNS < Async::DNS::Server
+    def initialize
+      addr = MKIt::Interface.ip
+      listen_addr = [
+        [:udp, addr, 53],
+        [:tcp, addr, 53],
+      ]
+      super(listen_addr)
+      @logger.info!
+      @resolver = RubyDNS::Resolver.new(Async::DNS::System.nameservers)
+    end
+    def process(name, resource_class, transaction)
+      host = DnsHost.find_by_name(name)
+      if host.nil?
+        transaction.passthrough!(@resolver)
+      else
+        ipaddr = IPAddr.new host.ip
+        if resource_class == Resolv::DNS::Resource::IN::A
+          transaction.respond!(ipaddr.to_s)
+        elsif resource_class == Resolv::DNS::Resource::IN::AAAA
+          transaction.respond!(ipaddr.ipv4_mapped.to_s)
+        else
+          transaction.fail!(:NXDomain)
+        end
+      end
+    end
+  end
+end
+
+

data/lib/mkit/mkit_interface.rb

@@ -0,0 +1,31 @@
+require 'mkit/status'
+require 'mkit/utils'
+require 'mkit/exceptions'
+require 'mkit/app/helpers/interface_helper'
+
+module MKIt
+  class Interface
+    def self.ip
+      main_pool = Pool.find_by_name(MKIt::Utils.me)
+      main_pool.ip
+    end
+
+    def self.up
+      main_pool = Pool.find_by_name(MKIt::Utils.me)
+      interface_name = "#{main_pool.name}0"
+      interface_type = "tap"
+      ip = main_pool.ip
+      mask = main_pool.netmask
+      MKIt::InterfaceHelper.create(name: interface_name, ctype: interface_type)
+      MKIt::InterfaceHelper.up(name: interface_name, ip: ip, mask: mask)
+    end
+
+    def self.down
+      main_pool = Pool.find_by_name(MKIt::Utils.me)
+      interface_name = "#{main_pool.name}0"
+      interface_type = "tap"
+      MKIt::InterfaceHelper.down(name: interface_name)
+      MKIt::InterfaceHelper.remove(name: interface_name, ctype: interface_type)
+    end
+  end
+end

data/lib/mkit/sagas/asaga.rb

@@ -0,0 +1,11 @@
+require 'mkit/status'
+module MKIt
+  class ASaga
+
+    # Must define topics method on subclass 
+    def initialize
+      System[:job_manager].register_worker(self, self.topics)
+    end 
+
+  end
+end

data/lib/mkit/sagas/create_pod_saga.rb

@@ -0,0 +1,28 @@
+module MKIt
+  class CreatePodSaga < ASaga
+
+    def topics
+      %w{create_pod_saga}
+    end
+
+    #
+    # create_pod_saga:
+    #
+    # payload:
+    #  * service_id
+    #
+    # triggers
+    #  * nothing
+    #
+    def do_the(job)
+      MKItLogger.info("#{self.class} <#{job.topic}> #{job.inspect}....")
+      service = Service.find(job.service_id)
+      # create pod
+
+      pd = Pod.new( service: service, status: MKIt::Status::CREATED, name: SecureRandom.uuid.gsub('-','')[0..11])
+      service.pod << pd
+      service.save
+      MkitJob.publish(topic: :start_pod, service_id: job.service_id, pod_id: pd.id)
+    end
+  end
+end

data/lib/mkit/sagas/saga_manager.rb

@@ -0,0 +1,10 @@
+require 'mkit/sagas/asaga'
+require 'mkit/sagas/create_pod_saga'
+
+module MKIt
+  class SagaManager
+    def self.register_workers
+      CreatePodSaga.new
+    end
+  end
+end

data/lib/mkit/status.rb

@@ -0,0 +1,47 @@
+#
+#
+#
+module MKIt
+  class MKItStatus
+    def initialize(status)
+      @status = status
+    end
+    def to_s
+      @status.to_s
+    end
+  end
+
+  module Status
+    # APP
+    CREATED   = 'CREATED'
+    CREATING  = 'CREATING'
+    DEPLOYING = 'DEPLOYING'
+    DEPLOYED  = 'DEPLOYED'
+    PENDING   = 'PENDING'
+    DEGRATED  = 'DEGRATED'
+
+    # network
+    RESERVED = 'RESERVED'
+    IN_USE   = 'IN_USE'
+    EXPIRED  = 'EXPIRED'
+
+    # pods
+    STARTING = 'STARTING'
+    RUNNING  = 'RUNNING'
+    STOPPED  = 'STOPPED'
+    STOPING  = 'STOPING'
+    PAUSED   = 'PAUSED'
+
+    # Service
+    RESTARTING = 'RESTARTING'
+    UPDATING   = 'UPDATING'
+  end
+
+  module PoolStatus
+    
+    RESERVED = 'RESERVED'
+    IN_USE   = 'IN_USE'
+    EXPIRED  = 'EXPIRED'
+    EXAUSTED = 'EXAUSTED'
+  end
+end

data/lib/mkit/utils.rb

@@ -0,0 +1,51 @@
+require 'erb'
+require 'mkit/config/config'
+module MKIt
+  module Utils
+    module_function
+
+    def me
+      'mkit'
+    end
+
+    def log
+      Console.logger
+    end
+
+    def root
+      File.expand_path("../../..", __FILE__)
+    end
+
+    def set_config_dir(config_dir)
+      @config_dir = config_dir
+    end
+
+    def config_dir
+      @config_dir.nil? ? "#{self.root}/config" : @config_dir
+    end
+
+    def load_db_config(db_config_dir = self.config_dir)
+      self.log.info "loading database configurations from '#{config_dir}'..."
+      YAML::load(ERB.new(IO.read("#{db_config_dir}/database.yml")).result)
+    end
+
+    def db_config_to_uri(env = MKIt::Config.mkit.database.env)
+      config = self.load_db_config[env]
+
+      if config["username"] || config["password"]
+        user_info = [ config["username"], config["password"] ].join(":")
+      else
+        user_info = nil
+      end
+      URI::Generic.new(config["adapter"],user_info,
+                       config["hostname"] || "localhost",
+                       config["port"],
+                       nil,
+                       "/#{config["database"]}",
+                       nil,
+                       nil,
+                       nil).to_s
+    end
+  end
+end
+

data/lib/mkit/version.rb

@@ -0,0 +1,4 @@
+module MKIt
+  VERSION = "0.2.0"
+end
+

data/lib/mkit/workers/aworker.rb

@@ -0,0 +1,11 @@
+require 'mkit/status'
+module MKIt
+  class AWorker
+
+    # Must define topics method on subclass 
+    def initialize
+      System[:job_manager].register_worker(self, self.topics)
+    end 
+
+  end
+end

data/lib/mkit/workers/haproxy_worker.rb

@@ -0,0 +1,35 @@
+module MKIt
+  class HAProxyWorker < AWorker
+
+    def topics
+      %w{create_proxy_config destroy_proxy_config update_proxy_config restart_proxy reload_proxy}
+    end
+
+    def do_the(job)
+      MKItLogger.info("#{self.class} working on the job #{job.inspect}....")
+      unless job.service_id.nil?
+        srv = Service.find(job.service_id)
+        config = srv.proxy_config
+      end
+      case job.topic.to_sym
+      when :update_proxy_config
+        MKItLogger.debug config.inspect
+        MKIt::HAProxy.create_config_file(filename: config[:filename], data: config[:data])
+        MKIt::HAProxy.reload
+      when :destroy_proxy_config
+        MKIt::HAProxy.delete_config_file(filename: job.data['filename'])
+        MKIt::HAProxy.reload
+      when :create_proxy_config
+        MKIt::HAProxy.create_config_file(filename: config[:filename], data: config[:data])
+        MKIt::HAProxy.reload
+      when :reload_proxy
+        MKIt::HAProxy.reload
+      when :restart_proxy
+        MKIt::HAProxy.restart
+      else
+        MKItLogger.warn("#{self.class} <<TODO>> job #{job.inspect}....")
+      end
+    end
+  end
+end
+

data/lib/mkit/workers/pod_worker.rb

@@ -0,0 +1,39 @@
+#
+module MKIt
+  class PodWorker < AWorker
+
+    def topics
+      %w{pod_network_connected pod_network_disconnected
+          pod_unhealthy 
+          start_pod stop_pod update_pod deploy_pod destroy_pod
+          pod_ip_updated pod_destroyed
+      }
+    end
+
+    def do_the(job)
+      MKItLogger.info("#{self.class} <#{job.topic}> job #{job.inspect}....")
+      pod = Pod.find(job.pod_id) unless job.pod_id.nil?
+      case job.topic.to_sym
+      when :deploy_pod
+        MKItLogger.warn("#{self.class} @deprecated job #{job.inspect}....")
+      when :start_pod
+        pod.start
+      when :stop_pod
+        pod.stop
+      when :destroy_pod
+        pod.stop
+        pod.destroy
+      when :pod_destroyed
+        if Service.exists?(job.service_id)
+          MkitJob.publish(topic: :update_proxy_config, service_id: job.service_id)
+        end
+      when :pod_ip_updated
+        MkitJob.publish(topic: :update_proxy_config, service_id: job.service_id)
+      else
+        MKItLogger.info("#{self.class} <<TODO>> job #{job.inspect}....")
+      end
+    end
+
+  end
+end
+

data/lib/mkit/workers/service_worker.rb

@@ -0,0 +1,27 @@
+module MKIt
+  class ServiceWorker < AWorker
+
+    def topics
+      %w{start_service stop_service update_service delete_service}
+    end
+
+    def do_the(job)
+      MKItLogger.info("#{self.class} <#{job.topic}> job #{job.inspect}....")
+      srv = Service.find(job.service_id)
+      case job.topic.to_sym
+      when :start_service
+        srv.start
+      when :stop_service
+        srv.stop
+      when :update_service
+        MKItLogger.info("#{self.class} <#{job.topic}> <<TODO>> job #{job.inspect}....")
+      when :delete_service
+        Service.destroy(job.service_id)
+      else
+        MKItLogger.info("#{self.class} <#{job.topic}> <<TODO>> job #{job.inspect}....")
+      end
+    end
+
+  end
+end
+

data/lib/mkit/workers/worker_manager.rb

@@ -0,0 +1,14 @@
+require 'mkit/workers/aworker'
+require 'mkit/workers/service_worker'
+require 'mkit/workers/pod_worker'
+require 'mkit/workers/haproxy_worker'
+
+module MKIt
+  class WorkerManager
+    def self.register_workers
+      ServiceWorker.new
+      PodWorker.new
+      HAProxyWorker.new
+    end
+  end
+end