mkit 0.2.0

data/lib/mkit/app/model/pool.rb

@@ -0,0 +1,60 @@
+require 'mkit/app/model/lease'
+require 'mkit/status'
+require 'mkit/exceptions'
+
+class Pool < ActiveRecord::Base
+  has_many :lease, dependent: :destroy
+
+  def check_status
+    if status == MKIt::PoolStatus::EXAUSTED
+      raise PoolExaustedException.new
+    end
+  end
+
+  def next_lease_ip
+    self.check_status
+    ips = range.split('-')
+    next_ip = ips[0]
+    next_ip = next_ip.to_i
+    lease.select(:status == MKIt::PoolStatus::IN_USE || :status == MKIt::PoolStatus::RESERVED).each { |l|
+      leased_ip = l.ip.split('.')[3]
+      leased_ip = leased_ip.to_i
+      if leased_ip >= next_ip
+        next_ip = leased_ip+1
+      end
+    }
+    if next_ip > ips[1].to_i
+      self.status = MKIt::PoolStatus::EXAUSTED
+      self.save
+      raise PoolExaustedException.new
+    end
+
+    ip_add = self.ip.split('.')
+
+    "#{ip_add[0]}.#{ip_add[1]}.#{ip_add[2]}.#{next_ip}"
+  end
+
+  def request(service:, status:)
+    lease_ip = next_lease_ip
+    idx = lease_ip.split('.')[3]
+    new_lease = Lease.new(
+      pool: self,
+      service: service,
+      interface_name: "vmkit#{idx}",
+      interface_type: 'tun',
+      status: status,
+      ip: lease_ip
+    )
+    new_lease.save
+    new_lease
+  end
+
+  def request_for(service)
+     request(service: service, status:  MKIt::PoolStatus::IN_USE)
+  end
+
+  def reserve_for(service)
+     request(service: service, status:  MKIt::PoolStatus::RESERVED)
+  end
+end
+

data/lib/mkit/app/model/service.rb

@@ -0,0 +1,266 @@
+require 'mkit/app/model/volume'
+require 'mkit/app/model/service_port'
+require 'mkit/app/model/service_config'
+require 'mkit/app/model/pod'
+require 'mkit/app/helpers/erb_helper'
+require 'mkit/app/helpers/docker_helper'
+require 'mkit/mkit_interface'
+require 'mkit/status'
+require 'mkit/utils'
+require 'mkit/ctypes'
+require 'mkit/app/model/pool'
+require 'mkit/app/model/service'
+require 'mkit/app/model/pod'
+require 'mkit/app/model/dns_host'
+require 'mkit/app/helpers/erb_helper'
+require 'mkit/app/helpers/docker_helper'
+require 'mkit/app/helpers/haproxy'
+
+#
+class Service < ActiveRecord::Base
+  has_many   :pod, dependent: :destroy
+  has_many   :volume, dependent: :destroy
+  has_many   :service_config, dependent: :destroy
+  has_many   :service_port, dependent: :destroy
+
+  has_one  :lease, dependent: :destroy
+  has_one  :dns_host, dependent: :destroy
+
+  before_destroy :clean_up
+
+  validates :name, uniqueness: true
+  validates :name, presence: true
+
+  include MKIt::ERBHelper
+  include MKIt::DockerHelper
+
+  def self.create(yaml)
+    config = yaml["service"]
+    raise MKIt::ServiceAlreadyExists.new unless Service.find_by_name(config.name).nil?
+
+    srv = Service.new(
+      name: config.name,
+      version: 1,
+      image: config.image,
+      command: config.command,
+      status: MKIt::Status::CREATING
+    )
+
+    # docker network
+    if config.network.nil? || config.network.empty?
+      srv.pods_network="mkit"
+    else
+      srv.pods_network=config.network
+    end
+
+    # reserve pool ip
+    srv.lease = Pool.find_by_name(MKIt::Utils.me).reserve_for(srv)
+
+    srv.dns_host = DnsHost.new(
+      service: srv,
+      name: srv.name,
+      ip: srv.lease.ip
+    )
+
+    # create service network
+    srv.deploy_network
+
+    # configure
+    srv.configure(config)
+    #
+    srv.status = MKIt::Status::CREATED
+    srv.save
+    data = { service_id: srv.id, version: srv.version }
+    # create pod
+    (1..srv.min_replicas).each { |i|
+      MkitJob.publish(topic: :create_pod_saga, service_id: srv.id, data: data)
+    }
+    srv
+  end
+
+  def configure(config)
+    self.image = config.image if config.image != self.image
+    self.command = config.command if config.command != self.command
+
+    unless config.resources.nil?
+      self.max_replicas = config.resources.max_replicas unless config.resources.max_replicas.nil? || config.resources.max_replicas < 1
+      self.min_replicas = config.resources.min_replicas unless config.resources.min_replicas.nil? || config.resources.min_replicas < 1
+    else
+      self.min_replicas = 1
+      self.max_replicas = 1
+    end
+    self.max_replicas = self.min_replicas if self.min_replicas > self.max_replicas
+
+    # haproxy ports
+    self.service_port = []
+    config.ports&.each do |p|
+      port = ServicePort.create(service: self, config: p)
+      self.service_port << port
+    end
+
+    # volumes
+    self.volume = []
+    config.volumes&.each { |volume|
+      self.add_volume(volume)
+    }
+    # environment
+    self.service_config=[]
+    config.environment&.each_pair { |key,value|
+      self.add_service_config(key, value)
+    }
+    self.volume.each { | volume |
+      volume.deploy
+    }
+  end
+
+  def update!(yaml)
+    config = yaml["service"]
+    raise MKIt::ServiceNameMismatch.new unless config.name == self.name
+    self.version+=1
+    self.configure(config)
+
+    # start new pod, destroy old pod...
+    self.pod.each { |pod| MkitJob.publish(topic: :destroy_pod, pod_id: pod.id, data: {}) }
+    # create pod
+    data = { service_id: self.id, version: self.version }
+    (1..self.min_replicas).each { |i|
+      MkitJob.publish(topic: :create_pod_saga, service_id: self.id, data: data)
+    }
+    self.save
+  end
+
+  def create_pods_network
+    netw = inspect_network(self.pods_network)
+    create_network(self.pods_network) if netw.nil?
+  end
+
+  def deploy_network
+    # create service interface...
+    self.lease.confirm
+    self.lease.up
+    # ...and pods network
+    self.create_pods_network
+  end
+
+  def add_volume(volume_config)
+    v = Volume.create(self, volume_config)
+    self.volume << v
+    v
+  end
+
+  def add_service_config(key, value)
+    v = ServiceConfig.create(service: self, key: key, value: value)
+    self.service_config << v
+    v
+  end
+
+  def current_configs
+    self.service_config&.select{ |x| x.ctype == MKIt::CType::ENVIRONMENT.to_s && x.version == self.version}
+  end
+
+  def current_ports
+    self.service_port&.select{ |x| x.version == self.version}
+  end
+
+  def my_dns
+    MKIt::Interface.ip
+  end
+
+  def update_status!
+    combined_status = nil
+    self.pod.each { |pod|
+      child_status = pod.set_status_from_docker
+      if combined_status
+        case combined_status
+        when MKIt::Status::RUNNING
+          case child_status
+          when MKIt::Status::STOPPED || MKIt::Status::PENDING
+            combined_status = MKIt::Status::DEGRATED
+          end
+        when MKIt::Status::STOPPED
+          case child_status
+          when MKIt::Status::RUNNING || MKIt::Status::PENDING
+            combined_status = MKIt::Status::DEGRATED
+          end
+        when MKIt::Status::PENDING
+          case child_status
+          when MKIt::Status::RUNNING || MKIt::Status::STOPPED
+            combined_status = MKIt::Status::DEGRATED
+          end
+        end
+      else
+        combined_status = child_status
+      end
+    }
+    combined_status = MKIt::Status::CREATING unless combined_status
+    self.status = combined_status
+    self.save
+    self.status
+  end
+
+  #
+  # ha proxy configs & template
+  #
+  def public_ports
+    self.service_port.each.map{|p| p.external_port}.uniq
+  end
+
+  def ports_by_external(external_port)
+    self.service_port.where('external_port = ?', external_port)
+  end
+
+  def ports_mode_by_external(external_port)
+    ports = self.service_port.where('external_port = ?', external_port).first
+    ports.mode if ports
+  end
+
+  def update_proxy
+    MkitJob.publish(topic: :update_proxy_config, application_id: self.id, data: proxy_config)
+  end
+
+  def proxy_config
+    # config
+    haproxy = parse
+    my_addr = self.lease.ip.split('.')[3]
+    filename = "#{'%04i' % my_addr.to_i}_#{self.name}.cfg"
+    MKItLogger.debug("haproxy config file: #{filename}")
+    {filename: filename, data: haproxy}
+  end
+
+  def parse
+    parse_model(MKIt::Templates::HAPROXY).result(binding)
+  end
+
+  def clean_up
+    my_addr = self.lease.ip.split('.')[3]
+    filename = "#{'%04i' % my_addr.to_i}_#{self.name}.cfg"
+    MkitJob.publish(topic: :destroy_proxy_config, data: {filename: filename})
+  end
+
+  #
+  # ctrl
+  #
+  def start
+    self.pod.each { |p|
+      MkitJob.publish(topic: :start_pod, service_id: self.id, pod_id: p.id)
+    }
+  end
+
+  def stop
+    self.pod.each { |p|
+      MkitJob.publish(topic: :stop_pod, service_id: self.id, pod_id: p.id)
+    }
+  end
+
+  def as_json(options = {})
+    srv = super
+    a=[:pod, :volume, :service_config, :service_port]
+    a.each { | k | 
+      srv[k] ||= []
+      self.send(k).each { |v| 
+        srv[k] << v.as_json
+      }
+    }
+    srv
+  end
+end

data/lib/mkit/app/model/service_config.rb

@@ -0,0 +1,16 @@
+require 'mkit/ctypes'
+
+class ServiceConfig < ActiveRecord::Base
+  belongs_to :service
+
+  def self.create(service:, key:, value:, ctype: MKIt::CType::ENVIRONMENT)
+    ServiceConfig.new(
+      service: service,
+      key: key,
+      value: value,
+      version: service.version,
+      ctype: ctype
+    )
+  end
+end
+

data/lib/mkit/app/model/service_port.rb

@@ -0,0 +1,30 @@
+require 'mkit/app/model/service'
+
+class ServicePort < ActiveRecord::Base
+  belongs_to :service
+
+  def self.create(service:, config:)
+    sp = ServicePort.new(service: service, version: service.version)
+    sp.parse_config(config)
+    sp
+  end
+
+  # haproxy support for port range - leave src blank
+  # service:
+  #   ports:
+  #     # src:dest:tcp|http:load-balancing
+  #     - 5532:5432:tcp:round_robin
+  # model:
+  #  service_ports:
+  #     - external: 5432
+  #       internal: 5432
+  #       mode: tcp|http
+  #       load_bal:
+  def parse_config(config)
+    ports = config.split(':')
+    self.external_port = ports[0]
+    self.internal_port = ports[1]
+    self.mode = ports[2]
+    self.load_bal = ports[3]
+  end
+end

data/lib/mkit/app/model/setting.rb

@@ -0,0 +1 @@
+class Setting < ActiveRecord::Base; end

data/lib/mkit/app/model/volume.rb

@@ -0,0 +1,53 @@
+require 'mkit/ctypes'
+require 'mkit/app/helpers/docker_helper'
+require 'fileutils'
+
+class Volume < ActiveRecord::Base
+  belongs_to :service
+  before_destroy :clean_up
+
+  def self.create(service, volume)
+        case volume
+        when /^docker:\/\//
+          ctype = MKIt::CType::DOCKER_STORAGE
+          paths = volume[9..].split(':')
+          # vname="#{service.name}.#{service.application.name}.#{paths[0]}"
+          vname = paths[0]
+        when /^\//
+          ctype = MKIt::CType::LOCAL_STORAGE
+          paths = volume.split(':')
+          vname = paths[0]
+        end
+        Volume.new(
+          service: service,
+          name: vname,
+          path: paths[1],
+          ctype: ctype
+        )
+  end
+
+  def deploy
+    create_volume
+  end
+
+  def create_volume
+    case self.ctype
+    when MKIt::CType::DOCKER_STORAGE
+      MKIt::DockerHelper.create_volume(self.name)
+    when MKIt::CType::LOCAL_STORAGE
+      # nop
+    end
+  end
+
+  def delete_volume
+    case self.ctype
+    when MKIt::CType::DOCKER_STORAGE
+      MKIt::DockerHelper.delete_volume(self.name)
+    end
+  end
+
+  def clean_up
+    # nop
+  end
+end
+

data/lib/mkit/app/templates/docker/docker_run.sh.erb

@@ -0,0 +1 @@
+docker run -d --name <%=name%>  <%service.service_config&.select{ |x| x.ctype == MKIt::CType::ENVIRONMENT.to_s }.each { |env|%><%=" -e #{env.key}=\"#{env.value}\""%><%}%> <%service.volume&.each { |vol|%><%=" -v \"#{vol.name}:#{vol.path}\""%><%}%>  --network <%=service.pods_network%> --dns <%=service.my_dns%> <%=service.image%> <%=service.command unless service.command.nil?%>

data/lib/mkit/app/templates/haproxy/0000_defaults.cfg

@@ -0,0 +1,23 @@
+global
+        log 127.0.0.1   local2
+        maxconn 4096
+        user nobody
+        group nobody
+
+defaults
+    log                     global
+    mode http
+    option                  tcplog
+    retries                 3
+    timeout http-request    10s
+    timeout queue           1m
+    timeout connect         10s
+    timeout client          1m
+    timeout server          1m
+    timeout http-keep-alive 10s
+    timeout check           10s
+    maxconn                 3000
+    stats enable
+    stats auth someuser:somepassword
+    stats uri /haproxy_stats
+

data/lib/mkit/app/templates/haproxy/xapp_haproxy.cfg.erb

@@ -0,0 +1,30 @@
+#
+# MKIt generated file
+#
+<% public_ports.each { |external_port|%>
+#
+# start <%=name%>-<%=external_port%>
+#
+frontend <%=name%>-<%=external_port%>-front
+  bind <%=lease.ip%>:<%=external_port%>
+  mode <%=ports_mode_by_external(external_port)%>
+  #
+  use_backend <%=name%>-<%=external_port%>-back
+
+backend <%=name%>-<%=external_port%>-back
+   mode <%=ports_mode_by_external(external_port)%>
+   #balance leastconn
+   balance  roundrobin
+ <%if ports_mode_by_external(external_port) == 'http'%>
+   option httpclose
+   option forwardfor
+   cookie JSESSIONID prefix
+ <%end%>
+<%ports_by_external(external_port).each { |port| %>
+   <%port.service.pod.each { | pod | %>
+   server <%=pod.name%>  <%=pod.ip%>:<%=port.internal_port%> <%if port.mode == 'http'%>cookie A<%end%>  check<%}%><%}%>
+# 
+# end of <%=name%>-<%=external_port%>
+#
+<%}%>
+

data/lib/mkit/cmd_runner.rb

@@ -0,0 +1,27 @@
+require 'pty'
+require 'mkit/exceptions'
+
+module MKIt
+  class CmdRunner
+    def self.run(cmd)
+      result=''
+      begin
+        shell = PTY.spawn( cmd )
+        begin
+          shell[0].each { |line| result << line.strip! }
+        rescue Errno::EIO
+          # nothing
+        ensure
+          shell[0].close
+        end
+        shell[1].close
+        Process.wait(shell[2])
+      rescue PTY::ChildExited
+        # nothing
+      end
+      raise CmdRunnerException.new("command '#{cmd[0..30]}...' returned an error response") if !$?.nil? && $?.exitstatus != 0
+      result
+    end
+  end
+end
+

data/lib/mkit/config/config.rb

@@ -0,0 +1,18 @@
+require 'yaml'
+#
+# MKIt::Config.load_yml!('samples/mkit.yml')
+# MKIt::Config.application.services
+#  requires Hash.to_o
+module MKIt
+  module Config
+    extend self
+    def load_yml!(path)
+      @config = YAML.load(File.new(path).read).to_o
+    end
+    # 
+    def method_missing(name,*args)
+      return @config.send(name,*args)
+      super.method_missing name
+    end
+  end
+end

data/lib/mkit/config/environment.rb

@@ -0,0 +1,26 @@
+require 'bundler/setup'
+require 'dry-container'
+require 'sinatra/activerecord'
+require 'rubydns'
+require 'sinatra'
+
+require_relative 'initializers/001_hash'
+require_relative 'initializers/002_openstruct'
+
+SOCKET_PATH = File.expand_path('/tmp/app.sock')
+
+# sinatra conf
+configure do
+  # set :public_folder, 'public'
+  # set :views, 'app/views'
+  set :server, :thin
+  # enable/disable the built-in web server
+  # set :run, :false
+  # server hostname or IP address
+  # set :bind, SOCKET_PATH, "localhost:4567"
+  # set :port, 4567
+  #enable :sessions
+  #set :session_secret, 'password_security'
+  set :default_content_type, :json
+end
+

data/lib/mkit/config/initializers/001_hash.rb

@@ -0,0 +1,11 @@
+class Hash
+  def to_o
+    JSON.parse to_json, object_class: OpenStruct
+  end
+
+  def to_os
+    self.each_with_object(OpenStruct.new) do |(key, val), memo|
+      memo[key] = val.is_a?(Hash) ? val.to_os : val
+    end
+  end
+end

data/lib/mkit/config/initializers/002_openstruct.rb

@@ -0,0 +1,7 @@
+class OpenStruct
+  def to_hash
+    self.each_pair.with_object({}) do |(key, value), hash|
+      hash[key] = value.is_a?(OpenStruct) ? value.to_hash : value
+    end
+  end
+end

data/lib/mkit/config/load_default_configs.rb

@@ -0,0 +1,29 @@
+require 'mkit/app/model/setting'
+require 'mkit/app/model/pool'
+require 'mkit/config/config'
+require 'fileutils'
+require 'mkit/utils'
+
+module MKIt
+  module Initializers
+
+    def self.load_my_configuration(config_dir: MKIt::Utils.config_dir)
+      MKIt::Utils.log.info "loading configurations from '#{config_dir}'..."
+      MKIt::Config.load_yml!("#{config_dir}/mkit_config.yml")
+    end
+
+    def self.load_default_configs
+      if Pool.find_by_name(MKIt::Utils.me).nil?
+        st = Pool.new(
+          name: MKIt::Utils.me,
+          ip: MKIt::Config.mkit.my_network.ip,
+          range: '10-200',
+          netmask: '24',
+          preferred: true
+        )
+        st.save
+      end
+    end
+  end
+end
+

data/lib/mkit/config/the_config.yml

@@ -0,0 +1,3 @@
+mkit:
+  public_pool: 10.20.30.1/24
+

data/lib/mkit/ctypes.rb

@@ -0,0 +1,31 @@
+#
+#
+module MKIt
+    class MKItCType
+      def initialize(status)
+        @status = status.to_s
+      end
+
+      def to_s
+        @status.downcase
+      end
+    end
+  
+    module CType
+        ENVIRONMENT = MKIt::MKItCType.new(:environment)
+        DOCKER_STORAGE = MKIt::MKItCType.new(:docker)
+        LOCAL_STORAGE = MKIt::MKItCType.new(:local)
+
+        NETWORK_SPECIFIC = MKIt::MKItCType.new(:specific)
+        NETWORK_BRIDGE = MKIt::MKItCType.new(:bridge)
+        TUN_INTERFACE = MKIt::MKItCType.new(:tun)
+        TAP_INTERFACE = MKIt::MKItCType.new(:tap)
+    end
+
+    module Templates
+      DOCKER_RUN = 'docker/docker_run.sh'
+      DOCKER_BUILD = 'docker/docker_build.sh'
+      HAPROXY = 'haproxy/xapp_haproxy.cfg'
+      HAPROXY_DEFAULTS = 'haproxy/0000_defaults.cfg'
+    end
+end