mkit 0.2.0

data/db/schema.rb

@@ -0,0 +1,108 @@
+# This file is auto-generated from the current state of the database. Instead
+# of editing this file, please use the migrations feature of Active Record to
+# incrementally modify your database, and then regenerate this schema definition.
+#
+# This file is the source Rails uses to define your schema when running `bin/rails
+# db:schema:load`. When creating a new database, `bin/rails db:schema:load` tends to
+# be faster and is potentially less error prone than running all of your
+# migrations from scratch. Old migrations may fail to apply correctly if those
+# migrations use external dependencies or application code.
+#
+# It's strongly recommended that you check this file into your version control system.
+
+ActiveRecord::Schema[7.0].define(version: 2) do
+  create_table "dns_hosts", force: :cascade do |t|
+    t.string "service_id"
+    t.string "pod_id"
+    t.string "name"
+    t.string "ip"
+    t.string "ipv6"
+  end
+
+  create_table "leases", force: :cascade do |t|
+    t.string "pool_id"
+    t.string "service_id"
+    t.string "interface_name"
+    t.string "interface_type"
+    t.string "status"
+    t.string "ip"
+    t.datetime "expires_at", precision: nil
+  end
+
+  create_table "mkit_jobs", force: :cascade do |t|
+    t.string "topic"
+    t.string "service_id"
+    t.string "pod_id"
+    t.string "status"
+    t.string "retries"
+    t.string "payload"
+    t.datetime "created_at", precision: nil
+    t.datetime "updated_at", precision: nil
+  end
+
+  create_table "pods", force: :cascade do |t|
+    t.string "service_id"
+    t.string "pod_id"
+    t.string "name"
+    t.string "ip"
+    t.string "image"
+    t.string "status"
+  end
+
+  create_table "pools", force: :cascade do |t|
+    t.string "name"
+    t.string "ip"
+    t.string "range"
+    t.string "netmask"
+    t.string "status"
+    t.boolean "preferred", default: false
+  end
+
+  create_table "service_configs", force: :cascade do |t|
+    t.string "service_id"
+    t.string "key"
+    t.string "value"
+    t.string "ctype"
+    t.string "version"
+  end
+
+  create_table "service_ports", force: :cascade do |t|
+    t.string "service_id"
+    t.string "internal_port"
+    t.string "external_port"
+    t.string "mode"
+    t.string "load_bal"
+    t.string "version"
+  end
+
+  create_table "services", force: :cascade do |t|
+    t.string "name"
+    t.string "image"
+    t.string "qdm"
+    t.string "command"
+    t.integer "max_replicas", default: 1
+    t.integer "min_replicas", default: 1
+    t.string "lease_id"
+    t.string "pods_network", default: "mkit"
+    t.string "version"
+    t.string "status"
+    t.datetime "created_at", precision: nil, default: "2023-12-28 12:41:59"
+    t.datetime "updated_at", precision: nil
+    t.index ["name"], name: "services_name_id"
+  end
+
+  create_table "settings", force: :cascade do |t|
+    t.string "key"
+    t.string "value"
+  end
+
+  create_table "volumes", force: :cascade do |t|
+    t.string "service_id"
+    t.string "name"
+    t.string "path"
+    t.string "ctype"
+    t.string "status"
+    t.string "version"
+  end
+
+end

data/lib/mkit/app/controllers/mkitjobs_controller.rb

@@ -0,0 +1,37 @@
+require 'mkit/app/model/mkit_job'
+
+class MkitJobsController < MKIt::Server
+
+  # curl localhost:4567/mkitjobs
+  get '/mkitjobs' do
+    JSON.pretty_generate(JSON.parse(MkitJob.all.to_json))
+  end
+
+  get '/mkitjobs/:id' do
+    JSON.pretty_generate(JSON.parse(MkitJob.find(params[:id]).to_json))
+  end
+
+  put '/mkitjobs/:id' do
+    "Not impleemnted\n"
+  end
+
+  delete '/mkitjobs/:id' do
+    JSON.pretty_generate(JSON.parse(MkitJob.destroy(params[:id]).to_json))
+  end
+
+  delete '/mkitjobs/clean/all' do
+    MkitJob.destroy_all
+  end
+
+  post '/mkitjobs' do
+    xx = "no file"
+    if params[:file]
+      tempfile = params[:file][:tempfile]
+      xx= YAML.load(tempfile.read)
+      puts xx
+    end
+    JSON.pretty_generate(JSON.parse(xx.to_json))
+  end
+
+end
+

data/lib/mkit/app/controllers/pods_controller.rb

@@ -0,0 +1,30 @@
+class PodsController < MKIt::Server
+
+  get '/services/:service_id/pods' do
+    "Not implemented\n"
+  end
+
+  get '/services/:service_id/pods/:pod_id' do
+    "Not implemented\n"
+  end
+
+  put '/services/:service_id/pods/:pod_id' do
+    "Not implemented\n"
+  end
+
+  delete '/services/:service_id/pods/:pod_id' do
+    "Not implemented\n"
+  end
+
+  post '/services/:service_id/pods' do
+    xx = "no file"
+    if params[:file]
+      tempfile = params[:file][:tempfile]
+      xx= YAML.load(tempfile.read)
+      puts xx
+    end
+    JSON.pretty_generate(JSON.parse(xx.to_json))
+  end
+
+end
+

data/lib/mkit/app/controllers/services_controller.rb

@@ -0,0 +1,87 @@
+require 'mkit/app/model/service'
+require 'mkit/app/helpers/services_helper'
+
+class ServicesController < MKIt::Server
+  helpers MKIt::ServicesHelper
+
+  # curl localhost:4567/services
+  get '/services' do
+    if params[:verbose]
+      verbose = params[:verbose] == 'true'
+    else
+      verbose = false
+    end
+    if params[:id]
+      redirect "/services/#{params[:id]}"
+    elsif params[:name]
+      srv = find_by_id_or_name
+      format_response(srv, verbose)
+    else
+      services = Service.all
+      format_response(services, verbose)
+    end
+  end
+
+  get '/services/:id' do
+    srv = find_by_id_or_name
+    if request.env['CONTENT_TYPE'] == 'application/json'
+      resp = srv.to_json
+    else
+      resp = format_response(srv)
+    end
+    resp
+  end
+
+  # curl -X PUT localhost:4567/services/1  -F "file=@mkit/samples/mkit.yml"
+  put '/services/:id' do
+    srv = find_by_id_or_name
+    if params[:file]
+      tempfile = params[:file][:tempfile]
+      yaml = YAML.load(tempfile.read)
+      srv.update!(yaml.to_o)
+    end
+    format_response(srv)
+  rescue MKIt::BaseException => e
+    MKItLogger.debug e
+    error e.error_code, e.message
+  end
+
+  # curl -X DELETE localhost:4567/services/1
+  delete '/services/:id' do
+    srv = find_by_id_or_name
+    MkitJob.publish(topic: :delete_service, service_id: srv.id)
+    format_response(srv)
+  end
+
+  # curl -X POST localhost:4567/services  -F "file=@mkit/samples/mkit.yml"
+  post '/services' do
+    srv = "no file"
+    if params[:file]
+      tempfile = params[:file][:tempfile]
+      yaml = YAML.load(tempfile.read)
+      srv = Service.create(yaml.to_o)
+    end
+    format_response(srv)
+  rescue MKIt::BaseException => e
+    MKItLogger.debug e
+    error e.error_code, e.message
+  end
+
+  #
+  # operations
+  #
+  # curl -X PUT localhost:4567/services/1/start
+  put '/services/:id/start' do
+    srv = find_by_id_or_name
+    MkitJob.publish(topic: :start_service, service_id: srv.id)
+    format_response(srv)
+  end
+
+  # curl -X PUT localhost:4567/services/1/stop
+  put '/services/:id/stop' do
+    srv = find_by_id_or_name
+    MkitJob.publish(topic: :stop_service, service_id: srv.id)
+    format_response(srv)
+  end
+end
+

data/lib/mkit/app/helpers/docker_helper.rb

@@ -0,0 +1,75 @@
+require 'mkit/cmd_runner'
+
+module MKIt
+  module DockerHelper
+
+    # from ERB template
+    def create_instance(cmd)
+      MKIt::CmdRunner.run(cmd)
+    end
+
+    def start_instance(instance_id)
+      MKIt::CmdRunner.run("docker start #{instance_id}")
+    end
+
+    def stop_instance(instance_id)
+      MKIt::CmdRunner.run("docker stop #{instance_id}")
+    end
+
+    def remove_instance(instance)
+      MKIt::CmdRunner.run("docker rm -f #{instance}")
+    end
+
+    def inspect_instance(instance_id)
+      # this one does not work on ubunto MKIt::CmdRunner.run("docker inspect #{instance_id}")
+      x = %x{docker inspect #{instance_id}}
+      JSON.parse(x).first
+    end
+
+    def execute_local(instance_id, cmd)
+      MKIt::CmdRunner.run("docker exec -it #{instance_id} #{cmd}")
+    end
+
+    #
+    # network
+    #
+
+    def create_network(network_name)
+      MKIt::CmdRunner.run("docker network create #{network_name}")
+    end
+
+    def inspect_network(network_name)
+      x = MKIt::CmdRunner.run("docker network inspect #{network_name}")
+      JSON.parse(x).first
+    end
+
+    def dettach_network(network_id, instance_id)
+      MKIt::CmdRunner.run("docker network disconnect #{network_id} #{instance_id}")
+    end
+
+    def attach_network(network_id, instance_id)
+      MKIt::CmdRunner.run("docker network connect #{network_id} #{instance_id}")
+    end
+
+    def remove_network(network_id)
+      MKIt::CmdRunner.run("docker network rm #{network_id}")
+    end
+
+    #
+    # volumes
+    #
+
+    def create_volume(volume_name)
+      MKIt::CmdRunner.run("docker volume create #{volume_name}")
+    end
+
+    def delete_volume(volume_name)
+      MKIt::CmdRunner.run("docker volume rm #{volume_name}")
+    end
+
+    def inspect_volume(volume_name)
+      x = MKIt::CmdRunner.run("docker volume inspect #{volume_name}")
+      JSON.parse(x).first
+    end
+  end
+end

data/lib/mkit/app/helpers/erb_helper.rb

@@ -0,0 +1,18 @@
+require 'mkit/utils'
+
+module MKIt
+  module ERBHelper
+    def read_template(template)
+      root = MKIt::Utils.root
+      File.read("#{root}/lib/mkit/app/templates/#{template}.erb")
+    end
+  
+    def parse_template(template, data = {})
+       ERB.new(read_template(template)).result_with_hash(data)
+    end
+
+    def parse_model(template)
+      ERB.new(read_template(template))
+    end
+  end
+end

data/lib/mkit/app/helpers/haproxy.rb

@@ -0,0 +1,41 @@
+require 'mkit/app/model/setting'
+require 'mkit/config/config'
+
+module MKIt
+  module HAProxy
+    module_function
+    
+    def start
+      %x{#{MKIt::Config.mkit.haproxy.ctrl.start}}
+    end
+  
+    def stop
+      %x{#{MKIt::Config.mkit.haproxy.ctrl.stop}}
+    end
+
+    def restart
+      stop
+      start
+    end
+
+    def status
+      %x{#{MKIt::Config.mkit.haproxy.ctrl.status}}
+    end
+
+    def reload
+      %x{#{MKIt::Config.mkit.haproxy.ctrl.reload}}
+    end
+
+    def create_config_file(filename:, data:)
+      File.write("#{MKIt::Config.mkit.haproxy.config_dir}/#{filename}", data)
+    end
+
+    def delete_config_file(filename:)
+      begin
+        File.delete("#{MKIt::Config.mkit.haproxy.config_dir}/#{filename}")
+      rescue => e
+        puts e
+      end
+    end
+  end
+end

data/lib/mkit/app/helpers/interface_helper.rb

@@ -0,0 +1,17 @@
+module MKIt
+  module InterfaceHelper
+    module_function
+    def create(name:, ctype:)
+      %x{ip tuntap add mode #{ctype} #{name}}
+    end
+    def remove(name:, ctype:)
+      %x{ip tuntap del mode #{ctype} #{name}}
+    end
+    def up(name:, ip:, mask:)
+      %x{ifconfig #{name} #{ip}/#{mask} up}
+    end
+    def down(name:)
+      %x{ifconfig #{name} 0.0.0.0 down}
+    end
+  end
+end

data/lib/mkit/app/helpers/services_helper.rb

@@ -0,0 +1,54 @@
+module MKIt
+  module ServicesHelper
+    def str_template
+      "%-5s   %-18s   %-15s  %-25s  %-10s"
+    end
+
+    def header_template
+      ["id", "name", "addr", "ports", "status"]
+    end
+
+    def _format(template, data)
+      template % data
+    end
+
+    def format_response(data, verbose = false)
+      resp = []
+      header = _format(str_template, header_template)
+      resp << header
+      if data.respond_to?"each"
+        data.each { | srv |
+          ports = srv.service_port&.each.map { |p| "#{p.mode}/#{p.external_port}"}.join(",")
+          resp << _format(str_template, [srv.id, srv.name, srv.lease&.ip, ports, srv.status])
+          resp+=service_pods(srv) if verbose
+        }
+      else
+        ports = data.service_port&.each.map { |p| "#{p.mode}/#{p.external_port}"}.join(",")
+        resp << _format(str_template, [data.id, data.name, data.lease&.ip, ports, data.status])
+        resp+=service_pods(data)
+      end
+      resp << ""
+      resp.join("\n")
+    end
+
+    def service_pods(srv)
+      resp=[]
+      resp << "  pods"
+      resp << _format("    %-5s   %-15s   %-15s   %-15s   %-10s", ["id", "pod_id", "pod_name", "pod_ip", "status"])
+      srv.pod.each { |pod|
+        name = pod.name.nil? ? "" : pod.name[0..11]
+        pod_id = pod.pod_id.nil? ? "" : pod.pod_id[0..11]
+        resp << _format("    %-5s   %-15s   %-15s   %-15s   %-10s", [pod.id, pod_id, name, pod.ip, pod.status])
+      }
+      resp
+    end
+
+    def find_by_id_or_name
+      srv = Service.find_by_id(params[:id])
+      srv = Service.find_by_name(params[:id]) unless srv
+      error 404, "Couldn't find Service '#{params[:id]}'\n" unless srv
+      srv
+    end
+
+  end
+end

data/lib/mkit/app/mkit_server.rb

@@ -0,0 +1,8 @@
+require 'mkit/exceptions'
+
+module MKIt
+  class Server < Sinatra::Base
+    set :default_content_type, :json
+    # set :dump_errors, false
+  end
+end

data/lib/mkit/app/model/dns_host.rb

@@ -0,0 +1,11 @@
+require 'mkit/status'
+require 'mkit/exceptions'
+require 'mkit/app/model/service'
+require 'mkit/app/model/pod'
+require 'mkit/app/helpers/interface_helper'
+
+class DnsHost < ActiveRecord::Base
+  belongs_to :service
+  belongs_to :pod
+end
+

data/lib/mkit/app/model/lease.rb

@@ -0,0 +1,26 @@
+require 'mkit/status'
+require 'mkit/exceptions'
+require 'mkit/app/helpers/interface_helper'
+
+class Lease < ActiveRecord::Base
+  belongs_to :pool
+  belongs_to :service
+
+  before_destroy :down
+
+  def confirm
+    self.status = MKIt::PoolStatus::IN_USE
+  end
+
+  def up
+    MKIt::InterfaceHelper.create(name: interface_name, ctype: interface_type)
+    MKIt::InterfaceHelper.up(name: interface_name, ip: ip, mask: pool.netmask)
+    self.status = MKIt::PoolStatus::IN_USE
+  end
+
+  def down
+    MKIt::InterfaceHelper.down(name: interface_name)
+    MKIt::InterfaceHelper.remove(name: interface_name, ctype: interface_type)
+    self.status = MKIt::PoolStatus::RESERVED
+  end
+end

data/lib/mkit/app/model/mkit_job.rb

@@ -0,0 +1,48 @@
+
+require 'mkit/app/helpers/erb_helper'
+require 'mkit/app/helpers/docker_helper'
+require 'mkit/app/model/service'
+require 'mkit/status'
+
+class MkitJob < ActiveRecord::Base
+  before_destroy :clean_up
+
+  STATUS = %w{READY PROCESSING FAILED}
+
+  def self.take
+    MkitJob.where(status: 'READY').group(:service_id, :pod_id).take
+  end
+
+  def self.publish(*args)
+    job = MkitJob.new(args[0])
+    job.status = 'READY'
+    job.save
+    job
+  end
+
+  def processing!
+    self.status='PROCESSING'
+    self.save
+  end
+
+  def error!
+    self.status='FAILED'
+    self.save
+  end
+
+  def data=(opt={})
+    self.payload=opt.to_json
+  end
+
+  def data
+    JSON.parse(self.payload)
+  end
+
+  def done!
+    self.destroy
+  end
+
+  def clean_up
+  end
+
+end

data/lib/mkit/app/model/pod.rb

@@ -0,0 +1,95 @@
+require 'mkit/app/helpers/erb_helper'
+require 'mkit/app/helpers/docker_helper'
+require 'mkit/app/model/service'
+require 'mkit/status'
+
+class Pod < ActiveRecord::Base
+  belongs_to :service
+  has_one    :dns_host, dependent: :destroy
+
+  before_destroy :clean_up
+
+  include MKIt::ERBHelper
+  include MKIt::DockerHelper
+
+  def update_ip
+    new_ip = nil
+    tries = 5
+    while (new_ip.nil? && tries > 0) do
+        instance = self.properties.to_o
+        new_ip = instance.NetworkSettings.Networks[self.service.pods_network].IPAddress
+        sleep(1) if new_ip.nil?
+        tries = tries - 1
+    end
+    if self.ip != new_ip
+      self.ip = new_ip
+      self.update_dns
+      MkitJob.publish(topic: :pod_ip_updated, service_id: self.service.id, pod_id: self.id)
+    end
+    MKItLogger.info("docker ip updated [#{self.ip}]")
+    self.ip
+  end
+
+  def update_dns
+    self.dns_host ||= DnsHost.new(
+      service: self.service,
+      name:    "#{name}.#{self.service.name}",
+      ip:      self.ip
+    )
+    self.dns_host.ip = self.ip
+    self.dns_host.save
+  end
+
+  def properties
+    inspect_instance(self.name)
+  end
+
+  def set_status_from_docker
+    instance = self.properties
+    if self.properties
+      instance = instance.to_o
+      if instance.State.Running
+        self.status = MKIt::Status::RUNNING
+      else
+        self.status = MKIt::Status::STOPPED
+      end
+    else
+      self.status = MKIt::Status::PENDING
+    end
+    self.save
+    self.status
+  end
+
+  def parse
+    parse_model(MKIt::Templates::DOCKER_RUN).result(binding)
+  end
+
+  def start
+    if self.pod_id.nil?
+      docker_run = parse
+      MKItLogger.info("deploying docker pod, cmd [#{docker_run}]")
+      create_instance(docker_run)
+    else
+      pre_check
+
+      instance = self.properties.to_o
+      start_instance(self.name) unless instance.State.Running
+    end
+  end
+
+  def stop
+    pre_check
+    stop_instance(self.name)
+  end
+
+  def pre_check
+    raise MKIt::PodNotFoundException.new('no pod_name found') if self.name.nil?
+    raise MKIt::PodNotFoundException.new("no properties found for #{self.name}") if self.properties.nil?
+  end
+
+  def clean_up
+    remove_instance(self.name) unless self.pod_id.nil?
+    MkitJob.publish(topic: :pod_destroyed, service_id: self.service.id, data: {pod_id: self.id})
+  end
+end
+