mkit 0.2.0

data/lib/mkit.rb

@@ -0,0 +1,158 @@
+require 'rubygems'
+require 'bundler'
+require 'json'
+require 'yaml'
+require 'optparse'
+require 'ostruct'
+require 'timeout'
+require 'net/http'
+require 'sinatra/base'
+require 'mkit/config/environment'
+require 'mkit/app/mkit_server'
+require 'mkit/config/load_default_configs'
+require_relative "mkit/version"
+require_relative 'mkit/mkit_interface'
+require_relative 'mkit/mkit_dns'
+require_relative 'mkit/docker_listener'
+require 'mkit/app/helpers/haproxy'
+require 'mkit/app/controllers/services_controller'
+require 'mkit/app/controllers/mkitjobs_controller'
+require 'mkit/mkit_interface'
+require 'mkit/mkit_dns'
+require 'mkit/job_manager'
+require 'mkit/workers/worker_manager'
+require 'mkit/sagas/saga_manager'
+require 'mkit/docker_listener'
+require 'mkit/app/helpers/haproxy'
+require 'active_record/tasks/database_tasks'
+require 'mkit/utils'
+
+MKItLogger = Console.logger
+
+module MKIt
+  class Error < StandardError; end
+  include ActiveRecord::Tasks
+
+  System = Dry::Container.new
+
+  def self.configure(options:)
+    @root = MKIt::Utils.root
+    MKItLogger.debug!
+    #
+    # config dir
+    if ENV["RACK_ENV"] != "development"
+      @config_dir = options[:config_dir].nil? ? '/etc/mkit' : options[:config_dir]
+    else
+      @config_dir = options[:config_dir].nil? ? "#{@root}/config" : options[:config_dir]
+    end
+    MKIt::Utils.set_config_dir(@config_dir)
+    # create dirs
+    if ENV["RACK_ENV"] != "development" || !options[:config_dir].nil?
+      check_config_files = false
+      if ! File.exists?(@config_dir)
+        FileUtils.mkdir_p(@config_dir)
+        check_config_files = true
+      end
+      FileUtils.mkdir_p('/var/lib/mkitd') unless File.exists?('/var/lib/mkitd')
+      FileUtils.cp( "#{@root}/config/mkit_config.yml", @config_dir) unless File.exists?("#{@config_dir}/mkit_config.yml")
+      FileUtils.cp( "#{@root}/config/database.yml", @config_dir) unless File.exists?("#{@config_dir}/database.yml")
+      FileUtils.cp( "#{@root}/config/mkitd_config.sh", @config_dir) unless File.exists?("#{@config_dir}/mkitd_config.sh")
+      if check_config_files
+        MKItLogger.info "Configuration files copied to #{@config_dir}. Please check it and restart."
+        exit
+      end
+    end
+    #
+    # load configuration
+    MKIt::Initializers.load_my_configuration
+    #
+    # run config based tasks
+    FileUtils.mkdir_p(MKIt::Config.mkit.haproxy.config_dir)
+    # ...haproxy defaults file
+    FileUtils.cp(
+      "#{MKIt::Utils.root}/lib/mkit/app/templates/haproxy/0000_defaults.cfg",
+      MKIt::Config.mkit.haproxy.config_dir
+    ) unless File.exists?("#{MKIt::Config.mkit.haproxy.config_dir}/0000_defaults.cfg")
+    #
+    #  conn = { adapter: "sqlite3", database: ":memory:" }
+    #  ActiveRecord::Base.establish_connection(conn)
+    #  include ActiveRecord::Tasks
+    #  DatabaseTasks.database_configuration = YAML.load_file('my_database_config.yml')
+    #  DatabaseTasks.db_dir = 'db'
+    #
+    DatabaseTasks.database_configuration = MKIt::Utils.load_db_config
+    DatabaseTasks.env=MKIt::Config.mkit.database.env
+    DatabaseTasks.migrations_paths=[ "#{@root}/db/migrate" ]
+    DatabaseTasks.db_dir="db"
+    DatabaseTasks.root=@root
+  end
+
+  def self.establish_db_connection
+    #
+    ActiveRecord::Base.establish_connection(DatabaseTasks.database_configuration[DatabaseTasks.env])
+    ActiveRecord::Base.connection.migration_context.migrations_paths.clear
+    ActiveRecord::Base.connection.migration_context.migrations_paths << "#{@root}/db/migrate"
+    #
+    MKItLogger.debug "database_tasks migration paths     #{DatabaseTasks.migrations_paths}"
+    MKItLogger.debug "active_record_base migration_paths #{ActiveRecord::Base.connection.migration_context.migrations_paths.inspect}"
+    MKItLogger.debug "active_record_base configurations  #{ActiveRecord::Base.configurations.inspect}"
+    MKItLogger.debug "active_record_base conn_db_config  #{ActiveRecord::Base.connection_db_config.inspect}"
+  end
+
+  def self.migrate
+    ActiveRecord::Base.connection.migration_context.migrate if ActiveRecord::Base.connection.migration_context.needs_migration?
+  end
+
+  def self.restore_operation
+    MKItLogger.info "restoring operations..."
+    # create interfaces of deployed apps  otherwise haproxy won't start
+    Service.all.each { | srv |
+      srv.deploy_network
+      srv.update_status!
+    }
+  end
+
+  def self.startup(options: {})
+    self.configure(options: options)
+    self.establish_db_connection
+    self.migrate
+
+    MKIt::Initializers.load_default_configs
+    MKIt::Interface.up
+
+    System.register(:job_manager, memoize: true) {
+      MKIt::JobManager.new
+    }
+    System.register(:mkit_dns, memoize: true) {
+      Thread.new {
+        dns = MKIt::DNS.new
+        dns.run
+      }
+    }
+    System.register(:docker_listener, memoize: true) {
+      MKIt::DockerListener.new
+    }
+    # watchdog feature is to be re-evaluated
+    # System.register(:watchdog, memoize: true) {
+    #  MKIt::WatchdogManager.new
+    # }
+
+    # register workers
+    WorkerManager.register_workers    
+    SagaManager.register_workers
+    #
+    System[:job_manager].start
+    System[:docker_listener].start
+    # watchdog feature is to be re-evaluated
+    # System[:watchdog].start
+    System[:mkit_dns].run
+    #
+    self.restore_operation
+    #
+    MKItLogger.debug "restarting proxy..."
+    MKIt::HAProxy.restart
+    MKItLogger.info "MKIt is up and running!"
+  end
+
+end
+

data/mkit.gemspec

@@ -0,0 +1,40 @@
+# encoding: UTF-8
+
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "mkit/version"
+
+Gem::Specification.new do |s|
+        s.name         = 'mkit'
+        s.summary      = 'micro kubernets'
+        s.bindir       = 'bin'
+        s.homepage     = 'http://vars.pt'
+        s.license      = 'Apache-2.0'
+        s.rubyforge_project = ''
+        s.description  = 'micro kubernets impl'
+        # s.require_paths = ["."]
+        s.author      = 'Vasco Santos'
+        s.email        = ['valexsantos@gmail.com']
+        s.version      = MKIt::VERSION
+        s.platform     = Gem::Platform::RUBY
+        s.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+          `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features|TODO|db/development.sqlite)}) }
+        end
+        s.executables  << 'mkitd'
+        s.executables  << 'mkitc'
+        s.add_runtime_dependency 'net-ping', '~> 2.0', '>= 2.0.8'
+        s.add_runtime_dependency 'dry-container', '~> 0.9', '>= 0.9.0'
+        s.add_runtime_dependency 'sqlite3', '~> 1.5', '>= 1.5.4'
+        s.add_runtime_dependency 'standalone_migrations', '~> 7.1', '>= 7.1.0'
+        s.add_runtime_dependency 'sinatra-activerecord', '~> 2.0', '>= 2.0.26'
+        s.add_runtime_dependency 'rack',            '~> 2.2', '>= 2.2.5'
+        s.add_runtime_dependency 'rack-protection', '~> 3.0',  '>= 3.0.5'
+        s.add_runtime_dependency 'rack-test',       '~> 2.0',  '>= 2.0.2'
+        s.add_runtime_dependency 'pry',             '~> 0.14', '>= 0.14.2'
+        s.add_runtime_dependency 'rubydns',         '~> 2.0',  '>= 2.0.2'
+        s.add_runtime_dependency 'async-dns',       '~> 1.3',  '>= 1.3.0'
+        s.add_runtime_dependency 'sinatra',         '~> 3.0',  '>= 3.0.5'
+        s.add_runtime_dependency 'thin',            '~> 1.8',  '>= 1.8.1'
+        s.add_runtime_dependency 'net_http_unix',   '~> 0.2',  '>= 0.2.2'
+end
+

data/mkitd

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+$LOAD_PATH.unshift File.expand_path('lib')
+#
+ENV['SINATRA_ENV'] ||= "development"
+ENV['RACK_ENV'] ||= "development"
+
+load 'bin/mkitd'
+

data/samples/apps/postgres.yml

@@ -0,0 +1,22 @@
+#
+service:
+  name: postgres
+  network: bridge
+  image: "postgres:14"
+  ports:
+    - 5432:5432:tcp:round_robin
+  environment:
+    POSTGRES_USER: postgres
+    POSTGRES_PASSWORD: postgres
+    POSTGRES_DB: postgres
+  volumes:
+    - docker://postgresql_14_data:/var/lib/postgresql/data
+  restart: always
+  resources:
+    max_replicas: 1
+    min_replicas: 1
+  limits:
+    cpu: 1000m
+    ram: 1Gi
+
+

data/samples/apps/rabbitmq.yml

@@ -0,0 +1,19 @@
+#
+service:
+  name: rabbitmq
+  image: rabbitmq:3-management-alpine
+  network: bridge # docker network
+  ports:
+    - 5672:5672:tcp:round_robin
+    - 80:15672:http:round_robin
+  resources:
+    max_replicas: 1
+    min_replicas: 1
+  volumes:
+    - docker://mkit_rabbitmq_data:/var/lib/rabbitmq
+    - docker://mkit_rabbitmq_logs:/var/log/rabbitmq
+  environment:
+    RABBITMQ_DEFAULT_USER: admin
+    RABBITMQ_DEFAULT_PASS: admin
+    RABBITMQ_DEFAULT_VHOST: mkit
+

data/samples/daemontools/log/run

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+# generic daemontools log run script
+
+# Copyright 2017-2018 Vasco Santos <valexsantos[at]gmail[dot]com>
+# All rights reserved.
+#
+# Redistribution and use of this script, with or without modification, is
+# permitted provided that the following conditions are met:
+#
+# 1. Redistributions of this script must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ''AS IS'' AND ANY EXPRESS OR IMPLIED
+# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+# EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#
+# cfgs
+#
+LOGUSER=nobody
+APP=mkitd
+LOGDIR=/var/log/$APP
+LOG_FILE_HISTORY=20
+LOG_FILE_SIZE=100000
+##
+# doit
+##
+PATH=/opt/daemontools/bin:/usr/local/bin:/usr/bin:/bin
+export PATH
+# create log dir
+[ -d $LOGDIR ] || mkdir $LOGDIR
+# set owner
+chown -R $LOGUSER $LOGDIR
+# go
+exec 2>&1
+exec setuidgid $LOGUSER multilog t s$LOG_FILE_SIZE n$LOG_FILE_HISTORY $LOGDIR

data/samples/daemontools/run

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+# mkitd daemontools run script
+
+# Copyright 2017-2018 Vasco Santos <valexsantos[at]gmail[dot]com>
+# All rights reserved.
+#
+# Redistribution and use of this script, with or without modification, is
+# permitted provided that the following conditions are met:
+#
+# 1. Redistributions of this script must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ''AS IS'' AND ANY EXPRESS OR IMPLIED
+# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+# EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+exec 2>&1
+
+source /etc/mkit/mkitd_config.sh
+
+CMD="mkitd $OPTIONS"
+echo "command-line $CMD"
+
+##
+# Doit
+##
+
+exec $CMD
+
+##
+# err
+echo ERROR: command did not run correctly
+sleep 5
+exit 1

data/samples/systemd/mkitd.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=Kids kube
+After=network-online.target
+
+[Service]
+EnvironmentFile=-/etc/mkit/mkitd_config.sh
+ExecStart=mkitd $OPTIONS
+Restart=always
+
+[Install]
+WantedBy=multi-user.target
+