mkit 0.9.1 → 0.10.0

data/lib/mkit/app/model/service.rb

@@ -1,4 +1,5 @@
 require 'mkit/app/model/volume'
+require 'mkit/app/model/ingress'
 require 'mkit/app/model/service_port'
 require 'mkit/app/model/service_config'
 require 'mkit/app/model/pod'
@@ -21,6 +22,7 @@ class Service < ActiveRecord::Base
 
   has_one  :lease, dependent: :destroy
   has_one  :dns_host, dependent: :destroy
+  has_one  :ingress, dependent: :destroy
 
   before_destroy :clean_up
 
@@ -63,7 +65,9 @@ class Service < ActiveRecord::Base
       data = { service_id: srv.id, version: srv.version }
       # create pod
       (1..srv.min_replicas).each { |i|
-        MkitJob.publish(topic: :create_pod_saga, service_id: srv.id, data: data)
+        pd = Pod.new( status: MKIt::Status::CREATED, name: SecureRandom.uuid.gsub('-','')[0..11])
+        srv.pod << pd
+        MkitJob.publish(topic: :create_pod_saga, data: {pod_name: pd.name})
       }
       srv
     end
@@ -90,12 +94,8 @@ class Service < ActiveRecord::Base
     end
     self.create_pods_network
 
-    # haproxy ports
-    self.service_port = []
-    config.ports&.each do |p|
-      port = ServicePort.create(service: self, config: p)
-      self.service_port << port
-    end
+    # haproxy config
+    self.ingress = Ingress.create(config.ingress)
 
     # volumes
     self.volume = []
@@ -119,12 +119,14 @@ class Service < ActiveRecord::Base
       self.version+=1
       self.configure(config)
 
-      # start new pod, destroy old pod...
-      self.pod.each { |pod| MkitJob.publish(topic: :destroy_pod, pod_id: pod.id, data: {}) }
+      # destroy old pods...
+      self.pod.destroy_all
       # create pod
       data = { service_id: self.id, version: self.version }
       (1..self.min_replicas).each { |i|
-        MkitJob.publish(topic: :create_pod_saga, service_id: self.id, data: data)
+        pd = Pod.new( status: MKIt::Status::CREATED, name: SecureRandom.uuid.gsub('-','')[0..11])
+        self.pod << pd
+        MkitJob.publish(topic: :create_pod_saga, data: {pod_name: pd.name})
       }
       self.save
     end
@@ -164,6 +166,9 @@ class Service < ActiveRecord::Base
     MKIt::Interface.ip
   end
 
+  # TODO
+  #  refactor to remove it from db model and check if it is needed
+  #  this will be the pod status
   def update_status!
     combined_status = nil
     self.pod.each { |pod|
@@ -252,6 +257,7 @@ class Service < ActiveRecord::Base
   end
   def to_h(options = {})
     details = options[:details] || false
+
     yaml = {}
     yaml['service'] = {}
     srv = yaml['service']
@@ -269,18 +275,10 @@ class Service < ActiveRecord::Base
         srv['pods'] << p.to_h
       }
     end
-    srv['ports'] = []
-    self.service_port.each { |p|
-      "#{p.external_port}:#{p.internal_port}:#{p.mode}:#{p.load_bal}".tap { |x|
-        if p.ssl == 'true'
-          x << ':ssl'
-          if !p.crt.nil? && p.crt != MKIt::Utils.proxy_cert
-            x << ":#{p.crt}"
-          end
-        end
-        srv['ports'] << x
-      }
-    }
+
+    # ingress
+    srv['ingress'] = self.ingress.to_h(options)
+
     srv['resources'] = {}
     srv['resources']['min_replicas'] = self.min_replicas
     srv['resources']['max_replicas'] = self.max_replicas
@@ -301,7 +299,7 @@ class Service < ActiveRecord::Base
 
   def as_json(options = {})
     srv = super
-    a=[:pod, :volume, :service_config, :service_port]
+    a=[:pod, :volume, :service_config, :ingress]
     a.each { | k | 
       srv[k] ||= []
       self.send(k).each { |v| 

data/lib/mkit/app/model/service_port.rb

@@ -34,7 +34,6 @@ class ServicePort < ActiveRecord::Base
     ports = config.match(CONFIG_EXPRESSION)
     raise MKIt::InvalidPortsConfiguration.new("no match with config expression $#{CONFIG_EXPRESSION}") if ports.nil?
 
-    puts ports
     self.external_port = ports[1]
     self.internal_port = ports[2]
     self.mode = ports[3]

data/lib/mkit/app/templates/haproxy/0000_defaults.cfg

@@ -1,8 +1,8 @@
 global
         log 127.0.0.1   local2
         maxconn 4096
-        user nobody
-        group nobody
+        user haproxy
+        group haproxy
 
 defaults
     log                     global

data/lib/mkit/app/templates/haproxy/app_haproxy.cfg.erb

@@ -0,0 +1,28 @@
+#
+# MKIt generated file
+# Service <%=name%>
+#
+
+#
+# frontends
+#
+<% ingress.frontends.each { |fe|%>
+frontend  <%=name%>-<%=fe.name%>-front
+  bind <%=lease.ip%>:<%=fe.port%> <%=if fe.ssl? then "ssl crt #{fe.crt}" end%> <%=fe.bind_options.join(' ')%>
+  mode <%=fe.mode%>
+  <%=fe.options.join("\n  ")%>
+
+  use_backend <%=name%>-<%=fe.default_backend%>-back
+<%}%>
+#
+# backends
+#
+<% ingress.backends.each { |be|%>
+backend <%=name%>-<%=be.name%>-back
+  mode <%=be.mode%>
+  balance <%=be.load_balance%>
+  <%=be.options.join("\n  ")%>
+<%pod.each { | pod | %>
+  server <%=pod.name%>  <%=pod.ip%><%unless be.port.nil? || be.port.empty? then%><%=":#{be.port}"%> <%=be.bind_options.join(' ')%> <%end%>
+<%}%>
+<%}%>

data/lib/mkit/client/commands.yaml

@@ -159,6 +159,19 @@
   request:
     verb: delete
     uri: "/services/<%=id%>"
+- cmd: migrate
+  usage:
+    - "<service.yaml>"
+  help: migrate local service definition to the new schema version
+  args:
+    - name: file
+      help:
+        - file
+        - Service definition
+      mandatory: true
+  request:
+    verb: post
+    uri: "/services/migrate"
 - cmd: exec
   usage:
     - "<service_id_or_name> [options] -- <command> [args...]"

data/lib/mkit/config/environment.rb

@@ -6,6 +6,7 @@ require 'sinatra'
 
 require_relative 'initializers/001_hash'
 require_relative 'initializers/002_openstruct'
+require_relative 'initializers/003_custom_exceptions'
 
 SOCKET_PATH = File.expand_path('/tmp/app.sock')
 

data/lib/mkit/config/initializers/001_hash.rb

@@ -8,4 +8,18 @@ class Hash
       memo[key] = val.is_a?(Hash) ? val.to_os : val
     end
   end
+
+  def remove_symbols_from_keys
+    self.each_with_object({}) do |(k, v), new_hash|
+      new_key = k.to_s
+      new_value = if v.is_a?(Hash)
+                    v.remove_symbols_from_keys
+                  elsif v.is_a?(Array)
+                    v.map { |item| item.is_a?(Hash) ? item.remove_symbols_from_keys : item }
+                  else
+                    v
+                  end
+      new_hash[new_key] = new_value
+    end
+  end
 end

data/lib/mkit/config/initializers/003_custom_exceptions.rb

@@ -0,0 +1,10 @@
+# Load custom exceptions
+require 'mkit/exceptions'
+
+module CustomExceptions
+  def raise_bad_configuration(message)
+    raise MKIt::InvalidConfigurationException, "MKIt :: Invalid Configuration :: #{message}"
+  end
+end
+
+Object.include(CustomExceptions)

data/lib/mkit/ctypes.rb

@@ -25,7 +25,7 @@ module MKIt
     module Templates
       DOCKER_RUN = 'docker/docker_run.sh'
       DOCKER_BUILD = 'docker/docker_build.sh'
-      HAPROXY = 'haproxy/xapp_haproxy.cfg'
+      HAPROXY = 'haproxy/app_haproxy.cfg'
       HAPROXY_DEFAULTS = 'haproxy/0000_defaults.cfg'
     end
 end

data/lib/mkit/exceptions.rb

@@ -22,7 +22,19 @@ module MKIt
       super(400, message)
     end
   end
-  class ServiceNotFoundException < StandardError; end
+
+  class ServiceNotFoundException < BaseException
+    def initialize(message = nil)
+      super(404, message)
+    end
+  end
+
+  class InvalidConfigurationException < BaseException
+    def initialize(message = nil)
+      super(400, message)
+    end
+  end
+
   class PodNotFoundException     < StandardError; end
   class AppAlreadyDeployedException < StandardError; end
   class InvalidPortMappingTypeException < StandardError; end

data/lib/mkit/pods/docker_listener.rb

@@ -6,7 +6,7 @@ require "mkit/cmd/shell_client"
 # https://docs.docker.com/engine/reference/commandline/events
 require 'mkit/app/helpers/docker_helper'
 module MKIt
-class StopThread < RuntimeError; end
+  class StopThread < RuntimeError; end
 
   class DockerListener
     include MKIt::DockerHelper
@@ -67,22 +67,27 @@ class StopThread < RuntimeError; end
         end
       when :network
         pod_id = msg.Actor.Attributes.container
-        inspect = inspect_instance(pod_id).to_o
-        pod_name = inspect.Name[1..]
-        pod = Pod.find_by(name: pod_name)
-        unless pod.nil?
-          case action
-          when :connect
-            MKItLogger.info("docker network #{action} received: #{msg} for pod #{pod_name}")
-            pod.update_ip
-            pod.save
-          when :disconnect
-            MKItLogger.debug("  #{type} #{action} <<NOOP / TODO>>")
+        inspect = inspect_instance(pod_id)
+        unless inspect.nil?
+          inspect = inspect.to_o
+          pod_name = inspect.Name[1..]
+          pod = Pod.find_by(name: pod_name)
+          unless pod.nil?
+            case action
+            when :connect
+              MKItLogger.info("docker network #{action} received: #{msg} for pod #{pod_name}")
+              pod.update_ip
+              pod.save
+            when :disconnect
+              MKItLogger.debug("  #{type} #{action} <<NOOP / TODO>>")
+            else
+              MKItLogger.debug("  #{type} #{action} <<TODO>>")
+            end
           else
-            MKItLogger.debug("  #{type} #{action} <<TODO>>")
+            MKItLogger.warn("docker <<#{type}>> <#{action}> received: #{msg}. But I don't know anything about pod #{pod_id}/#{pod_name}")
           end
         else
-          MKItLogger.warn("docker <<#{type}>> <#{action}> received: #{msg}. But I don't know anything about pod #{pod_id}/#{pod_name}")
+            MKItLogger.warn("docker <<#{type}>> <#{action}> received: #{msg}. But I don't know anything about pod #{pod_id}")
         end
       else
         MKItLogger.info("\t#{type} #{action} <<unknown>>")

data/lib/mkit/sagas/create_pod_saga.rb

@@ -1,5 +1,8 @@
+require 'mkit/app/helpers/docker_helper'
+
 module MKIt
   class CreatePodSaga < ASaga
+    include MKIt::DockerHelper
 
     def topics
       %w{create_pod_saga}
@@ -9,20 +12,19 @@ module MKIt
     # create_pod_saga:
     #
     # payload:
-    #  * service_id
+    #  * pod_name
     #
     # triggers
     #  * nothing
     #
     def do_the(job)
       MKItLogger.info("#{self.class} <#{job.topic}> #{job.inspect}....")
-      service = Service.find(job.service_id)
-      # create pod
 
-      pd = Pod.new( service: service, status: MKIt::Status::CREATED, name: SecureRandom.uuid.gsub('-','')[0..11])
-      service.pod << pd
-      service.save
-      MkitJob.publish(topic: :start_pod, service_id: job.service_id, pod_id: pd.id)
+      pod_name = job.data['pod_name']
+      pod = Pod.find_by_name(pod_name)
+      docker_run = pod.parse
+      MKItLogger.info("deploying docker pod, cmd [#{docker_run}]")
+      create_instance(docker_run)
     end
   end
 end

data/lib/mkit/sagas/destroy_pod_saga.rb

@@ -0,0 +1,32 @@
+require 'mkit/app/helpers/docker_helper'
+
+module MKIt
+  class DestroyPodSaga < ASaga
+    include MKIt::DockerHelper
+
+    def topics
+      %w{destroy_pod_saga}
+    end
+
+    #
+    # destroy_pod_saga:
+    #
+    # payload:
+    #  * pod_name
+    #  * pod.id
+    #
+    # triggers
+    #  * pod_destroyed?
+    #
+    def do_the(job)
+      MKItLogger.info("#{self.class} <#{job.topic}> #{job.inspect}....")
+
+      begin
+        remove_instance(job.data['pod_name'])
+      rescue => e
+        MKItLogger.warn(e)
+      end
+
+    end
+  end
+end

data/lib/mkit/sagas/saga_manager.rb

@@ -1,10 +1,12 @@
 require 'mkit/sagas/asaga'
 require 'mkit/sagas/create_pod_saga'
+require 'mkit/sagas/destroy_pod_saga'
 
 module MKIt
   class SagaManager
     def self.register_workers
       CreatePodSaga.new
+      DestroyPodSaga.new
     end
   end
 end

data/lib/mkit/version.rb

@@ -1,4 +1,4 @@
 module MKIt
-  VERSION = "0.9.1"
+  VERSION = "0.10.0"
 end
 

data/lib/mkit/workers/pod_worker.rb

@@ -28,7 +28,9 @@ module MKIt
           MkitJob.publish(topic: :update_proxy_config, service_id: job.service_id)
         end
       when :pod_ip_updated
-        MkitJob.publish(topic: :update_proxy_config, service_id: job.service_id)
+        if Service.exists?(job.service_id)
+          MkitJob.publish(topic: :update_proxy_config, service_id: job.service_id)
+        end
       else
         MKItLogger.info("#{self.class} <<TODO>> job #{job.inspect}....")
       end

data/lib/mkit/workers/service_worker.rb

@@ -1,3 +1,5 @@
+
+
 module MKIt
   class ServiceWorker < AWorker
 

data/samples/apps/httpbin.yml

@@ -0,0 +1,46 @@
+---
+service:
+  name: httpbin
+  image: mccutchen/go-httpbin
+  command: 
+  network: bridge
+  ingress:
+    frontend:
+      - name: http-in-ssl
+        options:
+          - option httpclose
+          - option forwardfor
+        bind:
+          port: 443
+          mode: http
+          ssl: true
+          options:
+            - accept-proxy
+            - transparent
+            - defer-accept
+        default_backend: server
+      - name: http-in
+        options:
+          - option forwardfor
+        bind:
+          port: 80
+          mode: http
+        default_backend: server
+    backend:
+    - name: server
+      balance: round_robin
+      options:
+      - cookie JSESSIONID prefix
+      bind:
+        port: 8080
+        mode: http
+        options:
+        - cookie A
+        - check
+  resources:
+    min_replicas: 1
+    max_replicas: 1
+  volumes: []
+  environment:
+    LOG4J_LEVEL: WARN
+    LOGGING_LEVEL_ROOT: WARN

data/samples/apps/kafka-cluster.yml

@@ -1,21 +1,31 @@
-#
+---
 service:
   name: kafka-cluster
   network: kafka-cluster
   image: confluentinc/cp-kafka:7.1.6
-  ports:
-    - 9092:9092:tcp:round_robin
   volumes:
-    - docker://kafka_cluster_secrets:/etc/kafka/secrets
-    - docker://kafka_cluster_data:/var/lib/kafka/data
+  - docker://kafka_cluster_secrets:/etc/kafka/secrets
+  - docker://kafka_cluster_data:/var/lib/kafka/data
   environment:
     KAFKA_BROKER_ID: 1
     KAFKA_ZOOKEEPER_CONNECT: kafka-zookeeper:2181
-    KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
+    KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
     KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-cluster:9092
     KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT
-      # KAFKA_CONFLUENT_SCHEMA_REGISTRY_URL: http://kafka-schema-registry
     KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
     KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
     KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
-
+  ingress:
+    frontend:
+    - name: frontend-9092
+      options: []
+      bind:
+        port: '9092'
+        mode: tcp
+      default_backend: backend-9092
+    backend:
+    - name: backend-9092
+      bind:
+        port: '9092'
+        mode: tcp
+      balance: round_robin

data/samples/apps/kafka-magic.yml

@@ -1,16 +1,33 @@
-#
+---
 service:
   name: kafka-magic
   network: kafka-cluster
-  ports:
-    - 80:80:http:round_robin
   image: digitsy/kafka-magic
   environment:
-    KMAGIC_ALLOW_TOPIC_DELETE: "true"
-    KMAGIC_ALLOW_SCHEMA_DELETE: "true"
-    KMAGIC_CONFIG_STORE_TYPE: "file"
-    KMAGIC_CONFIG_STORE_CONNECTION: "Data Source=/config/KafkaMagicConfig.db;"
-    KMAGIC_CONFIG_ENCRYPTION_KEY: "123456"
+    KMAGIC_ALLOW_TOPIC_DELETE: 'true'
+    KMAGIC_ALLOW_SCHEMA_DELETE: 'true'
+    KMAGIC_CONFIG_STORE_TYPE: file
+    KMAGIC_CONFIG_STORE_CONNECTION: Data Source=/config/KafkaMagicConfig.db;
+    KMAGIC_CONFIG_ENCRYPTION_KEY: '123456'
   volumes:
-    - docker://kafka_magic_config:/config
-
+  - docker://kafka_magic_config:/config
+  ingress:
+    frontend:
+    - name: frontend-80
+      options: []
+      bind:
+        port: '80'
+        mode: http
+      default_backend: backend-80
+    backend:
+    - name: backend-80
+      bind:
+        port: '80'
+        mode: http
+        options:
+        - cookie A check
+      balance: round_robin
+      options:
+      - option httpclose
+      - option forwardfor
+      - cookie JSESSIONID prefix

data/samples/apps/kafka-zookeeper.yml

@@ -1,15 +1,26 @@
-#
+---
 service:
   name: kafka-zookeeper
   image: confluentinc/cp-zookeeper:7.1.6
   network: kafka-cluster
-  ports:
-    - 2181:2181:tcp:round_robin
   volumes:
-    - docker://kafka_zookeeper_secrets:/etc/zookeeper/secrets
-    - docker://kafka_zookeeper_data:/var/lib/zookeeper/data
-    - docker://kafka_zookeeper_log:/var/lib/zookeeper/log
+  - docker://kafka_zookeeper_secrets:/etc/zookeeper/secrets
+  - docker://kafka_zookeeper_data:/var/lib/zookeeper/data
+  - docker://kafka_zookeeper_log:/var/lib/zookeeper/log
   environment:
     ZOOKEEPER_CLIENT_PORT: 2181
     ZOOKEEPER_TICK_TIME: 2000
-
+  ingress:
+    frontend:
+    - name: frontend-2181
+      options: []
+      bind:
+        port: '2181'
+        mode: tcp
+      default_backend: backend-2181
+    backend:
+    - name: backend-2181
+      bind:
+        port: '2181'
+        mode: tcp
+      balance: round_robin

data/samples/apps/minio.yml

@@ -1,16 +1,42 @@
-#
+---
 service:
   name: minio
   image: minio/minio
   command: server /data --console-address ":9001"
   network: bridge
-  ports:
-    - 9001:9001:http:round_robin
-    - 9000:9000:tcp:round_robin
   environment:
     MINIO_ACCESS_KEY: minio
     MINIO_SECRET_KEY: minio123
   volumes:
-    - docker://minio_data:/data
-    #- /tmp/minio_data:/data
-
+  - docker://minio_data:/data
+  ingress:
+    frontend:
+    - name: frontend-9001
+      options: []
+      bind:
+        port: '9001'
+        mode: http
+      default_backend: backend-9001
+    - name: frontend-9000
+      options: []
+      bind:
+        port: '9000'
+        mode: tcp
+      default_backend: backend-9000
+    backend:
+    - name: backend-9001
+      bind:
+        port: '9001'
+        mode: http
+        options:
+        - cookie A check
+      balance: round_robin
+      options:
+      - option httpclose
+      - option forwardfor
+      - cookie JSESSIONID prefix
+    - name: backend-9000
+      bind:
+        port: '9000'
+        mode: tcp
+      balance: round_robin

data/samples/apps/mongo.yml

@@ -1,9 +1,20 @@
-#
+---
 service:
   name: mongo
   image: mongo:4.0
   command: "--smallfiles"
   network: bridge
-  ports:
-    - 27017:27017:tcp:round_robin
-
+  ingress:
+    frontend:
+    - name: frontend-27017
+      options: []
+      bind:
+        port: '27017'
+        mode: tcp
+      default_backend: backend-27017
+    backend:
+    - name: backend-27017
+      bind:
+        port: '27017'
+        mode: tcp
+      balance: round_robin