mkit 0.9.1 → 0.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5c7293226251f8d031a6b864f4ef7e9932a6046be6a7e78e07e203b1d9e4ff9a
-  data.tar.gz: 8a4993da20b730eb02f3f0605e257d255348861f84c3b8412e7b46b94316886c
+  metadata.gz: 524cddb617c87b3a095f03139dbbd17d8069f15d52b451f704df658e94f2eebb
+  data.tar.gz: f8f86343cdbc20ad16236769e5d16963372b6b470501129c7e74a72d5de5ee3b
 SHA512:
-  metadata.gz: bdd7a977df1915b70778fb7626836d0718563efc523cd128c2ee797ace0da1cb4c0f20e6b9dc2667bc1cfb03ec8efd0447b891a3a62179af68c01e6418c1d2de
-  data.tar.gz: e2783cd197695782478e29c075343a2bd6a1d5b84f5fe9d97ac7701a9329b1d26f874ab50a4493f85c629cbf2f3f2cf2761223700806d7c7adc322138e0d6cdc
+  metadata.gz: 651a204440f41182b41c1ce1fc809a351692ff5e344f72cc2cf5211a42b3634423fdef7ee403ecc82b549ac9f6ca962d5f0ed853294766979fd8e78289c777f1
+  data.tar.gz: b27df3240e917b99a071008b7c440eb56fe41e6a9daa89ccb56fbcd925e212789df87347e64e01f8a21b768a499c7b5662878bdcaf6908531070ae1eaae83279

data/README.md

@@ -127,18 +127,52 @@ service:
   name: rabbitmq # unique. Available on internal DNS as 'rabbitmq'
   image: rabbitmq:3-management-alpine # image
   network: bridge # docker network - it will be created if it does not exists
-  ports:  # haproxy port mapping
-          #   <external_port>:[internal_port]:<tcp|http>:[round_robin (default)|leastconn]
-          # to define a range on `external_port`, leave `internal_port` blank
-          #   - 5000-5100::tcp:round_robin
-          #   range on `internal_port` is not supported
-          # ssl suport
-          #   <external_port>:[internal_port]:<tcp|http>:round_robin|leastconn[:ssl[,<cert.pem>(mkit.pem default)>]]
-          #   e.g.
-          #  - 443:80:http:round_robin:ssl # uses mkitd default crt file (mkit.pem)
-          #  - 443:80:http:round_robin:ssl,/etc/pki/foo.pem # custom crt file full path
-    - 5672:5672:tcp:round_robin
-    - 80:15672:http:round_robin
+  ingress:
+    # ingress configuration
+    # see https://docs.haproxy.org/3.1/configuration.html
+    frontend: # frontend section
+      - name: http-in-ssl # frontend name - must be unique
+        options: # frontend global section options
+          - option httpclose
+          - option forwardfor
+        bind:
+          port: 443 # to define a range (e.g. 8000-8080), backend `port` must be blank
+          ssl: true # ssl support - default is false
+          mode: http # tcp|http
+          cert: /etc/ssl/certs/server.crt # if empty uses mkit default crt file (mkit.pem)
+          options: # frontend bind section options
+            - accept-proxy
+            - transparent
+            - defer-accept
+        default_backend: admin # backend to use - must point to a backend name
+      - name: http-in
+        bind:
+          port: 80
+          mode: http
+        default_backend: admin
+      - name: server-in
+        bind:
+          port: 5672
+          mode: tcp
+        default_backend: server
+    backend: # backend section
+      - name: admin # backend name - must be unique
+        balance: round_robin # round_robin|least_conn - default is round_robin
+        options: # backend global section options
+          - option httpclose
+          - option forwardfor
+          - cookie JSESSIONID prefix
+        bind:
+          port: 15672 # if this backend is used by a frontend with a range port, this port must be blank
+          mode: http
+          options: # backend bind section options
+            - cookie A
+            - check
+      - name: server
+        balance: round_robin
+        bind:
+          port: 5672
+          mode: tcp
   resources:
     min_replicas: 1 # default value. Pods will be available on internal DNS as '<service_name>.internal'
     max_replicas: 1 # default value
@@ -163,9 +197,9 @@ Usage: mkitd [options]
     -b bind                          specify bind address (e.g. 0.0.0.0)
     -e env                           set the environment (default is development)
     -o addr                          alias for '-b' option
-        --no-ssl                     disable ssl - use http for local server. (default is https)
-        --ssl-key-file PATH          Path to private key (default mkit internal)
-        --ssl-cert-file PATH         Path to certificate (default mkit internal)
+    --no-ssl                         disable ssl - use http for local server. (default is https)
+    --ssl-key-file PATH              Path to private key (default mkit internal)
+    --ssl-cert-file PATH             Path to certificate (default mkit internal)
 ```
 
 There's also samples for [systemd](samples/systemd) and [daemontools](samples/daemontools) as well for some miscellaneous [spps](samples/apps).
@@ -193,6 +227,7 @@ create       create new service
 update       update service
 get          print service configuration
 rm           remove service
+migrate      migrate local service definition to the new schema version
 exec         execute a command in a running pod
 logs         view service logs
 version      prints mkit client and server version

data/db/migrate/004_create_ingress.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+class CreateIngress < ActiveRecord::Migration[5.1]
+  def up
+    create_table :ingresses do |t|
+      t.string :service_id
+      t.string :version
+      t.timestamp :created_at
+      t.timestamp :updated_at
+    end
+  end
+end

data/db/migrate/005_create_frontend.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+class CreateFrontend < ActiveRecord::Migration[5.1]
+  def up
+    create_table :frontends do |t|
+      t.string :ingress_id
+      t.string :name
+      t.string :mode
+      t.string :port
+      t.string :ssl
+      t.string :crt
+      t.text :options, default: '[]'
+      t.text :bind_options, default: '[]'
+      t.text :default_backend
+      t.string :version
+      t.timestamp :created_at
+      t.timestamp :updated_at
+    end
+  end
+end

data/db/migrate/006_create_backend.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+class CreateBackend < ActiveRecord::Migration[5.1]
+  def up
+    create_table :backends do |t|
+      t.string :ingress_id
+      t.string :name
+      t.string :mode
+      t.string :load_bal
+      t.string :port
+      t.text :options, default: '[]'
+      t.text :bind_options, default: '[]'
+      t.string :version
+      t.timestamp :created_at
+      t.timestamp :updated_at
+    end
+  end
+end

data/db/migrate/007_migrate_schema.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+class MigrateSchema < ActiveRecord::Migration[5.1]
+
+  #
+  # migrate the schema from service_ports to ingress
+  #
+  def up
+    Service.all.each do |service|
+      ingress_config = service_ports_to_ingress service.service_port
+      service.ingress = Ingress.create(ingress_config)
+      service.service_port.destroy_all
+    end
+  end
+
+  def service_ports_to_ingress(ports)
+    ingress = { frontend: [], backend: [] }
+
+    ports.each do |port|
+      frontend = {
+        name: "frontend-#{port.external_port}",
+        bind: {
+          port: port.external_port,
+          mode: port.mode
+        },
+        options: [],
+        default_backend: "backend-#{port.external_port}"
+      }
+
+      if port.ssl.start_with?('true')
+        frontend[:bind][:ssl] = true
+        frontend[:bind][:cert] = port.crt
+      end
+
+      backend = {
+        name: "backend-#{port.external_port}",
+        bind: {
+          port: port.internal_port,
+          mode: port.mode
+        },
+        balance: port.load_bal
+      }
+
+      if port.mode == 'http'
+        a= [
+          'option httpclose',
+          'option forwardfor',
+          'cookie JSESSIONID prefix'
+        ]
+        backend[:options] = a
+        backend[:bind][:options] = [ 'cookie A check' ]
+      end
+      ingress[:frontend] << frontend
+      ingress[:backend] << backend
+    end
+
+    ingress.remove_symbols_from_keys
+  end
+end

data/lib/mkit/app/controllers/services_controller.rb

@@ -3,12 +3,14 @@
 require 'mkit/app/model/service'
 require 'mkit/app/helpers/services_helper'
 require 'mkit/app/helpers/params_helper'
+require 'mkit/app/helpers/migrations_helper'
 require 'mkit/pods/docker_log_listener'
 require 'mkit/pods/docker_exec_command'
 
 class ServicesController < MKIt::Server
   helpers MKIt::ServicesHelper
   helpers MKIt::ParamsHelper
+  helpers MKIt::MigrationsHelper
 
   # curl localhost:4567/services
   get '/services' do
@@ -100,6 +102,16 @@ class ServicesController < MKIt::Server
     format_response(srv)
   end
 
+  post '/services/migrate' do
+    srv = 'no file'
+    if params[:file]
+      tempfile = params[:file][:tempfile]
+      yaml = YAML.safe_load(tempfile.read)
+      srv = migrate_service(yaml).to_yaml
+    end
+    srv
+  end
+
   #
   # operations
   #

data/lib/mkit/app/helpers/migrations_helper.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module MKIt
+  module MigrationsHelper
+
+    def migrate_service(yaml)
+      yaml['service']['ingress'] = ports_to_ingress(yaml['service']['ports'])
+      yaml['service'].delete('ports')
+      yaml
+    end
+
+    def ports_to_ingress(ports)
+      ingress = { frontend: [], backend: [] }
+
+      ports.each_with_index do |port_config, index|
+        match = port_config.match(/^(.*?):(.*?):(tcp|http):(.*?)($|:ssl$|:ssl,(.+))$/)
+        next unless match
+
+        external_port, internal_port, mode, load_bal, ssl, cert = match.captures
+
+        frontend = {
+          name: "frontend-#{external_port}",
+          options: [],
+          bind: {
+            port: external_port,
+            mode: mode
+          },
+          default_backend: "backend-#{external_port}"
+        }
+
+        if ssl.start_with?(':ssl')
+          frontend[:bind][:ssl] = true
+          frontend[:bind][:cert] = cert
+        end
+
+        backend = {
+          name: "backend-#{external_port}",
+          bind: {
+            port: internal_port,
+            mode: mode
+          },
+          balance: load_bal
+        }
+
+        if mode == 'http'
+          a= [
+            'option httpclose',
+            'option forwardfor',
+            'cookie JSESSIONID prefix'
+          ]
+          backend[:options] = a
+          backend[:bind][:options] = [ 'cookie A check' ]
+        end
+        ingress[:frontend] << frontend
+        ingress[:backend] << backend
+      end
+
+      ingress.remove_symbols_from_keys
+    end
+
+  end
+end

data/lib/mkit/app/helpers/services_helper.rb

@@ -34,27 +34,16 @@ module MKIt
     end
 
     def build_table_row(data)
-      ports = data.service_port&.each.map { |p| build_port(p) }.join(',')
+      ports = data.ingress.frontends&.each.map { |f| build_port(f) }.join(',')
       pods = data.pod.each.map { |p| p.name.to_s }.join(' ')
       [data.id, data.name, data.lease&.ip, ports, pods, data.status]
     end
 
     def build_port(p)
-      case p.mode
-      when 'http'
-        if p.ssl?
-          "#{p.mode}s/#{p.external_port}"
-        else
-          "#{p.mode}/#{p.external_port}"
-        end
-      when 'tcp'
-        if p.ssl?
-          "s#{p.mode}/#{p.external_port}"
-        else
-          "#{p.mode}/#{p.external_port}"
-        end
+      if p.ssl?
+        "#{p.mode}s/#{p.port}"
       else
-        "#{p.mode}/#{p.external_port}"
+        "#{p.mode}/#{p.port}"
       end
     end
   end

data/lib/mkit/app/model/backend.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+class Backend  < ActiveRecord::Base
+  belongs_to :ingress
+  serialize :options, JSON
+  serialize :bind_options, JSON
+
+  def self.create(yaml)
+    validate(yaml)
+    backend = Backend.new
+    backend.name = yaml["name"]
+    backend.port = yaml["bind"]["port"] if yaml["bind"]["port"]
+    backend.mode = yaml["bind"]["mode"] if yaml["bind"]["mode"]
+    backend.options = yaml["options"] if yaml["options"]
+    backend.load_bal = yaml["balance"] if yaml["balance"]
+    backend.bind_options = yaml["bind"]["options"] if yaml["bind"]["options"]
+    backend
+  end
+
+  def self.validate(yaml)
+    raise_bad_configuration "name is mandatory" unless yaml["name"]
+    raise_bad_configuration "bind is mandatory" unless yaml["bind"]
+    raise_bad_configuration "mode is mandatory" unless yaml["bind"]["mode"]
+    raise_bad_configuration "mode must match tcp|http" unless yaml["bind"]["mode"] =~ /^(tcp|http)$/
+    if yaml["balance"]
+      raise_bad_configuration "balance must match round_robin|least_conn" unless yaml["balance"] =~ /^(round_robin|least_conn)$/
+    end
+  end
+
+  def load_balance
+    case self.load_bal
+    when /^round_robin$/
+      "roundrobin"
+    when /^least_conn$/
+      "leastconn"
+    else
+      "roundrobin"
+    end
+  end
+
+  def to_h(options = {})
+    hash = {
+      name: self.name,
+      balance: self.load_bal,
+      options: self.options,
+      bind: {
+        port: self.port,
+        mode: self.mode,
+        options: self.bind_options
+      }
+    }
+
+    if self.port.nil? || self.port.empty?
+      hash[:bind].delete(:port)
+    end
+    hash.remove_symbols_from_keys
+  end
+end

data/lib/mkit/app/model/frontend.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+require 'mkit/exceptions'
+
+class Frontend  < ActiveRecord::Base
+  belongs_to :ingress
+  serialize :options, JSON
+  serialize :bind_options, JSON
+  has_one :backend
+
+  def self.create(yaml)
+    validate(yaml)
+    frontend = Frontend.new
+
+    frontend.name = yaml["name"]
+    frontend.port = yaml["bind"]["port"] if yaml["bind"]["port"]
+    frontend.mode = yaml["bind"]["mode"] if yaml["bind"]["mode"]
+    frontend.bind_options = yaml["bind"]["options"] if yaml["bind"]["options"]
+    frontend.options = yaml["options"] if yaml["options"]
+    frontend.default_backend = yaml["use_backend"]
+
+    has_ssl = !yaml["bind"]["ssl"].nil? && yaml["bind"]["ssl"].to_s.start_with?('true')
+    frontend.ssl = has_ssl ? 'true' : 'false'
+    if has_ssl
+      frontend.crt = yaml["bind"]["cert"].nil? ? MKIt::Utils.proxy_cert : yaml["bind"]["cert"]
+    end
+
+    frontend
+  end
+
+  def self.validate(yaml)
+    raise_bad_configuration "name is mandatory" unless yaml["name"]
+    raise_bad_configuration "default_backend is mandatory" unless yaml["default_backend"]
+    raise_bad_configuration "bind is mandatory" unless yaml["bind"]
+    raise_bad_configuration "mode is mandatory" unless yaml["bind"]["mode"]
+    raise_bad_configuration "frontend port is mandatory" unless yaml["bind"]["port"]
+    raise_bad_configuration "mode must match tcp|http" unless yaml["bind"]["mode"] =~ /^(tcp|http)$/
+  end
+
+  def ssl?
+    self.ssl == 'true'
+  end
+
+  def to_h(options = {})
+    hash = {
+      name: self.name,
+      options: self.options,
+      bind: {
+        port: self.port,
+        mode: self.mode,
+        ssl: self.ssl,
+        cert: self.ssl? ? self.crt : nil,
+        options: self.bind_options
+      },
+      default_backend: self.default_backend
+    }
+
+    unless self.ssl?
+      hash[:bind].delete(:ssl)
+      hash[:bind].delete(:cert)
+    end
+    hash.remove_symbols_from_keys
+  end
+end

data/lib/mkit/app/model/ingress.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+require 'mkit/app/model/backend'
+require 'mkit/app/model/frontend'
+
+class Ingress < ActiveRecord::Base
+  belongs_to :service
+  has_many :frontends, dependent: :destroy
+  has_many :backends, dependent: :destroy
+
+  def self.create(yaml)
+    validate(yaml)
+    ingress = Ingress.new
+
+    yaml["backend"].each do |back|
+      ingress.backends << Backend.create(back)
+    end
+
+    yaml["frontend"].each do |front|
+      ingress.frontends << Frontend.create(front)
+    end
+    ingress
+  end
+
+  def self.validate(yaml)
+    frontend_names = []
+    frontend_ports = []
+
+    # must have at least one frontend and one backend
+    raise_bad_configuration "Ingress section is mandatory" unless yaml
+    raise_bad_configuration "At least one frontend is mandatory" unless yaml["frontend"]
+    raise_bad_configuration "At least one backend is mandatory" unless yaml["backend"]
+    # frontend name is mandatory
+    yaml["frontend"].each { |front| raise "Frontend name is mandatory" unless front["name"] }
+    # backend name is mandatory
+    yaml["backend"].each { |back| raise "Backend name is mandatory" unless back["name"] }
+    # frontend must point to a valid backend name - backend names list
+    backend_names = yaml["backend"].map { |back| back["name"] }
+
+    # frontend validation
+    yaml["frontend"].each do |front|
+      # name is mandatory
+      raise_bad_configuration "Frontend name is mandatory" unless front["name"]
+      # frontend name must be unique
+      if frontend_names.include?(front["name"])
+        raise_bad_configuration "Frontend name '#{front["name"]}' must be unique"
+      end
+      frontend_names << front["name"]
+
+      # bind and mode are mandatory, port is not
+      raise_bad_configuration "Frontend bind and mode are mandatory" unless front["bind"] && front["bind"]["mode"]
+
+      # port must be unique
+      if frontend_ports.include?(front["bind"]["port"])
+        raise_bad_configuration "Frontend port '#{front["bind"]["port"]}' must be unique"
+      end
+      frontend_ports << front["bind"]["port"]
+
+      # default_backend must point to a valid backend name
+      unless backend_names.include?(front["default_backend"])
+        raise_bad_configuration "Frontend default_backend '#{front["default_backend"]}' must point to a valid backend name"
+      end
+
+    end
+
+    # backend validation
+    backend_names.each do |name|
+      if backend_names.count(name) > 1
+        raise_bad_configuration "Backend name '#{name}' must be unique"
+      end
+    end
+
+    # global validations
+    # each backend must point to a valid frontend default_backend
+    frontend_default_backends = yaml["frontend"].map { |front| front["default_backend"] }
+    yaml["backend"].each do |back|
+      unless frontend_default_backends.include?(back["name"])
+        raise_bad_configuration "Backend '#{back["name"]}' must be referenced by at least one frontend default_backend"
+      end
+    end
+
+    # when frontend port is range - e.g. 1200-1220, referred backend port must be empty
+    yaml["frontend"].each do |front|
+      if front["bind"]["port"] =~ /^\d+-\d+$/
+        referred_backend = yaml["backend"].find { |back| back["name"] == front["default_backend"] }
+        raise_bad_configuration "Frontend port range '#{front["bind"]["port"]}' must have an empty backend port" unless referred_backend && (referred_backend["bind"]["port"].nil?)
+      end
+    end
+
+    true
+  end
+
+  def to_h(options = {})
+    {
+      frontend: self.frontends.map { |front| front.to_h },
+      backend: self.backends.map { |back| back.to_h }
+    }.remove_symbols_from_keys
+  end
+end

data/lib/mkit/app/model/pod.rb

@@ -58,13 +58,7 @@ class Pod < ActiveRecord::Base
   end
 
   def start
-    if self.instance.nil?
-      docker_run = parse
-      MKItLogger.info("deploying docker pod, cmd [#{docker_run}]")
-      create_instance(docker_run)
-    else
-      start_instance(self.name) unless instance.State.Running
-    end
+    start_instance(self.name) unless instance.State.Running
   end
 
   def stop
@@ -78,12 +72,12 @@ class Pod < ActiveRecord::Base
   end
 
   def clean_up
-    begin
-      remove_instance(self.name) unless self.instance.nil?
-    rescue => e
-      MKItLogger.warn(e)
-    end
-    MkitJob.publish(topic: :pod_destroyed, service_id: self.service.id, data: {pod_id: self.id})
+    # stop and destroy pod
+    MkitJob.publish(topic: :destroy_pod_saga,
+                    service_id: self.service.id,
+                    pod_id: self.id,
+                    data: {pod_name: self.name}
+    )
   end
 
   def to_h