dapp 0.11.0 → 0.12.0

data/lib/dapp/deployment/kube_base.rb

@@ -0,0 +1,97 @@
+module Dapp
+  module Deployment
+    class KubeBase
+      def pod_exist?(name)
+        deployment.kubernetes.pod?(name, labelSelector: labelSelector)
+      end
+
+      def pod_succeeded?(name)
+        return false unless pod_exist?(name)
+        deployment.kubernetes.pod_status(name)['status']['phase'] == 'Succeeded'
+      end
+
+      def delete_pod!(name)
+        deployment.kubernetes.delete_pod!(name)
+        loop do
+          break unless deployment.kubernetes.pod?(name)
+          sleep 1
+        end
+      end
+
+      def run_job!(spec, name)
+        current_spec = deployment.kubernetes.create_pod!(spec)
+
+        deployment.dapp.log_process(:pending) do
+          loop do
+            current_spec = deployment.kubernetes.pod_status(name)
+            break if current_spec['status']['phase'] != 'Pending'
+            unless current_spec['status']['containerStatuses'].nil?
+              current_spec['status']['containerStatuses'].first['state'].each do |_, desc|
+                if ['ErrImagePull', 'ImagePullBackOff'].include? desc['reason']
+                  raise Error::Deployment,
+                        code: :bootstrap_image_not_found,
+                        data: { reason: desc['reason'], message: desc['message'] }
+                end
+              end
+            end
+            sleep 1
+          end
+        end
+
+        deployment.dapp.log_process(:running, verbose: true) do
+          log_thread = Thread.new do
+            begin
+              deployment.kubernetes.pod_log(name, follow: true) { |chunk| puts chunk }
+            rescue Kubernetes::Error::Timeout
+              deployment.dapp.log_warning('Pod log: read timeout reached!')
+            end
+          end
+
+          loop do
+            current_spec = deployment.kubernetes.pod_status(name)
+            break if current_spec['status']['phase'] != 'Running'
+            sleep 1
+          end
+
+          sleep 1 # last chance to get a message
+          log_thread.kill if log_thread.alive?
+
+          phase = current_spec['status']['phase']
+          if phase == 'Failed'
+            current_spec['status']['containerStatuses'].first['state'].each do |status, desc|
+              next if status == 'running'
+              raise Error::Deployment,
+                    code: :bootstrap_command_failed,
+                    data: { reason: desc['reason'], message: desc['message'], exit_code: desc['exitCode'] }
+            end
+          end
+          raise Error::Deployment,
+                code: :bootstrap_failed,
+                data: { phase: phase, message: current_spec['status']['message'] } unless phase == 'Succeeded'
+        end
+      end
+
+      def merge_kube_controller_spec(spec1, spec2)
+        spec1.kube_in_depth_merge(spec2).tap do |spec|
+          spec['spec']['template']['spec']['containers'] = begin
+            containers1 = spec1['spec']['template']['spec']['containers']
+            containers2 = spec2['spec']['template']['spec']['containers']
+            containers2.map do |container2|
+              if (container1 = containers1.find { |c| c['name'] == container2['name'] }).nil?
+                container2
+              else
+                container1.kube_in_depth_merge(container2)
+              end
+            end
+          end
+        end
+      end
+
+      protected
+
+      def labelSelector
+        labels.map {|key, value| "#{key}=#{value}"}.join(',')
+      end
+    end
+  end
+end

data/lib/dapp/deployment/kube_deployment.rb

@@ -0,0 +1,51 @@
+module Dapp
+  module Deployment
+    class KubeDeployment < KubeBase
+      attr_reader :deployment
+
+      def initialize(deployment)
+        @deployment = deployment
+      end
+
+      def labels
+        {'dapp' => deployment.dapp.name, 'dapp-deployment-version' => 1.to_s}
+      end
+
+      def delete_unknown_resources!
+        # Удаление объектов, связанных с несуществующими более app'ами
+        known_apps_names = deployment.apps.map(&:name)
+        deployment.kubernetes.deployment_list(labelSelector: labelSelector)['items']
+          .reject do |item|
+            known_apps_names.include? item['metadata']['labels']['dapp-app']
+          end
+          .each do |item|
+            deployment.dapp.log "Deleting Deployment '#{item['metadata']['name']}': unknown app '#{item['metadata']['labels']['dapp-app']}'"
+            deployment.kubernetes.delete_deployment!(item['metadata']['name'])
+          end
+        deployment.kubernetes.service_list(labelSelector: labelSelector)['items']
+          .reject do |item|
+            known_apps_names.include? item['metadata']['labels']['dapp-app']
+          end
+          .each do |item|
+            deployment.dapp.log "Deleting Service '#{item['metadata']['name']}': unknown app '#{item['metadata']['labels']['dapp-app']}'"
+            deployment.kubernetes.delete_service!(item['metadata']['name'])
+          end
+
+        # Версионирование объектов из кода dapp, чтобы
+        # иметь возможность автоматом удалить объекты,
+        # созданные по старой логике
+        old_versions_selector = "dapp=#{labels['dapp']},dapp-deployment-version notin (#{labels['dapp-deployment-version']})"
+        deployment.kubernetes.deployment_list(labelSelector: old_versions_selector)['items']
+          .each do |item|
+            deployment.dapp.log "Deleting Deployment '#{item['metadata']['name']}': dapp-deployment-version changed from #{item['metadata']['labels']['dapp-deployment-version']} to #{labels['dapp-deployment-version']}"
+            deployment.kubernetes.delete_deployment!(item['metadata']['name'])
+          end
+        deployment.kubernetes.service_list(labelSelector: old_versions_selector)['items']
+          .each do |item|
+            deployment.dapp.log "Deleting Service '#{item['metadata']['name']}': dapp-deployment-version changed from #{item['metadata']['labels']['dapp-deployment-version']} to #{labels['dapp-deployment-version']}"
+            deployment.kubernetes.delete_service!(item['metadata']['name'])
+          end
+      end
+    end
+  end
+end

data/lib/dapp/deployment/kubernetes.rb

@@ -0,0 +1,174 @@
+module Dapp
+  module Deployment
+    class Kubernetes
+      def initialize(namespace: nil)
+        @namespace = namespace
+        @query_parameters = {}
+      end
+
+      def namespace
+        @namespace || 'default'
+      end
+
+      # Чтобы не перегружать методы явной передачей namespace.
+      # Данный метод может пригодиться только в ситуации, когда надо указать другой namespace,
+      # в большинстве случаев используется namespace из конструктора.
+      def with_namespace(namespace, &blk)
+        old_namespace = @namespace
+        begin
+          @namespace = namespace
+          return yield
+        ensure
+          @namespace = old_namespace
+        end
+      end
+
+      def with_query(query, &blk)
+        old_query = @query_parameters
+        begin
+          @query_parameters = query
+          return yield
+        ensure
+          @query_parameters = old_query
+        end
+      end
+
+      # NOTICE: Название метода аналогично kind'у выдаваемого результата.
+      # NOTICE: В данном случае в результате kind=DeploymentList.
+      # NOTICE: Методы создания/обновления/удаления сущностей kubernetes заканчиваются на '!'. Например, create_deployment!.
+
+      {
+        '/api/v1' => [:service, :replicationcontroller, :pod],
+        '/apis/extensions/v1beta1' => [:deployment, :replicaset]
+      }.each do |api, objects|
+        objects.each do |object|
+          define_method :"#{object}_list" do |**query_parameters|
+            request!(:get, "#{api}/namespaces/#{namespace}/#{object}s", **query_parameters)
+          end
+
+          define_method object do |name, **query_parameters|
+            request!(:get, "#{api}/namespaces/#{namespace}/#{object}s/#{name}", **query_parameters)
+          end
+
+          define_method "#{object}_status" do |name, **query_parameters|
+            request!(:get, "#{api}/namespaces/#{namespace}/#{object}s/#{name}/status", **query_parameters)
+          end
+
+          define_method :"create_#{object}!" do |spec, **query_parameters|
+            request!(:post, "#{api}/namespaces/#{namespace}/#{object}s", body: spec, **query_parameters)
+          end
+
+          define_method :"replace_#{object}!" do |name, spec, **query_parameters|
+            request!(:put, "#{api}/namespaces/#{namespace}/#{object}s/#{name}", body: spec, **query_parameters)
+          end
+
+          define_method :"delete_#{object}!" do |name, **query_parameters|
+            request!(:delete, "#{api}/namespaces/#{namespace}/#{object}s/#{name}", **query_parameters)
+          end
+
+          define_method :"delete_#{object}s!" do |**query_parameters|
+            request!(:delete, "#{api}/namespaces/#{namespace}/#{object}s", **query_parameters)
+          end
+
+          define_method :"#{object}?" do |name, **query_parameters|
+            public_send(:"#{object}_list", **query_parameters)['items'].map { |item| item['metadata']['name'] }.include?(name)
+          end
+        end
+      end
+
+      def pod_log(name, follow: false, **query_parameters, &blk)
+        excon_parameters = follow ? { response_block: blk } : {}
+        request!(:get,
+                 "/api/v1/namespaces/#{namespace}/pods/#{name}/log",
+                 excon_parameters: excon_parameters,
+                 **{ follow: follow }.merge(query_parameters))
+      rescue Excon::Error::Timeout
+        raise Error::Timeout
+      end
+
+      def event_list(**query_parameters)
+        request!(:get, "/api/v1/namespaces/#{namespace}/events", **query_parameters)
+      end
+
+      protected
+
+      # query_parameters — соответствует 'Query Parameters' в документации kubernetes
+      # excon_parameters — соответствует опциям Excon
+      # body — hash для http-body, соответствует 'Body Parameters' в документации kubernetes, опционален
+      def request!(method, path, body: nil, excon_parameters: {}, **query_parameters)
+        with_connection(excon_parameters: excon_parameters) do |conn|
+          request_parameters = {method: method, path: path, query: @query_parameters.merge(query_parameters)}
+          request_parameters[:body] = JSON.dump(body) if body
+          load_body! conn.request(request_parameters), request_parameters
+        end
+      end
+
+      def load_body!(response, request_parameters)
+        response_ok = response.status.to_s.start_with? '2'
+
+        if response_ok
+          JSON.parse(response.body)
+        else
+          err_data = {}
+          err_data[:response_http_status] = response.status
+          if response_body = (JSON.parse(response.body) rescue nil)
+            err_data[:response_body] = response_body
+          else
+            err_data[:response_raw_body] = response.body
+          end
+          err_data[:request_parameters] = request_parameters
+
+          if response.status.to_s.start_with? '5'
+            raise Error::Base, code: :server_error, data: err_data
+          elsif response.status.to_s == '404'
+            raise Error::NotFound, data: err_data
+          else not response.status.to_s.start_with? '2'
+            raise Error::Base, code: :bad_request, data: err_data
+          end
+        end
+      end
+
+      def with_connection(excon_parameters: {}, &blk)
+        old_ssl_ca_file = Excon.defaults[:ssl_ca_file]
+        old_middlewares = Excon.defaults[:middlewares].dup
+
+        begin
+          Excon.defaults[:ssl_ca_file] = kube_config.fetch('clusters', [{}]).first.fetch('cluster', {}).fetch('certificate-authority', nil)
+          Excon.defaults[:middlewares] << Excon::Middleware::RedirectFollower
+
+          return yield Excon.new(kube_server_url, **kube_server_options(excon_parameters))
+        ensure
+          Excon.defaults[:ssl_ca_file] = old_ssl_ca_file
+          Excon.defaults[:middlewares] = old_middlewares
+        end
+      end
+
+      def kube_server_url
+        @kube_server_url ||= begin
+          kube_config.fetch('clusters', [{}]).first.fetch('cluster', {}).fetch('server', nil).tap do |url|
+            begin
+              Excon.new(url, **kube_server_options).get
+            rescue Excon::Error::Socket
+              raise Error::ConnectionRefused, code: :kube_server_connection_refused, data: { url: url }
+            end
+          end
+        end
+      end
+
+      def kube_server_options(excon_parameters = {})
+        { client_cert: kube_config.fetch('users', [{}]).first.fetch('user', {}).fetch('client-certificate', nil),
+          client_key: kube_config.fetch('users', [{}]).first.fetch('user', {}).fetch('client-key', nil) }.merge(excon_parameters)
+      end
+
+      def kube_config
+        @kube_config ||= begin
+          if File.exist?((kube_config_path = File.join(ENV['HOME'], '.kube/config')))
+            YAML.load_file(kube_config_path)
+          else
+            raise Error::Base, code: :kube_config_not_found, data: { path: kube_config_path }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dapp/deployment/kubernetes/error.rb

@@ -0,0 +1,20 @@
+module Dapp
+  module Deployment
+    module Kubernetes::Error
+      class Base < ::Dapp::Deployment::Error::Kubernetes
+        def initialize(**net_status)
+          super(**net_status, context: :kubernetes)
+        end
+      end
+
+      class NotFound < Base
+        def initialize(**net_status)
+          super({code: :not_found}.merge(net_status))
+        end
+      end
+
+      class Timeout < Base; end
+      class ConnectionRefused < Base; end
+    end
+  end
+end

data/lib/dapp/deployment/mod/jobs.rb

@@ -0,0 +1,37 @@
+module Dapp
+  module Deployment
+    module Mod
+      module Jobs
+        [:bootstrap, :before_apply_job].each do |job|
+          define_method :"to_kube_#{job}_pods" do |repo, image_version|
+            to_kube_default_job_pods(job, repo, image_version)
+          end
+        end
+
+        def to_kube_default_job_pods(directive, repo, image_version)
+          return {} if (directive_config = config.public_send("_#{directive}")).empty?
+          {}.tap do |hash|
+            hash[name(directive)] = {}.tap do |pod|
+              pod['metadata'] = {}.tap do |metadata|
+                metadata['name']   = name(directive)
+                metadata['labels'] = kube.labels
+              end
+              pod['spec'] = {}.tap do |spec|
+                spec['restartPolicy'] = 'Never'
+                spec['containers'] = [].tap do |containers|
+                  containers << {}.tap do |container|
+                    container['imagePullPolicy'] = 'Always'
+                    container['image']           = [repo, [directive_config._dimg || config._dimg, image_version].compact.join('-')].join(':')
+                    container['name']            = name(directive)
+                    container['command']         = directive_config._run unless directive_config._run.empty?
+                    container['env']             = environments unless environments.empty?
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dapp/deployment/mod/namespace.rb

@@ -0,0 +1,58 @@
+module Dapp
+  module Deployment
+    module Mod
+      module Namespace
+        def environment
+          config._environment
+            .merge(namespace_config_directive(:_environment) || {})
+            .merge(system_environments)
+        end
+
+        def secret_environment
+          return {} if secret.nil?
+          config._secret_environment
+            .merge(namespace_config_directive(:_secret_environment) || {})
+            .each_with_object({}) { |(k, v), environments| environments[k] = secret.extract(v) }
+        end
+
+        def environments
+          [environment, secret_environment]
+            .select { |env| !env.empty? }
+            .map { |h| h.map { |k, v| { 'name' => k.to_s, 'value' => v.to_s } } }
+            .flatten
+        end
+
+        def scale
+          namespace_config_directive(:_scale) || config._scale || 1
+        end
+
+        def namespace_config_directive(name)
+          return if namespace_config.nil?
+          namespace_config.send(name)
+        end
+
+        def namespace_config
+          @namespace_config = begin
+            error_class = Error.const_get(self.class.name.to_s.split('::').last)
+            if config._namespace.include?(namespace)
+              config._namespace[namespace]
+            elsif namespace.nil? || namespace == 'default'
+              if config._namespace.one?
+                config._namespace.values.first
+              elsif config._namespace.empty?
+              else
+                if namespace.nil?
+                  raise error_class, code: :namespace_not_defined, data: { name: name }
+                else
+                  raise error_class, code: :namespace_not_found, data: { namespace: namespace, name: name }
+                end
+              end
+            else
+              raise error_class, code: :namespace_not_found, data: { namespace: namespace, name: name }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dapp/deployment/mod/system_environments.rb

@@ -0,0 +1,30 @@
+module Dapp
+  module Deployment
+    module Mod
+      module SystemEnvironments
+        protected
+
+        def system_environments
+          @system_environments ||= begin
+            [namespace_system_env_name_prefix, system_env_name_prefix].each_with_object({}) do |prefix, environments|
+              ENV.map do |k, v|
+                if k.start_with?(prefix)
+                  key = k.sub(prefix, '')
+                  environments[key] = v unless environments.key?(key)
+                end
+              end
+            end
+          end
+        end
+
+        def namespace_system_env_name_prefix
+          [system_env_name_prefix, namespace.to_s.tr('-', '_').upcase].join('_')
+        end
+
+        def system_env_name_prefix
+          'DAPP_DEPLOYMENT'
+        end
+      end
+    end
+  end
+end