dapp 0.13.8 → 0.13.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4fa2488a50074d7d9e8fe4f203cd8e6f20029aea
-  data.tar.gz: ae116aebb7ee348914e388e989f417f6f3d23567
+  metadata.gz: 3b002b0baef639d31484f8a6dc71dfacaa786683
+  data.tar.gz: a19fc91d6667e26f4a64b4b232583553c5c16e9f
 SHA512:
-  metadata.gz: b48cf76b9e22b0af9aa9ba8de0fd11f8af0eb97d257877b0d8e15fb0640bbb6b389013338276ceefa93d6e81f1a854b9aa486006dd21ccf403c86c10920446e6
-  data.tar.gz: 016ac3f9bab694d8142b385fc55387d210069616d515aa43662b0487cfec5838457c1fad2ffe9a5f493808a8fedf1e8027e6472bf591e92afdb09aff2ca885b1
+  metadata.gz: 2b776834a1dbef3450b0d61d7bbefdb0afac5ee4cf80df121520e3fb540b9fcc0d3dc5567ac66ba2161df5880083ac687468c157200da64e6807f208bd401bdf
+  data.tar.gz: bed8a4f7b6fd98cb8b0dcbd7f3dfb357d61a05cb32a0c9b6173e6e5f013839ce0e3a3172d7653945c5339edb2168b2078c3594140c191a50a5887d0532196190

data/config/en/net_status.yml

@@ -39,6 +39,7 @@ en:
       secret_values_file_not_found: "Secret values file `%{path}` not found!"
       user_containers_detected: "User containers have been using images!\n%{ids}"
       incorrect_yaml_structure: "Incorrect YAML structure in `%{path}`!"
+      editor_not_found: "Editor Not found!"
     dapp:
       no_such_dimg: "No such dimg: `%{dimgs_patterns}`!"
       no_such_app: "No such app: `%{apps_patterns}`!"
@@ -48,6 +49,8 @@ en:
       docker_not_found: "Docker not found!"
       docker_version: "Required docker version: >= %{version}!"
       dapp_config_file_incorrect: "%{message}"
+      yaml_file_incorrect: "Yaml file `%{file}`: `%{message}`"
+      yaml_incorrect: "`%{message}`"
       incorrect_dapp_config_option: ".dapp_config includes option `%{option}` with incorrect value `%{value}`: expected one of the following types [%{expected}]"
       incorrect_dapp_config_option_color: ".dapp_config includes option `color` with incorrect value `%{value}`: expected one of the following values [%{expected}]"
     config:
@@ -66,6 +69,7 @@ en:
       mount_from_type_required: "Mount: `from` directive expect `build_dir` or `tmp_dir` type!"
       builder_type_conflict: 'Conflict between builder types!'
       docker_from_incorrect: "`docker.from` has incorrect value `%{name}`!"
+      docker_artifact_unsupported_directive: "`artifact.docker` supports only `from` subdirective!"
       stage_artifact_not_associated: "Artifact not associated with any stage: expected `before` or `after` attribute!"
       stage_artifact_double_associate: "Cannot use `%{stage}` stage for artifact, already used in `%{conflict_stage}` stage!"
       stage_artifact_not_supported_associated_stage: "Bad artifact stage `%{stage}`!"

data/lib/dapp.rb

@@ -32,6 +32,7 @@ require 'dapp/helper/trivia'
 require 'dapp/helper/sha256'
 require 'dapp/helper/net_status'
 require 'dapp/helper/tar'
+require 'dapp/helper/yaml'
 require 'dapp/prctl'
 require 'dapp/error/base'
 require 'dapp/error/dapp'
@@ -105,8 +106,17 @@ require 'dapp/kube'
 require 'dapp/kube/error/base'
 require 'dapp/kube/error/command'
 require 'dapp/kube/error/kubernetes'
-require 'dapp/kube/client'
-require 'dapp/kube/client/error'
+require 'dapp/kube/secret'
+require 'dapp/kube/kubernetes'
+require 'dapp/kube/kubernetes/client'
+require 'dapp/kube/kubernetes/client/error'
+require 'dapp/kube/kubernetes/client/resource/base'
+require 'dapp/kube/kubernetes/client/resource/pod'
+require 'dapp/kube/kubernetes/client/resource/job'
+require 'dapp/kube/kubernetes/manager/base'
+require 'dapp/kube/kubernetes/manager/pod'
+require 'dapp/kube/kubernetes/manager/container'
+require 'dapp/kube/kubernetes/manager/job'
 require 'dapp/kube/cli/command/base'
 require 'dapp/kube/cli/command/kube'
 require 'dapp/kube/cli/command/kube/deploy'
@@ -115,7 +125,9 @@ require 'dapp/kube/cli/command/kube/secret_key_generate'
 require 'dapp/kube/cli/command/kube/secret_generate'
 require 'dapp/kube/cli/command/kube/secret_extract'
 require 'dapp/kube/cli/command/kube/secret_regenerate'
+require 'dapp/kube/cli/command/kube/secret_edit'
 require 'dapp/kube/cli/command/kube/minikube_setup'
+require 'dapp/kube/cli/command/kube/chart_create'
 require 'dapp/kube/cli/cli'
 require 'dapp/kube/dapp/command/common'
 require 'dapp/kube/dapp/command/deploy'
@@ -124,9 +136,10 @@ require 'dapp/kube/dapp/command/secret_key_generate'
 require 'dapp/kube/dapp/command/secret_generate'
 require 'dapp/kube/dapp/command/secret_extract'
 require 'dapp/kube/dapp/command/secret_regenerate'
+require 'dapp/kube/dapp/command/secret_edit'
 require 'dapp/kube/dapp/command/minikube_setup'
+require 'dapp/kube/dapp/command/chart_create'
 require 'dapp/kube/dapp/dapp'
-require 'dapp/kube/secret'
 require 'dapp/dimg'
 require 'dapp/dimg/builder'
 require 'dapp/dimg/builder/base'

data/lib/dapp/cli/command/update.rb

@@ -7,6 +7,8 @@ module Dapp
             File.fnmatch(File.join(s.full_gem_path, '*'), __FILE__)
           end
           Gem.install(spec.name, approximate_recommendation(spec.version))
+        rescue Gem::FilePermissionError => e
+          raise Errno::EACCES, e.message
         end
 
         # get latest beta-version

data/lib/dapp/dapp/dapp_config.rb

@@ -42,7 +42,7 @@ module Dapp
           config_search_paths.reduce({}) do |options, dir|
             if (config_options_path = make_path(dir, '.dapp_config')).file?
               config_options = begin
-                YAML.load_file(config_options_path).tap do |c_options|
+                yaml_load_file(config_options_path).tap do |c_options|
                   c_options.merge!(c_options.in_depth_merge(c_options['ci'] || {})) if ENV['GITLAB_CI'] || ENV['TRAVIS']
                   c_options.delete('ci')
                 end

data/lib/dapp/deployment/kubernetes.rb

@@ -163,7 +163,7 @@ module Dapp
       def kube_config
         @kube_config ||= begin
           if File.exist?((kube_config_path = File.join(ENV['HOME'], '.kube/config')))
-            YAML.load_file(kube_config_path)
+            yaml_load_file(kube_config_path)
           else
             raise Error::Base, code: :kube_config_not_found, data: { path: kube_config_path }
           end

data/lib/dapp/dimg/build/stage/base.rb

@@ -231,7 +231,7 @@ module Dapp
           end
 
           def image_should_be_prepared?
-            !image.built? && !should_be_not_present? || image_should_be_introspected?
+            (!image.built? && !should_be_not_present? || image_should_be_introspected?) && !dimg.dapp.dry_run?
           end
 
           def should_be_not_present?

data/lib/dapp/dimg/config/directive/docker/artifact.rb

@@ -4,6 +4,11 @@ module Dapp
       module Directive
         module Docker
           class Artifact < Base
+            def method_missing(m, *args)
+              raise Error::Config, code: :docker_artifact_unsupported_directive if Dimg.instance_methods.include?(m)
+              super
+            end
+
             def _change_options
               {}
             end

data/lib/dapp/helper/yaml.rb

@@ -0,0 +1,23 @@
+module Dapp
+  module Helper
+    module YAML
+      def yaml_load_file(file_path, hash: true)
+        yaml_load(File.read(file_path), hash: hash)
+      rescue ::Dapp::Error::Dapp => e
+        raise ::Dapp::Error::Dapp, code: :yaml_file_incorrect, data: { file: file_path, message: e.net_status[:data][:message] }
+      end
+
+      def yaml_load(string, hash: true)
+        ::YAML::load(string).tap do |res|
+          if hash && !res.is_a?(Hash)
+            raise ::Dapp::Error::Dapp,
+                  code: :yaml_incorrect,
+                  data: { message: "unexpected json data \n>>> START YAML\n#{string.strip}\n>>> END YAML\n" }
+          end
+        end
+      rescue Psych::SyntaxError => e
+        raise ::Dapp::Error::Dapp, code: :yaml_incorrect, data: { message: e.message }
+      end
+    end
+  end # Helper
+end # Dapp

data/lib/dapp/kube/cli/command/kube.rb

@@ -1,7 +1,7 @@
 module Dapp::Kube::CLI
   module Command
     class Kube < ::Dapp::CLI
-      SUBCOMMANDS = ['secret generate', 'secret key generate', 'secret regenerate', 'deploy', 'dismiss', 'secret extract', 'minikube setup'].freeze
+      SUBCOMMANDS = ['secret generate', 'secret key generate', 'secret regenerate', 'deploy', 'dismiss', 'secret extract', 'secret edit', 'minikube setup', 'chart create'].freeze
 
       banner <<BANNER.freeze
 Usage: dapp kube subcommand [subcommand options]
@@ -14,7 +14,9 @@ Available subcommands: (for details, dapp kube SUB-COMMAND --help)
   dapp kube secret generate [FILE_PATH] [options]
   dapp kube secret extract [FILE_PATH] [options]
   dapp kube secret regenerate [SECRET_VALUES_FILE_PATH ...] [options]
+  dapp kube secret edit [FILE_PATH] [options]
   dapp kube minikube setup
+  dapp kube chart create [options]
 
 Options:
 BANNER

data/lib/dapp/kube/cli/command/kube/chart_create.rb

@@ -0,0 +1,19 @@
+module Dapp::Kube::CLI::Command
+  class Kube < ::Dapp::CLI
+    class ChartCreate < Base
+      banner <<BANNER.freeze
+Usage:
+
+  dapp kube chart create [options]
+
+Options:
+BANNER
+
+      option :force,
+             long: '--force',
+             short: '-f',
+             description: 'Delete existing chart',
+             default: false
+    end
+  end
+end

data/lib/dapp/kube/cli/command/kube/secret_edit.rb

@@ -0,0 +1,24 @@
+module Dapp::Kube::CLI::Command
+  class Kube < ::Dapp::CLI
+    class SecretEdit < Base
+      banner <<BANNER.freeze
+Usage:
+
+  dapp kube secret edit [FILE_PATH] [options]
+
+Options:
+BANNER
+
+      option :values,
+             long: '--values',
+             description: 'Edit secret values file',
+             default: false
+
+      def run(argv = ARGV)
+        self.class.parse_options(self, argv)
+        file_path = self.class.required_argument(self, 'FILE_PATH')
+        ::Dapp::Dapp.new(options: cli_options).public_send(run_method, file_path)
+      end
+    end
+  end
+end

data/lib/dapp/kube/dapp/command/chart_create.rb

@@ -0,0 +1,104 @@
+module Dapp
+  module Kube
+    module Dapp
+      module Command
+        module ChartCreate
+          def kube_chart_create
+            with_kube_tmp_chart_dir do
+              FileUtils.cp_r kube_chart_path, kube_tmp_chart_path(name) if kube_chart_path.directory? && !options[:force]
+
+              shellout!("helm create #{name}", cwd: kube_tmp_chart_path)
+              kube_create_chart_samples
+
+              FileUtils.rm_rf kube_chart_path
+              FileUtils.mv kube_tmp_chart_path(name), kube_chart_path
+            end
+          end
+
+          def kube_create_chart_samples
+            kube_tmp_chart_path(name, 'secret-values.yaml').tap { |f| FileUtils.touch(f) unless f.file? }
+            kube_tmp_chart_path(name, kube_chart_secret_dir_name).tap { |dir| FileUtils.mkdir(dir) unless dir.directory? }
+            kube_tmp_chart_path(name, 'templates',  '_envs.tpl').tap do |f|
+              f.write begin
+                <<-EOF
+{{- define "common_envs" }}
+- name: A
+  value: value
+{{-   if eq .Values.global.env "production" }}
+- name: B
+  value: value
+{{-   else }}
+- name: B
+  value: value2
+{{-     if or (eq .Values.global.env "staging") (eq .Values.global.env "testing") }}
+- name: C
+  value: value3
+{{-     end }}
+{{-   end }}
+{{- end }}
+                EOF
+              end unless f.file?
+            end
+            kube_tmp_chart_path(name, 'templates', 'backend.yaml').tap do |f|
+              f.write begin
+                <<-EOF
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: {{ .Chart.Name }}-backend
+  labels:
+    service: {{ .Chart.Name }}-backend
+spec:
+  minReadySeconds: 60
+  strategy:
+    type: RollingUpdate
+  replicas: 2
+  template:
+    metadata:
+      labels:
+        service: {{ .Chart.Name }}-backend
+    spec:
+      volumes:
+      - name: {{ .Chart.Name }}-backend
+        secret:
+          secretName: {{ .Chart.Name }}-backend
+      containers:
+      - command: [ '/bin/bash', '-l', '-c', 'bundle exec ctl start' ]
+        image: {{ tuple . 'specific_name' | include "dimg" }} # or nameless dimg {{ tuple . | include "dimg" }}
+        imagePullPolicy: Always
+        name: {{ .Chart.Name }}-backend
+        livenessProbe:
+          httpGet:
+            path: /assets/logo.png
+            port: 8080
+            scheme: HTTP
+          initialDelaySeconds: 10
+          periodSeconds: 3
+        ports:
+        - containerPort: 8080
+          name: http
+          protocol: TCP
+        env:
+{{- include "common_envs" . | indent 8 }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Chart.Name }}-backend
+spec:
+  type: ClusterIP
+  selector:
+    service: {{ .Chart.Name }}-backend
+  ports:
+  - name: http
+    port: 8080
+    protocol: TCP
+                EOF
+              end unless f.file?
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dapp/kube/dapp/command/common.rb

@@ -46,13 +46,32 @@ module Dapp
             ) if secret.nil?
           end
 
+          def kube_chart_secret_path(*path)
+            kube_chart_path(kube_chart_secret_dir_name, *path).expand_path
+          end
+
+          def kube_chart_secret_dir_name
+            'secret'
+          end
+
           def kube_chart_path(*path)
             self.path('.helm', *path).expand_path
           end
 
+          def with_kube_tmp_chart_dir
+            yield if block_given?
+          ensure
+            FileUtils.rm_rf(kube_tmp_chart_path)
+          end
+
+          def kube_tmp_chart_path(*path)
+            @kube_tmp_path ||= Dir.mktmpdir('dapp-helm-chart-', tmp_base_dir)
+            make_path(@kube_tmp_path, *path).expand_path.tap { |p| p.parent.mkpath }
+          end
+
           def secret
             @secret ||= begin
-              unless secret_key = ENV['DAPP_SECRET_KEY']
+              unless (secret_key = ENV['DAPP_SECRET_KEY'])
                 secret_key_not_found_in << '`DAPP_SECRET_KEY`'
 
                 if dappfile_exists?
@@ -76,7 +95,7 @@ module Dapp
           def kubernetes
             @kubernetes ||= begin
               namespace = options[:namespace].nil? ? nil : options[:namespace].tr('_', '-')
-              Client.new(namespace: namespace)
+              Kubernetes::Client.new(namespace: namespace)
             end
           end
         end

data/lib/dapp/kube/dapp/command/deploy.rb

@@ -13,7 +13,7 @@ module Dapp
             validate_repo_name!(repo)
             validate_tag_name!(image_version)
 
-            begin
+            with_kube_tmp_chart_dir do
               kube_copy_chart
               kube_helm_decode_secrets
               kube_generate_helm_chart_tpl
@@ -29,10 +29,11 @@ module Dapp
                 options.concat(self.options[:helm_set_options].map { |opt| "--set #{opt}" })
               end
 
-              kube_flush_hooks_jobs(additional_values, set_options)
-              kube_run_deploy(additional_values, set_options)
-            ensure
-              FileUtils.rm_rf(kube_tmp_chart_path)
+              hooks_jobs = kube_helm_hooks_jobs(additional_values, set_options)
+
+              kube_flush_hooks_jobs(hooks_jobs)
+
+              kube_run_deploy(additional_values, set_options, hooks_jobs: hooks_jobs)
             end
           end
 
@@ -54,7 +55,7 @@ module Dapp
 
           def kube_helm_decode_secret_values
             kube_secret_values_paths.each_with_index do |secret_values_file, index|
-              decoded_data = kube_helm_decode_json(secret, YAML::load(secret_values_file.read))
+              decoded_data = kube_helm_decode_json(secret, yaml_load_file(secret_values_file))
               kube_tmp_chart_secret_values_paths[index].write(decoded_data.to_yaml)
             end
           end
@@ -75,14 +76,14 @@ module Dapp
 
 {{- define "dimg" -}}
 {{- if (ge (len (index .)) 2) -}}
-{{- $name := index . 0 -}}                                                                          
-{{- $context := index . 1 -}}  
+{{- $name := index . 0 -}}
+{{- $context := index . 1 -}}
 {{- printf "%s:%s-%s" $context.Values.global.dapp.repo $name $context.Values.global.dapp.image_version -}}
 {{- else -}}
 {{- $context := index . 0 -}}
 {{- printf "%s:%s" $context.Values.global.dapp.repo $context.Values.global.dapp.image_version -}}
-{{- end -}}                                                                               
-{{- end -}}                                                                                         
+{{- end -}}
+{{- end -}}
 
 {{- define "dapp_secret_file" -}}
 {{- $relative_file_path := index . 0 -}}
@@ -93,16 +94,16 @@ module Dapp
             kube_tmp_chart_path('templates/_dapp_helpers.tpl').write(cont)
           end
 
-          def kube_flush_hooks_jobs(additional_values, set_options)
-            return if (config_jobs_names = kube_helm_hooks_jobs_to_delete(additional_values, set_options).keys).empty?
+          def kube_flush_hooks_jobs(hooks_jobs)
+            return if (config_jobs_names = hooks_jobs.keys).empty?
             config_jobs_names.select { |name| kube_job_list.include? name }.each do |name|
               log_process("Delete hooks job `#{name}` for release #{kube_release_name} ", short: true) { kube_delete_job!(name) }
             end
           end
 
-          def kube_helm_hooks_jobs_to_delete(additional_values, set_options)
+          def kube_helm_hooks_jobs(additional_values, set_options)
             generator = proc do |text|
-              text.split(/# Source.*|---/).reject {|c| c.strip.empty? }.map {|c| YAML::load(c) }.reduce({}) do |objects, c|
+              text.split(/# Source.*|---/).reject {|c| c.strip.empty? }.map {|c| yaml_load(c) }.reduce({}) do |objects, c|
                 objects[c['kind']] ||= {}
                 objects[c['kind']][(c['metadata'] || {})['name']] = c
                 objects
@@ -114,7 +115,7 @@ module Dapp
 
             manifest_start_index = output.lines.index("MANIFEST:\n") + 1
             hook_start_index     = output.lines.index("HOOKS:\n") + 1
-            configs = generator.call(output.lines[hook_start_index..manifest_start_index-1].join)
+            configs = generator.call(output.lines[hook_start_index..manifest_start_index-2].join)
 
             (configs['Job'] || {}).reject do |_, c|
               c['metadata'] ||= {}
@@ -135,11 +136,44 @@ module Dapp
             end
           end
 
-          def kube_run_deploy(additional_values, set_options)
+          def kube_run_deploy(additional_values, set_options, hooks_jobs: {})
             log_process("Deploy release #{kube_release_name}") do
+              release_exists = shellout("helm status #{kube_release_name}").status.success?
+
+              hooks_jobs_by_type = hooks_jobs
+                .reduce({}) do |res, (job_name, job_spec)|
+                  job = Kubernetes::Client::Resource::Job.new(job_spec)
+
+                  if job.annotations['dapp/watch-logs'].to_s == 'true'
+                    job.annotations['helm.sh/hook'].to_s.split(',').each do |hook_type|
+                      res[hook_type] ||= []
+                      res[hook_type] << job
+                    end
+                  end
+
+                  res
+                end
+                .tap do |res|
+                  res.values.each do |jobs|
+                    jobs.sort_by! {|job| job.annotations['helm.sh/hook-weight'].to_i}
+                  end
+                end
+
+              watch_jobs = if release_exists
+                hooks_jobs_by_type['pre-upgrade'].to_a + hooks_jobs_by_type['post-upgrade'].to_a
+              else
+                hooks_jobs_by_type['pre-install'].to_a + hooks_jobs_by_type['post-install'].to_a
+              end
+
+              watch_thr = Thread.new do
+                watch_jobs.each {|job| Kubernetes::Manager::Job.new(self, job.name).watch_till_done!}
+              end
+
               args = [kube_release_name, kube_tmp_chart_path, additional_values, set_options, kube_helm_extra_options].flatten
               kubernetes.create_namespace!(kube_namespace) unless kubernetes.namespace?(kube_namespace)
               shellout! "helm upgrade #{args.join(' ')}", verbose: true
+
+              watch_thr.join
             end
           end
 
@@ -163,11 +197,6 @@ module Dapp
             kube_tmp_chart_path('decoded-secret', *path).tap { |p| p.parent.mkpath }
           end
 
-          def kube_tmp_chart_path(*path)
-            @kube_tmp_path ||= Dir.mktmpdir('dapp-helm-chart-', tmp_base_dir)
-            make_path(@kube_tmp_path, *path).expand_path.tap { |p| p.parent.mkpath }
-          end
-
           def kube_values_paths
             self.options[:helm_values_options].map { |p| Pathname(p).expand_path }.each do |f|
               raise Error::Command, code: :values_file_not_found, data: { path: f } unless f.file?
@@ -190,10 +219,6 @@ module Dapp
           def kube_chart_secret_values_path
             kube_chart_path('secret-values.yaml').expand_path
           end
-
-          def kube_chart_secret_path(*path)
-            kube_chart_path('secret', *path).expand_path
-          end
         end
       end
     end