dapp 0.29.2 → 0.30.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA256:
-  metadata.gz: aca77a164ea6ba0c0cfaebd85974916fee6ece90215572ec4b69a4f525d4b343
-  data.tar.gz: 4b25cc1076c2130714b2210c22ac11ad59697af51bd60e91c0142e9d595bddbd
+SHA1:
+  metadata.gz: 259d9afe89f767678121d96c8ef2b64c1521207c
+  data.tar.gz: 667396adfa0c4f566621b4ed577636e1e82df25a
 SHA512:
-  metadata.gz: 36d882416ba486c071fe29f7891a5f96e8bd95e68d2e55b2d5bc2a8adf7e8bc470128a5d89853f9bb65b2b040abee2c5a779edeb3f7841130793ded23887d220
-  data.tar.gz: 65037b3db41398af71370f9ae8c2eb6769f96b3d3896f45bb479b5226550331e12b80aaac493c2627f02be2d42eabfcbb266a8926f626a86945276671ef6439b
+  metadata.gz: 434fe4c12cdc155d0ce888cce71948ac9fdf29daafcc8327d64d7649b78df948205ea76a2ee777488e1d12f32273178f5e7f03a64671cb5727b3bb207dd91cef
+  data.tar.gz: 97a9dc7ae4e16c42cb1e9002e8eff1dc90d572a013288638b2fc61a124862d37b5089f4f0d7afec841fdfa0413aba323033922b5ba54e8f002bda9cdcd030ed7

data/bin/dapp

@@ -10,29 +10,7 @@ Thread.abort_on_exception = true
 begin
   begin
     begin
-      begin
-        Dapp::CLI.new.run
-      ensure
-        if Time.now.to_date < Date.parse("2019-11-01")
-          STDERR.puts
-          STDERR.puts ::Paint["###################################################################", :red, :bold]
-          STDERR.puts ::Paint["###                   DEPRECATION WARNING!                      ###", :red, :bold]
-          STDERR.puts ::Paint["###  Dapp will be deprecated for use starting with 01.11.2019!  ###", :red, :bold]
-          STDERR.puts ::Paint["###            Please port your project to werf:                ###", :red, :bold]
-          STDERR.puts ::Paint["###                     https://werf.io                         ###", :red, :bold]
-          STDERR.puts ::Paint["###################################################################", :red, :bold]
-          STDERR.puts
-        else
-          STDERR.puts
-          STDERR.puts ::Paint["######################################################################", :red, :bold]
-          STDERR.puts ::Paint["###                     DEPRECATION WARNING!                       ###", :red, :bold]
-          STDERR.puts ::Paint["###  Dapp is deprecated for use and will not receive any support!  ###", :red, :bold]
-          STDERR.puts ::Paint["###               Please port your project to werf:                ###", :red, :bold]
-          STDERR.puts ::Paint["###                       https://werf.io                          ###", :red, :bold]
-          STDERR.puts ::Paint["######################################################################", :red, :bold]
-          STDERR.puts
-        end
-      end
+      Dapp::CLI.new.run
     rescue Dapp::Error::Base => e
       unless (message = Dapp::Helper::NetStatus.before_error_message(e)).empty?
         $stderr.puts(message)

data/config/en/net_status.yml

@@ -136,10 +136,11 @@ en:
       namespace_not_found: "App `%{name}`: namespace `%{namespace}` not found!"
       namespace_not_defined: "App `%{name}`: namespace not defined!"
     kubernetes:
-      config_not_found: "Kube config `%{config_path}` not found!"
-      user_config_not_found: "User's `%{user}` config is not found in config %{config_path}"
-      cluster_config_not_found: "Cluster's `%{cluster}` config is not found in config %{config_path}"
-      config_context_not_found: "Context `%{context_name}` is not found in config %{config_path}"
+      config_not_found: "Kube config not found in: KUBECONFIG, kubectl (through `kubectl config view`), `~/.kube/config`"
+      config_file_not_found: "Kube config file `%{config_path}` not found!"
+      user_config_not_found: "User's `%{user}` config is not found in config `%{config_path}`"
+      cluster_config_not_found: "Cluster's `%{cluster}` config is not found in config `%{config_path}`"
+      context_config_not_found: "Context `%{context_name}` is not found in config `%{config_path}`"
       server_connection_refused: "Kube server `%{url}` connection refused: %{error}"
       server_error: "Kube respond with server error code %{response_http_status}, request parameters: `%{request_parameters}`, response: `%{response_raw_body}`"
       container_stuck: "Pod's `%{pod_name}` container `%{container_name}` stuck in %{state} state: %{state_reason}: %{state_message}"

data/lib/dapp.rb

@@ -83,10 +83,10 @@ require 'dapp/dapp/command/slug'
 require 'dapp/dapp'
 require 'dapp/kube'
 require 'dapp/kube/error/default'
-require 'dapp/kube/error/kubernetes'
 require 'dapp/kube/secret'
 require 'dapp/kube/kubernetes'
 require 'dapp/kube/kubernetes/error'
+require 'dapp/kube/kubernetes/config'
 require 'dapp/kube/kubernetes/client'
 require 'dapp/kube/kubernetes/client/error'
 require 'dapp/kube/kubernetes/client/resource/base'

data/lib/dapp/dapp/sentry.rb

@@ -56,13 +56,16 @@ module Dapp
           },
         }.tap {|extra|
           if git_own_repo_exist?
-            extra["git"] = {
-              "remote_origin_url" => git_own_repo.remote_origin_url, # may contain https token
-              "name" => self.git_url_to_name(git_own_repo.remote_origin_url),
-              "path" => git_own_repo.path,
-              "workdir_path" => git_own_repo.workdir_path,
-              "latest_commit" => git_own_repo.latest_commit,
-            }
+            extra["git"] = {}.tap do |extra_git|
+              if git_own_repo.tracked_remote_repository?
+                extra_git["remote_origin_url"] = git_own_repo.remote_origin_url, # may contain https token
+                extra_git["name"] = self.git_url_to_name(git_own_repo.remote_origin_url)
+              end
+
+              extra_git["path"] = git_own_repo.path
+              extra_git["workdir_path"] = git_own_repo.workdir_path
+              extra_git["latest_commit"] = git_own_repo.latest_commit unless git_own_repo.empty?
+            end
           end
 
           extra["ci-env"] = {"CI" => ENV["CI"]}
@@ -80,7 +83,7 @@ module Dapp
           "dapp-build-cache-version" => ::Dapp::BUILD_CACHE_VERSION,
           "dapp-command" => self.options[:dapp_command],
         }.tap {|tags|
-          if git_own_repo_exist?
+          if git_own_repo_exist? && git_own_repo.tracked_remote_repository?
             tags["git-host"] = self.get_host_from_git_url(git_own_repo.remote_origin_url)
 
             git_name = self.git_url_to_name(git_own_repo.remote_origin_url)

data/lib/dapp/dimg/dapp/command/cleanup_repo.rb

@@ -9,6 +9,12 @@ module Dapp
           def cleanup_repo
             lock_repo(repo = option_repo) do
               log_step_with_indent(repo) do
+                log_step_with_indent("Searching for images being used in kubernetes clusters in one of the kube-config contexts") do
+                  deployed_docker_images.each do |deployed_img|
+                    log(deployed_img)
+                  end
+                end
+
                 registry = dimg_registry(repo)
 
                 if git_own_repo_exist?
@@ -29,18 +35,21 @@ module Dapp
             {}.tap do |detailed_dimgs_images_by_scheme|
               tagging_schemes.each { |scheme| detailed_dimgs_images_by_scheme[scheme] = [] }
               repo_detailed_dimgs_images(registry).each do |image|
-                next unless repo_dimg_image_should_be_ignored?(image)
+                image_repository = [option_repo, image[:dimg]].compact.join('/')
+                image_name = [image_repository, image[:tag]].join(':')
+
+                should_be_ignored = deployed_docker_images.include?(image_name)
+
+                if should_be_ignored
+                  log "Keep in repo image that is being used in kubernetes: #{image_name}"
+                  next
+                end
+
                 (detailed_dimgs_images_by_scheme[image[:labels]['dapp-tag-scheme']] ||= []) << image
               end
             end
           end
 
-          def repo_dimg_image_should_be_ignored?(image)
-            image_repository = [option_repo, image[:dimg]].compact.join('/')
-            image_name = [image_repository, image[:tag]].join(':')
-            !deployed_docker_images.include?(image_name)
-          end
-
           def cleanup_repo_by_nonexistent_git_primitive(registry, detailed_dimgs_images_by_scheme)
             %w(git_tag git_branch git_commit).each do |scheme|
               cleanup_repo_by_nonexistent_git_base(detailed_dimgs_images_by_scheme, scheme) do |detailed_dimg_image|
@@ -117,18 +126,23 @@ module Dapp
           end
 
           def deployed_docker_images
+            @deployed_docker_images ||= search_deployed_docker_images
+          end
+
+          def search_deployed_docker_images
             return [] if without_kube?
 
-            # open kube client, get all pods and select containers' images
-            ::Dapp::Kube::Kubernetes::Client.tap do |kube|
-              config_file = kube.kube_config_path
-              unless File.exist?(config_file)
-                return []
-              end
-            end
+            config = ::Dapp::Kube::Kubernetes::Config.new_auto_if_available
+            return [] if config.nil?
 
-            client = ::Dapp::Kube::Kubernetes::Client.new
+            config.context_names.
+              map {|context_name|
+                client = ::Dapp::Kube::Kubernetes::Client.new(config, context_name, "default")
+                search_deployed_docker_images_from_kube(client)
+              }.flatten.sort.uniq
+          end
 
+          def search_deployed_docker_images_from_kube(client)
             namespaces = []
             # check connectivity for 2 seconds
             begin
@@ -138,7 +152,7 @@ module Dapp
             end
 
             # get images from containers from pods from all namespaces.
-            @kube_images ||= namespaces['items'].map do |item|
+            namespaces['items'].map do |item|
               item['metadata']['name']
             end.map do |ns|
               [].tap do |arr|
@@ -152,9 +166,9 @@ module Dapp
                   arr << replicationcontroller_images(client)
                 end
               end
-            end.flatten.uniq.select do |image|
-              image.start_with?(option_repo)
-            end
+            end.flatten.select do |img|
+              img.start_with? option_repo
+            end.sort.uniq
           end
 
           # pod items[] spec containers[] image
@@ -206,8 +220,6 @@ module Dapp
             end
           end
 
-
-
           def without_kube?
             !!options[:without_kube]
           end

data/lib/dapp/dimg/dimg/git_artifact.rb

@@ -19,10 +19,7 @@ module Dapp
         def remote_git_artifacts
           @remote_git_artifact_list ||= [].tap do |artifacts|
             Array(config._git_artifact._remote).each do |ga_config|
-              repo = GitRepo::Remote.get_or_create(dapp, ga_config._name,
-                                                   url: ga_config._url,
-                                                   branch: ga_config._branch,
-                                                   ignore_git_fetch: ignore_git_fetch)
+              repo = GitRepo::Remote.get_or_create(dapp, ga_config._name, url: ga_config._url, ignore_git_fetch: ignore_git_fetch)
               artifacts.concat(generate_git_artifacts(repo, **ga_config._artifact_options)) unless repo.empty?
             end
           end
@@ -30,7 +27,7 @@ module Dapp
 
         def generate_git_artifacts(repo, **git_artifact_options)
           [].tap do |artifacts|
-            artifacts << (artifact = ::Dapp::Dimg::GitArtifact.new(repo, self, **git_artifact_options))
+            artifacts << (artifact = ::Dapp::Dimg::GitArtifact.new(repo, self, ignore_signature_auto_calculation: ignore_signature_auto_calculation, **git_artifact_options))
             if ENV['DAPP_DISABLE_GIT_SUBMODULES']
               artifacts
             else

data/lib/dapp/dimg/git_artifact.rb

@@ -14,11 +14,13 @@ module Dapp
       # rubocop:disable Metrics/ParameterLists
       def initialize(repo, dimg, to:, name: nil, branch: nil, commit: nil,
                      cwd: nil, include_paths: nil, exclude_paths: nil, owner: nil, group: nil, as: nil,
-                     stages_dependencies: {})
+                     stages_dependencies: {}, ignore_signature_auto_calculation: false)
         @repo = repo
         @dimg = dimg
         @name = name
 
+        @ignore_signature_auto_calculation = ignore_signature_auto_calculation
+
         @branch = branch || repo.dapp.options[:git_artifact_branch] || repo.branch
         @commit = commit
 
@@ -70,10 +72,7 @@ module Dapp
         embedded_rel_path = embedded_params[:path]
         embedded_repo     = begin
           if embedded_params[:type] == :remote
-            GitRepo::Remote.get_or_create(repo.dapp, embedded_rel_path,
-                                          url: embedded_params[:url],
-                                          branch: embedded_params[:branch],
-                                          ignore_git_fetch: dimg.ignore_git_fetch )
+            GitRepo::Remote.get_or_create(repo.dapp, embedded_rel_path, url: embedded_params[:url], ignore_git_fetch: dimg.ignore_git_fetch)
           elsif embedded_params[:type] == :local
             embedded_path = File.join(repo.workdir_path, embedded_params[:path])
             GitRepo::Local.new(repo.dapp, embedded_rel_path, embedded_path)
@@ -104,8 +103,11 @@ module Dapp
               .to_h
           end
           options[:branch]              = embedded_params[:branch]
+          options[:commit]              = embedded_params[:commit]
           options[:owner]               = owner
           options[:group]               = group
+
+          options[:ignore_signature_auto_calculation]= ignore_signature_auto_calculation
         end
       end
 
@@ -288,7 +290,11 @@ module Dapp
       end
 
       def latest_commit
-        @latest_commit ||= (commit || repo.latest_commit(branch))
+        @latest_commit ||= begin
+          (commit || repo.latest_commit(branch)).tap do |c|
+            repo.dapp.log_info("Repository `#{repo.name}`: latest commit `#{c}` to `#{to}`") unless ignore_signature_auto_calculation
+          end
+        end
       end
 
       def paramshash
@@ -325,6 +331,7 @@ module Dapp
       attr_reader :owner
       attr_reader :group
       attr_reader :stages_dependencies
+      attr_reader :ignore_signature_auto_calculation
 
       def sudo
         repo.dapp.sudo_command(owner: owner, group: group)
@@ -409,33 +416,40 @@ module Dapp
       # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
       def change_patch_new_file_path(stage, patch)
         patch.to_s.lines.tap do |lines|
+          prepare_native_path = proc do |path_with_cwd|
+            case archive_type(stage)
+            when :directory
+              path_with_cwd.sub(cwd, '')
+            when :file
+              File.basename(to)
+            else
+              raise
+            end
+          end
+
           modify_patch_line = proc do |line_number, path_char|
             action_part, path_part = lines[line_number].strip.split(' ', 2)
             if (path_with_cwd = path_part.partition("#{path_char}/").last).start_with?(cwd)
-              native_path = case archive_type(stage)
-              when :directory
-                path_with_cwd.sub(cwd, '')
-              when :file
-                File.basename(to)
-              else
-                raise
-              end
-
-              if native_path
-                expected_path = File.join(path_char, native_path)
-                lines[line_number] = [action_part, expected_path].join(' ') + "\n"
+              prepare_native_path.call(path_with_cwd).tap do |native_path|
+                if native_path
+                  expected_path = File.join(path_char, native_path)
+                  lines[line_number] = [action_part, expected_path].join(' ') + "\n"
+                end
               end
-
-              native_path
             end
           end
 
+          modify_patch_first_line = proc do
+            old_file_path = prepare_native_path.call(patch.delta.old_file[:path])
+            new_file_path = prepare_native_path.call(patch.delta.new_file[:path])
+            lines[0] = ['diff --git', File.join('a', old_file_path), File.join('b', new_file_path)].join(' ') + "\n"
+          end
+
           modify_patch = proc do |*modify_patch_line_args|
             native_paths = modify_patch_line_args.map { |args| modify_patch_line.call(*args) }
             unless (native_paths = native_paths.compact.uniq).empty?
               raise Error::Build, code: :unsupported_patch_format, data: { patch: patch.to_s } unless native_paths.one?
-              native_path = native_paths.first
-              lines[0] = ['diff --git', File.join('a', native_path), File.join('b', native_path)].join(' ') + "\n"
+              modify_patch_first_line.call
             end
           end
 
@@ -444,7 +458,11 @@ module Dapp
           when patch.delta.added? then modify_patch.call([4, 'b'])
           when patch.delta.modified?
             if patch_file_mode_changed?(patch)
-              modify_patch.call([4, 'a'], [5, 'b'])
+              if patch.changes.zero?
+                modify_patch_first_line.call
+              else
+                modify_patch.call([4, 'a'], [5, 'b'])
+              end
             else
               modify_patch.call([2, 'a'], [3, 'b'])
             end

data/lib/dapp/dimg/git_repo/base.rb

@@ -49,50 +49,26 @@ module Dapp
         end
 
         def submodules_params(commit, paths: [], exclude_paths: [])
-          raise "Workdir not supported for #{self.class}" if commit.nil?
+          raise "Workdir not supported for `#{self.class}` repository" if commit.nil?
+          submodules(commit, paths: paths, exclude_paths: exclude_paths).map(&method(:submodule_params))
+        end
 
-          entry = begin
-            lookup_commit(commit).tree.path('.gitmodules')
-          rescue Rugged::TreeError
-            return []
+        def submodule_params(submodule)
+          {}.tap do |params|
+            params[:path]   = submodule.path
+            params[:url]    = submodule_url(submodule.url)
+            params[:type]   = url_protocol(params[:url]) == :noname ? :local : :remote
+            params[:commit] = submodule.head_oid
           end
-
-          submodules_params_base(lookup_object(entry[:oid]).content, paths: paths, exclude_paths: exclude_paths)
         end
 
-        def submodules_params_base(gitsubmodule_content, paths: [], exclude_paths: [])
-          submodules_file_parse!(gitsubmodule_content)
-            .select { |_, params| !ignore_directory?(params['path'], paths: paths, exclude_paths: exclude_paths) }
-            .map do |_, params|
-              params = params.symbolize_keys
-              params[:branch] = params[:branch].to_s if params.key?(:branch)
-              params[:url] = submodule_url(params[:url])
-              params[:type] = url_protocol(params[:url]) == :noname ? :local : :remote
-              params
-            end
+        def submodules(commit, paths: [], exclude_paths: [])
+          Rugged::SubmoduleCollection.new(submodules_git(commit))
+            .select { |s| !ignore_directory?(s.path, paths: paths, exclude_paths: exclude_paths) }
         end
 
-        def submodules_file_parse!(gitsubmodule_content)
-          raise_error = proc do |error_message|
-            raise Error::Rugged, code: :incorrect_gitmodules_file, data: { name: self.name,
-                                                                           error: error_message,
-                                                                           content: gitsubmodule_content.strip }
-          end
-
-          begin
-            IniFile.new.parse(gitsubmodule_content).to_h.tap do |submodules_params|
-              submodules_params.each do |name, params|
-                %w(path url).each do |field|
-                  next unless params[field].nil? || params[field].empty?
-                  raise_error.call("field `#{field}` required (#{name})")
-                end
-
-                raise_error.call("path should be relative (#{name})") unless Pathname(params['path']).relative?
-              end
-            end
-          rescue IniFile::Error => e
-            raise_error.call(e.message)
-          end
+        def submodules_git(_)
+          git
         end
 
         def submodule_url(gitsubmodule_url)
@@ -178,7 +154,7 @@ module Dapp
             .select { |b| b.start_with?('origin/') }
             .map { |b| b.reverse.chomp('origin/'.reverse).reverse }
         end
-        
+
         def find_commit_id_by_message(regex)
           walker.each do |commit|
             msg = commit.message.encode('UTF-8', invalid: :replace, undef: :replace)
@@ -204,6 +180,10 @@ module Dapp
           git.empty?
         end
 
+        def tracked_remote_repository?
+          !git.remotes.to_a.empty?
+        end
+
         protected
 
         def git(**kwargs)

data/lib/dapp/dimg/git_repo/local.rb

@@ -26,22 +26,20 @@ module Dapp
           end
         end
 
-        # NOTICE: Параметры {from: nil, to: nil} можно указать только для Own repo.
-        # NOTICE: Для Remote repo такой вызов не имеет смысла и это ошибка пользователя класса Remote.
-
         def submodules_params(commit, paths: [], exclude_paths: [])
-          return super unless commit.nil?
-          return []    unless File.file?((gitmodules_file_path = File.join(workdir_path, '.gitmodules')))
-
-          submodules_params_base(File.read(gitmodules_file_path),
-                                 paths: paths,
-                                 exclude_paths: exclude_paths).each do |submodule_params|
-            submodule_path = File.join(workdir_path, submodule_params[:path])
-            if git_repo_exist?(submodule_path)
-              dapp.log_info("Using local submodule `#{submodule_path}`!")
-              submodule_params[:type] = :local
+          submodules(commit, paths: paths, exclude_paths: exclude_paths).map do |submodule|
+            next if commit.nil? && !submodule.in_config?
+            submodule_params(submodule).tap do |params|
+              if commit.nil?
+                submodule_path = File.join(workdir_path, params[:path])
+                params[:commit] = submodule.workdir_oid || submodule.head_oid
+                if git_repo_exist?(submodule_path)
+                  dapp.log_info("Using local submodule `#{params[:path]}`!")
+                  params[:type] = :local
+                end
+              end
             end
-          end
+          end.compact
         end
 
         def ignore_patch?(patch, paths: [], exclude_paths: [])
@@ -58,6 +56,9 @@ module Dapp
           mode == 0o040000
         end
 
+        # NOTICE: Параметры {from: nil, to: nil} можно указать только для Own repo.
+        # NOTICE: Для Remote repo такой вызов не имеет смысла и это ошибка пользователя класса Remote.
+
         def diff(from, to, **kwargs)
           if from.nil? and to.nil?
             mid_commit = latest_commit