dapp 0.22.6 → 0.22.7

data/lib/dapp/dimg/dapp/command/build.rb

@@ -11,7 +11,7 @@ module Dapp
                 Dimg.new(config: config, dapp: self).build!
               end
             end
-          rescue ::Dapp::Error::Shellout, Error::Base
+          rescue ::Dapp::Error::Shellout, Error::Default
             build_context_export unless options[:build_context_directory].nil?
             raise
           end

data/lib/dapp/dimg/dapp/command/build_context/export.rb

@@ -14,7 +14,7 @@ module Dapp
             def export_build_context_image_tar
               lock("#{name}.images", readonly: true) do
                 context_images_names = build_configs.map do |config|
-                  Dimg.new(config: config, dapp: self).tagged_images.map(&:name)
+                  Dimg.new(config: config, dapp: self, ignore_git_fetch: true).tagged_images.map(&:name)
                 end.flatten
 
                 log_secondary_process(:images, short: true) do
@@ -25,18 +25,22 @@ module Dapp
 
             def export_build_context_build_tar
               log_secondary_process(:build_dir, short: true) do
-                tar_write(build_context_build_tar) do |tar|
-                  Dir.glob(File.join(build_path, '**/*'), File::FNM_DOTMATCH).each do |path|
-                    archive_file_path = path
-                                          .reverse
-                                          .chomp(build_path.to_s.reverse)
-                                          .chomp('/')
-                                          .reverse
-                    if File.directory?(path)
-                      tar.mkdir archive_file_path, File.stat(path).mode
-                    else
-                      tar.add_file archive_file_path, File.stat(path).mode do |tf|
-                        tf.write File.read(path)
+                if !!options[:use_system_tar]
+                  shellout!("tar -C #{build_path} -cf #{build_context_build_tar} .")
+                else
+                  tar_write(build_context_build_tar) do |tar|
+                    Dir.glob(File.join(build_path, '**/*'), File::FNM_DOTMATCH).each do |path|
+                      archive_file_path = path
+                        .reverse
+                        .chomp(build_path.to_s.reverse)
+                        .chomp('/')
+                        .reverse
+                      if File.directory?(path)
+                        tar.mkdir archive_file_path, File.stat(path).mode
+                      else
+                        tar.add_file archive_file_path, File.stat(path).mode do |tf|
+                          tf.write File.read(path)
+                        end
                       end
                     end
                   end

data/lib/dapp/dimg/dapp/command/build_context/import.rb

@@ -5,8 +5,8 @@ module Dapp
         module BuildContext
           module Import
             def build_context_import
-              raise Error::Command, code: :context_directory_not_found,
-                                    data: { path: build_context_path } unless build_context_path.exist?
+              raise ::Dapp::Error::Command, code: :context_directory_not_found,
+                                            data: { path: build_context_path } unless build_context_path.exist?
 
               log_process(:'import context') do
                 import_build_context_build_tar
@@ -16,7 +16,7 @@ module Dapp
 
             def import_build_context_image_tar
               if build_context_images_tar.exist?
-                log_secondary_process(:images, short: true) do
+                log_secondary_process(:images) do
                   lock("#{name}.images") do
                     Image::Docker.load!(build_context_images_tar, verbose: true, quiet: log_quiet?)
                   end unless dry_run?
@@ -29,22 +29,29 @@ module Dapp
             def import_build_context_build_tar
               if build_context_build_tar.exist?
                 log_secondary_process(:build_dir, short: true) do
-                  store_current_build_dir
+                  unless dry_run?
+                    store_current_build_dir
 
-                  tar_read(build_context_build_tar) do |tar|
-                    tar.each_entry do |entry|
-                      header = entry.header
-                      path = File.join(build_path, entry.full_name)
+                    if !!options[:use_system_tar]
+                      FileUtils.mkpath build_path
+                      shellout!("tar -xf #{build_context_build_tar} -C #{build_path}")
+                    else
+                      tar_read(build_context_build_tar) do |tar|
+                        tar.each_entry do |entry|
+                          header = entry.header
+                          path = File.join(build_path, entry.full_name)
 
-                      if entry.directory?
-                        FileUtils.mkpath path, :mode => entry.header.mode
-                      else
-                        FileUtils.mkpath File.dirname(path)
-                        File.write(path, entry.read)
-                        File.chmod(header.mode, path)
+                          if entry.directory?
+                            FileUtils.mkpath path, :mode => entry.header.mode
+                          else
+                            FileUtils.mkpath File.dirname(path)
+                            File.write(path, entry.read)
+                            File.chmod(header.mode, path)
+                          end
+                        end
                       end
                     end
-                  end unless dry_run?
+                  end
                 end
               else
                 log_warning(desc: { code: :context_archive_not_found, data: { path: build_context_build_tar } })
@@ -53,8 +60,8 @@ module Dapp
 
             def store_current_build_dir
               return if build_path_empty?
-              raise Error::Command, code: :stored_build_dir_already_exist,
-                                    data: { path: "#{build_path}.old" } if File.exist?("#{build_path}.old")
+              raise ::Dapp::Error::Command, code: :stored_build_dir_already_exist,
+                                            data: { path: "#{build_path}.old" } if File.exist?("#{build_path}.old")
               FileUtils.mv(build_path, "#{build_path}.old")
             end
 

data/lib/dapp/dimg/dapp/command/cleanup_repo.rb

@@ -8,16 +8,18 @@ module Dapp
 
           def cleanup_repo
             lock_repo(repo = option_repo) do
-              registry = dimg_registry(repo)
+              log_step_with_indent(repo) do
+                registry = dimg_registry(repo)
 
-              cleanup_repo_by_nonexistent_git_primitive(registry, actual_detailed_dimgs_images_by_scheme(registry))
-              cleanup_repo_by_policies(registry, actual_detailed_dimgs_images_by_scheme(registry))
+                cleanup_repo_by_nonexistent_git_primitive(registry, actual_detailed_dimgs_images_by_scheme(registry))
+                cleanup_repo_by_policies(registry, actual_detailed_dimgs_images_by_scheme(registry))
 
-              begin
-                repo_dimgs      = repo_dimgs_images(registry)
-                repo_dimgstages = repo_dimgstages_images(registry)
-                repo_dimgstages_cleanup(registry, repo_dimgs, repo_dimgstages)
-              end if with_stages?
+                begin
+                  repo_dimgs      = repo_dimgs_images(registry)
+                  repo_dimgstages = repo_dimgstages_images(registry)
+                  repo_dimgstages_cleanup(registry, repo_dimgs, repo_dimgstages)
+                end if with_stages?
+              end
             end
           end
 
@@ -84,26 +86,25 @@ module Dapp
                   dimg
                 end
               end
-            end
-              .sort_by { |dimg| dimg[:created_at] }
-              .tap do |sorted_dimgs_images|
-                expired_dimgs_images, not_expired_dimgs_images = sorted_dimgs_images.partition do |dimg_image|
-                  dimg_image[:created_at] < DATE_POLICY
-                end
+            end.tap do |detailed_dimgs_images|
+              sorted_detailed_dimgs_images = detailed_dimgs_images.sort_by { |dimg| dimg[:created_at] }
+              expired_dimgs_images, not_expired_dimgs_images = sorted_detailed_dimgs_images.partition do |dimg_image|
+                dimg_image[:created_at] < DATE_POLICY
+              end
 
-                log_step_with_indent(:"date policy (before #{DateTime.strptime(DATE_POLICY.to_s, '%s')})") do
-                  expired_dimgs_images.each { |dimg| delete_repo_image(registry, dimg) }
-                end
+              log_step_with_indent(:"date policy (before #{DateTime.strptime(DATE_POLICY.to_s, '%s')})") do
+                expired_dimgs_images.each { |dimg| delete_repo_image(registry, dimg) }
+              end
 
-                {}.tap do |images_by_dimg|
-                  not_expired_dimgs_images.each { |dimg| (images_by_dimg[dimg[:dimg]] ||= []) << dimg }
-                  images_by_dimg.each do |dimg, images|
-                    log_step_with_indent(:"limit policy (> #{GIT_TAGS_LIMIT_POLICY}) (`#{dimg}`)") do
-                      images[GIT_TAGS_LIMIT_POLICY..-1].each { |dimg| delete_repo_image(registry, dimg) }
-                    end unless images[GIT_TAGS_LIMIT_POLICY..-1].nil?
-                  end
+              {}.tap do |images_by_dimg|
+                not_expired_dimgs_images.each { |dimg| (images_by_dimg[dimg[:dimg]] ||= []) << dimg }
+                images_by_dimg.each do |dimg, images|
+                  log_step_with_indent(:"limit policy (> #{GIT_TAGS_LIMIT_POLICY}) (`#{dimg}`)") do
+                    images[GIT_TAGS_LIMIT_POLICY..-1].each { |dimg| delete_repo_image(registry, dimg) }
+                  end unless images[GIT_TAGS_LIMIT_POLICY..-1].nil?
                 end
               end
+            end
           end
 
           def git_tag_by_consistent_git_tag(consistent_git_tag)
@@ -114,14 +115,6 @@ module Dapp
             @git_consistent_tags ||= git_local_repo.tags.map { |t| [consistent_uniq_slugify(t), t] }.to_h
           end
 
-          def repo_detailed_dimgs_images(registry)
-            repo_dimgs_images(registry).each do |dimg|
-              image_history = registry.image_history(dimg[:tag], dimg[:dimg])
-              dimg[:parent] = image_history['container_config']['Image']
-              dimg[:labels] = image_history['config']['Labels']
-            end
-          end
-
           def deployed_docker_images
             return [] if without_kube?
 
@@ -140,7 +133,7 @@ module Dapp
             begin
               namespaces = client.namespace_list(excon_parameters: {:connect_timeout => 30})
             rescue Excon::Error::Timeout
-              raise Kube::Error::Base, code: :connect_timeout
+              raise ::Dapp::Error::Default, code: :kube_connect_timeout
             end
 
             # get images from containers from pods from all namespaces.

data/lib/dapp/dimg/dapp/command/common.rb

@@ -5,74 +5,60 @@ module Dapp
         module Common
           protected
 
-          def dapp_project_image_labels(image_id)
-            dapp_project_image_inspect(image_id)['Config']['Labels']
+          def dapp_project_dimgstages
+            dapp_project_images.select { |image| image[:dimgstage] }
           end
 
-          def dapp_project_image_inspect(image_id_or_name)
-            return {} unless dapp_project_image_exist?(image_id_or_name)
-
-            dapp_project_images.find { |img| [img[:name], img[:id]].include?(image_id_or_name) }[:inspect] ||= begin
-              cmd = shellout!("#{::Dapp::Dapp.host_docker} inspect --type=image #{image_id_or_name}")
-              Array(JSON.parse(cmd.stdout.strip)).first || {}
-            end
+          def dapp_project_dimgs
+            dapp_project_images.select { |image| image[:dimg] }
           end
 
-          def dapp_project_image_exist?(image_id_or_name)
-            dapp_project_images.any? { |img| [img[:name], img[:id]].include?(image_id_or_name) }
+          def dapp_project_image_by_id(image_id)
+            dapp_project_images.find { |image| image[:id] == image_id }
           end
 
-          def dapp_project_images_ids
-            dapp_project_images.map { |img| img[:id] }
+          def dapp_project_image_labels(image)
+            dapp_project_image_inspect(image)['Config']['Labels']
           end
 
-          def dapp_project_dangling_images
-            dapp_project_images.select { |img| project_dangling_image?(img) }
-          end
-
-          def project_dangling_image?(image)
-            image[:name] == '<none>:<none>'
+          def dapp_project_image_inspect(image)
+            image[:inspect] ||= begin
+              cmd = shellout!("#{::Dapp::Dapp.host_docker} inspect --type=image #{image[:id]}")
+              Array(JSON.parse(cmd.stdout.strip)).first || {}
+            end
           end
 
           def dapp_project_images
             @dapp_project_images ||= [].tap do |images|
-              shellout!(%(#{host_docker} images --format="{{.ID}};{{.Repository}}:{{.Tag}};{{.CreatedAt}}" -f "label=dapp" --no-trunc #{stage_cache}))
-                  .stdout
-                  .lines
-                  .map(&:strip)
-                  .each do |l|
-                id, name, created_at = l.split(';')
-                images << { id: id, name: name, created_at: Time.parse(created_at) }
-              end
+              images.concat prepare_docker_images(stage_cache, dimgstage: true)
+              images.concat prepare_docker_images('-f label=dapp-dimg=true', dimg: true)
             end
           end
 
-          def dapp_project_containers_flush
-            remove_containers_by_query(%(#{host_docker} ps -a -f "label=dapp" -f "name=#{container_name_prefix}" -q --no-trunc))
-          end
-
-          def dapp_project_dangling_images_flush
-            remove_project_images(dapp_project_dangling_images.map { |img| img[:id] })
+          def prepare_docker_images(extra_args, **extra_fields)
+            [].tap do |images|
+              shellout!(%(#{host_docker} images --format="{{.ID}};{{.Repository}}:{{.Tag}};{{.CreatedAt}}" -f "dangling=false" -f "label=dapp=#{name}" --no-trunc #{extra_args}))
+                .stdout
+                .lines
+                .map(&:strip)
+                .each do |l|
+                id, name, created_at = l.split(';')
+                images << { id: id, name: name, created_at: Time.parse(created_at), **extra_fields }
+              end
+            end
           end
 
-          def remove_project_images(images_ids_or_names)
-            images_ids_or_names = convert_to_project_images_names(images_ids_or_names)
-            update_project_images(images_ids_or_names)
-            remove_images(images_ids_or_names)
+          def remove_project_images(project_images)
+            update_project_images_cache(project_images)
+            remove_images(project_images_to_delete(project_images))
           end
 
-          def update_project_images(images_ids_or_names)
-            dapp_project_images.delete_if do |img|
-              images_ids_or_names.include?(img[:id]) || images_ids_or_names.include?(img[:name])
-            end
+          def update_project_images_cache(project_images)
+            dapp_project_images.delete_if { |image| project_images.include?(image) }
           end
 
-          def convert_to_project_images_names(images_ids_or_names)
-            dapp_project_images.each_with_object([]) do |image, images|
-              if images_ids_or_names.include?(image[:id]) || images_ids_or_names.include?(image[:name])
-                images << (project_dangling_image?(image) ? image[:id] : image[:name])
-              end
-            end
+          def project_images_to_delete(project_images)
+            project_images.map { |image| image[:dangling] ? image[:id] : image[:name] }
           end
 
           def dapp_containers_flush
@@ -96,7 +82,7 @@ module Dapp
           def check_user_containers!(images_ids)
             return if images_ids.empty?
             run_command(%(#{host_docker} ps -a -q #{images_ids.uniq.map { |image_id| "--filter=ancestor=#{image_id}" }.join(' ')} --no-trunc)).tap do |res|
-              raise Error::Command, code: :user_containers_detected, data: { ids: res.stdout.strip } if res && !res.stdout.strip.empty? && !dry_run?
+              raise ::Dapp::Error::Command, code: :user_containers_detected, data: { ids: res.stdout.strip } if res && !res.stdout.strip.empty? && !dry_run?
             end
           end
 
@@ -124,20 +110,32 @@ module Dapp
             shellout!(cmd) unless dry_run?
           end
 
+          def dimg_import_export_base(should_be_built: true)
+            repo = option_repo
+            validate_repo_name!(repo)
+            build_configs.each do |config|
+              log_dimg_name_with_indent(config) do
+                Dimg.new(config: config, dapp: self, ignore_git_fetch: true, should_be_built: should_be_built).tap do |dimg|
+                  yield dimg
+                end
+              end
+            end
+          end
+
           def container_name_prefix
             name
           end
 
           def validate_repo_name!(repo)
-            raise Error::Command, code: :repo_name_incorrect, data: { name: repo } unless ::Dapp::Dimg::DockerRegistry.repo_name?(repo)
+            raise ::Dapp::Error::Command, code: :repo_name_incorrect, data: { name: repo } unless ::Dapp::Dimg::DockerRegistry.repo_name?(repo)
           end
 
           def validate_image_name!(image)
-            raise Error::Command, code: :image_name_incorrect, data: { name: image } unless ::Dapp::Dimg::Image::Docker.image_name?(image)
+            raise ::Dapp::Error::Command, code: :image_name_incorrect, data: { name: image } unless ::Dapp::Dimg::Image::Docker.image_name?(image)
           end
 
           def validate_tag_name!(tag)
-            raise Error::Command, code: :tag_name_incorrect, data: { name: tag } unless ::Dapp::Dimg::Image::Docker.tag?(tag)
+            raise ::Dapp::Error::Command, code: :tag_name_incorrect, data: { name: tag } unless ::Dapp::Dimg::Image::Docker.tag?(tag)
           end
 
           def proper_cache_version?
@@ -152,11 +150,6 @@ module Dapp
             log_step_with_indent(:'proper repo cache', &blk)
           end
 
-          def one_dimg!
-            return if build_configs.one?
-            raise Error::Command, code: :command_unexpected_dimgs_number, data: { dimgs_names: build_configs.map(&:_name).join(' ') }
-          end
-
           def push_format(dimg_name)
             if dimg_name.nil?
               spush_format
@@ -169,6 +162,10 @@ module Dapp
             '%{repo}:%{tag}'
           end
 
+          def dimgstage_push_format
+            '%{repo}:dimgstage-%{signature}'
+          end
+
           def with_stages?
             !!options[:with_stages]
           end

data/lib/dapp/dimg/dapp/command/flush_local.rb

@@ -0,0 +1,18 @@
+module Dapp
+  module Dimg
+    module Dapp
+      module Command
+        module FlushLocal
+          def flush_local
+            lock("#{name}.images") do
+              log_step_with_indent(:flush) do
+                remove_project_images(dapp_project_dimgs)
+                stages_flush_local if with_stages?
+              end
+            end
+          end
+        end
+      end
+    end
+  end # Dimg
+end # Dapp

data/lib/dapp/dimg/dapp/command/mrproper.rb

@@ -6,7 +6,7 @@ module Dapp
           # rubocop:disable Metrics/PerceivedComplexity, Metrics/CyclomaticComplexity
           def mrproper
             log_step_with_indent(:mrproper) do
-              raise Error::Command, code: :mrproper_required_option if mrproper_command_without_any_option?
+              raise ::Dapp::Error::Command, code: :mrproper_required_option if mrproper_command_without_any_option?
 
               if proper_all?
                 proper_all

data/lib/dapp/dimg/dapp/command/push.rb

@@ -6,15 +6,9 @@ module Dapp
           def push
             #require 'ruby-prof'
             #RubyProf.start
-            repo = option_repo
-            validate_repo_name!(repo)
             log_step_with_indent(:stages) { stages_push } if with_stages?
-            build_configs.each do |config|
-              log_dimg_name_with_indent(config) do
-                Dimg.new(config: config, dapp: self, ignore_git_fetch: true, should_be_built: true).tap do |dimg|
-                  dimg.export!(repo, format: push_format(config._name))
-                end
-              end
+            dimg_import_export_base do |dimg|
+              dimg.export!(option_repo, format: push_format(dimg.config._name))
             end
             # FIXME: rework images cache, then profile
             #result = RubyProf.stop

data/lib/dapp/dimg/dapp/command/spush.rb

@@ -4,11 +4,9 @@ module Dapp
       module Command
         module Spush
           def spush
-            repo = option_repo
-            validate_repo_name!(repo)
             one_dimg!
-            Dimg.new(config: build_configs.first, dapp: self, ignore_git_fetch: true, should_be_built: true).tap do |dimg|
-              dimg.export!(repo, format: spush_format)
+            dimg_import_export_base do |dimg|
+              dimg.export!(option_repo, format: spush_format)
             end
           end
         end