dapp 0.22.6 → 0.22.7

data/lib/dapp/dimg/dapp/command/stages/cleanup_local.rb

@@ -6,9 +6,7 @@ module Dapp
           module CleanupLocal
             def stages_cleanup_local
               lock_repo(option_repo, readonly: true) do
-                raise Error::Command, code: :stages_cleanup_required_option unless stages_cleanup_option?
-
-                dapp_project_containers_flush
+                raise ::Dapp::Error::Command, code: :stages_cleanup_required_option unless stages_cleanup_option?
 
                 proper_cache           if proper_cache_version?
                 stages_cleanup_by_repo if proper_repo_cache?
@@ -21,18 +19,13 @@ module Dapp
             def proper_cache
               log_proper_cache do
                 lock("#{name}.images") do
-                  remove_project_images(dapp_project_images_ids.select { |image_id| !actual_cache_project_images_ids.include?(image_id) })
+                  remove_project_images(dapp_project_dimgstages - actual_cache_project_dimgstages)
                 end
               end
             end
 
-            def actual_cache_project_images_ids
-              @actual_cache_project_images_ids ||= begin
-                shellout!(%(#{host_docker} images -f "label=dapp" -f "label=dapp-cache-version=#{::Dapp::BUILD_CACHE_VERSION}" -q --no-trunc #{stage_cache}))
-                  .stdout
-                  .lines
-                  .map(&:strip)
-              end
+            def actual_cache_project_dimgstages
+              @actual_cache_project_images_ids ||= prepare_docker_images("-f \"label=dapp-cache-version=#{::Dapp::BUILD_CACHE_VERSION}\" #{stage_cache}")
             end
 
             def stages_cleanup_by_repo
@@ -40,74 +33,76 @@ module Dapp
                 lock("#{name}.images") do
                   registry   = dimg_registry(option_repo)
                   repo_dimgs = repo_detailed_dimgs_images(registry)
+                  dimgstages = clone_dapp_project_dimgstages
 
-                  dapp_project_dangling_images_flush
-
-                  _, dimgstages = dapp_project_images.partition { |image| repo_dimgs.any? { |dimg| dimg[:id] == image[:id] } }
-                  repo_dimgs.each { |repo_dimg| except_dapp_project_image_with_parents(repo_dimg[:parent], dimgstages) }
+                  repo_dimgs.each { |repo_dimg| except_image_id_with_parents(repo_dimg[:parent], dimgstages) }
 
                   # Удаление только образов старше 2ч
                   dimgstages.delete_if do |dimgstage|
                     Time.now - dimgstage[:created_at] < 2*60*60
                   end
 
-                  remove_project_images(dimgstages.map { |dimgstage| dimgstage[:id]} )
+                  remove_project_images(dimgstages)
                 end
               end
             end
 
-            def except_dapp_project_image_with_parents(image_id, dimgstages)
-              if dapp_project_image_exist?(image_id)
-                dapp_project_image_artifacts_ids_in_labels(image_id).each { |aiid| except_dapp_project_image_with_parents(aiid, dimgstages) }
-                iid = image_id
-                loop do
-                  dimgstages.delete_if { |dimgstage| dimgstage[:id] == iid }
-                  break if (iid = dapp_project_image_parent_id(iid)).nil?
-                end
-              else
-                dimgstages.delete_if { |dimgstage| dimgstage[:id] == dimgstage }
+            def clone_dapp_project_dimgstages
+              Marshal.load(Marshal.dump(dapp_project_dimgstages))
+            end
+
+            def except_image_id_with_parents(image_id, dimgstages)
+              return unless (project_image = dapp_project_image_by_id(image_id))
+              except_dapp_project_image_with_parents(project_image, dimgstages)
+            end
+
+            def except_dapp_project_image_with_parents(image, dimgstages)
+              dapp_project_image_artifacts_ids_in_labels(image).each { |aiid| except_image_id_with_parents(aiid, dimgstages) }
+              i = image
+              loop do
+                dimgstages.delete_if { |dimgstage| dimgstage == i }
+                break if (i = dapp_project_image_parent(i)).nil?
               end
             end
 
-            def dapp_project_image_artifacts_ids_in_labels(image_id)
-              select_dapp_artifacts_ids(dapp_project_image_labels(image_id))
+            def dapp_project_image_artifacts_ids_in_labels(image)
+              select_dapp_artifacts_ids(dapp_project_image_labels(image))
             end
 
             def proper_git_commit
               log_proper_git_commit do
                 lock("#{name}.images") do
-                  dapp_project_dangling_images_flush
-
-                  unproper_images_ids = []
-                  dapp_project_images_ids.each do |image_id|
-                    dapp_project_image_labels(image_id).each do |repo_name, commit|
+                  unproper_images = []
+                  dapp_project_dimgstages.each do |dimgstage|
+                    dapp_project_image_labels(dimgstage).each do |repo_name, commit|
                       next if (repo = dapp_git_repositories[repo_name]).nil?
-                      unproper_images_ids.concat(dapp_project_image_hierarchy(image_id)) unless repo.commit_exists?(commit)
+                      unproper_images.concat(dapp_project_image_with_children(dimgstage)) unless repo.commit_exists?(commit)
                     end
                   end
-                  remove_project_images(unproper_images_ids)
+                  remove_project_images(unproper_images)
                 end
               end
             end
 
-            def dapp_project_image_hierarchy(image_id)
-              hierarchy = []
-              iids = [image_id]
+            def dapp_project_image_with_children(image)
+              children = []
+              images   = [image]
 
               loop do
-                hierarchy.concat(dapp_project_images_ids.select { |image_id| iids.include?(image_id) })
-                break if begin
-                  iids.map! do |iid|
-                    dapp_project_images_ids.select { |image_id| dapp_project_image_parent_id(image_id) == iid }
-                  end.flatten!.empty?
+                children.concat(dapp_project_images.select { |project_image| images.include?(project_image) })
+                images.map! do |parent_image|
+                  dapp_project_images
+                    .select { |project_image| dapp_project_image_parent(project_image) == parent_image }
                 end
+                images.flatten!
+                break if images.empty?
               end
 
-              hierarchy
+              children
             end
 
-            def dapp_project_image_parent_id(image_id)
-              dapp_project_image_inspect(image_id)['Parent']
+            def dapp_project_image_parent(image)
+              dapp_project_image_by_id(dapp_project_image_inspect(image)['Parent'])
             end
           end
         end

data/lib/dapp/dimg/dapp/command/stages/cleanup_repo.rb

@@ -6,17 +6,19 @@ module Dapp
           module CleanupRepo
             def stages_cleanup_repo
               lock_repo(repo = option_repo) do
-                raise Error::Command, code: :stages_cleanup_required_option unless stages_cleanup_option?
+                raise ::Dapp::Error::Command, code: :stages_cleanup_required_option unless stages_cleanup_option?
 
-                registry        = dimg_registry(repo)
-                repo_dimgs      = repo_dimgs_images(registry)
-                repo_dimgstages = repo_dimgstages_images(registry)
+                log_step_with_indent("#{repo} stages") do
+                  registry        = dimg_registry(repo)
+                  repo_dimgs      = repo_dimgs_images(registry)
+                  repo_dimgstages = repo_dimgstages_images(registry)
 
-                repo_dimgstages.delete_if { |dimgstage| repo_dimgs.any? { |dimg| dimgstage[:id] == dimg[:id] } } # ignoring stages with dimgs ids (v2)
+                  repo_dimgstages.delete_if { |dimgstage| repo_dimgs.any? { |dimg| dimgstage[:id] == dimg[:id] } } # ignoring stages with dimgs ids (v2)
 
-                proper_repo_cache(registry, repo_dimgstages)                   if proper_cache_version?
-                repo_dimgstages_cleanup(registry, repo_dimgs, repo_dimgstages) if proper_repo_cache?
-                proper_repo_git_commit(registry)                               if proper_git_commit?
+                  proper_repo_cache(registry, repo_dimgstages)                   if proper_cache_version?
+                  repo_dimgstages_cleanup(registry, repo_dimgs, repo_dimgstages) if proper_repo_cache?
+                  proper_repo_git_commit(registry)                               if proper_git_commit?
+                end
               end
             end
 
@@ -31,24 +33,24 @@ module Dapp
             end
 
             def repo_dimgstages_cleanup(registry, repo_dimgs, repo_dimgstages)
-              log_step_with_indent(option_repo) do
-                repo_dimgs.each { |dimg| except_repo_image_with_parents(registry, repo_dimgstages, dimg) }
+              log_proper_repo_cache do
+                repo_dimgs.each { |dimg| except_repo_image_with_parents(registry, dimg, repo_dimgstages) }
                 repo_dimgstages.each { |dimgstage| delete_repo_image(registry, dimgstage) }
               end
             end
 
-            def except_repo_image_with_parents(registry, repo_dimgstages, repo_image)
-              repo_image_dapp_artifacts_labels(registry, repo_image).each do |aid|
-                repo_image = find_repo_image_by_id(repo_dimgstages, aid)
-                except_repo_image_with_parents(registry, repo_dimgstages, repo_image) unless repo_image.nil?
+            def except_repo_image_with_parents(registry, repo_image, repo_dimgstages)
+              repo_image_dapp_artifacts_labels(registry, repo_image).each do |aiid|
+                unless (repo_artifact_image = repo_image_by_id(aiid, repo_dimgstages)).nil?
+                  except_repo_image_with_parents(registry, repo_artifact_image, repo_dimgstages)
+                end
               end
 
+              ri = repo_image
               loop do
-                repo_dimgstages.delete_if { |dimgstage| dimgstage[:id] == repo_image[:id] }
-                break if begin
-                  (iid = registry.image_parent_id(repo_image[:tag], repo_image[:dimg])).empty? ||
-                    (repo_image = find_repo_image_by_id(repo_dimgstages, iid)).nil?
-                end
+                repo_dimgstages.delete_if { |dimgstage| dimgstage == ri }
+                ri_parent_id = registry.image_parent_id(ri[:tag], ri[:dimg])
+                break if ri_parent_id.empty? || (ri = repo_image_by_id(ri_parent_id, repo_dimgstages)).nil?
               end
             end
 
@@ -60,20 +62,20 @@ module Dapp
               registry.image_labels(repo_image[:tag], repo_image[:dimg])['dapp-cache-version']
             end
 
-            def find_repo_image_by_id(repo_images, repo_image_id)
+            def repo_image_by_id(repo_image_id, repo_images)
               repo_images.find { |repo_image| repo_image[:id] == repo_image_id }
             end
 
             def proper_repo_git_commit(registry)
               log_proper_git_commit do
-                unproper_images = []
+                unproper_dimgstages = []
                 repo_detailed_dimgstage_images(registry).each do |dimgstage|
                   dimgstage[:labels].each do |repo_name, commit|
                     next if (repo = dapp_git_repositories[repo_name]).nil?
-                    unproper_images.concat(repo_image_tags_hierarchy(registry, dimgstage[:id])) unless repo.commit_exists?(commit)
+                    unproper_dimgstages.concat(repo_detailed_image_with_children(registry, dimgstage)) unless repo.commit_exists?(commit)
                   end
                 end
-                remove_repo_dapp_dimgstage_images(registry, unproper_images.uniq)
+                unproper_dimgstages.uniq.each { |dimgstage| delete_repo_image(registry, dimgstage) }
               end
             end
 
@@ -87,27 +89,21 @@ module Dapp
               end
             end
 
-            def repo_image_tags_hierarchy(registry, registry_image_id)
-              hierarchy = []
-              iids = [registry_image_id]
+            def repo_detailed_image_with_children(registry, image)
+              children        = []
+              detailed_images = [image]
 
               loop do
-                hierarchy.concat(iids)
-                break if begin
-                  iids.map! do |iid|
-                    repo_detailed_dimgstage_images(registry).map { |dimgstage| dimgstage[:id] if dimgstage[:parent] == iid }.compact
-                  end.flatten!.empty?
+                children.concat(detailed_images)
+                detailed_images.map! do |repo_image|
+                  repo_detailed_dimgstage_images(registry)
+                    .select { |dimgstage| dimgstage[:parent] == repo_image[:id] }
                 end
+                detailed_images.flatten!
+                break if detailed_images.empty?
               end
 
-              repo_detailed_dimgstage_images(registry).map { |dimgstage| dimgstage[:tag] if hierarchy.include? dimgstage[:id] }.compact
-            end
-
-            def remove_repo_dapp_dimgstage_images(registry, tags)
-              tags.each do |tag|
-                log(tag) if log_verbose? || dry_run?
-                registry.image_delete(tag, nil) unless dry_run?
-              end
+              children
             end
           end
         end

data/lib/dapp/dimg/dapp/command/stages/common.rb

@@ -6,6 +6,14 @@ module Dapp
           module Common
             protected
 
+            def repo_detailed_dimgs_images(registry)
+              repo_dimgs_images(registry).each do |dimg|
+                image_history = registry.image_history(dimg[:tag], dimg[:dimg])
+                dimg[:parent] = image_history['container_config']['Image']
+                dimg[:labels] = image_history['config']['Labels']
+              end
+            end
+
             def repo_dimgs_images(registry)
               [].tap do |dimgs_images|
                 {}.tap do |dimgs_tags|
@@ -53,9 +61,11 @@ module Dapp
                 {}.tap do |repositories|
                   dimgs = build_configs.map { |config| Dimg.new(config: config, dapp: self, ignore_git_fetch: true) }
                   dimgs.each do |dimg|
-                    [dimg, dimg.artifacts].flatten
-                                          .map(&:git_artifacts).flatten
-                                          .map { |git_artifact| repositories[dimgstage_g_a_commit_label(git_artifact.paramshash)] = git_artifact.repo }
+                    [dimg, dimg.artifacts]
+                      .flatten
+                      .map(&:git_artifacts)
+                      .flatten
+                      .map { |git_artifact| repositories[dimgstage_g_a_commit_label(git_artifact.paramshash)] = git_artifact.repo }
                   end
                 end
               end

data/lib/dapp/dimg/dapp/command/stages/flush_local.rb

@@ -6,9 +6,7 @@ module Dapp
           module FlushLocal
             def stages_flush_local
               lock("#{name}.images") do
-                dapp_project_containers_flush
-                dapp_project_dangling_images_flush
-                remove_project_images(dapp_project_images_ids)
+                log_step_with_indent('flush stages') { remove_project_images(dapp_project_dimgstages) }
               end
             end
           end

data/lib/dapp/dimg/dapp/command/stages/flush_repo.rb

@@ -5,9 +5,9 @@ module Dapp
         module Stages
           module FlushRepo
             def stages_flush_repo
-              lock_repo(option_repo) do
-                log_step_with_indent(option_repo) do
-                  registry = dimg_registry(option_repo)
+              lock_repo(repo = option_repo) do
+                log_step_with_indent("#{repo} stages") do
+                  registry = dimg_registry(repo)
                   repo_dimgstages_images(registry).each { |repo_image| delete_repo_image(registry, repo_image) }
                 end
               end

data/lib/dapp/dimg/dapp/command/stages/pull.rb

@@ -5,20 +5,10 @@ module Dapp
         module Stages
           module Pull
             def stages_pull
-              repo = option_repo
-              validate_repo_name!(repo)
-              build_configs.each do |config|
-                log_dimg_name_with_indent(config) do
-                  Dimg.new(config: config, dapp: self, ignore_git_fetch: true).tap do |dimg|
-                    dimg.import_stages!(repo, format: '%{repo}:dimgstage-%{signature}')
-                  end
-                end
+              dimg_import_export_base(should_be_built: false) do |dimg|
+                dimg.import_stages!(option_repo, format: dimgstage_push_format)
               end
             end
-
-            def pull_all_stages?
-              !!options[:pull_all_stages]
-            end
           end
         end
       end

data/lib/dapp/dimg/dapp/command/stages/push.rb

@@ -5,14 +5,8 @@ module Dapp
         module Stages
           module Push
             def stages_push
-              repo = option_repo
-              validate_repo_name!(repo)
-              build_configs.each do |config|
-                log_dimg_name_with_indent(config) do
-                  Dimg.new(config: config, dapp: self, ignore_git_fetch: true, should_be_built: true).tap do |dimg|
-                    dimg.export_stages!(repo, format: '%{repo}:dimgstage-%{signature}')
-                  end
-                end
+              dimg_import_export_base(should_be_built: false) do |dimg|
+                dimg.export_stages!(option_repo, format: dimgstage_push_format)
               end
             end
           end

data/lib/dapp/dimg/dapp/command/tag.rb

@@ -3,12 +3,9 @@ module Dapp
     module Dapp
       module Command
         module Tag
-          def tag(tag)
-            one_dimg!
-            validate_image_name!(tag)
-
-            Dimg.new(config: build_configs.first, dapp: self, ignore_git_fetch: true, should_be_built: true).tap do |app|
-              app.tag!(tag)
+          def tag
+            dimg_import_export_base do |dimg|
+              dimg.tag!(option_repo, format: push_format(dimg.config._name))
             end
           end
         end

data/lib/dapp/dimg/dapp/dapp.rb

@@ -20,6 +20,7 @@ module Dapp
         include Command::Stages::Pull
         include Command::Stages::Common
         include Command::CleanupRepo
+        include Command::FlushLocal
         include Command::FlushRepo
         include Command::Cleanup
         include Command::Mrproper

data/lib/dapp/dimg/dapp/dappfile.rb

@@ -6,6 +6,16 @@ module Dapp
           dimgs_names.first.nil?
         end
 
+        def dimg_name!
+          one_dimg!
+          build_configs.first._name
+        end
+
+        def one_dimg!
+          return if build_configs.one?
+          raise ::Dapp::Error::Command, code: :command_unexpected_dimgs_number, data: { dimgs_names: build_configs.map(&:_name).join('`, `') }
+        end
+
         def dimgs_names
           build_configs.map(&:_name)
         end
@@ -15,7 +25,7 @@ module Dapp
             config._dimg.select do |dimg|
               dimgs_patterns.any? { |pattern| dimg._name.nil? || File.fnmatch(pattern, dimg._name) }
             end.tap do |dimgs|
-              raise ::Dapp::Error::Dapp, code: :no_such_dimg, data: { dimgs_patterns: dimgs_patterns.join(', ') } if dimgs.empty?
+              raise ::Dapp::Error::Dapp, code: :no_such_dimg, data: { dimgs_patterns: dimgs_patterns.join('`, `') } if dimgs.empty?
             end
           end
         end

data/lib/dapp/dimg/dimg.rb

@@ -46,53 +46,60 @@ module Dapp
         artifacts.each { |artifact| artifact.last_stage.save_in_cache! }
       end
 
-      def tag!(tag)
+      def tag!(repo, format:)
+        dimg_export_base!(repo, export_format: format)
+      end
+
+      def export!(repo, format:)
+        dimg_export_base!(repo, export_format: format, push: true)
+      end
+
+      def export_stages!(repo, format:)
         dapp.lock("#{dapp.name}.images", readonly: true) do
-          dimg_name = config._name
-          if dapp.dry_run?
-            dapp.log_state(dimg_name, state: dapp.t(code: 'state.tag'), styles: { status: :success })
-          else
-            dapp.log_process(dimg_name, process: dapp.t(code: 'status.process.tagging')) do
-              last_stage.image.tag!(tag)
+          export_images.each do |stage_image|
+            image_name = format(format, repo: repo, signature: stage_image.name.split(':').last)
+            export_base!(image_name, push: true) do
+              stage_image.export!(image_name)
             end
           end
         end
       end
 
-      def export!(repo, format:)
+      def dimg_export_base!(repo, export_format:, push: false)
         dapp.lock("#{dapp.name}.images", readonly: true) do
           dapp.tags_by_scheme.each do |tag_scheme_name, tags|
-            tags.each do |tag|
-              image_name = format(format, repo: repo, dimg_name: config._name, tag: tag)
-              export_base!(image_name) do
-                last_stage.image.export_dimg!(image_name, sheme_name: tag_scheme_name)
+            dapp.log_step_with_indent(tag_scheme_name) do
+              tags.each do |tag|
+                image_name = format(export_format, repo: repo, dimg_name: config._name, tag: tag)
+                export_base!(image_name, push: push) do
+                  export_image = build_export_image!(image_name, scheme_name: tag_scheme_name)
+                  if push
+                    export_image.export!
+                  else
+                    export_image.tag!
+                  end
+                end
               end
-            end
+            end unless tags.empty?
           end
         end
       end
 
-      def export_stages!(repo, format:)
-        dapp.lock("#{dapp.name}.images", readonly: true) do
-          export_images.each do |image|
-            image_name = format(format, repo: repo, signature: image.name.split(':').last)
-            export_base!(image_name) do
-              image.export!(image_name)
-            end
-          end
+      def build_export_image!(image_name, scheme_name:)
+        Image::Dimg.image_by_name(name: image_name, from: last_stage.image, dapp: dapp).tap do |export_image|
+          export_image.add_service_change_label(:'dapp-tag-scheme' => scheme_name)
+          export_image.add_service_change_label(:'dapp-dimg' => true)
+          export_image.build!
         end
       end
 
-      def export_base!(image_name)
+      def export_base!(image_name, push: true)
         if dapp.dry_run?
-          dapp.log_state(image_name, state: dapp.t(code: 'state.push'), styles: { status: :success })
+          dapp.log_state(image_name, state: dapp.t(code: push ? 'state.push' : 'state.export'), styles: { status: :success })
         else
           dapp.lock("image.#{hashsum image_name}") do
             ::Dapp::Dimg::Image::Docker.reset_image_inspect(image_name)
-
-            dapp.log_process(image_name, process: dapp.t(code: 'status.process.pushing')) do
-              yield
-            end
+            dapp.log_process(image_name, process: dapp.t(code: push ? 'status.process.pushing' : 'status.process.exporting')) { yield }
           end
         end
       end
@@ -107,7 +114,7 @@ module Dapp
               dapp.log_info ::Dapp::Helper::NetStatus.message(e)
               next
             end
-            break unless dapp.pull_all_stages?
+            break unless !!dapp.options[:pull_all_stages]
           end
         end
       end
@@ -117,9 +124,10 @@ module Dapp
           dapp.log_state(image_name, state: dapp.t(code: 'state.pull'), styles: { status: :success })
         else
           dapp.lock("image.#{hashsum image_name}") do
-            dapp.log_process(image_name, process: dapp.t(code: 'status.process.pulling'),
-                                         status: { failed: dapp.t(code: 'status.failed.not_pulled') },
-                                         style: { failed: :secondary }) do
+            dapp.log_process(image_name,
+                             process: dapp.t(code: 'status.process.pulling'),
+                             status: { failed: dapp.t(code: 'status.failed.not_pulled') },
+                             style: { failed: :secondary }) do
               image.import!(image_name)
             end
           end