dapp 0.21.5 → 0.21.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3506c0e4ae752fd7925c65913df56c068a1118471fc8cdcb5b9bdf4006170412
-  data.tar.gz: ffd8f79655ce7025781fbf9d6edca70692351a1ef5f190899b9228ca43d8ff34
+  metadata.gz: 4e7d9a0dc7f56238457f89d9892f39a6f3e420d11fc67f575413248c44e105fa
+  data.tar.gz: a78133ee132bbef279c05cdcaf313dd09decd9e20eeb93325034cfd95979b69f
 SHA512:
-  metadata.gz: d9b92fd573d5d3a9aa7db24df92af823aeea1e4145f395d8ea893bf0b1d0f0dc8f3fb5df2855b8290d5ded79fbf196e7497bbb90c0a79ad5c2795c75a4cd939e
-  data.tar.gz: f52f8bc0ae1d12bf1b524a13df92ea38666a26458b1aa43d631ed961477e31c78dbfa3527578830cfa61e46228b194fd8f9c20deb73a03e042a22904c17135db
+  metadata.gz: 26138b6f38fe585774aa434b42379473d08b08663a9242d894201236ef1a5738f6ef0dfda78f62c61431b7794e54f1a16af7a325b322b7d565bcd3a381d8728d
+  data.tar.gz: bc959e938bee49762ef4711c3af7e6cc4205b1ccf520dd6f214c48670a0f6a01aec4da5572cdb97c6d770071daed7ee7ec2842f34c86fb61caec712afcc45948

data/config/en/net_status.yml

@@ -21,6 +21,7 @@ en:
     command:
       command_unexpected_dimgs_number: "Command can process only one dimg!\nAmbiguous dimg pattern: `%{dimgs_names}`!"
       mrproper_required_option: "Expected command option `--improper-dev-mode-cache`, `--improper-cache-version-stages` or `--all`!"
+      cleanup_repo_required_option: "Expected command option `--improper-nonexistent-git-tag-branch-commit`!"
       stages_cleanup_required_option: "Expected command option `--improper-cache-version`, `--improper-git-commit` or `--improper-repo-cache`!"
       repo_name_incorrect: "Incorrect repo name `%{name}` given!"
       image_name_incorrect: "Incorrect image name `%{name}` given!"

data/lib/dapp.rb

@@ -207,6 +207,8 @@ require 'dapp/dimg/cli/command/dimg/stages/cleanup_repo'
 require 'dapp/dimg/cli/command/dimg/stages/push'
 require 'dapp/dimg/cli/command/dimg/stages/pull'
 require 'dapp/dimg/cli/command/dimg/run'
+require 'dapp/dimg/cli/command/dimg/cleanup_repo'
+require 'dapp/dimg/cli/command/dimg/flush_repo'
 require 'dapp/dimg/cli/command/dimg/cleanup'
 require 'dapp/dimg/cli/command/dimg/bp'
 require 'dapp/dimg/cli/command/dimg/mrproper'
@@ -235,6 +237,8 @@ require 'dapp/dimg/config/config'
 require 'dapp/dimg/dapp/command/common'
 require 'dapp/dimg/dapp/command/build'
 require 'dapp/dimg/dapp/command/bp'
+require 'dapp/dimg/dapp/command/cleanup_repo'
+require 'dapp/dimg/dapp/command/flush_repo'
 require 'dapp/dimg/dapp/command/cleanup'
 require 'dapp/dimg/dapp/command/mrproper'
 require 'dapp/dimg/dapp/command/stage_image'

data/lib/dapp/dapp/option_tags.rb

@@ -5,32 +5,37 @@ module Dapp
         @git_repo ||= ::Dapp::Dimg::GitRepo::Own.new(self)
       end
 
-      def option_tags
-        @tags ||= begin
-          tags = simple_tags + branch_tags + commit_tags + build_tags + ci_tags
-          tags << :latest if tags.empty?
-          tags
+      def tags_by_scheme
+        @tags_by_scheme_name ||= begin
+          [simple_tags, branch_tags, commit_tags, build_tags, ci_tags].reduce({}) do |some_tags_by_scheme, tags_by_scheme|
+            tags_by_scheme.in_depth_merge(some_tags_by_scheme)
+          end.tap do |tags_by_scheme|
+            tags_by_scheme[:custom] = [:latest] if tags_by_scheme.values.flatten.empty?
+          end
         end
       end
 
+      def option_tags
+        tags_by_scheme.values.flatten
+      end
+
       def simple_tags
-        options[:tag]
+        { custom: options[:tag] }
       end
 
       def branch_tags
-        return [] unless options[:tag_branch]
+        return {} unless options[:tag_branch]
         raise Error::Dapp, code: :git_branch_without_name if (branch = git_local_repo.branch) == 'HEAD'
-        [branch]
+        { git_branch: [branch] }
       end
 
       def commit_tags
-        return [] unless options[:tag_commit]
-        commit = git_local_repo.latest_commit
-        [commit]
+        return {} unless options[:tag_commit]
+        { git_commit: [git_local_repo.latest_commit] }
       end
 
       def build_tags
-        return [] unless options[:tag_build_id]
+        return {} unless options[:tag_build_id]
 
         if ENV['GITLAB_CI']
           build_id = ENV['CI_BUILD_ID']
@@ -40,23 +45,25 @@ module Dapp
           raise Error::Dapp, code: :ci_environment_required
         end
 
-        [build_id]
+        { ci: [build_id] }
       end
 
       def ci_tags
-        return [] unless options[:tag_ci]
+        return {} unless options[:tag_ci]
 
-        if ENV['GITLAB_CI']
-          branch = ENV['CI_BUILD_REF_NAME']
-          tag = ENV['CI_BUILD_TAG']
-        elsif ENV['TRAVIS']
-          branch = ENV['TRAVIS_BRANCH']
-          tag = ENV['TRAVIS_TAG']
-        else
-          raise Error::Dapp, code: :ci_environment_required
-        end
+        {}.tap do |tags_by_scheme|
+          if ENV['GITLAB_CI']
+            tags_by_scheme[:git_branch] = [ENV['CI_BUILD_REF_NAME']]
+            tags_by_scheme[:git_tag]    = [ENV['CI_BUILD_TAG']]
+          elsif ENV['TRAVIS']
+            tags_by_scheme[:git_branch] = [ENV['TRAVIS_BRANCH']]
+            tags_by_scheme[:git_tag]    = [ENV['TRAVIS_TAG']]
+          else
+            raise Error::Dapp, code: :ci_environment_required
+          end
 
-        [branch, tag].compact
+          tags_by_scheme.delete_if { |_, tags| tags.first.nil? }
+        end
       end
     end # Tags
   end # Dapp

data/lib/dapp/dimg/build/stage/base.rb

@@ -81,9 +81,6 @@ module Dapp
           def prepare_image
             return if dimg.dapp.dry_run?
 
-            image.add_volumes_from dimg.dapp.toolchain_container
-            image.add_volumes_from dimg.dapp.base_container
-
             image_add_mounts
 
             image.add_service_change_label dapp: dimg.stage_dapp_label

data/lib/dapp/dimg/cli/command/dimg.rb

@@ -1,7 +1,7 @@
 module Dapp::Dimg::CLI
   module Command
     class Dimg < ::Dapp::CLI
-      SUBCOMMANDS = ['build', 'push', 'spush', 'list', 'run', 'stages', 'cleanup', 'bp', 'mrproper', 'stage image', 'tag', 'build-context'].freeze
+      SUBCOMMANDS = ['build', 'push', 'spush', 'list', 'run', 'stages', 'cleanup', 'bp', 'mrproper', 'stage image', 'tag', 'build-context', 'cleanup repo', 'flush repo'].freeze
 
       banner <<BANNER.freeze
 Usage: dapp dimg [options] subcommand [subcommand options]
@@ -15,7 +15,9 @@ Available subcommands: (for details, dapp dimg SUB-COMMAND --help)
   dapp dimg tag [options] [DIMG] TAG
   dapp dimg list [options] [DIMG ...]
   dapp dimg run [options] [DIMG] [DOCKER ARGS]
-  dapp dimg cleanup [options] [DIMG ...]
+  dapp dimg cleanup repo [options] [DIMG ...] REPO
+  dapp dimg flush repo [options] [DIMG ...] REPO
+  dapp dimg cleanup [options]
   dapp dimg mrproper [options]
   dapp dimg stage image [options] [DIMG]
   dapp dimg stages

data/lib/dapp/dimg/cli/command/dimg/cleanup_repo.rb

@@ -0,0 +1,30 @@
+module Dapp::Dimg::CLI
+  module Command
+    class Dimg < ::Dapp::CLI
+      class CleanupRepo < Base
+        banner <<BANNER.freeze
+Usage:
+
+  dapp dimg cleanup repo [options] [DIMG ...] REPO
+
+Options:
+BANNER
+
+        option :lock_timeout,
+               long: '--lock-timeout TIMEOUT',
+               description: 'Redefine resource locking timeout (in seconds)',
+               proc: ->(v) { v.to_i }
+
+        option :with_stages,
+               long: '--with-stages',
+               boolean: true
+
+        def run(argv = ARGV)
+          self.class.parse_options(self, argv)
+          repo = self.class.required_argument(self, 'repo')
+          run_dapp_command(run_method, options: cli_options(dimgs_patterns: cli_arguments, repo: repo))
+        end
+      end
+    end
+  end
+end

data/lib/dapp/dimg/cli/command/dimg/flush_repo.rb

@@ -0,0 +1,30 @@
+module Dapp::Dimg::CLI
+  module Command
+    class Dimg < ::Dapp::CLI
+      class FlushRepo < Base
+        banner <<BANNER.freeze
+Usage:
+
+  dapp dimg flush repo [options] [DIMG ...] REPO
+
+Options:
+BANNER
+
+        option :lock_timeout,
+               long: '--lock-timeout TIMEOUT',
+               description: 'Redefine resource locking timeout (in seconds)',
+               proc: ->(v) { v.to_i }
+
+        option :with_stages,
+               long: '--with-stages',
+               boolean: true
+
+        def run(argv = ARGV)
+          self.class.parse_options(self, argv)
+          repo = self.class.required_argument(self, 'repo')
+          run_dapp_command(run_method, options: cli_options(dimgs_patterns: cli_arguments, repo: repo))
+        end
+      end
+    end
+  end
+end

data/lib/dapp/dimg/dapp/command/cleanup_repo.rb

@@ -0,0 +1,71 @@
+module Dapp
+  module Dimg
+    module Dapp
+      module Command
+        module CleanupRepo
+          def cleanup_repo
+            lock_repo(repo = option_repo) do
+              registry = registry(repo)
+
+              repo_detailed_dimgs_images(registry).select do |image|
+                case image[:labels]['dapp-tag-scheme']
+                  when 'git_tag', 'git_branch', 'git_commit' then true
+                  else false
+                end && !deployed_docker_images.include?([image[:dimg], image[:tag]].join(':'))
+              end.tap do |dimgs_images|
+                cleanup_repo_by_nonexistent_git_tag(registry, dimgs_images)
+                cleanup_repo_by_nonexistent_git_branch(registry, dimgs_images)
+                cleanup_repo_by_nonexistent_git_commit(registry, dimgs_images)
+              end
+
+              begin
+                registry.reset_cache
+                repo_dimgs      = repo_dimgs_images(registry)
+                repo_dimgstages = repo_dimgstages_images(registry)
+                repo_dimgstages_cleanup(registry, repo_dimgs, repo_dimgstages)
+              end if with_stages?
+            end
+          end
+
+          def cleanup_repo_by_nonexistent_git_tag(registry, dimgs_images)
+            cleanup_repo_by_nonexistent_git_base(dimgs_images, 'git_tag') do |dimg_image|
+              delete_repo_image(registry, dimg_image) unless git_local_repo.tags.include?(dimg_image[:tag])
+            end
+          end
+
+          def cleanup_repo_by_nonexistent_git_branch(registry, dimgs_images)
+            cleanup_repo_by_nonexistent_git_base(dimgs_images, 'git_branch') do |dimg_image|
+              delete_repo_image(registry, dimg_image) unless git_local_repo.remote_branches.include?(dimg_image[:tag])
+            end
+          end
+
+          def cleanup_repo_by_nonexistent_git_commit(registry, dimgs_images)
+            cleanup_repo_by_nonexistent_git_base(dimgs_images, 'git_commit') do |dimg_image|
+              delete_repo_image(registry, dimg_image) unless git_local_repo.commit_exists?(dimg_image[:tag])
+            end
+          end
+
+          def cleanup_repo_by_nonexistent_git_base(dimgs_images, dapp_tag_scheme)
+            log_step_with_indent(:"nonexistent #{dapp_tag_scheme.split('_').join(' ')}") do
+              dimgs_images
+                .select { |dimg_image| dimg_image[:labels]['dapp-tag-scheme'] == dapp_tag_scheme }
+                .each { |dimg_image| yield dimg_image }
+            end
+          end
+
+          def repo_detailed_dimgs_images(registry)
+            repo_dimgs_images(registry).each do |dimg|
+              image_history = registry.image_history(dimg[:tag], dimg[:dimg])
+              dimg[:parent] = image_history['container_config']['Image']
+              dimg[:labels] = image_history['config']['Labels']
+            end
+          end
+
+          def deployed_docker_images
+            []
+          end
+        end
+      end
+    end
+  end # Dimg
+end # Dapp

data/lib/dapp/dimg/dapp/command/common.rb

@@ -173,6 +173,10 @@ module Dapp
               options[:repo]
             end
           end
+
+          def with_stages?
+            !!options[:with_stages]
+          end
         end
       end
     end

data/lib/dapp/dimg/dapp/command/flush_repo.rb

@@ -0,0 +1,19 @@
+module Dapp
+  module Dimg
+    module Dapp
+      module Command
+        module FlushRepo
+          def flush_repo
+            lock_repo(repo = option_repo) do
+              log_step_with_indent(option_repo) do
+                registry = registry(repo)
+                repo_dimgs_images(registry).each { |repo_image| delete_repo_image(registry, repo_image) }
+                stages_flush_repo if with_stages?
+              end
+            end
+          end
+        end
+      end
+    end
+  end # Dimg
+end # Dapp

data/lib/dapp/dimg/dapp/command/push.rb

@@ -21,13 +21,6 @@ module Dapp
             #printer = RubyProf::MultiPrinter.new(result)
             #printer.print(path: '/tmp/testdapp.push.profile', profile: 'profile')
           end
-
-
-          protected
-
-          def with_stages?
-            !!options[:with_stages]
-          end
         end
       end
     end

data/lib/dapp/dimg/dapp/command/stages/cleanup_local.rb

@@ -38,7 +38,7 @@ module Dapp
             end
 
             def stages_cleanup_by_repo
-              registry = registry(option_repo)
+              registry   = registry(option_repo)
               repo_dimgs = repo_dimgs_images(registry)
 
               lock("#{name}.images") do

data/lib/dapp/dimg/dapp/command/stages/cleanup_repo.rb

@@ -8,8 +8,8 @@ module Dapp
               lock_repo(repo = option_repo) do
                 raise Error::Command, code: :stages_cleanup_required_option unless stages_cleanup_option?
 
-                registry    = registry(repo)
-                repo_dimgs  = repo_dimgs_images(registry)
+                registry        = registry(repo)
+                repo_dimgs      = repo_dimgs_images(registry)
                 repo_dimgstages = repo_dimgstages_images(registry)
 
                 repo_dimgstages.delete_if { |dimgstage| repo_dimgs.any? { |dimg| dimgstage[:id] == dimg[:id] } } # ignoring stages with dimgs ids (v2)

data/lib/dapp/dimg/dapp/command/stages/common.rb

@@ -12,7 +12,7 @@ module Dapp
             end
 
             def repo_dimgs_images(registry)
-              @repo_dimgs_images ||= [].tap do |dimgs_images|
+              [].tap do |dimgs_images|
                 with_registry_wrapper do
                   {}.tap do |dimgs_tags|
                     dimgs_tags[nil] = registry.nameless_dimg_tags

data/lib/dapp/dimg/dapp/command/stages/flush_repo.rb

@@ -7,11 +7,8 @@ module Dapp
             def stages_flush_repo
               lock_repo(option_repo) do
                 log_step_with_indent(option_repo) do
-                  registry    = registry(option_repo)
-                  repo_dimgs  = repo_dimgs_images(registry)
-                  repo_dimgstages = repo_dimgstages_images(registry)
-
-                  repo_dimgs.concat(repo_dimgstages).each { |repo_image| delete_repo_image(registry, repo_image) }
+                  registry = registry(option_repo)
+                  repo_dimgstages_images(registry).each { |repo_image| delete_repo_image(registry, repo_image) }
                 end
               end
             end

data/lib/dapp/dimg/dapp/dapp.rb

@@ -19,6 +19,8 @@ module Dapp
         include Command::Stages::Push
         include Command::Stages::Pull
         include Command::Stages::Common
+        include Command::CleanupRepo
+        include Command::FlushRepo
         include Command::Cleanup
         include Command::Mrproper
         include Command::StageImage

data/lib/dapp/dimg/dimg.rb

@@ -61,9 +61,13 @@ module Dapp
 
       def export!(repo, format:)
         dapp.lock("#{dapp.name}.images", readonly: true) do
-          dapp.option_tags.each do |tag|
-            image_name = format(format, repo: repo, dimg_name: config._name, tag: tag)
-            export_base!(last_stage.image, image_name)
+          dapp.tags_by_scheme.each do |tag_scheme_name, tags|
+            tags.each do |tag|
+              image_name = format(format, repo: repo, dimg_name: config._name, tag: tag)
+              export_base!(image_name) do
+                last_stage.image.export_dimg!(image_name, sheme_name: tag_scheme_name)
+              end
+            end
           end
         end
       end
@@ -72,12 +76,14 @@ module Dapp
         dapp.lock("#{dapp.name}.images", readonly: true) do
           export_images.each do |image|
             image_name = format(format, repo: repo, signature: image.name.split(':').last)
-            export_base!(image, image_name)
+            export_base!(image_name) do
+              image.export!(image_name)
+            end
           end
         end
       end
 
-      def export_base!(image, image_name)
+      def export_base!(image_name)
         if dapp.dry_run?
           dapp.log_state(image_name, state: dapp.t(code: 'state.push'), styles: { status: :success })
         else
@@ -85,7 +91,7 @@ module Dapp
             ::Dapp::Dimg::Image::Docker.reset_image_inspect(image_name)
 
             dapp.log_process(image_name, process: dapp.t(code: 'status.process.pushing')) do
-              image.export!(image_name)
+              yield
             end
           end
         end

data/lib/dapp/dimg/docker_registry/base/authorization.rb

@@ -4,12 +4,12 @@ module Dapp
       class Base
         module Authorization
           def authorization_options(url)
-            @authorization_options ||= begin
+            (@authorization_options ||= {})[@repo_suffix] ||= begin
               case authenticate_header = raw_request(url).headers['Www-Authenticate']
-              when /Bearer/ then { headers: { Authorization: "Bearer #{authorization_token(authenticate_header)}" } }
-              when /Basic/ then { headers: { Authorization: "Basic #{authorization_auth}" } }
-              when nil then {}
-              else raise Error::Registry, code: :authenticate_type_not_supported, data: { registry: api_url }
+                when /Bearer/ then { headers: { Authorization: "Bearer #{authorization_token(authenticate_header)}" } }
+                when /Basic/ then { headers: { Authorization: "Basic #{authorization_auth}" } }
+                when nil then {}
+                else raise Error::Registry, code: :authenticate_type_not_supported, data: { registry: api_url }
               end
             end
           end

data/lib/dapp/dimg/docker_registry/dimg.rb

@@ -43,6 +43,11 @@ module Dapp
           end
         end
 
+        def reset_cache
+          @tags = nil
+          @image_history = nil
+        end
+
         protected
 
         def with_repo_suffix(extra_repo_suffix)

data/lib/dapp/dimg/git_repo/base.rb

@@ -84,6 +84,17 @@ module Dapp
           raise Error::Rugged, code: :git_repository_reference_error, data: { name: name, message: e.message.downcase }
         end
 
+        def tags
+          git.tags.map { |t| t.name }
+        end
+
+        def remote_branches
+          git.branches
+            .map { |b| b.name }
+            .select { |b| b.start_with?('origin/') }
+            .map { |b| b.reverse.chomp('origin/'.reverse).reverse }
+        end
+
         def commit_at(commit)
           lookup_commit(commit).time.to_i
         end

data/lib/dapp/dimg/image/stage.rb

@@ -22,16 +22,31 @@ module Dapp
         end
 
         def build!
+          add_build_service_volumes
           run!
           @built_id = commit!
         ensure
           dapp.shellout("#{dapp.host_docker} rm #{container_name}")
         end
 
+        def add_build_service_volumes
+          add_volumes_from dapp.toolchain_container
+          add_volumes_from dapp.base_container
+        end
+
         def built?
           !built_id.nil?
         end
 
+        def export_dimg!(name, sheme_name:)
+          self.class.new(name: name, dapp: dapp, from: self).tap do |image|
+            image.add_service_change_label(:'dapp-tag-scheme' => sheme_name)
+            image.add_service_change_label(:'dapp-dimg' => true)
+            image.build!
+            image.export!(name)
+          end
+        end
+
         def export!(name)
           tag!(name).tap do |image|
             image.push!

data/lib/dapp/version.rb

@@ -1,4 +1,4 @@
 module Dapp
-  VERSION = '0.21.5'.freeze
+  VERSION = '0.21.6'.freeze
   BUILD_CACHE_VERSION = 24
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dapp
 version: !ruby/object:Gem::Version
-  version: 0.21.5
+  version: 0.21.6
 platform: ruby
 authors:
 - Dmitry Stolyarov
@@ -534,6 +534,8 @@ files:
 - lib/dapp/dimg/cli/command/dimg/build_context/export.rb
 - lib/dapp/dimg/cli/command/dimg/build_context/import.rb
 - lib/dapp/dimg/cli/command/dimg/cleanup.rb
+- lib/dapp/dimg/cli/command/dimg/cleanup_repo.rb
+- lib/dapp/dimg/cli/command/dimg/flush_repo.rb
 - lib/dapp/dimg/cli/command/dimg/list.rb
 - lib/dapp/dimg/cli/command/dimg/mrproper.rb
 - lib/dapp/dimg/cli/command/dimg/push.rb
@@ -575,7 +577,9 @@ files:
 - lib/dapp/dimg/dapp/command/build_context/export.rb
 - lib/dapp/dimg/dapp/command/build_context/import.rb
 - lib/dapp/dimg/dapp/command/cleanup.rb
+- lib/dapp/dimg/dapp/command/cleanup_repo.rb
 - lib/dapp/dimg/dapp/command/common.rb
+- lib/dapp/dimg/dapp/command/flush_repo.rb
 - lib/dapp/dimg/dapp/command/list.rb
 - lib/dapp/dimg/dapp/command/mrproper.rb
 - lib/dapp/dimg/dapp/command/push.rb