dapp 0.7.16 → 0.7.17

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f5884787809c47ff8c6ced799f109912144814b0
-  data.tar.gz: 5b94502ee922f44d9adbabc87e74b3e29db2e80a
+  metadata.gz: ac3cccdbfe9bb2fb9aab05958f1fa1d8f2db5ab5
+  data.tar.gz: c8518534ebe63840f47ff76327186f6b78d542bc
 SHA512:
-  metadata.gz: 35020f1bd1c480ba266b0e500a8e41944de22ec889b4ae7c4e8617b0f9b81348be06ceb9962e526064e02464aac999199bb63b7e1b04745627380494d6931272
-  data.tar.gz: 9c1bb01f8c90ec1d00a3131ae65af7a82fce6bbf7863d6a2d5e2326eee5a55df6322b579be0ceb3771023f3e9e9c0962b849d28ececec46c6004874087671370
+  metadata.gz: 043df0615ad8084c2deca2fec4bb98f932a60a3ce7969f2ad84e4a14fef2643ed6e5889a8387c67701b287ef945d0f7198b1143be1fb3e24b78aaa9e5ae7bc6b
+  data.tar.gz: 12099bf042bb1282be953c32f12bce147fe2a6103a90a82eb88dcf3e296bb1af4d782fde0aa12988d6caad8dc2b881dad8a222f703f82ce4ba7caf7f4b8a28ef

data/config/en/net_status.yml

@@ -26,6 +26,7 @@ en:
       cannot_run_ssh_agent: "Cannot run ssh-agent"
       ssh_key_not_found: "Ssh key `%{path}` not exist!"
       mrproper_required_option: "Expected command option `--improper-dev-mode-cache`, `--improper-cache-version-stages` or `--all`!"
+      stages_cleanup_required_option: "Expected command option `--improper-cache-version`, `--improper-git-commit` or `--improper-repo-cache`!"
     config:
       dimg_name_required: 'Dimg name required!'
       dimg_name_incorrect: "Dimg has incorrect name `%{name}`: doesn't match regex `%{reg}`!"
@@ -42,6 +43,7 @@ en:
       stage_artifact_not_associated: "Artifact not associated with any stage: expected `before` or `after` attribute!"
       stage_artifact_double_associate: "Cannot use `%{stage}` stage for artifact, already used in `%{conflict_stage}` stage!"
       stage_artifact_not_supported_associated_stage: "Bad artifact stage `%{stage}`!"
+      git_artifact_remote_unsupported_protocol: "Remote git repo `%{url}`: unsupported protocol!"
       artifact_conflict: "Conflict between artifacts paths!"
       scratch_unsupported_directive: "Scratch dimg has unsupported directive `%{directive}`!"
       scratch_artifact_required: "Scratch dimg without artifacts!"
@@ -54,13 +56,13 @@ en:
       cookbook_berksfile_not_found: "Dapp cookbook Berksfile not found at %{path}"
       cookbook_metadata_not_found: "Dapp cookbook metadata.rb file not found at %{path}"
       cookbook_not_specified_in_berksfile: "Dapp cookbook `%{name}` not specified in Berksfile at %{path}"
-    registry:
-      incorrect_repo: 'Incorrect repository!'
-      no_such_dimg: 'No such dimg in registry!'
-      authenticate_type_not_supported: 'Registry authenticate type not supported!'
-      page_not_found: "Page `%{url}` not found!"
-      user_not_authorized: 'User not authorized!'
-      response_with_error_status: 'Response with error status!'
       berksfile_absolute_path_forbidden: "Absolute paths in Berksfile are not allowed (cookbook `%{cookbook}`, path: `%{path}`)"
+    registry:
+      no_such_dimg: 'Registry `%{registry}`: no such dimg in registry!'
+      authenticate_type_not_supported: 'Registry `%{registry}`: authenticate type not supported!'
+      page_not_found: "Registry `%{registry}`: page `%{url}` not found!"
+      user_not_authorized: 'Registry `%{registry}`: user not authorized!'
+    rugged:
+      rugged_network_error: "Remote git repo `%{url}`: `%{message}`!"
     lock:
       timeout: "Could not obtain lock for `%{name}` within %{timeout} seconds"

data/lib/dapp.rb

@@ -9,11 +9,13 @@ require 'mixlib/shellout'
 require 'securerandom'
 require 'excon'
 require 'json'
+require 'uri'
 require 'ostruct'
 require 'time'
 require 'i18n'
 require 'paint'
 require 'inifile'
+require 'rugged'
 
 require 'net_status'
 
@@ -32,6 +34,7 @@ require 'dapp/error/config'
 require 'dapp/error/project'
 require 'dapp/error/shellout'
 require 'dapp/error/registry'
+require 'dapp/error/rugged'
 require 'dapp/exception/base'
 require 'dapp/exception/introspect_image'
 require 'dapp/exception/registry'

data/lib/dapp/build/stage/artifact_base.rb

@@ -18,8 +18,19 @@ module Dapp
           image.add_service_change_label artifacts_labels
         end
 
-        def images
-          [image].concat(artifacts.map { |artifact| artifact[:dimg].images }.flatten)
+        def artifacts
+          @artifacts ||= begin
+            dimg.config.public_send("_#{name}").map do |artifact|
+              artifact_dimg = Dapp::Artifact.new(config: artifact._config,
+                                                 project: dimg.project,
+                                                 ignore_git_fetch: dimg.ignore_git_fetch)
+              { name: artifact._config._name, options: artifact._artifact_options, dimg: artifact_dimg }
+            end
+          end
+        end
+
+        def artifact?
+          true
         end
 
         protected
@@ -32,17 +43,6 @@ module Dapp
           true
         end
 
-        def artifacts
-          @artifacts ||= begin
-            dimg.config.public_send("_#{name}").map do |artifact|
-              artifact_dimg = Dapp::Artifact.new(config: artifact._config,
-                                                 project: dimg.project,
-                                                 ignore_git_fetch: dimg.ignore_git_fetch)
-              { name: artifact._config._name, options: artifact._artifact_options, dimg: artifact_dimg }
-            end
-          end
-        end
-
         def artifacts_signatures
           artifacts.map { |artifact| hashsum [artifact[:dimg].signature, artifact[:options]] }
         end

data/lib/dapp/build/stage/base.rb

@@ -142,6 +142,10 @@ module Dapp
           []
         end
 
+        def artifact?
+          false
+        end
+
         protected
 
         def image_build

data/lib/dapp/cli/stages.rb

@@ -9,7 +9,7 @@ Version: #{Dapp::VERSION}
 
 Available subcommands: (for details, dapp SUB-COMMAND --help)
 
-dapp stages cleanup local [options] [DIMG ...] REPO
+dapp stages cleanup local [options] [DIMG ...] [REPO]
 dapp stages cleanup repo [options] [DIMG ...] REPO
 dapp stages flush local [options] [DIMG ...]
 dapp stages flush repo [options] [DIMG ...] REPO

data/lib/dapp/cli/stages/cleanup_local.rb

@@ -7,7 +7,7 @@ module Dapp
 Version: #{Dapp::VERSION}
 
 Usage:
-  dapp stages cleanup local [options] [DIMG ...] REPO
+  dapp stages cleanup local [options] [DIMG ...] [REPO]
 
     DIMG                        Dapp image to process [default: *].
 
@@ -17,10 +17,26 @@ BANNER
                long: '--improper-cache-version',
                boolean: true
 
+        option :proper_git_commit,
+               long: '--improper-git-commit',
+               boolean: true
+
+        option :proper_repo_cache,
+               long: '--improper-repo-cache',
+               boolean: true
+
         def run(argv = ARGV)
           self.class.parse_options(self, argv)
-          repo = self.class.required_argument(self)
-          Project.new(cli_options: config, dimgs_patterns: cli_arguments).stages_cleanup_local(repo)
+          repository = repo
+          Project.new(cli_options: config, dimgs_patterns: cli_arguments).send(run_method, repository)
+        end
+
+        def repo
+          config[:proper_repo_cache] ? self.class.required_argument(self) : nil
+        end
+
+        def run_method
+          :stages_cleanup_local
         end
       end
     end

data/lib/dapp/cli/stages/cleanup_repo.rb

@@ -13,10 +13,12 @@ Usage:
 
 Options:
 BANNER
-        def run(argv = ARGV)
-          self.class.parse_options(self, argv)
-          repo = self.class.required_argument(self)
-          Project.new(cli_options: config, dimgs_patterns: cli_arguments).stages_cleanup_repo(repo)
+        def run_method
+          :stages_cleanup_repo
+        end
+
+        def repo
+          self.class.required_argument(self)
         end
       end
     end

data/lib/dapp/config/dimg/instance_methods.rb

@@ -43,7 +43,7 @@ module Dapp
 
         def git(url = nil, &blk)
           type = url.nil? ? :local : :remote
-          _git_artifact.send(type, url, &blk)
+          _git_artifact.send(type, url.to_s, &blk)
         end
 
         def mount(to, &blk)

data/lib/dapp/config/dimg/validation.rb

@@ -31,12 +31,13 @@ module Dapp
         end
 
         def validate_scratch_directives!
-          directives = [:_shell, :_chef, :_git_artifact, :_install_dependencies, :_setup_dependencies,
-                        :_tmp_dir_mount, :_build_dir_mount]
-          directives.each do |directive|
+          directives = [[:_shell, :shell], [:_chef, :chef], [:_git_artifact, :git],
+                        [:_install_dependencies, :install_depends_on], [:_setup_dependencies, :setup_depends_on],
+                        [:_tmp_dir_mount, :mount], [:_build_dir_mount, :mount]]
+          directives.each do |name, user_name|
             raise Error::Config,
                   code: :scratch_unsupported_directive,
-                  data: { directive: directive[1..-1] } unless public_send(directive).send(:empty?)
+                  data: { directive: user_name } unless public_send(name).send(:empty?)
           end
 
           docker_directives = [:_expose, :_env, :_cmd, :_onbuild, :_workdir, :_user, :_entrypoint]

data/lib/dapp/config/directive/git_artifact_remote.rb

@@ -6,6 +6,8 @@ module Dapp
         attr_reader :_url, :_name, :_branch, :_commit
 
         def initialize(url, **kwargs, &blk)
+          raise Error::Config, code: :git_artifact_remote_unsupported_protocol, data: { url: url } unless %w(http https).include? URI(url.to_s).scheme
+
           @_url  = url
           @_name = url.gsub(%r{.*?([^\/ ]+)\.git}, '\\1')
 

data/lib/dapp/dimg.rb

@@ -37,7 +37,7 @@ module Dapp
         end
       end
     ensure
-      FileUtils.rm_rf(tmp_path)
+      cleanup_tmp
     end
 
     def tag!(tag)
@@ -132,6 +132,10 @@ module Dapp
       @builder ||= Builder.const_get(config._builder.capitalize).new(self)
     end
 
+    def artifacts
+      @artifacts ||= artifacts_stages.map { |stage| stage.artifacts.map { |artifact| artifact[:dimg] } }.flatten
+    end
+
     def artifact?
       false
     end
@@ -153,6 +157,11 @@ module Dapp
       system(cmd)
     end
 
+    def cleanup_tmp
+      FileUtils.rm_rf(tmp_path)
+      artifacts.each(&:cleanup_tmp)
+    end
+
     protected
 
     def should_be_built?

data/lib/dapp/dimg/stages.rb

@@ -15,16 +15,8 @@ module Dapp
         project.stage_dapp_label
       end
 
-      def images
-        (@images ||= []).tap do |images|
-          stages.each do |stage|
-            if stage.respond_to?(:images)
-              images.concat(stage.images)
-            else
-              images << stage.image
-            end
-          end
-        end.uniq!(&:name)
+      def all_images
+        @all_images ||= all_stages.map(&:image).uniq!(&:name)
       end
 
       protected
@@ -38,11 +30,11 @@ module Dapp
       end
 
       def export_images
-        images.select(&:tagged?)
+        all_images.select(&:tagged?)
       end
 
       def import_images
-        images.select { |image| !image.tagged? }
+        all_images.select { |image| !image.tagged? }
       end
 
       def stages
@@ -54,6 +46,14 @@ module Dapp
           end
         end
       end
+
+      def artifacts_stages
+        @artifacts_stages ||= stages.select { |stage| stage.artifact? }
+      end
+
+      def all_stages
+        stages + artifacts.map(&:all_stages).flatten
+      end
     end # Stages
   end # Dimg
 end # Dapp

data/lib/dapp/docker_registry/base.rb

@@ -25,7 +25,7 @@ module Dapp
       def tags
         @tags ||= api_request(repo_suffix, 'tags/list')['tags'] || []
       rescue Error::Registry => e
-        raise Exception::Registry, code: :dimg_not_found_in_registry if e.net_status[:code] == :page_not_found
+        raise Exception::Registry, code: :no_such_dimg, data: { registry: api_url } if e.net_status[:code] == :page_not_found
         raise
       end
 
@@ -50,6 +50,11 @@ module Dapp
                     headers: { Accept: 'Accept: application/vnd.docker.distribution.manifest.v2+json' })
       end
 
+      def image_history(tag)
+        response = manifest_v1(tag)
+        JSON.load(response['history'].first['v1Compatibility'])
+      end
+
       protected
 
       def image_digest(tag)
@@ -70,11 +75,6 @@ module Dapp
                     headers: { Accept: 'application/vnd.docker.distribution.manifest.v2+json' })
       end
 
-      def image_history(tag)
-        response = manifest_v1(tag)
-        JSON.load(response['history'].first['v1Compatibility'])
-      end
-
       def blob_delete(id)
         api_request(repo_suffix, "/blobs/#{id}",
                     method: :delete, expects: [202, 404])
@@ -88,9 +88,9 @@ module Dapp
         url = api_url(*uri)
         request(url, **default_api_options.merge(options))
       rescue Excon::Error::NotFound
-        raise Error::Registry, code: :page_not_found, data: { url: url }
+        raise Error::Registry, code: :page_not_found, data: { url: url, registry: api_url }
       rescue Excon::Error::Unauthorized
-        raise Error::Registry, code: :user_not_authorized
+        user_not_authorized!
       end
 
       def api_url(*uri)
@@ -100,6 +100,10 @@ module Dapp
       def default_api_options
         { expects: [200] }
       end
+
+      def user_not_authorized!
+        raise Error::Registry, code: :user_not_authorized, data: { registry: api_url }
+      end
     end
   end # DockerRegistry
 end # Dapp

data/lib/dapp/docker_registry/mod/authorization.rb

@@ -10,7 +10,7 @@ module Dapp
             when /Bearer/ then { headers: { Authorization: "Bearer #{authorization_token(authenticate_header)}" } }
             when /Basic/ then { headers: { Authorization: "Basic #{authorization_auth}" } }
             when nil then {}
-            else raise Error::Registry, code: :authenticate_type_not_supported
+            else raise Error::Registry, code: :authenticate_type_not_supported, data: { registry: api_url }
             end
           end
         end
@@ -42,14 +42,14 @@ module Dapp
             r = chomp_name(r)
           end
           credential = (auths[r] || auths.find { |repo, _| repo == r })
-          raise Error::Registry, code: :user_not_authorized if credential.nil?
+          user_not_authorized! if credential.nil?
           credential['auth']
         end
 
         def auths_section_from_docker_config
           file = Pathname(File.join(Dir.home, '.docker', 'config.json'))
-          raise Error::Registry, code: :user_not_authorized unless file.exist?
-          JSON.load(file.read)['auths'].tap { |auths| raise Error::Registry, code: :user_not_authorized if auths.nil? }
+          user_not_authorized! unless file.exist?
+          JSON.load(file.read)['auths'].tap { |auths| user_not_authorized! if auths.nil? }
         end
 
         private

data/lib/dapp/error/rugged.rb

@@ -0,0 +1,6 @@
+module Dapp
+  module Error
+    # Rugged
+    class Rugged < Base; end
+  end
+end

data/lib/dapp/git_artifact.rb

@@ -55,11 +55,11 @@ module Dapp
     end
 
     def any_changes?(from, to = latest_commit)
-      !repo.git_bare(diff_command(from, to, quiet: true), returns: [0, 1]).status.success?
+      !repo.old_git_bare(diff_command(from, to, quiet: true), returns: [0, 1]).status.success?
     end
 
     def patch_size(from, to)
-      repo.git_bare("#{diff_command(from, to)} | wc -c").stdout.strip.to_i
+      repo.old_git_bare("#{diff_command(from, to)} | wc -c").stdout.strip.to_i
     end
 
     def latest_commit

data/lib/dapp/git_repo/base.rb

@@ -15,35 +15,40 @@ module Dapp
       end
 
       def path
-        dimg.tmp_path "#{name}.git"
+        dimg.tmp_path("#{name}.git").to_s
       end
 
-      def git_bare(command, **kwargs)
-        git "--git-dir=#{path} #{command}", **kwargs
+      def git_bare
+        @git_bare ||= Rugged::Repository.new(path, bare: true)
       end
 
-      def commit_at(commit)
-        Time.at Integer git_bare("show -s --format=%ct #{commit}").stdout.strip
+      def old_git_bare(command, **kwargs)
+        old_git "--git-dir=#{path} #{command}", **kwargs
       end
 
-      def latest_commit(branch)
-        git_bare("rev-parse #{branch}").stdout.strip
+      def commit_at(commit)
+        git_bare.lookup(commit).time.to_i
       end
 
-      def exist_in_commit?(path, commit)
-        git_bare("cat-file -e #{commit}:#{path}", returns: [0, 128]).status.success?
+      def latest_commit(branch)
+        return git_bare.head.target_id if branch == 'HEAD'
+        git_bare.branches[branch].target_id
       end
 
       def cleanup!
       end
 
       def branch
-        git_bare('rev-parse --abbrev-ref HEAD').stdout.strip
+        git_bare.head.name.sub(/^refs\/heads\//, '')
       end
 
       protected
 
-      def git(command, **kwargs)
+      def git
+        @git ||= Rugged::Repository.new(path)
+      end
+
+      def old_git(command, **kwargs)
         dimg.system_shellout! "#{dimg.project.git_bin} #{command}", **kwargs
       end
     end

data/lib/dapp/git_repo/own.rb

@@ -11,14 +11,7 @@ module Dapp
       end
 
       def path
-        @path ||= begin
-          git_repo_path = Pathname(git("-C #{dimg.home_path} rev-parse --git-dir").stdout.strip)
-          if git_repo_path.relative?
-            File.join(dimg.home_path, git_repo_path)
-          else
-            git_repo_path
-          end
-        end
+        @path ||= Rugged::Repository.discover(dimg.home_path.to_s).path
       end
 
       def latest_commit(branch = nil)

data/lib/dapp/git_repo/remote.rb

@@ -8,13 +8,18 @@ module Dapp
         @url = url
 
         dimg.project.log_secondary_process(dimg.project.t(code: 'process.git_artifact_clone', data: { name: name }), short: true) do
-          git "clone --bare --depth 1 #{url} #{path}"
+          begin
+            Rugged::Repository.clone_at(url, path, bare: true)
+          rescue Rugged::NetworkError => e
+            raise Error::Rugged, code: :rugged_network_error, data: { message: e.message, url: url }
+          end
         end unless File.directory?(path)
       end
 
-      def fetch!(branch = 'master')
+      def fetch!(branch = nil)
+        branch ||= self.branch
         dimg.project.log_secondary_process(dimg.project.t(code: 'process.git_artifact_fetch', data: { name: name }), short: true) do
-          git_bare "fetch origin #{branch}:#{branch}"
+          git_bare.fetch('origin', [branch])
         end unless dimg.ignore_git_fetch || dimg.project.dry_run?
       end
 

data/lib/dapp/project/command/bp.rb

@@ -8,12 +8,12 @@ module Dapp
         def bp(repo)
           bp_step(:build)
           bp_step(:push, repo)
-          bp_step(:stages_cleanup_local, repo)
+          bp_step(:stages_cleanup_by_repo, repo)
           bp_step(:cleanup)
         end
 
         def bp_step(step, *args)
-          log_step_with_indent(step) { public_send(step, *args) }
+          log_step_with_indent(step) { send(step, *args) }
         end
       end
     end

data/lib/dapp/project/command/cleanup.rb

@@ -12,7 +12,7 @@ module Dapp
               project_dangling_images_flush
               remove_images_by_query([
                 'docker images',
-                %(--format '{{if ne "#{stage_cache}" .Repository }}{{.ID}}{{ end }}'),
+                %(--format '{{if ne "#{stage_cache}" .Repository }}{{.Repository}}:{{.Tag}}{{ end }}'),
                 %(-f "label=dapp=#{stage_dapp_label}")
               ].join(' ')) # FIXME: negative filter is not currently supported by the Docker CLI
             end

data/lib/dapp/project/command/common.rb

@@ -7,8 +7,12 @@ module Dapp
       module Common
         protected
 
-        def project_images
-          shellout!(%(docker images --format="{{.Repository}}:{{.Tag}}" #{stage_cache})).stdout.strip
+        def project_images_names
+          shellout!(%(docker images --format="{{.Repository}}:{{.Tag}}" #{stage_cache})).stdout.lines.map(&:strip)
+        end
+
+        def project_images_ids
+          shellout!(%(docker images #{stage_cache} -q --no-trunc)).stdout.lines.map(&:strip)
         end
 
         def project_containers_flush
@@ -24,7 +28,7 @@ module Dapp
         end
 
         def remove_images(ids, force: false)
-          remove_base('docker rmi%{force_option} %{ids}', ids, force: force)
+          remove_base('docker rmi%{force_option} %{ids}', ids.uniq, force: force)
         end
 
         def remove_containers_by_query(containers_query, force: false)
@@ -32,7 +36,7 @@ module Dapp
         end
 
         def remove_containers(ids, force: false)
-          remove_base('docker rm%{force_option} %{ids}', ids, force: force)
+          remove_base('docker rm%{force_option} %{ids}', ids.uniq, force: force)
         end
 
         def remove_base(query_format, ids, force: false)
@@ -46,6 +50,10 @@ module Dapp
           yield(res)
         end
 
+        def image_labels(image_id)
+          Image::Stage.image_config_option(image_id: image_id, option: 'labels')
+        end
+
         def run_command(cmd)
           log(cmd) if log_verbose? || dry_run?
           shellout!(cmd) unless dry_run?

data/lib/dapp/project/command/mrproper.rb

@@ -14,7 +14,7 @@ module Dapp
             elsif proper_cache_version?
               log_proper_cache do
                 proper_cache_images = proper_cache_all_images
-                remove_images(dapp_images_by_label('dapp').lines.select { |id| !proper_cache_images.lines.include?(id) }.map(&:strip))
+                remove_images(dapp_images_by_label('dapp').select { |id| !proper_cache_images.include?(id) }.map(&:strip))
               end
             else
               raise Error::Project, code: :mrproper_required_option
@@ -26,7 +26,7 @@ module Dapp
 
         def flush_by_label(label)
           log_step_with_indent(:containers) { dapp_containers_flush_by_label(label) }
-          log_step_with_indent(:images) { dapp_images_flush_be_label(label) }
+          log_step_with_indent(:images) { dapp_images_flush_by_label(label) }
         end
 
         def proper_all?
@@ -45,13 +45,13 @@ module Dapp
           remove_images_by_query(%(docker images -f "dangling=true" -f "label=#{label}" -q), force: true)
         end
 
-        def dapp_images_flush_be_label(label)
+        def dapp_images_flush_by_label(label)
           dapp_dangling_images_flush_by_label(label)
-          remove_images(dapp_images_by_label(label).lines.map(&:strip), force: true)
+          remove_images(dapp_images_by_label(label), force: true)
         end
 
         def dapp_images_by_label(label)
-          @dapp_images ||= shellout!(%(docker images -f "dangling=false" --format="{{.Repository}}:{{.Tag}}" -f "label=#{label}")).stdout.strip
+          @dapp_images ||= shellout!(%(docker images -f "dangling=false" --format="{{.Repository}}:{{.Tag}}" -f "label=#{label}")).stdout.lines.map(&:strip)
         end
 
         def proper_cache_all_images
@@ -59,7 +59,7 @@ module Dapp
             'docker images',
             '--format="{{.Repository}}:{{.Tag}}"',
             %(-f "label=dapp-cache-version=#{Dapp::BUILD_CACHE_VERSION}" -f "dangling=false")
-          ].join(' ')).stdout.strip
+          ].join(' ')).stdout.lines.map(&:strip)
         end
       end
     end

data/lib/dapp/project/command/stages/cleanup_local.rb

@@ -8,43 +8,46 @@ module Dapp
         module CleanupLocal
           def stages_cleanup_local(repo)
             lock_repo(repo, readonly: true) do
-              registry = registry(repo)
-              repo_dimgs = repo_dimgs_images(registry)
-              proper_cache if proper_cache_version?
-              cleanup_project(repo_dimgs)
+              raise Error::Project, code: :stages_cleanup_required_option unless stages_cleanup_option?
+
+              project_containers_flush
+
+              proper_cache                 if proper_cache_version?
+              stages_cleanup_by_repo(repo) if proper_repo_cache?
+              proper_git_commit            if proper_git_commit?
             end
           end
 
           protected
 
-          def cleanup_project(repo_dimgs)
+          def proper_cache
+            log_proper_cache do
+              lock("#{name}.images") do
+                log_step_with_indent(name) do
+                  remove_images(project_images_names.select { |image_name| !actual_cache_images.include?(image_name) })
+                end
+              end
+            end
+          end
+
+          def stages_cleanup_by_repo(repo)
+            registry = registry(repo)
+            repo_dimgs = repo_dimgs_images(registry)
+
             lock("#{name}.images") do
               log_step_with_indent(name) do
-                project_containers_flush
                 project_dangling_images_flush
                 dimgs, stages = project_images_hash.partition { |_, image_id| repo_dimgs.values.include?(image_id) }
                 dimgs = dimgs.to_h
                 stages = stages.to_h
-                dimgs.each { |_, aiid| clear_stages(aiid, stages) }
+                dimgs.each { |_, aiid| except_image_with_parents(aiid, stages) }
                 remove_images(stages.keys)
               end
             end
           end
 
           def repo_dimgs_images(registry)
-            repo_images(registry).first
-          end
-
-          def proper_cache
-            log_proper_cache do
-              lock("#{name}.images") do
-                log_step_with_indent(name) do
-                  project_containers_flush
-                  actual_cache_images = actual_cache_images
-                  remove_images(project_images.lines.select { |id| !actual_cache_images.lines.include?(id) }.map(&:strip))
-                end
-              end
-            end
+            repo_dimgs_and_cache(registry).first
           end
 
           def actual_cache_images
@@ -53,7 +56,7 @@ module Dapp
               '--format="{{.Repository}}:{{.Tag}}"',
               %(-f "label=dapp-cache-version=#{Dapp::BUILD_CACHE_VERSION}"),
               stage_cache
-            ].join(' ')).stdout.strip
+            ].join(' ')).stdout.lines.map(&:strip)
           end
 
           def project_images_hash
@@ -62,9 +65,9 @@ module Dapp
             end.to_h
           end
 
-          def clear_stages(image_id, stages)
+          def except_image_with_parents(image_id, stages)
             if image_exist?(image_id)
-              image_dapp_artifacts_label(image_id).each { |aiid| clear_stages(aiid, stages) }
+              image_dapp_artifacts_label(image_id).each { |aiid| except_image_with_parents(aiid, stages) }
               iid = image_id
               loop do
                 stages.delete_if { |_, siid| siid == iid }
@@ -89,6 +92,46 @@ module Dapp
           def image_parent(image_id)
             shellout!(%(docker inspect -f {{.Parent}} #{image_id})).stdout.strip
           end
+
+          def proper_git_commit
+            log_proper_git_commit do
+              unproper_images = []
+              project_images_detailed.each do |_, attrs|
+                attrs['Config']['Labels'].each do |repo_name, commit|
+                  next if (repo = project_git_repositories[repo_name]).nil?
+                  git = repo.name == 'own' ? :git : :git_bare
+                  unproper_images.concat(image_hierarchy(attrs['Id'])) unless repo.send(git).exists?(commit)
+                end
+              end
+              remove_images(unproper_images.uniq)
+            end
+          end
+
+          def project_images_detailed
+            @project_images_detailed ||= {}.tap do |images|
+              project_images_names.each do |image_name|
+                shellout!(%(docker inspect --format='{{json .}}' #{image_name})).stdout.strip.tap do |output|
+                  images[image_name] = output == 'null' ? {} : JSON.parse(output)
+                end
+              end
+            end
+          end
+
+          def image_hierarchy(image_id)
+            hierarchy = []
+            iids = [image_id]
+
+            loop do
+              hierarchy.concat(iids)
+              break if begin
+                iids.map! do |iid|
+                  project_images_detailed.map { |_, attrs| attrs['Id'] if attrs['Parent'] == iid }.compact
+                end.flatten!.empty?
+              end
+            end
+
+            hierarchy
+          end
         end
       end
     end

data/lib/dapp/project/command/stages/cleanup_repo.rb

@@ -8,19 +8,37 @@ module Dapp
         module CleanupRepo
           def stages_cleanup_repo(repo)
             lock_repo(repo) do
+              raise Error::Project, code: :stages_cleanup_required_option unless stages_cleanup_option?
+
               registry = registry(repo)
-              repo_dimgs, repo_stages = repo_images(registry)
+              repo_dimgs, repo_stages = repo_dimgs_and_cache(registry)
               repo_stages.delete_if { |_, siid| repo_dimgs.values.include?(siid) } # ignoring stages with dimgs ids (v2)
-              proper_repo_cache(registry, repo_stages) if proper_cache_version?
-              log_step_with_indent(repo) do
-                repo_dimgs.each { |image_tag, image_id| clear_repo_stages(registry, repo_stages, image_tag, image_id) }
-                repo_stages.keys.each { |image_tag| delete_repo_image(registry, image_tag) }
-              end
+
+              proper_repo_cache(registry, repo_stages)               if proper_cache_version?
+              repo_stages_cleanup(registry, repo_dimgs, repo_stages) if proper_repo_cache?
+              proper_repo_git_commit(registry)                       if proper_git_commit?
             end
           end
 
           protected
 
+          def proper_repo_cache(registry, repo_stages)
+            log_proper_cache do
+              wrong_cache_images = repo_stages.select do |image_tag, _|
+                repo_image_dapp_cache_version_label(registry, image_tag) != Dapp::BUILD_CACHE_VERSION.to_s
+              end
+              wrong_cache_images.each { |image_tag, _| delete_repo_image(registry, image_tag) }
+              repo_stages.delete_if { |image_tag, _| wrong_cache_images.keys.include?(image_tag) }
+            end
+          end
+
+          def repo_stages_cleanup(registry, repo_dimgs, repo_stages)
+            log_step_with_indent(repo) do
+              repo_dimgs.each { |image_tag, image_id| clear_repo_stages(registry, repo_stages, image_tag, image_id) }
+              repo_stages.keys.each { |image_tag| delete_repo_image(registry, image_tag) }
+            end
+          end
+
           def clear_repo_stages(registry, repo_stages, image_tag, image_id)
             repo_image_dapp_artifacts_labels(registry, image_tag).each do |iid|
               itag = find_image_tag_by_id(repo_stages, iid)
@@ -49,13 +67,54 @@ module Dapp
             nil
           end
 
-          def proper_repo_cache(registry, repo_stages)
-            log_proper_cache do
-              wrong_cache_images = repo_stages.select do |image_tag, _|
-                repo_image_dapp_cache_version_label(registry, image_tag) != Dapp::BUILD_CACHE_VERSION.to_s
+          def proper_repo_git_commit(registry)
+            log_proper_git_commit do
+              unproper_images = []
+              repo_project_images_detailed(registry).each do |_, attrs|
+                attrs[:labels].each do |repo_name, commit|
+                  next if (repo = project_git_repositories[repo_name]).nil?
+                  git = repo.name == 'own' ? :git : :git_bare
+                  unproper_images.concat(repo_image_tags_hierarchy(registry, attrs[:id])) unless repo.send(git).exists?(commit)
+                end
               end
-              wrong_cache_images.each { |image_tag, _| delete_repo_image(registry, image_tag) }
-              repo_stages.delete_if { |image_tag, _| wrong_cache_images.keys.include?(image_tag) }
+              remove_repo_images(registry, unproper_images.uniq)
+            end
+          end
+
+          def repo_project_images_detailed(registry)
+            @repo_project_images_detailed ||= begin
+              registry.tags.map do |tag|
+                image_history = registry.image_history(tag)
+                attrs = {
+                  id: registry.image_id(tag),
+                  parent: image_history['container_config']['Image'],
+                  labels: image_history['config']['Labels']
+                }
+                [tag, attrs]
+              end
+            end
+          end
+
+          def repo_image_tags_hierarchy(registry, registry_image_id)
+            hierarchy = []
+            iids = [registry_image_id]
+
+            loop do
+              hierarchy.concat(iids)
+              break if begin
+                iids.map! do |iid|
+                  repo_project_images_detailed(registry).map { |_, attrs| attrs[:id] if attrs[:parent] == iid }.compact
+                end.flatten!.empty?
+              end
+            end
+
+            repo_project_images_detailed(registry).map { |tag, attrs| tag if hierarchy.include? attrs[:id] }.compact
+          end
+
+          def remove_repo_images(registry, tags)
+            tags.each do |tag|
+              log(tag) if dry_run? || log_verbose?
+              registry.image_delete(tag) unless dry_run?
             end
           end
         end

data/lib/dapp/project/command/stages/common.rb

@@ -13,7 +13,7 @@ module Dapp
             DockerRegistry.new(repo)
           end
 
-          def repo_images(registry)
+          def repo_dimgs_and_cache(registry)
             format = proc do |arr|
               arr.map do |tag|
                 if (id = registry.image_id(tag)).nil?
@@ -30,8 +30,8 @@ module Dapp
           def registry_tags(registry)
             registry.tags
           rescue Exception::Registry => e
-            raise unless e.net_status[:code] == :dimg_not_found_in_registry
-            log_warning(desc: { code: 'dimg_not_found_in_registry' })
+            raise unless e.net_status[:code] == :no_such_dimg
+            log_warning(desc: { code: :dimg_not_found_in_registry })
             []
           end
 
@@ -47,6 +47,35 @@ module Dapp
             hash.select { |k, _v| k.start_with?('dapp-artifact') }.values
           end
 
+          def project_git_repositories
+            @project_git_repositories ||= begin
+              {}.tap do |repositories|
+                dimgs = build_configs.map { |config| Dimg.new(config: config, project: self, ignore_git_fetch: true) }
+                dimgs.each do |dimg|
+                  [dimg, dimg.artifacts].flatten
+                    .map(&:git_artifacts).flatten
+                    .map { |ga_artifact| repositories[ga_artifact.full_name] = ga_artifact.repo }
+                end
+              end
+            end
+          end
+
+          def proper_repo_cache?
+            !!cli_options[:proper_repo_cache]
+          end
+
+          def proper_git_commit?
+            !!cli_options[:proper_git_commit]
+          end
+
+          def stages_cleanup_option?
+            proper_git_commit? || proper_cache_version? || proper_repo_cache?
+          end
+
+          def log_proper_git_commit(&blk)
+            log_step_with_indent(:'proper git commit', &blk)
+          end
+
           def lock_repo(repo, *args, &blk)
             lock("repo.#{hashsum repo}", *args, &blk)
           end

data/lib/dapp/project/command/stages/flush_local.rb

@@ -11,7 +11,7 @@ module Dapp
               log_step_with_indent(name) do
                 project_containers_flush
                 project_dangling_images_flush
-                remove_images(project_images.lines.map(&:strip))
+                remove_images(project_images_names)
               end
             end
           end

data/lib/dapp/project/command/stages/flush_repo.rb

@@ -10,7 +10,7 @@ module Dapp
             lock_repo(repo) do
               log_step_with_indent(repo) do
                 registry = registry(repo)
-                repo_dimgs, repo_stages = repo_images(registry)
+                repo_dimgs, repo_stages = repo_dimgs_and_cache(registry)
                 repo_dimgs.merge(repo_stages).keys.each { |image_tag| delete_repo_image(registry, image_tag) }
               end
             end

data/lib/dapp/project/logging/base.rb

@@ -62,9 +62,11 @@ module Dapp
 
         def log_warning(*args, **kwargs)
           kwargs[:style] = :warning
-          kwargs[:desc] ||= {}
-          kwargs[:desc][:context] ||= :warning
           kwargs[:stream] ||= $stderr
+          if args.empty?
+            kwargs[:desc] ||= {}
+            kwargs[:desc][:context] ||= :warning
+          end
           log(*args, **kwargs)
         end
 

data/lib/dapp/version.rb

@@ -1,5 +1,5 @@
 # Version
 module Dapp
-  VERSION = '0.7.16'.freeze
+  VERSION = '0.7.17'.freeze
   BUILD_CACHE_VERSION = 6
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dapp
 version: !ruby/object:Gem::Version
-  version: 0.7.16
+  version: 0.7.17
 platform: ruby
 authors:
 - Dmitry Stolyarov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-12-01 00:00:00.000000000 Z
+date: 2016-12-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mixlib-shellout
@@ -138,6 +138,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 3.0.0
+- !ruby/object:Gem::Dependency
+  name: rugged
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.24.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.24.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -460,6 +474,7 @@ files:
 - lib/dapp/error/image.rb
 - lib/dapp/error/project.rb
 - lib/dapp/error/registry.rb
+- lib/dapp/error/rugged.rb
 - lib/dapp/error/shellout.rb
 - lib/dapp/exception/base.rb
 - lib/dapp/exception/introspect_image.rb