dapp 0.26.14 → 0.27.0

data/lib/dapp/dimg/builder/chef.rb

@@ -127,7 +127,7 @@ module Dapp
       end
 
       def stage_cookbooks_checksum_path(stage)
-        dimg.build_path.join("#{builder_cookbook.checksum}.#{dimg.config._name}.#{stage}.checksum")
+        dimg.build_path.join("#{builder_cookbook.checksum}.#{dimg.name}.#{stage}.checksum")
       end
 
       def stage_cookbooks_checksum(stage)
@@ -216,7 +216,7 @@ module Dapp
       end
 
       def stage_build_path(stage)
-        dimg.tmp_path(dimg.config._name).join(stage.to_s)
+        dimg.tmp_path(dimg.name).join(stage.to_s)
       end
 
       def container_stage_build_path(_stage)

data/lib/dapp/dimg/cli/command/base.rb

@@ -1,6 +1,16 @@
 module Dapp::Dimg::CLI
   module Command
     class Base < ::Dapp::CLI::Command::Base
+      DIMG_STAGES = [
+        :from, :before_install, :before_install_artifact, :g_a_archive, :g_a_pre_install_patch, :install,
+        :g_a_post_install_patch, :after_install_artifact, :before_setup, :before_setup_artifact,
+        :g_a_pre_setup_patch, :setup, :g_a_post_setup_patch, :after_setup_artifact, :g_a_latest_patch, :docker_instructions
+      ].freeze
+
+      STAGE_PROC = proc do |stages|
+        proc { |val| val.to_sym.tap { |v| in_validate!(v, stages) } }
+      end
+
       option :build_dir,
              long: "--build-dir PATH",
              description: "Directory where build cache stored ($HOME/.dapp/builds/<dapp name> by default)."
@@ -9,6 +19,10 @@ module Dapp::Dimg::CLI
         self.class.parse_options(self, argv)
         run_dapp_command(run_method, options: cli_options(dimgs_patterns: cli_arguments))
       end
+
+      def run_method
+        class_to_lowercase
+      end
     end
   end
 end

data/lib/dapp/dimg/cli/command/dimg/build.rb

@@ -11,25 +11,16 @@ Usage:
 
 Options:
 BANNER
-        introspected_stages = [
-          :from, :before_install, :before_install_artifact, :g_a_archive, :g_a_pre_install_patch, :install,
-          :g_a_post_install_patch, :after_install_artifact, :before_setup, :before_setup_artifact,
-          :g_a_pre_setup_patch, :setup, :g_a_post_setup_patch, :after_setup_artifact, :g_a_latest_patch, :docker_instructions
-        ]
-        artifact_introspected_stages = [
+        artifact_stages = [
           :from, :before_install, :before_install_artifact, :g_a_archive, :g_a_pre_install_patch, :install,
           :g_a_post_install_patch, :after_install_artifact, :before_setup, :before_setup_artifact,
           :g_a_pre_setup_patch, :setup, :after_setup_artifact, :g_a_artifact_patch, :build_artifact
         ]
 
-        introspect_stage_proc = proc do |stages|
-          proc { |val| val.to_sym.tap { |v| in_validate!(v, stages) } }
-        end
-
-        introspect_before_proc = proc do |stages|
+        before_stage_proc = proc do |stages|
           proc do |val|
             val_sym = val.to_sym
-            introspect_stage_proc.call(stages[1..-1]).call(val_sym)
+            STAGE_PROC.call(stages[1..-1]).call(val_sym)
             stages[stages.index(val_sym) - 1]
           end
         end
@@ -58,24 +49,24 @@ BANNER
                default: false
 
         option :introspect_stage,
-               long: '--introspect-stage STAGE',
-               description: "Introspect one of the following stages (#{list_msg_format(introspected_stages)})",
-               proc: introspect_stage_proc.call(introspected_stages)
+               long:        '--introspect-stage STAGE',
+               description: "Introspect one of the following stages (#{list_msg_format(DIMG_STAGES)})",
+               proc:        STAGE_PROC.call(DIMG_STAGES)
 
         option :introspect_before,
-               long: '--introspect-before STAGE',
-               description: "Introspect stage before one of the following stages (#{list_msg_format(introspected_stages[1..-1])})",
-               proc: introspect_before_proc.call(introspected_stages)
+               long:        '--introspect-before STAGE',
+               description: "Introspect stage before one of the following stages (#{list_msg_format(DIMG_STAGES[1..-1])})",
+               proc:        before_stage_proc.call(DIMG_STAGES)
 
         option :introspect_artifact_stage,
-               long: '--introspect-artifact-stage STAGE',
-               description: "Introspect one of the following stages (#{list_msg_format(artifact_introspected_stages)})",
-               proc: introspect_stage_proc.call(artifact_introspected_stages)
+               long:        '--introspect-artifact-stage STAGE',
+               description: "Introspect one of the following stages (#{list_msg_format(artifact_stages)})",
+               proc:        STAGE_PROC.call(artifact_stages)
 
         option :introspect_artifact_before,
-               long: '--introspect-artifact-before STAGE',
-               description: "Introspect stage before one of the following stages (#{list_msg_format(artifact_introspected_stages[1..-1])})",
-               proc: introspect_before_proc.call(artifact_introspected_stages)
+               long:        '--introspect-artifact-before STAGE',
+               description: "Introspect stage before one of the following stages (#{list_msg_format(artifact_stages[1..-1])})",
+               proc:        before_stage_proc.call(artifact_stages)
 
         option :ssh_key,
                long: '--ssh-key SSH_KEY',

data/lib/dapp/dimg/cli/command/dimg/run.rb

@@ -12,6 +12,11 @@ Usage:
 
 Options:
 BANNER
+        option :stage,
+               long:        '--stage STAGE',
+               description: "Run one of the following stages (#{list_msg_format(DIMG_STAGES)})",
+               proc:        STAGE_PROC.call(DIMG_STAGES)
+
         option :ssh_key,
                long: '--ssh-key SSH_KEY',
                description: ['Enable only specified ssh keys ',
@@ -51,8 +56,16 @@ BANNER
           index = filtered_args.index('--') || filtered_args.count
           docker_options = index.nonzero? ? filtered_args.slice(0..index - 1) : []
           command = filtered_args.slice(index + 1..-1) || []
+
+          if docker_options.empty? && command.empty?
+            docker_options = %w(-ti --rm)
+            command = %w(/bin/bash)
+          end
+
+          stage_name = config.delete(:stage)
+
           run_dapp_command(nil, options: cli_options(dimgs_patterns: patterns), log_running_time: false) do |dapp|
-            dapp.run(docker_options, command)
+            dapp.run(stage_name, docker_options, command)
           end
         end
       end

data/lib/dapp/dimg/config/directive/dimg/instance_methods.rb

@@ -5,8 +5,9 @@ module Dapp
         class Dimg < Base
           module InstanceMethods
             attr_reader :_builder
-            attr_reader :_chef, :_shell, :_docker, :_git_artifact, :_mount, :_artifact
+            attr_reader :_chef, :_shell, :_docker, :_git_artifact, :_mount, :_artifact, :_ansible
             attr_reader :_artifact_groups
+            attr_reader :_from_dimg, :_from_dimg_artifact
 
             def chef(&blk)
               builder(:chef)

data/lib/dapp/dimg/config/directive/dimg_group_base.rb

@@ -20,11 +20,11 @@ module Dapp
           end
 
           def _dimg
-            (@_dimg + @_dimg_group.map(&:_dimg)).flatten
+            (@_dimg + _dimg_group.map(&:_dimg)).flatten
           end
 
           def _dimg_group
-            @_dimg_group
+            @_dimg_group||[]
           end
 
           protected

data/lib/dapp/dimg/config/directive/git_artifact_remote.rb

@@ -3,13 +3,14 @@ module Dapp
     module Config
       module Directive
         class GitArtifactRemote < GitArtifactLocal
-          include ::Dapp::Helper::Url
-
           attr_reader :_url, :_name, :_branch, :_commit
 
           def initialize(url, **kwargs, &blk)
             @_url  = url
-            @_name = git_url_to_name(url)
+
+            url_without_scheme = url.split("://", 2).last
+            url_without_creds = url_without_scheme.split(":", 2).last
+            @_name = url_without_creds.gsub(%r{.*?([^\/ ]+\/[^\/ ]+)\.git}, '\\1')
 
             super(**kwargs, &blk)
           end

data/lib/dapp/dimg/dapp/command/cleanup.rb

@@ -5,8 +5,8 @@ module Dapp
         module Cleanup
           def cleanup
             log_step_with_indent(:cleanup) do
-              dapp_containers_flush
-              dapp_dangling_images_flush
+              dapp_containers_flush_by_label('dapp')
+              dapp_dangling_images_flush_by_label('dapp')
             end
           end
         end

data/lib/dapp/dimg/dapp/command/common.rb

@@ -66,12 +66,16 @@ module Dapp
             project_images.map { |image| image[:dangling] ? image[:id] : image[:name] }
           end
 
-          def dapp_containers_flush
-            remove_containers_by_query(%(#{host_docker} ps -a -f "label=dapp" -q --no-trunc))
+          def dapp_containers_flush_by_label(label)
+            log_proper_containers do
+              remove_containers_by_query(%(#{host_docker} ps -a -f "label=#{label}" -q --no-trunc))
+            end
           end
 
-          def dapp_dangling_images_flush
-            remove_images_by_query(%(#{host_docker} images -f "dangling=true" -f "label=dapp" -q --no-trunc))
+          def dapp_dangling_images_flush_by_label(label)
+            log_proper_flush_dangling_images do
+              remove_images_by_query(%(#{host_docker} images -f "dangling=true" -f "label=#{label}" -q --no-trunc))
+            end
           end
 
           def dapp_tagless_images_flush
@@ -125,7 +129,6 @@ module Dapp
           end
 
           def remove_base(query_format, ids, force: false)
-            return if ids.empty?
             force_option = force ? ' -f' : ''
             log(ids.join("\n")) if log_verbose? || dry_run?
             run_command(format(query_format, force_option: force_option, ids: ids.join(' ')))
@@ -178,6 +181,14 @@ module Dapp
             log_step_with_indent(:'proper repo cache', &blk)
           end
 
+          def log_proper_containers(&blk)
+            log_step_with_indent(:'proper containers', &blk)
+          end
+
+          def log_proper_flush_dangling_images(&blk)
+            log_step_with_indent(:'proper dangling', &blk)
+          end
+
           def push_format(dimg_name)
             if dimg_name.nil?
               spush_format

data/lib/dapp/dimg/dapp/command/mrproper.rb

@@ -16,7 +16,7 @@ module Dapp
                 proper_cache_version
               end
 
-              dapp_dangling_images_flush
+              dapp_dangling_images_flush_by_label('dapp')
               dapp_tagless_images_flush
             end
           end
@@ -52,16 +52,14 @@ module Dapp
           end
 
           def flush_by_label(label)
-            log_step_with_indent(:containers) { dapp_containers_flush_by_label(label) }
-            log_step_with_indent(:images) { dapp_images_flush_by_label(label) }
-          end
-
-          def dapp_containers_flush_by_label(label)
-            remove_containers_by_query(%(#{host_docker} ps -a -f "label=dapp" -f "label=#{label}" -q --no-trunc))
+            dapp_containers_flush_by_label(label)
+            dapp_images_flush_by_label(label)
           end
 
           def dapp_images_flush_by_label(label)
-            remove_images(dapp_images_names_by_label(label))
+            log_step_with_indent('proper images') do
+              remove_images(dapp_images_names_by_label(label))
+            end
           end
 
           def proper_cache_version

data/lib/dapp/dimg/dapp/command/push.rb

@@ -8,7 +8,7 @@ module Dapp
             #RubyProf.start
             log_step_with_indent(:stages) { stages_push } if with_stages?
             dimg_import_export_base do |dimg|
-              dimg.export!(option_repo, format: push_format(dimg.config._name))
+              dimg.export!(option_repo, format: push_format(dimg.name))
             end
             # FIXME: rework images cache, then profile
             #result = RubyProf.stop

data/lib/dapp/dimg/dapp/command/run.rb

@@ -3,10 +3,11 @@ module Dapp
     module Dapp
       module Command
         module Run
-          def run(docker_options, command)
+          def run(stage_name, docker_options, command)
             one_dimg!
             setup_ssh_agent
-            dimg(config: build_configs.first, ignore_git_fetch: true, should_be_built: true).run(docker_options, command)
+            dimg(config: build_configs.first, ignore_git_fetch: true, should_be_built: stage_name.nil?)
+              .run_stage(stage_name, docker_options, command)
           end
         end
       end

data/lib/dapp/dimg/dapp/command/stages/cleanup_local.rb

@@ -11,6 +11,9 @@ module Dapp
                 proper_cache           if proper_cache_version?
                 stages_cleanup_by_repo if proper_repo_cache?
                 proper_git_commit      if proper_git_commit?
+
+                dapp_containers_flush_by_label("dapp=#{name}")
+                dapp_dangling_images_flush_by_label("dapp=#{name}")
               end
             end
 

data/lib/dapp/dimg/dapp/command/tag.rb

@@ -5,7 +5,7 @@ module Dapp
         module Tag
           def tag
             dimg_import_export_base do |dimg|
-              dimg.tag!(option_repo, format: push_format(dimg.config._name))
+              dimg.tag!(option_repo, format: push_format(dimg.name))
             end
           end
         end

data/lib/dapp/dimg/dapp/dimg.rb

@@ -10,6 +10,14 @@ module Dapp
           (@artifacts_dimgs ||= {})[config._name] ||= ::Dapp::Dimg::Artifact.new(config: config, dapp: self, **kwargs)
         end
 
+        def dimg_layer(config:, **kwargs)
+          (@dimg_layers ||= {})[config._name] ||= ::Dapp::Dimg::Dimg.new(config: config, dapp: self, **kwargs)
+        end
+
+        def artifact_dimg_layer(config:, **kwargs)
+          (@artifact_dimg_layers ||= {})[config._name] ||= ::Dapp::Dimg::Artifact.new(config: config, dapp: self, **kwargs)
+        end
+
         def _terminate_dimg_on_terminate(dimg)
           @_call_before_terminate << proc{dimg.terminate}
         end

data/lib/dapp/dimg/dimg.rb

@@ -25,6 +25,10 @@ module Dapp
         raise Error::Dimg, code: :dimg_not_built if should_be_built?
       end
 
+      def name
+        config._name
+      end
+
       def terminate
         cleanup_tmp
       end
@@ -74,7 +78,7 @@ module Dapp
           dapp.tags_by_scheme.each do |tag_scheme_name, tags|
             dapp.log_step_with_indent(tag_scheme_name) do
               tags.each do |tag|
-                image_name = format(export_format, repo: repo, dimg_name: config._name, tag: tag)
+                image_name = format(export_format, repo: repo, dimg_name: name, tag: tag)
                 export_base!(image_name, push: push) do
                   export_image = build_export_image!(image_name, scheme_name: tag_scheme_name)
                   if push
@@ -139,7 +143,13 @@ module Dapp
       end
 
       def run(docker_options, command)
-        cmd = "#{dapp.host_docker} run #{[docker_options, last_stage.image.built_id, command].flatten.compact.join(' ')}"
+        run_stage(nil, docker_options, command)
+      end
+
+      def run_stage(stage_name, docker_options, command)
+        stage_image = (stage_name.nil? ? last_stage : stage_by_name(stage_name)).image
+        raise Error::Dimg, code: :dimg_stage_not_built, data: { stage_name: stage_name } unless stage_image.built?
+        cmd = "#{dapp.host_docker} run #{[docker_options, stage_image.built_id, command].flatten.compact.join(' ')}"
         if dapp.dry_run?
           dapp.log(cmd)
         else
@@ -164,7 +174,7 @@ module Dapp
       end
 
       def scratch?
-        config._docker._from.nil?
+        config._docker._from.nil? && config._from_dimg.nil? && config._from_dimg_artifact.nil?
       end
 
       def dev_mode?
@@ -198,6 +208,7 @@ module Dapp
         cmd = "".tap do |cmd|
           cmd << "#{dapp.host_docker} run --rm"
           cmd << " --volume #{dapp.tmp_base_dir}:#{dapp.tmp_base_dir}"
+          cmd << " --label dapp=#{dapp.name}"
           cmd << " alpine:3.6"
           cmd << " rm -rf #{tmp_path}"
         end

data/lib/dapp/dimg/dimg/stages.rb

@@ -34,7 +34,7 @@ module Dapp
         def last_stage
           @last_stage || begin
             (self.last_stage = last_stage_class.new(self)).tap do |stage|
-              dapp.log_secondary_process("#{config._name || 'nameless'}: calculating stages signatures") do
+              dapp.log_secondary_process("#{name || 'nameless'}: calculating stages signatures") do
                 stage.signature
               end
             end

data/lib/dapp/dimg/docker_registry/base.rb

@@ -35,6 +35,7 @@ module Dapp
         end
 
         def image_delete(tag)
+          image_blobs(tag).each { |hash| blob_delete(hash.values.first) }
           api_request(repo_suffix, "/manifests/#{image_digest(tag)}",
                       method: :delete,
                       expects: [202, 404],
@@ -53,6 +54,10 @@ module Dapp
                           headers: { Accept: 'application/vnd.docker.distribution.manifest.v2+json' }).headers['Docker-Content-Digest']
         end
 
+        def image_blobs(tag)
+          manifest_v1(tag)['fsLayers']
+        end
+
         def manifest_v1(tag)
           api_request(repo_suffix, "/manifests/#{tag}")
         end
@@ -62,6 +67,11 @@ module Dapp
                       headers: { Accept: 'application/vnd.docker.distribution.manifest.v2+json' })
         end
 
+        def blob_delete(id)
+          api_request(repo_suffix, "/blobs/#{id}",
+                      method: :delete, expects: [202, 404])
+        end
+
         def api_request(*uri, **options)
           JSON.load(raw_api_request(*uri, **options).body)
         end

data/lib/dapp/dimg/docker_registry/base/authorization.rb

@@ -29,25 +29,10 @@ module Dapp
             [:realm, :service, :scope].map do |option|
               /#{option}="([[^"].]*)/ =~ header
               next unless Regexp.last_match(1)
-
-              option_value = begin
-                if option == :scope
-                  handle_scope_option(Regexp.last_match(1))
-                else
-                  Regexp.last_match(1)
-                end
-              end
-
-              [option, option_value]
+              [option, Regexp.last_match(1)]
             end.compact.to_h
           end
 
-          def handle_scope_option(resourcescope)
-            resource_type, resource_name, actions = resourcescope.split(":")
-            actions                               = actions.split(",").map { |action| action == "delete" ? "*" : action }.join(",")
-            [resource_type, resource_name, actions].join(":")
-          end
-
           def authorization_auth
             @authorization_auth ||= begin
               if ::Dapp::Dapp.options_with_docker_credentials?