dapp 0.10.2 → 0.36.24

data/lib/dapp/dimg/config/directive/dimg_group_base.rb

@@ -0,0 +1,46 @@
+module Dapp
+  module Dimg
+    module Config
+      module Directive
+        module DimgGroupBase
+          def dimg(name = nil, &blk)
+            Dimg.new(name, dapp: dapp).tap do |dimg|
+              before_dimg_eval(dimg)
+              dimg.instance_eval(&blk) if block_given?
+              @_dimg << dimg
+            end
+          end
+
+          def dimg_group(&blk)
+            DimgGroup.new(dapp: dapp).tap do |dimg_group|
+              before_dimg_group_eval(dimg_group)
+              dimg_group.instance_eval(&blk) if block_given?
+              @_dimg_group << dimg_group
+            end
+          end
+
+          def _dimg
+            (@_dimg + _dimg_group.map(&:_dimg)).flatten
+          end
+
+          def _dimg_group
+            @_dimg_group||[]
+          end
+
+          protected
+
+          def before_dimg_eval(dimg)
+          end
+
+          def before_dimg_group_eval(dimg_group)
+          end
+
+          def dimg_group_init_variables!
+            @_dimg       = []
+            @_dimg_group = []
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dapp/dimg/config/directive/docker/artifact.rb

@@ -4,6 +4,11 @@ module Dapp
       module Directive
         module Docker
           class Artifact < Base
+            def method_missing(m, *args)
+              raise ::Dapp::Error::Config, code: :docker_artifact_unsupported_directive if Dimg.instance_methods.include?(m)
+              super
+            end
+
             def _change_options
               {}
             end

data/lib/dapp/dimg/config/directive/docker/base.rb

@@ -9,9 +9,8 @@ module Dapp
             def from(image, cache_version: nil)
               sub_directive_eval do
                 image = image.to_s
-                raise(Error::Config, code: :docker_from_incorrect, data: { name: image }) unless ::Dapp::Dimg::Image::Docker.image_name?(image)
-                raise(Error::Config, code: :docker_from_without_tag, data: { name: image }) unless image.include?(':')
-                @_from = image
+                raise(::Dapp::Error::Config, code: :docker_from_incorrect, data: { name: image }) unless ::Dapp::Dimg::Image::Stage.image_name?(image)
+                @_from = image.include?(':') ? image : [image, 'latest'].join(':')
                 @_from_cache_version = cache_version
               end
             end

data/lib/dapp/dimg/config/directive/docker/dimg.rb

@@ -18,11 +18,11 @@ module Dapp
             end
 
             def volume(*args)
-              sub_directive_eval { @_volume.concat(args) }
+              sub_directive_eval { @_volume.concat(args.flatten) }
             end
 
             def expose(*args)
-              sub_directive_eval { @_expose.concat(args) }
+              sub_directive_eval { @_expose.concat(args.flatten) }
             end
 
             def env(**options)
@@ -34,11 +34,11 @@ module Dapp
             end
 
             def cmd(*args)
-              sub_directive_eval { @_cmd.concat(args) }
+              sub_directive_eval { @_cmd.concat(args.flatten) }
             end
 
             def onbuild(*args)
-              sub_directive_eval { @_onbuild.concat(args) }
+              sub_directive_eval { @_onbuild.concat(args.flatten) }
             end
 
             def workdir(path)

data/lib/dapp/dimg/config/directive/git_artifact_local.rb

@@ -3,20 +3,39 @@ module Dapp
     module Config
       module Directive
         class GitArtifactLocal < ArtifactBase
+          attr_reader :_as
+
+          def as(value)
+            @_as = value
+          end
+
+          def export(absolute_dir_path = '/', &blk)
+            super
+          end
           alias add export
           undef_method :export
 
+          def _export
+            super do |export|
+              export._as    = @_as
+
+              yield(export) if block_given?
+            end
+          end
+
           class Export < ArtifactBase::Export
+            attr_accessor :_as
+
             def stage_dependencies(&blk)
               @stage_dependencies ||= StageDependencies.new(&blk)
             end
 
             def validate!
-              raise Error::Config, code: :add_to_required if _to.nil?
+              raise ::Dapp::Error::Config, code: :add_to_required if _to.nil?
             end
 
             def _artifact_options
-              super.merge(stages_dependencies: stage_dependencies.to_h)
+              super.merge(stages_dependencies: stage_dependencies.to_h, as: _as)
             end
 
             class StageDependencies < Directive::Base
@@ -25,10 +44,10 @@ module Dapp
               STAGES.each do |stage|
                 define_method(stage) do |*glob|
                   sub_directive_eval do
-                    if (globs = glob.map { |g| path_format(g) }).any? { |g| Pathname(g).absolute? }
-                      raise Error::Config, code: :stages_dependencies_paths_relative_path_required, data: { stage: stage }
+                    if (globs = glob.flatten.map { |g| path_format(g) }).any? { |g| Pathname(g).absolute? }
+                      raise ::Dapp::Error::Config, code: :stages_dependencies_paths_relative_path_required, data: { stage: stage }
                     end
-                    instance_variable_set(:"@#{stage}", globs)
+                    instance_variable_set(:"@#{stage}", public_send("_#{stage}") + globs)
                   end
                 end
 

data/lib/dapp/dimg/config/directive/git_artifact_remote.rb

@@ -3,24 +3,27 @@ module Dapp
     module Config
       module Directive
         class GitArtifactRemote < GitArtifactLocal
-          attr_reader :_url, :_name, :_branch, :_commit
+          include ::Dapp::Helper::Url
 
-          def initialize(url, **kwargs, &blk)
-            raise Error::Config, code: :git_artifact_remote_unsupported_protocol,
-                                 data: { url: url } unless %w(http https).include? URI(url.to_s).scheme
+          attr_reader :_url, :_name, :_branch, :_tag, :_commit
 
+          def initialize(url, **kwargs, &blk)
             @_url  = url
-            @_name = url.gsub(%r{.*?([^\/ ]+)\.git}, '\\1')
+            @_name = git_url_to_name(url)
 
             super(**kwargs, &blk)
           end
 
           def branch(value)
-            sub_directive_eval { @_branch = value }
+            sub_directive_eval { @_branch = value.to_s }
+          end
+
+          def tag(value)
+            sub_directive_eval { @_tag = value.to_s }
           end
 
           def commit(value)
-            sub_directive_eval { @_commit = value }
+            sub_directive_eval { @_commit = value.to_s }
           end
 
           def _export
@@ -28,28 +31,36 @@ module Dapp
               export._url    = @_url
               export._name   = @_name
               export._branch ||= @_branch
+              export._tag    ||= @_tag
               export._commit ||= @_commit
+
+              yield(export) if block_given?
             end
           end
 
           class Export < GitArtifactLocal::Export
-            attr_accessor :_url, :_name, :_branch, :_commit
+            attr_accessor :_url, :_name, :_branch, :_tag, :_commit
 
             def _artifact_options
-              super.merge(name: _name, branch: _branch, commit: _commit)
+              super.merge(name: _name, branch: _branch, tag: _tag, commit: _commit)
             end
 
             def branch(value)
-              sub_directive_eval { @_branch = value }
+              sub_directive_eval { @_branch = value.to_s }
+            end
+
+            def tag(value)
+              sub_directive_eval { @_tag = value.to_s }
             end
 
             def commit(value)
-              sub_directive_eval { @_commit = value }
+              sub_directive_eval { @_commit = value.to_s }
             end
 
             def validate!
               super
-              raise Error::Config, code: :git_artifact_remote_branch_with_commit if !_branch.nil? && !_commit.nil?
+              refs = [_branch, _tag, _commit].compact
+              raise ::Dapp::Error::Config, code: :git_artifact_remote_with_refs if refs.length > 1
             end
           end
         end

data/lib/dapp/dimg/config/directive/mount.rb

@@ -4,10 +4,11 @@ module Dapp
       module Directive
         class Mount < Base
           attr_reader :_to
+          attr_reader :_from
           attr_reader :_type
 
           def initialize(to, **kwargs, &blk)
-            raise Error::Config, code: :mount_to_absolute_path_required unless Pathname((to = to.to_s)).absolute?
+            raise ::Dapp::Error::Config, code: :mount_to_absolute_path_required unless Pathname((to = to.to_s)).absolute?
             @_to = path_format(to)
 
             super(**kwargs, &blk)
@@ -16,13 +17,20 @@ module Dapp
           def from(type)
             sub_directive_eval do
               type = type.to_sym
-              raise Error::Config, code: :mount_from_type_required unless [:tmp_dir, :build_dir].include? type
+              raise ::Dapp::Error::Config, code: :mount_from_type_required unless [:tmp_dir, :build_dir].include? type
               @_type = type
             end
           end
 
+          def from_path(path)
+            sub_directive_eval do
+              @_from = path_format(path)
+              @_type = :custom_dir
+            end
+          end
+
           def validate!
-            raise Error::Config, code: :mount_from_required if _type.nil?
+            raise ::Dapp::Error::Config, code: :mount_from_or_from_path_required if _type.nil?
           end
         end
       end

data/lib/dapp/dimg/config/directive/shell/dimg.rb

@@ -25,7 +25,7 @@ module Dapp
 
               define_method "_#{stage}_version" do
                 return [] if (variable = instance_variable_get("@_#{stage}")).nil?
-                variable._version || _version
+                variable._version
               end
             end
             [:before_install, :before_setup, :install, :setup].each(&method(:stage_command_generator))
@@ -48,7 +48,7 @@ module Dapp
               end
 
               def run(*args)
-                sub_directive_eval { @_run.concat(args) }
+                sub_directive_eval { @_run.concat(args.flatten) }
               end
 
               def version(value)

data/lib/dapp/dimg/dapp/command/bp.rb

@@ -1,13 +1,11 @@
 module Dapp
   module Dimg
-    class Dapp
+    module Dapp
       module Command
         module Bp
-          def bp(repo)
+          def bp
             bp_step(:build)
-            bp_step(:push, repo)
-            bp_step(:stages_cleanup_by_repo, repo)
-            bp_step(:cleanup)
+            bp_step(:push)
           end
 
           def bp_step(step, *args)

data/lib/dapp/dimg/dapp/command/build.rb

@@ -1,18 +1,16 @@
 module Dapp
   module Dimg
-    class Dapp
+    module Dapp
       module Command
         module Build
           def build
-            setup_ssh_agent
-
             build_configs.each do |config|
               log_dimg_name_with_indent(config) do
-                Dimg.new(config: config, dapp: self).build!
+                dimg(config: config).build!
               end
             end
-          rescue ::Dapp::Error::Shellout, Error::Base
-            build_context_export unless cli_options[:build_context_directory].nil?
+          rescue ::Dapp::Error::Shellout, Error::Default
+            build_context_export unless options[:build_context_directory].nil?
             raise
           end
         end

data/lib/dapp/dimg/dapp/command/build_context/common.rb

@@ -1,6 +1,6 @@
 module Dapp
   module Dimg
-    class Dapp
+    module Dapp
       module Command
         module BuildContext
           module Common
@@ -17,7 +17,7 @@ module Dapp
             end
 
             def build_context_directory
-              File.expand_path(cli_options[:build_context_directory].to_s)
+              File.expand_path(options[:build_context_directory].to_s)
             end
           end
         end

data/lib/dapp/dimg/dapp/command/build_context/export.rb

@@ -1,40 +1,40 @@
 module Dapp
   module Dimg
-    class Dapp
+    module Dapp
       module Command
         module BuildContext
           module Export
             def build_context_export
               log_process(:'export context') do
-                with_log_indent do
-                  export_build_context_image_tar
-                  export_build_context_build_tar
-                end
+                export_build_context_image_tar
+                export_build_context_build_tar
               end
             end
 
             def export_build_context_image_tar
               lock("#{name}.images", readonly: true) do
                 context_images_names = build_configs.map do |config|
-                  Dimg.new(config: config, dapp: self).tagged_images.map(&:name)
+                  dimg(config: config).all_tagged_images.map(&:name)
                 end.flatten
 
                 log_secondary_process(:images, short: true) do
-                  Image::Docker.save!(context_images_names, build_context_images_tar)
+                  Image::Stage.save!(self, context_images_names, build_context_images_tar, verbose: true, quiet: log_quiet?) unless dry_run?
                 end unless context_images_names.empty?
               end
             end
 
             def export_build_context_build_tar
               log_secondary_process(:build_dir, short: true) do
-                File.open(build_context_build_tar, File::RDWR | File::CREAT) do |f|
-                  Gem::Package::TarWriter.new(f) do |tar|
+                if !!options[:use_system_tar]
+                  shellout!("tar -C #{build_path} -cf #{build_context_build_tar} .")
+                else
+                  tar_write(build_context_build_tar) do |tar|
                     Dir.glob(File.join(build_path, '**/*'), File::FNM_DOTMATCH).each do |path|
                       archive_file_path = path
-                                            .reverse
-                                            .chomp(build_path.to_s.reverse)
-                                            .chomp('/')
-                                            .reverse
+                        .reverse
+                        .chomp(build_path.to_s.reverse)
+                        .chomp('/')
+                        .reverse
                       if File.directory?(path)
                         tar.mkdir archive_file_path, File.stat(path).mode
                       else
@@ -44,7 +44,7 @@ module Dapp
                       end
                     end
                   end
-                end
+                end unless dry_run?
               end
             end
           end

data/lib/dapp/dimg/dapp/command/build_context/import.rb

@@ -1,27 +1,25 @@
 module Dapp
   module Dimg
-    class Dapp
+    module Dapp
       module Command
         module BuildContext
           module Import
             def build_context_import
-              raise Error::Command, code: :context_directory_not_found,
-                                    data: { path: build_context_path } unless build_context_path.exist?
+              raise ::Dapp::Error::Command, code: :context_directory_not_found,
+                                            data: { path: build_context_path } unless build_context_path.exist?
 
               log_process(:'import context') do
-                with_log_indent do
-                  import_build_context_build_tar
-		              import_build_context_image_tar
-                end
+                import_build_context_build_tar
+                import_build_context_image_tar
               end
             end
 
             def import_build_context_image_tar
               if build_context_images_tar.exist?
-                log_secondary_process(:images, short: true) do
+                log_secondary_process(:images) do
                   lock("#{name}.images") do
-                    Image::Docker.load!(build_context_images_tar)
-                  end
+                    Image::Stage.load!(self, build_context_images_tar, verbose: true, quiet: log_quiet?)
+                  end unless dry_run?
                 end
               else
                 log_warning(desc: { code: :context_archive_not_found, data: { path: build_context_images_tar } })
@@ -31,19 +29,26 @@ module Dapp
             def import_build_context_build_tar
               if build_context_build_tar.exist?
                 log_secondary_process(:build_dir, short: true) do
-                  store_current_build_dir
+                  unless dry_run?
+                    store_current_build_dir
 
-                  File.open(build_context_build_tar, File::RDONLY) do |f|
-                    Gem::Package::TarReader.new(f).each_entry do |entry|
-                      header = entry.header
-                      path = File.join(build_path, entry.full_name)
+                    if !!options[:use_system_tar]
+                      FileUtils.mkpath build_path
+                      shellout!("tar -xf #{build_context_build_tar} -C #{build_path}")
+                    else
+                      tar_read(build_context_build_tar) do |tar|
+                        tar.each_entry do |entry|
+                          header = entry.header
+                          path = File.join(build_path, entry.full_name)
 
-                      if entry.directory?
-                        FileUtils.mkpath path, :mode => entry.header.mode
-                      else
-                        FileUtils.mkpath File.dirname(path)
-                        File.write(path, entry.read)
-                        File.chmod(header.mode, path)
+                          if entry.directory?
+                            FileUtils.mkpath path, :mode => entry.header.mode
+                          else
+                            FileUtils.mkpath File.dirname(path)
+                            File.write(path, entry.read)
+                            File.chmod(header.mode, path)
+                          end
+                        end
                       end
                     end
                   end
@@ -55,8 +60,8 @@ module Dapp
 
             def store_current_build_dir
               return if build_path_empty?
-              raise Error::Command, code: :stored_build_dir_already_exist,
-                                    data: { path: "#{build_path}.old" } if File.exist?("#{build_path}.old")
+              raise ::Dapp::Error::Command, code: :stored_build_dir_already_exist,
+                                            data: { path: "#{build_path}.old" } if File.exist?("#{build_path}.old")
               FileUtils.mv(build_path, "#{build_path}.old")
             end
 

data/lib/dapp/dimg/dapp/command/cleanup.rb

@@ -1,19 +1,12 @@
 module Dapp
   module Dimg
-    class Dapp
+    module Dapp
       module Command
         module Cleanup
           def cleanup
-            lock("#{name}.images") do
-              log_step_with_indent(name) do
-                dapp_containers_flush
-                dapp_dangling_images_flush
-                remove_images_by_query([
-                  'docker images',
-                  %(--format '{{if ne "#{stage_cache}" .Repository }}{{.Repository}}:{{.Tag}}{{ end }}'),
-                  %(-f "label=dapp=#{stage_dapp_label}")
-                ].join(' ')) # FIXME: negative filter is not currently supported by the Docker CLI
-              end
+            log_step_with_indent(:cleanup) do
+              dapp_containers_flush_by_label('dapp')
+              dapp_dangling_images_flush_by_label('dapp')
             end
           end
         end

data/lib/dapp/dimg/dapp/command/cleanup_repo.rb

@@ -0,0 +1,143 @@
+module Dapp
+  module Dimg
+    module Dapp
+      module Command
+        module CleanupRepo
+          def cleanup_repo
+            git_repo_option = git_own_repo_exist? ? JSON.dump(git_own_repo.get_ruby2go_state_hash) : nil
+            ruby2go_cleanup_command(:gc, ruby2go_cleanup_gc_options_dump, local_repo: git_repo_option)
+          end
+
+          def ruby2go_cleanup_gc_options_dump
+            ruby2go_cleanup_common_repo_options.merge(
+              mode: {
+                with_stages: with_stages?
+              },
+              deployed_docker_images: deployed_docker_images
+            ).tap do |data|
+              break JSON.dump(data)
+            end
+          end
+
+          def deployed_docker_images
+            @deployed_docker_images ||= search_deployed_docker_images
+          end
+
+          def search_deployed_docker_images
+            return [] if without_kube?
+
+            config = ::Dapp::Kube::Kubernetes::Config.new_auto_if_available
+            return [] if config.nil?
+
+            config.context_names.
+              map {|context_name|
+                client = ::Dapp::Kube::Kubernetes::Client.new(
+                  config,
+                  context_name,
+                  "default",
+                  timeout: options[:kubernetes_timeout],
+                )
+                search_deployed_docker_images_from_kube(client)
+              }.flatten.sort.uniq
+          end
+
+          def search_deployed_docker_images_from_kube(client)
+            namespaces = []
+            # check connectivity for 2 seconds
+            begin
+              namespaces = client.namespace_list(excon_parameters: {:connect_timeout => 30})
+            rescue Excon::Error::Timeout
+              raise ::Dapp::Error::Default, code: :kube_connect_timeout
+            end
+
+            # get images from containers from pods from all namespaces.
+            namespaces['items'].map do |item|
+              item['metadata']['name']
+            end.map do |ns|
+              [].tap do |arr|
+                client.with_namespace(ns) do
+                  arr << pod_images(client)
+                  arr << deployment_images(client)
+                  arr << replicaset_images(client)
+                  arr << statefulset_images(client)
+                  arr << daemonset_images(client)
+                  arr << job_images(client)
+                  arr << cronjob_images(client)
+                  arr << replicationcontroller_images(client)
+                end
+              end
+            end.flatten.select do |img|
+              img.start_with? option_repo
+            end.sort.uniq
+          end
+
+          # pod items[] spec containers[] image
+          def pod_images(client)
+            client.pod_list['items'].map do |item|
+              images_from_pod_spec(item)
+            end
+          end
+
+          # cronjob items[] spec jobTemplate spec template spec containers[] image
+          def cronjob_images(client)
+            client.cronjob_list['items'].map do |item|
+              images_from_pod_spec(item['spec']['jobTemplate']['spec']['template'])
+            end
+          end
+
+          # daemonsets   items[] spec template spec containers[] image
+          def daemonset_images(client)
+            client.daemonset_list['items'].map do |item|
+              images_from_pod_spec(item['spec']['template'])
+            end
+          end
+
+          # deployment   items[] spec template spec containers[] image
+          def deployment_images(client)
+            client.deployment_list['items'].map do |item|
+              images_from_pod_spec(item['spec']['template'])
+            end
+          end
+
+          # job          items[] spec template spec containers[] image
+          def job_images(client)
+            client.job_list['items'].map do |item|
+              images_from_pod_spec(item['spec']['template'])
+            end
+          end
+
+          # replicasets  items[] spec template spec containers[] image
+          def replicaset_images(client)
+            client.replicaset_list['items'].map do |item|
+              images_from_pod_spec(item['spec']['template'])
+            end
+          end
+
+           # replicasets  items[] spec template spec containers[] image
+           def statefulset_images(client)
+            client.statefulset_list['items'].map do |item|
+              images_from_pod_spec(item['spec']['template'])
+            end
+          end
+
+          # replicationcontroller    items[] spec template spec containers[] image
+          def replicationcontroller_images(client)
+            client.replicationcontroller_list['items'].map do |item|
+              images_from_pod_spec(item['spec']['template'])
+            end
+          end
+
+          def images_from_pod_spec(pod_spec)
+            containers = Array(pod_spec['spec']['containers'])
+            initContainers = Array(pod_spec['spec']['initContainers'])
+            (containers + initContainers).map { |cont| cont['image'] }
+          end
+
+          def without_kube?
+            !!options[:without_kube]
+          end
+        end
+      end
+    end
+  end # Dimg
+end # Dapp