dapp 0.10.2 → 0.36.24

data/lib/dapp/dimg/artifact.rb

@@ -1,11 +1,19 @@
 module Dapp
   module Dimg
     class Artifact < Dimg
+      def name
+        dapp.consistent_uniq_slugify(config._name) unless config._name.nil?
+      end
+
       def after_stages_build!
       end
 
-      def stage_should_be_introspected?(name)
-        dapp.cli_options[:introspect_artifact_stage] == name
+      def stage_should_be_introspected_before_build?(name)
+        dapp.options[:introspect_artifact_before] == name
+      end
+
+      def stage_should_be_introspected_after_build?(name)
+        dapp.options[:introspect_artifact_stage] == name
       end
 
       def artifact?
@@ -16,8 +24,8 @@ module Dapp
         false
       end
 
-      def last_stage
-        @last_stage ||= Build::Stage::BuildArtifact.new(self)
+      def last_stage_class
+        Build::Stage::BuildArtifact
       end
     end # Artifact
   end # Dimg

data/lib/dapp/dimg/build/stage/artifact_base.rb

@@ -4,27 +4,27 @@ module Dapp
       module Stage
         class ArtifactBase < Base
           def dependencies
-            artifacts_signatures
+            @dependencies ||= artifacts_signatures
           end
 
           def prepare_image
-            super
-            artifacts_dimgs_build!
-            artifacts_labels = {}
-            artifacts.each do |artifact|
-              apply_artifact(artifact, image)
-              artifacts_labels["dapp-artifact-#{artifact[:name]}".to_sym] = artifact[:dimg].last_stage.image.built_id
+            super do
+              artifacts_dimgs_build!
+              artifacts_labels = {}
+              artifacts.each do |artifact|
+                apply_artifact(artifact, image)
+                artifacts_labels["dapp-artifact-#{artifact[:dimg].name}".to_sym] = artifact[:dimg].last_stage.image.built_id
+              end
+              image.add_service_change_label artifacts_labels
             end
-            image.add_service_change_label artifacts_labels
           end
 
           def artifacts
             @artifacts ||= begin
               dimg.config.public_send("_#{name}").map do |artifact|
-                artifact_dimg = ::Dapp::Dimg::Artifact.new(config: artifact._config,
-                                                           dapp: dimg.dapp,
-                                                           ignore_git_fetch: dimg.ignore_git_fetch)
-                { name: artifact._config._name, options: artifact._artifact_options, dimg: artifact_dimg }
+                artifact_dimg = dimg.dapp.artifact_dimg(config: artifact._config,
+                                                        ignore_signature_auto_calculation: dimg.ignore_signature_auto_calculation)
+                { options: artifact._artifact_options, dimg: artifact_dimg }
               end
             end
           end
@@ -48,29 +48,25 @@ module Dapp
           end
 
           def artifacts_dimgs_build!
-            artifacts.each do |artifact|
-              process = dimg.dapp.t(code: 'process.artifact_building', data: { name: artifact[:name] })
-              dimg.dapp.log_secondary_process(process,
-                                              short: !dimg.dapp.log_verbose?,
-                                              quiet: dimg.artifact? && !dimg.dapp.log_verbose?) do
-                dimg.dapp.with_log_indent do
-                  artifact[:dimg].build!
-                end
-              end
+            artifacts.uniq { |artifact| artifact[:dimg] }.each do |artifact|
+              process = dimg.dapp.t(code: 'process.artifact_building', data: { name: artifact[:dimg].name })
+              dimg.dapp.log_secondary_process(process) { artifact[:dimg].build! }
             end
           end
 
-          def run_artifact_dimg(artifact_dimg, artifact_name, commands)
+          def run_artifact_dimg(artifact_dimg, commands)
             docker_options = ['--rm',
-                              "--volume #{dimg.tmp_path('artifact', artifact_name)}:#{artifact_dimg.container_tmp_path(artifact_name)}",
-                              "--volumes-from #{dimg.dapp.base_container}",
-                              "--entrypoint #{dimg.dapp.bash_bin}"]
-            dimg.dapp.log_secondary_process(dimg.dapp.t(code: 'process.artifact_copy',
-                                                        data: { name: artifact_name }),
-                                            short: true,
-                                            quiet: dimg.artifact? && !dimg.dapp.log_verbose?) do
-              artifact_dimg.run(docker_options, [%(-ec '#{dimg.dapp.shellout_pack(commands)}')])
+                              "--volume=#{dimg.tmp_path('artifact', artifact_dimg.name )}:#{artifact_dimg.container_tmp_path(artifact_dimg.name)}",
+                              "--volumes-from=#{dimg.dapp.toolchain_container}",
+                              "--volumes-from=#{dimg.dapp.base_container}",
+                              "--entrypoint=#{dimg.dapp.bash_bin}"]
+            dimg.dapp.log_secondary_process(dimg.dapp.t(code: 'process.artifact_copy', data:
+              { name: artifact_dimg.name }), short: true) do
+              artifact_dimg.run(docker_options, ["-ec", dimg.dapp.shellout_pack(commands)])
             end
+          rescue Error::Dimg => e
+            raise unless e.net_status[:code] == :dimg_not_run
+            raise Error::Build, code: :export_failed, data: { artifact_name: artifact_dimg.name }
           end
           # rubocop:enable Metrics/AbcSize
         end # ArtifactBase

data/lib/dapp/dimg/build/stage/artifact_default.rb

@@ -8,7 +8,6 @@ module Dapp
           def apply_artifact(artifact, image)
             return if dimg.dapp.dry_run?
 
-            artifact_name = artifact[:name]
             artifact_dimg = artifact[:dimg]
             cwd = artifact[:options][:cwd]
             include_paths = artifact[:options][:include_paths]
@@ -17,12 +16,14 @@ module Dapp
             group = artifact[:options][:group]
             to = artifact[:options][:to]
 
-            command = safe_cp(cwd, artifact_dimg.container_tmp_path(artifact_name).to_s, Process.uid, Process.gid, include_paths, exclude_paths)
-            run_artifact_dimg(artifact_dimg, artifact_name, command)
+            export_folder_name = SecureRandom.uuid
 
-            command = safe_cp(dimg.container_tmp_path('artifact', artifact_name).to_s, to, owner, group, include_paths, exclude_paths)
+            command = safe_cp(cwd, artifact_dimg.container_tmp_path(artifact_dimg.name, export_folder_name).to_s, nil, nil, include_paths, exclude_paths)
+            run_artifact_dimg(artifact_dimg, command)
+
+            command = safe_cp(dimg.container_tmp_path('artifact', artifact_dimg.name, export_folder_name).to_s, to, owner, group, include_paths, exclude_paths)
             image.add_command command
-            image.add_volume "#{dimg.tmp_path('artifact', artifact_name)}:#{dimg.container_tmp_path('artifact', artifact_name)}:ro"
+            image.add_volume "#{dimg.tmp_path('artifact', artifact_dimg.name)}:#{dimg.container_tmp_path('artifact', artifact_dimg.name)}:ro"
           end
           # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
 
@@ -30,32 +31,53 @@ module Dapp
 
           # rubocop:disable Metrics/ParameterLists
           def safe_cp(from, to, owner, group, include_paths = [], exclude_paths = [])
-            credentials = ''
-            credentials += "-o #{owner} " if owner
-            credentials += "-g #{group} " if group
-            excludes = find_command_excludes(from, exclude_paths).join(' ')
-
-            copy_files = proc do |from_, path_ = ''|
-              "if [[ -d #{File.join(from_, path_)} ]] || [[ -f #{File.join(from_, path_)} ]]; then " \
-              "#{dimg.dapp.find_bin} #{File.join(from_, path_)} #{excludes} -type f -exec " \
-              "#{dimg.dapp.bash_bin} -ec '#{dimg.dapp.install_bin} -D #{credentials} \"{}\" " \
-              "\"#{File.join(to, '$(echo "{}" | ' \
-              "#{dimg.dapp.sed_bin} -e \"s/^#{from_.gsub('/', '\\/')}\\///g\")")}\"' \\; ;" \
-              'fi'
-            end
+            ''.tap do |cmd|
+              cmd << "#{dimg.dapp.mkdir_bin} -p #{File.dirname(to)}"
+              cmd << ' && '
+              cmd << dimg.dapp.rsync_bin
+              cmd << ' --archive --links --inplace'
+              cmd << " --chown=#{owner}:#{group}" if owner or group
 
-            commands = []
-            commands << [dimg.dapp.install_bin, credentials, '-d', to].join(' ')
-            commands.concat(include_paths.empty? ? Array(copy_files.call(from)) : include_paths.map { |path| copy_files.call(from, path) })
-            commands << "#{dimg.dapp.find_bin} #{to} -type d -exec " \
-                        "#{dimg.dapp.bash_bin} -ec '#{dimg.dapp.install_bin} -d #{credentials} {}' \\;"
-            commands.join(' && ')
-          end
-          # rubocop:enable Metrics/ParameterLists
+              if include_paths.any?
+                # Если указали include_paths — это означает, что надо копировать
+                # только указанные пути. Поэтому exclude_paths в приоритете, т.к. в данном режиме
+                # exclude_paths может относится только к путям, указанным в include_paths.
+                # При этом случай, когда в include_paths указали более специальный путь, чем в exclude_paths,
+                # будет обрабатываться в пользу exclude, этот путь не скопируется.
+                exclude_paths.each do |p|
+                  cmd << " --filter='-/ #{File.join(from, p)}'"
+                end
+
+                include_paths.each do |p|
+                  target_path = File.join(from, p)
 
-          def find_command_excludes(from, exclude_paths)
-            exclude_paths.map { |path| "-not \\( -path #{File.join(from, path)} -prune \\)" }
+                  # Генерируем разрешающее правило для каждого элемента пути
+                  Pathname.new(target_path).descend do |path_part|
+                    cmd << " --filter='+/ #{path_part}'"
+                  end
+
+                  # * На данный момент не знаем директорию или файл имел в виду пользователь,
+                  #   поэтому подставляем фильтры для обоих возможных случаев.
+                  # * Автоматом подставляем паттерн ** для включения файлов, содержащихся в
+                  #   директории, которую пользователь указал в include_paths.
+                  cmd << " --filter='+/ #{target_path}'"
+                  cmd << " --filter='+/ #{File.join(target_path, '**')}'"
+                end
+
+                # Все что не подошло по include — исключается
+                cmd << " --filter='-/ #{File.join(from, '**')}'"
+              else
+                exclude_paths.each do |p|
+                  cmd << " --filter='-/ #{File.join(from, p)}'"
+                end
+              end
+
+              # Слэш после from — это инструкция rsync'у для копирования
+              # содержимого директории from, а не самой директории.
+              cmd << " $(if [ -d #{from} ] ; then echo #{from}/ ; else echo #{from} ; fi) #{to}"
+            end
           end
+          # rubocop:enable Metrics/ParameterLists
         end # ArtifactDefault
       end # Stage
     end # Build

data/lib/dapp/dimg/build/stage/base.rb

@@ -17,6 +17,39 @@ module Dapp
             @next_stage.prev_stage = self
           end
 
+          def get_ruby2go_state_hash
+            {}.tap {|hash|
+              # NOTICE: Delete this when stage has been moved to go.
+              # NOTICE: This is data for build.StubStage and build.StubImage.
+
+              hash["Image"] = {
+                "Labels" => image.labels
+                  .map{|k,v| [k.to_s, v.to_s]}
+                  .to_h,
+                "ServiceChangeLabels" => image.send(:service_change_options)
+                  .fetch(:label, {})
+                  .map{|k,v| [k.to_s, v.to_s]}
+                  .to_h,
+              }
+
+              if prev_stage
+                hash["PrevStage"] = prev_stage.get_ruby2go_state_hash
+              end
+            }
+          end
+
+          def set_ruby2go_state_hash(hash)
+            if prev_stage
+              prev_stage.set_ruby2go_state_hash(hash["PrevStage"])
+            end
+
+            # NOTICE: This is data from build.StubImage.
+
+            hash["Image"]["ServiceChangeLabels"].each do |k, v|
+              image.add_service_change_label(k.to_sym => v)
+            end
+          end
+
           # rubocop:disable Metrics/PerceivedComplexity
           def build_lock!
             return yield if dimg.dapp.dry_run?
@@ -27,11 +60,14 @@ module Dapp
               no_lock = false
 
               dimg.dapp.lock("#{dimg.dapp.name}.image.#{image.name}") do
-                image.cache_reset
+                image.reset_image_inspect
 
                 if image_should_be_locked?
                   yield
                 else
+                  stage = self
+                  stage.renew until (stage = stage.next_stage).nil?
+
                   no_lock = true
                 end
               end
@@ -47,17 +83,11 @@ module Dapp
           end
           # rubocop:enable Metrics/MethodLength, Metrics/PerceivedComplexity
 
-          # rubocop:disable Metrics/CyclomaticComplexity
           def build!
-            return if build_should_be_skipped?
             prev_stage.build! if prev_stage
-            if image_should_be_build?
-              prepare_image if !image.built? && !should_be_not_present? || should_be_introspected?
-              log_image_build(&method(:image_build))
-            end
-            dimg.introspect_image!(image: image.built_id, options: image.send(:prepared_options)) if should_be_introspected?
+            renew             if should_be_renewed?
+            image_build
           end
-          # rubocop:enable Metrics/CyclomaticComplexity
 
           def save_in_cache!
             prev_stage.save_in_cache! if prev_stage
@@ -77,41 +107,78 @@ module Dapp
 
           def prepare_image
             return if dimg.dapp.dry_run?
-            image.add_volumes_from dimg.dapp.base_container
 
             image_add_mounts
 
             image.add_service_change_label dapp: dimg.stage_dapp_label
             image.add_service_change_label 'dapp-version'.to_sym => ::Dapp::VERSION
             image.add_service_change_label 'dapp-cache-version'.to_sym => ::Dapp::BUILD_CACHE_VERSION
+            image.add_service_change_label 'dapp-dimg'.to_sym => false
             image.add_service_change_label 'dapp-dev-mode'.to_sym => true if dimg.dev_mode?
 
             if dimg.dapp.ssh_auth_sock
               image.add_volume "#{dimg.dapp.ssh_auth_sock}:/tmp/dapp-ssh-agent"
-              image.add_env 'SSH_AUTH_SOCK', '/tmp/dapp-ssh-agent'
+              image.add_env SSH_AUTH_SOCK: '/tmp/dapp-ssh-agent'
             end
 
             yield if block_given?
           end
 
           def image_add_mounts
+            image_add_service_mounts
+            image_add_custom_mounts
+
+            image_add_mounts_labels
+          end
+
+          def image_add_service_mounts
             [:tmp_dir, :build_dir].each do |type|
               next if (mounts = adding_mounts_by_type(type)).empty?
 
               mounts.each do |path|
                 absolute_path = File.expand_path(File.join('/', path))
-                tmp_path = dimg.send(type, 'mount', absolute_path[1..-1]).tap(&:mkpath)
+                tmp_path = dimg.send(type, 'mount', dimg.dapp.consistent_uniq_slugify(absolute_path)).tap(&:mkpath)
                 image.add_volume "#{tmp_path}:#{absolute_path}"
               end
+            end
+          end
+
+          def image_add_custom_mounts
+            adding_custom_dir_mounts.each do |from, to_pathes|
+              FileUtils.mkdir_p(from) unless File.exist?(from)
+              to_pathes.tap(&:uniq!).map { |to_path| image.add_volume "#{from}:#{to_path}" }
+              image.add_service_change_label :"dapp-mount-custom-dir-#{from.gsub('/', '--')}" => to_pathes.join(';')
+            end
+          end
 
+          def image_add_mounts_labels
+            [:tmp_dir, :build_dir].each do |type|
+              next if (mounts = adding_mounts_by_type(type)).empty?
               image.add_service_change_label :"dapp-mount-#{type.to_s.tr('_', '-')}" => mounts.join(';')
             end
+
+            adding_custom_dir_mounts.each do |from, to_pathes|
+              image.add_service_change_label :"dapp-mount-custom-dir-#{from.gsub('/', '--')}" => to_pathes.join(';')
+            end
           end
 
           def adding_mounts_by_type(type)
             (config_mounts_by_type(type) + labels_mounts_by_type(type)).uniq
           end
 
+          def adding_custom_dir_mounts
+            config_custom_dir_mounts.in_depth_merge(labels_custom_dir_mounts)
+          end
+
+          def config_custom_dir_mounts
+            dimg.config._custom_dir_mount.reduce({}) do |mounts, mount|
+              from_path = File.expand_path(mount._from)
+              mounts[from_path] ||= []
+              mounts[from_path] << mount._to
+              mounts
+            end
+          end
+
           def config_mounts_by_type(type)
             dimg.config.public_send("_#{type}_mount").map(&:_to)
           end
@@ -120,8 +187,11 @@ module Dapp
             from_image.labels.select { |l, _| l == "dapp-mount-#{type.to_s.tr('_', '-')}" }.map { |_, value| value.split(';') }.flatten
           end
 
-          def image_should_be_build?
-            (!empty? && !image.built?) || dimg.dapp.log_verbose?
+          def labels_custom_dir_mounts
+            from_image.labels.map do |label, value|
+              next unless label =~ /dapp-mount-custom-dir-(?<from>.+)/
+              [File.expand_path(Regexp.last_match(:from).gsub('--', '/')), value.split(';')]
+            end.compact.to_h
           end
 
           def empty?
@@ -133,18 +203,16 @@ module Dapp
           end
 
           def signature
-            @signature ||= begin
-              if empty?
-                prev_stage.signature
-              else
-                args = []
-                args << prev_stage.signature unless prev_stage.nil?
-                args << dimg.build_cache_version
-                args << builder_checksum
-                args.concat(dependencies.flatten)
+            if empty?
+              prev_stage.signature
+            else
+              args = []
+              args << prev_stage.signature unless prev_stage.nil?
+              args << dimg.build_cache_version
+              args << builder_checksum
+              args.concat(dependencies.flatten)
 
-                hashsum args
-              end
+              hashsum args
             end
           end
 
@@ -156,25 +224,29 @@ module Dapp
           end
 
           def git_artifacts_dependencies
-            local_git_artifacts_dependencies + remote_git_artifacts_dependencies
+            dimg.git_artifacts.map { |git_artifact| git_artifact.stage_dependencies_checksum(self) }
           end
 
-          def local_git_artifacts_dependencies
-            default_git_artifacts_dependencies(dimg.local_git_artifacts)
-          end
-
-          def remote_git_artifacts_dependencies
-            default_git_artifacts_dependencies(dimg.remote_git_artifacts)
-          end
-
-          def default_git_artifacts_dependencies(git_artifacts)
-            git_artifacts.map { |git_artifact| git_artifact.stage_dependencies_checksums(self) }
+          def layer_commit(git_artifact)
+            commits[git_artifact] ||= begin
+              if image.built?
+                image.labels[dimg.dapp.dimgstage_g_a_commit_label(git_artifact.paramshash)]
+              elsif g_a_stage? && !empty?
+                git_artifact.latest_commit
+              elsif prev_stage
+                prev_stage.layer_commit(git_artifact)
+              end
+            end
           end
 
           def dependencies
             []
           end
 
+          def dependencies_discard
+            @dependencies = nil
+          end
+
           def artifact?
             false
           end
@@ -183,28 +255,52 @@ module Dapp
             false
           end
 
-          def prev_g_a_stage
-            stage = self
-            until (stage = stage.prev_stage).nil?
-              return stage if stage.g_a_stage?
-            end
+          protected
+
+          def renew
+            commits_discard
+            image_untag! if image_should_be_untagged?
+            image_reset
           end
 
-          def next_g_a_stage
-            stage = self
-            until (stage = stage.next_stage).nil?
-              return stage if stage.g_a_stage?
-            end
+          def image_reset
+            Image::Stage.image_reset(image_name)
+            @image = nil
           end
 
-          protected
+          def image_untag!
+            return if dimg.dapp.dry_run?
+            image.untag!
+          end
 
           def image_build
-            image.build!
+            prepare_image if image_should_be_prepared?
+
+            introspect_image_before_build if image_should_be_introspected_before_build?
+
+            log_image_build do
+              dimg.dapp.log_process(log_name,
+                                    process: dimg.dapp.t(code: 'status.process.building'),
+                                    short: should_not_be_detailed?) { image.build! }
+            end unless empty?
+
+            introspect_image_after_build if image_should_be_introspected_after_build?
+          end
+
+          def introspect_image_before_build
+            introspect_image_default(from_image)
           end
 
-          def build_should_be_skipped?
-            image.built? && !dimg.dapp.log_verbose? && !should_be_introspected?
+          def introspect_image_after_build
+            introspect_image_default(image)
+          end
+
+          def introspect_image_default(introspected_image)
+            if introspected_image.built?
+              introspected_image.introspect!
+            else
+              dimg.dapp.log_warning(desc: { code: :introspect_image_impossible, data: { name: name } })
+            end
           end
 
           def should_be_tagged?
@@ -212,12 +308,43 @@ module Dapp
           end
 
           def image_should_be_locked?
-            !(empty? || image.tagged? || should_be_not_present?)
+            !(empty? || image.built? || should_be_not_present?)
+          end
+
+          def image_should_be_prepared?
+            (!image.built? && !should_be_not_present? || image_should_be_introspected? && image.tagged?) && !dimg.dapp.dry_run?
+          end
+
+          def should_be_renewed?
+            current_or_related_image_should_be_untagged?
+          end
+
+          def image_should_be_untagged?
+            image.tagged? && current_or_related_image_should_be_untagged?
+          end
+
+          def current_or_related_image_should_be_untagged?
+            @current_or_related_image_should_be_untagged ||= begin
+              image_should_be_untagged_condition || begin
+                if prev_stage.nil?
+                  false
+                else
+                  prev_stage.current_or_related_image_should_be_untagged?
+                end
+              end
+            end
+          end
+
+          def image_should_be_untagged_condition
+            return false unless image.tagged?
+            dimg.git_artifacts.any? do |git_artifact|
+              !git_artifact.repo.commit_exists? layer_commit(git_artifact)
+            end
           end
 
           def should_be_not_present?
             return false if next_stage.nil?
-            next_stage.image.tagged? || next_stage.should_be_not_present?
+            !current_or_related_image_should_be_untagged? && (next_stage.image.tagged? && !next_stage.image_should_be_untagged_condition || next_stage.should_be_not_present?)
           end
 
           def image_name
@@ -237,6 +364,14 @@ module Dapp
               end
             end
           end
+
+          def commits_discard
+            @commits = nil
+          end
+
+          def commits
+            @commits ||= {}
+          end
         end # Base
       end # Stage
     end # Build

data/lib/dapp/dimg/build/stage/before_install.rb

@@ -2,30 +2,19 @@ module Dapp
   module Dimg
     module Build
       module Stage
-        class BeforeInstall < Base
+        class BeforeInstall < Instructions
           def initialize(dimg, next_stage)
             @prev_stage = From.new(dimg, self)
             super
           end
 
-          def empty?
-            super && !dimg.builder.before_install?
-          end
-
-          def context
+          def dependencies
             [builder_checksum]
           end
 
-          def builder_checksum
-            dimg.builder.before_install_checksum
-          end
-
-          def prepare_image
-            super
-            dimg.builder.before_install(image)
+          def image_should_be_untagged_condition
+            false
           end
-
-          alias dependencies context
         end # BeforeInstall
       end # Stage
     end # Build

data/lib/dapp/dimg/build/stage/before_install_artifact.rb

@@ -7,6 +7,10 @@ module Dapp
             @prev_stage = BeforeInstall.new(dimg, self)
             super
           end
+
+          def image_should_be_untagged_condition
+            false
+          end
         end # BeforeInstallArtifact
       end # Stage
     end # Build

data/lib/dapp/dimg/build/stage/before_setup.rb

@@ -2,30 +2,11 @@ module Dapp
   module Dimg
     module Build
       module Stage
-        class BeforeSetup < Base
-          include Mod::GitArtifactsDependencies
-
+        class BeforeSetup < Instructions
           def initialize(dimg, next_stage)
             @prev_stage = AfterInstallArtifact.new(dimg, self)
             super
           end
-
-          def empty?
-            !dimg.builder.before_setup?
-          end
-
-          def context
-            [git_artifacts_dependencies, builder_checksum]
-          end
-
-          def builder_checksum
-            dimg.builder.before_setup_checksum
-          end
-
-          def prepare_image
-            super
-            dimg.builder.before_setup(image)
-          end
         end # BeforeSetup
       end # Stage
     end # Build