dapp 0.7.36 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: adca48a7c835ec4933a2b8bb3c1f16de64a5221a
-  data.tar.gz: 724dfed9ee30db37ea650ec775908cbe76753ca8
+  metadata.gz: 92eea836b5eb9b9d0b542583679e8138609e37ad
+  data.tar.gz: f9b18f1faf86f4b0524e132d85783df1fddb905b
 SHA512:
-  metadata.gz: f43599e74343261ed0dbb5f8664f3145d944b247bb04088247cab42ae3c4b1e96e647b4c0832ecd2ead546276a5e4baca5df2850252ea543c9f2ee52420cb740
-  data.tar.gz: fcd6dd54e7d1e3f4718dea2b22fa77b5e1408ddea2bb66c1cf1b20f2bf53a6fa5388a0dcd29ce9823ad88b0fcd12aea3ed1d4fe5e2e1e609d50074f86329fa0a
+  metadata.gz: add25471d85a39f355507fb1a19d77d6673020c327997ee0a7ca89a2eb33141d28a8e41ac07e36e5f5d7fb9d4fbdef5a5fa32b69344f57541f16f3da78360ee5
+  data.tar.gz: d9cc3f25bde5fba86ec1f7b7e511ca2123a85cce668fb3049dad6f039c505661f5679e3e832541f057e0b71b8371dbdd08f3a6adbaf7699a0d8d15efa5851914

data/config/en/common.yml

@@ -15,8 +15,7 @@ en:
       artifact_building: "building artifact `%{name}`"
       git_artifact_clone: "cloning remote git_artifact `%{name}`"
       git_artifact_fetch: "fetching remote git_artifact `%{name}`"
-      berks_vendor: 'vendoring cookbooks'
-      chef_cookbooks_stage_berks_vendor: 'vendoring cookbooks for chef cookbooks stage'
+      vendoring_builder_cookbooks: 'vendoring builder cookbooks'
       waiting_resouce_lock: "waiting for locked resource `%{name}`"
       gitartifact_container_creating: 'creating dappdeps/gitartifact container'
       base_container_creating: 'creating dappdeps/base container'
@@ -76,7 +75,6 @@ en:
         g_a_pre_setup_patch_dependencies: 'Git artifacts dependencies'
         g_a_pre_setup_patch: 'Git artifacts: apply patches (before setup)'
         setup: 'Setup'
-        chef_cookbooks: 'Сhef сookbooks'
         g_a_post_setup_patch_dependencies: 'Git artifacts dependencies'
         g_a_post_setup_patch: 'Git artifacts: apply patches (after setup)'
       after_setup_artifact: 'After setup artifact'

data/config/en/net_status.yml

@@ -17,7 +17,7 @@ en:
       image_not_exist: "Image `%{name}` not exist!"
       built_id_not_defined: '`from.built_id` not defined!'
       from_image_not_found: 'Image `%{name}` not found!'
-      unsupported_docker_image_size_format: "Unsupported docker image size format `%{value}`"
+      unsupported_patch_format: "Unsupported patch format:\n\n%{patch}"
     project:
       command_unexpected_dimgs_number: "Command can process only one dimg!\nAmbiguous dimg pattern: `%{dimgs_names}`!"
       no_such_dimg: "No such dimg: `%{dimgs_patterns}`!"
@@ -45,7 +45,7 @@ en:
       stage_artifact_not_associated: "Artifact not associated with any stage: expected `before` or `after` attribute!"
       stage_artifact_double_associate: "Cannot use `%{stage}` stage for artifact, already used in `%{conflict_stage}` stage!"
       stage_artifact_not_supported_associated_stage: "Bad artifact stage `%{stage}`!"
-      git_artifact_remote_branch_with_commit: "Remote git repo: use `commit` or `branch` directive!"
+      git_artifact_remote_unsupported_protocol: "Remote git repo `%{url}`: unsupported protocol!"
       artifact_conflict: "Conflict between artifacts paths!"
       scratch_unsupported_directive: "Scratch dimg has unsupported directive `%{directive}`!"
       scratch_artifact_required: "Scratch dimg without artifacts!"
@@ -53,10 +53,9 @@ en:
       scratch_artifact_docker_from: "Scratch artifact must have directive `docker.from`!"
     chef:
       stage_path_overlap: "Cannot install `%{cookbook}` cookbook's path %{from} into %{to}: already exists"
-      mdapp_dependency_in_metadata_forbidden: "Using mdapp as dependency in metadata.rb forbidden: detected `%{dependency}`"
-      local_cookbook_path_not_found: "Dapp cookbook directory not found at %{path}"
-      cookbook_berksfile_not_found: "Dapp cookbook Berksfile not found at %{path}"
-      cookbook_metadata_not_found: "Dapp cookbook metadata.rb file not found at %{path}"
+      builder_cookbook_not_found: "Dapp cookbook directory not found at %{path}"
+      builder_cookbook_berksfile_not_found: "Dapp local cookbook Berksfile not found at %{path}"
+      builder_cookbook_metadata_not_found: "Dapp local cookbook metadata.rb file not found at %{path}"
       cookbook_not_specified_in_berksfile: "Dapp cookbook `%{name}` not specified in Berksfile at %{path}"
       berksfile_absolute_path_forbidden: "Absolute paths in Berksfile are not allowed (cookbook `%{cookbook}`, path: `%{path}`)"
     registry:
@@ -67,8 +66,7 @@ en:
     rugged:
       rugged_remote_error: "Remote git repo `%{url}`: `%{message}`!"
       local_git_repository_does_not_exist: "Local git repo: doesn't exist!"
-      commit_not_found_in_local_git_repository: "Local git repo: commit `%{commit}` not found!\nIf commit has been rebased: run command `dapp stages cleanup local --improper-git-commit`!"
-      commit_not_found_in_remote_git_repository: "Remote git repo `%{url}`: commit `%{commit}` not found!\nIf commit has been rebased: run command `dapp stages cleanup local --improper-git-commit`!"
-      branch_not_exist_in_remote_git_repository: "Remote git repo `%{url}`: branch `%{branch}` not exist!"
+      commit_not_found_in_local_git_repository: "Local git repo: commit `%{commit}` not found (run command `dapp stages cleanup local --improper-git-commit`)!"
+      commit_not_found_in_remote_git_repository: "Remote git repo `%{url}`: commit `%{commit}` not found (run command `dapp stages cleanup local --improper-git-commit`)!"
     lock:
       timeout: "Could not obtain lock for `%{name}` within %{timeout} seconds"

data/lib/dapp.rb

@@ -16,6 +16,7 @@ require 'i18n'
 require 'paint'
 require 'inifile'
 require 'rugged'
+require 'rubygems/package'
 
 require 'net_status'
 
@@ -36,6 +37,7 @@ require 'dapp/error/project'
 require 'dapp/error/shellout'
 require 'dapp/error/registry'
 require 'dapp/error/rugged'
+require 'dapp/error/tar_writer'
 require 'dapp/exception/base'
 require 'dapp/exception/introspect_image'
 require 'dapp/exception/registry'
@@ -88,6 +90,7 @@ require 'dapp/builder/chef'
 require 'dapp/builder/chef/error'
 require 'dapp/builder/chef/cookbook_metadata'
 require 'dapp/builder/chef/berksfile'
+require 'dapp/builder/chef/cookbook'
 require 'dapp/builder/shell'
 require 'dapp/builder/none'
 require 'dapp/build/stage/mod/logging'
@@ -114,7 +117,6 @@ require 'dapp/build/stage/before_setup_artifact'
 require 'dapp/build/stage/setup/ga_pre_setup_patch_dependencies'
 require 'dapp/build/stage/setup/ga_pre_setup_patch'
 require 'dapp/build/stage/setup/setup'
-require 'dapp/build/stage/setup/chef_cookbooks'
 require 'dapp/build/stage/setup/ga_post_setup_patch_dependencies'
 require 'dapp/build/stage/setup/ga_post_setup_patch'
 require 'dapp/build/stage/after_setup_artifact'

data/lib/dapp/artifact.rb

@@ -1,6 +1,13 @@
 module Dapp
   # Artifact
   class Artifact < Dimg
+    def after_stages_build!
+    end
+
+    def stage_should_be_introspected?(name)
+      project.cli_options[:introspect_artifact_stage] == name
+    end
+
     def artifact?
       true
     end
@@ -9,8 +16,6 @@ module Dapp
       false
     end
 
-    protected
-
     def last_stage
       @last_stage ||= Build::Stage::BuildArtifact.new(self)
     end

data/lib/dapp/build/stage/after_setup_artifact.rb

@@ -5,7 +5,7 @@ module Dapp
       class AfterSetupArtifact < ArtifactDefault
         def initialize(dimg, next_stage)
           @prev_stage = if dimg.artifact?
-                          SetupGroup::ChefCookbooks.new(dimg, self)
+                          SetupGroup::Setup.new(dimg, self)
                         else
                           SetupGroup::GAPostSetupPatch.new(dimg, self)
                         end

data/lib/dapp/build/stage/artifact_base.rb

@@ -13,7 +13,7 @@ module Dapp
           artifacts_labels = {}
           artifacts.each do |artifact|
             apply_artifact(artifact, image)
-            artifacts_labels["dapp-artifact-#{artifact[:name]}".to_sym] = artifact[:dimg].send(:last_stage).image.built_id
+            artifacts_labels["dapp-artifact-#{artifact[:name]}".to_sym] = artifact[:dimg].last_stage.image.built_id
           end
           image.add_service_change_label artifacts_labels
         end

data/lib/dapp/build/stage/artifact_default.rb

@@ -18,7 +18,7 @@ module Dapp
           group = artifact[:options][:group]
           to = artifact[:options][:to]
 
-          command = safe_cp(cwd, artifact_dimg.container_tmp_path(artifact_name).to_s, nil, nil, include_paths, exclude_paths)
+          command = safe_cp(cwd, artifact_dimg.container_tmp_path(artifact_name).to_s, Process.uid, Process.gid, include_paths, exclude_paths)
           run_artifact_dimg(artifact_dimg, artifact_name, command)
 
           command = safe_cp(dimg.container_tmp_path('artifact', artifact_name).to_s, to, owner, group, include_paths, exclude_paths)
@@ -31,51 +31,32 @@ module Dapp
 
         # rubocop:disable Metrics/ParameterLists, Metrics/AbcSize, Metrics/MethodLength
         def safe_cp(from, to, owner, group, include_paths = [], exclude_paths = [])
-          ''.tap do |cmd|
-            cmd << dimg.project.rsync_bin
-            cmd << ' --archive --links'
-            cmd << " --chown=#{owner}:#{group}" if owner or group
-
-            if include_paths.any?
-              # Если указали include_paths ­— это означает, что надо копировать
-              # только указанные пути. Поэтому exclude_paths в приоритете, т.к. в данном режиме
-              # exclude_paths может относится только к путям, указанным в include_paths.
-              # При этом случай, когда в include_paths указали более специальный путь, чем в exclude_paths,
-              # будет обрабатываться в пользу exclude, этот путь не скопируется.
-              exclude_paths.each do |p|
-                cmd << " --filter='-/ #{File.join(from, p)}'"
-              end
-
-              include_paths.each do |p|
-                target_path = File.join(from, p)
-
-                # Генерируем разрешающее правило для каждого элемента пути
-                Pathname.new(target_path).descend do |path_part|
-                  cmd << " --filter='+/ #{path_part}'"
-                end
-
-                # * На данный момент не знаем директорию или файл имел в виду пользователь,
-                #   поэтому подставляем фильтры для обоих возможных случаев.
-                # * Автоматом подставляем паттерн ** для включения файлов, содержащихся в
-                #   директории, которую пользователь указал в include_paths.
-                cmd << " --filter='+/ #{target_path}'"
-                cmd << " --filter='+/ #{File.join(target_path, '**')}'"
-              end
-
-              # Все что не подошло по include — исключается
-              cmd << " --filter='-/ #{File.join(from, '**')}'"
-            else
-              exclude_paths.each do |p|
-                cmd << " --filter='-/ #{File.join(from, p)}'"
-              end
-            end
-
-            # Слэш после from — это инструкция rsync'у для копирования
-            # содержимого директории from, а не самой директории.
-            cmd << " #{from}/ #{to}"
+          credentials = ''
+          credentials += "-o #{owner} " if owner
+          credentials += "-g #{group} " if group
+          excludes = find_command_excludes(from, exclude_paths).join(' ')
+
+          copy_files = proc do |from_, path_ = ''|
+            "if [[ -d #{File.join(from_, path_)} ]] || [[ -f #{File.join(from_, path_)} ]]; then " \
+            "#{dimg.project.find_bin} #{File.join(from_, path_)} #{excludes} -type f -exec " \
+            "#{dimg.project.bash_bin} -ec '#{dimg.project.install_bin} -D #{credentials} \"{}\" " \
+            "\"#{File.join(to, "$(echo \"{}\" | " \
+            "#{dimg.project.sed_bin} -e \"s/^#{from_.gsub('/', '\\/')}\\///g\")")}\"' \\; ;" \
+            'fi'
           end
+
+          commands = []
+          commands << [dimg.project.install_bin, credentials, '-d', to].join(' ')
+          commands.concat(include_paths.empty? ? Array(copy_files.call(from)) : include_paths.map { |path| copy_files.call(from, path) })
+          commands << "#{dimg.project.find_bin} #{to} -type d -exec " \
+                      "#{dimg.project.bash_bin} -ec '#{dimg.project.install_bin} -d #{credentials} {}' \\;"
+          commands.join(' && ')
         end
         # rubocop:enable Metrics/ParameterLists, Metrics/AbcSize, Metrics/MethodLength
+
+        def find_command_excludes(from, exclude_paths)
+          exclude_paths.map { |path| "-not \\( -path #{File.join(from, path)} -prune \\)" }
+        end
       end # ArtifactDefault
     end # Stage
   end # Build

data/lib/dapp/build/stage/base.rb

@@ -48,10 +48,10 @@ module Dapp
         # rubocop:enable Metrics/MethodLength, Metrics/PerceivedComplexity
 
         def build!
-          return if should_be_skipped?
+          return if build_should_be_skipped?
           prev_stage.build! if prev_stage
           if image_should_be_build?
-            prepare_image if !image.tagged? && !should_be_not_present?
+            prepare_image if !image.built? && !should_be_not_present?
             log_image_build(&method(:image_build))
           end
           dimg.introspect_image!(image: image.built_id, options: image.send(:prepared_options)) if should_be_introspected?
@@ -68,7 +68,7 @@ module Dapp
             if empty?
               prev_stage.image
             else
-              Image::Stage.new(name: image_name, from: from_image, project: dimg.project)
+              Image::Stage.image_by_name(name: image_name, from: from_image, project: dimg.project)
             end
           end
         end
@@ -76,8 +76,11 @@ module Dapp
         def prepare_image
           return if dimg.project.dry_run?
           image.add_volumes_from dimg.project.base_container
-          image_add_volumes
+
+          image_add_mounts
+
           image.add_service_change_label dapp: dimg.stage_dapp_label
+          image.add_service_change_label 'dapp-version'.to_sym => Dapp::VERSION
           image.add_service_change_label 'dapp-cache-version'.to_sym => Dapp::BUILD_CACHE_VERSION
           image.add_service_change_label 'dapp-dev-mode'.to_sym => true if dimg.dev_mode?
 
@@ -85,32 +88,31 @@ module Dapp
             image.add_volume "#{dimg.project.ssh_auth_sock}:/tmp/dapp-ssh-agent"
             image.add_env 'SSH_AUTH_SOCK', '/tmp/dapp-ssh-agent'
           end
-        end
 
-        def image_add_volumes
-          image_add_tmp_volumes
-          image_add_build_volumes
+          yield if block_given?
         end
 
-        def image_add_tmp_volumes
-          image_add_default_volumes(:tmp_dir)
-        end
+        def image_add_mounts
+          [:tmp_dir, :build_dir].each do |type|
+            next if (mounts = mounts_by_type(type)).empty?
+
+            mounts.each do |path|
+              absolute_path = File.expand_path(File.join('/', path))
+              tmp_path = dimg.send(type, 'mount', absolute_path[1..-1]).tap(&:mkpath)
+              image.add_volume "#{tmp_path}:#{absolute_path}"
+            end
 
-        def image_add_build_volumes
-          image_add_default_volumes(:build_dir)
+            image.add_service_change_label :"dapp-mount-#{type.to_s.tr('_', '-')}" => mounts.join(';')
+          end
         end
 
-        def image_add_default_volumes(type)
+        def mounts_by_type(type)
           (dimg.config.public_send("_#{type}_mount").map(&:_to) +
-            from_image.labels.select { |l, _| l == "dapp-#{type}-dir" }.map { |_, value| value.split(';') }.flatten).each do |path|
-            absolute_path = File.expand_path(File.join('/', path))
-            tmp_path = dimg.send(type, 'mount', absolute_path[1..-1]).tap(&:mkpath)
-            image.add_volume "#{tmp_path}:#{absolute_path}"
-          end
+            from_image.labels.select { |l, _| l == "dapp-mount-#{type.to_s.tr('_', '-')}" }.map { |_, value| value.split(';') }.flatten).uniq
         end
 
         def image_should_be_build?
-          !empty? || dimg.project.log_verbose?
+          (!empty? && !image.built?) || dimg.project.log_verbose?
         end
 
         def empty?
@@ -122,16 +124,18 @@ module Dapp
         end
 
         def signature
-          if empty?
-            prev_stage.signature
-          else
-            args = []
-            args << prev_stage.signature unless prev_stage.nil?
-            args << dimg.build_cache_version
-            args << builder_checksum
-            args.concat(dependencies.flatten)
+          @signature ||= begin
+            if empty?
+              prev_stage.signature
+            else
+              args = []
+              args << prev_stage.signature unless prev_stage.nil?
+              args << dimg.build_cache_version
+              args << builder_checksum
+              args.concat(dependencies.flatten)
 
-            hashsum args
+              hashsum args
+            end
           end
         end
 
@@ -152,8 +156,8 @@ module Dapp
           image.build!
         end
 
-        def should_be_skipped?
-          image.tagged? && !dimg.project.log_verbose? && !should_be_introspected?
+        def build_should_be_skipped?
+          image.built? && !dimg.project.log_verbose? && !should_be_introspected?
         end
 
         def should_be_tagged?

data/lib/dapp/build/stage/build_artifact.rb

@@ -28,7 +28,7 @@ module Dapp
         private
 
         def artifact_dependencies_files_checksum
-          @artifact_files_checksum ||= dependencies_files_checksum(dimg.config._artifact_dependencies)
+          dependencies_files_checksum(dimg.config._artifact_dependencies)
         end
       end # BuildArtifact
     end # Stage

data/lib/dapp/build/stage/from.rb

@@ -26,7 +26,7 @@ module Dapp
         end
 
         def from_image
-          @from_image ||= Image::Stage.new(name: from_image_name, project: dimg.project)
+          @from_image ||= Image::Stage.image_by_name(name: from_image_name, project: dimg.project)
         end
       end # Prepare
     end # Stage

data/lib/dapp/build/stage/ga_archive_dependencies.rb

@@ -3,13 +3,22 @@ module Dapp
     module Stage
       # GAArchiveDependencies
       class GAArchiveDependencies < GADependenciesBase
+        RESET_COMMIT_MESSAGES = ['[dapp reset]', '[reset dapp]']
+
         def initialize(dimg, next_stage)
           @prev_stage = BeforeInstallArtifact.new(dimg, self)
           super
         end
 
         def dependencies
-          [dimg.git_artifacts.map(&:paramshash).join]
+          [dimg.git_artifacts.map(&:paramshash).join, reset_commits.sort.uniq.compact]
+        end
+
+        protected
+
+        def reset_commits
+          regex = Regexp.union(RESET_COMMIT_MESSAGES)
+          dimg.git_artifacts.map { |git_artifact| git_artifact.repo.find_commit_id_by_message(regex) }
         end
       end # GAArchiveDependencies
     end # Stage

data/lib/dapp/build/stage/ga_artifact_patch.rb

@@ -4,7 +4,7 @@ module Dapp
       # GAArtifactPatch
       class GAArtifactPatch < GALatestPatch
         def initialize(dimg, next_stage)
-          @prev_stage = SetupGroup::ChefCookbooks.new(dimg, self)
+          @prev_stage = SetupGroup::Setup.new(dimg, self)
           super
         end
 
@@ -13,7 +13,7 @@ module Dapp
         end
 
         def prev_g_a_stage
-          super.prev_stage.prev_stage # GAPreSetupPatch
+          super.prev_stage # GAPreSetupPatch
         end
       end # GAArtifactPatch
     end # Stage

data/lib/dapp/build/stage/ga_base.rb

@@ -18,14 +18,27 @@ module Dapp
         end
 
         def prepare_image
-          super
-          image.add_volumes_from dimg.project.gitartifact_container
+          super do
+            image.add_volumes_from dimg.project.gitartifact_container
+            image.add_volume "#{dimg.tmp_path('archives')}:#{dimg.container_tmp_path('archives')}:ro"
+            image.add_volume "#{dimg.tmp_path('patches')}:#{dimg.container_tmp_path('patches')}:ro"
 
-          dimg.git_artifacts.each do |git_artifact|
-            image.add_volume "#{git_artifact.repo.path}:#{git_artifact.repo.container_path}:ro"
-            image.add_command git_artifact.send(apply_command_method, self)
+            prepare_local_git_artifacts_command
+            prepare_remote_git_artifacts_command
           end
         end
+        
+        def prepare_local_git_artifacts_command
+          prepare_base_git_artifacts_command(dimg.local_git_artifacts)
+        end
+
+        def prepare_remote_git_artifacts_command
+          prepare_base_git_artifacts_command(dimg.remote_git_artifacts)
+        end
+
+        def prepare_base_git_artifacts_command(git_artifacts)
+          git_artifacts.each { |git_artifact| image.add_command git_artifact.send(apply_command_method, self) }
+        end
 
         def empty?
           dependencies_stage.empty?

data/lib/dapp/build/stage/ga_dependencies_base.rb

@@ -11,7 +11,7 @@ module Dapp
         end
 
         def empty?
-          dimg.git_artifacts.empty? ? true : false
+          dimg.git_artifacts.empty? || super
         end
       end # GADependenciesBase
     end # Stage