dapp 0.7.36 → 0.8.0

data/lib/dapp/builder/none.rb

@@ -34,12 +34,6 @@ module Dapp
 
       def build_artifact_checksum
       end
-
-      def chef_cookbooks(_image)
-      end
-
-      def chef_cookbooks_checksum
-      end
     end # Base
   end # Builder
 end # Dapp

data/lib/dapp/cli/build.rb

@@ -40,7 +40,14 @@ BANNER
              proc: proc { |v| v.to_sym },
              in: [nil, :from, :before_install, :before_install_artifact, :g_a_archive, :g_a_pre_install_patch, :install,
                   :g_a_post_install_patch, :after_install_artifact, :before_setup, :before_setup_artifact, :g_a_pre_setup_patch,
-                  :chef_cookbooks, :setup, :g_a_post_setup_patch, :after_setup_artifact, :g_a_latest_patch, :docker_instructions]
+                  :setup, :g_a_post_setup_patch, :after_setup_artifact, :g_a_latest_patch, :docker_instructions]
+
+      option :introspect_artifact_stage,
+             long: '--introspect-artifact-stage STAGE',
+             proc: proc { |v| v.to_sym },
+             in: [nil, :from, :before_install, :before_install_artifact, :g_a_archive, :g_a_pre_install_patch, :install,
+                  :g_a_post_install_patch, :after_install_artifact, :before_setup, :before_setup_artifact, :g_a_pre_setup_patch,
+                  :setup, :after_setup_artifact, :g_a_artifact_patch, :build_artifact]
 
       option :ssh_key,
              long: '--ssh-key SSH_KEY',

data/lib/dapp/cli/stage_image.rb

@@ -18,7 +18,7 @@ BANNER
              default: :docker_instructions,
              in: [:from, :before_install, :before_install_artifact, :g_a_archive, :g_a_pre_install_patch, :install,
                   :g_a_post_install_patch, :after_install_artifact, :before_setup, :before_setup_artifact, :g_a_pre_setup_patch,
-                  :chef_cookbooks, :setup, :g_a_post_setup_patch, :after_setup_artifact, :g_a_latest_patch, :docker_instructions]
+                  :setup, :g_a_post_setup_patch, :after_setup_artifact, :g_a_latest_patch, :docker_instructions]
     end
   end
 end

data/lib/dapp/config/directive/base.rb

@@ -14,9 +14,7 @@ module Dapp
         end
 
         def path_format(path)
-          path = path.to_s
-          path = path.chomp('/') unless path == '/'
-          path
+          path.to_s.chomp('/')
         end
       end
     end

data/lib/dapp/config/directive/git_artifact_remote.rb

@@ -6,6 +6,8 @@ module Dapp
         attr_reader :_url, :_name, :_branch, :_commit
 
         def initialize(url, **kwargs, &blk)
+          raise Error::Config, code: :git_artifact_remote_unsupported_protocol, data: { url: url } unless %w(http https).include? URI(url.to_s).scheme
+
           @_url  = url
           @_name = url.gsub(%r{.*?([^\/ ]+)\.git}, '\\1')
 
@@ -26,7 +28,7 @@ module Dapp
           attr_accessor :_url, :_name, :_branch, :_commit
 
           def _artifact_options
-            super.merge(name: _name, branch: _branch, commit: _commit)
+            super.merge(name: _name, branch: _branch)
           end
 
           protected
@@ -38,11 +40,6 @@ module Dapp
           def commit(value)
             @_commit = value
           end
-
-          def validate!
-            super
-            raise Error::Config, code: :git_artifact_remote_branch_with_commit if !_branch.nil? && !_commit.nil?
-          end
         end
 
         protected

data/lib/dapp/dimg.rb

@@ -31,7 +31,7 @@ module Dapp
               builder.before_build_check
               last_stage.build!
             ensure
-              last_stage.save_in_cache! if last_stage.image.built? || dev_mode?
+              after_stages_build!
             end
           end
         end
@@ -40,6 +40,12 @@ module Dapp
       cleanup_tmp
     end
 
+    def after_stages_build!
+      return unless last_stage.image.built? || dev_mode?
+      last_stage.save_in_cache!
+      artifacts.each { |artifact| artifact.last_stage.save_in_cache! }
+    end
+
     def tag!(tag)
       project.lock("#{project.name}.images", readonly: true) do
         dimg_name = config._name
@@ -116,7 +122,7 @@ module Dapp
     end
 
     def run(docker_options, command)
-      cmd = "docker run #{[docker_options, last_stage.image.name, command].flatten.compact.join(' ')}"
+      cmd = "docker run #{[docker_options, last_stage.image.built_id, command].flatten.compact.join(' ')}"
       if project.dry_run?
         project.log(cmd)
       else
@@ -158,21 +164,14 @@ module Dapp
     end
 
     def cleanup_tmp
-      # В tmp-директории могли остаться файлы, владельцами которых мы не являемся.
-      # Такие файлы могут попасть туда при экспорте файлов артефакта.
-      # Чтобы от них избавиться — запускаем docker-контейнер под root-пользователем
-      # и удаляем примонтированную tmp-директорию.
-      cmd = "".tap do |cmd|
-        cmd << "docker run --rm"
-        cmd << " --volume #{tmp_base_dir}:#{tmp_base_dir}"
-        cmd << " ubuntu:16.04"
-        cmd << " rm -rf #{tmp_path}"
-      end
-      project.shellout! cmd
-
+      FileUtils.rm_rf(tmp_path)
       artifacts.each(&:cleanup_tmp)
     end
 
+    def stage_should_be_introspected?(name)
+      project.cli_options[:introspect_stage] == name
+    end
+
     protected
 
     def should_be_built?

data/lib/dapp/dimg/path.rb

@@ -7,16 +7,8 @@ module Dapp
         make_path(project.path, *path).expand_path
       end
 
-      def local_cookbook_path(*path)
-        make_path(project.local_cookbook_path, *path)
-      end
-
-      def tmp_base_dir
-        File.expand_path(project.cli_options[:tmp_dir_prefix] || '/tmp')
-      end
-
       def tmp_path(*path)
-        @tmp_path ||= Dir.mktmpdir('dapp-', tmp_base_dir)
+        @tmp_path ||= Dir.mktmpdir('dapp-', project.cli_options[:tmp_dir_prefix] || '/tmp')
         make_path(@tmp_path, *path).expand_path.tap { |p| p.parent.mkpath }
       end
 

data/lib/dapp/error/tar_writer.rb

@@ -0,0 +1,6 @@
+module Dapp
+  module Error
+    # TarWriter
+    class TarWriter < Base; end
+  end
+end

data/lib/dapp/git_artifact.rb

@@ -14,7 +14,14 @@ module Dapp
       @commit = commit
 
       @to = to
-      @cwd = (cwd.nil? || cwd.empty? || cwd == '/') ? '' : File.expand_path(File.join('/', cwd, '/'))[1..-1]
+      @cwd = begin
+        if cwd.nil? || cwd.empty?
+          ''
+        else
+          cwd = File.expand_path(File.join('/', cwd))[1..-1] # must be relative
+          "#{cwd.chomp('/')}/"
+        end
+      end
       @include_paths = include_paths
       @exclude_paths = exclude_paths
       @owner = owner
@@ -27,39 +34,16 @@ module Dapp
 
       [].tap do |commands|
         commands << "#{repo.dimg.project.install_bin} #{credentials.join(' ')} -d #{to}"
-
-        if include_paths_or_cwd.empty? || include_paths_or_cwd.any? { |path| file_exist_in_repo?(stage.layer_commit(self), path) }
-          commands << ["#{repo.dimg.project.git_bin} --git-dir=#{repo.container_path} archive #{stage.layer_commit(self)}:#{cwd} --prefix=/ #{include_paths.join(' ')}",
-                       "#{sudo}#{repo.dimg.project.tar_bin} -x -C #{to} #{archive_command_excludes.join(' ')}"].join(' | ')
-        end
+        commands << "#{sudo}#{repo.dimg.project.tar_bin} -xf #{archive_file(stage.layer_commit(self))} -C #{to}" if any_changes?(nil, stage.layer_commit(self))
       end
     end
 
     def apply_patch_command(stage)
-      current_commit = stage.layer_commit(self)
-      prev_commit = stage.prev_g_a_stage.layer_commit(self)
-
-      if any_changes?(prev_commit, current_commit)
-        [["#{repo.dimg.project.git_bin} --git-dir=#{repo.container_path} #{diff_command(prev_commit, current_commit)}",
-          "#{sudo}#{repo.dimg.project.git_bin} apply --whitespace=nowarn --directory=#{to} #{patch_command_excludes.join(' ')} --unsafe-paths"].join(' | ')]
-      else
-        []
-      end
+      patch_command(stage.prev_g_a_stage.layer_commit(self), stage.layer_commit(self))
     end
 
-    def archive_command_excludes
-      exclude_paths.map { |path| %(--exclude=#{File.join('/', path)}) }
-    end
-
-    def patch_command_excludes
-      exclude_paths.map do |path|
-        base = File.join(to, path)
-        path =~ /[\*\?\[\]\{\}]/ ? %(--exclude=#{base} ) : %(--exclude=#{base} --exclude=#{File.join(base, '*')})
-      end
-    end
-
-    def any_changes?(from, to = latest_commit)
-      diff_patches(from, to).any?
+    def apply_dev_patch_command(stage)
+      patch_command(stage.prev_g_a_stage.layer_commit(self), nil)
     end
 
     def patch_size(from, to)
@@ -79,12 +63,16 @@ module Dapp
       end
     end
 
+    def dev_patch_hash(stage)
+      Digest::SHA256.hexdigest diff_patches(stage.prev_g_a_stage.layer_commit(self), nil).map(&:to_s).join(':::')
+    end
+
     def latest_commit
       @latest_commit ||= commit || repo.latest_commit(branch)
     end
 
     def paramshash
-      Digest::SHA256.hexdigest [full_name, to, cwd, commit, branch, *include_paths, *exclude_paths, owner, group].map(&:to_s).join(':::')
+      Digest::SHA256.hexdigest [full_name, to, cwd, *include_paths, *exclude_paths, owner, group].map(&:to_s).join(':::')
     end
 
     def exclude_paths(with_cwd = false)
@@ -103,6 +91,10 @@ module Dapp
       "#{repo.name}#{name ? "_#{name}" : nil}"
     end
 
+    def any_changes?(from, to = latest_commit)
+      diff_patches(from, to).any?
+    end
+
     protected
 
     attr_reader :to
@@ -112,12 +104,108 @@ module Dapp
     attr_reader :owner
     attr_reader :group
 
+    def patch_command(prev_commit, current_commit)
+      [].tap do |commands|
+        if any_changes?(prev_commit, current_commit)
+          commands << "#{sudo}#{repo.dimg.project.git_bin} apply --whitespace=nowarn --directory=#{to} --unsafe-paths " \
+                      "#{patch_file(prev_commit, current_commit)}"
+        end
+      end
+    end
+
     def sudo
       repo.dimg.project.sudo_command(owner: owner, group: group)
     end
 
-    def diff_command(from, to, quiet: false)
-      "diff --binary #{'--quiet' if quiet} #{from}..#{to} #{"--relative=#{cwd}" if cwd} -- #{include_paths(true).join(' ')}"
+    def archive_file(commit)
+      create_file(repo.dimg.tmp_path('archives', archive_file_name(commit))) do |f|
+        Gem::Package::TarWriter.new(f) do |tar|
+          diff_patches(nil, commit).each do |patch|
+            entry = patch.delta.new_file
+            tar.add_file slice_cwd(entry[:path]), entry[:mode] do |tf|
+              tf.write repo.lookup_object(entry[:oid]).content
+            end
+          end
+        end
+      end
+      repo.dimg.container_tmp_path('archives', archive_file_name(commit))
+    rescue Gem::Package::TooLongFileName => e
+      raise Error::TarWriter, message: e.message
+    end
+
+    def slice_cwd(path)
+      return path if cwd.empty?
+      path
+        .reverse
+        .chomp(cwd.reverse)
+        .reverse
+    end
+
+    def archive_file_name(commit)
+      file_name(commit, 'tar')
+    end
+
+    def patch_file(from, to)
+      create_file(repo.dimg.tmp_path('patches', patch_file_name(from, to))) do |f|
+        diff_patches(from, to).each { |patch| f.write change_patch_new_file_path(patch) }
+      end
+      repo.dimg.container_tmp_path('patches', patch_file_name(from, to))
+    end
+
+    def change_patch_new_file_path(patch)
+      patch.to_s.lines.tap do |lines|
+        modify_patch_line = proc do |line_number, path_char|
+          action_part, path_part = lines[line_number].split
+          if (path_with_cwd = path_part.partition("#{path_char}/").last).start_with?(cwd)
+            path_with_cwd.sub(cwd, '').tap do |native_path|
+              expected_path = File.join(path_char, native_path)
+              lines[line_number] = [action_part, expected_path].join(' ') + "\n"
+            end
+          end
+        end
+
+        modify_patch = proc do |*modify_patch_line_args|
+          native_paths = modify_patch_line_args.map { |args| modify_patch_line.call(*args) }
+          unless (native_paths = native_paths.compact.uniq).empty?
+            raise Error::Build, code: :unsupported_patch_format, data: { patch: patch.to_s } unless native_paths.one?
+            native_path = native_paths.first
+            lines[0] = ['diff --git', File.join('a', native_path), File.join('b', native_path)].join(' ') + "\n"
+          end
+        end
+
+        case
+        when patch.delta.deleted? then modify_patch.call([3, 'a'])
+        when patch.delta.added? then modify_patch.call([4, 'b'])
+        when patch.delta.modified?
+          if patch_file_mode_changed?(patch)
+            modify_patch.call([4, 'a'], [5, 'b'])
+          else
+            modify_patch.call([2, 'a'], [3, 'b'])
+          end
+        else
+          raise
+        end
+      end.join
+    end
+
+    def patch_file_mode_changed?(patch)
+      patch.delta.old_file[:mode] != patch.delta.new_file[:mode]
+    end
+
+    def patch_file_name(from, to)
+      file_name(from, to, 'patch')
+    end
+
+    def file_name(*args, ext)
+      "#{[full_name, args].flatten.join('_')}.#{ext}"
+    end
+
+    def create_file(file_path, &blk)
+      File.open(file_path, File::RDWR|File::CREAT, &blk)
+    end
+
+    def diff_patches(from, to)
+      (@diff_patches ||= {})[[from, to]] = repo.patches(from, to, paths: include_paths_or_cwd, exclude_paths: exclude_paths(true))
     end
 
     def include_paths_or_cwd
@@ -129,14 +217,25 @@ module Dapp
       end
     end
 
-    def diff_patches(from, to)
-      repo.diff(from, to, paths: include_paths_or_cwd).patches.select do |p|
-        exclude_paths(true).any? { |path| !p.delta.new_file[:path].start_with?(path) }
-      end
+    def exclude_paths(with_cwd = false)
+      base_paths(repo.exclude_paths + @exclude_paths, with_cwd)
+    end
+
+    def include_paths(with_cwd = false)
+      base_paths(@include_paths, with_cwd)
     end
 
-    def file_exist_in_repo?(from, path)
-      repo.file_exist_in_tree?(repo.lookup_commit(from).tree, path.split('/'))
+    def base_paths(paths, with_cwd = false)
+      [paths].flatten.compact.map do |path|
+        if with_cwd && !cwd.empty?
+          File.join(cwd, path)
+        else
+          path
+        end
+          .chomp('/')
+          .reverse.chomp('/')
+          .reverse
+      end
     end
   end
 end

data/lib/dapp/git_repo/base.rb

@@ -14,58 +14,74 @@ module Dapp
         []
       end
 
-      def container_path
-        raise
+      def patches(from, to, exclude_paths: [], **kwargs)
+        diff(from, to, **kwargs).patches.select do |patch|
+          !exclude_paths.any? { |p| check_path?(patch.delta.new_file[:path], p) }
+        end
       end
 
-      def path
-        raise
-      end
-
-      def git_bare
-        @git_bare ||= Rugged::Repository.new(path, bare: true)
+      def diff(from, to, **kwargs)
+        if from.nil?
+          Rugged::Tree.diff(git, nil, to, **kwargs)
+        else
+          lookup_commit(from).diff(lookup_commit(to), **kwargs)
+        end
       end
 
-      def diff(from, to, **kwargs)
-        lookup_commit(from).diff(lookup_commit(to), **kwargs)
+      def commit_exists?(commit)
+        git.exists?(commit)
       end
 
-      def commit_at(commit)
-        lookup_commit(commit).time.to_i
+      def latest_commit(_branch)
+        raise
       end
 
-      def latest_commit(branch)
-        return git_bare.head.target_id if branch == 'HEAD'
-        git_bare.branches[branch].target_id
+      def branch
+        git.head.name.sub(/^refs\/heads\//, '')
       end
 
-      def cleanup!
+      def commit_at(commit)
+        lookup_commit(commit).time.to_i
       end
 
-      def branch
-        git_bare.head.name.sub(/^refs\/heads\//, '')
+      def find_commit_id_by_message(regex)
+        walker.each do |commit|
+          next unless commit.message =~ regex
+          return commit.oid
+        end
       end
 
-      def file_exist_in_tree?(tree, paths)
-        path = paths.shift
-        paths.empty? ?
-          tree.each { |obj| return true if File.fnmatch(path, obj[:name]) } :
-          tree.each_tree { |tree_obj| return file_exist_in_tree?(lookup_object(tree_obj[:oid]), paths) if File.fnmatch(path, tree_obj[:name]) }
-        false
+      def walker
+        walker = Rugged::Walker.new(git)
+        walker.push(git.head.target_id)
+        walker
       end
 
       def lookup_object(oid)
-        git_bare.lookup(oid)
+        git.lookup(oid)
       end
 
       def lookup_commit(commit)
-        git_bare.lookup(commit)
+        git.lookup(commit)
       end
 
       protected
 
-      def git
-        @git ||= Rugged::Repository.new(path)
+      def git(**kwargs)
+        @git ||= Rugged::Repository.new(path, **kwargs)
+      end
+
+      private
+
+      def check_path?(path, format)
+        path_parts = path.split('/')
+        checking_path = nil
+
+        until path_parts.empty?
+          checking_path = [checking_path, path_parts.shift].compact.join('/')
+          return true if File.fnmatch(format, checking_path)
+        end
+        false
       end
     end
   end