dapp 0.32.10 → 0.33.0

data/lib/dapp/dimg/dapp/command/mrproper.rb

@@ -38,13 +38,6 @@ module Dapp
 
           def proper_all
             flush_by_label('dapp')
-            remove_build_dir
-          end
-
-          def remove_build_dir
-            build_path.tap { |p| log_step_with_indent(:build_dir) { FileUtils.rm_rf(p) } }
-          rescue ::Dapp::Error::Dapp => e
-            raise unless e.net_status[:code] == :dappfile_not_found
           end
 
           def proper_dev_mode_cache
@@ -71,19 +64,11 @@ module Dapp
           end
 
           def proper_cache_all_images_names
-            shellout!(%(#{host_docker} images --format='{{if ne "<none>" .Tag }}{{.Repository}}:{{.Tag}}{{ end }}' -f "label=dapp" -f "label=dapp-cache-version=#{::Dapp::BUILD_CACHE_VERSION}"))
-              .stdout
-              .lines
-              .map(&:strip)
-              .reject(&:empty?)
+            ruby2go_image_images([{ label: "dapp" }, { label: "dapp-cache-version=#{::Dapp::BUILD_CACHE_VERSION}" }], ignore_tagless: true).map { |i| i["RepoTags"].empty? ? i["Id"] : i["RepoTags"] }.flatten
           end
 
           def dapp_images_names_by_label(label)
-            shellout!(%(#{host_docker} images --format='{{if ne "<none>" .Tag }}{{.Repository}}:{{.Tag}}{{ end }}' -f "label=dapp" -f "label=#{label}"))
-              .stdout
-              .lines
-              .map(&:strip)
-              .reject(&:empty?)
+            ruby2go_image_images([{ label: "dapp" }, { label: label }], ignore_tagless: true).map { |i| i["RepoTags"].empty? ? i["Id"] : i["RepoTags"] }.flatten
           end
         end
       end

data/lib/dapp/dimg/dapp/command/stages/cleanup_local.rb

@@ -24,7 +24,7 @@ module Dapp
                 lock("#{name}.images") do
                   remove_project_images begin
                     dapp_project_dimgstages.select do |image|
-                      !actual_cache_project_dimgstages.map { |dimgstage| dimgstage[:id] }.include?(image[:id])
+                      !actual_cache_project_dimgstages.map { |dimgstage| dimgstage["Id"] }.include?(image["Id"])
                     end
                   end
                 end
@@ -32,7 +32,10 @@ module Dapp
             end
 
             def actual_cache_project_dimgstages
-              @actual_cache_project_images_ids ||= prepare_docker_images("-f \"label=dapp-cache-version=#{::Dapp::BUILD_CACHE_VERSION}\" #{stage_cache}")
+              @actual_cache_project_dimgstages ||= begin
+                filters = [{ label: "dapp-cache-version=#{::Dapp::BUILD_CACHE_VERSION}", reference: stage_cache }]
+                prepare_docker_images(extra_filters: filters)
+              end
             end
 
             def stages_cleanup_by_repo
@@ -46,7 +49,7 @@ module Dapp
 
                   # Удаление только образов старше 2ч
                   dimgstages.delete_if do |dimgstage|
-                    Time.now.to_i - dimgstage[:created_at] < 2 * 60 * 60
+                    Time.now.to_i - dimgstage["Created"] < 2 * 60 * 60
                   end unless ENV['DAPP_STAGES_CLEANUP_LOCAL_DISABLED_DATE_POLICY']
 
                   remove_project_images(dimgstages)
@@ -73,7 +76,7 @@ module Dapp
             end
 
             def dapp_project_image_artifacts_ids_in_labels(image)
-              select_dapp_artifacts_ids(dapp_project_image_labels(image))
+              select_dapp_artifacts_ids(image['Labels'])
             end
 
             def proper_git_commit
@@ -81,7 +84,7 @@ module Dapp
                 lock("#{name}.images") do
                   unproper_images = []
                   dapp_project_dimgstages.each do |dimgstage|
-                    dapp_project_image_labels(dimgstage).each do |repo_name, commit|
+                    dimgstage["Labels"].each do |repo_name, commit|
                       next if (repo = dapp_git_repositories[repo_name]).nil?
                       unproper_images.concat(dapp_project_image_with_children(dimgstage)) unless repo.commit_exists?(commit)
                     end
@@ -109,7 +112,7 @@ module Dapp
             end
 
             def dapp_project_image_parent(image)
-              dapp_project_image_by_id(dapp_project_image_inspect(image)['Parent'])
+              dapp_project_image_by_id(image['ParentId'])
             end
           end
         end

data/lib/dapp/dimg/dimg.rb

@@ -13,6 +13,13 @@ module Dapp
       attr_reader :should_be_built
       attr_reader :dapp
 
+      def get_ruby2go_state_hash
+        {
+          "Dapp" => dapp.get_ruby2go_state_hash,
+          "TmpPath" => tmp_path.to_s,
+        }
+      end
+
       def initialize(config:, dapp:, should_be_built: false, ignore_signature_auto_calculation: false)
         @config = config
         @dapp = dapp
@@ -42,22 +49,19 @@ module Dapp
       end
 
       def build!
-        with_introspection do
-          dapp.lock("#{dapp.name}.images", readonly: true) do
-            last_stage.build_lock! do
-              begin
-                builder.before_build_check
-                last_stage.build!
-              ensure
-                after_stages_build!
-              end
+        dapp.lock("#{dapp.name}.images", readonly: true) do
+          last_stage.build_lock! do
+            begin
+              builder.before_build_check
+              last_stage.build!
+            ensure
+              after_stages_build!
             end
           end
         end
       end
 
       def after_stages_build!
-        return unless last_stage.image.built? || dev_mode? || force_save_cache?
         last_stage.save_in_cache!
         artifacts.each { |artifact| artifact.last_stage.save_in_cache! }
       end
@@ -158,7 +162,6 @@ module Dapp
           dapp.log_state(image_name, state: dapp.t(code: push ? 'state.push' : 'state.export'), styles: { status: :success })
         else
           dapp.lock("image.#{hashsum image_name}") do
-            ::Dapp::Dimg::Image::Docker.reset_image_inspect(image_name)
             dapp.log_process(image_name, process: dapp.t(code: push ? 'status.process.pushing' : 'status.process.exporting')) { yield }
           end
         end
@@ -209,11 +212,12 @@ module Dapp
       def run_stage(stage_name, docker_options, command)
         stage_image = (stage_name.nil? ? last_stage : stage_by_name(stage_name)).image
         raise Error::Dimg, code: :dimg_stage_not_built, data: { stage_name: stage_name } unless stage_image.built?
-        cmd = "#{dapp.host_docker} run #{[docker_options, stage_image.built_id, command].flatten.compact.join(' ')}"
+
+        args = [docker_options, stage_image.built_id, command].flatten.compact
         if dapp.dry_run?
-          dapp.log(cmd)
+          dapp.log("docker run #{args.join(' ')}")
         else
-          system(cmd) || raise(Error::Dimg, code: :dimg_not_run)
+          Image::Stage.ruby2go_command(dapp, command: :container_run, options: { args: args })
         end
       end
 
@@ -241,23 +245,10 @@ module Dapp
         dapp.dev_mode?
       end
 
-      def force_save_cache?
-        if ENV.key? "DAPP_FORCE_SAVE_CACHE"
-          %w(yes 1 true).include? ENV["DAPP_FORCE_SAVE_CACHE"].to_s
-        else
-          !!dapp.options[:force_save_cache]
-        end
-      end
-
       def build_cache_version
         [::Dapp::BUILD_CACHE_VERSION, dev_mode? ? 1 : 0]
       end
 
-      def introspect_image!(image:, options:)
-        cmd = "#{dapp.host_docker} run -ti --rm --entrypoint #{dapp.bash_bin} #{options} #{image}"
-        system(cmd)
-      end
-
       def cleanup_tmp
         return unless tmp_dir_exists?
 
@@ -265,14 +256,14 @@ module Dapp
         # Такие файлы могут попасть туда при экспорте файлов артефакта.
         # Чтобы от них избавиться — запускаем docker-контейнер под root-пользователем
         # и удаляем примонтированную tmp-директорию.
-        cmd = "".tap do |cmd|
-          cmd << "#{dapp.host_docker} run --rm"
-          cmd << " --volume #{dapp.tmp_base_dir}:#{dapp.tmp_base_dir}"
-          cmd << " --label dapp=#{dapp.name}"
-          cmd << " alpine:3.6"
-          cmd << " rm -rf #{tmp_path}"
-        end
-        dapp.shellout! cmd
+        args = [
+          "--rm",
+          "--volume=#{dapp.tmp_base_dir}:#{dapp.tmp_base_dir}",
+          "--label=dapp=#{dapp.name}",
+          "alpine:3.6",
+          "rm", "-rf", tmp_path
+        ]
+        Image::Stage.ruby2go_command(dapp, command: :container_run, options: { args: args })
       end
 
       def stage_should_be_introspected_before_build?(name)
@@ -291,14 +282,6 @@ module Dapp
           !last_stage.image.tagged?
         end
       end
-
-      def with_introspection
-        yield
-      rescue Exception::IntrospectImage => e
-        data = e.net_status[:data]
-        introspect_image!(image: data[:built_id], options: data[:options])
-        raise data[:error]
-      end
     end # Dimg
   end # Dimg
 end # Dapp

data/lib/dapp/dimg/docker_registry/base/authorization.rb

@@ -29,25 +29,10 @@ module Dapp
             [:realm, :service, :scope].map do |option|
               /#{option}="([[^"].]*)/ =~ header
               next unless Regexp.last_match(1)
-
-              option_value = begin
-                if option == :scope
-                  handle_scope_option(Regexp.last_match(1))
-                else
-                  Regexp.last_match(1)
-                end
-              end
-
-              [option, option_value]
+              [option, Regexp.last_match(1)]
             end.compact.to_h
           end
 
-          def handle_scope_option(resourcescope)
-            resource_type, resource_name, actions = resourcescope.split(":")
-            actions                               = actions.split(",").map { |action| action == "delete" ? "*" : action }.join(",")
-            [resource_type, resource_name, actions].join(":")
-          end
-
           def authorization_auth
             @authorization_auth ||= begin
               if ::Dapp::Dapp.options_with_docker_credentials?

data/lib/dapp/dimg/git_artifact.rb

@@ -14,7 +14,7 @@ module Dapp
       # rubocop:disable Metrics/ParameterLists
       def initialize(repo, dimg, to:, name: nil, branch: nil, tag: nil, commit: nil,
                      cwd: nil, include_paths: nil, exclude_paths: nil, owner: nil, group: nil, as: nil,
-                     stages_dependencies: {}, ignore_signature_auto_calculation: false)
+                     stages_dependencies: {}, ignore_signature_auto_calculation: false, disable_go_git: nil)
         @repo = repo
         @dimg = dimg
         @name = name
@@ -34,6 +34,7 @@ module Dapp
         @as = as
 
         @stages_dependencies = stages_dependencies
+        @disable_go_git = disable_go_git unless disable_go_git.nil?
       end
       # rubocop:enable Metrics/ParameterLists
 
@@ -49,7 +50,7 @@ module Dapp
       end
 
       def submodule_artifact(submodule_params)
-        embedded_artifact(submodule_params)
+        embedded_artifact(**submodule_params, disable_go_git: true)
       rescue Rugged::InvalidError => e
         raise Error::Rugged, code: :git_local_incorrect_gitmodules_params, data: { error: e.message }
       end
@@ -109,6 +110,7 @@ module Dapp
           options[:group]               = group
 
           options[:ignore_signature_auto_calculation] = ignore_signature_auto_calculation
+          options[:disable_go_git] = embedded_params[:disable_go_git]
         end
       end
 
@@ -181,7 +183,37 @@ module Dapp
         end
       end
 
+      def disable_go_git?
+        return @disable_go_git unless @disable_go_git.nil?
+
+        @disable_go_git = (dev_mode? || !!ENV["DAPP_DISABLE_GO_GIT"] || begin
+          commit = dev_mode? ? nil : latest_commit
+          repo.submodules(
+            commit,
+            paths: include_paths_or_cwd,
+            exclude_paths: exclude_paths(true)
+          ).any?
+        end)
+      end
+
       def apply_archive_command(stage)
+        return apply_archive_command_old(stage) if disable_go_git?
+
+        res = repo.dapp.ruby2go_git_artifact(
+          "GitArtifact" => JSON.dump(get_ruby2go_state_hash),
+          "method" => "ApplyArchiveCommand",
+          "Stage" => JSON.dump(get_stub_stage_state(stage)),
+        )
+
+        raise res["error"] if res["error"]
+
+        self.set_ruby2go_state_hash(JSON.load(res["data"]["GitArtifact"]))
+        stage.set_ruby2go_state_hash(JSON.load(res["data"]["Stage"]))
+
+        res["data"]["result"]
+      end
+
+      def apply_archive_command_old(stage)
         [].tap do |commands|
           if archive_any_changes?(stage)
             case cwd_type(stage)
@@ -189,12 +221,12 @@ module Dapp
               stage.image.add_service_change_label(repo.dapp.dimgstage_g_a_type_label(paramshash).to_sym => 'directory')
 
               commands << "#{repo.dapp.install_bin} #{credentials.join(' ')} -d \"#{to}\""
-              commands << "#{sudo}#{repo.dapp.tar_bin} -xf #{archive_file(stage)} -C \"#{to}\""
+              commands << "#{sudo} #{repo.dapp.tar_bin} -xf #{archive_file(stage)} -C \"#{to}\""
             when :file
               stage.image.add_service_change_label(repo.dapp.dimgstage_g_a_type_label(paramshash).to_sym => 'file')
 
               commands << "#{repo.dapp.install_bin} #{credentials.join(' ')} -d \"#{File.dirname(to)}\""
-              commands << "#{sudo}#{repo.dapp.tar_bin} -xf #{archive_file(stage)} -C \"#{File.dirname(to)}\""
+              commands << "#{sudo} #{repo.dapp.tar_bin} -xf #{archive_file(stage)} -C \"#{File.dirname(to)}\""
             end
           end
         end
@@ -204,7 +236,36 @@ module Dapp
         stage.prev_stage.image.labels[repo.dapp.dimgstage_g_a_type_label(paramshash)].to_s.to_sym
       end
 
+      def get_stub_stage_state(stage)
+        stage.get_ruby2go_state_hash.tap do |stage_state|
+          # Data for StubStage specific for ApplyPatchCommand
+          stage_state["LayerCommitMap"] = {
+            paramshash => stage.layer_commit(self),
+          }
+          stage_state["PrevStage"]["LayerCommitMap" ] = {
+            paramshash => stage.prev_stage.layer_commit(self),
+          }
+        end
+      end
+
       def apply_patch_command(stage)
+        return apply_patch_command_old(stage) if disable_go_git?
+
+        res = repo.dapp.ruby2go_git_artifact(
+          "GitArtifact" => JSON.dump(get_ruby2go_state_hash),
+          "method" => "ApplyPatchCommand",
+          "Stage" => JSON.dump(get_stub_stage_state(stage)),
+        )
+
+        raise res["error"] if res["error"]
+
+        self.set_ruby2go_state_hash(JSON.load(res["data"]["GitArtifact"]))
+        stage.set_ruby2go_state_hash(JSON.load(res["data"]["Stage"]))
+
+        res["data"]["result"]
+      end
+
+      def apply_patch_command_old(stage)
         [].tap do |commands|
           if dev_mode?
             if any_changes?(*dev_patch_stage_commits(stage))
@@ -219,12 +280,12 @@ module Dapp
 
                 commands << "#{repo.dapp.rm_bin} -rf $(#{repo.dapp.cat_bin} #{dimg.container_tmp_path('archives', files_to_remove_file_name)})"
                 commands << "#{repo.dapp.install_bin} #{credentials.join(' ')} -d \"#{to}\""
-                commands << "#{sudo}#{repo.dapp.tar_bin} -xf #{archive_file(stage)} -C \"#{to}\""
+                commands << "#{sudo} #{repo.dapp.tar_bin} -xf #{archive_file(stage)} -C \"#{to}\""
                 commands << "#{repo.dapp.find_bin} \"#{to}\" -empty -type d -delete"
               when :file
                 commands << "#{repo.dapp.rm_bin} -rf \"#{to}\""
                 commands << "#{repo.dapp.install_bin} #{credentials.join(' ')} -d \"#{File.dirname(to)}\""
-                commands << "#{sudo}#{repo.dapp.tar_bin} -xf #{archive_file(stage)} -C \"#{File.dirname(to)}\""
+                commands << "#{sudo} #{repo.dapp.tar_bin} -xf #{archive_file(stage)} -C \"#{File.dirname(to)}\""
               else
                 raise
               end
@@ -234,10 +295,10 @@ module Dapp
               case archive_type(stage)
               when :directory
                 commands << "#{repo.dapp.install_bin} #{credentials.join(' ')} -d \"#{to}\""
-                commands << "#{sudo}#{repo.dapp.git_bin} apply --whitespace=nowarn --directory=\"#{to}\" --unsafe-paths #{patch_file(stage, *patch_stage_commits(stage))}"
+                commands << "#{sudo} #{repo.dapp.git_bin} apply --whitespace=nowarn --directory=\"#{to}\" --unsafe-paths #{patch_file(stage, *patch_stage_commits(stage))}"
               when :file
                 commands << "#{repo.dapp.install_bin} #{credentials.join(' ')} -d \"#{File.dirname(to)}\""
-                commands << "#{sudo}#{repo.dapp.git_bin} apply --whitespace=nowarn --directory=\"#{File.dirname(to)}\" --unsafe-paths #{patch_file(stage, *patch_stage_commits(stage))}"
+                commands << "#{sudo} #{repo.dapp.git_bin} apply --whitespace=nowarn --directory=\"#{File.dirname(to)}\" --unsafe-paths #{patch_file(stage, *patch_stage_commits(stage))}"
               else
                 raise
               end
@@ -298,21 +359,81 @@ module Dapp
         end
       end
 
+      def get_ruby2go_state_hash
+        {
+          "Name" => @name.to_s,
+          "As" => @as.to_s,
+          "Branch" => @branch.to_s,
+          "Tag" => @tag.to_s,
+          "Commit" => @commit.to_s,
+          "To" => @to.to_s,
+          "Cwd" => @cwd.to_s,
+          "RepoPath" => File.join("/", @cwd.to_s),
+          "Owner" => @owner.to_s,
+          "Group" => @group.to_s,
+          "IncludePaths" => include_paths(true),
+          "ExcludePaths" => exclude_paths(true),
+          "StagesDependencies" => @stages_dependencies.map {|k, v| [_stages_map[k], Array(v).map(&:to_s)]}.to_h,
+          "Paramshash" => paramshash.to_s,
+          "PatchesDir" => dimg.tmp_path('patches'),
+          "ContainerPatchesDir" => dimg.container_tmp_path('patches'),
+          "ArchivesDir" => dimg.tmp_path('archives'),
+          "ContainerArchivesDir" => dimg.container_tmp_path('archives'),
+        }.tap {|res|
+          if repo.is_a? ::Dapp::Dimg::GitRepo::Local
+            res["LocalGitRepo"] = repo.get_ruby2go_state_hash
+          elsif repo.is_a? ::Dapp::Dimg::GitRepo::Remote
+            res["RemoteGitRepo"] = repo.get_ruby2go_state_hash
+          else
+            raise
+          end
+        }
+      end
+
+      def _stages_map
+        {
+          before_install: "beforeInstall",
+          install: "install",
+          before_setup: "beforeSetup",
+          setup: "setup",
+          build_artifact: "buildArtifact",
+        }
+      end
+
+      def _stages_map_reversed
+        _stages_map.map {|k, v| [v, k]}.to_h
+      end
+
+      def set_ruby2go_state_hash(new_state)
+        [
+          [:@name, new_state["Name"]],
+          [:@as, new_state["As"]],
+          [:@branch, new_state["Branch"]],
+          [:@tag, new_state["Tag"]],
+          [:@commit, new_state["Commit"]],
+          [:@cwd, new_state["Cwd"]],
+          [:@owner, new_state["Owner"]],
+          [:@group, new_state["Group"]],
+        ].each do |var, new_value|
+          if new_value != ""
+            instance_variable_set(var, new_value)
+          end
+        end
+
+        @stages_dependencies = new_state["StagesDependencies"].map do |k, v|
+          [_stages_map_reversed[k], v]
+        end.to_h
+      end
+
       def latest_commit
         @latest_commit ||= begin
-          commit || begin
-            if !tag.nil?
-              repo.latest_tag_commit(tag)
-            elsif !branch.nil?
-              repo.latest_branch_commit(branch)
-            else
-              if local?
-                repo.head_commit
-              else
-                repo.latest_branch_commit(repo.branch)
-              end
-            end
-          end
+          res = repo.dapp.ruby2go_git_artifact("GitArtifact" => JSON.dump(get_ruby2go_state_hash), "method" => "LatestCommit")
+
+          raise res["error"] if res["error"]
+
+          self.set_ruby2go_state_hash(JSON.load(res["data"]["GitArtifact"]))
+
+          res["data"]["result"]
         end.tap do |c|
           repo.dapp.log_info("Repository `#{repo.name}`: latest commit `#{c}` to `#{to}`") unless ignore_signature_auto_calculation
         end