dapp 0.10.2 → 0.36.24

data/lib/dapp/dimg/build/stage/setup/ga_post_setup_patch.rb

@@ -6,10 +6,33 @@ module Dapp
           class GAPostSetupPatch < GABase
             include Mod::Group
 
+            MAX_PATCH_SIZE = 1024 * 1024
+
             def initialize(dimg, next_stage)
-              @prev_stage = GAPostSetupPatchDependencies.new(dimg, self)
+              @prev_stage = Setup.new(dimg, self)
               super
             end
+
+            def renew
+              dependencies_discard
+              super
+            end
+
+            def dependencies
+              @dependencies ||= [(changes_size_since_g_a_pre_setup_patch / MAX_PATCH_SIZE).to_i]
+            end
+
+            private
+
+            def changes_size_since_g_a_pre_setup_patch
+              dimg.git_artifacts.map do |git_artifact|
+                if git_artifact.repo.commit_exists? prev_stage.layer_commit(git_artifact)
+                  git_artifact.patch_size(prev_stage.layer_commit(git_artifact))
+                else
+                  0
+                end
+              end.reduce(0, :+)
+            end
           end # GAPostSetupPatch
         end
       end # Stage

data/lib/dapp/dimg/build/stage/setup/ga_pre_setup_patch.rb

@@ -3,13 +3,17 @@ module Dapp
     module Build
       module Stage
         module Setup
-          class GAPreSetupPatch < GABase
+          class GAPreSetupPatch < GARelatedBase
             include Mod::Group
 
             def initialize(dimg, next_stage)
-              @prev_stage = GAPreSetupPatchDependencies.new(dimg, self)
+              @prev_stage = BeforeSetupArtifact.new(dimg, self)
               super
             end
+
+            def related_stage_name
+              :setup
+            end
           end # GAPrePatch
         end
       end # Stage

data/lib/dapp/dimg/build/stage/setup/setup.rb

@@ -3,31 +3,13 @@ module Dapp
     module Build
       module Stage
         module Setup
-          class Setup < Base
+          class Setup < Instructions
             include Mod::Group
-            include Mod::GitArtifactsDependencies
 
             def initialize(dimg, next_stage)
               @prev_stage = GAPreSetupPatch.new(dimg, self)
               super
             end
-
-            def empty?
-              !dimg.builder.setup?
-            end
-
-            def context
-              [git_artifacts_dependencies, builder_checksum]
-            end
-
-            def builder_checksum
-              dimg.builder.setup_checksum
-            end
-
-            def prepare_image
-              super
-              dimg.builder.setup(image)
-            end
           end # Setup
         end # Setup
       end # Stage

data/lib/dapp/dimg/builder/ansible.rb

@@ -0,0 +1,5 @@
+module Dapp
+  module Dimg
+    class Builder::Ansible < Builder::Ruby2Go; end
+  end # Dimg
+end # Dapp

data/lib/dapp/dimg/builder/chef/cookbook.rb

@@ -66,7 +66,7 @@ module Dapp
       end
 
       def vendor_path
-        builder.dimg.dapp.lock(_vendor_lock_name, default_timeout: 120) do
+        builder.dimg.dapp.lock(_vendor_lock_name, default_timeout: 600) do
           vendor! unless _vendor_path.join('.created_at').exist?
         end
         _vendor_path
@@ -74,11 +74,9 @@ module Dapp
 
       # rubocop:disable Metrics/AbcSize
       def vendor!
-        volumes_from = [builder.dimg.dapp.base_container, builder.chefdk_container]
+        volumes_from = [builder.dimg.dapp.toolchain_container, builder.dimg.dapp.base_container, builder.chefdk_container]
 
         builder.dimg.dapp.log_secondary_process(builder.dimg.dapp.t(code: _vendor_process_name)) do
-          volumes_from = [builder.dimg.dapp.base_container, builder.chefdk_container]
-
           tmp_berksfile_path = builder.dimg.tmp_path.join("Berksfile.#{SecureRandom.uuid}")
           File.open(tmp_berksfile_path, 'w') {|file| file.write berksfile.dump}
 
@@ -89,11 +87,11 @@ module Dapp
             "#{builder.dimg.dapp.mkdir_bin} -p ~/.ssh",
             'echo "Host *" >> ~/.ssh/config',
             'echo "    StrictHostKeyChecking no" >> ~/.ssh/config',
-            *local_paths.map {|path| "#{builder.dimg.dapp.rsync_bin} --archive --relative #{path} /tmp/local_cookbooks"},
+            *local_paths.map {|path| "#{builder.dimg.dapp.rsync_bin} --inplace --archive --relative #{path} /tmp/local_cookbooks"},
             "cd /tmp/local_cookbooks/#{path}",
             "cp #{tmp_berksfile_path} Berksfile",
             "cp #{tmp_metadata_path} metadata.rb",
-            '/.dapp/deps/chefdk/bin/berks vendor /tmp/cookbooks',
+            "#{builder.berks_bin} vendor /tmp/cookbooks",
             ["#{builder.dimg.dapp.find_bin} /tmp/cookbooks -type d -exec #{builder.dimg.dapp.bash_bin} -ec '",
              "#{builder.dimg.dapp.install_bin} -o #{Process.uid} -g #{Process.gid} ",
              "--mode $(#{builder.dimg.dapp.stat_bin} -c %a {}) -d ",
@@ -107,7 +105,7 @@ module Dapp
           ]
 
           builder.dimg.dapp.shellout!(
-            [ 'docker run --rm',
+            [ "#{builder.dimg.dapp.host_docker} run --rm",
               volumes_from.map {|container| "--volumes-from #{container}"}.join(' '),
               *local_paths.map {|path| "--volume #{path}:#{path}"},
               "--volume #{builder.dimg.tmp_path}:#{builder.dimg.tmp_path}",
@@ -118,7 +116,7 @@ module Dapp
                 cmd << "dappdeps/berksdeps:0.1.0 #{builder.dimg.dapp.bash_bin}"
                 cmd << " -ec '#{builder.dimg.dapp.shellout_pack(vendor_commands.join(' && '))}'"
               end ].compact.join(' '),
-            log_verbose: builder.dimg.dapp.log_verbose?
+            verbose: true
           )
         end
       end
@@ -187,6 +185,8 @@ module Dapp
                 Dir[from_path.join('**/*')]
                   .map(&Pathname.method(:new))
                   .each do |from_subpath|
+                    next if from_subpath.directory?
+
                     to_subpath = to_path.join(from_subpath.relative_path_from(from_path))
                     if to_subpath.exist?
                       raise(Error::Chef,

data/lib/dapp/dimg/builder/chef.rb

@@ -1,7 +1,15 @@
 module Dapp
   module Dimg
     class Builder::Chef < Builder::Base
-      DEFAULT_CHEFDK_IMAGE = 'dappdeps/chefdk:1.2.22-1'.freeze # TODO: config, DSL, DEFAULT_CHEFDK_IMAGE
+      CHEFDK_VERSION = "2.3.17-2"
+
+      def chef_solo_bin
+        "/.dapp/deps/chefdk/#{CHEFDK_VERSION}/bin/chef-solo"
+      end
+
+      def berks_bin
+        "/.dapp/deps/chefdk/#{CHEFDK_VERSION}/bin/berks"
+      end
 
       %i(before_install install before_setup setup build_artifact).each do |stage|
         define_method("#{stage}?") {!stage_empty?(stage)}
@@ -16,7 +24,7 @@ module Dapp
           unless stage_empty?(stage)
             image.add_volumes_from(chefdk_container)
             image.add_volume "#{stage_build_path(stage)}:#{container_stage_build_path(stage)}:ro"
-            image.add_command ['/.dapp/deps/chefdk/bin/chef-solo',
+            image.add_command [chef_solo_bin,
                                '--legacy-mode',
                                "--config #{container_stage_config_path(stage)}",
                                "--json-attributes #{container_stage_json_attributes_path(stage)}",
@@ -45,25 +53,29 @@ module Dapp
       end
 
       def chefdk_image
-        DEFAULT_CHEFDK_IMAGE # TODO: config, DSL, DEFAULT_CHEFDK_IMAGE
+        "dappdeps/chefdk:#{CHEFDK_VERSION}" # TODO: config, DSL, DEFAULT_CHEFDK_IMAGE
       end
 
       def chefdk_container_name # FIXME: hashsum(image) or dockersafe()
-        chefdk_image.tr('/', '_').tr(':', '_')
+        "dappdeps_chefdk_#{CHEFDK_VERSION}"
       end
 
       def chefdk_container
         @chefdk_container ||= begin
-          if dimg.dapp.shellout("docker inspect #{chefdk_container_name}").exitstatus.nonzero?
-            dimg.dapp.log_secondary_process(dimg.dapp.t(code: 'process.chefdk_container_creating'), short: true) do
-              dimg.dapp.shellout!(
-                ['docker create',
-                 "--name #{chefdk_container_name}",
-                 "--volume /.dapp/deps/chefdk #{chefdk_image}"].join(' ')
-              )
+          is_container_exist = proc{dimg.dapp.shellout("#{dimg.dapp.host_docker} inspect #{chefdk_container_name}").exitstatus.zero?}
+          if !is_container_exist.call
+            dimg.dapp.lock("dappdeps.container.#{chefdk_container_name}", default_timeout: 600) do
+              if !is_container_exist.call
+                dimg.dapp.log_secondary_process(dimg.dapp.t(code: 'process.chefdk_container_creating', data: {name: chefdk_container_name}), short: true) do
+                  dimg.dapp.shellout!(
+                    ["#{dimg.dapp.host_docker} create",
+                    "--name #{chefdk_container_name}",
+                    "--volume /.dapp/deps/chefdk/#{CHEFDK_VERSION} #{chefdk_image}"].join(' ')
+                  )
+                end
+              end
             end
           end
-
           chefdk_container_name
         end
       end
@@ -115,7 +127,7 @@ module Dapp
       end
 
       def stage_cookbooks_checksum_path(stage)
-        dimg.build_path.join("#{builder_cookbook.checksum}.#{dimg.config._name}.#{stage}.checksum")
+        dimg.build_path.join("#{builder_cookbook.checksum}.#{dimg.name}.#{stage}.checksum")
       end
 
       def stage_cookbooks_checksum(stage)
@@ -180,8 +192,9 @@ module Dapp
         @install_chef_solo_stage_config[stage] ||= true.tap do
           stage_build_path(stage).join('config.rb').write [
             "file_cache_path \"/.dapp/chef/cache\"\n",
-            "cookbook_path \"#{container_stage_build_path(stage).join('cookbooks')}\"\n"
-          ].join
+            "cookbook_path \"#{container_stage_build_path(stage).join('cookbooks')}\"\n",
+            ("log_level :debug\n" if ENV["DAPP_CHEF_DEBUG"]),
+          ].compact.join
         end
       end
 
@@ -203,7 +216,7 @@ module Dapp
       end
 
       def stage_build_path(stage)
-        dimg.tmp_path(dimg.config._name).join(stage.to_s)
+        dimg.tmp_path(dimg.name).join(stage.to_s)
       end
 
       def container_stage_build_path(_stage)

data/lib/dapp/dimg/builder/none.rb

@@ -1,38 +1,5 @@
 module Dapp
   module Dimg
-    class Builder::None < Builder::Base
-      def before_dimg_should_be_built_check
-      end
-
-      def before_install(_image)
-      end
-
-      def before_install_checksum
-      end
-
-      def before_setup(_image)
-      end
-
-      def before_setup_checksum
-      end
-
-      def install(_image)
-      end
-
-      def install_checksum
-      end
-
-      def setup(_image)
-      end
-
-      def setup_checksum
-      end
-
-      def build_artifact(_image)
-      end
-
-      def build_artifact_checksum
-      end
-    end # Builder::None
+    class Builder::None < Builder::Ruby2Go; end
   end # Dimg
 end # Dapp

data/lib/dapp/dimg/builder/ruby2go.rb

@@ -0,0 +1,57 @@
+module Dapp
+  module Dimg
+    class Builder::Ruby2Go < Builder::Base
+      [:before_install, :before_setup, :install, :setup, :build_artifact].each do |stage|
+        define_method("#{stage}_checksum") do
+          command = "#{snake_case_to_camel_case(stage)}Checksum"
+          ruby2go_builder_command(command: command)
+        end
+
+        define_method("#{stage}?") do
+          command = "Is#{snake_case_to_camel_case(stage)}Empty"
+          !ruby2go_builder_command(command: command)
+        end
+
+        define_method(stage.to_s) do |image|
+          command = snake_case_to_camel_case(stage)
+          ruby2go_builder_command(command: command, image: image.ruby2go_image_option).tap do |data|
+            image.set_ruby2go_state_hash(JSON.load(data["image"]))
+          end
+        end
+      end
+
+      def ruby2go_builder_command(command:, **options)
+        (options[:options] ||= {}).merge!(host_docker_config_dir: dimg.dapp.class.host_docker_config_dir)
+        builder = self.class.name.split("::").last.downcase
+        command_options = {
+          builder: builder,
+          command: command,
+          config: YAML.dump(dimg.config),
+          extra: get_ruby2go_state_hash,
+          artifact: dimg.artifact?
+        }.merge(options)
+
+        dimg.dapp.ruby2go_builder(command_options).tap do |res|
+          raise Error::Build, code: :ruby2go_builder_command_failed_unexpected_error, data: { command: command, message: res["error"] } unless res["error"].nil?
+          break res['data']
+        end
+      end
+
+      def get_ruby2go_state_hash
+        {
+          "TmpPath" => dimg.tmp_path.to_s,
+          "ContainerDappPath" => dimg.container_dapp_path.to_s,
+        }
+      end
+
+      def snake_case_to_camel_case(value)
+        ::Dapp::Helper::CaseConversion.snake_case_to_camel_case(value)
+      end
+
+      def snake_case_to_lower_camel_case(value)
+        ::Dapp::Helper::CaseConversion.snake_case_to_lower_camel_case(value)
+      end
+
+    end # Builder::Shell
+  end # Dimg
+end # Dapp

data/lib/dapp/dimg/builder/shell.rb

@@ -1,24 +1,5 @@
 module Dapp
   module Dimg
-    class Builder::Shell < Builder::Base
-      [:before_install, :before_setup, :install, :setup, :build_artifact].each do |stage|
-        define_method("#{stage}_checksum") do
-          [dimg.config._shell.public_send("_#{stage}_command"),
-           dimg.config._shell.public_send("_#{stage}_version")].flatten
-        end
-        define_method("#{stage}?") { !stage_empty?(stage) }
-        define_method(stage.to_s) do |image|
-          image.add_command(*stage_commands(stage)) unless stage_empty?(stage)
-        end
-      end
-
-      def stage_empty?(stage)
-        stage_commands(stage).empty?
-      end
-
-      def stage_commands(stage)
-        dimg.config._shell.public_send("_#{stage}_command")
-      end
-    end # Builder::Shell
+    class Builder::Shell < Builder::Ruby2Go; end
   end # Dimg
 end # Dapp

data/lib/dapp/dimg/cli/cli.rb

@@ -1 +1,2 @@
-::Dapp::CLI.send(:include, ::Dapp::Dimg::CLI)
+::Dapp::CLI.send(:include, ::Dapp::Dimg::CLI::Command)
+::Dapp::CLI.send(:include, ::Dapp::Dimg::CLI::Options)

data/lib/dapp/dimg/cli/command/base.rb

@@ -0,0 +1,24 @@
+module Dapp::Dimg::CLI
+  module Command
+    class Base < ::Dapp::CLI::Command::Base
+      DIMG_STAGES = [
+        :from, :before_install, :before_install_artifact, :g_a_archive, :g_a_pre_install_patch, :install,
+        :g_a_post_install_patch, :after_install_artifact, :before_setup, :before_setup_artifact,
+        :g_a_pre_setup_patch, :setup, :g_a_post_setup_patch, :after_setup_artifact, :g_a_latest_patch, :docker_instructions
+      ].freeze
+
+      STAGE_PROC = proc do |stages|
+        proc { |val| val.to_sym.tap { |v| in_validate!(v, stages) } }
+      end
+
+      option :build_dir,
+             long: "--build-dir PATH",
+             description: "Directory where build cache stored ($HOME/.dapp/builds/<dapp name> by default)."
+
+      def run(argv = ARGV)
+        self.class.parse_options(self, argv)
+        run_dapp_command(run_method, options: cli_options(dimgs_patterns: cli_arguments))
+      end
+    end
+  end
+end

data/lib/dapp/dimg/cli/command/dimg/bp.rb

@@ -0,0 +1,60 @@
+module Dapp::Dimg::CLI
+  module Command
+    class Dimg < ::Dapp::CLI
+      class Bp < Push
+        banner <<BANNER.freeze
+Usage:
+
+  dapp dimg bp [options] [DIMG ...] REPO
+
+    DIMG                        Dapp image to process [default: *].
+    REPO                        Pushed image name.
+
+Options:
+BANNER
+
+        # build options
+
+        extend ::Dapp::Dimg::CLI::Options::Introspection
+        extend ::Dapp::CLI::Options::Ssh
+
+        option :tmp_dir_prefix,
+               long: '--tmp-dir-prefix PREFIX',
+               description: 'Tmp directory prefix (/tmp by default). Used for build process service directories.'
+
+        option :lock_timeout,
+               long: '--lock-timeout TIMEOUT',
+               description: 'Redefine resource locking timeout (in seconds)',
+               proc: ->(v) { v.to_i }
+
+        option :build_context_directory,
+               long: '--build-context-directory DIR_PATH',
+               default: nil
+
+        option :use_system_tar,
+               long: '--use-system-tar',
+               boolean: true,
+               default: false
+
+        option :force_save_cache,
+               long: '--force-save-cache',
+               boolean: true,
+               default: false
+
+        # push options
+
+        extend ::Dapp::CLI::Options::Tag
+
+        option :with_stages,
+               long: '--with-stages',
+               boolean: true
+
+        option :registry_username,
+               long: '--registry-username USERNAME'
+
+        option :registry_password,
+               long: '--registry-password PASSWORD'
+      end
+    end
+  end
+end

data/lib/dapp/dimg/cli/command/dimg/build.rb

@@ -0,0 +1,43 @@
+module Dapp::Dimg::CLI
+  module Command
+    class Dimg < ::Dapp::CLI
+      class Build < Base
+        banner <<BANNER.freeze
+Usage:
+
+  dapp dimg build [options] [DIMG ...]
+
+    DIMG                        Dapp image to process [default: *].
+
+Options:
+BANNER
+
+        extend ::Dapp::Dimg::CLI::Options::Introspection
+        extend ::Dapp::CLI::Options::Ssh
+
+        option :tmp_dir_prefix,
+               long: '--tmp-dir-prefix PREFIX',
+               description: 'Tmp directory prefix (/tmp by default). Used for build process service directories.'
+
+        option :lock_timeout,
+               long: '--lock-timeout TIMEOUT',
+               description: 'Redefine resource locking timeout (in seconds)',
+               proc: ->(v) { v.to_i }
+
+        option :build_context_directory,
+               long: '--build-context-directory DIR_PATH',
+               default: nil
+
+        option :use_system_tar,
+               long: '--use-system-tar',
+               boolean: true,
+               default: false
+
+        option :force_save_cache,
+               long: '--force-save-cache',
+               boolean: true,
+               default: false
+      end
+    end
+  end
+end

data/lib/dapp/dimg/cli/{build_context → command/dimg/build_context}/export.rb

@@ -1,7 +1,7 @@
-module Dapp
-  module Dimg
-    module CLI
-      class BuildContext
+module Dapp::Dimg::CLI
+  module Command
+    class Dimg < ::Dapp::CLI
+      class BuildContext < ::Dapp::CLI
         class Export < Base
           banner <<BANNER.freeze
 Usage:
@@ -16,6 +16,11 @@ BANNER
                  long: '--build-context-directory DIR_PATH',
                  description: 'Path to the directory with context'
 
+          option :use_system_tar,
+                 long: '--use-system-tar',
+                 boolean: true,
+                 default: false
+
           def run_method
             :"build_context_#{class_to_lowercase}"
           end

data/lib/dapp/dimg/cli/{build_context → command/dimg/build_context}/import.rb

@@ -1,7 +1,7 @@
-module Dapp
-  module Dimg
-    module CLI
-      class BuildContext
+module Dapp::Dimg::CLI
+  module Command
+    class Dimg < ::Dapp::CLI
+      class BuildContext < ::Dapp::CLI
         class Import < Export
           banner <<BANNER.freeze
 Usage:

data/lib/dapp/dimg/cli/{build_context.rb → command/dimg/build_context.rb}

@@ -1,14 +1,14 @@
-module Dapp
-  module Dimg
-    module CLI
+module Dapp::Dimg::CLI
+  module Command
+    class Dimg < ::Dapp::CLI
       class BuildContext < ::Dapp::CLI
         SUBCOMMANDS = %w(import export).freeze
 
         banner <<BANNER.freeze
 Available subcommands: (for details, dapp dimg build-context SUB-COMMAND --help)
 
-dapp dimg build-context export [options] [DIMG ...]
-dapp dimg build-context import [options]
+  dapp dimg build-context export [options] [DIMG ...]
+  dapp dimg build-context import [options]
 
 Options:
 BANNER

data/lib/dapp/dimg/cli/{cleanup.rb → command/dimg/cleanup.rb}

@@ -1,13 +1,11 @@
-module Dapp
-  module Dimg
-    module CLI
+module Dapp::Dimg::CLI
+  module Command
+    class Dimg < ::Dapp::CLI
       class Cleanup < Base
         banner <<BANNER.freeze
 Usage:
 
-  dapp dimg cleanup [options] [DIMG ...]
-
-    DIMG                        Dapp image to process [default: *].
+  dapp dimg cleanup [options]
 
 Options:
 BANNER

data/lib/dapp/dimg/cli/command/dimg/cleanup_repo.rb

@@ -0,0 +1,47 @@
+module Dapp::Dimg::CLI
+  module Command
+    class Dimg < ::Dapp::CLI
+      class CleanupRepo < Base
+        banner <<BANNER.freeze
+Usage:
+
+  dapp dimg cleanup repo [options] [DIMG ...] REPO
+
+Options:
+BANNER
+
+        option :lock_timeout,
+               long: '--lock-timeout TIMEOUT',
+               description: 'Redefine resource locking timeout (in seconds)',
+               proc: ->(v) { v.to_i }
+
+        option :kubernetes_timeout,
+               long: '--kubernetes-timeout TIMEOUT',
+               description: 'Kubernetes api-server tcp connection, read and write timeout (in seconds)',
+               proc: ->(v) { v.to_i }
+
+        option :with_stages,
+               long: '--with-stages',
+               boolean: true
+
+        option :without_kube,
+               long: '--without-kube',
+               boolean: true
+
+        option :registry_username,
+               long: '--registry-username USERNAME',
+               default: (ENV.key?("DAPP_DIMG_CLEANUP_REGISTRY_PASSWORD") ? "dapp-cleanup-repo" : nil) # FIXME: https://gitlab.com/gitlab-org/gitlab-ce/issues/41384
+
+        option :registry_password,
+               long: '--registry-password PASSWORD',
+               default: ENV["DAPP_DIMG_CLEANUP_REGISTRY_PASSWORD"] # FIXME: https://gitlab.com/gitlab-org/gitlab-ce/issues/41384
+
+        def run(argv = ARGV)
+          self.class.parse_options(self, argv)
+          repo = self.class.required_argument(self, 'repo')
+          run_dapp_command(run_method, options: cli_options(dimgs_patterns: cli_arguments, repo: repo, verbose: true))
+        end
+      end
+    end
+  end
+end

data/lib/dapp/dimg/cli/command/dimg/flush_local.rb

@@ -0,0 +1,24 @@
+module Dapp::Dimg::CLI
+  module Command
+    class Dimg < ::Dapp::CLI
+      class FlushLocal < Base
+        banner <<BANNER.freeze
+Usage:
+
+  dapp dimg flush local [options] [DIMG ...]
+
+Options:
+BANNER
+
+        option :with_stages,
+               long: '--with-stages',
+               boolean: true
+
+        def run(argv = ARGV)
+          self.class.parse_options(self, argv)
+          run_dapp_command(run_method, options: cli_options(dimgs_patterns: cli_arguments, verbose: true))
+        end
+      end
+    end
+  end
+end