dapp 0.32.10 → 0.33.0

data/lib/dapp/dapp/shellout/base.rb

@@ -58,7 +58,7 @@ module Dapp
 
         protected
 
-        def _shellout_with_logging!(*args, verbose: false, quiet: true, time: false, **kwargs)
+        def _shellout_with_logging!(*args, verbose: false, quiet: true, time: false, raise_on_error: true, **kwargs)
           stream = Stream.new
           if verbose && !quiet
             kwargs[:live_stream] = Proxy::Base.new(stream, STDOUT, with_time: time)
@@ -68,7 +68,9 @@ module Dapp
             kwargs[:live_stderr] = Proxy::Error.new(stream, with_time: time)
           end
 
-          shellout(*args, **kwargs).tap(&:error!)
+          shellout(*args, **kwargs).tap do |res|
+            res.error! if raise_on_error
+          end
         rescue ::Mixlib::ShellOut::ShellCommandFailed => e
           raise Error::Shellout, code: Helper::Trivia.class_to_lowercase(e.class), data: { stream: stream.show }
         end

data/lib/dapp/dimg/build/stage/artifact_base.rb

@@ -56,12 +56,13 @@ module Dapp
 
           def run_artifact_dimg(artifact_dimg, commands)
             docker_options = ['--rm',
-                              "--volume #{dimg.tmp_path('artifact', artifact_dimg.name )}:#{artifact_dimg.container_tmp_path(artifact_dimg.name)}",
-                              "--volumes-from #{dimg.dapp.toolchain_container}",
-                              "--volumes-from #{dimg.dapp.base_container}",
-                              "--entrypoint #{dimg.dapp.bash_bin}"]
-            dimg.dapp.log_secondary_process(dimg.dapp.t(code: 'process.artifact_copy', data: { name: artifact_dimg.name }), short: true) do
-              artifact_dimg.run(docker_options, [%(-ec '#{dimg.dapp.shellout_pack(commands)}')])
+                              "--volume=#{dimg.tmp_path('artifact', artifact_dimg.name )}:#{artifact_dimg.container_tmp_path(artifact_dimg.name)}",
+                              "--volumes-from=#{dimg.dapp.toolchain_container}",
+                              "--volumes-from=#{dimg.dapp.base_container}",
+                              "--entrypoint=#{dimg.dapp.bash_bin}"]
+            dimg.dapp.log_secondary_process(dimg.dapp.t(code: 'process.artifact_copy', data:
+              { name: artifact_dimg.name }), short: true) do
+              artifact_dimg.run(docker_options, ["-ec", dimg.dapp.shellout_pack(commands)])
             end
           rescue Error::Dimg => e
             raise unless e.net_status[:code] == :dimg_not_run

data/lib/dapp/dimg/build/stage/base.rb

@@ -17,6 +17,39 @@ module Dapp
             @next_stage.prev_stage = self
           end
 
+          def get_ruby2go_state_hash
+            {}.tap {|hash|
+              # NOTICE: Delete this when stage has been moved to go.
+              # NOTICE: This is data for build.StubStage and build.StubImage.
+
+              hash["Image"] = {
+                "Labels" => image.labels
+                  .map{|k,v| [k.to_s, v.to_s]}
+                  .to_h,
+                "ServiceChangeLabels" => image.send(:service_change_options)
+                  .fetch(:label, {})
+                  .map{|k,v| [k.to_s, v.to_s]}
+                  .to_h,
+              }
+
+              if prev_stage
+                hash["PrevStage"] = prev_stage.get_ruby2go_state_hash
+              end
+            }
+          end
+
+          def set_ruby2go_state_hash(hash)
+            if prev_stage
+              prev_stage.set_ruby2go_state_hash(hash["PrevStage"])
+            end
+
+            # NOTICE: This is data from build.StubImage.
+
+            hash["Image"]["ServiceChangeLabels"].each do |k, v|
+              image.add_service_change_label(k.to_sym => v)
+            end
+          end
+
           # rubocop:disable Metrics/PerceivedComplexity
           def build_lock!
             return yield if dimg.dapp.dry_run?
@@ -27,14 +60,11 @@ module Dapp
               no_lock = false
 
               dimg.dapp.lock("#{dimg.dapp.name}.image.#{image.name}") do
-                image.class.reset_image_inspect(image.name)
+                image.reset_image_inspect
 
                 if image_should_be_locked?
                   yield
                 else
-                  stage = self
-                  stage.renew until (stage = stage.next_stage).nil?
-
                   no_lock = true
                 end
               end
@@ -84,7 +114,7 @@ module Dapp
 
             if dimg.dapp.ssh_auth_sock
               image.add_volume "#{dimg.dapp.ssh_auth_sock}:/tmp/dapp-ssh-agent"
-              image.add_env 'SSH_AUTH_SOCK', '/tmp/dapp-ssh-agent'
+              image.add_env SSH_AUTH_SOCK: '/tmp/dapp-ssh-agent'
             end
 
             yield if block_given?
@@ -211,8 +241,8 @@ module Dapp
 
           def renew
             commits_discard
-            image_reset
             image_untag! if image_should_be_untagged?
+            image_reset
           end
 
           def image_reset
@@ -249,7 +279,7 @@ module Dapp
 
           def introspect_image_default(introspected_image)
             if introspected_image.built?
-              dimg.introspect_image!(image: introspected_image.built_id, options: image.send(:prepared_options))
+              introspected_image.introspect!
             else
               dimg.dapp.log_warning(desc: { code: :introspect_image_impossible, data: { name: name } })
             end

data/lib/dapp/dimg/build/stage/from.rb

@@ -13,7 +13,7 @@ module Dapp
             if !from_dimg.nil?
               process = dimg.dapp.t(code: 'process.layer_building', data: { name: from_dimg.name })
               dimg.dapp.log_secondary_process(process) { from_dimg.build! }
-            else
+            elsif !from_image.tagged?
               try_host_docker_login
               from_image.pull!
               raise Error::Build, code: :from_image_not_found, data: { name: from_image_name } unless from_image.tagged?

data/lib/dapp/dimg/builder/ansible.rb

@@ -1,213 +1,5 @@
 module Dapp
   module Dimg
-    class Builder::Ansible < Builder::Base
-
-      ANSIBLE_IMAGE_VERSION = "2.4.4.0-10"
-
-      def ansible_bin
-        "/.dapp/deps/ansible/#{ANSIBLE_IMAGE_VERSION}/embedded/bin/ansible"
-      end
-
-      def ansible_playbook_bin
-        "/.dapp/deps/ansible/#{ANSIBLE_IMAGE_VERSION}/embedded/bin/ansible-playbook"
-      end
-
-      def python_path
-        "/.dapp/deps/ansible/#{ANSIBLE_IMAGE_VERSION}/embedded/bin/python"
-      end
-
-      def ansible_playbook_solo_cmd
-        "#{ansible_playbook_bin} -c local"
-      end
-
-      def ansible_image
-        "dappdeps/ansible:#{ANSIBLE_IMAGE_VERSION}"
-      end
-
-      def ansible_container_name
-        "dappdeps_ansible_#{ANSIBLE_IMAGE_VERSION}"
-      end
-
-      def ansible_container
-        @ansible_container ||= begin
-          is_container_exist = proc{dimg.dapp.shellout("#{dimg.dapp.host_docker} inspect #{ansible_container_name}").exitstatus.zero?}
-          if !is_container_exist.call
-            dimg.dapp.lock("dappdeps.container.#{ansible_container_name}", default_timeout: 600) do
-              if !is_container_exist.call
-                dimg.dapp.log_secondary_process(dimg.dapp.t(code: 'process.ansible_container_creating', data: {name: ansible_container_name}), short: true) do
-                  dimg.dapp.shellout!(
-                    ["#{dimg.dapp.host_docker} create",
-                     "--name #{ansible_container_name}",
-                     "--volume /.dapp/deps/ansible/#{ANSIBLE_IMAGE_VERSION} #{ansible_image}"].join(' ')
-                  )
-                end
-              end
-            end
-          end
-          ansible_container_name
-        end
-      end
-
-      # query tasks from ansible config
-      # create dump_config structure
-      # returns structure:
-      # { 'tasks' => [array of tasks for stage],
-      #   'dump_config' => {
-      #      'dump_config_doc' => 'dump of doc',
-      #      'dump_config_sections' => {'task_0'=>'dump for task 0', 'task_1'=>'dump for task 1', ... }}
-      def stage_config(stage)
-        @stage_configs ||= {}
-        @stage_configs[stage.to_s] ||= begin
-          {}.tap do |stage_config|
-            stage_config['dump_config'] = {
-              'dump_config_doc' => dimg.config._ansible['dump_config_doc'],
-              'dump_config_sections' => {},
-            }
-            stage_config['tasks'] = dimg.config._ansible[stage.to_s].map.with_index do |dapp_task, task_num|
-              {}.tap do |task|
-                task.merge!(dapp_task['config'])
-                task['tags'] = [].tap do |tags|
-                  tags << dapp_task['config']['tags']
-                  dump_tag = "task_#{task_num}"
-                  tags << dump_tag
-                  stage_config['dump_config']['dump_config_sections'][dump_tag] = dapp_task['dump_config_section']
-                end.flatten.compact
-              end
-            end || []
-          end
-        end
-      end
-
-      def stage_playbook(stage)
-        @stage_playbooks ||= {}
-        @stage_playbooks[stage.to_s] ||= begin
-          [{
-            'hosts' => 'all',
-            'gather_facts' => 'no',
-            'tasks' => [],
-           }].tap do |playbook|
-            playbook[0]['tasks'].concat(stage_config(stage)['tasks'])
-          end
-        end
-      end
-
-      def create_workdir_structure(stage)
-        @workdir_structures ||= {}
-        @workdir_structures[stage.to_s] ||= true.tap do
-          host_workdir(stage).tap do |workdir|
-            # playbook with tasks for a stage
-            workdir.join('playbook.yml').write YAML.dump(stage_playbook(stage))
-            #puts YAML.dump(stage_playbook(stage))
-
-            # generate inventory with localhost and python in dappdeps-ansible
-            workdir.join('hosts').write Assets.hosts(python_path)
-
-            # generate ansible config for solo mode
-            workdir.join('ansible.cfg').write Assets.ansible_cfg(container_workdir.join('hosts'),
-                                                                 container_workdir.join('lib', 'callback'),
-                                                                 dimg.dapp.sudo_bin,
-                                                                 container_tmpdir.join('local'),
-                                                                 container_tmpdir.join('remote'),
-                                                                 )
-
-            # save config dump for pretty errors
-            workdir.join('dump_config.json').write JSON.generate(stage_config(stage)['dump_config'])
-
-            # python modules
-            workdir.join('lib').tap do |libdir|
-              libdir.mkpath
-              # crypt.py hack
-              # TODO must be in dappdeps-ansible
-              libdir.join('crypt.py').write Assets.crypt_py
-              libdir.join('callback').tap do |callbackdir|
-                callbackdir.mkpath
-                callbackdir.join('__init__.py').write '# module callback'
-                callbackdir.join('live.py').write Assets.live_py
-                # add dapp specific stdout callback for ansible
-                callbackdir.join('dapp.py').write Assets.dapp_py
-              end
-            end
-          end
-        end
-      end
-
-      %i(before_install install before_setup setup build_artifact).each do |stage|
-        define_method("#{stage}?") {
-          !stage_empty?(stage)
-        }
-
-        define_method("#{stage}_checksum") do
-          checksum_args = []
-          (dimg.config._ansible[stage.to_s] || []).each do |task|
-            checksum_args << JSON.dump(task["config"])
-          end
-          checksum_args << public_send("#{stage}_version_checksum")
-          _checksum checksum_args
-        end
-
-        define_method("#{stage}_version_checksum") do
-          _checksum(dimg.config._ansible["#{stage}_version"], dimg.config._ansible['version'])
-        end
-
-        define_method(stage.to_s) do |image|
-          unless stage_empty?(stage)
-            create_workdir_structure(stage)
-            image.add_env('ANSIBLE_CONFIG', container_workdir.join('ansible.cfg'))
-            image.add_env('DAPP_DUMP_CONFIG_DOC_PATH', container_workdir.join('dump_config.json'))
-            image.add_env('PYTHONPATH', container_workdir.join('lib'))
-            image.add_env('PYTHONIOENCODING', 'utf-8')
-            image.add_env('ANSIBLE_PREPEND_SYSTEM_PATH', dimg.dapp.dappdeps_base_path)
-            image.add_volumes_from("#{ansible_container}:rw")
-            image.add_volume "#{host_workdir(stage)}:#{container_workdir}:ro"
-            image.add_volume "#{host_tmpdir(stage)}:#{container_tmpdir}:rw"
-            image.add_command [ansible_playbook_bin,
-                               container_workdir.join('playbook.yml'),
-                               ENV['ANSIBLE_ARGS']
-                              ].compact.join(' ')
-          end
-        end
-      end
-
-      def before_build_check
-      end
-
-      def before_dimg_should_be_built_check
-      end
-
-      def stage_empty?(stage)
-        stage_config(stage)['tasks'].empty? && public_send("#{stage}_version_checksum").nil?
-      end
-
-      # host directory in tmp_dir with directories structure
-      def host_workdir(stage)
-        @host_workdirs ||= {}
-        @host_workdirs[stage.to_s] ||= begin
-          dimg.tmp_path(dimg.dapp.consistent_uniq_slugify(dimg.config._name || "nameless"), "ansible-workdir-#{stage}").tap {|p| p.mkpath}
-        end
-      end
-
-      # temporary directories for ansible
-      def host_tmpdir(stage)
-        @host_tmpdirs ||= {}
-        @host_tmpdirs[stage.to_s] ||= begin
-          dimg.tmp_path(dimg.dapp.consistent_uniq_slugify(dimg.config._name || "nameless"), "ansible-tmpdir-#{stage}").tap do |p|
-            p.mkpath
-            p.join('local').mkpath
-            p.join('remote').mkpath
-          end
-        end
-      end
-
-      # directory with playbook in container
-      def container_workdir
-        dimg.container_dapp_path("ansible-workdir")
-      end
-
-      # temporary directory for ansible
-      def container_tmpdir
-        dimg.container_dapp_path("ansible-tmpdir")
-      end
-
-    end # Builder::Ansible
+    class Builder::Ansible < Builder::Ruby2Go; end
   end # Dimg
 end # Dapp

data/lib/dapp/dimg/builder/base.rb

@@ -72,11 +72,6 @@ module Dapp
       def build_artifact_checksum
         raise
       end
-
-      def _checksum(*args)
-        return if (format_args = args.flatten.compact.delete_if { |val| val.respond_to?(:empty?) && val.empty? }).empty?
-        dimg.hashsum format_args
-      end
     end # Builder::Base
   end # Dimg
 end # Dapp

data/lib/dapp/dimg/builder/none.rb

@@ -1,38 +1,5 @@
 module Dapp
   module Dimg
-    class Builder::None < Builder::Base
-      def before_dimg_should_be_built_check
-      end
-
-      def before_install(_image)
-      end
-
-      def before_install_checksum
-      end
-
-      def before_setup(_image)
-      end
-
-      def before_setup_checksum
-      end
-
-      def install(_image)
-      end
-
-      def install_checksum
-      end
-
-      def setup(_image)
-      end
-
-      def setup_checksum
-      end
-
-      def build_artifact(_image)
-      end
-
-      def build_artifact_checksum
-      end
-    end # Builder::None
+    class Builder::None < Builder::Ruby2Go; end
   end # Dimg
 end # Dapp

data/lib/dapp/dimg/builder/ruby2go.rb

@@ -0,0 +1,51 @@
+module Dapp
+  module Dimg
+    class Builder::Ruby2Go < Builder::Base
+      [:before_install, :before_setup, :install, :setup, :build_artifact].each do |stage|
+        define_method("#{stage}_checksum") do
+          command = "#{snake_case_to_camel_case(stage)}Checksum"
+          ruby2go_builder_command(command: command)
+        end
+
+        define_method("#{stage}?") do
+          command = "Is#{snake_case_to_camel_case(stage)}Empty"
+          !ruby2go_builder_command(command: command)
+        end
+
+        define_method(stage.to_s) do |image|
+          command = snake_case_to_camel_case(stage)
+          ruby2go_builder_command(command: command, image: image.ruby2go_image_option).tap do |data|
+            image.set_ruby2go_state_hash(JSON.load(data["image"]))
+          end
+        end
+      end
+
+      def ruby2go_builder_command(command:, **options)
+        builder = self.class.name.split("::").last.downcase
+        command_options = {
+          builder: builder,
+          command: command,
+          config: YAML.dump(dimg.config),
+          extra: get_ruby2go_state_hash,
+          artifact: dimg.artifact?
+        }.merge(options)
+
+        dimg.dapp.ruby2go_builder(command_options).tap do |res|
+          raise Error::Build, code: :ruby2go_builder_command_failed_unexpected_error, data: { command: command, message: res["error"] } unless res["error"].nil?
+          break res['data']
+        end
+      end
+
+      def get_ruby2go_state_hash
+        {
+          "TmpPath" => dimg.tmp_path.to_s,
+          "ContainerDappPath" => dimg.container_dapp_path.to_s,
+        }
+      end
+
+      def snake_case_to_camel_case(value)
+        value.to_s.split('_').collect(&:capitalize).join
+      end
+    end # Builder::Shell
+  end # Dimg
+end # Dapp