dapp 0.10.2 → 0.36.24

data/lib/dapp/dimg/dapp/dimg.rb

@@ -0,0 +1,41 @@
+module Dapp
+  module Dimg
+    module Dapp
+      module Dimg
+        def dimg(config:, **kwargs)
+          dimg_after_define_hook(config: config, **kwargs) do
+            (@dimg ||= {})[config._name] ||= ::Dapp::Dimg::Dimg.new(config: config, dapp: self, **kwargs)
+          end
+        end
+
+        def artifact_dimg(config:, **kwargs)
+          dimg_after_define_hook(config: config, **kwargs) do
+            (@artifacts_dimgs ||= {})[config._name] ||= ::Dapp::Dimg::Artifact.new(config: config, dapp: self, **kwargs)
+          end
+        end
+
+        def dimg_after_define_hook(**kwargs)
+          should_be_built = kwargs[:should_be_built] || false
+          yield.tap do |dimg|
+            if should_be_built && dimg.should_be_built != should_be_built
+              dimg.enable_should_be_built
+              dimg.should_be_built!
+            end
+          end
+        end
+
+        def dimg_layer(config:, **kwargs)
+          (@dimg_layers ||= {})[config._name] ||= ::Dapp::Dimg::Dimg.new(config: config, dapp: self, **kwargs)
+        end
+
+        def artifact_dimg_layer(config:, **kwargs)
+          (@artifact_dimg_layers ||= {})[config._name] ||= ::Dapp::Dimg::Artifact.new(config: config, dapp: self, **kwargs)
+        end
+
+        def _terminate_dimg_on_terminate(dimg)
+          @_call_before_terminate << proc{dimg.terminate}
+        end
+      end # Dimg
+    end # Dapp
+  end # Dimg
+end # Dapp

data/lib/dapp/dimg/dimg/git_artifact.rb

@@ -2,24 +2,38 @@ module Dapp
   module Dimg
     class Dimg
       module GitArtifact
-        def git_artifacts
-          [*local_git_artifacts, *remote_git_artifacts].compact
+        def git_artifacts(omit_empty: true)
+          [*local_git_artifacts(omit_empty: omit_empty), *remote_git_artifacts(omit_empty: omit_empty)].compact
         end
 
-        def local_git_artifacts
-          @local_git_artifact_list ||= Array(config._git_artifact._local).map do |ga_config|
-            repo = GitRepo::Own.new(self)
-            ::Dapp::Dimg::GitArtifact.new(repo, **ga_config._artifact_options)
+        def local_git_artifacts(omit_empty: true)
+          @local_git_artifact_list ||= [].tap do |artifacts|
+            break artifacts if (local_git_artifacts = Array(config._git_artifact._local)).empty?
+            repo = GitRepo::Own.new(dapp)
+            local_git_artifacts.map do |ga_config|
+              artifacts.concat(generate_git_artifacts(repo, omit_empty: omit_empty, **ga_config._artifact_options))
+            end unless repo.empty?
           end
         end
 
-        def remote_git_artifacts
-          @remote_git_artifact_list ||= Array(config._git_artifact._remote).map do |ga_config|
-            repo = GitRepo::Remote.new(self, ga_config._name, url: ga_config._url)
-            repo.fetch!(ga_config._branch)
-            ::Dapp::Dimg::GitArtifact.new(repo, **ga_config._artifact_options)
+        def remote_git_artifacts(omit_empty: true)
+          @remote_git_artifact_list ||= [].tap do |artifacts|
+            Array(config._git_artifact._remote).each do |ga_config|
+              repo = GitRepo::Remote.get_or_create(dapp, ga_config._name, url: ga_config._url)
+              artifacts.concat(generate_git_artifacts(repo, omit_empty: omit_empty, **ga_config._artifact_options)) unless repo.empty?
+            end
           end
         end
+
+        def generate_git_artifacts(repo, omit_empty: true, **git_artifact_options)
+          [].tap do |artifacts|
+            artifacts << (artifact = ::Dapp::Dimg::GitArtifact.new(repo, self, ignore_signature_auto_calculation: ignore_signature_auto_calculation, **git_artifact_options))
+            artifacts
+          end.select do |artifact|
+            !omit_empty || !artifact.is_empty
+          end
+        end
+
       end # GitArtifact
     end # Mod
   end # Dimg

data/lib/dapp/dimg/dimg/path.rb

@@ -3,16 +3,20 @@ module Dapp
     class Dimg
       module Path
         def home_path(*path)
-          make_path(dapp.path, *path).expand_path
+          dapp.path(*path).expand_path
+        end
+
+        def tmp_dir_exists?
+          @tmp_path != nil
         end
 
         def tmp_path(*path)
-          @tmp_path ||= Dir.mktmpdir('dapp-', dapp.cli_options[:tmp_dir_prefix] || '/tmp')
+          @tmp_path ||= Dir.mktmpdir('dapp-', dapp.tmp_base_dir)
           make_path(@tmp_path, *path).expand_path.tap { |p| p.parent.mkpath }
         end
 
         def build_path(*path)
-          make_path(dapp.build_path, *path).expand_path.tap { |p| p.parent.mkpath }
+          dapp.build_path(*path).expand_path.tap { |p| p.parent.mkpath }
         end
 
         def container_dapp_path(*path)
@@ -25,12 +29,6 @@ module Dapp
 
         alias build_dir build_path
         alias tmp_dir tmp_path
-
-        private
-
-        def make_path(base, *path)
-          path.compact.map(&:to_s).inject(Pathname.new(base), &:+)
-        end
       end # Path
     end
   end # Dimg

data/lib/dapp/dimg/dimg/stages.rb

@@ -3,7 +3,7 @@ module Dapp
     class Dimg
       module Stages
         def signature
-          last_stage.send(:signature)
+          last_stage.signature
         end
 
         def stage_by_name(name)
@@ -18,31 +18,39 @@ module Dapp
           dapp.stage_dapp_label
         end
 
-        def tagged_images
-          all_images.select(&:tagged?)
-        end
-        alias export_images tagged_images
-
-        def all_images
-          @all_images ||= all_stages.map(&:image).uniq!(&:name)
+        def all_tagged_images
+          tagged_images.concat(artifacts.map(&:all_tagged_images).flatten).uniq(&:name)
         end
 
-        def all_stages
-          stages + artifacts.map(&:all_stages).flatten
+        def last_stage
+          @last_stage || begin
+            (self.last_stage = last_stage_class.new(self)).tap do |stage|
+              dapp.log_secondary_process("#{name || 'nameless'}: calculating stages signatures") do
+                stage.signature
+              end unless ignore_signature_auto_calculation
+            end
+          end
         end
 
         protected
 
-        def last_stage
-          @last_stage ||= if scratch?
-            Build::Stage::ImportArtifact.new(self)
+        attr_writer :last_stage
+
+        def last_stage_class
+          if scratch?
+            Build::Stage::ImportArtifact
           else
-            Build::Stage::DockerInstructions.new(self)
+            Build::Stage::DockerInstructions
           end
         end
 
+        def tagged_images
+          images.select(&:tagged?)
+        end
+        alias export_images tagged_images
+
         def import_images
-          all_images.select { |image| !image.tagged? }
+          images.select { |image| !image.tagged? }
         end
 
         def artifacts_stages
@@ -58,6 +66,10 @@ module Dapp
             end
           end
         end
+
+        def images
+          stages.map(&:image).uniq(&:name)
+        end
       end # Stages
     end # Mod
   end # Dimg

data/lib/dapp/dimg/dimg.rb

@@ -3,91 +3,166 @@ module Dapp
     class Dimg
       include GitArtifact
       include Path
-      include Tags
       include Stages
 
       include Helper::Sha256
+      include Helper::Trivia
 
       attr_reader :config
-      attr_reader :ignore_git_fetch
+      attr_reader :ignore_signature_auto_calculation
       attr_reader :should_be_built
       attr_reader :dapp
 
-      def initialize(config:, dapp:, should_be_built: false, ignore_git_fetch: false)
+      def get_ruby2go_state_hash
+        {
+          "Dapp" => dapp.get_ruby2go_state_hash,
+          "TmpPath" => tmp_path.to_s,
+        }
+      end
+
+      def initialize(config:, dapp:, should_be_built: false, ignore_signature_auto_calculation: false)
         @config = config
         @dapp = dapp
 
-        @ignore_git_fetch = ignore_git_fetch
-        @should_be_built = should_be_built
+        @ignore_signature_auto_calculation = ignore_signature_auto_calculation
+
+        @dapp._terminate_dimg_on_terminate(self)
+
+        enable_should_be_built if should_be_built
+        should_be_built!
+      end
+
+      def enable_should_be_built
+        @should_be_built = true
+      end
 
+      def should_be_built!
         raise Error::Dimg, code: :dimg_not_built if should_be_built?
       end
 
+      def name
+        config._name
+      end
+
+      def terminate
+        cleanup_tmp
+      end
+
       def build!
-        with_introspection do
-          dapp.lock("#{dapp.name}.images", readonly: true) do
-            last_stage.build_lock! do
-              begin
-                builder.before_build_check
-                last_stage.build!
-              ensure
-                after_stages_build!
-              end
+        dapp.lock("#{dapp.name}.images", readonly: true) do
+          last_stage.build_lock! do
+            begin
+              builder.before_build_check
+              last_stage.build!
+            ensure
+              after_stages_build!
             end
           end
         end
-      ensure
-        cleanup_tmp
       end
 
       def after_stages_build!
-        return unless last_stage.image.built? || dev_mode?
+        return unless last_stage.image.built? || dev_mode? || force_save_cache?
         last_stage.save_in_cache!
         artifacts.each { |artifact| artifact.last_stage.save_in_cache! }
       end
 
-      def tag!(tag)
+      def tag!(repo, format:)
+        dimg_export_base!(repo, export_format: format)
+      end
+
+      def export!(repo, format:)
+        dimg_export_base!(repo, export_format: format, push: true)
+      end
+
+      def export_stages!(repo, format:)
         dapp.lock("#{dapp.name}.images", readonly: true) do
-          dimg_name = config._name
-          if dapp.dry_run?
-            dapp.log_state(dimg_name, state: dapp.t(code: 'state.tag'), styles: { status: :success })
-          else
-            dapp.log_process(dimg_name, process: dapp.t(code: 'status.process.tagging')) do
-              last_stage.image.tag!(tag)
+          export_images.each do |stage_image|
+            signature = stage_image.name.split(':').last
+            image_name = format(format, repo: repo, signature: signature)
+
+            if dimgstage_should_not_be_pushed?(format(dapp.dimgstage_push_tag_format, signature: signature))
+              dapp.log_state(image_name, state: dapp.t(code: 'state.exist'))
+              next
+            end
+
+            export_base!(image_name, push: true) do
+              stage_image.export!(image_name)
             end
           end
+          artifacts.each { |artifact| artifact.export_stages!(repo, format: format) }
         end
       end
 
-      def export!(repo, format:)
+      def dimg_export_base!(repo, export_format:, push: false)
         dapp.lock("#{dapp.name}.images", readonly: true) do
-          tags.each do |tag|
-            image_name = format(format, repo: repo, dimg_name: config._name, tag: tag)
-            export_base!(last_stage.image, image_name)
+          dapp.tags_by_scheme.each do |tag_scheme_name, tags|
+            dapp.log_step_with_indent(tag_scheme_name) do
+              tags.each do |tag|
+                image_name = format(export_format, repo: repo, dimg_name: name, tag: tag)
+
+                if push && tag_should_not_be_pushed?(tag.to_s)
+                  dapp.log_state(image_name, state: dapp.t(code: 'state.exist'))
+                  next
+                end
+
+                export_base!(image_name, push: push) do
+                  export_image = build_export_image!(image_name, scheme_name: tag_scheme_name)
+                  if push
+                    export_image.export!
+                  else
+                    export_image.tag!
+                  end
+                end
+              end
+            end unless tags.empty?
           end
         end
       end
 
-      def export_stages!(repo, format:)
-        dapp.lock("#{dapp.name}.images", readonly: true) do
-          export_images.each do |image|
-            image_name = format(format, repo: repo, signature: image.name.split(':').last)
-            export_base!(image, image_name)
+      def dimgstage_should_not_be_pushed?(signature)
+        registry_dimgstages_tags.include?(signature)
+      end
+
+      def tag_should_not_be_pushed?(tag)
+        registry_tags.include?(tag) && begin
+          registry_tag_parent = registry.image_parent_id(tag, name)
+          registry_tag_parent == last_stage.image.built_id
+        end
+      end
+
+      def registry_tags
+        @registry_tags ||= begin
+          if name.nil?
+            registry.nameless_dimg_tags
+          else
+            registry.dimg_tags(name)
           end
         end
       end
 
-      def export_base!(image, image_name)
+      def registry_dimgstages_tags
+        @registry_dimgstages_tags ||= registry.dimgstages_tags
+      end
+
+      def registry
+        @registry ||= dapp.dimg_registry
+      end
+
+      def build_export_image!(image_name, scheme_name:)
+        Image::Dimg.image_by_name(name: image_name, from: last_stage.image, dapp: dapp).tap do |export_image|
+          export_image.add_service_change_label(:'dapp-tag-scheme' => scheme_name)
+          export_image.add_service_change_label(:'dapp-dimg' => true)
+          export_image.build!
+        end
+      end
+
+      def export_base!(image_name, push: true)
         if dapp.dry_run?
-          dapp.log_state(image_name, state: dapp.t(code: 'state.push'), styles: { status: :success })
+          dapp.log_state(image_name, state: dapp.t(code: push ? 'state.push' : 'state.export'), styles: { status: :success })
         else
           dapp.lock("image.#{hashsum image_name}") do
-            ::Dapp::Dimg::Image::Stage.cache_reset(image_name)
-            dapp.log_process(image_name, process: dapp.t(code: 'status.process.pushing')) do
-              dapp.with_log_indent do
-                image.export!(image_name)
-              end
-            end
+            dapp.log_process(image_name, process: dapp.t(code: push ? 'status.process.pushing' : 'status.process.exporting')) { yield }
           end
         end
       end
@@ -95,14 +170,23 @@ module Dapp
       def import_stages!(repo, format:)
         dapp.lock("#{dapp.name}.images", readonly: true) do
           import_images.each do |image|
+            signature = image.name.split(':').last
+            image_name = format(format, repo: repo, signature:signature )
+
+            unless dimgstage_should_not_be_pushed?(format(dapp.dimgstage_push_tag_format, signature: signature))
+              dapp.log_state(image_name, state: dapp.t(code: 'state.not_exist'))
+              next
+            end
+
             begin
-              image_name = format(format, repo: repo, signature: image.name.split(':').last)
               import_base!(image, image_name)
-            rescue ::Dapp::Error::Shellout
+            rescue ::Dapp::Error::Shellout => e
+              dapp.log_info ::Dapp::Helper::NetStatus.message(e)
               next
             end
-            break unless dapp.pull_all_stages?
+            break unless !!dapp.options[:pull_all_stages]
           end
+          artifacts.each { |artifact| artifact.import_stages!(repo, format: format) }
         end
       end
 
@@ -111,9 +195,10 @@ module Dapp
           dapp.log_state(image_name, state: dapp.t(code: 'state.pull'), styles: { status: :success })
         else
           dapp.lock("image.#{hashsum image_name}") do
-            dapp.log_process(image_name, process: dapp.t(code: 'status.process.pulling'),
-                                         status: { failed: dapp.t(code: 'status.failed.not_pulled') },
-                                         style: { failed: :secondary }) do
+            dapp.log_process(image_name,
+                             process: dapp.t(code: 'status.process.pulling'),
+                             status: { failed: dapp.t(code: 'status.failed.not_pulled') },
+                             style: { failed: :secondary }) do
               image.import!(image_name)
             end
           end
@@ -121,16 +206,23 @@ module Dapp
       end
 
       def run(docker_options, command)
-        cmd = "docker run #{[docker_options, last_stage.image.built_id, command].flatten.compact.join(' ')}"
+        run_stage(nil, docker_options, command)
+      end
+
+      def run_stage(stage_name, docker_options, command)
+        stage_image = (stage_name.nil? ? last_stage : stage_by_name(stage_name)).image
+        raise Error::Dimg, code: :dimg_stage_not_built, data: { stage_name: stage_name } unless stage_image.built?
+
+        args = [docker_options, stage_image.built_id, command].flatten.compact
         if dapp.dry_run?
-          dapp.log(cmd)
+          dapp.log("docker run #{args.join(' ')}")
         else
-          system(cmd) || raise(Error::Dimg, code: :dimg_not_run)
+          Image::Stage.ruby2go_command(dapp, command: :container_run, options: { args: args })
         end
       end
 
       def stage_image_name(stage_name)
-        stages.find { |stage| stage.send(:name) == stage_name }.image.name
+        stages.find { |stage| stage.name == stage_name }.image.name
       end
 
       def builder
@@ -146,29 +238,48 @@ module Dapp
       end
 
       def scratch?
-        config._docker._from.nil?
+        config._docker._from.nil? && config._from_dimg.nil? && config._from_dimg_artifact.nil?
       end
 
       def dev_mode?
-        config._dev_mode || dapp.dev_mode?
+        dapp.dev_mode?
+      end
+
+      def force_save_cache?
+        if ENV.key? "DAPP_FORCE_SAVE_CACHE"
+          %w(yes 1 true).include? ENV["DAPP_FORCE_SAVE_CACHE"].to_s
+        else
+          !!dapp.options[:force_save_cache]
+        end
       end
 
       def build_cache_version
         [::Dapp::BUILD_CACHE_VERSION, dev_mode? ? 1 : 0]
       end
 
-      def introspect_image!(image:, options:)
-        cmd = "docker run -ti --rm --entrypoint #{dapp.bash_bin} #{options} #{image}"
-        system(cmd)
+      def cleanup_tmp
+        return unless tmp_dir_exists?
+
+        # В tmp-директории могли остаться файлы, владельцами которых мы не являемся.
+        # Такие файлы могут попасть туда при экспорте файлов артефакта.
+        # Чтобы от них избавиться — запускаем docker-контейнер под root-пользователем
+        # и удаляем примонтированную tmp-директорию.
+        args = [
+          "--rm",
+          "--volume=#{dapp.tmp_base_dir}:#{dapp.tmp_base_dir}",
+          "--label=dapp=#{dapp.name}",
+          "alpine:3.6",
+          "rm", "-rf", tmp_path
+        ]
+        Image::Stage.ruby2go_command(dapp, command: :container_run, options: { args: args })
       end
 
-      def cleanup_tmp
-        FileUtils.rm_rf(tmp_path)
-        artifacts.each(&:cleanup_tmp)
+      def stage_should_be_introspected_before_build?(name)
+        dapp.options[:introspect_before] == name
       end
 
-      def stage_should_be_introspected?(name)
-        dapp.cli_options[:introspect_stage] == name
+      def stage_should_be_introspected_after_build?(name)
+        dapp.options[:introspect_stage] == name
       end
 
       protected
@@ -179,14 +290,6 @@ module Dapp
           !last_stage.image.tagged?
         end
       end
-
-      def with_introspection
-        yield
-      rescue Exception::IntrospectImage => e
-        data = e.net_status[:data]
-        introspect_image!(image: data[:built_id], options: data[:options])
-        raise data[:error]
-      end
     end # Dimg
   end # Dimg
 end # Dapp