dapp 0.8.0 → 0.9.0

data/lib/dapp/dimg.rb

@@ -1,192 +1 @@
-module Dapp
-  # Dimg
-  class Dimg
-    include GitArtifact
-    include Path
-    include Tags
-    include Stages
-
-    include Helper::Sha256
-
-    attr_reader :config
-    attr_reader :ignore_git_fetch
-    attr_reader :should_be_built
-    attr_reader :project
-
-    def initialize(config:, project:, should_be_built: false, ignore_git_fetch: false)
-      @config = config
-      @project = project
-
-      @ignore_git_fetch = ignore_git_fetch
-      @should_be_built = should_be_built
-
-      raise Error::Dimg, code: :dimg_not_built if should_be_built?
-    end
-
-    def build!
-      with_introspection do
-        project.lock("#{project.name}.images", readonly: true) do
-          last_stage.build_lock! do
-            begin
-              builder.before_build_check
-              last_stage.build!
-            ensure
-              after_stages_build!
-            end
-          end
-        end
-      end
-    ensure
-      cleanup_tmp
-    end
-
-    def after_stages_build!
-      return unless last_stage.image.built? || dev_mode?
-      last_stage.save_in_cache!
-      artifacts.each { |artifact| artifact.last_stage.save_in_cache! }
-    end
-
-    def tag!(tag)
-      project.lock("#{project.name}.images", readonly: true) do
-        dimg_name = config._name
-        if project.dry_run?
-          project.log_state(dimg_name, state: project.t(code: 'state.tag'), styles: { status: :success })
-        else
-          project.log_process(dimg_name, process: project.t(code: 'status.process.tagging')) do
-            last_stage.image.tag!(tag)
-          end
-        end
-      end
-    end
-
-    def export!(repo, format:)
-      project.lock("#{project.name}.images", readonly: true) do
-        tags.each do |tag|
-          image_name = format % { repo: repo, dimg_name: config._name, tag: tag }
-          export_base!(last_stage.image, image_name)
-        end
-      end
-    end
-
-    def export_stages!(repo, format:)
-      project.lock("#{project.name}.images", readonly: true) do
-        export_images.each do |image|
-          image_name = format % { repo: repo, signature: image.name.split(':').last }
-          export_base!(image, image_name)
-        end
-      end
-    end
-
-    def export_base!(image, image_name)
-      if project.dry_run?
-        project.log_state(image_name, state: project.t(code: 'state.push'), styles: { status: :success })
-      else
-        project.lock("image.#{hashsum image_name}") do
-          Dapp::Image::Stage.cache_reset(image_name)
-          project.log_process(image_name, process: project.t(code: 'status.process.pushing')) do
-            project.with_log_indent do
-              image.export!(image_name)
-            end
-          end
-        end
-      end
-    end
-
-    def import_stages!(repo, format:)
-      project.lock("#{project.name}.images", readonly: true) do
-        import_images.each do |image|
-          begin
-            image_name = format % { repo: repo, signature: image.name.split(':').last }
-            import_base!(image, image_name)
-          rescue Error::Shellout
-            next
-          end
-          break unless project.pull_all_stages?
-        end
-      end
-    end
-
-    def import_base!(image, image_name)
-      if project.dry_run?
-        project.log_state(image_name, state: project.t(code: 'state.pull'), styles: { status: :success })
-      else
-        project.lock("image.#{hashsum image_name}") do
-          project.log_process(image_name,
-                              process: project.t(code: 'status.process.pulling'),
-                              status: { failed: project.t(code: 'status.failed.not_pulled') },
-                              style: { failed: :secondary }) do
-            image.import!(image_name)
-          end
-        end
-      end
-    end
-
-    def run(docker_options, command)
-      cmd = "docker run #{[docker_options, last_stage.image.built_id, command].flatten.compact.join(' ')}"
-      if project.dry_run?
-        project.log(cmd)
-      else
-        system(cmd) || raise(Error::Dimg, code: :dimg_not_run)
-      end
-    end
-
-    def stage_image_name(stage_name)
-      stages.find { |stage| stage.send(:name) == stage_name }.image.name
-    end
-
-    def builder
-      @builder ||= Builder.const_get(config._builder.capitalize).new(self)
-    end
-
-    def artifacts
-      @artifacts ||= artifacts_stages.map { |stage| stage.artifacts.map { |artifact| artifact[:dimg] } }.flatten
-    end
-
-    def artifact?
-      false
-    end
-
-    def scratch?
-      config._docker._from.nil?
-    end
-
-    def dev_mode?
-      config._dev_mode || project.dev_mode?
-    end
-
-    def build_cache_version
-      [Dapp::BUILD_CACHE_VERSION, dev_mode? ? 1 : 0]
-    end
-
-    def introspect_image!(image:, options:)
-      cmd = "docker run -ti --rm --entrypoint #{project.bash_bin} #{options} #{image}"
-      system(cmd)
-    end
-
-    def cleanup_tmp
-      FileUtils.rm_rf(tmp_path)
-      artifacts.each(&:cleanup_tmp)
-    end
-
-    def stage_should_be_introspected?(name)
-      project.cli_options[:introspect_stage] == name
-    end
-
-    protected
-
-    def should_be_built?
-      should_be_built && begin
-        builder.before_dimg_should_be_built_check
-        !last_stage.image.tagged?
-      end
-    end
-
-    def with_introspection
-      yield
-    rescue Exception::IntrospectImage => e
-      data = e.net_status[:data]
-      introspect_image!(image: data[:built_id], options: data[:options])
-      raise data[:error]
-    end
-  end # Dimg
-end # Dapp
+module Dapp::Dimg; end

data/lib/dapp/dimg/artifact.rb

@@ -0,0 +1,24 @@
+module Dapp
+  module Dimg
+    class Artifact < Dimg
+      def after_stages_build!
+      end
+
+      def stage_should_be_introspected?(name)
+        dapp.cli_options[:introspect_artifact_stage] == name
+      end
+
+      def artifact?
+        true
+      end
+
+      def should_be_built?
+        false
+      end
+
+      def last_stage
+        @last_stage ||= Build::Stage::BuildArtifact.new(self)
+      end
+    end # Artifact
+  end # Dimg
+end # Dapp

data/lib/dapp/dimg/build/stage/after_install_artifact.rb

@@ -0,0 +1,14 @@
+module Dapp
+  module Dimg
+    module Build
+      module Stage
+        class AfterInstallArtifact < ArtifactDefault
+          def initialize(dimg, next_stage)
+            @prev_stage = Install::GAPostInstallPatch.new(dimg, self)
+            super
+          end
+        end # AfterInstallArtifact
+      end # Stage
+    end # Build
+  end # Dimg
+end # Dapp

data/lib/dapp/dimg/build/stage/after_setup_artifact.rb

@@ -0,0 +1,18 @@
+module Dapp
+  module Dimg
+    module Build
+      module Stage
+        class AfterSetupArtifact < ArtifactDefault
+          def initialize(dimg, next_stage)
+            @prev_stage = if dimg.artifact?
+              Setup::Setup.new(dimg, self)
+            else
+              Setup::GAPostSetupPatch.new(dimg, self)
+            end
+            super
+          end
+        end # AfterSetupArtifact
+      end # Stage
+    end # Build
+  end # Dimg
+end # Dapp

data/lib/dapp/dimg/build/stage/artifact_base.rb

@@ -0,0 +1,80 @@
+module Dapp
+  module Dimg
+    module Build
+      module Stage
+        class ArtifactBase < Base
+          def dependencies
+            artifacts_signatures
+          end
+
+          def prepare_image
+            super
+            artifacts_dimgs_build!
+            artifacts_labels = {}
+            artifacts.each do |artifact|
+              apply_artifact(artifact, image)
+              artifacts_labels["dapp-artifact-#{artifact[:name]}".to_sym] = artifact[:dimg].last_stage.image.built_id
+            end
+            image.add_service_change_label artifacts_labels
+          end
+
+          def artifacts
+            @artifacts ||= begin
+              dimg.config.public_send("_#{name}").map do |artifact|
+                artifact_dimg = ::Dapp::Dimg::Artifact.new(config: artifact._config,
+                                                           dapp: dimg.dapp,
+                                                           ignore_git_fetch: dimg.ignore_git_fetch)
+                { name: artifact._config._name, options: artifact._artifact_options, dimg: artifact_dimg }
+              end
+            end
+          end
+
+          def artifact?
+            true
+          end
+
+          protected
+
+          def should_not_be_detailed?
+            true
+          end
+
+          def ignore_log_commands?
+            true
+          end
+
+          def artifacts_signatures
+            artifacts.map { |artifact| hashsum [artifact[:dimg].signature, artifact[:options]] }
+          end
+
+          def artifacts_dimgs_build!
+            artifacts.each do |artifact|
+              process = dimg.dapp.t(code: 'process.artifact_building', data: { name: artifact[:name] })
+              dimg.dapp.log_secondary_process(process,
+                                              short: !dimg.dapp.log_verbose?,
+                                              quiet: dimg.artifact? && !dimg.dapp.log_verbose?) do
+                dimg.dapp.with_log_indent do
+                  artifact[:dimg].build!
+                end
+              end
+            end
+          end
+
+          def run_artifact_dimg(artifact_dimg, artifact_name, commands)
+            docker_options = ['--rm',
+                              "--volume #{dimg.tmp_path('artifact', artifact_name)}:#{artifact_dimg.container_tmp_path(artifact_name)}",
+                              "--volumes-from #{dimg.dapp.base_container}",
+                              "--entrypoint #{dimg.dapp.bash_bin}"]
+            dimg.dapp.log_secondary_process(dimg.dapp.t(code: 'process.artifact_copy',
+                                                        data: { name: artifact_name }),
+                                            short: true,
+                                            quiet: dimg.artifact? && !dimg.dapp.log_verbose?) do
+              artifact_dimg.run(docker_options, [%(-ec '#{dimg.dapp.shellout_pack(commands)}')])
+            end
+          end
+          # rubocop:enable Metrics/AbcSize
+        end # ArtifactBase
+      end # Stage
+    end # Build
+  end # Dimg
+end # Dapp

data/lib/dapp/dimg/build/stage/artifact_default.rb

@@ -0,0 +1,63 @@
+module Dapp
+  module Dimg
+    module Build
+      module Stage
+        class ArtifactDefault < ArtifactBase
+          protected
+
+          def apply_artifact(artifact, image)
+            return if dimg.dapp.dry_run?
+
+            artifact_name = artifact[:name]
+            artifact_dimg = artifact[:dimg]
+            cwd = artifact[:options][:cwd]
+            include_paths = artifact[:options][:include_paths]
+            exclude_paths = artifact[:options][:exclude_paths]
+            owner = artifact[:options][:owner]
+            group = artifact[:options][:group]
+            to = artifact[:options][:to]
+
+            command = safe_cp(cwd, artifact_dimg.container_tmp_path(artifact_name).to_s, Process.uid, Process.gid, include_paths, exclude_paths)
+            run_artifact_dimg(artifact_dimg, artifact_name, command)
+
+            command = safe_cp(dimg.container_tmp_path('artifact', artifact_name).to_s, to, owner, group, include_paths, exclude_paths)
+            image.add_command command
+            image.add_volume "#{dimg.tmp_path('artifact', artifact_name)}:#{dimg.container_tmp_path('artifact', artifact_name)}:ro"
+          end
+          # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
+
+          private
+
+          # rubocop:disable Metrics/ParameterLists
+          def safe_cp(from, to, owner, group, include_paths = [], exclude_paths = [])
+            credentials = ''
+            credentials += "-o #{owner} " if owner
+            credentials += "-g #{group} " if group
+            excludes = find_command_excludes(from, exclude_paths).join(' ')
+
+            copy_files = proc do |from_, path_ = ''|
+              "if [[ -d #{File.join(from_, path_)} ]] || [[ -f #{File.join(from_, path_)} ]]; then " \
+              "#{dimg.dapp.find_bin} #{File.join(from_, path_)} #{excludes} -type f -exec " \
+              "#{dimg.dapp.bash_bin} -ec '#{dimg.dapp.install_bin} -D #{credentials} \"{}\" " \
+              "\"#{File.join(to, '$(echo "{}" | ' \
+              "#{dimg.dapp.sed_bin} -e \"s/^#{from_.gsub('/', '\\/')}\\///g\")")}\"' \\; ;" \
+              'fi'
+            end
+
+            commands = []
+            commands << [dimg.dapp.install_bin, credentials, '-d', to].join(' ')
+            commands.concat(include_paths.empty? ? Array(copy_files.call(from)) : include_paths.map { |path| copy_files.call(from, path) })
+            commands << "#{dimg.dapp.find_bin} #{to} -type d -exec " \
+                        "#{dimg.dapp.bash_bin} -ec '#{dimg.dapp.install_bin} -d #{credentials} {}' \\;"
+            commands.join(' && ')
+          end
+          # rubocop:enable Metrics/ParameterLists
+
+          def find_command_excludes(from, exclude_paths)
+            exclude_paths.map { |path| "-not \\( -path #{File.join(from, path)} -prune \\)" }
+          end
+        end # ArtifactDefault
+      end # Stage
+    end # Build
+  end # Dimg
+end # Dapp

data/lib/dapp/dimg/build/stage/base.rb

@@ -0,0 +1,210 @@
+module Dapp
+  module Dimg
+    module Build
+      module Stage
+        class Base
+          include Helper::Sha256
+          include Helper::Trivia
+          include Mod::Logging
+
+          attr_accessor :prev_stage, :next_stage
+          attr_reader :dimg
+
+          def initialize(dimg, next_stage)
+            @dimg = dimg
+
+            @next_stage = next_stage
+            @next_stage.prev_stage = self
+          end
+
+          # rubocop:disable Metrics/PerceivedComplexity
+          def build_lock!
+            return yield if dimg.dapp.dry_run?
+
+            try_lock = proc do
+              next yield unless should_be_tagged?
+
+              no_lock = false
+
+              dimg.dapp.lock("#{dimg.dapp.name}.image.#{image.name}") do
+                image.cache_reset
+
+                if should_be_tagged?
+                  yield
+                else
+                  no_lock = true
+                end
+              end
+
+              yield if no_lock
+            end
+
+            if prev_stage
+              prev_stage.build_lock! { try_lock.call }
+            else
+              try_lock.call
+            end
+          end
+          # rubocop:enable Metrics/MethodLength, Metrics/PerceivedComplexity
+
+          # rubocop:disable Metrics/CyclomaticComplexity
+          def build!
+            return if build_should_be_skipped?
+            prev_stage.build! if prev_stage
+            if image_should_be_build?
+              prepare_image if !image.built? && !should_be_not_present?
+              log_image_build(&method(:image_build))
+            end
+            dimg.introspect_image!(image: image.built_id, options: image.send(:prepared_options)) if should_be_introspected?
+          end
+          # rubocop:enable Metrics/CyclomaticComplexity
+
+          def save_in_cache!
+            prev_stage.save_in_cache! if prev_stage
+            return unless should_be_tagged?
+            image.save_in_cache! unless dimg.dapp.dry_run?
+          end
+
+          def image
+            @image ||= begin
+              if empty?
+                prev_stage.image
+              else
+                Image::Stage.image_by_name(name: image_name, from: from_image, dapp: dimg.dapp)
+              end
+            end
+          end
+
+          def prepare_image
+            return if dimg.dapp.dry_run?
+            image.add_volumes_from dimg.dapp.base_container
+
+            image_add_mounts
+
+            image.add_service_change_label dapp: dimg.stage_dapp_label
+            image.add_service_change_label 'dapp-version'.to_sym => ::Dapp::VERSION
+            image.add_service_change_label 'dapp-cache-version'.to_sym => ::Dapp::BUILD_CACHE_VERSION
+            image.add_service_change_label 'dapp-dev-mode'.to_sym => true if dimg.dev_mode?
+
+            if dimg.dapp.ssh_auth_sock
+              image.add_volume "#{dimg.dapp.ssh_auth_sock}:/tmp/dapp-ssh-agent"
+              image.add_env 'SSH_AUTH_SOCK', '/tmp/dapp-ssh-agent'
+            end
+
+            yield if block_given?
+          end
+
+          def image_add_mounts
+            [:tmp_dir, :build_dir].each do |type|
+              next if (mounts = adding_mounts_by_type(type)).empty?
+
+              mounts.each do |path|
+                absolute_path = File.expand_path(File.join('/', path))
+                tmp_path = dimg.send(type, 'mount', absolute_path[1..-1]).tap(&:mkpath)
+                image.add_volume "#{tmp_path}:#{absolute_path}"
+              end
+
+              image.add_service_change_label :"dapp-mount-#{type.to_s.tr('_', '-')}" => mounts.join(';')
+            end
+          end
+
+          def adding_mounts_by_type(type)
+            (config_mounts_by_type(type) + labels_mounts_by_type(type)).uniq
+          end
+
+          def config_mounts_by_type(type)
+            dimg.config.public_send("_#{type}_mount").map(&:_to)
+          end
+
+          def labels_mounts_by_type(type)
+            from_image.labels.select { |l, _| l == "dapp-mount-#{type.to_s.tr('_', '-')}" }.map { |_, value| value.split(';') }.flatten
+          end
+
+          def image_should_be_build?
+            (!empty? && !image.built?) || dimg.dapp.log_verbose?
+          end
+
+          def empty?
+            dependencies_empty?
+          end
+
+          def dependencies_empty?
+            dependencies.flatten.compact.delete_if { |val| val.respond_to?(:empty?) && val.empty? }.empty?
+          end
+
+          def signature
+            @signature ||= begin
+              if empty?
+                prev_stage.signature
+              else
+                args = []
+                args << prev_stage.signature unless prev_stage.nil?
+                args << dimg.build_cache_version
+                args << builder_checksum
+                args.concat(dependencies.flatten)
+
+                hashsum args
+              end
+            end
+          end
+
+          def name
+            class_to_lowercase.to_sym
+          end
+
+          def builder_checksum
+          end
+
+          def git_artifacts_dependencies
+            dimg.git_artifacts.map { |git_artifact| git_artifact.stage_dependencies_checksums(self) }
+          end
+
+          def dependencies
+            []
+          end
+
+          def artifact?
+            false
+          end
+
+          protected
+
+          def image_build
+            image.build!
+          end
+
+          def build_should_be_skipped?
+            image.built? && !dimg.dapp.log_verbose? && !should_be_introspected?
+          end
+
+          def should_be_tagged?
+            !(empty? || image.tagged? || should_be_not_present?) && image.built?
+          end
+
+          def should_be_not_present?
+            return false if next_stage.nil?
+            next_stage.image.tagged? || next_stage.should_be_not_present?
+          end
+
+          def image_name
+            format(dimg.stage_cache_format, signature: signature)
+          end
+
+          def from_image
+            prev_stage.image if prev_stage || begin
+              raise Error::Build, code: :from_image_required
+            end
+          end
+
+          def change_options
+            @change_options ||= begin
+              dimg.config._docker._change_options.to_h.delete_if do |_, val|
+                val.nil? || (val.respond_to?(:empty?) && val.empty?)
+              end
+            end
+          end
+        end # Base
+      end # Stage
+    end # Build
+  end # Dimg
+end # Dapp