dapp 0.8.0 → 0.9.0

data/lib/dapp/dimg/dapp/command/stages/cleanup_repo.rb

@@ -0,0 +1,124 @@
+module Dapp
+  module Dimg
+    class Dapp
+      module Command
+        module Stages
+          module CleanupRepo
+            def stages_cleanup_repo(repo)
+              lock_repo(repo) do
+                raise Error::Command, code: :stages_cleanup_required_option unless stages_cleanup_option?
+
+                registry = registry(repo)
+                repo_dimgs, repo_stages = repo_dimgs_and_cache(registry)
+                repo_stages.delete_if { |_, siid| repo_dimgs.values.include?(siid) } # ignoring stages with dimgs ids (v2)
+
+                proper_repo_cache(registry, repo_stages)               if proper_cache_version?
+                repo_stages_cleanup(registry, repo_dimgs, repo_stages) if proper_repo_cache?
+                proper_repo_git_commit(registry)                       if proper_git_commit?
+              end
+            end
+
+            protected
+
+            def proper_repo_cache(registry, repo_stages)
+              log_proper_cache do
+                wrong_cache_images = repo_stages.select do |image_tag, _|
+                  repo_image_dapp_cache_version_label(registry, image_tag) != ::Dapp::BUILD_CACHE_VERSION.to_s
+                end
+                wrong_cache_images.each { |image_tag, _| delete_repo_image(registry, image_tag) }
+                repo_stages.delete_if { |image_tag, _| wrong_cache_images.keys.include?(image_tag) }
+              end
+            end
+
+            def repo_stages_cleanup(registry, repo_dimgs, repo_stages)
+              log_step_with_indent(repo) do
+                repo_dimgs.each { |image_tag, image_id| clear_repo_stages(registry, repo_stages, image_tag, image_id) }
+                repo_stages.keys.each { |image_tag| delete_repo_image(registry, image_tag) }
+              end
+            end
+
+            def clear_repo_stages(registry, repo_stages, image_tag, image_id)
+              repo_image_dapp_artifacts_labels(registry, image_tag).each do |iid|
+                itag = find_image_tag_by_id(repo_stages, iid)
+                clear_repo_stages(registry, repo_stages, itag, iid) unless itag.nil?
+              end
+
+              itag = image_tag
+              iid = image_id
+              loop do
+                repo_stages.delete_if { |_, siid| siid == iid }
+                break if itag.nil? || (iid = registry.image_parent_id(itag)).empty?
+                itag = find_image_tag_by_id(repo_stages, iid)
+              end
+            end
+
+            def repo_image_dapp_artifacts_labels(registry, image_tag)
+              select_dapp_artifacts_ids(registry.image_labels(image_tag))
+            end
+
+            def repo_image_dapp_cache_version_label(registry, image_tag)
+              registry.image_labels(image_tag)['dapp-cache-version']
+            end
+
+            def find_image_tag_by_id(images, image_id)
+              images.each { |tag, id| return tag if id == image_id }
+              nil
+            end
+
+            def proper_repo_git_commit(registry)
+              log_proper_git_commit do
+                unproper_images = []
+                repo_dapp_dappstage_images_detailed(registry).each do |_, attrs|
+                  attrs[:labels].each do |repo_name, commit|
+                    next if (repo = dapp_git_repositories[repo_name]).nil?
+                    unproper_images.concat(repo_image_tags_hierarchy(registry, attrs[:id])) unless repo.commit_exists?(commit)
+                  end
+                end
+                remove_repo_images(registry, unproper_images.uniq)
+              end
+            end
+
+            def repo_dapp_dappstage_images_detailed(registry)
+              @repo_dapp_images_detailed ||= begin
+                registry.tags.map do |tag|
+                  next unless tag.start_with?('dimgstage')
+
+                  image_history = registry.image_history(tag)
+                  attrs = {
+                    id: registry.image_id(tag),
+                    parent: image_history['container_config']['Image'],
+                    labels: image_history['config']['Labels']
+                  }
+                  [tag, attrs]
+                end.compact
+              end
+            end
+
+            def repo_image_tags_hierarchy(registry, registry_image_id)
+              hierarchy = []
+              iids = [registry_image_id]
+
+              loop do
+                hierarchy.concat(iids)
+                break if begin
+                  iids.map! do |iid|
+                    repo_dapp_dappstage_images_detailed(registry).map { |_, attrs| attrs[:id] if attrs[:parent] == iid }.compact
+                  end.flatten!.empty?
+                end
+              end
+
+              repo_dapp_dappstage_images_detailed(registry).map { |tag, attrs| tag if hierarchy.include? attrs[:id] }.compact
+            end
+
+            def remove_repo_images(registry, tags)
+              tags.each do |tag|
+                log(tag) if dry_run? || log_verbose?
+                registry.image_delete(tag) unless dry_run?
+              end
+            end
+          end
+        end
+      end
+    end
+  end # Dimg
+end # Dapp

data/lib/dapp/dimg/dapp/command/stages/common.rb

@@ -0,0 +1,85 @@
+module Dapp
+  module Dimg
+    class Dapp
+      module Command
+        module Stages
+          module Common
+            protected
+
+            def registry(repo)
+              validate_repo_name(repo)
+              ::Dapp::Dimg::DockerRegistry.new(repo)
+            end
+
+            def repo_dimgs_and_cache(registry)
+              format = proc do |arr|
+                arr.map do |tag|
+                  if (id = registry.image_id(tag)).nil?
+                    log_warning(desc: { code: 'tag_ignored', data: { tag: tag } })
+                  else
+                    [tag, id]
+                  end
+                end.compact.to_h
+              end
+              dimgs, stages = registry.tags.partition { |tag| !tag.start_with?('dimgstage') }
+              [format.call(dimgs), format.call(stages)]
+            end
+
+            def registry_tags(registry)
+              registry.tags
+            rescue Exception::Registry => e
+              raise unless e.net_status[:code] == :no_such_dimg
+              log_warning(desc: { code: :dimg_not_found_in_registry })
+              []
+            end
+
+            def delete_repo_image(registry, image_tag)
+              if dry_run?
+                log(image_tag)
+              else
+                registry.image_delete(image_tag)
+              end
+            end
+
+            def select_dapp_artifacts_ids(hash)
+              hash.select { |k, _v| k.start_with?('dapp-artifact') }.values
+            end
+
+            def dapp_git_repositories
+              @dapp_git_repositories ||= begin
+                {}.tap do |repositories|
+                  dimgs = build_configs.map { |config| Dimg.new(config: config, dapp: self, ignore_git_fetch: true) }
+                  dimgs.each do |dimg|
+                    [dimg, dimg.artifacts].flatten
+                                          .map(&:git_artifacts).flatten
+                                          .map { |ga_artifact| repositories[ga_artifact.full_name] = ga_artifact.repo }
+                  end
+                end
+              end
+            end
+
+            def proper_repo_cache?
+              !!cli_options[:proper_repo_cache]
+            end
+
+            def proper_git_commit?
+              !!cli_options[:proper_git_commit]
+            end
+
+            def stages_cleanup_option?
+              proper_git_commit? || proper_cache_version? || proper_repo_cache?
+            end
+
+            def log_proper_git_commit(&blk)
+              log_step_with_indent(:'proper git commit', &blk)
+            end
+
+            def lock_repo(repo, *args, &blk)
+              lock("repo.#{hashsum repo}", *args, &blk)
+            end
+          end
+        end
+      end
+    end
+  end # Dimg
+end # Dapp

data/lib/dapp/dimg/dapp/command/stages/flush_local.rb

@@ -0,0 +1,21 @@
+module Dapp
+  module Dimg
+    class Dapp
+      module Command
+        module Stages
+          module FlushLocal
+            def stages_flush_local
+              lock("#{name}.images") do
+                log_step_with_indent(name) do
+                  dapp_containers_flush
+                  dapp_dangling_images_flush
+                  remove_images(dapp_images_names)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end # Dimg
+end # Dapp

data/lib/dapp/dimg/dapp/command/stages/flush_repo.rb

@@ -0,0 +1,21 @@
+module Dapp
+  module Dimg
+    class Dapp
+      module Command
+        module Stages
+          module FlushRepo
+            def stages_flush_repo(repo)
+              lock_repo(repo) do
+                log_step_with_indent(repo) do
+                  registry = registry(repo)
+                  repo_dimgs, repo_stages = repo_dimgs_and_cache(registry)
+                  repo_dimgs.merge(repo_stages).keys.each { |image_tag| delete_repo_image(registry, image_tag) }
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end # Dimg
+end # Dapp

data/lib/dapp/dimg/dapp/command/stages/pull.rb

@@ -0,0 +1,26 @@
+module Dapp
+  module Dimg
+    class Dapp
+      module Command
+        module Stages
+          module Pull
+            def stages_pull(repo)
+              validate_repo_name(repo)
+              build_configs.each do |config|
+                log_dimg_name_with_indent(config) do
+                  Dimg.new(config: config, dapp: self, ignore_git_fetch: true).tap do |dimg|
+                    dimg.import_stages!(repo, format: '%{repo}:dimgstage-%{signature}')
+                  end
+                end
+              end
+            end
+
+            def pull_all_stages?
+              !!cli_options[:pull_all_stages]
+            end
+          end
+        end
+      end
+    end
+  end # Dimg
+end # Dapp

data/lib/dapp/dimg/dapp/command/stages/push.rb

@@ -0,0 +1,22 @@
+module Dapp
+  module Dimg
+    class Dapp
+      module Command
+        module Stages
+          module Push
+            def stages_push(repo)
+              validate_repo_name(repo)
+              build_configs.each do |config|
+                log_dimg_name_with_indent(config) do
+                  Dimg.new(config: config, dapp: self, ignore_git_fetch: true, should_be_built: true).tap do |dimg|
+                    dimg.export_stages!(repo, format: '%{repo}:dimgstage-%{signature}')
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end # Dimg
+end # Dapp

data/lib/dapp/dimg/dapp/command/tag.rb

@@ -0,0 +1,17 @@
+module Dapp
+  module Dimg
+    class Dapp
+      module Command
+        module Tag
+          def tag(tag)
+            one_dimg!
+            raise Error::Command, code: :tag_command_incorrect_tag, data: { name: tag } unless ::Dapp::Dimg::Image::Docker.image_name?(tag)
+            Dimg.new(config: build_configs.first, dapp: self, ignore_git_fetch: true, should_be_built: true).tap do |app|
+              app.tag!(tag)
+            end
+          end
+        end
+      end
+    end
+  end # Dimg
+end # Dapp

data/lib/dapp/dimg/dimg.rb

@@ -0,0 +1,192 @@
+module Dapp
+  module Dimg
+    class Dimg
+      include GitArtifact
+      include Path
+      include Tags
+      include Stages
+
+      include Helper::Sha256
+
+      attr_reader :config
+      attr_reader :ignore_git_fetch
+      attr_reader :should_be_built
+      attr_reader :dapp
+
+      def initialize(config:, dapp:, should_be_built: false, ignore_git_fetch: false)
+        @config = config
+        @dapp = dapp
+
+        @ignore_git_fetch = ignore_git_fetch
+        @should_be_built = should_be_built
+
+        raise Error::Dimg, code: :dimg_not_built if should_be_built?
+      end
+
+      def build!
+        with_introspection do
+          dapp.lock("#{dapp.name}.images", readonly: true) do
+            last_stage.build_lock! do
+              begin
+                builder.before_build_check
+                last_stage.build!
+              ensure
+                after_stages_build!
+              end
+            end
+          end
+        end
+      ensure
+        cleanup_tmp
+      end
+
+      def after_stages_build!
+        return unless last_stage.image.built? || dev_mode?
+        last_stage.save_in_cache!
+        artifacts.each { |artifact| artifact.last_stage.save_in_cache! }
+      end
+
+      def tag!(tag)
+        dapp.lock("#{dapp.name}.images", readonly: true) do
+          dimg_name = config._name
+          if dapp.dry_run?
+            dapp.log_state(dimg_name, state: dapp.t(code: 'state.tag'), styles: { status: :success })
+          else
+            dapp.log_process(dimg_name, process: dapp.t(code: 'status.process.tagging')) do
+              last_stage.image.tag!(tag)
+            end
+          end
+        end
+      end
+
+      def export!(repo, format:)
+        dapp.lock("#{dapp.name}.images", readonly: true) do
+          tags.each do |tag|
+            image_name = format(format, repo: repo, dimg_name: config._name, tag: tag)
+            export_base!(last_stage.image, image_name)
+          end
+        end
+      end
+
+      def export_stages!(repo, format:)
+        dapp.lock("#{dapp.name}.images", readonly: true) do
+          export_images.each do |image|
+            image_name = format(format, repo: repo, signature: image.name.split(':').last)
+            export_base!(image, image_name)
+          end
+        end
+      end
+
+      def export_base!(image, image_name)
+        if dapp.dry_run?
+          dapp.log_state(image_name, state: dapp.t(code: 'state.push'), styles: { status: :success })
+        else
+          dapp.lock("image.#{hashsum image_name}") do
+            ::Dapp::Dimg::Image::Stage.cache_reset(image_name)
+            dapp.log_process(image_name, process: dapp.t(code: 'status.process.pushing')) do
+              dapp.with_log_indent do
+                image.export!(image_name)
+              end
+            end
+          end
+        end
+      end
+
+      def import_stages!(repo, format:)
+        dapp.lock("#{dapp.name}.images", readonly: true) do
+          import_images.each do |image|
+            begin
+              image_name = format(format, repo: repo, signature: image.name.split(':').last)
+              import_base!(image, image_name)
+            rescue Error::Shellout
+              next
+            end
+            break unless dapp.pull_all_stages?
+          end
+        end
+      end
+
+      def import_base!(image, image_name)
+        if dapp.dry_run?
+          dapp.log_state(image_name, state: dapp.t(code: 'state.pull'), styles: { status: :success })
+        else
+          dapp.lock("image.#{hashsum image_name}") do
+            dapp.log_process(image_name, process: dapp.t(code: 'status.process.pulling'),
+                                         status: { failed: dapp.t(code: 'status.failed.not_pulled') },
+                                         style: { failed: :secondary }) do
+              image.import!(image_name)
+            end
+          end
+        end
+      end
+
+      def run(docker_options, command)
+        cmd = "docker run #{[docker_options, last_stage.image.built_id, command].flatten.compact.join(' ')}"
+        if dapp.dry_run?
+          dapp.log(cmd)
+        else
+          system(cmd) || raise(Error::Dimg, code: :dimg_not_run)
+        end
+      end
+
+      def stage_image_name(stage_name)
+        stages.find { |stage| stage.send(:name) == stage_name }.image.name
+      end
+
+      def builder
+        @builder ||= Builder.const_get(config._builder.capitalize).new(self)
+      end
+
+      def artifacts
+        @artifacts ||= artifacts_stages.map { |stage| stage.artifacts.map { |artifact| artifact[:dimg] } }.flatten
+      end
+
+      def artifact?
+        false
+      end
+
+      def scratch?
+        config._docker._from.nil?
+      end
+
+      def dev_mode?
+        config._dev_mode || dapp.dev_mode?
+      end
+
+      def build_cache_version
+        [::Dapp::BUILD_CACHE_VERSION, dev_mode? ? 1 : 0]
+      end
+
+      def introspect_image!(image:, options:)
+        cmd = "docker run -ti --rm --entrypoint #{dapp.bash_bin} #{options} #{image}"
+        system(cmd)
+      end
+
+      def cleanup_tmp
+        FileUtils.rm_rf(tmp_path)
+        artifacts.each(&:cleanup_tmp)
+      end
+
+      def stage_should_be_introspected?(name)
+        dapp.cli_options[:introspect_stage] == name
+      end
+
+      protected
+
+      def should_be_built?
+        should_be_built && begin
+          builder.before_dimg_should_be_built_check
+          !last_stage.image.tagged?
+        end
+      end
+
+      def with_introspection
+        yield
+      rescue Exception::IntrospectImage => e
+        data = e.net_status[:data]
+        introspect_image!(image: data[:built_id], options: data[:options])
+        raise data[:error]
+      end
+    end # Dimg
+  end # Dimg
+end # Dapp