dapp 0.10.2 → 0.36.24

data/lib/dapp/dimg/dapp/command/stages/cleanup_local.rb

@@ -1,134 +1,32 @@
 module Dapp
   module Dimg
-    class Dapp
+    module Dapp
       module Command
         module Stages
           module CleanupLocal
-            def stages_cleanup_local(repo)
-              lock_repo(repo, readonly: true) do
-                raise Error::Command, code: :stages_cleanup_required_option unless stages_cleanup_option?
+            def stages_cleanup_local
+              raise ::Dapp::Error::Command, code: :stages_cleanup_required_option unless stages_cleanup_option?
 
-                dapp_containers_flush
-
-                proper_cache                 if proper_cache_version?
-                stages_cleanup_by_repo(repo) if proper_repo_cache?
-                proper_git_commit            if proper_git_commit?
+              if proper_cache_version?
+                ruby2go_cleanup_command(:sync, cleanup_local_proper_cache_version_options_dump)
               end
-            end
 
-            protected
-
-            def proper_cache
-              log_proper_cache do
-                lock("#{name}.images") do
-                  log_step_with_indent(name) do
-                    remove_images(dapp_images_names.select { |image_name| !actual_cache_images.include?(image_name) })
-                  end
-                end
+              if proper_repo_cache?
+                ruby2go_cleanup_command(:sync, cleanup_local_proper_repo_cache_options_dump)
               end
             end
 
-            def stages_cleanup_by_repo(repo)
-              registry = registry(repo)
-              repo_dimgs = repo_dimgs_images(registry)
-
-              lock("#{name}.images") do
-                log_step_with_indent(name) do
-                  dapp_dangling_images_flush
-                  dimgs, stages = dapp_images_hash.partition { |_, image_id| repo_dimgs.values.include?(image_id) }
-                  dimgs = dimgs.to_h
-                  stages = stages.to_h
-                  dimgs.each { |_, aiid| except_image_with_parents(aiid, stages) }
-                  remove_images(stages.keys)
-                end
+            def cleanup_local_proper_cache_version_options_dump
+              ruby2go_cleanup_common_project_options.merge(ruby2go_cleanup_cache_version_options).tap do |data|
+                break JSON.dump(data)
               end
             end
 
-            def repo_dimgs_images(registry)
-              repo_dimgs_and_cache(registry).first
-            end
-
-            def actual_cache_images
-              shellout!([
-                'docker images',
-                '--format="{{.Repository}}:{{.Tag}}"',
-                %(-f "label=dapp-cache-version=#{::Dapp::BUILD_CACHE_VERSION}"),
-                stage_cache
-              ].join(' ')).stdout.lines.map(&:strip)
-            end
-
-            def dapp_images_hash
-              shellout!(%(docker images --format "{{.Repository}}:{{.Tag}};{{.ID}}" --no-trunc #{stage_cache})).stdout.lines.map do |line|
-                line.strip.split(';')
-              end.to_h
-            end
-
-            def except_image_with_parents(image_id, stages)
-              if image_exist?(image_id)
-                image_dapp_artifacts_label(image_id).each { |aiid| except_image_with_parents(aiid, stages) }
-                iid = image_id
-                loop do
-                  stages.delete_if { |_, siid| siid == iid }
-                  break if (iid = image_parent(iid)).empty?
-                end
-              else
-                stages.delete_if { |_, siid| siid == image_id }
+            def cleanup_local_proper_repo_cache_options_dump
+              ruby2go_cleanup_common_project_options.merge(ruby2go_cleanup_common_repo_options).tap do |data|
+                break JSON.dump(data)
               end
             end
-
-            def image_exist?(image_id)
-              shellout!(%(docker inspect #{image_id}))
-              true
-            rescue ::Dapp::Error::Shellout
-              false
-            end
-
-            def image_dapp_artifacts_label(image_id)
-              select_dapp_artifacts_ids(::Dapp::Dimg::Image::Docker.image_config_option(image_id: image_id, option: 'labels'))
-            end
-
-            def image_parent(image_id)
-              shellout!(%(docker inspect -f {{.Parent}} #{image_id})).stdout.strip
-            end
-
-            def proper_git_commit
-              log_proper_git_commit do
-                unproper_images_names = []
-                dapp_images_detailed.each do |_, attrs|
-                  attrs['Config']['Labels'].each do |repo_name, commit|
-                    next if (repo = dapp_git_repositories[repo_name]).nil?
-                    unproper_images_names.concat(image_hierarchy_by_id(attrs['Id'])) unless repo.commit_exists?(commit)
-                  end
-                end
-                remove_images(unproper_images_names.uniq)
-              end
-            end
-
-            def dapp_images_detailed
-              @dapp_images_detailed ||= {}.tap do |images|
-                dapp_images_names.each do |image_name|
-                  shellout!(%(docker inspect --format='{{json .}}' #{image_name})).stdout.strip.tap do |output|
-                    images[image_name] = output == 'null' ? {} : JSON.parse(output)
-                  end
-                end
-              end
-            end
-
-            def image_hierarchy_by_id(image_id)
-              hierarchy = []
-              iids = [image_id]
-
-              loop do
-                hierarchy.concat(dapp_images_detailed.map { |name, attrs| name if iids.include?(attrs['Id']) }.compact)
-                break if begin
-                  iids.map! do |iid|
-                    dapp_images_detailed.map { |_, attrs| attrs['Id'] if attrs['Parent'] == iid }.compact
-                  end.flatten!.empty?
-                end
-              end
-
-              hierarchy
-            end
           end
         end
       end

data/lib/dapp/dimg/dapp/command/stages/cleanup_repo.rb

@@ -1,119 +1,40 @@
 module Dapp
   module Dimg
-    class Dapp
+    module Dapp
       module Command
         module Stages
           module CleanupRepo
-            def stages_cleanup_repo(repo)
-              lock_repo(repo) do
-                raise Error::Command, code: :stages_cleanup_required_option unless stages_cleanup_option?
+            def stages_cleanup_repo
+              lock_repo(repo = option_repo) do
+                raise ::Dapp::Error::Command, code: :stages_cleanup_required_option unless stages_cleanup_option?
 
-                registry = registry(repo)
-                repo_dimgs, repo_stages = repo_dimgs_and_cache(registry)
-                repo_stages.delete_if { |_, siid| repo_dimgs.values.include?(siid) } # ignoring stages with dimgs ids (v2)
-
-                proper_repo_cache(registry, repo_stages)               if proper_cache_version?
-                repo_stages_cleanup(registry, repo_dimgs, repo_stages) if proper_repo_cache?
-                proper_repo_git_commit(registry)                       if proper_git_commit?
-              end
-            end
-
-            protected
-
-            def proper_repo_cache(registry, repo_stages)
-              log_proper_cache do
-                wrong_cache_images = repo_stages.select do |image_tag, _|
-                  repo_image_dapp_cache_version_label(registry, image_tag) != ::Dapp::BUILD_CACHE_VERSION.to_s
-                end
-                wrong_cache_images.each { |image_tag, _| delete_repo_image(registry, image_tag) }
-                repo_stages.delete_if { |image_tag, _| wrong_cache_images.keys.include?(image_tag) }
-              end
-            end
-
-            def repo_stages_cleanup(registry, repo_dimgs, repo_stages)
-              log_step_with_indent(repo) do
-                repo_dimgs.each { |image_tag, image_id| clear_repo_stages(registry, repo_stages, image_tag, image_id) }
-                repo_stages.keys.each { |image_tag| delete_repo_image(registry, image_tag) }
-              end
-            end
-
-            def clear_repo_stages(registry, repo_stages, image_tag, image_id)
-              repo_image_dapp_artifacts_labels(registry, image_tag).each do |iid|
-                itag = find_image_tag_by_id(repo_stages, iid)
-                clear_repo_stages(registry, repo_stages, itag, iid) unless itag.nil?
-              end
-
-              itag = image_tag
-              iid = image_id
-              loop do
-                repo_stages.delete_if { |_, siid| siid == iid }
-                break if itag.nil? || (iid = registry.image_parent_id(itag)).empty?
-                itag = find_image_tag_by_id(repo_stages, iid)
+                ruby2go_cleanup_command(:sync, cleanup_repo_proper_cache_version_options_dump) if proper_cache_version?
+                repo_dimgstages_cleanup if proper_repo_cache?
               end
             end
 
-            def repo_image_dapp_artifacts_labels(registry, image_tag)
-              select_dapp_artifacts_ids(registry.image_labels(image_tag))
+            def repo_dimgstages_cleanup
+              ruby2go_cleanup_command(:sync, cleanup_repo_proper_repo_cache_options)
             end
 
-            def repo_image_dapp_cache_version_label(registry, image_tag)
-              registry.image_labels(image_tag)['dapp-cache-version']
-            end
-
-            def find_image_tag_by_id(images, image_id)
-              images.each { |tag, id| return tag if id == image_id }
-              nil
-            end
-
-            def proper_repo_git_commit(registry)
-              log_proper_git_commit do
-                unproper_images = []
-                repo_dapp_dappstage_images_detailed(registry).each do |_, attrs|
-                  attrs[:labels].each do |repo_name, commit|
-                    next if (repo = dapp_git_repositories[repo_name]).nil?
-                    unproper_images.concat(repo_image_tags_hierarchy(registry, attrs[:id])) unless repo.commit_exists?(commit)
-                  end
-                end
-                remove_repo_images(registry, unproper_images.uniq)
+            def cleanup_repo_proper_cache_version_options_dump
+              ruby2go_cleanup_common_repo_options.merge(ruby2go_cleanup_cache_version_options).merge(
+                mode: {
+                  sync_repo: true,
+                  only_cache_version: true
+               }
+              ).tap do |data|
+                break JSON.dump(data)
               end
             end
 
-            def repo_dapp_dappstage_images_detailed(registry)
-              @repo_dapp_images_detailed ||= begin
-                registry.tags.map do |tag|
-                  next unless tag.start_with?('dimgstage')
-
-                  image_history = registry.image_history(tag)
-                  attrs = {
-                    id: registry.image_id(tag),
-                    parent: image_history['container_config']['Image'],
-                    labels: image_history['config']['Labels']
-                  }
-                  [tag, attrs]
-                end.compact
-              end
-            end
-
-            def repo_image_tags_hierarchy(registry, registry_image_id)
-              hierarchy = []
-              iids = [registry_image_id]
-
-              loop do
-                hierarchy.concat(iids)
-                break if begin
-                  iids.map! do |iid|
-                    repo_dapp_dappstage_images_detailed(registry).map { |_, attrs| attrs[:id] if attrs[:parent] == iid }.compact
-                  end.flatten!.empty?
-                end
-              end
-
-              repo_dapp_dappstage_images_detailed(registry).map { |tag, attrs| tag if hierarchy.include? attrs[:id] }.compact
-            end
-
-            def remove_repo_images(registry, tags)
-              tags.each do |tag|
-                log(tag) if dry_run? || log_verbose?
-                registry.image_delete(tag) unless dry_run?
+            def cleanup_repo_proper_repo_cache_options
+              ruby2go_cleanup_common_repo_options.merge(
+                mode: {
+                  sync_repo: true
+                }
+              ).tap do |data|
+                break JSON.dump(data)
               end
             end
           end

data/lib/dapp/dimg/dapp/command/stages/common.rb

@@ -1,69 +1,87 @@
 module Dapp
   module Dimg
-    class Dapp
+    module Dapp
       module Command
         module Stages
           module Common
             protected
 
-            def registry(repo)
-              validate_repo_name(repo)
-              ::Dapp::Dimg::DockerRegistry.new(repo)
+            def repo_detailed_dimgs_images(registry)
+              repo_dimgs_images(registry).select do |dimg|
+                begin
+                  image_config = registry.image_config(dimg[:tag], dimg[:dimg])
+                  dimg[:created_at] = Time.parse(image_config["created"]).to_i
+                  dimg[:parent] = image_config["container_config"]["Image"]
+                  dimg[:labels] = image_config["config"]["Labels"]
+                  dimg[:labels]['dapp'] == name
+                rescue ::Dapp::Dimg::Error::Registry => e
+                  raise unless e.net_status[:data][:message].include?("MANIFEST_UNKNOWN")
+                  log_warning "WARNING: Ignore dimg `#{dimg[:dimg]}` tag `#{dimg[:tag]}`: got manifest-invalid-error from docker registry: #{e.net_status[:data][:message]}"
+                  false
+                end
+              end
             end
 
-            def repo_dimgs_and_cache(registry)
-              format = proc do |arr|
-                arr.map do |tag|
-                  if (id = registry.image_id(tag)).nil?
-                    log_warning(desc: { code: 'tag_ignored', data: { tag: tag } })
-                  else
-                    [tag, id]
-                  end
-                end.compact.to_h
+            def repo_dimgs_images(registry)
+              [].tap do |dimgs_images|
+                {}.tap do |dimgs_tags|
+                  dimgs_tags[nil] = registry.nameless_dimg_tags
+                  dimgs_names.each do |dimg_name|
+                    dimgs_tags[dimg_name] = registry.dimg_tags(dimg_name)
+                  end unless nameless_dimg?
+                end.each do |dimg_name, tags|
+                  dimgs_images.concat(tags_to_repo_images(registry, tags, dimg_name))
+                end
               end
-              dimgs, stages = registry.tags.partition { |tag| !tag.start_with?('dimgstage') }
-              [format.call(dimgs), format.call(stages)]
             end
 
-            def registry_tags(registry)
-              registry.tags
-            rescue Exception::Registry => e
-              raise unless e.net_status[:code] == :no_such_dimg
-              log_warning(desc: { code: :dimg_not_found_in_registry })
-              []
+            def repo_dimgstages_images(registry)
+              tags_to_repo_images(registry, registry.dimgstages_tags)
             end
 
-            def delete_repo_image(registry, image_tag)
-              if dry_run?
-                log(image_tag)
-              else
-                registry.image_delete(image_tag)
-              end
+            def tags_to_repo_images(registry, tags, dimg_name = nil)
+              tags.map { |tag| repo_image_format(registry, tag, dimg_name) }.compact
             end
 
-            def select_dapp_artifacts_ids(hash)
-              hash.select { |k, _v| k.start_with?('dapp-artifact') }.values
+            def repo_image_format(registry, tag, dimg_name = nil)
+              begin
+                id = registry.image_id(tag, dimg_name)
+
+                if id.nil?
+                  log_warning(desc: { code: 'tag_ignored', data: { tag: tag } })
+                  return nil
+                end
+
+                return { dimg: dimg_name, tag: tag, id: id }
+              rescue ::Dapp::Dimg::Error::Registry => e
+                raise unless e.net_status[:data][:message].include?("MANIFEST_UNKNOWN")
+                log_warning "WARNING: Ignore dimg `#{dimg_name}` tag `#{tag}`: got not-found-error from docker registry on get-image-manifest request: #{e.net_status[:data][:message]}"
+                return nil
+              end
             end
 
-            def dapp_git_repositories
-              @dapp_git_repositories ||= begin
-                {}.tap do |repositories|
-                  dimgs = build_configs.map { |config| Dimg.new(config: config, dapp: self, ignore_git_fetch: true) }
-                  dimgs.each do |dimg|
-                    [dimg, dimg.artifacts].flatten
-                                          .map(&:git_artifacts).flatten
-                                          .map { |ga_artifact| repositories[ga_artifact.full_name] = ga_artifact.repo }
+            def delete_repo_image(registry, repo_image)
+              log([repo_image[:dimg], repo_image[:tag]].compact.join(':')) if dry_run? || log_verbose?
+              unless dry_run?
+                begin
+                  if self.class.gcr_registry?
+                    registry.gcr_image_delete(repo_image[:tag], repo_image[:dimg])
+                  else
+                    registry.image_delete(repo_image[:tag], repo_image[:dimg])
                   end
+                rescue ::Dapp::Dimg::Error::Registry => e
+                  raise unless e.net_status[:data][:message].include?("MANIFEST_UNKNOWN")
+                  log_warning "WARNING: Ignore dimg `#{repo_image[:dimg]}` tag `#{repo_image[:tag]}`: got not-found-error from docker registry on image-delete request: #{e.net_status[:data][:message]}"
                 end
               end
             end
 
             def proper_repo_cache?
-              !!cli_options[:proper_repo_cache]
+              !!options[:proper_repo_cache]
             end
 
             def proper_git_commit?
-              !!cli_options[:proper_git_commit]
+              !!options[:proper_git_commit]
             end
 
             def stages_cleanup_option?

data/lib/dapp/dimg/dapp/command/stages/flush_local.rb

@@ -1,16 +1,22 @@
 module Dapp
   module Dimg
-    class Dapp
+    module Dapp
       module Command
         module Stages
           module FlushLocal
             def stages_flush_local
-              lock("#{name}.images") do
-                log_step_with_indent(name) do
-                  dapp_containers_flush
-                  dapp_dangling_images_flush
-                  remove_images(dapp_images_names)
-                end
+              ruby2go_cleanup_command(:flush, ruby2go_cleanup_stages_flush_local_options_dump)
+            end
+
+            def ruby2go_cleanup_stages_flush_local_options_dump
+              ruby2go_cleanup_common_project_options.merge(
+                mode: {
+                  with_dimgs: false,
+                  with_stages: true,
+                  only_repo: false,
+                }
+              ).tap do |json|
+                break JSON.dump(json)
               end
             end
           end

data/lib/dapp/dimg/dapp/command/stages/flush_repo.rb

@@ -1,16 +1,22 @@
 module Dapp
   module Dimg
-    class Dapp
+    module Dapp
       module Command
         module Stages
           module FlushRepo
-            def stages_flush_repo(repo)
-              lock_repo(repo) do
-                log_step_with_indent(repo) do
-                  registry = registry(repo)
-                  repo_dimgs, repo_stages = repo_dimgs_and_cache(registry)
-                  repo_dimgs.merge(repo_stages).keys.each { |image_tag| delete_repo_image(registry, image_tag) }
-                end
+            def stages_flush_repo
+              ruby2go_cleanup_command(:flush, ruby2go_cleanup_stages_flush_repo_options_dump)
+            end
+
+            def ruby2go_cleanup_stages_flush_repo_options_dump
+              ruby2go_cleanup_common_repo_options.merge(
+                mode: {
+                  with_dimgs: false,
+                  with_stages: true,
+                  only_repo: true,
+                },
+              ).tap do |json|
+                break JSON.dump(json)
               end
             end
           end

data/lib/dapp/dimg/dapp/command/stages/pull.rb

@@ -1,23 +1,14 @@
 module Dapp
   module Dimg
-    class Dapp
+    module Dapp
       module Command
         module Stages
           module Pull
-            def stages_pull(repo)
-              validate_repo_name(repo)
-              build_configs.each do |config|
-                log_dimg_name_with_indent(config) do
-                  Dimg.new(config: config, dapp: self, ignore_git_fetch: true).tap do |dimg|
-                    dimg.import_stages!(repo, format: '%{repo}:dimgstage-%{signature}')
-                  end
-                end
+            def stages_pull
+              dimg_import_export_base(should_be_built: false) do |dimg|
+                dimg.import_stages!(option_repo, format: dimgstage_push_format)
               end
             end
-
-            def pull_all_stages?
-              !!cli_options[:pull_all_stages]
-            end
           end
         end
       end

data/lib/dapp/dimg/dapp/command/stages/push.rb

@@ -1,17 +1,12 @@
 module Dapp
   module Dimg
-    class Dapp
+    module Dapp
       module Command
         module Stages
           module Push
-            def stages_push(repo)
-              validate_repo_name(repo)
-              build_configs.each do |config|
-                log_dimg_name_with_indent(config) do
-                  Dimg.new(config: config, dapp: self, ignore_git_fetch: true, should_be_built: true).tap do |dimg|
-                    dimg.export_stages!(repo, format: '%{repo}:dimgstage-%{signature}')
-                  end
-                end
+            def stages_push
+              dimg_import_export_base(should_be_built: false) do |dimg|
+                dimg.export_stages!(option_repo, format: dimgstage_push_format)
               end
             end
           end

data/lib/dapp/dimg/dapp/command/tag.rb

@@ -1,13 +1,11 @@
 module Dapp
   module Dimg
-    class Dapp
+    module Dapp
       module Command
         module Tag
-          def tag(tag)
-            one_dimg!
-            raise Error::Command, code: :tag_command_incorrect_tag, data: { name: tag } unless ::Dapp::Dimg::Image::Docker.image_name?(tag)
-            Dimg.new(config: build_configs.first, dapp: self, ignore_git_fetch: true, should_be_built: true).tap do |app|
-              app.tag!(tag)
+          def tag
+            dimg_import_export_base do |dimg|
+              dimg.tag!(option_repo, format: push_format(dimg.name))
             end
           end
         end

data/lib/dapp/dimg/dapp/config_artifact_group.rb

@@ -0,0 +1,20 @@
+module Dapp
+  module Dimg
+    module Dapp
+      module ConfigArtifactGroup
+        def artifact_config_by_name(name)
+          artifacts_configs[name] || raise(::Dapp::Error::Config, code: :artifact_not_found, data: { name: name })
+        end
+
+        def artifact_config(name, artifact_config)
+          raise(::Dapp::Error::Config, code: :artifact_already_exists, data: { name: name }) if artifacts_configs.key?(name)
+          artifacts_configs[name] = artifact_config
+        end
+
+        def artifacts_configs
+          @artifacts_configs ||= {}
+        end
+      end # Dimg
+    end # Dapp
+  end # Dimg
+end # Dapp

data/lib/dapp/dimg/dapp/dapp.rb

@@ -0,0 +1,40 @@
+module Dapp
+  module Dimg
+    module Dapp
+      module Dapp
+        include Dappfile
+        include Dimg
+        include ConfigArtifactGroup
+
+        include Command::Common
+        include Command::Run
+        include Command::Build
+        include Command::Bp
+        include Command::Push
+        include Command::Spush
+        include Command::Tag
+        include Command::List
+        include Command::Stages::CleanupLocal
+        include Command::Stages::CleanupRepo
+        include Command::Stages::FlushLocal
+        include Command::Stages::FlushRepo
+        include Command::Stages::Push
+        include Command::Stages::Pull
+        include Command::Stages::Common
+        include Command::CleanupRepo
+        include Command::FlushLocal
+        include Command::FlushRepo
+        include Command::Cleanup
+        include Command::Mrproper
+        include Command::StageImage
+        include Command::BuildContext::Import
+        include Command::BuildContext::Export
+        include Command::BuildContext::Common
+
+        include Command::Ruby2GoCleanup
+      end
+    end
+  end
+end
+
+::Dapp::Dapp.send(:include, ::Dapp::Dimg::Dapp::Dapp)

data/lib/dapp/dimg/dapp/dappfile.rb

@@ -0,0 +1,43 @@
+module Dapp
+  module Dimg
+    module Dapp
+      module Dappfile
+        def nameless_dimg?
+          dimgs_names.first.nil?
+        end
+
+        def dimg_name!
+          one_dimg!
+          build_configs.first._name
+        end
+
+        def one_dimg!
+          return if build_configs.one?
+          raise ::Dapp::Error::Command, code: :command_unexpected_dimgs_number, data: { dimgs_names: dimgs_names.join('`, `') }
+        end
+
+        def dimgs_names
+          build_configs.map(&:_name)
+        end
+
+        def build_configs
+          @build_configs ||= begin
+            config._dimg.select do |dimg|
+              dimgs_patterns.any? { |pattern| dimg._name.nil? || File.fnmatch(pattern, dimg._name) }
+            end.tap do |dimgs|
+              raise ::Dapp::Error::Dapp, code: :no_such_dimg, data: { dimgs_patterns: dimgs_patterns.join('`, `') } if dimgs.empty?
+            end
+          end
+        end
+
+        def dimgs_patterns
+          @dimgs_patterns ||= begin
+            (options[:dimgs_patterns] || []).tap do |dimgs_patterns|
+              dimgs_patterns << '*' unless dimgs_patterns.any?
+            end
+          end
+        end
+      end # Dappfile
+    end # Dapp
+  end # Dimg
+end # Dapp