dapp 0.8.0 → 0.9.0

data/lib/dapp/dimg/dapp/command/build_context/import.rb

@@ -0,0 +1,71 @@
+module Dapp
+  module Dimg
+    class Dapp
+      module Command
+        module BuildContext
+          module Import
+            def build_context_import
+              raise Error::Command, code: :context_directory_not_found,
+                                    data: { path: build_context_path } unless build_context_path.exist?
+
+              log_process(:'import context') do
+                with_log_indent do
+                  import_build_context_build_tar
+		              import_build_context_image_tar
+                end
+              end
+            end
+
+            def import_build_context_image_tar
+              if build_context_images_tar.exist?
+                log_secondary_process(:images, short: true) do
+                  lock("#{name}.images") do
+                    Image::Docker.load!(build_context_images_tar)
+                  end
+                end
+              else
+                log_warning(desc: { code: :context_archive_not_found, data: { path: build_context_images_tar } })
+              end
+            end
+
+            def import_build_context_build_tar
+              if build_context_build_tar.exist?
+                log_secondary_process(:build_dir, short: true) do
+                  store_current_build_dir
+
+                  File.open(build_context_build_tar, File::RDONLY) do |f|
+                    Gem::Package::TarReader.new(f).each_entry do |entry|
+                      header = entry.header
+                      path = File.join(build_path, entry.full_name)
+
+                      if entry.directory?
+                        FileUtils.mkpath path, :mode => entry.header.mode
+                      else
+                        FileUtils.mkpath File.dirname(path)
+                        File.write(path, entry.read)
+                        File.chmod(header.mode, path)
+                      end
+                    end
+                  end
+                end
+              else
+                log_warning(desc: { code: :context_archive_not_found, data: { path: build_context_build_tar } })
+              end
+            end
+
+            def store_current_build_dir
+              return if build_path_empty?
+              raise Error::Command, code: :stored_build_dir_already_exist,
+                                    data: { path: "#{build_path}.old" } if File.exist?("#{build_path}.old")
+              FileUtils.mv(build_path, "#{build_path}.old")
+            end
+
+            def build_path_empty?
+              (build_path.entries.map(&:to_s) - %w(. .. locks)).empty?
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dapp/dimg/dapp/command/cleanup.rb

@@ -0,0 +1,23 @@
+module Dapp
+  module Dimg
+    class Dapp
+      module Command
+        module Cleanup
+          def cleanup
+            lock("#{name}.images") do
+              log_step_with_indent(name) do
+                dapp_containers_flush
+                dapp_dangling_images_flush
+                remove_images_by_query([
+                  'docker images',
+                  %(--format '{{if ne "#{stage_cache}" .Repository }}{{.Repository}}:{{.Tag}}{{ end }}'),
+                  %(-f "label=dapp=#{stage_dapp_label}")
+                ].join(' ')) # FIXME: negative filter is not currently supported by the Docker CLI
+              end
+            end
+          end
+        end
+      end
+    end
+  end # Dimg
+end # Dapp

data/lib/dapp/dimg/dapp/command/common.rb

@@ -0,0 +1,96 @@
+module Dapp
+  module Dimg
+    class Dapp
+      module Command
+        module Common
+          protected
+
+          def dapp_images_names
+            shellout!(%(docker images --format="{{.Repository}}:{{.Tag}}" #{stage_cache})).stdout.lines.map(&:strip)
+          end
+
+          def dapp_images_ids
+            shellout!(%(docker images #{stage_cache} -q --no-trunc)).stdout.lines.map(&:strip)
+          end
+
+          def dapp_containers_flush
+            remove_containers_by_query(%(docker ps -a -f "label=dapp" -f "name=#{container_name_prefix}" -q), force: true)
+          end
+
+          def dapp_dangling_images_flush
+            remove_images_by_query(%(docker images -f "dangling=true" -f "label=dapp=#{stage_dapp_label}" -q), force: true)
+          end
+
+          def remove_images_by_query(images_query, force: false)
+            with_subquery(images_query) { |ids| remove_images(ids, force: force) }
+          end
+
+          def remove_images(ids, force: false)
+            remove_base('docker rmi%{force_option} %{ids}', ids.uniq, force: force)
+          end
+
+          def remove_containers_by_query(containers_query, force: false)
+            with_subquery(containers_query) { |ids| remove_containers(ids, force: force) }
+          end
+
+          def remove_containers(ids, force: false)
+            remove_base('docker rm%{force_option} %{ids}', ids.uniq, force: force)
+          end
+
+          def remove_base(query_format, ids, force: false)
+            return if ids.empty?
+            force_option = force ? ' -f' : ''
+            ids.each_slice(50) { |chunk| run_command(format(query_format, force_option: force_option, ids: chunk.join(' '))) }
+          end
+
+          def with_subquery(query)
+            return if (res = shellout!(query).stdout.strip.lines.map(&:strip)).empty?
+            yield(res)
+          end
+
+          def image_labels(image_id)
+            Image::Stage.image_config_option(image_id: image_id, option: 'labels')
+          end
+
+          def run_command(cmd)
+            log(cmd) if log_verbose? || dry_run?
+            shellout!(cmd) unless dry_run?
+          end
+
+          def container_name_prefix
+            name
+          end
+
+          def validate_repo_name(repo)
+            raise(Error::Command, code: :repo_name_incorrect, data: { name: repo }) unless ::Dapp::Dimg::DockerRegistry.repo_name?(repo)
+          end
+
+          def proper_cache_version?
+            !!cli_options[:proper_cache_version]
+          end
+
+          def log_proper_cache(&blk)
+            log_step_with_indent(:'proper cache', &blk)
+          end
+
+          def one_dimg!
+            return if build_configs.one?
+            raise Error::Command, code: :command_unexpected_dimgs_number, data: { dimgs_names: build_configs.map(&:_name).join(' ') }
+          end
+
+          def push_format(dimg_name)
+            if dimg_name.nil?
+              spush_format
+            else
+              '%{repo}:%{dimg_name}-%{tag}'
+            end
+          end
+
+          def spush_format
+            '%{repo}:%{tag}'
+          end
+        end
+      end
+    end
+  end # Dimg
+end # Dapp

data/lib/dapp/dimg/dapp/command/list.rb

@@ -0,0 +1,19 @@
+module Dapp
+  module Dimg
+    class Dapp
+      module Command
+        module List
+          def list
+            build_configs.each do |config|
+              if config._name.nil?
+                log_warning('Dappfile has unnamed dimg!') if log_verbose?
+              else
+                puts config._name
+              end
+            end
+          end
+        end
+      end
+    end
+  end # Dimg
+end # Dapp

data/lib/dapp/dimg/dapp/command/mrproper.rb

@@ -0,0 +1,75 @@
+module Dapp
+  module Dimg
+    class Dapp
+      module Command
+        module Mrproper
+          # rubocop:disable Metrics/PerceivedComplexity, Metrics/CyclomaticComplexity
+          def mrproper
+            log_step_with_indent(:mrproper) do
+              raise Error::Command, code: :mrproper_required_option unless proper_all? || proper_dev_mode_cache? || proper_cache_version?
+
+              dapp_dangling_images_flush
+
+              if proper_all?
+                flush_by_label('dapp')
+              elsif proper_dev_mode_cache?
+                flush_by_label('dapp-dev-mode')
+              elsif proper_cache_version?
+                log_proper_cache do
+                  proper_cache_images = proper_cache_all_images
+                  remove_images(dapp_images_by_label('dapp').select { |id| !proper_cache_images.include?(id) }.map(&:strip))
+                end
+              end
+            end
+          end
+          # rubocop:enable Metrics/PerceivedComplexity, Metrics/CyclomaticComplexity
+
+          protected
+
+          def flush_by_label(label)
+            log_step_with_indent(:containers) { dapp_containers_flush_by_label(label) }
+            log_step_with_indent(:images) { dapp_images_flush_by_label(label) }
+          end
+
+          def proper_all?
+            !!cli_options[:proper_all]
+          end
+
+          def proper_dev_mode_cache?
+            !!cli_options[:proper_dev_mode_cache]
+          end
+
+          def dapp_containers_flush_by_label(label)
+            remove_containers_by_query(%(docker ps -a -f "label=#{label}" -q), force: true)
+          end
+
+          def dapp_dangling_images_flush_by_label(label)
+            remove_images_by_query(%(docker images -f "dangling=true" -f "label=#{label}" -q), force: true)
+          end
+
+          def dapp_images_flush_by_label(label)
+            dapp_dangling_images_flush_by_label(label)
+            remove_images(dapp_images_by_label(label), force: true)
+          end
+
+          def dapp_images_by_label(label)
+            @dapp_images ||= begin
+              shellout!(%(docker images -f "dangling=false" --format="{{.Repository}}:{{.Tag}}" -f "label=#{label}"))
+                .stdout
+                .lines
+                .map(&:strip)
+            end
+          end
+
+          def proper_cache_all_images
+            shellout!([
+              'docker images',
+              '--format="{{.Repository}}:{{.Tag}}"',
+              %(-f "label=dapp-cache-version=#{::Dapp::BUILD_CACHE_VERSION}" -f "dangling=false")
+            ].join(' ')).stdout.lines.map(&:strip)
+          end
+        end
+      end
+    end
+  end # Dimg
+end # Dapp

data/lib/dapp/dimg/dapp/command/push.rb

@@ -0,0 +1,27 @@
+module Dapp
+  module Dimg
+    class Dapp
+      module Command
+        module Push
+          def push(repo)
+            validate_repo_name(repo)
+            log_step_with_indent(:stages) { stages_push(repo) } if with_stages?
+            build_configs.each do |config|
+              log_dimg_name_with_indent(config) do
+                Dimg.new(config: config, dapp: self, ignore_git_fetch: true, should_be_built: true).tap do |dimg|
+                  dimg.export!(repo, format: push_format(config._name))
+                end
+              end
+            end
+          end
+
+          protected
+
+          def with_stages?
+            !!cli_options[:with_stages]
+          end
+        end
+      end
+    end
+  end # Dimg
+end # Dapp

data/lib/dapp/dimg/dapp/command/run.rb

@@ -0,0 +1,15 @@
+module Dapp
+  module Dimg
+    class Dapp
+      module Command
+        module Run
+          def run(docker_options, command)
+            one_dimg!
+            setup_ssh_agent
+            Dimg.new(config: build_configs.first, dapp: self, ignore_git_fetch: true, should_be_built: true).run(docker_options, command)
+          end
+        end
+      end
+    end
+  end # Dimg
+end # Dapp

data/lib/dapp/dimg/dapp/command/spush.rb

@@ -0,0 +1,17 @@
+module Dapp
+  module Dimg
+    class Dapp
+      module Command
+        module Spush
+          def spush(repo)
+            validate_repo_name(repo)
+            one_dimg!
+            Dimg.new(config: build_configs.first, dapp: self, ignore_git_fetch: true, should_be_built: true).tap do |dimg|
+              dimg.export!(repo, format: spush_format)
+            end
+          end
+        end
+      end
+    end
+  end # Dimg
+end # Dapp

data/lib/dapp/dimg/dapp/command/stage_image.rb

@@ -0,0 +1,14 @@
+module Dapp
+  module Dimg
+    class Dapp
+      module Command
+        module StageImage
+          def stage_image
+            one_dimg!
+            puts Dimg.new(config: build_configs.first, dapp: self, ignore_git_fetch: true).stage_image_name(cli_options[:stage])
+          end
+        end
+      end
+    end
+  end # Dimg
+end # Dapp

data/lib/dapp/dimg/dapp/command/stages/cleanup_local.rb

@@ -0,0 +1,137 @@
+module Dapp
+  module Dimg
+    class Dapp
+      module Command
+        module Stages
+          module CleanupLocal
+            def stages_cleanup_local(repo)
+              lock_repo(repo, readonly: true) do
+                raise Error::Command, code: :stages_cleanup_required_option unless stages_cleanup_option?
+
+                dapp_containers_flush
+
+                proper_cache                 if proper_cache_version?
+                stages_cleanup_by_repo(repo) if proper_repo_cache?
+                proper_git_commit            if proper_git_commit?
+              end
+            end
+
+            protected
+
+            def proper_cache
+              log_proper_cache do
+                lock("#{name}.images") do
+                  log_step_with_indent(name) do
+                    remove_images(dapp_images_names.select { |image_name| !actual_cache_images.include?(image_name) })
+                  end
+                end
+              end
+            end
+
+            def stages_cleanup_by_repo(repo)
+              registry = registry(repo)
+              repo_dimgs = repo_dimgs_images(registry)
+
+              lock("#{name}.images") do
+                log_step_with_indent(name) do
+                  dapp_dangling_images_flush
+                  dimgs, stages = dapp_images_hash.partition { |_, image_id| repo_dimgs.values.include?(image_id) }
+                  dimgs = dimgs.to_h
+                  stages = stages.to_h
+                  dimgs.each { |_, aiid| except_image_with_parents(aiid, stages) }
+                  remove_images(stages.keys)
+                end
+              end
+            end
+
+            def repo_dimgs_images(registry)
+              repo_dimgs_and_cache(registry).first
+            end
+
+            def actual_cache_images
+              shellout!([
+                'docker images',
+                '--format="{{.Repository}}:{{.Tag}}"',
+                %(-f "label=dapp-cache-version=#{::Dapp::BUILD_CACHE_VERSION}"),
+                stage_cache
+              ].join(' ')).stdout.lines.map(&:strip)
+            end
+
+            def dapp_images_hash
+              shellout!(%(docker images --format "{{.Repository}}:{{.Tag}};{{.ID}}" --no-trunc #{stage_cache})).stdout.lines.map do |line|
+                line.strip.split(';')
+              end.to_h
+            end
+
+            def except_image_with_parents(image_id, stages)
+              if image_exist?(image_id)
+                image_dapp_artifacts_label(image_id).each { |aiid| except_image_with_parents(aiid, stages) }
+                iid = image_id
+                loop do
+                  stages.delete_if { |_, siid| siid == iid }
+                  break if (iid = image_parent(iid)).empty?
+                end
+              else
+                stages.delete_if { |_, siid| siid == image_id }
+              end
+            end
+
+            def image_exist?(image_id)
+              shellout!(%(docker inspect #{image_id}))
+              true
+            rescue Error::Shellout
+              false
+            end
+
+            def image_dapp_artifacts_label(image_id)
+              select_dapp_artifacts_ids(::Dapp::Dimg::Image::Docker.image_config_option(image_id: image_id, option: 'labels'))
+            end
+
+            def image_parent(image_id)
+              shellout!(%(docker inspect -f {{.Parent}} #{image_id})).stdout.strip
+            end
+
+            def proper_git_commit
+              log_proper_git_commit do
+                unproper_images_names = []
+                dapp_images_detailed.each do |_, attrs|
+                  attrs['Config']['Labels'].each do |repo_name, commit|
+                    next if (repo = dapp_git_repositories[repo_name]).nil?
+                    unproper_images_names.concat(image_hierarchy_by_id(attrs['Id'])) unless repo.commit_exists?(commit)
+                  end
+                end
+                remove_images(unproper_images_names.uniq)
+              end
+            end
+
+            def dapp_images_detailed
+              @dapp_images_detailed ||= {}.tap do |images|
+                dapp_images_names.each do |image_name|
+                  shellout!(%(docker inspect --format='{{json .}}' #{image_name})).stdout.strip.tap do |output|
+                    images[image_name] = output == 'null' ? {} : JSON.parse(output)
+                  end
+                end
+              end
+            end
+
+            def image_hierarchy_by_id(image_id)
+              hierarchy = []
+              iids = [image_id]
+
+              loop do
+                hierarchy.concat(dapp_images_detailed.map { |name, attrs| name if iids.include?(attrs['Id']) }.compact)
+                break if begin
+                  iids.map! do |iid|
+                    dapp_images_detailed.map { |_, attrs| attrs['Id'] if attrs['Parent'] == iid }.compact
+                  end.flatten!.empty?
+                end
+              end
+
+              hierarchy
+            end
+          end
+        end
+      end
+    end
+  end # Dimg
+end # Dapp