dapp 0.10.2 → 0.36.24

data/lib/dapp/dimg/git_repo/remote.rb

@@ -2,38 +2,69 @@ module Dapp
   module Dimg
     module GitRepo
       class Remote < Base
-        def initialize(dimg, name, url:)
-          super(dimg, name)
+        CACHE_VERSION = 4
+
+        attr_reader :url
+
+        class << self
+          def get_or_create(dapp, name, url:)
+            repositories[url] ||= new(dapp, name, url: url).tap(&:clone_and_fetch)
+          end
+
+          def repositories
+            @repositories ||= {}
+          end
+        end
+
+        def initialize(dapp, name, url:)
+          super(dapp, name)
 
           @url = url
+        end
 
-          dimg.dapp.log_secondary_process(dimg.dapp.t(code: 'process.git_artifact_clone', data: { name: name }), short: true) do
-            begin
-              Rugged::Repository.clone_at(url, path, bare: true)
-            rescue Rugged::NetworkError, Rugged::SslError => e
-              raise Error::Rugged, code: :rugged_remote_error, data: { message: e.message, url: url }
-            end
-          end unless File.directory?(path)
+        def url
+          @url ||= remote_origin_url
         end
 
-        def path
-          dimg.build_path("#{name}.git").to_s
+        def ruby2go_method(method, args_hash={})
+          res = dapp.ruby2go_git_repo(args_hash.merge("RemoteGitRepo" => JSON.dump(get_ruby2go_state_hash), "method" => method))
+
+          raise res["error"] if res["error"]
+
+          self.set_ruby2go_state_hash(JSON.load(res["data"]["RemoteGitRepo"]))
+
+          res["data"]["result"]
+        end
+
+        def get_ruby2go_state_hash
+          super.tap {|res|
+            res["Url"] = @url.to_s
+            res["ClonePath"] = dapp.build_path("remote_git_repo", CACHE_VERSION.to_s, dapp.consistent_uniq_slugify(name), url_protocol(url)).to_s # FIXME
+            res["IsDryRun"] = dapp.dry_run?
+          }
         end
 
-        def fetch!(branch = nil)
-          branch ||= self.branch
-          dimg.dapp.log_secondary_process(dimg.dapp.t(code: 'process.git_artifact_fetch', data: { name: name }), short: true) do
-            git.fetch('origin', [branch])
-            raise Error::Rugged, code: :branch_not_exist_in_remote_git_repository, data: { branch: branch, url: url } unless branch_exist?(branch)
-          end unless dimg.ignore_git_fetch || dimg.dapp.dry_run?
+        def set_ruby2go_state_hash(state)
+          super(state)
+          @url = state["Url"]
         end
 
-        def branch_exist?(name)
-          git.branches.exist?(branch_format(name))
+        def _rugged_credentials # TODO remove
+          @_rugged_credentials ||= begin
+            if url_protocol(url) == :ssh
+              host_with_user = url.split(':', 2).first
+              username = host_with_user.split('@', 2).reverse.last
+              Rugged::Credentials::SshKeyFromAgent.new(username: username)
+            end
+          end
         end
 
-        def latest_commit(name)
-          git.ref("refs/remotes/#{branch_format(name)}").target_id
+        def path
+          Pathname(dapp.build_path("remote_git_repo", CACHE_VERSION.to_s, dapp.consistent_uniq_slugify(name), url_protocol(url)).to_s)
+        end
+
+        def clone_and_fetch
+          return ruby2go_method("CloneAndFetch")
         end
 
         def lookup_commit(commit)
@@ -44,15 +75,23 @@ module Dapp
 
         protected
 
-        attr_reader :url
-
-        def git
-          super(bare: true)
+        def git # TODO remove
+          super(bare: true, credentials: _rugged_credentials)
         end
 
         private
 
-        def branch_format(name)
+        def url_protocol(url) # TODO remove
+          if (scheme = URI.parse(url).scheme).nil?
+            :noname
+          else
+            scheme.to_sym
+          end
+        rescue URI::InvalidURIError
+          :ssh
+        end
+
+        def branch_format(name) # TODO remove
           "origin/#{name.reverse.chomp('origin/'.reverse).reverse}"
         end
       end

data/lib/dapp/dimg/image/argument.rb

@@ -11,11 +11,11 @@ module Dapp
         end
 
         def add_change_env(**options)
-          add_change_option(:env, options)
+          (change_options[:env] ||= {}).merge!(options)
         end
 
         def add_change_label(**options)
-          add_change_option(:label, options)
+          (change_options[:label] ||= {}).merge!(options)
         end
 
         def add_change_cmd(value)
@@ -31,19 +31,19 @@ module Dapp
         end
 
         def add_change_workdir(value)
-          add_change_option(:workdir, value)
+          change_options[:workdir] = value
         end
 
         def add_change_user(value)
-          add_change_option(:user, value)
+          change_options[:user] = value
         end
 
         def add_service_change_label(**options)
-          add_service_change_option(:label, options)
+          (service_change_options[:label] ||= {}).merge!(options)
         end
 
-        def add_env(var, value)
-          add_option(:env, "#{var}=#{value}")
+        def add_env(**options)
+          (self.options[:env] ||= {}).merge!(options)
         end
 
         def add_volume(value)
@@ -65,8 +65,12 @@ module Dapp
         def prepare_instructions(options)
           options.map do |key, vals|
             case key
-            when :cmd, :entrypoint then [vals]
-            when :env, :label then vals.map(&method(:options_to_args)).flatten
+            when :workdir, :user
+              [vals]
+            when :cmd, :entrypoint
+              vals = [''] if vals == [] && ::Dapp::Dapp.host_docker_minor_version >= Gem::Version.new('17.10')
+              [vals]
+            when :env, :label then options_to_args(vals)
             else vals
             end.map { |val| %(#{key.to_s.upcase} #{val}) }
           end.flatten
@@ -75,7 +79,8 @@ module Dapp
         protected
 
         attr_reader :bash_commands, :service_bash_commands
-        attr_reader :change_options, :service_change_options
+        attr_reader :change_options
+        attr_reader :service_change_options
         attr_reader :options
 
         def add_option(key, value)
@@ -94,49 +99,9 @@ module Dapp
           hash[key] = (hash[key].nil? ? [value] : (hash[key] << value)).flatten
         end
 
-        def from_change_options
-          return {} if from.nil?
-          [:entrypoint, :cmd].each_with_object({}) do |option, options|
-            options[option] = self.class.image_config_option(image_id: from.built_id, option: option)
-          end
-        end
-
         def options_to_args(options)
           options.map { |key, value| "#{key}=#{value}" }
         end
-
-        def prepared_options
-          all_options.map { |key, vals| Array(vals).map { |val| "--#{key}=#{val}" } }.flatten.join(' ')
-        end
-
-        def all_options
-          service_options.merge(options)
-        end
-
-        def all_bash_commands
-          Array(bash_commands) + Array(service_bash_commands)
-        end
-
-        def service_options
-          { entrypoint: dapp.bash_bin, name: container_name }
-        end
-
-        def prepared_change
-          prepare_instructions(all_change_options).map { |instruction| %(-c '#{instruction}') }.join(' ')
-        end
-
-        def all_change_options
-          from_change_options.merge(change_options.merge(service_change_options) { |_, v1, v2| [v1, v2].flatten })
-        end
-
-        def prepared_bash_command
-          dapp.shellout_pack prepared_commands.join(' && ')
-        end
-
-        def prepared_commands
-          return [dapp.true_bin] if all_bash_commands.empty?
-          all_bash_commands
-        end
       end
     end # Image
   end # Dimg

data/lib/dapp/dimg/image/dimg.rb

@@ -0,0 +1,15 @@
+module Dapp
+  module Dimg
+    module Image
+      class Dimg < Stage
+        def export!(export_name = name)
+          super(export_name)
+        end
+
+        def tag!(tag_name = name)
+          super(tag_name)
+        end
+      end
+    end # Image
+  end # Dimg
+end # Dapp

data/lib/dapp/dimg/image/scratch.rb

@@ -2,6 +2,8 @@ module Dapp
   module Dimg
     module Image
       class Scratch < Stage
+        include Helper::Tar
+
         def initialize(**_kwargs)
           super
           @from_archives = []
@@ -12,20 +14,44 @@ module Dapp
         end
 
         def build!(**_kwargs)
-          raise
-
-          # FIXME: system-shellout rejected
-          # build_from_command = if from_archives.empty?
-          #                        "#{dapp.tar_bin} c --files-from /dev/null"
-          #                      else
-          #                        "#{dapp.cat_bin} #{from_archives.join(' ')}"
-          #                      end
-          # @built_id = dapp.system_shellout!("#{build_from_command} | docker import #{prepared_change} - ").stdout.strip
+          @built_id = dapp.shellout!("docker import #{prepared_change} #{archive}").stdout.strip
+        ensure
+          FileUtils.rm_rf(tmp_path)
         end
 
         protected
 
         attr_accessor :from_archives
+
+        def archive
+          tmp_path('archive.tar').tap do |archive_path|
+            tar_write(archive_path) do |common_tar|
+              from_archives.each do |from_archive|
+                tar_gz_read(from_archive) do |tar|
+                  tar.each_entry do |entry|
+                    mode = entry.header.mode
+                    path = entry.full_name
+
+                    if entry.directory?
+                      common_tar.mkdir path, mode
+                    elsif entry.symlink?
+                      common_tar.add_symlink path, entry.header.linkname, mode
+                    else
+                      common_tar.add_file path, mode do |tf|
+                        tf.write entry.read
+                      end
+                    end
+                  end
+                end
+              end
+            end
+          end
+        end
+
+        def tmp_path(*path)
+          @tmp_path ||= Dir.mktmpdir('dapp-scratch-', dapp.tmp_base_dir)
+          dapp.make_path(@tmp_path, *path).expand_path.tap { |p| p.parent.mkpath }
+        end
       end # Stage
     end # Image
   end # Dimg

data/lib/dapp/dimg/image/stage.rb

@@ -1,11 +1,60 @@
 module Dapp
   module Dimg
     module Image
-      class Stage < Docker
+      class Stage
         include Argument
 
+        attr_reader :from
+        attr_reader :name
+        attr_reader :dapp
+
+        class << self
+          def image_by_name(name:, **kwargs)
+            images[name] ||= new(name: name, **kwargs)
+          end
+
+          def image_reset(name)
+            images.delete(name)
+          end
+
+          def images
+            @images ||= {}
+          end
+
+          def image_name_format
+            "#{DockerRegistry.repo_name_format}(:(?<tag>#{tag_format}))?"
+          end
+
+          def tag_format
+            '(?![-.])[a-zA-Z0-9_.-]{1,127}'
+          end
+
+          def image_name?(name)
+            !(/^#{image_name_format}$/ =~ name).nil?
+          end
+
+          def tag?(name)
+            !(/^#{tag_format}$/ =~ name).nil?
+          end
+
+          def save!(dapp, image_or_images, file_path, verbose: false, quiet: false)
+            ruby2go_command(dapp, command: :save, options: { images: Array(image_or_images), file_path: file_path })
+          end
+
+          def load!(dapp, file_path, verbose: false, quiet: false)
+            ruby2go_command(dapp, command: :load, options: { file_path: file_path })
+          end
+
+          def ruby2go_command(dapp, command:, **options)
+            (options[:options] ||= {}).merge!(host_docker_config_dir: dapp.class.host_docker_config_dir)
+            dapp.ruby2go_image({ command: command }.merge(options)).tap do |res|
+              raise Error::Build, code: :ruby2go_image_command_failed_unexpected_error, data: { command: command, message: res["error"] } unless res["error"].nil?
+              break res['data']
+            end
+          end
+        end
+
         def initialize(name:, dapp:, built_id: nil, from: nil)
-          @container_name = "#{name[/[[^:].]*/]}.#{SecureRandom.hex(4)}"
           @built_id = built_id
 
           @bash_commands          = []
@@ -14,80 +63,210 @@ module Dapp
           @change_options         = {}
           @service_change_options = {}
 
-          super(name: name, dapp: dapp, from: from)
+          @from = from
+          @name = name
+          @dapp = dapp
+        end
+
+        def tagged?
+          not image_inspect.empty?
+        end
+
+        def built?
+          !built_id.nil?
         end
 
         def built_id
           @built_id ||= id
         end
 
+        def id
+          image_inspect["Id"]
+        end
+
+        def labels
+          built_image_inspect!.fetch('Config', {}).fetch('Labels', {}) || {}
+        end
+
+        def created_at
+          built_image_inspect!["Created"]
+        end
+
+        def size
+          Float(built_image_inspect!["Size"])
+        end
+
+        def built_image_inspect!
+          built_image_inspect
+        end
+
+        def built_image_inspect
+          @built_image_inspect || image_inspect
+        end
+
+        def reset_image_inspect
+          @image_inspect = nil
+        end
+
+        def image_inspect
+          ruby2go_command(:inspect, short_image_option: true) if @image_inspect.nil?
+          @image_inspect
+        end
+
+        def pull!
+          dapp.log_secondary_process(dapp.t(code: 'process.image_pull', data: { name: name })) do
+            ruby2go_command(:pull)
+          end
+        end
+
+        def push!
+          dapp.log_secondary_process(dapp.t(code: 'process.image_push', data: { name: name })) do
+            ruby2go_command(:push)
+          end
+        end
+
         def build!
-          run!
-          @built_id = commit!
-        ensure
-          dapp.shellout("docker rm #{container_name}")
+          res = self.dapp.ruby2go_image(**ruby2go_image_build_options)
+          if res["error"].nil?
+            set_ruby2go_state_hash(JSON.load(res['data']['image']))
+          elsif res["error"].start_with? "container run failed"
+            raise Error::Build, code: :ruby2go_image_command_failed, data: { command: "build" }
+          else
+            raise Error::Build, code: :ruby2go_image_command_failed_unexpected_error, data: { command: "build", message: res["error"] }
+          end
         end
 
-        def built?
-          !built_id.nil?
+        def ruby2go_image_build_options
+          {
+            image: ruby2go_image_option,
+            command: :build,
+            options: {
+              introspection: {
+                before: dapp.introspect_before_error?,
+                after: dapp.introspect_error?
+              },
+              host_docker_config_dir: dapp.class.host_docker_config_dir,
+            }
+          }
+        end
+
+        def introspect!
+          ruby2go_command(:introspect)
         end
 
         def export!(name)
-          tag!(name).tap do |image|
-            image.push!
-            image.untag!
-          end
+          ruby2go_command(:export, options: { name: name })
         end
 
         def tag!(name)
-          clone!(name).tap do |image|
-            self.class.tag!(id: image.built_id, tag: image.name)
-          end
+          ruby2go_command(:tag, options: { name: name })
         end
 
         def import!(name)
-          clone!(name).tap do |image|
-            image.pull!
-            @built_id = image.built_id
-            save_in_cache!
-            image.untag!
-          end
+          ruby2go_command(:import, options: { name: name })
         end
 
         def save_in_cache!
-          dapp.log_warning(desc: { code: :another_image_already_tagged }) if !(existed_id = id).nil? && built_id != existed_id
-          self.class.tag!(id: built_id, tag: name)
+          ruby2go_command(:save_in_cache)
         end
 
-        def labels
-          raise Error::Build, code: :image_not_exist, data: { name: name } if built_id.nil?
-          self.class.image_config_option(image_id: built_id, option: 'labels')
+        def untag!
+          ruby2go_command(:untag)
+        end
+
+        def ruby2go_command(command, short_image_option: false, options: {})
+          command_options = ruby2go_command_options(command, short_image_option: short_image_option).in_depth_merge(options: options)
+          self.class.ruby2go_command(dapp, **command_options).tap do |data|
+            set_ruby2go_state_hash(JSON.load(data['image']))
+          end
         end
 
-        protected
+        def ruby2go_command_options(command, short_image_option: false)
+          image = begin
+            if short_image_option
+              ruby2go_short_image_option
+            else
+              ruby2go_image_option
+            end
+          end
 
-        attr_reader :container_name
+          {
+            image: image,
+            command: command,
+          }
+        end
 
-        def run!
-          raise Error::Build, code: :built_id_not_defined if from.built_id.nil?
-          dapp.shellout!("docker run #{prepared_options} #{from.built_id} -ec '#{prepared_bash_command}'", log_verbose: true)
-        rescue ::Dapp::Error::Shellout => error
-          dapp.log_warning(desc: { code: :launched_command, data: { command: prepared_commands.join(' && ') }, context: :container })
+        def ruby2go_image_option
+          JSON.dump(get_ruby2go_state_hash)
+        end
 
-          raise unless dapp.introspect_error? || dapp.introspect_before_error?
-          built_id = dapp.introspect_error? ? commit! : from.built_id
-          raise Exception::IntrospectImage, data: { built_id: built_id,
-                                                    options: prepared_options,
-                                                    rmi: dapp.introspect_error?,
-                                                    error: error }
+        def ruby2go_short_image_option
+          JSON.dump(get_ruby2go_short_state_hash)
         end
 
-        def commit!
-          dapp.shellout!("docker commit #{prepared_change} #{container_name}").stdout.strip
+        def get_ruby2go_state_hash
+          [
+            :name,
+            :from,
+            :built_id,
+            :built_image_inspect,
+            :image_inspect,
+            :bash_commands,
+            :service_bash_commands,
+            :options,
+            :change_options,
+            :service_change_options,
+          ].map do |name|
+            if name == :from
+              [name, from.get_ruby2go_state_hash] unless from.nil?
+            elsif name == :built_image_inspect && built_image_inspect.empty?
+            elsif name == :image_inspect && image_inspect.empty?
+            else
+              [name, send(name)]
+            end
+          end
+            .compact
+            .to_h
         end
 
-        def clone!(name)
-          self.class.new(name: name, dapp: dapp, built_id: built_id)
+        def get_ruby2go_short_state_hash
+          [
+            :name,
+            :bash_commands,
+            :service_bash_commands,
+            :options,
+            :change_options,
+            :service_change_options,
+          ].map do |name|
+            [name, send(name)]
+          end
+            .compact
+            .to_h
+        end
+
+        def set_ruby2go_state_hash(state_hash)
+          state_hash.each do |name, value|
+            variable = "@#{name}".to_sym
+
+            case name
+            when "from"
+              from.set_ruby2go_state_hash(value) unless from.nil? || value.nil?
+            when "built_id"
+              if value.to_s.empty?
+                @built_id = nil
+              else
+                @built_id = value
+              end
+            when "image_inspect"
+              instance_variable_set(variable, (value || {}))
+            when "options", "change_options", "service_change_options"
+              instance_variable_set(variable, (value || {}).reject { |_, v| v.nil? || v.empty? }.symbolize_keys)
+            when "bash_commands", "service_bash_commands"
+              instance_variable_set(variable, value || [])
+            else
+              instance_variable_set(variable, value)
+            end
+          end
         end
       end # Stage
     end # Image

data/lib/dapp/dimg/lock/file.rb

@@ -10,18 +10,18 @@ module Dapp
           end
         end # << self
 
-        attr_reader :lock_path
+        attr_reader :locks_path
 
-        def initialize(lock_path, name)
+        def initialize(locks_path, name)
           super(name)
 
-          @lock_path = Pathname.new(lock_path).tap(&:mkpath)
+          @locks_path = Pathname.new(locks_path).tap(&:mkpath)
         end
 
         protected
 
         def lock_file_path
-          lock_path.join(MurmurHash3::V32.str_hexdigest(name))
+          locks_path.join(MurmurHash3::V32.str_hexdigest(name))
         end
 
         def _do_lock(timeout, on_wait, readonly)