dapp 0.10.2 → 0.36.24

data/lib/dapp/dapp/ruby2go.rb

@@ -0,0 +1,130 @@
+module Dapp
+  class Dapp
+    module Ruby2Go
+      class Error < ::Dapp::Error::Base
+      end
+
+      def ruby2go_image(args_hash)
+        _ruby2go("image", args_hash)
+      end
+
+      def ruby2go_docker_registry(args_hash)
+        _ruby2go("docker_registry", args_hash)
+      end
+
+      def ruby2go_builder(args_hash)
+        _ruby2go("builder", args_hash)
+      end
+
+      def ruby2go_git_artifact(args_hash)
+        _ruby2go("git-artifact", args_hash)
+      end
+
+      def ruby2go_dappdeps(args_hash)
+        _ruby2go("dappdeps", args_hash)
+      end
+
+      def ruby2go_git_repo(args_hash)
+        _ruby2go("git-repo", args_hash)
+      end
+
+      def ruby2go_cleanup(args_hash)
+        _ruby2go("cleanup", args_hash)
+      end
+
+      def ruby2go_slug(args_hash)
+        _ruby2go("slug", args_hash)
+      end
+
+      def ruby2go_deploy_watcher(args_hash, **kwargs)
+        _ruby2go("deploy-watcher", args_hash, **kwargs)
+      end
+
+      def ruby2go_deploy(args_hash)
+        _ruby2go("deploy", args_hash)
+      end
+
+      def ruby2go_init
+        @_call_after_before_terminate << proc {
+          FileUtils.rmtree(@_ruby2go_tmp_dir) if @_ruby2go_tmp_dir
+        }
+      end
+
+      def _ruby2go_bin_path_env_var_name(progname)
+        "DAPP_BIN_#{progname.gsub("-", "_").upcase}"
+      end
+
+      def _ruby2go(progname, args_hash, tmp_dir: nil)
+        tmp_dir = _ruby2go_tmp_dir if tmp_dir.nil?
+
+        call_id = SecureRandom.uuid
+
+        args_file = File.join(tmp_dir, "args.#{call_id}.json")
+        File.open(args_file, "w") {|f| f.write JSON.dump(args_hash)}
+
+        res_file = File.join(tmp_dir, "res.#{call_id}.json")
+
+        if bin_path = ENV[_ruby2go_bin_path_env_var_name(progname)]
+          unless File.exists? bin_path
+            raise ::Dapp::Error::Dapp,
+              code: :ruby2go_bin_path_not_found,
+              data: {env_var_name: _ruby2go_bin_path_env_var_name(progname), path: bin_path}
+          end
+        else
+          bin_path = File.join(::Dapp::Dapp.home_dir, "bin", progname, ::Dapp::VERSION, progname)
+          unless File.exists? bin_path
+            _download_ruby2go_bin(progname, bin_path)
+          end
+        end
+
+        env_hash = ENV.map {|k, v| [k, v]}.to_h
+
+        begin
+          exec(env_hash, "#{bin_path} -args-from-file #{args_file} -result-to-file #{res_file}") unless (pid = fork)
+          pid, status = Process.waitpid2(pid)
+        rescue Interrupt => _e
+          Process.kill('INT', pid)
+          raise
+        end
+
+        status_code = status.exitstatus
+        if [0, 16].include?(status_code)
+          res = nil
+          File.open(res_file, "r") {|f| res = JSON.load(f.read)}
+          res
+        else
+          raise ::Dapp::Dapp::Ruby2Go::Error, code: :ruby2go_command_unexpected_exitstatus, data: { progname: progname, status_code: status_code }
+        end
+      end
+
+      def _ruby2go_tmp_dir
+        @_ruby2go_tmp_dir ||= Dir.mktmpdir('dapp-ruby2go-', tmp_base_dir)
+      end
+
+      def _download_ruby2go_bin(progname, bin_path)
+        lock("downloader.bin.#{progname}", default_timeout: 1800) do
+          return if File.exists? bin_path
+
+          log_process("Downloading #{progname} dapp dependency") do
+            location = URI("https://dl.bintray.com/dapp/ruby2go/#{::Dapp::VERSION}/#{progname}")
+
+            tmp_bin_path = File.join(self.class.tmp_base_dir, "#{progname}-#{SecureRandom.uuid}")
+            ::Dapp::Downloader.download(location, tmp_bin_path, show_progress: true, progress_titile: bin_path)
+
+            checksum_location = URI("https://dl.bintray.com/dapp/ruby2go/#{::Dapp::VERSION}/#{progname}.sha")
+            tmp_bin_checksum_path = tmp_bin_path + ".checksum"
+            ::Dapp::Downloader.download(checksum_location, tmp_bin_checksum_path)
+
+            if Digest::SHA256.hexdigest(File.read(tmp_bin_path)) != File.read(tmp_bin_checksum_path).strip
+              raise ::Dapp::Error::Dapp, code: :ruby2go_download_failed_bad_checksum, data: {url: location.to_s, checksum_url: checksum_location.to_s, progname: progname}
+            end
+
+            File.chmod(0755, tmp_bin_path)
+            FileUtils.mkdir_p File.dirname(bin_path)
+            FileUtils.mv tmp_bin_path, bin_path
+          end # log_process
+        end # lock
+      end
+    end # Ruby2Go
+  end # Dapp
+end # Dapp

data/lib/dapp/dapp/sentry.rb

@@ -0,0 +1,97 @@
+module Dapp
+  class Dapp
+    module Sentry
+      def sentry_message(msg, **kwargs)
+        return if not ensure_sentry_configured
+        kwargs[:level] ||= "info"
+        Raven.capture_message(msg, _make_sentry_params(**kwargs))
+      end
+
+      def sentry_exception(exception, **kwargs)
+        return if not ensure_sentry_configured
+        (kwargs[:tags] ||= {})['error-code'] = begin
+          net_status = exception.net_status
+          [net_status[:context], net_status[:code]].compact.join('_')
+        end
+        Raven.capture_exception(exception, _make_sentry_params(**kwargs))
+      end
+
+      def ensure_sentry_configured
+        return false unless sentry_settings = settings["sentry"]
+
+        unless @sentry_settings_configured
+          Raven.configure do |config|
+            logger = ::Logger.new(STDOUT)
+            logger.level = ::Logger::WARN
+
+            config.logger = logger
+            config.dsn = sentry_settings["dsn"]
+          end
+
+          @sentry_settings_configured = true
+        end
+
+        return true
+      end
+
+      def _make_sentry_params(level: nil, tags: {}, extra: {}, user: {})
+        Marshal.load(Marshal.dump({
+          level: level,
+          tags:  _sentry_tags_context.merge(tags),
+          extra: _sentry_extra_context.merge(extra),
+          user:  _sentry_user_context.merge(user),
+        }))
+      end
+
+      def _sentry_extra_context
+        @_sentry_extra_context ||= {
+          "pwd" => Dir.pwd,
+          "dapp-dir" => self.work_dir,
+          "build-dir" => self.build_dir,
+          "options" => self.options,
+          "env-options" => {
+            "DAPP_FORCE_SAVE_CACHE" => ENV["DAPP_FORCE_SAVE_CACHE"],
+            "DAPP_BIN_DAPPFILE_YML" => ENV["DAPP_BIN_DAPPFILE_YML"],
+            "ANSIBLE_ARGS" => ENV["ANSIBLE_ARGS"],
+            "DAPP_CHEF_DEBUG" => ENV["DAPP_CHEF_DEBUG"],
+          },
+        }.tap {|extra|
+          extra["ci-env"] = {"CI" => ENV["CI"]}
+          ENV.select {|k, v| k.start_with?("CI_")}.each do |k, v|
+            extra["ci-env"][k] = v
+          end
+        }
+      end
+
+      def _sentry_tags_context
+        @_sentry_tags_context ||= {
+          "dapp-name" => self.name,
+          "dapp-short-version" => ::Dapp::VERSION.split(".")[0..1].join("."),
+          "dapp-version" => ::Dapp::VERSION,
+          "dapp-build-cache-version" => ::Dapp::BUILD_CACHE_VERSION,
+          "dapp-command" => self.options[:dapp_command],
+        }.tap {|tags|
+          # FIXME: precache this info after Dapp object initialization
+          #if git_own_repo_exist? && git_own_repo.remote_origin_url
+            #tags["git-host"] = self.get_host_from_git_url(git_own_repo.remote_origin_url)
+
+            #git_name = self.git_url_to_name(git_own_repo.remote_origin_url)
+
+            #tags["git-group"] = git_name.partition("/")[0]
+            #tags["git-name"] = git_name
+          #end
+
+          begin
+            ver = self.class.host_docker_minor_version
+            tags["docker-minor-version"] = ver.to_s
+          rescue ::Exception
+          end
+        }
+      end
+
+      def _sentry_user_context
+        @__sentry_user_context ||= {}
+      end
+    end # Sentry
+  end # Dapp
+end # Dapp

data/lib/dapp/dapp/shellout/base.rb

@@ -19,7 +19,7 @@ module Dapp
           env = (env || {}).merge(param_env) if param_env
           kwargs[:env] = env if env
 
-          do_shellout = -> { ::Mixlib::ShellOut.new(*args, timeout: 3600, **kwargs).run_command }
+          do_shellout = -> { ::Mixlib::ShellOut.new(*args, timeout: 86400, **kwargs).run_command }
           if defined? ::Bundler
             ::Bundler.with_clean_env { do_shellout.call }
           else
@@ -28,13 +28,16 @@ module Dapp
         end
 
         def shellout!(*args, **kwargs)
-          shellout_with_logging(**kwargs) do |options|
-            shellout(*args, **options).tap(&:error!)
+          if instance_of? Dapp
+            default_kwarg = proc { |key, value| kwargs[key] = value unless kwargs.key?(key) }
+            default_kwarg.call(:quiet, log_quiet?)
+            default_kwarg.call(:time, log_time?)
           end
+          _shellout_with_logging!(*args, **kwargs)
         end
 
         def shellout_pack(command)
-          "eval $(echo #{Base64.strict_encode64(command)} | base64 --decode)"
+          "eval $(echo #{Base64.strict_encode64(command)} | #{base64_bin} --decode)"
         end
 
         class << self
@@ -47,24 +50,29 @@ module Dapp
           end
         end # << self
 
-        protected
+        def shellout_cmd_should_succeed!(cmd)
+          return cmd.tap(&:error!)
+        rescue ::Mixlib::ShellOut::ShellCommandFailed => e
+          raise Error::Shellout, code: Helper::Trivia.class_to_lowercase(e.class), data: { stream: e.message }
+        end
 
-        def shellout_with_logging(log_verbose: false, **kwargs)
-          return yield(**kwargs) unless instance_of? Dapp
+        protected
 
-          begin
-            stream = Stream.new
-            if log_verbose && log_verbose?
-              kwargs[:live_stream] = Proxy::Base.new(stream, STDOUT, with_time: log_time?)
-            else
-              kwargs[:live_stdout] = Proxy::Base.new(stream, with_time: log_time?)
-            end
-            kwargs[:live_stderr] = Proxy::Error.new(stream, with_time: log_time?)
+        def _shellout_with_logging!(*args, verbose: false, quiet: true, time: false, raise_on_error: true, **kwargs)
+          stream = Stream.new
+          if verbose && !quiet
+            kwargs[:live_stream] = Proxy::Base.new(stream, STDOUT, with_time: time)
+            kwargs[:live_stderr] = Proxy::Error.new(stream, STDERR, with_time: time)
+          else
+            kwargs[:live_stdout] = Proxy::Base.new(stream, with_time: time)
+            kwargs[:live_stderr] = Proxy::Error.new(stream, with_time: time)
+          end
 
-            yield(**kwargs)
-          rescue ::Mixlib::ShellOut::ShellCommandFailed => e
-            raise Error::Shellout, code: class_to_lowercase(e.class), data: { stream: stream.show }
+          shellout(*args, **kwargs).tap do |res|
+            res.error! if raise_on_error
           end
+        rescue ::Mixlib::ShellOut::ShellCommandFailed => e
+          raise Error::Shellout, code: Helper::Trivia.class_to_lowercase(e.class), data: { stream: stream.show }
         end
       end
     end

data/lib/dapp/dapp/shellout/streaming.rb

@@ -28,13 +28,13 @@ module Dapp
             end
 
             def format_string(str)
-              str.lines.map { |l| "#{Dapp.log_time if @with_time}#{l.strip}\n" }.join
+              str.lines.map { |l| "#{Dapp.log_time if @with_time}#{l.chomp}\n" }.join
             end
           end
 
           class Error < Base
             def format_string(str)
-              "#{Dapp.paint_string(super.strip, :warning)}\n"
+              "#{Dapp.paint_string(super.chomp, :warning)}\n"
             end
           end
         end

data/lib/dapp/dapp/slug.rb

@@ -0,0 +1,12 @@
+module Dapp
+  class Dapp
+    module Slug
+      def consistent_uniq_slugify(s)
+        ruby2go_slug({ options: { data: s } }).tap do |res|
+          raise Error::Build, code: :ruby2go_slug_failed_unexpected_error, data: { message: res["error"] } unless res["error"].nil?
+          break res['data']
+        end
+      end
+    end # Slug
+  end # Dapp
+end # Dapp

data/lib/dapp/dapp/ssh_agent.rb

@@ -46,10 +46,24 @@ module Dapp
         shellout! "ssh-add #{ssh_key_path}", env: { SSH_AUTH_SOCK: ssh_auth_sock(force_run_agent: true) }
       end
 
+      def ssh_agent_exist?
+        ENV['SSH_AUTH_SOCK'] && File.exist?(ENV['SSH_AUTH_SOCK'])
+      end
+
+      def default_ssh_keys
+        @default_ssh_keys ||= begin
+          %w(id_rsa id_dsa).map do |path|
+            File.join(ENV["HOME"], ".ssh", path)
+          end.select do |path|
+            File.exist? path
+          end
+        end
+      end
+
       def ssh_auth_sock(force_run_agent: false)
         @ssh_auth_sock ||= begin
           system_ssh_auth_sock = nil
-          system_ssh_auth_sock = File.expand_path(ENV['SSH_AUTH_SOCK']) if ENV['SSH_AUTH_SOCK'] && File.exist?(ENV['SSH_AUTH_SOCK'])
+          system_ssh_auth_sock = File.expand_path(ENV['SSH_AUTH_SOCK']) if ssh_agent_exist?
 
           if force_run_agent
             run_ssh_agent.tap { |ssh_auth_sock| ENV['SSH_AUTH_SOCK'] = ssh_auth_sock }
@@ -60,13 +74,17 @@ module Dapp
       end
 
       def setup_ssh_agent
-        return unless cli_options[:ssh_key]
-
-        cli_options[:ssh_key].each do |ssh_key|
-          raise Error::Dapp, code: :ssh_key_not_found, data: { path: ssh_key } unless File.exist? ssh_key
+        if options[:ssh_key]
+          options[:ssh_key].each do |ssh_key|
+            raise Error::Dapp, code: :ssh_key_not_found, data: { path: ssh_key } unless File.exist? ssh_key
 
-          File.chmod 0o600, ssh_key
-          add_ssh_key ssh_key
+            File.chmod 0o600, ssh_key
+            add_ssh_key ssh_key
+          end
+        elsif (not ssh_agent_exist?) and default_ssh_keys.any?
+          default_ssh_keys.each do |ssh_key|
+            add_ssh_key ssh_key
+          end
         end
       end
     end # SshAgent

data/lib/dapp/dapp.rb

@@ -4,6 +4,15 @@ module Dapp
     include GitArtifact
     include Dappfile
     include Chef
+    include DappConfig
+    include OptionTags
+    include Slug
+
+    include Command::Common
+    include Command::Slug
+    include Command::Sample::Common
+    include Command::Sample::List
+    include Command::Sample::Create
 
     include Logging::Base
     include Logging::Process
@@ -11,78 +20,127 @@ module Dapp
     include Logging::Paint
 
     include SshAgent
+    include Sentry
+    include Ruby2Go
+
     include Helper::Sha256
+    extend  Helper::Trivia
     include Helper::Trivia
+    include Helper::Tar
+    include Helper::Url
 
+    include Deps::Common
+    include Deps::Toolchain
     include Deps::Gitartifact
     include Deps::Base
 
     include Shellout::Base
 
-    attr_reader :cli_options
-    attr_reader :dimgs_patterns
+    GCR_REGISTRIES = %w(asia.gcr.io eu.gcr.io gcr.io marketplace.gcr.io staging-k8s.gcr.io us.gcr.io)
 
-    def initialize(cli_options: {}, dimgs_patterns: nil)
-      @cli_options = cli_options
-      @dimgs_patterns = dimgs_patterns || []
-      @dimgs_patterns << '*' unless @dimgs_patterns.any?
+    attr_reader :options
+
+    def get_ruby2go_state_hash
+      {
+        "Name" => name.to_s,
+        "WorkDir" => work_dir.to_s,
+      }
+    end
 
-      Logging::Paint.initialize(cli_options[:log_color])
+    def initialize(options: {})
+      self.class.options.merge!(options)
       Logging::I18n.initialize
+      validate_config_options!
+      Logging::Paint.initialize(option_color)
+
+      @_call_before_terminate = []
+      @_call_after_before_terminate = []
+
+      ruby2go_init
+
+      setup_ssh_agent
+      try_host_docker_login unless !!options[:ignore_try_host_docker_login]
+    end
+
+    def options
+      self.class.options
+    end
+
+    def settings
+      @settings ||= begin
+        settings_path = File.join(self.class.home_dir, "settings.toml")
+
+        if File.exists? settings_path
+          TomlRB.load_file(settings_path)
+        else
+          {}
+        end
+      end
     end
 
     def name
       @name ||= begin
-        if git_url
-          repo_name = git_url.split('/').last
-          repo_name = repo_name[/.*(?=\.git)/] if repo_name.end_with? '.git'
-          repo_name
-        elsif git_path
-          File.basename(File.dirname(git_path))
-        else
-          File.basename(path)
+        n = begin
+          if (name = options[:name])
+            name
+          elsif git_own_repo_exist?
+            if git_url
+              repo_name = git_url.split('/').last
+              repo_name = repo_name[/.*(?=\.git)/] if repo_name.end_with? '.git'
+              repo_name
+            else
+              File.basename(File.dirname(git_own_repo.path)).to_s
+            end
+          else
+            path.basename.to_s
+          end
         end
+        consistent_uniq_slugify(n)
       end
     end
 
     def git_url
-      return unless git_config
-      (git_config['remote "origin"'] || {})['url']
+      return unless git_own_repo_exist?
+      git_own_repo.remote_origin_url
     end
 
-    def git_config
-      @git_config ||= begin
-        IniFile.load(File.join(git_path, 'config')) if git_path
-      end
+    def git_own_repo_exist?
+      !git_own_repo.nil?
     end
 
-    def git_path
-      defined?(@git_path) ? @git_path : begin
-        @git_path = search_file_upward('.git')
-      end
+    def git_own_repo
+      @git_own_repo ||= Dimg::GitRepo::Own.new(self)
+    rescue Dimg::Error::Rugged => e
+      raise unless e.net_status[:code] == :local_git_repository_does_not_exist
+      nil
+    end
+
+    def work_dir
+      File.expand_path(options[:dir] || Dir.pwd)
+    end
+
+    def path(*path)
+      @path ||= make_path(work_dir)
+      make_path(@path, *path)
     end
 
-    def path
-      @path ||= expand_path(dappfile_path)
+    def tmp_base_dir
+      self.class.tmp_base_dir
     end
 
-    def build_path
-      @build_path ||= begin
-        if cli_options[:build_dir]
-          Pathname.new(cli_options[:build_dir])
+    def build_dir
+      @build_dir ||= begin
+        if option_build_dir
+          Pathname.new(option_build_dir)
         else
-          Pathname.new(path).join('.dapp_build')
+          dir = File.join(self.class.home_dir, "builds", self.name)
+          Pathname.new(dir)
         end.expand_path.tap(&:mkpath)
       end
     end
 
-    def local_git_artifact_exclude_paths(&blk)
-      super do |exclude_paths|
-        build_path_relpath = Pathname.new(build_path).subpath_of(File.dirname(git_path))
-        exclude_paths << build_path_relpath.to_s if build_path_relpath
-
-        yield exclude_paths if block_given?
-      end
+    def build_path(*path)
+      make_path(build_dir, *path)
     end
 
     def stage_cache
@@ -93,8 +151,101 @@ module Dapp
       name
     end
 
-    def dev_mode?
-      !!cli_options[:dev]
+    def terminate
+      @_call_before_terminate.each {|on_terminate| on_terminate.call(self)}
+      @_call_after_before_terminate.each {|on_terminate| on_terminate.call(self)}
+      FileUtils.rmtree(host_docker_tmp_config_dir)
+    end
+
+    def host_docker
+      self.class.host_docker
+    end
+
+    def host_docker_tmp_config_dir
+      self.class.host_docker_tmp_config_dir
+    end
+
+    def try_host_docker_login
+      return unless option_repo
+      validate_repo_name!(option_repo)
+      host_docker_login(option_repo)
+    end
+
+    def host_docker_login(repo)
+      return unless self.class.options_with_docker_credentials?
+
+      username, password = self.class.docker_credentials
+      ::Dapp::Dimg::Image::Stage.ruby2go_command(self, command: :login, options: { username: username, password: password, repository: repo })
+    end
+
+    class << self
+      def home_dir
+        ENV["DAPP_HOME"] || File.join(Dir.home, ".dapp")
+      end
+
+      def options
+        @options ||= {}
+      end
+
+      def host_docker
+        @host_docker ||= begin
+          min_docker_minor_version = Gem::Version.new('1.10')
+          unless host_docker_minor_version > min_docker_minor_version
+            raise Error::Dapp, code: :docker_version, data: { min_version: min_docker_minor_version.to_s,
+                                                              version:     host_docker_minor_version.to_s }
+          end
+
+          [].tap do |cmd|
+            cmd << host_docker_bin
+            cmd << "--config #{host_docker_config_dir}"
+          end.join(' ')
+        end
+      end
+
+      def host_docker_bin
+        raise Error::Dapp, code: :docker_not_found if (res = shellout('which docker')).exitstatus.nonzero?
+        res.stdout.strip
+      end
+
+      def host_docker_minor_version
+        Gem::Version.new(shellout!("#{host_docker_bin} --version").stdout.strip[/\d+\.\d+/])
+      end
+
+      def host_docker_config_dir
+        if options_with_docker_credentials? && !options[:repo].nil?
+          host_docker_tmp_config_dir
+        elsif ENV.key?('DAPP_DOCKER_CONFIG')
+          ENV['DAPP_DOCKER_CONFIG']
+        else
+          File.join(Dir.home, '.docker')
+        end
+      end
+
+      def options_with_docker_credentials?
+        !docker_credentials.nil?
+      end
+
+      def docker_credentials
+        if options[:registry_username] && options[:registry_password]
+          [options[:registry_username], options[:registry_password]]
+        elsif ENV.key?('DAPP_DOCKER_CONFIG')
+        elsif !gcr_registry? && !ENV.key?('DAPP_IGNORE_CI_DOCKER_AUTOLOGIN') && ENV.key?('CI_REGISTRY') && ENV.key?('CI_JOB_TOKEN')
+          ['gitlab-ci-token', ENV['CI_JOB_TOKEN']]
+        end
+      end
+
+      def gcr_registry?
+        return false if options[:repo].nil?
+        GCR_REGISTRIES.any? { |registry| options[:repo].to_s.start_with?(registry) }
+      end
+
+      def host_docker_tmp_config_dir
+        @host_docker_tmp_config_dir ||= Dir.mktmpdir('dapp-', tmp_base_dir)
+      end
+
+      def tmp_base_dir
+        File.expand_path(options[:tmp_dir_prefix] || '/tmp')
+      end
     end
   end # Dapp
 end # Dapp

data/lib/dapp/deployment.rb

@@ -0,0 +1 @@
+module Dapp::Deployment; end