dapp 0.30.2 → 0.30.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 85e24f5f787e11e46f34a01a946986396ead3830
-  data.tar.gz: 075072783e8d9bb26fcf4545901a637bfb1847c0
+  metadata.gz: 5d13edf9865121c85fcc72bf4bbcfeb691fb7d08
+  data.tar.gz: 7a12de433ca2f9427cd66a916666628925d953e6
 SHA512:
-  metadata.gz: 33db546ddeed11f985386d7c55aa48bc3704054745bb83a77e359060eb01d6e7ececf02470933e93776a4b826801fe19702a871d08c7906972ddfa14ae35bcaf
-  data.tar.gz: 25bbfc1a6f66fa507f344939b2e1584f1938f5ad12d62a95cadfe5d4409e894fbcffa9b1e58bbd719f61e65ec43f6c3b6301d169f322e0ca2e694e6760458a94
+  metadata.gz: 1d2988cad1ab659edc3b190425e329a6b3f595d306f4e21a304745505ccf84d1aac52a9d1abf6cbfeade8944731e6dc4f0cd94411ffb25855d84b14ace22f8ea
+  data.tar.gz: 72435eac2f93bd12638d2d7b87bc948e165d3a876bf40c1f9b460b66972918353fe53fbb8647b28f650c92cb92cf1be12761724ba2c01386bfbf3b36693a0e97

data/config/en/net_status.yml

@@ -47,6 +47,8 @@ en:
       kube_deploy_timeout: "Deploy timeout!"
       kube_connect_timeout: "Connect timeout when connecting to kube API."
       kube_helm_failed: "Helm failed: %{output}"
+      samples_directory_not_exist: "Samples directory `%{path}` not exist in repository `%{url}`!"
+      sample_not_exist: "Sample `%{name}` not exist in repository `%{url}` (inside samples directory `%{path}`)!"
     dapp:
       no_such_dimg: "No such dimg: `%{dimgs_patterns}`!"
       no_such_app: "No such app: `%{apps_patterns}`!"

data/lib/dapp.rb

@@ -56,6 +56,10 @@ require 'dapp/cli/options/tag'
 require 'dapp/cli/options/ssh'
 require 'dapp/cli/command/base'
 require 'dapp/cli/command/update'
+require 'dapp/cli/command/sample'
+require 'dapp/cli/command/sample/base'
+require 'dapp/cli/command/sample/list'
+require 'dapp/cli/command/sample/create'
 require 'dapp/cli/command/slug'
 require 'dapp/cli/cli'
 require 'dapp/config/directive/base'
@@ -80,6 +84,9 @@ require 'dapp/dapp/shellout/streaming'
 require 'dapp/dapp/shellout/base'
 require 'dapp/dapp/command/common'
 require 'dapp/dapp/command/slug'
+require 'dapp/dapp/command/sample/common'
+require 'dapp/dapp/command/sample/create'
+require 'dapp/dapp/command/sample/list'
 require 'dapp/dapp'
 require 'dapp/kube'
 require 'dapp/kube/error/default'

data/lib/dapp/cli.rb

@@ -5,7 +5,7 @@ module Dapp
     extend Helper::Cli
     include Helper::Trivia
 
-    SUBCOMMANDS = ['dimg', 'kube', 'update', 'slug'].freeze
+    SUBCOMMANDS = %w(dimg kube update sample slug).freeze
 
     banner <<BANNER.freeze
 Usage: dapp subcommand [subcommand options]
@@ -15,6 +15,7 @@ Available subcommands: (for details, dapp SUB-COMMAND --help)
   dapp dimg
   dapp kube
   dapp update
+  dapp sample
   dapp slug STRING
 
 Options:

data/lib/dapp/cli/command/sample.rb

@@ -0,0 +1,20 @@
+module Dapp
+  class CLI
+    module Command
+      class Sample < ::Dapp::CLI
+        SUBCOMMANDS = %w(list create).freeze
+
+        banner <<BANNER.freeze
+Usage: dapp sample [options] subcommand [subcommand options]
+
+Available subcommands: (for details, dapp sample SUB-COMMAND --help)
+
+  dapp sample list [options]
+  dapp sample create SAMPLE_NAME [options] 
+
+Options:
+BANNER
+      end
+    end
+  end
+end

data/lib/dapp/cli/command/sample/base.rb

@@ -0,0 +1,28 @@
+module Dapp
+  class CLI
+    module Command
+      class Sample < ::Dapp::CLI
+        class Base < Base
+          option :samples_repo,
+                 long:        '--samples-repo GIT_REPO',
+                 description: 'Git repository with samples (\'https://github.com/flant/dapp.git\' by default).',
+                 default:     'https://github.com/flant/dapp.git'
+
+          option :samples_branch,
+                 long:        '--samples-branch GIT_BRANCH',
+                 description: 'Specific git branch (\'master\' by default).',
+                 default:     'master'
+
+          option :samples_dir,
+                 long:        '--samples-dir DIR',
+                 description: 'Directory with samples (\'samples\' by default)',
+                 default:     'samples'
+
+          def run_method
+            "sample_#{super}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dapp/cli/command/sample/create.rb

@@ -0,0 +1,24 @@
+module Dapp
+  class CLI
+    module Command
+      class Sample < ::Dapp::CLI
+        class Create < Base
+          banner <<BANNER.freeze
+Usage:
+
+  dapp sample create SAMPLE_NAME [options]
+
+Options:
+BANNER
+          def run(argv = ARGV)
+            self.class.parse_options(self, argv)
+            sample_name = self.class.required_argument(self, 'sample_name')
+            run_dapp_command(nil, options: cli_options) do |dapp|
+              dapp.sample_create(sample_name)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dapp/cli/command/sample/list.rb

@@ -0,0 +1,21 @@
+module Dapp
+  class CLI
+    module Command
+      class Sample < ::Dapp::CLI
+        class List < Base
+          banner <<BANNER.freeze
+Usage:
+
+  dapp sample list [options]
+
+Options:
+BANNER
+          def run(argv = ARGV)
+            self.class.parse_options(self, argv)
+            run_dapp_command(run_method, options: cli_options)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dapp/dapp.rb

@@ -10,6 +10,9 @@ module Dapp
 
     include Command::Common
     include Command::Slug
+    include Command::Sample::Common
+    include Command::Sample::List
+    include Command::Sample::Create
 
     include Logging::Base
     include Logging::Process

data/lib/dapp/dapp/command/sample/common.rb

@@ -0,0 +1,54 @@
+module Dapp
+  class Dapp
+    module Command
+      module Sample
+        module Common
+          def _sample_list
+            @sample_list ||= [].tap do |list|
+              tree = begin
+                latest_commit = _sample_git_repo.latest_commit(_samples_git_repo_branch)
+                latest_commit_tree = _sample_git_repo.lookup_commit(latest_commit).tree
+
+                if _samples_dir == '.'
+                  latest_commit_tree
+                else
+                  begin
+                    oid = latest_commit_tree.path(_samples_dir)[:oid]
+                  rescue Rugged::TreeError
+                    raise Error::Command, code: :samples_directory_not_exist, data: { url: _samples_git_repo_url, path: _samples_dir }
+                  end
+
+                  _sample_git_repo.lookup_object(oid)
+                end
+              end
+
+              tree.each_tree { |entry| list << entry[:name] }
+            end
+          end
+
+          def _sample_git_repo
+            @sample_repo ||= begin
+              Dimg::GitRepo::Remote.get_or_create(
+                self,
+                git_url_to_name(_samples_git_repo_url),
+                url: _samples_git_repo_url
+              )
+            end
+          end
+
+          def _samples_git_repo_url
+            options[:samples_repo]
+          end
+
+          def _samples_git_repo_branch
+            options[:samples_branch]
+          end
+
+          def _samples_dir
+            options[:samples_dir]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dapp/dapp/command/sample/create.rb

@@ -0,0 +1,57 @@
+module Dapp
+  class Dapp
+    module Command
+      module Sample
+        module Create
+          def sample_create(sample_name)
+            sample_exist!(sample_name)
+
+            _sample_repo_blobs_entries(sample_name).each do |root, entry|
+              file_path = _sample_slice_cwd(sample_name, File.join(root, entry[:name]))
+              content = _sample_git_repo.lookup_object(entry[:oid]).content
+
+              begin
+                FileUtils.mkdir_p(File.dirname(file_path))
+                if entry[:filemode] == 0o120000 # symlink
+                  FileUtils.symlink(content, file_path)
+                else
+                  IO.write(file_path, content)
+                  FileUtils.chmod(entry[:filemode], file_path)
+                end
+              rescue Errno::EEXIST => e
+                log_warning("File `#{file_path}` skipped: `#{e.message}`")
+              end
+            end
+          end
+
+          def sample_exist!(sample_name)
+            return if sample_exist?(sample_name)
+            raise Error::Command, code: :sample_not_exist, data: { name: sample_name, url: _samples_git_repo_url, path: _samples_dir }
+          end
+
+          def sample_exist?(sample_name)
+            _sample_list.include?(sample_name)
+          end
+
+          def _sample_repo_blobs_entries(sample_name)
+            _sample_git_repo
+              .blobs_entries(_sample_git_repo.latest_commit(_samples_git_repo_branch), paths: [_sample_directory(sample_name)])
+              .reject { |_, entry| entry[:filemode] == 0o160000 }
+          end
+
+          def _sample_directory(sample_name)
+            File.expand_path(File.join('/', _samples_dir, sample_name))[1..-1]
+          end
+
+          def _sample_slice_cwd(sample_name, path)
+            path
+              .reverse
+              .chomp(_sample_directory(sample_name).reverse)
+              .chomp('/')
+              .reverse
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dapp/dapp/command/sample/list.rb

@@ -0,0 +1,13 @@
+module Dapp
+  class Dapp
+    module Command
+      module Sample
+        module List
+          def sample_list
+            _sample_list.each(&method(:puts))
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dapp/dimg/builder/ansible.rb

@@ -137,7 +137,11 @@ module Dapp
         }
 
         define_method("#{stage}_checksum") do
-          dimg.hashsum [JSON.dump(stage_config(stage)['tasks'])]
+          dimg.hashsum [
+            JSON.dump(stage_config(stage)['tasks']),
+            dimg.config._ansible["#{stage}_version"],
+            dimg.config._ansible["version"]
+          ]
         end
 
         define_method(stage.to_s) do |image|

data/lib/dapp/dimg/builder/shell.rb

@@ -4,7 +4,8 @@ module Dapp
       [:before_install, :before_setup, :install, :setup, :build_artifact].each do |stage|
         define_method("#{stage}_checksum") do
           [dimg.config._shell.public_send("_#{stage}_command"),
-           dimg.config._shell.public_send("_#{stage}_version")].flatten
+           dimg.config._shell.public_send("_#{stage}_version"),
+           dimg.config._shell._version].flatten
         end
         define_method("#{stage}?") { !stage_empty?(stage) }
         define_method(stage.to_s) do |image|

data/lib/dapp/dimg/cli/command/dimg/cleanup_repo.rb

@@ -15,6 +15,11 @@ BANNER
                description: 'Redefine resource locking timeout (in seconds)',
                proc: ->(v) { v.to_i }
 
+        option :kubernetes_timeout,
+               long: '--kubernetes-timeout TIMEOUT',
+               description: 'Kubernetes api-server tcp connection, read and write timeout (in seconds)',
+               proc: ->(v) { v.to_i }
+
         option :with_stages,
                long: '--with-stages',
                boolean: true

data/lib/dapp/dimg/dapp/command/cleanup_repo.rb

@@ -137,7 +137,12 @@ module Dapp
 
             config.context_names.
               map {|context_name|
-                client = ::Dapp::Kube::Kubernetes::Client.new(config, context_name, "default")
+                client = ::Dapp::Kube::Kubernetes::Client.new(
+                  config,
+                  context_name,
+                  "default",
+                  timeout: options[:kubernetes_timeout],
+                )
                 search_deployed_docker_images_from_kube(client)
               }.flatten.sort.uniq
           end
@@ -158,11 +163,12 @@ module Dapp
               [].tap do |arr|
                 client.with_namespace(ns) do
                   arr << pod_images(client)
-                  arr << cronjob_images(client)
-                  arr << daemonset_images(client)
                   arr << deployment_images(client)
-                  arr << job_images(client)
                   arr << replicaset_images(client)
+                  arr << statefulset_images(client)
+                  arr << daemonset_images(client)
+                  arr << job_images(client)
+                  arr << cronjob_images(client)
                   arr << replicationcontroller_images(client)
                 end
               end
@@ -213,6 +219,13 @@ module Dapp
             end
           end
 
+           # replicasets  items[] spec template spec containers[] image
+           def statefulset_images(client)
+            client.statefulset_list['items'].map do |item|
+              item['spec']['template']['spec']['containers'].map{ |cont| cont['image'] }
+            end
+          end
+
           # replicationcontroller    items[] spec template spec containers[] image
           def replicationcontroller_images(client)
             client.replicationcontroller_list['items'].map do |item|

data/lib/dapp/dimg/git_artifact.rb

@@ -488,7 +488,7 @@ module Dapp
       def repo_entries(commit, paths: include_paths_or_cwd)
         (@repo_entries ||= {})[[commit, paths]] ||= begin
           repo
-            .entries(commit, paths: paths, exclude_paths: exclude_paths(true))
+            .blobs_entries(commit, paths: paths, exclude_paths: exclude_paths(true))
             .select { |_, entry| !submodule_mode?(entry[:filemode]) }
         end
       end

data/lib/dapp/dimg/git_repo/base.rb

@@ -110,11 +110,11 @@ module Dapp
           ignore_path?(patch.delta.new_file[:path], paths: paths, exclude_paths: exclude_paths)
         end
 
-        def entries(commit, paths: [], exclude_paths: [])
+        def blobs_entries(commit, paths: [], exclude_paths: [])
           [].tap do |entries|
-            lookup_commit(commit).tree.walk(:preorder) do |root, entry|
+            lookup_commit(commit).tree.walk_blobs(:preorder) do |root, entry|
               fullpath = File.join(root, entry[:name]).reverse.chomp('/').reverse
-              next if entry[:type] == :tree || ignore_path?(fullpath, paths: paths, exclude_paths: exclude_paths)
+              next if ignore_path?(fullpath, paths: paths, exclude_paths: exclude_paths)
               entries << [root, entry]
             end
           end

data/lib/dapp/dimg/git_repo/remote.rb

@@ -80,7 +80,7 @@ module Dapp
             dapp.log_secondary_process(dapp.t(code: 'process.git_artifact_fetch', data: { url: url }), short: true) do
               begin
                 git.remotes.each { |remote| remote.fetch(credentials: _rugged_credentials) }
-              rescue Rugged::SshError => e
+              rescue Rugged::SshError, Rugged::NetworkError => e
                 raise Error::Rugged, code: :rugged_remote_error, data: { url: url, message: e.message.strip }
               end
             end

data/lib/dapp/kube/cli/command/kube/deploy.rb

@@ -50,6 +50,11 @@ BANNER
              description: 'Default timeout to wait for resources to become ready, 300 seconds by default.',
              proc: proc {|v| Integer(v)}
 
+      option :kubernetes_timeout,
+             long: '--kubernetes-timeout TIMEOUT',
+             description: 'Kubernetes api-server tcp connection, read and write timeout (in seconds)',
+             proc: ->(v) { v.to_i }
+
       option :registry_username,
              long: '--registry-username USERNAME'
 

data/lib/dapp/kube/cli/command/kube/dismiss.rb

@@ -20,6 +20,12 @@ BANNER
       option :with_namespace,
              long: '--with-namespace',
              default: false
+
+      option :kubernetes_timeout,
+             long: '--kubernetes-timeout TIMEOUT',
+             description: 'Kubernetes api-server tcp connection, read and write timeout (in seconds)',
+             proc: ->(v) { v.to_i }
+
     end
   end
 end

data/lib/dapp/kube/cli/command/kube/minikube_setup.rb

@@ -8,6 +8,12 @@ Usage:
 
 Options:
 BANNER
+
+      option :kubernetes_timeout,
+        long: '--kubernetes-timeout TIMEOUT',
+        description: 'Kubernetes api-server tcp connection, read and write timeout (in seconds)',
+        proc: ->(v) { v.to_i }
+
     end
   end
 end

data/lib/dapp/kube/dapp/command/common.rb

@@ -366,6 +366,7 @@ image: {{ tuple $name $context | include "_dimg2" }}
               kubernetes_config,
               kube_context,
               kube_namespace,
+              timeout: options[:kubernetes_timeout],
             )
           end
         end

data/lib/dapp/kube/dapp/command/deploy.rb

@@ -217,7 +217,11 @@ module Dapp
               unless dry_run?
                 begin
                   ::Timeout::timeout(self.options[:timeout] || 300) do
-                    deployment_managers.each {|deployment_manager| deployment_manager.watch_till_ready!}
+                    deployment_managers.each {|deployment_manager|
+                      if deployment_manager.should_watch?
+                        deployment_manager.watch_till_ready!
+                      end
+                    }
                   end
                 rescue ::Timeout::Error
                   raise ::Dapp::Error::Command, code: :kube_deploy_timeout

data/lib/dapp/kube/dapp/command/minikube_setup.rb

@@ -254,7 +254,8 @@ module Dapp
             @_minikube_kubernetes ||= Kubernetes::Client.new(
               kubernetes_config,
               kubernetes_config.current_context_name,
-              'kube-system'
+              'kube-system',
+              timeout: options[:kubernetes_timeout],
             )
           end
 

data/lib/dapp/kube/kubernetes/client.rb

@@ -50,11 +50,13 @@ module Dapp
         attr_reader :config
         attr_reader :context
         attr_reader :namespace
+        attr_reader :timeout
 
-        def initialize(config, context, namespace)
+        def initialize(config, context, namespace, timeout: nil)
           @config = config
           @context = context
           @namespace = namespace
+          @timeout = timeout
           @query_parameters = {}
           @cluster_version
         end
@@ -203,6 +205,16 @@ module Dapp
         # excon_parameters — соответствует connection-опциям Excon
         # body — hash для http-body, соответствует 'Body Parameters' в документации kubernetes, опционален
         def request!(method, path, body: nil, excon_parameters: {}, response_body_parameters: {}, **query_parameters)
+          if method == :get
+            excon_parameters[:idempotent] = true
+            excon_parameters[:retry_limit] = 6
+            excon_parameters[:retry_interval] = 5
+          end
+
+          excon_parameters[:connect_timeout] ||= timeout
+          excon_parameters[:read_timeout] ||= timeout
+          excon_parameters[:write_timeout] ||= timeout
+
           with_connection(excon_parameters: excon_parameters) do |conn|
             request_parameters = {method: method, path: path, query: @query_parameters.merge(query_parameters)}
             request_parameters[:body] = JSON.dump(body) if body

data/lib/dapp/kube/kubernetes/manager/deployment.rb

@@ -30,6 +30,14 @@ module Dapp
           @deployed_at = Time.now
         end
 
+        def should_watch?
+          d = Kubernetes::Client::Resource::Deployment.new(dapp.kubernetes.deployment(name))
+
+          ["dapp/watch", "dapp/watch-logs"].all? do |anno|
+            d.annotations[anno] != "false"
+          end
+        end
+
         def watch_till_ready!
           dapp.log_process("Watch deployment '#{name}' till ready") do
             known_events_by_pod = {}

data/lib/dapp/version.rb

@@ -1,4 +1,4 @@
 module Dapp
-  VERSION = "0.30.2"
+  VERSION = "0.30.3"
   BUILD_CACHE_VERSION = 30
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dapp
 version: !ruby/object:Gem::Version
-  version: 0.30.2
+  version: 0.30.3
 platform: ruby
 authors:
 - Dmitry Stolyarov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-06-27 00:00:00.000000000 Z
+date: 2018-06-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mixlib-shellout
@@ -467,6 +467,10 @@ files:
 - lib/dapp/cli.rb
 - lib/dapp/cli/cli.rb
 - lib/dapp/cli/command/base.rb
+- lib/dapp/cli/command/sample.rb
+- lib/dapp/cli/command/sample/base.rb
+- lib/dapp/cli/command/sample/create.rb
+- lib/dapp/cli/command/sample/list.rb
 - lib/dapp/cli/command/slug.rb
 - lib/dapp/cli/command/update.rb
 - lib/dapp/cli/options/ssh.rb
@@ -478,6 +482,9 @@ files:
 - lib/dapp/dapp.rb
 - lib/dapp/dapp/chef.rb
 - lib/dapp/dapp/command/common.rb
+- lib/dapp/dapp/command/sample/common.rb
+- lib/dapp/dapp/command/sample/create.rb
+- lib/dapp/dapp/command/sample/list.rb
 - lib/dapp/dapp/command/slug.rb
 - lib/dapp/dapp/dapp_config.rb
 - lib/dapp/dapp/dappfile.rb