kontena-cli 1.4.3 → 1.5.0.pre1

data/lib/kontena/cli/stacks/yaml/reader.rb

@@ -213,7 +213,11 @@ module Kontena::Cli::Stacks
           result['dependencies']  = dependencies
           result['source']        = raw_content
           result['variables']     = variable_values(without_defaults: true, without_vault: true)
-          result['parent_name']   = parent_name
+        end
+        if parent_name
+          result['parent'] = { 'name' => parent_name }
+        else
+          result['parent'] = nil
         end
         if service_name.nil?
           result['services'].each do |service|
@@ -349,8 +353,8 @@ module Kontena::Cli::Stacks
         @services ||= fully_interpolated_yaml.fetch('services', {})
       end
 
-      def from_external_file(filename, service_name)
-        external_reader = FileLoader.new(filename, loader).reader
+      def from_external_stack(name, service_name)
+        external_reader = StackFileLoader.for(name, loader).reader
         variables.to_a(with_value: true).each do |var|
           external_reader.variables.build_option(var)
         end
@@ -445,7 +449,8 @@ module Kontena::Cli::Stacks
           parent_config = process_config(services[extends])
         when Hash
           target = extends['file'] || extends['stack']
-          parent_config = from_external_file(target, extends['service'])
+          raise ("Service '#{extends}' does not define file: or stack: source") if target.nil?
+          parent_config = from_external_stack(target, extends['service'])
         else
           raise TypeError, "Extends must be a hash or string"
         end

data/lib/kontena/cli/vault/remove_command.rb

@@ -3,18 +3,20 @@ module Kontena::Cli::Vault
     include Kontena::Cli::Common
     include Kontena::Cli::GridOptions
 
-    parameter "NAME", "Secret name"
+    parameter "NAME ...", "Secret name", attribute_name: :names
     option "--force", :flag, "Force remove", default: false, attribute_name: :forced
     option "--silent", :flag, "Reduce output verbosity"
 
     def execute
       require_api_url
       require_current_grid
-      confirm_command(name) unless forced?
+      names.each do |name|
+        confirm_command(name) unless forced?
 
-      token = require_token
-      vspinner "Removing #{name.colorize(:cyan)} from the vault " do
-        client(token).delete("secrets/#{current_grid}/#{name}")
+        token = require_token
+        vspinner "Removing #{pastel.cyan(name)} from the vault " do
+          client(token).delete("secrets/#{current_grid}/#{name}")
+        end
       end
     end
   end

data/lib/kontena/cli/vault/update_command.rb

@@ -16,7 +16,7 @@ module Kontena::Cli::Vault
     end
 
     def execute
-      vspinner "Updating #{name.colorize(:cyan)} value in the vault " do
+      vspinner "Updating #{pastel.cyan(name)} value in the vault " do
         client.put("secrets/#{current_grid}/#{name}", {name: name, value: value, upsert: upsert? })
       end
     end

data/lib/kontena/cli/vault/write_command.rb

@@ -15,7 +15,7 @@ module Kontena::Cli::Vault
     end
 
     def execute
-      vspinner "Writing #{name.colorize(:cyan)} to the vault " do
+      vspinner "Writing #{pastel.cyan(name)} to the vault " do
         client.post("grids/#{current_grid}/secrets", { name: name, value: value })
       end
     end

data/lib/kontena/cli/volumes/remove_command.rb

@@ -6,17 +6,19 @@ module Kontena::Cli::Volumes
 
 
     banner "Removes a volume"
-    parameter 'VOLUME', 'Volume'
+    parameter 'VOLUME ...', 'Volume name', attribute_name: :volumes
     option "--force", :flag, "Force remove", default: false, attribute_name: :forced
 
     requires_current_master
     requires_current_master_token
 
     def execute
-      confirm_command(volume) unless forced?
+      volumes.each do |volume|
+        confirm_command(volume) unless forced?
 
-      spinner "Removing volume #{pastel.cyan(volume)} " do
-        remove_volume(volume)
+        spinner "Removing volume #{pastel.cyan(volume)} " do
+          remove_volume(volume)
+        end
       end
     end
 

data/lib/kontena/cli/vpn/create_command.rb

@@ -53,7 +53,7 @@ module Kontena::Cli::Vpn
       spinner "Generating #{pastel.cyan(name)} keys (this will take a while) " do
         wait_for_configuration_to_finish(token)
       end
-      puts "#{name.colorize(:cyan)} service is now started (udp://#{vpn_ip}:1194)."
+      puts "#{pastel.cyan(name)} service is now started (udp://#{vpn_ip}:1194)."
       puts "use 'kontena vpn config' to fetch OpenVPN client config to your machine."
     end
 

data/lib/kontena/cli/vpn/remove_command.rb

@@ -14,7 +14,7 @@ module Kontena::Cli::Vpn
       vpn = client(token).get("stacks/#{current_grid}/#{name}") rescue nil
       exit_with_error("VPN stack does not exist") if vpn.nil?
 
-      spinner "Removing #{name.colorize(:cyan)} service " do
+      spinner "Removing #{pastel.cyan(name)} service " do
         client(token).delete("stacks/#{current_grid}/#{name}")
       end
     end

data/lib/kontena/client.rb

@@ -11,6 +11,8 @@ module Kontena
     X_KONTENA_VERSION  = 'X-Kontena-Version'.freeze
     ACCEPT             = 'Accept'.freeze
     AUTHORIZATION      = 'Authorization'.freeze
+    ACCEPT_ENCODING    = 'Accept-Encoding'.freeze
+    GZIP               = 'gzip'.freeze
 
     attr_accessor :default_headers
     attr_accessor :path_prefix
@@ -53,7 +55,8 @@ module Kontena
         connect_timeout: ENV["EXCON_CONNECT_TIMEOUT"] ? ENV["EXCON_CONNECT_TIMEOUT"].to_i : 10,
         read_timeout:    ENV["EXCON_READ_TIMEOUT"]    ? ENV["EXCON_READ_TIMEOUT"].to_i    : 30,
         write_timeout:   ENV["EXCON_WRITE_TIMEOUT"]   ? ENV["EXCON_WRITE_TIMEOUT"].to_i   : 10,
-        ssl_verify_peer: ignore_ssl_errors? ? false : true
+        ssl_verify_peer: ignore_ssl_errors? ? false : true,
+        middlewares:     Excon.defaults[:middlewares] + [Excon::Middleware::Decompress]
       }
       if Kontena.debug?
         require 'kontena/debug_instrumentor'
@@ -248,7 +251,7 @@ module Kontena
     # @param [Hash,NilClass] params
     # @param [Hash] headers
     def get_stream(path, response_block, params = nil, headers = {}, auth = true)
-      request(path: path, query: params, headers: headers, response_block: response_block, auth: auth)
+      request(path: path, query: params, headers: headers, response_block: response_block, auth: auth, gzip: false)
     end
 
     def token_expired?
@@ -279,15 +282,15 @@ module Kontena
     # @param expects [Array] raises unless response status code matches this list.
     # @param auth [Boolean] use token authentication default = true
     # @return [Hash, String] response parsed response object
-    def request(http_method: :get, path:'/', body: nil, query: {}, headers: {}, response_block: nil, expects: [200, 201, 204], host: nil, port: nil, auth: true)
+    def request(http_method: :get, path:'/', body: nil, query: {}, headers: {}, response_block: nil, expects: [200, 201, 204], host: nil, port: nil, auth: true, gzip: true)
 
       retried ||= false
 
       if auth && token_expired?
-        raise Excon::Errors::Unauthorized, "Token expired or not valid, you need to login again, use: kontena #{token_is_for_master? ? "master" : "cloud"} login"
+        raise Excon::Error::Unauthorized, "Token expired or not valid, you need to login again, use: kontena #{token_is_for_master? ? "master" : "cloud"} login"
       end
 
-      request_headers = request_headers(headers, auth)
+      request_headers = request_headers(headers, auth: auth, gzip: gzip)
 
       if body.nil?
         body_content = ''
@@ -325,7 +328,7 @@ module Kontena
       @last_response = http_client.request(request_options)
 
       parse_response(@last_response)
-    rescue Excon::Errors::Unauthorized
+    rescue Excon::Error::Unauthorized
       if token
         debug { 'Server reports access token expired' }
 
@@ -337,7 +340,11 @@ module Kontena
         retry if refresh_token
       end
       raise Kontena::Errors::StandardError.new(401, 'Unauthorized')
-    rescue Excon::Errors::HTTPStatusError => error
+    rescue Excon::Error::HTTPStatus => error
+      if error.response.headers['Content-Encoding'] == 'gzip'
+        error.response.body = Zlib::GzipReader.new(StringIO.new(error.response.body)).read
+      end
+
       debug { "Request #{error.request[:method].upcase} #{error.request[:path]}: #{error.response.status} #{error.response.reason_phrase}: #{error.response.body}" }
 
       handle_error_response(error.response)
@@ -447,9 +454,10 @@ module Kontena
     #
     # @param [Hash] headers
     # @return [Hash]
-    def request_headers(headers = {}, auth = true)
+    def request_headers(headers = {}, auth: true, gzip: true)
       headers = default_headers.merge(headers)
       headers.merge!(bearer_authorization_header) if auth
+      headers[ACCEPT_ENCODING] = GZIP if gzip
       headers.reject{|_,v| v.nil? || (v.respond_to?(:empty?) && v.empty?)}
     end
 
@@ -479,7 +487,7 @@ module Kontena
       check_version_and_warn(response.headers[X_KONTENA_VERSION])
 
       if response.headers[CONTENT_TYPE] =~ JSON_REGEX
-        parse_json(response.body)
+        parse_json(response)
       else
         response.body
       end
@@ -503,13 +511,14 @@ module Kontena
 
     # Parse json
     #
-    # @param [String] json
+    # @param response [Excon::Response]
     # @return [Hash,Object,NilClass]
-    def parse_json(json)
-      JSON.parse(json)
+    def parse_json(response)
+      return nil if response.body.empty?
+      
+      JSON.parse(response.body)
     rescue => ex
-      debug { "JSON parse exception: #{ex.class.name} : #{ex.message}" }
-      nil
+      raise Kontena::Errors::StandardError.new(520, "Invalid response JSON from server for #{response.path}: #{ex.class.name}: #{ex.message}")
     end
 
     # Dump json

data/lib/kontena/command.rb

@@ -3,7 +3,7 @@ require 'kontena/cli/subcommand_loader'
 require 'kontena/util'
 require 'kontena/cli/bytes_helper'
 require 'kontena/cli/grid_options'
-require 'excon/errors'
+require 'excon/error'
 
 class Kontena::Command < Clamp::Command
 
@@ -221,7 +221,7 @@ class Kontena::Command < Clamp::Command
     run_callbacks :after unless help_requested?
     exit(@exit_code) if @exit_code.to_i > 0
     @result
-  rescue Excon::Errors::SocketError => ex
+  rescue Excon::Error::Socket => ex
     if ex.message.include?('Unable to verify certificate')
       $stderr.puts " [#{Kontena.pastel.red('error')}] The server uses a certificate signed by an unknown authority."
       $stderr.puts "         You can trust this server by copying server CA pem file to: #{Kontena.pastel.yellow("~/.kontena/certs/<hostname>.pem")}"

data/lib/kontena/debug_instrumentor.rb

@@ -20,7 +20,9 @@ module Kontena
         str = "Headers: {"
         heads = []
         heads << "Accept: #{params[:headers]['Accept']}" if params[:headers]['Accept']
+        heads << "Accept-Encoding: #{params[:headers]['Accept-Encoding']}" if params[:headers]['Accept-Encoding']
         heads << "Content-Type: #{params[:headers]['Content-Type']}" if params[:headers]['Content-Type']
+        heads << "Content-Encoding: #{params[:headers]['Content-Encoding']}" if params[:headers]['Content-Encoding']
         heads << "Authorization: #{params[:headers]['Authorization'].split(' ', 2).first}" if params[:headers]['Authorization']
         heads << "X-Kontena-Version: #{params[:headers]['X-Kontena-Version']}" if params[:headers]['X-Kontena-Version']
         str << heads.join(', ')
@@ -39,12 +41,19 @@ module Kontena
       end
 
       if params[:body] && !params[:body].empty?
+        if params[:headers]['Content-Encoding'].to_s =~ /gzip/
+          body_content = Zlib::GzipReader.new(StringIO.new(params[:body])).read
+          body = "(GZIPPED 1:%d) %s" % [body_content.bytesize / params[:body].bytesize, body_content]
+        else
+          body = params[:body]
+        end
+
         str = "Body: "
         if ENV["DEBUG"] == "api"
           str << "\n"
-          str << params[:body]
+          str << body
         else
-          body = params[:body].inspect.strip
+          body = body.inspect.strip
           str << body[0,80]
           if body.length > 80
             str << "...\""

data/lib/kontena/plugin_manager/common.rb

@@ -6,6 +6,8 @@ module Kontena
       Gem.autoload :DefaultUserInteraction, 'rubygems/user_interaction'
       Gem.autoload :StreamUI, 'rubygems/user_interaction'
 
+      KONTENA_PLUGIN = 'kontena-plugin-%s'
+
       # @return [Boolean] is the CLI in plugin debugging mode?
       def plugin_debug?
         @plugin_debug ||= ENV['DEBUG'] == 'plugin'
@@ -26,8 +28,9 @@ module Kontena
 
       # Prefix a plugin name into a gem name (hello to kontena-plugin-hello)
       def prefix(plugin_name)
-        return plugin_name if plugin_name.to_s.start_with?('kontena-plugin-')
-        "kontena-plugin-#{plugin_name}"
+        return KONTENA_PLUGIN % nil if plugin_name.nil? || plugin_name.empty?
+        return plugin_name if plugin_name.start_with?('kontena-plugin-') || plugin_name.include?('.')
+        KONTENA_PLUGIN % plugin_name
       end
       module_function :prefix
 

data/lib/kontena/plugin_manager/installer.rb

@@ -7,7 +7,8 @@ module Kontena
     class Installer
       include Common
 
-      attr_reader :plugin_name, :pre, :version
+      attr_reader :plugin_name, :pre
+      attr_accessor :version
 
       # Create a new instance of plugin installer
       # @param plugin_name [String]
@@ -19,6 +20,10 @@ module Kontena
         @version = version
       end
 
+      def pre?
+        !!@pre
+      end
+
       def command
         @command ||= Gem::DependencyInstaller.new(
           document: false,
@@ -30,23 +35,42 @@ module Kontena
 
       # Install a plugin
       def install
+        return install_uri if plugin_name.include?('://')
         plugin_version = version.nil? ? Gem::Requirement.default : Gem::Requirement.new(version)
         command.install(prefix(plugin_name), plugin_version)
         command.installed_gems
       end
 
-      # Upgrade an installed plugin
-      # @param plugin_name [String]
-      # @param pre [Boolean] upgrade to a prerelease version if available. Will happen always when the installed version is a prerelease version.
-      def upgrade
-        return install if version
+      def install_uri
+        require 'tempfile'
+        require 'open-uri'
+        file = Tempfile.new(['kontena_plugin', '.gem'])
+        open(plugin_name) do |input|
+          file.write input.read
+          file.close
+        end
+        self.class.new(file.path).install
+      ensure
+        file.unlink
+      end
+
+      def available_upgrade
         installed = installed(plugin_name)
-        pre = installed.version.prerelease?
+        return false unless installed
 
-        raise "Plugin #{plugin_name} not installed" unless installed
-        latest = rubygems_client.latest_version(prefix(plugin_name), pre: pre)
+        pre = installed.version.prerelease?
+        latest = rubygems_client.latest_version(prefix(plugin_name), pre: pre? || pre)
         if latest > installed.version
-          Installer.new(plugin_name, version: latest.to_s).install
+          latest.to_s
+        end
+      end
+
+      # Upgrade an installed plugin
+      def upgrade
+        return install if version
+        upgrade_to = available_upgrade
+        if upgrade_to
+          Installer.new(plugin_name, version: upgrade_to).install
         end
       end
     end

data/lib/kontena/scripts/completer.rb

@@ -1,4 +1,5 @@
 require 'kontena/cli/common'
+require 'kontena/stacks_client'
 
 class Helper
   include Kontena::Cli::Common
@@ -27,12 +28,6 @@ class Helper
     client_config['current_server']
   end
 
-  def client
-    $VERSION_WARNING_ADDED=true
-    token = require_token
-    super(token)
-  end
-
   def grids
     client.get("grids")['grids'].map{|grid| grid['id']}
   rescue => ex
@@ -57,6 +52,37 @@ class Helper
     []
   end
 
+  def stack_registry_usable?
+    return false if current_account.nil? || current_account.stacks_url.nil?
+    return false if current_account.stacks_read_authentication && current_account.token.nil? || current_account.token.access_token.nil?
+    true
+  end
+
+  def stacks_client
+    Kontena::StacksClient.new(current_account.stacks_url, current_account.token, read_requires_token: current_account.stacks_read_authentication)
+  end
+
+  def registry_stacks(query = '')
+    return [] unless stack_registry_usable?
+    results = stacks_client.search(query).map { |s| s['stack'] }
+    if results.empty? && !query.empty? # this is here because old stack registry does not return anything for "org/"
+      results = stacks_client.search('').map { |s| s['stack'] }.select { |s| s.start_with?(query) }
+    end
+    results
+  rescue => ex
+    logger.debug ex
+    []
+  end
+
+  def registry_stack_versions(stackname)
+    return [] unless stack_registry_usable?
+    logger.debug stackname.inspect
+    stacks_client.versions(stackname).map { |v| [stackname, v['version']].join(':') }
+  rescue => ex
+    logger.debug ex
+    []
+  end
+
   def services
     services = client.get("grids/#{current_grid}/services")['services']
     results = []
@@ -99,8 +125,21 @@ class Helper
     []
   end
 
-  def yml_files
-    Dir["./*.yml"].map{|file| file.sub('./', '')}
+  def directories(word)
+    if word && File.directory?(word) && !word.end_with?('/')
+      ['%s/' % word]
+    else
+      Dir[File.join('.', '%s*' % word)].select { |file| File.directory?(file) }.map { |file| '%s/' % file.sub('./', '') }
+    end
+  end
+
+  def yml_files(word)
+    if word && File.directory?(word) && word.end_with?('/')
+      glob = File.join(word, '*.{yml,yaml}')
+    else
+      glob = File.join('.', '%s*.{yml,yaml}' % word)
+    end
+    Dir[glob].map { |file| file.sub('./', '') } + directories(word)
   rescue => ex
     logger.debug ex
     []
@@ -139,30 +178,30 @@ helper.logger.debug { "Completing #{words.inspect}" }
 
 begin
   completion = []
-  completion.push %w(cloud grid app service stack vault certificate node master vpn registry container etcd external-registry whoami plugin version) if words.size < 2
+  completion.push %w(cloud grid service stack vault certificate node master vpn registry container etcd external-registry whoami plugin version) if words.size < 2
   if words.size > 0
     case words[0]
       when 'plugin'
         completion.clear
-        sub_commands = %w(list ls search install uninstall)
+        sub_commands = %w(list search install uninstall)
         if words[1]
-          completion.push(sub_commands) unless sub_commands.include?(words[1])
+          completion.push(sub_commands) unless (sub_commands + %w(ls)).include?(words[1])
         else
           completion.push sub_commands
         end
       when 'etcd'
         completion.clear
-        sub_commands = %w(get set mkdir mk list ls rm)
+        sub_commands = %w(get set mkdir list rm)
         if words[1]
-          completion.push(sub_commands) unless sub_commands.include?(words[1])
+          completion.push(sub_commands) unless (sub_commands + %w(ls)).include?(words[1])
         else
           completion.push sub_commands
         end
       when 'registry'
         completion.clear
-        sub_commands = %w(create remove rm)
+        sub_commands = %w(create remove)
         if words[1]
-          completion.push(sub_commands) unless sub_commands.include?(words[1])
+          completion.push(sub_commands) unless (sub_commands + %w(rm)).include?(words[1])
         else
           completion.push sub_commands
         end
@@ -171,7 +210,7 @@ begin
         sub_commands = %w(add-user audit-log create current list user remove show use)
         if words[1] && words[1] == 'use'
           completion.push helper.grids.reject { |g| g == helper.current_grid }
-        elsif words[1] && %w(update show rm remove env cloud-config health).include?(words[1])
+        elsif words[1] && %w(update show remove env cloud-config health).include?(words[1])
           completion.push helper.grids
         else
           completion.push sub_commands
@@ -186,7 +225,7 @@ begin
         end
       when 'master'
         completion.clear
-        sub_commands = %w(list use user current remove rm config cfg login logout token join audit-log init-cloud)
+        sub_commands = %w(list use user current remove config login logout token join audit-log init-cloud)
         if words[1] && words[1] == 'use'
           completion.push helper.master_names.reject { |n| n == helper.current_master_name }
         elsif words[1] && %w(remove rm).include?(words[1])
@@ -194,8 +233,8 @@ begin
         elsif words[1] && words[1] == 'user'
           users_sub_commands = %w(invite list role)
           if words[2] == 'role'
-            role_subcommands = %w(add remove rm)
-            if !words[3] || !role_subcommands.include?(words[3])
+            role_subcommands = %w(add remove)
+            if !words[3] || !(role_subcommands + %w(rm)).include?(words[3])
               completion.push role_subcommands
             end
           else
@@ -205,10 +244,10 @@ begin
           config_sub_commands = %(set get dump load import export unset)
           completion.push config_sub_commands
         elsif words[1] && words[1] == 'token'
-          token_sub_commands = %(list ls rm remove show current create)
+          token_sub_commands = %(list remove show current create)
           completion.push token_sub_commands
         elsif words[1]
-          completion.push(sub_commands) unless sub_commands.include?(words[1])
+          completion.push(sub_commands) unless (sub_commands + %w(ls rm)).include?(words[1])
         else
           completion.push sub_commands
         end
@@ -216,10 +255,10 @@ begin
         completion.clear
         sub_commands = %w(login logout master)
         if words[1] && words[1] == 'master'
-          cloud_master_sub_commands = %(list ls remove rm add show update)
+          cloud_master_sub_commands = %(list remove add show update)
           completion.push cloud_master_sub_commands
         elsif words[1]
-          completion.push(sub_commands) unless sub_commands.include?(words[1])
+          completion.push(sub_commands) unless (sub_commands + %w(ls rm)).include?(words[1])
         else
           completion.push sub_commands
         end
@@ -247,31 +286,40 @@ begin
       when 'external-registry'
         completion.clear
         completion.push %w(add list delete)
-      when 'app'
-        completion.clear
-        sub_commands = %w(init build config deploy start stop remove rm ps list
-                          logs monitor show)
-        if words[1] && sub_commands.include?(words[1])
-          completion.push helper.yml_services
-        else
-          completion.push sub_commands
-        end
       when 'stack'
         completion.clear
-        sub_commands = %w(build install upgrade deploy start stop remove rm ls list
-                          logs monitor show registry)
+        sub_commands = %w(build install upgrade deploy start stop remove restart list
+                          logs monitor show registry inspect)
         if words[1]
-          if words[1] == 'registry'
-            registry_sub_commands = %(push pull search show rm)
-            completion.push registry_sub_commands
-          elsif %w(install).include?(words[1])
-              completion.push helper.yml_files
-          elsif words[1] == 'upgrade' && words[3]
-            completion.push helper.yml_files
-          elsif words[1] && sub_commands.include?(words[1])
+          if words[1] == 'registry' || words[1] == 'reg'
+            registry_sub_commands = %(push pull search show remove)
+            if words[2]
+              if words[2] == 'push'
+                completion.push helper.yml_files(words[3])
+              elsif %w(pull search show remove rm).include?(words[2]) && words[4].nil?
+                completion.push helper.registry_stacks(words[3].to_s)
+              else
+                completion.push registry_sub_commands
+              end
+            else
+              completion.push registry_sub_commands
+            end
+          elsif %w(install validate build).include?(words[1])
+            completion.push helper.yml_files(words[2])
+            if words[1] == 'install'
+              completion.push helper.registry_stacks(words[2].to_s)
+            end
+          elsif words[1] == 'upgrade'
+            if words[3]
+              completion.push helper.yml_files(words[4])
+              completion.push helper.registry_stacks(words[4].to_s)
+            else
+              completion.push helper.stacks
+            end
+          elsif %w(deploy start stop remove rm restart logs monitor show inspect).include?(words[1])
             completion.push helper.stacks
           else
-            completion.push(sub_commands)
+            completion.push(sub_commands) unless (sub_commands + %w(rm ls)).include?(words[1])
           end
         else
           completion.push sub_commands