kontena-cli 1.4.0.pre6 → 1.4.0.pre7

data/lib/kontena/cli/stacks/yaml/service_extender.rb

@@ -1,5 +1,3 @@
-require_relative '../../../util'
-
 module Kontena::Cli::Stacks
   module YAML
     class ServiceExtender
@@ -14,7 +12,7 @@ module Kontena::Cli::Stacks
       # @param [Hash] from
       # @return [Hash]
       def extend_from(from)
-        service_config['environment'] = extend_env_vars(from['environment'], service_config['environment'])
+        service_config['environment'] = extend_env_vars(from['env'], service_config['environment'])
         service_config['secrets']     = extend_secrets( from['secrets'], service_config['secrets'])
         build_args                    = extend_build_args(safe_dig(from, 'build', 'args'), safe_dig(service_config, 'build', 'args'))
         unless build_args.empty?
@@ -26,6 +24,10 @@ module Kontena::Cli::Stacks
 
       private
 
+      def env_to_hash(env_array)
+        env_array.map { |env| env.split('=', 2) }.to_h
+      end
+
       # Takes two arrays of "key=value" pairs and merges them. Keys in "from"-array
       # will not overwrite keys that already exist in "to"-array.
       #
@@ -33,15 +35,7 @@ module Kontena::Cli::Stacks
       # @param [Array] to
       # @return [Array]
       def extend_env_vars(from, to)
-        env_vars = to || []
-        if from
-          from.each do |env|
-            env_vars << env unless to && to.find do |key|
-              key.split('=').first == env.split('=').first
-            end
-          end
-        end
-        env_vars
+        env_to_hash(from || []).merge(env_to_hash(to || [])).map { |k,v| [k.to_s, v.to_s].join('=') }
       end
 
       # Takes two arrays of hashes containing { 'secret' => 'str', 'type' => 'str', 'name' => 'str' }

data/lib/kontena/cli/stacks/yaml/stack_file_loader.rb

@@ -0,0 +1,97 @@
+require_relative '../stack_name'
+require 'yaml'
+require_relative 'reader'
+
+module Kontena::Cli::Stacks
+  module YAML
+    class StackFileLoader
+      # A base class for loading stack files. You can define more loaders by
+      # inheriting from this class.
+      #
+      # The purpose of StackFileLoader is to provide a generic interface for
+      # loading stack YAML's from different sources, such as local files,
+      # stack registry or URLs
+
+      def self.inherited(where)
+        loaders << where
+      end
+
+      def self.loaders
+        @loaders ||= []
+      end
+
+      # The main interface for getting a new loader
+      #
+      # @param source [String] stack file source string (filename, url, ..)
+      # @param parent [StackFileLoader] define a parent for recursion
+      # @return [StackFileLoader]
+      def self.for(source, parent = nil)
+        loader = loaders.find { |l| l.match?(source, parent) }
+        if loader.nil?
+          raise RuntimeError, "Not found: no such file #{source} or invalid uri scheme"
+        end
+        loader.new(source, parent)
+      end
+
+      attr_reader :source, :parent
+
+      # @param source [String] stack file source string (filename, url, ..)
+      # @param parent [StackFileLoader] define a parent for recursion
+      # @return [StackFileLoader]
+      def initialize(source, parent = nil)
+        @source = source
+        @parent = parent
+      end
+
+      # @return [Hash] a hash parsed from the YAML content
+      def yaml
+        @yaml ||= ::YAML.safe_load(content, [], [], true, source)
+      end
+
+      # @return [String] raw file content
+      def content
+        @content ||= read_content
+      end
+
+      def read_content
+        raise "Implement in inheriting class"
+      end
+
+      # @return [StackName] an accessor to StackName for the target file
+      def stack_name
+        @stack_name ||= Kontena::Cli::Stacks::StackName.new(yaml['stack'], yaml['version'])
+      end
+
+      # @return [Reader] an accessor to YAML::Reader for the target file
+      def reader(*args)
+        @reader ||= Reader.new(self, *args)
+      end
+
+      # Builds an array of hashes that represent the dependency tree starting
+      # from the target file. Unless recurse is set to false, the tree will
+      # contain also nested dependencies from any child stacks.
+      #
+      # @param recurse [TrueClass,FalseClass] recurse child dependencies?
+      # @return [Array<Hash>] an array of hashes ('name', 'stack', 'variables', and 'depends')
+      def dependencies(recurse: true)
+        return @dependencies if @dependencies
+        depends = yaml['depends']
+        if depends.nil? || depends.empty?
+          @dependencies = nil
+        else
+          @dependencies = depends.map do |name, dependency|
+            reader = StackFileLoader.for(dependency['stack'], self)
+            deps = { 'name' => name, 'stack' => reader.source, 'variables' => dependency.fetch('variables', Hash.new) }
+            if recurse
+              child_deps = reader.dependencies
+              deps['depends'] = child_deps unless child_deps.nil?
+            end
+            deps
+          end
+        end
+      end
+    end
+  end
+end
+
+Dir[File.expand_path('../stack_file_loader/*.rb', __FILE__)].each { |f| require f }

data/lib/kontena/cli/stacks/yaml/stack_file_loader/file_loader.rb

@@ -0,0 +1,41 @@
+require 'pathname'
+
+module Kontena::Cli::Stacks
+  module YAML
+    class FileLoader < StackFileLoader
+      def self.match?(source, parent = nil)
+        ::File.exist?(with_context(source, parent))
+      end
+
+      def self.is_file?(parent)
+        parent.is_a?(FileLoader)
+      end
+
+      def self.with_context(source, parent = nil)
+        if is_file?(parent)
+          File.join(File.dirname(parent.source),  source)
+        else
+          File.absolute_path(source)
+        end
+      end
+
+      def initialize(*args)
+        super
+        @source = self.class.with_context(@source, parent)
+      end
+
+      def read_content
+        ::File.read(source)
+      end
+
+      def origin
+        "file"
+      end
+
+      def registry
+        "file://"
+      end
+
+    end
+  end
+end

data/lib/kontena/cli/stacks/yaml/stack_file_loader/registry_loader.rb

@@ -0,0 +1,24 @@
+module Kontena::Cli::Stacks
+  module YAML
+    class RegistryLoader < StackFileLoader
+      def self.match?(source, parent = nil)
+        source =~ /\A[a-zA-Z0-9\_\.\-]+\/[a-zA-Z0-9\_\.\-]+(?::.*)?\z/ && !FileLoader.match?(source, parent)
+      end
+
+      def read_content
+        Kontena::StacksCache.pull(source)
+      end
+
+      def origin
+        "registry"
+      end
+
+      def registry
+        account = Kontena::Cli::Config.current_account
+        raise "Current account not set" if account.nil?
+        account.stacks_url
+      end
+    end
+  end
+end
+

data/lib/kontena/cli/stacks/yaml/stack_file_loader/uri_loader.rb

@@ -0,0 +1,23 @@
+module Kontena::Cli::Stacks
+  module YAML
+    class UriLoader < StackFileLoader
+      def self.match?(source, parent = nil)
+        source.include?('://') && !FileLoader.match?(source, parent)
+      end
+
+      def read_content
+        require 'open-uri'
+        stream = open(source)
+        stream.read
+      end
+
+      def origin
+        "uri"
+      end
+
+      def registry
+        "file://"
+      end
+    end
+  end
+end

data/lib/kontena/cli/stacks/yaml/validations.rb

@@ -6,6 +6,7 @@ module Kontena::Cli::Stacks::YAML
      require_relative 'custom_validators/extends_validator'
      require_relative 'custom_validators/hooks_validator'
      require_relative 'custom_validators/secrets_validator'
+     require_relative 'custom_validators/certificates_validator'
 
      def self.load
        return if @loaded
@@ -13,6 +14,7 @@ module Kontena::Cli::Stacks::YAML
        HashValidator.append_validator(BuildValidator.new)
        HashValidator.append_validator(ExtendsValidator.new)
        HashValidator.append_validator(SecretsValidator.new)
+       HashValidator.append_validator(CertificatesValidator.new)
        HashValidator.append_validator(HooksValidator.new)
        @loaded = true
      end
@@ -27,10 +29,12 @@ module Kontena::Cli::Stacks::YAML
         'cap_add' => optional('array'),
         'cap_drop' => optional('array'),
         'command' => optional('string'),
+        'cpus' => optional('float'),
         'cpu_shares' => optional('integer'),
         'external_links' => optional('array'),
         'mem_limit' => optional('string'),
         'mem_swaplimit' => optional('string'),
+        'shm_size' => optional('string'),
         'environment' => optional(-> (value) {
           if value.is_a?(Hash)
             value.all? do |k,v|
@@ -58,6 +62,7 @@ module Kontena::Cli::Stacks::YAML
         'volumes' => optional('array'),
         'volumes_from' => optional('array'),
         'secrets' => optional('stacks_valid_secrets'),
+        'certificates' => optional('stacks_valid_certificates'),
         'hooks' => optional('stacks_valid_hooks'),
         'only_if' => optional(-> (value) { value.is_a?(String) || value.is_a?(Hash) || value.is_a?(Array) }),
         'skip_if' => optional(-> (value) { value.is_a?(String) || value.is_a?(Hash) || value.is_a?(Array) }),
@@ -92,10 +97,21 @@ module Kontena::Cli::Stacks::YAML
       HashValidator.validate(volume_config, volume_schema, true)
     end
 
+    def validate_dependencies(dependency_config)
+      HashValidator.validate(dependency_config, dependency_schema, true)
+    end
+
     def volume_schema
       {
         'external' => optional(-> (value) { value.is_a?(TrueClass) || (value.is_a?(Hash) && value['name'].is_a?(String)) })
       }
     end
+
+    def dependency_schema
+      {
+        'stack' => optional('string'),
+        'variables' => optional(-> (value) { value.is_a?(Hash) })
+      }
+    end
   end
 end

data/lib/kontena/cli/stacks/yaml/validator_v3.rb

@@ -6,7 +6,7 @@ module Kontena::Cli::Stacks
       require_relative 'validations'
       include Validations
 
-      KNOWN_TOP_LEVEL_KEYS = %w(
+      KNOWN_TOP_LEVEL_KEYS = %i(
         services
         errors
         volumes
@@ -17,6 +17,7 @@ module Kontena::Cli::Stacks
         data
         description
         expose
+        depends
       )
 
       def initialize
@@ -26,7 +27,7 @@ module Kontena::Cli::Stacks
         @schema['network_mode'] = optional(%w(host bridge))
         @schema['logging'] = optional({
           'driver' => optional('string'),
-          'options' => optional(-> (value) { value.is_a?(Hash) })
+          'options' => optional(-> (value) { value.kind_of?(Hash) })
           })
         Validations::CustomValidators.load
       end
@@ -59,14 +60,16 @@ module Kontena::Cli::Stacks
           notifications: []
         }
 
-        result[:notifications] += (yaml.keys - KNOWN_TOP_LEVEL_KEYS).map do |key|
-          { key => "unknown top level key" }
+        yaml.keys.each do |key|
+          unless KNOWN_TOP_LEVEL_KEYS.include?(key) || KNOWN_TOP_LEVEL_KEYS.include?(key.to_sym)
+            result[:notifications] << { key.to_s => "unknown top level key" }
+          end
         end
 
         if yaml.key?('services')
-          if yaml['services'].is_a?(Hash)
+          if yaml['services'].kind_of?(Hash)
             yaml['services'].each do |service, options|
-              unless options.is_a?(Hash)
+              unless options.kind_of?(Hash)
                 result[:errors] << { 'services' => { service => { 'options' => "must be a mapping not a #{options.class}"}  } }
                 next
               end
@@ -107,9 +110,9 @@ module Kontena::Cli::Stacks
         end
 
         if yaml.key?('volumes')
-          if yaml['volumes'].is_a?(Hash)
+          if yaml['volumes'].kind_of?(Hash)
             yaml['volumes'].each do |volume, options|
-              if options.is_a?(Hash)
+              if options.kind_of?(Hash)
                 option_errors = validate_volume_options(options)
                 unless option_errors.valid?
                   result[:errors] << { 'volumes' => { volume => option_errors.errors } }
@@ -130,6 +133,20 @@ module Kontena::Cli::Stacks
         if (yaml['volumes'].nil? || yaml['volumes'].empty?) && (yaml['services'].nil? || yaml['services'].empty?)
           result[:errors] << { 'file' => 'does not list any services or volumes' }
         end
+
+        if yaml.key?('depends')
+          unless yaml['depends'].kind_of?(Hash)
+            result[:errors] << { 'depends' => "Must be a mapping, not #{yaml['depends'].class}" }
+          end
+
+          yaml['depends'].each do |name, dependency_options|
+            validator = validate_dependencies(dependency_options)
+            result[:errors] << { 'depends' => { name => validator.errors } } unless validator.valid?
+            if yaml.key?('services') && yaml['services'][name]
+              result[:errors] << { 'depends' => { name => 'is defined both as service and dependency name' } }
+            end
+          end
+        end
         result
       end
     end

data/lib/kontena/client.rb

@@ -50,9 +50,9 @@ module Kontena
 
       excon_opts = {
         omit_default_port: true,
-        connect_timeout: ENV["EXCON_CONNECT_TIMEOUT"] ? ENV["EXCON_CONNECT_TIMEOUT"].to_i : 5,
+        connect_timeout: ENV["EXCON_CONNECT_TIMEOUT"] ? ENV["EXCON_CONNECT_TIMEOUT"].to_i : 10,
         read_timeout:    ENV["EXCON_READ_TIMEOUT"]    ? ENV["EXCON_READ_TIMEOUT"].to_i    : 30,
-        write_timeout:   ENV["EXCON_WRITE_TIMEOUT"]   ? ENV["EXCON_WRITE_TIMEOUT"].to_i   : 5,
+        write_timeout:   ENV["EXCON_WRITE_TIMEOUT"]   ? ENV["EXCON_WRITE_TIMEOUT"].to_i   : 10,
         ssl_verify_peer: ignore_ssl_errors? ? false : true
       }
       if ENV["DEBUG"]

data/lib/kontena/command.rb

@@ -183,6 +183,17 @@ class Kontena::Command < Clamp::Command
     false
   end
 
+  # Returns an instance of the command, just like with Kontena.run! but before calling "execute"
+  # You can use it for specs or reuse of instancemethods.
+  # Example:
+  #   cmd = Kontena::FooCommand.instance(['-n', 'foo'])
+  #   cmd.fetch_stuff
+  def instance(arguments)
+    @arguments = arguments
+    parse @arguments
+    self
+  end
+
   def run(arguments)
     Kontena.logger.debug { "Running #{self.class.name} with #{arguments.inspect} -- callback matcher = '#{self.class.callback_matcher.nil? ? "nil" : self.class.callback_matcher.map(&:to_s).join(' ')}'" }
     @arguments = arguments

data/lib/kontena/main_command.rb

@@ -29,7 +29,6 @@ class Kontena::MainCommand < Kontena::Command
   subcommand "registry", "Registry specific commands", load_subcommand('registry_command')
   subcommand "external-registry", "External registry specific commands", load_subcommand('external_registry_command')
   subcommand "volume", "Volume specific commands [EXPERIMENTAL]", load_subcommand('volume_command')
-  subcommand "app", "App specific commands", load_subcommand('app_command')
   subcommand "plugin", "Plugin specific commands", load_subcommand('plugin_command')
   subcommand "whoami", "Shows current logged in user", load_subcommand('whoami_command')
   subcommand "version", "Show CLI and current master version", load_subcommand('version_command')
@@ -54,6 +53,9 @@ class Kontena::MainCommand < Kontena::Command
     elsif name == 'logout'
       exit_with_error "Use 'kontena master logout' to log out from a Kontena Master\n"+
              "         or 'kontena cloud logout' for logging out from your Kontena Cloud account"
+    elsif name == 'app'
+      exit_with_error "The deprecated app subcommand has been moved into a plugin. You can install\n" +
+             "         it by using 'kontena plugin install app-command'"
     end
     super
   end

data/lib/kontena/plugin_manager.rb

@@ -1,213 +1,26 @@
-require 'singleton'
-
 module Kontena
-  class PluginManager
+  module PluginManager
     autoload :RubygemsClient, 'kontena/plugin_manager/rubygems_client'
-    Gem.autoload :DependencyInstaller, 'rubygems/dependency_installer'
-    Gem.autoload :Requirement, 'rubygems/requirement'
-    Gem.autoload :Uninstaller, 'rubygems/uninstaller'
-    Gem.autoload :Commands, 'rubygems/command'
-    Gem.autoload :DefaultUserInteraction, 'rubygems/user_interaction'
-    Gem.autoload :StreamUI, 'rubygems/user_interaction'
-    Gem::Commands.autoload :CleanupCommand, 'rubygems/commands/cleanup_command'
-
-    include Singleton
-
-    CLI_GEM = 'kontena-cli'.freeze
-    MIN_CLI_VERSION = '0.15.99'.freeze
+    autoload :Loader, 'kontena/plugin_manager/loader'
+    autoload :Installer, 'kontena/plugin_manager/installer'
+    autoload :Uninstaller, 'kontena/plugin_manager/uninstaller'
+    autoload :Cleaner, 'kontena/plugin_manager/cleaner'
+    autoload :Common, 'kontena/plugin_manager/common'
 
     # Initialize plugin manager
     def init
-      ENV["GEM_HOME"] = install_dir
+      ENV["GEM_HOME"] = Common.install_dir
       Gem.paths = ENV
+      Common.use_dummy_ui unless ENV["DEBUG"]
       plugins
-      use_dummy_ui unless ENV["DEBUG"]
       true
     end
-
-    # Install a plugin
-    # @param plugin_name [String]
-    # @param pre [Boolean] install a prerelease version if available
-    # @param version [String] install a specific version
-    def install_plugin(plugin_name, pre: false, version: nil)
-      cmd = Gem::DependencyInstaller.new(
-        document: false,
-        force: true,
-        prerelease: pre,
-        minimal_deps: true
-      )
-      plugin_version = version.nil? ? Gem::Requirement.default : Gem::Requirement.new(version)
-      cmd.install(prefix(plugin_name), plugin_version)
-      cmd.installed_gems
-    end
-
-    # Uninstall a plugin
-    # @param plugin_name [String]
-    def uninstall_plugin(plugin_name)
-      installed = installed(plugin_name)
-      raise "Plugin #{plugin_name} not installed" unless installed
-
-      cmd = Gem::Uninstaller.new(
-        installed.name,
-        all: true,
-        executables: true,
-        force: true,
-        install_dir: installed.base_dir
-      )
-      cmd.uninstall
-    end
-
-    # Search rubygems for kontena plugins
-    # @param pattern [String] optional search pattern
-    def search_plugins(pattern = nil)
-      RubygemsClient.new.search(prefix(pattern))
-    end
-
-    # Retrieve plugin versions from rubygems
-    # @param plugin_name [String]
-    def gem_versions(plugin_name)
-      RubygemsClient.new.versions(prefix(plugin_name))
-    end
-
-    # Get the latest version number from rubygems
-    # @param plugin_name [String]
-    # @param pre [Boolean] include prerelease versions
-    def latest_version(plugin_name, pre: false)
-      return gem_versions(plugin_name).first if pre
-      gem_versions(plugin_name).find { |version| !version.prerelease? }
-    end
-
-    # Find a plugin by name from installed plugins
-    # @param plugin_name [String]
-    def installed(plugin_name)
-      search = prefix(plugin_name)
-      plugins.find {|plugin| plugin.name == search }
-    end
-
-    # Upgrade an installed plugin
-    # @param plugin_name [String]
-    # @param pre [Boolean] upgrade to a prerelease version if available. Will happen always when the installed version is a prerelease version.
-    def upgrade_plugin(plugin_name, pre: false)
-      installed = installed(plugin_name)
-      if installed.version.prerelease?
-        pre = true
-      end
-
-      if installed
-        latest = latest_version(plugin_name, pre: pre)
-        if latest > installed.version
-          install_plugin(plugin_name, version: latest.to_s)
-        end
-      else
-        raise "Plugin #{plugin_name} not installed"
-      end
-    end
-
-    # Runs gem cleanup, removes remains from previous versions
-    # @param plugin_name [String]
-    def cleanup_plugin(plugin_name)
-      cmd = Gem::Commands::CleanupCommand.new
-      options = []
-      options += ['-q', '--no-verbose'] unless ENV["DEBUG"]
-      cmd.handle_options options
-      cmd.execute
-    rescue Gem::SystemExitException => e
-      return true if e.exit_code == 0
-      raise
-    end
-
-    # Gem installation directory
-    # @return [String]
-    def install_dir
-      return @install_dir if @install_dir
-      install_dir = File.join(Dir.home, '.kontena', 'gems', RUBY_VERSION)
-      unless File.directory?(install_dir)
-        require 'fileutils'
-        FileUtils.mkdir_p(install_dir, mode: 0700)
-      end
-      @install_dir = install_dir
-    end
-
+    module_function :init
 
     # @return [Array<Gem::Specification>]
     def plugins
-      @plugins ||= load_plugins
-    end
-
-    private
-
-    def plugin_debug?
-      @plugin_debug ||= ENV['DEBUG'] == 'plugin'
-    end
-
-    def load_plugins
-      plugins = []
-      Gem::Specification.to_a.each do |spec|
-        spec.require_paths.to_a.each do |require_path|
-          plugin = File.join(spec.gem_dir, require_path, 'kontena_cli_plugin.rb')
-          if File.exist?(plugin) && !plugins.find{ |p| p.name == spec.name }
-            begin
-              if spec_has_valid_dependency?(spec)
-
-                loaded_features_before = $LOADED_FEATURES.dup
-                load_path_before = $LOAD_PATH.dup
-
-                Kontena.logger.debug { "Activating plugin #{spec.name}" } if plugin_debug?
-                spec.activate
-                spec.activate_dependencies
-
-                Kontena.logger.debug { "Loading plugin #{spec.name}" }
-                require(plugin)
-                Kontena.logger.debug { "Loaded plugin #{spec.name}" } if plugin_debug?
-
-                if plugin_debug?
-                  added_features = ($LOADED_FEATURES - loaded_features_before).map {|feat| "- #{feat}"}
-                  added_paths = ($LOAD_PATH - load_path_before).map {|feat| "- #{feat}"}
-                  Kontena.logger.debug { "Plugin manager loaded features for #{spec.name}:" } unless added_features.empty?
-                  added_features.each { |feat| Kontena.logger.debug { feat } }
-                  Kontena.logger.debug { "Plugin manager load paths added for #{spec.name}:" } unless added_paths.empty?
-                  added_paths.each { |path| Kontena.logger.debug { path } }
-                end
-
-                plugins << spec
-              else
-                plugin_name = spec.name.sub('kontena-plugin-', '')
-                $stderr.puts " [#{Kontena.pastel.red('error')}] Plugin #{Kontena.pastel.cyan(plugin_name)} (#{spec.version}) is not compatible with the current cli version."
-                $stderr.puts "         To update the plugin, run 'kontena plugin install #{plugin_name}'"
-              end
-            rescue ScriptError, StandardError => ex
-              warn " [#{Kontena.pastel.red('error')}] Failed to load plugin: #{spec.name} from #{spec.gem_dir}\n\tRerun the command with environment DEBUG=true set to get the full exception."
-              Kontena.logger.error(ex)
-            end
-          end
-        end
-      end
-      plugins
-    rescue => ex
-      $stderr.puts Kontena.pastel.red(ex.message)
-      Kontena.logger.error(ex)
-    end
-
-    def prefix(plugin_name)
-      return plugin_name if plugin_name.to_s.start_with?('kontena-plugin-')
-      "kontena-plugin-#{plugin_name}"
-    end
-
-    def dummy_ui
-      Gem::StreamUI.new(StringIO.new, StringIO.new, StringIO.new, false)
-    end
-
-    def use_dummy_ui
-      Gem::DefaultUserInteraction.ui = dummy_ui
-    end
-
-    # @param [Gem::Specification] spec
-    # @return [Boolean]
-    def spec_has_valid_dependency?(spec)
-      kontena_cli = spec.runtime_dependencies.find{ |d| d.name == CLI_GEM }
-      !kontena_cli.match?(CLI_GEM, MIN_CLI_VERSION)
-    rescue
-      false
+      @plugins ||= Loader.new.load_plugins
     end
+    module_function :plugins
   end
 end