krates 1.6.0

data/lib/kontena/cli/stacks/yaml/stack_file_loader.rb

@@ -0,0 +1,152 @@
+require_relative '../stack_name'
+require 'yaml'
+require_relative 'reader'
+
+module Kontena::Cli::Stacks
+  module YAML
+    class StackFileLoader
+      # A base class for loading stack files. You can define more loaders by
+      # inheriting from this class.
+      #
+      # The purpose of StackFileLoader is to provide a generic interface for
+      # loading stack YAML's from different sources, such as local files,
+      # stack registry or URLs
+
+      def self.inherited(where)
+        loaders << where
+      end
+
+      def self.loaders
+        @loaders ||= []
+      end
+
+      # The main interface for getting a new loader
+      #
+      # @param source [String] stack file source string (filename, url, ..)
+      # @param parent [StackFileLoader] define a parent for recursion
+      # @return [StackFileLoader]
+      def self.for(source, parent = nil)
+        loader = loaders.find { |l| l.match?(source, parent) }
+        if loader.nil?
+          raise RuntimeError, "Not found: no such file #{source} or invalid uri scheme"
+        end
+        loader.new(source, parent)
+      end
+
+      attr_reader :source, :parent
+
+      # @param source [String] stack file source string (filename, url, ..)
+      # @param parent [StackFileLoader] define a parent for recursion
+      # @return [StackFileLoader]
+      def initialize(source, parent = nil)
+        @source = source
+        @parent = parent
+      end
+
+      # @return [String] a stripped down version of inspect without all the yaml source
+      def inspect
+        "#<#{self.class.name}:#{object_id} @source=#{source.inspect} @parent=#{parent.nil? ? 'nil' : parent.source}>"
+      end
+
+      # @return [Hash] a hash parsed from the YAML content
+      def yaml
+        @yaml ||= ::YAML.safe_load(content, [], [], true, source)
+      end
+
+      # @return [String] raw file content
+      def content
+        @content ||= read_content
+      end
+
+      def read_content
+        raise "Implement in inheriting class"
+      end
+
+      # @return [StackName] an accessor to StackName for the target file
+      def stack_name
+        @stack_name = Kontena::Cli::Stacks::StackName.new(yaml['stack'], yaml['version'])
+      end
+
+      # @return [Reader] an accessor to YAML::Reader for the target file
+      def reader(*args)
+        @reader ||= Reader.new(self, *args)
+      end
+
+      # Builds an array of hashes that represent the dependency tree starting
+      # from the target file. Unless recurse is set to false, the tree will
+      # contain also nested dependencies from any child stacks.
+      #
+      # @param recurse [TrueClass,FalseClass] recurse child dependencies?
+      # @return [Array<Hash>] an array of hashes ('name', 'stack', 'variables', and 'depends')
+      def dependencies(recurse: true)
+        return @dependencies if @dependencies
+        if depends.nil? || depends.empty?
+          @dependencies = nil
+        else
+          @dependencies = depends.map do |name, dependency|
+            loader = StackFileLoader.for(dependency['stack'], self)
+            deps = { 'name' => name, 'stack' => loader.source, 'variables' => dependency.fetch('variables', Hash.new) }
+            if recurse
+              child_deps = loader.dependencies
+              deps['depends'] = child_deps unless child_deps.nil?
+            end
+            deps
+          end
+        end
+      end
+
+      def to_h
+        {
+          'stack' => stack_name.stack_name,
+          :loader => self,
+        }
+      end
+
+      # Returns a non nested hash of all dependencies.
+      # Processes :variables hash and moves the related variables to children
+      #
+      # @param basename [String] installed stack name
+      # @param opts [Hash] extra data such as variable lists
+      # @return [Hash] { installation_name => { 'name' => installation-name, 'stack' => stack_name, :loader => self }, child_install_name => { ... } }
+      def flat_dependencies(basename, opts = {})
+        opt_variables = opts[:variables] || {}
+
+        result = {
+          basename => self.to_h.merge(opts).merge(
+            name: basename,
+            variables: opt_variables.reject { |k, _| k.include?('.') }
+          )
+        }
+
+        depends.each do |as_name, data|
+          variables = {}
+
+          opt_variables.select { |k, _| k.start_with?(as_name + '.') }.each do |k,v|
+            variables[k.split('.', 2).last] = v
+          end
+
+          data['variables'] ||= {}
+
+          loader = StackFileLoader.for(data['stack'], self)
+          result.merge!(
+           loader.flat_dependencies(
+             basename + '-' + as_name,
+             variables: data['variables'].merge(variables),
+             parent_name: basename
+           )
+          )
+        end
+
+        result
+      end
+
+      private
+
+      def depends
+        yaml['depends'] || {}
+      end
+    end
+  end
+end
+
+Dir[File.expand_path('../stack_file_loader/*.rb', __FILE__)].each { |f| require f }

data/lib/kontena/cli/stacks/yaml/validations.rb

@@ -0,0 +1,119 @@
+module Kontena::Cli::Stacks::YAML
+ module Validations
+   module CustomValidators
+     require_relative 'custom_validators/affinities_validator'
+     require_relative 'custom_validators/build_validator'
+     require_relative 'custom_validators/extends_validator'
+     require_relative 'custom_validators/hooks_validator'
+     require_relative 'custom_validators/secrets_validator'
+     require_relative 'custom_validators/certificates_validator'
+
+     def self.load
+       return if @loaded
+       HashValidator.append_validator(AffinitiesValidator.new)
+       HashValidator.append_validator(BuildValidator.new)
+       HashValidator.append_validator(ExtendsValidator.new)
+       HashValidator.append_validator(SecretsValidator.new)
+       HashValidator.append_validator(CertificatesValidator.new)
+       HashValidator.append_validator(HooksValidator.new)
+       @loaded = true
+     end
+   end
+
+   def common_validations
+     {
+        'image' => optional('string'), # it's optional because some base yml file might contain image option
+        'extends' => optional('stacks_valid_extends'),
+        'stateful' => optional('boolean'),
+        'affinity' => optional('stacks_valid_affinities'),
+        'cap_add' => optional('array'),
+        'cap_drop' => optional('array'),
+        'command' => optional('string'),
+        'cpus' => optional('float'),
+        'cpu_shares' => optional('integer'),
+        'external_links' => optional('array'),
+        'mem_limit' => optional('string'),
+        'mem_swaplimit' => optional('string'),
+        'shm_size' => optional('string'),
+        'environment' => optional(-> (value) {
+          if value.is_a?(Hash)
+            value.all? do |k,v|
+              k.kind_of?(String) && (
+                v.kind_of?(String) ||
+                v.kind_of?(Integer) ||
+                v.kind_of?(TrueClass) ||
+                v.kind_of?(FalseClass) ||
+                v.nil?
+              )
+            end
+          elsif value.is_a?(Array)
+            value.all? { |v| v.kind_of?(String) && v =~ /\A[^=]+=/ }
+          else
+            false
+          end
+        }),
+        'env_file' => optional(-> (value) { value.is_a?(String) || value.is_a?(Array) }),
+        'instances' => optional('integer'),
+        'links' => optional(-> (value) { value.is_a?(Array) || value.nil? }),
+        'ports' => optional(-> (value) { value.is_a?(Array) && value.all? { |v| v.is_a?(String) && v.match(/\A(\d+\.\d+\.\d+\.\d+)?:?(\d+)\:(\d+)\/?(\w+)?\z/) } }),
+        'pid' => optional('string'),
+        'privileged' => optional('boolean'),
+        'user' => optional('string'),
+        'volumes' => optional('array'),
+        'volumes_from' => optional('array'),
+        'secrets' => optional('stacks_valid_secrets'),
+        'certificates' => optional('stacks_valid_certificates'),
+        'hooks' => optional('stacks_valid_hooks'),
+        'only_if' => optional(-> (value) { value.is_a?(String) || value.is_a?(Hash) || value.is_a?(Array) }),
+        'skip_if' => optional(-> (value) { value.is_a?(String) || value.is_a?(Hash) || value.is_a?(Array) }),
+        'deploy' => optional({
+          'strategy' => optional(%w(ha daemon random)),
+          'wait_for_port' => optional('integer'),
+          'min_health' => optional('float'),
+          'interval' => optional(/^\d+(min|h|d|)$/)
+        }),
+        'health_check' => optional({
+          'protocol' => /^(http|tcp)$/,
+          'port' => 'integer',
+          'uri' => optional(/\/[\S]*/),
+          'timeout' => optional('integer'),
+          'interval' => optional('integer'),
+          'initial_delay' => optional('integer')
+        }),
+        'stop_signal' => optional('string'),
+        'stop_grace_period' => optional(/(\d+(?:\.\d+)?)([hms])/),
+        'read_only' => optional('boolean'),
+        'entrypoint' => optional('string')
+      }
+    end
+
+    def optional(type)
+      HashValidator.optional(type)
+    end
+
+    def validate_options(service_config)
+      HashValidator.validate(service_config, @schema, true)
+    end
+
+    def validate_volume_options(volume_config)
+      HashValidator.validate(volume_config, volume_schema, true)
+    end
+
+    def validate_dependencies(dependency_config)
+      HashValidator.validate(dependency_config, dependency_schema, true)
+    end
+
+    def volume_schema
+      {
+        'external' => optional(-> (value) { value.is_a?(TrueClass) || (value.is_a?(Hash) && value['name'].is_a?(String)) })
+      }
+    end
+
+    def dependency_schema
+      {
+        'stack' => optional('string'),
+        'variables' => optional(-> (value) { value.is_a?(Hash) })
+      }
+    end
+  end
+end

data/lib/kontena/cli/stacks/yaml/validator_v3.rb

@@ -0,0 +1,164 @@
+require 'hash_validator'
+
+module Kontena::Cli::Stacks
+  module YAML
+    class ValidatorV3
+      require_relative 'validations'
+      include Validations
+
+      KNOWN_TOP_LEVEL_KEYS = %i(
+        services
+        errors
+        volumes
+        networks
+        variables
+        stack
+        version
+        data
+        description
+        expose
+        depends
+        labels
+      )
+
+      def initialize
+        @schema = common_validations
+        @schema['build'] = optional('stacks_valid_build')
+        @schema['depends_on'] = optional('array')
+        @schema['network_mode'] = optional(%w(host bridge))
+        @schema['logging'] = optional({
+          'driver' => optional('string'),
+          'options' => optional(-> (value) { value.kind_of?(Hash) })
+          })
+        Validations::CustomValidators.load
+      end
+
+      # borrowed from server/app/helpers/volumes_helpers.rb
+      def parse_volume(vol)
+        elements = vol.split(':')
+        if elements.size >= 2 # Bind mount or volume used
+          if elements[0].start_with?('/') && elements[1] && elements[1].start_with?('/') # Bind mount
+            {bind_mount: elements[0], path: elements[1], flags: elements[2..-1].join(',')}
+          elsif !elements[0].start_with?('/') && elements[1].start_with?('/') # Real volume
+            {volume: elements[0], path: elements[1], flags: elements[2..-1].join(',')}
+          else
+            {error: "volume definition not in right format: #{vol}" }
+          end
+        elsif elements.size == 1 && elements[0].start_with?('/') # anon volume
+          {bind_mount: nil, path: elements[0], flags: nil} # anon vols do not support flags
+        else
+          {error: "volume definition not in right format: #{vol}" }
+        end
+      end
+
+      ##
+      # @param [Hash] yaml
+      # @param [TrueClass|FalseClass] strict
+      # @return [Array] validation_errors
+      def validate(yaml)
+        result = {
+          errors: [],
+          notifications: []
+        }
+
+        yaml.keys.each do |key|
+          unless KNOWN_TOP_LEVEL_KEYS.include?(key) || KNOWN_TOP_LEVEL_KEYS.include?(key.to_sym)
+            result[:notifications] << { key.to_s => "unknown top level key" }
+          end
+        end
+
+        if yaml.key?('stack')
+          unless yaml['stack'] =~ /\A(?:.+?\/)?(?!-)[a-z0-9\-]+\z/
+            result[:notifications] << { 'stack' => 'A stack name should only include a-z, 0-9 and - characters and not start with the - character' }
+          end
+        end
+
+        if yaml.key?('services')
+          if yaml['services'].kind_of?(Hash)
+            yaml['services'].each do |service, options|
+              unless service =~ /\A(?!-)[a-z0-9\-]+\z/
+                result[:notifications] << { 'services' => { service => { 'name' => 'A service name should only include a-z, 0-9 and - characters and not start with the - character' } } }
+              end
+              unless options.kind_of?(Hash)
+                result[:errors] << { 'services' => { service => { 'options' => "must be a mapping not a #{options.class}"}  } }
+                next
+              end
+              option_errors = validate_options(options)
+              result[:errors] << { 'services' => { service => option_errors.errors } } unless option_errors.valid?
+              if options['volumes']
+                mount_path_occurences = Hash.new(0)
+                options['volumes'].each do |volume|
+                  parsed = parse_volume(volume)
+                  if parsed[:error]
+                    result[:errors] << { 'services' => { service => { 'volumes' => { volume => parsed[:error] } } } }
+                  elsif parsed[:path]
+                    mount_path_occurences[parsed[:path]] += 1
+                    volume_name = parsed[:volume]
+                    if volume_name && !volume_name.start_with?('/')
+                      if yaml.key?('volumes')
+                        unless yaml['volumes'][volume_name]
+                          result[:errors] << { 'services' => { service => { 'volumes' => { volume_name => 'not found in top level volumes list' } } } }
+                        end
+                      else
+                        result[:errors] << { 'services' => { service => { 'volumes' => { volume => 'defines volume name, but file does not contain volumes definitions' } } } }
+                      end
+                    end
+                  else
+                    result[:errors] << { 'services' => { service => { 'volumes' => { volume => 'mount point missing' } } } }
+                  end
+                end
+                mount_path_occurences.select {|path, occurences| occurences > 1 }.each do |path, occurences|
+                  result[:errors] << { 'services' => { service => { 'volumes' => { path => "mount point defined #{occurences} times" } } } }
+                end
+              end
+            end
+          else
+            result[:errors] << { 'services' => "must be a mapping, not #{yaml['services'].class}" }
+          end
+        else
+          result[:notifications] << { 'file' => 'does not define any services' }
+        end
+
+        if yaml.key?('volumes')
+          if yaml['volumes'].kind_of?(Hash)
+            yaml['volumes'].each do |volume, options|
+              if options.kind_of?(Hash)
+                option_errors = validate_volume_options(options)
+                unless option_errors.valid?
+                  result[:errors] << { 'volumes' => { volume => option_errors.errors } }
+                end
+              else
+                result[:errors] << { 'volumes' => { volume => { 'options' => "must be a mapping, not #{options.class}" } } }
+              end
+            end
+          else
+            result[:errors] << { 'volumes' => "must be a mapping, not #{yaml['volumes'].class}" }
+          end
+        end
+
+        if yaml.key?('networks')
+          result[:notifications] << { 'networks' => 'Kontena does not support multiple networks yet. You can reference services with Kontena\'s internal DNS (service_name.kontena.local)' }
+        end
+
+        if (yaml['volumes'].nil? || yaml['volumes'].empty?) && (yaml['services'].nil? || yaml['services'].empty?)
+          result[:errors] << { 'file' => 'does not list any services or volumes' }
+        end
+
+        if yaml.key?('depends')
+          unless yaml['depends'].kind_of?(Hash)
+            result[:errors] << { 'depends' => "Must be a mapping, not #{yaml['depends'].class}" }
+          end
+
+          yaml['depends'].each do |name, dependency_options|
+            validator = validate_dependencies(dependency_options)
+            result[:errors] << { 'depends' => { name => validator.errors } } unless validator.valid?
+            if yaml.key?('services') && yaml['services'][name]
+              result[:errors] << { 'depends' => { name => 'is defined both as service and dependency name' } }
+            end
+          end
+        end
+        result
+      end
+    end
+  end
+end

data/lib/kontena/cli/subcommand_loader.rb

@@ -0,0 +1,83 @@
+require 'kontena_cli'
+module Kontena::Cli
+  class SubcommandLoader
+    attr_reader :path
+
+    # Create a subcommand loader instance
+    #
+    # @param [String] path path to command definition
+    def initialize(path)
+      @path = path
+    end
+
+    # Takes something like /foo/bar/cli/master/foo_coimmand and returns [:Master, :FooCommand]
+    #
+    # @param path [String]
+    # @return [Array<Symbol>]
+    def symbolize_path(path)
+      path.gsub(/.*\/cli\//, '').split('/').map do |path_part|
+        path_part.split('_').map{ |e| e.capitalize }.join
+      end.map(&:to_sym)
+    end
+
+    # Takes an array such as [:Foo] or [:Cli, :Foo] and returns [:Kontena, :Cli, :Foo]
+    def prepend_kontena_cli(tree)
+      [:Kontena, :Cli] + (tree - [:Cli])
+    end
+
+    # Takes an array such as [:Master, :FooCommand] and returns Master::FooCommand
+    #
+    # @param tree [Array<Symbol]
+    # @return [Class]
+    def const_get_tree(tree)
+      if tree.size == 1
+        Object.const_get(tree.first)
+      else
+        tree[1..-1].inject(Object.const_get(tree.first)) { |new_base, part| new_base.const_get(part) }
+      end
+    rescue
+      raise ArgumentError, "Can't figure out command class name from path #{path} - tried #{tree}"
+    end
+
+    # Tries to require a file, returns false instead of raising LoadError unless succesful
+    #
+    # @param path [String]
+    # @return [TrueClass,FalseClass]
+    def safe_require(path)
+      require path
+      true
+    rescue LoadError
+      false
+    end
+
+    def klass
+      return @subcommand_class if @subcommand_class
+      unless safe_require(path) || safe_require(Kontena.cli_root(path))
+        raise ArgumentError, "Can't load #{path} or #{Kontena.cli_root(path)}"
+      end
+      @subcommand_class = const_get_tree(prepend_kontena_cli(symbolize_path(path)))
+    end
+
+    def new(*args)
+      klass.new(*args)
+    end
+
+    def method_missing(meth, *args)
+      klass.send(meth, *args)
+    end
+
+    def respond_to_missing?(meth)
+      klass.respond_to?(meth)
+    end
+
+    def const_get(const)
+      klass.const_get(const)
+    end
+
+    def const_defined?(const)
+      klass.const_defined?(const)
+    end
+
+    alias_method :class, :klass
+  end
+end

data/lib/kontena/cli/table_generator.rb

@@ -0,0 +1,128 @@
+require 'tty-table'
+
+module Kontena
+  module Cli
+    class TableGenerator
+
+      attr_reader :data
+      attr_reader :fields
+      attr_reader :header
+      attr_reader :row_format_proc, :header_format_proc, :render_options, :render_mode
+
+      DEFAULT_HEADER_FORMAT_PROC = lambda { |header| header.to_s.capitalize }
+
+      DEFAULT_RENDER_OPTS = {padding: [0,2,0,0]}
+
+      module Helper
+        def self.included(base)
+          if base.respond_to?(:option)
+            base.option ['-q', '--quiet'], :flag, "Output the identifying column only"
+          end
+        end
+
+        def table_generator
+          Kontena::Cli::TableGenerator
+        end
+
+        def generate_table(array, fields = nil, &block)
+          fields ||= self.fields if self.respond_to?(:fields)
+          table_generator.new(
+            array,
+            fields,
+            row_format_proc: block_given? ? block.to_proc : nil,
+            header_format_proc: lambda { |item| pastel.bold(item.to_s.upcase) },
+            render_options: self.respond_to?(:render_options) ? DEFAULT_RENDER_OPTS.merge(self.render_options) : DEFAULT_RENDER_OPTS
+          ).render
+        end
+
+        def print_table(array, fields = nil, &block)
+          output = generate_table(array, fields, &block)
+          puts output unless output.strip.empty?
+        end
+      end
+
+      # @param data [Array<Hash>,Array<Array>] an array of hashes or arrays
+      # @param fields [Array] an array of field names found in the data hashes.
+      # @param fields [Hash] a hash of field_title => field_name_in_the_data_hash, for example 'Users' => 'user_count'
+      # @param fields [NilClass] try to auto detect fields (all fields!) from the data hashes
+      # @return [TTY::Table]
+      def initialize(data, fields = nil, row_format_proc: nil, header_format_proc: nil, render_options: nil)
+        @data = data
+        @render_options = render_options || { }
+        @render_mode = @render_options.delete(:mode) || :basic
+        @row_format_proc = row_format_proc
+        @header_format_proc = header_format_proc || DEFAULT_HEADER_FORMAT_PROC
+        @fields = parse_fields(fields)
+        @header = generate_header(fields || @fields)
+      end
+
+      def create_table(header, rows)
+        TTY::Table.new(
+          header: header,
+          rows: rows
+        )
+      end
+
+      def table
+        create_table(header, rows)
+      end
+
+      def render
+        if data.empty?
+          fields.size > 1 ? fields.map(&method(:format_header_item)).join('  ') : ''
+        else
+          table.render(render_mode, render_options).gsub(/\s+$/, '')
+        end
+      end
+
+      def format_row(row)
+        return row if row_format_proc.nil?
+        row_clone = row.dup
+        row_format_proc.call(row_clone)
+        row_clone
+      end
+
+      def format_header_item(field_name)
+        header_format_proc.call(field_name)
+      end
+
+      def rows
+        fields.empty? ? data.map { |row| format_row(row).map(&:values) } : data.map { |row| format_row(row).values_at(*fields) }
+      end
+
+      # Collect all the unique keys from the hashes if the data
+      # is an array of hashes.
+      def detect_fields
+        if data.first.respond_to?(:keys)
+          data.flat_map(&:keys).uniq
+        else
+          []
+        end
+      end
+
+      def parse_fields(fields)
+        if fields.nil? || fields.empty?
+          detect_fields
+        elsif fields.kind_of?(Hash)
+          fields.values
+        else
+          fields
+        end
+      end
+
+      def generate_header(fields)
+        if fields.kind_of?(Hash)
+          header = fields.keys
+        else
+          header = Array(fields)
+        end
+
+        if header.size < 2
+          nil
+        else
+          header.map { |head| format_header_item(head) }
+        end
+      end
+    end
+  end
+end

data/lib/kontena/cli/vault/export_command.rb

@@ -0,0 +1,24 @@
+module Kontena::Cli::Vault
+  class ExportCommand < Kontena::Command
+    include Kontena::Cli::Common
+    include Kontena::Cli::GridOptions
+
+    banner "Exports secrets from Vault to STDOUT as YAML or JSON."
+
+    requires_current_master
+
+    option '--json', :flag, "Output JSON"
+
+    def execute
+      require 'shellwords'
+      require 'json'
+      require 'yaml'
+      meth = json? ? :to_json : :to_yaml
+      puts(
+        Kontena.run!(['vault', 'ls', '--return']).sort.map do |secret|
+          [secret, Kontena.run!(['vault', 'read', '--return', secret])]
+        end.to_h.send(meth)
+      )
+    end
+  end
+end