leap_cli 1.5.6 → 1.6.2

data/lib/leap_cli/config/manager.rb

@@ -20,6 +20,16 @@ module LeapCli
 
       def initialize
         @environments = {} # hash of `Environment` objects, keyed by name.
+
+        # load macros and other custom ruby in provider base
+        platform_ruby_files = Dir[Path.provider_base + '/lib/*.rb']
+        if platform_ruby_files.any?
+          $: << Path.provider_base + '/lib'
+          platform_ruby_files.each do |rb_file|
+            require rb_file
+          end
+        end
+        Config::Object.send(:include, LeapCli::Macro)
       end
 
       ##
@@ -54,9 +64,24 @@ module LeapCli
         e
       end
 
-      def services; env('default').services; end
-      def tags; env('default').tags; end
-      def provider; env('default').provider; end
+      #
+      # The default accessors for services, tags, and provider.
+      # For these defaults, use 'default' environment, or whatever
+      # environment is pinned.
+      #
+      def services
+        env(default_environment).services
+      end
+      def tags
+        env(default_environment).tags
+      end
+      def provider
+        env(default_environment).provider
+      end
+
+      def default_environment
+        LeapCli.leapfile.environment
+      end
 
       ##
       ## IMPORT EXPORT
@@ -80,8 +105,8 @@ module LeapCli
         @secrets  = load_json(    Path.named_path(:secrets_config,      @provider_dir), Config::Secrets)
         @common.inherit_from! @base_common
 
-        # load provider services, tags, and provider.json, DEFAULT environment
-        log 3, :loading, 'default environment.........'
+        # For the default environment, load provider services, tags, and provider.json
+        log 3, :loading, 'default environment...'
         env('default') do |e|
           e.services = load_all_json(Path.named_path([:service_config, '*'], @provider_dir), Config::Tag, :no_dots => true)
           e.tags     = load_all_json(Path.named_path([:tag_config, '*'],     @provider_dir), Config::Tag, :no_dots => true)
@@ -92,28 +117,54 @@ module LeapCli
           validate_provider(e.provider)
         end
 
-        # load provider services, tags, and provider.json, OTHER environments
+        # create a special '_all_' environment, used for tracking the union
+        # of all the environments
+        env('_all_') do |e|
+          e.services = Config::ObjectList.new
+          e.tags     = Config::ObjectList.new
+          e.provider = Config::Provider.new
+          e.services.inherit_from! env('default').services
+          e.tags.inherit_from!     env('default').tags
+          e.provider.inherit_from! env('default').provider
+        end
+
+        # For each defined environment, load provider services, tags, and provider.json.
         environment_names.each do |ename|
           next unless ename
-          log 3, :loading, '%s environment.........' % ename
+          log 3, :loading, '%s environment...' % ename
           env(ename) do |e|
             e.services = load_all_json(Path.named_path([:service_env_config, '*', ename], @provider_dir), Config::Tag)
             e.tags     = load_all_json(Path.named_path([:tag_env_config, '*', ename],     @provider_dir), Config::Tag)
             e.provider = load_json(    Path.named_path([:provider_env_config, ename],     @provider_dir), Config::Provider)
-            e.services.inherit_from! env.services
-            e.tags.inherit_from!     env.tags
-            e.provider.inherit_from! env.provider
+            e.services.inherit_from! env('default').services
+            e.tags.inherit_from!     env('default').tags
+            e.provider.inherit_from! env('default').provider
             validate_provider(e.provider)
           end
         end
 
+        # apply inheritance
         @nodes.each do |name, node|
           Util::assert! name =~ /^[0-9a-z-]+$/, "Illegal character(s) used in node name '#{name}'"
           @nodes[name] = apply_inheritance(node)
         end
 
-        unless options[:include_disabled]
-          remove_disabled_nodes
+        # do some node-list post-processing
+        cleanup_node_lists(options)
+
+        # apply control files
+        @nodes.each do |name, node|
+          control_files(node).each do |file|
+            begin
+              node.eval_file file
+            rescue ConfigError => exc
+              if options[:continue_on_error]
+                exc.log
+              else
+                raise exc
+              end
+            end
+          end
         end
       end
 
@@ -178,35 +229,19 @@ module LeapCli
       # returns a node list consisting only of nodes that satisfy the filter criteria.
       #
       # filter: condition [condition] [condition] [+condition]
-      # condition: [node_name | service_name | tag_name]
+      # condition: [node_name | service_name | tag_name | environment_name]
       #
       # if conditions is prefixed with +, then it works like an AND. Otherwise, it works like an OR.
       #
-      def filter(filters)
-        if filters.empty?
-          return nodes
-        end
-        if filters[0] =~ /^\+/
-          # don't let the first filter have a + prefix
-          filters[0] = filters[0][1..-1]
-        end
-
-        node_list = Config::ObjectList.new
-        filters.each do |filter|
-          if filter =~ /^\+/
-            keep_list = nodes_for_name(filter[1..-1])
-            node_list.delete_if do |name, node|
-              if keep_list[name]
-                false
-              else
-                true
-              end
-            end
-          else
-            node_list.merge!(nodes_for_name(filter))
-          end
-        end
-        return node_list
+      # args:
+      # filter -- array of filter terms, one per item
+      #
+      # options:
+      # :local -- if :local is false and the filter is empty, then local nodes are excluded.
+      # :nopin -- if true, ignore environment pinning
+      #
+      def filter(filters=nil, options={})
+        Filter.new(filters, options, self).nodes()
       end
 
       #
@@ -248,6 +283,28 @@ module LeapCli
         @nodes[node.name] = apply_inheritance!(node)
       end
 
+      #
+      # returns all the partial data for the specified partial path.
+      # partial path is always relative to provider root, but there must be multiple files
+      # that match because provider root might be the base provider or the local provider.
+      #
+      def partials(partial_path)
+        @partials ||= {}
+        if @partials[partial_path].nil?
+          [Path.provider_base, Path.provider].each do |provider_dir|
+            path = File.join(provider_dir, partial_path)
+            if File.exists?(path)
+              @partials[partial_path] ||= []
+              @partials[partial_path] << load_json(path, Config::Object)
+            end
+          end
+          if @partials[partial_path].nil?
+            raise RuntimeError, 'no such partial path `%s`' % partial_path, caller
+          end
+        end
+        @partials[partial_path]
+      end
+
       private
 
       def load_all_json(pattern, object_class, options={})
@@ -358,7 +415,6 @@ module LeapCli
               raise LeapCli::ConfigError.new(node, "error " + msg) if throw_exceptions
             else
               new_node.deep_merge!(service)
-              self.services[node_service].node_list.add(name, new_node)
             end
           end
         end
@@ -376,7 +432,6 @@ module LeapCli
               raise LeapCli::ConfigError.new(node, "error " + msg) if throw_exceptions
             else
               new_node.deep_merge!(tag)
-              self.tags[node_tag].node_list.add(name, new_node)
             end
           end
         end
@@ -390,45 +445,59 @@ module LeapCli
         apply_inheritance(node, true)
       end
 
-      def remove_disabled_nodes
+      #
+      # does some final clean at the end of loading nodes.
+      # this includes removing disabled nodes, and populating
+      # the services[x].node_list and tags[x].node_list
+      #
+      def cleanup_node_lists(options)
         @disabled_nodes = Config::ObjectList.new
         @nodes.each do |name, node|
-          unless node.enabled
-            log 2, :skipping, "disabled node #{name}."
-            @nodes.delete(name)
-            @disabled_nodes[name] = node
+          if node.enabled || options[:include_disabled]
             if node['services']
               node['services'].to_a.each do |node_service|
-                self.services[node_service].node_list.delete(node.name)
+                env(node.environment).services[node_service].node_list.add(node.name, node)
+                env('_all_').services[node_service].node_list.add(node.name, node)
               end
             end
             if node['tags']
               node['tags'].to_a.each do |node_tag|
-                self.tags[node_tag].node_list.delete(node.name)
+                env(node.environment).tags[node_tag].node_list.add(node.name, node)
+                env('_all_').tags[node_tag].node_list.add(node.name, node)
               end
             end
+          elsif !options[:include_disabled]
+            log 2, :skipping, "disabled node #{name}."
+            @nodes.delete(name)
+            @disabled_nodes[name] = node
           end
         end
       end
 
+      def validate_provider(provider)
+        # nothing yet.
+      end
 
       #
-      # returns a set of nodes corresponding to a single name, where name could be a node name, service name, or tag name.
+      # returns a list of 'control' files for this node.
+      # a control file is like a service or a tag JSON file, but it contains
+      # raw ruby code that gets evaluated in the context of the node.
+      # Yes, this entirely breaks our functional programming model
+      # for JSON generation.
       #
-      def nodes_for_name(name)
-        if node = self.nodes[name]
-          Config::ObjectList.new(node)
-        elsif service = self.services[name]
-          service.node_list
-        elsif tag = self.tags[name]
-          tag.node_list
-        else
-          {}
+      def control_files(node)
+        files = []
+        [Path.provider_base, @provider_dir].each do |provider_dir|
+          [['services', :service_config], ['tags', :tag_config]].each do |attribute, path_sym|
+            node[attribute].each do |attr_value|
+              path = Path.named_path([path_sym, "#{attr_value}.rb"], provider_dir).sub(/\.json$/,'')
+              if File.exists?(path)
+                files << path
+              end
+            end
+          end
         end
-      end
-
-      def validate_provider(provider)
-        # nothing yet.
+        return files
       end
 
     end

data/lib/leap_cli/config/node.rb

@@ -33,6 +33,29 @@ module LeapCli; module Config
       return vagrant_range.include?(ip_address)
     end
 
+    #
+    # Return a hash table representation of ourselves, with the key equal to the @node.name,
+    # and the value equal to the fields specified in *keys.
+    #
+    # Also, the result is flattened to a single hash, so a key of 'a.b' becomes 'a_b'
+    #
+    # compare to Object#pick(*keys). This method is the sames as Config::ObjectList#pick_fields,
+    # but works on a single node.
+    #
+    # Example:
+    #
+    #  node.pick('domain.internal') =>
+    #
+    #    {
+    #      'node1': {
+    #        'domain_internal': 'node1.example.i'
+    #      }
+    #    }
+    #
+    def pick_fields(*keys)
+      {@node.name => self.pick(*keys)}
+    end
+
     #
     # can be overridden by the platform.
     # returns a list of node names that should be tested before this node
@@ -40,6 +63,15 @@ module LeapCli; module Config
     def test_dependencies
       []
     end
+
+    # returns a string list of supported ssh host key algorithms for this node.
+    # or an empty string if it could not be determined
+    def supported_ssh_host_key_algorithms
+      @host_key_algo ||= SshKey.supported_host_key_algorithms(
+        Util.read_file([:node_ssh_pub_key, @node.name])
+      )
+    end
+
   end
 
 end; end

data/lib/leap_cli/config/object.rb

@@ -8,8 +8,6 @@ if $ruby_version < [1,9]
 end
 require 'ya2yaml' # pure ruby yaml
 
-require 'leap_cli/config/macros'
-
 module LeapCli
   module Config
     #
@@ -20,8 +18,6 @@ module LeapCli
     #
     class Object < Hash
 
-      include Config::Macros
-
       attr_reader :node
       attr_reader :manager
       alias :global :manager
@@ -44,7 +40,7 @@ module LeapCli
       #
       def dump_yaml
         evaluate(@node)
-        ya2yaml(:syck_compatible => true)
+        sorted_ya2yaml(:syck_compatible => true)
       end
 
       #
@@ -68,6 +64,11 @@ module LeapCli
         get(key)
       end
 
+      # Overrride some default methods in Hash that are likely to
+      # be used as attributes.
+      alias_method :hkey, :key
+      def key; get('key'); end
+
       #
       # make hash addressable like an object (e.g. obj['name'] available as obj.name)
       #
@@ -134,7 +135,18 @@ module LeapCli
       #
       def deep_merge!(object, prefer_self=false)
         object.each do |key,new_value|
-          old_value = self.fetch key, nil
+          if self.has_key?('+'+key)
+            mode = :add
+            old_value = self.fetch '+'+key, nil
+            self.delete('+'+key)
+          elsif self.has_key?('-'+key)
+            mode = :subtract
+            old_value = self.fetch '-'+key, nil
+            self.delete('-'+key)
+          else
+            mode = :normal
+            old_value = self.fetch key, nil
+          end
 
           # clean up boolean
           new_value = true  if new_value == "true"
@@ -160,6 +172,18 @@ module LeapCli
           elsif new_value.is_a?(Array) && !old_value.is_a?(Array)
             (value = (new_value.dup << old_value).compact.uniq).delete('REQUIRED')
 
+          # merge two arrays
+          elsif old_value.is_a?(Array) && new_value.is_a?(Array)
+            if mode == :add
+              value = (old_value + new_value).sort.uniq
+            elsif mode == :subtract
+              value = new_value - old_value
+            elsif prefer_self
+              value = old_value
+            else
+              value = new_value
+            end
+
           # catch errors
           elsif type_mismatch?(old_value, new_value)
             raise 'Type mismatch. Cannot merge %s (%s) with %s (%s). Key is "%s", name is "%s".' % [
@@ -168,7 +192,7 @@ module LeapCli
               key, self.class
             ]
 
-          # merge strings, numbers, and sometimes arrays
+          # merge simple strings & numbers
           else
             if prefer_self
               value = old_value
@@ -206,6 +230,10 @@ module LeapCli
         end
       end
 
+      def eval_file(filename)
+        evaluate_ruby(filename, File.read(filename))
+      end
+
       protected
 
       #
@@ -246,45 +274,42 @@ module LeapCli
       # (`key` is just passed for debugging purposes)
       #
       def evaluate_ruby(key, value)
-        result = nil
-        if LeapCli.log_level >= 2
-          result = self.instance_eval(value)
-        else
-          begin
-            result = self.instance_eval(value)
-          rescue SystemStackError => exc
-            Util::log 0, :error, "while evaluating node '#{self.name}'"
-            Util::log 0, "offending key: #{key}", :indent => 1
-            Util::log 0, "offending string: #{value}", :indent => 1
-            Util::log 0, "STACK OVERFLOW, BAILING OUT. There must be an eval loop of death (variables with circular dependencies).", :indent => 1
-            raise SystemExit.new(1)
-          rescue FileMissing => exc
-            Util::bail! do
-              if exc.options[:missing]
-                Util::log :missing, exc.options[:missing].gsub('$node', self.name)
-              else
-                Util::log :error, "while evaluating node '#{self.name}'"
-                Util::log "offending key: #{key}", :indent => 1
-                Util::log "offending string: #{value}", :indent => 1
-                Util::log "error message: no file '#{exc}'", :indent => 1
-              end
-            end
-          rescue AssertionFailed => exc
-            Util.bail! do
-              Util::log :failed, "assertion while evaluating node '#{self.name}'"
-              Util::log 'assertion: %s' % exc.assertion, :indent => 1
-              Util::log "offending key: #{key}", :indent => 1
-            end
-          rescue SyntaxError, StandardError => exc
-            Util::bail! do
-              Util::log :error, "while evaluating node '#{self.name}'"
-              Util::log "offending key: #{key}", :indent => 1
-              Util::log "offending string: #{value}", :indent => 1
-              Util::log "error message: #{exc.inspect}", :indent => 1
-            end
+        self.instance_eval(value, key, 1)
+      rescue ConfigError => exc
+        raise exc # pass through
+      rescue SystemStackError => exc
+        Util::log 0, :error, "while evaluating node '#{self.name}'"
+        Util::log 0, "offending key: #{key}", :indent => 1
+        Util::log 0, "offending string: #{value}", :indent => 1
+        Util::log 0, "STACK OVERFLOW, BAILING OUT. There must be an eval loop of death (variables with circular dependencies).", :indent => 1
+        raise SystemExit.new(1)
+      rescue FileMissing => exc
+        Util::bail! do
+          if exc.options[:missing]
+            Util::log :missing, exc.options[:missing].gsub('$node', self.name).gsub('$file', exc.path)
+          else
+            Util::log :error, "while evaluating node '#{self.name}'"
+            Util::log "offending key: #{key}", :indent => 1
+            Util::log "offending string: #{value}", :indent => 1
+            Util::log "error message: no file '#{exc}'", :indent => 1
           end
+          raise exc if DEBUG
+        end
+      rescue AssertionFailed => exc
+        Util.bail! do
+          Util::log :failed, "assertion while evaluating node '#{self.name}'"
+          Util::log 'assertion: %s' % exc.assertion, :indent => 1
+          Util::log "offending key: #{key}", :indent => 1
+          raise exc if DEBUG
+        end
+      rescue SyntaxError, StandardError => exc
+        Util::bail! do
+          Util::log :error, "while evaluating node '#{self.name}'"
+          Util::log "offending key: #{key}", :indent => 1
+          Util::log "offending string: #{value}", :indent => 1
+          Util::log "error message: #{exc.inspect}", :indent => 1
+          raise exc if DEBUG
         end
-        return result
       end
 
       private

data/lib/leap_cli/config/object_list.rb

@@ -28,42 +28,22 @@ module LeapCli
       # nodes[:public_dns => true]
       #   all nodes with public dns
       #
-      # nodes[:services => 'openvpn', :services => 'tor']
+      # nodes[:services => 'openvpn', 'location.country_code' => 'US']
+      #   all nodes with services containing 'openvpn' OR country code of US
+      #
+      # Sometimes, you want to do an OR condition with multiple conditions
+      # for the same field. Since hash keys must be unique, you can use
+      # an array representation instead:
+      #
+      # nodes[[:services, 'openvpn'], [:services, 'tor']]
       #   nodes with openvpn OR tor service
       #
       # nodes[:services => 'openvpn'][:tags => 'production']
       #   nodes with openvpn AND are production
       #
       def [](key)
-        if key.is_a? Hash
-          results = Config::ObjectList.new
-          key.each do |field, match_value|
-            field = field.is_a?(Symbol) ? field.to_s : field
-            match_value = match_value.is_a?(Symbol) ? match_value.to_s : match_value
-            if match_value.is_a?(String) && match_value =~ /^!/
-              operator = :not_equal
-              match_value = match_value.sub(/^!/, '')
-            else
-              operator = :equal
-            end
-            each do |name, config|
-              value = config[field]
-              if value.is_a? Array
-                if operator == :equal && value.include?(match_value)
-                  results[name] = config
-                elsif operator == :not_equal && !value.include?(match_value)
-                  results[name] = config
-                end
-              else
-                if operator == :equal && value == match_value
-                  results[name] = config
-                elsif operator == :not_equal && value != match_value
-                  results[name] = config
-                end
-              end
-            end
-          end
-          results
+        if key.is_a?(Hash) || key.is_a?(Array)
+          filter(key)
         else
           super key.to_s
         end
@@ -81,15 +61,40 @@ module LeapCli
         end
       end
 
-      # def <<(object)
-      #   if object.is_a? Config::ObjectList
-      #     self.merge!(object)
-      #   elsif object['name']
-      #     self[object['name']] = object
-      #   else
-      #     raise ArgumentError.new('argument must be a Config::Object or a Config::ObjectList')
-      #   end
-      # end
+      #
+      # filters this object list, producing a new list.
+      # filter is an array or a hash. see []
+      #
+      def filter(filter)
+        results = Config::ObjectList.new
+        filter.each do |field, match_value|
+          field = field.is_a?(Symbol) ? field.to_s : field
+          match_value = match_value.is_a?(Symbol) ? match_value.to_s : match_value
+          if match_value.is_a?(String) && match_value =~ /^!/
+            operator = :not_equal
+            match_value = match_value.sub(/^!/, '')
+          else
+            operator = :equal
+          end
+          each do |name, config|
+            value = config[field]
+            if value.is_a? Array
+              if operator == :equal && value.include?(match_value)
+                results[name] = config
+              elsif operator == :not_equal && !value.include?(match_value)
+                results[name] = config
+              end
+            else
+              if operator == :equal && value == match_value
+                results[name] = config
+              elsif operator == :not_equal && value != match_value
+                results[name] = config
+              end
+            end
+          end
+        end
+        results
+      end
 
       def add(name, object)
         self[name] = object

data/lib/leap_cli/config/secrets.rb

@@ -13,6 +13,11 @@ module LeapCli; module Config
       @discovered_keys = {}
     end
 
+     # we can't use fetch() or get(), since those already have special meanings
+     def retrieve(key, environment=nil)
+       self.fetch(environment||'default', {})[key.to_s]
+     end
+
     def set(key, value, environment=nil)
       environment ||= 'default'
       key = key.to_s
@@ -23,22 +28,29 @@ module LeapCli; module Config
     end
 
     #
-    # if only_discovered_keys is true, then we will only export
-    # those secrets that have been discovered and the prior ones will be cleaned out.
+    # if clean is true, then only secrets that have been discovered
+    # during this run will be exported.
+    #
+    # if environment is also pinned, then we will clean those secrets
+    # just for that environment.
     #
-    # this should only be triggered when all nodes have been processed, otherwise
-    # secrets that are actually in use will get mistakenly removed.
+    # the clean argument should only be used when all nodes have
+    # been processed, otherwise secrets that are actually in use will
+    # get mistakenly removed.
     #
-    def dump_json(only_discovered_keys=false)
-      if only_discovered_keys
+    def dump_json(clean=false)
+      pinned_env = LeapCli.leapfile.environment
+      if clean
         self.each_key do |environment|
-          self[environment].each_key do |key|
-            unless @discovered_keys[environment] && @discovered_keys[environment][key]
-              self[environment].delete(key)
+          if pinned_env.nil? || pinned_env == environment
+            self[environment].each_key do |key|
+              unless @discovered_keys[environment] && @discovered_keys[environment][key]
+                self[environment].delete(key)
+              end
+            end
+            if self[environment].empty?
+              self.delete(environment)
             end
-          end
-          if self[environment].empty?
-            self.delete(environment)
           end
         end
       end

data/lib/leap_cli/config/tag.rb

@@ -13,6 +13,13 @@ module LeapCli; module Config
       super(manager)
       @node_list = Config::ObjectList.new
     end
+
+    # don't copy the node list pointer when this object is dup'ed.
+    def initialize_copy(orig)
+      super
+      @node_list = Config::ObjectList.new
+    end
+
   end
 
 end; end