leap_cli 1.2.5

data/lib/leap_cli/config/object.rb

@@ -0,0 +1,336 @@
+require 'erb'
+require 'json/pure'  # pure ruby implementation is required for our sorted trick to work.
+
+if $ruby_version < [1,9]
+  $KCODE = 'UTF8'
+end
+require 'ya2yaml' # pure ruby yaml
+
+require 'leap_cli/config/macros'
+
+module LeapCli
+  module Config
+    #
+    # This class represents the configuration for a single node, service, or tag.
+    # Also, all the nested hashes are also of this type.
+    #
+    # It is called 'object' because it corresponds to an Object in JSON.
+    #
+    class Object < Hash
+
+      include Config::Macros
+
+      attr_reader :node
+      attr_reader :manager
+      alias :global :manager
+
+      def initialize(manager=nil, node=nil)
+        # keep a global pointer around to the config manager. used a lot in the eval strings and templates
+        # (which are evaluated in the context of Config::Object)
+        @manager = manager
+
+        # an object that is a node as @node equal to self, otherwise all the child objects point back to the top level node.
+        @node = node || self
+      end
+
+      #
+      # We use pure ruby yaml exporter ya2yaml instead of SYCK or PSYCH because it
+      # allows us greater compatibility regardless of installed ruby version and
+      # greater control over how the yaml is exported (sorted keys, in particular).
+      #
+      def dump
+        evaluate
+        ya2yaml(:syck_compatible => true)
+      end
+
+      def dump_json
+        evaluate
+        JSON.sorted_generate(self)
+      end
+
+      def evaluate
+        evaluate_everything
+        late_evaluate_everything
+      end
+
+      ##
+      ## FETCHING VALUES
+      ##
+
+      def [](key)
+        get(key)
+      end
+
+      #
+      # make hash addressable like an object (e.g. obj['name'] available as obj.name)
+      #
+      def method_missing(method, *args, &block)
+        get!(method)
+      end
+
+      def get(key)
+        begin
+          get!(key)
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      # override behavior of #default() from Hash
+      def default
+        get!('default')
+      end
+
+      #
+      # Like a normal Hash#[], except:
+      #
+      # (1) lazily eval dynamic values when we encounter them. (i.e. strings that start with "= ")
+      #
+      # (2) support for nested references in a single string (e.g. ['a.b'] is the same as ['a']['b'])
+      #     the dot path is always absolute, starting at the top-most object.
+      #
+      def get!(key)
+        key = key.to_s
+        if key =~ /\./
+          # for keys with with '.' in them, we start from the root object (@node).
+          keys = key.split('.')
+          value = @node.get!(keys.first)
+          if value.is_a? Config::Object
+            value.get!(keys[1..-1].join('.'))
+          else
+            value
+          end
+        elsif self.has_key?(key)
+          fetch_value(key)
+        else
+          raise NoMethodError.new(key, "No method '#{key}' for #{self.class}")
+        end
+      end
+
+      ##
+      ## COPYING
+      ##
+
+      #
+      # A deep (recursive) merge with another Config::Object.
+      #
+      # If prefer_self is set to true, the value from self will be picked when there is a conflict
+      # that cannot be merged.
+      #
+      # Merging rules:
+      #
+      # - If a value is a hash, we recursively merge it.
+      # - If the value is simple, like a string, the new one overwrites the value.
+      # - If the value is an array:
+      #   - If both old and new values are arrays, the new one replaces the old.
+      #   - If one of the values is simple but the other is an array, the simple is added to the array.
+      #
+      def deep_merge!(object, prefer_self=false)
+        object.each do |key,new_value|
+          old_value = self.fetch key, nil
+
+          # clean up boolean
+          new_value = true  if new_value == "true"
+          new_value = false if new_value == "false"
+          old_value = true  if old_value == "true"
+          old_value = false if old_value == "false"
+
+          # merge hashes
+          if old_value.is_a?(Hash) || new_value.is_a?(Hash)
+            value = Config::Object.new(@manager, @node)
+            old_value.is_a?(Hash) ? value.deep_merge!(old_value) : (value[key] = old_value if !old_value.nil?)
+            new_value.is_a?(Hash) ? value.deep_merge!(new_value, prefer_self) : (value[key] = new_value if !new_value.nil?)
+
+          # merge nil
+          elsif new_value.nil?
+            value = old_value
+          elsif old_value.nil?
+            value = new_value
+
+          # merge arrays when one value is not an array
+          elsif old_value.is_a?(Array) && !new_value.is_a?(Array)
+            (value = (old_value.dup << new_value).compact.uniq).delete('REQUIRED')
+          elsif new_value.is_a?(Array) && !old_value.is_a?(Array)
+            (value = (new_value.dup << old_value).compact.uniq).delete('REQUIRED')
+
+          # catch errors
+          elsif type_mismatch?(old_value, new_value)
+            raise 'Type mismatch. Cannot merge %s (%s) with %s (%s). Key is "%s", name is "%s".' % [
+              old_value.inspect, old_value.class,
+              new_value.inspect, new_value.class,
+              key, self.class
+            ]
+
+          # merge strings, numbers, and sometimes arrays
+          else
+            if prefer_self
+              value = old_value
+            else
+              value = new_value
+            end
+          end
+
+          # save value
+          self[key] = value
+        end
+        self
+      end
+
+      #
+      # like a reverse deep merge
+      # (self takes precedence)
+      #
+      def inherit_from!(object)
+        self.deep_merge!(object, true)
+      end
+
+      #
+      # Make a copy of ourselves, except only including the specified keys.
+      #
+      # Also, the result is flattened to a single hash, so a key of 'a.b' becomes 'a_b'
+      #
+      def pick(*keys)
+        keys.map(&:to_s).inject(self.class.new(@manager)) do |hsh, key|
+          value = self.get(key)
+          if !value.nil?
+            hsh[key.gsub('.','_')] = value
+          end
+          hsh
+        end
+      end
+
+      protected
+
+      #
+      # walks the object tree, eval'ing all the attributes that are dynamic ruby (e.g. value starts with '= ')
+      #
+      def evaluate_everything
+        keys.each do |key|
+          obj = fetch_value(key)
+          if is_required_value_not_set?(obj)
+            Util::log 0, :warning, "required key \"#{key}\" is not set in node \"#{node.name}\"."
+          elsif obj.is_a? Config::Object
+            obj.evaluate_everything
+          end
+        end
+      end
+
+      #
+      # some keys need to be evaluated 'late', after all the other keys have been evaluated.
+      #
+      def late_evaluate_everything
+        if @late_eval_list
+          @late_eval_list.each do |key, value|
+            self[key] = evaluate_now(key, value)
+            if is_required_value_not_set?(self[key])
+              Util::log 0, :warning, "required key \"#{key}\" is not set in node \"#{node.name}\"."
+            end
+          end
+        end
+        values.each do |obj|
+          if obj.is_a? Config::Object
+            obj.late_evaluate_everything
+          end
+        end
+      end
+
+      private
+
+      #
+      # fetches the value for the key, evaluating the value as ruby if it begins with '='
+      #
+      def fetch_value(key)
+        value = fetch(key, nil)
+        if value.is_a?(String) && value =~ /^=/
+          if value =~ /^=> (.*)$/
+            value = evaluate_later(key, $1)
+          elsif value =~ /^= (.*)$/
+            value = evaluate_now(key, $1)
+          end
+          self[key] = value
+        end
+        return value
+      end
+
+      def evaluate_later(key, value)
+        @late_eval_list ||= []
+        @late_eval_list << [key, value]
+        '<evaluate later>'
+      end
+
+      def evaluate_now(key, value)
+        result = nil
+        if LeapCli.log_level >= 2
+          result = @node.instance_eval(value)
+        else
+          begin
+            result = @node.instance_eval(value)
+          rescue SystemStackError => exc
+            Util::log 0, :error, "while evaluating node '#{@node.name}'"
+            Util::log 0, "offending key: #{key}", :indent => 1
+            Util::log 0, "offending string: #{value}", :indent => 1
+            Util::log 0, "STACK OVERFLOW, BAILING OUT. There must be an eval loop of death (variables with circular dependencies).", :indent => 1
+            raise SystemExit.new(1)
+          rescue FileMissing => exc
+            Util::bail! do
+              if exc.options[:missing]
+                Util::log :missing, exc.options[:missing].gsub('$node', @node.name)
+              else
+                Util::log :error, "while evaluating node '#{@node.name}'"
+                Util::log "offending key: #{key}", :indent => 1
+                Util::log "offending string: #{value}", :indent => 1
+                Util::log "error message: no file '#{exc}'", :indent => 1
+              end
+            end
+          rescue AssertionFailed => exc
+            Util.bail! do
+              Util::log :failed, "assertion while evaluating node '#{@node.name}'"
+              Util::log 'assertion: %s' % exc.assertion, :indent => 1
+              Util::log "offending key: #{key}", :indent => 1
+            end
+          rescue SyntaxError, StandardError => exc
+            Util::bail! do
+              Util::log :error, "while evaluating node '#{@node.name}'"
+              Util::log "offending key: #{key}", :indent => 1
+              Util::log "offending string: #{value}", :indent => 1
+              Util::log "error message: #{exc.inspect}", :indent => 1
+            end
+          end
+        end
+        return result
+      end
+
+      #
+      # when merging, we raise an error if this method returns true for the two values.
+      #
+      def type_mismatch?(old_value, new_value)
+        if old_value.is_a?(Boolean) && new_value.is_a?(Boolean)
+          # note: FalseClass and TrueClass are different classes
+          # so we can't do old_value.class == new_value.class
+          return false
+        elsif old_value.is_a?(String) && old_value =~ /^=/
+          # pass through macros, since we don't know what the type will eventually be.
+          return false
+        elsif new_value.is_a?(String) && new_value =~ /^=/
+          return false
+        elsif old_value.class == new_value.class
+          return false
+        else
+          return true
+        end
+      end
+
+      #
+      # returns true if the value has not been changed and the default is "REQUIRED"
+      #
+      def is_required_value_not_set?(value)
+        if value.is_a? Array
+          value == ["REQUIRED"]
+        else
+          value == "REQUIRED"
+        end
+      end
+
+    end # class
+  end # module
+end # module

data/lib/leap_cli/config/object_list.rb

@@ -0,0 +1,174 @@
+module LeapCli
+  module Config
+    #
+    # A list of Config::Object instances (internally stored as a hash)
+    #
+    class ObjectList < Hash
+
+      def initialize(config=nil)
+        if config
+          self.add(config['name'], config)
+        end
+      end
+
+      #
+      # If the key is a string, the Config::Object it references is returned.
+      #
+      # If the key is a hash, we treat it as a condition and filter all the Config::Objects using the condition.
+      # A new ObjectList is returned.
+      #
+      # Examples:
+      #
+      # nodes['vpn1']
+      #   node named 'vpn1'
+      #
+      # nodes[:public_dns => true]
+      #   all nodes with public dns
+      #
+      # nodes[:services => 'openvpn', :services => 'tor']
+      #   nodes with openvpn OR tor service
+      #
+      # nodes[:services => 'openvpn'][:tags => 'production']
+      #   nodes with openvpn AND are production
+      #
+      def [](key)
+        if key.is_a? Hash
+          results = Config::ObjectList.new
+          key.each do |field, match_value|
+            field = field.is_a?(Symbol) ? field.to_s : field
+            match_value = match_value.is_a?(Symbol) ? match_value.to_s : match_value
+            if match_value.is_a?(String) && match_value =~ /^!/
+              operator = :not_equal
+              match_value = match_value.sub(/^!/, '')
+            else
+              operator = :equal
+            end
+            each do |name, config|
+              value = config[field]
+              if value.is_a? Array
+                if value.include?(match_value)
+                  results[name] = config
+                end
+              else
+                if operator == :equal && value == match_value
+                  results[name] = config
+                elsif operator == :not_equal && value != match_value
+                  results[name] = config
+                end
+              end
+            end
+          end
+          results
+        else
+          super key.to_s
+        end
+      end
+
+      def exclude(node)
+        list = self.dup
+        list.delete(node.name)
+        return list
+      end
+
+      def each_node(&block)
+        self.keys.sort.each do |node_name|
+          yield self[node_name]
+        end
+      end
+
+      # def <<(object)
+      #   if object.is_a? Config::ObjectList
+      #     self.merge!(object)
+      #   elsif object['name']
+      #     self[object['name']] = object
+      #   else
+      #     raise ArgumentError.new('argument must be a Config::Object or a Config::ObjectList')
+      #   end
+      # end
+
+      def add(name, object)
+        self[name] = object
+      end
+
+      #
+      # converts the hash of configs into an array of hashes, with ONLY the specified fields
+      #
+      def fields(*fields)
+        result = []
+        keys.sort.each do |name|
+          result << self[name].pick(*fields)
+        end
+        result
+      end
+
+      #
+      # like fields(), but returns an array of values instead of an array of hashes.
+      #
+      def field(field)
+        field = field.to_s
+        result = []
+        keys.sort.each do |name|
+          result << self[name].get(field)
+        end
+        result
+      end
+
+      #
+      # pick_fields(field1, field2, ...)
+      #
+      # generates a Hash from the object list, but with only the fields that are picked.
+      #
+      # If there are more than one field, then the result is a Hash of Hashes.
+      # If there is just one field, it is a simple map to the value.
+      #
+      # For example:
+      #
+      #   "neighbors" = "= nodes_like_me[:services => :couchdb].pick_fields('domain.full', 'ip_address')"
+      #
+      # generates this:
+      #
+      #   neighbors:
+      #     couch1:
+      #       domain_full: couch1.bitmask.net
+      #       ip_address: "10.5.5.44"
+      #     couch2:
+      #       domain_full: couch2.bitmask.net
+      #       ip_address: "10.5.5.52"
+      #
+      # But this:
+      #
+      #   "neighbors": "= nodes_like_me[:services => :couchdb].pick_fields('domain.full')"
+      #
+      # will generate this:
+      #
+      #   neighbors:
+      #     couch1: couch1.bitmask.net
+      #     couch2: couch2.bitmask.net
+      #
+      def pick_fields(*fields)
+        self.values.inject({}) do |hsh, node|
+          value = self[node.name].pick(*fields)
+          if fields.size == 1
+            value = value.values.first
+          end
+          hsh[node.name] = value
+          hsh
+        end
+      end
+
+      #
+      # applies inherit_from! to all objects.
+      #
+      def inherit_from!(object_list)
+        object_list.each do |name, object|
+          if self[name]
+            self[name].inherit_from!(object)
+          else
+            self[name] = object.dup
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/leap_cli/config/secrets.rb

@@ -0,0 +1,43 @@
+#
+#
+# A class for the secrets.json file
+#
+#
+
+module LeapCli; module Config
+
+  class Secrets < Object
+    attr_reader :node_list
+
+    def initialize(manager=nil)
+      super(manager)
+      @discovered_keys = {}
+    end
+
+    def set(key, value)
+      key = key.to_s
+      @discovered_keys[key] = true
+      self[key] ||= value
+    end
+
+    #
+    # if only_discovered_keys is true, then we will only export
+    # those secrets that have been discovered and the prior ones will be cleaned out.
+    #
+    # this should only be triggered when all nodes have been processed, otherwise
+    # secrets that are actually in use will get mistakenly removed.
+    #
+    #
+    def dump_json(only_discovered_keys=false)
+      if only_discovered_keys
+        self.each_key do |key|
+          unless @discovered_keys[key]
+            self.delete(key)
+          end
+        end
+      end
+      super()
+    end
+  end
+
+end; end

data/lib/leap_cli/config/tag.rb

@@ -0,0 +1,18 @@
+#
+#
+# A class for node services or node tags.
+#
+#
+
+module LeapCli; module Config
+
+  class Tag < Object
+    attr_reader :node_list
+
+    def initialize(manager=nil)
+      super(manager)
+      @node_list = Config::ObjectList.new
+    end
+  end
+
+end; end

data/lib/leap_cli/constants.rb

@@ -0,0 +1,7 @@
+module LeapCli
+
+  PUPPET_DESTINATION = '/srv/leap'
+  INITIALIZED_FILE = "#{PUPPET_DESTINATION}/initialized"
+  DEFAULT_TAGS = ['leap_base','leap_service']
+
+end

data/lib/leap_cli/leapfile.rb

@@ -0,0 +1,97 @@
+#
+# The Leapfile is the bootstrap configuration file for a LEAP provider.
+#
+# It is akin to a Gemfile, Rakefile, or Capfile (e.g. it is a ruby file that gets eval'ed)
+#
+
+module LeapCli
+  def self.leapfile
+    @leapfile ||= Leapfile.new
+  end
+
+  class Leapfile
+    attr_accessor :platform_directory_path
+    attr_accessor :provider_directory_path
+    attr_accessor :custom_vagrant_vm_line
+    attr_accessor :leap_version
+    attr_accessor :log
+    attr_accessor :vagrant_network
+    attr_accessor :platform_branch
+    attr_accessor :allow_production_deploy
+
+    def initialize
+      @vagrant_network = '10.5.5.0/24'
+    end
+
+    def load(search_directory=nil)
+      directory = File.expand_path(find_in_directory_tree('Leapfile', search_directory))
+      if directory == '/'
+        return nil
+      else
+        #
+        # set up paths
+        #
+        @provider_directory_path = directory
+        read_settings(directory + '/Leapfile')
+        read_settings(ENV['HOME'] + '/.leaprc')
+        @platform_directory_path = File.expand_path(@platform_directory_path || '../leap_platform', @provider_directory_path)
+
+        #
+        # load the platform
+        #
+        require "#{@platform_directory_path}/platform.rb"
+        if !Leap::Platform.compatible_with_cli?(LeapCli::VERSION)
+          Util.bail! "This leap command (v#{LeapCli::VERSION}) " +
+                     "is not compatible with the platform #{@platform_directory_path} (v#{Leap::Platform.version}). " +
+                     "You need leap command #{Leap::Platform.compatible_cli.first} to #{Leap::Platform.compatible_cli.last}."
+        end
+        if !Leap::Platform.version_in_range?(LeapCli::COMPATIBLE_PLATFORM_VERSION)
+          Util.bail! "This leap command (v#{LeapCli::VERSION}) " +
+                     "is not compatible with the platform #{@platform_directory_path} (v#{Leap::Platform.version}). " +
+                     "You need platform version #{LeapCli::COMPATIBLE_PLATFORM_VERSION.first} to #{LeapCli::COMPATIBLE_PLATFORM_VERSION.last}."
+        end
+
+        #
+        # set defaults
+        #
+        if @allow_production_deploy.nil?
+          # by default, only allow production deploys from 'master' or if not a git repo
+          @allow_production_deploy = !LeapCli::Util.is_git_directory?(@provider_directory_path) ||
+            LeapCli::Util.current_git_branch(@provider_directory_path) == 'master'
+        end
+        return true
+      end
+    end
+
+    private
+
+    def read_settings(file)
+      if File.exists? file
+        Util::log 2, :read, file
+        instance_eval(File.read(file), file)
+        validate(file)
+      end
+    end
+
+    def find_in_directory_tree(filename, directory_tree=nil)
+      search_dir = directory_tree || Dir.pwd
+      while search_dir != "/"
+        Dir.foreach(search_dir) do |f|
+          return search_dir if f == filename
+        end
+        search_dir = File.dirname(search_dir)
+      end
+      return search_dir
+    end
+
+    PRIVATE_IP_RANGES = /(^127\.0\.0\.1)|(^10\.)|(^172\.1[6-9]\.)|(^172\.2[0-9]\.)|(^172\.3[0-1]\.)|(^192\.168\.)/
+
+    def validate(file)
+      Util::assert! vagrant_network =~ PRIVATE_IP_RANGES do
+        Util::log 0, :error, "in #{file}: vagrant_network is not a local private network"
+      end
+    end
+
+  end
+end
+

data/lib/leap_cli/load_paths.rb

@@ -0,0 +1,15 @@
+#
+# Bundle and rubygems each have their own way of modifying $LOAD_PATH.
+#
+# We want to make sure that the right paths are loaded, including the
+# vendored gems, regardless of how leap is run.
+#
+#
+
+require File.expand_path('../version', __FILE__)
+
+base_leap_dir = File.expand_path('../../..', __FILE__)
+LeapCli::LOAD_PATHS.each do |path|
+  path = File.expand_path(path, base_leap_dir)
+  $LOAD_PATH.unshift(path) unless $LOAD_PATH.include?(path)
+end