cisco_node_utils 1.1.0 → 1.2.0

data/lib/cisco_node_utils/cmd_ref/vrf.yaml

@@ -0,0 +1,88 @@
+# vrf.yaml
+---
+_template:
+  config_get: "show running all | section '^vrf context'"
+  config_get_token: '/^vrf context <vrf>$/'
+  config_get_token_append:
+    - '/^address-family <afi> <safi>$/'
+  config_set: 'vrf context <vrf>'
+  config_set_append:
+    - 'address-family <afi> <safi>'
+
+address_family:
+  config_set: ['vrf context <vrf>', '<state> address-family <afi> <safi>']
+
+all_vrf_afs:
+  multiple: true
+  config_get_token_append: '/^address-family (\S+) (\S+)$/'
+
+all_vrfs:
+  multiple: true
+  config_get_token: '/^vrf context (.*)/'
+
+create:
+  config_set:  'vrf context <vrf>'
+
+description:
+  config_get_token_append: '/^description (.*)/'
+  config_set_append: '<state> description <desc>'
+  kind: string
+  default_value: ''
+
+destroy:
+  config_set: 'no vrf context <vrf>'
+
+route_distinguisher:
+  config_get_token_append: '/^rd (\S+)$/'
+  config_set_append: '<state> rd <rd>'
+  default_value: ''
+
+route_target_both_auto:
+  kind: boolean
+  config_get_token_append: '/^route-target both auto$/'
+  config_set_append: '<state> route-target both auto'
+  default_value: false
+
+route_target_both_auto_evpn:
+  kind: boolean
+  config_get_token_append: '/^route-target both auto evpn$/'
+  config_set_append: '<state> route-target both auto evpn'
+  default_value: false
+
+route_target_export:
+  multiple: true
+  config_get_token_append: '/^route-target export (\S+)$/'
+  config_set_append: '<state> route-target export <community>'
+  default_value: []
+
+route_target_export_evpn:
+  multiple: true
+  config_get_token_append: '/^route-target export (\S+) evpn$/'
+  config_set_append: '<state> route-target export <community> evpn'
+  default_value: []
+
+route_target_import:
+  multiple: true
+  config_get_token_append: '/^route-target import (\S+)$/'
+  config_set_append: '<state> route-target import <community>'
+  default_value: []
+
+route_target_import_evpn:
+  multiple: true
+  config_get_token_append: '/^route-target import (\S+) evpn$/'
+  config_set_append: '<state> route-target import <community> evpn'
+  default_value: []
+
+shutdown:
+  kind: boolean
+  config_get_token_append: '/^shutdown$/'
+  config_set_append: '<state> shutdown'
+  default_value: false
+
+vni: # TBD Should this move to the vni provider as vrf_vni?
+  # MT-lite only
+  /N9/:
+    kind: int
+    config_get_token_append: '/^vni (\d+)$/'
+    config_set_append: '<state> vni <id>'
+    default_value: false

data/lib/cisco_node_utils/cmd_ref/vtp.yaml

@@ -0,0 +1,38 @@
+# vtp
+---
+domain:
+  config_get: "show vtp status"
+  config_get_token: "domain_name"
+  config_set: "vtp domain %s"
+
+feature:
+  kind: boolean
+  config_get: "show running vtp"
+  config_get_token: '/^feature vtp$/'
+  config_set: "%s feature vtp"
+  default_value: false
+
+filename:
+  config_get: "show running vtp"
+  config_get_token: '/vtp file (\S+)/'
+  config_set: "%s vtp file %s"
+  default_value: "bootflash:/vlan.dat"
+
+password:
+  config_get: "show vtp password"
+  config_get_token: "passwd"
+  config_set: "%s vtp password %s"
+  default_value: ""
+
+version:
+  kind: int
+  config_get: "show vtp status"
+  config_get_token: '/VTP\s+version\s+running\s+:\s+(\d+)/'
+  config_set: "vtp version %s"
+  default_value: 1
+  /N7/:
+    test_config_result:
+      3: 3
+  else:
+    test_config_result:
+      3: "Cisco::CliError"

data/lib/cisco_node_utils/cmd_ref/vxlan_vtep.yaml

@@ -0,0 +1,60 @@
+# vxlan_vtep
+---
+_exclude: [/N(3|5|6)/]
+
+_template:
+  config_get: "show running-config interface all | section 'interface nve'"
+  config_get_token: '/^interface <name>$/i'
+  config_set: 'interface <name>'
+
+all_interfaces:
+  multiple:
+  config_get_token: '/^interface (.*)$/'
+
+host_reachability:
+  /N(7|9)/:
+    config_get_token_append: '/^host-reachability protocol (\S+)/'
+    config_set_append: '<state> host-reachability protocol <proto>'
+    default_value: 'flood'
+
+mt_full_support:
+  # This is only used for determining support for Multi-Tenancy Full
+  kind: boolean
+  /N(7)/:
+    default_only: true
+  else:
+    # this feature is always off on these platforms and cannot be changed
+    default_only: false
+
+mt_lite_support:
+  # This is only used for determining support for Multi-Tenancy Lite
+  kind: boolean
+  /N9/:
+    default_only: true
+  else:
+    # this feature is always off on these platforms and cannot be changed
+    default_only: false
+
+shutdown:
+  kind: boolean
+  config_get_token_append: '/^no shutdown$/'
+  config_set_append: '<state> shutdown'
+  default_value: true
+
+source_intf:
+  config_get_token_append: '/^source\-interface (\S+)$/'
+  config_set_append: '<state> source-interface <lpbk_intf>'
+  default_value: ''
+
+vni:
+  config_set_append: '<state> member vni <vni> <assoc_vrf>'
+
+vni_with_vrf:
+  kind: boolean
+  config_get_token_append: '/^member vni <vni> associate-vrf$/'
+  default_value: false
+
+vni_without_vrf:
+  kind: boolean
+  config_get_token_append: '/^member vni <vni>$/'
+  default_value: false

data/lib/cisco_node_utils/cmd_ref/vxlan_vtep_vni.yaml

@@ -0,0 +1,39 @@
+# vxlan_vtep_vni
+---
+_exclude: [/N(3|5|6)/]
+
+_template:
+  config_get: "show running-config interface all | section 'interface nve'"
+  config_get_token: '/^interface <name>$/i'
+  config_get_token_append:
+    - '/^member vni <vni> ?(associate-vrf)?$/'
+  config_set: 'interface <name>'
+  config_set_append:
+    - 'member vni <vni> <assoc_vrf>'
+
+all_vnis:
+  multiple:
+  config_get_token_append: '/^member vni (\d+|\d+-\d+) ?(associate-vrf)?$/'
+
+ingress_replication:
+  kind: string
+  config_get_token_append: '/^ingress-replication protocol (\S+)$/'
+  config_set_append: '<state> ingress-replication protocol <protocol>'
+  default_value: ''
+
+multicast_group:
+  config_get_token_append: '/^mcast-group (\S+)\s?(\S+)?$/'
+  config_set_append: '<state> mcast-group <ip_start> <ip_end>'
+  default_value: ''
+
+peer_list:
+  multiple:
+  config_get_token_append: ['/^ingress-replication protocol static$/', '/^peer-ip (\S+)$/']
+  config_set_append: ['ingress-replication protocol static', '<state> peer-ip <peer>']
+  default_value: []
+
+suppress_arp:
+  kind: boolean
+  config_get_token_append: '/^suppress-arp$/'
+  config_set_append: '<state> suppress-arp'
+  default_value: false

data/lib/cisco_node_utils/cmd_ref/yum.yaml

@@ -0,0 +1,13 @@
+# yum
+---
+install:
+  config_set: "install add %s %s activate"
+
+query:
+  multiple: true
+  config_get: "show install packages"
+  # pass in the pkg name, retrieve version
+  config_get_token: '/^%s\S*\s+(\S+)\s+(?:installed|@\S+)/'
+
+remove:
+  config_set: "install deactivate %s"

data/lib/cisco_node_utils/command_reference.rb

@@ -1,6 +1,6 @@
 # CommandReference module for testing.
 #
-# Copyright (c) 2014-2015 Cisco and/or its affiliates.
+# Copyright (c) 2014-2016 Cisco and/or its affiliates.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,81 +16,154 @@
 
 require 'yaml'
 
-module CommandReference
-  # Helper class to match product id with reference files.
-  class CommandPlatformFile
-    attr_reader :regex, :file
-
-    def initialize(match_expression, reference_file)
-      self.regex = match_expression
-      self.file = reference_file
-    end
-
-    def regex=(expression)
-      if expression.is_a? Regexp
-        @regex = expression
-      else
-        fail ArgumentError
-      end
-    end
-
-    def file=(file)
-      if file.is_a? String
-        @file = file
-      else
-        fail ArgumentError
-      end
-    end
-
-    def match(product)
-      @regex.match(product)
-    end
-  end
-
+module Cisco
   # Control a reference for an attribute.
   class CmdRef
-    attr_reader :feature
-    attr_reader :name
-    attr_reader :sources
-    attr_reader :hash
+    attr_reader :feature, :name, :hash
+    attr_reader :auto_default, :multiple, :kind, :default_only
+    alias_method :auto_default?, :auto_default
+    alias_method :default_only?, :default_only
+    alias_method :multiple?, :multiple
+
+    KEYS = %w(default_value default_only
+              config_set config_set_append
+              config_get config_get_token config_get_token_append
+              auto_default multiple kind
+              test_config_get test_config_get_regex test_config_result)
+
+    def self.keys
+      KEYS
+    end
 
-    # rubocop:disable Style/ClassVars
-    @@keys = %w(default_value
-                config_set config_set_append
-                config_get config_get_token config_get_token_append
-                test_config_get test_config_get_regex test_config_result)
-    # rubocop:enable Style/ClassVars
+    KINDS = %w(boolean int string)
 
-    def initialize(feature, name, ref, source)
-      fail ArgumentError, "'#{ref}' is not a hash." unless ref.is_a? Hash
+    # Construct a CmdRef describing the given (feature, name) pair.
+    # Param "values" is a hash with keys as described in KEYS.
+    # Param "file" is for debugging purposes only.
+    def initialize(feature, name, values, file)
+      fail ArgumentError, "'#{values}' is not a hash." unless values.is_a? Hash
 
       @feature = feature
       @name = name
       @hash = {}
+      @auto_default = true
+      @default_only = false
+      @multiple = false
+      @kind = nil
 
-      @sources = []
-      merge(ref, source)
-    end
-
-    # Overwrite values from more specific references.
-    def merge(values, file)
       values.each do |key, value|
-        unless @@keys.include?(key)
+        unless KEYS.include?(key)
           fail "Unrecognized key #{key} for #{feature}, #{name} in #{file}"
         end
-        if value.nil?
-          # Some attributes can store an explicit nil.
-          # Others treat this as unset (allowing a platform to override common).
-          if key == 'default_value'
-            @hash[key] = value
-          else
-            @hash.delete(key)
-          end
+        if key == 'config_get_token' || key == 'config_set'
+          # For simplicity, these are ALWAYS arrays
+          value = [value] unless value.is_a?(Array)
+          define_getter(key, value)
+          # We intentionally do this *after* the define_getter() call
+          @hash[key] = preprocess_value(value)
+        elsif key == 'auto_default'
+          @auto_default = value ? true : false
+        elsif key == 'default_only'
+          @default_only = true
+          # default_value overrides default_only
+          @hash['default_value'] ||= preprocess_value(value)
+        elsif key == 'multiple'
+          @multiple = boolean_default_true(value)
+        elsif key == 'kind'
+          fail "Unknown 'kind': '#{value}'" unless KINDS.include?(value)
+          @kind = value.to_sym
         else
-          @hash[key] = value
+          # default_value overrides default_only
+          @default_only = false if key == 'default_value'
+          @hash[key] = preprocess_value(value)
+        end
+      end
+
+      if @default_only # rubocop:disable Style/GuardClause
+        %w(config_get_token config_set).each do |key|
+          instance_eval "undef #{key}" if @hash.key?(key)
+        end
+        @hash.delete_if { |key, _| key != 'default_value' }
+      end
+    end
+
+    # Property with an implicit value of 'true' if no value is given
+    def boolean_default_true(value)
+      value.nil? || value
+    end
+
+    # Create a getter method for the given key.
+    # This getter method will automatically handle wildcard arguments.
+    def define_getter(key, value)
+      return unless value.is_a?(Array)
+      if value.any? { |item| item.is_a?(String) && /<\S+>/ =~ item }
+        # Key-value substitution
+        define_singleton_method(key.to_sym, key_substitutor(key, value))
+      elsif value.any? { |item| item.is_a?(String) && /%/ =~ item }
+        # printf-style substitution
+        define_singleton_method(key.to_sym, printf_substitutor(key, value))
+      else
+        # simple static token(s)
+        value = preprocess_value(value)
+        define_singleton_method key.to_sym, -> { value }
+      end
+    end
+
+    # curried function to define a getter method body that performs key-value
+    # substitution
+    def key_substitutor(config_key, value)
+      lambda do |**args|
+        result = []
+        value.each do |line|
+          replace = line.scan(/<(\S+)>/).flatten.map(&:to_sym)
+          replace.each do |item|
+            line = line.sub("<#{item}>", args[item].to_s) if args.key?(item)
+          end
+          result.push(line) unless /<\S+>/.match(line)
+        end
+        if result.empty?
+          fail ArgumentError,
+               "Arguments given to #{config_key} yield empty result"
+        end
+        preprocess_value(result)
+      end
+    end
+
+    # curried function to define a getter method body that performs
+    # printf-style substitution
+    def printf_substitutor(config_key, value)
+      arg_c = value.join.scan(/%/).length
+      lambda do |*args|
+        unless args.length == arg_c
+          fail ArgumentError,
+               "Given #{args.length} args, but #{config_key} requires #{arg_c}"
+        end
+        # Fill in the parameters
+        result = value.map do |line|
+          sprintf(line, *args.shift(line.scan(/%/).length))
         end
+        preprocess_value(result)
       end
-      @sources << file
+    end
+
+    # Helper method.
+    # Converts a regexp-like string (or array thereof) into a proper
+    # Regexp object (or array thereof)
+    def preprocess_value(value)
+      if value.is_a?(Array)
+        # Recurse!
+        return value.map { |item| preprocess_value(item) }
+      elsif value.is_a?(String)
+        # Some 'Strings' in YAML are actually intended to be regexps
+        if value[0] == '/' && value[-1] == '/'
+          # '/foo/' => %r{foo}
+          return Regexp.new(value[1..-2])
+        elsif value[0] == '/' && value[-2..-1] == '/i'
+          # '/foo/i' => %r{foo}i
+          return Regexp.new(value[1..-3], Regexp::IGNORECASE)
+        end
+      end
+      value
     end
 
     def convert_to_constant(value)
@@ -104,7 +177,11 @@ module CommandReference
       # to a Constant.
       if value.is_a?(String) && !value.empty?
         if value[0].chr == value[0].chr.upcase
-          value = Object.const_get(value) if Object.const_defined?(value)
+          begin
+            value = Object.const_get(value) if Object.const_defined?(value)
+          rescue NameError
+            debug("'#{value}' is not a constant")
+          end
         end
       end
       value
@@ -116,13 +193,25 @@ module CommandReference
     end
 
     def method_missing(method_name, *args, &block)
-      super(method_name, *args, &block) unless @@keys.include?(method_name.to_s)
-      method_name = method_name.to_s
-      unless @hash.include?(method_name)
-        fail IndexError, "No #{method_name} defined for #{@feature}, #{@name}"
+      if KEYS.include?(method_name.to_s)
+        # ref.foo -> return @hash[foo] or fail IndexError
+        method_name = method_name.to_s
+        unless @hash.include?(method_name)
+          if @default_only
+            fail UnsupportedError.new(@feature, @name, method_name)
+          end
+          fail IndexError, "No #{method_name} defined for #{@feature}, #{@name}"
+        end
+        # puts("get #{method_name}: '#{@hash[method_name]}'")
+        @hash[method_name]
+      elsif method_name.to_s[-1] == '?' && \
+            KEYS.include?(method_name.to_s[0..-2])
+        # ref.foo? -> return true if @hash[foo], else false
+        method_name = method_name.to_s[0..-2]
+        @hash.include?(method_name)
+      else
+        super(method_name, *args, &block)
       end
-      # puts("get #{method_name}: '#{@hash[method_name]}'")
-      @hash[method_name]
     end
 
     # Print useful debugging information about the object.
@@ -132,11 +221,29 @@ module CommandReference
       @hash.each { |key, value| str << "  #{key}: #{value}\n" }
       str
     end
+  end
 
-    # Check that all necessary values have been populated.
-    def valid?
-      return false unless @feature && @name
-      true
+  # Exception class raised when a particular feature/attribute
+  # is explicitly excluded on the given node.
+  class UnsupportedError < RuntimeError
+    def initialize(feature, name, oper=nil, msg=nil)
+      @feature = feature
+      @name = name
+      @oper = oper
+      message = "Feature '#{feature}'"
+      message += ", attribute '#{name}'" unless name.nil?
+      message += ", operation '#{oper}'" unless oper.nil?
+      message += ' is unsupported on this node'
+      message += ": #{msg}" unless msg.nil?
+      super(message)
+    end
+  end
+
+  # Placeholder for known but explicitly excluded entry
+  # For these, we have an implied default_only value of nil.
+  class UnsupportedCmdRef < CmdRef
+    def initialize(feature, name, file)
+      super(feature, name, { 'default_only' => nil }, file)
     end
   end
 
@@ -150,46 +257,26 @@ module CommandReference
       @@debug = value # rubocop:disable Style/ClassVars
     end
 
-    attr_reader :files, :product_id
+    attr_reader :cli, :files, :platform, :product_id
 
     # Constructor.
-    # Normal usage is to pass product_id only, in which case all usual YAML
+    # Normal usage is to pass product, platform, cli, in which case usual YAML
     # files will be located then the list will be filtered down to only those
-    # matching the given product_id.
+    # matching the given settings.
     # For testing purposes (only!) you can pass an explicit list of files to
     # load instead. This list will NOT be filtered further by product_id.
-    def initialize(product_id, files=nil)
-      @product_id = product_id
+    def initialize(product:  nil,
+                   platform: nil,
+                   cli:      false,
+                   files:    nil)
+      @product_id = product
+      @platform = platform
+      @cli = cli
       @hash = {}
       if files
         @files = files
       else
-        @files = []
-        # Hashes are unordered in Ruby 1.8.7, so instead, we use an array
-        # of objects.
-        # rubocop:disable Metrics/LineLength
-        platforms = [
-          CommandPlatformFile.new(//,
-                                  File.join(File.dirname(__FILE__),
-                                            'command_reference_common.yaml')),
-          CommandPlatformFile.new(//,
-                                  File.join(File.dirname(__FILE__),
-                                            'command_reference_common_bgp.yaml')),
-          CommandPlatformFile.new(/N9K/,
-                                  File.join(File.dirname(__FILE__),
-                                            'command_reference_n9k.yaml')),
-          CommandPlatformFile.new(/N7K/,
-                                  File.join(File.dirname(__FILE__),
-                                            'command_reference_n7k.yaml')),
-          CommandPlatformFile.new(/C3064/,
-                                  File.join(File.dirname(__FILE__),
-                                            'command_reference_n3064.yaml')),
-        ]
-        # rubocop:enable Metrics/LineLength
-        # Build array
-        platforms.each do |reference|
-          @files << reference.file if reference.match(@product_id)
-        end
+        @files = Dir.glob(__dir__ + '/cmd_ref/*.yaml')
       end
 
       build_cmd_ref
@@ -202,48 +289,43 @@ module CommandReference
       debug "Product: #{@product_id}"
       debug "Files being used: #{@files.join(', ')}"
 
-      reference_yaml = {}
-
       @files.each do |file|
-        debug "Processing file '#{file}'"
-        reference_yaml = load_yaml(file)
-
-        reference_yaml.each do |feature, names|
-          if names.nil? || names.empty?
-            fail "No names under feature #{feature}: #{names}"
-          elsif @hash[feature].nil?
-            @hash[feature] = {}
+        feature = File.basename(file).split('.')[0]
+        debug "Processing file '#{file}' as feature '#{feature}'"
+        feature_hash = load_yaml(file)
+        if feature_hash.empty?
+          debug "Feature #{feature} is empty"
+          next
+        end
+        feature_hash = filter_hash(feature_hash)
+        if feature_hash.empty?
+          debug "Feature #{feature} is excluded"
+          @hash[feature] = UnsupportedCmdRef.new(feature, nil, file)
+          next
+        end
+
+        base_hash = {}
+        if feature_hash.key?('_template')
+          base_hash = CommandReference.hash_merge(feature_hash['_template'])
+        end
+
+        feature_hash.each do |name, value|
+          fail "No entries under '#{name}' in '#{file}'" if value.nil?
+          @hash[feature] ||= {}
+          if value.empty?
+            @hash[feature][name] = UnsupportedCmdRef.new(feature, name, file)
           else
-            debug "  Merging feature '#{feature}' retrieved from '#{file}'."
-          end
-          names.each do |name, values|
-            debug "  Processing feature '#{feature}' name '#{name}'"
-            if @hash[feature][name].nil?
-              begin
-                @hash[feature][name] = CmdRef.new(feature, name, values, file)
-              rescue ArgumentError => e
-                raise "Invalid data for '#{feature}', '#{name}': #{e}"
-              end
-            else
-              debug "  Merging feature '#{feature}' name '#{name}' " \
-                    "from '#{file}'."
-              @hash[feature][name].merge(values, file)
-            end
+            values = CommandReference.hash_merge(value, base_hash.clone)
+            @hash[feature][name] = CmdRef.new(feature, name, values, file)
           end
         end
       end
-
-      fail 'Missing values in CommandReference.' unless valid?
     end
 
     # Get the command reference
     def lookup(feature, name)
-      begin
-        value = @hash[feature][name]
-      rescue NoMethodError
-        # happens if @hash[feature] doesn't exist
-        value = nil
-      end
+      value = @hash[feature]
+      value = value[name] if value.is_a? Hash
       fail IndexError, "No CmdRef defined for #{feature}, #{name}" if value.nil?
       value
     end
@@ -257,24 +339,143 @@ module CommandReference
       puts "DEBUG: #{text}" if @@debug
     end
 
+    KNOWN_FILTERS = %w(cli_nexus)
+
+    def self.key_match(key, platform, product_id, cli)
+      if key[0] == '/' && key[-1] == '/'
+        # It's a product-id regexp. Does it match our given product_id?
+        return Regexp.new(key[1..-2]) =~ product_id ? true : false
+      elsif KNOWN_FILTERS.include?(key)
+        return false if key.match(/cli/) && !cli
+        return Regexp.new(platform.to_s) =~ key ? true : false
+      else
+        return :unknown
+      end
+    end
+
+    # Helper method
+    # Given a Hash of command reference data as read from YAML, does:
+    # - Filter out any API-specific data not applicable to this API
+    # - Filter any platform-specific data not applicable to this product_id
+    # Returns the filtered hash (possibly empty)
+    def self.filter_hash(hash,
+                         platform:           nil,
+                         product_id:         nil,
+                         cli:                false,
+                         allow_unknown_keys: true)
+      result = {}
+
+      exclude = hash.delete('_exclude') || []
+      exclude.each do |value|
+        if key_match(value, platform, product_id, cli) == true
+          debug 'Exclude this product (#{product_id}, #{value})'
+          return result
+        end
+      end
+
+      # to_inspect: sub-keys we want to recurse into
+      to_inspect = []
+      # regexp_match: did we find a product_id regexp that matches?
+      regexp_match = false
+
+      hash.each do |key, value|
+        if CmdRef.keys.include?(key)
+          result[key] = value
+        elsif key != 'else'
+          match = key_match(key, platform, product_id, cli)
+          next if match == false
+          if match == :unknown
+            fail "Unrecognized key '#{key}'" unless allow_unknown_keys
+          end
+          regexp_match = true if match == true
+          to_inspect << key
+        end
+      end
+      # If we didn't find any platform regexp match,
+      # and an 'else' sub-hash is provided, descend into 'else'
+      to_inspect << 'else' if hash.key?('else') && !regexp_match
+      # Recurse! Sub-hashes can override the base hash
+      to_inspect.each do |key|
+        unless hash[key].is_a?(Hash)
+          result[key] = hash[key]
+          next
+        end
+        begin
+          result[key] = filter_hash(hash[key],
+                                    platform:           platform,
+                                    product_id:         product_id,
+                                    cli:                cli,
+                                    allow_unknown_keys: false)
+        rescue RuntimeError => e
+          raise "[#{key}]: #{e}"
+        end
+      end
+      result
+    end
+
+    def filter_hash(input_hash)
+      CommandReference.filter_hash(input_hash,
+                                   platform:   platform,
+                                   product_id: product_id,
+                                   cli:        cli)
+    end
+
+    # Helper method
+    # Given a suitably filtered Hash of command reference data, does:
+    # - Inherit data from the given base_hash (if any) and extend/override it
+    #   with the given input data.
+    # - Append 'config_set_append' data to any existing 'config_set' data
+    # - Append 'config_get_token_append' data to 'config_get_token', ditto
+    def self.hash_merge(input_hash, base_hash=nil)
+      result = base_hash
+      result ||= {}
+      # to_inspect: sub-hashes we want to recurse into
+      to_inspect = []
+
+      input_hash.each do |key, value|
+        if CmdRef.keys.include?(key)
+          if key == 'config_set_append'
+            result['config_set'] = value_append(result['config_set'], value)
+          elsif key == 'config_get_token_append'
+            result['config_get_token'] = value_append(
+              result['config_get_token'], value)
+          else
+            result[key] = value
+          end
+        elsif value.is_a?(Hash)
+          to_inspect << value
+        else
+          fail "Unexpected non-hash data: #{value}"
+        end
+      end
+      # Recurse! Sub-hashes can override the base hash
+      to_inspect.each do |hash|
+        result = hash_merge(hash, result)
+      end
+      result
+    end
+
+    # Helper method.
+    # Combines the two given values (either or both of which may be arrays)
+    # into a single combined array
+    # value_append('foo', 'bar') ==> ['foo', 'bar']
+    # value_append('foo', ['bar', 'baz']) ==> ['foo', 'bar', 'baz']
+    def self.value_append(base_value, new_value)
+      base_value = [base_value] unless base_value.is_a?(Array)
+      new_value = [new_value] unless new_value.is_a?(Array)
+      base_value + new_value
+    end
+
     def mapping?(node)
-      # Need to handle both Syck::Map and Psych::Nodes::Mapping
       node.class.ancestors.any? { |name| /Map/ =~ name.to_s }
     end
     private :mapping?
 
     def get_keys_values_from_map(node)
-      if node.class.ancestors.any? { |name| /Psych/ =~ name.to_s }
-        # A Psych::Node::Mapping instance has an Array of children in
-        # the format [key1, val1, key2, val2]
-        key_children = node.children.select.each_with_index { |_, i| i.even? }
-        val_children = node.children.select.each_with_index { |_, i| i.odd? }
-      else
-        # Syck::Map nodes have a .children method but it doesn't work :-(
-        # Instead we access the node.value which is a hash.
-        key_children = node.value.keys
-        val_children = node.value.values
-      end
+      # A Psych::Node::Mapping instance has an Array of children in
+      # the format [key1, val1, key2, val2]
+      key_children = node.children.select.each_with_index { |_, i| i.even? }
+      val_children = node.children.select.each_with_index { |_, i| i.odd? }
       debug "children of #{node} mapping: #{key_children}, #{val_children}"
       [key_children, val_children]
     end
@@ -291,9 +492,7 @@ module CommandReference
     # @param parents String describing parents of this node, for messages
     def validate_yaml(node, filename, depth=0, parents=nil)
       return unless node && (mapping?(node) || node.children)
-      # Psych wraps everything in a Document instance, while
-      # Syck does not. To keep the "depth" counting consistent,
-      # we need to ignore Documents.
+      # Psych wraps everything in a Document instance, which we ignore.
       unless node.class.ancestors.any? { |name| /Document/ =~ name.to_s }
         depth += 1
       end
@@ -308,7 +507,7 @@ module CommandReference
       end
 
       # For Mappings, we validate more extensively:
-      # 1. no duplicate keys are allowed (Syck/Psych don't catch this)
+      # 1. no duplicate keys are allowed (Psych doesn't catch this)
       # 2. Features must be listed in alphabetical order for maintainability
 
       # Take advantage of our known YAML structure to assign labels by depth
@@ -329,10 +528,6 @@ module CommandReference
         fail msg
       end
 
-=begin
-# Syck does not necessarily preserve ordering of keys in a mapping even during
-# the initial parsing stage. To avoid spurious failures, this is disabled
-# for now. Fixing this may require restructuring our YAML...
       # Enforce alphabetical ordering of features (only).
       # We can extend this later to enforce ordering of names if desired
       # by checking at depth 2 as well.
@@ -340,12 +535,11 @@ module CommandReference
         last_key = nil
         key_arr.each do |key|
           if last_key && key < last_key
-            raise RuntimeError, "features out of order (#{last_key} > #{key})"
+            fail "features out of order in #{filename}: (#{last_key} > #{key})"
           end
           last_key = key
         end
       end
-=end
 
       # Recurse to the children. We get a little fancy here so as to be able
       # to provide more meaningful debug/error messages, such as:
@@ -363,16 +557,13 @@ module CommandReference
     private :validate_yaml
 
     # Read in yaml file.
+    # The expectation is that a file corresponds to a feature
     def load_yaml(yaml_file)
       fail "File #{yaml_file} doesn't exist." unless File.exist?(yaml_file)
       # Parse YAML file into a tree of nodes
       # Psych::SyntaxError doesn't inherit from StandardError in some versions,
       # so we want to explicitly catch it if using Psych.
-      if defined?(::Psych::SyntaxError)
-        rescue_errors = [::StandardError, ::Psych::SyntaxError]
-      else
-        rescue_errors = [::StandardError]
-      end
+      rescue_errors = [::StandardError, ::Psych::SyntaxError]
       yaml_parsed = File.open(yaml_file, 'r') do |f|
         begin
           YAML.parse(f)
@@ -380,30 +571,11 @@ module CommandReference
           raise "unable to parse #{yaml_file}: #{e}"
         end
       end
-      if yaml_parsed
-        # Validate the node tree
-        validate_yaml(yaml_parsed, yaml_file)
-        # If validation passed, convert the node tree to a Ruby Hash.
-        return yaml_parsed.transform
-      else
-        # if yaml_file is empty, YAML.parse() returns false.
-        # Change this to an empty hash.
-        return {}
-      end
-    end
-
-    # Check that all resources were pulled in correctly.
-    def valid?
-      complete_status = true
-      @hash.each_value do |names|
-        names.each_value do |ref|
-          status = ref.valid?
-          debug('Reference does not contain all supported values:' \
-                "\n#{ref}") unless status
-          complete_status = (status && complete_status)
-        end
-      end
-      complete_status
+      return {} unless yaml_parsed
+      # Validate the node tree
+      validate_yaml(yaml_parsed, yaml_file)
+      # If validation passed, convert the node tree to a Ruby Hash.
+      yaml_parsed.transform
     end
 
     def to_s