arborist 0.0.1.pre20160106113421

data/lib/arborist/monitor_runner.rb

@@ -0,0 +1,217 @@
+# -*- ruby -*-
+#encoding: utf-8
+
+require 'rbczmq'
+require 'loggability'
+
+require 'arborist' unless defined?( Arborist )
+require 'arborist/client'
+
+
+# Undo the useless scoping
+class ZMQ::Loop
+	public_class_method :instance
+end
+
+
+# An event-driven runner for Arborist::Monitors.
+class Arborist::MonitorRunner
+	extend Loggability
+
+	log_to :arborist
+
+
+	# A ZMQ::Handler object for managing IO for all running monitors.
+	class Handler < ZMQ::Handler
+		extend Loggability,
+		       Arborist::MethodUtilities
+
+		log_to :arborist
+
+		### Create a ZMQ::Handler that acts as the agent that runs the specified
+		### +monitor+.
+		def initialize( reactor )
+			@reactor = reactor
+			@client = Arborist::Client.new
+			@pollitem = ZMQ::Pollitem.new( @client.tree_api, ZMQ::POLLOUT )
+			@pollitem.handler = self
+
+			@request_queue = {}
+			@registered = false
+		end
+
+
+		######
+		public
+		######
+
+		# The ZMQ::Loop that this runner is registered with
+		attr_reader :reactor
+
+		# The Queue of pending requests, keyed by the callback that should be called with the
+		# results.
+		attr_reader :request_queue
+
+		# The Arborist::Client that will provide the message packing and unpacking
+		attr_reader :client
+
+		##
+		# True if the Handler is registered to write one or more requests
+		attr_predicate :registered
+
+
+		### Run the specified +monitor+ and update nodes with the results.
+		def run_monitor( monitor )
+			criteria     = monitor.positive_criteria
+			include_down = monitor.include_down?
+			props        = monitor.node_properties
+
+			self.fetch( criteria, include_down, props ) do |nodes|
+				# :FIXME: Doesn't apply negative criteria
+				results = monitor.run( nodes )
+				self.update( results ) do
+					self.log.debug "Updated %d via the '%s' monitor" %
+						[ results.length, monitor.description ]
+				end
+			end
+		end
+
+
+		### Create a fetch request using the runner's client, then queue the request up
+		### with the specified +block+ as the callback.
+		def fetch( criteria, include_down, properties, &block )
+			fetch = self.client.make_fetch_request( criteria,
+				include_down: include_down,
+				properties: properties
+			)
+			self.queue_request( fetch, &block )
+		end
+
+
+		### Create an update request using the runner's client, then queue the request up
+		### with the specified +block+ as the callback.
+		def update( nodemap, &block )
+			update = self.client.make_update_request( nodemap )
+			self.queue_request( update, &block )
+		end
+
+
+		### Add the specified +event+ to the queue to be published to the console event
+		### socket
+		def queue_request( request, &callback )
+			self.request_queue[ callback ] = request
+			self.register
+		end
+
+
+		### Register the handler's pollitem as being ready to write if it isn't already.
+		def register
+			# self.log.debug "Registering for writing."
+			self.reactor.register( self.pollitem ) unless @registered
+			@registered = true
+		end
+
+
+		### Unregister the handler's pollitem from the reactor when there's nothing ready
+		### to write.
+		def unregister
+			# self.log.debug "Unregistering for writing."
+			self.reactor.remove( self.pollitem ) if @registered
+			@registered = false
+		end
+
+
+		### Write commands from the queue
+		def on_writable
+			if (( pair = self.request_queue.shift ))
+				callback, request = *pair
+				res = self.client.send_tree_api_request( request )
+				callback.call( res )
+			end
+
+			self.unregister if self.request_queue.empty?
+			return true
+		end
+
+	end # class Handler
+
+
+	### Create a new Arborist::MonitorRunner
+	def initialize
+		@monitors = []
+		@handler = nil
+		@reactor = ZMQ::Loop.new
+	end
+
+
+	######
+	public
+	######
+
+	# The Array of loaded Arborist::Monitors the runner should run.
+	attr_reader :monitors
+
+	# The ZMQ::Handler subclass that handles all async IO
+	attr_accessor :handler
+
+	# The reactor (a ZMQ::Loop) the runner uses to drive everything
+	attr_accessor :reactor
+
+
+	### Load monitors from the specified +enumerator+.
+	def load_monitors( enumerator )
+		@monitors += enumerator.to_a
+	end
+
+
+	### Run the specified +monitors+
+	def run
+		self.handler = Arborist::MonitorRunner::Handler.new( self.reactor )
+
+		self.monitors.each do |mon|
+			self.add_timer_for( mon )
+		end
+
+		self.reactor.start
+	end
+
+
+	### Register a timer for the specified +monitor+.
+	def add_timer_for( monitor )
+		interval = monitor.interval
+
+		timer = if monitor.splay.nonzero?
+				self.splay_timer_for( monitor )
+			else
+				self.interval_timer_for( monitor )
+			end
+
+		self.reactor.register_timer( timer )
+	end
+
+
+	### Create a repeating ZMQ::Timer that will run the specified monitor on its interval.
+	def interval_timer_for( monitor )
+		interval = monitor.interval
+		self.log.info "Creating timer for %s monitor to run every %ds" % [ monitor, interval ]
+
+		return ZMQ::Timer.new( interval, 0 ) do
+			self.handler.run_monitor( monitor )
+		end
+	end
+
+
+	### Create a one-shot ZMQ::Timer that will register the interval timer for the specified
+	### +monitor+ after a random number of seconds no greater than its splay.
+	def splay_timer_for( monitor )
+		delay = rand( monitor.splay )
+		self.log.info "Splaying registration of %s monitor for %ds" % [ monitor, delay ]
+
+		return ZMQ::Timer.new( delay, 1 ) do
+			interval_timer = self.interval_timer_for( monitor )
+			self.reactor.register_timer( interval_timer )
+		end
+	end
+
+end # class Arborist::MonitorRunner
+

data/lib/arborist/node.rb

@@ -0,0 +1,700 @@
+# -*- ruby -*-
+#encoding: utf-8
+
+require 'set'
+require 'uri'
+require 'time'
+require 'pathname'
+require 'state_machines'
+
+require 'loggability'
+require 'pluggability'
+require 'arborist' unless defined?( Arborist )
+require 'arborist/mixins'
+
+using Arborist::TimeRefinements
+
+
+# The basic node class for an Arborist tree
+class Arborist::Node
+	include Enumerable,
+	        Arborist::HashUtilities
+	extend Loggability,
+	       Pluggability,
+	       Arborist::MethodUtilities
+
+
+	##
+	# The key for the thread local that is used to track instances as they're
+	# loaded.
+	LOADED_INSTANCE_KEY = :loaded_node_instances
+
+	##
+	# The glob pattern to use for searching for node
+	NODE_FILE_PATTERN = '**/*.rb'
+
+
+	##
+	# The struct for the 'ack' operational property
+	ACK = Struct.new( 'ArboristNodeACK', :message, :via, :sender, :time )
+
+	##
+	# The keys required to be set for an ACK
+	ACK_REQUIRED_PROPERTIES = %w[ message sender ]
+
+
+	##
+	# Log via the Arborist logger
+	log_to :arborist
+
+	##
+	# Search for plugins in lib/arborist/node directories in loaded gems
+	plugin_prefixes 'arborist/node'
+
+
+	state_machine( :status, initial: :unknown ) do
+
+		state :unknown,
+			:up,
+			:down,
+			:acked
+
+		event :update do
+			transition any - [:acked] => :acked, if: :ack_set?
+			transition any - [:up] => :up, if: :last_contact_successful?
+			transition any - [:down, :acked] => :down, unless: :last_contact_successful?
+		end
+
+		after_transition any => :acked, do: :on_ack
+		after_transition :acked => :up, do: :on_ack_cleared
+		after_transition :down => :up, do: :on_node_up
+		after_transition [:unknown, :up] => :down, do: :on_node_down
+
+		after_transition do: :add_status_to_update_delta
+	end
+
+
+	### Return a curried Proc for the ::create method for the specified +type+.
+	def self::curried_create( type )
+		return self.method( :create ).to_proc.curry( 2 )[ type ]
+	end
+
+
+	### Overridden to track instances of created nodes for the DSL.
+	def self::new( * )
+		new_instance = super
+		Arborist::Node.add_loaded_instance( new_instance )
+		return new_instance
+	end
+
+
+	### Create a new node with its state read from the specified +hash+.
+	def self::from_hash( hash )
+		return self.new( hash[:identifier] ) do
+			self.marshal_load( hash )
+		end
+	end
+
+
+	### Record a new loaded instance if the Thread-local variable is set up to track
+	### them.
+	def self::add_loaded_instance( new_instance )
+		instances = Thread.current[ LOADED_INSTANCE_KEY ] or return
+		instances << new_instance
+	end
+
+
+	### Inheritance hook -- add a DSL declarative function for the given +subclass+.
+	def self::inherited( subclass )
+		super
+
+		if name = subclass.name
+			name.sub!( /.*::/, '' )
+			body = self.curried_create( subclass )
+			Arborist.add_dsl_constructor( name, &body )
+		else
+			self.log.info "Skipping DSL constructor for anonymous class."
+		end
+
+	end
+
+
+	### Load the specified +file+ and return any new Nodes created as a result.
+	def self::load( file )
+		self.log.info "Loading node file %s..." % [ file ]
+		Thread.current[ LOADED_INSTANCE_KEY ] = []
+
+		begin
+			Kernel.load( file )
+		rescue => err
+			self.log.error "%p while loading %s: %s" % [ err.class, file, err.message ]
+			raise
+		end
+
+		return Thread.current[ LOADED_INSTANCE_KEY ]
+	ensure
+		Thread.current[ LOADED_INSTANCE_KEY ] = nil
+	end
+
+
+	### Return an iterator for all the node files in the specified +directory+.
+	def self::each_in( directory )
+		path = Pathname( directory )
+		paths = if path.directory?
+				Pathname.glob( directory + NODE_FILE_PATTERN )
+			else
+				[ path ]
+			end
+
+		return paths.flat_map do |file|
+			file_url = "file://%s" % [ file.expand_path ]
+			nodes = self.load( file )
+			self.log.debug "Loaded nodes %p..." % [ nodes ]
+			nodes.each do |node|
+				node.source = file_url
+			end
+			nodes
+		end
+	end
+
+
+	### Create a new Node with the specified +identifier+, which must be unique to the
+	### loaded tree.
+	def initialize( identifier, &block )
+		raise "Invalid identifier %p" % [identifier] unless
+			identifier =~ /^\w[\w\-]*$/
+
+		@identifier     = identifier
+		@parent         = '_'
+		@description    = nil
+		@tags           = Set.new
+		@source         = nil
+		@children       = {}
+
+		@status         = 'unknown'
+		@status_changed = Time.at( 0 )
+
+		@error          = nil
+		@ack            = nil
+		@properties     = {}
+		@last_contacted = Time.at( 0 )
+
+		@update_delta   = Hash.new do |h,k|
+			h[ k ] = Hash.new( &h.default_proc )
+		end
+		@pending_update_events = []
+		@subscriptions  = {}
+
+		self.instance_eval( &block ) if block
+	end
+
+
+	######
+	public
+	######
+
+	##
+	# The node's identifier
+	attr_reader :identifier
+
+	##
+	# The URI of the source the object was read from
+	attr_reader :source
+
+	##
+	# The Hash of nodes which are children of this node, keyed by identifier
+	attr_reader :children
+
+	##
+	# Arbitrary attributes attached to this node via the manager API
+	attr_reader :properties
+
+	##
+	# The Time the node was last contacted
+	attr_accessor :last_contacted
+
+	##
+	# The Time the node's status last changed.
+	attr_accessor :status_changed
+
+	##
+	# The last error encountered by a monitor attempting to update this node.
+	attr_accessor :error
+
+	##
+	# The acknowledgement currently in effect. Should be an instance of Arborist::Node::ACK
+	attr_accessor :ack
+
+	##
+	# The Hash of changes tracked during an #update.
+	attr_reader :update_delta
+
+	##
+	# The Array of events generated by the current update event
+	attr_reader :pending_update_events
+
+	##
+	# The Hash of Subscription objects observing this node and its children, keyed by
+	# subscription ID.
+	attr_reader :subscriptions
+
+
+	### Set the source of the node to +source+, which should be a valid URI.
+	def source=( source )
+		@source = URI( source )
+	end
+
+
+	#
+	# :section: DSLish declaration methods
+	# These methods are both getter and setter for a node's attributes, used
+	# in the node source.
+	#
+
+	### Get/set the node's parent node, which should either be an identifier or an object
+	### that responds to #identifier with one.
+	def parent( new_parent=nil )
+		return @parent if new_parent.nil?
+
+		@parent = if new_parent.respond_to?( :identifier )
+				new_parent.identifier.to_s
+			else
+				@parent = new_parent.to_s
+			end
+	end
+
+
+	### Get/set the node's description.
+	def description( new_description=nil )
+		return @description unless new_description
+		@description = new_description.to_s
+	end
+
+
+	### Declare one or more +tags+ for this node.
+	def tags( *tags )
+		@tags.merge( tags.map(&:to_s) ) unless tags.empty?
+		return @tags.to_a
+	end
+
+
+	#
+	# :section: Manager API
+	# Methods used by the manager to manage its nodes.
+	#
+
+
+	### Return the simple type of this node (e.g., Arborist::Node::Host => 'host')
+	def type
+		return 'anonymous' unless self.class.name
+		return self.class.name.sub( /.*::/, '' ).downcase
+	end
+
+
+	### Add the specified +subscription+ (an Arborist::Subscription) to the node.
+	def add_subscription( subscription )
+		self.subscriptions[ subscription.id ] = subscription
+	end
+
+
+	### Remove the specified +subscription+ (an Arborist::Subscription) from the node.
+	def remove_subscription( subscription_id )
+		return self.subscriptions.delete( subscription_id )
+	end
+
+
+	### Return subscriptions matching the specified +event+ on the receiving node.
+	def find_matching_subscriptions( event )
+		return self.subscriptions.values.find_all {|sub| event =~ sub }
+	end
+
+
+	### Publish the specified +events+ to any subscriptions the node has which match them.
+	def publish_events( *events )
+		self.subscriptions.each_value do |sub|
+			sub.on_events( *events )
+		end
+	end
+
+
+	### Update specified +properties+ for the node.
+	def update( new_properties )
+		new_properties = stringify_keys( new_properties )
+		self.log.debug "Updated: %p" % [ new_properties ]
+
+		self.last_contacted = Time.now
+		self.error          = new_properties.delete( 'error' )
+		self.ack            = new_properties.delete( 'ack' ) if new_properties.key?( 'ack' )
+
+		self.properties.merge!( new_properties, &self.method(:merge_and_record_delta) )
+		compact_hash( self.properties )
+
+		# Super to the state machine event method
+		super
+
+		events = self.pending_update_events.clone
+		events << self.make_update_event
+		events << self.make_delta_event unless self.update_delta.empty?
+
+		return events
+	ensure
+		self.update_delta.clear
+		self.pending_update_events.clear
+	end
+
+
+	### Merge the specified +new_properties+ into the node's properties, recording
+	### each change in the node's #update_delta.
+	def merge_and_record_delta( key, oldval, newval, prefixes=[] )
+		self.log.debug "Merging property %s: %p -> %p" % [
+			(prefixes + [key]).join('.'),
+			oldval,
+			newval
+		]
+
+		# Merge them (recursively) if they're both merge-able
+		if oldval.respond_to?( :merge! ) && newval.respond_to?( :merge! )
+			return oldval.merge( newval ) do |ikey, ioldval, inewval|
+				self.merge_and_record_delta( ikey, ioldval, inewval, prefixes + [key] )
+			end
+
+		# Otherwise just directly compare them and record any changes
+		else
+			unless oldval == newval
+				prefixed_delta = prefixes.inject( self.update_delta ) do |hash, key|
+					hash[ key ]
+				end
+				prefixed_delta[ key ] = [ oldval, newval ]
+			end
+
+			return newval
+		end
+	end
+
+
+	### Return the node's state in an Arborist::Event of type 'node.update'.
+	def make_update_event
+		return Arborist::Event.create( 'node_update', self )
+	end
+
+
+	### Return an Event generated from the node's state changes.
+	def make_delta_event
+		self.log.debug "Making node.delta event: %p" % [ self.update_delta ]
+		return Arborist::Event.create( 'node_delta', self, self.update_delta )
+	end
+
+
+	### Returns +true+ if the node's state has changed since the last time
+	### #snapshot_state was called.
+	def state_has_changed?
+		return ! self.update_delta.empty?
+	end
+
+
+	### Returns +true+ if the specified search +criteria+ all match this node.
+	def matches?( criteria )
+		self.log.debug "Matching %p against criteria: %p" % [ self, criteria ]
+		return criteria.all? do |key, val|
+			self.match_criteria?( key, val )
+		end.tap {|match| self.log.debug "  node %s match." % [ match ? "DID" : "did not"] }
+	end
+
+
+	### Returns +true+ if the node matches the specified +key+ and +val+ criteria.
+	def match_criteria?( key, val )
+		return case key
+			when 'status'
+				self.status == val
+			when 'type'
+				self.log.debug "Checking node type %p against %p" % [ self.type, val ]
+				self.type == val
+			when 'tag' then @tags.include?( val.to_s )
+			when 'tags' then Array(val).all? {|tag| @tags.include?(tag) }
+			when 'identifier' then @identifier == val
+			else
+				hash_matches( @properties, key, val )
+			end
+	end
+
+
+	### Return a Hash of node state values that match the specified +value_spec+.
+	def fetch_values( value_spec=nil )
+		state = self.properties.merge( self.operational_values )
+		state = stringify_keys( state )
+
+		if value_spec
+			self.log.debug "Eliminating all values except: %p (from keys: %p)" %
+				[ value_spec, state.keys ]
+			state.delete_if {|key, _| !value_spec.include?(key) }
+		end
+
+		return state
+	end
+
+
+	### Return a Hash of the operational values that are included with the node's
+	### monitor state.
+	def operational_values
+		values = {
+			type: self.type,
+			status: self.status,
+			tags: self.tags
+		}
+		values[:ack] = self.ack.to_h if self.ack
+
+		return values
+	end
+
+
+	#
+	# :section: Hierarchy API
+	#
+
+	### Enumerable API -- iterate over the children of this node.
+	def each( &block )
+		return self.children.values.each( &block )
+	end
+
+
+	### Returns +true+ if the node has one or more child nodes.
+	def has_children?
+		return !self.children.empty?
+	end
+
+
+	### Returns +true+ if the node is considered operational.
+	def operational?
+		return self.identifier.start_with?( '_' )
+	end
+
+
+	### Register the specified +node+ as a child of this node, replacing any existing
+	### node with the same identifier.
+	def add_child( node )
+		self.log.debug "Adding node %p as a child. Parent = %p" % [ node, node.parent ]
+		raise "%p is not a child of %p" % [ node, self ] if
+			node.parent && node.parent != self.identifier
+		self.children[ node.identifier ] = node
+	end
+
+
+	### Append operator -- add the specified +node+ as a child and return +self+.
+	def <<( node )
+		self.add_child( node )
+		return self
+	end
+
+
+	### Unregister the specified +node+ as a child of this node.
+	def remove_child( node )
+		self.log.debug "Removing node %p from children" % [ node ]
+		return self.children.delete( node.identifier )
+	end
+
+
+	#
+	# :section: Utility methods
+	#
+
+	### Return a string describing the node's status.
+	def status_description
+		case self.status
+		when 'up', 'down'
+			return "%s as of %s" % [ self.status.upcase, self.last_contacted ]
+		when 'acked'
+			return "ACKed by %s %s" % [ self.ack.sender, self.ack.time.as_delta ]
+		else
+			return "in an unknown state"
+		end
+	end
+
+
+	### Return a string describing node details; returns +nil+ for the base class. Subclasses
+	### may override this to add to the output of #inspect.
+	def node_description
+		return nil
+	end
+
+
+	### Return a String representation of the object suitable for debugging.
+	def inspect
+		return "#<%p:%#x [%s] -> %s %p %s%s, %d children, %s>" % [
+			self.class,
+			self.object_id * 2,
+			self.identifier,
+			self.parent || 'root',
+			self.description || "(no description)",
+			self.node_description.to_s,
+			self.source,
+			self.children.length,
+			self.status_description,
+		]
+	end
+
+
+	#
+	# :section: Serialization API
+	#
+
+	### Return a Hash of the node's state.
+	def to_hash
+		return {
+			identifier: self.identifier,
+			type: self.class.name.to_s.sub( /.+::/, '' ).downcase,
+			parent: self.parent,
+			description: self.description,
+			tags: self.tags,
+			properties: self.properties.dup,
+			status: self.status,
+			ack: self.ack ? self.ack.to_h : nil,
+			last_contacted: self.last_contacted ? self.last_contacted.iso8601 : nil,
+			status_changed: self.status_changed ? self.status_changed.iso8601 : nil,
+			error: self.error,
+		}
+	end
+
+
+	### Marshal API -- return the node as an object suitable for marshalling.
+	def marshal_dump
+		return self.to_hash
+	end
+
+
+	### Marshal API -- set up the object's state using the +hash+ from a
+	### previously-marshalled node.
+	def marshal_load( hash )
+		@identifier = hash[:identifier]
+		@properties = hash[:properties]
+
+		@parent         = hash[:parent]
+		@description    = hash[:description]
+		@tags           = Set.new( hash[:tags] )
+		@children       = {}
+
+		@status         = hash[:status]
+		@status_changed = Time.parse( hash[:status_changed] )
+
+		@error          = hash[:error]
+		@properties     = hash[:properties]
+		@last_contacted = Time.parse( hash[:last_contacted] )
+
+		if hash[:ack]
+			ack_values = hash[:ack].values_at( *Arborist::Node::ACK.members )
+			@ack = Arborist::Node::ACK.new( *ack_values )
+		end
+	end
+
+
+	### Equality operator -- returns +true+ if +other_node+ has the same identifier, parent, and
+	### state as the receiving one.
+	def ==( other_node )
+		return \
+			other_node.identifier == self.identifier &&
+			other_node.parent == self.parent &&
+			other_node.description == self.description &&
+			other_node.tags == self.tags &&
+			other_node.properties == self.properties &&
+			other_node.status == self.status &&
+			other_node.ack == self.ack &&
+			other_node.error == self.error
+	end
+
+
+	#########
+	protected
+	#########
+
+	### Ack the node with the specified +ack_data+, which should contain
+	def ack=( ack_data )
+		self.log.debug "ACKed with data: %p" % [ ack_data ]
+
+		ack_data['time'] ||= Time.now
+		ack_values = ack_data.values_at( *Arborist::Node::ACK.members.map(&:to_s) )
+		new_ack = Arborist::Node::ACK.new( *ack_values )
+
+		if missing = ACK_REQUIRED_PROPERTIES.find {|prop| new_ack[prop].nil? }
+			raise "Missing required ACK attribute %s" % [ missing ]
+		end
+
+		@ack = new_ack
+	end
+
+
+	### State machine guard predicate -- Returns +true+ if the node has an ACK status set.
+	def ack_set?
+		self.log.debug "Checking to see if this node has been ACKed (it %s)" %
+			[ @ack ? "has" : "has not" ]
+		return @ack ? true : false
+	end
+
+
+	### State machine guard predicate -- Returns +true+ if the last time the node
+	### was monitored resulted in an update.
+	def last_contact_successful?
+		self.log.debug "Checking to see if last contact was successful (it %s)" %
+			[ self.error ? "wasn't" : "was" ]
+		return !self.error
+	end
+
+
+	#
+	# :section: State Callbacks
+	#
+
+	### Callback for when an acknowledgement is set.
+	def on_ack( transition )
+		self.log.warn "ACKed: %s" % [ self.status_description ]
+		self.pending_update_events <<
+			Arborist::Event.create( 'node_acked', self.fetch_values, self.ack.to_h )
+	end
+
+
+	### Callback for when an acknowledgement is cleared.
+	def on_ack_cleared( transition )
+		self.log.warn "ACK cleared for %s" % [ self.identifier ]
+	end
+
+
+	### Callback for when a node goes from down to up
+	def on_node_up( transition )
+		self.log.warn "%s is %s" % [ self.identifier, self.status_description ]
+	end
+
+
+	### Callback for when a node goes from up to down
+	def on_node_down( transition )
+		self.log.error "%s is %s" % [ self.identifier, self.status_description ]
+		self.update_delta[ 'error' ] = [ nil, self.error ]
+	end
+
+
+	### Add the transition from one state to another to the data used to build
+	### deltas for the #update event.
+	def add_status_to_update_delta( transition )
+		self.update_delta[ 'status' ] = [ transition.from, transition.to ]
+	end
+
+
+	#######
+	private
+	#######
+
+	### Returns true if the specified +hash+ includes the specified +key+, and the value
+	### associated with the +key+ either includes +val+ if it is a Hash, or equals +val+ if it's
+	### anything but a Hash.
+	def hash_matches( hash, key, val )
+		actual = hash[ key ] or return false
+
+		if actual.is_a?( Hash )
+			if val.is_a?( Hash )
+				return val.all? {|subkey, subval| hash_matches(actual, subkey, subval) }
+			else
+				return false
+			end
+		else
+			return actual == val
+		end
+	end
+
+end # class Arborist::Node