arborist 0.0.1.pre20160606141735 → 0.0.1.pre20160829140603

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7a9424679d95a6df37569e5020831d81643fb54d
-  data.tar.gz: c5c116c695eab33c5a81f79acfebaebcf9a8396a
+  metadata.gz: c13d6d8931471437945bbe16acd57fa7f17a01f8
+  data.tar.gz: 38d663d0f7fb9105727613bf61ff62a1496062f6
 SHA512:
-  metadata.gz: cab6cbc4c4ed3d7a194ba614cd808155c2f3563bbe1b10b90ac33fc2513ce9d8abed321fd8ea901abd361544eeaa8a564faf904a9c46e932ef1034529fea0164
-  data.tar.gz: ce941a82bacba31cfee78ce458df86abc9579e6763b0af249de2442fc2eea3a85f298cb2bf4828855cb5272b62648e153942b26c2c94b7513cfbc1efef960564
+  metadata.gz: 49431289b8d44fc52f40cdf60062905666a6f9de12c7c6ff8bbfb04d62e2f782f084fd824df08271ed264b513953b23baae0ccf9b4fd36b390a5d4f072c5b07f
+  data.tar.gz: 436d3fadba15758c0e9bd12656ada1ef3b7c5471c4b8ccaba431e02e338cae3576e3acbd579ea5f440937ff1af7b4717f118b6d4723691913f320c05afb82565

data/ChangeLog

@@ -1,3 +1,82 @@
+2016-08-29  Michael Granger  <ged@FaerieMUD.org>
+
+	* spec/data/nodes/duir.rb:
+	Merge with a5a6e5092024
+	[c961355d10c8] [tip]
+
+2016-08-29  Mahlon E. Smith  <mahlon@martini.nu>
+
+	* lib/arborist/loader/file.rb, spec/data/nodes/duir.rb,
+	spec/data/nodes/sub/duir.rb:
+	Fix the file loader so it correctly crawls subdirectories.
+	[a5a6e5092024]
+
+2016-08-03  Michael Granger  <ged@FaerieMUD.org>
+
+	* lib/arborist/event/sys_node_added.rb,
+	lib/arborist/event/sys_node_removed.rb,
+	lib/arborist/event/sys_reloaded.rb,
+	lib/arborist/event/sys_shutdown.rb:
+	Remove system event node classes
+	[a34a23ef095e]
+
+2016-08-02  Michael Granger  <ged@FaerieMUD.org>
+
+	* Manifest.txt, TODO.md, lib/arborist/mixins.rb,
+	lib/arborist/monitor.rb, lib/arborist/monitor_runner.rb,
+	lib/arborist/node.rb, lib/arborist/node/host.rb,
+	lib/arborist/node/service.rb, spec/arborist/node/service_spec.rb,
+	spec/arborist/node_spec.rb, spec/arborist/subscription_spec.rb:
+	- Update the TO-DO list
+	- Factor out some network functions into a common mixin
+	- Add Arborist::Monitor#inspect
+	- Restore ack+status of serialized nodes
+	- Allow services to set their own address when not bound to INADDR_ANY
+	- Finish separating node events from system events
+	[f853508dba8c] [github/master]
+
+	* lib/arborist/monitor/socket.rb:
+	Fix race to calculate select timeout in socket monitor
+	[56a6dd2a8e40]
+
+2016-07-20  Michael Granger  <ged@FaerieMUD.org>
+
+	* TODO.md:
+	Update the To-Do file
+	[53ca09b4d50e]
+
+2016-07-20  Mahlon E. Smith  <mahlon@martini.nu>
+
+	* lib/arborist/node/resource.rb, spec/arborist/node/resource_spec.rb:
+	Add a `category` attribute to the resource node type.
+	[ae3aa8435e15]
+
+2016-07-20  Michael Granger  <ged@FaerieMUD.org>
+
+	* .gems, .tm_properties, Events.md, TODO.md, arborist.gemspec,
+	lib/arborist.rb, lib/arborist/command/watch.rb,
+	lib/arborist/event/sys_shutdown.rb, lib/arborist/manager.rb,
+	lib/arborist/manager/event_publisher.rb,
+	lib/arborist/manager/tree_api.rb, lib/arborist/node.rb,
+	lib/arborist/observer_runner.rb, spec/arborist/client_spec.rb,
+	spec/arborist/manager/event_publisher_spec.rb,
+	spec/arborist/manager/tree_api_spec.rb,
+	spec/arborist/manager_spec.rb,
+	spec/arborist/observer_runner_spec.rb, spec/spec_helper.rb:
+	Send system events from the manager for status changes
+	[4cc9924e6b0c]
+
+2016-06-15  Michael Granger  <ged@FaerieMUD.org>
+
+	* spec/arborist/observer_spec.rb, spec/data/observers/auditor.rb:
+	Fix signature of observer actions in specs
+	[196443e52d87]
+
+	* lib/arborist/node.rb, lib/arborist/node/host.rb,
+	spec/arborist/node_spec.rb:
+	Add a config hash to node operational metadata
+	[196223da8ba4]
+
 2016-05-19  Mahlon E. Smith  <mahlon@martini.nu>
 
 	* lib/arborist/node/resource.rb, spec/arborist/node/resource_spec.rb:
@@ -9,7 +88,7 @@
 	isn't a service with a protocol and port, but a necessary piece of
 	information for overall host and service health, that should be
 	monitored (and potentially ack'ed/disabled) independently.
-	[b4324ab8853e] [tip]
+	[b4324ab8853e]
 
 2016-05-18  Mahlon E. Smith  <mahlon@laika.com>
 
@@ -23,7 +102,7 @@
 	lib/arborist/node.rb, spec/arborist/dependency_spec.rb,
 	spec/arborist/node_spec.rb:
 	Fix the way nodes' dependency state is restored
-	[325443abe6df] [github/master]
+	[325443abe6df]
 
 2016-04-27  Michael Granger  <ged@FaerieMUD.org>
 

data/Events.md

@@ -2,19 +2,19 @@
 
 ## Event Types
 
-«type».«subtype».«identifier»
+«type».«subtype»
 
-    sys.reload
-    sys.shutdown
+    node.acked
     node.delta
+    node.disabled
+    node.down
+    node.quieted
+    node.unknown
+    node.up
     node.update
-    node.ack
-
-Second part of the event name is the verb.
-
-- update
-- change (delta)
-- reload
-- shutdown / startup
+    sys.node_added
+    sys.node_removed
+    sys.reloaded
+    sys.hearbeat
 
 

data/Manifest.txt

@@ -31,9 +31,6 @@ lib/arborist/event/node_quieted.rb
 lib/arborist/event/node_unknown.rb
 lib/arborist/event/node_up.rb
 lib/arborist/event/node_update.rb
-lib/arborist/event/sys_node_added.rb
-lib/arborist/event/sys_node_removed.rb
-lib/arborist/event/sys_reloaded.rb
 lib/arborist/exceptions.rb
 lib/arborist/loader.rb
 lib/arborist/loader/file.rb
@@ -77,6 +74,7 @@ spec/arborist/node/service_spec.rb
 spec/arborist/node_spec.rb
 spec/arborist/observer/action_spec.rb
 spec/arborist/observer/summarize_spec.rb
+spec/arborist/observer_runner_spec.rb
 spec/arborist/observer_spec.rb
 spec/arborist/subscription_spec.rb
 spec/arborist_spec.rb
@@ -84,9 +82,9 @@ spec/data/monitors/pings.rb
 spec/data/monitors/port_checks.rb
 spec/data/monitors/system_resources.rb
 spec/data/monitors/web_services.rb
-spec/data/nodes/duir.rb
 spec/data/nodes/localhost.rb
 spec/data/nodes/sidonie.rb
+spec/data/nodes/sub/duir.rb
 spec/data/nodes/yevaud.rb
 spec/data/observers/auditor.rb
 spec/data/observers/webservices.rb

data/TODO.md

@@ -7,43 +7,23 @@
 * Performance/profiling examination
 
 
-### Manager
-
-* Only restore timestamps from serialized node dependencies, not the deps themselves.
-
-* Broadcast system events:
-    - `sys.node.added`
-    - `sys.node.removed`
-    - `sys.startup`
-    - `sys.shutdown`
-
-
 ### Observers
 
-* Re-subscribe on `sys.startup`, `sys.reloaded`, `sys.node.added`
-* Add `except` to observers DSL
+* Add `exclude` to observers DSL
+  * modify tree api to accept negative criteria to subscribe
+	* pass to manager's create_subscription()
+	* alter subscription to no-op if event matches negative stuff
 
 
-### Nodes
-
-* Allow a service node to not inherit all of its host's addresses (i.e., be bound to one address only or whatever)
-* Resource nodes: disk, load, process checks, etc.  Anything that might
-  be considered a problem, that you'd want to ack independantly of the
-  Host node they are attached to.
-
 ### Monitor
 
 * Add some default monitor types and utilities
-  - UDP socket check
-  - Basic monitors for stdlib Net::* protocols/services
-  - 
-
-* Gems for monitor types that have external dependency
-  - SNMP
-
-### Watch Command
+  - ftp
+  - imap
+  - pop
+  - smtp
 
-* Re-subscribe on `sys.startup`, `sys.reloaded`, `sys.node.added`
+* Write a gem for `fping` monitor 
 
 
 ## Second Release (0.2)

data/lib/arborist.rb

@@ -17,7 +17,7 @@ module Arborist
 	VERSION = '0.0.1'
 
 	# Version control revision
-	REVISION = %q$Revision: ed4f0655b4b6 $
+	REVISION = %q$Revision: 4cc9924e6b0c $
 
 
 	# The name of the environment variable which can be used to set the config path
@@ -168,7 +168,11 @@ module Arborist
 
 	### Fetch the ZMQ context for Arborist.
 	def self::zmq_context
-		return @zmq_context ||= ZMQ::Context.new
+		return @zmq_context ||= begin
+			self.log.info "Using ZeroMQ %s/CZMQ %s" %
+				[ ZMQ.version.join('.'), ZMQ.czmq_version.join('.') ]
+			ZMQ::Context.new
+		end
 	end
 
 

data/lib/arborist/command/watch.rb

@@ -10,31 +10,65 @@ require 'arborist/client'
 module Arborist::CLI::Watch
 	extend Arborist::CLI::Subcommand
 
+	HEARTBEAT_CHARACTERS = %w[💓 💗]
+
 	desc 'Watch events in an Arborist manager'
 
 	command :watch do |cmd|
 		cmd.action do |globals, options, args|
 			client = Arborist::Client.new
-			subid  = client.subscribe( identifier: '_' )
-
 			sock = client.event_api
-			sock.subscribe( subid )
+
+			subid = subscribe_to_node_events( client )
 			prompt.say "Subscription %p" % [ subid ]
 
+			# Watch for system events as well
+			prompt.say "Subscribing to manager heartbeat events."
+			sock.subscribe( 'sys.heartbeat' )
+
 			begin
+				last_runid = nil
 				prompt.say "Watching for events on manager at %s" % [ client.event_api_url ]
 				loop do
 					msgsubid = sock.recv
 					raise "Partial write?!" unless sock.rcvmore?
 					raw_event = sock.recv
-
 					event = MessagePack.unpack( raw_event )
-					prompt.say "[%s] %s\n" % [
-						hl(Time.now.strftime('%Y-%m-%d %H:%M:%S %Z')).color(:dark, :white),
-						hl(dump_event( event )).color( :dark, :white )
-					]
+
+					case msgsubid
+					when 'sys.heartbeat'
+						this_runid = event['run_id']
+
+						if last_runid && last_runid != this_runid
+							self.log.warn "Manager restart: re-subscribing."
+							sock.unsubscribe( subid )
+							subid = subscribe_to_node_events( client )
+							prompt.say "New subscription %p" % [ subid ]
+						else
+							self.log.debug "Manager is alive (runid: %s)" % [ this_runid ]
+						end
+
+						$stderr.print( heartbeat() )
+						last_runid = this_runid
+					when 'sys.node_added'
+						prompt.say "[%s] «Node added» %s\n" % [
+							hl(Time.now.strftime('%Y-%m-%d %H:%M:%S %Z')).color(:dark, :white),
+							hl(event['node']).color( :bold, :cyan )
+						]
+					when 'sys.node_removed'
+						prompt.say "[%s] »Node removed« %s\n" % [
+							hl(Time.now.strftime('%Y-%m-%d %H:%M:%S %Z')).color(:dark, :white),
+							hl(event['node']).color( :dark, :cyan )
+						]
+					else
+						prompt.say "[%s] %s\n" % [
+							hl(Time.now.strftime('%Y-%m-%d %H:%M:%S %Z')).color(:dark, :white),
+							hl(dump_event( event )).color( :dark, :white )
+						]
+					end
 				end
 			ensure
+				self.log.info "Unsubscribing from subscription %p" % [ subid ]
 				client.unsubscribe( subid )
 			end
 		end
@@ -45,6 +79,16 @@ module Arborist::CLI::Watch
 	module_function
 	###############
 
+	### Establish a subscription to all node events via the specified +client+.
+	def subscribe_to_node_events( client )
+		subid = client.subscribe( identifier: '_' )
+		sock = client.event_api
+
+		sock.subscribe( subid )
+		return subid
+	end
+
+
 	### Return a String representation of the specified +event+.
 	def dump_event( event )
 		event_type = event['type']
@@ -84,5 +128,12 @@ module Arborist::CLI::Watch
 		end.join( hl(", ").color(:dark, :white) )
 	end
 
+
+	### Return a heartbeat string for the current time.
+	def heartbeat
+		idx = (Time.now.to_i % HEARTBEAT_CHARACTERS.length) - 1
+		return " " + HEARTBEAT_CHARACTERS[ idx ] + "\x08\x08"
+	end
+
 end # module Arborist::CLI::Watch
 

data/lib/arborist/loader/file.rb

@@ -27,7 +27,7 @@ class Arborist::Loader::File < Arborist::Loader
 	def paths
 		path = Pathname( self.directory )
 		if path.directory?
-			return Pathname.glob( directory + FILE_PATTERN ).each
+			return Pathname.glob( path + FILE_PATTERN ).each
 		else
 			return [ path ].each
 		end

data/lib/arborist/manager.rb

@@ -1,6 +1,7 @@
 # -*- ruby -*-
 #encoding: utf-8
 
+require 'securerandom'
 require 'pathname'
 require 'tempfile'
 require 'configurability'
@@ -30,7 +31,9 @@ class Arborist::Manager
 	# Configurability API -- set config defaults
 	CONFIG_DEFAULTS = {
 		state_file: nil,
-		checkpoint_frequency: 30
+		checkpoint_frequency: 30000,
+		heartbeat_frequency: 1000,
+		linger: 5000
 	}
 
 
@@ -46,16 +49,34 @@ class Arborist::Manager
 	singleton_attr_accessor :state_file
 
 	##
-	# The number of seconds between automatic state checkpoints
+	# The number of milliseconds between automatic state checkpoints
 	singleton_attr_accessor :checkpoint_frequency
 
+	##
+	# The number of milliseconds between heartbeat events
+	singleton_attr_accessor :heartbeat_frequency
+
+	##
+	# The maximum amount of time to wait for pending events to be delivered during
+	# shutdown, in milliseconds.
+	singleton_attr_accessor :linger
+
 
 	### Configurability API -- configure the manager
 	def self::configure( config=nil )
 		config ||= {}
 		config = self.defaults.merge( config[:manager] || {} )
 
+		self.log.debug "Config is: %p" % [ config ]
+
 		self.state_file = config[:state_file] && Pathname( config[:state_file] )
+		self.linger = config[:linger].to_i
+		self.log.info "Linger configured to %p" % [ self.linger ]
+
+		self.heartbeat_frequency = config[:heartbeat_frequency].to_i ||
+			CONFIG_DEFAULTS[:heartbeat_frequency]
+		raise Arborist::ConfigError, "heartbeat frequency must be a positive non-zero integer" if
+			self.heartbeat_frequency <= 0
 
 		interval = config[:checkpoint_frequency].to_i
 		if interval && interval.nonzero?
@@ -72,10 +93,10 @@ class Arborist::Manager
 
 	### Create a new Arborist::Manager.
 	def initialize
+		@run_id = SecureRandom.hex( 16 )
 		@root = Arborist::Node.create( :root )
-		@nodes = {
-			'_' => @root,
-		}
+		@nodes = { '_' => @root }
+
 		@subscriptions = {}
 		@tree_built = false
 
@@ -83,12 +104,27 @@ class Arborist::Manager
 		@signal_timer = nil
 		@start_time   = nil
 
-		Thread.main[:signal_queue] = []
-		@zmq_loop     = nil
-
-		@api_handler = nil
-		@event_publisher = nil
 		@checkpoint_timer = nil
+		@linger = self.class.linger || Arborist::Manager::CONFIG_DEFAULTS[ :linger ]
+		self.log.info "Linger set to %p" % [ @linger ]
+
+		@zmq_loop = ZMQ::Loop.new
+		# @zmq_loop.verbose = true
+		@tree_sock = self.setup_tree_socket
+		@event_sock = self.setup_event_socket
+
+		@api_handler = Arborist::Manager::TreeAPI.new( @tree_sock, self )
+		@tree_sock.handler = @api_handler
+		@zmq_loop.register( @tree_sock )
+
+		@event_publisher = Arborist::Manager::EventPublisher.new( @event_sock, self, @zmq_loop )
+		@event_sock.handler = @event_publisher
+		@zmq_loop.register( @event_sock )
+
+		@heartbeat_timer = self.make_heartbeat_timer
+		@checkpoint_timer = self.make_checkpoint_timer
+
+		Thread.main[:signal_queue] = []
 	end
 
 
@@ -96,6 +132,10 @@ class Arborist::Manager
 	public
 	######
 
+	##
+	# A unique string used to identify different runs of the Manager
+	attr_reader :run_id
+
 	##
 	# The root node of the tree.
 	attr_accessor :root
@@ -128,6 +168,23 @@ class Arborist::Manager
 	# Flag for marking when the tree is built successfully the first time
 	attr_predicate_accessor :tree_built
 
+	##
+	# The maximum amount of time to wait for pending events to be delivered during
+	# shutdown, in milliseconds.
+	attr_reader :linger
+
+	##
+	# The ZMQ::Timer that processes signals
+	attr_reader :signal_timer
+
+	##
+	# The ZMQ::Timer that periodically checkpoints the manager's state (if it's configured to do so)
+	attr_reader :checkpoint_timer
+
+	##
+	# The ZMQ::Timer that periodically publishes a heartbeat event
+	attr_reader :heartbeat_timer
+
 
 	#
 	# :section: Startup/Shutdown
@@ -136,20 +193,21 @@ class Arborist::Manager
 	### Setup sockets and start the event loop.
 	def run
 		self.log.info "Getting ready to start the manager."
-		self.setup_sockets
+		self.publish_system_event( 'startup', start_time: Time.now.to_s, version: Arborist::VERSION )
+		self.register_timers
 		self.set_signal_handlers
 		self.start_accepting_requests
 
 		return self # For chaining
 	ensure
 		self.restore_signal_handlers
-		if @zmq_loop
+		if self.zmq_loop
 			self.log.debug "Unregistering sockets."
-			@zmq_loop.remove( @tree_sock )
+			self.zmq_loop.remove( @tree_sock )
 			@tree_sock.pollable.close
-			@zmq_loop.remove( @event_sock )
+			self.zmq_loop.remove( @event_sock )
 			@event_sock.pollable.close
-			@zmq_loop.cancel_timer( @checkpoint_timer ) if @checkpoint_timer
+			self.zmq_loop.cancel_timer( @checkpoint_timer ) if @checkpoint_timer
 		end
 
 		self.save_node_states
@@ -161,7 +219,14 @@ class Arborist::Manager
 
 	### Returns true if the Manager is running.
 	def running?
-		return @zmq_loop && @zmq_loop.running?
+		return self.zmq_loop && self.zmq_loop.running?
+	end
+
+
+	### Register the Manager's timers.
+	def register_timers
+		self.zmq_loop.register_timer( self.heartbeat_timer )
+		self.zmq_loop.register_timer( self.checkpoint_timer ) if self.checkpoint_timer
 	end
 
 
@@ -169,32 +234,11 @@ class Arborist::Manager
 	def start_accepting_requests
 		self.log.debug "Starting the main loop"
 
-		@zmq_loop = ZMQ::Loop.new
-
-		@api_handler = Arborist::Manager::TreeAPI.new( @tree_sock, self )
-		@tree_sock.handler = @api_handler
-		@zmq_loop.register( @tree_sock )
-
-		@event_publisher = Arborist::Manager::EventPublisher.new( @event_sock, self, @zmq_loop )
-		@event_sock.handler = @event_publisher
-		@zmq_loop.register( @event_sock )
-
-		@checkpoint_timer = self.start_state_checkpointing
-		@zmq_loop.register_timer( @checkpoint_timer ) if @checkpoint_timer
-
 		self.setup_signal_timer
 		self.start_time = Time.now
 
 		self.log.debug "Manager running."
-		@zmq_loop.start
-	end
-
-
-	### Create the ZMQ API socket if necessary.
-	def setup_sockets
-		self.log.debug "Setting up sockets"
-		@tree_sock = self.setup_tree_socket
-		@event_sock = self.setup_event_socket
+		return self.zmq_loop.start
 	end
 
 
@@ -214,7 +258,7 @@ class Arborist::Manager
 		sock = Arborist.zmq_context.socket( :PUB )
 		self.log.debug "  binding the event socket (%#0x) to %p" %
 			[ sock.object_id * 2, Arborist.event_api_url ]
-		sock.linger = 0
+		sock.linger = self.linger
 		sock.bind( Arborist.event_api_url )
 		return ZMQ::Pollitem.new( sock, ZMQ::POLLOUT )
 	end
@@ -231,7 +275,11 @@ class Arborist::Manager
 		self.log.info "Stopping the manager."
 		self.ignore_signals
 		self.cancel_signal_timer
-		@zmq_loop.stop if @zmq_loop
+
+		@api_handler.shutdown
+		@event_publisher.shutdown
+
+		self.zmq_loop.stop
 	end
 
 
@@ -287,14 +335,26 @@ class Arborist::Manager
 	end
 
 
-	### Start a timer that will save a snapshot of the node tree's state to the state
+	### Make a ZMQ::Timer that will publish a heartbeat event at a configurable interval.
+	def make_heartbeat_timer
+		interval = self.class.heartbeat_frequency || CONFIG_DEFAULTS[ :heartbeat_frequency ]
+
+		self.log.info "Setting up to heartbeat every %dms" % [ interval ]
+		heartbeat_timer = ZMQ::Timer.new( (interval/1000.0), 0 ) do
+			self.publish_heartbeat_event
+		end
+		return heartbeat_timer
+	end
+
+
+	### Make a ZMQ::Timer that will save a snapshot of the node tree's state to the state
 	### file on a configured interval if it's configured.
-	def start_state_checkpointing
+	def make_checkpoint_timer
 		return nil unless self.class.state_file
 		interval = self.class.checkpoint_frequency or return nil
 
-		self.log.info "Setting up node state checkpoint every %ds" % [ interval ]
-		checkpoint_timer = ZMQ::Timer.new( interval, 0 ) do
+		self.log.info "Setting up node state checkpoint every %dms" % [ interval ]
+		checkpoint_timer = ZMQ::Timer.new( (interval/1000.0), 0 ) do
 			self.save_node_states
 		end
 		return checkpoint_timer
@@ -311,15 +371,15 @@ class Arborist::Manager
 	### Set up a periodic ZMQ timer to check for queued signals and handle them.
 	def setup_signal_timer
 		@signal_timer = ZMQ::Timer.new( SIGNAL_INTERVAL, 0, self.method(:process_signal_queue) )
-		@zmq_loop.register_timer( @signal_timer )
+		self.zmq_loop.register_timer( @signal_timer )
 	end
 
 
 	### Disable the timer that checks for incoming signals
 	def cancel_signal_timer
-		if @signal_timer
-			@signal_timer.cancel
-			@zmq_loop.cancel_timer( @signal_timer )
+		if self.signal_timer
+			self.signal_timer.cancel
+			self.zmq_loop.cancel_timer( self.signal_timer )
 		end
 	end
 
@@ -403,6 +463,13 @@ class Arborist::Manager
 	end
 
 
+	### Simulate the receipt of the specified +signal+ (probably only useful
+	### in testing).
+	def simulate_signal( signal )
+		Thread.main[:signal_queue] << signal.to_sym
+	end
+
+
 	#
 	# :section: Tree API
 	#
@@ -461,7 +528,7 @@ class Arborist::Manager
 
 		if self.tree_built?
 			self.link_node( node )
-			node.handle_event( Arborist::Event.create(:sys_node_added, node) )
+			self.publish_system_event( 'node_added', node: identifier )
 		end
 	end
 
@@ -484,7 +551,7 @@ class Arborist::Manager
 		raise "Can't remove an operational node" if node.operational?
 
 		self.log.info "Removing node %p" % [ node ]
-		node.handle_event( Arborist::Event.create(:sys_node_removed, node) )
+		self.publish_system_event( 'node_removed', node: node.identifier )
 		node.children.each do |identifier, child_node|
 			self.remove_node( child_node )
 		end
@@ -555,6 +622,13 @@ class Arborist::Manager
 	# Tree-traversal API
 	#
 
+
+	### Return the current root node.
+	def root_node
+		return self.nodes[ '_' ]
+	end
+
+
 	### Yield each node in a depth-first traversal of the manager's tree
 	### to the specified +block+, or return an Enumerator if no block is given.
 	def all_nodes( &block )
@@ -639,6 +713,21 @@ class Arborist::Manager
 	end
 
 
+	### Publish a system event that observers can watch for to detect restarts.
+	def publish_heartbeat_event
+		self.publish_system_event( 'heartbeat', run_id: self.run_id )
+	end
+
+
+	### Publish an event with the specified +eventname+ and +data+.
+	def publish_system_event( eventname, **data )
+		eventname = eventname.to_s
+		eventname = 'sys.' + eventname unless eventname.start_with?( 'sys.' )
+		self.log.debug "Publishing %s event: %p." % [ eventname, data ]
+		self.event_publisher.publish( eventname, data )
+	end
+
+
 	### Create a subscription that publishes to the Manager's event publisher for
 	### the node with the specified +identifier+ and +event_pattern+, using the
 	### given +criteria+ when considering an event.