observability 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9e8e5db4293106c0b48ed3ea428c4a786eec948d11f19e8a78950ee49383655b
-  data.tar.gz: a9195f349e14fbc0721c22af67f8cb115bb15264a68dbf12ffd4243418f62167
+  metadata.gz: eb2bcfb9c815937bbc46798e8645ccc77e0a7e87dfa5024d327c9e9d4db5a1ff
+  data.tar.gz: c561389042c72073f21a5e18c06252a36177cd19022e6330c2633f8368953a8f
 SHA512:
-  metadata.gz: a851f201c9ff6757fb53bd14455b062fec2e4923d1386ee0192bc7f3033b3bb74ce48549b2b21ce4888ed674cb6f48050c89039b9d678d32baf85721fde965db
-  data.tar.gz: b7eee539d2e8a7ea9ae40262071b21f986945786f2409e3721342e182e5acb6bee27c25a2dc35ae5db1952aa434c32faec23203946b345cdfb88201fc7f4b4f9
+  metadata.gz: 4295af2f27a6d6eed728608896ae92fd79736d159c9b2f4c0595b5918708e6b6f63cde95d9824450daa741db481b0ecc7331c842fd43006eb03e704a35b9e603
+  data.tar.gz: e98b10c101567afa169e55d4b75dffa400d06491ff0320e3c9f01c72d73349ca413655697bb4e69b6493bfa676b2682ff1a48a035f3474de9552e349f9734fbf

data/History.md

@@ -1,3 +1,18 @@
+# Release History for observability
+
+---
+
+## v0.2.0 [2019-10-16] Michael Granger <ged@faeriemud.org>
+
+Improvements:
+
+- Add an experimental rabbitmq collector
+- Add a udp multicast sender
+- Handle exceptions with a #cause
+- Add instrumentation API
+- Update the timescale store schema
+
+
 ## v0.1.0 [2019-07-23] Michael Granger <ged@FaerieMUD.org>
 
 Initial release.

data/README.md

@@ -1,7 +1,10 @@
 # Observability
 
+home
+: https://hg.sr.ht/~ged/Observability
+
 code
-: http://bitbucket.org/ged/observability
+: https://hg.sr.ht/~ged/Observability/browse
 
 github
 : https://github.com/ged/observability
@@ -12,8 +15,8 @@ docs
 
 ## Description
 
-Observability is a toolkit for instrumenting code to make it more observable,
-following the principle of Observability-Oriented Design as expressed by Charity
+Observability is a toolkit for instrumenting code to make it more observable.
+It follows the principle of Observability-Oriented Design as expressed by Charity
 Majors (@mipsytipsy).
 
 Its goals are [stolen from https://charity.wtf/2019/02/05/logs-vs-structured-events/]:
@@ -46,7 +49,7 @@ Its goals are [stolen from https://charity.wtf/2019/02/05/logs-vs-structured-eve
 ## Contributing
 
 You can check out the current development source with Mercurial via its
-[project page][bitbucket]. Or if you prefer Git, via 
+[project page][sourcehut]. Or if you prefer Git, via
 [its Github mirror][github].
 
 After checking out the source, run:
@@ -57,6 +60,11 @@ This task will install any missing dependencies, run the tests/specs,
 and generate the API documentation.
 
 
+## Author
+
+- Michael Granger <ged@faeriemud.org>
+
+
 ## License
 
 Copyright (c) 2019, Michael Granger
@@ -88,6 +96,6 @@ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
-[bitbucket]: http://bitbucket.org/ged/observability
+[sourcehut]: https://hg.sr.ht/~ged/Observability
 [github]: https://github.com/ged/observability
 

data/lib/observability.rb

@@ -13,7 +13,7 @@ module Observability
 
 
 	# Package version
-	VERSION = '0.1.0'
+	VERSION = '0.2.0'
 
 	# Version control revision
 	REVISION = %q$Revision$
@@ -27,6 +27,7 @@ module Observability
 
 	autoload :Collector, 'observability/collector'
 	autoload :Event, 'observability/event'
+	autoload :Instrumentation, 'observability/instrumentation'
 	autoload :ObserverHooks, 'observability/observer_hooks'
 	autoload :Observer, 'observability/observer'
 	autoload :Sender, 'observability/sender'
@@ -59,6 +60,13 @@ module Observability
 	end
 
 
+	### Install the default instrumentatin for one or more +libraries+.
+	def self::install_instrumentation( *libraries )
+		Observability::Instrumentation.load( *libraries )
+		Observability::Instrumentation.install
+	end
+
+
 	### Return the current Observer, creating it if necessary.
 	def self::observer
 		unless @observer.complete?

data/lib/observability/collector/rabbitmq.rb

@@ -0,0 +1,196 @@
+# -*- ruby -*-
+# frozen_string_literal: true
+
+require 'json'
+require 'configurability'
+require 'bunny'
+
+require 'observability/collector' unless defined?( Observability::Collector )
+
+
+# A collector that re-injects events over AMQP to a RabbitMQ cluster.
+class Observability::Collector::RabbitMQ < Observability::Collector
+	extend Configurability,
+		Loggability
+
+	# The maximum size of event messages
+	MAX_EVENT_BYTES = 64 * 1024
+
+	# The number of seconds to wait between IO loops
+	LOOP_TIMER = 0.25
+
+	# Default options for publication
+	DEFAULT_PUBLISH_OPTIONS = {
+		mandatory:  false,
+		persistent: true
+	}
+
+
+	log_to :observability
+
+	configurability( 'observability.collector.rabbitmq' ) do
+
+		##
+		# The host to bind to
+		setting :host, default: 'localhost'
+
+		##
+		# The port to bind to
+		setting :port, default: 15775
+
+		##
+		# The broker_uri to use when connecting to RabbitMQ
+		setting :broker_uri
+
+		##
+		# The exchange to use when connecting to RabbitMQ
+		setting :exchange, default: 'events'
+
+		##
+		# The vhost to use when connecting to RabbitMQ
+		setting :vhost, default: '/telemetry'
+
+		##
+		# The heartbeat to use when connecting to RabbitMQ
+		setting :heartbeat, default: 'server' do |value|
+			value.to_sym if value
+		end
+
+		##
+		# Use single-threaded connections when set to `false`
+		setting :threaded, default: false
+
+	end
+
+
+	### Fetch a Hash of AMQP options.
+	def self::amqp_session_options
+		return {
+			logger: Loggability[ Observability ],
+			heartbeat: self.heartbeat,
+			exchange: self.exchange,
+			vhost: self.vhost,
+			threaded: self.threaded,
+		}
+	end
+
+
+	### Return a formatted list of the server's capabilities listed in +server_info+.
+	def self::capabilities_list( server_info )
+		server_info.
+			map {|name,enabled| enabled ? name : nil }.
+			compact.join(', ')
+	end
+
+
+	### Establish the connection to RabbitMQ based on the loaded configuration.
+	def self::configured_amqp_session
+		uri = self.broker_uri or raise "No broker_uri configured."
+		options = self.amqp_session_options
+
+		session = Bunny.new( uri, options )
+		session.start
+
+		self.log.info "Connected to %s v%s server: %s" % [
+			session.server_properties['product'],
+			session.server_properties['version'],
+			self.capabilities_list( session.server_properties['capabilities'] ),
+		]
+
+		return session
+	end
+
+
+
+	### Create a new UDP collector
+	def initialize
+		super
+
+		@socket        = UDPSocket.new
+		@amqp_session  = nil
+		@amqp_channel  = Concurrent::ThreadLocalVar.new { @amqp_session.create_channel }
+		@amqp_exchange = Concurrent::ThreadLocalVar.new do
+			@amqp_channel.value.headers( self.class.exchange, passive: true )
+		end
+		@processing    = false
+	end
+
+
+	######
+	public
+	######
+
+	### Start receiving events.
+	def start
+		self.log.info "Starting up."
+
+		@amqp_session = self.class.configured_amqp_session
+		@socket.bind( self.class.host, self.class.port )
+
+		self.start_processing
+	end
+
+
+	### Stop receiving events.
+	def stop
+		self.stop_processing
+
+		@socket.shutdown( :SHUT_RDWR )
+		@amqp_session.close
+	end
+
+
+	### Start consuming incoming events and storing them.
+	def start_processing
+		@processing = true
+		while @processing
+			event = self.read_next_event or next
+			self.log.debug "Read event: %p" % [ event ]
+			self.store_event( event )
+		end
+	end
+
+
+	### Stop consuming events.
+	def stop_processing
+		@processing = false
+	end
+
+
+	### Read the next event from the socket
+	def read_next_event
+		self.log.debug "Reading next event."
+		data = @socket.recv_nonblock( MAX_EVENT_BYTES, exception: false )
+
+		if data == :wait_readable
+			IO.select( [@socket], nil, nil, LOOP_TIMER )
+			return nil
+		elsif data.empty?
+			return nil
+		else
+			self.log.info "Read %d bytes" % [ data.bytesize ]
+			return JSON.parse( data )
+		end
+	end
+
+
+	### Store the specified +event+.
+	def store_event( event )
+		time    = event.delete( '@timestamp' )
+		type    = event.delete( '@type' )
+		version = event.delete( '@version' )
+
+		data = JSON.generate( event )
+		headers = {
+			time: time,
+			type: type,
+			version: version,
+			content_type: 'application/json',
+			content_encoding: data.encoding.name,
+			timestamp: Time.now.to_f,
+		}
+
+		@amqp_exchange.value.publish( data, headers )
+	end
+
+end # class Observability::Collector::RabbitMQ

data/lib/observability/collector/timescale.rb

@@ -83,7 +83,7 @@ class Observability::Collector::Timescale < Observability::Collector
 		self.stop_processing
 
 		@cursor = nil
-		@db.disconnct
+		@db.disconnect
 	end
 
 

data/lib/observability/instrumentation.rb

@@ -0,0 +1,118 @@
+# -*- ruby -*-
+# frozen_string_literal: true
+
+require 'set'
+require 'loggability'
+
+require 'observability' unless defined?( Observability )
+
+
+# Utilities for loading and installing pre-packaged instrumentation for common
+# libraries.
+module Observability::Instrumentation
+	extend Loggability
+
+
+	# Loggability API -- use the Observability logger for instrumentation
+	log_to :observability
+
+
+	##
+	# :singleton-method:
+	# The Set of Instrumentation modules that are laoded
+	singleton_class.attr_reader :modules
+	@modules = Set.new
+
+
+	### Load instrumentation for the specified +libraries+.
+	def self::load( *libraries )
+		libraries.flatten.each do |library|
+			libfile = "observability/instrumentation/%s" % [ library ]
+			require( libfile )
+		end
+
+		return self
+	end
+
+
+	### Extension callback -- declare some instance variables in the extending
+	### +mod+.
+	def self::extended( mod )
+		super
+
+		if self.modules.add?( mod )
+			self.log.info "Loaded %p" % [ mod ]
+			mod.extend( Loggability )
+			mod.log_to( :observability )
+			mod.instance_variable_set( :@depends_on, Set.new )
+			mod.instance_variable_set( :@installation_callbacks, Set.new )
+			mod.singleton_class.attr_reader( :installation_callbacks )
+		else
+			self.log.warn "Already loaded %p" % [ mod ]
+		end
+	end
+
+
+	### Install loaded instrumentation if the requisite modules are present.
+	def self::install
+		self.modules.each do |mod|
+			mod.install if mod.available?
+		end
+	end
+
+
+	### Returns +true+ if each of the given +module_names+ are defined and are
+	### Module objects.
+	def self::dependencies_met?( *module_names )
+		return module_names.flatten.all? do |mod|
+			self.check_for_module( mod )
+		end
+	end
+
+
+	### Returns +true+ if +mod+ is defined and is a Module.
+	def self::check_for_module( mod )
+		self.log.debug "Checking for presence of `%s` module..." % [ mod ]
+		Object.const_defined?( mod ) && Object.const_get( mod ).is_a?( Module )
+	end
+
+
+	### Declare +modules+ that must be available for instrumentation to be loaded.
+	### If they are not present when the instrumentation loads, it will be skipped
+	### entirely.
+	def depends_on( *modules )
+		@depends_on.merge( modules.flatten(1) )
+		return @depends_on
+	end
+
+
+	### Register a +callback+ that will be called when instrumentation is installed,
+	### if and only if all of the given +modules+ are present (may be empty).
+	def when_installed( *modules, &callback )
+		self.installation_callbacks.add( [callback, modules] )
+	end
+
+
+	### Returns +true+ if all of the modules registered with #depends_on are defined.
+	def available?
+		return Observability::Instrumentation.dependencies_met?( self.depends_on.to_a )
+	end
+
+
+	### Call installation callbacks which meet their prerequistes.
+	def install
+		self.installation_callbacks.each do |callback, dependencies|
+			missing = dependencies.
+				reject {|mod| Observability::Instrumentation.check_for_module(mod) }
+
+			if missing.empty?
+				self.log.debug "Instrumenting %s: %p" % [ dependencies.join(', '), callback ]
+				callback.call
+			else
+				self.log.info "Skipping %p: missing %s" % [ callback, missing.join(', ') ]
+			end
+		end
+	end
+
+end # module Observability::Instrumentation
+

data/lib/observability/instrumentation/bunny.rb

@@ -0,0 +1,24 @@
+# -*- ruby -*-
+# frozen_string_literal: true
+
+require 'observability/instrumentation' unless defined?( Observability::Instrumentation )
+
+
+# Instrumentation for the Bunny RabbitMQ client library
+# Refs:
+# - http://rubybunny.info/
+module Observability::Instrumentation::Bunny
+	extend Observability::Instrumentation
+
+	depends_on 'Bunny'
+
+
+	when_installed( 'Bunny::Session' ) do
+		Bunny::Session.extend( Observability )
+		Bunny::Session.observe_class_method( :new )
+		Bunny::Session.observe_method( :start )
+	end
+
+
+end # module Observability::Instrumentation::Bunny
+