loggability 0.15.0.pre20190714094638 → 0.18.1

data/lib/loggability.rb

@@ -9,10 +9,7 @@ require 'date'
 module Loggability
 
 	# Package version constant
-	VERSION = '0.14.0'
-
-	# VCS revision
-	REVISION = %q$Revision$
+	VERSION = '0.18.1'
 
 	# The key for the global logger (Loggability's own logger)
 	GLOBAL_KEY = :__global__
@@ -29,22 +26,28 @@ module Loggability
 	LOGSPEC_PATTERN = %r{
 		^
 			\s*
-			((?i:debug|info|warn|error|fatal))   # severity
+			(?<severity>(?i:debug|info|warn|error|fatal))
 		    (?:
 				\s+
-				((?:[\w\-/:\.]|\\[ ])+)
+				(?<target>(?:[\w\-/:\.\[\]]|\\[ ])+)
 			)?
 			(?: \s+\(
-				(\w+)
+				(?<format>\w+)
 			\) )?
 			\s*
 		$
 	}x
 
-	require 'loggability/constants'
-	include Loggability::Constants
 
-	require 'loggability/logger'
+	# Automatically load subordinate classes/modules
+	autoload :Constants, 'loggability/constants'
+	autoload :LogDevice, 'loggability/log_device'
+	autoload :Logger, 'loggability/logger'
+	autoload :LogHost, 'loggability/loghost'
+	autoload :LogClient, 'loggability/logclient'
+	autoload :Override, 'loggability/override'
+
+	include Loggability::Constants
 
 
 	##
@@ -58,18 +61,6 @@ module Loggability
 	@config = CONFIG_DEFAULTS.dup.freeze
 
 
-	# Automatically log the log host and log client mixins when they're referenced
-	autoload :LogHost, 'loggability/loghost'
-	autoload :LogClient, 'loggability/logclient'
-	autoload :Override, 'loggability/override'
-
-
-	### Return the library's version string
-	def self::version_string( include_buildnum=false )
-		vstring = "%s %s" % [ self.name, VERSION ]
-		vstring << " (build %s)" % [ REVISION[/: ([[:xdigit:]]+)/, 1] || '0' ] if include_buildnum
-		return vstring
-	end
 
 
 	### Cast the given +device+ to a Loggability::Logger, if possible, and return it. If
@@ -155,12 +146,12 @@ module Loggability
 
 	### Call the method with the given +methodname+ across the loggers of all loghosts with
 	### the given +arg+ and/or +block+.
-	def self::aggregate( methodname, arg, &block )
+	def self::aggregate( methodname, *args, &block )
 		# self.log.debug "Aggregating a call to %p with %p to %d log hosts" %
 		#	[ methodname, arg, Loggability.log_hosts.length ]
 		Loggability.log_hosts.values.each do |loghost|
 			# self.log.debug "  %p.logger.%s( %p )" % [ loghost, methodname, arg ]
-			loghost.logger.send( methodname, arg, &block )
+			loghost.logger.send( methodname, *args, &block )
 		end
 	end
 
@@ -200,8 +191,8 @@ module Loggability
 	#
 	# Aggregate method: set all loggers to log to +destination+. See Loggability::Logger#output_to
 	# for more info.
-	def self::output_to( newdevice )
-		self.aggregate( :output_to, newdevice )
+	def self::output_to( newdevice, *args )
+		self.aggregate( :output_to, newdevice, *args )
 	end
 	class << self
 		alias_method :write_to, :output_to
@@ -332,6 +323,7 @@ module Loggability
 		target = case target
 			when 'STDOUT' then $stdout
 			when 'STDERR' then $stderr
+			when /:/ then Loggability::LogDevice.parse_device_spec( target )
 			else
 				target
 			end

data/lib/loggability/formatter/structured.rb

@@ -0,0 +1,35 @@
+# -*- ruby -*-
+# vim: set nosta noet ts=4 sw=4:
+# frozen_string_literal: true
+
+require 'time'
+require 'json'
+
+require 'loggability/formatter' unless defined?( Loggability::Formatter )
+
+
+# Output logs as JSON.
+class Loggability::Formatter::Structured < Loggability::Formatter
+
+	# The version of the format output
+	LOG_FORMAT_VERSION = 1
+
+
+	### Format a message of the specified +severity+ using the given +time+,
+	### +progname+, and +message+.
+	def call( severity, time, progname, message )
+		severity ||= 'DEBUG'
+		time ||= Time.now
+		entry = {
+			'@version' => LOG_FORMAT_VERSION,
+			'@timestamp' => time.iso8601( 3 ),
+			'level' => severity,
+			'progname' => progname,
+			'message' => message,
+		}
+
+		return JSON.generate( entry )
+	end
+
+end # class Loggability::Formatter::Default
+

data/lib/loggability/log_device.rb

@@ -0,0 +1,86 @@
+#!/usr/bin/env ruby
+# vim: set nosta noet ts=4 sw=4:
+# encoding: utf-8
+
+require 'loggability' unless defined?( Loggability )
+
+
+# An abstract base class for logging devices. A device manages the actual writing of messages
+# to whatever destination logs are supposed to be shipped to, along with any buffering,
+# encoding, or serialization that needs to be done.
+#
+# Log devices are loadable by name via the ::create method if they are declared in a
+# directory named `loggability/log_device/` in the gem path.
+#
+# Concrete log devices are required to implement two methods: #write and #close.
+#
+# [write]
+#   Takes one argument, which is the message that needs to be written.
+#
+# [close]
+#   Close any open filehandles or connections established by the device.
+class Loggability::LogDevice
+
+
+	# Regexp used to split up logging devices in config lines
+	DEVICE_TARGET_REGEX = /^([\s*a-z]\w*)(?:\[(.*)\])?/
+
+
+	### Parses out the target class name and its arguments from the +target_spec+
+	### then requires the subclass and instantiates it by passing the arguments.
+	### The +target_spec+ comes from a config file in the format of:
+	###
+	###   logging:
+	###     datadog[data_dog_api_key]
+	###
+	### In the above example:
+	### * "datadog" is the log device to send logs to
+	### * "data_dog_api_key" is the argument that will be passed onto the datadog
+	###   log device's constructor
+	def self::parse_device_spec( target_spec )
+		targets = target_spec.split( ';' ).compact
+		return targets.map do |t|
+			target_subclass = t[ DEVICE_TARGET_REGEX, 1 ]&.strip.to_sym
+			target_subclass_args = t[ DEVICE_TARGET_REGEX, 2 ]
+
+			self.create( target_subclass, target_subclass_args )
+		end
+	end
+
+
+	### Requires the subclass and instantiates it with the passed-in arguments and
+	### then returns an instance of it.
+	def self::create( target, *target_args )
+		modname = target.to_s.capitalize
+
+		self.load_device_type( target ) unless self.const_defined?( modname, false )
+		subclass = self.const_get( modname, false )
+
+		return subclass.new( *target_args )
+	rescue NameError => err
+		raise LoadError, "failed to load %s LogDevice: %s" % [ target, err.message ]
+	end
+
+
+	### Attempt to load a LogDevice of the given +type+.
+	def self::load_device_type( type )
+		require_path = "loggability/log_device/%s" % [ type.to_s.downcase ]
+		require( require_path )
+	end
+
+
+	### Write a +message+ to the device. This needs to be overridden by concrete
+	### subclasses; calling this implementation will raise an NotImplementedError.
+	def write( message )
+		raise NotImplementedError, "%s is not implemented by %s" % [ __callee__, self.class.name ]
+	end
+
+
+	### Close the device. This needs to be overridden by concrete subclasses;
+	### calling this implementation will raise an NotImplementedError.
+	def close
+		raise NotImplementedError, "%s is not implemented by %s" % [ __callee__, self.class.name ]
+	end
+
+end # class Loggability::LogDevice
+

data/lib/loggability/log_device/appending.rb

@@ -0,0 +1,34 @@
+#!/usr/bin/env ruby
+# vim: set nosta noet ts=4 sw=4:
+# encoding: utf-8
+
+require 'loggability/logger' unless defined?( Loggability::Logger )
+
+# A log device that appends to the object it's constructed with instead of writing
+# to a file descriptor or a file.
+class Loggability::LogDevice::Appending < Loggability::LogDevice
+
+	### Create a new +Appending+ log device that will append content to +array+.
+	def initialize( target )
+		@target = target || []
+	end
+
+
+	######
+	public
+	######
+
+	# The target of the log device
+	attr_reader :target
+
+
+	### Append the specified +message+ to the target.
+	def write( message )
+		@target << message
+	end
+
+
+	### No-op -- this is here just so Logger doesn't complain
+	def close; end
+
+end # class Loggability::LogDevice::Appending

data/lib/loggability/log_device/datadog.rb

@@ -0,0 +1,90 @@
+#!/usr/bin/env ruby
+# vim: set nosta noet ts=4 sw=4:
+# encoding: utf-8
+
+require 'uri'
+require 'socket'
+require 'net/https'
+require 'json'
+require 'concurrent'
+require 'loggability/logger' unless defined?( Loggability::Logger )
+
+require 'loggability/log_device/http'
+
+
+# A log device that sends logs to Datadog's HTTP endpoint
+# for receiving logs
+class Loggability::LogDevice::Datadog < Loggability::LogDevice::Http
+
+	### Datadog's HTTP endpoint URL for sending logs to
+	DEFAULT_ENDPOINT = URI( "https://http-intake.logs.datadoghq.com/v1/input" )
+
+	### The max number of messages that can be sent to datadog in a single payload
+	MAX_BATCH_SIZE = 480
+
+	### The max size in bytes for a single message.
+	### Limiting the message size to 200kB	to leave room for other info such as
+	### tags, metadata, etc.
+	### DataDog's max size for a single log entry is 256kB
+	MAX_MESSAGE_BYTESIZE = 204_800
+
+	### The max size in bytes of all messages in the batch.
+	### Limiting the total messages size to 4MB to leave room for other info such as
+	### tags, metadata, etc.
+	### Datadog's max size for the entire payload is 5MB
+	MAX_BATCH_BYTESIZE = 4_194_304
+
+	# Override the default HTTP device options for sending logs to DD
+	DEFAULT_OPTIONS = {
+		max_batch_size: MAX_BATCH_SIZE,
+		max_message_bytesize: MAX_MESSAGE_BYTESIZE,
+		max_batch_bytesize: MAX_BATCH_BYTESIZE,
+	}
+
+
+	### Create a new Datadog
+	def initialize( api_key, endpoint=DEFAULT_ENDPOINT, options={} )
+		if endpoint.is_a?( Hash )
+			options = endpoint
+			endpoint = DEFAULT_ENDPOINT
+		end
+
+		super( endpoint, options )
+
+		@api_key = api_key
+		@hostname = Socket.gethostname
+	end
+
+
+	######
+	public
+	######
+
+	##
+	# The name of the current host
+	attr_reader :hostname
+
+	##
+	# The configured Datadog API key
+	attr_reader :api_key
+
+
+	### Format an individual log +message+ for Datadog.
+	def format_log_message( message )
+		return {
+			hostname: self.hostname,
+			message: message
+		}.to_json
+	end
+
+
+	### Overridden to add the configured API key to the headers of each request.
+	def make_batch_request
+		request = super
+
+		request[ 'DD-API-KEY' ] = self.api_key
+
+		return request
+	end
+
+end # class Loggability::LogDevice::Datadog

data/lib/loggability/log_device/file.rb

@@ -0,0 +1,37 @@
+#!/usr/bin/env ruby
+# vim: set nosta noet ts=4 sw=4:
+# encoding: utf-8
+
+require 'logger'
+require 'loggability/logger' unless defined?( Loggability::Logger )
+
+# A log device that delegates to the ruby's default Logger's file device and writes
+# to a file descriptor or a file.
+class Loggability::LogDevice::File < Loggability::LogDevice
+
+	### Create a new +File+ device that will write to the file using the built-in ruby's +File+ log device
+	def initialize( target )
+		@target = ::Logger::LogDevice.new( target )
+	end
+
+
+	######
+	public
+	######
+
+	# The target of the log device
+	attr_reader :target
+
+
+	### Append the specified +message+ to the target.
+	def write( message )
+		self.target.write( message )
+	end
+
+
+	### close the file
+	def close
+		self.target.close
+	end
+
+end # class Loggability::LogDevice::File

data/lib/loggability/log_device/http.rb

@@ -0,0 +1,310 @@
+#!/usr/bin/env ruby
+# vim: set nosta noet ts=4 sw=4:
+# encoding: utf-8
+
+require 'socket'
+require 'uri'
+require 'net/https'
+require 'json'
+require 'concurrent'
+require 'loggability/logger' unless defined?( Loggability::Logger )
+
+require 'loggability/log_device'
+
+# This is the a generalized class that allows its subclasses to send log
+# messages to HTTP endpoints asynchronously on a separate thread.
+class Loggability::LogDevice::Http < Loggability::LogDevice
+
+	# The default HTTP endpoint URL to send logs to
+	DEFAULT_ENDPOINT = "http://localhost:12775/v1/logs"
+
+	# The default maximum number of messages that can be sent to the server in a single payload
+	DEFAULT_MAX_BATCH_SIZE = 100
+
+	# The default max size in bytes for a single message.
+	DEFAULT_MAX_MESSAGE_BYTESIZE = 2 * 16
+
+	# The default number of seconds between batches
+	DEFAULT_BATCH_INTERVAL = 60
+
+	# The default number of seconds to wait for data to be written before timing out
+	DEFAULT_WRITE_TIMEOUT = 15
+
+	# The default Executor class to use for asynchronous tasks
+	DEFAULT_EXECUTOR_CLASS = Concurrent::SingleThreadExecutor
+
+	# The default for the maximum bytesize of the queue (1 GB)
+	DEFAULT_MAX_QUEUE_BYTESIZE = ( 2 ** 10 ) * ( 2 ** 10 ) * ( 2 ** 10 )
+
+	# The default options for new instances
+	DEFAULT_OPTIONS = {
+		execution_interval: DEFAULT_BATCH_INTERVAL,
+		write_timeout: DEFAULT_WRITE_TIMEOUT,
+		max_batch_size: DEFAULT_MAX_BATCH_SIZE,
+		max_message_bytesize: DEFAULT_MAX_MESSAGE_BYTESIZE,
+		executor_class: DEFAULT_EXECUTOR_CLASS,
+	}
+
+
+	### Initialize the HTTP log device to send to the specified +endpoint+ with the
+	### given +options+. Valid options are:
+	###
+	### [:batch_interval]
+	###   Maximum number of seconds between batches
+	### [:write_timeout]
+	###   How many seconds to wait for data to be written while sending a batch
+	### [:max_batch_size]
+	###   The maximum number of messages that can be in a single batch
+	### [:max_batch_bytesize]
+	###   The maximum number of bytes that can be in the payload of a single batch
+	### [:max_message_bytesize]
+	###   The maximum number of bytes that can be in a single message
+	### [:executor_class]
+	###   The Concurrent executor class to use for asynchronous tasks.
+	def initialize( endpoint=DEFAULT_ENDPOINT, opts={} )
+		if endpoint.is_a?( Hash )
+			opts = endpoint
+			endpoint = DEFAULT_ENDPOINT
+		end
+
+		opts = DEFAULT_OPTIONS.merge( opts )
+
+		@endpoint             = URI( endpoint ).freeze
+		@logs_queue           = Queue.new
+
+		@logs_queue_bytesize  = 0
+		@max_queue_bytesize   = opts[:max_queue_bytesize] || DEFAULT_MAX_QUEUE_BYTESIZE
+		@batch_interval       = opts[:batch_interval] || DEFAULT_BATCH_INTERVAL
+		@write_timeout        = opts[:write_timeout] || DEFAULT_WRITE_TIMEOUT
+		@max_batch_size       = opts[:max_batch_size] || DEFAULT_MAX_BATCH_SIZE
+		@max_message_bytesize = opts[:max_message_bytesize] || DEFAULT_MAX_MESSAGE_BYTESIZE
+		@executor_class       = opts[:executor_class] || DEFAULT_EXECUTOR_CLASS
+
+		@max_batch_bytesize   = opts[:max_batch_bytesize] || @max_batch_size * @max_message_bytesize
+		@last_send_time       = Concurrent.monotonic_time
+	end
+
+
+	######
+	public
+	######
+
+	##
+	# The single thread pool executor
+	attr_reader :executor
+
+	##
+	# The URI of the endpoint to send messages to
+	attr_reader :endpoint
+
+	##
+	# The Queue that contains any log messages which have not yet been sent to the
+	# logging service.
+	attr_reader :logs_queue
+
+	##
+	# The max bytesize of the queue. Will not queue more messages if this threshold is hit
+	attr_reader :max_queue_bytesize
+
+	##
+	# The size of +logs_queue+ in bytes
+	attr_accessor :logs_queue_bytesize
+
+	##
+	# The monotonic clock time when the last batch of logs were sent
+	attr_accessor :last_send_time
+
+	##
+	# Number of seconds after the task completes before the task is performed again.
+	attr_reader :batch_interval
+
+	##
+	# How many seconds to wait for data to be written while sending a batch
+	attr_reader :write_timeout
+
+	##
+	# The maximum number of messages to post at one time
+	attr_reader :max_batch_size
+
+	##
+	# The maximum number of bytes of a single message to include in a batch
+	attr_reader :max_message_bytesize
+
+	##
+	# The maximum number of bytes that will be included in a single POST
+	attr_reader :max_batch_bytesize
+
+	##
+	# The Concurrent executor class to use for asynchronous tasks
+	attr_reader :executor_class
+
+	##
+	# The timer task thread
+	attr_reader :timer_task
+
+
+	### LogDevice API -- write a message to the HTTP device.
+	def write( message )
+		self.start unless self.running?
+		if message.is_a?( Hash )
+			message_size = message.to_json.bytesize
+		else
+			message_size = message.bytesize
+		end
+		return if ( self.logs_queue_bytesize + message_size ) >= self.max_queue_bytesize
+		self.logs_queue_bytesize += message_size
+		self.logs_queue.enq( message )
+		self.send_logs
+	end
+
+
+	### LogDevice API -- stop the batch thread and close the http connection
+	def close
+		self.stop
+		self.http_client.finish
+	rescue IOError
+		# ignore it since http session has not yet started.
+	end
+
+
+	### Starts a thread pool with a single thread.
+	def start
+		self.start_executor
+		self.start_timer_task
+	end
+
+
+	### Returns +true+ if the device has started sending messages to the logging endpoint.
+	def running?
+		return self.executor&.running?
+	end
+
+
+	### Shutdown the executor, which is a pool of single thread
+	### waits 3 seconds for shutdown to complete
+	def stop
+		return unless self.running?
+
+		self.timer_task.shutdown if self.timer_task&.running?
+		self.executor.shutdown
+
+		unless self.executor.wait_for_termination( 3 )
+			self.executor.halt
+			self.executor.wait_for_termination( 3 )
+		end
+	end
+
+
+	### Start the background thread that sends messages.
+	def start_executor
+		@executor = self.executor_class.new
+		@executor.auto_terminate = true unless @executor.serialized?
+	end
+
+
+	### Create a timer task that calls that sends logs at regular interval
+	def start_timer_task
+		@timer_task = Concurrent::TimerTask.execute( execution_interval: self.batch_interval ) do
+			self.send_logs
+		end
+	end
+
+
+	### Sends a batch of log messages to the logging service. This executes inside
+	### the sending thread.
+	def send_logs
+		self.executor.post do
+			if self.batch_ready?
+				# p "Batch ready; sending."
+				request = self.make_batch_request
+				request.body = self.get_next_log_payload
+
+				# p "Sending request", request
+
+				self.http_client.request( request ) do |res|
+					p( res ) if $DEBUG
+				end
+
+				self.last_send_time = Concurrent.monotonic_time
+			else
+				# p "Batch not ready yet."
+			end
+		end
+	end
+
+
+	### Returns +true+ if a batch of logs is ready to be sent.
+	def batch_ready?
+		seconds_since_last_send = Concurrent.monotonic_time - self.last_send_time
+
+		return self.logs_queue.size >= self.max_batch_size ||
+			seconds_since_last_send >= self.batch_interval
+	end
+	alias_method :has_batch_ready?, :batch_ready?
+
+
+	### Returns a new HTTP request (a subclass of Net::HTTPRequest) suitable for
+	### sending the next batch of logs to the service. Defaults to a POST of JSON data. This
+	### executes inside the sending thread.
+	def make_batch_request
+		request = Net::HTTP::Post.new( self.endpoint.path )
+		request[ 'Content-Type' ] = 'application/json'
+
+		return request
+	end
+
+
+	### Dequeue pending log messages to send to the service and return them as a
+	### suitably-encoded String. The default is a JSON Array. This executes inside
+	### the sending thread.
+	def get_next_log_payload
+		buf = []
+		count = 0
+		bytes = 0
+
+		# Be conservative so as not to overflow
+		max_size = self.max_batch_bytesize - self.max_message_bytesize - 2 # for the outer Array
+
+		while count < self.max_batch_size && bytes < max_size && !self.logs_queue.empty?
+			message = self.logs_queue.deq
+			formatted_message = self.format_log_message( message )
+			self.logs_queue_bytesize -= message.bytesize
+
+			count += 1
+			bytes += formatted_message.bytesize + 3 # comma and delimiters
+
+			buf << formatted_message
+		end
+
+		return '[' + buf.join(',') + ']'
+	end
+
+
+	### Returns the given +message+ in whatever format individual log messages are
+	### expected to be in by the service. The default just returns the stringified
+	### +message+. This executes inside the sending thread.
+	def format_log_message( message )
+		return message.to_s[ 0 ... self.max_message_bytesize ].dump
+	end
+
+
+	### sets up a configured http object ready to instantiate connections
+	def http_client
+		return @http_client ||= begin
+			uri = URI( self.endpoint )
+
+			http = Net::HTTP.new( uri.host, uri.port )
+			http.write_timeout = self.write_timeout
+
+			if uri.scheme == 'https'
+				http.use_ssl = true
+				http.verify_mode = OpenSSL::SSL::VERIFY_PEER
+			end
+
+			http
+		end
+	end
+
+
+end # class Loggability::LogDevice::Http
+