loggability 0.15.1 → 0.16.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4dd5d6f5605a1a639a0143107b4079272e35bb7ffba9e2b76b150c1fab57e11f
-  data.tar.gz: aef9016eb5f208d0d168d8e4aac0b03047851b582f8afcb437b37bb9f9b5ad4d
+  metadata.gz: '08efe977f97d6ff78421b1ff180716378aa56afb83b0aac80cf36550f4509e00'
+  data.tar.gz: 1af9769dad5f2eca223b413f1ead75dcbe2e322dc96f4d9598d3f275d3fa661a
 SHA512:
-  metadata.gz: e14c6919279635835bd126a7c878ef69149811640cd4c632022e1677c18abfb3ebadc67fe8a12e9d35e35a94296e2c3ca32614c0c998ed9089b8e9b72d25ec56
-  data.tar.gz: d2481edb5e1a26dcc7f2a78420831d127b9951dd36e02e898bbd7fded829895083071627d9d78616fbad692937eb05edbdd39456efa0034aae4647cce597c484
+  metadata.gz: 9fd3be386367fab5fbf7b1afba29e234f452160af4514098880483d6d4c377bc201c4558aad2e1e19e8ddbd2c24afbef34cf25fe6047a7fbaa2a08c742141de5
+  data.tar.gz: e96d9bd362d3709c3035938e03fa303c635846c1722a270b2d9286c0736c46d103f00d05f5391ad96b3dbcbcc5c3f5262378e62a38ac704ccaddc53dc9644089

data/History.rdoc

@@ -1,6 +1,15 @@
 = Release History for loggability
 
 ---
+== v0.16.0 [2020-02-24] Michael Granger <ged@faeriemud.org>
+
+Improvements:
+
+- Expose a "log device" API and add the ability to write to multiple devices
+- Add a device for logging to an HTTP service
+
+Thanks to Mahmood Khan <mkhan1484@gmail.com> for the patch.
+
 
 == v0.15.1 [2020-01-09] Michael Granger <ged@faeriemud.org>
 

data/Manifest.txt

@@ -1,9 +1,5 @@
 .simplecov
-ChangeLog
 History.rdoc
-Manifest.txt
-README.md
-Rakefile
 lib/loggability.rb
 lib/loggability/constants.rb
 lib/loggability/formatter.rb
@@ -14,8 +10,16 @@ lib/loggability/formatter/structured.rb
 lib/loggability/logclient.rb
 lib/loggability/logger.rb
 lib/loggability/loghost.rb
+lib/loggability/log_device.rb
+lib/loggability/log_device/appending.rb
+lib/loggability/log_device/datadog.rb
+lib/loggability/log_device/file.rb
+lib/loggability/log_device/http.rb
 lib/loggability/override.rb
 lib/loggability/spechelpers.rb
+Manifest.txt
+Rakefile
+README.md
 spec/helpers.rb
 spec/loggability/formatter/color_spec.rb
 spec/loggability/formatter/default_spec.rb
@@ -24,6 +28,10 @@ spec/loggability/formatter/structured_spec.rb
 spec/loggability/formatter_spec.rb
 spec/loggability/logger_spec.rb
 spec/loggability/loghost_spec.rb
+spec/loggability/log_device/appending_spec.rb
+spec/loggability/log_device/datadog_spec.rb
+spec/loggability/log_device/file_spec.rb
+spec/loggability/log_device/http_spec.rb
 spec/loggability/override_spec.rb
 spec/loggability/spechelpers_spec.rb
 spec/loggability_spec.rb

data/lib/loggability.rb

@@ -9,7 +9,7 @@ require 'date'
 module Loggability
 
 	# Package version constant
-	VERSION = '0.15.1'
+	VERSION = '0.16.0'
 
 	# The key for the global logger (Loggability's own logger)
 	GLOBAL_KEY = :__global__
@@ -29,7 +29,7 @@ module Loggability
 			(?<severity>(?i:debug|info|warn|error|fatal))
 		    (?:
 				\s+
-				(?<target>(?:[\w\-/:\.]|\\[ ])+)
+				(?<target>(?:[\w\-/:\.\[\]]|\\[ ])+)
 			)?
 			(?: \s+\(
 				(?<format>\w+)
@@ -38,10 +38,16 @@ module Loggability
 		$
 	}x
 
-	require 'loggability/constants'
-	include Loggability::Constants
 
-	require 'loggability/logger'
+	# Automatically load subordinate classes/modules
+	autoload :Constants, 'loggability/constants'
+	autoload :LogDevice, 'loggability/log_device'
+	autoload :Logger, 'loggability/logger'
+	autoload :LogHost, 'loggability/loghost'
+	autoload :LogClient, 'loggability/logclient'
+	autoload :Override, 'loggability/override'
+
+	include Loggability::Constants
 
 
 	##
@@ -55,10 +61,6 @@ module Loggability
 	@config = CONFIG_DEFAULTS.dup.freeze
 
 
-	# Automatically log the log host and log client mixins when they're referenced
-	autoload :LogHost, 'loggability/loghost'
-	autoload :LogClient, 'loggability/logclient'
-	autoload :Override, 'loggability/override'
 
 
 	### Cast the given +device+ to a Loggability::Logger, if possible, and return it. If
@@ -321,6 +323,7 @@ module Loggability
 		target = case target
 			when 'STDOUT' then $stdout
 			when 'STDERR' then $stderr
+			when /:/ then Loggability::LogDevice.parse_device_spec( target )
 			else
 				target
 			end

data/lib/loggability/log_device.rb

@@ -0,0 +1,86 @@
+#!/usr/bin/env ruby
+# vim: set nosta noet ts=4 sw=4:
+# encoding: utf-8
+
+require 'loggability' unless defined?( Loggability )
+
+
+# An abstract base class for logging devices. A device manages the actual writing of messages
+# to whatever destination logs are supposed to be shipped to, along with any buffering,
+# encoding, or serialization that needs to be done.
+#
+# Log devices are loadable by name via the ::create method if they are declared in a
+# directory named `loggability/log_device/` in the gem path.
+#
+# Concrete log devices are required to implement two methods: #write and #close.
+#
+# [write]
+#   Takes one argument, which is the message that needs to be written.
+#
+# [close]
+#   Close any open filehandles or connections established by the device.
+class Loggability::LogDevice
+
+
+	# Regexp used to split up logging devices in config lines
+	DEVICE_TARGET_REGEX = /^([\s*a-z]\w*)(?:\[(.*)\])?/
+
+
+	### Parses out the target class name and its arguments from the +target_spec+
+	### then requires the subclass and instantiates it by passing the arguments.
+	### The +target_spec+ comes from a config file in the format of:
+	###
+	###   logging:
+	###     datadog[data_dog_api_key]
+	###
+	### In the above example:
+	### * "datadog" is the log device to send logs to
+	### * "data_dog_api_key" is the argument that will be passed onto the datadog
+	###   log device's constructor
+	def self::parse_device_spec( target_spec )
+		targets = target_spec.split( ':' ).compact
+		return targets.map do |t|
+			target_subclass = t[ DEVICE_TARGET_REGEX, 1 ]&.strip().to_sym
+			target_subclass_args = t[ DEVICE_TARGET_REGEX, 2 ]
+
+			self.create( target_subclass, target_subclass_args )
+		end
+	end
+
+
+	### Requires the subclass and instantiates it with the passed-in arguments and
+	### then returns an instance of it.
+	def self::create( target, *target_args )
+		modname = target.to_s.capitalize
+
+		self.load_device_type( target ) unless self.const_defined?( modname, false )
+		subclass = self.const_get( modname, false )
+
+		return subclass.new( *target_args )
+	rescue NameError => err
+		raise LoadError, "failed to load %s LogDevice: %s" % [ target, err.message ]
+	end
+
+
+	### Attempt to load a LogDevice of the given +type+.
+	def self::load_device_type( type )
+		require_path = "loggability/log_device/%s" % [ type.to_s.downcase ]
+		require( require_path )
+	end
+
+
+	### Write a +message+ to the device. This needs to be overridden by concrete
+	### subclasses; calling this implementation will raise an NotImplementedError.
+	def write( message )
+		raise NotImplementedError, "%s is not implemented by %s" % [ __callee__, self.class.name ]
+	end
+
+
+	### Close the device. This needs to be overridden by concrete subclasses;
+	### calling this implementation will raise an NotImplementedError.
+	def close
+		raise NotImplementedError, "%s is not implemented by %s" % [ __callee__, self.class.name ]
+	end
+
+end # class Loggability::LogDevice
+

data/lib/loggability/log_device/appending.rb

@@ -0,0 +1,34 @@
+#!/usr/bin/env ruby
+# vim: set nosta noet ts=4 sw=4:
+# encoding: utf-8
+
+require 'loggability/logger' unless defined?( Loggability::Logger )
+
+# A log device that appends to the object it's constructed with instead of writing
+# to a file descriptor or a file.
+class Loggability::LogDevice::Appending < Loggability::LogDevice
+
+	### Create a new +Appending+ log device that will append content to +array+.
+	def initialize( target )
+		@target = target || []
+	end
+
+
+	######
+	public
+	######
+
+	# The target of the log device
+	attr_reader :target
+
+
+	### Append the specified +message+ to the target.
+	def write( message )
+		@target << message
+	end
+
+
+	### No-op -- this is here just so Logger doesn't complain
+	def close; end
+
+end # class Loggability::LogDevice::Appending

data/lib/loggability/log_device/datadog.rb

@@ -0,0 +1,90 @@
+#!/usr/bin/env ruby
+# vim: set nosta noet ts=4 sw=4:
+# encoding: utf-8
+
+require 'uri'
+require 'socket'
+require 'net/https'
+require 'json'
+require 'concurrent'
+require 'loggability/logger' unless defined?( Loggability::Logger )
+
+require 'loggability/log_device/http'
+
+
+# A log device that sends logs to Datadog's HTTP endpoint
+# for receiving logs
+class Loggability::LogDevice::Datadog < Loggability::LogDevice::Http
+
+	### Datadog's HTTP endpoint URL for sending logs to
+	DEFAULT_ENDPOINT = URI( "https://http-intake.logs.datadoghq.com/v1/input" )
+
+	### The max number of messages that can be sent to datadog in a single payload
+	MAX_BATCH_SIZE = 480
+
+	### The max size in bytes for a single message.
+	### Limiting the message size to 200kB	to leave room for other info such as
+	### tags, metadata, etc.
+	### DataDog's max size for a single log entry is 256kB
+	MAX_MESSAGE_BYTESIZE = 204_800
+
+	### The max size in bytes of all messages in the batch.
+	### Limiting the total messages size to 4MB to leave room for other info such as
+	### tags, metadata, etc.
+	### Datadog's max size for the entire payload is 5MB
+	MAX_BATCH_BYTESIZE = 4_194_304
+
+	# Override the default HTTP device options for sending logs to DD
+	DEFAULT_OPTIONS = {
+		max_batch_size: MAX_BATCH_SIZE,
+		max_message_bytesize: MAX_MESSAGE_BYTESIZE,
+		max_batch_bytesize: MAX_BATCH_BYTESIZE,
+	}
+
+
+	### Create a new Datadog
+	def initialize( api_key, endpoint=DEFAULT_ENDPOINT, options={} )
+		if endpoint.is_a?( Hash )
+			options = endpoint
+			endpoint = DEFAULT_ENDPOINT
+		end
+
+		super( endpoint, options )
+
+		@api_key = api_key
+		@hostname = Socket.gethostname
+	end
+
+
+	######
+	public
+	######
+
+	##
+	# The name of the current host
+	attr_reader :hostname
+
+	##
+	# The configured Datadog API key
+	attr_reader :api_key
+
+
+	### Format an individual log +message+ for Datadog.
+	def format_log_message( message )
+		return {
+			hostname: self.hostname,
+			message: message
+		}.to_json
+	end
+
+
+	### Overridden to add the configured API key to the headers of each request.
+	def make_batch_request
+		request = super
+
+		request[ 'DD-API-KEY' ] = self.api_key
+
+		return request
+	end
+
+end # class Loggability::LogDevice::Datadog

data/lib/loggability/log_device/file.rb

@@ -0,0 +1,37 @@
+#!/usr/bin/env ruby
+# vim: set nosta noet ts=4 sw=4:
+# encoding: utf-8
+
+require 'logger'
+require 'loggability/logger' unless defined?( Loggability::Logger )
+
+# A log device that delegates to the ruby's default Logger's file device and writes
+# to a file descriptor or a file.
+class Loggability::LogDevice::File < Loggability::LogDevice
+
+	### Create a new +File+ device that will write to the file using the built-in ruby's +File+ log device
+	def initialize( target )
+		@target = ::Logger::LogDevice.new( target )
+	end
+
+
+	######
+	public
+	######
+
+	# The target of the log device
+	attr_reader :target
+
+
+	### Append the specified +message+ to the target.
+	def write( message )
+		self.target.write( message )
+	end
+
+
+	### close the file
+	def close
+		self.target.close
+	end
+
+end # class Loggability::LogDevice::File

data/lib/loggability/log_device/http.rb

@@ -0,0 +1,288 @@
+#!/usr/bin/env ruby
+# vim: set nosta noet ts=4 sw=4:
+# encoding: utf-8
+
+require 'socket'
+require 'uri'
+require 'net/https'
+require 'json'
+require 'concurrent'
+require 'loggability/logger' unless defined?( Loggability::Logger )
+
+require 'loggability/log_device'
+
+# This is the a generalized class that allows its subclasses to send log
+# messages to HTTP endpoints asynchronously on a separate thread.
+class Loggability::LogDevice::Http < Loggability::LogDevice
+
+	# The default HTTP endpoint URL to send logs to
+	DEFAULT_ENDPOINT = "http://localhost:12775/v1/logs"
+
+	# The default maximum number of messages that can be sent to the server in a single payload
+	DEFAULT_MAX_BATCH_SIZE = 100
+
+	# The default max size in bytes for a single message.
+	DEFAULT_MAX_MESSAGE_BYTESIZE = 2 * 16
+
+	# The default number of seconds between batches
+	DEFAULT_BATCH_INTERVAL = 60
+
+	# The default number of seconds to wait for data to be written before timing out
+	DEFAULT_WRITE_TIMEOUT = 15
+
+	# The default Executor class to use for asynchronous tasks
+	DEFAULT_EXECUTOR_CLASS = Concurrent::SingleThreadExecutor
+
+	# The default options for new instances
+	DEFAULT_OPTIONS = {
+		execution_interval: DEFAULT_BATCH_INTERVAL,
+		write_timeout: DEFAULT_WRITE_TIMEOUT,
+		max_batch_size: DEFAULT_MAX_BATCH_SIZE,
+		max_message_bytesize: DEFAULT_MAX_MESSAGE_BYTESIZE,
+		executor_class: DEFAULT_EXECUTOR_CLASS,
+	}
+
+
+	### Initialize the HTTP log device to send to the specified +endpoint+ with the
+	### given +options+. Valid options are:
+	###
+	### [:batch_interval]
+	###   Maximum number of seconds between batches
+	### [:write_timeout]
+	###   How many seconds to wait for data to be written while sending a batch
+	### [:max_batch_size]
+	###   The maximum number of messages that can be in a single batch
+	### [:max_batch_bytesize]
+	###   The maximum number of bytes that can be in the payload of a single batch
+	### [:max_message_bytesize]
+	###   The maximum number of bytes that can be in a single message
+	### [:executor_class]
+	###   The Concurrent executor class to use for asynchronous tasks.
+	def initialize( endpoint=DEFAULT_ENDPOINT, opts={} )
+		if endpoint.is_a?( Hash )
+			opts = endpoint
+			endpoint = DEFAULT_ENDPOINT
+		end
+
+		opts = DEFAULT_OPTIONS.merge( opts )
+
+		@endpoint             = URI( endpoint ).freeze
+		@logs_queue           = Queue.new
+
+		@batch_interval       = opts[:batch_interval] || DEFAULT_BATCH_INTERVAL
+		@write_timeout        = opts[:write_timeout] || DEFAULT_WRITE_TIMEOUT
+		@max_batch_size       = opts[:max_batch_size] || DEFAULT_MAX_BATCH_SIZE
+		@max_message_bytesize = opts[:max_message_bytesize] || DEFAULT_MAX_MESSAGE_BYTESIZE
+		@executor_class       = opts[:executor_class] || DEFAULT_EXECUTOR_CLASS
+
+		@max_batch_bytesize   = opts[:max_batch_bytesize] || @max_batch_size * @max_message_bytesize
+		@last_send_time       = Concurrent.monotonic_time
+	end
+
+
+	######
+	public
+	######
+
+	##
+	# The single thread pool executor
+	attr_reader :executor
+
+	##
+	# The URI of the endpoint to send messages to
+	attr_reader :endpoint
+
+	##
+	# The Queue that contains any log messages which have not yet been sent to the
+	# logging service.
+	attr_reader :logs_queue
+
+	##
+	# The monotonic clock time when the last batch of logs were sent
+	attr_accessor :last_send_time
+
+	##
+	# Number of seconds after the task completes before the task is performed again.
+	attr_reader :batch_interval
+
+	##
+	# How many seconds to wait for data to be written while sending a batch
+	attr_reader :write_timeout
+
+	##
+	# The maximum number of messages to post at one time
+	attr_reader :max_batch_size
+
+	##
+	# The maximum number of bytes of a single message to include in a batch
+	attr_reader :max_message_bytesize
+
+	##
+	# The maximum number of bytes that will be included in a single POST
+	attr_reader :max_batch_bytesize
+
+	##
+	# The Concurrent executor class to use for asynchronous tasks
+	attr_reader :executor_class
+
+	##
+	# The timer task thread
+	attr_reader :timer_task
+
+
+	### LogDevice API -- write a message to the HTTP device.
+	def write( message )
+		self.start unless self.running?
+		self.logs_queue.enq( message )
+		self.send_logs
+	end
+
+
+	### LogDevice API -- stop the batch thread and close the http connection
+	def close
+		self.stop
+		self.http_client.finish
+	rescue IOError
+		# ignore it since http session has not yet started.
+	end
+
+
+	### Starts a thread pool with a single thread.
+	def start
+		self.start_executor
+		self.start_timer_task
+	end
+
+
+	### Returns +true+ if the device has started sending messages to the logging endpoint.
+	def running?
+		return self.executor&.running?
+	end
+
+
+	### Shutdown the executor, which is a pool of single thread
+	### waits 3 seconds for shutdown to complete
+	def stop
+		return unless self.running?
+
+		self.timer_task.shutdown if self.timer_task&.running?
+		self.executor.shutdown
+
+		unless self.executor.wait_for_termination( 3 )
+			self.executor.halt
+			self.executor.wait_for_termination( 3 )
+		end
+	end
+
+
+	### Start the background thread that sends messages.
+	def start_executor
+		@executor = self.executor_class.new
+		@executor.auto_terminate = true unless @executor.serialized?
+	end
+
+
+	### Create a timer task that calls that sends logs at regular interval
+	def start_timer_task
+		@timer_task = Concurrent::TimerTask.execute( execution_interval: self.batch_interval ) do
+			self.send_logs
+		end
+	end
+
+
+	### Sends a batch of log messages to the logging service. This executes inside
+	### the sending thread.
+	def send_logs
+		self.executor.post do
+			if self.batch_ready?
+				# p "Batch ready; sending."
+				request = self.make_batch_request
+				request.body = self.get_next_log_payload
+
+				# p "Sending request", request
+
+				self.http_client.request( request ) do |res|
+					p( res ) if $DEBUG
+				end
+
+				self.last_send_time = Concurrent.monotonic_time
+			else
+				# p "Batch not ready yet."
+			end
+		end
+	end
+
+
+	### Returns +true+ if a batch of logs is ready to be sent.
+	def batch_ready?
+		seconds_since_last_send = Concurrent.monotonic_time - self.last_send_time
+
+		return self.logs_queue.size >= self.max_batch_size ||
+			seconds_since_last_send >= self.batch_interval
+	end
+	alias_method :has_batch_ready?, :batch_ready?
+
+
+	### Returns a new HTTP request (a subclass of Net::HTTPRequest) suitable for
+	### sending the next batch of logs to the service. Defaults to a POST of JSON data. This
+	### executes inside the sending thread.
+	def make_batch_request
+		request = Net::HTTP::Post.new( self.endpoint.path )
+		request[ 'Content-Type' ] = 'application/json'
+
+		return request
+	end
+
+
+	### Dequeue pending log messages to send to the service and return them as a
+	### suitably-encoded String. The default is a JSON Array. This executes inside
+	### the sending thread.
+	def get_next_log_payload
+		buf = []
+		count = 0
+		bytes = 0
+
+		# Be conservative so as not to overflow
+		max_size = self.max_batch_bytesize - self.max_message_bytesize - 2 # for the outer Array
+
+		while count < self.max_batch_size && bytes < max_size && !self.logs_queue.empty?
+			formatted_message = self.format_log_message( self.logs_queue.deq )
+
+			count += 1
+			bytes += formatted_message.bytesize + 3 # comma and delimiters
+
+			buf << formatted_message
+		end
+
+		return '[' + buf.join(',') + ']'
+	end
+
+
+	### Returns the given +message+ in whatever format individual log messages are
+	### expected to be in by the service. The default just returns the stringified
+	### +message+. This executes inside the sending thread.
+	def format_log_message( message )
+		return message.to_s[ 0 ... self.max_message_bytesize ].dump
+	end
+
+
+	### sets up a configured http object ready to instantiate connections
+	def http_client
+		return @http_client ||= begin
+			uri = URI.parse( self.endpoint )
+
+			http = Net::HTTP.new( uri.host, uri.port )
+			http.write_timeout = self.write_timeout
+
+			if uri.scheme == 'https'
+				http.use_ssl = true
+				http.verify_mode = OpenSSL::SSL::VERIFY_PEER
+			end
+
+			http
+		end
+	end
+
+
+end # class Loggability::LogDevice::Http
+