loggability 0.15.0.pre20190714094638 → 0.18.1

data/lib/loggability/logger.rb

@@ -24,35 +24,6 @@ class Loggability::Logger < ::Logger
 	DEFAULT_SHIFT_SIZE = 1048576
 
 
-	# A log device that appends to the object it's constructed with instead of writing
-	# to a file descriptor or a file.
-	class AppendingLogDevice
-
-		### Create a new AppendingLogDevice that will append content to +array+.
-		def initialize( target )
-			@target = target
-		end
-
-
-		######
-		public
-		######
-
-		# The target of the log device
-		attr_reader :target
-
-
-		### Append the specified +message+ to the target.
-		def write( message )
-			@target << message
-		end
-
-		### No-op -- this is here just so Logger doesn't complain
-		def close; end
-
-	end # class AppendingLogDevice
-
-
 	# Proxy for the Logger that injects the name of the object it wraps as the 'progname'
 	# of each log message.
 	class ObjectNameProxy
@@ -182,6 +153,28 @@ class Loggability::Logger < ::Logger
 	end
 
 
+	### Log a message if the +severity+ is high enough. Overridden to account for
+	### the overridden #level.
+	def add( severity, message=nil, progname=nil )
+		return true if severity < self.sev_threshold
+		progname ||= @progname
+
+		unless message
+			if block_given?
+				message = yield
+			else
+				message = progname
+				progname = @progname
+			end
+		end
+
+		msg = self.format_message( self.format_severity(severity), Time.now, progname, message )
+		self.logdev.write( msg )
+
+		return true
+	end
+
+
 	### Append operator -- Override Logger's append so log messages always have
 	### formatting, and are always appended at :debug level.
 	def <<( message )
@@ -224,8 +217,7 @@ class Loggability::Logger < ::Logger
 
 	### Return the logger's level as a Symbol.
 	def level
-		numeric_level = super
-		return LOG_LEVEL_NAMES[ numeric_level ]
+		return LOG_LEVEL_NAMES[ self.sev_threshold ]
 	end
 
 
@@ -251,14 +243,17 @@ class Loggability::Logger < ::Logger
 	### logging to IO objects and files (given a filename in a String), this method can also
 	### set up logging to any object that responds to #<<.
 	def output_to( target, *args )
-		if target.is_a?( Logger::LogDevice ) ||
-		   target.is_a?( Loggability::Logger::AppendingLogDevice )
+		if target.is_a?( Logger::LogDevice ) || target.is_a?( Loggability::LogDevice )
 			self.logdev = target
 		elsif target.respond_to?( :write ) || target.is_a?( String )
 			opts = { :shift_age => args.shift || 0, :shift_size => args.shift || 1048576 }
-			self.logdev = Logger::LogDevice.new( target, opts )
+			self.logdev = Logger::LogDevice.new( target, **opts )
+		elsif target.respond_to?( :any? ) && target.any?( Loggability::LogDevice )
+			self.logdev = MultiDevice.new( target )
 		elsif target.respond_to?( :<< )
-			self.logdev = AppendingLogDevice.new( target )
+			self.logdev = Loggability::LogDevice.create( :appending, target )
+		elsif target.is_a?( Symbol )
+			self.logdev = Loggability::LogDevice.create( target, *args )
 		else
 			raise ArgumentError, "don't know how to output to %p (a %p)" % [ target, target.class ]
 		end
@@ -282,10 +277,18 @@ class Loggability::Logger < ::Logger
 	end
 
 
-	### Set a new +formatter+ for the logger. If +formatter+ is +nil+ or +:default+, this causes the
-	### logger to fall back to its default formatter. If it's a Symbol other than +:default+, it looks
-	### for a similarly-named formatter under loggability/formatter/ and uses that. If +formatter+ is
-	### an object that responds to #call (e.g., a Proc or a Method object), that object is used directly.
+	### Return the formatted name of the given +severity+.
+	def format_severity( severity )
+		name = LOG_LEVEL_NAMES[ severity ] || severity.to_s
+		return name.upcase
+	end
+
+
+	### Set a new +formatter+ for the logger. If +formatter+ is +nil+ or +:default+, this causes
+	### the logger to fall back to its default formatter. If it's a Symbol other than +:default+,
+	### it looks for a similarly-named formatter under loggability/formatter/ and uses that. If
+	### +formatter+ is an object that responds to #call (e.g., a Proc or a Method object), that
+	### object is used directly.
 	###
 	### Procs and methods should have the method signature: (severity, datetime, progname, msg).
 	###

data/spec/helpers.rb

@@ -20,7 +20,6 @@ require 'timecop'
 require 'loggability'
 require 'loggability/spechelpers'
 
-
 # Helpers specific to Loggability specs
 module SpecHelpers
 
@@ -99,10 +98,13 @@ RSpec.configure do |c|
 	c.mock_with( :rspec ) do |mock|
 		mock.syntax = :expect
 	end
+	c.warnings = true
+	c.profile_examples = 5
 
 	c.include( SpecHelpers )
 	c.include( Loggability::SpecHelpers )
 	c.filter_run_excluding( :configurability ) unless defined?( Configurability )
 
+
 end
 

data/spec/loggability/formatter/default_spec.rb

@@ -0,0 +1,50 @@
+# -*- ruby -*-
+# vim: set nosta noet ts=4 sw=4:
+# frozen_string_literal: true
+
+require_relative '../../helpers'
+
+require 'loggability/formatter/default'
+
+
+describe Loggability::Formatter::Default do
+
+	it "formats messages with the pattern it's constructed with" do
+		formatter = described_class.new( '[%5$s] %7$s' )
+		result = formatter.call( 'INFO', Time.at(1336286481), nil, 'Foom.' )
+		expect( result ).to match(/\[INFO\] Foom./i)
+	end
+
+
+	it "formats exceptions into useful messages" do
+		formatter = described_class.new( '[%5$s] %7$s' )
+		msg = nil
+
+		begin
+			raise ArgumentError, "invalid argument"
+		rescue => err
+			msg = formatter.call( 'INFO', Time.at(1336286481), nil, err )
+		end
+
+		expect( msg ).to match(/\[INFO\] ArgumentError: invalid argument/i)
+	end
+
+
+	it "formats regular objects into useful messages" do
+		formatter = described_class.new( '[%5$s] %7$s' )
+		result = formatter.call( 'INFO', Time.at(1336286481), nil, Object.new )
+
+		expect( result ).to match(/\[INFO\] #<Object:0x[[:xdigit:]]+>/i)
+	end
+
+
+	it "includes the thread ID if logging from a thread other than the main thread" do
+		formatter = described_class.new( '%4$d' )
+		thr = Thread.new do
+			formatter.call( 'INFO', Time.now, nil, 'Foom.' )
+		end
+		expect( thr.value ).to eq( thr.object_id.to_s )
+	end
+
+end
+

data/spec/loggability/formatter/structured_spec.rb

@@ -0,0 +1,61 @@
+# -*- ruby -*-
+# vim: set nosta noet ts=4 sw=4:
+# frozen_string_literal: true
+
+require_relative '../../helpers'
+
+require 'loggability/formatter/structured'
+
+
+describe Loggability::Formatter::Structured do
+
+	before( :each ) do
+		ENV['TZ'] = 'UTC'
+	end
+
+
+	it "outputs a stream of JSON objects" do
+		expect(
+			subject.call('INFO', Time.at(1336286481), nil, "Foom.")
+		).to eq(
+			%q|{"@version":1,"@timestamp":"2012-05-06T06:41:21.000+00:00"| +
+			%q|,"level":"INFO","progname":null,"message":"Foom."}|
+		)
+	end
+
+
+	it "includes a time even if called without one" do
+		Timecop.freeze( Time.at(1563114765.123) ) do
+			expect(
+				subject.call('WARN', nil, nil, "Right.")
+			).to match( %r(
+				\{
+					"@version":1,
+					"@timestamp":"2019-07-14T14:32:45\.\d{3}\+00:00",
+					"level":"WARN",
+					"progname":null,
+					"message":"Right\."
+				\}
+			)x )
+		end
+	end
+
+
+	it "defaults to DEBUG severity" do
+		Timecop.freeze( Time.at(1563114765.123) ) do
+			expect(
+				subject.call(nil, nil, nil, "Crane.")
+			).to match( %r(
+				\{
+					"@version":1,
+					"@timestamp":"2019-07-14T14:32:45\.\d{3}\+00:00",
+					"level":"DEBUG",
+					"progname":null,
+					"message":"Crane\."
+				\}
+			)x )
+		end
+	end
+
+end
+

data/spec/loggability/log_device/appending_spec.rb

@@ -0,0 +1,27 @@
+# -*- rspec -*-
+#encoding: utf-8
+
+require 'rspec'
+
+require 'loggability/logger'
+require 'loggability/log_device/appending'
+
+
+describe Loggability::LogDevice::Appending do
+
+	let ( :logger ) { described_class.new( [] ) }
+
+
+	it "The target is an array" do
+		expect( logger.target ).to be_instance_of( Array )
+	end
+
+
+	it "can append to the array" do
+		logger.write("log message one")
+		logger.write("log message two")
+		expect( logger.target.size ).to eq( 2 )
+	end
+
+end
+

data/spec/loggability/log_device/datadog_spec.rb

@@ -0,0 +1,67 @@
+# -*- rspec -*-
+#encoding: utf-8
+
+require 'securerandom'
+require 'rspec'
+
+require 'loggability/logger'
+require 'loggability/log_device/datadog'
+
+
+describe Loggability::LogDevice::Datadog do
+
+
+	let( :api_key ) { SecureRandom.hex(24) }
+	let( :http_client ) { instance_double(Net::HTTP) }
+
+
+	it "includes the configured API key in request headers" do
+		device = described_class.new(
+			api_key,
+			max_batch_size: 3,
+			batch_interval: 0.1,
+			executor_class: Concurrent::ImmediateExecutor )
+		device.instance_variable_set( :@http_client, http_client )
+
+		expect( http_client ).to receive( :request ) do |request|
+			expect( request ).to be_a( Net::HTTP::Post )
+			expect( request['Content-type'] ).to match( %r|application/json|i )
+			expect( request['DD-API-KEY'] ).to eq( api_key )
+		end.at_least( :once )
+
+		device.write( "message data" * 10 )  # 120 bytes
+		device.write( "message data" * 100 ) # 1200 bytes
+		device.write( "message data" * 85 )  # 1020 bytes
+		device.write( "message data" * 86 )  # 1032 bytes
+
+		sleep( 0.1 ) until device.logs_queue.empty?
+	end
+
+
+	it "includes the hostname in individual log messages" do
+		device = described_class.new(
+			api_key,
+			max_batch_size: 3,
+			batch_interval: 0.1,
+			executor_class: Concurrent::ImmediateExecutor )
+		device.instance_variable_set( :@http_client, http_client )
+
+		expect( http_client ).to receive( :request ) do |request|
+			expect( request ).to be_a( Net::HTTP::Post )
+
+			data = JSON.parse( request.body )
+
+			expect( data ).to all( be_a Hash )
+			expect( data ).to all( include('hostname' => device.hostname) )
+		end.at_least( :once )
+
+		device.write( "message data" * 10 )  # 120 bytes
+		device.write( "message data" * 100 ) # 1200 bytes
+		device.write( "message data" * 85 )  # 1020 bytes
+		device.write( "message data" * 86 )  # 1032 bytes
+
+		sleep( 0.1 ) until device.logs_queue.empty?
+	end
+
+end
+

data/spec/loggability/log_device/file_spec.rb

@@ -0,0 +1,27 @@
+# -*- rspec -*-
+#encoding: utf-8
+
+require 'tempfile'
+require 'rspec'
+
+require 'loggability/logger'
+require 'loggability/log_device/file'
+
+
+describe Loggability::LogDevice::File do
+
+	let( :logfile ) { Tempfile.new( 'test.log' ) }
+	let( :logger ) { described_class.new( logfile.path ) }
+
+
+	it "The logger is an instance of Loggability::LogDevice::File" do
+		expect( logger ).to be_instance_of( Loggability::LogDevice::File )
+	end
+
+
+	it "The log device is delegated to Ruby's built-in log device" do
+		expect( logger.target ).to be_instance_of( ::Logger::LogDevice )
+	end
+
+end
+

data/spec/loggability/log_device/http_spec.rb

@@ -0,0 +1,217 @@
+# -*- rspec -*-
+
+require_relative '../../helpers'
+
+require 'tempfile'
+require 'rspec'
+
+require 'loggability/log_device/http'
+
+
+describe Loggability::LogDevice::Http do
+
+
+	let( :http_client ) { instance_double(Net::HTTP) }
+
+
+	it "can be created with defaults" do
+		result = described_class.new
+
+		expect( result ).to be_an_instance_of( described_class )
+		expect( result.batch_interval ).to eq( described_class::DEFAULT_BATCH_INTERVAL )
+		expect( result.write_timeout ).to eq( described_class::DEFAULT_WRITE_TIMEOUT )
+	end
+
+
+	it "doesn't start when created" do
+		result = described_class.new
+
+		expect( result ).to_not be_running
+	end
+
+
+	it "sends logs when a full batch is ready" do
+		device = described_class.new( max_batch_size: 3, executor_class: Concurrent::ImmediateExecutor )
+		device.instance_variable_set( :@http_client, http_client )
+
+		expect( http_client ).to receive( :request ) do |request|
+			expect( request ).to be_a( Net::HTTP::Post )
+			expect( request['Content-type'] ).to match( %r|application/json|i )
+			expect( request.body ).to match( /message 1/i )
+			expect( request.body ).to match( /message 2/i )
+			expect( request.body ).to match( /message 3/i )
+		end
+
+		device.write( "Message 1" )
+		device.write( "Message 2" )
+		device.write( "Message 3" )
+		device.write( "Message 4" )
+
+		expect( device.logs_queue ).to have_attributes( length: 1 )
+	end
+
+
+	it "sends logs when enough time has elapsed since the last message" do
+		device = described_class.new(
+			max_batch_size: 3, batch_interval: 0.1, executor_class: Concurrent::ImmediateExecutor )
+		device.instance_variable_set( :@http_client, http_client )
+		device.start
+		device.timer_task.shutdown # Don't let the timer fire
+
+		expect( http_client ).to receive( :request ) do |request|
+			expect( request ).to be_a( Net::HTTP::Post )
+			expect( request['Content-type'] ).to match( %r|application/json|i )
+			expect( request.body ).to match( /message 1/i )
+			expect( request.body ).to match( /message 2/i )
+		end
+
+		device.write( "Message 1" )
+
+		# Now wait for the batch interval to pass and send another
+		sleep device.batch_interval
+		expect( device ).to have_batch_ready
+		device.write( "Message 2" )
+
+		expect( device.logs_queue ).to have_attributes( length: 0 )
+	end
+
+
+	it "sends logs on the batch interval even when messages aren't arriving" do
+		device = described_class.new(
+			max_batch_size: 3, batch_interval: 0.1, executor_class: Concurrent::ImmediateExecutor )
+		device.instance_variable_set( :@http_client, http_client )
+
+		expect( http_client ).to receive( :request ) do |request|
+			expect( request ).to be_a( Net::HTTP::Post )
+			expect( request['Content-type'] ).to match( %r|application/json|i )
+			expect( request.body ).to match( /message 1/i )
+		end
+
+		device.write( "Message 1" )
+
+		# Now wait for the batch interval to pass and send another
+		sleep device.batch_interval * 2
+
+		expect( device.logs_queue ).to have_attributes( length: 0 )
+	end
+
+
+	it "limits messages to the configured byte size constraints" do
+		device = described_class.new(
+			max_batch_size: 3,
+			max_message_bytesize: 1024,
+			batch_interval: 0.1,
+			executor_class: Concurrent::ImmediateExecutor )
+		device.instance_variable_set( :@http_client, http_client )
+
+		expect( http_client ).to receive( :request ) do |request|
+			expect( request ).to be_a( Net::HTTP::Post )
+			expect( request['Content-type'] ).to match( %r|application/json|i )
+
+			data = JSON.parse( request.body )
+
+			expect( data ).to all( have_attributes(bytesize: a_value <= 1024) )
+		end.at_least( :once )
+
+		device.write( "message data" * 10 )  # 120 bytes
+		device.write( "message data" * 100 ) # 1200 bytes
+		device.write( "message data" * 85 )  # 1020 bytes
+		device.write( "message data" * 86 )  # 1032 bytes
+
+		sleep( 0.1 ) until device.logs_queue.empty?
+	end
+
+
+	it "limits the batch to the configured byte size constraints" do
+		device = described_class.new(
+			max_batch_bytesize: 1024,
+			batch_interval: 0.1,
+			executor_class: Concurrent::ImmediateExecutor )
+		device.instance_variable_set( :@http_client, http_client )
+
+		expect( http_client ).to receive( :request ) do |request|
+			expect( request ).to be_a( Net::HTTP::Post )
+			expect( request['Content-type'] ).to match( %r|application/json|i )
+
+			expect( request.body.bytesize ).to be <= 1024
+		end.at_least( :once )
+
+		20.times { device.write( "message data" * 10 )  } # 120 bytes
+		20.times { device.write( "message data" * 100 ) } # 1200 bytes
+		20.times { device.write( "message data" * 85 )  } # 1020 bytes
+		20.times { device.write( "message data" * 86 )  } # 1032 bytes
+
+		sleep( 0.1 ) until device.logs_queue.empty?
+	end
+
+
+	it "uses an HTTP client for the appropriate host and port" do
+		device = described_class.new(
+			'http://logs.example.com:12881/v1/log_ingester',
+			executor_class: Concurrent::ImmediateExecutor )
+		http = device.http_client
+
+		expect( http.address ).to eq( 'logs.example.com' )
+		expect( http.port ).to eq( 12881 )
+	end
+
+
+	it "verifies the peer cert if sending to an HTTPS endpoint" do
+		device = described_class.new(
+			'https://logs.example.com:12881/v1/log_ingester',
+			executor_class: Concurrent::ImmediateExecutor )
+		http = device.http_client
+
+		expect( http.use_ssl? ).to be_truthy
+		expect( http.verify_mode ).to eq( OpenSSL::SSL::VERIFY_PEER )
+	end
+
+
+	it "stops queuing more messages if max queue size is reached" do
+		device = described_class.new(
+			max_batch_bytesize: 1024,
+			batch_interval: 100,
+			max_queue_bytesize: 100,
+			executor_class: Concurrent::ImmediateExecutor )
+		device.instance_variable_set( :@http_client, http_client )
+
+		expect( device ).to receive( :send_logs ).at_least( :once )
+
+		msg = "test message"
+		device.write(msg)
+		expect( device.logs_queue_bytesize == msg.bytesize )
+
+		hash_msg = { message: "This is a test log message", tags: ["tag1", "tag2"] }
+		device.write( hash_msg )
+		previous_bytesize = device.logs_queue_bytesize - hash_msg.to_json.bytesize
+		expect( device.logs_queue_bytesize ).to eq( hash_msg.to_json.bytesize + previous_bytesize )
+
+		queue_current_bytesize = device.logs_queue_bytesize
+		hash_msg = { message: "This is a test log message", tags: ["tag1", "tag2"] }
+		device.write( hash_msg )
+		expect( device.logs_queue_bytesize ).to eq( queue_current_bytesize )
+	end
+
+
+	it "reduces the queue bytesize once messages are sent" do
+		device = described_class.new(
+			max_batch_bytesize: 1024,
+			batch_interval: 100,
+			max_queue_bytesize: 100,
+			executor_class: Concurrent::ImmediateExecutor )
+		device.instance_variable_set( :@http_client, http_client )
+
+		expect( device ).to receive( :send_logs ).at_least( :once )
+		msg = "test message"
+		device.write(msg)
+		expect( device.logs_queue_bytesize == msg.bytesize )
+
+		msg = "this is just a test message"
+		device.write( msg )
+		previous_bytesize = device.logs_queue_bytesize - msg.bytesize
+		expect( device.logs_queue_bytesize ).to eq( msg.bytesize + previous_bytesize )
+
+		expect { device.get_next_log_payload }.to change { device.logs_queue_bytesize }.to( 0 )
+	end
+
+end