loggability 0.15.1 → 0.18.2

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,310 @@
1
+ #!/usr/bin/env ruby
2
+ # vim: set nosta noet ts=4 sw=4:
3
+ # encoding: utf-8
4
+
5
+ require 'socket'
6
+ require 'uri'
7
+ require 'net/https'
8
+ require 'json'
9
+ require 'concurrent'
10
+ require 'loggability/logger' unless defined?( Loggability::Logger )
11
+
12
+ require 'loggability/log_device'
13
+
14
+ # This is the a generalized class that allows its subclasses to send log
15
+ # messages to HTTP endpoints asynchronously on a separate thread.
16
+ class Loggability::LogDevice::Http < Loggability::LogDevice
17
+
18
+ # The default HTTP endpoint URL to send logs to
19
+ DEFAULT_ENDPOINT = "http://localhost:12775/v1/logs"
20
+
21
+ # The default maximum number of messages that can be sent to the server in a single payload
22
+ DEFAULT_MAX_BATCH_SIZE = 100
23
+
24
+ # The default max size in bytes for a single message.
25
+ DEFAULT_MAX_MESSAGE_BYTESIZE = 2 * 16
26
+
27
+ # The default number of seconds between batches
28
+ DEFAULT_BATCH_INTERVAL = 60
29
+
30
+ # The default number of seconds to wait for data to be written before timing out
31
+ DEFAULT_WRITE_TIMEOUT = 15
32
+
33
+ # The default Executor class to use for asynchronous tasks
34
+ DEFAULT_EXECUTOR_CLASS = Concurrent::SingleThreadExecutor
35
+
36
+ # The default for the maximum bytesize of the queue (1 GB)
37
+ DEFAULT_MAX_QUEUE_BYTESIZE = ( 2 ** 10 ) * ( 2 ** 10 ) * ( 2 ** 10 )
38
+
39
+ # The default options for new instances
40
+ DEFAULT_OPTIONS = {
41
+ execution_interval: DEFAULT_BATCH_INTERVAL,
42
+ write_timeout: DEFAULT_WRITE_TIMEOUT,
43
+ max_batch_size: DEFAULT_MAX_BATCH_SIZE,
44
+ max_message_bytesize: DEFAULT_MAX_MESSAGE_BYTESIZE,
45
+ executor_class: DEFAULT_EXECUTOR_CLASS,
46
+ }
47
+
48
+
49
+ ### Initialize the HTTP log device to send to the specified +endpoint+ with the
50
+ ### given +options+. Valid options are:
51
+ ###
52
+ ### [:batch_interval]
53
+ ### Maximum number of seconds between batches
54
+ ### [:write_timeout]
55
+ ### How many seconds to wait for data to be written while sending a batch
56
+ ### [:max_batch_size]
57
+ ### The maximum number of messages that can be in a single batch
58
+ ### [:max_batch_bytesize]
59
+ ### The maximum number of bytes that can be in the payload of a single batch
60
+ ### [:max_message_bytesize]
61
+ ### The maximum number of bytes that can be in a single message
62
+ ### [:executor_class]
63
+ ### The Concurrent executor class to use for asynchronous tasks.
64
+ def initialize( endpoint=DEFAULT_ENDPOINT, opts={} )
65
+ if endpoint.is_a?( Hash )
66
+ opts = endpoint
67
+ endpoint = DEFAULT_ENDPOINT
68
+ end
69
+
70
+ opts = DEFAULT_OPTIONS.merge( opts )
71
+
72
+ @endpoint = URI( endpoint ).freeze
73
+ @logs_queue = Queue.new
74
+
75
+ @logs_queue_bytesize = 0
76
+ @max_queue_bytesize = opts[:max_queue_bytesize] || DEFAULT_MAX_QUEUE_BYTESIZE
77
+ @batch_interval = opts[:batch_interval] || DEFAULT_BATCH_INTERVAL
78
+ @write_timeout = opts[:write_timeout] || DEFAULT_WRITE_TIMEOUT
79
+ @max_batch_size = opts[:max_batch_size] || DEFAULT_MAX_BATCH_SIZE
80
+ @max_message_bytesize = opts[:max_message_bytesize] || DEFAULT_MAX_MESSAGE_BYTESIZE
81
+ @executor_class = opts[:executor_class] || DEFAULT_EXECUTOR_CLASS
82
+
83
+ @max_batch_bytesize = opts[:max_batch_bytesize] || @max_batch_size * @max_message_bytesize
84
+ @last_send_time = Concurrent.monotonic_time
85
+ end
86
+
87
+
88
+ ######
89
+ public
90
+ ######
91
+
92
+ ##
93
+ # The single thread pool executor
94
+ attr_reader :executor
95
+
96
+ ##
97
+ # The URI of the endpoint to send messages to
98
+ attr_reader :endpoint
99
+
100
+ ##
101
+ # The Queue that contains any log messages which have not yet been sent to the
102
+ # logging service.
103
+ attr_reader :logs_queue
104
+
105
+ ##
106
+ # The max bytesize of the queue. Will not queue more messages if this threshold is hit
107
+ attr_reader :max_queue_bytesize
108
+
109
+ ##
110
+ # The size of +logs_queue+ in bytes
111
+ attr_accessor :logs_queue_bytesize
112
+
113
+ ##
114
+ # The monotonic clock time when the last batch of logs were sent
115
+ attr_accessor :last_send_time
116
+
117
+ ##
118
+ # Number of seconds after the task completes before the task is performed again.
119
+ attr_reader :batch_interval
120
+
121
+ ##
122
+ # How many seconds to wait for data to be written while sending a batch
123
+ attr_reader :write_timeout
124
+
125
+ ##
126
+ # The maximum number of messages to post at one time
127
+ attr_reader :max_batch_size
128
+
129
+ ##
130
+ # The maximum number of bytes of a single message to include in a batch
131
+ attr_reader :max_message_bytesize
132
+
133
+ ##
134
+ # The maximum number of bytes that will be included in a single POST
135
+ attr_reader :max_batch_bytesize
136
+
137
+ ##
138
+ # The Concurrent executor class to use for asynchronous tasks
139
+ attr_reader :executor_class
140
+
141
+ ##
142
+ # The timer task thread
143
+ attr_reader :timer_task
144
+
145
+
146
+ ### LogDevice API -- write a message to the HTTP device.
147
+ def write( message )
148
+ self.start unless self.running?
149
+ if message.is_a?( Hash )
150
+ message_size = message.to_json.bytesize
151
+ else
152
+ message_size = message.bytesize
153
+ end
154
+ return if ( self.logs_queue_bytesize + message_size ) >= self.max_queue_bytesize
155
+ self.logs_queue_bytesize += message_size
156
+ self.logs_queue.enq( message )
157
+ self.send_logs
158
+ end
159
+
160
+
161
+ ### LogDevice API -- stop the batch thread and close the http connection
162
+ def close
163
+ self.stop
164
+ self.http_client.finish
165
+ rescue IOError
166
+ # ignore it since http session has not yet started.
167
+ end
168
+
169
+
170
+ ### Starts a thread pool with a single thread.
171
+ def start
172
+ self.start_executor
173
+ self.start_timer_task
174
+ end
175
+
176
+
177
+ ### Returns +true+ if the device has started sending messages to the logging endpoint.
178
+ def running?
179
+ return self.executor&.running?
180
+ end
181
+
182
+
183
+ ### Shutdown the executor, which is a pool of single thread
184
+ ### waits 3 seconds for shutdown to complete
185
+ def stop
186
+ return unless self.running?
187
+
188
+ self.timer_task.shutdown if self.timer_task&.running?
189
+ self.executor.shutdown
190
+
191
+ unless self.executor.wait_for_termination( 3 )
192
+ self.executor.halt
193
+ self.executor.wait_for_termination( 3 )
194
+ end
195
+ end
196
+
197
+
198
+ ### Start the background thread that sends messages.
199
+ def start_executor
200
+ @executor = self.executor_class.new
201
+ @executor.auto_terminate = true unless @executor.serialized?
202
+ end
203
+
204
+
205
+ ### Create a timer task that calls that sends logs at regular interval
206
+ def start_timer_task
207
+ @timer_task = Concurrent::TimerTask.execute( execution_interval: self.batch_interval ) do
208
+ self.send_logs
209
+ end
210
+ end
211
+
212
+
213
+ ### Sends a batch of log messages to the logging service. This executes inside
214
+ ### the sending thread.
215
+ def send_logs
216
+ self.executor.post do
217
+ if self.batch_ready?
218
+ # p "Batch ready; sending."
219
+ request = self.make_batch_request
220
+ request.body = self.get_next_log_payload
221
+
222
+ # p "Sending request", request
223
+
224
+ self.http_client.request( request ) do |res|
225
+ p( res ) if $DEBUG
226
+ end
227
+
228
+ self.last_send_time = Concurrent.monotonic_time
229
+ else
230
+ # p "Batch not ready yet."
231
+ end
232
+ end
233
+ end
234
+
235
+
236
+ ### Returns +true+ if a batch of logs is ready to be sent.
237
+ def batch_ready?
238
+ seconds_since_last_send = Concurrent.monotonic_time - self.last_send_time
239
+
240
+ return self.logs_queue.size >= self.max_batch_size ||
241
+ seconds_since_last_send >= self.batch_interval
242
+ end
243
+ alias_method :has_batch_ready?, :batch_ready?
244
+
245
+
246
+ ### Returns a new HTTP request (a subclass of Net::HTTPRequest) suitable for
247
+ ### sending the next batch of logs to the service. Defaults to a POST of JSON data. This
248
+ ### executes inside the sending thread.
249
+ def make_batch_request
250
+ request = Net::HTTP::Post.new( self.endpoint.path )
251
+ request[ 'Content-Type' ] = 'application/json'
252
+
253
+ return request
254
+ end
255
+
256
+
257
+ ### Dequeue pending log messages to send to the service and return them as a
258
+ ### suitably-encoded String. The default is a JSON Array. This executes inside
259
+ ### the sending thread.
260
+ def get_next_log_payload
261
+ buf = []
262
+ count = 0
263
+ bytes = 0
264
+
265
+ # Be conservative so as not to overflow
266
+ max_size = self.max_batch_bytesize - self.max_message_bytesize - 2 # for the outer Array
267
+
268
+ while count < self.max_batch_size && bytes < max_size && !self.logs_queue.empty?
269
+ message = self.logs_queue.deq
270
+ formatted_message = self.format_log_message( message )
271
+ self.logs_queue_bytesize -= message.bytesize
272
+
273
+ count += 1
274
+ bytes += formatted_message.bytesize + 3 # comma and delimiters
275
+
276
+ buf << formatted_message
277
+ end
278
+
279
+ return '[' + buf.join(',') + ']'
280
+ end
281
+
282
+
283
+ ### Returns the given +message+ in whatever format individual log messages are
284
+ ### expected to be in by the service. The default just returns the stringified
285
+ ### +message+. This executes inside the sending thread.
286
+ def format_log_message( message )
287
+ return message.to_s[ 0 ... self.max_message_bytesize ].dump
288
+ end
289
+
290
+
291
+ ### sets up a configured http object ready to instantiate connections
292
+ def http_client
293
+ return @http_client ||= begin
294
+ uri = URI( self.endpoint )
295
+
296
+ http = Net::HTTP.new( uri.host, uri.port )
297
+ http.write_timeout = self.write_timeout
298
+
299
+ if uri.scheme == 'https'
300
+ http.use_ssl = true
301
+ http.verify_mode = OpenSSL::SSL::VERIFY_PEER
302
+ end
303
+
304
+ http
305
+ end
306
+ end
307
+
308
+
309
+ end # class Loggability::LogDevice::Http
310
+
@@ -24,36 +24,6 @@ class Loggability::Logger < ::Logger
24
24
  DEFAULT_SHIFT_SIZE = 1048576
25
25
 
26
26
 
27
- # A log device that appends to the object it's constructed with instead of writing
28
- # to a file descriptor or a file.
29
- class AppendingLogDevice
30
-
31
- ### Create a new AppendingLogDevice that will append content to +target+ (a
32
- ### object that responds to #>>).
33
- def initialize( target )
34
- @target = target
35
- end
36
-
37
-
38
- ######
39
- public
40
- ######
41
-
42
- # The target of the log device
43
- attr_reader :target
44
-
45
-
46
- ### Append the specified +message+ to the target.
47
- def write( message )
48
- @target << message
49
- end
50
-
51
- ### No-op -- this is here just so Logger doesn't complain
52
- def close; end
53
-
54
- end # class AppendingLogDevice
55
-
56
-
57
27
  # Proxy for the Logger that injects the name of the object it wraps as the 'progname'
58
28
  # of each log message.
59
29
  class ObjectNameProxy
@@ -273,14 +243,17 @@ class Loggability::Logger < ::Logger
273
243
  ### logging to IO objects and files (given a filename in a String), this method can also
274
244
  ### set up logging to any object that responds to #<<.
275
245
  def output_to( target, *args )
276
- if target.is_a?( Logger::LogDevice ) ||
277
- target.is_a?( Loggability::Logger::AppendingLogDevice )
246
+ if target.is_a?( Logger::LogDevice ) || target.is_a?( Loggability::LogDevice )
278
247
  self.logdev = target
279
248
  elsif target.respond_to?( :write ) || target.is_a?( String )
280
249
  opts = { :shift_age => args.shift || 0, :shift_size => args.shift || 1048576 }
281
250
  self.logdev = Logger::LogDevice.new( target, **opts )
251
+ elsif target.respond_to?( :any? ) && target.any?( Loggability::LogDevice )
252
+ self.logdev = MultiDevice.new( target )
282
253
  elsif target.respond_to?( :<< )
283
- self.logdev = AppendingLogDevice.new( target )
254
+ self.logdev = Loggability::LogDevice.create( :appending, target )
255
+ elsif target.is_a?( Symbol )
256
+ self.logdev = Loggability::LogDevice.create( target, *args )
284
257
  else
285
258
  raise ArgumentError, "don't know how to output to %p (a %p)" % [ target, target.class ]
286
259
  end
@@ -311,10 +284,11 @@ class Loggability::Logger < ::Logger
311
284
  end
312
285
 
313
286
 
314
- ### Set a new +formatter+ for the logger. If +formatter+ is +nil+ or +:default+, this causes the
315
- ### logger to fall back to its default formatter. If it's a Symbol other than +:default+, it looks
316
- ### for a similarly-named formatter under loggability/formatter/ and uses that. If +formatter+ is
317
- ### an object that responds to #call (e.g., a Proc or a Method object), that object is used directly.
287
+ ### Set a new +formatter+ for the logger. If +formatter+ is +nil+ or +:default+, this causes
288
+ ### the logger to fall back to its default formatter. If it's a Symbol other than +:default+,
289
+ ### it looks for a similarly-named formatter under loggability/formatter/ and uses that. If
290
+ ### +formatter+ is an object that responds to #call (e.g., a Proc or a Method object), that
291
+ ### object is used directly.
318
292
  ###
319
293
  ### Procs and methods should have the method signature: (severity, datetime, progname, msg).
320
294
  ###
@@ -20,7 +20,6 @@ require 'timecop'
20
20
  require 'loggability'
21
21
  require 'loggability/spechelpers'
22
22
 
23
-
24
23
  # Helpers specific to Loggability specs
25
24
  module SpecHelpers
26
25
 
@@ -106,5 +105,6 @@ RSpec.configure do |c|
106
105
  c.include( Loggability::SpecHelpers )
107
106
  c.filter_run_excluding( :configurability ) unless defined?( Configurability )
108
107
 
108
+
109
109
  end
110
110
 
@@ -0,0 +1,27 @@
1
+ # -*- rspec -*-
2
+ #encoding: utf-8
3
+
4
+ require 'rspec'
5
+
6
+ require 'loggability/logger'
7
+ require 'loggability/log_device/appending'
8
+
9
+
10
+ describe Loggability::LogDevice::Appending do
11
+
12
+ let ( :logger ) { described_class.new( [] ) }
13
+
14
+
15
+ it "The target is an array" do
16
+ expect( logger.target ).to be_instance_of( Array )
17
+ end
18
+
19
+
20
+ it "can append to the array" do
21
+ logger.write("log message one")
22
+ logger.write("log message two")
23
+ expect( logger.target.size ).to eq( 2 )
24
+ end
25
+
26
+ end
27
+
@@ -0,0 +1,67 @@
1
+ # -*- rspec -*-
2
+ #encoding: utf-8
3
+
4
+ require 'securerandom'
5
+ require 'rspec'
6
+
7
+ require 'loggability/logger'
8
+ require 'loggability/log_device/datadog'
9
+
10
+
11
+ describe Loggability::LogDevice::Datadog do
12
+
13
+
14
+ let( :api_key ) { SecureRandom.hex(24) }
15
+ let( :http_client ) { instance_double(Net::HTTP) }
16
+
17
+
18
+ it "includes the configured API key in request headers" do
19
+ device = described_class.new(
20
+ api_key,
21
+ max_batch_size: 3,
22
+ batch_interval: 0.1,
23
+ executor_class: Concurrent::ImmediateExecutor )
24
+ device.instance_variable_set( :@http_client, http_client )
25
+
26
+ expect( http_client ).to receive( :request ) do |request|
27
+ expect( request ).to be_a( Net::HTTP::Post )
28
+ expect( request['Content-type'] ).to match( %r|application/json|i )
29
+ expect( request['DD-API-KEY'] ).to eq( api_key )
30
+ end.at_least( :once )
31
+
32
+ device.write( "message data" * 10 ) # 120 bytes
33
+ device.write( "message data" * 100 ) # 1200 bytes
34
+ device.write( "message data" * 85 ) # 1020 bytes
35
+ device.write( "message data" * 86 ) # 1032 bytes
36
+
37
+ sleep( 0.1 ) until device.logs_queue.empty?
38
+ end
39
+
40
+
41
+ it "includes the hostname in individual log messages" do
42
+ device = described_class.new(
43
+ api_key,
44
+ max_batch_size: 3,
45
+ batch_interval: 0.1,
46
+ executor_class: Concurrent::ImmediateExecutor )
47
+ device.instance_variable_set( :@http_client, http_client )
48
+
49
+ expect( http_client ).to receive( :request ) do |request|
50
+ expect( request ).to be_a( Net::HTTP::Post )
51
+
52
+ data = JSON.parse( request.body )
53
+
54
+ expect( data ).to all( be_a Hash )
55
+ expect( data ).to all( include('hostname' => device.hostname) )
56
+ end.at_least( :once )
57
+
58
+ device.write( "message data" * 10 ) # 120 bytes
59
+ device.write( "message data" * 100 ) # 1200 bytes
60
+ device.write( "message data" * 85 ) # 1020 bytes
61
+ device.write( "message data" * 86 ) # 1032 bytes
62
+
63
+ sleep( 0.1 ) until device.logs_queue.empty?
64
+ end
65
+
66
+ end
67
+