loggability 0.15.1 → 0.16.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data.tar.gz.sig +0 -0
- data/History.rdoc +9 -0
- data/Manifest.txt +12 -4
- data/lib/loggability.rb +12 -9
- data/lib/loggability/log_device.rb +86 -0
- data/lib/loggability/log_device/appending.rb +34 -0
- data/lib/loggability/log_device/datadog.rb +90 -0
- data/lib/loggability/log_device/file.rb +37 -0
- data/lib/loggability/log_device/http.rb +288 -0
- data/lib/loggability/logger.rb +11 -37
- data/spec/helpers.rb +1 -1
- data/spec/loggability/log_device/appending_spec.rb +27 -0
- data/spec/loggability/log_device/datadog_spec.rb +67 -0
- data/spec/loggability/log_device/file_spec.rb +27 -0
- data/spec/loggability/log_device/http_spec.rb +148 -0
- data/spec/loggability/logger_spec.rb +43 -2
- data/spec/loggability_spec.rb +7 -0
- metadata +27 -5
- metadata.gz.sig +0 -0
- data/ChangeLog +0 -667
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: '08efe977f97d6ff78421b1ff180716378aa56afb83b0aac80cf36550f4509e00'
|
4
|
+
data.tar.gz: 1af9769dad5f2eca223b413f1ead75dcbe2e322dc96f4d9598d3f275d3fa661a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9fd3be386367fab5fbf7b1afba29e234f452160af4514098880483d6d4c377bc201c4558aad2e1e19e8ddbd2c24afbef34cf25fe6047a7fbaa2a08c742141de5
|
7
|
+
data.tar.gz: e96d9bd362d3709c3035938e03fa303c635846c1722a270b2d9286c0736c46d103f00d05f5391ad96b3dbcbcc5c3f5262378e62a38ac704ccaddc53dc9644089
|
checksums.yaml.gz.sig
CHANGED
Binary file
|
data.tar.gz.sig
CHANGED
Binary file
|
data/History.rdoc
CHANGED
@@ -1,6 +1,15 @@
|
|
1
1
|
= Release History for loggability
|
2
2
|
|
3
3
|
---
|
4
|
+
== v0.16.0 [2020-02-24] Michael Granger <ged@faeriemud.org>
|
5
|
+
|
6
|
+
Improvements:
|
7
|
+
|
8
|
+
- Expose a "log device" API and add the ability to write to multiple devices
|
9
|
+
- Add a device for logging to an HTTP service
|
10
|
+
|
11
|
+
Thanks to Mahmood Khan <mkhan1484@gmail.com> for the patch.
|
12
|
+
|
4
13
|
|
5
14
|
== v0.15.1 [2020-01-09] Michael Granger <ged@faeriemud.org>
|
6
15
|
|
data/Manifest.txt
CHANGED
@@ -1,9 +1,5 @@
|
|
1
1
|
.simplecov
|
2
|
-
ChangeLog
|
3
2
|
History.rdoc
|
4
|
-
Manifest.txt
|
5
|
-
README.md
|
6
|
-
Rakefile
|
7
3
|
lib/loggability.rb
|
8
4
|
lib/loggability/constants.rb
|
9
5
|
lib/loggability/formatter.rb
|
@@ -14,8 +10,16 @@ lib/loggability/formatter/structured.rb
|
|
14
10
|
lib/loggability/logclient.rb
|
15
11
|
lib/loggability/logger.rb
|
16
12
|
lib/loggability/loghost.rb
|
13
|
+
lib/loggability/log_device.rb
|
14
|
+
lib/loggability/log_device/appending.rb
|
15
|
+
lib/loggability/log_device/datadog.rb
|
16
|
+
lib/loggability/log_device/file.rb
|
17
|
+
lib/loggability/log_device/http.rb
|
17
18
|
lib/loggability/override.rb
|
18
19
|
lib/loggability/spechelpers.rb
|
20
|
+
Manifest.txt
|
21
|
+
Rakefile
|
22
|
+
README.md
|
19
23
|
spec/helpers.rb
|
20
24
|
spec/loggability/formatter/color_spec.rb
|
21
25
|
spec/loggability/formatter/default_spec.rb
|
@@ -24,6 +28,10 @@ spec/loggability/formatter/structured_spec.rb
|
|
24
28
|
spec/loggability/formatter_spec.rb
|
25
29
|
spec/loggability/logger_spec.rb
|
26
30
|
spec/loggability/loghost_spec.rb
|
31
|
+
spec/loggability/log_device/appending_spec.rb
|
32
|
+
spec/loggability/log_device/datadog_spec.rb
|
33
|
+
spec/loggability/log_device/file_spec.rb
|
34
|
+
spec/loggability/log_device/http_spec.rb
|
27
35
|
spec/loggability/override_spec.rb
|
28
36
|
spec/loggability/spechelpers_spec.rb
|
29
37
|
spec/loggability_spec.rb
|
data/lib/loggability.rb
CHANGED
@@ -9,7 +9,7 @@ require 'date'
|
|
9
9
|
module Loggability
|
10
10
|
|
11
11
|
# Package version constant
|
12
|
-
VERSION = '0.
|
12
|
+
VERSION = '0.16.0'
|
13
13
|
|
14
14
|
# The key for the global logger (Loggability's own logger)
|
15
15
|
GLOBAL_KEY = :__global__
|
@@ -29,7 +29,7 @@ module Loggability
|
|
29
29
|
(?<severity>(?i:debug|info|warn|error|fatal))
|
30
30
|
(?:
|
31
31
|
\s+
|
32
|
-
(?<target>(?:[\w
|
32
|
+
(?<target>(?:[\w\-/:\.\[\]]|\\[ ])+)
|
33
33
|
)?
|
34
34
|
(?: \s+\(
|
35
35
|
(?<format>\w+)
|
@@ -38,10 +38,16 @@ module Loggability
|
|
38
38
|
$
|
39
39
|
}x
|
40
40
|
|
41
|
-
require 'loggability/constants'
|
42
|
-
include Loggability::Constants
|
43
41
|
|
44
|
-
|
42
|
+
# Automatically load subordinate classes/modules
|
43
|
+
autoload :Constants, 'loggability/constants'
|
44
|
+
autoload :LogDevice, 'loggability/log_device'
|
45
|
+
autoload :Logger, 'loggability/logger'
|
46
|
+
autoload :LogHost, 'loggability/loghost'
|
47
|
+
autoload :LogClient, 'loggability/logclient'
|
48
|
+
autoload :Override, 'loggability/override'
|
49
|
+
|
50
|
+
include Loggability::Constants
|
45
51
|
|
46
52
|
|
47
53
|
##
|
@@ -55,10 +61,6 @@ module Loggability
|
|
55
61
|
@config = CONFIG_DEFAULTS.dup.freeze
|
56
62
|
|
57
63
|
|
58
|
-
# Automatically log the log host and log client mixins when they're referenced
|
59
|
-
autoload :LogHost, 'loggability/loghost'
|
60
|
-
autoload :LogClient, 'loggability/logclient'
|
61
|
-
autoload :Override, 'loggability/override'
|
62
64
|
|
63
65
|
|
64
66
|
### Cast the given +device+ to a Loggability::Logger, if possible, and return it. If
|
@@ -321,6 +323,7 @@ module Loggability
|
|
321
323
|
target = case target
|
322
324
|
when 'STDOUT' then $stdout
|
323
325
|
when 'STDERR' then $stderr
|
326
|
+
when /:/ then Loggability::LogDevice.parse_device_spec( target )
|
324
327
|
else
|
325
328
|
target
|
326
329
|
end
|
@@ -0,0 +1,86 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
# vim: set nosta noet ts=4 sw=4:
|
3
|
+
# encoding: utf-8
|
4
|
+
|
5
|
+
require 'loggability' unless defined?( Loggability )
|
6
|
+
|
7
|
+
|
8
|
+
# An abstract base class for logging devices. A device manages the actual writing of messages
|
9
|
+
# to whatever destination logs are supposed to be shipped to, along with any buffering,
|
10
|
+
# encoding, or serialization that needs to be done.
|
11
|
+
#
|
12
|
+
# Log devices are loadable by name via the ::create method if they are declared in a
|
13
|
+
# directory named `loggability/log_device/` in the gem path.
|
14
|
+
#
|
15
|
+
# Concrete log devices are required to implement two methods: #write and #close.
|
16
|
+
#
|
17
|
+
# [write]
|
18
|
+
# Takes one argument, which is the message that needs to be written.
|
19
|
+
#
|
20
|
+
# [close]
|
21
|
+
# Close any open filehandles or connections established by the device.
|
22
|
+
class Loggability::LogDevice
|
23
|
+
|
24
|
+
|
25
|
+
# Regexp used to split up logging devices in config lines
|
26
|
+
DEVICE_TARGET_REGEX = /^([\s*a-z]\w*)(?:\[(.*)\])?/
|
27
|
+
|
28
|
+
|
29
|
+
### Parses out the target class name and its arguments from the +target_spec+
|
30
|
+
### then requires the subclass and instantiates it by passing the arguments.
|
31
|
+
### The +target_spec+ comes from a config file in the format of:
|
32
|
+
###
|
33
|
+
### logging:
|
34
|
+
### datadog[data_dog_api_key]
|
35
|
+
###
|
36
|
+
### In the above example:
|
37
|
+
### * "datadog" is the log device to send logs to
|
38
|
+
### * "data_dog_api_key" is the argument that will be passed onto the datadog
|
39
|
+
### log device's constructor
|
40
|
+
def self::parse_device_spec( target_spec )
|
41
|
+
targets = target_spec.split( ':' ).compact
|
42
|
+
return targets.map do |t|
|
43
|
+
target_subclass = t[ DEVICE_TARGET_REGEX, 1 ]&.strip().to_sym
|
44
|
+
target_subclass_args = t[ DEVICE_TARGET_REGEX, 2 ]
|
45
|
+
|
46
|
+
self.create( target_subclass, target_subclass_args )
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
|
51
|
+
### Requires the subclass and instantiates it with the passed-in arguments and
|
52
|
+
### then returns an instance of it.
|
53
|
+
def self::create( target, *target_args )
|
54
|
+
modname = target.to_s.capitalize
|
55
|
+
|
56
|
+
self.load_device_type( target ) unless self.const_defined?( modname, false )
|
57
|
+
subclass = self.const_get( modname, false )
|
58
|
+
|
59
|
+
return subclass.new( *target_args )
|
60
|
+
rescue NameError => err
|
61
|
+
raise LoadError, "failed to load %s LogDevice: %s" % [ target, err.message ]
|
62
|
+
end
|
63
|
+
|
64
|
+
|
65
|
+
### Attempt to load a LogDevice of the given +type+.
|
66
|
+
def self::load_device_type( type )
|
67
|
+
require_path = "loggability/log_device/%s" % [ type.to_s.downcase ]
|
68
|
+
require( require_path )
|
69
|
+
end
|
70
|
+
|
71
|
+
|
72
|
+
### Write a +message+ to the device. This needs to be overridden by concrete
|
73
|
+
### subclasses; calling this implementation will raise an NotImplementedError.
|
74
|
+
def write( message )
|
75
|
+
raise NotImplementedError, "%s is not implemented by %s" % [ __callee__, self.class.name ]
|
76
|
+
end
|
77
|
+
|
78
|
+
|
79
|
+
### Close the device. This needs to be overridden by concrete subclasses;
|
80
|
+
### calling this implementation will raise an NotImplementedError.
|
81
|
+
def close
|
82
|
+
raise NotImplementedError, "%s is not implemented by %s" % [ __callee__, self.class.name ]
|
83
|
+
end
|
84
|
+
|
85
|
+
end # class Loggability::LogDevice
|
86
|
+
|
@@ -0,0 +1,34 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
# vim: set nosta noet ts=4 sw=4:
|
3
|
+
# encoding: utf-8
|
4
|
+
|
5
|
+
require 'loggability/logger' unless defined?( Loggability::Logger )
|
6
|
+
|
7
|
+
# A log device that appends to the object it's constructed with instead of writing
|
8
|
+
# to a file descriptor or a file.
|
9
|
+
class Loggability::LogDevice::Appending < Loggability::LogDevice
|
10
|
+
|
11
|
+
### Create a new +Appending+ log device that will append content to +array+.
|
12
|
+
def initialize( target )
|
13
|
+
@target = target || []
|
14
|
+
end
|
15
|
+
|
16
|
+
|
17
|
+
######
|
18
|
+
public
|
19
|
+
######
|
20
|
+
|
21
|
+
# The target of the log device
|
22
|
+
attr_reader :target
|
23
|
+
|
24
|
+
|
25
|
+
### Append the specified +message+ to the target.
|
26
|
+
def write( message )
|
27
|
+
@target << message
|
28
|
+
end
|
29
|
+
|
30
|
+
|
31
|
+
### No-op -- this is here just so Logger doesn't complain
|
32
|
+
def close; end
|
33
|
+
|
34
|
+
end # class Loggability::LogDevice::Appending
|
@@ -0,0 +1,90 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
# vim: set nosta noet ts=4 sw=4:
|
3
|
+
# encoding: utf-8
|
4
|
+
|
5
|
+
require 'uri'
|
6
|
+
require 'socket'
|
7
|
+
require 'net/https'
|
8
|
+
require 'json'
|
9
|
+
require 'concurrent'
|
10
|
+
require 'loggability/logger' unless defined?( Loggability::Logger )
|
11
|
+
|
12
|
+
require 'loggability/log_device/http'
|
13
|
+
|
14
|
+
|
15
|
+
# A log device that sends logs to Datadog's HTTP endpoint
|
16
|
+
# for receiving logs
|
17
|
+
class Loggability::LogDevice::Datadog < Loggability::LogDevice::Http
|
18
|
+
|
19
|
+
### Datadog's HTTP endpoint URL for sending logs to
|
20
|
+
DEFAULT_ENDPOINT = URI( "https://http-intake.logs.datadoghq.com/v1/input" )
|
21
|
+
|
22
|
+
### The max number of messages that can be sent to datadog in a single payload
|
23
|
+
MAX_BATCH_SIZE = 480
|
24
|
+
|
25
|
+
### The max size in bytes for a single message.
|
26
|
+
### Limiting the message size to 200kB to leave room for other info such as
|
27
|
+
### tags, metadata, etc.
|
28
|
+
### DataDog's max size for a single log entry is 256kB
|
29
|
+
MAX_MESSAGE_BYTESIZE = 204_800
|
30
|
+
|
31
|
+
### The max size in bytes of all messages in the batch.
|
32
|
+
### Limiting the total messages size to 4MB to leave room for other info such as
|
33
|
+
### tags, metadata, etc.
|
34
|
+
### Datadog's max size for the entire payload is 5MB
|
35
|
+
MAX_BATCH_BYTESIZE = 4_194_304
|
36
|
+
|
37
|
+
# Override the default HTTP device options for sending logs to DD
|
38
|
+
DEFAULT_OPTIONS = {
|
39
|
+
max_batch_size: MAX_BATCH_SIZE,
|
40
|
+
max_message_bytesize: MAX_MESSAGE_BYTESIZE,
|
41
|
+
max_batch_bytesize: MAX_BATCH_BYTESIZE,
|
42
|
+
}
|
43
|
+
|
44
|
+
|
45
|
+
### Create a new Datadog
|
46
|
+
def initialize( api_key, endpoint=DEFAULT_ENDPOINT, options={} )
|
47
|
+
if endpoint.is_a?( Hash )
|
48
|
+
options = endpoint
|
49
|
+
endpoint = DEFAULT_ENDPOINT
|
50
|
+
end
|
51
|
+
|
52
|
+
super( endpoint, options )
|
53
|
+
|
54
|
+
@api_key = api_key
|
55
|
+
@hostname = Socket.gethostname
|
56
|
+
end
|
57
|
+
|
58
|
+
|
59
|
+
######
|
60
|
+
public
|
61
|
+
######
|
62
|
+
|
63
|
+
##
|
64
|
+
# The name of the current host
|
65
|
+
attr_reader :hostname
|
66
|
+
|
67
|
+
##
|
68
|
+
# The configured Datadog API key
|
69
|
+
attr_reader :api_key
|
70
|
+
|
71
|
+
|
72
|
+
### Format an individual log +message+ for Datadog.
|
73
|
+
def format_log_message( message )
|
74
|
+
return {
|
75
|
+
hostname: self.hostname,
|
76
|
+
message: message
|
77
|
+
}.to_json
|
78
|
+
end
|
79
|
+
|
80
|
+
|
81
|
+
### Overridden to add the configured API key to the headers of each request.
|
82
|
+
def make_batch_request
|
83
|
+
request = super
|
84
|
+
|
85
|
+
request[ 'DD-API-KEY' ] = self.api_key
|
86
|
+
|
87
|
+
return request
|
88
|
+
end
|
89
|
+
|
90
|
+
end # class Loggability::LogDevice::Datadog
|
@@ -0,0 +1,37 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
# vim: set nosta noet ts=4 sw=4:
|
3
|
+
# encoding: utf-8
|
4
|
+
|
5
|
+
require 'logger'
|
6
|
+
require 'loggability/logger' unless defined?( Loggability::Logger )
|
7
|
+
|
8
|
+
# A log device that delegates to the ruby's default Logger's file device and writes
|
9
|
+
# to a file descriptor or a file.
|
10
|
+
class Loggability::LogDevice::File < Loggability::LogDevice
|
11
|
+
|
12
|
+
### Create a new +File+ device that will write to the file using the built-in ruby's +File+ log device
|
13
|
+
def initialize( target )
|
14
|
+
@target = ::Logger::LogDevice.new( target )
|
15
|
+
end
|
16
|
+
|
17
|
+
|
18
|
+
######
|
19
|
+
public
|
20
|
+
######
|
21
|
+
|
22
|
+
# The target of the log device
|
23
|
+
attr_reader :target
|
24
|
+
|
25
|
+
|
26
|
+
### Append the specified +message+ to the target.
|
27
|
+
def write( message )
|
28
|
+
self.target.write( message )
|
29
|
+
end
|
30
|
+
|
31
|
+
|
32
|
+
### close the file
|
33
|
+
def close
|
34
|
+
self.target.close
|
35
|
+
end
|
36
|
+
|
37
|
+
end # class Loggability::LogDevice::File
|
@@ -0,0 +1,288 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
# vim: set nosta noet ts=4 sw=4:
|
3
|
+
# encoding: utf-8
|
4
|
+
|
5
|
+
require 'socket'
|
6
|
+
require 'uri'
|
7
|
+
require 'net/https'
|
8
|
+
require 'json'
|
9
|
+
require 'concurrent'
|
10
|
+
require 'loggability/logger' unless defined?( Loggability::Logger )
|
11
|
+
|
12
|
+
require 'loggability/log_device'
|
13
|
+
|
14
|
+
# This is the a generalized class that allows its subclasses to send log
|
15
|
+
# messages to HTTP endpoints asynchronously on a separate thread.
|
16
|
+
class Loggability::LogDevice::Http < Loggability::LogDevice
|
17
|
+
|
18
|
+
# The default HTTP endpoint URL to send logs to
|
19
|
+
DEFAULT_ENDPOINT = "http://localhost:12775/v1/logs"
|
20
|
+
|
21
|
+
# The default maximum number of messages that can be sent to the server in a single payload
|
22
|
+
DEFAULT_MAX_BATCH_SIZE = 100
|
23
|
+
|
24
|
+
# The default max size in bytes for a single message.
|
25
|
+
DEFAULT_MAX_MESSAGE_BYTESIZE = 2 * 16
|
26
|
+
|
27
|
+
# The default number of seconds between batches
|
28
|
+
DEFAULT_BATCH_INTERVAL = 60
|
29
|
+
|
30
|
+
# The default number of seconds to wait for data to be written before timing out
|
31
|
+
DEFAULT_WRITE_TIMEOUT = 15
|
32
|
+
|
33
|
+
# The default Executor class to use for asynchronous tasks
|
34
|
+
DEFAULT_EXECUTOR_CLASS = Concurrent::SingleThreadExecutor
|
35
|
+
|
36
|
+
# The default options for new instances
|
37
|
+
DEFAULT_OPTIONS = {
|
38
|
+
execution_interval: DEFAULT_BATCH_INTERVAL,
|
39
|
+
write_timeout: DEFAULT_WRITE_TIMEOUT,
|
40
|
+
max_batch_size: DEFAULT_MAX_BATCH_SIZE,
|
41
|
+
max_message_bytesize: DEFAULT_MAX_MESSAGE_BYTESIZE,
|
42
|
+
executor_class: DEFAULT_EXECUTOR_CLASS,
|
43
|
+
}
|
44
|
+
|
45
|
+
|
46
|
+
### Initialize the HTTP log device to send to the specified +endpoint+ with the
|
47
|
+
### given +options+. Valid options are:
|
48
|
+
###
|
49
|
+
### [:batch_interval]
|
50
|
+
### Maximum number of seconds between batches
|
51
|
+
### [:write_timeout]
|
52
|
+
### How many seconds to wait for data to be written while sending a batch
|
53
|
+
### [:max_batch_size]
|
54
|
+
### The maximum number of messages that can be in a single batch
|
55
|
+
### [:max_batch_bytesize]
|
56
|
+
### The maximum number of bytes that can be in the payload of a single batch
|
57
|
+
### [:max_message_bytesize]
|
58
|
+
### The maximum number of bytes that can be in a single message
|
59
|
+
### [:executor_class]
|
60
|
+
### The Concurrent executor class to use for asynchronous tasks.
|
61
|
+
def initialize( endpoint=DEFAULT_ENDPOINT, opts={} )
|
62
|
+
if endpoint.is_a?( Hash )
|
63
|
+
opts = endpoint
|
64
|
+
endpoint = DEFAULT_ENDPOINT
|
65
|
+
end
|
66
|
+
|
67
|
+
opts = DEFAULT_OPTIONS.merge( opts )
|
68
|
+
|
69
|
+
@endpoint = URI( endpoint ).freeze
|
70
|
+
@logs_queue = Queue.new
|
71
|
+
|
72
|
+
@batch_interval = opts[:batch_interval] || DEFAULT_BATCH_INTERVAL
|
73
|
+
@write_timeout = opts[:write_timeout] || DEFAULT_WRITE_TIMEOUT
|
74
|
+
@max_batch_size = opts[:max_batch_size] || DEFAULT_MAX_BATCH_SIZE
|
75
|
+
@max_message_bytesize = opts[:max_message_bytesize] || DEFAULT_MAX_MESSAGE_BYTESIZE
|
76
|
+
@executor_class = opts[:executor_class] || DEFAULT_EXECUTOR_CLASS
|
77
|
+
|
78
|
+
@max_batch_bytesize = opts[:max_batch_bytesize] || @max_batch_size * @max_message_bytesize
|
79
|
+
@last_send_time = Concurrent.monotonic_time
|
80
|
+
end
|
81
|
+
|
82
|
+
|
83
|
+
######
|
84
|
+
public
|
85
|
+
######
|
86
|
+
|
87
|
+
##
|
88
|
+
# The single thread pool executor
|
89
|
+
attr_reader :executor
|
90
|
+
|
91
|
+
##
|
92
|
+
# The URI of the endpoint to send messages to
|
93
|
+
attr_reader :endpoint
|
94
|
+
|
95
|
+
##
|
96
|
+
# The Queue that contains any log messages which have not yet been sent to the
|
97
|
+
# logging service.
|
98
|
+
attr_reader :logs_queue
|
99
|
+
|
100
|
+
##
|
101
|
+
# The monotonic clock time when the last batch of logs were sent
|
102
|
+
attr_accessor :last_send_time
|
103
|
+
|
104
|
+
##
|
105
|
+
# Number of seconds after the task completes before the task is performed again.
|
106
|
+
attr_reader :batch_interval
|
107
|
+
|
108
|
+
##
|
109
|
+
# How many seconds to wait for data to be written while sending a batch
|
110
|
+
attr_reader :write_timeout
|
111
|
+
|
112
|
+
##
|
113
|
+
# The maximum number of messages to post at one time
|
114
|
+
attr_reader :max_batch_size
|
115
|
+
|
116
|
+
##
|
117
|
+
# The maximum number of bytes of a single message to include in a batch
|
118
|
+
attr_reader :max_message_bytesize
|
119
|
+
|
120
|
+
##
|
121
|
+
# The maximum number of bytes that will be included in a single POST
|
122
|
+
attr_reader :max_batch_bytesize
|
123
|
+
|
124
|
+
##
|
125
|
+
# The Concurrent executor class to use for asynchronous tasks
|
126
|
+
attr_reader :executor_class
|
127
|
+
|
128
|
+
##
|
129
|
+
# The timer task thread
|
130
|
+
attr_reader :timer_task
|
131
|
+
|
132
|
+
|
133
|
+
### LogDevice API -- write a message to the HTTP device.
|
134
|
+
def write( message )
|
135
|
+
self.start unless self.running?
|
136
|
+
self.logs_queue.enq( message )
|
137
|
+
self.send_logs
|
138
|
+
end
|
139
|
+
|
140
|
+
|
141
|
+
### LogDevice API -- stop the batch thread and close the http connection
|
142
|
+
def close
|
143
|
+
self.stop
|
144
|
+
self.http_client.finish
|
145
|
+
rescue IOError
|
146
|
+
# ignore it since http session has not yet started.
|
147
|
+
end
|
148
|
+
|
149
|
+
|
150
|
+
### Starts a thread pool with a single thread.
|
151
|
+
def start
|
152
|
+
self.start_executor
|
153
|
+
self.start_timer_task
|
154
|
+
end
|
155
|
+
|
156
|
+
|
157
|
+
### Returns +true+ if the device has started sending messages to the logging endpoint.
|
158
|
+
def running?
|
159
|
+
return self.executor&.running?
|
160
|
+
end
|
161
|
+
|
162
|
+
|
163
|
+
### Shutdown the executor, which is a pool of single thread
|
164
|
+
### waits 3 seconds for shutdown to complete
|
165
|
+
def stop
|
166
|
+
return unless self.running?
|
167
|
+
|
168
|
+
self.timer_task.shutdown if self.timer_task&.running?
|
169
|
+
self.executor.shutdown
|
170
|
+
|
171
|
+
unless self.executor.wait_for_termination( 3 )
|
172
|
+
self.executor.halt
|
173
|
+
self.executor.wait_for_termination( 3 )
|
174
|
+
end
|
175
|
+
end
|
176
|
+
|
177
|
+
|
178
|
+
### Start the background thread that sends messages.
|
179
|
+
def start_executor
|
180
|
+
@executor = self.executor_class.new
|
181
|
+
@executor.auto_terminate = true unless @executor.serialized?
|
182
|
+
end
|
183
|
+
|
184
|
+
|
185
|
+
### Create a timer task that calls that sends logs at regular interval
|
186
|
+
def start_timer_task
|
187
|
+
@timer_task = Concurrent::TimerTask.execute( execution_interval: self.batch_interval ) do
|
188
|
+
self.send_logs
|
189
|
+
end
|
190
|
+
end
|
191
|
+
|
192
|
+
|
193
|
+
### Sends a batch of log messages to the logging service. This executes inside
|
194
|
+
### the sending thread.
|
195
|
+
def send_logs
|
196
|
+
self.executor.post do
|
197
|
+
if self.batch_ready?
|
198
|
+
# p "Batch ready; sending."
|
199
|
+
request = self.make_batch_request
|
200
|
+
request.body = self.get_next_log_payload
|
201
|
+
|
202
|
+
# p "Sending request", request
|
203
|
+
|
204
|
+
self.http_client.request( request ) do |res|
|
205
|
+
p( res ) if $DEBUG
|
206
|
+
end
|
207
|
+
|
208
|
+
self.last_send_time = Concurrent.monotonic_time
|
209
|
+
else
|
210
|
+
# p "Batch not ready yet."
|
211
|
+
end
|
212
|
+
end
|
213
|
+
end
|
214
|
+
|
215
|
+
|
216
|
+
### Returns +true+ if a batch of logs is ready to be sent.
|
217
|
+
def batch_ready?
|
218
|
+
seconds_since_last_send = Concurrent.monotonic_time - self.last_send_time
|
219
|
+
|
220
|
+
return self.logs_queue.size >= self.max_batch_size ||
|
221
|
+
seconds_since_last_send >= self.batch_interval
|
222
|
+
end
|
223
|
+
alias_method :has_batch_ready?, :batch_ready?
|
224
|
+
|
225
|
+
|
226
|
+
### Returns a new HTTP request (a subclass of Net::HTTPRequest) suitable for
|
227
|
+
### sending the next batch of logs to the service. Defaults to a POST of JSON data. This
|
228
|
+
### executes inside the sending thread.
|
229
|
+
def make_batch_request
|
230
|
+
request = Net::HTTP::Post.new( self.endpoint.path )
|
231
|
+
request[ 'Content-Type' ] = 'application/json'
|
232
|
+
|
233
|
+
return request
|
234
|
+
end
|
235
|
+
|
236
|
+
|
237
|
+
### Dequeue pending log messages to send to the service and return them as a
|
238
|
+
### suitably-encoded String. The default is a JSON Array. This executes inside
|
239
|
+
### the sending thread.
|
240
|
+
def get_next_log_payload
|
241
|
+
buf = []
|
242
|
+
count = 0
|
243
|
+
bytes = 0
|
244
|
+
|
245
|
+
# Be conservative so as not to overflow
|
246
|
+
max_size = self.max_batch_bytesize - self.max_message_bytesize - 2 # for the outer Array
|
247
|
+
|
248
|
+
while count < self.max_batch_size && bytes < max_size && !self.logs_queue.empty?
|
249
|
+
formatted_message = self.format_log_message( self.logs_queue.deq )
|
250
|
+
|
251
|
+
count += 1
|
252
|
+
bytes += formatted_message.bytesize + 3 # comma and delimiters
|
253
|
+
|
254
|
+
buf << formatted_message
|
255
|
+
end
|
256
|
+
|
257
|
+
return '[' + buf.join(',') + ']'
|
258
|
+
end
|
259
|
+
|
260
|
+
|
261
|
+
### Returns the given +message+ in whatever format individual log messages are
|
262
|
+
### expected to be in by the service. The default just returns the stringified
|
263
|
+
### +message+. This executes inside the sending thread.
|
264
|
+
def format_log_message( message )
|
265
|
+
return message.to_s[ 0 ... self.max_message_bytesize ].dump
|
266
|
+
end
|
267
|
+
|
268
|
+
|
269
|
+
### sets up a configured http object ready to instantiate connections
|
270
|
+
def http_client
|
271
|
+
return @http_client ||= begin
|
272
|
+
uri = URI.parse( self.endpoint )
|
273
|
+
|
274
|
+
http = Net::HTTP.new( uri.host, uri.port )
|
275
|
+
http.write_timeout = self.write_timeout
|
276
|
+
|
277
|
+
if uri.scheme == 'https'
|
278
|
+
http.use_ssl = true
|
279
|
+
http.verify_mode = OpenSSL::SSL::VERIFY_PEER
|
280
|
+
end
|
281
|
+
|
282
|
+
http
|
283
|
+
end
|
284
|
+
end
|
285
|
+
|
286
|
+
|
287
|
+
end # class Loggability::LogDevice::Http
|
288
|
+
|