logtail-ruby 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.github/workflows/main.yml +33 -0
- data/.gitignore +24 -0
- data/.rspec +2 -0
- data/CHANGELOG.md +12 -0
- data/Gemfile +10 -0
- data/LICENSE.md +15 -0
- data/README.md +4 -0
- data/Rakefile +72 -0
- data/lib/logtail.rb +36 -0
- data/lib/logtail/config.rb +154 -0
- data/lib/logtail/config/integrations.rb +17 -0
- data/lib/logtail/context.rb +9 -0
- data/lib/logtail/contexts.rb +12 -0
- data/lib/logtail/contexts/http.rb +31 -0
- data/lib/logtail/contexts/release.rb +52 -0
- data/lib/logtail/contexts/runtime.rb +23 -0
- data/lib/logtail/contexts/session.rb +24 -0
- data/lib/logtail/contexts/system.rb +29 -0
- data/lib/logtail/contexts/user.rb +28 -0
- data/lib/logtail/current_context.rb +168 -0
- data/lib/logtail/event.rb +36 -0
- data/lib/logtail/events.rb +10 -0
- data/lib/logtail/events/controller_call.rb +44 -0
- data/lib/logtail/events/error.rb +40 -0
- data/lib/logtail/events/exception.rb +10 -0
- data/lib/logtail/events/sql_query.rb +26 -0
- data/lib/logtail/events/template_render.rb +25 -0
- data/lib/logtail/integration.rb +40 -0
- data/lib/logtail/integrator.rb +50 -0
- data/lib/logtail/log_devices.rb +8 -0
- data/lib/logtail/log_devices/http.rb +368 -0
- data/lib/logtail/log_devices/http/flushable_dropping_sized_queue.rb +52 -0
- data/lib/logtail/log_devices/http/request_attempt.rb +20 -0
- data/lib/logtail/log_entry.rb +110 -0
- data/lib/logtail/logger.rb +270 -0
- data/lib/logtail/logtail.rb +36 -0
- data/lib/logtail/timer.rb +21 -0
- data/lib/logtail/util.rb +7 -0
- data/lib/logtail/util/non_nil_hash_builder.rb +40 -0
- data/lib/logtail/version.rb +3 -0
- data/logtail-ruby.gemspec +43 -0
- data/spec/README.md +13 -0
- data/spec/logtail/current_context_spec.rb +113 -0
- data/spec/logtail/events/controller_call_spec.rb +12 -0
- data/spec/logtail/events/error_spec.rb +15 -0
- data/spec/logtail/log_devices/http_spec.rb +185 -0
- data/spec/logtail/log_entry_spec.rb +22 -0
- data/spec/logtail/logger_spec.rb +227 -0
- data/spec/spec_helper.rb +22 -0
- data/spec/support/logtail.rb +5 -0
- data/spec/support/socket_hostname.rb +12 -0
- data/spec/support/timecop.rb +3 -0
- data/spec/support/webmock.rb +3 -0
- metadata +238 -0
@@ -0,0 +1,52 @@
|
|
1
|
+
module Logtail
|
2
|
+
module LogDevices
|
3
|
+
class HTTP
|
4
|
+
# A simple thread-safe queue implementation that provides a #flush method.
|
5
|
+
# The built-in ruby `Queue` class does not provide a #flush method that allows
|
6
|
+
# the caller to retrieve all items on the queue in one call. The Ruby `SizedQueue` also
|
7
|
+
# implements thread waiting, which is something we want to avoid. To keep things
|
8
|
+
# simple and straight-forward, we designed this queue class.
|
9
|
+
# @private
|
10
|
+
class FlushableDroppingSizedQueue
|
11
|
+
def initialize(max_size)
|
12
|
+
@lock = Mutex.new
|
13
|
+
@max_size = max_size
|
14
|
+
@array = []
|
15
|
+
end
|
16
|
+
|
17
|
+
# Adds a message to the queue
|
18
|
+
def enq(msg)
|
19
|
+
@lock.synchronize do
|
20
|
+
if !full?
|
21
|
+
@array << msg
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
# Removes a single item from the queue
|
27
|
+
def deq
|
28
|
+
@lock.synchronize do
|
29
|
+
@array.pop
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
# Flushes all message from the queue and returns them.
|
34
|
+
def flush
|
35
|
+
@lock.synchronize do
|
36
|
+
old = @array
|
37
|
+
@array = []
|
38
|
+
return old
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def full?
|
43
|
+
size >= @max_size
|
44
|
+
end
|
45
|
+
|
46
|
+
def size
|
47
|
+
@array.size
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
module Logtail
|
2
|
+
module LogDevices
|
3
|
+
class HTTP
|
4
|
+
# Represents an attempt to deliver a request. Requests can be retried, hence
|
5
|
+
# why we keep track of the number of attempts.
|
6
|
+
class RequestAttempt
|
7
|
+
attr_reader :attempts, :request
|
8
|
+
|
9
|
+
def initialize(req)
|
10
|
+
@attempts = 0
|
11
|
+
@request = req
|
12
|
+
end
|
13
|
+
|
14
|
+
def attempted!
|
15
|
+
@attempts += 1
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,110 @@
|
|
1
|
+
require "socket"
|
2
|
+
require "time"
|
3
|
+
|
4
|
+
require "logtail/contexts"
|
5
|
+
require "logtail/events"
|
6
|
+
|
7
|
+
module Logtail
|
8
|
+
# Represents a new log entry into the log. This is an intermediary class between
|
9
|
+
# `Logger` and the log device that you set it up with.
|
10
|
+
class LogEntry #:nodoc:
|
11
|
+
BINARY_LIMIT_THRESHOLD = 1_000.freeze
|
12
|
+
DT_PRECISION = 6.freeze
|
13
|
+
MESSAGE_MAX_BYTES = 8192.freeze
|
14
|
+
|
15
|
+
attr_reader :context_snapshot, :event, :level, :message, :progname, :tags, :time
|
16
|
+
|
17
|
+
# Creates a log entry suitable to be sent to the Logtail API.
|
18
|
+
# @param level [Integer] the log level / severity
|
19
|
+
# @param time [Time] the exact time the log message was written
|
20
|
+
# @param progname [String] the progname scope for the log message
|
21
|
+
# @param message [String] Human readable log message.
|
22
|
+
# @param context_snapshot [Hash] structured data representing a snapshot of the context at
|
23
|
+
# the given point in time.
|
24
|
+
# @param event [Logtail.Event] structured data representing the log line event. This should be
|
25
|
+
# an instance of {Logtail.Event}.
|
26
|
+
# @return [LogEntry] the resulting LogEntry object
|
27
|
+
def initialize(level, time, progname, message, context_snapshot, event, options = {})
|
28
|
+
@level = level
|
29
|
+
@time = time.utc
|
30
|
+
@progname = progname
|
31
|
+
|
32
|
+
# If the message is not a string we call inspect to ensure it is a string.
|
33
|
+
# This follows the default behavior set by ::Logger
|
34
|
+
# See: https://github.com/ruby/ruby/blob/trunk/lib/logger.rb#L615
|
35
|
+
@message = message.is_a?(String) ? message : message.inspect
|
36
|
+
@message = @message.byteslice(0, MESSAGE_MAX_BYTES)
|
37
|
+
@tags = options[:tags]
|
38
|
+
@context_snapshot = context_snapshot
|
39
|
+
@event = event
|
40
|
+
end
|
41
|
+
|
42
|
+
# Builds a hash representation containing simple objects, suitable for serialization (JSON).
|
43
|
+
def to_hash(options = {})
|
44
|
+
options ||= {}
|
45
|
+
hash = {
|
46
|
+
:level => level,
|
47
|
+
:dt => formatted_dt,
|
48
|
+
:message => message
|
49
|
+
}
|
50
|
+
|
51
|
+
if !tags.nil? && tags.length > 0
|
52
|
+
hash[:tags] = tags
|
53
|
+
end
|
54
|
+
|
55
|
+
if !event.nil?
|
56
|
+
hash.merge!(event)
|
57
|
+
end
|
58
|
+
|
59
|
+
if !context_snapshot.nil? && context_snapshot.length > 0
|
60
|
+
hash[:context] = context_snapshot
|
61
|
+
end
|
62
|
+
|
63
|
+
if options[:only]
|
64
|
+
hash.select do |key, _value|
|
65
|
+
options[:only].include?(key)
|
66
|
+
end
|
67
|
+
elsif options[:except]
|
68
|
+
hash.select do |key, _value|
|
69
|
+
!options[:except].include?(key)
|
70
|
+
end
|
71
|
+
else
|
72
|
+
hash
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
def inspect
|
77
|
+
to_s
|
78
|
+
end
|
79
|
+
|
80
|
+
def to_json(options = {})
|
81
|
+
to_hash.to_json
|
82
|
+
end
|
83
|
+
|
84
|
+
def to_msgpack(*args)
|
85
|
+
to_hash.to_msgpack(*args)
|
86
|
+
end
|
87
|
+
|
88
|
+
# This is used when LogEntry objects make it to a non-Logtail logger.
|
89
|
+
def to_s
|
90
|
+
message + "\n"
|
91
|
+
end
|
92
|
+
|
93
|
+
private
|
94
|
+
def formatted_dt
|
95
|
+
@formatted_dt ||= time.iso8601(DT_PRECISION)
|
96
|
+
end
|
97
|
+
|
98
|
+
# Attempts to encode a non UTF-8 string into UTF-8, discarding invalid characters.
|
99
|
+
# If it fails, a nil is returned.
|
100
|
+
def encode_string(string)
|
101
|
+
string.encode('UTF-8', {
|
102
|
+
:invalid => :replace,
|
103
|
+
:undef => :replace,
|
104
|
+
:replace => '?'
|
105
|
+
})
|
106
|
+
rescue Exception
|
107
|
+
nil
|
108
|
+
end
|
109
|
+
end
|
110
|
+
end
|
@@ -0,0 +1,270 @@
|
|
1
|
+
require "logger"
|
2
|
+
require "msgpack"
|
3
|
+
|
4
|
+
require "logtail/config"
|
5
|
+
require "logtail/current_context"
|
6
|
+
require "logtail/log_devices"
|
7
|
+
require "logtail/log_entry"
|
8
|
+
|
9
|
+
module Logtail
|
10
|
+
# The Logtail Logger behaves exactly like the standard Ruby `::Logger`, except that it supports a
|
11
|
+
# transparent API for logging structured data and events.
|
12
|
+
#
|
13
|
+
# @example Basic logging
|
14
|
+
# logger.info "Payment rejected for customer #{customer_id}"
|
15
|
+
#
|
16
|
+
# @example Logging an event
|
17
|
+
# logger.info "Payment rejected", payment_rejected: {customer_id: customer_id, amount: 100}
|
18
|
+
class Logger < ::Logger
|
19
|
+
|
20
|
+
# @private
|
21
|
+
class Formatter
|
22
|
+
# Formatters get the formatted level from the logger.
|
23
|
+
SEVERITY_MAP = {
|
24
|
+
"DEBUG" => :debug,
|
25
|
+
"INFO" => :info,
|
26
|
+
"WARN" => :warn,
|
27
|
+
"ERROR" => :error,
|
28
|
+
"FATAL" => :fatal,
|
29
|
+
"UNKNOWN" => :unknown
|
30
|
+
}
|
31
|
+
EMPTY_ARRAY = []
|
32
|
+
|
33
|
+
private
|
34
|
+
def build_log_entry(severity, time, progname, logged_obj)
|
35
|
+
context_snapshot = CurrentContext.instance.snapshot
|
36
|
+
level = SEVERITY_MAP.fetch(severity)
|
37
|
+
tags = extract_active_support_tagged_logging_tags
|
38
|
+
|
39
|
+
if logged_obj.is_a?(Event)
|
40
|
+
LogEntry.new(level, time, progname, logged_obj.message, context_snapshot, logged_obj,
|
41
|
+
tags: tags)
|
42
|
+
elsif logged_obj.is_a?(Hash)
|
43
|
+
# Extract the tags
|
44
|
+
tags = tags.clone
|
45
|
+
tags.push(logged_obj.delete(:tag)) if logged_obj.key?(:tag)
|
46
|
+
tags.concat(logged_obj.delete(:tags)) if logged_obj.key?(:tags)
|
47
|
+
tags.uniq!
|
48
|
+
|
49
|
+
message = logged_obj.delete(:message)
|
50
|
+
|
51
|
+
LogEntry.new(level, time, progname, message, context_snapshot, logged_obj, tags: tags)
|
52
|
+
else
|
53
|
+
LogEntry.new(level, time, progname, logged_obj, context_snapshot, nil, tags: tags)
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
# Because of all the crazy ways Rails has attempted tags, we need this crazy method.
|
58
|
+
def extract_active_support_tagged_logging_tags
|
59
|
+
Thread.current[:activesupport_tagged_logging_tags] ||
|
60
|
+
Thread.current[tagged_logging_object_key_name] ||
|
61
|
+
EMPTY_ARRAY
|
62
|
+
end
|
63
|
+
|
64
|
+
def tagged_logging_object_key_name
|
65
|
+
@tagged_logging_object_key_name ||= "activesupport_tagged_logging_tags:#{object_id}"
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
# For use in development and test environments where you do not want metadata
|
70
|
+
# included in the log lines.
|
71
|
+
class MessageOnlyFormatter < Formatter
|
72
|
+
# This method is invoked when a log event occurs
|
73
|
+
def call(severity, timestamp, progname, msg)
|
74
|
+
log_entry = build_log_entry(severity, timestamp, progname, msg)
|
75
|
+
log_entry.to_s
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
# Structures your log messages as strings and appends metadata if
|
80
|
+
# `Logtail::Config.instance.append_metadata?` is true.
|
81
|
+
#
|
82
|
+
# Example message with metdata:
|
83
|
+
#
|
84
|
+
# My log message @metadata {"level":"info","dt":"2016-09-01T07:00:00.000000-05:00"}
|
85
|
+
#
|
86
|
+
class AugmentedFormatter < Formatter
|
87
|
+
METADATA_CALLOUT = " @metadata ".freeze
|
88
|
+
NEW_LINE = "\n".freeze
|
89
|
+
ESCAPED_NEW_LINE = "\\n".freeze
|
90
|
+
|
91
|
+
def call(severity, time, progname, msg)
|
92
|
+
log_entry = build_log_entry(severity, time, progname, msg)
|
93
|
+
metadata = log_entry.to_json(:except => [:message])
|
94
|
+
# use << for concatenation for performance reasons
|
95
|
+
log_entry.message.gsub(NEW_LINE, ESCAPED_NEW_LINE) << METADATA_CALLOUT <<
|
96
|
+
metadata << NEW_LINE
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
# Structures your log messages into JSON.
|
101
|
+
#
|
102
|
+
# logger = Logtail::Logger.new(STDOUT)
|
103
|
+
# logger.formatter = Logtail::JSONFormatter.new
|
104
|
+
#
|
105
|
+
# Example message:
|
106
|
+
#
|
107
|
+
# {"level":"info","dt":"2016-09-01T07:00:00.000000-05:00","message":"My log message"}
|
108
|
+
#
|
109
|
+
class JSONFormatter < Formatter
|
110
|
+
def call(severity, time, progname, msg)
|
111
|
+
# use << for concatenation for performance reasons
|
112
|
+
build_log_entry(severity, time, progname, msg).to_json << "\n"
|
113
|
+
end
|
114
|
+
end
|
115
|
+
|
116
|
+
# Passes through the LogEntry object. This is specifically used for the {Logtail::LogDevices::HTTP}
|
117
|
+
# class. This allows the IO device to format it however it wants. This is necessary for
|
118
|
+
# MessagePack because it requires a fixed array size before encoding. And since HTTP is
|
119
|
+
# sending data in batches, the encoding should happen there.
|
120
|
+
class PassThroughFormatter < Formatter
|
121
|
+
def call(severity, time, progname, msg)
|
122
|
+
build_log_entry(severity, time, progname, msg)
|
123
|
+
end
|
124
|
+
end
|
125
|
+
|
126
|
+
# Creates a new Logtail::Logger instance where the passed argument is an IO device. That is,
|
127
|
+
# anything that responds to `#write` and `#close`.
|
128
|
+
#
|
129
|
+
# Note, this method does *not* accept the same arguments as the standard Ruby `::Logger`.
|
130
|
+
# The Ruby `::Logger` accepts additional options controlling file rotation if the first argument
|
131
|
+
# is a file *name*. This is a design flaw that Logtail does not assume. Logging to a file, or
|
132
|
+
# multiple IO devices is demonstrated in the examples below.
|
133
|
+
#
|
134
|
+
# @example Logging to STDOUT
|
135
|
+
# logger = Logtail::Logger.new(STDOUT)
|
136
|
+
#
|
137
|
+
# @example Logging to the Logtail HTTP device
|
138
|
+
# http_device = Logtail::LogDevices::HTTP.new("my-logtail-api-key")
|
139
|
+
# logger = Logtail::Logger.new(http_device)
|
140
|
+
#
|
141
|
+
# @example Logging to a file (with rotation)
|
142
|
+
# file_device = Logger::LogDevice.new("path/to/file.log")
|
143
|
+
# logger = Logtail::Logger.new(file_device)
|
144
|
+
#
|
145
|
+
# @example Logging to a file and the Logtail HTTP device (multiple log devices)
|
146
|
+
# http_device = Logtail::LogDevices::HTTP.new("my-logtail-api-key")
|
147
|
+
# file_logger = ::Logger.new("path/to/file.log")
|
148
|
+
# logger = Logtail::Logger.new(http_device, file_logger)
|
149
|
+
def initialize(*io_devices_and_loggers)
|
150
|
+
if io_devices_and_loggers.size == 0
|
151
|
+
raise ArgumentError.new("At least one IO device or Logger must be provided when " +
|
152
|
+
"instantiating a Logtail::Logger. Ex: Logtail::Logger.new(STDOUT).")
|
153
|
+
end
|
154
|
+
|
155
|
+
@extra_loggers = io_devices_and_loggers[1..-1].collect do |obj|
|
156
|
+
if is_a_logger?(obj)
|
157
|
+
obj
|
158
|
+
else
|
159
|
+
self.class.new(obj)
|
160
|
+
end
|
161
|
+
end
|
162
|
+
|
163
|
+
io_device = io_devices_and_loggers[0]
|
164
|
+
|
165
|
+
super(io_device)
|
166
|
+
|
167
|
+
# Ensure we sync STDOUT to avoid buffering
|
168
|
+
if io_device.respond_to?(:"sync=")
|
169
|
+
io_device.sync = true
|
170
|
+
end
|
171
|
+
|
172
|
+
# Set the default formatter. The formatter cannot be set during
|
173
|
+
# initialization, and can be changed with #formatter=.
|
174
|
+
if io_device.is_a?(LogDevices::HTTP)
|
175
|
+
self.formatter = PassThroughFormatter.new
|
176
|
+
elsif Config.instance.development? || Config.instance.test?
|
177
|
+
self.formatter = MessageOnlyFormatter.new
|
178
|
+
else
|
179
|
+
self.formatter = JSONFormatter.new
|
180
|
+
end
|
181
|
+
|
182
|
+
self.level = environment_level
|
183
|
+
|
184
|
+
after_initialize if respond_to?(:after_initialize)
|
185
|
+
|
186
|
+
Logtail::Config.instance.debug { "Logtail::Logger instantiated, level: #{level}, formatter: #{formatter.class}" }
|
187
|
+
|
188
|
+
@initialized = true
|
189
|
+
end
|
190
|
+
|
191
|
+
# Sets a new formatted on the logger.
|
192
|
+
#
|
193
|
+
# @note The formatter cannot be changed if you are using the HTTP logger backend.
|
194
|
+
def formatter=(value)
|
195
|
+
if @initialized && @logdev && @logdev.dev.is_a?(Logtail::LogDevices::HTTP) && !value.is_a?(PassThroughFormatter)
|
196
|
+
raise ArgumentError.new("The formatter cannot be changed when using the " +
|
197
|
+
"Logtail::LogDevices::HTTP log device. The PassThroughFormatter must be used for proper " +
|
198
|
+
"delivery.")
|
199
|
+
end
|
200
|
+
|
201
|
+
super
|
202
|
+
end
|
203
|
+
|
204
|
+
def level=(value)
|
205
|
+
if value.is_a?(Symbol)
|
206
|
+
value = level_from_symbol(value)
|
207
|
+
end
|
208
|
+
super
|
209
|
+
end
|
210
|
+
|
211
|
+
# @private
|
212
|
+
def with_context(context, &block)
|
213
|
+
Logtail::CurrentContext.with(context, &block)
|
214
|
+
end
|
215
|
+
|
216
|
+
# Patch to ensure that the {#level} method is used instead of `@level`.
|
217
|
+
# This is required because of Rails' monkey patching on Logger via `::LoggerSilence`.
|
218
|
+
def add(severity, message = nil, progname = nil, &block)
|
219
|
+
return true if @logdev.nil? || (severity || UNKNOWN) < level
|
220
|
+
|
221
|
+
@extra_loggers.each do |logger|
|
222
|
+
logger.add(severity, message, progname, &block)
|
223
|
+
end
|
224
|
+
|
225
|
+
super
|
226
|
+
end
|
227
|
+
|
228
|
+
# Backwards compatibility with older ActiveSupport::Logger versions
|
229
|
+
Logger::Severity.constants.each do |severity|
|
230
|
+
class_eval(<<-EOT, __FILE__, __LINE__ + 1)
|
231
|
+
def #{severity.downcase}(*args, &block)
|
232
|
+
progname = args.first
|
233
|
+
options = args.last
|
234
|
+
|
235
|
+
if args.length == 2 and options.is_a?(Hash) && options.length > 0
|
236
|
+
progname = options.merge(message: progname)
|
237
|
+
end
|
238
|
+
|
239
|
+
add(#{severity}, nil, progname, &block)
|
240
|
+
end
|
241
|
+
|
242
|
+
def #{severity.downcase}? # def debug?
|
243
|
+
Logger::#{severity} >= level # DEBUG >= level
|
244
|
+
end # end
|
245
|
+
EOT
|
246
|
+
end
|
247
|
+
|
248
|
+
private
|
249
|
+
def environment_level
|
250
|
+
level = ([ENV['LOG_LEVEL'].to_s.upcase, "DEBUG"] & %w[DEBUG INFO WARN ERROR FATAL UNKNOWN]).compact.first
|
251
|
+
self.class.const_get(level)
|
252
|
+
end
|
253
|
+
|
254
|
+
def level_from_symbol(value)
|
255
|
+
case value
|
256
|
+
when :debug; DEBUG
|
257
|
+
when :info; INFO
|
258
|
+
when :warn; WARN
|
259
|
+
when :error; ERROR
|
260
|
+
when :fatal; FATAL
|
261
|
+
when :unknown; UNKNOWN
|
262
|
+
else; raise ArgumentError.new("level #{value.inspect} is not a valid logger level")
|
263
|
+
end
|
264
|
+
end
|
265
|
+
|
266
|
+
def is_a_logger?(obj)
|
267
|
+
obj.respond_to?(:debug) && obj.respond_to?(:info) && obj.respond_to?(:warn)
|
268
|
+
end
|
269
|
+
end
|
270
|
+
end
|