logtail-ruby 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. checksums.yaml +7 -0
  2. data/.github/workflows/main.yml +33 -0
  3. data/.gitignore +24 -0
  4. data/.rspec +2 -0
  5. data/CHANGELOG.md +12 -0
  6. data/Gemfile +10 -0
  7. data/LICENSE.md +15 -0
  8. data/README.md +4 -0
  9. data/Rakefile +72 -0
  10. data/lib/logtail.rb +36 -0
  11. data/lib/logtail/config.rb +154 -0
  12. data/lib/logtail/config/integrations.rb +17 -0
  13. data/lib/logtail/context.rb +9 -0
  14. data/lib/logtail/contexts.rb +12 -0
  15. data/lib/logtail/contexts/http.rb +31 -0
  16. data/lib/logtail/contexts/release.rb +52 -0
  17. data/lib/logtail/contexts/runtime.rb +23 -0
  18. data/lib/logtail/contexts/session.rb +24 -0
  19. data/lib/logtail/contexts/system.rb +29 -0
  20. data/lib/logtail/contexts/user.rb +28 -0
  21. data/lib/logtail/current_context.rb +168 -0
  22. data/lib/logtail/event.rb +36 -0
  23. data/lib/logtail/events.rb +10 -0
  24. data/lib/logtail/events/controller_call.rb +44 -0
  25. data/lib/logtail/events/error.rb +40 -0
  26. data/lib/logtail/events/exception.rb +10 -0
  27. data/lib/logtail/events/sql_query.rb +26 -0
  28. data/lib/logtail/events/template_render.rb +25 -0
  29. data/lib/logtail/integration.rb +40 -0
  30. data/lib/logtail/integrator.rb +50 -0
  31. data/lib/logtail/log_devices.rb +8 -0
  32. data/lib/logtail/log_devices/http.rb +368 -0
  33. data/lib/logtail/log_devices/http/flushable_dropping_sized_queue.rb +52 -0
  34. data/lib/logtail/log_devices/http/request_attempt.rb +20 -0
  35. data/lib/logtail/log_entry.rb +110 -0
  36. data/lib/logtail/logger.rb +270 -0
  37. data/lib/logtail/logtail.rb +36 -0
  38. data/lib/logtail/timer.rb +21 -0
  39. data/lib/logtail/util.rb +7 -0
  40. data/lib/logtail/util/non_nil_hash_builder.rb +40 -0
  41. data/lib/logtail/version.rb +3 -0
  42. data/logtail-ruby.gemspec +43 -0
  43. data/spec/README.md +13 -0
  44. data/spec/logtail/current_context_spec.rb +113 -0
  45. data/spec/logtail/events/controller_call_spec.rb +12 -0
  46. data/spec/logtail/events/error_spec.rb +15 -0
  47. data/spec/logtail/log_devices/http_spec.rb +185 -0
  48. data/spec/logtail/log_entry_spec.rb +22 -0
  49. data/spec/logtail/logger_spec.rb +227 -0
  50. data/spec/spec_helper.rb +22 -0
  51. data/spec/support/logtail.rb +5 -0
  52. data/spec/support/socket_hostname.rb +12 -0
  53. data/spec/support/timecop.rb +3 -0
  54. data/spec/support/webmock.rb +3 -0
  55. metadata +238 -0
@@ -0,0 +1,52 @@
1
+ module Logtail
2
+ module LogDevices
3
+ class HTTP
4
+ # A simple thread-safe queue implementation that provides a #flush method.
5
+ # The built-in ruby `Queue` class does not provide a #flush method that allows
6
+ # the caller to retrieve all items on the queue in one call. The Ruby `SizedQueue` also
7
+ # implements thread waiting, which is something we want to avoid. To keep things
8
+ # simple and straight-forward, we designed this queue class.
9
+ # @private
10
+ class FlushableDroppingSizedQueue
11
+ def initialize(max_size)
12
+ @lock = Mutex.new
13
+ @max_size = max_size
14
+ @array = []
15
+ end
16
+
17
+ # Adds a message to the queue
18
+ def enq(msg)
19
+ @lock.synchronize do
20
+ if !full?
21
+ @array << msg
22
+ end
23
+ end
24
+ end
25
+
26
+ # Removes a single item from the queue
27
+ def deq
28
+ @lock.synchronize do
29
+ @array.pop
30
+ end
31
+ end
32
+
33
+ # Flushes all message from the queue and returns them.
34
+ def flush
35
+ @lock.synchronize do
36
+ old = @array
37
+ @array = []
38
+ return old
39
+ end
40
+ end
41
+
42
+ def full?
43
+ size >= @max_size
44
+ end
45
+
46
+ def size
47
+ @array.size
48
+ end
49
+ end
50
+ end
51
+ end
52
+ end
@@ -0,0 +1,20 @@
1
+ module Logtail
2
+ module LogDevices
3
+ class HTTP
4
+ # Represents an attempt to deliver a request. Requests can be retried, hence
5
+ # why we keep track of the number of attempts.
6
+ class RequestAttempt
7
+ attr_reader :attempts, :request
8
+
9
+ def initialize(req)
10
+ @attempts = 0
11
+ @request = req
12
+ end
13
+
14
+ def attempted!
15
+ @attempts += 1
16
+ end
17
+ end
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,110 @@
1
+ require "socket"
2
+ require "time"
3
+
4
+ require "logtail/contexts"
5
+ require "logtail/events"
6
+
7
+ module Logtail
8
+ # Represents a new log entry into the log. This is an intermediary class between
9
+ # `Logger` and the log device that you set it up with.
10
+ class LogEntry #:nodoc:
11
+ BINARY_LIMIT_THRESHOLD = 1_000.freeze
12
+ DT_PRECISION = 6.freeze
13
+ MESSAGE_MAX_BYTES = 8192.freeze
14
+
15
+ attr_reader :context_snapshot, :event, :level, :message, :progname, :tags, :time
16
+
17
+ # Creates a log entry suitable to be sent to the Logtail API.
18
+ # @param level [Integer] the log level / severity
19
+ # @param time [Time] the exact time the log message was written
20
+ # @param progname [String] the progname scope for the log message
21
+ # @param message [String] Human readable log message.
22
+ # @param context_snapshot [Hash] structured data representing a snapshot of the context at
23
+ # the given point in time.
24
+ # @param event [Logtail.Event] structured data representing the log line event. This should be
25
+ # an instance of {Logtail.Event}.
26
+ # @return [LogEntry] the resulting LogEntry object
27
+ def initialize(level, time, progname, message, context_snapshot, event, options = {})
28
+ @level = level
29
+ @time = time.utc
30
+ @progname = progname
31
+
32
+ # If the message is not a string we call inspect to ensure it is a string.
33
+ # This follows the default behavior set by ::Logger
34
+ # See: https://github.com/ruby/ruby/blob/trunk/lib/logger.rb#L615
35
+ @message = message.is_a?(String) ? message : message.inspect
36
+ @message = @message.byteslice(0, MESSAGE_MAX_BYTES)
37
+ @tags = options[:tags]
38
+ @context_snapshot = context_snapshot
39
+ @event = event
40
+ end
41
+
42
+ # Builds a hash representation containing simple objects, suitable for serialization (JSON).
43
+ def to_hash(options = {})
44
+ options ||= {}
45
+ hash = {
46
+ :level => level,
47
+ :dt => formatted_dt,
48
+ :message => message
49
+ }
50
+
51
+ if !tags.nil? && tags.length > 0
52
+ hash[:tags] = tags
53
+ end
54
+
55
+ if !event.nil?
56
+ hash.merge!(event)
57
+ end
58
+
59
+ if !context_snapshot.nil? && context_snapshot.length > 0
60
+ hash[:context] = context_snapshot
61
+ end
62
+
63
+ if options[:only]
64
+ hash.select do |key, _value|
65
+ options[:only].include?(key)
66
+ end
67
+ elsif options[:except]
68
+ hash.select do |key, _value|
69
+ !options[:except].include?(key)
70
+ end
71
+ else
72
+ hash
73
+ end
74
+ end
75
+
76
+ def inspect
77
+ to_s
78
+ end
79
+
80
+ def to_json(options = {})
81
+ to_hash.to_json
82
+ end
83
+
84
+ def to_msgpack(*args)
85
+ to_hash.to_msgpack(*args)
86
+ end
87
+
88
+ # This is used when LogEntry objects make it to a non-Logtail logger.
89
+ def to_s
90
+ message + "\n"
91
+ end
92
+
93
+ private
94
+ def formatted_dt
95
+ @formatted_dt ||= time.iso8601(DT_PRECISION)
96
+ end
97
+
98
+ # Attempts to encode a non UTF-8 string into UTF-8, discarding invalid characters.
99
+ # If it fails, a nil is returned.
100
+ def encode_string(string)
101
+ string.encode('UTF-8', {
102
+ :invalid => :replace,
103
+ :undef => :replace,
104
+ :replace => '?'
105
+ })
106
+ rescue Exception
107
+ nil
108
+ end
109
+ end
110
+ end
@@ -0,0 +1,270 @@
1
+ require "logger"
2
+ require "msgpack"
3
+
4
+ require "logtail/config"
5
+ require "logtail/current_context"
6
+ require "logtail/log_devices"
7
+ require "logtail/log_entry"
8
+
9
+ module Logtail
10
+ # The Logtail Logger behaves exactly like the standard Ruby `::Logger`, except that it supports a
11
+ # transparent API for logging structured data and events.
12
+ #
13
+ # @example Basic logging
14
+ # logger.info "Payment rejected for customer #{customer_id}"
15
+ #
16
+ # @example Logging an event
17
+ # logger.info "Payment rejected", payment_rejected: {customer_id: customer_id, amount: 100}
18
+ class Logger < ::Logger
19
+
20
+ # @private
21
+ class Formatter
22
+ # Formatters get the formatted level from the logger.
23
+ SEVERITY_MAP = {
24
+ "DEBUG" => :debug,
25
+ "INFO" => :info,
26
+ "WARN" => :warn,
27
+ "ERROR" => :error,
28
+ "FATAL" => :fatal,
29
+ "UNKNOWN" => :unknown
30
+ }
31
+ EMPTY_ARRAY = []
32
+
33
+ private
34
+ def build_log_entry(severity, time, progname, logged_obj)
35
+ context_snapshot = CurrentContext.instance.snapshot
36
+ level = SEVERITY_MAP.fetch(severity)
37
+ tags = extract_active_support_tagged_logging_tags
38
+
39
+ if logged_obj.is_a?(Event)
40
+ LogEntry.new(level, time, progname, logged_obj.message, context_snapshot, logged_obj,
41
+ tags: tags)
42
+ elsif logged_obj.is_a?(Hash)
43
+ # Extract the tags
44
+ tags = tags.clone
45
+ tags.push(logged_obj.delete(:tag)) if logged_obj.key?(:tag)
46
+ tags.concat(logged_obj.delete(:tags)) if logged_obj.key?(:tags)
47
+ tags.uniq!
48
+
49
+ message = logged_obj.delete(:message)
50
+
51
+ LogEntry.new(level, time, progname, message, context_snapshot, logged_obj, tags: tags)
52
+ else
53
+ LogEntry.new(level, time, progname, logged_obj, context_snapshot, nil, tags: tags)
54
+ end
55
+ end
56
+
57
+ # Because of all the crazy ways Rails has attempted tags, we need this crazy method.
58
+ def extract_active_support_tagged_logging_tags
59
+ Thread.current[:activesupport_tagged_logging_tags] ||
60
+ Thread.current[tagged_logging_object_key_name] ||
61
+ EMPTY_ARRAY
62
+ end
63
+
64
+ def tagged_logging_object_key_name
65
+ @tagged_logging_object_key_name ||= "activesupport_tagged_logging_tags:#{object_id}"
66
+ end
67
+ end
68
+
69
+ # For use in development and test environments where you do not want metadata
70
+ # included in the log lines.
71
+ class MessageOnlyFormatter < Formatter
72
+ # This method is invoked when a log event occurs
73
+ def call(severity, timestamp, progname, msg)
74
+ log_entry = build_log_entry(severity, timestamp, progname, msg)
75
+ log_entry.to_s
76
+ end
77
+ end
78
+
79
+ # Structures your log messages as strings and appends metadata if
80
+ # `Logtail::Config.instance.append_metadata?` is true.
81
+ #
82
+ # Example message with metdata:
83
+ #
84
+ # My log message @metadata {"level":"info","dt":"2016-09-01T07:00:00.000000-05:00"}
85
+ #
86
+ class AugmentedFormatter < Formatter
87
+ METADATA_CALLOUT = " @metadata ".freeze
88
+ NEW_LINE = "\n".freeze
89
+ ESCAPED_NEW_LINE = "\\n".freeze
90
+
91
+ def call(severity, time, progname, msg)
92
+ log_entry = build_log_entry(severity, time, progname, msg)
93
+ metadata = log_entry.to_json(:except => [:message])
94
+ # use << for concatenation for performance reasons
95
+ log_entry.message.gsub(NEW_LINE, ESCAPED_NEW_LINE) << METADATA_CALLOUT <<
96
+ metadata << NEW_LINE
97
+ end
98
+ end
99
+
100
+ # Structures your log messages into JSON.
101
+ #
102
+ # logger = Logtail::Logger.new(STDOUT)
103
+ # logger.formatter = Logtail::JSONFormatter.new
104
+ #
105
+ # Example message:
106
+ #
107
+ # {"level":"info","dt":"2016-09-01T07:00:00.000000-05:00","message":"My log message"}
108
+ #
109
+ class JSONFormatter < Formatter
110
+ def call(severity, time, progname, msg)
111
+ # use << for concatenation for performance reasons
112
+ build_log_entry(severity, time, progname, msg).to_json << "\n"
113
+ end
114
+ end
115
+
116
+ # Passes through the LogEntry object. This is specifically used for the {Logtail::LogDevices::HTTP}
117
+ # class. This allows the IO device to format it however it wants. This is necessary for
118
+ # MessagePack because it requires a fixed array size before encoding. And since HTTP is
119
+ # sending data in batches, the encoding should happen there.
120
+ class PassThroughFormatter < Formatter
121
+ def call(severity, time, progname, msg)
122
+ build_log_entry(severity, time, progname, msg)
123
+ end
124
+ end
125
+
126
+ # Creates a new Logtail::Logger instance where the passed argument is an IO device. That is,
127
+ # anything that responds to `#write` and `#close`.
128
+ #
129
+ # Note, this method does *not* accept the same arguments as the standard Ruby `::Logger`.
130
+ # The Ruby `::Logger` accepts additional options controlling file rotation if the first argument
131
+ # is a file *name*. This is a design flaw that Logtail does not assume. Logging to a file, or
132
+ # multiple IO devices is demonstrated in the examples below.
133
+ #
134
+ # @example Logging to STDOUT
135
+ # logger = Logtail::Logger.new(STDOUT)
136
+ #
137
+ # @example Logging to the Logtail HTTP device
138
+ # http_device = Logtail::LogDevices::HTTP.new("my-logtail-api-key")
139
+ # logger = Logtail::Logger.new(http_device)
140
+ #
141
+ # @example Logging to a file (with rotation)
142
+ # file_device = Logger::LogDevice.new("path/to/file.log")
143
+ # logger = Logtail::Logger.new(file_device)
144
+ #
145
+ # @example Logging to a file and the Logtail HTTP device (multiple log devices)
146
+ # http_device = Logtail::LogDevices::HTTP.new("my-logtail-api-key")
147
+ # file_logger = ::Logger.new("path/to/file.log")
148
+ # logger = Logtail::Logger.new(http_device, file_logger)
149
+ def initialize(*io_devices_and_loggers)
150
+ if io_devices_and_loggers.size == 0
151
+ raise ArgumentError.new("At least one IO device or Logger must be provided when " +
152
+ "instantiating a Logtail::Logger. Ex: Logtail::Logger.new(STDOUT).")
153
+ end
154
+
155
+ @extra_loggers = io_devices_and_loggers[1..-1].collect do |obj|
156
+ if is_a_logger?(obj)
157
+ obj
158
+ else
159
+ self.class.new(obj)
160
+ end
161
+ end
162
+
163
+ io_device = io_devices_and_loggers[0]
164
+
165
+ super(io_device)
166
+
167
+ # Ensure we sync STDOUT to avoid buffering
168
+ if io_device.respond_to?(:"sync=")
169
+ io_device.sync = true
170
+ end
171
+
172
+ # Set the default formatter. The formatter cannot be set during
173
+ # initialization, and can be changed with #formatter=.
174
+ if io_device.is_a?(LogDevices::HTTP)
175
+ self.formatter = PassThroughFormatter.new
176
+ elsif Config.instance.development? || Config.instance.test?
177
+ self.formatter = MessageOnlyFormatter.new
178
+ else
179
+ self.formatter = JSONFormatter.new
180
+ end
181
+
182
+ self.level = environment_level
183
+
184
+ after_initialize if respond_to?(:after_initialize)
185
+
186
+ Logtail::Config.instance.debug { "Logtail::Logger instantiated, level: #{level}, formatter: #{formatter.class}" }
187
+
188
+ @initialized = true
189
+ end
190
+
191
+ # Sets a new formatted on the logger.
192
+ #
193
+ # @note The formatter cannot be changed if you are using the HTTP logger backend.
194
+ def formatter=(value)
195
+ if @initialized && @logdev && @logdev.dev.is_a?(Logtail::LogDevices::HTTP) && !value.is_a?(PassThroughFormatter)
196
+ raise ArgumentError.new("The formatter cannot be changed when using the " +
197
+ "Logtail::LogDevices::HTTP log device. The PassThroughFormatter must be used for proper " +
198
+ "delivery.")
199
+ end
200
+
201
+ super
202
+ end
203
+
204
+ def level=(value)
205
+ if value.is_a?(Symbol)
206
+ value = level_from_symbol(value)
207
+ end
208
+ super
209
+ end
210
+
211
+ # @private
212
+ def with_context(context, &block)
213
+ Logtail::CurrentContext.with(context, &block)
214
+ end
215
+
216
+ # Patch to ensure that the {#level} method is used instead of `@level`.
217
+ # This is required because of Rails' monkey patching on Logger via `::LoggerSilence`.
218
+ def add(severity, message = nil, progname = nil, &block)
219
+ return true if @logdev.nil? || (severity || UNKNOWN) < level
220
+
221
+ @extra_loggers.each do |logger|
222
+ logger.add(severity, message, progname, &block)
223
+ end
224
+
225
+ super
226
+ end
227
+
228
+ # Backwards compatibility with older ActiveSupport::Logger versions
229
+ Logger::Severity.constants.each do |severity|
230
+ class_eval(<<-EOT, __FILE__, __LINE__ + 1)
231
+ def #{severity.downcase}(*args, &block)
232
+ progname = args.first
233
+ options = args.last
234
+
235
+ if args.length == 2 and options.is_a?(Hash) && options.length > 0
236
+ progname = options.merge(message: progname)
237
+ end
238
+
239
+ add(#{severity}, nil, progname, &block)
240
+ end
241
+
242
+ def #{severity.downcase}? # def debug?
243
+ Logger::#{severity} >= level # DEBUG >= level
244
+ end # end
245
+ EOT
246
+ end
247
+
248
+ private
249
+ def environment_level
250
+ level = ([ENV['LOG_LEVEL'].to_s.upcase, "DEBUG"] & %w[DEBUG INFO WARN ERROR FATAL UNKNOWN]).compact.first
251
+ self.class.const_get(level)
252
+ end
253
+
254
+ def level_from_symbol(value)
255
+ case value
256
+ when :debug; DEBUG
257
+ when :info; INFO
258
+ when :warn; WARN
259
+ when :error; ERROR
260
+ when :fatal; FATAL
261
+ when :unknown; UNKNOWN
262
+ else; raise ArgumentError.new("level #{value.inspect} is not a valid logger level")
263
+ end
264
+ end
265
+
266
+ def is_a_logger?(obj)
267
+ obj.respond_to?(:debug) && obj.respond_to?(:info) && obj.respond_to?(:warn)
268
+ end
269
+ end
270
+ end