semantic_logger 5.2.0 → 5.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. checksums.yaml +4 -4
  2. data/docs/api.md +160 -9
  3. data/docs/appenders.md +461 -34
  4. data/docs/config.md +543 -36
  5. data/docs/log.md +38 -7
  6. data/docs/metrics.md +139 -28
  7. data/docs/operations.md +293 -24
  8. data/docs/rails.md +319 -5
  9. data/docs/security.md +135 -107
  10. data/docs/testing.md +23 -0
  11. data/docs/upgrading.md +644 -1
  12. data/lib/semantic_logger/appender/async.rb +29 -11
  13. data/lib/semantic_logger/appender/bugsnag.rb +7 -2
  14. data/lib/semantic_logger/appender/cloudwatch_logs.rb +46 -6
  15. data/lib/semantic_logger/appender/elasticsearch_base.rb +197 -29
  16. data/lib/semantic_logger/appender/elasticsearch_http.rb +22 -5
  17. data/lib/semantic_logger/appender/file.rb +29 -14
  18. data/lib/semantic_logger/appender/graylog.rb +81 -7
  19. data/lib/semantic_logger/appender/honeybadger.rb +7 -1
  20. data/lib/semantic_logger/appender/honeybadger_insights.rb +8 -2
  21. data/lib/semantic_logger/appender/http.rb +244 -41
  22. data/lib/semantic_logger/appender/io.rb +11 -6
  23. data/lib/semantic_logger/appender/kafka.rb +192 -46
  24. data/lib/semantic_logger/appender/loki.rb +42 -6
  25. data/lib/semantic_logger/appender/mongodb.rb +94 -9
  26. data/lib/semantic_logger/appender/new_relic.rb +2 -3
  27. data/lib/semantic_logger/appender/new_relic_logs.rb +30 -8
  28. data/lib/semantic_logger/appender/open_telemetry.rb +37 -35
  29. data/lib/semantic_logger/appender/rabbitmq.rb +241 -17
  30. data/lib/semantic_logger/appender/sentry.rb +2 -3
  31. data/lib/semantic_logger/appender/sentry_ruby.rb +82 -15
  32. data/lib/semantic_logger/appender/splunk.rb +4 -4
  33. data/lib/semantic_logger/appender/splunk_http.rb +92 -1
  34. data/lib/semantic_logger/appender/syslog.rb +89 -10
  35. data/lib/semantic_logger/appender/tcp.rb +68 -98
  36. data/lib/semantic_logger/appender/text_output.rb +65 -0
  37. data/lib/semantic_logger/appender/udp.rb +130 -18
  38. data/lib/semantic_logger/appender/wrapper.rb +10 -7
  39. data/lib/semantic_logger/appender.rb +115 -10
  40. data/lib/semantic_logger/appenders.rb +136 -4
  41. data/lib/semantic_logger/backtrace_cleaning.rb +77 -0
  42. data/lib/semantic_logger/base.rb +53 -71
  43. data/lib/semantic_logger/call_site.rb +235 -0
  44. data/lib/semantic_logger/context.rb +222 -0
  45. data/lib/semantic_logger/core_ext/process.rb +20 -5
  46. data/lib/semantic_logger/core_ext/warning.rb +70 -0
  47. data/lib/semantic_logger/deprecation.rb +139 -0
  48. data/lib/semantic_logger/exception_details.rb +4 -3
  49. data/lib/semantic_logger/formatters/base.rb +139 -19
  50. data/lib/semantic_logger/formatters/color.rb +3 -4
  51. data/lib/semantic_logger/formatters/default.rb +35 -7
  52. data/lib/semantic_logger/formatters/ecs.rb +9 -4
  53. data/lib/semantic_logger/formatters/json.rb +9 -0
  54. data/lib/semantic_logger/formatters/logfmt.rb +9 -7
  55. data/lib/semantic_logger/formatters/loki/labels.rb +282 -0
  56. data/lib/semantic_logger/formatters/loki.rb +412 -24
  57. data/lib/semantic_logger/formatters/new_relic_logs.rb +15 -13
  58. data/lib/semantic_logger/formatters/one_line.rb +44 -10
  59. data/lib/semantic_logger/formatters/open_telemetry.rb +186 -29
  60. data/lib/semantic_logger/formatters/pattern.rb +60 -8
  61. data/lib/semantic_logger/formatters/raw.rb +38 -8
  62. data/lib/semantic_logger/formatters/signalfx.rb +55 -15
  63. data/lib/semantic_logger/formatters/syslog.rb +4 -2
  64. data/lib/semantic_logger/formatters/syslog_cee.rb +10 -4
  65. data/lib/semantic_logger/formatters/syslog_header.rb +49 -0
  66. data/lib/semantic_logger/formatters.rb +1 -0
  67. data/lib/semantic_logger/log.rb +236 -80
  68. data/lib/semantic_logger/loggable.rb +55 -95
  69. data/lib/semantic_logger/logger.rb +31 -5
  70. data/lib/semantic_logger/metric/new_relic.rb +32 -17
  71. data/lib/semantic_logger/metric/signalfx.rb +21 -12
  72. data/lib/semantic_logger/metric/statsd.rb +26 -8
  73. data/lib/semantic_logger/processor.rb +194 -5
  74. data/lib/semantic_logger/queue_processor/backoff.rb +140 -0
  75. data/lib/semantic_logger/queue_processor/batch_failure.rb +69 -0
  76. data/lib/semantic_logger/queue_processor/held_write.rb +62 -0
  77. data/lib/semantic_logger/queue_processor/inbox.rb +97 -0
  78. data/lib/semantic_logger/queue_processor/reopener.rb +56 -0
  79. data/lib/semantic_logger/queue_processor/reporter.rb +113 -0
  80. data/lib/semantic_logger/queue_processor/request.rb +48 -10
  81. data/lib/semantic_logger/queue_processor/throttled_counter.rb +60 -17
  82. data/lib/semantic_logger/queue_processor.rb +426 -176
  83. data/lib/semantic_logger/ractor_copy.rb +120 -0
  84. data/lib/semantic_logger/ractor_processor.rb +82 -0
  85. data/lib/semantic_logger/ractor_receiver.rb +282 -0
  86. data/lib/semantic_logger/ractor_support.rb +28 -0
  87. data/lib/semantic_logger/redactor.rb +310 -0
  88. data/lib/semantic_logger/semantic_logger.rb +313 -79
  89. data/lib/semantic_logger/subscriber.rb +169 -15
  90. data/lib/semantic_logger/sync_processor.rb +226 -26
  91. data/lib/semantic_logger/test/capture_log_events.rb +20 -2
  92. data/lib/semantic_logger/test/minitest.rb +2 -1
  93. data/lib/semantic_logger/test/rspec.rb +1 -1
  94. data/lib/semantic_logger/trap_safe.rb +113 -0
  95. data/lib/semantic_logger/utils.rb +266 -79
  96. data/lib/semantic_logger/version.rb +1 -1
  97. data/lib/semantic_logger.rb +35 -2
  98. metadata +22 -2
data/docs/operations.md CHANGED
@@ -30,12 +30,55 @@ thread, so unless those are re-opened in the child, logging quietly stops.
30
30
  ### It is automatic (default)
31
31
 
32
32
  As of v5 you do not need to do anything. Semantic Logger installs a `Process._fork` hook (Ruby 3.1
33
- and later) that calls `SemanticLogger.reopen` in the child after `fork`, `Process.daemon`,
34
- `IO.popen`, `Kernel#system`, and backticks. That covers every forking framework (Puma, Unicorn,
35
- Resque, Spring, Phusion Passenger, parallel tests, and so on).
33
+ and later) that calls `SemanticLogger.reopen` in the child after `fork`, `Process.fork`,
34
+ `Process.daemon`, and `IO.popen("-")`, the calls that fork the Ruby process. That covers every
35
+ forking framework (Puma, Unicorn, Resque, Spring, Phusion Passenger, parallel tests, and so on).
36
+ `Kernel#system`, backticks, `spawn`, and `IO.popen` with a command start another program instead,
37
+ which has nothing to reopen. The process can also fork in a signal trap.
36
38
 
37
39
  `reopen` runs only once per process after a fork, so it is safe even if something else also calls it.
38
40
 
41
+ Each appender is reopened by the thread that writes its entries, before it writes the next one, so
42
+ that an appender is never reopened while it is writing (see [Reopen manually](#reopen-manually)).
43
+ After a fork, that is the child's own thread, which reopens the appender before it writes anything.
44
+
45
+ In the child, the appenders that keep a connection open to their server leave the one they inherited
46
+ to the parent, without closing it or writing to it, and connect again: `:http` and the appenders
47
+ built on it (`:elasticsearch_http`, `:splunk_http`, `:loki`, and `metric: :signalfx`), `:tcp`,
48
+ `:syslog` and `:graylog` over TCP, `:kafka`, and `:rabbitmq`. Closing a TLS connection sends the
49
+ server a `close_notify`, which ends the connection for the parent too, and over TLS the server
50
+ cannot read what the parent writes after the child has written to it. When one of them cannot
51
+ connect in the child, such as while its server cannot be reached, it connects before it writes the
52
+ next entry. They also leave the connection they inherited to the parent when they are closed in the
53
+ child before they have reopened, such as with `SemanticLogger.reopen_on_fork = false`.
54
+ `SemanticLogger.remove_appender` in the child first waits, when the thread that writes the entries
55
+ has not yet reopened the appenders, until it has, so that the appender it removes closes only what
56
+ the child built. That matters for an appender of your own whose `#reopen` leaves the connection it
57
+ inherited to the parent, but whose `#close` closes whatever connection it holds.
58
+ (`:kafka` connects only when it delivers; when it cannot build a new producer in the child, such as
59
+ for invalid options, logging to it raises `IOError` until it is reopened.)
60
+
61
+ `:cloudwatch_logs` and `:mongodb` cannot do the same, since their gems keep their connections, and
62
+ still act on the parent's in the child. The AWS SDK keeps one pool of connections for all its
63
+ clients with the same options, so in the child it can reuse a connection the parent left idle in the
64
+ pool, and before each request it finishes those idle for longer than its `http_idle_timeout` (5
65
+ seconds by default), including the parent's. The MongoDB driver closes the parent's connections in the child, when the
66
+ child uses its client, and when the child collects the client's connection pool, or exits. Over TLS,
67
+ finishing or closing a connection sends the server a `close_notify`, which ends the connection for
68
+ the parent too. Closed in the child before it has reopened, `:cloudwatch_logs` leaves the buffered
69
+ entries it inherited to the parent, which sends them. Closed in the child, even once it has reopened
70
+ there, `:mongodb` forgets the client it inherited without closing it, since closing it would end
71
+ the parent's sessions on the server; reopening it then connects a new client.
72
+
73
+ An appender that cannot be reopened, such as `:udp` once its server's name no longer resolves, or
74
+ `:http` while its server cannot be reached, is reported to the internal logger, as
75
+ `Failed to re-open appender`, or `Async: Failed to reopen the appender` for one added with
76
+ `async: true` or in batches, and carries on with what it had, apart from a connection inherited from
77
+ the parent (see above). The other appenders are still reopened, and the thread that writes the
78
+ entries of an appender added with `async: true` or in batches is still started in the child, so its
79
+ entries are written, or discarded and counted like any other entry it cannot write (see
80
+ [Retry a failing appender](#retry-a-failing-appender)), rather than left on its queue.
81
+
39
82
  ### Reopen manually
40
83
 
41
84
  You only need the steps below if you turned the automatic hook off, or you rotated logs in a way that
@@ -55,9 +98,79 @@ did not fork (see [Log rotation](#log-rotation)).
55
98
  SemanticLogger.reopen(force: true) # same process, e.g. after external log rotation
56
99
  ~~~
57
100
 
101
+ Within the same process, `reopen` keeps the log entries that are still queued, and the thread
102
+ that writes them keeps running. That thread is only started again if it has stopped, such as
103
+ after `async_max_retries`. After a fork, the child starts with an empty queue, since the parent
104
+ process writes the entries it had queued.
105
+
106
+ Within the same process, the appenders close the file or the connection that a reopen replaces.
107
+ `:syslog` and `:graylog` over TCP, `:udp`, and `:rabbitmq` connect again before they close the
108
+ connection they had, and keep it when they cannot connect. `:cloudwatch_logs` shuts down the
109
+ timer that sends its buffered entries, and keeps those entries for the new timer.
110
+
111
+ `reopen` only asks for the appenders to be reopened: each one is reopened by the thread that
112
+ writes its entries, before it writes the next one, so that a file is never closed, nor a
113
+ connection ended, while an entry is being written to it. So `reopen` returns before the appenders
114
+ have reopened; a `SemanticLogger.flush` after it returns once they have. An appender added in
115
+ batches reopens before its next batch, which it writes at most `batch_seconds` later, or when it
116
+ is flushed. An entry still queued when `reopen` is called can be written after the reopen, such
117
+ as to the new log file. That also lets `reopen` be called from a signal trap, such as one that a
118
+ log rotation tool signals, since no appender connects or locks anything there. In synchronous mode
119
+ (see [Synchronous operation](#synchronous-operation)), `reopen` reopens on the calling thread each
120
+ appender that has no thread of its own, or, in a signal trap, on a thread that the trap starts
121
+ and waits for, at most one second; an appender added with `async: true` or in batches is still
122
+ reopened by its own thread.
123
+
58
124
  You might opt out if another library forks in ways you do not want to trigger a reopen, or you need
59
125
  to control exactly when the appender thread restarts.
60
126
 
127
+ ## The json gem's warning about binary text
128
+
129
+ json 2.8.0 through 3.0.x print this warning each time they serialize a binary String that holds
130
+ UTF-8, such as a response body or Base64-decoded text:
131
+
132
+ ```
133
+ warning: JSON.generate: UTF-8 string passed as BINARY, this will raise an encoding error in json 3.0
134
+ ```
135
+
136
+ Semantic Logger writes such text as UTF-8 with every json version, so with those versions it
137
+ prepends a module to Ruby's `Warning` when it is loaded. The module drops this one warning when it
138
+ comes from Semantic Logger serializing log data. Every other warning is passed on, and so is this
139
+ one from the application's own JSON, including JSON that an exception's `#message` builds. A
140
+ module that the application prepends to `Warning` after loading Semantic Logger, such as a test
141
+ helper that raises on warnings, sees the warning first. The module is not installed with json 3.1
142
+ and later, which raise on such a String instead (Semantic Logger then writes a converted copy),
143
+ or on JRuby.
144
+
145
+ ## Payload text where no locale is set
146
+
147
+ Where no UTF-8 locale is set, as in cron jobs, many container images, and systems with no locale
148
+ configured (`LC_ALL`, `LC_CTYPE`, and `LANG` unset, or set to `C` or `POSIX`), CRuby's default
149
+ external encoding is US-ASCII. The text formatters (`:default`, `:color`, `:one_line`, `:syslog`,
150
+ and `%{payload}` in `:pattern`) write the payload with Ruby's `#inspect`, which then writes each
151
+ character outside ASCII as an escape:
152
+
153
+ ```
154
+ 2026-11-16 09:10:53.874476 I [86699:1312] Checkout -- José -- {name: "Jos\u00E9"}
155
+ ```
156
+
157
+ The message, tags, and named tags whose values are Strings are written as they are. A named tag
158
+ whose value is an Array or a Hash is written with `#inspect`, and so is escaped as the payload is.
159
+ The JSON formatters are not affected, and neither are Rails applications, since Rails sets the
160
+ default external encoding to UTF-8 (`config.encoding`), nor JRuby, whose default is UTF-8.
161
+
162
+ To write these characters, run Ruby with a UTF-8 locale or default external encoding, for example:
163
+
164
+ ```shell
165
+ export LC_ALL=C.UTF-8
166
+ # or
167
+ export RUBYOPT=-Eutf-8
168
+ ```
169
+
170
+ or set `Encoding.default_external = Encoding::UTF_8` when the application starts. `LC_ALL`, then
171
+ `LC_CTYPE`, take precedence over `LANG`, so setting `LANG=C.UTF-8` is not enough when either is set
172
+ to `C` or `POSIX`.
173
+
61
174
  ## Log rotation
62
175
 
63
176
  For performance the log file is **not** re-opened on every write, so a log file must be rotated with
@@ -151,9 +264,15 @@ the internal logger at most once every 30 seconds, and the count is available fr
151
264
  `SemanticLogger.stats[:dropped]`. It is not supported in synchronous mode (`SemanticLogger.sync!`),
152
265
  which has no queue.
153
266
 
267
+ When the queue is full, a message logged in a signal trap, such as by the handlers of
268
+ [`add_signal_handler`](#linux-signals), is dropped, counted, and reported the same way, with the
269
+ report made just after the trap rather than in it.
270
+
154
271
  Either setting keeps logging calls from blocking; neither makes `SemanticLogger.flush` return sooner.
155
272
  A flush, including the one that runs automatically when the process exits, still waits for every
156
- message already on the queue to be written.
273
+ message already on the queue to be written, unless the destination of an appender added with
274
+ `async: true` or in batches fails meanwhile, or, at exit, that of any other appender (see
275
+ [Retry a failing appender](#retry-a-failing-appender)).
157
276
 
158
277
  `non_blocking` applies only to a capped queue. An uncapped queue (`max_queue_size: -1`) never blocks
159
278
  and never drops, but can grow without bound.
@@ -163,28 +282,55 @@ and never drops, but can grow without bound.
163
282
  If an appender raises while the worker thread is writing, the thread logs the error and restarts, so
164
283
  a transient failure (such as a brief network blip to a remote appender) does not permanently stop
165
284
  logging. Each restart sleeps with an increasing back-off (1 second, then 2, ...), reset as soon as a
166
- message is processed successfully.
167
-
168
- A message that fails because of its own content, rather than the destination, is not retried: for
169
- example a value the formatter cannot serialize (a `JSON::GeneratorError`, `JSON::NestingError`,
170
- encoding error, or `FloatDomainError`). That one message is discarded, and the thread carries on
171
- without backing off. When batching with an appender that sends the whole batch in a single request,
172
- such as the HTTP, Loki, Splunk HTTP, Elasticsearch, and OpenSearch appenders, only that message is
173
- discarded and the rest of the batch is still written. With an appender that may have written part
285
+ message is processed successfully. The messages logged meanwhile wait on the queue.
286
+
287
+ When the appender says that the write that failed wrote none of the message (or, when batching, of
288
+ the batch), and is worth trying again, the thread writes the same message or batch again after the
289
+ back-off, ahead of the messages on the queue. Once the destination has answered it with a failure
290
+ after it was written again 3 times, it is discarded, and the thread carries on with the next one
291
+ once it has backed off, so that a message the destination fails every time does not hold up the
292
+ others until the thread gives up. A write that never reached the destination, such as one that
293
+ could not connect, is not counted, so an outage loses no messages until the thread gives up. The
294
+ HTTP based appenders say so for a `408`, `429`, or `503` response, and for a request they could not
295
+ send because they could not connect to the server; see
296
+ [When the server does not accept a request](appenders.html#when-the-server-does-not-accept-a-request).
297
+ Otherwise the message or batch is discarded straight away, since writing it again could write some
298
+ of it twice, or would fail the same way, and the thread carries on with the next one once it has
299
+ backed off.
300
+
301
+ A message that fails because of its own content, rather than the destination, is discarded and the
302
+ thread carries on straight away, without backing off: for example a value the formatter cannot
303
+ serialize (a `JSON::GeneratorError`, `JSON::NestingError`, encoding error, or `FloatDomainError`).
304
+ When batching with an appender that sends the whole batch in a single request, such as the HTTP,
305
+ Loki, Splunk HTTP, Elasticsearch, and OpenSearch appenders, only that message is discarded and the
306
+ rest of the batch is still written. With an appender that may have written part
174
307
  of the batch before failing, the whole batch is discarded, so that no message is written twice.
175
308
 
176
- The first discarded message is reported to the internal logger straight away, with its error. After
177
- that, the number discarded is reported, with the latest error, at most once every
178
- `dropped_message_report_seconds` (default `30`). The total appears as `failed` for that appender in
179
- `SemanticLogger.stats[:appenders]`.
309
+ A message that fails with an error that writing it again cannot fix, such as a request the
310
+ destination refuses for its credentials or its url, is discarded straight away too, without backing
311
+ off, as one that fails because of its content. When the appender says that such an error would fail
312
+ any batch, whatever it holds, as the HTTP based appenders do for a `401`, `403`, or `404`, the whole
313
+ batch is discarded at once, rather than split up to find the messages at fault.
314
+
315
+ Whether its content or the destination was at fault, the first discarded message is reported to the
316
+ internal logger straight away, with its error. After that, the number discarded is reported, with the
317
+ latest error, at most once every `dropped_message_report_seconds` (default `30`). The total appears
318
+ as `failed` for that appender in `SemanticLogger.stats[:appenders]`.
180
319
 
181
320
  A `JSON::ParserError`, such as from a destination that answered with an HTML error page, is a
182
- problem with the destination, so it is retried like any other error. The appender decides which
183
- errors are caused by message content; see [Custom appenders](config.html#custom-appenders).
321
+ problem with the destination, so the thread backs off as for any other error. The appender decides
322
+ which errors are caused by message content; see [Custom appenders](config.html#custom-appenders).
323
+ The HTTP based appenders decide by the status of the response; see
324
+ [When the server does not accept a request](appenders.html#when-the-server-does-not-accept-a-request).
184
325
 
185
326
  After `async_max_retries` (default `100`) consecutive failed restarts the worker thread gives up,
186
- rather than spinning forever on a persistent failure. From then on new log messages for that
187
- appender are dropped once its queue is full, instead of blocking the application:
327
+ rather than spinning forever on a persistent failure, and discards the message or batch it was
328
+ writing again. From then on new log messages for that
329
+ appender are dropped once its queue is full, instead of blocking the application. So is a message
330
+ already waiting for room on the full queue when the thread gives up, within a second, which lets the
331
+ thread that logged it carry on: normally the thread that writes to every appender. The messages
332
+ already on the queue are kept, and the dropped ones are counted as `dropped` for that appender in
333
+ `SemanticLogger.stats[:appenders]`:
188
334
 
189
335
  ~~~ruby
190
336
  SemanticLogger.add_appender(
@@ -196,7 +342,53 @@ SemanticLogger.add_appender(
196
342
  ~~~
197
343
 
198
344
  Set `async_max_retries: -1` to retry indefinitely. The back-off still applies and still resets after
199
- a successful message.
345
+ a successful message. `SemanticLogger.reopen(force: true)` starts a worker thread that gave up again,
346
+ which writes the messages kept on the queue. It carries on with the count of failed retries: one
347
+ that gave up after all `async_max_retries` gives up again at its first failure while the destination
348
+ is still unavailable, rather than hold up logging with a back-off as long again, and one that stopped
349
+ before using them all, such as after a fatal exception, has only the retries it had left. Once a
350
+ message is written, the count starts again. After a fork, the worker thread of the child process
351
+ starts counting afresh.
352
+
353
+ A flush or close does not wait out the back-offs of the worker thread, nor for each message ahead of
354
+ it to fail in turn. Submitting one ends the back-off under way, and until it has run, a message or
355
+ batch that the destination fails is discarded, and counted as `failed`, rather than written again
356
+ after a back-off. So is every message queued after that one, ahead of the flush or close, without
357
+ being written: during an outage, each would take as long to fail as the first, such as up to the
358
+ `open_timeout` (2 seconds by default) of an HTTP based appender whose server does not answer, or
359
+ longer for an appender that retries on its own, such as `:tcp` with its `connect_retry_count`. Those
360
+ failures do not count towards `async_max_retries`. Once the flush or close has run, the messages
361
+ queued after it are written as usual. A flush still waits while the first message is tried, and
362
+ while a destination that takes the messages writes every one of them.
363
+
364
+ When the process exits, Semantic Logger flushes every appender. Before that flush, it tells each
365
+ appender added with `async: true` or in batches that the process is exiting, and each one then acts
366
+ as though a flush were already waiting: it stops backing off and writing again what failed, and once
367
+ a write fails because of its destination, discards and counts the rest of its queue, without sending
368
+ it. Each one is told directly, rather than through the queue of the thread that writes to every
369
+ appender, since that thread can be held up handing a message to the full queue of an appender that
370
+ is backing off, and so would reach the flush only once that appender stopped backing off. So during
371
+ an outage the process exits within seconds, the time one write takes to fail, rather than minutes or
372
+ hours, and an appender whose destination takes the messages still writes every one of them. A
373
+ `SemanticLogger.flush` called while the process runs reaches each appender through that thread,
374
+ behind the messages logged before it.
375
+
376
+ An appender added without `async: true`, and not in batches, such as a plain `:http`,
377
+ `:splunk_http`, `:tcp`, or `:udp`, has no queue of its own: the thread that writes to every
378
+ appender writes each message to it in turn, and carries on with the next one when it fails, without
379
+ backing off. So during an outage, the flush at exit would wait while each message queued ahead of
380
+ it failed in turn, up to the `read_timeout` or `open_timeout` of an HTTP based appender whose server
381
+ does not answer. While the process exits, once a write to such an appender fails because of its
382
+ destination, rather than the content of the message, it is given no more messages for the rest of
383
+ the flush at exit. The messages it was not given are discarded, and the flush at exit reports their
384
+ number to the internal logger, with the error, as
385
+ `Discarded 29 log message(s) during the flush at exit, after the destination failed, for appender: SemanticLogger::Appender::Http`.
386
+ The flush at exit then still flushes it, so that it delivers the messages it had already accepted,
387
+ such as those that the producer of a `:kafka` appender holds; for `:http`, `:tcp`, or `:udp` that
388
+ flush does nothing. The other appenders are still given every message. Such an appender has no count
389
+ of its own in `SemanticLogger.stats`, so that report is the only record of them. While the process
390
+ runs, it is given every message, as before, and so is it in a child process forked during the flush
391
+ at exit.
200
392
 
201
393
  A flush or close is never left waiting on a worker thread that cannot answer it. When a batch cannot
202
394
  be written, a flush or close taken off the queue with it still runs before the thread backs off; a
@@ -204,6 +396,65 @@ close then stops the thread instead of restarting it. If the worker thread ends
204
396
  for example once it gives up, `SemanticLogger.flush` (including the flush at exit) stops waiting on
205
397
  that appender.
206
398
 
399
+ ### When every log entry must be delivered
400
+
401
+ Semantic Logger keeps the log entries it has not yet written in memory: on the queue of the thread
402
+ that writes to every appender, and on the queue of each appender added with `async: true` or in
403
+ batches. It does not store them anywhere that outlives the process. So the entries still queued are
404
+ lost when the process is killed (`kill -9`, the out of memory killer, or the `SIGKILL` that
405
+ Kubernetes and other process managers send once the grace period after `SIGTERM` runs out), or when
406
+ it ends without running `at_exit`, such as on a segmentation fault or `exit!`. An uncaught
407
+ exception, `exit`, and the default handling of `SIGTERM` still run the flush at exit, which writes
408
+ the queued entries to every destination that takes them.
409
+
410
+ During an outage of a remote destination, a flush, a close, and the flush at exit discard the
411
+ entries that destination fails, as described above, so that the process exits within seconds rather
412
+ than waiting for the outage to end. That holds for every `SemanticLogger.flush`, including one an
413
+ application or job framework calls while the process runs. There is no option to wait for every
414
+ entry instead: such a process would not stop until the destination came back, or until it was
415
+ killed, losing the entries anyway.
416
+
417
+ When every entry must reach its destination, such as for an audit log, write it to a local file
418
+ before the log call returns, and let a separate process deliver it:
419
+
420
+ - Log in synchronous mode (see [Synchronous operation](#synchronous-operation)), so that each entry
421
+ is written to the file before the log call returns, rather than waiting on a queue. The file
422
+ appender hands each entry to the operating system straight away, though without `fsync`, so a
423
+ crash of the host itself can still lose the last entries. A write that fails, such as on a full
424
+ disk, is reported to the internal logger rather than raised, so the log call still returns.
425
+ - In synchronous mode every appender writes on the application's thread, so add any remote appender,
426
+ such as `:http` or `:splunk_http`, with `async: true` or in batches, or during its outage every log
427
+ call waits out its timeouts.
428
+ - Ship the file with a log collector, such as [Vector](https://vector.dev),
429
+ [Fluent Bit](https://fluentbit.io), or the
430
+ [OpenTelemetry Collector](https://opentelemetry.io/docs/collector/), which keep their place in the
431
+ file and retry until the destination accepts the entries.
432
+
433
+ ~~~ruby
434
+ # Each entry is in the file before the log call returns...
435
+ SemanticLogger.sync!
436
+ SemanticLogger.add_appender(file_name: "log/production.json", formatter: :json)
437
+ # A remote appender in synchronous mode gets a thread of its own:
438
+ SemanticLogger.add_appender(appender: :http, url: "https://logs.example.com", async: true)
439
+
440
+ # ...and a collector running beside the application, such as Vector or Fluent Bit, reads the file
441
+ # and delivers every entry to the remote service, retrying through an outage.
442
+ ~~~
443
+
444
+ Sending the entries to a collector or agent on the same host, such as the OpenTelemetry Collector or
445
+ Fluent Bit with its own disk buffer, narrows the window, since a connection to the same host rarely
446
+ fails, but does not close it: those entries still wait in memory, and are discarded or lost as above.
447
+
448
+ Within those limits, an appender added with `async: true` or in batches loses no entries during an
449
+ outage while the process runs and nothing flushes or closes it, until its worker thread gives up
450
+ after `async_max_retries` (see above). With `async_max_retries: -1` it never gives up, so give it a
451
+ `max_queue_size` large enough to hold the entries logged during the longest outage it should ride
452
+ out: once its queue is full, logging waits for room for as long as the outage lasts, first the
453
+ thread that writes to every appender, so that the other appenders stop receiving entries too, then
454
+ the application's log calls, once the main queue is full as well. With `non_blocking: true` on the
455
+ appender, and `SemanticLogger.non_blocking = true` for the main queue, the entries are dropped
456
+ instead (see [Drop messages instead of blocking](#drop-messages-instead-of-blocking)).
457
+
207
458
  ### Give a slow appender its own thread
208
459
 
209
460
  If one destination is slow, such as a remote HTTP service, run just that appender on its own thread
@@ -223,6 +474,9 @@ burst of logging occurs. Check the queue at runtime:
223
474
  SemanticLogger.queue_size
224
475
  ~~~
225
476
 
477
+ In [synchronous mode](#synchronous-operation) entries are written on the calling thread, so it counts
478
+ only those logged in a signal trap that another thread has yet to write.
479
+
226
480
  For a fuller operational picture, including per-appender queues, use `SemanticLogger.stats`. It
227
481
  returns a Hash describing the main pipeline and every appender, handy for exporting Semantic Logger's
228
482
  own health to a monitoring system such as Prometheus or statsd:
@@ -246,7 +500,7 @@ SemanticLogger.stats
246
500
  # max_queue_size: 10_000,
247
501
  # processed: 1_529,
248
502
  # dropped: 0,
249
- # failed: 0 } # discarded because their content could not be written
503
+ # failed: 0 } # discarded because they could not be written
250
504
  # ]
251
505
  # }
252
506
  ~~~
@@ -272,8 +526,9 @@ number of appenders, or speed up the slow appender.
272
526
 
273
527
  Synchronous mode bypasses the background thread and logs inline on the calling thread. This disables
274
528
  a core design principle of Semantic Logger and slows the calling thread, so it is not recommended for
275
- most applications. It can suit short-lived or single-threaded programs, or forked environments where
276
- you would rather not re-create the logging thread.
529
+ most applications. It can suit short-lived or single-threaded programs, forked environments where
530
+ you would rather not re-create the logging thread, or an audit log that must be in a file before each
531
+ log call returns (see [When every log entry must be delivered](#when-every-log-entry-must-be-delivered)).
277
532
 
278
533
  Enable it **before** adding any appenders:
279
534
 
@@ -293,6 +548,20 @@ Or, in a Gemfile:
293
548
  gem "semantic_logger", require: "semantic_logger/sync"
294
549
  ~~~
295
550
 
551
+ In synchronous mode, one thread at a time writes to the appenders, so a log call waits while another
552
+ thread writes. Ruby does not allow that wait in a signal trap, nor an appender to lock a Mutex of its
553
+ own there, as a Ruby `Logger` given to `add_appender(logger:)` does when it writes. So a message
554
+ logged in a trap, such as by the handlers of [`add_signal_handler`](#linux-signals), is written just
555
+ after, by another thread, before any message logged after the trap; the trap does not wait for it.
556
+ That includes a trap that interrupted its own thread while it was writing a message: the trap's
557
+ message is written once that one is, never in the middle of it.
558
+
559
+ In a trap, `SemanticLogger.flush`, `close`, `reopen`, and `remove_appender` run on a thread that the
560
+ trap starts, after the messages logged in the trap before them, and the trap waits for it, at most
561
+ one second, and returns what it returned. When it has not finished by then, such as when the trap
562
+ interrupted its own thread while it was writing a message, which cannot go on before the trap
563
+ returns, the call returns `nil` and finishes just after. `add_appender` adds the appender at once.
564
+
296
565
  ## Linux signals
297
566
 
298
567
  On Linux, Unix, and Mac, Semantic Logger can respond to signals, for example to change the log level