pinqloq 1.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +68 -0
- data/LICENSE +21 -0
- data/README.md +177 -0
- data/lib/pinqloq/buffering/callback.rb +29 -0
- data/lib/pinqloq/buffering/log_buffer.rb +73 -0
- data/lib/pinqloq/buffering/log_dispatcher.rb +86 -0
- data/lib/pinqloq/buffering/queued_log.rb +7 -0
- data/lib/pinqloq/client.rb +28 -0
- data/lib/pinqloq/http/ingest_api_client.rb +157 -0
- data/lib/pinqloq/internal/throttled_warn.rb +24 -0
- data/lib/pinqloq/log_entry.rb +37 -0
- data/lib/pinqloq/log_failure_reason.rb +14 -0
- data/lib/pinqloq/log_level.rb +11 -0
- data/lib/pinqloq/log_source_type.rb +13 -0
- data/lib/pinqloq/logger.rb +40 -0
- data/lib/pinqloq/options.rb +40 -0
- data/lib/pinqloq/rack/captured_body.rb +41 -0
- data/lib/pinqloq/rack/path_match.rb +25 -0
- data/lib/pinqloq/rack/request_logging.rb +240 -0
- data/lib/pinqloq/redaction/redaction.rb +90 -0
- data/lib/pinqloq/redaction/redaction_plan.rb +120 -0
- data/lib/pinqloq/version.rb +5 -0
- data/lib/pinqloq.rb +19 -0
- metadata +132 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: b34593f2d1e32afe9a03124e36c7b3b824eb8155ba7a16156f113928e605cddf
|
|
4
|
+
data.tar.gz: f0ac5ed9f372d187d8d15be83e4a8832bc1637e190c18773fbf36925bd8b9b60
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: bcfb49cb5f6c33f5aa27c1f52e94e3d42f53efad2aee1808bb4c866fc36517e386bf12875257c2abf2e190a21d1b685e8d82d5c8fdf988b9ba8da424c089c0a6
|
|
7
|
+
data.tar.gz: 976f6719d2d3ffddc12c54f9ac98f50676b1c948fd5dad6ccd516afd16d78d1c451f12eac8cae271b733626e1fa36cdd9cd28afc57bdb39903e0b3f1d0777e2e
|
data/CHANGELOG.md
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to the `pinqloq` gem are documented here. This gem follows
|
|
4
|
+
[Semantic Versioning](https://semver.org). Its version numbers are independent of the .NET
|
|
5
|
+
`pinqloq` NuGet package and the `pinqloq` npm package — all three ship on separate cadences for
|
|
6
|
+
the same platform, and mirror each other's feature set rather than their version numbers.
|
|
7
|
+
|
|
8
|
+
## 1.1.2 — 2026-09-08
|
|
9
|
+
|
|
10
|
+
**Fixed:**
|
|
11
|
+
|
|
12
|
+
- Every batch failed to send, silently from the app's perspective (only a throttled warning in
|
|
13
|
+
the application log), whenever nothing else in the host process had already `require`d Ruby's
|
|
14
|
+
`time` standard library. `IngestApiClient` called `Time#iso8601` — a method the `time` library
|
|
15
|
+
monkey-patches onto `Time`, not one Ruby's core `Time` class defines — without ever requiring
|
|
16
|
+
it itself. It worked by accident in this gem's own test suite (`webmock` pulls in `time`
|
|
17
|
+
transitively) and in apps that happened to load another gem which requires `time`, and failed
|
|
18
|
+
silently in a plain Sinatra app that doesn't. `ingest_api_client.rb` now requires `time`
|
|
19
|
+
directly.
|
|
20
|
+
|
|
21
|
+
## 1.1.1 — 2026-09-08
|
|
22
|
+
|
|
23
|
+
**Fixed:**
|
|
24
|
+
|
|
25
|
+
- The request-logging middleware raised `NoMethodError` on the first request whenever
|
|
26
|
+
`rack.input` did not respond to `rewind` — notably `Rack::Lint::Wrapper::InputWrapper`, which
|
|
27
|
+
Sinatra (and many Rack apps) enable by default in the development environment. The middleware
|
|
28
|
+
now falls back to buffering the full body and replacing `rack.input` with a rewindable
|
|
29
|
+
`StringIO` when the original input can't be rewound in place, so downstream handlers still see
|
|
30
|
+
the complete request body either way.
|
|
31
|
+
|
|
32
|
+
## 1.1.0 — 2026-09-08
|
|
33
|
+
|
|
34
|
+
**Added:**
|
|
35
|
+
|
|
36
|
+
- `LogEntry#path` — a fixed, indexed field for the request path, matching the .NET SDK's 4.3.0
|
|
37
|
+
wire contract change ([pinqponq/pinqloq#108](https://github.com/pinqponq/pinqloq/issues/108)).
|
|
38
|
+
The request-logging middleware fills it automatically; on a manual `enqueue` it stays `nil`
|
|
39
|
+
unless set explicitly.
|
|
40
|
+
|
|
41
|
+
**Changed:**
|
|
42
|
+
|
|
43
|
+
- `event` and `path` are no longer duplicated into `metadata`. The request-logging middleware
|
|
44
|
+
used to also write `metadata["event"]`, `metadata["path"]`, and `metadata["RequestPath"]` —
|
|
45
|
+
those keys are gone from newly-sent logs now that `event` and `path` travel as their own
|
|
46
|
+
fixed, indexed fields. A `metadata: { "event" => ... }` enricher still overrides the event
|
|
47
|
+
title; it just no longer leaves a copy behind in `metadata`.
|
|
48
|
+
|
|
49
|
+
If you built a saved filter or dashboard on `metadata.event`, `metadata.path`, or
|
|
50
|
+
`metadata.RequestPath`, switch it to the `Event` / `Path` fields directly. No other consumer
|
|
51
|
+
action is needed — existing code compiles unchanged.
|
|
52
|
+
|
|
53
|
+
## 1.0.0 — 2026-09-08
|
|
54
|
+
|
|
55
|
+
**Added:**
|
|
56
|
+
|
|
57
|
+
- Initial release: feature parity with the .NET and Node.js SDKs' core surface.
|
|
58
|
+
- `Pinqloq.create(...)` — buffered, batched manual structured logging (`logger.enqueue` /
|
|
59
|
+
`logger.enqueue_many`), delivered to the same `/bulk` ingest endpoint the other SDKs use.
|
|
60
|
+
- `Pinqloq::Rack::RequestLogging` — a Rack middleware (works with Rails, Sinatra, Hanami, Grape,
|
|
61
|
+
or any Rack app) for automatic HTTP request/response logging: captures method/path/status/
|
|
62
|
+
duration, request/response bodies (32KB cap) and headers, resolves `device_identifier`
|
|
63
|
+
(resolver → `device-identifier` header → global fallback, HTTP 400 if none resolve) and
|
|
64
|
+
`correlation_id` (`correlation-id` header → generated UUID).
|
|
65
|
+
- Redaction: a built-in, unconditional credential-name floor (password, token, Authorization,
|
|
66
|
+
...) plus `redact_fields` (name-based, any nesting depth) and `redact_paths` (whole-body/header
|
|
67
|
+
masking, the `[PinqloqRedactEndpoint]` equivalent). Fails closed on a sensitive body that can't
|
|
68
|
+
be parsed as JSON, matching the .NET and Node SDKs' behavior.
|
data/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 pinqponq
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
data/README.md
ADDED
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
# Pinqloq (Ruby / Rack)
|
|
2
|
+
|
|
3
|
+
[](https://rubygems.org/gems/pinqloq)
|
|
4
|
+
[](https://opensource.org/licenses/MIT)
|
|
5
|
+
|
|
6
|
+
Pinqloq is a structured logging and log shipping SDK for centralized application logs. It
|
|
7
|
+
captures HTTP request/response logs through a Rack middleware and sends manual application events
|
|
8
|
+
to the Pinqloq log management platform using in-memory buffering, batching, and HTTPS delivery.
|
|
9
|
+
This is the Ruby counterpart of the [.NET](https://www.nuget.org/packages/pinqloq) and
|
|
10
|
+
[Node.js](https://www.npmjs.com/package/pinqloq) `pinqloq` SDKs — same platform, same wire
|
|
11
|
+
protocol, idiomatic API on each side.
|
|
12
|
+
|
|
13
|
+
Because the middleware targets [Rack](https://github.com/rack/rack) rather than one framework, it
|
|
14
|
+
works unmodified with Rails, Sinatra, Hanami, Grape, or a bare Rack app.
|
|
15
|
+
|
|
16
|
+
## Features
|
|
17
|
+
|
|
18
|
+
- Automatic Rack request/response logging — works with any Rack-based framework
|
|
19
|
+
- Correlation id read from the caller's header, falling back to a generated UUID
|
|
20
|
+
- Name-based redaction of sensitive fields, headers, and whole endpoints
|
|
21
|
+
- Manual structured application events
|
|
22
|
+
- Buffered and batched HTTPS delivery, backed by a single background thread
|
|
23
|
+
- Graceful shutdown flush
|
|
24
|
+
|
|
25
|
+
## Requirements
|
|
26
|
+
|
|
27
|
+
- Ruby 3.1 or later
|
|
28
|
+
- A Pinqloq project and secret key
|
|
29
|
+
|
|
30
|
+
## Installation
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
gem install pinqloq
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
Or in a `Gemfile`:
|
|
37
|
+
|
|
38
|
+
```ruby
|
|
39
|
+
gem "pinqloq"
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
## Quick Start
|
|
43
|
+
|
|
44
|
+
Store your secret key in an environment variable or a secret manager. Do not hardcode production
|
|
45
|
+
credentials.
|
|
46
|
+
|
|
47
|
+
```ruby
|
|
48
|
+
require "pinqloq"
|
|
49
|
+
|
|
50
|
+
pinqloq = Pinqloq.create(
|
|
51
|
+
secret_key: ENV.fetch("PINQLOQ_SECRET_KEY"),
|
|
52
|
+
api_logs_collection_name: "myapp_api_logs",
|
|
53
|
+
device_identifier: "myapp-instance-1"
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
# Rack app / Rails config/application.rb / config.ru
|
|
57
|
+
use Pinqloq::Rack::RequestLogging,
|
|
58
|
+
logger: pinqloq.logger,
|
|
59
|
+
pinqloq_options: pinqloq.options,
|
|
60
|
+
exclude_paths: ["/health"]
|
|
61
|
+
|
|
62
|
+
at_exit { pinqloq.shutdown }
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
The middleware captures the HTTP method, path, and status code as searchable metadata. The
|
|
66
|
+
request body, response body, request headers, and response headers go to the log detail as
|
|
67
|
+
`InputJson`, `OutputJson`, `RequestHeaders`, and `ResponseHeaders`. Bodies are truncated at 32 KB.
|
|
68
|
+
|
|
69
|
+
## Manual Logging
|
|
70
|
+
|
|
71
|
+
Use `pinqloq.logger` to send structured application events:
|
|
72
|
+
|
|
73
|
+
```ruby
|
|
74
|
+
pinqloq.logger.enqueue(
|
|
75
|
+
Pinqloq::LogEntry.new(
|
|
76
|
+
event: "order.created",
|
|
77
|
+
device_identifier: order.customer_id,
|
|
78
|
+
log_level: Pinqloq::LogLevel::INFORMATION,
|
|
79
|
+
log_source_type: Pinqloq::LogSourceType::BACKEND,
|
|
80
|
+
metadata: { "orderId" => order.id }
|
|
81
|
+
)
|
|
82
|
+
)
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
`event` and `device_identifier` are required on every entry. Leave `device_identifier` unset on
|
|
86
|
+
an entry to inherit the global `device_identifier` option. `enqueue` raises if an entry has no
|
|
87
|
+
`device_identifier` and no global fallback is set — a missing required field fails loudly rather
|
|
88
|
+
than being silently dropped.
|
|
89
|
+
|
|
90
|
+
## Add Request Metadata
|
|
91
|
+
|
|
92
|
+
By default the middleware reads the required `device_identifier` from the `device-identifier`
|
|
93
|
+
request header automatically. Override how it is resolved with `resolve_device_identifier`; the
|
|
94
|
+
override wins, and if it returns nil/blank the middleware falls back to the `device-identifier`
|
|
95
|
+
header, then to the global `device_identifier` option. If none of these resolve a value, the
|
|
96
|
+
middleware rejects the request with **HTTP 400** before it runs.
|
|
97
|
+
|
|
98
|
+
```ruby
|
|
99
|
+
use Pinqloq::Rack::RequestLogging,
|
|
100
|
+
logger: pinqloq.logger,
|
|
101
|
+
pinqloq_options: pinqloq.options,
|
|
102
|
+
exclude_paths: ["/health"],
|
|
103
|
+
resolve_device_identifier: ->(request) { request.session[:user_id] },
|
|
104
|
+
resolve_app_version_name: ->(request) { request.get_header("HTTP_X_APP_VERSION") },
|
|
105
|
+
metadata: {
|
|
106
|
+
"userId" => ->(request, _status, _headers) { request.session[:user_id] }
|
|
107
|
+
}
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
Use `metadata` for searchable values such as user and tenant IDs. Use `detail` for additional
|
|
111
|
+
drill-down information. The `event` key (the panel title) defaults to `"{method} {path}"` and can
|
|
112
|
+
be overridden via `metadata["event"]`.
|
|
113
|
+
|
|
114
|
+
## Correlation ID
|
|
115
|
+
|
|
116
|
+
Every log carries a `correlation_id` that ties together the records of a single request or flow.
|
|
117
|
+
The request-logging middleware fills it with no configuration: the caller's `correlation-id`
|
|
118
|
+
request header when present, otherwise a generated UUID.
|
|
119
|
+
|
|
120
|
+
```ruby
|
|
121
|
+
pinqloq.logger.enqueue(
|
|
122
|
+
Pinqloq::LogEntry.new(
|
|
123
|
+
event: "order.created",
|
|
124
|
+
device_identifier: order.customer_id,
|
|
125
|
+
correlation_id: current_correlation_id
|
|
126
|
+
)
|
|
127
|
+
)
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
## Redacting Sensitive Values
|
|
131
|
+
|
|
132
|
+
Request and response bodies and headers may contain credentials, tokens, or personal information.
|
|
133
|
+
Unlike the .NET SDK's attribute-based redaction (which relies on C# reflection over typed DTOs —
|
|
134
|
+
not available the same way in Ruby's dynamically-typed request/response objects), this SDK
|
|
135
|
+
redacts by **name**, exactly like the Node.js SDK:
|
|
136
|
+
|
|
137
|
+
- `redact_fields` — case-insensitive field/header names masked with `*****REDACTED*****` wherever
|
|
138
|
+
they appear in a captured body or header, at any nesting depth.
|
|
139
|
+
- `redact_paths` — path prefixes (matched the same way as `exclude_paths`) where every value in
|
|
140
|
+
`InputJson`, `OutputJson`, `RequestHeaders`, and `ResponseHeaders` is masked, keeping the JSON
|
|
141
|
+
structure and header names intact.
|
|
142
|
+
|
|
143
|
+
```ruby
|
|
144
|
+
use Pinqloq::Rack::RequestLogging,
|
|
145
|
+
logger: pinqloq.logger,
|
|
146
|
+
pinqloq_options: pinqloq.options,
|
|
147
|
+
redact_fields: ["ssn_last_four"],
|
|
148
|
+
redact_paths: ["/payment"]
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
A built-in, unconditional floor of common credential names (password, token, `Authorization`,
|
|
152
|
+
card numbers, ...) is always masked, even with no configuration — see
|
|
153
|
+
[`lib/pinqloq/redaction/redaction_plan.rb`](lib/pinqloq/redaction/redaction_plan.rb) for the full
|
|
154
|
+
list.
|
|
155
|
+
|
|
156
|
+
## Security and Reliability
|
|
157
|
+
|
|
158
|
+
Logs are buffered in memory and sent in batches by a single background thread. Buffered logs may
|
|
159
|
+
be lost if the process is terminated without a graceful shutdown — call `pinqloq.shutdown` on
|
|
160
|
+
exit (`at_exit`, a `Rails.application.config.after_initialize` shutdown hook, or your process
|
|
161
|
+
manager's stop signal handler).
|
|
162
|
+
|
|
163
|
+
Delivery failures are reported through `on_failed` callbacks and, even without callbacks, as
|
|
164
|
+
throttled warnings via Ruby's `Kernel#warn` — never silently discarded, but also never blocking.
|
|
165
|
+
If your secret key is authorized for more than one collection, set `api_logs_collection_name` (or
|
|
166
|
+
a per-entry `collection_name`); otherwise the batch is rejected.
|
|
167
|
+
|
|
168
|
+
## Documentation
|
|
169
|
+
|
|
170
|
+
- [.NET SDK](https://www.nuget.org/packages/pinqloq), [Node.js SDK](https://www.npmjs.com/package/pinqloq),
|
|
171
|
+
[Go SDK](https://pkg.go.dev/github.com/pinqponq/pinqloq-go-sdk) — the other implementations of
|
|
172
|
+
this platform's wire protocol and feature set.
|
|
173
|
+
- [Full documentation](https://pinqloq.pinqponq.io/documentation.html)
|
|
174
|
+
|
|
175
|
+
## License
|
|
176
|
+
|
|
177
|
+
MIT
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pinqloq
|
|
4
|
+
module Buffering
|
|
5
|
+
module Callback
|
|
6
|
+
module_function
|
|
7
|
+
|
|
8
|
+
def raise_sent(on_sent, entry)
|
|
9
|
+
return unless on_sent
|
|
10
|
+
|
|
11
|
+
begin
|
|
12
|
+
on_sent.call(entry)
|
|
13
|
+
rescue StandardError => e
|
|
14
|
+
warn "Pinqloq: the on_sent callback raised an exception; swallowed. #{e.class}: #{e.message}"
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def raise_failed(on_failed, entry, error)
|
|
19
|
+
return unless on_failed
|
|
20
|
+
|
|
21
|
+
begin
|
|
22
|
+
on_failed.call(entry, error)
|
|
23
|
+
rescue StandardError => e
|
|
24
|
+
warn "Pinqloq: the on_failed callback raised an exception; swallowed. #{e.class}: #{e.message}"
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../log_failure_reason"
|
|
4
|
+
require_relative "queued_log"
|
|
5
|
+
require_relative "callback"
|
|
6
|
+
|
|
7
|
+
module Pinqloq
|
|
8
|
+
module Buffering
|
|
9
|
+
class LogBuffer
|
|
10
|
+
def initialize(capacity)
|
|
11
|
+
@capacity = capacity
|
|
12
|
+
@queue = []
|
|
13
|
+
@dropped_count = 0
|
|
14
|
+
@closed = false
|
|
15
|
+
@mutex = Mutex.new
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def size
|
|
19
|
+
@mutex.synchronize { @queue.size }
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def enqueue(entry, on_sent: nil, on_failed: nil)
|
|
23
|
+
raise "Pinqloq: the log queue is closed." if @closed
|
|
24
|
+
return false if entry.nil?
|
|
25
|
+
|
|
26
|
+
entry.date ||= Time.now.utc
|
|
27
|
+
|
|
28
|
+
dropped = false
|
|
29
|
+
@mutex.synchronize do
|
|
30
|
+
if @queue.size >= @capacity
|
|
31
|
+
@dropped_count += 1
|
|
32
|
+
dropped = true
|
|
33
|
+
else
|
|
34
|
+
@queue.push(QueuedLog.new(entry: entry, on_sent: on_sent, on_failed: on_failed))
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
return true unless dropped
|
|
39
|
+
|
|
40
|
+
warn "Pinqloq: log queue is full; #{@dropped_count} logs dropped so far." if @dropped_count == 1 || (@dropped_count % 1000).zero?
|
|
41
|
+
|
|
42
|
+
Callback.raise_failed(
|
|
43
|
+
on_failed,
|
|
44
|
+
entry,
|
|
45
|
+
LogError.new(
|
|
46
|
+
reason: LogFailureReason::QUEUE_FULL,
|
|
47
|
+
message: "Log queue is full; the log was dropped. Increase queue_capacity or lower flush_interval."
|
|
48
|
+
)
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
false
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def enqueue_many(entries, on_sent: nil, on_failed: nil)
|
|
55
|
+
return 0 if entries.nil?
|
|
56
|
+
|
|
57
|
+
entries.count { |entry| enqueue(entry, on_sent: on_sent, on_failed: on_failed) }
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def close!
|
|
61
|
+
@mutex.synchronize { @closed = true }
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def closed?
|
|
65
|
+
@mutex.synchronize { @closed }
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def drain(max)
|
|
69
|
+
@mutex.synchronize { @queue.shift(max) }
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
end
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pinqloq
|
|
4
|
+
module Buffering
|
|
5
|
+
class LogDispatcher
|
|
6
|
+
def initialize(buffer, api_client, options)
|
|
7
|
+
@buffer = buffer
|
|
8
|
+
@api_client = api_client
|
|
9
|
+
@options = options
|
|
10
|
+
@mutex = Mutex.new
|
|
11
|
+
@condition = ConditionVariable.new
|
|
12
|
+
@stopping = false
|
|
13
|
+
@thread = nil
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def start
|
|
17
|
+
return if @thread
|
|
18
|
+
|
|
19
|
+
@thread = Thread.new { run }
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def notify_enqueued
|
|
23
|
+
@mutex.synchronize { @condition.signal }
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def shutdown
|
|
27
|
+
@mutex.synchronize do
|
|
28
|
+
@stopping = true
|
|
29
|
+
@condition.signal
|
|
30
|
+
end
|
|
31
|
+
@thread&.join
|
|
32
|
+
@thread = nil
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
private
|
|
36
|
+
|
|
37
|
+
def run
|
|
38
|
+
loop do
|
|
39
|
+
wait_for_first_entry
|
|
40
|
+
return if finished?
|
|
41
|
+
|
|
42
|
+
batch_window
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def wait_for_first_entry
|
|
47
|
+
@mutex.synchronize do
|
|
48
|
+
@condition.wait(@mutex) while @buffer.size.zero? && !@stopping
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def finished?
|
|
53
|
+
@buffer.size.zero? && @stopping
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def batch_window
|
|
57
|
+
deadline = monotonic_now + @options.flush_interval
|
|
58
|
+
|
|
59
|
+
@mutex.synchronize do
|
|
60
|
+
while @buffer.size < @options.batch_size && !@stopping
|
|
61
|
+
remaining = deadline - monotonic_now
|
|
62
|
+
break if remaining <= 0
|
|
63
|
+
|
|
64
|
+
@condition.wait(@mutex, remaining)
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
send_batch(@buffer.drain(@options.batch_size))
|
|
69
|
+
|
|
70
|
+
send_batch(@buffer.drain(@options.batch_size)) while @stopping && @buffer.size.positive?
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def send_batch(batch)
|
|
74
|
+
return if batch.empty?
|
|
75
|
+
|
|
76
|
+
@api_client.send_batch(batch)
|
|
77
|
+
rescue StandardError => e
|
|
78
|
+
warn "Pinqloq: batch of #{batch.length} logs could not be sent; dropped. #{e.class}: #{e.message}"
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
def monotonic_now
|
|
82
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
end
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "options"
|
|
4
|
+
require_relative "logger"
|
|
5
|
+
require_relative "buffering/log_buffer"
|
|
6
|
+
require_relative "buffering/log_dispatcher"
|
|
7
|
+
require_relative "http/ingest_api_client"
|
|
8
|
+
require_relative "rack/request_logging"
|
|
9
|
+
|
|
10
|
+
module Pinqloq
|
|
11
|
+
class Client
|
|
12
|
+
attr_reader :logger, :options
|
|
13
|
+
|
|
14
|
+
def initialize(options)
|
|
15
|
+
@options = options
|
|
16
|
+
@buffer = Buffering::LogBuffer.new(options.queue_capacity)
|
|
17
|
+
api_client = Http::IngestApiClient.new(options)
|
|
18
|
+
@dispatcher = Buffering::LogDispatcher.new(@buffer, api_client, options)
|
|
19
|
+
@logger = Logger.new(@buffer, @dispatcher, options)
|
|
20
|
+
|
|
21
|
+
@dispatcher.start
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def shutdown
|
|
25
|
+
@dispatcher.shutdown
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "net/http"
|
|
4
|
+
require "json"
|
|
5
|
+
require "uri"
|
|
6
|
+
require "time"
|
|
7
|
+
require_relative "../options"
|
|
8
|
+
require_relative "../log_failure_reason"
|
|
9
|
+
require_relative "../log_source_type"
|
|
10
|
+
require_relative "../buffering/callback"
|
|
11
|
+
|
|
12
|
+
module Pinqloq
|
|
13
|
+
module Http
|
|
14
|
+
class IngestApiClient
|
|
15
|
+
SECRET_KEY_HEADER = "X-Secret-Key"
|
|
16
|
+
HTTP_WARNING_THROTTLE_SECONDS = 60
|
|
17
|
+
MAX_ERROR_BODY_CHARACTERS = 512
|
|
18
|
+
|
|
19
|
+
def initialize(options)
|
|
20
|
+
@options = options
|
|
21
|
+
@uri = URI.join("#{INGEST_BASE_ADDRESS}/", options.bulk_path.sub(%r{\A/+}, ""))
|
|
22
|
+
@next_http_warning_at = Time.at(0)
|
|
23
|
+
@warning_mutex = Mutex.new
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def send_batch(items)
|
|
27
|
+
return if items.empty?
|
|
28
|
+
|
|
29
|
+
items.group_by { |item| resolve_collection_name(item.entry) }.each do |collection_name, group_items|
|
|
30
|
+
send_group(collection_name, group_items)
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
private
|
|
35
|
+
|
|
36
|
+
def send_group(collection_name, group_items)
|
|
37
|
+
payload = {
|
|
38
|
+
collectionName: collection_name,
|
|
39
|
+
logs: group_items.map { |item| to_wire_item(item.entry) }
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
request = Net::HTTP::Post.new(@uri)
|
|
43
|
+
request[SECRET_KEY_HEADER] = @options.secret_key
|
|
44
|
+
request["Content-Type"] = "application/json"
|
|
45
|
+
request.body = JSON.generate(payload)
|
|
46
|
+
|
|
47
|
+
response = Net::HTTP.start(@uri.host, @uri.port, use_ssl: @uri.scheme == "https", read_timeout: @options.http_timeout, open_timeout: @options.http_timeout) do |http|
|
|
48
|
+
http.request(request)
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
if response.is_a?(Net::HTTPSuccess)
|
|
52
|
+
group_items.each { |item| Buffering::Callback.raise_sent(item.on_sent, item.entry) }
|
|
53
|
+
return
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
error = build_http_error(response, collection_name)
|
|
57
|
+
group_items.each { |item| Buffering::Callback.raise_failed(item.on_failed, item.entry, error) }
|
|
58
|
+
rescue StandardError => e
|
|
59
|
+
error = build_exception_error(e)
|
|
60
|
+
warn "Pinqloq: group of #{group_items.length} logs could not be sent (#{format_collection_name(collection_name)}). #{e.class}: #{e.message}"
|
|
61
|
+
group_items.each { |item| Buffering::Callback.raise_failed(item.on_failed, item.entry, error) }
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def build_http_error(response, collection_name)
|
|
65
|
+
status = response.code.to_i
|
|
66
|
+
reason =
|
|
67
|
+
case status
|
|
68
|
+
when 401 then LogFailureReason::UNAUTHORIZED
|
|
69
|
+
when 403 then LogFailureReason::FORBIDDEN
|
|
70
|
+
else LogFailureReason::HTTP_ERROR
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
message =
|
|
74
|
+
case reason
|
|
75
|
+
when LogFailureReason::UNAUTHORIZED
|
|
76
|
+
"Unauthorized (HTTP 401): the secret key is invalid or missing."
|
|
77
|
+
when LogFailureReason::FORBIDDEN
|
|
78
|
+
"Forbidden (HTTP 403): the secret key is not authorized for the '#{format_collection_name(collection_name)}' collection."
|
|
79
|
+
else
|
|
80
|
+
if status == 400
|
|
81
|
+
"The server rejected the request (HTTP 400, '#{format_collection_name(collection_name)}'): " \
|
|
82
|
+
"check the collection_name (required for keys allowed on multiple collections) or event fields."
|
|
83
|
+
else
|
|
84
|
+
"The server returned an error (HTTP #{status})."
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
error_body = read_error_body(response)
|
|
89
|
+
message += " Server response: #{error_body}" unless error_body.empty?
|
|
90
|
+
|
|
91
|
+
if should_log_http_failure?
|
|
92
|
+
warn "Pinqloq: batch send rejected (HTTP #{status}, collection '#{format_collection_name(collection_name)}'); logs in this group were dropped. #{message}"
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
LogError.new(reason: reason, status_code: status, message: message)
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def build_exception_error(exception)
|
|
99
|
+
timeout = exception.is_a?(Net::OpenTimeout) || exception.is_a?(Net::ReadTimeout) || exception.is_a?(Timeout::Error)
|
|
100
|
+
|
|
101
|
+
if timeout
|
|
102
|
+
LogError.new(reason: LogFailureReason::TIMEOUT, message: "The request timed out.", cause: exception)
|
|
103
|
+
else
|
|
104
|
+
LogError.new(reason: LogFailureReason::NETWORK, message: "Network error: #{exception.message}", cause: exception)
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
def read_error_body(response)
|
|
109
|
+
body = (response.body || "").strip
|
|
110
|
+
body.length <= MAX_ERROR_BODY_CHARACTERS ? body : body[0, MAX_ERROR_BODY_CHARACTERS]
|
|
111
|
+
rescue StandardError => e
|
|
112
|
+
warn "Pinqloq: could not read the error response body; continuing without it. #{e.class}: #{e.message}"
|
|
113
|
+
""
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def resolve_collection_name(entry)
|
|
117
|
+
name = entry.collection_name&.strip
|
|
118
|
+
(name && !name.empty?) ? name : @options.api_logs_collection_name
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
def should_log_http_failure?
|
|
122
|
+
@warning_mutex.synchronize do
|
|
123
|
+
now = Time.now
|
|
124
|
+
return false if now < @next_http_warning_at
|
|
125
|
+
|
|
126
|
+
@next_http_warning_at = now + HTTP_WARNING_THROTTLE_SECONDS
|
|
127
|
+
true
|
|
128
|
+
end
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
def format_collection_name(collection_name)
|
|
132
|
+
(collection_name && !collection_name.strip.empty?) ? collection_name : "(not resolved server-side)"
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
def to_wire_item(entry)
|
|
136
|
+
app_version_name = entry.app_version_name&.strip
|
|
137
|
+
app_version_name = (app_version_name && !app_version_name.empty?) ? entry.app_version_name : @options.app_version_name
|
|
138
|
+
|
|
139
|
+
device_identifier = entry.device_identifier&.strip
|
|
140
|
+
device_identifier = (device_identifier && !device_identifier.empty?) ? entry.device_identifier : (@options.device_identifier || "")
|
|
141
|
+
|
|
142
|
+
{
|
|
143
|
+
logLevel: entry.log_level,
|
|
144
|
+
event: entry.event,
|
|
145
|
+
date: entry.date&.iso8601(3),
|
|
146
|
+
appVersionName: app_version_name,
|
|
147
|
+
deviceIdentifier: device_identifier,
|
|
148
|
+
logSourceType: LogSourceType::NAMES.fetch(entry.log_source_type, "Backend"),
|
|
149
|
+
correlationId: entry.correlation_id,
|
|
150
|
+
path: entry.path,
|
|
151
|
+
metadata: entry.metadata,
|
|
152
|
+
detail: entry.detail
|
|
153
|
+
}
|
|
154
|
+
end
|
|
155
|
+
end
|
|
156
|
+
end
|
|
157
|
+
end
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pinqloq
|
|
4
|
+
module Internal
|
|
5
|
+
module ThrottledWarn
|
|
6
|
+
@next_allowed_at = {}
|
|
7
|
+
@mutex = Mutex.new
|
|
8
|
+
|
|
9
|
+
class << self
|
|
10
|
+
def warn_throttled(key, interval_seconds, message)
|
|
11
|
+
should_warn = @mutex.synchronize do
|
|
12
|
+
now = Time.now
|
|
13
|
+
next false if now < @next_allowed_at.fetch(key, Time.at(0))
|
|
14
|
+
|
|
15
|
+
@next_allowed_at[key] = now + interval_seconds
|
|
16
|
+
true
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
warn message if should_warn
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|