fractal-ruby 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +30 -0
- data/LICENSE +21 -0
- data/README.md +151 -0
- data/VERSION +1 -0
- data/lib/fractal/analytics.rb +437 -0
- data/lib/fractal/breaker.rb +69 -0
- data/lib/fractal/client.rb +106 -0
- data/lib/fractal/configuration.rb +164 -0
- data/lib/fractal/error_reporter.rb +338 -0
- data/lib/fractal/event_builder.rb +95 -0
- data/lib/fractal/fingerprint.rb +287 -0
- data/lib/fractal/identity.rb +52 -0
- data/lib/fractal/rails/error_subscriber.rb +14 -0
- data/lib/fractal/rails/middleware.rb +41 -0
- data/lib/fractal/rails/railtie.rb +36 -0
- data/lib/fractal/rails/warden_hooks.rb +39 -0
- data/lib/fractal/scope.rb +95 -0
- data/lib/fractal/scrubber.rb +253 -0
- data/lib/fractal/transport.rb +25 -0
- data/lib/fractal/version.rb +6 -0
- data/lib/fractal-ruby.rb +4 -0
- data/lib/fractal.rb +111 -0
- metadata +63 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: fb85bbae2629a4f35ac315bda48069d46f952a44391422d162e88d05814d6724
|
|
4
|
+
data.tar.gz: 68f9802f6b2e0db444ce472670269dab8e626d13370adcd47037c362430831ce
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: a228762b9ee48fe5e87c4d12b4dcda604a0138b57610c9e7ff5cbdcd0f194b001437fc2e4c51ca18824a3ad98c4f5371d18f1d51b5eb77db4729e664a0bf8229
|
|
7
|
+
data.tar.gz: b43092221f112e42367e98c4f8cebb200ce6369c4f2a598395aab3abeb0474e411fe95177c822e465d9a63a970a4b882ad71b416b5c7465a294cbd5e7b147593
|
data/CHANGELOG.md
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to the `fractal-ruby` gem. The version lives in `VERSION`;
|
|
4
|
+
`ci/check_version_sync.sh` keeps `lib/fractal/version.rb`, this file and the lockfiles in step.
|
|
5
|
+
|
|
6
|
+
## 0.1.0
|
|
7
|
+
|
|
8
|
+
First version (#495, ADR 0003). Not yet published to rubygems.org (#497); Fractal's own app uses it
|
|
9
|
+
from the path.
|
|
10
|
+
|
|
11
|
+
- **Errors**, extracted from Fractal's in-repo self-reporter (`lib/self_report/`, #426): a Railtie
|
|
12
|
+
adds an outermost Rack middleware and a `Rails.error` subscriber; events are built from an
|
|
13
|
+
allowlist, scrubbed (Rails `filter_parameters`, extra keys, secret ENV values, credential and
|
|
14
|
+
email patterns, structured Postgres messages), coalesced by the collector's fingerprint (ported,
|
|
15
|
+
pinned by the shared vectors) and delivered from a background thread with a circuit breaker,
|
|
16
|
+
fork reset and a 2 s shutdown flush.
|
|
17
|
+
- **Configuration instead of constants**: `Fractal.configure` takes the API key, collector URL,
|
|
18
|
+
app version, environment, a runtime `enabled` switch, a `user_resolver`, extra filter keys,
|
|
19
|
+
`message_filters` and a `before_send_error` hook.
|
|
20
|
+
- **401 stops sending** for the rest of the process, for errors and analytics alike; 402 still
|
|
21
|
+
turns error reporting off.
|
|
22
|
+
- **Request-scoped context**: `Fractal.set_user`, `set_tag`, `add_breadcrumb` and `set_session`,
|
|
23
|
+
per request and per job, never global.
|
|
24
|
+
- **Analytics** (new): `Fractal.analytics.track` with its own buffer and background flush, a
|
|
25
|
+
gem-generated `event_id`, the 128-byte `event_type` and 64 KB payload checks before queueing,
|
|
26
|
+
per-event `player_id`, `session_token` and `client_ip`, and `server_side: true` on the envelope.
|
|
27
|
+
- **Packaging** (#497): MIT licensed, with the `LICENSE` file in the gem. Rails only: plain Rack
|
|
28
|
+
apps and Sidekiq workers outside Active Job are documented gaps.
|
|
29
|
+
- **Sessions and sign-in**: the default session is a hash of the Rack session id; Warden/Devise
|
|
30
|
+
sign-in tracks `identify`, linking the logged-out session to the user.
|
data/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Fractal
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
data/README.md
ADDED
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
# fractal-ruby
|
|
2
|
+
|
|
3
|
+
Error tracking and server-side analytics for Rails backends, reporting to
|
|
4
|
+
[Fractal](https://getfractal.dev). The Ruby peer of the JavaScript, Godot and LÖVE SDKs: same
|
|
5
|
+
facade, same wire protocol (`sdks/shared/ANALYTICS_PROTOCOL.md`, `sdks/shared/CRASH_PROTOCOL.md`),
|
|
6
|
+
same conformance check.
|
|
7
|
+
|
|
8
|
+
Requires Ruby 3.2+ and Rails 7.1+ (ADR 0003). The core (transport, buffers, scrubber, fingerprint,
|
|
9
|
+
`track`) is plain Ruby; the Railtie is the supported entry point for the first release.
|
|
10
|
+
|
|
11
|
+
> Not published yet (#497). Fractal's own app uses it from the path:
|
|
12
|
+
> `gem "fractal-ruby", path: "sdks/ruby"`.
|
|
13
|
+
|
|
14
|
+
## Setup
|
|
15
|
+
|
|
16
|
+
```ruby
|
|
17
|
+
# Gemfile
|
|
18
|
+
gem "fractal-ruby"
|
|
19
|
+
|
|
20
|
+
# config/initializers/fractal.rb
|
|
21
|
+
Fractal.configure do |c|
|
|
22
|
+
c.api_key = ENV["FRACTAL_API_KEY"] # a key created for this backend
|
|
23
|
+
c.app_version = ENV["GIT_SHA"] # resolve/reopen compare versions
|
|
24
|
+
c.user_resolver = ->(controller) { "user-#{controller.current_user.id}" if controller.current_user }
|
|
25
|
+
end
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
Without an `api_key` the gem does nothing, which is the usual setting for development and test.
|
|
29
|
+
Create a **dedicated key** for the server: until key scopes exist (#490) a key has the same power
|
|
30
|
+
wherever it is used.
|
|
31
|
+
|
|
32
|
+
## Errors
|
|
33
|
+
|
|
34
|
+
The Railtie installs an outermost Rack middleware and a `Rails.error` subscriber, so unhandled
|
|
35
|
+
exceptions in requests and Active Job jobs are reported with no further code. Rails' own filter
|
|
36
|
+
(404s and other rescue responses) is respected.
|
|
37
|
+
|
|
38
|
+
```ruby
|
|
39
|
+
Fractal.errors.capture_error(exception) # one you rescued (handled: true)
|
|
40
|
+
Fractal.errors.capture_message("payment provider slow")
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
What is sent is built from an allowlist (error class, message, frames, controller#action, route,
|
|
44
|
+
request id, job class, queue, executions) and then scrubbed: keys in `filter_parameters` plus
|
|
45
|
+
`extra_filter_keys`, the values of secret-looking ENV variables, credentials in URLs, bearer and
|
|
46
|
+
basic tokens, JWTs, Stripe keys, emails and long hex strings. Postgres errors are reduced to
|
|
47
|
+
`<class> <SQLSTATE> <constraint>`, other Active Record errors to their first line, and errors that
|
|
48
|
+
echo their input (JSON parse errors and the like) have their message withheld. Add your own rules:
|
|
49
|
+
|
|
50
|
+
```ruby
|
|
51
|
+
c.message_filters << ->(exception, raw) { "Redis timeout" if exception.is_a?(Redis::TimeoutError) }
|
|
52
|
+
c.before_send_error = ->(event) { event["tags"]["region"] = "eu"; event } # nil drops the event
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
Repeats of one error are coalesced into one event with an `occurrence_count`, keyed by the
|
|
56
|
+
collector's own fingerprint. Delivery runs on a background thread: at most 100 distinct errors
|
|
57
|
+
buffered, 5 requests per second, a circuit breaker after 5 failures, a fresh buffer after fork
|
|
58
|
+
(Puma, Solid Queue) and a 2-second flush at exit.
|
|
59
|
+
|
|
60
|
+
## Analytics
|
|
61
|
+
|
|
62
|
+
```ruby
|
|
63
|
+
Fractal.analytics.track("checkout_completed", { "total_cents" => 1299 })
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
`track` never sends on the caller's thread: events are buffered (5,000 events or 8 MB, oldest
|
|
67
|
+
dropped first) and flushed every 10 seconds or at 100 events, up to 1,000 per request. Each event
|
|
68
|
+
gets its own `event_id` (UUID). The collector's limits are checked first: an empty or over-128-byte
|
|
69
|
+
`event_type`, a payload that is not a Hash or is over 64 KB serialised, a `player_id` over 256
|
|
70
|
+
bytes, a `timestamp` that is not a `Time`, or an `event_type`, `player_id` or `session_token`
|
|
71
|
+
that is not valid UTF-8, raises `Fractal::Analytics::InvalidEvent` outside production and is
|
|
72
|
+
logged and dropped in production. A `client_ip` never costs the event: one that is not an IPv4 or
|
|
73
|
+
IPv6 address is sent as given (the collector keeps the event and stores it without geo), and one
|
|
74
|
+
that is not valid UTF-8 is left out with a warning.
|
|
75
|
+
|
|
76
|
+
Delivery is **at least once**: a request whose response is lost is retried and stored twice.
|
|
77
|
+
`event_id` lets you tell duplicates apart; the collector does not deduplicate yet (#513).
|
|
78
|
+
|
|
79
|
+
### Who and which session
|
|
80
|
+
|
|
81
|
+
| | Default | Override |
|
|
82
|
+
| --- | --- | --- |
|
|
83
|
+
| Player | the scope's user (`Fractal.set_user`, or Warden), else `user_resolver`, else `"anonymous"` inside a request and `"system"` outside one (jobs, rake, boot) | `track(..., player_id:)`, `anonymous_player_id`, `system_player_id` |
|
|
84
|
+
| Session | a hash of the Rack session id (the raw id is never sent); with no session (jobs, cookie-less APIs), one session per player per UTC day | `track(..., session_token:)`, `Fractal.set_session(token)`, `session_resolver` |
|
|
85
|
+
| End-user address | the request's address (`request.remote_ip`, which honours your trusted proxies) | `track(..., client_ip:)`, `send_client_ip = false` |
|
|
86
|
+
|
|
87
|
+
With Warden (Devise), the gem sets the user as soon as the app authenticates, and a sign-in
|
|
88
|
+
tracks an `identify` event: it is recorded in the session that was current before sign-in, under
|
|
89
|
+
the user's player id, with `previous_player_id` in the payload. Warden gives the session a new id
|
|
90
|
+
at sign-in, so the logged-out and logged-in halves of a visit are two sessions, linked by that
|
|
91
|
+
event. Only a sign-in stored in the session counts: a stateless strategy (a token or HTTP auth,
|
|
92
|
+
which Warden runs with `store: false` on every request) sets the user but tracks no `identify`.
|
|
93
|
+
Turn it off with `identify_on_login = false`, or call `Fractal.identify(player_id)` yourself when
|
|
94
|
+
you do not use Warden.
|
|
95
|
+
|
|
96
|
+
The collector geolocates `client_ip` only when the project has IP tracking on, and stores the
|
|
97
|
+
address hashed. Your app owns consent for its users: this gem has no consent switch of its own.
|
|
98
|
+
|
|
99
|
+
### Request scope
|
|
100
|
+
|
|
101
|
+
`Fractal.set_user`, `set_tag`, `add_breadcrumb` and `set_session` apply to the current request or
|
|
102
|
+
job only. Tags and breadcrumbs go on errors; the user and session go on both errors and events.
|
|
103
|
+
|
|
104
|
+
## Configuration reference
|
|
105
|
+
|
|
106
|
+
See `lib/fractal/configuration.rb`. Highlights: `enabled` may be a callable checked on every call
|
|
107
|
+
(a runtime kill switch), `errors_enabled` / `analytics_enabled` turn one half off (it becomes a
|
|
108
|
+
set of no-ops), and `strict` controls whether invalid events raise.
|
|
109
|
+
|
|
110
|
+
## Not covered yet
|
|
111
|
+
|
|
112
|
+
- Sidekiq workers that do not go through Active Job: the `Rails.error` subscriber does not see
|
|
113
|
+
them. Decided as a documented gap for the first release (#497).
|
|
114
|
+
- Plain Rack apps without Rails. The first release supports Rails only (#497); the core runs
|
|
115
|
+
without Rails, but nothing installs the middleware or the subscriber for you.
|
|
116
|
+
- Per-event-type sampling.
|
|
117
|
+
|
|
118
|
+
## Development
|
|
119
|
+
|
|
120
|
+
```bash
|
|
121
|
+
sdks/ruby/ci/test.sh 3.2 rails_7_1 # the floor lane
|
|
122
|
+
sdks/ruby/ci/test.sh 3.4 rails_8_1 # what Fractal runs
|
|
123
|
+
sdks/ruby/ci/check_version_sync.sh
|
|
124
|
+
sdks/ruby/ci/conformance_capture.sh # writes tmp/conformance/ruby/
|
|
125
|
+
sdks/ruby/ci/run_e2e.sh # against a real collector, Postgres and ClickHouse
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
The fingerprint spec reads `collector/internal/ingest/fingerprint_vectors.json` from the repo,
|
|
129
|
+
so the specs run from a checkout of the Fractal repo.
|
|
130
|
+
|
|
131
|
+
## Releasing
|
|
132
|
+
|
|
133
|
+
Merging a `VERSION` bump to `main` is the release: `.github/workflows/ruby_sdk_tag.yml` tags
|
|
134
|
+
`ruby-sdk/vX.Y.Z`, builds the gem with `gem build --strict` and pushes it to rubygems.org. A
|
|
135
|
+
version rubygems.org already has is skipped. Without the `RUBYGEMS_API_KEY` secret the workflow
|
|
136
|
+
tags and skips the push with a warning; re-run it by hand (`workflow_dispatch`) once the secret is
|
|
137
|
+
set. The key needs the "push rubygem" scope, on an account whose MFA level is "UI and gem signin":
|
|
138
|
+
at "UI and API" every push asks for an OTP that CI cannot give.
|
|
139
|
+
|
|
140
|
+
**Before the first deploy that advertises the gem.** The Settings quick start and the help article
|
|
141
|
+
(`content/help/{en,es}/sdk-integration/ruby.md`) tell customers to add `gem "fractal-ruby"`, and
|
|
142
|
+
the name is not registered on rubygems.org until the first push. Push 0.1.0 by hand from the
|
|
143
|
+
owning account (or set the secret and dispatch the workflow) before running `bin/deploy`: until
|
|
144
|
+
then the install line fails and the name is free for anyone to claim. Shipping the copy ahead of
|
|
145
|
+
the push was an owner decision (2026-10-12), not an oversight.
|
|
146
|
+
|
|
147
|
+
`spec/fractal/package_spec.rb` builds the gem the same way and checks what is inside it.
|
|
148
|
+
|
|
149
|
+
## License
|
|
150
|
+
|
|
151
|
+
MIT. See `LICENSE`.
|
data/VERSION
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
0.1.0
|
|
@@ -0,0 +1,437 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require "resolv"
|
|
5
|
+
require "securerandom"
|
|
6
|
+
|
|
7
|
+
module Fractal
|
|
8
|
+
# Server-side `track`: buffers events in memory and delivers them to POST /v1/batch from its own
|
|
9
|
+
# background thread. Its buffer is separate from the error reporter's, so a burst of events can
|
|
10
|
+
# never evict an error report; the circuit breaker and the 401 revocation are shared.
|
|
11
|
+
#
|
|
12
|
+
# One request carries many players and sessions: each event names its own player_id,
|
|
13
|
+
# session_token and client_ip (#511, #512), and the envelope is marked server_side so the
|
|
14
|
+
# collector never geolocates the server's own address.
|
|
15
|
+
#
|
|
16
|
+
# Delivery is at-least-once. A request whose 202 is lost is retried and stored twice; event_id
|
|
17
|
+
# is set here so duplicates can be told apart (#513).
|
|
18
|
+
class Analytics
|
|
19
|
+
MAX_EVENT_TYPE_BYTES = 128
|
|
20
|
+
MAX_PLAYER_ID_BYTES = 256
|
|
21
|
+
MAX_PAYLOAD_BYTES = 65_536
|
|
22
|
+
MAX_BATCH = 1_000 # collector limit per request
|
|
23
|
+
BACKOFF_BASE = 1
|
|
24
|
+
BACKOFF_CAP = 60
|
|
25
|
+
|
|
26
|
+
InvalidEvent = Class.new(ArgumentError)
|
|
27
|
+
|
|
28
|
+
def initialize(config:, transport: Transport.new, breaker: Breaker.new, start_worker: true, clock: nil)
|
|
29
|
+
@config = config
|
|
30
|
+
@transport = transport
|
|
31
|
+
@breaker = breaker
|
|
32
|
+
@start_worker = start_worker
|
|
33
|
+
@clock = clock || -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }
|
|
34
|
+
@identity = Identity.new(config)
|
|
35
|
+
reset_state
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def enabled?
|
|
39
|
+
@config.analytics_enabled && @config.enabled? && !@breaker.revoked?
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# Records one event and returns at once; nothing is sent on the caller's thread.
|
|
43
|
+
#
|
|
44
|
+
# player_id, session_token and client_ip default to the current scope's (see Identity and
|
|
45
|
+
# Configuration#session_resolver). An event the collector would reject (empty or over-128-byte
|
|
46
|
+
# event_type, a non-Hash payload, a payload over 64 KB, a player_id over 256 bytes), that can
|
|
47
|
+
# never be serialised (an event_type, player_id or session_token that is not valid UTF-8) or
|
|
48
|
+
# whose timestamp is not a Time raises InvalidEvent in strict mode and is otherwise logged at
|
|
49
|
+
# error level and dropped. A client_ip never costs the event: one that is not an address is
|
|
50
|
+
# sent as given (the collector keeps the event and stores it without geo), and one that is not
|
|
51
|
+
# valid UTF-8 is left out with a warning.
|
|
52
|
+
def track(event_type, payload = nil, player_id: nil, session_token: nil, client_ip: nil, timestamp: nil)
|
|
53
|
+
reset_after_fork
|
|
54
|
+
return false unless enabled?
|
|
55
|
+
|
|
56
|
+
payload_json = validate!(event_type, payload)
|
|
57
|
+
enqueue(build(event_type.to_s, payload_json, player_id: player_id, session_token: session_token,
|
|
58
|
+
client_ip: client_ip, timestamp: timestamp))
|
|
59
|
+
true
|
|
60
|
+
rescue InvalidEvent => e
|
|
61
|
+
bump(:dropped_invalid)
|
|
62
|
+
raise if @config.strict?
|
|
63
|
+
|
|
64
|
+
log(:error, "event dropped: #{e.message}")
|
|
65
|
+
false
|
|
66
|
+
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
67
|
+
log(:error, "track failed: #{e.class}")
|
|
68
|
+
false
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# The logged-out and logged-in halves of a visit are two sessions (the session id changes at
|
|
72
|
+
# sign-in). This event links them: tracked in the session that was current before sign-in,
|
|
73
|
+
# under the player that just signed in.
|
|
74
|
+
def identify(player_id, previous_player_id:)
|
|
75
|
+
track("identify", { "previous_player_id" => previous_player_id }, player_id: player_id)
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# Delivers up to one batch. Never raises.
|
|
79
|
+
def flush(deadline: nil)
|
|
80
|
+
reset_after_fork
|
|
81
|
+
unless enabled?
|
|
82
|
+
discard_buffer
|
|
83
|
+
return
|
|
84
|
+
end
|
|
85
|
+
return if @breaker.open? || now < @retry_at
|
|
86
|
+
|
|
87
|
+
loop do
|
|
88
|
+
break if @breaker.open? || !enabled? || (deadline && now > deadline)
|
|
89
|
+
|
|
90
|
+
batch = take_batch
|
|
91
|
+
break if batch.empty?
|
|
92
|
+
|
|
93
|
+
break unless deliver(batch)
|
|
94
|
+
end
|
|
95
|
+
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
96
|
+
log(:error, "flush failed: #{e.class}")
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def shutdown(timeout: @config.shutdown_timeout)
|
|
100
|
+
return unless @pid == Process.pid
|
|
101
|
+
|
|
102
|
+
@stopping = true
|
|
103
|
+
@mutex.synchronize { @wake.broadcast }
|
|
104
|
+
@thread&.join(0.5)
|
|
105
|
+
@retry_at = 0
|
|
106
|
+
flush(deadline: now + timeout)
|
|
107
|
+
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
108
|
+
log(:error, "shutdown failed: #{e.class}")
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
def stats
|
|
112
|
+
@mutex.synchronize { @stats.dup }
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
def pending
|
|
116
|
+
@mutex.synchronize { @queue.size }
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
private
|
|
120
|
+
|
|
121
|
+
def reset_state
|
|
122
|
+
@pid = Process.pid
|
|
123
|
+
@mutex = Mutex.new
|
|
124
|
+
@wake = ConditionVariable.new
|
|
125
|
+
@queue = []
|
|
126
|
+
@queue_bytes = 0
|
|
127
|
+
@seq = 0
|
|
128
|
+
@stats = Hash.new(0)
|
|
129
|
+
@thread = nil
|
|
130
|
+
@stopping = false
|
|
131
|
+
@attempts = 0
|
|
132
|
+
@retry_at = 0
|
|
133
|
+
@flush_requested = false
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
# A forked child inherits the parent's queue; sending it again would duplicate every event.
|
|
137
|
+
def reset_after_fork
|
|
138
|
+
reset_state unless @pid == Process.pid
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
def now = @clock.call
|
|
142
|
+
|
|
143
|
+
def validate!(event_type, payload)
|
|
144
|
+
unless event_type.is_a?(String) || event_type.is_a?(Symbol)
|
|
145
|
+
raise InvalidEvent, "event_type must be a String, got #{event_type.class}"
|
|
146
|
+
end
|
|
147
|
+
name = event_type.to_s
|
|
148
|
+
if name.empty? || name.bytesize > MAX_EVENT_TYPE_BYTES
|
|
149
|
+
raise InvalidEvent, "event_type must be 1 to #{MAX_EVENT_TYPE_BYTES} bytes, got #{name.bytesize}"
|
|
150
|
+
end
|
|
151
|
+
return nil if payload.nil? || (payload.is_a?(Hash) && payload.empty?)
|
|
152
|
+
raise InvalidEvent, "payload for #{name.inspect} must be a Hash, got #{payload.class}" unless payload.is_a?(Hash)
|
|
153
|
+
|
|
154
|
+
json = JSON.generate(payload)
|
|
155
|
+
if json.bytesize > MAX_PAYLOAD_BYTES
|
|
156
|
+
raise InvalidEvent, "payload for #{name.inspect} is #{json.bytesize} bytes, over the #{MAX_PAYLOAD_BYTES}-byte limit"
|
|
157
|
+
end
|
|
158
|
+
json
|
|
159
|
+
rescue JSON::JSONError, EncodingError => e # JSONError: also NestingError (circular or over 100 deep)
|
|
160
|
+
raise InvalidEvent, "payload for #{name.inspect} cannot be serialised as JSON (#{e.class})"
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
def build(event_type, payload_json, player_id:, session_token:, client_ip:, timestamp:)
|
|
164
|
+
unless timestamp.nil? || timestamp.respond_to?(:getutc)
|
|
165
|
+
raise InvalidEvent, "timestamp must be a Time, got #{timestamp.class}"
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
scope = Scope.current
|
|
169
|
+
context = ExecutionContext.current
|
|
170
|
+
player = player_id.to_s.empty? ? @identity.player_id(scope: scope, controller: context[:controller]) : player_id.to_s
|
|
171
|
+
event = {
|
|
172
|
+
"event_id" => SecureRandom.uuid,
|
|
173
|
+
"event_type" => event_type,
|
|
174
|
+
"event_timestamp" => (timestamp&.getutc || Time.now.utc).iso8601(3),
|
|
175
|
+
"player_id" => player,
|
|
176
|
+
"session_token" => session_token.to_s.empty? ? session_for(scope, player) : session_token.to_s
|
|
177
|
+
}
|
|
178
|
+
ip = client_ip.to_s.empty? ? client_ip_for(scope) : client_ip.to_s
|
|
179
|
+
ip = serialisable_ip(ip) if ip
|
|
180
|
+
event["client_ip"] = ip if ip
|
|
181
|
+
reject_invalid_identity!(event)
|
|
182
|
+
event["payload"] = JSON.parse(payload_json) if payload_json
|
|
183
|
+
[ event, payload_json ? payload_json.bytesize : 0 ]
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
# These strings come from the caller, the scope or a resolver. One that is not valid UTF-8 can
|
|
187
|
+
# never be serialised, and the collector rejects an over-long player_id, so each is refused
|
|
188
|
+
# here like a bad payload, not queued. client_ip never gets here unserialisable (see
|
|
189
|
+
# serialisable_ip): the collector never rejects an event over it (ANALYTICS_PROTOCOL.md), so
|
|
190
|
+
# neither does the gem.
|
|
191
|
+
def reject_invalid_identity!(event)
|
|
192
|
+
event.each do |field, value|
|
|
193
|
+
JSON.generate(value)
|
|
194
|
+
rescue JSON::JSONError, EncodingError
|
|
195
|
+
raise InvalidEvent, "#{field} is not valid UTF-8 and cannot be serialised as JSON"
|
|
196
|
+
end
|
|
197
|
+
return if event["player_id"].bytesize <= MAX_PLAYER_ID_BYTES
|
|
198
|
+
|
|
199
|
+
raise InvalidEvent, "player_id must be at most #{MAX_PLAYER_ID_BYTES} bytes, got #{event['player_id'].bytesize}"
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
# An SDK MUST NOT drop an event over its client_ip (ANALYTICS_PROTOCOL.md). One that is not
|
|
203
|
+
# valid UTF-8 can never be serialised, so the address is what is left out, not the event.
|
|
204
|
+
def serialisable_ip(ip)
|
|
205
|
+
JSON.generate(ip)
|
|
206
|
+
ip
|
|
207
|
+
rescue JSON::JSONError, EncodingError
|
|
208
|
+
log(:warn, "client_ip is not valid UTF-8 and was left out; the event is kept without geo")
|
|
209
|
+
nil
|
|
210
|
+
end
|
|
211
|
+
|
|
212
|
+
# Only decides whether the request's own address is worth sending (see remote_ip). It does not
|
|
213
|
+
# have to agree with the collector's parser: a miss costs one event its geo, never the event.
|
|
214
|
+
def address?(ip)
|
|
215
|
+
return false if ip.include?("%")
|
|
216
|
+
|
|
217
|
+
ip.match?(Resolv::IPv4::Regex) || ip.match?(Resolv::IPv6::Regex)
|
|
218
|
+
end
|
|
219
|
+
|
|
220
|
+
# The scope's explicit session, else the configured resolver over the request, else one
|
|
221
|
+
# session per player per UTC day: a job or a cookie-less API call has no session of its own,
|
|
222
|
+
# and an empty token would make the collector invent a new session every second.
|
|
223
|
+
def session_for(scope, player)
|
|
224
|
+
return scope.session_token.to_s unless scope.session_token.to_s.empty?
|
|
225
|
+
|
|
226
|
+
resolved = resolve_session(scope)
|
|
227
|
+
return resolved unless resolved.to_s.empty?
|
|
228
|
+
|
|
229
|
+
"day_#{Time.now.utc.strftime('%Y%m%d')}_#{player}"
|
|
230
|
+
end
|
|
231
|
+
|
|
232
|
+
def resolve_session(scope)
|
|
233
|
+
return nil unless scope.env && @config.session_resolver
|
|
234
|
+
|
|
235
|
+
@config.session_resolver.call(scope.env)&.to_s
|
|
236
|
+
rescue StandardError => e
|
|
237
|
+
log(:error, "session_resolver raised #{e.class}; using the daily session")
|
|
238
|
+
nil
|
|
239
|
+
end
|
|
240
|
+
|
|
241
|
+
def client_ip_for(scope)
|
|
242
|
+
return scope.client_ip unless scope.client_ip.to_s.empty?
|
|
243
|
+
return nil unless @config.send_client_ip && scope.env
|
|
244
|
+
|
|
245
|
+
scope.client_ip = remote_ip(scope.env)
|
|
246
|
+
rescue StandardError => e
|
|
247
|
+
log(:warn, "could not read the client address: #{e.class}")
|
|
248
|
+
nil
|
|
249
|
+
end
|
|
250
|
+
|
|
251
|
+
# ActionDispatch's remote_ip honours the app's trusted proxies; plain Rack's ip is the fallback.
|
|
252
|
+
# A request address that is not one (a proxy's "unknown") is left out: the event matters more.
|
|
253
|
+
def remote_ip(env)
|
|
254
|
+
ip = if defined?(::ActionDispatch::Request)
|
|
255
|
+
::ActionDispatch::Request.new(env).remote_ip
|
|
256
|
+
elsif defined?(::Rack::Request)
|
|
257
|
+
::Rack::Request.new(env).ip
|
|
258
|
+
else
|
|
259
|
+
env["REMOTE_ADDR"]
|
|
260
|
+
end
|
|
261
|
+
address?(ip.to_s) ? ip.to_s : nil
|
|
262
|
+
end
|
|
263
|
+
|
|
264
|
+
def enqueue((event, bytes))
|
|
265
|
+
@mutex.synchronize do
|
|
266
|
+
@seq += 1
|
|
267
|
+
event["client_seq"] = @seq
|
|
268
|
+
@queue << [ event, bytes ]
|
|
269
|
+
@queue_bytes += bytes
|
|
270
|
+
while @queue.size > @config.analytics_max_queue || @queue_bytes > @config.analytics_max_queue_bytes
|
|
271
|
+
_, dropped_bytes = @queue.shift
|
|
272
|
+
@queue_bytes -= dropped_bytes
|
|
273
|
+
@stats[:dropped_queue_full] += 1
|
|
274
|
+
end
|
|
275
|
+
if @queue.size >= @config.analytics_flush_at
|
|
276
|
+
@flush_requested = true
|
|
277
|
+
@wake.signal
|
|
278
|
+
end
|
|
279
|
+
end
|
|
280
|
+
start_worker if @start_worker
|
|
281
|
+
end
|
|
282
|
+
|
|
283
|
+
def take_batch
|
|
284
|
+
@mutex.synchronize do
|
|
285
|
+
batch = @queue.shift(MAX_BATCH)
|
|
286
|
+
@queue_bytes -= batch.sum { |_, bytes| bytes }
|
|
287
|
+
batch
|
|
288
|
+
end
|
|
289
|
+
end
|
|
290
|
+
|
|
291
|
+
# Puts a batch back at the front for a retry, keeping the queue within its caps by dropping
|
|
292
|
+
# the oldest events.
|
|
293
|
+
def restore(batch)
|
|
294
|
+
@mutex.synchronize do
|
|
295
|
+
@queue.unshift(*batch)
|
|
296
|
+
@queue_bytes += batch.sum { |_, bytes| bytes }
|
|
297
|
+
while @queue.size > @config.analytics_max_queue || @queue_bytes > @config.analytics_max_queue_bytes
|
|
298
|
+
_, dropped_bytes = @queue.shift
|
|
299
|
+
@queue_bytes -= dropped_bytes
|
|
300
|
+
@stats[:dropped_queue_full] += 1
|
|
301
|
+
end
|
|
302
|
+
end
|
|
303
|
+
end
|
|
304
|
+
|
|
305
|
+
# Returns true when the batch is settled (delivered or dropped for good), false when it was
|
|
306
|
+
# put back for a retry.
|
|
307
|
+
def deliver(batch)
|
|
308
|
+
body = serialise(batch)
|
|
309
|
+
return true unless body
|
|
310
|
+
|
|
311
|
+
response = @transport.post(@config.endpoint("/v1/batch"), @config.api_key, body)
|
|
312
|
+
handle_response(response, batch)
|
|
313
|
+
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
314
|
+
retry_later(batch, "delivery failed: #{e.class}")
|
|
315
|
+
end
|
|
316
|
+
|
|
317
|
+
# A batch that cannot be serialised now never will be. It is dropped, not retried: a retry
|
|
318
|
+
# would hold every later event behind it and open the breaker the error reporter shares.
|
|
319
|
+
def serialise(batch)
|
|
320
|
+
envelope(batch.map(&:first)).to_json
|
|
321
|
+
rescue StandardError => e
|
|
322
|
+
bump(:dropped_invalid, batch.size)
|
|
323
|
+
log(:error, "batch of #{batch.size} event(s) dropped: cannot be serialised (#{e.class})")
|
|
324
|
+
nil
|
|
325
|
+
end
|
|
326
|
+
|
|
327
|
+
def handle_response(response, batch)
|
|
328
|
+
case response.code
|
|
329
|
+
when 200..299
|
|
330
|
+
succeeded(batch, response.body)
|
|
331
|
+
true
|
|
332
|
+
when 401
|
|
333
|
+
@breaker.revoke!
|
|
334
|
+
discard_buffer
|
|
335
|
+
log(:error, "collector answered 401: the API key is unknown or inactive. Fractal stops sending for the rest of this process")
|
|
336
|
+
true
|
|
337
|
+
when 429, 500..599
|
|
338
|
+
retry_later(batch, "collector answered #{response.code}")
|
|
339
|
+
else
|
|
340
|
+
@breaker.failure!
|
|
341
|
+
bump(:dropped_rejected, batch.size)
|
|
342
|
+
log(:error, "collector answered #{response.code}; batch of #{batch.size} event(s) dropped")
|
|
343
|
+
true
|
|
344
|
+
end
|
|
345
|
+
end
|
|
346
|
+
|
|
347
|
+
def succeeded(batch, body)
|
|
348
|
+
@breaker.success!
|
|
349
|
+
@attempts = 0
|
|
350
|
+
@retry_at = 0
|
|
351
|
+
json = parse(body)
|
|
352
|
+
if json.is_a?(Hash) && json["dropped"] == true
|
|
353
|
+
bump(:dropped_over_cap, batch.size)
|
|
354
|
+
log(:warn, "collector dropped #{batch.size} event(s): the organization is over its plan's event cap")
|
|
355
|
+
return
|
|
356
|
+
end
|
|
357
|
+
rejected = Rejections.parse(body, batch.size)
|
|
358
|
+
bump(:sent_events, batch.size - rejected.size)
|
|
359
|
+
return if rejected.empty?
|
|
360
|
+
|
|
361
|
+
bump(:rejected_events, rejected.size)
|
|
362
|
+
rejected.each do |r|
|
|
363
|
+
log(:error, "collector rejected event #{batch[r['index']].first['event_type'].inspect}: #{r['reason']}")
|
|
364
|
+
end
|
|
365
|
+
end
|
|
366
|
+
|
|
367
|
+
# Exponential backoff with jitter, capped, and never giving up on the events: they stay queued
|
|
368
|
+
# (within the queue caps) for the next attempt.
|
|
369
|
+
def retry_later(batch, reason)
|
|
370
|
+
@breaker.failure!
|
|
371
|
+
bump(:failed_posts)
|
|
372
|
+
restore(batch)
|
|
373
|
+
delay = [ BACKOFF_BASE * (2**@attempts), BACKOFF_CAP ].min
|
|
374
|
+
@attempts += 1
|
|
375
|
+
@retry_at = now + delay * (1 + (rand * 0.1))
|
|
376
|
+
log(:warn, "#{reason}; #{batch.size} event(s) kept, retrying in #{delay.round}s")
|
|
377
|
+
false
|
|
378
|
+
end
|
|
379
|
+
|
|
380
|
+
def parse(body)
|
|
381
|
+
JSON.parse(body.to_s)
|
|
382
|
+
rescue JSON::ParserError
|
|
383
|
+
nil
|
|
384
|
+
end
|
|
385
|
+
|
|
386
|
+
def envelope(events)
|
|
387
|
+
{
|
|
388
|
+
sent_at: Time.now.utc.iso8601(3),
|
|
389
|
+
context: {
|
|
390
|
+
player_id: @config.system_player_id, session_token: "", platform: "server",
|
|
391
|
+
app_version: @config.app_version.to_s, environment: @config.environment,
|
|
392
|
+
sdk_version: Fractal.sdk_version, server_side: true
|
|
393
|
+
},
|
|
394
|
+
events: events
|
|
395
|
+
}
|
|
396
|
+
end
|
|
397
|
+
|
|
398
|
+
def discard_buffer
|
|
399
|
+
@mutex.synchronize do
|
|
400
|
+
@queue.clear
|
|
401
|
+
@queue_bytes = 0
|
|
402
|
+
end
|
|
403
|
+
end
|
|
404
|
+
|
|
405
|
+
def start_worker
|
|
406
|
+
return if @thread&.alive?
|
|
407
|
+
|
|
408
|
+
@mutex.synchronize do
|
|
409
|
+
return if @thread&.alive?
|
|
410
|
+
|
|
411
|
+
@thread = Thread.new { work }
|
|
412
|
+
@thread.report_on_exception = false
|
|
413
|
+
end
|
|
414
|
+
end
|
|
415
|
+
|
|
416
|
+
def work
|
|
417
|
+
until @stopping
|
|
418
|
+
# The flag, not just the signal: a signal sent before this thread first waits is lost.
|
|
419
|
+
@mutex.synchronize do
|
|
420
|
+
@wake.wait(@mutex, @config.analytics_flush_interval) unless @flush_requested
|
|
421
|
+
@flush_requested = false
|
|
422
|
+
end
|
|
423
|
+
flush unless @stopping
|
|
424
|
+
end
|
|
425
|
+
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
426
|
+
log(:error, "analytics worker died: #{e.class}")
|
|
427
|
+
end
|
|
428
|
+
|
|
429
|
+
def bump(stat, by = 1)
|
|
430
|
+
@mutex.synchronize { @stats[stat] += by }
|
|
431
|
+
end
|
|
432
|
+
|
|
433
|
+
def log(level, message)
|
|
434
|
+
@config.logger&.public_send(level, "[fractal] #{message}")
|
|
435
|
+
end
|
|
436
|
+
end
|
|
437
|
+
end
|