pug-sh 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/README.md +247 -0
- data/lib/pug/batcher.rb +384 -0
- data/lib/pug/client.rb +248 -0
- data/lib/pug/connect.rb +291 -0
- data/lib/pug/countries.rb +19 -0
- data/lib/pug/errors.rb +16 -0
- data/lib/pug/event_builder.rb +522 -0
- data/lib/pug/gen/buf/validate/validate_pb.rb +59 -0
- data/lib/pug/gen/common/events/v1/api_events_pb.rb +26 -0
- data/lib/pug/gen/common/events/v1/app_events_pb.rb +33 -0
- data/lib/pug/gen/common/events/v1/auth_events_pb.rb +32 -0
- data/lib/pug/gen/common/events/v1/billing_events_pb.rb +41 -0
- data/lib/pug/gen/common/events/v1/chat_events_pb.rb +56 -0
- data/lib/pug/gen/common/events/v1/commerce_events_pb.rb +38 -0
- data/lib/pug/gen/common/events/v1/discovery_events_pb.rb +31 -0
- data/lib/pug/gen/common/events/v1/error_events_pb.rb +26 -0
- data/lib/pug/gen/common/events/v1/file_events_pb.rb +29 -0
- data/lib/pug/gen/common/events/v1/form_events_pb.rb +27 -0
- data/lib/pug/gen/common/events/v1/integration_events_pb.rb +27 -0
- data/lib/pug/gen/common/events/v1/invitation_events_pb.rb +26 -0
- data/lib/pug/gen/common/events/v1/media_events_pb.rb +36 -0
- data/lib/pug/gen/common/events/v1/navigation_events_pb.rb +31 -0
- data/lib/pug/gen/common/events/v1/notification_events_pb.rb +28 -0
- data/lib/pug/gen/common/events/v1/options_pb.rb +25 -0
- data/lib/pug/gen/common/events/v1/social_events_pb.rb +25 -0
- data/lib/pug/gen/common/events/v1/support_events_pb.rb +33 -0
- data/lib/pug/gen/common/events/v1/workspace_events_pb.rb +30 -0
- data/lib/pug/gen/common/v1/filter_schema_pb.rb +29 -0
- data/lib/pug/gen/common/v1/filters_pb.rb +27 -0
- data/lib/pug/gen/common/v1/property_value_pb.rb +24 -0
- data/lib/pug/gen/common/v1/time_pb.rb +25 -0
- data/lib/pug/gen/sdk/devices/v1/devices_pb.rb +35 -0
- data/lib/pug/gen/sdk/events/v1/events_pb.rb +30 -0
- data/lib/pug/gen/sdk/profiles/v1/profiles_pb.rb +28 -0
- data/lib/pug/gen/shared/activity/v1/activity_pb.rb +46 -0
- data/lib/pug/gen/shared/insights/v1/insights_pb.rb +69 -0
- data/lib/pug/gen/shared/profiles/v1/profiles_pb.rb +43 -0
- data/lib/pug/log.rb +35 -0
- data/lib/pug/proto.rb +22 -0
- data/lib/pug/reads.rb +118 -0
- data/lib/pug/rpc.rb +73 -0
- data/lib/pug/validator.rb +475 -0
- data/lib/pug/version.rb +5 -0
- data/lib/pug/well_known.rb +33 -0
- data/lib/pug-sh.rb +3 -0
- data/lib/pug.rb +5 -0
- metadata +122 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: 893420d16827c4d7472d599fde6fc5278e354cc559ffcd217b5ff34151c0d5c5
|
|
4
|
+
data.tar.gz: d9f76d529e9e0fda1fc678a6096cc9922076e84e0a7c53515e5c6ec98eee4eed
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: 9194a40e3a2eb238d5dce9f1db3803ca4fd103a0ae366ef27d8dfd266f93fff923fba12afbbce82796af440175a0563bc39495b4e12e007fe32bb7fbb3fb3540
|
|
7
|
+
data.tar.gz: 4db8dc628a8f10582f53eae2242bd6e35c8e1511669cf643e81b1c095615353fcbbc58ca362e058cbbe72bd81eb4f35b886d34cb88c0c8209688f4764b6a7d7f
|
data/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Pug
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
data/README.md
ADDED
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
<p align="center">
|
|
2
|
+
<img src="https://app.pug.sh/logo.svg" alt="Pug" width="120" height="120" />
|
|
3
|
+
</p>
|
|
4
|
+
|
|
5
|
+
# pug-sh
|
|
6
|
+
|
|
7
|
+
Pug server-side SDK for Ruby — event tracking, profile identify, and analytics reads.
|
|
8
|
+
|
|
9
|
+
It is instance-based and authenticates with a **private** project API key (`prv_…`). Ingestion
|
|
10
|
+
never raises, so a tracking call can't take down your request path.
|
|
11
|
+
|
|
12
|
+
## Install
|
|
13
|
+
|
|
14
|
+
```ruby
|
|
15
|
+
# Gemfile
|
|
16
|
+
gem "pug-sh"
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
Requires Ruby 3.3 or newer.
|
|
20
|
+
|
|
21
|
+
## Quick start
|
|
22
|
+
|
|
23
|
+
```ruby
|
|
24
|
+
require "pug"
|
|
25
|
+
|
|
26
|
+
pug = Pug::Client.new(api_key: ENV.fetch("PUG_API_KEY"))
|
|
27
|
+
|
|
28
|
+
# Track an event. distinct_id (who the event is for) comes first — a server has no ambient user.
|
|
29
|
+
pug.track("user_42", "order.completed", amount: 49.0, currency: "USD")
|
|
30
|
+
|
|
31
|
+
# Well-known events get their properties type-checked and validated against the event's schema.
|
|
32
|
+
pug.track("user_42", "feature_used", feature_id: "export")
|
|
33
|
+
|
|
34
|
+
# Identify a profile (never raises — failures are logged).
|
|
35
|
+
pug.identify("user_42", email: "ada@example.com", plan: "pro")
|
|
36
|
+
|
|
37
|
+
# Reads raise Pug::Error.
|
|
38
|
+
profile = pug.profiles.get_by_external_id("user_42")
|
|
39
|
+
pug.profiles.list.each { |p| puts p.external_id }
|
|
40
|
+
trend = pug.insights.query(
|
|
41
|
+
spec: { insight_type: :INSIGHT_TYPE_TRENDS, events: [{ event: { kind: "order.completed" } }] },
|
|
42
|
+
time_range: { from: Time.now - (30 * 86_400), to: Time.now },
|
|
43
|
+
granularity: :GRANULARITY_DAY,
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
# Drain on shutdown so no buffered events are lost.
|
|
47
|
+
pug.close
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
## Behaviour
|
|
51
|
+
|
|
52
|
+
- **Ingestion never raises.** `track` and `identify` validate input client-side and log and drop
|
|
53
|
+
anything invalid. `track` is non-blocking: a background thread sends events in batches, by size
|
|
54
|
+
(`max_size`) or interval (`max_wait`).
|
|
55
|
+
- **Delivery is best-effort with retry.** A transient send failure keeps events queued and retries
|
|
56
|
+
`max_wait` after the failure, but no sooner than a second; a permanent one (a fixed set of
|
|
57
|
+
client-error Connect codes, plus any non-transport error) dead-letters them via `on_error`.
|
|
58
|
+
`close` drains and reports anything still undelivered through `on_error` and the log.
|
|
59
|
+
`on_error` covers buffered `track` events only — `identify` failures are logged, and so are
|
|
60
|
+
events the server drops from a batch it accepts (`BatchCreateResponse.dropped`), at WARN.
|
|
61
|
+
- **Reads are request/response and do raise** `Pug::Error` — they run in your control flow, so
|
|
62
|
+
you own retry. They are not auto-retried.
|
|
63
|
+
- **Well-known events** have typed, validated properties; any other `kind` is accepted as a
|
|
64
|
+
custom event with loose properties.
|
|
65
|
+
- **Thread- and fork-safe.** One client can be shared by every thread. A client created before a
|
|
66
|
+
fork (Puma, Unicorn, Sidekiq) starts empty in each child, with its own sender thread and
|
|
67
|
+
connections; events buffered in the parent stay with the parent.
|
|
68
|
+
|
|
69
|
+
## API
|
|
70
|
+
|
|
71
|
+
### `Pug::Client.new(**options)`
|
|
72
|
+
|
|
73
|
+
| Option | Type | Default | Description |
|
|
74
|
+
| ---------- | ------------- | --------------------------- | --------------------------------------------------------------------------------------------------------------------------------- |
|
|
75
|
+
| `api_key` | `String` | — | **Required.** Private project key (`prv_…`). |
|
|
76
|
+
| `endpoint` | `String` | hosted endpoint | Pug server origin. |
|
|
77
|
+
| `batch` | `Hash` | see below | Ingestion buffer: `max_size` (100, capped at 1000), `max_wait` (5 seconds), `max_queue_size` (10,000). |
|
|
78
|
+
| `on_error` | callable | none | Dead-letter hook for undeliverable buffered `track` events, called with `(error, events)`. |
|
|
79
|
+
| `logger` | `Logger`-like | `Logger.new($stderr)`, WARN | Gets the SDK's warnings and errors: anything with public `warn` and `error`, such as `Rails.logger` (wrap an IO in `Logger.new`). |
|
|
80
|
+
| `timeout` | `Numeric` | 10 | Request timeout, in seconds. |
|
|
81
|
+
|
|
82
|
+
```ruby
|
|
83
|
+
Pug::Client.new(
|
|
84
|
+
api_key: ENV.fetch("PUG_API_KEY"),
|
|
85
|
+
batch: { max_size: 100, max_wait: 5, max_queue_size: 10_000 },
|
|
86
|
+
on_error: lambda do |error, events|
|
|
87
|
+
# dead-letter sink: error is a Pug::QueueOverflowError, a Pug::ClosedError, a Pug::Error
|
|
88
|
+
# with a permanent Connect code, the transient Pug::Error that failed close's last attempt,
|
|
89
|
+
# or the exception a bug raised
|
|
90
|
+
end,
|
|
91
|
+
)
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
`on_error` runs on the background sender thread; on the thread calling `close`, for what it
|
|
95
|
+
reports undelivered; or, when the queue overflows, on the thread that called `track`. It must be
|
|
96
|
+
thread-safe and return promptly. Called from inside it on the sender or the closing thread, `flush`
|
|
97
|
+
and `close` don't wait for what is waiting on `on_error`: they return `false` at once, except that
|
|
98
|
+
`flush` returns `true` while `close` reports. On the overflow path nothing waits on `on_error`, so
|
|
99
|
+
they behave as anywhere else, blocking the thread that called `track`.
|
|
100
|
+
|
|
101
|
+
### Ingestion (never raises)
|
|
102
|
+
|
|
103
|
+
- `pug.track(distinct_id, kind, properties = nil, options = nil)` — enqueue an event. Non-blocking.
|
|
104
|
+
- `pug.identify(external_id, traits = nil, options = nil)` — create or update a profile. Blocks
|
|
105
|
+
until the server answers.
|
|
106
|
+
- `pug.flush(timeout: nil)` — send the currently buffered events now and wait for that attempt.
|
|
107
|
+
Returns `false` if `timeout` (seconds) passed first. `true` means the attempt finished, not that
|
|
108
|
+
every event was delivered: a transient failure keeps them queued for a retry.
|
|
109
|
+
- `pug.close(timeout: nil)` — drain and shut down; call on graceful exit so nothing buffered is lost.
|
|
110
|
+
Returns `false` if `timeout` passed first. Either way, by the time it returns every event it
|
|
111
|
+
couldn't send has gone to `on_error` and the log, including a batch still being sent when the
|
|
112
|
+
timeout ran out, which may yet arrive.
|
|
113
|
+
|
|
114
|
+
A `timeout` that isn't a number waits without one, with a warning.
|
|
115
|
+
|
|
116
|
+
`distinct_id` and `external_id` are non-empty Strings, or Integers such as a record's id, sent as
|
|
117
|
+
their decimal string; `session_id`, `anonymous_id` and `device_id` must be Strings. Properties go
|
|
118
|
+
in a Hash, braces optional, and options follow as keywords. Without braces, a key named after an
|
|
119
|
+
option sets that option, so a property with an option's name needs the braces:
|
|
120
|
+
|
|
121
|
+
```ruby
|
|
122
|
+
pug.track("user_42", "signup", plan: "pro")
|
|
123
|
+
pug.track("user_42", "signup", plan: "pro", timestamp: Time.now - 3600)
|
|
124
|
+
pug.track("user_42", "signup", { plan: "pro", location: "landing" }, location: { country: "DE" })
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
`track` options: `timestamp` (a `Time`; defaults to now), `session_id` (a UUID overriding the
|
|
128
|
+
client's per-instance session), `location` (the visitor's location — see below).
|
|
129
|
+
`identify` options, which work the same way with traits: `anonymous_id` (must start with `anon-`;
|
|
130
|
+
triggers anon→identified merge), `device_id`. Traits must be JSON-serializable, with nested values
|
|
131
|
+
mapped as for properties below; they travel as a protobuf `Struct`, whose numbers are doubles, so
|
|
132
|
+
an integer past 2^53 is refused.
|
|
133
|
+
|
|
134
|
+
Custom properties map by Ruby type:
|
|
135
|
+
|
|
136
|
+
| Ruby value | Sent as |
|
|
137
|
+
| ------------------------------------------------ | ----------------------------------------- |
|
|
138
|
+
| `String`, `Symbol` | string |
|
|
139
|
+
| `true`, `false` | bool |
|
|
140
|
+
| `Integer` | int (outside the int64 range: dropped) |
|
|
141
|
+
| `Float`, `BigDecimal`, `Rational` | double (NaN and ±Infinity: dropped) |
|
|
142
|
+
| `Time`, `DateTime`, `Google::Protobuf::Timestamp`| timestamp |
|
|
143
|
+
| `Date` | ISO 8601 date string, e.g. `"2026-10-10"` |
|
|
144
|
+
| `Google::Protobuf::Duration` | duration string, e.g. `"1.5s"` |
|
|
145
|
+
| `Hash`, `Array` | JSON string |
|
|
146
|
+
| `nil` | omitted |
|
|
147
|
+
|
|
148
|
+
Anything else is dropped with a warning. Inside a `Hash` or `Array`, a `Time`, `DateTime` or `Date`
|
|
149
|
+
becomes an ISO 8601 string and a `BigDecimal` or `Rational` a number, while a value JSON has no
|
|
150
|
+
form for, such as a `Set`, drops the whole property with a warning. Strings are capped at 1024
|
|
151
|
+
bytes (truncated with a warning), and invalid UTF-8 is repaired.
|
|
152
|
+
|
|
153
|
+
### Well-known events
|
|
154
|
+
|
|
155
|
+
For a well-known kind such as `purchase`, properties are checked against the event's schema.
|
|
156
|
+
Keys may be the field's name (`product_id`) or its JSON name (`productId`); either way the
|
|
157
|
+
property is sent under the JSON name, like every other Pug SDK. A field's type comes from the
|
|
158
|
+
schema rather than the Ruby value, so `amount: 10` is sent as a double. A value of the wrong type,
|
|
159
|
+
or one failing the schema's rules, drops the event with an error; other keys ride along as custom
|
|
160
|
+
properties.
|
|
161
|
+
|
|
162
|
+
```ruby
|
|
163
|
+
pug.track("user_42", "purchase", product_id: "p1", amount: 49, currency: "USD", coupon: "FALL")
|
|
164
|
+
pug.track("user_42", "video_play", video_id: "v1", position: 92.5) # a Duration field, in seconds
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
Duration fields take seconds as a number, `{ seconds:, nanos: }`, or a `Google::Protobuf::Duration`.
|
|
168
|
+
|
|
169
|
+
### Location
|
|
170
|
+
|
|
171
|
+
Pug only derives geo from CDN headers on browser (public-key) requests. This SDK uses a private
|
|
172
|
+
key, so the server adds no geo of its own — without `location`, an event has no geo at all:
|
|
173
|
+
|
|
174
|
+
```ruby
|
|
175
|
+
pug.track("user_42", "checkout_completed", { plan: "pro" }, location: { country: "DE", city: "Berlin" })
|
|
176
|
+
```
|
|
177
|
+
|
|
178
|
+
Fields: `continent`, `country`, `region`, `city`, `postal_code`, `metro_code`, `timezone`,
|
|
179
|
+
`latitude`, `longitude` — strings, except the two coordinates. `country` must be an ISO 3166-1
|
|
180
|
+
alpha-2 code (case and surrounding spaces are normalized); `latitude` and `longitude` are only
|
|
181
|
+
sent as a pair.
|
|
182
|
+
|
|
183
|
+
A field that is missing, nil or empty is omitted silently. One that looks like a mistake —
|
|
184
|
+
unknown key, wrong type, unknown country, coordinate out of range — is dropped with a warning and
|
|
185
|
+
the rest is still sent. If nothing is usable the event goes out with no geo.
|
|
186
|
+
|
|
187
|
+
### Reads (raise `Pug::Error`)
|
|
188
|
+
|
|
189
|
+
These run in your control flow:
|
|
190
|
+
|
|
191
|
+
- `pug.profiles` — `get(id, include_bots: false)`, `get_by_external_id(external_id, include_bots: false)`,
|
|
192
|
+
`delete(id)`, `list(request = nil, timeout: nil)` (auto-paginating: yields each profile to a
|
|
193
|
+
block, or returns an `Enumerator`). The timeout bounds the whole stream, the time your block
|
|
194
|
+
takes included, so `timeout:` (seconds) can give one long `list` more than the client's.
|
|
195
|
+
- `pug.activity` — `feed`, `event_explorer`, `heatmap`, `profile_sessions`, `profile_stats`,
|
|
196
|
+
`filter_schema`, `property_values`.
|
|
197
|
+
- `pug.insights` — `query`, `segment_users`, `filter_schema`, `property_values`.
|
|
198
|
+
|
|
199
|
+
Requests are a Hash or the generated request message, and responses are the generated messages,
|
|
200
|
+
under `Pug::Proto` (`Pug::Proto::Shared::Insights::V1::QueryResponse`, …); call `to_h` on one
|
|
201
|
+
for plain Ruby data.
|
|
202
|
+
|
|
203
|
+
```ruby
|
|
204
|
+
sessions = pug.activity.profile_sessions(distinct_id: "user_42", page_size: 50)
|
|
205
|
+
sessions.sessions.each { |s| puts s.session_id }
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
Reads leave out traffic tagged as a bot at ingest unless you ask for it: pass
|
|
209
|
+
`include_bots: true` to `profiles.get` and `get_by_external_id`, set it on the request of
|
|
210
|
+
`profiles.list` or of an `activity` read other than `filter_schema` and `property_values`, or set
|
|
211
|
+
it on the `spec` of `insights.query`. The other requests have no such field.
|
|
212
|
+
|
|
213
|
+
`Pug::Error#code` is the Connect code as a Symbol (`:not_found`, `:unavailable`, …), or `nil`
|
|
214
|
+
when the failure didn't come from the RPC layer; `cause` is the underlying exception. The SDK
|
|
215
|
+
doesn't follow redirects, so one is `:unimplemented`, like a wrong path: point `endpoint` at the
|
|
216
|
+
final URL.
|
|
217
|
+
|
|
218
|
+
```ruby
|
|
219
|
+
begin
|
|
220
|
+
pug.profiles.get("p_123")
|
|
221
|
+
rescue Pug::Error => e
|
|
222
|
+
raise unless e.code == :not_found
|
|
223
|
+
end
|
|
224
|
+
```
|
|
225
|
+
|
|
226
|
+
## Rails
|
|
227
|
+
|
|
228
|
+
```ruby
|
|
229
|
+
# config/initializers/pug.rb
|
|
230
|
+
unless ENV["SECRET_KEY_BASE_DUMMY"]
|
|
231
|
+
PUG = Pug::Client.new(api_key: Rails.application.credentials.pug_api_key, logger: Rails.logger)
|
|
232
|
+
at_exit { PUG.close(timeout: 5) }
|
|
233
|
+
end
|
|
234
|
+
```
|
|
235
|
+
|
|
236
|
+
`Pug::Client.new` raises without an API key, so the guard skips it during the `assets:precompile`
|
|
237
|
+
step of Rails' generated Dockerfile, which boots the app with `SECRET_KEY_BASE_DUMMY=1` and no
|
|
238
|
+
credentials. The client is safe to create before Puma or Unicorn fork their workers.
|
|
239
|
+
|
|
240
|
+
## Contributing
|
|
241
|
+
|
|
242
|
+
See [`docs/development.md`](./docs/development.md) for building, testing, and how the generated
|
|
243
|
+
code and the well-known event catalog are maintained.
|
|
244
|
+
|
|
245
|
+
## License
|
|
246
|
+
|
|
247
|
+
MIT
|
data/lib/pug/batcher.rb
ADDED
|
@@ -0,0 +1,384 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "errors"
|
|
4
|
+
require_relative "log"
|
|
5
|
+
|
|
6
|
+
module Pug
|
|
7
|
+
# In-memory batching transport. Events accumulate and are sent by a single background thread
|
|
8
|
+
# on a size or interval trigger, so batches never overlap and go out in order. A transient
|
|
9
|
+
# failure re-queues them for a retry after max_wait; a permanent one dead-letters them to
|
|
10
|
+
# on_error, as does close for anything still undelivered.
|
|
11
|
+
class Batcher
|
|
12
|
+
DEFAULTS = { max_size: 100, max_wait: 5.0, max_queue_size: 10_000 }.freeze
|
|
13
|
+
|
|
14
|
+
# BatchCreateRequest.events is capped at 1000 by the events proto.
|
|
15
|
+
MAX_BATCH_EVENTS = 1000
|
|
16
|
+
|
|
17
|
+
# The server refuses request bodies over 8 MiB with resource_exhausted, a transient code, so
|
|
18
|
+
# an oversized batch would be retried forever. Half the cap leaves room for the envelope.
|
|
19
|
+
MAX_BATCH_BYTES = 4 * 1024 * 1024
|
|
20
|
+
|
|
21
|
+
# Retries wait max_wait but at least this long, or a max_wait of 0 would retry a server that
|
|
22
|
+
# refuses connections in a tight loop.
|
|
23
|
+
MIN_RETRY_WAIT = 1
|
|
24
|
+
|
|
25
|
+
# Retrying can't fix these client errors, nor an error that didn't come from the RPC layer,
|
|
26
|
+
# which is a bug.
|
|
27
|
+
PERMANENT_CODES = %i[invalid_argument not_found already_exists permission_denied failed_precondition
|
|
28
|
+
unimplemented unauthenticated].freeze
|
|
29
|
+
|
|
30
|
+
Item = Struct.new(:event, :bytesize)
|
|
31
|
+
|
|
32
|
+
def self.permanent?(error)
|
|
33
|
+
!error.is_a?(Error) || error.code.nil? || PERMANENT_CODES.include?(error.code)
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
attr_reader :max_size, :max_wait, :max_queue_size
|
|
37
|
+
|
|
38
|
+
def initialize(sink, config, on_error, log)
|
|
39
|
+
@sink = sink
|
|
40
|
+
@on_error = on_error
|
|
41
|
+
@log = log
|
|
42
|
+
configure(config)
|
|
43
|
+
@lock = Mutex.new
|
|
44
|
+
@closed = false
|
|
45
|
+
@closer = nil
|
|
46
|
+
reset
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def enqueue(event, bytesize)
|
|
50
|
+
dropped = nil
|
|
51
|
+
closed = @lock.synchronize do
|
|
52
|
+
after_fork
|
|
53
|
+
next true if @closed
|
|
54
|
+
|
|
55
|
+
@buffer << Item.new(event, bytesize)
|
|
56
|
+
dropped = trim
|
|
57
|
+
arm
|
|
58
|
+
@kick = true if @buffer.size >= @max_size && !@backoff
|
|
59
|
+
start
|
|
60
|
+
@wake.signal
|
|
61
|
+
false
|
|
62
|
+
end
|
|
63
|
+
return dead_letter(ClosedError.new("client is closed"), [event]) if closed
|
|
64
|
+
|
|
65
|
+
report_overflow(dropped)
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# Resolves once the attempt it requested has finished, or returns false when timeout (in
|
|
69
|
+
# seconds) runs out first. A transient failure keeps events queued, so an empty queue is not
|
|
70
|
+
# guaranteed; close drains.
|
|
71
|
+
def flush(timeout: nil)
|
|
72
|
+
waiter = Queue.new
|
|
73
|
+
@lock.synchronize do
|
|
74
|
+
after_fork
|
|
75
|
+
return true if @stopping || (@buffer.empty? && !@sending && @requests.empty?)
|
|
76
|
+
# Called from on_error, which runs on the sender: waiting for it would wait forever.
|
|
77
|
+
return false if Thread.current.equal?(@thread)
|
|
78
|
+
|
|
79
|
+
@requests << waiter
|
|
80
|
+
start
|
|
81
|
+
@wake.signal
|
|
82
|
+
end
|
|
83
|
+
!waiter.pop(timeout: timeout).nil?
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
# Drains the buffer and stops the sender. Returns false when timeout (in seconds) runs out
|
|
87
|
+
# first; everything still undelivered then is dead-lettered before it returns, including a
|
|
88
|
+
# batch whose send goes on in the background and may yet succeed.
|
|
89
|
+
def close(timeout: nil)
|
|
90
|
+
deadline = timeout && (monotonic + timeout)
|
|
91
|
+
first = @lock.synchronize do
|
|
92
|
+
after_fork
|
|
93
|
+
next false if @closed
|
|
94
|
+
|
|
95
|
+
@closed = true
|
|
96
|
+
@closer = Thread.current
|
|
97
|
+
disarm
|
|
98
|
+
true
|
|
99
|
+
end
|
|
100
|
+
unless first
|
|
101
|
+
# Called from on_error, on the sender or the closing thread: the close underway waits on this call.
|
|
102
|
+
return false if !@done.closed? && (Thread.current.equal?(@thread) || Thread.current.equal?(@closer))
|
|
103
|
+
|
|
104
|
+
@done.pop(timeout: remaining(deadline))
|
|
105
|
+
return @done.closed? && @drained
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
drained = flush(timeout: remaining(deadline))
|
|
109
|
+
thread = @lock.synchronize do
|
|
110
|
+
@stopping = true
|
|
111
|
+
@wake.signal
|
|
112
|
+
@thread
|
|
113
|
+
end
|
|
114
|
+
stopped = thread.nil? || (!thread.equal?(Thread.current) && !thread.join(remaining(deadline)).nil?)
|
|
115
|
+
report_undelivered
|
|
116
|
+
@drained = drained && stopped
|
|
117
|
+
@done.close
|
|
118
|
+
@drained
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
# The default would list every buffered event.
|
|
122
|
+
def inspect = "#<#{self.class.name}>"
|
|
123
|
+
|
|
124
|
+
private
|
|
125
|
+
|
|
126
|
+
def configure(config)
|
|
127
|
+
config = (config || {}).transform_keys(&:to_sym)
|
|
128
|
+
(config.keys - DEFAULTS.keys).each { |key| @log.warn("Ignoring unknown batch option #{key.inspect}") }
|
|
129
|
+
settings = DEFAULTS.merge(config)
|
|
130
|
+
@max_size = setting(:max_size, settings[:max_size], 1)
|
|
131
|
+
if @max_size > MAX_BATCH_EVENTS
|
|
132
|
+
@log.warn("batch.max_size exceeds the #{MAX_BATCH_EVENTS}-event proto cap, clamping to #{MAX_BATCH_EVENTS}.")
|
|
133
|
+
@max_size = MAX_BATCH_EVENTS
|
|
134
|
+
end
|
|
135
|
+
@max_wait = setting(:max_wait, settings[:max_wait], 0)
|
|
136
|
+
@retry_wait = [@max_wait, MIN_RETRY_WAIT].max
|
|
137
|
+
@max_queue_size = setting(:max_queue_size, settings[:max_queue_size], 1)
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
# Counts must be integers; max_wait, in seconds, may be fractional.
|
|
141
|
+
def setting(name, value, min)
|
|
142
|
+
if min >= 1 && !value.is_a?(Integer)
|
|
143
|
+
@log.warn("batch.#{name} must be an integer, using default.")
|
|
144
|
+
return DEFAULTS[name]
|
|
145
|
+
end
|
|
146
|
+
unless value.is_a?(Numeric) && value.real? && value.finite? && value >= min
|
|
147
|
+
@log.warn("batch.#{name} must be >= #{min}, using default.")
|
|
148
|
+
return DEFAULTS[name]
|
|
149
|
+
end
|
|
150
|
+
value
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
def reset
|
|
154
|
+
@buffer = []
|
|
155
|
+
@inflight = nil
|
|
156
|
+
@requests = []
|
|
157
|
+
@wake = ConditionVariable.new
|
|
158
|
+
@done = Queue.new
|
|
159
|
+
# A child forked from a closed client has nothing to drain, and no closer to wait for.
|
|
160
|
+
@done.close if @closed
|
|
161
|
+
@drained = true
|
|
162
|
+
@deadline = nil
|
|
163
|
+
@kick = false
|
|
164
|
+
@backoff = false
|
|
165
|
+
@sending = false
|
|
166
|
+
@stopping = false
|
|
167
|
+
@thread = nil
|
|
168
|
+
@pid = Process.pid
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
# A forked child inherits the buffer but not the sender thread. The parent still owns and
|
|
172
|
+
# sends those events, so the child starts empty with a sender of its own.
|
|
173
|
+
def after_fork
|
|
174
|
+
reset unless @pid == Process.pid
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
def start
|
|
178
|
+
return if @thread&.alive?
|
|
179
|
+
|
|
180
|
+
@thread = Thread.new { run }
|
|
181
|
+
@thread.name = "pug-batcher"
|
|
182
|
+
@thread.report_on_exception = false
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
def run
|
|
186
|
+
loop do
|
|
187
|
+
requests = @lock.synchronize { next_job }
|
|
188
|
+
break if requests.nil?
|
|
189
|
+
|
|
190
|
+
begin
|
|
191
|
+
send_buffer
|
|
192
|
+
ensure
|
|
193
|
+
requests.each { |waiter| waiter << true }
|
|
194
|
+
end
|
|
195
|
+
end
|
|
196
|
+
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
197
|
+
# Nothing else would report it: the thread doesn't, and on_error or a logger may raise anything.
|
|
198
|
+
@log.error("Event sender stopped unexpectedly; it restarts with the next event: #{Log.describe(e)}")
|
|
199
|
+
ensure
|
|
200
|
+
# An idle exit has already handed over to whichever thread starts next, so only a sender
|
|
201
|
+
# that is stopping or crashed answers the pending flushes.
|
|
202
|
+
@lock.synchronize do
|
|
203
|
+
if @thread.equal?(Thread.current)
|
|
204
|
+
@thread = nil unless @stopping
|
|
205
|
+
@requests.each { |waiter| waiter << true }
|
|
206
|
+
@requests.clear
|
|
207
|
+
end
|
|
208
|
+
end
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
# Called under the lock. Returns the flush requests the next send answers, or nil to stop.
|
|
212
|
+
def next_job
|
|
213
|
+
loop do
|
|
214
|
+
return if @stopping
|
|
215
|
+
|
|
216
|
+
unless @requests.empty?
|
|
217
|
+
requests = @requests
|
|
218
|
+
@requests = []
|
|
219
|
+
return requests
|
|
220
|
+
end
|
|
221
|
+
if @kick
|
|
222
|
+
@kick = false
|
|
223
|
+
# Stale if the send that set it off has since taken those events, or if a backoff began;
|
|
224
|
+
# once closed, close's flush sends.
|
|
225
|
+
return [] if @buffer.size >= @max_size && !@backoff && !@closed
|
|
226
|
+
|
|
227
|
+
next
|
|
228
|
+
end
|
|
229
|
+
if @deadline.nil?
|
|
230
|
+
# Idle: nothing is buffered without a timer armed, so the thread exits rather than
|
|
231
|
+
# holding on to the client, and the next event starts a new one.
|
|
232
|
+
@thread = nil
|
|
233
|
+
return
|
|
234
|
+
end
|
|
235
|
+
wait = @deadline - monotonic
|
|
236
|
+
return [] unless wait.positive?
|
|
237
|
+
|
|
238
|
+
@wake.wait(@lock, wait)
|
|
239
|
+
end
|
|
240
|
+
end
|
|
241
|
+
|
|
242
|
+
# One attempt at what is buffered now, a chunk at a time. Events stay in the buffer until
|
|
243
|
+
# their chunk is taken, and the chunk being sent is tracked, so a close that runs out of time
|
|
244
|
+
# can report every event. Stops at the first transient failure, since the rest would likely
|
|
245
|
+
# fail the same way.
|
|
246
|
+
def send_buffer
|
|
247
|
+
pending = @lock.synchronize do
|
|
248
|
+
disarm
|
|
249
|
+
@backoff = false
|
|
250
|
+
@sending = true
|
|
251
|
+
@buffer.size
|
|
252
|
+
end
|
|
253
|
+
while pending.positive?
|
|
254
|
+
chunk = @lock.synchronize { take_chunk(pending) unless @stopping }
|
|
255
|
+
break if chunk.nil?
|
|
256
|
+
|
|
257
|
+
pending -= chunk.size
|
|
258
|
+
break unless %i[sent permanent].include?(settle(chunk, send_chunk(chunk)))
|
|
259
|
+
end
|
|
260
|
+
ensure
|
|
261
|
+
@lock.synchronize { @sending = false }
|
|
262
|
+
end
|
|
263
|
+
|
|
264
|
+
# Called under the lock: the oldest events, at most limit of them, cut at max_size and
|
|
265
|
+
# MAX_BATCH_BYTES.
|
|
266
|
+
def take_chunk(limit)
|
|
267
|
+
cap = [limit, @max_size].min
|
|
268
|
+
count = 0
|
|
269
|
+
bytes = 0
|
|
270
|
+
@buffer.each do |item|
|
|
271
|
+
break if count == cap || (count.positive? && bytes + item.bytesize > MAX_BATCH_BYTES)
|
|
272
|
+
|
|
273
|
+
count += 1
|
|
274
|
+
bytes += item.bytesize
|
|
275
|
+
end
|
|
276
|
+
@inflight = @buffer.shift(count) if count.positive?
|
|
277
|
+
end
|
|
278
|
+
|
|
279
|
+
# Not just StandardError: one that ended the thread would strand the chunk. Any error that
|
|
280
|
+
# didn't come from the RPC layer is permanent, so the chunk is dead-lettered.
|
|
281
|
+
def send_chunk(chunk)
|
|
282
|
+
@sink.call(chunk.map(&:event))
|
|
283
|
+
nil
|
|
284
|
+
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
285
|
+
e
|
|
286
|
+
end
|
|
287
|
+
|
|
288
|
+
# Returns the outcome of a chunk's send. Open, a transient failure re-queues the chunk for a
|
|
289
|
+
# retry; closed, it dead-letters it and leaves the rest for close to report.
|
|
290
|
+
def settle(chunk, error)
|
|
291
|
+
outcome, dropped = @lock.synchronize do
|
|
292
|
+
# close ran out of time during the send, and has reported the chunk itself.
|
|
293
|
+
next [:reported] unless @inflight.equal?(chunk)
|
|
294
|
+
|
|
295
|
+
@inflight = nil
|
|
296
|
+
if error.nil? then [:sent]
|
|
297
|
+
elsif self.class.permanent?(error) then [:permanent]
|
|
298
|
+
elsif @closed then [:closed]
|
|
299
|
+
else [:retry, requeue(chunk)]
|
|
300
|
+
end
|
|
301
|
+
end
|
|
302
|
+
events = chunk.map(&:event)
|
|
303
|
+
case outcome
|
|
304
|
+
when :permanent
|
|
305
|
+
@log.error("Permanent error, #{events.size} event(s) dropped (will NOT retry): #{Log.describe(error)}")
|
|
306
|
+
dead_letter(error, events)
|
|
307
|
+
when :closed
|
|
308
|
+
@log.error("Client closed, #{events.size} event(s) dropped after transient error: #{Log.describe(error)}")
|
|
309
|
+
dead_letter(error, events)
|
|
310
|
+
when :retry
|
|
311
|
+
@log.warn("Transient error sending batch, will retry: #{Log.describe(error)}")
|
|
312
|
+
report_overflow(dropped)
|
|
313
|
+
end
|
|
314
|
+
outcome
|
|
315
|
+
end
|
|
316
|
+
|
|
317
|
+
# Called under the lock. The retry is timed from the failure: a timer an enqueue armed during
|
|
318
|
+
# the send could already be due.
|
|
319
|
+
def requeue(chunk)
|
|
320
|
+
@buffer.unshift(*chunk)
|
|
321
|
+
@backoff = true
|
|
322
|
+
disarm
|
|
323
|
+
arm(@retry_wait)
|
|
324
|
+
trim
|
|
325
|
+
end
|
|
326
|
+
|
|
327
|
+
# Called under the lock. Drops the oldest events past max_queue_size and returns them.
|
|
328
|
+
def trim
|
|
329
|
+
over = @buffer.size - @max_queue_size
|
|
330
|
+
@buffer.shift(over).map(&:event) if over.positive?
|
|
331
|
+
end
|
|
332
|
+
|
|
333
|
+
def report_overflow(dropped)
|
|
334
|
+
return unless dropped
|
|
335
|
+
|
|
336
|
+
@log.warn("Queue full, dropping #{dropped.size} oldest event(s)")
|
|
337
|
+
dead_letter(QueueOverflowError.new("queue overflow: dropped oldest event"), dropped)
|
|
338
|
+
end
|
|
339
|
+
|
|
340
|
+
def report_undelivered
|
|
341
|
+
inflight, unsent = @lock.synchronize do
|
|
342
|
+
taken = [@inflight, @buffer.slice!(0..)]
|
|
343
|
+
@inflight = nil
|
|
344
|
+
taken
|
|
345
|
+
end
|
|
346
|
+
if inflight
|
|
347
|
+
@log.error("Close timed out with #{inflight.size} event(s) still being sent, which may yet arrive")
|
|
348
|
+
dead_letter(ClosedError.new("close timed out with #{inflight.size} event(s) still being sent"),
|
|
349
|
+
inflight.map(&:event))
|
|
350
|
+
end
|
|
351
|
+
return if unsent.empty?
|
|
352
|
+
|
|
353
|
+
@log.error("Client closed with #{unsent.size} undelivered event(s), dropped")
|
|
354
|
+
dead_letter(ClosedError.new("client closed with #{unsent.size} undelivered event(s)"), unsent.map(&:event))
|
|
355
|
+
end
|
|
356
|
+
|
|
357
|
+
def arm(wait = @max_wait)
|
|
358
|
+
@deadline = monotonic + wait if @deadline.nil? && !@closed
|
|
359
|
+
end
|
|
360
|
+
|
|
361
|
+
def disarm
|
|
362
|
+
@deadline = nil
|
|
363
|
+
end
|
|
364
|
+
|
|
365
|
+
# A raising callback must not abort the sender or a drain: the remaining chunks would be lost
|
|
366
|
+
# with neither a log nor a callback. NotImplementedError, a common placeholder, is a
|
|
367
|
+
# ScriptError.
|
|
368
|
+
def dead_letter(error, events)
|
|
369
|
+
@on_error&.call(error, events)
|
|
370
|
+
nil
|
|
371
|
+
rescue StandardError, ScriptError => e
|
|
372
|
+
@log.error("on_error callback raised; #{events.size} event(s) may be lost: #{Log.describe(e)}")
|
|
373
|
+
nil
|
|
374
|
+
end
|
|
375
|
+
|
|
376
|
+
def remaining(deadline)
|
|
377
|
+
deadline && [deadline - monotonic, 0].max
|
|
378
|
+
end
|
|
379
|
+
|
|
380
|
+
def monotonic
|
|
381
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
382
|
+
end
|
|
383
|
+
end
|
|
384
|
+
end
|