pug-sh 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +21 -0
  3. data/README.md +247 -0
  4. data/lib/pug/batcher.rb +384 -0
  5. data/lib/pug/client.rb +248 -0
  6. data/lib/pug/connect.rb +291 -0
  7. data/lib/pug/countries.rb +19 -0
  8. data/lib/pug/errors.rb +16 -0
  9. data/lib/pug/event_builder.rb +522 -0
  10. data/lib/pug/gen/buf/validate/validate_pb.rb +59 -0
  11. data/lib/pug/gen/common/events/v1/api_events_pb.rb +26 -0
  12. data/lib/pug/gen/common/events/v1/app_events_pb.rb +33 -0
  13. data/lib/pug/gen/common/events/v1/auth_events_pb.rb +32 -0
  14. data/lib/pug/gen/common/events/v1/billing_events_pb.rb +41 -0
  15. data/lib/pug/gen/common/events/v1/chat_events_pb.rb +56 -0
  16. data/lib/pug/gen/common/events/v1/commerce_events_pb.rb +38 -0
  17. data/lib/pug/gen/common/events/v1/discovery_events_pb.rb +31 -0
  18. data/lib/pug/gen/common/events/v1/error_events_pb.rb +26 -0
  19. data/lib/pug/gen/common/events/v1/file_events_pb.rb +29 -0
  20. data/lib/pug/gen/common/events/v1/form_events_pb.rb +27 -0
  21. data/lib/pug/gen/common/events/v1/integration_events_pb.rb +27 -0
  22. data/lib/pug/gen/common/events/v1/invitation_events_pb.rb +26 -0
  23. data/lib/pug/gen/common/events/v1/media_events_pb.rb +36 -0
  24. data/lib/pug/gen/common/events/v1/navigation_events_pb.rb +31 -0
  25. data/lib/pug/gen/common/events/v1/notification_events_pb.rb +28 -0
  26. data/lib/pug/gen/common/events/v1/options_pb.rb +25 -0
  27. data/lib/pug/gen/common/events/v1/social_events_pb.rb +25 -0
  28. data/lib/pug/gen/common/events/v1/support_events_pb.rb +33 -0
  29. data/lib/pug/gen/common/events/v1/workspace_events_pb.rb +30 -0
  30. data/lib/pug/gen/common/v1/filter_schema_pb.rb +29 -0
  31. data/lib/pug/gen/common/v1/filters_pb.rb +27 -0
  32. data/lib/pug/gen/common/v1/property_value_pb.rb +24 -0
  33. data/lib/pug/gen/common/v1/time_pb.rb +25 -0
  34. data/lib/pug/gen/sdk/devices/v1/devices_pb.rb +35 -0
  35. data/lib/pug/gen/sdk/events/v1/events_pb.rb +30 -0
  36. data/lib/pug/gen/sdk/profiles/v1/profiles_pb.rb +28 -0
  37. data/lib/pug/gen/shared/activity/v1/activity_pb.rb +46 -0
  38. data/lib/pug/gen/shared/insights/v1/insights_pb.rb +69 -0
  39. data/lib/pug/gen/shared/profiles/v1/profiles_pb.rb +43 -0
  40. data/lib/pug/log.rb +35 -0
  41. data/lib/pug/proto.rb +22 -0
  42. data/lib/pug/reads.rb +118 -0
  43. data/lib/pug/rpc.rb +73 -0
  44. data/lib/pug/validator.rb +475 -0
  45. data/lib/pug/version.rb +5 -0
  46. data/lib/pug/well_known.rb +33 -0
  47. data/lib/pug-sh.rb +3 -0
  48. data/lib/pug.rb +5 -0
  49. metadata +122 -0
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 893420d16827c4d7472d599fde6fc5278e354cc559ffcd217b5ff34151c0d5c5
4
+ data.tar.gz: d9f76d529e9e0fda1fc678a6096cc9922076e84e0a7c53515e5c6ec98eee4eed
5
+ SHA512:
6
+ metadata.gz: 9194a40e3a2eb238d5dce9f1db3803ca4fd103a0ae366ef27d8dfd266f93fff923fba12afbbce82796af440175a0563bc39495b4e12e007fe32bb7fbb3fb3540
7
+ data.tar.gz: 4db8dc628a8f10582f53eae2242bd6e35c8e1511669cf643e81b1c095615353fcbbc58ca362e058cbbe72bd81eb4f35b886d34cb88c0c8209688f4764b6a7d7f
data/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Pug
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,247 @@
1
+ <p align="center">
2
+ <img src="https://app.pug.sh/logo.svg" alt="Pug" width="120" height="120" />
3
+ </p>
4
+
5
+ # pug-sh
6
+
7
+ Pug server-side SDK for Ruby — event tracking, profile identify, and analytics reads.
8
+
9
+ It is instance-based and authenticates with a **private** project API key (`prv_…`). Ingestion
10
+ never raises, so a tracking call can't take down your request path.
11
+
12
+ ## Install
13
+
14
+ ```ruby
15
+ # Gemfile
16
+ gem "pug-sh"
17
+ ```
18
+
19
+ Requires Ruby 3.3 or newer.
20
+
21
+ ## Quick start
22
+
23
+ ```ruby
24
+ require "pug"
25
+
26
+ pug = Pug::Client.new(api_key: ENV.fetch("PUG_API_KEY"))
27
+
28
+ # Track an event. distinct_id (who the event is for) comes first — a server has no ambient user.
29
+ pug.track("user_42", "order.completed", amount: 49.0, currency: "USD")
30
+
31
+ # Well-known events get their properties type-checked and validated against the event's schema.
32
+ pug.track("user_42", "feature_used", feature_id: "export")
33
+
34
+ # Identify a profile (never raises — failures are logged).
35
+ pug.identify("user_42", email: "ada@example.com", plan: "pro")
36
+
37
+ # Reads raise Pug::Error.
38
+ profile = pug.profiles.get_by_external_id("user_42")
39
+ pug.profiles.list.each { |p| puts p.external_id }
40
+ trend = pug.insights.query(
41
+ spec: { insight_type: :INSIGHT_TYPE_TRENDS, events: [{ event: { kind: "order.completed" } }] },
42
+ time_range: { from: Time.now - (30 * 86_400), to: Time.now },
43
+ granularity: :GRANULARITY_DAY,
44
+ )
45
+
46
+ # Drain on shutdown so no buffered events are lost.
47
+ pug.close
48
+ ```
49
+
50
+ ## Behaviour
51
+
52
+ - **Ingestion never raises.** `track` and `identify` validate input client-side and log and drop
53
+ anything invalid. `track` is non-blocking: a background thread sends events in batches, by size
54
+ (`max_size`) or interval (`max_wait`).
55
+ - **Delivery is best-effort with retry.** A transient send failure keeps events queued and retries
56
+ `max_wait` after the failure, but no sooner than a second; a permanent one (a fixed set of
57
+ client-error Connect codes, plus any non-transport error) dead-letters them via `on_error`.
58
+ `close` drains and reports anything still undelivered through `on_error` and the log.
59
+ `on_error` covers buffered `track` events only — `identify` failures are logged, and so are
60
+ events the server drops from a batch it accepts (`BatchCreateResponse.dropped`), at WARN.
61
+ - **Reads are request/response and do raise** `Pug::Error` — they run in your control flow, so
62
+ you own retry. They are not auto-retried.
63
+ - **Well-known events** have typed, validated properties; any other `kind` is accepted as a
64
+ custom event with loose properties.
65
+ - **Thread- and fork-safe.** One client can be shared by every thread. A client created before a
66
+ fork (Puma, Unicorn, Sidekiq) starts empty in each child, with its own sender thread and
67
+ connections; events buffered in the parent stay with the parent.
68
+
69
+ ## API
70
+
71
+ ### `Pug::Client.new(**options)`
72
+
73
+ | Option | Type | Default | Description |
74
+ | ---------- | ------------- | --------------------------- | --------------------------------------------------------------------------------------------------------------------------------- |
75
+ | `api_key` | `String` | — | **Required.** Private project key (`prv_…`). |
76
+ | `endpoint` | `String` | hosted endpoint | Pug server origin. |
77
+ | `batch` | `Hash` | see below | Ingestion buffer: `max_size` (100, capped at 1000), `max_wait` (5 seconds), `max_queue_size` (10,000). |
78
+ | `on_error` | callable | none | Dead-letter hook for undeliverable buffered `track` events, called with `(error, events)`. |
79
+ | `logger` | `Logger`-like | `Logger.new($stderr)`, WARN | Gets the SDK's warnings and errors: anything with public `warn` and `error`, such as `Rails.logger` (wrap an IO in `Logger.new`). |
80
+ | `timeout` | `Numeric` | 10 | Request timeout, in seconds. |
81
+
82
+ ```ruby
83
+ Pug::Client.new(
84
+ api_key: ENV.fetch("PUG_API_KEY"),
85
+ batch: { max_size: 100, max_wait: 5, max_queue_size: 10_000 },
86
+ on_error: lambda do |error, events|
87
+ # dead-letter sink: error is a Pug::QueueOverflowError, a Pug::ClosedError, a Pug::Error
88
+ # with a permanent Connect code, the transient Pug::Error that failed close's last attempt,
89
+ # or the exception a bug raised
90
+ end,
91
+ )
92
+ ```
93
+
94
+ `on_error` runs on the background sender thread; on the thread calling `close`, for what it
95
+ reports undelivered; or, when the queue overflows, on the thread that called `track`. It must be
96
+ thread-safe and return promptly. Called from inside it on the sender or the closing thread, `flush`
97
+ and `close` don't wait for what is waiting on `on_error`: they return `false` at once, except that
98
+ `flush` returns `true` while `close` reports. On the overflow path nothing waits on `on_error`, so
99
+ they behave as anywhere else, blocking the thread that called `track`.
100
+
101
+ ### Ingestion (never raises)
102
+
103
+ - `pug.track(distinct_id, kind, properties = nil, options = nil)` — enqueue an event. Non-blocking.
104
+ - `pug.identify(external_id, traits = nil, options = nil)` — create or update a profile. Blocks
105
+ until the server answers.
106
+ - `pug.flush(timeout: nil)` — send the currently buffered events now and wait for that attempt.
107
+ Returns `false` if `timeout` (seconds) passed first. `true` means the attempt finished, not that
108
+ every event was delivered: a transient failure keeps them queued for a retry.
109
+ - `pug.close(timeout: nil)` — drain and shut down; call on graceful exit so nothing buffered is lost.
110
+ Returns `false` if `timeout` passed first. Either way, by the time it returns every event it
111
+ couldn't send has gone to `on_error` and the log, including a batch still being sent when the
112
+ timeout ran out, which may yet arrive.
113
+
114
+ A `timeout` that isn't a number waits without one, with a warning.
115
+
116
+ `distinct_id` and `external_id` are non-empty Strings, or Integers such as a record's id, sent as
117
+ their decimal string; `session_id`, `anonymous_id` and `device_id` must be Strings. Properties go
118
+ in a Hash, braces optional, and options follow as keywords. Without braces, a key named after an
119
+ option sets that option, so a property with an option's name needs the braces:
120
+
121
+ ```ruby
122
+ pug.track("user_42", "signup", plan: "pro")
123
+ pug.track("user_42", "signup", plan: "pro", timestamp: Time.now - 3600)
124
+ pug.track("user_42", "signup", { plan: "pro", location: "landing" }, location: { country: "DE" })
125
+ ```
126
+
127
+ `track` options: `timestamp` (a `Time`; defaults to now), `session_id` (a UUID overriding the
128
+ client's per-instance session), `location` (the visitor's location — see below).
129
+ `identify` options, which work the same way with traits: `anonymous_id` (must start with `anon-`;
130
+ triggers anon→identified merge), `device_id`. Traits must be JSON-serializable, with nested values
131
+ mapped as for properties below; they travel as a protobuf `Struct`, whose numbers are doubles, so
132
+ an integer past 2^53 is refused.
133
+
134
+ Custom properties map by Ruby type:
135
+
136
+ | Ruby value | Sent as |
137
+ | ------------------------------------------------ | ----------------------------------------- |
138
+ | `String`, `Symbol` | string |
139
+ | `true`, `false` | bool |
140
+ | `Integer` | int (outside the int64 range: dropped) |
141
+ | `Float`, `BigDecimal`, `Rational` | double (NaN and ±Infinity: dropped) |
142
+ | `Time`, `DateTime`, `Google::Protobuf::Timestamp`| timestamp |
143
+ | `Date` | ISO 8601 date string, e.g. `"2026-10-10"` |
144
+ | `Google::Protobuf::Duration` | duration string, e.g. `"1.5s"` |
145
+ | `Hash`, `Array` | JSON string |
146
+ | `nil` | omitted |
147
+
148
+ Anything else is dropped with a warning. Inside a `Hash` or `Array`, a `Time`, `DateTime` or `Date`
149
+ becomes an ISO 8601 string and a `BigDecimal` or `Rational` a number, while a value JSON has no
150
+ form for, such as a `Set`, drops the whole property with a warning. Strings are capped at 1024
151
+ bytes (truncated with a warning), and invalid UTF-8 is repaired.
152
+
153
+ ### Well-known events
154
+
155
+ For a well-known kind such as `purchase`, properties are checked against the event's schema.
156
+ Keys may be the field's name (`product_id`) or its JSON name (`productId`); either way the
157
+ property is sent under the JSON name, like every other Pug SDK. A field's type comes from the
158
+ schema rather than the Ruby value, so `amount: 10` is sent as a double. A value of the wrong type,
159
+ or one failing the schema's rules, drops the event with an error; other keys ride along as custom
160
+ properties.
161
+
162
+ ```ruby
163
+ pug.track("user_42", "purchase", product_id: "p1", amount: 49, currency: "USD", coupon: "FALL")
164
+ pug.track("user_42", "video_play", video_id: "v1", position: 92.5) # a Duration field, in seconds
165
+ ```
166
+
167
+ Duration fields take seconds as a number, `{ seconds:, nanos: }`, or a `Google::Protobuf::Duration`.
168
+
169
+ ### Location
170
+
171
+ Pug only derives geo from CDN headers on browser (public-key) requests. This SDK uses a private
172
+ key, so the server adds no geo of its own — without `location`, an event has no geo at all:
173
+
174
+ ```ruby
175
+ pug.track("user_42", "checkout_completed", { plan: "pro" }, location: { country: "DE", city: "Berlin" })
176
+ ```
177
+
178
+ Fields: `continent`, `country`, `region`, `city`, `postal_code`, `metro_code`, `timezone`,
179
+ `latitude`, `longitude` — strings, except the two coordinates. `country` must be an ISO 3166-1
180
+ alpha-2 code (case and surrounding spaces are normalized); `latitude` and `longitude` are only
181
+ sent as a pair.
182
+
183
+ A field that is missing, nil or empty is omitted silently. One that looks like a mistake —
184
+ unknown key, wrong type, unknown country, coordinate out of range — is dropped with a warning and
185
+ the rest is still sent. If nothing is usable the event goes out with no geo.
186
+
187
+ ### Reads (raise `Pug::Error`)
188
+
189
+ These run in your control flow:
190
+
191
+ - `pug.profiles` — `get(id, include_bots: false)`, `get_by_external_id(external_id, include_bots: false)`,
192
+ `delete(id)`, `list(request = nil, timeout: nil)` (auto-paginating: yields each profile to a
193
+ block, or returns an `Enumerator`). The timeout bounds the whole stream, the time your block
194
+ takes included, so `timeout:` (seconds) can give one long `list` more than the client's.
195
+ - `pug.activity` — `feed`, `event_explorer`, `heatmap`, `profile_sessions`, `profile_stats`,
196
+ `filter_schema`, `property_values`.
197
+ - `pug.insights` — `query`, `segment_users`, `filter_schema`, `property_values`.
198
+
199
+ Requests are a Hash or the generated request message, and responses are the generated messages,
200
+ under `Pug::Proto` (`Pug::Proto::Shared::Insights::V1::QueryResponse`, …); call `to_h` on one
201
+ for plain Ruby data.
202
+
203
+ ```ruby
204
+ sessions = pug.activity.profile_sessions(distinct_id: "user_42", page_size: 50)
205
+ sessions.sessions.each { |s| puts s.session_id }
206
+ ```
207
+
208
+ Reads leave out traffic tagged as a bot at ingest unless you ask for it: pass
209
+ `include_bots: true` to `profiles.get` and `get_by_external_id`, set it on the request of
210
+ `profiles.list` or of an `activity` read other than `filter_schema` and `property_values`, or set
211
+ it on the `spec` of `insights.query`. The other requests have no such field.
212
+
213
+ `Pug::Error#code` is the Connect code as a Symbol (`:not_found`, `:unavailable`, …), or `nil`
214
+ when the failure didn't come from the RPC layer; `cause` is the underlying exception. The SDK
215
+ doesn't follow redirects, so one is `:unimplemented`, like a wrong path: point `endpoint` at the
216
+ final URL.
217
+
218
+ ```ruby
219
+ begin
220
+ pug.profiles.get("p_123")
221
+ rescue Pug::Error => e
222
+ raise unless e.code == :not_found
223
+ end
224
+ ```
225
+
226
+ ## Rails
227
+
228
+ ```ruby
229
+ # config/initializers/pug.rb
230
+ unless ENV["SECRET_KEY_BASE_DUMMY"]
231
+ PUG = Pug::Client.new(api_key: Rails.application.credentials.pug_api_key, logger: Rails.logger)
232
+ at_exit { PUG.close(timeout: 5) }
233
+ end
234
+ ```
235
+
236
+ `Pug::Client.new` raises without an API key, so the guard skips it during the `assets:precompile`
237
+ step of Rails' generated Dockerfile, which boots the app with `SECRET_KEY_BASE_DUMMY=1` and no
238
+ credentials. The client is safe to create before Puma or Unicorn fork their workers.
239
+
240
+ ## Contributing
241
+
242
+ See [`docs/development.md`](./docs/development.md) for building, testing, and how the generated
243
+ code and the well-known event catalog are maintained.
244
+
245
+ ## License
246
+
247
+ MIT
@@ -0,0 +1,384 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "errors"
4
+ require_relative "log"
5
+
6
+ module Pug
7
+ # In-memory batching transport. Events accumulate and are sent by a single background thread
8
+ # on a size or interval trigger, so batches never overlap and go out in order. A transient
9
+ # failure re-queues them for a retry after max_wait; a permanent one dead-letters them to
10
+ # on_error, as does close for anything still undelivered.
11
+ class Batcher
12
+ DEFAULTS = { max_size: 100, max_wait: 5.0, max_queue_size: 10_000 }.freeze
13
+
14
+ # BatchCreateRequest.events is capped at 1000 by the events proto.
15
+ MAX_BATCH_EVENTS = 1000
16
+
17
+ # The server refuses request bodies over 8 MiB with resource_exhausted, a transient code, so
18
+ # an oversized batch would be retried forever. Half the cap leaves room for the envelope.
19
+ MAX_BATCH_BYTES = 4 * 1024 * 1024
20
+
21
+ # Retries wait max_wait but at least this long, or a max_wait of 0 would retry a server that
22
+ # refuses connections in a tight loop.
23
+ MIN_RETRY_WAIT = 1
24
+
25
+ # Retrying can't fix these client errors, nor an error that didn't come from the RPC layer,
26
+ # which is a bug.
27
+ PERMANENT_CODES = %i[invalid_argument not_found already_exists permission_denied failed_precondition
28
+ unimplemented unauthenticated].freeze
29
+
30
+ Item = Struct.new(:event, :bytesize)
31
+
32
+ def self.permanent?(error)
33
+ !error.is_a?(Error) || error.code.nil? || PERMANENT_CODES.include?(error.code)
34
+ end
35
+
36
+ attr_reader :max_size, :max_wait, :max_queue_size
37
+
38
+ def initialize(sink, config, on_error, log)
39
+ @sink = sink
40
+ @on_error = on_error
41
+ @log = log
42
+ configure(config)
43
+ @lock = Mutex.new
44
+ @closed = false
45
+ @closer = nil
46
+ reset
47
+ end
48
+
49
+ def enqueue(event, bytesize)
50
+ dropped = nil
51
+ closed = @lock.synchronize do
52
+ after_fork
53
+ next true if @closed
54
+
55
+ @buffer << Item.new(event, bytesize)
56
+ dropped = trim
57
+ arm
58
+ @kick = true if @buffer.size >= @max_size && !@backoff
59
+ start
60
+ @wake.signal
61
+ false
62
+ end
63
+ return dead_letter(ClosedError.new("client is closed"), [event]) if closed
64
+
65
+ report_overflow(dropped)
66
+ end
67
+
68
+ # Resolves once the attempt it requested has finished, or returns false when timeout (in
69
+ # seconds) runs out first. A transient failure keeps events queued, so an empty queue is not
70
+ # guaranteed; close drains.
71
+ def flush(timeout: nil)
72
+ waiter = Queue.new
73
+ @lock.synchronize do
74
+ after_fork
75
+ return true if @stopping || (@buffer.empty? && !@sending && @requests.empty?)
76
+ # Called from on_error, which runs on the sender: waiting for it would wait forever.
77
+ return false if Thread.current.equal?(@thread)
78
+
79
+ @requests << waiter
80
+ start
81
+ @wake.signal
82
+ end
83
+ !waiter.pop(timeout: timeout).nil?
84
+ end
85
+
86
+ # Drains the buffer and stops the sender. Returns false when timeout (in seconds) runs out
87
+ # first; everything still undelivered then is dead-lettered before it returns, including a
88
+ # batch whose send goes on in the background and may yet succeed.
89
+ def close(timeout: nil)
90
+ deadline = timeout && (monotonic + timeout)
91
+ first = @lock.synchronize do
92
+ after_fork
93
+ next false if @closed
94
+
95
+ @closed = true
96
+ @closer = Thread.current
97
+ disarm
98
+ true
99
+ end
100
+ unless first
101
+ # Called from on_error, on the sender or the closing thread: the close underway waits on this call.
102
+ return false if !@done.closed? && (Thread.current.equal?(@thread) || Thread.current.equal?(@closer))
103
+
104
+ @done.pop(timeout: remaining(deadline))
105
+ return @done.closed? && @drained
106
+ end
107
+
108
+ drained = flush(timeout: remaining(deadline))
109
+ thread = @lock.synchronize do
110
+ @stopping = true
111
+ @wake.signal
112
+ @thread
113
+ end
114
+ stopped = thread.nil? || (!thread.equal?(Thread.current) && !thread.join(remaining(deadline)).nil?)
115
+ report_undelivered
116
+ @drained = drained && stopped
117
+ @done.close
118
+ @drained
119
+ end
120
+
121
+ # The default would list every buffered event.
122
+ def inspect = "#<#{self.class.name}>"
123
+
124
+ private
125
+
126
+ def configure(config)
127
+ config = (config || {}).transform_keys(&:to_sym)
128
+ (config.keys - DEFAULTS.keys).each { |key| @log.warn("Ignoring unknown batch option #{key.inspect}") }
129
+ settings = DEFAULTS.merge(config)
130
+ @max_size = setting(:max_size, settings[:max_size], 1)
131
+ if @max_size > MAX_BATCH_EVENTS
132
+ @log.warn("batch.max_size exceeds the #{MAX_BATCH_EVENTS}-event proto cap, clamping to #{MAX_BATCH_EVENTS}.")
133
+ @max_size = MAX_BATCH_EVENTS
134
+ end
135
+ @max_wait = setting(:max_wait, settings[:max_wait], 0)
136
+ @retry_wait = [@max_wait, MIN_RETRY_WAIT].max
137
+ @max_queue_size = setting(:max_queue_size, settings[:max_queue_size], 1)
138
+ end
139
+
140
+ # Counts must be integers; max_wait, in seconds, may be fractional.
141
+ def setting(name, value, min)
142
+ if min >= 1 && !value.is_a?(Integer)
143
+ @log.warn("batch.#{name} must be an integer, using default.")
144
+ return DEFAULTS[name]
145
+ end
146
+ unless value.is_a?(Numeric) && value.real? && value.finite? && value >= min
147
+ @log.warn("batch.#{name} must be >= #{min}, using default.")
148
+ return DEFAULTS[name]
149
+ end
150
+ value
151
+ end
152
+
153
+ def reset
154
+ @buffer = []
155
+ @inflight = nil
156
+ @requests = []
157
+ @wake = ConditionVariable.new
158
+ @done = Queue.new
159
+ # A child forked from a closed client has nothing to drain, and no closer to wait for.
160
+ @done.close if @closed
161
+ @drained = true
162
+ @deadline = nil
163
+ @kick = false
164
+ @backoff = false
165
+ @sending = false
166
+ @stopping = false
167
+ @thread = nil
168
+ @pid = Process.pid
169
+ end
170
+
171
+ # A forked child inherits the buffer but not the sender thread. The parent still owns and
172
+ # sends those events, so the child starts empty with a sender of its own.
173
+ def after_fork
174
+ reset unless @pid == Process.pid
175
+ end
176
+
177
+ def start
178
+ return if @thread&.alive?
179
+
180
+ @thread = Thread.new { run }
181
+ @thread.name = "pug-batcher"
182
+ @thread.report_on_exception = false
183
+ end
184
+
185
+ def run
186
+ loop do
187
+ requests = @lock.synchronize { next_job }
188
+ break if requests.nil?
189
+
190
+ begin
191
+ send_buffer
192
+ ensure
193
+ requests.each { |waiter| waiter << true }
194
+ end
195
+ end
196
+ rescue Exception => e # rubocop:disable Lint/RescueException
197
+ # Nothing else would report it: the thread doesn't, and on_error or a logger may raise anything.
198
+ @log.error("Event sender stopped unexpectedly; it restarts with the next event: #{Log.describe(e)}")
199
+ ensure
200
+ # An idle exit has already handed over to whichever thread starts next, so only a sender
201
+ # that is stopping or crashed answers the pending flushes.
202
+ @lock.synchronize do
203
+ if @thread.equal?(Thread.current)
204
+ @thread = nil unless @stopping
205
+ @requests.each { |waiter| waiter << true }
206
+ @requests.clear
207
+ end
208
+ end
209
+ end
210
+
211
+ # Called under the lock. Returns the flush requests the next send answers, or nil to stop.
212
+ def next_job
213
+ loop do
214
+ return if @stopping
215
+
216
+ unless @requests.empty?
217
+ requests = @requests
218
+ @requests = []
219
+ return requests
220
+ end
221
+ if @kick
222
+ @kick = false
223
+ # Stale if the send that set it off has since taken those events, or if a backoff began;
224
+ # once closed, close's flush sends.
225
+ return [] if @buffer.size >= @max_size && !@backoff && !@closed
226
+
227
+ next
228
+ end
229
+ if @deadline.nil?
230
+ # Idle: nothing is buffered without a timer armed, so the thread exits rather than
231
+ # holding on to the client, and the next event starts a new one.
232
+ @thread = nil
233
+ return
234
+ end
235
+ wait = @deadline - monotonic
236
+ return [] unless wait.positive?
237
+
238
+ @wake.wait(@lock, wait)
239
+ end
240
+ end
241
+
242
+ # One attempt at what is buffered now, a chunk at a time. Events stay in the buffer until
243
+ # their chunk is taken, and the chunk being sent is tracked, so a close that runs out of time
244
+ # can report every event. Stops at the first transient failure, since the rest would likely
245
+ # fail the same way.
246
+ def send_buffer
247
+ pending = @lock.synchronize do
248
+ disarm
249
+ @backoff = false
250
+ @sending = true
251
+ @buffer.size
252
+ end
253
+ while pending.positive?
254
+ chunk = @lock.synchronize { take_chunk(pending) unless @stopping }
255
+ break if chunk.nil?
256
+
257
+ pending -= chunk.size
258
+ break unless %i[sent permanent].include?(settle(chunk, send_chunk(chunk)))
259
+ end
260
+ ensure
261
+ @lock.synchronize { @sending = false }
262
+ end
263
+
264
+ # Called under the lock: the oldest events, at most limit of them, cut at max_size and
265
+ # MAX_BATCH_BYTES.
266
+ def take_chunk(limit)
267
+ cap = [limit, @max_size].min
268
+ count = 0
269
+ bytes = 0
270
+ @buffer.each do |item|
271
+ break if count == cap || (count.positive? && bytes + item.bytesize > MAX_BATCH_BYTES)
272
+
273
+ count += 1
274
+ bytes += item.bytesize
275
+ end
276
+ @inflight = @buffer.shift(count) if count.positive?
277
+ end
278
+
279
+ # Not just StandardError: one that ended the thread would strand the chunk. Any error that
280
+ # didn't come from the RPC layer is permanent, so the chunk is dead-lettered.
281
+ def send_chunk(chunk)
282
+ @sink.call(chunk.map(&:event))
283
+ nil
284
+ rescue Exception => e # rubocop:disable Lint/RescueException
285
+ e
286
+ end
287
+
288
+ # Returns the outcome of a chunk's send. Open, a transient failure re-queues the chunk for a
289
+ # retry; closed, it dead-letters it and leaves the rest for close to report.
290
+ def settle(chunk, error)
291
+ outcome, dropped = @lock.synchronize do
292
+ # close ran out of time during the send, and has reported the chunk itself.
293
+ next [:reported] unless @inflight.equal?(chunk)
294
+
295
+ @inflight = nil
296
+ if error.nil? then [:sent]
297
+ elsif self.class.permanent?(error) then [:permanent]
298
+ elsif @closed then [:closed]
299
+ else [:retry, requeue(chunk)]
300
+ end
301
+ end
302
+ events = chunk.map(&:event)
303
+ case outcome
304
+ when :permanent
305
+ @log.error("Permanent error, #{events.size} event(s) dropped (will NOT retry): #{Log.describe(error)}")
306
+ dead_letter(error, events)
307
+ when :closed
308
+ @log.error("Client closed, #{events.size} event(s) dropped after transient error: #{Log.describe(error)}")
309
+ dead_letter(error, events)
310
+ when :retry
311
+ @log.warn("Transient error sending batch, will retry: #{Log.describe(error)}")
312
+ report_overflow(dropped)
313
+ end
314
+ outcome
315
+ end
316
+
317
+ # Called under the lock. The retry is timed from the failure: a timer an enqueue armed during
318
+ # the send could already be due.
319
+ def requeue(chunk)
320
+ @buffer.unshift(*chunk)
321
+ @backoff = true
322
+ disarm
323
+ arm(@retry_wait)
324
+ trim
325
+ end
326
+
327
+ # Called under the lock. Drops the oldest events past max_queue_size and returns them.
328
+ def trim
329
+ over = @buffer.size - @max_queue_size
330
+ @buffer.shift(over).map(&:event) if over.positive?
331
+ end
332
+
333
+ def report_overflow(dropped)
334
+ return unless dropped
335
+
336
+ @log.warn("Queue full, dropping #{dropped.size} oldest event(s)")
337
+ dead_letter(QueueOverflowError.new("queue overflow: dropped oldest event"), dropped)
338
+ end
339
+
340
+ def report_undelivered
341
+ inflight, unsent = @lock.synchronize do
342
+ taken = [@inflight, @buffer.slice!(0..)]
343
+ @inflight = nil
344
+ taken
345
+ end
346
+ if inflight
347
+ @log.error("Close timed out with #{inflight.size} event(s) still being sent, which may yet arrive")
348
+ dead_letter(ClosedError.new("close timed out with #{inflight.size} event(s) still being sent"),
349
+ inflight.map(&:event))
350
+ end
351
+ return if unsent.empty?
352
+
353
+ @log.error("Client closed with #{unsent.size} undelivered event(s), dropped")
354
+ dead_letter(ClosedError.new("client closed with #{unsent.size} undelivered event(s)"), unsent.map(&:event))
355
+ end
356
+
357
+ def arm(wait = @max_wait)
358
+ @deadline = monotonic + wait if @deadline.nil? && !@closed
359
+ end
360
+
361
+ def disarm
362
+ @deadline = nil
363
+ end
364
+
365
+ # A raising callback must not abort the sender or a drain: the remaining chunks would be lost
366
+ # with neither a log nor a callback. NotImplementedError, a common placeholder, is a
367
+ # ScriptError.
368
+ def dead_letter(error, events)
369
+ @on_error&.call(error, events)
370
+ nil
371
+ rescue StandardError, ScriptError => e
372
+ @log.error("on_error callback raised; #{events.size} event(s) may be lost: #{Log.describe(e)}")
373
+ nil
374
+ end
375
+
376
+ def remaining(deadline)
377
+ deadline && [deadline - monotonic, 0].max
378
+ end
379
+
380
+ def monotonic
381
+ Process.clock_gettime(Process::CLOCK_MONOTONIC)
382
+ end
383
+ end
384
+ end