railwatch 1.1.1 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +46 -0
- data/app/controllers/railwatch/beacon_controller.rb +33 -1
- data/docs/records.md +14 -4
- data/lib/generators/railwatch/install/templates/railwatch.ts +61 -2
- data/lib/railwatch/execution.rb +35 -3
- data/lib/railwatch/health.rb +38 -2
- data/lib/railwatch/subscribers/views.rb +3 -1
- data/lib/railwatch/version.rb +1 -1
- data/lib/railwatch/wire_fixtures.json +9 -86
- data/lib/railwatch.rb +8 -1
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: af26db042e24211ffde4ad4e82dd90a7bc27e37c0bc9f74674947a30109b0036
|
|
4
|
+
data.tar.gz: 8c8033a1fa3f4b97142dd6b3593cf4bcc7fa16d2bea97fed6d580ee1584133de
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 1ab758eed04fbcc9088989d5c411c88e0e80d8c3fbc08195c2908f04fcf38a21ad6b9ed8ec3ab8363081fcad59886f1abff8efb93bd6abfaeb156bada0bb66dc
|
|
7
|
+
data.tar.gz: 81e0e6e114d46a70619e21ae97816f451b83ccee7ee3f8e8aad8f2ccce8bd487beb31bfdea4cbbdf467357da7e7c4ae1b32dad69eae317e55083755897f3aa23
|
data/CHANGELOG.md
CHANGED
|
@@ -6,6 +6,52 @@
|
|
|
6
6
|
filled in by the release commit, which is also the only commit that
|
|
7
7
|
touches lib/railwatch/version.rb and Gemfile.lock. See CONTRIBUTING.md. -->
|
|
8
8
|
|
|
9
|
+
## 1.3.0 (2026-10-25)
|
|
10
|
+
|
|
11
|
+
- The initial-load visit splits its TTFB into `redirect_ms`,
|
|
12
|
+
`connect_ms` and `server_ms`, which add up to `ttfb`, plus
|
|
13
|
+
`redirect_count`. TTFB is measured from navigation start, so it holds
|
|
14
|
+
every redirect before the page: a 1.2 s TTFB on a route the server
|
|
15
|
+
answers in 100 ms was a redirect chain, and nothing said so.
|
|
16
|
+
`redirect_ms` (navigation start to `fetchStart`) is still measured when
|
|
17
|
+
a hop is cross-origin, where browsers report `redirectCount` 0; the
|
|
18
|
+
client then leaves `redirect_count` out rather than claim no redirect.
|
|
19
|
+
The beacon accepts and clamps the four fields. Receivers must store them
|
|
20
|
+
for them to show anywhere. Apps that copied the client
|
|
21
|
+
(`app/frontend/lib/railwatch.ts`) need the same change in their copy.
|
|
22
|
+
|
|
23
|
+
## 1.2.0 (2026-10-25)
|
|
24
|
+
|
|
25
|
+
- Executions send only the counters that fired. Every parent record
|
|
26
|
+
carried all 18 of `Execution::COUNTERS`, most of them zero: 281 bytes of
|
|
27
|
+
`counters` JSON on every request, job and command. A request that ran
|
|
28
|
+
two queries, a log line and a view now sends 59 bytes; a channel action
|
|
29
|
+
that did nothing sends `{}`. A counter missing from the hash is zero.
|
|
30
|
+
Needs a railwatch-rs that reads it that way (Rebulk/railwatch-rs#43);
|
|
31
|
+
older servers show a blank query count on the route page and `null`
|
|
32
|
+
from the MCP tools for a request that ran none.
|
|
33
|
+
|
|
34
|
+
- Health samples stop repeating the recurring task manifest. `detail`
|
|
35
|
+
carried every Solid Queue recurring task key and schedule on every
|
|
36
|
+
sample, every 15 seconds, from every process: one unchanging string
|
|
37
|
+
that was 94% of the column (1,312 of 1,417 bytes a row on a 28-task
|
|
38
|
+
app). It is now sent when it changes, on a process's first sample, and
|
|
39
|
+
otherwise once every five minutes per process. railwatch-rs's
|
|
40
|
+
scheduled-task check reads the newest sample that carries one from the
|
|
41
|
+
last ten minutes, so it always finds one.
|
|
42
|
+
|
|
43
|
+
- An execution whose record tree outgrew `execution_buffer_bytes` (8 MiB)
|
|
44
|
+
says so. The buffer keeps the earliest child records and drops the
|
|
45
|
+
rest, and the parent said nothing: a job that issued 5,048 queries
|
|
46
|
+
showed a plausible 3,155-query trace, and the loss was visible only as
|
|
47
|
+
the batch's dropped count. The parent record now carries
|
|
48
|
+
`dropped_records` and `dropped_bytes` when the tree lost anything.
|
|
49
|
+
railwatch-rs shows them on the execution page (Rebulk/railwatch-rs#44).
|
|
50
|
+
- Stop building records the execution is about to drop. Once its buffer
|
|
51
|
+
has overflowed, `Execution#recording?` answers no and counts the record
|
|
52
|
+
the subscriber was about to build, so the SQL normalisation, redaction,
|
|
53
|
+
backtrace walk and weighing for it never run.
|
|
54
|
+
|
|
9
55
|
## 1.1.1 (2026-10-24)
|
|
10
56
|
|
|
11
57
|
- The browser client names the first page load after its Inertia
|
|
@@ -10,6 +10,8 @@ module Railwatch
|
|
|
10
10
|
# forged beacon, not a page load, so it is clamped rather than stored.
|
|
11
11
|
MAX_METRIC_MS = 120_000
|
|
12
12
|
MAX_CLS = 100.0
|
|
13
|
+
# Browsers stop following a redirect chain at 20 (Fetch's limit).
|
|
14
|
+
MAX_REDIRECTS = 20
|
|
13
15
|
|
|
14
16
|
MAX_REQUEST_BYTES = 256 * 1024
|
|
15
17
|
MAX_VISITS = 50
|
|
@@ -50,6 +52,7 @@ module Railwatch
|
|
|
50
52
|
cls: cumulative_layout_shift(v["cls"]),
|
|
51
53
|
inp: metric_ms(v["inp"]),
|
|
52
54
|
ttfb: metric_ms(v["ttfb"]),
|
|
55
|
+
**ttfb_phases(v),
|
|
53
56
|
user: user_id,
|
|
54
57
|
tenant: tenant,
|
|
55
58
|
user_agent: safe_string(request.user_agent, 256))
|
|
@@ -274,9 +277,38 @@ module Railwatch
|
|
|
274
277
|
# Web vitals only ride along on the initial-load visit, and only from
|
|
275
278
|
# browsers that support the entry type behind them, so every one of these
|
|
276
279
|
# is nil far more often than not.
|
|
280
|
+
# A value that is not a finite number ("1e309" parses to Infinity, which
|
|
281
|
+
# #round refuses) is nil, like a missing one: raising here would drop the
|
|
282
|
+
# whole beacon at the rescue below, every other visit in it included.
|
|
277
283
|
def metric_ms(value)
|
|
284
|
+
finite_ms(value)&.round&.clamp(0, MAX_METRIC_MS)
|
|
285
|
+
end
|
|
286
|
+
|
|
287
|
+
# The initial load's TTFB split into phases. They add up to ttfb, so if
|
|
288
|
+
# clamping any of them (or ttfb itself) would break that sum, the split is
|
|
289
|
+
# dropped rather than stored inconsistent: ttfb stays, the phases are nil.
|
|
290
|
+
def ttfb_phases(visit)
|
|
291
|
+
raw = %w[redirect_ms connect_ms server_ms].to_h { |key| [ key, finite_ms(visit[key]) ] }
|
|
292
|
+
phases = raw.transform_values { |ms| ms&.round&.clamp(0, MAX_METRIC_MS) }
|
|
293
|
+
total = finite_ms(visit["ttfb"])
|
|
294
|
+
clamped = raw.any? { |key, ms| ms && ms.round != phases[key] }
|
|
295
|
+
over = total && total.round > MAX_METRIC_MS
|
|
296
|
+
phases = phases.transform_values { nil } if clamped || over
|
|
297
|
+
phases.transform_keys(&:to_sym).merge(redirect_count: redirect_count(visit["redirect_count"]))
|
|
298
|
+
end
|
|
299
|
+
|
|
300
|
+
def finite_ms(value)
|
|
301
|
+
return nil if value.nil? || value == ""
|
|
302
|
+
float = safe_float(value)
|
|
303
|
+
float.finite? ? float : nil
|
|
304
|
+
end
|
|
305
|
+
|
|
306
|
+
# How many redirects the browser SHOWED. It hides them when any hop was
|
|
307
|
+
# cross-origin, so the client sends nothing then rather than 0, and a
|
|
308
|
+
# nil here must stay nil: "none" and "hidden" are different answers.
|
|
309
|
+
def redirect_count(value)
|
|
278
310
|
return nil if value.nil? || value == ""
|
|
279
|
-
|
|
311
|
+
safe_integer(value).clamp(0, MAX_REDIRECTS)
|
|
280
312
|
end
|
|
281
313
|
|
|
282
314
|
def cumulative_layout_shift(value)
|
data/docs/records.md
CHANGED
|
@@ -56,12 +56,14 @@ these fields, from `Railwatch.build_parent` in `lib/railwatch.rb`:
|
|
|
56
56
|
|---|---|
|
|
57
57
|
| `duration` | Wall time in microseconds, execution start to finish. |
|
|
58
58
|
| `stages` | Hash of stage name → microseconds spent in it (e.g. `{"middleware_before" => 120, "action" => 4300, "render" => 900}`). |
|
|
59
|
-
| `counters` | Hash of child-record counts for this execution — `queries`, `cached_queries`, `exceptions`, `logs`, `cache_events`, `jobs_enqueued`, `mail`, `broadcasts`, `notifications`, `outgoing_requests`, `storage_ops`, `view_renders`, `transactions`, `hydrated_models`, `lazy_loads`, `deprecations`, `spans` (`Execution::COUNTERS`). Counted even when the execution is sampled out, so aggregate rates don't depend on the sample rate. |
|
|
59
|
+
| `counters` | Hash of child-record counts for this execution — `queries`, `cached_queries`, `exceptions`, `logs`, `cache_events`, `jobs_enqueued`, `mail`, `broadcasts`, `notifications`, `outgoing_requests`, `storage_ops`, `view_renders`, `transactions`, `hydrated_models`, `lazy_loads`, `deprecations`, `spans` (`Execution::COUNTERS`). Counted even when the execution is sampled out, so aggregate rates don't depend on the sample rate. Only the counters that fired are sent: a counter missing from the hash is zero, and readers must treat it that way. |
|
|
60
60
|
| `peak_memory` | RSS in bytes, sampled at most once per second process-wide (`Execution.sampled_memory`) — cheap enough to read but not per-execution-accurate to the microsecond. |
|
|
61
61
|
| `allocations` | Objects allocated during the execution (`GC.stat(:total_allocated_objects)` delta). |
|
|
62
62
|
| `gc_time` | GC time in the execution's window, when the Ruby build exposes `GC.stat(:time)`. |
|
|
63
63
|
| `exception_preview` | First unhandled exception's `"Class: message"`, truncated to 255 chars, or nil. |
|
|
64
64
|
| `context` | Serialized `Railwatch.context(...)` key/values active for this execution, parameter-filtered like request params. `"_railwatch_truncated": true` when it did not fit in 64KB. |
|
|
65
|
+
| `dropped_records` | Child records this execution's tree lost to `execution_buffer_bytes` (or `Execution::MAX_RECORDS`): the earliest records are kept and everything after the first overflow is counted here instead of built. Present only when non-zero, and only on a tree that shipped. View renders past `max_view_renders_per_execution` are capped, not dropped, and not counted here. |
|
|
66
|
+
| `dropped_bytes` | Estimated in-memory weight of those dropped records (records refused after the overflow are weighed at the buffer's mean). Present alongside `dropped_records`. |
|
|
65
67
|
|
|
66
68
|
A sampled-out execution still ships its parent record if it raised an
|
|
67
69
|
unhandled exception. See `Railwatch.finish_execution`. Sampling controls
|
|
@@ -766,7 +768,11 @@ entirely if `config.beacon_enabled` is off.
|
|
|
766
768
|
| `lcp` | Largest Contentful Paint, integer ms, clamped to 0–120000. Initial load only. |
|
|
767
769
|
| `cls` | Cumulative Layout Shift — the largest session window, float rounded to 4 decimals, clamped to 0–100. Initial load only. |
|
|
768
770
|
| `inp` | Interaction to Next Paint, integer ms, clamped to 0–120000. The slowest interaction, not the spec's high percentile. Initial load only. |
|
|
769
|
-
| `ttfb` | Time to First Byte from navigation timing's `responseStart`, integer ms, clamped to 0–120000. Initial load only. |
|
|
771
|
+
| `ttfb` | Time to First Byte from navigation timing's `responseStart`, integer ms, clamped to 0–120000. Initial load only. Includes every redirect before the page. |
|
|
772
|
+
| `redirect_ms` | The part of `ttfb` before the final request started: navigation start to `fetchStart` (`workerStart` when a service worker started first). Every redirect hop, plus a few ms of the browser's own work. Still measured when a hop is cross-origin, when the browser hides the count. Integer ms, clamped to 0–120000. Initial load only. |
|
|
773
|
+
| `redirect_count` | Redirects the browser **showed** (`redirectCount`), clamped to 0–20. Nil when it showed none, which is either no redirect or a cross-origin hop it hid: never read nil as "no redirect", read `redirect_ms`. Initial load only. |
|
|
774
|
+
| `connect_ms` | `ttfb` from the final request's start to `requestStart`: service worker startup, HTTP cache, DNS, TCP and TLS. Integer ms, clamped to 0–120000. Initial load only. |
|
|
775
|
+
| `server_ms` | `ttfb` from `requestStart` to `responseStart`: the network round trip plus the server's time. Integer ms, clamped to 0–120000. Initial load only. |
|
|
770
776
|
| `user` | Resolved server-side from the beacon request's session/cookies, same resolver as every other record. |
|
|
771
777
|
| `tenant` | Current tenant context. |
|
|
772
778
|
| `user_agent` | Truncated to 256 chars. |
|
|
@@ -779,7 +785,11 @@ never routed it. It has `method` `"GET"`, `status` `"success"`, and
|
|
|
779
785
|
four Core Web Vitals. It is held back until the page is first hidden, on
|
|
780
786
|
`visibilitychange`/`pagehide`, so those numbers are final when it ships.
|
|
781
787
|
Every vital is nil on a browser that doesn't support the
|
|
782
|
-
`PerformanceObserver` entry type behind it.
|
|
788
|
+
`PerformanceObserver` entry type behind it. `redirect_ms`, `connect_ms`
|
|
789
|
+
and `server_ms` add up to `ttfb` exactly (the boundaries are rounded
|
|
790
|
+
before subtracting); all three are nil when the browser's timestamps are
|
|
791
|
+
missing or out of order, and when clamping `ttfb` or any phase to the
|
|
792
|
+
120,000 ms cap would leave a split that no longer adds up.
|
|
783
793
|
|
|
784
794
|
### `session`
|
|
785
795
|
|
|
@@ -870,7 +880,7 @@ watches. The record still ships with whatever it did manage to read.
|
|
|
870
880
|
| `pool_waiting` | Threads blocked waiting for a connection — sustained non-zero means the pool is undersized for the thread count. |
|
|
871
881
|
| `queue_depth` | `SolidQueue::ReadyExecution.count` — jobs ready to run right now. |
|
|
872
882
|
| `queue_latency` | Microseconds since the oldest ready job was created, i.e. the backlog's head-of-line wait. nil when the queue is empty. |
|
|
873
|
-
| `detail` | JSON string: `queues` (ready count per queue name), `workers` (`SolidQueue::Process` rows of kind `Worker`), `requests_count` (Puma's lifetime request count for this process), `running` (threads Puma has spawned), `max_threads_reached` (true when Puma's `pool_capacity` was 0 at sample time, i.e. no spare thread), `recurring_tasks` (Solid Queue recurring task key => schedule, from `SolidQueue::RecurringTask`; left out when there are none or the table could not be read, so the platform can tell a task removed from `config/recurring.yml` apart from one that stopped running). |
|
|
883
|
+
| `detail` | JSON string: `queues` (ready count per queue name), `workers` (`SolidQueue::Process` rows of kind `Worker`), `requests_count` (Puma's lifetime request count for this process), `running` (threads Puma has spawned), `max_threads_reached` (true when Puma's `pool_capacity` was 0 at sample time, i.e. no spare thread), `recurring_tasks` (Solid Queue recurring task key => schedule, from `SolidQueue::RecurringTask`; left out when there are none or the table could not be read, so the platform can tell a task removed from `config/recurring.yml` apart from one that stopped running). **Not on every sample.** The manifest is identical in every process and changes only with `config/recurring.yml`, so repeating it made it 94% of this column; it is re-sent when it changes and otherwise once every five minutes per process. The reader takes the newest sample that carries one, from inside a ten-minute window. |
|
|
874
884
|
|
|
875
885
|
Every Puma field is nil when no `Puma::Server` exists in the process, and
|
|
876
886
|
every Solid Queue field is nil when `SolidQueue` isn't loaded.
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
// Railwatch browser client for Inertia. Reports each visit's duration,
|
|
2
2
|
// component, and prop payload size to /railwatch/beacon so the platform can
|
|
3
|
-
// show real page-load timing, plus Core Web Vitals (LCP, CLS, INP, TTFB
|
|
4
|
-
//
|
|
3
|
+
// show real page-load timing, plus Core Web Vitals (LCP, CLS, INP, TTFB,
|
|
4
|
+
// with TTFB split into redirect / connection / server time) for the
|
|
5
|
+
// initial page load, plus every JavaScript error the page throws
|
|
5
6
|
// with the breadcrumb trail that led to it.
|
|
6
7
|
// No dependencies -- every metric comes from PerformanceObserver or the
|
|
7
8
|
// navigation timing entry, and every API is feature-detected, so browsers
|
|
@@ -23,6 +24,11 @@ interface Visit {
|
|
|
23
24
|
cls?: number
|
|
24
25
|
inp?: number
|
|
25
26
|
ttfb?: number
|
|
27
|
+
// TTFB split into phases (initial load only; see ttfbPhases).
|
|
28
|
+
redirect_ms?: number
|
|
29
|
+
redirect_count?: number
|
|
30
|
+
connect_ms?: number
|
|
31
|
+
server_ms?: number
|
|
26
32
|
}
|
|
27
33
|
|
|
28
34
|
// The tab's session, for release health. sessionStorage scopes it to the
|
|
@@ -679,6 +685,58 @@ function pageComponent(): string | undefined {
|
|
|
679
685
|
}
|
|
680
686
|
}
|
|
681
687
|
|
|
688
|
+
// Where the initial load's TTFB went, in three phases that add up to it:
|
|
689
|
+
//
|
|
690
|
+
// redirect_ms navigation start -> the final request's start: every
|
|
691
|
+
// redirect hop, plus the browser's own few ms (so never
|
|
692
|
+
// quite 0).
|
|
693
|
+
// connect_ms -> requestStart: service worker startup, HTTP cache, DNS,
|
|
694
|
+
// TCP and TLS for the final request.
|
|
695
|
+
// server_ms -> responseStart: the network round trip and the server's
|
|
696
|
+
// time.
|
|
697
|
+
//
|
|
698
|
+
// The final request starts at fetchStart, or at workerStart when a service
|
|
699
|
+
// worker had to start first (the web-vitals library cuts there too).
|
|
700
|
+
//
|
|
701
|
+
// redirect_ms stays honest when a hop is cross-origin. The browser then
|
|
702
|
+
// zeroes redirectCount, redirectStart and redirectEnd (Navigation Timing 2),
|
|
703
|
+
// but fetchStart is still the final request's start (Fetch's "post-redirect
|
|
704
|
+
// start time"), so the hidden hops' time is still in it.
|
|
705
|
+
//
|
|
706
|
+
// redirect_count is sent only when the browser shows the redirects. A 0
|
|
707
|
+
// means "no redirect" or "hidden", and a hidden redirect must never be
|
|
708
|
+
// reported as no redirect.
|
|
709
|
+
//
|
|
710
|
+
// The boundaries are rounded before subtracting, so the three phases add up
|
|
711
|
+
// to the reported ttfb exactly. A browser whose timestamps are missing or
|
|
712
|
+
// out of order sends no phases rather than a wrong split.
|
|
713
|
+
function ttfbPhases(nav: PerformanceNavigationTiming): Partial<Visit> {
|
|
714
|
+
const fetchStart =
|
|
715
|
+
nav.workerStart > 0 && nav.workerStart < nav.fetchStart
|
|
716
|
+
? nav.workerStart
|
|
717
|
+
: nav.fetchStart
|
|
718
|
+
// Order is judged on the raw timestamps: rounding first would let a
|
|
719
|
+
// requestStart a hair before fetchStart through as a 0 ms phase.
|
|
720
|
+
const ordered =
|
|
721
|
+
nav.responseStart > 0 &&
|
|
722
|
+
fetchStart >= nav.startTime &&
|
|
723
|
+
nav.requestStart >= fetchStart &&
|
|
724
|
+
nav.responseStart >= nav.requestStart
|
|
725
|
+
if (!ordered) return {}
|
|
726
|
+
const [start, fetched, requested, responded] = [
|
|
727
|
+
nav.startTime,
|
|
728
|
+
fetchStart,
|
|
729
|
+
nav.requestStart,
|
|
730
|
+
nav.responseStart,
|
|
731
|
+
].map(Math.round)
|
|
732
|
+
return {
|
|
733
|
+
redirect_ms: fetched - start,
|
|
734
|
+
redirect_count: nav.redirectCount > 0 ? nav.redirectCount : undefined,
|
|
735
|
+
connect_ms: requested - fetched,
|
|
736
|
+
server_ms: responded - requested,
|
|
737
|
+
}
|
|
738
|
+
}
|
|
739
|
+
|
|
682
740
|
// The first page load is a visit too -- it just wasn't routed by Inertia, so
|
|
683
741
|
// the component name comes off the root element's serialized page object.
|
|
684
742
|
function initialVisit(): Visit | null {
|
|
@@ -696,6 +754,7 @@ function initialVisit(): Visit | null {
|
|
|
696
754
|
component,
|
|
697
755
|
duration_ms: duration,
|
|
698
756
|
status: "success",
|
|
757
|
+
...ttfbPhases(nav),
|
|
699
758
|
}
|
|
700
759
|
}
|
|
701
760
|
|
data/lib/railwatch/execution.rb
CHANGED
|
@@ -114,6 +114,7 @@ module Railwatch
|
|
|
114
114
|
@buffered_bytes = 0
|
|
115
115
|
@dropped_records = 0
|
|
116
116
|
@dropped_bytes = 0
|
|
117
|
+
@full = false
|
|
117
118
|
@keep = false
|
|
118
119
|
# Tail sampling keeps buffering child records for a head-sampled-out
|
|
119
120
|
# execution so the ship/discard decision can be made at the end. Read
|
|
@@ -145,8 +146,28 @@ module Railwatch
|
|
|
145
146
|
@paused_depth.positive?
|
|
146
147
|
end
|
|
147
148
|
|
|
149
|
+
# The gate every subscriber asks before it builds a child record. Once
|
|
150
|
+
# the tree has overflowed (#buffer) the answer is no, and the record the
|
|
151
|
+
# caller was about to build is counted as dropped right here: building
|
|
152
|
+
# it -- SQL normalisation, redaction, a backtrace walk, weighing -- only
|
|
153
|
+
# to throw it away cost ~15us per query, and a job issuing 5,000 queries
|
|
154
|
+
# into an 8 MiB buffer drops most of them. A gate with a side effect,
|
|
155
|
+
# deliberately: each caller asks once per record it would build, so the
|
|
156
|
+
# count stays exact. Bytes are estimated at the buffer's mean record
|
|
157
|
+
# weight (never a division by zero: @full is only set once a record is
|
|
158
|
+
# held), so the dropped-bytes figure keeps meaning something.
|
|
148
159
|
def recording?
|
|
149
|
-
(sampled? || @tail_buffering) && !paused?
|
|
160
|
+
return false unless (sampled? || @tail_buffering) && !paused?
|
|
161
|
+
return true unless @full
|
|
162
|
+
|
|
163
|
+
@dropped_records += 1
|
|
164
|
+
@dropped_bytes += @buffered_bytes / @records.size
|
|
165
|
+
false
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
# Whether the buffer has overflowed and #recording? is now refusing.
|
|
169
|
+
def full?
|
|
170
|
+
@full
|
|
150
171
|
end
|
|
151
172
|
|
|
152
173
|
# Ship this execution's whole tree regardless of the head sampling
|
|
@@ -222,13 +243,16 @@ module Railwatch
|
|
|
222
243
|
|
|
223
244
|
# A failure-context ring keeps the LAST record_limit records: the ones
|
|
224
245
|
# just before the exception are the ones worth having. Every other
|
|
225
|
-
# buffer keeps the earliest and rejects the overflow
|
|
226
|
-
#
|
|
246
|
+
# buffer keeps the earliest and rejects the overflow -- and from the
|
|
247
|
+
# first rejection on, refuses at the gate (#recording?) so nothing
|
|
248
|
+
# else is built for it. Either way the loss is counted onto the parent
|
|
249
|
+
# record and its batch (Railwatch.finish_execution).
|
|
227
250
|
if @failure_context
|
|
228
251
|
drop_oldest while @records.any? && (@records.size >= @record_limit || @buffered_bytes + bytes > @byte_limit)
|
|
229
252
|
elsif @records.size >= @record_limit || @buffered_bytes + bytes > @byte_limit
|
|
230
253
|
@dropped_records += 1
|
|
231
254
|
@dropped_bytes += bytes
|
|
255
|
+
@full = true
|
|
232
256
|
return
|
|
233
257
|
end
|
|
234
258
|
|
|
@@ -279,6 +303,14 @@ module Railwatch
|
|
|
279
303
|
@counters[counter] += by
|
|
280
304
|
end
|
|
281
305
|
|
|
306
|
+
# The counters that actually fired. Most executions touch a handful of the
|
|
307
|
+
# 18, so sending the rest as explicit zeros costs roughly 240 bytes on the
|
|
308
|
+
# wire and in storage, per execution, to say nothing happened. A reader
|
|
309
|
+
# treats an absent counter as zero, exactly as it treats a zero.
|
|
310
|
+
def counted
|
|
311
|
+
@counters.reject { |_, value| value.zero? }
|
|
312
|
+
end
|
|
313
|
+
|
|
282
314
|
# Rails.error and an outer middleware can observe the same error. Count
|
|
283
315
|
# that occurrence once even when sampling or pause prevents its report.
|
|
284
316
|
# Reporting has separate flags so a suppressed observation can still be
|
data/lib/railwatch/health.rb
CHANGED
|
@@ -19,6 +19,8 @@ module Railwatch
|
|
|
19
19
|
@thread = nil
|
|
20
20
|
@pid = nil
|
|
21
21
|
@stopping = false
|
|
22
|
+
@manifest_digest = nil
|
|
23
|
+
@manifest_sent_at = nil
|
|
22
24
|
|
|
23
25
|
module_function
|
|
24
26
|
|
|
@@ -51,6 +53,8 @@ module Railwatch
|
|
|
51
53
|
@thread = nil
|
|
52
54
|
@pid = nil
|
|
53
55
|
@stopping = false
|
|
56
|
+
@manifest_digest = nil
|
|
57
|
+
@manifest_sent_at = nil
|
|
54
58
|
remove_instance_variable(:@puma_server) if defined?(@puma_server)
|
|
55
59
|
start!
|
|
56
60
|
end
|
|
@@ -81,6 +85,7 @@ module Railwatch
|
|
|
81
85
|
puma = puma_stats
|
|
82
86
|
pool = pool_stats
|
|
83
87
|
queue = solid_queue_stats
|
|
88
|
+
fields = detail(puma, queue)
|
|
84
89
|
Railwatch.record(:health,
|
|
85
90
|
pid: Process.pid,
|
|
86
91
|
role: Subscribers::ProcessInfo.role,
|
|
@@ -93,7 +98,10 @@ module Railwatch
|
|
|
93
98
|
pool_waiting: pool[:waiting],
|
|
94
99
|
queue_depth: queue[:queue_depth],
|
|
95
100
|
queue_latency: queue[:queue_latency],
|
|
96
|
-
detail: JSON.generate(
|
|
101
|
+
detail: JSON.generate(fields))
|
|
102
|
+
# Only once the sample is recorded: a sample that raised must not
|
|
103
|
+
# count as having sent the manifest, or the next one leaves it out.
|
|
104
|
+
manifest_sent(fields[:recurring_tasks]) if fields[:recurring_tasks]
|
|
97
105
|
end
|
|
98
106
|
rescue StandardError => e
|
|
99
107
|
Railwatch.debug { "health sample failed: #{e.class}: #{e.message}" }
|
|
@@ -122,13 +130,41 @@ module Railwatch
|
|
|
122
130
|
# with scheduled-task detection). Left out rather than sent empty when
|
|
123
131
|
# there are none or the table could not be read: Jobs folds a failed
|
|
124
132
|
# read into an empty set, and "no manifest" must not read as "no tasks".
|
|
133
|
+
#
|
|
134
|
+
# The manifest is a property of the deploy, not of the sample: it is
|
|
135
|
+
# identical in every process and changes only when config/recurring.yml
|
|
136
|
+
# does. Sending it every interval made it 94% of every health sample --
|
|
137
|
+
# 1,312 of 1,417 bytes per row on a 28-task app, the same string stored
|
|
138
|
+
# 32,694 times over nineteen hours. So it is re-sent when it changes, and
|
|
139
|
+
# otherwise only once per MANIFEST_INTERVAL. Nothing is lost: the
|
|
140
|
+
# server's scheduled-task check takes the newest sample that *carries* a
|
|
141
|
+
# manifest from the last ten minutes, and the floor is half that.
|
|
125
142
|
def recurring_tasks
|
|
126
143
|
schedules = Subscribers::Jobs.recurring_tasks[:schedules]
|
|
127
|
-
|
|
144
|
+
return nil if schedules.empty?
|
|
145
|
+
return nil unless manifest_due?(schedules)
|
|
146
|
+
|
|
147
|
+
schedules
|
|
128
148
|
rescue StandardError
|
|
129
149
|
nil
|
|
130
150
|
end
|
|
131
151
|
|
|
152
|
+
# Five minutes, as a duration rather than a count of samples:
|
|
153
|
+
# health_interval is configurable, so any count would outrun the reader's
|
|
154
|
+
# ten-minute window once an app slowed its sampling down.
|
|
155
|
+
MANIFEST_INTERVAL = 300
|
|
156
|
+
|
|
157
|
+
# Monotonic, so a clock step cannot park the manifest for five minutes or
|
|
158
|
+
# put it on every sample. Read and written only from the health thread.
|
|
159
|
+
def manifest_due?(schedules)
|
|
160
|
+
!(schedules.hash == @manifest_digest && @manifest_sent_at && Clock.monotonic - @manifest_sent_at < MANIFEST_INTERVAL)
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
def manifest_sent(schedules)
|
|
164
|
+
@manifest_digest = schedules.hash
|
|
165
|
+
@manifest_sent_at = Clock.monotonic
|
|
166
|
+
end
|
|
167
|
+
|
|
132
168
|
# Puma::Server#stats is the only public API exposing busy_threads, so it is
|
|
133
169
|
# used in preference to the individual readers. It also resets Puma's
|
|
134
170
|
# own since-last-read backlog_max/reactor_max gauges, which Railwatch does
|
|
@@ -33,8 +33,10 @@ module Railwatch
|
|
|
33
33
|
subscribe("#{kind}.action_view") do |event|
|
|
34
34
|
exe = execution
|
|
35
35
|
exe&.count(:view_renders)
|
|
36
|
-
|
|
36
|
+
# The cap first: a render past it is never built, so a full
|
|
37
|
+
# buffer's gate (Execution#recording?) must not count it dropped.
|
|
37
38
|
next if exe && exe.counters[:view_renders] > Railwatch.config.max_view_renders_per_execution
|
|
39
|
+
next unless recording?
|
|
38
40
|
p = event.payload
|
|
39
41
|
identifier = p[:identifier].to_s.delete_prefix(Backtrace.app_root)
|
|
40
42
|
Railwatch.record(:view_render,
|
data/lib/railwatch/version.rb
CHANGED
|
@@ -23,23 +23,9 @@
|
|
|
23
23
|
},
|
|
24
24
|
"counters": {
|
|
25
25
|
"queries": 2,
|
|
26
|
-
"cached_queries": 0,
|
|
27
|
-
"exceptions": 0,
|
|
28
26
|
"logs": 1,
|
|
29
|
-
"cache_events": 0,
|
|
30
|
-
"jobs_enqueued": 0,
|
|
31
|
-
"mail": 0,
|
|
32
|
-
"broadcasts": 0,
|
|
33
|
-
"notifications": 0,
|
|
34
|
-
"outgoing_requests": 0,
|
|
35
|
-
"storage_ops": 0,
|
|
36
27
|
"view_renders": 1,
|
|
37
|
-
"
|
|
38
|
-
"hydrated_models": 1,
|
|
39
|
-
"lazy_loads": 0,
|
|
40
|
-
"deprecations": 0,
|
|
41
|
-
"spans": 0,
|
|
42
|
-
"llm_calls": 0
|
|
28
|
+
"hydrated_models": 1
|
|
43
29
|
},
|
|
44
30
|
"peak_memory": 1,
|
|
45
31
|
"allocations": 1,
|
|
@@ -99,23 +85,7 @@
|
|
|
99
85
|
},
|
|
100
86
|
"counters": {
|
|
101
87
|
"queries": 1,
|
|
102
|
-
"
|
|
103
|
-
"exceptions": 0,
|
|
104
|
-
"logs": 1,
|
|
105
|
-
"cache_events": 0,
|
|
106
|
-
"jobs_enqueued": 0,
|
|
107
|
-
"mail": 0,
|
|
108
|
-
"broadcasts": 0,
|
|
109
|
-
"notifications": 0,
|
|
110
|
-
"outgoing_requests": 0,
|
|
111
|
-
"storage_ops": 0,
|
|
112
|
-
"view_renders": 0,
|
|
113
|
-
"transactions": 0,
|
|
114
|
-
"hydrated_models": 0,
|
|
115
|
-
"lazy_loads": 0,
|
|
116
|
-
"deprecations": 0,
|
|
117
|
-
"spans": 0,
|
|
118
|
-
"llm_calls": 0
|
|
88
|
+
"logs": 1
|
|
119
89
|
},
|
|
120
90
|
"peak_memory": 1,
|
|
121
91
|
"allocations": 1,
|
|
@@ -160,23 +130,7 @@
|
|
|
160
130
|
},
|
|
161
131
|
"counters": {
|
|
162
132
|
"queries": 1,
|
|
163
|
-
"
|
|
164
|
-
"exceptions": 0,
|
|
165
|
-
"logs": 1,
|
|
166
|
-
"cache_events": 0,
|
|
167
|
-
"jobs_enqueued": 0,
|
|
168
|
-
"mail": 0,
|
|
169
|
-
"broadcasts": 0,
|
|
170
|
-
"notifications": 0,
|
|
171
|
-
"outgoing_requests": 0,
|
|
172
|
-
"storage_ops": 0,
|
|
173
|
-
"view_renders": 0,
|
|
174
|
-
"transactions": 0,
|
|
175
|
-
"hydrated_models": 0,
|
|
176
|
-
"lazy_loads": 0,
|
|
177
|
-
"deprecations": 0,
|
|
178
|
-
"spans": 0,
|
|
179
|
-
"llm_calls": 0
|
|
133
|
+
"logs": 1
|
|
180
134
|
},
|
|
181
135
|
"peak_memory": 1,
|
|
182
136
|
"allocations": 1,
|
|
@@ -222,23 +176,7 @@
|
|
|
222
176
|
"stages": {},
|
|
223
177
|
"counters": {
|
|
224
178
|
"queries": 1,
|
|
225
|
-
"
|
|
226
|
-
"exceptions": 0,
|
|
227
|
-
"logs": 0,
|
|
228
|
-
"cache_events": 0,
|
|
229
|
-
"jobs_enqueued": 0,
|
|
230
|
-
"mail": 0,
|
|
231
|
-
"broadcasts": 0,
|
|
232
|
-
"notifications": 0,
|
|
233
|
-
"outgoing_requests": 0,
|
|
234
|
-
"storage_ops": 0,
|
|
235
|
-
"view_renders": 0,
|
|
236
|
-
"transactions": 1,
|
|
237
|
-
"hydrated_models": 0,
|
|
238
|
-
"lazy_loads": 0,
|
|
239
|
-
"deprecations": 0,
|
|
240
|
-
"spans": 0,
|
|
241
|
-
"llm_calls": 0
|
|
179
|
+
"transactions": 1
|
|
242
180
|
},
|
|
243
181
|
"peak_memory": 1,
|
|
244
182
|
"allocations": 1,
|
|
@@ -267,26 +205,7 @@
|
|
|
267
205
|
"tenant": null,
|
|
268
206
|
"duration": 1,
|
|
269
207
|
"stages": {},
|
|
270
|
-
"counters": {
|
|
271
|
-
"queries": 0,
|
|
272
|
-
"cached_queries": 0,
|
|
273
|
-
"exceptions": 0,
|
|
274
|
-
"logs": 0,
|
|
275
|
-
"cache_events": 0,
|
|
276
|
-
"jobs_enqueued": 0,
|
|
277
|
-
"mail": 0,
|
|
278
|
-
"broadcasts": 0,
|
|
279
|
-
"notifications": 0,
|
|
280
|
-
"outgoing_requests": 0,
|
|
281
|
-
"storage_ops": 0,
|
|
282
|
-
"view_renders": 0,
|
|
283
|
-
"transactions": 0,
|
|
284
|
-
"hydrated_models": 0,
|
|
285
|
-
"lazy_loads": 0,
|
|
286
|
-
"deprecations": 0,
|
|
287
|
-
"spans": 0,
|
|
288
|
-
"llm_calls": 0
|
|
289
|
-
},
|
|
208
|
+
"counters": {},
|
|
290
209
|
"peak_memory": 1,
|
|
291
210
|
"allocations": 1,
|
|
292
211
|
"gc_time": 1,
|
|
@@ -686,6 +605,10 @@
|
|
|
686
605
|
"cls": null,
|
|
687
606
|
"inp": null,
|
|
688
607
|
"ttfb": null,
|
|
608
|
+
"redirect_ms": null,
|
|
609
|
+
"redirect_count": null,
|
|
610
|
+
"connect_ms": null,
|
|
611
|
+
"server_ms": null,
|
|
689
612
|
"user": null,
|
|
690
613
|
"tenant": null,
|
|
691
614
|
"user_agent": ""
|
data/lib/railwatch.rb
CHANGED
|
@@ -167,6 +167,13 @@ module Railwatch
|
|
|
167
167
|
group ||= fields.delete(:group)
|
|
168
168
|
fields[:tail_sampled] = true if tail
|
|
169
169
|
fields[:profiled] = true if profiled
|
|
170
|
+
# What the tree lost to its own buffer (Execution#buffer), so the
|
|
171
|
+
# parent says so where someone opening it will look. Only on a tree
|
|
172
|
+
# that ships: one that is discarded lost nothing anyone would see.
|
|
173
|
+
if shipping && exe.dropped_records.positive?
|
|
174
|
+
fields[:dropped_records] = exe.dropped_records
|
|
175
|
+
fields[:dropped_bytes] = exe.dropped_bytes
|
|
176
|
+
end
|
|
170
177
|
parent = build_parent(parent_type, exe, group: group, **fields)
|
|
171
178
|
end
|
|
172
179
|
if shipping
|
|
@@ -224,7 +231,7 @@ module Railwatch
|
|
|
224
231
|
def build_parent(type, exe, group: nil, **fields)
|
|
225
232
|
rec = Record.build(type, exe, group: group, timestamp: exe.started_at,
|
|
226
233
|
duration: exe.duration, stages: exe.stage_durations.compact.transform_keys(&:to_s),
|
|
227
|
-
counters: exe.
|
|
234
|
+
counters: exe.counted, peak_memory: exe.peak_memory, allocations: exe.allocations,
|
|
228
235
|
gc_time: exe.gc_time, exception_preview: exe.exception_preview,
|
|
229
236
|
context: Context.serialized, **fields)
|
|
230
237
|
run_redactors(type, rec)
|