railwatch 0.9.2 → 0.9.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f68b0938b3d6520230abbfb7fbb4f4fac4ed1f052a6d8a4c57c24152ef364d27
-  data.tar.gz: a6807b8d823cadb1c02d5f22ae045694470aa4e274b998549168b7ce89b2ee22
+  metadata.gz: bfbb6b3ab550297baaa042889c6e98fe0b5fc1528acf6696fa343631b05936bd
+  data.tar.gz: 37b7b7bce05a1e0b19ed8051f9dcc7cd09f3ca0c8d638404c9fa55bb227c44c4
 SHA512:
-  metadata.gz: eb3ba23b7ecb61dda1792c3d19e6dfe3d6a8f94d87159d476962e6b4430e6914aa88517fe4bb52a1440415948df1fb408f99bf028e546b317e427e86d6561078
-  data.tar.gz: 9fc99ae33e5b538ef7ac8c92a043fa8ee688b7869ae344109254334654a1b96a7b7f048078b0c97ffaf165208b98128c2eae9f8556e4b219db08dbea91036868
+  metadata.gz: dc66f26f906bdab5b2be01c151c81e40767004508f94c1c7860d0a590272daa7469ec3a1a66cb8d5992e1f8ff7f080d6280049afe932b0582859b693c2801655
+  data.tar.gz: 9296b26cf0c6c0eb837fdc85f7b69cd4f589c2ebfe5579e30401d3b16819e653dd86ad52764cff5d2ab6e768bf72c5b505ec4255d128a0f3334bfcefe0c41b72

data/CHANGELOG.md

@@ -6,6 +6,31 @@
      filled in by the release commit, which is also the only commit that
      touches lib/railwatch/version.rb and Gemfile.lock. See CONTRIBUTING.md. -->
 
+## 0.9.4 (2026-10-22)
+
+- Sampling recovers as soon as ingest does. A receiver refusing with a
+  `Retry-After` (a Railwatch server below its disk floor answers 503 with
+  `Retry-After: 60`) kept the app at 1/8 sampling for 50-60 s after it was
+  accepting again: the factor only halved per accepted delivery, and those
+  waited on 32-60 s backoff rungs. A refusal is now retried after its
+  `Retry-After` or 7.5-15 s, whichever is sooner (never sooner than the
+  backoff rung), and the first accepted delivery puts the factor back to
+  1x.
+- A refusal is no longer silent. One `[railwatch]` line when ingest starts
+  refusing, and one when it accepts again, with how many executions
+  backpressure sampled out and how many records were dropped meanwhile.
+  Both go through `on_unrecoverable` / `warn_on_data_loss`, like lost
+  batches.
+
+## 0.9.3 (2026-10-18)
+
+- The dashboard's route helpers and its browser beacon follow the
+  `railwatch-mount` meta tag wherever the page is served from: the embedded
+  engine mounted somewhere other than `/railwatch`, or a Railwatch server
+  mounted inside another app under a base path. The mount is read by one
+  helper (`lib/mount.ts`) that refuses anything but a single absolute path, so
+  a tag like `//host` cannot point links or the beacon at another origin.
+
 ## 0.9.2 (2026-10-17)
 
 - Changing the dashboard's window (1h/6h/24h/7d, a custom range, step or

data/app/frontend/entrypoints/inertia.tsx

@@ -5,16 +5,13 @@ import { createRoot } from "react-dom/client"
 
 import { initializeTheme } from "@/hooks/use-appearance"
 import PersistentLayout from "@/layouts/persistent-layout"
+import { applyMount } from "@/lib/mount"
 import { railwatchRootOptions, startRailwatch } from "@/lib/railwatch"
-import { configure as configureRoutes } from "@/routes"
 
-// When served by the embedded engine, every route helper must carry the
-// engine's mount point. The engine layout emits it; the hosted platform does
-// not, so this is a no-op there.
-const mount = document.querySelector<HTMLMetaElement>(
-  'meta[name="railwatch-mount"]',
-)?.content
-if (mount) configureRoutes({ default_url_options: { script_name: mount } })
+// When served under a mount point (the embedded engine, or a Railwatch server
+// mounted inside another app), every route helper must carry it. The layout
+// emits it; the hosted platform does not, so this is a no-op there.
+applyMount()
 
 const appName = import.meta.env.VITE_APP_NAME ?? "Railwatch"
 

data/app/frontend/lib/mount.ts

@@ -0,0 +1,23 @@
+import { configure } from "@/routes"
+
+// The path the dashboard is served under, from the layout's
+// <meta name="railwatch-mount">: the embedded engine's mount point
+// (request.script_name), or the base path of a Railwatch server mounted
+// inside another app. "" without the tag (the hosted platform) or for
+// anything that is not one absolute path: "//host" or "https://..." would
+// point every link at another origin.
+export function mountPath(doc: Document = document): string {
+  const raw =
+    doc.querySelector<HTMLMetaElement>('meta[name="railwatch-mount"]')
+      ?.content ?? ""
+  const path = raw.trim().replace(/\/+$/, "")
+  return /^\/(?![/\\])/.test(path) ? path : ""
+}
+
+// Every route helper carries the mount point (js-routes' script_name).
+// Returns the mount, "" when there is none and nothing changed.
+export function applyMount(doc: Document = document): string {
+  const mount = mountPath(doc)
+  if (mount) configure({ default_url_options: { script_name: mount } })
+  return mount
+}

data/app/frontend/lib/railwatch.ts

@@ -127,8 +127,17 @@ let foregroundRequest: { id?: string } | null = null
 // whether or not the first was reported.
 const requestFailures = new WeakSet<object>()
 
+// The engine's beacon, under the dashboard's mount point when the page has
+// one (<meta name="railwatch-mount">, lib/mount.ts; inlined because this
+// file is copied into apps on its own). An app's own pages have no such tag
+// and post to the engine at its default mount.
 function endpoint() {
-  return "/railwatch/beacon"
+  const mount =
+    document
+      .querySelector<HTMLMetaElement>('meta[name="railwatch-mount"]')
+      ?.content.trim()
+      .replace(/\/+$/, "") ?? ""
+  return /^\/(?![/\\])/.test(mount) ? `${mount}/beacon` : "/railwatch/beacon"
 }
 
 function csrf() {

data/docs/configuration.md

@@ -411,7 +411,12 @@ result without inserting the payload twice. Records written while a request is i
 separate bounded buffer, so they never change the retained request's
 identity. At most one retained batch plus one live buffer are held in
 memory. The reporter retries with jittered exponential backoff, from one
-second up to 60 seconds. It does not busy-loop. A 401 marks the transport
+second up to 60 seconds. It does not busy-loop. A response with a
+`Retry-After` (a 503 for a full disk or busy database, a 429 rate limit) is a
+refusal: the next attempt waits for its `Retry-After` instead, but never
+less than the backoff rung and never more than 15 seconds (jittered down to
+7.5), because Railwatch's receivers send a fixed 60 there, not a forecast.
+A 401 marks the transport
 permanently unauthorized: no further HTTP attempts for the process's
 lifetime. It and other permanent client rejections are reported through
 `on_unrecoverable`. Delivery never raises into app code.
@@ -433,6 +438,15 @@ affects one probabilistic decision. Set `backpressure` false to keep the
 factor at 1. The current value is sent as
 `X-Railwatch-Backpressure-Factor` whenever it is greater than 1.
 
+A refusal from ingest (see above) sets the factor straight to 8x, and the
+first delivery accepted after it sets it straight back to 1x, so sampling
+recovers within one retry of the receiver recovering. Each refusal prints
+two `[railwatch]` lines, the same way lost records do (see
+`warn_on_data_loss`): one when it starts, and one when ingest accepts
+again, with how long it lasted, how many executions backpressure sampled
+out that the configured rates would have kept, and how many records were
+dropped.
+
 `Railwatch.flush` forces an immediate flush, on the calling thread and
 without a time limit, so it is yours to call when you know you want to
 wait. Nothing in the gem calls it for you: a short-lived process keeps its
@@ -827,16 +841,19 @@ Railwatch.on_unrecoverable { |error| Rails.error.report(error, handled: true) }
 ```
 
 Called whenever Railwatch rescues one of its own internal errors, ingest
-permanently rejects a batch, or shutdown expires with retained records
-that could not be sent. Retryable delivery failures stay buffered and do
-not fire the callback on every attempt. With no callback registered, a
+permanently rejects a batch, ingest starts or stops refusing deliveries, or
+shutdown expires with retained records that could not be sent. Retryable
+delivery failures stay buffered and do not fire the callback on every
+attempt. With no callback registered, a
 recovered internal error falls back to `Railwatch.debug`, gated on
 `RAILWATCH_DEBUG` — the gem carried on and there is nothing to do about it.
 
 **Lost records are different, and are reported by default.** A batch
 dropped after its retry ladder, one the receiver permanently refused, or
 records still unsent when the bounded shutdown ran out of time each print
-one `[railwatch]` stderr line. Telemetry that vanishes silently looks
+one `[railwatch]` stderr line. So do the start and end of an ingest
+refusal, which samples down until it is over (see backpressure above).
+Telemetry that vanishes silently looks
 exactly like having nothing to report, and a short-lived process — a rake
 task, a `rails runner`, a cron job — gets one bounded shutdown and no
 second chance to mention it.

data/lib/railwatch/buffer.rb

@@ -20,6 +20,10 @@ module Railwatch
       @bytes = 0
       @dropped = 0
       @dropped_bytes = 0
+      # Every record this queue has ever had to drop for room. Unlike
+      # @dropped it is never drained, so a caller can difference it across a
+      # span of time.
+      @overflowed = 0
       @mutex = Mutex.new
     end
 
@@ -58,6 +62,10 @@ module Railwatch
       @mutex.synchronize { @bytes }
     end
 
+    def overflowed
+      @mutex.synchronize { @overflowed }
+    end
+
     def stats
       @mutex.synchronize { [ @records.size, @dropped, @bytes, @dropped_bytes ] }
     end
@@ -110,6 +118,7 @@ module Railwatch
     def drop(bytes)
       @dropped += 1
       @dropped_bytes += bytes
+      @overflowed += 1
     end
 
     def drop_oldest

data/lib/railwatch/reporter.rb

@@ -13,7 +13,8 @@ module Railwatch
     # counted) so the buffer's newest records win again. Without the cap a
     # batch that keeps failing would be pinned forever while everything
     # newer was discarded around it. Eight attempts on the backoff ladder is
-    # roughly four minutes of outage.
+    # roughly four minutes of outage; one to two when every attempt is
+    # refused with a Retry-After (MAX_REFUSED_RETRY_DELAY).
     MAX_RETRY_ATTEMPTS = 8
     # An unhandled exception asks for an immediate flush so it reaches the
     # platform without waiting out flush_interval. "Immediate" is this many
@@ -26,7 +27,23 @@ module Railwatch
     # is enough to turn a saturated stream into breathing room without the
     # long recovery and sparse telemetry a 16x cap would impose.
     MAX_BACKPRESSURE_FACTOR = 8.0
+    # A receiver that answers with a Retry-After (503 "disk almost full" or
+    # "database busy", 429 "rate limit exceeded") is refusing, not failing.
+    # The next attempt waits for its Retry-After instead of the ladder, up
+    # to this long. Railwatch's receivers send 15 s for a busy database,
+    # honoured as given, and 60 s for the disk floor and the rate-limit
+    # window, which are not forecasts: the floor is re-checked every 10 s
+    # and lifts the moment space is back, and a window can end at any time.
+    # The ladder alone was worse than either: a minute into a refusal its
+    # rungs were 16-60 s apart, and a recovered app stayed sampled down
+    # until the next one. A longer wait is cut to between half and all of
+    # this, jittered so processes refused together do not return together:
+    # one POST per process every 7.5-15 s.
+    MAX_REFUSED_RETRY_DELAY = 15.0
     DeliveryBatch = Data.define(:id, :records, :bytes, :dropped, :dropped_bytes, :prepared)
+    # From the first refusal to the next accepted delivery: when it began,
+    # what the receiver said, and the loss counters as they stood then.
+    Refusal = Struct.new(:started_at, :reason, :overflowed, :sampled_out, :gave_up)
 
     class DeliveryError < StandardError
       attr_reader :status, :records, :bytes, :dropped, :dropped_bytes
@@ -66,6 +83,8 @@ module Railwatch
       @in_flight_bytes = 0
       @in_flight_dropped_bytes = 0
       @shutdown_notified = false
+      @refusal = nil
+      @sampled_out = 0
     end
 
     def buffer
@@ -75,6 +94,13 @@ module Railwatch
 
     attr_reader :backpressure_factor
 
+    # The sampler calls this for an execution the configured rate kept and
+    # backpressure did not. Only reached while the factor is above one.
+    def note_sampled_out
+      ensure_process!
+      @mutex.synchronize { @sampled_out += 1 }
+    end
+
     def write(record, bytes = nil)
       ensure_process!
       size = @buffer.push(record, bytes)
@@ -146,6 +172,8 @@ module Railwatch
       @in_flight_bytes = 0
       @in_flight_dropped_bytes = 0
       @shutdown_notified = false
+      @refusal = nil
+      @sampled_out = 0
       remove_instance_variable(:@shutdown_deadline) if defined?(@shutdown_deadline)
       self
     end
@@ -348,6 +376,7 @@ module Railwatch
         # raise here and be retained as if it had failed.
         Railwatch.debug { "ingest rejected #{result.rejected} of #{batch.records.size} records" } if result.rejected.to_i.positive?
         delivery_succeeded
+        refusal_ended if @refusal
       elsif retryable?(result)
         retain(batch, result)
       else
@@ -370,22 +399,33 @@ module Railwatch
     end
 
     def retain(batch, result)
+      refused_until = result.retry_after_at if result.respond_to?(:retry_after_at)
+      refusal = nil
       gave_up = @mutex.synchronize do
         @in_flight_records = 0
         @in_flight_dropped = 0
         @in_flight_bytes = 0
         @in_flight_dropped_bytes = 0
+        if refused_until && !@refusal
+          refusal = @refusal = Refusal.new(started_at: Clock.monotonic, reason: failure_detail(result),
+                                           overflowed: @buffer.overflowed, sampled_out: @sampled_out, gave_up: 0)
+          # Nothing leaves this buffer until the receiver accepts again, and
+          # with retries now seconds apart, so are the flush ticks that would
+          # otherwise double the factor up to here. They hold it there.
+          @backpressure_factor = MAX_BACKPRESSURE_FACTOR if @config.backpressure
+        end
         @retry_attempt += 1
         if @retry_attempt > MAX_RETRY_ATTEMPTS
           @buffer.account_dropped(batch.records.size + batch.dropped,
                                   bytes: batch.bytes + batch.dropped_bytes)
+          @refusal.gave_up += batch.records.size if @refusal
           @retry_attempt = 0
           @retry_at = nil
           Railwatch.debug { "gave up on a batch of #{batch.records.size} records after #{MAX_RETRY_ATTEMPTS} retries (#{result.error || result.status}); dropped and counted" }
           next true
         end
         @retry_batch = batch
-        delay = retry_delay(@retry_attempt)
+        delay = refused_until ? refused_delay(refused_until) : retry_delay(@retry_attempt)
         @retry_at = Clock.monotonic + delay
         @wakeup.signal
         Railwatch.debug do
@@ -394,6 +434,17 @@ module Railwatch
         end
         false
       end
+      # From here backpressure samples out executions the app asked to keep,
+      # and the receiver refusing them has no way to count them. Said once
+      # per refusal, and refusal_ended says what it cost.
+      if refusal
+        defer_notification(
+          DeliveryError.new("Railwatch ingest is refusing deliveries (#{refusal.reason}); holding " \
+                            "#{batch.records.size} #{batch.records.size == 1 ? "record" : "records"}" \
+                            "#{" and sampling down" if @config.backpressure} until it accepts again",
+                            status: result.status)
+        )
+      end
       return unless gave_up
 
       # Losing a batch is not a debug-level event: with an ingest (or an
@@ -420,6 +471,42 @@ module Railwatch
       )
     end
 
+    # The first delivery ingest accepted after refusing. The refusal was the
+    # pressure, so the divisor goes straight back to one, as it went up,
+    # rather than halving over three more flush ticks. A buffer still over
+    # its high-water mark raises it again on the next tick, as it would have
+    # anyway.
+    def refusal_ended
+      refusal, sampled_out, overflowed = @mutex.synchronize do
+        ended = @refusal
+        @refusal = nil
+        [ ended, @sampled_out - ended.sampled_out, @buffer.overflowed - ended.overflowed ]
+      end
+      @backpressure_factor = 1.0
+      dropped = overflowed + refusal.gave_up
+      seconds = (Clock.monotonic - refusal.started_at).round
+      defer_notification(
+        DeliveryError.new("Railwatch ingest is accepting deliveries again after refusing for #{seconds}s; " \
+                          "meanwhile backpressure sampled out #{sampled_out} " \
+                          "#{sampled_out == 1 ? "execution" : "executions"} and #{dropped} " \
+                          "#{dropped == 1 ? "record was" : "records were"} dropped",
+                          records: dropped)
+      )
+    end
+
+    # Never sooner than the ladder's rung either, so a receiver answering
+    # "Retry-After: 0" gets backoff, not a loop.
+    def refused_delay(retry_after_at)
+      wanted = retry_after_at - Time.now
+      return MAX_REFUSED_RETRY_DELAY * (0.5 + @random.rand * 0.5) if wanted > MAX_REFUSED_RETRY_DELAY
+
+      [ wanted, retry_delay(@retry_attempt) ].max.clamp(0.0, MAX_REFUSED_RETRY_DELAY)
+    end
+
+    def failure_detail(result)
+      [ result.status && "HTTP #{result.status}", result.error ].compact.reject { |part| part.to_s.empty? }.join(": ")
+    end
+
     def discard_unauthorized
       batch = drain_into_flight
       delivery_succeeded

data/lib/railwatch/sampler.rb

@@ -8,10 +8,18 @@ module Railwatch
     module_function
 
     def decide(kind)
-      rate = Railwatch.config.sample_rate(kind) / Railwatch.reporter.backpressure_factor
+      configured = Railwatch.config.sample_rate(kind)
+      rate = configured / Railwatch.reporter.backpressure_factor
       return true if rate >= 1.0
       return false if rate <= 0.0
-      Random.rand < rate
+
+      roll = Random.rand
+      return true if roll < rate
+
+      # Kept by the configured rate, lost to backpressure: counted, so the
+      # reporter can say what an ingest refusal cost once it is over.
+      Railwatch.reporter.note_sampled_out if roll < configured
+      false
     end
   end
 end

data/lib/railwatch/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Railwatch
-  VERSION = "0.9.2"
+  VERSION = "0.9.4"
 end

data/lib/railwatch.rb

@@ -355,8 +355,10 @@ module Railwatch
     # the gem recovered, and a subscriber that raises once is noise in
     # someone else's cron output. A DeliveryError is different. It is raised
     # only when records are already gone -- a batch dropped after the retry
-    # ladder, one the receiver permanently refused, or the ones still unsent
-    # when at_exit's bounded shutdown ran out of time -- and since 0.3.7 that
+    # ladder, one the receiver permanently refused, the ones still unsent
+    # when at_exit's bounded shutdown ran out of time, or the executions an
+    # ingest refusal sampled out (said when it starts, counted when it
+    # ends) -- and since 0.3.7 that
     # shutdown is the only delivery a rake task or runner gets. Behind
     # RAILWATCH_DEBUG that loss was invisible: a cron job whose exception
     # never reached the platform looked exactly like one that had nothing to