railwatch 1.3.0 → 1.5.0

data/lib/generators/railwatch/agents/templates/agents_block.md.tt

@@ -0,0 +1,26 @@
+## Railwatch (performance and errors, in development)
+
+Every request, job and command this app runs in development is recorded by
+the `railwatch` gem into the server `railwatch-server` runs beside it: SQL
+(with the `file:line` that ran it), N+1s, views, logs and exceptions, all
+linked to one execution.
+
+- MCP server `<%= SERVER_NAME %>` at <%= mcp_url %> (in `.mcp.json`). No token from
+  this machine in development. Tools take `application_slug: "<%= app_slug %>"`
+  and `environment: "development"`; `list_applications` lists them.
+- Dashboard, for the human: <%= dashboard_url %>
+
+The loop after a change:
+
+1. Hit the page: `curl -si <%= app_url %>/path | grep -i -e server-timing -e x-railwatch`.
+   `Server-Timing` has total, db (with the query count), view and other;
+   `X-Railwatch-Execution-Id` names the execution.
+2. `get_execution` with that id, or `last_execution` (optionally with a
+   `path`): a summary of where the time went, the repeated queries and N+1s
+   with their `source` (`file:line`), then a page of the timeline.
+3. `get_route` for the route: its N+1s, each with the source line and an
+   `includes` to add, and its slowest queries.
+4. Fix, hit the page again, compare the query count and db time.
+
+`search_requests`, `list_slow_routes`, `list_issues` and `get_issue` cover the
+rest.

data/lib/generators/railwatch/install/install_generator.rb

@@ -23,11 +23,12 @@ module Railwatch
       class_option :doctor, type: :boolean, default: true,
                             desc: "Run railwatch:doctor when the install finishes."
 
-      # Whichever of these exists is where `startRailwatch()` is added.
+      # Whichever of these exists, under the app's frontend source directory
+      # (Railwatch.frontend_dir), is where `startRailwatch()` is added.
       INERTIA_ENTRYPOINTS = %w[
-        app/frontend/entrypoints/inertia.tsx
-        app/frontend/entrypoints/inertia.ts
-        app/frontend/entrypoints/inertia.jsx
+        entrypoints/inertia.tsx
+        entrypoints/inertia.ts
+        entrypoints/inertia.jsx
       ].freeze
 
       IMPORT_LINE = 'import { startRailwatch } from "@/lib/railwatch"'
@@ -68,19 +69,22 @@ module Railwatch
 
       # The one browser client (Railwatch.browser_client_path). Copied rather
       # than templated: there is nothing to interpolate, and a copy means the
-      # host sees exactly what the gem tests.
+      # host sees exactly what the gem tests. Re-running the generator after
+      # a gem upgrade is how an app picks up a new client: Thor asks before
+      # overwriting a copy that differs (--force overwrites, --skip keeps it).
       def create_browser_client
-        return unless File.directory?("app/frontend")
-        copy_file Railwatch.browser_client_path.to_s, "app/frontend/lib/railwatch.ts"
+        return unless frontend_dir
+        copy_file Railwatch.browser_client_path.to_s, "#{frontend_dir}/lib/railwatch.ts"
       end
 
       # Adds the import and the call to the Inertia entrypoint, so page-visit
       # timing and Core Web Vitals report with no further editing.
       def start_browser_client
-        return unless File.directory?("app/frontend")
+        return unless frontend_dir
 
-        path = INERTIA_ENTRYPOINTS.find { |candidate| File.exist?(candidate) }
-        return say(browser_client_instructions(INERTIA_ENTRYPOINTS.first), :yellow) unless path
+        candidates = INERTIA_ENTRYPOINTS.map { |entrypoint| "#{frontend_dir}/#{entrypoint}" }
+        path = candidates.find { |candidate| File.exist?(candidate) }
+        return say(browser_client_instructions(candidates.first), :yellow) unless path
 
         contents = File.read(path)
         return say_status(:identical, path, :blue) if contents.include?(START_CALL)
@@ -279,6 +283,10 @@ module Railwatch
           "The token value is never printed:\n#{lines.join("\n")}"
       end
 
+      def frontend_dir
+        @frontend_dir ||= Railwatch.frontend_dir(destination_root)
+      end
+
       def browser_client_instructions(path)
         "Add these two lines to #{path} to report Inertia visits and Core Web Vitals:\n" \
           "  #{IMPORT_LINE}\n  #{START_CALL}"

data/lib/generators/railwatch/install/templates/post-deploy

@@ -37,6 +37,12 @@ case "$INGEST" in
       exit 0
     fi
     ;;
+  *)
+    # An upper-case scheme or a bare host: curl would send the token over
+    # plain HTTP, which none of the branches above allowed.
+    echo "Railwatch deploy marker skipped: unsupported ingest URL (use https://)" >&2
+    exit 0
+    ;;
 esac
 
 post() {

data/lib/generators/railwatch/install/templates/railwatch.ts

@@ -29,10 +29,13 @@ interface Visit {
   redirect_count?: number
   connect_ms?: number
   server_ms?: number
+  // The browser session and the tab the visit happened in, for the journey.
+  session_id?: string
+  tab_id?: string
 }
 
-// The tab's session, for release health. sessionStorage scopes it to the
-// tab and it dies with the tab, which is what a browser session is.
+// The browser's session, for release health and the session journey, on the
+// wire. `ended` only when this tab has seen it time out (see `begin`).
 interface Session {
   id: string
   started_at: number
@@ -50,6 +53,10 @@ interface JsError {
   component?: string
   url: string
   visit?: string
+  // The session and tab it was thrown in, taken when it was: another tab
+  // may move the session on before this one flushes.
+  session_id?: string
+  tab_id?: string
   breadcrumbs?: Crumb[]
   context?: Record<string, unknown>
 }
@@ -90,14 +97,26 @@ const DEFAULT_DENY_URLS = [
 ]
 
 const SESSION_ID_KEY = "railwatch.session"
+const SESSION_SEEN_KEY = "railwatch.session.seen"
 const SESSION_STARTED_KEY = "railwatch.session.at"
+const TAB_KEY = "railwatch.tab"
+// The GA4/Amplitude rules: a session is over after 30 minutes with no
+// activity in any tab (config.session_timeout's default, which the server
+// half times out on too), and after 24 hours whatever happens.
+const SESSION_TIMEOUT_MS = 30 * 60 * 1000
+const SESSION_CAP_MS = 24 * 60 * 60 * 1000
+// Railwatch::Sessions reads these, ahead of the cookie.
+const SESSION_HEADER = "X-Railwatch-Session"
+const TAB_HEADER = "X-Railwatch-Tab"
 
 const queue: Visit[] = []
 const errors: JsError[] = []
 const crumbs: Crumb[] = []
 let current: Visit | null = null
 let initial: Visit | null = null
-let session: Session | null = null
+// `seen` is the session's last activity in any tab, as far as this one knows.
+let session: { id: string; started_at: number; seen: number } | null = null
+let tab = ""
 // The Inertia page component the user is on, so an error that fires between
 // visits still says which screen it broke.
 let component: string | undefined
@@ -170,12 +189,13 @@ function post(payload: {
   }).catch(() => undefined)
 }
 
-function flush(ended = false) {
-  if (queue.length === 0 && errors.length === 0 && !ended) return
+function flush(force = false) {
+  if (queue.length === 0 && errors.length === 0 && !force) return
+  touch(false)
   post({
     visits: queue.splice(0, queue.length),
     errors: errors.splice(0, errors.length),
-    session: beat(ended),
+    session: beat(),
     tenant: tenant(),
   })
 }
@@ -199,40 +219,220 @@ function randomId() {
   return Array.from(bytes, (b) => b.toString(16).padStart(2, "0")).join("")
 }
 
-// Mints the tab's session on its first load, or picks up the one an earlier
-// page in this tab minted, and mirrors the id into a cookie so every request
-// the tab makes carries it -- that is what lets the server side of the
-// session (Railwatch::Sessions) join the browser side under one id.
-function startSession() {
+interface Store {
+  getItem(key: string): string | null
+  setItem(key: string, value: string): void
+}
+
+// Where the session lives: localStorage, so every tab of the browser shares
+// one; sessionStorage when localStorage throws (storage disabled, a
+// sandboxed frame), which narrows the session to the tab; memory when both
+// throw, which narrows it to the page. The tab id lives in sessionStorage,
+// which survives a reload of the tab and nothing else, or in memory.
+let sessions: Store = memoryStore()
+let tabs: Store = memoryStore()
+
+function storage(kind: "localStorage" | "sessionStorage"): Store | undefined {
+  try {
+    const store = window[kind]
+    store.setItem("railwatch.probe", "1")
+    store.removeItem("railwatch.probe")
+    return store
+  } catch {
+    // Disabled, or denied to this frame. The caller tries the next one.
+    return undefined
+  }
+}
+
+function memoryStore(): Store {
+  const values = new Map<string, string>()
+  return {
+    getItem: (key) => values.get(key) ?? null,
+    setItem: (key, value) => void values.set(key, value),
+  }
+}
+
+// A full store throws on write. The session then lasts as long as this
+// page's memory of it, which is all a write could have bought.
+function save(store: Store, key: string, value: string) {
+  try {
+    store.setItem(key, value)
+  } catch {
+    // Quota exceeded.
+  }
+}
+
+// Mirrors the id into a cookie so every request the browser makes carries
+// it, full page loads included -- that is what lets the server side of the
+// session (Railwatch::Sessions) join the browser side under one id. It
+// lapses with the session's idle timeout, so the first page load after one
+// does not carry the dead id; every activity in any tab renews it.
+function remember(id: string) {
   try {
-    const existing = sessionStorage.getItem(SESSION_ID_KEY)
-    const id = existing ?? randomId()
-    const startedAt =
-      Number(sessionStorage.getItem(SESSION_STARTED_KEY)) || Date.now()
-    if (!existing) {
-      sessionStorage.setItem(SESSION_ID_KEY, id)
-      sessionStorage.setItem(SESSION_STARTED_KEY, String(startedAt))
-    }
     // Railwatch::Sessions::COOKIE reads this name to stitch server requests to the browser session.
-    document.cookie = `railwatch_session=${id}; path=/; SameSite=Lax`
-    session = { id, started_at: startedAt }
-    // No duration on the first beat: that is what opens the session.
-    if (!existing)
-      post({
-        visits: [],
-        errors: [],
-        session: { ...session },
-        tenant: tenant(),
-      })
+    document.cookie = `railwatch_session=${id}; path=/; max-age=${SESSION_TIMEOUT_MS / 1000}; SameSite=Lax`
+  } catch {
+    // A sandboxed frame has no cookies. Visits still carry the headers.
+  }
+}
+
+// A store that passed the probe can still throw later: the user turns
+// storage off, or the browser revokes it. From then on the page keeps its
+// own copy, seeded with the session it is in, so losing storage neither
+// stops the client nor ends the session.
+function read(key: string): string | null {
+  try {
+    return sessions.getItem(key)
+  } catch {
+    sessions = memoryStore()
+    if (session) {
+      sessions.setItem(SESSION_ID_KEY, session.id)
+      sessions.setItem(SESSION_STARTED_KEY, String(session.started_at))
+      sessions.setItem(SESSION_SEEN_KEY, String(session.seen))
+    }
+    return sessions.getItem(key)
+  }
+}
+
+function readTab(): string | null {
+  try {
+    return tabs.getItem(TAB_KEY)
   } catch {
-    // sessionStorage is unavailable (private mode, storage disabled).
-    // Everything else still reports; this tab just has no session.
+    tabs = memoryStore()
+    return null
   }
 }
 
-function beat(ended: boolean): Session | undefined {
+function startSession() {
+  sessions =
+    storage("localStorage") ?? storage("sessionStorage") ?? memoryStore()
+  tabs = storage("sessionStorage") ?? memoryStore()
+  tab = readTab() ?? randomId()
+  save(tabs, TAB_KEY, tab)
+  touch(true)
+}
+
+// Brings this tab's session up to date with the browser's, joining the one
+// another tab opened meanwhile. `active` is the user doing something -- the
+// page loading, a visit they wait on, the tab coming back into view -- which
+// keeps the session alive, or opens a new one when the last went 30 minutes
+// without activity in any tab or is 24 hours old. Anything else (a
+// background poll, a prefetch, a flush) only joins a live session, so a tab
+// left polling cannot hold its session open until the cap.
+function touch(active: boolean) {
+  const now = Date.now()
+  const id = read(SESSION_ID_KEY)
+  const seen = Number(read(SESSION_SEEN_KEY)) || 0
+  const startedAt = Number(read(SESSION_STARTED_KEY)) || 0
+  const live = id !== null && !expired(startedAt, seen, now)
+  if (live && id === session?.id) session.seen = Math.max(session.seen, seen)
+  else if (live) begin(id, startedAt, seen, false)
+  else if (active) begin(minted(id) ?? randomId(), now, now, true)
+  if (!active || !session) return
+  session.seen = now
+  save(sessions, SESSION_SEEN_KEY, String(now))
+  remember(session.id)
+}
+
+// The id the server put in the cookie for this browser's first page load
+// (Railwatch::Sessions.mint), so the session starts with that request rather
+// than after it. Not the session that just expired, whose id the cookie
+// can still hold.
+function minted(expiredId: string | null): string | undefined {
+  try {
+    const id = /(?:^|;\s*)railwatch_session=([A-Za-z0-9_-]{1,64})(?:;|$)/.exec(
+      document.cookie,
+    )?.[1]
+    return id && id !== expiredId && id !== session?.id ? id : undefined
+  } catch {
+    // A sandboxed frame has no cookies.
+    return undefined
+  }
+}
+
+function expired(startedAt: number, seen: number, now: number) {
+  return now - seen >= SESSION_TIMEOUT_MS || now - startedAt >= SESSION_CAP_MS
+}
+
+// The session this tab is in, while it is live. A background visit after it
+// timed out belongs to no session: it is neither tagged nor recorded as one.
+function liveSession(): string | undefined {
+  if (!session || expired(session.started_at, session.seen, Date.now()))
+    return undefined
+  return session.id
+}
+
+// Moves this tab into a session. The one it was in is over -- a session is
+// only replaced once it has timed out or reached the cap -- so this tab
+// says so with a last, `ended` beat. Closing a tab sends no such thing: the
+// browser's other tabs may still be in the session, and nothing a closing
+// tab knows says otherwise. A session no tab comes back to ends where the
+// server half's does, on its own timeout.
+function begin(id: string, startedAt: number, seen: number, minted: boolean) {
+  if (session)
+    post({
+      visits: [],
+      errors: [],
+      session: {
+        id: session.id,
+        started_at: session.started_at,
+        duration_ms: session.seen - session.started_at,
+        ended: true,
+      },
+      tenant: tenant(),
+    })
+  session = { id, started_at: startedAt, seen }
+  if (!minted) return
+  save(sessions, SESSION_ID_KEY, id)
+  save(sessions, SESSION_STARTED_KEY, String(startedAt))
+  save(sessions, SESSION_SEEN_KEY, String(seen))
+  // No duration on the first beat: that is what opens the session.
+  post({
+    visits: [],
+    errors: [],
+    session: { id, started_at: startedAt },
+    tenant: tenant(),
+  })
+}
+
+// While the session is live it runs to now, whichever tab is beating it. One
+// this tab has watched time out ran to its last activity, and is `ended`.
+function beat(): Session | undefined {
   if (!session) return undefined
-  return { ...session, duration_ms: Date.now() - session.started_at, ended }
+  const now = Date.now()
+  const ended = expired(session.started_at, session.seen, now)
+  return {
+    id: session.id,
+    started_at: session.started_at,
+    duration_ms: (ended ? session.seen : now) - session.started_at,
+    ended,
+  }
+}
+
+// Inertia sends a visit with whatever headers the visit object holds once
+// `before` has fired, so setting them here tags every visit, prefetch, poll
+// and partial reload with the session and the tab -- which is how a request
+// record joins its journey under the tab it came from. Only for the page's
+// own origin: a custom header makes a cross-origin request preflight, and
+// another origin is not this app's server anyway. The headers object is
+// replaced, not written into, because it may be the app's own options. The
+// prefetch cache compares headers; a prefetch and the click it serves are
+// tagged alike, so the prefetch still serves.
+function tag(visit: { url: unknown; headers?: Record<string, string> }) {
+  const id = liveSession()
+  if (!id) return
+  try {
+    if (new URL(String(visit.url), location.href).origin !== location.origin)
+      return
+  } catch {
+    // Not a URL Inertia could visit either.
+    return
+  }
+  visit.headers = {
+    ...visit.headers,
+    [SESSION_HEADER]: id,
+    [TAB_HEADER]: tab,
+  }
 }
 
 // --- Breadcrumbs -------------------------------------------------------
@@ -349,6 +549,8 @@ function capture(
     component,
     url: location.pathname + location.search,
     visit: current?.url,
+    session_id: liveSession(),
+    tab_id: tab,
     breadcrumbs: breadcrumbs.length > 0 ? breadcrumbs : undefined,
     context,
   }
@@ -755,6 +957,8 @@ function initialVisit(): Visit | null {
     duration_ms: duration,
     status: "success",
     ...ttfbPhases(nav),
+    session_id: liveSession(),
+    tab_id: tab,
   }
 }
 
@@ -774,7 +978,7 @@ function finalize() {
   }
   // Still flushes on every later hide: a tab can be backgrounded, brought
   // back, navigated some more, and then closed. Each of those carries a
-  // final session beat, which the platform dedupes by session id.
+  // session beat, which the platform dedupes by session id.
   flush(true)
 }
 
@@ -798,7 +1002,10 @@ export function startRailwatch(options: RailwatchOptions = {}) {
   // final there.
   router.on("before", (event) => {
     const v = event.detail.visit
-    if (!waiting(v) || event.defaultPrevented) return
+    if (event.defaultPrevented) return
+    touch(waiting(v))
+    tag(v)
+    if (!waiting(v)) return
     const request = (foregroundRequest = { id: idOf(v, "id") })
     queueMicrotask(() => {
       if (event.defaultPrevented && foregroundRequest === request)
@@ -817,6 +1024,8 @@ export function startRailwatch(options: RailwatchOptions = {}) {
       method: v.method,
       partial: Boolean(v.only?.length || v.except?.length),
       only: v.only,
+      session_id: liveSession(),
+      tab_id: tab,
     }
     crumb("navigate", `${v.method.toUpperCase()} ${current.url}`)
   })
@@ -847,6 +1056,7 @@ export function startRailwatch(options: RailwatchOptions = {}) {
   timer ??= window.setInterval(flush, 5000)
   document.addEventListener("visibilitychange", () => {
     if (document.visibilityState === "hidden") finalize()
+    else touch(true)
   })
   window.addEventListener("pagehide", finalize)
 }

data/lib/railwatch/configuration.rb

@@ -86,7 +86,7 @@ module Railwatch
                   :capture_job_arguments, :capture_job_retry_errors, :capture_response_body_on_error, :max_attachment_bytes,
                   :track_sessions, :session_flush_interval, :session_timeout,
                   :capture_console, :interactive_runner_paths, :ignored_request_paths,
-                  :capture_llm_content
+                  :capture_llm_content, :timing_headers, :browser_client
 
     attr_reader :deploy, :deploy_source, :detect_deploy, :user_resolver, :beacon_user_resolver,
                 :fingerprint_resolver, :redactors, :rejectors, :before_ingest, :backpressure_high_water
@@ -238,6 +238,18 @@ module Railwatch
         DEFAULT_INTERACTIVE_RUNNER_PATHS.dup
       @ignored_request_paths = ENV["RAILWATCH_IGNORED_REQUEST_PATHS"]&.split(",")&.map(&:strip)&.reject(&:empty?) ||
         DEFAULT_IGNORED_REQUEST_PATHS.dup
+      # Server-Timing (total, db with the query count, view, other) and
+      # X-Railwatch-Execution-Id on each recorded request's response, for
+      # `curl -i`, the browser's network panel, and an agent that wants the
+      # execution it just caused. nil: on in development only
+      # (#timing_headers?), since elsewhere they tell every visitor how the
+      # app spends its time.
+      @timing_headers = env_bool("RAILWATCH_TIMING_HEADERS", nil)
+      # Whether the app's pages run the browser client, which is what makes
+      # it safe to hand a first page load a session cookie (#browser_client?).
+      # nil: whether lib/railwatch.ts exists in the app's frontend source
+      # directory (Railwatch.frontend_dir), where railwatch:install puts it.
+      @browser_client = env_bool("RAILWATCH_BROWSER_CLIENT", nil)
       @user_resolver = nil
       @beacon_user_resolver = nil
       @fingerprint_resolver = nil
@@ -364,6 +376,27 @@ module Railwatch
       @environment || (defined?(Rails) ? Rails.env.to_s : "production")
     end
 
+    # Whether the request middleware adds its timing headers (#timing_headers).
+    def timing_headers?
+      @timing_headers.nil? ? defined?(Rails) && Rails.env.development? : @timing_headers
+    end
+
+    # Whether the browser client is installed (#browser_client). Looked for
+    # once, on the first request that asks, when Rails.root is known.
+    def browser_client?
+      return @browser_client unless @browser_client.nil?
+
+      @browser_client_installed = browser_client_installed? if @browser_client_installed.nil?
+      @browser_client_installed
+    end
+
+    def browser_client_installed?
+      return false unless defined?(Rails) && Rails.root
+
+      dir = Railwatch.frontend_dir(Rails.root)
+      dir ? Rails.root.join(dir, "lib/railwatch.ts").exist? : false
+    end
+
     # Matches Railwatch.reject_cache_keys entries and DEFAULT_VENDOR_CACHE_KEYS
     # against a cache key. A Regexp is used as-is. A String starting with "^"
     # or containing another regex metacharacter is compiled as a Regexp; a

data/lib/railwatch/engine.rb

@@ -1,10 +1,14 @@
 # frozen_string_literal: true
 
+require "railwatch/middleware/beacon_limit"
+
 module Railwatch
   # Mounted for the browser beacon (POST <mount>/beacon), its only route.
   class Engine < ::Rails::Engine
     isolate_namespace Railwatch
 
+    config.middleware.use Railwatch::Middleware::BeaconLimit
+
     # The request middleware goes first so wall time includes every other
     # middleware, exactly like Nightwatch's GlobalMiddleware.
     initializer "railwatch.middleware", before: :load_config_initializers do |app|

data/lib/railwatch/middleware/beacon_limit.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Railwatch
+  module Middleware
+    # Refuses an oversized beacon body before Rails parses it. The check
+    # cannot live in BeaconController: ActionController parses the params
+    # (ParamsWrapper, parameter filtering for the log line) before any
+    # before_action, and Request#content_length on a chunked body reads it
+    # whole. In the engine's own stack, so it sees only the beacon route.
+    class BeaconLimit
+      MAX_BYTES = 256 * 1024
+
+      def initialize(app, limit = MAX_BYTES)
+        @app = app
+        @limit = limit
+      end
+
+      def call(env)
+        return too_large if env["CONTENT_LENGTH"].to_i > @limit
+        return @app.call(env) if env["CONTENT_LENGTH"] || !(input = env["rack.input"])
+
+        # Chunked: no length up front, so read at most one byte past the
+        # limit and hand the rest of the stack a body it can read again.
+        body = read_bounded(input)
+        return too_large if body.bytesize > @limit
+
+        env["rack.input"] = StringIO.new(body)
+        env["CONTENT_LENGTH"] = body.bytesize.to_s
+        env.delete("HTTP_TRANSFER_ENCODING")
+        @app.call(env)
+      end
+
+      private
+
+      # Rack lets read(n) return fewer than n bytes before the end, so keep
+      # reading until the end or one byte past the limit, never further.
+      def read_bounded(input)
+        body = +"".b
+        while body.bytesize <= @limit
+          chunk = input.read(@limit + 1 - body.bytesize) or break
+          break if chunk.empty?
+
+          body << chunk
+        end
+        body
+      end
+
+      def too_large
+        [ 413, { "content-type" => "text/plain" }, [] ]
+      end
+    end
+  end
+end