railwatch 1.3.0 → 1.5.0

data/lib/railwatch/middleware/request.rb

@@ -20,6 +20,10 @@ module Railwatch
       # CONTENT_TYPE test Subscribers::Requests uses before its params walk.
       MULTIPART = "multipart/form-data"
       NO_FILES = [].freeze
+      # Response headers written with config.timing_headers. Lower case, as
+      # Rack 3 requires; Rack::Headers matches any case.
+      SERVER_TIMING = "server-timing"
+      EXECUTION_ID = "x-railwatch-execution-id"
 
       # Returns [trace_id, parent_id, sampled] from an inbound traceparent,
       # or nil when the header is absent or malformed.
@@ -55,21 +59,59 @@ module Railwatch
         exe.keep! if upstream_sampled
         exe.enter_stage(:middleware_before)
         env["railwatch.execution"] = exe
-        status = headers = body = nil
+        status = headers = body = parent = nil
+        timing = Railwatch.config.timing_headers?
         begin
           status, headers, body = @app.call(env)
+          server_timing(env, exe, headers) if timing && headers
+          Sessions.mint(env, status, headers) if headers
         rescue Exception => e # rubocop:disable Lint/RescueException
           Subscribers::Exceptions.capture(e, handled: false, severity: :error, source: "railwatch.middleware")
           raise
         ensure
           exe.enter_stage(:middleware_after) unless exe.stage == :middleware_after
-          finish(env, exe, status, headers)
+          parent = finish(env, exe, status, headers)
         end
+        execution_id(exe, headers) if timing && headers && parent
         [ status, headers, body ]
       end
 
       private
 
+      # config.timing_headers (development by default): what this request
+      # cost, on the response, so `curl -i` or the browser's network panel
+      # shows it without opening the dashboard. Everything here is already
+      # known: the counters, and the runtimes the process_action subscriber
+      # put in env. Appended to a Server-Timing the app (Rails' own, with
+      # config.server_timing) already set, whether one string or several.
+      def server_timing(env, exe, headers)
+        total = exe.duration / 1000.0
+        db = env["railwatch.db_runtime"]
+        view = env["railwatch.view_runtime"]
+        queries = exe.counters[:queries]
+        cached = exe.counters[:cached_queries]
+        desc = cached.positive? ? "#{queries} queries, #{cached} cached" : "#{queries} #{queries == 1 ? "query" : "queries"}"
+        metrics = [ format("total;dur=%.1f", total), db ? format("db;dur=%.1f;desc=\"%s\"", db, desc) : "db;desc=\"#{desc}\"" ]
+        metrics << format("view;dur=%.1f", view) if view
+        metrics << format("other;dur=%.1f;desc=\"not db or view\"", [ total - db.to_f - view.to_f, 0 ].max) if db || view
+        # A plain Hash may hold it title-cased; Rack::Headers finds either.
+        existing = Array(headers[SERVER_TIMING] || headers.delete("Server-Timing")).join(", ")
+        headers[SERVER_TIMING] = existing.empty? ? metrics.join(", ") : "#{existing}, #{metrics.join(", ")}"
+      rescue StandardError => e
+        # A frozen header hash (a Rack app that returns a constant): the
+        # response is the app's, unchanged.
+        Railwatch.debug { "timing headers skipped: #{e.class}: #{e.message}" }
+      end
+
+      # The execution an agent passes to get_execution. Only once the
+      # request record has shipped (head-sampled in, kept, or rescued by the
+      # tail decision): any other id is one the server never has.
+      def execution_id(exe, headers)
+        headers[EXECUTION_ID] = exe.id
+      rescue StandardError => e
+        Railwatch.debug { "execution id header skipped: #{e.class}: #{e.message}" }
+      end
+
       def ignored_request?(env)
         path = env["PATH_INFO"].to_s
         Railwatch.config.ignored_request_paths.any? do |pattern|
@@ -111,13 +153,16 @@ module Railwatch
         # The block is only called when the request record is going to ship;
         # a head-sampled-out request nothing rescued skips the
         # ActionDispatch::Request and the header walk entirely.
-        Railwatch.finish_execution(:request) { parent_fields(env, exe, status, headers) }
+        parent = Railwatch.finish_execution(:request) { parent_fields(env, exe, status, headers) }
         # A request with no user and no session cookie has no session, and
         # Sessions.touch returns without writing anything.
         Sessions.touch(exe, env, status) if Railwatch.config.track_sessions
+        # The request record, when one shipped.
+        parent
       rescue StandardError => e
         Railwatch.debug { "request finish failed: #{e.class}: #{e.message}" }
         Railwatch.finish_execution
+        nil
       end
 
       def parent_fields(env, exe, status, headers)
@@ -158,7 +203,8 @@ module Railwatch
           payload: payload,
           queue_time: queue_time(env, exe),
           user_agent: req.user_agent.to_s[0, 256],
-          files: env["railwatch.files"] || uploaded_files_fallback(req, env)
+          files: env["railwatch.files"] || uploaded_files_fallback(req, env),
+          **Sessions.browser_fields(env)
         }
       end
 

data/lib/railwatch/sessions.rb

@@ -26,6 +26,18 @@ module Railwatch
     # a tab carries the id its beacons already use.
     COOKIE = /(?:\A|;\s*)railwatch_session=([^;]+)/
     KEY_LIMIT = 64
+    # What the browser client mints for a session or a tab id. Anything else
+    # in a header, cookie, or beacon is not one, and is not recorded as one.
+    BROWSER_ID = /\A[A-Za-z0-9_-]{1,64}\z/
+    # The id Sessions.mint handed this request, read by key_for and
+    # browser_fields as if the browser had sent it.
+    MINTED = "railwatch.session_id"
+    # The client's idle timeout, which the cookie lapses with (railwatch.ts).
+    COOKIE_MAX_AGE = 30 * 60
+    HTML = "text/html"
+    # A response a shared cache may store and serve to anyone. One visitor's
+    # cookie on it would be every visitor's.
+    SHARED_CACHE = /\b(?:public|s-maxage)\b/i
 
     @mutex = Mutex.new
     @wakeup = ConditionVariable.new
@@ -78,13 +90,61 @@ module Railwatch
     end
 
     # The browser client's id when the request carries one -- so the two
-    # halves of the same session share a key -- else the resolved user.
+    # halves of the same session share a key -- else the resolved user. A
+    # first page load the middleware minted an id for keys on that id, but
+    # only where it would have keyed on the user: an anonymous request with
+    # no cookie (a crawler, curl) opened no session before and opens none
+    # now. Its browser, if it runs the client, opens it with the first beat.
     def key_for(exe, env)
-      id = env["HTTP_X_RAILWATCH_SESSION"] || COOKIE.match(env["HTTP_COOKIE"])&.[](1)
+      id = sent_id(env)
       return id[0, KEY_LIMIT] unless id.nil? || id.empty?
 
       user = exe.user_id
-      user && "user:#{user}"[0, KEY_LIMIT]
+      return nil unless user
+      env[MINTED] || "user:#{user}"[0, KEY_LIMIT]
+    end
+
+    # The browser session and tab a request came from, for the request
+    # record: only what the browser client sent, never the user fallback
+    # above, which names a person rather than a session.
+    def browser_fields(env)
+      session = browser_id(sent_id(env) || env[MINTED])
+      tab = browser_id(env["HTTP_X_RAILWATCH_TAB"])
+      { session_id: session, tab_id: tab }.compact
+    end
+
+    def sent_id(env)
+      env["HTTP_X_RAILWATCH_SESSION"] || COOKIE.match(env["HTTP_COOKIE"])&.[](1)
+    end
+
+    # A browser's first page load carries no session cookie: the client sets
+    # it once the page runs, after this request is over. Without one the
+    # request would key on the user, a session no browser ever beats, and
+    # the journey would start a page late. So a page the client will run on
+    # -- a successful GET of an HTML document, not an Inertia visit, from an
+    # app with the client installed -- that arrives with no id gets one here,
+    # in the cookie the client reads and adopts. Never on a response a
+    # shared cache may keep (Cache-Control public or s-maxage).
+    def mint(env, status, headers)
+      return unless Railwatch.config.browser_client?
+      return unless env["REQUEST_METHOD"] == "GET" && status.to_i == 200
+      return if env["HTTP_X_INERTIA"] || sent_id(env)
+      return unless headers[Rack::CONTENT_TYPE].to_s.start_with?(HTML)
+      return if SHARED_CACHE.match?(headers[Rack::CACHE_CONTROL].to_s)
+
+      id = env[MINTED] = SecureRandom.hex(8)
+      # Secure over HTTPS (behind a TLS proxy too: ssl? reads
+      # X-Forwarded-Proto), so the id never rides a plain-HTTP request.
+      Rack::Utils.set_cookie_header!(headers, "railwatch_session",
+        { value: id, path: "/", max_age: COOKIE_MAX_AGE, same_site: :lax, secure: Rack::Request.new(env).ssl? })
+    rescue StandardError => e
+      # A frozen header hash: the client mints the id once the page runs.
+      env.delete(MINTED)
+      Railwatch.debug { "session cookie skipped: #{e.class}: #{e.message}" }
+    end
+
+    def browser_id(value)
+      value if value.is_a?(String) && BROWSER_ID.match?(value)
     end
 
     def emit(key, entry, ended)

data/lib/railwatch/transport/http.rb

@@ -69,9 +69,9 @@ module Railwatch
       end
 
       def deliver(records, dropped: 0, dropped_bytes: 0, backpressure_factor: 1.0, batch_id: SecureRandom.uuid)
-        unless destination_allowed?
-          return Result.new(ok: false, error: "plain HTTP ingest is disabled; use HTTPS or set RAILWATCH_ALLOW_HTTP=true")
-        end
+        # Permanent: it cannot change between attempts, and with no status
+        # the result would read as retryable and walk the backoff ladder.
+        return permanent("plain HTTP ingest is disabled; use HTTPS or set RAILWATCH_ALLOW_HTTP=true") unless destination_allowed?
         return Result.new(ok: false, status: UNAUTHORIZED_STATUS, error: "unauthorized, flushing stopped") if @unauthorized
 
         encoded = @encoder.encode(records)

data/lib/railwatch/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Railwatch
-  VERSION = "1.3.0"
+  VERSION = "1.5.0"
 end

data/lib/railwatch/wire_fixtures.json

@@ -57,12 +57,16 @@
     "headers": {
       "Host": "www.example.com",
       "Accept": "text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5",
+      "X-Railwatch-Session": "3f2a9c1e7b4d6a08",
+      "X-Railwatch-Tab": "9e8d7c6b5a4f3e21",
       "Cookie": "[FILTERED]"
     },
     "payload": null,
     "queue_time": null,
     "user_agent": "",
-    "files": []
+    "files": [],
+    "session_id": "3f2a9c1e7b4d6a08",
+    "tab_id": "9e8d7c6b5a4f3e21"
   },
   "job_attempt": {
     "v": 1,
@@ -609,6 +613,8 @@
     "redirect_count": null,
     "connect_ms": null,
     "server_ms": null,
+    "session_id": "3f2a9c1e7b4d6a08",
+    "tab_id": "9e8d7c6b5a4f3e21",
     "user": null,
     "tenant": null,
     "user_agent": ""

data/lib/railwatch.rb

@@ -607,13 +607,28 @@ require "railwatch/subscribers"
 
 module Railwatch
   # The Inertia browser client, relative to the gem root. The install
-  # generator copies it into a host's app/frontend/lib.
+  # generator copies it into a host's frontend source directory, at lib/.
   BROWSER_CLIENT = "lib/generators/railwatch/install/templates/railwatch.ts"
+  # Where a Rails app keeps its Vite sources when ViteRuby does not say:
+  # vite_ruby's default, then the jsbundling/rails_vite one.
+  FRONTEND_DIRS = %w[app/frontend app/javascript].freeze
 
   def self.browser_client_path
     File.expand_path("../#{BROWSER_CLIENT}", __dir__)
   end
 
+  # The app's frontend source directory, relative to `root`: ViteRuby's
+  # configured sourceCodeDir when ViteRuby is loaded, else the first of
+  # FRONTEND_DIRS that exists, else nil. The install generator copies the
+  # client into it, and Configuration#browser_client? looks for it there.
+  def self.frontend_dir(root)
+    root = Pathname(root)
+    configured = defined?(ViteRuby) && (ViteRuby.config.source_code_dir rescue nil)
+    return configured.to_s if configured.present? && root.join(configured.to_s).directory?
+
+    FRONTEND_DIRS.find { |dir| root.join(dir).directory? }
+  end
+
   # One realistic wire record per record type, exactly as the gem's own
   # subscribers emit it, generated by spec/railwatch/wire_fixtures_spec.rb.
   # A receiver tests its mapper against these instead of hand-writing what

data/lib/tasks/railwatch_tasks.rake

@@ -144,21 +144,22 @@ namespace :railwatch do
     check.call(hook.exist? && hook.read.include?("railwatch"), "kamal post-deploy hook",
                hook.exist? ? hook.to_s : "not found (only needed when deploying with Kamal)")
 
-    client = Rails.root.join("app/frontend/lib/railwatch.ts")
+    frontend = Railwatch.frontend_dir(Rails.root) || "app/frontend"
+    client = Rails.root.join(frontend, "lib/railwatch.ts")
     check.call(client.exist?, "browser client",
                client.exist? ? "#{client} (call startRailwatch() from your Inertia entrypoint)" : "not found (only needed for Inertia visit timing)")
 
     # The client file existing is not the same as it running: startRailwatch()
     # has to be called from an entrypoint or no visit is ever timed.
-    entrypoints = Dir[Rails.root.join("app/frontend/entrypoints/*")].select { |f| File.file?(f) }
+    entrypoints = Dir[Rails.root.join(frontend, "entrypoints/*")].select { |f| File.file?(f) }
     started = entrypoints.select { |f| File.read(f).include?("startRailwatch") }
     check.call(started.any?, "browser client imported",
                if started.any?
                  started.map { |f| Pathname.new(f).relative_path_from(Rails.root).to_s }.join(", ")
                elsif entrypoints.any?
-                 "no entrypoint in app/frontend/entrypoints calls startRailwatch()"
+                 "no entrypoint in #{frontend}/entrypoints calls startRailwatch()"
                else
-                 "no app/frontend/entrypoints (only needed for Inertia visit timing)"
+                 "no #{frontend}/entrypoints (only needed for Inertia visit timing)"
                end)
 
     backend = Railwatch::Profiler.backend

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: railwatch
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 1.5.0
 platform: ruby
 authors:
 - Cole Robertson
@@ -70,6 +70,9 @@ files:
 - docs/source-maps.md
 - docs/testing.md
 - docs/troubleshooting.md
+- lib/generators/railwatch/agents/agents_generator.rb
+- lib/generators/railwatch/agents/templates/SKILL.md.tt
+- lib/generators/railwatch/agents/templates/agents_block.md.tt
 - lib/generators/railwatch/install/install_generator.rb
 - lib/generators/railwatch/install/templates/initializer.rb.tt
 - lib/generators/railwatch/install/templates/post-deploy
@@ -91,6 +94,7 @@ files:
 - lib/railwatch/health.rb
 - lib/railwatch/job_tracing.rb
 - lib/railwatch/json_compat.rb
+- lib/railwatch/middleware/beacon_limit.rb
 - lib/railwatch/middleware/request.rb
 - lib/railwatch/minitest.rb
 - lib/railwatch/patches.rb