railwatch 0.1.3 → 0.2.0.pre1

data/lib/railwatch/engine.rb

@@ -1,11 +1,56 @@
 # frozen_string_literal: true
 
+# The dashboard controllers are Inertia controllers; a host that does not use
+# Inertia itself never requires the gem, so the engine does.
+require "inertia_rails"
+# The engine's jobs (grouping, rollups, scans) are Active Job classes even
+# though embedded mode calls their perform directly, so a host built with
+# `rails new --minimal` (no Active Job) still needs the framework loaded.
+# Part of the rails gem this one depends on, so nothing new is pulled in.
+require "active_job/railtie"
+
 module Railwatch
+  # isolate_namespace would give every model a railwatch_ prefix. Only the
+  # engine's own records (issues, comments, deploys: Railwatch::ApplicationRecord)
+  # carry it; the telemetry tables are the hosted platform's schema and stay
+  # unprefixed. Defined before isolate_namespace runs, which only adds a
+  # prefix when none is set.
+  def self.table_name_prefix = ""
+
   class Engine < ::Rails::Engine
     isolate_namespace Railwatch
 
     config.railwatch = ActiveSupport::OrderedOptions.new
 
+    # The dashboard bundle is built into the gem at release time; serve it
+    # from here so a host app needs no asset pipeline integration at all.
+    # Where it goes in the stack is DashboardAssets.install!'s decision.
+    initializer "railwatch.dashboard_assets" do |app|
+      Railwatch::DashboardAssets.install!(app, root: root.join("public/railwatch").to_s)
+    end
+
+    # Action Cable is optional: live dashboard updates need it, nothing else
+    # does. A host without it (`rails new --minimal`, --skip-action-cable)
+    # must not eager-load the engine's channel, which inherits from a class
+    # that does not exist there.
+    initializer "railwatch.channels", before: :setup_main_autoloader do
+      Rails.autoloaders.main.ignore(root.join("app/channels")) unless defined?(::ActionCable)
+    end
+
+    # The dashboard bundle subscribes to "EnvironmentChannel" by name; Action
+    # Cable constantizes that, so the engine's channel needs the bare name.
+    # Only defined when the host has not got one of its own.
+    initializer "railwatch.live_channel" do
+      next unless defined?(::ActionCable)
+
+      # Action Cable constantizes the subscription's channel name at
+      # subscribe time; the alias only has to exist by then, and it must
+      # not clobber a host channel of the same name.
+      config.to_prepare do
+        Object.const_set(:EnvironmentChannel, Railwatch::EnvironmentChannel) unless Object.const_defined?(:EnvironmentChannel)
+      end
+    end
+
     # The request middleware goes first so wall time includes every other
     # middleware, exactly like Nightwatch's GlobalMiddleware.
     initializer "railwatch.middleware", before: :load_config_initializers do |app|
@@ -18,8 +63,39 @@ module Railwatch
       Railwatch::Console.silence!
     end
 
+    # Same source Mission Control Jobs reads: railwatch.http_basic_auth_user
+    # and _password in Rails credentials, which `bin/rails
+    # railwatch:authentication:configure` writes. An initializer or env var
+    # that already set them wins.
+    initializer "railwatch.http_basic_auth", after: :load_config_initializers do |app|
+      config = Railwatch.config
+      config.http_basic_auth_user ||= app.credentials.dig(:railwatch, :http_basic_auth_user)
+      config.http_basic_auth_password ||= app.credentials.dig(:railwatch, :http_basic_auth_password)
+    end
+
+    # Two things worth one line in the log at boot, because both are
+    # invisible until something is already wrong: a Rails/json pair that
+    # cannot decode, and an embedded dashboard with nothing declared in
+    # front of it.
+    initializer "railwatch.warnings", after: :load_config_initializers do
+      config.after_initialize do
+        next unless Railwatch.enabled?
+
+        Rails.logger.warn("[railwatch] #{Railwatch::JsonCompat.advice}") if Railwatch::JsonCompat.broken?
+
+        if Railwatch.config.local? && Railwatch.config.dashboard_gate == :undeclared && !Rails.env.local?
+          Rails.logger.warn(
+            "[railwatch] the dashboard at the engine's mount has no gate this gem can see: HTTP Basic is off and " \
+            "no base_controller_class, dashboard_user or dashboard_open is set. If a routes constraint or your " \
+            "network already gates it, set `c.dashboard_open = true` to say so (it also enables live updates); " \
+            "otherwise anyone who can reach the URL can read every query, log line and exception this app records."
+          )
+        end
+      end
+    end
+
     initializer "railwatch.transport_security", after: :load_config_initializers do
-      next if Railwatch.config.ingest_url_allowed?
+      next if Railwatch.config.local? || Railwatch.config.ingest_url_allowed?
 
       Rails.logger.warn(
         "Railwatch will not send telemetry to #{Railwatch.config.ingest_url}: plain HTTP is allowed only for loopback " \
@@ -129,6 +205,17 @@ module Railwatch
       at_exit { Railwatch::Sessions.stop! }
     end
 
+    # The embedded install's maintenance clock (release health, scans,
+    # pruning). Same shape and ordering rationale as "railwatch.health":
+    # stopped before the reporter's final flush. Maintenance.start! is a
+    # no-op unless the transport is local.
+    initializer "railwatch.maintenance" do
+      next unless Railwatch.enabled?
+
+      Railwatch::Maintenance.start!
+      at_exit { Railwatch::Maintenance.stop! }
+    end
+
     # lib/tasks/railwatch_tasks.rake is picked up by Rails::Engine's default
     # lib/tasks convention; the rake_tasks block above only installs the
     # Rake::Task patch.
@@ -138,6 +225,7 @@ end
 require "railwatch/console"
 require "railwatch/health"
 require "railwatch/sessions"
+require "railwatch/maintenance"
 require "railwatch/middleware/request"
 require "railwatch/job_tracing"
 require "railwatch/controller_helpers"

data/lib/railwatch/ingest_request_body_limit.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+# The hosted platform bounds ingest request bodies with Rack middleware. In
+# an embedded install nothing arrives over HTTP, but Ingest::Payload and
+# Telemetry::Attachment still size their limits from this constant.
+module Railwatch
+  module IngestRequestBodyLimit
+    MAX_BYTES = 32 * 1024 * 1024
+  end
+end

data/lib/railwatch/json_compat.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require "active_support/json"
+
+module Railwatch
+  # Rails 8.1, up to and including 8.1.3.1, calls `JSON.parse` with a
+  # positional options hash. json 3 made those options keyword arguments
+  # (rails/rails#58784), so on that pair every signed cookie, every JSON
+  # column, and this gem's own SQLite migrations raise ArgumentError. Ruby
+  # 3.4.10 ships json 3.0.2 as a default gem, so a fresh machine meets it
+  # without choosing to. The fix is merged on Rails' 8-1-stable branch and
+  # unreleased as of 8.1.3.1.
+  #
+  # Detected by behaviour, not by version numbers: the pair is asked to
+  # decode a document once. That way the check is right about combinations
+  # nobody has enumerated (a patched Rails, a backport, a future json that
+  # restores the old signature), and it goes quiet by itself the day the
+  # host upgrades, with no release of this gem required.
+  #
+  # Railwatch does not pin `json` for the host. The breakage is the host
+  # application's either way -- its sessions are already failing -- so the
+  # gem reports it and the installer offers the pin, rather than quietly
+  # constraining everybody's bundle for a bug that is not ours and is on its
+  # way out.
+  module JsonCompat
+    PIN = %(gem "json", "< 3")
+    ISSUE = "rails/rails#58784"
+
+    module_function
+
+    def broken?
+      return @broken unless @broken.nil?
+
+      @broken = begin
+        ActiveSupport::JSON.decode("{}")
+        false
+      rescue ArgumentError
+        true
+      rescue StandardError
+        # Anything else is not this bug.
+        false
+      end
+    end
+
+    # One line, usable from the installer, the doctor and a boot log.
+    def advice
+      "json #{json_version} cannot be decoded by Rails #{rails_version} (#{ISSUE}): signed cookies, JSON " \
+        "columns and Railwatch's own migrations all raise ArgumentError on this pair. Add #{PIN} to your " \
+        "Gemfile until Rails ships the fix, then remove it."
+    end
+
+    def json_version = defined?(::JSON::VERSION) ? ::JSON::VERSION : "unknown"
+    def rails_version = defined?(::Rails) && ::Rails.respond_to?(:version) ? ::Rails.version : "unknown"
+
+    # Test hook.
+    def reset!
+      @broken = nil
+    end
+  end
+end

data/lib/railwatch/maintenance.rb

@@ -0,0 +1,183 @@
+# frozen_string_literal: true
+
+# TASKS is built from duration literals at load time; the engine has these
+# loaded already, but a writer process required ahead of the app must not
+# depend on that.
+require "active_support/core_ext/numeric/time"
+
+module Railwatch
+  # The embedded install's own clock. One background thread per web/worker
+  # process wakes every TICK seconds and runs whichever maintenance task is
+  # due: release-health rollups, threshold and anomaly scans, missed
+  # scheduled tasks, auto-resolve, pruning. Every process runs the clock;
+  # one process at a time runs a given task, claimed through a lease row in
+  # the railwatch database (MaintenanceTask), so a Puma cluster and a Solid
+  # Queue worker on the same host do not all prune at once.
+  #
+  # Same thread shape as Railwatch::Health: parked on a ConditionVariable,
+  # started from the engine, re-armed in every forked child. Nothing here
+  # goes through Active Job -- the bodies call the job classes' perform
+  # directly -- so an embedded install needs no worker, no recurring.yml,
+  # and never writes the host's queue adapter.
+  #
+  # A task must never be visible to the app it is maintaining: each body runs
+  # inside Railwatch.ignore and rescues everything.
+  module Maintenance
+    ROLES = %w[web worker writer].freeze
+    TICK = 30
+    FOLLOWUP_BATCHES_PER_TICK = 200
+
+    # name => [interval, lease, body]. The lease is how long a claim is held
+    # by a process that never releases it (crashed mid-task); it is a ceiling
+    # on the work, not an estimate of it.
+    TASKS = {
+      # A batch's exceptions are grouped into issues right after it commits;
+      # a process that dies in between leaves the work recorded on the
+      # batch's ledger row. Bounded per tick so a long outage drains over a
+      # few ticks rather than one long one.
+      "drain_followups" => [ 1.minute, 5.minutes, lambda { |env|
+        env.with_telemetry do
+          Telemetry::IngestBatch.with_pending_followups.limit(FOLLOWUP_BATCHES_PER_TICK).each do |batch|
+            batch.drain_followups!(env)
+          end
+        end
+      } ],
+      "release_health" => [ 1.minute, 5.minutes, lambda { |env|
+        now = Time.current
+        [ now.beginning_of_hour, (now - 1.hour).beginning_of_hour ].each do |bucket|
+          ReleaseHealthRollupJob.new.perform(env, bucket)
+        end
+      } ],
+      # Embedded batches fold themselves into the current hour as they land
+      # (Ingest::RollupAbsorber), so the reconciler only ever needs to catch
+      # rows that arrived after their hour closed.
+      "rollup_reconcile" => [ 1.hour, 10.minutes, lambda { |env|
+        RollupJob.new.perform(env, (Time.current - 1.hour).beginning_of_hour)
+      } ],
+      "performance_scan" => [ 5.minutes, 10.minutes, lambda { |env|
+        DetectPerformanceIssuesJob.new.perform(env)
+      } ],
+      "anomaly_scan" => [ 5.minutes, 10.minutes, lambda { |env|
+        DetectAnomaliesJob.new.perform(env) if AnomalyRule.where(enabled: true).exists?
+      } ],
+      "scheduled_tasks" => [ 10.minutes, 10.minutes, lambda { |env|
+        CheckScheduledTasksJob.new.perform(env)
+      } ],
+      "auto_resolve" => [ 24.hours, 10.minutes, lambda { |_env|
+        AutoResolveIssuesJob.new.perform
+      } ],
+      # PASSIVE, not TRUNCATE: this runs inside a Puma worker, and a
+      # truncating checkpoint blocks every reader and writer on the file.
+      "prune" => [ 24.hours, 60.minutes, lambda { |env|
+        PruneTelemetryJob.new.perform(env, checkpoint: "PASSIVE")
+        OptimizeTelemetryJob.new.perform(env)
+        FollowupReceipt.prune!
+      } ]
+    }.freeze
+
+    @mutex = Mutex.new
+    @wakeup = ConditionVariable.new
+    @thread = nil
+    @pid = nil
+    @stopping = false
+
+    module_function
+
+    # Idempotent; Railwatch.restart_after_fork! calls it again in every forked
+    # child.
+    def start!
+      return unless Railwatch.enabled? && Railwatch.config.local?
+      return if defined?(Rails) && Rails.env.test?
+      return unless ROLES.include?(Subscribers::ProcessInfo.role)
+      # With a writer process listening, it is the one clock. The lease table
+      # would keep two clocks honest, but there is no reason to run a second.
+      return if !Writer.running? && Writer.listening?
+      return if @thread&.alive? && @pid == Process.pid
+
+      @mutex.synchronize do
+        return if @thread&.alive? && @pid == Process.pid
+
+        @pid = Process.pid
+        @stopping = false
+        @thread = Thread.new { run }
+        @thread.name = "railwatch-maintenance"
+        @thread.abort_on_exception = false
+        @thread.report_on_exception = false
+      end
+    end
+
+    # A forked child inherits a dead thread and may inherit a mutex held by a
+    # vanished parent thread; every primitive is replaced before start!.
+    def restart_after_fork!
+      @mutex = Mutex.new
+      @wakeup = ConditionVariable.new
+      @thread = nil
+      @pid = nil
+      @stopping = false
+      start!
+    end
+
+    def stop!
+      return unless @thread
+
+      @stopping = true
+      @mutex.synchronize { @wakeup.signal }
+      @thread.join(1)
+      @thread = nil
+    end
+
+    def run
+      until @stopping
+        @mutex.synchronize { @wakeup.wait(@mutex, TICK) unless @stopping }
+        tick unless @stopping
+      end
+    end
+
+    # Runs every task that is due and unclaimed. Returns the names it ran.
+    # Public so a spec, or an operator in a console, can drive the clock by
+    # hand. One failing task is reported and does not stop the others.
+    #
+    # A web worker's clock starts at boot, before the Puma plugin has forked
+    # the writer, so the start-time check in start! cannot see it. Checked
+    # again on every tick: once a writer is listening this process's clock
+    # stands down and stays down (the lease table would keep both honest,
+    # but there is no reason to run a second one).
+    def tick(now: Time.current)
+      return [] if !Writer.running? && Writer.listening?
+
+      ran = []
+      Rails.application.executor.wrap do
+        env = Environment.current
+        TASKS.each do |name, (every, lease, body)|
+          token = MaintenanceTask.claim(name, every: every, lease: lease, owner: owner, now: now) or next
+
+          succeeded = false
+          begin
+            Railwatch.ignore { body.call(env) }
+            succeeded = true
+            ran << name
+          rescue StandardError => e
+            # Not Rails.error: this thread has no execution for Railwatch.ignore
+            # to pause, so a report there would be captured by Railwatch's own
+            # subscriber and opened as an application issue about Railwatch.
+            Railwatch.debug { "maintenance #{name} failed: #{e.class}: #{e.message}" }
+            Railwatch.notify_unrecoverable(TaskError.new("maintenance task #{name} failed: #{e.class}: #{e.message}"))
+          ensure
+            MaintenanceTask.release(name, token: token, ran_at: now, succeeded: succeeded)
+          end
+        end
+      end
+      ran
+    rescue StandardError => e
+      # The lease table not being migrated yet is the usual way to get here.
+      Railwatch.debug { "maintenance tick failed: #{e.class}: #{e.message}" }
+      ran
+    end
+
+    def owner
+      "#{Railwatch.config.server}:#{Process.pid}"
+    end
+
+    class TaskError < StandardError; end
+  end
+end

data/lib/railwatch/patches/runner_command.rb

@@ -107,13 +107,33 @@ module Railwatch
       # Expanded so a relative path is judged by where it actually resolves.
       # An argument File.expand_path refuses (a "~nobody/x.rb") is matched
       # as-is rather than assumed interactive: when in doubt, report.
+      #
+      # A scratch directory that contains the application itself is not a
+      # scratch directory for the application's own files: an app checked out
+      # under /tmp (a CI runner, a throwaway worktree) must not have its
+      # script/ classified as a shell session because of where the checkout
+      # lives. A scratch path inside the app (an operator's own
+      # `Rails.root/tmp/`) is left alone by this rule and still matches.
       def self.scratch?(argument)
         path = begin
           File.expand_path(argument)
         rescue StandardError
           argument
         end
-        Railwatch.config.interactive_runner_paths.any? { |directory| path.start_with?(directory) }
+        root = application_root
+        Railwatch.config.interactive_runner_paths.any? do |directory|
+          next false if root && "#{root}/".start_with?(directory) && path.start_with?("#{root}/")
+
+          path.start_with?(directory)
+        end
+      end
+
+      def self.application_root
+        return nil unless defined?(Rails) && Rails.respond_to?(:root) && Rails.root
+
+        Rails.root.to_s
+      rescue StandardError
+        nil
       end
     end
   end

data/lib/railwatch/record.rb

@@ -54,32 +54,57 @@ module Railwatch
     # Counting stops as soon as `limit` is exceeded: past that the only fact
     # the caller uses is "too big", so there is no reason to keep walking.
     def buffered_bytes(value, limit:, depth: 0)
-      return limit + 1 if depth > MAX_SIZING_DEPTH
+      bytes = weigh(value, limit, depth)
+      bytes > limit ? limit + 1 : bytes
+    end
 
-      bytes = case value
+    # The walk itself. Positional arguments, and scalars weighed inline in
+    # the container loops rather than through a call per value: a record is
+    # thirty-odd scalars under one Hash, so the common case is one call and
+    # one loop, and only a nested header or payload hash recurses. Halves the
+    # per-record cost against the one-method-per-value version (measured:
+    # 6.2 to 2.8 us for a query record).
+    def weigh(value, limit, depth)
+      case value
       when String then 40 + value.bytesize
       when Hash then hash_bytes(value, limit, depth)
       when Array then array_bytes(value, limit, depth)
-      when Symbol then 16
       else 16
       end
-      bytes > limit ? limit + 1 : bytes
     end
 
     def hash_bytes(hash, limit, depth)
+      # Children are weighed inline below, so the bound is checked for them
+      # here: a container whose contents would sit past MAX_SIZING_DEPTH is
+      # over the limit by definition, which is what a per-value depth check
+      # produced before the contents were inlined.
+      return limit + 1 if depth > MAX_SIZING_DEPTH || (depth == MAX_SIZING_DEPTH && !hash.empty?)
+
       bytes = 80 + (hash.size * 40)
-      hash.each do |key, value|
-        bytes += buffered_bytes(key, limit: limit, depth: depth + 1)
-        bytes += buffered_bytes(value, limit: limit, depth: depth + 1)
+      hash.each_pair do |key, value|
+        bytes += key.is_a?(Symbol) ? 16 : weigh(key, limit, depth + 1)
+        bytes += case value
+        when String then 40 + value.bytesize
+        when Hash then hash_bytes(value, limit, depth + 1)
+        when Array then array_bytes(value, limit, depth + 1)
+        else 16
+        end
         break if bytes > limit
       end
       bytes
     end
 
     def array_bytes(array, limit, depth)
+      return limit + 1 if depth > MAX_SIZING_DEPTH || (depth == MAX_SIZING_DEPTH && !array.empty?)
+
       bytes = 40 + (array.size * 8)
       array.each do |value|
-        bytes += buffered_bytes(value, limit: limit, depth: depth + 1)
+        bytes += case value
+        when String then 40 + value.bytesize
+        when Hash then hash_bytes(value, limit, depth + 1)
+        when Array then array_bytes(value, limit, depth + 1)
+        else 16
+        end
         break if bytes > limit
       end
       bytes

data/lib/railwatch/reporter.rb

@@ -187,6 +187,12 @@ module Railwatch
     end
 
     def forked_transport
+      # A child may be a different kind of process from its parent: the
+      # writer forked from a Puma master must write SQLite itself, not hand
+      # batches back to the socket it is about to serve.
+      reselected = Railwatch.local_transport if @config.local?
+      return reselected if reselected && reselected.class != @transport.class
+
       transport = @transport.dup
       transport.reset_after_fork! if transport.respond_to?(:reset_after_fork!)
       transport
@@ -327,6 +333,16 @@ module Railwatch
           @retry_attempt = 0
           @retry_at = nil
           Railwatch.debug { "gave up on a batch of #{batch.records.size} records after #{MAX_RETRY_ATTEMPTS} retries (#{result.error || result.status}); dropped and counted" }
+          # Losing a batch is not a debug-level event: with an ingest (or an
+          # embedded writer) that never comes back this is the only place the
+          # loss is ever reported, and the dropped counter it leaves behind
+          # rides on the NEXT successful delivery, which may never happen.
+          Railwatch.notify_unrecoverable(
+            DeliveryError.new("Railwatch dropped #{batch.records.size} records after #{MAX_RETRY_ATTEMPTS} failed delivery attempts: " \
+                              "#{result.error || result.status}",
+                              status: result.status, records: batch.records.size, bytes: batch.bytes,
+                              dropped: batch.dropped, dropped_bytes: batch.dropped_bytes)
+          )
           next
         end
         @retry_batch = batch