railwatch 0.1.0

data/lib/railwatch/patches/net_http.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require "net/http"
+
+module Railwatch
+  module Patches
+    # Outgoing HTTP. Net::HTTP is under Faraday's default adapter, HTTParty,
+    # RestClient, and ruby-llm, so one prepend covers most of the ecosystem.
+    # Requests to the Railwatch ingest itself are skipped.
+    module NetHttp
+      REENTRY = :railwatch_net_http
+
+      def request(req, body = nil, &block)
+        return super if Thread.current[REENTRY] || !Railwatch.enabled? || railwatch_self_request?
+        exe = Railwatch.execution
+        # Before the recording? gate: a sampled-out execution still propagates
+        # its trace context, just with the "not sampled" flag.
+        Railwatch::Patches::NetHttp.propagate_trace(req, address)
+        return super if exe && !exe.recording?
+
+        Thread.current[REENTRY] = true
+        start = Clock.monotonic
+        started_at = Clock.now
+        response = nil
+        error = nil
+        begin
+          response = super
+        rescue StandardError => e
+          error = e
+          raise
+        ensure
+          Thread.current[REENTRY] = nil
+          exe&.count(:outgoing_requests)
+          Railwatch::Patches::NetHttp.record(self, req, response, error, start, started_at)
+        end
+      end
+
+      def railwatch_self_request?
+        ingest = URI(Railwatch.config.ingest_url)
+        address == ingest.host && port == ingest.port
+      rescue StandardError
+        false
+      end
+
+      # Never overwrites a traceparent the app set itself.
+      def self.propagate_trace(req, host)
+        return if req.key?("traceparent")
+
+        traceparent = Railwatch.traceparent(host)
+        req["traceparent"] = traceparent if traceparent
+      rescue StandardError => e
+        Railwatch.debug { "traceparent propagation failed: #{e.message}" }
+      end
+
+      def self.record(http, req, response, error, start, started_at)
+        host = http.address
+        default_port = http.use_ssl? ? 443 : 80
+        url = "#{http.use_ssl? ? 'https' : 'http'}://#{host}#{http.port == default_port ? '' : ":#{http.port}"}#{req.path}"
+        Railwatch.record(:outgoing_request,
+          group: Record.group_hash(host, req.method),
+          timestamp: started_at,
+          host: host,
+          method: req.method,
+          url: Record.url_without_sensitive_components(url, limit: 2048),
+          duration: Clock.micros_since(start),
+          status_code: response&.code.to_i,
+          request_size: (req.body || "").bytesize,
+          response_size: response ? (response["Content-Length"]&.to_i || response.body&.bytesize rescue nil) : nil,
+          error: error && "#{error.class}: #{error.message}"[0, 255],
+          response_body: response_body(response, error),
+          source: Backtrace.caller_location(skip: 4))
+      rescue StandardError => e
+        Railwatch.debug { "outgoing request record failed: #{e.message}" }
+      end
+
+      RESPONSE_BODY_MAX = 4096
+
+      def self.response_body(response, error)
+        return nil unless Railwatch.config.capture_response_body_on_error
+        return nil unless error || response&.code.to_i >= 400
+
+        # Net::HTTPResponse#body reads from the socket the first time it is
+        # called, which would consume a response the caller is streaming out
+        # of #read_body. @body holds a String only once Net::HTTP has already
+        # buffered the whole body (which #request does for every response it
+        # isn't streaming), so reading it here can never touch the socket.
+        captured_response_body(response&.instance_variable_get(:@body))
+      end
+
+      # Shared with Railwatch::Faraday. A JSON object body goes through the
+      # same parameter filter as request params and is re-serialized; any
+      # other body has no keys to match, so it is stored as it arrived.
+      def self.captured_response_body(body)
+        return nil unless body.is_a?(String) && !body.empty?
+
+        parsed = (JSON.parse(body) rescue nil)
+        body = JSON.generate(Railwatch.redactor.params(parsed)) if parsed.is_a?(Hash)
+        body[0, RESPONSE_BODY_MAX]
+      end
+    end
+  end
+end

data/lib/railwatch/patches/rake_task.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module Railwatch
+  module Patches
+    # Rake tasks are Rails' commands. Each top-level task invocation is a
+    # `command` execution; nested tasks (prerequisites) run inside it.
+    module RakeTask
+      SKIP = %w[environment].freeze
+
+      def execute(args = nil)
+        run_as_command(args_suffix(args)) { super }
+      end
+
+      # Rake dispatches a task's prerequisites via #invoke (Task#invoke_with_call_chain
+      # calls invoke_prerequisites, which fully invokes -- and executes -- each
+      # prerequisite, BEFORE calling the dependent task's own #execute). Patching
+      # #invoke means the top-level command execution is already open by the time
+      # prerequisites run, so their own #invoke/#execute calls see Railwatch.execution
+      # is not nil and just run plain, nesting inside the one command record instead
+      # of each starting (and finishing) their own.
+      def invoke(*args)
+        run_as_command(args.any? ? "[#{args.join(',')}]" : "") { super }
+      end
+
+      private
+
+      # A vendor-excluded task (db:migrate, say) can internally call
+      # `Rake::Task["db:_dump"].invoke` from its own action body -- an
+      # implementation detail, not a prerequisite -- to run a task that isn't
+      # itself vendor-excluded. With no execution open (the outer task never
+      # started one), that inner call looks exactly like a fresh top-level
+      # invocation and would ship its own unwanted command record. This flag
+      # marks "we're inside a task we deliberately chose not to track," so
+      # anything invoked underneath it is left untracked too.
+      def run_as_command(command_suffix)
+        return yield unless Railwatch.enabled? && !SKIP.include?(name) && Railwatch.execution.nil?
+        return yield if Thread.current[:railwatch_vendor_excluded_rake]
+
+        if vendor_excluded?
+          Thread.current[:railwatch_vendor_excluded_rake] = true
+          begin
+            return yield
+          ensure
+            Thread.current[:railwatch_vendor_excluded_rake] = false
+          end
+        end
+
+        exe = Railwatch.start_execution(source: :command, sample_kind: :commands, preview: "rake #{name}")
+        exe.enter_stage(:action)
+        exit_code = 0
+        begin
+          yield
+        rescue SystemExit => e
+          exit_code = e.status
+          raise
+        rescue SignalException => e
+          # SIGTERM/SIGINT is how Kamal, systemd and Ctrl-C stop a long-running
+          # task (a litestream replicator, a queue worker); it is a shutdown,
+          # not a failure, so the command closes with the signal's exit code
+          # and no exception is reported.
+          exit_code = 128 + (e.signo || 0)
+          raise
+        rescue Exception => e # rubocop:disable Lint/RescueException
+          exit_code = 1
+          Railwatch::Subscribers::Exceptions.capture(e, handled: false, severity: :error, source: "application.rake")
+          raise
+        ensure
+          exe.finish_stages
+          Railwatch.finish_execution(:command,
+            group: Record.group_hash(name),
+            class: "Rake::Task",
+            name: name,
+            command: "rake #{name}#{command_suffix}",
+            exit_code: exit_code.to_i.clamp(0, 255))
+          Railwatch.flush
+        end
+      end
+
+      def args_suffix(args)
+        args.respond_to?(:to_a) && args.to_a.any? ? "[#{args.to_a.join(',')}]" : ""
+      end
+
+      def vendor_excluded?
+        !Railwatch.config.capture_default_vendor_commands && Configuration::DEFAULT_VENDOR_COMMANDS.include?(name)
+      end
+    end
+  end
+end

data/lib/railwatch/patches/runner_command.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+module Railwatch
+  module Patches
+    # `bin/rails runner` is a command execution, same as a rake task, but
+    # Rails::Command::RunnerCommand#perform isn't a Rake::Task so it needs its
+    # own prepend.
+    #
+    # Two very different things arrive here, and only one of them is an issue.
+    # A DEPLOYED script -- `rails runner script/nightly.rb` from cron, a
+    # release step, a container entrypoint -- must report: a nightly job that
+    # starts dying is exactly what monitoring is for. An INTERACTIVE run is an
+    # engineer at a shell typing at production, and their typo (a misspelled
+    # attribute, a tenant slug that does not exist, an `unless ... next` that
+    # does not parse) is the ops equivalent of a shell error, not a bug in the
+    # app. Under Sentry those typos opened four of fifteen unresolved issues
+    # in this app and woke an automated responder each time.
+    #
+    # The line is WHERE THE CODE CAME FROM, which railties makes plain -- its
+    # #perform reaches the operator's code through three call sites:
+    #
+    #   rails runner -            -> eval($stdin.read, TOPLEVEL_BINDING, "stdin")
+    #   rails runner 'Some.code'  -> eval(code_or_file, TOPLEVEL_BINDING, __FILE__, __LINE__)
+    #   rails runner script.rb    -> Kernel.load(expanded_file_path)
+    #
+    # so the argument alone answers it: `-` is piped, anything that is not a
+    # `.rb` path was typed inline, and a `.rb` path is a file -- deployed
+    # unless it sits in a scratch directory (config.interactive_runner_paths),
+    # because nothing an app deploys lives in /tmp.
+    #
+    # An interactive run still opens its execution and still ships the
+    # `command` record (with exit_code, and `interactive: true`) -- you can
+    # see that someone ran it, and what it cost. Only the exception is
+    # withheld, and that is done by flagging the execution rather than by
+    # skipping the capture below: the Rails executor hands the error to
+    # Rails.error inside `super`, so Subscribers::Exceptions has already seen
+    # it by the time this rescue runs.
+    module RunnerCommand
+      def perform(code_or_file = nil, *command_argv)
+        return super unless Railwatch.enabled? && Railwatch.execution.nil?
+
+        RunnerCommand.instrument(code_or_file) { super }
+      end
+
+      # The prepend happens from the engine's runner hook, which #perform
+      # reaches (through boot_application! and load_runner) after it has
+      # already started -- so in a real `bin/rails runner` process the
+      # invocation on the stack is the unpatched one and the override above
+      # never runs (it does when railties' runner command was loaded before
+      # boot, as in this gem's own specs). Method lookup is dynamic, though:
+      # by the time that #perform reaches conditional_executor the prepend
+      # is in place, so this is where a runner started from the shell gets
+      # its execution. Thor keeps the positional arguments on #args, and the
+      # first one is code_or_file.
+      def conditional_executor(enabled, **kwargs, &block)
+        return super unless Railwatch.enabled? && Railwatch.execution.nil?
+
+        RunnerCommand.instrument(args.first) { super }
+      end
+
+      # Opens the command execution around the runner's work, withholding the
+      # exception (but not the command record) for an interactive run.
+      def self.instrument(code_or_file)
+        preview = code_or_file.to_s[0, 200]
+        interactive = interactive?(code_or_file)
+        exe = Railwatch.start_execution(source: :command, sample_kind: :commands, preview: "rails runner #{preview}")
+        exe.interactive = true if interactive
+        exe.enter_stage(:action)
+        exit_code = 0
+        begin
+          yield
+        rescue SystemExit => e
+          exit_code = e.status
+          raise
+        rescue SignalException => e
+          # A signal ends the runner by design (see the rake patch); not reported.
+          exit_code = 128 + (e.signo || 0)
+          raise
+        rescue Exception => e # rubocop:disable Lint/RescueException
+          exit_code = 1
+          Railwatch::Subscribers::Exceptions.capture(e, handled: false, severity: :error, source: "application.runner")
+          raise
+        ensure
+          exe.finish_stages
+          fields = {
+            group: Record.group_hash("runner"),
+            class: "Rails::Command::RunnerCommand",
+            name: "runner",
+            command: "rails runner #{preview}",
+            exit_code: exit_code.to_i.clamp(0, 255)
+          }
+          fields[:interactive] = true if interactive
+          Railwatch.finish_execution(:command, **fields)
+          Railwatch.flush
+        end
+      end
+
+      # Typed or piped by a human, rather than loaded from a deployed file.
+      def self.interactive?(code_or_file)
+        argument = code_or_file.to_s
+        # "-" (stdin), "" (railties prints help and exits), and inline code.
+        return true unless argument.end_with?(".rb")
+
+        scratch?(argument)
+      end
+
+      # Expanded so a relative path is judged by where it actually resolves.
+      # An argument File.expand_path refuses (a "~nobody/x.rb") is matched
+      # as-is rather than assumed interactive: when in doubt, report.
+      def self.scratch?(argument)
+        path = begin
+          File.expand_path(argument)
+        rescue StandardError
+          argument
+        end
+        Railwatch.config.interactive_runner_paths.any? { |directory| path.start_with?(directory) }
+      end
+    end
+  end
+end

data/lib/railwatch/patches.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require "railwatch/patches/net_http"
+require "railwatch/patches/rake_task"
+require "railwatch/patches/runner_command"
+require "railwatch/patches/inertia"
+
+module Railwatch
+  module Patches
+    module_function
+
+    # The patches every process needs. Rake and the runner command are
+    # installed from the engine's rake_tasks and runner hooks instead, which
+    # fire only in a process that is actually about to run one: requiring
+    # rake and railties' runner command in every web and worker boot cost
+    # about 170 ms for code those processes never call.
+    def install!
+      ::Net::HTTP.prepend(NetHttp) unless ::Net::HTTP.ancestors.include?(NetHttp)
+      Inertia.install!
+    end
+
+    # From Rails::Engine#load_tasks, which has already required rake.
+    def install_rake_task!
+      ::Rake::Task.prepend(RakeTask) unless ::Rake::Task.ancestors.include?(RakeTask)
+    end
+
+    # From Rails::Application#load_runner, which RunnerCommand#perform calls
+    # after boot_application! -- so the class is loaded by the time this
+    # runs, and the prepend is in place before conditional_executor.
+    def install_runner_command!
+      require "rails/command"
+      require "rails/commands/runner/runner_command"
+      unless ::Rails::Command::RunnerCommand.ancestors.include?(RunnerCommand)
+        ::Rails::Command::RunnerCommand.prepend(RunnerCommand)
+      end
+    rescue LoadError, StandardError => e
+      # A railties rename would land here; say so rather than leaving every
+      # deployed `rails runner` script silently untraced.
+      Railwatch.debug { "runner command patch not installed: #{e.class}: #{e.message}" }
+      nil
+    end
+  end
+end

data/lib/railwatch/profiler.rb

@@ -0,0 +1,270 @@
+# frozen_string_literal: true
+
+module Railwatch
+  # Sampling profiler for one execution, on top of whichever backend the app
+  # has in its Gemfile: Vernier (preferred) or StackProf. Both are optional
+  # dependencies, and every path here degrades to nil rather than raising --
+  # a profile is a nice-to-have, never a reason to break a request.
+  #
+  # Both backends are process-global: there is one profiler per process, not
+  # one per thread. An execution that starts while another one is being
+  # profiled is simply not profiled, and is counted in `skipped`.
+  module Profiler
+    # What `stop` hands back. `interval` and `duration` are microseconds;
+    # `collapsed` is the folded-stack text described on `collapse`.
+    Profile = Struct.new(:profiler, :mode, :interval, :duration, :samples, :collapsed)
+
+    # The profile currently running in this process: which backend started
+    # it, when, and on which thread. Vernier samples every thread in the
+    # process, and only the thread that asked for the profile is running
+    # this execution.
+    Handle = Struct.new(:backend, :mode, :interval, :started, :thread_id)
+
+    # Preference order when config.profiler doesn't pin one.
+    BACKENDS = %i[vernier stackprof].freeze
+
+    # Uncompressed cap on the collapsed text. Rails stacks run 30-200 frames
+    # deep, so a busy request folds to a few hundred KiB; gzip takes that
+    # down ~30x, well inside a batch. Over the cap the least frequent stacks
+    # are dropped: the long tail of one-sample stacks, not the profile's
+    # shape.
+    MAX_COLLAPSED_BYTES = 4 * 1024 * 1024
+
+    # Frames with no Ruby file of their own (C functions). Vernier reports
+    # those as "<cfunc>", StackProf as "<cfunc>" with a nil line.
+    CFUNC = "<cfunc>"
+
+    # Ruby's own stdlib directory, e.g. .../lib/ruby/3.4.0/.
+    RUBY_LIB_PREFIX = "#{RbConfig::CONFIG['rubylibdir']}/"
+
+    @lock = Mutex.new
+    @running = nil
+    @loadable = {}
+    @skipped = 0
+
+    class << self
+      # Executions that wanted a profile while another one was already being
+      # profiled in this process. Read by tests and diagnostics.
+      attr_reader :skipped
+
+      # Whether this process can profile at all. Memoised: the require is
+      # the expensive part (see loadable?), choosing between two symbols is
+      # not, so config.profiler stays live and overridable.
+      def available?
+        !backend.nil?
+      end
+
+      # :vernier, :stackprof, or nil when neither gem is installed.
+      # config.profiler pins one by name (an unknown name simply doesn't
+      # load, so profiling stays off); otherwise the first backend that
+      # loads wins.
+      def backend
+        wanted = Railwatch.config.profiler
+        return loadable?(wanted.to_sym) ? wanted.to_sym : nil if wanted
+
+        BACKENDS.find { |name| loadable?(name) }
+      end
+
+      # Starts the process-global profiler and returns a handle, or nil when
+      # no backend is installed, one is already running, or the backend
+      # refused to start.
+      def start(mode: :wall)
+        return nil unless available?
+
+        @lock.synchronize do
+          if @running
+            @skipped += 1
+            next nil
+          end
+
+          handle = Handle.new(backend, mode, Railwatch.config.profile_interval_us,
+                              Clock.monotonic, Thread.current.object_id)
+          # StackProf returns false rather than raising when something else
+          # in the process is already profiling.
+          if start_backend(handle) == false
+            @skipped += 1
+            next nil
+          end
+          @running = handle
+        end
+      rescue StandardError => e
+        Railwatch.debug { "profiler start failed: #{e.class}: #{e.message}" }
+        nil
+      end
+
+      # Stops the running profile and folds it into a Profile, or nil when
+      # nothing was running or anything at all went wrong on the way.
+      def stop
+        handle = @lock.synchronize { @running.tap { @running = nil } }
+        return nil unless handle
+
+        duration = Clock.micros_since(handle.started)
+        result = stop_backend(handle)
+        return nil unless result
+
+        counts = sample_counts(result, handle.thread_id)
+        Profile.new(handle.backend, handle.mode, handle.interval, duration,
+                    counts.each_value.sum, format_counts(counts))
+      rescue StandardError => e
+        Railwatch.debug { "profiler stop failed: #{e.class}: #{e.message}" }
+        nil
+      end
+
+      # Folded-stack text for a backend result: one `outermost;...;leaf
+      # count` line per unique stack, most sampled first with ties broken by
+      # the stack text, so the same profile always serialises to the same
+      # bytes. Capped at MAX_COLLAPSED_BYTES.
+      def collapse(result, thread_id: nil)
+        format_counts(sample_counts(result, thread_id))
+      end
+
+      # Test hook: forgets which backends loaded, plus any state a killed
+      # profile left behind.
+      def reset!
+        @lock.synchronize do
+          @running = nil
+          @loadable = {}
+          @skipped = 0
+        end
+      end
+
+      # Process._fork hook. A child inherits @running still holding the
+      # Handle for a profile the parent was taking, and nothing in the child
+      # ever stops it: `start` then sees a profile already running and every
+      # execution in that worker is counted as skipped and never profiled
+      # again. Clear the process-global state without taking @lock -- the
+      # child is single-threaded here, and Ruby has already abandoned any
+      # mutex a parent thread held across the fork.
+      def restart_after_fork!
+        @running = nil
+        @loadable = {}
+        @skipped = 0
+        self
+      end
+
+      private
+
+      # Whether a backend gem can be required, resolved once per process:
+      # requiring is what has to be memoised, not the choice between two
+      # symbols.
+      def loadable?(name)
+        return @loadable[name] if @loadable.key?(name)
+
+        @loadable[name] = begin
+          require name.to_s
+          true
+        rescue LoadError
+          false
+        end
+      end
+
+      def start_backend(handle)
+        case handle.backend
+        when :vernier then ::Vernier.start_profile(mode: handle.mode, interval: handle.interval)
+        when :stackprof then ::StackProf.start(mode: handle.mode, interval: handle.interval, raw: true)
+        end
+      end
+
+      def stop_backend(handle)
+        case handle.backend
+        when :vernier
+          ::Vernier.stop_profile
+        when :stackprof
+          ::StackProf.stop
+          ::StackProf.results
+        end
+      end
+
+      # {collapsed stack => sample count}. StackProf hands back a Hash,
+      # Vernier a Vernier::Result.
+      def sample_counts(result, thread_id)
+        result.is_a?(Hash) ? stackprof_counts(result) : vernier_counts(result, thread_id)
+      end
+
+      # Vernier samples every thread in the process -- an idle thread
+      # sleeping through the whole window otherwise outweighs the work being
+      # profiled -- so only the thread that started the profile is folded in.
+      def vernier_counts(result, thread_id)
+        counts = Hash.new(0)
+        thread = result.threads[thread_id] || result.main_thread
+        return counts unless thread
+
+        table = result.stack_table
+        labels = {}
+        lines = {}
+        weights = thread[:weights]
+        thread[:samples].each_with_index do |stack_idx, i|
+          counts[lines[stack_idx] ||= vernier_line(table, stack_idx, labels)] += weights[i]
+        end
+        counts
+      end
+
+      # Vernier's stack table is a tree of leaf -> parent links, so the walk
+      # comes out leaf-first and is reversed into folded order.
+      def vernier_line(table, stack_idx, labels)
+        frames = []
+        while stack_idx
+          func_idx = table.frame_func_idx(table.stack_frame_idx(stack_idx))
+          frames << (labels[func_idx] ||= label(table.func_name(func_idx), table.func_filename(func_idx),
+                                                table.func_first_lineno(func_idx)))
+          stack_idx = table.stack_parent_idx(stack_idx)
+        end
+        frames.reverse.join(";")
+      end
+
+      # StackProf's :raw is a flat array of `depth, frame_id * depth, weight`
+      # groups with the outermost frame first -- decoded exactly as
+      # StackProf::Report#print_stackcollapse decodes it.
+      def stackprof_counts(result)
+        counts = Hash.new(0)
+        frames = result[:frames]
+        raw = result[:raw]
+        return counts unless frames && raw
+
+        labels = {}
+        i = 0
+        while (depth = raw[i])
+          line = raw[i + 1, depth].map { |id| labels[id] ||= stackprof_label(frames[id]) }.join(";")
+          counts[line] += raw[i + depth + 1].to_i
+          i += depth + 2
+        end
+        counts
+      end
+
+      def stackprof_label(frame)
+        label(frame[:name], frame[:file], frame[:line])
+      end
+
+      # "Class#method (path:line)".
+      def label(name, file, line)
+        "#{name} (#{short_path(file)}:#{line.to_i})"
+      end
+
+      # App paths lose the Rails root, installed-gem paths become
+      # "<gem>/relative/path" (version dropped: it repeats on every frame of
+      # every line and the deploy already records it), Ruby's own lib
+      # becomes "ruby/...". A frame with no file at all is a C function.
+      def short_path(file)
+        file = file.to_s
+        return CFUNC if file.empty? || file == CFUNC
+        return file.delete_prefix(Backtrace.app_root) if file.start_with?(Backtrace.app_root)
+        return "ruby/#{file.delete_prefix(RUBY_LIB_PREFIX)}" if file.start_with?(RUBY_LIB_PREFIX)
+        return file unless Backtrace.installed_gem_path?(file)
+
+        dir = Gem.path.find { |path| file.start_with?("#{path}/") }
+        rest = file.delete_prefix("#{dir}/").delete_prefix("gems/")
+        rest.sub(/\A([^\/]+?)-\d[^\/]*\//, '\1/')
+      end
+
+      def format_counts(counts)
+        text = +""
+        counts.sort_by { |stack, count| [ -count, stack ] }.each do |stack, count|
+          line = "#{stack} #{count}\n"
+          break if text.bytesize + line.bytesize > MAX_COLLAPSED_BYTES
+          text << line
+        end
+        text
+      end
+    end
+  end
+end