railwatch 0.1.0

data/lib/railwatch/attachments.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+require "base64"
+require "pathname"
+require "zlib"
+
+module Railwatch
+  # Railwatch.attach: ship an arbitrary blob -- the JSON payload that failed to
+  # parse, a rendered PDF, the webhook body a customer swears they sent -- as
+  # its own `attachment` record. Sentry's `Sentry.add_attachment` equivalent,
+  # except an attachment is a first-class record linked to the execution it
+  # was made in, and optionally to the exception it explains.
+  #
+  #   Railwatch.attach("payload.json", request.raw_post)
+  #   Railwatch.attach("invoice.pdf", Rails.root.join("tmp/invoice.pdf"))
+  #   Railwatch.attach("payload.json", body, exception: error)
+  #
+  # The wire field `data` is base64 of gzip, so a text payload costs a
+  # fraction of its size in the batch.
+  module Attachments
+    DEFAULT_CONTENT_TYPE = "application/octet-stream"
+    MAX_NAME = 255
+    MAX_CONTENT_TYPE = 128
+
+    module_function
+
+    def attach(name, data, content_type: nil, exception: nil)
+      return nil unless Railwatch.enabled?
+
+      # cap + 1 so "was it truncated?" is still answerable without ever
+      # holding more than the cap in memory. A 2GB log file used to be read
+      # whole and then sliced.
+      cap = Railwatch.config.max_attachment_bytes
+      bytes = read(data, cap + 1)
+      return nil if bytes.nil? || bytes.empty?
+
+      name = name.to_s[0, MAX_NAME]
+      truncated = bytes.bytesize > cap
+      bytes = bytes.byteslice(0, cap) if truncated
+
+      fields = {
+        name: name,
+        content_type: (content_type || content_type_for(name)).to_s[0, MAX_CONTENT_TYPE],
+        bytes: bytes.bytesize,
+        data: Base64.strict_encode64(Zlib.gzip(bytes)),
+        exception_group_hash: exception && Subscribers::Exceptions.group_for(exception)
+      }
+      fields[:truncated] = true if truncated
+      # `attachment` is in Railwatch::STANDALONE_TYPES, so this buffers as a
+      # child of the current execution when one is recording and ships on its
+      # own (from a boot hook, a console, a rescue with nothing executing)
+      # when there isn't one.
+      Railwatch.record(:attachment, group: Record.group_hash(name), **fields)
+    end
+
+    # A String is the data itself; a Pathname is a file to read; anything else
+    # that responds to #read (File, StringIO, an uploaded file) is read.
+    def read(data, limit)
+      case data
+      when nil then nil
+      when String then data.byteslice(0, limit)
+      when Pathname then File.binread(data, limit)
+      else
+        value = data.respond_to?(:read) ? data.read(limit) : data.to_s
+        value.to_s.byteslice(0, limit)
+      end
+    rescue StandardError => e
+      Railwatch.debug { "attachment read failed: #{e.class}: #{e.message}" }
+      nil
+    end
+
+    # Marcel comes with Rails (Active Storage depends on it), but Railwatch's
+    # only declared dependency is rails itself, so an app that has somehow
+    # dropped it still attaches -- just with the generic type.
+    def content_type_for(name)
+      return DEFAULT_CONTENT_TYPE unless defined?(::Marcel::MimeType)
+
+      ::Marcel::MimeType.for(name: name) || DEFAULT_CONTENT_TYPE
+    rescue StandardError
+      DEFAULT_CONTENT_TYPE
+    end
+  end
+end

data/lib/railwatch/backtrace.rb

@@ -0,0 +1,158 @@
+# frozen_string_literal: true
+
+module Railwatch
+  # Locates the app frame that caused a query or outgoing request, and
+  # serialises exception frames with source snippets for app code.
+  module Backtrace
+    module_function
+
+    def app_root
+      @app_root ||= (defined?(Rails) && Rails.root ? Rails.root.to_s + "/" : Dir.pwd + "/")
+    end
+
+    def clean(frames)
+      cleaner = defined?(Rails) && Rails.respond_to?(:backtrace_cleaner) ? Rails.backtrace_cleaner : nil
+      cleaner ? cleaner.clean(frames) : frames
+    end
+
+    # First application frame as "app/models/user.rb:12", or nil. 100 frames
+    # clears framework internals in practice (Rails 8.1's instrumentation and
+    # query-cache wrapping alone run 40+ frames deep before reaching app code,
+    # more once a view template is on the stack) and keeps the walk (and the
+    # array it allocates) cheap on the hot query path.
+    # The gem's own lib/ directory, so frames inside Railwatch are skipped by
+    # prefix rather than by a substring that would also match any app whose
+    # checkout happens to live under a folder named "railwatch" (including
+    # this repo's own spec/dummy, which lives under the repo root but not
+    # under its lib/).
+    GEM_ROOT = File.expand_path("..", __dir__) + "/"
+
+    def gem_root
+      GEM_ROOT
+    end
+
+    def caller_location(skip: 2)
+      locations = caller_locations(skip, 100) or return nil
+      locations.each do |loc|
+        path = loc.path
+        next if path.start_with?(gem_root)
+        next if installed_gem_path?(path)
+        return "#{path.delete_prefix(app_root)}:#{loc.lineno}"
+      end
+      nil
+    end
+
+    # True for frames inside an installed gem (net-http, webmock, rspec-core,
+    # etc). Railwatch itself is excluded separately via gem_root because in
+    # this repo's own test suite (and in a `path:`/`git:` Gemfile checkout)
+    # it is loaded straight from a working tree, not from a Gem.path install.
+    def installed_gem_path?(path)
+      Gem.path.any? { |p| path.start_with?("#{p}/") }
+    end
+
+    # One line of a String backtrace: "path:line:in 'label'" (Ruby 3.4),
+    # "path:line:in `label'" (earlier), or a bare "path:line".
+    BACKTRACE_LINE = /\A(.+?):(\d+)(?::in [`'](.*)')?\z/
+
+    def frames(exception, with_source: true, limit: 50)
+      raw_frames(exception, limit).map do |path, lineno, label|
+        in_app = path.to_s.start_with?(app_root)
+        frame = {
+          file: in_app ? path.delete_prefix(app_root) : path,
+          line: lineno,
+          function: label,
+          in_app: in_app
+        }
+        frame[:code] = source_snippet(path, lineno) if with_source && in_app
+        frame
+      end
+    end
+
+    # [path, line, label] per frame. backtrace_locations is nil for any
+    # exception whose backtrace was assigned rather than raised into it --
+    # ActiveRecord::StatementInvalid (set_backtrace from the driver error),
+    # Faraday::Error (delegates #backtrace to the wrapped exception) -- which
+    # are the most common production exceptions, so fall back to parsing the
+    # strings rather than shipping them with no frames, no culprit, and a
+    # fingerprint of nothing but class and message.
+    def raw_frames(exception, limit)
+      locations = exception.backtrace_locations
+      if locations
+        locations.first(limit).map { |loc| [ loc.absolute_path || loc.path, loc.lineno, loc.label ] }
+      else
+        Array(exception.backtrace).first(limit).filter_map do |line|
+          match = BACKTRACE_LINE.match(line.to_s) or next
+          [ match[1], match[2].to_i, match[3] ]
+        end
+      end
+    end
+
+    # --- Browser stacks ---------------------------------------------------
+
+    # One frame of a JavaScript stack, in either of the two shapes engines
+    # write: V8's "at fn (https://host/assets/app-abc.js:1:2)" (and the same
+    # line without the function name), or SpiderMonkey and JavaScriptCore's
+    # "fn@https://host/assets/app-abc.js:1:2". A line with no file:line on it
+    # -- V8's leading "TypeError: ..." header, "at new Promise (<anonymous>)"
+    # -- matches neither and is dropped.
+    JS_FRAME = /
+      \A
+      (?:at\s+)?                       # V8 indents every frame with "at "
+      (?:(?<function>[^@]*?)\s*[@(])?  # "fn@" (Firefox, Safari) or "fn (" (V8)
+      (?<file>\S+?)
+      :(?<line>\d+)(?::(?<column>\d+))? # line and column (both one-based)
+      \)?
+      \z
+    /x
+
+    # Frames from the app's own origin that are still not the app's code.
+    VENDOR_PATH = %r{(?:\A|/)(?:node_modules|vendor)\b}
+
+    MAX_JS_FRAMES = 50
+
+    # A browser stack, exactly as the engine wrote it, in the same frame
+    # shape as a Ruby backtrace. `origin` is the app's own scheme and host: a
+    # script served from it is the app's own, so its file is stored relative
+    # to that origin the way a Ruby frame is stored relative to Rails.root,
+    # and anything from a CDN, an extension, or a third-party tag keeps its
+    # whole URL and is not in_app.
+    def js_frames(stack, origin: nil, limit: MAX_JS_FRAMES)
+      frames = []
+      stack.to_s.each_line do |raw|
+        break if frames.size >= limit
+        match = JS_FRAME.match(raw.strip) or next
+        url = match[:file].split("?", 2).first.to_s
+        own = js_own_origin?(url, origin)
+        file = own ? url.delete_prefix(origin.to_s).delete_prefix("/") : url
+        function = match[:function].to_s.strip
+        frame = {
+          file: file[0, 255],
+          line: match[:line].to_i,
+          function: function.empty? ? "(anonymous)" : function[0, 255],
+          in_app: own && !VENDOR_PATH.match?(file)
+        }
+        frame[:column] = match[:column].to_i if match[:column]
+        frames << frame
+      end
+      frames
+    end
+
+    # A bare path ("/assets/app.js") can only be the app's own; an absolute
+    # URL is only the app's own when it is on the app's origin.
+    def js_own_origin?(url, origin)
+      return true if url.start_with?("/")
+      return false if origin.nil? || origin.empty?
+      url.start_with?("#{origin}/")
+    end
+
+    def source_snippet(path, line, context: 5)
+      return nil unless path && File.readable?(path)
+      lines = File.readlines(path, chomp: true)
+      from = [ line - context - 1, 0 ].max
+      to = [ line + context - 1, lines.size - 1 ].min
+      (from..to).to_h { |i| [ i + 1, lines[i].to_s[0, 200] ] }
+    rescue StandardError
+      nil
+    end
+  end
+end

data/lib/railwatch/buffer.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+module Railwatch
+  # Bounded, thread-safe queue of records. When full, the oldest record is
+  # dropped and counted; the drop count is reported with the next batch so
+  # loss is visible on the platform instead of silent.
+  #
+  # "Full" is two limits, not one. A record count alone does not bound memory:
+  # 10,000 records is a few megabytes of ordinary telemetry and a gigabyte of
+  # captured attachments or a query record carrying a multi-megabyte SQL
+  # string. Each record's weight is measured once, when it is pushed, and
+  # carried alongside it so shifting one out is arithmetic rather than a
+  # re-measure.
+  class Buffer
+    def initialize(capacity, byte_capacity: Float::INFINITY)
+      @capacity = capacity
+      @byte_capacity = byte_capacity
+      @records = []
+      @record_bytes = []
+      @bytes = 0
+      @dropped = 0
+      @dropped_bytes = 0
+      @mutex = Mutex.new
+    end
+
+    # `bytes` is the caller's already-measured weight (Execution weighed the
+    # record when it buffered it); anything else is weighed here. Weighing
+    # stops at the ceiling, so the dropped-byte counter is a floor for an
+    # over-ceiling record; the dropped-record counter is always exact.
+    def push(record, bytes = nil)
+      bytes ||= Record.buffered_bytes(record, limit: @byte_capacity)
+      @mutex.synchronize do
+        # A single record heavier than the whole queue can only be dropped:
+        # making room for it would mean emptying the queue and still not
+        # fitting.
+        if bytes > @byte_capacity
+          drop(bytes)
+          next @records.size
+        end
+
+        drop_oldest while @records.any? && (@records.size >= @capacity || @bytes + bytes > @byte_capacity)
+        @records << record
+        @record_bytes << bytes
+        @bytes += bytes
+        @records.size
+      end
+    end
+
+    def size
+      @mutex.synchronize { @records.size }
+    end
+
+    def dropped
+      @mutex.synchronize { @dropped }
+    end
+
+    def bytes
+      @mutex.synchronize { @bytes }
+    end
+
+    def stats
+      @mutex.synchronize { [ @records.size, @dropped, @bytes, @dropped_bytes ] }
+    end
+
+    def account_dropped(count, bytes: 0)
+      @mutex.synchronize do
+        @dropped += count
+        @dropped_bytes += bytes
+      end
+    end
+
+    def full?(threshold)
+      size >= threshold
+    end
+
+    # Atomically take everything, resetting the drop counters. Returns the
+    # records with the weights measured when they were pushed, so the reporter
+    # can split a batch by bytes without weighing anything twice.
+    def drain
+      @mutex.synchronize do
+        drained = [ @records, @dropped, @dropped_bytes, @record_bytes ]
+        @records = []
+        @record_bytes = []
+        @bytes = 0
+        @dropped = 0
+        @dropped_bytes = 0
+        drained
+      end
+    end
+
+    # Put an unsuccessfully delivered batch back ahead of records written
+    # while it was in flight. The queue stays bounded: if both generations no
+    # longer fit, the oldest restored records are discarded first so fresh
+    # application telemetry wins under sustained ingest failure.
+    def restore(records, sizes = nil, dropped: 0, dropped_bytes: 0)
+      sizes ||= records.map { |record| Record.buffered_bytes(record, limit: @byte_capacity) }
+      @mutex.synchronize do
+        @records = records + @records
+        @record_bytes = sizes + @record_bytes
+        @bytes += sizes.sum
+        @dropped += dropped
+        @dropped_bytes += dropped_bytes
+        drop_oldest while @records.any? && (@records.size > @capacity || @bytes > @byte_capacity)
+        @records.size
+      end
+    end
+
+    private
+
+    def drop(bytes)
+      @dropped += 1
+      @dropped_bytes += bytes
+    end
+
+    def drop_oldest
+      @records.shift
+      bytes = @record_bytes.shift
+      @bytes -= bytes
+      drop(bytes)
+    end
+  end
+end

data/lib/railwatch/clock.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Railwatch
+  # Wall time for timestamps, monotonic time for durations. Durations are
+  # integers in microseconds everywhere, matching Nightwatch.
+  module Clock
+    module_function
+
+    def now
+      Process.clock_gettime(Process::CLOCK_REALTIME)
+    end
+
+    def monotonic
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+
+    def micros_since(monotonic_start)
+      ((monotonic - monotonic_start) * 1_000_000).round
+    end
+
+    def ms_to_micros(ms)
+      (ms.to_f * 1_000).round
+    end
+  end
+end