railwatch 0.1.0

data/lib/railwatch/subscribers/users.rb

@@ -0,0 +1,159 @@
+# frozen_string_literal: true
+
+module Railwatch
+  module Subscribers
+    # Resolves the current user. Default order: the app's Railwatch.user block,
+    # then Current.user (authentication-zero, Rails 8 auth generator), then
+    # Warden (Devise). Emits a `user` record once per user per process hour so
+    # the platform can show names without every record carrying them.
+    module Users
+      extend Base
+
+      module_function
+
+      def install!(_app)
+        subscribe("start_processing.action_controller") do |event|
+          exe = execution or next
+          exe.user_id = resolve_id(event.payload[:request]&.env)
+        end
+      end
+
+      def resolve_id(env = nil)
+        user = resolve_object(env) or return nil
+        details = describe(user) or return nil
+        remember(details)
+        details[:id]
+      rescue StandardError
+        nil
+      end
+
+      def resolve_from_current
+        resolve_id(nil)
+      end
+
+      # The beacon's resolution: the app's beacon_user block first, then the
+      # same Current.user / Warden lookup a request gets.
+      def resolve_beacon_id(request)
+        if (resolver = Railwatch.config.beacon_user_resolver)
+          user = resolver.call(request)
+          if user
+            details = describe(user) or return nil
+            remember(details)
+            return details[:id]
+          end
+        end
+        resolve_id(request.env)
+      rescue StandardError
+        nil
+      end
+
+      def resolve_object(env)
+        if defined?(::Current) && ::Current.respond_to?(:user) && ::Current.user
+          ::Current.user
+        elsif env && env["warden"].respond_to?(:user) && env["warden"].user
+          env["warden"].user
+        end
+      end
+
+      def describe(user)
+        details = if (resolver = Railwatch.config.user_resolver)
+          resolver.call(user)
+        else
+          {
+            id: user.respond_to?(:id) ? user.id : user.to_s,
+            name: user.respond_to?(:name) ? user.name : nil,
+            email: user.respond_to?(:email) ? user.email : nil
+          }
+        end
+        return nil unless details.is_a?(Hash) && details[:id]
+        details = details.transform_values { |v| v&.to_s&.[](0, 255) }
+        # Binding a known tenant onto the execution now (rather than leaving
+        # it to Execution#envelope's lazy bind) is what makes the reference
+        # below final, which is what lets `remember` trust its cache.
+        if (tenant = Context.current_tenant)
+          exe = execution
+          exe.tenant = tenant if exe && exe.tenant.nil?
+          details[:id] = Execution.qualified_user(details[:id], tenant)
+        end
+        details
+      end
+
+      # One `user` entity per id per process-hour. The cache entry is written
+      # only once the entity has actually shipped, which is why the record is
+      # parked on the execution (Execution#pending_users) and committed from
+      # finish_execution instead of here: a first sighting inside a
+      # sampled-out or paused execution writes no record, and must not
+      # suppress the next sighting that would.
+      #
+      # @seen is a plain unsynchronized Hash, as it was before: in CRuby a
+      # Hash store runs to completion under the GVL, so concurrent web
+      # threads cannot corrupt it, and the worst a lost race costs is one
+      # duplicate `user` record -- which the platform upserts by id.
+      def remember(details)
+        @seen ||= {}
+        key = details[:id]
+        now = Clock.now
+        return if recently_seen?(key, now)
+
+        exe = execution
+        pending = exe&.pending_users
+        return if pending&.key?(key)
+
+        record = Railwatch.record(:user, id: key, name: details[:name], email: details[:email],
+                                tenant: Context.current_tenant)
+        return unless record
+
+        exe ? (exe.pending_users ||= {})[key] = record : mark_seen(key, now)
+        record
+      end
+
+      # Called from Railwatch.finish_execution, for a tree that is being handed
+      # to the reporter, just before its records are written.
+      def commit_execution!(exe)
+        pending = exe.pending_users
+        exe.pending_users = nil
+        now = Clock.now
+        pending.each do |key, record|
+          # An over-full execution buffer drops the record it was handed, and
+          # a failure-context ring can later shift it back out; either way the
+          # entity never shipped. Identity, not `==`: two `user` records for
+          # the same person are equal hashes.
+          index = exe.records.index { |buffered| buffered.equal?(record) } or next
+
+          # A tenant bound after the entity was resolved changes its
+          # reference ("1" becomes "acme:1"), so this -- not the provisional
+          # key `remember` checked -- is what the cache is keyed on. Such an
+          # app therefore rebuilds the entity each request and discards the
+          # duplicate here; that is one small hash, and the alternative
+          # (trusting the provisional key) is what suppressed a second
+          # tenant's user 1 entirely.
+          reference = Execution.qualified_user(key, exe.tenant)
+          if recently_seen?(reference, now)
+            exe.records.delete_at(index)
+          else
+            record[:id] = reference
+            record[:tenant] = exe.tenant if record[:tenant].nil?
+            mark_seen(reference, now)
+          end
+        end
+      end
+
+      # A fork inherits this cache but not the reporter buffer the cached
+      # entities were written to, so the child has to emit its own.
+      def restart_after_fork!
+        @seen = {}
+        execution&.pending_users = nil
+      end
+
+      def recently_seen?(key, now)
+        last = @seen[key]
+        last && now - last < 3600
+      end
+
+      def mark_seen(key, now)
+        @seen[key] = now
+        @seen.delete(@seen.keys.first) if @seen.size > 10_000
+      end
+    end
+  end
+end

data/lib/railwatch/subscribers/views.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Railwatch
+  module Subscribers
+    # Template, partial, and collection renders. Only the first N per
+    # execution are stored (config.max_view_renders_per_execution); all are counted.
+    module Views
+      extend Base
+
+      # A process renders a small, fixed set of templates, so the identifier
+      # -> group hash is computed once per template rather than per render.
+      # Frozen because the same string goes out on every record as _group.
+      GROUP_CACHE_LIMIT = 2_048
+      @group_cache = {}
+      @group_mutex = Mutex.new
+
+      module_function
+
+      def group_for(identifier)
+        cached = @group_cache[identifier]
+        return cached if cached
+
+        group = Record.group_hash(identifier).freeze
+        @group_mutex.synchronize do
+          @group_cache.clear if @group_cache.size >= GROUP_CACHE_LIMIT
+          @group_cache[identifier] = group
+        end
+        group
+      end
+
+      def install!(_app)
+        %w[render_template render_partial render_collection render_layout].each do |kind|
+          subscribe("#{kind}.action_view") do |event|
+            exe = execution
+            exe&.count(:view_renders)
+            next unless recording?
+            next if exe && exe.counters[:view_renders] > Railwatch.config.max_view_renders_per_execution
+            p = event.payload
+            identifier = p[:identifier].to_s.delete_prefix(Backtrace.app_root)
+            Railwatch.record(:view_render,
+              group: group_for(identifier),
+              timestamp: started_at(event),
+              identifier: identifier[0, 255],
+              kind: kind.delete_prefix("render_"),
+              layout: p[:layout]&.to_s,
+              count: p[:count],
+              cache_hits: p[:cache_hits],
+              duration: micros(event))
+          end
+        end
+      end
+    end
+  end
+end

data/lib/railwatch/subscribers.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require "railwatch/subscribers/base"
+require "railwatch/subscribers/requests"
+require "railwatch/subscribers/queries"
+require "railwatch/subscribers/exceptions"
+require "railwatch/subscribers/cache"
+require "railwatch/subscribers/mail"
+require "railwatch/subscribers/broadcasts"
+require "railwatch/subscribers/notifications"
+require "railwatch/subscribers/storage"
+require "railwatch/subscribers/views"
+require "railwatch/subscribers/logs"
+require "railwatch/subscribers/jobs"
+require "railwatch/subscribers/deprecations"
+require "railwatch/subscribers/users"
+require "railwatch/subscribers/process_info"
+
+module Railwatch
+  module Subscribers
+    ALL = [ Requests, Queries, Exceptions, Cache, Mail, Broadcasts, Notifications,
+            Storage, Views, Logs, Jobs, Deprecations, Users, ProcessInfo ].freeze
+
+    module_function
+
+    def install!(app = nil)
+      ALL.each do |subscriber|
+        subscriber.install!(app)
+      rescue StandardError => e
+        Railwatch.debug { "failed to install #{subscriber.name}: #{e.class}: #{e.message}" }
+      end
+    end
+  end
+end

data/lib/railwatch/transport/http.rb

@@ -0,0 +1,208 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "openssl"
+require "zlib"
+require "json"
+
+module Railwatch
+  module Transport
+    # POSTs gzip NDJSON batches to the platform. Each call retries one raised
+    # error or 5xx response, then returns a classified, non-raising result;
+    # Reporter owns retention and backoff between calls. A 401 marks the
+    # transport unauthorized so no further requests are made.
+    class Http
+      RETRYABLE_STATUSES = [ 402, 408, 429 ].freeze
+      UNAUTHORIZED_STATUS = 401
+
+      Result = Struct.new(:ok, :status, :accepted, :rejected, :rejections, :error, :retryable_error, keyword_init: true) do
+        def retryable?
+          !ok && (retryable_error || Http.retryable_status?(status))
+        end
+      end
+
+      def self.retryable_status?(status)
+        status.nil? || RETRYABLE_STATUSES.include?(status) || (500..599).cover?(status)
+      end
+
+      def initialize(config)
+        @config = config
+        @uri = URI.join(config.ingest_url, "/ingest")
+        @unauthorized = false
+      end
+
+      def unauthorized?
+        @unauthorized
+      end
+
+      # The object itself is copied into a forked process, but its policy
+      # state belongs to the parent that observed those responses.
+      def reset_after_fork!
+        @unauthorized = false
+        self
+      end
+
+      def deliver(records, dropped: 0, dropped_bytes: 0, backpressure_factor: 1.0, batch_id: SecureRandom.uuid)
+        unless @config.ingest_url_allowed?
+          return Result.new(ok: false, error: "plain HTTP ingest is disabled; use HTTPS or set RAILWATCH_ALLOW_HTTP=true")
+        end
+        return Result.new(ok: false, status: UNAUTHORIZED_STATUS, error: "unauthorized, flushing stopped") if @unauthorized
+
+        body, sent, over_cap, over_cap_bytes = encode(records)
+        if over_cap.positive?
+          # Not a delivery failure: a batch this large will be exactly as
+          # large on every retry, so raising a retryable error here would burn
+          # the whole ladder and drop the records at the end anyway. Drop them
+          # now, and count them onto this batch so the loss is visible.
+          Railwatch.debug { "dropped #{over_cap} records that did not fit in batch_bytes (#{@config.batch_bytes})" }
+          dropped += over_cap
+          dropped_bytes += over_cap_bytes
+        end
+        attempt = 0
+        begin
+          attempt += 1
+          result = parse(post(body, dropped, dropped_bytes, backpressure_factor, batch_id), expected_count: sent)
+          if attempt < 2 && (500..599).cover?(result.status)
+            result = parse(post(body, dropped, dropped_bytes, backpressure_factor, batch_id), expected_count: sent)
+          end
+          apply_status_policy(result)
+          result
+        rescue StandardError => e
+          retry if attempt < 2
+          Result.new(ok: false, error: "#{e.class}: #{e.message}")
+        end
+      end
+
+      def ping
+        return false unless @config.ingest_url_allowed?
+
+        response = request(Net::HTTP::Get.new(URI.join(@config.ingest_url, "/ingest/ping")))
+        response.is_a?(Net::HTTPSuccess)
+      rescue StandardError
+        false
+      end
+
+      private
+
+      # The one serialization of the batch, so it is also where its exact
+      # uncompressed size is known. Records past config.batch_bytes are left
+      # out and reported back to the caller rather than growing the request
+      # without limit. Returns [body, records written, records left out,
+      # bytes left out].
+      def encode(records)
+        io = StringIO.new
+        gz = Zlib::GzipWriter.new(io)
+        bytes = 0
+        sent = 0
+        over_cap = 0
+        over_cap_bytes = 0
+        records.each do |record|
+          json = JSON.generate(record)
+          size = json.bytesize + 1
+          if bytes + size > @config.batch_bytes
+            over_cap += 1
+            over_cap_bytes += size
+            next
+          end
+          gz.write(json)
+          gz.write("\n")
+          bytes += size
+          sent += 1
+        end
+        gz.close
+        [ io.string, sent, over_cap, over_cap_bytes ]
+      end
+
+      def post(body, dropped, dropped_bytes, backpressure_factor, batch_id)
+        req = Net::HTTP::Post.new(@uri)
+        req["Content-Type"] = "application/x-ndjson"
+        req["Content-Encoding"] = "gzip"
+        req["X-Railwatch-Dropped"] = dropped.to_s if dropped.positive?
+        req["X-Railwatch-Dropped-Bytes"] = dropped_bytes.to_s if dropped_bytes.positive?
+        if backpressure_factor > 1.0
+          req["X-Railwatch-Backpressure-Factor"] = backpressure_factor.to_s
+        end
+        req["X-Railwatch-Version"] = Railwatch::VERSION
+        req["X-Railwatch-Batch-Id"] = batch_id
+        req.body = body
+        request(req)
+      end
+
+      def request(req)
+        req["Authorization"] = "Bearer #{@config.token}"
+        req["User-Agent"] = "railwatch-ruby/#{Railwatch::VERSION}"
+        options = {
+          use_ssl: @uri.scheme == "https",
+          open_timeout: @config.connect_timeout,
+          read_timeout: @config.timeout,
+          write_timeout: @config.timeout
+        }
+        # Net::HTTP currently defaults HTTPS clients to VERIFY_PEER. Set it
+        # explicitly so a Ruby default change cannot silently weaken ingest.
+        options[:verify_mode] = OpenSSL::SSL::VERIFY_PEER if options[:use_ssl]
+        Net::HTTP.start(@uri.host, @uri.port, **options) do |http|
+          http.request(req)
+        end
+      end
+
+      def parse(response, expected_count:)
+        if response.is_a?(Net::HTTPSuccess)
+          parse_acknowledgement(response, expected_count)
+        else
+          Result.new(ok: false, status: response.code.to_i, error: response.body.to_s[0, 200])
+        end
+      end
+
+      def parse_acknowledgement(response, expected_count)
+        data = JSON.parse(response.body)
+        return invalid_acknowledgement(response, "response must be a JSON object") unless data.is_a?(Hash)
+
+        accepted = data["accepted"]
+        rejected = data["rejected"]
+        unless accepted.is_a?(Integer) && accepted >= 0 && rejected.is_a?(Integer) && rejected >= 0
+          return invalid_acknowledgement(response, "accepted and rejected must be non-negative integers")
+        end
+        unless drained?(data, accepted, rejected) || accepted + rejected == expected_count
+          return invalid_acknowledgement(response,
+                                         "accepted + rejected was #{accepted + rejected}, expected #{expected_count}")
+        end
+
+        rejections = data["rejections"]
+        unless rejections.nil? || rejections.is_a?(Array)
+          return invalid_acknowledgement(response, "rejections must be an array when present")
+        end
+
+        Result.new(ok: true, status: response.code.to_i, accepted: accepted, rejected: rejected,
+                   rejections: Array(rejections).first(10))
+      rescue JSON::ParserError => error
+        invalid_acknowledgement(response, "invalid JSON (#{error.message})")
+      end
+
+      # Ingest can take a whole batch off our hands without storing any of it:
+      # a paused or over-quota environment answers 200 with
+      # {"accepted":0,"rejected":0,"reason":"paused"}. That batch IS delivered
+      # -- the platform decided its fate -- so retrying it would burn eight
+      # attempts and drop the records anyway. Any acknowledgement carrying a
+      # `reason`, and any all-zero acknowledgement, drains the batch.
+      def drained?(data, accepted, rejected)
+        data.key?("reason") || (accepted.zero? && rejected.zero?)
+      end
+
+      # A proxy-generated 2xx page or a contract mismatch cannot acknowledge
+      # the submitted records. Keep the batch for Reporter retry instead of
+      # silently treating it as delivered.
+      def invalid_acknowledgement(response, detail)
+        Result.new(ok: false, status: response.code.to_i, error: "invalid ingest acknowledgement: #{detail}",
+                   retryable_error: true)
+      end
+
+      def apply_status_policy(result)
+        case result.status
+        when UNAUTHORIZED_STATUS
+          @unauthorized = true
+          Railwatch.debug { "ingest returned 401 -- marking transport unauthorized, no further flushes will be attempted" }
+        end
+      end
+    end
+  end
+end

data/lib/railwatch/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Railwatch
+  VERSION = "0.1.0"
+end