fractal-ruby 0.1.0

data/lib/fractal/breaker.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module Fractal
+  # Delivery health shared by the error reporter and analytics: one circuit breaker, because both
+  # talk to the same collector with the same key, and one revocation flag, because a 401 means the
+  # key itself is bad.
+  #
+  # Breaker: FAILURE_THRESHOLD consecutive failed POSTs (from either half) open it for COOLDOWN
+  # seconds, then one probe is let through. A failed probe leaves the count at the threshold, so it
+  # reopens at once; any success resets it.
+  #
+  # Revocation is for the life of the process (ANALYTICS_PROTOCOL.md, "401"): a revoked key is not
+  # retried once a second forever. It survives a fork, since the child holds the same key.
+  class Breaker
+    FAILURE_THRESHOLD = 5
+    COOLDOWN = 60
+
+    def initialize(clock: nil)
+      @clock = clock || -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }
+      @revoked = false
+      reset
+    end
+
+    def open?
+      reset_after_fork
+      @mutex.synchronize { !@opened_at.nil? && now - @opened_at < COOLDOWN }
+    end
+
+    def revoked?
+      @revoked
+    end
+
+    def success!
+      reset_after_fork
+      @mutex.synchronize do
+        @consecutive_failures = 0
+        @opened_at = nil
+      end
+    end
+
+    def failure!
+      reset_after_fork
+      @mutex.synchronize do
+        @consecutive_failures += 1
+        @opened_at = now if @consecutive_failures >= FAILURE_THRESHOLD
+      end
+    end
+
+    def revoke!
+      @revoked = true
+    end
+
+    private
+
+    def now = @clock.call
+
+    def reset
+      @pid = Process.pid
+      @mutex = Mutex.new
+      @consecutive_failures = 0
+      @opened_at = nil
+    end
+
+    # A mutex inherited across fork may be held by a thread that does not exist in the child.
+    def reset_after_fork
+      reset unless @pid == Process.pid
+    end
+  end
+end

data/lib/fractal/client.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+module Fractal
+  # One configured instance: the two subsystems, sharing a transport and a breaker. A subsystem
+  # turned off in the configuration (or with no API key at all) is a NullErrors / NullAnalytics,
+  # whose methods do nothing, so calling code never checks for nil.
+  class Client
+    attr_reader :config, :errors, :analytics, :breaker
+
+    def initialize(config, transport: Transport.new, breaker: Breaker.new, start_workers: true)
+      @config = config
+      @breaker = breaker
+      on = config.api_key?
+      reporter = ErrorReporter.new(config: config, transport: transport, breaker: breaker, start_worker: start_workers)
+      @errors = on && config.errors_enabled ? Errors.new(reporter) : NullErrors.new
+      @analytics = on && config.analytics_enabled ? Analytics.new(config: config, transport: transport, breaker: breaker, start_worker: start_workers) : NullAnalytics.new
+    end
+
+    def flush
+      @errors.flush
+      @analytics.flush
+    end
+
+    def shutdown
+      @errors.shutdown
+      @analytics.shutdown
+    end
+
+    # at_exit runs in every forked child too; one registration covers all of them, and it always
+    # shuts down whichever client is active at exit.
+    def register_at_exit
+      return if Client.instance_variable_get(:@at_exit_registered)
+
+      Client.instance_variable_set(:@at_exit_registered, true)
+      at_exit { Fractal.shutdown }
+    end
+  end
+
+  # Fractal.errors when error reporting is on.
+  class Errors
+    attr_reader :reporter
+
+    def initialize(reporter)
+      @reporter = reporter
+    end
+
+    # Reports an exception you rescued. Never raises.
+    def capture_error(exception, handled: true, severity: :error, context: {})
+      @reporter.report(exception, handled: handled, severity: severity, source: "fractal.capture_error",
+                                  context: ExecutionContext.current.merge(context))
+    end
+
+    # Reports a message with no exception; the stack is where this was called from.
+    def capture_message(message, severity: :info, context: {})
+      @reporter.report_message(message, severity: severity, frames: caller(1),
+                                        context: ExecutionContext.current.merge(context))
+    end
+
+    # Rails.error subscriber entry point.
+    def report(exception, handled:, severity:, source:, context: {})
+      @reporter.report(exception, handled: handled, severity: severity, source: source, context: context)
+    end
+
+    def enabled? = @reporter.enabled?
+    def flush(...) = @reporter.flush(...)
+    def shutdown(...) = @reporter.shutdown(...)
+    def stats = @reporter.stats
+  end
+
+  class NullErrors
+    def capture_error(*, **) = nil
+    def capture_message(*, **) = nil
+    def report(*, **) = nil
+    def enabled? = false
+    def flush(...) = nil
+    def shutdown(...) = nil
+    def stats = {}
+  end
+
+  class NullAnalytics
+    def track(*, **) = false
+    def identify(*, **) = false
+    def enabled? = false
+    def flush(...) = nil
+    def shutdown(...) = nil
+    def stats = {}
+    def pending = 0
+  end
+
+  # Before Fractal.configure: everything is a no-op.
+  class NullClient
+    attr_reader :config, :errors, :analytics
+
+    def initialize
+      @config = Configuration.new
+      @errors = NullErrors.new
+      @analytics = NullAnalytics.new
+    end
+
+    def flush = nil
+    def shutdown = nil
+    def register_at_exit = nil
+  end
+
+  NULL_CLIENT = NullClient.new
+end

data/lib/fractal/configuration.rb

@@ -0,0 +1,164 @@
+# frozen_string_literal: true
+
+module Fractal
+  # Everything an app can set in `Fractal.configure`. Values that default to something Rails knows
+  # (filter_parameters, root, environment, logger) are read lazily, on first use, so an initializer
+  # that runs before Rails has finished booting still gets them.
+  class Configuration
+    DEFAULT_COLLECTOR_URL = "https://collector.getfractal.dev"
+
+    # Rails' own default config/initializers/filter_parameter_logging.rb list, used when the app
+    # is not a Rails app or Rails has no filter_parameters configured.
+    DEFAULT_FILTER_PARAMETERS = %w[passw email secret token _key crypt salt certificate otp ssn cvv cvc].freeze
+
+    # The API key of a Fractal project. Use a key created for this backend, not the one the game
+    # client ships with. Without one, the gem does nothing.
+    attr_accessor :api_key
+    attr_accessor :collector_url
+    # The app's own version. Fractal resolves and reopens error groups by comparing versions, so
+    # an empty value is accepted with a warning but makes both unreliable.
+    attr_accessor :app_version
+    attr_writer :environment
+
+    # true, false or a callable checked on every call, so a kill switch read from ENV takes
+    # effect without a restart. It is process-wide and also read on the sending threads, where
+    # there is no request: turning it off discards what is buffered, so it is no place for a
+    # per-user check such as consent (that belongs at the track call site).
+    attr_accessor :enabled
+    attr_accessor :errors_enabled, :analytics_enabled
+
+    # ->(controller) { "user-42" or nil }: the player id for the signed-in user of a request.
+    # Consulted when nothing called Fractal.set_user (or the Warden hook) in this request.
+    attr_accessor :user_resolver
+    # ->(user) { id string }: how a Warden/Devise user becomes a player id.
+    attr_accessor :player_id_for_user
+    # Player ids for traffic with no user: a request nobody is signed in for, and code outside
+    # any request (jobs, rake tasks, boot).
+    attr_accessor :anonymous_player_id, :system_player_id
+    # ->(rack_env) { token or nil }: the session an analytics event belongs to.
+    attr_accessor :session_resolver
+    # Send each analytics event's end-user address so the collector can geolocate it. The
+    # collector only uses it when the project has IP tracking on.
+    attr_accessor :send_client_ip
+    # Track an `identify` event when Warden signs a user in, linking the pre-login session to
+    # the user.
+    attr_accessor :identify_on_login
+
+    attr_writer :filter_parameters, :project_root, :logger
+    # Extra keys to mask, on top of filter_parameters and the gem's own list.
+    attr_accessor :extra_filter_keys
+    # Callables ->(exception, raw_message) { replacement or nil }, tried in order before the
+    # gem's built-in database rules. The first non-nil answer is used (then scrubbed as usual).
+    attr_accessor :message_filters
+    # ->(event_hash) { event_hash or nil }: a last look at each error event; nil drops it.
+    attr_accessor :before_send_error
+
+    # Raise ArgumentError from `track` on an event the collector would reject, instead of
+    # logging and dropping it. Defaults to on outside production.
+    attr_writer :strict
+
+    attr_accessor :errors_flush_interval, :analytics_flush_interval, :analytics_flush_at
+    attr_accessor :analytics_max_queue, :analytics_max_queue_bytes, :shutdown_timeout
+
+    def initialize
+      @collector_url = DEFAULT_COLLECTOR_URL
+      @app_version = ""
+      @enabled = true
+      @errors_enabled = true
+      @analytics_enabled = true
+      @anonymous_player_id = "anonymous"
+      @system_player_id = "system"
+      @player_id_for_user = ->(user) { user.respond_to?(:id) ? user.id.to_s : user.to_s }
+      @session_resolver = SessionResolver.method(:from_rack_session)
+      @send_client_ip = true
+      @identify_on_login = true
+      @extra_filter_keys = []
+      @message_filters = []
+      @errors_flush_interval = 1
+      @analytics_flush_interval = 10
+      @analytics_flush_at = 100
+      @analytics_max_queue = 5_000
+      @analytics_max_queue_bytes = 8 * 1024 * 1024
+      @shutdown_timeout = 2
+    end
+
+    # Fails loudly on a configuration that could never deliver anything, rather than letting the
+    # app run believing it reports errors.
+    def validate!
+      return unless api_key?
+
+      raise ArgumentError, "Fractal: collector_url is required when api_key is set" if blank?(collector_url)
+      raise ArgumentError, "Fractal: collector_url must be an http(s) URL, got #{collector_url.inspect}" unless collector_url.to_s.match?(%r{\Ahttps?://}i)
+
+      %i[user_resolver session_resolver player_id_for_user before_send_error].each do |name|
+        value = public_send(name)
+        raise ArgumentError, "Fractal: #{name} must respond to #call" if value && !value.respond_to?(:call)
+      end
+      raise ArgumentError, "Fractal: message_filters must be callables" unless Array(message_filters).all? { |f| f.respond_to?(:call) }
+      if blank?(anonymous_player_id) || blank?(system_player_id)
+        raise ArgumentError, "Fractal: anonymous_player_id and system_player_id must be non-empty"
+      end
+      return unless blank?(app_version)
+
+      logger.warn("[fractal] app_version is empty: error groups cannot be resolved or reopened by version")
+    end
+
+    def api_key? = !blank?(api_key)
+
+    # Checked on every call: a callable `enabled` is a runtime kill switch.
+    def enabled?
+      return false unless api_key?
+
+      enabled.respond_to?(:call) ? !!enabled.call : !!enabled
+    end
+
+    def environment
+      @environment ||= if defined?(::Rails) && ::Rails.respond_to?(:env)
+        ::Rails.env.to_s
+      else
+        ENV.fetch("RACK_ENV") { ENV.fetch("RAILS_ENV", "") }
+      end
+    end
+
+    def strict?
+      return @strict unless @strict.nil?
+
+      !%w[production staging].include?(environment)
+    end
+
+    def filter_parameters
+      return @filter_parameters if @filter_parameters
+      return ::Rails.application.config.filter_parameters if rails_app? && ::Rails.application.config.filter_parameters.any?
+
+      DEFAULT_FILTER_PARAMETERS
+    end
+
+    def project_root
+      @project_root ||= rails_app? ? ::Rails.root.to_s : Dir.pwd
+    end
+
+    def logger
+      @logger ||= if defined?(::Rails) && ::Rails.respond_to?(:logger) && ::Rails.logger
+        ::Rails.logger
+      else
+        require "logger"
+        Logger.new($stderr, progname: "fractal")
+      end
+    end
+
+    def endpoint(path) = "#{collector_url.to_s.chomp('/')}#{path}"
+
+    # Built on first use, not at configure time: it reads Rails' filter_parameters and ENV, which
+    # are only complete once the app has booted.
+    def scrubber
+      @scrubber ||= Scrubber.new(filter_parameters: filter_parameters, root: project_root,
+                                 extra_keys: extra_filter_keys, message_filters: message_filters)
+    end
+
+    private
+
+    def rails_app? = defined?(::Rails) && ::Rails.respond_to?(:application) && ::Rails.application
+
+    def blank?(value) = value.nil? || value.to_s.strip.empty?
+  end
+end

data/lib/fractal/error_reporter.rb

@@ -0,0 +1,338 @@
+# frozen_string_literal: true
+
+require "etc"
+require "json"
+
+module Fractal
+  # Buffers scrubbed error events in memory and delivers them to POST /v1/errors from one
+  # background thread. Nothing here may ever slow down or break the request or job that raised:
+  # every public method rescues everything, and delivery is bounded in memory, time and rate.
+  #
+  # Deliberately not a job queue: those are usually database-backed, and the database is
+  # frequently the very thing that is broken when the report matters most.
+  #
+  # The buffer is a coalescing map, not a queue. Repeats of one error (same collector fingerprint,
+  # same player) increment occurrence_count instead of taking a slot, which is the storm control:
+  # a database outage that fails every request sends one event per distinct error per second.
+  class ErrorReporter
+    MAX_ENTRIES = 100            # distinct errors buffered; the collector caps a request at 100
+    MAX_POSTS_PER_FLUSH = 5      # collector limit is 10 req/s per key, burst 30
+
+    Entry = Struct.new(:event, :player_id, :fingerprint, :count, :first_at)
+
+    def initialize(config:, transport: Transport.new, breaker: Breaker.new, builder: nil,
+                   flush_interval: config.errors_flush_interval, start_worker: true, clock: nil)
+      @config = config
+      @transport = transport
+      @breaker = breaker
+      @builder = builder
+      @clock = clock || -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }
+      @flush_interval = flush_interval
+      @start_worker = start_worker
+      reset_state
+    end
+
+    def enabled?
+      @config.errors_enabled && @config.enabled? && !@breaker.revoked? && !@disabled
+    end
+
+    # Never raises. Called from Rails.error subscribers, i.e. from inside a failing request or job.
+    def report(exception, handled:, severity:, source:, context: {})
+      reset_after_fork
+      return unless enabled?
+      return if seen?(exception)
+
+      built = build { builder.build(exception, handled: handled, severity: severity, source: source, context: context) }
+      enqueue(built)
+    rescue Exception => e # rubocop:disable Lint/RescueException
+      log(:error, "report failed: #{e.class}")
+    end
+
+    # Never raises.
+    def report_message(message, severity:, frames:, context: {})
+      reset_after_fork
+      return unless enabled?
+
+      built = build { builder.build_message(message, severity: severity, frames: frames, context: context) }
+      enqueue(built)
+    rescue Exception => e # rubocop:disable Lint/RescueException
+      log(:error, "report failed: #{e.class}")
+    end
+
+    # Delivers up to MAX_POSTS_PER_FLUSH batches (one per player: /v1/errors has no per-event
+    # player). Never raises.
+    def flush(deadline: nil)
+      reset_after_fork
+      unless enabled?
+        discard_buffer
+        return
+      end
+      return if @breaker.open?
+
+      batches = take_batches
+      batches.each_with_index do |(player_id, entries), index|
+        if @breaker.open? || !enabled? || (deadline && now > deadline)
+          batches.drop(index).each { |_, rest| restore(rest) }
+          break
+        end
+        deliver(player_id, entries)
+      end
+    rescue Exception => e # rubocop:disable Lint/RescueException
+      log(:error, "flush failed: #{e.class}")
+    end
+
+    # Stops the worker and delivers what is left, within the deadline. Never raises.
+    def shutdown(timeout: @config.shutdown_timeout)
+      return unless @pid == Process.pid
+
+      @stopping = true
+      @mutex.synchronize { @wake.broadcast }
+      @thread&.join(0.5)
+      flush(deadline: now + timeout)
+    rescue Exception => e # rubocop:disable Lint/RescueException
+      log(:error, "shutdown failed: #{e.class}")
+    end
+
+    def stats
+      @mutex.synchronize { @stats.dup }
+    end
+
+    private
+
+    def reset_state
+      @pid = Process.pid
+      @mutex = Mutex.new
+      @wake = ConditionVariable.new
+      @buffer = {}
+      @stats = Hash.new(0)
+      @seen = ObjectSpace::WeakMap.new
+      @thread = nil
+      @stopping = false
+      @disabled = false
+    end
+
+    # A forked child (Puma worker, Solid Queue process) inherits the parent's buffer and a thread
+    # that no longer exists. Sending that buffer again would duplicate every event, so start clean.
+    def reset_after_fork
+      reset_state unless @pid == Process.pid
+    end
+
+    def builder
+      @builder ||= EventBuilder.new(config: @config, scrubber: @config.scrubber)
+    end
+
+    def now = @clock.call
+
+    # One exception object is reported at most once. Rails marks it with an ivar, but that cannot
+    # work on a frozen exception, and does not cover a wrapper whose cause was reported first.
+    def seen?(exception)
+      chain = [ exception ]
+      chain << chain.last.cause while chain.last.cause && chain.size < 10
+      @mutex.synchronize do
+        return true if chain.any? { |e| @seen.key?(e) }
+
+        chain.each { |e| @seen[e] = true }
+      end
+      false
+    end
+
+    # Fail closed: if the event cannot be built and scrubbed, it is dropped, never sent raw. The
+    # app's before_send_error hook gets the last look; nil drops the event.
+    def build
+      built = yield
+      hook = @config.before_send_error
+      return built unless hook
+
+      event = hook.call(built.event)
+      if event.nil?
+        bump(:dropped_by_hook)
+        return nil
+      end
+      built.with(event: event)
+    rescue Exception => e # rubocop:disable Lint/RescueException
+      bump(:dropped_scrub)
+      log(:error, "event dropped, could not be built: #{e.class}")
+      nil
+    end
+
+    def enqueue(built)
+      return unless built
+
+      add(built)
+      start_worker if @start_worker
+    end
+
+    def add(built)
+      key = [ built.player_id, built.fingerprint ]
+      @mutex.synchronize do
+        if (entry = @buffer[key])
+          entry.count += 1
+        elsif @buffer.size >= MAX_ENTRIES
+          @stats[:dropped_buffer_full] += 1
+        else
+          @buffer[key] = Entry.new(built.event, built.player_id, built.fingerprint, 1, Time.now.utc)
+        end
+      end
+    end
+
+    def take_batches
+      @mutex.synchronize do
+        batches = @buffer.values.group_by(&:player_id).first(MAX_POSTS_PER_FLUSH)
+        batches.each { |_, entries| entries.each { |entry| @buffer.delete(key_for(entry)) } }
+        batches
+      end
+    end
+
+    def key_for(entry) = [ entry.player_id, entry.fingerprint ]
+
+    # Puts undelivered entries back, merging with any repeats that arrived in the meantime.
+    def restore(entries)
+      @mutex.synchronize do
+        entries.each do |entry|
+          key = key_for(entry)
+          if (existing = @buffer[key])
+            existing.count += entry.count
+            existing.first_at = [ existing.first_at, entry.first_at ].min
+          elsif @buffer.size < MAX_ENTRIES
+            @buffer[key] = entry
+          else
+            @stats[:dropped_buffer_full] += entry.count
+          end
+        end
+      end
+    end
+
+    def deliver(player_id, entries)
+      body = serialise(player_id, entries)
+      return unless body
+
+      response = @transport.post(@config.endpoint("/v1/errors"), @config.api_key, body)
+      handle_response(response, entries)
+    rescue Exception => e # rubocop:disable Lint/RescueException
+      failed(entries, requeue: true)
+      log(:warn, "delivery failed: #{e.class}")
+    end
+
+    # Entries that cannot be serialised now never will be. They are dropped, not requeued: a
+    # requeue would fail on every flush and open the breaker analytics shares.
+    def serialise(player_id, entries)
+      payload(player_id, entries).to_json
+    rescue StandardError => e
+      bump(:dropped_invalid, entries.size)
+      log(:error, "batch of #{entries.size} error(s) dropped: cannot be serialised (#{e.class})")
+      nil
+    end
+
+    def handle_response(response, entries)
+      case response.code
+      when 200..299
+        succeeded(entries, response.body)
+      when 401
+        @breaker.revoke!
+        discard_buffer
+        log(:error, "collector answered 401: the API key is unknown or inactive. Fractal stops sending for the rest of this process")
+      when 402
+        @mutex.synchronize { @disabled = true }
+        discard_buffer
+        log(:warn, "collector answered 402: this plan has no error tracking. Error reporting is off for the rest of this process")
+      when 429, 500..599
+        failed(entries, requeue: true)
+        log(:warn, "collector answered #{response.code}; will retry")
+      else
+        failed(entries, requeue: false)
+        log(:error, "collector answered #{response.code}; batch of #{entries.size} error(s) dropped")
+      end
+    end
+
+    def succeeded(entries, body)
+      @breaker.success!
+      bump(:sent_events, entries.size)
+      rejected = Rejections.parse(body, entries.size)
+      return if rejected.empty?
+
+      bump(:rejected_events, rejected.size)
+      rejected.each do |r|
+        log(:error, "collector rejected error event #{r['index']} (#{entries[r['index']].event['error_type']}): #{r['reason']}")
+      end
+    end
+
+    def failed(entries, requeue:)
+      @breaker.failure!
+      bump(:failed_posts)
+      restore(entries) if requeue
+    end
+
+    def discard_buffer
+      @mutex.synchronize { @buffer.clear }
+    end
+
+    def payload(player_id, entries)
+      os = Etc.uname
+      {
+        sent_at: Time.now.utc.iso8601(3),
+        context: {
+          player_id: player_id, session_token: "", platform: "server", app_version: @config.app_version.to_s,
+          environment: @config.environment, runtime_version: "ruby #{RUBY_VERSION}", os_name: os[:sysname],
+          os_version: os[:release], sdk_version: Fractal.sdk_version
+        },
+        errors: entries.map do |entry|
+          entry.event.merge("occurrence_count" => entry.count, "timestamp" => entry.first_at.iso8601(3))
+        end
+      }
+    end
+
+    def start_worker
+      return if @thread&.alive?
+
+      @mutex.synchronize do
+        return if @thread&.alive?
+
+        @thread = Thread.new { work }
+        @thread.report_on_exception = false
+      end
+    end
+
+    def work
+      until @stopping
+        @mutex.synchronize { @wake.wait(@mutex, @flush_interval) }
+        flush unless @stopping
+      end
+    rescue Exception => e # rubocop:disable Lint/RescueException
+      log(:error, "worker died: #{e.class}")
+    end
+
+    def bump(stat, by = 1)
+      @mutex.synchronize { @stats[stat] += by }
+    end
+
+    # Class names only: an exception message may carry exactly what the scrubber exists to remove.
+    def log(level, message)
+      @config.logger&.public_send(level, "[fractal] #{message}")
+    end
+  end
+
+  # Reads rejected[] out of a 202 body, defensively (ANALYTICS_PROTOCOL.md, CRASH_PROTOCOL.md): an
+  # entry whose index is missing, not a whole number, out of range or a duplicate is ignored, and
+  # a malformed or empty body rejects nothing.
+  module Rejections
+    module_function
+
+    def parse(body, size)
+      json = JSON.parse(body.to_s)
+      list = json.is_a?(Hash) ? json["rejected"] : nil
+      return [] unless list.is_a?(Array)
+
+      seen = {}
+      list.select do |entry|
+        next false unless entry.is_a?(Hash)
+
+        index = entry["index"]
+        next false unless index.is_a?(Integer) && index >= 0 && index < size && !seen[index]
+
+        seen[index] = true
+      end
+    rescue JSON::ParserError
+      []
+    end
+  end
+end