flagnetic 0.1.0

data/lib/flagnetic/local_overrides.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Flagnetic
+  # LocalOverrides substitutes flag values inside this process, on a
+  # developer's machine or in a non-production build. A substituted value
+  # never reaches the server. An overridden flag reads reason
+  # local_override, with no experiment block and the stage unchanged.
+  #
+  # Local overrides exist only on a client constructed with
+  # allow_overrides: true, and are refused when the held ruleset belongs to
+  # a protected environment: set raises LocalOverrideRefused and logs
+  # override_refused, and a protected ruleset arriving later drops every
+  # override. Setting or removing one fires on_change.
+  #
+  #   client.local_overrides.set("new_checkout", true)
+  class LocalOverrides
+    def initialize(client)
+      @client = client
+    end
+
+    # set substitutes value for the flag key. The value must be of the
+    # flag's type: true or false, a String, a Numeric, or for a JSON flag
+    # anything JSON can encode. A key the ruleset lacks takes the type of
+    # the value. Raises LocalOverridesNotAllowed, LocalOverrideRefused, or
+    # ArgumentError for a value of the wrong type.
+    def set(key, value)
+      @client.__send__(:set_local_override, key.to_s, value)
+      nil
+    end
+
+    # remove drops the local override of key, firing on_change when there
+    # was one.
+    def remove(key)
+      @client.__send__(:remove_local_overrides, [key.to_s])
+      nil
+    end
+
+    # clear drops every local override, firing on_change with their keys.
+    def clear
+      @client.__send__(:remove_local_overrides, nil)
+      nil
+    end
+
+    # list returns the local overrides by flag key.
+    def list = @client.__send__(:local_override_values)
+
+    # normalize checks value against a flag type and returns the frozen
+    # value to serve.
+    def self.normalize(type, value)
+      case type
+      when "boolean" then return value if [true, false].include?(value)
+      when "string" then return value.dup.freeze if value.is_a?(String)
+      when "number" then return value if value.is_a?(Numeric) && value.real? && value.to_f.finite?
+      when "json" then return JSON.parse(JSON.generate(value), freeze: true)
+      else raise ArgumentError, "flag type #{type.inspect} is unknown"
+      end
+      raise ArgumentError, "a #{value.class} is not a value of a #{type} flag"
+    rescue JSON::GeneratorError, JSON::ParserError => e
+      raise ArgumentError, "a #{value.class} is not a JSON value: #{e.message}"
+    end
+    private_class_method :normalize
+  end
+end

data/lib/flagnetic/logging.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Flagnetic
+  # EVENTS are the stable names of log events: alert on them. The fields
+  # each carries are listed in the README.
+  EVENTS = %i[
+    ruleset_loaded fetch_failed sse_connected sse_dropped stage_changed key_revoked schema_unsupported
+    rule_unsupported override_set override_refused local_overrides_allowed type_mismatch evaluation_failed
+    listener_failed store_failed worker_failed queue_dropped exposure_skipped ingest_failed exposures_disabled
+    client_not_configured
+  ].freeze
+
+  # LEVELS are the log levels, lowest first.
+  LEVELS = %i[debug info warn error].freeze
+
+  # Logging adapts the logger option: a callable receiving
+  # (level, event, fields), or a standard library ::Logger (anything
+  # responding to add), which receives every level and applies its own.
+  module Logging
+    module_function
+
+    # SEVERITIES are the standard library Logger's severities by level.
+    SEVERITIES = { debug: 0, info: 1, warn: 2, error: 3 }.freeze
+
+    # standard is the default logger: warn and above to $stderr, through
+    # the standard library's Logger when the application has loaded it (a
+    # Rails application has), else as the same lines through Kernel#warn.
+    # The gem never requires logger itself: from Ruby 3.5 it is a bundled
+    # gem, which would be a runtime dependency.
+    def standard(level: :warn)
+      logger = defined?(::Logger) ? ::Logger.new($stderr, progname: "flagnetic") : nil
+      min = LEVELS.index(level)
+      lambda do |lvl, event, fields|
+        next if LEVELS.index(lvl) < min
+
+        line = format_line(event, fields)
+        logger ? logger.add(SEVERITIES[lvl], line) : Kernel.warn("flagnetic #{lvl.upcase}: #{line}")
+      end
+    end
+
+    # adapt turns the logger option into a callable.
+    def adapt(logger)
+      return standard if logger.nil?
+      return logger if logger.respond_to?(:call)
+
+      if logger.respond_to?(:add)
+        return lambda { |lvl, event, fields|
+          logger.add(SEVERITIES[lvl], format_line(event, fields))
+        }
+      end
+
+      raise ArgumentError, "logger must respond to call(level, event, fields) or be a Logger, got #{logger.class}"
+    end
+
+    def format_line(event, fields)
+      ([event.to_s] + fields.map { |k, v| "#{k}=#{v.is_a?(String) ? v.inspect : v}" }).join(" ")
+    end
+
+    private_constant :SEVERITIES
+    private_class_method :standard, :format_line
+  end
+end

data/lib/flagnetic/railtie.rb

@@ -0,0 +1,146 @@
+# frozen_string_literal: true
+
+require "rails/railtie"
+require_relative "../flagnetic"
+require_relative "controller_helpers"
+
+module Flagnetic
+  # Railtie builds Flagnetic.client from the Rails configuration. The gem
+  # loads it only when Rails is loaded; nothing else in the gem knows Rails.
+  #
+  # Configuration, all under config.flagnetic, with the SDK key and base
+  # URL taken from the first of config, the FLAGNETIC_SDK_KEY and
+  # FLAGNETIC_BASE_URL environment variables, and credentials
+  # (flagnetic.sdk_key, flagnetic.base_url):
+  #
+  # - sdk_key, base_url: see above. Without both the client serves
+  #   built-in defaults and logs client_not_configured.
+  # - offline_file: a ruleset document to serve instead; no network.
+  # - start: nil (default) starts the client at boot in a server or job
+  #   process and on first use in rails console, rails runner and rake;
+  #   true starts it at boot everywhere; false only on first use.
+  # - ready_timeout: seconds a boot waits for the ruleset (default 5).
+  # - close_timeout: seconds the exit flush may take (default 5).
+  # - allow_overrides: false disables local overrides, which are otherwise
+  #   allowed in every Rails environment but production, where no option
+  #   enables them.
+  # - logger: the logger option of Client.new (default Rails.logger).
+  # - client_options: any other Client.new option, as a Hash.
+  # - platform, app_version: predeclared attributes of every request
+  #   context, and the client's platform and app_version options, so
+  #   User-Agent, evaluation summaries and exposures outside a request
+  #   carry them too.
+  # - user_unit_type, user_id, anonymous_unit_type, anonymous_cookie,
+  #   attributes: how the controller helper builds a request's context;
+  #   see ControllerHelpers.
+  #
+  # The client is started per process: a Puma or Unicorn worker starts its
+  # own background thread the moment it is forked, and every process
+  # closes its client at exit, which flushes what it has queued.
+  class Railtie < ::Rails::Railtie
+    config.flagnetic = ActiveSupport::OrderedOptions.new
+    config.flagnetic.start = nil
+    config.flagnetic.ready_timeout = 5
+    config.flagnetic.close_timeout = 5
+    config.flagnetic.client_options = {}
+    config.flagnetic.anonymous_cookie = "flagnetic_anonymous_id"
+    config.flagnetic.user_id = ControllerHelpers::SIGNED_IN_USER_ID
+
+    # EMPTY is the ruleset an unconfigured client serves: no flags, so every
+    # getter returns its built-in default.
+    EMPTY = { "schema_version" => 1, "version" => 0, "scope" => "server", "flags" => [] }.freeze
+    private_constant :EMPTY
+
+    # A Rakefile loads the application's tasks before any task boots the
+    # environment, so this runs first: rake is a one-off process.
+    rake_tasks { Railtie.one_off = true }
+
+    initializer "flagnetic.controller_helpers" do
+      ActiveSupport.on_load(:action_controller) { include Flagnetic::ControllerHelpers }
+    end
+
+    config.after_initialize { |app| Railtie.boot(app) }
+
+    class << self
+      attr_writer :one_off
+
+      # boot installs the builder of Flagnetic.client and, in a server or
+      # job process, builds the client now and waits for its ruleset.
+      def boot(app)
+        config = app.config.flagnetic
+        Flagnetic.client_builder = -> { build(app) }
+        Process.singleton_class.prepend(ForkHook) unless Process.singleton_class < ForkHook
+        start = config.start.nil? ? !one_off? : config.start
+        Flagnetic.client if start
+      end
+
+      # build constructs the client from the configuration and registers
+      # its close at exit.
+      def build(app)
+        config = app.config.flagnetic
+        client = new_client(app)
+        at_exit { client.close(timeout: config.close_timeout) }
+        client
+      end
+
+      # after_fork starts the background thread of a forked child: a Puma
+      # or Unicorn worker then follows the change stream before its first
+      # request. A client not built yet is left to be built on first use.
+      def after_fork
+        Flagnetic.client.after_fork if Flagnetic.client?
+      end
+
+      private
+
+      # new_client constructs the client the configuration describes.
+      # Local overrides stay off in production whatever client_options say.
+      def new_client(app)
+        config = app.config.flagnetic
+        log = Logging.adapt(config.logger || ::Rails.logger)
+        sdk_key = setting(app, :sdk_key)
+        base_url = setting(app, :base_url)
+        allow_overrides = config.allow_overrides != false && !::Rails.env.production?
+        options = { **config.client_options, logger: log, allow_overrides: allow_overrides }
+        options[:platform] = config.platform if config.platform
+        options[:app_version] = config.app_version if config.app_version
+        if config.offline_file
+          Client.new(offline_file: config.offline_file, **options)
+        elsif sdk_key && base_url
+          Client.new(sdk_key, base_url: base_url, **options).tap { |c| c.ready(timeout: config.ready_timeout) }
+        else
+          log.call(:warn, :client_not_configured, { rails_env: ::Rails.env.to_s })
+          Client.new(offline: EMPTY, **options, send_summaries: false)
+        end
+      end
+
+      # setting reads name from config, the environment variable, then
+      # credentials; blank is unset.
+      def setting(app, name)
+        [app.config.flagnetic[name], ENV.fetch("FLAGNETIC_#{name.upcase}", nil), credential(app, name)].find(&:present?)
+      end
+
+      def credential(app, name)
+        app.credentials.dig(:flagnetic, name)
+      rescue StandardError
+        nil
+      end
+
+      # one_off tells whether this process is rails console, rails runner or
+      # rake: a process the client must not start in unless it is asked
+      # for.
+      def one_off?
+        @one_off || defined?(::Rails::Console) || defined?(::Rails::Command::RunnerCommand) ? true : false
+      end
+    end
+
+    # ForkHook runs Railtie.after_fork in every forked child (Ruby 3.1's
+    # Process._fork, which every fork goes through).
+    module ForkHook
+      def _fork
+        pid = super
+        Railtie.after_fork if pid.zero?
+        pid
+      end
+    end
+  end
+end

data/lib/flagnetic/recorder.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+require "time"
+
+module Flagnetic
+  # Recorder holds in memory what an offline client would report, so a
+  # test can assert on it: the running evaluation counts, every evaluation
+  # summary a flush or close would have sent, and every exposure and alias
+  # the client recorded. It is thread-safe.
+  class Recorder
+    # SummaryCount is one row of an evaluation summary: how often a flag
+    # served a variation for an evaluation reason. It carries no unit IDs
+    # and no values; variation is nil for local_override and fallback.
+    # count is the wire name; Struct#count is not used on it.
+    SummaryCount = Struct.new(:flag, :variation, :reason, :count, keyword_init: true) # rubocop:disable Lint/StructNewOverride
+
+    # EvaluationSummary is one evaluation summary an offline client would
+    # have sent, as its own event: a deduplication key, the interval and
+    # ruleset version it reports, the app_version option, and its rows.
+    EvaluationSummary = Struct.new(:event_id, :interval_start, :interval_end, :ruleset_version, :app_version, :counts,
+                                   keyword_init: true) do
+      def initialize(**fields)
+        super
+        freeze
+      end
+    end
+
+    # Exposure is one exposure an offline client would have sent.
+    Exposure = Struct.new(:event_id, :experiment_key, :flag_key, :variation_key, :unit_type, :unit_id, :platform,
+                          :app_version, :reason, :override, :ruleset_version, :occurred_at, keyword_init: true) do
+      def initialize(**fields)
+        super
+        freeze
+      end
+    end
+
+    # Alias is one alias an offline client would have sent.
+    Alias = Struct.new(:event_id, :from_unit_type, :from_unit_id, :to_unit_type, :to_unit_id, :occurred_at,
+                       keyword_init: true) do
+      def initialize(**fields)
+        super
+        freeze
+      end
+    end
+
+    def initialize
+      @mutex = Mutex.new
+      @counts = Hash.new(0)
+      @summaries = []
+      @exposures = []
+      @aliases = []
+    end
+
+    # summary returns the running evaluation counts since construction or
+    # the last reset, ordered by flag, variation and reason. Closing a
+    # window (flush, close) does not touch it.
+    def summary
+      rows = @mutex.synchronize { @counts.dup }
+      rows.map { |(flag, variation, reason), n| SummaryCount.new(flag:, variation:, reason:, count: n).freeze }
+          .sort_by { |r| [r.flag, r.variation.to_s, r.reason.to_s] }
+    end
+
+    # reset forgets everything recorded.
+    def reset
+      @mutex.synchronize do
+        @counts.clear
+        @summaries.clear
+        @exposures.clear
+        @aliases.clear
+      end
+    end
+
+    # count records one evaluation into the running total.
+    def count(flag, detail)
+      key = [flag, detail.variation, detail.reason]
+      @mutex.synchronize { @counts[key] += 1 }
+    end
+
+    # record keeps an evaluation summary the client would have sent once a
+    # window closed.
+    def record(summary)
+      @mutex.synchronize { @summaries << summary }
+    end
+
+    # evaluation_summaries returns the evaluation summaries the client
+    # would have sent since construction or the last reset, in the order
+    # they were reported.
+    def evaluation_summaries
+      @mutex.synchronize { @summaries.dup }
+    end
+
+    # record_event keeps an exposure or an alias the client would have
+    # sent, from the wire-shaped Hash Exposures builds. An offline client
+    # records it at the call itself, so a test asserts on it without
+    # flushing.
+    def record_event(event)
+      case event["type"]
+      when "exposure"
+        @mutex.synchronize do
+          @exposures << Exposure.new(event_id: event["event_id"], experiment_key: event["experiment_key"],
+                                     flag_key: event["flag_key"], variation_key: event["variation_key"],
+                                     unit_type: event["unit_type"], unit_id: event["unit_id"],
+                                     platform: event["platform"], app_version: event["app_version"],
+                                     reason: event["evaluation_reason"].to_sym, override: event["override"],
+                                     ruleset_version: event["ruleset_version"],
+                                     occurred_at: Time.parse(event["occurred_at"]))
+        end
+      when "alias"
+        @mutex.synchronize do
+          @aliases << Alias.new(event_id: event["event_id"], from_unit_type: event["from_unit_type"],
+                                from_unit_id: event["from_unit_id"], to_unit_type: event["to_unit_type"],
+                                to_unit_id: event["to_unit_id"], occurred_at: Time.parse(event["occurred_at"]))
+        end
+      end
+    end
+
+    # exposures returns the exposures the client would have sent since
+    # construction or the last reset, in the order they were recorded.
+    # Repeats per experiment, unit and variation are suppressed before they
+    # reach the recorder, exactly as they are online.
+    def exposures
+      @mutex.synchronize { @exposures.dup }
+    end
+
+    # aliases returns the aliases the client would have sent since
+    # construction or the last reset, in the order they were recorded.
+    def aliases
+      @mutex.synchronize { @aliases.dup }
+    end
+
+    # The client hands the recorder what it counts and reports; a test only
+    # reads it.
+    private :count, :record, :record_event
+  end
+end

data/lib/flagnetic/reporter.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require_relative "event_batch"
+
+module Flagnetic
+  # Reporter is the send-with-backoff machinery shared by Exposures and
+  # Summaries: both post batches through EventBatch under the same retry
+  # policy (429, 5xx and network errors retried with backoff bounded by
+  # the flush's own budget; 401/403 backed off to the unauthorized cap
+  # and logged once) and the same bounded wait for another flush already
+  # in progress. schedule_retry is the one place they differ (the protocol
+  # allows honouring Retry-After or backing off on its own schedule); each
+  # includer defines its own. The includer supplies @transport, @timing,
+  # @log, @backoff, @not_before, @revoked_logged, @send_mutex and a
+  # private #kind.
+  module Reporter
+    # MIN_ATTEMPT_TIME is the least budget left an attempt needs to be
+    # worth making at all, so a very short flush_interval (tests) still
+    # gets one real attempt instead of giving up before trying.
+    MIN_ATTEMPT_TIME = 0.01
+
+    private
+
+    def now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+    # wait_for_send_lock waits for @send_mutex, up to deadline, so a flush
+    # bounded by its own budget never blocks on another flush already in
+    # progress for longer than that budget. Returns whether it was
+    # acquired; the caller must unlock it.
+    def wait_for_send_lock(deadline)
+      until @send_mutex.try_lock
+        return false if now >= deadline
+
+        sleep([deadline - now, 0.01].min)
+      end
+      true
+    end
+
+    # attempt sends group, retrying it whole (same events, same event IDs)
+    # on 429, 5xx and network errors, until the platform answers it, the
+    # deadline is up or the key is refused. Each call is itself bounded by
+    # whatever is left of the deadline, so a hung platform never holds a
+    # flush past its budget.
+    def attempt(group, deadline)
+      loop do
+        remaining = deadline - now
+        if remaining <= MIN_ATTEMPT_TIME
+          result = EventBatch::Outcome.new(kind: :retry, error: "flush budget exhausted")
+          # schedule_retry's own doing: without it @not_before never
+          # advances, and the next due? is immediately true again.
+          schedule_retry(group, result)
+          return result
+        end
+
+        result = EventBatch.send_batch(@transport, group, [remaining, @timing.request_timeout].min)
+        case result.kind
+        when :retry then wait = schedule_retry(group, result)
+        when :unauthorized then (schedule_unauthorized(result)
+                                 return result)
+        else return result
+        end
+        return result if now + wait > deadline
+
+        sleep(wait)
+      end
+    end
+
+    def schedule_unauthorized(result)
+      wait = @backoff.next_delay(cap: @timing.unauthorized_cap)
+      @not_before = now + wait
+      return if @revoked_logged
+
+      @revoked_logged = true
+      log(:error, :key_revoked, status: result.status, source: "events")
+    end
+
+    def log_ingest_failed(group, result, wait)
+      log(:warn, :ingest_failed, kind: kind, events: group.size, retry_in: wait.round(3), status: result.status,
+                                 error: result.error)
+    end
+
+    def report_dropped(group, result)
+      log(:warn, :queue_dropped, kind: kind, cause: "client_error",
+                                 events: group.map { |e| e["event_id"] }, status: result.status, error: result.error)
+    end
+
+    def log(level, event, **fields)
+      @log.call(level, event, fields.compact)
+    rescue StandardError
+      nil
+    end
+  end
+  private_constant :Reporter
+end

data/lib/flagnetic/sse_parser.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module Flagnetic
+  # SSEParser reads the text/event-stream format of the change stream by the
+  # WHATWG rules: CRLF, LF or CR end a line, even when split between reads;
+  # a blank line dispatches the event gathered so far; lines starting with
+  # a colon are comments; data lines join with newlines; the last event ID
+  # carries over between events. The server only ever writes `retry`, a
+  # `ruleset` event and `: hb` heartbeats, but a proxy may reframe them.
+  class SSEParser
+    # Event is one dispatched event. type defaults to "message"; id is the
+    # last event ID seen, or nil.
+    Event = Struct.new(:type, :data, :id, keyword_init: true)
+
+    # Error is a stream the parser refuses: a line longer than max_line.
+    class Error < StandardError; end
+
+    BOM = "\xEF\xBB\xBF".b.freeze
+    LINE_END = /[\r\n]/n
+    private_constant :BOM, :LINE_END
+
+    def initialize(max_line: 1 << 20)
+      @max_line = max_line
+      @buffer = +"".b
+      @skip_lf = false
+      @started = false
+      @type = +""
+      @data = +""
+      @id = nil
+    end
+
+    # feed parses the next bytes of the stream, yielding [:event, Event],
+    # [:retry, milliseconds] and [:comment, text] as they complete.
+    def feed(chunk, &)
+      bytes = chunk.b
+      if @skip_lf && !bytes.empty?
+        bytes = bytes.byteslice(1..) if bytes.getbyte(0) == 10
+        @skip_lf = false
+      end
+      @buffer << bytes
+      strip_bom
+      while (i = @buffer.index(LINE_END))
+        line = @buffer.byteslice(0, i)
+        @buffer = @buffer.byteslice((i + line_end_length(i))..)
+        line(line.force_encoding(Encoding::UTF_8).scrub, &)
+      end
+      raise Error, "an event-stream line exceeds #{@max_line} bytes" if @buffer.bytesize > @max_line
+    end
+
+    private
+
+    def strip_bom
+      return if @started || @buffer.bytesize < BOM.bytesize
+
+      @started = true
+      @buffer = @buffer.byteslice(BOM.bytesize..) if @buffer.start_with?(BOM)
+    end
+
+    # line_end_length is 2 for CRLF, else 1. A CR ending the buffer may be
+    # the first half of a CRLF: the LF is skipped when it arrives.
+    def line_end_length(index)
+      return 1 unless @buffer.getbyte(index) == 13
+      return 2 if @buffer.getbyte(index + 1) == 10
+
+      @skip_lf = index + 1 == @buffer.bytesize
+      1
+    end
+
+    def line(line)
+      return dispatch { |e| yield :event, e } if line.empty?
+      return yield(:comment, line[1..]) if line.start_with?(":")
+
+      field, value = line.split(":", 2)
+      value = value.to_s.delete_prefix(" ")
+      case field
+      when "event" then @type = value
+      when "data" then @data << value << "\n"
+      when "id" then @id = value unless value.include?("\u0000")
+      when "retry" then yield :retry, value.to_i if value.match?(/\A\d+\z/)
+      end
+    end
+
+    def dispatch
+      data = @data
+      type = @type
+      @data = +""
+      @type = +""
+      return if data.empty?
+
+      yield Event.new(type: type.empty? ? "message" : type, data: data.chomp, id: @id)
+    end
+  end
+  private_constant :SSEParser
+end

data/lib/flagnetic/store.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Flagnetic
+  # FileStore keeps the last good ruleset in one file: the ruleset document
+  # exactly as the endpoint served it, so the file also works as an offline
+  # document. Its modification time is when the ruleset was received.
+  # Writes go to a unique temporary file renamed over the target, so
+  # processes sharing the path (Puma workers) never read a torn file.
+  class FileStore
+    def initialize(path)
+      @path = path.to_s
+    end
+
+    # load returns [json, received_at], or nil when there is no file.
+    def load
+      json = File.binread(@path)
+      [json, File.mtime(@path)]
+    rescue Errno::ENOENT
+      nil
+    end
+
+    def save(json)
+      tmp = "#{@path}.#{Process.pid}.#{Thread.current.object_id}.tmp"
+      # The server ruleset holds private segments: readable by the owner only.
+      File.open(tmp, File::WRONLY | File::CREAT | File::TRUNC | File::BINARY, 0o600) { |f| f.write(json) }
+      File.rename(tmp, @path)
+    ensure
+      File.delete(tmp) if tmp && File.exist?(tmp)
+    end
+
+    def to_s = @path
+  end
+  private_constant :FileStore
+end