flagnetic 0.1.0

data/lib/flagnetic/exposures.rb

@@ -0,0 +1,342 @@
+# frozen_string_literal: true
+
+require_relative "generations"
+require_relative "event_batch"
+require_relative "reporter"
+require_relative "uuid"
+
+module Flagnetic
+  # Exposures queues exposures and aliases: the explicit
+  # exposure call, automatic exposure, the alias call, suppression of
+  # repeats bounded in memory, and the flush that the background thread and
+  # Client#flush/#close drive with backoff. Offline: nothing is queued,
+  # every event goes to the Recorder at the call itself.
+  #
+  # Thread-safe: evaluated, exposure and add_alias run on whatever thread
+  # called a getter or the exposure or alias method; flush_once runs on the
+  # background thread and, for an explicit flush or close, whatever thread
+  # called it.
+  class Exposures
+    include Reporter
+
+    QUEUE_CAPACITY = 10_000
+    SUPPRESSION_CAPACITY = 10_000
+    DETAIL_CAPACITY = 10_000
+
+    # ExposureRecord is what evaluated remembers for a later exposure call:
+    # the experiment block, the evaluation reason and the ruleset version
+    # the evaluation used.
+    ExposureRecord = Struct.new(:experiment, :reason, :ruleset_version, keyword_init: true)
+
+    # The predeclared attributes an exposure carries from the context.
+    ATTRIBUTE_PLATFORM = "platform"
+    ATTRIBUTE_APP_VERSION = "app_version"
+    private_constant :ATTRIBUTE_PLATFORM, :ATTRIBUTE_APP_VERSION
+
+    # wake: a callable poked after enqueuing, so a background thread
+    # sleeping on a longer, unrelated deadline notices the new event
+    # without waiting for it; a no-op where nothing needs waking (offline,
+    # or no worker yet). platform and app_version stand in for a context
+    # that names neither.
+    def initialize(timing:, log:, transport: nil, record: nil, platform: nil, app_version: nil,
+                   capacity: QUEUE_CAPACITY, wake: -> {})
+      @transport = transport
+      @timing = timing
+      @log = log
+      @record = record
+      @platform = platform
+      @app_version = app_version
+      @capacity = capacity
+      @wake = wake
+      @mutex = Mutex.new
+      @queue = []
+      @suppressed = Generations.new(capacity: SUPPRESSION_CAPACITY)
+      @suppressed_since = now
+      @details = Generations.new(capacity: DETAIL_CAPACITY)
+      @experiment_flags = {}
+      @backoff = Backoff.new(base: timing.fetch_base, cap: timing.fetch_cap)
+      @not_before = 0
+      @flushed_at = now
+      @revoked_logged = false
+      @send_mutex = Mutex.new
+    end
+
+    # offline? reports whether events go to the Recorder rather than to the
+    # events endpoint. An offline queue is driven by nothing: every event is
+    # recorded at the call itself.
+    def offline? = !@record.nil?
+
+    # evaluated remembers the detail of one evaluation for a later exposure
+    # call, and records the exposure at once for a flag with
+    # automatic_exposure. A flag whose evaluation stops carrying an
+    # experiment block is forgotten, so a later exposure call reports
+    # nothing rather than the last assignment of an experiment the ruleset
+    # no longer serves. The fingerprint of context is computed only when a
+    # detail actually needs remembering or forgetting: the great majority
+    # of evaluations, of flags that have never carried an experiment,
+    # touch nothing but @experiment_flags.
+    def evaluated(context, key, detail, automatic)
+      unless detail.experiment
+        forget(detail_key(key, context), detail.ruleset_version) if known?(key)
+        return
+      end
+      mark_known(key)
+      record = ExposureRecord.new(experiment: detail.experiment, reason: detail.reason,
+                                  ruleset_version: detail.ruleset_version)
+      remember(detail_key(key, context), record)
+      emit(context, key, record) if automatic
+    end
+
+    # exposure looks up the most recent detail of key for context and
+    # enqueues an exposure only when it carries an experiment block;
+    # otherwise nothing is sent and exposure_skipped names why.
+    def exposure(context, key)
+      record, found = recall(detail_key(key, context))
+      return skipped(key, cause(key)) unless found
+
+      emit(context, key, record)
+    end
+
+    # add_alias enqueues an alias event linking two unit IDs.
+    def add_alias(from_unit_type, from_unit_id, to_unit_type, to_unit_id)
+      t = Time.now
+      enqueue({ "type" => "alias", "event_id" => UUID.v7(time: t), "from_unit_type" => from_unit_type.to_s,
+                "from_unit_id" => from_unit_id.to_s, "to_unit_type" => to_unit_type.to_s,
+                "to_unit_id" => to_unit_id.to_s, "occurred_at" => EventBatch.format_time(t) })
+    end
+
+    # due? reports whether a flush should run now: something is queued, the
+    # flush interval has passed since the last flush or a full batch is
+    # waiting, and the backoff since the last failure has elapsed.
+    def due?(mono = now)
+      return false if offline? || mono < @not_before
+
+      size = @mutex.synchronize { @queue.size }
+      size.positive? && (mono - @flushed_at >= @timing.exposure_flush || size >= EventBatch::MAX_EVENTS_PER_BATCH)
+    end
+
+    # next_deadline is when due? might next turn true, for a worker's wait
+    # to schedule around; nil when nothing is queued (offline or not). An
+    # enqueue wakes the worker, so a batch filling up is seen at once.
+    def next_deadline
+      size = @mutex.synchronize { @queue.size }
+      return nil if offline? || size.zero?
+      return @not_before if size >= EventBatch::MAX_EVENTS_PER_BATCH
+
+      [@not_before, @flushed_at + @timing.exposure_flush].max
+    end
+
+    # flush_once sends what is queued, retrying with backoff within budget
+    # seconds; whatever is not sent stays queued for a later attempt. Safe
+    # to call from any thread; concurrent callers wait for each other only
+    # up to budget, so one never blocks behind another past its own.
+    def flush_once(budget)
+      return true if offline?
+
+      deadline = now + budget
+      return true unless wait_for_send_lock(deadline)
+
+      begin
+        @flushed_at = now
+        send_queue(deadline)
+      ensure
+        @send_mutex.unlock
+      end
+    end
+
+    # reset_after_fork empties the queue in a forked child, which starts
+    # empty; the parent, which never calls this, keeps its own.
+    def reset_after_fork
+      @mutex.synchronize { @queue.clear }
+      @backoff.reset
+      @not_before = 0
+    end
+
+    private
+
+    def kind = "exposure_or_alias"
+
+    def known?(key) = @mutex.synchronize { @experiment_flags[key] }
+    def mark_known(key) = @mutex.synchronize { @experiment_flags[key] = true }
+
+    # emit builds the exposure and enqueues it, unless the same unit has
+    # already been recorded seeing this variation of this experiment within
+    # the suppression window.
+    def emit(context, key, record)
+      exp = record.experiment
+      if suppressed?([exp.key, exp.unit_type, exp.unit_id, exp.variation])
+        skipped(key, "repeat")
+        return
+      end
+      t = Time.now
+      coerced = begin
+        Eval::Context.__send__(:coerce, context)
+      rescue StandardError
+        Eval::Context::EMPTY
+      end
+      enqueue({ "type" => "exposure", "event_id" => UUID.v7(time: t), "experiment_key" => exp.key, "flag_key" => key,
+                "variation_key" => exp.variation, "unit_type" => exp.unit_type, "unit_id" => exp.unit_id,
+                "platform" => predeclared(coerced, ATTRIBUTE_PLATFORM) || @platform,
+                "app_version" => predeclared(coerced, ATTRIBUTE_APP_VERSION) || @app_version,
+                "evaluation_reason" => record.reason.to_s, "override" => exp.override,
+                "ruleset_version" => record.ruleset_version, "occurred_at" => EventBatch.format_time(t) })
+    end
+
+    def skipped(key, cause)
+      log(:debug, :exposure_skipped, flag: key, cause: cause)
+    end
+
+    # cause tells the two mistakes behind an exposure call with nothing to
+    # report apart, as far as the SDK can: a flag no evaluation of which
+    # has ever carried an experiment block, and a flag that has, but not
+    # for this context.
+    def cause(key) = known?(key) ? "no_detail" : "no_experiment"
+
+    def suppressed?(key)
+      @mutex.synchronize do
+        t = now
+        if t - @suppressed_since >= @timing.exposure_flush
+          @suppressed.expire
+          @suppressed_since = t
+        end
+        _, seen = @suppressed.load(key)
+        @suppressed.store(key, true) unless seen
+        seen
+      end
+    end
+
+    def detail_key(flag, context) = [flag, fingerprint(context)]
+
+    def remember(key, record)
+      @mutex.synchronize do
+        held, ok = @details.load(key)
+        next if ok && held.ruleset_version > record.ruleset_version
+
+        @details.store(key, record)
+      end
+    end
+
+    def forget(key, ruleset_version)
+      @mutex.synchronize do
+        held, ok = @details.load(key)
+        next if ok && held.ruleset_version > ruleset_version
+
+        @details.delete(key)
+      end
+    end
+
+    def recall(key) = @mutex.synchronize { @details.load(key) }
+
+    # fingerprint is a canonical, in-process encoding of a context, so an
+    # exposure call finds the detail of the very context the flag was
+    # evaluated for. It never leaves the process and needs no cross-language
+    # stability.
+    def fingerprint(context)
+      c = Eval::Context.__send__(:coerce, context)
+      units = c.units.sort.flatten
+      attributes = c.attributes.sort.map { |k, v| [k, attribute_token(v)] }.flatten
+      (units + [""] + attributes).join(" ")
+    rescue StandardError
+      context.object_id.to_s
+    end
+
+    def attribute_token(value)
+      case value
+      when String, Numeric, true, false then value.inspect
+      when Array then value.map(&:inspect).join(",")
+      when Time then value.utc.iso8601(3)
+      else value.to_s
+      end
+    end
+
+    # predeclared reads one predeclared attribute as a string from an
+    # already-coerced context; nil when the context names none or names it
+    # as another type, and the client's option then stands in.
+    def predeclared(coerced_context, name)
+      v = coerced_context.attributes[name]
+      v.is_a?(String) ? v : nil
+    end
+
+    # enqueue puts one event in the queue, dropping the oldest when the
+    # queue is full, and wakes whatever is waiting for a longer deadline.
+    # An offline queue hands the event to the Recorder instead.
+    def enqueue(event)
+      if offline?
+        @record.call(event)
+        return
+      end
+      dropped = @mutex.synchronize do
+        @queue << event
+        excess = @queue.size - @capacity
+        excess.positive? ? @queue.shift(excess).size : 0
+      end
+      @wake.call
+      return unless dropped.positive?
+
+      log(:warn, :queue_dropped, kind: kind, cause: "queue_full", events: dropped, capacity: @capacity)
+    end
+
+    def take = @mutex.synchronize { @queue.dup.tap { @queue.clear } }
+
+    # put_back returns events the platform did not answer to the front of
+    # the queue, so the next flush sends them again with the same event
+    # IDs. They are the oldest, so they are the first dropped when the
+    # queue is full.
+    def put_back(events)
+      return if events.empty?
+
+      dropped = @mutex.synchronize do
+        @queue = events + @queue
+        excess = @queue.size - @capacity
+        excess.positive? ? @queue.shift(excess).size : 0
+      end
+      return unless dropped.positive?
+
+      log(:warn, :queue_dropped, kind: kind, cause: "queue_full", events: dropped, capacity: @capacity)
+    end
+
+    def send_queue(deadline)
+      events = take
+      return true if events.empty?
+
+      batches = EventBatch.pack(events, @transport.identity)
+      batches.each_with_index do |group, i|
+        result = attempt(group, deadline)
+        case result.kind
+        when :accepted
+          @backoff.reset
+          @not_before = 0
+          @revoked_logged = false
+          report_accepted(group, result)
+        when :dropped
+          report_dropped(group, result)
+        else # :retry or :unauthorized ran out of the flush's budget
+          put_back(batches[i..].flatten(1))
+          return false
+        end
+      end
+      true
+    end
+
+    # schedule_retry backs off on its own schedule rather than honouring
+    # Retry-After: the flush budget is seconds, and a header naming
+    # minutes must not hold exposures back for minutes (the protocol
+    # allows either choice).
+    def schedule_retry(group, result)
+      wait = @backoff.next_delay(cap: @timing.fetch_cap)
+      @not_before = now + wait
+      log_ingest_failed(group, result, wait)
+      wait
+    end
+
+    def report_accepted(group, result)
+      return if result.rejected.to_a.empty?
+
+      log(:warn, :queue_dropped, kind: kind, cause: "refused_by_the_platform",
+                                 events: group.map { |e| e["event_id"] },
+                                 accepted: result.accepted, duplicates: result.duplicates,
+                                 rejected: result.rejected.map { |r| r["code"] })
+    end
+  end
+  private_constant :Exposures
+end

data/lib/flagnetic/fallback_chain.rb

@@ -0,0 +1,230 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Flagnetic
+  # FallbackChain is the part of Client that holds the ruleset: it loads
+  # the store and offline documents, applies each fetch outcome and moves
+  # the stage. Every method is private; the background thread calls handle_fetch,
+  # held_version and key_revoked.
+  module FallbackChain
+    # Loaded is the held ruleset and when it was received.
+    Loaded = Struct.new(:ruleset, :received_at, keyword_init: true)
+
+    private
+
+    def held_version = @loaded&.ruleset&.version
+
+    def load_offline
+      ruleset = decode(@offline_source.call)
+      unless ruleset.is_a?(Eval::Ruleset)
+        cause = ruleset.is_a?(Array) ? "schema_version #{ruleset[1]} is unsupported" : ruleset
+        raise OfflineError, "offline ruleset: #{cause}"
+      end
+
+      swap(ruleset, received_at: Time.now, stage: :fresh, source: "offline")
+    rescue OfflineError
+      raise
+    rescue StandardError => e
+      # The source may be the application's own callable: whatever it
+      # raises is a failed document, never an error escaping Client.new.
+      raise OfflineError, "offline ruleset: #{e.class}: #{e.message}"
+    end
+
+    # load_store serves a persisted ruleset as last_good before any network
+    # call. A corrupt file leaves the stage built_in, with a log.
+    def load_store
+      json, received_at = @store.load
+      return unless json
+
+      ruleset = decode(json)
+      unless ruleset.is_a?(Eval::Ruleset)
+        cause = ruleset.is_a?(Array) ? "schema_version #{ruleset[1]} is unsupported" : ruleset
+        return log(:error, :store_failed, path: @store.to_s, cause: cause)
+      end
+
+      swap(ruleset, received_at: received_at, stage: :last_good, source: "store")
+    rescue SystemCallError, IOError => e
+      log(:error, :store_failed, path: @store.to_s, cause: "#{e.class}: #{e.message}")
+    end
+
+    # decode returns the ruleset a fetched or stored document holds, or the
+    # reason it is refused: it must parse, carry a supported schema_version
+    # and be of the server scope.
+    def decode(json)
+      # The nesting limit is the evaluator's own, so a document it reads is
+      # never refused here first.
+      doc = JSON.parse(json, max_nesting: 10_000)
+      return "parse: the document is not a JSON object" unless doc.is_a?(Hash)
+
+      schema = doc["schema_version"]
+      return "parse: schema_version is missing" unless schema.is_a?(Integer)
+      return [:schema, schema] unless schema.between?(SCHEMA_VERSION_MIN, SCHEMA_VERSION_MAX)
+      return "parse: scope is #{doc["scope"].inspect}, want \"server\"" unless doc["scope"] == "server"
+
+      Eval::Ruleset.decode(json)
+    rescue JSON::ParserError, EncodingError => e
+      "parse: #{e.message.lines.first&.strip}"
+    rescue Eval::DecodeError => e
+      "parse: #{e.message}"
+    end
+
+    # swap installs a ruleset, moves to stage and returns the flags whose
+    # evaluation may have changed, local overrides dropped by a protected
+    # environment included.
+    def swap(ruleset, received_at:, stage:, source:)
+      # The ruleset and its stage change together, so no getter sees one
+      # without the other.
+      previous, previous_stage = @mutex.synchronize do
+        p = [@loaded, @stage]
+        @loaded = Loaded.new(ruleset: ruleset, received_at: received_at).freeze
+        @stage = stage
+        p
+      end
+      log(:info, :ruleset_loaded, source: source, ruleset_version: ruleset.version, flags: ruleset.flags.size)
+      report_unsupported(ruleset)
+      stage_changed(previous_stage, stage)
+      (Changes.flags(previous&.ruleset, ruleset) | refuse_local_overrides(ruleset)).sort
+    end
+
+    # report_unsupported logs rule_unsupported for every flag of ruleset
+    # holding a construct this evaluator does not know, once per flag and
+    # ruleset version: the evaluator finds them when it decodes, so a flag
+    # nobody evaluates is reported too, and an SDK that must be upgraded
+    # says so without a line per evaluation.
+    def report_unsupported(ruleset)
+      ruleset.unsupported_rules.sort.each do |key, constructs|
+        next unless first_report?(key, ruleset.version)
+
+        log(:error, :rule_unsupported, flag: key, constructs: constructs.map { |c| "#{c.kind} #{c.name}" },
+                                       ruleset_version: ruleset.version)
+      end
+    end
+
+    def first_report?(key, version)
+      @mutex.synchronize do
+        next false if @unsupported_logged[key] == version
+
+        @unsupported_logged[key] = version
+        true
+      end
+    end
+
+    def move_to(stage)
+      previous = @mutex.synchronize do
+        p = @stage
+        @stage = stage
+        p
+      end
+      stage_changed(previous, stage)
+    end
+
+    def stage_changed(previous, stage)
+      return if previous == stage
+
+      log(stage == :fresh ? :info : :warn, :stage_changed, from: previous, to: stage)
+      fire(@stage_listeners, stage)
+    end
+
+    # handle_fetch applies a fetch outcome (a Transport::Response or the
+    # exception raised) to the fallback chain and returns :ok, :failed or
+    # :revoked. Called by the background thread.
+    def handle_fetch(response)
+      return fetch_failed("#{response.class}: #{response.message}") if response.is_a?(Exception)
+
+      case response.status
+      when 200 then accept(response.body)
+      when 304 then not_modified
+      when 401, 403 then key_revoked(response.status, "fetch")
+      else fetch_failed("HTTP #{response.status}")
+      end
+    rescue StandardError => e
+      # An unexpected failure applying the outcome is a failed fetch, so the
+      # stage and the retry schedule move as for any other.
+      fetch_failed("#{e.class}: #{e.message}")
+    ensure
+      # ready waits for the first outcome, whatever it was.
+      mark_ready
+    end
+
+    def accept(json)
+      ruleset = decode(json)
+      return schema_unsupported(ruleset[1]) if ruleset.is_a?(Array)
+      return fetch_failed(ruleset) if ruleset.is_a?(String)
+
+      held = held_version
+      return fetch_failed("version #{ruleset.version} is below the held #{held}") if held && ruleset.version < held
+
+      changed = swap(ruleset, received_at: Time.now, stage: :fresh, source: "fetch")
+      succeeded
+      persist(json)
+      fire_change(changed)
+      :ok
+    end
+
+    def not_modified
+      return fetch_failed("304 without a held ruleset") unless @loaded
+
+      move_to(:fresh)
+      succeeded
+      :ok
+    end
+
+    def fetch_failed(cause)
+      log(:warn, :fetch_failed, cause: cause, stage: @stage)
+      failed
+      :failed
+    end
+
+    # failed moves fresh to last_good and marks the held ruleset stale;
+    # last_good and built_in stay.
+    def failed
+      @mutex.synchronize { @stale_since ||= Time.now }
+      move_to(:last_good) if @stage == :fresh
+    end
+
+    def succeeded
+      @mutex.synchronize do
+        @stale_since = nil
+        @once.delete([:key_revoked, "fetch"])
+      end
+    end
+
+    def schema_unsupported(schema)
+      once(:schema_unsupported) do
+        log(:error, :schema_unsupported, schema_version: schema,
+                                         supported: (SCHEMA_VERSION_MIN..SCHEMA_VERSION_MAX).to_a)
+      end
+      fetch_failed("schema_version #{schema} is unsupported")
+    end
+
+    # key_revoked keeps serving: the stage leaves fresh as for any failure,
+    # key_revoked is logged once at error per source, until a fetch
+    # succeeds again or the stream connects again. A refused fetch is a
+    # failure of the fallback chain; a refused stream only closes it.
+    def key_revoked(status, source)
+      once([:key_revoked, source]) { log(:error, :key_revoked, status: status, source: source) }
+      failed if source == "fetch"
+      :revoked
+    end
+
+    def stream_connected
+      @mutex.synchronize { @once.delete([:key_revoked, "stream"]) }
+      log(:info, :sse_connected)
+    end
+
+    def persist(json)
+      @store&.save(json)
+    rescue SystemCallError, IOError => e
+      log(:error, :store_failed, path: @store.to_s, cause: "#{e.class}: #{e.message}")
+    end
+
+    def mark_ready
+      @ready_mutex.synchronize do
+        @ready = true
+        @ready_cond.broadcast
+      end
+    end
+  end
+  private_constant :FallbackChain
+end

data/lib/flagnetic/generations.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module Flagnetic
+  # Generations is a bounded map whose entries survive one rotation: a
+  # lookup sees the current generation and the one before it, and a
+  # rotation forgets only the older one. It bounds memory without a timer
+  # and without the bookkeeping of an eviction order, which is what a
+  # suppression set the platform does not rely on needs.
+  #
+  # It holds no clock and no window: when a generation has lived long
+  # enough is the caller's to decide, by calling expire. A full generation
+  # rotates on its own, because the bound is the whole point.
+  #
+  # Not safe for concurrent use; the caller holds a mutex over it.
+  class Generations
+    def initialize(capacity:)
+      @capacity = capacity
+      @current = {}
+      @previous = {}
+    end
+
+    # load returns [value, true] from either generation, or [nil, false].
+    def load(key)
+      return [@current[key], true] if @current.key?(key)
+      return [@previous[key], true] if @previous.key?(key)
+
+      [nil, false]
+    end
+
+    # store writes key into the current generation, expiring the older one
+    # first when the current one is full.
+    def store(key, value)
+      expire if !@current.key?(key) && @current.size >= @capacity
+      @current[key] = value
+    end
+
+    # delete forgets key in both generations.
+    def delete(key)
+      @current.delete(key)
+      @previous.delete(key)
+    end
+
+    # expire forgets the older generation and makes the current one older.
+    def expire
+      @previous = @current
+      @current = {}
+    end
+  end
+  private_constant :Generations
+end

data/lib/flagnetic/global.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+# Flagnetic.client: the process-wide client.
+module Flagnetic
+  # NotConfigured is raised by Flagnetic.client when no client was assigned
+  # and no builder was set.
+  class NotConfigured < Error; end
+
+  @global_mutex = Mutex.new
+  @client = nil
+  @client_builder = nil
+
+  class << self
+    # client returns the process-wide client: the one assigned with
+    # client=, or the one client_builder builds on first use, once, however
+    # many threads ask at the same time. The Rails integration sets either.
+    # Raises NotConfigured when neither was set.
+    #
+    #   Flagnetic.client = Flagnetic::Client.new(ENV.fetch("FLAGNETIC_SDK_KEY"), base_url: "https://flags.example.com")
+    #   Flagnetic.client.bool("new_checkout", default: false, context: { units: { user: "u-1001" } })
+    def client
+      @client || @global_mutex.synchronize do
+        @client ||= @client_builder&.call or
+          raise NotConfigured, "Flagnetic.client is not set: assign one with Flagnetic.client= " \
+                               "or use the Rails integration"
+      end
+    end
+
+    # client= assigns the process-wide client. It does not close the one it
+    # replaces.
+    def client=(client)
+      @global_mutex.synchronize { @client = client }
+    end
+
+    # client_builder= sets a callable building the process-wide client on
+    # the first call to client, unless one is assigned before.
+    def client_builder=(builder)
+      @global_mutex.synchronize { @client_builder = builder }
+    end
+
+    # client? tells whether the process-wide client exists, without
+    # building it.
+    def client? = !@client.nil?
+
+    private
+
+    # replace_client swaps the assigned client, and the builder unless it
+    # is :keep, returning the previous pair; Testing.with_flags restores it
+    # without building a client that was never built.
+    def replace_client(client, builder = :keep)
+      @global_mutex.synchronize do
+        previous = [@client, @client_builder]
+        @client = client
+        @client_builder = builder unless builder == :keep
+        previous
+      end
+    end
+  end
+end