flagnetic 0.1.0

data/lib/flagnetic/stream.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require "net/http"
+
+module Flagnetic
+  # Stream is one request of the change stream, read on a thread of its own
+  # so that the worker can keep its timers while the stream is quiet. What
+  # happens is posted to the worker's inbox as [:stream, id, kind, ...]:
+  #
+  # - :head, status: the response head arrived.
+  # - :frame, kind, value: a parsed frame of a 200 response.
+  # - :ended, cause, error: the request is over; error is false for the
+  #   server closing the stream, true for anything else, a connection that
+  #   fails, a malformed stream, or silence for longer than the read
+  #   timeout the stream is held with.
+  #
+  # The read timeout is the silence limit: Net::HTTP raises when no byte,
+  # a heartbeat included, arrives within it.
+  class Stream
+    def initialize(id, transport, held_version, silence:, inbox:)
+      @id = id
+      @transport = transport
+      @request = transport.stream_request(held_version)
+      @silence = silence
+      @inbox = inbox
+      @thread = Thread.new { run }
+      @thread.name = "flagnetic-stream"
+      @thread.report_on_exception = false
+    end
+
+    attr_reader :id
+
+    # close ends the request. Killing the thread runs the ensure of
+    # Net::HTTP#start, which closes the socket.
+    def close
+      @thread.kill
+      @thread.join(1)
+    end
+
+    private
+
+    def run
+      http = @transport.http_for(@transport.stream_uri, @transport.timeout)
+      http.start { |h| h.request(@request) { |response| read(h, response) } }
+      post(:ended, "closed by the server", false)
+    rescue Net::ReadTimeout
+      post(:ended, "no frame for #{@silence} s", true)
+    rescue StandardError => e
+      post(:ended, "#{e.class}: #{e.message}", true)
+    end
+
+    def read(http, response)
+      status = response.code.to_i
+      post(:head, status)
+      return unless status == 200
+
+      http.read_timeout = @silence
+      parser = SSEParser.new
+      response.read_body { |chunk| parser.feed(chunk) { |kind, value| post(:frame, kind, value) } }
+    end
+
+    def post(*message) = @inbox << [:stream, @id, *message]
+  end
+  private_constant :Stream
+end

data/lib/flagnetic/summaries.rb

@@ -0,0 +1,257 @@
+# frozen_string_literal: true
+
+require "time"
+require_relative "event_batch"
+require_relative "reporter"
+require_relative "uuid"
+
+module Flagnetic
+  # Summaries counts every evaluation and reports each interval's window as
+  # one or more evaluation summary events: counts per (flag, variation,
+  # reason), the ruleset version at the interval's end, no unit IDs. A
+  # window whose report failed merges into the next one rather than being
+  # retried under its own event ID; one older than an hour is dropped
+  # instead. Offline: the window rotates only on flush and close, and goes
+  # to the Recorder instead of the events endpoint.
+  class Summaries
+    include Reporter
+
+    MAX_SUMMARY_ROWS = 1000
+    # MAX_WINDOW_AGE is how old a window whose report failed may be before
+    # it is dropped rather than merged forward: an hour, so a
+    # long outage does not eventually deliver counts nobody can place.
+    MAX_WINDOW_AGE = 3600
+
+    # LogicalEvent is one evaluation summary before it is wire-encoded:
+    # counts as [flag, variation, reason, count] tuples, reason a symbol.
+    LogicalEvent = Struct.new(:event_id, :interval_start, :interval_end, :ruleset_version, :counts, keyword_init: true)
+
+    # app_version, when given, goes with every summary.
+    def initialize(timing:, log:, ruleset_version:, transport: nil, record: nil, app_version: nil)
+      @transport = transport
+      @app_version = app_version
+      @timing = timing
+      @log = log
+      @record = record
+      @ruleset_version = ruleset_version
+      @mutex = Mutex.new
+      @wall_start = Time.now
+      @window_due_since = now
+      @current = Hash.new(0)
+      @kept = nil
+      @backoff = Backoff.new(base: timing.fetch_base, cap: timing.fetch_cap)
+      @not_before = 0
+      @revoked_logged = false
+      @send_mutex = Mutex.new
+    end
+
+    # offline? reports whether the window goes to the Recorder rather than
+    # to the events endpoint.
+    def offline? = !@record.nil?
+
+    # count records one served value: the flag, the variation and the
+    # evaluation reason, never the value and never the context.
+    def count(flag, detail)
+      key = [flag, detail.variation, detail.reason]
+      @mutex.synchronize { @current[key] += 1 }
+    end
+
+    # due? reports whether the reporting interval has elapsed and the
+    # backoff since the last failure, if any, has too.
+    def due?(mono = now)
+      return false if offline?
+
+      mono >= @not_before && mono - @window_due_since >= @timing.summary_flush
+    end
+
+    # next_deadline is when due? might next turn true, for a worker's wait
+    # to schedule around; nil offline, where nothing drives it.
+    def next_deadline
+      return nil if offline?
+
+      [@not_before, @window_due_since + @timing.summary_flush].max
+    end
+
+    # flush_once closes the current window and delivers it (or, offline,
+    # hands it to the Recorder), retrying with backoff within budget
+    # seconds. Safe to call from any thread; concurrent callers wait for
+    # each other only up to budget, so one never blocks behind another
+    # past its own.
+    def flush_once(budget)
+      deadline = now + budget
+      return true unless wait_for_send_lock(deadline)
+
+      begin
+        w = take_window
+        w ? send_window(w, deadline) : true
+      ensure
+        @send_mutex.unlock
+      end
+    end
+
+    # reset_after_fork empties the window in a forked child, which starts
+    # empty; the parent, which never calls this, keeps its own.
+    def reset_after_fork
+      @mutex.synchronize do
+        @current = Hash.new(0)
+        @kept = nil
+        @wall_start = Time.now
+        @window_due_since = now
+      end
+      @backoff.reset
+      @not_before = 0
+    end
+
+    private
+
+    def kind = "evaluation_summary"
+
+    # take_window closes the current window, merging in one a failed
+    # report kept unless it began over an hour ago, and returns nil when
+    # nothing was evaluated.
+    def take_window
+      wall_end = Time.now
+      win, kept = @mutex.synchronize do
+        window = { start: @wall_start, end: wall_end, counts: @current }
+        held = @kept
+        @wall_start = wall_end
+        @window_due_since = now
+        @current = Hash.new(0)
+        @kept = nil
+        [window, held]
+      end
+      merge_kept(win, kept, wall_end) if kept
+      win[:counts].empty? ? nil : win
+    end
+
+    def merge_kept(win, kept, wall_end)
+      age = wall_end - kept[:start]
+      if age > MAX_WINDOW_AGE
+        log(:warn, :queue_dropped, kind: kind, rows: kept[:counts].size, cause: "window_older_than_one_hour",
+                                   age: age.round(3))
+        return
+      end
+      win[:start] = kept[:start] if kept[:start] < win[:start]
+      kept[:counts].each { |k, n| win[:counts][k] = (win[:counts][k] || 0) + n }
+    end
+
+    # keep puts a window back, to merge into the next one taken.
+    def keep(window)
+      @mutex.synchronize do
+        if @kept.nil?
+          @kept = window
+          next
+        end
+        @kept[:start] = window[:start] if window[:start] < @kept[:start]
+        window[:counts].each { |k, n| @kept[:counts][k] = (@kept[:counts][k] || 0) + n }
+      end
+    end
+
+    def send_window(win, deadline)
+      events = build_events(win)
+      if offline?
+        events.each { |e| @record.call(public_summary(e)) }
+        return true
+      end
+
+      wire_events = events.map { |e| wire(e) }
+      batches = EventBatch.pack(wire_events, @transport.identity)
+      batches.each_with_index do |group, i|
+        result = attempt(group, deadline)
+        case result.kind
+        when :accepted
+          @backoff.reset
+          @not_before = 0
+          @revoked_logged = false
+          report_accepted(group, result)
+        when :dropped
+          report_dropped(group, result)
+        else
+          keep(window_of(batches[i..].flatten(1)))
+          return false
+        end
+      end
+      true
+    end
+
+    # build_events turns a window into logical summary events: the rows
+    # ordered, at most 1000 to an event, each with its own event ID and all
+    # naming the window's interval and the ruleset version held at its end.
+    def build_events(win)
+      rows = win[:counts].map { |(flag, variation, reason), n| { flag:, variation:, reason:, count: n } }
+      rows.sort_by! { |r| [r[:flag], r[:variation].to_s, r[:reason].to_s] }
+      # Before any ruleset was held the version is 0, which no ruleset has.
+      version = @ruleset_version.call || 0
+      events = []
+      events << LogicalEvent.new(event_id: UUID.v7, interval_start: win[:start], interval_end: win[:end],
+                                 ruleset_version: version, counts: rows.shift(MAX_SUMMARY_ROWS)) until rows.empty?
+      events
+    end
+
+    def wire(event)
+      doc = { "type" => "evaluation_summary", "event_id" => event.event_id,
+              "interval_start" => EventBatch.format_time(event.interval_start),
+              "interval_end" => EventBatch.format_time(event.interval_end),
+              "ruleset_version" => event.ruleset_version,
+              "counts" => event.counts.map do |r|
+                { "flag" => r[:flag], "variation" => r[:variation], "reason" => r[:reason].to_s, "count" => r[:count] }
+              end }
+      doc["app_version"] = @app_version if @app_version
+      doc
+    end
+
+    def public_summary(event)
+      Recorder::EvaluationSummary.new(
+        event_id: event.event_id, interval_start: event.interval_start, interval_end: event.interval_end,
+        ruleset_version: event.ruleset_version, app_version: @app_version,
+        counts: event.counts.map do |r|
+          Recorder::SummaryCount.new(flag: r[:flag], variation: r[:variation],
+                                     reason: r[:reason], count: r[:count])
+        end
+      )
+    end
+
+    # window_of rebuilds a window from the summary events that were not
+    # delivered, so their rows merge into the next one.
+    def window_of(wire_events)
+      counts = Hash.new(0)
+      start_t = nil
+      end_t = nil
+      wire_events.each do |e|
+        s = Time.parse(e["interval_start"])
+        en = Time.parse(e["interval_end"])
+        start_t = s if start_t.nil? || s < start_t
+        end_t = en if end_t.nil? || en > end_t
+        e["counts"].each { |r| counts[[r["flag"], r["variation"], r["reason"].to_sym]] += r["count"] }
+      end
+      { start: start_t || Time.now, end: end_t || Time.now, counts: counts }
+    end
+
+    # schedule_retry honours Retry-After, capped at the ordinary backoff
+    # ceiling: a header naming minutes must not silence reporting for
+    # minutes (or, from a misbehaving proxy, far longer), so it bounds
+    # rather than replaces the backoff (the protocol allows either
+    # choice; Exposures ignores the header entirely instead).
+    def schedule_retry(group, result)
+      wait = if result.retry_after
+               [result.retry_after,
+                @timing.fetch_cap].min
+             else
+               @backoff.next_delay(cap: @timing.fetch_cap)
+             end
+      @not_before = now + wait
+      log_ingest_failed(group, result, wait)
+      wait
+    end
+
+    def report_accepted(group, result)
+      return if result.dropped_rows.to_i.zero? && result.rejected.to_a.empty?
+
+      log(:warn, :queue_dropped, kind: kind, cause: "refused_by_the_platform",
+                                 events: group.map { |e| e["event_id"] }, accepted: result.accepted,
+                                 duplicates: result.duplicates, dropped_rows: result.dropped_rows,
+                                 rejected: result.rejected.map { |r| r["code"] })
+    end
+  end
+  private_constant :Summaries
+end

data/lib/flagnetic/testing.rb

@@ -0,0 +1,160 @@
+# frozen_string_literal: true
+
+require "json"
+require_relative "../flagnetic"
+
+module Flagnetic
+  # Testing builds offline clients for deterministic tests: from a ruleset
+  # document, a file, or a Hash of flag key to value. No client it builds
+  # ever touches the network; the recorder holds what it would report.
+  #
+  #   require "flagnetic/testing"
+  #
+  #   client = Flagnetic::Testing.offline(values: { "new_checkout" => true })
+  #   client.set("new_checkout", false) # now served to everyone as fallthrough
+  #   client.recorder.summary           # evaluation counts
+  #
+  # Helpers gives RSpec examples and Minitest tests with_flags and
+  # with_ruleset, which swap such a client in as Flagnetic.client for a
+  # block.
+  module Testing
+    # VARIATION names the one variation set and values: give a flag.
+    VARIATION = "value"
+    private_constant :VARIATION
+
+    # offline builds an OfflineClient from a ruleset document (JSON text or
+    # a Hash, exactly as the ruleset endpoint serves it), from file:, or
+    # from values:, a Hash of flag key to value where the value's class
+    # decides the flag type. Other options go to Client.new. It raises
+    # OfflineError when the document does not load.
+    def self.offline(document = nil, file: nil, values: nil, **)
+      given = [document, file, values].compact.size
+      raise ArgumentError, "pass one of a document, file: or values:" unless given == 1
+
+      source =
+        if file then -> { File.binread(file) }
+        elsif values then document_from(values).then { |doc| -> { doc } }
+        else
+          (document.is_a?(Hash) ? JSON.generate(document) : document.to_s).then { |doc| -> { doc } }
+        end
+      OfflineClient.new(source, **)
+    end
+
+    # document_from builds a ruleset document serving each value as the
+    # fallthrough of a flag with one variation.
+    def self.document_from(values)
+      doc = { "schema_version" => 1, "version" => 1, "scope" => "server", "flags" => [] }
+      values.each { |key, value| set_flag(doc, key.to_s, value) }
+      JSON.generate(doc)
+    end
+
+    # set_flag replaces or adds the flag key in a parsed ruleset document.
+    # An existing flag keeps its type and value must be of it.
+    def self.set_flag(doc, key, value)
+      flags = (doc["flags"] ||= [])
+      at = flags.index { |f| f.is_a?(Hash) && f["key"] == key }
+      type = Client.__send__(:type_of, value)
+      if at && (existing = flags[at]["type"]) != "json" && existing != type
+        raise ArgumentError, "flag #{key.inspect} is #{existing}, a #{value.class} is #{type}"
+      end
+
+      type = flags[at]["type"] if at
+      flag = { "key" => key, "type" => type, "on" => true, "variations" => [{ "key" => VARIATION, "value" => value }],
+               "off_variation" => VARIATION, "fallthrough" => VARIATION }
+      at ? flags[at] = flag : flags << flag
+    end
+
+    # with_flags makes an offline client serving the given flag values
+    # Flagnetic.client for the block, as offline(values:) builds it, and
+    # restores the previous one afterwards, even when the block raises; a
+    # client the Rails integration had not built yet stays unbuilt. Keys
+    # may be Strings or Symbols. It yields the client, whose recorder holds
+    # what the code under test reported. The swap is process-wide: tests
+    # running in parallel threads must not use it.
+    #
+    #   Flagnetic::Testing.with_flags("new_checkout" => true) do |client|
+    #     get "/checkout"
+    #     assert client.recorder.summary.any? { |row| row.flag == "new_checkout" }
+    #   end
+    def self.with_flags(values = {}, **more, &)
+      swap(offline(values: values.merge(more)), &)
+    end
+
+    # with_ruleset is with_flags for a ruleset document (JSON text or a
+    # Hash, exactly as the ruleset endpoint serves it) or file:, for a test
+    # needing rules, rollouts or an experiment: the recorder then holds the
+    # exposures the code under test reported.
+    def self.with_ruleset(document = nil, file: nil, &)
+      raise ArgumentError, "pass a document or file:" if document.nil? == file.nil?
+
+      swap(offline(document, file: file), &)
+    end
+
+    def self.swap(client)
+      raise ArgumentError, "a block is required" unless block_given?
+
+      previous = Flagnetic.__send__(:replace_client, client)
+      begin
+        yield client
+      ensure
+        Flagnetic.__send__(:replace_client, previous.first)
+        client.close(timeout: 0)
+      end
+    end
+    private_class_method :document_from, :set_flag, :swap
+
+    # Helpers is with_flags and with_ruleset for test classes:
+    #
+    #   RSpec.configure { |config| config.include Flagnetic::Testing::Helpers }
+    #
+    #   class ActiveSupport::TestCase
+    #     include Flagnetic::Testing::Helpers
+    #   end
+    module Helpers
+      # with_flags is Flagnetic::Testing.with_flags.
+      def with_flags(...) = Testing.with_flags(...)
+
+      # with_ruleset is Flagnetic::Testing.with_ruleset.
+      def with_ruleset(...) = Testing.with_ruleset(...)
+    end
+
+    # OfflineClient is an offline Client whose document a test can rewrite.
+    class OfflineClient < Client
+      def initialize(read, **)
+        @read = read
+        @document_mutex = Mutex.new
+        @document = read.call
+        begin
+          Eval::Ruleset.decode(@document)
+        rescue Eval::DecodeError, TypeError => e
+          raise OfflineError, "offline ruleset: #{e.message}"
+        end
+        super(offline: -> { @document_mutex.synchronize { @document } }, **)
+      end
+
+      alias reload_document reload
+      private :reload_document
+
+      # set rewrites the flag key to serve value to everyone: one variation,
+      # served as fallthrough, so the reason reads fallthrough. A key the
+      # document lacks is added, typed by the value; an existing flag keeps
+      # its type. on_change fires.
+      def set(key, value)
+        @document_mutex.synchronize do
+          doc = JSON.parse(@document)
+          Testing.__send__(:set_flag, doc, key.to_s, value)
+          @document = JSON.generate(doc)
+        end
+        reload_document
+      end
+
+      # reload re-reads the source the client was built from, discarding
+      # what set changed. on_change fires with the flags that changed.
+      def reload
+        document = @read.call
+        @document_mutex.synchronize { @document = document }
+        reload_document
+      end
+    end
+  end
+end

data/lib/flagnetic/transport.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "uri"
+
+module Flagnetic
+  # Transport speaks the SDK API under SDK_API_PATH for one server key. Every
+  # request, the change stream included, goes through Net::HTTP, so the
+  # proxy the process is configured with (the http_proxy, https_proxy and
+  # no_proxy environment variables) applies to all of them.
+  class Transport
+    # Response is the outcome of a ruleset fetch; body is decompressed.
+    Response = Struct.new(:status, :body, keyword_init: true)
+
+    # EventsResponse is the outcome of an events POST; EventBatch classifies
+    # it. body is decompressed; retry_after is the raw header value.
+    EventsResponse = Struct.new(:status, :body, :retry_after, keyword_init: true)
+
+    attr_reader :stream_uri, :timeout
+
+    def initialize(sdk_key:, base_url:, platform:, timeout:)
+      base = URI(base_url.to_s.chomp("/"))
+      raise ArgumentError, "base_url must be an http or https URL, got #{base_url.inspect}" unless
+        base.is_a?(URI::HTTP) && base.host
+
+      @ruleset_uri = URI("#{base}#{SDK_API_PATH}/ruleset")
+      @stream_uri = URI("#{base}#{SDK_API_PATH}/stream")
+      @events_uri = URI("#{base}#{SDK_API_PATH}/events")
+      @platform = platform
+      @sdk_key = sdk_key.to_s
+      @user_agent = "flagnetic-ruby/#{VERSION} (#{platform})"
+      @timeout = timeout
+    end
+
+    # identity is this SDK's entry in an events envelope's sdk member.
+    def identity
+      { "name" => "flagnetic-ruby", "version" => VERSION, "platform" => @platform }
+    end
+
+    # post_events POSTs a gzipped events batch and returns the raw
+    # response; EventBatch classifies it. It raises on network errors and
+    # timeouts, as fetch_ruleset does. timeout bounds this one call, so a
+    # flush bounded by a budget shorter than the client's own
+    # request_timeout still returns within it.
+    def post_events(gzipped_body, timeout = @timeout)
+      request = Net::HTTP::Post.new(@events_uri)
+      headers.each { |k, v| request[k] = v }
+      request["Content-Type"] = "application/json"
+      request["Content-Encoding"] = "gzip"
+      request["Flagnetic-Schema-Version"] = SCHEMA_VERSION_MAX.to_s
+      request.body = gzipped_body
+      response = http_for(@events_uri, timeout).start { |h| h.request(request) }
+      EventsResponse.new(status: response.code.to_i, body: response.body.to_s, retry_after: response["Retry-After"])
+    end
+
+    # fetch_ruleset GETs the ruleset, revalidating the held version with
+    # If-None-Match. It raises on network errors and timeouts; Net::HTTP
+    # asks for and inflates gzip.
+    def fetch_ruleset(held_version)
+      request = Net::HTTP::Get.new(@ruleset_uri)
+      headers.each { |k, v| request[k] = v }
+      request["Accept"] = "application/json"
+      request["Flagnetic-Schema-Version"] = SCHEMA_VERSION_MAX.to_s
+      request["If-None-Match"] = %("#{held_version}") if held_version
+      http = http_for(@ruleset_uri, @timeout)
+      response = http.start { |h| h.request(request) }
+      Response.new(status: response.code.to_i, body: response.body.to_s)
+    end
+
+    # stream_request is the change stream's request; Last-Event-ID is the
+    # held version, so the platform emits only what is newer. The stream is
+    # asked for uncompressed: a compressed one would hold frames back until
+    # the compressor flushed.
+    def stream_request(held_version)
+      request = Net::HTTP::Get.new(@stream_uri)
+      headers.each { |k, v| request[k] = v }
+      request["Accept"] = "text/event-stream"
+      request["Cache-Control"] = "no-cache"
+      request["Accept-Encoding"] = "identity"
+      request["Last-Event-ID"] = held_version.to_s if held_version
+      request
+    end
+
+    # http_for builds the connection to uri's host. Net::HTTP.new takes the
+    # proxy from the environment, honouring no_proxy; retries are off
+    # because a failed request is retried by the client's backoff alone.
+    def http_for(uri, timeout)
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = uri.scheme == "https"
+      http.open_timeout = http.read_timeout = http.write_timeout = http.ssl_timeout = timeout
+      http.max_retries = 0
+      http
+    end
+
+    # inspect leaves the server key out, so a client printed in a console
+    # or attached to an error report does not disclose it.
+    def inspect = "#<#{self.class} #{@ruleset_uri.host}:#{@ruleset_uri.port}>"
+
+    private
+
+    def headers
+      { "Authorization" => "Bearer #{@sdk_key}", "User-Agent" => @user_agent }
+    end
+  end
+  private_constant :Transport
+end

data/lib/flagnetic/uuid.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require "securerandom"
+
+module Flagnetic
+  # UUID generates the event IDs of the events endpoint.
+  module UUID
+    module_function
+
+    # v7 returns a lowercase UUIDv7 whose first 48 bits are milliseconds
+    # since the epoch, so the platform reads an event's time from the ID
+    # itself and drops a retry as a duplicate.
+    def v7(time: Time.now)
+      ms = [(time.to_r * 1000).to_i, 0].max
+      timestamp = [ms].pack("Q>").byteslice(2, 6) # the top 48 bits, big-endian
+      entropy = SecureRandom.random_bytes(10)
+      version_and_rand_a = ((entropy.getbyte(0) & 0x0f) | 0x70).chr
+      variant_and_rand_b = ((entropy.getbyte(1) & 0x3f) | 0x80).chr
+      bytes = timestamp + version_and_rand_a + entropy.byteslice(2, 1) + variant_and_rand_b + entropy.byteslice(3, 7)
+      hex = bytes.unpack1("H*")
+      "#{hex[0, 8]}-#{hex[8, 4]}-#{hex[12, 4]}-#{hex[16, 4]}-#{hex[20, 12]}"
+    end
+  end
+  private_constant :UUID
+end

data/lib/flagnetic/version.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Flagnetic
+  VERSION = "0.1.0"
+
+  # The schema_version range of the documents this SDK reads and the SDK API
+  # path it speaks. A ruleset outside the range is a fallback chain failure,
+  # logged once as schema_unsupported; every ruleset and events request asks
+  # for SCHEMA_VERSION_MAX in Flagnetic-Schema-Version (the change stream
+  # carries no document). A change to any of these needs a row in
+  # docs/sdks/compatibility.md, and a new SDK_API_PATH is a new major
+  # VERSION.
+  SCHEMA_VERSION_MIN = 1
+  SCHEMA_VERSION_MAX = 1
+  SDK_API_PATH = "/sdk/v1"
+end