quonfig 1.8.0 → 1.10.0

data/lib/quonfig/client.rb

@@ -21,6 +21,12 @@ module Quonfig
   class Client
     LOG = Quonfig::InternalLogger.new(self)
 
+    # Init fetch loop cadence (qfg-9p4i.10; init-failure design B.3, C).
+    INIT_PROGRESS_EVERY_S = 5
+    INIT_WAIT_PROGRESS_EVERY_S = 30
+    INIT_BACKOFF_CAP_S = 30.0
+    private_constant :INIT_PROGRESS_EVERY_S, :INIT_WAIT_PROGRESS_EVERY_S, :INIT_BACKOFF_CAP_S
+
     # Auth-failed probe period bounds, rule R3 (qfg-goi1.10.26).
     AUTH_PROBE_MIN_SECONDS = 150.0
     AUTH_PROBE_MAX_SECONDS = 300.0
@@ -62,6 +68,7 @@ module Quonfig
           Quonfig::Options.new(option_kwargs)
         end
       Quonfig::InternalLogger.user_logger = @options.logger if @options.logger
+      warn_on_init_failure_spelling(@options)
       @global_context = build_initial_global_context(@options)
       @instance_hash = SecureRandom.uuid
       @store = store || Quonfig::ConfigStore.new
@@ -129,7 +136,8 @@ module Quonfig
 
       # Register only for non-store-injected clients (a caller-supplied store
       # is the test/bootstrap path; the fork hook does not apply there).
-      self.class.register_instance(self) unless store
+      # Always the base class registry: a subclass has no @instances of its own.
+      Quonfig::Client.register_instance(self) unless store
     end
 
     # ---- Lookup --------------------------------------------------------
@@ -621,6 +629,33 @@ module Quonfig
       !last_successful_refresh.nil?
     end
 
+    # Block until the client has config (see #ready?), up to +timeout_ms+ (nil
+    # = no bound). Returns true once ready, false if the timeout ran out or the
+    # client was stopped first (qfg-9p4i.10; init-failure design B.4). Never
+    # raises for an outage: under on_init_failure: :continue this is how a
+    # caller waits for the background retry to heal.
+    #
+    # In a forked child that has not used the client yet, this runs the
+    # child's re-initialization (as any first use does) and keeps waiting if
+    # that fails.
+    def wait_for_ready(timeout_ms: nil)
+      deadline = timeout_ms && (monotonic_now + (timeout_ms / 1000.0))
+      begin
+        ensure_initialized_after_fork
+      rescue Quonfig::Errors::InitializationError
+        nil
+      end
+      loop do
+        return true if ready?
+        return false if @stopped
+        return false if deadline && monotonic_now >= deadline
+
+        sleep WAIT_FOR_READY_POLL_S
+      end
+    end
+
+    WAIT_FOR_READY_POLL_S = 0.05
+
     # Meta.generation of the currently-held envelope (0 before the first install
     # or when the backend does not emit a generation). Canonical ordering: an
     # established client never regresses to a lower generation.
@@ -794,6 +829,10 @@ module Quonfig
 
       cancel_fallback_engage_timer
 
+      # Sleep-only thread with no I/O and nothing to drain: killing it is safe.
+      @not_ready_reporter&.kill
+      @not_ready_reporter = nil
+
       begin
         @poll_supervisor&.stop
       rescue StandardError => e
@@ -829,6 +868,8 @@ module Quonfig
       @telemetry_reporter = nil
       @datadir_watcher = nil
       @fallback_engage_timer = nil
+      @not_ready_reporter = nil
+      @init_heal_pending = false
 
       # Dropping our reference is not enough for the reporter: its
       # `Kernel.at_exit { final_drain_on_exit }` closure is process-wide, it
@@ -939,9 +980,9 @@ module Quonfig
         end
       end
 
-      return unless @options.on_init_failure == Quonfig::Options::ON_INITIALIZATION_FAILURE::RAISE
+      return unless @options.on_init_failure == Quonfig::Options::ON_INITIALIZATION_FAILURE::FAIL
 
-      # Parity with a fresh Client.new, which raises under :raise. Stored so
+      # Parity with a fresh Client.new, which raises under :fail. Stored so
       # later calls keep raising rather than re-running the fetch on every
       # lookup.
       @fork_rebuild_error = err
@@ -1049,7 +1090,18 @@ module Quonfig
     # path that hands an envelope to the cache: datadir load, initial HTTP
     # fetch, SSE event apply, and polling worker fetch.
     def record_refresh!
-      @state_mutex.synchronize { @last_successful_refresh = Time.now.utc }
+      healed_from = @state_mutex.synchronize do
+        @last_successful_refresh = Time.now.utc
+        next nil unless @init_heal_pending
+
+        # First install after an init that continued without config. Cleared
+        # under the lock so racing installers log the heal exactly once.
+        @init_heal_pending = false
+        @init_started_at || monotonic_now
+      end
+      # The section C heal line, at WARN so the filter level that saw the
+      # failure also sees the recovery. Logged outside the lock.
+      log_init_recovered(healed_from) if healed_from
     end
 
     def sse_restart_total
@@ -1090,6 +1142,10 @@ module Quonfig
       @sse_error_callback ||= ->(error) { handle_sse_error(error) }
     end
 
+    def sse_connected_callback
+      @sse_connected_callback ||= -> { handle_sse_state_change(:connected) unless @stopped }
+    end
+
     # Every SSE error, auth-class included, is an ordinary drop: SSE alone
     # never enters auth-failed (rule R2, qfg-goi1.10.26; this reverses the
     # qfg-i5xv latch). The fallback poller engages on its usual schedule and
@@ -1273,10 +1329,8 @@ module Quonfig
     # propagates through the auto-reload path. Records a refresh timestamp.
     # Caller is responsible for firing on_update.
     def apply_datadir_envelope(envelope)
-      new_keys = envelope.configs.map { |cfg| cfg['key'] }.compact.to_set
-      old_keys = @store.keys.to_set
-      (old_keys - new_keys).each { |k| @store.delete(k) }
-      envelope.configs.each { |cfg| @store.set(cfg['key'], cfg) }
+      # One atomic swap, like a network install (qfg-i86q).
+      @store.replace_all(envelope.configs.to_h { |cfg| [cfg['key'], cfg] })
       # qfg-pinh: evaluate against the installed envelope's meta.environment,
       # matching sdk-go. In datadir mode the loader stamps meta.environment =
       # the resolved env (the `environment:` pin or QUONFIG_ENVIRONMENT), so
@@ -1336,6 +1390,7 @@ module Quonfig
       warn_if_hedge_abort_exceeds_init_timeout
       warn_if_explicit_api_urls_disables_failover
 
+      @network_mode = true
       # ||=: after a fork the child already built its loader over its fresh
       # store (see #reset_store_in_child!).
       @config_loader ||= Quonfig::ConfigLoader.new(@store, @options, failover_aggregator: @failover_aggregator)
@@ -1365,29 +1420,285 @@ module Quonfig
       start_polling if @options.enable_polling && !sse_started
     end
 
+    # The init call's fetch loop (qfg-9p4i.10; init-failure design B.3 and the
+    # section C log contract). Attempts the hedged primary+secondary fetch, and
+    # between failed attempts sleeps a jittered 0.5s, 1s, 2s ... (30s cap) until
+    # one installs. At init_timeout_ms, on_init_failure decides:
+    #
+    # - :fail     raise InitializationError (reason :unavailable), or
+    #             InitializationTimeoutError when an attempt was still in flight
+    # - :wait     keep going with no upper bound
+    # - :continue return over the empty store; the update channel started by
+    #             the caller keeps retrying and heals it
+    #
+    # Every leg answering an auth-class status (401/403/404/410) short-circuits
+    # :fail and :continue at once; :wait logs it and keeps retrying (the key
+    # may be provisioned late).
+    #
+    # init_timeout_ms <= 0 runs exactly one attempt, lets it finish (bounded by
+    # the per-leg fetch timeouts, not the init deadline), then applies
+    # on_init_failure. :wait keeps retrying after it (decision 8, qfg-9p4i.17).
     def perform_initial_fetch
-      timeout = (@options.init_timeout_ms || 10_000) / 1000.0
-      result = :failed
+      mode = @options.on_init_failure
+      timeout_ms = @options.init_timeout_ms || Quonfig::Options::DEFAULT_INIT_TIMEOUT_MS
+      timeout_ms = 0 if timeout_ms.negative?
+      one_shot = timeout_ms.zero?
+      timeout_s = timeout_ms / 1000.0
+      start = monotonic_now
+      @init_started_at = start
+      deadline = start + timeout_s
+      urls = Array(@options.config_api_urls)
+      LOG.debug "[quonfig] init: fetching config (initTimeout=#{format_secs(timeout_s)}, onInitFailure=#{mode})",
+                event: 'quonfig.init.start'
+
+      waiting = mode == Quonfig::Options::ON_INITIALIZATION_FAILURE::WAIT
+      attempt = 0
+      errs = ''
+      next_progress = start + init_progress_every_s
+      timed_out_logged = false
+
+      # The section C progress lines, on their own cadence: called after every
+      # attempt and at each wake-up inside the backoff sleep.
+      report = lambda do |now|
+        if waiting && !timed_out_logged && now >= deadline
+          timed_out_logged = true
+          next_progress = now + INIT_WAIT_PROGRESS_EVERY_S
+          LOG.warn "[quonfig] init: no config after #{format('%.1f', now - start)}s. onInitFailure=wait: still " \
+                   "blocking startup and retrying indefinitely. #{errs}. Use onInitFailure=fail or continue to bound startup.",
+                   event: 'quonfig.init.wait'
+        end
+        next unless now >= next_progress
+        next unless now < deadline || waiting
 
-      begin
-        Timeout.timeout(timeout) do
-          result = @config_loader.fetch!
+        if now < deadline
+          LOG.warn "[quonfig] init: still waiting for config (#{format('%.1f', now - start)}s elapsed of " \
+                   "#{format_secs(timeout_s)}, attempt #{attempt}). #{errs}", event: 'quonfig.init.waiting'
+          next_progress += init_progress_every_s while next_progress <= now
+        else
+          LOG.warn "[quonfig] init: still blocked waiting for config (#{(now - start).round}s elapsed, " \
+                   "attempt #{attempt}). #{errs}", event: 'quonfig.init.waiting'
+          next_progress += INIT_WAIT_PROGRESS_EVERY_S while next_progress <= now
         end
-      rescue Timeout::Error
-        handle_init_failure(
-          Quonfig::Errors::InitializationTimeoutError.new(timeout, nil)
-        )
-        return
       end
 
-      if result == :failed
-        handle_init_failure(RuntimeError.new('Config fetch failed against all api_urls'))
-      else
-        sync_evaluator_env_id!
-        record_refresh!
+      loop do
+        return if @stopped
+
+        attempt += 1
+        cycle = nil
+        cycle_log = @config_loader.new_cycle_log
+        leg_errors = Concurrent::Map.new
+        begin
+          if waiting || one_shot
+            cycle = @config_loader.fetch_cycle!(cycle_log: cycle_log, leg_errors: leg_errors)
+          else
+            # Never 0: Timeout.timeout(0) means "no timeout".
+            budget = [deadline - monotonic_now, 0.001].max
+            Timeout.timeout(budget) { cycle = @config_loader.fetch_cycle!(cycle_log: cycle_log, leg_errors: leg_errors) }
+          end
+        rescue Timeout::Error
+          # An attempt was still in flight at init_timeout_ms. Its legs run on
+          # until the hedge abort; their late lines go to DEBUG. Keep the leg
+          # errors already known: a leg that failed fast means the attempt did
+          # not hang, so the reason is :unavailable (decision 7).
+          cycle_log.silence!
+          errs = format_leg_errors(urls, leg_errors, in_flight: true)
+          return init_timed_out(mode, timeout_s, errs, hung: attempt_hung?(urls, leg_errors))
+        end
+
+        if cycle.result != :failed
+          log_init_recovered(start, attempt) if attempt > 1
+          sync_evaluator_env_id!
+          record_refresh!
+          return
+        end
+
+        errs = format_leg_errors(urls, cycle.leg_errors)
+        last_hung = attempt_hung?(urls, cycle.leg_errors)
+        now = monotonic_now
+        if cycle.all_legs_unauthorized?(urls.length)
+          # Every leg is auth-class here (decision 6); report the primary's.
+          status = cycle.primary_status
+          legs = leg_names_phrase(urls.length)
+          if waiting
+            LOG.error "[quonfig] init: SDK key rejected (#{status}) by #{legs}. Check the key and " \
+                      "environment. onInitFailure=wait: still retrying. #{errs}", event: 'quonfig.init.unauthorized'
+          else
+            msg = "[quonfig] init FAILED: SDK key rejected (#{status}) by #{legs}. Check the key " \
+                  "and environment. onInitFailure=#{mode}: " \
+                  "#{mode == Quonfig::Options::ON_INITIALIZATION_FAILURE::FAIL ? 'raising now' : 'starting on code defaults now'}."
+            LOG.error msg, event: 'quonfig.init.unauthorized'
+            return init_continue_on_defaults(errs) if mode == Quonfig::Options::ON_INITIALIZATION_FAILURE::CONTINUE
+
+            raise Quonfig::Errors::InitializationError.new(
+              "Quonfig init failed: SDK key rejected (#{status}) by #{legs}. #{errs}",
+              reason: :unauthorized, errors: errs
+            )
+          end
+        elsif attempt == 1 && !one_shot
+          LOG.warn "[quonfig] init: config not loaded yet, retrying until initTimeout (#{format_secs(timeout_s)}). #{errs}",
+                   event: 'quonfig.init.retrying'
+        end
+
+        if !waiting && now >= deadline
+          # A one-shot attempt reports how long it actually took.
+          elapsed_s = one_shot ? now - start : timeout_s
+          return init_timed_out(mode, elapsed_s, errs, hung: last_hung)
+        end
+
+        report.call(now)
+
+        # Decision 10 (sdk-go's rule): a new attempt starts whenever time
+        # remains before the deadline, however little. If the deadline cuts it
+        # off, it is the last attempt and decides the reason (the rescue
+        # above); if the deadline lands in this backoff, this attempt decides.
+        wake = now + init_backoff_s(attempt)
+        wake = deadline if !waiting && wake > deadline
+        wake_points = -> { [next_progress, (deadline if waiting && !timed_out_logged)].compact }
+        init_sleep_until(wake, wake_points) { |t| report.call(t) }
+        return init_timed_out(mode, timeout_s, errs, hung: last_hung) if !waiting && monotonic_now >= deadline
+      end
+    end
+
+    # Sleep until +wake+, waking early at each of +wake_points+ (re-read every
+    # time) to yield the current time, so progress lines keep their cadence
+    # through a long backoff. Returns early once the client is stopped.
+    def init_sleep_until(wake, wake_points)
+      loop do
+        return if @stopped
+
+        now = monotonic_now
+        return if now >= wake
+
+        nxt = (wake_points.call.select { |t| t > now } << wake).min
+        sleep(nxt - now)
+        yield monotonic_now
       end
     end
 
+    # Seconds between "still waiting for config" lines before the timeout.
+    def init_progress_every_s
+      INIT_PROGRESS_EVERY_S
+    end
+
+    # "primary", "primary and secondary", "primary, secondary and url2".
+    def leg_names_phrase(count)
+      names = Array.new([count, 1].max) { |i| leg_label(i) }
+      names.length == 1 ? names.first : "#{names[0..-2].join(', ')} and #{names.last}"
+    end
+
+    def leg_label(index)
+      case index
+      when 0 then 'primary'
+      when 1 then 'secondary'
+      else "url#{index}"
+      end
+    end
+
+    # 0.5s, 1s, 2s, 4s ... capped at 30s, jittered +/-20%.
+    def init_backoff_s(attempt)
+      base = [0.5 * (2**(attempt - 1)), INIT_BACKOFF_CAP_S].min
+      base * (0.8 + (rand * 0.4))
+    end
+
+    def init_timed_out(mode, timeout_s, errs, hung:)
+      elapsed = format('%.1f', timeout_s)
+      if mode == Quonfig::Options::ON_INITIALIZATION_FAILURE::CONTINUE
+        LOG.error "[quonfig] init: no config after #{elapsed}s. onInitFailure=continue: STARTING ON CODE DEFAULTS. " \
+                  'This instance may disagree with the rest of your fleet. Retrying in background.',
+                  event: 'quonfig.init.continue'
+        return init_continue_on_defaults(errs)
+      end
+
+      LOG.error "[quonfig] init FAILED: no config after #{elapsed}s. #{errs}. onInitFailure=fail: raising " \
+                'Quonfig::Errors::InitializationError. Options: on_init_failure: :wait to keep waiting, ' \
+                'on_init_failure: :continue to start on code defaults, or raise init_timeout_ms.',
+                event: 'quonfig.init.failed'
+      message = "Quonfig init failed: no config after #{elapsed}s. #{errs}"
+      raise Quonfig::Errors::InitializationTimeoutError.new(timeout_s, nil, message: message, errors: errs) if hung
+
+      raise Quonfig::Errors::InitializationError.new(message, reason: :unavailable, errors: errs)
+    end
+
+    # :continue over an empty store. The update channel the caller starts next
+    # keeps retrying; #record_refresh! logs the heal, and the not-ready reporter
+    # logs the outage on the section C cadence until then.
+    def init_continue_on_defaults(errs)
+      @state_mutex.synchronize do
+        @init_heal_pending = true
+        @init_last_errs = errs
+      end
+      start_not_ready_reporter
+      nil
+    end
+
+    # WARN every 30s for 5 minutes, then every 5 minutes, while a :continue
+    # client has no config (section C, "Background retry while not ready").
+    def start_not_ready_reporter
+      return if @not_ready_reporter&.alive?
+
+      @not_ready_reporter = Thread.new do
+        Thread.current.name = 'quonfig-not-ready-reporter' if Thread.current.respond_to?(:name=)
+        loop do
+          elapsed = monotonic_now - @init_started_at
+          sleep(elapsed < 300 ? 30 : 300)
+          break if @stopped || ready?
+
+          since = (monotonic_now - @init_started_at).round
+          errs = @state_mutex.synchronize { @init_last_errs }
+          LOG.warn "[quonfig] still not connected to Quonfig delivery (#{since / 60}m#{since % 60}s since start); " \
+                   "serving defaults. #{errs}", event: 'quonfig.init.not_connected'
+        end
+      rescue StandardError
+        nil
+      end
+    end
+
+    def log_init_recovered(start, _attempt = nil)
+      via = @config_loader.respond_to?(:resolved_from) ? @config_loader.resolved_from : ''
+      # A stream install leaves resolved_from empty: name the SSE leg.
+      via = sse_failed_over_to_secondary? ? leg_label(1) : leg_label(0) if via.to_s.empty?
+      version = @config_loader.respond_to?(:version) ? @config_loader.version : nil
+      LOG.warn "[quonfig] init recovered: config loaded after #{format('%.1f', monotonic_now - start)}s via #{via}" \
+               "#{" (version #{version})" if version}. Now serving live config.", event: 'quonfig.init.recovered'
+    end
+
+    # Whether an attempt hung (decision 7): no leg failed fast. Every leg that
+    # settled ended on a timeout, and a leg with no error yet was still in
+    # flight when the init deadline cut the attempt off.
+    def attempt_hung?(urls, leg_errors)
+      urls.each_index.all? do |i|
+        err = leg_errors[i]
+        err.nil? || err == Quonfig::ConfigLoader::LEG_TIMED_OUT
+      end
+    end
+
+    # primary=<host>: <error>; secondary=<host>: <error>. Hosts only, never the
+    # SDK key or URL credentials.
+    def format_leg_errors(urls, leg_errors, in_flight: false)
+      urls.each_with_index.map do |url, i|
+        label = leg_label(i)
+        err = leg_errors[i] || (in_flight ? 'no response before initTimeout' : 'no response')
+        "#{label}=#{safe_host(url)}: #{err}"
+      end.join('; ')
+    end
+
+    def safe_host(url)
+      uri = URI.parse(url.to_s)
+      uri.host ? "#{uri.host}:#{uri.port}" : '?'
+    rescue URI::InvalidURIError
+      '?'
+    end
+
+    # "10s" for whole seconds, "0.5s" otherwise.
+    def format_secs(secs)
+      secs == secs.to_i ? "#{secs.to_i}s" : "#{secs}s"
+    end
+
+    def monotonic_now
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+
     # qfg-pinh: In SDK-key DELIVERY mode the server's `meta.environment` is
     # AUTHORITATIVE. The server scopes each config to a single environment and
     # reports the active env id in `meta.environment` (the loader captures it
@@ -1429,6 +1740,26 @@ module Quonfig
       )
     end
 
+    DEPRECATED_INIT_FAILURE_WARNED = Set.new
+    DEPRECATED_INIT_FAILURE_MUTEX = Mutex.new
+    private_constant :DEPRECATED_INIT_FAILURE_WARNED, :DEPRECATED_INIT_FAILURE_MUTEX
+
+    # One WARN per process per deprecated spelling (section C), and a WARN for
+    # an unrecognized value, which runs as :fail.
+    def warn_on_init_failure_spelling(options)
+      if (bad = options.on_init_failure_unknown)
+        LOG.warn "[quonfig] unknown on_init_failure #{bad.inspect}; using :fail (expected :fail, :wait or :continue)"
+      end
+      old = options.on_init_failure_alias
+      return if old.nil?
+
+      first = DEPRECATED_INIT_FAILURE_MUTEX.synchronize { DEPRECATED_INIT_FAILURE_WARNED.add?(old) }
+      return unless first
+
+      LOG.warn "[quonfig] on_init_failure=:#{old} is deprecated; use :#{options.on_init_failure}",
+               event: 'quonfig.init.deprecated_alias'
+    end
+
     # qfg-7h5d.1.14: the per-leg hedge abort MUST be < init_timeout_ms, otherwise
     # the init-path heal leg is clipped by the overall init deadline before it can
     # heal forward. Mirrors sdk-go's construction-time warning in options.go. Warn
@@ -1443,6 +1774,8 @@ module Quonfig
       abort_ms = @options.config_fetch_hedge_abort_ms
       init_ms = @options.init_timeout_ms
       return if abort_ms.nil? || init_ms.nil?
+      # <= 0 runs one attempt to completion; no init deadline clips its legs.
+      return if init_ms <= 0
       return if init_ms > abort_ms
 
       LOG.warn(
@@ -1474,15 +1807,6 @@ module Quonfig
       )
     end
 
-    def handle_init_failure(err)
-      if @options.on_init_failure == Quonfig::Options::ON_INITIALIZATION_FAILURE::RETURN
-        LOG.warn "[quonfig] Initialization did not complete cleanly; continuing with empty store: #{err.message}"
-        return
-      end
-
-      raise err
-    end
-
     # Returns true if SSE started successfully, false otherwise. A false here
     # signals the caller to fall back to polling.
     def start_sse
@@ -1494,7 +1818,12 @@ module Quonfig
         @config_loader,
         nil,
         nil,
-        on_error: sse_error_callback
+        on_error: sse_error_callback,
+        # First-frame rule (qfg-goi1.10.43): connected at the connection's
+        # first valid frame, keepalive included, never on the 200 headers.
+        # Cancels the engage timer / disengages the poller without stamping
+        # last_successful_refresh (only an install or fetch does that).
+        on_connected: sse_connected_callback
       )
       @sse_client.start do |envelope, _event, _source|
         next if @stopped
@@ -1755,6 +2084,25 @@ module Quonfig
       nil
     end
 
+    # A delivery-mode client that has never installed config (init continued
+    # on code defaults, or the caller caught a failed init): the *_details
+    # getters answer ERROR / PROVIDER_NOT_READY (init-failure design B.3)
+    # rather than claiming the key does not exist.
+    def provider_not_ready?
+      @network_mode && !ready?
+    end
+
+    def provider_not_ready_details
+      Quonfig::EvaluationDetails.new(
+        value: nil,
+        reason: Quonfig::EvaluationDetails::REASON_ERROR,
+        error_code: Quonfig::EvaluationDetails::ERROR_PROVIDER_NOT_READY,
+        error_message: 'Quonfig has not loaded config yet',
+        variant: build_variant(Quonfig::EvaluationDetails::REASON_ERROR, nil, nil),
+        flag_metadata: build_flag_metadata(nil, nil, nil, nil, nil)
+      )
+    end
+
     # Build a Quonfig::EvaluationDetails for +key+, evaluated against the
     # caller's context, after coercing/checking +expected_type+. Never
     # raises; all exceptions become ERROR details.
@@ -1763,6 +2111,7 @@ module Quonfig
       jit = context == NO_DEFAULT_PROVIDED ? nil : context
       ctx = build_context(jit)
       record_context_for_telemetry(ctx)
+      return provider_not_ready_details if provider_not_ready?
 
       result =
         begin