quonfig 1.9.0 → 1.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +36 -0
- data/README.md +48 -3
- data/lib/quonfig/client.rb +406 -29
- data/lib/quonfig/config_loader.rb +137 -30
- data/lib/quonfig/dev_context.rb +8 -1
- data/lib/quonfig/errors/initialization_error.rb +31 -0
- data/lib/quonfig/errors/initialization_timeout_error.rb +9 -4
- data/lib/quonfig/evaluation_details.rb +3 -0
- data/lib/quonfig/http_connection.rb +118 -3
- data/lib/quonfig/internal_logger.rb +21 -14
- data/lib/quonfig/options.rb +50 -2
- data/lib/quonfig/sse_config_client.rb +62 -3
- data/lib/quonfig/version.rb +1 -1
- data/lib/quonfig.rb +1 -0
- metadata +3 -2
data/lib/quonfig/client.rb
CHANGED
|
@@ -21,6 +21,18 @@ module Quonfig
|
|
|
21
21
|
class Client
|
|
22
22
|
LOG = Quonfig::InternalLogger.new(self)
|
|
23
23
|
|
|
24
|
+
# The +callback+ label values of quonfig_sdk_callback_errors_total
|
|
25
|
+
# (supervisor contract). sdk-ruby has no public connection-state callback,
|
|
26
|
+
# so +on_connection_state_change+ stays 0; it is listed so a reader of
|
|
27
|
+
# either name gets the contract's answer.
|
|
28
|
+
CALLBACK_NAMES = %w[on_config_update on_connection_state_change].freeze
|
|
29
|
+
|
|
30
|
+
# Init fetch loop cadence (qfg-9p4i.10; init-failure design B.3, C).
|
|
31
|
+
INIT_PROGRESS_EVERY_S = 5
|
|
32
|
+
INIT_WAIT_PROGRESS_EVERY_S = 30
|
|
33
|
+
INIT_BACKOFF_CAP_S = 30.0
|
|
34
|
+
private_constant :INIT_PROGRESS_EVERY_S, :INIT_WAIT_PROGRESS_EVERY_S, :INIT_BACKOFF_CAP_S
|
|
35
|
+
|
|
24
36
|
# Auth-failed probe period bounds, rule R3 (qfg-goi1.10.26).
|
|
25
37
|
AUTH_PROBE_MIN_SECONDS = 150.0
|
|
26
38
|
AUTH_PROBE_MAX_SECONDS = 300.0
|
|
@@ -62,6 +74,7 @@ module Quonfig
|
|
|
62
74
|
Quonfig::Options.new(option_kwargs)
|
|
63
75
|
end
|
|
64
76
|
Quonfig::InternalLogger.user_logger = @options.logger if @options.logger
|
|
77
|
+
warn_on_init_failure_spelling(@options)
|
|
65
78
|
@global_context = build_initial_global_context(@options)
|
|
66
79
|
@instance_hash = SecureRandom.uuid
|
|
67
80
|
@store = store || Quonfig::ConfigStore.new
|
|
@@ -90,6 +103,9 @@ module Quonfig
|
|
|
90
103
|
@sse_state = :idle
|
|
91
104
|
@sse_ever_connected = false
|
|
92
105
|
@fallback_engage_timer = nil
|
|
106
|
+
# quonfig_sdk_callback_errors_total{callback} (supervisor contract Test 5,
|
|
107
|
+
# qfg-6vno.6.1). Guarded by @state_mutex. See #callback_errors_total.
|
|
108
|
+
@callback_errors = CALLBACK_NAMES.to_h { |name| [name, 0] }
|
|
93
109
|
# Auth-failed regime (rule R2-R4, qfg-goi1.10.26). Set by the poller's
|
|
94
110
|
# fetch cycle on entry, cleared by the first accepted probe. While set:
|
|
95
111
|
# no SSE, no polling, one probe per [150s, 300s]. Read by
|
|
@@ -129,7 +145,8 @@ module Quonfig
|
|
|
129
145
|
|
|
130
146
|
# Register only for non-store-injected clients (a caller-supplied store
|
|
131
147
|
# is the test/bootstrap path; the fork hook does not apply there).
|
|
132
|
-
|
|
148
|
+
# Always the base class registry: a subclass has no @instances of its own.
|
|
149
|
+
Quonfig::Client.register_instance(self) unless store
|
|
133
150
|
end
|
|
134
151
|
|
|
135
152
|
# ---- Lookup --------------------------------------------------------
|
|
@@ -553,6 +570,29 @@ module Quonfig
|
|
|
553
570
|
end
|
|
554
571
|
end
|
|
555
572
|
|
|
573
|
+
# quonfig_sdk_callback_errors_total counter (supervisor contract Test 5,
|
|
574
|
+
# qfg-6vno.6.1). Counts each call of your +on_update+ block that raised a
|
|
575
|
+
# StandardError the SDK caught and logged at ERROR. Such a raise is not a
|
|
576
|
+
# worker restart: #worker_restart_total does not change, the stream keeps
|
|
577
|
+
# running, and the next update still reaches your block.
|
|
578
|
+
#
|
|
579
|
+
# Pass +callback:+ ('on_config_update' or 'on_connection_state_change',
|
|
580
|
+
# String or Symbol) for one label; the default returns the sum. sdk-ruby
|
|
581
|
+
# has no connection-state callback, so 'on_connection_state_change' is
|
|
582
|
+
# always 0. An unknown name raises ArgumentError. Starts at 0 when the
|
|
583
|
+
# client is built.
|
|
584
|
+
def callback_errors_total(callback: nil)
|
|
585
|
+
@state_mutex.synchronize do
|
|
586
|
+
next @callback_errors.values.sum if callback.nil?
|
|
587
|
+
|
|
588
|
+
name = callback.to_s
|
|
589
|
+
raise ArgumentError, "unknown callback #{callback.inspect}; expected one of #{CALLBACK_NAMES.join(', ')}" unless
|
|
590
|
+
@callback_errors.key?(name)
|
|
591
|
+
|
|
592
|
+
@callback_errors[name]
|
|
593
|
+
end
|
|
594
|
+
end
|
|
595
|
+
|
|
556
596
|
# Wall-clock time of the last installed envelope (any source: datadir,
|
|
557
597
|
# initial HTTP fetch, SSE, or polling fallback). +nil+ before the first
|
|
558
598
|
# install. Preserved after +stop+.
|
|
@@ -621,6 +661,33 @@ module Quonfig
|
|
|
621
661
|
!last_successful_refresh.nil?
|
|
622
662
|
end
|
|
623
663
|
|
|
664
|
+
# Block until the client has config (see #ready?), up to +timeout_ms+ (nil
|
|
665
|
+
# = no bound). Returns true once ready, false if the timeout ran out or the
|
|
666
|
+
# client was stopped first (qfg-9p4i.10; init-failure design B.4). Never
|
|
667
|
+
# raises for an outage: under on_init_failure: :continue this is how a
|
|
668
|
+
# caller waits for the background retry to heal.
|
|
669
|
+
#
|
|
670
|
+
# In a forked child that has not used the client yet, this runs the
|
|
671
|
+
# child's re-initialization (as any first use does) and keeps waiting if
|
|
672
|
+
# that fails.
|
|
673
|
+
def wait_for_ready(timeout_ms: nil)
|
|
674
|
+
deadline = timeout_ms && (monotonic_now + (timeout_ms / 1000.0))
|
|
675
|
+
begin
|
|
676
|
+
ensure_initialized_after_fork
|
|
677
|
+
rescue Quonfig::Errors::InitializationError
|
|
678
|
+
nil
|
|
679
|
+
end
|
|
680
|
+
loop do
|
|
681
|
+
return true if ready?
|
|
682
|
+
return false if @stopped
|
|
683
|
+
return false if deadline && monotonic_now >= deadline
|
|
684
|
+
|
|
685
|
+
sleep WAIT_FOR_READY_POLL_S
|
|
686
|
+
end
|
|
687
|
+
end
|
|
688
|
+
|
|
689
|
+
WAIT_FOR_READY_POLL_S = 0.05
|
|
690
|
+
|
|
624
691
|
# Meta.generation of the currently-held envelope (0 before the first install
|
|
625
692
|
# or when the backend does not emit a generation). Canonical ordering: an
|
|
626
693
|
# established client never regresses to a lower generation.
|
|
@@ -794,6 +861,10 @@ module Quonfig
|
|
|
794
861
|
|
|
795
862
|
cancel_fallback_engage_timer
|
|
796
863
|
|
|
864
|
+
# Sleep-only thread with no I/O and nothing to drain: killing it is safe.
|
|
865
|
+
@not_ready_reporter&.kill
|
|
866
|
+
@not_ready_reporter = nil
|
|
867
|
+
|
|
797
868
|
begin
|
|
798
869
|
@poll_supervisor&.stop
|
|
799
870
|
rescue StandardError => e
|
|
@@ -829,6 +900,8 @@ module Quonfig
|
|
|
829
900
|
@telemetry_reporter = nil
|
|
830
901
|
@datadir_watcher = nil
|
|
831
902
|
@fallback_engage_timer = nil
|
|
903
|
+
@not_ready_reporter = nil
|
|
904
|
+
@init_heal_pending = false
|
|
832
905
|
|
|
833
906
|
# Dropping our reference is not enough for the reporter: its
|
|
834
907
|
# `Kernel.at_exit { final_drain_on_exit }` closure is process-wide, it
|
|
@@ -939,9 +1012,9 @@ module Quonfig
|
|
|
939
1012
|
end
|
|
940
1013
|
end
|
|
941
1014
|
|
|
942
|
-
return unless @options.on_init_failure == Quonfig::Options::ON_INITIALIZATION_FAILURE::
|
|
1015
|
+
return unless @options.on_init_failure == Quonfig::Options::ON_INITIALIZATION_FAILURE::FAIL
|
|
943
1016
|
|
|
944
|
-
# Parity with a fresh Client.new, which raises under :
|
|
1017
|
+
# Parity with a fresh Client.new, which raises under :fail. Stored so
|
|
945
1018
|
# later calls keep raising rather than re-running the fetch on every
|
|
946
1019
|
# lookup.
|
|
947
1020
|
@fork_rebuild_error = err
|
|
@@ -1049,7 +1122,18 @@ module Quonfig
|
|
|
1049
1122
|
# path that hands an envelope to the cache: datadir load, initial HTTP
|
|
1050
1123
|
# fetch, SSE event apply, and polling worker fetch.
|
|
1051
1124
|
def record_refresh!
|
|
1052
|
-
@state_mutex.synchronize
|
|
1125
|
+
healed_from = @state_mutex.synchronize do
|
|
1126
|
+
@last_successful_refresh = Time.now.utc
|
|
1127
|
+
next nil unless @init_heal_pending
|
|
1128
|
+
|
|
1129
|
+
# First install after an init that continued without config. Cleared
|
|
1130
|
+
# under the lock so racing installers log the heal exactly once.
|
|
1131
|
+
@init_heal_pending = false
|
|
1132
|
+
@init_started_at || monotonic_now
|
|
1133
|
+
end
|
|
1134
|
+
# The section C heal line, at WARN so the filter level that saw the
|
|
1135
|
+
# failure also sees the recovery. Logged outside the lock.
|
|
1136
|
+
log_init_recovered(healed_from) if healed_from
|
|
1053
1137
|
end
|
|
1054
1138
|
|
|
1055
1139
|
def sse_restart_total
|
|
@@ -1338,6 +1422,7 @@ module Quonfig
|
|
|
1338
1422
|
warn_if_hedge_abort_exceeds_init_timeout
|
|
1339
1423
|
warn_if_explicit_api_urls_disables_failover
|
|
1340
1424
|
|
|
1425
|
+
@network_mode = true
|
|
1341
1426
|
# ||=: after a fork the child already built its loader over its fresh
|
|
1342
1427
|
# store (see #reset_store_in_child!).
|
|
1343
1428
|
@config_loader ||= Quonfig::ConfigLoader.new(@store, @options, failover_aggregator: @failover_aggregator)
|
|
@@ -1367,29 +1452,285 @@ module Quonfig
|
|
|
1367
1452
|
start_polling if @options.enable_polling && !sse_started
|
|
1368
1453
|
end
|
|
1369
1454
|
|
|
1455
|
+
# The init call's fetch loop (qfg-9p4i.10; init-failure design B.3 and the
|
|
1456
|
+
# section C log contract). Attempts the hedged primary+secondary fetch, and
|
|
1457
|
+
# between failed attempts sleeps a jittered 0.5s, 1s, 2s ... (30s cap) until
|
|
1458
|
+
# one installs. At init_timeout_ms, on_init_failure decides:
|
|
1459
|
+
#
|
|
1460
|
+
# - :fail raise InitializationError (reason :unavailable), or
|
|
1461
|
+
# InitializationTimeoutError when an attempt was still in flight
|
|
1462
|
+
# - :wait keep going with no upper bound
|
|
1463
|
+
# - :continue return over the empty store; the update channel started by
|
|
1464
|
+
# the caller keeps retrying and heals it
|
|
1465
|
+
#
|
|
1466
|
+
# Every leg answering an auth-class status (401/403/404/410) short-circuits
|
|
1467
|
+
# :fail and :continue at once; :wait logs it and keeps retrying (the key
|
|
1468
|
+
# may be provisioned late).
|
|
1469
|
+
#
|
|
1470
|
+
# init_timeout_ms <= 0 runs exactly one attempt, lets it finish (bounded by
|
|
1471
|
+
# the per-leg fetch timeouts, not the init deadline), then applies
|
|
1472
|
+
# on_init_failure. :wait keeps retrying after it (decision 8, qfg-9p4i.17).
|
|
1370
1473
|
def perform_initial_fetch
|
|
1371
|
-
|
|
1372
|
-
|
|
1474
|
+
mode = @options.on_init_failure
|
|
1475
|
+
timeout_ms = @options.init_timeout_ms || Quonfig::Options::DEFAULT_INIT_TIMEOUT_MS
|
|
1476
|
+
timeout_ms = 0 if timeout_ms.negative?
|
|
1477
|
+
one_shot = timeout_ms.zero?
|
|
1478
|
+
timeout_s = timeout_ms / 1000.0
|
|
1479
|
+
start = monotonic_now
|
|
1480
|
+
@init_started_at = start
|
|
1481
|
+
deadline = start + timeout_s
|
|
1482
|
+
urls = Array(@options.config_api_urls)
|
|
1483
|
+
LOG.debug "[quonfig] init: fetching config (initTimeout=#{format_secs(timeout_s)}, onInitFailure=#{mode})",
|
|
1484
|
+
event: 'quonfig.init.start'
|
|
1485
|
+
|
|
1486
|
+
waiting = mode == Quonfig::Options::ON_INITIALIZATION_FAILURE::WAIT
|
|
1487
|
+
attempt = 0
|
|
1488
|
+
errs = ''
|
|
1489
|
+
next_progress = start + init_progress_every_s
|
|
1490
|
+
timed_out_logged = false
|
|
1491
|
+
|
|
1492
|
+
# The section C progress lines, on their own cadence: called after every
|
|
1493
|
+
# attempt and at each wake-up inside the backoff sleep.
|
|
1494
|
+
report = lambda do |now|
|
|
1495
|
+
if waiting && !timed_out_logged && now >= deadline
|
|
1496
|
+
timed_out_logged = true
|
|
1497
|
+
next_progress = now + INIT_WAIT_PROGRESS_EVERY_S
|
|
1498
|
+
LOG.warn "[quonfig] init: no config after #{format('%.1f', now - start)}s. onInitFailure=wait: still " \
|
|
1499
|
+
"blocking startup and retrying indefinitely. #{errs}. Use onInitFailure=fail or continue to bound startup.",
|
|
1500
|
+
event: 'quonfig.init.wait'
|
|
1501
|
+
end
|
|
1502
|
+
next unless now >= next_progress
|
|
1503
|
+
next unless now < deadline || waiting
|
|
1373
1504
|
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1505
|
+
if now < deadline
|
|
1506
|
+
LOG.warn "[quonfig] init: still waiting for config (#{format('%.1f', now - start)}s elapsed of " \
|
|
1507
|
+
"#{format_secs(timeout_s)}, attempt #{attempt}). #{errs}", event: 'quonfig.init.waiting'
|
|
1508
|
+
next_progress += init_progress_every_s while next_progress <= now
|
|
1509
|
+
else
|
|
1510
|
+
LOG.warn "[quonfig] init: still blocked waiting for config (#{(now - start).round}s elapsed, " \
|
|
1511
|
+
"attempt #{attempt}). #{errs}", event: 'quonfig.init.waiting'
|
|
1512
|
+
next_progress += INIT_WAIT_PROGRESS_EVERY_S while next_progress <= now
|
|
1377
1513
|
end
|
|
1378
|
-
rescue Timeout::Error
|
|
1379
|
-
handle_init_failure(
|
|
1380
|
-
Quonfig::Errors::InitializationTimeoutError.new(timeout, nil)
|
|
1381
|
-
)
|
|
1382
|
-
return
|
|
1383
1514
|
end
|
|
1384
1515
|
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1516
|
+
loop do
|
|
1517
|
+
return if @stopped
|
|
1518
|
+
|
|
1519
|
+
attempt += 1
|
|
1520
|
+
cycle = nil
|
|
1521
|
+
cycle_log = @config_loader.new_cycle_log
|
|
1522
|
+
leg_errors = Concurrent::Map.new
|
|
1523
|
+
begin
|
|
1524
|
+
if waiting || one_shot
|
|
1525
|
+
cycle = @config_loader.fetch_cycle!(cycle_log: cycle_log, leg_errors: leg_errors)
|
|
1526
|
+
else
|
|
1527
|
+
# Never 0: Timeout.timeout(0) means "no timeout".
|
|
1528
|
+
budget = [deadline - monotonic_now, 0.001].max
|
|
1529
|
+
Timeout.timeout(budget) { cycle = @config_loader.fetch_cycle!(cycle_log: cycle_log, leg_errors: leg_errors) }
|
|
1530
|
+
end
|
|
1531
|
+
rescue Timeout::Error
|
|
1532
|
+
# An attempt was still in flight at init_timeout_ms. Its legs run on
|
|
1533
|
+
# until the hedge abort; their late lines go to DEBUG. Keep the leg
|
|
1534
|
+
# errors already known: a leg that failed fast means the attempt did
|
|
1535
|
+
# not hang, so the reason is :unavailable (decision 7).
|
|
1536
|
+
cycle_log.silence!
|
|
1537
|
+
errs = format_leg_errors(urls, leg_errors, in_flight: true)
|
|
1538
|
+
return init_timed_out(mode, timeout_s, errs, hung: attempt_hung?(urls, leg_errors))
|
|
1539
|
+
end
|
|
1540
|
+
|
|
1541
|
+
if cycle.result != :failed
|
|
1542
|
+
log_init_recovered(start, attempt) if attempt > 1
|
|
1543
|
+
sync_evaluator_env_id!
|
|
1544
|
+
record_refresh!
|
|
1545
|
+
return
|
|
1546
|
+
end
|
|
1547
|
+
|
|
1548
|
+
errs = format_leg_errors(urls, cycle.leg_errors)
|
|
1549
|
+
last_hung = attempt_hung?(urls, cycle.leg_errors)
|
|
1550
|
+
now = monotonic_now
|
|
1551
|
+
if cycle.all_legs_unauthorized?(urls.length)
|
|
1552
|
+
# Every leg is auth-class here (decision 6); report the primary's.
|
|
1553
|
+
status = cycle.primary_status
|
|
1554
|
+
legs = leg_names_phrase(urls.length)
|
|
1555
|
+
if waiting
|
|
1556
|
+
LOG.error "[quonfig] init: SDK key rejected (#{status}) by #{legs}. Check the key and " \
|
|
1557
|
+
"environment. onInitFailure=wait: still retrying. #{errs}", event: 'quonfig.init.unauthorized'
|
|
1558
|
+
else
|
|
1559
|
+
msg = "[quonfig] init FAILED: SDK key rejected (#{status}) by #{legs}. Check the key " \
|
|
1560
|
+
"and environment. onInitFailure=#{mode}: " \
|
|
1561
|
+
"#{mode == Quonfig::Options::ON_INITIALIZATION_FAILURE::FAIL ? 'raising now' : 'starting on code defaults now'}."
|
|
1562
|
+
LOG.error msg, event: 'quonfig.init.unauthorized'
|
|
1563
|
+
return init_continue_on_defaults(errs) if mode == Quonfig::Options::ON_INITIALIZATION_FAILURE::CONTINUE
|
|
1564
|
+
|
|
1565
|
+
raise Quonfig::Errors::InitializationError.new(
|
|
1566
|
+
"Quonfig init failed: SDK key rejected (#{status}) by #{legs}. #{errs}",
|
|
1567
|
+
reason: :unauthorized, errors: errs
|
|
1568
|
+
)
|
|
1569
|
+
end
|
|
1570
|
+
elsif attempt == 1 && !one_shot
|
|
1571
|
+
LOG.warn "[quonfig] init: config not loaded yet, retrying until initTimeout (#{format_secs(timeout_s)}). #{errs}",
|
|
1572
|
+
event: 'quonfig.init.retrying'
|
|
1573
|
+
end
|
|
1574
|
+
|
|
1575
|
+
if !waiting && now >= deadline
|
|
1576
|
+
# A one-shot attempt reports how long it actually took.
|
|
1577
|
+
elapsed_s = one_shot ? now - start : timeout_s
|
|
1578
|
+
return init_timed_out(mode, elapsed_s, errs, hung: last_hung)
|
|
1579
|
+
end
|
|
1580
|
+
|
|
1581
|
+
report.call(now)
|
|
1582
|
+
|
|
1583
|
+
# Decision 10 (sdk-go's rule): a new attempt starts whenever time
|
|
1584
|
+
# remains before the deadline, however little. If the deadline cuts it
|
|
1585
|
+
# off, it is the last attempt and decides the reason (the rescue
|
|
1586
|
+
# above); if the deadline lands in this backoff, this attempt decides.
|
|
1587
|
+
wake = now + init_backoff_s(attempt)
|
|
1588
|
+
wake = deadline if !waiting && wake > deadline
|
|
1589
|
+
wake_points = -> { [next_progress, (deadline if waiting && !timed_out_logged)].compact }
|
|
1590
|
+
init_sleep_until(wake, wake_points) { |t| report.call(t) }
|
|
1591
|
+
return init_timed_out(mode, timeout_s, errs, hung: last_hung) if !waiting && monotonic_now >= deadline
|
|
1592
|
+
end
|
|
1593
|
+
end
|
|
1594
|
+
|
|
1595
|
+
# Sleep until +wake+, waking early at each of +wake_points+ (re-read every
|
|
1596
|
+
# time) to yield the current time, so progress lines keep their cadence
|
|
1597
|
+
# through a long backoff. Returns early once the client is stopped.
|
|
1598
|
+
def init_sleep_until(wake, wake_points)
|
|
1599
|
+
loop do
|
|
1600
|
+
return if @stopped
|
|
1601
|
+
|
|
1602
|
+
now = monotonic_now
|
|
1603
|
+
return if now >= wake
|
|
1604
|
+
|
|
1605
|
+
nxt = (wake_points.call.select { |t| t > now } << wake).min
|
|
1606
|
+
sleep(nxt - now)
|
|
1607
|
+
yield monotonic_now
|
|
1608
|
+
end
|
|
1609
|
+
end
|
|
1610
|
+
|
|
1611
|
+
# Seconds between "still waiting for config" lines before the timeout.
|
|
1612
|
+
def init_progress_every_s
|
|
1613
|
+
INIT_PROGRESS_EVERY_S
|
|
1614
|
+
end
|
|
1615
|
+
|
|
1616
|
+
# "primary", "primary and secondary", "primary, secondary and url2".
|
|
1617
|
+
def leg_names_phrase(count)
|
|
1618
|
+
names = Array.new([count, 1].max) { |i| leg_label(i) }
|
|
1619
|
+
names.length == 1 ? names.first : "#{names[0..-2].join(', ')} and #{names.last}"
|
|
1620
|
+
end
|
|
1621
|
+
|
|
1622
|
+
def leg_label(index)
|
|
1623
|
+
case index
|
|
1624
|
+
when 0 then 'primary'
|
|
1625
|
+
when 1 then 'secondary'
|
|
1626
|
+
else "url#{index}"
|
|
1627
|
+
end
|
|
1628
|
+
end
|
|
1629
|
+
|
|
1630
|
+
# 0.5s, 1s, 2s, 4s ... capped at 30s, jittered +/-20%.
|
|
1631
|
+
def init_backoff_s(attempt)
|
|
1632
|
+
base = [0.5 * (2**(attempt - 1)), INIT_BACKOFF_CAP_S].min
|
|
1633
|
+
base * (0.8 + (rand * 0.4))
|
|
1634
|
+
end
|
|
1635
|
+
|
|
1636
|
+
def init_timed_out(mode, timeout_s, errs, hung:)
|
|
1637
|
+
elapsed = format('%.1f', timeout_s)
|
|
1638
|
+
if mode == Quonfig::Options::ON_INITIALIZATION_FAILURE::CONTINUE
|
|
1639
|
+
LOG.error "[quonfig] init: no config after #{elapsed}s. onInitFailure=continue: STARTING ON CODE DEFAULTS. " \
|
|
1640
|
+
'This instance may disagree with the rest of your fleet. Retrying in background.',
|
|
1641
|
+
event: 'quonfig.init.continue'
|
|
1642
|
+
return init_continue_on_defaults(errs)
|
|
1643
|
+
end
|
|
1644
|
+
|
|
1645
|
+
LOG.error "[quonfig] init FAILED: no config after #{elapsed}s. #{errs}. onInitFailure=fail: raising " \
|
|
1646
|
+
'Quonfig::Errors::InitializationError. Options: on_init_failure: :wait to keep waiting, ' \
|
|
1647
|
+
'on_init_failure: :continue to start on code defaults, or raise init_timeout_ms.',
|
|
1648
|
+
event: 'quonfig.init.failed'
|
|
1649
|
+
message = "Quonfig init failed: no config after #{elapsed}s. #{errs}"
|
|
1650
|
+
raise Quonfig::Errors::InitializationTimeoutError.new(timeout_s, nil, message: message, errors: errs) if hung
|
|
1651
|
+
|
|
1652
|
+
raise Quonfig::Errors::InitializationError.new(message, reason: :unavailable, errors: errs)
|
|
1653
|
+
end
|
|
1654
|
+
|
|
1655
|
+
# :continue over an empty store. The update channel the caller starts next
|
|
1656
|
+
# keeps retrying; #record_refresh! logs the heal, and the not-ready reporter
|
|
1657
|
+
# logs the outage on the section C cadence until then.
|
|
1658
|
+
def init_continue_on_defaults(errs)
|
|
1659
|
+
@state_mutex.synchronize do
|
|
1660
|
+
@init_heal_pending = true
|
|
1661
|
+
@init_last_errs = errs
|
|
1662
|
+
end
|
|
1663
|
+
start_not_ready_reporter
|
|
1664
|
+
nil
|
|
1665
|
+
end
|
|
1666
|
+
|
|
1667
|
+
# WARN every 30s for 5 minutes, then every 5 minutes, while a :continue
|
|
1668
|
+
# client has no config (section C, "Background retry while not ready").
|
|
1669
|
+
def start_not_ready_reporter
|
|
1670
|
+
return if @not_ready_reporter&.alive?
|
|
1671
|
+
|
|
1672
|
+
@not_ready_reporter = Thread.new do
|
|
1673
|
+
Thread.current.name = 'quonfig-not-ready-reporter' if Thread.current.respond_to?(:name=)
|
|
1674
|
+
loop do
|
|
1675
|
+
elapsed = monotonic_now - @init_started_at
|
|
1676
|
+
sleep(elapsed < 300 ? 30 : 300)
|
|
1677
|
+
break if @stopped || ready?
|
|
1678
|
+
|
|
1679
|
+
since = (monotonic_now - @init_started_at).round
|
|
1680
|
+
errs = @state_mutex.synchronize { @init_last_errs }
|
|
1681
|
+
LOG.warn "[quonfig] still not connected to Quonfig delivery (#{since / 60}m#{since % 60}s since start); " \
|
|
1682
|
+
"serving defaults. #{errs}", event: 'quonfig.init.not_connected'
|
|
1683
|
+
end
|
|
1684
|
+
rescue StandardError
|
|
1685
|
+
nil
|
|
1686
|
+
end
|
|
1687
|
+
end
|
|
1688
|
+
|
|
1689
|
+
def log_init_recovered(start, _attempt = nil)
|
|
1690
|
+
via = @config_loader.respond_to?(:resolved_from) ? @config_loader.resolved_from : ''
|
|
1691
|
+
# A stream install leaves resolved_from empty: name the SSE leg.
|
|
1692
|
+
via = sse_failed_over_to_secondary? ? leg_label(1) : leg_label(0) if via.to_s.empty?
|
|
1693
|
+
version = @config_loader.respond_to?(:version) ? @config_loader.version : nil
|
|
1694
|
+
LOG.warn "[quonfig] init recovered: config loaded after #{format('%.1f', monotonic_now - start)}s via #{via}" \
|
|
1695
|
+
"#{" (version #{version})" if version}. Now serving live config.", event: 'quonfig.init.recovered'
|
|
1696
|
+
end
|
|
1697
|
+
|
|
1698
|
+
# Whether an attempt hung (decision 7): no leg failed fast. Every leg that
|
|
1699
|
+
# settled ended on a timeout, and a leg with no error yet was still in
|
|
1700
|
+
# flight when the init deadline cut the attempt off.
|
|
1701
|
+
def attempt_hung?(urls, leg_errors)
|
|
1702
|
+
urls.each_index.all? do |i|
|
|
1703
|
+
err = leg_errors[i]
|
|
1704
|
+
err.nil? || err == Quonfig::ConfigLoader::LEG_TIMED_OUT
|
|
1390
1705
|
end
|
|
1391
1706
|
end
|
|
1392
1707
|
|
|
1708
|
+
# primary=<host>: <error>; secondary=<host>: <error>. Hosts only, never the
|
|
1709
|
+
# SDK key or URL credentials.
|
|
1710
|
+
def format_leg_errors(urls, leg_errors, in_flight: false)
|
|
1711
|
+
urls.each_with_index.map do |url, i|
|
|
1712
|
+
label = leg_label(i)
|
|
1713
|
+
err = leg_errors[i] || (in_flight ? 'no response before initTimeout' : 'no response')
|
|
1714
|
+
"#{label}=#{safe_host(url)}: #{err}"
|
|
1715
|
+
end.join('; ')
|
|
1716
|
+
end
|
|
1717
|
+
|
|
1718
|
+
def safe_host(url)
|
|
1719
|
+
uri = URI.parse(url.to_s)
|
|
1720
|
+
uri.host ? "#{uri.host}:#{uri.port}" : '?'
|
|
1721
|
+
rescue URI::InvalidURIError
|
|
1722
|
+
'?'
|
|
1723
|
+
end
|
|
1724
|
+
|
|
1725
|
+
# "10s" for whole seconds, "0.5s" otherwise.
|
|
1726
|
+
def format_secs(secs)
|
|
1727
|
+
secs == secs.to_i ? "#{secs.to_i}s" : "#{secs}s"
|
|
1728
|
+
end
|
|
1729
|
+
|
|
1730
|
+
def monotonic_now
|
|
1731
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
1732
|
+
end
|
|
1733
|
+
|
|
1393
1734
|
# qfg-pinh: In SDK-key DELIVERY mode the server's `meta.environment` is
|
|
1394
1735
|
# AUTHORITATIVE. The server scopes each config to a single environment and
|
|
1395
1736
|
# reports the active env id in `meta.environment` (the loader captures it
|
|
@@ -1431,6 +1772,26 @@ module Quonfig
|
|
|
1431
1772
|
)
|
|
1432
1773
|
end
|
|
1433
1774
|
|
|
1775
|
+
DEPRECATED_INIT_FAILURE_WARNED = Set.new
|
|
1776
|
+
DEPRECATED_INIT_FAILURE_MUTEX = Mutex.new
|
|
1777
|
+
private_constant :DEPRECATED_INIT_FAILURE_WARNED, :DEPRECATED_INIT_FAILURE_MUTEX
|
|
1778
|
+
|
|
1779
|
+
# One WARN per process per deprecated spelling (section C), and a WARN for
|
|
1780
|
+
# an unrecognized value, which runs as :fail.
|
|
1781
|
+
def warn_on_init_failure_spelling(options)
|
|
1782
|
+
if (bad = options.on_init_failure_unknown)
|
|
1783
|
+
LOG.warn "[quonfig] unknown on_init_failure #{bad.inspect}; using :fail (expected :fail, :wait or :continue)"
|
|
1784
|
+
end
|
|
1785
|
+
old = options.on_init_failure_alias
|
|
1786
|
+
return if old.nil?
|
|
1787
|
+
|
|
1788
|
+
first = DEPRECATED_INIT_FAILURE_MUTEX.synchronize { DEPRECATED_INIT_FAILURE_WARNED.add?(old) }
|
|
1789
|
+
return unless first
|
|
1790
|
+
|
|
1791
|
+
LOG.warn "[quonfig] on_init_failure=:#{old} is deprecated; use :#{options.on_init_failure}",
|
|
1792
|
+
event: 'quonfig.init.deprecated_alias'
|
|
1793
|
+
end
|
|
1794
|
+
|
|
1434
1795
|
# qfg-7h5d.1.14: the per-leg hedge abort MUST be < init_timeout_ms, otherwise
|
|
1435
1796
|
# the init-path heal leg is clipped by the overall init deadline before it can
|
|
1436
1797
|
# heal forward. Mirrors sdk-go's construction-time warning in options.go. Warn
|
|
@@ -1445,6 +1806,8 @@ module Quonfig
|
|
|
1445
1806
|
abort_ms = @options.config_fetch_hedge_abort_ms
|
|
1446
1807
|
init_ms = @options.init_timeout_ms
|
|
1447
1808
|
return if abort_ms.nil? || init_ms.nil?
|
|
1809
|
+
# <= 0 runs one attempt to completion; no init deadline clips its legs.
|
|
1810
|
+
return if init_ms <= 0
|
|
1448
1811
|
return if init_ms > abort_ms
|
|
1449
1812
|
|
|
1450
1813
|
LOG.warn(
|
|
@@ -1476,15 +1839,6 @@ module Quonfig
|
|
|
1476
1839
|
)
|
|
1477
1840
|
end
|
|
1478
1841
|
|
|
1479
|
-
def handle_init_failure(err)
|
|
1480
|
-
if @options.on_init_failure == Quonfig::Options::ON_INITIALIZATION_FAILURE::RETURN
|
|
1481
|
-
LOG.warn "[quonfig] Initialization did not complete cleanly; continuing with empty store: #{err.message}"
|
|
1482
|
-
return
|
|
1483
|
-
end
|
|
1484
|
-
|
|
1485
|
-
raise err
|
|
1486
|
-
end
|
|
1487
|
-
|
|
1488
1842
|
# Returns true if SSE started successfully, false otherwise. A false here
|
|
1489
1843
|
# signals the caller to fall back to polling.
|
|
1490
1844
|
def start_sse
|
|
@@ -1685,7 +2039,10 @@ module Quonfig
|
|
|
1685
2039
|
begin
|
|
1686
2040
|
cb.call
|
|
1687
2041
|
rescue StandardError => e
|
|
1688
|
-
|
|
2042
|
+
# Counted, not a worker restart (supervisor contract Test 5).
|
|
2043
|
+
@state_mutex.synchronize { @callback_errors['on_config_update'] += 1 }
|
|
2044
|
+
bt = (e.backtrace || []).first(10).join("\n ")
|
|
2045
|
+
LOG.error "[quonfig] onConfigUpdate callback raised: #{e.class}: #{e.message}\n #{bt}"
|
|
1689
2046
|
end
|
|
1690
2047
|
end
|
|
1691
2048
|
|
|
@@ -1762,6 +2119,25 @@ module Quonfig
|
|
|
1762
2119
|
nil
|
|
1763
2120
|
end
|
|
1764
2121
|
|
|
2122
|
+
# A delivery-mode client that has never installed config (init continued
|
|
2123
|
+
# on code defaults, or the caller caught a failed init): the *_details
|
|
2124
|
+
# getters answer ERROR / PROVIDER_NOT_READY (init-failure design B.3)
|
|
2125
|
+
# rather than claiming the key does not exist.
|
|
2126
|
+
def provider_not_ready?
|
|
2127
|
+
@network_mode && !ready?
|
|
2128
|
+
end
|
|
2129
|
+
|
|
2130
|
+
def provider_not_ready_details
|
|
2131
|
+
Quonfig::EvaluationDetails.new(
|
|
2132
|
+
value: nil,
|
|
2133
|
+
reason: Quonfig::EvaluationDetails::REASON_ERROR,
|
|
2134
|
+
error_code: Quonfig::EvaluationDetails::ERROR_PROVIDER_NOT_READY,
|
|
2135
|
+
error_message: 'Quonfig has not loaded config yet',
|
|
2136
|
+
variant: build_variant(Quonfig::EvaluationDetails::REASON_ERROR, nil, nil),
|
|
2137
|
+
flag_metadata: build_flag_metadata(nil, nil, nil, nil, nil)
|
|
2138
|
+
)
|
|
2139
|
+
end
|
|
2140
|
+
|
|
1765
2141
|
# Build a Quonfig::EvaluationDetails for +key+, evaluated against the
|
|
1766
2142
|
# caller's context, after coercing/checking +expected_type+. Never
|
|
1767
2143
|
# raises; all exceptions become ERROR details.
|
|
@@ -1770,6 +2146,7 @@ module Quonfig
|
|
|
1770
2146
|
jit = context == NO_DEFAULT_PROVIDED ? nil : context
|
|
1771
2147
|
ctx = build_context(jit)
|
|
1772
2148
|
record_context_for_telemetry(ctx)
|
|
2149
|
+
return provider_not_ready_details if provider_not_ready?
|
|
1773
2150
|
|
|
1774
2151
|
result =
|
|
1775
2152
|
begin
|