statsig 1.10.0 → 1.20.0

data/lib/layer.rb

@@ -1,7 +1,22 @@
+# typed: false
+
+##
+# Contains the current values from Statsig.
+# Will contain layer default values for all shared parameters in that layer.
+# If a parameter is in an active experiment, and the current user is allocated to that experiment,
+# those parameters will be updated to reflect the experiment values not the layer defaults.
+#
+# Layers Documentation: https://docs.statsig.com/layers
 class Layer
+  extend T::Sig
+
+  sig { returns(String) }
   attr_accessor :name
+
+  sig { returns(String) }
   attr_accessor :rule_id
 
+  sig { params(name: String, value: T::Hash[String, T.untyped], rule_id: String, exposure_log_func: T.any(Method, Proc, NilClass)).void }
   def initialize(name, value = {}, rule_id = '', exposure_log_func = nil)
     @name = name
     @value = value
@@ -9,6 +24,12 @@ class Layer
     @exposure_log_func = exposure_log_func
   end
 
+  sig { params(index: String, default_value: T.untyped).returns(T.untyped) }
+  ##
+  # Get the value for the given key (index), falling back to the default_value if it cannot be found.
+  #
+  # @param index The name of parameter being fetched
+  # @param default_value The fallback value if the name cannot be found
   def get(index, default_value)
     return default_value if @value.nil? || !@value.key?(index)
 
@@ -18,4 +39,22 @@ class Layer
 
     @value[index]
   end
+
+  sig { params(index: String, default_value: T.untyped).returns(T.untyped) }
+  ##
+  # Get the value for the given key (index), falling back to the default_value if it cannot be found
+  # or is found to have a different type from the default_value.
+  #
+  # @param index The name of parameter being fetched
+  # @param default_value The fallback value if the name cannot be found
+  def get_typed(index, default_value)
+    return default_value if @value.nil? || !@value.key?(index)
+    return default_value if @value[index].class != default_value.class and default_value.class != TrueClass and default_value.class != FalseClass
+
+    if @exposure_log_func.is_a? Proc
+      @exposure_log_func.call(self, index)
+    end
+
+    @value[index]
+  end
 end

data/lib/network.rb

@@ -1,28 +1,56 @@
+# typed: true
+
 require 'http'
 require 'json'
 require 'securerandom'
+require 'sorbet-runtime'
 
 $retry_codes = [408, 500, 502, 503, 504, 522, 524, 599]
 
 module Statsig
+  class NetworkError < StandardError
+    attr_reader :http_code
+
+    def initialize(msg = nil, http_code = nil)
+      super(msg)
+      @http_code = http_code
+    end
+  end
+
   class Network
-    def initialize(server_secret, api, backoff_mult = 10)
+    extend T::Sig
+
+    sig { params(server_secret: String, api: String, local_mode: T::Boolean, backoff_mult: Integer).void }
+
+    def initialize(server_secret, api, local_mode, backoff_mult = 10)
       super()
       unless api.end_with?('/')
         api += '/'
       end
       @server_secret = server_secret
       @api = api
+      @local_mode = local_mode
       @backoff_multiplier = backoff_mult
       @session_id = SecureRandom.uuid
     end
 
+    sig { params(endpoint: String, body: String, retries: Integer, backoff: Integer)
+            .returns([T.any(HTTP::Response, NilClass), T.any(StandardError, NilClass)]) }
+
     def post_helper(endpoint, body, retries = 0, backoff = 1)
+      if @local_mode
+        return nil, nil
+      end
+
+      meta = Statsig.get_statsig_metadata
       http = HTTP.headers(
-        {"STATSIG-API-KEY" => @server_secret,
-        "STATSIG-CLIENT-TIME" => (Time.now.to_f * 1000).to_s,
-         "STATSIG-SERVER-SESSION-ID" => @session_id,
-        "Content-Type" => "application/json; charset=UTF-8"
+        {
+          "STATSIG-API-KEY" => @server_secret,
+          "STATSIG-CLIENT-TIME" => (Time.now.to_f * 1000).to_i.to_s,
+          "STATSIG-SERVER-SESSION-ID" => @session_id,
+          "Content-Type" => "application/json; charset=UTF-8",
+          "STATSIG-SDK-TYPE" => meta['sdkType'],
+          "STATSIG-SDK-VERSION" => meta['sdkVersion'],
         }).accept(:json)
       begin
         res = http.post(@api + endpoint, body: body)
@@ -32,8 +60,8 @@ module Statsig
         sleep backoff
         return post_helper(endpoint, body, retries - 1, backoff * @backoff_multiplier)
       end
-      return res, nil unless !res.status.success?
-      return nil, StandardError.new("Got an exception when making request to #{@api + endpoint}: #{res.to_s}") unless retries > 0 && $retry_codes.include?(res.code)
+      return res, nil if res.status.success?
+      return nil, NetworkError.new("Got an exception when making request to #{@api + endpoint}: #{res.to_s}", res.status.to_i) unless retries > 0 && $retry_codes.include?(res.code)
       ## status code retry
       sleep backoff
       post_helper(endpoint, body, retries - 1, backoff * @backoff_multiplier)
@@ -41,7 +69,7 @@ module Statsig
 
     def check_gate(user, gate_name)
       begin
-        request_body = JSON.generate({'user' => user&.serialize(false), 'gateName' => gate_name})
+        request_body = JSON.generate({ 'user' => user&.serialize(false), 'gateName' => gate_name })
         response, _ = post_helper('check_gate', request_body)
         return JSON.parse(response.body) unless response.nil?
         false
@@ -52,7 +80,7 @@ module Statsig
 
     def get_config(user, dynamic_config_name)
       begin
-        request_body = JSON.generate({'user' => user&.serialize(false), 'configName' => dynamic_config_name})
+        request_body = JSON.generate({ 'user' => user&.serialize(false), 'configName' => dynamic_config_name })
         response, _ = post_helper('get_config', request_body)
         return JSON.parse(response.body) unless response.nil?
         nil
@@ -63,8 +91,8 @@ module Statsig
 
     def post_logs(events)
       begin
-        json_body = JSON.generate({'events' => events, 'statsigMetadata' => Statsig.get_statsig_metadata})
-        post_helper('log_event', json_body, retries: 5)
+        json_body = JSON.generate({ 'events' => events, 'statsigMetadata' => Statsig.get_statsig_metadata })
+        post_helper('log_event', json_body, 5)
       rescue
       end
     end

data/lib/spec_store.rb

@@ -1,71 +1,157 @@
+# typed: false
 require 'net/http'
 require 'uri'
-
+require 'evaluation_details'
 require 'id_list'
+require 'concurrent-ruby'
 
 module Statsig
   class SpecStore
-    def initialize(network, error_callback = nil, rulesets_sync_interval = 10, id_lists_sync_interval = 60)
+
+    CONFIG_SPECS_KEY = "statsig.cache"
+
+    attr_accessor :last_config_sync_time
+    attr_accessor :initial_config_sync_time
+    attr_accessor :init_reason
+
+    def initialize(network, options, error_callback, init_diagnostics = nil)
+      @init_reason = EvaluationReason::UNINITIALIZED
       @network = network
-      @last_sync_time = 0
-      @rulesets_sync_interval = rulesets_sync_interval
-      @id_lists_sync_interval = id_lists_sync_interval
-      @store = {
+      @options = options
+      @error_callback = error_callback
+      @last_config_sync_time = 0
+      @initial_config_sync_time = 0
+      @rulesets_sync_interval = options.rulesets_sync_interval
+      @id_lists_sync_interval = options.idlists_sync_interval
+      @rules_updated_callback = options.rules_updated_callback
+      @specs = {
         :gates => {},
         :configs => {},
         :layers => {},
         :id_lists => {},
+        :experiment_to_layer => {}
       }
-      e = download_config_specs
-      error_callback.call(e) unless error_callback.nil?
-      get_id_lists
+
+      @id_list_thread_pool = Concurrent::FixedThreadPool.new(
+        options.idlist_threadpool_size,
+        max_queue: 100,
+        fallback_policy: :discard,
+      )
+
+      unless @options.bootstrap_values.nil?
+        begin
+          if !@options.data_store.nil?
+            puts 'data_store gets priority over bootstrap_values. bootstrap_values will be ignored'
+          else
+            init_diagnostics&.mark("bootstrap", "start", "load")
+            if process(options.bootstrap_values)
+              @init_reason = EvaluationReason::BOOTSTRAP
+            end
+            init_diagnostics&.mark("bootstrap", "end", "load", @init_reason == EvaluationReason::BOOTSTRAP)
+          end
+        rescue
+          puts 'the provided bootstrapValues is not a valid JSON string'
+        end
+      end
+
+      unless @options.data_store.nil?
+        init_diagnostics&.mark("data_store", "start", "load")
+        @options.data_store.init
+        load_from_storage_adapter
+        init_diagnostics&.mark("data_store", "end", "load", @init_reason == EvaluationReason::DATA_ADAPTER)
+      end
+
+      if @init_reason == EvaluationReason::UNINITIALIZED
+        download_config_specs(init_diagnostics)
+      end
+
+      @initial_config_sync_time = @last_config_sync_time == 0 ? -1 : @last_config_sync_time
+      get_id_lists(init_diagnostics)
 
       @config_sync_thread = sync_config_specs
       @id_lists_sync_thread = sync_id_lists
     end
 
+    def is_ready_for_checks
+      @last_config_sync_time != 0
+    end
+
     def shutdown
       @config_sync_thread&.exit
       @id_lists_sync_thread&.exit
+      @id_list_thread_pool.shutdown
+      @id_list_thread_pool.wait_for_termination(timeout = 3)
+      unless @options.data_store.nil?
+        @options.data_store.shutdown
+      end
     end
 
     def has_gate?(gate_name)
-      @store[:gates].key?(gate_name)
+      @specs[:gates].key?(gate_name)
     end
 
     def has_config?(config_name)
-      @store[:configs].key?(config_name)
+      @specs[:configs].key?(config_name)
     end
 
     def has_layer?(layer_name)
-      @store[:layers].key?(layer_name)
+      @specs[:layers].key?(layer_name)
     end
 
     def get_gate(gate_name)
       return nil unless has_gate?(gate_name)
-      @store[:gates][gate_name]
+      @specs[:gates][gate_name]
     end
 
     def get_config(config_name)
       return nil unless has_config?(config_name)
-      @store[:configs][config_name]
+      @specs[:configs][config_name]
     end
 
     def get_layer(layer_name)
       return nil unless has_layer?(layer_name)
-      @store[:layers][layer_name]
+      @specs[:layers][layer_name]
     end
 
     def get_id_list(list_name)
-      @store[:id_lists][list_name]
+      @specs[:id_lists][list_name]
+    end
+
+    def get_raw_specs
+      @specs
+    end
+
+    def maybe_restart_background_threads
+      if @config_sync_thread.nil? or !@config_sync_thread.alive?
+        @config_sync_thread = sync_config_specs
+      end
+      if @id_lists_sync_thread.nil? or !@id_lists_sync_thread.alive?
+        @id_lists_sync_thread = sync_id_lists
+      end
     end
 
     private
 
+    def load_from_storage_adapter
+      cached_values = @options.data_store.get(CONFIG_SPECS_KEY)
+      if cached_values.nil?
+        return
+      end
+      process(cached_values, true)
+      @init_reason = EvaluationReason::DATA_ADAPTER
+    end
+
+    def save_to_storage_adapter(specs_string)
+      if @options.data_store.nil?
+        return
+      end
+      @options.data_store.set(CONFIG_SPECS_KEY, specs_string)
+    end
+
     def sync_config_specs
       Thread.new do
         loop do
-          sleep @rulesets_sync_interval
+          sleep @options.rulesets_sync_interval
           download_config_specs
         end
       end
@@ -80,55 +166,102 @@ module Statsig
       end
     end
 
-    def download_config_specs
+    def download_config_specs(init_diagnostics = nil)
+      init_diagnostics&.mark("download_config_specs", "start", "network_request")
+
+      error = nil
       begin
-        response, e = @network.post_helper('download_config_specs', JSON.generate({'sinceTime' => @last_sync_time}))
+        response, e = @network.post_helper('download_config_specs', JSON.generate({ 'sinceTime' => @last_config_sync_time }))
+        code = response&.status.to_i
+        if e.is_a? NetworkError
+          code = e.http_code
+        end
+        init_diagnostics&.mark("download_config_specs", "end", "network_request", code)
+
         if e.nil?
-          process(JSON.parse(response.body))
+          unless response.nil?
+            init_diagnostics&.mark("download_config_specs", "start", "process")
+
+            if process(response.body)
+              @init_reason = EvaluationReason::NETWORK
+              @rules_updated_callback.call(response.body.to_s, @last_config_sync_time) unless response.body.nil? or @rules_updated_callback.nil?
+            end
+
+            init_diagnostics&.mark("download_config_specs", "end", "process", @init_reason == EvaluationReason::NETWORK)
+          end
+
+          nil
         else
-          e
+          error = e
         end
       rescue StandardError => e
-        e
+        error = e
       end
+
+      @error_callback.call(error) unless error.nil? or @error_callback.nil?
     end
 
-    def process(specs_json)
-      if specs_json.nil?
-        return
+    def process(specs_string, from_adapter = false)
+      if specs_string.nil?
+        return false
       end
 
-      @last_sync_time = specs_json['time'] || @last_sync_time
-      return unless specs_json['has_updates'] == true &&
+      specs_json = JSON.parse(specs_string)
+      return false unless specs_json.is_a? Hash
+
+      @last_config_sync_time = specs_json['time'] || @last_config_sync_time
+      return false unless specs_json['has_updates'] == true &&
         !specs_json['feature_gates'].nil? &&
         !specs_json['dynamic_configs'].nil? &&
-        !specs_json['layer_configs'].nil? 
+        !specs_json['layer_configs'].nil?
 
       new_gates = {}
       new_configs = {}
       new_layers = {}
+      new_exp_to_layer = {}
+
+      specs_json['feature_gates'].each { |gate| new_gates[gate['name']] = gate }
+      specs_json['dynamic_configs'].each { |config| new_configs[config['name']] = config }
+      specs_json['layer_configs'].each { |layer| new_layers[layer['name']] = layer }
+
+      if specs_json['layers'].is_a?(Hash)
+        specs_json['layers'].each { |layer_name, experiments|
+          experiments.each { |experiment_name| new_exp_to_layer[experiment_name] = layer_name }
+        }
+      end
 
-      specs_json['feature_gates'].map{|gate|  new_gates[gate['name']] = gate }
-      specs_json['dynamic_configs'].map{|config|  new_configs[config['name']] = config }
-      specs_json['layer_configs'].map{|layer|  new_layers[layer['name']] = layer }
-      @store[:gates] = new_gates
-      @store[:configs] = new_configs
-      @store[:layers] = new_layers
+      @specs[:gates] = new_gates
+      @specs[:configs] = new_configs
+      @specs[:layers] = new_layers
+      @specs[:experiment_to_layer] = new_exp_to_layer
+
+      unless from_adapter
+        save_to_storage_adapter(specs_string)
+      end
+      true
     end
 
-    def get_id_lists
-      response, e = @network.post_helper('get_id_lists', JSON.generate({'statsigMetadata' => Statsig.get_statsig_metadata}))
+    def get_id_lists(init_diagnostics = nil)
+      init_diagnostics&.mark("get_id_lists", "start", "network_request")
+      response, e = @network.post_helper('get_id_lists', JSON.generate({ 'statsigMetadata' => Statsig.get_statsig_metadata }))
       if !e.nil? || response.nil?
         return
       end
+      init_diagnostics&.mark("get_id_lists", "end", "network_request", response.status.to_i)
 
       begin
         server_id_lists = JSON.parse(response)
-        local_id_lists = @store[:id_lists]
+        local_id_lists = @specs[:id_lists]
         if !server_id_lists.is_a?(Hash) || !local_id_lists.is_a?(Hash)
           return
         end
-        threads = []
+        tasks = []
+
+        if server_id_lists.length == 0
+          return
+        end
+
+        init_diagnostics&.mark("get_id_lists", "start", "process", server_id_lists.length)
 
         server_id_lists.each do |list_name, list|
           server_list = IDList.new(list)
@@ -157,11 +290,17 @@ module Statsig
             next
           end
 
-          threads << Thread.new do
+          tasks << Concurrent::Promise.execute(:executor => @id_list_thread_pool) do
             download_single_id_list(local_list)
           end
         end
-        threads.each(&:join)
+
+        result = Concurrent::Promise.all?(*tasks).execute.wait(@id_lists_sync_interval)
+        if result.state != :fulfilled
+          init_diagnostics&.mark("get_id_lists", "end", "process", false)
+          return # timed out
+        end
+
         delete_lists = []
         local_id_lists.each do |list_name, list|
           unless server_id_lists.key? list_name
@@ -171,6 +310,7 @@ module Statsig
         delete_lists.each do |list_name|
           local_id_lists.delete list_name
         end
+        init_diagnostics&.mark("get_id_lists", "end", "process", true)
       rescue
         # Ignored, will try again
       end
@@ -178,7 +318,7 @@ module Statsig
 
     def download_single_id_list(list)
       nil unless list.is_a? IDList
-      http = HTTP.headers({'Range' => "bytes=#{list&.size || 0}-"}).accept(:json)
+      http = HTTP.headers({ 'Range' => "bytes=#{list&.size || 0}-" }).accept(:json)
       begin
         res = http.get(list.url)
         nil unless res.status.success?
@@ -186,7 +326,7 @@ module Statsig
         nil if content_length.nil? || content_length <= 0
         content = res.body.to_s
         unless content.is_a?(String) && (content[0] == '-' || content[0] == '+')
-          @store[:id_lists].delete(list.name)
+          @specs[:id_lists].delete(list.name)
           return
         end
         ids_clone = list.ids # clone the list, operate on the new list, and swap out the old list, so the operation is thread-safe
@@ -195,7 +335,7 @@ module Statsig
           line = li.strip
           next if line.length <= 1
           op = line[0]
-          id = line[1..]
+          id = line[1..line.length]
           if op == '+'
             ids_clone.add(id)
           elsif op == '-'