launchdarkly-server-sdk 5.6.2 → 5.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0f95b72da90326fccd6fb823407b61ca1c686e53
-  data.tar.gz: 661768aaa51261feba0018cbc0f3ba793580c3e6
+  metadata.gz: 78fa0fd5890ae19f95b19c915694bca33187ef23
+  data.tar.gz: 5dae22ebbc4a8c1d2e6deb3e206ff4d27937da02
 SHA512:
-  metadata.gz: a409099d6bf0e5fb75b8d64ad05e52983bafb22f645620c8adb7403a0435a0ba9bd14068868b2ef1419f291431d6d517dad74284289f166e7ca21ca7156e0462
-  data.tar.gz: 72c021d8519a455da47ad3c5d8eacbbb79ef80ee068be1b37a5f200abdf8248e7741638de1f8811cbde03638635bc6a9029d90f90f2b6474b5d5e2fe8aa82b1e
+  metadata.gz: 888c327731d50e3d4869b2c6fe72748a9c9bb7f2c80123194eb2d32f1b365a613dbf7bec78a38fcf4f35b16e2ec634725fde25681bad0225dbe9fee41ce674f8
+  data.tar.gz: b8fcaa392b4940838daa060d66eea7dc2413d458875de3bb36905e70b194b9e34f6707d1d014c6e297ac81d396123411060c18761fe1a05c6287419cd902ca27

data/CHANGELOG.md

@@ -2,6 +2,10 @@
 
 All notable changes to the LaunchDarkly Ruby SDK will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org).
 
+## [5.6.2] - 2020-01-15
+### Fixed:
+- The SDK now specifies a uniquely identifiable request header when sending events to LaunchDarkly to ensure that events are only processed once, even if the SDK sends them two times due to a failed initial attempt.
+
 ## [5.6.1] - 2020-01-06
 ### Fixed:
 - In rare circumstances (depending on the exact data in the flag configuration, the flag's salt value, and the user properties), a percentage rollout could fail and return a default value, logging the error "Data inconsistency in feature flag ... variation/rollout object with no variation or rollout". This would happen if the user's hashed value fell exactly at the end of the last "bucket" (the last variation defined in the rollout). This has been fixed so that the user will get the last variation.

data/Gemfile.lock

@@ -1,10 +1,10 @@
 PATH
   remote: .
   specs:
-    launchdarkly-server-sdk (5.6.2)
+    launchdarkly-server-sdk (5.7.0)
       concurrent-ruby (~> 1.0)
       json (>= 1.8, < 3)
-      ld-eventsource (= 1.0.1)
+      ld-eventsource (= 1.0.2)
       semantic (~> 1.6)
 
 GEM
@@ -23,7 +23,7 @@ GEM
     aws-sigv4 (1.0.3)
     codeclimate-test-reporter (0.6.0)
       simplecov (>= 0.7.1, < 1.0.0)
-    concurrent-ruby (1.1.5)
+    concurrent-ruby (1.1.6)
     connection_pool (2.2.1)
     diff-lcs (1.3)
     diplomat (2.0.2)
@@ -40,7 +40,7 @@ GEM
     jmespath (1.4.0)
     json (1.8.6)
     json (1.8.6-java)
-    ld-eventsource (1.0.1)
+    ld-eventsource (1.0.2)
       concurrent-ruby (~> 1.0)
       http_tools (~> 0.4.5)
       socketry (~> 0.5.1)
@@ -49,7 +49,6 @@ GEM
       rb-inotify (~> 0.9, >= 0.9.7)
       ruby_dep (~> 1.2)
     multipart-post (2.0.0)
-    rake (10.5.0)
     rb-fsevent (0.10.3)
     rb-inotify (0.9.10)
       ffi (>= 0.5.0, < 2)
@@ -92,7 +91,6 @@ DEPENDENCIES
   diplomat (>= 2.0.2)
   launchdarkly-server-sdk!
   listen (~> 3.0)
-  rake (~> 10.0)
   redis (~> 3.3.5)
   rspec (~> 3.2)
   rspec_junit_formatter (~> 0.3.0)

data/launchdarkly-server-sdk.gemspec

@@ -28,7 +28,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "diplomat", ">= 2.0.2"
   spec.add_development_dependency "redis", "~> 3.3.5"
   spec.add_development_dependency "connection_pool", ">= 2.1.2"
-  spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec_junit_formatter", "~> 0.3.0"
   spec.add_development_dependency "timecop", "~> 0.9.1"
   spec.add_development_dependency "listen", "~> 3.0" # see file_data_source.rb
@@ -36,5 +35,5 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "json", [">= 1.8", "< 3"]
   spec.add_runtime_dependency "semantic", "~> 1.6"
   spec.add_runtime_dependency "concurrent-ruby", "~> 1.0"
-  spec.add_runtime_dependency "ld-eventsource", "1.0.1"
+  spec.add_runtime_dependency "ld-eventsource", "1.0.2"
 end

data/lib/ldclient-rb/config.rb

@@ -37,6 +37,10 @@ module LaunchDarkly
     # @option opts [Object] :data_source See {#data_source}.
     # @option opts [Object] :update_processor Obsolete synonym for `data_source`.
     # @option opts [Object] :update_processor_factory Obsolete synonym for `data_source`.
+    # @option opts [Boolean] :diagnostic_opt_out (false) See {#diagnostic_opt_out?}.
+    # @option opts [Float] :diagnostic_recording_interval (900) See {#diagnostic_recording_interval}.
+    # @option opts [String] :wrapper_name See {#wrapper_name}.
+    # @option opts [String] :wrapper_version See {#wrapper_version}.
     #
     def initialize(opts = {})
       @base_uri = (opts[:base_uri] || Config.default_base_uri).chomp("/")
@@ -62,6 +66,11 @@ module LaunchDarkly
       @data_source = opts[:data_source] || opts[:update_processor] || opts[:update_processor_factory]
       @update_processor = opts[:update_processor]
       @update_processor_factory = opts[:update_processor_factory]
+      @diagnostic_opt_out = opts.has_key?(:diagnostic_opt_out) && opts[:diagnostic_opt_out]
+      @diagnostic_recording_interval = opts.has_key?(:diagnostic_recording_interval) && opts[:diagnostic_recording_interval] > Config.minimum_diagnostic_recording_interval ?
+        opts[:diagnostic_recording_interval] : Config.default_diagnostic_recording_interval
+      @wrapper_name = opts[:wrapper_name]
+      @wrapper_version = opts[:wrapper_version]
     end
 
     #
@@ -257,6 +266,45 @@ module LaunchDarkly
     # @deprecated This is replaced by {#data_source}.
     attr_reader :update_processor_factory
 
+    #
+    # Set to true to opt out of sending diagnostics data.
+    #
+    # Unless `diagnostic_opt_out` is set to true, the client will send some diagnostics data to the LaunchDarkly servers
+    # in order to assist in the development of future SDK improvements. These diagnostics consist of an initial payload
+    # containing some details of the SDK in use, the SDK's configuration, and the platform the SDK is being run on, as
+    # well as periodic information on irregular occurrences such as dropped events.
+    # @return [Boolean]
+    #
+    def diagnostic_opt_out?
+      @diagnostic_opt_out
+    end
+
+    #
+    # The interval at which periodic diagnostic data is sent, in seconds.
+    #
+    # The default is 900 (every 15 minutes) and the minimum value is 60 (every minute).
+    # @return [Float]
+    #
+    attr_reader :diagnostic_recording_interval
+
+    #
+    # For use by wrapper libraries to set an identifying name for the wrapper being used.
+    #
+    # This will be sent in User-Agent headers during requests to the LaunchDarkly servers to allow recording
+    # metrics on the usage of these wrapper libraries.
+    # @return [String]
+    #
+    attr_reader :wrapper_name
+
+    #
+    # For use by wrapper libraries to report the version of the library in use.
+    #
+    # If `wrapper_name` is not set, this field will be ignored. Otherwise the version string will be included in
+    # the User-Agent headers along with the `wrapper_name` during requests to the LaunchDarkly servers.
+    # @return [String]
+    #
+    attr_reader :wrapper_version
+
     #
     # The default LaunchDarkly client configuration. This configuration sets
     # reasonable defaults for most users.
@@ -407,5 +455,21 @@ module LaunchDarkly
     def self.default_user_keys_flush_interval
       300
     end
+
+    #
+    # The default value for {#diagnostic_recording_interval}.
+    # @return [Float] 900
+    #
+    def self.default_diagnostic_recording_interval
+      900
+    end
+
+    #
+    # The minimum value for {#diagnostic_recording_interval}.
+    # @return [Float] 60
+    #
+    def self.minimum_diagnostic_recording_interval
+      60
+    end
   end
 end

data/lib/ldclient-rb/events.rb

@@ -1,7 +1,10 @@
+require "ldclient-rb/impl/diagnostic_events"
+require "ldclient-rb/impl/event_sender"
+require "ldclient-rb/impl/util"
+
 require "concurrent"
 require "concurrent/atomics"
 require "concurrent/executors"
-require "securerandom"
 require "thread"
 require "time"
 
@@ -24,12 +27,10 @@ require "time"
 
 module LaunchDarkly
   MAX_FLUSH_WORKERS = 5
-  CURRENT_SCHEMA_VERSION = 3
   USER_ATTRS_TO_STRINGIFY_FOR_EVENTS = [ :key, :secondary, :ip, :country, :email, :firstName, :lastName,
     :avatar, :name ]
 
   private_constant :MAX_FLUSH_WORKERS
-  private_constant :CURRENT_SCHEMA_VERSION
   private_constant :USER_ATTRS_TO_STRINGIFY_FOR_EVENTS
 
   # @private
@@ -60,6 +61,10 @@ module LaunchDarkly
   class FlushUsersMessage
   end
 
+  # @private
+  class DiagnosticEventMessage
+  end
+
   # @private
   class SynchronousMessage
     def initialize
@@ -85,9 +90,9 @@ module LaunchDarkly
 
   # @private
   class EventProcessor
-    def initialize(sdk_key, config, client = nil)
+    def initialize(sdk_key, config, client = nil, diagnostic_accumulator = nil, test_properties = nil)
       @logger = config.logger
-      @inbox = SizedQueue.new(config.capacity)
+      @inbox = SizedQueue.new(config.capacity < 100 ? 100 : config.capacity)
       @flush_task = Concurrent::TimerTask.new(execution_interval: config.flush_interval) do
         post_to_inbox(FlushMessage.new)
       end
@@ -96,14 +101,29 @@ module LaunchDarkly
         post_to_inbox(FlushUsersMessage.new)
       end
       @users_flush_task.execute
+      if !diagnostic_accumulator.nil?
+        interval = test_properties && test_properties.has_key?(:diagnostic_recording_interval) ?
+          test_properties[:diagnostic_recording_interval] :
+          config.diagnostic_recording_interval
+        @diagnostic_event_task = Concurrent::TimerTask.new(execution_interval: interval) do
+          post_to_inbox(DiagnosticEventMessage.new)
+        end
+        @diagnostic_event_task.execute
+      else
+        @diagnostic_event_task = nil
+      end
       @stopped = Concurrent::AtomicBoolean.new(false)
       @inbox_full = Concurrent::AtomicBoolean.new(false)
 
-      EventDispatcher.new(@inbox, sdk_key, config, client)
+      event_sender = test_properties && test_properties.has_key?(:event_sender) ?
+        test_properties[:event_sender] :
+        Impl::EventSender.new(sdk_key, config, client ? client : Util.new_http_client(config.events_uri, config))
+
+      EventDispatcher.new(@inbox, sdk_key, config, diagnostic_accumulator, event_sender)
     end
 
     def add_event(event)
-      event[:creationDate] = (Time.now.to_f * 1000).to_i
+      event[:creationDate] = Impl::Util.current_time_millis
       post_to_inbox(EventMessage.new(event))
     end
 
@@ -117,6 +137,7 @@ module LaunchDarkly
       if @stopped.make_true
         @flush_task.shutdown
         @users_flush_task.shutdown
+        @diagnostic_event_task.shutdown if !@diagnostic_event_task.nil?
         # Note that here we are not calling post_to_inbox, because we *do* want to wait if the inbox
         # is full; an orderly shutdown can't happen unless these messages are received.
         @inbox << FlushMessage.new
@@ -152,34 +173,36 @@ module LaunchDarkly
 
   # @private
   class EventDispatcher
-    def initialize(inbox, sdk_key, config, client)
+    def initialize(inbox, sdk_key, config, diagnostic_accumulator, event_sender)
       @sdk_key = sdk_key
       @config = config
-
-      if client
-        @client = client
-      else
-        @client = Util.new_http_client(@config.events_uri, @config)
-      end
+      @diagnostic_accumulator = config.diagnostic_opt_out? ? nil : diagnostic_accumulator
+      @event_sender = event_sender
 
       @user_keys = SimpleLRUCacheSet.new(config.user_keys_capacity)
       @formatter = EventOutputFormatter.new(config)
       @disabled = Concurrent::AtomicBoolean.new(false)
       @last_known_past_time = Concurrent::AtomicReference.new(0)
-
+      @deduplicated_users = 0
+      @events_in_last_batch = 0
+      
       outbox = EventBuffer.new(config.capacity, config.logger)
       flush_workers = NonBlockingThreadPool.new(MAX_FLUSH_WORKERS)
 
-      Thread.new { main_loop(inbox, outbox, flush_workers) }
+      if !@diagnostic_accumulator.nil?
+        diagnostic_event_workers = NonBlockingThreadPool.new(1)
+        init_event = @diagnostic_accumulator.create_init_event(config)
+        send_diagnostic_event(init_event, diagnostic_event_workers)
+      else
+        diagnostic_event_workers = nil
+      end
+
+      Thread.new { main_loop(inbox, outbox, flush_workers, diagnostic_event_workers) }
     end
 
     private
 
-    def now_millis()
-      (Time.now.to_f * 1000).to_i
-    end
-
-    def main_loop(inbox, outbox, flush_workers)
+    def main_loop(inbox, outbox, flush_workers, diagnostic_event_workers)
       running = true
       while running do
         begin
@@ -191,11 +214,13 @@ module LaunchDarkly
             trigger_flush(outbox, flush_workers)
           when FlushUsersMessage
             @user_keys.clear
+          when DiagnosticEventMessage
+            send_and_reset_diagnostics(outbox, diagnostic_event_workers)
           when TestSyncMessage
-            synchronize_for_testing(flush_workers)
+            synchronize_for_testing(flush_workers, diagnostic_event_workers)
             message.completed
           when StopMessage
-            do_shutdown(flush_workers)
+            do_shutdown(flush_workers, diagnostic_event_workers)
             running = false
             message.completed
           end
@@ -205,18 +230,23 @@ module LaunchDarkly
       end
     end
 
-    def do_shutdown(flush_workers)
+    def do_shutdown(flush_workers, diagnostic_event_workers)
       flush_workers.shutdown
       flush_workers.wait_for_termination
+      if !diagnostic_event_workers.nil?
+        diagnostic_event_workers.shutdown
+        diagnostic_event_workers.wait_for_termination
+      end
       begin
         @client.finish
       rescue
       end
     end
 
-    def synchronize_for_testing(flush_workers)
+    def synchronize_for_testing(flush_workers, diagnostic_event_workers)
       # Used only by unit tests. Wait until all active flush workers have finished.
       flush_workers.wait_all
+      diagnostic_event_workers.wait_all if !diagnostic_event_workers.nil?
     end
 
     def dispatch_event(event, outbox)
@@ -260,7 +290,9 @@ module LaunchDarkly
       if user.nil? || !user.has_key?(:key)
         true
       else
-        @user_keys.add(user[:key].to_s)
+        known = @user_keys.add(user[:key].to_s)
+        @deduplicated_users += 1 if known
+        known
       end
     end
 
@@ -268,7 +300,7 @@ module LaunchDarkly
       debug_until = event[:debugEventsUntilDate]
       if !debug_until.nil?
         last_past = @last_known_past_time.value
-        debug_until > last_past && debug_until > now_millis
+        debug_until > last_past && debug_until > Impl::Util.current_time_millis
       else
         false
       end
@@ -281,34 +313,44 @@ module LaunchDarkly
 
       payload = outbox.get_payload  
       if !payload.events.empty? || !payload.summary.counters.empty?
+        count = payload.events.length + (payload.summary.counters.empty? ? 0 : 1)
+        @events_in_last_batch = count
         # If all available worker threads are busy, success will be false and no job will be queued.
         success = flush_workers.post do
           begin
-            resp = EventPayloadSendTask.new.run(@sdk_key, @config, @client, payload, @formatter)
-            handle_response(resp) if !resp.nil?
+            events_out = @formatter.make_output_events(payload.events, payload.summary)
+            result = @event_sender.send_event_data(events_out.to_json, false)
+            @disabled.value = true if result.must_shutdown
+            if !result.time_from_server.nil?
+              @last_known_past_time.value = (result.time_from_server.to_f * 1000).to_i
+            end
           rescue => e
             Util.log_exception(@config.logger, "Unexpected error in event processor", e)
           end
         end
         outbox.clear if success # Reset our internal state, these events now belong to the flush worker
+      else
+        @events_in_last_batch = 0
       end
     end
 
-    def handle_response(res)
-      status = res.code.to_i
-      if status >= 400
-        message = Util.http_error_message(status, "event delivery", "some events were dropped")
-        @config.logger.error { "[LDClient] #{message}" }
-        if !Util.http_error_recoverable?(status)
-          @disabled.value = true
-        end
-      else
-        if !res["date"].nil?
-          begin
-            res_time = (Time.httpdate(res["date"]).to_f * 1000).to_i
-            @last_known_past_time.value = res_time
-          rescue ArgumentError
-          end
+    def send_and_reset_diagnostics(outbox, diagnostic_event_workers)
+      return if @diagnostic_accumulator.nil?
+      dropped_count = outbox.get_and_clear_dropped_count
+      event = @diagnostic_accumulator.create_periodic_event_and_reset(dropped_count, @deduplicated_users, @events_in_last_batch)
+      @deduplicated_users = 0
+      @events_in_last_batch = 0
+      send_diagnostic_event(event, diagnostic_event_workers)
+    end
+
+    def send_diagnostic_event(event, diagnostic_event_workers)
+      return if diagnostic_event_workers.nil?
+      uri = URI(@config.events_uri + "/diagnostic")
+      diagnostic_event_workers.post do
+        begin
+          @event_sender.send_event_data(event.to_json, true)
+        rescue => e
+          Util.log_exception(@config.logger, "Unexpected error in event processor", e)
         end
       end
     end
@@ -323,6 +365,7 @@ module LaunchDarkly
       @capacity = capacity
       @logger = logger
       @capacity_exceeded = false
+      @dropped_events = 0
       @events = []
       @summarizer = EventSummarizer.new
     end
@@ -333,6 +376,7 @@ module LaunchDarkly
         @events.push(event)
         @capacity_exceeded = false
       else
+        @dropped_events += 1
         if !@capacity_exceeded
           @capacity_exceeded = true
           @logger.warn { "[LDClient] Exceeded event queue capacity. Increase capacity to avoid dropping events." }
@@ -348,54 +392,18 @@ module LaunchDarkly
       return FlushPayload.new(@events, @summarizer.snapshot)
     end
 
+    def get_and_clear_dropped_count
+      ret = @dropped_events
+      @dropped_events = 0
+      ret
+    end
+
     def clear
       @events = []
       @summarizer.clear
     end
   end
 
-  # @private
-  class EventPayloadSendTask
-    def run(sdk_key, config, client, payload, formatter)
-      events_out = formatter.make_output_events(payload.events, payload.summary)
-      res = nil
-      body = events_out.to_json
-      payload_id = SecureRandom.uuid
-      (0..1).each do |attempt|
-        if attempt > 0
-          config.logger.warn { "[LDClient] Will retry posting events after 1 second" }
-          sleep(1)
-        end
-        begin
-          client.start if !client.started?
-          config.logger.debug { "[LDClient] sending #{events_out.length} events: #{body}" }
-          uri = URI(config.events_uri + "/bulk")
-          req = Net::HTTP::Post.new(uri)
-          req.content_type = "application/json"
-          req.body = body
-          req["Authorization"] = sdk_key
-          req["User-Agent"] = "RubyClient/" + LaunchDarkly::VERSION
-          req["X-LaunchDarkly-Event-Schema"] = CURRENT_SCHEMA_VERSION.to_s
-          req["X-LaunchDarkly-Payload-ID"] = payload_id
-          req["Connection"] = "keep-alive"
-          res = client.request(req)
-        rescue StandardError => exn
-          config.logger.warn { "[LDClient] Error flushing events: #{exn.inspect}." }
-          next
-        end
-        status = res.code.to_i
-        if status < 200 || status >= 300
-          if Util.http_error_recoverable?(status)
-            next
-          end
-        end
-        break
-      end
-      # used up our retries, return the last response if any
-      res
-    end
-  end
-
   # @private
   class EventOutputFormatter
     def initialize(config)