splitclient-rb 8.11.1 → 8.11.2.pre.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f75f1490dbebf79ecb7799bb96a29f180ed5259036edbd4351b3db9f1f2e1cba
-  data.tar.gz: e8f83fe01180e3a4a7487c82fd2a2c58cafdc971b951b9eb0c0c472351b00c4d
+  metadata.gz: 3f9a0707737415faaa93d90c63acefa18a154e72974150009071d8b7764735a4
+  data.tar.gz: 50fc9fe6dc2c51dd07bffc00b80a8f1fbcb82e7c2e51f2817666beaba5d8e5dc
 SHA512:
-  metadata.gz: 7d8a983e3f53437d2749ac0a7f60b83e2d124001f396b19389ee19cb63d2cee404ef4253835586414a793fbb737c803869db51f2114b0931ea7f474aa9bdfc1d
-  data.tar.gz: b6cc38e9b8d9d54b6d7858f02780059dc9b26dab95e9c022c5f92311621fc0057c4cddc1a1c2ea07b9831a27799ec5665cf5ef4b6cbc43dc0e27b4264179f51c
+  metadata.gz: 9ee3627c64357eb9ebd6ec7365371841387faa310d8bdcc8c012968a4274a50d7981551eb4e4bf632de562ff1141f9b4e2bf4ba504959bae3b54e1ef08519563
+  data.tar.gz: 5524bc84ef91500be12e09d9e457aec36c3d6058c37975ad2e6e7193fcbab45cd8cfbc8f99aa06e6ac2cb25696a3cfdb660c1a039003151809fba36c45b852c4

data/.gitignore

@@ -56,3 +56,5 @@ dump.rdb
 release.sh
 
 .scannerwork
+docs
+examples

data/.harness/orgs/PROD/projects/Harness_Split/pipelines/rubyclientrepo/input_sets/rubyclient.yaml

@@ -0,0 +1,14 @@
+inputSet:
+  name: ruby-client
+  identifier: rubyclient
+  orgIdentifier: PROD
+  projectIdentifier: Harness_Split
+  pipeline:
+    identifier: rubyclientrepo
+    properties:
+      ci:
+        codebase:
+          build:
+            type: PR
+            spec:
+              number: <+trigger.prNumber>

data/.harness/pipeline.yaml

@@ -0,0 +1,182 @@
+pipeline:
+  name: ruby-client-repo
+  identifier: rubyclientrepo
+  projectIdentifier: Harness_Split
+  orgIdentifier: PROD
+  tags: {}
+  properties:
+    ci:
+      codebase:
+        connectorRef: fmegithubharnessgitops
+        repoName: ruby-client
+        build:
+          type: branch
+          spec:
+            branch: <+input>
+  stages:
+    - stage:
+        name: Build and Test
+        identifier: Build_and_Test
+        description: ""
+        type: CI
+        spec:
+          cloneCodebase: true
+          caching:
+            enabled: true
+            override: true
+          platform:
+            os: Linux
+            arch: Amd64
+          runtime:
+            type: Cloud
+            spec:
+              size: flex
+          execution:
+            steps:
+              - step:
+                  type: Run
+                  name: Install Ruby
+                  identifier: Install_Ruby
+                  spec:
+                    shell: Bash
+                    command: |-
+                      curl https://mise.run | sh
+                      export PATH="$HOME/.local/bin:$PATH"
+                      mise use --global ruby@<+matrix.rubyVersion>
+                      ruby --version
+                      gem --version
+              - step:
+                  type: Run
+                  name: Start Redis
+                  identifier: Start_Redis
+                  spec:
+                    shell: Sh
+                    command: |-
+                      apt-get install -y redis-server -qq
+                      redis-server --daemonize yes --port 6379
+                      redis-cli ping
+              - step:
+                  type: Run
+                  name: Wait for Redis
+                  identifier: Wait_for_Redis
+                  spec:
+                    shell: Sh
+                    command: |-
+                      for i in $(seq 1 15); do
+                        redis-cli ping && break
+                        echo "Waiting for Redis... attempt $i"
+                        sleep 1
+                      done
+              - step:
+                  type: Run
+                  name: Install Dependencies
+                  identifier: Install_Dependencies
+                  spec:
+                    shell: Bash
+                    command: |-
+                      export PATH="$HOME/.local/bin:$PATH"
+                      eval "$(mise activate bash)"
+                      gem install bundler:2.4.19 --no-document
+                      bundle install --jobs 4 --retry 3
+              - step:
+                  type: Run
+                  name: Run Tests
+                  identifier: Run_Tests
+                  spec:
+                    shell: Bash
+                    command: |-
+                      export PATH="$HOME/.local/bin:$PATH"
+                      eval "$(mise activate bash)"
+                      bundle exec rake
+              - step:
+                  type: Run
+                  name: Fix Coverage Paths
+                  identifier: Fix_Coverage_Paths
+                  when:
+                    stageStatus: Success
+                    condition: <+matrix.rubyVersion> == "3.2.2"
+                  spec:
+                    shell: Bash
+                    command: |-
+                      export PATH="$HOME/.local/bin:$PATH"
+                      eval "$(mise activate bash)"
+                      cd coverage
+                      sed -i "s@${HARNESS_WORKSPACE}@/harness/workspace/@g" .resultset.json
+                      ruby -rjson -e '
+                        sqube = JSON.load(File.read(".resultset.json"))["RSpec"]["coverage"].transform_values {|lines| lines["lines"]}
+                        total = { "RSpec" => { "coverage" => sqube, "timestamp" => Time.now.to_i }}
+                        puts JSON.dump(total)
+                      ' > .resultset.sonarqube.json
+              - step:
+                  type: Run
+                  name: SonarQube Scan
+                  identifier: SonarQube_Scan
+                  when:
+                    stageStatus: Success
+                    condition: <+matrix.rubyVersion> == "3.2.2"
+                  spec:
+                    shell: Sh
+                    command: |-
+                      export SONAR_SCANNER_VERSION=5.0.1.3006
+                      curl -sSLo sonar-scanner.zip \
+                        "https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-${SONAR_SCANNER_VERSION}-linux.zip"
+                      unzip -q sonar-scanner.zip
+                      export PATH="$PWD/sonar-scanner-${SONAR_SCANNER_VERSION}-linux/bin:$PATH"
+                      apt-get install -y openjdk-17-jdk -qq
+                      sonar-scanner \
+                        -Dsonar.projectKey=ruby-client \
+                        -Dsonar.sources=lib \
+                        -Dsonar.tests=spec \
+                        -Dsonar.host.url=https://sonar.harness.io \
+                        -Dsonar.token=<+secrets.getValue("sonarqube-token")> \
+                        -Dsonar.ruby.coverage.reportPaths=coverage/.resultset.sonarqube.json \
+                        -Dsonar.coverage.exclusions="spec/**,**/vendor/**" \
+                        -Dsonar.exclusions="**/vendor/**,**/.git/**"
+              - step:
+                  type: Run
+                  name: Post Quality Gate
+                  identifier: Post_Quality_Gate
+                  when:
+                    stageStatus: Success
+                    condition: <+matrix.rubyVersion> == "3.2.2"
+                  spec:
+                    shell: Bash
+                    command: |-
+                      #!/bin/bash
+                      set -e
+
+                      SONAR_TOKEN="<+secrets.getValue("sonarqube-token")>"
+                      SONAR_URL="https://sonar.harness.io"
+                      SONAR_PROJECT_KEY="ruby-client"
+                      GITHUB_TOKEN="<+secrets.getValue("github-devops-token")>"
+                      GITHUB_REPO="ruby-client"
+                      COMMIT_SHA="<+codebase.commitSha>"
+
+                      STATUS_JSON=$(curl -sf \
+                        -H "Authorization: Bearer ${SONAR_TOKEN}" \
+                        "${SONAR_URL}/api/qualitygates/project_status?projectKey=${SONAR_PROJECT_KEY}")
+
+                      QG_STATUS=$(echo "$STATUS_JSON" | python3 -c "import sys,json; print(json.load(sys.stdin)['projectStatus']['status'])")
+
+                      case "$QG_STATUS" in
+                        OK)    GH_STATE="success"; DESCRIPTION="Quality gate passed" ;;
+                        ERROR) GH_STATE="failure"; DESCRIPTION="Quality gate failed" ;;
+                        WARN)  GH_STATE="success"; DESCRIPTION="Quality gate passed with warnings" ;;
+                        *)     GH_STATE="pending"; DESCRIPTION="Quality gate status unknown" ;;
+                      esac
+
+                      TARGET_URL="${SONAR_URL}/dashboard?id=${SONAR_PROJECT_KEY}"
+
+                      curl -sf -X POST \
+                        -H "Authorization: token ${GITHUB_TOKEN}" \
+                        -H "Accept: application/vnd.github.v3+json" \
+                        -H "Content-Type: application/json" \
+                        -d "{\"state\":\"${GH_STATE}\",\"description\":\"${DESCRIPTION}\",\"context\":\"sonarqube/quality-gate\",\"target_url\":\"${TARGET_URL}\"}" \
+                        "https://api.github.com/repos/splitio/${GITHUB_REPO}/statuses/${COMMIT_SHA}"
+
+                      echo "Posted SonarQube quality gate status: ${GH_STATE}"
+        strategy:
+          matrix:
+            rubyVersion:
+              - "2.7.8"
+              - "3.2.2"

data/.rubocop.yml

@@ -25,7 +25,7 @@ Metrics/ParameterLists:
   Exclude:
     - lib/splitclient-rb/engine/sync_manager.rb
 
-Metrics/LineLength:
+Layout/LineLength:
   Max: 135
 
 Metrics/BlockLength:

data/lib/splitclient-rb/engine/push_manager.rb

@@ -10,9 +10,25 @@ module SplitIoClient
         @api_key = api_key
         @back_off = Engine::BackOff.new(@config.auth_retry_back_off_base, 1)
         @telemetry_runtime_producer = telemetry_runtime_producer
+        # Serializes start_sse: it is reachable from the push status handler thread and
+        # from the token refresh thread concurrently.
+        @start_sse_mutex = Mutex.new
       end
 
       def start_sse
+        @start_sse_mutex.synchronize { start_sse_unsynchronized }
+      end
+
+      def stop_sse
+        @sse_handler.stop
+        SplitIoClient::Helpers::ThreadHelper.stop(:schedule_next_token_refresh, @config)
+      rescue StandardError => e
+        @config.logger.error(e.inspect)
+      end
+
+      private
+
+      def start_sse_unsynchronized
         response = @auth_api_client.authenticate(@api_key)
         @config.logger.debug("Auth service response push_enabled: #{response[:push_enabled]}") if @config.debug_enabled
 
@@ -33,18 +49,14 @@ module SplitIoClient
         true
       rescue StandardError => e
         @config.logger.error("start_sse: #{e.inspect}")
+        false
       end
 
-      def stop_sse
-        @sse_handler.stop
-        SplitIoClient::Helpers::ThreadHelper.stop(:schedule_next_token_refresh, @config)
-      rescue StandardError => e
-        @config.logger.error(e.inspect)
-      end
-
-      private
-
       def schedule_next_token_refresh(time)
+        # Reap first: start_sse is reachable both from the push status handler thread and
+        # from refresh_token_task itself, so without this an orphaned timer thread can
+        # survive with no handle to cancel it and fire its own reconnect later.
+        SplitIoClient::Helpers::ThreadHelper.reap(:schedule_next_token_refresh, @config)
         @config.threads[:schedule_next_token_refresh] = Thread.new { refresh_token_task(time) }
       end
 

data/lib/splitclient-rb/engine/reconnect_policy.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+module SplitIoClient
+  module Engine
+    # Governs how aggressively streaming reconnects are attempted.
+    #
+    # Extracted for FME-18143, where a customer's SDK reconnected up to ~64 times per
+    # second and accumulated ~10,000 threads. Two properties matter here:
+    #
+    #   * A connection only earns a back off reset if it stayed up long enough to count
+    #     as a genuine success. Resetting on every PUSH_CONNECTED meant a connection
+    #     that lived for milliseconds still produced a zero second retry.
+    #   * Only one reconnect may be in flight at a time, and errors describing an
+    #     already-replaced connection must not each start another one.
+    class ReconnectPolicy
+      # How long a connection must survive before it counts as stable.
+      MIN_UPTIME_FOR_BACKOFF_RESET = 30
+
+      def initialize(config, status_queue, back_off = nil)
+        @config = config
+        @status_queue = status_queue
+        @back_off = back_off || Engine::BackOff.new(1, 3)
+        @connected_at = Concurrent::AtomicReference.new(nil)
+        @reconnecting = Concurrent::AtomicBoolean.new(false)
+      end
+
+      # Monotonic clock: a backwards wall-clock jump (NTP correction, container
+      # migration) must not make uptime go negative and permanently block the
+      # back off reset.
+      def connected
+        @connected_at.set(monotonic_now)
+      end
+
+      # Whether a disconnect should be acted on at all. Returns false for a disconnect
+      # of an already-disconnected stream, and for a reconnect when one is already in
+      # flight (typically driven by the token refresh thread) -- running two
+      # concurrently is what multiplied connection attempts.
+      def actionable?(sse_connected, reconnect)
+        unless sse_connected || reconnect
+          log_debug('Streaming already disconnected.')
+          return false
+        end
+        return true unless reconnect && !acquire
+
+        log_debug('Streaming reconnect already in progress, ignoring.')
+        false
+      end
+
+      # Returns true only for the caller that acquired the right to reconnect; a
+      # concurrent caller gets false. AtomicBoolean#make_true is the compare-and-set:
+      # it returns true only for the transition from false to true.
+      def acquire
+        @reconnecting.make_true
+      end
+
+      def release
+        @reconnecting.make_false
+      end
+
+      # Resets the back off only if the connection that just dropped was stable, so a
+      # flapping connection keeps backing off instead of retrying immediately.
+      def record_disconnect
+        connected_at = @connected_at.get_and_set(nil)
+        return if connected_at.nil?
+
+        uptime = monotonic_now - connected_at
+        return if uptime < MIN_UPTIME_FOR_BACKOFF_RESET
+
+        log_debug("Streaming connection was up for #{uptime.round(1)}s, resetting back off.")
+        @back_off.reset
+      end
+
+      def interval
+        @back_off.interval
+      end
+
+      # Drops queued retryable errors, which refer to the connection just replaced.
+      # Acting on each of them would start one more reconnect apiece. Other statuses
+      # are preserved in order.
+      def discard_stale_errors
+        kept = []
+        discarded = 0
+
+        until @status_queue.empty?
+          status = @status_queue.pop(true)
+          if status == Constants::PUSH_RETRYABLE_ERROR
+            discarded += 1
+          else
+            kept << status
+          end
+        end
+
+        kept.each { |kept_status| @status_queue.push(kept_status) }
+        log_debug("Discarded #{discarded} stale streaming errors.") if discarded.positive?
+        discarded
+      rescue ThreadError
+        discarded
+      end
+
+      private
+
+      def log_debug(message)
+        @config.logger.debug(message) if @config.debug_enabled
+      end
+
+      def monotonic_now
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+    end
+  end
+end

data/lib/splitclient-rb/engine/sync_manager.rb

@@ -17,7 +17,7 @@ module SplitIoClient
         @push_manager = push_manager
         @status_queue = status_queue
         @sse_connected = Concurrent::AtomicBoolean.new(false)
-        @back_off = Engine::BackOff.new(1, 3)
+        @reconnect_policy = Engine::ReconnectPolicy.new(config, status_queue)
       end
 
       def start
@@ -110,30 +110,37 @@ module SplitIoClient
         @synchronizer.start_periodic_fetch
         record_telemetry(Telemetry::Domain::Constants::SYNC_MODE, SYNC_MODE_POLLING)
       rescue StandardError => e
-        @config.logger.error("process_connected error: #{e.inspect}")
+        @config.logger.error("process_forced_stop error: #{e.inspect}")
       end
 
       def process_disconnect(reconnect)
-        unless @sse_connected.value || reconnect
-          @config.logger.debug('Streaming already disconnected.') if @config.debug_enabled
-          return
-        end
+        return unless @reconnect_policy.actionable?(@sse_connected.value, reconnect)
 
+        @reconnect_policy.record_disconnect
         @sse_connected.make_false
         @sse_handler.stop_workers
         @synchronizer.start_periodic_fetch
         record_telemetry(Telemetry::Domain::Constants::SYNC_MODE, SYNC_MODE_POLLING)
 
-        if reconnect
-          wait_interval = @back_off.interval
-          @config.logger.debug("Retrying streaming connection in: #{wait_interval} seconds")
-          sleep(wait_interval)
-          @push_manager.stop_sse
-          @synchronizer.sync_all
-          @push_manager.start_sse
-        end
+        reconnect_streaming if reconnect
       rescue StandardError => e
         @config.logger.error("process_disconnect error: #{e.inspect}")
+        @reconnect_policy.release if reconnect
+      end
+
+      def reconnect_streaming
+        wait_interval = @reconnect_policy.interval
+        log_if_debug("Retrying streaming connection in: #{wait_interval} seconds")
+        sleep(wait_interval)
+        @push_manager.stop_sse
+        @synchronizer.sync_all
+
+        # Only discard queued errors once streaming is actually back up. If the retry
+        # failed, those errors are still the only thing that will drive the next
+        # attempt, so they must be left alone.
+        @reconnect_policy.discard_stale_errors if @push_manager.start_sse
+      ensure
+        @reconnect_policy.release
       end
 
       def record_telemetry(type, data)
@@ -148,36 +155,39 @@ module SplitIoClient
       end
 
       def incoming_push_status_handler
+        # Rescue lives inside the loop: a bug in any handler (including one
+        # that lacks its own rescue) would otherwise exit the loop and
+        # permanently stop the SDK from reacting to push status changes.
         while (status = @status_queue.pop)
-          @config.logger.debug("Push status handler dequeue #{status}") if @config.debug_enabled
-
-          case status
-          when Constants::PUSH_CONNECTED
-            @back_off.reset
-            process_connected
-          when Constants::PUSH_RETRYABLE_ERROR
-            process_disconnect(true)
-          when Constants::PUSH_FORCED_STOP
-            process_forced_stop
-          when Constants::PUSH_NONRETRYABLE_ERROR
-            process_disconnect(false)
-          when Constants::PUSH_SUBSYSTEM_DOWN
-            process_subsystem_down
-          when Constants::PUSH_SUBSYSTEM_READY
-            process_subsystem_ready
-          when Constants::PUSH_SUBSYSTEM_OFF
-            process_push_shutdown
-          else
-            log_if_debug('Incorrect push status type.')
+          begin
+            @config.logger.debug("Push status handler dequeue #{status}") if @config.debug_enabled
+            case status
+            when Constants::PUSH_CONNECTED
+              # Deliberately not resetting the back off here. Whether this
+              # connection earns a reset depends on how long it survives,
+              # which is only known once it drops -- see
+              # ReconnectPolicy#record_disconnect.
+              @reconnect_policy.connected
+              process_connected
+            when Constants::PUSH_RETRYABLE_ERROR    then process_disconnect(true)
+            when Constants::PUSH_FORCED_STOP        then process_forced_stop
+            when Constants::PUSH_NONRETRYABLE_ERROR then process_disconnect(false)
+            when Constants::PUSH_SUBSYSTEM_DOWN     then process_subsystem_down
+            when Constants::PUSH_SUBSYSTEM_READY    then process_subsystem_ready
+            when Constants::PUSH_SUBSYSTEM_OFF      then process_push_shutdown
+            else log_if_debug('Incorrect push status type.')
+            end
+          rescue StandardError => e
+            @config.logger.error("Push status handler error for #{status}: #{e.inspect}")
           end
         end
-      rescue StandardError => e
-        @config.logger.error("Push status handler error: #{e.inspect}")
       end
-    end
 
-    def log_if_debug(msg)
-      @config.logger.debug(msg) if @config.debug_enabled
+      # Was defined on the enclosing module, so every call raised NoMethodError and
+      # killed the push status handler thread (its rescue sits outside the loop).
+      def log_if_debug(msg)
+        @config.logger.debug(msg) if @config.debug_enabled
+      end
     end
   end
 end

data/lib/splitclient-rb/helpers/thread_helper.rb

@@ -6,6 +6,10 @@ module SplitIoClient
       def self.stop(thread_sym, config)
         thread = config.threads[thread_sym]
 
+        # Never kill the calling thread: stop_sse is reachable from inside the token
+        # refresh thread, which would otherwise terminate itself mid-flight.
+        return if thread == Thread.current
+
         unless thread.nil?
           config.logger.debug("Stopping #{thread_sym} thread...") if config.debug_enabled
           Thread.kill(thread)
@@ -14,6 +18,19 @@ module SplitIoClient
         config.logger.error(e.inspect)
       end
 
+      # Terminates the thread currently held in +thread_sym+ before its handle is
+      # replaced. Thread handles live in a single slot per symbol, so overwriting one
+      # while the thread is still running makes it unreachable and leaks it forever.
+      def self.reap(thread_sym, config, timeout = 1)
+        thread = config.threads[thread_sym]
+        return if thread.nil? || thread == Thread.current || !thread.alive?
+
+        config.logger.warn("#{thread_sym} thread is still alive, terminating it before starting a new one.")
+        Thread.kill(thread) unless thread.join(timeout)
+      rescue StandardError => e
+        config.logger.error(e.inspect)
+      end
+
       def self.alive?(thread_sym, config)
         thread = config.threads[thread_sym]
 

data/lib/splitclient-rb/sse/event_source/client.rb

@@ -32,20 +32,36 @@ module SplitIoClient
           @status_queue = status_queue
           @read_timeout = read_timeout
           @connected = Concurrent::AtomicBoolean.new(false)
-          @first_event = Concurrent::AtomicBoolean.new(true)
+          # Set while close is tearing a connection down on purpose, so the reader
+          # thread does not report the resulting socket error as a retryable failure.
+          @shutdown = Concurrent::AtomicBoolean.new(false)
+          # Incremented for every connection attempt. A reader thread whose generation
+          # is no longer current has been superseded and must exit without side effects.
+          @generation = Concurrent::AtomicFixnum.new(0)
           @socket = nil
+          # Serialises concurrent close calls so the second caller does not
+          # re-push the status or double-close a socket the first caller
+          # already claimed.
+          @close_mutex = Mutex.new
         end
 
         def close(status = nil)
-          return if @socket.nil?
-
-          @config.logger.debug("Closing SSEClient socket") if @config.debug_enabled
-          push_status(status)
-          @connected.make_false
-          @socket.sync_close = true if @socket.is_a? OpenSSL::SSL::SSLSocket
-          @socket.close
-          @config.logger.debug("SSEClient socket state #{@socket.state}") if @socket.is_a?(OpenSSL::SSL::SSLSocket) && @config.debug_enabled
-          @socket = nil
+          socket = nil
+          @close_mutex.synchronize do
+            return if @socket.nil?
+
+            @config.logger.debug('Closing SSEClient socket') if @config.debug_enabled
+            @shutdown.make_true
+            push_status(status)
+            @connected.make_false
+
+            socket = @socket
+            @socket = nil
+          end
+
+          socket.sync_close = true if socket.is_a? OpenSSL::SSL::SSLSocket
+          socket.close
+          @config.logger.debug("SSEClient socket state #{socket.state}") if socket.is_a?(OpenSSL::SSL::SSLSocket) && @config.debug_enabled
         rescue StandardError => e
           @config.logger.error("SSEClient close Error: #{e.inspect}")
         end
@@ -60,7 +76,10 @@ module SplitIoClient
           latch = Concurrent::CountDownLatch.new(1)
           connect_thread(latch)
 
-          return false unless latch.wait(CONNECT_TIMEOUT)
+          unless latch.wait(CONNECT_TIMEOUT)
+            @config.logger.warn("SSE connection attempt did not complete within #{CONNECT_TIMEOUT} seconds.")
+            return false
+          end
 
           connected?
         rescue StandardError => e
@@ -75,95 +94,143 @@ module SplitIoClient
         private
 
         def connect_thread(latch)
+          # Never orphan the previous reader thread: its handle lives in a single slot
+          # and would otherwise become unreachable, leaking the thread forever.
+          Helpers::ThreadHelper.reap(:connect_stream, @config)
+
+          generation = @generation.increment
+          @shutdown.make_false
+
           @config.threads[:connect_stream] = Thread.new do
             @config.logger.info('Starting connect_stream thread ...')
-            new_status = connect_stream(latch)
+            new_status = connect_stream(latch, generation)
             push_status(new_status) unless new_status.nil?
             @config.logger.info('connect_stream thread finished.')
           end
         end
 
-        def connect_stream(latch)
-          return Constants::PUSH_RETRYABLE_ERROR unless socket_write(latch)
-          while connected? || @first_event.value
-            begin 
-              if IO.select([@socket], nil, nil, @read_timeout)
+        # Each reader thread owns the socket it opened. Nothing about the connection is
+        # shared with later attempts, so a superseded thread can neither read from nor
+        # steal bytes off a socket belonging to a newer connection.
+        def connect_stream(latch, generation = nil)
+          generation ||= @generation.value
+          socket = open_socket(latch)
+          return retryable_error(generation) if socket.nil?
+
+          first_event = true
+
+          while current_generation?(generation)
+            begin
+              if IO.select([socket], nil, nil, @read_timeout)
                 begin
-                  partial_data = @socket.readpartial(10_000)
+                  partial_data = socket.readpartial(10_000)
 
-                  first_event_status = read_first_event(partial_data, latch)
-                  return first_event_status unless first_event_status.nil?
+                  if first_event
+                    first_event = false
+                    first_event_status = read_first_event(partial_data, latch, generation)
+                    return first_event_status unless first_event_status.nil?
+                  end
                 rescue IO::WaitReadable => e
                   @config.logger.debug("SSE client IO::WaitReadable transient error: #{e.inspect}") if @config.debug_enabled
-                  IO.select([@socket], nil, nil, @read_timeout)
+                  IO.select([socket], nil, nil, @read_timeout)
                   retry
-                rescue  Errno::EAGAIN => e
+                rescue Errno::EAGAIN => e
                   @config.logger.debug("SSE client transient error: #{e.inspect}") if @config.debug_enabled
-                  IO.select([@socket], nil, nil, @read_timeout)
+                  IO.select([socket], nil, nil, @read_timeout)
                   retry
                 rescue Errno::ETIMEDOUT => e
                   @config.logger.error("SSE read operation timed out!: #{e.inspect}")
-                  return Constants::PUSH_RETRYABLE_ERROR
+                  return retryable_error(generation)
                 rescue EOFError => e
                   @config.logger.error("SSE read operation EOF, server closed the connection, will reconnect: #{e.inspect}")
-                  return Constants::PUSH_RETRYABLE_ERROR
+                  return retryable_error(generation)
                 rescue Errno::EBADF, IOError => e
                   @config.logger.error("SSE read operation EBADF or IOError: #{e.inspect}")
-                  return Constants::PUSH_RETRYABLE_ERROR
+                  return retryable_error(generation)
                 rescue StandardError => e
                   @config.logger.error("SSE read operation StandardError: #{e.inspect}")
                   return nil if ENV['SPLITCLIENT_ENV'] == 'test'
 
                   @config.logger.error("Error reading partial data: #{e.inspect}")
-                  return Constants::PUSH_RETRYABLE_ERROR
+                  return retryable_error(generation)
                 end
               else
-                @config.logger.error("SSE read operation timed out, no data available.")
-                return Constants::PUSH_RETRYABLE_ERROR
+                @config.logger.error('SSE read operation timed out, no data available.')
+                return retryable_error(generation)
               end
-            rescue Exception => e
-              @config.logger.debug("SSE socket is not connected: #{e.inspect}") if @config.debug_enabled
-              return Constants::PUSH_RETRYABLE_ERROR
+            # Was `rescue Exception` logging at debug level only. That combination hid
+            # the reason for ~14,880 of ~15,000 reconnects during the FME-18143 incident.
+            rescue StandardError => e
+              @config.logger.warn("SSE socket is not connected: #{e.inspect}")
+              return retryable_error(generation)
             end
 
             process_data(partial_data)
           end
-          @config.logger.info("SSE read operation exited: #{connected?}")
 
+          @config.logger.debug('connect_stream superseded by a newer connection, exiting.') if @config.debug_enabled
           nil
         end
 
-        def socket_write(latch)
-          @first_event.make_true
-          @socket = socket_connect
-          @socket.puts(build_request(@uri))
-          true
+        # Single decision point for "should this thread ask SyncManager to reconnect?".
+        # An intentional close, or a connection that has already been superseded, must
+        # stay silent -- reporting either as a retryable error is what turned one
+        # dropped connection into a self-sustaining reconnect storm.
+        def retryable_error(generation)
+          unless report_error?(generation)
+            @config.logger.debug('SSE connection ended intentionally, not requesting a reconnect.') if @config.debug_enabled
+            return nil
+          end
+
+          Constants::PUSH_RETRYABLE_ERROR
+        end
+
+        def report_error?(generation)
+          current_generation?(generation) && !@shutdown.value
+        end
+
+        def current_generation?(generation)
+          generation == @generation.value
+        end
+
+        def open_socket(latch)
+          socket = socket_connect
+          raise IOError, "unable to open socket to #{@uri.host}" if socket.nil?
+
+          @socket = socket
+          socket.puts(build_request(@uri))
+          socket
         rescue StandardError => e
           @config.logger.error("Error during connecting to #{@uri.host}. Error: #{e.inspect}")
+          close_quietly(socket)
+          @socket = nil if @socket.equal?(socket)
           latch.count_down
-          false
+          nil
         end
 
-        def read_first_event(data, latch)
-          return unless @first_event.value
-
+        def read_first_event(data, latch, generation = nil)
+          generation ||= @generation.value
           response_code = @event_parser.first_event(data)
           @config.logger.debug("SSE client first event code: #{response_code}") if @config.debug_enabled
 
-          @first_event.make_false
-
           if response_code != OK_CODE
             @config.logger.error("SSE first event failed, code: #{response_code}")
             latch.count_down
-            return Constants::PUSH_RETRYABLE_ERROR
+            return retryable_error(generation)
+          end
+
+          # Only the connection that is still current may announce itself as connected.
+          if current_generation?(generation)
+            @connected.make_true
+            @config.logger.debug('SSE client first event Connected is true') if @config.debug_enabled
+            @telemetry_runtime_producer.record_streaming_event(Telemetry::Domain::Constants::SSE_CONNECTION_ESTABLISHED, nil)
+            push_status(Constants::PUSH_CONNECTED)
+          elsif @config.debug_enabled
+            @config.logger.debug('Ignoring first event from a superseded SSE connection.')
           end
 
-          @connected.make_true
-          @config.logger.debug("SSE client first event Connected is true") if @config.debug_enabled
-          @telemetry_runtime_producer.record_streaming_event(Telemetry::Domain::Constants::SSE_CONNECTION_ESTABLISHED, nil)
-          push_status(Constants::PUSH_CONNECTED)
           latch.count_down
-          return nil
+          nil
         end
 
         def socket_connect
@@ -184,9 +251,11 @@ module SplitIoClient
                 retry
               end
               return ssl_socket
-
-            rescue Exception => e
+            rescue StandardError => e
               @config.logger.error("socket connect error: #{e.inspect}")
+              # Without this the underlying descriptor is leaked on every failed
+              # attempt, which exhausts file descriptors during a reconnect storm.
+              close_quietly(tcp_socket)
               return nil
             end
           end
@@ -194,6 +263,12 @@ module SplitIoClient
           tcp_socket
         end
 
+        def close_quietly(socket)
+          socket.close unless socket.nil? || socket.closed?
+        rescue StandardError => e
+          @config.logger.debug("Error closing socket: #{e.inspect}") if @config.debug_enabled
+        end
+
         def process_data(partial_data)
           @config.logger.debug("Event partial data: #{partial_data}") if @config.debug_enabled
           return if partial_data.nil? || partial_data == KEEP_ALIVE_RESPONSE
@@ -247,7 +322,7 @@ module SplitIoClient
 
         def push_status(status)
           return if status.nil?
-          
+
           @config.logger.debug("Pushing new sse status: #{status}") if @config.debug_enabled
           @status_queue.push(status)
         end

data/lib/splitclient-rb/version.rb

@@ -1,3 +1,3 @@
 module SplitIoClient
-  VERSION = '8.11.1'
+  VERSION = '8.11.2.pre.rc1'
 end

data/lib/splitclient-rb.rb

@@ -126,6 +126,7 @@ require 'splitclient-rb/engine/models/valid_sdk_event.rb'
 require 'splitclient-rb/engine/models/event_active_subscriptions.rb'
 require 'splitclient-rb/engine/auth_api_client'
 require 'splitclient-rb/engine/back_off'
+require 'splitclient-rb/engine/reconnect_policy'
 require 'splitclient-rb/engine/fallback_treatment_calculator.rb'
 require 'splitclient-rb/engine/push_manager'
 require 'splitclient-rb/engine/status_manager'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: splitclient-rb
 version: !ruby/object:Gem::Version
-  version: 8.11.1
+  version: 8.11.2.pre.rc1
 platform: ruby
 authors:
 - Split Software
@@ -383,11 +383,9 @@ extensions:
 - ext/murmurhash/extconf.rb
 extra_rdoc_files: []
 files:
-- ".github/CODEOWNERS"
-- ".github/pull_request_template.md"
-- ".github/workflows/ci.yml"
-- ".github/workflows/update-license-year.yml"
 - ".gitignore"
+- ".harness/orgs/PROD/projects/Harness_Split/pipelines/rubyclientrepo/input_sets/rubyclient.yaml"
+- ".harness/pipeline.yaml"
 - ".rubocop.yml"
 - ".simplecov"
 - CHANGES.txt
@@ -518,6 +516,7 @@ files:
 - lib/splitclient-rb/engine/parser/evaluator.rb
 - lib/splitclient-rb/engine/parser/partition.rb
 - lib/splitclient-rb/engine/push_manager.rb
+- lib/splitclient-rb/engine/reconnect_policy.rb
 - lib/splitclient-rb/engine/status_manager.rb
 - lib/splitclient-rb/engine/sync_manager.rb
 - lib/splitclient-rb/engine/synchronizer.rb

data/.github/CODEOWNERS

@@ -1 +0,0 @@
-* @splitio/sdk

data/.github/pull_request_template.md

@@ -1,9 +0,0 @@
-# Ruby SDK
-
-## What did you accomplish?
-
-
-## How to test new changes?
-
-
-## Extra Notes

data/.github/workflows/ci.yml

@@ -1,90 +0,0 @@
-name: ci
-on:
-  pull_request:
-    branches:
-      - development
-      - master
-  push:
-    branches:
-      - development
-      - master
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
-  cancel-in-progress: true
-
-jobs:
-  test:
-    name: Test
-    runs-on: ubuntu-latest
-    services:
-      redis:
-        image: redis
-        ports:
-          - 6379:6379
-    strategy:
-      fail-fast: false
-      matrix:
-        version:
-          - '2.5.0'
-          - '3.2.2'
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-
-      - name: Set up Java
-        uses: actions/setup-java@v2
-        with:
-          java-version: 17
-          distribution: "temurin"
-
-      - name: Setup Ruby ${{ matrix.version }}
-        uses: ruby/setup-ruby@v1
-        with:
-          ruby-version: ${{ matrix.version }}
-
-      - name: Install dependencies
-        run: bundle install
-
-      - name: Run tests
-        run: bundle exec rake
-
-      - name: Fix code coverage paths
-        working-directory: coverage
-        run: |
-          sed -i 's@'$GITHUB_WORKSPACE'@/github/workspace/@g' .resultset.json
-          ruby -rjson -e 'sqube = JSON.load(File.read(".resultset.json"))["RSpec"]["coverage"].transform_values {|lines| lines["lines"]}; total = { "RSpec" => { "coverage" => sqube, "timestamp" => Time.now.to_i }}; puts JSON.dump(total)' > .resultset.sonarqube.json
-
-      - name: Set VERSION env
-        run: echo "VERSION=$(cat lib/splitclient-rb/version.rb | grep VERSION | awk -F "'" '{print $2}')" >> $GITHUB_ENV
-
-      - name: SonarQube Scan (Push)
-        if: matrix.version == '3.2.2' && github.event_name == 'push'
-        uses: SonarSource/sonarcloud-github-action@v5.0.0
-        env:
-          SONAR_TOKEN: ${{ secrets.SONARQUBE_TOKEN }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          projectBaseDir: .
-          args: >
-            -Dsonar.host.url=${{ secrets.SONARQUBE_HOST }}
-            -Dsonar.projectVersion=${{ env.VERSION }}
-
-      - name: SonarQube Scan (Pull Request)
-        if: matrix.version == '3.2.2' && github.event_name == 'pull_request'
-        uses: SonarSource/sonarcloud-github-action@v5.0.0
-        env:
-          SONAR_TOKEN: ${{ secrets.SONARQUBE_TOKEN }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          projectBaseDir: .
-          args: >
-            -Dsonar.java.source=17
-            -Dsonar.host.url=${{ secrets.SONARQUBE_HOST }}
-            -Dsonar.projectVersion=${{ env.VERSION }}
-            -Dsonar.pullrequest.key=${{ github.event.pull_request.number }}
-            -Dsonar.pullrequest.branch=${{ github.event.pull_request.head.ref }}
-            -Dsonar.pullrequest.base=${{ github.event.pull_request.base.ref }}

data/.github/workflows/update-license-year.yml

@@ -1,45 +0,0 @@
-name: Update License Year
-
-on:
-  schedule:
-    - cron: "0 3 1 1 *" # 03:00 AM on January 1
-
-permissions:
-  contents: write
-  pull-requests: write
-
-jobs:
-  test:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-
-      - name: Set Current year
-        run: "echo CURRENT=$(date +%Y) >> $GITHUB_ENV"
-
-      - name: Set Previous Year
-        run: "echo PREVIOUS=$(($CURRENT-1)) >> $GITHUB_ENV"
-
-      - name: Update LICENSE
-        uses: jacobtomlinson/gha-find-replace@v2
-        with:
-          find: ${{ env.PREVIOUS }}
-          replace: ${{ env.CURRENT }}
-          include: "LICENSE"
-          regex: false
-
-      - name: Commit files
-        run: |
-          git config user.name 'github-actions[bot]'
-          git config user.email 'github-actions[bot]@users.noreply.github.com'
-          git commit -m "Updated License Year" -a
-
-      - name: Create Pull Request
-        uses: peter-evans/create-pull-request@v3
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          title: Update License Year
-          branch: update-license