promoted-ruby-client 0.1.6 → 0.1.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 30ba0e172809a6aa76284b4b88980fa45dea5b2bf28cdda4fbe8e672b7510a83
-  data.tar.gz: e28586ed23f7db16b959bb9819072e502fc7af3805e68938f2bbf421fdc9e4d8
+  metadata.gz: 6339ed023cca804b02e1c118466d80dba1635d603f69c3d9a3789c9be8be1895
+  data.tar.gz: 75dce2d318c3d817bf0b4a6bcea20a4d6629cdf160e1a1f7973985c5cd8b57ae
 SHA512:
-  metadata.gz: 4e6b42f73ff30726aa81af1c08b97a0cd0e0307685dec1f90dcce5d0d61d4009c3a2e6179af77a600038531101db187a83dc52f46a7c0ad4a99f7c6c2530b43e
-  data.tar.gz: f3d304f0a2aad7acc15229750f97b021da4c0bfe8deae40c6ac1110bc7c1621ab0db19635464ba835e4e9a9ec07b9cfa0c231a9163fa0e1dcffeeebfdc1e51fc
+  metadata.gz: 4f73e120f23ec6e4c5049f26c6ca6d3418d660514d063e192631c9200fee5c165433e6ebb29858c7171a9c65b187eceae670ef57a2721835f733c2aaccf5e6aa
+  data.tar.gz: e3bad8867640b2ba17d92f7154366fa41a25b7eb16e9dac7a4b0a5ba9ffed52a1da749ee9aece0b6c05956158569539725306751baf576debac5fa5f6b922e6e

data/.gitignore

@@ -57,4 +57,7 @@ build-iPhoneSimulator/
 # .rubocop-https?--*
 
 # IDE
-.vscode/
+.vscode/
+
+# Dev
+dev/

data/Gemfile

@@ -8,6 +8,7 @@ gemspec
 gem 'faraday', '~> 1.4.1'
 gem 'faraday_middleware'
 gem 'faraday-net_http'
+gem 'net-http-persistent'
 gem 'concurrent-ruby', require: 'concurrent'
 
 group :development do

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    promoted-ruby-client (0.1.6)
+    promoted-ruby-client (0.1.10)
 
 GEM
   remote: https://rubygems.org/
@@ -10,6 +10,7 @@ GEM
     backport (1.2.0)
     benchmark (0.1.1)
     concurrent-ruby (1.1.9)
+    connection_pool (2.2.5)
     debase (0.2.5.beta2)
       debase-ruby_core_source (>= 0.10.12)
     debase-ruby_core_source (0.10.12)
@@ -38,6 +39,8 @@ GEM
       kramdown (~> 2.0)
     mini_portile2 (2.5.3)
     multipart-post (2.1.1)
+    net-http-persistent (4.0.1)
+      connection_pool (~> 2.2)
     nokogiri (1.11.7)
       mini_portile2 (~> 2.5.0)
       racc (~> 1.4)
@@ -116,6 +119,7 @@ DEPENDENCIES
   faraday-net_http
   faraday_middleware
   jaro_winkler
+  net-http-persistent
   promoted-ruby-client!
   rake (~> 10.0)
   rspec (~> 3.0)

data/README.md

@@ -28,10 +28,10 @@ This client will suffice for building log requests. To send actually send traffi
 
 ```rb
 client = Promoted::Ruby::Client::PromotedClient.new({
-  :metrics_endpoint = "https://<get this from Promoted>",
-  :delivery_endpoint = "https://<get this from Promoted>",
-  :metrics_api_key = "<get this from Promoted>",
-  :delivery_api_key = "<get this from Promoted>"
+  :metrics_endpoint => "https://<get this from Promoted>",
+  :delivery_endpoint => "https://<get this from Promoted>",
+  :metrics_api_key => "<get this from Promoted>",
+  :delivery_api_key => "<get this from Promoted>"
 })
 ```
 
@@ -88,7 +88,7 @@ Field Name | Type | Optional? | Description
 ---------- | ---- | --------- | -----------
 ```:user_info``` | UserInfo | Yes | The user info structure.
 ```:insertion_id``` | String | Yes | Generated by the SDK (*do not set*)
-```:request_id``` | String | Yes | Generated by the SDK (*do not set*)
+```:request_id``` | String | Yes | Generated by the SDK when needed (*do not set*)
 ```:content_id``` | String | No | Identifier for the content to be shown, must be set.
 ```:properties``` | Properties | Yes | Any additional custom properties to associate. For v1 integrations, it is fine not to fill in all the properties.
 
@@ -101,7 +101,7 @@ A request for content insertions.
 Field Name | Type | Optional? | Description
 ---------- | ---- | --------- | -----------
 ```:user_info``` | UserInfo | Yes | The user info structure.
-```:request_id``` | String | Yes | Generated by the SDK (*do not set*)
+```:request_id``` | String | Yes | Generated by the SDK when needed (*do not set*)
 ```:use_case``` | String | Yes | One of the use case values, i.e. 'FEED' (see [constants.rb](https://github.com/promotedai/promoted-ruby-client/blob/main/lib/promoted/ruby/client/constants.rb)).
 ```:properties``` | Properties | Yes | Any additional custom properties to associate.
 ```:paging``` | Paging | Yes | Paging parameters (see TODO)

data/dev.md

@@ -4,5 +4,5 @@
 2. Get credentials for deployment from 1password.
 3. Modify `promoted-ruby-client.gemspec`'s push block.
 4. Run `gem build promoted-ruby-client.gemspec` to generate `gem`.
-5. Run (using new output) `gem push promoted-ruby-client-0.1.6.gem`
+5. Run (using new output) `gem push promoted-ruby-client-0.1.10.gem`
 6. Update README with new version.

data/lib/promoted/ruby/client.rb

@@ -19,8 +19,16 @@ module Promoted
         class Error < StandardError; end
 
         attr_reader :perform_checks, :default_only_log, :delivery_timeout_millis, :metrics_timeout_millis, :should_apply_treatment_func,
-                    :default_request_headers, :http_client
+                    :default_request_headers, :http_client, :logger, :shadow_traffic_delivery_percent, :async_shadow_traffic
+                    
+        attr_accessor :request_logging_on, :enabled
         
+        ##
+        # Whether or not the client is currently enabled for execution.
+        def enabled?
+          @enabled
+        end
+
         ##            
         # A common compact method implementation.
         def self.copy_and_remove_properties
@@ -39,7 +47,8 @@ module Promoted
             @perform_checks = params[:perform_checks]
           end
 
-          @logger = params[:logger] # Example:  Logger.new(STDERR, :progname => "promotedai")
+          @logger               = params[:logger] # Example:  Logger.new(STDERR, :progname => "promotedai")
+          @request_logging_on   = params[:request_logging_on] || false
 
           @default_request_headers = params[:default_request_headers] || {}
           @metrics_api_key = params[:metrics_api_key] || ''
@@ -52,6 +61,7 @@ module Promoted
           raise ArgumentError.new("Invalid shadow_traffic_delivery_percent, must be between 0 and 1") if @shadow_traffic_delivery_percent < 0 || @shadow_traffic_delivery_percent > 1.0
 
           @sampler = Sampler.new
+          @pager   = Pager.new
 
           # HTTP Client creation
           @delivery_endpoint = params[:delivery_endpoint] || DEFAULT_DELIVERY_ENDPOINT
@@ -66,21 +76,36 @@ module Promoted
           @http_client = FaradayHTTPClient.new
           @validator = Promoted::Ruby::Client::Validator.new
 
-          # Thread pool to process delivery of shadow traffic. Will silently drop excess requests beyond the queue
-          # size, and silently eat errors on the background threads.
-          @pool = Concurrent::ThreadPoolExecutor.new(
-            min_threads: 0,
-            max_threads: 10,
-            max_queue: 100,
-            fallback_policy: :discard
-          )
+          @async_shadow_traffic = true
+          if params[:async_shadow_traffic] != nil
+            @async_shadow_traffic = params[:async_shadow_traffic] || false
+          end
+
+          @pool = nil
+          if @async_shadow_traffic
+            # Thread pool to process delivery of shadow traffic. Will silently drop excess requests beyond the queue
+            # size, and silently eat errors on the background threads.
+            @pool = Concurrent::ThreadPoolExecutor.new(
+              min_threads: 0,
+              max_threads: 10,
+              max_queue: 100,
+              fallback_policy: :discard
+            )
+          end
+
+          @enabled = true
+          if params[:enabled] != nil
+            @enabled = params[:enabled] || false
+          end
         end
         
         ##
         # Politely shut down a Promoted client.
         def close
-          @pool.shutdown
-          @pool.wait_for_termination
+          if @pool
+            @pool.shutdown
+            @pool.wait_for_termination
+          end
         end
 
         ##
@@ -88,19 +113,34 @@ module Promoted
         def deliver args, headers={}
           args = Promoted::Ruby::Client::Util.translate_args(args)
 
+          # Respect the enabled state
+          if !@enabled
+            return {
+              insertion: @pager.apply_paging(args[:full_insertion], Promoted::Ruby::Client::INSERTION_PAGING_TYPE['UNPAGED'], args[:request][:paging])
+              # No log request returned when disabled
+            }
+          end
+          
           delivery_request_builder = RequestBuilder.new
           delivery_request_builder.set_request_params(args)
 
           perform_common_checks!(args) if @perform_checks
 
-          pre_delivery_fillin_fields delivery_request_builder
+          delivery_request_builder.ensure_client_timestamp
   
           response_insertions = []
           cohort_membership_to_log = nil
-          insertions_from_promoted = false
+          insertions_from_delivery = false
 
           only_log = delivery_request_builder.only_log != nil ? delivery_request_builder.only_log : @default_only_log
           deliver_err = false
+
+          if !@pager.validate_paging(delivery_request_builder.full_insertion, delivery_request_builder.request[:paging])
+            # Invalid input, log and do SDK-side delivery.
+            @logger.warn("Invalid paging parameters") if @logger
+            only_log = true
+          end
+
           if !only_log
             cohort_membership_to_log = delivery_request_builder.new_cohort_membership_to_log
 
@@ -117,22 +157,16 @@ module Promoted
                 @logger.error("Error calling delivery: " + err.message) if @logger
               end
               
-              insertions_from_promoted = (response != nil && !deliver_err);
+              insertions_from_delivery = (response != nil && !deliver_err);
               response_insertions = delivery_request_builder.fill_details_from_response(
                 response ? response[:insertion] : [])
             end
           end
   
           request_to_log = nil
-          if !insertions_from_promoted then
+          if !insertions_from_delivery then
             request_to_log = delivery_request_builder.request
-            size = delivery_request_builder.request.dig(:paging, :size)
-            response_insertions = size != nil ? delivery_request_builder.full_insertion[0..size] : delivery_request_builder.full_insertion
-          end
-
-          if request_to_log
-            request_to_log[:request_id] = SecureRandom.uuid if not request_to_log[:request_id]
-            add_missing_ids_on_insertions! request_to_log, response_insertions
+            response_insertions = @pager.apply_paging(delivery_request_builder.full_insertion, Promoted::Ruby::Client::INSERTION_PAGING_TYPE['UNPAGED'], delivery_request_builder.request[:paging])
           end
 
           log_req = nil
@@ -150,14 +184,12 @@ module Promoted
             log_request_builder.platform_id = delivery_request_builder.platform_id
             log_request_builder.timing      = delivery_request_builder.timing
             log_request_builder.user_info   = delivery_request_builder.user_info
-            pre_delivery_fillin_fields log_request_builder
-
 
             # On a successful delivery request, we don't log the insertions
             # or the request since they are logged on the server-side.
             log_req = log_request_builder.log_request_params(
-              include_insertions: !insertions_from_promoted, 
-              include_request: !insertions_from_promoted)
+              include_insertions: !insertions_from_delivery, 
+              include_request: !insertions_from_delivery)
           end
 
           client_response = {
@@ -173,6 +205,12 @@ module Promoted
         def prepare_for_logging args, headers={}
           args = Promoted::Ruby::Client::Util.translate_args(args)
 
+          if !@enabled
+            return {
+              insertion: args[:full_insertion]
+            }
+          end
+
           log_request_builder = RequestBuilder.new
 
           # Note: This method expects as JSON (string keys) but internally, RequestBuilder
@@ -188,7 +226,7 @@ module Promoted
             end
           end
           
-          pre_delivery_fillin_fields log_request_builder
+          log_request_builder.ensure_client_timestamp
 
           if !shadow_traffic_err && should_send_as_shadow_traffic?
             deliver_shadow_traffic args, headers
@@ -211,29 +249,44 @@ module Promoted
         private
 
         def send_request payload, endpoint, timeout_millis, api_key, headers={}, send_async=false
+          resp = nil
+
           headers["x-api-key"] = api_key
           use_headers = @default_request_headers.merge headers
           
-          if send_async
+          if @request_logging_on && @logger
+            @logger.info("promotedai") {
+              "Sending #{payload.to_json} to #{endpoint}"
+            }
+          end
+
+          if send_async && @pool
             @pool.post do
-              @http_client.send(endpoint, timeout_millis, payload, use_headers)
+              start_time = Time.now
+              begin
+                resp = @http_client.send(endpoint, timeout_millis, payload, use_headers)
+              rescue Faraday::Error => err
+                @logger.warn("Async send_request failed with #{err}") if @logger
+                return
+              end
+
+              ellapsed_time = Time.now - start_time
+              @logger.debug("Async send_request completed in #{ellapsed_time.to_f * 1000} ms") if @logger
             end
           else
+            start_time = Time.now
             begin
-              @http_client.send(endpoint, timeout_millis, payload, use_headers)
+              resp = @http_client.send(endpoint, timeout_millis, payload, use_headers)
             rescue Faraday::Error => err
+              @logger.warn("Sync send_request failed with #{err}") if @logger
               raise EndpointError.new(err)
             end
-          end
-        end
 
+            ellapsed_time = Time.now - start_time
+            @logger.debug("Sync send_request completed in #{ellapsed_time.to_f * 1000} ms") if @logger
+        end
 
-        def add_missing_ids_on_insertions! request, insertions
-          insertions.each do |insertion|
-            insertion[:insertion_id] = SecureRandom.uuid if not insertion[:insertion_id]
-            insertion[:session_id] = request[:session_id] if request[:session_id]
-            insertion[:request_id] = request[:request_id] if request[:request_id]
-          end
+          return resp
         end
 
         def should_send_as_shadow_traffic?
@@ -250,8 +303,19 @@ module Promoted
           delivery_request_params = delivery_request_builder.delivery_request_params(should_compact: false)
           delivery_request_params[:client_info][:traffic_type] = Promoted::Ruby::Client::TRAFFIC_TYPE['SHADOW']
 
-          # Call Delivery API async (fire and forget)
-          send_request(delivery_request_params, @delivery_endpoint, @delivery_timeout_millis, @delivery_api_key, headers, true)
+          # Call Delivery API and log/ignore errors.
+          start_time = Time.now
+          begin
+            send_request(delivery_request_params, @delivery_endpoint, @delivery_timeout_millis, @delivery_api_key, headers, @async_shadow_traffic)
+          rescue StandardError => err
+            @logger.warn("Shadow traffic call failed with #{err}") if @logger
+            return
+          end
+          
+          if !@async_shadow_traffic
+            ellapsed_time = Time.now - start_time
+            @logger.info("Shadow traffic call completed in #{ellapsed_time.to_f * 1000} ms") if @logger
+          end
         end
 
         def perform_common_checks!(req)
@@ -272,14 +336,7 @@ module Promoted
             return true if !cohort_membership[:arm]
             return cohort_membership[:arm] != Promoted::Ruby::Client::COHORT_ARM['CONTROL']
           end
-        end
-        
-        # TODO: This probably just goes better in the RequestBuilder class.
-        def pre_delivery_fillin_fields(log_request_builder)
-          if log_request_builder.timing[:client_log_timestamp].nil?
-            log_request_builder.timing[:client_log_timestamp] = Time.now.to_i
-          end
-        end
+        end        
       end
     end
   end
@@ -287,7 +344,9 @@ end
 
 # dependent /libs
 require "promoted/ruby/client/request_builder"
+require "promoted/ruby/client/pager"
 require "promoted/ruby/client/sampler"
 require "promoted/ruby/client/util"
 require "promoted/ruby/client/validator"
-require 'securerandom'
+require 'securerandom'
+require 'time'

data/lib/promoted/ruby/client/faraday_http_client.rb

@@ -11,7 +11,7 @@ module Promoted
                     f.request :json
                     f.request :retry, max: 3
                     f.use Faraday::Response::RaiseError # raises on 4xx and 5xx responses
-                    f.adapter :net_http
+                    f.adapter :net_http_persistent
                 end
             end
 

data/lib/promoted/ruby/client/id_generator.rb

@@ -0,0 +1,15 @@
+module Promoted
+    module Ruby
+      module Client
+        class IdGenerator
+            def initialize;end
+
+            def newID
+                SecureRandom.uuid
+            end
+        end
+    end
+  end
+end
+
+require 'securerandom'

data/lib/promoted/ruby/client/pager.rb

@@ -0,0 +1,57 @@
+module Promoted
+    module Ruby
+      module Client
+        class Pager
+            def validate_paging (insertions, paging)
+              if paging && paging[:offset]
+                return paging[:offset] < insertions.length
+              end
+              return true          
+            end
+
+            def apply_paging (insertions, insertion_page_type, paging = nil)
+              # This is invalid input, stop it before it goes to the server.
+              if !validate_paging(insertions, paging)
+                return []
+              end
+
+              if !paging
+                paging = {
+                  :offset => 0,
+                  :size => insertions.length
+                }
+              end
+
+              offset = [0, paging[:offset]].max
+
+              index = offset
+              if insertion_page_type == Promoted::Ruby::Client::INSERTION_PAGING_TYPE['PRE_PAGED']
+                # When insertions are pre-paged, we don't use offset to
+                # window into the provided insertions, although we do use it when
+                # assigning positions.
+                index = 0
+              end
+
+              size = paging[:size]
+              if size <= 0
+                size = insertions.length
+              end
+
+              final_insertion_size = [size, insertions.length].min
+              insertion_page = Array.new(final_insertion_size)
+              0.upto(final_insertion_size - 1) {|i|
+                insertion = insertions[index]
+                if insertion[:position] == nil
+                  insertion[:position] = offset
+                end
+                insertion_page[i] = insertion
+                index = index + 1
+                offset = offset + 1
+              }
+
+              return insertion_page
+            end
+        end
+      end
+   end
+end

data/lib/promoted/ruby/client/request_builder.rb

@@ -8,7 +8,13 @@ module Promoted
 
         attr_accessor :timing, :user_info, :platform_id
 
-        def initialize;end
+        def initialize args = {}
+          if args[:id_generator]
+            @id_generator = args[:id_generator]
+          else
+            @id_generator = IdGenerator.new
+          end
+        end
 
         # Populates request parameters from the given arguments, presumed to be a hash of symbols.
         def set_request_params args = {}
@@ -21,11 +27,13 @@ module Promoted
           @view_id                 = request[:view_id]
           @use_case                = Promoted::Ruby::Client::USE_CASES[request[:use_case]] || Promoted::Ruby::Client::USE_CASES['UNKNOWN_USE_CASE']
           @full_insertion          = args[:full_insertion]
-          @request_id              = SecureRandom.uuid
           @user_info               = request[:user_info] || { :user_id => nil, :log_user_id => nil}
           @timing                  = request[:timing] || { :client_log_timestamp => Time.now.to_i }
           @to_compact_metrics_insertion_func       = args[:to_compact_metrics_insertion_func]
           @to_compact_delivery_insertion_func      = args[:to_compact_delivery_insertion_func]
+
+          # If the user didn't create a client request id, we do it for them.
+          request[:client_request_id] = request[:client_request_id] || @id_generator.newID
         end
 
         # Only used in delivery
@@ -50,13 +58,13 @@ module Promoted
             timing: timing,
             client_info: @client_info.merge({ :client_type => Promoted::Ruby::Client::CLIENT_TYPE['PLATFORM_SERVER'] }),
             platform_id: @platform_id,
-            request_id: @request_id,
             view_id: @view_id,
             session_id: @session_id,
             use_case: @use_case,
             search_query: request[:search_query],
             properties: request[:properties],
-            paging: request[:paging]
+            paging: request[:paging],
+            client_request_id: request[:client_request_id]
           }
           params[:insertion] = should_compact ? compact_delivery_insertions : full_insertion
 
@@ -94,8 +102,17 @@ module Promoted
             cohort_membership: @experiment,
             client_info: @client_info
           }
-          params[:request] = [request] if include_request
-          params[:insertion] = compact_metrics_insertions if include_insertions
+
+          # Log request allows for multiple requests but here we only send one.
+          if include_request
+            request[:request_id] = request[:request_id] || @id_generator.newID
+            params[:request] = [request]
+          end
+
+          if include_insertions
+            params[:insertion] = compact_metrics_insertions if include_insertions
+            add_missing_ids_on_insertions! request, params[:insertion]
+          end
           
           params.clean!
         end
@@ -108,6 +125,12 @@ module Promoted
           end
         end
 
+        def ensure_client_timestamp
+          if timing[:client_log_timestamp].nil?
+            timing[:client_log_timestamp] = Time.now.to_i
+          end
+        end
+
         # TODO: This looks overly complicated.
         def compact_metrics_insertions
           @insertion            = [] # insertion should be set according to the compact insertion
@@ -126,7 +149,7 @@ module Promoted
             insertion_obj                = Hash[insertion_obj]
             insertion_obj[:user_info]    = user_info
             insertion_obj[:timing]       = timing
-            insertion_obj[:insertion_id] = SecureRandom.uuid # generate random UUID
+            insertion_obj[:insertion_id] = @id_generator.newID
             insertion_obj[:request_id]   = request_id
             insertion_obj[:position]     = offset + index
             insertion_obj                = @to_compact_metrics_insertion_func.call(insertion_obj) if @to_compact_metrics_insertion_func
@@ -137,6 +160,14 @@ module Promoted
 
         private
 
+        def add_missing_ids_on_insertions! request, insertions
+          insertions.each do |insertion|
+            insertion[:insertion_id] = @id_generator.newID if not insertion[:insertion_id]
+            insertion[:session_id] = request[:session_id] if request[:session_id]
+            insertion[:request_id] = request[:request_id] if request[:request_id]
+          end
+        end
+
         # A list of the response Insertions.  This client expects lists to be truncated
         # already to request.paging.size.  If not truncated, this client will truncate
         # the list.
@@ -148,6 +179,6 @@ module Promoted
   end
 end
 
-require 'securerandom'
 require "promoted/ruby/client/constants"
 require "promoted/ruby/client/extensions"
+require "promoted/ruby/client/id_generator"

data/lib/promoted/ruby/client/version.rb

@@ -1,7 +1,7 @@
 module Promoted
   module Ruby
     module Client
-      VERSION = "0.1.6"
+      VERSION = "0.1.10"
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: promoted-ruby-client
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.10
 platform: ruby
 authors:
 - scottmcmaster
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-07-07 00:00:00.000000000 Z
+date: 2021-07-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -75,6 +75,8 @@ files:
 - lib/promoted/ruby/client/errors.rb
 - lib/promoted/ruby/client/extensions.rb
 - lib/promoted/ruby/client/faraday_http_client.rb
+- lib/promoted/ruby/client/id_generator.rb
+- lib/promoted/ruby/client/pager.rb
 - lib/promoted/ruby/client/request_builder.rb
 - lib/promoted/ruby/client/sampler.rb
 - lib/promoted/ruby/client/util.rb