moesif_rack 1.3.9 → 1.4.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8f5cd1d13df1da4833bc55a10462721bdabae0a09cb6d8ab6398c378bd546975
-  data.tar.gz: ab2e62d3c6c4f76532bf8dbad38b4fdb2747413897e72228dc1ac23265d134c9
+  metadata.gz: e8bde32f3f60b62bfe1dfd5aef52ba6c56f21db3e2f8ac78743972c227ff1d2b
+  data.tar.gz: d0fe24cc62aa794dab07b04fa6ce943f85d015db045eb654cd66e9827e9b97fe
 SHA512:
-  metadata.gz: f42082b67c965208ba8d13bb2552a2b44ee08c3930ed58825838804431bfda1655acbcd82099ea0d0baec4573e31c0db4dc91dc95a7c2e7b5f23fb401e5f7076
-  data.tar.gz: ac4f6d15abbb9cce97e8206a158666883a002575d9921b4c64c54a3f38f384034e189c5dbb26ae44ac6636b66fb8a842fee1ad02b09e3c59b9fd9408815f813e
+  metadata.gz: 0db637f28ba8d87d7e5e8a81b9642eeeb64f2c701d862c941e90ac0bdc5a2d15584f597df82420b736020dcd8f426c87e1e727cb9dcf081214e9aeecc07fdcd9
+  data.tar.gz: 134f36829793337b6d78a7a51dedddeadd2d4e2fc1353e5b0ae5962f7ddb72f08562c5ff9f5f93248b7ee80db14a987f4935c58eb152b0229bfae9f7ea9e128c

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2019 Moesif, Inc
+Copyright (c) 2020 Moesif, Inc
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

data/README.md

@@ -22,7 +22,7 @@ gem install moesif_rack
 and if you have a `Gemfile` in your project, please add this line to
 
 ```
-gem 'moesif_rack', '~> 1.3.9'
+gem 'moesif_rack', '~> 1.4.2'
 
 ```
 
@@ -126,12 +126,10 @@ Optional.
 identify_user is a Proc that takes env, headers, and body as arguments and returns a user_id string. This helps us attribute requests to unique users. Even though Moesif can automatically retrieve the user_id without this, this is highly recommended to ensure accurate attribution.
 
 ```ruby
-
 moesif_options['identify_user'] = Proc.new { |env, headers, body|
 
-  #snip
-
-  'my_user_id'
+  # Add your custom code that returns a string for user id
+  '12345'
 }
 
 ```
@@ -145,13 +143,24 @@ identify_company is a Proc that takes env, headers, and body as arguments and re
 
 moesif_options['identify_company'] = Proc.new { |env, headers, body|
 
-  #snip
-
-  'my_company_id'
+  # Add your custom code that returns a string for company id
+  '67890'
 }
 
 ```
 
+#### __`identify_session`__
+
+Optional. A Proc that takes env, headers, body and returns a string.
+
+```ruby
+
+moesif_options['identify_session'] = Proc.new { |env, headers, body|
+    # Add your custom code that returns a string for session/API token
+    'XXXXXXXXX'
+}
+```
+
 #### __`get_metadata`__
 
 Optional.
@@ -162,31 +171,15 @@ metadata to this event.
 ```ruby
 
 moesif_options['get_metadata'] = Proc.new { |env, headers, body|
-
-  #snip
+  # Add your custom code that returns a dictionary
   value = {
-      'foo'  => 'abc',
-      'bar'  => '123'
+      'datacenter'  => 'westus',
+      'deployment_version'  => 'v1.2.3'
   }
-
   value
 }
 ```
 
-#### __`identify_session`__
-
-Optional. A Proc that takes env, headers, body and returns a string.
-
-```ruby
-
-moesif_options['identify_session'] = Proc.new { |env, headers, body|
-
-  #snip
-
-  'the_session_token'
-}
-
-```
 
 #### __`mask_data`__
 
@@ -196,9 +189,8 @@ With mask_data, you can make modifications to headers or body of the event befor
 ```ruby
 
 moesif_options['mask_data'] = Proc.new { |event_model|
-
-  #snip
-
+  # Add your custom code that returns a event_model after modifying any fields
+  event_model.response.body.password = nil
   event_model
 }
 
@@ -213,9 +205,7 @@ Optional. A Proc that takes env, headers, body and returns a boolean.
 ```ruby
 
 moesif_options['skip'] = Proc.new { |env, headers, body|
-
-  #snip
-
+  # Add your custom code that returns true to skip logging the API call
   false
 }
 
@@ -232,7 +222,7 @@ Optional. Boolean. Default false. If true, it will print out debug messages. In
 
 Optional. Boolean. Default true. If false, will not log request and response body to Moesif.
 
-#### __`capture_outoing_requests`__
+#### __`capture_outgoing_requests`__
 Optional. boolean, Default `false`. Set to `true` to capture all outgoing API calls from your app to third parties like Stripe, Github or to your own dependencies while using [Net::HTTP](https://ruby-doc.org/stdlib-2.6.3/libdoc/net/http/rdoc/Net/HTTP.html) package. The options below is applied to outgoing API calls. When the request is outgoing, for options functions that take request and response as input arguments, the request and response objects passed in are [Request](https://www.rubydoc.info/stdlib/net/Net/HTTPRequest) request and [Response](https://www.rubydoc.info/stdlib/net/Net/HTTPResponse) response objects.
 
 
@@ -245,9 +235,8 @@ identify_user_outgoing is a Proc that takes request and response as arguments an
 
 moesif_options['identify_user_outgoing'] = Proc.new { |request, response|
 
-  #snip
-
-  'the_user_id'
+  # Add your custom code that returns a string for user id
+  '12345'
 }
 
 ```
@@ -261,9 +250,8 @@ identify_company_outgoing is a Proc that takes request and response as arguments
 
 moesif_options['identify_company_outgoing'] = Proc.new { |request, response|
 
-  #snip
-
-  'the_company_id'
+  # Add your custom code that returns a string for company id
+  '67890'
 }
 
 ```
@@ -279,12 +267,11 @@ metadata to this event.
 
 moesif_options['get_metadata_outgoing'] = Proc.new { |request, response|
 
-  #snip
+  # Add your custom code that returns a dictionary
   value = {
-      'foo'  => 'abc',
-      'bar'  => '123'
+      'datacenter'  => 'westus',
+      'deployment_version'  => 'v1.2.3'
   }
-
   value
 }
 ```
@@ -297,9 +284,8 @@ Optional. A Proc that takes request, response and returns a string.
 
 moesif_options['identify_session_outgoing'] = Proc.new { |request, response|
 
-  #snip
-
-  'the_session_token'
+    # Add your custom code that returns a string for session/API token
+    'XXXXXXXXX'
 }
 
 ```
@@ -312,8 +298,7 @@ Optional. A Proc that takes request, response and returns a boolean. If `true` w
 
 moesif_options['skip_outgoing'] = Proc.new{ |request, response|
 
-  #snip
-
+  # Add your custom code that returns true to skip logging the API call
   false
 }
 
@@ -328,8 +313,8 @@ With mask_data_outgoing, you can make modifications to headers or body of the ev
 
 moesif_options['mask_data_outgoing'] = Proc.new { |event_model|
 
-  #snip
-
+  # Add your custom code that returns a event_model after modifying any fields
+  event_model.response.body.password = nil
   event_model
 }
 
@@ -341,103 +326,169 @@ Optional. Boolean. Default true. If false, will not log request and response bod
 
 ## Update User
 
-### update_user method
-A method is attached to the moesif middleware object to update the user profile or metadata.
-The metadata field can be any custom data you want to set on the user. The `user_id` field is required.
+### Update a Single User
+Create or update a user profile in Moesif.
+The metadata field can be any customer demographic or other info you want to store.
+Only the `user_id` field is required.
+This method is a convenient helper that calls the Moesif API lib.
+For details, visit the [Ruby API Reference](https://www.moesif.com/docs/api?ruby#update-a-user).
 
 ```ruby
-metadata = JSON.parse('{'\
-      '"email": "testrubyapi@user.com",'\
-      '"name": "ruby api user",'\
-      '"custom": "testdata"'\
-    '}')
-
-campaign_model = {"utm_source" => "Newsletter",
-                  "utm_medium" => "Email"}
+metadata = {
+  :email => 'john@acmeinc.com',
+  :first_name => 'John',
+  :last_name => 'Doe',
+  :title => 'Software Engineer',
+  :salesInfo => {
+      :stage => 'Customer',
+      :lifetime_value => 24000,
+      :accountOwner => 'mary@contoso.com',
+  }
+}
 
-user_model = { "user_id" => "12345",
-                "company_id" => "67890",
-                "modified_time" => Time.now.utc.iso8601, 
-                "metadata" => metadata,
-                "campaign" => campaign_model }
+# Campaign object is optional, but useful if you want to track ROI of acquisition channels
+# See https://www.moesif.com/docs/api#users for campaign schema
+campaign = MoesifApi::CampaignModel.new()
+campaign.utm_source = "google"
+campaign.utm_medium = "cpc"
+campaign.utm_campaign = "adwords"
+campaign.utm_term = "api+tooling"
+campaign.utm_content = "landing"
+
+# Only user_id is required.
+# metadata can be any custom object
+user = MoesifApi::UserModel.new()
+user.user_id = "12345"
+user.company_id = "67890" # If set, associate user with a company object
+user.campaign = campaign
+user.metadata = metadata
 
 update_user = MoesifRack::MoesifMiddleware.new(@app, @options).update_user(user_model)
 ```
 
-### update_users_batch method
-A method is attached to the moesif middleware object to update the users profile or metadata in batch.
-The metadata field can be any custom data you want to set on the user. The `user_id` field is required.
+### Update Users in Batch
+Similar to update_user, but used to update a list of users in one batch. 
+Only the `user_id` field is required.
+This method is a convenient helper that calls the Moesif API lib.
+For details, visit the [Ruby API Reference](https://www.moesif.com/docs/api?ruby#update-users-in-batch).
 
 ```ruby
-metadata = JSON.parse('{'\
-      '"email": "testrubyapi@user.com",'\
-      '"name": "ruby api user",'\
-      '"custom": "testdata"'\
-    '}')
-
-user_models = []
-
-user_model_A = { "user_id" => "12345",
-                "company_id" => "67890", 
-                "modified_time" => Time.now.utc.iso8601, 
-                "metadata" => metadata }
-
-user_model_B = { "user_id" => "1234", 
-                "company_id" => "6789",
-                "modified_time" => Time.now.utc.iso8601, 
-                "metadata" => metadata }
-
-user_models << user_model_A << user_model_B
-response = MoesifRack::MoesifMiddleware.new(@app, @options).update_users_batch(user_models)
+users = []
+
+metadata = {
+  :email => 'john@acmeinc.com',
+  :first_name => 'John',
+  :last_name => 'Doe',
+  :title => 'Software Engineer',
+  :salesInfo => {
+      :stage => 'Customer',
+      :lifetime_value => 24000,
+      :accountOwner => 'mary@contoso.com',
+  }
+}
+
+# Campaign object is optional, but useful if you want to track ROI of acquisition channels
+# See https://www.moesif.com/docs/api#users for campaign schema
+campaign = MoesifApi::CampaignModel.new()
+campaign.utm_source = "google"
+campaign.utm_medium = "cpc"
+campaign.utm_campaign = "adwords"
+campaign.utm_term = "api+tooling"
+campaign.utm_content = "landing"
+
+# Only user_id is required.
+# metadata can be any custom object
+user = MoesifApi::UserModel.new()
+user.user_id = "12345"
+user.company_id = "67890" # If set, associate user with a company object
+user.campaign = campaign
+user.metadata = metadata
+
+users << user
+
+response = MoesifRack::MoesifMiddleware.new(@app, @options).update_users_batch(users)
 ```
 
 ## Update Company
 
-### update_company method
-A method is attached to the moesif middleware object to update the company profile or metadata.
-The metadata field can be any custom data you want to set on the company. The `company_id` field is required.
+### Update a Single Company
+Create or update a company profile in Moesif.
+The metadata field can be any company demographic or other info you want to store.
+Only the `company_id` field is required.
+This method is a convenient helper that calls the Moesif API lib.
+For details, visit the [Ruby API Reference](https://www.moesif.com/docs/api?ruby#update-a-company).
 
 ```ruby
-metadata = JSON.parse('{'\
-      '"email": "testrubyapi@company.com",'\
-      '"name": "ruby api company",'\
-      '"custom": "testdata"'\
-    '}')
-
-campaign_model = {"utm_source" => "Adwords",
-                  "utm_medium" => "Twitter"}
+metadata = {
+  :org_name => 'Acme, Inc',
+  :plan_name => 'Free',
+  :deal_stage => 'Lead',
+  :mrr => 24000,
+  :demographics => {
+      :alexa_ranking => 500000,
+      :employee_count => 47
+  }
+}
 
-company_model = { "company_id" => "12345", 
-                  "company_domain" => "acmeinc.com",
-                  "metadata" => metadata,
-                  "campaign" => campaign_model }
+# Campaign object is optional, but useful if you want to track ROI of acquisition channels
+# See https://www.moesif.com/docs/api#update-a-company for campaign schema
+campaign = MoesifApi::CampaignModel.new()
+campaign.utm_source = "google"
+campaign.utm_medium = "cpc"
+campaign.utm_campaign = "adwords"
+campaign.utm_term = "api+tooling"
+campaign.utm_content = "landing"
+
+# Only company_id is required.
+# metadata can be any custom object
+company = MoesifApi::CompanyModel.new()
+company.company_id = "67890"
+company.company_domain = "acmeinc.com" # If domain is set, Moesif will enrich your profiles with publicly available info 
+company.campaign = campaign
+company.metadata = metadata
 
 update_company = MoesifRack::MoesifMiddleware.new(@app, @options).update_company(company_model)
 ```
 
-### update_companies_batch method
-A method is attached to the moesif middleware object to update the companies profile or metadata in batch.
-The metadata field can be any custom data you want to set on the company. The `company_id` field is required.
+### Update Companies in Batch
+Similar to update_company, but used to update a list of companies in one batch. 
+Only the `company_id` field is required.
+This method is a convenient helper that calls the Moesif API lib.
+For details, visit the [Ruby API Reference](https://www.moesif.com/docs/api?ruby#update-companies-in-batch).
 
 ```ruby
-metadata = JSON.parse('{'\
-      '"email": "testrubyapi@user.com",'\
-      '"name": "ruby api user",'\
-      '"custom": "testdata"'\
-    '}')
-
-company_models = []
-
-company_model_A = { "company_id" => "12345",
-                    "company_domain" => "nowhere.com", 
-                "metadata" => metadata }
-
-company_model_B = { "company_id" => "67890",
-                    "company_domain" => "acmeinc.com",
-                "metadata" => metadata }
+companies = []
+
+metadata = {
+  :org_name => 'Acme, Inc',
+  :plan_name => 'Free',
+  :deal_stage => 'Lead',
+  :mrr => 24000,
+  :demographics => {
+      :alexa_ranking => 500000,
+      :employee_count => 47
+  }
+}
 
-company_models << company_model_A << company_model_B
-response = MoesifRack::MoesifMiddleware.new(@app, @options).update_companies_batch(company_models)
+# Campaign object is optional, but useful if you want to track ROI of acquisition channels
+# See https://www.moesif.com/docs/api#update-a-company for campaign schema
+campaign = MoesifApi::CampaignModel.new()
+campaign.utm_source = "google"
+campaign.utm_medium = "cpc"
+campaign.utm_campaign = "adwords"
+campaign.utm_term = "api+tooling"
+campaign.utm_content = "landing"
+
+# Only company_id is required.
+# metadata can be any custom object
+company = MoesifApi::CompanyModel.new()
+company.company_id = "67890"
+company.company_domain = "acmeinc.com" # If domain is set, Moesif will enrich your profiles with publicly available info 
+company.campaign = campaign
+company.metadata = metadata
+
+companies << company
+response = MoesifRack::MoesifMiddleware.new(@app, @options).update_companies_batch(companies)
 ```
 
 ## How to test

data/lib/moesif_rack/app_config.rb

@@ -19,7 +19,12 @@ class AppConfig
                 puts 'Error getting application configuration, with status code:'
                 puts e.response_code
             end
+        rescue => e
+            if debug
+                puts e.to_s
+            end
         end
+        rescue
     end
 
     def parse_configuration(config_api_response, debug)
@@ -118,4 +123,8 @@ class AppConfig
             return nil
         end
     end
+
+    def calculate_weight(sample_rate)
+        return sample_rate == 0 ? 1 : (100 / sample_rate).floor
+    end
 end

data/lib/moesif_rack/client_ip.rb

@@ -1,8 +1,8 @@
 
 def is_ip?(value)
+    ipv4 = /^(?:(?:\d|[1-9]\d|1\d{2}|2[0-4]\d|25[0-5])\.){3}(?:\d|[1-9]\d|1\d{2}|2[0-4]\d|25[0-5])$/
+    ipv6 = /^\s*((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?\s*$/
     if 
-        ipv4 = /^(?:(?:\d|[1-9]\d|1\d{2}|2[0-4]\d|25[0-5])\.){3}(?:\d|[1-9]\d|1\d{2}|2[0-4]\d|25[0-5])$/
-        ipv6 = /^\s*((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?\s*$/
         # We use !! to convert the return value to a boolean
         !!(value =~ ipv4 or value=~ ipv6)
     end

data/lib/moesif_rack/moesif_middleware.rb

@@ -6,6 +6,8 @@ require_relative './client_ip.rb'
 require_relative './app_config.rb'
 require_relative './update_user.rb'
 require_relative './update_company.rb'
+require 'zlib'
+require 'stringio'
 
 module MoesifRack
 
@@ -30,25 +32,29 @@ module MoesifRack
       @config = @app_config.get_config(@api_controller, @debug)
       @config_etag = nil
       @sampling_percentage = 100
-      @last_updated_time = Time.now.utc
+      @last_config_download_time = Time.now.utc
+      @last_worker_run = Time.now.utc
       @config_dict = Hash.new
       @disable_transaction_id = options['disable_transaction_id'] || false
       @log_body = options.fetch('log_body', true)
+      @batch_size = options['batch_size'] || 25
+      @batch_max_time = options['batch_max_time'] || 2
+      @events_queue = Queue.new
+      @event_response_config_etag = nil
+      start_worker()
+
       begin
         if !@config.nil?
-          @config_etag, @sampling_percentage, @last_updated_time = @app_config.parse_configuration(@config, @debug)
+          @config_etag, @sampling_percentage, @last_config_download_time = @app_config.parse_configuration(@config, @debug)
         end
       rescue => exception
-        if @debug
-          puts 'Error while parsing application configuration on initialization'
-          puts exception.to_s
-        end
+        log_debug 'Error while parsing application configuration on initialization'
+        log_debug exception.to_s
       end
       @capture_outoing_requests = options['capture_outoing_requests']
-      if @capture_outoing_requests
-        if @debug
-          puts 'Start Capturing outgoing requests'
-        end
+      @capture_outgoing_requests = options['capture_outgoing_requests']
+      if @capture_outoing_requests || @capture_outgoing_requests
+        log_debug 'Start Capturing outgoing requests'
         require_relative '../../moesif_capture_outgoing/httplog.rb'
         MoesifCaptureOutgoing.start_capture_outgoing(options)
       end
@@ -70,12 +76,84 @@ module MoesifRack
       CompanyHelper.new.update_companies_batch(@api_controller, @debug, company_profiles)
     end
 
-    def call env
-      start_time = Time.now.utc.iso8601
+    def start_with_json(body)
+      body.start_with?('{') || body.start_with?('[')
+    end
+
+    def decompress_body(body)
+      Zlib::GzipReader.new(StringIO.new(body)).read
+    end
+
+    def transform_headers(headers)
+      Hash[headers.map { |k, v| [k.downcase, v]}]
+    end
+
+    def base64_encode_body(body)
+      return Base64.encode64(body), 'base64'
+    end
 
+    def log_debug(message)
       if @debug
-        puts 'inside moesif middleware'
+        puts("#{Time.now.to_s} [Moesif Middleware] PID #{Process.pid} TID #{Thread.current.object_id} #{message}")
       end
+    end
+
+    def parse_body(body, headers)
+      begin
+        if start_with_json(body)
+          parsed_body = JSON.parse(body)
+          transfer_encoding = 'json'
+        elsif headers.key?('content-encoding') && ((headers['content-encoding'].downcase).include? "gzip")
+          uncompressed_string = decompress_body(body)
+          parsed_body, transfer_encoding = base64_encode_body(uncompressed_string)
+        else
+          parsed_body, transfer_encoding = base64_encode_body(body)
+        end
+      rescue
+        parsed_body, transfer_encoding = base64_encode_body(body)
+      end
+      return parsed_body, transfer_encoding
+    end
+
+    def start_worker
+      Thread::new do
+        @last_worker_run = Time.now.utc
+        loop do
+          begin
+            until @events_queue.empty? do
+                batch_events = []
+                until batch_events.size == @batch_size || @events_queue.empty? do 
+                  batch_events << @events_queue.pop
+                end 
+                log_debug("Sending #{batch_events.size.to_s} events to Moesif")
+                event_api_response =  @api_controller.create_events_batch(batch_events)
+                @event_response_config_etag = event_api_response[:x_moesif_config_etag]
+                log_debug(event_api_response.to_s)
+                log_debug("Events successfully sent to Moesif")
+            end
+            
+            if @events_queue.empty?
+              log_debug("No events to read from the queue")
+            end
+  
+            sleep @batch_max_time
+          rescue MoesifApi::APIException => e
+            if e.response_code.between?(401, 403)
+              puts "Unathorized accesss sending event to Moesif. Please verify your Application Id."
+              log_debug(e.to_s)
+            end
+            log_debug("Error sending event to Moesif, with status code #{e.response_code.to_s}")
+          rescue => e
+            log_debug(e.to_s)
+          end
+        end
+      end
+    end
+
+    def call env
+      start_time = Time.now.utc.iso8601
+
+      log_debug('Calling Moesif middleware')
 
       status, headers, body = @app.call env
       end_time = Time.now.utc.iso8601
@@ -85,7 +163,7 @@ module MoesifRack
         complex_copy = env.dup
 
         req_headers = {}
-        complex_copy.select {|k,v| k.start_with? 'HTTP_'}.each do |key, val|
+        complex_copy.select {|k,v| k.start_with?('HTTP_', 'CONTENT_') }.each do |key, val|
           new_key = key.sub(/^HTTP_/, '')
           new_key = new_key.sub('_', '-')
           req_headers[new_key] = val
@@ -98,12 +176,7 @@ module MoesifRack
 
         if @log_body
           if req_body_string && req_body_string.length != 0
-            begin
-              req_body = JSON.parse(req_body_string)
-            rescue
-              req_body = Base64.encode64(req_body_string)
-              req_body_transfer_encoding = 'base64'
-            end
+            req_body, req_body_transfer_encoding = parse_body(req_body_string, transform_headers(req_headers))
           end
         end
 
@@ -115,12 +188,7 @@ module MoesifRack
 
         if @log_body
           if rsp_body_string && rsp_body_string.length != 0
-            begin
-              rsp_body = JSON.parse(rsp_body_string)
-            rescue
-              rsp_body = Base64.encode64(rsp_body_string)
-              rsp_body_transfer_encoding = 'base64'
-            end
+            rsp_body, rsp_body_transfer_encoding = parse_body(rsp_body_string, transform_headers(rsp_headers))
           end
         end
 
@@ -180,43 +248,31 @@ module MoesifRack
         event_model.direction = "Incoming"
         
         if @identify_user
-          if @debug
-            puts "calling identify user proc"
-          end
+          log_debug "calling identify user proc"
           event_model.user_id = @identify_user.call(env, headers, body)
         end
 
         if @identify_company
-          if @debug
-            puts "calling identify company proc"
-          end
+          log_debug "calling identify company proc"
           event_model.company_id = @identify_company.call(env, headers, body)
         end
 
         if @get_metadata
-          if @debug
-            puts "calling get_metadata proc"
-          end
+          log_debug "calling get_metadata proc"
           event_model.metadata = @get_metadata.call(env, headers, body)
         end
 
         if @identify_session
-          if @debug
-            puts "calling identify session proc"
-          end
+          log_debug "calling identify session proc"
           event_model.session_token = @identify_session.call(env, headers, body)
         end
         if @mask_data
-          if @debug
-            puts "calling mask_data proc"
-          end
+          log_debug "calling mask_data proc"
           event_model = @mask_data.call(event_model)
         end
 
-        if @debug
-          puts "sending data to moesif"
-          puts event_model.to_json
-        end
+        log_debug "sending data to moesif"
+        log_debug event_model.to_json
         # Perform the API call through the SDK function
         begin
           @random_percentage = Random.rand(0.00..100.00)
@@ -224,44 +280,35 @@ module MoesifRack
           begin 
             @sampling_percentage = @app_config.get_sampling_percentage(@config, event_model.user_id, event_model.company_id, @debug)
           rescue => exception
-            if @debug
-              puts 'Error while getting sampling percentage, assuming default behavior'
-              puts exception.to_s
-            end
+            log_debug 'Error while getting sampling percentage, assuming default behavior'
+            log_debug exception.to_s
             @sampling_percentage = 100
           end
 
           if @sampling_percentage > @random_percentage
-            event_api_response = @api_controller.create_event(event_model)
-            event_response_config_etag = event_api_response[:x_moesif_config_etag]
+            event_model.weight = @app_config.calculate_weight(@sampling_percentage)
+            # Add Event to the queue
+            @events_queue << event_model
+            log_debug("Event added to the queue ")
+            if Time.now.utc > (@last_config_download_time + 60)
+              start_worker()
+            end
 
-            if !event_response_config_etag.nil? && !@config_etag.nil? && @config_etag != event_response_config_etag && Time.now.utc > @last_updated_time + 300
+            if !@event_response_config_etag.nil? && !@config_etag.nil? && @config_etag != @event_response_config_etag && Time.now.utc > (@last_config_download_time + 300)
               begin 
                 @config = @app_config.get_config(@api_controller, @debug)
-                @config_etag, @sampling_percentage, @last_updated_time = @app_config.parse_configuration(@config, @debug)
+                @config_etag, @sampling_percentage, @last_config_download_time = @app_config.parse_configuration(@config, @debug)
               rescue => exception
-                if @debug
-                  puts 'Error while updating the application configuration'
-                  puts exception.to_s
-                end    
+                log_debug 'Error while updating the application configuration'
+                log_debug exception.to_s
               end
             end
-            if @debug
-              puts("Event successfully sent to Moesif")
-            end
           else
-            if @debug
-              puts("Skipped Event due to sampling percentage: " + @sampling_percentage.to_s + " and random percentage: " + @random_percentage.to_s)
-            end
-          end
-        rescue MoesifApi::APIException => e
-          if e.response_code.between?(401, 403)
-            puts "Unathorized accesss sending event to Moesif. Please verify your Application Id."
-          end
-          if @debug
-            puts "Error sending event to Moesif, with status code: "
-            puts e.response_code
+            log_debug("Skipped Event due to sampling percentage: " + @sampling_percentage.to_s + " and random percentage: " + @random_percentage.to_s)
           end
+        rescue => exception
+          log_debug "Error adding event to the queue "
+          log_debug exception.to_s
         end
 
       end
@@ -275,11 +322,7 @@ module MoesifRack
       end
 
       if !should_skip
-        if @debug
-          process_send.call
-        else
-          Thread.start(&process_send)
-        end
+        process_send.call
       end
 
       [status, headers, body]

data/moesif_capture_outgoing/httplog/http_log.rb

@@ -3,6 +3,7 @@ require 'rack'
 require 'moesif_api'
 require 'json'
 require 'base64'
+require_relative '../../lib/moesif_rack/app_config.rb'
 
 module MoesifCaptureOutgoing
 
@@ -23,6 +24,22 @@ module MoesifCaptureOutgoing
       @skip_outgoing = options['skip_outgoing']
       @mask_data_outgoing = options['mask_data_outgoing']
       @log_body_outgoing = options.fetch('log_body_outgoing', true)
+      @app_config = AppConfig.new
+      @config = @app_config.get_config(@api_controller, @debug)
+      @config_etag = nil
+      @sampling_percentage = 100
+      @last_updated_time = Time.now.utc
+      @config_dict = Hash.new
+      begin
+        if !@config.nil?
+          @config_etag, @sampling_percentage, @last_updated_time = @app_config.parse_configuration(@config, @debug)
+        end
+      rescue => exception
+        if @debug
+          puts 'Error while parsing application configuration on initialization'
+          puts exception.to_s
+        end
+      end
     end
 
     def call (url, request, request_time, response, response_time)
@@ -157,11 +174,45 @@ module MoesifCaptureOutgoing
 
           # Send Event to Moesif
           begin
-            if @debug
-              puts 'Sending Outgoing Request Data to Moesif'
-              puts event_model.to_json
+            @random_percentage = Random.rand(0.00..100.00)
+            begin 
+              @sampling_percentage = @app_config.get_sampling_percentage(@config, event_model.user_id, event_model.company_id, @debug)
+            rescue => exception
+              if @debug
+                puts 'Error while getting sampling percentage, assuming default behavior'
+                puts exception.to_s
+              end
+              @sampling_percentage = 100
+            end
+
+            if @sampling_percentage > @random_percentage
+              event_model.weight = @app_config.calculate_weight(@sampling_percentage)
+              if @debug
+                puts 'Sending Outgoing Request Data to Moesif'
+                puts event_model.to_json
+              end
+              event_api_response = @api_controller.create_event(event_model)
+              event_response_config_etag = event_api_response[:x_moesif_config_etag]
+
+              if !event_response_config_etag.nil? && !@config_etag.nil? && @config_etag != event_response_config_etag && Time.now.utc > @last_updated_time + 300
+                begin 
+                  @config = @app_config.get_config(@api_controller, @debug)
+                  @config_etag, @sampling_percentage, @last_updated_time = @app_config.parse_configuration(@config, @debug)
+                rescue => exception
+                  if @debug
+                    puts 'Error while updating the application configuration'
+                    puts exception.to_s
+                  end
+                end
+              end
+              if @debug
+                puts("Event successfully sent to Moesif")
+              end
+            else
+              if @debug
+                puts("Skipped outgoing Event due to sampling percentage: " + @sampling_percentage.to_s + " and random percentage: " + @random_percentage.to_s)
+              end
             end
-            @api_controller.create_event(event_model)
           rescue MoesifApi::APIException => e
             if e.response_code.between?(401, 403)
               puts "Unathorized accesss sending event to Moesif. Please verify your Application Id."
@@ -170,6 +221,10 @@ module MoesifCaptureOutgoing
               puts "Error sending event to Moesif, with status code: "
               puts e.response_code
             end
+          rescue => e
+            if debug
+                puts e.to_s
+            end
           end
         else 
           if @debug

data/test/moesif_rack_test.rb

@@ -10,7 +10,7 @@ class MoesifRackTest < Test::Unit::TestCase
     @options = { 'application_id' => 'Your Moesif Application Id',
     'debug' => true,
     'disable_transaction_id' => true,
-    'capture_outoing_requests' => true,
+    'capture_outgoing_requests' => true,
     'get_metadata' => Proc.new {|request, response|
       {
         'foo'  => 'abc',
@@ -108,6 +108,8 @@ class MoesifRackTest < Test::Unit::TestCase
 
   def test_log_event
     response = @moesif_rack_app.call(Rack::MockRequest.env_for("https://acmeinc.com/items/42752/reviews"))
+    # Sleep to allow queue to flush for testing purpose
+    sleep 5
     assert_equal response, @app.call(nil)
   end
 

metadata

@@ -1,28 +1,34 @@
 --- !ruby/object:Gem::Specification
 name: moesif_rack
 version: !ruby/object:Gem::Version
-  version: 1.3.9
+  version: 1.4.3
 platform: ruby
 authors:
 - Moesif, Inc
 - Xing Wang
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-12-13 00:00:00.000000000 Z
+date: 2020-06-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: test-unit
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+    - - ">="
       - !ruby/object:Gem::Version
         version: 3.1.5
-  type: :runtime
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+    - - ">="
       - !ruby/object:Gem::Version
         version: 3.1.5
 - !ruby/object:Gem::Dependency
@@ -31,15 +37,21 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2.10
+        version: '1.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.12
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2.10
-description: Collection/Data Ingestion SDK for Rack (also Rails) Middleware / RoR
+        version: '1.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.12
+description: Rack/Rails middleware to log API calls to Moesif API analytics and monitoring
 email: xing@moesif.com
 executables: []
 extensions: []
@@ -61,7 +73,7 @@ homepage: https://moesif.com
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -76,9 +88,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
+rubyforge_project:
 rubygems_version: 2.7.7
-signing_key: 
+signing_key:
 specification_version: 4
 summary: moesif_rack
 test_files: []