zuora_connect 1.8.71s → 2.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f47f7dd4479d1fdc48a9b4ad63b1426a11c0da6b438b4a5e9f0d97ea64cba8d5
-  data.tar.gz: 1b5c6f2cd77b5b5122127f2201ac7265a98889f03ccdc796dc5dd0d1e2bb78c4
+  metadata.gz: 3f0a05ee8e03bf92f69dfa82546fb5a917d9bdd3cc9350b7ddc688462789a7a5
+  data.tar.gz: a5a85a0db4bb49697aaeb408337e5dca8c860ba9f3f45153a98d8ad545e96a5d
 SHA512:
-  metadata.gz: bb3214541ca5cc6b51f23a46cbdc23f72d118905e6224f23f7ab4ff48d4792f1bd693698baeda046e956eb13b67d4f867d89be817f723d21d4e089321ee0774d
-  data.tar.gz: ecac2c02fff9530cffee1093d1c4a611e855d593d5634b82cd46abcab311946813719dc4e22fd3c63f32e981505fb228afb1d7929d205cfdd37c5ebc75ff995b
+  metadata.gz: 66f02578165cd4da25089c786090ec3c63a8d312c127c943a8cd939abc74fbdb7c696490a9f5955d41818e43852720bb93da449e0a217076b22ac7899204d594
+  data.tar.gz: f9899bd76b9111abd1ff2d3256f94b87e3660f12c8412382b758adaec6b480a74cbaf817108a4da95e6dce5e39735c04440194101ff768295503d723c13bc261

data/app/controllers/zuora_connect/static_controller.rb

@@ -26,6 +26,19 @@ module ZuoraConnect
     end
 
     def health
+      if params[:error].present?
+        begin 
+          raise ZuoraConnect::Exceptions::Error.new('This is an error')
+        rescue => ex
+          case params[:error]
+          when 'Log'  
+            Rails.logger.error(ex)
+          when 'Exception'
+            raise
+          end
+        end
+      end
+
       render json: {
         message: "Alive",
         status: 200

data/app/models/zuora_connect/app_instance_base.rb

@@ -16,6 +16,7 @@ module ZuoraConnect
     BLANK_OBJECT_ID_LOOKUP = 'BlankValueSupplied'
     HOLDING_PATTERN_SLEEP = 5.seconds
     CONNECT_COMMUNICATION_SLEEP= 5.seconds
+    IGNORED_LOCALS = ['fr', 'ja', 'sp']
 
     def init
       self.connect_user = 'Nobody'
@@ -29,7 +30,7 @@ module ZuoraConnect
       PaperTrail.whodunnit = "Backend" if defined?(PaperTrail)
       if defined?(ElasticAPM) && ElasticAPM.running?
         ElasticAPM.set_user("Backend")
-        ElasticAPM.set_tag(:app_instance, self.id) 
+        ElasticAPM.set_tag(:app_instance, self.id)
       end
 
       if INSTANCE_REFRESH_WINDOW > INSTANCE_REDIS_CACHE_PERIOD
@@ -60,13 +61,13 @@ module ZuoraConnect
         Resque.redis.zrem("PauseQueue", id)
       end
       return true
-    end 
+    end
 
     def apartment_switch(method = nil, migrate = false)
       switch_count ||= 0
       if self.persisted?
         begin
-          Apartment::Tenant.switch!(self.id) 
+          Apartment::Tenant.switch!(self.id)
         rescue Apartment::TenantNotFound => ex
           sleep(2)
           begin
@@ -74,9 +75,9 @@ module ZuoraConnect
           rescue Apartment::TenantExists => ex
           end
           if (switch_count += 1) < 2
-            retry 
+            retry
           else
-            raise 
+            raise
           end
         end
         if migrate && ActiveRecord::Migrator.needs_migration?
@@ -159,7 +160,7 @@ module ZuoraConnect
           ZuoraConnect.logger.add(log_level, self.new_session_message)
           self.build_task(task_data: session["#{self.id}::task_data"], session: session)
         end
-      end      
+      end
       return self
     rescue ZuoraConnect::Exceptions::HoldingPattern => ex
       while self.marked_for_refresh?
@@ -169,9 +170,9 @@ module ZuoraConnect
       self.reload_attributes([:refresh_token, :oauth_expires_at, :access_token])
       session = self.data_lookup(session: session)
       retry
-    rescue => ex 
+    rescue => ex
       if recoverable_session
-        ZuoraConnect.logger.error("REBUILDING - Using backup expired cache")
+        ZuoraConnect.logger.warn("REBUILDING - Using backup expired cache")
         self.build_task(task_data: session["#{self.id}::task_data"], session: session)
         return self
       else
@@ -181,7 +182,7 @@ module ZuoraConnect
       begin
         I18n.locale = self.locale
       rescue I18n::InvalidLocale => ex
-        ZuoraConnect.logger.debug("Invalid Locale: #{ex.message}")
+        ZuoraConnect.logger.error(ex) if !IGNORED_LOCALS.include?(ex.locale.to_s.downcase)
       end
       Time.zone = self.timezone
       tenants = self.task_data.dig('tenant_ids') || []
@@ -191,7 +192,7 @@ module ZuoraConnect
         ElasticAPM.set_tag(:organization, organizations.first) 
       end
       self.logitem(item: {tenant_ids: tenants, organization: organizations})
-      self.update_column(:name, self.task_data.dig('name')) if ZuoraConnect::AppInstance.column_names.include?('name') && self.task_data.dig('name') != self.name 
+      self.update_column(:name, self.task_data.dig('name')) if ZuoraConnect::AppInstance.column_names.include?('name') && self.task_data.dig('name') != self.name
     end
 
     def refresh(session: {}, session_fallback: false)
@@ -207,7 +208,6 @@ module ZuoraConnect
         self.cache_app_instance
         self.reset_mark_for_refresh
       else
-        ZuoraConnect.logger.fatal("[#{self.id}] REFRESH TASK - Failed Code #{response.code}")
         raise ZuoraConnect::Exceptions::ConnectCommunicationError.new("Error Communicating with Connect", response.body, response.code)
       end
     rescue *(ZuoraAPI::Login::CONNECTION_EXCEPTIONS).concat(ZuoraAPI::Login::CONNECTION_READ_EXCEPTIONS) => ex
@@ -220,13 +220,15 @@ module ZuoraConnect
       end
     rescue ZuoraConnect::Exceptions::ConnectCommunicationError => ex
       if (refresh_count += 1) < 3
-        ZuoraConnect.logger.info("[#{self.id}] REFRESH TASK - Failed Retrying(#{refresh_count})")
         if ex.code == 401
+          ZuoraConnect.logger.info("[#{self.id}] REFRESH TASK - Failed #{ex.code} - Retrying(#{refresh_count})")
           self.refresh_oauth
+        else
+          ZuoraConnect.logger.warn("[#{self.id}] REFRESH TASK - Failed #{ex.code} - Retrying(#{refresh_count})")
         end
         retry
       else
-        ZuoraConnect.logger.fatal("[#{self.id}] REFRESH TASK - Failed #{refresh_count}x")
+        ZuoraConnect.logger.fatal("[#{self.id}] REFRESH TASK - Failed #{ex.code} - #{refresh_count}x")
         raise
       end
     end
@@ -243,10 +245,45 @@ module ZuoraConnect
       def self.write_to_telegraf(*args)
         if ZuoraConnect.configuration.enable_metrics
           @@telegraf_host = ZuoraConnect::Telegraf.new() if @@telegraf_host == nil
+          unicorn_stats = self.unicorn_listener_stats() if defined?(Unicorn) && Unicorn.respond_to?(:listener_names)
+          @@telegraf_host.write(direction: 'Raindrops', tags: {}, values: unicorn_stats)  unless unicorn_stats.blank?
           return @@telegraf_host.write(*args)
         end
       end
 
+      def self.unicorn_listener_stats ()
+        stats_hash = {}
+        stats_hash["total_active"] = 0
+        stats_hash["total_queued"] = 0
+
+        begin 
+          tmp = Unicorn.listener_names
+          unix = tmp.grep(%r{\A/})
+          tcp = tmp.grep(/\A.+:\d+\z/)
+          tcp = nil if tcp.empty?
+          unix = nil if unix.empty?
+
+
+          Raindrops::Linux.tcp_listener_stats(tcp).each do |addr,stats|
+            stats_hash["active_#{addr}"] = stats.active
+            stats_hash["queued_#{addr}"] = stats.queued
+            stats_hash["total_active"] = stats.active + stats_hash["total_active"]
+            stats_hash["total_queued"] = stats.queued + stats_hash["total_queued"]
+          end if tcp
+
+          Raindrops::Linux.unix_listener_stats(unix).each do |addr,stats|
+            stats_hash["active_#{addr}"] = stats.active
+            stats_hash["queued_#{addr}"] = stats.queued
+            stats_hash["total_active"] = stats.active + stats_hash["total_active"]
+            stats_hash["total_queued"] = stats.queued + stats_hash["total_queued"]
+          end if unix
+        rescue IOError => ex
+        rescue => ex
+          ZuoraConnect.logger.error(ex)
+        end
+        return stats_hash
+      end
+
       def self.get_metrics(type)
         @data = {}
 
@@ -301,7 +338,7 @@ module ZuoraConnect
               else
                 tmp.client.current_session                       = session["#{self.id}::#{k}:current_session"]                             if session["#{self.id}::#{k}:current_session"]
                 tmp.client.bearer_token                          = session["#{self.id}::#{k}:bearer_token"]                                if session["#{self.id}::#{k}:bearer_token"] && tmp.client.respond_to?(:bearer_token) ## need incase session id goes from basic to aouth in same redis store
-                tmp.client.oauth_session_expires_at              = session["#{self.id}::#{k}:oauth_session_expires_at"]                    if session["#{self.id}::#{k}:oauth_session_expires_at"]  && tmp.client.respond_to?(:oauth_session_expires_at) 
+                tmp.client.oauth_session_expires_at              = session["#{self.id}::#{k}:oauth_session_expires_at"]                    if session["#{self.id}::#{k}:oauth_session_expires_at"]  && tmp.client.respond_to?(:oauth_session_expires_at)
               end
             end
             self.logins[k] = tmp
@@ -317,7 +354,10 @@ module ZuoraConnect
         end
       rescue => ex
         ZuoraConnect.logger.error("Task Data: #{task_data}") if task_data.present?
-        ZuoraConnect.logger.error("Task Session: #{session.to_h}") if session.present?
+        if session.present?
+          ZuoraConnect.logger.error("Task Session: #{session.to_h}")  if session.methods.include?(:to_h)
+          ZuoraConnect.logger.error("Task Session: #{session.to_hash}") if session.methods.include?(:to_hash)
+        end
         raise
       end
 
@@ -422,8 +462,7 @@ module ZuoraConnect
           self.oauth_expires_at = Time.at(response_body["created_at"].to_i) + response_body["expires_in"].seconds
           self.save(:validate => false)
         else
-          ZuoraConnect.logger.fatal("[#{self.id}] REFRESH OAUTH - Failed Code #{response.code}")
-          raise ZuoraConnect::Exceptions::ConnectCommunicationError.new("Error Refreshing Access Token", response.body, response.code)
+          raise ZuoraConnect::Exceptions::ConnectCommunicationError.new("Error Refreshing Access Token for #{self.id}", response.body, response.code)
         end
       rescue *(ZuoraAPI::Login::CONNECTION_EXCEPTIONS).concat(ZuoraAPI::Login::CONNECTION_READ_EXCEPTIONS) => ex
         if (refresh_oauth_count += 1) < 3
@@ -441,10 +480,10 @@ module ZuoraConnect
         return if !self.oauth_expired?
 
         if (refresh_oauth_count += 1) < 3
-          ZuoraConnect.logger.info("[#{self.id}] REFRESH OAUTH - Failed Retrying(#{refresh_oauth_count})")
+          ZuoraConnect.logger.info("[#{self.id}] REFRESH OAUTH - Failed #{ex.code} - Retrying(#{refresh_oauth_count})")
           retry
         else
-          ZuoraConnect.logger.fatal("[#{self.id}] REFRESH OAUTH - Failed #{refresh_oauth_count}x")
+          ZuoraConnect.logger.fatal("[#{self.id}] REFRESH OAUTH - Failed #{ex.code} - #{refresh_oauth_count}x")
           raise
         end
       end
@@ -561,16 +600,16 @@ module ZuoraConnect
       def decrypt_data(data: nil, rescue_return: nil, log_fatal: true)
         return data if data.blank?
         if Rails.env == 'development'
-          begin 
+          begin
             return JSON.parse(data)
           rescue JSON::ParserError => ex
             return data
           end
         else
-          begin 
+          begin
             return JSON.parse(encryptor.decrypt_and_verify(CGI::unescape(data)))
           rescue ActiveSupport::MessageVerifier::InvalidSignature => ex
-            ZuoraConnect.logger.add(Logger::FATAL, 'Error Decrypting') if log_fatal
+            ZuoraConnect.logger.add(Logger::ERROR, "Error Decrypting for #{self.id}") if log_fatal
             return rescue_return
           rescue JSON::ParserError => ex
             return encryptor.decrypt_and_verify(CGI::unescape(data))
@@ -622,7 +661,7 @@ module ZuoraConnect
         if paused_user == current_user || paused_user.blank?
           Resque.redis.zrem("PauseQueue", "#{self.id}__#{paused_user}")
         else
-          raise "Can only unpause for user #{paused_user}." 
+          raise "Can only unpause for user #{paused_user}."
         end
       end
     ### END Resque Helping Methods ####
@@ -836,7 +875,7 @@ module ZuoraConnect
           if cache
             Redis.current.sadd("Catalog:#{self.id}:Keys", ["Catalog:#{self.id}:#{object_id}:Hierarchy", "Catalog:#{self.id}:#{object_id}:Children:#{child_objects}"])
             Redis.current.set("Catalog:#{self.id}:#{object_id}:Hierarchy", encrypt_data(data: object_hierarchy))
-            Redis.current.set("Catalog:#{self.id}:#{object_id}:Children:#{child_objects}", encrypt_data(data: stub_catalog)) 
+            Redis.current.set("Catalog:#{self.id}:#{object_id}:Children:#{child_objects}", encrypt_data(data: stub_catalog))
           else
             Redis.current.sadd("Catalog:#{self.id}:Keys", ["Catalog:#{self.id}:#{object_id}:Hierarchy"])
             Redis.current.set("Catalog:#{self.id}:#{object_id}:Hierarchy", encrypt_data(data: object_hierarchy))
@@ -875,7 +914,7 @@ module ZuoraConnect
         self.update_functions
         if index_table
           ActiveRecord::Base.connection.execute('SELECT "shared_extensions".refresh_aggregate_table(\'%s\', \'%s\', %s, \'Index\');' % [aggregate_name, table_name, ActiveRecord::Base.connection.quote(where_clause)])
-        else 
+        else
           ActiveRecord::Base.connection.execute('SELECT "shared_extensions".refresh_aggregate_table(\'%s\', \'%s\', %s, \'NO\');' % [aggregate_name, table_name, ActiveRecord::Base.connection.quote(where_clause)])
         end
       end
@@ -949,4 +988,4 @@ module ZuoraConnect
     method_hook :refresh, :updateOption, :update_logins, :before => :check_oauth_state
     method_hook :new_session, :refresh, :build_task, :after => :apartment_switch
   end
-end
+end

data/config/initializers/resque.rb

@@ -22,10 +22,10 @@ Resque.module_eval do
 end
 
 if defined?(Resque)
-  Resque.logger = ZuoraConnect.custom_logger(name: "Resque") 
+  Resque.logger = ZuoraConnect.custom_logger(name: "Resque", type: 'Monologger', level: MonoLogger::INFO) 
   Resque::Scheduler.logger = ZuoraConnect.custom_logger(name: "ResqueScheduler") if defined?(Resque::Scheduler)
 end
 
 Makara::Logging::Logger.logger = ZuoraConnect.custom_logger(name: "Makara") if defined?(Makara)
 ElasticAPM.agent.config.logger = ZuoraConnect.custom_logger(name: "ElasticAPM", level: MonoLogger::WARN) if defined?(ElasticAPM) && ElasticAPM.running?
-ActionMailer::Base.logger = ZuoraConnect.custom_logger(name: "ActionMailer") if defined?(ActionMailer)
+ActionMailer::Base.logger = ZuoraConnect.custom_logger(name: "ActionMailer", type: 'Monologger') if defined?(ActionMailer)

data/config/initializers/unicorn.rb

@@ -2,8 +2,7 @@ if defined?(Unicorn::WorkerKiller)
   Unicorn::WorkerKiller.module_eval do
     self.singleton_class.send(:alias_method, :kill_self_old, :kill_self)
     def self.kill_self(logger, start_time)
-      @unicorn_logger ||= ZuoraConnect.custom_logger(name: "UnicornKiller")
-      self.kill_self_old(@unicorn_logger, start_time)
+      self.kill_self_old(logger, start_time)
       ZuoraConnect::AppInstance.write_to_telegraf(direction: 'Unicorn-Killer', tags: {app_instance: 0}, values: {kill: 1})
     end
   end

data/db/migrate/20100718151733_create_connect_app_instances.rb

@@ -1,4 +1,4 @@
-class CreateConnectAppInstances < ActiveRecord::Migration
+class CreateConnectAppInstances < ActiveRecord::Migration[4.2]
   def change
     if !ActiveRecord::Base.connection.table_exists?('zuora_connect_app_instances')
       create_table :zuora_connect_app_instances do |t|

data/db/migrate/20101024162319_add_tokens_to_app_instance.rb

@@ -1,4 +1,4 @@
-class AddTokensToAppInstance < ActiveRecord::Migration
+class AddTokensToAppInstance < ActiveRecord::Migration[4.2]
   def change
     add_column :zuora_connect_app_instances, :access_token, :string unless column_exists? :zuora_connect_app_instances, :access_token
     add_column :zuora_connect_app_instances, :refresh_token, :string unless column_exists? :zuora_connect_app_instances, :refresh_token

data/db/migrate/20101024220705_add_token_to_app_instance.rb

@@ -1,4 +1,4 @@
-class AddTokenToAppInstance < ActiveRecord::Migration
+class AddTokenToAppInstance < ActiveRecord::Migration[4.2]
   def change
     add_column :zuora_connect_app_instances, :token, :string  unless column_exists? :zuora_connect_app_instances, :token
   end

data/db/migrate/20110131211919_add_sessions_table.rb

@@ -1,4 +1,4 @@
-class AddSessionsTable < ActiveRecord::Migration
+class AddSessionsTable < ActiveRecord::Migration[4.2]
   def change
     if !ActiveRecord::Base.connection.table_exists?('sessions')
       create_table :sessions do |t|

data/db/migrate/20110411200303_add_expiration_to_app_instance.rb

@@ -1,4 +1,4 @@
-class AddExpirationToAppInstance < ActiveRecord::Migration
+class AddExpirationToAppInstance < ActiveRecord::Migration[4.2]
   def change
     add_column :zuora_connect_app_instances, :oauth_expires_at, :datetime  unless column_exists? :zuora_connect_app_instances, :oauth_expires_at
   end

data/db/migrate/20110413191512_add_new_api_token.rb

@@ -1,4 +1,4 @@
-class AddNewApiToken < ActiveRecord::Migration
+class AddNewApiToken < ActiveRecord::Migration[4.2]
   def change
     add_column :zuora_connect_app_instances, :api_token, :string  unless column_exists? :zuora_connect_app_instances, :api_token
   end

data/db/migrate/20110503003602_add_catalog_data_to_app_instance.rb

@@ -1,4 +1,4 @@
-class AddCatalogDataToAppInstance < ActiveRecord::Migration
+class AddCatalogDataToAppInstance < ActiveRecord::Migration[4.2]
   def change
     add_column :zuora_connect_app_instances, :catalog_updated_at, :datetime  unless column_exists? :zuora_connect_app_instances, :catalog_updated_at
     add_column :zuora_connect_app_instances, :catalog, :jsonb, default: {}  unless column_exists? :zuora_connect_app_instances, :catalog

data/db/migrate/20110503003603_add_catalog_mappings_to_app_instance.rb

@@ -1,4 +1,4 @@
-class AddCatalogMappingsToAppInstance < ActiveRecord::Migration
+class AddCatalogMappingsToAppInstance < ActiveRecord::Migration[4.2]
   def change
     add_column :zuora_connect_app_instances, :catalog_mapping, :jsonb, default: {} unless column_exists? :zuora_connect_app_instances, :catalog_mapping
   end

data/db/migrate/20110503003604_catalog_default.rb

@@ -1,4 +1,4 @@
-class CatalogDefault < ActiveRecord::Migration
+class CatalogDefault < ActiveRecord::Migration[4.2]
   def change
     change_column :zuora_connect_app_instances, :catalog, :jsonb, default: {}
   end

data/db/migrate/20180301052853_add_catalog_attempted_at.rb

@@ -1,4 +1,4 @@
-class AddCatalogAttemptedAt < ActiveRecord::Migration
+class AddCatalogAttemptedAt < ActiveRecord::Migration[4.2]
   def change
     add_column :zuora_connect_app_instances, :catalog_update_attempt_at, :datetime  unless column_exists? :zuora_connect_app_instances, :catalog_update_attempt_at
   end

data/db/migrate/20181206162339_add_fields_to_instance.rb

@@ -1,4 +1,4 @@
-class AddFieldsToInstance < ActiveRecord::Migration
+class AddFieldsToInstance < ActiveRecord::Migration[4.2]
   def change
   	add_column :zuora_connect_app_instances, :name, :string, default: "" unless column_exists? :zuora_connect_app_instances, :name
   end

data/db/migrate/20190520232221_add_zuora_user_table_and_alter_app_instance_id_table.rb

@@ -0,0 +1,18 @@
+class AddZuoraUserTableAndAlterAppInstanceIdTable < ActiveRecord::Migration[5.0]
+  def change
+    if !ActiveRecord::Base.connection.table_exists?('zuora_users')
+      create_table :zuora_users do |t|
+        t.string  :zuora_user_id
+        t.jsonb   :zuora_identity_response, default: {}
+        t.jsonb   :app_permissions, default: {}
+        t.integer :scope, default: 0
+        t.timestamps
+      end
+    end
+
+    add_column :zuora_connect_app_instances, :zuora_domain, :text, default: "" unless column_exists? :zuora_connect_app_instances, :zuora_domain
+    add_column :zuora_connect_app_instances, :zuora_entity_ids, :jsonb, default: [] unless column_exists? :zuora_connect_app_instances, :zuora_entity_ids
+    add_column :zuora_connect_app_instances, :zuora_tenant_ids, :jsonb, default: [] unless column_exists? :zuora_connect_app_instances, :zuora_tenant_ids
+    add_column :zuora_connect_app_instances, :zuora_logins, :text, default: "" unless column_exists? :zuora_connect_app_instances, :zuora_logins
+  end
+end

data/lib/logging/connect_formatter.rb

@@ -0,0 +1,39 @@
+require 'ougai/formatters/base'
+require 'ougai/formatters/for_json'
+
+module Ougai
+  module Formatters
+    # A JSON formatter compatible with node-bunyan
+    class ConnectFormatter < Base
+      include ForJson
+
+      # Intialize a formatter
+      # @param [String] app_name application name (execution program name if nil)
+      # @param [String] hostname hostname (hostname if nil)
+      # @param [Hash] opts the initial values of attributes
+      # @option opts [String] :trace_indent (2) the value of trace_indent attribute
+      # @option opts [String] :trace_max_lines (100) the value of trace_max_lines attribute
+      # @option opts [String] :serialize_backtrace (true) the value of serialize_backtrace attribute
+      # @option opts [String] :jsonize (true) the value of jsonize attribute
+      # @option opts [String] :with_newline (true) the value of with_newline attribute
+      def initialize(app_name = nil, hostname = nil, opts = {})
+        aname, hname, opts = Base.parse_new_params([app_name, hostname, opts])
+        super(aname, hname, opts)
+        init_opts_for_json(opts)
+      end
+
+      def _call(severity, time, progname, data)
+        dump({
+          name: progname || @app_name,
+          pid: $$,
+          level: severity,
+          timestamp: time.utc.strftime('%FT%T.%6NZ'),
+        }.merge(data.merge({message: data.delete(:msg)})))
+      end
+
+      def convert_time(data)
+        #data[:timestamp] = format_datetime(data[:time])
+      end
+    end
+  end
+end

data/lib/metrics/net.rb

@@ -162,7 +162,7 @@ class HttpLogger
   end
 
   def format_log_entry(message, dump = nil)
-    if self.class.colorize
+    if self.class.colorize && Rails.configuration.colorize_logging
       message_color, dump_color = "4;32;1", "0;1"
       log_entry = "  \e[#{message_color}m#{message}\e[0m   "
       log_entry << "\e[#{dump_color}m%#{String === dump ? 's' : 'p'}\e[0m" % dump if dump

data/lib/middleware/metrics_middleware.rb

@@ -1,4 +1,4 @@
-module Middleware
+module ZuoraConnect
   require 'uri'
 
   # Object of this class is passed to the ActiveSupport::Notification hook
@@ -53,7 +53,10 @@ module Middleware
           env['REQUEST_PATH'] = env['REQUEST_PATH'].gsub(Thread.current[:isHallway], '')
 
           #We need the forwarded host header to identify location of tenant
-          @bad_headers.delete('HTTP_X_FORWARDED_HOST')
+          whitelist = Regexp.new(".*[\.]zuora[\.]com$|^zuora[\.]com$")
+          if whitelist.match(env['HTTP_X_FORWARDED_HOST']).present?
+            @bad_headers.delete('HTTP_X_FORWARDED_HOST')
+          end
         else
           Thread.current[:isHallway] = nil
         end

data/lib/middleware/request_id_middleware.rb

@@ -0,0 +1,16 @@
+module ZuoraConnect
+  class RequestIdMiddleware
+    mattr_accessor :request_id
+    mattr_accessor :zuora_request_id
+
+    def initialize(app)
+      @app = app
+    end
+
+    def call(env)
+      self.request_id = env['action_dispatch.request_id']
+      self.zuora_request_id = env["HTTP_ZUORA_REQUEST_ID"]
+      @app.call(env)
+    end
+  end
+end

data/lib/resque/plugins/custom_logger.rb

@@ -7,27 +7,7 @@ module Resque
   module Plugins
     module CustomLogger
       def before_perform(*args)
-        marker = SecureRandom.uuid
-        
-        logger = MonoLogger.new(STDOUT)
-        logger.level = Rails.application.config.log_level
-        logger.formatter = proc do |serverity, datetime, progname, msg|
-          begin
-            msg = JSON.parse(msg)
-          rescue JSON::ParserError => ex
-          end
-
-          require 'json'
-          JSON.dump(
-            trace_id: marker,
-            level: serverity,
-            timestamp: datetime.strftime('%FT%T.%6N'),
-            pid: Process.pid,
-            msg: msg
-          ) + "\n"
-        end
-        
-        Rails.logger = logger
+        Rails.logger.with_fields = { trace_id: SecureRandom.uuid, name: "RailsWorker"} if Rails.logger.class.to_s == 'Ougai::Logger'
         case args.class.to_s
         when "Array"
           if args.first.class == Hash
@@ -38,8 +18,10 @@ module Resque
         when "Hash"
           data = args.merge({:worker_class => self.to_s})
         end
-        Rails.logger.info(data.to_json) if data.present?
+        data = {:msg => 'Starting job', :job => data}
+        data.merge!({:app_instance_id => data.dig(:job, 'app_instance_id')}) if data.dig(:job, 'app_instance_id').present?
+        Rails.logger.info(data) if data.present?
       end
     end
   end
-end
+end

data/lib/zuora_connect.rb

@@ -9,6 +9,7 @@ require 'resque/dynamic_queues'
 require 'resque/silence_done'
 require 'resque/self_lookup'
 require 'resque/plugins/custom_logger'
+require 'logging/connect_formatter'
 require 'metrics/influx/point_value'
 require 'metrics/net'
 
@@ -18,26 +19,66 @@ module ZuoraConnect
     attr_writer :logger
 
     def logger
-      @logger ||= custom_logger(name: "Connect", level: Rails.logger.level)
-    end
+      case Rails.env.to_s
+      when 'development'
+        Rails.logger
+      else
+        @logger ||= custom_logger(name: "Connect", level: Rails.logger.level)
+      end
+    end    
 
-    def custom_logger(name: "", level: Rails.logger.level)
-      logger = MonoLogger.new(STDOUT)
-      logger.level = level
-      logger.formatter =  proc do |serverity, datetime, progname, msg|
-        begin
-          msg = JSON.parse(msg)
-        rescue JSON::ParserError => ex
+    def custom_logger(name: "", level: Rails.logger.present? ? Rails.logger.level : MonoLogger::INFO, type: :ougai)
+      #puts name + ' - ' + {Logger::WARN => 'Logger::WARN', Logger::ERROR => 'Logger::ERROR', Logger::DEBUG => 'Logger::DEBUG', Logger::INFO => 'Logger::INFO' }[level] + ' - '
+      if type == :ougai
+        require 'ougai'
+        #logger = Ougai::Logger.new(MonoLogger.new(STDOUT))
+        logger = Ougai::Logger.new(STDOUT) 
+        logger.formatter = Ougai::Formatters::ConnectFormatter.new(name)
+        logger.level = level
+        logger.before_log = lambda do |data|
+          data[:trace_id] = ZuoraConnect::RequestIdMiddleware.request_id if ZuoraConnect::RequestIdMiddleware.request_id.present?
+          data[:zuora_trace_id] = ZuoraConnect::RequestIdMiddleware.zuora_request_id if ZuoraConnect::RequestIdMiddleware.zuora_request_id.present?
+          #data[:traces] = {amazon_id: data[:trace_id], zuora_id: data[:zuora_trace_id]}
+          if !['ElasticAPM', 'ResqueScheduler', 'ResquePool', 'Resque', 'Makara'].include?(name) 
+            if Thread.current[:appinstance].present?
+              data[:app_instance_id] = Thread.current[:appinstance].id
+              logitems = Thread.current[:appinstance].logitems
+              if logitems.present? && logitems.class == Hash
+                data[:tenant_ids] = logitems[:tenant_ids] if logitems[:tenant_ids].present?
+                data[:organization] = logitems[:organization] if logitems[:organization].present?
+              end
+            end
+          end
         end
+      else
+        logger = MonoLogger.new(STDOUT)
+        logger.level = level
+        logger.formatter =  proc do |serverity, datetime, progname, msg|
+          begin
+            msg = JSON.parse(msg)
+          rescue JSON::ParserError => ex
+          end
 
-        require 'json'
-        JSON.dump(
-          name: name,
-          level: serverity,
-          timestamp: datetime.strftime('%FT%T.%6N'),
-          pid: Process.pid,
-          message: msg
-        ) + "\n"
+          require 'json'
+          store = {
+            name: name,
+            level: serverity,
+            timestamp: datetime.strftime('%FT%T.%6NZ'),
+            pid: Process.pid,
+            message: name == "ActionMailer" ? msg.strip : msg
+          }
+          if !['ElasticAPM', 'ResqueScheduler', 'ResquePool','Resque', 'Makara'].include?(name) 
+            if Thread.current[:appinstance].present? 
+              store[:app_instance_id] = Thread.current[:appinstance].id
+              logitems = Thread.current[:appinstance].logitems
+              if logitems.present? && logitems.class == Hash
+                store[:tenant_ids] = logitems[:tenant_ids] if logitems[:tenant_ids].present?
+                store[:organization] = logitems[:organization] if logitems[:organization].present?
+              end
+            end
+          end
+          JSON.dump(store) + "\n"
+        end
       end
       return logger
     end      
@@ -94,6 +135,7 @@ module ZuoraConnect
     end
 
     defaults.merge!({
+      disable_start_message: true,
       pool_size: 1, 
       transaction_max_spans: 500, 
       ignore_url_patterns: ['^\/admin\/resque.*', '^\/admin\/redis.*', '^\/admin\/peek.*', '^\/peek.*'], 

data/lib/zuora_connect/controllers/helpers.rb

@@ -86,7 +86,7 @@ module ZuoraConnect
                 @appinstance.cache_app_instance
                 session["appInstance"] = app_instance_ids[0]
               else
-                ZuoraConnect.logger.fatal("Launch Error: Param Instance didnt match session data")
+                ZuoraConnect.logger.error("Launch Error: Param Instance didnt match session data")
                 render "zuora_connect/static/invalid_launch_request"
                 return
               end
@@ -95,7 +95,7 @@ module ZuoraConnect
               return
             end
           rescue => ex
-            ZuoraConnect.logger.fatal("Launch Error: #{ex.message}")
+            ZuoraConnect.logger.error(ex)
             render "zuora_connect/static/invalid_launch_request"
             return
           end
@@ -113,7 +113,7 @@ module ZuoraConnect
               return
             end
           rescue => ex
-            ZuoraConnect.logger.fatal("Launch Error: #{ex.message}")
+            ZuoraConnect.logger.error(ex)
             render "zuora_connect/static/invalid_launch_request"
             return
           end
@@ -142,7 +142,7 @@ module ZuoraConnect
         begin
           I18n.locale = session["#{@appinstance.id}::user::locale"] ?  session["#{@appinstance.id}::user::locale"] : @appinstance.locale
         rescue I18n::InvalidLocale => ex
-          ZuoraConnect.logger.error("Invalid Locale: #{ex.message}")
+          ZuoraConnect.logger.error(ex) if !ZuoraConnect::AppInstance::IGNORED_LOCALS.include?(ex.locale.to_s.downcase)
         end
         Time.zone = session["#{@appinstance.id}::user::timezone"] ? session["#{@appinstance.id}::user::timezone"] : @appinstance.timezone
         ZuoraConnect.logger.debug("[#{@appinstance.blank? ? "N/A" : @appinstance.id}] Authenticate App Request Completed In - #{(Time.now - start_time).round(2)}s")
@@ -170,7 +170,6 @@ module ZuoraConnect
       def setup_instance_via_data
         session.clear
         values = JSON.parse(ZuoraConnect::AppInstance.decrypt_response(Base64.urlsafe_decode64(request["data"])))
-        ZuoraConnect.logger.debug("Data: #{values.to_json}")
         if values["param_data"]
           values["param_data"].each do |k ,v|
             params[k] = v
@@ -185,7 +184,7 @@ module ZuoraConnect
           session["#{values["appInstance"]}::user::email"] = values["current_user"]["email"]
         end
 
-        ZuoraConnect.logger.debug("App Params: #{values.to_json}}") if Rails.env != "production"
+        ZuoraConnect.logger.debug({msg: 'Setup values', connect: values}) if Rails.env != "production"
 
         @appinstance = ZuoraConnect::AppInstance.where(:id => values["appInstance"].to_i).first
         if @appinstance.blank?
@@ -253,7 +252,7 @@ module ZuoraConnect
           ElasticAPM.set_user("API User")  if defined?(ElasticAPM) && ElasticAPM.running?
           return true
         else
-          render text: "Access Denied", status: :unauthorized
+          render plain: "Access Denied", status: :unauthorized
         end
       end
     end

data/lib/zuora_connect/railtie.rb

@@ -1,4 +1,5 @@
 require 'middleware/metrics_middleware'
+require 'middleware/request_id_middleware'
 
 module ZuoraConnect
   class Railtie < Rails::Railtie
@@ -22,33 +23,38 @@ module ZuoraConnect
       end
     end
     initializer "zuora_connect.configure_rails_initialization" do |app|
-      app.middleware.insert_after Rack::Sendfile, Middleware::MetricsMiddleware
+      app.middleware.insert_after Rack::Sendfile, ZuoraConnect::MetricsMiddleware
+      app.middleware.insert_after ActionDispatch::RequestId, ZuoraConnect::RequestIdMiddleware
     end
 
     # hook to process_action
-    ActiveSupport::Notifications.subscribe('process_action.action_controller', Middleware::PageRequest.new)
+    ActiveSupport::Notifications.subscribe('process_action.action_controller', ZuoraConnect::PageRequest.new)
 
     initializer(:rails_stdout_logging, before: :initialize_logger) do
       if Rails.env != 'development' && !ENV['DEIS_APP'].blank?
         require 'lograge'
-        logger           = ActiveSupport::Logger.new(STDOUT)
-        logger.formatter = ::Logger::Formatter.new
-        # logger.formatter  = proc do |severity, datetime, progname, msg|
-        #   {severity: severity, time: datetime.strftime('%FT%T.%6N'), process_id: Process.pid, message: msg }.to_json
-        # end
-        Rails.configuration.logger = ActiveSupport::TaggedLogging.new(logger)
-        Rails.configuration.log_tags = [:uuid]
+  
+        Rails.configuration.logger = ZuoraConnect.custom_logger(name: "Rails") 
+
         Rails.configuration.lograge.enabled = true
-        Rails.configuration.lograge.formatter = Lograge::Formatters::Json.new
+        Rails.configuration.colorize_logging = false
+        if Rails.configuration.logger.class.to_s == 'Ougai::Logger'
+          Rails.configuration.lograge.formatter = Class.new do |fmt|
+            def fmt.call(data)
+              { msg: 'Rails Request', request: data }
+            end
+          end
+        end
+        #Rails.configuration.lograge.formatter = Lograge::Formatters::Json.new
         Rails.configuration.lograge.custom_options = lambda do |event|
           exceptions = %w(controller action format id)
           items = {
             #time: event.time.strftime('%FT%T.%6N'),  
-            params: event.payload[:params].except(*exceptions),
-            exception: event.payload[:exception],
-            exception_object: event.payload[:exception_object],
-            process_id: Process.pid
+            params: event.payload[:params].except(*exceptions).to_json.to_s
           }
+          items.merge!({exception_object: event.payload[:exception_object]}) if event.payload[:exception_object].present?
+          items.merge!({exception: event.payload[:exception]}) if event.payload[:exception].present?
+
           if Thread.current[:appinstance].present? 
             items.merge!({appinstance_id: Thread.current[:appinstance].id, connect_user: Thread.current[:appinstance].connect_user, new_session: Thread.current[:appinstance].new_session_message})
             if Thread.current[:appinstance].logitems.present? && Thread.current[:appinstance].logitems.class == Hash

data/lib/zuora_connect/version.rb

@@ -1,3 +1,3 @@
 module ZuoraConnect
-  VERSION = "1.8.71s"
+  VERSION = "2.0.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: zuora_connect
 version: !ruby/object:Gem::Version
-  version: 1.8.71s
+  version: 2.0.1
 platform: ruby
 authors:
 - Connect Team
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-03-22 00:00:00.000000000 Z
+date: 2019-05-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: apartment
@@ -24,6 +24,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: ougai
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: zuora_api
   requirement: !ruby/object:Gem::Requirement
@@ -129,7 +143,7 @@ dependencies:
         version: 4.1.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: '5.1'
+        version: '5.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -139,7 +153,21 @@ dependencies:
         version: 4.1.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: '5.1'
+        version: '5.3'
+- !ruby/object:Gem::Dependency
+  name: raindrops
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -304,9 +332,12 @@ files:
 - db/migrate/20110503003604_catalog_default.rb
 - db/migrate/20180301052853_add_catalog_attempted_at.rb
 - db/migrate/20181206162339_add_fields_to_instance.rb
+- db/migrate/20190520232221_add_zuora_user_table_and_alter_app_instance_id_table.rb
+- lib/logging/connect_formatter.rb
 - lib/metrics/influx/point_value.rb
 - lib/metrics/net.rb
 - lib/middleware/metrics_middleware.rb
+- lib/middleware/request_id_middleware.rb
 - lib/resque/additions.rb
 - lib/resque/dynamic_queues.rb
 - lib/resque/plugins/custom_logger.rb
@@ -376,9 +407,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubygems_version: 3.0.3
 signing_key: