zuora_connect 3.0.0.pre.i → 3.0.0.pre.p

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d50ef7f71f058f62d07317b7cbd801b858b49b5068472170fb067f14904ae0b6
-  data.tar.gz: bb79eeaaeb9e3923b30ae8727f0faee76bb8c4d7f99ba51db0d82391f14f9483
+  metadata.gz: 48c5c6950c9bae664c28fa538b8e5893fc2e477d2d0f1cb609076fe7d465ad76
+  data.tar.gz: 00dabe2bc4bfe823c9c17cd85e335b802c543b84b04b7ced36f9492277d45c18
 SHA512:
-  metadata.gz: ef4103085477c552e5d35b5c3969f5095cf67506c177ce2dce9d78ddb86a27b80c5a975b4facb646c60deaf68e5a4d7b4ee1d688d7a465fdd065bb91c105efce
-  data.tar.gz: 8f49d6547fca5bc40769b229beb59c9f95a6cb07db1de4c76d4ef972279add50a6ee804eb5793ca92e2d7a78d2601a88933d674881c963beebe8998531d695c7
+  metadata.gz: f0a69fc47ca4a1835e8712cc4d88c32d4208407a7bbd7e89df37d2f4bd8915b1a0665d5bf780313b40777c1b4c3c35438e908fcdeabefcca5109e8de7e2dedb6
+  data.tar.gz: 55e492a0fb4cc472ce7fe3f09978032ea48a4eda2f2890efeb3b681b7b55b170b7ed3867d9fab5ba44777c0437d6d4329dd060b0fb24fc959546a0e149ef61fa

data/app/models/zuora_connect/app_instance_base.rb

@@ -12,6 +12,7 @@ module ZuoraConnect
 
     self.table_name = "zuora_connect_app_instances"
     attr_accessor :options, :mode, :logins, :task_data, :last_refresh, :username, :password, :s3_client, :api_version, :drop_message, :new_session_message, :connect_user, :logitems, :user_timezone
+    @@telegraf_host = nil
     REFRESH_TIMEOUT = 2.minute                #Used to determine how long to wait on current refresh call before executing another
     INSTANCE_REFRESH_WINDOW = 1.hours         #Used to set how how long till app starts attempting to refresh cached task connect data
     INSTANCE_REDIS_CACHE_PERIOD = 24.hours    #Used to determine how long to cached task data will live for
@@ -348,7 +349,7 @@ module ZuoraConnect
 
     def refresh(session: {})
       refresh_count ||= 0
-      skip_connect ||= false
+      skip_connect ||= ZuoraConnect.configuration.skip_connect
       begin
         #Check how app was deployed
         if !self.auto_deployed? && !skip_connect
@@ -362,9 +363,9 @@ module ZuoraConnect
               raise ZuoraConnect::Exceptions::ConnectCommunicationError.new("JSON parse error", response.body, response.code)
             end
 
-            self.build_task(task_data: parsed_json, session: session)  
+            self.build_task(task_data: parsed_json, session: session)
             self.set_backup_creds
-            self.save(validate: false) if self.changed?    
+            self.save(validate: false) if self.changed?
           else
             raise ZuoraConnect::Exceptions::ConnectCommunicationError.new("Error Communicating with Connect", response.body, response.code)
           end
@@ -412,8 +413,8 @@ module ZuoraConnect
       def set_backup_creds
         if self.kms_key.present? && self.kms_key.match(/^arn:aws:.*/) && self.task_data.present?
           self.zuora_logins = self.strip_cache_data(object: self.task_data.dup, keys: ['applications', 'tokens', 'user_settings'])
-        end  
-      end 
+        end
+      end
 
       def zuora_logins=(val)
         write_attribute(:zuora_logins, kms_encrypt(val.to_json))
@@ -478,6 +479,15 @@ module ZuoraConnect
         end
         Thread.current[:appinstance] = self
       end
+
+      def self.write_to_telegraf(*args)
+        if ZuoraConnect.configuration.enable_metrics && !defined?(Prometheus)
+          @@telegraf_host = ZuoraConnect::Telegraf.new() if @@telegraf_host == nil
+          unicorn_stats = ZuoraObservability::Metrics.unicorn_listener if defined?(Unicorn) && Unicorn.respond_to?(:listener_names)
+          @@telegraf_host.write(direction: 'Raindrops', tags: {}, values: unicorn_stats)  unless unicorn_stats.blank?
+          return @@telegraf_host.write(*args)
+        end
+      end
     #### END Task Methods ####
 
     #### START Task Methods ####
@@ -600,7 +610,10 @@ module ZuoraConnect
               self.provisioned_app = nil
             end
           end
-          self.save(:validate => false)
+          if self.changed? 
+            self.save(:validate => false) 
+            self.refresh
+          end
 
           return parsed_json
         end
@@ -1029,7 +1042,7 @@ module ZuoraConnect
             catalog_container = catalog_container[object_id]
           end
 
-          return catalog_container
+          return catalog_container || {}
         end
 
         if defined?(Redis.current) && object_id.present? && object_id.class == String && object_id.present?

data/app/models/zuora_connect/telegraf.rb

@@ -0,0 +1,73 @@
+module ZuoraConnect
+  class Telegraf
+    attr_accessor :host
+
+    OUTBOUND_METRICS = true
+    OUTBOUND_METRICS_NAME = "request-outbound"
+    INBOUND_METRICS = true
+    INBOUND_METRICS_NAME = "request-inbound"
+
+    def initialize
+      self.connect
+    end
+
+    def connect
+      ZuoraConnect.logger.debug(self.format_metric_log('Telegraf','Need new connection')) if ZuoraConnect.configuration.telegraf_debug
+      uri = URI.parse(ZuoraConnect.configuration.telegraf_endpoint)
+      self.host = UDPSocket.new.tap do |socket|
+        socket.connect uri.host, uri.port
+      end
+    rescue => ex
+      self.host = nil
+      ZuoraConnect.logger.warn(self.format_metric_log('Telegraf', "Failed to connect: #{ex.class}")) if Rails.env.to_s != 'production'
+    end
+
+    def write(direction: 'Unknown', tags: {}, values: {})
+      time = Benchmark.measure do |bench|
+        # To avoid writing metrics from rspec tests
+        if Rails.env.to_sym != :test
+          app_instance = Thread.current[:appinstance].present? ? Thread.current[:appinstance].id : 0
+          tags = { app_name: ZuoraObservability::Env.app_name, process_type: ZuoraObservability::Env.process_type, app_instance: app_instance, pod_name: ZuoraObservability::Env.pod_name}.merge(tags)
+
+          if direction == :inbound
+            if INBOUND_METRICS && !Thread.current[:inbound_metric].to_bool
+              self.write_udp(series: INBOUND_METRICS_NAME, tags: tags, values: values)
+              Thread.current[:inbound_metric] = true
+            else
+              return
+            end
+          elsif direction == :outbound
+            self.write_udp(series: OUTBOUND_METRICS_NAME, tags: tags, values: values) if OUTBOUND_METRICS
+          else
+            self.write_udp(series: direction, tags: tags, values: values)
+          end
+        end
+      end
+      if ZuoraConnect.configuration.telegraf_debug
+        ZuoraConnect.logger.debug(self.format_metric_log('Telegraf', tags.to_s))
+        ZuoraConnect.logger.debug(self.format_metric_log('Telegraf', values.to_s))
+        ZuoraConnect.logger.debug(self.format_metric_log('Telegraf', "Writing '#{direction.capitalize}': #{time.real.round(5)} ms"))
+      end
+    end
+
+
+    def write_udp(series: '', tags: {}, values: {})
+      return if !values.present?
+      self.host.write InfluxDB::PointValue.new({series: series, tags: tags, values: values}).dump
+    rescue => ex
+      self.connect
+      ZuoraConnect.logger.warn(self.format_metric_log('Telegraf',"Failed to write udp: #{ex.class}")) if Rails.env.to_s != 'production'
+    end
+
+    def format_metric_log(message, dump = nil)
+      message_color, dump_color = "1;91", "0;1"
+      log_entry = "  \e[#{message_color}m#{message}\e[0m   "
+      log_entry << "\e[#{dump_color}m%#{String === dump ? 's' : 'p'}\e[0m" % dump if dump
+      if Rails.env == :development
+        log_entry
+      else
+        [message, dump].compact.join(' - ')
+      end
+    end
+  end
+end

data/config/initializers/prometheus.rb

@@ -15,29 +15,6 @@ if defined? Prometheus
       dir: prometheus_path
     )
 
-    class ResqueExporter
-      require 'prometheus/client/formats/text'
-      require 'fileutils'
-
-      def initialize
-        @lock = Monitor.new
-        @registry = Prometheus::Client.registry
-        @path = ENV['RESQUE_EXPORTER_PATH'] || Rails.root.join('tmp/resque_exporter')
-        FileUtils.mkdir_p(@path)
-      end
-
-      def export
-        filename = File.join(@path, 'resque_export.prom')
-        @lock.synchronize do
-          File.open(filename, 'w+') do |file|
-            file.write(Prometheus::Client::Formats::Text.marshal(@registry))
-          end
-        end
-      rescue
-        # Ignored
-      end
-    end
-
     most_recent_aggregation = {}
     sum_aggregation = {}
     if defined?(Unicorn)

data/config/initializers/resque.rb

@@ -5,20 +5,6 @@ if defined?(Resque::Worker)
   Resque::Job.send(:include, Resque::SelfLookup)
 end
 
-if defined?(Resque::Job) && defined?(Prometheus)
-  module ResquePrometheusExtensions
-    EXPORTER = Prometheus::ResqueExporter.new
-    def perform
-      super
-      EXPORTER.export
-    end
-  end
-
-  class Resque::Job
-    prepend ResquePrometheusExtensions
-  end
-end
-
 Resque.module_eval do
   # Returns a hash, mapping queue names to queue sizes
   def queue_sizes

data/config/initializers/unicorn.rb

@@ -6,7 +6,7 @@ if defined?(Unicorn::WorkerKiller)
       if defined?(Prometheus)
         Prometheus::UNICORN_KILLS.set(1)
       else
-        ZuoraObservability::Metrics.write_to_telegraf(direction: 'Unicorn-Killer', tags: {app_instance: 0}, values: {kill: 1})
+        ZuoraConnect::AppInstance.write_to_telegraf(direction: 'Unicorn-Killer', tags: {app_instance: 0}, values: {kill: 1})
       end
     end
   end

data/lib/metrics/influx/point_value.rb

@@ -0,0 +1,79 @@
+module InfluxDB
+  # Convert data point to string using Line protocol
+  class PointValue
+    attr_reader :series, :values, :tags, :timestamp
+
+    def initialize(data)
+      @series    = escape data[:series], :measurement
+      @values    = escape_values data[:values]
+      @tags      = escape_tags data[:tags]
+      @timestamp = data[:timestamp]
+    end
+
+    def dump
+      dump =  @series.dup
+      dump << ",#{@tags}" if @tags
+      dump << " #{@values}"
+      dump << " #{@timestamp}" if @timestamp
+      dump
+    end
+
+    private
+
+    ESCAPES = {
+      measurement:  [' '.freeze, ','.freeze],
+      tag_key:      ['='.freeze, ' '.freeze, ','.freeze],
+      tag_value:    ['='.freeze, ' '.freeze, ','.freeze],
+      field_key:    ['='.freeze, ' '.freeze, ','.freeze, '"'.freeze],
+      field_value:  ["\\".freeze, '"'.freeze],
+    }.freeze
+
+    private_constant :ESCAPES
+
+    def escape(str, type)
+      # rubocop:disable Layout/AlignParameters
+      str = str.encode "UTF-8".freeze, "UTF-8".freeze,
+        invalid: :replace,
+        undef: :replace,
+        replace: "".freeze
+      # rubocop:enable Layout/AlignParameters
+
+      ESCAPES[type].each do |ch|
+        str = str.gsub(ch) { "\\#{ch}" }
+      end
+      str
+    end
+
+    def escape_values(values)
+      return if values.nil?
+      values.map do |k, v|
+        key = escape(k.to_s, :field_key)
+        val = escape_value(v)
+        "#{key}=#{val}"
+      end.join(",".freeze)
+    end
+
+    def escape_value(value)
+      if value.is_a?(String)
+        '"'.freeze + escape(value, :field_value) + '"'.freeze
+      elsif value.is_a?(Integer)
+        "#{value}i"
+      else
+        value.to_s
+      end
+    end
+
+    def escape_tags(tags)
+      return if tags.nil?
+
+      tags = tags.map do |k, v|
+        key = escape(k.to_s, :tag_key)
+        val = escape(v.to_s, :tag_value)
+
+        "#{key}=#{val}" unless key == "".freeze || val == "".freeze
+      end.compact
+
+      tags.join(",") unless tags.empty?
+    end
+  end
+end

data/lib/metrics/net.rb

@@ -68,12 +68,12 @@ class HttpLogger
       log_request_body(request)
       log_request_headers(request)
       if defined?(response) && response
-        tags = tags.merge({status: response.code.to_i})
+        tags = tags.merge({status: response.code.to_i}) 
         log_response_code(response)
         log_response_headers(response)
         log_response_body(response.body)
       end
-      ZuoraObservability::Metrics.write_to_telegraf(direction: :outbound, tags: tags, values: values)
+      ZuoraConnect::AppInstance.write_to_telegraf(direction: :outbound, tags: tags, values: values)
     end
   end
 
@@ -95,7 +95,7 @@ class HttpLogger
   end
 
   HTTP_METHODS_WITH_BODY = Set.new(%w(POST PUT GET PATCH))
-
+  
   def log_request_body(request)
     if self.class.log_request_body
       if HTTP_METHODS_WITH_BODY.include?(request.method)
@@ -149,8 +149,8 @@ class HttpLogger
   def truncate_body(body)
     if collapse_body_limit && collapse_body_limit > 0 && body && body.size >= collapse_body_limit
       body_piece_size = collapse_body_limit / 2
-      body[0..body_piece_size] +
-        "\n\n<some data truncated>\n\n" +
+      body[0..body_piece_size] + 
+        "\n\n<some data truncated>\n\n" + 
         body[(body.size - body_piece_size)..body.size]
     else
       body
@@ -203,7 +203,7 @@ class Net::HTTP
 
   def request(request, body = nil, &block)
     HttpLogger.perform(self, request, body) do
-      request_without_logging(request, body, &block)
+      request_without_logging(request, body, &block) 
     end
   end
 end
@@ -215,4 +215,4 @@ if defined?(Rails)
       HttpLogger.logger = ZuoraConnect.logger unless HttpLogger.logger
     end
   end
-end
+end

data/lib/middleware/metrics_middleware.rb

@@ -1,6 +1,37 @@
 module ZuoraConnect
   require 'uri'
 
+  # Object of this class is passed to the ActiveSupport::Notification hook
+  class PageRequest
+
+    # This method is triggered when a non error page is loaded (not 404)
+    def call(name, started, finished, unique_id, payload)
+      # If the url contains any css or JavaScript files then do not collect metrics for them
+      return nil if ["css", "assets", "jpg", "png", "jpeg", "ico"].any? { |word| payload[:path].include?(word) }
+
+      # Getting the endpoint and the content_type
+      content_hash = {:html => "text/html", :js => "application/javascript", :json => "application/json", :csv => "text/csv"}
+      content_type = content_hash.key?(payload[:format]) ? content_hash[payload[:format]] : payload[:format]
+      content_type = content_type.to_s.gsub('text/javascript', 'application/javascript')
+
+      # payloads with 500 requests do not have status as it is not set by the controller
+      # https://github.com/rails/rails/issues/33335
+      #status_code = payload[:status] ? payload[:status] : payload[:exception_object].present? ? 500 : ""
+      if payload[:exception].present?
+        status_code, exception = [500, payload[:exception].first]
+      else
+        status_code, exception = [payload[:status], nil]
+      end
+
+      tags = {method: payload[:method], status: status_code, error_type: exception, content_type: content_type, controller: payload[:controller], action: payload[:action]}.compact
+
+      values = {view_time: payload[:view_runtime], db_time: payload[:db_runtime], response_time: ((finished-started)*1000)}.compact
+      values = values.map{ |k,v| [k,v.round(2)]}.to_h
+
+      ZuoraConnect::AppInstanceBase.write_to_telegraf(direction: :inbound, tags: tags, values: values)
+    end
+  end
+
   class MetricsMiddleware
 
     require "zuora_connect/version"
@@ -36,9 +67,9 @@ module ZuoraConnect
         if defined?(Prometheus) && env['PATH_INFO'] == '/connect/internal/metrics'
 
           # Prometheus Stuff
-          metrics = ZuoraObservability::Metrics.resque 
+          metrics = ZuoraObservability::Metrics.resque
           metrics = defined?(ZuoraConnect::AppInstance.get_metrics) ? ZuoraConnect::AppInstance.get_metrics(metrics) : metrics
-         
+
           redis_up = metrics.present? && metrics.dig(:Resque, :Workers_Total).present? ? 1 : 0
           Prometheus::REDIS_CONNECTION.set(redis_up)
 
@@ -77,7 +108,7 @@ module ZuoraConnect
 
               values = {response_time: ((Time.now - start_time)*1000).round(2) }
 
-              ZuoraObservability::Metrics.write_to_telegraf(direction: :inbound, tags: tags, values: values)
+              ZuoraConnect::AppInstanceBase.write_to_telegraf(direction: :inbound, tags: tags, values: values)
             end
           end
           Thread.current[:inbound_metric] = nil

data/lib/zuora_connect.rb

@@ -9,6 +9,7 @@ require 'resque/silence_done'
 require 'resque/self_lookup'
 require 'resque/plugins/custom_logger'
 require 'resque/plugins/app_instance_job'
+require 'metrics/influx/point_value'
 require 'metrics/net'
 require 'mono_logger'
 require 'zuora_connect/zuora_audit'

data/lib/zuora_connect/configuration.rb

@@ -7,7 +7,7 @@ module ZuoraConnect
 
     attr_accessor :oauth_client_id, :oauth_client_secret, :oauth_client_redirect_uri
 
-    attr_accessor :dev_mode_logins, :dev_mode_options, :dev_mode_mode, :dev_mode_appinstance, :dev_mode_user, :dev_mode_pass, :dev_mode_admin, :dev_mode_secret_access_key,:dev_mode_access_key_id,:aws_region, :s3_bucket_name, :s3_folder_name, :json_logging, :insert_migrations
+    attr_accessor :dev_mode_logins, :dev_mode_options, :dev_mode_mode, :dev_mode_appinstance, :dev_mode_user, :dev_mode_pass, :dev_mode_admin, :dev_mode_secret_access_key,:dev_mode_access_key_id,:aws_region, :s3_bucket_name, :s3_folder_name, :json_logging, :insert_migrations, :skip_connect
 
     def initialize
       @default_locale = :en
@@ -20,6 +20,7 @@ module ZuoraConnect
       @silencer_resque_finish = true
       @blpop_queue = false
       @insert_migrations = true
+      @skip_connect = false
 
       # Setting the app name for telegraf write
       @enable_metrics = false

data/lib/zuora_connect/controllers/helpers.rb

@@ -125,7 +125,7 @@ module ZuoraConnect
             PaperTrail.whodunnit =  session["#{@appinstance.id}::user::email"] if defined?(PaperTrail)
           end
 
-          locale = (session["#{@appinstance.id}::user::locale"] || "").gsub("_", "-") 
+          locale = (session["#{@appinstance.id}::user::locale"] || "").gsub("_", "-")
           begin
             I18n.locale = locale.present? ? locale : @appinstance.locale
           rescue I18n::InvalidLocale => ex
@@ -280,6 +280,7 @@ module ZuoraConnect
                 next_instance_id,
                 zuora_entity_id,
                 rest_domain,
+                tenant_id: request.headers['zuora-tenant-id'],
                 retry_count: retry_count
               )
             rescue ActiveRecord::RecordNotUnique
@@ -302,6 +303,7 @@ module ZuoraConnect
     private
       def setup_instance_via_prod_mode
         zuora_entity_id = request.headers['ZuoraCurrentEntity'] || cookies['ZuoraCurrentEntity']
+        ZuoraConnect::ZuoraUser.current_user_id = nil
 
         if zuora_entity_id.present?
           zuora_tenant_id = cookies['Zuora-Tenant-Id']
@@ -417,7 +419,6 @@ module ZuoraConnect
             if defined?(@appinstance) && !@appinstance['zuora_logins'].nil?
               #Add user/update
               begin
-                ZuoraConnect::ZuoraUser.reset_table_name
                 @zuora_user = ZuoraConnect::ZuoraUser.where(:zuora_user_id => zuora_user_id).first
               rescue ActiveRecord::StatementInvalid => ex
                 if ex.message.include?("PG::UndefinedTable") && ex.message.include?("zuora_users")
@@ -636,7 +637,7 @@ module ZuoraConnect
         (ZuoraConnect::AppInstance.all.where("id > #{min_instance_id}").order(id: :desc).limit(1).pluck(:id).first || min_instance_id) + 1
       end
 
-      def new_instance(id, zuora_entity_id, rest_domain, task_data: nil, retry_count: 0)
+      def new_instance(id, zuora_entity_id, rest_domain, tenant_id: nil, task_data: nil, retry_count: 0)
         app_instance = ZuoraConnect::AppInstance.new(
           :id => id,
           :api_token => generate_token,
@@ -646,6 +647,8 @@ module ZuoraConnect
           :zuora_entity_ids => [zuora_entity_id]
         )
 
+        app_instance[:zuora_tenant_ids] = [tenant_id.to_s] if tenant_id.present?
+
         if task_data.nil?
           # no encryption
           app_instance['zuora_logins'] = task_data
@@ -678,15 +681,15 @@ module ZuoraConnect
       def setup_instance_via_dev_mode
         session["appInstance"] = ZuoraConnect.configuration.dev_mode_appinstance
         session["#{ZuoraConnect.configuration.dev_mode_appinstance}::admin"] =  ZuoraConnect.configuration.dev_mode_admin
- 
+
         values = {
           id: ZuoraConnect.configuration.dev_mode_appinstance,
-          access_token: ZuoraConnect.configuration.dev_mode_user, 
-          refresh_token: ZuoraConnect.configuration.dev_mode_pass, 
-          token: ZuoraConnect.configuration.dev_mode_pass+ZuoraConnect.configuration.dev_mode_pass, 
+          access_token: ZuoraConnect.configuration.dev_mode_user,
+          refresh_token: ZuoraConnect.configuration.dev_mode_pass,
+          token: ZuoraConnect.configuration.dev_mode_pass+ZuoraConnect.configuration.dev_mode_pass,
           api_token: ZuoraConnect.configuration.dev_mode_pass+ZuoraConnect.configuration.dev_mode_pass
         }
-        
+
         @appinstance = ZuoraConnect::AppInstance.find_by(:id => ZuoraConnect.configuration.dev_mode_appinstance.to_i)
         ZuoraConnect::ZuoraUser.current_user_id = 0
         if @appinstance.blank?
@@ -704,4 +707,4 @@ module ZuoraConnect
       end
     end
   end
-end
+end