zuora_connect 3.0.0.pre.j → 3.0.0.pre.q

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ac62f8c020016b3ac368a50dcdc08888b93b985e17ab64b7f2f25fd4c9caccf5
-  data.tar.gz: 5ff7f12a0bd0de56cb069aec3aefa09521c150bf5ff9dc29e0d4299d0f1efd3a
+  metadata.gz: 9348d7aa606a5c6a3100f94afd22efdb2ad2c8405342708420e0e15753cb4816
+  data.tar.gz: ea4ece99e78a056adb90fd21c4308e374365f0c64036e118fc78e83bdea441b2
 SHA512:
-  metadata.gz: f03732fb29484dfa483bc958a6b2079728363252327b156d306acaa898a3b74eb82c1e5083d03c4a79cbb92625ed23edb89be44c37e23e7491556cda22e0c5c3
-  data.tar.gz: 5bd4b3954208453eb3dc089b8bb0b95ae912439c6c54f3b6b1383878dba20026adea0b3ca9d9ce7963f5780a99f942f9ce63c4273d0cea499090a15af7ab9b3a
+  metadata.gz: ec5539d89ee66fa7d02dc64add2073c5281cd138342e28600f54085d2664bc7db7df2f9995617f5fd79607a80eb3c28e09dd0893b68d22a6355f8965a8333ff3
+  data.tar.gz: a3c30217361264f6c934484e550e26d9183cb7aeec362abd2da73be352dba20564f0b2dc6252af15355d307ff64440b7d9c619eab380e8ce2d0b95116a35cc3f

data/app/models/zuora_connect/app_instance_base.rb

@@ -12,6 +12,7 @@ module ZuoraConnect
 
     self.table_name = "zuora_connect_app_instances"
     attr_accessor :options, :mode, :logins, :task_data, :last_refresh, :username, :password, :s3_client, :api_version, :drop_message, :new_session_message, :connect_user, :logitems, :user_timezone
+    @@telegraf_host = nil
     REFRESH_TIMEOUT = 2.minute                #Used to determine how long to wait on current refresh call before executing another
     INSTANCE_REFRESH_WINDOW = 1.hours         #Used to set how how long till app starts attempting to refresh cached task connect data
     INSTANCE_REDIS_CACHE_PERIOD = 24.hours    #Used to determine how long to cached task data will live for
@@ -348,7 +349,7 @@ module ZuoraConnect
 
     def refresh(session: {})
       refresh_count ||= 0
-      skip_connect ||= false
+      skip_connect ||= ZuoraConnect.configuration.skip_connect
       begin
         #Check how app was deployed
         if !self.auto_deployed? && !skip_connect
@@ -362,9 +363,9 @@ module ZuoraConnect
               raise ZuoraConnect::Exceptions::ConnectCommunicationError.new("JSON parse error", response.body, response.code)
             end
 
-            self.build_task(task_data: parsed_json, session: session)  
+            self.build_task(task_data: parsed_json, session: session)
             self.set_backup_creds
-            self.save(validate: false) if self.changed?    
+            self.save(validate: false) if self.changed?
           else
             raise ZuoraConnect::Exceptions::ConnectCommunicationError.new("Error Communicating with Connect", response.body, response.code)
           end
@@ -412,8 +413,8 @@ module ZuoraConnect
       def set_backup_creds
         if self.kms_key.present? && self.kms_key.match(/^arn:aws:.*/) && self.task_data.present?
           self.zuora_logins = self.strip_cache_data(object: self.task_data.dup, keys: ['applications', 'tokens', 'user_settings'])
-        end  
-      end 
+        end
+      end
 
       def zuora_logins=(val)
         write_attribute(:zuora_logins, kms_encrypt(val.to_json))
@@ -478,6 +479,15 @@ module ZuoraConnect
         end
         Thread.current[:appinstance] = self
       end
+
+      def self.write_to_telegraf(*args)
+        if ZuoraConnect.configuration.enable_metrics && !defined?(Prometheus)
+          @@telegraf_host = ZuoraConnect::Telegraf.new() if @@telegraf_host == nil
+          unicorn_stats = ZuoraObservability::Metrics.unicorn_listener if defined?(Unicorn) && Unicorn.respond_to?(:listener_names)
+          @@telegraf_host.write(direction: 'Raindrops', tags: {}, values: unicorn_stats)  unless unicorn_stats.blank?
+          return @@telegraf_host.write(*args)
+        end
+      end
     #### END Task Methods ####
 
     #### START Task Methods ####
@@ -600,7 +610,10 @@ module ZuoraConnect
               self.provisioned_app = nil
             end
           end
-          self.save(:validate => false)
+          if self.changed? 
+            self.save(:validate => false) 
+            self.refresh
+          end
 
           return parsed_json
         end
@@ -1029,7 +1042,7 @@ module ZuoraConnect
             catalog_container = catalog_container[object_id]
           end
 
-          return catalog_container
+          return catalog_container || {}
         end
 
         if defined?(Redis.current) && object_id.present? && object_id.class == String && object_id.present?

data/app/models/zuora_connect/telegraf.rb

@@ -0,0 +1,73 @@
+module ZuoraConnect
+  class Telegraf
+    attr_accessor :host
+
+    OUTBOUND_METRICS = true
+    OUTBOUND_METRICS_NAME = "request-outbound"
+    INBOUND_METRICS = true
+    INBOUND_METRICS_NAME = "request-inbound"
+
+    def initialize
+      self.connect
+    end
+
+    def connect
+      ZuoraConnect.logger.debug(self.format_metric_log('Telegraf','Need new connection')) if ZuoraConnect.configuration.telegraf_debug
+      uri = URI.parse(ZuoraConnect.configuration.telegraf_endpoint)
+      self.host = UDPSocket.new.tap do |socket|
+        socket.connect uri.host, uri.port
+      end
+    rescue => ex
+      self.host = nil
+      ZuoraConnect.logger.warn(self.format_metric_log('Telegraf', "Failed to connect: #{ex.class}")) if Rails.env.to_s != 'production'
+    end
+
+    def write(direction: 'Unknown', tags: {}, values: {})
+      time = Benchmark.measure do |bench|
+        # To avoid writing metrics from rspec tests
+        if Rails.env.to_sym != :test
+          app_instance = Thread.current[:appinstance].present? ? Thread.current[:appinstance].id : 0
+          tags = { app_name: ZuoraObservability::Env.app_name, process_type: ZuoraObservability::Env.process_type, app_instance: app_instance, pod_name: ZuoraObservability::Env.pod_name}.merge(tags)
+
+          if direction == :inbound
+            if INBOUND_METRICS && !Thread.current[:inbound_metric].to_bool
+              self.write_udp(series: INBOUND_METRICS_NAME, tags: tags, values: values)
+              Thread.current[:inbound_metric] = true
+            else
+              return
+            end
+          elsif direction == :outbound
+            self.write_udp(series: OUTBOUND_METRICS_NAME, tags: tags, values: values) if OUTBOUND_METRICS
+          else
+            self.write_udp(series: direction, tags: tags, values: values)
+          end
+        end
+      end
+      if ZuoraConnect.configuration.telegraf_debug
+        ZuoraConnect.logger.debug(self.format_metric_log('Telegraf', tags.to_s))
+        ZuoraConnect.logger.debug(self.format_metric_log('Telegraf', values.to_s))
+        ZuoraConnect.logger.debug(self.format_metric_log('Telegraf', "Writing '#{direction.capitalize}': #{time.real.round(5)} ms"))
+      end
+    end
+
+
+    def write_udp(series: '', tags: {}, values: {})
+      return if !values.present?
+      self.host.write InfluxDB::PointValue.new({series: series, tags: tags, values: values}).dump
+    rescue => ex
+      self.connect
+      ZuoraConnect.logger.warn(self.format_metric_log('Telegraf',"Failed to write udp: #{ex.class}")) if Rails.env.to_s != 'production'
+    end
+
+    def format_metric_log(message, dump = nil)
+      message_color, dump_color = "1;91", "0;1"
+      log_entry = "  \e[#{message_color}m#{message}\e[0m   "
+      log_entry << "\e[#{dump_color}m%#{String === dump ? 's' : 'p'}\e[0m" % dump if dump
+      if Rails.env == :development
+        log_entry
+      else
+        [message, dump].compact.join(' - ')
+      end
+    end
+  end
+end

data/config/initializers/prometheus.rb

@@ -15,29 +15,6 @@ if defined? Prometheus
       dir: prometheus_path
     )
 
-    class ResqueExporter
-      require 'prometheus/client/formats/text'
-      require 'fileutils'
-
-      def initialize
-        @lock = Monitor.new
-        @registry = Prometheus::Client.registry
-        @path = ENV['RESQUE_EXPORTER_PATH'] || Rails.root.join('tmp/resque_exporter')
-        FileUtils.mkdir_p(@path)
-      end
-
-      def export
-        filename = File.join(@path, 'resque_export.prom')
-        @lock.synchronize do
-          File.open(filename, 'w+') do |file|
-            file.write(Prometheus::Client::Formats::Text.marshal(@registry))
-          end
-        end
-      rescue
-        # Ignored
-      end
-    end
-
     most_recent_aggregation = {}
     sum_aggregation = {}
     if defined?(Unicorn)

data/config/initializers/resque.rb

@@ -5,20 +5,6 @@ if defined?(Resque::Worker)
   Resque::Job.send(:include, Resque::SelfLookup)
 end
 
-if defined?(Resque::Job) && defined?(Prometheus)
-  module ResquePrometheusExtensions
-    EXPORTER = Prometheus::ResqueExporter.new
-    def perform
-      super
-      EXPORTER.export
-    end
-  end
-
-  class Resque::Job
-    prepend ResquePrometheusExtensions
-  end
-end
-
 Resque.module_eval do
   # Returns a hash, mapping queue names to queue sizes
   def queue_sizes

data/config/initializers/unicorn.rb

@@ -6,7 +6,7 @@ if defined?(Unicorn::WorkerKiller)
       if defined?(Prometheus)
         Prometheus::UNICORN_KILLS.set(1)
       else
-        ZuoraObservability::Metrics.write_to_telegraf(direction: 'Unicorn-Killer', tags: {app_instance: 0}, values: {kill: 1})
+        ZuoraConnect::AppInstance.write_to_telegraf(direction: 'Unicorn-Killer', tags: {app_instance: 0}, values: {kill: 1})
       end
     end
   end

data/config/initializers/zuora_observability.rb

@@ -0,0 +1,21 @@
+ZuoraObservability.configure do |config|
+  config.zecs_service_hook = lambda do |controller|
+    custom_payload = {}
+    if controller.instance_variable_defined?(:@appinstance)
+      appinstance = controller.instance_variable_get(:@appinstance)
+      if appinstance.logitems.present? && appinstance.logitems.class == Hash
+        ActiveSupport::Deprecation.warn("logitems is deprecated, please use zecs_service_hook. See 'https://gitlab.zeta.tools/extension-products/incubating-projects/zuora_observability/-/blob/master/doc/logging.md' for more details.")
+        custom_payload.merge!(appinstance.logitems)
+      end
+    end
+    custom_payload
+  end
+  config.add_custom_payload_hook do |controller|
+    custom_payload = {}
+    if controller.instance_variable_defined?(:@appinstance)
+      appinstance = controller.instance_variable_get(:@appinstance)
+      custom_payload.merge!({ email: appinstance.connect_user })
+    end
+    custom_payload
+  end
+end

data/lib/metrics/influx/point_value.rb

@@ -0,0 +1,79 @@
+module InfluxDB
+  # Convert data point to string using Line protocol
+  class PointValue
+    attr_reader :series, :values, :tags, :timestamp
+
+    def initialize(data)
+      @series    = escape data[:series], :measurement
+      @values    = escape_values data[:values]
+      @tags      = escape_tags data[:tags]
+      @timestamp = data[:timestamp]
+    end
+
+    def dump
+      dump =  @series.dup
+      dump << ",#{@tags}" if @tags
+      dump << " #{@values}"
+      dump << " #{@timestamp}" if @timestamp
+      dump
+    end
+
+    private
+
+    ESCAPES = {
+      measurement:  [' '.freeze, ','.freeze],
+      tag_key:      ['='.freeze, ' '.freeze, ','.freeze],
+      tag_value:    ['='.freeze, ' '.freeze, ','.freeze],
+      field_key:    ['='.freeze, ' '.freeze, ','.freeze, '"'.freeze],
+      field_value:  ["\\".freeze, '"'.freeze],
+    }.freeze
+
+    private_constant :ESCAPES
+
+    def escape(str, type)
+      # rubocop:disable Layout/AlignParameters
+      str = str.encode "UTF-8".freeze, "UTF-8".freeze,
+        invalid: :replace,
+        undef: :replace,
+        replace: "".freeze
+      # rubocop:enable Layout/AlignParameters
+
+      ESCAPES[type].each do |ch|
+        str = str.gsub(ch) { "\\#{ch}" }
+      end
+      str
+    end
+
+    def escape_values(values)
+      return if values.nil?
+      values.map do |k, v|
+        key = escape(k.to_s, :field_key)
+        val = escape_value(v)
+        "#{key}=#{val}"
+      end.join(",".freeze)
+    end
+
+    def escape_value(value)
+      if value.is_a?(String)
+        '"'.freeze + escape(value, :field_value) + '"'.freeze
+      elsif value.is_a?(Integer)
+        "#{value}i"
+      else
+        value.to_s
+      end
+    end
+
+    def escape_tags(tags)
+      return if tags.nil?
+
+      tags = tags.map do |k, v|
+        key = escape(k.to_s, :tag_key)
+        val = escape(v.to_s, :tag_value)
+
+        "#{key}=#{val}" unless key == "".freeze || val == "".freeze
+      end.compact
+
+      tags.join(",") unless tags.empty?
+    end
+  end
+end

data/lib/metrics/net.rb

@@ -68,12 +68,12 @@ class HttpLogger
       log_request_body(request)
       log_request_headers(request)
       if defined?(response) && response
-        tags = tags.merge({status: response.code.to_i})
+        tags = tags.merge({status: response.code.to_i}) 
         log_response_code(response)
         log_response_headers(response)
         log_response_body(response.body)
       end
-      ZuoraObservability::Metrics.write_to_telegraf(direction: :outbound, tags: tags, values: values)
+      ZuoraConnect::AppInstance.write_to_telegraf(direction: :outbound, tags: tags, values: values)
     end
   end
 
@@ -95,7 +95,7 @@ class HttpLogger
   end
 
   HTTP_METHODS_WITH_BODY = Set.new(%w(POST PUT GET PATCH))
-
+  
   def log_request_body(request)
     if self.class.log_request_body
       if HTTP_METHODS_WITH_BODY.include?(request.method)
@@ -149,8 +149,8 @@ class HttpLogger
   def truncate_body(body)
     if collapse_body_limit && collapse_body_limit > 0 && body && body.size >= collapse_body_limit
       body_piece_size = collapse_body_limit / 2
-      body[0..body_piece_size] +
-        "\n\n<some data truncated>\n\n" +
+      body[0..body_piece_size] + 
+        "\n\n<some data truncated>\n\n" + 
         body[(body.size - body_piece_size)..body.size]
     else
       body
@@ -203,7 +203,7 @@ class Net::HTTP
 
   def request(request, body = nil, &block)
     HttpLogger.perform(self, request, body) do
-      request_without_logging(request, body, &block)
+      request_without_logging(request, body, &block) 
     end
   end
 end
@@ -215,4 +215,4 @@ if defined?(Rails)
       HttpLogger.logger = ZuoraConnect.logger unless HttpLogger.logger
     end
   end
-end
+end

data/lib/middleware/metrics_middleware.rb

@@ -1,6 +1,37 @@
 module ZuoraConnect
   require 'uri'
 
+  # Object of this class is passed to the ActiveSupport::Notification hook
+  class PageRequest
+
+    # This method is triggered when a non error page is loaded (not 404)
+    def call(name, started, finished, unique_id, payload)
+      # If the url contains any css or JavaScript files then do not collect metrics for them
+      return nil if ["css", "assets", "jpg", "png", "jpeg", "ico"].any? { |word| payload[:path].include?(word) }
+
+      # Getting the endpoint and the content_type
+      content_hash = {:html => "text/html", :js => "application/javascript", :json => "application/json", :csv => "text/csv"}
+      content_type = content_hash.key?(payload[:format]) ? content_hash[payload[:format]] : payload[:format]
+      content_type = content_type.to_s.gsub('text/javascript', 'application/javascript')
+
+      # payloads with 500 requests do not have status as it is not set by the controller
+      # https://github.com/rails/rails/issues/33335
+      #status_code = payload[:status] ? payload[:status] : payload[:exception_object].present? ? 500 : ""
+      if payload[:exception].present?
+        status_code, exception = [500, payload[:exception].first]
+      else
+        status_code, exception = [payload[:status], nil]
+      end
+
+      tags = {method: payload[:method], status: status_code, error_type: exception, content_type: content_type, controller: payload[:controller], action: payload[:action]}.compact
+
+      values = {view_time: payload[:view_runtime], db_time: payload[:db_runtime], response_time: ((finished-started)*1000)}.compact
+      values = values.map{ |k,v| [k,v.round(2)]}.to_h
+
+      ZuoraConnect::AppInstanceBase.write_to_telegraf(direction: :inbound, tags: tags, values: values)
+    end
+  end
+
   class MetricsMiddleware
 
     require "zuora_connect/version"
@@ -36,9 +67,9 @@ module ZuoraConnect
         if defined?(Prometheus) && env['PATH_INFO'] == '/connect/internal/metrics'
 
           # Prometheus Stuff
-          metrics = ZuoraObservability::Metrics.resque 
+          metrics = ZuoraObservability::Metrics.resque
           metrics = defined?(ZuoraConnect::AppInstance.get_metrics) ? ZuoraConnect::AppInstance.get_metrics(metrics) : metrics
-         
+
           redis_up = metrics.present? && metrics.dig(:Resque, :Workers_Total).present? ? 1 : 0
           Prometheus::REDIS_CONNECTION.set(redis_up)
 
@@ -77,7 +108,7 @@ module ZuoraConnect
 
               values = {response_time: ((Time.now - start_time)*1000).round(2) }
 
-              ZuoraObservability::Metrics.write_to_telegraf(direction: :inbound, tags: tags, values: values)
+              ZuoraConnect::AppInstanceBase.write_to_telegraf(direction: :inbound, tags: tags, values: values)
             end
           end
           Thread.current[:inbound_metric] = nil

data/lib/zuora_connect.rb

@@ -9,8 +9,8 @@ require 'resque/silence_done'
 require 'resque/self_lookup'
 require 'resque/plugins/custom_logger'
 require 'resque/plugins/app_instance_job'
+require 'metrics/influx/point_value'
 require 'metrics/net'
-require 'mono_logger'
 require 'zuora_connect/zuora_audit'
 require 'active_record'
 require 'zuora_observability'
@@ -87,7 +87,7 @@ module ZuoraConnect
       verify_server_cert: false,
       log_level: Logger::INFO,
       service_name: ENV['DEIS_APP'].present? ? ENV['DEIS_APP'] : Rails.application.class.parent_name,
-      logger: ZuoraObservability::Logger.custom_logger(name: "ElasticAPM", level: MonoLogger::WARN)
+      logger: ZuoraObservability::Logger.custom_logger(name: "ElasticAPM", level: Logger::WARN)
     })
     defaults.merge!({disable_send: true}) if defined?(Rails::Console)