zuora_connect 3.0.0.pre.m → 3.0.0.pre.n

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fa8acf1336b53b9d3407afba6a37d962a74434e087974a5810955c20c7e5e2f1
-  data.tar.gz: e3c8f1a30f247c68e78891244e01fdded4c1c5b76ede6540f1808aed255b987d
+  metadata.gz: e800e6da56132f3a8b5e0f52d2953347af0e73db371cef49421128b483cecb93
+  data.tar.gz: 24b4315f400c88afcd3916548887208b1a10b19a841617df4007beb0280cdf42
 SHA512:
-  metadata.gz: 525866e7f02a4ad82b6d8920ee372f091a6f1216c6ae851149b5e5835f71db237c2d4313039abf0b30279f030f7f6994669a8dc38728fea3c427a1402d40bc44
-  data.tar.gz: fec0a295b4a3493c9b15320e4cd4578396795f63af4e9ad58fbcd88627b16ba7e7ab72bf7826efc85efe3501c26ba65cab47a88c8bf06a8f8fce7df7417eb7ce
+  metadata.gz: d82d6ded09b50ae677194d3219b3c8d7204f8f18b9ede85431269ff3f677afa205a17ddb20f25760dd35e3d87da35ebac471d86e8d861d0fe5585b4671ddbdc3
+  data.tar.gz: b899378707679de0076efdfc1b14a73e781f03216f2ac1391c176567034a436b1074e9f71966e9efb59490442fe22621a3da19481b0397410a1e4bad97657477

data/app/models/zuora_connect/app_instance_base.rb

@@ -12,6 +12,7 @@ module ZuoraConnect
 
     self.table_name = "zuora_connect_app_instances"
     attr_accessor :options, :mode, :logins, :task_data, :last_refresh, :username, :password, :s3_client, :api_version, :drop_message, :new_session_message, :connect_user, :logitems, :user_timezone
+    @@telegraf_host = nil
     REFRESH_TIMEOUT = 2.minute                #Used to determine how long to wait on current refresh call before executing another
     INSTANCE_REFRESH_WINDOW = 1.hours         #Used to set how how long till app starts attempting to refresh cached task connect data
     INSTANCE_REDIS_CACHE_PERIOD = 24.hours    #Used to determine how long to cached task data will live for
@@ -362,9 +363,9 @@ module ZuoraConnect
               raise ZuoraConnect::Exceptions::ConnectCommunicationError.new("JSON parse error", response.body, response.code)
             end
 
-            self.build_task(task_data: parsed_json, session: session)  
+            self.build_task(task_data: parsed_json, session: session)
             self.set_backup_creds
-            self.save(validate: false) if self.changed?    
+            self.save(validate: false) if self.changed?
           else
             raise ZuoraConnect::Exceptions::ConnectCommunicationError.new("Error Communicating with Connect", response.body, response.code)
           end
@@ -412,8 +413,8 @@ module ZuoraConnect
       def set_backup_creds
         if self.kms_key.present? && self.kms_key.match(/^arn:aws:.*/) && self.task_data.present?
           self.zuora_logins = self.strip_cache_data(object: self.task_data.dup, keys: ['applications', 'tokens', 'user_settings'])
-        end  
-      end 
+        end
+      end
 
       def zuora_logins=(val)
         write_attribute(:zuora_logins, kms_encrypt(val.to_json))
@@ -478,6 +479,15 @@ module ZuoraConnect
         end
         Thread.current[:appinstance] = self
       end
+
+      def self.write_to_telegraf(*args)
+        if ZuoraConnect.configuration.enable_metrics && !defined?(Prometheus)
+          @@telegraf_host = ZuoraConnect::Telegraf.new() if @@telegraf_host == nil
+          unicorn_stats = ZuoraObservability::Metrics.unicorn_listener if defined?(Unicorn) && Unicorn.respond_to?(:listener_names)
+          @@telegraf_host.write(direction: 'Raindrops', tags: {}, values: unicorn_stats)  unless unicorn_stats.blank?
+          return @@telegraf_host.write(*args)
+        end
+      end
     #### END Task Methods ####
 
     #### START Task Methods ####
@@ -600,7 +610,10 @@ module ZuoraConnect
               self.provisioned_app = nil
             end
           end
-          self.save(:validate => false)
+          if self.changed? 
+            self.save(:validate => false) 
+            self.refresh
+          end
 
           return parsed_json
         end

data/app/models/zuora_connect/telegraf.rb

@@ -0,0 +1,73 @@
+module ZuoraConnect
+  class Telegraf
+    attr_accessor :host
+
+    OUTBOUND_METRICS = true
+    OUTBOUND_METRICS_NAME = "request-outbound"
+    INBOUND_METRICS = true
+    INBOUND_METRICS_NAME = "request-inbound"
+
+    def initialize
+      self.connect
+    end
+
+    def connect
+      ZuoraConnect.logger.debug(self.format_metric_log('Telegraf','Need new connection')) if ZuoraConnect.configuration.telegraf_debug
+      uri = URI.parse(ZuoraConnect.configuration.telegraf_endpoint)
+      self.host = UDPSocket.new.tap do |socket|
+        socket.connect uri.host, uri.port
+      end
+    rescue => ex
+      self.host = nil
+      ZuoraConnect.logger.warn(self.format_metric_log('Telegraf', "Failed to connect: #{ex.class}")) if Rails.env.to_s != 'production'
+    end
+
+    def write(direction: 'Unknown', tags: {}, values: {})
+      time = Benchmark.measure do |bench|
+        # To avoid writing metrics from rspec tests
+        if Rails.env.to_sym != :test
+          app_instance = Thread.current[:appinstance].present? ? Thread.current[:appinstance].id : 0
+          tags = { app_name: ZuoraObservability::Env.app_name, process_type: ZuoraObservability::Env.process_type, app_instance: app_instance, pod_name: ZuoraObservability::Env.pod_name}.merge(tags)
+
+          if direction == :inbound
+            if INBOUND_METRICS && !Thread.current[:inbound_metric].to_bool
+              self.write_udp(series: INBOUND_METRICS_NAME, tags: tags, values: values)
+              Thread.current[:inbound_metric] = true
+            else
+              return
+            end
+          elsif direction == :outbound
+            self.write_udp(series: OUTBOUND_METRICS_NAME, tags: tags, values: values) if OUTBOUND_METRICS
+          else
+            self.write_udp(series: direction, tags: tags, values: values)
+          end
+        end
+      end
+      if ZuoraConnect.configuration.telegraf_debug
+        ZuoraConnect.logger.debug(self.format_metric_log('Telegraf', tags.to_s))
+        ZuoraConnect.logger.debug(self.format_metric_log('Telegraf', values.to_s))
+        ZuoraConnect.logger.debug(self.format_metric_log('Telegraf', "Writing '#{direction.capitalize}': #{time.real.round(5)} ms"))
+      end
+    end
+
+
+    def write_udp(series: '', tags: {}, values: {})
+      return if !values.present?
+      self.host.write InfluxDB::PointValue.new({series: series, tags: tags, values: values}).dump
+    rescue => ex
+      self.connect
+      ZuoraConnect.logger.warn(self.format_metric_log('Telegraf',"Failed to write udp: #{ex.class}")) if Rails.env.to_s != 'production'
+    end
+
+    def format_metric_log(message, dump = nil)
+      message_color, dump_color = "1;91", "0;1"
+      log_entry = "  \e[#{message_color}m#{message}\e[0m   "
+      log_entry << "\e[#{dump_color}m%#{String === dump ? 's' : 'p'}\e[0m" % dump if dump
+      if Rails.env == :development
+        log_entry
+      else
+        [message, dump].compact.join(' - ')
+      end
+    end
+  end
+end

data/config/initializers/unicorn.rb

@@ -6,7 +6,7 @@ if defined?(Unicorn::WorkerKiller)
       if defined?(Prometheus)
         Prometheus::UNICORN_KILLS.set(1)
       else
-        ZuoraObservability::Metrics.write_to_telegraf(direction: 'Unicorn-Killer', tags: {app_instance: 0}, values: {kill: 1})
+        ZuoraConnect::AppInstance.write_to_telegraf(direction: 'Unicorn-Killer', tags: {app_instance: 0}, values: {kill: 1})
       end
     end
   end

data/lib/metrics/influx/point_value.rb

@@ -0,0 +1,79 @@
+module InfluxDB
+  # Convert data point to string using Line protocol
+  class PointValue
+    attr_reader :series, :values, :tags, :timestamp
+
+    def initialize(data)
+      @series    = escape data[:series], :measurement
+      @values    = escape_values data[:values]
+      @tags      = escape_tags data[:tags]
+      @timestamp = data[:timestamp]
+    end
+
+    def dump
+      dump =  @series.dup
+      dump << ",#{@tags}" if @tags
+      dump << " #{@values}"
+      dump << " #{@timestamp}" if @timestamp
+      dump
+    end
+
+    private
+
+    ESCAPES = {
+      measurement:  [' '.freeze, ','.freeze],
+      tag_key:      ['='.freeze, ' '.freeze, ','.freeze],
+      tag_value:    ['='.freeze, ' '.freeze, ','.freeze],
+      field_key:    ['='.freeze, ' '.freeze, ','.freeze, '"'.freeze],
+      field_value:  ["\\".freeze, '"'.freeze],
+    }.freeze
+
+    private_constant :ESCAPES
+
+    def escape(str, type)
+      # rubocop:disable Layout/AlignParameters
+      str = str.encode "UTF-8".freeze, "UTF-8".freeze,
+        invalid: :replace,
+        undef: :replace,
+        replace: "".freeze
+      # rubocop:enable Layout/AlignParameters
+
+      ESCAPES[type].each do |ch|
+        str = str.gsub(ch) { "\\#{ch}" }
+      end
+      str
+    end
+
+    def escape_values(values)
+      return if values.nil?
+      values.map do |k, v|
+        key = escape(k.to_s, :field_key)
+        val = escape_value(v)
+        "#{key}=#{val}"
+      end.join(",".freeze)
+    end
+
+    def escape_value(value)
+      if value.is_a?(String)
+        '"'.freeze + escape(value, :field_value) + '"'.freeze
+      elsif value.is_a?(Integer)
+        "#{value}i"
+      else
+        value.to_s
+      end
+    end
+
+    def escape_tags(tags)
+      return if tags.nil?
+
+      tags = tags.map do |k, v|
+        key = escape(k.to_s, :tag_key)
+        val = escape(v.to_s, :tag_value)
+
+        "#{key}=#{val}" unless key == "".freeze || val == "".freeze
+      end.compact
+
+      tags.join(",") unless tags.empty?
+    end
+  end
+end

data/lib/metrics/net.rb

@@ -68,12 +68,12 @@ class HttpLogger
       log_request_body(request)
       log_request_headers(request)
       if defined?(response) && response
-        tags = tags.merge({status: response.code.to_i})
+        tags = tags.merge({status: response.code.to_i}) 
         log_response_code(response)
         log_response_headers(response)
         log_response_body(response.body)
       end
-      ZuoraObservability::Metrics.write_to_telegraf(direction: :outbound, tags: tags, values: values)
+      ZuoraConnect::AppInstance.write_to_telegraf(direction: :outbound, tags: tags, values: values)
     end
   end
 
@@ -95,7 +95,7 @@ class HttpLogger
   end
 
   HTTP_METHODS_WITH_BODY = Set.new(%w(POST PUT GET PATCH))
-
+  
   def log_request_body(request)
     if self.class.log_request_body
       if HTTP_METHODS_WITH_BODY.include?(request.method)
@@ -149,8 +149,8 @@ class HttpLogger
   def truncate_body(body)
     if collapse_body_limit && collapse_body_limit > 0 && body && body.size >= collapse_body_limit
       body_piece_size = collapse_body_limit / 2
-      body[0..body_piece_size] +
-        "\n\n<some data truncated>\n\n" +
+      body[0..body_piece_size] + 
+        "\n\n<some data truncated>\n\n" + 
         body[(body.size - body_piece_size)..body.size]
     else
       body
@@ -203,7 +203,7 @@ class Net::HTTP
 
   def request(request, body = nil, &block)
     HttpLogger.perform(self, request, body) do
-      request_without_logging(request, body, &block)
+      request_without_logging(request, body, &block) 
     end
   end
 end
@@ -215,4 +215,4 @@ if defined?(Rails)
       HttpLogger.logger = ZuoraConnect.logger unless HttpLogger.logger
     end
   end
-end
+end

data/lib/middleware/metrics_middleware.rb

@@ -1,6 +1,37 @@
 module ZuoraConnect
   require 'uri'
 
+  # Object of this class is passed to the ActiveSupport::Notification hook
+  class PageRequest
+
+    # This method is triggered when a non error page is loaded (not 404)
+    def call(name, started, finished, unique_id, payload)
+      # If the url contains any css or JavaScript files then do not collect metrics for them
+      return nil if ["css", "assets", "jpg", "png", "jpeg", "ico"].any? { |word| payload[:path].include?(word) }
+
+      # Getting the endpoint and the content_type
+      content_hash = {:html => "text/html", :js => "application/javascript", :json => "application/json", :csv => "text/csv"}
+      content_type = content_hash.key?(payload[:format]) ? content_hash[payload[:format]] : payload[:format]
+      content_type = content_type.to_s.gsub('text/javascript', 'application/javascript')
+
+      # payloads with 500 requests do not have status as it is not set by the controller
+      # https://github.com/rails/rails/issues/33335
+      #status_code = payload[:status] ? payload[:status] : payload[:exception_object].present? ? 500 : ""
+      if payload[:exception].present?
+        status_code, exception = [500, payload[:exception].first]
+      else
+        status_code, exception = [payload[:status], nil]
+      end
+
+      tags = {method: payload[:method], status: status_code, error_type: exception, content_type: content_type, controller: payload[:controller], action: payload[:action]}.compact
+
+      values = {view_time: payload[:view_runtime], db_time: payload[:db_runtime], response_time: ((finished-started)*1000)}.compact
+      values = values.map{ |k,v| [k,v.round(2)]}.to_h
+
+      ZuoraConnect::AppInstanceBase.write_to_telegraf(direction: :inbound, tags: tags, values: values)
+    end
+  end
+
   class MetricsMiddleware
 
     require "zuora_connect/version"
@@ -36,9 +67,9 @@ module ZuoraConnect
         if defined?(Prometheus) && env['PATH_INFO'] == '/connect/internal/metrics'
 
           # Prometheus Stuff
-          metrics = ZuoraObservability::Metrics.resque 
+          metrics = ZuoraObservability::Metrics.resque
           metrics = defined?(ZuoraConnect::AppInstance.get_metrics) ? ZuoraConnect::AppInstance.get_metrics(metrics) : metrics
-         
+
           redis_up = metrics.present? && metrics.dig(:Resque, :Workers_Total).present? ? 1 : 0
           Prometheus::REDIS_CONNECTION.set(redis_up)
 
@@ -77,7 +108,7 @@ module ZuoraConnect
 
               values = {response_time: ((Time.now - start_time)*1000).round(2) }
 
-              ZuoraObservability::Metrics.write_to_telegraf(direction: :inbound, tags: tags, values: values)
+              ZuoraConnect::AppInstanceBase.write_to_telegraf(direction: :inbound, tags: tags, values: values)
             end
           end
           Thread.current[:inbound_metric] = nil

data/lib/zuora_connect.rb

@@ -9,6 +9,7 @@ require 'resque/silence_done'
 require 'resque/self_lookup'
 require 'resque/plugins/custom_logger'
 require 'resque/plugins/app_instance_job'
+require 'metrics/influx/point_value'
 require 'metrics/net'
 require 'mono_logger'
 require 'zuora_connect/zuora_audit'

data/lib/zuora_connect/railtie.rb

@@ -32,5 +32,8 @@ module ZuoraConnect
         app.config.middleware.use Rack::Deflater, if: ->(env, *) { env['PATH_INFO'] == '/connect/internal/metrics' }
       end
     end
+
+    # hook to process_action
+    ActiveSupport::Notifications.subscribe('process_action.action_controller', ZuoraConnect::PageRequest.new)
   end
 end

data/lib/zuora_connect/version.rb

@@ -1,3 +1,3 @@
 module ZuoraConnect
-  VERSION = "3.0.0-m"
+  VERSION = "3.0.0-n"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: zuora_connect
 version: !ruby/object:Gem::Version
-  version: 3.0.0.pre.m
+  version: 3.0.0.pre.n
 platform: ruby
 authors:
 - Connect Team
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-12-23 00:00:00.000000000 Z
+date: 2021-01-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: apartment
@@ -360,6 +360,7 @@ files:
 - app/models/zuora_connect/app_instance.rb
 - app/models/zuora_connect/app_instance_base.rb
 - app/models/zuora_connect/login.rb
+- app/models/zuora_connect/telegraf.rb
 - app/models/zuora_connect/zuora_user.rb
 - app/views/layouts/zuora_connect/application.html.erb
 - app/views/sql/refresh_aggregate_table.txt
@@ -394,6 +395,7 @@ files:
 - db/migrate/20190520232222_add_unique_index.rb
 - db/migrate/20190520232223_add_provisioning_fields.rb
 - db/migrate/20190520232224_add_environment_fields.rb
+- lib/metrics/influx/point_value.rb
 - lib/metrics/net.rb
 - lib/middleware/bad_multipart_form_data_sanitizer.rb
 - lib/middleware/json_parse_errors.rb