zuora_connect 3.0.0.pre.g → 3.0.0.pre.n

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7443cb29882ccc12bc04d9e58f21fe54dbae32e6b4e7ffe12b699b63854cce55
-  data.tar.gz: 9b87ed757abeca777bff58cf469f6e7d918aeacf304ec98bcd6bb02458195754
+  metadata.gz: e800e6da56132f3a8b5e0f52d2953347af0e73db371cef49421128b483cecb93
+  data.tar.gz: 24b4315f400c88afcd3916548887208b1a10b19a841617df4007beb0280cdf42
 SHA512:
-  metadata.gz: 9ddc22c116f93881f4f7c0901ee9e220087e038392238fd4681c899725fcf72836203eafe88a59b2a5f025bc36f3594e743b298a47b5aa1d1ae2ed21ffb1de57
-  data.tar.gz: a9d5f1aaeddd76c772eacfbcdbaa6404a6edeeef8194a7626252f2df55afc7c08b6c962c64be01139c5c4f1f0609a17a4cd101fca16bff2011ab657b23a52cc8
+  metadata.gz: d82d6ded09b50ae677194d3219b3c8d7204f8f18b9ede85431269ff3f677afa205a17ddb20f25760dd35e3d87da35ebac471d86e8d861d0fe5585b4671ddbdc3
+  data.tar.gz: b899378707679de0076efdfc1b14a73e781f03216f2ac1391c176567034a436b1074e9f71966e9efb59490442fe22621a3da19481b0397410a1e4bad97657477

data/Rakefile

@@ -15,7 +15,7 @@ RDoc::Task.new(:rdoc) do |rdoc|
   rdoc.rdoc_files.include('lib/**/*.rb')
 end
 
-APP_RAKEFILE = File.expand_path("../test/dummy/Rakefile", __FILE__)
+APP_RAKEFILE = File.expand_path("../spec/dummy/Rakefile", __FILE__)
 load 'rails/tasks/engine.rake'
 
 

data/app/models/zuora_connect/app_instance_base.rb

@@ -12,6 +12,7 @@ module ZuoraConnect
 
     self.table_name = "zuora_connect_app_instances"
     attr_accessor :options, :mode, :logins, :task_data, :last_refresh, :username, :password, :s3_client, :api_version, :drop_message, :new_session_message, :connect_user, :logitems, :user_timezone
+    @@telegraf_host = nil
     REFRESH_TIMEOUT = 2.minute                #Used to determine how long to wait on current refresh call before executing another
     INSTANCE_REFRESH_WINDOW = 1.hours         #Used to set how how long till app starts attempting to refresh cached task connect data
     INSTANCE_REDIS_CACHE_PERIOD = 24.hours    #Used to determine how long to cached task data will live for
@@ -362,9 +363,9 @@ module ZuoraConnect
               raise ZuoraConnect::Exceptions::ConnectCommunicationError.new("JSON parse error", response.body, response.code)
             end
 
-            self.build_task(task_data: parsed_json, session: session)  
+            self.build_task(task_data: parsed_json, session: session)
             self.set_backup_creds
-            self.save(validate: false) if self.changed?    
+            self.save(validate: false) if self.changed?
           else
             raise ZuoraConnect::Exceptions::ConnectCommunicationError.new("Error Communicating with Connect", response.body, response.code)
           end
@@ -412,8 +413,8 @@ module ZuoraConnect
       def set_backup_creds
         if self.kms_key.present? && self.kms_key.match(/^arn:aws:.*/) && self.task_data.present?
           self.zuora_logins = self.strip_cache_data(object: self.task_data.dup, keys: ['applications', 'tokens', 'user_settings'])
-        end  
-      end 
+        end
+      end
 
       def zuora_logins=(val)
         write_attribute(:zuora_logins, kms_encrypt(val.to_json))
@@ -478,6 +479,15 @@ module ZuoraConnect
         end
         Thread.current[:appinstance] = self
       end
+
+      def self.write_to_telegraf(*args)
+        if ZuoraConnect.configuration.enable_metrics && !defined?(Prometheus)
+          @@telegraf_host = ZuoraConnect::Telegraf.new() if @@telegraf_host == nil
+          unicorn_stats = ZuoraObservability::Metrics.unicorn_listener if defined?(Unicorn) && Unicorn.respond_to?(:listener_names)
+          @@telegraf_host.write(direction: 'Raindrops', tags: {}, values: unicorn_stats)  unless unicorn_stats.blank?
+          return @@telegraf_host.write(*args)
+        end
+      end
     #### END Task Methods ####
 
     #### START Task Methods ####
@@ -600,7 +610,10 @@ module ZuoraConnect
               self.provisioned_app = nil
             end
           end
-          self.save(:validate => false)
+          if self.changed? 
+            self.save(:validate => false) 
+            self.refresh
+          end
 
           return parsed_json
         end
@@ -1029,7 +1042,7 @@ module ZuoraConnect
             catalog_container = catalog_container[object_id]
           end
 
-          return catalog_container
+          return catalog_container || {}
         end
 
         if defined?(Redis.current) && object_id.present? && object_id.class == String && object_id.present?

data/app/models/zuora_connect/telegraf.rb

@@ -0,0 +1,73 @@
+module ZuoraConnect
+  class Telegraf
+    attr_accessor :host
+
+    OUTBOUND_METRICS = true
+    OUTBOUND_METRICS_NAME = "request-outbound"
+    INBOUND_METRICS = true
+    INBOUND_METRICS_NAME = "request-inbound"
+
+    def initialize
+      self.connect
+    end
+
+    def connect
+      ZuoraConnect.logger.debug(self.format_metric_log('Telegraf','Need new connection')) if ZuoraConnect.configuration.telegraf_debug
+      uri = URI.parse(ZuoraConnect.configuration.telegraf_endpoint)
+      self.host = UDPSocket.new.tap do |socket|
+        socket.connect uri.host, uri.port
+      end
+    rescue => ex
+      self.host = nil
+      ZuoraConnect.logger.warn(self.format_metric_log('Telegraf', "Failed to connect: #{ex.class}")) if Rails.env.to_s != 'production'
+    end
+
+    def write(direction: 'Unknown', tags: {}, values: {})
+      time = Benchmark.measure do |bench|
+        # To avoid writing metrics from rspec tests
+        if Rails.env.to_sym != :test
+          app_instance = Thread.current[:appinstance].present? ? Thread.current[:appinstance].id : 0
+          tags = { app_name: ZuoraObservability::Env.app_name, process_type: ZuoraObservability::Env.process_type, app_instance: app_instance, pod_name: ZuoraObservability::Env.pod_name}.merge(tags)
+
+          if direction == :inbound
+            if INBOUND_METRICS && !Thread.current[:inbound_metric].to_bool
+              self.write_udp(series: INBOUND_METRICS_NAME, tags: tags, values: values)
+              Thread.current[:inbound_metric] = true
+            else
+              return
+            end
+          elsif direction == :outbound
+            self.write_udp(series: OUTBOUND_METRICS_NAME, tags: tags, values: values) if OUTBOUND_METRICS
+          else
+            self.write_udp(series: direction, tags: tags, values: values)
+          end
+        end
+      end
+      if ZuoraConnect.configuration.telegraf_debug
+        ZuoraConnect.logger.debug(self.format_metric_log('Telegraf', tags.to_s))
+        ZuoraConnect.logger.debug(self.format_metric_log('Telegraf', values.to_s))
+        ZuoraConnect.logger.debug(self.format_metric_log('Telegraf', "Writing '#{direction.capitalize}': #{time.real.round(5)} ms"))
+      end
+    end
+
+
+    def write_udp(series: '', tags: {}, values: {})
+      return if !values.present?
+      self.host.write InfluxDB::PointValue.new({series: series, tags: tags, values: values}).dump
+    rescue => ex
+      self.connect
+      ZuoraConnect.logger.warn(self.format_metric_log('Telegraf',"Failed to write udp: #{ex.class}")) if Rails.env.to_s != 'production'
+    end
+
+    def format_metric_log(message, dump = nil)
+      message_color, dump_color = "1;91", "0;1"
+      log_entry = "  \e[#{message_color}m#{message}\e[0m   "
+      log_entry << "\e[#{dump_color}m%#{String === dump ? 's' : 'p'}\e[0m" % dump if dump
+      if Rails.env == :development
+        log_entry
+      else
+        [message, dump].compact.join(' - ')
+      end
+    end
+  end
+end

data/config/initializers/postgresql_adapter.rb

@@ -26,7 +26,7 @@ module ActiveRecord
           if oids
             query += "WHERE t.oid::integer IN (%s)" % oids.join(", ")
           else
-            query += initializer.query_conditions_for_initial_load(type_map)
+            query += initializer.query_conditions_for_initial_load
           end
 
           execute_and_clear(query, "SCHEMA", []) do |records|

data/config/initializers/prometheus.rb

@@ -15,29 +15,6 @@ if defined? Prometheus
       dir: prometheus_path
     )
 
-    class ResqueExporter
-      require 'prometheus/client/formats/text'
-      require 'fileutils'
-
-      def initialize
-        @lock = Monitor.new
-        @registry = Prometheus::Client.registry
-        @path = ENV['RESQUE_EXPORTER_PATH'] || Rails.root.join('tmp/resque_exporter')
-        FileUtils.mkdir_p(@path)
-      end
-
-      def export
-        filename = File.join(@path, 'resque_export.prom')
-        @lock.synchronize do
-          File.open(filename, 'w+') do |file|
-            file.write(Prometheus::Client::Formats::Text.marshal(@registry))
-          end
-        end
-      rescue
-        # Ignored
-      end
-    end
-
     most_recent_aggregation = {}
     sum_aggregation = {}
     if defined?(Unicorn)

data/config/initializers/resque.rb

@@ -5,20 +5,6 @@ if defined?(Resque::Worker)
   Resque::Job.send(:include, Resque::SelfLookup)
 end
 
-if defined?(Resque::Job) && defined?(Prometheus)
-  module ResquePrometheusExtensions
-    EXPORTER = Prometheus::ResqueExporter.new
-    def perform
-      super
-      EXPORTER.export
-    end
-  end
-
-  class Resque::Job
-    prepend ResquePrometheusExtensions
-  end
-end
-
 Resque.module_eval do
   # Returns a hash, mapping queue names to queue sizes
   def queue_sizes

data/config/initializers/unicorn.rb

@@ -6,7 +6,7 @@ if defined?(Unicorn::WorkerKiller)
       if defined?(Prometheus)
         Prometheus::UNICORN_KILLS.set(1)
       else
-        ZuoraObservability::Metrics.write_to_telegraf(direction: 'Unicorn-Killer', tags: {app_instance: 0}, values: {kill: 1})
+        ZuoraConnect::AppInstance.write_to_telegraf(direction: 'Unicorn-Killer', tags: {app_instance: 0}, values: {kill: 1})
       end
     end
   end

data/lib/metrics/influx/point_value.rb

@@ -0,0 +1,79 @@
+module InfluxDB
+  # Convert data point to string using Line protocol
+  class PointValue
+    attr_reader :series, :values, :tags, :timestamp
+
+    def initialize(data)
+      @series    = escape data[:series], :measurement
+      @values    = escape_values data[:values]
+      @tags      = escape_tags data[:tags]
+      @timestamp = data[:timestamp]
+    end
+
+    def dump
+      dump =  @series.dup
+      dump << ",#{@tags}" if @tags
+      dump << " #{@values}"
+      dump << " #{@timestamp}" if @timestamp
+      dump
+    end
+
+    private
+
+    ESCAPES = {
+      measurement:  [' '.freeze, ','.freeze],
+      tag_key:      ['='.freeze, ' '.freeze, ','.freeze],
+      tag_value:    ['='.freeze, ' '.freeze, ','.freeze],
+      field_key:    ['='.freeze, ' '.freeze, ','.freeze, '"'.freeze],
+      field_value:  ["\\".freeze, '"'.freeze],
+    }.freeze
+
+    private_constant :ESCAPES
+
+    def escape(str, type)
+      # rubocop:disable Layout/AlignParameters
+      str = str.encode "UTF-8".freeze, "UTF-8".freeze,
+        invalid: :replace,
+        undef: :replace,
+        replace: "".freeze
+      # rubocop:enable Layout/AlignParameters
+
+      ESCAPES[type].each do |ch|
+        str = str.gsub(ch) { "\\#{ch}" }
+      end
+      str
+    end
+
+    def escape_values(values)
+      return if values.nil?
+      values.map do |k, v|
+        key = escape(k.to_s, :field_key)
+        val = escape_value(v)
+        "#{key}=#{val}"
+      end.join(",".freeze)
+    end
+
+    def escape_value(value)
+      if value.is_a?(String)
+        '"'.freeze + escape(value, :field_value) + '"'.freeze
+      elsif value.is_a?(Integer)
+        "#{value}i"
+      else
+        value.to_s
+      end
+    end
+
+    def escape_tags(tags)
+      return if tags.nil?
+
+      tags = tags.map do |k, v|
+        key = escape(k.to_s, :tag_key)
+        val = escape(v.to_s, :tag_value)
+
+        "#{key}=#{val}" unless key == "".freeze || val == "".freeze
+      end.compact
+
+      tags.join(",") unless tags.empty?
+    end
+  end
+end

data/lib/metrics/net.rb

@@ -68,12 +68,12 @@ class HttpLogger
       log_request_body(request)
       log_request_headers(request)
       if defined?(response) && response
-        tags = tags.merge({status: response.code.to_i})
+        tags = tags.merge({status: response.code.to_i}) 
         log_response_code(response)
         log_response_headers(response)
         log_response_body(response.body)
       end
-      ZuoraObservability::Metrics.write_to_telegraf(direction: :outbound, tags: tags, values: values)
+      ZuoraConnect::AppInstance.write_to_telegraf(direction: :outbound, tags: tags, values: values)
     end
   end
 
@@ -95,7 +95,7 @@ class HttpLogger
   end
 
   HTTP_METHODS_WITH_BODY = Set.new(%w(POST PUT GET PATCH))
-
+  
   def log_request_body(request)
     if self.class.log_request_body
       if HTTP_METHODS_WITH_BODY.include?(request.method)
@@ -149,8 +149,8 @@ class HttpLogger
   def truncate_body(body)
     if collapse_body_limit && collapse_body_limit > 0 && body && body.size >= collapse_body_limit
       body_piece_size = collapse_body_limit / 2
-      body[0..body_piece_size] +
-        "\n\n<some data truncated>\n\n" +
+      body[0..body_piece_size] + 
+        "\n\n<some data truncated>\n\n" + 
         body[(body.size - body_piece_size)..body.size]
     else
       body
@@ -203,7 +203,7 @@ class Net::HTTP
 
   def request(request, body = nil, &block)
     HttpLogger.perform(self, request, body) do
-      request_without_logging(request, body, &block)
+      request_without_logging(request, body, &block) 
     end
   end
 end
@@ -215,4 +215,4 @@ if defined?(Rails)
       HttpLogger.logger = ZuoraConnect.logger unless HttpLogger.logger
     end
   end
-end
+end

data/lib/middleware/metrics_middleware.rb

@@ -1,6 +1,37 @@
 module ZuoraConnect
   require 'uri'
 
+  # Object of this class is passed to the ActiveSupport::Notification hook
+  class PageRequest
+
+    # This method is triggered when a non error page is loaded (not 404)
+    def call(name, started, finished, unique_id, payload)
+      # If the url contains any css or JavaScript files then do not collect metrics for them
+      return nil if ["css", "assets", "jpg", "png", "jpeg", "ico"].any? { |word| payload[:path].include?(word) }
+
+      # Getting the endpoint and the content_type
+      content_hash = {:html => "text/html", :js => "application/javascript", :json => "application/json", :csv => "text/csv"}
+      content_type = content_hash.key?(payload[:format]) ? content_hash[payload[:format]] : payload[:format]
+      content_type = content_type.to_s.gsub('text/javascript', 'application/javascript')
+
+      # payloads with 500 requests do not have status as it is not set by the controller
+      # https://github.com/rails/rails/issues/33335
+      #status_code = payload[:status] ? payload[:status] : payload[:exception_object].present? ? 500 : ""
+      if payload[:exception].present?
+        status_code, exception = [500, payload[:exception].first]
+      else
+        status_code, exception = [payload[:status], nil]
+      end
+
+      tags = {method: payload[:method], status: status_code, error_type: exception, content_type: content_type, controller: payload[:controller], action: payload[:action]}.compact
+
+      values = {view_time: payload[:view_runtime], db_time: payload[:db_runtime], response_time: ((finished-started)*1000)}.compact
+      values = values.map{ |k,v| [k,v.round(2)]}.to_h
+
+      ZuoraConnect::AppInstanceBase.write_to_telegraf(direction: :inbound, tags: tags, values: values)
+    end
+  end
+
   class MetricsMiddleware
 
     require "zuora_connect/version"
@@ -34,8 +65,11 @@ module ZuoraConnect
         @bad_headers.each { |header| env.delete(header) }
 
         if defined?(Prometheus) && env['PATH_INFO'] == '/connect/internal/metrics'
+
           # Prometheus Stuff
           metrics = ZuoraObservability::Metrics.resque
+          metrics = defined?(ZuoraConnect::AppInstance.get_metrics) ? ZuoraConnect::AppInstance.get_metrics(metrics) : metrics
+
           redis_up = metrics.present? && metrics.dig(:Resque, :Workers_Total).present? ? 1 : 0
           Prometheus::REDIS_CONNECTION.set(redis_up)
 
@@ -74,7 +108,7 @@ module ZuoraConnect
 
               values = {response_time: ((Time.now - start_time)*1000).round(2) }
 
-              ZuoraObservability::Metrics.write_to_telegraf(direction: :inbound, tags: tags, values: values)
+              ZuoraConnect::AppInstanceBase.write_to_telegraf(direction: :inbound, tags: tags, values: values)
             end
           end
           Thread.current[:inbound_metric] = nil

data/lib/resque/plugins/app_instance_job.rb

@@ -17,9 +17,9 @@ module Resque
         end
 
         begin
-          connection_count ||= 0 
+          connection_count ||= 0
           @appinstance = ZuoraConnect::AppInstance.find(args['app_instance_id'].to_i)
-          Rails.logger.info('Starting job') 
+          job_start_log(args)
 
           @appinstance.new_session(holding_pattern: true)
         rescue ActiveRecord::RecordNotFound => exception
@@ -52,12 +52,12 @@ module Resque
           return
         rescue ZuoraConnect::Exceptions::ConnectCommunicationError => exception
           Rails.logger.warn("Enqueue Job Again ~ 2 mins", exception)
-          @appinstance.queue_pause(time: 2.minutes.to_i) 
+          @appinstance.queue_pause(time: 2.minutes.to_i)
           Resque.enqueue_to(self.job.queue, self.job.payload['class'], args)
           return
         rescue Net::ReadTimeout, Net::OpenTimeout, Errno::ECONNRESET, Errno::ECONNREFUSED, SocketError => exception
           Rails.logger.warn("Enqueue Job Again ~ 2 mins", exception)
-          @appinstance.queue_pause(time: 2.minutes.to_i) 
+          @appinstance.queue_pause(time: 2.minutes.to_i)
           Resque.enqueue_to(self.job.queue, self.job.payload['class'], args)
           return
         end
@@ -68,6 +68,10 @@ module Resque
         @appinstance.cache_app_instance if defined?(@appinstance)
         Rails.logger.flush if Rails.logger.methods.include?(:flush)
       end
+
+      def job_start_log(args)
+       Rails.logger.info("Starting job")
+      end
     end
   end
-end
+end