zuora_connect 3.0.0.pre.m → 3.0.0.pre.n

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: fa8acf1336b53b9d3407afba6a37d962a74434e087974a5810955c20c7e5e2f1
4
- data.tar.gz: e3c8f1a30f247c68e78891244e01fdded4c1c5b76ede6540f1808aed255b987d
3
+ metadata.gz: e800e6da56132f3a8b5e0f52d2953347af0e73db371cef49421128b483cecb93
4
+ data.tar.gz: 24b4315f400c88afcd3916548887208b1a10b19a841617df4007beb0280cdf42
5
5
  SHA512:
6
- metadata.gz: 525866e7f02a4ad82b6d8920ee372f091a6f1216c6ae851149b5e5835f71db237c2d4313039abf0b30279f030f7f6994669a8dc38728fea3c427a1402d40bc44
7
- data.tar.gz: fec0a295b4a3493c9b15320e4cd4578396795f63af4e9ad58fbcd88627b16ba7e7ab72bf7826efc85efe3501c26ba65cab47a88c8bf06a8f8fce7df7417eb7ce
6
+ metadata.gz: d82d6ded09b50ae677194d3219b3c8d7204f8f18b9ede85431269ff3f677afa205a17ddb20f25760dd35e3d87da35ebac471d86e8d861d0fe5585b4671ddbdc3
7
+ data.tar.gz: b899378707679de0076efdfc1b14a73e781f03216f2ac1391c176567034a436b1074e9f71966e9efb59490442fe22621a3da19481b0397410a1e4bad97657477
@@ -12,6 +12,7 @@ module ZuoraConnect
12
12
 
13
13
  self.table_name = "zuora_connect_app_instances"
14
14
  attr_accessor :options, :mode, :logins, :task_data, :last_refresh, :username, :password, :s3_client, :api_version, :drop_message, :new_session_message, :connect_user, :logitems, :user_timezone
15
+ @@telegraf_host = nil
15
16
  REFRESH_TIMEOUT = 2.minute #Used to determine how long to wait on current refresh call before executing another
16
17
  INSTANCE_REFRESH_WINDOW = 1.hours #Used to set how how long till app starts attempting to refresh cached task connect data
17
18
  INSTANCE_REDIS_CACHE_PERIOD = 24.hours #Used to determine how long to cached task data will live for
@@ -362,9 +363,9 @@ module ZuoraConnect
362
363
  raise ZuoraConnect::Exceptions::ConnectCommunicationError.new("JSON parse error", response.body, response.code)
363
364
  end
364
365
 
365
- self.build_task(task_data: parsed_json, session: session)
366
+ self.build_task(task_data: parsed_json, session: session)
366
367
  self.set_backup_creds
367
- self.save(validate: false) if self.changed?
368
+ self.save(validate: false) if self.changed?
368
369
  else
369
370
  raise ZuoraConnect::Exceptions::ConnectCommunicationError.new("Error Communicating with Connect", response.body, response.code)
370
371
  end
@@ -412,8 +413,8 @@ module ZuoraConnect
412
413
  def set_backup_creds
413
414
  if self.kms_key.present? && self.kms_key.match(/^arn:aws:.*/) && self.task_data.present?
414
415
  self.zuora_logins = self.strip_cache_data(object: self.task_data.dup, keys: ['applications', 'tokens', 'user_settings'])
415
- end
416
- end
416
+ end
417
+ end
417
418
 
418
419
  def zuora_logins=(val)
419
420
  write_attribute(:zuora_logins, kms_encrypt(val.to_json))
@@ -478,6 +479,15 @@ module ZuoraConnect
478
479
  end
479
480
  Thread.current[:appinstance] = self
480
481
  end
482
+
483
+ def self.write_to_telegraf(*args)
484
+ if ZuoraConnect.configuration.enable_metrics && !defined?(Prometheus)
485
+ @@telegraf_host = ZuoraConnect::Telegraf.new() if @@telegraf_host == nil
486
+ unicorn_stats = ZuoraObservability::Metrics.unicorn_listener if defined?(Unicorn) && Unicorn.respond_to?(:listener_names)
487
+ @@telegraf_host.write(direction: 'Raindrops', tags: {}, values: unicorn_stats) unless unicorn_stats.blank?
488
+ return @@telegraf_host.write(*args)
489
+ end
490
+ end
481
491
  #### END Task Methods ####
482
492
 
483
493
  #### START Task Methods ####
@@ -600,7 +610,10 @@ module ZuoraConnect
600
610
  self.provisioned_app = nil
601
611
  end
602
612
  end
603
- self.save(:validate => false)
613
+ if self.changed?
614
+ self.save(:validate => false)
615
+ self.refresh
616
+ end
604
617
 
605
618
  return parsed_json
606
619
  end
@@ -0,0 +1,73 @@
1
+ module ZuoraConnect
2
+ class Telegraf
3
+ attr_accessor :host
4
+
5
+ OUTBOUND_METRICS = true
6
+ OUTBOUND_METRICS_NAME = "request-outbound"
7
+ INBOUND_METRICS = true
8
+ INBOUND_METRICS_NAME = "request-inbound"
9
+
10
+ def initialize
11
+ self.connect
12
+ end
13
+
14
+ def connect
15
+ ZuoraConnect.logger.debug(self.format_metric_log('Telegraf','Need new connection')) if ZuoraConnect.configuration.telegraf_debug
16
+ uri = URI.parse(ZuoraConnect.configuration.telegraf_endpoint)
17
+ self.host = UDPSocket.new.tap do |socket|
18
+ socket.connect uri.host, uri.port
19
+ end
20
+ rescue => ex
21
+ self.host = nil
22
+ ZuoraConnect.logger.warn(self.format_metric_log('Telegraf', "Failed to connect: #{ex.class}")) if Rails.env.to_s != 'production'
23
+ end
24
+
25
+ def write(direction: 'Unknown', tags: {}, values: {})
26
+ time = Benchmark.measure do |bench|
27
+ # To avoid writing metrics from rspec tests
28
+ if Rails.env.to_sym != :test
29
+ app_instance = Thread.current[:appinstance].present? ? Thread.current[:appinstance].id : 0
30
+ tags = { app_name: ZuoraObservability::Env.app_name, process_type: ZuoraObservability::Env.process_type, app_instance: app_instance, pod_name: ZuoraObservability::Env.pod_name}.merge(tags)
31
+
32
+ if direction == :inbound
33
+ if INBOUND_METRICS && !Thread.current[:inbound_metric].to_bool
34
+ self.write_udp(series: INBOUND_METRICS_NAME, tags: tags, values: values)
35
+ Thread.current[:inbound_metric] = true
36
+ else
37
+ return
38
+ end
39
+ elsif direction == :outbound
40
+ self.write_udp(series: OUTBOUND_METRICS_NAME, tags: tags, values: values) if OUTBOUND_METRICS
41
+ else
42
+ self.write_udp(series: direction, tags: tags, values: values)
43
+ end
44
+ end
45
+ end
46
+ if ZuoraConnect.configuration.telegraf_debug
47
+ ZuoraConnect.logger.debug(self.format_metric_log('Telegraf', tags.to_s))
48
+ ZuoraConnect.logger.debug(self.format_metric_log('Telegraf', values.to_s))
49
+ ZuoraConnect.logger.debug(self.format_metric_log('Telegraf', "Writing '#{direction.capitalize}': #{time.real.round(5)} ms"))
50
+ end
51
+ end
52
+
53
+
54
+ def write_udp(series: '', tags: {}, values: {})
55
+ return if !values.present?
56
+ self.host.write InfluxDB::PointValue.new({series: series, tags: tags, values: values}).dump
57
+ rescue => ex
58
+ self.connect
59
+ ZuoraConnect.logger.warn(self.format_metric_log('Telegraf',"Failed to write udp: #{ex.class}")) if Rails.env.to_s != 'production'
60
+ end
61
+
62
+ def format_metric_log(message, dump = nil)
63
+ message_color, dump_color = "1;91", "0;1"
64
+ log_entry = " \e[#{message_color}m#{message}\e[0m "
65
+ log_entry << "\e[#{dump_color}m%#{String === dump ? 's' : 'p'}\e[0m" % dump if dump
66
+ if Rails.env == :development
67
+ log_entry
68
+ else
69
+ [message, dump].compact.join(' - ')
70
+ end
71
+ end
72
+ end
73
+ end
@@ -6,7 +6,7 @@ if defined?(Unicorn::WorkerKiller)
6
6
  if defined?(Prometheus)
7
7
  Prometheus::UNICORN_KILLS.set(1)
8
8
  else
9
- ZuoraObservability::Metrics.write_to_telegraf(direction: 'Unicorn-Killer', tags: {app_instance: 0}, values: {kill: 1})
9
+ ZuoraConnect::AppInstance.write_to_telegraf(direction: 'Unicorn-Killer', tags: {app_instance: 0}, values: {kill: 1})
10
10
  end
11
11
  end
12
12
  end
@@ -0,0 +1,79 @@
1
+ module InfluxDB
2
+ # Convert data point to string using Line protocol
3
+ class PointValue
4
+ attr_reader :series, :values, :tags, :timestamp
5
+
6
+ def initialize(data)
7
+ @series = escape data[:series], :measurement
8
+ @values = escape_values data[:values]
9
+ @tags = escape_tags data[:tags]
10
+ @timestamp = data[:timestamp]
11
+ end
12
+
13
+ def dump
14
+ dump = @series.dup
15
+ dump << ",#{@tags}" if @tags
16
+ dump << " #{@values}"
17
+ dump << " #{@timestamp}" if @timestamp
18
+ dump
19
+ end
20
+
21
+ private
22
+
23
+ ESCAPES = {
24
+ measurement: [' '.freeze, ','.freeze],
25
+ tag_key: ['='.freeze, ' '.freeze, ','.freeze],
26
+ tag_value: ['='.freeze, ' '.freeze, ','.freeze],
27
+ field_key: ['='.freeze, ' '.freeze, ','.freeze, '"'.freeze],
28
+ field_value: ["\\".freeze, '"'.freeze],
29
+ }.freeze
30
+
31
+ private_constant :ESCAPES
32
+
33
+ def escape(str, type)
34
+ # rubocop:disable Layout/AlignParameters
35
+ str = str.encode "UTF-8".freeze, "UTF-8".freeze,
36
+ invalid: :replace,
37
+ undef: :replace,
38
+ replace: "".freeze
39
+ # rubocop:enable Layout/AlignParameters
40
+
41
+ ESCAPES[type].each do |ch|
42
+ str = str.gsub(ch) { "\\#{ch}" }
43
+ end
44
+ str
45
+ end
46
+
47
+ def escape_values(values)
48
+ return if values.nil?
49
+ values.map do |k, v|
50
+ key = escape(k.to_s, :field_key)
51
+ val = escape_value(v)
52
+ "#{key}=#{val}"
53
+ end.join(",".freeze)
54
+ end
55
+
56
+ def escape_value(value)
57
+ if value.is_a?(String)
58
+ '"'.freeze + escape(value, :field_value) + '"'.freeze
59
+ elsif value.is_a?(Integer)
60
+ "#{value}i"
61
+ else
62
+ value.to_s
63
+ end
64
+ end
65
+
66
+ def escape_tags(tags)
67
+ return if tags.nil?
68
+
69
+ tags = tags.map do |k, v|
70
+ key = escape(k.to_s, :tag_key)
71
+ val = escape(v.to_s, :tag_value)
72
+
73
+ "#{key}=#{val}" unless key == "".freeze || val == "".freeze
74
+ end.compact
75
+
76
+ tags.join(",") unless tags.empty?
77
+ end
78
+ end
79
+ end
@@ -68,12 +68,12 @@ class HttpLogger
68
68
  log_request_body(request)
69
69
  log_request_headers(request)
70
70
  if defined?(response) && response
71
- tags = tags.merge({status: response.code.to_i})
71
+ tags = tags.merge({status: response.code.to_i})
72
72
  log_response_code(response)
73
73
  log_response_headers(response)
74
74
  log_response_body(response.body)
75
75
  end
76
- ZuoraObservability::Metrics.write_to_telegraf(direction: :outbound, tags: tags, values: values)
76
+ ZuoraConnect::AppInstance.write_to_telegraf(direction: :outbound, tags: tags, values: values)
77
77
  end
78
78
  end
79
79
 
@@ -95,7 +95,7 @@ class HttpLogger
95
95
  end
96
96
 
97
97
  HTTP_METHODS_WITH_BODY = Set.new(%w(POST PUT GET PATCH))
98
-
98
+
99
99
  def log_request_body(request)
100
100
  if self.class.log_request_body
101
101
  if HTTP_METHODS_WITH_BODY.include?(request.method)
@@ -149,8 +149,8 @@ class HttpLogger
149
149
  def truncate_body(body)
150
150
  if collapse_body_limit && collapse_body_limit > 0 && body && body.size >= collapse_body_limit
151
151
  body_piece_size = collapse_body_limit / 2
152
- body[0..body_piece_size] +
153
- "\n\n<some data truncated>\n\n" +
152
+ body[0..body_piece_size] +
153
+ "\n\n<some data truncated>\n\n" +
154
154
  body[(body.size - body_piece_size)..body.size]
155
155
  else
156
156
  body
@@ -203,7 +203,7 @@ class Net::HTTP
203
203
 
204
204
  def request(request, body = nil, &block)
205
205
  HttpLogger.perform(self, request, body) do
206
- request_without_logging(request, body, &block)
206
+ request_without_logging(request, body, &block)
207
207
  end
208
208
  end
209
209
  end
@@ -215,4 +215,4 @@ if defined?(Rails)
215
215
  HttpLogger.logger = ZuoraConnect.logger unless HttpLogger.logger
216
216
  end
217
217
  end
218
- end
218
+ end
@@ -1,6 +1,37 @@
1
1
  module ZuoraConnect
2
2
  require 'uri'
3
3
 
4
+ # Object of this class is passed to the ActiveSupport::Notification hook
5
+ class PageRequest
6
+
7
+ # This method is triggered when a non error page is loaded (not 404)
8
+ def call(name, started, finished, unique_id, payload)
9
+ # If the url contains any css or JavaScript files then do not collect metrics for them
10
+ return nil if ["css", "assets", "jpg", "png", "jpeg", "ico"].any? { |word| payload[:path].include?(word) }
11
+
12
+ # Getting the endpoint and the content_type
13
+ content_hash = {:html => "text/html", :js => "application/javascript", :json => "application/json", :csv => "text/csv"}
14
+ content_type = content_hash.key?(payload[:format]) ? content_hash[payload[:format]] : payload[:format]
15
+ content_type = content_type.to_s.gsub('text/javascript', 'application/javascript')
16
+
17
+ # payloads with 500 requests do not have status as it is not set by the controller
18
+ # https://github.com/rails/rails/issues/33335
19
+ #status_code = payload[:status] ? payload[:status] : payload[:exception_object].present? ? 500 : ""
20
+ if payload[:exception].present?
21
+ status_code, exception = [500, payload[:exception].first]
22
+ else
23
+ status_code, exception = [payload[:status], nil]
24
+ end
25
+
26
+ tags = {method: payload[:method], status: status_code, error_type: exception, content_type: content_type, controller: payload[:controller], action: payload[:action]}.compact
27
+
28
+ values = {view_time: payload[:view_runtime], db_time: payload[:db_runtime], response_time: ((finished-started)*1000)}.compact
29
+ values = values.map{ |k,v| [k,v.round(2)]}.to_h
30
+
31
+ ZuoraConnect::AppInstanceBase.write_to_telegraf(direction: :inbound, tags: tags, values: values)
32
+ end
33
+ end
34
+
4
35
  class MetricsMiddleware
5
36
 
6
37
  require "zuora_connect/version"
@@ -36,9 +67,9 @@ module ZuoraConnect
36
67
  if defined?(Prometheus) && env['PATH_INFO'] == '/connect/internal/metrics'
37
68
 
38
69
  # Prometheus Stuff
39
- metrics = ZuoraObservability::Metrics.resque
70
+ metrics = ZuoraObservability::Metrics.resque
40
71
  metrics = defined?(ZuoraConnect::AppInstance.get_metrics) ? ZuoraConnect::AppInstance.get_metrics(metrics) : metrics
41
-
72
+
42
73
  redis_up = metrics.present? && metrics.dig(:Resque, :Workers_Total).present? ? 1 : 0
43
74
  Prometheus::REDIS_CONNECTION.set(redis_up)
44
75
 
@@ -77,7 +108,7 @@ module ZuoraConnect
77
108
 
78
109
  values = {response_time: ((Time.now - start_time)*1000).round(2) }
79
110
 
80
- ZuoraObservability::Metrics.write_to_telegraf(direction: :inbound, tags: tags, values: values)
111
+ ZuoraConnect::AppInstanceBase.write_to_telegraf(direction: :inbound, tags: tags, values: values)
81
112
  end
82
113
  end
83
114
  Thread.current[:inbound_metric] = nil
@@ -9,6 +9,7 @@ require 'resque/silence_done'
9
9
  require 'resque/self_lookup'
10
10
  require 'resque/plugins/custom_logger'
11
11
  require 'resque/plugins/app_instance_job'
12
+ require 'metrics/influx/point_value'
12
13
  require 'metrics/net'
13
14
  require 'mono_logger'
14
15
  require 'zuora_connect/zuora_audit'
@@ -32,5 +32,8 @@ module ZuoraConnect
32
32
  app.config.middleware.use Rack::Deflater, if: ->(env, *) { env['PATH_INFO'] == '/connect/internal/metrics' }
33
33
  end
34
34
  end
35
+
36
+ # hook to process_action
37
+ ActiveSupport::Notifications.subscribe('process_action.action_controller', ZuoraConnect::PageRequest.new)
35
38
  end
36
39
  end
@@ -1,3 +1,3 @@
1
1
  module ZuoraConnect
2
- VERSION = "3.0.0-m"
2
+ VERSION = "3.0.0-n"
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: zuora_connect
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.0.0.pre.m
4
+ version: 3.0.0.pre.n
5
5
  platform: ruby
6
6
  authors:
7
7
  - Connect Team
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-12-23 00:00:00.000000000 Z
11
+ date: 2021-01-06 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: apartment
@@ -360,6 +360,7 @@ files:
360
360
  - app/models/zuora_connect/app_instance.rb
361
361
  - app/models/zuora_connect/app_instance_base.rb
362
362
  - app/models/zuora_connect/login.rb
363
+ - app/models/zuora_connect/telegraf.rb
363
364
  - app/models/zuora_connect/zuora_user.rb
364
365
  - app/views/layouts/zuora_connect/application.html.erb
365
366
  - app/views/sql/refresh_aggregate_table.txt
@@ -394,6 +395,7 @@ files:
394
395
  - db/migrate/20190520232222_add_unique_index.rb
395
396
  - db/migrate/20190520232223_add_provisioning_fields.rb
396
397
  - db/migrate/20190520232224_add_environment_fields.rb
398
+ - lib/metrics/influx/point_value.rb
397
399
  - lib/metrics/net.rb
398
400
  - lib/middleware/bad_multipart_form_data_sanitizer.rb
399
401
  - lib/middleware/json_parse_errors.rb