zuora_connect 3.0.0.pre.m → 3.0.0.pre.n
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/app/models/zuora_connect/app_instance_base.rb +18 -5
- data/app/models/zuora_connect/telegraf.rb +73 -0
- data/config/initializers/unicorn.rb +1 -1
- data/lib/metrics/influx/point_value.rb +79 -0
- data/lib/metrics/net.rb +7 -7
- data/lib/middleware/metrics_middleware.rb +34 -3
- data/lib/zuora_connect.rb +1 -0
- data/lib/zuora_connect/railtie.rb +3 -0
- data/lib/zuora_connect/version.rb +1 -1
- metadata +4 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e800e6da56132f3a8b5e0f52d2953347af0e73db371cef49421128b483cecb93
|
4
|
+
data.tar.gz: 24b4315f400c88afcd3916548887208b1a10b19a841617df4007beb0280cdf42
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d82d6ded09b50ae677194d3219b3c8d7204f8f18b9ede85431269ff3f677afa205a17ddb20f25760dd35e3d87da35ebac471d86e8d861d0fe5585b4671ddbdc3
|
7
|
+
data.tar.gz: b899378707679de0076efdfc1b14a73e781f03216f2ac1391c176567034a436b1074e9f71966e9efb59490442fe22621a3da19481b0397410a1e4bad97657477
|
@@ -12,6 +12,7 @@ module ZuoraConnect
|
|
12
12
|
|
13
13
|
self.table_name = "zuora_connect_app_instances"
|
14
14
|
attr_accessor :options, :mode, :logins, :task_data, :last_refresh, :username, :password, :s3_client, :api_version, :drop_message, :new_session_message, :connect_user, :logitems, :user_timezone
|
15
|
+
@@telegraf_host = nil
|
15
16
|
REFRESH_TIMEOUT = 2.minute #Used to determine how long to wait on current refresh call before executing another
|
16
17
|
INSTANCE_REFRESH_WINDOW = 1.hours #Used to set how how long till app starts attempting to refresh cached task connect data
|
17
18
|
INSTANCE_REDIS_CACHE_PERIOD = 24.hours #Used to determine how long to cached task data will live for
|
@@ -362,9 +363,9 @@ module ZuoraConnect
|
|
362
363
|
raise ZuoraConnect::Exceptions::ConnectCommunicationError.new("JSON parse error", response.body, response.code)
|
363
364
|
end
|
364
365
|
|
365
|
-
self.build_task(task_data: parsed_json, session: session)
|
366
|
+
self.build_task(task_data: parsed_json, session: session)
|
366
367
|
self.set_backup_creds
|
367
|
-
self.save(validate: false) if self.changed?
|
368
|
+
self.save(validate: false) if self.changed?
|
368
369
|
else
|
369
370
|
raise ZuoraConnect::Exceptions::ConnectCommunicationError.new("Error Communicating with Connect", response.body, response.code)
|
370
371
|
end
|
@@ -412,8 +413,8 @@ module ZuoraConnect
|
|
412
413
|
def set_backup_creds
|
413
414
|
if self.kms_key.present? && self.kms_key.match(/^arn:aws:.*/) && self.task_data.present?
|
414
415
|
self.zuora_logins = self.strip_cache_data(object: self.task_data.dup, keys: ['applications', 'tokens', 'user_settings'])
|
415
|
-
end
|
416
|
-
end
|
416
|
+
end
|
417
|
+
end
|
417
418
|
|
418
419
|
def zuora_logins=(val)
|
419
420
|
write_attribute(:zuora_logins, kms_encrypt(val.to_json))
|
@@ -478,6 +479,15 @@ module ZuoraConnect
|
|
478
479
|
end
|
479
480
|
Thread.current[:appinstance] = self
|
480
481
|
end
|
482
|
+
|
483
|
+
def self.write_to_telegraf(*args)
|
484
|
+
if ZuoraConnect.configuration.enable_metrics && !defined?(Prometheus)
|
485
|
+
@@telegraf_host = ZuoraConnect::Telegraf.new() if @@telegraf_host == nil
|
486
|
+
unicorn_stats = ZuoraObservability::Metrics.unicorn_listener if defined?(Unicorn) && Unicorn.respond_to?(:listener_names)
|
487
|
+
@@telegraf_host.write(direction: 'Raindrops', tags: {}, values: unicorn_stats) unless unicorn_stats.blank?
|
488
|
+
return @@telegraf_host.write(*args)
|
489
|
+
end
|
490
|
+
end
|
481
491
|
#### END Task Methods ####
|
482
492
|
|
483
493
|
#### START Task Methods ####
|
@@ -600,7 +610,10 @@ module ZuoraConnect
|
|
600
610
|
self.provisioned_app = nil
|
601
611
|
end
|
602
612
|
end
|
603
|
-
self.
|
613
|
+
if self.changed?
|
614
|
+
self.save(:validate => false)
|
615
|
+
self.refresh
|
616
|
+
end
|
604
617
|
|
605
618
|
return parsed_json
|
606
619
|
end
|
@@ -0,0 +1,73 @@
|
|
1
|
+
module ZuoraConnect
|
2
|
+
class Telegraf
|
3
|
+
attr_accessor :host
|
4
|
+
|
5
|
+
OUTBOUND_METRICS = true
|
6
|
+
OUTBOUND_METRICS_NAME = "request-outbound"
|
7
|
+
INBOUND_METRICS = true
|
8
|
+
INBOUND_METRICS_NAME = "request-inbound"
|
9
|
+
|
10
|
+
def initialize
|
11
|
+
self.connect
|
12
|
+
end
|
13
|
+
|
14
|
+
def connect
|
15
|
+
ZuoraConnect.logger.debug(self.format_metric_log('Telegraf','Need new connection')) if ZuoraConnect.configuration.telegraf_debug
|
16
|
+
uri = URI.parse(ZuoraConnect.configuration.telegraf_endpoint)
|
17
|
+
self.host = UDPSocket.new.tap do |socket|
|
18
|
+
socket.connect uri.host, uri.port
|
19
|
+
end
|
20
|
+
rescue => ex
|
21
|
+
self.host = nil
|
22
|
+
ZuoraConnect.logger.warn(self.format_metric_log('Telegraf', "Failed to connect: #{ex.class}")) if Rails.env.to_s != 'production'
|
23
|
+
end
|
24
|
+
|
25
|
+
def write(direction: 'Unknown', tags: {}, values: {})
|
26
|
+
time = Benchmark.measure do |bench|
|
27
|
+
# To avoid writing metrics from rspec tests
|
28
|
+
if Rails.env.to_sym != :test
|
29
|
+
app_instance = Thread.current[:appinstance].present? ? Thread.current[:appinstance].id : 0
|
30
|
+
tags = { app_name: ZuoraObservability::Env.app_name, process_type: ZuoraObservability::Env.process_type, app_instance: app_instance, pod_name: ZuoraObservability::Env.pod_name}.merge(tags)
|
31
|
+
|
32
|
+
if direction == :inbound
|
33
|
+
if INBOUND_METRICS && !Thread.current[:inbound_metric].to_bool
|
34
|
+
self.write_udp(series: INBOUND_METRICS_NAME, tags: tags, values: values)
|
35
|
+
Thread.current[:inbound_metric] = true
|
36
|
+
else
|
37
|
+
return
|
38
|
+
end
|
39
|
+
elsif direction == :outbound
|
40
|
+
self.write_udp(series: OUTBOUND_METRICS_NAME, tags: tags, values: values) if OUTBOUND_METRICS
|
41
|
+
else
|
42
|
+
self.write_udp(series: direction, tags: tags, values: values)
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
if ZuoraConnect.configuration.telegraf_debug
|
47
|
+
ZuoraConnect.logger.debug(self.format_metric_log('Telegraf', tags.to_s))
|
48
|
+
ZuoraConnect.logger.debug(self.format_metric_log('Telegraf', values.to_s))
|
49
|
+
ZuoraConnect.logger.debug(self.format_metric_log('Telegraf', "Writing '#{direction.capitalize}': #{time.real.round(5)} ms"))
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
|
54
|
+
def write_udp(series: '', tags: {}, values: {})
|
55
|
+
return if !values.present?
|
56
|
+
self.host.write InfluxDB::PointValue.new({series: series, tags: tags, values: values}).dump
|
57
|
+
rescue => ex
|
58
|
+
self.connect
|
59
|
+
ZuoraConnect.logger.warn(self.format_metric_log('Telegraf',"Failed to write udp: #{ex.class}")) if Rails.env.to_s != 'production'
|
60
|
+
end
|
61
|
+
|
62
|
+
def format_metric_log(message, dump = nil)
|
63
|
+
message_color, dump_color = "1;91", "0;1"
|
64
|
+
log_entry = " \e[#{message_color}m#{message}\e[0m "
|
65
|
+
log_entry << "\e[#{dump_color}m%#{String === dump ? 's' : 'p'}\e[0m" % dump if dump
|
66
|
+
if Rails.env == :development
|
67
|
+
log_entry
|
68
|
+
else
|
69
|
+
[message, dump].compact.join(' - ')
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
@@ -6,7 +6,7 @@ if defined?(Unicorn::WorkerKiller)
|
|
6
6
|
if defined?(Prometheus)
|
7
7
|
Prometheus::UNICORN_KILLS.set(1)
|
8
8
|
else
|
9
|
-
|
9
|
+
ZuoraConnect::AppInstance.write_to_telegraf(direction: 'Unicorn-Killer', tags: {app_instance: 0}, values: {kill: 1})
|
10
10
|
end
|
11
11
|
end
|
12
12
|
end
|
@@ -0,0 +1,79 @@
|
|
1
|
+
module InfluxDB
|
2
|
+
# Convert data point to string using Line protocol
|
3
|
+
class PointValue
|
4
|
+
attr_reader :series, :values, :tags, :timestamp
|
5
|
+
|
6
|
+
def initialize(data)
|
7
|
+
@series = escape data[:series], :measurement
|
8
|
+
@values = escape_values data[:values]
|
9
|
+
@tags = escape_tags data[:tags]
|
10
|
+
@timestamp = data[:timestamp]
|
11
|
+
end
|
12
|
+
|
13
|
+
def dump
|
14
|
+
dump = @series.dup
|
15
|
+
dump << ",#{@tags}" if @tags
|
16
|
+
dump << " #{@values}"
|
17
|
+
dump << " #{@timestamp}" if @timestamp
|
18
|
+
dump
|
19
|
+
end
|
20
|
+
|
21
|
+
private
|
22
|
+
|
23
|
+
ESCAPES = {
|
24
|
+
measurement: [' '.freeze, ','.freeze],
|
25
|
+
tag_key: ['='.freeze, ' '.freeze, ','.freeze],
|
26
|
+
tag_value: ['='.freeze, ' '.freeze, ','.freeze],
|
27
|
+
field_key: ['='.freeze, ' '.freeze, ','.freeze, '"'.freeze],
|
28
|
+
field_value: ["\\".freeze, '"'.freeze],
|
29
|
+
}.freeze
|
30
|
+
|
31
|
+
private_constant :ESCAPES
|
32
|
+
|
33
|
+
def escape(str, type)
|
34
|
+
# rubocop:disable Layout/AlignParameters
|
35
|
+
str = str.encode "UTF-8".freeze, "UTF-8".freeze,
|
36
|
+
invalid: :replace,
|
37
|
+
undef: :replace,
|
38
|
+
replace: "".freeze
|
39
|
+
# rubocop:enable Layout/AlignParameters
|
40
|
+
|
41
|
+
ESCAPES[type].each do |ch|
|
42
|
+
str = str.gsub(ch) { "\\#{ch}" }
|
43
|
+
end
|
44
|
+
str
|
45
|
+
end
|
46
|
+
|
47
|
+
def escape_values(values)
|
48
|
+
return if values.nil?
|
49
|
+
values.map do |k, v|
|
50
|
+
key = escape(k.to_s, :field_key)
|
51
|
+
val = escape_value(v)
|
52
|
+
"#{key}=#{val}"
|
53
|
+
end.join(",".freeze)
|
54
|
+
end
|
55
|
+
|
56
|
+
def escape_value(value)
|
57
|
+
if value.is_a?(String)
|
58
|
+
'"'.freeze + escape(value, :field_value) + '"'.freeze
|
59
|
+
elsif value.is_a?(Integer)
|
60
|
+
"#{value}i"
|
61
|
+
else
|
62
|
+
value.to_s
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
def escape_tags(tags)
|
67
|
+
return if tags.nil?
|
68
|
+
|
69
|
+
tags = tags.map do |k, v|
|
70
|
+
key = escape(k.to_s, :tag_key)
|
71
|
+
val = escape(v.to_s, :tag_value)
|
72
|
+
|
73
|
+
"#{key}=#{val}" unless key == "".freeze || val == "".freeze
|
74
|
+
end.compact
|
75
|
+
|
76
|
+
tags.join(",") unless tags.empty?
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
data/lib/metrics/net.rb
CHANGED
@@ -68,12 +68,12 @@ class HttpLogger
|
|
68
68
|
log_request_body(request)
|
69
69
|
log_request_headers(request)
|
70
70
|
if defined?(response) && response
|
71
|
-
tags = tags.merge({status: response.code.to_i})
|
71
|
+
tags = tags.merge({status: response.code.to_i})
|
72
72
|
log_response_code(response)
|
73
73
|
log_response_headers(response)
|
74
74
|
log_response_body(response.body)
|
75
75
|
end
|
76
|
-
|
76
|
+
ZuoraConnect::AppInstance.write_to_telegraf(direction: :outbound, tags: tags, values: values)
|
77
77
|
end
|
78
78
|
end
|
79
79
|
|
@@ -95,7 +95,7 @@ class HttpLogger
|
|
95
95
|
end
|
96
96
|
|
97
97
|
HTTP_METHODS_WITH_BODY = Set.new(%w(POST PUT GET PATCH))
|
98
|
-
|
98
|
+
|
99
99
|
def log_request_body(request)
|
100
100
|
if self.class.log_request_body
|
101
101
|
if HTTP_METHODS_WITH_BODY.include?(request.method)
|
@@ -149,8 +149,8 @@ class HttpLogger
|
|
149
149
|
def truncate_body(body)
|
150
150
|
if collapse_body_limit && collapse_body_limit > 0 && body && body.size >= collapse_body_limit
|
151
151
|
body_piece_size = collapse_body_limit / 2
|
152
|
-
body[0..body_piece_size] +
|
153
|
-
"\n\n<some data truncated>\n\n" +
|
152
|
+
body[0..body_piece_size] +
|
153
|
+
"\n\n<some data truncated>\n\n" +
|
154
154
|
body[(body.size - body_piece_size)..body.size]
|
155
155
|
else
|
156
156
|
body
|
@@ -203,7 +203,7 @@ class Net::HTTP
|
|
203
203
|
|
204
204
|
def request(request, body = nil, &block)
|
205
205
|
HttpLogger.perform(self, request, body) do
|
206
|
-
request_without_logging(request, body, &block)
|
206
|
+
request_without_logging(request, body, &block)
|
207
207
|
end
|
208
208
|
end
|
209
209
|
end
|
@@ -215,4 +215,4 @@ if defined?(Rails)
|
|
215
215
|
HttpLogger.logger = ZuoraConnect.logger unless HttpLogger.logger
|
216
216
|
end
|
217
217
|
end
|
218
|
-
end
|
218
|
+
end
|
@@ -1,6 +1,37 @@
|
|
1
1
|
module ZuoraConnect
|
2
2
|
require 'uri'
|
3
3
|
|
4
|
+
# Object of this class is passed to the ActiveSupport::Notification hook
|
5
|
+
class PageRequest
|
6
|
+
|
7
|
+
# This method is triggered when a non error page is loaded (not 404)
|
8
|
+
def call(name, started, finished, unique_id, payload)
|
9
|
+
# If the url contains any css or JavaScript files then do not collect metrics for them
|
10
|
+
return nil if ["css", "assets", "jpg", "png", "jpeg", "ico"].any? { |word| payload[:path].include?(word) }
|
11
|
+
|
12
|
+
# Getting the endpoint and the content_type
|
13
|
+
content_hash = {:html => "text/html", :js => "application/javascript", :json => "application/json", :csv => "text/csv"}
|
14
|
+
content_type = content_hash.key?(payload[:format]) ? content_hash[payload[:format]] : payload[:format]
|
15
|
+
content_type = content_type.to_s.gsub('text/javascript', 'application/javascript')
|
16
|
+
|
17
|
+
# payloads with 500 requests do not have status as it is not set by the controller
|
18
|
+
# https://github.com/rails/rails/issues/33335
|
19
|
+
#status_code = payload[:status] ? payload[:status] : payload[:exception_object].present? ? 500 : ""
|
20
|
+
if payload[:exception].present?
|
21
|
+
status_code, exception = [500, payload[:exception].first]
|
22
|
+
else
|
23
|
+
status_code, exception = [payload[:status], nil]
|
24
|
+
end
|
25
|
+
|
26
|
+
tags = {method: payload[:method], status: status_code, error_type: exception, content_type: content_type, controller: payload[:controller], action: payload[:action]}.compact
|
27
|
+
|
28
|
+
values = {view_time: payload[:view_runtime], db_time: payload[:db_runtime], response_time: ((finished-started)*1000)}.compact
|
29
|
+
values = values.map{ |k,v| [k,v.round(2)]}.to_h
|
30
|
+
|
31
|
+
ZuoraConnect::AppInstanceBase.write_to_telegraf(direction: :inbound, tags: tags, values: values)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
4
35
|
class MetricsMiddleware
|
5
36
|
|
6
37
|
require "zuora_connect/version"
|
@@ -36,9 +67,9 @@ module ZuoraConnect
|
|
36
67
|
if defined?(Prometheus) && env['PATH_INFO'] == '/connect/internal/metrics'
|
37
68
|
|
38
69
|
# Prometheus Stuff
|
39
|
-
metrics = ZuoraObservability::Metrics.resque
|
70
|
+
metrics = ZuoraObservability::Metrics.resque
|
40
71
|
metrics = defined?(ZuoraConnect::AppInstance.get_metrics) ? ZuoraConnect::AppInstance.get_metrics(metrics) : metrics
|
41
|
-
|
72
|
+
|
42
73
|
redis_up = metrics.present? && metrics.dig(:Resque, :Workers_Total).present? ? 1 : 0
|
43
74
|
Prometheus::REDIS_CONNECTION.set(redis_up)
|
44
75
|
|
@@ -77,7 +108,7 @@ module ZuoraConnect
|
|
77
108
|
|
78
109
|
values = {response_time: ((Time.now - start_time)*1000).round(2) }
|
79
110
|
|
80
|
-
|
111
|
+
ZuoraConnect::AppInstanceBase.write_to_telegraf(direction: :inbound, tags: tags, values: values)
|
81
112
|
end
|
82
113
|
end
|
83
114
|
Thread.current[:inbound_metric] = nil
|
data/lib/zuora_connect.rb
CHANGED
@@ -9,6 +9,7 @@ require 'resque/silence_done'
|
|
9
9
|
require 'resque/self_lookup'
|
10
10
|
require 'resque/plugins/custom_logger'
|
11
11
|
require 'resque/plugins/app_instance_job'
|
12
|
+
require 'metrics/influx/point_value'
|
12
13
|
require 'metrics/net'
|
13
14
|
require 'mono_logger'
|
14
15
|
require 'zuora_connect/zuora_audit'
|
@@ -32,5 +32,8 @@ module ZuoraConnect
|
|
32
32
|
app.config.middleware.use Rack::Deflater, if: ->(env, *) { env['PATH_INFO'] == '/connect/internal/metrics' }
|
33
33
|
end
|
34
34
|
end
|
35
|
+
|
36
|
+
# hook to process_action
|
37
|
+
ActiveSupport::Notifications.subscribe('process_action.action_controller', ZuoraConnect::PageRequest.new)
|
35
38
|
end
|
36
39
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: zuora_connect
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.0.0.pre.
|
4
|
+
version: 3.0.0.pre.n
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Connect Team
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-01-06 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: apartment
|
@@ -360,6 +360,7 @@ files:
|
|
360
360
|
- app/models/zuora_connect/app_instance.rb
|
361
361
|
- app/models/zuora_connect/app_instance_base.rb
|
362
362
|
- app/models/zuora_connect/login.rb
|
363
|
+
- app/models/zuora_connect/telegraf.rb
|
363
364
|
- app/models/zuora_connect/zuora_user.rb
|
364
365
|
- app/views/layouts/zuora_connect/application.html.erb
|
365
366
|
- app/views/sql/refresh_aggregate_table.txt
|
@@ -394,6 +395,7 @@ files:
|
|
394
395
|
- db/migrate/20190520232222_add_unique_index.rb
|
395
396
|
- db/migrate/20190520232223_add_provisioning_fields.rb
|
396
397
|
- db/migrate/20190520232224_add_environment_fields.rb
|
398
|
+
- lib/metrics/influx/point_value.rb
|
397
399
|
- lib/metrics/net.rb
|
398
400
|
- lib/middleware/bad_multipart_form_data_sanitizer.rb
|
399
401
|
- lib/middleware/json_parse_errors.rb
|