zuora_connect 3.0.0.pre.m → 3.0.0.pre.n
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/app/models/zuora_connect/app_instance_base.rb +18 -5
- data/app/models/zuora_connect/telegraf.rb +73 -0
- data/config/initializers/unicorn.rb +1 -1
- data/lib/metrics/influx/point_value.rb +79 -0
- data/lib/metrics/net.rb +7 -7
- data/lib/middleware/metrics_middleware.rb +34 -3
- data/lib/zuora_connect.rb +1 -0
- data/lib/zuora_connect/railtie.rb +3 -0
- data/lib/zuora_connect/version.rb +1 -1
- metadata +4 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e800e6da56132f3a8b5e0f52d2953347af0e73db371cef49421128b483cecb93
|
4
|
+
data.tar.gz: 24b4315f400c88afcd3916548887208b1a10b19a841617df4007beb0280cdf42
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d82d6ded09b50ae677194d3219b3c8d7204f8f18b9ede85431269ff3f677afa205a17ddb20f25760dd35e3d87da35ebac471d86e8d861d0fe5585b4671ddbdc3
|
7
|
+
data.tar.gz: b899378707679de0076efdfc1b14a73e781f03216f2ac1391c176567034a436b1074e9f71966e9efb59490442fe22621a3da19481b0397410a1e4bad97657477
|
@@ -12,6 +12,7 @@ module ZuoraConnect
|
|
12
12
|
|
13
13
|
self.table_name = "zuora_connect_app_instances"
|
14
14
|
attr_accessor :options, :mode, :logins, :task_data, :last_refresh, :username, :password, :s3_client, :api_version, :drop_message, :new_session_message, :connect_user, :logitems, :user_timezone
|
15
|
+
@@telegraf_host = nil
|
15
16
|
REFRESH_TIMEOUT = 2.minute #Used to determine how long to wait on current refresh call before executing another
|
16
17
|
INSTANCE_REFRESH_WINDOW = 1.hours #Used to set how how long till app starts attempting to refresh cached task connect data
|
17
18
|
INSTANCE_REDIS_CACHE_PERIOD = 24.hours #Used to determine how long to cached task data will live for
|
@@ -362,9 +363,9 @@ module ZuoraConnect
|
|
362
363
|
raise ZuoraConnect::Exceptions::ConnectCommunicationError.new("JSON parse error", response.body, response.code)
|
363
364
|
end
|
364
365
|
|
365
|
-
self.build_task(task_data: parsed_json, session: session)
|
366
|
+
self.build_task(task_data: parsed_json, session: session)
|
366
367
|
self.set_backup_creds
|
367
|
-
self.save(validate: false) if self.changed?
|
368
|
+
self.save(validate: false) if self.changed?
|
368
369
|
else
|
369
370
|
raise ZuoraConnect::Exceptions::ConnectCommunicationError.new("Error Communicating with Connect", response.body, response.code)
|
370
371
|
end
|
@@ -412,8 +413,8 @@ module ZuoraConnect
|
|
412
413
|
def set_backup_creds
|
413
414
|
if self.kms_key.present? && self.kms_key.match(/^arn:aws:.*/) && self.task_data.present?
|
414
415
|
self.zuora_logins = self.strip_cache_data(object: self.task_data.dup, keys: ['applications', 'tokens', 'user_settings'])
|
415
|
-
end
|
416
|
-
end
|
416
|
+
end
|
417
|
+
end
|
417
418
|
|
418
419
|
def zuora_logins=(val)
|
419
420
|
write_attribute(:zuora_logins, kms_encrypt(val.to_json))
|
@@ -478,6 +479,15 @@ module ZuoraConnect
|
|
478
479
|
end
|
479
480
|
Thread.current[:appinstance] = self
|
480
481
|
end
|
482
|
+
|
483
|
+
def self.write_to_telegraf(*args)
|
484
|
+
if ZuoraConnect.configuration.enable_metrics && !defined?(Prometheus)
|
485
|
+
@@telegraf_host = ZuoraConnect::Telegraf.new() if @@telegraf_host == nil
|
486
|
+
unicorn_stats = ZuoraObservability::Metrics.unicorn_listener if defined?(Unicorn) && Unicorn.respond_to?(:listener_names)
|
487
|
+
@@telegraf_host.write(direction: 'Raindrops', tags: {}, values: unicorn_stats) unless unicorn_stats.blank?
|
488
|
+
return @@telegraf_host.write(*args)
|
489
|
+
end
|
490
|
+
end
|
481
491
|
#### END Task Methods ####
|
482
492
|
|
483
493
|
#### START Task Methods ####
|
@@ -600,7 +610,10 @@ module ZuoraConnect
|
|
600
610
|
self.provisioned_app = nil
|
601
611
|
end
|
602
612
|
end
|
603
|
-
self.
|
613
|
+
if self.changed?
|
614
|
+
self.save(:validate => false)
|
615
|
+
self.refresh
|
616
|
+
end
|
604
617
|
|
605
618
|
return parsed_json
|
606
619
|
end
|
@@ -0,0 +1,73 @@
|
|
1
|
+
module ZuoraConnect
|
2
|
+
class Telegraf
|
3
|
+
attr_accessor :host
|
4
|
+
|
5
|
+
OUTBOUND_METRICS = true
|
6
|
+
OUTBOUND_METRICS_NAME = "request-outbound"
|
7
|
+
INBOUND_METRICS = true
|
8
|
+
INBOUND_METRICS_NAME = "request-inbound"
|
9
|
+
|
10
|
+
def initialize
|
11
|
+
self.connect
|
12
|
+
end
|
13
|
+
|
14
|
+
def connect
|
15
|
+
ZuoraConnect.logger.debug(self.format_metric_log('Telegraf','Need new connection')) if ZuoraConnect.configuration.telegraf_debug
|
16
|
+
uri = URI.parse(ZuoraConnect.configuration.telegraf_endpoint)
|
17
|
+
self.host = UDPSocket.new.tap do |socket|
|
18
|
+
socket.connect uri.host, uri.port
|
19
|
+
end
|
20
|
+
rescue => ex
|
21
|
+
self.host = nil
|
22
|
+
ZuoraConnect.logger.warn(self.format_metric_log('Telegraf', "Failed to connect: #{ex.class}")) if Rails.env.to_s != 'production'
|
23
|
+
end
|
24
|
+
|
25
|
+
def write(direction: 'Unknown', tags: {}, values: {})
|
26
|
+
time = Benchmark.measure do |bench|
|
27
|
+
# To avoid writing metrics from rspec tests
|
28
|
+
if Rails.env.to_sym != :test
|
29
|
+
app_instance = Thread.current[:appinstance].present? ? Thread.current[:appinstance].id : 0
|
30
|
+
tags = { app_name: ZuoraObservability::Env.app_name, process_type: ZuoraObservability::Env.process_type, app_instance: app_instance, pod_name: ZuoraObservability::Env.pod_name}.merge(tags)
|
31
|
+
|
32
|
+
if direction == :inbound
|
33
|
+
if INBOUND_METRICS && !Thread.current[:inbound_metric].to_bool
|
34
|
+
self.write_udp(series: INBOUND_METRICS_NAME, tags: tags, values: values)
|
35
|
+
Thread.current[:inbound_metric] = true
|
36
|
+
else
|
37
|
+
return
|
38
|
+
end
|
39
|
+
elsif direction == :outbound
|
40
|
+
self.write_udp(series: OUTBOUND_METRICS_NAME, tags: tags, values: values) if OUTBOUND_METRICS
|
41
|
+
else
|
42
|
+
self.write_udp(series: direction, tags: tags, values: values)
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
if ZuoraConnect.configuration.telegraf_debug
|
47
|
+
ZuoraConnect.logger.debug(self.format_metric_log('Telegraf', tags.to_s))
|
48
|
+
ZuoraConnect.logger.debug(self.format_metric_log('Telegraf', values.to_s))
|
49
|
+
ZuoraConnect.logger.debug(self.format_metric_log('Telegraf', "Writing '#{direction.capitalize}': #{time.real.round(5)} ms"))
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
|
54
|
+
def write_udp(series: '', tags: {}, values: {})
|
55
|
+
return if !values.present?
|
56
|
+
self.host.write InfluxDB::PointValue.new({series: series, tags: tags, values: values}).dump
|
57
|
+
rescue => ex
|
58
|
+
self.connect
|
59
|
+
ZuoraConnect.logger.warn(self.format_metric_log('Telegraf',"Failed to write udp: #{ex.class}")) if Rails.env.to_s != 'production'
|
60
|
+
end
|
61
|
+
|
62
|
+
def format_metric_log(message, dump = nil)
|
63
|
+
message_color, dump_color = "1;91", "0;1"
|
64
|
+
log_entry = " \e[#{message_color}m#{message}\e[0m "
|
65
|
+
log_entry << "\e[#{dump_color}m%#{String === dump ? 's' : 'p'}\e[0m" % dump if dump
|
66
|
+
if Rails.env == :development
|
67
|
+
log_entry
|
68
|
+
else
|
69
|
+
[message, dump].compact.join(' - ')
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
@@ -6,7 +6,7 @@ if defined?(Unicorn::WorkerKiller)
|
|
6
6
|
if defined?(Prometheus)
|
7
7
|
Prometheus::UNICORN_KILLS.set(1)
|
8
8
|
else
|
9
|
-
|
9
|
+
ZuoraConnect::AppInstance.write_to_telegraf(direction: 'Unicorn-Killer', tags: {app_instance: 0}, values: {kill: 1})
|
10
10
|
end
|
11
11
|
end
|
12
12
|
end
|
@@ -0,0 +1,79 @@
|
|
1
|
+
module InfluxDB
|
2
|
+
# Convert data point to string using Line protocol
|
3
|
+
class PointValue
|
4
|
+
attr_reader :series, :values, :tags, :timestamp
|
5
|
+
|
6
|
+
def initialize(data)
|
7
|
+
@series = escape data[:series], :measurement
|
8
|
+
@values = escape_values data[:values]
|
9
|
+
@tags = escape_tags data[:tags]
|
10
|
+
@timestamp = data[:timestamp]
|
11
|
+
end
|
12
|
+
|
13
|
+
def dump
|
14
|
+
dump = @series.dup
|
15
|
+
dump << ",#{@tags}" if @tags
|
16
|
+
dump << " #{@values}"
|
17
|
+
dump << " #{@timestamp}" if @timestamp
|
18
|
+
dump
|
19
|
+
end
|
20
|
+
|
21
|
+
private
|
22
|
+
|
23
|
+
ESCAPES = {
|
24
|
+
measurement: [' '.freeze, ','.freeze],
|
25
|
+
tag_key: ['='.freeze, ' '.freeze, ','.freeze],
|
26
|
+
tag_value: ['='.freeze, ' '.freeze, ','.freeze],
|
27
|
+
field_key: ['='.freeze, ' '.freeze, ','.freeze, '"'.freeze],
|
28
|
+
field_value: ["\\".freeze, '"'.freeze],
|
29
|
+
}.freeze
|
30
|
+
|
31
|
+
private_constant :ESCAPES
|
32
|
+
|
33
|
+
def escape(str, type)
|
34
|
+
# rubocop:disable Layout/AlignParameters
|
35
|
+
str = str.encode "UTF-8".freeze, "UTF-8".freeze,
|
36
|
+
invalid: :replace,
|
37
|
+
undef: :replace,
|
38
|
+
replace: "".freeze
|
39
|
+
# rubocop:enable Layout/AlignParameters
|
40
|
+
|
41
|
+
ESCAPES[type].each do |ch|
|
42
|
+
str = str.gsub(ch) { "\\#{ch}" }
|
43
|
+
end
|
44
|
+
str
|
45
|
+
end
|
46
|
+
|
47
|
+
def escape_values(values)
|
48
|
+
return if values.nil?
|
49
|
+
values.map do |k, v|
|
50
|
+
key = escape(k.to_s, :field_key)
|
51
|
+
val = escape_value(v)
|
52
|
+
"#{key}=#{val}"
|
53
|
+
end.join(",".freeze)
|
54
|
+
end
|
55
|
+
|
56
|
+
def escape_value(value)
|
57
|
+
if value.is_a?(String)
|
58
|
+
'"'.freeze + escape(value, :field_value) + '"'.freeze
|
59
|
+
elsif value.is_a?(Integer)
|
60
|
+
"#{value}i"
|
61
|
+
else
|
62
|
+
value.to_s
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
def escape_tags(tags)
|
67
|
+
return if tags.nil?
|
68
|
+
|
69
|
+
tags = tags.map do |k, v|
|
70
|
+
key = escape(k.to_s, :tag_key)
|
71
|
+
val = escape(v.to_s, :tag_value)
|
72
|
+
|
73
|
+
"#{key}=#{val}" unless key == "".freeze || val == "".freeze
|
74
|
+
end.compact
|
75
|
+
|
76
|
+
tags.join(",") unless tags.empty?
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
data/lib/metrics/net.rb
CHANGED
@@ -68,12 +68,12 @@ class HttpLogger
|
|
68
68
|
log_request_body(request)
|
69
69
|
log_request_headers(request)
|
70
70
|
if defined?(response) && response
|
71
|
-
tags = tags.merge({status: response.code.to_i})
|
71
|
+
tags = tags.merge({status: response.code.to_i})
|
72
72
|
log_response_code(response)
|
73
73
|
log_response_headers(response)
|
74
74
|
log_response_body(response.body)
|
75
75
|
end
|
76
|
-
|
76
|
+
ZuoraConnect::AppInstance.write_to_telegraf(direction: :outbound, tags: tags, values: values)
|
77
77
|
end
|
78
78
|
end
|
79
79
|
|
@@ -95,7 +95,7 @@ class HttpLogger
|
|
95
95
|
end
|
96
96
|
|
97
97
|
HTTP_METHODS_WITH_BODY = Set.new(%w(POST PUT GET PATCH))
|
98
|
-
|
98
|
+
|
99
99
|
def log_request_body(request)
|
100
100
|
if self.class.log_request_body
|
101
101
|
if HTTP_METHODS_WITH_BODY.include?(request.method)
|
@@ -149,8 +149,8 @@ class HttpLogger
|
|
149
149
|
def truncate_body(body)
|
150
150
|
if collapse_body_limit && collapse_body_limit > 0 && body && body.size >= collapse_body_limit
|
151
151
|
body_piece_size = collapse_body_limit / 2
|
152
|
-
body[0..body_piece_size] +
|
153
|
-
"\n\n<some data truncated>\n\n" +
|
152
|
+
body[0..body_piece_size] +
|
153
|
+
"\n\n<some data truncated>\n\n" +
|
154
154
|
body[(body.size - body_piece_size)..body.size]
|
155
155
|
else
|
156
156
|
body
|
@@ -203,7 +203,7 @@ class Net::HTTP
|
|
203
203
|
|
204
204
|
def request(request, body = nil, &block)
|
205
205
|
HttpLogger.perform(self, request, body) do
|
206
|
-
request_without_logging(request, body, &block)
|
206
|
+
request_without_logging(request, body, &block)
|
207
207
|
end
|
208
208
|
end
|
209
209
|
end
|
@@ -215,4 +215,4 @@ if defined?(Rails)
|
|
215
215
|
HttpLogger.logger = ZuoraConnect.logger unless HttpLogger.logger
|
216
216
|
end
|
217
217
|
end
|
218
|
-
end
|
218
|
+
end
|
@@ -1,6 +1,37 @@
|
|
1
1
|
module ZuoraConnect
|
2
2
|
require 'uri'
|
3
3
|
|
4
|
+
# Object of this class is passed to the ActiveSupport::Notification hook
|
5
|
+
class PageRequest
|
6
|
+
|
7
|
+
# This method is triggered when a non error page is loaded (not 404)
|
8
|
+
def call(name, started, finished, unique_id, payload)
|
9
|
+
# If the url contains any css or JavaScript files then do not collect metrics for them
|
10
|
+
return nil if ["css", "assets", "jpg", "png", "jpeg", "ico"].any? { |word| payload[:path].include?(word) }
|
11
|
+
|
12
|
+
# Getting the endpoint and the content_type
|
13
|
+
content_hash = {:html => "text/html", :js => "application/javascript", :json => "application/json", :csv => "text/csv"}
|
14
|
+
content_type = content_hash.key?(payload[:format]) ? content_hash[payload[:format]] : payload[:format]
|
15
|
+
content_type = content_type.to_s.gsub('text/javascript', 'application/javascript')
|
16
|
+
|
17
|
+
# payloads with 500 requests do not have status as it is not set by the controller
|
18
|
+
# https://github.com/rails/rails/issues/33335
|
19
|
+
#status_code = payload[:status] ? payload[:status] : payload[:exception_object].present? ? 500 : ""
|
20
|
+
if payload[:exception].present?
|
21
|
+
status_code, exception = [500, payload[:exception].first]
|
22
|
+
else
|
23
|
+
status_code, exception = [payload[:status], nil]
|
24
|
+
end
|
25
|
+
|
26
|
+
tags = {method: payload[:method], status: status_code, error_type: exception, content_type: content_type, controller: payload[:controller], action: payload[:action]}.compact
|
27
|
+
|
28
|
+
values = {view_time: payload[:view_runtime], db_time: payload[:db_runtime], response_time: ((finished-started)*1000)}.compact
|
29
|
+
values = values.map{ |k,v| [k,v.round(2)]}.to_h
|
30
|
+
|
31
|
+
ZuoraConnect::AppInstanceBase.write_to_telegraf(direction: :inbound, tags: tags, values: values)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
4
35
|
class MetricsMiddleware
|
5
36
|
|
6
37
|
require "zuora_connect/version"
|
@@ -36,9 +67,9 @@ module ZuoraConnect
|
|
36
67
|
if defined?(Prometheus) && env['PATH_INFO'] == '/connect/internal/metrics'
|
37
68
|
|
38
69
|
# Prometheus Stuff
|
39
|
-
metrics = ZuoraObservability::Metrics.resque
|
70
|
+
metrics = ZuoraObservability::Metrics.resque
|
40
71
|
metrics = defined?(ZuoraConnect::AppInstance.get_metrics) ? ZuoraConnect::AppInstance.get_metrics(metrics) : metrics
|
41
|
-
|
72
|
+
|
42
73
|
redis_up = metrics.present? && metrics.dig(:Resque, :Workers_Total).present? ? 1 : 0
|
43
74
|
Prometheus::REDIS_CONNECTION.set(redis_up)
|
44
75
|
|
@@ -77,7 +108,7 @@ module ZuoraConnect
|
|
77
108
|
|
78
109
|
values = {response_time: ((Time.now - start_time)*1000).round(2) }
|
79
110
|
|
80
|
-
|
111
|
+
ZuoraConnect::AppInstanceBase.write_to_telegraf(direction: :inbound, tags: tags, values: values)
|
81
112
|
end
|
82
113
|
end
|
83
114
|
Thread.current[:inbound_metric] = nil
|
data/lib/zuora_connect.rb
CHANGED
@@ -9,6 +9,7 @@ require 'resque/silence_done'
|
|
9
9
|
require 'resque/self_lookup'
|
10
10
|
require 'resque/plugins/custom_logger'
|
11
11
|
require 'resque/plugins/app_instance_job'
|
12
|
+
require 'metrics/influx/point_value'
|
12
13
|
require 'metrics/net'
|
13
14
|
require 'mono_logger'
|
14
15
|
require 'zuora_connect/zuora_audit'
|
@@ -32,5 +32,8 @@ module ZuoraConnect
|
|
32
32
|
app.config.middleware.use Rack::Deflater, if: ->(env, *) { env['PATH_INFO'] == '/connect/internal/metrics' }
|
33
33
|
end
|
34
34
|
end
|
35
|
+
|
36
|
+
# hook to process_action
|
37
|
+
ActiveSupport::Notifications.subscribe('process_action.action_controller', ZuoraConnect::PageRequest.new)
|
35
38
|
end
|
36
39
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: zuora_connect
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.0.0.pre.
|
4
|
+
version: 3.0.0.pre.n
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Connect Team
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-01-06 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: apartment
|
@@ -360,6 +360,7 @@ files:
|
|
360
360
|
- app/models/zuora_connect/app_instance.rb
|
361
361
|
- app/models/zuora_connect/app_instance_base.rb
|
362
362
|
- app/models/zuora_connect/login.rb
|
363
|
+
- app/models/zuora_connect/telegraf.rb
|
363
364
|
- app/models/zuora_connect/zuora_user.rb
|
364
365
|
- app/views/layouts/zuora_connect/application.html.erb
|
365
366
|
- app/views/sql/refresh_aggregate_table.txt
|
@@ -394,6 +395,7 @@ files:
|
|
394
395
|
- db/migrate/20190520232222_add_unique_index.rb
|
395
396
|
- db/migrate/20190520232223_add_provisioning_fields.rb
|
396
397
|
- db/migrate/20190520232224_add_environment_fields.rb
|
398
|
+
- lib/metrics/influx/point_value.rb
|
397
399
|
- lib/metrics/net.rb
|
398
400
|
- lib/middleware/bad_multipart_form_data_sanitizer.rb
|
399
401
|
- lib/middleware/json_parse_errors.rb
|