zuora_connect 2.1.1 → 3.0.0.pre.a
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/app/controllers/zuora_connect/static_controller.rb +84 -17
- data/app/models/zuora_connect/app_instance_base.rb +168 -97
- data/app/models/zuora_connect/zuora_user.rb +1 -1
- data/config/initializers/postgresql_adapter.rb +71 -1
- data/config/initializers/prometheus.rb +80 -23
- data/config/initializers/redis.rb +4 -4
- data/config/initializers/resque.rb +20 -6
- data/config/initializers/unicorn.rb +30 -2
- data/config/routes.rb +5 -1
- data/lib/metrics/net.rb +7 -7
- data/lib/middleware/json_parse_errors.rb +13 -2
- data/lib/middleware/metrics_middleware.rb +52 -38
- data/lib/resque/dynamic_queues.rb +1 -1
- data/lib/resque/plugins/app_instance_job.rb +6 -10
- data/lib/zuora_connect.rb +6 -63
- data/lib/zuora_connect/controllers/helpers.rb +212 -76
- data/lib/zuora_connect/engine.rb +2 -1
- data/lib/zuora_connect/railtie.rb +9 -53
- data/lib/zuora_connect/version.rb +1 -1
- data/lib/zuora_connect/zuora_audit.rb +31 -0
- metadata +19 -7
- data/app/models/zuora_connect/telegraf.rb +0 -97
- data/lib/logging/connect_formatter.rb +0 -44
- data/lib/metrics/influx/point_value.rb +0 -79
data/lib/zuora_connect/engine.rb
CHANGED
@@ -10,9 +10,10 @@ module ZuoraConnect
|
|
10
10
|
|
11
11
|
initializer "connect", before: :load_config_initializers do |app|
|
12
12
|
Rails.application.routes.prepend do
|
13
|
+
get '/connect/internal/data' => 'zuora_observability/metrics#metrics'
|
13
14
|
mount ZuoraConnect::Engine, at: "/connect"
|
14
15
|
match '/api/connect/health', via: :all, to: 'zuora_connect/static#health'
|
15
|
-
match '/api/connect/internal/data', via: :all, to: '
|
16
|
+
match '/api/connect/internal/data', via: :all, to: 'zuora_observability/metrics#metrics'
|
16
17
|
end
|
17
18
|
end
|
18
19
|
|
@@ -28,11 +28,6 @@ module ZuoraConnect
|
|
28
28
|
::Rails.configuration.action_dispatch.x_sendfile_header = nil
|
29
29
|
end
|
30
30
|
|
31
|
-
if defined? Prometheus
|
32
|
-
initializer "prometheus.configure_rails_initialization" do |app|
|
33
|
-
app.middleware.use Prometheus::Middleware::Exporter,(options ={:path => '/connect/internal/metrics'})
|
34
|
-
end
|
35
|
-
end
|
36
31
|
initializer "zuora_connect.configure_rails_initialization" do |app|
|
37
32
|
app.middleware.insert_after Rack::Sendfile, ZuoraConnect::MetricsMiddleware
|
38
33
|
app.middleware.insert_after ActionDispatch::RequestId, ZuoraConnect::RequestIdMiddleware
|
@@ -40,55 +35,16 @@ module ZuoraConnect
|
|
40
35
|
app.config.middleware.use ZuoraConnect::JsonParseErrors
|
41
36
|
end
|
42
37
|
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
Rails.configuration.logger = ZuoraConnect.custom_logger(name: "Rails")
|
50
|
-
if !Rails.env.test? && !Rails.env.development?
|
51
|
-
Rails.configuration.lograge.enabled = true
|
52
|
-
Rails.configuration.colorize_logging = false
|
53
|
-
end
|
54
|
-
|
55
|
-
if Rails.configuration.lograge.enabled
|
56
|
-
if Rails.configuration.logger.class.to_s == 'Ougai::Logger'
|
57
|
-
Rails.configuration.lograge.formatter = Class.new do |fmt|
|
58
|
-
def fmt.call(data)
|
59
|
-
{ msg: 'Rails Request', request: data }
|
60
|
-
end
|
61
|
-
end
|
62
|
-
end
|
63
|
-
#Rails.configuration.lograge.formatter = Lograge::Formatters::Json.new
|
64
|
-
Rails.configuration.lograge.custom_options = lambda do |event|
|
65
|
-
exceptions = %w(controller action format)
|
66
|
-
items = {
|
67
|
-
#time: event.time.strftime('%FT%T.%6N'),
|
68
|
-
params: event.payload[:params].as_json(except: exceptions).to_json.to_s
|
69
|
-
}
|
70
|
-
items.merge!({exception_object: event.payload[:exception_object]}) if event.payload[:exception_object].present?
|
71
|
-
items.merge!({exception: event.payload[:exception]}) if event.payload[:exception].present?
|
72
|
-
|
73
|
-
if event.payload[:headers].present?
|
74
|
-
# By convertion, headers usually do not have dots. Nginx even rejects headers with dots
|
75
|
-
# All Rails headers are namespaced, like 'rack.input'.
|
76
|
-
# Thus, we can obtain the client headers by rejecting dots
|
77
|
-
request_headers =
|
78
|
-
event.payload[:headers].env.
|
79
|
-
reject { |key| key.to_s.include?('.') || REQUEST_HEADERS_TO_IGNORE.include?(key.to_s) }
|
80
|
-
items.merge!({ headers: request_headers.to_s })
|
81
|
-
end
|
82
|
-
|
83
|
-
if Thread.current[:appinstance].present?
|
84
|
-
items.merge!({connect_user: Thread.current[:appinstance].connect_user, new_session: Thread.current[:appinstance].new_session_message})
|
85
|
-
if Thread.current[:appinstance].logitems.present? && Thread.current[:appinstance].logitems.class == Hash
|
86
|
-
items.merge!(Thread.current[:appinstance].logitems)
|
87
|
-
end
|
88
|
-
end
|
89
|
-
return items
|
90
|
-
end
|
38
|
+
if defined? Prometheus
|
39
|
+
require 'rack'
|
40
|
+
require 'prometheus/middleware/exporter'
|
41
|
+
initializer "prometheus.configure_rails_initialization" do |app|
|
42
|
+
app.middleware.insert_after ZuoraConnect::MetricsMiddleware, Prometheus::Middleware::Exporter, path: '/connect/internal/metrics'
|
43
|
+
app.config.middleware.use Rack::Deflater, if: ->(env, *) { env['PATH_INFO'] == '/connect/internal/metrics' }
|
91
44
|
end
|
92
45
|
end
|
46
|
+
|
47
|
+
# hook to process_action
|
48
|
+
ActiveSupport::Notifications.subscribe('process_action.action_controller', ZuoraConnect::PageRequest.new)
|
93
49
|
end
|
94
50
|
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
# Added by @Vina
|
2
|
+
# Description: This automatically stamp user created/updated the record for DataQuery Audit
|
3
|
+
# Usage: add 'zuora_audit' to your model.rb that you would like to track
|
4
|
+
|
5
|
+
module ZuoraConnect
|
6
|
+
module ZuoraAudit
|
7
|
+
extend ActiveSupport::Concern
|
8
|
+
|
9
|
+
module ClassMethods
|
10
|
+
def zuora_audit
|
11
|
+
include ZuoraConnect::ZuoraAudit::ZuoraAuditInstanceMethods
|
12
|
+
before_create :set_created_by_id
|
13
|
+
before_update :set_updated_by_id
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
module ZuoraAuditInstanceMethods
|
18
|
+
def set_created_by_id
|
19
|
+
self.created_by_id = current_user_id if defined?(self.created_by_id)
|
20
|
+
end
|
21
|
+
|
22
|
+
def set_updated_by_id
|
23
|
+
self.updated_by_id = current_user_id if defined?(self.updated_by_id)
|
24
|
+
end
|
25
|
+
|
26
|
+
def current_user_id
|
27
|
+
return ZuoraConnect::ZuoraUser.current_user_id
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: zuora_connect
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 3.0.0.pre.a
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Connect Team
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-11-
|
11
|
+
date: 2020-11-30 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: apartment
|
@@ -182,6 +182,20 @@ dependencies:
|
|
182
182
|
- - ">="
|
183
183
|
- !ruby/object:Gem::Version
|
184
184
|
version: '0'
|
185
|
+
- !ruby/object:Gem::Dependency
|
186
|
+
name: zuora_observability
|
187
|
+
requirement: !ruby/object:Gem::Requirement
|
188
|
+
requirements:
|
189
|
+
- - ">="
|
190
|
+
- !ruby/object:Gem::Version
|
191
|
+
version: '0'
|
192
|
+
type: :runtime
|
193
|
+
prerelease: false
|
194
|
+
version_requirements: !ruby/object:Gem::Requirement
|
195
|
+
requirements:
|
196
|
+
- - ">="
|
197
|
+
- !ruby/object:Gem::Version
|
198
|
+
version: '0'
|
185
199
|
- !ruby/object:Gem::Dependency
|
186
200
|
name: rspec
|
187
201
|
requirement: !ruby/object:Gem::Requirement
|
@@ -318,7 +332,6 @@ files:
|
|
318
332
|
- app/models/zuora_connect/app_instance.rb
|
319
333
|
- app/models/zuora_connect/app_instance_base.rb
|
320
334
|
- app/models/zuora_connect/login.rb
|
321
|
-
- app/models/zuora_connect/telegraf.rb
|
322
335
|
- app/models/zuora_connect/zuora_user.rb
|
323
336
|
- app/views/layouts/zuora_connect/application.html.erb
|
324
337
|
- app/views/sql/refresh_aggregate_table.txt
|
@@ -353,8 +366,6 @@ files:
|
|
353
366
|
- db/migrate/20190520232222_add_unique_index.rb
|
354
367
|
- db/migrate/20190520232223_add_provisioning_fields.rb
|
355
368
|
- db/migrate/20190520232224_add_environment_fields.rb
|
356
|
-
- lib/logging/connect_formatter.rb
|
357
|
-
- lib/metrics/influx/point_value.rb
|
358
369
|
- lib/metrics/net.rb
|
359
370
|
- lib/middleware/bad_multipart_form_data_sanitizer.rb
|
360
371
|
- lib/middleware/json_parse_errors.rb
|
@@ -374,6 +385,7 @@ files:
|
|
374
385
|
- lib/zuora_connect/exceptions.rb
|
375
386
|
- lib/zuora_connect/railtie.rb
|
376
387
|
- lib/zuora_connect/version.rb
|
388
|
+
- lib/zuora_connect/zuora_audit.rb
|
377
389
|
- test/controllers/zuora_connect/api/v1/app_instance_controller_test.rb
|
378
390
|
- test/dummy/README.rdoc
|
379
391
|
- test/dummy/Rakefile
|
@@ -429,9 +441,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
429
441
|
version: '0'
|
430
442
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
431
443
|
requirements:
|
432
|
-
- - "
|
444
|
+
- - ">"
|
433
445
|
- !ruby/object:Gem::Version
|
434
|
-
version:
|
446
|
+
version: 1.3.1
|
435
447
|
requirements: []
|
436
448
|
rubygems_version: 3.0.3
|
437
449
|
signing_key:
|
@@ -1,97 +0,0 @@
|
|
1
|
-
module ZuoraConnect
|
2
|
-
class Telegraf
|
3
|
-
attr_accessor :host
|
4
|
-
|
5
|
-
OUTBOUND_METRICS = true
|
6
|
-
OUTBOUND_METRICS_NAME = "request-outbound"
|
7
|
-
INBOUND_METRICS = true
|
8
|
-
INBOUND_METRICS_NAME = "request-inbound"
|
9
|
-
|
10
|
-
def initialize
|
11
|
-
self.connect
|
12
|
-
end
|
13
|
-
|
14
|
-
def connect
|
15
|
-
ZuoraConnect.logger.debug(self.format_metric_log('Telegraf','Need new connection')) if ZuoraConnect.configuration.telegraf_debug
|
16
|
-
uri = URI.parse(ZuoraConnect.configuration.telegraf_endpoint)
|
17
|
-
self.host = UDPSocket.new.tap do |socket|
|
18
|
-
socket.connect uri.host, uri.port
|
19
|
-
end
|
20
|
-
rescue => ex
|
21
|
-
self.host = nil
|
22
|
-
ZuoraConnect.logger.warn(self.format_metric_log('Telegraf', "Failed to connect: #{ex.class}")) if Rails.env.to_s != 'production'
|
23
|
-
end
|
24
|
-
|
25
|
-
def write(direction: 'Unknown', tags: {}, values: {})
|
26
|
-
time = Benchmark.measure do |bench|
|
27
|
-
# To avoid writing metrics from rspec tests
|
28
|
-
if Rails.env.to_sym != :test
|
29
|
-
app_instance = Thread.current[:appinstance].present? ? Thread.current[:appinstance].id : 0
|
30
|
-
tags = { app_name: self.class.app_name, process_type: self.class.process_type, app_instance: app_instance, pod_name: self.class.pod_name}.merge(tags)
|
31
|
-
|
32
|
-
if direction == :inbound
|
33
|
-
if INBOUND_METRICS && !Thread.current[:inbound_metric].to_bool
|
34
|
-
self.write_udp(series: INBOUND_METRICS_NAME, tags: tags, values: values)
|
35
|
-
Thread.current[:inbound_metric] = true
|
36
|
-
else
|
37
|
-
return
|
38
|
-
end
|
39
|
-
elsif direction == :outbound
|
40
|
-
self.write_udp(series: OUTBOUND_METRICS_NAME, tags: tags, values: values) if OUTBOUND_METRICS
|
41
|
-
else
|
42
|
-
self.write_udp(series: direction, tags: tags, values: values)
|
43
|
-
end
|
44
|
-
end
|
45
|
-
end
|
46
|
-
if ZuoraConnect.configuration.telegraf_debug
|
47
|
-
ZuoraConnect.logger.debug(self.format_metric_log('Telegraf', tags.to_s))
|
48
|
-
ZuoraConnect.logger.debug(self.format_metric_log('Telegraf', values.to_s))
|
49
|
-
ZuoraConnect.logger.debug(self.format_metric_log('Telegraf', "Writing '#{direction.capitalize}': #{time.real.round(5)} ms"))
|
50
|
-
end
|
51
|
-
end
|
52
|
-
|
53
|
-
|
54
|
-
def write_udp(series: '', tags: {}, values: {})
|
55
|
-
return if !values.present?
|
56
|
-
self.host.write InfluxDB::PointValue.new({series: series, tags: tags, values: values}).dump
|
57
|
-
rescue => ex
|
58
|
-
self.connect
|
59
|
-
ZuoraConnect.logger.warn(self.format_metric_log('Telegraf',"Failed to write udp: #{ex.class}")) if Rails.env.to_s != 'production'
|
60
|
-
end
|
61
|
-
|
62
|
-
def format_metric_log(message, dump = nil)
|
63
|
-
message_color, dump_color = "1;91", "0;1"
|
64
|
-
log_entry = " \e[#{message_color}m#{message}\e[0m "
|
65
|
-
log_entry << "\e[#{dump_color}m%#{String === dump ? 's' : 'p'}\e[0m" % dump if dump
|
66
|
-
if Rails.env == :development
|
67
|
-
log_entry
|
68
|
-
else
|
69
|
-
[message, dump].compact.join(' - ')
|
70
|
-
end
|
71
|
-
end
|
72
|
-
|
73
|
-
def self.app_name
|
74
|
-
return ENV['DEIS_APP'].present? ? ENV['DEIS_APP'] : Rails.application.class.parent_name
|
75
|
-
end
|
76
|
-
|
77
|
-
def self.pod_name
|
78
|
-
return ENV['HOSTNAME'].present? ? ENV['HOSTNAME'] : Socket.gethostname
|
79
|
-
end
|
80
|
-
|
81
|
-
def self.full_process_name(process_name: nil, function: nil)
|
82
|
-
keys = [self.pod_name, process_name.present? ? process_name : self.process_type, Process.pid, function]
|
83
|
-
return keys.compact.join('][').prepend('[').concat(']')
|
84
|
-
end
|
85
|
-
|
86
|
-
# Returns the process type if any
|
87
|
-
def self.process_type(default: 'Unknown')
|
88
|
-
p_type = default
|
89
|
-
if ENV['HOSTNAME'] && ENV['DEIS_APP']
|
90
|
-
temp = ENV['HOSTNAME'].split(ENV['DEIS_APP'])[1]
|
91
|
-
temp = temp.split(/(-[0-9a-zA-Z]{5})$/)[0] # remove the 5 char hash
|
92
|
-
p_type = temp[1, temp.rindex("-")-1]
|
93
|
-
end
|
94
|
-
return p_type
|
95
|
-
end
|
96
|
-
end
|
97
|
-
end
|
@@ -1,44 +0,0 @@
|
|
1
|
-
require 'ougai/formatters/base'
|
2
|
-
require 'ougai/formatters/for_json'
|
3
|
-
|
4
|
-
module Ougai
|
5
|
-
module Formatters
|
6
|
-
# A JSON formatter compatible with node-bunyan
|
7
|
-
class ConnectFormatter < Base
|
8
|
-
include ForJson
|
9
|
-
|
10
|
-
# Intialize a formatter
|
11
|
-
# @param [String] app_name application name (execution program name if nil)
|
12
|
-
# @param [String] hostname hostname (hostname if nil)
|
13
|
-
# @param [Hash] opts the initial values of attributes
|
14
|
-
# @option opts [String] :trace_indent (2) the value of trace_indent attribute
|
15
|
-
# @option opts [String] :trace_max_lines (100) the value of trace_max_lines attribute
|
16
|
-
# @option opts [String] :serialize_backtrace (true) the value of serialize_backtrace attribute
|
17
|
-
# @option opts [String] :jsonize (true) the value of jsonize attribute
|
18
|
-
# @option opts [String] :with_newline (true) the value of with_newline attribute
|
19
|
-
def initialize(app_name = nil, hostname = nil, opts = {})
|
20
|
-
aname, hname, opts = Base.parse_new_params([app_name, hostname, opts])
|
21
|
-
super(aname, hname, opts)
|
22
|
-
init_opts_for_json(opts)
|
23
|
-
end
|
24
|
-
|
25
|
-
def _call(severity, time, progname, data)
|
26
|
-
data.merge!({message: data.delete(:msg)})
|
27
|
-
if data[:timestamp].present?
|
28
|
-
time = data[:timestamp]
|
29
|
-
data.delete(:timestamp)
|
30
|
-
end
|
31
|
-
dump({
|
32
|
-
name: progname || @app_name,
|
33
|
-
pid: $$,
|
34
|
-
level: severity,
|
35
|
-
timestamp: time.utc.strftime('%FT%T.%6NZ'),
|
36
|
-
}.merge(data))
|
37
|
-
end
|
38
|
-
|
39
|
-
def convert_time(data)
|
40
|
-
#data[:timestamp] = format_datetime(data[:time])
|
41
|
-
end
|
42
|
-
end
|
43
|
-
end
|
44
|
-
end
|
@@ -1,79 +0,0 @@
|
|
1
|
-
module InfluxDB
|
2
|
-
# Convert data point to string using Line protocol
|
3
|
-
class PointValue
|
4
|
-
attr_reader :series, :values, :tags, :timestamp
|
5
|
-
|
6
|
-
def initialize(data)
|
7
|
-
@series = escape data[:series], :measurement
|
8
|
-
@values = escape_values data[:values]
|
9
|
-
@tags = escape_tags data[:tags]
|
10
|
-
@timestamp = data[:timestamp]
|
11
|
-
end
|
12
|
-
|
13
|
-
def dump
|
14
|
-
dump = @series.dup
|
15
|
-
dump << ",#{@tags}" if @tags
|
16
|
-
dump << " #{@values}"
|
17
|
-
dump << " #{@timestamp}" if @timestamp
|
18
|
-
dump
|
19
|
-
end
|
20
|
-
|
21
|
-
private
|
22
|
-
|
23
|
-
ESCAPES = {
|
24
|
-
measurement: [' '.freeze, ','.freeze],
|
25
|
-
tag_key: ['='.freeze, ' '.freeze, ','.freeze],
|
26
|
-
tag_value: ['='.freeze, ' '.freeze, ','.freeze],
|
27
|
-
field_key: ['='.freeze, ' '.freeze, ','.freeze, '"'.freeze],
|
28
|
-
field_value: ["\\".freeze, '"'.freeze],
|
29
|
-
}.freeze
|
30
|
-
|
31
|
-
private_constant :ESCAPES
|
32
|
-
|
33
|
-
def escape(str, type)
|
34
|
-
# rubocop:disable Layout/AlignParameters
|
35
|
-
str = str.encode "UTF-8".freeze, "UTF-8".freeze,
|
36
|
-
invalid: :replace,
|
37
|
-
undef: :replace,
|
38
|
-
replace: "".freeze
|
39
|
-
# rubocop:enable Layout/AlignParameters
|
40
|
-
|
41
|
-
ESCAPES[type].each do |ch|
|
42
|
-
str = str.gsub(ch) { "\\#{ch}" }
|
43
|
-
end
|
44
|
-
str
|
45
|
-
end
|
46
|
-
|
47
|
-
def escape_values(values)
|
48
|
-
return if values.nil?
|
49
|
-
values.map do |k, v|
|
50
|
-
key = escape(k.to_s, :field_key)
|
51
|
-
val = escape_value(v)
|
52
|
-
"#{key}=#{val}"
|
53
|
-
end.join(",".freeze)
|
54
|
-
end
|
55
|
-
|
56
|
-
def escape_value(value)
|
57
|
-
if value.is_a?(String)
|
58
|
-
'"'.freeze + escape(value, :field_value) + '"'.freeze
|
59
|
-
elsif value.is_a?(Integer)
|
60
|
-
"#{value}i"
|
61
|
-
else
|
62
|
-
value.to_s
|
63
|
-
end
|
64
|
-
end
|
65
|
-
|
66
|
-
def escape_tags(tags)
|
67
|
-
return if tags.nil?
|
68
|
-
|
69
|
-
tags = tags.map do |k, v|
|
70
|
-
key = escape(k.to_s, :tag_key)
|
71
|
-
val = escape(v.to_s, :tag_value)
|
72
|
-
|
73
|
-
"#{key}=#{val}" unless key == "".freeze || val == "".freeze
|
74
|
-
end.compact
|
75
|
-
|
76
|
-
tags.join(",") unless tags.empty?
|
77
|
-
end
|
78
|
-
end
|
79
|
-
end
|