zuora_connect 2.0.60v → 3.0.0.pre.c
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/app/controllers/zuora_connect/static_controller.rb +3 -7
- data/app/models/zuora_connect/app_instance_base.rb +24 -103
- data/config/initializers/prometheus.rb +13 -13
- data/config/initializers/redis.rb +4 -4
- data/config/initializers/resque.rb +0 -12
- data/config/initializers/unicorn.rb +2 -2
- data/config/routes.rb +0 -1
- data/lib/metrics/net.rb +7 -7
- data/lib/middleware/metrics_middleware.rb +6 -44
- data/lib/zuora_connect.rb +3 -63
- data/lib/zuora_connect/controllers/helpers.rb +14 -11
- data/lib/zuora_connect/engine.rb +2 -1
- data/lib/zuora_connect/railtie.rb +0 -77
- data/lib/zuora_connect/version.rb +1 -1
- metadata +16 -5
- data/app/models/zuora_connect/telegraf.rb +0 -97
- data/lib/logging/connect_formatter.rb +0 -44
- data/lib/metrics/influx/point_value.rb +0 -79
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 27fb22389a0778791fc2ecab5d1ad5bbd651f1b647712830b4e006102e28eecd
|
4
|
+
data.tar.gz: c5aaa8e152b214b12abf26383d46371377c4ef0fca324929e9ffc33952663600
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: edd9d1b187c9a517eaf91d643718ada56deec42eec78f6f570a93bcdd83e99d1b6ab1ffe41c2bb86f763139d1ab6e6ea66c36f5854a815356a5563494b990c11
|
7
|
+
data.tar.gz: 59a8465e0558022d280b08100c91ac77bdc955e55f24c19201c4d198b5720c4a3afdfa3650520fca8edcdc883fd81e2a3d152d1b1ba8b8f3c80a83d3e84e6c17
|
@@ -1,16 +1,12 @@
|
|
1
1
|
module ZuoraConnect
|
2
2
|
class StaticController < ApplicationController
|
3
|
-
before_action :authenticate_connect_app_request, :except => [:
|
4
|
-
before_action :clear_connect_app_session, :only => [:
|
5
|
-
after_action :persist_connect_app_session, :except => [:
|
3
|
+
before_action :authenticate_connect_app_request, :except => [:health, :initialize_app, :provision, :instance_user]
|
4
|
+
before_action :clear_connect_app_session, :only => [:health, :initialize_app, :provision, :instance_user]
|
5
|
+
after_action :persist_connect_app_session, :except => [:health, :initialize_app, :provision, :instance_user]
|
6
6
|
|
7
7
|
skip_before_action :verify_authenticity_token, :only => [:initialize_app, :provision]
|
8
8
|
http_basic_authenticate_with name: ENV['PROVISION_USER'], password: ENV['PROVISION_SECRET'], :only => [:provision, :instance_user]
|
9
9
|
|
10
|
-
def metrics
|
11
|
-
type = params[:type].present? ? params[:type] : "versions"
|
12
|
-
render json: ZuoraConnect::AppInstance.get_metrics(type).to_json, status: 200
|
13
|
-
end
|
14
10
|
|
15
11
|
def health
|
16
12
|
if params[:error].present?
|
@@ -12,7 +12,6 @@ module ZuoraConnect
|
|
12
12
|
|
13
13
|
self.table_name = "zuora_connect_app_instances"
|
14
14
|
attr_accessor :options, :mode, :logins, :task_data, :last_refresh, :username, :password, :s3_client, :api_version, :drop_message, :new_session_message, :connect_user, :logitems, :user_timezone
|
15
|
-
@@telegraf_host = nil
|
16
15
|
REFRESH_TIMEOUT = 2.minute #Used to determine how long to wait on current refresh call before executing another
|
17
16
|
INSTANCE_REFRESH_WINDOW = 1.hours #Used to set how how long till app starts attempting to refresh cached task connect data
|
18
17
|
INSTANCE_REDIS_CACHE_PERIOD = 24.hours #Used to determine how long to cached task data will live for
|
@@ -35,7 +34,7 @@ module ZuoraConnect
|
|
35
34
|
Aws::Errors::MissingCredentialsError,
|
36
35
|
Aws::S3::Errors::AccessDenied,
|
37
36
|
Aws::SES::Errors::AccessDenied,
|
38
|
-
Aws::KMS::Errors::AccessDeniedException
|
37
|
+
Aws::KMS::Errors::AccessDeniedException
|
39
38
|
].freeze
|
40
39
|
AWS_AUTH_ERRORS_MSG = "AWS Auth Errors".freeze
|
41
40
|
|
@@ -169,6 +168,7 @@ module ZuoraConnect
|
|
169
168
|
end
|
170
169
|
|
171
170
|
self.build_task(task_data: mock_task_data, session: session)
|
171
|
+
self.set_backup_creds
|
172
172
|
self.last_refresh = Time.now.to_i
|
173
173
|
else
|
174
174
|
time_expire = (session["#{self.id}::last_refresh"] || Time.now).to_i - INSTANCE_REFRESH_WINDOW.ago.to_i
|
@@ -263,7 +263,7 @@ module ZuoraConnect
|
|
263
263
|
end
|
264
264
|
|
265
265
|
params = {
|
266
|
-
name: self.task_data.dig('name'),
|
266
|
+
name: self.task_data.dig('name'),
|
267
267
|
zuora_entity_ids: (self.task_data.dig(LOGIN_TENANT_DESTINATION,'entities') || []).map{|e| e['id']}.uniq,
|
268
268
|
zuora_tenant_ids: tenants.map(&:to_s).uniq,
|
269
269
|
organizations: organizations
|
@@ -272,7 +272,7 @@ module ZuoraConnect
|
|
272
272
|
client = self.send(LOGIN_TENANT_DESTINATION).client
|
273
273
|
if defined?(client.rest_domain)
|
274
274
|
ZuoraConnect::RequestIdMiddleware.zuora_rest_domain = client.rest_domain
|
275
|
-
params.merge!({zuora_domain: client.rest_domain, environment: client.environment })
|
275
|
+
params.merge!({zuora_domain: client.rest_domain, environment: client.environment })
|
276
276
|
end
|
277
277
|
end
|
278
278
|
params = params.reject{|k,v| !self.attributes.keys.member?(k.to_s) || self[k] == v}
|
@@ -290,10 +290,7 @@ module ZuoraConnect
|
|
290
290
|
elsif type == :user
|
291
291
|
begin
|
292
292
|
sql = <<-eos
|
293
|
-
SELECT zuora_users.zuora_identity_response
|
294
|
-
FROM "#{self.id}".zuora_users
|
295
|
-
ORDER BY zuora_users.updated_at DESC
|
296
|
-
LIMIT 1;
|
293
|
+
SELECT zuora_users.zuora_identity_response FROM "#{self.id}".zuora_users ORDER BY zuora_users.updated_at DESC LIMIT 1;
|
297
294
|
eos
|
298
295
|
user = ActiveRecord::Base.connection.execute(sql).to_a.first
|
299
296
|
|
@@ -348,12 +345,12 @@ module ZuoraConnect
|
|
348
345
|
self.id >= 25000000
|
349
346
|
end
|
350
347
|
|
351
|
-
def refresh(session: {})
|
348
|
+
def refresh(session: {})
|
352
349
|
refresh_count ||= 0
|
353
350
|
skip_connect ||= false
|
354
351
|
begin
|
355
352
|
#Check how app was deployed
|
356
|
-
if self.
|
353
|
+
if !self.auto_deployed? && !skip_connect
|
357
354
|
self.check_oauth_state
|
358
355
|
response = HTTParty.get(ZuoraConnect.configuration.url + "/api/#{self.api_version}/tools/tasks/#{self.id}.json",:body => {:access_token => self.access_token})
|
359
356
|
|
@@ -365,16 +362,8 @@ module ZuoraConnect
|
|
365
362
|
end
|
366
363
|
|
367
364
|
self.build_task(task_data: parsed_json, session: session)
|
368
|
-
|
369
|
-
|
370
|
-
self.zuora_logins = self.strip_cache_data(object: parsed_json.dup, keys: ['applications', 'tokens', 'user_settings'])
|
371
|
-
self.save(:validate => false)
|
372
|
-
rescue Aws::KMS::Errors::ValidationException, *AWS_AUTH_ERRORS => ex
|
373
|
-
Rails.logger.warn(AWS_AUTH_ERRORS_MSG, ex)
|
374
|
-
rescue => ex
|
375
|
-
Rails.logger.error(AWS_AUTH_ERRORS_MSG, ex)
|
376
|
-
end
|
377
|
-
end
|
365
|
+
self.set_backup_creds
|
366
|
+
self.save(validate: false) if self.changed?
|
378
367
|
else
|
379
368
|
raise ZuoraConnect::Exceptions::ConnectCommunicationError.new("Error Communicating with Connect", response.body, response.code)
|
380
369
|
end
|
@@ -419,8 +408,16 @@ module ZuoraConnect
|
|
419
408
|
end
|
420
409
|
|
421
410
|
#### START KMS ENCRYPTION Methods ####
|
411
|
+
def set_backup_creds
|
412
|
+
if self.kms_key.present? && self.kms_key.match(/^arn:aws:.*/) && self.task_data.present?
|
413
|
+
self.zuora_logins = self.strip_cache_data(object: self.task_data.dup, keys: ['applications', 'tokens', 'user_settings'])
|
414
|
+
end
|
415
|
+
end
|
416
|
+
|
422
417
|
def zuora_logins=(val)
|
423
418
|
write_attribute(:zuora_logins, kms_encrypt(val.to_json))
|
419
|
+
rescue Aws::KMS::Errors::ValidationException, Aws::KMS::Errors::NotFoundException, *AWS_AUTH_ERRORS => ex
|
420
|
+
Rails.logger.warn(AWS_AUTH_ERRORS_MSG, ex)
|
424
421
|
end
|
425
422
|
|
426
423
|
def zuora_logins
|
@@ -480,82 +477,6 @@ module ZuoraConnect
|
|
480
477
|
end
|
481
478
|
Thread.current[:appinstance] = self
|
482
479
|
end
|
483
|
-
|
484
|
-
def self.write_to_telegraf(*args)
|
485
|
-
if ZuoraConnect.configuration.enable_metrics && !defined?(Prometheus)
|
486
|
-
@@telegraf_host = ZuoraConnect::Telegraf.new() if @@telegraf_host == nil
|
487
|
-
unicorn_stats = self.unicorn_listener_stats() if defined?(Unicorn) && Unicorn.respond_to?(:listener_names)
|
488
|
-
@@telegraf_host.write(direction: 'Raindrops', tags: {}, values: unicorn_stats) unless unicorn_stats.blank?
|
489
|
-
return @@telegraf_host.write(*args)
|
490
|
-
end
|
491
|
-
end
|
492
|
-
|
493
|
-
def self.unicorn_listener_stats ()
|
494
|
-
stats_hash = {}
|
495
|
-
stats_hash["total_active"] = 0
|
496
|
-
stats_hash["total_queued"] = 0
|
497
|
-
|
498
|
-
begin
|
499
|
-
tmp = Unicorn.listener_names
|
500
|
-
unix = tmp.grep(%r{\A/})
|
501
|
-
tcp = tmp.grep(/\A.+:\d+\z/)
|
502
|
-
tcp = nil if tcp.empty?
|
503
|
-
unix = nil if unix.empty?
|
504
|
-
|
505
|
-
|
506
|
-
Raindrops::Linux.tcp_listener_stats(tcp).each do |addr,stats|
|
507
|
-
stats_hash["active_#{addr}"] = stats.active
|
508
|
-
stats_hash["queued_#{addr}"] = stats.queued
|
509
|
-
stats_hash["total_active"] = stats.active + stats_hash["total_active"]
|
510
|
-
stats_hash["total_queued"] = stats.queued + stats_hash["total_queued"]
|
511
|
-
end if tcp
|
512
|
-
|
513
|
-
Raindrops::Linux.unix_listener_stats(unix).each do |addr,stats|
|
514
|
-
stats_hash["active_#{addr}"] = stats.active
|
515
|
-
stats_hash["queued_#{addr}"] = stats.queued
|
516
|
-
stats_hash["total_active"] = stats.active + stats_hash["total_active"]
|
517
|
-
stats_hash["total_queued"] = stats.queued + stats_hash["total_queued"]
|
518
|
-
end if unix
|
519
|
-
rescue IOError => ex
|
520
|
-
rescue => ex
|
521
|
-
ZuoraConnect.logger.error(ex)
|
522
|
-
end
|
523
|
-
return stats_hash
|
524
|
-
end
|
525
|
-
|
526
|
-
def self.get_metrics(type)
|
527
|
-
@data = {}
|
528
|
-
|
529
|
-
if type == "versions"
|
530
|
-
@data = {
|
531
|
-
app_name: ZuoraConnect::Telegraf.app_name,
|
532
|
-
url: "dummy",
|
533
|
-
Version_Gem: ZuoraConnect::VERSION,
|
534
|
-
Version_Zuora: ZuoraAPI::VERSION ,
|
535
|
-
Version_Ruby: RUBY_VERSION,
|
536
|
-
Version_Rails: Rails.version,
|
537
|
-
hold: 1
|
538
|
-
}
|
539
|
-
elsif type == "stats"
|
540
|
-
begin
|
541
|
-
Resque.redis.ping
|
542
|
-
@resque = Resque.info
|
543
|
-
@data = {
|
544
|
-
app_name: ZuoraConnect::Telegraf.app_name,
|
545
|
-
url: "dummy",
|
546
|
-
Resque:{
|
547
|
-
Jobs_Finished: @resque[:processed] ,
|
548
|
-
Jobs_Failed: @resque[:failed],
|
549
|
-
Jobs_Pending: @resque[:pending],
|
550
|
-
Workers_Active: @resque[:working],
|
551
|
-
Workers_Total: @resque[:workers]
|
552
|
-
}
|
553
|
-
}
|
554
|
-
rescue
|
555
|
-
end
|
556
|
-
end
|
557
|
-
return @data
|
558
|
-
end
|
559
480
|
#### END Task Methods ####
|
560
481
|
|
561
482
|
#### START Task Methods ####
|
@@ -598,7 +519,7 @@ module ZuoraConnect
|
|
598
519
|
end
|
599
520
|
rescue ZuoraConnect::Exceptions::MissMatch => ex
|
600
521
|
raise
|
601
|
-
rescue ZuoraConnect::Exceptions::InvalidCredentialSet => ex
|
522
|
+
rescue ZuoraConnect::Exceptions::InvalidCredentialSet => ex
|
602
523
|
raise
|
603
524
|
rescue => ex
|
604
525
|
ZuoraConnect.logger.error("Build Task Error", ex)
|
@@ -654,12 +575,12 @@ module ZuoraConnect
|
|
654
575
|
self.refresh if !defined?(self.target_login)
|
655
576
|
|
656
577
|
response = HTTParty.get("#{ZuoraConnect.configuration.url}/api/#{self.api_version}/tenants/search?hostname=#{self.target_login.client.hostname}&node_id=#{self.zuora_entity_ids.first}")
|
657
|
-
|
578
|
+
|
658
579
|
if response.success?
|
659
580
|
parsed_json = JSON.parse(response.body)
|
660
|
-
|
581
|
+
|
661
582
|
#Set Org
|
662
|
-
if self.
|
583
|
+
if self.auto_deployed? && parsed_json['organization'].present?
|
663
584
|
login_cache = self.zuora_logins
|
664
585
|
login_cache.delete('organization')
|
665
586
|
self.zuora_logins = login_cache.merge({'organizations' => [parsed_json['organization']]})
|
@@ -679,7 +600,7 @@ module ZuoraConnect
|
|
679
600
|
end
|
680
601
|
end
|
681
602
|
self.save(:validate => false)
|
682
|
-
|
603
|
+
|
683
604
|
return parsed_json
|
684
605
|
end
|
685
606
|
rescue *(ZuoraAPI::Login::CONNECTION_EXCEPTIONS + ZuoraAPI::Login::CONNECTION_READ_EXCEPTIONS) => ex
|
@@ -733,7 +654,7 @@ module ZuoraConnect
|
|
733
654
|
end
|
734
655
|
|
735
656
|
def refresh_oauth
|
736
|
-
refresh_oauth_count ||= 0
|
657
|
+
refresh_oauth_count ||= 0
|
737
658
|
response = HTTParty.post("#{ZuoraConnect.configuration.url}/oauth/token", body: {
|
738
659
|
:grant_type => "refresh_token",
|
739
660
|
:redirect_uri => ZuoraConnect.configuration.oauth_client_redirect_uri,
|
@@ -1369,7 +1290,7 @@ module ZuoraConnect
|
|
1369
1290
|
|
1370
1291
|
def self.without_sticking
|
1371
1292
|
if self.connection.respond_to?(:without_sticking)
|
1372
|
-
self.connection.without_sticking do
|
1293
|
+
self.connection.without_sticking do
|
1373
1294
|
yield
|
1374
1295
|
end
|
1375
1296
|
else
|
@@ -49,10 +49,10 @@ if defined? Prometheus
|
|
49
49
|
prometheus = Prometheus::Client.registry
|
50
50
|
|
51
51
|
# Create your metrics.
|
52
|
-
ZUORA_VERSION = prometheus.gauge(:zuora_version, docstring: 'The current Zuora Gem version.', labels: %i(version name), preset_labels: { version: ZuoraAPI::VERSION, name:
|
53
|
-
CONNECT_VERSION = prometheus.gauge(:gem_version, docstring: 'The current Connect Gem version.', labels: %i(version name), preset_labels: { version: ZuoraConnect::VERSION, name:
|
54
|
-
RAILS_VERSION = prometheus.gauge(:rails_version, docstring: 'The current Rails version.', labels: %i(version name), preset_labels: { version: Rails.version, name:
|
55
|
-
RUBY_V = prometheus.gauge(:ruby_version, docstring: 'The current Ruby version.', labels: %i(version name), preset_labels: { version: RUBY_VERSION, name:
|
52
|
+
ZUORA_VERSION = prometheus.gauge(:zuora_version, docstring: 'The current Zuora Gem version.', labels: %i(version name), preset_labels: { version: ZuoraAPI::VERSION, name: ZuoraObservability::Env.app_name }, store_settings: most_recent_aggregation)
|
53
|
+
CONNECT_VERSION = prometheus.gauge(:gem_version, docstring: 'The current Connect Gem version.', labels: %i(version name), preset_labels: { version: ZuoraConnect::VERSION, name: ZuoraObservability::Env.app_name }, store_settings: most_recent_aggregation)
|
54
|
+
RAILS_VERSION = prometheus.gauge(:rails_version, docstring: 'The current Rails version.', labels: %i(version name), preset_labels: { version: Rails.version, name: ZuoraObservability::Env.app_name }, store_settings: most_recent_aggregation)
|
55
|
+
RUBY_V = prometheus.gauge(:ruby_version, docstring: 'The current Ruby version.', labels: %i(version name), preset_labels: { version: RUBY_VERSION, name: ZuoraObservability::Env.app_name }, store_settings: most_recent_aggregation)
|
56
56
|
|
57
57
|
ZUORA_VERSION.set(0)
|
58
58
|
CONNECT_VERSION.set(0)
|
@@ -61,12 +61,12 @@ if defined? Prometheus
|
|
61
61
|
|
62
62
|
# Do they have resque jobs?
|
63
63
|
if defined? Resque.redis
|
64
|
-
REDIS_CONNECTION = prometheus.gauge(:redis_connection, docstring: 'The status of the redis connection, 0 or 1', labels: %i(connection name), preset_labels: {connection:'redis', name:
|
65
|
-
JOBS_FINISHED = prometheus.gauge(:jobs_finished, docstring: 'Done resque jobs', labels: %i(type name), preset_labels: {type:'resque', name:
|
66
|
-
WORKERS_TOTAL = prometheus.gauge(:workers_total, docstring: 'Total resque workers', labels: %i(type name), preset_labels: {type:'resque', name:
|
67
|
-
WORKERS_ACTIVE = prometheus.gauge(:workers_active, docstring: 'Active resque workers', labels: %i(type name), preset_labels: {type:'resque', name:
|
68
|
-
JOBS_FAILED = prometheus.gauge(:jobs_failed, docstring: 'Failed resque jobs', labels: %i(type name), preset_labels: {type:'resque', name:
|
69
|
-
JOBS_PENDING = prometheus.gauge(:jobs_pending, docstring: 'Pending resque jobs', labels: %i(type name), preset_labels: {type:'resque', name:
|
64
|
+
REDIS_CONNECTION = prometheus.gauge(:redis_connection, docstring: 'The status of the redis connection, 0 or 1', labels: %i(connection name), preset_labels: {connection:'redis', name: ZuoraObservability::Env.app_name}, store_settings: most_recent_aggregation)
|
65
|
+
JOBS_FINISHED = prometheus.gauge(:jobs_finished, docstring: 'Done resque jobs', labels: %i(type name), preset_labels: {type:'resque', name: ZuoraObservability::Env.app_name}, store_settings: most_recent_aggregation)
|
66
|
+
WORKERS_TOTAL = prometheus.gauge(:workers_total, docstring: 'Total resque workers', labels: %i(type name), preset_labels: {type:'resque', name: ZuoraObservability::Env.app_name}, store_settings: most_recent_aggregation)
|
67
|
+
WORKERS_ACTIVE = prometheus.gauge(:workers_active, docstring: 'Active resque workers', labels: %i(type name), preset_labels: {type:'resque', name: ZuoraObservability::Env.app_name}, store_settings: most_recent_aggregation)
|
68
|
+
JOBS_FAILED = prometheus.gauge(:jobs_failed, docstring: 'Failed resque jobs', labels: %i(type name), preset_labels: {type:'resque', name: ZuoraObservability::Env.app_name}, store_settings: most_recent_aggregation)
|
69
|
+
JOBS_PENDING = prometheus.gauge(:jobs_pending, docstring: 'Pending resque jobs', labels: %i(type name), preset_labels: {type:'resque', name: ZuoraObservability::Env.app_name}, store_settings: most_recent_aggregation)
|
70
70
|
end
|
71
71
|
|
72
72
|
if defined?(Unicorn) && Unicorn.respond_to?(:listener_names)
|
@@ -74,17 +74,17 @@ if defined? Prometheus
|
|
74
74
|
:unicorn_kills,
|
75
75
|
docstring: 'Unicorn Kills',
|
76
76
|
labels: %i(type name),
|
77
|
-
preset_labels: {type:'Unicorn-Killer', name:
|
77
|
+
preset_labels: {type:'Unicorn-Killer', name: ZuoraObservability::Env.app_name},
|
78
78
|
store_settings: sum_aggregation
|
79
79
|
)
|
80
80
|
|
81
|
-
|
81
|
+
ZuoraObservability::Metrics.unicorn_listener.each do |key, _|
|
82
82
|
gauge_name = "unicorn_#{key}".gsub(/[^a-zA-Z0-9_]/, '_')
|
83
83
|
gauge = prometheus.gauge(
|
84
84
|
gauge_name.to_sym,
|
85
85
|
docstring: 'Unicorn Stats',
|
86
86
|
labels: %i(type name),
|
87
|
-
preset_labels: { type: 'unicorn', name:
|
87
|
+
preset_labels: { type: 'unicorn', name: ZuoraObservability::Env.app_name },
|
88
88
|
store_settings: most_recent_aggregation
|
89
89
|
)
|
90
90
|
Prometheus.const_set(
|
@@ -12,21 +12,21 @@ class RedisFlash
|
|
12
12
|
end
|
13
13
|
|
14
14
|
if defined?(Redis.current)
|
15
|
-
Redis.current = Redis.new(:id => "#{
|
15
|
+
Redis.current = Redis.new(:id => "#{ZuoraObservability::Env.full_process_name(process_name: 'Redis')}", :url => redis_url, :timeout => 6, :reconnect_attempts => 2)
|
16
16
|
browser_urls['Redis'] = { "url" => redis_url }
|
17
17
|
if defined?(Resque.redis)
|
18
18
|
if resque_url != redis_url
|
19
|
-
Resque.redis = Redis.new(:id => "#{
|
19
|
+
Resque.redis = Redis.new(:id => "#{ZuoraObservability::Env.full_process_name(process_name: 'Resque')}", :url => resque_url, :timeout => 6, :reconnect_attempts => 2)
|
20
20
|
browser_urls['Resque'] = { "url" => resque_url }
|
21
21
|
else
|
22
22
|
Resque.redis = Redis.current
|
23
23
|
end
|
24
24
|
end
|
25
25
|
if defined?(flash_url.present?)
|
26
|
-
RedisFlash.current = Redis.new(:id => "#{
|
26
|
+
RedisFlash.current = Redis.new(:id => "#{ZuoraObservability::Env.full_process_name(process_name: 'Flash')}", :url => flash_url, :timeout => 6, :reconnect_attempts => 2)
|
27
27
|
browser_urls['Flash'] = { "url" => flash_url }
|
28
28
|
end
|
29
29
|
end
|
30
30
|
if defined?(RedisBrowser)
|
31
31
|
RedisBrowser.configure("connections" => browser_urls)
|
32
|
-
end
|
32
|
+
end
|
@@ -34,15 +34,3 @@ Resque.module_eval do
|
|
34
34
|
Hash[queue_names.zip(sizes)]
|
35
35
|
end
|
36
36
|
end
|
37
|
-
|
38
|
-
if defined?(Resque.logger)
|
39
|
-
Resque.logger = ZuoraConnect.custom_logger(name: "Resque", type: 'Monologger', level: MonoLogger::INFO)
|
40
|
-
Resque::Scheduler.logger = ZuoraConnect.custom_logger(name: "ResqueScheduler") if defined?(Resque::Scheduler)
|
41
|
-
end
|
42
|
-
if defined?(Delayed::Worker.logger)
|
43
|
-
Delayed::Worker.logger = ZuoraConnect.custom_logger(name: "DelayedJob", type: 'Monologger', level: MonoLogger::INFO)
|
44
|
-
end
|
45
|
-
|
46
|
-
Makara::Logging::Logger.logger = ZuoraConnect.custom_logger(name: "Makara") if defined?(Makara)
|
47
|
-
ElasticAPM.agent.config.logger = ZuoraConnect.custom_logger(name: "ElasticAPM", level: MonoLogger::WARN) if defined?(ElasticAPM) && ElasticAPM.running?
|
48
|
-
ActionMailer::Base.logger = ZuoraConnect.custom_logger(name: "ActionMailer", type: 'Monologger') if defined?(ActionMailer)
|
@@ -6,7 +6,7 @@ if defined?(Unicorn::WorkerKiller)
|
|
6
6
|
if defined?(Prometheus)
|
7
7
|
Prometheus::UNICORN_KILLS.set(1)
|
8
8
|
else
|
9
|
-
|
9
|
+
ZuoraObservability::Metrics.write_to_telegraf(direction: 'Unicorn-Killer', tags: {app_instance: 0}, values: {kill: 1})
|
10
10
|
end
|
11
11
|
end
|
12
12
|
end
|
@@ -34,4 +34,4 @@ if defined?(Unicorn::HttpServer) && defined?(Prometheus)
|
|
34
34
|
class Unicorn::Worker
|
35
35
|
prepend WorkerExtensions
|
36
36
|
end
|
37
|
-
end
|
37
|
+
end
|
data/config/routes.rb
CHANGED
data/lib/metrics/net.rb
CHANGED
@@ -68,12 +68,12 @@ class HttpLogger
|
|
68
68
|
log_request_body(request)
|
69
69
|
log_request_headers(request)
|
70
70
|
if defined?(response) && response
|
71
|
-
tags = tags.merge({status: response.code.to_i})
|
71
|
+
tags = tags.merge({status: response.code.to_i})
|
72
72
|
log_response_code(response)
|
73
73
|
log_response_headers(response)
|
74
74
|
log_response_body(response.body)
|
75
75
|
end
|
76
|
-
|
76
|
+
ZuoraObservability::Metrics.write_to_telegraf(direction: :outbound, tags: tags, values: values)
|
77
77
|
end
|
78
78
|
end
|
79
79
|
|
@@ -95,7 +95,7 @@ class HttpLogger
|
|
95
95
|
end
|
96
96
|
|
97
97
|
HTTP_METHODS_WITH_BODY = Set.new(%w(POST PUT GET PATCH))
|
98
|
-
|
98
|
+
|
99
99
|
def log_request_body(request)
|
100
100
|
if self.class.log_request_body
|
101
101
|
if HTTP_METHODS_WITH_BODY.include?(request.method)
|
@@ -149,8 +149,8 @@ class HttpLogger
|
|
149
149
|
def truncate_body(body)
|
150
150
|
if collapse_body_limit && collapse_body_limit > 0 && body && body.size >= collapse_body_limit
|
151
151
|
body_piece_size = collapse_body_limit / 2
|
152
|
-
body[0..body_piece_size] +
|
153
|
-
"\n\n<some data truncated>\n\n" +
|
152
|
+
body[0..body_piece_size] +
|
153
|
+
"\n\n<some data truncated>\n\n" +
|
154
154
|
body[(body.size - body_piece_size)..body.size]
|
155
155
|
else
|
156
156
|
body
|
@@ -203,7 +203,7 @@ class Net::HTTP
|
|
203
203
|
|
204
204
|
def request(request, body = nil, &block)
|
205
205
|
HttpLogger.perform(self, request, body) do
|
206
|
-
request_without_logging(request, body, &block)
|
206
|
+
request_without_logging(request, body, &block)
|
207
207
|
end
|
208
208
|
end
|
209
209
|
end
|
@@ -215,4 +215,4 @@ if defined?(Rails)
|
|
215
215
|
HttpLogger.logger = ZuoraConnect.logger unless HttpLogger.logger
|
216
216
|
end
|
217
217
|
end
|
218
|
-
end
|
218
|
+
end
|
@@ -1,37 +1,6 @@
|
|
1
1
|
module ZuoraConnect
|
2
2
|
require 'uri'
|
3
3
|
|
4
|
-
# Object of this class is passed to the ActiveSupport::Notification hook
|
5
|
-
class PageRequest
|
6
|
-
|
7
|
-
# This method is triggered when a non error page is loaded (not 404)
|
8
|
-
def call(name, started, finished, unique_id, payload)
|
9
|
-
# If the url contains any css or JavaScript files then do not collect metrics for them
|
10
|
-
return nil if ["css", "assets", "jpg", "png", "jpeg", "ico"].any? { |word| payload[:path].include?(word) }
|
11
|
-
|
12
|
-
# Getting the endpoint and the content_type
|
13
|
-
content_hash = {:html => "text/html", :js => "application/javascript", :json => "application/json", :csv => "text/csv"}
|
14
|
-
content_type = content_hash.key?(payload[:format]) ? content_hash[payload[:format]] : payload[:format]
|
15
|
-
content_type = content_type.to_s.gsub('text/javascript', 'application/javascript')
|
16
|
-
|
17
|
-
# payloads with 500 requests do not have status as it is not set by the controller
|
18
|
-
# https://github.com/rails/rails/issues/33335
|
19
|
-
#status_code = payload[:status] ? payload[:status] : payload[:exception_object].present? ? 500 : ""
|
20
|
-
if payload[:exception].present?
|
21
|
-
status_code, exception = [500, payload[:exception].first]
|
22
|
-
else
|
23
|
-
status_code, exception = [payload[:status], nil]
|
24
|
-
end
|
25
|
-
|
26
|
-
tags = {method: payload[:method], status: status_code, error_type: exception, content_type: content_type, controller: payload[:controller], action: payload[:action]}.compact
|
27
|
-
|
28
|
-
values = {view_time: payload[:view_runtime], db_time: payload[:db_runtime], response_time: ((finished-started)*1000)}.compact
|
29
|
-
values = values.map{ |k,v| [k,v.round(2)]}.to_h
|
30
|
-
|
31
|
-
ZuoraConnect::AppInstanceBase.write_to_telegraf(direction: :inbound, tags: tags, values: values)
|
32
|
-
end
|
33
|
-
end
|
34
|
-
|
35
4
|
class MetricsMiddleware
|
36
5
|
|
37
6
|
require "zuora_connect/version"
|
@@ -42,7 +11,7 @@ module ZuoraConnect
|
|
42
11
|
end
|
43
12
|
|
44
13
|
def call(env)
|
45
|
-
@bad_headers = ["HTTP_X_FORWARDED_FOR", "HTTP_X_FORWARDED_HOST", "HTTP_X_FORWARDED_PORT", "HTTP_X_FORWARDED_PROTO", "HTTP_X_FORWARDED_SCHEME", "HTTP_X_FORWARDED_SSL"]
|
14
|
+
@bad_headers = ["HTTP_X_FORWARDED_FOR", "HTTP_X_FORWARDED_HOST", "HTTP_X_FORWARDED_PORT", "HTTP_X_FORWARDED_PROTO", "HTTP_X_FORWARDED_SCHEME", "HTTP_X_FORWARDED_SSL"]
|
46
15
|
if !ActionDispatch::Request::HTTP_METHODS.include?(env["REQUEST_METHOD"].upcase)
|
47
16
|
[405, {"Content-Type" => "text/plain"}, ["Method Not Allowed"]]
|
48
17
|
else
|
@@ -66,14 +35,14 @@ module ZuoraConnect
|
|
66
35
|
|
67
36
|
if defined?(Prometheus) && env['PATH_INFO'] == '/connect/internal/metrics'
|
68
37
|
# Prometheus Stuff
|
69
|
-
metrics =
|
38
|
+
metrics = ZuoraObservability::Metrics.resque
|
70
39
|
redis_up = metrics.present? && metrics.dig(:Resque, :Workers_Total).present? ? 1 : 0
|
71
40
|
Prometheus::REDIS_CONNECTION.set(redis_up)
|
72
41
|
|
73
42
|
process_prometheus_metric(metrics: metrics)
|
74
43
|
|
75
44
|
if defined?(Unicorn) && Unicorn.respond_to?(:listener_names)
|
76
|
-
|
45
|
+
ZuoraObservability::Metrics.unicorn_listener.each do |key, value|
|
77
46
|
gauge = Prometheus.const_get("unicorn_#{key}".gsub(/[^a-zA-Z0-9_]/, '_').upcase)
|
78
47
|
gauge.set(value) if gauge.present?
|
79
48
|
end
|
@@ -84,14 +53,7 @@ module ZuoraConnect
|
|
84
53
|
start_time = Time.now
|
85
54
|
begin
|
86
55
|
@status, @headers, @response = @app.call(env)
|
87
|
-
ensure
|
88
|
-
|
89
|
-
# If the url contains any CSS or JavaScript files then do not collect metrics for them
|
90
|
-
if ["css", "assets", "jpg", "png", "jpeg", "ico"].any? { |word| env['PATH_INFO'].include?(word) } || /.*\.js$/.match(env['PATH_INFO'])
|
91
|
-
tags = {status: @status, controller: 'ActionController', action: 'Assets', app_instance: 0}
|
92
|
-
values = {response_time: ((Time.now - start_time)*1000).round(2) }
|
93
|
-
ZuoraConnect::AppInstanceBase.write_to_telegraf(direction: 'request-inbound-assets', tags: tags, values: values)
|
94
|
-
end
|
56
|
+
ensure
|
95
57
|
|
96
58
|
# Uncomment following block of code for handling engine requests/requests without controller
|
97
59
|
# else
|
@@ -112,7 +74,7 @@ module ZuoraConnect
|
|
112
74
|
|
113
75
|
values = {response_time: ((Time.now - start_time)*1000).round(2) }
|
114
76
|
|
115
|
-
|
77
|
+
ZuoraObservability::Metrics.write_to_telegraf(direction: :inbound, tags: tags, values: values)
|
116
78
|
end
|
117
79
|
end
|
118
80
|
Thread.current[:inbound_metric] = nil
|
@@ -140,7 +102,7 @@ module ZuoraConnect
|
|
140
102
|
gauge_name.to_sym,
|
141
103
|
docstring: "#{key} metric",
|
142
104
|
labels: %i(type name),
|
143
|
-
preset_labels: { type: type, name:
|
105
|
+
preset_labels: { type: type, name: ZuoraObservability::Env.app_name },
|
144
106
|
store_settings: most_recent_aggregation
|
145
107
|
)
|
146
108
|
gauge.set(value)
|
data/lib/zuora_connect.rb
CHANGED
@@ -9,12 +9,11 @@ require 'resque/silence_done'
|
|
9
9
|
require 'resque/self_lookup'
|
10
10
|
require 'resque/plugins/custom_logger'
|
11
11
|
require 'resque/plugins/app_instance_job'
|
12
|
-
require 'logging/connect_formatter'
|
13
|
-
require 'metrics/influx/point_value'
|
14
12
|
require 'metrics/net'
|
15
13
|
require 'mono_logger'
|
16
14
|
require 'zuora_connect/zuora_audit'
|
17
15
|
require 'active_record'
|
16
|
+
require 'zuora_observability'
|
18
17
|
::ActiveRecord::Base.send :include, ZuoraConnect::ZuoraAudit
|
19
18
|
|
20
19
|
module ZuoraConnect
|
@@ -29,68 +28,9 @@ module ZuoraConnect
|
|
29
28
|
when 'test'
|
30
29
|
Rails.logger
|
31
30
|
else
|
32
|
-
@logger ||= custom_logger(name: "Connect", level: Rails.logger.level)
|
31
|
+
@logger ||= ZuoraObservability::Logger.custom_logger(name: "Connect", level: Rails.logger.level)
|
33
32
|
end
|
34
33
|
end
|
35
|
-
|
36
|
-
def custom_logger(name: "", level: Rails.logger.present? ? Rails.logger.level : MonoLogger::INFO, type: :ougai)
|
37
|
-
#puts name + ' - ' + {Logger::WARN => 'Logger::WARN', Logger::ERROR => 'Logger::ERROR', Logger::DEBUG => 'Logger::DEBUG', Logger::INFO => 'Logger::INFO' }[level] + ' - '
|
38
|
-
if type == :ougai
|
39
|
-
require 'ougai'
|
40
|
-
require "ougai/formatters/customizable"
|
41
|
-
#logger = Ougai::Logger.new(MonoLogger.new(STDOUT))
|
42
|
-
logger = Ougai::Logger.new(STDOUT)
|
43
|
-
logger.level = level
|
44
|
-
if ZuoraConnect.configuration.json_logging
|
45
|
-
logger.formatter = Ougai::Formatters::ConnectFormatter.new(name)
|
46
|
-
logger.before_log = lambda do |data|
|
47
|
-
data[:trace_id] = ZuoraConnect::RequestIdMiddleware.request_id if ZuoraConnect::RequestIdMiddleware.request_id.present?
|
48
|
-
data[:zuora_trace_id] = ZuoraConnect::RequestIdMiddleware.zuora_request_id if ZuoraConnect::RequestIdMiddleware.zuora_request_id.present?
|
49
|
-
#data[:traces] = {amazon_id: data[:trace_id], zuora_id: data[:zuora_trace_id]}
|
50
|
-
end
|
51
|
-
else
|
52
|
-
logger.formatter = Ougai::Formatters::Customizable.new(
|
53
|
-
format_err: proc do |data|
|
54
|
-
next nil unless data.key?(:err)
|
55
|
-
err = data.delete(:err)
|
56
|
-
" #{err[:name]} (#{err[:message]})\n #{err[:stack]}"
|
57
|
-
end,
|
58
|
-
format_data: proc do |data|
|
59
|
-
data.delete(:app_instance_id); data.delete(:tenant_ids); data.delete(:organization); data.delete(:environment)
|
60
|
-
format('%s %s: %s', 'DATA'.ljust(6), Time.now.strftime('%FT%T.%6NZ'), "#{data.to_json}") if data.present?
|
61
|
-
end,
|
62
|
-
format_msg: proc do |severity, datetime, _progname, data|
|
63
|
-
msg = data.delete(:msg)
|
64
|
-
format('%s %s: %s', severity.ljust(6), datetime, msg)
|
65
|
-
end
|
66
|
-
)
|
67
|
-
logger.formatter.datetime_format = '%FT%T.%6NZ'
|
68
|
-
end
|
69
|
-
else
|
70
|
-
logger = MonoLogger.new(STDOUT)
|
71
|
-
logger.level = level
|
72
|
-
logger.formatter = proc do |serverity, datetime, progname, msg|
|
73
|
-
begin
|
74
|
-
msg = JSON.parse(msg)
|
75
|
-
rescue JSON::ParserError => ex
|
76
|
-
end
|
77
|
-
if ZuoraConnect.configuration.json_logging
|
78
|
-
require 'json'
|
79
|
-
store = {
|
80
|
-
name: name,
|
81
|
-
level: serverity,
|
82
|
-
timestamp: datetime.strftime('%FT%T.%6NZ'),
|
83
|
-
pid: Process.pid,
|
84
|
-
message: name == "ActionMailer" ? msg.strip : msg
|
85
|
-
}
|
86
|
-
JSON.dump(store) + "\n"
|
87
|
-
else
|
88
|
-
format('%s %s: %s', serverity.ljust(6), datetime, msg) + "\n"
|
89
|
-
end
|
90
|
-
end
|
91
|
-
end
|
92
|
-
return logger
|
93
|
-
end
|
94
34
|
end
|
95
35
|
|
96
36
|
module Controllers
|
@@ -147,7 +87,7 @@ module ZuoraConnect
|
|
147
87
|
verify_server_cert: false,
|
148
88
|
log_level: Logger::INFO,
|
149
89
|
service_name: ENV['DEIS_APP'].present? ? ENV['DEIS_APP'] : Rails.application.class.parent_name,
|
150
|
-
logger:
|
90
|
+
logger: ZuoraObservability::Logger.custom_logger(name: "ElasticAPM", level: MonoLogger::WARN)
|
151
91
|
})
|
152
92
|
defaults.merge!({disable_send: true}) if defined?(Rails::Console)
|
153
93
|
|
@@ -677,10 +677,17 @@ module ZuoraConnect
|
|
677
677
|
|
678
678
|
def setup_instance_via_dev_mode
|
679
679
|
session["appInstance"] = ZuoraConnect.configuration.dev_mode_appinstance
|
680
|
-
|
681
|
-
|
682
|
-
values = {
|
683
|
-
|
680
|
+
session["#{ZuoraConnect.configuration.dev_mode_appinstance}::admin"] = ZuoraConnect.configuration.dev_mode_admin
|
681
|
+
|
682
|
+
values = {
|
683
|
+
id: ZuoraConnect.configuration.dev_mode_appinstance,
|
684
|
+
access_token: ZuoraConnect.configuration.dev_mode_user,
|
685
|
+
refresh_token: ZuoraConnect.configuration.dev_mode_pass,
|
686
|
+
token: ZuoraConnect.configuration.dev_mode_pass+ZuoraConnect.configuration.dev_mode_pass,
|
687
|
+
api_token: ZuoraConnect.configuration.dev_mode_pass+ZuoraConnect.configuration.dev_mode_pass
|
688
|
+
}
|
689
|
+
|
690
|
+
@appinstance = ZuoraConnect::AppInstance.find_by(:id => ZuoraConnect.configuration.dev_mode_appinstance.to_i)
|
684
691
|
ZuoraConnect::ZuoraUser.current_user_id = 0
|
685
692
|
if @appinstance.blank?
|
686
693
|
Apartment::Tenant.switch!("public")
|
@@ -690,14 +697,10 @@ module ZuoraConnect
|
|
690
697
|
Apartment::Tenant.drop(values[:appinstance].to_s)
|
691
698
|
retry
|
692
699
|
end
|
693
|
-
|
694
|
-
@appinstance = ZuoraConnect::AppInstance.new(:id => values[:appinstance].to_i, :access_token => values[:user], :refresh_token => values[:key], :token => "#{values[:key]}#{values[:key]}", :api_token => "#{values[:key]}#{values[:key]}")
|
695
|
-
@appinstance.save(:validate => false)
|
696
|
-
end
|
697
|
-
if @appinstance.access_token.blank? || @appinstance.refresh_token.blank? || @appinstance.token.blank? || @appinstance.api_token.blank?
|
698
|
-
@appinstance.update_attributes!(:access_token => values["user"], :refresh_token => values["key"], :token => "#{values[:key]}#{values[:key]}", :api_token => "#{values[:key]}#{values[:key]}")
|
700
|
+
@appinstance = ZuoraConnect::AppInstance.new()
|
699
701
|
end
|
700
|
-
|
702
|
+
@appinstance.assign_attributes(values)
|
703
|
+
@appinstance.save(:validate => false)
|
701
704
|
end
|
702
705
|
end
|
703
706
|
end
|
data/lib/zuora_connect/engine.rb
CHANGED
@@ -10,9 +10,10 @@ module ZuoraConnect
|
|
10
10
|
|
11
11
|
initializer "connect", before: :load_config_initializers do |app|
|
12
12
|
Rails.application.routes.prepend do
|
13
|
+
get '/connect/internal/data' => 'zuora_observability/metrics#metrics'
|
13
14
|
mount ZuoraConnect::Engine, at: "/connect"
|
14
15
|
match '/api/connect/health', via: :all, to: 'zuora_connect/static#health'
|
15
|
-
match '/api/connect/internal/data', via: :all, to: '
|
16
|
+
match '/api/connect/internal/data', via: :all, to: 'zuora_observability/metrics#metrics'
|
16
17
|
end
|
17
18
|
end
|
18
19
|
|
@@ -5,17 +5,6 @@ require 'middleware/bad_multipart_form_data_sanitizer'
|
|
5
5
|
|
6
6
|
module ZuoraConnect
|
7
7
|
class Railtie < Rails::Railtie
|
8
|
-
REQUEST_HEADERS_TO_IGNORE = %W(
|
9
|
-
RAW_POST_DATA
|
10
|
-
REQUEST_METHOD
|
11
|
-
REQUEST_URI
|
12
|
-
REQUEST_PATH
|
13
|
-
PATH_INFO
|
14
|
-
CONTENT_TYPE
|
15
|
-
ORIGINAL_FULLPATH
|
16
|
-
QUERY_STRING
|
17
|
-
)
|
18
|
-
|
19
8
|
config.before_initialize do
|
20
9
|
version = Rails.version
|
21
10
|
if version >= "5.0.0"
|
@@ -43,71 +32,5 @@ module ZuoraConnect
|
|
43
32
|
app.config.middleware.use Rack::Deflater, if: ->(env, *) { env['PATH_INFO'] == '/connect/internal/metrics' }
|
44
33
|
end
|
45
34
|
end
|
46
|
-
|
47
|
-
# hook to process_action
|
48
|
-
ActiveSupport::Notifications.subscribe('process_action.action_controller', ZuoraConnect::PageRequest.new)
|
49
|
-
|
50
|
-
initializer(:rails_stdout_logging, before: :initialize_logger) do
|
51
|
-
require 'lograge'
|
52
|
-
|
53
|
-
Rails.configuration.logger = ZuoraConnect.custom_logger(name: "Rails")
|
54
|
-
if !Rails.env.test? && !Rails.env.development?
|
55
|
-
Rails.configuration.lograge.enabled = true
|
56
|
-
Rails.configuration.colorize_logging = false
|
57
|
-
end
|
58
|
-
|
59
|
-
if Rails.configuration.lograge.enabled
|
60
|
-
if Rails.configuration.logger.class.to_s == 'Ougai::Logger'
|
61
|
-
Rails.configuration.lograge.formatter = Class.new do |fmt|
|
62
|
-
def fmt.call(data)
|
63
|
-
{ msg: 'Rails Request', request: data }
|
64
|
-
end
|
65
|
-
end
|
66
|
-
end
|
67
|
-
#Rails.configuration.lograge.formatter = Lograge::Formatters::Json.new
|
68
|
-
Rails.configuration.lograge.custom_options = lambda do |event|
|
69
|
-
exceptions = %w(controller action format)
|
70
|
-
items = {
|
71
|
-
#time: event.time.strftime('%FT%T.%6N'),
|
72
|
-
params: event.payload[:params].as_json(except: exceptions).to_json.to_s
|
73
|
-
}
|
74
|
-
items.merge!({exception_object: event.payload[:exception_object]}) if event.payload[:exception_object].present?
|
75
|
-
items.merge!({exception: event.payload[:exception]}) if event.payload[:exception].present?
|
76
|
-
|
77
|
-
if event.payload[:headers].present?
|
78
|
-
# By convertion, headers usually do not have dots. Nginx even rejects headers with dots
|
79
|
-
# All Rails headers are namespaced, like 'rack.input'.
|
80
|
-
# Thus, we can obtain the client headers by rejecting dots
|
81
|
-
request_headers =
|
82
|
-
event.payload[:headers].env.
|
83
|
-
reject { |key| key.to_s.include?('.') || REQUEST_HEADERS_TO_IGNORE.include?(key.to_s) }
|
84
|
-
begin
|
85
|
-
if request_headers["HTTP_AUTHORIZATION"].present?
|
86
|
-
if request_headers["HTTP_AUTHORIZATION"].include?("Basic")
|
87
|
-
user_password = request_headers["HTTP_AUTHORIZATION"].split("Basic").last.strip
|
88
|
-
user, password = Base64.decode64(user_password).split(":")
|
89
|
-
request_headers["HTTP_AUTHORIZATION"] = "Basic #{user}:ValueFiltered"
|
90
|
-
elsif
|
91
|
-
request_headers["HTTP_AUTHORIZATION"] = "ValueFiltered"
|
92
|
-
end
|
93
|
-
end
|
94
|
-
request_headers["HTTP_API_TOKEN"] = "ValueFiltered" if request_headers["HTTP_API_TOKEN"].present?
|
95
|
-
rescue
|
96
|
-
request_headers.delete("HTTP_API_TOKEN")
|
97
|
-
request_headers.delete("HTTP_AUTHORIZATION")
|
98
|
-
end
|
99
|
-
items.merge!({ headers: request_headers.to_s })
|
100
|
-
end
|
101
|
-
|
102
|
-
if Thread.current[:appinstance].present?
|
103
|
-
items.merge!({connect_user: Thread.current[:appinstance].connect_user, new_session: Thread.current[:appinstance].new_session_message})
|
104
|
-
if Thread.current[:appinstance].logitems.present? && Thread.current[:appinstance].logitems.class == Hash
|
105
|
-
items.merge!(Thread.current[:appinstance].logitems)
|
106
|
-
end
|
107
|
-
end
|
108
|
-
return items
|
109
|
-
end
|
110
|
-
end
|
111
|
-
end
|
112
35
|
end
|
113
36
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: zuora_connect
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 3.0.0.pre.c
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Connect Team
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-12-03 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: apartment
|
@@ -182,6 +182,20 @@ dependencies:
|
|
182
182
|
- - ">="
|
183
183
|
- !ruby/object:Gem::Version
|
184
184
|
version: '0'
|
185
|
+
- !ruby/object:Gem::Dependency
|
186
|
+
name: zuora_observability
|
187
|
+
requirement: !ruby/object:Gem::Requirement
|
188
|
+
requirements:
|
189
|
+
- - '='
|
190
|
+
- !ruby/object:Gem::Version
|
191
|
+
version: 0.1.0.pre.b
|
192
|
+
type: :runtime
|
193
|
+
prerelease: false
|
194
|
+
version_requirements: !ruby/object:Gem::Requirement
|
195
|
+
requirements:
|
196
|
+
- - '='
|
197
|
+
- !ruby/object:Gem::Version
|
198
|
+
version: 0.1.0.pre.b
|
185
199
|
- !ruby/object:Gem::Dependency
|
186
200
|
name: rspec
|
187
201
|
requirement: !ruby/object:Gem::Requirement
|
@@ -318,7 +332,6 @@ files:
|
|
318
332
|
- app/models/zuora_connect/app_instance.rb
|
319
333
|
- app/models/zuora_connect/app_instance_base.rb
|
320
334
|
- app/models/zuora_connect/login.rb
|
321
|
-
- app/models/zuora_connect/telegraf.rb
|
322
335
|
- app/models/zuora_connect/zuora_user.rb
|
323
336
|
- app/views/layouts/zuora_connect/application.html.erb
|
324
337
|
- app/views/sql/refresh_aggregate_table.txt
|
@@ -353,8 +366,6 @@ files:
|
|
353
366
|
- db/migrate/20190520232222_add_unique_index.rb
|
354
367
|
- db/migrate/20190520232223_add_provisioning_fields.rb
|
355
368
|
- db/migrate/20190520232224_add_environment_fields.rb
|
356
|
-
- lib/logging/connect_formatter.rb
|
357
|
-
- lib/metrics/influx/point_value.rb
|
358
369
|
- lib/metrics/net.rb
|
359
370
|
- lib/middleware/bad_multipart_form_data_sanitizer.rb
|
360
371
|
- lib/middleware/json_parse_errors.rb
|
@@ -1,97 +0,0 @@
|
|
1
|
-
module ZuoraConnect
|
2
|
-
class Telegraf
|
3
|
-
attr_accessor :host
|
4
|
-
|
5
|
-
OUTBOUND_METRICS = true
|
6
|
-
OUTBOUND_METRICS_NAME = "request-outbound"
|
7
|
-
INBOUND_METRICS = true
|
8
|
-
INBOUND_METRICS_NAME = "request-inbound"
|
9
|
-
|
10
|
-
def initialize
|
11
|
-
self.connect
|
12
|
-
end
|
13
|
-
|
14
|
-
def connect
|
15
|
-
ZuoraConnect.logger.debug(self.format_metric_log('Telegraf','Need new connection')) if ZuoraConnect.configuration.telegraf_debug
|
16
|
-
uri = URI.parse(ZuoraConnect.configuration.telegraf_endpoint)
|
17
|
-
self.host = UDPSocket.new.tap do |socket|
|
18
|
-
socket.connect uri.host, uri.port
|
19
|
-
end
|
20
|
-
rescue => ex
|
21
|
-
self.host = nil
|
22
|
-
ZuoraConnect.logger.warn(self.format_metric_log('Telegraf', "Failed to connect: #{ex.class}")) if Rails.env.to_s != 'production'
|
23
|
-
end
|
24
|
-
|
25
|
-
def write(direction: 'Unknown', tags: {}, values: {})
|
26
|
-
time = Benchmark.measure do |bench|
|
27
|
-
# To avoid writing metrics from rspec tests
|
28
|
-
if Rails.env.to_sym != :test
|
29
|
-
app_instance = Thread.current[:appinstance].present? ? Thread.current[:appinstance].id : 0
|
30
|
-
tags = { app_name: self.class.app_name, process_type: self.class.process_type, app_instance: app_instance, pod_name: self.class.pod_name}.merge(tags)
|
31
|
-
|
32
|
-
if direction == :inbound
|
33
|
-
if INBOUND_METRICS && !Thread.current[:inbound_metric].to_bool
|
34
|
-
self.write_udp(series: INBOUND_METRICS_NAME, tags: tags, values: values)
|
35
|
-
Thread.current[:inbound_metric] = true
|
36
|
-
else
|
37
|
-
return
|
38
|
-
end
|
39
|
-
elsif direction == :outbound
|
40
|
-
self.write_udp(series: OUTBOUND_METRICS_NAME, tags: tags, values: values) if OUTBOUND_METRICS
|
41
|
-
else
|
42
|
-
self.write_udp(series: direction, tags: tags, values: values)
|
43
|
-
end
|
44
|
-
end
|
45
|
-
end
|
46
|
-
if ZuoraConnect.configuration.telegraf_debug
|
47
|
-
ZuoraConnect.logger.debug(self.format_metric_log('Telegraf', tags.to_s))
|
48
|
-
ZuoraConnect.logger.debug(self.format_metric_log('Telegraf', values.to_s))
|
49
|
-
ZuoraConnect.logger.debug(self.format_metric_log('Telegraf', "Writing '#{direction.capitalize}': #{time.real.round(5)} ms"))
|
50
|
-
end
|
51
|
-
end
|
52
|
-
|
53
|
-
|
54
|
-
def write_udp(series: '', tags: {}, values: {})
|
55
|
-
return if !values.present?
|
56
|
-
self.host.write InfluxDB::PointValue.new({series: series, tags: tags, values: values}).dump
|
57
|
-
rescue => ex
|
58
|
-
self.connect
|
59
|
-
ZuoraConnect.logger.warn(self.format_metric_log('Telegraf',"Failed to write udp: #{ex.class}")) if Rails.env.to_s != 'production'
|
60
|
-
end
|
61
|
-
|
62
|
-
def format_metric_log(message, dump = nil)
|
63
|
-
message_color, dump_color = "1;91", "0;1"
|
64
|
-
log_entry = " \e[#{message_color}m#{message}\e[0m "
|
65
|
-
log_entry << "\e[#{dump_color}m%#{String === dump ? 's' : 'p'}\e[0m" % dump if dump
|
66
|
-
if Rails.env == :development
|
67
|
-
log_entry
|
68
|
-
else
|
69
|
-
[message, dump].compact.join(' - ')
|
70
|
-
end
|
71
|
-
end
|
72
|
-
|
73
|
-
def self.app_name
|
74
|
-
return ENV['DEIS_APP'].present? ? ENV['DEIS_APP'] : Rails.application.class.parent_name
|
75
|
-
end
|
76
|
-
|
77
|
-
def self.pod_name
|
78
|
-
return ENV['HOSTNAME'].present? ? ENV['HOSTNAME'] : Socket.gethostname
|
79
|
-
end
|
80
|
-
|
81
|
-
def self.full_process_name(process_name: nil, function: nil)
|
82
|
-
keys = [self.pod_name, process_name.present? ? process_name : self.process_type, Process.pid, function]
|
83
|
-
return keys.compact.join('][').prepend('[').concat(']')
|
84
|
-
end
|
85
|
-
|
86
|
-
# Returns the process type if any
|
87
|
-
def self.process_type(default: 'Unknown')
|
88
|
-
p_type = default
|
89
|
-
if ENV['HOSTNAME'] && ENV['DEIS_APP']
|
90
|
-
temp = ENV['HOSTNAME'].split(ENV['DEIS_APP'])[1]
|
91
|
-
temp = temp.split(/(-[0-9a-zA-Z]{5})$/)[0] # remove the 5 char hash
|
92
|
-
p_type = temp[1, temp.rindex("-")-1]
|
93
|
-
end
|
94
|
-
return p_type
|
95
|
-
end
|
96
|
-
end
|
97
|
-
end
|
@@ -1,44 +0,0 @@
|
|
1
|
-
require 'ougai/formatters/base'
|
2
|
-
require 'ougai/formatters/for_json'
|
3
|
-
|
4
|
-
module Ougai
|
5
|
-
module Formatters
|
6
|
-
# A JSON formatter compatible with node-bunyan
|
7
|
-
class ConnectFormatter < Base
|
8
|
-
include ForJson
|
9
|
-
|
10
|
-
# Intialize a formatter
|
11
|
-
# @param [String] app_name application name (execution program name if nil)
|
12
|
-
# @param [String] hostname hostname (hostname if nil)
|
13
|
-
# @param [Hash] opts the initial values of attributes
|
14
|
-
# @option opts [String] :trace_indent (2) the value of trace_indent attribute
|
15
|
-
# @option opts [String] :trace_max_lines (100) the value of trace_max_lines attribute
|
16
|
-
# @option opts [String] :serialize_backtrace (true) the value of serialize_backtrace attribute
|
17
|
-
# @option opts [String] :jsonize (true) the value of jsonize attribute
|
18
|
-
# @option opts [String] :with_newline (true) the value of with_newline attribute
|
19
|
-
def initialize(app_name = nil, hostname = nil, opts = {})
|
20
|
-
aname, hname, opts = Base.parse_new_params([app_name, hostname, opts])
|
21
|
-
super(aname, hname, opts)
|
22
|
-
init_opts_for_json(opts)
|
23
|
-
end
|
24
|
-
|
25
|
-
def _call(severity, time, progname, data)
|
26
|
-
data.merge!({message: data.delete(:msg)})
|
27
|
-
if data[:timestamp].present?
|
28
|
-
time = data[:timestamp]
|
29
|
-
data.delete(:timestamp)
|
30
|
-
end
|
31
|
-
dump({
|
32
|
-
name: progname || @app_name,
|
33
|
-
pid: $$,
|
34
|
-
level: severity,
|
35
|
-
timestamp: time.utc.strftime('%FT%T.%6NZ'),
|
36
|
-
}.merge(data))
|
37
|
-
end
|
38
|
-
|
39
|
-
def convert_time(data)
|
40
|
-
#data[:timestamp] = format_datetime(data[:time])
|
41
|
-
end
|
42
|
-
end
|
43
|
-
end
|
44
|
-
end
|
@@ -1,79 +0,0 @@
|
|
1
|
-
module InfluxDB
|
2
|
-
# Convert data point to string using Line protocol
|
3
|
-
class PointValue
|
4
|
-
attr_reader :series, :values, :tags, :timestamp
|
5
|
-
|
6
|
-
def initialize(data)
|
7
|
-
@series = escape data[:series], :measurement
|
8
|
-
@values = escape_values data[:values]
|
9
|
-
@tags = escape_tags data[:tags]
|
10
|
-
@timestamp = data[:timestamp]
|
11
|
-
end
|
12
|
-
|
13
|
-
def dump
|
14
|
-
dump = @series.dup
|
15
|
-
dump << ",#{@tags}" if @tags
|
16
|
-
dump << " #{@values}"
|
17
|
-
dump << " #{@timestamp}" if @timestamp
|
18
|
-
dump
|
19
|
-
end
|
20
|
-
|
21
|
-
private
|
22
|
-
|
23
|
-
ESCAPES = {
|
24
|
-
measurement: [' '.freeze, ','.freeze],
|
25
|
-
tag_key: ['='.freeze, ' '.freeze, ','.freeze],
|
26
|
-
tag_value: ['='.freeze, ' '.freeze, ','.freeze],
|
27
|
-
field_key: ['='.freeze, ' '.freeze, ','.freeze, '"'.freeze],
|
28
|
-
field_value: ["\\".freeze, '"'.freeze],
|
29
|
-
}.freeze
|
30
|
-
|
31
|
-
private_constant :ESCAPES
|
32
|
-
|
33
|
-
def escape(str, type)
|
34
|
-
# rubocop:disable Layout/AlignParameters
|
35
|
-
str = str.encode "UTF-8".freeze, "UTF-8".freeze,
|
36
|
-
invalid: :replace,
|
37
|
-
undef: :replace,
|
38
|
-
replace: "".freeze
|
39
|
-
# rubocop:enable Layout/AlignParameters
|
40
|
-
|
41
|
-
ESCAPES[type].each do |ch|
|
42
|
-
str = str.gsub(ch) { "\\#{ch}" }
|
43
|
-
end
|
44
|
-
str
|
45
|
-
end
|
46
|
-
|
47
|
-
def escape_values(values)
|
48
|
-
return if values.nil?
|
49
|
-
values.map do |k, v|
|
50
|
-
key = escape(k.to_s, :field_key)
|
51
|
-
val = escape_value(v)
|
52
|
-
"#{key}=#{val}"
|
53
|
-
end.join(",".freeze)
|
54
|
-
end
|
55
|
-
|
56
|
-
def escape_value(value)
|
57
|
-
if value.is_a?(String)
|
58
|
-
'"'.freeze + escape(value, :field_value) + '"'.freeze
|
59
|
-
elsif value.is_a?(Integer)
|
60
|
-
"#{value}i"
|
61
|
-
else
|
62
|
-
value.to_s
|
63
|
-
end
|
64
|
-
end
|
65
|
-
|
66
|
-
def escape_tags(tags)
|
67
|
-
return if tags.nil?
|
68
|
-
|
69
|
-
tags = tags.map do |k, v|
|
70
|
-
key = escape(k.to_s, :tag_key)
|
71
|
-
val = escape(v.to_s, :tag_value)
|
72
|
-
|
73
|
-
"#{key}=#{val}" unless key == "".freeze || val == "".freeze
|
74
|
-
end.compact
|
75
|
-
|
76
|
-
tags.join(",") unless tags.empty?
|
77
|
-
end
|
78
|
-
end
|
79
|
-
end
|