newrelic-management 0.1.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +9 -0
- data/.gitlab-ci.yml +36 -0
- data/.rubocop.yml +18 -0
- data/.travis.yml +38 -0
- data/CHANGELOG.md +14 -0
- data/Gemfile +2 -0
- data/Gemfile.lock +84 -0
- data/LICENSE.txt +21 -0
- data/README.md +105 -0
- data/Rakefile +6 -0
- data/bin/console +32 -0
- data/bin/setup +8 -0
- data/exe/newrelic-management +15 -0
- data/lib/newrelic-management.rb +10 -0
- data/lib/newrelic-management/cli.rb +91 -0
- data/lib/newrelic-management/client.rb +138 -0
- data/lib/newrelic-management/config.rb +107 -0
- data/lib/newrelic-management/controller.rb +61 -0
- data/lib/newrelic-management/helpers/configuration.rb +57 -0
- data/lib/newrelic-management/manager.rb +171 -0
- data/lib/newrelic-management/notifier.rb +64 -0
- data/lib/newrelic-management/util.rb +101 -0
- data/lib/newrelic-management/version.rb +3 -0
- data/newrelic-management.gemspec +51 -0
- metadata +252 -0
@@ -0,0 +1,138 @@
|
|
1
|
+
# Encoding: UTF-8
|
2
|
+
# rubocop: disable LineLength
|
3
|
+
#
|
4
|
+
# Gem Name:: newrelic-management
|
5
|
+
# NewRelicManagement:: Client
|
6
|
+
#
|
7
|
+
# Copyright (C) 2017 Brian Dwyer - Intelligent Digital Services
|
8
|
+
#
|
9
|
+
# All rights reserved - Do Not Redistribute
|
10
|
+
#
|
11
|
+
|
12
|
+
require 'faraday'
|
13
|
+
require 'faraday_middleware'
|
14
|
+
require 'newrelic-management/config'
|
15
|
+
require 'uri'
|
16
|
+
|
17
|
+
module NewRelicManagement
|
18
|
+
# => NewRelic Manager Client
|
19
|
+
module Client
|
20
|
+
module_function
|
21
|
+
|
22
|
+
# => Build the HTTP Connection
|
23
|
+
def nr_api
|
24
|
+
# => Build the Faraday Connection
|
25
|
+
@conn ||= Faraday::Connection.new('https://api.newrelic.com', conn_opts) do |client|
|
26
|
+
client.use Faraday::Response::RaiseError
|
27
|
+
client.use FaradayMiddleware::EncodeJson
|
28
|
+
client.use FaradayMiddleware::ParseJson, content_type: /\bjson$/
|
29
|
+
client.response :logger if Config.environment.to_s.casecmp('development').zero? # => Log Requests to STDOUT
|
30
|
+
client.adapter Faraday.default_adapter #:net_http_persistent
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
def conn_opts
|
35
|
+
{
|
36
|
+
headers: {
|
37
|
+
'Accept' => 'application/json',
|
38
|
+
'Content-Type' => 'application/json',
|
39
|
+
'X-api-key' => Config.nr_api_key
|
40
|
+
},
|
41
|
+
# => ssl: ssl_options
|
42
|
+
}
|
43
|
+
end
|
44
|
+
|
45
|
+
#
|
46
|
+
# => Alerts
|
47
|
+
#
|
48
|
+
|
49
|
+
# => List Alert Policies
|
50
|
+
def alert_policies
|
51
|
+
nr_api.get(url('alerts_policies')).body['policies']
|
52
|
+
rescue NoMethodError
|
53
|
+
[]
|
54
|
+
end
|
55
|
+
|
56
|
+
# => List Alert Conditions
|
57
|
+
def alert_conditions(policy)
|
58
|
+
nr_api.get(url('alerts_conditions'), policy_id: policy).body
|
59
|
+
end
|
60
|
+
|
61
|
+
# => Add an Entitity to an Existing Alert Policy
|
62
|
+
def alert_add_entity(entity_id, condition_id, entity_type = 'Server')
|
63
|
+
nr_api.put do |req|
|
64
|
+
req.url url('alerts_entity_conditions', entity_id)
|
65
|
+
req.params['entity_type'] = entity_type
|
66
|
+
req.params['condition_id'] = condition_id
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
# => Delete an Entitity from an Existing Alert Policy
|
71
|
+
def alert_delete_entity(entity_id, condition_id, entity_type = 'Server')
|
72
|
+
nr_api.delete do |req|
|
73
|
+
req.url url('alerts_entity_conditions', entity_id)
|
74
|
+
req.params['entity_type'] = entity_type
|
75
|
+
req.params['condition_id'] = condition_id
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
# => List the Labels
|
80
|
+
def labels
|
81
|
+
nr_api.get(url('labels')).body['labels']
|
82
|
+
rescue NoMethodError
|
83
|
+
[]
|
84
|
+
end
|
85
|
+
|
86
|
+
#
|
87
|
+
# => Servers
|
88
|
+
#
|
89
|
+
|
90
|
+
# => List the Servers Reporting to NewRelic
|
91
|
+
def servers
|
92
|
+
nr_api.get(url('servers')).body['servers']
|
93
|
+
rescue NoMethodError
|
94
|
+
[]
|
95
|
+
end
|
96
|
+
|
97
|
+
# => Get Info for Specific Server
|
98
|
+
def get_server(server)
|
99
|
+
srv = get_server_id(server)
|
100
|
+
srv ? srv : get_server_name(server)
|
101
|
+
end
|
102
|
+
|
103
|
+
# => Get a Server based on ID
|
104
|
+
def get_server_id(server_id)
|
105
|
+
return nil unless server_id =~ /^[0-9]+$/
|
106
|
+
ret = nr_api.get(url('servers', server_id)).body
|
107
|
+
ret['server']
|
108
|
+
rescue Faraday::ResourceNotFound, NoMethodError
|
109
|
+
nil
|
110
|
+
end
|
111
|
+
|
112
|
+
# => Get a Server based on Name
|
113
|
+
def get_server_name(server, exact = true)
|
114
|
+
ret = nr_api.get(url('servers'), 'filter[name]' => server).body
|
115
|
+
return ret['servers'] unless exact
|
116
|
+
ret['servers'].find { |x| x['name'].casecmp(server).zero? }
|
117
|
+
rescue NoMethodError
|
118
|
+
nil
|
119
|
+
end
|
120
|
+
|
121
|
+
# => List the Servers with a Label
|
122
|
+
def get_servers_labeled(labels)
|
123
|
+
label_query = Array(labels).reject { |x| !x.include?(':') }.join(';')
|
124
|
+
return [] unless label_query
|
125
|
+
nr_api.get(url('servers'), 'filter[labels]' => label_query).body
|
126
|
+
end
|
127
|
+
|
128
|
+
# => Delete a Server from NewRelic
|
129
|
+
def delete_server(server_id)
|
130
|
+
nr_api.delete(url('servers', server_id)).body
|
131
|
+
end
|
132
|
+
|
133
|
+
def url(*args)
|
134
|
+
'/v2/' + args.map { |a| URI.encode_www_form_component a.to_s }.join('/') + '.json'
|
135
|
+
end
|
136
|
+
private :url
|
137
|
+
end
|
138
|
+
end
|
@@ -0,0 +1,107 @@
|
|
1
|
+
# Encoding: UTF-8
|
2
|
+
#
|
3
|
+
# Gem Name:: newrelic-management
|
4
|
+
# NewRelicManagement:: Config
|
5
|
+
#
|
6
|
+
# Copyright (C) 2017 Brian Dwyer - Intelligent Digital Services
|
7
|
+
#
|
8
|
+
# All rights reserved - Do Not Redistribute
|
9
|
+
#
|
10
|
+
|
11
|
+
require 'newrelic-management/helpers/configuration'
|
12
|
+
require 'pathname'
|
13
|
+
|
14
|
+
module NewRelicManagement
|
15
|
+
# => This is the Configuration module.
|
16
|
+
module Config
|
17
|
+
module_function
|
18
|
+
|
19
|
+
extend Configuration
|
20
|
+
|
21
|
+
# => Gem Root Directory
|
22
|
+
define_setting :root, Pathname.new(File.expand_path('../../../', __FILE__))
|
23
|
+
|
24
|
+
# => My Name
|
25
|
+
define_setting :author, 'Brian Dwyer - Intelligent Digital Services'
|
26
|
+
|
27
|
+
# => Application Environment
|
28
|
+
define_setting :environment, :production
|
29
|
+
|
30
|
+
# => Config File
|
31
|
+
define_setting :config_file, File.join(root, 'config', 'config.json')
|
32
|
+
|
33
|
+
# => NewRelic API Key
|
34
|
+
define_setting :nr_api_key, nil
|
35
|
+
|
36
|
+
# => Daemonization
|
37
|
+
define_setting :daemonize, false
|
38
|
+
|
39
|
+
# => Silence Notifications
|
40
|
+
define_setting :silent, false
|
41
|
+
|
42
|
+
#
|
43
|
+
# => Alert Management
|
44
|
+
#
|
45
|
+
|
46
|
+
# => Array of Alerts to Manage
|
47
|
+
define_setting :alerts, []
|
48
|
+
|
49
|
+
# => How often to run when Daemonized
|
50
|
+
define_setting :alert_management_interval, '1m'
|
51
|
+
|
52
|
+
# => Find entities matching any tag, instead of all tags
|
53
|
+
define_setting :alert_match_any, false
|
54
|
+
|
55
|
+
#
|
56
|
+
# => Stale Server Management
|
57
|
+
#
|
58
|
+
|
59
|
+
# => Enable Stale Server Cleanup
|
60
|
+
define_setting :cleanup, false
|
61
|
+
|
62
|
+
# => Set a Time to keep Non-Reporting Servers
|
63
|
+
define_setting :cleanup_age, nil
|
64
|
+
|
65
|
+
# => How often to run when Daemonized
|
66
|
+
define_setting :cleanup_interval, '1m'
|
67
|
+
|
68
|
+
#
|
69
|
+
# => Transient Configuration
|
70
|
+
#
|
71
|
+
define_setting :transient, {}
|
72
|
+
|
73
|
+
#
|
74
|
+
# => Facilitate Dynamic Addition of Configuration Values
|
75
|
+
#
|
76
|
+
# => @return [class_variable]
|
77
|
+
#
|
78
|
+
def add(config = {})
|
79
|
+
config.each do |key, value|
|
80
|
+
define_setting key.to_sym, value
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
#
|
85
|
+
# => Facilitate Dynamic Removal of Configuration Values
|
86
|
+
#
|
87
|
+
# => @return nil
|
88
|
+
#
|
89
|
+
def clear(config)
|
90
|
+
Array(config).each do |setting|
|
91
|
+
delete_setting setting
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
#
|
96
|
+
# => List the Configurable Keys as a Hash
|
97
|
+
#
|
98
|
+
# @return [Hash]
|
99
|
+
#
|
100
|
+
def options
|
101
|
+
map = Config.class_variables.map do |key|
|
102
|
+
[key.to_s.tr('@', '').to_sym, class_variable_get(:"#{key}")]
|
103
|
+
end
|
104
|
+
Hash[map]
|
105
|
+
end
|
106
|
+
end
|
107
|
+
end
|
@@ -0,0 +1,61 @@
|
|
1
|
+
# Encoding: UTF-8
|
2
|
+
#
|
3
|
+
# Gem Name:: newrelic-management
|
4
|
+
# NewRelicManagement:: Controller
|
5
|
+
#
|
6
|
+
# Copyright (C) 2017 Brian Dwyer - Intelligent Digital Services
|
7
|
+
#
|
8
|
+
# All rights reserved - Do Not Redistribute
|
9
|
+
#
|
10
|
+
|
11
|
+
require 'newrelic-management/config'
|
12
|
+
require 'newrelic-management/manager'
|
13
|
+
require 'newrelic-management/notifier'
|
14
|
+
require 'os'
|
15
|
+
require 'rufus-scheduler'
|
16
|
+
|
17
|
+
module NewRelicManagement
|
18
|
+
# => Utility Methods
|
19
|
+
module Controller
|
20
|
+
module_function
|
21
|
+
|
22
|
+
# => Daemonization for Periodic Management
|
23
|
+
def daemon # rubocop: disable AbcSize, MethodLength
|
24
|
+
# => Windows Workaround (https://github.com/bdwyertech/newrelic-management/issues/1)
|
25
|
+
ENV['TZ'] = 'UTC' if OS.windows? && !ENV['TZ']
|
26
|
+
|
27
|
+
scheduler = Rufus::Scheduler.new
|
28
|
+
Notifier.msg('Daemonizing Process')
|
29
|
+
|
30
|
+
# => Alerts Management
|
31
|
+
alerts_interval = Config.alert_management_interval
|
32
|
+
scheduler.every alerts_interval, overlap: false do
|
33
|
+
Manager.manage_alerts
|
34
|
+
end
|
35
|
+
|
36
|
+
# => Cleanup Stale Servers
|
37
|
+
if Config.cleanup
|
38
|
+
cleanup_interval = Config.cleanup_interval
|
39
|
+
cleanup_age = Config.cleanup_age
|
40
|
+
|
41
|
+
scheduler.every cleanup_interval, overlap: false do
|
42
|
+
Manager.remove_nonreporting_servers(cleanup_age)
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
# => Join the Current Thread to the Scheduler Thread
|
47
|
+
scheduler.join
|
48
|
+
end
|
49
|
+
|
50
|
+
# => Run the Application
|
51
|
+
def run
|
52
|
+
daemon if Config.daemonize
|
53
|
+
|
54
|
+
# => Manage Alerts
|
55
|
+
Manager.manage_alerts
|
56
|
+
|
57
|
+
# => Manage
|
58
|
+
Manager.remove_nonreporting_servers(Config.cleanup_age) if Config.cleanup
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
@@ -0,0 +1,57 @@
|
|
1
|
+
# Encoding: UTF-8
|
2
|
+
#
|
3
|
+
# Gem Name:: newrelic-management
|
4
|
+
# Helper:: Configuration
|
5
|
+
#
|
6
|
+
# Author: Eli Fatsi - https://www.viget.com/articles/easy-gem-configuration-variables-with-defaults
|
7
|
+
# Contributor: Brian Dwyer - Intelligent Digital Services
|
8
|
+
#
|
9
|
+
|
10
|
+
# => Configuration Helper Module
|
11
|
+
module Configuration
|
12
|
+
#
|
13
|
+
# => Provides a method to configure an Application
|
14
|
+
# => Example:
|
15
|
+
# NewRelicManagement::Config.setup do |cfg|
|
16
|
+
# cfg.config_file = 'abc.json'
|
17
|
+
# cfg.app_name = 'GemBase'
|
18
|
+
# end
|
19
|
+
#
|
20
|
+
def setup
|
21
|
+
yield self
|
22
|
+
end
|
23
|
+
|
24
|
+
def define_setting(name, default = nil)
|
25
|
+
class_variable_set("@@#{name}", default)
|
26
|
+
|
27
|
+
define_class_method "#{name}=" do |value|
|
28
|
+
class_variable_set("@@#{name}", value)
|
29
|
+
end
|
30
|
+
|
31
|
+
define_class_method name do
|
32
|
+
class_variable_get("@@#{name}")
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def delete_setting(name)
|
37
|
+
remove_class_variable("@@#{name}")
|
38
|
+
|
39
|
+
delete_class_method(name)
|
40
|
+
rescue NameError # => Handle Non-Existent Settings
|
41
|
+
return
|
42
|
+
end
|
43
|
+
|
44
|
+
private
|
45
|
+
|
46
|
+
def define_class_method(name, &block)
|
47
|
+
(class << self; self; end).instance_eval do
|
48
|
+
define_method name, &block
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
def delete_class_method(name)
|
53
|
+
(class << self; self; end).instance_eval do
|
54
|
+
undef_method name
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
@@ -0,0 +1,171 @@
|
|
1
|
+
# Encoding: UTF-8
|
2
|
+
# rubocop: disable LineLength
|
3
|
+
#
|
4
|
+
# Gem Name:: newrelic-management
|
5
|
+
# NewRelicManagement:: Manager
|
6
|
+
#
|
7
|
+
# Copyright (C) 2017 Brian Dwyer - Intelligent Digital Services
|
8
|
+
#
|
9
|
+
# All rights reserved - Do Not Redistribute
|
10
|
+
#
|
11
|
+
|
12
|
+
require 'chronic_duration'
|
13
|
+
require 'json'
|
14
|
+
require 'newrelic-management/client'
|
15
|
+
require 'newrelic-management/config'
|
16
|
+
require 'newrelic-management/notifier'
|
17
|
+
|
18
|
+
module NewRelicManagement
|
19
|
+
# => Manager Methods
|
20
|
+
module Manager
|
21
|
+
module_function
|
22
|
+
|
23
|
+
######################
|
24
|
+
# => Alerts <= #
|
25
|
+
######################
|
26
|
+
|
27
|
+
# => Manage Alerts
|
28
|
+
def manage_alerts
|
29
|
+
Array(Config.alerts).each do |alert|
|
30
|
+
# => Set the Filtering Policy
|
31
|
+
Config.transient[:alert_match_any] = alert[:match_any] ? true : false
|
32
|
+
|
33
|
+
# => Manage the Alerts
|
34
|
+
manage_alert(alert[:name], alert[:labels], alert[:exclude])
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
def manage_alert(alert, labels, exclude = []) # rubocop: disable AbcSize
|
39
|
+
conditions = find_alert_conditions(alert) || return
|
40
|
+
tagged_entities = find_labeled(labels)
|
41
|
+
excluded = find_excluded(exclude)
|
42
|
+
|
43
|
+
conditions.each do |condition|
|
44
|
+
next unless condition['type'] == 'servers_metric'
|
45
|
+
existing_entities = condition['entities']
|
46
|
+
|
47
|
+
to_add = tagged_entities.map(&:to_i) - existing_entities.map(&:to_i) - excluded
|
48
|
+
to_delete = excluded & existing_entities.map(&:to_i)
|
49
|
+
|
50
|
+
add_to_alert(to_add, condition['id'])
|
51
|
+
delete_from_alert(to_delete, condition['id'])
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
def add_to_alert(entities, condition_id, type = 'Server')
|
56
|
+
return if entities.empty?
|
57
|
+
Notifier.add_servers(entities)
|
58
|
+
Array(entities).each do |entity|
|
59
|
+
Client.alert_add_entity(entity, condition_id, type)
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
def delete_from_alert(entities, condition_id, type = 'Server')
|
64
|
+
return if entities.empty?
|
65
|
+
Notifier.remove_servers(entities)
|
66
|
+
Array(entities).each do |entity|
|
67
|
+
Client.alert_delete_entity(entity, condition_id, type)
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
# => Find Matching Alert (Name or ID)
|
72
|
+
def find_alert(alert)
|
73
|
+
id = Integer(alert) rescue nil # rubocop: disable RescueModifier
|
74
|
+
list_alerts.find do |policy|
|
75
|
+
return policy if id && policy['id'] == id
|
76
|
+
policy['name'].casecmp(alert).zero?
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
# => Find Alert Conditions for a Matching Alert Policy
|
81
|
+
def find_alert_conditions(alert)
|
82
|
+
alert = find_alert(alert)
|
83
|
+
list_alert_conditions(alert['id'])['conditions'] if alert
|
84
|
+
end
|
85
|
+
|
86
|
+
# => Simply List Alerts
|
87
|
+
def list_alerts
|
88
|
+
Util.cachier('list_alerts') { Client.alert_policies }
|
89
|
+
end
|
90
|
+
|
91
|
+
# => List All Alert Conditions for an Alert Policy
|
92
|
+
def list_alert_conditions(policy_id)
|
93
|
+
Util.cachier("alert_conditions_#{policy_id}") { Client.alert_conditions(policy_id) }
|
94
|
+
end
|
95
|
+
|
96
|
+
#######################
|
97
|
+
# => Servers <= #
|
98
|
+
#######################
|
99
|
+
|
100
|
+
# => Servers with the oldest `last_reported_at` will be at the top
|
101
|
+
def list_servers
|
102
|
+
Util.cachier('list_servers') do
|
103
|
+
Client.servers.sort_by { |hsh| hsh['last_reported_at'] }.collect do |server|
|
104
|
+
{
|
105
|
+
name: server['name'],
|
106
|
+
last_reported_at: server['last_reported_at'],
|
107
|
+
id: server['id'],
|
108
|
+
reporting: server['reporting']
|
109
|
+
}
|
110
|
+
end
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
def list_nonreporting_servers
|
115
|
+
list_servers.reject { |server| server[:reporting] }
|
116
|
+
end
|
117
|
+
|
118
|
+
# => Remove Non-Reporting Servers
|
119
|
+
def remove_nonreporting_servers(keeptime = nil)
|
120
|
+
list_nonreporting_servers.each do |server|
|
121
|
+
next if keeptime && Time.parse(server[:last_reported_at]) >= Time.now - ChronicDuration.parse(keeptime)
|
122
|
+
Notifier.msg(server[:name], 'Removing Stale, Non-Reporting Server')
|
123
|
+
Client.delete_server(server[:id])
|
124
|
+
end
|
125
|
+
end
|
126
|
+
|
127
|
+
# => Find Servers which should be excluded from Management
|
128
|
+
def find_excluded(excluded)
|
129
|
+
result = []
|
130
|
+
Array(excluded).each do |exclude|
|
131
|
+
if exclude.include?(':')
|
132
|
+
find_labeled(exclude).each { |x| result << x }
|
133
|
+
next
|
134
|
+
end
|
135
|
+
res = Client.get_server(exclude)
|
136
|
+
result << res['id'] if res
|
137
|
+
end
|
138
|
+
result
|
139
|
+
end
|
140
|
+
|
141
|
+
######################
|
142
|
+
# => Labels <= #
|
143
|
+
######################
|
144
|
+
|
145
|
+
def list_labels
|
146
|
+
Util.cachier('list_labels') { Client.labels }
|
147
|
+
end
|
148
|
+
|
149
|
+
# => Find Servers Matching a Label
|
150
|
+
# => Example: find_labeled(['Role:API', 'Environment:Production'])
|
151
|
+
def find_labeled(labels, match_any = Config.transient[:alert_match_any]) # rubocop: disable AbcSize
|
152
|
+
list = list_labels
|
153
|
+
labeled = []
|
154
|
+
Array(labels).select do |lbl|
|
155
|
+
list.select { |x| x['key'].casecmp(lbl).zero? }.each do |mtch|
|
156
|
+
labeled.push(Array(mtch['links']['servers']))
|
157
|
+
end
|
158
|
+
end
|
159
|
+
|
160
|
+
unless match_any
|
161
|
+
# => Array(labeled) should contain one array per label
|
162
|
+
# => # => If it does not, it means the label is missing or misspelled
|
163
|
+
return [] unless labeled.count == Array(labels).count
|
164
|
+
|
165
|
+
# => Return Only those matching All Labels
|
166
|
+
return Util.common_array(labeled)
|
167
|
+
end
|
168
|
+
labeled.flatten.uniq
|
169
|
+
end
|
170
|
+
end
|
171
|
+
end
|