kennel 2.19.1 → 2.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Readme.md +3 -0
- data/lib/kennel/importer.rb +8 -1
- data/lib/kennel/models/project.rb +11 -7
- data/lib/kennel/models/slo.rb +14 -9
- data/lib/kennel/tasks/dump.rb +54 -0
- data/lib/kennel/tasks/import.rb +17 -0
- data/lib/kennel/tasks/nodata.rb +53 -0
- data/lib/kennel/tasks/tracking_id.rb +14 -0
- data/lib/kennel/tasks/validate_mentions.rb +41 -0
- data/lib/kennel/tasks/validate_plan.rb +114 -0
- data/lib/kennel/tasks.rb +2 -169
- data/lib/kennel/version.rb +1 -1
- data/template/Readme.md +3 -0
- metadata +7 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 5f604eb100e46e40ba76333717c9ddb99e06cd85be98f7eb19217b7be1b193b2
|
|
4
|
+
data.tar.gz: 352ead0a6c2ce295f14113d03f9ab956278851d55457340dffa9d6fdf042856e
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 5d0e3477bcef3043433fca52d3e3c3940f7cd315c7c89ecebb7a07eb7a50d56adcb46522a51492ef47c72346b10bec7446c495c8d368c5ff741a510e2c8885b9
|
|
7
|
+
data.tar.gz: 0e71a447829ac1dcc996b08858e7dcd720b3e3f6f9e5b2e008bc0e187e952e1cafaa561b62152a37e547053d033ef0c2e37746419205eb1788d859c536f5d77a
|
data/Readme.md
CHANGED
|
@@ -444,6 +444,9 @@ Run `rake kennel:alerts TAG=service:my-service` to see all un-muted alerts for a
|
|
|
444
444
|
Use `KNOWN=foo@bar.com,baz@bar.com` to exempt mentions that are not returned by the API.
|
|
445
445
|
Use `KNOWN_RANDOM=@sns-foo,@sns-bar` to ignore for example SNS handles that are randomly invalid in the API.
|
|
446
446
|
|
|
447
|
+
### Validating planned changes
|
|
448
|
+
`rake kennel:validate_plan` validates planned monitor and dashboard changes against the Datadog API.
|
|
449
|
+
|
|
447
450
|
### Grepping through all of datadog
|
|
448
451
|
```Bash
|
|
449
452
|
rake kennel:dump > tmp/dump
|
data/lib/kennel/importer.rb
CHANGED
|
@@ -86,7 +86,14 @@ module Kennel
|
|
|
86
86
|
end
|
|
87
87
|
when "synthetics/tests"
|
|
88
88
|
data[:locations] = :all if data[:locations].sort == Kennel::Models::SyntheticTest::LOCATIONS.sort
|
|
89
|
-
|
|
89
|
+
when "slo"
|
|
90
|
+
# sli_specification it is only used by datadog if the user switched to "Bad Events"
|
|
91
|
+
# otherwise user would be trying to set sli_specification but the slo does not change since query is used
|
|
92
|
+
if data.key?(:query) && data.key?(:sli_specification)
|
|
93
|
+
delete = (data.dig(:sli_specification, :count, :bad_events_formula) ? :query : :sli_specification)
|
|
94
|
+
data.delete delete
|
|
95
|
+
end
|
|
96
|
+
else # uncovered
|
|
90
97
|
# noop
|
|
91
98
|
end
|
|
92
99
|
|
|
@@ -10,10 +10,16 @@ module Kennel
|
|
|
10
10
|
|
|
11
11
|
def self.file_location
|
|
12
12
|
return @file_location if defined?(@file_location)
|
|
13
|
-
|
|
14
|
-
|
|
13
|
+
methods = instance_methods(false)
|
|
14
|
+
if methods.any?
|
|
15
|
+
@file_location = methods.detect do |method|
|
|
16
|
+
location = instance_method(method).source_location.first
|
|
17
|
+
if (path = find_relative_path(location))
|
|
18
|
+
break path
|
|
19
|
+
end
|
|
20
|
+
end || raise("Unable to find file_location for #{name}")
|
|
15
21
|
else
|
|
16
|
-
@file_location = nil
|
|
22
|
+
@file_location = nil # not sure if this is actually needed
|
|
17
23
|
end
|
|
18
24
|
end
|
|
19
25
|
|
|
@@ -29,11 +35,9 @@ module Kennel
|
|
|
29
35
|
|
|
30
36
|
private
|
|
31
37
|
|
|
32
|
-
private_class_method def self.
|
|
38
|
+
private_class_method def self.find_relative_path(path)
|
|
33
39
|
return path unless File.absolute_path?(path)
|
|
34
|
-
path.dup.sub!("#{Bundler.root}/", "") ||
|
|
35
|
-
path.dup.sub!("#{Dir.pwd}/", "") ||
|
|
36
|
-
raise("Unable to make path #{path} relative with bundler root #{Bundler.root} or pwd #{Dir.pwd}")
|
|
40
|
+
path.dup.sub!("#{Bundler.root}/", "") || path.dup.sub!("#{Dir.pwd}/", "")
|
|
37
41
|
end
|
|
38
42
|
|
|
39
43
|
# hook for users to add custom filtering via `prepend`
|
data/lib/kennel/models/slo.rb
CHANGED
|
@@ -15,7 +15,8 @@ module Kennel
|
|
|
15
15
|
groups: nil,
|
|
16
16
|
monitor_ids: [],
|
|
17
17
|
thresholds: [],
|
|
18
|
-
primary: nil
|
|
18
|
+
primary: nil,
|
|
19
|
+
sli_specification: nil
|
|
19
20
|
}.freeze
|
|
20
21
|
|
|
21
22
|
settings :type, :description, :thresholds, :query, :tags, :monitor_ids, :monitor_tags, :name, :groups, :sli_specification, :primary
|
|
@@ -27,7 +28,8 @@ module Kennel
|
|
|
27
28
|
monitor_ids: -> { DEFAULTS.fetch(:monitor_ids) },
|
|
28
29
|
thresholds: -> { DEFAULTS.fetch(:thresholds) },
|
|
29
30
|
groups: -> { DEFAULTS.fetch(:groups) },
|
|
30
|
-
primary: -> { DEFAULTS.fetch(:primary) }
|
|
31
|
+
primary: -> { DEFAULTS.fetch(:primary) },
|
|
32
|
+
sli_specification: -> { DEFAULTS.fetch(:sli_specification) }
|
|
31
33
|
)
|
|
32
34
|
|
|
33
35
|
def build_json
|
|
@@ -50,9 +52,8 @@ module Kennel
|
|
|
50
52
|
data[:target_threshold] = threshold[:target]
|
|
51
53
|
end
|
|
52
54
|
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
data[:sli_specification] = sli_specification
|
|
55
|
+
if (v = sli_specification)
|
|
56
|
+
data[:sli_specification] = v
|
|
56
57
|
elsif (v = query)
|
|
57
58
|
data[:query] = v
|
|
58
59
|
end
|
|
@@ -102,10 +103,14 @@ module Kennel
|
|
|
102
103
|
[:timeframe, :warning_threshold, :target_threshold].each { |k| actual.delete k }
|
|
103
104
|
end
|
|
104
105
|
|
|
105
|
-
#
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
106
|
+
# discard deprecated query which stays in datadog forever when we are trying to set sli_specification
|
|
107
|
+
# (downgrading to query by setting sli_specification=nil is not supported in the api)
|
|
108
|
+
actual.delete :query if expected[:sli_specification]
|
|
109
|
+
|
|
110
|
+
# user set query so let's not worry about sli_specification even if this might be hiding bugs
|
|
111
|
+
# ideally we'd validate and tell the user that this will have no effect (see importer logic)
|
|
112
|
+
# but I'm not confident this will always be right
|
|
113
|
+
actual.delete :sli_specification if expected[:query]
|
|
109
114
|
|
|
110
115
|
ignore_default(expected, actual, DEFAULTS)
|
|
111
116
|
end
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
namespace :kennel do
|
|
4
|
+
desc "Dump ALL of datadog config as raw json ... useful for grep/search [TYPE=slo|monitor|dashboard]"
|
|
5
|
+
task dump: :environment do
|
|
6
|
+
resources =
|
|
7
|
+
if (type = ENV["TYPE"])
|
|
8
|
+
[type]
|
|
9
|
+
else
|
|
10
|
+
Kennel::Models::Record.api_resource_map.keys
|
|
11
|
+
end
|
|
12
|
+
api = Kennel::Api.new
|
|
13
|
+
list = nil
|
|
14
|
+
first = true
|
|
15
|
+
|
|
16
|
+
Kennel.out.puts "["
|
|
17
|
+
resources.each do |resource|
|
|
18
|
+
Kennel::Progress.progress("Downloading #{resource}") do
|
|
19
|
+
list = api.list(resource)
|
|
20
|
+
api.fill_details!(resource, list) if resource == "dashboard"
|
|
21
|
+
end
|
|
22
|
+
list.each do |r|
|
|
23
|
+
r[:api_resource] = resource
|
|
24
|
+
if first
|
|
25
|
+
first = false
|
|
26
|
+
else
|
|
27
|
+
Kennel.out.puts ","
|
|
28
|
+
end
|
|
29
|
+
Kennel.out.print JSON.pretty_generate(r)
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
Kennel.out.puts "\n]"
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
desc "Find items from dump by pattern DUMP= PATTERN= [URLS=true]"
|
|
36
|
+
task dump_grep: :environment do
|
|
37
|
+
file = ENV.fetch("DUMP")
|
|
38
|
+
pattern = Regexp.new ENV.fetch("PATTERN")
|
|
39
|
+
items = File.read(file)[2..-2].gsub("},\n{", "}--SPLIT--{").split("--SPLIT--")
|
|
40
|
+
models = Kennel::Models::Record.api_resource_map
|
|
41
|
+
found = items.grep(pattern)
|
|
42
|
+
exit 1 if found.empty?
|
|
43
|
+
found.each do |resource|
|
|
44
|
+
if ENV["URLS"]
|
|
45
|
+
parsed = JSON.parse(resource)
|
|
46
|
+
url = models[parsed.fetch("api_resource")].url(parsed.fetch("id"))
|
|
47
|
+
title = parsed["title"] || parsed["name"]
|
|
48
|
+
Kennel.out.puts "#{url} # #{title}"
|
|
49
|
+
else
|
|
50
|
+
Kennel.out.puts resource
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
namespace :kennel do
|
|
4
|
+
desc "Convert existing resources to copy-pasteable definitions to import existing resources (call with URL= or call with RESOURCE= and ID=)"
|
|
5
|
+
task import: :environment do
|
|
6
|
+
if (id = ENV["ID"]) && (resource = ENV["RESOURCE"])
|
|
7
|
+
id = Integer(id) if id =~ /^\d+$/
|
|
8
|
+
elsif (url = ENV["URL"])
|
|
9
|
+
resource, id = Kennel::Models::Record.parse_any_url(url) || Kennel::Tasks.abort("Unable to parse url")
|
|
10
|
+
else
|
|
11
|
+
possible_resources = Kennel::Models::Record.subclasses.map(&:api_resource)
|
|
12
|
+
Kennel::Tasks.abort("Call with URL= or call with RESOURCE=#{possible_resources.join(" or ")} and ID=")
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
Kennel.out.puts Kennel::Importer.new(Kennel::Api.new).import(resource, id)
|
|
16
|
+
end
|
|
17
|
+
end
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
namespace :kennel do
|
|
4
|
+
desc "show monitors with no data by TAG, for example TAG=team:foo [THRESHOLD_DAYS=7] [FORMAT=json]"
|
|
5
|
+
task nodata: :environment do
|
|
6
|
+
tag = ENV["TAG"] || Kennel::Tasks.abort("Call with TAG=foo:bar")
|
|
7
|
+
monitors = Kennel::Api.new.list("monitor", monitor_tags: tag, group_states: "no data")
|
|
8
|
+
monitors.select! { |m| m[:overall_state] == "No Data" }
|
|
9
|
+
monitors.reject! { |m| m[:tags].include? "nodata:ignore" }
|
|
10
|
+
if monitors.any?
|
|
11
|
+
Kennel.err.puts <<~TEXT
|
|
12
|
+
To ignore monitors with expected nodata, tag it with "nodata:ignore"
|
|
13
|
+
|
|
14
|
+
TEXT
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
now = Time.now
|
|
18
|
+
monitors.each do |m|
|
|
19
|
+
m[:days_in_no_data] =
|
|
20
|
+
if m[:overall_state_modified]
|
|
21
|
+
since = Date.parse(m[:overall_state_modified]).to_time
|
|
22
|
+
((now - since) / (24 * 60 * 60)).to_i
|
|
23
|
+
else
|
|
24
|
+
999
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
if (threshold = ENV["THRESHOLD_DAYS"])
|
|
29
|
+
monitors.select! { |m| m[:days_in_no_data] > Integer(threshold) }
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
monitors.each { |m| m[:url] = Kennel::Utils.path_to_url("/monitors/#{m[:id]}") }
|
|
33
|
+
|
|
34
|
+
if ENV["FORMAT"] == "json"
|
|
35
|
+
report = monitors.map do |m|
|
|
36
|
+
match = m[:message].to_s.match(/-- #{Regexp.escape(Kennel::Models::Record::MARKER_TEXT)} (\S+:\S+) in (\S+), /) || []
|
|
37
|
+
m.slice(:url, :name, :tags, :days_in_no_data).merge(
|
|
38
|
+
kennel_tracking_id: match[1],
|
|
39
|
+
kennel_source: match[2]
|
|
40
|
+
)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
Kennel.out.puts JSON.pretty_generate(report)
|
|
44
|
+
else
|
|
45
|
+
monitors.each do |m|
|
|
46
|
+
Kennel.out.puts m[:name]
|
|
47
|
+
Kennel.out.puts Kennel::Utils.path_to_url("/monitors/#{m[:id]}")
|
|
48
|
+
Kennel.out.puts "No data since #{m[:days_in_no_data]}d"
|
|
49
|
+
Kennel.out.puts
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
namespace :kennel do
|
|
4
|
+
desc "Resolve given id to kennel tracking-id RESOURCE= ID="
|
|
5
|
+
task tracking_id: "kennel:environment" do
|
|
6
|
+
resource = ENV.fetch("RESOURCE")
|
|
7
|
+
id = ENV.fetch("ID")
|
|
8
|
+
klass =
|
|
9
|
+
Kennel::Models::Record.subclasses.detect { |s| s.api_resource == resource } ||
|
|
10
|
+
raise("resource #{resource} not know")
|
|
11
|
+
object = Kennel::Api.new.show(resource, id)
|
|
12
|
+
Kennel.out.puts klass.parse_tracking_id(object)
|
|
13
|
+
end
|
|
14
|
+
end
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
namespace :kennel do
|
|
4
|
+
desc "Verify that all used monitor mentions are valid"
|
|
5
|
+
task validate_mentions: :environment do
|
|
6
|
+
known = []
|
|
7
|
+
|
|
8
|
+
# @slack- @team- @webhook- @sns- user-emails
|
|
9
|
+
known += Kennel::Api.new.send(:request, :get, "/api/v2/notifications/handles?group_limit=99999")
|
|
10
|
+
.fetch(:data)
|
|
11
|
+
.flat_map { |d| d.dig(:attributes, :handles) }
|
|
12
|
+
.map { |v| v.fetch(:value) }
|
|
13
|
+
|
|
14
|
+
# group emails or other 1-off things we know are valid
|
|
15
|
+
manual = ENV["KNOWN"].to_s.split(",")
|
|
16
|
+
dupes = (manual & known)
|
|
17
|
+
Kennel::Tasks.abort "KNOWN=#{dupes.join(",")} values are already known and should be removed" if dupes.any?
|
|
18
|
+
known += manual
|
|
19
|
+
|
|
20
|
+
# @sns- handles are randomly invalid so we need to ignore them without checking if the ignore is needed
|
|
21
|
+
# https://help.datadoghq.com/hc/en-us/requests/2310423
|
|
22
|
+
known += ENV["KNOWN_RANDOM"].to_s.split(",")
|
|
23
|
+
|
|
24
|
+
bad = []
|
|
25
|
+
Dir["generated/**/*.json"].each do |f|
|
|
26
|
+
next unless (message = JSON.parse(File.read(f))["message"])
|
|
27
|
+
used = message
|
|
28
|
+
.scan(/(?:^|\s)(@[^\s{,'"]+)/)
|
|
29
|
+
.flatten(1)
|
|
30
|
+
.grep(/^@.*@|^@.*-/) # ignore @here etc handles ... datadog uses @foo@bar.com for emails and @foo-bar for integrations
|
|
31
|
+
(used - known).each { |v| bad << [f, v] }
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
if bad.any?
|
|
35
|
+
url = Kennel::Utils.path_to_url "/account/settings"
|
|
36
|
+
Kennel.err.puts "Invalid mentions found, either ignore them by adding to `KNOWN` env var or add them via #{url}"
|
|
37
|
+
bad.each { |f, v| Kennel.err.puts "Invalid mention #{v} in monitor message of #{f}" }
|
|
38
|
+
Kennel::Tasks.abort ENV["KNOWN_WARNING"]
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
module Kennel
|
|
3
|
+
module ValidatePlan
|
|
4
|
+
class MonitorValidator
|
|
5
|
+
COSMETIC_FIELDS = ["name", "message", "tags"].freeze
|
|
6
|
+
|
|
7
|
+
def initialize(item)
|
|
8
|
+
@item = item
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
def validate(api)
|
|
12
|
+
data = @item.expected.as_json
|
|
13
|
+
|
|
14
|
+
# ignore unresolved ids from yet to be created monitors
|
|
15
|
+
return nil if ["composite", "slo alert"].include?(data[:type]) && data[:query].include?("%")
|
|
16
|
+
|
|
17
|
+
api.send(:request, :post, "/api/v1/monitor/validate", body: data)
|
|
18
|
+
nil
|
|
19
|
+
rescue StandardError => e
|
|
20
|
+
"#{Kennel::Console.color(:yellow, "#{@item.api_resource} #{@item.tracking_id}:")}\n#{e.message}"
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
class DashboardValidator
|
|
25
|
+
COSMETIC_FIELDS = ["title", "description", "tags"].freeze
|
|
26
|
+
|
|
27
|
+
def initialize(item)
|
|
28
|
+
@item = item
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
# datadog does not offer a validation api for dashboards,
|
|
32
|
+
# so we insert an invalid widget at the end and see if that is the invalid widget it complains about
|
|
33
|
+
# this will break if they ever start from the back or return errors for everything that is invalid
|
|
34
|
+
#
|
|
35
|
+
# we do not need to worry about unresolved ids because:
|
|
36
|
+
# - alert_graph widgets allows kennel style ids
|
|
37
|
+
# - slo widgets allow kennel style ids
|
|
38
|
+
# - uptime widgets allow kennel style ids
|
|
39
|
+
def validate(api)
|
|
40
|
+
json = @item.expected.as_json
|
|
41
|
+
json = Marshal.load(Marshal.dump(json))
|
|
42
|
+
|
|
43
|
+
# add a semi-valid (does not fail immediately on missing definition) widget still blocks the request
|
|
44
|
+
placeholder = "invalid_metric_do_not_update"
|
|
45
|
+
json.fetch(:widgets) << {
|
|
46
|
+
definition: {
|
|
47
|
+
type: "timeseries", requests: [{
|
|
48
|
+
response_format: "timeseries",
|
|
49
|
+
queries: [{ data_source: "metrics", name: "restarts", query: placeholder }]
|
|
50
|
+
}]
|
|
51
|
+
},
|
|
52
|
+
layout: { x: 0, y: 0, height: 0, width: 0 } # needed for `layout_type: free` and valid for all
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
begin
|
|
56
|
+
if @item.class::TYPE == :update
|
|
57
|
+
api.update("dashboard", @item.actual.fetch(:id), json)
|
|
58
|
+
else
|
|
59
|
+
api.create("dashboard", json)
|
|
60
|
+
end
|
|
61
|
+
raise "Dashboard validation should have failed, live dashboard was update/created by accident"
|
|
62
|
+
rescue StandardError => e
|
|
63
|
+
# parse the JSON in the error message and see if there is anything except our error
|
|
64
|
+
raise "Unreadable error format: #{e.message}" unless (json = e.message[/^\{"errors":.*}$/m])
|
|
65
|
+
data =
|
|
66
|
+
begin
|
|
67
|
+
JSON.parse(json)
|
|
68
|
+
rescue JSON::ParserError
|
|
69
|
+
raise "Unreadable error format: #{json}"
|
|
70
|
+
end
|
|
71
|
+
raise "Unreadable error format: #{data}" unless (errors = data["errors"]) # uncovered
|
|
72
|
+
return if errors.size == 1 && errors.all? { |m| m.include?("unable to parse #{placeholder}") }
|
|
73
|
+
"#{@item.tracking_id}: #{e.message}"
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
VALIDATORS = {
|
|
79
|
+
"monitor" => MonitorValidator,
|
|
80
|
+
"dashboard" => DashboardValidator
|
|
81
|
+
}.freeze
|
|
82
|
+
|
|
83
|
+
def self.validate(plan)
|
|
84
|
+
changes = (plan.creates + plan.updates)
|
|
85
|
+
|
|
86
|
+
validators = changes.filter_map do |item|
|
|
87
|
+
next unless (validator = VALIDATORS[item.api_resource]&.new(item))
|
|
88
|
+
|
|
89
|
+
if item.class::TYPE == :update
|
|
90
|
+
# ignore if nothing can break
|
|
91
|
+
modified_fields = item.diff.map { |_, f, *| f }
|
|
92
|
+
next nil if modified_fields.all? { |f| validator.class::COSMETIC_FIELDS.include?(f) }
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
validator
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
api = Kennel::Api.new
|
|
99
|
+
errors = validators.filter_map { |v| v.validate(api) }
|
|
100
|
+
return if errors.empty?
|
|
101
|
+
|
|
102
|
+
abort "#{Kennel::Console.color(:red, "#{errors.size} validation(s) failed:")}\n#{errors.join("\n")}"
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
namespace :kennel do
|
|
108
|
+
desc "Validate planned changes against the Datadog API [PROJECT=]"
|
|
109
|
+
task "validate_plan" => "kennel:environment" do
|
|
110
|
+
kennel = Kennel::Tasks.kennel
|
|
111
|
+
kennel.preload
|
|
112
|
+
Kennel::ValidatePlan.validate(kennel.plan)
|
|
113
|
+
end
|
|
114
|
+
end
|
data/lib/kennel/tasks.rb
CHANGED
|
@@ -5,6 +5,8 @@ require "kennel/unmuted_alerts"
|
|
|
5
5
|
require "kennel/importer"
|
|
6
6
|
require "json"
|
|
7
7
|
|
|
8
|
+
Dir.children("#{__dir__}/tasks").each { |f| require_relative "tasks/#{File.basename(f, ".rb")}" }
|
|
9
|
+
|
|
8
10
|
module Kennel
|
|
9
11
|
module Tasks
|
|
10
12
|
class << self
|
|
@@ -71,49 +73,6 @@ namespace :kennel do
|
|
|
71
73
|
Kennel::Tasks.abort "Error during diffing" unless $CHILD_STATUS.success?
|
|
72
74
|
end
|
|
73
75
|
|
|
74
|
-
# ideally do this on every run, but it's slow (~1.5s) and brittle
|
|
75
|
-
# (might not find all via the regex + might find false-positives because random emails can also be sent to)
|
|
76
|
-
# https://help.datadoghq.com/hc/en-us/requests/254114 for automatic validation
|
|
77
|
-
# /monitor/notifications has users+slack+sns but not @team- and @webhook-
|
|
78
|
-
# got a support ticket open to get sns into api/v2 too
|
|
79
|
-
desc "Verify that all used monitor mentions are valid"
|
|
80
|
-
task validate_mentions: :environment do
|
|
81
|
-
known = []
|
|
82
|
-
|
|
83
|
-
# @slack- @team- @webhook- @sns- user-emails
|
|
84
|
-
known += Kennel::Api.new.send(:request, :get, "/api/v2/notifications/handles?group_limit=99999")
|
|
85
|
-
.fetch(:data)
|
|
86
|
-
.flat_map { |d| d.dig(:attributes, :handles) }
|
|
87
|
-
.map { |v| v.fetch(:value) }
|
|
88
|
-
|
|
89
|
-
# group emails or other 1-off things we know are valid
|
|
90
|
-
manual = ENV["KNOWN"].to_s.split(",")
|
|
91
|
-
dupes = (manual & known)
|
|
92
|
-
Kennel::Tasks.abort "KNOWN=#{dupes.join(",")} values are already known and should be removed" if dupes.any?
|
|
93
|
-
known += manual
|
|
94
|
-
|
|
95
|
-
# @sns- handles are randomly invalid so we need to ignore them without checking if the ignore is needed
|
|
96
|
-
# https://help.datadoghq.com/hc/en-us/requests/2310423
|
|
97
|
-
known += ENV["KNOWN_RANDOM"].to_s.split(",")
|
|
98
|
-
|
|
99
|
-
bad = []
|
|
100
|
-
Dir["generated/**/*.json"].each do |f|
|
|
101
|
-
next unless (message = JSON.parse(File.read(f))["message"])
|
|
102
|
-
used = message
|
|
103
|
-
.scan(/(?:^|\s)(@[^\s{,'"]+)/)
|
|
104
|
-
.flatten(1)
|
|
105
|
-
.grep(/^@.*@|^@.*-/) # ignore @here etc handles ... datadog uses @foo@bar.com for emails and @foo-bar for integrations
|
|
106
|
-
(used - known).each { |v| bad << [f, v] }
|
|
107
|
-
end
|
|
108
|
-
|
|
109
|
-
if bad.any?
|
|
110
|
-
url = Kennel::Utils.path_to_url "/account/settings"
|
|
111
|
-
Kennel.err.puts "Invalid mentions found, either ignore them by adding to `KNOWN` env var or add them via #{url}"
|
|
112
|
-
bad.each { |f, v| Kennel.err.puts "Invalid mention #{v} in monitor message of #{f}" }
|
|
113
|
-
Kennel::Tasks.abort ENV["KNOWN_WARNING"]
|
|
114
|
-
end
|
|
115
|
-
end
|
|
116
|
-
|
|
117
76
|
desc "store definitions in generated/"
|
|
118
77
|
task generate: :environment do
|
|
119
78
|
Kennel::Tasks.kennel.generate
|
|
@@ -146,132 +105,6 @@ namespace :kennel do
|
|
|
146
105
|
Kennel::UnmutedAlerts.print(Kennel::Api.new, tag)
|
|
147
106
|
end
|
|
148
107
|
|
|
149
|
-
desc "show monitors with no data by TAG, for example TAG=team:foo [THRESHOLD_DAYS=7] [FORMAT=json]"
|
|
150
|
-
task nodata: :environment do
|
|
151
|
-
tag = ENV["TAG"] || Kennel::Tasks.abort("Call with TAG=foo:bar")
|
|
152
|
-
monitors = Kennel::Api.new.list("monitor", monitor_tags: tag, group_states: "no data")
|
|
153
|
-
monitors.select! { |m| m[:overall_state] == "No Data" }
|
|
154
|
-
monitors.reject! { |m| m[:tags].include? "nodata:ignore" }
|
|
155
|
-
if monitors.any?
|
|
156
|
-
Kennel.err.puts <<~TEXT
|
|
157
|
-
To ignore monitors with expected nodata, tag it with "nodata:ignore"
|
|
158
|
-
|
|
159
|
-
TEXT
|
|
160
|
-
end
|
|
161
|
-
|
|
162
|
-
now = Time.now
|
|
163
|
-
monitors.each do |m|
|
|
164
|
-
m[:days_in_no_data] =
|
|
165
|
-
if m[:overall_state_modified]
|
|
166
|
-
since = Date.parse(m[:overall_state_modified]).to_time
|
|
167
|
-
((now - since) / (24 * 60 * 60)).to_i
|
|
168
|
-
else
|
|
169
|
-
999
|
|
170
|
-
end
|
|
171
|
-
end
|
|
172
|
-
|
|
173
|
-
if (threshold = ENV["THRESHOLD_DAYS"])
|
|
174
|
-
monitors.select! { |m| m[:days_in_no_data] > Integer(threshold) }
|
|
175
|
-
end
|
|
176
|
-
|
|
177
|
-
monitors.each { |m| m[:url] = Kennel::Utils.path_to_url("/monitors/#{m[:id]}") }
|
|
178
|
-
|
|
179
|
-
if ENV["FORMAT"] == "json"
|
|
180
|
-
report = monitors.map do |m|
|
|
181
|
-
match = m[:message].to_s.match(/-- #{Regexp.escape(Kennel::Models::Record::MARKER_TEXT)} (\S+:\S+) in (\S+), /) || []
|
|
182
|
-
m.slice(:url, :name, :tags, :days_in_no_data).merge(
|
|
183
|
-
kennel_tracking_id: match[1],
|
|
184
|
-
kennel_source: match[2]
|
|
185
|
-
)
|
|
186
|
-
end
|
|
187
|
-
|
|
188
|
-
Kennel.out.puts JSON.pretty_generate(report)
|
|
189
|
-
else
|
|
190
|
-
monitors.each do |m|
|
|
191
|
-
Kennel.out.puts m[:name]
|
|
192
|
-
Kennel.out.puts Kennel::Utils.path_to_url("/monitors/#{m[:id]}")
|
|
193
|
-
Kennel.out.puts "No data since #{m[:days_in_no_data]}d"
|
|
194
|
-
Kennel.out.puts
|
|
195
|
-
end
|
|
196
|
-
end
|
|
197
|
-
end
|
|
198
|
-
|
|
199
|
-
desc "Convert existing resources to copy-pasteable definitions to import existing resources (call with URL= or call with RESOURCE= and ID=)"
|
|
200
|
-
task import: :environment do
|
|
201
|
-
if (id = ENV["ID"]) && (resource = ENV["RESOURCE"])
|
|
202
|
-
id = Integer(id) if id =~ /^\d+$/ # dashboards can have alphanumeric ids
|
|
203
|
-
elsif (url = ENV["URL"])
|
|
204
|
-
resource, id = Kennel::Models::Record.parse_any_url(url) || Kennel::Tasks.abort("Unable to parse url")
|
|
205
|
-
else
|
|
206
|
-
possible_resources = Kennel::Models::Record.subclasses.map(&:api_resource)
|
|
207
|
-
Kennel::Tasks.abort("Call with URL= or call with RESOURCE=#{possible_resources.join(" or ")} and ID=")
|
|
208
|
-
end
|
|
209
|
-
|
|
210
|
-
Kennel.out.puts Kennel::Importer.new(Kennel::Api.new).import(resource, id)
|
|
211
|
-
end
|
|
212
|
-
|
|
213
|
-
desc "Dump ALL of datadog config as raw json ... useful for grep/search [TYPE=slo|monitor|dashboard]"
|
|
214
|
-
task dump: :environment do
|
|
215
|
-
resources =
|
|
216
|
-
if (type = ENV["TYPE"])
|
|
217
|
-
[type]
|
|
218
|
-
else
|
|
219
|
-
Kennel::Models::Record.api_resource_map.keys
|
|
220
|
-
end
|
|
221
|
-
api = Kennel::Api.new
|
|
222
|
-
list = nil
|
|
223
|
-
first = true
|
|
224
|
-
|
|
225
|
-
Kennel.out.puts "["
|
|
226
|
-
resources.each do |resource|
|
|
227
|
-
Kennel::Progress.progress("Downloading #{resource}") do
|
|
228
|
-
list = api.list(resource)
|
|
229
|
-
api.fill_details!(resource, list) if resource == "dashboard"
|
|
230
|
-
end
|
|
231
|
-
list.each do |r|
|
|
232
|
-
r[:api_resource] = resource
|
|
233
|
-
if first
|
|
234
|
-
first = false
|
|
235
|
-
else
|
|
236
|
-
Kennel.out.puts ","
|
|
237
|
-
end
|
|
238
|
-
Kennel.out.print JSON.pretty_generate(r)
|
|
239
|
-
end
|
|
240
|
-
end
|
|
241
|
-
Kennel.out.puts "\n]"
|
|
242
|
-
end
|
|
243
|
-
|
|
244
|
-
desc "Find items from dump by pattern DUMP= PATTERN= [URLS=true]"
|
|
245
|
-
task dump_grep: :environment do
|
|
246
|
-
file = ENV.fetch("DUMP")
|
|
247
|
-
pattern = Regexp.new ENV.fetch("PATTERN")
|
|
248
|
-
items = File.read(file)[2..-2].gsub("},\n{", "}--SPLIT--{").split("--SPLIT--")
|
|
249
|
-
models = Kennel::Models::Record.api_resource_map
|
|
250
|
-
found = items.grep(pattern)
|
|
251
|
-
exit 1 if found.empty?
|
|
252
|
-
found.each do |resource|
|
|
253
|
-
if ENV["URLS"]
|
|
254
|
-
parsed = JSON.parse(resource)
|
|
255
|
-
url = models[parsed.fetch("api_resource")].url(parsed.fetch("id"))
|
|
256
|
-
title = parsed["title"] || parsed["name"]
|
|
257
|
-
Kennel.out.puts "#{url} # #{title}"
|
|
258
|
-
else
|
|
259
|
-
Kennel.out.puts resource
|
|
260
|
-
end
|
|
261
|
-
end
|
|
262
|
-
end
|
|
263
|
-
|
|
264
|
-
desc "Resolve given id to kennel tracking-id RESOURCE= ID="
|
|
265
|
-
task tracking_id: "kennel:environment" do
|
|
266
|
-
resource = ENV.fetch("RESOURCE")
|
|
267
|
-
id = ENV.fetch("ID")
|
|
268
|
-
klass =
|
|
269
|
-
Kennel::Models::Record.subclasses.detect { |s| s.api_resource == resource } ||
|
|
270
|
-
raise("resource #{resource} not know")
|
|
271
|
-
object = Kennel::Api.new.show(resource, id)
|
|
272
|
-
Kennel.out.puts klass.parse_tracking_id(object)
|
|
273
|
-
end
|
|
274
|
-
|
|
275
108
|
task :environment do
|
|
276
109
|
Kennel::Tasks.load_environment
|
|
277
110
|
end
|
data/lib/kennel/version.rb
CHANGED
data/template/Readme.md
CHANGED
|
@@ -426,6 +426,9 @@ Run `rake kennel:alerts TAG=service:my-service` to see all un-muted alerts for a
|
|
|
426
426
|
Use `KNOWN=foo@bar.com,baz@bar.com` to exempt mentions that are not returned by the API.
|
|
427
427
|
Use `KNOWN_RANDOM=@sns-foo,@sns-bar` to ignore for example SNS handles that are randomly invalid in the API.
|
|
428
428
|
|
|
429
|
+
### Validating planned changes
|
|
430
|
+
`rake kennel:validate_plan` validates planned monitor and dashboard changes against the Datadog API.
|
|
431
|
+
|
|
429
432
|
### Grepping through all of datadog
|
|
430
433
|
```Bash
|
|
431
434
|
rake kennel:dump > tmp/dump
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: kennel
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 2.
|
|
4
|
+
version: 2.21.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Michael Grosser
|
|
@@ -117,6 +117,12 @@ files:
|
|
|
117
117
|
- lib/kennel/syncer/types.rb
|
|
118
118
|
- lib/kennel/tags_validation.rb
|
|
119
119
|
- lib/kennel/tasks.rb
|
|
120
|
+
- lib/kennel/tasks/dump.rb
|
|
121
|
+
- lib/kennel/tasks/import.rb
|
|
122
|
+
- lib/kennel/tasks/nodata.rb
|
|
123
|
+
- lib/kennel/tasks/tracking_id.rb
|
|
124
|
+
- lib/kennel/tasks/validate_mentions.rb
|
|
125
|
+
- lib/kennel/tasks/validate_plan.rb
|
|
120
126
|
- lib/kennel/template_variables.rb
|
|
121
127
|
- lib/kennel/unmuted_alerts.rb
|
|
122
128
|
- lib/kennel/utils.rb
|