kennel 1.74.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/Readme.md +289 -0
- data/lib/kennel.rb +90 -0
- data/lib/kennel/api.rb +83 -0
- data/lib/kennel/file_cache.rb +53 -0
- data/lib/kennel/github_reporter.rb +49 -0
- data/lib/kennel/importer.rb +135 -0
- data/lib/kennel/models/base.rb +29 -0
- data/lib/kennel/models/dashboard.rb +209 -0
- data/lib/kennel/models/monitor.rb +219 -0
- data/lib/kennel/models/project.rb +31 -0
- data/lib/kennel/models/record.rb +94 -0
- data/lib/kennel/models/slo.rb +92 -0
- data/lib/kennel/models/team.rb +12 -0
- data/lib/kennel/optional_validations.rb +21 -0
- data/lib/kennel/progress.rb +34 -0
- data/lib/kennel/settings_as_methods.rb +86 -0
- data/lib/kennel/subclass_tracking.rb +19 -0
- data/lib/kennel/syncer.rb +260 -0
- data/lib/kennel/tasks.rb +147 -0
- data/lib/kennel/template_variables.rb +38 -0
- data/lib/kennel/unmuted_alerts.rb +89 -0
- data/lib/kennel/utils.rb +159 -0
- data/lib/kennel/version.rb +4 -0
- data/template/Readme.md +247 -0
- metadata +109 -0
@@ -0,0 +1,53 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# cache that reads everything from a single file
|
4
|
+
# to avoid doing multiple disk reads while iterating all definitions
|
5
|
+
# it also replaces updated keys and has an overall expiry to not keep deleted things forever
|
6
|
+
module Kennel
|
7
|
+
class FileCache
|
8
|
+
def initialize(file, cache_version)
|
9
|
+
@file = file
|
10
|
+
@cache_version = cache_version
|
11
|
+
@now = Time.now.to_i
|
12
|
+
@expires = @now + (30 * 24 * 60 * 60) # 1 month
|
13
|
+
end
|
14
|
+
|
15
|
+
def open
|
16
|
+
load_data
|
17
|
+
expire_old_data
|
18
|
+
yield self
|
19
|
+
ensure
|
20
|
+
persist
|
21
|
+
end
|
22
|
+
|
23
|
+
def fetch(key, key_version)
|
24
|
+
old_value, old_version = @data[key]
|
25
|
+
return old_value if old_version == [key_version, @cache_version]
|
26
|
+
|
27
|
+
new_value = yield
|
28
|
+
@data[key] = [new_value, [key_version, @cache_version], @expires]
|
29
|
+
new_value
|
30
|
+
end
|
31
|
+
|
32
|
+
private
|
33
|
+
|
34
|
+
def load_data
|
35
|
+
@data =
|
36
|
+
begin
|
37
|
+
Marshal.load(File.read(@file)) # rubocop:disable Security/MarshalLoad
|
38
|
+
rescue StandardError
|
39
|
+
{}
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
def persist
|
44
|
+
dir = File.dirname(@file)
|
45
|
+
FileUtils.mkdir_p(dir) unless File.directory?(dir)
|
46
|
+
File.write(@file, Marshal.dump(@data))
|
47
|
+
end
|
48
|
+
|
49
|
+
def expire_old_data
|
50
|
+
@data.reject! { |_, (_, _, ex)| ex < @now }
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
@@ -0,0 +1,49 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
# Not used in here, but in our templated repo ... so keeping it around for now.
|
3
|
+
module Kennel
|
4
|
+
class GithubReporter
|
5
|
+
MAX_COMMENT_SIZE = 65536
|
6
|
+
TRUNCATED_MSG = "\n```\n... (truncated)" # finish the code block so it look nice
|
7
|
+
|
8
|
+
class << self
|
9
|
+
def report(token, &block)
|
10
|
+
return yield unless token
|
11
|
+
new(token, Utils.capture_sh("git rev-parse HEAD").strip).report(&block)
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
def initialize(token, git_sha)
|
16
|
+
@token = token
|
17
|
+
@git_sha = git_sha
|
18
|
+
origin = ENV["PROJECT_REPOSITORY"] || Utils.capture_sh("git remote -v").split("\n").first
|
19
|
+
@repo_part = origin[%r{github\.com[:/](.+?)(\.git|$)}, 1] || raise("no origin found")
|
20
|
+
end
|
21
|
+
|
22
|
+
def report(&block)
|
23
|
+
output = Utils.strip_shell_control(Utils.tee_output(&block).strip)
|
24
|
+
rescue StandardError
|
25
|
+
output = "Error:\n#{$ERROR_INFO.message}"
|
26
|
+
raise
|
27
|
+
ensure
|
28
|
+
comment "```\n#{output || "Error"}\n```"
|
29
|
+
end
|
30
|
+
|
31
|
+
# https://developer.github.com/v3/repos/comments/#create-a-commit-comment
|
32
|
+
def comment(body)
|
33
|
+
# truncate to maximum allowed comment size for github to avoid 422
|
34
|
+
if body.bytesize > MAX_COMMENT_SIZE
|
35
|
+
body = body.byteslice(0, MAX_COMMENT_SIZE - TRUNCATED_MSG.bytesize) + TRUNCATED_MSG
|
36
|
+
end
|
37
|
+
|
38
|
+
post "commits/#{@git_sha}/comments", body: body
|
39
|
+
end
|
40
|
+
|
41
|
+
private
|
42
|
+
|
43
|
+
def post(path, data)
|
44
|
+
url = "https://api.github.com/repos/#{@repo_part}/#{path}"
|
45
|
+
response = Faraday.post(url, data.to_json, authorization: "token #{@token}")
|
46
|
+
raise "failed to POST to github:\n#{url} -> #{response.status}\n#{response.body}" unless response.status == 201
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
@@ -0,0 +1,135 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Kennel
|
4
|
+
class Importer
|
5
|
+
TITLES = [:name, :title].freeze
|
6
|
+
SORT_ORDER = [*TITLES, :id, :kennel_id, :type, :tags, :query, *Syncer::TRACKING_FIELDS, :template_variables].freeze
|
7
|
+
|
8
|
+
def initialize(api)
|
9
|
+
@api = api
|
10
|
+
end
|
11
|
+
|
12
|
+
def import(resource, id)
|
13
|
+
if ["screen", "dash"].include?(resource)
|
14
|
+
raise ArgumentError, "resource 'screen' and 'dash' are deprecated, use 'dashboard'"
|
15
|
+
end
|
16
|
+
|
17
|
+
model =
|
18
|
+
begin
|
19
|
+
Kennel::Models.const_get(resource.capitalize)
|
20
|
+
rescue NameError
|
21
|
+
raise ArgumentError, "#{resource} is not supported"
|
22
|
+
end
|
23
|
+
|
24
|
+
data = @api.show(model.api_resource, id)
|
25
|
+
id = data.fetch(:id) # keep native value
|
26
|
+
model.normalize({}, data) # removes id
|
27
|
+
data[:id] = id
|
28
|
+
|
29
|
+
title_field = TITLES.detect { |f| data[f] }
|
30
|
+
title = data.fetch(title_field)
|
31
|
+
title.tr!(Kennel::Models::Record::LOCK, "") # avoid double lock icon
|
32
|
+
|
33
|
+
# calculate or reuse kennel_id
|
34
|
+
# TODO: this is copy-pasted from syncer, need to find a nice way to reuse it
|
35
|
+
tracking_field = Syncer::TRACKING_FIELDS.detect { |f| data[f] }
|
36
|
+
data[:kennel_id] =
|
37
|
+
if tracking_field && data[tracking_field].sub!(/\n?-- Managed by kennel (\S+:\S+).*/, "")
|
38
|
+
$1.split(":").last
|
39
|
+
else
|
40
|
+
Kennel::Utils.parameterize(title)
|
41
|
+
end
|
42
|
+
|
43
|
+
case resource
|
44
|
+
when "monitor"
|
45
|
+
# flatten monitor options so they are all on the base
|
46
|
+
data.merge!(data.delete(:options))
|
47
|
+
data.merge!(data.delete(:thresholds) || {})
|
48
|
+
[:notify_no_data, :notify_audit].each { |k| data.delete(k) if data[k] } # monitor uses true by default
|
49
|
+
data = data.slice(*model.instance_methods)
|
50
|
+
|
51
|
+
# make query use critical method if it matches
|
52
|
+
critical = data[:critical]
|
53
|
+
query = data[:query]
|
54
|
+
if query && critical
|
55
|
+
query.sub!(/([><=]) (#{Regexp.escape(critical.to_f.to_s)}|#{Regexp.escape(critical.to_i.to_s)})$/, "\\1 \#{critical}")
|
56
|
+
end
|
57
|
+
|
58
|
+
data[:type] = "query alert" if data[:type] == "metric alert"
|
59
|
+
when "dashboard"
|
60
|
+
widgets = data[:widgets]&.flat_map { |widget| widget.dig(:definition, :widgets) || [widget] }
|
61
|
+
widgets&.each { |widget| dry_up_query!(widget) }
|
62
|
+
end
|
63
|
+
|
64
|
+
# simplify template_variables to array of string when possible
|
65
|
+
if vars = data[:template_variables]
|
66
|
+
vars.map! { |v| v[:default] == "*" && v[:prefix] == v[:name] ? v[:name] : v }
|
67
|
+
end
|
68
|
+
|
69
|
+
pretty = pretty_print(data).lstrip.gsub("\\#", "#")
|
70
|
+
<<~RUBY
|
71
|
+
#{model.name}.new(
|
72
|
+
self,
|
73
|
+
#{pretty}
|
74
|
+
)
|
75
|
+
RUBY
|
76
|
+
end
|
77
|
+
|
78
|
+
private
|
79
|
+
|
80
|
+
# reduce duplication in imports by using dry `q: :metadata` when possible
|
81
|
+
def dry_up_query!(widget)
|
82
|
+
(widget.dig(:definition, :requests) || []).each do |request|
|
83
|
+
next unless request.is_a?(Hash)
|
84
|
+
next unless metadata = request[:metadata]
|
85
|
+
next unless query = request[:q]&.dup
|
86
|
+
metadata.each do |m|
|
87
|
+
next unless exp = m[:expression]
|
88
|
+
query.sub!(exp, "")
|
89
|
+
end
|
90
|
+
request[:q] = :metadata if query.delete(", ") == ""
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
def pretty_print(hash)
|
95
|
+
sort_widgets hash
|
96
|
+
|
97
|
+
sort_hash(hash).map do |k, v|
|
98
|
+
pretty_value =
|
99
|
+
if v.is_a?(Hash) || (v.is_a?(Array) && !v.all? { |e| e.is_a?(String) })
|
100
|
+
# update answer here when changing https://stackoverflow.com/questions/8842546/best-way-to-pretty-print-a-hash
|
101
|
+
# (exclude last indent gsub)
|
102
|
+
pretty = JSON.pretty_generate(v)
|
103
|
+
.gsub(": null", ": nil")
|
104
|
+
.gsub(/(^\s*)"([a-zA-Z][a-zA-Z\d_]*)":/, "\\1\\2:") # "foo": 1 -> foo: 1
|
105
|
+
.gsub(/: \[\n\s+\]/, ": []") # empty arrays on a single line
|
106
|
+
.gsub(/^/, " ") # indent
|
107
|
+
.gsub('q: "metadata"', "q: :metadata") # bring symbols back
|
108
|
+
|
109
|
+
"\n#{pretty}\n "
|
110
|
+
elsif k == :message
|
111
|
+
"\n <<~TEXT\n#{v.each_line.map { |l| l.strip.empty? ? "\n" : " #{l}" }.join}\n \#{super()}\n TEXT\n "
|
112
|
+
elsif k == :tags
|
113
|
+
" super() + #{v.inspect} "
|
114
|
+
else
|
115
|
+
" #{v.inspect} "
|
116
|
+
end
|
117
|
+
" #{k}: -> {#{pretty_value}}"
|
118
|
+
end.join(",\n")
|
119
|
+
end
|
120
|
+
|
121
|
+
# sort dashboard widgets + nesting
|
122
|
+
def sort_widgets(outer)
|
123
|
+
outer[:widgets]&.each do |widgets|
|
124
|
+
definition = widgets[:definition]
|
125
|
+
definition.replace sort_hash(definition)
|
126
|
+
sort_widgets definition
|
127
|
+
end
|
128
|
+
end
|
129
|
+
|
130
|
+
# important to the front and rest deterministic
|
131
|
+
def sort_hash(hash)
|
132
|
+
Hash[hash.sort_by { |k, _| [SORT_ORDER.index(k) || 999, k] }]
|
133
|
+
end
|
134
|
+
end
|
135
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
require "hashdiff"
|
3
|
+
|
4
|
+
module Kennel
|
5
|
+
module Models
|
6
|
+
class Base
|
7
|
+
extend SubclassTracking
|
8
|
+
include SettingsAsMethods
|
9
|
+
|
10
|
+
SETTING_OVERRIDABLE_METHODS = [:name, :kennel_id].freeze
|
11
|
+
|
12
|
+
def kennel_id
|
13
|
+
name = self.class.name
|
14
|
+
if name.start_with?("Kennel::")
|
15
|
+
raise_with_location ArgumentError, "Set :kennel_id"
|
16
|
+
end
|
17
|
+
@kennel_id ||= Utils.snake_case name
|
18
|
+
end
|
19
|
+
|
20
|
+
def name
|
21
|
+
self.class.name
|
22
|
+
end
|
23
|
+
|
24
|
+
def to_json # rubocop:disable Lint/ToJSON
|
25
|
+
raise NotImplementedError, "Use as_json"
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,209 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
module Kennel
|
3
|
+
module Models
|
4
|
+
class Dashboard < Record
|
5
|
+
include TemplateVariables
|
6
|
+
include OptionalValidations
|
7
|
+
|
8
|
+
API_LIST_INCOMPLETE = true
|
9
|
+
DASHBOARD_DEFAULTS = { template_variables: [] }.freeze
|
10
|
+
READONLY_ATTRIBUTES = superclass::READONLY_ATTRIBUTES + [
|
11
|
+
:author_handle, :author_name, :modified_at, :url, :is_read_only, :notify_list
|
12
|
+
]
|
13
|
+
REQUEST_DEFAULTS = {
|
14
|
+
style: { line_width: "normal", palette: "dog_classic", line_type: "solid" }
|
15
|
+
}.freeze
|
16
|
+
WIDGET_DEFAULTS = {
|
17
|
+
"timeseries" => { show_legend: false, legend_size: "0" },
|
18
|
+
"note" => { background_color: "white", font_size: "14", show_tick: false, tick_edge: "left", tick_pos: "50%", text_align: "left" }
|
19
|
+
}.freeze
|
20
|
+
SUPPORTED_DEFINITION_OPTIONS = [:events, :markers, :precision].freeze
|
21
|
+
|
22
|
+
DEFAULTS = {
|
23
|
+
template_variable_presets: nil
|
24
|
+
}.freeze
|
25
|
+
|
26
|
+
settings :title, :description, :definitions, :widgets, :layout_type, :template_variable_presets
|
27
|
+
|
28
|
+
defaults(
|
29
|
+
description: -> { "" },
|
30
|
+
definitions: -> { [] },
|
31
|
+
widgets: -> { [] },
|
32
|
+
template_variable_presets: -> { DEFAULTS.fetch(:template_variable_presets) },
|
33
|
+
id: -> { nil }
|
34
|
+
)
|
35
|
+
|
36
|
+
class << self
|
37
|
+
def api_resource
|
38
|
+
"dashboard"
|
39
|
+
end
|
40
|
+
|
41
|
+
def normalize(expected, actual)
|
42
|
+
super
|
43
|
+
|
44
|
+
ignore_default(expected, actual, DEFAULTS)
|
45
|
+
|
46
|
+
widgets_pairs(expected, actual).each do |pair|
|
47
|
+
# conditional_formats ordering is randomly changed by datadog, compare a stable ordering
|
48
|
+
pair.each do |widgets|
|
49
|
+
widgets.each do |widget|
|
50
|
+
if formats = widget.dig(:definition, :conditional_formats)
|
51
|
+
widget[:definition][:conditional_formats] = formats.sort_by(&:hash)
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
ignore_widget_defaults pair
|
57
|
+
|
58
|
+
ignore_request_defaults(*pair)
|
59
|
+
|
60
|
+
# ids are kinda random so we always discard them
|
61
|
+
pair.each { |widgets| widgets.each { |w| w.delete(:id) } }
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
private
|
66
|
+
|
67
|
+
def ignore_widget_defaults(pair)
|
68
|
+
pair.map(&:size).max.times do |i|
|
69
|
+
types = pair.map { |w| w.dig(i, :definition, :type) }.uniq
|
70
|
+
next unless types.size == 1
|
71
|
+
next unless defaults = WIDGET_DEFAULTS[types.first]
|
72
|
+
ignore_defaults(pair[0], pair[1], defaults, nesting: :definition)
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
# discard styles/conditional_formats/aggregator if nothing would change when we applied (both are default or nil)
|
77
|
+
def ignore_request_defaults(expected, actual)
|
78
|
+
[expected.size, actual.size].max.times do |i|
|
79
|
+
a_r = actual.dig(i, :definition, :requests) || []
|
80
|
+
e_r = expected.dig(i, :definition, :requests) || []
|
81
|
+
ignore_defaults e_r, a_r, REQUEST_DEFAULTS
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
def ignore_defaults(expected, actual, defaults, nesting: nil)
|
86
|
+
[expected.size, actual.size].max.times do |i|
|
87
|
+
e = expected.dig(i, *nesting) || {}
|
88
|
+
a = actual.dig(i, *nesting) || {}
|
89
|
+
ignore_default(e, a, defaults)
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
# expand nested widgets into expected/actual pairs for default resolution
|
94
|
+
# [a, e] -> [[a-w, e-w], [a-w1-w1, e-w1-w1], ...]
|
95
|
+
def widgets_pairs(*pair)
|
96
|
+
result = [pair.map { |d| d[:widgets] || [] }]
|
97
|
+
slots = result[0].map(&:size).max
|
98
|
+
slots.times do |i|
|
99
|
+
nested = pair.map { |d| d.dig(:widgets, i, :definition, :widgets) || [] }
|
100
|
+
result << nested if nested.any?(&:any?)
|
101
|
+
end
|
102
|
+
result
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
106
|
+
def as_json
|
107
|
+
return @json if @json
|
108
|
+
all_widgets = render_definitions + widgets
|
109
|
+
expand_q all_widgets
|
110
|
+
|
111
|
+
@json = {
|
112
|
+
layout_type: layout_type,
|
113
|
+
title: "#{title}#{LOCK}",
|
114
|
+
description: description,
|
115
|
+
template_variables: render_template_variables,
|
116
|
+
template_variable_presets: template_variable_presets,
|
117
|
+
widgets: all_widgets
|
118
|
+
}
|
119
|
+
|
120
|
+
@json[:id] = id if id
|
121
|
+
|
122
|
+
validate_json(@json) if validate
|
123
|
+
|
124
|
+
@json
|
125
|
+
end
|
126
|
+
|
127
|
+
def url(id)
|
128
|
+
Utils.path_to_url "/dashboard/#{id}"
|
129
|
+
end
|
130
|
+
|
131
|
+
def self.parse_url(url)
|
132
|
+
url[/\/dashboard\/([a-z\d-]+)/, 1]
|
133
|
+
end
|
134
|
+
|
135
|
+
def resolve_linked_tracking_ids!(id_map, **args)
|
136
|
+
widgets = as_json[:widgets].flat_map { |w| [w, *w.dig(:definition, :widgets) || []] }
|
137
|
+
widgets.each do |widget|
|
138
|
+
next unless definition = widget[:definition]
|
139
|
+
case definition[:type]
|
140
|
+
when "uptime"
|
141
|
+
if ids = definition[:monitor_ids]
|
142
|
+
definition[:monitor_ids] = ids.map do |id|
|
143
|
+
tracking_id?(id) ? resolve_link(id, :monitor, id_map, **args) : id
|
144
|
+
end
|
145
|
+
end
|
146
|
+
when "alert_graph"
|
147
|
+
if (id = definition[:alert_id]) && tracking_id?(id)
|
148
|
+
definition[:alert_id] = resolve_link(id, :monitor, id_map, **args).to_s
|
149
|
+
end
|
150
|
+
when "slo"
|
151
|
+
if (id = definition[:slo_id]) && tracking_id?(id)
|
152
|
+
definition[:slo_id] = resolve_link(id, :slo, id_map, **args).to_s
|
153
|
+
end
|
154
|
+
end
|
155
|
+
end
|
156
|
+
end
|
157
|
+
|
158
|
+
private
|
159
|
+
|
160
|
+
def tracking_id?(id)
|
161
|
+
id.is_a?(String) && id.include?(":")
|
162
|
+
end
|
163
|
+
|
164
|
+
# creates queries from metadata to avoid having to keep q and expression in sync
|
165
|
+
#
|
166
|
+
# {q: :metadata, metadata: [{expression: "sum:bar", alias_name: "foo"}, ...], }
|
167
|
+
# -> {q: "sum:bar, ...", metadata: ..., }
|
168
|
+
def expand_q(widgets)
|
169
|
+
widgets = widgets.flat_map { |w| w.dig(:definition, :widgets) || w } # expand groups
|
170
|
+
widgets.each do |w|
|
171
|
+
w.dig(:definition, :requests)&.each do |request|
|
172
|
+
next unless request.is_a?(Hash) && request[:q] == :metadata
|
173
|
+
request[:q] = request.fetch(:metadata).map { |m| m.fetch(:expression) }.join(", ")
|
174
|
+
end
|
175
|
+
end
|
176
|
+
end
|
177
|
+
|
178
|
+
def validate_json(data)
|
179
|
+
super
|
180
|
+
|
181
|
+
validate_template_variables data, :widgets
|
182
|
+
|
183
|
+
# Avoid diff from datadog presets sorting.
|
184
|
+
presets = data[:template_variable_presets]
|
185
|
+
invalid! "template_variable_presets must be sorted by name" if presets && presets != presets.sort_by { |p| p[:name] }
|
186
|
+
end
|
187
|
+
|
188
|
+
def render_definitions
|
189
|
+
definitions.map do |title, type, display_type, queries, options = {}, ignored = nil|
|
190
|
+
# validate inputs
|
191
|
+
if ignored || (!title || !type || !queries || !options.is_a?(Hash))
|
192
|
+
raise ArgumentError, "Expected exactly 5 arguments for each definition (title, type, display_type, queries, options)"
|
193
|
+
end
|
194
|
+
if (SUPPORTED_DEFINITION_OPTIONS | options.keys) != SUPPORTED_DEFINITION_OPTIONS
|
195
|
+
raise ArgumentError, "Supported options are: #{SUPPORTED_DEFINITION_OPTIONS.map(&:inspect).join(", ")}"
|
196
|
+
end
|
197
|
+
|
198
|
+
# build definition
|
199
|
+
requests = Array(queries).map do |q|
|
200
|
+
request = { q: q }
|
201
|
+
request[:display_type] = display_type if display_type
|
202
|
+
request
|
203
|
+
end
|
204
|
+
{ definition: { title: title, type: type, requests: requests, **options } }
|
205
|
+
end
|
206
|
+
end
|
207
|
+
end
|
208
|
+
end
|
209
|
+
end
|