@intentius/chant-lexicon-prometheus 0.97.0 → 0.99.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -1
- package/dist/alertmanager.d.ts +4 -2
- package/dist/alertmanager.d.ts.map +1 -1
- package/dist/build.d.ts.map +1 -1
- package/dist/codegen/docs.d.ts.map +1 -1
- package/dist/import/embedded.d.ts +14 -0
- package/dist/import/embedded.d.ts.map +1 -0
- package/dist/import/generator.d.ts +50 -0
- package/dist/import/generator.d.ts.map +1 -0
- package/dist/import/parser.d.ts +52 -0
- package/dist/import/parser.d.ts.map +1 -0
- package/dist/import/slo.d.ts +15 -0
- package/dist/import/slo.d.ts.map +1 -0
- package/dist/integrations.d.ts +531 -0
- package/dist/integrations.d.ts.map +1 -0
- package/dist/integrity.json +7 -7
- package/dist/lint/rules/literal-credential.d.ts.map +1 -1
- package/dist/lint/rules/prom-ast.d.ts +8 -0
- package/dist/lint/rules/prom-ast.d.ts.map +1 -1
- package/dist/lint/rules/promql-literal.d.ts.map +1 -1
- package/dist/manifest.json +1 -1
- package/dist/model.d.ts +77 -121
- package/dist/model.d.ts.map +1 -1
- package/dist/plugin.d.ts.map +1 -1
- package/dist/rules/literal-credential.ts +52 -15
- package/dist/rules/prom-ast.ts +36 -0
- package/dist/rules/promql-literal.ts +15 -1
- package/dist/skills/chant-prometheus-alertmanager.md +4 -0
- package/dist/skills/chant-prometheus.md +11 -0
- package/package.json +2 -2
- package/src/alertmanager.ts +4 -1
- package/src/build.ts +2 -0
- package/src/codegen/docs.ts +5 -1
- package/src/import/cli.test.ts +98 -0
- package/src/import/embedded.test.ts +56 -0
- package/src/import/embedded.ts +44 -0
- package/src/import/generated-types.e2e.test.ts +77 -0
- package/src/import/generator.test.ts +138 -0
- package/src/import/generator.ts +578 -0
- package/src/import/parser.test.ts +133 -0
- package/src/import/parser.ts +365 -0
- package/src/import/roundtrip.test.ts +358 -0
- package/src/import/slo.ts +175 -0
- package/src/import/testdata/alertmanager-full.yml +89 -0
- package/src/import/testdata/alertmanager-integrations.yml +215 -0
- package/src/import/testdata/fixtures.ts +122 -0
- package/src/import/testdata/rules-full.yml +42 -0
- package/src/import/testdata/upstream/alertmanager-conf-good.yml +120 -0
- package/src/import/testdata/upstream/alertmanager-mattermost-default-webhook-url-file.yml +16 -0
- package/src/import/testdata/upstream/alertmanager-opsgenie-default-apikey-file.yml +27 -0
- package/src/import/testdata/upstream/alertmanager-rocketchat-default-token-file.yml +25 -0
- package/src/import/testdata/upstream/alertmanager-route-labels.yml +65 -0
- package/src/import/testdata/upstream/alertmanager-simple.yml +130 -0
- package/src/import/testdata/upstream/alertmanager-sns-topic-arn.yml +17 -0
- package/src/import/testdata/upstream/alertmanager-telegram-default-bot-token-file.yml +16 -0
- package/src/import/testdata/upstream/alertmanager-victorops-default-apikey-file.yml +23 -0
- package/src/import/testdata/upstream/alertmanager-wechat-default-api-secret-file.yml +18 -0
- package/src/import/testdata/upstream/prometheus-alerting-rules.yml +15 -0
- package/src/import/testdata/upstream/prometheus-alerting-templates.yml +23 -0
- package/src/import/testdata/upstream/prometheus-recording-rules.yml +7 -0
- package/src/integrations.ts +538 -0
- package/src/lint/rules/literal-credential.ts +52 -15
- package/src/lint/rules/prom-ast.ts +36 -0
- package/src/lint/rules/promql-literal.ts +15 -1
- package/src/lint/rules/rules.test.ts +58 -0
- package/src/model.ts +157 -109
- package/src/plugin.ts +15 -0
- package/src/skills/chant-prometheus-alertmanager.md +4 -0
- package/src/skills/chant-prometheus.md +11 -0
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
# Every receiver integration Alertmanager v0.34.1 defines, each with the
|
|
2
|
+
# fields its struct lists beyond the destination, every credential read from
|
|
3
|
+
# a *_file, and every global field that has a default for one of them.
|
|
4
|
+
global:
|
|
5
|
+
resolve_timeout: 5m
|
|
6
|
+
http_config:
|
|
7
|
+
follow_redirects: true
|
|
8
|
+
enable_http2: false
|
|
9
|
+
http_headers:
|
|
10
|
+
X-Tenant:
|
|
11
|
+
values: [platform]
|
|
12
|
+
X-Api-Key:
|
|
13
|
+
files: [/etc/alertmanager/secrets/api-key-header]
|
|
14
|
+
jira_api_url: https://jira.example.com/rest/api/2/
|
|
15
|
+
smtp_from: alertmanager@example.com
|
|
16
|
+
smtp_smarthost: smtp.example.com:465
|
|
17
|
+
smtp_auth_username: alertmanager
|
|
18
|
+
smtp_auth_password_file: /etc/alertmanager/secrets/smtp-password
|
|
19
|
+
smtp_require_tls: false
|
|
20
|
+
smtp_force_implicit_tls: true
|
|
21
|
+
smtp_tls_config:
|
|
22
|
+
ca_file: /etc/alertmanager/tls/ca.pem
|
|
23
|
+
min_version: TLS12
|
|
24
|
+
slack_app_token_file: /etc/alertmanager/secrets/slack-app-token
|
|
25
|
+
slack_app_url: https://slack.com/api/chat.postMessage
|
|
26
|
+
opsgenie_api_url: https://api.eu.opsgenie.com/
|
|
27
|
+
opsgenie_api_key_file: /etc/alertmanager/secrets/opsgenie-key
|
|
28
|
+
wechat_api_url: https://qyapi.weixin.qq.com/cgi-bin/
|
|
29
|
+
wechat_api_secret_file: /etc/alertmanager/secrets/wechat-secret
|
|
30
|
+
wechat_api_corp_id: corp-1
|
|
31
|
+
victorops_api_url: https://alert.victorops.com/integrations/generic/20131114/alert/
|
|
32
|
+
victorops_api_key_file: /etc/alertmanager/secrets/victorops-key
|
|
33
|
+
telegram_api_url: https://api.telegram.org
|
|
34
|
+
telegram_bot_token_file: /etc/alertmanager/secrets/telegram-token
|
|
35
|
+
webex_api_url: https://webexapis.com/v1/messages
|
|
36
|
+
rocketchat_api_url: https://open.rocket.chat
|
|
37
|
+
rocketchat_token_file: /etc/alertmanager/secrets/rocketchat-token
|
|
38
|
+
rocketchat_token_id_file: /etc/alertmanager/secrets/rocketchat-token-id
|
|
39
|
+
mattermost_webhook_url_file: /etc/alertmanager/secrets/mattermost-url
|
|
40
|
+
route:
|
|
41
|
+
receiver: chat
|
|
42
|
+
group_by: [alertname]
|
|
43
|
+
routes:
|
|
44
|
+
- matchers: ['team="ops"']
|
|
45
|
+
receiver: ops
|
|
46
|
+
- matchers: ['team="tickets"']
|
|
47
|
+
receiver: tickets
|
|
48
|
+
- matchers: ['team="mobile"']
|
|
49
|
+
receiver: mobile
|
|
50
|
+
receivers:
|
|
51
|
+
- name: chat
|
|
52
|
+
slack_configs:
|
|
53
|
+
- channel: '#alerts'
|
|
54
|
+
message_text: '{{ .CommonLabels.alertname }}'
|
|
55
|
+
callback_id: alerts
|
|
56
|
+
image_url: https://example.com/graph.png
|
|
57
|
+
thumb_url: https://example.com/thumb.png
|
|
58
|
+
update_message: true
|
|
59
|
+
timeout: 10s
|
|
60
|
+
fields:
|
|
61
|
+
- title: Severity
|
|
62
|
+
value: '{{ .CommonLabels.severity }}'
|
|
63
|
+
short: true
|
|
64
|
+
actions:
|
|
65
|
+
- type: button
|
|
66
|
+
text: Runbook
|
|
67
|
+
url: '{{ .CommonAnnotations.runbook_url }}'
|
|
68
|
+
style: primary
|
|
69
|
+
confirm:
|
|
70
|
+
text: Open the runbook?
|
|
71
|
+
ok_text: 'Yes'
|
|
72
|
+
dismiss_text: 'No'
|
|
73
|
+
discord_configs:
|
|
74
|
+
- webhook_url_file: /etc/alertmanager/secrets/discord-url
|
|
75
|
+
title: '{{ .CommonLabels.alertname }}'
|
|
76
|
+
message: '{{ .CommonAnnotations.summary }}'
|
|
77
|
+
content: '<@&123456>'
|
|
78
|
+
username: alertmanager
|
|
79
|
+
avatar_url: https://example.com/avatar.png
|
|
80
|
+
msteams_configs:
|
|
81
|
+
- webhook_url_file: /etc/alertmanager/secrets/teams-url
|
|
82
|
+
title: '{{ .CommonLabels.alertname }}'
|
|
83
|
+
summary: '{{ .CommonAnnotations.summary }}'
|
|
84
|
+
text: '{{ .CommonAnnotations.description }}'
|
|
85
|
+
msteamsv2_configs:
|
|
86
|
+
- webhook_url_file: /etc/alertmanager/secrets/teams-workflow-url
|
|
87
|
+
title: '{{ .CommonLabels.alertname }}'
|
|
88
|
+
text: '{{ .CommonAnnotations.description }}'
|
|
89
|
+
mattermost_configs:
|
|
90
|
+
- channel: alerts
|
|
91
|
+
username: alertmanager
|
|
92
|
+
text: '{{ .CommonAnnotations.summary }}'
|
|
93
|
+
icon_emoji: ':rotating_light:'
|
|
94
|
+
attachments:
|
|
95
|
+
- title: '{{ .CommonLabels.alertname }}'
|
|
96
|
+
color: danger
|
|
97
|
+
fields:
|
|
98
|
+
- title: Severity
|
|
99
|
+
value: '{{ .CommonLabels.severity }}'
|
|
100
|
+
short: true
|
|
101
|
+
props:
|
|
102
|
+
card: '{{ .CommonAnnotations.description }}'
|
|
103
|
+
priority:
|
|
104
|
+
priority: urgent
|
|
105
|
+
requested_ack: true
|
|
106
|
+
webex_configs:
|
|
107
|
+
- room_id: room-1
|
|
108
|
+
message: '{{ .CommonAnnotations.summary }}'
|
|
109
|
+
http_config:
|
|
110
|
+
authorization:
|
|
111
|
+
credentials_file: /etc/alertmanager/secrets/webex-token
|
|
112
|
+
rocketchat_configs:
|
|
113
|
+
- channel: '#alerts'
|
|
114
|
+
title: '{{ .CommonLabels.alertname }}'
|
|
115
|
+
emoji: ':fire:'
|
|
116
|
+
short_fields: true
|
|
117
|
+
fields:
|
|
118
|
+
- title: Severity
|
|
119
|
+
value: '{{ .CommonLabels.severity }}'
|
|
120
|
+
short: true
|
|
121
|
+
actions:
|
|
122
|
+
- type: button
|
|
123
|
+
text: Silence
|
|
124
|
+
url: https://alertmanager.example.com/#/silences/new
|
|
125
|
+
telegram_configs:
|
|
126
|
+
- chat_id: -1001234567890
|
|
127
|
+
message_thread_id: 7
|
|
128
|
+
parse_mode: HTML
|
|
129
|
+
disable_notifications: false
|
|
130
|
+
message: '{{ .CommonAnnotations.summary }}'
|
|
131
|
+
- name: ops
|
|
132
|
+
email_configs:
|
|
133
|
+
- to: ops@example.com
|
|
134
|
+
force_implicit_tls: true
|
|
135
|
+
threading:
|
|
136
|
+
enabled: true
|
|
137
|
+
thread_by_date: daily
|
|
138
|
+
opsgenie_configs:
|
|
139
|
+
- message: '{{ .CommonLabels.alertname }}'
|
|
140
|
+
priority: P2
|
|
141
|
+
tags: 'team,ops'
|
|
142
|
+
details:
|
|
143
|
+
cluster: '{{ .CommonLabels.cluster }}'
|
|
144
|
+
responders:
|
|
145
|
+
- name: ops
|
|
146
|
+
type: team
|
|
147
|
+
update_alerts: true
|
|
148
|
+
victorops_configs:
|
|
149
|
+
- routing_key: ops
|
|
150
|
+
message_type: CRITICAL
|
|
151
|
+
entity_display_name: '{{ .CommonLabels.alertname }}'
|
|
152
|
+
custom_fields:
|
|
153
|
+
cluster: '{{ .CommonLabels.cluster }}'
|
|
154
|
+
pagerduty_configs:
|
|
155
|
+
- routing_key_file: /etc/alertmanager/secrets/pagerduty-key
|
|
156
|
+
timeout: 20s
|
|
157
|
+
details:
|
|
158
|
+
firing: '{{ .Alerts.Firing | len }}'
|
|
159
|
+
incidentio_configs:
|
|
160
|
+
- url: https://api.incident.io/v2/alert_events/http/01ABC
|
|
161
|
+
alert_source_token_file: /etc/alertmanager/secrets/incidentio-token
|
|
162
|
+
max_alerts: 20
|
|
163
|
+
timeout: 15s
|
|
164
|
+
webhook_configs:
|
|
165
|
+
- url_file: /etc/alertmanager/secrets/webhook-url
|
|
166
|
+
timeout: 5s
|
|
167
|
+
http_config:
|
|
168
|
+
oauth2:
|
|
169
|
+
client_id: alertmanager
|
|
170
|
+
client_secret_file: /etc/alertmanager/secrets/oauth-secret
|
|
171
|
+
token_url: https://auth.example.com/token
|
|
172
|
+
scopes: [alerts]
|
|
173
|
+
- name: tickets
|
|
174
|
+
jira_configs:
|
|
175
|
+
- project: OPS
|
|
176
|
+
issue_type: Bug
|
|
177
|
+
summary: '{{ template "jira.default.summary" . }}'
|
|
178
|
+
description:
|
|
179
|
+
template: '{{ template "jira.default.description" . }}'
|
|
180
|
+
enable_update: false
|
|
181
|
+
labels: [alertmanager]
|
|
182
|
+
priority: High
|
|
183
|
+
resolve_transition: Done
|
|
184
|
+
reopen_transition: Reopen
|
|
185
|
+
wont_fix_resolution: "Won't Fix"
|
|
186
|
+
reopen_duration: 1h
|
|
187
|
+
fields:
|
|
188
|
+
customfield_10001: ops
|
|
189
|
+
http_config:
|
|
190
|
+
basic_auth:
|
|
191
|
+
username: alertmanager
|
|
192
|
+
password_file: /etc/alertmanager/secrets/jira-password
|
|
193
|
+
sns_configs:
|
|
194
|
+
- topic_arn: arn:aws:sns:us-east-1:123456789012:alerts
|
|
195
|
+
subject: '{{ .CommonLabels.alertname }}'
|
|
196
|
+
sigv4:
|
|
197
|
+
region: us-east-1
|
|
198
|
+
role_arn: arn:aws:iam::123456789012:role/alertmanager
|
|
199
|
+
attributes:
|
|
200
|
+
team: ops
|
|
201
|
+
- name: mobile
|
|
202
|
+
pushover_configs:
|
|
203
|
+
- user_key_file: /etc/alertmanager/secrets/pushover-user
|
|
204
|
+
token_file: /etc/alertmanager/secrets/pushover-token
|
|
205
|
+
priority: '{{ if eq .Status "firing" }}2{{ else }}0{{ end }}'
|
|
206
|
+
retry: 1m
|
|
207
|
+
expire: 1h
|
|
208
|
+
ttl: 24h
|
|
209
|
+
sound: siren
|
|
210
|
+
html: true
|
|
211
|
+
wechat_configs:
|
|
212
|
+
- to_user: '@all'
|
|
213
|
+
agent_id: '1000002'
|
|
214
|
+
message_type: markdown
|
|
215
|
+
message: '{{ .CommonAnnotations.summary }}'
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Fixtures shared by the import round-trip tests (roundtrip.test.ts,
|
|
3
|
+
* cli.test.ts) and the type-check of the generated source
|
|
4
|
+
* (generated-types.e2e.test.ts).
|
|
5
|
+
*/
|
|
6
|
+
import { readdirSync, readFileSync, statSync } from "fs";
|
|
7
|
+
import { join, resolve } from "path";
|
|
8
|
+
import { build } from "@intentius/chant/build";
|
|
9
|
+
import type { Serializer, SerializerResult } from "@intentius/chant/serializer";
|
|
10
|
+
import { k8sSerializer } from "@intentius/chant-lexicon-k8s";
|
|
11
|
+
import { prometheusSerializer, ALERTMANAGER_FILE } from "../../serializer";
|
|
12
|
+
import { ruleFileYaml } from "../../build";
|
|
13
|
+
import { Slo } from "../../composites/slo";
|
|
14
|
+
|
|
15
|
+
export const pkgDir = resolve(import.meta.dirname, "../../..");
|
|
16
|
+
export const repoRoot = resolve(pkgDir, "../..");
|
|
17
|
+
export const read = (...p: string[]) => readFileSync(join(import.meta.dirname, ...p), "utf-8");
|
|
18
|
+
|
|
19
|
+
/** One file a build wrote: the rule file or `alertmanager.yml`. */
|
|
20
|
+
export interface BuiltFile {
|
|
21
|
+
name: string;
|
|
22
|
+
yaml: string;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/** The rule file and `alertmanager.yml` of one build output, whichever it has. */
|
|
26
|
+
export function builtFiles(out: string | SerializerResult | undefined): { rules?: string; alertmanager?: string } {
|
|
27
|
+
if (out === undefined || out === "") return {};
|
|
28
|
+
if (typeof out === "string") return out.startsWith("groups:") ? { rules: out } : { alertmanager: out };
|
|
29
|
+
const primaryIsRules = out.primary.startsWith("groups:");
|
|
30
|
+
return {
|
|
31
|
+
...(primaryIsRules ? { rules: out.primary } : { alertmanager: out.primary }),
|
|
32
|
+
...(out.files?.[ALERTMANAGER_FILE] !== undefined ? { alertmanager: out.files[ALERTMANAGER_FILE] } : {}),
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/** Every file each prometheus example builds, named `<example>/rules.yml` or `<example>/alertmanager.yml`. */
|
|
37
|
+
export async function exampleOutputs(): Promise<BuiltFile[]> {
|
|
38
|
+
const examplesDir = join(pkgDir, "examples");
|
|
39
|
+
const out: BuiltFile[] = [];
|
|
40
|
+
for (const name of readdirSync(examplesDir).sort()) {
|
|
41
|
+
const srcDir = join(examplesDir, name, "src");
|
|
42
|
+
try {
|
|
43
|
+
if (!statSync(srcDir).isDirectory()) continue;
|
|
44
|
+
} catch {
|
|
45
|
+
continue;
|
|
46
|
+
}
|
|
47
|
+
// k3d-stack declares its Kubernetes workloads beside the rules.
|
|
48
|
+
const serializers: Serializer[] = name === "k3d-stack" ? [k8sSerializer, prometheusSerializer] : [prometheusSerializer];
|
|
49
|
+
const result = await build(srcDir, serializers);
|
|
50
|
+
if (result.errors.length > 0) throw new Error(`${name}: ${result.errors.map(String).join("; ")}`);
|
|
51
|
+
const files = builtFiles(result.outputs.get("prometheus"));
|
|
52
|
+
if (files.rules !== undefined) out.push({ name: `${name}/rules.yml`, yaml: files.rules });
|
|
53
|
+
if (files.alertmanager !== undefined) out.push({ name: `${name}/alertmanager.yml`, yaml: files.alertmanager });
|
|
54
|
+
}
|
|
55
|
+
return out;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
const calls = "traces_span_metrics_calls_total";
|
|
59
|
+
|
|
60
|
+
/** Rule files `Slo()` builds, over the props that change its shape. */
|
|
61
|
+
export function sloOutputs(): BuiltFile[] {
|
|
62
|
+
const good = {
|
|
63
|
+
good: `sum(rate(${calls}{span_name="checkout",status_code!="STATUS_CODE_ERROR"}[{{window}}]))`,
|
|
64
|
+
total: `sum(rate(${calls}{span_name="checkout"}[{{window}}]))`,
|
|
65
|
+
};
|
|
66
|
+
const errors = {
|
|
67
|
+
errors: `sum(rate(http_requests_total{code=~"5.."}[{{window}}]))`,
|
|
68
|
+
total: "sum(rate(http_requests_total[{{window}}]))",
|
|
69
|
+
};
|
|
70
|
+
const slos = [
|
|
71
|
+
Slo({ name: "defaults", objective: 0.999, window: "30d", sli: errors }),
|
|
72
|
+
Slo({
|
|
73
|
+
name: "order-ack",
|
|
74
|
+
objective: 0.995,
|
|
75
|
+
window: "28d",
|
|
76
|
+
description: "Orders are acknowledged without an error span.",
|
|
77
|
+
sli: good,
|
|
78
|
+
alerting: {
|
|
79
|
+
page: { burnRates: "default", annotations: { runbook_url: "https://runbooks.example.com/order-ack" } },
|
|
80
|
+
ticket: { burnRates: "default", for: "15m", labels: { team: "orders" } },
|
|
81
|
+
},
|
|
82
|
+
labels: { team: "orders" },
|
|
83
|
+
interval: "1m",
|
|
84
|
+
}),
|
|
85
|
+
Slo({
|
|
86
|
+
name: "custom",
|
|
87
|
+
objective: 0.99,
|
|
88
|
+
window: "7d",
|
|
89
|
+
sli: errors,
|
|
90
|
+
alerting: {
|
|
91
|
+
alertName: "LatencyBudgetBurn",
|
|
92
|
+
page: {
|
|
93
|
+
severity: "critical",
|
|
94
|
+
burnRates: [
|
|
95
|
+
{ long: "1h", short: "5m", factor: 10 },
|
|
96
|
+
{ long: "6h", short: "30m", budgetConsumed: 0.1 },
|
|
97
|
+
],
|
|
98
|
+
},
|
|
99
|
+
ticket: false,
|
|
100
|
+
},
|
|
101
|
+
}),
|
|
102
|
+
Slo({ name: "no-alerts", objective: 0.95, window: "1w", sli: good, alerting: { page: false, ticket: false } }),
|
|
103
|
+
];
|
|
104
|
+
return slos.map((s) => ({ name: `Slo ${s.rules.props.name}`, yaml: ruleFileYaml([s.rules]) }));
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** The vendored upstream samples, each with the source it came from in its header. */
|
|
108
|
+
export const UPSTREAM = [
|
|
109
|
+
"prometheus-recording-rules.yml",
|
|
110
|
+
"prometheus-alerting-rules.yml",
|
|
111
|
+
"prometheus-alerting-templates.yml",
|
|
112
|
+
"alertmanager-simple.yml",
|
|
113
|
+
"alertmanager-route-labels.yml",
|
|
114
|
+
"alertmanager-conf-good.yml",
|
|
115
|
+
"alertmanager-mattermost-default-webhook-url-file.yml",
|
|
116
|
+
"alertmanager-opsgenie-default-apikey-file.yml",
|
|
117
|
+
"alertmanager-rocketchat-default-token-file.yml",
|
|
118
|
+
"alertmanager-sns-topic-arn.yml",
|
|
119
|
+
"alertmanager-telegram-default-bot-token-file.yml",
|
|
120
|
+
"alertmanager-victorops-default-apikey-file.yml",
|
|
121
|
+
"alertmanager-wechat-default-api-secret-file.yml",
|
|
122
|
+
];
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# Every rule file field, multi-line expressions (block scalars), a PromQL
|
|
2
|
+
# raw string in backticks, Go templates in annotations and a label value
|
|
3
|
+
# written as a number.
|
|
4
|
+
groups:
|
|
5
|
+
- name: node
|
|
6
|
+
interval: 1m
|
|
7
|
+
query_offset: 30s
|
|
8
|
+
limit: 100
|
|
9
|
+
labels:
|
|
10
|
+
team: platform
|
|
11
|
+
tier: 1
|
|
12
|
+
rules:
|
|
13
|
+
- record: instance:node_cpu_utilisation:rate5m
|
|
14
|
+
expr: |
|
|
15
|
+
1 - avg without (cpu, mode) (
|
|
16
|
+
rate(node_cpu_seconds_total{mode="idle"}[5m])
|
|
17
|
+
)
|
|
18
|
+
- record: instance:node_host:label
|
|
19
|
+
expr: label_replace(up{job="node"}, "host", "$1", "instance", `(.*):.*`)
|
|
20
|
+
labels:
|
|
21
|
+
source: node-exporter
|
|
22
|
+
- alert: NodeHighCpu
|
|
23
|
+
expr: instance:node_cpu_utilisation:rate5m > 0.9
|
|
24
|
+
for: 15m
|
|
25
|
+
keep_firing_for: 5m
|
|
26
|
+
labels:
|
|
27
|
+
severity: ticket
|
|
28
|
+
annotations:
|
|
29
|
+
summary: "{{ $labels.instance }} CPU above 90%"
|
|
30
|
+
description: >-
|
|
31
|
+
CPU on {{ $labels.instance }} has been above 90% for 15 minutes
|
|
32
|
+
(currently {{ $value | humanizePercentage }}).
|
|
33
|
+
- name: blackbox
|
|
34
|
+
rules:
|
|
35
|
+
- alert: ProbeFailing
|
|
36
|
+
expr: probe_success == 0
|
|
37
|
+
for: 2m
|
|
38
|
+
labels:
|
|
39
|
+
severity: page
|
|
40
|
+
annotations:
|
|
41
|
+
summary: "{{ $labels.instance }} is not answering"
|
|
42
|
+
runbook_url: https://runbooks.example.com/probe
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
# Vendored from github.com/prometheus/alertmanager v0.34.1, config/testdata/conf.good.yml
|
|
2
|
+
# (https://github.com/prometheus/alertmanager/blob/v0.34.1/config/testdata/conf.good.yml). Apache-2.0.
|
|
3
|
+
# Unchanged below this header.
|
|
4
|
+
global:
|
|
5
|
+
# The smarthost and SMTP sender used for mail notifications.
|
|
6
|
+
smtp_smarthost: 'localhost:25'
|
|
7
|
+
smtp_from: 'alertmanager@example.org'
|
|
8
|
+
smtp_auth_username: 'alertmanager'
|
|
9
|
+
smtp_auth_password: "multiline\nmysecret"
|
|
10
|
+
smtp_hello: "host.example.org"
|
|
11
|
+
slack_api_url: "http://mysecret.example.com/"
|
|
12
|
+
http_config:
|
|
13
|
+
proxy_url: 'http://127.0.0.1:1025'
|
|
14
|
+
# The directory from which notification templates are read.
|
|
15
|
+
templates:
|
|
16
|
+
- '/etc/alertmanager/template/*.tmpl'
|
|
17
|
+
# The root route on which each incoming alert enters.
|
|
18
|
+
route:
|
|
19
|
+
# The labels by which incoming alerts are grouped together. For example,
|
|
20
|
+
# multiple alerts coming in for cluster=A and alertname=LatencyHigh would
|
|
21
|
+
# be batched into a single group.
|
|
22
|
+
group_by: ['alertname', 'cluster', 'service']
|
|
23
|
+
# When a new group of alerts is created by an incoming alert, wait at
|
|
24
|
+
# least 'group_wait' to send the initial notification.
|
|
25
|
+
# This way ensures that you get multiple alerts for the same group that start
|
|
26
|
+
# firing shortly after another are batched together on the first
|
|
27
|
+
# notification.
|
|
28
|
+
group_wait: 30s
|
|
29
|
+
# When the first notification was sent, wait 'group_interval' to send a batch
|
|
30
|
+
# of new alerts that started firing for that group.
|
|
31
|
+
group_interval: 5m
|
|
32
|
+
# If an alert has successfully been sent, wait 'repeat_interval' to
|
|
33
|
+
# resend them.
|
|
34
|
+
repeat_interval: 3h
|
|
35
|
+
# A default receiver
|
|
36
|
+
receiver: team-X-mails
|
|
37
|
+
# All the above attributes are inherited by all child routes and can
|
|
38
|
+
# overwritten on each.
|
|
39
|
+
|
|
40
|
+
# The child route trees.
|
|
41
|
+
routes:
|
|
42
|
+
# This routes performs a regular expression match on alert labels to
|
|
43
|
+
# catch alerts that are related to a list of services.
|
|
44
|
+
- match_re:
|
|
45
|
+
service: ^(foo1|foo2|baz)$
|
|
46
|
+
receiver: team-X-mails
|
|
47
|
+
# The service has a sub-route for critical alerts, any alerts
|
|
48
|
+
# that do not match, i.e. severity != critical, fall-back to the
|
|
49
|
+
# parent node and are sent to 'team-X-mails'
|
|
50
|
+
routes:
|
|
51
|
+
- match:
|
|
52
|
+
severity: critical
|
|
53
|
+
receiver: team-X-pager
|
|
54
|
+
- match:
|
|
55
|
+
service: files
|
|
56
|
+
receiver: team-Y-mails
|
|
57
|
+
routes:
|
|
58
|
+
- match:
|
|
59
|
+
severity: critical
|
|
60
|
+
receiver: team-Y-pager
|
|
61
|
+
# This route handles all alerts coming from a database service. If there's
|
|
62
|
+
# no team to handle it, it defaults to the DB team.
|
|
63
|
+
- match:
|
|
64
|
+
service: database
|
|
65
|
+
receiver: team-DB-pager
|
|
66
|
+
# Also group alerts by affected database.
|
|
67
|
+
group_by: [alertname, cluster, database]
|
|
68
|
+
routes:
|
|
69
|
+
- match:
|
|
70
|
+
owner2: team-X
|
|
71
|
+
receiver: team-X-pager
|
|
72
|
+
continue: true
|
|
73
|
+
- match:
|
|
74
|
+
owner: team-Y
|
|
75
|
+
receiver: team-Y-pager
|
|
76
|
+
# continue: true
|
|
77
|
+
# Inhibition rules allow to mute a set of alerts given that another alert is
|
|
78
|
+
# firing.
|
|
79
|
+
# We use this to mute any warning-level notifications if the same alert is
|
|
80
|
+
# already critical.
|
|
81
|
+
inhibit_rules:
|
|
82
|
+
- source_match:
|
|
83
|
+
severity: 'critical'
|
|
84
|
+
target_match:
|
|
85
|
+
severity: 'warning'
|
|
86
|
+
# Apply inhibition if the alertname is the same.
|
|
87
|
+
equal: ['alertname', 'cluster', 'service']
|
|
88
|
+
receivers:
|
|
89
|
+
- name: 'team-X-mails'
|
|
90
|
+
email_configs:
|
|
91
|
+
- to: 'team-X+alerts@example.org'
|
|
92
|
+
- name: 'team-X-pager'
|
|
93
|
+
email_configs:
|
|
94
|
+
- to: 'team-X+alerts-critical@example.org'
|
|
95
|
+
pagerduty_configs:
|
|
96
|
+
- routing_key: "mysecret"
|
|
97
|
+
- name: 'team-Y-mails'
|
|
98
|
+
email_configs:
|
|
99
|
+
- to: 'team-Y+alerts@example.org'
|
|
100
|
+
- name: 'team-Y-pager'
|
|
101
|
+
pagerduty_configs:
|
|
102
|
+
- routing_key: "mysecret"
|
|
103
|
+
- name: 'team-DB-pager'
|
|
104
|
+
pagerduty_configs:
|
|
105
|
+
- routing_key: "mysecret"
|
|
106
|
+
- name: victorOps-receiver
|
|
107
|
+
victorops_configs:
|
|
108
|
+
- api_key: mysecret
|
|
109
|
+
routing_key: Sample_route
|
|
110
|
+
- name: opsGenie-receiver
|
|
111
|
+
opsgenie_configs:
|
|
112
|
+
- api_key: mysecret
|
|
113
|
+
- name: pushover-receiver
|
|
114
|
+
pushover_configs:
|
|
115
|
+
- token: mysecret
|
|
116
|
+
user_key: key
|
|
117
|
+
- name: slack-receiver
|
|
118
|
+
slack_configs:
|
|
119
|
+
- channel: '#my-channel'
|
|
120
|
+
image_url: 'http://some.img.com/img.png'
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# Vendored from github.com/prometheus/alertmanager v0.34.1, config/testdata/conf.mattermost-default-webhook-url-file.yml
|
|
2
|
+
# (https://github.com/prometheus/alertmanager/blob/v0.34.1/config/testdata/conf.mattermost-default-webhook-url-file.yml). Apache-2.0.
|
|
3
|
+
# Unchanged below this header.
|
|
4
|
+
global:
|
|
5
|
+
mattermost_webhook_url_file: /global_file
|
|
6
|
+
route:
|
|
7
|
+
receiver: team-X-mattermost
|
|
8
|
+
|
|
9
|
+
receivers:
|
|
10
|
+
- name: team-X-mattermost
|
|
11
|
+
mattermost_configs:
|
|
12
|
+
- channel: team-X
|
|
13
|
+
- name: team-Y-mattermost
|
|
14
|
+
mattermost_configs:
|
|
15
|
+
- channel: team-Y
|
|
16
|
+
webhook_url_file: /override_file
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# Vendored from github.com/prometheus/alertmanager v0.34.1, config/testdata/conf.opsgenie-default-apikey-file.yml
|
|
2
|
+
# (https://github.com/prometheus/alertmanager/blob/v0.34.1/config/testdata/conf.opsgenie-default-apikey-file.yml). Apache-2.0.
|
|
3
|
+
# Unchanged below this header.
|
|
4
|
+
global:
|
|
5
|
+
opsgenie_api_key_file: '/global_file'
|
|
6
|
+
route:
|
|
7
|
+
group_by: ['alertname', 'cluster', 'service']
|
|
8
|
+
group_wait: 30s
|
|
9
|
+
group_interval: 5m
|
|
10
|
+
repeat_interval: 3h
|
|
11
|
+
receiver: escalation-Y-opsgenie
|
|
12
|
+
routes:
|
|
13
|
+
- match:
|
|
14
|
+
service: foo
|
|
15
|
+
receiver: team-X-opsgenie
|
|
16
|
+
receivers:
|
|
17
|
+
- name: 'team-X-opsgenie'
|
|
18
|
+
opsgenie_configs:
|
|
19
|
+
- responders:
|
|
20
|
+
- name: 'team-X'
|
|
21
|
+
type: 'team'
|
|
22
|
+
- name: 'escalation-Y-opsgenie'
|
|
23
|
+
opsgenie_configs:
|
|
24
|
+
- responders:
|
|
25
|
+
- name: 'escalation-Y'
|
|
26
|
+
type: 'escalation'
|
|
27
|
+
api_key_file: /override_file
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# Vendored from github.com/prometheus/alertmanager v0.34.1, config/testdata/conf.rocketchat-default-token-file.yml
|
|
2
|
+
# (https://github.com/prometheus/alertmanager/blob/v0.34.1/config/testdata/conf.rocketchat-default-token-file.yml). Apache-2.0.
|
|
3
|
+
# Unchanged below this header.
|
|
4
|
+
global:
|
|
5
|
+
rocketchat_token_file: /global_file
|
|
6
|
+
rocketchat_token_id_file: /etc/alertmanager/rocketchat_token_id
|
|
7
|
+
route:
|
|
8
|
+
group_by: ['alertname', 'cluster', 'service']
|
|
9
|
+
group_wait: 30s
|
|
10
|
+
group_interval: 5m
|
|
11
|
+
repeat_interval: 3h
|
|
12
|
+
receiver: team-Y-rocketchat
|
|
13
|
+
routes:
|
|
14
|
+
- match:
|
|
15
|
+
service: foo
|
|
16
|
+
receiver: team-X-rocketchat
|
|
17
|
+
receivers:
|
|
18
|
+
- name: 'team-X-rocketchat'
|
|
19
|
+
rocketchat_configs:
|
|
20
|
+
- channel: '#team-X'
|
|
21
|
+
- name: 'team-Y-rocketchat'
|
|
22
|
+
rocketchat_configs:
|
|
23
|
+
- channel: '#team-Y'
|
|
24
|
+
token_file: /override_file
|
|
25
|
+
token_id_file: /override_file
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
# Vendored from github.com/prometheus/alertmanager v0.34.1, doc/examples/route_labels.yml
|
|
2
|
+
# (https://github.com/prometheus/alertmanager/blob/v0.34.1/doc/examples/route_labels.yml). Apache-2.0.
|
|
3
|
+
# Unchanged below this header.
|
|
4
|
+
# Example showing route `labels`.
|
|
5
|
+
#
|
|
6
|
+
# Route labels are attached to a route, inherited by child routes, and may be
|
|
7
|
+
# overridden per route. They are rendered per alert group and exposed to
|
|
8
|
+
# notification templates via the `routeLabels` function (and in the
|
|
9
|
+
# `routeLabels` field of the /api/v2/alerts/groups API response).
|
|
10
|
+
#
|
|
11
|
+
# The pattern shown here: a `description` is composed once at the root from a
|
|
12
|
+
# `reason` sub-label. Each subtree overrides only `reason`, computing it from
|
|
13
|
+
# the labels that branch matched on. The shared description picks up the new
|
|
14
|
+
# reason automatically, so the per-branch routes never restate the surrounding
|
|
15
|
+
# wording.
|
|
16
|
+
|
|
17
|
+
templates:
|
|
18
|
+
- '/etc/alertmanager/template/*.tmpl'
|
|
19
|
+
|
|
20
|
+
route:
|
|
21
|
+
group_by: ['alertname']
|
|
22
|
+
receiver: default
|
|
23
|
+
# `description` is defined once and built from `reason`. Child routes only
|
|
24
|
+
# override `reason`; they inherit this description unchanged.
|
|
25
|
+
labels:
|
|
26
|
+
reason: '{{ .GroupLabels.alertname }}'
|
|
27
|
+
description: '{{ .GroupLabels.alertname }} firing ({{ routeLabels "reason" }})'
|
|
28
|
+
routes:
|
|
29
|
+
# Database alerts are grouped by the affected database, so the reason can be
|
|
30
|
+
# computed from that branch's grouping label.
|
|
31
|
+
- matchers:
|
|
32
|
+
- service="database"
|
|
33
|
+
receiver: dba
|
|
34
|
+
group_by: ['alertname', 'database']
|
|
35
|
+
labels:
|
|
36
|
+
reason: 'database {{ .GroupLabels.database }}'
|
|
37
|
+
|
|
38
|
+
# Network link alerts match on both endpoints; the reason names the link.
|
|
39
|
+
- matchers:
|
|
40
|
+
- link_a_device=~".+"
|
|
41
|
+
- link_z_device=~".+"
|
|
42
|
+
receiver: network
|
|
43
|
+
group_by: ['alertname', 'link_a_device', 'link_z_device']
|
|
44
|
+
labels:
|
|
45
|
+
reason: '{{ .GroupLabels.link_a_device }} <-> {{ .GroupLabels.link_z_device }}'
|
|
46
|
+
|
|
47
|
+
receivers:
|
|
48
|
+
# The webhook payload includes the rendered route labels under "routeLabels".
|
|
49
|
+
- name: default
|
|
50
|
+
webhook_configs:
|
|
51
|
+
- url: 'http://127.0.0.1:5001/'
|
|
52
|
+
- name: dba
|
|
53
|
+
webhook_configs:
|
|
54
|
+
- url: 'http://127.0.0.1:5001/'
|
|
55
|
+
- name: network
|
|
56
|
+
webhook_configs:
|
|
57
|
+
- url: 'http://127.0.0.1:5001/'
|
|
58
|
+
|
|
59
|
+
# A notification template just renders the shared description:
|
|
60
|
+
#
|
|
61
|
+
# {{ define "route_labels.text" }}{{ routeLabels "description" }}{{ end }}
|
|
62
|
+
#
|
|
63
|
+
# For a generic alert this renders e.g. "HighLatency firing (HighLatency)";
|
|
64
|
+
# for the database branch "DiskFull firing (database orders)"; for the link
|
|
65
|
+
# branch "LinkDown firing (switch-a <-> switch-b)" -- only `reason` changed.
|