@intentius/chant-lexicon-prometheus 0.96.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +58 -0
- package/dist/alertmanager.d.ts +81 -0
- package/dist/alertmanager.d.ts.map +1 -0
- package/dist/build.d.ts +33 -0
- package/dist/build.d.ts.map +1 -0
- package/dist/catalog.d.ts +17 -0
- package/dist/catalog.d.ts.map +1 -0
- package/dist/codegen/docs-cli.d.ts +3 -0
- package/dist/codegen/docs-cli.d.ts.map +1 -0
- package/dist/codegen/docs.d.ts +9 -0
- package/dist/codegen/docs.d.ts.map +1 -0
- package/dist/codegen/generate-cli.d.ts +3 -0
- package/dist/codegen/generate-cli.d.ts.map +1 -0
- package/dist/codegen/generate.d.ts +19 -0
- package/dist/codegen/generate.d.ts.map +1 -0
- package/dist/codegen/package.d.ts +7 -0
- package/dist/codegen/package.d.ts.map +1 -0
- package/dist/composites/catalog.d.ts +3 -0
- package/dist/composites/catalog.d.ts.map +1 -0
- package/dist/composites/index.d.ts +6 -0
- package/dist/composites/index.d.ts.map +1 -0
- package/dist/composites/slo.d.ts +205 -0
- package/dist/composites/slo.d.ts.map +1 -0
- package/dist/detect.d.ts +2 -0
- package/dist/detect.d.ts.map +1 -0
- package/dist/duration.d.ts +16 -0
- package/dist/duration.d.ts.map +1 -0
- package/dist/index.d.ts +14 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/init-templates.d.ts +13 -0
- package/dist/init-templates.d.ts.map +1 -0
- package/dist/integrity.json +33 -0
- package/dist/lint/audit-catalog.d.ts +13 -0
- package/dist/lint/audit-catalog.d.ts.map +1 -0
- package/dist/lint/post-synth/index.d.ts +3 -0
- package/dist/lint/post-synth/index.d.ts.map +1 -0
- package/dist/lint/post-synth/prom-helpers.d.ts +33 -0
- package/dist/lint/post-synth/prom-helpers.d.ts.map +1 -0
- package/dist/lint/post-synth/prom101.d.ts +8 -0
- package/dist/lint/post-synth/prom101.d.ts.map +1 -0
- package/dist/lint/post-synth/prom102.d.ts +8 -0
- package/dist/lint/post-synth/prom102.d.ts.map +1 -0
- package/dist/lint/post-synth/prom103.d.ts +8 -0
- package/dist/lint/post-synth/prom103.d.ts.map +1 -0
- package/dist/lint/post-synth/prom104.d.ts +8 -0
- package/dist/lint/post-synth/prom104.d.ts.map +1 -0
- package/dist/lint/post-synth/prom105.d.ts +8 -0
- package/dist/lint/post-synth/prom105.d.ts.map +1 -0
- package/dist/lint/post-synth/prom106.d.ts +8 -0
- package/dist/lint/post-synth/prom106.d.ts.map +1 -0
- package/dist/lint/post-synth/prom107.d.ts +8 -0
- package/dist/lint/post-synth/prom107.d.ts.map +1 -0
- package/dist/lint/post-synth/prom201.d.ts +8 -0
- package/dist/lint/post-synth/prom201.d.ts.map +1 -0
- package/dist/lint/post-synth/prom202.d.ts +8 -0
- package/dist/lint/post-synth/prom202.d.ts.map +1 -0
- package/dist/lint/post-synth/prom203.d.ts +8 -0
- package/dist/lint/post-synth/prom203.d.ts.map +1 -0
- package/dist/lint/post-synth/prom204.d.ts +8 -0
- package/dist/lint/post-synth/prom204.d.ts.map +1 -0
- package/dist/lint/post-synth/prom205.d.ts +8 -0
- package/dist/lint/post-synth/prom205.d.ts.map +1 -0
- package/dist/lint/post-synth/prom206.d.ts +8 -0
- package/dist/lint/post-synth/prom206.d.ts.map +1 -0
- package/dist/lint/post-synth/prom207.d.ts +8 -0
- package/dist/lint/post-synth/prom207.d.ts.map +1 -0
- package/dist/lint/post-synth/prom208.d.ts +8 -0
- package/dist/lint/post-synth/prom208.d.ts.map +1 -0
- package/dist/lint/post-synth/prom209.d.ts +8 -0
- package/dist/lint/post-synth/prom209.d.ts.map +1 -0
- package/dist/lint/rules/index.d.ts +7 -0
- package/dist/lint/rules/index.d.ts.map +1 -0
- package/dist/lint/rules/literal-credential.d.ts +18 -0
- package/dist/lint/rules/literal-credential.d.ts.map +1 -0
- package/dist/lint/rules/prom-ast.d.ts +14 -0
- package/dist/lint/rules/prom-ast.d.ts.map +1 -0
- package/dist/lint/rules/promql-literal.d.ts +11 -0
- package/dist/lint/rules/promql-literal.d.ts.map +1 -0
- package/dist/lint/rules/slo-literal.d.ts +13 -0
- package/dist/lint/rules/slo-literal.d.ts.map +1 -0
- package/dist/lsp/completions.d.ts +4 -0
- package/dist/lsp/completions.d.ts.map +1 -0
- package/dist/lsp/hover.d.ts +4 -0
- package/dist/lsp/hover.d.ts.map +1 -0
- package/dist/manifest.json +8 -0
- package/dist/matchers.d.ts +29 -0
- package/dist/matchers.d.ts.map +1 -0
- package/dist/meta.json +32 -0
- package/dist/model.d.ts +262 -0
- package/dist/model.d.ts.map +1 -0
- package/dist/okf/index.md +34 -0
- package/dist/okf/rules/PROM001.md +16 -0
- package/dist/okf/rules/PROM002.md +15 -0
- package/dist/okf/rules/PROM003.md +11 -0
- package/dist/okf/rules/PROM101.md +11 -0
- package/dist/okf/rules/PROM102.md +11 -0
- package/dist/okf/rules/PROM103.md +11 -0
- package/dist/okf/rules/PROM104.md +11 -0
- package/dist/okf/rules/PROM105.md +11 -0
- package/dist/okf/rules/PROM106.md +11 -0
- package/dist/okf/rules/PROM107.md +11 -0
- package/dist/okf/rules/PROM201.md +11 -0
- package/dist/okf/rules/PROM202.md +11 -0
- package/dist/okf/rules/PROM203.md +11 -0
- package/dist/okf/rules/PROM204.md +11 -0
- package/dist/okf/rules/PROM205.md +11 -0
- package/dist/okf/rules/PROM206.md +11 -0
- package/dist/okf/rules/PROM207.md +11 -0
- package/dist/okf/rules/PROM208.md +11 -0
- package/dist/okf/rules/PROM209.md +11 -0
- package/dist/okf/types/AlertmanagerSettings.md +13 -0
- package/dist/okf/types/InhibitRule.md +9 -0
- package/dist/okf/types/Receiver.md +13 -0
- package/dist/okf/types/Route.md +9 -0
- package/dist/okf/types/RuleGroup.md +13 -0
- package/dist/okf/types/TimeInterval.md +9 -0
- package/dist/package-cli.d.ts +3 -0
- package/dist/package-cli.d.ts.map +1 -0
- package/dist/pin.d.ts +20 -0
- package/dist/pin.d.ts.map +1 -0
- package/dist/plugin.d.ts +10 -0
- package/dist/plugin.d.ts.map +1 -0
- package/dist/promql.d.ts +25 -0
- package/dist/promql.d.ts.map +1 -0
- package/dist/rule-eval.d.ts +67 -0
- package/dist/rule-eval.d.ts.map +1 -0
- package/dist/rules/literal-credential.ts +84 -0
- package/dist/rules/prom-ast.ts +30 -0
- package/dist/rules/prom-helpers.ts +89 -0
- package/dist/rules/prom101.ts +17 -0
- package/dist/rules/prom102.ts +17 -0
- package/dist/rules/prom103.ts +17 -0
- package/dist/rules/prom104.ts +17 -0
- package/dist/rules/prom105.ts +17 -0
- package/dist/rules/prom106.ts +17 -0
- package/dist/rules/prom107.ts +17 -0
- package/dist/rules/prom201.ts +17 -0
- package/dist/rules/prom202.ts +17 -0
- package/dist/rules/prom203.ts +17 -0
- package/dist/rules/prom204.ts +17 -0
- package/dist/rules/prom205.ts +17 -0
- package/dist/rules/prom206.ts +17 -0
- package/dist/rules/prom207.ts +17 -0
- package/dist/rules/prom208.ts +17 -0
- package/dist/rules/prom209.ts +17 -0
- package/dist/rules/promql-literal.ts +55 -0
- package/dist/rules/slo-literal.ts +81 -0
- package/dist/rules.d.ts +59 -0
- package/dist/rules.d.ts.map +1 -0
- package/dist/serializer.d.ts +27 -0
- package/dist/serializer.d.ts.map +1 -0
- package/dist/skill-defs.d.ts +3 -0
- package/dist/skill-defs.d.ts.map +1 -0
- package/dist/skills/chant-prometheus-alertmanager.md +58 -0
- package/dist/skills/chant-prometheus-kubernetes.md +48 -0
- package/dist/skills/chant-prometheus.md +91 -0
- package/dist/tools.d.ts +30 -0
- package/dist/tools.d.ts.map +1 -0
- package/dist/types/index.d.ts +3 -0
- package/dist/validate-cli.d.ts +3 -0
- package/dist/validate-cli.d.ts.map +1 -0
- package/dist/validate-config.d.ts +35 -0
- package/dist/validate-config.d.ts.map +1 -0
- package/dist/validate.d.ts +11 -0
- package/dist/validate.d.ts.map +1 -0
- package/package.json +77 -0
- package/src/alertmanager.ts +143 -0
- package/src/build.ts +199 -0
- package/src/catalog.ts +65 -0
- package/src/codegen/docs-cli.ts +4 -0
- package/src/codegen/docs.ts +104 -0
- package/src/codegen/generate-cli.ts +8 -0
- package/src/codegen/generate.ts +49 -0
- package/src/codegen/package.ts +41 -0
- package/src/composites/catalog.test.ts +37 -0
- package/src/composites/catalog.ts +65 -0
- package/src/composites/composites.test.ts +256 -0
- package/src/composites/index.ts +23 -0
- package/src/composites/slo-burn.test.ts +289 -0
- package/src/composites/slo.ts +471 -0
- package/src/detect.ts +12 -0
- package/src/duration.ts +51 -0
- package/src/generated/lexicon-prometheus.json +32 -0
- package/src/index.ts +97 -0
- package/src/init-templates.test.ts +29 -0
- package/src/init-templates.ts +76 -0
- package/src/lint/audit-catalog.ts +123 -0
- package/src/lint/post-synth/index.ts +37 -0
- package/src/lint/post-synth/post-synth.test.ts +256 -0
- package/src/lint/post-synth/prom-helpers.ts +89 -0
- package/src/lint/post-synth/prom101.ts +17 -0
- package/src/lint/post-synth/prom102.ts +17 -0
- package/src/lint/post-synth/prom103.ts +17 -0
- package/src/lint/post-synth/prom104.ts +17 -0
- package/src/lint/post-synth/prom105.ts +17 -0
- package/src/lint/post-synth/prom106.ts +17 -0
- package/src/lint/post-synth/prom107.ts +17 -0
- package/src/lint/post-synth/prom201.ts +17 -0
- package/src/lint/post-synth/prom202.ts +17 -0
- package/src/lint/post-synth/prom203.ts +17 -0
- package/src/lint/post-synth/prom204.ts +17 -0
- package/src/lint/post-synth/prom205.ts +17 -0
- package/src/lint/post-synth/prom206.ts +17 -0
- package/src/lint/post-synth/prom207.ts +17 -0
- package/src/lint/post-synth/prom208.ts +17 -0
- package/src/lint/post-synth/prom209.ts +17 -0
- package/src/lint/rules/index.ts +11 -0
- package/src/lint/rules/literal-credential.ts +84 -0
- package/src/lint/rules/prom-ast.ts +30 -0
- package/src/lint/rules/promql-literal.ts +55 -0
- package/src/lint/rules/rules.test.ts +114 -0
- package/src/lint/rules/slo-literal.ts +81 -0
- package/src/lsp/completions.test.ts +23 -0
- package/src/lsp/completions.ts +15 -0
- package/src/lsp/hover.test.ts +27 -0
- package/src/lsp/hover.ts +26 -0
- package/src/matchers.ts +132 -0
- package/src/model.test.ts +104 -0
- package/src/model.ts +290 -0
- package/src/package-cli.ts +17 -0
- package/src/pin.ts +13 -0
- package/src/plugin.test.ts +73 -0
- package/src/plugin.ts +129 -0
- package/src/promql.ts +46 -0
- package/src/rule-eval.ts +397 -0
- package/src/rules.ts +127 -0
- package/src/serializer.test.ts +179 -0
- package/src/serializer.ts +55 -0
- package/src/skill-defs.ts +43 -0
- package/src/skills/chant-prometheus-alertmanager.md +58 -0
- package/src/skills/chant-prometheus-kubernetes.md +48 -0
- package/src/skills/chant-prometheus.md +91 -0
- package/src/tools.test.ts +119 -0
- package/src/tools.ts +70 -0
- package/src/validate-cli.ts +7 -0
- package/src/validate-config.ts +400 -0
- package/src/validate.ts +66 -0
package/src/rules.ts
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `RuleGroup`: one group of recording and alerting rules.
|
|
3
|
+
*
|
|
4
|
+
* A group is the unit Prometheus evaluates together, on one interval, and
|
|
5
|
+
* the unit both destinations take: an entry under `groups:` in a rule file,
|
|
6
|
+
* and an entry in a Prometheus Operator `PrometheusRule`'s `spec.groups`. The
|
|
7
|
+
* same declaration renders to either; `ruleGroupConfig()` is the one
|
|
8
|
+
* conversion both go through.
|
|
9
|
+
*
|
|
10
|
+
* Rules are plain objects in the rule file's own shape, so a composite (the
|
|
11
|
+
* SLO declaration, say) can build them as data and hand a list to one or
|
|
12
|
+
* more groups.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import { createResource } from "@intentius/chant/runtime";
|
|
16
|
+
import type { Declarable } from "@intentius/chant/declarable";
|
|
17
|
+
import {
|
|
18
|
+
isAlertingRuleConfig,
|
|
19
|
+
isRecordingRuleConfig,
|
|
20
|
+
type AlertingRuleConfig,
|
|
21
|
+
type LabelSet,
|
|
22
|
+
type RecordingRuleConfig,
|
|
23
|
+
type RuleConfig,
|
|
24
|
+
type RuleGroupConfig,
|
|
25
|
+
} from "./model";
|
|
26
|
+
|
|
27
|
+
export type RecordingRule = RecordingRuleConfig;
|
|
28
|
+
export type AlertingRule = AlertingRuleConfig;
|
|
29
|
+
export type Rule = RuleConfig;
|
|
30
|
+
|
|
31
|
+
/** What `new RuleGroup(...)` takes: the group exactly as it appears in a rule file. */
|
|
32
|
+
export type RuleGroupProps = RuleGroupConfig;
|
|
33
|
+
|
|
34
|
+
export interface RuleGroupEntity extends Declarable {
|
|
35
|
+
readonly props: RuleGroupProps;
|
|
36
|
+
/** The group's `name`. */
|
|
37
|
+
readonly groupName: string;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export const RULE_GROUP_TYPE = "Prometheus::Rules::RuleGroup";
|
|
41
|
+
|
|
42
|
+
const RuleGroupBase = createResource(RULE_GROUP_TYPE, "prometheus", {}) as unknown as (
|
|
43
|
+
this: object,
|
|
44
|
+
props: Record<string, unknown>,
|
|
45
|
+
) => void;
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* A group of recording and alerting rules.
|
|
49
|
+
*
|
|
50
|
+
* @example
|
|
51
|
+
* ```ts
|
|
52
|
+
* export const api = new RuleGroup({
|
|
53
|
+
* name: "api",
|
|
54
|
+
* interval: "30s",
|
|
55
|
+
* rules: [
|
|
56
|
+
* { record: "job:http_requests:rate5m", expr: "sum by (job) (rate(http_requests_total[5m]))" },
|
|
57
|
+
* {
|
|
58
|
+
* alert: "ApiHighErrorRate",
|
|
59
|
+
* expr: 'sum(rate(http_requests_total{code=~"5.."}[5m])) / sum(rate(http_requests_total[5m])) > 0.05',
|
|
60
|
+
* for: "10m",
|
|
61
|
+
* labels: { severity: "page" },
|
|
62
|
+
* annotations: { summary: "API 5xx ratio above 5%" },
|
|
63
|
+
* },
|
|
64
|
+
* ],
|
|
65
|
+
* });
|
|
66
|
+
* ```
|
|
67
|
+
*/
|
|
68
|
+
export const RuleGroup = function (this: object, props: RuleGroupProps) {
|
|
69
|
+
RuleGroupBase.call(this, props as unknown as Record<string, unknown>);
|
|
70
|
+
Object.defineProperty(this, "groupName", { value: props?.name, enumerable: false });
|
|
71
|
+
} as unknown as new (props: RuleGroupProps) => RuleGroupEntity;
|
|
72
|
+
Object.defineProperty(RuleGroup, "name", { value: "RuleGroup" });
|
|
73
|
+
|
|
74
|
+
/** True when `value` is a declared `RuleGroup`. */
|
|
75
|
+
export function isRuleGroup(value: unknown): value is RuleGroupEntity {
|
|
76
|
+
return typeof value === "object" && value !== null && (value as Declarable).entityType === RULE_GROUP_TYPE;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
function labelSet(value: LabelSet | undefined): LabelSet | undefined {
|
|
80
|
+
if (value === undefined || value === null) return undefined;
|
|
81
|
+
const out: LabelSet = {};
|
|
82
|
+
for (const [k, v] of Object.entries(value)) {
|
|
83
|
+
if (v !== undefined) out[k] = String(v);
|
|
84
|
+
}
|
|
85
|
+
return out;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
function defined<T extends Record<string, unknown>>(obj: T): T {
|
|
89
|
+
for (const k of Object.keys(obj)) if (obj[k] === undefined) delete obj[k];
|
|
90
|
+
return obj;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/** A rule in canonical key order, `undefined` fields dropped. */
|
|
94
|
+
export function ruleConfig(rule: Rule): Rule {
|
|
95
|
+
if (isRecordingRuleConfig(rule)) {
|
|
96
|
+
return defined({ record: rule.record, expr: rule.expr, labels: labelSet(rule.labels) }) as RecordingRule;
|
|
97
|
+
}
|
|
98
|
+
if (isAlertingRuleConfig(rule)) {
|
|
99
|
+
return defined({
|
|
100
|
+
alert: rule.alert,
|
|
101
|
+
expr: rule.expr,
|
|
102
|
+
for: rule.for,
|
|
103
|
+
keep_firing_for: rule.keep_firing_for,
|
|
104
|
+
labels: labelSet(rule.labels),
|
|
105
|
+
annotations: labelSet(rule.annotations),
|
|
106
|
+
}) as AlertingRule;
|
|
107
|
+
}
|
|
108
|
+
// Neither `record` nor `alert`: keep it as written, and let PROM105 name it.
|
|
109
|
+
return { ...(rule as object) } as Rule;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* The plain group a `RuleGroup` (or group props) renders to, in canonical key
|
|
114
|
+
* order. This is what goes under `groups:` in a rule file and into a
|
|
115
|
+
* `PrometheusRule`'s `spec.groups`.
|
|
116
|
+
*/
|
|
117
|
+
export function ruleGroupConfig(group: RuleGroupEntity | RuleGroupProps): RuleGroupConfig {
|
|
118
|
+
const p = isRuleGroup(group) ? group.props : group;
|
|
119
|
+
return defined({
|
|
120
|
+
name: p.name,
|
|
121
|
+
interval: p.interval,
|
|
122
|
+
query_offset: p.query_offset,
|
|
123
|
+
limit: p.limit,
|
|
124
|
+
labels: labelSet(p.labels),
|
|
125
|
+
rules: (p.rules ?? []).map(ruleConfig),
|
|
126
|
+
}) as RuleGroupConfig;
|
|
127
|
+
}
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
import { describe, expect, test } from "vitest";
|
|
2
|
+
import { load } from "js-yaml";
|
|
3
|
+
import type { Declarable } from "@intentius/chant/declarable";
|
|
4
|
+
import { prometheusSerializer, ALERTMANAGER_FILE } from "./serializer";
|
|
5
|
+
import { AlertmanagerSettings, InhibitRule, Receiver, Route, RuleGroup, TimeInterval } from "./index";
|
|
6
|
+
import type { AlertmanagerConfig, RuleFileConfig } from "./model";
|
|
7
|
+
import type { SerializerResult } from "@intentius/chant/serializer";
|
|
8
|
+
|
|
9
|
+
function entities(record: Record<string, unknown>): Map<string, Declarable> {
|
|
10
|
+
return new Map(Object.entries(record) as Array<[string, Declarable]>);
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
function primary(out: ReturnType<typeof prometheusSerializer.serialize>): string {
|
|
14
|
+
return typeof out === "string" ? out : out.primary;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
const api = () =>
|
|
18
|
+
new RuleGroup({
|
|
19
|
+
name: "api",
|
|
20
|
+
interval: "30s",
|
|
21
|
+
rules: [
|
|
22
|
+
{ record: "job:http_requests:rate5m", expr: "sum by (job) (rate(http_requests_total[5m]))" },
|
|
23
|
+
{ alert: "ApiDown", expr: 'up{job="api"} == 0', for: "5m", labels: { severity: "page" }, annotations: { summary: "api is down" } },
|
|
24
|
+
],
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
describe("prometheus serializer", () => {
|
|
28
|
+
// 1, 2
|
|
29
|
+
test("name and rule prefix", () => {
|
|
30
|
+
expect(prometheusSerializer.name).toBe("prometheus");
|
|
31
|
+
expect(prometheusSerializer.rulePrefix).toBe("PROM");
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
// 3
|
|
35
|
+
test("an empty map serializes to the empty string", () => {
|
|
36
|
+
expect(prometheusSerializer.serialize(new Map())).toBe("");
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
// 4
|
|
40
|
+
test("one rule group is a rule file", () => {
|
|
41
|
+
expect(prometheusSerializer.serialize(entities({ api: api() }))).toBe(`groups:
|
|
42
|
+
- name: api
|
|
43
|
+
interval: 30s
|
|
44
|
+
rules:
|
|
45
|
+
- record: job:http_requests:rate5m
|
|
46
|
+
expr: sum by (job) (rate(http_requests_total[5m]))
|
|
47
|
+
- alert: ApiDown
|
|
48
|
+
expr: up{job="api"} == 0
|
|
49
|
+
for: 5m
|
|
50
|
+
labels:
|
|
51
|
+
severity: page
|
|
52
|
+
annotations:
|
|
53
|
+
summary: api is down
|
|
54
|
+
`);
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
// 5, 6: the export name never appears; the group's own name is used
|
|
58
|
+
test("the group name comes from props, not the export name", () => {
|
|
59
|
+
const out = primary(prometheusSerializer.serialize(entities({ someExportName: api() })));
|
|
60
|
+
expect(out).not.toContain("someExportName");
|
|
61
|
+
expect((load(out) as RuleFileConfig).groups[0].name).toBe("api");
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
// 7
|
|
65
|
+
test("several groups come out sorted by name", () => {
|
|
66
|
+
const b = new RuleGroup({ name: "b", rules: [{ record: "b:x", expr: "1" }] });
|
|
67
|
+
const a = new RuleGroup({ name: "a", rules: [{ record: "a:x", expr: "1" }] });
|
|
68
|
+
const file = load(primary(prometheusSerializer.serialize(entities({ b, a })))) as RuleFileConfig;
|
|
69
|
+
expect(file.groups.map((g) => g.name)).toEqual(["a", "b"]);
|
|
70
|
+
});
|
|
71
|
+
|
|
72
|
+
// 8, 9: nothing is injected; what is set is what comes out
|
|
73
|
+
test("nothing is defaulted in", () => {
|
|
74
|
+
const g = new RuleGroup({ name: "g", rules: [{ alert: "A", expr: "vector(1)" }] });
|
|
75
|
+
expect(load(primary(prometheusSerializer.serialize(entities({ g }))))).toEqual({
|
|
76
|
+
groups: [{ name: "g", rules: [{ alert: "A", expr: "vector(1)" }] }],
|
|
77
|
+
});
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
// 10: non-prometheus entities are ignored
|
|
81
|
+
test("entities from other lexicons are skipped", () => {
|
|
82
|
+
const foreign = { lexicon: "k8s", entityType: "K8s::Core::ConfigMap", kind: "resource", props: {} };
|
|
83
|
+
expect(prometheusSerializer.serialize(entities({ foreign }))).toBe("");
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
// 11
|
|
87
|
+
test("keys come out in rule-file order whatever order they were written in", () => {
|
|
88
|
+
const g = new RuleGroup({
|
|
89
|
+
rules: [{ annotations: { summary: "s" }, labels: { severity: "page" }, for: "1m", expr: "up == 0", alert: "A" }],
|
|
90
|
+
interval: "1m",
|
|
91
|
+
name: "g",
|
|
92
|
+
} as never);
|
|
93
|
+
const text = primary(prometheusSerializer.serialize(entities({ g })));
|
|
94
|
+
expect(text.indexOf("name: g")).toBeLessThan(text.indexOf("interval"));
|
|
95
|
+
const rule = text.slice(text.indexOf("- alert"));
|
|
96
|
+
const order = ["alert", "expr", "for", "labels", "annotations"].map((k) => rule.indexOf(`${k}:`));
|
|
97
|
+
expect([...order].sort((x, y) => x - y)).toEqual(order);
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
// 12: format-specific
|
|
101
|
+
describe("alertmanager.yml", () => {
|
|
102
|
+
const hook = () => new Receiver({ name: "hook", webhook_configs: [{ url: "http://hook:8080/" }] });
|
|
103
|
+
|
|
104
|
+
test("alone, it is the primary output", () => {
|
|
105
|
+
const h = hook();
|
|
106
|
+
const root = new Route({ receiver: h, group_by: ["alertname"] });
|
|
107
|
+
const out = prometheusSerializer.serialize(entities({ h, root }));
|
|
108
|
+
expect(typeof out).toBe("string");
|
|
109
|
+
expect(load(out as string)).toEqual({
|
|
110
|
+
route: { receiver: "hook", group_by: ["alertname"] },
|
|
111
|
+
receivers: [{ name: "hook", webhook_configs: [{ url: "http://hook:8080/" }] }],
|
|
112
|
+
});
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
test("beside a rule file, it is written as alertmanager.yml", () => {
|
|
116
|
+
const h = hook();
|
|
117
|
+
const out = prometheusSerializer.serialize(entities({ api: api(), h, root: new Route({ receiver: h }) })) as SerializerResult;
|
|
118
|
+
expect((load(out.primary) as RuleFileConfig).groups).toHaveLength(1);
|
|
119
|
+
expect(Object.keys(out.files ?? {})).toEqual([ALERTMANAGER_FILE]);
|
|
120
|
+
expect((load(out.files![ALERTMANAGER_FILE]) as AlertmanagerConfig).route?.receiver).toBe("hook");
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
test("sections come out in Alertmanager's order, references by name", () => {
|
|
124
|
+
const h = hook();
|
|
125
|
+
const quiet = new TimeInterval({ name: "weekend", time_intervals: [{ weekdays: ["saturday", "sunday"] }] });
|
|
126
|
+
const out = primary(
|
|
127
|
+
prometheusSerializer.serialize(
|
|
128
|
+
entities({
|
|
129
|
+
quiet,
|
|
130
|
+
inhibit: new InhibitRule({ source_matchers: ['severity="page"'], target_matchers: ['severity="ticket"'], equal: ["alertname"] }),
|
|
131
|
+
h,
|
|
132
|
+
root: new Route({ receiver: h, routes: [{ matchers: ['severity="ticket"'], receiver: "hook", mute_time_intervals: [quiet] }] }),
|
|
133
|
+
settings: new AlertmanagerSettings({ global: { resolve_timeout: "5m" }, templates: ["/etc/am/*.tmpl"] }),
|
|
134
|
+
}),
|
|
135
|
+
),
|
|
136
|
+
);
|
|
137
|
+
const keys = Object.keys(load(out) as object);
|
|
138
|
+
expect(keys).toEqual(["global", "templates", "route", "inhibit_rules", "receivers", "time_intervals"]);
|
|
139
|
+
expect((load(out) as AlertmanagerConfig).route?.routes?.[0].mute_time_intervals).toEqual(["weekend"]);
|
|
140
|
+
});
|
|
141
|
+
|
|
142
|
+
test("a receiver or interval reached only through a route is still emitted", () => {
|
|
143
|
+
const h = hook();
|
|
144
|
+
const quiet = new TimeInterval({ name: "night", time_intervals: [{ times: [{ start_time: "22:00", end_time: "24:00" }] }] });
|
|
145
|
+
const out = load(primary(prometheusSerializer.serialize(entities({ root: new Route({ receiver: h, mute_time_intervals: [quiet] }) })))) as AlertmanagerConfig;
|
|
146
|
+
expect(out.receivers?.map((r) => r.name)).toEqual(["hook"]);
|
|
147
|
+
expect(out.time_intervals?.map((t) => t.name)).toEqual(["night"]);
|
|
148
|
+
});
|
|
149
|
+
|
|
150
|
+
test("a child Route entity is not a second root", () => {
|
|
151
|
+
const h = hook();
|
|
152
|
+
const child = new Route({ matchers: ['severity="page"'], receiver: h });
|
|
153
|
+
const out = prometheusSerializer.serialize(entities({ h, child, root: new Route({ receiver: h, routes: [child] }) }));
|
|
154
|
+
expect(typeof out).toBe("string");
|
|
155
|
+
expect((load(out as string) as AlertmanagerConfig).route?.routes).toHaveLength(1);
|
|
156
|
+
});
|
|
157
|
+
|
|
158
|
+
test("two roots warn and the first wins", () => {
|
|
159
|
+
const h = hook();
|
|
160
|
+
const out = prometheusSerializer.serialize(entities({ h, a: new Route({ receiver: h }), b: new Route({ receiver: "other" }) })) as SerializerResult;
|
|
161
|
+
expect(out.warnings?.[0]).toContain("2 root Routes");
|
|
162
|
+
expect((load(out.primary) as AlertmanagerConfig).route?.receiver).toBe("hook");
|
|
163
|
+
});
|
|
164
|
+
});
|
|
165
|
+
|
|
166
|
+
test("round-trip: the rule file parses back to the declared group", () => {
|
|
167
|
+
const g = api();
|
|
168
|
+
const file = load(primary(prometheusSerializer.serialize(entities({ g })))) as RuleFileConfig;
|
|
169
|
+
expect(file.groups[0]).toEqual(g.props);
|
|
170
|
+
});
|
|
171
|
+
|
|
172
|
+
test("a multi-line expr survives as a block scalar", () => {
|
|
173
|
+
const expr = "sum(rate(a[5m]))\n /\nsum(rate(b[5m]))";
|
|
174
|
+
const g = new RuleGroup({ name: "g", rules: [{ record: "a:ratio", expr }] });
|
|
175
|
+
const text = primary(prometheusSerializer.serialize(entities({ g })));
|
|
176
|
+
expect(text).toContain("expr: |-");
|
|
177
|
+
expect((load(text) as RuleFileConfig).groups[0].rules[0].expr).toBe(expr);
|
|
178
|
+
});
|
|
179
|
+
});
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Prometheus serializer.
|
|
3
|
+
*
|
|
4
|
+
* Emits the two files a Prometheus setup reads:
|
|
5
|
+
*
|
|
6
|
+
* - the rule file (`groups:`), from every `RuleGroup` in the build, the file
|
|
7
|
+
* `rule_files:` points at and `promtool check rules` reads;
|
|
8
|
+
* - `alertmanager.yml`, from the `Route`, `Receiver`, `InhibitRule`,
|
|
9
|
+
* `TimeInterval` and `AlertmanagerSettings` entities, the file
|
|
10
|
+
* `amtool check-config` reads.
|
|
11
|
+
*
|
|
12
|
+
* The rule file is the primary output when the build declares any rule
|
|
13
|
+
* groups, and `alertmanager.yml` is written beside it when the build also
|
|
14
|
+
* declares Alertmanager entities. A build with only Alertmanager entities
|
|
15
|
+
* emits `alertmanager.yml` as the primary output. Rules inside a group and
|
|
16
|
+
* child routes keep the order they are written in, which is the order that
|
|
17
|
+
* matters to Prometheus and Alertmanager; groups, receivers and time
|
|
18
|
+
* intervals, whose order means nothing, are sorted by name.
|
|
19
|
+
*
|
|
20
|
+
* Neither file has a metadata channel, so there is no ownership marker to
|
|
21
|
+
* stamp.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
import type { Declarable } from "@intentius/chant/declarable";
|
|
25
|
+
import type { Serializer, SerializerResult } from "@intentius/chant/serializer";
|
|
26
|
+
import type { LexiconOutput } from "@intentius/chant/lexicon-output";
|
|
27
|
+
import { buildAlertmanagerConfig, buildRuleFile, emitYaml } from "./build";
|
|
28
|
+
|
|
29
|
+
/** The filename `alertmanager.yml` is written under when it sits beside a rule file. */
|
|
30
|
+
export const ALERTMANAGER_FILE = "alertmanager.yml";
|
|
31
|
+
|
|
32
|
+
export const prometheusSerializer: Serializer = {
|
|
33
|
+
name: "prometheus",
|
|
34
|
+
rulePrefix: "PROM",
|
|
35
|
+
|
|
36
|
+
serialize(entities: Map<string, Declarable>, _outputs?: LexiconOutput[]): string | SerializerResult {
|
|
37
|
+
const rules = buildRuleFile(entities);
|
|
38
|
+
const am = buildAlertmanagerConfig(entities);
|
|
39
|
+
const hasRules = rules.groups.length > 0;
|
|
40
|
+
const hasAm = am.count > 0;
|
|
41
|
+
const warnings = am.warnings;
|
|
42
|
+
|
|
43
|
+
if (!hasRules && !hasAm) return "";
|
|
44
|
+
if (!hasAm) return emitYaml(rules.config);
|
|
45
|
+
if (!hasRules) {
|
|
46
|
+
const text = emitYaml(am.config);
|
|
47
|
+
return warnings.length === 0 ? text : { primary: text, warnings };
|
|
48
|
+
}
|
|
49
|
+
return {
|
|
50
|
+
primary: emitYaml(rules.config),
|
|
51
|
+
files: { [ALERTMANAGER_FILE]: emitYaml(am.config) },
|
|
52
|
+
...(warnings.length > 0 ? { warnings } : {}),
|
|
53
|
+
};
|
|
54
|
+
},
|
|
55
|
+
};
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { createSkillsLoader } from "@intentius/chant/lexicon-plugin-helpers";
|
|
2
|
+
|
|
3
|
+
/** The prometheus lexicon's AI skills, read from src/skills/. */
|
|
4
|
+
export const prometheusSkills = createSkillsLoader(import.meta.url, [
|
|
5
|
+
{
|
|
6
|
+
file: "chant-prometheus.md",
|
|
7
|
+
name: "chant-prometheus",
|
|
8
|
+
description: "Declare Prometheus recording and alerting rule groups as typed chant entities and build a rule file that passes promtool",
|
|
9
|
+
triggers: [
|
|
10
|
+
{ type: "context" as const, value: "prometheus rules" },
|
|
11
|
+
{ type: "context" as const, value: "alerting rule" },
|
|
12
|
+
{ type: "context" as const, value: "recording rule" },
|
|
13
|
+
],
|
|
14
|
+
examples: [
|
|
15
|
+
{
|
|
16
|
+
title: "A recording rule and an alert on it",
|
|
17
|
+
output:
|
|
18
|
+
"export const api = new RuleGroup({\n" +
|
|
19
|
+
' name: "api",\n' +
|
|
20
|
+
" rules: [\n" +
|
|
21
|
+
' { record: "job:http_errors:ratio5m", expr: \'sum by (job) (rate(http_requests_total{code=~"5.."}[5m])) / sum by (job) (rate(http_requests_total[5m]))\' },\n' +
|
|
22
|
+
' { alert: "ApiErrors", expr: "job:http_errors:ratio5m > 0.05", for: "10m", labels: { severity: "page" }, annotations: { summary: "5xx ratio above 5%" } },\n' +
|
|
23
|
+
" ],\n" +
|
|
24
|
+
"});",
|
|
25
|
+
},
|
|
26
|
+
],
|
|
27
|
+
},
|
|
28
|
+
{
|
|
29
|
+
file: "chant-prometheus-alertmanager.md",
|
|
30
|
+
name: "chant-prometheus-alertmanager",
|
|
31
|
+
description: "Declare Alertmanager routing (routes, receivers, inhibit rules, time intervals) and keep every alert severity routed",
|
|
32
|
+
triggers: [
|
|
33
|
+
{ type: "context" as const, value: "alertmanager" },
|
|
34
|
+
{ type: "context" as const, value: "alert routing" },
|
|
35
|
+
],
|
|
36
|
+
},
|
|
37
|
+
{
|
|
38
|
+
file: "chant-prometheus-kubernetes.md",
|
|
39
|
+
name: "chant-prometheus-kubernetes",
|
|
40
|
+
description: "Render the same RuleGroups into a Prometheus Operator PrometheusRule, or into ConfigMaps for a plain Prometheus and Alertmanager on Kubernetes",
|
|
41
|
+
triggers: [{ type: "context" as const, value: "PrometheusRule" }],
|
|
42
|
+
},
|
|
43
|
+
]);
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
---
|
|
2
|
+
skill: chant-prometheus-alertmanager
|
|
3
|
+
description: Declare Alertmanager routing (routes, receivers, inhibit rules, time intervals) and keep every alert severity routed
|
|
4
|
+
user-invocable: true
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
# Alertmanager routing with chant
|
|
8
|
+
|
|
9
|
+
`Route`, `Receiver`, `InhibitRule`, `TimeInterval` and `AlertmanagerSettings` build `alertmanager.yml`. When the same build root declares `RuleGroup`s, the rule file is the primary output and `alertmanager.yml` is written beside it.
|
|
10
|
+
|
|
11
|
+
```ts
|
|
12
|
+
import { AlertmanagerSettings, InhibitRule, Receiver, Route, TimeInterval } from "@intentius/chant-lexicon-prometheus";
|
|
13
|
+
|
|
14
|
+
export const settings = new AlertmanagerSettings({ global: { resolve_timeout: "5m" } });
|
|
15
|
+
|
|
16
|
+
export const oncall = new Receiver({
|
|
17
|
+
name: "oncall",
|
|
18
|
+
pagerduty_configs: [{ routing_key_file: "/etc/alertmanager/secrets/pagerduty-key" }],
|
|
19
|
+
});
|
|
20
|
+
export const team = new Receiver({
|
|
21
|
+
name: "team-slack",
|
|
22
|
+
slack_configs: [{ api_url_file: "/etc/alertmanager/secrets/slack-url", channel: "#alerts", send_resolved: true }],
|
|
23
|
+
});
|
|
24
|
+
export const sink = new Receiver({ name: "default", webhook_configs: [{ url: "http://alert-sink.monitoring:8080/" }] });
|
|
25
|
+
|
|
26
|
+
export const offHours = new TimeInterval({
|
|
27
|
+
name: "off-hours",
|
|
28
|
+
time_intervals: [{ weekdays: ["saturday", "sunday"] }, { times: [{ start_time: "18:00", end_time: "24:00" }] }],
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
export const root = new Route({
|
|
32
|
+
receiver: sink,
|
|
33
|
+
group_by: ["alertname", "job"],
|
|
34
|
+
group_wait: "30s",
|
|
35
|
+
group_interval: "5m",
|
|
36
|
+
repeat_interval: "4h",
|
|
37
|
+
routes: [
|
|
38
|
+
{ matchers: ['severity="page"'], receiver: oncall },
|
|
39
|
+
{ matchers: ['severity="ticket"'], receiver: team, mute_time_intervals: [offHours] },
|
|
40
|
+
],
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
export const pageMutesTicket = new InhibitRule({
|
|
44
|
+
source_matchers: ['severity="page"'],
|
|
45
|
+
target_matchers: ['severity="ticket"'],
|
|
46
|
+
equal: ["alertname", "job"],
|
|
47
|
+
});
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
## What the checks hold you to
|
|
51
|
+
|
|
52
|
+
- One root route (the `Route` no other route nests), with a receiver and no matchers (PROM205).
|
|
53
|
+
- Routes name receivers and time intervals that exist (PROM201, PROM204). Reference the entity rather than a string and TypeScript does most of this.
|
|
54
|
+
- Every `severity` an alert in the same build root carries is matched by some route below the root (PROM202). The check only sees one build root (chant #1939): keep the rules and the Alertmanager config in the same one, or it goes quiet.
|
|
55
|
+
- Credentials go in `*_file` fields. Alertmanager does not expand environment variables in its config, and a literal `api_url`, `routing_key` or `auth_password` is flagged (PROM001).
|
|
56
|
+
- Each integration has somewhere to send (PROM209): a webhook `url`, a Slack `api_url` (or `global.slack_api_url`), a PagerDuty key, an email `to`/`smarthost`/`from` (or the global SMTP defaults).
|
|
57
|
+
|
|
58
|
+
Run `amtoolCheckConfig(alertmanagerYaml(entities))` in a test to have `amtool check-config` confirm it when installed.
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
---
|
|
2
|
+
skill: chant-prometheus-kubernetes
|
|
3
|
+
description: Render the same RuleGroups into a Prometheus Operator PrometheusRule, or into ConfigMaps for a plain Prometheus and Alertmanager on Kubernetes
|
|
4
|
+
user-invocable: true
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
# Prometheus rules on Kubernetes
|
|
8
|
+
|
|
9
|
+
A `RuleGroup` from `@intentius/chant-lexicon-prometheus` renders to two places: the rule file, and a Prometheus Operator `PrometheusRule` from `@intentius/chant-lexicon-k8s`.
|
|
10
|
+
|
|
11
|
+
## Into a PrometheusRule
|
|
12
|
+
|
|
13
|
+
```ts
|
|
14
|
+
import { PrometheusRule } from "@intentius/chant-lexicon-k8s";
|
|
15
|
+
import { RuleGroup } from "@intentius/chant-lexicon-prometheus";
|
|
16
|
+
|
|
17
|
+
const api = new RuleGroup({
|
|
18
|
+
name: "api",
|
|
19
|
+
rules: [{ alert: "ApiDown", expr: 'up{job="api"} == 0', for: "5m", labels: { severity: "page" }, annotations: { summary: "api is down" } }],
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
export const apiRules = new PrometheusRule({
|
|
23
|
+
metadata: { name: "api-rules", labels: { release: "kube-prometheus-stack" } },
|
|
24
|
+
spec: { groups: [api] },
|
|
25
|
+
});
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
The k8s serializer writes each `RuleGroup` in `spec.groups` as the same group the rule file would hold. Declare the group with `const` (not exported) when it should only appear inside the CRD; export it too when the build should also write the rule file.
|
|
29
|
+
|
|
30
|
+
`MonitoredService` takes the groups directly:
|
|
31
|
+
|
|
32
|
+
```ts
|
|
33
|
+
MonitoredService({ name: "api", image: "api:1.0", ruleGroups: [api] });
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
Its older `alertRules` prop still works, and both may be set; the groups come after the `alertRules` group.
|
|
37
|
+
|
|
38
|
+
## Without the operator
|
|
39
|
+
|
|
40
|
+
For a plain Prometheus or Alertmanager Deployment, put the files in ConfigMaps with `ruleFileYaml(groups)` and `alertmanagerYaml(entities)`:
|
|
41
|
+
|
|
42
|
+
```ts
|
|
43
|
+
import { ConfigMap } from "@intentius/chant-lexicon-k8s";
|
|
44
|
+
import { ruleFileYaml, alertmanagerYaml } from "@intentius/chant-lexicon-prometheus";
|
|
45
|
+
|
|
46
|
+
export const rulesCm = new ConfigMap({ metadata: { name: "prometheus-rules" }, data: { "rules.yml": ruleFileYaml([api]) } });
|
|
47
|
+
export const amCm = new ConfigMap({ metadata: { name: "alertmanager" }, data: { "alertmanager.yml": alertmanagerYaml([root, oncall, sink]) } });
|
|
48
|
+
```
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
---
|
|
2
|
+
skill: chant-prometheus
|
|
3
|
+
description: Declare Prometheus recording and alerting rule groups as typed chant entities and build a rule file that passes promtool
|
|
4
|
+
user-invocable: true
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
# Prometheus rules with chant
|
|
8
|
+
|
|
9
|
+
The prometheus lexicon (`@intentius/chant-lexicon-prometheus`) types Prometheus rule groups. Each `RuleGroup` is an entity; `chant build` writes the rule file Prometheus loads through `rule_files:`.
|
|
10
|
+
|
|
11
|
+
## Project setup
|
|
12
|
+
|
|
13
|
+
```ts
|
|
14
|
+
// chant.config.ts
|
|
15
|
+
export default { lexicons: ["prometheus"] };
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
## Declaring a group
|
|
19
|
+
|
|
20
|
+
A group takes the rule file's own keys (`name`, `interval`, `query_offset`, `limit`, `labels`, `rules`). Rules are plain objects: `record` + `expr` for a recording rule, `alert` + `expr` (+ `for`, `keep_firing_for`, `labels`, `annotations`) for an alerting rule.
|
|
21
|
+
|
|
22
|
+
```ts
|
|
23
|
+
import { RuleGroup } from "@intentius/chant-lexicon-prometheus";
|
|
24
|
+
|
|
25
|
+
export const api = new RuleGroup({
|
|
26
|
+
name: "api",
|
|
27
|
+
interval: "30s",
|
|
28
|
+
rules: [
|
|
29
|
+
{
|
|
30
|
+
record: "job:http_errors:ratio5m",
|
|
31
|
+
expr: 'sum by (job) (rate(http_requests_total{code=~"5.."}[5m])) / sum by (job) (rate(http_requests_total[5m]))',
|
|
32
|
+
},
|
|
33
|
+
{
|
|
34
|
+
alert: "ApiErrorRatioHigh",
|
|
35
|
+
expr: "job:http_errors:ratio5m > 0.05",
|
|
36
|
+
for: "10m",
|
|
37
|
+
labels: { severity: "page" },
|
|
38
|
+
annotations: { summary: "{{ $labels.job }} 5xx ratio above 5%", runbook_url: "https://runbooks.example.com/api-errors" },
|
|
39
|
+
},
|
|
40
|
+
],
|
|
41
|
+
});
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
Build it with `chant build src --lexicon prometheus -o dist/rules.yml`.
|
|
45
|
+
|
|
46
|
+
## Rules of thumb
|
|
47
|
+
|
|
48
|
+
- Name recording rules `level:metric:operations` (`job:http_errors:ratio5m`), and alert on the recorded series rather than repeating the expression.
|
|
49
|
+
- Give every alert a `severity` label (PROM106) that an Alertmanager route matches (PROM202), and a `summary` annotation (PROM107).
|
|
50
|
+
- Alerts may share a name when their labels differ, e.g. the same alert at `severity: "page"` and `severity: "ticket"`. Two rules with the same name *and* labels overwrite each other (PROM102).
|
|
51
|
+
- Durations are Prometheus durations: `30s`, `5m`, `1h30m`. Not `1.5h`, not `90 seconds` (PROM103).
|
|
52
|
+
- Every `expr` is parsed with the Prometheus project's PromQL grammar at build time (PROM104) and, for literals, in the editor (PROM002).
|
|
53
|
+
|
|
54
|
+
## Checking with promtool
|
|
55
|
+
|
|
56
|
+
```ts
|
|
57
|
+
import { ruleFileYaml, promtoolCheckRules } from "@intentius/chant-lexicon-prometheus";
|
|
58
|
+
|
|
59
|
+
const result = promtoolCheckRules(ruleFileYaml([api]));
|
|
60
|
+
if (result.ran && !result.ok) throw new Error(result.output);
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
`ran` is false when `promtool` isn't on PATH (or `$PROMTOOL`), so a test can skip instead of failing.
|
|
64
|
+
|
|
65
|
+
## Building rules from data
|
|
66
|
+
|
|
67
|
+
`RuleGroup` takes plain rule objects, so a composite can generate them: map over services or windows, return one or more `RuleGroup`s, and the rule file and a `PrometheusRule` both pick them up. `ruleGroupConfig(group)` gives the plain group as it appears in the file.
|
|
68
|
+
|
|
69
|
+
## SLOs
|
|
70
|
+
|
|
71
|
+
`Slo` builds an SLO to one `RuleGroup`: error ratios per window, the error budget left, and the SRE Workbook's multiwindow burn-rate alerts (page at 1h/5m and 6h/30m, ticket at 1d/2h and 3d/6h, factors scaled to the SLO window). It is a composite, called without `new`.
|
|
72
|
+
|
|
73
|
+
```ts
|
|
74
|
+
import { Slo, sloMetrics } from "@intentius/chant-lexicon-prometheus";
|
|
75
|
+
|
|
76
|
+
export const checkout = Slo({
|
|
77
|
+
name: "checkout",
|
|
78
|
+
objective: 0.999,
|
|
79
|
+
window: "30d",
|
|
80
|
+
sli: {
|
|
81
|
+
good: 'sum(rate(http_requests_total{job="checkout",code!~"5.."}[{{window}}]))',
|
|
82
|
+
total: 'sum(rate(http_requests_total{job="checkout"}[{{window}}]))',
|
|
83
|
+
},
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
sloMetrics(checkout).errorRatio["1h"]; // "slo:sli_error:ratio_rate1h"
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
- Write the objective as a fraction strictly between 0 and 1, and put `{{window}}` where each SLI expression's range goes (PROM003).
|
|
90
|
+
- Alerts carry `severity` `page` or `ticket`; route both (PROM202).
|
|
91
|
+
- Read series names from `sloMetrics()` in dashboards instead of repeating them.
|