@intentius/chant-lexicon-prometheus 0.102.0 → 0.104.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (124) hide show
  1. package/README.md +3 -1
  2. package/dist/collector-metrics.d.ts +58 -0
  3. package/dist/collector-metrics.d.ts.map +1 -0
  4. package/dist/index.d.ts +1 -1
  5. package/dist/index.d.ts.map +1 -1
  6. package/dist/integrity.json +19 -4
  7. package/dist/lint/audit-catalog.d.ts +2 -0
  8. package/dist/lint/audit-catalog.d.ts.map +1 -1
  9. package/dist/lint/audit-lineage.d.ts +21 -0
  10. package/dist/lint/audit-lineage.d.ts.map +1 -1
  11. package/dist/lint/post-synth/index.d.ts.map +1 -1
  12. package/dist/lint/post-synth/prom-helpers.d.ts +9 -2
  13. package/dist/lint/post-synth/prom-helpers.d.ts.map +1 -1
  14. package/dist/lint/post-synth/prom211.d.ts +12 -0
  15. package/dist/lint/post-synth/prom211.d.ts.map +1 -0
  16. package/dist/lint/post-synth/prom212.d.ts +9 -0
  17. package/dist/lint/post-synth/prom212.d.ts.map +1 -0
  18. package/dist/lint/post-synth/prom213.d.ts +11 -0
  19. package/dist/lint/post-synth/prom213.d.ts.map +1 -0
  20. package/dist/lint/post-synth/prom214.d.ts +9 -0
  21. package/dist/lint/post-synth/prom214.d.ts.map +1 -0
  22. package/dist/lint/post-synth/prom215.d.ts +9 -0
  23. package/dist/lint/post-synth/prom215.d.ts.map +1 -0
  24. package/dist/lint/post-synth/prom216.d.ts +10 -0
  25. package/dist/lint/post-synth/prom216.d.ts.map +1 -0
  26. package/dist/lint/post-synth/prom217.d.ts +9 -0
  27. package/dist/lint/post-synth/prom217.d.ts.map +1 -0
  28. package/dist/lint/post-synth/prom218.d.ts +10 -0
  29. package/dist/lint/post-synth/prom218.d.ts.map +1 -0
  30. package/dist/lint/post-synth/prom219.d.ts +9 -0
  31. package/dist/lint/post-synth/prom219.d.ts.map +1 -0
  32. package/dist/lint/post-synth/prom220.d.ts +9 -0
  33. package/dist/lint/post-synth/prom220.d.ts.map +1 -0
  34. package/dist/lint/post-synth/prom221.d.ts +10 -0
  35. package/dist/lint/post-synth/prom221.d.ts.map +1 -0
  36. package/dist/lint/post-synth/prom222.d.ts +9 -0
  37. package/dist/lint/post-synth/prom222.d.ts.map +1 -0
  38. package/dist/lint/post-synth/prom223.d.ts +10 -0
  39. package/dist/lint/post-synth/prom223.d.ts.map +1 -0
  40. package/dist/lint/post-synth/prom224.d.ts +9 -0
  41. package/dist/lint/post-synth/prom224.d.ts.map +1 -0
  42. package/dist/lint/post-synth/prom301.d.ts +14 -0
  43. package/dist/lint/post-synth/prom301.d.ts.map +1 -0
  44. package/dist/lint/rules/literal-credential.d.ts.map +1 -1
  45. package/dist/manifest.json +1 -1
  46. package/dist/okf/index.md +15 -0
  47. package/dist/okf/rules/PROM211.md +11 -0
  48. package/dist/okf/rules/PROM212.md +11 -0
  49. package/dist/okf/rules/PROM213.md +11 -0
  50. package/dist/okf/rules/PROM214.md +11 -0
  51. package/dist/okf/rules/PROM215.md +11 -0
  52. package/dist/okf/rules/PROM216.md +11 -0
  53. package/dist/okf/rules/PROM217.md +11 -0
  54. package/dist/okf/rules/PROM218.md +11 -0
  55. package/dist/okf/rules/PROM219.md +11 -0
  56. package/dist/okf/rules/PROM220.md +11 -0
  57. package/dist/okf/rules/PROM221.md +11 -0
  58. package/dist/okf/rules/PROM222.md +11 -0
  59. package/dist/okf/rules/PROM223.md +11 -0
  60. package/dist/okf/rules/PROM224.md +11 -0
  61. package/dist/okf/rules/PROM301.md +11 -0
  62. package/dist/plugin.d.ts.map +1 -1
  63. package/dist/promql-analysis.d.ts +82 -0
  64. package/dist/promql-analysis.d.ts.map +1 -0
  65. package/dist/rules/literal-credential.ts +4 -0
  66. package/dist/rules/prom-helpers.ts +41 -2
  67. package/dist/rules/prom211.ts +21 -0
  68. package/dist/rules/prom212.ts +18 -0
  69. package/dist/rules/prom213.ts +20 -0
  70. package/dist/rules/prom214.ts +18 -0
  71. package/dist/rules/prom215.ts +18 -0
  72. package/dist/rules/prom216.ts +19 -0
  73. package/dist/rules/prom217.ts +18 -0
  74. package/dist/rules/prom218.ts +19 -0
  75. package/dist/rules/prom219.ts +18 -0
  76. package/dist/rules/prom220.ts +18 -0
  77. package/dist/rules/prom221.ts +19 -0
  78. package/dist/rules/prom222.ts +18 -0
  79. package/dist/rules/prom223.ts +19 -0
  80. package/dist/rules/prom224.ts +18 -0
  81. package/dist/rules/prom301.ts +23 -0
  82. package/dist/validate-config.d.ts +9 -3
  83. package/dist/validate-config.d.ts.map +1 -1
  84. package/dist/validate-integrations.d.ts +1 -0
  85. package/dist/validate-integrations.d.ts.map +1 -1
  86. package/dist/validate-security.d.ts +13 -0
  87. package/dist/validate-security.d.ts.map +1 -0
  88. package/package.json +2 -2
  89. package/src/collector-metrics.ts +262 -0
  90. package/src/composites/composites.test.ts +2 -5
  91. package/src/import/testdata/alertmanager-integrations.yml +5 -1
  92. package/src/index.ts +1 -0
  93. package/src/init-templates.test.ts +3 -1
  94. package/src/lint/audit-catalog.ts +102 -0
  95. package/src/lint/audit-lineage.test.ts +74 -0
  96. package/src/lint/audit-lineage.ts +28 -0
  97. package/src/lint/post-synth/checks.test.ts +257 -0
  98. package/src/lint/post-synth/index.ts +30 -0
  99. package/src/lint/post-synth/post-synth.test.ts +3 -1
  100. package/src/lint/post-synth/prom-helpers.ts +41 -2
  101. package/src/lint/post-synth/prom211.ts +21 -0
  102. package/src/lint/post-synth/prom212.ts +18 -0
  103. package/src/lint/post-synth/prom213.ts +20 -0
  104. package/src/lint/post-synth/prom214.ts +18 -0
  105. package/src/lint/post-synth/prom215.ts +18 -0
  106. package/src/lint/post-synth/prom216.ts +19 -0
  107. package/src/lint/post-synth/prom217.ts +18 -0
  108. package/src/lint/post-synth/prom218.ts +19 -0
  109. package/src/lint/post-synth/prom219.ts +18 -0
  110. package/src/lint/post-synth/prom220.ts +18 -0
  111. package/src/lint/post-synth/prom221.ts +19 -0
  112. package/src/lint/post-synth/prom222.ts +18 -0
  113. package/src/lint/post-synth/prom223.ts +19 -0
  114. package/src/lint/post-synth/prom224.ts +18 -0
  115. package/src/lint/post-synth/prom301.test.ts +157 -0
  116. package/src/lint/post-synth/prom301.ts +23 -0
  117. package/src/lint/rules/literal-credential.ts +4 -0
  118. package/src/lint/rules/rules.test.ts +16 -0
  119. package/src/plugin.test.ts +22 -0
  120. package/src/plugin.ts +12 -1
  121. package/src/promql-analysis.ts +365 -0
  122. package/src/validate-config.ts +274 -5
  123. package/src/validate-integrations.ts +1 -1
  124. package/src/validate-security.ts +143 -0
@@ -0,0 +1,257 @@
1
+ /**
2
+ * PROM211-PROM224 (#3363): one failing and one passing case each, and the
3
+ * lexicon's own composites, init templates and examples reporting none.
4
+ */
5
+ import { describe, expect, test } from "vitest";
6
+ import { join } from "path";
7
+ import { readdirSync } from "fs";
8
+ import { makePostSynthCtx } from "@intentius/chant-test-utils";
9
+ import { build } from "@intentius/chant/build";
10
+ import { runPostSynthChecks } from "@intentius/chant/lint/post-synth";
11
+ import { genAiMetrics } from "@intentius/chant-lexicon-otel/genai";
12
+ import { postSynthChecks } from "./index";
13
+ import { OPT_IN_CHECKS } from "../audit-catalog";
14
+ import { prometheusSerializer } from "../../serializer";
15
+ import { emitYaml } from "../../build";
16
+ import { ruleGroupConfig } from "../../rules";
17
+ import { GenAiRules, type GenAiAlerting } from "../../composites/genai";
18
+ import { Slo } from "../../composites/slo";
19
+ import type { RuleGroupConfig } from "../../model";
20
+
21
+ const NEW_CHECKS = postSynthChecks.filter((c) => c.id >= "PROM211" && c.id <= "PROM224");
22
+
23
+ /** The PROM211-PROM224 findings on a document, by check id. */
24
+ function found(text: string, ids?: string[]): string[] {
25
+ const ctx = makePostSynthCtx("prometheus", text);
26
+ return NEW_CHECKS.filter((c) => !ids || ids.includes(c.id))
27
+ .flatMap((c) => c.check(ctx))
28
+ .map((d) => d.checkId);
29
+ }
30
+
31
+ /** A rule file of one group holding `rule`, as YAML. */
32
+ function rules(rule: Record<string, unknown>, groupLabels?: Record<string, string>): string {
33
+ return JSON.stringify({ groups: [{ name: "g", ...(groupLabels ? { labels: groupLabels } : {}), rules: [rule] }] });
34
+ }
35
+
36
+ /** An alert that passes every check but the one a case varies. */
37
+ function alert(over: Record<string, unknown> = {}): string {
38
+ return rules({
39
+ alert: "A",
40
+ expr: "sum by (job) (rate(errors_total[5m])) > 1",
41
+ for: "5m",
42
+ labels: { severity: "page" },
43
+ annotations: { summary: "errors on {{ $labels.job }}", runbook_url: "https://runbooks/a" },
44
+ ...over,
45
+ });
46
+ }
47
+
48
+ const record = (name: string, expr: string) => rules({ record: name, expr });
49
+
50
+ const am = (over: Record<string, unknown> = {}) =>
51
+ JSON.stringify({ route: { receiver: "r" }, receivers: [{ name: "r", webhook_configs: [{ url: "https://hook" }] }], ...over });
52
+
53
+ describe("the rule-file checks", () => {
54
+ test("the baseline alert and recording rule report nothing", () => {
55
+ expect(found(alert())).toEqual([]);
56
+ expect(found(record("job:errors:rate5m", "sum by (job) (rate(errors_total[5m]))"))).toEqual([]);
57
+ });
58
+
59
+ test("PROM211 no for, or for: 0s", () => {
60
+ expect(found(alert({ for: undefined }))).toEqual(["PROM211"]);
61
+ expect(found(alert({ for: "0s" }))).toEqual(["PROM211"]);
62
+ expect(found(alert({ for: "1m" }))).toEqual([]);
63
+ });
64
+
65
+ test("PROM211 leaves alone an expression over a window, a multi-window and, and a heartbeat", () => {
66
+ expect(found(alert({ for: undefined, expr: "avg_over_time(job:errors:rate5m[1h]) > 1" }))).toEqual([]);
67
+ expect(found(alert({ for: undefined, expr: "(job:errors:rate1h > 1) and (job:errors:rate5m > 1)" }))).toEqual([]);
68
+ expect(found(alert({ for: undefined, expr: "vector(1)" }), ["PROM211", "PROM213"])).toEqual([]);
69
+ // A for that is not a duration is PROM103's alone.
70
+ expect(found(alert({ for: "1.5h" }))).toEqual([]);
71
+ });
72
+
73
+ test("PROM212 is the only finding on an alert with no runbook_url", () => {
74
+ expect(found(alert({ annotations: { summary: "x" } }))).toEqual(["PROM212"]);
75
+ expect(found(record("job:errors:rate5m", "sum by (job) (rate(errors_total[5m]))"), ["PROM212"])).toEqual([]);
76
+ });
77
+
78
+ test("PROM213 no comparison", () => {
79
+ expect(found(alert({ expr: "sum by (job) (rate(errors_total[5m]))" }))).toEqual(["PROM213"]);
80
+ expect(found(alert({ expr: "sum by (job) (rate(errors_total[5m])) > bool 1" }))).toEqual(["PROM213"]);
81
+ expect(found(alert({ expr: "job:errors:rate5m > 1 or job:errors:rate1h" }))).toEqual(["PROM213"]);
82
+ expect(found(alert({ expr: "absent(up{job=\"api\"})" }))).toEqual([]);
83
+ expect(found(alert({ expr: "up{job=\"api\"} unless on (instance) maintenance" }))).toEqual([]);
84
+ expect(found(alert({ expr: "sum by (job) (up == 0)" }))).toEqual([]);
85
+ });
86
+
87
+ test("PROM214 a template label the aggregation drops", () => {
88
+ const reads = (expr: string) => alert({ expr, annotations: { summary: "{{ $labels.instance }} on {{ .Labels.job }}", runbook_url: "u" } });
89
+ expect(found(reads("sum by (job) (up) == 0"))).toEqual(["PROM214"]);
90
+ expect(found(reads("sum without (instance) (up) == 0"))).toEqual(["PROM214"]);
91
+ expect(found(reads("sum by (job, instance) (up) == 0"))).toEqual([]);
92
+ expect(found(reads("up == 0"))).toEqual([]);
93
+ expect(found(reads("sum without (pod) (up) == 0"))).toEqual([]);
94
+ // One-to-one matching on (job) keeps only job.
95
+ expect(found(reads("sum by (job, instance) (up) / on (job) sum by (job) (up) < 0.5"))).toEqual(["PROM214"]);
96
+ // A label the rule sets itself is left alone.
97
+ expect(found(alert({ expr: "sum(up) == 0", labels: { severity: "page", instance: "all" }, annotations: { summary: "{{ $labels.instance }}", runbook_url: "u" } }))).toEqual([]);
98
+ });
99
+
100
+ test("PROM215 rate over a name that is not a counter's", () => {
101
+ expect(found(record("job:mem:rate5m", "sum by (job) (rate(process_resident_memory_bytes[5m]))"))).toEqual(["PROM215"]);
102
+ expect(found(record("job:mem:rate5m", "sum by (job) (increase(queue_depth[5m]))"))).toEqual(["PROM215"]);
103
+ for (const name of ["errors_total", "req_seconds_count", "req_seconds_sum", "req_seconds_bucket"]) {
104
+ expect(found(record("job:x:rate5m", `sum by (job) (rate(${name}[5m]))`))).toEqual([]);
105
+ }
106
+ expect(found(record("job:mem:deriv5m", "sum by (job) (deriv(process_resident_memory_bytes[5m]))"))).toEqual([]);
107
+ });
108
+
109
+ test("PROM216 histogram_quantile without _bucket or without le", () => {
110
+ expect(found(record("job:lat:p95", "histogram_quantile(0.95, sum by (job) (rate(req_seconds_bucket[5m])))"))).toEqual(["PROM216"]);
111
+ expect(found(record("job:lat:p95", "histogram_quantile(0.95, sum without (le) (rate(req_seconds_bucket[5m])))"))).toEqual(["PROM216"]);
112
+ expect(found(record("job:lat:p95", "histogram_quantile(0.95, sum by (job, le) (rate(req_seconds_count[5m])))"))).toEqual(["PROM216"]);
113
+ expect(found(record("job:lat:p95", "histogram_quantile(0.95, sum by (job, le) (rate(req_seconds_bucket[5m])))"))).toEqual([]);
114
+ expect(found(record("job:lat:p95", "histogram_quantile(0.95, rate(req_seconds_bucket[5m]))"))).toEqual([]);
115
+ });
116
+
117
+ test("PROM217 recording rule names", () => {
118
+ expect(found(record("errors_rate", "sum(rate(errors_total[5m]))"))).toEqual(["PROM217"]);
119
+ expect(found(record("job:errors", "sum by (job) (rate(errors_total[5m]))"))).toEqual(["PROM217"]);
120
+ expect(found(record("job::rate5m", "sum by (job) (rate(errors_total[5m]))"))).toEqual(["PROM217"]);
121
+ expect(found(record("instance_path:requests:rate5m", "sum by (instance, path) (rate(requests_total[5m]))"))).toEqual([]);
122
+ });
123
+
124
+ test("PROM218 a regex that needs none, or is anchored", () => {
125
+ expect(found(record("job:e:rate5m", 'sum by (job) (rate(errors_total{job=~"api"}[5m]))'))).toEqual(["PROM218"]);
126
+ expect(found(record("job:e:rate5m", 'sum by (job) (rate(errors_total{job!~"api"}[5m]))'))).toEqual(["PROM218"]);
127
+ expect(found(record("job:e:rate5m", 'sum by (job) (rate(errors_total{job=~"^api.*"}[5m]))'))).toEqual(["PROM218"]);
128
+ expect(found(record("job:e:rate5m", 'sum by (job) (rate(errors_total{job=~"api.*$"}[5m]))'))).toEqual(["PROM218"]);
129
+ expect(found(record("job:e:rate5m", 'sum by (job) (rate(errors_total{job=~"api|web", code=~"5.."}[5m]))'))).toEqual([]);
130
+ });
131
+
132
+ test("PROM219 alertname set on the rule or the group", () => {
133
+ expect(found(alert({ labels: { severity: "page", alertname: "B" } }))).toEqual(["PROM219"]);
134
+ expect(found(rules(JSON.parse(alert()).groups[0].rules[0], { alertname: "B" }))).toEqual(["PROM219"]);
135
+ });
136
+
137
+ test("an expression that does not parse is PROM104's alone", () => {
138
+ expect(found(alert({ expr: "sum by (job) (rate(x[5m]) >" }))).toEqual([]);
139
+ });
140
+ });
141
+
142
+ describe("the alertmanager.yml checks", () => {
143
+ test("the baseline config reports nothing", () => {
144
+ expect(found(am())).toEqual([]);
145
+ });
146
+
147
+ test("PROM220 insecure_skip_verify on a receiver or in global", () => {
148
+ const insecure = { url: "https://hook", http_config: { tls_config: { insecure_skip_verify: true } } };
149
+ expect(found(am({ receivers: [{ name: "r", webhook_configs: [insecure] }] }))).toEqual(["PROM220"]);
150
+ expect(found(am({ global: { http_config: { tls_config: { insecure_skip_verify: true } } } }))).toEqual(["PROM220"]);
151
+ expect(found(am({ receivers: [{ name: "r", email_configs: [{ to: "a@b.c", smarthost: "s:25", from: "x@b.c", tls_config: { insecure_skip_verify: true } }] }] }))).toEqual(["PROM220"]);
152
+ expect(found(am({ receivers: [{ name: "r", webhook_configs: [{ url: "https://hook", http_config: { tls_config: { insecure_skip_verify: false } } }] }] }))).toEqual([]);
153
+ });
154
+
155
+ test("PROM221 SMTP auth with require_tls false", () => {
156
+ const email = (entry: Record<string, unknown>, global: Record<string, unknown> = {}) =>
157
+ am({ global, receivers: [{ name: "r", email_configs: [{ to: "a@b.c", smarthost: "s:25", from: "x@b.c", ...entry }] }] });
158
+ expect(found(email({ require_tls: false, auth_username: "u", auth_password_file: "/p" }))).toEqual(["PROM221"]);
159
+ expect(found(email({ auth_username: "u" }, { smtp_require_tls: false, smtp_auth_password_file: "/p" }))).toEqual(["PROM221"]);
160
+ expect(found(email({ require_tls: false }))).toEqual([]);
161
+ expect(found(email({ auth_password_file: "/p" }))).toEqual([]);
162
+ expect(found(email({ require_tls: false, auth_password_file: "/p", force_implicit_tls: true }))).toEqual([]);
163
+ });
164
+
165
+ test("PROM222 credentials sent to an http:// URL", () => {
166
+ const hook = (entry: Record<string, unknown>, global: Record<string, unknown> = {}) => am({ global, receivers: [{ name: "r", webhook_configs: [entry] }] });
167
+ expect(found(hook({ url: "http://user:pass@hook/" }))).toEqual(["PROM222"]);
168
+ expect(found(hook({ url: "http://hook/", http_config: { bearer_token_file: "/t" } }))).toEqual(["PROM222"]);
169
+ expect(found(hook({ url: "http://hook/" }, { http_config: { basic_auth: { username: "u", password_file: "/p" } } }))).toEqual(["PROM222"]);
170
+ expect(found(hook({ url: "http://hook/" }))).toEqual([]);
171
+ expect(found(hook({ url: "https://hook/", http_config: { bearer_token_file: "/t" } }))).toEqual([]);
172
+ const opsgenie = (global: Record<string, unknown>) => am({ global, receivers: [{ name: "r", opsgenie_configs: [{ api_key_file: "/k" }] }] });
173
+ expect(found(opsgenie({ opsgenie_api_url: "http://opsgenie.internal/" }))).toEqual(["PROM222"]);
174
+ expect(found(opsgenie({}))).toEqual([]);
175
+ });
176
+
177
+ test("PROM223 repeat_interval under group_interval, inherited down the tree", () => {
178
+ expect(found(am({ route: { receiver: "r", group_interval: "10m", repeat_interval: "5m" } }))).toEqual(["PROM223"]);
179
+ expect(found(am({ route: { receiver: "r", repeat_interval: "1m" } }))).toEqual(["PROM223"]);
180
+ expect(found(am({ route: { receiver: "r", group_interval: "1h", routes: [{ receiver: "r", repeat_interval: "30m" }] } }))).toEqual(["PROM223"]);
181
+ expect(found(am({ route: { receiver: "r", group_interval: "5m", repeat_interval: "1h" } }))).toEqual([]);
182
+ });
183
+
184
+ test("PROM224 an inhibit rule whose sides can match one alert, with no equal", () => {
185
+ const inhibit = (rule: Record<string, unknown>) => am({ inhibit_rules: [rule] });
186
+ expect(found(inhibit({ source_matchers: ['severity="page"'], target_matchers: ['team="db"'] }))).toEqual(["PROM224"]);
187
+ expect(found(inhibit({ source_matchers: ['severity=~"page|ticket"'], target_matchers: ['severity="ticket"'] }))).toEqual(["PROM224"]);
188
+ expect(found(inhibit({ source_matchers: ['severity="page"'], target_matchers: ['team="db"'], equal: ["alertname"] }))).toEqual([]);
189
+ expect(found(inhibit({ source_matchers: ['severity="page"'], target_matchers: ['severity="ticket"'] }))).toEqual([]);
190
+ expect(found(inhibit({ source_matchers: ['severity=~"critical"'], target_matchers: ['severity=~"warning|info"'] }))).toEqual([]);
191
+ expect(found(inhibit({ source_matchers: ['severity="page"'], target_matchers: ['severity!="page"'] }))).toEqual([]);
192
+ });
193
+ });
194
+
195
+ /** Every finding the `recommended` preset reports (PROM212, opt-in, is left out). */
196
+ function recommended(text: string): string[] {
197
+ const ctx = makePostSynthCtx("prometheus", text);
198
+ return postSynthChecks
199
+ .filter((c) => !OPT_IN_CHECKS.has(c.id))
200
+ .flatMap((c) => c.check(ctx))
201
+ .map((d) => `${d.checkId} ${d.message}`);
202
+ }
203
+
204
+ const groupYaml = (group: RuleGroupConfig) => emitYaml({ groups: [group] });
205
+
206
+ describe("the lexicon's own rules report none of the checks", () => {
207
+ const prices = [{ provider: "anthropic", model: "m1", inputPerMTok: 3, outputPerMTok: 15, currency: "USD", source: "https://example.com", asOf: "2026-09-29" }];
208
+ const alerts: GenAiAlerting = {
209
+ errorRatio: true,
210
+ latency: true,
211
+ toolErrorRatio: true,
212
+ budgets: [
213
+ { amount: 5, currency: "USD", per: "hour" },
214
+ { amount: 50, currency: "USD", per: "day" },
215
+ ],
216
+ };
217
+
218
+ test.each([
219
+ ["spans", genAiMetrics()],
220
+ ["client", genAiMetrics({ clientMetrics: "derive" })],
221
+ ] as const)("GenAiRules over the %s metrics, with and without its alerts", (_label, genAi) => {
222
+ expect(recommended(groupYaml(ruleGroupConfig(GenAiRules({ genAi, prices }).rules)))).toEqual([]);
223
+ expect(recommended(groupYaml(ruleGroupConfig(GenAiRules({ genAi, prices, alerts }).rules)))).toEqual([]);
224
+ });
225
+
226
+ test("Slo, with each SLI form and with tickets off", () => {
227
+ const calls = "traces_span_metrics_calls_total";
228
+ const slos = [
229
+ Slo({
230
+ name: "a",
231
+ objective: 0.995,
232
+ window: "28d",
233
+ sli: { good: `sum(rate(${calls}{status_code!="STATUS_CODE_ERROR"}[{{window}}]))`, total: `sum(rate(${calls}[{{window}}]))` },
234
+ }),
235
+ Slo({
236
+ name: "b",
237
+ objective: 0.999,
238
+ window: "30d",
239
+ sli: { errors: `sum(rate(${calls}{status_code="STATUS_CODE_ERROR"}[{{window}}]))`, total: `sum(rate(${calls}[{{window}}]))` },
240
+ alerting: { ticket: false },
241
+ }),
242
+ ];
243
+ for (const slo of slos) expect(recommended(groupYaml(ruleGroupConfig(slo.rules)))).toEqual([]);
244
+ });
245
+
246
+ const examplesDir = join(import.meta.dirname, "..", "..", "..", "examples");
247
+ const examples = readdirSync(examplesDir, { withFileTypes: true })
248
+ .filter((d) => d.isDirectory())
249
+ .map((d) => d.name);
250
+
251
+ test.each(examples)("the %s example", async (name) => {
252
+ const result = await build(join(examplesDir, name, "src"), [prometheusSerializer]);
253
+ expect(result.errors).toEqual([]);
254
+ const diags = runPostSynthChecks(postSynthChecks, result).filter((d) => !OPT_IN_CHECKS.has(d.checkId));
255
+ expect(diags).toEqual([]);
256
+ });
257
+ });
@@ -17,6 +17,21 @@ import { prom207 } from "./prom207";
17
17
  import { prom208 } from "./prom208";
18
18
  import { prom209 } from "./prom209";
19
19
  import { prom210 } from "./prom210";
20
+ import { prom211 } from "./prom211";
21
+ import { prom212 } from "./prom212";
22
+ import { prom213 } from "./prom213";
23
+ import { prom214 } from "./prom214";
24
+ import { prom215 } from "./prom215";
25
+ import { prom216 } from "./prom216";
26
+ import { prom217 } from "./prom217";
27
+ import { prom218 } from "./prom218";
28
+ import { prom219 } from "./prom219";
29
+ import { prom220 } from "./prom220";
30
+ import { prom221 } from "./prom221";
31
+ import { prom222 } from "./prom222";
32
+ import { prom223 } from "./prom223";
33
+ import { prom224 } from "./prom224";
34
+ import { prom301 } from "./prom301";
20
35
 
21
36
  export const postSynthChecks: PostSynthCheck[] = [
22
37
  prom101,
@@ -36,4 +51,19 @@ export const postSynthChecks: PostSynthCheck[] = [
36
51
  prom208,
37
52
  prom209,
38
53
  prom210,
54
+ prom211,
55
+ prom212,
56
+ prom213,
57
+ prom214,
58
+ prom215,
59
+ prom216,
60
+ prom217,
61
+ prom218,
62
+ prom219,
63
+ prom220,
64
+ prom221,
65
+ prom222,
66
+ prom223,
67
+ prom224,
68
+ prom301,
39
69
  ];
@@ -1,6 +1,7 @@
1
1
  import { describe, expect, test } from "vitest";
2
2
  import { makePostSynthCtx, makePostSynthCtxFromFiles } from "@intentius/chant-test-utils";
3
3
  import { postSynthChecks } from "./index";
4
+ import { OPT_IN_CHECKS } from "../audit-catalog";
4
5
  import { RECEIVER_INTEGRATION_TYPES } from "../../integrations";
5
6
  import { INTEGRATION_CASES, SLACK_APP_URL, amWith } from "../../testdata/integration-cases";
6
7
  import { prom101 } from "./prom101";
@@ -76,7 +77,8 @@ const rulesOnly = (rules: string) => makePostSynthCtx("prometheus", rules);
76
77
  const amOnly = (am: string) => makePostSynthCtx("prometheus", am);
77
78
 
78
79
  describe("a clean rule file and alertmanager.yml", () => {
79
- test.each(postSynthChecks.map((c) => [c.id, c] as const))("%s finds nothing", (_id, check) => {
80
+ // PROM212 (no runbook_url) is opt-in and RULES carries none; see checks.test.ts.
81
+ test.each(postSynthChecks.filter((c) => !OPT_IN_CHECKS.has(c.id)).map((c) => [c.id, c] as const))("%s finds nothing", (_id, check) => {
80
82
  expect(check.check(both())).toEqual([]);
81
83
  });
82
84
  });
@@ -13,10 +13,12 @@
13
13
  import type { PostSynthContext, PostSynthDiagnostic } from "@intentius/chant/lint/post-synth";
14
14
  import type { SerializerResult } from "@intentius/chant/serializer";
15
15
  import { loadAll } from "js-yaml";
16
+ import { buildCollectorConfigs, collectorMetricIssues, collectorMetrics } from "../../collector-metrics";
16
17
  import { looksLikeAlertmanagerConfig, looksLikeRuleFile, type AlertmanagerConfig, type RuleFileConfig } from "../../model";
17
18
  import {
18
19
  validateAlertmanagerConfig,
19
20
  validateRuleFile,
21
+ validateRunbookUrls,
20
22
  validateSeverityRouting,
21
23
  type PrometheusIssue,
22
24
  type PrometheusIssueCode,
@@ -62,7 +64,7 @@ function toDiagnostic(issue: PrometheusIssue, source: string): PostSynthDiagnost
62
64
  };
63
65
  }
64
66
 
65
- /** Diagnostics for one rule-file code (PROM101-PROM107) across every rule file in the output. */
67
+ /** Diagnostics for one rule-file code (PROM101-PROM107, PROM211, PROM213-PROM219) across every rule file in the output. */
66
68
  export function ruleFileDiagnostics(ctx: PostSynthContext, code: PrometheusIssueCode): PostSynthDiagnostic[] {
67
69
  return prometheusDocs(ctx).ruleFiles.flatMap(({ source, config }) =>
68
70
  validateRuleFile(config)
@@ -71,7 +73,7 @@ export function ruleFileDiagnostics(ctx: PostSynthContext, code: PrometheusIssue
71
73
  );
72
74
  }
73
75
 
74
- /** Diagnostics for one Alertmanager code (PROM201, PROM203-PROM210) across every `alertmanager.yml` in the output. */
76
+ /** Diagnostics for one Alertmanager code (PROM201, PROM203-PROM210, PROM220-PROM224) across every `alertmanager.yml` in the output. */
75
77
  export function alertmanagerDiagnostics(ctx: PostSynthContext, code: PrometheusIssueCode): PostSynthDiagnostic[] {
76
78
  return prometheusDocs(ctx).alertmanager.flatMap(({ source, config }) =>
77
79
  validateAlertmanagerConfig(config)
@@ -80,6 +82,43 @@ export function alertmanagerDiagnostics(ctx: PostSynthContext, code: PrometheusI
80
82
  );
81
83
  }
82
84
 
85
+ /** PROM212 across every rule file in the output. Opt-in: the lexicon's `recommended` lint preset leaves it out (see ../../plugin.ts). */
86
+ export function runbookDiagnostics(ctx: PostSynthContext): PostSynthDiagnostic[] {
87
+ return prometheusDocs(ctx).ruleFiles.flatMap(({ source, config }) => validateRunbookUrls(config).map((i) => toDiagnostic(i, source)));
88
+ }
89
+
90
+ /**
91
+ * PROM301: every rule expression in the output against the metrics the
92
+ * build's collector configs emit. Silent when the build has none.
93
+ */
94
+ export function collectorMetricDiagnostics(ctx: PostSynthContext): PostSynthDiagnostic[] {
95
+ const { ruleFiles } = prometheusDocs(ctx);
96
+ if (ruleFiles.length === 0) return [];
97
+ const emitted = collectorMetrics(buildCollectorConfigs(ctx));
98
+ if (!emitted) return [];
99
+ const out: PostSynthDiagnostic[] = [];
100
+ for (const { source, config } of ruleFiles) {
101
+ for (const group of Array.isArray(config?.groups) ? config.groups : []) {
102
+ for (const rule of Array.isArray(group?.rules) ? group.rules : []) {
103
+ const r = rule as unknown as Record<string, unknown>;
104
+ if (typeof r.expr !== "string") continue;
105
+ const subject = `${group.name}/${typeof r.record === "string" ? r.record : String(r.alert ?? "")}`;
106
+ for (const issue of collectorMetricIssues(r.expr, emitted)) {
107
+ const message = `rule ${subject} ${issue.message}`;
108
+ out.push({
109
+ checkId: "PROM301",
110
+ severity: "warning",
111
+ message: source && source !== "prometheus" ? `${source}: ${message}` : message,
112
+ entity: subject,
113
+ lexicon: "prometheus",
114
+ });
115
+ }
116
+ }
117
+ }
118
+ }
119
+ return out;
120
+ }
121
+
83
122
  /** PROM202: joins every rule file in the output against every Alertmanager config in it. Silent when either is absent. */
84
123
  export function routingDiagnostics(ctx: PostSynthContext): PostSynthDiagnostic[] {
85
124
  const { ruleFiles, alertmanager } = prometheusDocs(ctx);
@@ -0,0 +1,21 @@
1
+ /**
2
+ * PROM211: An alerting rule has no for, or for: 0s
3
+ *
4
+ * The alert fires on the first evaluation that matches, so one bad scrape
5
+ * pages. Left alone when the expression reads no series (vector(1), a dead
6
+ * man's switch) or already spans a window: every selector read through a
7
+ * *_over_time function, or two conditions joined by and (the multi-window
8
+ * burn-rate form, where the short window does what for would).
9
+ */
10
+
11
+ import type { PostSynthCheck, PostSynthContext, PostSynthDiagnostic } from "@intentius/chant/lint/post-synth";
12
+ import { ruleFileDiagnostics } from "./prom-helpers";
13
+
14
+ export const prom211: PostSynthCheck = {
15
+ id: "PROM211",
16
+ description: "An alerting rule has no for, or for: 0s",
17
+
18
+ check(ctx: PostSynthContext): PostSynthDiagnostic[] {
19
+ return ruleFileDiagnostics(ctx, "PROM211");
20
+ },
21
+ };
@@ -0,0 +1,18 @@
1
+ /**
2
+ * PROM212: An alerting rule has no runbook_url annotation (opt-in)
3
+ *
4
+ * Off unless enabled: the lint preset all, or a lint.rules entry for PROM212,
5
+ * turns it on. See validateRunbookUrls.
6
+ */
7
+
8
+ import type { PostSynthCheck, PostSynthContext, PostSynthDiagnostic } from "@intentius/chant/lint/post-synth";
9
+ import { runbookDiagnostics } from "./prom-helpers";
10
+
11
+ export const prom212: PostSynthCheck = {
12
+ id: "PROM212",
13
+ description: "An alerting rule has no runbook_url annotation (opt-in)",
14
+
15
+ check(ctx: PostSynthContext): PostSynthDiagnostic[] {
16
+ return runbookDiagnostics(ctx);
17
+ },
18
+ };
@@ -0,0 +1,20 @@
1
+ /**
2
+ * PROM213: An alert expression has no comparison
3
+ *
4
+ * Every series the expression returns is an alert, so an expression with no
5
+ * comparison fires for all of them. absent(), absent_over_time() and unless
6
+ * count as a condition; a comparison with bool does not. An expression that
7
+ * reads no series (vector(1), a dead man's switch) is left alone.
8
+ */
9
+
10
+ import type { PostSynthCheck, PostSynthContext, PostSynthDiagnostic } from "@intentius/chant/lint/post-synth";
11
+ import { ruleFileDiagnostics } from "./prom-helpers";
12
+
13
+ export const prom213: PostSynthCheck = {
14
+ id: "PROM213",
15
+ description: "An alert expression has no comparison",
16
+
17
+ check(ctx: PostSynthContext): PostSynthDiagnostic[] {
18
+ return ruleFileDiagnostics(ctx, "PROM213");
19
+ },
20
+ };
@@ -0,0 +1,18 @@
1
+ /**
2
+ * PROM214: An alert template reads a label the expression aggregates away
3
+ *
4
+ * A $labels.x whose x the expression's by (...) leaves out, or its without
5
+ * (...) names, renders empty. Labels the rule or its group set are left alone.
6
+ */
7
+
8
+ import type { PostSynthCheck, PostSynthContext, PostSynthDiagnostic } from "@intentius/chant/lint/post-synth";
9
+ import { ruleFileDiagnostics } from "./prom-helpers";
10
+
11
+ export const prom214: PostSynthCheck = {
12
+ id: "PROM214",
13
+ description: "An alert template reads a label the expression aggregates away",
14
+
15
+ check(ctx: PostSynthContext): PostSynthDiagnostic[] {
16
+ return ruleFileDiagnostics(ctx, "PROM214");
17
+ },
18
+ };
@@ -0,0 +1,18 @@
1
+ /**
2
+ * PROM215: rate, irate or increase over a name that is not a counter's
3
+ *
4
+ * These functions treat every drop as a counter reset. A name not ending in
5
+ * _total, _count, _sum or _bucket is read as a gauge.
6
+ */
7
+
8
+ import type { PostSynthCheck, PostSynthContext, PostSynthDiagnostic } from "@intentius/chant/lint/post-synth";
9
+ import { ruleFileDiagnostics } from "./prom-helpers";
10
+
11
+ export const prom215: PostSynthCheck = {
12
+ id: "PROM215",
13
+ description: "rate, irate or increase over a name that is not a counter's",
14
+
15
+ check(ctx: PostSynthContext): PostSynthDiagnostic[] {
16
+ return ruleFileDiagnostics(ctx, "PROM215");
17
+ },
18
+ };
@@ -0,0 +1,19 @@
1
+ /**
2
+ * PROM216: histogram_quantile over a series without _bucket, or without le
3
+ *
4
+ * A classic histogram's quantile needs its _bucket series with the le label
5
+ * kept. A native histogram has no _bucket series, so this check does not apply
6
+ * to one.
7
+ */
8
+
9
+ import type { PostSynthCheck, PostSynthContext, PostSynthDiagnostic } from "@intentius/chant/lint/post-synth";
10
+ import { ruleFileDiagnostics } from "./prom-helpers";
11
+
12
+ export const prom216: PostSynthCheck = {
13
+ id: "PROM216",
14
+ description: "histogram_quantile over a series without _bucket, or without le",
15
+
16
+ check(ctx: PostSynthContext): PostSynthDiagnostic[] {
17
+ return ruleFileDiagnostics(ctx, "PROM216");
18
+ },
19
+ };
@@ -0,0 +1,18 @@
1
+ /**
2
+ * PROM217: A recording rule name is not in level:metric:operations form
3
+ *
4
+ * Prometheus's naming convention for recorded series, e.g.
5
+ * job:http_requests:rate5m.
6
+ */
7
+
8
+ import type { PostSynthCheck, PostSynthContext, PostSynthDiagnostic } from "@intentius/chant/lint/post-synth";
9
+ import { ruleFileDiagnostics } from "./prom-helpers";
10
+
11
+ export const prom217: PostSynthCheck = {
12
+ id: "PROM217",
13
+ description: "A recording rule name is not in level:metric:operations form",
14
+
15
+ check(ctx: PostSynthContext): PostSynthDiagnostic[] {
16
+ return ruleFileDiagnostics(ctx, "PROM217");
17
+ },
18
+ };
@@ -0,0 +1,19 @@
1
+ /**
2
+ * PROM218: A regex matcher needs no regex, or is anchored
3
+ *
4
+ * A =~ or !~ with no regex metacharacters is an equality matcher written the
5
+ * slow way. Prometheus anchors every regex, so a leading ^ or trailing $ does
6
+ * nothing.
7
+ */
8
+
9
+ import type { PostSynthCheck, PostSynthContext, PostSynthDiagnostic } from "@intentius/chant/lint/post-synth";
10
+ import { ruleFileDiagnostics } from "./prom-helpers";
11
+
12
+ export const prom218: PostSynthCheck = {
13
+ id: "PROM218",
14
+ description: "A regex matcher needs no regex, or is anchored",
15
+
16
+ check(ctx: PostSynthContext): PostSynthDiagnostic[] {
17
+ return ruleFileDiagnostics(ctx, "PROM218");
18
+ },
19
+ };
@@ -0,0 +1,18 @@
1
+ /**
2
+ * PROM219: An alerting rule sets the alertname label
3
+ *
4
+ * Prometheus sets alertname to the rule's name after applying the rule's
5
+ * labels, so the value is overwritten.
6
+ */
7
+
8
+ import type { PostSynthCheck, PostSynthContext, PostSynthDiagnostic } from "@intentius/chant/lint/post-synth";
9
+ import { ruleFileDiagnostics } from "./prom-helpers";
10
+
11
+ export const prom219: PostSynthCheck = {
12
+ id: "PROM219",
13
+ description: "An alerting rule sets the alertname label",
14
+
15
+ check(ctx: PostSynthContext): PostSynthDiagnostic[] {
16
+ return ruleFileDiagnostics(ctx, "PROM219");
17
+ },
18
+ };
@@ -0,0 +1,18 @@
1
+ /**
2
+ * PROM220: A receiver turns off TLS certificate verification
3
+ *
4
+ * tls_config.insecure_skip_verify: true on a receiver, or in global, accepts
5
+ * any certificate the endpoint presents.
6
+ */
7
+
8
+ import type { PostSynthCheck, PostSynthContext, PostSynthDiagnostic } from "@intentius/chant/lint/post-synth";
9
+ import { alertmanagerDiagnostics } from "./prom-helpers";
10
+
11
+ export const prom220: PostSynthCheck = {
12
+ id: "PROM220",
13
+ description: "A receiver turns off TLS certificate verification",
14
+
15
+ check(ctx: PostSynthContext): PostSynthDiagnostic[] {
16
+ return alertmanagerDiagnostics(ctx, "PROM220");
17
+ },
18
+ };
@@ -0,0 +1,19 @@
1
+ /**
2
+ * PROM221: SMTP credentials sent with require_tls false
3
+ *
4
+ * An email receiver with SMTP auth and require_tls false (its own, or
5
+ * global.smtp_require_tls) can send the credentials in clear text. Implicit
6
+ * TLS is left alone.
7
+ */
8
+
9
+ import type { PostSynthCheck, PostSynthContext, PostSynthDiagnostic } from "@intentius/chant/lint/post-synth";
10
+ import { alertmanagerDiagnostics } from "./prom-helpers";
11
+
12
+ export const prom221: PostSynthCheck = {
13
+ id: "PROM221",
14
+ description: "SMTP credentials sent with require_tls false",
15
+
16
+ check(ctx: PostSynthContext): PostSynthDiagnostic[] {
17
+ return alertmanagerDiagnostics(ctx, "PROM221");
18
+ },
19
+ };
@@ -0,0 +1,18 @@
1
+ /**
2
+ * PROM222: Credentials sent to an http:// receiver URL
3
+ *
4
+ * An integration whose URL is http:// and whose request carries a credential:
5
+ * user info in the URL, http_config auth, or a key or token field.
6
+ */
7
+
8
+ import type { PostSynthCheck, PostSynthContext, PostSynthDiagnostic } from "@intentius/chant/lint/post-synth";
9
+ import { alertmanagerDiagnostics } from "./prom-helpers";
10
+
11
+ export const prom222: PostSynthCheck = {
12
+ id: "PROM222",
13
+ description: "Credentials sent to an http:// receiver URL",
14
+
15
+ check(ctx: PostSynthContext): PostSynthDiagnostic[] {
16
+ return alertmanagerDiagnostics(ctx, "PROM222");
17
+ },
18
+ };
@@ -0,0 +1,19 @@
1
+ /**
2
+ * PROM223: repeat_interval is shorter than group_interval
3
+ *
4
+ * Alertmanager resends only when it flushes a group, every group_interval, so
5
+ * a shorter repeat_interval never takes effect. Intervals are inherited down
6
+ * the routing tree from Alertmanager's defaults.
7
+ */
8
+
9
+ import type { PostSynthCheck, PostSynthContext, PostSynthDiagnostic } from "@intentius/chant/lint/post-synth";
10
+ import { alertmanagerDiagnostics } from "./prom-helpers";
11
+
12
+ export const prom223: PostSynthCheck = {
13
+ id: "PROM223",
14
+ description: "repeat_interval is shorter than group_interval",
15
+
16
+ check(ctx: PostSynthContext): PostSynthDiagnostic[] {
17
+ return alertmanagerDiagnostics(ctx, "PROM223");
18
+ },
19
+ };