@intentius/chant-lexicon-prometheus 0.124.30 → 0.124.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/alertmanager.js +61 -0
- package/dist/api/alertmanager.js +98 -0
- package/dist/api/apply.js +306 -0
- package/dist/api/bind.js +66 -0
- package/dist/api/client.js +110 -0
- package/dist/api/evaluated.js +86 -0
- package/dist/api/fake-servers.js +165 -0
- package/dist/api/index.js +17 -0
- package/dist/api/ruler.js +122 -0
- package/dist/build.js +240 -0
- package/dist/catalog.js +66 -0
- package/dist/codegen/docs-cli.js +3 -0
- package/dist/codegen/docs.js +105 -0
- package/dist/codegen/generate-cli.js +7 -0
- package/dist/codegen/generate.js +42 -0
- package/dist/codegen/package.js +28 -0
- package/dist/collector-metrics.js +225 -0
- package/dist/composites/alert-routing.js +210 -0
- package/dist/composites/catalog.js +344 -0
- package/dist/composites/genai.js +454 -0
- package/dist/composites/index.js +8 -0
- package/dist/composites/red-alerts.js +135 -0
- package/dist/composites/slo.js +353 -0
- package/dist/composites/watchdog.js +110 -0
- package/dist/config-model.js +52 -0
- package/dist/config.js +225 -0
- package/dist/describe-resources.js +299 -0
- package/dist/detect.js +12 -0
- package/dist/duration.js +54 -0
- package/dist/export-resources.js +163 -0
- package/dist/import/alertmanager-live.js +309 -0
- package/dist/import/embedded.js +100 -0
- package/dist/import/generator.js +683 -0
- package/dist/import/parser.js +428 -0
- package/dist/import/slo.js +207 -0
- package/dist/index.js +33 -0
- package/dist/init-templates.js +119 -0
- package/dist/integrations.js +33 -0
- package/dist/integrity.json +2 -2
- package/dist/lint/audit-catalog.js +60 -0
- package/dist/lint/audit-lineage.js +42 -0
- package/dist/lint/post-synth/index.js +68 -0
- package/dist/lint/post-synth/prom-helpers.js +157 -0
- package/dist/lint/post-synth/prom101.js +13 -0
- package/dist/lint/post-synth/prom102.js +13 -0
- package/dist/lint/post-synth/prom103.js +13 -0
- package/dist/lint/post-synth/prom104.js +13 -0
- package/dist/lint/post-synth/prom105.js +13 -0
- package/dist/lint/post-synth/prom106.js +13 -0
- package/dist/lint/post-synth/prom107.js +13 -0
- package/dist/lint/post-synth/prom201.js +13 -0
- package/dist/lint/post-synth/prom202.js +13 -0
- package/dist/lint/post-synth/prom203.js +13 -0
- package/dist/lint/post-synth/prom204.js +13 -0
- package/dist/lint/post-synth/prom205.js +13 -0
- package/dist/lint/post-synth/prom206.js +13 -0
- package/dist/lint/post-synth/prom207.js +13 -0
- package/dist/lint/post-synth/prom208.js +13 -0
- package/dist/lint/post-synth/prom209.js +13 -0
- package/dist/lint/post-synth/prom210.js +13 -0
- package/dist/lint/post-synth/prom211.js +17 -0
- package/dist/lint/post-synth/prom212.js +14 -0
- package/dist/lint/post-synth/prom213.js +16 -0
- package/dist/lint/post-synth/prom214.js +14 -0
- package/dist/lint/post-synth/prom215.js +14 -0
- package/dist/lint/post-synth/prom216.js +15 -0
- package/dist/lint/post-synth/prom217.js +14 -0
- package/dist/lint/post-synth/prom218.js +15 -0
- package/dist/lint/post-synth/prom219.js +14 -0
- package/dist/lint/post-synth/prom220.js +14 -0
- package/dist/lint/post-synth/prom221.js +15 -0
- package/dist/lint/post-synth/prom222.js +14 -0
- package/dist/lint/post-synth/prom223.js +15 -0
- package/dist/lint/post-synth/prom224.js +14 -0
- package/dist/lint/post-synth/prom225.js +15 -0
- package/dist/lint/post-synth/prom301.js +19 -0
- package/dist/lint/rules/index.js +8 -0
- package/dist/lint/rules/literal-credential.js +127 -0
- package/dist/lint/rules/prom-ast.js +65 -0
- package/dist/lint/rules/promql-literal.js +65 -0
- package/dist/lint/rules/slo-literal.js +79 -0
- package/dist/lsp/completions.js +11 -0
- package/dist/lsp/hover.js +21 -0
- package/dist/manifest.json +1 -1
- package/dist/matchers.js +131 -0
- package/dist/model.js +108 -0
- package/dist/op/activities/index.js +20 -0
- package/dist/op/activities/promtool.js +87 -0
- package/dist/op/activities/rule-audit.js +128 -0
- package/dist/op/activities/rules-loaded.js +73 -0
- package/dist/op/activities/silence.js +123 -0
- package/dist/op/activity-contracts.js +51 -0
- package/dist/op/audit-op.js +28 -0
- package/dist/op/builders.js +57 -0
- package/dist/op/slo-rule-tests.js +148 -0
- package/dist/package-cli.js +14 -0
- package/dist/pin.js +13 -0
- package/dist/plugin.js +143 -0
- package/dist/prometheus-config.js +38 -0
- package/dist/promql-analysis.js +343 -0
- package/dist/promql.js +44 -0
- package/dist/rule-eval.js +466 -0
- package/dist/rules.js +97 -0
- package/dist/serializer.js +59 -0
- package/dist/skill-defs.js +41 -0
- package/dist/tools.js +85 -0
- package/dist/validate-cli.js +7 -0
- package/dist/validate-config.js +556 -0
- package/dist/validate-integrations.js +388 -0
- package/dist/validate-relabel.js +52 -0
- package/dist/validate-security.js +135 -0
- package/dist/validate.js +69 -0
- package/package.json +10 -10
- package/src/codegen/package.ts +2 -2
|
@@ -0,0 +1,683 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `TemplateIR` -> TypeScript, for `chant import`.
|
|
3
|
+
*
|
|
4
|
+
* Writes the files the way the lexicon's examples are written: nested
|
|
5
|
+
* settings lifted into named consts typed by the model (COR001), at most
|
|
6
|
+
* eight declarables per module (COR009), and an `export { ... }` list at the
|
|
7
|
+
* end of each module (COR004 flags an exported declaration nothing in its
|
|
8
|
+
* file uses).
|
|
9
|
+
*
|
|
10
|
+
* A rule file becomes `rules.ts`: per group, its rules as a `Rule[]` const
|
|
11
|
+
* of plain objects, the same shape `ruleGroupConfig` emits, and a
|
|
12
|
+
* `RuleGroup`. A group an `Slo()` built is written as that `Slo` call in
|
|
13
|
+
* `slos.ts` instead (see ./slo.ts).
|
|
14
|
+
*
|
|
15
|
+
* An `alertmanager.yml` becomes `receivers.ts`, `time-intervals.ts`,
|
|
16
|
+
* `routes.ts` (the root `Route`, its children as `RouteProps` objects),
|
|
17
|
+
* `inhibit-rules.ts` and `settings.ts`. Routes name their receivers and time
|
|
18
|
+
* intervals by the declared variable, imported from the module that
|
|
19
|
+
* declares it; a name nothing declares stays a string, and PROM201 / PROM204
|
|
20
|
+
* report it.
|
|
21
|
+
*/
|
|
22
|
+
import { ALERTMANAGER_GLOBAL_FIELDS, RECEIVER_INTEGRATION_TYPES } from "../model.js";
|
|
23
|
+
import { PROMETHEUS_PIN } from "../pin.js";
|
|
24
|
+
import { ALERTMANAGER_RESOURCE_TYPE, PROMETHEUS_RESOURCE_TYPE, RULE_FILE_RESOURCE_TYPE, } from "./parser.js";
|
|
25
|
+
import { recognizeSlo } from "./slo.js";
|
|
26
|
+
const PACKAGE = "@intentius/chant-lexicon-prometheus";
|
|
27
|
+
const RESERVED = new Set(("break case catch class const continue debugger default delete do else enum export extends false finally for " +
|
|
28
|
+
"function if import in instanceof new null return super switch this throw true try typeof var void while with " +
|
|
29
|
+
"yield let static implements interface package private protected public await arguments eval undefined").split(" "));
|
|
30
|
+
const IDENT = /^[A-Za-z_$][A-Za-z0-9_$]*$/;
|
|
31
|
+
function words(text) {
|
|
32
|
+
return text.split(/[^A-Za-z0-9]+/).filter((w) => w !== "");
|
|
33
|
+
}
|
|
34
|
+
function camel(text) {
|
|
35
|
+
return words(text)
|
|
36
|
+
.map((w, i) => (i === 0 ? w.charAt(0).toLowerCase() + w.slice(1) : w.charAt(0).toUpperCase() + w.slice(1)))
|
|
37
|
+
.join("");
|
|
38
|
+
}
|
|
39
|
+
function pascal(text) {
|
|
40
|
+
return words(text)
|
|
41
|
+
.map((w) => w.charAt(0).toUpperCase() + w.slice(1))
|
|
42
|
+
.join("");
|
|
43
|
+
}
|
|
44
|
+
// ── TypeScript literals ──────────────────────────────────────────────
|
|
45
|
+
/** An expression written as is: a reference to a declared variable. */
|
|
46
|
+
class Code {
|
|
47
|
+
constructor(code) {
|
|
48
|
+
this.code = code;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
const MAX_LINE = 100;
|
|
52
|
+
function propertyKey(k) {
|
|
53
|
+
return IDENT.test(k) ? k : JSON.stringify(k);
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* A string literal: double-quoted, single-quoted when that saves escaping
|
|
57
|
+
* `"` (PromQL label matchers read better), or a template literal when the
|
|
58
|
+
* string spans lines, as a multi-line `expr: |` does.
|
|
59
|
+
*/
|
|
60
|
+
function stringLiteral(s) {
|
|
61
|
+
if (s.includes("\n"))
|
|
62
|
+
return `\`${s.replace(/\\/g, "\\\\").replace(/`/g, "\\`").replace(/\$\{/g, "\\${")}\``;
|
|
63
|
+
const double = JSON.stringify(s);
|
|
64
|
+
if (!s.includes('"') || s.includes("'"))
|
|
65
|
+
return double;
|
|
66
|
+
return `'${double.slice(1, -1).replace(/\\"/g, '"')}'`;
|
|
67
|
+
}
|
|
68
|
+
function scalarLiteral(v) {
|
|
69
|
+
if (v instanceof Code)
|
|
70
|
+
return v.code;
|
|
71
|
+
if (v === null)
|
|
72
|
+
return "null";
|
|
73
|
+
if (typeof v === "string")
|
|
74
|
+
return stringLiteral(v);
|
|
75
|
+
if (typeof v === "number" || typeof v === "boolean")
|
|
76
|
+
return String(v);
|
|
77
|
+
return JSON.stringify(String(v));
|
|
78
|
+
}
|
|
79
|
+
function inline(v) {
|
|
80
|
+
if (Array.isArray(v))
|
|
81
|
+
return `[${v.map(inline).join(", ")}]`;
|
|
82
|
+
if (typeof v === "object" && v !== null && !(v instanceof Code)) {
|
|
83
|
+
const items = Object.entries(v)
|
|
84
|
+
.filter(([, x]) => x !== undefined)
|
|
85
|
+
.map(([k, x]) => `${propertyKey(k)}: ${inline(x)}`);
|
|
86
|
+
return items.length === 0 ? "{}" : `{ ${items.join(", ")} }`;
|
|
87
|
+
}
|
|
88
|
+
return scalarLiteral(v);
|
|
89
|
+
}
|
|
90
|
+
/** A value as a TypeScript expression, starting at column `col`, indented by `indent` spaces. */
|
|
91
|
+
export function tsLiteral(v, indent, col = indent) {
|
|
92
|
+
const one = inline(v);
|
|
93
|
+
if (!one.includes("\n") && col + one.length + 2 <= MAX_LINE)
|
|
94
|
+
return one;
|
|
95
|
+
const pad = " ".repeat(indent + 2);
|
|
96
|
+
const close = " ".repeat(indent);
|
|
97
|
+
if (Array.isArray(v)) {
|
|
98
|
+
if (v.length === 0)
|
|
99
|
+
return "[]";
|
|
100
|
+
return `[\n${v.map((x) => `${pad}${tsLiteral(x, indent + 2)},`).join("\n")}\n${close}]`;
|
|
101
|
+
}
|
|
102
|
+
if (typeof v === "object" && v !== null && !(v instanceof Code)) {
|
|
103
|
+
const lines = [];
|
|
104
|
+
for (const [k, x] of Object.entries(v)) {
|
|
105
|
+
if (x === undefined)
|
|
106
|
+
continue;
|
|
107
|
+
const key = `${pad}${propertyKey(k)}: `;
|
|
108
|
+
lines.push(`${key}${tsLiteral(x, indent + 2, key.length)},`);
|
|
109
|
+
}
|
|
110
|
+
return lines.length === 0 ? "{}" : `{\n${lines.join("\n")}\n${close}}`;
|
|
111
|
+
}
|
|
112
|
+
return scalarLiteral(v);
|
|
113
|
+
}
|
|
114
|
+
/** An object literal whose entries are already-rendered expressions, starting at column `col`. */
|
|
115
|
+
function objectOf(entries, indent, col) {
|
|
116
|
+
if (entries.length === 0)
|
|
117
|
+
return "{}";
|
|
118
|
+
const prop = ([k, v]) => k === v && IDENT.test(k) ? k : k.startsWith("...") ? k : `${propertyKey(k)}: ${v}`;
|
|
119
|
+
const one = `{ ${entries.map(prop).join(", ")} }`;
|
|
120
|
+
if (!one.includes("\n") && col + one.length + 2 <= MAX_LINE)
|
|
121
|
+
return one;
|
|
122
|
+
const pad = " ".repeat(indent + 2);
|
|
123
|
+
return `{\n${entries.map((e) => `${pad}${prop(e)},`).join("\n")}\n${" ".repeat(indent)}}`;
|
|
124
|
+
}
|
|
125
|
+
/** True for a value COR001 wants in a named const rather than inline in a constructor. */
|
|
126
|
+
function needsHoist(v) {
|
|
127
|
+
if (v instanceof Code)
|
|
128
|
+
return false;
|
|
129
|
+
if (Array.isArray(v))
|
|
130
|
+
return v.some((x) => typeof x === "object" && x !== null && !(x instanceof Code));
|
|
131
|
+
return typeof v === "object" && v !== null;
|
|
132
|
+
}
|
|
133
|
+
// ── modules ──────────────────────────────────────────────────────────
|
|
134
|
+
/** COR009's default: at most this many declarables per file. */
|
|
135
|
+
const DECLARABLES_PER_FILE = 8;
|
|
136
|
+
/** One generated module: what it imports, declares and exports. */
|
|
137
|
+
class Module {
|
|
138
|
+
constructor(path, summary) {
|
|
139
|
+
this.path = path;
|
|
140
|
+
this.summary = summary;
|
|
141
|
+
this.values = new Set();
|
|
142
|
+
this.types = new Set();
|
|
143
|
+
this.local = new Map();
|
|
144
|
+
this.body = [];
|
|
145
|
+
this.exports = [];
|
|
146
|
+
}
|
|
147
|
+
use(name, from) {
|
|
148
|
+
if (!from)
|
|
149
|
+
this.values.add(name);
|
|
150
|
+
else if (from !== this) {
|
|
151
|
+
const spec = `./${from.path.replace(/\.ts$/, "")}`;
|
|
152
|
+
const set = this.local.get(spec) ?? new Set();
|
|
153
|
+
set.add(name);
|
|
154
|
+
this.local.set(spec, set);
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
block(lines) {
|
|
158
|
+
if (this.body.length > 0)
|
|
159
|
+
this.body.push("");
|
|
160
|
+
this.body.push(...lines);
|
|
161
|
+
}
|
|
162
|
+
render() {
|
|
163
|
+
const byName = (a, b) => a.localeCompare(b, "en", { sensitivity: "base" });
|
|
164
|
+
const names = [...this.values, ...[...this.types].map((t) => `type ${t}`)].sort((a, b) => byName(a.replace(/^type /, ""), b.replace(/^type /, "")));
|
|
165
|
+
const lines = [`/** ${this.summary} */`];
|
|
166
|
+
if (names.length > 0) {
|
|
167
|
+
const one = `import { ${names.join(", ")} } from "${PACKAGE}";`;
|
|
168
|
+
lines.push(one.length <= MAX_LINE ? one : `import {\n${names.map((n) => ` ${n},`).join("\n")}\n} from "${PACKAGE}";`);
|
|
169
|
+
}
|
|
170
|
+
for (const [spec, set] of [...this.local].sort(([a], [b]) => byName(a, b))) {
|
|
171
|
+
const list = [...set].sort(byName);
|
|
172
|
+
const one = `import { ${list.join(", ")} } from "${spec}";`;
|
|
173
|
+
lines.push(one.length <= MAX_LINE ? one : `import {\n${list.map((n) => ` ${n},`).join("\n")}\n} from "${spec}";`);
|
|
174
|
+
}
|
|
175
|
+
lines.push("", ...this.body);
|
|
176
|
+
if (this.exports.length > 0) {
|
|
177
|
+
const one = `export { ${this.exports.join(", ")} };`;
|
|
178
|
+
lines.push("", one.length <= MAX_LINE ? one : `export {\n${this.exports.map((e) => ` ${e},`).join("\n")}\n};`);
|
|
179
|
+
}
|
|
180
|
+
return `${lines.join("\n")}\n`;
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
/** Split `items` into modules of at most `DECLARABLES_PER_FILE`, named `base.ts` or `base-1.ts`, `base-2.ts`, … */
|
|
184
|
+
function chunk(items, base, summary) {
|
|
185
|
+
if (items.length === 0)
|
|
186
|
+
return [];
|
|
187
|
+
if (items.length <= DECLARABLES_PER_FILE)
|
|
188
|
+
return [[new Module(`${base}.ts`, summary), items]];
|
|
189
|
+
const out = [];
|
|
190
|
+
for (let i = 0; i < items.length; i += DECLARABLES_PER_FILE) {
|
|
191
|
+
out.push([new Module(`${base}-${out.length + 1}.ts`, summary), items.slice(i, i + DECLARABLES_PER_FILE)]);
|
|
192
|
+
}
|
|
193
|
+
return out;
|
|
194
|
+
}
|
|
195
|
+
/** Variable names, unique across every module of one import. */
|
|
196
|
+
class Names {
|
|
197
|
+
constructor(reserved) {
|
|
198
|
+
this.taken = new Set();
|
|
199
|
+
for (const r of reserved)
|
|
200
|
+
this.taken.add(r);
|
|
201
|
+
}
|
|
202
|
+
/** `base`, or `base` + `suffix` when that is taken or not an identifier, then numbered. */
|
|
203
|
+
claim(text, suffix, fallback) {
|
|
204
|
+
return this.claimWith(text, suffix, fallback, []);
|
|
205
|
+
}
|
|
206
|
+
/**
|
|
207
|
+
* Like `claim`, and also reserves `name + member` for each member: a
|
|
208
|
+
* composite's members are declared under those names.
|
|
209
|
+
*/
|
|
210
|
+
claimWith(text, suffix, fallback, members) {
|
|
211
|
+
const free = (n) => !this.taken.has(n) && !RESERVED.has(n) && members.every((m) => !this.taken.has(`${n}${m}`));
|
|
212
|
+
let base = camel(text);
|
|
213
|
+
if (base === "" || /^[0-9]/.test(base))
|
|
214
|
+
base = camel(`${fallback} ${text}`);
|
|
215
|
+
if (!free(base))
|
|
216
|
+
base = `${base}${suffix}`;
|
|
217
|
+
let out = base;
|
|
218
|
+
for (let n = 2; !free(out); n++)
|
|
219
|
+
out = `${base}${n}`;
|
|
220
|
+
this.taken.add(out);
|
|
221
|
+
for (const m of members)
|
|
222
|
+
this.taken.add(`${out}${m}`);
|
|
223
|
+
return out;
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
/** The lexicon's exports a generated module may import; no variable takes one of these names. */
|
|
227
|
+
const LEXICON_NAMES = [
|
|
228
|
+
"RuleGroup",
|
|
229
|
+
"Slo",
|
|
230
|
+
"Rule",
|
|
231
|
+
"LabelSet",
|
|
232
|
+
"Route",
|
|
233
|
+
"RouteProps",
|
|
234
|
+
"Receiver",
|
|
235
|
+
"TimeInterval",
|
|
236
|
+
"TimePeriodConfig",
|
|
237
|
+
"InhibitRule",
|
|
238
|
+
"ScrapeConfig",
|
|
239
|
+
"ScrapeConfigProps",
|
|
240
|
+
"PrometheusConfig",
|
|
241
|
+
"StaticConfig",
|
|
242
|
+
"RelabelConfig",
|
|
243
|
+
"PrometheusGlobalConfig",
|
|
244
|
+
"PrometheusAlertingConfig",
|
|
245
|
+
"RemoteWriteConfig",
|
|
246
|
+
"RemoteReadConfig",
|
|
247
|
+
"OtlpConfig",
|
|
248
|
+
"KubernetesSDConfig",
|
|
249
|
+
"FileSDConfig",
|
|
250
|
+
"HttpSDConfig",
|
|
251
|
+
"DnsSDConfig",
|
|
252
|
+
"Ec2SDConfig",
|
|
253
|
+
"ConsulSDConfig",
|
|
254
|
+
"AlertmanagerSettings",
|
|
255
|
+
"AlertmanagerGlobalConfig",
|
|
256
|
+
"AlertmanagerTracingConfig",
|
|
257
|
+
"WebhookConfig",
|
|
258
|
+
"EmailConfig",
|
|
259
|
+
"SlackConfig",
|
|
260
|
+
"PagerDutyConfig",
|
|
261
|
+
];
|
|
262
|
+
// ── rule files ───────────────────────────────────────────────────────
|
|
263
|
+
/** Generate the TypeScript declaring one rule file. */
|
|
264
|
+
export function generateRuleFileFiles(file) {
|
|
265
|
+
return generateRuleFile(file).files;
|
|
266
|
+
}
|
|
267
|
+
/**
|
|
268
|
+
* Generate the TypeScript declaring one rule file, and say where each of its
|
|
269
|
+
* groups is declared, in the file's order (#2962: a PrometheusRule's
|
|
270
|
+
* `spec.groups` refers to them).
|
|
271
|
+
*/
|
|
272
|
+
export function generateRuleFile(file) {
|
|
273
|
+
const names = new Names(LEXICON_NAMES);
|
|
274
|
+
const declared = new Map();
|
|
275
|
+
const slos = [];
|
|
276
|
+
const groups = [];
|
|
277
|
+
for (const g of file.groups) {
|
|
278
|
+
const props = recognizeSlo(g);
|
|
279
|
+
if (props)
|
|
280
|
+
slos.push({ group: g, props });
|
|
281
|
+
else
|
|
282
|
+
groups.push(g);
|
|
283
|
+
}
|
|
284
|
+
// Each Slo declares its rule group as `<name>Rules`, so those names are claimed first.
|
|
285
|
+
const sloVars = slos.map(({ props }) => names.claimWith(props.name, "Slo", "slo", ["Rules"]));
|
|
286
|
+
const modules = [];
|
|
287
|
+
for (const [mod, items] of chunk(groups, "rules", "Rule groups")) {
|
|
288
|
+
modules.push(mod);
|
|
289
|
+
mod.use("RuleGroup");
|
|
290
|
+
for (const g of items) {
|
|
291
|
+
const v = names.claim(g.name, "Group", "group");
|
|
292
|
+
const lines = [];
|
|
293
|
+
const entries = [["name", stringLiteral(g.name)]];
|
|
294
|
+
if (g.interval !== undefined)
|
|
295
|
+
entries.push(["interval", stringLiteral(g.interval)]);
|
|
296
|
+
if (g.query_offset !== undefined)
|
|
297
|
+
entries.push(["query_offset", stringLiteral(g.query_offset)]);
|
|
298
|
+
if (g.limit !== undefined)
|
|
299
|
+
entries.push(["limit", String(g.limit)]);
|
|
300
|
+
if (g.labels !== undefined) {
|
|
301
|
+
mod.types.add("LabelSet");
|
|
302
|
+
const c = names.claim(`${v} labels`, "", "group");
|
|
303
|
+
const head = `const ${c}: LabelSet = `;
|
|
304
|
+
lines.push(`${head}${tsLiteral(g.labels, 0, head.length)};`);
|
|
305
|
+
entries.push(["labels", c]);
|
|
306
|
+
}
|
|
307
|
+
mod.types.add("Rule");
|
|
308
|
+
const rulesConst = names.claim(`${v} rules`, "", "group");
|
|
309
|
+
const head = `const ${rulesConst}: Rule[] = `;
|
|
310
|
+
lines.push(`${head}${tsLiteral(g.rules, 0, head.length)};`);
|
|
311
|
+
entries.push(["rules", rulesConst]);
|
|
312
|
+
const ctor = `const ${v} = new RuleGroup(`;
|
|
313
|
+
lines.push(`${ctor}${objectOf(entries, 0, ctor.length)});`);
|
|
314
|
+
mod.block(lines);
|
|
315
|
+
mod.exports.push(v);
|
|
316
|
+
declared.set(g, { path: mod.path, name: v });
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
for (const [mod, items] of chunk(slos, "slos", "Service level objectives, each built by Slo() to its rule group")) {
|
|
320
|
+
modules.push(mod);
|
|
321
|
+
mod.use("Slo");
|
|
322
|
+
for (const { group, props } of items) {
|
|
323
|
+
const v = sloVars[slos.findIndex((s) => s.props === props)];
|
|
324
|
+
const head = `const ${v} = Slo(`;
|
|
325
|
+
mod.block([`${head}${tsLiteral(props, 0, head.length)});`]);
|
|
326
|
+
mod.exports.push(v);
|
|
327
|
+
declared.set(group, { path: mod.path, name: v, member: "rules" });
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
return {
|
|
331
|
+
files: modules.map((m) => ({ path: m.path, content: m.render() })),
|
|
332
|
+
groups: file.groups.map((g) => declared.get(g)),
|
|
333
|
+
};
|
|
334
|
+
}
|
|
335
|
+
// ── alertmanager.yml ─────────────────────────────────────────────────
|
|
336
|
+
/** Receiver fields the lexicon types, with the type each list is declared as. */
|
|
337
|
+
const RECEIVER_FIELD_TYPES = {
|
|
338
|
+
...Object.fromEntries(Object.entries(RECEIVER_INTEGRATION_TYPES).map(([k, t]) => [k, `${t}[]`])),
|
|
339
|
+
labels: "LabelSet",
|
|
340
|
+
};
|
|
341
|
+
/** `global:` fields the lexicon types. */
|
|
342
|
+
const GLOBAL_FIELDS = new Set(ALERTMANAGER_GLOBAL_FIELDS);
|
|
343
|
+
const AM_VERSION = PROMETHEUS_PIN.alertmanager.version;
|
|
344
|
+
/** A comment naming the fields carried as data, wrapped at the line limit. */
|
|
345
|
+
function untypedComment(what, keys) {
|
|
346
|
+
const one = keys.length === 1;
|
|
347
|
+
const text = `${what} ${keys.join(", ")} ${one ? "is not a field" : "are not fields"} Alertmanager ${AM_VERSION} defines, so ${one ? "it is" : "they are"} carried as data, untyped.`;
|
|
348
|
+
const lines = [];
|
|
349
|
+
let cur = "//";
|
|
350
|
+
for (const w of text.split(" ")) {
|
|
351
|
+
if (cur.length + 1 + w.length > MAX_LINE) {
|
|
352
|
+
lines.push(cur);
|
|
353
|
+
cur = "//";
|
|
354
|
+
}
|
|
355
|
+
cur += ` ${w}`;
|
|
356
|
+
}
|
|
357
|
+
lines.push(cur);
|
|
358
|
+
return lines;
|
|
359
|
+
}
|
|
360
|
+
/** Generate the TypeScript declaring one `alertmanager.yml`. */
|
|
361
|
+
export function generateAlertmanagerFiles(config) {
|
|
362
|
+
return generateAlertmanager(config).files;
|
|
363
|
+
}
|
|
364
|
+
/**
|
|
365
|
+
* Generate the TypeScript declaring one `alertmanager.yml`, and list every
|
|
366
|
+
* declaration it exports, module by module (#3031: a ConfigMap holding the
|
|
367
|
+
* file becomes `alertmanagerYaml([...])` over them).
|
|
368
|
+
*/
|
|
369
|
+
export function generateAlertmanager(config) {
|
|
370
|
+
const names = new Names([...LEXICON_NAMES, "global", "tracing"]);
|
|
371
|
+
const modules = [];
|
|
372
|
+
const receiverVars = new Map();
|
|
373
|
+
const intervalVars = new Map();
|
|
374
|
+
// Receivers.
|
|
375
|
+
for (const [mod, items] of chunk(config.receivers ?? [], "receivers", "Receivers: where notifications go")) {
|
|
376
|
+
modules.push(mod);
|
|
377
|
+
mod.use("Receiver");
|
|
378
|
+
for (const r of items) {
|
|
379
|
+
const v = names.claim(r.name, "Receiver", "receiver");
|
|
380
|
+
receiverVars.set(r.name, { v, mod });
|
|
381
|
+
const lines = [];
|
|
382
|
+
const entries = [["name", stringLiteral(r.name)]];
|
|
383
|
+
const untyped = {};
|
|
384
|
+
for (const [key, value] of Object.entries(r)) {
|
|
385
|
+
if (key === "name" || value === undefined)
|
|
386
|
+
continue;
|
|
387
|
+
const type = RECEIVER_FIELD_TYPES[key];
|
|
388
|
+
if (!type) {
|
|
389
|
+
untyped[key] = value;
|
|
390
|
+
continue;
|
|
391
|
+
}
|
|
392
|
+
if (!needsHoist(value)) {
|
|
393
|
+
entries.push([key, tsLiteral(value, 2, 4 + key.length)]);
|
|
394
|
+
continue;
|
|
395
|
+
}
|
|
396
|
+
mod.types.add(type.replace(/\[\]$/, ""));
|
|
397
|
+
const c = names.claim(`${v} ${key.replace(/_configs$/, "")}`, "Config", "receiver");
|
|
398
|
+
const head = `const ${c}: ${type} = `;
|
|
399
|
+
lines.push(`${head}${tsLiteral(value, 0, head.length)};`);
|
|
400
|
+
entries.push([key, c]);
|
|
401
|
+
}
|
|
402
|
+
if (Object.keys(untyped).length > 0) {
|
|
403
|
+
const c = names.claim(`${v} untyped`, "", "receiver");
|
|
404
|
+
lines.push(...untypedComment(`Receiver "${r.name}":`, Object.keys(untyped)));
|
|
405
|
+
const head = `const ${c} = `;
|
|
406
|
+
lines.push(`${head}${tsLiteral(untyped, 0, head.length)};`);
|
|
407
|
+
entries.push([`...${c}`, c]);
|
|
408
|
+
}
|
|
409
|
+
const ctor = `const ${v} = new Receiver(`;
|
|
410
|
+
lines.push(`${ctor}${objectOf(entries, 0, ctor.length)});`);
|
|
411
|
+
mod.block(lines);
|
|
412
|
+
mod.exports.push(v);
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
// Time intervals.
|
|
416
|
+
for (const [mod, items] of chunk(config.time_intervals ?? [], "time-intervals", "Time intervals routes mute or activate on")) {
|
|
417
|
+
modules.push(mod);
|
|
418
|
+
mod.use("TimeInterval");
|
|
419
|
+
for (const t of items) {
|
|
420
|
+
const v = names.claim(t.name, "Interval", "interval");
|
|
421
|
+
intervalVars.set(t.name, { v, mod });
|
|
422
|
+
const lines = [];
|
|
423
|
+
let periods = tsLiteral(t.time_intervals, 2, 20);
|
|
424
|
+
if (needsHoist(t.time_intervals)) {
|
|
425
|
+
mod.types.add("TimePeriodConfig");
|
|
426
|
+
const c = names.claim(`${v} periods`, "", "interval");
|
|
427
|
+
const head = `const ${c}: TimePeriodConfig[] = `;
|
|
428
|
+
lines.push(`${head}${tsLiteral(t.time_intervals, 0, head.length)};`);
|
|
429
|
+
periods = c;
|
|
430
|
+
}
|
|
431
|
+
const ctor = `const ${v} = new TimeInterval(`;
|
|
432
|
+
lines.push(`${ctor}${objectOf([["name", stringLiteral(t.name)], ["time_intervals", periods]], 0, ctor.length)});`);
|
|
433
|
+
mod.block(lines);
|
|
434
|
+
mod.exports.push(v);
|
|
435
|
+
}
|
|
436
|
+
}
|
|
437
|
+
// The routing tree: one Route, its children as plain RouteProps.
|
|
438
|
+
if (config.route) {
|
|
439
|
+
const mod = new Module("routes.ts", "The routing tree: the root route and its children, tried in order");
|
|
440
|
+
modules.push(mod);
|
|
441
|
+
mod.use("Route");
|
|
442
|
+
const receiverRef = (name) => {
|
|
443
|
+
const r = receiverVars.get(name);
|
|
444
|
+
if (!r)
|
|
445
|
+
return name;
|
|
446
|
+
mod.use(r.v, r.mod);
|
|
447
|
+
return new Code(r.v);
|
|
448
|
+
};
|
|
449
|
+
const intervalRef = (name) => {
|
|
450
|
+
const t = intervalVars.get(name);
|
|
451
|
+
if (!t)
|
|
452
|
+
return name;
|
|
453
|
+
mod.use(t.v, t.mod);
|
|
454
|
+
return new Code(t.v);
|
|
455
|
+
};
|
|
456
|
+
const withRefs = (route) => {
|
|
457
|
+
const out = {};
|
|
458
|
+
for (const [k, x] of Object.entries(route)) {
|
|
459
|
+
if (x === undefined)
|
|
460
|
+
continue;
|
|
461
|
+
if (k === "receiver")
|
|
462
|
+
out[k] = receiverRef(x);
|
|
463
|
+
else if (k === "mute_time_intervals" || k === "active_time_intervals")
|
|
464
|
+
out[k] = x.map(intervalRef);
|
|
465
|
+
else if (k === "routes")
|
|
466
|
+
out[k] = x.map(withRefs);
|
|
467
|
+
else
|
|
468
|
+
out[k] = x;
|
|
469
|
+
}
|
|
470
|
+
return out;
|
|
471
|
+
};
|
|
472
|
+
const root = withRefs(config.route);
|
|
473
|
+
const v = names.claim("root", "Route", "route");
|
|
474
|
+
const lines = [];
|
|
475
|
+
const entries = [];
|
|
476
|
+
for (const [key, value] of Object.entries(root)) {
|
|
477
|
+
if (!needsHoist(value)) {
|
|
478
|
+
entries.push([key, tsLiteral(value, 2, 4 + key.length)]);
|
|
479
|
+
continue;
|
|
480
|
+
}
|
|
481
|
+
const type = key === "routes" ? "RouteProps[]" : "LabelSet";
|
|
482
|
+
mod.types.add(type.replace(/\[\]$/, ""));
|
|
483
|
+
const c = names.claim(`${v} ${key === "routes" ? "children" : key}`, "", "route");
|
|
484
|
+
const head = `const ${c}: ${type} = `;
|
|
485
|
+
lines.push(`${head}${tsLiteral(value, 0, head.length)};`);
|
|
486
|
+
entries.push([key, c]);
|
|
487
|
+
}
|
|
488
|
+
const ctor = `const ${v} = new Route(`;
|
|
489
|
+
lines.push(`${ctor}${objectOf(entries, 0, ctor.length)});`);
|
|
490
|
+
mod.block(lines);
|
|
491
|
+
mod.exports.push(v);
|
|
492
|
+
}
|
|
493
|
+
// Inhibit rules.
|
|
494
|
+
let inhibitCount = 0;
|
|
495
|
+
for (const [mod, items] of chunk(config.inhibit_rules ?? [], "inhibit-rules", "Inhibit rules: alerts that mute others while they fire")) {
|
|
496
|
+
modules.push(mod);
|
|
497
|
+
mod.use("InhibitRule");
|
|
498
|
+
items.forEach((rule) => {
|
|
499
|
+
inhibitCount++;
|
|
500
|
+
const v = names.claim(rule.name ?? `inhibit rule ${inhibitCount}`, "Rule", "inhibit");
|
|
501
|
+
const entries = Object.entries(rule)
|
|
502
|
+
.filter(([, x]) => x !== undefined)
|
|
503
|
+
.map(([k, x]) => [k, tsLiteral(x, 2, 4 + k.length)]);
|
|
504
|
+
const ctor = `const ${v} = new InhibitRule(`;
|
|
505
|
+
mod.block([`${ctor}${objectOf(entries, 0, ctor.length)});`]);
|
|
506
|
+
mod.exports.push(v);
|
|
507
|
+
});
|
|
508
|
+
}
|
|
509
|
+
// global, templates and tracing.
|
|
510
|
+
if (config.global !== undefined || config.templates !== undefined || config.tracing !== undefined) {
|
|
511
|
+
const mod = new Module("settings.ts", "global settings, notification templates and tracing");
|
|
512
|
+
modules.push(mod);
|
|
513
|
+
mod.use("AlertmanagerSettings");
|
|
514
|
+
const lines = [];
|
|
515
|
+
const entries = [];
|
|
516
|
+
if (config.global !== undefined) {
|
|
517
|
+
const typed = [];
|
|
518
|
+
const untyped = {};
|
|
519
|
+
for (const [k, x] of Object.entries(config.global)) {
|
|
520
|
+
if (GLOBAL_FIELDS.has(k))
|
|
521
|
+
typed.push([k, x]);
|
|
522
|
+
else
|
|
523
|
+
untyped[k] = x;
|
|
524
|
+
}
|
|
525
|
+
mod.types.add("AlertmanagerGlobalConfig");
|
|
526
|
+
if (Object.keys(untyped).length > 0) {
|
|
527
|
+
lines.push(...untypedComment("global:", Object.keys(untyped)));
|
|
528
|
+
const head = "const globalUntyped = ";
|
|
529
|
+
lines.push(`${head}${tsLiteral(untyped, 0, head.length)};`);
|
|
530
|
+
const body = [["...globalUntyped", "globalUntyped"], ...typed.map(([k, x]) => [k, tsLiteral(x, 2, 4 + k.length)])];
|
|
531
|
+
lines.push(`const global: AlertmanagerGlobalConfig = ${objectOf(body, 0, 40)};`);
|
|
532
|
+
}
|
|
533
|
+
else {
|
|
534
|
+
const head = "const global: AlertmanagerGlobalConfig = ";
|
|
535
|
+
lines.push(`${head}${tsLiteral(Object.fromEntries(typed), 0, head.length)};`);
|
|
536
|
+
}
|
|
537
|
+
entries.push(["global", "global"]);
|
|
538
|
+
}
|
|
539
|
+
if (config.templates !== undefined)
|
|
540
|
+
entries.push(["templates", tsLiteral(config.templates, 2, 15)]);
|
|
541
|
+
if (config.tracing !== undefined) {
|
|
542
|
+
mod.types.add("AlertmanagerTracingConfig");
|
|
543
|
+
const head = "const tracing: AlertmanagerTracingConfig = ";
|
|
544
|
+
lines.push(`${head}${tsLiteral(config.tracing, 0, head.length)};`);
|
|
545
|
+
entries.push(["tracing", "tracing"]);
|
|
546
|
+
}
|
|
547
|
+
const v = names.claim("settings", "", "settings");
|
|
548
|
+
const ctor = `const ${v} = new AlertmanagerSettings(`;
|
|
549
|
+
lines.push(`${ctor}${objectOf(entries, 0, ctor.length)});`);
|
|
550
|
+
mod.block(lines);
|
|
551
|
+
mod.exports.push(v);
|
|
552
|
+
}
|
|
553
|
+
return {
|
|
554
|
+
files: modules.map((m) => ({ path: m.path, content: m.render() })),
|
|
555
|
+
declarations: modules.flatMap((m) => m.exports.map((name) => ({ path: m.path, name }))),
|
|
556
|
+
};
|
|
557
|
+
}
|
|
558
|
+
// ── prometheus.yml ───────────────────────────────────────────────────
|
|
559
|
+
/** The type each hoisted scrape config field is declared as; others use `ScrapeConfigProps["field"]`. */
|
|
560
|
+
const SCRAPE_FIELD_TYPES = {
|
|
561
|
+
static_configs: "StaticConfig[]",
|
|
562
|
+
relabel_configs: "RelabelConfig[]",
|
|
563
|
+
metric_relabel_configs: "RelabelConfig[]",
|
|
564
|
+
params: "Record<string, string[]>",
|
|
565
|
+
kubernetes_sd_configs: "KubernetesSDConfig[]",
|
|
566
|
+
file_sd_configs: "FileSDConfig[]",
|
|
567
|
+
http_sd_configs: "HttpSDConfig[]",
|
|
568
|
+
dns_sd_configs: "DnsSDConfig[]",
|
|
569
|
+
ec2_sd_configs: "Ec2SDConfig[]",
|
|
570
|
+
consul_sd_configs: "ConsulSDConfig[]",
|
|
571
|
+
};
|
|
572
|
+
/** The type each top-level section is declared as. */
|
|
573
|
+
const SECTION_TYPES = {
|
|
574
|
+
global: "PrometheusGlobalConfig",
|
|
575
|
+
alerting: "PrometheusAlertingConfig",
|
|
576
|
+
remote_write: "RemoteWriteConfig[]",
|
|
577
|
+
remote_read: "RemoteReadConfig[]",
|
|
578
|
+
otlp: "OtlpConfig",
|
|
579
|
+
storage: "Record<string, unknown>",
|
|
580
|
+
tracing: "Record<string, unknown>",
|
|
581
|
+
runtime: "Record<string, unknown>",
|
|
582
|
+
};
|
|
583
|
+
/** Generate the TypeScript declaring one `prometheus.yml`. */
|
|
584
|
+
export function generatePrometheusConfigFiles(config) {
|
|
585
|
+
return generatePrometheusConfig(config).files;
|
|
586
|
+
}
|
|
587
|
+
/**
|
|
588
|
+
* Generate the TypeScript declaring one `prometheus.yml`, and list every
|
|
589
|
+
* declaration it exports, module by module (a ConfigMap holding the file
|
|
590
|
+
* becomes `prometheusConfigYaml([...])` over them).
|
|
591
|
+
*/
|
|
592
|
+
export function generatePrometheusConfig(config) {
|
|
593
|
+
const names = new Names([...LEXICON_NAMES, ...Object.keys(SECTION_TYPES)]);
|
|
594
|
+
const modules = [];
|
|
595
|
+
for (const [mod, items] of chunk(config.scrape_configs ?? [], "scrape-configs", "Scrape jobs: where Prometheus pulls metrics from")) {
|
|
596
|
+
modules.push(mod);
|
|
597
|
+
mod.use("ScrapeConfig");
|
|
598
|
+
for (const job of items) {
|
|
599
|
+
const v = names.claim(job.job_name, "Scrape", "scrape");
|
|
600
|
+
const lines = [];
|
|
601
|
+
const entries = [["job_name", stringLiteral(job.job_name)]];
|
|
602
|
+
for (const [key, value] of Object.entries(job)) {
|
|
603
|
+
if (key === "job_name" || value === undefined)
|
|
604
|
+
continue;
|
|
605
|
+
if (!needsHoist(value)) {
|
|
606
|
+
entries.push([key, tsLiteral(value, 2, 4 + key.length)]);
|
|
607
|
+
continue;
|
|
608
|
+
}
|
|
609
|
+
const known = SCRAPE_FIELD_TYPES[key];
|
|
610
|
+
const type = known ?? (key.endsWith("_sd_configs") ? undefined : `ScrapeConfigProps["${key}"]`);
|
|
611
|
+
const c = names.claim(`${v} ${key.replace(/_configs$/, "")}`, "Config", "scrape");
|
|
612
|
+
if (type) {
|
|
613
|
+
if (type.startsWith("ScrapeConfigProps"))
|
|
614
|
+
mod.types.add("ScrapeConfigProps");
|
|
615
|
+
else if (!type.startsWith("Record<"))
|
|
616
|
+
mod.types.add(type.replace(/\[\]$/, ""));
|
|
617
|
+
const head = `const ${c}: ${type} = `;
|
|
618
|
+
lines.push(`${head}${tsLiteral(value, 0, head.length)};`);
|
|
619
|
+
}
|
|
620
|
+
else {
|
|
621
|
+
lines.push(`// ${key} is not a discovery kind the lexicon types, so it is carried as data, untyped.`);
|
|
622
|
+
const head = `const ${c} = `;
|
|
623
|
+
lines.push(`${head}${tsLiteral(value, 0, head.length)};`);
|
|
624
|
+
}
|
|
625
|
+
entries.push([key, c]);
|
|
626
|
+
}
|
|
627
|
+
const ctor = `const ${v} = new ScrapeConfig(`;
|
|
628
|
+
lines.push(`${ctor}${objectOf(entries, 0, ctor.length)});`);
|
|
629
|
+
mod.block(lines);
|
|
630
|
+
mod.exports.push(v);
|
|
631
|
+
}
|
|
632
|
+
}
|
|
633
|
+
const sectionKeys = Object.keys(config).filter((k) => k !== "scrape_configs");
|
|
634
|
+
if (sectionKeys.length > 0) {
|
|
635
|
+
const mod = new Module("prometheus.ts", "The prometheus.yml sections besides the scrape jobs");
|
|
636
|
+
modules.push(mod);
|
|
637
|
+
mod.use("PrometheusConfig");
|
|
638
|
+
const lines = [];
|
|
639
|
+
const entries = [];
|
|
640
|
+
for (const key of sectionKeys) {
|
|
641
|
+
const value = config[key];
|
|
642
|
+
const type = SECTION_TYPES[key];
|
|
643
|
+
if (!type || !needsHoist(value)) {
|
|
644
|
+
entries.push([key, tsLiteral(value, 2, 4 + key.length)]);
|
|
645
|
+
continue;
|
|
646
|
+
}
|
|
647
|
+
if (!type.startsWith("Record<"))
|
|
648
|
+
mod.types.add(type.replace(/\[\]$/, ""));
|
|
649
|
+
const c = names.claim(key.replace(/_/g, " "), "Config", "prometheus");
|
|
650
|
+
const head = `const ${c}: ${type} = `;
|
|
651
|
+
lines.push(`${head}${tsLiteral(value, 0, head.length)};`);
|
|
652
|
+
entries.push([key, c]);
|
|
653
|
+
}
|
|
654
|
+
const v = names.claim("prometheus", "Settings", "prometheus");
|
|
655
|
+
const ctor = `const ${v} = new PrometheusConfig(`;
|
|
656
|
+
lines.push(`${ctor}${objectOf(entries, 0, ctor.length)});`);
|
|
657
|
+
mod.block(lines);
|
|
658
|
+
mod.exports.push(v);
|
|
659
|
+
}
|
|
660
|
+
return {
|
|
661
|
+
files: modules.map((m) => ({ path: m.path, content: m.render() })),
|
|
662
|
+
declarations: modules.flatMap((m) => m.exports.map((name) => ({ path: m.path, name }))),
|
|
663
|
+
};
|
|
664
|
+
}
|
|
665
|
+
/** The rule file and `alertmanager.yml` TypeScript generator `chant import` runs. */
|
|
666
|
+
export class PrometheusGenerator {
|
|
667
|
+
generate(ir) {
|
|
668
|
+
const files = [];
|
|
669
|
+
for (const r of ir.resources) {
|
|
670
|
+
if (r.type === RULE_FILE_RESOURCE_TYPE) {
|
|
671
|
+
files.push(...generateRuleFileFiles(r.properties.file));
|
|
672
|
+
}
|
|
673
|
+
else if (r.type === PROMETHEUS_RESOURCE_TYPE) {
|
|
674
|
+
files.push(...generatePrometheusConfigFiles(r.properties.config));
|
|
675
|
+
}
|
|
676
|
+
else if (r.type === ALERTMANAGER_RESOURCE_TYPE) {
|
|
677
|
+
files.push(...generateAlertmanagerFiles(r.properties.config));
|
|
678
|
+
}
|
|
679
|
+
}
|
|
680
|
+
// Core writes the first file of a generate() call as a whole module for some layouts; never hand it nothing.
|
|
681
|
+
return files.length > 0 ? files : [{ path: "rules.ts", content: "// The imported file declares nothing.\n" }];
|
|
682
|
+
}
|
|
683
|
+
}
|