@intentius/chant-lexicon-k8s 0.95.1 → 0.97.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/composites/catalog.d.ts.map +1 -1
- package/dist/composites/index.d.ts +4 -0
- package/dist/composites/index.d.ts.map +1 -1
- package/dist/composites/monitored-service.d.ts +19 -0
- package/dist/composites/monitored-service.d.ts.map +1 -1
- package/dist/composites/otel-collector-agent.d.ts +22 -0
- package/dist/composites/otel-collector-agent.d.ts.map +1 -1
- package/dist/composites/otel-collector-gateway.d.ts +173 -0
- package/dist/composites/otel-collector-gateway.d.ts.map +1 -0
- package/dist/composites/otel-collector-shape.d.ts +73 -0
- package/dist/composites/otel-collector-shape.d.ts.map +1 -0
- package/dist/composites/otel-collector.d.ts +19 -1
- package/dist/composites/otel-collector.d.ts.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/integrity.json +7 -3
- package/dist/lint/audit-catalog.d.ts.map +1 -1
- package/dist/lint/post-synth/index.d.ts.map +1 -1
- package/dist/lint/post-synth/otel-placement-helpers.d.ts +103 -0
- package/dist/lint/post-synth/otel-placement-helpers.d.ts.map +1 -0
- package/dist/lint/post-synth/wk8601.d.ts +17 -0
- package/dist/lint/post-synth/wk8601.d.ts.map +1 -0
- package/dist/lint/post-synth/wk8602.d.ts +23 -0
- package/dist/lint/post-synth/wk8602.d.ts.map +1 -0
- package/dist/lint/post-synth/wk8603.d.ts +19 -0
- package/dist/lint/post-synth/wk8603.d.ts.map +1 -0
- package/dist/manifest.json +1 -1
- package/dist/okf/index.md +3 -0
- package/dist/okf/rules/WK8601.md +15 -0
- package/dist/okf/rules/WK8602.md +16 -0
- package/dist/okf/rules/WK8603.md +15 -0
- package/dist/okf/types/DaemonSet.md +3 -0
- package/dist/okf/types/Service.md +1 -0
- package/dist/rules/otel-placement-helpers.ts +277 -0
- package/dist/rules/wk8601.ts +44 -0
- package/dist/rules/wk8602.ts +133 -0
- package/dist/rules/wk8603.ts +66 -0
- package/dist/serializer.d.ts.map +1 -1
- package/dist/skills/chant-k8s.md +2 -0
- package/package.json +4 -3
- package/src/composites/__snapshots__/gke-otel-collector.snapshot.test.ts.snap +401 -0
- package/src/composites/catalog.ts +109 -0
- package/src/composites/gke-otel-collector.snapshot.test.ts +51 -0
- package/src/composites/index.ts +9 -0
- package/src/composites/monitored-service.ts +40 -16
- package/src/composites/otel-collector-agent.ts +62 -26
- package/src/composites/otel-collector-gateway.test.ts +260 -0
- package/src/composites/otel-collector-gateway.ts +325 -0
- package/src/composites/otel-collector-shape.ts +141 -0
- package/src/composites/otel-collector.ts +74 -31
- package/src/index.ts +3 -0
- package/src/lint/audit-catalog.ts +3 -0
- package/src/lint/post-synth/index.ts +6 -0
- package/src/lint/post-synth/otel-placement-helpers.ts +277 -0
- package/src/lint/post-synth/otel-placement.test.ts +401 -0
- package/src/lint/post-synth/wk8601.ts +44 -0
- package/src/lint/post-synth/wk8602.ts +133 -0
- package/src/lint/post-synth/wk8603.ts +66 -0
- package/src/prometheus-rule-groups.test.ts +135 -0
- package/src/serializer.ts +9 -1
- package/src/skills/chant-k8s.md +2 -0
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Where each OpenTelemetry Collector config in a k8s build runs: the join
|
|
3
|
+
* WK8601, WK8602 and WK8603 read (chant #2899).
|
|
4
|
+
*
|
|
5
|
+
* The collector config is a YAML string inside a ConfigMap, and whether it
|
|
6
|
+
* runs once per node or as N replicas lives on another document, the workload
|
|
7
|
+
* that mounts it. A config is joined to its workload in this order:
|
|
8
|
+
*
|
|
9
|
+
* 1. The workload's `otel.chant.dev/config` annotation, written by
|
|
10
|
+
* `OtelCollector` and `OtelCollectorGateway`, naming the ConfigMap whose
|
|
11
|
+
* `config.yaml` it runs.
|
|
12
|
+
* 2. Otherwise the ConfigMaps the pod spec mounts as volumes (how
|
|
13
|
+
* `GkeOtelCollector` and hand-written manifests link them), keeping only
|
|
14
|
+
* data keys that parse as a collector config (a `service.pipelines` map).
|
|
15
|
+
* When a container passes `--config=<path>`, only keys named by those
|
|
16
|
+
* paths count.
|
|
17
|
+
*
|
|
18
|
+
* Anything this can't resolve (a ConfigMap or workload outside the build, a
|
|
19
|
+
* replica count that isn't a number) is left out, so the checks stay silent
|
|
20
|
+
* rather than guess. Like every bundle-join check they see one build root at
|
|
21
|
+
* a time (chant #1939).
|
|
22
|
+
*
|
|
23
|
+
* The annotation keys are repeated here rather than imported from the
|
|
24
|
+
* composites, whose module pulls in the otel lexicon; a test pins them to
|
|
25
|
+
* `OTEL_COLLECTOR_ANNOTATIONS`.
|
|
26
|
+
*/
|
|
27
|
+
import { type K8sManifest } from "./k8s-helpers.js";
|
|
28
|
+
export declare const PLACEMENT_ANNOTATIONS: {
|
|
29
|
+
readonly role: "otel.chant.dev/role";
|
|
30
|
+
readonly workload: "otel.chant.dev/workload";
|
|
31
|
+
readonly config: "otel.chant.dev/config";
|
|
32
|
+
readonly gateways: "otel.chant.dev/gateways";
|
|
33
|
+
};
|
|
34
|
+
/** The part of a collector config the placement checks read. */
|
|
35
|
+
export interface CollectorConfigShape {
|
|
36
|
+
receivers?: Record<string, unknown>;
|
|
37
|
+
processors?: Record<string, unknown>;
|
|
38
|
+
exporters?: Record<string, unknown>;
|
|
39
|
+
extensions?: Record<string, unknown>;
|
|
40
|
+
service: {
|
|
41
|
+
extensions?: unknown;
|
|
42
|
+
pipelines: Record<string, {
|
|
43
|
+
receivers?: unknown;
|
|
44
|
+
processors?: unknown;
|
|
45
|
+
exporters?: unknown;
|
|
46
|
+
}>;
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
/** One collector config and the workload that runs it. */
|
|
50
|
+
export interface CollectorPlacement {
|
|
51
|
+
workload: K8sManifest;
|
|
52
|
+
kind: "DaemonSet" | "Deployment" | "StatefulSet";
|
|
53
|
+
name: string;
|
|
54
|
+
namespace: string;
|
|
55
|
+
/**
|
|
56
|
+
* How many copies run at most: `spec.replicas` (1 when unset), raised to a
|
|
57
|
+
* HorizontalPodAutoscaler's `maxReplicas` when one targets the workload.
|
|
58
|
+
* Undefined for a DaemonSet, or when the count isn't a number.
|
|
59
|
+
*/
|
|
60
|
+
replicas: number | undefined;
|
|
61
|
+
configMap: string;
|
|
62
|
+
key: string;
|
|
63
|
+
config: CollectorConfigShape;
|
|
64
|
+
}
|
|
65
|
+
/** Parse a ConfigMap value as a collector config, or undefined when it isn't one. */
|
|
66
|
+
export declare function parseCollectorConfig(text: unknown): CollectorConfigShape | undefined;
|
|
67
|
+
/** The `type` of a component id: `tail_sampling/errors` is `tail_sampling`. */
|
|
68
|
+
export declare function componentType(id: string): string;
|
|
69
|
+
/** The pipelines of one signal (`traces`, `traces/sampled`, ...) as [id, pipeline] pairs. */
|
|
70
|
+
export declare function pipelinesOf(config: CollectorConfigShape, signal: string): Array<[string, {
|
|
71
|
+
receivers: string[];
|
|
72
|
+
processors: string[];
|
|
73
|
+
exporters: string[];
|
|
74
|
+
}]>;
|
|
75
|
+
/** Every pipeline, whatever its signal. */
|
|
76
|
+
export declare function allPipelines(config: CollectorConfigShape): Array<[string, {
|
|
77
|
+
receivers: string[];
|
|
78
|
+
processors: string[];
|
|
79
|
+
exporters: string[];
|
|
80
|
+
}]>;
|
|
81
|
+
/** Processor ids of `type` that a traces pipeline runs. */
|
|
82
|
+
export declare function tracePipelineProcessors(config: CollectorConfigShape, type: string): string[];
|
|
83
|
+
/** Join every collector config in the build to the workload that runs it. */
|
|
84
|
+
export declare function collectorPlacements(manifests: K8sManifest[]): CollectorPlacement[];
|
|
85
|
+
/** How a placement is described in a message: `DaemonSet observability/otel-agent`. */
|
|
86
|
+
export declare function describePlacement(p: CollectorPlacement): string;
|
|
87
|
+
/** A Service that selects a workload's pods. */
|
|
88
|
+
export interface SelectingService {
|
|
89
|
+
name: string;
|
|
90
|
+
namespace: string;
|
|
91
|
+
headless: boolean;
|
|
92
|
+
}
|
|
93
|
+
/** The Services in a workload's namespace whose selector matches its pod template labels. */
|
|
94
|
+
export declare function servicesSelecting(manifests: K8sManifest[], workload: K8sManifest): SelectingService[];
|
|
95
|
+
/**
|
|
96
|
+
* The Service a hostname names, as `name.namespace`, when it has the shape of
|
|
97
|
+
* cluster DNS: `name`, `name.ns`, `name.ns.svc` or `name.ns.svc.<domain>`.
|
|
98
|
+
* A bare `name` resolves in the caller's namespace.
|
|
99
|
+
*/
|
|
100
|
+
export declare function serviceOfHost(host: string, callerNamespace: string): string | undefined;
|
|
101
|
+
/** The host of an exporter endpoint: `host:port`, `scheme://host:port/path` or `dns:///host:port`. */
|
|
102
|
+
export declare function hostOfEndpoint(endpoint: unknown): string | undefined;
|
|
103
|
+
//# sourceMappingURL=otel-placement-helpers.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"otel-placement-helpers.d.ts","sourceRoot":"","sources":["../../../src/lint/post-synth/otel-placement-helpers.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAGH,OAAO,EAAqC,KAAK,WAAW,EAAE,MAAM,eAAe,CAAC;AAEpF,eAAO,MAAM,qBAAqB;;;;;CAKxB,CAAC;AAEX,gEAAgE;AAChE,MAAM,WAAW,oBAAoB;IACnC,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACpC,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACrC,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACpC,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACrC,OAAO,EAAE;QACP,UAAU,CAAC,EAAE,OAAO,CAAC;QACrB,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE;YAAE,SAAS,CAAC,EAAE,OAAO,CAAC;YAAC,UAAU,CAAC,EAAE,OAAO,CAAC;YAAC,SAAS,CAAC,EAAE,OAAO,CAAA;SAAE,CAAC,CAAC;KAC/F,CAAC;CACH;AAED,0DAA0D;AAC1D,MAAM,WAAW,kBAAkB;IACjC,QAAQ,EAAE,WAAW,CAAC;IACtB,IAAI,EAAE,WAAW,GAAG,YAAY,GAAG,aAAa,CAAC;IACjD,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB;;;;OAIG;IACH,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,oBAAoB,CAAC;CAC9B;AAkBD,qFAAqF;AACrF,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,OAAO,GAAG,oBAAoB,GAAG,SAAS,CAUpF;AAED,+EAA+E;AAC/E,wBAAgB,aAAa,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,CAGhD;AAMD,6FAA6F;AAC7F,wBAAgB,WAAW,CAAC,MAAM,EAAE,oBAAoB,EAAE,MAAM,EAAE,MAAM,GAAG,KAAK,CAAC,CAAC,MAAM,EAAE;IAAE,SAAS,EAAE,MAAM,EAAE,CAAC;IAAC,UAAU,EAAE,MAAM,EAAE,CAAC;IAAC,SAAS,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC,CAAC,CAO7J;AAED,2CAA2C;AAC3C,wBAAgB,YAAY,CAAC,MAAM,EAAE,oBAAoB,GAAG,KAAK,CAAC,CAAC,MAAM,EAAE;IAAE,SAAS,EAAE,MAAM,EAAE,CAAC;IAAC,UAAU,EAAE,MAAM,EAAE,CAAC;IAAC,SAAS,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC,CAAC,CAE9I;AAED,2DAA2D;AAC3D,wBAAgB,uBAAuB,CAAC,MAAM,EAAE,oBAAoB,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAM5F;AAkDD,6EAA6E;AAC7E,wBAAgB,mBAAmB,CAAC,SAAS,EAAE,WAAW,EAAE,GAAG,kBAAkB,EAAE,CA6ClF;AAED,uFAAuF;AACvF,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,kBAAkB,GAAG,MAAM,CAE/D;AAID,gDAAgD;AAChD,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED,6FAA6F;AAC7F,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,WAAW,EAAE,EAAE,QAAQ,EAAE,WAAW,GAAG,gBAAgB,EAAE,CAYrG;AAED;;;;GAIG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,eAAe,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAOvF;AAED,sGAAsG;AACtG,wBAAgB,cAAc,CAAC,QAAQ,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CAOpE"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* WK8601: `tail_sampling` in a collector that runs once per node
|
|
3
|
+
*
|
|
4
|
+
* A DaemonSet collector sees the spans produced on its own node. A trace
|
|
5
|
+
* whose services run on several nodes arrives in pieces, one piece per
|
|
6
|
+
* agent, and `tail_sampling` decides on each piece alone: an error span on
|
|
7
|
+
* node A doesn't keep the rest of the trace on node B, and latency policies
|
|
8
|
+
* measure a fragment. Nothing fails; the sampled traces are just incomplete.
|
|
9
|
+
* Tail sampling belongs on a gateway that receives whole traces (see WK8602
|
|
10
|
+
* for the multi-replica gateway case).
|
|
11
|
+
*
|
|
12
|
+
* Reads the collector config from the ConfigMap the DaemonSet runs, joined
|
|
13
|
+
* through `otel-placement-helpers.ts`. One build root at a time (chant #1939).
|
|
14
|
+
*/
|
|
15
|
+
import type { PostSynthCheck } from "@intentius/chant/lint/post-synth";
|
|
16
|
+
export declare const wk8601: PostSynthCheck;
|
|
17
|
+
//# sourceMappingURL=wk8601.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"wk8601.d.ts","sourceRoot":"","sources":["../../../src/lint/post-synth/wk8601.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAyC,MAAM,kCAAkC,CAAC;AAI9G,eAAO,MAAM,MAAM,EAAE,cAwBpB,CAAC"}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* WK8602: `tail_sampling` on a multi-replica gateway that collectors reach
|
|
3
|
+
* without trace-aware load balancing
|
|
4
|
+
*
|
|
5
|
+
* Tail sampling needs every span of a trace in the same process. Behind a
|
|
6
|
+
* ClusterIP Service, connections are spread across the gateway replicas with
|
|
7
|
+
* no regard for trace ids, so the spans of one trace land on different
|
|
8
|
+
* replicas and each samples a fragment. The fix is a `loadbalancing`
|
|
9
|
+
* exporter in front, routing by `traceID` to the replicas themselves (the
|
|
10
|
+
* `k8s` resolver on any Service that selects them, or the `dns` resolver on
|
|
11
|
+
* a headless one). A single-replica gateway passes.
|
|
12
|
+
*
|
|
13
|
+
* The check fires on evidence, not on absence: another collector in the
|
|
14
|
+
* build whose traces pipeline exports to one of the gateway's Services
|
|
15
|
+
* through `otlp`/`otlphttp`, through a `loadbalancing` exporter routed by
|
|
16
|
+
* something other than `traceID`, or through the `dns` resolver on a
|
|
17
|
+
* ClusterIP Service (one virtual IP, so one backend per connection). When
|
|
18
|
+
* the senders live in another build root, or are applications talking to
|
|
19
|
+
* the gateway directly, it stays silent (chant #1939).
|
|
20
|
+
*/
|
|
21
|
+
import type { PostSynthCheck } from "@intentius/chant/lint/post-synth";
|
|
22
|
+
export declare const wk8602: PostSynthCheck;
|
|
23
|
+
//# sourceMappingURL=wk8602.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"wk8602.d.ts","sourceRoot":"","sources":["../../../src/lint/post-synth/wk8602.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAyC,MAAM,kCAAkC,CAAC;AAsE9G,eAAO,MAAM,MAAM,EAAE,cAyCpB,CAAC"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* WK8603: `k8s_cluster` receiver in a collector that runs more than one copy
|
|
3
|
+
* without a leader elector
|
|
4
|
+
*
|
|
5
|
+
* The `k8s_cluster` receiver reports cluster-level metrics and entity events
|
|
6
|
+
* (nodes, deployments, pods, resource quotas) by watching the API server.
|
|
7
|
+
* Every copy of the collector reports every object, so in a DaemonSet the
|
|
8
|
+
* cluster is reported once per node, and in a Deployment once per replica:
|
|
9
|
+
* duplicated series and N watches against the API server. Run it in a
|
|
10
|
+
* single-replica Deployment, or set the receiver's `k8s_leader_elector` to a
|
|
11
|
+
* `k8s_leader_elector` extension enabled in `service.extensions` (both ship in
|
|
12
|
+
* otelcol-contrib and otelcol-k8s at the pinned v0.130.0), so only the
|
|
13
|
+
* leader collects.
|
|
14
|
+
*
|
|
15
|
+
* One build root at a time (chant #1939).
|
|
16
|
+
*/
|
|
17
|
+
import type { PostSynthCheck } from "@intentius/chant/lint/post-synth";
|
|
18
|
+
export declare const wk8603: PostSynthCheck;
|
|
19
|
+
//# sourceMappingURL=wk8603.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"wk8603.d.ts","sourceRoot":"","sources":["../../../src/lint/post-synth/wk8603.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAyC,MAAM,kCAAkC,CAAC;AAiB9G,eAAO,MAAM,MAAM,EAAE,cA+BpB,CAAC"}
|
package/dist/manifest.json
CHANGED
package/dist/okf/index.md
CHANGED
|
@@ -280,3 +280,6 @@ okf_version: '0.2'
|
|
|
280
280
|
* [WK8503](/rules/WK8503.md) - Workload consumes a Secret nothing in the output produces — produce it (Secret, ExternalSecret, InfisicalSecret, Certificate) or declare its provenance with...
|
|
281
281
|
* [WK8504](/rules/WK8504.md) - committed-encrypted secret declaration does not resolve — the declared file is missing, is not the named Secret, or is not encrypted
|
|
282
282
|
* [WK8505](/rules/WK8505.md) - committed-encrypted secret with no Flux decryption wiring — add decryption: 'sops' to the FluxAppFor reconciling the path that carries it. Sees one build roo...
|
|
283
|
+
* [WK8601](/rules/WK8601.md) - OpenTelemetry Collector runs tail_sampling as a per-node DaemonSet; each agent sees only its node's spans, so decisions are made on partial traces. Sees one...
|
|
284
|
+
* [WK8602](/rules/WK8602.md) - OpenTelemetry Collector gateway runs tail_sampling on more than one replica, but collectors send it traces without a loadbalancing exporter routed by traceID...
|
|
285
|
+
* [WK8603](/rules/WK8603.md) - OpenTelemetry Collector runs the k8s_cluster receiver in a DaemonSet or multi-replica workload without a k8s_leader_elector; every copy reports every cluster...
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
---
|
|
2
|
+
type: post-synth-check
|
|
3
|
+
title: WK8601
|
|
4
|
+
description: 'OpenTelemetry Collector runs tail_sampling as a per-node DaemonSet; each agent sees only its node''s spans, so decisions are made on partial traces. Sees one build root at a time (chant #1939).'
|
|
5
|
+
id: WK8601
|
|
6
|
+
severity: error
|
|
7
|
+
category: post-synth
|
|
8
|
+
lexicon: k8s
|
|
9
|
+
docs: https://intentius.io/chant/lexicons/k8s/rules/
|
|
10
|
+
---
|
|
11
|
+
OpenTelemetry Collector runs tail_sampling as a per-node DaemonSet; each agent sees only its node's spans, so decisions are made on partial traces. Sees one build root at a time (chant #1939).
|
|
12
|
+
|
|
13
|
+
## Applies to
|
|
14
|
+
|
|
15
|
+
- [DaemonSet](/types/DaemonSet.md)
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
---
|
|
2
|
+
type: post-synth-check
|
|
3
|
+
title: WK8602
|
|
4
|
+
description: 'OpenTelemetry Collector gateway runs tail_sampling on more than one replica, but collectors send it traces without a loadbalancing exporter routed by traceID. Sees one build root at a time (chant #...'
|
|
5
|
+
id: WK8602
|
|
6
|
+
severity: error
|
|
7
|
+
category: post-synth
|
|
8
|
+
lexicon: k8s
|
|
9
|
+
docs: https://intentius.io/chant/lexicons/k8s/rules/
|
|
10
|
+
---
|
|
11
|
+
OpenTelemetry Collector gateway runs tail_sampling on more than one replica, but collectors send it traces without a loadbalancing exporter routed by traceID. Sees one build root at a time (chant #1939).
|
|
12
|
+
|
|
13
|
+
## Applies to
|
|
14
|
+
|
|
15
|
+
- [DaemonSet](/types/DaemonSet.md)
|
|
16
|
+
- [Service](/types/Service.md)
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
---
|
|
2
|
+
type: post-synth-check
|
|
3
|
+
title: WK8603
|
|
4
|
+
description: OpenTelemetry Collector runs the k8s_cluster receiver in a DaemonSet or multi-replica workload without a k8s_leader_elector; every copy reports every cluster object. Sees one build root at a time (...
|
|
5
|
+
id: WK8603
|
|
6
|
+
severity: error
|
|
7
|
+
category: post-synth
|
|
8
|
+
lexicon: k8s
|
|
9
|
+
docs: https://intentius.io/chant/lexicons/k8s/rules/
|
|
10
|
+
---
|
|
11
|
+
OpenTelemetry Collector runs the k8s_cluster receiver in a DaemonSet or multi-replica workload without a k8s_leader_elector; every copy reports every cluster object. Sees one build root at a time (chant #1939).
|
|
12
|
+
|
|
13
|
+
## Applies to
|
|
14
|
+
|
|
15
|
+
- [DaemonSet](/types/DaemonSet.md)
|
|
@@ -17,3 +17,6 @@ resource_type: K8s::Apps::DaemonSet
|
|
|
17
17
|
|
|
18
18
|
- [WK8002](/rules/WK8002.md): Latest Image Tag
|
|
19
19
|
- [WK8003](/rules/WK8003.md): Missing Resource Limits
|
|
20
|
+
- [WK8601](/rules/WK8601.md): OpenTelemetry Collector runs tail_sampling as a per-node DaemonSet; each agent sees only its node's spans, so decisions are made on partial traces. Sees one build root at a time (chant #1939).
|
|
21
|
+
- [WK8602](/rules/WK8602.md): OpenTelemetry Collector gateway runs tail_sampling on more than one replica, but collectors send it traces without a loadbalancing exporter routed by traceID. Sees one build root at a time (chant #1939).
|
|
22
|
+
- [WK8603](/rules/WK8603.md): OpenTelemetry Collector runs the k8s_cluster receiver in a DaemonSet or multi-replica workload without a k8s_leader_elector; every copy reports every cluster object. Sees one build root at a time (chant #1939).
|
|
@@ -18,3 +18,4 @@ resource_type: K8s::Core::Service
|
|
|
18
18
|
- [WK8104](/rules/WK8104.md): Ports should be named — named ports improve Service and NetworkPolicy configuration
|
|
19
19
|
- [WK8301](/rules/WK8301.md): Probes required — containers should have livenessProbe and readinessProbe
|
|
20
20
|
- [WK8305](/rules/WK8305.md): Ingress port not matching Service — backend port must match a declared Service port
|
|
21
|
+
- [WK8602](/rules/WK8602.md): OpenTelemetry Collector gateway runs tail_sampling on more than one replica, but collectors send it traces without a loadbalancing exporter routed by traceID. Sees one build root at a time (chant #1939).
|
|
@@ -0,0 +1,277 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Where each OpenTelemetry Collector config in a k8s build runs: the join
|
|
3
|
+
* WK8601, WK8602 and WK8603 read (chant #2899).
|
|
4
|
+
*
|
|
5
|
+
* The collector config is a YAML string inside a ConfigMap, and whether it
|
|
6
|
+
* runs once per node or as N replicas lives on another document, the workload
|
|
7
|
+
* that mounts it. A config is joined to its workload in this order:
|
|
8
|
+
*
|
|
9
|
+
* 1. The workload's `otel.chant.dev/config` annotation, written by
|
|
10
|
+
* `OtelCollector` and `OtelCollectorGateway`, naming the ConfigMap whose
|
|
11
|
+
* `config.yaml` it runs.
|
|
12
|
+
* 2. Otherwise the ConfigMaps the pod spec mounts as volumes (how
|
|
13
|
+
* `GkeOtelCollector` and hand-written manifests link them), keeping only
|
|
14
|
+
* data keys that parse as a collector config (a `service.pipelines` map).
|
|
15
|
+
* When a container passes `--config=<path>`, only keys named by those
|
|
16
|
+
* paths count.
|
|
17
|
+
*
|
|
18
|
+
* Anything this can't resolve (a ConfigMap or workload outside the build, a
|
|
19
|
+
* replica count that isn't a number) is left out, so the checks stay silent
|
|
20
|
+
* rather than guess. Like every bundle-join check they see one build root at
|
|
21
|
+
* a time (chant #1939).
|
|
22
|
+
*
|
|
23
|
+
* The annotation keys are repeated here rather than imported from the
|
|
24
|
+
* composites, whose module pulls in the otel lexicon; a test pins them to
|
|
25
|
+
* `OTEL_COLLECTOR_ANNOTATIONS`.
|
|
26
|
+
*/
|
|
27
|
+
|
|
28
|
+
import { load } from "js-yaml";
|
|
29
|
+
import { extractContainers, extractPodSpec, type K8sManifest } from "./k8s-helpers";
|
|
30
|
+
|
|
31
|
+
export const PLACEMENT_ANNOTATIONS = {
|
|
32
|
+
role: "otel.chant.dev/role",
|
|
33
|
+
workload: "otel.chant.dev/workload",
|
|
34
|
+
config: "otel.chant.dev/config",
|
|
35
|
+
gateways: "otel.chant.dev/gateways",
|
|
36
|
+
} as const;
|
|
37
|
+
|
|
38
|
+
/** The part of a collector config the placement checks read. */
|
|
39
|
+
export interface CollectorConfigShape {
|
|
40
|
+
receivers?: Record<string, unknown>;
|
|
41
|
+
processors?: Record<string, unknown>;
|
|
42
|
+
exporters?: Record<string, unknown>;
|
|
43
|
+
extensions?: Record<string, unknown>;
|
|
44
|
+
service: {
|
|
45
|
+
extensions?: unknown;
|
|
46
|
+
pipelines: Record<string, { receivers?: unknown; processors?: unknown; exporters?: unknown }>;
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/** One collector config and the workload that runs it. */
|
|
51
|
+
export interface CollectorPlacement {
|
|
52
|
+
workload: K8sManifest;
|
|
53
|
+
kind: "DaemonSet" | "Deployment" | "StatefulSet";
|
|
54
|
+
name: string;
|
|
55
|
+
namespace: string;
|
|
56
|
+
/**
|
|
57
|
+
* How many copies run at most: `spec.replicas` (1 when unset), raised to a
|
|
58
|
+
* HorizontalPodAutoscaler's `maxReplicas` when one targets the workload.
|
|
59
|
+
* Undefined for a DaemonSet, or when the count isn't a number.
|
|
60
|
+
*/
|
|
61
|
+
replicas: number | undefined;
|
|
62
|
+
configMap: string;
|
|
63
|
+
key: string;
|
|
64
|
+
config: CollectorConfigShape;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
const PLACED_KINDS = new Set(["DaemonSet", "Deployment", "StatefulSet"]);
|
|
68
|
+
|
|
69
|
+
function ns(m: K8sManifest): string {
|
|
70
|
+
const n = m.metadata?.namespace;
|
|
71
|
+
return typeof n === "string" && n.length > 0 ? n : "default";
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
function annotations(m: K8sManifest): Record<string, unknown> {
|
|
75
|
+
const a = (m.metadata as { annotations?: unknown } | undefined)?.annotations;
|
|
76
|
+
return a && typeof a === "object" ? (a as Record<string, unknown>) : {};
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
function isRecord(v: unknown): v is Record<string, unknown> {
|
|
80
|
+
return typeof v === "object" && v !== null && !Array.isArray(v);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** Parse a ConfigMap value as a collector config, or undefined when it isn't one. */
|
|
84
|
+
export function parseCollectorConfig(text: unknown): CollectorConfigShape | undefined {
|
|
85
|
+
if (typeof text !== "string") return undefined;
|
|
86
|
+
let value: unknown;
|
|
87
|
+
try {
|
|
88
|
+
value = load(text);
|
|
89
|
+
} catch {
|
|
90
|
+
return undefined;
|
|
91
|
+
}
|
|
92
|
+
if (!isRecord(value) || !isRecord(value.service) || !isRecord(value.service.pipelines)) return undefined;
|
|
93
|
+
return value as unknown as CollectorConfigShape;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** The `type` of a component id: `tail_sampling/errors` is `tail_sampling`. */
|
|
97
|
+
export function componentType(id: string): string {
|
|
98
|
+
const slash = id.indexOf("/");
|
|
99
|
+
return slash === -1 ? id : id.slice(0, slash);
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
function stringList(v: unknown): string[] {
|
|
103
|
+
return Array.isArray(v) ? v.filter((x): x is string => typeof x === "string") : [];
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/** The pipelines of one signal (`traces`, `traces/sampled`, ...) as [id, pipeline] pairs. */
|
|
107
|
+
export function pipelinesOf(config: CollectorConfigShape, signal: string): Array<[string, { receivers: string[]; processors: string[]; exporters: string[] }]> {
|
|
108
|
+
const out: Array<[string, { receivers: string[]; processors: string[]; exporters: string[] }]> = [];
|
|
109
|
+
for (const [id, p] of Object.entries(config.service.pipelines)) {
|
|
110
|
+
if (componentType(id) !== signal || !isRecord(p)) continue;
|
|
111
|
+
out.push([id, { receivers: stringList(p.receivers), processors: stringList(p.processors), exporters: stringList(p.exporters) }]);
|
|
112
|
+
}
|
|
113
|
+
return out;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/** Every pipeline, whatever its signal. */
|
|
117
|
+
export function allPipelines(config: CollectorConfigShape): Array<[string, { receivers: string[]; processors: string[]; exporters: string[] }]> {
|
|
118
|
+
return ["traces", "metrics", "logs"].flatMap((s) => pipelinesOf(config, s));
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/** Processor ids of `type` that a traces pipeline runs. */
|
|
122
|
+
export function tracePipelineProcessors(config: CollectorConfigShape, type: string): string[] {
|
|
123
|
+
const out = new Set<string>();
|
|
124
|
+
for (const [, p] of pipelinesOf(config, "traces")) {
|
|
125
|
+
for (const id of p.processors) if (componentType(id) === type) out.add(id);
|
|
126
|
+
}
|
|
127
|
+
return [...out];
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/** The ConfigMap file names a workload's containers pass with `--config`. */
|
|
131
|
+
function configArgNames(workload: K8sManifest): string[] {
|
|
132
|
+
const names: string[] = [];
|
|
133
|
+
for (const c of extractContainers(workload)) {
|
|
134
|
+
const argv = [...stringList(c.command), ...stringList(c.args)];
|
|
135
|
+
for (let i = 0; i < argv.length; i++) {
|
|
136
|
+
let path: string | undefined;
|
|
137
|
+
if (argv[i].startsWith("--config=")) path = argv[i].slice("--config=".length);
|
|
138
|
+
else if (argv[i] === "--config") path = argv[i + 1];
|
|
139
|
+
if (!path) continue;
|
|
140
|
+
// `--config=env:...`, `--config=yaml:...` and URLs are not files.
|
|
141
|
+
if (/^[a-z]+:/.test(path)) continue;
|
|
142
|
+
names.push(path.slice(path.lastIndexOf("/") + 1));
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
return names;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/** ConfigMap names mounted by a workload's pod spec, directly or through a projected volume. */
|
|
149
|
+
function mountedConfigMaps(workload: K8sManifest): string[] {
|
|
150
|
+
const pod = extractPodSpec(workload);
|
|
151
|
+
const out: string[] = [];
|
|
152
|
+
for (const v of Array.isArray(pod?.volumes) ? pod.volumes : []) {
|
|
153
|
+
if (!isRecord(v)) continue;
|
|
154
|
+
const cm = isRecord(v.configMap) ? v.configMap.name : undefined;
|
|
155
|
+
if (typeof cm === "string") out.push(cm);
|
|
156
|
+
const sources = isRecord(v.projected) && Array.isArray(v.projected.sources) ? v.projected.sources : [];
|
|
157
|
+
for (const s of sources) {
|
|
158
|
+
const name = isRecord(s) && isRecord(s.configMap) ? s.configMap.name : undefined;
|
|
159
|
+
if (typeof name === "string") out.push(name);
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
return out;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/** The max replica count an HPA allows for kind/name in a namespace, if one targets it. */
|
|
166
|
+
function hpaMax(manifests: K8sManifest[], kind: string, name: string, namespace: string): number | undefined {
|
|
167
|
+
let max: number | undefined;
|
|
168
|
+
for (const m of manifests) {
|
|
169
|
+
if (m.kind !== "HorizontalPodAutoscaler" || ns(m) !== namespace) continue;
|
|
170
|
+
const target = isRecord(m.spec?.scaleTargetRef) ? m.spec.scaleTargetRef : undefined;
|
|
171
|
+
if (!target || target.kind !== kind || target.name !== name) continue;
|
|
172
|
+
const n = m.spec?.maxReplicas;
|
|
173
|
+
if (typeof n === "number") max = Math.max(max ?? 0, n);
|
|
174
|
+
}
|
|
175
|
+
return max;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/** Join every collector config in the build to the workload that runs it. */
|
|
179
|
+
export function collectorPlacements(manifests: K8sManifest[]): CollectorPlacement[] {
|
|
180
|
+
const configMaps = new Map<string, K8sManifest>();
|
|
181
|
+
for (const m of manifests) {
|
|
182
|
+
if (m.kind === "ConfigMap" && typeof m.metadata?.name === "string") configMaps.set(`${ns(m)}/${m.metadata.name}`, m);
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
const out: CollectorPlacement[] = [];
|
|
186
|
+
for (const workload of manifests) {
|
|
187
|
+
if (!workload.kind || !PLACED_KINDS.has(workload.kind)) continue;
|
|
188
|
+
const name = workload.metadata?.name;
|
|
189
|
+
if (typeof name !== "string") continue;
|
|
190
|
+
const namespace = ns(workload);
|
|
191
|
+
const kind = workload.kind as CollectorPlacement["kind"];
|
|
192
|
+
|
|
193
|
+
let replicas: number | undefined;
|
|
194
|
+
if (kind !== "DaemonSet") {
|
|
195
|
+
const r = workload.spec?.replicas;
|
|
196
|
+
const base = r === undefined || r === null ? 1 : typeof r === "number" ? r : undefined;
|
|
197
|
+
const hpa = hpaMax(manifests, kind, name, namespace);
|
|
198
|
+
replicas = base === undefined ? undefined : Math.max(base, hpa ?? 0);
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
const annotated = annotations(workload)[PLACEMENT_ANNOTATIONS.config];
|
|
202
|
+
const candidates: Array<{ configMap: string; keys?: string[] }> =
|
|
203
|
+
typeof annotated === "string"
|
|
204
|
+
? [{ configMap: annotated, keys: ["config.yaml"] }]
|
|
205
|
+
: mountedConfigMaps(workload).map((configMap) => ({ configMap }));
|
|
206
|
+
const argNames = configArgNames(workload);
|
|
207
|
+
|
|
208
|
+
for (const { configMap, keys } of candidates) {
|
|
209
|
+
const cm = configMaps.get(`${namespace}/${configMap}`);
|
|
210
|
+
if (!cm || !isRecord(cm.data)) continue;
|
|
211
|
+
const parsed: Array<[string, CollectorConfigShape]> = [];
|
|
212
|
+
for (const [key, text] of Object.entries(cm.data)) {
|
|
213
|
+
if (keys && !keys.includes(key)) continue;
|
|
214
|
+
const config = parseCollectorConfig(text);
|
|
215
|
+
if (config) parsed.push([key, config]);
|
|
216
|
+
}
|
|
217
|
+
const named = parsed.filter(([key]) => argNames.includes(key));
|
|
218
|
+
for (const [key, config] of named.length > 0 ? named : parsed) {
|
|
219
|
+
out.push({ workload, kind, name, namespace, replicas, configMap, key, config });
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
return out;
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
/** How a placement is described in a message: `DaemonSet observability/otel-agent`. */
|
|
227
|
+
export function describePlacement(p: CollectorPlacement): string {
|
|
228
|
+
return `${p.kind} ${p.namespace}/${p.name}`;
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
// ── Services and exporters (WK8602) ─────────────────────────────────
|
|
232
|
+
|
|
233
|
+
/** A Service that selects a workload's pods. */
|
|
234
|
+
export interface SelectingService {
|
|
235
|
+
name: string;
|
|
236
|
+
namespace: string;
|
|
237
|
+
headless: boolean;
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
/** The Services in a workload's namespace whose selector matches its pod template labels. */
|
|
241
|
+
export function servicesSelecting(manifests: K8sManifest[], workload: K8sManifest): SelectingService[] {
|
|
242
|
+
const template = isRecord(workload.spec?.template) ? workload.spec.template : undefined;
|
|
243
|
+
const labels = isRecord(template?.metadata) && isRecord(template.metadata.labels) ? template.metadata.labels : {};
|
|
244
|
+
const out: SelectingService[] = [];
|
|
245
|
+
for (const m of manifests) {
|
|
246
|
+
if (m.kind !== "Service" || ns(m) !== ns(workload) || typeof m.metadata?.name !== "string") continue;
|
|
247
|
+
const selector = isRecord(m.spec?.selector) ? m.spec.selector : undefined;
|
|
248
|
+
if (!selector || Object.keys(selector).length === 0) continue;
|
|
249
|
+
if (!Object.entries(selector).every(([k, v]) => labels[k] === v)) continue;
|
|
250
|
+
out.push({ name: m.metadata.name, namespace: ns(m), headless: m.spec?.clusterIP === "None" });
|
|
251
|
+
}
|
|
252
|
+
return out;
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
/**
|
|
256
|
+
* The Service a hostname names, as `name.namespace`, when it has the shape of
|
|
257
|
+
* cluster DNS: `name`, `name.ns`, `name.ns.svc` or `name.ns.svc.<domain>`.
|
|
258
|
+
* A bare `name` resolves in the caller's namespace.
|
|
259
|
+
*/
|
|
260
|
+
export function serviceOfHost(host: string, callerNamespace: string): string | undefined {
|
|
261
|
+
const parts = host.split(".");
|
|
262
|
+
if (parts.some((p) => p.length === 0)) return undefined;
|
|
263
|
+
if (parts.length === 1) return `${parts[0]}.${callerNamespace}`;
|
|
264
|
+
if (parts.length === 2) return `${parts[0]}.${parts[1]}`;
|
|
265
|
+
if (parts[2] === "svc") return `${parts[0]}.${parts[1]}`;
|
|
266
|
+
return undefined;
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
/** The host of an exporter endpoint: `host:port`, `scheme://host:port/path` or `dns:///host:port`. */
|
|
270
|
+
export function hostOfEndpoint(endpoint: unknown): string | undefined {
|
|
271
|
+
if (typeof endpoint !== "string" || endpoint.length === 0) return undefined;
|
|
272
|
+
let rest = endpoint.replace(/^[a-z][a-z0-9+.-]*:\/\/\/?/i, "");
|
|
273
|
+
rest = rest.split("/")[0];
|
|
274
|
+
if (rest.startsWith("[")) return undefined;
|
|
275
|
+
const colon = rest.lastIndexOf(":");
|
|
276
|
+
return colon === -1 ? rest : rest.slice(0, colon);
|
|
277
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* WK8601: `tail_sampling` in a collector that runs once per node
|
|
3
|
+
*
|
|
4
|
+
* A DaemonSet collector sees the spans produced on its own node. A trace
|
|
5
|
+
* whose services run on several nodes arrives in pieces, one piece per
|
|
6
|
+
* agent, and `tail_sampling` decides on each piece alone: an error span on
|
|
7
|
+
* node A doesn't keep the rest of the trace on node B, and latency policies
|
|
8
|
+
* measure a fragment. Nothing fails; the sampled traces are just incomplete.
|
|
9
|
+
* Tail sampling belongs on a gateway that receives whole traces (see WK8602
|
|
10
|
+
* for the multi-replica gateway case).
|
|
11
|
+
*
|
|
12
|
+
* Reads the collector config from the ConfigMap the DaemonSet runs, joined
|
|
13
|
+
* through `otel-placement-helpers.ts`. One build root at a time (chant #1939).
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import type { PostSynthCheck, PostSynthContext, PostSynthDiagnostic } from "@intentius/chant/lint/post-synth";
|
|
17
|
+
import { docsToManifests } from "./k8s-helpers";
|
|
18
|
+
import { collectorPlacements, describePlacement, tracePipelineProcessors } from "./otel-placement-helpers";
|
|
19
|
+
|
|
20
|
+
export const wk8601: PostSynthCheck = {
|
|
21
|
+
id: "WK8601",
|
|
22
|
+
description:
|
|
23
|
+
"OpenTelemetry Collector runs tail_sampling as a per-node DaemonSet; each agent sees only its node's spans, so decisions are made on partial traces. Sees one build root at a time (chant #1939).",
|
|
24
|
+
|
|
25
|
+
check(ctx: PostSynthContext): PostSynthDiagnostic[] {
|
|
26
|
+
const diagnostics: PostSynthDiagnostic[] = [];
|
|
27
|
+
for (const p of collectorPlacements(docsToManifests(ctx))) {
|
|
28
|
+
if (p.kind !== "DaemonSet") continue;
|
|
29
|
+
const ids = tracePipelineProcessors(p.config, "tail_sampling");
|
|
30
|
+
if (ids.length === 0) continue;
|
|
31
|
+
diagnostics.push({
|
|
32
|
+
checkId: "WK8601",
|
|
33
|
+
severity: "warning",
|
|
34
|
+
message:
|
|
35
|
+
`${describePlacement(p)} runs ${ids.join(", ")} (ConfigMap ${p.configMap}, ${p.key}) on every node. ` +
|
|
36
|
+
`Each agent sees only its own node's spans, so sampling decisions are made on partial traces. ` +
|
|
37
|
+
`Move tail sampling to a gateway (OtelCollectorGateway) and send traces to it through a loadbalancing exporter routed by traceID.`,
|
|
38
|
+
entity: p.name,
|
|
39
|
+
lexicon: "k8s",
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
return diagnostics;
|
|
43
|
+
},
|
|
44
|
+
};
|