@intentius/chant-lexicon-otel 0.124.30 → 0.124.32

package/dist/composites/catalog.js

@@ -0,0 +1,368 @@
+export const compositeCatalog = [
+    {
+        "name": "GenAiPipeline",
+        "lexicon": "otel",
+        "description": "The collector config of an OTLP collector for GenAI workloads, from the GenAI preset.",
+        "bundles": [
+            "BatchProcessor",
+            "DebugExporter",
+            "DeltaToCumulativeProcessor",
+            "FilterProcessor",
+            "ForwardConnector",
+            "HealthCheckExtension",
+            "MemoryLimiterProcessor",
+            "OtlpReceiver",
+            "Pipeline",
+            "RedactionProcessor",
+            "SignalToMetricsConnector",
+            "SpanMetricsConnector",
+            "SumConnector",
+            "TransformProcessor"
+        ],
+        "params": [
+            {
+                "name": "traceExporters",
+                "type": "Exporter[]",
+                "required": false,
+                "description": "Where traces go."
+            },
+            {
+                "name": "metricExporters",
+                "type": "Exporter[]",
+                "required": false,
+                "description": "Where the span and token metrics go, and with `clientMetrics` the SDK's metrics too."
+            },
+            {
+                "name": "logExporters",
+                "type": "Exporter[]",
+                "required": false,
+                "description": "Where logs (and events sent as logs) go."
+            },
+            {
+                "name": "logs",
+                "type": "boolean",
+                "required": false,
+                "description": "Give logs a pipeline, so event-based content is removed too."
+            },
+            {
+                "name": "sampling",
+                "type": "Processor[]",
+                "required": false,
+                "description": "Processors that thin exported traces, such as `tail_sampling`."
+            },
+            {
+                "name": "healthCheck",
+                "type": "boolean",
+                "required": false,
+                "description": "Serve `health_check` on 0.0.0.0:13133."
+            },
+            {
+                "name": "deltaToCumulative",
+                "type": "boolean | \"auto\"",
+                "required": false,
+                "description": "Put a `deltatocumulative/genai` processor in front of `batch` on `metrics/genai`, so the delta token sums of the `sum` connector, and with `clientMetrics."
+            },
+            {
+                "name": "keepContent",
+                "type": "boolean",
+                "required": false,
+                "description": "Keep content attributes and bodies."
+            },
+            {
+                "name": "contentAttributes",
+                "type": "string[]",
+                "required": false,
+                "description": "More attribute keys that hold content, deleted along with the convention ones."
+            },
+            {
+                "name": "maskValues",
+                "type": "string[]",
+                "required": false,
+                "description": "RE2 patterns masked in every attribute value (redaction `blocked_values`), whether content is kept or not."
+            },
+            {
+                "name": "hashFunction",
+                "type": "RedactionProcessorConfig[\"hash_function\"]",
+                "required": false,
+                "description": "Hash masked values with this function instead of writing `****`."
+            },
+            {
+                "name": "buckets",
+                "type": "Duration[]",
+                "required": false,
+                "description": "Duration histogram buckets."
+            },
+            {
+                "name": "metricsFlushInterval",
+                "type": "Duration",
+                "required": false,
+                "description": "How often span and token metrics are flushed."
+            },
+            {
+                "name": "namespace",
+                "type": "string",
+                "required": false,
+                "description": "Prefix of the emitted metric names."
+            },
+            {
+                "name": "dimensions",
+                "type": "SpanMetricsDimension[]",
+                "required": false,
+                "description": "Dimensions added to the span metrics beyond the GenAI ones."
+            },
+            {
+                "name": "providerDimensions",
+                "type": "boolean",
+                "required": false,
+                "description": "Add `gen_ai.provider.name` and `gen_ai.response.model` to `genai.calls` and `genai.duration`."
+            },
+            {
+                "name": "clientMetrics",
+                "type": "GenAiClientMetricsSource",
+                "required": false,
+                "description": "Also produce the conventions' client metrics, `gen_ai.client.operation.duration` and `gen_ai.client.token.usage`."
+            }
+        ]
+    },
+    {
+        "name": "NodeAgent",
+        "lexicon": "otel",
+        "description": "The collector config of a per-node Kubernetes agent.",
+        "bundles": [
+            "BatchProcessor",
+            "FileLogReceiver",
+            "HealthCheckExtension",
+            "HostMetricsReceiver",
+            "K8sAttributesProcessor",
+            "KubeletStatsReceiver",
+            "MemoryLimiterProcessor",
+            "OtlpReceiver",
+            "Pipeline",
+            "ResourceDetectionProcessor",
+            "ResourceProcessor"
+        ],
+        "params": [
+            {
+                "name": "exporters",
+                "type": "Exporter[]",
+                "required": true,
+                "description": "Where traces and logs go: usually the gateway (the k8s lexicon's `gatewayExporter()`), or an OTLP backend."
+            },
+            {
+                "name": "metricExporters",
+                "type": "Exporter[]",
+                "required": false,
+                "description": "Where metrics go, e.g. a `prometheus` exporter for a scrape on each node."
+            },
+            {
+                "name": "clusterName",
+                "type": "string",
+                "required": false,
+                "description": "Sets `k8s.cluster.name` on everything the agent sends."
+            },
+            {
+                "name": "nodeNameEnv",
+                "type": "string",
+                "required": false,
+                "description": "The environment variable holding this node's name, set from `spec.nodeName` (default `K8S_NODE_NAME`)."
+            },
+            {
+                "name": "hostMetrics",
+                "type": "boolean | { interval?: Duration; rootPath?: string }",
+                "required": false,
+                "description": "Node CPU, memory, load, filesystem and network from the host root at `/hostfs` (default: on, every 30s)."
+            },
+            {
+                "name": "containerLogs",
+                "type": "boolean | { selfContainer?: string }",
+                "required": false,
+                "description": "Container logs from `/var/log/pods` (default: on)."
+            },
+            {
+                "name": "kubeletStats",
+                "type": "boolean | { interval?: Duration }",
+                "required": false,
+                "description": "Pod and container resource metrics from this node's kubelet (default: off)."
+            },
+            {
+                "name": "memoryLimitMib",
+                "type": "number",
+                "required": false,
+                "description": "`memory_limiter`'s hard limit in MiB (default 400; the spike limit is a quarter of it)."
+            },
+            {
+                "name": "healthCheck",
+                "type": "boolean",
+                "required": false,
+                "description": "Serve `health_check` on 0.0.0.0:13133 (default: on)."
+            }
+        ]
+    },
+    {
+        "name": "OtlpCollector",
+        "lexicon": "otel",
+        "description": "A small OTLP collector config: OTLP in, `memory_limiter` and `batch`, the given exporters out, for each signal.",
+        "bundles": [
+            "BatchProcessor",
+            "DebugExporter",
+            "HealthCheckExtension",
+            "MemoryLimiterProcessor",
+            "OtlpReceiver",
+            "Pipeline"
+        ],
+        "params": [
+            {
+                "name": "exporters",
+                "type": "OTelComponent<\"exporter\", string, any>[]",
+                "required": false,
+                "description": "Where telemetry goes."
+            },
+            {
+                "name": "signals",
+                "type": "Signal[]",
+                "required": false,
+                "description": "Which signals get a pipeline."
+            },
+            {
+                "name": "healthCheck",
+                "type": "boolean",
+                "required": false,
+                "description": "Serve `health_check` on 0.0.0.0:13133."
+            }
+        ]
+    },
+    {
+        "name": "RedMetrics",
+        "lexicon": "otel",
+        "description": "RED metrics from traces: `spanmetrics` and `servicegraph` connectors feeding a `prometheus` exporter.",
+        "bundles": [
+            "BatchProcessor",
+            "MemoryLimiterProcessor",
+            "OtlpReceiver",
+            "Pipeline",
+            "PrometheusExporter",
+            "ServiceGraphConnector",
+            "SpanMetricsConnector"
+        ],
+        "params": [
+            {
+                "name": "receivers",
+                "type": "TraceSource[]",
+                "required": false,
+                "description": "Where spans come from: receivers, or a connector from another traces pipeline."
+            },
+            {
+                "name": "traceExporters",
+                "type": "Exporter[]",
+                "required": false,
+                "description": "Where traces go besides the connectors, e.g. a tracing backend."
+            },
+            {
+                "name": "exporter",
+                "type": "OTelComponent<\"exporter\", \"prometheus\", PrometheusExporterConfig>",
+                "required": false,
+                "description": "The `prometheus` exporter the metrics are served on."
+            },
+            {
+                "name": "spanMetrics",
+                "type": "SpanMetricsConnectorConfig",
+                "required": false,
+                "description": "The `spanmetrics` connector's config: namespace, dimensions, histogram unit and buckets (default: the connector's defaults)."
+            },
+            {
+                "name": "serviceGraph",
+                "type": "boolean | ServiceGraphConnectorConfig",
+                "required": false,
+                "description": "Service-to-service edge metrics from a `servicegraph` connector (default: on)."
+            },
+            {
+                "name": "memoryLimitMib",
+                "type": "number",
+                "required": false,
+                "description": "`memory_limiter`'s hard limit in MiB (default: 80% of the container's memory, with a 20% spike limit)."
+            },
+            {
+                "name": "name",
+                "type": "string",
+                "required": false,
+                "description": "The instance name of the connectors, the exporter and the two pipelines (default `red`)."
+            }
+        ]
+    },
+    {
+        "name": "TailSamplingTier",
+        "lexicon": "otel",
+        "description": "The collector config of a tail sampling tier: whole traces in, the ones a policy keeps out.",
+        "bundles": [
+            "BatchProcessor",
+            "DebugExporter",
+            "HealthCheckExtension",
+            "MemoryLimiterProcessor",
+            "OtlpReceiver",
+            "Pipeline",
+            "TailSamplingProcessor"
+        ],
+        "params": [
+            {
+                "name": "exporters",
+                "type": "Exporter[]",
+                "required": false,
+                "description": "Where the kept traces go."
+            },
+            {
+                "name": "receivers",
+                "type": "TraceSource[]",
+                "required": false,
+                "description": "Where spans come from."
+            },
+            {
+                "name": "decisionWait",
+                "type": "Duration",
+                "required": false,
+                "description": "How long after a trace's first span the decision is made (default `10s`)."
+            },
+            {
+                "name": "errors",
+                "type": "boolean",
+                "required": false,
+                "description": "Keep every trace with a span ended in error (default: on)."
+            },
+            {
+                "name": "slowerThanMs",
+                "type": "number | false",
+                "required": false,
+                "description": "Keep every trace that took at least this many milliseconds (default 1000; `false` for none)."
+            },
+            {
+                "name": "percentage",
+                "type": "number | false",
+                "required": false,
+                "description": "Keep this percentage of all other traces, by trace id hash (default 10; `false` for none)."
+            },
+            {
+                "name": "policies",
+                "type": "TailSamplingPolicy[]",
+                "required": false,
+                "description": "More policies, evaluated after the ones above."
+            },
+            {
+                "name": "numTraces",
+                "type": "number",
+                "required": false,
+                "description": "How many traces to hold in memory at once (default: the collector's 50000)."
+            },
+            {
+                "name": "memoryLimitMib",
+                "type": "number",
+                "required": false,
+                "description": "`memory_limiter`'s hard limit in MiB (default: 80% of the container's memory, with a 20% spike limit)."
+            },
+            {
+                "name": "healthCheck",
+                "type": "boolean",
+                "required": false,
+                "description": "Serve `health_check` on 0.0.0.0:13133 (default: on)."
+            }
+        ]
+    }
+];

package/dist/composites/genai-pipeline.js

@@ -0,0 +1,28 @@
+/**
+ * `GenAiPipeline`: `genAiPipeline()` as a composite.
+ *
+ * The same collector config `genAiPipeline(options)` returns as a list (an
+ * `otlp` receiver, content removal on traces and logs, a `traces/genai`
+ * branch that turns GenAI spans into metrics before any sampling, and a
+ * `metrics/genai` pipeline that exports them), with each entity under a
+ * member name, so a project exports one declaration and reads the parts by
+ * name. The options are `genAiPipeline()`'s; every one has its default.
+ *
+ * The prometheus lexicon's `GenAiRules` reads the metric names from
+ * `genAiMetrics(options)` with the same options.
+ */
+import { Composite } from "@intentius/chant/composite";
+import { genAiPipelineParts } from "../genai.js";
+/**
+ * The collector config of an OTLP collector for GenAI workloads, from the
+ * GenAI preset.
+ *
+ * @example
+ * ```ts
+ * import { GenAiPipeline, PrometheusExporter } from "@intentius/chant-lexicon-otel";
+ *
+ * const scrape = new PrometheusExporter({ endpoint: "0.0.0.0:8889" });
+ * export const genai = GenAiPipeline({ metricExporters: [scrape], clientMetrics: "derive" });
+ * ```
+ */
+export const GenAiPipeline = Composite((props) => genAiPipelineParts(props ?? {}), "GenAiPipeline");

package/dist/composites/index.js

@@ -0,0 +1,8 @@
+/**
+ * otel lexicon composites: declarations that expand to a collector config's components and pipelines.
+ */
+export { NodeAgent, nodeAgentPropsProblem } from "./node-agent.js";
+export { RedMetrics, redMetricsNames, redMetricsPropsProblem } from "./red-metrics.js";
+export { GenAiPipeline } from "./genai-pipeline.js";
+export { OtlpCollector, otlpCollectorPropsProblem } from "./otlp-collector.js";
+export { TailSamplingTier, tailSamplingTierPropsProblem, tailSamplingLoadBalancer } from "./tail-sampling-tier.js";

package/dist/composites/node-agent.js

@@ -0,0 +1,156 @@
+/**
+ * `NodeAgent`: the collector config a per-node agent runs on Kubernetes.
+ *
+ * One collector per node (a DaemonSet) takes OTLP from the pods on its node,
+ * reads the node's own telemetry, puts Kubernetes metadata on all of it, and
+ * hands it on, usually to a gateway tier. This composite declares that
+ * config:
+ *
+ * - receivers: `otlp` (gRPC on 4317, HTTP on 4318), and by default
+ *   `hostmetrics` (CPU, memory, load, filesystem, network of the node, read
+ *   under `/hostfs`) and `filelog` (container logs under `/var/log/pods`,
+ *   parsed by the `container` operator). `kubeletstats` (pod and container
+ *   resource metrics from the local kubelet) is opt-in.
+ * - processors, in order: `memory_limiter`; `k8sattributes`, filtered to
+ *   pods on this node and matching by pod IP, pod UID or the connection;
+ *   `resourcedetection` (`env`, `system`); a `resource` processor setting
+ *   `k8s.cluster.name` when `clusterName` is given; `batch`.
+ * - a `traces`, a `metrics` and a `logs` pipeline, each through those
+ *   processors to the given exporters.
+ * - a `health_check` extension on 0.0.0.0:13133.
+ *
+ * Tail sampling is left out on purpose: an agent sees only its node's
+ * spans of a trace, so a sampling decision there is made on partial traces.
+ * Sample on the gateway.
+ *
+ * What the pod has to provide: the node name in an environment variable
+ * (`K8S_NODE_NAME` by default, from `spec.nodeName`), the host root mounted
+ * read-only at `/hostfs` for `hostmetrics`, `/var/log/pods` mounted
+ * read-only for `filelog`, and a service account that may get, list and
+ * watch pods, namespaces and nodes (and replicasets, for
+ * `k8s.deployment.name`). The k8s lexicon's `OtelCollector` takes this
+ * composite's entities as its `config` and derives all of that from them:
+ * the variable, the read-only host mounts, group 0 for the root-owned log
+ * files, the RBAC and the ports.
+ */
+import { Composite } from "@intentius/chant/composite";
+import { OtlpReceiver, HostMetricsReceiver, FileLogReceiver } from "../components/receivers.js";
+import { KubeletStatsReceiver } from "../components/k8s-receivers.js";
+import { BatchProcessor, K8sAttributesProcessor, MemoryLimiterProcessor, ResourceDetectionProcessor, ResourceProcessor, } from "../components/processors.js";
+import { HealthCheckExtension } from "../components/extensions.js";
+import { Pipeline } from "../pipeline.js";
+const HOST_SCRAPERS = { cpu: {}, memory: {}, load: {}, filesystem: {}, network: {} };
+/** Pod metadata for every signal. `k8s.pod.uid` is what the `container` log parser sets on log records. */
+const POD_METADATA = {
+    metadata: ["k8s.namespace.name", "k8s.pod.name", "k8s.pod.uid", "k8s.deployment.name", "k8s.node.name"],
+};
+const POD_ASSOCIATION = [
+    { sources: [{ from: "resource_attribute", name: "k8s.pod.ip" }] },
+    { sources: [{ from: "resource_attribute", name: "k8s.pod.uid" }] },
+    { sources: [{ from: "connection" }] },
+];
+function option(value, onByDefault) {
+    if (value === undefined)
+        return onByDefault ? {} : undefined;
+    if (value === false)
+        return undefined;
+    return value === true ? {} : value;
+}
+/** Why a set of `NodeAgent` props can't build, or undefined when they can. */
+export function nodeAgentPropsProblem(props) {
+    if (!Array.isArray(props.exporters) || props.exporters.length === 0)
+        return "exporters must name at least one exporter";
+    if (props.metricExporters !== undefined && props.metricExporters.length === 0)
+        return "metricExporters must name at least one exporter when set";
+    if (props.memoryLimitMib !== undefined && !(Number.isInteger(props.memoryLimitMib) && props.memoryLimitMib > 0)) {
+        return "memoryLimitMib must be a positive whole number";
+    }
+    if (props.nodeNameEnv !== undefined && !/^[A-Za-z_][A-Za-z0-9_]*$/.test(props.nodeNameEnv)) {
+        return `nodeNameEnv "${props.nodeNameEnv}" is not an environment variable name`;
+    }
+    return undefined;
+}
+/**
+ * The collector config of a per-node Kubernetes agent: OTLP from the node's
+ * pods, host metrics and container logs from the node itself, Kubernetes
+ * metadata on everything, sent on to the given exporters.
+ *
+ * @example
+ * ```ts
+ * import { NodeAgent, OtlpExporter } from "@intentius/chant-lexicon-otel";
+ *
+ * const gateway = new OtlpExporter({ name: "gateway", endpoint: "otel-gateway.observability.svc:4317", tls: { insecure: true } });
+ * export const agent = NodeAgent({ exporters: [gateway], clusterName: "prod-eu-1" });
+ * ```
+ */
+export const NodeAgent = Composite((props) => {
+    const problem = nodeAgentPropsProblem(props);
+    if (problem)
+        throw new Error(`NodeAgent: ${problem}`);
+    const nodeNameEnv = props.nodeNameEnv ?? "K8S_NODE_NAME";
+    const limit = props.memoryLimitMib ?? 400;
+    const host = option(props.hostMetrics, true);
+    const logFiles = option(props.containerLogs, true);
+    const kubelet = option(props.kubeletStats, false);
+    const otlp = new OtlpReceiver({
+        protocols: { grpc: { endpoint: "0.0.0.0:4317" }, http: { endpoint: "0.0.0.0:4318" } },
+    });
+    const hostMetrics = host
+        ? new HostMetricsReceiver({ collection_interval: host.interval ?? "30s", root_path: host.rootPath ?? "/hostfs", scrapers: HOST_SCRAPERS })
+        : undefined;
+    const containerLogs = logFiles
+        ? new FileLogReceiver({
+            include: ["/var/log/pods/*/*/*.log"],
+            exclude: [`/var/log/pods/*/${logFiles.selfContainer ?? "otel-collector"}/*.log`],
+            start_at: "end",
+            include_file_path: true,
+            operators: [{ type: "container", id: "container-parser" }],
+        })
+        : undefined;
+    const kubeletStats = kubelet
+        ? new KubeletStatsReceiver({
+            collection_interval: kubelet.interval ?? "30s",
+            auth_type: "serviceAccount",
+            endpoint: `https://\${env:${nodeNameEnv}}:10250`,
+            insecure_skip_verify: true,
+            node: `\${env:${nodeNameEnv}}`,
+        })
+        : undefined;
+    const memoryLimiter = new MemoryLimiterProcessor({ check_interval: "1s", limit_mib: limit, spike_limit_mib: Math.ceil(limit / 4) });
+    const k8sAttributes = new K8sAttributesProcessor({
+        auth_type: "serviceAccount",
+        filter: { node_from_env_var: nodeNameEnv },
+        extract: POD_METADATA,
+        pod_association: POD_ASSOCIATION,
+    });
+    const resourceDetection = new ResourceDetectionProcessor({ detectors: ["env", "system"], timeout: "5s", override: false });
+    const cluster = props.clusterName
+        ? new ResourceProcessor({ attributes: [{ key: "k8s.cluster.name", value: props.clusterName, action: "upsert" }] })
+        : undefined;
+    const batch = new BatchProcessor({ timeout: "10s" });
+    const processors = [memoryLimiter, k8sAttributes, resourceDetection, ...(cluster ? [cluster] : []), batch];
+    const exporters = props.exporters;
+    const metricExporters = props.metricExporters ?? exporters;
+    const traces = new Pipeline({ signal: "traces", receivers: [otlp], processors, exporters });
+    const metrics = new Pipeline({
+        signal: "metrics",
+        receivers: [otlp, ...(hostMetrics ? [hostMetrics] : []), ...(kubeletStats ? [kubeletStats] : [])],
+        processors,
+        exporters: metricExporters,
+    });
+    const logs = new Pipeline({ signal: "logs", receivers: [...(containerLogs ? [containerLogs] : []), otlp], processors, exporters });
+    const health = props.healthCheck === false ? undefined : new HealthCheckExtension({ endpoint: "0.0.0.0:13133" });
+    // A composite member must be a declarable, so the components that are off are left out rather than undefined.
+    const members = { otlp, memoryLimiter, k8sAttributes, resourceDetection, batch, traces, metrics, logs };
+    if (hostMetrics)
+        members.hostMetrics = hostMetrics;
+    if (containerLogs)
+        members.containerLogs = containerLogs;
+    if (kubeletStats)
+        members.kubeletStats = kubeletStats;
+    if (cluster)
+        members.cluster = cluster;
+    if (health)
+        members.health = health;
+    return members;
+}, "NodeAgent");

package/dist/composites/otlp-collector.js

@@ -0,0 +1,57 @@
+/**
+ * `OtlpCollector`: `otlpCollector()` as a composite.
+ *
+ * The small OTLP collector the docker, k8s and fly platform composites start
+ * from: an `otlp` receiver on 4317 (gRPC) and 4318 (HTTP), `memory_limiter`
+ * then `batch`, the given exporters (default: one `debug` exporter), one
+ * pipeline per signal, and a `health_check` extension on 0.0.0.0:13133.
+ * Each part is a member, so a project exports one declaration and hands
+ * the parts on by name.
+ */
+import { Composite } from "@intentius/chant/composite";
+import { otlpCollectorParts } from "../platform.js";
+/** Why a set of `OtlpCollector` props can't build, or undefined when they can. */
+export function otlpCollectorPropsProblem(props = {}) {
+    if (props.exporters !== undefined && props.exporters.length === 0)
+        return "exporters must name at least one exporter when set";
+    if (props.signals !== undefined) {
+        if (props.signals.length === 0)
+            return "signals must name at least one signal when set";
+        const bad = props.signals.find((s) => s !== "traces" && s !== "metrics" && s !== "logs");
+        if (bad !== undefined)
+            return `signals: "${String(bad)}" is not traces, metrics or logs`;
+    }
+    return undefined;
+}
+/**
+ * A small OTLP collector config: OTLP in, `memory_limiter` and `batch`, the
+ * given exporters out, for each signal.
+ *
+ * @example
+ * ```ts
+ * import { OtlpCollector, OtlpExporter } from "@intentius/chant-lexicon-otel";
+ *
+ * const backend = new OtlpExporter({ name: "backend", endpoint: "collector.example.com:4317" });
+ * export const collector = OtlpCollector({ exporters: [backend], signals: ["traces", "logs"] });
+ * ```
+ */
+export const OtlpCollector = Composite((input) => {
+    const props = input ?? {};
+    const problem = otlpCollectorPropsProblem(props);
+    if (problem)
+        throw new Error(`OtlpCollector: ${problem}`);
+    const p = otlpCollectorParts(props);
+    // A composite member must be a declarable, so the parts that are off are left out rather than undefined.
+    const members = { otlp: p.otlp, memoryLimiter: p.memoryLimiter, batch: p.batch };
+    if (props.exporters === undefined)
+        members.debug = p.exporters[0];
+    if (p.health)
+        members.health = p.health;
+    if (p.pipelines.traces)
+        members.traces = p.pipelines.traces;
+    if (p.pipelines.metrics)
+        members.metrics = p.pipelines.metrics;
+    if (p.pipelines.logs)
+        members.logs = p.pipelines.logs;
+    return members;
+}, "OtlpCollector");