@intentius/chant-lexicon-otel 0.95.1 → 0.96.0

package/src/skills/chant-otel.md

@@ -6,7 +6,7 @@ user-invocable: true
 
 # OpenTelemetry Collector config with chant
 
-The otel lexicon (`@intentius/chant-lexicon-otel`) types collector config. Each receiver, processor, exporter and extension is an entity, pipelines reference them, and `chant build` writes one collector config file.
+The otel lexicon (`@intentius/chant-lexicon-otel`) types collector config. Each receiver, processor, exporter, connector and extension is an entity, pipelines reference them, and `chant build` writes one collector config file.
 
 ## Project setup
 
@@ -40,13 +40,36 @@ Emits `receivers.otlp`, `processors.memory_limiter` and `processors.batch`, `exp
 
 | Kind | Types |
 |---|---|
-| receivers | otlp, prometheus, hostmetrics, filelog |
-| processors | batch, memory_limiter, resource, attributes, k8sattributes, resourcedetection |
-| exporters | otlp, otlphttp, debug, prometheus, googlecloud |
+| receivers | otlp, prometheus, hostmetrics, filelog, k8s_cluster, kubeletstats |
+| processors | batch, memory_limiter, resource, attributes, k8sattributes, resourcedetection, filter, transform, redaction, tail_sampling, probabilistic_sampler |
+| exporters | otlp, otlphttp, debug, prometheus, googlecloud, loadbalancing |
+| connectors | spanmetrics, servicegraph, routing, forward, count, sum |
 | extensions | health_check, pprof, zpages |
 
 Anything else goes through `defineComponent`; see the `chant-otel-custom-components` skill.
 
+## Connectors
+
+A connector joins two pipelines. List the same entity in `exporters` of the pipeline that feeds it and in `receivers` of the pipeline it feeds:
+
+```ts
+export const spanmetrics = new SpanMetricsConnector({ dimensions: [{ name: "http.route" }] });
+export const traces = new Pipeline({ signal: "traces", receivers: [otlp], exporters: [backend, spanmetrics] });
+export const red = new Pipeline({ signal: "metrics", name: "red", receivers: [spanmetrics], exporters: [prom] });
+```
+
+`spanmetrics` and `servicegraph` turn traces into metrics, `count` and `sum` turn any signal into metrics (`sum` adds up a numeric attribute), `routing` and `forward` keep the signal. A connector on one side only fails OTEL101; a pipeline whose signal the connector can't pair fails OTEL112.
+
+## GenAI workloads
+
+For services that emit GenAI spans, start from the preset instead of writing the processors by hand:
+
+```ts
+export const collector = genAiPipeline({ traceExporters: [tempo], metricExporters: [prom] });
+```
+
+It deletes prompt, completion, system-instruction and tool-call content from spans, span events and logs, and derives call, error, duration and token metrics (`genai_calls_total`, `genai_duration_seconds`, `genai_tokens_input_total`, `genai_tokens_output_total`) from every GenAI span before sampling. Keep content only when asked, with `keepContent: true`. `genAiComponents()` returns the pieces for hand-built pipelines.
+
 ## Rules
 
 - Reference components by entity where you can. A string id (`"otlp/backend"`) is allowed for a component declared elsewhere, and OTEL101 fails the build if nothing declares it.
@@ -57,4 +80,4 @@ Anything else goes through `defineComponent`; see the `chant-otel-custom-compone
 
 ## Reading the result
 
-`collectorTopologyOf(entities)` returns the pipelines, each component's endpoints and schema pin, and for each exporter the signals it carries, as plain data.
+`collectorTopologyOf(entities)` returns the pipelines, each component's endpoints and schema pin, for each exporter the signals it carries, the connector `edges` between pipelines, and under `semconv` the semantic-conventions version (`GENAI_SEMCONV_PIN`) the config's `gen_ai.` keys follow, as plain data.

package/src/topology.ts

@@ -3,16 +3,21 @@
  *
  * `collectorTopology()` reads a collector config (declared, or parsed from a
  * YAML file) and returns its pipelines, its components with their endpoints
- * and schema pins, and for each exporter the pipelines and signals it
- * carries. It is the surface a reader such as `chant workspace graph` uses to
- * say where a member's telemetry is sent, and what a declared telemetry
- * endpoint link can point at.
+ * and schema pins, for each exporter the pipelines and signals it carries,
+ * the edges connectors make from one pipeline to another, and the
+ * semantic-convention versions its attribute keys follow. It is the
+ * surface a reader such as `chant workspace graph` uses to say where a
+ * member's telemetry is sent, and what a declared telemetry endpoint link can
+ * point at.
  */
 
 import type { Declarable } from "@intentius/chant/declarable";
 import { buildCollectorConfig } from "./collector";
 import { definitionOf, type SchemaPin } from "./define";
 import { parseComponentId, pipelineSignal, SECTION_OF, type CollectorConfig, type ComponentKind } from "./model";
+import { semconvUsage, type SemconvUsage } from "./semconv";
+
+export type { SemconvUsage } from "./semconv";
 
 export interface TopologyPipeline {
   /** The id under `service.pipelines`, e.g. `traces` or `traces/backend`. */
@@ -36,10 +41,24 @@ export interface TopologyComponent {
   schema?: SchemaPin;
   /** Addresses it listens on or sends to, as the config states them. Empty when the config names none. */
   endpoints: string[];
-  /** The pipelines that use it. For an extension, empty. */
+  /** The pipelines that use it. For a connector, those on either side; for an extension, empty. */
   pipelines: string[];
 }
 
+/** One hop from a pipeline to another through a connector. */
+export interface TopologyEdge {
+  /** The connector's id. */
+  connector: string;
+  /** The pipeline that lists the connector as an exporter. */
+  from: string;
+  /** That pipeline's signal. */
+  fromSignal: string;
+  /** The pipeline that lists the connector as a receiver. */
+  to: string;
+  /** That pipeline's signal. */
+  toSignal: string;
+}
+
 export interface TopologyExporter {
   id: string;
   type: string;
@@ -54,6 +73,18 @@ export interface CollectorTopology {
   components: TopologyComponent[];
   /** The exporters again, grouped for the question "where does this telemetry go". */
   exporters: TopologyExporter[];
+  /**
+   * Pipeline-to-pipeline edges through connectors, one per (from, to) pair the
+   * connector supports. For a connector whose definition this process lacks,
+   * every pair.
+   */
+  edges: TopologyEdge[];
+  /**
+   * The semantic-convention vocabularies the config's attribute keys come
+   * from, each with the pin this package follows for it (`GENAI_SEMCONV_PIN`
+   * for `gen_ai`) and the components that use it. Empty when none is used.
+   */
+  semconv: SemconvUsage[];
 }
 
 function endpointsOf(kind: ComponentKind, type: string, config: Record<string, unknown> | null | undefined): string[] {
@@ -86,9 +117,12 @@ export function collectorTopology(config: CollectorConfig): CollectorTopology {
       const parsed = parseComponentId(id);
       const type = parsed?.type ?? id;
       const def = definitionOf(kind, type);
-      const field = section as "receivers" | "processors" | "exporters" | "extensions";
       const inPipelines =
-        field === "extensions" ? [] : pipelines.filter((p) => p[field].includes(id)).map((p) => p.id);
+        section === "extensions"
+          ? []
+          : section === "connectors"
+            ? pipelines.filter((p) => p.receivers.includes(id) || p.exporters.includes(id)).map((p) => p.id)
+            : pipelines.filter((p) => p[section].includes(id)).map((p) => p.id);
       components.push({
         id,
         kind,
@@ -113,7 +147,19 @@ export function collectorTopology(config: CollectorConfig): CollectorTopology {
       return { id: c.id, type: c.type, endpoints: c.endpoints, pipelines: c.pipelines, signals };
     });
 
-  return { pipelines, components, exporters };
+  const edges: TopologyEdge[] = [];
+  for (const c of components) {
+    if (c.kind !== "connector") continue;
+    const pairs = definitionOf("connector", c.type)?.connects;
+    for (const from of pipelines.filter((p) => p.exporters.includes(c.id))) {
+      for (const to of pipelines.filter((p) => p.receivers.includes(c.id))) {
+        if (pairs && pairs.length > 0 && !pairs.some((pair) => pair.from === from.signal && pair.to === to.signal)) continue;
+        edges.push({ connector: c.id, from: from.id, fromSignal: from.signal, to: to.id, toSignal: to.signal });
+      }
+    }
+  }
+
+  return { pipelines, components, exporters, edges, semconv: semconvUsage(config) };
 }
 
 /** The topology of declared entities, i.e. of the config the serializer would emit for them. */

package/src/validate-config.ts

@@ -9,10 +9,12 @@
  */
 
 import type { Declarable } from "@intentius/chant/declarable";
-import { definitionFor, isOTelComponent, isUsablePin, runValidator } from "./define";
+import { definitionFor, definitionOf, isOTelComponent, isUsablePin, runValidator } from "./define";
 import { componentConfig } from "./collector";
-import { isComponentId, pipelineSignal, SIGNALS, type CollectorConfig } from "./model";
+import { isComponentId, parseComponentId, pipelineSignal, SIGNALS, type CollectorConfig, type ConnectorSignalPair } from "./model";
 import { isPipelineEntity } from "./pipeline";
+// OTEL112 reads the built-in connectors' signal pairs from the registry.
+import "./components/connectors";
 
 export type CollectorIssueCode =
   | "OTEL101"
@@ -23,7 +25,8 @@ export type CollectorIssueCode =
   | "OTEL106"
   | "OTEL107"
   | "OTEL108"
-  | "OTEL109";
+  | "OTEL109"
+  | "OTEL112";
 
 export interface CollectorIssue {
   code: CollectorIssueCode;
@@ -41,7 +44,20 @@ function ids(section: Record<string, unknown> | undefined): Set<string> {
   return new Set(Object.keys(section ?? {}));
 }
 
-/** Check a collector config's references and pipeline shape (OTEL101-OTEL106). */
+/** Where a connector appears: the pipelines it is an exporter in, and those it is a receiver in. */
+interface ConnectorUse {
+  asExporter: string[];
+  asReceiver: string[];
+}
+
+function describePairs(pairs: ReadonlyArray<ConnectorSignalPair>): string {
+  return pairs.map((p) => `${p.from} to ${p.to}`).join(", ");
+}
+
+/**
+ * Check a collector config's references and pipeline shape (OTEL101-OTEL106),
+ * and each connector's signals against its definition (OTEL112).
+ */
 export function validateCollectorConfig(config: CollectorConfig): CollectorIssue[] {
   const issues: CollectorIssue[] = [];
   const receivers = ids(config.receivers);
@@ -52,6 +68,7 @@ export function validateCollectorConfig(config: CollectorConfig): CollectorIssue
   const pipelines = config.service?.pipelines ?? {};
 
   const used = { receivers: new Set<string>(), processors: new Set<string>(), exporters: new Set<string>() };
+  const connectorUse = new Map<string, ConnectorUse>();
 
   for (const [pipelineId, pipeline] of Object.entries(pipelines)) {
     const signal = pipelineSignal(pipelineId);
@@ -84,6 +101,11 @@ export function validateCollectorConfig(config: CollectorConfig): CollectorIssue
           continue;
         }
         const viaConnector = field !== "processors" && connectors.has(id);
+        if (viaConnector) {
+          const use = connectorUse.get(id) ?? { asExporter: [], asReceiver: [] };
+          (field === "exporters" ? use.asExporter : use.asReceiver).push(pipelineId);
+          connectorUse.set(id, use);
+        }
         if (!declared.has(id) && !viaConnector) {
           issues.push({
             code: "OTEL101",
@@ -124,10 +146,35 @@ export function validateCollectorConfig(config: CollectorConfig): CollectorIssue
     }
   }
 
+  // A connector joins two pipelines, so the collector refuses one that is
+  // listed on only one side.
+  for (const [id, use] of connectorUse) {
+    if (use.asReceiver.length === 0) {
+      issues.push({
+        code: "OTEL101",
+        severity: "error",
+        pipeline: use.asExporter[0],
+        component: id,
+        message: `connector "${id}" is an exporter in pipeline "${use.asExporter[0]}" but no pipeline lists it as a receiver; a connector must appear on both sides, and the collector refuses to start`,
+      });
+    } else if (use.asExporter.length === 0) {
+      issues.push({
+        code: "OTEL101",
+        severity: "error",
+        pipeline: use.asReceiver[0],
+        component: id,
+        message: `connector "${id}" is a receiver in pipeline "${use.asReceiver[0]}" but no pipeline lists it as an exporter; a connector must appear on both sides, and the collector refuses to start`,
+      });
+    } else {
+      issues.push(...connectorSignalIssues(id, use));
+    }
+  }
+
   const unused: Array<[Set<string>, Set<string>, string]> = [
     [receivers, used.receivers, "receiver"],
     [processors, used.processors, "processor"],
     [exporters, used.exporters, "exporter"],
+    [connectors, new Set(connectorUse.keys()), "connector"],
   ];
   for (const [declared, usedIds, noun] of unused) {
     for (const id of declared) {
@@ -167,6 +214,47 @@ export function validateCollectorConfig(config: CollectorConfig): CollectorIssue
   return issues;
 }
 
+/**
+ * OTEL112: every pipeline a connector joins must pair with a pipeline on the
+ * other side through a signal pair the connector supports. This is the
+ * collector's own rule: `spanmetrics` fed by a traces pipeline needs a metrics
+ * pipeline to receive from it, and cannot be the receiver of a traces
+ * pipeline. A connector whose definition this process doesn't have, or whose
+ * definition lists no pairs, is not checked.
+ */
+function connectorSignalIssues(id: string, use: ConnectorUse): CollectorIssue[] {
+  const type = parseComponentId(id)?.type ?? id;
+  const pairs = definitionOf("connector", type)?.connects;
+  if (!pairs || pairs.length === 0) return [];
+  const supported = (from: string, to: string) => pairs.some((p) => p.from === from && p.to === to);
+  const inSignals = use.asExporter.map(pipelineSignal);
+  const outSignals = use.asReceiver.map(pipelineSignal);
+  const issues: CollectorIssue[] = [];
+  for (const pipeline of use.asExporter) {
+    const from = pipelineSignal(pipeline);
+    if (outSignals.some((to) => supported(from, to))) continue;
+    issues.push({
+      code: "OTEL112",
+      severity: "error",
+      pipeline,
+      component: id,
+      message: `connector "${id}" is an exporter in ${from} pipeline "${pipeline}", but no pipeline it feeds carries a signal ${type} makes from ${from} (it supports ${describePairs(pairs)}); the collector refuses to start`,
+    });
+  }
+  for (const pipeline of use.asReceiver) {
+    const to = pipelineSignal(pipeline);
+    if (inSignals.some((from) => supported(from, to))) continue;
+    issues.push({
+      code: "OTEL112",
+      severity: "error",
+      pipeline,
+      component: id,
+      message: `connector "${id}" is a receiver in ${to} pipeline "${pipeline}", but no pipeline feeding it carries a signal ${type} turns into ${to} (it supports ${describePairs(pairs)}); the collector refuses to start`,
+    });
+  }
+  return issues;
+}
+
 /**
  * Check declared entities for what only the declaration knows (OTEL107-OTEL109):
  * each component's own config rules, duplicate ids, and custom components'

package/src/validate.ts

@@ -18,17 +18,25 @@ export const REQUIRED_NAMES = [
   "PrometheusReceiver",
   "HostMetricsReceiver",
   "FileLogReceiver",
+  "K8sClusterReceiver",
+  "KubeletStatsReceiver",
   "BatchProcessor",
   "MemoryLimiterProcessor",
   "ResourceProcessor",
   "AttributesProcessor",
   "K8sAttributesProcessor",
   "ResourceDetectionProcessor",
+  "FilterProcessor",
+  "TransformProcessor",
+  "RedactionProcessor",
+  "TailSamplingProcessor",
+  "ProbabilisticSamplerProcessor",
   "OtlpExporter",
   "OtlpHttpExporter",
   "DebugExporter",
   "PrometheusExporter",
   "GoogleCloudExporter",
+  "LoadBalancingExporter",
   "HealthCheckExtension",
   "PprofExtension",
   "ZPagesExtension",