@intentius/chant-lexicon-otel 0.99.0 → 0.101.0

package/src/validate-config.ts

@@ -13,6 +13,9 @@ import { definitionFor, definitionOf, isOTelComponent, isUsablePin, runValidator
 import { componentConfig } from "./collector";
 import { isComponentId, parseComponentId, pipelineSignal, SIGNALS, type CollectorConfig, type ConnectorSignalPair } from "./model";
 import { isPipelineEntity } from "./pipeline";
+import { signalToMetricsEntries, type SignalToMetricsConnectorConfig } from "./components/connectors";
+import { genAiCardinalityRisk } from "./genai";
+import { collectorTopology, type TopologyEdge } from "./topology";
 // OTEL112 reads the built-in connectors' signal pairs from the registry.
 import "./components/connectors";
 
@@ -26,7 +29,12 @@ export type CollectorIssueCode =
   | "OTEL107"
   | "OTEL108"
   | "OTEL109"
-  | "OTEL112";
+  | "OTEL112"
+  | "OTEL113"
+  | "OTEL114"
+  | "OTEL115"
+  | "OTEL116"
+  | "OTEL117";
 
 export interface CollectorIssue {
   code: CollectorIssueCode;
@@ -56,7 +64,11 @@ function describePairs(pairs: ReadonlyArray<ConnectorSignalPair>): string {
 
 /**
  * Check a collector config's references and pipeline shape (OTEL101-OTEL106),
- * and each connector's signals against its definition (OTEL112).
+ * each connector's signals against its definition (OTEL112), cycles through
+ * connectors (OTEL113), connector ids shared with a receiver or exporter
+ * (OTEL114), `routing` connector targets (OTEL115), the attributes
+ * connectors split metrics by (OTEL116), and started components that listen
+ * on the same address (OTEL117).
  */
 export function validateCollectorConfig(config: CollectorConfig): CollectorIssue[] {
   const issues: CollectorIssue[] = [];
@@ -211,6 +223,407 @@ export function validateCollectorConfig(config: CollectorConfig): CollectorIssue
     }
   }
 
+  issues.push(
+    ...connectorCycleIssues(config),
+    ...connectorIdIssues(config),
+    ...routingTargetIssues(config),
+    ...metricAttributeIssues(config),
+    ...listenerIssues(config),
+  );
+
+  return issues;
+}
+
+/**
+ * OTEL113: the pipelines and the connectors between them must form a graph
+ * with no cycle, or the collector refuses to start. Reads the edges
+ * `collectorTopology()` returns, so an edge counts only for a signal pair the
+ * connector supports, as in the collector's own graph. Reports each strongly
+ * connected group of pipelines once, with one cycle through it.
+ */
+function connectorCycleIssues(config: CollectorConfig): CollectorIssue[] {
+  const { pipelines, edges } = collectorTopology(config);
+  const out = new Map<string, TopologyEdge[]>();
+  for (const e of edges) out.set(e.from, [...(out.get(e.from) ?? []), e]);
+
+  // Tarjan's strongly connected components, in pipeline order.
+  const order = pipelines.map((p) => p.id);
+  const index = new Map<string, number>();
+  const low = new Map<string, number>();
+  const stack: string[] = [];
+  const onStack = new Set<string>();
+  const groups: string[][] = [];
+  const visit = (v: string) => {
+    index.set(v, index.size);
+    low.set(v, index.get(v)!);
+    stack.push(v);
+    onStack.add(v);
+    for (const { to } of out.get(v) ?? []) {
+      if (!index.has(to)) {
+        visit(to);
+        low.set(v, Math.min(low.get(v)!, low.get(to)!));
+      } else if (onStack.has(to)) {
+        low.set(v, Math.min(low.get(v)!, index.get(to)!));
+      }
+    }
+    if (low.get(v) === index.get(v)) {
+      const group: string[] = [];
+      let w: string;
+      do {
+        w = stack.pop()!;
+        onStack.delete(w);
+        group.push(w);
+      } while (w !== v);
+      groups.push(group);
+    }
+  };
+  for (const id of order) if (!index.has(id)) visit(id);
+
+  const issues: CollectorIssue[] = [];
+  for (const group of groups) {
+    const members = new Set(group);
+    const start = order.find((id) => members.has(id))!;
+    const path = shortestCycle(start, members, out);
+    if (!path) continue; // a single pipeline with no edge to itself
+    const hops = path.map((e) => `${e.from} -> ${e.connector}`).join(" -> ");
+    issues.push({
+      code: "OTEL113",
+      severity: "error",
+      pipeline: start,
+      message: `pipelines form a cycle through connectors: ${hops} -> ${start}; the collector refuses to start`,
+    });
+  }
+  return issues;
+}
+
+/** The shortest path of edges from `start` back to itself, staying inside `members`. */
+function shortestCycle(start: string, members: Set<string>, out: Map<string, TopologyEdge[]>): TopologyEdge[] | undefined {
+  const via = new Map<string, TopologyEdge>();
+  const queue = [start];
+  while (queue.length > 0) {
+    const v = queue.shift()!;
+    for (const e of out.get(v) ?? []) {
+      if (!members.has(e.to)) continue;
+      if (e.to === start) {
+        const path = [e];
+        for (let at = v; at !== start; at = via.get(at)!.from) path.unshift(via.get(at)!);
+        return path;
+      }
+      if (!via.has(e.to)) {
+        via.set(e.to, e);
+        queue.push(e.to);
+      }
+    }
+  }
+  return undefined;
+}
+
+/**
+ * OTEL114: a pipeline names a connector the same way it names a receiver or
+ * an exporter, so the collector refuses a connector id that is also declared
+ * under receivers or exporters, used or not.
+ */
+function connectorIdIssues(config: CollectorConfig): CollectorIssue[] {
+  const receivers = ids(config.receivers);
+  const exporters = ids(config.exporters);
+  const issues: CollectorIssue[] = [];
+  for (const id of ids(config.connectors)) {
+    const clashes = [receivers.has(id) ? "receiver" : "", exporters.has(id) ? "exporter" : ""].filter(Boolean);
+    if (clashes.length === 0) continue;
+    const type = parseComponentId(id)?.type ?? id;
+    issues.push({
+      code: "OTEL114",
+      severity: "error",
+      component: id,
+      message: `connector "${id}" has the same id as a declared ${clashes.join(" and ")}, so a pipeline listing "${id}" is ambiguous and the collector refuses to start; rename one of them (e.g. "${type}/connector")`,
+    });
+  }
+  return issues;
+}
+
+/**
+ * OTEL115: each pipeline a `routing` connector routes to, in `table[].pipelines`
+ * or `default_pipelines`, must list that connector in its receivers. The
+ * connector can only hand data to the pipelines it feeds, and the collector
+ * refuses to start when a route names any other.
+ */
+function routingTargetIssues(config: CollectorConfig): CollectorIssue[] {
+  const pipelines = config.service?.pipelines ?? {};
+  const issues: CollectorIssue[] = [];
+  for (const [id, raw] of Object.entries(config.connectors ?? {})) {
+    if (parseComponentId(id)?.type !== "routing") continue;
+    const cfg = (raw ?? {}) as { table?: Array<{ pipelines?: unknown }>; default_pipelines?: unknown };
+    const targets: Array<[string, string]> = [];
+    (Array.isArray(cfg.table) ? cfg.table : []).forEach((item, i) => {
+      for (const t of Array.isArray(item?.pipelines) ? item.pipelines : []) targets.push([String(t), `table[${i}].pipelines`]);
+    });
+    for (const t of Array.isArray(cfg.default_pipelines) ? cfg.default_pipelines : []) targets.push([String(t), "default_pipelines"]);
+
+    const reported = new Set<string>();
+    for (const [target, where] of targets) {
+      if (reported.has(target)) continue;
+      const pipeline = Object.prototype.hasOwnProperty.call(pipelines, target) ? pipelines[target] : undefined;
+      if (pipeline && (pipeline.receivers ?? []).map(String).includes(id)) continue;
+      reported.add(target);
+      issues.push({
+        code: "OTEL115",
+        severity: "error",
+        pipeline: target,
+        component: id,
+        message: pipeline
+          ? `routing connector "${id}" routes to pipeline "${target}" (${where}), which does not list "${id}" in its receivers; the collector refuses to start`
+          : `routing connector "${id}" routes to pipeline "${target}" (${where}), which is not declared under service.pipelines; the collector refuses to start`,
+      });
+    }
+  }
+  return issues;
+}
+
+/** Where a connector lists the attribute keys it splits metrics by. */
+interface MetricAttributeUse {
+  key: string;
+  /** The config field, e.g. `dimensions` or `spans.genai.tokens.attributes`. */
+  field: string;
+}
+
+function list(value: unknown): unknown[] {
+  return Array.isArray(value) ? value : [];
+}
+
+function keysOf(items: unknown, prop: "name" | "key", field: string): MetricAttributeUse[] {
+  return list(items).flatMap((item) => {
+    const key = typeof item === "string" ? item : (item as Record<string, unknown> | null)?.[prop];
+    return typeof key === "string" ? [{ key, field }] : [];
+  });
+}
+
+/**
+ * The attribute keys a built-in metrics connector puts on the metrics it
+ * makes, by type. An empty or unset `include_resource_attributes` keeps
+ * every resource attribute, whichever they are, so only listed keys can be
+ * named.
+ */
+function metricAttributeUses(type: string, config: Record<string, unknown>): MetricAttributeUse[] {
+  switch (type) {
+    case "spanmetrics": {
+      const histogram = config.histogram as Record<string, unknown> | undefined;
+      const events = config.events as Record<string, unknown> | undefined;
+      return [
+        ...keysOf(config.dimensions, "name", "dimensions"),
+        ...keysOf(config.calls_dimensions, "name", "calls_dimensions"),
+        ...keysOf(histogram?.dimensions, "name", "histogram.dimensions"),
+        ...keysOf(events?.dimensions, "name", "events.dimensions"),
+      ];
+    }
+    case "servicegraph":
+      return keysOf(config.dimensions, "name", "dimensions");
+    case "count":
+    case "sum":
+      return Object.entries(config).flatMap(([section, metrics]) =>
+        typeof metrics === "object" && metrics !== null && !Array.isArray(metrics)
+          ? Object.entries(metrics as Record<string, unknown>).flatMap(([name, info]) =>
+              keysOf((info as Record<string, unknown> | null)?.attributes, "key", `${section}.${name}.attributes`),
+            )
+          : [],
+      );
+    case "signaltometrics":
+      return signalToMetricsEntries(config as SignalToMetricsConnectorConfig).flatMap(({ signal, index, metric }) => {
+        const at = metric.name ? `${signal}[${index}] (${metric.name})` : `${signal}[${index}]`;
+        return [
+          ...keysOf(metric.attributes, "key", `${at}.attributes`),
+          ...keysOf(metric.include_resource_attributes, "key", `${at}.include_resource_attributes`),
+        ];
+      });
+    default:
+      return [];
+  }
+}
+
+/**
+ * OTEL116: a connector splits metrics by a GenAI attribute that takes a new
+ * value per response, tool call, conversation, session or user, or by a
+ * content attribute. Each value starts new time series, so the metric's
+ * series grow with traffic. The keys are `GENAI_HIGH_CARDINALITY_ATTRIBUTES`
+ * and the content keys in `genai.ts`.
+ */
+function metricAttributeIssues(config: CollectorConfig): CollectorIssue[] {
+  const issues: CollectorIssue[] = [];
+  for (const [id, raw] of Object.entries(config.connectors ?? {})) {
+    const type = parseComponentId(id)?.type ?? id;
+    const body = typeof raw === "object" && raw !== null ? (raw as Record<string, unknown>) : {};
+    for (const { key, field } of metricAttributeUses(type, body)) {
+      const risk = genAiCardinalityRisk(key);
+      if (!risk) continue;
+      const why =
+        risk === "content"
+          ? "it carries message content, which is unbounded and sensitive"
+          : "it takes a new value per request, conversation or user";
+      issues.push({
+        code: "OTEL116",
+        severity: "warning",
+        component: id,
+        message: `connector "${id}" splits metrics by "${key}" (${field}); ${why}, so every value starts new time series. Drop it from the metric's attributes, or keep it on spans and logs`,
+      });
+    }
+  }
+  return issues;
+}
+
+/** One address a started component listens on. */
+interface Listener {
+  /** Who listens: `receiver "otlp" (protocols.grpc.endpoint)`. */
+  owner: string;
+  component?: string;
+  host: string;
+  port: number;
+  transport: "tcp" | "udp";
+}
+
+/**
+ * The listeners OTEL117 knows, by component type: the config key path to an
+ * address, its default at `COLLECTOR_PIN` (collector and contrib v0.130.0,
+ * each component's `factory.go`), and the transport. `present` means the
+ * default applies only when the key's parent is in the config: an otlp or
+ * jaeger protocol that isn't listed doesn't start. A type not listed here is
+ * not checked, since its `endpoint` may be a server it connects to (the
+ * kubeletstats receiver's is the kubelet).
+ */
+interface ListenerSpec {
+  path: string[];
+  default?: string;
+  transport?: "udp";
+  present?: boolean;
+}
+
+const LISTENERS: Record<"receiver" | "exporter" | "extension", Record<string, ListenerSpec[]>> = {
+  receiver: {
+    otlp: [
+      { path: ["protocols", "grpc", "endpoint"], default: "localhost:4317", present: true },
+      { path: ["protocols", "http", "endpoint"], default: "localhost:4318", present: true },
+    ],
+    zipkin: [{ path: ["endpoint"], default: "localhost:9411" }],
+    jaeger: [
+      { path: ["protocols", "grpc", "endpoint"], default: "localhost:14250", present: true },
+      { path: ["protocols", "thrift_http", "endpoint"], default: "localhost:14268", present: true },
+      { path: ["protocols", "thrift_binary", "endpoint"], default: "localhost:6832", present: true, transport: "udp" },
+      { path: ["protocols", "thrift_compact", "endpoint"], default: "localhost:6831", present: true, transport: "udp" },
+    ],
+  },
+  exporter: {
+    prometheus: [{ path: ["endpoint"] }],
+  },
+  extension: {
+    health_check: [{ path: ["endpoint"], default: "localhost:13133" }],
+    zpages: [{ path: ["endpoint"], default: "localhost:55679" }],
+    pprof: [{ path: ["endpoint"], default: "localhost:1777" }],
+  },
+};
+
+/** The collector's own metrics endpoint when `service.telemetry.metrics` names no reader. */
+const DEFAULT_TELEMETRY_METRICS = { host: "localhost", port: 8888 };
+
+const WILDCARD_HOSTS = new Set(["", "0.0.0.0", "::", "[::]"]);
+
+function splitAddress(address: unknown): { host: string; port: number } | undefined {
+  if (typeof address !== "string") return undefined;
+  const m = address.match(/^(.*):(\d+)$/);
+  if (!m) return undefined;
+  return { host: m[1], port: Number(m[2]) };
+}
+
+function at(value: unknown, path: string[]): { found: boolean; value: unknown } {
+  let cur: unknown = value;
+  for (const key of path) {
+    if (typeof cur !== "object" || cur === null || !(key in (cur as Record<string, unknown>))) return { found: false, value: undefined };
+    cur = (cur as Record<string, unknown>)[key];
+  }
+  return { found: true, value: cur };
+}
+
+function componentListeners(kind: "receiver" | "exporter" | "extension", id: string, body: unknown): Listener[] {
+  const type = parseComponentId(id)?.type ?? id;
+  const out: Listener[] = [];
+  for (const spec of LISTENERS[kind][type] ?? []) {
+    if (spec.present && !at(body, spec.path.slice(0, -1)).found) continue;
+    const { found, value } = at(body, spec.path);
+    const address = splitAddress(found && value !== undefined && value !== null ? value : spec.default);
+    if (!address) continue;
+    out.push({ owner: `${kind} "${id}" (${spec.path.join(".")})`, component: id, ...address, transport: spec.transport ?? "tcp" });
+  }
+  return out;
+}
+
+/** The collector's own metrics endpoints, from `service.telemetry.metrics`. */
+function telemetryListeners(config: CollectorConfig): Listener[] {
+  const metrics = (config.service?.telemetry as Record<string, unknown> | undefined)?.metrics as Record<string, unknown> | undefined;
+  if (metrics?.level === "none") return [];
+  const readers = Array.isArray(metrics?.readers) ? (metrics.readers as unknown[]) : undefined;
+  if (!readers) return [{ owner: "the collector's own metrics (service.telemetry.metrics, by default)", ...DEFAULT_TELEMETRY_METRICS, transport: "tcp" }];
+  const out: Listener[] = [];
+  readers.forEach((reader, i) => {
+    const prom = at(reader, ["pull", "exporter", "prometheus"]);
+    if (!prom.found || typeof prom.value !== "object" || prom.value === null) return;
+    const { host, port } = prom.value as { host?: unknown; port?: unknown };
+    const portNumber = typeof port === "number" ? port : typeof port === "string" && /^\d+$/.test(port) ? Number(port) : undefined;
+    if (portNumber === undefined) return;
+    out.push({
+      owner: `the collector's own metrics (service.telemetry.metrics.readers[${i}])`,
+      host: typeof host === "string" ? host : "localhost",
+      port: portNumber,
+      transport: "tcp",
+    });
+  });
+  return out;
+}
+
+function sameAddress(a: Listener, b: Listener): boolean {
+  if (a.port !== b.port || a.transport !== b.transport) return false;
+  return a.host === b.host || WILDCARD_HOSTS.has(a.host) || WILDCARD_HOSTS.has(b.host);
+}
+
+/**
+ * OTEL117: two things the collector starts listen on the same address, so
+ * the second bind fails with "address already in use" and the collector
+ * exits. `otelcol validate` builds the config without binding, so it does
+ * not catch this. Started means: a receiver or exporter some pipeline lists,
+ * an extension in `service.extensions`, and the collector's own metrics
+ * endpoint. A wildcard host (`0.0.0.0`, `::`, empty) overlaps every host on
+ * its port; an `${env:...}` host is compared as written.
+ */
+function listenerIssues(config: CollectorConfig): CollectorIssue[] {
+  const pipelines = Object.values(config.service?.pipelines ?? {});
+  const started = (refs: Array<string[] | undefined>) => new Set(refs.flatMap((r) => (r ?? []).map(String)));
+  const receivers = started(pipelines.map((p) => p?.receivers));
+  const exporters = started(pipelines.map((p) => p?.exporters));
+  const extensions = new Set((config.service?.extensions ?? []).map(String));
+
+  const listeners: Listener[] = [];
+  const sections: Array<["receiver" | "exporter" | "extension", Record<string, unknown> | undefined, Set<string>]> = [
+    ["receiver", config.receivers, receivers],
+    ["exporter", config.exporters, exporters],
+    ["extension", config.extensions, extensions],
+  ];
+  for (const [kind, section, startedIds] of sections) {
+    for (const [id, body] of Object.entries(section ?? {})) {
+      if (startedIds.has(id)) listeners.push(...componentListeners(kind, id, body));
+    }
+  }
+  listeners.push(...telemetryListeners(config));
+
+  const issues: CollectorIssue[] = [];
+  for (let i = 0; i < listeners.length; i++) {
+    for (let j = 0; j < i; j++) {
+      const [first, second] = [listeners[j], listeners[i]];
+      if (!sameAddress(first, second)) continue;
+      const udp = first.transport === "udp" ? " (UDP)" : "";
+      issues.push({
+        code: "OTEL117",
+        severity: "error",
+        ...(second.component ? { component: second.component } : first.component ? { component: first.component } : {}),
+        message: `${first.owner} listens on ${first.host}:${first.port}${udp} and ${second.owner} on ${second.host}:${second.port}${udp}; the second bind fails with "address already in use" and the collector exits. Move one of them to another port`,
+      });
+    }
+  }
   return issues;
 }
 

package/src/validate.ts

@@ -31,15 +31,24 @@ export const REQUIRED_NAMES = [
   "RedactionProcessor",
   "TailSamplingProcessor",
   "ProbabilisticSamplerProcessor",
+  "DeltaToCumulativeProcessor",
   "OtlpExporter",
   "OtlpHttpExporter",
   "DebugExporter",
   "PrometheusExporter",
   "GoogleCloudExporter",
   "LoadBalancingExporter",
+  "SpanMetricsConnector",
+  "ServiceGraphConnector",
+  "RoutingConnector",
+  "ForwardConnector",
+  "CountConnector",
+  "SumConnector",
+  "SignalToMetricsConnector",
   "HealthCheckExtension",
   "PprofExtension",
   "ZPagesExtension",
+  "K8sLeaderElectorExtension",
   "Pipeline",
   "Service",
 ];
@@ -55,6 +64,14 @@ export async function validate(): Promise<ValidateResult> {
       : { name: "required-names", ok: false, error: `Missing required names: ${missing.join(", ")}` },
   );
 
+  // The list names every entity the registry has, so a new built-in is held to it from the start.
+  const unlisted = Object.keys(registry).filter((n) => !REQUIRED_NAMES.includes(n));
+  checks.push(
+    unlisted.length === 0
+      ? { name: "required-names-complete", ok: true }
+      : { name: "required-names-complete", ok: false, error: `In the registry but not in REQUIRED_NAMES: ${unlisted.join(", ")}` },
+  );
+
   const classes = (Object.entries(components) as Array<[string, unknown]>).filter(
     (e): e is [string, ComponentClass] => typeof e[1] === "function" && "definition" in (e[1] as object),
   );