@intentius/chant-lexicon-otel 0.81.0

package/src/index.ts

@@ -0,0 +1,73 @@
+// OpenTelemetry Collector lexicon.
+
+// Plugin and serializer
+export { otelPlugin } from "./plugin";
+export { otelSerializer } from "./serializer";
+
+// Built-in components and their config types
+export * from "./components";
+
+// Pipelines and the service block
+export {
+  Pipeline,
+  Service,
+  PIPELINE_TYPE,
+  SERVICE_TYPE,
+  isPipelineEntity,
+  isServiceEntity,
+  type ComponentRef,
+  type PipelineProps,
+  type PipelineEntity,
+  type ServiceProps,
+  type ServiceEntity,
+  type ServiceTelemetry,
+} from "./pipeline";
+
+// The extension point: components chant doesn't ship
+export {
+  defineComponent,
+  definitionFor,
+  definitionOf,
+  registeredDefinitions,
+  componentEntityType,
+  isOTelComponent,
+  COLLECTOR_PIN,
+  type SchemaPin,
+  type SafeParseSchema,
+  type ConfigValidator,
+  type ComponentDefinition,
+  type CustomComponentOptions,
+  type ComponentProps,
+  type ComponentClass,
+  type OTelComponent,
+} from "./define";
+
+// The plain-data model, YAML, checks and topology
+export * from "./model";
+export { buildCollectorConfig, collectorYaml, componentConfig, type BuiltCollector } from "./collector";
+export { emitCollectorYaml, type EmitOptions } from "./yaml";
+export {
+  validateCollectorConfig,
+  validateCollectorEntities,
+  type CollectorIssue,
+  type CollectorIssueCode,
+} from "./validate-config";
+export {
+  collectorTopology,
+  collectorTopologyOf,
+  type CollectorTopology,
+  type TopologyPipeline,
+  type TopologyComponent,
+  type TopologyExporter,
+} from "./topology";
+
+// What the platform collector composites (docker, k8s, fly) share
+export {
+  otlpCollector,
+  collectorEndpoints,
+  COLLECTOR_IMAGE,
+  COLLECTOR_CONFIG_PATH,
+  type OtlpCollectorOptions,
+  type CollectorPort,
+  type CollectorEndpoints,
+} from "./platform";

package/src/lint/audit-catalog.ts

@@ -0,0 +1,102 @@
+/**
+ * The otel lexicon's chant audit catalog, contributed via
+ * `otelPlugin.auditCatalog()` (#687, #1346).
+ *
+ * OTEL101-OTEL106 read the emitted collector YAML, so they are `yamlBased`.
+ * OTEL107-OTEL109 read the declared entities (a component's definition, its
+ * schema pin), which a standalone YAML file does not carry, so they are
+ * constructed with `yamlBased: false`. The two source-level lint rules are
+ * listed too, for a reader who meets them in a lint report.
+ */
+
+import { auditRule, type RuleMeta } from "@intentius/chant/audit/catalog";
+
+function entityRule(
+  id: string,
+  category: RuleMeta["category"],
+  title: string,
+  remediation: string,
+): RuleMeta {
+  return { id, tier: "merge-worthy", fixKind: "guidance", category, title, remediation, yamlBased: false };
+}
+
+export const otelAuditCatalog: Record<string, RuleMeta> = {
+  OTEL001: entityRule(
+    "OTEL001",
+    "correctness",
+    "Collector component id or instance name is not valid syntax",
+    'Write component ids as `type` or `type/name` (e.g. "otlp/backend") and give instances a non-empty name with no whitespace.',
+  ),
+  OTEL002: entityRule(
+    "OTEL002",
+    "security",
+    "Collector credential declared as a literal",
+    'Replace the value with `${env:NAME}` or `${file:/path}` so the collector reads it at start-up.',
+  ),
+  OTEL101: auditRule(
+    "OTEL101",
+    "merge-worthy",
+    "guidance",
+    "Pipeline uses an undeclared component",
+    "Declare the receiver, processor or exporter under its section, or reference the declared entity instead of an id string.",
+    { category: "correctness" },
+  ),
+  OTEL102: auditRule(
+    "OTEL102",
+    "merge-worthy",
+    "guidance",
+    "Pipeline has no receivers or no exporters",
+    "Give every pipeline at least one receiver and at least one exporter, or remove it.",
+    { category: "correctness" },
+  ),
+  OTEL103: auditRule(
+    "OTEL103",
+    "report-only",
+    "guidance",
+    "Declared collector component is never used",
+    "List the component in a pipeline (or the extension in service.extensions), or remove its declaration.",
+    { category: "best-practice" },
+  ),
+  OTEL104: auditRule(
+    "OTEL104",
+    "merge-worthy",
+    "guidance",
+    "service.extensions enables an undeclared extension",
+    "Declare the extension under extensions, or drop it from service.extensions.",
+    { category: "correctness" },
+  ),
+  OTEL105: auditRule(
+    "OTEL105",
+    "report-only",
+    "guidance",
+    "memory_limiter is not the first processor",
+    "Move memory_limiter to the front of the pipeline's processors list.",
+    { category: "best-practice" },
+  ),
+  OTEL106: auditRule(
+    "OTEL106",
+    "merge-worthy",
+    "guidance",
+    "Pipeline id or component reference is not valid collector syntax",
+    "Name pipelines traces, metrics or logs (optionally /name) and reference components as type or type/name.",
+    { category: "correctness" },
+  ),
+  OTEL107: entityRule(
+    "OTEL107",
+    "correctness",
+    "Collector component config breaks its definition's rules",
+    "Fix the config the message names; for a custom component the rule comes from its defineComponent validate.",
+  ),
+  OTEL108: entityRule(
+    "OTEL108",
+    "correctness",
+    "Two collector components or pipelines declare the same id",
+    "Give each instance its own name so their ids (type/name) differ.",
+  ),
+  OTEL109: entityRule(
+    "OTEL109",
+    "correctness",
+    "Custom collector component has no schema pin",
+    "Pass defineComponent a pin with the source and version the component's config type follows.",
+  ),
+};

package/src/lint/post-synth/index.ts

@@ -0,0 +1,23 @@
+// Code generated by chant generate. DO NOT EDIT.
+import type { PostSynthCheck } from "@intentius/chant/lint/post-synth";
+import { otel101 } from "./otel101";
+import { otel102 } from "./otel102";
+import { otel103 } from "./otel103";
+import { otel104 } from "./otel104";
+import { otel105 } from "./otel105";
+import { otel106 } from "./otel106";
+import { otel107 } from "./otel107";
+import { otel108 } from "./otel108";
+import { otel109 } from "./otel109";
+
+export const postSynthChecks: PostSynthCheck[] = [
+  otel101,
+  otel102,
+  otel103,
+  otel104,
+  otel105,
+  otel106,
+  otel107,
+  otel108,
+  otel109,
+];

package/src/lint/post-synth/otel-helpers.ts

@@ -0,0 +1,70 @@
+/**
+ * Shared plumbing for the otel post-synth checks: find the collector configs
+ * in a build's output, and run the plain-function checks over them.
+ */
+
+import type { PostSynthContext, PostSynthDiagnostic } from "@intentius/chant/lint/post-synth";
+import type { SerializerResult } from "@intentius/chant/serializer";
+import { loadAll } from "js-yaml";
+import type { Declarable } from "@intentius/chant/declarable";
+import { looksLikeCollectorConfig, type CollectorConfig } from "../../model";
+import { validateCollectorConfig, validateCollectorEntities, type CollectorIssueCode, type CollectorIssue } from "../../validate-config";
+
+/**
+ * Every collector config in the output: the otel lexicon's own, and any YAML
+ * document shaped like one. Parsed with js-yaml rather than `ctx.docs`,
+ * because collector configs are written with flow lists (`[otlp, batch]`)
+ * that core's small YAML reader keeps as strings.
+ */
+export function collectorConfigs(ctx: PostSynthContext): Array<{ source: string; config: CollectorConfig }> {
+  const out: Array<{ source: string; config: CollectorConfig }> = [];
+  for (const [lexicon, output] of ctx.outputs) {
+    const texts: Array<[string, string]> =
+      typeof output === "string"
+        ? [[lexicon, output]]
+        : [[lexicon, (output as SerializerResult).primary], ...Object.entries((output as SerializerResult).files ?? {})];
+    for (const [source, text] of texts) {
+      if (!text) continue;
+      let docs: unknown[];
+      try {
+        docs = loadAll(text);
+      } catch {
+        continue; // not YAML, or not ours
+      }
+      for (const doc of docs) {
+        if (typeof doc !== "object" || doc === null || Array.isArray(doc)) continue;
+        if ((lexicon === "otel" && source === lexicon) || looksLikeCollectorConfig(doc)) {
+          out.push({ source, config: doc as CollectorConfig });
+        }
+      }
+    }
+  }
+  return out;
+}
+
+function toDiagnostic(issue: CollectorIssue, source?: string): PostSynthDiagnostic {
+  return {
+    checkId: issue.code,
+    severity: issue.severity,
+    message: source && source !== "otel" ? `${source}: ${issue.message}` : issue.message,
+    ...(issue.component ? { entity: issue.component } : issue.pipeline ? { entity: issue.pipeline } : {}),
+    lexicon: "otel",
+  };
+}
+
+/** Diagnostics for one config-level code (OTEL101-OTEL106) across every collector config in the output. */
+export function configDiagnostics(ctx: PostSynthContext, code: CollectorIssueCode): PostSynthDiagnostic[] {
+  return collectorConfigs(ctx).flatMap(({ source, config }) =>
+    validateCollectorConfig(config)
+      .filter((i) => i.code === code)
+      .map((i) => toDiagnostic(i, source)),
+  );
+}
+
+/** Diagnostics for one entity-level code (OTEL107-OTEL109) over the build's otel entities. */
+export function entityDiagnostics(ctx: PostSynthContext, code: CollectorIssueCode): PostSynthDiagnostic[] {
+  const otel: Declarable[] = [...(ctx.entities?.values() ?? [])].filter((e) => e?.lexicon === "otel");
+  return validateCollectorEntities(otel)
+    .filter((i) => i.code === code)
+    .map((i) => toDiagnostic(i));
+}

package/src/lint/post-synth/otel101.ts

@@ -0,0 +1,17 @@
+/**
+ * OTEL101: A pipeline uses a receiver, processor or exporter that is not declared
+ *
+ * The collector refuses to start on this. Typed entity references cannot go wrong this way, so it mostly catches id strings (`"otlp/backend"`) and hand-edited or parsed configs. A connector id is accepted in receivers and exporters.
+ */
+
+import type { PostSynthCheck, PostSynthContext, PostSynthDiagnostic } from "@intentius/chant/lint/post-synth";
+import { configDiagnostics } from "./otel-helpers";
+
+export const otel101: PostSynthCheck = {
+  id: "OTEL101",
+  description: "A pipeline uses a receiver, processor or exporter that is not declared",
+
+  check(ctx: PostSynthContext): PostSynthDiagnostic[] {
+    return configDiagnostics(ctx, "OTEL101");
+  },
+};

package/src/lint/post-synth/otel102.ts

@@ -0,0 +1,17 @@
+/**
+ * OTEL102: A pipeline has no receivers or no exporters
+ *
+ * A pipeline with no receivers takes in nothing, and one with no exporters sends what it takes in nowhere. The collector rejects both at start.
+ */
+
+import type { PostSynthCheck, PostSynthContext, PostSynthDiagnostic } from "@intentius/chant/lint/post-synth";
+import { configDiagnostics } from "./otel-helpers";
+
+export const otel102: PostSynthCheck = {
+  id: "OTEL102",
+  description: "A pipeline has no receivers or no exporters",
+
+  check(ctx: PostSynthContext): PostSynthDiagnostic[] {
+    return configDiagnostics(ctx, "OTEL102");
+  },
+};

package/src/lint/post-synth/otel103.ts

@@ -0,0 +1,17 @@
+/**
+ * OTEL103: A declared component is never used
+ *
+ * A receiver, processor or exporter no pipeline lists, or an extension missing from service.extensions, is ignored by the collector. Usually it means a pipeline was meant to list it. A warning, since the config still runs.
+ */
+
+import type { PostSynthCheck, PostSynthContext, PostSynthDiagnostic } from "@intentius/chant/lint/post-synth";
+import { configDiagnostics } from "./otel-helpers";
+
+export const otel103: PostSynthCheck = {
+  id: "OTEL103",
+  description: "A declared component is never used",
+
+  check(ctx: PostSynthContext): PostSynthDiagnostic[] {
+    return configDiagnostics(ctx, "OTEL103");
+  },
+};

package/src/lint/post-synth/otel104.ts

@@ -0,0 +1,17 @@
+/**
+ * OTEL104: service.extensions enables an extension that is not declared
+ *
+ * The collector refuses to start when service.extensions names an id with no entry under extensions.
+ */
+
+import type { PostSynthCheck, PostSynthContext, PostSynthDiagnostic } from "@intentius/chant/lint/post-synth";
+import { configDiagnostics } from "./otel-helpers";
+
+export const otel104: PostSynthCheck = {
+  id: "OTEL104",
+  description: "service.extensions enables an extension that is not declared",
+
+  check(ctx: PostSynthContext): PostSynthDiagnostic[] {
+    return configDiagnostics(ctx, "OTEL104");
+  },
+};

package/src/lint/post-synth/otel105.ts

@@ -0,0 +1,17 @@
+/**
+ * OTEL105: memory_limiter is not the first processor in a pipeline
+ *
+ * The collector's own guidance puts memory_limiter first, so it can refuse data before any other processor buffers it. Later in the chain it limits too late to protect the process.
+ */
+
+import type { PostSynthCheck, PostSynthContext, PostSynthDiagnostic } from "@intentius/chant/lint/post-synth";
+import { configDiagnostics } from "./otel-helpers";
+
+export const otel105: PostSynthCheck = {
+  id: "OTEL105",
+  description: "memory_limiter is not the first processor in a pipeline",
+
+  check(ctx: PostSynthContext): PostSynthDiagnostic[] {
+    return configDiagnostics(ctx, "OTEL105");
+  },
+};

package/src/lint/post-synth/otel106.ts

@@ -0,0 +1,17 @@
+/**
+ * OTEL106: A pipeline id or component reference is not valid collector syntax
+ *
+ * A pipeline id is a signal (traces, metrics, logs) optionally followed by /name, and every reference is type or type/name. Anything else is a config error at collector start.
+ */
+
+import type { PostSynthCheck, PostSynthContext, PostSynthDiagnostic } from "@intentius/chant/lint/post-synth";
+import { configDiagnostics } from "./otel-helpers";
+
+export const otel106: PostSynthCheck = {
+  id: "OTEL106",
+  description: "A pipeline id or component reference is not valid collector syntax",
+
+  check(ctx: PostSynthContext): PostSynthDiagnostic[] {
+    return configDiagnostics(ctx, "OTEL106");
+  },
+};

package/src/lint/post-synth/otel107.ts

@@ -0,0 +1,17 @@
+/**
+ * OTEL107: A component's config breaks its definition's rules
+ *
+ * Each component definition, built-in or custom, can check its own config beyond what the TypeScript type says: memory_limiter needs a limit, filelog needs an include, and so on. A custom component supplies its checks through defineComponent's validate.
+ */
+
+import type { PostSynthCheck, PostSynthContext, PostSynthDiagnostic } from "@intentius/chant/lint/post-synth";
+import { entityDiagnostics } from "./otel-helpers";
+
+export const otel107: PostSynthCheck = {
+  id: "OTEL107",
+  description: "A component's config breaks its definition's rules",
+
+  check(ctx: PostSynthContext): PostSynthDiagnostic[] {
+    return entityDiagnostics(ctx, "OTEL107");
+  },
+};

package/src/lint/post-synth/otel108.ts

@@ -0,0 +1,17 @@
+/**
+ * OTEL108: Two components or two pipelines declare the same id
+ *
+ * The collector config is keyed by id, so the second declaration would silently replace the first. The serializer keeps the first and this check fails the build.
+ */
+
+import type { PostSynthCheck, PostSynthContext, PostSynthDiagnostic } from "@intentius/chant/lint/post-synth";
+import { entityDiagnostics } from "./otel-helpers";
+
+export const otel108: PostSynthCheck = {
+  id: "OTEL108",
+  description: "Two components or two pipelines declare the same id",
+
+  check(ctx: PostSynthContext): PostSynthDiagnostic[] {
+    return entityDiagnostics(ctx, "OTEL108");
+  },
+};

package/src/lint/post-synth/otel109.ts

@@ -0,0 +1,17 @@
+/**
+ * OTEL109: A custom component has no schema pin
+ *
+ * defineComponent requires a pin naming the source and version its config type follows, so a reader can tell which schema the component was checked against. This check fails a build whose custom component lost its pin, for instance through an untyped call.
+ */
+
+import type { PostSynthCheck, PostSynthContext, PostSynthDiagnostic } from "@intentius/chant/lint/post-synth";
+import { entityDiagnostics } from "./otel-helpers";
+
+export const otel109: PostSynthCheck = {
+  id: "OTEL109",
+  description: "A custom component has no schema pin",
+
+  check(ctx: PostSynthContext): PostSynthDiagnostic[] {
+    return entityDiagnostics(ctx, "OTEL109");
+  },
+};

package/src/lint/post-synth/post-synth.test.ts

@@ -0,0 +1,184 @@
+import { describe, expect, test } from "vitest";
+import { makePostSynthCtx } from "@intentius/chant-test-utils";
+import type { Declarable } from "@intentius/chant/declarable";
+import { otel101 } from "./otel101";
+import { otel102 } from "./otel102";
+import { otel103 } from "./otel103";
+import { otel104 } from "./otel104";
+import { otel105 } from "./otel105";
+import { otel106 } from "./otel106";
+import { otel107 } from "./otel107";
+import { otel108 } from "./otel108";
+import { otel109 } from "./otel109";
+import { BatchProcessor, DebugExporter, FileLogReceiver, MemoryLimiterProcessor, OtlpReceiver } from "../../components";
+import { defineComponent } from "../../define";
+import { Pipeline } from "../../pipeline";
+
+const GOOD = `receivers:
+  otlp:
+    protocols:
+      grpc: {}
+
+processors:
+  memory_limiter:
+    check_interval: 1s
+    limit_mib: 400
+  batch: {}
+
+exporters:
+  debug: {}
+
+extensions:
+  health_check: {}
+
+service:
+  extensions: [health_check]
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [memory_limiter, batch]
+      exporters: [debug]
+`;
+
+const configChecks = [otel101, otel102, otel103, otel104, otel105, otel106];
+
+function entities(list: unknown[]): Map<string, Declarable> {
+  return new Map(list.map((e, i) => [`e${i}`, e as Declarable]));
+}
+
+describe("config-level checks on a clean config", () => {
+  test.each(configChecks.map((c) => [c.id, c] as const))("%s finds nothing", (_id, check) => {
+    expect(check.check(makePostSynthCtx("otel", GOOD))).toEqual([]);
+  });
+});
+
+describe("OTEL101 undeclared component", () => {
+  test("flags an exporter and a processor nobody declares", () => {
+    const yaml = GOOD.replace("exporters: [debug]", "exporters: [debug, otlp/backend]").replace(
+      "processors: [memory_limiter, batch]",
+      "processors: [memory_limiter, batch, attributes/redact]",
+    );
+    const diags = otel101.check(makePostSynthCtx("otel", yaml));
+    expect(diags.map((d) => d.entity)).toEqual(["attributes/redact", "otlp/backend"]);
+    expect(diags[0].severity).toBe("error");
+  });
+
+  test("accepts a connector id as a receiver or exporter", () => {
+    const yaml = `connectors:
+  forward: {}
+receivers:
+  otlp: {protocols: {grpc: {}}}
+exporters:
+  debug: {}
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      exporters: [forward]
+    traces/2:
+      receivers: [forward]
+      exporters: [debug]
+`;
+    expect(otel101.check(makePostSynthCtx("otel", yaml))).toEqual([]);
+  });
+
+  test("finds a collector config in another lexicon's output by its shape", () => {
+    const yaml = GOOD.replace("exporters: [debug]", "exporters: [nope]");
+    expect(otel101.check(makePostSynthCtx("docker", yaml))).toHaveLength(1);
+  });
+});
+
+describe("OTEL102 empty ends", () => {
+  test("flags a pipeline with no receivers and one with no exporters", () => {
+    const yaml = `receivers:
+  otlp: {protocols: {grpc: {}}}
+exporters:
+  debug: {}
+service:
+  pipelines:
+    traces:
+      receivers: []
+      exporters: [debug]
+    logs:
+      receivers: [otlp]
+`;
+    const diags = otel102.check(makePostSynthCtx("otel", yaml));
+    expect(diags.map((d) => d.message)).toEqual([
+      'pipeline "traces" has no receivers, so nothing enters it',
+      'pipeline "logs" has no exporters, so what enters it goes nowhere',
+    ]);
+  });
+});
+
+describe("OTEL103 unused", () => {
+  test("warns about an unused exporter and an extension never enabled", () => {
+    const yaml = GOOD.replace("  debug: {}\n", "  debug: {}\n  otlp/spare:\n    endpoint: x:4317\n").replace(
+      "  health_check: {}\n",
+      "  health_check: {}\n  zpages: {}\n",
+    );
+    const diags = otel103.check(makePostSynthCtx("otel", yaml));
+    expect(diags.map((d) => [d.entity, d.severity])).toEqual([
+      ["otlp/spare", "warning"],
+      ["zpages", "warning"],
+    ]);
+  });
+});
+
+describe("OTEL104 undeclared extension", () => {
+  test("flags service.extensions naming an extension nobody declares", () => {
+    const yaml = GOOD.replace("extensions: [health_check]", "extensions: [health_check, pprof]");
+    expect(otel104.check(makePostSynthCtx("otel", yaml)).map((d) => d.entity)).toEqual(["pprof"]);
+  });
+});
+
+describe("OTEL105 memory_limiter position", () => {
+  test("warns when memory_limiter is not first", () => {
+    const yaml = GOOD.replace("processors: [memory_limiter, batch]", "processors: [batch, memory_limiter]");
+    const diags = otel105.check(makePostSynthCtx("otel", yaml));
+    expect(diags).toHaveLength(1);
+    expect(diags[0].severity).toBe("warning");
+  });
+});
+
+describe("OTEL106 syntax", () => {
+  test("flags a pipeline id that names no signal", () => {
+    const yaml = GOOD.replace("    traces:\n", "    trace:\n");
+    expect(otel106.check(makePostSynthCtx("otel", yaml)).map((d) => d.entity)).toEqual(["trace"]);
+  });
+});
+
+describe("entity-level checks", () => {
+  test("OTEL107 reports a built-in's own config rules", () => {
+    const limiter = new MemoryLimiterProcessor({ check_interval: "1s" });
+    const logs = new FileLogReceiver({ include: [] });
+    const diags = otel107.check(makePostSynthCtx("otel", "", entities([limiter, logs])));
+    expect(diags.map((d) => d.entity)).toEqual(["memory_limiter", "filelog"]);
+    expect(diags[0].message).toContain("limit_mib or limit_percentage");
+  });
+
+  test("OTEL107 is quiet for valid config", () => {
+    const limiter = new MemoryLimiterProcessor({ check_interval: "1s", limit_percentage: 80 });
+    const otlp = new OtlpReceiver({ protocols: { grpc: {} } });
+    expect(otel107.check(makePostSynthCtx("otel", "", entities([limiter, otlp, new BatchProcessor({})])))).toEqual([]);
+  });
+
+  test("OTEL108 flags duplicate component and pipeline ids", () => {
+    const a = new DebugExporter({});
+    const b = new DebugExporter({ verbosity: "detailed" });
+    const otlp = new OtlpReceiver({ protocols: { grpc: {} } });
+    const p1 = new Pipeline({ signal: "traces", receivers: [otlp], exporters: [a] });
+    const p2 = new Pipeline({ signal: "traces", receivers: [otlp], exporters: [b] });
+    const diags = otel108.check(makePostSynthCtx("otel", "", entities([a, b, otlp, p1, p2])));
+    expect(diags.map((d) => d.entity)).toEqual(["debug", "traces"]);
+  });
+
+  test("OTEL109 flags a custom component whose pin is unusable", () => {
+    const Loose = defineComponent<{ endpoint?: string }>()({
+      kind: "exporter",
+      type: "loosepin2559",
+      pin: { source: "", version: "" },
+    });
+    const diags = otel109.check(makePostSynthCtx("otel", "", entities([new Loose({})])));
+    expect(diags.map((d) => d.checkId)).toEqual(["OTEL109"]);
+  });
+});