@intentius/chant-lexicon-otel 0.97.0 → 0.99.0

package/src/import/parser.test.ts

@@ -0,0 +1,174 @@
+import { describe, expect, test } from "vitest";
+import { COLLECTOR_RESOURCE_TYPE, OtelCollectorParser, parseCollectorYaml } from "./parser";
+
+const parser = new OtelCollectorParser();
+
+describe("parseCollectorYaml", () => {
+  test("reads every section as the collector's own layout", () => {
+    const { config, warnings } = parseCollectorYaml(`
+receivers:
+  otlp:
+    protocols:
+      grpc:
+processors:
+  batch:
+exporters:
+  otlp/tempo:
+    endpoint: tempo:4317
+connectors:
+  forward/x: {}
+extensions:
+  health_check:
+service:
+  extensions: [health_check]
+  telemetry:
+    logs: { level: debug }
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [otlp/tempo, forward/x]
+`);
+    expect(warnings).toEqual([]);
+    expect(config.receivers).toEqual({ otlp: { protocols: { grpc: null } } });
+    expect(config.processors).toEqual({ batch: null });
+    expect(config.exporters).toEqual({ "otlp/tempo": { endpoint: "tempo:4317" } });
+    expect(config.connectors).toEqual({ "forward/x": {} });
+    expect(config.extensions).toEqual({ health_check: null });
+    expect(config.service).toEqual({
+      extensions: ["health_check"],
+      telemetry: { logs: { level: "debug" } },
+      pipelines: { traces: { receivers: ["otlp"], processors: ["batch"], exporters: ["otlp/tempo", "forward/x"] } },
+    });
+  });
+
+  test("keeps ${env:VAR} references and quoted scalars as strings", () => {
+    const { config } = parseCollectorYaml(`
+exporters:
+  otlphttp:
+    endpoint: \${env:OTLP_ENDPOINT}
+    headers:
+      api-key: "\${env:API_KEY}"
+    compression: "none"
+  prometheus:
+    endpoint: "0.0.0.0:8889"
+    const_labels:
+      port: "8080"
+      started: 2024-01-01
+`);
+    expect(config.exporters?.otlphttp).toEqual({
+      endpoint: "${env:OTLP_ENDPOINT}",
+      headers: { "api-key": "${env:API_KEY}" },
+      compression: "none",
+    });
+    // No YAML 1.1 dates: the collector reads this as a string, and so does the importer.
+    expect(config.exporters?.prometheus).toEqual({ endpoint: "0.0.0.0:8889", const_labels: { port: "8080", started: "2024-01-01" } });
+  });
+
+  test("resolves anchors and merge keys", () => {
+    const { config } = parseCollectorYaml(`
+exporters:
+  otlp/a: &base
+    endpoint: a:4317
+    tls: { insecure: true }
+  otlp/b:
+    <<: *base
+    endpoint: b:4317
+`);
+    expect(config.exporters?.["otlp/b"]).toEqual({ endpoint: "b:4317", tls: { insecure: true } });
+  });
+
+  test("reads the # chant: header back", () => {
+    const { pins, semconv, warnings } = parseCollectorYaml(
+      [
+        "# chant: exporter splunk_hec/security schema github.com/open-telemetry/opentelemetry-collector-contrib/exporter/splunkhecexporter@v0.130.0",
+        "# chant: processor acme schema @acme/otel-components@1.2.3 sha256:deadbeef",
+        "# chant: semconv gen_ai github.com/open-telemetry/semantic-conventions@v1.41.1 (transform, redaction)",
+        "receivers: {}",
+      ].join("\n"),
+    );
+    expect(pins).toEqual([
+      {
+        kind: "exporter",
+        id: "splunk_hec/security",
+        pin: { source: "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/splunkhecexporter", version: "v0.130.0" },
+      },
+      { kind: "processor", id: "acme", pin: { source: "@acme/otel-components", version: "1.2.3", digest: "sha256:deadbeef" } },
+    ]);
+    expect(semconv).toEqual([{ namespace: "gen_ai", pin: { source: "github.com/open-telemetry/semantic-conventions", version: "v1.41.1" } }]);
+    expect(warnings).toEqual([]);
+  });
+
+  test("only the leading comment block is a header, and an unpinned line pins nothing", () => {
+    const { pins } = parseCollectorYaml(
+      [
+        "# chant: exporter x schema (unpinned)@(unpinned)",
+        "receivers: {}",
+        "# chant: exporter y schema src@v1",
+      ].join("\n"),
+    );
+    expect(pins).toEqual([]);
+  });
+
+  test("names what it cannot carry", () => {
+    const { config, warnings } = parseCollectorYaml(
+      [
+        "# chant: semconv gen_ai github.com/open-telemetry/semantic-conventions@v1.30.0 (x)",
+        "# chant: something else",
+        "receivers:",
+        "  thing:",
+        "    name: dropped",
+        "    other: kept",
+        "  bad id: {}",
+        "exporters:",
+        "  debug: 3",
+        "connectors: []",
+        "extras: {}",
+        "service:",
+        "  pipelines:",
+        "    profiles:",
+        "      receivers: [thing]",
+        "      exporters: [debug]",
+        "      extra: [x]",
+        "  unknown: true",
+      ].join("\n"),
+    );
+    expect(warnings).toEqual([
+      "the config was built against gen_ai semantic conventions github.com/open-telemetry/semantic-conventions@v1.30.0; this lexicon follows github.com/open-telemetry/semantic-conventions@v1.41.1, which the rebuilt config will name",
+      'header line "# chant: something else" is not a schema pin chant recognises; it is not carried',
+      'receivers.thing has a config key "name", which chant uses for the instance name; the key is not carried',
+      'receivers: "bad id" is not a type[/name] component id',
+      "exporters.debug is not a mapping; it is carried as an empty config",
+      '"connectors" is not a mapping; it is not carried',
+      'top-level section "extras" is not part of a collector config; it is not carried',
+      'pipeline "profiles" carries signal "profiles", which Pipeline does not type; it is imported with a type suppression',
+      "service.pipelines.profiles.extra is not carried; a Pipeline has receivers, processors and exporters",
+      "service.unknown is not carried; chant's Service declares extensions and telemetry",
+    ]);
+    expect(config.receivers?.thing).toEqual({ name: "dropped", other: "kept" });
+  });
+
+  test("an empty file is an empty config; a non-mapping is an error", () => {
+    expect(parseCollectorYaml("").config).toEqual({});
+    expect(parseCollectorYaml("# just a comment\n").config).toEqual({});
+    expect(() => parseCollectorYaml("- a\n- b\n")).toThrow(/is a YAML mapping/);
+  });
+});
+
+describe("OtelCollectorParser", () => {
+  test("carries the whole config as one OTel::Collector resource, with the header in metadata", () => {
+    const ir = parser.parse(
+      "# chant: exporter x/y schema src@v1\nexporters:\n  x/y: {}\nservice:\n  pipelines:\n    logs:\n      receivers: []\n      exporters: [x/y]\n",
+    );
+    expect(ir.resources).toHaveLength(1);
+    const [r] = ir.resources;
+    expect(r.type).toBe(COLLECTOR_RESOURCE_TYPE);
+    expect(r.logicalId).toBe("collector");
+    expect(r.properties).toEqual({
+      config: { exporters: { "x/y": {} }, service: { pipelines: { logs: { receivers: [], exporters: ["x/y"] } } } },
+    });
+    expect(r.metadata).toEqual({ pins: [{ kind: "exporter", id: "x/y", pin: { source: "src", version: "v1" } }], semconv: [] });
+    expect(ir.parameters).toEqual([]);
+    expect(ir.warnings).toEqual([]);
+  });
+});

package/src/import/parser.ts

@@ -0,0 +1,236 @@
+/**
+ * Collector YAML -> `TemplateIR`, for `chant import`.
+ *
+ * A collector config is one document whose parts name each other: pipelines
+ * list component ids, a connector sits in two pipelines, `service.extensions`
+ * lists extension ids. So the IR carries the whole config as one resource of
+ * type `OTel::Collector`, and the generator lays out the modules itself, with
+ * those names turned into imports of declared entities. (Core splits an IR
+ * with more than three resources into one generate() call per category and
+ * keeps only the first file of each, which would lose the imports between a
+ * pipeline and the components it references.)
+ *
+ * The config is carried as parsed, with nothing dropped from a component's
+ * config. What the lexicon cannot express is named in `warnings`.
+ *
+ * The `# chant:` lines the serializer writes above a config are read back:
+ * a custom component's schema pin becomes the pin of the `defineComponent`
+ * the generator writes for it. A config without them imports against
+ * `COLLECTOR_PIN`.
+ */
+
+import * as jsYaml from "js-yaml";
+import type { TemplateIR, TemplateParser } from "@intentius/chant/import/parser";
+import { GENAI_SEMCONV_PIN, type SchemaPin } from "../define";
+import {
+  COMPONENT_KINDS,
+  SECTION_OF,
+  SIGNALS,
+  parseComponentId,
+  pipelineSignal,
+  type CollectorConfig,
+  type ComponentKind,
+} from "../model";
+
+/** The IR resource type for a whole collector config. */
+export const COLLECTOR_RESOURCE_TYPE = "OTel::Collector";
+
+/** One `# chant: <kind> <id> schema <source>@<version>` header line, read back. */
+export interface HeaderPin {
+  kind: ComponentKind;
+  id: string;
+  pin: SchemaPin;
+}
+
+/** One `# chant: semconv <namespace> <source>@<version>` header line, read back. */
+export interface HeaderSemconv {
+  namespace: string;
+  pin: SchemaPin;
+}
+
+/** What the parser read from a collector YAML file. */
+export interface ParsedCollector {
+  config: CollectorConfig;
+  pins: HeaderPin[];
+  semconv: HeaderSemconv[];
+  warnings: string[];
+}
+
+/** `properties` of the `OTel::Collector` resource. */
+export interface CollectorResourceProperties {
+  config: CollectorConfig;
+}
+
+/** `metadata` of the `OTel::Collector` resource. */
+export interface CollectorResourceMetadata {
+  pins: HeaderPin[];
+  semconv: HeaderSemconv[];
+}
+
+// YAML 1.2 core types plus `<<` merge keys. The default schema would also
+// turn an unquoted date into a Date, which the collector never does.
+// js-yaml exports its built-in types at runtime; @types/js-yaml does not declare them.
+const MERGE_TYPE = (jsYaml as unknown as { types: { merge: jsYaml.Type } }).types.merge;
+const COLLECTOR_YAML_SCHEMA = jsYaml.CORE_SCHEMA.extend({ implicit: [MERGE_TYPE] });
+
+const SECTIONS = new Set<string>(["receivers", "processors", "exporters", "connectors", "extensions", "service"]);
+const SERVICE_KEYS = new Set(["extensions", "telemetry", "pipelines"]);
+const PIPELINE_KEYS = new Set(["receivers", "processors", "exporters"]);
+
+const PIN_LINE = /^#\s*chant:\s*(receiver|processor|exporter|connector|extension)\s+(\S+)\s+schema\s+(\S+)@([^@\s]+)(?:\s+(\S+))?\s*$/;
+const SEMCONV_LINE = /^#\s*chant:\s*semconv\s+(\S+)\s+(\S+)@([^@\s]+)(?:\s+([^\s(]+))?(?:\s+\(.*\))?\s*$/;
+
+function isPlainObject(v: unknown): v is Record<string, unknown> {
+  return typeof v === "object" && v !== null && !Array.isArray(v);
+}
+
+/** The `# chant:` lines of the comment block at the top of the file. */
+function readHeader(content: string, warnings: string[]): { pins: HeaderPin[]; semconv: HeaderSemconv[] } {
+  const pins: HeaderPin[] = [];
+  const semconv: HeaderSemconv[] = [];
+  for (const raw of content.split(/\r?\n/)) {
+    const line = raw.trim();
+    if (line === "") continue;
+    if (!line.startsWith("#")) break;
+    if (!/^#\s*chant:/.test(line)) continue;
+    const pin = PIN_LINE.exec(line);
+    if (pin) {
+      const [, kind, id, source, version, digest] = pin;
+      if (source === "(unpinned)" || version === "(unpinned)") continue;
+      pins.push({ kind: kind as ComponentKind, id, pin: digest ? { source, version, digest } : { source, version } });
+      continue;
+    }
+    const sc = SEMCONV_LINE.exec(line);
+    if (sc) {
+      const [, namespace, source, version, digest] = sc;
+      semconv.push({ namespace, pin: digest ? { source, version, digest } : { source, version } });
+      if (namespace === "gen_ai" && (source !== GENAI_SEMCONV_PIN.source || version !== GENAI_SEMCONV_PIN.version)) {
+        warnings.push(
+          `the config was built against ${namespace} semantic conventions ${source}@${version}; ` +
+            `this lexicon follows ${GENAI_SEMCONV_PIN.source}@${GENAI_SEMCONV_PIN.version}, which the rebuilt config will name`,
+        );
+      }
+      continue;
+    }
+    warnings.push(`header line "${line}" is not a schema pin chant recognises; it is not carried`);
+  }
+  return { pins, semconv };
+}
+
+/** Parse collector YAML into the plain config model, the header pins and what cannot be carried. */
+export function parseCollectorYaml(content: string): ParsedCollector {
+  const warnings: string[] = [];
+  const { pins, semconv } = readHeader(content, warnings);
+  const doc = content.trim() === "" ? {} : jsYaml.load(content, { schema: COLLECTOR_YAML_SCHEMA });
+  if (doc === null || doc === undefined) return { config: {}, pins, semconv, warnings };
+  if (!isPlainObject(doc)) {
+    throw new Error("an OpenTelemetry Collector config is a YAML mapping; this document is not one");
+  }
+
+  const config: CollectorConfig = {};
+  for (const [key, value] of Object.entries(doc)) {
+    if (!SECTIONS.has(key)) {
+      warnings.push(`top-level section "${key}" is not part of a collector config; it is not carried`);
+      continue;
+    }
+    if (value === null || value === undefined) continue;
+    if (!isPlainObject(value)) {
+      warnings.push(`"${key}" is not a mapping; it is not carried`);
+      continue;
+    }
+    if (key === "service") continue;
+    const section: Record<string, Record<string, unknown> | null> = {};
+    for (const [id, cfg] of Object.entries(value)) {
+      if (!parseComponentId(id)) warnings.push(`${key}: "${id}" is not a type[/name] component id`);
+      if (cfg !== null && cfg !== undefined && !isPlainObject(cfg)) {
+        warnings.push(`${key}.${id} is not a mapping; it is carried as an empty config`);
+        section[id] = null;
+        continue;
+      }
+      if (isPlainObject(cfg) && "name" in cfg) {
+        warnings.push(
+          `${key}.${id} has a config key "name", which chant uses for the instance name; the key is not carried`,
+        );
+      }
+      section[id] = (cfg as Record<string, unknown> | null | undefined) ?? null;
+    }
+    (config as Record<string, unknown>)[key] = section;
+  }
+
+  const service = (doc as Record<string, unknown>).service;
+  if (isPlainObject(service)) {
+    config.service = {};
+    for (const [key, value] of Object.entries(service)) {
+      if (!SERVICE_KEYS.has(key)) {
+        warnings.push(`service.${key} is not carried; chant's Service declares extensions and telemetry`);
+        continue;
+      }
+      if (value === null || value === undefined) continue;
+      if (key === "extensions") {
+        if (!Array.isArray(value)) warnings.push("service.extensions is not a list; it is not carried");
+        else config.service.extensions = value.map(String);
+      } else if (key === "telemetry") {
+        if (!isPlainObject(value)) warnings.push("service.telemetry is not a mapping; it is not carried");
+        else config.service.telemetry = value;
+      } else if (key === "pipelines") {
+        if (!isPlainObject(value)) {
+          warnings.push("service.pipelines is not a mapping; it is not carried");
+          continue;
+        }
+        config.service.pipelines = {};
+        for (const [pid, p] of Object.entries(value)) {
+          const signal = pipelineSignal(pid);
+          if (!(SIGNALS as readonly string[]).includes(signal)) {
+            warnings.push(
+              `pipeline "${pid}" carries signal "${signal}", which Pipeline does not type; it is imported with a type suppression`,
+            );
+          }
+          const entry: Record<string, string[]> = {};
+          if (isPlainObject(p)) {
+            for (const [k, list] of Object.entries(p)) {
+              if (!PIPELINE_KEYS.has(k)) {
+                warnings.push(`service.pipelines.${pid}.${k} is not carried; a Pipeline has receivers, processors and exporters`);
+                continue;
+              }
+              if (list === null || list === undefined) continue;
+              if (!Array.isArray(list)) {
+                warnings.push(`service.pipelines.${pid}.${k} is not a list; it is not carried`);
+                continue;
+              }
+              entry[k] = list.map(String);
+            }
+          }
+          config.service.pipelines[pid] = entry;
+        }
+      }
+    }
+  }
+
+  return { config, pins, semconv, warnings };
+}
+
+/** Which kind a section's components are. */
+export const KIND_OF_SECTION: Record<string, ComponentKind> = Object.fromEntries(
+  COMPONENT_KINDS.map((k) => [SECTION_OF[k], k]),
+);
+
+/** The OpenTelemetry Collector config parser `chant import` runs. */
+export class OtelCollectorParser implements TemplateParser {
+  parse(content: string): TemplateIR {
+    const parsed = parseCollectorYaml(content);
+    const properties: CollectorResourceProperties = { config: parsed.config };
+    const metadata: CollectorResourceMetadata = { pins: parsed.pins, semconv: parsed.semconv };
+    return {
+      resources: [
+        {
+          logicalId: "collector",
+          type: COLLECTOR_RESOURCE_TYPE,
+          properties: properties as unknown as Record<string, unknown>,
+          metadata: metadata as unknown as Record<string, unknown>,
+        },
+      ],
+      parameters: [],
+      warnings: parsed.warnings,
+    };
+  }
+}