@intentius/chant-lexicon-otel 0.97.0 → 0.99.0

package/src/codegen/docs.ts

@@ -44,7 +44,8 @@ RED and token metrics.
 Their config types follow the collector-contrib release named on the
 Custom components page. A component chant doesn't ship is added with
 \`defineComponent\`, and it is serialized and checked the same way as the
-built-ins.
+built-ins. \`chant import\` turns an existing collector config file into this
+TypeScript (see Importing a Collector Config).
 
 Checks catch a pipeline that uses a component nobody declared (OTEL101), a
 pipeline with no receivers or exporters (OTEL102), a declared component no

package/src/components/common.ts

@@ -5,8 +5,13 @@
  * unchanged.
  */
 
-/** A duration string the collector parses, e.g. `5s`, `250ms`, `1m30s`. */
-export type Duration = string;
+/**
+ * A duration the collector parses: a string such as `5s`, `250ms` or
+ * `1m30s`, or a bare integer, which the collector reads as nanoseconds
+ * (collector-contrib's own servicegraph test config writes
+ * `latency_histogram_buckets: [1,2,3,4,5]`). Prefer the string form.
+ */
+export type Duration = string | number;
 
 export interface TLSClientSettings {
   insecure?: boolean;

package/src/components/filtering.test.ts

@@ -81,6 +81,33 @@ describe("filter processor", () => {
     );
     expect(problems(new FilterProcessor({ logs: { log_record: ["true"] } }))).toEqual([]);
   });
+
+  test("the legacy include/exclude match blocks serialize as written and count as conditions", () => {
+    const filter = new FilterProcessor({
+      metrics: { exclude: { match_type: "strict", metric_names: ["up", "scrape_duration_seconds"] } },
+      logs: { include: { match_type: "regexp", severity_number: { min: "WARN", match_undefined: true }, bodies: ["^panic"] } },
+      spans: { exclude: { match_type: "strict", services: ["healthcheck"], span_kinds: ["SPAN_KIND_INTERNAL"] } },
+    });
+    expect(problems(filter)).toEqual([]);
+    const parsed = load(primary(otelSerializer.serialize(entities({ filter })))) as any;
+    expect(parsed.processors.filter.metrics.exclude.metric_names).toEqual(["up", "scrape_duration_seconds"]);
+    expect(parsed.processors.filter.logs.include.severity_number).toEqual({ min: "WARN", match_undefined: true });
+    expect(parsed.processors.filter.spans.exclude.span_kinds).toEqual(["SPAN_KIND_INTERNAL"]);
+  });
+
+  test("a signal that mixes OTTL conditions with include/exclude is reported, as the collector rejects it", () => {
+    const mixed = new FilterProcessor({
+      traces: { span: ["true"] },
+      spans: { include: { match_type: "strict", services: ["a"] } },
+      metrics: { metric: ["true"], exclude: { match_type: "expr", expressions: ["MetricName == 'up'"] } },
+      logs: { log_record: ["true"], exclude: { match_type: "strict", bodies: ["x"] } },
+    });
+    expect(problems(mixed)).toEqual([
+      "traces mixes OTTL conditions with include/exclude, which the collector rejects",
+      "metrics mixes OTTL conditions with include/exclude, which the collector rejects",
+      "logs mixes OTTL conditions with include/exclude, which the collector rejects",
+    ]);
+  });
 });
 
 describe("transform processor", () => {

package/src/components/filtering.ts

@@ -8,13 +8,56 @@
  */
 
 import { defineBuiltin } from "../define";
+import type { AttributesMatch } from "./processors";
 
 /** What an OTTL processor does when a condition or statement errors at run time. */
 export type OttlErrorMode = "ignore" | "silent" | "propagate";
 
 // ── filter ───────────────────────────────────────────────────────────
 
-/** OTTL conditions per context. Telemetry matching any condition is dropped. */
+/** An attribute a legacy include/exclude block matches on (`filterconfig.Attribute`). */
+export interface FilterAttributeMatch {
+  key: string;
+  value?: unknown;
+}
+
+/** Options for the `regexp` match type (`filterset/regexp.Config`). */
+export interface FilterRegexpSettings {
+  cacheenabled?: boolean;
+  cachemaxnumentries?: number;
+}
+
+/**
+ * The filter processor's legacy `metrics.include` / `metrics.exclude` block
+ * (`filterconfig.MetricMatchProperties`). `expr` takes `expressions` instead
+ * of `metric_names`.
+ */
+export interface FilterMetricMatch {
+  match_type: "strict" | "regexp" | "expr";
+  regexp?: FilterRegexpSettings;
+  metric_names?: string[];
+  expressions?: string[];
+  resource_attributes?: FilterAttributeMatch[];
+}
+
+/** The filter processor's legacy `logs.include` / `logs.exclude` block (`LogMatchProperties`). */
+export interface FilterLogMatch {
+  match_type: "strict" | "regexp";
+  resource_attributes?: FilterAttributeMatch[];
+  record_attributes?: FilterAttributeMatch[];
+  severity_texts?: string[];
+  /** Matches records at or above `min`, a severity number or name such as `INFO`. */
+  severity_number?: { min: number | string; match_undefined?: boolean };
+  bodies?: string[];
+}
+
+/**
+ * Conditions per signal. Telemetry matching any condition is dropped.
+ *
+ * Each signal takes OTTL conditions or the legacy `include` / `exclude`
+ * match blocks (for traces, the legacy blocks sit under `spans`), not both:
+ * the collector rejects a signal that mixes them.
+ */
 export interface FilterProcessorConfig {
   /** Default `propagate`: an erroring condition fails the batch. `ignore` logs and moves on. */
   error_mode?: OttlErrorMode;
@@ -24,15 +67,27 @@ export interface FilterProcessorConfig {
     /** Drops the span event. */
     spanevent?: string[];
   };
+  /** Legacy span matching. With `include`, spans that do not match are dropped; with `exclude`, spans that match. */
+  spans?: { include?: AttributesMatch; exclude?: AttributesMatch };
   metrics?: {
     /** Drops the whole metric, e.g. `name == "http.server.duration" and type == METRIC_DATA_TYPE_HISTOGRAM`. */
     metric?: string[];
     /** Drops the data point. */
     datapoint?: string[];
+    /** Legacy: keeps only the metrics that match. */
+    include?: FilterMetricMatch;
+    /** Legacy: drops the metrics that match. */
+    exclude?: FilterMetricMatch;
+    /** Legacy: options for the `regexp` match type. */
+    regexp?: FilterRegexpSettings;
   };
   logs?: {
     /** Drops the log record, e.g. `severity_number < SEVERITY_NUMBER_WARN`. */
     log_record?: string[];
+    /** Legacy: keeps only the log records that match. */
+    include?: FilterLogMatch;
+    /** Legacy: drops the log records that match. */
+    exclude?: FilterLogMatch;
   };
 }
 
@@ -55,7 +110,15 @@ export const FilterProcessor = defineBuiltin<FilterProcessorConfig, "processor",
         if (typeof cond !== "string" || cond.trim() === "") problems.push(`${path}[${i}] is an empty condition`);
       });
     }
-    if (lists.every(([, list]) => !list || list.length === 0)) {
+    const legacy: Array<[string, boolean, boolean]> = [
+      ["traces", !!(c.traces?.span || c.traces?.spanevent), !!(c.spans?.include || c.spans?.exclude)],
+      ["metrics", !!(c.metrics?.metric || c.metrics?.datapoint), !!(c.metrics?.include || c.metrics?.exclude)],
+      ["logs", !!c.logs?.log_record, !!(c.logs?.include || c.logs?.exclude)],
+    ];
+    for (const [signal, ottl, matchBlocks] of legacy) {
+      if (ottl && matchBlocks) problems.push(`${signal} mixes OTTL conditions with include/exclude, which the collector rejects`);
+    }
+    if (lists.every(([, list]) => !list || list.length === 0) && legacy.every(([, , matchBlocks]) => !matchBlocks)) {
       problems.push("no condition is set for any signal, so the processor drops nothing");
     }
     return problems;

package/src/components/processors.ts

@@ -73,12 +73,16 @@ export const ResourceProcessor = defineBuiltin<ResourceProcessorConfig, "process
 
 // ── attributes ───────────────────────────────────────────────────────
 
+/** An include/exclude block of the attributes processor and the filter processor's legacy `spans` (`filterconfig.MatchProperties`). */
 export interface AttributesMatch {
   match_type: "strict" | "regexp";
+  regexp?: { cacheenabled?: boolean; cachemaxnumentries?: number };
   services?: string[];
   span_names?: string[];
+  span_kinds?: string[];
   log_bodies?: string[];
   log_severity_texts?: string[];
+  log_severity_number?: { min: number | string; match_undefined?: boolean };
   metric_names?: string[];
   attributes?: Array<{ key: string; value?: unknown }>;
   resources?: Array<{ key: string; value?: unknown }>;

package/src/components/receivers.ts

@@ -44,25 +44,109 @@ export interface PrometheusStaticConfig {
   labels?: Record<string, string>;
 }
 
-/** One Prometheus scrape job. Service-discovery and relabel blocks pass through as written. */
-export interface PrometheusScrapeConfig {
+/** Prometheus's `tls_config` (prometheus/common `TLSConfig`). Versions are spelled `TLS12`, `TLS13`. */
+export interface PrometheusTLSConfig {
+  ca?: string;
+  cert?: string;
+  key?: string;
+  ca_file?: string;
+  cert_file?: string;
+  key_file?: string;
+  ca_ref?: string;
+  cert_ref?: string;
+  key_ref?: string;
+  server_name?: string;
+  insecure_skip_verify?: boolean;
+  min_version?: string;
+  max_version?: string;
+}
+
+/** Prometheus's proxy settings, inlined into the HTTP client and OAuth2 blocks. */
+export interface PrometheusProxySettings {
+  proxy_url?: string;
+  no_proxy?: string;
+  proxy_from_environment?: boolean;
+  proxy_connect_header?: Record<string, string[]>;
+}
+
+/** Scrape authentication with a user name and password. Prefer `password_file` or `${env:...}` to a literal password. */
+export interface PrometheusBasicAuth {
+  username?: string;
+  username_file?: string;
+  username_ref?: string;
+  password?: string;
+  password_file?: string;
+  password_ref?: string;
+}
+
+/** An `Authorization` header: `type` defaults to `Bearer`. */
+export interface PrometheusAuthorization {
+  type?: string;
+  credentials?: string;
+  credentials_file?: string;
+  credentials_ref?: string;
+}
+
+export interface PrometheusOAuth2 extends PrometheusProxySettings {
+  client_id: string;
+  client_secret?: string;
+  client_secret_file?: string;
+  client_secret_ref?: string;
+  scopes?: string[];
+  token_url: string;
+  endpoint_params?: Record<string, string>;
+  tls_config?: PrometheusTLSConfig;
+}
+
+/**
+ * One Prometheus scrape job, as the Prometheus release collector-contrib
+ * v0.130.0 embeds reads it (scrape settings plus prometheus/common's HTTP
+ * client settings). Service-discovery (`*_sd_configs`) and relabel blocks
+ * pass through as written.
+ */
+export interface PrometheusScrapeConfig extends PrometheusProxySettings {
   job_name: string;
   scrape_interval?: Duration;
   scrape_timeout?: Duration;
+  scrape_protocols?: string[];
+  fallback_scrape_protocol?: string;
+  always_scrape_classic_histograms?: boolean;
+  convert_classic_histograms_to_nhcb?: boolean;
+  scrape_failure_log_file?: string;
   metrics_path?: string;
   scheme?: "http" | "https";
   honor_labels?: boolean;
   honor_timestamps?: boolean;
+  track_timestamps_staleness?: boolean;
+  enable_compression?: boolean;
   params?: Record<string, string[]>;
   static_configs?: PrometheusStaticConfig[];
   kubernetes_sd_configs?: Array<Record<string, unknown>>;
   file_sd_configs?: Array<Record<string, unknown>>;
+  /** Any other service-discovery block (`dns_sd_configs`, `ec2_sd_configs`, `consul_sd_configs`, ...). */
+  [sd: `${string}_sd_configs`]: Array<Record<string, unknown>> | undefined;
   relabel_configs?: Array<Record<string, unknown>>;
   metric_relabel_configs?: Array<Record<string, unknown>>;
-  tls_config?: Record<string, unknown>;
-  authorization?: { type?: string; credentials_file?: string };
+  basic_auth?: PrometheusBasicAuth;
+  authorization?: PrometheusAuthorization;
+  oauth2?: PrometheusOAuth2;
+  bearer_token?: string;
   bearer_token_file?: string;
+  tls_config?: PrometheusTLSConfig;
+  follow_redirects?: boolean;
+  enable_http2?: boolean;
+  http_headers?: Record<string, { values?: string[]; secrets?: string[]; files?: string[] }>;
+  body_size_limit?: string;
   sample_limit?: number;
+  target_limit?: number;
+  label_limit?: number;
+  label_name_length_limit?: number;
+  label_value_length_limit?: number;
+  native_histogram_bucket_limit?: number;
+  native_histogram_min_bucket_factor?: number;
+  keep_dropped_targets?: number;
+  metric_name_validation_scheme?: "utf8" | "legacy";
+  metric_name_escaping_scheme?: "allow-utf-8" | "underscores" | "dots" | "values";
 }
 
 export interface PrometheusReceiverConfig {

package/src/import/embedded.test.ts

@@ -0,0 +1,56 @@
+import { describe, expect, test } from "vitest";
+import { embeddedDocument, type EmbeddedContent } from "@intentius/chant/import/embedded";
+import { otelPlugin } from "../plugin";
+import { collectorConfigImporter } from "./embedded";
+
+const CONFIG = `receivers:
+  otlp:
+    protocols:
+      grpc: {}
+  mycorp: {}
+exporters:
+  debug: {}
+service:
+  pipelines:
+    traces:
+      receivers: [otlp, mycorp]
+      exporters: [debug]
+`;
+
+const site = (text: string, over: Partial<EmbeddedContent> = {}): EmbeddedContent => ({
+  host: "k8s",
+  hostType: "K8s::Core::ConfigMap",
+  location: 'ConfigMap agent data["config.yaml"]',
+  directory: "agent",
+  text,
+  document: embeddedDocument(text),
+  ...over,
+});
+
+describe("a collector config embedded in another lexicon's resource (#2962)", () => {
+  test("the plugin registers the importer", () => {
+    expect(otelPlugin.embeddedImporters?.()).toEqual([collectorConfigImporter]);
+  });
+
+  test("matches collector configs held as text, nothing else", () => {
+    expect(collectorConfigImporter.matches(site(CONFIG))).toBe(true);
+    expect(collectorConfigImporter.matches(site("groups: []\n"))).toBe(false);
+    expect(collectorConfigImporter.matches(site(CONFIG, { select: "config" }))).toBe(false);
+  });
+
+  test("imports the config as `chant import` does, referenced through collectorYaml", () => {
+    const out = collectorConfigImporter.import(site(CONFIG));
+    expect(out.files.map((f) => f.path)).toEqual(["custom-components.ts", "receivers.ts", "exporters.ts", "pipelines.ts"]);
+    // Every declared component and pipeline, not the custom component's class.
+    expect(out.value).toEqual({
+      bindings: [
+        { from: "receivers.ts", name: "otlp" },
+        { from: "receivers.ts", name: "mycorp" },
+        { from: "exporters.ts", name: "debug" },
+        { from: "pipelines.ts", name: "traces" },
+      ],
+      shape: "list",
+      through: { from: "@intentius/chant-lexicon-otel", name: "collectorYaml" },
+    });
+  });
+});

package/src/import/embedded.ts

@@ -0,0 +1,41 @@
+/**
+ * A collector config embedded in another lexicon's resource, e.g. the
+ * `config.yaml` of a k8s ConfigMap, for `chant import` (#2962).
+ *
+ * The config is imported exactly as `chant import config.yaml` would import
+ * it, into a directory of its own, and the host's value becomes
+ * `collectorYaml([...])` over every component, pipeline and service the
+ * import declares, which renders the same config text at build time.
+ */
+
+import type { EmbeddedContentImporter, EmbeddedImport } from "@intentius/chant/import/embedded";
+import { exportedNames } from "@intentius/chant/import/embedded";
+import { looksLikeCollectorConfig } from "../model";
+import { OtelCollectorParser } from "./parser";
+import { OtelCollectorGenerator } from "./generator";
+
+const PACKAGE = "@intentius/chant-lexicon-otel";
+
+/** The module holding `defineComponent` classes: types, not config entities. */
+const CUSTOM_COMPONENTS = "custom-components.ts";
+
+export const collectorConfigImporter: EmbeddedContentImporter = {
+  what: "an OpenTelemetry Collector config",
+
+  matches(content) {
+    return typeof content.text === "string" && content.select === undefined && looksLikeCollectorConfig(content.document);
+  },
+
+  import(content): EmbeddedImport {
+    const ir = new OtelCollectorParser().parse(content.text!);
+    const files = new OtelCollectorGenerator().generate(ir);
+    const bindings = files
+      .filter((f) => f.path !== CUSTOM_COMPONENTS)
+      .flatMap((f) => exportedNames(f.content).map((name) => ({ from: f.path, name })));
+    return {
+      files,
+      value: { bindings, shape: "list", through: { from: PACKAGE, name: "collectorYaml" } },
+      warnings: ir.warnings ?? [],
+    };
+  },
+};

package/src/import/generated-types.e2e.test.ts

@@ -0,0 +1,110 @@
+/**
+ * The generated source type-checks against the lexicon's config types.
+ * Compiling a dozen generated projects takes longer than the unit-test
+ * budget allows, so this runs with the e2e tests.
+ */
+import { describe, expect, test } from "vitest";
+import { mkdirSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from "fs";
+import { join } from "path";
+import * as ts from "typescript";
+import { collectorYaml } from "../collector";
+import { genAiPipeline } from "../genai";
+import { OtelCollectorParser } from "./parser";
+import { OtelCollectorGenerator } from "./generator";
+import { everyBuiltin, exampleOutputs, pkgDir, read, repoRoot } from "./testdata/fixtures";
+
+/** Type errors in generated projects, compiled together against the lexicon's source with the repo's compiler options. */
+function typeErrors(projects: Record<string, Array<{ path: string; content: string }>>): string[] {
+  const dir = mkdtempSync(join(pkgDir, ".roundtrip-tmp-"));
+  try {
+    const files: string[] = [];
+    for (const [name, generated] of Object.entries(projects)) {
+      const sub = join(dir, name.replace(/[^A-Za-z0-9]+/g, "-"));
+      mkdirSync(sub);
+      for (const f of generated) {
+        writeFileSync(join(sub, f.path), f.content);
+        files.push(join(sub, f.path));
+      }
+    }
+    const configFile = ts.readConfigFile(join(repoRoot, "tsconfig.json"), ts.sys.readFile);
+    const parsed = ts.parseJsonConfigFileContent(configFile.config, ts.sys, repoRoot);
+    const program = ts.createProgram(files, { ...parsed.options, noEmit: true });
+    const real = (f: string) => ts.sys.realpath?.(f) ?? f;
+    const fileSet = new Set(files.map(real));
+    const diagnostics = ts.getPreEmitDiagnostics(program).filter((d) => d.file && fileSet.has(real(d.file.fileName)));
+    // A program that saw none of the files would pass vacuously.
+    expect(files.every((f) => program.getSourceFile(f) !== undefined)).toBe(true);
+    return diagnostics.map((d) => {
+      const at = real(d.file!.fileName).slice(real(dir).length + 1);
+      return `${at}: ${ts.flattenDiagnosticMessageText(d.messageText, "\n")}`;
+    });
+  } finally {
+    rmSync(dir, { recursive: true, force: true });
+  }
+}
+
+const generate = (yaml: string) => new OtelCollectorGenerator().generate(new OtelCollectorParser().parse(yaml));
+
+/** Every vendored import fixture, by path under testdata/: the chant-built configs and the collector-contrib ones. */
+function vendoredFixtures(): string[] {
+  const dir = join(import.meta.dirname, "testdata");
+  const yamlIn = (sub: string) =>
+    readdirSync(join(dir, sub))
+      .filter((f) => f.endsWith(".yaml"))
+      .sort()
+      .map((f) => (sub ? `${sub}/${f}` : f));
+  return [...yamlIn(""), ...yamlIn("upstream")];
+}
+
+describe("the generated source type-checks against the lexicon's config types", () => {
+  test("for every vendored fixture, every example's output, genAiPipeline() and every built-in's typed fields", async () => {
+    const fixtures = vendoredFixtures();
+    // The corpus the round-trip tests read; a missing directory would pass vacuously.
+    expect(fixtures).toEqual(expect.arrayContaining(["gateway.yaml", "upstream/couchbase.yaml", "upstream/servicegraph-nop.yaml"]));
+    expect(fixtures.length).toBeGreaterThanOrEqual(12);
+    const projects: Record<string, Array<{ path: string; content: string }>> = {
+      ...Object.fromEntries(fixtures.map((f) => [f, generate(read(...f.split("/")))])),
+      genai: generate(collectorYaml(genAiPipeline())),
+      everyBuiltin: generate(collectorYaml(everyBuiltin())),
+      ...Object.fromEntries((await exampleOutputs()).map(([n, y]) => [`example-${n}`, generate(y)])),
+    };
+    expect(typeErrors(projects)).toEqual([]);
+  }, 120_000);
+
+  test("a value outside a built-in's typed config is carried as found, and tsc points at it", () => {
+    // The type check is not vacuous: a batch timeout of `true` is neither a duration string nor
+    // nanoseconds, and a scrape job stays closed to keys Prometheus does not read.
+    const yaml = [
+      "receivers:",
+      "  otlp:",
+      "    protocols:",
+      "      grpc: {}",
+      "  prometheus:",
+      "    config:",
+      "      scrape_configs:",
+      "        - job_name: self",
+      "          static_configs: [{ targets: [\"localhost:8888\"] }]",
+      "          basic_auth: { username: u, password_file: /etc/p }",
+      "          scrape_everything: true",
+      "processors:",
+      "  batch:",
+      "    timeout: true",
+      "exporters:",
+      "  debug: {}",
+      "service:",
+      "  pipelines:",
+      "    traces:",
+      "      receivers: [otlp]",
+      "      processors: [batch]",
+      "      exporters: [debug]",
+      "    metrics:",
+      "      receivers: [prometheus]",
+      "      exporters: [debug]",
+      "",
+    ].join("\n");
+    expect(typeErrors({ bad: generate(yaml) })).toEqual([
+      expect.stringMatching(/^bad\/processors\.ts: Type 'true' is not assignable to type 'Duration/),
+      expect.stringMatching(/^bad\/receivers\.ts: .*'scrape_everything' does not exist in type 'PrometheusScrapeConfig'/),
+    ]);
+  }, 120_000);
+});