@intentius/chant-lexicon-otel 0.124.45 → 0.124.48

package/src/spec/spec.test.ts

@@ -0,0 +1,144 @@
+import { describe, expect, test } from "vitest";
+import { CONTRIB_MODULE, CORE_MODULE, locate, parseManifest, refsIn, resolveRef } from "./fetch";
+import { documentAlias, generateTypes, isDeprecated, propertyDocs } from "./parse";
+import {
+  OTEL_SPEC_PIN,
+  OTELCOL_CONTRIB_VERSION,
+  OTELCOL_K8S_VERSION,
+  assertPinnedSnapshot,
+  assertSnapshotVersions,
+  canonicalJson,
+  specDigest,
+} from "./pin";
+import { loadSnapshot } from "./snapshot";
+import { BUILTIN_SPECS } from "./builtins";
+import { BUILTIN_CATALOG } from "../catalog";
+
+const OTLP = `${CORE_MODULE}/receiver/otlpreceiver`;
+const TAIL = `${CONTRIB_MODULE}/processor/tailsamplingprocessor`;
+
+describe("manifests and refs", () => {
+  test("a builder manifest lists component modules by kind, without providers", () => {
+    const modules = parseManifest(`
+dist: { name: otelcol-contrib }
+receivers:
+  - gomod: go.opentelemetry.io/collector/receiver/otlpreceiver v0.162.0
+connectors:
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/connector/countconnector v0.162.0
+providers:
+  - gomod: go.opentelemetry.io/collector/confmap/provider/envprovider v1.48.0
+`);
+    expect(modules).toEqual([
+      { kind: "receiver", module: OTLP, version: "v0.162.0" },
+      { kind: "connector", module: `${CONTRIB_MODULE}/connector/countconnector`, version: "v0.162.0" },
+    ]);
+  });
+
+  test("each $ref form resolves to a package and a definition", () => {
+    expect(resolveRef(TAIL, "and_cfg")).toEqual({ doc: TAIL, def: "and_cfg" });
+    expect(resolveRef(TAIL, "./internal/metadata.config")).toEqual({ doc: `${TAIL}/internal/metadata`, def: "config" });
+    expect(resolveRef(OTLP, "/config/configgrpc.server_config")).toEqual({ doc: `${CORE_MODULE}/config/configgrpc`, def: "server_config" });
+    expect(resolveRef(TAIL, "/pkg/ottl.error_mode")).toEqual({ doc: `${CONTRIB_MODULE}/pkg/ottl`, def: "error_mode" });
+    expect(resolveRef(TAIL, "go.opentelemetry.io/collector/config/configopaque.string")).toEqual({
+      doc: `${CORE_MODULE}/config/configopaque`,
+      def: "string",
+    });
+    expect(refsIn({ a: { $ref: "x" }, b: [{ $ref: "y" }] })).toEqual(["x", "y"]);
+  });
+
+  test("core packages read from opentelemetry-collector, contrib ones from contrib", () => {
+    const tags = { core: "v0.162.0", contrib: "v0.162.0" };
+    expect(locate(OTLP, tags)).toMatchObject({ repo: "open-telemetry/opentelemetry-collector", path: "receiver/otlpreceiver" });
+    expect(locate(TAIL, tags)).toMatchObject({ repo: "open-telemetry/opentelemetry-collector-contrib", path: "processor/tailsamplingprocessor" });
+    expect(locate("go.opentelemetry.io/obi", tags)).toBeUndefined();
+  });
+});
+
+describe("schema to TypeScript", () => {
+  const docs = {
+    "example.com/c": {
+      $defs: {
+        node: { type: "object", properties: { next: { $ref: "node" }, level: { $ref: "go.opentelemetry.io/collector/config/configtelemetry.level" } } },
+      },
+      description: "Config of c.",
+      type: "object",
+      properties: {
+        timeout: { type: "string", format: "duration", description: "Timeout. Deprecated: use deadline." },
+        grpc: { "x-optional": true, $ref: "node" },
+        mode: { type: "string", enum: ["a", "b"] },
+        count: { type: "integer", "x-customType": "uint64" },
+        id: { type: "string", "x-customType": "go.opentelemetry.io/collector/component.ID" },
+        labels: { type: "object", additionalProperties: { type: "string" } },
+        missing: { $ref: "/nowhere.thing" },
+      },
+      allOf: [{ $ref: "go.opentelemetry.io/collector/config/confighttp.server_config" }],
+    },
+    "go.opentelemetry.io/collector/config/confighttp": { $defs: { server_config: { type: "object", properties: { endpoint: { type: "string" } } } } },
+    "go.opentelemetry.io/collector/config/configtelemetry": { $defs: { level: { type: "integer" } } },
+  };
+
+  test("definitions become named aliases; extensions map to TypeScript", () => {
+    const g = generateTypes(docs, ["example.com/c"]);
+    const root = g.roots.get("example.com/c")!;
+    expect(root).toBe("ExampleComCSchema");
+    expect(g.declarations).toContain(`export type ${root} = CollectorConfigConfighttpServerConfig & {`);
+    expect(g.declarations).toContain("timeout?: Duration;");
+    expect(g.declarations).toContain("@deprecated");
+    expect(g.declarations).toContain("grpc?: ExampleComCNode | null;");
+    expect(g.declarations).toContain('mode?: "a" | "b";');
+    expect(g.declarations).toContain("count?: number;");
+    expect(g.declarations).toContain("id?: string;");
+    expect(g.declarations).toContain("labels?: Record<string, string>;");
+    expect(g.declarations).toContain("missing?: unknown;");
+    // Recursive definitions refer to themselves by name.
+    expect(g.declarations).toContain("next?: ExampleComCNode;");
+    // configtelemetry.Level is read from its name, not its number.
+    expect(g.declarations).toContain('export type CollectorConfigConfigtelemetryLevel = "none" | "basic" | "normal" | "detailed";');
+    expect([...g.unresolved.keys()]).toEqual(["/nowhere.thing"]);
+  });
+
+  test("property docs carry descriptions, short types and deprecation, embedded structs flattened", () => {
+    const props = propertyDocs(docs, "example.com/c");
+    expect(props.endpoint).toEqual({ type: "string" });
+    expect(props.timeout).toMatchObject({ type: "duration", deprecated: true });
+    expect(props.grpc.properties?.level.type).toBe('"none" | "basic" | "normal" | "detailed"');
+    expect(props.labels.type).toBe("object");
+  });
+
+  test("deprecation is read from Go comment forms", () => {
+    expect(isDeprecated("Deprecated: use X")).toBe(true);
+    expect(isDeprecated("Size of cache. Deprecated [v0.130.0]: use Y")).toBe(true);
+    expect(isDeprecated("Not deprecated at all")).toBe(false);
+    expect(documentAlias(`${CONTRIB_MODULE}/pkg/ottl`)).toBe("PkgOttl");
+  });
+});
+
+describe("the committed snapshot", () => {
+  const snapshot = loadSnapshot();
+
+  test("matches the pin and the version constants", () => {
+    expect(snapshot.versions["otelcol-contrib"]).toBe(OTELCOL_CONTRIB_VERSION);
+    expect(snapshot.versions["otelcol-k8s"]).toBe(OTELCOL_K8S_VERSION);
+    expect(specDigest(snapshot)).toBe(OTEL_SPEC_PIN.digest);
+    expect(Object.keys(snapshot.schemas)).toHaveLength(OTEL_SPEC_PIN.documents);
+    expect(() => assertPinnedSnapshot(snapshot)).not.toThrow();
+  });
+
+  test("a changed snapshot or version refuses, and the accept env var warns instead", () => {
+    const edited = { ...snapshot, schemas: { ...snapshot.schemas, "example.com/x": {} } };
+    expect(() => assertPinnedSnapshot(edited, { env: {} })).toThrow(/does not match OTEL_SPEC_PIN/);
+    const warnings: string[] = [];
+    assertPinnedSnapshot(edited, { env: { CHANT_ACCEPT_OTEL_SPEC: "1" }, warn: (m) => warnings.push(m) });
+    expect(warnings[0]).toContain("documents: 134");
+    expect(() => assertSnapshotVersions({ "otelcol-contrib": "v0.1.0", "otelcol-k8s": OTELCOL_K8S_VERSION })).toThrow(/just snapshot/);
+  });
+
+  test("the digest ignores key order", () => {
+    expect(canonicalJson({ b: 1, a: [{ d: 2, c: 3 }] })).toBe('{"a":[{"c":3,"d":2}],"b":1}');
+  });
+
+  test("BUILTIN_SPECS lists the exported component classes", () => {
+    const fromCatalog = BUILTIN_CATALOG.filter((c) => c.type !== undefined).map((c) => `${c.className} ${c.kind} ${c.type}`);
+    expect(BUILTIN_SPECS.map((b) => `${b.className} ${b.kind} ${b.type}`).sort()).toEqual(fromCatalog.sort());
+  });
+});

package/src/validate-config.ts

@@ -45,7 +45,8 @@ export type CollectorIssueCode =
   | "OTEL124"
   | "OTEL125"
   | "OTEL126"
-  | "OTEL127";
+  | "OTEL127"
+  | "OTEL128";
 
 export interface CollectorIssue {
   code: CollectorIssueCode;
@@ -494,8 +495,8 @@ export interface Listener {
 
 /**
  * The listeners OTEL117 knows, by component type: the config key path to an
- * address, its default at `COLLECTOR_PIN` (collector and contrib v0.130.0,
- * each component's `factory.go`), and the transport. `present` means the
+ * address, its default (read from each component's `factory.go` at collector
+ * and contrib v0.130.0), and the transport. `present` means the
  * default applies only when the key's parent is in the config: an otlp or
  * jaeger protocol that isn't listed doesn't start. A type not listed here is
  * not checked, since its `endpoint` may be a server it connects to (the

package/src/validate.ts

@@ -38,6 +38,7 @@ export const REQUIRED_NAMES = [
   "PrometheusExporter",
   "GoogleCloudExporter",
   "LoadBalancingExporter",
+  "ClickHouseExporter",
   "SpanMetricsConnector",
   "ServiceGraphConnector",
   "RoutingConnector",

package/src/workspace-links.e2e.test.ts

@@ -0,0 +1,91 @@
+/**
+ * An otel member's clickhouse exporter linked to the sql member that declares
+ * its tables (#4397), end to end through `chant workspace check`. Both
+ * members are folded, never run: the sql member's tables and columns are its
+ * outputs (#3931), and the otel lexicon's `workspaceLinks` hook states a link
+ * to each table the exporter writes and each column its insert names.
+ * Renaming one of those tables in the sql member fails the check (WSP093) at
+ * the exporter's declaration.
+ */
+
+import { execFileSync } from "node:child_process";
+import { mkdirSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import { afterAll, describe, expect, test } from "vitest";
+import { runDeclarationChecks } from "@intentius/chant/workspace/checks";
+
+const scratch: string[] = [];
+afterAll(() => {
+  for (const d of scratch) rmSync(d, { recursive: true, force: true });
+});
+
+function repo(files: Record<string, string>): string {
+  const root = realpathSync(mkdtempSync(join(tmpdir(), "chant-4397-")));
+  scratch.push(root);
+  execFileSync("git", ["init", "-q"], { cwd: root });
+  for (const [path, text] of Object.entries(files)) {
+    mkdirSync(dirname(join(root, path)), { recursive: true });
+    writeFileSync(join(root, path), text);
+  }
+  return root;
+}
+
+const WORKSPACE = JSON.stringify(
+  {
+    name: "observability",
+    schema: 1,
+    members: [
+      { name: "warehouse", dir: "warehouse", kind: "chant" },
+      { name: "collector", dir: "collector", kind: "chant" },
+    ],
+  },
+  null,
+  2,
+);
+
+const COLLECTOR = [
+  `import { ClickHouseExporter, OtlpReceiver, Pipeline } from "@intentius/chant-lexicon-otel";`,
+  ``,
+  `export const otlp = new OtlpReceiver({ protocols: { grpc: { endpoint: "0.0.0.0:4317" } } });`,
+  ``,
+  `export const clickhouse = new ClickHouseExporter({ endpoint: "tcp://clickhouse:9000", database: "otel", create_schema: false });`,
+  ``,
+  `export const logs = new Pipeline({ signal: "logs", receivers: [otlp], exporters: [clickhouse] });`,
+  `export const traces = new Pipeline({ signal: "traces", receivers: [otlp], exporters: [clickhouse] });`,
+  `export const metrics = new Pipeline({ signal: "metrics", receivers: [otlp], exporters: [clickhouse] });`,
+  ``,
+].join("\n");
+
+const warehouse = (tables: string) =>
+  [`import { OtelTables } from "@intentius/chant-lexicon-sql/clickhouse";`, ``, `export const otel = OtelTables({ database: "otel"${tables} });`, ``].join("\n");
+
+function observability(tables = ""): string {
+  return repo({
+    "chant.workspace.json": WORKSPACE,
+    "warehouse/chant.config.ts": `export default { lexicons: ["sql"], sql: { dialect: "clickhouse" } };\n`,
+    "warehouse/src/otel.ts": warehouse(tables),
+    "collector/chant.config.ts": `export default { lexicons: ["otel"] };\n`,
+    "collector/src/collector.ts": COLLECTOR,
+  });
+}
+
+describe("an otel member's clickhouse exporter links to the sql member's tables (#4397)", () => {
+  test("every table and column the exporter writes resolves against OtelTables", async () => {
+    const report = await runDeclarationChecks(observability());
+    expect(report.diagnostics.filter((d) => d.ruleId.startsWith("WSP09"))).toEqual([]);
+    const rows = report.links.filter((r) => r.consumer === "collector");
+    expect(rows.length).toBeGreaterThan(7);
+    expect(rows.every((r) => r.status === "resolved" && r.producer === "warehouse")).toBe(true);
+    expect(rows.map((r) => ("output" in r ? r.output : undefined))).toEqual(expect.arrayContaining(["otel.otel_logs", "otel.otel_traces.Events", "otel.otel_metrics_exponential_histogram.Scale"]));
+  });
+
+  test("renaming a table the exporter writes fails workspace check at the exporter", async () => {
+    const report = await runDeclarationChecks(observability(`, tables: { logs: "otel_logs_v2" }`));
+    const missing = report.diagnostics.filter((d) => d.ruleId === "WSP093");
+    expect(missing.length).toBeGreaterThan(0);
+    expect(missing.every((d) => d.file === "collector/src/collector.ts" && d.line === 5)).toBe(true);
+    expect(missing[0].message).toMatch(/collector's link to warehouse output otel\.otel_logs.* \(read by lexicon otel from collector\/src\/collector\.ts:5\) does not resolve/);
+    expect(report.ok).toBe(false);
+  });
+});

package/src/workspace-links.ts

@@ -0,0 +1,50 @@
+/**
+ * The workspace links an otel member's declarations state (#4397): each
+ * table a `clickhouse` exporter writes, and each column its insert names, as
+ * a link to the sql member that declares them (#3931).
+ *
+ * | Link | Output |
+ * |---|---|
+ * | the table | `<database>.<table>` |
+ * | a column | `<database>.<table>.<column>`, anchored on the table |
+ *
+ * The exporter's config does not name the sql member, so each link is
+ * matched to the one member that exposes the table (`resolveLinks` in
+ * chant's `workspace/links.ts`). `chant workspace check` then fails (WSP093)
+ * when a table the exporter writes, or a column its insert names, is renamed
+ * or dropped in the sql member. The tables and columns come from
+ * `clickhouse-tables.ts`, which OTEL129 reads in one build root.
+ *
+ * A config value chant can't read without the environment, such as a
+ * database set from `${env:CH_DB}`, is a gap (WSP099), and the tables it
+ * names are not linked.
+ */
+
+import type { Declarable } from "@intentius/chant/declarable";
+import type { LexiconWorkspaceLink, LexiconWorkspaceLinks } from "@intentius/chant/graph-ir";
+import { clickhouseExportersOf } from "./clickhouse-tables";
+import { collectorConfig } from "./collector";
+import type { OTelComponent } from "./define";
+
+export function otelWorkspaceLinks(entities: ReadonlyMap<string, Declarable>): LexiconWorkspaceLinks | undefined {
+  const keyOf = new Map<string, string>();
+  for (const [key, e] of entities) {
+    const c = e as Partial<OTelComponent>;
+    if (c.componentKind === "exporter" && c.componentType === "clickhouse" && typeof c.componentId === "string") keyOf.set(c.componentId, key);
+  }
+  if (keyOf.size === 0) return undefined;
+
+  const links: LexiconWorkspaceLink[] = [];
+  const gaps: string[] = [];
+  for (const exp of clickhouseExportersOf(collectorConfig(entities.values()))) {
+    const entity = keyOf.get(exp.id) ?? exp.id;
+    for (const u of exp.unread) gaps.push(`exporter ${exp.id} (${entity}) sets ${u}, which is read from the environment, so the tables it names are not linked`);
+    for (const t of exp.tables) {
+      const table = `${exp.database}.${t.name}`;
+      links.push({ output: table, entity });
+      for (const column of t.columns) links.push({ output: `${table}.${column}`, anchor: table, entity });
+    }
+  }
+  if (links.length === 0 && gaps.length === 0) return undefined;
+  return { links, ...(gaps.length ? { gaps } : {}) };
+}