@intentius/chant-lexicon-otel 0.97.0 → 0.98.0

package/src/import/testdata/upstream/couchbase.yaml

@@ -0,0 +1,150 @@
+# Vendored from opentelemetry-collector-contrib v0.130.0 (Apache-2.0), unchanged below this header:
+# https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/v0.130.0/examples/couchbase/otel-collector-config.yaml
+
+receivers:
+  prometheus/couchbase:
+    config:
+      scrape_configs:
+        - job_name: 'couchbase'
+          scrape_interval: 5s
+          static_configs:
+            - targets: ['couchbase:8091']
+          basic_auth:
+            username: 'otelu'
+            password: 'otelpassword'
+          metric_relabel_configs:
+            # Include only a few key metrics
+            - source_labels: [ __name__ ]
+              regex: "(kv_ops)|\
+                (kv_vb_curr_items)|\
+                (kv_num_vbuckets)|\
+                (kv_ep_cursor_memory_freed_bytes)|\
+                (kv_total_memory_used_bytes)|\
+                (kv_ep_num_value_ejects)|\
+                (kv_ep_mem_high_wat)|\
+                (kv_ep_mem_low_wat)|\
+                (kv_ep_tmp_oom_errors)|\
+                (kv_ep_oom_errors)"
+              action: keep
+
+processors:
+  filter/couchbase:
+    # Filter out prometheus scraping meta-metrics.
+    metrics:
+      exclude:
+        match_type: strict
+        metric_names:
+          - scrape_samples_post_metric_relabeling
+          - scrape_series_added
+          - scrape_duration_seconds
+          - scrape_samples_scraped
+          - up
+
+  metricstransform/couchbase:
+    transforms:
+      # Rename from prometheus metric name to OTel metric name.
+      # We cannot do this with metric_relabel_configs, as the prometheus receiver does not
+      # allow metric renames at this time.
+      - include: kv_ops
+        match_type: strict
+        action: update
+        new_name: "couchbase.bucket.operation.count"
+      - include: kv_vb_curr_items
+        match_type: strict
+        action: update
+        new_name: "couchbase.bucket.item.count"
+      - include: kv_num_vbuckets
+        match_type: strict
+        action: update
+        new_name: "couchbase.bucket.vbucket.count"
+      - include: kv_ep_cursor_memory_freed_bytes
+        match_type: strict
+        action: update
+        new_name: "couchbase.bucket.memory.usage.free"
+      - include: kv_total_memory_used_bytes
+        match_type: strict
+        action: update
+        new_name: "couchbase.bucket.memory.usage.used"
+      - include: kv_ep_num_value_ejects
+        match_type: strict
+        action: update
+        new_name: "couchbase.bucket.item.ejection.count"
+      - include: kv_ep_mem_high_wat
+        match_type: strict
+        action: update
+        new_name: "couchbase.bucket.memory.high_water_mark.limit"
+      - include: kv_ep_mem_low_wat
+        match_type: strict
+        action: update
+        new_name: "couchbase.bucket.memory.low_water_mark.limit"
+      - include: kv_ep_tmp_oom_errors
+        match_type: strict
+        action: update
+        new_name: "couchbase.bucket.error.oom.count.recoverable"
+      - include: kv_ep_oom_errors
+        match_type: strict
+        action: update
+        new_name: "couchbase.bucket.error.oom.count.unrecoverable"
+      # Combine couchbase.bucket.error.oom.count.x and couchbase.bucket.memory.usage.x
+      # metrics.
+      - include: '^couchbase\.bucket\.error\.oom\.count\.(?P<error_type>unrecoverable|recoverable)$$'
+        match_type: regexp
+        action: combine
+        new_name: "couchbase.bucket.error.oom.count"
+      - include: '^couchbase\.bucket\.memory\.usage\.(?P<state>free|used)$$'
+        match_type: regexp
+        action: combine
+        new_name: "couchbase.bucket.memory.usage"
+      # Aggregate "result" label on operation count to keep label sets consistent across the metric datapoints
+      - include: 'couchbase.bucket.operation.count'
+        match_type: strict
+        action: update
+        operations:
+          - action: aggregate_labels
+            label_set: ["bucket", "op"]
+            aggregation_type: sum
+
+  transform/couchbase:
+    metric_statements:
+    - context: datapoint
+      statements:
+        - convert_gauge_to_sum("cumulative", true) where metric.name == "couchbase.bucket.operation.count"
+        - set(metric.description, "Number of operations on the bucket.") where metric.name == "couchbase.bucket.operation.count"
+        - set(metric.unit, "{operations}") where metric.name == "couchbase.bucket.operation.count"
+
+        - convert_gauge_to_sum("cumulative", false) where metric.name == "couchbase.bucket.item.count"
+        - set(metric.description, "Number of items that belong to the bucket.") where metric.name == "couchbase.bucket.item.count"
+        - set(metric.unit, "{items}") where metric.name == "couchbase.bucket.item.count"
+
+        - convert_gauge_to_sum("cumulative", false) where metric.name == "couchbase.bucket.vbucket.count"
+        - set(metric.description, "Number of non-resident vBuckets.") where metric.name == "couchbase.bucket.vbucket.count"
+        - set(metric.unit, "{vbuckets}") where metric.name == "couchbase.bucket.vbucket.count"
+
+        - convert_gauge_to_sum("cumulative", false) where metric.name == "couchbase.bucket.memory.usage"
+        - set(metric.description, "Usage of total memory available to the bucket.") where metric.name == "couchbase.bucket.memory.usage"
+        - set(metric.unit, "By") where metric.name == "couchbase.bucket.memory.usage"
+
+        - convert_gauge_to_sum("cumulative", true) where metric.name == "couchbase.bucket.item.ejection.count"
+        - set(metric.description, "Number of item value ejections from memory to disk.") where metric.name == "couchbase.bucket.item.ejection.count"
+        - set(metric.unit, "{ejections}") where metric.name == "couchbase.bucket.item.ejection.count"
+
+        - convert_gauge_to_sum("cumulative", true) where metric.name == "couchbase.bucket.error.oom.count"
+        - set(metric.description, "Number of out of memory errors.") where metric.name == "couchbase.bucket.error.oom.count"
+        - set(metric.unit, "{errors}") where metric.name == "couchbase.bucket.error.oom.count"
+
+        - set(metric.description, "The memory usage at which items will be ejected.") where metric.name == "couchbase.bucket.memory.high_water_mark.limit"
+        - set(metric.unit, "By") where metric.name == "couchbase.bucket.memory.high_water_mark.limit"
+
+        - set(metric.description, "The memory usage at which ejections will stop that were previously triggered by a high water mark breach.") where metric.name == "couchbase.bucket.memory.low_water_mark.limit"
+        - set(metric.unit, "By") where metric.name == "couchbase.bucket.memory.low_water_mark.limit"
+
+exporters:
+  prometheus:
+    endpoint: "0.0.0.0:9123"
+
+service:
+  pipelines:
+    metrics/couchbase:
+      receivers: [prometheus/couchbase]
+      processors: [filter/couchbase, metricstransform/couchbase, transform/couchbase]
+      exporters: [prometheus]

package/src/import/testdata/upstream/fault-tolerant-logs.yaml

@@ -0,0 +1,27 @@
+# Vendored from opentelemetry-collector-contrib v0.130.0 (Apache-2.0), unchanged below this header:
+# https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/v0.130.0/examples/fault-tolerant-logs-collection/otel-col-config.yaml
+
+receivers:
+  filelog:
+    include: [/var/log/busybox/simple.log]
+    storage: file_storage/filelogreceiver
+
+extensions:
+  file_storage/filelogreceiver:
+    directory: /var/lib/otelcol/file_storage/receiver
+  file_storage/otlpoutput:
+    directory: /var/lib/otelcol/file_storage/output
+
+service:
+  extensions: [file_storage/filelogreceiver, file_storage/otlpoutput]
+  pipelines:
+    logs:
+      receivers: [filelog]
+      exporters: [otlp/custom]
+      processors: []
+
+exporters:
+  otlp/custom:
+    endpoint: http://0.0.0.0:4242
+    sending_queue:
+      storage: file_storage/otlpoutput

package/src/import/testdata/upstream/kubernetes-filelog.yaml

@@ -0,0 +1,86 @@
+# Vendored from opentelemetry-collector-contrib v0.130.0 (Apache-2.0), unchanged below this header:
+# https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/v0.130.0/examples/kubernetes/otel-collector-config.yml
+
+receivers:
+  filelog:
+    include:
+      - /var/log/pods/*/*/*.log
+    exclude:
+      # Exclude logs from all containers named otel-collector
+      - /var/log/pods/*/otel-collector/*.log
+    start_at: end
+    include_file_path: true
+    include_file_name: false
+    operators:
+      # Find out which format is used by kubernetes
+      - type: router
+        id: get-format
+        routes:
+          - output: parser-docker
+            expr: 'body matches "^\\{"'
+          - output: parser-crio
+            expr: 'body matches "^[^ Z]+ "'
+          - output: parser-containerd
+            expr: 'body matches "^[^ Z]+Z"'
+      # Parse CRI-O format
+      - type: regex_parser
+        id: parser-crio
+        regex: '^(?P<time>[^ Z]+) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*) ?(?P<log>.*)$'
+        output: extract_metadata_from_filepath
+        timestamp:
+          parse_from: attributes.time
+          layout_type: gotime
+          layout: '2006-01-02T15:04:05.999999999Z07:00'
+      # Parse CRI-Containerd format
+      - type: regex_parser
+        id: parser-containerd
+        regex: '^(?P<time>[^ ^Z]+Z) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*) ?(?P<log>.*)$'
+        output: extract_metadata_from_filepath
+        timestamp:
+          parse_from: attributes.time
+          layout: '%Y-%m-%dT%H:%M:%S.%LZ'
+      # Parse Docker format
+      - type: json_parser
+        id: parser-docker
+        output: extract_metadata_from_filepath
+        timestamp:
+          parse_from: attributes.time
+          layout: '%Y-%m-%dT%H:%M:%S.%LZ'
+      # Extract metadata from file path
+      - type: regex_parser
+        id: extract_metadata_from_filepath
+        regex: '^.*\/(?P<namespace>[^_]+)_(?P<pod_name>[^_]+)_(?P<uid>[a-f0-9\-]{36})\/(?P<container_name>[^\._]+)\/(?P<restart_count>\d+)\.log$'
+        parse_from: attributes["log.file.path"]
+        cache:
+          size: 128  # default maximum amount of Pods per Node is 110
+      # Update body field after finishing all parsing
+      - type: move
+        from: attributes.log
+        to: body
+      # Rename attributes
+      - type: move
+        from: attributes.stream
+        to: attributes["log.iostream"]
+      - type: move
+        from: attributes.container_name
+        to: resource["k8s.container.name"]
+      - type: move
+        from: attributes.namespace
+        to: resource["k8s.namespace.name"]
+      - type: move
+        from: attributes.pod_name
+        to: resource["k8s.pod.name"]
+      - type: move
+        from: attributes.restart_count
+        to: resource["k8s.container.restart_count"]
+      - type: move
+        from: attributes.uid
+        to: resource["k8s.pod.uid"]
+exporters:
+  debug:
+    verbosity: detailed
+service:
+  pipelines:
+    logs:
+      receivers: [filelog]
+      exporters: [debug]

package/src/import/testdata/upstream/loadbalancing-agent.yaml

@@ -0,0 +1,38 @@
+# Vendored from opentelemetry-collector-contrib v0.130.0 (Apache-2.0), unchanged below this header:
+# https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/v0.130.0/exporter/loadbalancingexporter/example/otel-agent-config.yaml
+
+receivers:
+  fluentforward:
+    endpoint: 0.0.0.0:24224
+
+processors:
+  batch:
+
+exporters:
+  debug:
+    verbosity: detailed
+  loadbalancing:
+    protocol:
+      otlp:
+        timeout: 1s
+        tls:
+          insecure: true
+    resolver:
+      static:
+        hostnames:
+          - otel-collector-1:4317
+          - otel-collector-2:4317
+          - otel-collector-3:4317
+
+extensions:
+  health_check:
+  pprof:
+  zpages:
+
+service:
+  extensions: [pprof, zpages, health_check]
+  pipelines:
+    logs:
+      receivers: [fluentforward]
+      processors: [batch]
+      exporters: [loadbalancing, debug]

package/src/import/testdata/upstream/loadbalancing-backend.yaml

@@ -0,0 +1,27 @@
+# Vendored from opentelemetry-collector-contrib v0.130.0 (Apache-2.0), unchanged below this header:
+# https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/v0.130.0/exporter/loadbalancingexporter/example/otel-collector-config.yaml
+
+receivers:
+  otlp:
+    protocols:
+      grpc:
+
+processors:
+  batch:
+
+exporters:
+  debug:
+    verbosity: detailed
+
+extensions:
+  health_check:
+  pprof:
+  zpages:
+
+service:
+  extensions: [pprof, zpages, health_check]
+  pipelines:
+    logs:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [debug]

package/src/import/testdata/upstream/logline-filter-in.yaml

@@ -0,0 +1,20 @@
+# Vendored from opentelemetry-collector-contrib v0.130.0 (Apache-2.0), unchanged below this header:
+# https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/v0.130.0/examples/logline-filtering/otel-col-config-filter-in-logs.yaml
+
+receivers:
+  filelog:
+    include: [/var/log/busybox/simple.log]
+    operators:
+      - type: filter
+        expr: 'body not matches "^WARN:"'
+
+service:
+  pipelines:
+    logs:
+      receivers: [filelog]
+      exporters: [otlp/custom]
+      processors: []
+
+exporters:
+  otlp/custom:
+    endpoint: http://0.0.0.0:4242

package/src/import/testdata/upstream/logline-filter-out.yaml

@@ -0,0 +1,20 @@
+# Vendored from opentelemetry-collector-contrib v0.130.0 (Apache-2.0), unchanged below this header:
+# https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/v0.130.0/examples/logline-filtering/otel-col-config-filter-out-logs.yaml
+
+receivers:
+  filelog:
+    include: [/var/log/busybox/simple.log]
+    operators:
+      - type: filter
+        expr: 'body matches "^INFO:"'
+
+service:
+  pipelines:
+    logs:
+      receivers: [filelog]
+      exporters: [otlp/custom]
+      processors: []
+
+exporters:
+  otlp/custom:
+    endpoint: http://0.0.0.0:4242

package/src/import/testdata/upstream/secure-tracing.yaml

@@ -0,0 +1,26 @@
+# Vendored from opentelemetry-collector-contrib v0.130.0 (Apache-2.0), unchanged below this header:
+# https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/v0.130.0/examples/secure-tracing/otel-collector-config.yaml
+
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:55690
+        tls:
+          cert_file: /etc/otel-collector.crt
+          key_file: /etc/otel-collector.key
+
+processors:
+  batch:
+
+exporters:
+  debug:
+    verbosity: detailed
+
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [debug]
+

package/src/import/testdata/upstream/servicegraph-nop.yaml

@@ -0,0 +1,28 @@
+# Vendored from opentelemetry-collector-contrib v0.130.0 (Apache-2.0), unchanged below this header:
+# https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/v0.130.0/connector/servicegraphconnector/testdata/service-graph-connector-config.yaml
+
+receivers:
+  nop:
+
+exporters:
+  nop:
+
+connectors:
+  servicegraph:
+    latency_histogram_buckets: [1,2,3,4,5]
+    dimensions:
+      - dimension-1
+      - dimension-2
+    store:
+      ttl: 1s
+      max_items: 10
+    database_name_attributes: [db.name]
+
+service:
+  pipelines:
+    traces:
+      receivers: [nop]
+      exporters: [servicegraph]
+    metrics:
+      receivers: [servicegraph]
+      exporters: [nop]

package/src/lint/rules/literal-credential.ts

@@ -25,25 +25,54 @@ export const literalCredentialRule: LintRule = {
     const diagnostics: LintDiagnostic[] = [];
     const source = context.sourceFile;
 
+    // `const x = { ... }` (or `[...]`, or a string) anywhere in the file, so a
+    // config value lifted out of the constructor into a named const, the way
+    // COR001 asks, is scanned where it is used.
+    const consts = new Map<string, ts.Expression>();
+    const collect = (node: ts.Node) => {
+      if (ts.isVariableDeclarationList(node) && node.flags & ts.NodeFlags.Const) {
+        for (const d of node.declarations) {
+          if (ts.isIdentifier(d.name) && d.initializer) consts.set(d.name.text, d.initializer);
+        }
+      }
+      ts.forEachChild(node, collect);
+    };
+    collect(source);
+
+    const flag = (key: string, node: ts.Node) =>
+      diagnostics.push({
+        ruleId: "OTEL002",
+        severity: "error",
+        message: `\`${key}\` is a literal credential; write "\${env:NAME}" (or "\${file:/path}") and let the collector read it at start-up`,
+        file: context.filePath,
+        ...position(source, node),
+      });
+
+    const scanned = new Set<ts.Node>();
+    const scanValue = (key: string | undefined, value: ts.Expression) => {
+      let init: ts.Expression = value;
+      if (ts.isIdentifier(init)) {
+        const resolved = consts.get(init.text);
+        if (!resolved) return;
+        init = resolved;
+      }
+      if (ts.isObjectLiteralExpression(init) || ts.isArrayLiteralExpression(init)) {
+        if (scanned.has(init)) return;
+        scanned.add(init);
+        if (ts.isObjectLiteralExpression(init)) scan(init);
+        else for (const el of init.elements) scanValue(undefined, el as ts.Expression);
+        return;
+      }
+      if (!key || !SECRET_KEY.test(key) || /_file$/i.test(key)) return;
+      const text = literalText(init);
+      if (text === undefined || text === "" || text.includes("${")) return;
+      flag(key, value);
+    };
+
     const scan = (obj: ts.ObjectLiteralExpression) => {
       for (const prop of obj.properties) {
-        if (!ts.isPropertyAssignment(prop)) continue;
-        const key = propertyName(prop);
-        const init = prop.initializer;
-        if (ts.isObjectLiteralExpression(init)) {
-          scan(init);
-          continue;
-        }
-        if (!key || !SECRET_KEY.test(key) || /_file$/i.test(key)) continue;
-        const text = literalText(init);
-        if (text === undefined || text === "" || text.includes("${")) continue;
-        diagnostics.push({
-          ruleId: "OTEL002",
-          severity: "error",
-          message: `\`${key}\` is a literal credential; write "\${env:NAME}" (or "\${file:/path}") and let the collector read it at start-up`,
-          file: context.filePath,
-          ...position(source, init),
-        });
+        if (ts.isShorthandPropertyAssignment(prop)) scanValue(prop.name.text, prop.name);
+        else if (ts.isPropertyAssignment(prop)) scanValue(propertyName(prop), prop.initializer);
       }
     };
 

package/src/lint/rules/rules.test.ts

@@ -67,4 +67,20 @@ describe("OTEL002 literal credential", () => {
     );
     expect(diags).toEqual([]);
   });
+
+  test("follows config lifted into named consts, and looks inside lists", () => {
+    const diags = literalCredentialRule.check(
+      ctx(`
+        const headers: OtlpExporterConfig["headers"] = { authorization: "Bearer abc123" };
+        const safe = { authorization: "Bearer \${env:TOKEN}" };
+        const token = "literal";
+        const config = { scrape_configs: [{ job_name: "x", basic_auth: { username: "u", password: "p" } }] };
+        new OtlpExporter({ endpoint: "x:4317", headers });
+        new OtlpExporter({ name: "b", endpoint: "x:4317", headers: safe });
+        new SplunkHecExporter({ token, endpoint: "https://hec" });
+        new PrometheusReceiver({ config });
+      `),
+    );
+    expect(diags.map((d) => d.message.split("`")[1])).toEqual(["authorization", "token", "password"]);
+  });
 });

package/src/plugin.ts

@@ -9,6 +9,8 @@ import { otelAuditCatalog } from "./lint/audit-catalog";
 import { completions } from "./lsp/completions";
 import { hover } from "./lsp/hover";
 import { detectTemplate } from "./detect";
+import { OtelCollectorParser } from "./import/parser";
+import { OtelCollectorGenerator } from "./import/generator";
 import { otelSkills } from "./skill-defs";
 import { BUILTIN_CATALOG } from "./catalog";
 import { COLLECTOR_PIN } from "./define";
@@ -96,6 +98,14 @@ export const otelPlugin: LexiconPlugin = {
     return detectTemplate(data);
   },
 
+  templateParser() {
+    return new OtelCollectorParser();
+  },
+
+  templateGenerator() {
+    return new OtelCollectorGenerator();
+  },
+
   completionProvider(ctx: CompletionContext) {
     return completions(ctx);
   },

package/src/skills/chant-otel.md

@@ -78,6 +78,17 @@ It deletes prompt, completion, system-instruction and tool-call content from spa
 - Every pipeline needs at least one receiver and one exporter (OTEL102). A declared component no pipeline uses is a warning (OTEL103).
 - Declared extensions are enabled in declaration order unless a `Service` lists `extensions` itself.
 
+## Starting from an existing config
+
+When the user already has a collector config file, import it rather than retyping it:
+
+```bash
+chant import otel-collector-config.yaml --output src
+chant build src --lexicon otel -o collector.yaml   # the same config back
+```
+
+The importer writes `receivers.ts`, `processors.ts`, `exporters.ts`, `connectors.ts`, `extensions.ts`, `pipelines.ts`, and `service.ts` when extensions or telemetry need a `Service`. A component type with no built-in class lands in `custom-components.ts` as a `defineComponent` whose config is untyped data; offer to give it a config interface. Settings outside a built-in's config type still import and build, but `tsc` flags them. Read the warnings `chant import` prints: they name anything the lexicon has no place for.
+
 ## Reading the result
 
 `collectorTopologyOf(entities)` returns the pipelines, each component's endpoints and schema pin, for each exporter the signals it carries, the connector `edges` between pipelines, and under `semconv` the semantic-conventions version (`GENAI_SEMCONV_PIN`) the config's `gen_ai.` keys follow, as plain data.