@smthrs/observability 0.35.0 → 1.0.0-rc.12

package/src/Resource.ts

@@ -0,0 +1,325 @@
+/**
+ * Explicit, validated OpenTelemetry resource metadata for Smithers.
+ *
+ * @since 0.1.0
+ */
+
+import * as OtelResource from "@effect/opentelemetry/Resource"
+import type { Attributes as OtelAttributes } from "@opentelemetry/api"
+import * as Effect from "effect/Effect"
+import * as Layer from "effect/Layer"
+import * as OtlpResource from "effect/observability/OtlpResource"
+import * as Schema from "effect/Schema"
+import { schemaIssueMessage, schemaIssuePath } from "./internal/schemaIssuePath.ts"
+
+/**
+ * Largest service-name or service-version field accepted by a resource.
+ *
+ * @category constants
+ * @since 1.0.0-rc.0
+ */
+export const maximumIdentityLength = 1_024
+
+/**
+ * Largest attribute key accepted by a resource.
+ *
+ * @category constants
+ * @since 1.0.0-rc.0
+ */
+export const maximumAttributeKeyLength = 1_024
+
+/**
+ * Largest string value accepted by one resource attribute.
+ *
+ * @category constants
+ * @since 1.0.0-rc.0
+ */
+export const maximumAttributeStringLength = 65_536
+
+/**
+ * Largest number of elements accepted in one array attribute value.
+ *
+ * @category constants
+ * @since 1.0.0-rc.0
+ */
+export const maximumAttributeArrayLength = 256
+
+/**
+ * Largest number of attributes accepted by one resource.
+ *
+ * @category constants
+ * @since 1.0.0-rc.0
+ */
+export const maximumAttributes = 256
+
+/**
+ * Largest OTLP/JSON encoding accepted for one whole resource, in UTF-8 bytes.
+ *
+ * The measure is the OTLP `Resource` message the exporters place in every
+ * request, `{"attributes":[{"key":…,"value":{…}}],"droppedAttributesCount":0}`
+ * with service identity and SDK-added `telemetry.sdk.name`,
+ * `telemetry.sdk.language` and `telemetry.sdk.version` included, after JSON
+ * escaping and UTF-8 encoding. The SDK's own values are reserved even for the
+ * default Effect exporter, whose values are shorter. A caller attribute that
+ * replaces an SDK field, or that service identity overwrites, is counted
+ * beside it. It is the bound the per-field limits cannot express: 256
+ * attributes at the string ceiling would encode to 16 MiB, and the resource
+ * rides in every export request, so the default transport, which discards a
+ * request over 1 MiB, would deliver nothing at all. 128 KiB admits one
+ * ASCII attribute at the string ceiling and leaves that transport 888 KiB of every
+ * request for the signal batch; see `Otlp.reservedBatchBytes`.
+ *
+ * @category constants
+ * @since 1.0.0-rc.0
+ */
+export const maximumResourceBytes = 131_072
+
+const loneSurrogate = /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/
+const embeddedNul = new RegExp(String.fromCharCode(0))
+
+const isWellFormed = (value: string): boolean => !loneSurrogate.test(value) && !embeddedNul.test(value)
+
+const identity = Schema.NonEmptyString.check(
+  Schema.isMaxLength(maximumIdentityLength),
+  Schema.makeFilter((value: string) => isWellFormed(value), { title: "wellFormedResourceIdentity" })
+)
+
+const attributeString = Schema.String.check(
+  Schema.isMaxLength(maximumAttributeStringLength),
+  Schema.makeFilter((value: string) => isWellFormed(value), { title: "wellFormedAttributeString" })
+)
+
+const attributeNumber = Schema.Number.check(
+  Schema.makeFilter((value: number) => Number.isFinite(value), { title: "finiteAttributeNumber" })
+)
+
+/**
+ * Runtime schema for one OpenTelemetry resource attribute value.
+ *
+ * @category schemas
+ * @since 1.0.0-rc.0
+ */
+export const AttributeValue = Schema.Union([
+  attributeString,
+  attributeNumber,
+  Schema.Boolean,
+  Schema.Array(attributeString),
+  Schema.Array(attributeNumber),
+  Schema.Array(Schema.Boolean)
+]).check(
+  Schema.makeFilter(
+    (value) =>
+      !Array.isArray(value) || value.length <= maximumAttributeArrayLength || {
+        path: [],
+        issue: `holds ${value.length} elements, over the ${maximumAttributeArrayLength} element limit`
+      },
+    { title: "boundedResourceAttributeArrays" }
+  )
+)
+
+/**
+ * Runtime schema for OpenTelemetry resource attributes.
+ *
+ * @category schemas
+ * @since 1.0.0-rc.0
+ */
+export const Attributes = Schema.Record(Schema.String, AttributeValue).check(
+  Schema.makeFilter(
+    (attributes: Readonly<Record<string, unknown>>) => Object.keys(attributes).length <= maximumAttributes,
+    { title: "boundedResourceAttributes" }
+  ),
+  Schema.makeFilter(
+    (attributes: Readonly<Record<string, unknown>>) =>
+      Object.keys(attributes).every(
+        (key) => key.length > 0 && key.length <= maximumAttributeKeyLength && isWellFormed(key)
+      ),
+    { title: "wellFormedResourceAttributeKeys" }
+  )
+)
+
+const encoder = new TextEncoder()
+
+/**
+ * Counts OTLP JSON fragments, including keys, wrappers, escaping and UTF-8.
+ * Stops at the first fragment over budget: the refusal reports a lower bound.
+ * A configuration may reuse one large string in thousands of array slots, so
+ * serializing the whole resource before checking could allocate gigabytes.
+ */
+const encodedResourceBytes = (configuration: {
+  readonly serviceName: string
+  readonly serviceVersion?: string | undefined
+  readonly attributes?: Record<string, unknown> | undefined
+}): number => {
+  const bytes = (value: unknown) => encoder.encode(JSON.stringify(value)).byteLength
+  const identityAttributes = OtelResource.configToAttributes({
+    serviceName: configuration.serviceName,
+    ...(configuration.serviceVersion === undefined ? {} : { serviceVersion: configuration.serviceVersion })
+  })
+  // Use the pinned SDK projection so its added metadata cannot escape the
+  // complete-resource ceiling. Both exporters use these OTLP AnyValue shapes.
+  let total = bytes({
+    attributes: OtlpResource.entriesToAttributes(Object.entries(identityAttributes)),
+    droppedAttributesCount: 0
+  })
+  for (const [key, value] of Object.entries(configuration.attributes ?? {})) {
+    // An identity attribute is always present, so every addition needs a comma.
+    total += 1 + bytes({ key, value: Array.isArray(value) ? { arrayValue: { values: [] } } : null })
+    if (Array.isArray(value)) {
+      for (const [index, element] of value.entries()) {
+        total += (index === 0 ? 0 : 1) + bytes(OtlpResource.unknownToAttributeValue(element))
+        if (total > maximumResourceBytes) return total
+      }
+    } else {
+      total += bytes(OtlpResource.unknownToAttributeValue(value)) - 4 // replace JSON null
+    }
+    if (total > maximumResourceBytes) return total
+  }
+  return total
+}
+
+/**
+ * Runtime schema for the service identity attached to exported telemetry.
+ *
+ * @category schemas
+ * @since 1.0.0-rc.0
+ */
+export const Configuration = Schema.Struct({
+  serviceName: identity,
+  serviceVersion: Schema.optional(identity),
+  attributes: Schema.optional(Attributes)
+}).check(
+  Schema.makeFilter(
+    (configuration: {
+      readonly serviceName: string
+      readonly serviceVersion?: string | undefined
+      readonly attributes?: Record<string, unknown> | undefined
+    }) => {
+      const bytes = encodedResourceBytes(configuration)
+      // Only attributes can carry a resource past the budget: the two identity
+      // fields encode to at most 6 KiB each, so the refusal points there.
+      return bytes <= maximumResourceBytes || {
+        path: ["attributes"],
+        issue: `bring the resource to at least ${bytes} bytes of OTLP JSON, over the ${maximumResourceBytes} byte limit`
+      }
+    },
+    { title: "boundedResourceBytes" }
+  )
+)
+
+/**
+ * Configuration used to identify the service emitting telemetry.
+ *
+ * @category models
+ * @since 1.0.0-rc.0
+ */
+export type Configuration = typeof Configuration.Type
+
+/**
+ * Stable resource-configuration refusal shared by every OTEL layer.
+ *
+ * @category errors
+ * @since 1.0.0-rc.0
+ */
+export class InvalidResourceConfiguration extends Schema.TaggedError<InvalidResourceConfiguration>()(
+  "@smthrs/observability/InvalidResourceConfiguration",
+  {
+    code: Schema.Literal("invalid_resource_configuration"),
+    path: Schema.String,
+    message: Schema.String
+  }
+) {}
+
+const invalid = (cause: unknown): InvalidResourceConfiguration => {
+  const path = schemaIssuePath(cause, "resource")
+  return new InvalidResourceConfiguration({
+    code: "invalid_resource_configuration",
+    path,
+    message: `OpenTelemetry resource ${path} ${schemaIssueMessage(cause) ?? "is invalid"}`
+  })
+}
+
+/**
+ * Decodes one resource configuration without retaining rejected values.
+ *
+ * @category decoding
+ * @since 1.0.0-rc.0
+ */
+export const decode = (
+  configuration: unknown
+): Effect.Effect<Configuration, InvalidResourceConfiguration> =>
+  Schema.decodeUnknownEffect(Configuration)(configuration).pipe(Effect.mapError(invalid))
+
+/**
+ * Decodes one resource configuration for the package's pure projection API.
+ *
+ * @category decoding
+ * @since 1.0.0-rc.0
+ */
+export const decodeSync = (configuration: unknown): Configuration => {
+  try {
+    return Schema.decodeUnknownSync(Configuration)(configuration)
+  } catch (cause) {
+    throw invalid(cause)
+  }
+}
+
+/**
+ * Projects a decoded resource into the exact optional-property shape expected
+ * by the OpenTelemetry SDK.
+ *
+ * @category conversions
+ * @since 1.0.0-rc.0
+ */
+export const toOpenTelemetryConfiguration = (
+  configuration: Configuration
+): {
+  readonly serviceName: string
+  readonly serviceVersion?: string
+  readonly attributes?: OtelAttributes
+} => ({
+  serviceName: configuration.serviceName,
+  ...(configuration.serviceVersion === undefined ? {} : { serviceVersion: configuration.serviceVersion }),
+  ...(configuration.attributes === undefined
+    ? {}
+    : {
+      attributes: Object.fromEntries(
+        Object.entries(configuration.attributes).map(([key, value]) => [
+          key,
+          Array.isArray(value) ? [...value] : value
+        ])
+      ) as OtelAttributes
+    })
+})
+
+/**
+ * Converts explicit service metadata into OpenTelemetry resource attributes.
+ *
+ * No environment variables are read. Invalid metadata throws the same typed
+ * {@link InvalidResourceConfiguration} every layer returns during acquisition.
+ *
+ * @category conversions
+ * @since 1.0.0-rc.0
+ */
+export const configToAttributes = (configuration: Configuration): OtelAttributes =>
+  OtelResource.configToAttributes(toOpenTelemetryConfiguration(decodeSync(configuration)))
+
+/**
+ * Provides an explicitly validated OpenTelemetry resource.
+ *
+ * @category layers
+ * @since 1.0.0-rc.0
+ */
+export const layer = (
+  configuration: Configuration
+): Layer.Layer<OtelResource.Resource, InvalidResourceConfiguration> =>
+  Layer.unwrap(
+    Effect.map(decode(configuration), (decoded) => OtelResource.layer(toOpenTelemetryConfiguration(decoded)))
+  )
+
+/**
+ * The OpenTelemetry resource service.
+ *
+ * @category services
+ * @since 1.0.0-rc.0
+ */
+export const Resource = OtelResource.Resource

package/src/index.ts

@@ -0,0 +1,67 @@
+/**
+ * Observability for flows: the default OTLP export wiring plus the
+ * Effect-native logger, metric, and OpenTelemetry resource layers.
+ *
+ * The store packages define and update their own metric handles
+ * (`JournalMetrics`, `RunStoreMetrics`, `CacheStoreMetrics`,
+ * `ArtifactStoreMetrics`, `DatabaseMetrics`) and open spans through Effect's
+ * tracer; this package is the exporter they deliberately leave out. `Otlp`
+ * only composes what `effect` already ships, so it is browser-bundleable. The
+ * remaining modules are the fuller stack folded in from the agent workspace:
+ * a structured `Logger`, a `JournalLogger` that mirrors log events into the
+ * run journal, `Metric` helpers, the `Resource` descriptor, and the `Otel`
+ * layer the OpenTelemetry SDK wiring installs behind.
+ *
+ * `NodeOtel` and `BrowserOtel` are deliberately NOT re-exported here. Each
+ * resolves a host-specific OpenTelemetry SDK. `NodeOtel` reaches Node-only
+ * host modules through `@effect/opentelemetry/NodeSdk` and
+ * `@opentelemetry/sdk-trace-node`, including the bare `async_hooks` specifier
+ * `@opentelemetry/context-async-hooks` imports, so re-exporting it would put a
+ * module a browser bundler cannot resolve in the root entry and break the
+ * browser bundle this package guarantees. Import them by subpath instead:
+ * `@smthrs/observability/NodeOtel`.
+ *
+ * @since 0.1.0
+ */
+
+/**
+ * @category layers
+ * @since 0.1.0
+ */
+export * as Otlp from "./Otlp.ts"
+
+/**
+ * @category observability
+ * @since 1.0.0-rc.0
+ */
+export * as Endpoint from "./Endpoint.ts"
+
+/**
+ * @category observability
+ * @since 0.1.0
+ */
+export * as JournalLogger from "./JournalLogger.ts"
+
+/**
+ * @category observability
+ * @since 0.1.0
+ */
+export * as Logger from "./Logger.ts"
+
+/**
+ * @category observability
+ * @since 0.1.0
+ */
+export * as Metric from "./Metric.ts"
+
+/**
+ * @category observability
+ * @since 0.1.0
+ */
+export * as Otel from "./Otel.ts"
+
+/**
+ * @category observability
+ * @since 0.1.0
+ */
+export * as Resource from "./Resource.ts"

package/src/internal/schemaIssuePath.ts

@@ -0,0 +1,65 @@
+/**
+ * The decoder-issue walk shared by this package's refusals.
+ *
+ * `JournalLogger` and `Resource` both publish the path of the option a decode
+ * rejected, and `Resource` also publishes the measured size a budget filter
+ * attached to its refusal. Effect's issues nest the same way for both, so the
+ * traversal lives once and a decoder change is followed once.
+ *
+ * @private
+ * @since 1.0.0-rc.0
+ */
+
+interface IssueLeaf {
+  readonly segments: ReadonlyArray<string>
+  readonly leaf: unknown
+}
+
+/**
+ * Walks a decode failure to its first offending leaf, collecting the path
+ * segments passed on the way. The walk is bounded so a self-referential issue
+ * tree cannot spin.
+ */
+const walk = (error: unknown): IssueLeaf => {
+  let issue = (error as { readonly issue?: unknown } | null)?.issue
+  const segments: Array<string> = []
+  for (let depth = 0; depth < 64 && typeof issue === "object" && issue !== null; depth++) {
+    const node = issue as { readonly path?: unknown; readonly issue?: unknown; readonly issues?: unknown }
+    if (Array.isArray(node.path)) segments.push(...node.path.map(String))
+    if (node.issue !== undefined) {
+      issue = node.issue
+      continue
+    }
+    if (Array.isArray(node.issues) && node.issues[0] !== undefined) {
+      issue = node.issues[0]
+      continue
+    }
+    break
+  }
+  return { segments, leaf: issue }
+}
+
+/**
+ * Walks a decode failure to the dotted path of the value it refused.
+ *
+ * Returns the caller's domain name when the failure carries no path at all.
+ *
+ * @private
+ * @since 1.0.0-rc.0
+ */
+export const schemaIssuePath = (error: unknown, fallback: string): string => walk(error).segments.join(".") || fallback
+
+/**
+ * Reads the message a filter attached to the value a decode refused, when the
+ * filter attached one. None of the Effect checks this package composes attach
+ * one, so a refusal only ever carries text this package wrote, never the
+ * rejected value.
+ *
+ * @private
+ * @since 1.0.0-rc.0
+ */
+export const schemaIssueMessage = (error: unknown): string | undefined => {
+  const leaf = walk(error).leaf as { readonly annotations?: { readonly message?: unknown } } | null | undefined
+  const message = leaf?.annotations?.message
+  return typeof message === "string" ? message : undefined
+}

package/docker-compose.otel.yml

@@ -1,104 +0,0 @@
-version: "3.9"
-
-services:
-  otel-collector:
-    image: otel/opentelemetry-collector-contrib:0.148.0
-    restart: unless-stopped
-    command: ["--config", "/etc/otelcol/config.yml"]
-    volumes:
-      - ./otel-collector-config.yml:/etc/otelcol/config.yml:ro
-    ports:
-      - "4317:4317"   # gRPC
-      - "4318:4318"   # HTTP (Smithers sends here)
-      - "8889:8889"   # Prometheus exporter scrape endpoint
-    depends_on:
-      tempo:
-        condition: service_healthy
-      loki:
-        condition: service_healthy
-    healthcheck:
-      test: ["CMD", "/otelcol-contrib", "--version"]
-      interval: 10s
-      timeout: 5s
-      retries: 12
-      start_period: 10s
-
-  loki:
-    image: grafana/loki:3.3.2
-    restart: unless-stopped
-    command: ["-config.file=/etc/loki/local-config.yaml"]
-    volumes:
-      - ./loki/loki-config.yaml:/etc/loki/local-config.yaml:ro
-      - ./data/loki:/loki
-    ports:
-      - "3100:3100"
-    healthcheck:
-      test: ["CMD", "wget", "-q", "-O-", "http://127.0.0.1:3100/ready"]
-      interval: 10s
-      timeout: 5s
-      retries: 18
-      start_period: 10s
-
-  prometheus:
-    image: prom/prometheus:v3.2.0
-    restart: unless-stopped
-    volumes:
-      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
-      - ./data/prometheus:/prometheus
-    ports:
-      - "9090:9090"
-    depends_on:
-      otel-collector:
-        condition: service_healthy
-    healthcheck:
-      test: ["CMD", "wget", "-q", "-O-", "http://127.0.0.1:9090/-/ready"]
-      interval: 10s
-      timeout: 5s
-      retries: 18
-      start_period: 10s
-
-  tempo:
-    image: grafana/tempo:2.7.0
-    restart: unless-stopped
-    command: ["-config.file=/etc/tempo/tempo.yml"]
-    volumes:
-      - ./tempo/tempo.yml:/etc/tempo/tempo.yml:ro
-      - ./data/tempo:/tmp/tempo
-    ports:
-      - "3200:3200"   # Tempo query API
-      - "4317"        # OTLP gRPC (internal)
-    healthcheck:
-      test: ["CMD", "wget", "-q", "-O-", "http://127.0.0.1:3200/api/search"]
-      interval: 10s
-      timeout: 5s
-      retries: 24
-      start_period: 20s
-
-  grafana:
-    image: grafana/grafana:11.5.0
-    restart: unless-stopped
-    environment:
-      # Local dev default is "admin"; override via env for anything shared.
-      - GF_SECURITY_ADMIN_PASSWORD=${GF_SECURITY_ADMIN_PASSWORD:-admin}
-      # Anonymous browsing stays on for frictionless local dev, but read-only.
-      - GF_AUTH_ANONYMOUS_ENABLED=${GF_AUTH_ANONYMOUS_ENABLED:-true}
-      - GF_AUTH_ANONYMOUS_ORG_ROLE=${GF_AUTH_ANONYMOUS_ORG_ROLE:-Viewer}
-    volumes:
-      - ./grafana/provisioning:/etc/grafana/provisioning:ro
-      - ./grafana/dashboards:/var/lib/grafana/dashboards:ro
-      - ./data/grafana:/var/lib/grafana
-    ports:
-      - "3001:3000"
-    depends_on:
-      prometheus:
-        condition: service_healthy
-      tempo:
-        condition: service_healthy
-      loki:
-        condition: service_healthy
-    healthcheck:
-      test: ["CMD", "wget", "-q", "-O-", "http://127.0.0.1:3000/api/health"]
-      interval: 10s
-      timeout: 5s
-      retries: 24
-      start_period: 20s