@equipe-tech/observability 0.2.0 → 0.3.5

package/dist/node/LocalCollector.js

@@ -0,0 +1,188 @@
+import { Schema } from "effect";
+import { spawn } from "node:child_process";
+import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { assessTelemetryDestination, } from "../testing/conformance/TelemetryEvidence.js";
+import { startOtlpCaptureServer, } from "../testing/index.js";
+export class LocalCollectorError extends Schema.TaggedError()("LocalCollectorError", {
+    code: Schema.Literal("OBS_CONFORMANCE_LOCAL_COLLECTOR_FAILED"),
+    message: Schema.String,
+    cause: Schema.Defect(),
+}) {
+}
+const collectorError = (message, cause) => new LocalCollectorError({
+    code: "OBS_CONFORMANCE_LOCAL_COLLECTOR_FAILED",
+    message,
+    cause,
+});
+const DockerPort = Schema.String.check(Schema.isPattern(/127[.]0[.]0[.]1:[0-9]+/));
+const DockerGateway = Schema.String.check(Schema.isPattern(/^(?:[0-9]{1,3}[.]){3}[0-9]{1,3}$/));
+const decodeDockerPort = Schema.decodeUnknownSync(DockerPort);
+const decodeDockerGateway = Schema.decodeUnknownSync(DockerGateway);
+const collectorTraversalAttribute = "observability.collector.traversal";
+const traversedTelemetry = (telemetry, traversal) => ({
+    spans: telemetry.spans.filter((span) => span.resourceAttributes.get(collectorTraversalAttribute) === traversal),
+    logs: telemetry.logs.filter((log) => log.resourceAttributes.get(collectorTraversalAttribute) === traversal),
+    metrics: telemetry.metrics.filter((metric) => metric.resourceAttributes.get(collectorTraversalAttribute) === traversal),
+});
+const command = (args) => new Promise((resolve, reject) => {
+    const process = spawn(args[0] ?? "", args.slice(1));
+    let stdout = "";
+    let stderr = "";
+    process.stdout.on("data", (chunk) => {
+        stdout += chunk.toString();
+    });
+    process.stderr.on("data", (chunk) => {
+        stderr += chunk.toString();
+    });
+    process.on("error", reject);
+    process.on("close", (exitCode) => {
+        if (exitCode === 0)
+            resolve(`${stdout}${stderr}`.trim());
+        else {
+            reject(collectorError(`${args.join(" ")} failed: ${stderr.trim()}`, {
+                exitCode,
+                command: args[0] ?? "",
+            }));
+        }
+    });
+});
+const waitForCollector = async (endpoint) => {
+    for (let attempt = 0; attempt < 40; attempt++) {
+        const reachable = await fetch(endpoint, {
+            signal: AbortSignal.timeout(1_000),
+        }).then(() => true, () => false);
+        if (reachable)
+            return;
+        await new Promise((resolve) => setTimeout(resolve, 250));
+    }
+    throw collectorError(`The isolated Collector did not listen at ${endpoint.origin}.`, endpoint.origin);
+};
+export const startLocalCollectorDestination = async () => {
+    const collectorInstance = `observability-conformance-${crypto.randomUUID()}`;
+    const collectorTraversal = crypto.randomUUID();
+    let destination;
+    let directory;
+    let running = false;
+    try {
+        const linuxGateway = process.platform === "linux"
+            ? decodeDockerGateway(await command([
+                "docker",
+                "network",
+                "inspect",
+                "bridge",
+                "--format",
+                "{{(index .IPAM.Config 0).Gateway}}",
+            ]))
+            : undefined;
+        directory = await mkdtemp(join(tmpdir(), "observability-conformance-"));
+        destination =
+            linuxGateway === undefined
+                ? await startOtlpCaptureServer()
+                : await startOtlpCaptureServer({ host: linuxGateway });
+        const destinationHost = linuxGateway ?? "host.docker.internal";
+        const configPath = join(directory, "collector.yaml");
+        await writeFile(configPath, [
+            "receivers:",
+            "  otlp:",
+            "    protocols:",
+            "      http:",
+            "        endpoint: 0.0.0.0:4318",
+            "exporters:",
+            "  otlphttp/destination:",
+            `    endpoint: http://${destinationHost}:${destination.endpoint.port}`,
+            "    compression: none",
+            "    encoding: json",
+            "processors:",
+            "  resource/local_traversal:",
+            "    attributes:",
+            `      - key: ${collectorTraversalAttribute}`,
+            `        value: ${collectorTraversal}`,
+            "        action: upsert",
+            "service:",
+            "  pipelines:",
+            "    traces:",
+            "      receivers: [otlp]",
+            "      processors: [resource/local_traversal]",
+            "      exporters: [otlphttp/destination]",
+            "    metrics:",
+            "      receivers: [otlp]",
+            "      processors: [resource/local_traversal]",
+            "      exporters: [otlphttp/destination]",
+            "    logs:",
+            "      receivers: [otlp]",
+            "      processors: [resource/local_traversal]",
+            "      exporters: [otlphttp/destination]",
+            "",
+        ].join("\n"));
+        await command([
+            "docker",
+            "run",
+            "--detach",
+            "--rm",
+            "--name",
+            collectorInstance,
+            "--add-host",
+            "host.docker.internal:host-gateway",
+            "--publish",
+            "127.0.0.1::4318",
+            "--volume",
+            `${configPath}:/etc/otelcol/config.yaml:ro`,
+            "otel/opentelemetry-collector-contrib:0.159.0",
+            "--config=/etc/otelcol/config.yaml",
+        ]);
+        running = true;
+        const address = decodeDockerPort(await command(["docker", "port", collectorInstance, "4318/tcp"]));
+        const endpoint = new URL(`http://${address}`);
+        await waitForCollector(endpoint);
+        let stopPromise;
+        const acquiredDestination = destination;
+        const acquiredDirectory = directory;
+        const collectorTelemetry = () => traversedTelemetry(acquiredDestination.telemetry(), collectorTraversal);
+        return {
+            endpoint,
+            telemetry: collectorTelemetry,
+            destinationTelemetry: collectorTelemetry,
+            destinationEndpoint: acquiredDestination.endpoint,
+            collectorInstance,
+            awaitDestination: async (runId, eventRunIdAttribute = "run.id") => {
+                for (let attempt = 0; attempt < 40; attempt++) {
+                    if (collectorTelemetry().logs.some((log) => log.attributes.get(eventRunIdAttribute) === runId)) {
+                        return;
+                    }
+                    await new Promise((resolve) => setTimeout(resolve, 100));
+                }
+                const logs = await command(["docker", "logs", collectorInstance]);
+                throw collectorError(`Collector destination did not observe run ${runId}. Collector logs: ${logs}`, runId);
+            },
+            destinationReceipt: (runId, binding) => assessTelemetryDestination({
+                topology: "local",
+                runId,
+                identity: binding.identity,
+                observationId: collectorInstance,
+                readback: collectorTelemetry,
+            }),
+            stop: () => {
+                stopPromise ??= Promise.all([
+                    running
+                        ? command(["docker", "rm", "--force", collectorInstance]).then(() => undefined)
+                        : Promise.resolve(),
+                    acquiredDestination.stop(),
+                ]).then(() => rm(acquiredDirectory, { recursive: true, force: true }));
+                running = false;
+                return stopPromise;
+            },
+        };
+    }
+    catch (cause) {
+        if (running) {
+            await command(["docker", "rm", "--force", collectorInstance]).catch(() => undefined);
+        }
+        if (destination !== undefined)
+            await destination.stop();
+        if (directory !== undefined)
+            await rm(directory, { recursive: true, force: true });
+        throw collectorError(`The isolated Collector could not start: ${String(cause)}`, cause);
+    }
+};

package/dist/node/Observability.d.ts

@@ -0,0 +1,49 @@
+import { Context, Effect, Layer, ManagedRuntime } from "effect";
+import { OtlpExporter } from "effect/unstable/observability";
+import { AuditPublisher } from "../audit/AuditPublisher.js";
+import { TelemetryEventSink } from "../contract/EventProducer.js";
+import type { Metrics } from "../Metrics.js";
+import { type EnvBootstrapInput, type NodeObservabilityConfig, type NodeObservabilityConfigEnabled } from "../profile/ObservabilityConfig.js";
+import type { AdapterRegistration, LifecycleReport, TestingAdapterRegistration } from "../profile/ObservabilityAdapter.js";
+import { ObservabilityLifecycleError } from "../profile/LifecycleRegistry.js";
+import type { DuplicateReleaseVariable } from "../profile/ObservabilityConfigError.js";
+import { InvalidObservabilityConfig } from "../profile/ObservabilityConfigError.js";
+export type NodeObservabilityDisabled = {
+    readonly enabled: false;
+    readonly eventLayer: Layer.Layer<TelemetryEventSink>;
+    readonly auditLayer: Layer.Layer<AuditPublisher>;
+    readonly metrics: Metrics;
+    readonly flush: () => Promise<LifecycleReport>;
+    readonly close: () => Promise<LifecycleReport>;
+    readonly dispose: () => Promise<LifecycleReport>;
+    readonly [Symbol.asyncDispose]: () => Promise<void>;
+};
+export type NodeObservabilityEnabled = {
+    readonly enabled: true;
+    readonly config: NodeObservabilityConfigEnabled;
+    readonly runtime: ManagedRuntime.ManagedRuntime<OtlpExporter.Flusher, InvalidObservabilityConfig>;
+    readonly eventLayer: Layer.Layer<TelemetryEventSink>;
+    readonly auditLayer: Layer.Layer<AuditPublisher>;
+    readonly metrics: Metrics;
+    readonly flush: () => Promise<LifecycleReport>;
+    readonly close: () => Promise<LifecycleReport>;
+    readonly dispose: () => Promise<LifecycleReport>;
+    readonly [Symbol.asyncDispose]: () => Promise<void>;
+};
+export type NodeObservability = NodeObservabilityDisabled | NodeObservabilityEnabled;
+export declare const acquireRuntimeFlusher: (runtime: ManagedRuntime.ManagedRuntime<OtlpExporter.Flusher, InvalidObservabilityConfig>) => Effect.Effect<{
+    readonly flush: Effect.Effect<void>;
+    readonly register: (run: Effect.Effect<void>) => Effect.Effect<void, never, import("effect/Scope").Scope>;
+}, InvalidObservabilityConfig | ObservabilityLifecycleError, never>;
+export declare const makeNodeObservability: (config: NodeObservabilityConfig, registrations: ReadonlyArray<AdapterRegistration>) => Effect.Effect<NodeObservability, InvalidObservabilityConfig | ObservabilityLifecycleError>;
+export declare const createNodeObservabilityFromConfig: (config: NodeObservabilityConfig, registrations: ReadonlyArray<AdapterRegistration>) => Promise<NodeObservability>;
+export declare const createTestingNodeObservabilityFromConfig: (config: NodeObservabilityConfig, registrations: ReadonlyArray<TestingAdapterRegistration>) => Promise<NodeObservability>;
+export type CreateNodeObservabilityInput = EnvBootstrapInput & {
+    readonly adapters: ReadonlyArray<AdapterRegistration>;
+};
+export declare const createNodeObservability: (input: CreateNodeObservabilityInput) => Promise<NodeObservability>;
+declare const NodeObservabilityService_base: Context.ServiceClass<NodeObservabilityService, "@equipe-tech/observability/node/NodeObservability", NodeObservability>;
+export declare class NodeObservabilityService extends NodeObservabilityService_base {
+}
+export declare const layerNodeObservability: (input: CreateNodeObservabilityInput) => Layer.Layer<NodeObservabilityService, InvalidObservabilityConfig | DuplicateReleaseVariable | ObservabilityLifecycleError>;
+export {};

package/dist/node/Observability.js

@@ -0,0 +1,190 @@
+import { Cause, Context, Duration, Effect, Layer, ManagedRuntime, Option } from "effect";
+import { OtlpExporter } from "effect/unstable/observability";
+import { AuditPublisher, unboundAuditPublisher } from "../audit/AuditPublisher.js";
+import { TelemetryEventSink } from "../contract/EventProducer.js";
+import * as Telemetry from "../Telemetry.js";
+import { createDisabledMetrics, LayerMetricsRuntime, releaseMetricsLease, } from "../MetricsRuntime.js";
+import { nodeObservabilityConfigFromEnv, } from "../profile/ObservabilityConfig.js";
+import { createLifecycleRegistry, ObservabilityLifecycleError, rollbackStartedAdapters, validateAdapterRegistrationKinds, validateAdapterRegistrations, } from "../profile/LifecycleRegistry.js";
+import { profileCapabilityRank } from "../profile/ObservabilityProfile.js";
+const emptyReport = (operation) => ({
+    operation,
+    outcomes: [],
+    durationMillis: 0,
+    degraded: false,
+});
+const closedFlush = () => Effect.runPromise(new ObservabilityLifecycleError({
+    code: "OBS_OBSERVABILITY_CLOSED",
+    message: "Observability is closed. Create a new runtime before flushing again.",
+    adapter: Option.none(),
+    cause: "flush after close",
+}));
+const noopEventLayer = Layer.succeed(TelemetryEventSink, TelemetryEventSink.of({
+    record: () => Effect.void,
+    admitBrowserBatch: () => Effect.succeed({ commit: Effect.void }),
+}));
+const noopAuditLayer = () => Layer.succeed(AuditPublisher, unboundAuditPublisher());
+const disabledHandle = () => {
+    const report = emptyReport("close");
+    const metrics = createDisabledMetrics();
+    let closePromise;
+    let closed = false;
+    const close = () => {
+        if (closePromise !== undefined)
+            return closePromise;
+        closed = true;
+        closePromise = metrics.close().then(() => report);
+        return closePromise;
+    };
+    return {
+        enabled: false,
+        eventLayer: noopEventLayer,
+        auditLayer: noopAuditLayer(),
+        metrics,
+        flush: () => (closed ? closedFlush() : Promise.resolve(emptyReport("flush"))),
+        close,
+        dispose: close,
+        [Symbol.asyncDispose]: async () => {
+            await close();
+        },
+    };
+};
+class LiveNodeObservability {
+    config;
+    runtime;
+    eventLayer;
+    auditLayer;
+    metrics;
+    runLifecycle;
+    enabled = true;
+    #closePromise;
+    #flushPromise;
+    #closed = false;
+    constructor(config, runtime, eventLayer, auditLayer, metrics, runLifecycle) {
+        this.config = config;
+        this.runtime = runtime;
+        this.eventLayer = eventLayer;
+        this.auditLayer = auditLayer;
+        this.metrics = metrics;
+        this.runLifecycle = runLifecycle;
+    }
+    flush() {
+        if (this.#closed) {
+            return closedFlush();
+        }
+        if (this.#flushPromise !== undefined) {
+            return this.#flushPromise;
+        }
+        const pending = Effect.runPromise(this.runLifecycle("flush"));
+        this.#flushPromise = pending;
+        const clear = () => {
+            if (this.#flushPromise === pending) {
+                this.#flushPromise = undefined;
+            }
+        };
+        pending.then(clear, clear);
+        return pending;
+    }
+    close() {
+        if (this.#closePromise !== undefined) {
+            return this.#closePromise;
+        }
+        this.#closed = true;
+        const close = () => releaseMetricsLease(this.metrics).then(() => Effect.runPromise(this.runLifecycle("close")));
+        const pendingFlush = this.#flushPromise;
+        this.#closePromise = pendingFlush === undefined ? close() : pendingFlush.then(close, close);
+        return this.#closePromise;
+    }
+    dispose() {
+        return this.close();
+    }
+    async [Symbol.asyncDispose]() {
+        await this.close();
+    }
+}
+const nodeMetricsFlushTimeoutMilliseconds = 400;
+export const acquireRuntimeFlusher = Effect.fn("acquireRuntimeFlusher")(function* (runtime) {
+    const acquisition = yield* Effect.promise(() => runtime.runPromiseExit(OtlpExporter.Flusher));
+    if (acquisition._tag === "Failure") {
+        yield* runtime.disposeEffect.pipe(Effect.catchCause(() => Effect.void));
+        return yield* Option.getOrElse(Cause.findErrorOption(acquisition.cause), () => new ObservabilityLifecycleError({
+            code: "OBS_OBSERVABILITY_STARTUP_FAILED",
+            message: "The built-in OpenTelemetry runtime failed to start. Verify the endpoint configuration.",
+            adapter: Option.none(),
+            cause: acquisition.cause,
+        }));
+    }
+    return acquisition.value;
+});
+const makeNodeObservabilityWithOptions = Effect.fn("makeNodeObservability")(function* (config, registrations, options) {
+    if (!config.enabled) {
+        yield* validateAdapterRegistrationKinds(registrations, options);
+        return disabledHandle();
+    }
+    yield* validateAdapterRegistrations(config.profile, config.identity.environment, registrations, options);
+    const runtime = ManagedRuntime.make(Telemetry.layer(config.telemetry, {
+        shutdownTimeout: Duration.millis(nodeMetricsFlushTimeoutMilliseconds),
+        policy: config.evlog.policy,
+        contract: config.evlog.contract,
+    }));
+    const flusher = yield* acquireRuntimeFlusher(runtime);
+    const metricsRuntime = yield* Effect.promise(() => runtime.runPromise(Effect.gen(function* () {
+        return yield* LayerMetricsRuntime;
+    })));
+    const metrics = metricsRuntime.acquireMetrics();
+    const context = {
+        profile: config.profile,
+        identity: config.identity,
+        telemetryConfig: config.telemetry,
+        contract: config.evlog.contract,
+        policy: config.evlog.policy,
+        sentry: config.sentry,
+        runtime,
+    };
+    const started = [];
+    for (const registration of registrations.toSorted((left, right) => {
+        const leftStage = config.profile.stages.indexOf(left.adapter.stage);
+        const rightStage = config.profile.stages.indexOf(right.adapter.stage);
+        return (leftStage - rightStage ||
+            profileCapabilityRank(config.profile, left.adapter.stage, left.adapter.capability) -
+                profileCapabilityRank(config.profile, right.adapter.stage, right.adapter.capability));
+    })) {
+        const result = yield* registration.adapter.start(context).pipe(Effect.exit);
+        if (result._tag === "Failure") {
+            yield* rollbackStartedAdapters(started);
+            yield* Effect.promise(() => releaseMetricsLease(metrics));
+            yield* runtime.disposeEffect;
+            return yield* new ObservabilityLifecycleError({
+                code: "OBS_OBSERVABILITY_STARTUP_FAILED",
+                message: `Observability adapter "${registration.adapter.name}" failed during startup. Fix its local configuration before retrying.`,
+                adapter: Option.some(registration.adapter.name),
+                cause: Option.getOrElse(Cause.findErrorOption(result.cause), () => result.cause),
+            });
+        }
+        started.push({ registration, handle: result.value });
+    }
+    const eventHandle = started.find((entry) => entry.registration.adapter.capability === "events")?.handle;
+    const eventLayer = eventHandle?.eventLayer ?? Option.none();
+    const auditLayer = Option.getOrElse(eventHandle?.auditLayer ?? Option.none(), noopAuditLayer);
+    if (Option.isNone(eventLayer)) {
+        yield* rollbackStartedAdapters(started);
+        yield* Effect.promise(() => releaseMetricsLease(metrics));
+        yield* runtime.disposeEffect;
+        return yield* new ObservabilityLifecycleError({
+            code: "OBS_OBSERVABILITY_STARTUP_FAILED",
+            message: "The events adapter did not provide its TelemetryEventSink layer. Use an official events adapter that owns event delivery.",
+            adapter: Option.fromNullishOr(started.find((entry) => entry.registration.adapter.capability === "events")?.registration
+                .adapter.name),
+            cause: "missing events service layer",
+        });
+    }
+    const registry = createLifecycleRegistry(config.profile, started, flusher, runtime.disposeEffect);
+    return new LiveNodeObservability(config, runtime, eventLayer.value, auditLayer, metrics, registry.run);
+});
+export const makeNodeObservability = (config, registrations) => makeNodeObservabilityWithOptions(config, registrations, { allowTesting: false });
+export const createNodeObservabilityFromConfig = (config, registrations) => Effect.runPromise(makeNodeObservability(config, registrations));
+export const createTestingNodeObservabilityFromConfig = (config, registrations) => Effect.runPromise(makeNodeObservabilityWithOptions(config, registrations, { allowTesting: true }));
+export const createNodeObservability = (input) => Effect.runPromise(nodeObservabilityConfigFromEnv(input).pipe(Effect.flatMap((config) => makeNodeObservability(config, input.adapters))));
+export class NodeObservabilityService extends Context.Service()("@equipe-tech/observability/node/NodeObservability") {
+}
+export const layerNodeObservability = (input) => Layer.effect(NodeObservabilityService, Effect.acquireRelease(nodeObservabilityConfigFromEnv(input).pipe(Effect.flatMap((config) => makeNodeObservability(config, input.adapters))), (handle) => Effect.promise(() => handle.close()).pipe(Effect.asVoid)));

package/dist/node/Runtime.d.ts

@@ -1,7 +1,10 @@
 import { Effect, Runtime } from "effect";
 import type { Layer } from "effect";
+import type { InvalidResourceIdentity } from "../ResourceIdentity.js";
+import type { InvalidDataPolicy } from "../policy/DataPolicyError.js";
+import type { DuplicateReleaseVariable } from "../profile/ObservabilityConfigError.js";
 import type { EnvironmentVariables, InvalidTelemetryEnvironment } from "../TelemetryConfig.js";
-export declare const layer: (env?: EnvironmentVariables) => Layer.Layer<never, InvalidTelemetryEnvironment>;
+export declare const layer: (env?: EnvironmentVariables) => Layer.Layer<never, InvalidTelemetryEnvironment | InvalidResourceIdentity | DuplicateReleaseVariable | InvalidDataPolicy>;
 export type RunMainOptions = {
     readonly env?: EnvironmentVariables;
     readonly disableErrorReporting?: boolean;

package/dist/node/index.d.ts

@@ -1,2 +1,6 @@
+export { layerNodeAuditDigest } from "./AuditDigest.js";
 export { ingestBrowserEventBatch, ingestBrowserEvents, InvalidBrowserEventBatch, parseBrowserEventBatch, type BrowserEventIngestReceipt, } from "./BrowserEventIngest.js";
+export { createNodeObservability, createNodeObservabilityFromConfig, layerNodeObservability, makeNodeObservability, NodeObservabilityService, type CreateNodeObservabilityInput, type NodeObservability, type NodeObservabilityDisabled, type NodeObservabilityEnabled, } from "./Observability.js";
+export { ObservabilityLifecycleError } from "../profile/LifecycleRegistry.js";
+export type { AdapterOutcome, AdapterRegistration, LifecycleCleanupResult, LifecycleOutcome, LifecycleOutcomeResult, LifecycleReport, OfficialAdapterRegistration, RuntimeDisposalOutcome, } from "../profile/ObservabilityAdapter.js";
 export { layer, runMain, type RunMainOptions } from "./Runtime.js";

package/dist/node/index.js

@@ -1,2 +1,5 @@
+export { layerNodeAuditDigest } from "./AuditDigest.js";
 export { ingestBrowserEventBatch, ingestBrowserEvents, InvalidBrowserEventBatch, parseBrowserEventBatch, } from "./BrowserEventIngest.js";
+export { createNodeObservability, createNodeObservabilityFromConfig, layerNodeObservability, makeNodeObservability, NodeObservabilityService, } from "./Observability.js";
+export { ObservabilityLifecycleError } from "../profile/LifecycleRegistry.js";
 export { layer, runMain } from "./Runtime.js";

package/dist/policy/BrowserFieldPolicy.d.ts

@@ -0,0 +1,7 @@
+import { type BrowserEventFields } from "../BrowserEvents.js";
+import type { EventAttributes } from "../contract/TelemetryEvent.js";
+export declare const maxOriginalStringLength = 16384;
+export declare const replaceStructuredAssignments: (value: string) => string;
+export declare const replaceStructuredText: (value: string) => string;
+export declare const sanitizeBrowserFields: (fields: EventAttributes) => BrowserEventFields;
+export declare const sanitizeEventName: (name: string) => string;