@cratis/chronicle 6.33.0 → 6.35.0

package/connection/for_TraceContextMiddleware/when_calling/with_active_context.spec.ts

@@ -0,0 +1,235 @@
+// Copyright (c) Cratis. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+import { afterEach, beforeEach, chai, describe, it, vi } from 'vitest';
+import { context, propagation, trace } from '@opentelemetry/api';
+import { createServer, ServerError, Status, type ServiceImplementation } from 'nice-grpc';
+import { Metadata } from 'nice-grpc-common';
+import { ConnectionServiceDefinition, EventSequencesDefinition, EventStoresDefinition, ReactorsDefinition } from '@cratis/chronicle.contracts';
+import { ChronicleConnection } from '../../ChronicleConnection.js';
+import { OAuthTokenProvider } from '../../TokenProvider.js';
+import { CorrelationId, correlationIdManager } from '../../../correlation/index.js';
+import { telemetrySession } from '../../../telemetry/for_ChronicleTelemetry/given/a_telemetry_session.fixture.js';
+
+const should = chai.should();
+const telemetry = telemetrySession();
+const correlation = 'f5a978af-cd69-4a98-9f1e-7f656cc8ba2a';
+let server: ReturnType<typeof createServer>;
+let connection: ChronicleConnection;
+let received: Metadata[];
+let original: Metadata;
+let traceparent: string;
+let address: string;
+let rejectNext: boolean;
+
+beforeEach(async () => {
+    received = [];
+    rejectNext = false;
+    server = createServer().use(async function* (call, callContext) {
+        received.push(Metadata(callContext.metadata));
+        return yield* call.next(call.request, callContext);
+    });
+    const unused = async () => { throw new Error('Unexpected RPC'); };
+    server.add(ConnectionServiceDefinition, {
+        ...Object.fromEntries(Object.keys(ConnectionServiceDefinition.methods).map(name => [name, unused])),
+        checkCompatibility: async () => ({ IsCompatible: true, Incompatibilities: [], ServerVersion: 'test' }),
+        connectionKeepAlive: async () => ({})
+    } as ServiceImplementation<typeof ConnectionServiceDefinition>);
+    server.add(EventStoresDefinition, {
+        allEventStores: async () => {
+            if (rejectNext) {
+                rejectNext = false;
+                throw new ServerError(Status.UNAUTHENTICATED, 'Refresh required');
+            }
+            return { IsAuthorized: true, Data: [] };
+        },
+        ensureEventStore: unused,
+        observeEventStores: async function* () { yield { IsAuthorized: true, Data: [] }; }
+    } as ServiceImplementation<typeof EventStoresDefinition>);
+    server.add(EventSequencesDefinition, {
+        ...Object.fromEntries(Object.keys(EventSequencesDefinition.methods).map(name => [name, unused])),
+        append: async () => ({ Response: { SequenceNumber: 0n } })
+    } as ServiceImplementation<typeof EventSequencesDefinition>);
+    server.add(ReactorsDefinition, {
+        ...Object.fromEntries(Object.keys(ReactorsDefinition.methods).map(name => [name, unused])),
+        observe: async function* (requests) {
+            for await (const _request of requests) {
+                yield { Events: [], Partition: '', ReplayState: 0 };
+                return;
+            }
+        }
+    } as ServiceImplementation<typeof ReactorsDefinition>);
+    const port = await server.listen('127.0.0.1:0');
+    address = `chronicle://127.0.0.1:${port}?disableTls=true`;
+    connection = new ChronicleConnection({ connectionString: `${address}&apiKey=api-secret` });
+    original = Metadata({ authorization: 'Bearer caller-token', 'x-custom': 'preserved',
+        'custom-bin': new Uint8Array([1, 2, 3]), baggage: 'email=private', traceparent: 'stale' });
+});
+
+afterEach(async () => {
+    connection?.dispose();
+    await server?.shutdown();
+    vi.restoreAllMocks();
+});
+
+async function inContext(action: (parent: string) => Promise<void>, correlationValue = correlation) {
+    const span = telemetry.traces.getTracer('host').startSpan('host');
+    const spanContext = span.spanContext();
+    const parent = `00-${spanContext.traceId}-${spanContext.spanId}-01`;
+    traceparent = parent;
+    const active = propagation.setBaggage(trace.setSpan(context.active(), span), propagation.createBaggage({
+        'cratis.correlation_id': { value: correlationValue }, email: { value: 'private@example.com' },
+        'cratis.event_source.id': { value: 'private-source' }, 'cratis.tenant.id': { value: 'private-tenant' }
+    }));
+    try {
+        await context.with(active, async () => {
+            await Promise.resolve();
+            await action(parent);
+            propagation.getBaggage(context.active())!.getEntry('email')!.value.should.equal('private@example.com');
+        });
+    } finally { span.end(); }
+}
+
+describe('when calling unary RPCs with active trace context and sensitive baggage', () => {
+    beforeEach(async () => {
+        await inContext(async () => {
+            await connection.connect();
+            await connection.eventStores.allEventStores({}, { metadata: original });
+            await connection.connections.connectionKeepAlive({ ConnectionId: 'test' });
+            await connection.reconnect();
+            await connection.eventStores.allEventStores({}, { metadata: original });
+        });
+    });
+    it('should detach connect and reconnect compatibility checks from caller context', () => {
+        received.should.have.lengthOf(5);
+        for (const index of [0, 3]) {
+            should.equal(received[index].get('traceparent'), undefined);
+            should.equal(received[index].get('baggage'), undefined);
+            should.equal(received[index].get('api-key'), 'api-secret');
+        }
+    });
+    it('should inject trace context and only correlation baggage on business calls and keep-alive', () => {
+        for (const index of [1, 2, 4]) {
+            should.equal(received[index].get('traceparent'), traceparent);
+            should.equal(received[index].get('baggage'), `cratis.correlation_id=${correlation}`);
+            should.equal(received[index].get('api-key'), 'api-secret');
+        }
+    });
+    it('should preserve authentication and unrelated string and binary metadata', () => {
+        const metadata = received[1];
+        should.equal(metadata.get('authorization'), 'Bearer caller-token');
+        should.equal(metadata.get('x-custom'), 'preserved');
+        Array.from(metadata.get('custom-bin')!).should.deep.equal([1, 2, 3]);
+    });
+    it('should not mutate the caller carrier', () => {
+        should.equal(original.get('traceparent'), 'stale');
+        should.equal(original.get('baggage'), 'email=private');
+        should.equal(original.get('api-key'), undefined);
+    });
+});
+
+describe('when calling an event sequence without a cached compatibility verdict', () => {
+    beforeEach(async () => {
+        await connection.resetChannel();
+        await correlationIdManager.run(new CorrelationId(correlation), () => inContext(async () => {
+            await connection.eventSequences.append({}, { metadata: original });
+            correlationIdManager.scoped!.toString().should.equal(correlation);
+        }));
+    });
+    it('should detach the shared compatibility check while preserving the business caller context', () => {
+        received.should.have.lengthOf(2);
+        should.equal(received[0].get('traceparent'), undefined);
+        should.equal(received[0].get('baggage'), undefined);
+        should.equal(received[1].get('traceparent'), traceparent);
+        should.equal(received[1].get('baggage'), `cratis.correlation_id=${correlation}`);
+        should.equal(received[1].get('authorization'), 'Bearer caller-token');
+    });
+});
+
+describe('when opening server and bidirectional streams', () => {
+    beforeEach(async () => {
+        await connection.connect();
+        received.length = 0;
+        await inContext(async () => {
+            for await (const _response of connection.eventStores.observeEventStores({}, { metadata: original })) { break; }
+            async function* requests() { yield { Content: undefined }; }
+            for await (const _response of connection.reactors.observe(requests(), { metadata: original })) { break; }
+        });
+    });
+    it('should propagate the stream-opening context and preserve authentication', () => {
+        received.should.have.lengthOf(2);
+        for (const metadata of received) {
+            should.equal(metadata.get('traceparent'), traceparent);
+            should.equal(metadata.get('baggage'), `cratis.correlation_id=${correlation}`);
+            should.equal(metadata.get('authorization'), 'Bearer caller-token');
+        }
+    });
+});
+
+describe('when authentication refresh retries a unary call', () => {
+    beforeEach(async () => {
+        connection.dispose();
+        vi.spyOn(OAuthTokenProvider.prototype, 'getAccessToken').mockResolvedValue('old-token');
+        vi.spyOn(OAuthTokenProvider.prototype, 'refresh').mockResolvedValue('refreshed-token');
+        connection = new ChronicleConnection({ connectionString: address });
+        await connection.connect();
+        received.length = 0;
+        rejectNext = true;
+        await inContext(async () => {
+            await connection.eventStores.allEventStores({}, { metadata: original });
+        });
+    });
+    it('should keep trace context and filtered baggage on both attempts with the refreshed authorization', () => {
+        received.should.have.lengthOf(2);
+        for (const metadata of received) {
+            should.equal(metadata.get('traceparent'), traceparent);
+            should.equal(metadata.get('baggage'), `cratis.correlation_id=${correlation}`);
+        }
+        should.equal(received[0].get('authorization'), 'Bearer old-token');
+        should.equal(received[1].get('authorization'), 'Bearer refreshed-token');
+    });
+});
+
+describe('when callers in two contexts use the same connection concurrently', () => {
+    const expected = new Map<string, { parent: string; correlationId: string }>();
+    beforeEach(async () => {
+        await connection.connect();
+        received.length = 0;
+        expected.clear();
+        const correlations = [correlation, '7144abf3-94bd-4429-93e5-3faf23f0bad1'];
+        let ready = 0;
+        let release!: () => void;
+        const bothReady = new Promise<void>(resolve => { release = resolve; });
+        await Promise.all(correlations.map((correlationId, index) =>
+            correlationIdManager.run(new CorrelationId(correlationId), () => inContext(async parent => {
+                const id = index.toString();
+                expected.set(id, { parent, correlationId });
+                if (++ready === 2) release();
+                await bothReady;
+                await connection.eventStores.allEventStores({}, { metadata: Metadata({ 'x-call': id }) });
+                correlationIdManager.scoped!.toString().should.equal(correlationId);
+            }, correlationId))));
+    });
+    it('should propagate each caller trace and business correlation without cross-contamination', () => {
+        received.should.have.lengthOf(2);
+        for (const metadata of received) {
+            const caller = expected.get(metadata.get('x-call')!)!;
+            should.equal(metadata.get('traceparent'), caller.parent);
+            should.equal(metadata.get('baggage'), `cratis.correlation_id=${caller.correlationId}`);
+        }
+        new Set(received.map(metadata => metadata.get('traceparent'))).size.should.equal(2);
+    });
+});
+
+describe('when calling without an active span or baggage', () => {
+    beforeEach(async () => {
+        await connection.connect();
+        received.length = 0;
+        await connection.eventStores.allEventStores({}, { metadata: original });
+    });
+    it('should remove stale propagation without losing other metadata', () => {
+        should.equal(received[0].get('traceparent'), undefined);
+        should.equal(received[0].get('baggage'), undefined);
+        should.equal(received[0].get('authorization'), 'Bearer caller-token');
+    });
+});

package/connection/for_TraceContextMiddleware/when_calling/with_grpc_instrumentation.spec.ts

@@ -0,0 +1,102 @@
+// Copyright (c) Cratis. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+import { afterAll, beforeAll, beforeEach, chai, describe, it } from 'vitest';
+import { context, propagation, trace } from '@opentelemetry/api';
+import { GrpcInstrumentation } from '@opentelemetry/instrumentation-grpc';
+import type { ServiceImplementation } from 'nice-grpc';
+import { Metadata } from 'nice-grpc-common';
+import type { ChronicleConnection } from '../../ChronicleConnection.js';
+import { telemetrySession } from '../../../telemetry/for_ChronicleTelemetry/given/a_telemetry_session.fixture.js';
+
+const should = chai.should();
+const telemetry = telemetrySession();
+const instrumentation = new GrpcInstrumentation({ enabled: false });
+const correlation = 'f5a978af-cd69-4a98-9f1e-7f656cc8ba2a';
+let server: ReturnType<typeof import('nice-grpc').createServer>;
+let connection: ChronicleConnection;
+const received: Metadata[] = [];
+let hostTraceId: string;
+
+beforeAll(async () => {
+    instrumentation.setTracerProvider(telemetry.traces);
+    instrumentation.enable();
+    // Load grpc-js only after the real instrumentation's require hooks are installed.
+    const { createServer } = await import('nice-grpc');
+    const { ConnectionServiceDefinition, EventStoresDefinition, ReactorsDefinition } = await import('@cratis/chronicle.contracts');
+    const { ChronicleConnection } = await import('../../ChronicleConnection.js');
+    server = createServer().use(async function* (call, callContext) {
+        received.push(Metadata(callContext.metadata));
+        return yield* call.next(call.request, callContext);
+    });
+    const unused = async () => { throw new Error('Unexpected RPC'); };
+    server.add(ConnectionServiceDefinition, {
+        ...Object.fromEntries(Object.keys(ConnectionServiceDefinition.methods).map(name => [name, unused])),
+        checkCompatibility: async () => ({ IsCompatible: true, Incompatibilities: [], ServerVersion: 'test' })
+    } as ServiceImplementation<typeof ConnectionServiceDefinition>);
+    server.add(EventStoresDefinition, {
+        allEventStores: async () => ({ IsAuthorized: true, Data: [] }),
+        ensureEventStore: unused,
+        observeEventStores: async function* () {
+            yield { IsAuthorized: true, Data: [] };
+            yield { IsAuthorized: true, Data: [] };
+        }
+    } as ServiceImplementation<typeof EventStoresDefinition>);
+    server.add(ReactorsDefinition, {
+        ...Object.fromEntries(Object.keys(ReactorsDefinition.methods).map(name => [name, unused])),
+        observe: async function* (requests) {
+            for await (const _request of requests) {
+                yield { Events: [], Partition: '', ReplayState: 0 };
+            }
+        }
+    } as ServiceImplementation<typeof ReactorsDefinition>);
+    const port = await server.listen('127.0.0.1:0');
+    connection = new ChronicleConnection({ connectionString: `chronicle://127.0.0.1:${port}?disableTls=true&apiKey=test` });
+    await connection.connect();
+});
+
+afterAll(async () => {
+    connection?.dispose();
+    await server?.shutdown();
+    instrumentation.disable();
+});
+
+describe('when gRPC auto-instrumentation reinjects active context for unary and streaming calls', () => {
+    beforeEach(async () => {
+        received.length = 0;
+        const span = telemetry.traces.getTracer('host').startSpan('host');
+        hostTraceId = span.spanContext().traceId;
+        const active = propagation.setBaggage(trace.setSpan(context.active(), span), propagation.createBaggage({
+            'cratis.correlation_id': { value: correlation },
+            email: { value: 'private@example.com' },
+            'cratis.event_source.id': { value: 'private-source' },
+            'cratis.tenant.id': { value: 'private-tenant' }
+        }));
+        try {
+            await context.with(active, async () => {
+                await connection.eventStores.allEventStores({});
+                for await (const _response of connection.eventStores.observeEventStores({})) {
+                    context.active().should.equal(active);
+                    await Promise.resolve();
+                }
+                async function* requests() { yield { Content: undefined }; yield { Content: undefined }; }
+                for await (const _response of connection.reactors.observe(requests())) {
+                    context.active().should.equal(active);
+                    await Promise.resolve();
+                }
+                propagation.getBaggage(context.active())!.getEntry('email')!.value.should.equal('private@example.com');
+            });
+        } finally { span.end(); }
+    });
+    it('should never let auto-instrumentation restore forbidden baggage on the wire', () => {
+        received.should.have.lengthOf(3);
+        for (const metadata of received) {
+            should.equal(metadata.get('baggage'), `cratis.correlation_id=${correlation}`);
+            metadata.get('traceparent')!.should.contain(hostTraceId);
+        }
+    });
+    it('should actually execute the enabled gRPC instrumentation', () => {
+        telemetry.spans.getFinishedSpans().filter(span => span.instrumentationScope.name === '@opentelemetry/instrumentation-grpc')
+            .length.should.be.greaterThan(0);
+    });
+});

package/correlation/CorrelationIdManager.ts

@@ -18,11 +18,21 @@ export class CorrelationIdManager implements ICorrelationIdAccessor, ICorrelatio
         return this._storage.getStore() ?? CorrelationId.create();
     }
 
+    /** The explicitly scoped identifier, without generating an identifier when no scope exists. */
+    get scoped(): CorrelationId | undefined {
+        return this._storage.getStore();
+    }
+
     /** Runs an async or synchronous operation with a correlation id, restoring the caller afterward. */
     run<T>(correlationId: CorrelationId, callback: () => T): T {
         return this._storage.run(correlationId, callback);
     }
 
+    /** Runs work without inheriting a business correlation, restoring the caller afterward. */
+    runWithout<T>(callback: () => T): T {
+        return this._storage.exit(callback);
+    }
+
     /** @inheritdoc */
     setCurrent(correlationId: CorrelationId): void {
         this._storage.enterWith(correlationId);

package/dist/ChronicleClient.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"ChronicleClient.d.ts","sourceRoot":"","sources":["../ChronicleClient.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAUzD,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;AACvE,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAO/C;;;;;;;;;;;;GAYG;AACH,qBAAa,eAAgB,YAAW,gBAAgB;IA0BxC,QAAQ,CAAC,OAAO,EAAE,gBAAgB;IAzB9C,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,sBAAsB,CAAQ;IACtD,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAU;IAE/C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAsB;IAClD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAsC;IAC9D,OAAO,CAAC,QAAQ,CAAC,UAAU,CAA6B;IAExD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAErB;IAEH,OAAO,CAAC,eAAe,CAAC,CAAiC;IACzD,OAAO,CAAC,iBAAiB,CAAC,CAAgB;IAC1C,OAAO,CAAC,mBAAmB,CAAC,CAAgB;IAC5C,OAAO,CAAC,mBAAmB,CAAC,CAAgB;IAC5C,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,yBAAyB,CAAC,CAAkB;IACpD,OAAO,CAAC,oBAAoB,CAAS;IACrC,OAAO,CAAC,kBAAkB,CAAC,CAAQ;IACnC,OAAO,CAAC,gCAAgC,CAAK;IAE7C;;;OAGG;IACH,YAAqB,OAAO,EAAE,gBAAgB,EA8C7C;IAED,kBAAkB;IACZ,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,cAAc,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,uBAAuB,GAAG,OAAO,CAAC,WAAW,CAAC,CAwDrH;IAED,kBAAkB;IACZ,cAAc,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC,CAwBhD;IAED,kBAAkB;IAClB,OAAO,IAAI,IAAI,CAqBd;YAEa,gBAAgB;YA4ChB,eAAe;YAuBf,SAAS;IAmDvB;;;;OAIG;YACW,OAAO;YAcP,aAAa;IAyB3B,OAAO,CAAC,uBAAuB;IAW/B,OAAO,CAAC,2BAA2B;IAOnC,OAAO,CAAC,eAAe;IA2BvB,OAAO,CAAC,cAAc;IAQtB,OAAO,CAAC,uBAAuB;YAQjB,cAAc;YAkBd,oBAAoB;IAmClC,OAAO,CAAC,cAAc;IAUtB,OAAO,CAAC,0BAA0B;YAQpB,yBAAyB;CA4B1C"}
+{"version":3,"file":"ChronicleClient.d.ts","sourceRoot":"","sources":["../ChronicleClient.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAUzD,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;AACvE,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAQ/C;;;;;;;;;;;;GAYG;AACH,qBAAa,eAAgB,YAAW,gBAAgB;IAwBxC,QAAQ,CAAC,OAAO,EAAE,gBAAgB;IAvB9C,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,sBAAsB,CAAQ;IACtD,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAU;IAE/C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAsB;IAClD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAsC;IAC9D,OAAO,CAAC,QAAQ,CAAC,UAAU,CAA6B;IAExD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAkC;IAE1D,OAAO,CAAC,eAAe,CAAC,CAAiC;IACzD,OAAO,CAAC,iBAAiB,CAAC,CAAgB;IAC1C,OAAO,CAAC,mBAAmB,CAAC,CAAgB;IAC5C,OAAO,CAAC,mBAAmB,CAAC,CAAgB;IAC5C,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,yBAAyB,CAAC,CAAkB;IACpD,OAAO,CAAC,oBAAoB,CAAS;IACrC,OAAO,CAAC,kBAAkB,CAAC,CAAQ;IACnC,OAAO,CAAC,gCAAgC,CAAK;IAE7C;;;OAGG;IACH,YAAqB,OAAO,EAAE,gBAAgB,EAiD7C;IAED,kBAAkB;IACZ,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,cAAc,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,uBAAuB,GAAG,OAAO,CAAC,WAAW,CAAC,CA6DrH;IAED,kBAAkB;IACZ,cAAc,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC,CAuBhD;IAED,kBAAkB;IAClB,OAAO,IAAI,IAAI,CAqBd;YAEa,gBAAgB;YA4ChB,eAAe;YAuBf,SAAS;IAmDvB;;;;OAIG;YACW,OAAO;YAcP,aAAa;IAyB3B,OAAO,CAAC,uBAAuB;IAW/B,OAAO,CAAC,2BAA2B;IAOnC,OAAO,CAAC,eAAe;IA2BvB,OAAO,CAAC,cAAc;IAQtB,OAAO,CAAC,uBAAuB;YAQjB,cAAc;YAkBd,oBAAoB;IAmClC,OAAO,CAAC,cAAc;IAUtB,OAAO,CAAC,0BAA0B;YAQpB,yBAAyB;CA4B1C"}

package/dist/ChronicleClient.js

@@ -1,7 +1,9 @@
 // Copyright (c) Cratis. All rights reserved.
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 import * as os from 'os';
-import { diag } from '@opentelemetry/api';
+import { createLogger } from './logging/createLogger.js';
+import { setTelemetryAttribute, recordSafeException } from './telemetry/spanAttributes.js';
+import { WellKnownTelemetryNames } from './WellKnownTelemetryNames.js';
 import { SpanStatusCode } from '@opentelemetry/api';
 import { ChronicleConnection } from './connection/index.js';
 import { IncompatibleChronicleServer } from './connection/IncompatibleChronicleServer.js';
@@ -15,7 +17,8 @@ import { EventStore } from './EventStore.js';
 import { EventStoreName } from './EventStoreName.js';
 import { EventStoreNamespaceName } from './EventStoreNamespaceName.js';
 import { ChronicleMetrics } from './Metrics.js';
-import { ChronicleTracer } from './Tracing.js';
+import { observeOperation } from './telemetry/observeOperation.js';
+import { runInBackgroundContext } from './telemetry/runInBackgroundContext.js';
 import { TypeDiscoverer } from './types/index.js';
 import { takeUnregisteredModelBoundMappings } from './types/modelBoundPropertyMetadata.js';
 import { reachableReadModelTypes } from './readModels/rootReadModelTypes.js';
@@ -39,9 +42,7 @@ export class ChronicleClient {
     _connection;
     _stores = new Map();
     _lifecycle = new ConnectionLifecycle();
-    _logger = diag.createComponentLogger({
-        namespace: '@cratis/chronicle/ChronicleClient'
-    });
+    _logger;
     _watchdogHandle;
     _connectOperation;
     _reconnectOperation;
@@ -57,6 +58,7 @@ export class ChronicleClient {
      */
     constructor(options) {
         this.options = options;
+        this._logger = createLogger('@cratis/chronicle/ChronicleClient', options.logger);
         // When TLS is disabled, pass pre-built insecure credentials directly.
         // @cratis/chronicle.contracts ≤15.24.3 always composes call credentials with
         // channel credentials, which gRPC forbids for insecure channels. Passing
@@ -65,15 +67,17 @@ export class ChronicleClient {
         const connectionOptions = options.connectionString.disableTls
             ? { connectionString: options.connectionString, credentials: options.connectionString.createCredentials() }
             : { connectionString: options.connectionString };
-        this._connection = new ChronicleConnection(connectionOptions);
+        this._connection = new ChronicleConnection({ ...connectionOptions, logger: options.logger });
         this._logger.info('Created Chronicle client', {
             serverAddress: `${options.connectionString.serverAddress.host}:${options.connectionString.serverAddress.port}`,
             disableTls: options.connectionString.disableTls
         });
         if (options.discoveryPatterns.length > 0) {
-            this._discoveryOperation = TypeDiscoverer.default.discover(options.discoveryPatterns);
-            this._discoveryOperation.catch(error => {
-                this._logger.error('Artifact file discovery failed', { error: String(error) });
+            runInBackgroundContext(() => {
+                this._discoveryOperation = TypeDiscoverer.default.discover(options.discoveryPatterns);
+                this._discoveryOperation.catch(error => {
+                    this._logger.error('Artifact file discovery failed', { error });
+                });
             });
         }
         this._lifecycle.onConnected(async () => {
@@ -104,9 +108,9 @@ export class ChronicleClient {
             : typeof namespace === 'string'
                 ? new EventStoreNamespaceName(namespace)
                 : namespace;
-        return ChronicleTracer.startActiveSpan('chronicle.client.get_event_store', async (span) => {
-            span.setAttribute('chronicle.event_store', storeName.value);
-            span.setAttribute('chronicle.namespace', namespaceName.value);
+        return observeOperation(WellKnownTelemetryNames.spans.getEventStore, async (span) => {
+            setTelemetryAttribute(span, 'eventStore', storeName.value);
+            setTelemetryAttribute(span, 'namespace', namespaceName.value);
             try {
                 const store = await this.withReconnect('get_event_store', async () => {
                     await this.ensureConnected();
@@ -123,10 +127,18 @@ export class ChronicleClient {
                         eventStore: storeName.value
                     });
                     ensureCommandSuccess('ensure event store', await this._connection.eventStores.ensureEventStore({ Name: storeName.value }));
-                    const created = new EventStore(storeName, namespaceName, this._connection, this._lifecycle, this.options.defaultSinkTypeId, this.options.clientArtifactsProvider, this.options.reactorResultHandler, this.options.artifactActivator, this.options.readModelNamingPolicy);
+                    const created = new EventStore(storeName, namespaceName, this._connection, this._lifecycle, this.options.defaultSinkTypeId, this.options.clientArtifactsProvider, this.options.reactorResultHandler, this.options.artifactActivator, this.options.readModelNamingPolicy, this.options.telemetry);
                     this._stores.set(key, created);
-                    await this.registerArtifactsForStore(created, 'new-store');
-                    return created;
+                    try {
+                        await this.registerArtifactsForStore(created, 'new-store');
+                        return created;
+                    }
+                    catch (error) {
+                        if (this._stores.get(key) === created)
+                            this._stores.delete(key);
+                        created.disposeObservations();
+                        throw error;
+                    }
                 });
                 ChronicleMetrics.eventStoreRetrievals.add(1, {
                     'chronicle.event_store': storeName.value,
@@ -136,12 +148,11 @@ export class ChronicleClient {
                 return store;
             }
             catch (error) {
-                span.setStatus({ code: SpanStatusCode.ERROR, message: String(error) });
-                span.recordException(error);
+                recordSafeException(span, error);
                 this._logger.error('Failed getting event store', {
                     eventStore: storeName.value,
                     namespace: namespaceName.value,
-                    error: this.toErrorMessage(error)
+                    error
                 });
                 throw error;
             }
@@ -152,7 +163,7 @@ export class ChronicleClient {
     }
     /** @inheritdoc */
     async getEventStores() {
-        return ChronicleTracer.startActiveSpan('chronicle.client.get_event_stores', async (span) => {
+        return observeOperation(WellKnownTelemetryNames.spans.getEventStores, async (span) => {
             try {
                 const response = await this.withReconnect('get_event_stores', async () => {
                     await this.ensureConnected();
@@ -166,10 +177,9 @@ export class ChronicleClient {
                 return result;
             }
             catch (error) {
-                span.setStatus({ code: SpanStatusCode.ERROR, message: String(error) });
-                span.recordException(error);
+                recordSafeException(span, error);
                 this._logger.error('Failed retrieving event stores', {
-                    error: this.toErrorMessage(error)
+                    error
                 });
                 throw error;
             }
@@ -190,7 +200,7 @@ export class ChronicleClient {
         if (this._lifecycle.isConnected) {
             void this._lifecycle.disconnected(error => {
                 this._logger.error('Disconnected lifecycle callback failed during dispose', {
-                    error: this.toErrorMessage(error)
+                    error
                 });
             });
         }
@@ -218,7 +228,7 @@ export class ChronicleClient {
                 await this.startKernelKeepAlive();
                 await this._lifecycle.connected(error => {
                     this._logger.error('Connected lifecycle callback failed', {
-                        error: this.toErrorMessage(error)
+                        error
                     });
                 });
                 return;
@@ -249,7 +259,7 @@ export class ChronicleClient {
             return;
         }
         if (!this._connectOperation) {
-            this._connectOperation = this.connectWithRetry().finally(() => {
+            this._connectOperation = runInBackgroundContext(() => this.connectWithRetry()).finally(() => {
                 this._connectOperation = undefined;
             });
         }
@@ -259,15 +269,15 @@ export class ChronicleClient {
         if (this._connectionFailure)
             throw this._connectionFailure;
         if (!this._reconnectOperation) {
-            this._reconnectOperation = (async () => {
+            this._reconnectOperation = runInBackgroundContext(async () => {
                 this._logger.warn('Reconnecting to Chronicle kernel', {
                     reason,
-                    error: this.toErrorMessage(error)
+                    error
                 });
                 if (this._lifecycle.isConnected) {
                     await this._lifecycle.disconnected(disconnectError => {
                         this._logger.error('Disconnected lifecycle callback failed', {
-                            error: this.toErrorMessage(disconnectError)
+                            error: disconnectError
                         });
                     });
                 }
@@ -282,7 +292,7 @@ export class ChronicleClient {
                         await this.startKernelKeepAlive();
                         await this._lifecycle.connected(connectedError => {
                             this._logger.error('Connected lifecycle callback failed after reconnect', {
-                                error: this.toErrorMessage(connectedError)
+                                error: connectedError
                             });
                         });
                         return;
@@ -297,7 +307,7 @@ export class ChronicleClient {
                         await this.backOff(attempt, 'Reconnect attempt failed, retrying', reconnectError);
                     }
                 }
-            })().finally(() => {
+            }).finally(() => {
                 this._reconnectOperation = undefined;
             });
         }
@@ -314,7 +324,7 @@ export class ChronicleClient {
         this._logger.warn(message, {
             attempt,
             delayMs,
-            error: this.toErrorMessage(error)
+            error
         });
         await new Promise(resolve => setTimeout(resolve, delayMs));
         return delayMs;
@@ -394,9 +404,9 @@ export class ChronicleClient {
         return String(error);
     }
     startConnectionWatchdog() {
-        this._watchdogHandle = setInterval(() => {
+        this._watchdogHandle = runInBackgroundContext(() => setInterval(() => {
             void this.runHealthCheck().catch(error => this.backgroundConnectionFailed('watchdog-health-check', error));
-        }, ChronicleClient._healthCheckIntervalMs);
+        }, ChronicleClient._healthCheckIntervalMs));
         this._watchdogHandle.unref?.();
     }
     async runHealthCheck() {
@@ -430,7 +440,7 @@ export class ChronicleClient {
                 return;
             }
             void this.reconnect(reason, error).catch(failure => this.backgroundConnectionFailed(reason, failure));
-        });
+        }, {}, this.options.logger);
         await keepAlive.start({
             ConnectionId: this._lifecycle.connectionId,
             ClientVersion: clientVersion,
@@ -457,7 +467,7 @@ export class ChronicleClient {
         this.failConnection(error instanceof Error ? error : new Error(String(error)));
         this._logger.error('Background connection recovery failed; create a new client after correcting the server', {
             reason,
-            error: this.toErrorMessage(error)
+            error
         });
     }
     async registerArtifactsForStore(store, reason) {