@dxos/observability 0.8.3 → 0.8.4-main.1068cf700f

package/src/providers/client-observability.ts

@@ -0,0 +1,253 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Effect from 'effect/Effect';
+
+import { Event, scheduleTaskInterval } from '@dxos/async';
+import { type Client, type ClientServices } from '@dxos/client';
+import { type Space } from '@dxos/client/echo';
+import { DeviceKind } from '@dxos/client/halo';
+import { Context } from '@dxos/context';
+import { invariant } from '@dxos/invariant';
+import { log } from '@dxos/log';
+import { ConnectionState, type NetworkStatus, Platform } from '@dxos/protocols/proto/dxos/client/services';
+
+import { type DataProvider } from '../observability';
+
+const SPACE_METRICS_MIN_INTERVAL = 1000 * 60 * 10; // 10 minutes
+const NETWORK_METRICS_MIN_INTERVAL = 1000 * 60 * 10; // 10 minutes
+const RUNTIME_METRICS_MIN_INTERVAL = 1000 * 60 * 10; // 10 minutes
+
+// TODO(wittjosiah): Improve privacy of telemetry identifiers.
+//  - Identifier should be generated client-side with no attachment to identity.
+//  - Identifier can then be reset by user.
+//  - Identifier can be synced via HALO to allow for correlation of events bewteen devices.
+//  - Identifier should also be stored outside of HALO such that it is available immediately on startup.
+/** Subscribes to identity and device changes and sets observability tags accordingly. */
+export const identityProvider = (clientServices: Partial<ClientServices>): DataProvider =>
+  Effect.fn(function* (observability) {
+    // TODO(wittjosiah): RPC subscribe returns void; cleanup requires upstream API change.
+    clientServices.IdentityService!.queryIdentity().subscribe((idqr) => {
+      if (!idqr?.identity?.did) {
+        return;
+      }
+
+      observability.identify(idqr.identity.did);
+      observability.setTags({ did: idqr.identity.did });
+    });
+
+    // TODO(wittjosiah): RPC subscribe returns void; cleanup requires upstream API change.
+    clientServices.DevicesService!.queryDevices().subscribe((dqr) => {
+      if (!dqr?.devices || dqr.devices.length === 0) {
+        return;
+      }
+
+      const thisDevice = dqr.devices.find((device) => device.kind === DeviceKind.CURRENT);
+      if (!thisDevice) {
+        return;
+      }
+
+      observability.setTags({ deviceKey: thisDevice.deviceKey.truncate() });
+      if (thisDevice.profile?.label) {
+        observability.setTags({ deviceProfile: thisDevice.profile.label });
+      }
+    });
+  });
+
+/** Periodically publishes network connection and buffer metrics. */
+export const networkMetricsProvider = (clientServices: Partial<ClientServices>): DataProvider =>
+  Effect.fn(function* (observability) {
+    const ctx = new Context();
+    let lastNetworkStatus: NetworkStatus | undefined;
+
+    // TODO(nf): support type in debounce()
+    const updateSignalMetrics = new Event<NetworkStatus>().debounce(NETWORK_METRICS_MIN_INTERVAL);
+    updateSignalMetrics.on(ctx, async () => {
+      log('send signal metrics');
+      (lastNetworkStatus?.signaling as NetworkStatus.Signal[])?.forEach(({ server, state }) => {
+        observability.metrics.gauge('dxos.client.network.signal.connectionState', state, { server });
+      });
+
+      let swarmCount = 0;
+      const connectionStates = new Map<string, number>();
+      for (const state in ConnectionState) {
+        connectionStates.set(state, 0);
+      }
+
+      let totalReadBufferSize = 0;
+      let totalWriteBufferSize = 0;
+      let totalChannelBufferSize = 0;
+
+      lastNetworkStatus?.connectionInfo?.forEach((connectionInfo) => {
+        swarmCount++;
+
+        for (const conn of connectionInfo.connections ?? []) {
+          connectionStates.set(conn.state, (connectionStates.get(conn.state) ?? 0) + 1);
+          totalReadBufferSize += conn.readBufferSize ?? 0;
+          totalWriteBufferSize += conn.writeBufferSize ?? 0;
+          for (const stream of conn.streams ?? []) {
+            totalChannelBufferSize += stream.writeBufferSize ?? 0;
+          }
+        }
+
+        observability.metrics.gauge('dxos.client.network.swarm.count', swarmCount);
+        for (const state in ConnectionState) {
+          observability.metrics.gauge('dxos.client.network.connection.count', connectionStates.get(state) ?? 0, {
+            state,
+          });
+        }
+        observability.metrics.gauge('dxos.client.network.totalReadBufferSize', totalReadBufferSize);
+        observability.metrics.gauge('dxos.client.network.totalWriteBufferSize', totalWriteBufferSize);
+        observability.metrics.gauge('dxos.client.network.totalChannelBufferSize', totalChannelBufferSize);
+      });
+    });
+
+    clientServices.NetworkService!.queryStatus().subscribe((networkStatus) => {
+      lastNetworkStatus = networkStatus;
+      updateSignalMetrics.emit();
+    });
+
+    scheduleTaskInterval(ctx, async () => updateSignalMetrics.emit(), NETWORK_METRICS_MIN_INTERVAL);
+
+    return async () => {
+      await ctx.dispose();
+    };
+  });
+
+/** Periodically publishes platform and heap memory metrics. */
+export const runtimeMetricsProvider = (clientServices: Partial<ClientServices>): DataProvider =>
+  Effect.fn(function* (observability) {
+    const ctx = new Context();
+    const platform = yield* Effect.promise(() => clientServices.SystemService!.getPlatform());
+    invariant(platform, 'platform is required');
+
+    observability.setTags({
+      platformType: Platform.PLATFORM_TYPE[platform.type as number].toLowerCase(),
+      platform: platform.platform,
+      arch: platform.arch,
+      runtime: platform.runtime,
+    });
+
+    scheduleTaskInterval(
+      ctx,
+      async () => {
+        if (clientServices.constructor.name === 'WorkerClientServices') {
+          const memory = (window.performance as any).memory;
+          if (memory) {
+            observability.metrics.gauge('dxos.client.runtime.heapTotal', memory.totalJSHeapSize);
+            observability.metrics.gauge('dxos.client.runtime.heapUsed', memory.usedJSHeapSize);
+            observability.metrics.gauge('dxos.client.runtime.heapSizeLimit', memory.jsHeapSizeLimit);
+          }
+        }
+
+        clientServices.SystemService?.getPlatform()
+          .then((platform) => {
+            if (platform.memory) {
+              observability.metrics.gauge('dxos.client.services.runtime.rss', platform.memory.rss);
+              observability.metrics.gauge('dxos.client.services.runtime.heapTotal', platform.memory.heapTotal);
+              observability.metrics.gauge('dxos.client.services.runtime.heapUsed', platform.memory.heapUsed);
+            }
+          })
+          .catch((error) => log('platform error', { error }));
+      },
+      RUNTIME_METRICS_MIN_INTERVAL,
+    );
+
+    return async () => {
+      await ctx.dispose();
+    };
+  });
+
+/** Periodically publishes space membership, object count, and pipeline progress metrics. */
+export const spacesMetricsProvider = (client: Client): DataProvider =>
+  Effect.fn(function* (observability) {
+    const ctx = new Context();
+    // TODO(nf): update subscription on new spaces
+    const spaces = client.spaces.get();
+    const subscriptions = new Map<string, { unsubscribe: () => void }>();
+    ctx.onDispose(() => subscriptions.forEach((subscription) => subscription.unsubscribe()));
+
+    const updateSpaceMetrics = new Event<Space>().debounce(SPACE_METRICS_MIN_INTERVAL);
+    updateSpaceMetrics.on(ctx, async () => {
+      log('send space metrics');
+      for (const data of mapSpaces(spaces, { truncateKeys: true })) {
+        observability.metrics.gauge('dxos.client.space.members', data.members, { key: data.key });
+        observability.metrics.gauge('dxos.client.space.objects', data.objects, { key: data.key });
+        observability.metrics.gauge('dxos.client.space.epoch', data.epoch, { key: data.key });
+        observability.metrics.gauge('dxos.client.space.currentDataMutations', data.currentDataMutations, {
+          key: data.key,
+        });
+      }
+    });
+
+    const subscribeToSpaceUpdate = (space: Space) =>
+      space.pipeline.subscribe({
+        next: () => {
+          updateSpaceMetrics.emit();
+        },
+      });
+
+    spaces.forEach((space) => {
+      subscriptions.set(space.id, subscribeToSpaceUpdate(space));
+    });
+
+    client.spaces.subscribe({
+      next: async (spaces) => {
+        spaces
+          .filter((space) => !subscriptions.has(space.id))
+          .forEach((space) => {
+            subscriptions.set(space.id, subscribeToSpaceUpdate(space));
+          });
+      },
+    });
+
+    scheduleTaskInterval(ctx, async () => updateSpaceMetrics.emit(), SPACE_METRICS_MIN_INTERVAL);
+
+    return async () => {
+      await ctx.dispose();
+    };
+  });
+
+type MapSpacesOptions = {
+  verbose?: boolean;
+  truncateKeys?: boolean;
+};
+
+const mapSpaces = (spaces: Space[], options: MapSpacesOptions = { verbose: false, truncateKeys: false }) => {
+  return spaces.map((space) => {
+    // TODO(burdon): Factor out.
+    // TODO(burdon): Agent needs to restart before `ready` is available.
+    const { open, ready } = space.internal.data.metrics ?? {};
+    const startup = open && ready && ready.getTime() - open.getTime();
+
+    // TODO(burdon): Get feeds from client-services if verbose (factor out from devtools/diagnostics).
+    // const host = client.services.services.DevtoolsHost!;
+    const pipeline = space.internal.data.pipeline;
+    const startDataMutations = pipeline?.currentEpoch?.subject.assertion.timeframe.totalMessages() ?? 0;
+    const epoch = pipeline?.currentEpoch?.subject.assertion.number;
+    // const appliedEpoch = pipeline?.appliedEpoch?.subject.assertion.number;
+    const currentDataMutations = pipeline?.currentDataTimeframe?.totalMessages() ?? 0;
+    const totalDataMutations = pipeline?.targetDataTimeframe?.totalMessages() ?? 0;
+
+    return {
+      // TODO(nf): truncate keys for DD?
+      key: space.key.truncate(),
+      open: space.isOpen,
+      members: space.members.get().length,
+      objects: space.internal.db.coreDatabase.getAllObjectIds().length,
+      startup,
+      epoch,
+      // appliedEpoch,
+      startDataMutations,
+      currentDataMutations,
+      totalDataMutations,
+
+      // TODO(burdon): Negative?
+      progress: (
+        Math.min(Math.abs((currentDataMutations - startDataMutations) / (totalDataMutations - startDataMutations)), 1) *
+        100
+      ).toFixed(0),
+    };
+  });
+};

package/src/providers/index.ts

@@ -0,0 +1,7 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+export * as Client from './client-observability';
+export * as IPData from './ip-data';
+export * as Storage from './storage';

package/src/providers/ip-data.ts

@@ -0,0 +1,88 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as FetchHttpClient from '@effect/platform/FetchHttpClient';
+import * as HttpClient from '@effect/platform/HttpClient';
+import * as HttpClientRequest from '@effect/platform/HttpClientRequest';
+import * as Effect from 'effect/Effect';
+import * as Schema from 'effect/Schema';
+// NOTE: localStorage is not available in web workers.
+import * as localForage from 'localforage';
+
+import { type Config } from '@dxos/config';
+import { log } from '@dxos/log';
+
+import { type DataProvider } from '../observability';
+
+const IP_DATA_CACHE_TIMEOUT = 6 * 60 * 60 * 1000; // 6 hours
+
+const IPData = Schema.Struct({
+  city: Schema.String,
+  region: Schema.String,
+  country: Schema.String,
+  latitude: Schema.optional(Schema.Number),
+  longitude: Schema.optional(Schema.Number),
+});
+type IPData = Schema.Schema.Type<typeof IPData>;
+
+type CachedIPData = {
+  data: IPData;
+  timestamp: number;
+};
+
+const getIPData = Effect.fn(function* (config: Config) {
+  const httpClient = yield* HttpClient.HttpClient;
+
+  // Check cache first.
+  const cachedData = yield* Effect.promise(() => localForage.getItem<CachedIPData>('dxos:observability:ipdata'));
+  if (cachedData && cachedData.timestamp > Date.now() - IP_DATA_CACHE_TIMEOUT) {
+    return cachedData.data;
+  }
+
+  // Fetch data if not cached.
+  const IPDATA_API_KEY = config.get('runtime.app.env.DX_IPDATA_API_KEY');
+  if (IPDATA_API_KEY) {
+    const data = yield* HttpClientRequest.get(`https://api.ipdata.co?api-key=${IPDATA_API_KEY}`).pipe(
+      httpClient.execute,
+      Effect.flatMap((res) => res.json),
+      Effect.flatMap(Schema.decodeUnknown(IPData)),
+    );
+
+    // Cache data.
+    yield* Effect.promise(() =>
+      localForage.setItem('dxos:observability:ipdata', {
+        data,
+        timestamp: Date.now(),
+      }),
+    );
+
+    return data;
+  }
+});
+
+/** Fetches IP geolocation data and sets city/region/country tags on the observability instance. */
+export const provider =
+  (config: Config): DataProvider =>
+  (observability) =>
+    Effect.gen(function* () {
+      const ipData = yield* getIPData(config);
+      if (!ipData) {
+        return;
+      }
+
+      observability.setTags({
+        city: ipData.city,
+        region: ipData.region,
+        country: ipData.country,
+        latitude: ipData.latitude,
+        longitude: ipData.longitude,
+      });
+    }).pipe(
+      Effect.provide(FetchHttpClient.layer),
+      Effect.catchAll((err) =>
+        Effect.sync(() => {
+          log.catch(err);
+        }),
+      ),
+    );

package/src/providers/storage.ts

@@ -0,0 +1,23 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Duration from 'effect/Duration';
+import * as Effect from 'effect/Effect';
+import * as Fiber from 'effect/Fiber';
+import * as Schedule from 'effect/Schedule';
+
+import { type DataProvider } from '../observability';
+
+export const provider: DataProvider = Effect.fn(function* (observability) {
+  if (typeof navigator !== 'undefined' && navigator.storage?.estimate) {
+    const action = Effect.gen(function* () {
+      const storageEstimate = yield* Effect.tryPromise(() => navigator.storage.estimate());
+      storageEstimate.usage && observability.metrics.gauge('storageUsage', storageEstimate.usage);
+      storageEstimate.quota && observability.metrics.gauge('storageQuota', storageEstimate.quota);
+    });
+
+    const fiber = action.pipe(Effect.repeat(Schedule.fixed(Duration.hours(1))), Effect.runFork);
+    return () => Effect.runSync(Fiber.interrupt(fiber));
+  }
+});

package/src/storage/browser.ts

@@ -0,0 +1,61 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+// NOTE: localStorage is not available in web workers.
+import * as localForage from 'localforage';
+
+import { log } from '@dxos/log';
+
+const OBSERVABILITY_DISABLED_KEY = 'observability-disabled';
+const OBSERVABILITY_GROUP_KEY = 'observability-group';
+
+/** No-op in browser contexts. */
+export const showObservabilityBanner = () => {
+  log.warn('showObservabilityBanner is not supported in browser contexts.');
+};
+
+/**
+ * @param namespace - localForage key prefix used to scope the observability state in browser storage.
+ */
+export const isObservabilityDisabled = async (namespace: string): Promise<boolean> => {
+  try {
+    return (await localForage.getItem(`${namespace}:${OBSERVABILITY_DISABLED_KEY}`)) === 'true';
+  } catch (err) {
+    log.catch('Failed to check if observability is disabled, assuming it is', err);
+    return true;
+  }
+};
+
+/**
+ * @param namespace - localForage key prefix used to scope the observability state in browser storage.
+ */
+export const storeObservabilityDisabled = async (namespace: string, value: boolean) => {
+  try {
+    await localForage.setItem(`${namespace}:${OBSERVABILITY_DISABLED_KEY}`, String(value));
+  } catch (err) {
+    log.catch('Failed to store observability disabled', err);
+  }
+};
+
+/**
+ * @param namespace - localForage key prefix used to scope the observability state in browser storage.
+ */
+export const getObservabilityGroup = async (namespace: string): Promise<string | undefined> => {
+  try {
+    return (await localForage.getItem(`${namespace}:${OBSERVABILITY_GROUP_KEY}`)) ?? undefined;
+  } catch (err) {
+    log.catch('Failed to get observability group', err);
+  }
+};
+
+/**
+ * @param namespace - localForage key prefix used to scope the observability state in browser storage.
+ */
+export const storeObservabilityGroup = async (namespace: string, value: string) => {
+  try {
+    await localForage.setItem(`${namespace}:${OBSERVABILITY_GROUP_KEY}`, value);
+  } catch (err) {
+    log.catch('Failed to store observability group', err);
+  }
+};

package/src/{sentry → storage}/index.ts

@@ -3,4 +3,3 @@
 //
 
 export * from './node';
-export * from './types';

package/src/storage/node.test.ts

@@ -0,0 +1,130 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { existsSync } from 'node:fs';
+import { mkdir, readFile, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+
+import yaml from 'js-yaml';
+import { validate as validateUuid } from 'uuid';
+import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';
+
+import {
+  type PersistentObservabilityState,
+  getObservabilityGroup,
+  isObservabilityDisabled,
+  showObservabilityBanner,
+  storeObservabilityDisabled,
+  storeObservabilityGroup,
+} from './node';
+
+let configDir: string;
+
+beforeEach(async () => {
+  configDir = join(tmpdir(), `observability-test-${Date.now()}-${Math.random().toString(36).slice(2)}`);
+  // Clean env vars.
+  delete process.env.DX_DISABLE_OBSERVABILITY;
+  delete process.env.DX_OBSERVABILITY_GROUP;
+});
+
+afterEach(async () => {
+  if (existsSync(configDir)) {
+    await rm(configDir, { recursive: true, force: true });
+  }
+});
+
+describe('node storage', () => {
+  test('creates config directory if missing', async () => {
+    expect(existsSync(configDir)).toBe(false);
+    await isObservabilityDisabled(configDir);
+    expect(existsSync(configDir)).toBe(true);
+  });
+
+  test('initializes state with UUID and writes observability.yml', async () => {
+    await isObservabilityDisabled(configDir);
+    const content = await readFile(join(configDir, 'observability.yml'), 'utf-8');
+    const state = yaml.load(content) as PersistentObservabilityState;
+    expect(validateUuid(state.installationId)).toBe(true);
+    expect(state.disabled).toBe(false);
+  });
+
+  test('isObservabilityDisabled returns false by default', async () => {
+    const disabled = await isObservabilityDisabled(configDir);
+    expect(disabled).toBe(false);
+  });
+
+  test('storeObservabilityDisabled persists to file', async () => {
+    await storeObservabilityDisabled(configDir, true);
+    const content = await readFile(join(configDir, 'observability.yml'), 'utf-8');
+    const state = yaml.load(content) as PersistentObservabilityState;
+    expect(state.disabled).toBe(true);
+  });
+
+  test('isObservabilityDisabled reads persisted value', async () => {
+    await storeObservabilityDisabled(configDir, true);
+    const disabled = await isObservabilityDisabled(configDir);
+    expect(disabled).toBe(true);
+  });
+
+  test('getObservabilityGroup returns undefined by default', async () => {
+    const group = await getObservabilityGroup(configDir);
+    expect(group).toBeUndefined();
+  });
+
+  test('storeObservabilityGroup persists and reads back', async () => {
+    await storeObservabilityGroup(configDir, 'beta-testers');
+    const group = await getObservabilityGroup(configDir);
+    expect(group).toBe('beta-testers');
+  });
+
+  test('respects DX_DISABLE_OBSERVABILITY env var', async () => {
+    process.env.DX_DISABLE_OBSERVABILITY = 'true';
+    const disabled = await isObservabilityDisabled(configDir);
+    expect(disabled).toBe(true);
+  });
+
+  test('respects DX_OBSERVABILITY_GROUP env var on initial creation', async () => {
+    process.env.DX_OBSERVABILITY_GROUP = 'alpha';
+    const group = await getObservabilityGroup(configDir);
+    expect(group).toBe('alpha');
+  });
+
+  test('validates UUID in existing state file', async () => {
+    await mkdir(configDir, { recursive: true });
+    const state: PersistentObservabilityState = {
+      installationId: '550e8400-e29b-41d4-a716-446655440000',
+      disabled: false,
+    };
+    await writeFile(join(configDir, 'observability.yml'), yaml.dump(state), 'utf-8');
+    const disabled = await isObservabilityDisabled(configDir);
+    expect(disabled).toBe(false);
+  });
+
+  test('reinitializes if UUID is invalid', async () => {
+    await mkdir(configDir, { recursive: true });
+    const badState = { installationId: 'not-a-uuid', disabled: true };
+    await writeFile(join(configDir, 'observability.yml'), yaml.dump(badState), 'utf-8');
+
+    // Should reinitialize with a fresh UUID.
+    await isObservabilityDisabled(configDir);
+    const content = await readFile(join(configDir, 'observability.yml'), 'utf-8');
+    const state = yaml.load(content) as PersistentObservabilityState;
+    expect(validateUuid(state.installationId)).toBe(true);
+    // Reinitialized state uses env var defaults, not the persisted bad state.
+    expect(state.disabled).toBe(false);
+  });
+
+  test('showObservabilityBanner prints once then is silent', async () => {
+    await mkdir(configDir, { recursive: true });
+    const bannerCb = vi.fn();
+
+    await showObservabilityBanner(configDir, bannerCb);
+    expect(bannerCb).toHaveBeenCalledTimes(1);
+
+    bannerCb.mockClear();
+    await showObservabilityBanner(configDir, bannerCb);
+    expect(bannerCb).not.toHaveBeenCalled();
+  });
+});