@dxos/observability 0.8.4-main.ae835ea → 0.8.4-main.bc2380dfbc

package/src/helpers/browser-observability.ts

@@ -1,178 +0,0 @@
-//
-// Copyright 2022 DXOS.org
-//
-
-// NOTE: localStorage is not available in web workers.
-import * as localForage from 'localforage';
-
-// import { type Platform } from '@dxos/client-services';
-// import { type InitOptions as TelemetryInitOptions } from '@dxos/telemetry';
-
-import type { Config } from '@dxos/client';
-import { log } from '@dxos/log';
-
-import type { Mode, Observability } from '../observability';
-
-import type { IPData } from './common';
-
-export const OBSERVABILITY_DISABLED_KEY = 'observability-disabled';
-export const OBSERVABILITY_GROUP_KEY = 'observability-group';
-
-export const isObservabilityDisabled = async (namespace: string): Promise<boolean> => {
-  try {
-    return (await localForage.getItem(`${namespace}:${OBSERVABILITY_DISABLED_KEY}`)) === 'true';
-  } catch (err) {
-    log.catch('Failed to check if observability is disabled, assuming it is', err);
-    return true;
-  }
-};
-
-export const storeObservabilityDisabled = async (namespace: string, value: boolean) => {
-  try {
-    await localForage.setItem(`${namespace}:${OBSERVABILITY_DISABLED_KEY}`, String(value));
-  } catch (err) {
-    log.catch('Failed to store observability disabled', err);
-  }
-};
-
-export const getObservabilityGroup = async (namespace: string): Promise<string | undefined> => {
-  try {
-    return (await localForage.getItem(`${namespace}:${OBSERVABILITY_GROUP_KEY}`)) ?? undefined;
-  } catch (err) {
-    log.catch('Failed to get observability group', err);
-  }
-};
-
-export const storeObservabilityGroup = async (namespace: string, value: string) => {
-  try {
-    await localForage.setItem(`${namespace}:${OBSERVABILITY_GROUP_KEY}`, value);
-  } catch (err) {
-    log.catch('Failed to store observability group', err);
-  }
-};
-
-export type AppObservabilityOptions = {
-  namespace: string;
-  config: Config;
-  mode?: Mode;
-  tracingEnable?: boolean;
-  replayEnable?: boolean;
-  // TODO(nf): options for providers?
-};
-
-// TODO(wittjosiah): Store preference for disabling observability.
-//   At minimum should be stored locally (i.e., localstorage), possibly in halo preference.
-//   Needs to be hooked up to settings page for user visibility.
-export const initializeAppObservability = async ({
-  namespace,
-  config,
-  // TODO(nf): Configure mode.
-  mode = 'basic',
-  tracingEnable = false,
-  replayEnable = false,
-}: AppObservabilityOptions): Promise<Observability> => {
-  log('initializeAppObservability', { config });
-
-  // const platform = (await client.services.services.SystemService?.getPlatform()) as Platform;
-  // if (!platform) {
-  //   log.error('failed to get platform, could not initialize observability');
-  //   return undefined;
-  // }
-
-  const group = (await getObservabilityGroup(namespace)) ?? undefined;
-  const release = `${namespace}@${config.get('runtime.app.build.version')}`;
-  const environment = config.get('runtime.app.env.DX_ENVIRONMENT');
-
-  const { Observability } = await import('../observability');
-  const observability = new Observability({
-    namespace,
-    release,
-    environment,
-    group,
-    mode,
-    config,
-    errorLog: {
-      sentryInitOptions: {
-        environment,
-        release,
-        tracing: tracingEnable,
-        replay: replayEnable,
-        // TODO(wittjosiah): Configure these.
-        //   Consider using a sampling function to dynamically configure these values.
-        //   https://docs.sentry.io/platforms/javascript/configuration/sampling/#setting-a-sampling-function
-        sampleRate: 1.0,
-        replaySampleRate: 1.0,
-        replaySampleRateOnError: 1.0,
-      },
-    },
-  });
-
-  // Global kill switch.
-  const observabilityDisabled = await isObservabilityDisabled(namespace);
-  if (observabilityDisabled) {
-    observability.setMode('disabled');
-    log.info('observability disabled');
-    return observability;
-  }
-
-  try {
-    const getIPData = async (config: Config): Promise<IPData | void> => {
-      const IP_DATA_CACHE_TIMEOUT = 6 * 60 * 60 * 1000; // 6 hours
-      type CachedIPData = {
-        data: IPData;
-        timestamp: number;
-      };
-
-      // Check cache first.
-      const cachedData: null | CachedIPData = await localForage.getItem('dxos:observability:ipdata');
-      if (cachedData && cachedData.timestamp > Date.now() - IP_DATA_CACHE_TIMEOUT) {
-        return cachedData.data;
-      }
-
-      // Fetch data if not cached.
-      const IPDATA_API_KEY = config.get('runtime.app.env.DX_IPDATA_API_KEY');
-      if (IPDATA_API_KEY) {
-        return fetch(`https://api.ipdata.co?api-key=${IPDATA_API_KEY}`)
-          .then((res) => res.json())
-          .then((data) => {
-            // Cache data.
-            localForage
-              .setItem('dxos:observability:ipdata', {
-                data,
-                timestamp: Date.now(),
-              })
-              .catch((err) => observability.captureException(err));
-
-            return data;
-          })
-          .catch((err) => observability.captureException(err));
-      }
-    };
-
-    // TODO(nf): plugin state?
-    // TODO(nf): should provide capability to init Sentry earlier in booting process to capture errors during initialization.
-
-    await observability.initialize();
-    observability.startErrorLogs();
-
-    const ipData = await getIPData(config);
-    ipData && observability.setIPDataTelemetryTags(ipData);
-
-    if (typeof navigator !== 'undefined' && navigator.storage?.estimate) {
-      // TODO(burdon): Need to close.
-      setInterval(async () => {
-        try {
-          const storageEstimate = await navigator.storage.estimate();
-          storageEstimate.usage && observability.setTag('storageUsage', storageEstimate.usage.toString(), 'telemetry');
-          storageEstimate.quota && observability.setTag('storageQuota', storageEstimate.quota.toString(), 'telemetry');
-        } catch (error) {
-          log.warn('Failed to run estimate()', error);
-        }
-      }, 10_000);
-    }
-  } catch (err: any) {
-    log.error('Failed to initialize app observability', err);
-  }
-
-  return observability;
-};

package/src/helpers/common.ts

@@ -1,38 +0,0 @@
-//
-// Copyright 2022 DXOS.org
-//
-
-import type { Client } from '@dxos/client';
-
-import { type IdentityOptions } from '../segment';
-
-export type IPData = {
-  city: string;
-  region: string;
-  country: string;
-  latitude: number;
-  longitude: number;
-};
-
-// TODO(wittjosiah): Improve privacy of telemetry identifiers.
-//  - Identifier should be generated client-side with no attachment to identity.
-//  - Identifier can then be reset by user.
-//  - Identifier can be synced via HALO to allow for correlation of events bewteen devices.
-//  - Identifier should also be stored outside of HALO such that it is available immediately on startup.
-export const getTelemetryIdentifier = (client: Client): string | undefined => {
-  if (!client?.initialized) {
-    return undefined;
-  }
-
-  const identity = client.halo.identity.get();
-  if (identity) {
-    return identity.did;
-  }
-
-  return undefined;
-};
-
-export const getTelemetryIdentity = (client: Client): IdentityOptions => {
-  const did = getTelemetryIdentifier(client);
-  return { did };
-};

package/src/helpers/index.ts

@@ -1,9 +0,0 @@
-//
-// Copyright 2022 DXOS.org
-//
-
-export * from './browser-observability';
-export * from './common';
-export * from './node-observability';
-export * from './map-spaces';
-export * from './setup-telemetry-listeners';

package/src/helpers/map-spaces.ts

@@ -1,48 +0,0 @@
-//
-// Copyright 2023 DXOS.org
-//
-
-import type { Space } from '@dxos/client-protocol';
-
-export type MapSpacesOptions = {
-  verbose?: boolean;
-  truncateKeys?: boolean;
-};
-
-export const mapSpaces = (spaces: Space[], options: MapSpacesOptions = { verbose: false, truncateKeys: false }) => {
-  return spaces.map((space) => {
-    // TODO(burdon): Factor out.
-    // TODO(burdon): Agent needs to restart before `ready` is available.
-    const { open, ready } = space.internal.data.metrics ?? {};
-    const startup = open && ready && ready.getTime() - open.getTime();
-
-    // TODO(burdon): Get feeds from client-services if verbose (factor out from devtools/diagnostics).
-    // const host = client.services.services.DevtoolsHost!;
-    const pipeline = space.internal.data.pipeline;
-    const startDataMutations = pipeline?.currentEpoch?.subject.assertion.timeframe.totalMessages() ?? 0;
-    const epoch = pipeline?.currentEpoch?.subject.assertion.number;
-    // const appliedEpoch = pipeline?.appliedEpoch?.subject.assertion.number;
-    const currentDataMutations = pipeline?.currentDataTimeframe?.totalMessages() ?? 0;
-    const totalDataMutations = pipeline?.targetDataTimeframe?.totalMessages() ?? 0;
-
-    return {
-      // TODO(nf): truncate keys for DD?
-      key: space.key.truncate(),
-      open: space.isOpen,
-      members: space.members.get().length,
-      objects: space.db.coreDatabase.getAllObjectIds().length,
-      startup,
-      epoch,
-      // appliedEpoch,
-      startDataMutations,
-      currentDataMutations,
-      totalDataMutations,
-
-      // TODO(burdon): Negative?
-      progress: (
-        Math.min(Math.abs((currentDataMutations - startDataMutations) / (totalDataMutations - startDataMutations)), 1) *
-        100
-      ).toFixed(0),
-    };
-  });
-};

package/src/helpers/setup-telemetry-listeners.ts

@@ -1,109 +0,0 @@
-//
-// Copyright 2022 DXOS.org
-//
-
-import type { Client } from '@dxos/client';
-
-import { type Observability } from '../observability';
-
-import { getTelemetryIdentity } from './common';
-
-let lastFocusEvent = new Date();
-let totalTime = 0;
-
-export const setupTelemetryListeners = (namespace: string, client: Client, observability: Observability) => {
-  const clickCallback = (event: Event) => {
-    const id = (event.target as HTMLElement)?.id;
-    if (!id) {
-      return;
-    }
-
-    setTimeout(() => {
-      observability.track({
-        ...getTelemetryIdentity(client),
-        action: 'window.click',
-        properties: {
-          id: (event.target as HTMLElement)?.id,
-          path: (event.composedPath() as HTMLElement[])
-            .filter((el) => Boolean(el.tagName))
-            .map((el) => `${el.tagName.toLowerCase()}${el.id ? `#${el.id}` : ''}`)
-            .reverse()
-            .join('>'),
-        },
-      });
-    });
-  };
-
-  const focusCallback = () => {
-    const now = new Date();
-    setTimeout(() => {
-      observability.track({
-        ...getTelemetryIdentity(client),
-        action: 'window.focus',
-        properties: {
-          timeAway: now.getTime() - lastFocusEvent.getTime(),
-        },
-      });
-    });
-
-    lastFocusEvent = now;
-  };
-
-  const blurCallback = () => {
-    const now = new Date();
-    const duration = now.getTime() - lastFocusEvent.getTime();
-    setTimeout(() => {
-      observability.track({
-        ...getTelemetryIdentity(client),
-        action: 'window.blur',
-        properties: {
-          duration,
-        },
-      });
-    });
-
-    lastFocusEvent = now;
-    totalTime = totalTime + duration;
-  };
-
-  const unloadCallback = () => {
-    setTimeout(() => {
-      observability.track({
-        ...getTelemetryIdentity(client),
-        action: 'page.unload',
-        properties: {
-          duration: totalTime,
-        },
-      });
-    });
-  };
-
-  const errorCallback = (event: ErrorEvent) => {
-    setTimeout(() => {
-      observability.track({
-        ...getTelemetryIdentity(client),
-        action: 'window.error',
-        properties: {
-          message: event.message,
-          filename: event.filename,
-          stack: (event.error as Error)?.stack,
-          cause: (event.error as Error)?.cause,
-        },
-      });
-    });
-  };
-
-  window.addEventListener('click', clickCallback, true);
-  window.addEventListener('focus', focusCallback);
-  window.addEventListener('blur', blurCallback);
-  window.addEventListener('beforeunload', unloadCallback);
-  window.addEventListener('error', errorCallback);
-
-  return () => {
-    window.removeEventListener('click', clickCallback, true);
-    window.removeEventListener('focus', focusCallback);
-    window.removeEventListener('blur', blurCallback);
-    window.removeEventListener('beforeunload', unloadCallback);
-    window.removeEventListener('error', errorCallback);
-  };
-};

package/src/otel/index.ts

@@ -1,8 +0,0 @@
-//
-// Copyright 2024 DXOS.org
-//
-
-export * from './otel';
-export * from './logs';
-export * from './metrics';
-export * from './traces';

package/src/otel/logs.ts

@@ -1,102 +0,0 @@
-//
-// Copyright 2024 DXOS.org
-//
-
-import { SeverityNumber } from '@opentelemetry/api-logs';
-import { OTLPLogExporter } from '@opentelemetry/exporter-logs-otlp-http';
-import { defaultResource, resourceFromAttributes } from '@opentelemetry/resources';
-import { BatchLogRecordProcessor, LoggerProvider } from '@opentelemetry/sdk-logs';
-import { SEMRESATTRS_SERVICE_NAME, SEMRESATTRS_SERVICE_VERSION } from '@opentelemetry/semantic-conventions';
-
-import {
-  type LogConfig,
-  type LogEntry,
-  LogLevel,
-  type LogProcessor,
-  getContextFromEntry,
-  getRelativeFilename,
-} from '@dxos/log';
-import { jsonlogify } from '@dxos/util';
-
-import { type OtelOptions, setDiagLogger } from './otel';
-
-export type OtelLogOptions = OtelOptions & {
-  logLevel: LogLevel;
-  /**
-   * Set `true` to capture logs sent through LoggingService from shared worker.
-   * Better to set to `false` because shared worker is initializing its own logger.
-   */
-  includeSharedWorkerLogs: boolean;
-};
-
-export class OtelLogs {
-  private _loggerProvider: LoggerProvider;
-  constructor(private readonly options: OtelLogOptions) {
-    setDiagLogger(options.consoleDiagLogLevel);
-    const resource = defaultResource().merge(
-      resourceFromAttributes({
-        [SEMRESATTRS_SERVICE_NAME]: this.options.serviceName,
-        [SEMRESATTRS_SERVICE_VERSION]: this.options.serviceVersion,
-      }),
-    );
-    const logExporter = new OTLPLogExporter({
-      url: this.options.endpoint + '/v1/logs',
-      headers: {
-        Authorization: this.options.authorizationHeader,
-      },
-      concurrencyLimit: 10, // an optional limit on pending requests
-    });
-    this._loggerProvider = new LoggerProvider({
-      resource,
-      processors: [new BatchLogRecordProcessor(logExporter)],
-    });
-  }
-
-  public readonly logProcessor: LogProcessor = (config: LogConfig, entry: LogEntry) => {
-    const logger = this._loggerProvider.getLogger('dxos-observability', this.options.serviceVersion);
-
-    if (
-      entry.level < this.options.logLevel ||
-      (!this.options.includeSharedWorkerLogs && entry.meta?.S?.remoteSessionId)
-    ) {
-      return;
-    }
-
-    const record = {
-      ...entry,
-      ...(entry.meta ? { meta: { file: getRelativeFilename(entry.meta.F), line: entry.meta.L } } : {}),
-      context: jsonlogify(getContextFromEntry(entry)),
-    };
-
-    logger.emit({
-      severityNumber: convertLevel(entry.level),
-      body: JSON.stringify(record),
-      attributes: this.options.getTags(),
-    });
-  };
-
-  flush(): Promise<void> {
-    return this._loggerProvider.forceFlush();
-  }
-
-  close(): Promise<void> {
-    return this._loggerProvider.shutdown();
-  }
-}
-
-const convertLevel = (level: LogLevel): SeverityNumber => {
-  switch (level) {
-    case LogLevel.DEBUG:
-      return SeverityNumber.DEBUG;
-    case LogLevel.VERBOSE:
-      return SeverityNumber.INFO;
-    case LogLevel.INFO:
-      return SeverityNumber.INFO;
-    case LogLevel.WARN:
-      return SeverityNumber.WARN;
-    case LogLevel.ERROR:
-      return SeverityNumber.ERROR;
-    default:
-      return SeverityNumber.ERROR;
-  }
-};

package/src/otel/otel.ts

@@ -1,21 +0,0 @@
-//
-// Copyright 2024 DXOS.org
-//
-
-import { DiagConsoleLogger, DiagLogLevel, diag } from '@opentelemetry/api';
-
-export type OtelOptions = {
-  endpoint: string;
-  authorizationHeader: string;
-  serviceName: string; // For the Otel API, the name of the entity for which signals (metrics or trace) are collected.
-  serviceVersion: string; // For the Otel API, The name of the entity for which signals (metrics or trace) are collected.
-  getTags: () => { [key: string]: string };
-  consoleDiagLogLevel?: string;
-};
-
-export const setDiagLogger = (level?: string) => {
-  const logLevel = DiagLogLevel[level as keyof typeof DiagLogLevel];
-  if (logLevel) {
-    diag.setLogger(new DiagConsoleLogger(), logLevel);
-  }
-};

package/src/otel/traces-browser.ts

@@ -1,62 +0,0 @@
-//
-// Copyright 2024 DXOS.org
-//
-
-import { type Tracer, trace } from '@opentelemetry/api';
-import { getWebAutoInstrumentations } from '@opentelemetry/auto-instrumentations-web';
-import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
-import { registerInstrumentations } from '@opentelemetry/instrumentation';
-import { defaultResource, resourceFromAttributes } from '@opentelemetry/resources';
-import { BatchSpanProcessor, ConsoleSpanExporter, SimpleSpanProcessor } from '@opentelemetry/sdk-trace-base';
-import { WebTracerProvider } from '@opentelemetry/sdk-trace-web';
-import { SEMRESATTRS_SERVICE_NAME, SEMRESATTRS_SERVICE_VERSION } from '@opentelemetry/semantic-conventions';
-
-import { log } from '@dxos/log';
-import { type StartSpanOptions, TRACE_PROCESSOR } from '@dxos/tracing';
-
-import { type OtelOptions } from './otel';
-
-export class OtelTraces {
-  private _tracer: Tracer;
-  constructor(private readonly options: OtelOptions) {
-    const resource = defaultResource().merge(
-      resourceFromAttributes({
-        [SEMRESATTRS_SERVICE_NAME]: this.options.serviceName,
-        [SEMRESATTRS_SERVICE_VERSION]: this.options.serviceVersion,
-      }),
-    );
-
-    const tracerProvider = new WebTracerProvider({
-      resource,
-      spanProcessors: [
-        new SimpleSpanProcessor(new ConsoleSpanExporter()),
-        new BatchSpanProcessor(
-          new OTLPTraceExporter({
-            url: this.options.endpoint + '/v1/traces',
-            headers: {
-              Authorization: this.options.authorizationHeader,
-            },
-            concurrencyLimit: 10, // an optional limit on pending requests
-          }),
-        ),
-      ],
-    });
-
-    // TODO(nf): ContextManager? Propogator?
-    trace.setGlobalTracerProvider(tracerProvider);
-    this._tracer = trace.getTracer('dxos-observability', this.options.serviceVersion);
-  }
-
-  public start(): void {
-    registerInstrumentations({
-      instrumentations: [getWebAutoInstrumentations()],
-    });
-    log('trace processor registered');
-    TRACE_PROCESSOR.remoteTracing.registerProcessor({
-      startSpan: (options: StartSpanOptions) => {
-        log('begin otel trace', { options });
-        return this._tracer.startSpan(options.name, options);
-      },
-    });
-  }
-}

package/src/otel/traces.ts

@@ -1,60 +0,0 @@
-//
-// Copyright 2024 DXOS.org
-//
-
-import { type Tracer, trace } from '@opentelemetry/api';
-import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
-import { defaultResource, resourceFromAttributes } from '@opentelemetry/resources';
-import {
-  BasicTracerProvider,
-  BatchSpanProcessor,
-  ConsoleSpanExporter,
-  SimpleSpanProcessor,
-} from '@opentelemetry/sdk-trace-base';
-import { SEMRESATTRS_SERVICE_NAME, SEMRESATTRS_SERVICE_VERSION } from '@opentelemetry/semantic-conventions';
-import { log } from 'debug';
-
-import { type StartSpanOptions, TRACE_PROCESSOR } from '@dxos/tracing';
-
-import { type OtelOptions } from './otel';
-
-export class OtelTraces {
-  private _tracer: Tracer;
-  constructor(private readonly options: OtelOptions) {
-    const resource = defaultResource().merge(
-      resourceFromAttributes({
-        [SEMRESATTRS_SERVICE_NAME]: this.options.serviceName,
-        [SEMRESATTRS_SERVICE_VERSION]: this.options.serviceVersion,
-      }),
-    );
-
-    const tracerProvider = new BasicTracerProvider({
-      resource,
-      spanProcessors: [
-        new SimpleSpanProcessor(new ConsoleSpanExporter()),
-        new BatchSpanProcessor(
-          new OTLPTraceExporter({
-            url: this.options.endpoint + '/v1/traces',
-            headers: {
-              Authorization: this.options.authorizationHeader,
-            },
-            concurrencyLimit: 10, // an optional limit on pending requests
-          }),
-        ),
-      ],
-    });
-
-    trace.setGlobalTracerProvider(tracerProvider);
-    this._tracer = trace.getTracer('dxos-observability', this.options.serviceVersion);
-  }
-
-  public start(): void {
-    log('trace processor registered');
-    TRACE_PROCESSOR.remoteTracing.registerProcessor({
-      startSpan: (options: StartSpanOptions) => {
-        log('begin otel trace', { options });
-        return this._tracer.startSpan(options.name, options);
-      },
-    });
-  }
-}