@dxos/observability 0.3.11-main.0252f93

package/src/sentry/browser.ts

@@ -0,0 +1,100 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+import {
+  init as naturalInit,
+  setTag,
+  addBreadcrumb as naturalAddBreadcrumb,
+  captureException as naturalCaptureException,
+  BrowserTracing,
+  replayIntegration,
+} from '@sentry/browser';
+import { captureConsoleIntegration, httpClientIntegration } from '@sentry/integrations';
+
+import { log } from '@dxos/log';
+
+import { type InitOptions } from './types';
+
+// Browser only feature.
+export * from './tracing';
+
+// Polyfill export.
+export { setTag, setTags, setUser } from '@sentry/browser';
+
+/**
+ * To use this SDK, call the init function as early as possible when loading the web page.
+ * To set context information or send manual events, use the provided methods.
+ *
+ * @param options {InitOptions}
+ */
+export const init = (options: InitOptions) => {
+  try {
+    log('sentry init', options);
+    naturalInit({
+      enabled: options.enable ?? true,
+      dsn: options.destination,
+      release: options.release,
+      environment: options.environment,
+      integrations: [
+        captureConsoleIntegration({ levels: ['error', 'warn'] }),
+        httpClientIntegration({ failedRequestStatusCodes: [[400, 599]] }),
+        ...(options.tracing ? [new BrowserTracing()] : []),
+        ...(options.replay ? [replayIntegration({ blockAllMedia: true, maskAllText: true })] : []),
+      ],
+      replaysSessionSampleRate: options.replaySampleRate,
+      replaysOnErrorSampleRate: options.replaySampleRateOnError,
+      tracesSampleRate: options.sampleRate,
+      transport: options.transport,
+      beforeSend: (event) => {
+        options.onError?.(event);
+        return event;
+      },
+    });
+
+    Object.entries(options.properties ?? {}).forEach(([key, value]) => {
+      setTag(key, value);
+    });
+  } catch (err) {
+    log.catch('Failed to initialize sentry', err);
+  }
+};
+
+/**
+ * Records a new breadcrumb which will be attached to future events.
+ *
+ * Breadcrumbs will be added to subsequent events to provide more context on user's actions prior to an error or crash.
+ *
+ * https://docs.sentry.io/platforms/javascript/enriching-events/breadcrumbs/
+ *
+ * @param breadcrumb — The breadcrumb to record.
+ */
+export const addBreadcrumb: typeof naturalAddBreadcrumb = (breadcrumb) => {
+  try {
+    naturalAddBreadcrumb(breadcrumb);
+    log('add breadcrumb', breadcrumb);
+  } catch (err) {
+    log.catch('Failed to add breadcrumb', err);
+  }
+};
+
+/**
+ * Captures an exception event and sends it to Sentry.
+ *
+ * @param exception — An exception-like object.
+ * @param captureContext — Additional scope data to apply to exception event.
+ * @returns — The generated eventId.
+ */
+export const captureException: typeof naturalCaptureException = (exception, captureContext) => {
+  try {
+    const eventId = naturalCaptureException(exception, captureContext);
+    log('capture exception', { exception, eventId, ...captureContext });
+    return eventId;
+  } catch (err) {
+    log.catch('Failed to capture exception', err);
+    return 'unknown';
+  }
+};
+
+// Not needed for browser.
+export const enableSentryLogProcessor = () => {};

package/src/sentry/index.ts

@@ -0,0 +1,9 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+export * from './node';
+export * from './types';
+
+// TODO(mykola): Tracing does not work in the node. Delete this export.
+export { configureTracing } from './tracing';

package/src/sentry/node.test.ts

@@ -0,0 +1,46 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+// @dxos/test platform=nodejs
+
+import type { Stacktrace } from '@sentry/types';
+import { expect } from 'chai';
+import waitForExpect from 'wait-for-expect';
+
+import { beforeAll, beforeEach, describe, test } from '@dxos/test';
+
+import * as Sentry from './node';
+import { sentryTestkit } from '../testing';
+
+const { testkit, sentryTransport } = sentryTestkit();
+
+// TODO(burdon): https://example.com?
+const MOCK_DESTINATION = 'https://acacaeaccacacacabcaacdacdacadaca@sentry.io/000001';
+
+describe('Node error reporting', () => {
+  beforeAll(() => {
+    Sentry.init({
+      destination: MOCK_DESTINATION,
+      release: 'test',
+      transport: sentryTransport,
+      scrubFilenames: true,
+    });
+  });
+
+  beforeEach(() => {
+    testkit.reset();
+  });
+
+  test('should scrub stacktraces', async () => {
+    const err = new Error('error to look for');
+    Sentry.captureException(err);
+    await waitForExpect(() => {
+      expect(testkit.reports()).to.be.lengthOf(1);
+    });
+    const report = testkit.findReport(err);
+    (report?.error?.stacktrace as Stacktrace).frames?.forEach((frame) => {
+      expect(frame.filename?.includes('/')).to.be.false;
+    });
+  });
+});

package/src/sentry/node.ts

@@ -0,0 +1,153 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+import { captureConsoleIntegration } from '@sentry/integrations';
+import {
+  init as naturalInit,
+  setTag,
+  addBreadcrumb as naturalAddBreadcrumb,
+  captureException as naturalCaptureException,
+  captureMessage as naturalCaptureMessage,
+  withScope,
+} from '@sentry/node';
+import type { Event } from '@sentry/node';
+import type { SeverityLevel } from '@sentry/types';
+
+import { log, LogLevel, shouldLog } from '@dxos/log';
+
+import { type InitOptions } from './types';
+
+// Polyfill export.
+export { setTag, setTags, setUser } from '@sentry/node';
+
+/**
+ * To use this SDK, call the init function as early as possible in the main entry module.
+ * To set context information or send manual events, use the provided methods.
+ *
+ * @param options {InitOptions}
+ */
+export const init = (options: InitOptions) => {
+  try {
+    if (options.tracing) {
+      void import('@sentry/tracing');
+    }
+
+    naturalInit({
+      enabled: options.enable ?? true,
+      dsn: options.destination,
+      serverName: options.installationId,
+      release: options.release,
+      environment: options.environment ?? process.env.DX_ENVIRONMENT,
+      integrations: [captureConsoleIntegration({ levels: ['error', 'warn'] })],
+      tracesSampleRate: options.sampleRate,
+      transport: options.transport,
+      beforeSend: (event) => {
+        options.scrubFilenames && scrub(event);
+        options.onError?.(event);
+
+        return event;
+      },
+    });
+
+    Object.entries(options.properties ?? {}).forEach(([key, value]) => {
+      setTag(key, value);
+    });
+  } catch (err) {
+    log.catch('Failed to initialize sentry', err);
+  }
+};
+
+const scrub = (event: Event) => {
+  event.exception?.values?.forEach((value) => {
+    value.stacktrace?.frames?.forEach((frame) => {
+      const filename = frame.filename?.split('/');
+      frame.filename = filename && filename[filename.length - 1];
+    });
+  });
+};
+
+/**
+ * Records a new breadcrumb which will be attached to future events.
+ *
+ * Breadcrumbs will be added to subsequent events to provide more context on user's actions prior to an error or crash.
+ *
+ * https://docs.sentry.io/platforms/javascript/enriching-events/breadcrumbs/
+ *
+ * @param breadcrumb — The breadcrumb to record.
+ */
+export const addBreadcrumb: typeof naturalAddBreadcrumb = (breadcrumb) => {
+  try {
+    naturalAddBreadcrumb(breadcrumb);
+    log('add breadcrumb', breadcrumb);
+  } catch (err) {
+    log.catch('Failed to add breadcrumb', err);
+  }
+};
+
+/**
+ * Captures an exception event and sends it to Sentry.
+ *
+ * @param exception — An exception-like object.
+ * @param captureContext — Additional scope data to apply to exception event.
+ * @returns — The generated eventId.
+ */
+export const captureException: typeof naturalCaptureException = (exception, captureContext) => {
+  try {
+    const eventId = naturalCaptureException(exception, captureContext);
+    log('capture exception', { exception, eventId, ...captureContext });
+    return eventId;
+  } catch (err) {
+    log.catch('Failed to capture exception', err);
+    return 'unknown';
+  }
+};
+
+// TODO(nf): move to log package as a first-class processor.
+export const enableSentryLogProcessor = (filter?: string | string[] | LogLevel) => {
+  log.runtimeConfig.processors.push(SENTRY_LOG_PROCESSOR);
+};
+
+export const SENTRY_LOG_PROCESSOR = (config: any, entry: any) => {
+  const { message, level, meta, error } = entry;
+  if (!shouldLog(entry, config.captureFilters)) {
+    return;
+  }
+  // TODO(nf): add rate limiting to avoid spamming Sentry/consuming excessive quota.
+  withScope((scope) => {
+    scope.setLevel(convertLevel(level));
+    scope.setContext('dxoslog', entry.context);
+    if (meta) {
+      scope.setTransactionName(`${getRelativeFilename(meta.F)}:${meta.L}`);
+    }
+    // Is this ever used?
+    if (error) {
+      scope.setExtra('message', message);
+      return naturalCaptureException(error);
+    }
+
+    naturalCaptureMessage(message);
+  });
+};
+
+const convertLevel = (level: LogLevel): SeverityLevel => {
+  if (level === LogLevel.TRACE) {
+    return 'debug';
+  }
+  if (level === LogLevel.WARN) {
+    return 'warning';
+  }
+  return LogLevel[level].toLowerCase() as SeverityLevel;
+};
+
+const getRelativeFilename = (filename: string) => {
+  // TODO(burdon): Hack uses "packages" as an anchor (pre-parse NX?)
+  // Including `packages/` part of the path so that excluded paths (e.g. from dist) are clickable in vscode.
+  const match = filename.match(/.+\/(packages\/.+\/.+)/);
+  if (match) {
+    const [, filePath] = match;
+    return filePath;
+  }
+
+  return filename;
+};

package/src/sentry/sentry.test.ts

@@ -0,0 +1,39 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+import { expect } from 'chai';
+import waitForExpect from 'wait-for-expect';
+
+import { beforeAll, beforeEach, describe, test } from '@dxos/test';
+
+import * as Sentry from './node';
+import { sentryTestkit } from '../testing';
+
+const { testkit, sentryTransport } = sentryTestkit();
+
+const MOCK_DESTINATION = 'https://acacaeaccacacacabcaacdacdacadaca@sentry.io/000001';
+
+describe('Error reporting', () => {
+  beforeAll(() => {
+    Sentry.init({
+      destination: MOCK_DESTINATION,
+      release: 'test',
+      transport: sentryTransport,
+    });
+  });
+
+  beforeEach(() => {
+    testkit.reset();
+  });
+
+  test('should capture errors', async () => {
+    const err = new Error('error to look for');
+    Sentry.captureException(err);
+    await waitForExpect(() => {
+      expect(testkit.reports()).to.be.lengthOf(1);
+    });
+    const report = testkit.findReport(err);
+    expect(report).to.exist;
+  });
+});

package/src/sentry/tracing.test.ts

@@ -0,0 +1,62 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+// @dxos/test platform=browser
+
+import { expect } from 'chai';
+
+import { sleep } from '@dxos/async';
+import { log } from '@dxos/log';
+import { afterAll, beforeAll, describe, test } from '@dxos/test';
+
+import * as Sentry from './browser';
+import * as Tracing from './tracing';
+import { sentryTestkit } from '../testing';
+
+const { testkit, sentryTransport } = sentryTestkit();
+
+// TODO(burdon): https://example.com?
+const MOCK_DESTINATION = 'https://acacaeaccacacacabcaacdacdacadaca@sentry.io/000001';
+
+describe('Logger tracing', () => {
+  beforeAll(() => {
+    Sentry.init({
+      destination: MOCK_DESTINATION,
+      release: 'test',
+      scrubFilenames: true,
+      tracing: true,
+      transport: sentryTransport,
+    });
+
+    Tracing.configureTracing();
+  });
+
+  afterAll(() => {
+    Tracing.finish();
+  });
+
+  test('begin and end span', async () => {
+    log.trace('test.trace', {
+      span: {
+        command: 'begin',
+        id: 'test',
+      },
+    });
+
+    log.trace('test.trace', {
+      span: {
+        command: 'end',
+        id: 'test',
+        status: 'error',
+      },
+      error: new Error('test'),
+    });
+    // Sleep to allow Sentry tracing to flush.
+    await sleep(10);
+    Tracing.finish();
+
+    expect(testkit.transactions().length).to.eq(1);
+    expect(testkit.transactions()[0].spans[0].op).to.eq('test.trace');
+  });
+});

package/src/sentry/tracing.ts

@@ -0,0 +1,153 @@
+//
+// Copyright 2023 DXOS.org
+//
+
+// NOTE: Browser only API. There is no node polyfill.
+
+import { getCurrentHub } from '@sentry/browser';
+import { type Transaction, type Span } from '@sentry/types';
+
+import { runInContext, scheduleMicroTask, Trigger } from '@dxos/async';
+import { Context } from '@dxos/context';
+import { invariant } from '@dxos/invariant';
+import { getContextFromEntry, log, LogLevel, type LogProcessor } from '@dxos/log';
+
+const REPORT_SPANS = false;
+
+let TX!: Transaction;
+const SPAN_MAP = new Map<string, Span>();
+const SENTRY_INITIALIZED = new Trigger();
+const ctx = new Context({ onError: (err) => log.warn('Unhandled error in Sentry context', err) });
+ctx.maxSafeDisposeCallbacks = 10_000;
+
+let tracingConfigured = false;
+
+export const configureTracing = () => {
+  tracingConfigured = true;
+  runInContext(ctx, () => {
+    // Configure root transaction.
+    TX = getCurrentHub().startTransaction({
+      name: 'DXOS Core Tracing',
+      op: 'dxos',
+    });
+    invariant(TX, 'Failed to create trace');
+    if (typeof window !== 'undefined') {
+      window.addEventListener('beforeunload', () => {
+        finish();
+      });
+    }
+    if (typeof process !== 'undefined') {
+      process.on('exit', () => {
+        finish();
+      });
+    }
+
+    SENTRY_INITIALIZED.wake();
+  });
+};
+
+export const finish = () => {
+  for (const span of Array.from(SPAN_MAP.values()).reverse()) {
+    try {
+      span.finish();
+    } catch (err) {
+      log.warn('Failed to finish span', err);
+    }
+  }
+  TX.finish();
+};
+
+export const SENTRY_PROCESSOR: LogProcessor = (config, entry) => {
+  if (!tracingConfigured) {
+    return;
+  }
+
+  const { message, level, error } = entry;
+  const context = getContextFromEntry(entry);
+
+  if (level !== LogLevel.TRACE) {
+    return;
+  }
+
+  // NOTE: Make sure `entry` is not captured in this closure to avoid a memory leak.
+  scheduleMicroTask(ctx, async () => {
+    await SENTRY_INITIALIZED.wait();
+
+    if (REPORT_SPANS && context?.span) {
+      switch (context.span.command) {
+        case 'begin': {
+          const id = context.span.id;
+
+          if (!id || SPAN_MAP.has(id)) {
+            log.warn('Cannot begin span', id);
+            return;
+          }
+
+          let parentSpan: Span = TX;
+          if (context.span.parent) {
+            parentSpan = SPAN_MAP.get(context.span.parent) ?? TX;
+          }
+
+          let logContext: string;
+          try {
+            logContext = JSON.stringify({ ...context, message, level, error });
+          } catch (err) {
+            logContext = JSON.stringify(context);
+          }
+
+          const span = parentSpan.startChild({
+            op: message,
+            data: {
+              ...context.span.data,
+              '@dxos/log': logContext,
+            },
+          });
+          SPAN_MAP.set(context.span.id, span);
+          break;
+        }
+
+        case 'end': {
+          const span = SPAN_MAP.get(context.span.id);
+          if (span) {
+            span.setStatus(getSpanStatus(context.span.status));
+            context.span.data && Object.entries(context.span.data).forEach(([key, value]) => span.setData(key, value));
+            span.finish();
+            SPAN_MAP.delete(context.span.id);
+          } else {
+            log.warn('Cannot end span', context.span.id);
+          }
+          break;
+        }
+
+        case 'update': {
+          const span = SPAN_MAP.get(context.span.id);
+          if (span) {
+            context.span.data && Object.entries(context.span.data).forEach(([key, value]) => span.setData(key, value));
+          } else {
+            log.warn('Cannot update span', context.span.id);
+          }
+          break;
+        }
+
+        default: {
+          log.warn('Unknown span command', context.span.command);
+        }
+      }
+    }
+  });
+};
+
+log.runtimeConfig.processors.push(SENTRY_PROCESSOR);
+
+/**
+ * @see https://develop.sentry.dev/sdk/event-payloads/span/#:~:text=this%20value%20explicitly.-,status,-Optional.%20Describes%20the
+ */
+const getSpanStatus = (status?: 'ok' | 'error') => {
+  switch (status) {
+    case 'ok':
+      return 'ok';
+    case 'error':
+      return 'not_found';
+  }
+  return status ?? 'ok';
+};

package/src/sentry/types.ts

@@ -0,0 +1,22 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+import type { Event, Transport } from '@sentry/types';
+
+export type InitOptions = {
+  enable?: boolean;
+  destination?: string;
+  installationId?: string;
+  release?: string;
+  environment?: string;
+  tracing?: boolean;
+  sampleRate?: number;
+  replay?: boolean;
+  replaySampleRate?: number;
+  replaySampleRateOnError?: number;
+  scrubFilenames?: boolean;
+  properties?: object;
+  transport?: () => Transport;
+  onError?: (event: Event) => void;
+};

package/src/testing/index.ts

@@ -0,0 +1,5 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+export * from './testkit';

package/src/testing/testkit/browser.ts

@@ -0,0 +1,8 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+// eslint-disable-next-line no-restricted-imports
+import testkit from 'sentry-testkit/dist/browser';
+
+export const sentryTestkit = testkit;

package/src/testing/testkit/index.ts

@@ -0,0 +1,7 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+import testkit from 'sentry-testkit';
+
+export const sentryTestkit = testkit;

package/src/testing/testkit/shims.d.ts

@@ -0,0 +1,5 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+declare module 'sentry-testkit/dist/browser';

package/src/util.ts

@@ -0,0 +1,49 @@
+//
+// Copyright 2023 DXOS.org
+//
+
+import { type Space } from '@dxos/client-protocol';
+
+export type MapSpacesOptions = {
+  verbose?: boolean;
+  truncateKeys?: boolean;
+};
+
+export const mapSpaces = (spaces: Space[], options: MapSpacesOptions = { verbose: false, truncateKeys: false }) => {
+  return spaces.map((space) => {
+    // TODO(burdon): Factor out.
+    // TODO(burdon): Agent needs to restart before `ready` is available.
+    const { open, ready } = space.internal.data.metrics ?? {};
+    const startup = open && ready && ready.getTime() - open.getTime();
+
+    // TODO(burdon): Get feeds from client-services if verbose (factor out from devtools/diagnostics).
+    // const host = client.services.services.DevtoolsHost!;
+    const pipeline = space.internal.data.pipeline;
+    const startDataMutations = pipeline?.currentEpoch?.subject.assertion.timeframe.totalMessages() ?? 0;
+    const epoch = pipeline?.currentEpoch?.subject.assertion.number;
+    // const appliedEpoch = pipeline?.appliedEpoch?.subject.assertion.number;
+    const currentDataMutations = pipeline?.currentDataTimeframe?.totalMessages() ?? 0;
+    const totalDataMutations = pipeline?.targetDataTimeframe?.totalMessages() ?? 0;
+
+    return {
+      // TODO(nf): truncate keys for DD?
+      key: space.key.truncate(),
+      open: space.isOpen,
+      name: space.properties.name,
+      members: space.members.get().length,
+      objects: space.db.query().objects.length,
+      startup,
+      epoch,
+      // appliedEpoch,
+      startDataMutations,
+      currentDataMutations,
+      totalDataMutations,
+
+      // TODO(burdon): Negative?
+      progress: (
+        Math.min(Math.abs((currentDataMutations - startDataMutations) / (totalDataMutations - startDataMutations)), 1) *
+        100
+      ).toFixed(0),
+    };
+  });
+};