@dxos/observability 0.3.11-main.2f1331c → 0.3.11-main.463e5cd

package/src/observability.ts

@@ -14,6 +14,7 @@ import { isNode } from '@dxos/util';
 
 import buildSecrets from './cli-observability-secrets.json';
 import { DatadogMetrics } from './datadog';
+import { type IPData, getTelemetryIdentifier, mapSpaces } from './helpers';
 import { SegmentTelemetry, type EventOptions, type PageOptions } from './segment';
 import {
   captureException as sentryCaptureException,
@@ -23,15 +24,11 @@ import {
   type InitOptions,
   setTag as sentrySetTag,
 } from './sentry';
-import { mapSpaces } from './util';
 
 const SPACE_METRICS_MIN_INTERVAL = 1000 * 60;
 // const DATADOG_IDLE_INTERVAL = 1000 * 60 * 5;
 const DATADOG_IDLE_INTERVAL = 1000 * 60 * 1;
 
-// TODO(nf): add allowlist for telemetry tags?
-const ERROR_TAGS = ['identityKey', 'username', 'deviceKey', 'group'];
-
 // Secrets? EnvironmentConfig?
 
 export type ObservabilitySecrets = {
@@ -45,6 +42,7 @@ export type ObservabilitySecrets = {
 };
 
 export type Mode = 'basic' | 'full' | 'disabled';
+export type TagScope = 'errors' | 'telemetry' | 'metrics' | 'all';
 
 export type ObservabilityOptions = {
   /// The webapp (e.g. 'composer.dxos.org'), 'cli', or 'agent'.
@@ -62,8 +60,8 @@ export type ObservabilityOptions = {
 
   errorLog?: {
     sentryInitOptions?: InitOptions;
+    logProcessor?: boolean;
   };
-  logProcessor?: boolean;
 };
 
 /*
@@ -87,10 +85,10 @@ export class Observability {
   private _group?: string;
   // TODO(nf): accept upstream context?
   private _ctx = new Context();
-  private _tags = new Map<string, string>();
+  private _tags = new Map<string, { value: string; scope: TagScope }>();
 
   // TODO(nf): make platform a required extension?
-  constructor({ namespace, config, secrets, group, mode, telemetry, errorLog, logProcessor }: ObservabilityOptions) {
+  constructor({ namespace, config, secrets, group, mode, telemetry, errorLog }: ObservabilityOptions) {
     this._namespace = namespace;
     this._config = config;
     this._mode = mode ?? 'disabled';
@@ -98,7 +96,7 @@ export class Observability {
     this._secrets = this._loadSecrets(config, secrets);
     this._telemetryBatchSize = telemetry?.batchSize ?? 30;
     this._errorReportingOptions = errorLog?.sentryInitOptions;
-    this._errorLogProcessor = logProcessor ?? false;
+    this._errorLogProcessor = errorLog?.logProcessor ?? false;
 
     if (this._group) {
       this.setTag('group', this._group);
@@ -149,6 +147,10 @@ export class Observability {
     return this._mode;
   }
 
+  get group() {
+    return this._group;
+  }
+
   get enabled() {
     return this._mode !== 'disabled';
   }
@@ -157,18 +159,25 @@ export class Observability {
   // Tags
   //
 
-  /**
-   * Set a tag for all observability services.
-   */
-  setTag(key: string, value: string) {
-    if (this.enabled && ERROR_TAGS.includes(key)) {
+  setTag(key: string, value: string, scope?: TagScope) {
+    if (this.enabled && (scope === undefined || scope === 'all' || scope === 'errors')) {
       sentrySetTag(key, value);
     }
-
-    this._tags.set(key, value);
+    if (!scope) {
+      scope = 'all';
+    }
+    this._tags.set(key, { value, scope });
   }
 
-  // TODO(nf): Combine with setDeviceTags.
+  addIPDataTelemetryTags = (ipData: IPData) => {
+    this.setTag('city', ipData.city, 'telemetry');
+    this.setTag('region', ipData.region, 'telemetry');
+    this.setTag('country', ipData.country, 'telemetry');
+    ipData.latitude && this.setTag('latitude', ipData.latitude.toString(), 'telemetry');
+    ipData.longitude && this.setTag('longitude', ipData.longitude.toString(), 'telemetry');
+  };
+
+  // TODO(wittjosiah): Improve privacy of telemetry identifiers. See `getTelemetryIdentifier`.
   async setIdentityTags(client: Client) {
     client.services.services.IdentityService!.queryIdentity().subscribe((idqr) => {
       if (!idqr?.identity?.identityKey) {
@@ -176,16 +185,9 @@ export class Observability {
         return;
       }
 
-      // TODO(nf): check mode
-      // TODO(nf): cardinality
-      this.setTag('identityKey', idqr?.identity?.identityKey.truncate());
-      if (idqr?.identity?.profile?.displayName) {
-        this.setTag('username', idqr?.identity?.profile?.displayName);
-      }
+      this.setTag('identityKey', idqr.identity.identityKey.truncate());
     });
-  }
 
-  async setDeviceTags(client: Client) {
     client.services.services.DevicesService!.queryDevices().subscribe((dqr) => {
       if (!dqr || !dqr.devices || dqr.devices.length === 0) {
         log('empty response from device service', { device: dqr });
@@ -213,7 +215,14 @@ export class Observability {
     if (this.enabled && this._secrets.DATADOG_API_KEY) {
       this._metrics = new DatadogMetrics({
         apiKey: this._secrets.DATADOG_API_KEY,
-        getTags: () => this._tags,
+        getTags: () =>
+          Object.fromEntries(
+            Array.from(this._tags)
+              .filter(([key, value]) => {
+                return value.scope === 'all' || value.scope === 'metrics';
+              })
+              .map(([key, value]) => [key, value.value]),
+          ),
         // TODO(nf): move/refactor from telementryContext, needed to read CORS proxy
         config: this._config!,
       });
@@ -283,19 +292,27 @@ export class Observability {
     // scheduleTaskInterval(ctx, async () => updateSignalMetrics.emit(), DATADOG_IDLE_INTERVAL);
   }
 
-  startSpacesMetrics(client: Client) {
-    let spaces = client.spaces.get();
+  startSpacesMetrics(client: Client, namespace: string) {
+    const spaces = client.spaces.get();
     const subscriptions = new Map<string, { unsubscribe: () => void }>();
     this._ctx.onDispose(() => subscriptions.forEach((subscription) => subscription.unsubscribe()));
 
     const updateSpaceMetrics = new Event<Space>().debounce(SPACE_METRICS_MIN_INTERVAL);
-    updateSpaceMetrics.on(this._ctx, async (space) => {
+    updateSpaceMetrics.on(this._ctx, async () => {
       log('send space update');
-      for (const sp of mapSpaces(spaces, { truncateKeys: true })) {
-        this.gauge('dxos.client.space.members', sp.members, { key: sp.key });
-        this.gauge('dxos.client.space.objects', sp.objects, { key: sp.key });
-        this.gauge('dxos.client.space.epoch', sp.epoch, { key: sp.key });
-        this.gauge('dxos.client.space.currentDataMutations', sp.currentDataMutations, { key: sp.key });
+      for (const data of mapSpaces(spaces, { truncateKeys: true })) {
+        // Metrics
+        this.gauge('dxos.client.space.members', data.members, { key: data.key });
+        this.gauge('dxos.client.space.objects', data.objects, { key: data.key });
+        this.gauge('dxos.client.space.epoch', data.epoch, { key: data.key });
+        this.gauge('dxos.client.space.currentDataMutations', data.currentDataMutations, { key: data.key });
+
+        // Telemetry
+        this.event({
+          identityId: getTelemetryIdentifier(client),
+          name: `${namespace}.space.update`,
+          properties: data,
+        });
       }
     });
 
@@ -311,9 +328,7 @@ export class Observability {
     });
 
     client.spaces.subscribe({
-      next: async () => {
-        spaces = client.spaces.get();
-        // spaces = await this.getSpaces(this._agent.client);
+      next: async (spaces) => {
         spaces
           .filter((space) => !subscriptions.has(space.key.toHex()))
           .forEach((space) => {
@@ -370,7 +385,14 @@ export class Observability {
       this._telemetry = new SegmentTelemetry({
         apiKey: this._secrets.TELEMETRY_API_KEY,
         batchSize: this._telemetryBatchSize,
-        getTags: () => this._tags,
+        getTags: () =>
+          Object.fromEntries(
+            Array.from(this._tags)
+              .filter(([key, value]) => {
+                return value.scope === 'all' || value.scope === 'telemetry';
+              })
+              .map(([key, value]) => [key, value.value]),
+          ),
       });
     } else {
       log('segment disabled');
@@ -420,8 +442,8 @@ export class Observability {
       }
       // TODO(nf): is this different than passing as properties in options?
       this._tags.forEach((v, k) => {
-        if (ERROR_TAGS.includes(k)) {
-          sentrySetTag(k, v);
+        if (v.scope === 'all' || v.scope === 'errors') {
+          sentrySetTag(k, v.value);
         }
       });
     } else {

package/src/segment/browser.ts

@@ -10,7 +10,7 @@ import type { EventOptions, SegmentTelemetryOptions, PageOptions } from './types
 import { captureException } from '../sentry';
 
 export class SegmentTelemetry {
-  private _getTags: () => Map<string, string>;
+  private _getTags: () => { [key: string]: string };
 
   constructor({ apiKey, batchSize, getTags }: SegmentTelemetryOptions) {
     this._getTags = getTags;
@@ -31,7 +31,7 @@ export class SegmentTelemetry {
   page({ identityId: userId, ...options }: PageOptions = {}) {
     try {
       (window as any).analytics?.page({
-        context: Object.fromEntries(this._getTags().entries()),
+        context: this._getTags(),
         ...options,
         userId,
       });
@@ -44,7 +44,7 @@ export class SegmentTelemetry {
     try {
       options.properties = {
         ...options.properties,
-        ...Object.fromEntries(this._getTags().entries()),
+        ...this._getTags(),
       };
       (window as any).analytics?.track({
         ...options,

package/src/segment/node.ts

@@ -12,7 +12,7 @@ import { captureException } from '../sentry';
 
 export class SegmentTelemetry {
   private _analytics?: Analytics;
-  private _getTags: () => Map<string, string>;
+  private _getTags: () => { [key: string]: string };
 
   constructor({ apiKey, batchSize, getTags }: SegmentTelemetryOptions) {
     this._getTags = getTags;
@@ -52,7 +52,7 @@ export class SegmentTelemetry {
    * Track an event.
    */
   event({ installationId: anonymousId, identityId: userId, name: event, ...options }: EventOptions) {
-    log('sending event to telemetry', { event, options, tags: Object.fromEntries(this._getTags().entries()) });
+    log('sending event to telemetry', { event, options, tags: this._getTags() });
     if (!this._analytics) {
       log('Analytics not initialized', { action: 'event' });
       return;
@@ -61,7 +61,7 @@ export class SegmentTelemetry {
     try {
       this._analytics.track({
         ...options,
-        context: Object.fromEntries(this._getTags().entries()),
+        context: this._getTags(),
         userId,
         anonymousId: anonymousId!,
         event,

package/src/segment/types.ts

@@ -6,7 +6,7 @@ export type SegmentTelemetryOptions = {
   apiKey?: string;
   batchSize?: number;
   enable?: boolean;
-  getTags: () => Map<string, string>;
+  getTags: () => { [key: string]: string };
 };
 
 type CommonOptions = {

package/dist/types/src/util.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"util.d.ts","sourceRoot":"","sources":["../../../src/util.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,KAAK,KAAK,EAAE,MAAM,uBAAuB,CAAC;AAEnD,MAAM,MAAM,gBAAgB,GAAG;IAC7B,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB,CAAC;AAEF,eAAO,MAAM,SAAS,WAAY,KAAK,EAAE,YAAW,gBAAgB;;;;;;;;;;;;GAqCnE,CAAC"}

package/dist/types/src/util.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"util.js","sourceRoot":"","sources":["../../../src/util.ts"],"names":[],"mappings":";AAAA,EAAE;AACF,0BAA0B;AAC1B,EAAE;;;AASK,MAAM,SAAS,GAAG,CAAC,MAAe,EAAE,UAA4B,EAAE,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,KAAK,EAAE,EAAE,EAAE;IAChH,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;;QAC1B,4BAA4B;QAC5B,oEAAoE;QACpE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAA,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,mCAAI,EAAE,CAAC;QAC1D,MAAM,OAAO,GAAG,IAAI,IAAI,KAAK,IAAI,KAAK,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;QAElE,kGAAkG;QAClG,uDAAuD;QACvD,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC;QAC9C,MAAM,kBAAkB,GAAG,MAAA,MAAA,QAAQ,aAAR,QAAQ,uBAAR,QAAQ,CAAE,YAAY,0CAAE,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,aAAa,EAAE,mCAAI,CAAC,CAAC;QACpG,MAAM,KAAK,GAAG,MAAA,QAAQ,aAAR,QAAQ,uBAAR,QAAQ,CAAE,YAAY,0CAAE,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC;QAC/D,yEAAyE;QACzE,MAAM,oBAAoB,GAAG,MAAA,MAAA,QAAQ,aAAR,QAAQ,uBAAR,QAAQ,CAAE,oBAAoB,0CAAE,aAAa,EAAE,mCAAI,CAAC,CAAC;QAClF,MAAM,kBAAkB,GAAG,MAAA,MAAA,QAAQ,aAAR,QAAQ,uBAAR,QAAQ,CAAE,mBAAmB,0CAAE,aAAa,EAAE,mCAAI,CAAC,CAAC;QAE/E,OAAO;YACL,kCAAkC;YAClC,GAAG,EAAE,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE;YACzB,IAAI,EAAE,KAAK,CAAC,MAAM;YAClB,IAAI,EAAE,KAAK,CAAC,UAAU,CAAC,IAAI;YAC3B,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,MAAM;YACnC,OAAO,EAAE,KAAK,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,OAAO,CAAC,MAAM;YACxC,OAAO;YACP,KAAK;YACL,gBAAgB;YAChB,kBAAkB;YAClB,oBAAoB;YACpB,kBAAkB;YAElB,0BAA0B;YAC1B,QAAQ,EAAE,CACR,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,oBAAoB,GAAG,kBAAkB,CAAC,GAAG,CAAC,kBAAkB,GAAG,kBAAkB,CAAC,CAAC,EAAE,CAAC,CAAC;gBAC9G,GAAG,CACJ,CAAC,OAAO,CAAC,CAAC,CAAC;SACb,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC,CAAC;AArCW,QAAA,SAAS,aAqCpB"}