@observertc/observer-js 1.0.0-beta.18 → 1.0.0-beta.21

package/dist/index.js

@@ -43,6 +43,7 @@ __export(src_exports, {
   ClientSampleSink: () => ClientSampleSink,
   CodecConsistencyValidator: () => CodecConsistencyValidator,
   Detectors: () => Detectors,
+  GEOHASH_CELL_SIZES: () => GEOHASH_CELL_SIZES,
   InMemorySink: () => InMemorySink,
   IssueFanOutDetector: () => IssueFanOutDetector,
   IssueFanOutTypes: () => IssueFanOutTypes,
@@ -103,6 +104,7 @@ __export(src_exports, {
   createP2pRemoteTrackResolverFactory: () => createP2pRemoteTrackResolverFactory,
   defaultCallSummaryConfig: () => defaultCallSummaryConfig,
   defaultClientHealthThresholds: () => defaultClientHealthThresholds,
+  geohash: () => geohash,
   isClientIssueResolutionEntry: () => isClientIssueResolutionEntry,
   issuePayloadAsString: () => issuePayloadAsString,
   mannKendall: () => mannKendall,
@@ -157,8 +159,8 @@ function createLogger(moduleName) {
     }
   }();
 }
-function setObserverLogger(logger10) {
-  mainLogger = logger10;
+function setObserverLogger(logger11) {
+  mainLogger = logger11;
 }
 
 // src/ObservedCall.ts
@@ -5706,7 +5708,54 @@ var ObserverConcurrentIssueDetector = class _ObserverConcurrentIssueDetector {
   }
 };
 
+// src/utils/geohash.ts
+var BASE32 = "0123456789bcdefghjkmnpqrstuvwxyz";
+var GEOHASH_CELL_SIZES = [
+  "",
+  "~5000 km",
+  "~1250 km",
+  "~156 km",
+  "~39 km",
+  "~5 km",
+  "~1.2 km",
+  "~150 m"
+];
+function geohash(location, precision = 3) {
+  const { latitude, longitude } = location;
+  if (!Number.isFinite(latitude) || !Number.isFinite(longitude)) return void 0;
+  if (latitude < -90 || 90 < latitude) return void 0;
+  if (longitude < -180 || 180 < longitude) return void 0;
+  if (!Number.isFinite(precision) || precision < 1) return void 0;
+  const chars = Math.min(12, Math.floor(precision));
+  const latRange = { min: -90, max: 90 };
+  const lonRange = { min: -180, max: 180 };
+  let hash = "";
+  let bits = 0;
+  let value = 0;
+  let isLongitude = true;
+  while (hash.length < chars) {
+    const range = isLongitude ? lonRange : latRange;
+    const mid = (range.min + range.max) / 2;
+    const coordinate = isLongitude ? longitude : latitude;
+    if (mid <= coordinate) {
+      value = (value << 1) + 1;
+      range.min = mid;
+    } else {
+      value <<= 1;
+      range.max = mid;
+    }
+    isLongitude = !isLongitude;
+    if (++bits === 5) {
+      hash += BASE32[value];
+      bits = 0;
+      value = 0;
+    }
+  }
+  return hash;
+}
+
 // src/detectors/ClientPopulationIssueDetector.ts
+var logger9 = createLogger("ClientPopulationIssueDetector");
 var ClientPopulationIssueTypes = {
   /** One issue type is concentrated on one client population while the rest of the fleet is fine. */
   clientPopulationIssue: "CLIENT_POPULATION_ISSUE"
@@ -5717,6 +5766,7 @@ var ClientPopulationIssueDetector = class _ClientPopulationIssueDetector {
     this._config = {
       issueTypes: [],
       groupBy: "browser",
+      locationPrecision: 3,
       includeVersion: true,
       minPopulationSize: 20,
       minAffectedClients: 5,
@@ -5726,6 +5776,12 @@ var ClientPopulationIssueDetector = class _ClientPopulationIssueDetector {
       cooldownMs: 3e5,
       ...config
     };
+    if (this._config.groupBy === "location" && !this._config.resolveClientLocation) {
+      logger9.warn(
+        "%s is configured with groupBy: 'location' but no resolveClientLocation, so no client can be placed and nothing will ever be detected",
+        this.name
+      );
+    }
     for (const type of this._config.issueTypes) {
       this._observer.activeIssuesRegistry.addIssueTracker(type, this);
     }
@@ -5806,13 +5862,17 @@ var ClientPopulationIssueDetector = class _ClientPopulationIssueDetector {
         const key = `${population}:${issueType}`;
         if (now - (this._lastRaisedAt.get(key) ?? 0) < this._config.cooldownMs) continue;
         this._lastRaisedAt.set(key, now);
+        const isLocation = this._config.groupBy === "location";
         this._observer.addIssue({
           type: ClientPopulationIssueTypes.clientPopulationIssue,
           timestamp: now,
           conclusion: {
-            faultDomain: "client-population",
-            summary: `'${issueType}' is ${this._riskText(rollup.relativeRisk)} more likely on ${population} than on the rest of the fleet (${rollup.affectedClients}/${rollup.clients} vs ${rollup.controlAffectedClients}/${rollup.controlClients})`,
-            recommendation: "this is not an SFU symptom \u2014 look at what those clients share: a recent release, a browser version, or shared/virtualised hardware",
+            // Geography is not something endpoints own. A cluster of network symptoms in one
+            // place is a path/transit story, so it is attributed to the infrastructure rather
+            // than to a client build.
+            faultDomain: isLocation ? "infrastructure" : "client-population",
+            summary: `'${issueType}' is ${this._riskText(rollup.relativeRisk)} more likely ${isLocation ? `in cell ${population}` : `on ${population}`} than across the rest of the fleet (${rollup.affectedClients}/${rollup.clients} vs ${rollup.controlAffectedClients}/${rollup.controlClients})`,
+            recommendation: isLocation ? "look at the path into that region \u2014 ISP, transit or peering \u2014 but first rule out the servers those clients share, since a region often maps onto one SFU or TURN deployment" : "this is not an SFU symptom \u2014 look at what those clients share: a recent release, a browser version, or shared/virtualised hardware",
             confidence: this._confidenceOf(rollup)
           },
           payload: { ...rollup }
@@ -5827,6 +5887,10 @@ var ClientPopulationIssueDetector = class _ClientPopulationIssueDetector {
   }
   /** `undefined` when the client never reported this attribute — see the class description. */
   _populationOf(client) {
+    if (this._config.groupBy === "location") {
+      const location = this._config.resolveClientLocation?.(client);
+      return location ? geohash(location, this._config.locationPrecision) : void 0;
+    }
     const attribute = client[this._config.groupBy];
     if (!attribute?.name) return void 0;
     return this._config.includeVersion && attribute.version ? `${attribute.name} ${attribute.version}` : attribute.name;
@@ -6547,7 +6611,7 @@ function mediaKindOf(mimeType) {
 }
 
 // src/Observer.ts
-var logger9 = createLogger("Observer");
+var logger10 = createLogger("Observer");
 var Observer = class extends import_events6.EventEmitter {
   observedTURN = new ObservedTURN();
   observedCalls = /* @__PURE__ */ new Map();
@@ -6666,7 +6730,7 @@ var Observer = class extends import_events6.EventEmitter {
         break;
       }
       default: {
-        logger9.warn("Unknown detector name %s; skipping", name);
+        logger10.warn("Unknown detector name %s; skipping", name);
         return this;
       }
     }
@@ -6751,7 +6815,7 @@ var Observer = class extends import_events6.EventEmitter {
         break;
     }
     if (!validator) {
-      logger9.warn("Unknown validator name %s; skipping", name);
+      logger10.warn("Unknown validator name %s; skipping", name);
       return this;
     }
     this.validators.add(validator);
@@ -6782,7 +6846,7 @@ var Observer = class extends import_events6.EventEmitter {
       try {
         validator.cancel(reason);
       } catch (err) {
-        logger9.warn("Error cancelling validator %s: %o", validator.name, err);
+        logger10.warn("Error cancelling validator %s: %o", validator.name, err);
       }
     }
     return matching.length;
@@ -6798,11 +6862,11 @@ var Observer = class extends import_events6.EventEmitter {
    */
   createObservedCall(settings, acceptCtx) {
     if (this.closed) {
-      logger9.warn("Attempted to create a call (callId: %s) on a closed observer", settings.callId);
+      logger10.warn("Attempted to create a call (callId: %s) on a closed observer", settings.callId);
       return void 0;
     }
     if (this.observedCalls.has(settings.callId)) {
-      logger9.warn("Observed Call with id %s already exists; returning the existing instance", settings.callId);
+      logger10.warn("Observed Call with id %s already exists; returning the existing instance", settings.callId);
       return this.observedCalls.get(settings.callId);
     }
     const callSettings = {
@@ -6840,11 +6904,11 @@ var Observer = class extends import_events6.EventEmitter {
   }
   createObservedMediasoupRouter(settings) {
     if (this.closed) {
-      logger9.warn("Attempted to create mediasoup router (id: %d) on a closed observer", settings.router.id);
+      logger10.warn("Attempted to create mediasoup router (id: %d) on a closed observer", settings.router.id);
       return void 0;
     }
     if (this.observedMediasoupRouters.has(settings.router.id)) {
-      logger9.warn("Observed Mediasoup Router (id %s) already exists; returning the existing instance", settings.router.id);
+      logger10.warn("Observed Mediasoup Router (id %s) already exists; returning the existing instance", settings.router.id);
       return this.observedMediasoupRouters.get(settings.router.id);
     }
     const observedMediasoupRouter = new ObservedMediasoupRouter(settings);
@@ -6875,7 +6939,7 @@ var Observer = class extends import_events6.EventEmitter {
   }
   close() {
     if (this.closed) {
-      return logger9.debug("Attempted to close twice");
+      return logger10.debug("Attempted to close twice");
     }
     this.closed = true;
     for (const call of [...this.observedCalls.values()]) call.close();
@@ -6893,7 +6957,7 @@ var Observer = class extends import_events6.EventEmitter {
     try {
       this.acceptMiddlewares.process({ sample, context });
     } catch (err) {
-      logger9.warn("An accept middleware threw; dropping the sample. %o", err);
+      logger10.warn("An accept middleware threw; dropping the sample. %o", err);
     }
     if (!sample.callId) {
       this._notify("sample-rejected", { ...this.eventScope, reason: "missing-callId", sample });
@@ -6933,7 +6997,7 @@ var Observer = class extends import_events6.EventEmitter {
       try {
         validator.update();
       } catch (err) {
-        logger9.warn("Error running validator %s: %o", validator.name, err);
+        logger10.warn("Error running validator %s: %o", validator.name, err);
       }
     }
     this._notify("observer-updated", { ...this.eventScope });
@@ -7247,21 +7311,56 @@ var RemoteTrackResolver = class {
     const onOutboundRemoved = (p) => {
       if (p.observedCall === this.observedCall) this._removeOutboundTrack(p.observedOutboundTrack);
     };
+    const onInboundUpdated = (p) => {
+      if (p.observedCall !== this.observedCall) return;
+      if (this._pendingInboundTracks.has(p.observedInboundTrack)) this._addInboundTrack(p.observedInboundTrack);
+    };
+    const onOutboundUpdated = (p) => {
+      if (p.observedCall !== this.observedCall) return;
+      if (this._pendingOutboundTracks.has(p.observedOutboundTrack)) this._addOutboundTrack(p.observedOutboundTrack);
+    };
     observedCall.once("close", () => {
       observer.off("inbound-track-added", onInboundAdded);
       observer.off("inbound-track-removed", onInboundRemoved);
       observer.off("outbound-track-added", onOutboundAdded);
       observer.off("outbound-track-removed", onOutboundRemoved);
+      observer.off("inbound-track-updated", onInboundUpdated);
+      observer.off("outbound-track-updated", onOutboundUpdated);
+      this._pendingInboundTracks.clear();
+      this._pendingOutboundTracks.clear();
     });
     observer.on("inbound-track-added", onInboundAdded);
     observer.on("inbound-track-removed", onInboundRemoved);
     observer.on("outbound-track-added", onOutboundAdded);
     observer.on("outbound-track-removed", onOutboundRemoved);
+    observer.on("inbound-track-updated", onInboundUpdated);
+    observer.on("outbound-track-updated", onOutboundUpdated);
   }
   observedCall;
   resolvers;
   _publisherIdToOutboundTrack = /* @__PURE__ */ new Map();
   _subscriberIdToInboundTrack = /* @__PURE__ */ new Map();
+  /**
+   * Tracks whose publisher id the strategy could not resolve **yet**.
+   *
+   * A track announces itself once, but its `attachments` are replaced on every sample, so a key that
+   * is missing from the first sample can appear on the second — and a strategy backed by an
+   * application's own mapping (a server-side `ssrc -> producerId` table, say) is inherently racy
+   * against sample arrival. Resolving only at `*-track-added` meant losing those tracks for their
+   * entire life, silently: an unresolvable outbound track never even reaches
+   * `unconsumedOutboundTracks`, so it is invisible to `UnconsumedTrackDetector` too.
+   *
+   * So they wait here and are retried on their own `*-track-updated`, i.e. exactly when new stats
+   * arrived for them. A track leaves on its first successful resolution or on removal, which makes
+   * a linked track cost one `Set.has` per update and bounds these sets by the unresolved tracks
+   * alive right now.
+   */
+  _pendingInboundTracks = /* @__PURE__ */ new Set();
+  _pendingOutboundTracks = /* @__PURE__ */ new Set();
+  /** Tracks still waiting for a resolvable publisher id. Diagnostics; normally both are empty. */
+  get pendingTrackCounts() {
+    return { inbound: this._pendingInboundTracks.size, outbound: this._pendingOutboundTracks.size };
+  }
   /** The published (outbound) track for a publisher id, if any. */
   getOutboundTrackByPublisherId(publisherId) {
     return this._publisherIdToOutboundTrack.get(publisherId);
@@ -7279,7 +7378,11 @@ var RemoteTrackResolver = class {
   }
   _addInboundTrack(inboundTrack) {
     const publisherId = this.resolvers.resolveInboundTrackPublisherId(inboundTrack);
-    if (!publisherId) return;
+    if (!publisherId) {
+      this._pendingInboundTracks.add(inboundTrack);
+      return;
+    }
+    this._pendingInboundTracks.delete(inboundTrack);
     const subscriberId = this.resolvers.resolveInboundTrackSubscriberId?.(inboundTrack);
     if (subscriberId) {
       this._subscriberIdToInboundTrack.set(subscriberId, inboundTrack);
@@ -7291,6 +7394,7 @@ var RemoteTrackResolver = class {
     this.observedCall.unconsumedOutboundTracks.delete(outboundTrack);
   }
   _removeInboundTrack(inboundTrack) {
+    this._pendingInboundTracks.delete(inboundTrack);
     const outboundTrack = inboundTrack.remoteOutboundTrack;
     outboundTrack?.remoteInboundTracks.delete(inboundTrack);
     inboundTrack.remoteOutboundTrack = void 0;
@@ -7304,7 +7408,11 @@ var RemoteTrackResolver = class {
   }
   _addOutboundTrack(outboundTrack) {
     const publisherId = this.resolvers.resolveOutboundTrackPublisherId(outboundTrack);
-    if (!publisherId) return;
+    if (!publisherId) {
+      this._pendingOutboundTracks.add(outboundTrack);
+      return;
+    }
+    this._pendingOutboundTracks.delete(outboundTrack);
     this._publisherIdToOutboundTrack.set(publisherId, outboundTrack);
     this.observedCall.unconsumedOutboundTracks.add(outboundTrack);
     for (const observedClient of this.observedCall.observedClients.values()) {
@@ -7319,6 +7427,7 @@ var RemoteTrackResolver = class {
     }
   }
   _removeOutboundTrack(outboundTrack) {
+    this._pendingOutboundTracks.delete(outboundTrack);
     const publisherId = this.resolvers.resolveOutboundTrackPublisherId(outboundTrack);
     if (publisherId && this._publisherIdToOutboundTrack.get(publisherId) === outboundTrack) {
       this._publisherIdToOutboundTrack.delete(publisherId);
@@ -7371,6 +7480,7 @@ function createP2pRemoteTrackResolverFactory() {
   ClientSampleSink,
   CodecConsistencyValidator,
   Detectors,
+  GEOHASH_CELL_SIZES,
   InMemorySink,
   IssueFanOutDetector,
   IssueFanOutTypes,
@@ -7431,6 +7541,7 @@ function createP2pRemoteTrackResolverFactory() {
   createP2pRemoteTrackResolverFactory,
   defaultCallSummaryConfig,
   defaultClientHealthThresholds,
+  geohash,
   isClientIssueResolutionEntry,
   issuePayloadAsString,
   mannKendall,