@mentra/engine 3.4.0-dev.1207 → 3.4.0-dev.1209

package/src/services/fleet/FleetReporter.ts

@@ -217,6 +217,8 @@ export function createFleetReporter(deps: FleetReporterDeps = {}): FleetReporter
     clearPartition: (partition) => queue.clearPartition(partition),
     size: (partition) => queue.size(partition),
     nextSequence: (partition, collectorSessionId) => queue.nextSequence(partition, collectorSessionId),
+    noteTelemetryLoss: (partition, packageName, kind, count, at) => queue.noteTelemetryLoss(partition, packageName, kind, count, at),
+    takeTelemetryLosses: (partition, at) => queue.takeTelemetryLosses(partition, at),
   }
   const collector = (deps.createCollector ?? ((q: FleetQueue) => createFleetCollector({queue: q})))(collectorQueue)
 
@@ -429,6 +431,7 @@ export function createFleetReporter(deps: FleetReporterDeps = {}): FleetReporter
     const at = now()
     // takeLosses clears the counts it returns, so their gap is queued before anything else can happen.
     recordLosses(r, queue.takeLosses(r.partition, at), at)
+    recordTelemetryLosses(r, at)
     const records = queue.nextBatch(r.partition, r.batchLimit, MAX_BATCH_BYTES)
     if (records.length === 0) return ASLEEP
     let outcome: FleetSendOutcome
@@ -685,9 +688,46 @@ export function createFleetReporter(deps: FleetReporterDeps = {}): FleetReporter
     }
   }
 
+  /**
+   * The miniapp telemetry lost since the last upload, per package, as one `telemetry_loss` record. Telemetry losses
+   * never become coverage gaps. Counts that cannot be queued are lost with the events they counted.
+   */
+  function recordTelemetryLosses(r: Run, at: number): void {
+    const losses = queue.takeTelemetryLosses(r.partition, at)
+    if (losses === null) return
+    const entries = Object.entries(losses.packages)
+      .map(([packageName, counts]) => ({packageName, ...counts}))
+      .filter((loss) => loss.overLimit + loss.evicted + loss.expired + loss.rejected > 0)
+      .slice(0, FLEET_LIMITS.maxTelemetryLossPackages)
+    if (entries.length === 0) return
+    const end = iso(at)
+    const from = losses.from !== null && timeOf(losses.from) !== null && timeOf(losses.from)! <= at ? losses.from : end
+    try {
+      queue.enqueue(r.partition, {
+        schemaVersion: FLEET_SCHEMA_VERSION,
+        recordId: newId(),
+        phoneDeviceId: phoneDeviceId(),
+        collectorSessionId: r.collectorSessionId,
+        sequence: queue.nextSequence(r.partition, r.collectorSessionId),
+        observedAt: end,
+        capabilities: ["miniapp_telemetry"],
+        kind: "telemetry_loss",
+        payload: {from, to: end, losses: entries},
+      })
+    } catch (error) {
+      warn("recording telemetry losses", error)
+    }
+  }
+
   function recordRejected(r: Run, record: FleetRecord, at: number): void {
     // A refused gap gets no gap of its own, so a rejection can never feed itself.
     if (record.kind === "coverage_gap") return
+    // Rejected telemetry is counted for its miniapp, never as a gap; a refused loss report is not counted again.
+    if (record.kind === "telemetry_event") {
+      queue.noteTelemetryLoss(r.partition, record.payload.packageName, "rejected", 1, at)
+      return
+    }
+    if (record.kind === "telemetry_loss") return
     const time = iso(timeOf(record.observedAt) ?? at)
     enqueueGap(r, {reason: "record_rejected", from: time, to: time, droppedRecordCount: 1}, at)
   }

package/src/services/fleet/fleetCollector.ts

@@ -72,9 +72,18 @@ import {useGlassesStore} from "../../stores/glasses"
 import {BgTimer} from "../../utils/timers"
 import {miniappRunningRegistry} from "../MiniappRunningRegistry"
 import {nativeMiniappExecution} from "../NativeMiniappExecution"
-import {fleetEvents, type FleetLaunchEvent, type FleetPhotoEvent} from "./fleetEvents"
+import {
+  fleetEvents,
+  type FleetConnectionEvent,
+  type FleetCrashEvent,
+  type FleetInstallEvent,
+  type FleetLaunchEvent,
+  type FleetPhotoEvent,
+  type FleetTelemetryEvent,
+} from "./fleetEvents"
 import {newId, phoneDeviceId} from "./fleetIds"
-import type {FleetQueue} from "./fleetQueue"
+import {TELEMETRY_RECORD_PREFIX, type FleetQueue} from "./fleetQueue"
+import {limitEvent, TelemetryRate} from "./telemetryLimits"
 import {appPhoneBattery} from "./phoneBatterySource"
 import {
   glassesProduct,
@@ -208,6 +217,9 @@ const MODELS_WITHOUT_BATTERY: ReadonlySet<string> = new Set<string>()
  * What the phone measures without glasses. With glasses connected it measures all of them, less the battery of glasses
  * that cannot report one. A phone that cannot report its own battery does not claim `phone_battery`.
  */
+/** A disconnect this soon after the person asked for one is theirs. */
+const USER_DISCONNECT_WINDOW_MS = 10_000
+
 const PHONE_CAPABILITIES: Capability[] = [
   "glasses_connection",
   "miniapp_execution",
@@ -215,6 +227,10 @@ const PHONE_CAPABILITIES: Capability[] = [
   "engine_active",
   "photo_operations",
   "phone_battery",
+  // Phase 12: what this Mentra App measures whatever is connected.
+  "miniapp_failures",
+  "glasses_connectivity",
+  "miniapp_telemetry",
 ]
 
 function warn(what: string, error: unknown): void {
@@ -592,6 +608,12 @@ class CollectorRun {
   private lastSnapshotKey: string | null = null
 
   private known: KnownGlasses | null = null
+  /** The store's last connection state, and when the glasses last dropped (for a reconnect's time). */
+  private rawConnection: string | null = null
+  private disconnectedAt: number | null = null
+  private connectFailed = false
+  /** Each miniapp's telemetry rate on this phone, for this run. */
+  private readonly telemetryRate = new TelemetryRate()
   private heldPhotos: HeldPhoto[] = []
   private battery: BatteryReading | null = null
   private phoneBattery: PhoneBatteryHeld | null = null
@@ -624,6 +646,10 @@ class CollectorRun {
       fleetEvents.subscribe({
         launch: (event) => this.guard("launch", () => this.onLaunch(event)),
         photo: (event) => this.guard("photo", () => this.onPhoto(event)),
+        telemetry: (event) => this.guard("telemetry", () => this.onTelemetry(event)),
+        crash: (event) => this.guard("crash", () => this.onCrash(event)),
+        install: (event) => this.guard("install", () => this.onInstall(event)),
+        connection: (event) => this.guard("connection", () => this.onConnection(event)),
       }),
     )
     const appState = AppState.addEventListener("change", (state) =>
@@ -712,10 +738,42 @@ class CollectorRun {
     this.evaluateSnapshotTriggers()
   }
 
+  /**
+   * Phase 12's connection events, from the store's own states: connected to disconnected is a disconnect (the
+   * person's own within 10 s of their asking, otherwise a lost link); connecting or bonding to disconnected without
+   * connecting is a failed connection, once until the glasses connect again and never while a lost link waits to come
+   * back; connected again after a disconnect is a reconnect, with the time it took.
+   * Simulated glasses have none.
+   */
+  private noteConnectionChange(raw: string, simulated: boolean, wall: number): void {
+    const previous = this.rawConnection
+    this.rawConnection = raw
+    if (previous === null || previous === raw || simulated) return
+    if (raw === "disconnected" && previous === "connected") {
+      const theirs = wall - fleetEvents.lastUserDisconnect() < USER_DISCONNECT_WINDOW_MS
+      this.onConnection({event: "disconnected", category: theirs ? "user_action" : "link_lost"})
+      this.disconnectedAt = theirs ? null : wall
+    } else if (raw === "disconnected" && (previous === "connecting" || previous === "bonding")) {
+      // Glasses away retry all the time: a lost link waiting to come back is its disconnect, and only the first failed
+      // connection between two connected spells is one, so an hour of retries is not an hour of records.
+      if (this.disconnectedAt === null && !this.connectFailed) {
+        this.connectFailed = true
+        this.onConnection({event: "connect_failed", category: "unknown"})
+      }
+    } else if (raw === "connected") {
+      this.connectFailed = false
+      if (this.disconnectedAt !== null) {
+        this.onConnection({event: "reconnected", category: "unknown", reconnectMs: wall - this.disconnectedAt})
+        this.disconnectedAt = null
+      }
+    }
+  }
+
   /** Notes what the glasses store says now: the connection, the glasses' identity and versions, and the battery. */
   private trackGlasses(wall: number): void {
     const s = useGlassesStore.getState()
     const state = mapConnection(s.connection.state)
+    this.noteConnectionChange(s.connection.state, isSimulatedGlasses(s.deviceModel), wall)
     if (state !== this.connection.state) {
       // Glasses that start connecting are not assumed to be the pair that was last here: a new connection is
       // identified afresh, from what the glasses say, before anything is attributed to a device. Only a new
@@ -1402,6 +1460,102 @@ class CollectorRun {
         ...(versionName ? {versionName} : {}),
         start: event.start,
         outcome: event.outcome,
+        ...(event.outcome === "failed" && event.failureReason ? {failureReason: event.failureReason} : {}),
+      },
+    }))
+  }
+
+  // --- Phase 12: a miniapp's own events, crashes, installs and connection changes -------------------------------
+
+  /** The open execution session of `packageName`, which a telemetry event or crash is attributed to. */
+  private executionOf(packageName: string): string | undefined {
+    return this.sessions.get(`miniapp_execution:${packageName}`)?.id
+  }
+
+  private onTelemetry(event: FleetTelemetryEvent): void {
+    const packageName = text(event.packageName)
+    if (!packageName) return
+    const at = this.env.now()
+    const limited = limitEvent(event.event, event.properties)
+    if (limited === null || !this.telemetryRate.admit(packageName, at)) {
+      this.env.queue.noteTelemetryLoss(this.partition, packageName, "overLimit", 1, at)
+      return
+    }
+    const versionName = text(event.versionName)
+    const executionSessionId = this.executionOf(packageName)
+    this.emit(
+      at,
+      (base) => ({
+        ...base,
+        kind: "telemetry_event",
+        payload: {
+          packageName,
+          ...(versionName ? {versionName} : {}),
+          ...(executionSessionId ? {executionSessionId} : {}),
+          event: limited.event,
+          properties: limited.properties,
+          droppedProperties: limited.droppedProperties,
+          cutValues: limited.cutValues,
+        },
+      }),
+      TELEMETRY_RECORD_PREFIX,
+    )
+  }
+
+  private onCrash(event: FleetCrashEvent): void {
+    const packageName = text(event.packageName)
+    if (!packageName) return
+    const versionName = text(event.versionName)
+    const executionSessionId = this.executionOf(packageName)
+    this.emit(this.env.now(), (base) => ({
+      ...base,
+      kind: "miniapp_crash",
+      payload: {
+        packageName,
+        ...(versionName ? {versionName} : {}),
+        ...(executionSessionId ? {executionSessionId} : {}),
+        surface: event.surface,
+        category: event.category,
+        outcome: event.outcome,
+        crashesInWindow: Math.min(FLEET_LIMITS.maxCrashesInWindow, Math.max(1, Math.round(event.crashesInWindow))),
+      },
+    }))
+  }
+
+  private onInstall(event: FleetInstallEvent): void {
+    const packageName = text(event.packageName)
+    if (!packageName) return
+    const fromVersion = text(event.fromVersion)
+    const toVersion = text(event.toVersion)
+    this.emit(this.env.now(), (base) => ({
+      ...base,
+      kind: "miniapp_install",
+      payload: {
+        packageName,
+        operation: event.operation,
+        source: event.source,
+        ...(fromVersion ? {fromVersion} : {}),
+        ...(toVersion ? {toVersion} : {}),
+        outcome: event.outcome,
+        ...(event.outcome === "succeeded" ? {} : {reason: event.reason ?? "unknown"}),
+      },
+    }))
+  }
+
+  private onConnection(event: FleetConnectionEvent): void {
+    const binding = this.currentBinding() ?? this.known?.binding ?? null
+    const glassesModel = this.known?.model
+    const appVersion = phoneInfo()?.appVersion ?? "unknown"
+    this.emit(this.env.now(), (base) => ({
+      ...base,
+      kind: "connection_event",
+      ...(binding ? {deviceBinding: binding} : {}),
+      payload: {
+        event: event.event,
+        category: event.category,
+        ...(glassesModel ? {glassesModel} : {}),
+        appVersion,
+        ...(event.event === "reconnected" ? {reconnectMs: Math.max(0, Math.round(event.reconnectMs ?? 0))} : {}),
       },
     }))
   }
@@ -1451,10 +1605,10 @@ class CollectorRun {
     return [...measured]
   }
 
-  private base(observedAt: number): RecordBase {
+  private base(observedAt: number, idPrefix = ""): RecordBase {
     return {
       schemaVersion: FLEET_SCHEMA_VERSION,
-      recordId: newId(),
+      recordId: `${idPrefix}${newId()}`,
       phoneDeviceId: this.phoneId,
       collectorSessionId: this.collectorSessionId,
       sequence: this.env.queue.nextSequence(this.partition, this.collectorSessionId),
@@ -1464,9 +1618,9 @@ class CollectorRun {
   }
 
   /** Builds a record and queues it in this run's partition. A record that cannot be built or stored is skipped. */
-  private emit(observedAt: number, make: (base: RecordBase) => FleetRecord): void {
+  private emit(observedAt: number, make: (base: RecordBase) => FleetRecord, idPrefix = ""): void {
     try {
-      this.env.queue.enqueue(this.partition, make(this.base(observedAt)))
+      this.env.queue.enqueue(this.partition, make(this.base(observedAt, idPrefix)))
     } catch (error) {
       warn("recording", error)
     }

package/src/services/fleet/fleetEvents.ts

@@ -15,9 +15,20 @@
  * the engine restores miniapps while the account it collects for may still be being resolved, and those launches are
  * that account's. The reporter discards them when the account changes. Photo stages are not kept.
  *
+ * Phase 12 adds a miniapp's own events (`telemetry`, from `session.telemetry.capture`, attributed by the runtime to the
+ * package whose bridge sent it), its crashes, install attempts, and the glasses' pairing and connection changes. None of
+ * these is kept while nothing listens: with no run collecting (Fleet not installed on the phone's Core, nobody signed
+ * in) they are dropped, and telemetry dropped this way is not counted.
+ *
  * This module imports nothing at run time, so the hot paths that import it carry no Fleet weight.
  */
-import type {FleetRecord} from "@mentra/cloud-protocol"
+import type {
+  FleetConnectionEventPayload,
+  FleetLaunchFailureReason,
+  FleetMiniappCrashPayload,
+  FleetMiniappInstallPayload,
+  FleetRecord,
+} from "@mentra/cloud-protocol"
 
 /** A miniapp start the engine attempted, and how it went. */
 export type FleetLaunchEvent = Extract<FleetRecord, {kind: "miniapp_launch"}>["payload"]
@@ -34,18 +45,42 @@ export interface UndeliveredLaunch {
   observedAt: number
 }
 
+/** A miniapp's own event as the bridge delivered it: unchecked, the collector applies the limits. */
+export interface FleetTelemetryEvent {
+  packageName: string
+  versionName?: string
+  event: unknown
+  properties: unknown
+}
+
+/** A crash: the collector adds the miniapp's execution session. */
+export type FleetCrashEvent = Omit<FleetMiniappCrashPayload, "executionSessionId">
+export type FleetInstallEvent = FleetMiniappInstallPayload
+/** A pairing or connection change: the collector adds the glasses, their model and the Mentra App's version. */
+export type FleetConnectionEvent = Pick<FleetConnectionEventPayload, "event" | "category" | "reconnectMs">
+
 export interface FleetEventListener {
   launch?: (event: FleetLaunchEvent) => void
   photo?: (event: FleetPhotoEvent) => void
+  telemetry?: (event: FleetTelemetryEvent) => void
+  crash?: (event: FleetCrashEvent) => void
+  install?: (event: FleetInstallEvent) => void
+  connection?: (event: FleetConnectionEvent) => void
 }
 
 /** More launches than this before the collector starts are not one boot's restores; the oldest go first. */
 const MAX_UNDELIVERED_LAUNCHES = 32
 
 const listeners = new Set<FleetEventListener>()
+let lastUserDisconnectAt = 0
 const undelivered: UndeliveredLaunch[] = []
+/** A start under way that reports its own launch: why a spawn it held back failed, if one did. */
+interface LaunchClaim {
+  failureReason?: FleetLaunchFailureReason
+}
+
 /** Starts under way that report their own launch, by package. */
-const claims = new Map<string, number>()
+const claims = new Map<string, Set<LaunchClaim>>()
 
 function warnListenerFailed(error: unknown): void {
   // The event is left out of the warning: it names a package or an operation.
@@ -86,20 +121,34 @@ export const fleetEvents = {
    * same miniapp is under way, which reports it. Never throws.
    */
   launchUnlessClaimed(event: FleetLaunchEvent): void {
-    if ((claims.get(event.packageName) ?? 0) > 0) return
-    deliverLaunch(event)
+    const held = claims.get(event.packageName)
+    if (!held) {
+      deliverLaunch(event)
+      return
+    }
+    // The start reports this attempt as its own, so it keeps the reason the spawn failed for.
+    const reason = event.failureReason
+    if (reason === undefined) return
+    for (const claim of held) if (claim.failureReason === undefined || claim.failureReason === "unknown") claim.failureReason = reason
   },
 
-  /** A start of `packageName` that reports its own launch is under way, until the returned function is called. */
-  claimLaunch(packageName: string): () => void {
-    claims.set(packageName, (claims.get(packageName) ?? 0) + 1)
+  /**
+   * A start of `packageName` that reports its own launch is under way, until the returned function is called. That
+   * function returns why a spawn the start held back failed, for the start's own report, or undefined.
+   */
+  claimLaunch(packageName: string): () => FleetLaunchFailureReason | undefined {
+    const claim: LaunchClaim = {}
+    const held = claims.get(packageName) ?? new Set<LaunchClaim>()
+    held.add(claim)
+    claims.set(packageName, held)
     let released = false
     return () => {
-      if (released) return
-      released = true
-      const left = (claims.get(packageName) ?? 1) - 1
-      if (left > 0) claims.set(packageName, left)
-      else claims.delete(packageName)
+      if (!released) {
+        released = true
+        held.delete(claim)
+        if (held.size === 0 && claims.get(packageName) === held) claims.delete(packageName)
+      }
+      return claim.failureReason
     }
   },
 
@@ -115,13 +164,48 @@ export const fleetEvents = {
 
   /** A photo operation reached a stage. Never throws. */
   photo(event: FleetPhotoEvent): void {
-    if (listeners.size === 0) return
-    for (const listener of [...listeners]) {
-      try {
-        listener.photo?.(event)
-      } catch (error) {
-        warnListenerFailed(error)
-      }
-    }
+    deliver("photo", event)
+  },
+
+  /** A miniapp captured one of its own events. Never throws. */
+  telemetry(event: FleetTelemetryEvent): void {
+    deliver("telemetry", event)
+  },
+
+  /** A miniapp crashed. Never throws. */
+  crash(event: FleetCrashEvent): void {
+    deliver("crash", event)
   },
+
+  /** An install, update or removal of a miniapp was attempted. Never throws. */
+  install(event: FleetInstallEvent): void {
+    deliver("install", event)
+  },
+
+  /** The person asked the glasses to disconnect (or forgot or unpaired them): the next disconnect is theirs. */
+  userDisconnect(): void {
+    lastUserDisconnectAt = Date.now()
+  },
+
+  /** When the person last asked the glasses to disconnect (epoch ms), or 0. */
+  lastUserDisconnect(): number {
+    return lastUserDisconnectAt
+  },
+
+  /** The glasses' pairing or connection changed. Never throws. */
+  connection(event: FleetConnectionEvent): void {
+    deliver("connection", event)
+  },
+}
+
+/** Hands `event` to every listener of `kind`; with none, it is dropped. */
+function deliver<K extends "photo" | "telemetry" | "crash" | "install" | "connection">(kind: K, event: Parameters<NonNullable<FleetEventListener[K]>>[0]): void {
+  if (listeners.size === 0) return
+  for (const listener of [...listeners]) {
+    try {
+      ;(listener[kind] as ((value: typeof event) => void) | undefined)?.(event)
+    } catch (error) {
+      warnListenerFailed(error)
+    }
+  }
 }