@glassly/engine 0.1.0-dev.0

package/src/utils/hardware/hardware.ts

@@ -0,0 +1,171 @@
+import {
+  Capabilities,
+  HardwareRequirement,
+  HardwareType,
+  HardwareRequirementLevel,
+  DeviceTypes,
+  simulatedGlasses,
+} from "../../types"
+
+/**
+ * Result of a hardware compatibility check
+ */
+export interface CompatibilityResult {
+  isCompatible: boolean
+  missingRequired: HardwareRequirement[]
+  missingOptional: HardwareRequirement[]
+  warnings: string[]
+}
+
+/**
+ * Service for checking hardware compatibility between apps and glasses
+ */
+export class HardwareCompatibility {
+  /**
+   * Check if app is compatible with given device capabilities
+   * @param app The app to check
+   * @param capabilities The device capabilities (null if no device connected)
+   * @returns Detailed compatibility result
+   */
+  static checkCompatibility(
+    hardwareRequirements: HardwareRequirement[],
+    capabilities: Capabilities | null,
+  ): CompatibilityResult {
+    const result: CompatibilityResult = {
+      isCompatible: true,
+      missingRequired: [],
+      missingOptional: [],
+      warnings: [],
+    }
+
+    // If no hardware requirements specified, app is compatible with any hardware
+    if (hardwareRequirements.length === 0) {
+      return result
+    }
+
+    // If no capabilities available assume simulated glasses:
+    if (!capabilities) {
+      capabilities = simulatedGlasses
+    }
+
+    // Check each hardware requirement
+    for (const requirement of hardwareRequirements) {
+      const hasHardware = this.checkHardwareAvailable(requirement.type, capabilities!)
+      if (!hasHardware) {
+        if (requirement.level === HardwareRequirementLevel.REQUIRED) {
+          result.missingRequired.push(requirement)
+          result.isCompatible = false
+        } else {
+          result.missingOptional.push(requirement)
+        }
+      }
+    }
+
+    return result
+  }
+
+  /**
+   * Check if specific hardware is available in capabilities
+   * @param hardwareType The type of hardware to check
+   * @param capabilities The device capabilities
+   * @returns true if hardware is available
+   */
+  private static checkHardwareAvailable(hardwareType: HardwareType, capabilities: Capabilities): boolean {
+    if (hardwareType === HardwareType.EXIST) {
+      if (capabilities.modelName === DeviceTypes.NONE) {
+        return false
+      }
+      return true
+    }
+
+    switch (hardwareType) {
+      case HardwareType.CAMERA:
+        return capabilities.hasCamera
+
+      case HardwareType.DISPLAY:
+        return capabilities.hasDisplay
+
+      case HardwareType.MICROPHONE:
+        // GlasslyOS always provides an audio-input path. When the connected
+        // glasses do not have an onboard microphone, the engine falls back to
+        // the phone microphone. Keep `capabilities.hasMicrophone` reserved for
+        // physical-device/source selection; app compatibility is about whether
+        // the requested input is available to the miniapp.
+        return true
+
+      case HardwareType.SPEAKER:
+        return capabilities.hasSpeaker
+
+      case HardwareType.IMU:
+        return capabilities.hasIMU
+
+      case HardwareType.BUTTON:
+        return capabilities.hasButton
+
+      case HardwareType.LIGHT:
+        return capabilities.hasLight
+
+      case HardwareType.WIFI:
+        return capabilities.hasWifi
+
+      default:
+        // Unknown hardware type - assume not available
+        return false
+    }
+  }
+
+  /**
+   * Get human-readable compatibility message
+   * @param result The compatibility check result
+   * @returns User-friendly message about compatibility
+   */
+  static getCompatibilityMessage(result: CompatibilityResult): string {
+    if (result.isCompatible) {
+      if (result.missingOptional.length > 0) {
+        const optionalHardware = result.missingOptional.map((req) => req.type.toLowerCase()).join(", ")
+        return `This app works with your glasses but has optional features that require: ${optionalHardware}`
+      }
+      return "This app is fully compatible with your glasses"
+    }
+
+    const requiredHardware = result.missingRequired.map((req) => req.type.toLowerCase()).join(", ")
+
+    if (result.missingRequired.length === 1) {
+      return `This app requires a ${requiredHardware} which is not available on your connected glasses`
+    } else {
+      return `This app requires the following hardware which is not available on your connected glasses: ${requiredHardware}`
+    }
+  }
+
+  /**
+   * Get detailed compatibility messages including descriptions
+   * @param result The compatibility check result
+   * @returns Array of detailed messages
+   */
+  static getDetailedMessages(result: CompatibilityResult): string[] {
+    const messages: string[] = []
+
+    // Add warnings first
+    messages.push(...result.warnings)
+
+    // Add missing required hardware
+    for (const req of result.missingRequired) {
+      let message = `❌ Missing required ${req.type.toLowerCase()}`
+      if (req.description) {
+        message += `: ${req.description}`
+      }
+      messages.push(message)
+    }
+
+    // Add missing optional hardware
+    for (const req of result.missingOptional) {
+      let message = `⚠️ Missing optional ${req.type.toLowerCase()}`
+      if (req.description) {
+        message += `: ${req.description}`
+      }
+      messages.push(message)
+    }
+
+    return messages
+  }
+}

package/src/utils/hardware/index.ts

@@ -0,0 +1 @@
+export * from "./hardware"

package/src/utils/memory/search.ts

@@ -0,0 +1,204 @@
+/**
+ * Pure memory-timeline search — the matching engine behind MemoryStore.query
+ * and the assistant's get_transcripts tool.
+ *
+ * Lives outside MemoryStore so it is unit-testable and importable by the
+ * assistant's eval harness (which replays the exact host semantics against a
+ * mock corpus). Keep it dependency-free and runtime-neutral: it also runs
+ * under bun for tests.
+ *
+ * TWO MATCH MODES, deliberately different:
+ *  - `contains`: exact case-insensitive substring — the pre-existing contract,
+ *    kept verbatim for callers that need precision (codes, URLs, exact phrases).
+ *  - `terms`: ANY-of term matching with per-word fuzziness. Speech-to-text
+ *    mangles proper nouns ("Sompo" for "Sampo", "Korwin" for "Corwin"), so a
+ *    term word matches a transcript token on exact/substring, small edit
+ *    distance, OR same Soundex key. A multi-word term ("ring size") requires
+ *    all of its words to match somewhere in the entry.
+ *
+ * The scan walks BACKWARDS from the newest in-window entry and stops at
+ * `limit` matches: the caller keeps the most recent matches anyway, and the
+ * common "last 90 minutes" query then touches dozens of entries instead of
+ * the full 4000-entry timeline. Window bounds are found by binary search on
+ * `at` (the timeline is maintained time-sorted).
+ */
+
+export interface MemorySearchableEntry {
+  text: string
+  at: number
+}
+
+export interface MemorySearchQuery {
+  /** Epoch ms lower bound (inclusive). */
+  sinceMs?: number
+  /** Epoch ms upper bound (inclusive). */
+  untilMs?: number
+  /** Exact case-insensitive substring filter. */
+  contains?: string
+  /** Fuzzy any-of terms (see file header). Combined with `contains` as AND. */
+  terms?: string[]
+  /** Max matches kept (newest win). Caller enforces its own cap. */
+  limit: number
+}
+
+export interface MemorySearchResult<T> {
+  /** Matches in chronological order (newest last), at most `limit`. */
+  entries: T[]
+  /**
+   * Entries inside the TIME window before keyword filtering. Lets the caller
+   * tell "keyword matched nothing" apart from "window itself is empty" — the
+   * distinction that saves an LLM a blind retry round.
+   */
+  windowCount: number
+}
+
+/** Tokenized lowercase text per entry, cached for the entry object's lifetime. */
+const tokenCache = new WeakMap<object, string[]>()
+
+export function searchMemoryEntries<T extends MemorySearchableEntry>(
+  entries: readonly T[],
+  q: MemorySearchQuery,
+): MemorySearchResult<T> {
+  const since = q.sinceMs ?? 0
+  const until = q.untilMs ?? Number.MAX_SAFE_INTEGER
+  const lo = lowerBound(entries, since)
+  const hi = upperBound(entries, until) // exclusive
+  const windowCount = Math.max(0, hi - lo)
+
+  const needle = q.contains?.trim().toLowerCase() || null
+  const termWords = (q.terms ?? [])
+    .map((t) => tokenize(t))
+    .filter((words) => words.length > 0)
+
+  const picked: T[] = []
+  for (let i = hi - 1; i >= lo && picked.length < q.limit; i--) {
+    const e = entries[i]
+    if (needle && !e.text.toLowerCase().includes(needle)) continue
+    if (termWords.length > 0 && !matchesAnyTerm(e, termWords)) continue
+    picked.push(e)
+  }
+  picked.reverse()
+  return {entries: picked, windowCount}
+}
+
+/** True when any term matches — a term matches iff ALL its words match a token. */
+function matchesAnyTerm(entry: MemorySearchableEntry, termWords: string[][]): boolean {
+  let tokens = tokenCache.get(entry)
+  if (!tokens) {
+    tokens = tokenize(entry.text)
+    tokenCache.set(entry, tokens)
+  }
+  outer: for (const words of termWords) {
+    for (const word of words) {
+      if (!tokens.some((tok) => fuzzyWordMatch(word, tok))) continue outer
+    }
+    return true
+  }
+  return false
+}
+
+/**
+ * Does query word `w` match transcript token `t`?
+ *  - exact, or substring for w of 3+ chars ("kestrel" in "kestrels")
+ *  - for words of 5+ chars: bounded edit distance (1, or 2 for 7+ chars)
+ *    — catches one-vowel STT swaps ("sompo"/"sampo", "korwin"/"corwin").
+ *    5 is the floor because at 4 chars a one-edit radius swallows unrelated
+ *    common words ("code"→"mode"); mistranscribed NAMES are almost always 5+.
+ *  - same Soundex key with similar length — catches phonetic respellings that
+ *    edit distance misses ("warryn"/"warren" style doublings)
+ */
+export function fuzzyWordMatch(w: string, t: string): boolean {
+  if (w === t) return true
+  if (w.length >= 3 && t.includes(w)) return true
+  if (w.length < 5 || t.length < 4) return false
+  const maxDist = w.length >= 7 ? 2 : 1
+  if (Math.abs(w.length - t.length) <= maxDist && editDistanceAtMost(w, t, maxDist)) return true
+  return Math.abs(w.length - t.length) <= 2 && soundex(w) === soundex(t)
+}
+
+/** Lowercase, strip diacritics, split on non-alphanumerics. */
+export function tokenize(text: string): string[] {
+  return text
+    .toLowerCase()
+    .normalize("NFD")
+    .replace(/[\u0300-\u036f]/g, "")
+    .split(/[^a-z0-9]+/)
+    .filter((t) => t.length > 0)
+}
+
+/**
+ * Optimal-string-alignment edit distance, early-exited: returns whether
+ * dist(a, b) <= max. Transpositions count as one edit (common in STT output).
+ */
+export function editDistanceAtMost(a: string, b: string, max: number): boolean {
+  if (Math.abs(a.length - b.length) > max) return false
+  let prev2: number[] | null = null
+  let prev = Array.from({length: b.length + 1}, (_, j) => j)
+  for (let i = 1; i <= a.length; i++) {
+    const cur = [i]
+    let rowMin = i
+    for (let j = 1; j <= b.length; j++) {
+      const cost = a[i - 1] === b[j - 1] ? 0 : 1
+      let v = Math.min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + cost)
+      if (prev2 && i > 1 && j > 1 && a[i - 1] === b[j - 2] && a[i - 2] === b[j - 1]) {
+        v = Math.min(v, prev2[j - 2] + 1)
+      }
+      cur.push(v)
+      if (v < rowMin) rowMin = v
+    }
+    if (rowMin > max) return false
+    prev2 = prev
+    prev = cur
+  }
+  return prev[b.length] <= max
+}
+
+const SOUNDEX_CODES: Record<string, string> = {
+  b: "1", f: "1", p: "1", v: "1",
+  c: "2", g: "2", j: "2", k: "2", q: "2", s: "2", x: "2", z: "2",
+  d: "3", t: "3",
+  l: "4",
+  m: "5", n: "5",
+  r: "6",
+}
+
+/** Classic Soundex. Empty input returns "". */
+export function soundex(word: string): string {
+  const w = word.toLowerCase().replace(/[^a-z]/g, "")
+  if (!w) return ""
+  let out = w[0].toUpperCase()
+  let lastCode = SOUNDEX_CODES[w[0]] ?? ""
+  for (let i = 1; i < w.length && out.length < 4; i++) {
+    const ch = w[i]
+    const code = SOUNDEX_CODES[ch] ?? ""
+    // h/w are transparent: they don't reset the adjacent-duplicate rule.
+    if (ch === "h" || ch === "w") continue
+    if (code && code !== lastCode) out += code
+    lastCode = code
+  }
+  return out.padEnd(4, "0")
+}
+
+/** First index with at >= value (entries time-sorted ascending). */
+function lowerBound(entries: readonly MemorySearchableEntry[], value: number): number {
+  let lo = 0
+  let hi = entries.length
+  while (lo < hi) {
+    const mid = (lo + hi) >> 1
+    if (entries[mid].at < value) lo = mid + 1
+    else hi = mid
+  }
+  return lo
+}
+
+/** First index with at > value (i.e. exclusive end for an inclusive bound). */
+function upperBound(entries: readonly MemorySearchableEntry[], value: number): number {
+  let lo = 0
+  let hi = entries.length
+  while (lo < hi) {
+    const mid = (lo + hi) >> 1
+    if (entries[mid].at <= value) lo = mid + 1
+    else hi = mid
+  }
+  return lo
+}

package/src/utils/miniappDiagnostics.ts

@@ -0,0 +1,226 @@
+import type {ClientApp} from "../types/applet"
+
+const MAX_DIAGNOSTIC_STRING_LENGTH = 512
+const MAX_DIAGNOSTIC_LIST_ITEMS = 100
+const MAX_RECENT_MINIAPPS = 20
+
+export interface MiniappManifestSnapshot {
+  packageName: string
+  name?: string
+  version?: string
+  sdkVersion?: string
+  minHostVersion?: string
+  type?: string
+  entry?: {
+    background?: string
+    ui?: string
+  }
+  permissions: Array<{type: string; required?: boolean}>
+  hardwareRequirements: Array<{type: string; level: string}>
+  actionIds: string[]
+}
+
+export interface MiniappRuntimeAppSnapshot {
+  packageName: string
+  handshakeComplete: boolean
+  subscriptions: string[]
+  lastMessageAgeMs: number
+  requestedLocationRate: string | null
+  hasActiveSpeakerStream: boolean
+  manifest?: MiniappManifestSnapshot
+}
+
+export interface MiniappRuntimeDiagnosticSnapshot {
+  connectedApps: MiniappRuntimeAppSnapshot[]
+  subscriptionsByStream: Record<string, string[]>
+  resources: {
+    display: Record<string, unknown>
+    camera: Record<string, unknown>
+    video: Record<string, unknown>
+    stream: Record<string, unknown>
+    cameraFov: Record<string, unknown>
+  }
+}
+
+type ManifestFallback = {
+  packageName: string
+  name?: string
+  version?: string
+  type?: string
+}
+
+function boundedString(value: unknown): string | undefined {
+  if (typeof value !== "string") return undefined
+  const trimmed = value.trim()
+  if (!trimmed) return undefined
+  return trimmed.slice(0, MAX_DIAGNOSTIC_STRING_LENGTH)
+}
+
+function recordValue(value: unknown): Record<string, unknown> | null {
+  return value && typeof value === "object" && !Array.isArray(value) ? (value as Record<string, unknown>) : null
+}
+
+function diagnosticPermissions(value: unknown): Array<{type: string; required?: boolean}> {
+  if (!Array.isArray(value)) return []
+  const permissions: Array<{type: string; required?: boolean}> = []
+  for (const raw of value) {
+    if (typeof raw === "string") {
+      const type = boundedString(raw)
+      if (type) permissions.push({type})
+      continue
+    }
+    const permission = recordValue(raw)
+    const type = boundedString(permission?.type)
+    if (!type) continue
+    permissions.push({
+      type,
+      ...(typeof permission?.required === "boolean" ? {required: permission.required} : {}),
+    })
+  }
+  return permissions.slice(0, MAX_DIAGNOSTIC_LIST_ITEMS)
+}
+
+function diagnosticHardwareRequirements(value: unknown): Array<{type: string; level: string}> {
+  if (!Array.isArray(value)) return []
+  const requirements: Array<{type: string; level: string}> = []
+  for (const raw of value) {
+    const requirement = recordValue(raw)
+    const type = boundedString(requirement?.type)
+    const level = boundedString(requirement?.level)
+    if (type && level) requirements.push({type, level})
+  }
+  return requirements.slice(0, MAX_DIAGNOSTIC_LIST_ITEMS)
+}
+
+function diagnosticActionIds(value: unknown): string[] {
+  if (!Array.isArray(value)) return []
+  return value
+    .map((raw) => boundedString(typeof raw === "string" ? raw : recordValue(raw)?.id))
+    .filter((id): id is string => id !== undefined)
+    .sort()
+    .slice(0, MAX_DIAGNOSTIC_LIST_ITEMS)
+}
+
+/**
+ * Keep the routing/build fields from miniapp.json while excluding arbitrary
+ * free-form manifest content from reports. Descriptions, URLs, and unknown
+ * extension fields are intentionally omitted.
+ */
+export function buildMiniappManifestSnapshot(raw: unknown, fallback: ManifestFallback): MiniappManifestSnapshot {
+  const manifest = recordValue(raw)
+  const entry = recordValue(manifest?.entry)
+  const packageName = boundedString(manifest?.packageName) ?? boundedString(fallback.packageName) ?? "unknown"
+  const name = boundedString(manifest?.name) ?? boundedString(fallback.name)
+  const version = boundedString(manifest?.version) ?? boundedString(fallback.version)
+  const type = boundedString(manifest?.type) ?? boundedString(fallback.type)
+  const background = boundedString(entry?.background)
+  const ui = boundedString(entry?.ui)
+
+  return {
+    packageName,
+    ...(name ? {name} : {}),
+    ...(version ? {version} : {}),
+    ...(boundedString(manifest?.sdkVersion) ? {sdkVersion: boundedString(manifest?.sdkVersion)} : {}),
+    ...(boundedString(manifest?.minHostVersion) ? {minHostVersion: boundedString(manifest?.minHostVersion)} : {}),
+    ...(type ? {type} : {}),
+    ...(background || ui ? {entry: {...(background ? {background} : {}), ...(ui ? {ui} : {})}} : {}),
+    permissions: diagnosticPermissions(manifest?.permissions),
+    hardwareRequirements: diagnosticHardwareRequirements(manifest?.hardwareRequirements),
+    actionIds: diagnosticActionIds(manifest?.actions ?? manifest?.actionIds),
+  }
+}
+
+function executionMode(app: ClientApp): "built_in" | "dev_server" | "dev_snapshot" | "installed_bundle" {
+  if (app.offline) return "built_in"
+  if (app.isMiniappDev && app.devUrl && !app.version) return "dev_server"
+  if (app.isMiniappDev) return "dev_snapshot"
+  return "installed_bundle"
+}
+
+function safeReleaseIdentity(value: unknown): Record<string, string> | undefined {
+  const identity = recordValue(value)
+  if (!identity) return undefined
+  const out: Record<string, string> = {}
+  for (const key of ["source", "releaseId", "bundleSha256", "channel"] as const) {
+    const field = boundedString(identity[key])
+    if (field) out[key] = field
+  }
+  return Object.keys(out).length > 0 ? out : undefined
+}
+
+export function buildMiniappDiagnosticContext(input: {
+  apps: ClientApp[]
+  foregroundedPackage: string | null
+  runtime: MiniappRuntimeDiagnosticSnapshot
+  getLastOpenedAtMs: (app: ClientApp) => number
+  getManifest: (app: ClientApp) => unknown
+  getReleaseIdentity: (app: ClientApp) => unknown
+}): Record<string, unknown> {
+  const runtimeByPackage = new Map(input.runtime.connectedApps.map((app) => [app.packageName, app]))
+  const appSnapshots = input.apps.map((app) => {
+    const lastOpenedAtMs = input.getLastOpenedAtMs(app)
+    const packageName = boundedString(app.packageName) ?? "unknown"
+    return {
+      packageName,
+      name: boundedString(app.name) ?? packageName,
+      version: boundedString(app.version) ?? null,
+      executionMode: executionMode(app),
+      running: app.running,
+      foregrounded: app.packageName === input.foregroundedPackage,
+      loading: app.loading,
+      healthy: app.healthy,
+      hidden: app.hidden,
+      type: app.type,
+      offline: app.offline,
+      local: app.local,
+      lastOpenedAtMs: Number.isFinite(lastOpenedAtMs) ? lastOpenedAtMs : 0,
+    }
+  })
+
+  const recentlyInteracted = [...appSnapshots]
+    .filter((app) => app.lastOpenedAtMs > 0)
+    .sort((a, b) => b.lastOpenedAtMs - a.lastOpenedAtMs)
+    .slice(0, MAX_RECENT_MINIAPPS)
+    .map((app) => ({packageName: app.packageName, lastOpenedAtMs: app.lastOpenedAtMs}))
+
+  const runningMiniapps = input.apps
+    .filter((app) => app.running)
+    .map((app) => {
+      const runtime = runtimeByPackage.get(app.packageName)
+      const releaseIdentity = safeReleaseIdentity(input.getReleaseIdentity(app))
+      const packageName = boundedString(app.packageName) ?? "unknown"
+      const manifest = buildMiniappManifestSnapshot(input.getManifest(app) ?? runtime?.manifest, {
+        packageName,
+        name: app.name,
+        version: app.version,
+        type: app.type,
+      })
+      return {
+        packageName,
+        version: boundedString(app.version) ?? manifest.version ?? null,
+        executionMode: executionMode(app),
+        foregrounded: app.packageName === input.foregroundedPackage,
+        manifest,
+        ...(releaseIdentity ? {releaseIdentity} : {}),
+        runtime: runtime
+          ? {
+              handshakeComplete: runtime.handshakeComplete,
+              subscriptions: runtime.subscriptions,
+              lastMessageAgeMs: runtime.lastMessageAgeMs,
+              requestedLocationRate: runtime.requestedLocationRate,
+              hasActiveSpeakerStream: runtime.hasActiveSpeakerStream,
+            }
+          : null,
+      }
+    })
+
+  return {
+    apps: appSnapshots,
+    installed: appSnapshots.map((app) => app.packageName),
+    running: runningMiniapps.map((app) => app.packageName),
+    foregroundedPackage: boundedString(input.foregroundedPackage) ?? null,
+    mostRecentlyInteractedPackage: recentlyInteracted[0]?.packageName ?? null,
+    recentlyInteracted,
+    runningMiniapps,
+  }
+}