@glassly/engine 0.1.0-dev.0

package/src/services/JSCrashController.ts

@@ -0,0 +1,189 @@
+/**
+ * JSCrashController — crash recovery state machine for the
+ * per-miniapp JSContexts owned by JSRouter.
+ *
+ *   RUNNING
+ *     │  crash detected (uncaught exception, native bridge throw,
+ *     │   ping miss, soft-watchdog kill, etc.)
+ *     ▼
+ *   CRASHED ──── respawn allowed? ──── no ──► CRASHLOOP_DISABLED
+ *     │                                          (persistent banner)
+ *     ▼ yes
+ *   BACKOFF(2s → 8s → 30s)
+ *     │  timer fires
+ *     ▼
+ *   RUNNING (and stays here on a clean 60s window;
+ *            otherwise crash count increments)
+ *
+ * Retry policy: zero retries by default — the first crash escalates
+ * straight to CRASHLOOP_DISABLED, matching how Android / iOS treat crashed
+ * apps. Callers that prefer auto-respawn opt in by passing
+ * `{maxRetries: 3, backoffMs: [2_000, 8_000, 30_000]}`; the BACKOFF state
+ * machine and exponential delays are implemented and unit-tested for that
+ * case.
+ *
+ * The controller is purely a policy engine — it doesn't actually call
+ * jsSpawn / jsKill. Callers (JSRouter) read the
+ * Outcome returned from onCrash() and drive the host accordingly. This
+ * makes the controller unit-testable with synthetic timer.
+ */
+
+export type CrashState =
+  | {kind: "RUNNING"; sinceMs: number}
+  | {kind: "CRASHED"; reason: string; crashedAtMs: number}
+  | {kind: "BACKOFF"; until: number; attempt: number; reason: string}
+  | {kind: "CRASHLOOP_DISABLED"; sinceMs: number; reason: string}
+
+export interface CrashOutcome {
+  /** Caller must call `respawn(packageName)` after this delay (ms from now). */
+  scheduleRespawnAfterMs?: number
+  /** Caller should surface "X restarted" toast (2nd-within-5m crash). */
+  showRestartToast?: boolean
+  /** Caller should surface persistent "crashloop disabled" banner. */
+  surfaceCrashloopBanner?: boolean
+  /** Caller may push the crash event to dev portal / Sentry as warn-level. */
+  sentryLevel?: "info" | "warning" | "error"
+}
+
+export interface CrashControllerOptions {
+  /** Override the wall clock for tests. Defaults to Date.now. */
+  now?: () => number
+  /** Backoff sequence in ms. Defaults to [2_000, 8_000, 30_000]. */
+  backoffMs?: number[]
+  /** Window in ms — N crashes inside this window escalates. Defaults to 5min. */
+  crashWindowMs?: number
+  /** Max retries before CRASHLOOP_DISABLED. Defaults to 0 — first crash
+   *  escalates immediately, matching Android / iOS app conventions. */
+  maxRetries?: number
+  /** Clean-uptime threshold that resets the retry counter. Defaults to 60s. */
+  cleanUptimeResetMs?: number
+}
+
+interface PackageState {
+  state: CrashState
+  /** Crash timestamps inside the current window (sorted ascending). */
+  crashesInWindow: number[]
+  /** Total respawns since the package was first registered (for telemetry). */
+  totalRespawns: number
+}
+
+const DEFAULT_BACKOFFS = [2_000, 8_000, 30_000]
+const DEFAULT_WINDOW_MS = 5 * 60_000
+const DEFAULT_MAX_RETRIES = 0
+const DEFAULT_CLEAN_UPTIME_RESET_MS = 60_000
+
+export class JSCrashController {
+  private readonly now: () => number
+  private readonly backoffMs: number[]
+  private readonly crashWindowMs: number
+  private readonly maxRetries: number
+  private readonly cleanUptimeResetMs: number
+  private readonly packages: Map<string, PackageState> = new Map()
+
+  constructor(opts: CrashControllerOptions = {}) {
+    this.now = opts.now ?? (() => Date.now())
+    this.backoffMs = opts.backoffMs ?? DEFAULT_BACKOFFS
+    this.crashWindowMs = opts.crashWindowMs ?? DEFAULT_WINDOW_MS
+    this.maxRetries = opts.maxRetries ?? DEFAULT_MAX_RETRIES
+    this.cleanUptimeResetMs = opts.cleanUptimeResetMs ?? DEFAULT_CLEAN_UPTIME_RESET_MS
+  }
+
+  /** Tracking starts when JSRouter spawns a fresh JSContext. */
+  onSpawn(packageName: string): void {
+    const at = this.now()
+    const prev = this.packages.get(packageName)
+    if (prev) {
+      // Carry forward the crash history; a respawn doesn't reset retry
+      // budget unless the previous RUNNING state survived >= cleanUptimeResetMs.
+      prev.state = {kind: "RUNNING", sinceMs: at}
+      prev.totalRespawns += 1
+    } else {
+      this.packages.set(packageName, {
+        state: {kind: "RUNNING", sinceMs: at},
+        crashesInWindow: [],
+        totalRespawns: 0,
+      })
+    }
+  }
+
+  /** Called when JSRouter explicitly tears down (user-initiated). */
+  onKill(packageName: string): void {
+    this.packages.delete(packageName)
+  }
+
+  /**
+   * Record a crash and return the policy outcome. Caller is responsible for
+   * scheduling the respawn (timer ownership belongs at the orchestration
+   * layer that holds the host context, not here).
+   */
+  onCrash(packageName: string, reason: string): CrashOutcome {
+    const at = this.now()
+    const entry = this.packages.get(packageName) ?? {
+      state: {kind: "RUNNING", sinceMs: at},
+      crashesInWindow: [],
+      totalRespawns: 0,
+    }
+
+    // If the previous RUNNING window was long enough, reset the retry
+    // counter — a crash after a stable session shouldn't burn budget.
+    if (entry.state.kind === "RUNNING") {
+      const uptime = at - entry.state.sinceMs
+      if (uptime >= this.cleanUptimeResetMs) {
+        entry.crashesInWindow = []
+      }
+    }
+
+    // Drop crashes that fell out of the window before counting the new one.
+    const cutoff = at - this.crashWindowMs
+    entry.crashesInWindow = entry.crashesInWindow.filter((t) => t >= cutoff)
+    entry.crashesInWindow.push(at)
+    this.packages.set(packageName, entry)
+
+    const crashCount = entry.crashesInWindow.length
+
+    if (crashCount > this.maxRetries) {
+      entry.state = {kind: "CRASHLOOP_DISABLED", sinceMs: at, reason}
+      return {
+        surfaceCrashloopBanner: true,
+        sentryLevel: "error",
+      }
+    }
+
+    const attempt = crashCount - 1
+    const delayMs = this.backoffMs[Math.min(attempt, this.backoffMs.length - 1)]!
+    entry.state = {kind: "BACKOFF", until: at + delayMs, attempt: crashCount, reason}
+
+    return {
+      scheduleRespawnAfterMs: delayMs,
+      showRestartToast: crashCount >= 2,
+      sentryLevel: crashCount === 1 ? "info" : "warning",
+    }
+  }
+
+  /** Inspect current state — useful for tests and diagnostics. */
+  stateFor(packageName: string): CrashState | null {
+    return this.packages.get(packageName)?.state ?? null
+  }
+
+  /** Total respawns since registration — surfaced to Sentry as a counter. */
+  respawnCount(packageName: string): number {
+    return this.packages.get(packageName)?.totalRespawns ?? 0
+  }
+
+  /** Currently-disabled packages. Host UI uses this to show banners. */
+  disabledPackages(): string[] {
+    const out: string[] = []
+    for (const [name, entry] of this.packages) {
+      if (entry.state.kind === "CRASHLOOP_DISABLED") out.push(name)
+    }
+    return out
+  }
+
+  /** Manually re-enable a disabled package (user "Try again" tap). */
+  reset(packageName: string): void {
+    const entry = this.packages.get(packageName)
+    if (!entry) return
+    entry.crashesInWindow = []
+    entry.state = {kind: "RUNNING", sinceMs: this.now()}
+  }
+}

package/src/services/JSCrashloopReportService.ts

@@ -0,0 +1,66 @@
+import {submitAutomaticReport} from "../facades/reports"
+import {useAppStatusStore} from "../stores/apps"
+import {
+  logAutomaticReportSubmissionStatus,
+  toAutomaticReportSubmissionStatus,
+  type AutomaticReportSubmissionStatus,
+} from "./AutomaticReportResult"
+import {getMiniappEngine} from "./MiniappEngine"
+import {islandNotifications, type IslandNotification} from "./NotificationsEmitter"
+
+const LOG_TAG = "JSCrashloopReport"
+
+let unsubscribe: (() => void) | null = null
+
+export async function submitJSCrashloopReport(params: {
+  packageName: string
+  reason: string
+  lastLogLines?: string[]
+}): Promise<AutomaticReportSubmissionStatus> {
+  const {packageName, reason, lastLogLines = []} = params
+  const app = useAppStatusStore.getState().apps.find((a) => a.packageName === packageName)
+  const appName = app?.name ?? packageName
+  const result = await submitAutomaticReport({
+    kind: "automatic",
+    trigger: {
+      type: "automatic",
+      source: "miniapp_crashloop",
+      reason: "glasslyjs_crashloop_disabled",
+      sourceAppletPackageName: packageName,
+      sourceAppletName: appName,
+    },
+    report: {
+      expectedBehavior: `${appName} should run without crashing.`,
+      actualBehavior: JSON.stringify({reason, lastLogLines}, null, 2),
+      systemPriority: "critical",
+    },
+    throttleKey: `glasslyjs_crashloop:${packageName}`,
+  })
+
+  const status = toAutomaticReportSubmissionStatus(result)
+  logAutomaticReportSubmissionStatus(LOG_TAG, status, packageName)
+  return status
+}
+
+function handleNotification(notification: IslandNotification): void {
+  if (notification.kind !== "miniapp_crashloop" || !notification.packageName) return
+  const packageName = notification.packageName
+  const lastLogLines = getMiniappEngine()?.router.logRing.snapshot(packageName) ?? []
+  void submitJSCrashloopReport({
+    packageName,
+    reason: notification.reason,
+    lastLogLines,
+  }).catch((error) => {
+    console.error(`[${LOG_TAG}] Unexpected error:`, error)
+  })
+}
+
+export function startJSCrashloopReportService(): void {
+  if (unsubscribe) return
+  unsubscribe = islandNotifications.subscribe(handleNotification)
+}
+
+export function stopJSCrashloopReportService(): void {
+  unsubscribe?.()
+  unsubscribe = null
+}

package/src/services/JSLogPipeline.ts

@@ -0,0 +1,166 @@
+/**
+ * JSLogPipeline — the three log-side utilities that
+ * JSRouter's `__log` handler runs every miniapp log line through:
+ *
+ *   1. redactSecrets — walks a value (string / array / object) and
+ *      replaces anything matching the secret-key regex
+ *      (token|password|secret|auth|bearer|key|api[_-]?key) with
+ *      "[REDACTED]". Conservative: over-redaction is preferable to
+ *      leaking a real secret into Sentry breadcrumbs.
+ *
+ *   2. JSLogThrottle — token-bucket rate limiter, 100 lines/min
+ *      sustained with a 500-line burst, per miniapp. Excess is dropped
+ *      and surfaced once as a `[throttled N]` summary. Stops a
+ *      misbehaving miniapp from drowning the host's log stream.
+ *
+ *   3. JSLogRingBuffer — fixed-size circular buffer (default 200)
+ *      holding the most recent log lines per miniapp. Dumped into the
+ *      crash report so post-mortem tooling sees the miniapp's last words.
+ *
+ * All three are pure (no React Native imports) and unit-tested in
+ * JSLogPipeline.test.ts.
+ */
+
+const SECRET_KEY_PATTERN = /\b(token|password|secret|auth|bearer|key|api[_-]?key)\b/i
+
+/**
+ * Redact obvious secret-like strings from a log line. Walks the JSON
+ * argument list, replacing any value that looks like a secret with
+ * `[REDACTED]`. Conservative — false negatives are fine, false positives
+ * are bad (devs lose visibility into their own data).
+ *
+ * Heuristics:
+ *   1. Top-level strings matching SECRET_KEY_PATTERN get the entire value
+ *      redacted (e.g. `console.log("token=abc123")` → `[REDACTED]`).
+ *   2. Object keys matching the pattern get their values replaced.
+ *   3. Arrays are walked element-by-element.
+ *
+ * Returns a new structure — never mutates the input.
+ */
+export function redactSecrets(value: unknown): unknown {
+  if (typeof value === "string") {
+    return SECRET_KEY_PATTERN.test(value) ? "[REDACTED]" : value
+  }
+  if (Array.isArray(value)) {
+    return value.map((v) => redactSecrets(v))
+  }
+  if (value && typeof value === "object") {
+    const out: Record<string, unknown> = {}
+    for (const [k, v] of Object.entries(value as Record<string, unknown>)) {
+      if (SECRET_KEY_PATTERN.test(k)) {
+        out[k] = "[REDACTED]"
+      } else {
+        out[k] = redactSecrets(v)
+      }
+    }
+    return out
+  }
+  return value
+}
+
+/**
+ * Token bucket throttler. Per-package: 100 logs/min sustained with a 500
+ * burst. Excess lines are dropped; the first drop in each refill window
+ * emits a `[throttled N]` synthetic line so the dev knows messages
+ * were lost.
+ */
+export interface ThrottleOptions {
+  /** Tokens added per second. Default 100/60 = ~1.67. */
+  tokensPerSecond?: number
+  /** Bucket capacity. Default 500. */
+  bucketCapacity?: number
+  /** Clock override for tests. */
+  now?: () => number
+}
+
+interface PackageBucket {
+  tokens: number
+  lastRefillAtMs: number
+  /** Drops accumulated since the last `[throttled N]` synthetic line. */
+  pendingDrops: number
+}
+
+export class JSLogThrottle {
+  private readonly tokensPerSecond: number
+  private readonly bucketCapacity: number
+  private readonly now: () => number
+  private readonly buckets: Map<string, PackageBucket> = new Map()
+
+  constructor(opts: ThrottleOptions = {}) {
+    this.tokensPerSecond = opts.tokensPerSecond ?? 100 / 60
+    this.bucketCapacity = opts.bucketCapacity ?? 500
+    this.now = opts.now ?? (() => Date.now())
+  }
+
+  /**
+   * Try to consume one token for the named package. Returns either:
+   *   - `{allowed: true}` — caller emits the original log line.
+   *   - `{allowed: false, throttledLine}` — caller may emit the
+   *     synthetic throttled line (the first drop in each window).
+   *     If `throttledLine` is undefined the caller drops silently
+   *     (subsequent drops in the same window).
+   */
+  consume(packageName: string): {allowed: true} | {allowed: false; throttledLine?: string} {
+    const at = this.now()
+    let bucket = this.buckets.get(packageName)
+    if (!bucket) {
+      bucket = {
+        tokens: this.bucketCapacity,
+        lastRefillAtMs: at,
+        pendingDrops: 0,
+      }
+      this.buckets.set(packageName, bucket)
+    } else {
+      const elapsedSec = (at - bucket.lastRefillAtMs) / 1000
+      bucket.tokens = Math.min(this.bucketCapacity, bucket.tokens + elapsedSec * this.tokensPerSecond)
+      bucket.lastRefillAtMs = at
+    }
+    if (bucket.tokens >= 1) {
+      bucket.tokens -= 1
+      if (bucket.pendingDrops > 0) {
+        const dropped = bucket.pendingDrops
+        bucket.pendingDrops = 0
+        return {allowed: false, throttledLine: `[throttled ${dropped}]`}
+      }
+      return {allowed: true}
+    }
+    bucket.pendingDrops += 1
+    return {allowed: false}
+  }
+
+  /** Test/dev — reset all buckets. */
+  resetForTests(): void {
+    this.buckets.clear()
+  }
+}
+
+/**
+ * In-memory ring buffer holding the last N lines per package. Useful as a
+ * "last words" context attached to crash reports.
+ */
+export class JSLogRingBuffer {
+  private readonly capacity: number
+  private readonly buffers: Map<string, string[]> = new Map()
+
+  constructor(capacityPerPackage = 200) {
+    this.capacity = capacityPerPackage
+  }
+
+  push(packageName: string, line: string): void {
+    let buf = this.buffers.get(packageName)
+    if (!buf) {
+      buf = []
+      this.buffers.set(packageName, buf)
+    }
+    buf.push(line)
+    if (buf.length > this.capacity) buf.shift()
+  }
+
+  snapshot(packageName: string): string[] {
+    return [...(this.buffers.get(packageName) ?? [])]
+  }
+
+  clear(packageName: string): void {
+    this.buffers.delete(packageName)
+  }
+}