@bobfrankston/msgapidefs 0.1.38 → 0.1.40

package/README.md

@@ -150,12 +150,17 @@ nothing about capability.
 | `tcp.close()` | ✅ | ✅ | ✅ | Close connection |
 | **HTTP Fetch** | | | | **`window.msgapi.http.*`** |
 | `http.fetch()` | ✅ | ✅ | ✅ | Native HTTP — bypasses CORS/mixed-content |
+| **Media capture (mic/camera)** | ❌ | ❌ | ❌ | Not an msgapi API — `getUserMedia` is the engine's. What's missing is the permission grant, which no host makes: wry's WebKitGTK backend has no `permission-request` handling (WebKitGTK denies by default), msger ignores WebView2's `PermissionRequested`, msga lacks `RECORD_AUDIO`. Deferred; direction is an external speech package, not in-page capture — see [msgapi-plan.md](./msgapi-plan.md#not-planned-recorded-so-it-doesnt-get-re-investigated). |
 | **Host Launcher Features** | | | | *Not part of `window.msgapi`; CLI / launcher-side* |
 | Markdown rendering (`.md`/`.markdown` auto-render) | ✅ | ✅ | ❌ | Local `file://` URL → rendered HTML via `msgcommon/markdown` (marked + light/dark CSS). msga could match it in `MainPage.xaml.cs` URL interception — not done yet. |
 | Unpacked Chrome extension loading (`-extension <dir>`) | ✅ | ❌ | ❌ | msgview-only via `session.extensions.loadExtension`. wry has no public extension API; WebKitGTK has none at all. **Not the add-in mechanism here** — app add-ins are `window.msgapi` + injected init scripts (`initScript` / `initScriptPath`), which work in msger and msga without Chromium. Chrome-extension-specific needs go to Chrome. |
 | Headless mode (`-noshow` without `-save`) | ✅ | ✅ | ❌ | msger creates the WebView invisible (`with_visible(false)`) but still loads HTML and runs scripts — an embedded `<script>` can query the engine and `window.ipc.postMessage` a result back. Combined with `-result <fieldname>` for clean stdout, this lets msger act as a headless CSS/JS resolver (`contrast-color()`, `getComputedStyle()`, `CSS.supports()`). msgview matches with `BrowserWindow({show:false})` + `backgroundThrottling:false`. |
 | Render to bitmap (`-render [file]`, `render` option) | ✅ | ✅ | ❌ | Headless page→image capture: the window is never shown; the page loads, settles (`renderDelay`, default 100ms after `load` + two rAFs), is screenshotted, and the host exits. `-render <file>` writes the image (format from extension: `.png`/`.jpg`/`.bmp`) and the JSON result carries `render: {path, width, height, format}`; bare `-render` (or API `render: true`) returns it as `render: {data (base64), width, height, format}`. Library API: `showMessageBox({render: "shot.png" \| true, renderDelay?, renderFormat?})`; a failed capture rejects (msger) or sets `renderError` in the result. Both hosts fall back to capturing whatever rendered if the page hasn't loaded within the timeout (default 30s in render mode). msger: WebView2 DevTools `Page.captureScreenshot` on Windows, `webkit_web_view_get_snapshot()` → cairo PNG on Linux/Pi (2026-08-01); msgview: `webContents.capturePage()`. Both msger paths produce the same base64 PNG, so format transcoding and file writing are shared. **Linux caveat:** WebKitGTK cannot snapshot a window that was never mapped, so on Linux the render window IS mapped — made invisible by going fully transparent and skipping the taskbar, since Wayland has no way to park a window off-screen. Width/height are physical pixels (include DPI scale) and cover the WebView viewport, not the window frame. |
 
+| Desktop overlay (`-overlay`, `overlay` option) | ❌ | ✅ | ❌ | msger-only (2026-08-09). Frameless + transparent window drawn over the desktop, click-through, always-on-top, kept out of the taskbar and never focused — for transient notices such as per-monitor identification. Needs all four of `with_decorations(false)`, `with_transparent(true)` on the tao window, `with_transparent(true)` on the wry WebView, and a page that declares its own transparent background (the built-in template does; `-url`/`-raw` pages must). Click-through is `set_ignore_cursor_events` (Windows/Mac/X11/Wayland; best-effort, warns and stays clickable if refused). No taskbar slot on Windows/Linux only — macOS has no per-window equivalent, Dock presence being a process-level activation policy. Pair with `timeout`: an overlay has no close button and swallows no clicks, so nothing in it can be operated. Each overlay defaults to its own WebView2 profile keyed by screen, because concurrent WebView2 creations against one user-data dir fail with `0x80070057` — which is what launching one overlay per display does. **Verified on Windows; Linux/Pi and Mac are built but not yet exercised.** Linux/X11 additionally needs a running compositor or transparency renders black. |
+| Monitor enumeration (`-list`, `listMonitors` option) | ❌ | ✅ | ❌ | msger-only (2026-08-09). Prints `{"monitors":[{index,name,x,y,width,height,scale,primary}]}` and exits without creating a window. Indices match `-screen`, ordered bottom-row-first then left-to-right (the ordering `-screen` has always used on Windows, preserved so existing indices keep pointing at the same display). Exists so a caller can discover how many displays there are, and what to label them, before spawning one overlay per display. |
+| Screen selection (`-screen <n>`, `screen` option) | ❌ | ✅ | ❌ | Offsets `-pos` into the named monitor **and** picks which display `-fullscreen` covers. Was Windows-only for `-pos`; the gate was wrong rather than protective, so Mac/X11 now honor it too (2026-08-09). `fullscreen` + `screen` is the only portable way to target a display: Wayland does not let a client position its own window at absolute coordinates, but it does honor fullscreen on a chosen output. |
+
 > **msger** now injects `window.msgapi` via `msger-api.js` with window control, UDP, and HTTP. File system and shell are not yet implemented in msger or msgview.
 >
 > **Markdown rendering** (last two rows): both msger and msgview detect `.md`/`.markdown` URLs pointing at local files and render them via the shared `renderMarkdown()` helper in `msgcommon/markdown` — no extensions, no per-machine setup, GFM with light/dark CSS that follows `prefers-color-scheme`. Library callers can `import { renderMarkdown, looksLikeMarkdown } from '@bobfrankston/msgcommon/markdown'` directly. **Extension loading** is msgview-only by design: see the row's note.

package/msgapi-plan.md

@@ -242,6 +242,53 @@ interface MsgAPI {
 
 ---
 
+## Not Planned (recorded so it doesn't get re-investigated)
+
+### Microphone / camera capture — deferred, direction is an external package
+
+**Status 2026-08-01: not in msgapi, and no host grants the permission.** Recorded
+after the wallclock app showed *"Initialization failed: Microphone constraints not
+supported"* under msger on a Pi 5.
+
+**Why it isn't an msgapi API.** The mic is reached with the plain web API
+`navigator.mediaDevices.getUserMedia` — the engine already provides it. msgapi
+exists for what a browser *can't* do. What's actually missing is a **permission
+decision**, and none of the hosts makes one:
+
+| host | today |
+|------|-------|
+| msger / WebKitGTK (Linux, Pi) | wry 0.54's WebKitGTK backend has no permission-request handling at all; WebKitGTK denies an unhandled `permission-request` by default |
+| msger / WebView2 (Windows) | msger never handles `PermissionRequested`; whatever WebView2 defaults to applies |
+| msga (Android) | manifest doesn't declare `RECORD_AUDIO` (the only hits are inside `Mono.Android.dll`); no `onPermissionRequest` handler |
+
+**Unresolved:** whether the Pi failure is the permission gate or the app's
+constraints — WebKitGTK is pickier than Chromium about `echoCancellation`,
+`sampleRate`, etc. One probe settles it: call `getUserMedia({audio: true})` with
+bare constraints via `initScript` and read the error name — `NotAllowedError`
+means permission, `OverconstrainedError` means constraints.
+
+**If it ever becomes a priority**, the work is:
+
+1. **msger (Rust)** — connect WebKitGTK's `permission-request` signal and WebView2's
+   `PermissionRequested` event, granting media for content the caller trusts.
+   Opt-in per the no-regression rule in CLAUDE.md (e.g. `permissions:
+   ["microphone","camera"]` / `-allow mic`), defaulting to today's behavior so no
+   existing caller changes.
+2. **msga** — `RECORD_AUDIO` in the manifest, the Android runtime permission
+   request, and a `WebChromeClient.onPermissionRequest` handler.
+3. **msgapidefs** — a matrix row for which host grants what. No new
+   `window.msgapi` call is needed.
+
+**Direction (Bob, 2026-08-01): don't build this speculatively.** The original
+motivation was speech processing, which never worked well enough in-browser to be
+worth it. The likely future is delegating to an **external speech package** —
+msger/msga hand off to a native or networked processor (reachable today through
+`msgapi.shell` / `msgapi.http` / `msgapi.udp`) rather than capturing audio in the
+page. In-page mic support is only worth building if a concrete use case shows up
+that genuinely needs the audio inside the WebView.
+
+---
+
 ## Implementation Priorities
 
 ### Phase 1: msga msgapi + UDP ✅ Done

package/msgapidefs.ts

@@ -90,6 +90,29 @@ export interface MsgVersionInfo {  // Structure of versions.json
 }
 
 // msgapi JavaScript API — available via window.msgapi inside msgview/msger/msga
+/** Why an update prompted, and whether the post-update relaunch can reach the foreground.
+ *  Android permits a silent self-update only when the updating app is the installer of record, so
+ *  a browser or adb sideload makes the NEXT update prompt (selfInstalled false). Separately, a
+ *  background process may not start an activity without canDrawOverlays, which is why a silent
+ *  update can leave the user on the home screen. */
+export interface MsgInstallInfo {
+    package: string;
+    installer: string;          // installing package name, or "none"
+    selfInstalled: boolean;     // true => next update can be silent
+    canDrawOverlays: boolean;   // true => post-update relaunch is allowed to foreground
+    canInstallPackages: boolean;// "Install unknown apps" for this app
+    sdkInt: number;
+}
+
+/** Handle returned by udp.listen(). send/broadcast leave from this listener's own socket, so the
+ *  device's reply arrives back at this listener rather than at whichever one registered first.
+ *  send/broadcast are optional: msga builds before 1.5.45 return a handle with only close(). */
+export interface UdpListenHandle {
+    close(): void;
+    send?(host: string, port: number, data: string | Uint8Array): Promise<void>;
+    broadcast?(port: number, data: string | Uint8Array): Promise<void>;
+}
+
 export interface MsgAPI {
 
     version?: string;  // Host app version (e.g. "1.3.0") — semver string, compare numerically not lexicographically
@@ -109,6 +132,8 @@ export interface MsgAPI {
 
     diag?: {
         getExitReasons(max?: number): Promise<MsgExitInfo[]>;  // Previous-run exit records, newest first (default max 5). msga Android 11+ only; [] elsewhere.
+        getInstallInfo?(): Promise<MsgInstallInfo>;   // Installer of record + permission state (msga 1.5.48+)
+        requestOverlay?(): Promise<'already' | 'opened' | 'n/a'>;  // Open the Display-over-other-apps settings page
     };
 
     // ── Window Control ──────────────────────────────────────────
@@ -142,7 +167,10 @@ export interface MsgAPI {
 
     udp?: {
         send(host: string, port: number, data: string | Uint8Array): Promise<void>;  // Send UDP packet to host:port
-        listen(port: number, callback: (data: Uint8Array, sender: { host: string; port: number }) => void): Promise<{ close: () => void }>;  // Listen on port (0=ephemeral), returns {close()}
+        // Listen on port (0=ephemeral). The returned handle's send/broadcast go out THIS listener's
+        // socket, so replies come back to this listener — prefer them over the bare send/broadcast
+        // above, which cannot tell which listener the caller meant and use the first one registered.
+        listen(port: number, callback: (data: Uint8Array, sender: { host: string; port: number }) => void): Promise<UdpListenHandle>;
         sendReceive(host: string, port: number, data: string | Uint8Array, timeoutMs?: number): Promise<Uint8Array>;  // Send and wait for response
         broadcast(port: number, data: string | Uint8Array): Promise<void>;  // Broadcast to local network
     };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bobfrankston/msgapidefs",
-  "version": "0.1.38",
+  "version": "0.1.40",
   "description": "TypeScript definitions for msgapi JavaScript API (msgview/msger)",
   "type": "module",
   "main": "./msgapidefs.js",