@swiftbrowser/web 0.17.5 → 0.17.7

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@swiftbrowser/web",
-  "version": "0.17.5",
+  "version": "0.17.7",
   "type": "module",
   "description": "SwiftBrowser web renderer and dev server: applies render ops from a Swift/Wasm app to the DOM inside an iPhone frame.",
   "repository": {

package/src/hostRequests.ts

@@ -32,6 +32,7 @@ import type {
   OpenURLRequestParams,
   WebAuthRequestParams,
   MediaRequestParams,
+  MediaSessionRequestParams,
   DownloadRequestParams,
   ClipboardRequestParams,
   TranslateRequestParams,
@@ -40,6 +41,7 @@ import type {
 } from './protocol';
 import { browserClipboardHost, performClipboard, type ClipboardHost } from './clipboard';
 import { browserMediaHost, performMedia, type MediaHost } from './media';
+import { browserMediaSessionHost, MediaSessionBridge, performMediaSession } from './mediaSession';
 import { browserScannerHost, performScanner, type ScannerHost, type ScannerResult } from './scanner';
 import { browserTranslationHost, performTranslate, type TranslationHost } from './translation';
 import { chooseColor, performPickColor, type ColorChooser } from './colorPicker';
@@ -123,6 +125,8 @@ export interface HostRequestEnvironment {
   webAuth?: WebAuthSessions;
   /** AVFoundation's media files and probes, QuickLook's previews (`media`, media.ts). */
   media?: MediaHost;
+  /** MediaPlayer's now-playing info and remote commands (src/mediaSession.ts); absent without the Media Session API. */
+  mediaSession?: MediaSessionBridge;
   /** The system clipboard (`clipboard`, clipboard.ts). */
   clipboard?: ClipboardHost;
   /** The browser's built-in translator (`translate`, translation.ts), where it has one. */
@@ -177,6 +181,7 @@ const notificationHost = browserNotificationHost();
 const scannerHost = browserScannerHost();
 const webAuthHost = browserWebAuthHost();
 const mediaHost = browserMediaHost(typeof document !== 'undefined' ? downloadBlob : undefined);
+const mediaSessionHost = browserMediaSessionHost();
 const clipboardHost = browserClipboardHost();
 const translationHost = browserTranslationHost();
 
@@ -209,6 +214,7 @@ export const browserEnvironment: HostRequestEnvironment = {
     : {}),
   ...(webAuthHost ? { webAuth: new WebAuthSessions(webAuthHost) } : {}),
   ...(mediaHost ? { media: mediaHost } : {}),
+  ...(mediaSessionHost ? { mediaSession: new MediaSessionBridge(mediaSessionHost) } : {}),
   ...(clipboardHost ? { clipboard: clipboardHost } : {}),
   ...(translationHost ? { translation: translationHost } : {}),
   ...(typeof document !== 'undefined' ? { pickColor: chooseColor } : {}),
@@ -256,6 +262,8 @@ export async function performHostRequest(
     case 'media':
       // `open` is called before any await, while the tap's activation may still count.
       return performMedia(op.params as MediaRequestParams, env.media);
+    case 'mediaSession':
+      return performMediaSession(op.params as MediaSessionRequestParams, env.mediaSession);
     case 'download':
       return performDownload(op.params as DownloadRequestParams, env);
     case 'clipboard': {

package/src/main.ts

@@ -19,6 +19,7 @@ import { Renderer, RendererError } from './renderer';
 import { withBlockedHosts } from './blockedHosts';
 import { withCORSProxy } from './corsProxy';
 import { browserEnvironment } from './hostRequests';
+import { locateBundleFiles } from './media';
 import { callbackPageURL } from './webAuth';
 import { bitmapStore } from './bitmaps';
 import { DEFAULT_TYPE_SIZE, applyTypeSizeVariables } from './typography';
@@ -460,6 +461,11 @@ function saveHome(home: ReturnType<typeof homeDirectory>): void {
   files?.apply(changes).catch((error: unknown) => console.warn('[sb] could not save the app\'s files to IndexedDB', error));
 }
 
+/** Where the page serves the resource at `path` (inside the bundle) of `app`. */
+function bundleFileURL(app: string, path: string): string {
+  return `${import.meta.env.BASE_URL}bundle/${app}/${path.split('/').map(encodeURIComponent).join('/')}`;
+}
+
 /** The app's resources (`/bundle/<App>.json` and its files), for the WASI `/bundle` mount; empty without a manifest. */
 async function loadBundle(app: string): Promise<Map<string, Uint8Array>> {
   const files = new Map<string, Uint8Array>();
@@ -470,8 +476,7 @@ async function loadBundle(app: string): Promise<Map<string, Uint8Array>> {
     if (!Array.isArray(manifest.files)) return files;
     await Promise.all(
       manifest.files.map(async (file) => {
-        const url = `${import.meta.env.BASE_URL}bundle/${app}/${file.path.split('/').map(encodeURIComponent).join('/')}`;
-        const data = await fetch(url, { cache: 'no-cache' });
+        const data = await fetch(bundleFileURL(app, file.path), { cache: 'no-cache' });
         if (data.ok) files.set(file.path, new Uint8Array(await data.arrayBuffer()));
         else console.warn(`[sb] resource ${file.path} of ${app}: HTTP ${data.status}`);
       }),
@@ -601,6 +606,8 @@ async function boot(): Promise<void> {
     console.info(`[sb] restoring state for ${requestedApp}: ${restored}`);
   }
   const bundle = await loadBundle(requestedApp);
+  // A player given a bundle file streams it from here rather than through a copy of its bytes (src/media.ts).
+  locateBundleFiles((path) => (bundle.has(path) ? bundleFileURL(requestedApp, path) : null));
   const stores = launchValue(await loadSwiftData());
   if (stores !== null) env.push(`SB_SWIFTDATA=${stores}`);
   savedHome = await loadHome();

package/src/media.ts

@@ -3,7 +3,11 @@
  * AVFoundation's files and metadata probes, and QuickLook's previews.
  *
  * - `file` (`name`, `type`, `data` base64): a blob of the bytes; answers
- *   `{ url }`, an object URL a `video` element can play.
+ *   `{ url }`, an object URL a `video` element can play. With `path` in
+ *   place of `data` (a resource of the app's bundle), the URL the page
+ *   serves that resource at, so the browser streams it and nothing is
+ *   copied; fails `unsupported: …` when the page has no such resource,
+ *   and the app sends the bytes instead.
  * - `probe` (`url`): reads the media's metadata in a detached `<video>`;
  *   answers `{ duration, width, height }` (`duration` null for a stream),
  *   fails (`network: …`) when the browser cannot load or decode it.
@@ -37,12 +41,29 @@ export interface MediaHost {
   /** Opens `url` in a new tab; false when the browser blocked it. */
   open(url: string): boolean;
   download?(blob: Blob, filename: string): void;
+  /** The URL the page serves the bundle resource at `path` from, or null for a resource it did not load. */
+  bundleURL?(path: string): string | null;
+}
+
+let bundleLocator: ((path: string) => string | null) | null = null;
+
+/**
+ * Tells the media host where the app's bundle resources are served from
+ * (src/main.ts, once the bundle is loaded), for `file` requests by `path`.
+ */
+export function locateBundleFiles(locator: ((path: string) => string | null) | null): void {
+  bundleLocator = locator;
 }
 
 export async function performMedia(params: MediaRequestParams, host: MediaHost | undefined): Promise<Record<string, unknown>> {
   if (!host) throw new Error('unsupported: this page has no media host');
   switch (params.action) {
     case 'file': {
+      if (params.path !== undefined) {
+        const url = host.bundleURL?.(params.path) ?? null;
+        if (url === null) throw new Error(`unsupported: the page has no bundle resource ${params.path}`);
+        return { url };
+      }
       const blob = new Blob([decode(params.data ?? '') as BlobPart], { type: params.type ?? '' });
       return { url: host.objectURL(blob) };
     }
@@ -128,5 +149,6 @@ export function browserMediaHost(download?: (blob: Blob, filename: string) => vo
       return true;
     },
     ...(download ? { download } : {}),
+    bundleURL: (path) => bundleLocator?.(path) ?? null,
   };
 }

package/src/mediaSession.ts

@@ -0,0 +1,163 @@
+/**
+ * `mediaSession` host requests (docs/ops-protocol.md, "Now playing"):
+ * MediaPlayer's now-playing info and remote commands over the browser's
+ * Media Session API, which puts what plays on the lock screen and the
+ * system's media controls and routes their buttons (and hardware media
+ * keys) back to the page.
+ *
+ * - `update` (`metadata`, `playbackState`, `position`, `commands`, `skip`)
+ *   sets what the session shows and which commands the app handles; the
+ *   browser extrapolates the position at the rate, so the app sends it
+ *   only when it changes (a play, a pause, a seek). Answers `{}`.
+ * - `receive` answers the next command the viewer gave: `{ command }`, with
+ *   `position` for `seekTo` (seconds) and `interval` for the skips (the
+ *   offset the system asked for, else the app's preferred one). Commands
+ *   that arrive with nobody receiving wait in a queue, as WebSocket
+ *   messages do (src/webSocket.ts). Fails `unsupported: …` on a page with
+ *   no Media Session API, which ends the app's listening.
+ */
+
+import type { MediaSessionCommand, MediaSessionRequestParams } from './protocol';
+
+/** The part of `navigator.mediaSession` the requests use; tests substitute a fake. */
+export interface MediaSessionHost {
+  setMetadata(metadata: { title: string; artist: string; album: string; artwork: { src: string }[] } | null): void;
+  setPlaybackState(state: 'none' | 'playing' | 'paused'): void;
+  setPositionState(state: { duration: number; playbackRate: number; position: number } | null): void;
+  /** Registers (or, with null, removes) the handler of a Media Session action; may throw for an action the browser lacks. */
+  setActionHandler(action: MediaSessionAction, handler: ((details: MediaSessionActionDetails) => void) | null): void;
+}
+
+/** A command the viewer gave, as `receive` answers it. */
+export interface ReceivedCommand {
+  command: MediaSessionCommand;
+  position?: number;
+  interval?: number;
+}
+
+/** The Media Session action each command listens to. */
+const ACTIONS: Record<MediaSessionCommand, MediaSessionAction> = {
+  play: 'play',
+  pause: 'pause',
+  stop: 'stop',
+  nextTrack: 'nexttrack',
+  previousTrack: 'previoustrack',
+  seekTo: 'seekto',
+  skipForward: 'seekforward',
+  skipBackward: 'seekbackward',
+};
+
+/** One page's media session: what it shows, the commands it listens to, and the ones received. */
+export class MediaSessionBridge {
+  private readonly registered = new Set<MediaSessionCommand>();
+  private readonly queue: ReceivedCommand[] = [];
+  private readonly waiting: ((command: ReceivedCommand) => void)[] = [];
+  private skip: { forward?: number; backward?: number } = {};
+
+  constructor(private readonly host: MediaSessionHost) {}
+
+  perform(params: MediaSessionRequestParams): Promise<Record<string, unknown>> {
+    switch (params.action) {
+      case 'update':
+        this.update(params);
+        return Promise.resolve({});
+      case 'receive':
+        return this.receive();
+      default:
+        return Promise.reject(new Error(`unsupported: mediaSession action ${String((params as { action: unknown }).action)}`));
+    }
+  }
+
+  private update(params: MediaSessionRequestParams): void {
+    if (params.metadata !== undefined) {
+      const metadata = params.metadata;
+      this.host.setMetadata(
+        metadata === null
+          ? null
+          : { title: metadata.title ?? '', artist: metadata.artist ?? '', album: metadata.album ?? '', artwork: metadata.artwork ? [{ src: metadata.artwork }] : [] },
+      );
+    }
+    if (params.playbackState !== undefined) this.host.setPlaybackState(params.playbackState);
+    if (params.position !== undefined) {
+      const position = params.position;
+      this.host.setPositionState(position === null ? null : { duration: position.duration, playbackRate: position.rate, position: position.position });
+    }
+    if (params.skip !== undefined) this.skip = params.skip;
+    if (params.commands !== undefined) this.listen(new Set(params.commands));
+  }
+
+  /** Listens to exactly `wanted`: handlers for the new ones, none for the dropped ones. */
+  private listen(wanted: Set<MediaSessionCommand>): void {
+    for (const command of this.registered) {
+      if (wanted.has(command)) continue;
+      this.setHandler(command, null);
+      this.registered.delete(command);
+    }
+    for (const command of wanted) {
+      if (this.registered.has(command)) continue;
+      this.setHandler(command, (details) => this.deliver(this.received(command, details)));
+      this.registered.add(command);
+    }
+  }
+
+  private setHandler(command: MediaSessionCommand, handler: ((details: MediaSessionActionDetails) => void) | null): void {
+    try {
+      this.host.setActionHandler(ACTIONS[command], handler);
+    } catch {
+      // An action this browser does not know (`seekto` on an older Safari): the button never shows.
+    }
+  }
+
+  private received(command: MediaSessionCommand, details: MediaSessionActionDetails): ReceivedCommand {
+    const received: ReceivedCommand = { command };
+    if (command === 'seekTo' && typeof details.seekTime === 'number') received.position = details.seekTime;
+    if (command === 'skipForward' || command === 'skipBackward') {
+      const preferred = command === 'skipForward' ? this.skip.forward : this.skip.backward;
+      const interval = typeof details.seekOffset === 'number' ? details.seekOffset : preferred;
+      if (interval !== undefined) received.interval = interval;
+    }
+    return received;
+  }
+
+  private deliver(command: ReceivedCommand): void {
+    const wake = this.waiting.shift();
+    if (wake) wake(command);
+    else this.queue.push(command);
+  }
+
+  private receive(): Promise<Record<string, unknown>> {
+    const queued = this.queue.shift();
+    if (queued) return Promise.resolve({ ...queued });
+    return new Promise((resolve) => this.waiting.push((command) => resolve({ ...command })));
+  }
+}
+
+export async function performMediaSession(params: MediaSessionRequestParams, bridge: MediaSessionBridge | undefined): Promise<Record<string, unknown>> {
+  if (!bridge) {
+    // No Media Session API: nothing to show, and no command will ever come.
+    if (params.action === 'update') return {};
+    throw new Error('unsupported: this browser has no Media Session');
+  }
+  return bridge.perform(params);
+}
+
+/** `navigator.mediaSession`, where the browser has it. */
+export function browserMediaSessionHost(): MediaSessionHost | undefined {
+  if (typeof navigator === 'undefined' || !('mediaSession' in navigator) || typeof MediaMetadata === 'undefined') return undefined;
+  const session = navigator.mediaSession;
+  return {
+    setMetadata: (metadata) => {
+      session.metadata = metadata === null ? null : new MediaMetadata(metadata);
+    },
+    setPlaybackState: (state) => {
+      session.playbackState = state;
+    },
+    setPositionState: (state) => {
+      if (typeof session.setPositionState !== 'function') return;
+      if (state === null) session.setPositionState();
+      // A position past the duration, or a duration that is not a finite number, throws.
+      else if (Number.isFinite(state.duration) && state.duration >= 0) session.setPositionState({ ...state, position: Math.min(Math.max(state.position, 0), state.duration) });
+    },
+    setActionHandler: (action, handler) => session.setActionHandler(action, handler),
+  };
+}

package/src/protocol.ts

@@ -1076,7 +1076,7 @@ export interface DefaultsOp {
 export interface RequestOp {
   op: 'request';
   id: number;
-  kind: 'fetch' | 'image' | 'share' | 'pickFiles' | 'notification' | 'scanner' | 'websocket' | 'openURL' | 'webAuth' | 'media' | 'download' | 'clipboard' | 'translate' | 'pickColor';
+  kind: 'fetch' | 'image' | 'share' | 'pickFiles' | 'notification' | 'scanner' | 'websocket' | 'openURL' | 'webAuth' | 'media' | 'download' | 'clipboard' | 'translate' | 'pickColor' | 'mediaSession';
   params:
     | FetchRequestParams
     | ImageRequestParams
@@ -1091,7 +1091,8 @@ export interface RequestOp {
     | DownloadRequestParams
     | ClipboardRequestParams
     | TranslateRequestParams
-    | PickColorRequestParams;
+    | PickColorRequestParams
+    | MediaSessionRequestParams;
 }
 /**
  * `translate` (Translation; src/translation.ts): `availability` of a pair
@@ -2127,6 +2128,8 @@ export interface SlideEvent {
   type: 'slide';
   id: ElementId;
   value: number;
+  /** `onEditingChanged`: true with the first value of a drag, false with its last; absent for the values between. */
+  editing?: boolean;
 }
 /** A `datepicker` changed: seconds since 1970 (an instant; the zone only shaped the reading). */
 export interface DateEvent {
@@ -2556,10 +2559,34 @@ export interface VideoEvent {
 }
 
 /** `media` requests (src/media.ts): AVFoundation's files and probes, QuickLook's previews. */
+/** A remote command MediaPlayer's `MPRemoteCommandCenter` handles (docs/ops-protocol.md, "Now playing"). */
+export type MediaSessionCommand = 'play' | 'pause' | 'stop' | 'nextTrack' | 'previousTrack' | 'seekTo' | 'skipForward' | 'skipBackward';
+
+/**
+ * `mediaSession` (src/mediaSession.ts): MediaPlayer's now-playing info and
+ * remote commands over the browser's Media Session API. `update` sets what
+ * the session shows and which commands the app handles (a field left out
+ * stays as it was); `receive` answers the next command the viewer gave.
+ */
+export interface MediaSessionRequestParams {
+  action: 'update' | 'receive';
+  /** `update`: what plays, for the lock screen and the media controls (`artwork` a URL); null clears it. */
+  metadata?: { title?: string; artist?: string; album?: string; artwork?: string } | null;
+  playbackState?: 'none' | 'playing' | 'paused';
+  /** `update`: the position in seconds, which the browser extrapolates at `rate`; null clears it. */
+  position?: { duration: number; position: number; rate: number } | null;
+  /** `update`: the commands the app handles; the others get no button. */
+  commands?: MediaSessionCommand[];
+  /** `update`: the app's preferred skip intervals in seconds, answered with a skip the system gives no offset for. */
+  skip?: { forward?: number; backward?: number };
+}
+
 export interface MediaRequestParams {
   action: 'file' | 'probe' | 'release' | 'open';
   url?: string;
   name?: string;
   type?: string;
   data?: string;
+  /** `file`: a resource's path inside the app's bundle, in place of `data`: the page answers the URL it serves the resource at. */
+  path?: string;
 }

package/src/renderer.ts

@@ -786,7 +786,7 @@ export class Renderer {
     const id = Number(slider.dataset[ID_ATTR]);
     if (!track || !Number.isInteger(id)) return;
     event.preventDefault();
-    const drag: SliderDrag = { slider, track, id, pointerId: event.pointerId, pending: null, sent: null, frame: null };
+    const drag: SliderDrag = { slider, track, id, pointerId: event.pointerId, pending: null, sent: null, frame: null, starting: true };
     this.sliderDrag = drag;
     window.addEventListener('pointermove', this.onSliderMove);
     window.addEventListener('pointerup', this.onSliderUp);
@@ -817,9 +817,11 @@ export class Renderer {
   }
 
   private flushSlide(drag: SliderDrag): void {
-    if (drag.pending === null || drag.pending === drag.sent) return;
+    if (drag.pending === null || (drag.pending === drag.sent && !drag.starting)) return;
     drag.sent = drag.pending;
-    this.onEvent({ type: 'slide', id: drag.id, value: drag.pending });
+    // The drag's first value says editing began (`onEditingChanged(true)`); the last, on release, that it ended.
+    this.onEvent(drag.starting ? { type: 'slide', id: drag.id, value: drag.pending, editing: true } : { type: 'slide', id: drag.id, value: drag.pending });
+    drag.starting = false;
   }
 
   private readonly onSliderMove = (event: PointerEvent): void => {
@@ -838,6 +840,7 @@ export class Renderer {
     if (drag.frame !== null) window.cancelAnimationFrame(drag.frame);
     drag.frame = null;
     this.flushSlide(drag);
+    if (drag.sent !== null) this.onEvent({ type: 'slide', id: drag.id, value: drag.sent, editing: false });
     // The click that follows is the slide itself, not a tap on an enclosing control.
     this.swallowNextClick();
   };
@@ -3351,6 +3354,8 @@ interface SliderDrag {
   /** The last value sent, so an unchanged frame sends nothing. */
   sent: number | null;
   frame: number | null;
+  /** Whether the drag's first `slide` (with `editing: true`) is still to be sent. */
+  starting: boolean;
 }
 
 function applyAlert(el: HTMLElement, props: AlertProps): void {