@swiftbrowser/web 0.1.0 → 0.2.0

package/src/translation.ts

@@ -0,0 +1,126 @@
+/**
+ * The Translation framework's page side (docs/ops-protocol.md, "Translation
+ * and the on-device language model"): `translate` host requests, answered
+ * with the browser's built-in `Translator` (and `LanguageDetector` for an
+ * unnamed source language) where the browser has them — Chrome 138 and later
+ * on desktop. Without them every pair is unsupported.
+ *
+ * - `availability` → `{ status: "installed" | "supported" | "unsupported" }`
+ *   (`available` / `downloadable` or `downloading` / `unavailable`).
+ * - `translate` → `{ source, texts }`, or rejects with `unsupported: …`,
+ *   `unidentified: …` (no source language could be detected) or
+ *   `download: …` (the pair's model needs a download the session may not ask
+ *   for). A translator is made once per pair and kept.
+ */
+
+import type { TranslateRequestParams } from './protocol';
+
+export type TranslationAvailability = 'available' | 'downloadable' | 'downloading' | 'unavailable';
+
+/** A translator for one pair (Chrome's `Translator` instance). */
+export interface TranslatorLike {
+  translate(text: string): Promise<string>;
+}
+
+/** The browser's translation facilities; tests substitute them. */
+export interface TranslationHost {
+  availability(source: string, target: string): Promise<TranslationAvailability>;
+  create(source: string, target: string): Promise<TranslatorLike>;
+  /** The most likely language of the text (BCP 47), or null. Absent without a detector. */
+  detect?(text: string): Promise<string | null>;
+}
+
+export type TranslateResult = {
+  status?: 'installed' | 'supported' | 'unsupported';
+  source?: string;
+  texts?: string[];
+};
+
+const STATUS: Record<TranslationAvailability, 'installed' | 'supported' | 'unsupported'> = {
+  available: 'installed',
+  downloadable: 'supported',
+  downloading: 'supported',
+  unavailable: 'unsupported',
+};
+
+/** Translators made so far, per `source>target`. */
+const translators = new Map<string, Promise<TranslatorLike>>();
+
+async function sourceLanguage(params: TranslateRequestParams, host: TranslationHost, text: string): Promise<string> {
+  if (params.source) return params.source;
+  const detected = host.detect && text.trim() ? await host.detect(text) : null;
+  if (!detected || detected === 'und') throw new Error('unidentified: the language of the text could not be detected');
+  return detected;
+}
+
+export async function performTranslate(
+  params: TranslateRequestParams,
+  host: TranslationHost | undefined,
+): Promise<TranslateResult> {
+  const target = params.target;
+  if (params.action === 'availability') {
+    if (!host) return { status: 'unsupported' };
+    const source = await sourceLanguage(params, host, params.text ?? '');
+    if (source === target) return { status: 'installed' };
+    return { status: STATUS[await host.availability(source, target)] ?? 'unsupported' };
+  }
+  if (params.action !== 'translate') {
+    throw new Error(`unsupported: translate action ${String((params as { action: unknown }).action)}`);
+  }
+  if (!host) throw new Error('unsupported: this browser has no built-in translator');
+  const texts = params.texts ?? [];
+  const source = await sourceLanguage(params, host, texts.join('\n'));
+  if (source === target) return { source, texts };
+  const key = `${source}>${target}`;
+  let translator = translators.get(key);
+  if (!translator) {
+    const availability = await host.availability(source, target);
+    if (availability === 'unavailable') throw new Error(`unsupported: no translator from ${source} to ${target}`);
+    if (availability !== 'available' && params.allowDownload !== true) {
+      throw new Error(`download: the ${source} to ${target} model is not installed`);
+    }
+    translator = host.create(source, target);
+    translators.set(key, translator);
+    translator.catch(() => translators.delete(key));
+  }
+  const ready = await translator;
+  const translated = await Promise.all(texts.map((text) => (text ? ready.translate(text) : Promise.resolve(''))));
+  return { source, texts: translated };
+}
+
+/** Forgets the translators made so far (tests). */
+export function resetTranslators(): void {
+  translators.clear();
+}
+
+interface TranslatorGlobal {
+  availability(options: { sourceLanguage: string; targetLanguage: string }): Promise<TranslationAvailability>;
+  create(options: { sourceLanguage: string; targetLanguage: string }): Promise<TranslatorLike>;
+}
+
+interface LanguageDetectorGlobal {
+  create(): Promise<{ detect(text: string): Promise<{ detectedLanguage: string; confidence: number }[]> }>;
+}
+
+/** Chrome's `Translator` and `LanguageDetector`, where the page has them. */
+export function browserTranslationHost(): TranslationHost | undefined {
+  const scope = globalThis as unknown as { Translator?: TranslatorGlobal; LanguageDetector?: LanguageDetectorGlobal };
+  const translator = scope.Translator;
+  if (!translator || typeof translator.create !== 'function') return undefined;
+  const detectorGlobal = scope.LanguageDetector;
+  let detector: ReturnType<LanguageDetectorGlobal['create']> | undefined;
+  return {
+    availability: (sourceLanguage, targetLanguage) => translator.availability({ sourceLanguage, targetLanguage }),
+    create: (sourceLanguage, targetLanguage) => translator.create({ sourceLanguage, targetLanguage }),
+    ...(detectorGlobal && typeof detectorGlobal.create === 'function'
+      ? {
+          detect: async (text: string) => {
+            detector ??= detectorGlobal.create();
+            const results = await (await detector).detect(text);
+            const best = results[0];
+            return best && best.confidence >= 0.5 ? best.detectedLanguage : null;
+          },
+        }
+      : {}),
+  };
+}

package/src/video.ts

@@ -0,0 +1,299 @@
+/**
+ * The `video` element (docs/ops-protocol.md, "Video and audio"): an HTML
+ * `<video>` inside the element's box, driven by the props AVFoundation works
+ * out from an `AVPlayer`, reporting back with `video` events.
+ *
+ * `applyVideo` carries out props: source, mute and volume (before any
+ * `play()`, so muted autoplay is allowed), rate, loop, native controls,
+ * gravity (CSS `object-fit`), each new `seek` serial once, then plays or
+ * pauses. It never talks to Swift itself: what needs reporting is a DOM event
+ * on the `<video>` (media events, or this module's `sb:video-*` events),
+ * which `VideoEvents`, listening on the renderer root in the capture phase
+ * (media events do not bubble), turns into `video` events.
+ *
+ * Browsers start unmuted playback only after the viewer has interacted with
+ * the page; a refused `play()` is reported as `pause` with `reason:
+ * "blocked"`, so the player's rate goes back to 0 and the native controls
+ * (or the app's own button, a tap) can start it.
+ */
+
+import type { ElementId, RendererEvent, VideoEvent, VideoProps } from './protocol';
+
+interface VideoState {
+  /** The props last applied. */
+  props: VideoProps | null;
+  /** The highest `seek` serial carried out (-1: none yet). */
+  seekSerial: number;
+  /** The serial of the seek in flight, reported with `seeked`. */
+  pendingSeek: number | null;
+  /** `performance.now()` of the last `time` event. */
+  lastTime: number;
+  /** The `requestAnimationFrame` loop reporting at `report`, if running. */
+  ticker: number | null;
+}
+
+const STATES = new WeakMap<HTMLVideoElement, VideoState>();
+/** `video` elements made so far; `stopVideos` has nothing to do while there are none. */
+let made = 0;
+
+/** At most this often (ms) a `timeupdate` becomes a `time` event when no `report` asks for more. */
+const TIME_EVENT_MIN_MS = 100;
+
+const SEEKED = 'sb:video-seeked';
+const BLOCKED = 'sb:video-blocked';
+const TICK = 'sb:video-tick';
+
+function objectFit(gravity: VideoProps['gravity']): string {
+  if (gravity === 'resizeAspectFill') return 'cover';
+  if (gravity === 'resize') return 'fill';
+  return 'contain';
+}
+
+/** The `<video>` of a `video` element, made on first use. */
+function mediaOf(el: HTMLElement): HTMLVideoElement {
+  const existing = el.firstElementChild;
+  if (existing instanceof HTMLVideoElement) return existing;
+  const video = document.createElement('video');
+  video.className = 'sb-video-media';
+  video.playsInline = true;
+  video.setAttribute('playsinline', '');
+  video.setAttribute('webkit-playsinline', '');
+  video.preload = 'metadata';
+  video.style.cssText = 'display:block;width:100%;height:100%;background:#000;object-fit:contain';
+  el.replaceChildren(video);
+  STATES.set(video, { props: null, seekSerial: -1, pendingSeek: null, lastTime: 0, ticker: null });
+  made += 1;
+  return video;
+}
+
+export function applyVideo(el: HTMLElement, props: VideoProps): void {
+  const video = mediaOf(el);
+  const state = STATES.get(video)!;
+  state.props = props;
+  // Mute first: a muted video may start without a tap.
+  const muted = props.muted === true;
+  if (video.muted !== muted) video.muted = muted;
+  video.defaultMuted = muted;
+  const volume = Math.min(1, Math.max(0, typeof props.volume === 'number' ? props.volume : 1));
+  if (video.volume !== volume) video.volume = volume;
+  const loop = props.loop === true;
+  if (video.loop !== loop) video.loop = loop;
+  const controls = props.controls === true;
+  if (video.controls !== controls) video.controls = controls;
+  const fit = objectFit(props.gravity);
+  if (video.style.objectFit !== fit) video.style.objectFit = fit;
+  el.dataset['sbGravity'] = props.gravity ?? 'resizeAspect';
+
+  const url = typeof props.url === 'string' ? props.url : '';
+  if ((video.getAttribute('src') ?? '') !== url) {
+    if (url) {
+      video.src = url;
+    } else {
+      video.removeAttribute('src');
+      video.load();
+    }
+  }
+  const rate = typeof props.rate === 'number' && props.rate > 0 ? props.rate : 1;
+  if (video.playbackRate !== rate) video.playbackRate = rate;
+  if (video.defaultPlaybackRate !== rate) video.defaultPlaybackRate = rate;
+
+  // Each seek serial once (the first props carry the player's position).
+  let seeked = false;
+  const seek = props.seek;
+  if (seek && typeof seek.serial === 'number' && seek.serial !== state.seekSerial) {
+    state.seekSerial = seek.serial;
+    const time = Math.max(0, typeof seek.time === 'number' && Number.isFinite(seek.time) ? seek.time : 0);
+    seeked = true;
+    if (video.readyState === 0 || (Math.abs(video.currentTime - time) < 0.001 && !video.ended)) {
+      // Nothing to wait for: before the metadata the time is where playback
+      // will start; at the time already, the browser fires no `seeked`.
+      if (video.currentTime !== time) video.currentTime = time;
+      state.pendingSeek = null;
+      // After the ops are applied: an event must not reach Swift mid-batch.
+      const serial = seek.serial;
+      queueMicrotask(() => video.dispatchEvent(new CustomEvent(SEEKED, { detail: { serial } })));
+    } else {
+      state.pendingSeek = seek.serial;
+      video.currentTime = time;
+    }
+  }
+
+  const playing = props.playing === true && url !== '';
+  video.preload = playing ? 'auto' : 'metadata';
+  if (playing) {
+    // An ended video plays again only after a seek (play() would restart it).
+    if (video.paused && (seeked || !video.ended)) startPlaying(video);
+  } else if (!video.paused) {
+    video.pause();
+  }
+  updateTicker(video, state);
+}
+
+function startPlaying(video: HTMLVideoElement): void {
+  let result: Promise<void> | undefined;
+  try {
+    result = video.play();
+  } catch {
+    return;
+  }
+  result?.catch((error: unknown) => {
+    // `AbortError`: a pause or a new source interrupted it, which is fine.
+    if (error instanceof Error && error.name === 'NotAllowedError') {
+      video.dispatchEvent(new CustomEvent(BLOCKED));
+    }
+  });
+}
+
+/** While playing with a `report` interval, `time` events at that pace (else at `timeupdate`'s). */
+function updateTicker(video: HTMLVideoElement, state: VideoState): void {
+  const interval = state.props?.report;
+  const wanted = typeof interval === 'number' && interval > 0 && state.props?.playing === true;
+  if (!wanted) {
+    if (state.ticker !== null) cancelAnimationFrame(state.ticker);
+    state.ticker = null;
+    return;
+  }
+  if (state.ticker !== null || typeof requestAnimationFrame !== 'function') return;
+  const frame = (): void => {
+    state.ticker = null;
+    const report = state.props?.report;
+    if (!video.isConnected || state.props?.playing !== true || typeof report !== 'number' || report <= 0) return;
+    if (!video.paused && performance.now() - state.lastTime >= report * 1000) video.dispatchEvent(new CustomEvent(TICK));
+    state.ticker = requestAnimationFrame(frame);
+  };
+  state.ticker = requestAnimationFrame(frame);
+}
+
+/**
+ * Stops the videos in a removed subtree (`el` included): a detached
+ * `<video>` would go on playing its sound. Their sources are dropped, which
+ * also ends their downloads.
+ */
+export function stopVideos(el: HTMLElement): void {
+  if (made === 0) return;
+  const videos = el instanceof HTMLVideoElement ? [el] : Array.from(el.querySelectorAll('video'));
+  for (const video of videos) {
+    const state = STATES.get(video);
+    if (!state) continue;
+    if (state.ticker !== null) cancelAnimationFrame(state.ticker);
+    state.ticker = null;
+    state.props = null;
+    video.pause();
+    video.removeAttribute('src');
+    video.load();
+  }
+}
+
+/** Turns media events of the root's `video` elements into `video` events. */
+export class VideoEvents {
+  private static readonly TYPES = [
+    'loadedmetadata',
+    'timeupdate',
+    'playing',
+    'waiting',
+    'pause',
+    'play',
+    'ended',
+    'error',
+    'volumechange',
+    'ratechange',
+    'seeked',
+    SEEKED,
+    BLOCKED,
+    TICK,
+  ];
+
+  constructor(
+    private readonly root: HTMLElement,
+    private readonly send: (event: RendererEvent) => void,
+  ) {
+    for (const type of VideoEvents.TYPES) root.addEventListener(type, this.onEvent, true);
+  }
+
+  dispose(): void {
+    for (const type of VideoEvents.TYPES) this.root.removeEventListener(type, this.onEvent, true);
+  }
+
+  private readonly onEvent = (event: Event): void => {
+    const video = event.target;
+    if (!(video instanceof HTMLVideoElement)) return;
+    const state = STATES.get(video);
+    const host = video.parentElement;
+    const id = Number(host?.dataset['sbId']);
+    if (!state || !state.props || !Number.isInteger(id)) return;
+    const message = this.message(event, video, state);
+    if (message) this.send({ type: 'video', id: id as ElementId, ...message } as VideoEvent);
+  };
+
+  private message(event: Event, video: HTMLVideoElement, state: VideoState): Omit<VideoEvent, 'type' | 'id'> | null {
+    const props = state.props!;
+    const time = round(video.currentTime);
+    const url = video.getAttribute('src') ?? '';
+    switch (event.type) {
+      case 'loadedmetadata':
+        return {
+          event: 'ready',
+          url,
+          time,
+          duration: Number.isFinite(video.duration) ? round(video.duration) : null,
+          width: video.videoWidth,
+          height: video.videoHeight,
+        };
+      case 'timeupdate':
+        if (typeof props.report === 'number' && props.report > 0 && !video.paused) return null;
+        if (!video.paused && performance.now() - state.lastTime < TIME_EVENT_MIN_MS) return null;
+        state.lastTime = performance.now();
+        return { event: 'time', time };
+      case TICK:
+        state.lastTime = performance.now();
+        return { event: 'time', time };
+      case 'playing':
+        return { event: 'status', status: 'playing', time };
+      case 'waiting':
+        return { event: 'status', status: 'waiting', time };
+      case 'pause':
+        // The end pauses too; `ended` follows.
+        if (video.ended) return null;
+        // Paused while the player wants it playing: the viewer did it.
+        if (props.playing) return { event: 'pause', time };
+        return { event: 'status', status: 'paused', time };
+      case 'play':
+        // Started while the player wants it paused: the viewer did it.
+        return props.playing ? null : { event: 'play', time };
+      case BLOCKED:
+        return props.playing ? { event: 'pause', reason: 'blocked', time } : null;
+      case 'ended':
+        return { event: 'ended', url, time };
+      case 'error': {
+        const error = video.error;
+        if (!url || !error) return null;
+        return { event: 'error', url, code: error.code, message: error.message ?? '' };
+      }
+      case 'volumechange': {
+        const volume = Math.min(1, Math.max(0, props.volume ?? 1));
+        if (video.muted === props.muted && Math.abs(video.volume - volume) < 0.001) return null;
+        return { event: 'volume', muted: video.muted, volume: round(video.volume) };
+      }
+      case 'ratechange':
+        if (video.playbackRate === (props.rate > 0 ? props.rate : 1)) return null;
+        return { event: 'rate', rate: video.playbackRate };
+      case 'seeked': {
+        const serial = state.pendingSeek;
+        state.pendingSeek = null;
+        // Without a seek in flight the viewer scrubbed: report where to.
+        return serial === null ? { event: 'time', time } : { event: 'seeked', serial, time };
+      }
+      case SEEKED: {
+        const serial = (event as CustomEvent<{ serial: number }>).detail?.serial;
+        return typeof serial === 'number' ? { event: 'seeked', serial, time } : null;
+      }
+      default:
+        return null;
+    }
+  }
+}
+
+/** Seconds to the millisecond, so events stay short and stable. */
+function round(value: number): number {
+  return Math.round(value * 1000) / 1000;
+}

package/src/vite-env.d.ts

@@ -14,6 +14,12 @@ interface SBLandingApp {
   icon?: string;
   /** Languages the module carries locale data for; the page runs it in the viewer's when among them. */
   locales?: string[];
+  /** The app's URL schemes (CFBundleURLTypes), whose OAuth redirects the page receives (`SB_URL_SCHEMES`). */
+  urlSchemes?: string[];
+  /** A CORS proxy (packages/cors-proxy) through which a refused cross-origin GET is retried (src/corsProxy.ts). */
+  corsProxy?: string;
+  /** Hosts the app's requests may not reach (src/blockedHosts.ts). */
+  blockedHosts?: string[];
 }
 
 /** The registered apps, in registry order. */