@swiftbrowser/web 0.17.7 → 0.18.0

package/src/audioRecorder.ts

@@ -0,0 +1,392 @@
+/**
+ * Recording from the microphone, as AVFoundation's `AVAudioRecorder` does it
+ * for an iOS app: the page side of `useAudioRecorder` (@swiftbrowser/react).
+ * The browser's `getUserMedia` and `MediaRecorder` record; an `AnalyserNode`
+ * meters the input as `averagePower(forChannel:)` would. A recording ends as
+ * a file in the app's `Documents/Recordings` (files.ts: the app's IndexedDB
+ * home), a `file:` URL that plays in a `video` element and that `discard`
+ * deletes, as the native app's does; a page that keeps no files answers a
+ * `data:` URL instead, which goes with its string. `trim` cuts a recording
+ * with the Web Audio API (decoded, the kept parts copied, written as WAV, to
+ * a new file). The native app answers the same calls with `AVAudioRecorder`
+ * and `AVAssetExportSession` (packages/native, AudioRecorderBridge.swift).
+ */
+
+import { fileURL, type FilesHost } from './files';
+
+/** A finished recording: a URL that plays it (a `file:` URL in `Documents/Recordings`, or a `data:` URL where the page keeps no files) and its length. */
+export interface AudioRecording {
+  url: string;
+  /** Seconds. */
+  duration: number;
+  /** The MIME type it was recorded in. */
+  type: string;
+}
+
+/** The input now: its level (0 silent … 1 loud) and the seconds recorded so far. */
+export interface AudioMeter {
+  level: number;
+  time: number;
+}
+
+/** A stretch of a recording, in seconds from its start. */
+export interface AudioRange {
+  start: number;
+  end: number;
+}
+
+/**
+ * One recording at a time. `start` asks for the microphone (rejecting with
+ * `permission: …` when it is refused); `stop` ends it and resolves with the
+ * recording; `cancel` ends it and keeps nothing; `discard` deletes one made
+ * before; `trim` makes a new recording out of parts of one.
+ */
+export interface AudioRecorderHost {
+  start(): Promise<void>;
+  pause(): Promise<void>;
+  resume(): Promise<void>;
+  stop(): Promise<AudioRecording>;
+  cancel(): Promise<void>;
+  meter(): Promise<AudioMeter>;
+  /** Deletes a recording `stop` made (its file; a `data:` URL is nothing to delete). */
+  discard(url: string): Promise<void>;
+  /**
+   * A new recording made of the parts of `url` in `keep` (seconds, in
+   * order, concatenated): `[{ start, end }]` trims to that range, two parts
+   * around a range delete it. The original is left as it was. Rejects with
+   * `unsupported: …` where the host cannot cut audio, `empty: …` when
+   * nothing of the recording falls in `keep`.
+   */
+  trim(url: string, keep: readonly AudioRange[]): Promise<AudioRecording>;
+}
+
+/** The formats to ask for, best first: AAC in MP4 (what Voice Memos keeps; Safari), else Opus in WebM. */
+const TYPES = ['audio/mp4', 'audio/webm;codecs=opus', 'audio/webm', 'audio/ogg;codecs=opus'];
+
+/** A level from an RMS amplitude: -50 dB and below is silence, 0 dB full, as Voice Memos' bars scale. */
+export function levelOf(rms: number): number {
+  if (rms <= 0) return 0;
+  const db = 20 * Math.log10(rms);
+  return Math.max(0, Math.min(1, (db + 50) / 50));
+}
+
+interface Session {
+  stream: MediaStream;
+  recorder: MediaRecorder;
+  chunks: Blob[];
+  context: AudioContext | null;
+  analyser: AnalyserNode | null;
+  samples: Float32Array<ArrayBuffer> | null;
+  /** Seconds recorded before the current stretch. */
+  recorded: number;
+  /** When the current stretch began (ms), or null while paused. */
+  since: number | null;
+}
+
+/** The file extension a recording of `type` is named with. */
+export function extensionOf(type: string): string {
+  const base = (type.split(';')[0] ?? type).trim().toLowerCase();
+  switch (base) {
+    case 'audio/mp4': case 'audio/x-m4a': case 'audio/aac': return 'm4a';
+    case 'audio/webm': return 'webm';
+    case 'audio/ogg': return 'ogg';
+    case 'audio/wav': case 'audio/wave': case 'audio/x-wav': return 'wav';
+    case 'audio/mpeg': return 'mp3';
+    default: return 'bin';
+  }
+}
+
+/** The app's `Documents/Recordings`, made if need be. */
+async function recordingsFolder(files: FilesHost): Promise<string> {
+  const folder = fileURL(files.directories.documents, 'Recordings');
+  await files.createDirectory(folder);
+  return folder;
+}
+
+/** Writes `bytes` as a new file in `Documents/Recordings`, named by a UUID and the type's extension: its URL. */
+export async function recordingFile(files: FilesHost, bytes: Uint8Array, type: string): Promise<string> {
+  const folder = await recordingsFolder(files);
+  const name = typeof crypto !== 'undefined' && typeof crypto.randomUUID === 'function' ? crypto.randomUUID().toUpperCase() : `${Date.now()}-${Math.random().toString(36).slice(2)}`;
+  const url = fileURL(folder, `${name}.${extensionOf(type)}`);
+  await files.write(url, bytes);
+  return url;
+}
+
+export class BrowserAudioRecorder implements AudioRecorderHost {
+  private session: Session | null = null;
+
+  /** `files` is the app's files host, looked up at each call (the environment fills it in after this is made). */
+  constructor(private readonly files: () => FilesHost | undefined = () => undefined) {}
+
+  /** The recording's URL: a file in `Documents/Recordings` where the page keeps files, else a `data:` URL. */
+  private async keep(blob: Blob, type: string): Promise<string> {
+    const files = this.files();
+    if (files) {
+      try {
+        return await recordingFile(files, new Uint8Array(await blob.arrayBuffer()), type);
+      } catch {
+        // The file could not be written (the page's storage refused it): the sound stays with its URL.
+      }
+    }
+    return dataURL(blob);
+  }
+
+  /** The bytes of a recording: its file, or what its `data:` URL holds. */
+  private async bytesOf(url: string): Promise<ArrayBuffer> {
+    const files = this.files();
+    if (url.startsWith('file:') && files) {
+      const bytes = await files.read(url);
+      return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength) as ArrayBuffer;
+    }
+    return (await fetch(url)).arrayBuffer();
+  }
+
+  async start(): Promise<void> {
+    if (this.session) throw new Error('busy: already recording');
+    // The context first, in the tap that asked to record: Safari starts one
+    // made after the microphone prompt (outside the gesture) suspended, and
+    // its analyser then reads silence, a flat waveform for a loud recording.
+    let context: AudioContext | null = null;
+    try {
+      context = new AudioContext();
+      void context.resume();
+    } catch {
+      // No Web Audio: the recording goes on, its level stays 0.
+    }
+    let stream: MediaStream;
+    try {
+      stream = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: false, noiseSuppression: false, autoGainControl: true } });
+    } catch (error) {
+      void context?.close();
+      const name = (error as { name?: string }).name;
+      if (name === 'NotAllowedError' || name === 'SecurityError') throw new Error('permission: the microphone was not allowed');
+      throw new Error(`unavailable: no microphone (${String(error)})`);
+    }
+    const mimeType = TYPES.find((type) => MediaRecorder.isTypeSupported(type));
+    const recorder = new MediaRecorder(stream, mimeType ? { mimeType } : undefined);
+    const session: Session = { stream, recorder, chunks: [], context: null, analyser: null, samples: null, recorded: 0, since: performance.now() };
+    recorder.addEventListener('dataavailable', (event) => {
+      if (event.data.size > 0) session.chunks.push(event.data);
+    });
+    if (context) {
+      try {
+        const analyser = context.createAnalyser();
+        analyser.fftSize = 1024;
+        context.createMediaStreamSource(stream).connect(analyser);
+        session.context = context;
+        session.analyser = analyser;
+        session.samples = new Float32Array(analyser.fftSize);
+      } catch {
+        void context.close();
+      }
+    }
+    recorder.start(250);
+    this.session = session;
+  }
+
+  async pause(): Promise<void> {
+    const session = this.active();
+    if (session.since === null) return;
+    session.recorder.pause();
+    session.recorded += (performance.now() - session.since) / 1000;
+    session.since = null;
+  }
+
+  async resume(): Promise<void> {
+    const session = this.active();
+    if (session.since !== null) return;
+    session.recorder.resume();
+    session.since = performance.now();
+  }
+
+  async stop(): Promise<AudioRecording> {
+    const session = this.active();
+    const duration = this.time(session);
+    const stopped = new Promise<void>((resolve) => session.recorder.addEventListener('stop', () => resolve(), { once: true }));
+    session.recorder.stop();
+    await stopped;
+    this.end(session);
+    const type = session.recorder.mimeType || session.chunks[0]?.type || 'audio/webm';
+    const blob = new Blob(session.chunks, { type });
+    const plain = type.split(';')[0] ?? type;
+    return { url: await this.keep(blob, plain), duration, type: plain };
+  }
+
+  async cancel(): Promise<void> {
+    const session = this.session;
+    if (!session) return;
+    if (session.recorder.state !== 'inactive') session.recorder.stop();
+    this.end(session);
+  }
+
+  async meter(): Promise<AudioMeter> {
+    const session = this.active();
+    let level = 0;
+    if (session.analyser && session.samples && session.since !== null) {
+      // A context the browser suspended anyway (a long permission prompt) is asked again.
+      if (session.context?.state === 'suspended') void session.context.resume();
+      session.analyser.getFloatTimeDomainData(session.samples);
+      let sum = 0;
+      for (const sample of session.samples) sum += sample * sample;
+      level = levelOf(Math.sqrt(sum / session.samples.length));
+    }
+    return { level, time: this.time(session) };
+  }
+
+  async discard(url: string): Promise<void> {
+    // A `data:` URL's sound goes with the URL; a file in Recordings is deleted (one gone already is nothing to delete).
+    const files = this.files();
+    if (!url.startsWith('file:') || !files) return;
+    await files.remove(url).catch((error: unknown) => {
+      if (!(error instanceof Error && error.message.startsWith('notFound:'))) throw error;
+    });
+  }
+
+  async trim(url: string, keep: readonly AudioRange[]): Promise<AudioRecording> {
+    const decoder = audioDecoder();
+    if (!decoder) throw new Error('unsupported: this page cannot cut audio');
+    let decoded: AudioBuffer;
+    try {
+      decoded = await decoder.context.decodeAudioData(await this.bytesOf(url));
+    } catch (error) {
+      throw new Error(`decode: the recording could not be read (${String(error)})`);
+    } finally {
+      decoder.done();
+    }
+    const channels: Float32Array[] = [];
+    for (let channel = 0; channel < decoded.numberOfChannels; channel += 1) channels.push(decoded.getChannelData(channel));
+    const kept = sliceSamples(channels, decoded.sampleRate, keptRanges(keep, decoded.duration));
+    const frames = kept[0]?.length ?? 0;
+    if (frames === 0) throw new Error('empty: nothing of the recording was kept');
+    const wav = encodeWAV(kept, decoded.sampleRate);
+    return { url: await this.keep(new Blob([wav], { type: 'audio/wav' }), 'audio/wav'), duration: frames / decoded.sampleRate, type: 'audio/wav' };
+  }
+
+  private active(): Session {
+    if (!this.session) throw new Error('idle: not recording');
+    return this.session;
+  }
+
+  private time(session: Session): number {
+    return session.recorded + (session.since === null ? 0 : (performance.now() - session.since) / 1000);
+  }
+
+  private end(session: Session): void {
+    for (const track of session.stream.getTracks()) track.stop();
+    void session.context?.close().catch(() => undefined);
+    if (this.session === session) this.session = null;
+  }
+}
+
+/**
+ * The ranges of a `duration`-second recording that `trim` keeps: each clamped
+ * to the recording, the empty ones dropped, in time order, and overlapping
+ * ones merged, so the kept parts never repeat a sound.
+ */
+export function keptRanges(keep: readonly AudioRange[], duration: number): AudioRange[] {
+  const clamped = keep
+    .map((range) => ({ start: Math.max(0, Math.min(duration, range.start)), end: Math.max(0, Math.min(duration, range.end)) }))
+    .filter((range) => Number.isFinite(range.start) && Number.isFinite(range.end) && range.end > range.start)
+    .sort((a, b) => a.start - b.start);
+  const merged: AudioRange[] = [];
+  for (const range of clamped) {
+    const last = merged[merged.length - 1];
+    if (last && range.start <= last.end) last.end = Math.max(last.end, range.end);
+    else merged.push({ ...range });
+  }
+  return merged;
+}
+
+/**
+ * The frames of `channels` (one array of samples per channel, at
+ * `sampleRate`) that fall in `ranges` (seconds), concatenated in order: one
+ * new array per channel, all the same length.
+ */
+export function sliceSamples(channels: readonly Float32Array[], sampleRate: number, ranges: readonly AudioRange[]): Float32Array[] {
+  const length = channels[0]?.length ?? 0;
+  const frames = ranges.map((range) => {
+    const start = Math.min(length, Math.max(0, Math.round(range.start * sampleRate)));
+    const end = Math.min(length, Math.max(start, Math.round(range.end * sampleRate)));
+    return { start, end };
+  });
+  const total = frames.reduce((sum, range) => sum + (range.end - range.start), 0);
+  return channels.map((samples) => {
+    const kept = new Float32Array(total);
+    let at = 0;
+    for (const range of frames) {
+      kept.set(samples.subarray(range.start, range.end), at);
+      at += range.end - range.start;
+    }
+    return kept;
+  });
+}
+
+/**
+ * `channels` (one array of -1…1 samples per channel, all the same length) as
+ * a WAV file: a RIFF header and the samples interleaved as 16-bit PCM, which
+ * every media element plays.
+ */
+export function encodeWAV(channels: readonly Float32Array[], sampleRate: number): ArrayBuffer {
+  const channelCount = channels.length;
+  const frames = channels[0]?.length ?? 0;
+  const blockAlign = channelCount * 2;
+  const dataSize = frames * blockAlign;
+  const buffer = new ArrayBuffer(44 + dataSize);
+  const view = new DataView(buffer);
+  const ascii = (offset: number, text: string): void => {
+    for (let i = 0; i < text.length; i += 1) view.setUint8(offset + i, text.charCodeAt(i));
+  };
+  ascii(0, 'RIFF');
+  view.setUint32(4, 36 + dataSize, true);
+  ascii(8, 'WAVE');
+  ascii(12, 'fmt ');
+  view.setUint32(16, 16, true); // The fmt chunk's size.
+  view.setUint16(20, 1, true); // PCM.
+  view.setUint16(22, channelCount, true);
+  view.setUint32(24, sampleRate, true);
+  view.setUint32(28, sampleRate * blockAlign, true);
+  view.setUint16(32, blockAlign, true);
+  view.setUint16(34, 16, true); // Bits per sample.
+  ascii(36, 'data');
+  view.setUint32(40, dataSize, true);
+  let offset = 44;
+  for (let frame = 0; frame < frames; frame += 1) {
+    for (const samples of channels) {
+      const sample = Math.max(-1, Math.min(1, samples[frame] ?? 0));
+      view.setInt16(offset, sample < 0 ? Math.round(sample * 0x8000) : Math.round(sample * 0x7fff), true);
+      offset += 2;
+    }
+  }
+  return buffer;
+}
+
+/**
+ * A context to decode with, and how to let it go: an `OfflineAudioContext`
+ * where there is one (it touches no hardware and needs no closing), else an
+ * `AudioContext`, closed once it has decoded (it holds the hardware, and
+ * Safari allows a page few); nothing where the page has no Web Audio.
+ */
+function audioDecoder(): { context: BaseAudioContext; done(): void } | undefined {
+  if (typeof OfflineAudioContext !== 'undefined') return { context: new OfflineAudioContext(1, 1, 44_100), done: () => undefined };
+  if (typeof AudioContext !== 'undefined') {
+    const context = new AudioContext();
+    return { context, done: () => void context.close().catch(() => undefined) };
+  }
+  return undefined;
+}
+
+function dataURL(blob: Blob): Promise<string> {
+  return new Promise((resolve, reject) => {
+    const reader = new FileReader();
+    reader.onload = () => resolve(String(reader.result));
+    reader.onerror = () => reject(reader.error ?? new Error('read: the recording could not be read'));
+    reader.readAsDataURL(blob);
+  });
+}
+
+/** The browser's recorder, where it has a microphone API and `MediaRecorder`; `files` is where recordings go (the environment's files host). */
+export function browserAudioRecorder(files: () => FilesHost | undefined = () => undefined): AudioRecorderHost | undefined {
+  if (typeof navigator === 'undefined' || typeof navigator.mediaDevices?.getUserMedia !== 'function') return undefined;
+  if (typeof MediaRecorder === 'undefined') return undefined;
+  return new BrowserAudioRecorder(files);
+}

package/src/editorSelection.ts

@@ -0,0 +1,80 @@
+/**
+ * The selection a bound `texteditor` shares with the app (docs/ops-protocol.md,
+ * "Rich text editors"), the same for the rich editor (src/richEditor.ts) and
+ * the plain textarea (src/plainEditor.ts): what the editor last reported, or
+ * was last moved to, so a selection coming back from the app is not applied
+ * again and the viewer's is not echoed; and one the app set while the editor
+ * was not focused, kept for when the app focuses it.
+ */
+
+import type { TextSelectionRange } from './protocol';
+
+/** The selection each bound editor last reported, or was last moved to. */
+const reported = new WeakMap<HTMLElement, string>();
+/** A selection the app set while the editor was not focused: where its caret goes when the app focuses it. */
+const pending = new WeakMap<HTMLElement, TextSelectionRange>();
+
+const key = (range: TextSelectionRange): string => `${range.start}:${range.end}`;
+
+/** Whether the editor's `selection` is bound (the prop is in its props): only then is the viewer's reported. */
+export function markSelectionBound(el: HTMLElement, bound: boolean): void {
+  if (bound) el.dataset['sbSelection'] = '';
+  else delete el.dataset['sbSelection'];
+}
+
+export function isSelectionBound(el: HTMLElement): boolean {
+  return el.dataset['sbSelection'] !== undefined;
+}
+
+/**
+ * Whether the field or editor has `deleteBackward` in its props: Backspace
+ * with nothing selected and the caret at offset 0 then sends `deletebackward`
+ * and deletes nothing (docs/ops-protocol.md, "The delete key at the start of
+ * a field"); the same mark for a `textfield`, a plain editor and a rich one.
+ */
+export function markDeleteBackward(el: HTMLElement, wanted: boolean): void {
+  if (wanted) el.dataset['sbDeleteBackward'] = '';
+  else delete el.dataset['sbDeleteBackward'];
+}
+
+export function wantsDeleteBackward(el: HTMLElement): boolean {
+  return el.dataset['sbDeleteBackward'] !== undefined;
+}
+
+/**
+ * The app sent `range` (or no selection): the range the editor moves to now,
+ * when it is focused; otherwise null, the range kept for `takePendingSelection`
+ * when the app focuses the editor. Nothing for the selection the editor
+ * itself reported, coming back.
+ */
+export function selectionToApply(el: HTMLElement, range: TextSelectionRange | undefined): TextSelectionRange | null {
+  if (!range || reported.get(el) === key(range)) return null;
+  reported.set(el, key(range));
+  if (el.ownerDocument.activeElement === el) return range;
+  pending.set(el, range);
+  return null;
+}
+
+/** The selection the app set while the editor was not focused, taken as the app focuses it. */
+export function takePendingSelection(el: HTMLElement): TextSelectionRange | undefined {
+  const range = pending.get(el);
+  pending.delete(el);
+  return range;
+}
+
+/** The viewer's selection: whether it is new (not the one the app set or the editor last reported), and so to report. */
+export function noteViewerSelection(el: HTMLElement, range: TextSelectionRange): boolean {
+  if (reported.get(el) === key(range)) return false;
+  reported.set(el, key(range));
+  return true;
+}
+
+/** The viewer focused the editor: their caret, not one the app set before. */
+export function viewerFocused(el: HTMLElement): void {
+  pending.delete(el);
+}
+
+/** The editor lost focus: it has no caret, so the selection the app holds is one to move to again. */
+export function editorBlurred(el: HTMLElement): void {
+  reported.delete(el);
+}