@swiftbrowser/native 0.18.1 → 0.18.2

package/src/speech.ts

@@ -0,0 +1,312 @@
+/**
+ * The Web Speech API over the host: `speechSynthesis` and
+ * `SpeechSynthesisUtterance` on `AVSpeechSynthesizer` (the bridge's
+ * `speak(id, json)` and `speechControl(action)`, events at
+ * `__sbSpeechEvent(id, json)`), and `SpeechRecognition` on
+ * `SFSpeechRecognizer` (`recognize(id, action, json)`, events at
+ * `__sbRecognition(id, json)`; the app's Info.plist names
+ * `NSSpeechRecognitionUsageDescription` and `NSMicrophoneUsageDescription`).
+ * The shapes apps use: an utterance's `text`, `lang`, `rate`, `pitch`,
+ * `volume` and its `start`, `end`, `error`, `pause`, `resume` and `boundary`
+ * events; a recognition's `lang`, `continuous`, `interimResults`, `start()`,
+ * `stop()`, `abort()` and its `start`, `result` (with `results[i][0].transcript`
+ * and `isFinal`), `error` (`error`: `not-allowed`, `audio-capture`,
+ * `no-speech`, `network`, `aborted`) and `end` events.
+ */
+
+import type { NativeBridge } from './bridge';
+
+type Listener<E> = (event: E) => void;
+
+/** Handlers and listeners by event type, as the DOM's targets have them. */
+class Target<E extends { type: string }> {
+  private readonly listeners = new Map<string, Set<Listener<E>>>();
+  constructor(private readonly handlers: Record<string, unknown>) {}
+  add(type: string, listener: Listener<E>): void {
+    if (typeof listener !== 'function') return;
+    this.listeners.set(type, (this.listeners.get(type) ?? new Set()).add(listener));
+  }
+  remove(type: string, listener: Listener<E>): void {
+    this.listeners.get(type)?.delete(listener);
+  }
+  fire(event: E): void {
+    const handler = this.handlers[`on${event.type}`];
+    if (typeof handler === 'function') (handler as Listener<E>)(event);
+    for (const listener of [...(this.listeners.get(event.type) ?? [])]) listener(event);
+  }
+}
+
+// MARK: Synthesis
+
+export interface SpeechSynthesisEvent {
+  type: 'start' | 'end' | 'error' | 'pause' | 'resume' | 'boundary';
+  utterance: SpeechSynthesisUtterance;
+  charIndex: number;
+  charLength: number;
+  elapsedTime: number;
+  name: string;
+  error?: string;
+}
+
+export class SpeechSynthesisUtterance {
+  text: string;
+  lang = '';
+  voice: { name: string; lang: string } | null = null;
+  volume = 1;
+  rate = 1;
+  pitch = 1;
+  onstart: Listener<SpeechSynthesisEvent> | null = null;
+  onend: Listener<SpeechSynthesisEvent> | null = null;
+  onerror: Listener<SpeechSynthesisEvent> | null = null;
+  onpause: Listener<SpeechSynthesisEvent> | null = null;
+  onresume: Listener<SpeechSynthesisEvent> | null = null;
+  onboundary: Listener<SpeechSynthesisEvent> | null = null;
+  onmark: Listener<SpeechSynthesisEvent> | null = null;
+  /** @internal */
+  readonly target = new Target<SpeechSynthesisEvent>(this as unknown as Record<string, unknown>);
+  constructor(text = '') {
+    this.text = String(text);
+  }
+  addEventListener(type: string, listener: Listener<SpeechSynthesisEvent>): void {
+    this.target.add(type, listener);
+  }
+  removeEventListener(type: string, listener: Listener<SpeechSynthesisEvent>): void {
+    this.target.remove(type, listener);
+  }
+}
+
+export interface SpeechSynthesis {
+  readonly speaking: boolean;
+  readonly pending: boolean;
+  readonly paused: boolean;
+  onvoiceschanged: (() => void) | null;
+  speak(utterance: SpeechSynthesisUtterance): void;
+  cancel(): void;
+  pause(): void;
+  resume(): void;
+  getVoices(): { name: string; lang: string; default: boolean; localService: boolean; voiceURI: string }[];
+  addEventListener(type: string, listener: () => void): void;
+  removeEventListener(type: string, listener: () => void): void;
+}
+
+/** The host's event for utterance `id`. */
+export interface NativeSpeechEvent {
+  type: 'start' | 'end' | 'cancel' | 'pause' | 'resume' | 'boundary' | 'error';
+  charIndex?: number;
+  charLength?: number;
+  message?: string;
+}
+
+export function createSpeechSynthesis(native: NativeBridge): { speechSynthesis: SpeechSynthesis; receive(id: number, json: string): void } {
+  /** The utterances handed to the host, by id, in order: the first is speaking once it started. */
+  const queue = new Map<number, SpeechSynthesisUtterance>();
+  let nextId = 1;
+  let speakingId: number | null = null;
+  let paused = false;
+  const started = new Map<number, number>();
+  const event = (type: SpeechSynthesisEvent['type'], utterance: SpeechSynthesisUtterance, id: number, extra: Partial<SpeechSynthesisEvent> = {}): SpeechSynthesisEvent => ({
+    type,
+    utterance,
+    charIndex: 0,
+    charLength: 0,
+    elapsedTime: (native.now() - (started.get(id) ?? native.now())) / 1000,
+    name: '',
+    ...extra,
+  });
+  const synthesis: SpeechSynthesis = {
+    get speaking() {
+      return speakingId !== null;
+    },
+    get pending() {
+      return queue.size > (speakingId === null ? 0 : 1);
+    },
+    get paused() {
+      return paused;
+    },
+    onvoiceschanged: null,
+    speak(utterance) {
+      if (!(utterance instanceof SpeechSynthesisUtterance)) throw new TypeError('speechSynthesis.speak: not a SpeechSynthesisUtterance');
+      const id = nextId++;
+      queue.set(id, utterance);
+      native.speak!(id, JSON.stringify({ text: utterance.text, lang: utterance.lang || null, voice: utterance.voice?.name ?? null, rate: utterance.rate, pitch: utterance.pitch, volume: utterance.volume }));
+    },
+    cancel() {
+      native.speechControl!('cancel');
+      // The host answers each with `cancel`; the queue empties as they come.
+    },
+    pause() {
+      native.speechControl!('pause');
+    },
+    resume() {
+      native.speechControl!('resume');
+    },
+    getVoices() {
+      return [];
+    },
+    addEventListener() {},
+    removeEventListener() {},
+  };
+  return {
+    speechSynthesis: synthesis,
+    receive(id, json) {
+      const utterance = queue.get(id);
+      if (!utterance) return;
+      const e = JSON.parse(json) as NativeSpeechEvent;
+      switch (e.type) {
+        case 'start':
+          speakingId = id;
+          paused = false;
+          started.set(id, native.now());
+          utterance.target.fire(event('start', utterance, id));
+          break;
+        case 'boundary':
+          utterance.target.fire(event('boundary', utterance, id, { name: 'word', charIndex: e.charIndex ?? 0, charLength: e.charLength ?? 0 }));
+          break;
+        case 'pause':
+          paused = true;
+          utterance.target.fire(event('pause', utterance, id, { charIndex: e.charIndex ?? 0 }));
+          break;
+        case 'resume':
+          paused = false;
+          utterance.target.fire(event('resume', utterance, id, { charIndex: e.charIndex ?? 0 }));
+          break;
+        case 'end':
+        case 'cancel':
+        case 'error': {
+          queue.delete(id);
+          if (speakingId === id) speakingId = null;
+          if (queue.size === 0) paused = false;
+          const done = e.type === 'end' ? event('end', utterance, id) : event('error', utterance, id, { error: e.type === 'cancel' ? 'canceled' : (e.message ?? 'synthesis-failed') });
+          started.delete(id);
+          utterance.target.fire(done);
+          break;
+        }
+      }
+    },
+  };
+}
+
+// MARK: Recognition
+
+export interface SpeechRecognitionAlternative {
+  transcript: string;
+  confidence: number;
+}
+
+export interface SpeechRecognitionResult {
+  readonly isFinal: boolean;
+  readonly length: number;
+  [index: number]: SpeechRecognitionAlternative;
+  item(index: number): SpeechRecognitionAlternative;
+}
+
+export interface SpeechRecognitionEvent {
+  type: 'start' | 'audiostart' | 'result' | 'error' | 'end' | 'audioend' | 'nomatch';
+  resultIndex: number;
+  results: SpeechRecognitionResult[];
+  error?: string;
+  message?: string;
+}
+
+/** The host's event for recognition `id`. */
+export interface NativeRecognitionEvent {
+  type: 'start' | 'result' | 'error' | 'end';
+  transcript?: string;
+  isFinal?: boolean;
+  confidence?: number;
+  error?: string;
+  message?: string;
+}
+
+export type RecognitionBridge = Required<Pick<NativeBridge, 'recognize'>>;
+
+const recognitions = new Map<number, SpeechRecognition>();
+let recognitionBridge: RecognitionBridge | null = null;
+let nextRecognition = 1;
+
+export class SpeechRecognition {
+  lang = '';
+  continuous = false;
+  interimResults = false;
+  maxAlternatives = 1;
+  onstart: Listener<SpeechRecognitionEvent> | null = null;
+  onaudiostart: Listener<SpeechRecognitionEvent> | null = null;
+  onresult: Listener<SpeechRecognitionEvent> | null = null;
+  onerror: Listener<SpeechRecognitionEvent> | null = null;
+  onend: Listener<SpeechRecognitionEvent> | null = null;
+  onaudioend: Listener<SpeechRecognitionEvent> | null = null;
+  onnomatch: Listener<SpeechRecognitionEvent> | null = null;
+  private readonly target = new Target<SpeechRecognitionEvent>(this as unknown as Record<string, unknown>);
+  private id: number | null = null;
+  /** The final results so far in this session, then the interim one (if any) last. */
+  private results: SpeechRecognitionResult[] = [];
+
+  static install(bridge: RecognitionBridge): void {
+    recognitionBridge = bridge;
+  }
+
+  start(): void {
+    if (!recognitionBridge) throw new Error('SpeechRecognition: no host');
+    if (this.id !== null) throw Object.assign(new Error('SpeechRecognition: already started'), { name: 'InvalidStateError' });
+    this.id = nextRecognition++;
+    this.results = [];
+    recognitions.set(this.id, this);
+    recognitionBridge.recognize(this.id, 'start', JSON.stringify({ lang: this.lang || null, continuous: this.continuous, interimResults: this.interimResults }));
+  }
+
+  /** Stops listening; what was heard is still delivered, then `end`. */
+  stop(): void {
+    if (this.id === null) return;
+    recognitionBridge?.recognize(this.id, 'stop', '{}');
+  }
+
+  /** Stops listening and keeps nothing more. */
+  abort(): void {
+    if (this.id === null) return;
+    recognitionBridge?.recognize(this.id, 'abort', '{}');
+  }
+
+  addEventListener(type: string, listener: Listener<SpeechRecognitionEvent>): void {
+    this.target.add(type, listener);
+  }
+  removeEventListener(type: string, listener: Listener<SpeechRecognitionEvent>): void {
+    this.target.remove(type, listener);
+  }
+
+  /** `__sbRecognition(id, json)`. */
+  static receive(id: number, json: string): void {
+    recognitions.get(id)?.handle(JSON.parse(json) as NativeRecognitionEvent);
+  }
+
+  private event(type: SpeechRecognitionEvent['type'], extra: Partial<SpeechRecognitionEvent> = {}): SpeechRecognitionEvent {
+    return { type, resultIndex: 0, results: this.results, ...extra };
+  }
+
+  private handle(e: NativeRecognitionEvent): void {
+    switch (e.type) {
+      case 'start':
+        this.target.fire(this.event('start'));
+        this.target.fire(this.event('audiostart'));
+        break;
+      case 'result': {
+        const alternative: SpeechRecognitionAlternative = { transcript: e.transcript ?? '', confidence: e.confidence ?? 0 };
+        const result: SpeechRecognitionResult = { isFinal: e.isFinal === true, length: 1, 0: alternative, item: () => alternative };
+        // An interim result replaces the last interim one; a final one takes its place for good.
+        const last = this.results[this.results.length - 1];
+        const index = last && !last.isFinal ? this.results.length - 1 : this.results.length;
+        this.results = [...this.results.slice(0, index), result];
+        this.target.fire(this.event('result', { resultIndex: index }));
+        break;
+      }
+      case 'error':
+        this.target.fire(this.event('error', { error: e.error ?? 'aborted', message: e.message ?? '' }));
+        break;
+      case 'end':
+        if (this.id !== null) recognitions.delete(this.id);
+        this.id = null;
+        this.target.fire(this.event('audioend'));
+        this.target.fire(this.event('end'));
+        break;
+    }
+  }
+}

package/src/wakeLock.ts

@@ -0,0 +1,63 @@
+/**
+ * `navigator.wakeLock` (the Screen Wake Lock API) over the host's idle timer
+ * (`wakeLock(active)` on the bridge: `UIApplication.isIdleTimerDisabled`, a
+ * power assertion on macOS). `request('screen')` resolves with a
+ * `WakeLockSentinel`; the screen stays awake while any sentinel is held and
+ * `release()` lets it go. Unlike a page's, which the browser releases when
+ * the document hides, a native sentinel stays held in the background: the
+ * system owns the screen there, and the lock holds again when the app returns.
+ */
+
+import type { NativeBridge } from './bridge';
+
+type ReleaseListener = (event: { type: 'release'; target: WakeLockSentinel }) => void;
+
+export class WakeLockSentinel {
+  readonly type = 'screen' as const;
+  released = false;
+  onrelease: ReleaseListener | null = null;
+  private readonly listeners = new Set<ReleaseListener>();
+  /** @internal */
+  constructor(private readonly onRelease: (sentinel: WakeLockSentinel) => void) {}
+
+  release(): Promise<void> {
+    if (this.released) return Promise.resolve();
+    this.released = true;
+    this.onRelease(this);
+    const event = { type: 'release' as const, target: this };
+    this.onrelease?.(event);
+    for (const listener of [...this.listeners]) listener(event);
+    return Promise.resolve();
+  }
+
+  addEventListener(type: string, listener: ReleaseListener): void {
+    if (type === 'release' && typeof listener === 'function') this.listeners.add(listener);
+  }
+
+  removeEventListener(type: string, listener: ReleaseListener): void {
+    if (type === 'release') this.listeners.delete(listener);
+  }
+}
+
+export interface WakeLock {
+  request(type?: string): Promise<WakeLockSentinel>;
+}
+
+export function createWakeLock(native: NativeBridge): WakeLock {
+  const held = new Set<WakeLockSentinel>();
+  const release = (sentinel: WakeLockSentinel): void => {
+    held.delete(sentinel);
+    if (held.size === 0) native.wakeLock!(false);
+  };
+  return {
+    request(type = 'screen') {
+      if (type !== 'screen') {
+        return Promise.reject(Object.assign(new TypeError(`wakeLock.request: '${String(type)}' is not a wake lock type ('screen')`), { name: 'TypeError' }));
+      }
+      const sentinel = new WakeLockSentinel(release);
+      if (held.size === 0) native.wakeLock!(true);
+      held.add(sentinel);
+      return Promise.resolve(sentinel);
+    },
+  };
+}

package/swift/Sources/SwiftBrowserNative/Host/AppleSignInBridge.swift

@@ -0,0 +1,139 @@
+#if canImport(SwiftUI) && canImport(AuthenticationServices) && !canImport(FoundationBridge)
+import AuthenticationServices
+import Foundation
+import SwiftBrowserProtocol
+#if canImport(UIKit)
+import UIKit
+#elseif canImport(AppKit)
+import AppKit
+#endif
+
+/// Sign in with Apple for the app (the `appleSignIn` host request,
+/// packages/native/src/hostRequests.ts): `signIn` shows the system sheet
+/// through `ASAuthorizationController` with the scopes, nonce and state asked
+/// for (the app's entitlement: `com.apple.developer.applesignin`,
+/// `swiftbrowser native init --sign-in-with-apple`) and answers the
+/// credential as the page's `AppleIDCredential`; `credentialState` asks the
+/// provider what a user identifier's credential is now. One sign-in at a time.
+@MainActor
+final class AppleSignInBridge: NSObject, ASAuthorizationControllerDelegate, ASAuthorizationControllerPresentationContextProviding {
+    private var continuation: CheckedContinuation<JSONValue, any Error>?
+    private var controller: ASAuthorizationController?
+
+    func handle(_ params: JSONObject) async throws -> JSONValue {
+        switch params["action"]?.string {
+        case "signIn":
+            return try await signIn(params)
+        case "credentialState":
+            guard let user = params["user"]?.string else { throw HostRequestFailure("appleSignIn: credentialState needs a user") }
+            return .object(["state": .string(try await credentialState(user))])
+        default:
+            throw HostRequestFailure("unsupported: appleSignIn: unknown action \(params["action"]?.string ?? "")")
+        }
+    }
+
+    private func signIn(_ params: JSONObject) async throws -> JSONValue {
+        guard continuation == nil else { throw HostRequestFailure("busy: a sign-in is already showing") }
+        let request = ASAuthorizationAppleIDProvider().createRequest()
+        let scopes = params["scopes"]?.array?.compactMap(\.string) ?? []
+        request.requestedScopes = scopes.compactMap { scope in
+            switch scope {
+            case "fullName": return .fullName
+            case "email": return .email
+            default: return nil
+            }
+        }
+        request.nonce = params["nonce"]?.string
+        request.state = params["state"]?.string
+        let controller = ASAuthorizationController(authorizationRequests: [request])
+        controller.delegate = self
+        controller.presentationContextProvider = self
+        self.controller = controller
+        return try await withCheckedThrowingContinuation { continuation in
+            self.continuation = continuation
+            controller.performRequests()
+        }
+    }
+
+    private func credentialState(_ user: String) async throws -> String {
+        let state = try await ASAuthorizationAppleIDProvider().credentialState(forUserID: user)
+        switch state {
+        case .authorized: return "authorized"
+        case .revoked: return "revoked"
+        case .notFound: return "notFound"
+        case .transferred: return "transferred"
+        @unknown default: return "unknown"
+        }
+    }
+
+    private func finish(_ result: Result<JSONValue, any Error>) {
+        controller = nil
+        guard let continuation else { return }
+        self.continuation = nil
+        continuation.resume(with: result)
+    }
+
+    /// The credential as JSON (read off the main actor's data, where the delegate is called).
+    private nonisolated static func json(_ credential: ASAuthorizationAppleIDCredential) -> JSONValue {
+        var fields: JSONObject = [
+            "user": .string(credential.user),
+            "email": credential.email.map(JSONValue.string) ?? .null,
+            "identityToken": credential.identityToken.flatMap { String(data: $0, encoding: .utf8) }.map(JSONValue.string) ?? .null,
+            "authorizationCode": credential.authorizationCode.flatMap { String(data: $0, encoding: .utf8) }.map(JSONValue.string) ?? .null,
+            "state": credential.state.map(JSONValue.string) ?? .null,
+        ]
+        switch credential.realUserStatus {
+        case .likelyReal: fields["realUserStatus"] = .string("likelyReal")
+        case .unknown: fields["realUserStatus"] = .string("unknown")
+        case .unsupported: fields["realUserStatus"] = .string("unsupported")
+        @unknown default: fields["realUserStatus"] = .string("unknown")
+        }
+        if let name = credential.fullName {
+            fields["fullName"] = .object([
+                "givenName": name.givenName.map(JSONValue.string) ?? .null,
+                "familyName": name.familyName.map(JSONValue.string) ?? .null,
+                "middleName": name.middleName.map(JSONValue.string) ?? .null,
+                "namePrefix": name.namePrefix.map(JSONValue.string) ?? .null,
+                "nameSuffix": name.nameSuffix.map(JSONValue.string) ?? .null,
+                "nickname": name.nickname.map(JSONValue.string) ?? .null,
+            ])
+        } else {
+            fields["fullName"] = .null
+        }
+        return .object(fields)
+    }
+
+    // MARK: ASAuthorizationControllerDelegate (on the main thread, where the controller was made)
+
+    nonisolated func authorizationController(controller: ASAuthorizationController, didCompleteWithAuthorization authorization: ASAuthorization) {
+        let result: Result<JSONValue, any Error>
+        if let credential = authorization.credential as? ASAuthorizationAppleIDCredential {
+            result = .success(Self.json(credential))
+        } else {
+            result = .failure(HostRequestFailure("appleSignIn: the credential is not an Apple ID"))
+        }
+        MainActor.assumeIsolated { finish(result) }
+    }
+
+    nonisolated func authorizationController(controller: ASAuthorizationController, didCompleteWithError error: any Error) {
+        let cancelled = (error as? ASAuthorizationError)?.code == .canceled
+        let message = error.localizedDescription
+        MainActor.assumeIsolated {
+            finish(.failure(HostRequestFailure(cancelled ? "cancelled: the viewer closed Sign in with Apple" : "appleSignIn: \(message)")))
+        }
+    }
+
+    // MARK: ASAuthorizationControllerPresentationContextProviding
+
+    nonisolated func presentationAnchor(for controller: ASAuthorizationController) -> ASPresentationAnchor {
+        MainActor.assumeIsolated {
+            #if canImport(UIKit)
+            let windows = UIApplication.shared.connectedScenes.compactMap { $0 as? UIWindowScene }.flatMap(\.windows)
+            return windows.first { $0.isKeyWindow } ?? windows.first ?? ASPresentationAnchor()
+            #else
+            return NSApplication.shared.keyWindow ?? ASPresentationAnchor()
+            #endif
+        }
+    }
+}
+#endif

package/swift/Sources/SwiftBrowserNative/Host/AudioRecorderBridge.swift

@@ -39,6 +39,11 @@ final class AudioRecorderBridge {
 
     private func start() async throws {
         guard recorder == nil else { throw HostRequestFailure("busy: already recording") }
+        // Asking for the microphone without the usage string in the app's
+        // Info.plist ends the app (iOS's privacy check): an error instead.
+        guard Bundle.main.object(forInfoDictionaryKey: "NSMicrophoneUsageDescription") != nil else {
+            throw HostRequestFailure("permission: the app's Info.plist has no NSMicrophoneUsageDescription")
+        }
         guard await AVAudioApplication.requestRecordPermission() else {
             throw HostRequestFailure("permission: the microphone was not allowed")
         }