@swiftbrowser/native 0.18.3 → 0.19.0

package/src/webSocket.ts

@@ -11,7 +11,7 @@
 
 import { SimpleBlob } from './blob';
 import type { NativeBridge } from './bridge';
-import { base64Decode, base64Encode } from './encoding';
+import { base64Encode, bridgedBytes } from './encoding';
 
 export interface WebSocketEventLike {
   type: string;
@@ -31,7 +31,10 @@ export interface NativeWebSocketEvent {
   protocol?: string;
   text?: string;
   /** Base64. */
-  binary?: string;
+  /** Base64, or `{ $bytes: 0 }` naming the `Uint8Array` after the event's JSON. */
+  binary?: string | { $bytes: number };
+  /** The views that came after the JSON. */
+  bytes?: ArrayBufferView[];
   message?: string;
   code?: number;
   reason?: string;
@@ -43,6 +46,8 @@ export type WebSocketBridge = Required<Pick<NativeBridge, 'wsOpen' | 'wsSend' |
 const sockets = new Map<number, NativeWebSocket>();
 let nextId = 1;
 let bridge: WebSocketBridge | null = null;
+/** Whether binary crosses as views beside the JSON (a host that carries bytes) rather than base64. */
+let carriesBytes = false;
 
 export class NativeWebSocket {
   static readonly CONNECTING = 0;
@@ -75,7 +80,9 @@ export class NativeWebSocket {
     this.url = href;
     this.id = nextId++;
     sockets.set(this.id, this);
-    bridge.wsOpen(this.id, href, typeof protocols === 'string' ? [protocols] : [...protocols]);
+    const list = typeof protocols === 'string' ? [protocols] : [...protocols];
+    if (carriesBytes) bridge.wsOpen(this.id, href, list, true);
+    else bridge.wsOpen(this.id, href, list);
   }
 
   send(data: string | ArrayBuffer | ArrayBufferView | SimpleBlob): void {
@@ -85,12 +92,19 @@ export class NativeWebSocket {
       bridge?.wsSend(this.id, JSON.stringify({ text: data }));
       return;
     }
-    const bytes = data instanceof SimpleBlob ? data.bytes() : data instanceof ArrayBuffer ? new Uint8Array(data) : new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
-    bridge?.wsSend(this.id, JSON.stringify({ binary: base64Encode(bytes) }));
+    const bytes =
+      data instanceof SimpleBlob
+        ? data.bytes()
+        : data instanceof ArrayBuffer
+          ? new Uint8Array(data)
+          : new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
+    if (carriesBytes) bridge?.wsSend(this.id, JSON.stringify({ binary: { $bytes: 0 } }), bytes);
+    else bridge?.wsSend(this.id, JSON.stringify({ binary: base64Encode(bytes) }));
   }
 
   close(code?: number, reason?: string): void {
-    if (code !== undefined && code !== 1000 && !(code >= 3000 && code <= 4999)) throw new DOMExceptionLike('InvalidAccessError', 'WebSocket: the close code must be 1000 or in 3000–4999');
+    if (code !== undefined && code !== 1000 && !(code >= 3000 && code <= 4999))
+      throw new DOMExceptionLike('InvalidAccessError', 'WebSocket: the close code must be 1000 or in 3000–4999');
     if (this.readyState === NativeWebSocket.CLOSING || this.readyState === NativeWebSocket.CLOSED) return;
     this.readyState = NativeWebSocket.CLOSING;
     bridge?.wsClose(this.id, code ?? 1000, reason ?? '');
@@ -129,7 +143,7 @@ export class NativeWebSocket {
         if (this.readyState !== NativeWebSocket.OPEN) return;
         let data: unknown = event.text ?? '';
         if (event.binary !== undefined) {
-          const bytes = base64Decode(event.binary);
+          const bytes = bridgedBytes(typeof event.binary === 'string' ? event.binary : event.bytes?.[event.binary.$bytes]);
           data = this.binaryType === 'arraybuffer' ? bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength) : new SimpleBlob([bytes]);
         }
         this.dispatchEvent({ type: 'message', target: this, data });
@@ -147,8 +161,8 @@ export class NativeWebSocket {
     }
   }
 
-  /** The bridge's `__sbWebSocketEvent(id, json)`. */
-  static receive(id: number, json: string | NativeWebSocketEvent): void {
+  /** The bridge's `__sbWebSocketEvent(id, json, ...bytes)`: a binary message's bytes after the JSON that names them. */
+  static receive(id: number, json: string | NativeWebSocketEvent, ...bytes: ArrayBufferView[]): void {
     const socket = sockets.get(id);
     if (!socket) return;
     let event: NativeWebSocketEvent;
@@ -157,12 +171,13 @@ export class NativeWebSocket {
     } catch {
       return;
     }
-    socket.receive(event);
+    socket.receive(bytes.length ? { ...event, bytes } : event);
   }
 
-  /** Points the sockets at the bridge (the runtime, once per install). */
-  static install(native: WebSocketBridge): void {
+  /** Points the sockets at the bridge (the runtime, once per install); `bytes`: the host carries typed arrays. */
+  static install(native: WebSocketBridge, options: { bytes?: boolean } = {}): void {
     bridge = native;
+    carriesBytes = options.bytes === true;
   }
 }
 

package/swift/Packages/SwiftBrowserDSP/Package.swift

@@ -0,0 +1,31 @@
+// swift-tools-version: 6.0
+import PackageDescription
+
+// SwiftBrowserDSP: the native side of @swiftbrowser/dsp. A program written
+// in the DSP language and compiled to bytecode by the package's TypeScript
+// compiler runs here, on the audio thread, in an `AVAudioSourceNode` of the
+// app's `AVAudioEngine`; JavaScript attaches it as a `DSPNode` and plays
+// notes on it. A package of its own, so only an app that synthesizes sound
+// links it; it installs itself as the `program` node kind
+// (`SwiftBrowserDSP.install()`, SwiftBrowserNative's AudioEngineNodes.swift).
+//
+// SwiftBrowserDSPCore is the machine alone (the bytecode, its checks, the
+// interpreter), with no Apple framework, so it builds and tests on Linux too.
+let package = Package(
+    name: "SwiftBrowserDSP",
+    platforms: [.iOS(.v17), .macOS(.v14)],
+    products: [
+        .library(name: "SwiftBrowserDSP", targets: ["SwiftBrowserDSP"]),
+        .library(name: "SwiftBrowserDSPCore", targets: ["SwiftBrowserDSPCore"]),
+    ],
+    dependencies: [
+        .package(path: "../.."),
+    ],
+    targets: [
+        .target(name: "SwiftBrowserDSPCore"),
+        .target(
+            name: "SwiftBrowserDSP",
+            dependencies: ["SwiftBrowserDSPCore", .product(name: "SwiftBrowserNative", package: "swift")]
+        ),
+    ]
+)

package/swift/Packages/SwiftBrowserDSP/README.md

@@ -0,0 +1,42 @@
+# SwiftBrowserDSP
+
+The native side of [@swiftbrowser/dsp](../../../packages/dsp): a program in
+the DSP language, compiled to bytecode by the package's TypeScript compiler,
+rendered on the audio thread by `DSPMachine` in an `AVAudioSourceNode` of
+the app's `AVAudioEngine`. The JavaScript attaches it as a `DSPNode` and
+plays notes on it.
+
+It is a package of its own, as SwiftBrowserLocalModel is, so only an app
+that synthesizes sound links it. @swiftbrowser/native ships it beside the
+core (`node_modules/@swiftbrowser/native/swift/Packages/SwiftBrowserDSP`; it
+depends on `SwiftBrowserNative` at `../..`, here and there). Link it into the
+app's Xcode project with `npx swiftbrowser native add dsp` (or `native init
+--with dsp`) and install it at launch:
+
+```swift
+import SwiftBrowserDSP
+
+init() {
+    SwiftBrowserDSP.install()
+}
+```
+
+That registers the `program` node kind with SwiftBrowserNative's audio
+engine (`AudioEngineNodes`). Without it, attaching a `DSPNode` fails
+`unsupported: no node kind program (the app installs SwiftBrowserDSP for it)`.
+
+- **SwiftBrowserDSPCore** is the machine alone: the program's checks
+  (`DSPProgram`, every register, operand and instance in range), and the
+  interpreter, 16 lanes a register, a lane per voice, in blocks of 128
+  frames. No Apple framework, so it builds and tests on Linux.
+- **SwiftBrowserDSP** is the node: the program, parameters and notes the
+  app sends are handed to the audio thread through a lock the audio thread
+  only ever tries (`withLockIfAvailable`), applied at its next render; a
+  replaced machine is released on the main thread. Nothing on the audio
+  thread allocates, locks or calls Objective-C.
+
+The tests (`swift test`) render @swiftbrowser/dsp's parity fixtures and
+compare them with the JavaScript machine's samples, check the program
+validation, and on macOS play programs through a real `AVAudioEngine`
+rendering offline. `swift test -c release -Xswiftc -enable-testing --filter
+sixteenVoices` prints the machine's speed.

package/swift/Packages/SwiftBrowserDSP/Sources/SwiftBrowserDSP/ProgramNode.swift

@@ -0,0 +1,203 @@
+#if canImport(AVFoundation) && canImport(SwiftUI)
+import AVFoundation
+import os
+@_exported import SwiftBrowserDSPCore
+import SwiftBrowserNative
+import SwiftBrowserProtocol
+
+/// The native side of @swiftbrowser/dsp: installs the `program` node kind
+/// for the app's `AVAudioEngine`, so a `DSPNode` the JavaScript attaches is
+/// rendered here, by `DSPMachine`, on the audio thread.
+///
+/// ```swift
+/// import SwiftBrowserDSP
+///
+/// init() {
+///     SwiftBrowserDSP.install()
+/// }
+/// ```
+@MainActor
+public enum SwiftBrowserDSP {
+    /// Registers the `program` kind with SwiftBrowserNative's audio engine; call at launch, before the bundle loads.
+    public static func install() {
+        AudioEngineNodes.register("program") { context in ProgramNode(format: context.format) }
+    }
+}
+
+/// A program as a node: an `AVAudioSourceNode` whose render block runs the
+/// machine. What the app sends (a program, its parameters, notes) is handed
+/// to the audio thread through `Renderer` without a lock the audio thread
+/// waits on, and applies at its next render.
+@MainActor
+final class ProgramNode: AudioEngineCustomNode {
+    let node: AVAudioNode
+    private let renderer: Renderer
+    private let sampleRate: Double
+    /// The program the audio thread runs (or is about to), for parameter names.
+    private var program: DSPProgram?
+    /// Every parameter the app set, by name: a program loaded later takes the values it has names for.
+    private var values: [String: Float] = [:]
+
+    init(format: AVAudioFormat) {
+        sampleRate = format.sampleRate
+        renderer = Renderer()
+        node = Self.source(renderer, format: format)
+    }
+
+    /// Made off the main actor: the render block runs on the audio thread.
+    nonisolated private static func source(_ renderer: Renderer, format: AVAudioFormat) -> AVAudioSourceNode {
+        AVAudioSourceNode(format: format) { silence, _, frames, list in
+            let buffers = UnsafeMutableAudioBufferListPointer(list)
+            guard let first = buffers.first?.mData?.assumingMemoryBound(to: Float.self) else { return noErr }
+            let count = Int(frames)
+            if !renderer.render(into: first, frames: count) {
+                silence.pointee = true
+                for buffer in buffers { buffer.mData?.initializeMemory(as: Float.self, repeating: 0, count: count) }
+                return noErr
+            }
+            for buffer in buffers.dropFirst() {
+                buffer.mData?.assumingMemoryBound(to: Float.self).update(from: first, count: count)
+            }
+            return noErr
+        }
+    }
+
+    func set(_ key: String, _ value: JSONValue) throws -> Bool {
+        defer { renderer.collect() }
+        if key == "program" {
+            guard let json = value.string else { throw DSPProgramError("a program crosses as JSON") }
+            let program = try DSPProgram(json: json)
+            // Made and given its parameters here, then handed over whole.
+            let machine = DSPMachine(program: program, sampleRate: sampleRate)
+            for (name, value) in values {
+                if let index = machine.parameterIndex(name) { machine.setParameter(at: index, value) }
+            }
+            self.program = program
+            renderer.replace(machine)
+            return true
+        }
+        if key.hasPrefix("param.") {
+            guard let number = value.double else { throw ProgramNodeError("unsupported: \(key) takes a number") }
+            let name = String(key.dropFirst(6))
+            values[name] = Float(number)
+            if let index = program?.parameters.firstIndex(where: { $0.name == name }) {
+                renderer.send(.parameter(Int32(index), Float(number)))
+            }
+            return true
+        }
+        return false
+    }
+
+    func note(_ action: String, _ params: JSONObject) throws -> Bool {
+        defer { renderer.collect() }
+        let int = { (key: String) in Int32(truncatingIfNeeded: params.int(key) ?? 0) }
+        switch action {
+        case "startNote": renderer.send(.noteOn(int("note"), int("velocity")))
+        case "stopNote": renderer.send(.noteOff(int("note")))
+        case "controller": renderer.send(.controller(int("controller"), int("value")))
+        case "pitchBend": renderer.send(.pitchBend(Int32(truncatingIfNeeded: params.int("value") ?? 8192)))
+        default: return false
+        }
+        return true
+    }
+
+    func halt() {
+        renderer.send(.reset)
+        renderer.collect()
+    }
+}
+
+/// A failure the app reads as `kind: detail`.
+struct ProgramNodeError: Error, CustomStringConvertible {
+    let description: String
+    init(_ description: String) { self.description = description }
+}
+
+/// What the main actor sends the audio thread.
+enum ProgramEvent: Sendable {
+    case noteOn(Int32, Int32)
+    case noteOff(Int32)
+    case controller(Int32, Int32)
+    case pitchBend(Int32)
+    case parameter(Int32, Float)
+    case reset
+}
+
+/// The machine and what is sent to it, between the main actor and the audio
+/// thread. The audio thread only ever tries the lock (`withLockIfAvailable`):
+/// when the main actor holds it, this render goes on with the events it has
+/// and takes the rest at the next. A machine the audio thread lets go of
+/// waits in `retired` for the main actor to release, so nothing is freed on
+/// the audio thread.
+final class Renderer: @unchecked Sendable {
+    private struct Shared {
+        var events: [ProgramEvent] = []
+        var machine: DSPMachine?
+        var retired: DSPMachine?
+    }
+
+    private let lock = OSAllocatedUnfairLock(initialState: Shared())
+    // The audio thread's own.
+    private var machine: DSPMachine?
+    private var inbox: [ProgramEvent] = []
+
+    init() {
+        inbox.reserveCapacity(256)
+        lock.withLockUnchecked { $0.events.reserveCapacity(256) }
+    }
+
+    func send(_ event: ProgramEvent) {
+        lock.withLockUnchecked { $0.events.append(event) }
+    }
+
+    func replace(_ machine: DSPMachine) {
+        // A machine that never reached the audio thread is released here, after the lock.
+        let superseded = lock.withLockUnchecked { shared -> DSPMachine? in
+            defer {
+                shared.machine = machine
+                shared.events.removeAll(keepingCapacity: true)
+            }
+            return shared.machine
+        }
+        _ = superseded
+    }
+
+    /// Releases, on the caller's thread, a machine the audio thread let go of.
+    func collect() {
+        let retired = lock.withLockUnchecked { shared -> DSPMachine? in
+            defer { shared.retired = nil }
+            return shared.retired
+        }
+        _ = retired
+    }
+
+    /// The audio thread: false when there is no program yet (silence).
+    func render(into out: UnsafeMutablePointer<Float>, frames: Int) -> Bool {
+        _ = lock.withLockIfAvailableUnchecked { shared in
+            if let next = shared.machine, shared.retired == nil {
+                shared.retired = machine
+                machine = next
+                shared.machine = nil
+            }
+            swap(&shared.events, &inbox)
+        }
+        guard let machine else {
+            inbox.removeAll(keepingCapacity: true)
+            return false
+        }
+        for event in inbox {
+            switch event {
+            case let .noteOn(note, velocity): machine.noteOn(note, velocity: velocity)
+            case let .noteOff(note): machine.noteOff(note)
+            case let .controller(number, value): machine.controller(number, value: value)
+            case let .pitchBend(value): machine.pitchBend(value)
+            case let .parameter(index, value): machine.setParameter(at: Int(index), value)
+            case .reset: machine.reset()
+            }
+        }
+        inbox.removeAll(keepingCapacity: true)
+        machine.render(into: out, frames: frames)
+        return true
+    }
+}
+#endif