@swiftbrowser/native 0.18.3 → 0.21.0

package/src/webSocket.ts

@@ -9,9 +9,9 @@
  * `binaryType` (`blob` by default, or `arraybuffer`).
  */
 
-import { SimpleBlob } from './blob';
+import { blobBytes, LazyFile, SimpleBlob } from './blob';
 import type { NativeBridge } from './bridge';
-import { base64Decode, base64Encode } from './encoding';
+import { base64Encode, bridgedBytes } from './encoding';
 
 export interface WebSocketEventLike {
   type: string;
@@ -31,7 +31,10 @@ export interface NativeWebSocketEvent {
   protocol?: string;
   text?: string;
   /** Base64. */
-  binary?: string;
+  /** Base64, or `{ $bytes: 0 }` naming the `Uint8Array` after the event's JSON. */
+  binary?: string | { $bytes: number };
+  /** The views that came after the JSON. */
+  bytes?: ArrayBufferView[];
   message?: string;
   code?: number;
   reason?: string;
@@ -43,6 +46,8 @@ export type WebSocketBridge = Required<Pick<NativeBridge, 'wsOpen' | 'wsSend' |
 const sockets = new Map<number, NativeWebSocket>();
 let nextId = 1;
 let bridge: WebSocketBridge | null = null;
+/** Whether binary crosses as views beside the JSON (a host that carries bytes) rather than base64. */
+let carriesBytes = false;
 
 export class NativeWebSocket {
   static readonly CONNECTING = 0;
@@ -75,7 +80,9 @@ export class NativeWebSocket {
     this.url = href;
     this.id = nextId++;
     sockets.set(this.id, this);
-    bridge.wsOpen(this.id, href, typeof protocols === 'string' ? [protocols] : [...protocols]);
+    const list = typeof protocols === 'string' ? [protocols] : [...protocols];
+    if (carriesBytes) bridge.wsOpen(this.id, href, list, true);
+    else bridge.wsOpen(this.id, href, list);
   }
 
   send(data: string | ArrayBuffer | ArrayBufferView | SimpleBlob): void {
@@ -85,12 +92,28 @@ export class NativeWebSocket {
       bridge?.wsSend(this.id, JSON.stringify({ text: data }));
       return;
     }
-    const bytes = data instanceof SimpleBlob ? data.bytes() : data instanceof ArrayBuffer ? new Uint8Array(data) : new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
-    bridge?.wsSend(this.id, JSON.stringify({ binary: base64Encode(bytes) }));
+    if (data instanceof LazyFile) {
+      // A picked file on disk: read it, then send (the frame goes out when the read lands; order among lazy sends is the reads').
+      void blobBytes(data).then((bytes) => {
+        if (this.readyState !== NativeWebSocket.OPEN) return;
+        if (carriesBytes) bridge?.wsSend(this.id, JSON.stringify({ binary: { $bytes: 0 } }), bytes);
+        else bridge?.wsSend(this.id, JSON.stringify({ binary: base64Encode(bytes) }));
+      });
+      return;
+    }
+    const bytes =
+      data instanceof SimpleBlob
+        ? data.bytes()
+        : data instanceof ArrayBuffer
+          ? new Uint8Array(data)
+          : new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
+    if (carriesBytes) bridge?.wsSend(this.id, JSON.stringify({ binary: { $bytes: 0 } }), bytes);
+    else bridge?.wsSend(this.id, JSON.stringify({ binary: base64Encode(bytes) }));
   }
 
   close(code?: number, reason?: string): void {
-    if (code !== undefined && code !== 1000 && !(code >= 3000 && code <= 4999)) throw new DOMExceptionLike('InvalidAccessError', 'WebSocket: the close code must be 1000 or in 3000–4999');
+    if (code !== undefined && code !== 1000 && !(code >= 3000 && code <= 4999))
+      throw new DOMExceptionLike('InvalidAccessError', 'WebSocket: the close code must be 1000 or in 3000–4999');
     if (this.readyState === NativeWebSocket.CLOSING || this.readyState === NativeWebSocket.CLOSED) return;
     this.readyState = NativeWebSocket.CLOSING;
     bridge?.wsClose(this.id, code ?? 1000, reason ?? '');
@@ -129,7 +152,7 @@ export class NativeWebSocket {
         if (this.readyState !== NativeWebSocket.OPEN) return;
         let data: unknown = event.text ?? '';
         if (event.binary !== undefined) {
-          const bytes = base64Decode(event.binary);
+          const bytes = bridgedBytes(typeof event.binary === 'string' ? event.binary : event.bytes?.[event.binary.$bytes]);
           data = this.binaryType === 'arraybuffer' ? bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength) : new SimpleBlob([bytes]);
         }
         this.dispatchEvent({ type: 'message', target: this, data });
@@ -147,8 +170,8 @@ export class NativeWebSocket {
     }
   }
 
-  /** The bridge's `__sbWebSocketEvent(id, json)`. */
-  static receive(id: number, json: string | NativeWebSocketEvent): void {
+  /** The bridge's `__sbWebSocketEvent(id, json, ...bytes)`: a binary message's bytes after the JSON that names them. */
+  static receive(id: number, json: string | NativeWebSocketEvent, ...bytes: ArrayBufferView[]): void {
     const socket = sockets.get(id);
     if (!socket) return;
     let event: NativeWebSocketEvent;
@@ -157,12 +180,13 @@ export class NativeWebSocket {
     } catch {
       return;
     }
-    socket.receive(event);
+    socket.receive(bytes.length ? { ...event, bytes } : event);
   }
 
-  /** Points the sockets at the bridge (the runtime, once per install). */
-  static install(native: WebSocketBridge): void {
+  /** Points the sockets at the bridge (the runtime, once per install); `bytes`: the host carries typed arrays. */
+  static install(native: WebSocketBridge, options: { bytes?: boolean } = {}): void {
     bridge = native;
+    carriesBytes = options.bytes === true;
   }
 }
 

package/swift/Packages/SwiftBrowserDSP/Package.swift

@@ -0,0 +1,31 @@
+// swift-tools-version: 6.0
+import PackageDescription
+
+// SwiftBrowserDSP: the native side of @swiftbrowser/dsp. A program written
+// in the DSP language and compiled to bytecode by the package's TypeScript
+// compiler runs here, on the audio thread, in an `AVAudioSourceNode` of the
+// app's `AVAudioEngine`; JavaScript attaches it as a `DSPNode` and plays
+// notes on it. A package of its own, so only an app that synthesizes sound
+// links it; it installs itself as the `program` node kind
+// (`SwiftBrowserDSP.install()`, SwiftBrowserNative's AudioEngineNodes.swift).
+//
+// SwiftBrowserDSPCore is the machine alone (the bytecode, its checks, the
+// interpreter), with no Apple framework, so it builds and tests on Linux too.
+let package = Package(
+    name: "SwiftBrowserDSP",
+    platforms: [.iOS(.v17), .macOS(.v14)],
+    products: [
+        .library(name: "SwiftBrowserDSP", targets: ["SwiftBrowserDSP"]),
+        .library(name: "SwiftBrowserDSPCore", targets: ["SwiftBrowserDSPCore"]),
+    ],
+    dependencies: [
+        .package(path: "../.."),
+    ],
+    targets: [
+        .target(name: "SwiftBrowserDSPCore"),
+        .target(
+            name: "SwiftBrowserDSP",
+            dependencies: ["SwiftBrowserDSPCore", .product(name: "SwiftBrowserNative", package: "swift")]
+        ),
+    ]
+)

package/swift/Packages/SwiftBrowserDSP/README.md

@@ -0,0 +1,42 @@
+# SwiftBrowserDSP
+
+The native side of [@swiftbrowser/dsp](../../../packages/dsp): a program in
+the DSP language, compiled to bytecode by the package's TypeScript compiler,
+rendered on the audio thread by `DSPMachine` in an `AVAudioSourceNode` of
+the app's `AVAudioEngine`. The JavaScript attaches it as a `DSPNode` and
+plays notes on it.
+
+It is a package of its own, as SwiftBrowserLocalModel is, so only an app
+that synthesizes sound links it. @swiftbrowser/native ships it beside the
+core (`node_modules/@swiftbrowser/native/swift/Packages/SwiftBrowserDSP`; it
+depends on `SwiftBrowserNative` at `../..`, here and there). Link it into the
+app's Xcode project with `npx swiftbrowser native add dsp` (or `native init
+--with dsp`) and install it at launch:
+
+```swift
+import SwiftBrowserDSP
+
+init() {
+    SwiftBrowserDSP.install()
+}
+```
+
+That registers the `program` node kind with SwiftBrowserNative's audio
+engine (`AudioEngineNodes`). Without it, attaching a `DSPNode` fails
+`unsupported: no node kind program (the app installs SwiftBrowserDSP for it)`.
+
+- **SwiftBrowserDSPCore** is the machine alone: the program's checks
+  (`DSPProgram`, every register, operand and instance in range), and the
+  interpreter, 16 lanes a register, a lane per voice, in blocks of 128
+  frames. No Apple framework, so it builds and tests on Linux.
+- **SwiftBrowserDSP** is the node: the program, parameters and notes the
+  app sends are handed to the audio thread through a lock the audio thread
+  only ever tries (`withLockIfAvailable`), applied at its next render; a
+  replaced machine is released on the main thread. Nothing on the audio
+  thread allocates, locks or calls Objective-C.
+
+The tests (`swift test`) render @swiftbrowser/dsp's parity fixtures and
+compare them with the JavaScript machine's samples, check the program
+validation, and on macOS play programs through a real `AVAudioEngine`
+rendering offline. `swift test -c release -Xswiftc -enable-testing --filter
+sixteenVoices` prints the machine's speed.

package/swift/Packages/SwiftBrowserDSP/Sources/SwiftBrowserDSP/ProgramNode.swift

@@ -0,0 +1,306 @@
+#if canImport(AVFoundation) && canImport(SwiftUI)
+import AVFoundation
+import os
+@_exported import SwiftBrowserDSPCore
+import SwiftBrowserNative
+import SwiftBrowserProtocol
+
+/// The native side of @swiftbrowser/dsp: installs the `program` node kind
+/// for the app's `AVAudioEngine`, so a `DSPNode` the JavaScript attaches is
+/// rendered here, by `DSPMachine`, on the audio thread.
+///
+/// ```swift
+/// import SwiftBrowserDSP
+///
+/// init() {
+///     SwiftBrowserDSP.install()
+/// }
+/// ```
+@MainActor
+public enum SwiftBrowserDSP {
+    /// Registers the `program` kind with SwiftBrowserNative's audio engine, and `ScopeView`'s SwiftUI view; call at
+    /// launch, before the bundle loads.
+    public static func install() {
+        AudioEngineNodes.register("program") { context in ProgramNode(format: context.format) }
+        ViewRegistry.register("SwiftBrowserDSP.scope") { context in DSPScopeView(props: context.props) }
+    }
+}
+
+/// A program as a node: an `AVAudioSourceNode` whose render block runs the
+/// machine. What the app sends (a program, its parameters, notes) is handed
+/// to the audio thread through `Renderer` without a lock the audio thread
+/// waits on, and applies at its next render.
+@MainActor
+final class ProgramNode: AudioEngineCustomNode {
+    let node: AVAudioNode
+    private let renderer: Renderer
+    private let sampleRate: Double
+    /// The program the audio thread runs (or is about to), for parameter names.
+    private var program: DSPProgram?
+    /// Every parameter the app set, by name: a program loaded later takes the values it has names for.
+    private var values: [String: Float] = [:]
+    /// The transport's tempo as last set, for a program loaded later.
+    private var tempo: Double = 120
+
+    init(format: AVAudioFormat) {
+        sampleRate = format.sampleRate
+        renderer = Renderer()
+        node = Self.source(renderer, format: format)
+    }
+
+    /// Made off the main actor: the render block runs on the audio thread.
+    nonisolated private static func source(_ renderer: Renderer, format: AVAudioFormat) -> AVAudioSourceNode {
+        AVAudioSourceNode(format: format) { silence, _, frames, list in
+            let buffers = UnsafeMutableAudioBufferListPointer(list)
+            guard let first = buffers.first?.mData?.assumingMemoryBound(to: Float.self) else { return noErr }
+            let count = Int(frames)
+            // Two channels or more: the left side, then the right; one: their average.
+            let second = buffers.count > 1 ? buffers[1].mData?.assumingMemoryBound(to: Float.self) : nil
+            if !renderer.render(into: first, right: second, frames: count) {
+                silence.pointee = true
+                for buffer in buffers { buffer.mData?.initializeMemory(as: Float.self, repeating: 0, count: count) }
+                return noErr
+            }
+            for buffer in buffers.dropFirst(2) {
+                buffer.mData?.initializeMemory(as: Float.self, repeating: 0, count: count)
+            }
+            return noErr
+        }
+    }
+
+    func set(_ key: String, _ value: JSONValue) throws -> Bool {
+        defer { renderer.collect() }
+        if key == "view" {
+            // What a view finds the node's frames by.
+            guard let token = value.string else { throw ProgramNodeError("unsupported: view takes a string") }
+            ScopeTaps.register(token, renderer.tap)
+            return true
+        }
+        if key == "program" {
+            guard let json = value.string else { throw DSPProgramError("a program crosses as JSON") }
+            let program = try DSPProgram(json: json)
+            // Made and given its parameters here, then handed over whole.
+            let machine = DSPMachine(program: program, sampleRate: sampleRate)
+            for (name, value) in values {
+                if let index = machine.parameterIndex(name) { machine.setParameter(at: index, value) }
+            }
+            machine.setTempo(tempo)
+            self.program = program
+            renderer.replace(machine)
+            return true
+        }
+        if key == "tempo" || key == "beat" {
+            guard let number = value.double else { throw ProgramNodeError("unsupported: \(key) takes a number") }
+            if key == "tempo" {
+                tempo = number
+                renderer.send(.tempo(number))
+            } else {
+                renderer.send(.seek(number))
+            }
+            return true
+        }
+        if key.hasPrefix("param.") {
+            guard let number = value.double else { throw ProgramNodeError("unsupported: \(key) takes a number") }
+            let name = String(key.dropFirst(6))
+            values[name] = Float(number)
+            if let index = program?.parameters.firstIndex(where: { $0.name == name }) {
+                renderer.send(.parameter(Int32(index), Float(number)))
+            }
+            return true
+        }
+        return false
+    }
+
+    func note(_ action: String, _ params: JSONObject) throws -> Bool {
+        defer { renderer.collect() }
+        let int = { (key: String) in Int32(truncatingIfNeeded: params.int(key) ?? 0) }
+        // When on the program's transport, for a command that says.
+        let beat = params.double("beat")
+        switch action {
+        case "startNote": renderer.send(.noteOn(int("note"), int("velocity"), beat))
+        case "stopNote": renderer.send(.noteOff(int("note"), beat))
+        case "controller": renderer.send(.controller(int("controller"), int("value"), beat))
+        case "pitchBend": renderer.send(.pitchBend(Int32(truncatingIfNeeded: params.int("value") ?? 8192), beat))
+        default: return false
+        }
+        return true
+    }
+
+    func halt() {
+        renderer.send(.reset)
+        renderer.collect()
+    }
+}
+
+/// A failure the app reads as `kind: detail`.
+struct ProgramNodeError: Error, CustomStringConvertible {
+    let description: String
+    init(_ description: String) { self.description = description }
+}
+
+/// What the main actor sends the audio thread.
+enum ProgramEvent: Sendable {
+    case noteOn(Int32, Int32, Double?)
+    case noteOff(Int32, Double?)
+    case controller(Int32, Int32, Double?)
+    case pitchBend(Int32, Double?)
+    case parameter(Int32, Float)
+    case tempo(Double)
+    case seek(Double)
+    case reset
+}
+
+/// The machine and what is sent to it, between the main actor and the audio
+/// thread. The audio thread only ever tries the lock (`withLockIfAvailable`):
+/// when the main actor holds it, this render goes on with the events it has
+/// and takes the rest at the next. A machine the audio thread lets go of
+/// waits in `retired` for the main actor to release, so nothing is freed on
+/// the audio thread.
+final class Renderer: @unchecked Sendable {
+    private struct Shared {
+        var events: [ProgramEvent] = []
+        var machine: DSPMachine?
+        var retired: DSPMachine?
+    }
+
+    private let lock = OSAllocatedUnfairLock(initialState: Shared())
+    /// The frames rendered last, for views.
+    let tap = ScopeTap()
+    // The audio thread's own.
+    private var machine: DSPMachine?
+    private var inbox: [ProgramEvent] = []
+
+    init() {
+        inbox.reserveCapacity(256)
+        lock.withLockUnchecked { $0.events.reserveCapacity(256) }
+    }
+
+    func send(_ event: ProgramEvent) {
+        lock.withLockUnchecked { $0.events.append(event) }
+    }
+
+    func replace(_ machine: DSPMachine) {
+        // A machine that never reached the audio thread is released here, after the lock.
+        let superseded = lock.withLockUnchecked { shared -> DSPMachine? in
+            defer {
+                shared.machine = machine
+                shared.events.removeAll(keepingCapacity: true)
+            }
+            return shared.machine
+        }
+        _ = superseded
+    }
+
+    /// Releases, on the caller's thread, a machine the audio thread let go of.
+    func collect() {
+        let retired = lock.withLockUnchecked { shared -> DSPMachine? in
+            defer { shared.retired = nil }
+            return shared.retired
+        }
+        _ = retired
+    }
+
+    /// The audio thread: false when there is no program yet (silence). With `right`, the left side into `out`.
+    func render(into out: UnsafeMutablePointer<Float>, right: UnsafeMutablePointer<Float>? = nil, frames: Int) -> Bool {
+        _ = lock.withLockIfAvailableUnchecked { shared in
+            if let next = shared.machine, shared.retired == nil {
+                // The transport carries on through a new program.
+                if let machine { next.seek(toBeat: machine.position) }
+                shared.retired = machine
+                machine = next
+                shared.machine = nil
+            }
+            swap(&shared.events, &inbox)
+        }
+        guard let machine else {
+            inbox.removeAll(keepingCapacity: true)
+            return false
+        }
+        for event in inbox {
+            switch event {
+            case let .noteOn(note, velocity, beat): machine.noteOn(note, velocity: velocity, atBeat: beat)
+            case let .noteOff(note, beat): machine.noteOff(note, atBeat: beat)
+            case let .controller(number, value, beat): machine.controller(number, value: value, atBeat: beat)
+            case let .pitchBend(value, beat): machine.pitchBend(value, atBeat: beat)
+            case let .parameter(index, value): machine.setParameter(at: Int(index), value)
+            case let .tempo(bpm): machine.setTempo(bpm)
+            case let .seek(beat): machine.seek(toBeat: beat)
+            case .reset: machine.reset()
+            }
+        }
+        inbox.removeAll(keepingCapacity: true)
+        if let right {
+            machine.render(left: out, right: right, frames: frames)
+        } else {
+            machine.render(into: out, frames: frames)
+        }
+        tap.write(out, right, frames: frames)
+        return true
+    }
+}
+
+/// A node's last 2048 frames, its two sides mixed, for views (`DSPScopeView`): the audio thread writes them when the
+/// lock is free (never waiting, and a frame a view misses is only a frame), a view copies the newest out.
+final class ScopeTap: @unchecked Sendable {
+    static let capacity = 2048
+    /// Frames written so far; the samples are only touched under it.
+    private let lock = OSAllocatedUnfairLock(initialState: 0)
+    private let samples: UnsafeMutablePointer<Float>
+
+    init() {
+        samples = .allocate(capacity: Self.capacity)
+        samples.initialize(repeating: 0, count: Self.capacity)
+    }
+
+    deinit {
+        samples.deallocate()
+    }
+
+    /// The audio thread.
+    func write(_ left: UnsafePointer<Float>, _ right: UnsafePointer<Float>?, frames: Int) {
+        let samples = samples
+        _ = lock.withLockIfAvailableUnchecked { written in
+            let mask = Self.capacity - 1
+            if let right {
+                for i in 0..<frames { samples[(written + i) & mask] = (left[i] + right[i]) * 0.5 }
+            } else {
+                for i in 0..<frames { samples[(written + i) & mask] = left[i] }
+            }
+            written += frames
+        }
+    }
+
+    /// The newest `buffer.count` frames (at most 2048), oldest first.
+    func read(into buffer: inout [Float]) {
+        let count = Swift.min(buffer.count, Self.capacity)
+        let samples = samples
+        buffer.withUnsafeMutableBufferPointer { out in
+            lock.withLockUnchecked { written in
+                let mask = Self.capacity - 1
+                for i in 0..<count { out[i] = samples[(written - count + i) & mask] }
+            }
+        }
+    }
+}
+
+/// The program nodes' taps by view token, for views made before or after the node.
+enum ScopeTaps {
+    private struct Weak {
+        weak var tap: ScopeTap?
+    }
+
+    private static let lock = OSAllocatedUnfairLock(initialState: [String: Weak]())
+
+    static func register(_ token: String, _ tap: ScopeTap) {
+        lock.withLockUnchecked { taps in
+            taps = taps.filter { $0.value.tap != nil }
+            taps[token] = Weak(tap: tap)
+        }
+    }
+
+    static func tap(_ token: String) -> ScopeTap? {
+        lock.withLockUnchecked { $0[token]?.tap }
+    }
+}
+#endif
+

package/swift/Packages/SwiftBrowserDSP/Sources/SwiftBrowserDSP/ScopeView.swift

@@ -0,0 +1,79 @@
+#if canImport(AVFoundation) && canImport(SwiftUI)
+import SwiftBrowserProtocol
+import SwiftUI
+
+/// `ScopeView` (@swiftbrowser/dsp): a program node's output as a waveform,
+/// drawn every display frame from what the audio thread keeps for it
+/// (`ScopeTap`), its frames never crossing to the app's JavaScript. The
+/// trace starts at a rising zero crossing, so a steady tone stands still;
+/// it is the page's canvas (packages/dsp/src/scope.ts), step for step.
+struct DSPScopeView: View {
+    let token: String
+    let color: Color
+    let gain: Float
+    @State private var reader = ScopeReader()
+
+    init(props: JSONValue) {
+        token = props["view"].string ?? ""
+        gain = Float(props["gain"].double ?? 1)
+        color = props["color"].string.flatMap(Self.color(hex:)) ?? .accentColor
+    }
+
+    var body: some View {
+        TimelineView(.animation) { timeline in
+            // The tick's date is the trace's own input: SwiftUI draws a view again only when its inputs change, and
+            // a canvas that read nothing from the timeline was drawn once and left.
+            ScopeTrace(date: timeline.date, token: token, color: color, gain: gain, reader: reader)
+        }
+    }
+
+    /// The first rising zero crossing in the first half (scope.ts `traceStart`).
+    static func traceStart(_ samples: [Float], _ length: Int) -> Int {
+        let last = Swift.max(1, samples.count - length)
+        for i in 1..<last where samples[i - 1] < 0 && samples[i] >= 0 { return i }
+        return 0
+    }
+
+    /// `#rrggbb`.
+    static func color(hex: String) -> Color? {
+        let digits = hex.hasPrefix("#") ? String(hex.dropFirst()) : hex
+        guard digits.count == 6, let value = UInt32(digits, radix: 16) else { return nil }
+        return Color(red: Double((value >> 16) & 0xFF) / 255, green: Double((value >> 8) & 0xFF) / 255, blue: Double(value & 0xFF) / 255)
+    }
+}
+
+/// One display frame's trace: what the tap holds at `date`.
+private struct ScopeTrace: View {
+    let date: Date
+    let token: String
+    let color: Color
+    let gain: Float
+    let reader: ScopeReader
+
+    var body: some View {
+        Canvas { context, size in
+            let samples = reader.read(ScopeTaps.tap(token))
+            let length = samples.count / 2
+            let start = DSPScopeView.traceStart(samples, length)
+            var path = Path()
+            for i in 0..<length {
+                let x = CGFloat(i) / CGFloat(length - 1) * size.width
+                let value = Swift.max(-1, Swift.min(1, samples[start + i] * gain))
+                let y = size.height / 2 - CGFloat(value) * size.height / 2
+                if i == 0 { path.move(to: CGPoint(x: x, y: y)) } else { path.addLine(to: CGPoint(x: x, y: y)) }
+            }
+            context.stroke(path, with: .color(color), style: StrokeStyle(lineWidth: 1.5, lineJoin: .round))
+        }
+    }
+}
+
+/// The view's copy of the newest frames, made again each display frame.
+final class ScopeReader {
+    private var samples = [Float](repeating: 0, count: 1024)
+
+    func read(_ tap: ScopeTap?) -> [Float] {
+        tap?.read(into: &samples)
+        return samples
+    }
+}
+#endif