@swiftbrowser/native 0.18.3 → 0.21.0

package/swift/Sources/SwiftBrowserNative/Host/LanguageModelProvider.swift

@@ -0,0 +1,195 @@
+#if canImport(SwiftUI) && !canImport(FoundationBridge)
+import Foundation
+import SwiftBrowserProtocol
+
+/// A language model an app brings for `LanguageModelSession`
+/// (LanguageModelBridge.swift) where Apple Intelligence cannot answer: set
+/// `LanguageModels.fallback` at launch, and the app's sessions use it
+/// whenever the system model is unavailable for a reason other than still
+/// downloading. SwiftBrowserLocalModel (Native/Packages) is one, running an
+/// open model with llama.cpp; an app that adds no such package pays nothing
+/// for this hook.
+@MainActor
+public protocol LanguageModelProvider: AnyObject {
+    /// Whether it can answer now, asked each time the app asks. A provider
+    /// whose model is not on the device yet answers `.unavailable(.modelNotReady)`
+    /// and says how its download stands in `downloadState`.
+    func availability() async -> LanguageModelAvailability
+
+    /// For a model the app fetches: its size, whether it is on the device,
+    /// and how far a download under way has come. Nil (the default) for a
+    /// model that is simply there.
+    var downloadState: LanguageModelDownload? { get }
+
+    /// Starts fetching the model, which the viewer asked for (the app shows
+    /// the size first). Nothing for a model without a download.
+    func startDownload()
+
+    /// Stops a download under way and removes the model from the device,
+    /// freeing its space; it can be fetched again.
+    func deleteDownload() async throws
+
+    /// A conversation with `instructions` followed throughout, and `tools`
+    /// the model may call while it answers. Making one should be cheap: load
+    /// what it needs on `prewarm` or the first prompt.
+    func makeSession(instructions: String?, tools: [LanguageModelTool]) throws -> any LanguageModelProviderSession
+
+    /// The models it can run, for the app to offer: empty (the default) for
+    /// a provider of one model.
+    var choices: [LanguageModelChoice] { get }
+
+    /// The choice that answers, by its `id`.
+    var selectedChoice: String? { get }
+
+    /// Makes the choice with this `id` the one that answers, from the next session on.
+    func select(_ id: String)
+}
+
+/// One conversation: each prompt follows the ones before it and their responses.
+@MainActor
+public protocol LanguageModelProviderSession: AnyObject {
+    /// Loads what the first response needs, ahead of it.
+    func prewarm()
+
+    /// The response to `prompt`, each element the whole response so far.
+    /// Ending the iteration (or cancelling the task iterating it) stops the
+    /// model. Failures are best thrown as `LanguageModelError`s, whose kinds
+    /// the app reads as `GenerationError` cases.
+    func streamResponse(to prompt: String, options: LanguageModelOptions) -> AsyncThrowingStream<String, Error>
+
+    /// The app is done with the conversation.
+    func close()
+}
+
+@MainActor
+extension LanguageModelProvider {
+    /// A conversation without tools.
+    public func makeSession(instructions: String?) throws -> any LanguageModelProviderSession {
+        try makeSession(instructions: instructions, tools: [])
+    }
+
+    public var choices: [LanguageModelChoice] { [] }
+    public var selectedChoice: String? { nil }
+    public func select(_ id: String) {}
+    public var downloadState: LanguageModelDownload? { nil }
+    public func startDownload() {}
+    public func deleteDownload() async throws {}
+}
+
+/// A function the model may call while it answers (FoundationModels' `Tool`):
+/// its name and what it does, its arguments as a JSON schema (an object of
+/// properties; strings, numbers, integers, booleans, arrays and enums), and
+/// `call`, which runs it (in the app) and answers what the model reads.
+public struct LanguageModelTool: Sendable {
+    public let name: String
+    public let description: String
+    public let parameters: JSONValue
+    public let call: @Sendable (JSONValue) async throws -> String
+
+    public init(name: String, description: String, parameters: JSONValue, call: @escaping @Sendable (JSONValue) async throws -> String) {
+        self.name = name
+        self.description = description
+        self.parameters = parameters
+        self.call = call
+    }
+}
+
+/// A model the app can offer to choose (`LanguageModelProvider.choices`).
+public struct LanguageModelChoice: Sendable, Equatable {
+    public var id: String
+    public var name: String
+    /// A line about it: what it is good at.
+    public var summary: String?
+    /// Where its download stands, for a model the app fetches.
+    public var download: LanguageModelDownload?
+
+    public init(id: String, name: String, summary: String? = nil, download: LanguageModelDownload? = nil) {
+        self.id = id
+        self.name = name
+        self.summary = summary
+        self.download = download
+    }
+}
+
+/// Where a model the app fetches stands.
+public struct LanguageModelDownload: Sendable, Equatable {
+    /// The file's size in bytes, where known.
+    public var size: Int64?
+    /// Whether the model is on the device.
+    public var isDownloaded: Bool
+    /// From 0 to 1 while a download is under way; nil when none is.
+    public var progress: Double?
+    /// Why the last download failed, until the next one starts.
+    public var failure: String?
+
+    public init(size: Int64? = nil, isDownloaded: Bool, progress: Double? = nil, failure: String? = nil) {
+        self.size = size
+        self.isDownloaded = isDownloaded
+        self.progress = progress
+        self.failure = failure
+    }
+}
+
+/// Whether a model can answer, as `SystemLanguageModel.Availability`.
+public enum LanguageModelAvailability: Sendable, Equatable {
+    case available
+    case unavailable(Reason)
+
+    public enum Reason: String, Sendable {
+        case deviceNotEligible
+        case appleIntelligenceNotEnabled
+        case modelNotReady
+    }
+}
+
+/// `GenerationOptions`, as the app sends them.
+public struct LanguageModelOptions: Sendable, Equatable {
+    public var temperature: Double?
+    public var maximumResponseTokens: Int?
+    /// Always the likeliest token, rather than sampling.
+    public var greedy: Bool
+
+    public init(temperature: Double? = nil, maximumResponseTokens: Int? = nil, greedy: Bool = false) {
+        self.temperature = temperature
+        self.maximumResponseTokens = maximumResponseTokens
+        self.greedy = greedy
+    }
+}
+
+/// A failed response, by the `GenerationError` case the app reads it as.
+public struct LanguageModelError: Error, CustomStringConvertible, Sendable {
+    public enum Kind: String, Sendable {
+        /// `assetsUnavailable`
+        case unavailable
+        /// `exceededContextWindowSize`
+        case context
+        /// `guardrailViolation`
+        case guardrail
+        /// `unsupportedLanguageOrLocale`
+        case language
+        /// `concurrentRequests`
+        case concurrent
+        case rateLimited
+        case refusal
+    }
+
+    public let kind: Kind
+    public let message: String
+
+    public init(_ kind: Kind, _ message: String) {
+        self.kind = kind
+        self.message = message
+    }
+
+    /// The tagged message the app reads the case from (`context: …`).
+    public var description: String { "\(kind.rawValue): \(message)" }
+}
+
+/// The models `LanguageModelSession` can use besides Apple Intelligence's.
+public enum LanguageModels {
+    /// Used where the system model is unavailable for a reason other than
+    /// still downloading (an ineligible device, Apple Intelligence off, a
+    /// system before iOS 26). Set it at launch, before the app asks.
+    @MainActor public static var fallback: (any LanguageModelProvider)?
+}
+#endif

package/swift/Sources/SwiftBrowserNative/Host/WebSocketBridge.swift

@@ -10,22 +10,26 @@ import FoundationBridge
 /// first ping is answered (the handshake is done by then), `wsSend` sends
 /// text or bytes, `wsClose` closes it. What a socket receives, and how it
 /// ends, goes to the app as `__sbWebSocketEvent(id, json)`: `open`,
-/// `message` (`text` or base64 `binary`), `error` and `close` (`code`,
-/// `reason`, `wasClean`). Against the shim (the round trip) the task is the
+/// `message` (`text` or `binary`), `error` and `close` (`code`, `reason`,
+/// `wasClean`). Binary crosses as base64, or, for a socket the app opened
+/// saying it reads bytes, as a `Uint8Array` after the JSON, which names it
+/// `{ "$bytes": 0 }` (docs/notes/bytes.md); `wsSend` takes it either way. Against the shim (the round trip) the task is the
 /// shim's, over the page's WebSocket.
 @MainActor
 final class WebSocketBridge {
     private let session: URLSession
-    private let send: (Int, JSONValue) -> Void
+    private let send: (Int, JSONValue, [HostBytes]) -> Void
     private var tasks: [Int: URLSessionWebSocketTask] = [:]
+    /// The sockets whose app reads binary messages as bytes.
+    private var bytesSockets: Set<Int> = []
 
-    /// `send` hands an event for socket `id` to the app.
-    init(session: URLSession, send: @escaping (Int, JSONValue) -> Void) {
+    /// `send` hands an event for socket `id` to the app, with the bytes its JSON names.
+    init(session: URLSession, send: @escaping (Int, JSONValue, [HostBytes]) -> Void) {
         self.session = session
         self.send = send
     }
 
-    func open(id: Int, url string: String, protocols: [String]) {
+    func open(id: Int, url string: String, protocols: [String], bytes: Bool = false) {
         guard let url = URL(string: string) else {
             emit(id, ["type": .string("error"), "message": .string("invalid URL")])
             emit(id, ["type": .string("close"), "code": .number(1006), "reason": .string(""), "wasClean": .bool(false)])
@@ -33,6 +37,7 @@ final class WebSocketBridge {
         }
         let task = protocols.isEmpty ? session.webSocketTask(with: url) : session.webSocketTask(with: url, protocols: protocols)
         tasks[id] = task
+        if bytes { bytesSockets.insert(id) }
         task.resume()
         task.sendPing { [weak self] error in
             Task { @MainActor [weak self] in
@@ -47,13 +52,13 @@ final class WebSocketBridge {
         }
     }
 
-    /// `{ text }` or `{ binary }` (base64).
-    func send(id: Int, json: String) {
+    /// `{ text }` or `{ binary }`: base64, or `{ "$bytes": 0 }` and the view in `bytes`.
+    func send(id: Int, json: String, bytes: [HostBytes] = []) {
         guard let task = tasks[id], let fields = (try? JSONValue.parse(json))?.object else { return }
         let message: URLSessionWebSocketTask.Message
         if let text = fields["text"]?.string {
             message = .string(text)
-        } else if let encoded = fields["binary"]?.string, let data = Data(base64Encoded: encoded) {
+        } else if let data = fields["binary"]?.bytes(from: bytes) {
             message = .data(data)
         } else {
             return
@@ -69,6 +74,7 @@ final class WebSocketBridge {
 
     func close(id: Int, code: Int, reason: String) {
         guard let task = tasks.removeValue(forKey: id) else { return }
+        defer { bytesSockets.remove(id) }
         task.cancel(with: URLSessionWebSocketTask.CloseCode(rawValue: code) ?? .normalClosure, reason: reason.isEmpty ? nil : Data(reason.utf8))
         emit(id, ["type": .string("close"), "code": .number(Double(code)), "reason": .string(reason), "wasClean": .bool(true)])
     }
@@ -77,6 +83,7 @@ final class WebSocketBridge {
     func invalidate() {
         for task in tasks.values { task.cancel(with: .goingAway, reason: nil) }
         tasks.removeAll()
+        bytesSockets.removeAll()
     }
 
     private func receive(id: Int, task: URLSessionWebSocketTask) {
@@ -87,7 +94,12 @@ final class WebSocketBridge {
                 case .success(let message):
                     switch message {
                     case .string(let text): self.emit(id, ["type": .string("message"), "text": .string(text)])
-                    case .data(let data): self.emit(id, ["type": .string("message"), "binary": .string(data.base64EncodedString())])
+                    case .data(let data):
+                        if self.bytesSockets.contains(id) {
+                            self.send(id, .object(["type": .string("message"), "binary": .object(["$bytes": .number(0)])]), [HostBytes(data, kind: .uint8)])
+                        } else {
+                            self.emit(id, ["type": .string("message"), "binary": .string(data.base64EncodedString())])
+                        }
                     @unknown default: break
                     }
                     self.receive(id: id, task: task)
@@ -101,6 +113,7 @@ final class WebSocketBridge {
     /// The socket ended: a close the server sent (its code and reason), else an error and an unclean close.
     private func fail(id: Int, task: URLSessionWebSocketTask, error: any Error) {
         tasks[id] = nil
+        bytesSockets.remove(id)
         let code = task.closeCode
         if code != .invalid {
             let reason = task.closeReason.map { String(decoding: $0, as: UTF8.self) } ?? ""
@@ -112,7 +125,7 @@ final class WebSocketBridge {
     }
 
     private func emit(_ id: Int, _ fields: JSONObject) {
-        send(id, .object(fields))
+        send(id, .object(fields), [])
     }
 }
 #endif

package/swift/Sources/SwiftBrowserNative/Host/WorkerHost.swift

@@ -62,7 +62,8 @@ final class WorkerHost: @unchecked Sendable {
     // MARK: On the worker's thread
 
     private func setUp() {
-        let context = JSContext(virtualMachine: JSVirtualMachine())!
+        // Its own virtual machine, made one at a time with the app's (JavaScriptCoreHost.swift).
+        let context = JavaScriptCoreHost.withVirtualMachineLock { JSContext(virtualMachine: JSVirtualMachine())! }
         context.name = "SwiftBrowser worker \(name)"
         context.exceptionHandler = { [weak self] _, exception in
             guard let self else { return }
@@ -85,7 +86,7 @@ final class WorkerHost: @unchecked Sendable {
         for timer in timers.values { timer.invalidate() }
         timers.removeAll()
         context?.exceptionHandler = nil
-        context = nil
+        JavaScriptCoreHost.withVirtualMachineLock { context = nil }
     }
 
     private func install(_ native: JSValue, _ name: String, _ body: @escaping ([JSONValue]) -> JSONValue?) {

package/swift/Sources/SwiftBrowserNative/Runtime/NativeFunctions.swift

@@ -35,7 +35,7 @@ public struct NativeFunctionError: Error, CustomStringConvertible {
 }
 
 /// Swift the app's JavaScript can call: the function half of the escape hatch
-/// (docs/native.md; `NativeRegistry` is the view half). The Swift app
+/// (docs/native.md; `ViewRegistry` is the view half). The Swift app
 /// registers functions once, before the app loads, and `@swiftbrowser/native`
 /// calls them: `callNative(name, params)` resolves with what an async function
 /// returns, through the host request `app:<name>` (HostRequestBridge.swift);

package/swift/Sources/SwiftBrowserNative/Runtime/ViewRegistry.swift

@@ -0,0 +1,57 @@
+#if canImport(SwiftUI)
+import Foundation
+import SwiftBrowserProtocol
+import SwiftBrowserTree
+import SwiftUI
+
+/// A view a package or the app draws itself, by name: the `registered`
+/// element kind (docs/native.md), which `RegisteredView` in
+/// @swiftbrowser/react emits. A page registers its own element under the
+/// same name (@swiftbrowser/web's `ViewRegistry`), so one component is a
+/// SwiftUI view here and the package's DOM there, drawing what never crosses
+/// the bridge (an audio node's frames, a map's tiles).
+///
+/// ```swift
+/// ViewRegistry.register("MapView") { context in
+///     MapView(region: context.props["region"]) { context.send(["tapped": true]) }
+/// }
+/// ```
+///
+/// ```tsx
+/// <RegisteredView name="MapView" props={{ region }} onEvent={(value) => …} />
+/// ```
+@MainActor
+public enum ViewRegistry {
+    /// What a registered view gets: the element's `props` as the app wrote
+    /// them, its children (SwiftUI views the app put inside it) and a way to
+    /// send a value back, which reaches the element's `onEvent` in JavaScript.
+    public struct Context {
+        public let id: Int
+        public let props: JSONValue
+        public let children: [ElementNode]
+        public let send: (JSONValue) -> Void
+
+        /// The children as a view (built on the main actor, as views are).
+        @MainActor public var content: some View {
+            ForEach(children) { ElementView(node: $0) }
+        }
+    }
+
+    public typealias Builder = (Context) -> AnyView
+    private static var builders: [String: Builder] = [:]
+
+    public static func register<V: View>(_ name: String, @ViewBuilder _ builder: @escaping (Context) -> V) {
+        builders[name] = { AnyView(builder($0)) }
+    }
+
+    public static func unregister(_ name: String) {
+        builders[name] = nil
+    }
+
+    public static func isRegistered(_ name: String) -> Bool { builders[name] != nil }
+
+    static func view(name: String, context: Context) -> AnyView? {
+        builders[name]?(context)
+    }
+}
+#endif

package/swift/Sources/SwiftBrowserNative/Views/ElementView.swift

@@ -61,6 +61,7 @@ struct ElementView: View {
         // Lists
         case "list": return AnyView(ListElement(node: node))
         case "section": return AnyView(SectionElement(node: node))
+        case "disclosuregroup": return AnyView(DisclosureGroupElement(node: node))
         case "listrow": return AnyView(ListRowElement(node: node))
         case "swipeactions": return AnyView(SwipeActionsElement(node: node))
         // Navigation and presentation
@@ -88,8 +89,12 @@ struct ElementView: View {
         // Shapes and colors
         case "color": return AnyView(ColorElement(node: node))
         case "shape": return ShapeElement(node: node).reportingSize(of: node)
+        case "canvas": return AnyView(CanvasElement(node: node))
+        case "layout": return AnyView(LayoutElement(node: node))
+        case "draggable": return AnyView(DraggableElement(node: node))
+        case "dropdestination": return AnyView(DropDestinationElement(node: node))
         // The escape hatch
-        case "native": return AnyView(NativeElement(node: node))
+        case "registered": return AnyView(RegisteredElement(node: node))
         default: return AnyView(UnsupportedElement(node: node))
         }
     }
@@ -130,18 +135,18 @@ struct UnsupportedElement: View {
     }
 }
 
-/// The `native` kind: a view the Swift app registered under `name`.
-struct NativeElement: View {
+/// The `registered` kind: a view registered under `name` (`ViewRegistry`).
+struct RegisteredElement: View {
     let node: ElementNode
     @Environment(\.nativeApp) private var app
 
     var body: some View {
-        let props = NativeProps(props: node.props)
+        let props = RegisteredProps(props: node.props)
         let app = app
-        let context = NativeViewContext(id: node.id, props: props.props, children: node.children) { value in
-            app?.dispatch(.native(node.id, name: props.name, value: value))
+        let context = ViewRegistry.Context(id: node.id, props: props.props, children: node.children) { value in
+            app?.dispatch(.registered(node.id, name: props.name, value: value))
         }
-        if let view = NativeRegistry.view(name: props.name, context: context) {
+        if let view = ViewRegistry.view(name: props.name, context: context) {
             view
         } else {
             UnsupportedElement(node: node)

package/swift/Sources/SwiftBrowserNative/Views/Kinds/Controls.swift

@@ -527,7 +527,24 @@ struct SliderElement: View {
         let range = props.min...max(props.min, props.max)
         // The drag's start and end, with the value then (`onEditingChanged`).
         let editing = { (editing: Bool) -> Void in app?.dispatch(.slide(node.id, SliderProps(props: node.props).value, editing: editing)) }
-        if node.children.isEmpty {
+        // The children by what `labels` says each is; the value labels sit at
+        // the track's ends (the label itself iOS keeps for accessibility).
+        let part = { (name: String) -> ElementNode? in
+            guard let index = props.labels.firstIndex(of: name), node.children.indices.contains(index) else { return nil }
+            return node.children[index]
+        }
+        let low = part("min"), high = part("max")
+        if low != nil || high != nil {
+            let label = part("label")
+            let labelView = { label.map { AnyView(ElementView(node: $0)) } ?? AnyView(EmptyView()) }
+            let lowView = { low.map { AnyView(ElementView(node: $0)) } ?? AnyView(EmptyView()) }
+            let highView = { high.map { AnyView(ElementView(node: $0)) } ?? AnyView(EmptyView()) }
+            if let step = props.step, step > 0 {
+                Slider(value: value, in: range, step: step, label: labelView, minimumValueLabel: lowView, maximumValueLabel: highView, onEditingChanged: editing)
+            } else {
+                Slider(value: value, in: range, label: labelView, minimumValueLabel: lowView, maximumValueLabel: highView, onEditingChanged: editing)
+            }
+        } else if node.children.isEmpty {
             if let step = props.step, step > 0 {
                 Slider(value: value, in: range, step: step, onEditingChanged: editing)
             } else {