@swiftbrowser/native 0.18.2 → 0.19.0

package/swift/Packages/SwiftBrowserDSP/Sources/SwiftBrowserDSPCore/Program.swift

@@ -0,0 +1,188 @@
+import Foundation
+
+/// A compiled DSP program: the JSON @swiftbrowser/dsp's compiler writes
+/// (packages/dsp/src/program.ts, which documents the format). The machine
+/// keeps a register file of `registers` registers of 16 lanes, a lane per
+/// voice; `block` runs once per block of up to 128 frames and `sample` once
+/// per frame, each a flat list of instructions: an opcode, a destination
+/// register, the opcode's operands and, for a stateful builtin, its instance.
+public struct DSPProgram: Sendable, Equatable {
+    public struct Parameter: Sendable, Equatable {
+        public let name: String
+        public let register: Int
+        public let defaultValue: Float
+        public let min: Float
+        public let max: Float
+    }
+
+    public struct State: Sendable, Equatable {
+        public let name: String
+        public let register: Int
+        public let initial: Float
+    }
+
+    public let voices: Int
+    public let registers: Int
+    public let constants: [(register: Int, value: Float)]
+    public let parameters: [Parameter]
+    public let state: [State]
+    public let instances: [DSPBuiltin]
+    public let block: [Int32]
+    public let sample: [Int32]
+    public let output: Int
+
+    public static func == (a: DSPProgram, b: DSPProgram) -> Bool {
+        a.voices == b.voices && a.registers == b.registers && a.constants.map(\.register) == b.constants.map(\.register)
+            && a.constants.map(\.value) == b.constants.map(\.value) && a.parameters == b.parameters && a.state == b.state
+            && a.instances == b.instances && a.block == b.block && a.sample == b.sample && a.output == b.output
+    }
+
+    public static let format = "swiftbrowser-dsp"
+    public static let version = 1
+    public static let lanes = 16
+    public static let blockFrames = 128
+    /// The machine's own registers: `sampleRate`, then a voice's `freq`, `gate`, `velocity` and `note`.
+    public static let machineRegisters = 5
+
+    /// The program the JSON describes, checked: every register, operand and instance in range.
+    public init(json: String) throws {
+        guard let data = json.data(using: .utf8) else { throw DSPProgramError("not UTF-8") }
+        try self.init(data: data)
+    }
+
+    public init(data: Data) throws {
+        let parsed: Any
+        do {
+            parsed = try JSONSerialization.jsonObject(with: data)
+        } catch {
+            throw DSPProgramError("not JSON (\(error.localizedDescription))")
+        }
+        try self.init(object: parsed)
+    }
+
+    /// From a decoded JSON object (`JSONSerialization`'s).
+    public init(object: Any) throws {
+        guard let root = object as? [String: Any] else { throw DSPProgramError("not an object") }
+        guard root["format"] as? String == Self.format else { throw DSPProgramError("format is not \(Self.format)") }
+        guard Self.int(root["version"]) == Self.version else { throw DSPProgramError("version is not \(Self.version)") }
+        guard let voices = Self.int(root["voices"]), (1...Self.lanes).contains(voices) else { throw DSPProgramError("voices must be 1 … \(Self.lanes)") }
+        guard let registers = Self.int(root["registers"]), (Self.machineRegisters...65_536).contains(registers) else { throw DSPProgramError("registers out of range") }
+        func register(_ value: Any?, _ what: String) throws -> Int {
+            guard let r = Self.int(value), r >= 0, r < registers else { throw DSPProgramError("\(what): register out of range") }
+            return r
+        }
+        func number(_ value: Any?, _ what: String) throws -> Float {
+            guard let n = (value as? NSNumber)?.doubleValue, n.isFinite || n.isInfinite else { throw DSPProgramError("\(what) must be a number") }
+            return Float(n)
+        }
+        guard let instanceNames = root["instances"] as? [String] else { throw DSPProgramError("instances must be an array of names") }
+        let instances = try instanceNames.map { name in
+            guard let builtin = DSPBuiltin(rawValue: name) else { throw DSPProgramError("unknown builtin \(name)") }
+            return builtin
+        }
+        guard let constants = root["constants"] as? [[Any]] else { throw DSPProgramError("constants missing") }
+        let parsedConstants: [(register: Int, value: Float)] = try constants.map { entry in
+            guard entry.count == 2 else { throw DSPProgramError("a constant is not [register, value]") }
+            return (try register(entry[0], "constant"), try number(entry[1], "a constant"))
+        }
+        guard let parameters = root["parameters"] as? [[String: Any]] else { throw DSPProgramError("parameters missing") }
+        let parsedParameters = try parameters.map { p -> Parameter in
+            guard let name = p["name"] as? String else { throw DSPProgramError("a parameter has no name") }
+            let parameter = Parameter(
+                name: name, register: try register(p["register"], "parameter \(name)"),
+                defaultValue: try number(p["default"], "parameter \(name)'s default"),
+                min: try number(p["min"], "parameter \(name)'s min"), max: try number(p["max"], "parameter \(name)'s max")
+            )
+            guard parameter.min <= parameter.max, [parameter.defaultValue, parameter.min, parameter.max].allSatisfy(\.isFinite) else {
+                throw DSPProgramError("parameter \(name): its range is not numbers from min to max")
+            }
+            return parameter
+        }
+        guard let state = root["state"] as? [[String: Any]] else { throw DSPProgramError("state missing") }
+        let parsedState = try state.map { s -> State in
+            let name = s["name"] as? String ?? "state"
+            return State(name: name, register: try register(s["register"], "state \(name)"), initial: try number(s["initial"], "state \(name)'s initial value"))
+        }
+        let output = try register(root["output"], "output")
+        func code(_ key: String) throws -> [Int32] {
+            guard let words = root[key] as? [Any] else { throw DSPProgramError("\(key) must be an array") }
+            let code = try words.map { word -> Int32 in
+                guard let n = Self.int(word), n >= Int(Int32.min), n <= Int(Int32.max) else { throw DSPProgramError("\(key): a word is not a whole number") }
+                return Int32(n)
+            }
+            var pc = 0
+            while pc < code.count {
+                guard let op = DSPOpcode(rawValue: code[pc]) else { throw DSPProgramError("\(key): unknown opcode \(code[pc]) at \(pc)") }
+                let length = 2 + op.arity + (op.builtin == nil ? 0 : 1)
+                guard pc + length <= code.count else { throw DSPProgramError("\(key): \(op) at \(pc) is cut off") }
+                for i in 1..<(2 + op.arity) { _ = try register(Int(code[pc + i]), "\(key) \(op) at \(pc)") }
+                if let builtin = op.builtin {
+                    let instance = Int(code[pc + length - 1])
+                    guard instances.indices.contains(instance), instances[instance] == builtin else {
+                        throw DSPProgramError("\(key): \(op) at \(pc) names instance \(instance), not a \(builtin)")
+                    }
+                }
+                pc += length
+            }
+            return code
+        }
+        self.voices = voices
+        self.registers = registers
+        self.constants = parsedConstants
+        self.parameters = parsedParameters
+        self.state = parsedState
+        self.instances = instances
+        self.block = try code("block")
+        self.sample = try code("sample")
+        self.output = output
+    }
+
+    private static func int(_ value: Any?) -> Int? {
+        guard let number = value as? NSNumber else { return nil }
+        let double = number.doubleValue
+        guard double.rounded() == double, abs(double) < 1e12 else { return nil }
+        return Int(double)
+    }
+}
+
+public struct DSPProgramError: Error, CustomStringConvertible, Equatable {
+    public let description: String
+    public init(_ detail: String) { description = "invalidProgram: \(detail)" }
+}
+
+/// The stateful builtins, by the name a program's `instances` gives them.
+public enum DSPBuiltin: String, Sendable, CaseIterable {
+    case phasor, sine, saw, square, triangle, noise, adsr, lowpass, highpass, bandpass
+
+    /// Words of state per lane (program.ts BUILTIN_STATE).
+    var words: Int {
+        switch self {
+        case .phasor, .sine, .saw, .square, .triangle, .noise: 1
+        case .adsr, .lowpass, .highpass, .bandpass: 8
+        }
+    }
+}
+
+/// The opcodes, v1: the numbers are the wire format (program.ts OPCODES).
+public enum DSPOpcode: Int32, Sendable, CaseIterable {
+    case mov = 0, add, sub, mul, div, mod, neg, min, max, abs, floor, ceil, round, sqrt, sin, cos, tan, exp, log, pow, tanh
+    case lt = 21, le, gt, ge, eq, ne, and, or, not, select, clamp, mix, wrap, sign, atan
+    case phasor = 40, sine, saw, square, triangle, noise, adsr, lowpass, highpass, bandpass
+
+    /// Operands after the destination.
+    public var arity: Int {
+        switch self {
+        case .noise: 0
+        case .mov, .neg, .abs, .floor, .ceil, .round, .sqrt, .sin, .cos, .tan, .exp, .log, .tanh, .not, .wrap, .sign, .atan,
+             .phasor, .sine, .saw, .triangle: 1
+        case .select, .clamp, .mix, .lowpass, .highpass, .bandpass: 3
+        case .adsr: 5
+        default: 2
+        }
+    }
+
+    /// The builtin whose state it keeps, for a stateful one.
+    public var builtin: DSPBuiltin? {
+        rawValue >= 40 ? DSPBuiltin.allCases[Int(rawValue) - 40] : nil
+    }
+}

package/swift/Packages/SwiftBrowserLocalModel/Package.swift

@@ -0,0 +1,35 @@
+// swift-tools-version: 6.0
+import PackageDescription
+
+// SwiftBrowserLocalModel: an open language model on the device, through
+// llama.cpp, for the `LanguageModelSession`s of a SwiftBrowser Native app
+// where Apple Intelligence cannot answer (an iPhone before the 15 Pro, Apple
+// Intelligence off, a system before iOS 26). A package of its own, so only an
+// app that adds it fetches, builds and links llama.cpp; it installs itself as
+// `LanguageModels.fallback` (SwiftBrowserNative, LanguageModelProvider.swift).
+//
+// llama.cpp is its release XCFramework, b10456: the newest whose release
+// carries the iOS Simulator beside devices and macOS (later ones ship devices
+// and macOS only). SwiftPM fetches it once (286 MB, tvOS and visionOS
+// included) and caches it; an iPhone app links about 6 MB of it.
+let package = Package(
+    name: "SwiftBrowserLocalModel",
+    platforms: [.iOS(.v17), .macOS(.v14)],
+    products: [
+        .library(name: "SwiftBrowserLocalModel", targets: ["SwiftBrowserLocalModel"]),
+    ],
+    dependencies: [
+        .package(path: "../.."),
+    ],
+    targets: [
+        .binaryTarget(
+            name: "llama",
+            url: "https://github.com/ggml-org/llama.cpp/releases/download/b10456/llama-b10456-xcframework.zip",
+            checksum: "0223bedd0a01232399d943dcb72bc227882bc90df98e29d7a92343531a88cc02"
+        ),
+        .target(
+            name: "SwiftBrowserLocalModel",
+            dependencies: [.product(name: "SwiftBrowserNative", package: "swift"), "llama"]
+        ),
+    ]
+)

package/swift/Packages/SwiftBrowserLocalModel/README.md

@@ -0,0 +1,66 @@
+# SwiftBrowserLocalModel
+
+An open language model on the device, run with
+[llama.cpp](https://github.com/ggml-org/llama.cpp), for a SwiftBrowser Native
+app's `LanguageModelSession`s where Apple Intelligence cannot answer: an
+iPhone before the 15 Pro, Apple Intelligence turned off, a system before
+iOS 26. Where Apple Intelligence answers, it still does.
+
+It is a package of its own so that only an app that adds it fetches, builds
+and links llama.cpp. @swiftbrowser/native ships it beside the core
+(`node_modules/@swiftbrowser/native/swift/Packages/SwiftBrowserLocalModel`;
+it depends on `SwiftBrowserNative` at `../..`, here and there). Link it into
+the app's Xcode project with
+
+```sh
+npx swiftbrowser native add local-model        # or native init --with local-model
+```
+
+and install a model at launch:
+
+```swift
+import SwiftBrowserLocalModel
+
+init() {
+    LocalLanguageModel.install([.llama3_2_1B, .qwen2_5_1_5B, .llama3_2_3B])
+}
+```
+
+With several, the first is the default and availability lists them as
+`models` for the viewer to choose from (`selectModel(id)`), the choice
+remembered across launches; each downloads on its own.
+
+On the JavaScript side, `useSystemLanguageModel()` reports `modelNotReady`
+with `download: { size, downloaded: false }` until the viewer asks for the
+model: show the size and call `SystemLanguageModel.default.download()` from
+a button. The download's `progress` follows, then `available`, and
+`LanguageModelSession` answers with it. `deleteDownload()` stops a download
+or takes the model off the device (the Showcase's Chat has both).
+
+- **Tools** a session is given are offered in its instructions, and a reply
+  that is just `{"tool": name, "arguments": {...}}` is run as a call, its
+  result handed back for the model to go on. Small models call tools less
+  reliably than Apple Intelligence does; Qwen 2.5 1.5B is the steadiest here.
+- **The models**: `.llama3_2_1B` (808 MB, fastest), `.qwen2_5_1_5B` (1.1 GB,
+  Apache 2.0, best with tools), `.llama3_2_3B` (2.0 GB, most capable; a phone
+  with 6 GB of memory runs it near its limit).
+- **The model** is a GGUF file, fetched when the app calls `download()`, kept
+  in Application Support's
+  `SwiftBrowserLocalModel` folder and excluded from backups. `.llama3_2_1B`
+  is Meta's Llama 3.2 1B Instruct at 4 bits (808 MB; Meta's Llama 3.2
+  Community License applies); any other is a `LocalLanguageModel.Model`
+  with its URL, size and context length. `allowsCellularAccess: false`
+  waits for Wi-Fi.
+- **Running it**: loaded once on the first prompt or `prewarm()`, on the GPU
+  with Metal, on a queue of its own. A session keeps its messages, formatted
+  with the model's chat template (ChatML for a model without one); the
+  context keeps the tokens of the last conversation, so a prompt that
+  continues it decodes only what is new. A conversation longer than the
+  context fails as `exceededContextWindowSize`.
+- **Size**: llama.cpp is its release XCFramework, b10456, the newest whose
+  release has the iOS Simulator beside devices and macOS. SwiftPM downloads
+  it once (286 MB, with tvOS and visionOS) and caches it; an iPhone app
+  links about 6 MB.
+
+The tests (`swift test`, macOS) run llama.cpp on the CPU with a 1.2 MB toy
+model they download.

package/swift/Packages/SwiftBrowserLocalModel/Sources/SwiftBrowserLocalModel/LlamaEngine.swift

@@ -0,0 +1,223 @@
+import Foundation
+import llama
+import SwiftBrowserNative
+
+/// A model loaded with llama.cpp and the one context it answers in, used
+/// from a serial queue of its own (every call into llama.cpp blocks). The
+/// context keeps the tokens of the last conversation it saw, so a prompt that
+/// continues it decodes only what is new.
+final class LlamaEngine: @unchecked Sendable {
+    struct Message: Sendable, Equatable {
+        let role: String
+        let content: String
+    }
+
+    let contextLength: Int
+    private let queue = DispatchQueue(label: "dev.swiftbrowser.localmodel.llama", qos: .userInitiated)
+    private let model: OpaquePointer
+    private let context: OpaquePointer
+    private let vocab: OpaquePointer
+    /// The model's chat template, or ChatML for a model without one.
+    private let template: String
+    /// The tokens the context holds, in order.
+    private var cached: [llama_token] = []
+
+    /// llama.cpp's backends, started once, with its logging silenced.
+    private static let backend: Void = {
+        llama_backend_init()
+        llama_log_set({ _, _, _ in }, nil)
+    }()
+
+    /// Loads the model at `path`. Blocking: call it off the main thread.
+    init(path: String, contextLength: Int, gpuLayers: Int32) throws {
+        _ = Self.backend
+        var modelParams = llama_model_default_params()
+        modelParams.n_gpu_layers = gpuLayers
+        guard let model = llama_model_load_from_file(path, modelParams) else {
+            throw LanguageModelError(.unavailable, "the model could not be loaded")
+        }
+        var contextParams = llama_context_default_params()
+        contextParams.n_ctx = UInt32(contextLength)
+        contextParams.n_batch = 512
+        let threads = Int32(max(1, min(8, ProcessInfo.processInfo.activeProcessorCount - 2)))
+        contextParams.n_threads = threads
+        contextParams.n_threads_batch = threads
+        guard let context = llama_init_from_model(model, contextParams) else {
+            llama_model_free(model)
+            throw LanguageModelError(.unavailable, "the model's context could not be made")
+        }
+        guard let vocab = llama_model_get_vocab(model) else {
+            llama_free(context)
+            llama_model_free(model)
+            throw LanguageModelError(.unavailable, "the model has no vocabulary")
+        }
+        self.model = model
+        self.context = context
+        self.vocab = vocab
+        self.contextLength = Int(llama_n_ctx(context))
+        template = llama_model_chat_template(model, nil).map { String(cString: $0) } ?? "chatml"
+    }
+
+    /// The role a tool's result goes back to the model as: Llama 3's `ipython`, else `tool`.
+    var toolRole: String {
+        template.contains("<|start_header_id|>") ? "ipython" : "tool"
+    }
+
+    deinit {
+        llama_free(context)
+        llama_model_free(model)
+    }
+
+    /// The reply to the last message of `messages`, calling `onText` with the
+    /// reply so far as it grows; stops early once `cancelled` is set, with
+    /// what came by then.
+    func generate(
+        _ messages: [Message],
+        options: LanguageModelOptions,
+        cancelled: CancelFlag,
+        onText: @escaping @Sendable (String) -> Void
+    ) async throws -> String {
+        try await withCheckedThrowingContinuation { continuation in
+            queue.async {
+                do {
+                    continuation.resume(returning: try self.generateNow(messages, options: options, cancelled: cancelled, onText: onText))
+                } catch {
+                    continuation.resume(throwing: error)
+                }
+            }
+        }
+    }
+
+    private func generateNow(_ messages: [Message], options: LanguageModelOptions, cancelled: CancelFlag, onText: (String) -> Void) throws -> String {
+        let tokens = tokenize(try format(messages))
+        guard tokens.count + 16 < contextLength else {
+            throw LanguageModelError(.context, "the conversation is longer than the model's \(contextLength) tokens")
+        }
+        // Keep what the context already holds of this conversation; decode the rest.
+        var kept = 0
+        while kept < min(cached.count, tokens.count), cached[kept] == tokens[kept] { kept += 1 }
+        if kept == tokens.count { kept -= 1 }
+        let memory = llama_get_memory(context)
+        if !llama_memory_seq_rm(memory, 0, llama_pos(kept), -1) {
+            llama_memory_clear(memory, true)
+            kept = 0
+        }
+        cached.removeSubrange(min(kept, cached.count)...)
+        var start = kept
+        while start < tokens.count {
+            if cancelled.isCancelled { return "" }
+            let end = min(start + 512, tokens.count)
+            try decode(Array(tokens[start..<end]))
+            cached.append(contentsOf: tokens[start..<end])
+            start = end
+        }
+
+        let sampler = llama_sampler_chain_init(llama_sampler_chain_default_params())
+        defer { llama_sampler_free(sampler) }
+        if options.greedy {
+            llama_sampler_chain_add(sampler, llama_sampler_init_greedy())
+        } else {
+            llama_sampler_chain_add(sampler, llama_sampler_init_top_k(40))
+            llama_sampler_chain_add(sampler, llama_sampler_init_top_p(0.9, 1))
+            llama_sampler_chain_add(sampler, llama_sampler_init_temp(Float(options.temperature ?? 0.7)))
+            llama_sampler_chain_add(sampler, llama_sampler_init_dist(UInt32.random(in: 0...UInt32.max)))
+        }
+
+        var bytes: [UInt8] = []
+        let limit = options.maximumResponseTokens ?? contextLength
+        for _ in 0..<limit {
+            if cancelled.isCancelled || cached.count >= contextLength - 1 { break }
+            let token = llama_sampler_sample(sampler, context, -1)
+            if llama_vocab_is_eog(vocab, token) { break }
+            bytes += piece(token)
+            // A character split across tokens waits for its last byte.
+            if let text = String(bytes: bytes, encoding: .utf8) { onText(Self.trimmed(text)) }
+            try decode([token])
+            cached.append(token)
+        }
+        return Self.trimmed(String(decoding: bytes, as: UTF8.self))
+    }
+
+    /// The conversation in the model's chat template, ending where the assistant's reply starts.
+    func format(_ messages: [Message]) throws -> String {
+        if let text = format(messages, template: template) ?? format(messages, template: "chatml") { return text }
+        throw LanguageModelError(.unavailable, "the model's chat template is not supported")
+    }
+
+    private func format(_ messages: [Message], template: String) -> String? {
+        let roles = messages.map { strdup($0.role) }
+        let contents = messages.map { strdup($0.content) }
+        defer {
+            roles.forEach { free($0) }
+            contents.forEach { free($0) }
+        }
+        var chat = messages.indices.map { llama_chat_message(role: UnsafePointer(roles[$0]), content: UnsafePointer(contents[$0])) }
+        let count = chat.count
+        var capacity = max(1024, messages.reduce(0) { $0 + $1.content.utf8.count } * 2 + 512)
+        var buffer = [CChar](repeating: 0, count: capacity)
+        var length = llama_chat_apply_template(template, &chat, count, true, &buffer, Int32(capacity))
+        if length > capacity {
+            capacity = Int(length) + 1
+            buffer = [CChar](repeating: 0, count: capacity)
+            length = llama_chat_apply_template(template, &chat, count, true, &buffer, Int32(capacity))
+        }
+        guard length >= 0 else { return nil }
+        return String(decoding: buffer.prefix(Int(length)).map { UInt8(bitPattern: $0) }, as: UTF8.self)
+    }
+
+    private func tokenize(_ text: String) -> [llama_token] {
+        let length = Int32(text.utf8.count)
+        var capacity = length + 8
+        var tokens = [llama_token](repeating: 0, count: Int(capacity))
+        var count = llama_tokenize(vocab, text, length, &tokens, capacity, true, true)
+        if count < 0 {
+            capacity = -count
+            tokens = [llama_token](repeating: 0, count: Int(capacity))
+            count = llama_tokenize(vocab, text, length, &tokens, capacity, true, true)
+        }
+        return Array(tokens.prefix(Int(max(count, 0))))
+    }
+
+    private func piece(_ token: llama_token) -> [UInt8] {
+        var capacity: Int32 = 64
+        var buffer = [CChar](repeating: 0, count: Int(capacity))
+        var count = llama_token_to_piece(vocab, token, &buffer, capacity, 0, false)
+        if count < 0 {
+            capacity = -count
+            buffer = [CChar](repeating: 0, count: Int(capacity))
+            count = llama_token_to_piece(vocab, token, &buffer, capacity, 0, false)
+        }
+        return buffer.prefix(Int(max(count, 0))).map { UInt8(bitPattern: $0) }
+    }
+
+    /// Reads `tokens` into the context; a failure leaves it empty, so the next prompt starts over.
+    private func decode(_ tokens: [llama_token]) throws {
+        var tokens = tokens
+        let result = tokens.withUnsafeMutableBufferPointer { buffer in
+            llama_decode(context, llama_batch_get_one(buffer.baseAddress, Int32(buffer.count)))
+        }
+        guard result == 0 else {
+            llama_memory_clear(llama_get_memory(context), true)
+            cached = []
+            throw LanguageModelError(result == 1 ? .context : .unavailable, "the model could not read the conversation (\(result))")
+        }
+    }
+
+    private static func trimmed(_ text: String) -> String {
+        String(text.drop(while: \.isWhitespace))
+    }
+}
+
+/// Set once a response should stop; read from the engine's queue.
+final class CancelFlag: @unchecked Sendable {
+    private let lock = NSLock()
+    private var cancelled = false
+
+    var isCancelled: Bool {
+        lock.withLock { cancelled }
+    }
+
+    func cancel() {
+        lock.withLock { cancelled = true }
+    }
+}