@swiftbrowser/native 0.18.3 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +53 -11
- package/bin/swiftbrowser-native.mjs +47 -0
- package/dist/blob.d.ts +85 -11
- package/dist/blob.d.ts.map +1 -1
- package/dist/blob.js +184 -34
- package/dist/blob.js.map +1 -1
- package/dist/bridge.d.ts +66 -8
- package/dist/bridge.d.ts.map +1 -1
- package/dist/bridge.js.map +1 -1
- package/dist/encoding.d.ts +13 -0
- package/dist/encoding.d.ts.map +1 -1
- package/dist/encoding.js +24 -0
- package/dist/encoding.js.map +1 -1
- package/dist/fetch.d.ts +27 -13
- package/dist/fetch.d.ts.map +1 -1
- package/dist/fetch.js +117 -45
- package/dist/fetch.js.map +1 -1
- package/dist/hostRequests.d.ts +24 -6
- package/dist/hostRequests.d.ts.map +1 -1
- package/dist/hostRequests.js +108 -15
- package/dist/hostRequests.js.map +1 -1
- package/dist/index.d.ts +1 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -2
- package/dist/index.js.map +1 -1
- package/dist/ios.d.ts +79 -0
- package/dist/ios.d.ts.map +1 -0
- package/dist/ios.js +181 -0
- package/dist/ios.js.map +1 -0
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +16 -7
- package/dist/runtime.js.map +1 -1
- package/dist/subtle.d.ts +1 -1
- package/dist/subtle.d.ts.map +1 -1
- package/dist/subtle.js +9 -5
- package/dist/subtle.js.map +1 -1
- package/dist/webSocket.d.ts +12 -5
- package/dist/webSocket.d.ts.map +1 -1
- package/dist/webSocket.js +37 -11
- package/dist/webSocket.js.map +1 -1
- package/package.json +7 -3
- package/src/blob.ts +215 -34
- package/src/bridge.ts +56 -8
- package/src/encoding.ts +22 -0
- package/src/fetch.ts +111 -48
- package/src/hostRequests.ts +129 -22
- package/src/index.ts +1 -2
- package/src/ios.ts +253 -0
- package/src/runtime.ts +23 -8
- package/src/subtle.ts +10 -6
- package/src/webSocket.ts +37 -13
- package/swift/Packages/SwiftBrowserDSP/Package.swift +31 -0
- package/swift/Packages/SwiftBrowserDSP/README.md +42 -0
- package/swift/Packages/SwiftBrowserDSP/Sources/SwiftBrowserDSP/ProgramNode.swift +306 -0
- package/swift/Packages/SwiftBrowserDSP/Sources/SwiftBrowserDSP/ScopeView.swift +79 -0
- package/swift/Packages/SwiftBrowserDSP/Sources/SwiftBrowserDSPCore/Machine.swift +1561 -0
- package/swift/Packages/SwiftBrowserDSP/Sources/SwiftBrowserDSPCore/Program.swift +483 -0
- package/swift/Packages/SwiftBrowserLocalModel/Package.swift +35 -0
- package/swift/Packages/SwiftBrowserLocalModel/README.md +66 -0
- package/swift/Packages/SwiftBrowserLocalModel/Sources/SwiftBrowserLocalModel/LlamaEngine.swift +223 -0
- package/swift/Packages/SwiftBrowserLocalModel/Sources/SwiftBrowserLocalModel/LocalLanguageModel.swift +409 -0
- package/swift/Packages/SwiftBrowserShader/Package.swift +31 -0
- package/swift/Packages/SwiftBrowserShader/README.md +45 -0
- package/swift/Packages/SwiftBrowserShader/Sources/SwiftBrowserShader/ShaderRenderer.swift +103 -0
- package/swift/Packages/SwiftBrowserShader/Sources/SwiftBrowserShader/ShaderView.swift +168 -0
- package/swift/Packages/SwiftBrowserShader/Sources/SwiftBrowserShaderCore/ShaderProps.swift +120 -0
- package/swift/Sources/SwiftBrowserNative/Host/AudioEngineBridge.swift +921 -0
- package/swift/Sources/SwiftBrowserNative/Host/AudioEngineNodes.swift +70 -0
- package/swift/Sources/SwiftBrowserNative/Host/BodyStream.swift +166 -0
- package/swift/Sources/SwiftBrowserNative/Host/FilesBridge.swift +144 -10
- package/swift/Sources/SwiftBrowserNative/Host/HostBridge.swift +151 -27
- package/swift/Sources/SwiftBrowserNative/Host/HostRequestBridge.swift +139 -40
- package/swift/Sources/SwiftBrowserNative/Host/JSHost.swift +116 -0
- package/swift/Sources/SwiftBrowserNative/Host/JavaScriptCoreHost.swift +141 -2
- package/swift/Sources/SwiftBrowserNative/Host/LanguageModelBridge.swift +511 -0
- package/swift/Sources/SwiftBrowserNative/Host/LanguageModelProvider.swift +195 -0
- package/swift/Sources/SwiftBrowserNative/Host/WebSocketBridge.swift +24 -11
- package/swift/Sources/SwiftBrowserNative/Host/WorkerHost.swift +3 -2
- package/swift/Sources/SwiftBrowserNative/Runtime/NativeFunctions.swift +1 -1
- package/swift/Sources/SwiftBrowserNative/Runtime/ViewRegistry.swift +57 -0
- package/swift/Sources/SwiftBrowserNative/Views/ElementView.swift +12 -7
- package/swift/Sources/SwiftBrowserNative/Views/Kinds/Controls.swift +18 -1
- package/swift/Sources/SwiftBrowserNative/Views/Kinds/Drawing.swift +378 -0
- package/swift/Sources/SwiftBrowserNative/Views/Kinds/Lists.swift +41 -1
- package/swift/Sources/SwiftBrowserNative/Views/Kinds/More.swift +88 -17
- package/swift/Sources/SwiftBrowserNative/Views/Kinds/Navigation.swift +87 -21
- package/swift/Sources/SwiftBrowserNative/Views/Kinds/PinchTracker.swift +200 -0
- package/swift/Sources/SwiftBrowserNative/Views/Kinds/PopTap.swift +103 -0
- package/swift/Sources/SwiftBrowserNative/Views/Kinds/Stacks.swift +3 -2
- package/swift/Sources/SwiftBrowserNative/Views/StyleModifier.swift +146 -0
- package/swift/Sources/SwiftBrowserProtocol/Drawing.swift +306 -0
- package/swift/Sources/SwiftBrowserProtocol/Props.swift +31 -2
- package/swift/Sources/SwiftBrowserProtocol/RendererEvent.swift +24 -3
- package/swift/Sources/SwiftBrowserProtocol/Style.swift +87 -0
- package/swift/Sources/SwiftBrowserTree/OpTree.swift +25 -0
- package/wasm/SwiftBrowserNative.wasm +2 -2
- package/dist/native.d.ts +0 -19
- package/dist/native.d.ts.map +0 -1
- package/dist/native.js +0 -17
- package/dist/native.js.map +0 -1
- package/src/native.ts +0 -35
- package/swift/Sources/SwiftBrowserNative/Runtime/NativeRegistry.swift +0 -53
|
@@ -0,0 +1,483 @@
|
|
|
1
|
+
import Foundation
|
|
2
|
+
|
|
3
|
+
/// A compiled DSP program: the JSON @swiftbrowser/dsp's compiler writes
|
|
4
|
+
/// (packages/dsp/src/program.ts, which documents the format). The machine
|
|
5
|
+
/// keeps a register file of `registers` registers of 16 lanes, a lane per
|
|
6
|
+
/// voice; `block` runs once per block of up to 128 frames and `sample` once
|
|
7
|
+
/// per frame, each a flat list of instructions: an opcode, a destination
|
|
8
|
+
/// register, the opcode's operands and, for a stateful builtin, its instance.
|
|
9
|
+
/// A voice may have a right side (`outputRight`), and the `master` stage
|
|
10
|
+
/// runs after the voices (in lane 0). The machine runs `sample` and `master`
|
|
11
|
+
/// a chunk of frames at a time, all but their `serial` parts (word offsets),
|
|
12
|
+
/// which touch state and run frame by frame (block processing).
|
|
13
|
+
public struct DSPProgram: Sendable, Equatable {
|
|
14
|
+
public struct Parameter: Sendable, Equatable {
|
|
15
|
+
public let name: String
|
|
16
|
+
public let register: Int
|
|
17
|
+
public let defaultValue: Float
|
|
18
|
+
public let min: Float
|
|
19
|
+
public let max: Float
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
public struct State: Sendable, Equatable {
|
|
23
|
+
public let name: String
|
|
24
|
+
public let register: Int
|
|
25
|
+
public let initial: Float
|
|
26
|
+
/// The master stage's: reset when the program loads and on `reset`, not when a voice starts.
|
|
27
|
+
public var master = false
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/// A wavetable: its frames, each the amplitudes of its sine harmonics from the fundamental up, and with
|
|
31
|
+
/// `cosines` (one list a frame, each as long as its frame's) their cosine parts.
|
|
32
|
+
public struct Table: Sendable, Equatable {
|
|
33
|
+
public let name: String
|
|
34
|
+
public let frames: [[Float]]
|
|
35
|
+
public var cosines: [[Float]]? = nil
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/// A drawn curve: values at evenly spaced points from x = 0 to x = 1.
|
|
39
|
+
public struct Shape: Sendable, Equatable {
|
|
40
|
+
public let name: String
|
|
41
|
+
public let points: [Float]
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/// Audio: mono, `rate` samples a second.
|
|
45
|
+
public struct Sample: Sendable, Equatable {
|
|
46
|
+
public let name: String
|
|
47
|
+
public let rate: Double
|
|
48
|
+
public let data: [Float]
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/// The master stage: the voices' sum goes into `inputs` (left, right) each frame, `code` runs in lane 0, and
|
|
52
|
+
/// `outputs` is the program's output. Its notes are the newest one's (registers 7 … 10), and its envelopes start
|
|
53
|
+
/// over at every note.
|
|
54
|
+
public struct Master: Sendable, Equatable {
|
|
55
|
+
public let code: [Int32]
|
|
56
|
+
/// The part of `code` that runs frame by frame, in words.
|
|
57
|
+
public let serial: Range<Int>?
|
|
58
|
+
public let inputs: [Int]
|
|
59
|
+
public let outputs: [Int]
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/// One voice playing the newest held note: `mono` starts its envelopes over at each note, `legato` only when
|
|
63
|
+
/// no other note was held.
|
|
64
|
+
public enum Mode: String, Sendable, Equatable {
|
|
65
|
+
case mono, legato
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
public let voices: Int
|
|
69
|
+
/// Absent, polyphonic.
|
|
70
|
+
public let mode: Mode?
|
|
71
|
+
public let registers: Int
|
|
72
|
+
public let constants: [(register: Int, value: Float)]
|
|
73
|
+
public let parameters: [Parameter]
|
|
74
|
+
public let state: [State]
|
|
75
|
+
public let instances: [DSPBuiltin]
|
|
76
|
+
public let tables: [Table]
|
|
77
|
+
public let shapes: [Shape]
|
|
78
|
+
public let samples: [Sample]
|
|
79
|
+
/// `modsum`'s route lists: a source's register, then its amount's, pair after pair.
|
|
80
|
+
public let routes: [[Int]]
|
|
81
|
+
/// Envelopes with curves (program.ts `curves`): an `adsr` instance and its attack, decay and release exponents.
|
|
82
|
+
public let curves: [Curve]
|
|
83
|
+
public let block: [Int32]
|
|
84
|
+
public let sample: [Int32]
|
|
85
|
+
/// The part of `sample` that runs frame by frame, in words.
|
|
86
|
+
public let serial: Range<Int>?
|
|
87
|
+
/// The voice's left side, or both with no `outputRight`.
|
|
88
|
+
public let output: Int
|
|
89
|
+
public let outputRight: Int?
|
|
90
|
+
public let master: Master?
|
|
91
|
+
|
|
92
|
+
public static func == (a: DSPProgram, b: DSPProgram) -> Bool {
|
|
93
|
+
a.voices == b.voices && a.mode == b.mode && a.registers == b.registers && a.constants.map(\.register) == b.constants.map(\.register)
|
|
94
|
+
&& a.constants.map(\.value) == b.constants.map(\.value) && a.parameters == b.parameters && a.state == b.state
|
|
95
|
+
&& a.instances == b.instances && a.tables == b.tables && a.shapes == b.shapes && a.samples == b.samples && a.routes == b.routes && a.curves == b.curves
|
|
96
|
+
&& a.block == b.block && a.sample == b.sample && a.serial == b.serial && a.output == b.output && a.outputRight == b.outputRight && a.master == b.master
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
public static let format = "swiftbrowser-dsp"
|
|
100
|
+
/// The format's version: the machine runs only its own.
|
|
101
|
+
public static let version = 1
|
|
102
|
+
public static let lanes = 16
|
|
103
|
+
public static let blockFrames = 128
|
|
104
|
+
/// The machine's own registers: `sampleRate`, a voice's `freq`, `gate`, `velocity` and `note`, then the
|
|
105
|
+
/// transport's `tempo` and `beat`.
|
|
106
|
+
public static let machineRegisters = 11
|
|
107
|
+
/// Samples a wavetable frame has, at each of its band-limited levels (program.ts `TABLE_FRAME`, `TABLE_LEVELS`).
|
|
108
|
+
public static let tableFrame = 2048
|
|
109
|
+
public static let tableLevels = 10
|
|
110
|
+
/// The most harmonics a frame gives, and frames a program's tables have between them.
|
|
111
|
+
public static let tableHarmonics = 512
|
|
112
|
+
public static let tableFramesMax = 256
|
|
113
|
+
/// The bounds of an envelope curve's exponents (program.ts `CURVE_MIN`, `CURVE_MAX`).
|
|
114
|
+
public static let curveRange: ClosedRange<Float> = 1.0 / 16 ... 16
|
|
115
|
+
|
|
116
|
+
/// An envelope's curves: the exponents its attack, decay and release are bent by (1 as without).
|
|
117
|
+
public struct Curve: Equatable, Sendable {
|
|
118
|
+
public let instance: Int
|
|
119
|
+
public let attack: Float
|
|
120
|
+
public let decay: Float
|
|
121
|
+
public let release: Float
|
|
122
|
+
}
|
|
123
|
+
/// The longest a delay may be, in seconds.
|
|
124
|
+
public static let delaySecondsMax: Float = 2
|
|
125
|
+
/// Points a shape has, and its shapes between them; samples a program's samples have between them.
|
|
126
|
+
public static let shapePointsMax = 4096
|
|
127
|
+
public static let shapesPointsMax = 65_536
|
|
128
|
+
public static let samplesMax = 8_000_000
|
|
129
|
+
|
|
130
|
+
/// The program the JSON describes, checked: every register, operand and instance in range.
|
|
131
|
+
public init(json: String) throws {
|
|
132
|
+
guard let data = json.data(using: .utf8) else { throw DSPProgramError("not UTF-8") }
|
|
133
|
+
try self.init(data: data)
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
public init(data: Data) throws {
|
|
137
|
+
let parsed: Any
|
|
138
|
+
do {
|
|
139
|
+
parsed = try JSONSerialization.jsonObject(with: data)
|
|
140
|
+
} catch {
|
|
141
|
+
throw DSPProgramError("not JSON (\(error.localizedDescription))")
|
|
142
|
+
}
|
|
143
|
+
try self.init(object: parsed)
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/// From a decoded JSON object (`JSONSerialization`'s).
|
|
147
|
+
public init(object: Any) throws {
|
|
148
|
+
guard let root = object as? [String: Any] else { throw DSPProgramError("not an object") }
|
|
149
|
+
guard root["format"] as? String == Self.format else { throw DSPProgramError("format is not \(Self.format)") }
|
|
150
|
+
guard Self.int(root["version"]) == Self.version else {
|
|
151
|
+
throw DSPProgramError("version \(root["version"].map { "\($0)" } ?? "none") is not \(Self.version)")
|
|
152
|
+
}
|
|
153
|
+
guard let voices = Self.int(root["voices"]), (1...Self.lanes).contains(voices) else { throw DSPProgramError("voices must be 1 … \(Self.lanes)") }
|
|
154
|
+
var mode: Mode?
|
|
155
|
+
if let value = root["mode"] {
|
|
156
|
+
guard let name = value as? String, let parsed = Mode(rawValue: name) else { throw DSPProgramError("mode is not mono or legato") }
|
|
157
|
+
guard voices == 1 else { throw DSPProgramError("a mono or legato program has 1 voice") }
|
|
158
|
+
mode = parsed
|
|
159
|
+
}
|
|
160
|
+
guard let registers = Self.int(root["registers"]), (Self.machineRegisters...65_536).contains(registers) else { throw DSPProgramError("registers out of range") }
|
|
161
|
+
func register(_ value: Any?, _ what: String) throws -> Int {
|
|
162
|
+
guard let r = Self.int(value), r >= 0, r < registers else { throw DSPProgramError("\(what): register out of range") }
|
|
163
|
+
return r
|
|
164
|
+
}
|
|
165
|
+
func number(_ value: Any?, _ what: String) throws -> Float {
|
|
166
|
+
guard let n = (value as? NSNumber)?.doubleValue, n.isFinite || n.isInfinite else { throw DSPProgramError("\(what) must be a number") }
|
|
167
|
+
return Float(n)
|
|
168
|
+
}
|
|
169
|
+
guard let instanceNames = root["instances"] as? [String] else { throw DSPProgramError("instances must be an array of names") }
|
|
170
|
+
let instances = try instanceNames.map { name in
|
|
171
|
+
guard let builtin = DSPBuiltin(rawValue: name) else { throw DSPProgramError("unknown builtin \(name)") }
|
|
172
|
+
return builtin
|
|
173
|
+
}
|
|
174
|
+
guard let constants = root["constants"] as? [[Any]] else { throw DSPProgramError("constants missing") }
|
|
175
|
+
let parsedConstants: [(register: Int, value: Float)] = try constants.map { entry in
|
|
176
|
+
guard entry.count == 2 else { throw DSPProgramError("a constant is not [register, value]") }
|
|
177
|
+
return (try register(entry[0], "constant"), try number(entry[1], "a constant"))
|
|
178
|
+
}
|
|
179
|
+
guard let parameters = root["parameters"] as? [[String: Any]] else { throw DSPProgramError("parameters missing") }
|
|
180
|
+
let parsedParameters = try parameters.map { p -> Parameter in
|
|
181
|
+
guard let name = p["name"] as? String else { throw DSPProgramError("a parameter has no name") }
|
|
182
|
+
let parameter = Parameter(
|
|
183
|
+
name: name, register: try register(p["register"], "parameter \(name)"),
|
|
184
|
+
defaultValue: try number(p["default"], "parameter \(name)'s default"),
|
|
185
|
+
min: try number(p["min"], "parameter \(name)'s min"), max: try number(p["max"], "parameter \(name)'s max")
|
|
186
|
+
)
|
|
187
|
+
guard parameter.min <= parameter.max, [parameter.defaultValue, parameter.min, parameter.max].allSatisfy(\.isFinite) else {
|
|
188
|
+
throw DSPProgramError("parameter \(name): its range is not numbers from min to max")
|
|
189
|
+
}
|
|
190
|
+
return parameter
|
|
191
|
+
}
|
|
192
|
+
guard let state = root["state"] as? [[String: Any]] else { throw DSPProgramError("state missing") }
|
|
193
|
+
let parsedState = try state.map { s -> State in
|
|
194
|
+
let name = s["name"] as? String ?? "state"
|
|
195
|
+
return State(
|
|
196
|
+
name: name, register: try register(s["register"], "state \(name)"), initial: try number(s["initial"], "state \(name)'s initial value"),
|
|
197
|
+
master: s["master"] as? Bool ?? false
|
|
198
|
+
)
|
|
199
|
+
}
|
|
200
|
+
let output = try register(root["output"], "output")
|
|
201
|
+
let outputRight = try root["outputRight"].map { try register($0, "outputRight") }
|
|
202
|
+
var masterParts: (inputs: [Int], outputs: [Int])?
|
|
203
|
+
if root["master"] != nil {
|
|
204
|
+
guard let inputs = root["masterInputs"] as? [Any], inputs.count == 2 else { throw DSPProgramError("master needs masterInputs [left, right]") }
|
|
205
|
+
guard let outputs = root["masterOutput"] as? [Any], outputs.count == 2 else { throw DSPProgramError("master needs masterOutput [left, right]") }
|
|
206
|
+
masterParts = (try inputs.map { try register($0, "master") }, try outputs.map { try register($0, "master") })
|
|
207
|
+
}
|
|
208
|
+
var parsedShapes: [Shape] = []
|
|
209
|
+
if let shapes = root["shapes"] {
|
|
210
|
+
guard let list = shapes as? [[String: Any]] else { throw DSPProgramError("shapes must be an array") }
|
|
211
|
+
var points = 0
|
|
212
|
+
for shape in list {
|
|
213
|
+
guard let name = shape["name"] as? String, let values = shape["points"] as? [Any], (2...Self.shapePointsMax).contains(values.count) else {
|
|
214
|
+
throw DSPProgramError("a shape is { name, points }, 2 … \(Self.shapePointsMax) numbers")
|
|
215
|
+
}
|
|
216
|
+
let parsed = try values.map { value in
|
|
217
|
+
let point = try number(value, "shape \(name)'s point")
|
|
218
|
+
guard point.isFinite else { throw DSPProgramError("shape \(name): a point is not finite") }
|
|
219
|
+
return point
|
|
220
|
+
}
|
|
221
|
+
points += parsed.count
|
|
222
|
+
parsedShapes.append(Shape(name: name, points: parsed))
|
|
223
|
+
}
|
|
224
|
+
guard points <= Self.shapesPointsMax else { throw DSPProgramError("the shapes have more than \(Self.shapesPointsMax) points between them") }
|
|
225
|
+
}
|
|
226
|
+
var parsedSamples: [Sample] = []
|
|
227
|
+
if let samples = root["samples"] {
|
|
228
|
+
guard let list = samples as? [[String: Any]] else { throw DSPProgramError("samples must be an array") }
|
|
229
|
+
var total = 0
|
|
230
|
+
for sample in list {
|
|
231
|
+
guard let name = sample["name"] as? String, let text = sample["data"] as? String, let rate = (sample["rate"] as? NSNumber)?.doubleValue,
|
|
232
|
+
rate >= 1000, rate <= 384_000
|
|
233
|
+
else { throw DSPProgramError("a sample is { name, rate, data }") }
|
|
234
|
+
// Base64 of 32-bit little-endian floats; at least two.
|
|
235
|
+
guard let bytes = Data(base64Encoded: text), bytes.count % 4 == 0, bytes.count >= 8 else {
|
|
236
|
+
throw DSPProgramError("sample \(name)'s data is not base64 of at least 2 32-bit floats")
|
|
237
|
+
}
|
|
238
|
+
total += bytes.count / 4
|
|
239
|
+
guard total <= Self.samplesMax else { throw DSPProgramError("the samples have more than \(Self.samplesMax) samples between them") }
|
|
240
|
+
let data = bytes.withUnsafeBytes { raw in
|
|
241
|
+
(0..<(bytes.count / 4)).map { Float(bitPattern: UInt32(littleEndian: raw.loadUnaligned(fromByteOffset: $0 * 4, as: UInt32.self))) }
|
|
242
|
+
}
|
|
243
|
+
parsedSamples.append(Sample(name: name, rate: rate, data: data))
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
var parsedTables: [Table] = []
|
|
247
|
+
if let tables = root["tables"] {
|
|
248
|
+
guard let list = tables as? [[String: Any]] else { throw DSPProgramError("tables must be an array") }
|
|
249
|
+
var frameCount = 0
|
|
250
|
+
for table in list {
|
|
251
|
+
guard let name = table["name"] as? String, let frames = table["frames"] as? [[Any]], !frames.isEmpty else { throw DSPProgramError("a table is { name, frames }") }
|
|
252
|
+
let parsedFrames = try frames.map { frame -> [Float] in
|
|
253
|
+
guard !frame.isEmpty, frame.count <= Self.tableHarmonics else { throw DSPProgramError("table \(name): a frame is 1 … \(Self.tableHarmonics) harmonic amplitudes") }
|
|
254
|
+
return try frame.map { value in
|
|
255
|
+
let amplitude = try number(value, "table \(name)'s amplitude")
|
|
256
|
+
guard amplitude.isFinite else { throw DSPProgramError("table \(name): an amplitude is not finite") }
|
|
257
|
+
return amplitude
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
var parsedCosines: [[Float]]?
|
|
261
|
+
if let cosines = table["cosines"] {
|
|
262
|
+
guard let lists = cosines as? [[Any]], lists.count == parsedFrames.count else { throw DSPProgramError("table \(name): cosines are one list a frame") }
|
|
263
|
+
parsedCosines = try lists.enumerated().map { f, list in
|
|
264
|
+
guard list.count == parsedFrames[f].count else { throw DSPProgramError("table \(name): a frame's cosines are as many numbers as its sines") }
|
|
265
|
+
return try list.map { value in
|
|
266
|
+
let amplitude = try number(value, "table \(name)'s cosine")
|
|
267
|
+
guard amplitude.isFinite else { throw DSPProgramError("table \(name): a cosine is not finite") }
|
|
268
|
+
return amplitude
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
frameCount += parsedFrames.count
|
|
273
|
+
parsedTables.append(Table(name: name, frames: parsedFrames, cosines: parsedCosines))
|
|
274
|
+
}
|
|
275
|
+
guard frameCount <= Self.tableFramesMax else { throw DSPProgramError("the tables have more than \(Self.tableFramesMax) frames between them") }
|
|
276
|
+
}
|
|
277
|
+
var parsedRoutes: [[Int]] = []
|
|
278
|
+
if let routes = root["routes"] {
|
|
279
|
+
guard let lists = routes as? [[Any]] else { throw DSPProgramError("routes must be an array of lists") }
|
|
280
|
+
parsedRoutes = try lists.map { list in
|
|
281
|
+
guard list.count % 2 == 0 else { throw DSPProgramError("a route list is source and amount registers, pair after pair") }
|
|
282
|
+
let registers = try list.map { try register($0, "route") }
|
|
283
|
+
// An amount is a parameter, the same in every lane: the machine tests it once for all of them.
|
|
284
|
+
for k in stride(from: 1, to: registers.count, by: 2) where !parsedParameters.contains(where: { $0.register == registers[k] }) {
|
|
285
|
+
throw DSPProgramError("route amount register \(registers[k]) is not a parameter")
|
|
286
|
+
}
|
|
287
|
+
return registers
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
var parsedCurves: [Curve] = []
|
|
291
|
+
if let curves = root["curves"] {
|
|
292
|
+
guard let entries = curves as? [[Any]] else { throw DSPProgramError("curves must be an array") }
|
|
293
|
+
parsedCurves = try entries.map { entry in
|
|
294
|
+
guard entry.count == 4, let instance = Self.int(entry[0]), instances.indices.contains(instance), instances[instance] == .adsr else {
|
|
295
|
+
throw DSPProgramError("a curve is [instance, attack, decay, release] for an adsr")
|
|
296
|
+
}
|
|
297
|
+
let exponents = try entry.dropFirst().map { try number($0, "curve") }
|
|
298
|
+
guard exponents.allSatisfy({ Self.curveRange.contains($0) }) else { throw DSPProgramError("a curve's exponents are 1/16 … 16") }
|
|
299
|
+
return Curve(instance: instance, attack: exponents[0], decay: exponents[1], release: exponents[2])
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
let constantValues = Dictionary(parsedConstants.map { ($0.register, $0.value) }, uniquingKeysWith: { _, last in last })
|
|
303
|
+
let stateRegisters = Set(parsedState.map(\.register))
|
|
304
|
+
for r in [output, outputRight].compactMap({ $0 }) + (masterParts?.outputs ?? []) where stateRegisters.contains(r) {
|
|
305
|
+
throw DSPProgramError("output register \(r) is state")
|
|
306
|
+
}
|
|
307
|
+
func serial(_ key: String) throws -> Range<Int>? {
|
|
308
|
+
guard let value = root[key] else { return nil }
|
|
309
|
+
guard let pair = value as? [Any], pair.count == 2, let from = Self.int(pair[0]), let to = Self.int(pair[1]), from >= 0, from <= to else {
|
|
310
|
+
throw DSPProgramError("\(key) is not [from, to)")
|
|
311
|
+
}
|
|
312
|
+
return from..<to
|
|
313
|
+
}
|
|
314
|
+
func code(_ key: String, serial: Range<Int>?) throws -> [Int32] {
|
|
315
|
+
guard let words = root[key] as? [Any] else { throw DSPProgramError("\(key) must be an array") }
|
|
316
|
+
let code = try words.map { word -> Int32 in
|
|
317
|
+
guard let n = Self.int(word), n >= Int(Int32.min), n <= Int(Int32.max) else { throw DSPProgramError("\(key): a word is not a whole number") }
|
|
318
|
+
return Int32(n)
|
|
319
|
+
}
|
|
320
|
+
if let serial, serial.upperBound > code.count { throw DSPProgramError("\(key)'s serial part is not [from, to) within it") }
|
|
321
|
+
var boundaries = 0
|
|
322
|
+
var pc = 0
|
|
323
|
+
while pc < code.count {
|
|
324
|
+
if let serial { boundaries += (pc == serial.lowerBound ? 1 : 0) + (pc == serial.upperBound ? 1 : 0) }
|
|
325
|
+
guard let op = DSPOpcode(rawValue: code[pc]) else { throw DSPProgramError("\(key): unknown opcode \(code[pc]) at \(pc)") }
|
|
326
|
+
let length = 2 + op.arity + (op.builtin == nil ? 0 : 1)
|
|
327
|
+
guard pc + length <= code.count else { throw DSPProgramError("\(key): \(op) at \(pc) is cut off") }
|
|
328
|
+
for i in 1..<(2 + op.arity) { _ = try register(Int(code[pc + i]), "\(key) \(op) at \(pc)") }
|
|
329
|
+
// State is a voice's value, not a frame's: only the part that runs frame by frame reads or writes it.
|
|
330
|
+
let inSerial = serial?.contains(pc) ?? false
|
|
331
|
+
if !inSerial && (1..<(2 + op.arity)).contains(where: { stateRegisters.contains(Int(code[pc + $0])) }) {
|
|
332
|
+
throw DSPProgramError("\(key): \(op) at \(pc) reads or writes state outside the serial part")
|
|
333
|
+
}
|
|
334
|
+
if let builtin = op.builtin {
|
|
335
|
+
let instance = Int(code[pc + length - 1])
|
|
336
|
+
guard instances.indices.contains(instance), instances[instance] == builtin else {
|
|
337
|
+
throw DSPProgramError("\(key): \(op) at \(pc) names instance \(instance), not a \(builtin)")
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
// What sizes memory comes from a constant register.
|
|
341
|
+
if op == .wavetable || op == .tableread || op == .shape || op == .play || op == .loop || op == .grains {
|
|
342
|
+
let (count, what) = op == .wavetable || op == .tableread ? (parsedTables.count, "a table") : op == .shape ? (parsedShapes.count, "a shape") : (parsedSamples.count, "a sample")
|
|
343
|
+
guard let index = constantValues[Int(code[pc + 2])], index.rounded() == index, index >= 0, index < Float(count) else {
|
|
344
|
+
throw DSPProgramError("\(key): \(op) at \(pc) does not name \(what)")
|
|
345
|
+
}
|
|
346
|
+
} else if op == .modsum {
|
|
347
|
+
guard let index = constantValues[Int(code[pc + 2])], index.rounded() == index, index >= 0, index < Float(parsedRoutes.count) else {
|
|
348
|
+
throw DSPProgramError("\(key): modsum at \(pc) does not name a route list")
|
|
349
|
+
}
|
|
350
|
+
// It sums into its destination route by route, so the destination is none of the registers it reads.
|
|
351
|
+
guard !parsedRoutes[Int(index)].contains(Int(code[pc + 1])) else {
|
|
352
|
+
throw DSPProgramError("\(key): modsum at \(pc) writes a register its routes read")
|
|
353
|
+
}
|
|
354
|
+
guard inSerial || !parsedRoutes[Int(index)].contains(where: { stateRegisters.contains($0) }) else {
|
|
355
|
+
throw DSPProgramError("\(key): modsum at \(pc) reads state outside the serial part")
|
|
356
|
+
}
|
|
357
|
+
} else if let longestAt = op.longest {
|
|
358
|
+
guard let longest = constantValues[Int(code[pc + 1 + longestAt])], longest > 0, longest <= Self.delaySecondsMax else {
|
|
359
|
+
throw DSPProgramError("\(key): \(op) at \(pc)'s longest time is not a constant up to \(Self.delaySecondsMax) s")
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
pc += length
|
|
363
|
+
}
|
|
364
|
+
if let serial {
|
|
365
|
+
boundaries += (pc == serial.lowerBound ? 1 : 0) + (pc == serial.upperBound ? 1 : 0)
|
|
366
|
+
guard boundaries == 2 else { throw DSPProgramError("\(key)'s serial part does not fall between instructions") }
|
|
367
|
+
}
|
|
368
|
+
return code
|
|
369
|
+
}
|
|
370
|
+
self.voices = voices
|
|
371
|
+
self.mode = mode
|
|
372
|
+
self.registers = registers
|
|
373
|
+
self.constants = parsedConstants
|
|
374
|
+
self.parameters = parsedParameters
|
|
375
|
+
self.state = parsedState
|
|
376
|
+
self.instances = instances
|
|
377
|
+
self.tables = parsedTables
|
|
378
|
+
self.shapes = parsedShapes
|
|
379
|
+
self.samples = parsedSamples
|
|
380
|
+
self.routes = parsedRoutes
|
|
381
|
+
self.curves = parsedCurves
|
|
382
|
+
self.block = try code("block", serial: nil)
|
|
383
|
+
let sampleSerial = try serial("serial")
|
|
384
|
+
self.serial = sampleSerial
|
|
385
|
+
self.sample = try code("sample", serial: sampleSerial)
|
|
386
|
+
self.output = output
|
|
387
|
+
self.outputRight = outputRight
|
|
388
|
+
let masterSerial = try serial("masterSerial")
|
|
389
|
+
self.master = try masterParts.map { Master(code: try code("master", serial: masterSerial), serial: masterSerial, inputs: $0.inputs, outputs: $0.outputs) }
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
private static func int(_ value: Any?) -> Int? {
|
|
393
|
+
guard let number = value as? NSNumber else { return nil }
|
|
394
|
+
let double = number.doubleValue
|
|
395
|
+
guard double.rounded() == double, abs(double) < 1e12 else { return nil }
|
|
396
|
+
return Int(double)
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
public struct DSPProgramError: Error, CustomStringConvertible, Equatable {
|
|
401
|
+
public let description: String
|
|
402
|
+
public init(_ detail: String) { description = "invalidProgram: \(detail)" }
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
/// The stateful builtins, by the name a program's `instances` gives them.
|
|
406
|
+
public enum DSPBuiltin: String, Sendable, CaseIterable {
|
|
407
|
+
case phasor, sine, saw, square, triangle, noise, adsr, lowpass, highpass, bandpass, wavetable, delay, play, grains, smooth, hold
|
|
408
|
+
case onepole, dcblock, comb, allpass, tableread, loop
|
|
409
|
+
|
|
410
|
+
/// Words of state per lane (program.ts BUILTIN_STATE): a delay's, a comb's and an all-pass's samples are a
|
|
411
|
+
/// buffer of their own.
|
|
412
|
+
var words: Int {
|
|
413
|
+
switch self {
|
|
414
|
+
case .phasor, .sine, .saw, .square, .triangle, .noise, .delay, .allpass: 1
|
|
415
|
+
case .hold, .dcblock, .comb: 2
|
|
416
|
+
// A table read's frequency last seen and the level it reads at that frequency.
|
|
417
|
+
case .tableread: 2
|
|
418
|
+
// A one-pole's place, then the cutoff last seen and its coefficient.
|
|
419
|
+
case .onepole: 3
|
|
420
|
+
// A wavetable's phase, then the frequency last seen and the level it reads at that frequency.
|
|
421
|
+
case .play, .loop, .smooth, .wavetable: 3
|
|
422
|
+
// An envelope's stage, level, trigger and gate before, its decay and release times and coefficients, then
|
|
423
|
+
// (with curves) what it put out last and where its release started.
|
|
424
|
+
case .adsr: 10
|
|
425
|
+
case .lowpass, .highpass, .bandpass: 8
|
|
426
|
+
case .grains: 2 + 4 * 8
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
/// The opcodes, v1: the numbers are the wire format (program.ts OPCODES).
|
|
432
|
+
public enum DSPOpcode: Int32, Sendable, CaseIterable {
|
|
433
|
+
case mov = 0, add, sub, mul, div, mod, neg, min, max, abs, floor, ceil, round, sqrt, sin, cos, tan, exp, log, pow, tanh
|
|
434
|
+
case lt = 21, le, gt, ge, eq, ne, and, or, not, select, clamp, mix, wrap, sign, atan
|
|
435
|
+
case phasor = 40, sine, saw, square, triangle, noise, adsr, lowpass, highpass, bandpass, wavetable, delay, shape, play, grains, smooth, hold, modsum
|
|
436
|
+
/// The superinstructions the compiler fuses: a × b + c and (a + b) × c.
|
|
437
|
+
case madd = 58, amul
|
|
438
|
+
/// The rest of Math, and softclip.
|
|
439
|
+
case log2 = 60, log10, exp2, trunc, asin, acos, atan2, sinh, cosh, cbrt, hypot, expm1, log1p, softclip
|
|
440
|
+
case onepole = 74, dcblock, comb, allpass, tableread, loop
|
|
441
|
+
|
|
442
|
+
/// Operands after the destination.
|
|
443
|
+
public var arity: Int {
|
|
444
|
+
switch self {
|
|
445
|
+
case .noise: 0
|
|
446
|
+
case .modsum: 1
|
|
447
|
+
case .mov, .neg, .abs, .floor, .ceil, .round, .sqrt, .sin, .cos, .tan, .exp, .log, .tanh, .not, .wrap, .sign, .atan,
|
|
448
|
+
.phasor, .sine, .saw, .triangle, .log2, .log10, .exp2, .trunc, .asin, .acos, .sinh, .cosh, .cbrt, .expm1, .log1p,
|
|
449
|
+
.softclip, .dcblock: 1
|
|
450
|
+
case .select, .clamp, .mix, .lowpass, .highpass, .bandpass, .wavetable, .delay, .play, .loop, .madd, .amul: 3
|
|
451
|
+
case .allpass, .tableread: 4
|
|
452
|
+
case .adsr, .grains, .comb: 5
|
|
453
|
+
default: 2
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
/// The builtin whose state it keeps, for a stateful one (`shape` keeps none).
|
|
458
|
+
public var builtin: DSPBuiltin? {
|
|
459
|
+
switch self {
|
|
460
|
+
case .play: .play
|
|
461
|
+
case .grains: .grains
|
|
462
|
+
case .smooth: .smooth
|
|
463
|
+
case .hold: .hold
|
|
464
|
+
case .onepole: .onepole
|
|
465
|
+
case .dcblock: .dcblock
|
|
466
|
+
case .comb: .comb
|
|
467
|
+
case .allpass: .allpass
|
|
468
|
+
case .tableread: .tableread
|
|
469
|
+
case .loop: .loop
|
|
470
|
+
default: rawValue >= 40 && rawValue <= 51 ? DSPBuiltin.allCases[Int(rawValue) - 40] : nil
|
|
471
|
+
}
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
/// A delay line's operand (after the destination) that names its longest time: a delay's, a comb's, an all-pass's.
|
|
475
|
+
public var longest: Int? {
|
|
476
|
+
switch self {
|
|
477
|
+
case .delay: 3
|
|
478
|
+
case .comb: 5
|
|
479
|
+
case .allpass: 4
|
|
480
|
+
default: nil
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
// swift-tools-version: 6.0
|
|
2
|
+
import PackageDescription
|
|
3
|
+
|
|
4
|
+
// SwiftBrowserLocalModel: an open language model on the device, through
|
|
5
|
+
// llama.cpp, for the `LanguageModelSession`s of a SwiftBrowser Native app
|
|
6
|
+
// where Apple Intelligence cannot answer (an iPhone before the 15 Pro, Apple
|
|
7
|
+
// Intelligence off, a system before iOS 26). A package of its own, so only an
|
|
8
|
+
// app that adds it fetches, builds and links llama.cpp; it installs itself as
|
|
9
|
+
// `LanguageModels.fallback` (SwiftBrowserNative, LanguageModelProvider.swift).
|
|
10
|
+
//
|
|
11
|
+
// llama.cpp is its release XCFramework, b10456: the newest whose release
|
|
12
|
+
// carries the iOS Simulator beside devices and macOS (later ones ship devices
|
|
13
|
+
// and macOS only). SwiftPM fetches it once (286 MB, tvOS and visionOS
|
|
14
|
+
// included) and caches it; an iPhone app links about 6 MB of it.
|
|
15
|
+
let package = Package(
|
|
16
|
+
name: "SwiftBrowserLocalModel",
|
|
17
|
+
platforms: [.iOS(.v17), .macOS(.v14)],
|
|
18
|
+
products: [
|
|
19
|
+
.library(name: "SwiftBrowserLocalModel", targets: ["SwiftBrowserLocalModel"]),
|
|
20
|
+
],
|
|
21
|
+
dependencies: [
|
|
22
|
+
.package(path: "../.."),
|
|
23
|
+
],
|
|
24
|
+
targets: [
|
|
25
|
+
.binaryTarget(
|
|
26
|
+
name: "llama",
|
|
27
|
+
url: "https://github.com/ggml-org/llama.cpp/releases/download/b10456/llama-b10456-xcframework.zip",
|
|
28
|
+
checksum: "0223bedd0a01232399d943dcb72bc227882bc90df98e29d7a92343531a88cc02"
|
|
29
|
+
),
|
|
30
|
+
.target(
|
|
31
|
+
name: "SwiftBrowserLocalModel",
|
|
32
|
+
dependencies: [.product(name: "SwiftBrowserNative", package: "swift"), "llama"]
|
|
33
|
+
),
|
|
34
|
+
]
|
|
35
|
+
)
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
# SwiftBrowserLocalModel
|
|
2
|
+
|
|
3
|
+
An open language model on the device, run with
|
|
4
|
+
[llama.cpp](https://github.com/ggml-org/llama.cpp), for a SwiftBrowser Native
|
|
5
|
+
app's `LanguageModelSession`s where Apple Intelligence cannot answer: an
|
|
6
|
+
iPhone before the 15 Pro, Apple Intelligence turned off, a system before
|
|
7
|
+
iOS 26. Where Apple Intelligence answers, it still does.
|
|
8
|
+
|
|
9
|
+
It is a package of its own so that only an app that adds it fetches, builds
|
|
10
|
+
and links llama.cpp. @swiftbrowser/native ships it beside the core
|
|
11
|
+
(`node_modules/@swiftbrowser/native/swift/Packages/SwiftBrowserLocalModel`;
|
|
12
|
+
it depends on `SwiftBrowserNative` at `../..`, here and there). Link it into
|
|
13
|
+
the app's Xcode project with
|
|
14
|
+
|
|
15
|
+
```sh
|
|
16
|
+
npx swiftbrowser native add local-model # or native init --with local-model
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
and install a model at launch:
|
|
20
|
+
|
|
21
|
+
```swift
|
|
22
|
+
import SwiftBrowserLocalModel
|
|
23
|
+
|
|
24
|
+
init() {
|
|
25
|
+
LocalLanguageModel.install([.llama3_2_1B, .qwen2_5_1_5B, .llama3_2_3B])
|
|
26
|
+
}
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
With several, the first is the default and availability lists them as
|
|
30
|
+
`models` for the viewer to choose from (`selectModel(id)`), the choice
|
|
31
|
+
remembered across launches; each downloads on its own.
|
|
32
|
+
|
|
33
|
+
On the JavaScript side, `useSystemLanguageModel()` reports `modelNotReady`
|
|
34
|
+
with `download: { size, downloaded: false }` until the viewer asks for the
|
|
35
|
+
model: show the size and call `SystemLanguageModel.default.download()` from
|
|
36
|
+
a button. The download's `progress` follows, then `available`, and
|
|
37
|
+
`LanguageModelSession` answers with it. `deleteDownload()` stops a download
|
|
38
|
+
or takes the model off the device (the Showcase's Chat has both).
|
|
39
|
+
|
|
40
|
+
- **Tools** a session is given are offered in its instructions, and a reply
|
|
41
|
+
that is just `{"tool": name, "arguments": {...}}` is run as a call, its
|
|
42
|
+
result handed back for the model to go on. Small models call tools less
|
|
43
|
+
reliably than Apple Intelligence does; Qwen 2.5 1.5B is the steadiest here.
|
|
44
|
+
- **The models**: `.llama3_2_1B` (808 MB, fastest), `.qwen2_5_1_5B` (1.1 GB,
|
|
45
|
+
Apache 2.0, best with tools), `.llama3_2_3B` (2.0 GB, most capable; a phone
|
|
46
|
+
with 6 GB of memory runs it near its limit).
|
|
47
|
+
- **The model** is a GGUF file, fetched when the app calls `download()`, kept
|
|
48
|
+
in Application Support's
|
|
49
|
+
`SwiftBrowserLocalModel` folder and excluded from backups. `.llama3_2_1B`
|
|
50
|
+
is Meta's Llama 3.2 1B Instruct at 4 bits (808 MB; Meta's Llama 3.2
|
|
51
|
+
Community License applies); any other is a `LocalLanguageModel.Model`
|
|
52
|
+
with its URL, size and context length. `allowsCellularAccess: false`
|
|
53
|
+
waits for Wi-Fi.
|
|
54
|
+
- **Running it**: loaded once on the first prompt or `prewarm()`, on the GPU
|
|
55
|
+
with Metal, on a queue of its own. A session keeps its messages, formatted
|
|
56
|
+
with the model's chat template (ChatML for a model without one); the
|
|
57
|
+
context keeps the tokens of the last conversation, so a prompt that
|
|
58
|
+
continues it decodes only what is new. A conversation longer than the
|
|
59
|
+
context fails as `exceededContextWindowSize`.
|
|
60
|
+
- **Size**: llama.cpp is its release XCFramework, b10456, the newest whose
|
|
61
|
+
release has the iOS Simulator beside devices and macOS. SwiftPM downloads
|
|
62
|
+
it once (286 MB, with tvOS and visionOS) and caches it; an iPhone app
|
|
63
|
+
links about 6 MB.
|
|
64
|
+
|
|
65
|
+
The tests (`swift test`, macOS) run llama.cpp on the CPU with a 1.2 MB toy
|
|
66
|
+
model they download.
|