@bendyline/gezel-capacitor 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/Package.swift +26 -0
- package/README.md +230 -2
- package/android/build.gradle +30 -0
- package/android/src/main/AndroidManifest.xml +1 -0
- package/android/src/main/java/com/bendyline/gezel/capacitor/GezelKnowledgePlugin.java +153 -0
- package/android/src/main/java/com/bendyline/gezel/capacitor/GezelOfflineSpeechPlugin.java +216 -0
- package/android/src/main/java/com/bendyline/gezel/capacitor/GezelRuntimePlugin.java +84 -0
- package/android/src/main/java/com/bendyline/gezel/capacitor/KnowledgeFiles.java +101 -0
- package/android/src/main/java/com/bendyline/gezel/capacitor/SpeechModels.java +230 -0
- package/android/src/main/java/com/bendyline/gezel/speech/SpeechRuntime.java +38 -0
- package/dist/index.d.ts +561 -0
- package/dist/index.js +7512 -0
- package/dist/knowledge-worker.js +211 -0
- package/dist/licenses/npm/manifest.json +76 -0
- package/dist/licenses/npm/texts/1583809bf1155510d2961f94aa9bca6dc4b44b6f59fd773de33bd6c20860d781.txt +29 -0
- package/dist/licenses/npm/texts/1cd39aa4a566eba49eb206b969ef423295e79b50503df567bbdb2e8877d78892.txt +8 -0
- package/dist/licenses/npm/texts/3f1189b28e3866e0d979968d466b78f813f76827cfdca1fbb124cc0a5c8841f8.txt +21 -0
- package/dist/licenses/npm/texts/5b37224c080cdcc97c871ada971c224e9926370fe74f11b539aa1cf9f3b1aca1.txt +21 -0
- package/dist/licenses/npm/texts/c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4.txt +201 -0
- package/dist/sqlite3.wasm +0 -0
- package/embedding-manifest.json +210 -0
- package/ios/Sources/GezelCapacitor/GezelKnowledgePlugin.swift +200 -0
- package/ios/Sources/GezelCapacitor/GezelOfflineSpeechPlugin.swift +484 -0
- package/ios/Sources/GezelCapacitor/GezelRuntimePlugin.swift +102 -0
- package/kokoro-lexicon/LICENSE +202 -0
- package/kokoro-lexicon/lexicon-gb-en.txt.gz +0 -0
- package/kokoro-lexicon/lexicon-us-en.txt.gz +0 -0
- package/native/android/LICENSE-gezel.txt +21 -0
- package/native/android/PUBLIC-DISTRIBUTION.md +97 -0
- package/native/android/THIRD_PARTY_LICENSES/LICENSE-arm-optimized-routines-MIT-OR-Apache-2.0.txt +249 -0
- package/native/android/THIRD_PARTY_LICENSES/LICENSE-cpp-httplib-MIT.txt +22 -0
- package/native/android/THIRD_PARTY_LICENSES/LICENSE-fp16-MIT.txt +11 -0
- package/native/android/THIRD_PARTY_LICENSES/LICENSE-ggml-MIT.txt +21 -0
- package/native/android/THIRD_PARTY_LICENSES/LICENSE-llama-cpp-llamafile-sgemm-MIT.txt +21 -0
- package/native/android/THIRD_PARTY_LICENSES/LICENSE-llama-cpp-openvino-APACHE.txt +201 -0
- package/native/android/THIRD_PARTY_LICENSES/LICENSE-llama-cpp-pillow-MIT-CMU.txt +30 -0
- package/native/android/THIRD_PARTY_LICENSES/LICENSE-llama-cpp-rotate-bits-MIT.txt +21 -0
- package/native/android/THIRD_PARTY_LICENSES/LICENSE-llama-cpp-unicode-data-Unicode-3.0.txt +39 -0
- package/native/android/THIRD_PARTY_LICENSES/LICENSE-nlohmann-json-MIT.txt +21 -0
- package/native/android/THIRD_PARTY_LICENSES/LICENSE-nvidia-cub-BSD-3-Clause.txt +24 -0
- package/native/android/THIRD_PARTY_LICENSES/LICENSE-vulkan-headers-Apache-2.0-OR-MIT.txt +33 -0
- package/native/android/THIRD_PARTY_LICENSES/LICENSE-vulkan-loader-APACHE.txt +201 -0
- package/native/android/THIRD_PARTY_LICENSES/LICENSE-vulkan-loader-cjson-MIT.txt +32 -0
- package/native/android/THIRD_PARTY_LICENSES/LICENSE-yarn-MIT.txt +21 -0
- package/native/android/THIRD_PARTY_LICENSES/manifest.json +21 -0
- package/native/android/engine-manifest.json +88 -0
- package/native/android/maven/com/bendyline/gezel/gezel-llama/0.1.57/gezel-llama-0.1.57.aar +0 -0
- package/native/android/maven/com/bendyline/gezel/gezel-llama/0.1.57/gezel-llama-0.1.57.aar.sha256 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-llama/0.1.57/gezel-llama-0.1.57.pom +8 -0
- package/native/android/maven/com/bendyline/gezel/gezel-llama/0.1.57/gezel-llama-0.1.57.pom.sha256 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57-sources.jar +0 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57-sources.jar.md5 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57-sources.jar.sha1 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57-sources.jar.sha256 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57-sources.jar.sha512 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.aar +0 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.aar.md5 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.aar.sha1 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.aar.sha256 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.aar.sha512 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.module +118 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.module.md5 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.module.sha1 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.module.sha256 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.module.sha512 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.pom +51 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.pom.md5 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.pom.sha1 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.pom.sha256 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.pom.sha512 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/maven-metadata.xml +13 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/maven-metadata.xml.md5 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/maven-metadata.xml.sha1 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/maven-metadata.xml.sha256 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/maven-metadata.xml.sha512 +1 -0
- package/native/android/sdk-manifest.json +98 -0
- package/native/ios/GezelLlama.xcframework/Info.plist +47 -0
- package/native/ios/GezelLlama.xcframework/ios-arm64/Headers/gezel_llama.h +290 -0
- package/native/ios/GezelLlama.xcframework/ios-arm64/Headers/module.modulemap +9 -0
- package/native/ios/GezelLlama.xcframework/ios-arm64/libGezelLlama.a +0 -0
- package/native/ios/GezelLlama.xcframework/ios-arm64-simulator/Headers/gezel_llama.h +290 -0
- package/native/ios/GezelLlama.xcframework/ios-arm64-simulator/Headers/module.modulemap +9 -0
- package/native/ios/GezelLlama.xcframework/ios-arm64-simulator/libGezelLlama.a +0 -0
- package/native/ios/LICENSE-gezel.txt +21 -0
- package/native/ios/LICENSE-ggml.txt +21 -0
- package/native/ios/LICENSE-llama-cpp.txt +21 -0
- package/native/ios/PUBLIC-DISTRIBUTION.md +97 -0
- package/native/ios/Package.swift +17 -0
- package/native/ios/Sources/GezelModelStorage/MobileModelStore.swift +286 -0
- package/native/ios/Sources/GezelModelStorage/ModelDownloadHTTP.swift +55 -0
- package/native/ios/Sources/GezelModelStorage/ModelDownloadSource.swift +46 -0
- package/native/ios/Sources/GezelModelStorage/ModelDownloads.swift +289 -0
- package/native/ios/Sources/GezelModelStorage/PrivacyInfo.xcprivacy +32 -0
- package/native/ios/Sources/GezelRuntime/AppleFoundationProvider.swift +292 -0
- package/native/ios/Sources/GezelRuntime/AppleNativeTools.swift +152 -0
- package/native/ios/Sources/GezelRuntime/GezelNativeRuntime.swift +1349 -0
- package/native/ios/Sources/GezelRuntime/NativeCall.swift +35 -0
- package/native/ios/Sources/GezelRuntime/PrivacyInfo.xcprivacy +31 -0
- package/native/ios/THIRD_PARTY_LICENSES/LICENSE-arm-optimized-routines-MIT-OR-Apache-2.0.txt +249 -0
- package/native/ios/THIRD_PARTY_LICENSES/LICENSE-cpp-httplib-MIT.txt +22 -0
- package/native/ios/THIRD_PARTY_LICENSES/LICENSE-fp16-MIT.txt +11 -0
- package/native/ios/THIRD_PARTY_LICENSES/LICENSE-ggml-MIT.txt +21 -0
- package/native/ios/THIRD_PARTY_LICENSES/LICENSE-llama-cpp-llamafile-sgemm-MIT.txt +21 -0
- package/native/ios/THIRD_PARTY_LICENSES/LICENSE-llama-cpp-openvino-APACHE.txt +201 -0
- package/native/ios/THIRD_PARTY_LICENSES/LICENSE-llama-cpp-pillow-MIT-CMU.txt +30 -0
- package/native/ios/THIRD_PARTY_LICENSES/LICENSE-llama-cpp-rotate-bits-MIT.txt +21 -0
- package/native/ios/THIRD_PARTY_LICENSES/LICENSE-llama-cpp-unicode-data-Unicode-3.0.txt +39 -0
- package/native/ios/THIRD_PARTY_LICENSES/LICENSE-nlohmann-json-MIT.txt +21 -0
- package/native/ios/THIRD_PARTY_LICENSES/LICENSE-nvidia-cub-BSD-3-Clause.txt +24 -0
- package/native/ios/THIRD_PARTY_LICENSES/LICENSE-vulkan-headers-Apache-2.0-OR-MIT.txt +33 -0
- package/native/ios/THIRD_PARTY_LICENSES/LICENSE-vulkan-loader-APACHE.txt +201 -0
- package/native/ios/THIRD_PARTY_LICENSES/LICENSE-vulkan-loader-cjson-MIT.txt +32 -0
- package/native/ios/THIRD_PARTY_LICENSES/LICENSE-yarn-MIT.txt +21 -0
- package/native/ios/THIRD_PARTY_LICENSES/manifest.json +21 -0
- package/native/ios/engine-manifest.json +102 -0
- package/native/ios/sdk-manifest.json +89 -0
- package/package.json +94 -3
- package/scripts/embedding-package.d.mts +23 -0
- package/scripts/embedding-package.mjs +185 -0
- package/scripts/stage-native.mjs +168 -0
|
@@ -0,0 +1,1349 @@
|
|
|
1
|
+
import Foundation
|
|
2
|
+
import ImageIO
|
|
3
|
+
import UIKit
|
|
4
|
+
import GezelLlama
|
|
5
|
+
import os
|
|
6
|
+
|
|
7
|
+
private final class NativeChatStream {
|
|
8
|
+
weak var plugin: GezelNativeRuntime?
|
|
9
|
+
let requestId: String
|
|
10
|
+
var text = ""
|
|
11
|
+
/** Set on the first chunk, which is when decoding began. */
|
|
12
|
+
var generating = false
|
|
13
|
+
init(plugin: GezelNativeRuntime, requestId: String) {
|
|
14
|
+
self.plugin = plugin
|
|
15
|
+
self.requestId = requestId
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
private func receiveLlamaChunk(_ bytes: UnsafePointer<CChar>?, _ length: Int, _ context: UnsafeMutableRawPointer?) -> Int32 {
|
|
20
|
+
guard let bytes, let context else { return 1 }
|
|
21
|
+
let stream = Unmanaged<NativeChatStream>.fromOpaque(context).takeUnretainedValue()
|
|
22
|
+
guard let plugin = stream.plugin, !plugin.isCancelled(stream.requestId) else { return 1 }
|
|
23
|
+
let delta = String(decoding: UnsafeRawBufferPointer(start: bytes, count: length), as: UTF8.self)
|
|
24
|
+
stream.text.append(delta)
|
|
25
|
+
let first = !stream.generating
|
|
26
|
+
stream.generating = true
|
|
27
|
+
DispatchQueue.main.async {
|
|
28
|
+
if first { plugin.notifyListeners("enginePhase", data: ["requestId": stream.requestId, "phase": "generating"]) }
|
|
29
|
+
plugin.notifyListeners("chatDelta", data: ["requestId": stream.requestId, "delta": delta])
|
|
30
|
+
}
|
|
31
|
+
return 0
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Chat chunks travel to JavaScript in small batches: one bridge event per token
|
|
36
|
+
* costs more than the token, and the order must hold. Touched only on the
|
|
37
|
+
* inference thread; flushed about every 24 ms while tokens stream, and once
|
|
38
|
+
* more when the request ends.
|
|
39
|
+
*/
|
|
40
|
+
private final class NativeChatBatch {
|
|
41
|
+
weak var plugin: GezelNativeRuntime?
|
|
42
|
+
let requestId: String
|
|
43
|
+
private var pending: [String] = []
|
|
44
|
+
private var flushedAt = DispatchTime.now().uptimeNanoseconds
|
|
45
|
+
var generating = false
|
|
46
|
+
init(plugin: GezelNativeRuntime, requestId: String) {
|
|
47
|
+
self.plugin = plugin
|
|
48
|
+
self.requestId = requestId
|
|
49
|
+
}
|
|
50
|
+
func add(_ chunk: String) {
|
|
51
|
+
pending.append(chunk)
|
|
52
|
+
if DispatchTime.now().uptimeNanoseconds - flushedAt >= 24_000_000 { flush() }
|
|
53
|
+
}
|
|
54
|
+
func flush() {
|
|
55
|
+
flushedAt = DispatchTime.now().uptimeNanoseconds
|
|
56
|
+
guard !pending.isEmpty, let plugin else { return }
|
|
57
|
+
let chunks = pending, requestId = requestId
|
|
58
|
+
pending.removeAll()
|
|
59
|
+
DispatchQueue.main.async { plugin.notifyListeners("chatChunk", data: ["requestId": requestId, "chunks": chunks]) }
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/// The vision describer's text, collected without streaming it to JavaScript.
|
|
64
|
+
private final class NativeTextCollector {
|
|
65
|
+
weak var plugin: GezelNativeRuntime?
|
|
66
|
+
let requestId: String
|
|
67
|
+
var text = ""
|
|
68
|
+
init(plugin: GezelNativeRuntime, requestId: String) {
|
|
69
|
+
self.plugin = plugin
|
|
70
|
+
self.requestId = requestId
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
private func receiveLlamaText(_ bytes: UnsafePointer<CChar>?, _ length: Int, _ context: UnsafeMutableRawPointer?) -> Int32 {
|
|
75
|
+
guard let bytes, let context else { return 1 }
|
|
76
|
+
let collector = Unmanaged<NativeTextCollector>.fromOpaque(context).takeUnretainedValue()
|
|
77
|
+
guard let plugin = collector.plugin, !plugin.isCancelled(collector.requestId) else { return 1 }
|
|
78
|
+
collector.text.append(String(decoding: UnsafeRawBufferPointer(start: bytes, count: length), as: UTF8.self))
|
|
79
|
+
return 0
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
private func receiveLlamaJSON(_ bytes: UnsafePointer<CChar>?, _ length: Int, _ context: UnsafeMutableRawPointer?) -> Int32 {
|
|
83
|
+
guard let bytes, let context else { return 1 }
|
|
84
|
+
let batch = Unmanaged<NativeChatBatch>.fromOpaque(context).takeUnretainedValue()
|
|
85
|
+
guard let plugin = batch.plugin, !plugin.isCancelled(batch.requestId) else { return 1 }
|
|
86
|
+
if !batch.generating {
|
|
87
|
+
batch.generating = true
|
|
88
|
+
plugin.notifyPhase(batch.requestId, "generating")
|
|
89
|
+
}
|
|
90
|
+
batch.add(String(decoding: UnsafeRawBufferPointer(start: bytes, count: length), as: UTF8.self))
|
|
91
|
+
return plugin.isCancelled(batch.requestId) ? 1 : 0
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
#if canImport(GezelModelStorage)
|
|
95
|
+
import GezelModelStorage
|
|
96
|
+
#endif
|
|
97
|
+
|
|
98
|
+
/// One process-owned provider/model/lifecycle host shared by native and Capacitor clients.
|
|
99
|
+
public final class GezelNativeRuntime: @unchecked Sendable {
|
|
100
|
+
private static let sharedLock = NSLock()
|
|
101
|
+
private static var sharedRuntime: GezelNativeRuntime?
|
|
102
|
+
private static var sharedRoot: URL?
|
|
103
|
+
public static func shared(root: URL) throws -> GezelNativeRuntime {
|
|
104
|
+
sharedLock.lock(); defer { sharedLock.unlock() }
|
|
105
|
+
let canonical = try MobileModelStore.validatedRoot(root)
|
|
106
|
+
if let existing = sharedRuntime {
|
|
107
|
+
guard sharedRoot == canonical else { throw MobileInferenceError(code: "ALREADY_CONFIGURED", message: "The process runtime already owns another model root") }
|
|
108
|
+
return existing
|
|
109
|
+
}
|
|
110
|
+
let runtime = GezelNativeRuntime(root: canonical)
|
|
111
|
+
sharedRoot = canonical; sharedRuntime = runtime
|
|
112
|
+
return runtime
|
|
113
|
+
}
|
|
114
|
+
public static func shared() throws -> GezelNativeRuntime {
|
|
115
|
+
sharedLock.lock(); let existing = sharedRuntime; sharedLock.unlock()
|
|
116
|
+
if let existing { return existing }
|
|
117
|
+
let support = try FileManager.default.url(for: .applicationSupportDirectory, in: .userDomainMask, appropriateFor: nil, create: true)
|
|
118
|
+
return try shared(root: support.appendingPathComponent("Gezel", isDirectory: true))
|
|
119
|
+
}
|
|
120
|
+
private var listeners: [UUID: (String, [String: Any]) -> Void] = [:]
|
|
121
|
+
public func listen(_ callback: @escaping (String, [String: Any]) -> Void) -> UUID {
|
|
122
|
+
operationLock.lock(); defer { operationLock.unlock() }
|
|
123
|
+
let id = UUID(); listeners[id] = callback; return id
|
|
124
|
+
}
|
|
125
|
+
public func removeListener(_ id: UUID) {
|
|
126
|
+
operationLock.lock(); defer { operationLock.unlock() }; listeners.removeValue(forKey: id)
|
|
127
|
+
}
|
|
128
|
+
/**
|
|
129
|
+
* Model-loading and prompt-processing progress for the status pill, polled
|
|
130
|
+
* from the cancellation timer. Reports only changes.
|
|
131
|
+
*/
|
|
132
|
+
fileprivate func reportProgress(_ requestId: String, last: inout (phase: UInt32, value: UInt32)) {
|
|
133
|
+
guard let engine else { return }
|
|
134
|
+
var progress = gezel_llama_progress()
|
|
135
|
+
progress.struct_size = UInt32(MemoryLayout<gezel_llama_progress>.size)
|
|
136
|
+
progress.abi_version = gezel_llama_abi_version()
|
|
137
|
+
guard gezel_llama_get_progress(engine, &progress) == 0 else { return }
|
|
138
|
+
let value: UInt32
|
|
139
|
+
switch progress.phase {
|
|
140
|
+
case 1: value = UInt32(progress.load_fraction * 1000)
|
|
141
|
+
case 2: value = progress.processed_tokens
|
|
142
|
+
default: return
|
|
143
|
+
}
|
|
144
|
+
guard progress.phase != last.phase || value != last.value else { return }
|
|
145
|
+
last = (progress.phase, value)
|
|
146
|
+
var data: [String: Any] = ["requestId": requestId]
|
|
147
|
+
if progress.phase == 1 {
|
|
148
|
+
data["phase"] = "loading_model"
|
|
149
|
+
data["progress"] = min(1.0, Double(progress.load_fraction))
|
|
150
|
+
} else {
|
|
151
|
+
data["phase"] = "prefill"
|
|
152
|
+
data["promptTokens"] = Int(progress.prompt_tokens)
|
|
153
|
+
data["processedTokens"] = Int(progress.processed_tokens)
|
|
154
|
+
data["reusedTokens"] = Int(progress.reused_tokens)
|
|
155
|
+
if progress.prompt_tokens > 0 {
|
|
156
|
+
data["progress"] = min(1.0, Double(progress.processed_tokens) / Double(progress.prompt_tokens))
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
DispatchQueue.main.async {
|
|
160
|
+
guard !self.isCancelled(requestId) else { return }
|
|
161
|
+
self.notifyListeners("enginePhase", data: data)
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/** Engine phase for the status pill, ordered with `chatDelta` on the main queue. */
|
|
166
|
+
fileprivate func notifyPhase(_ requestId: String, _ phase: String) {
|
|
167
|
+
DispatchQueue.main.async {
|
|
168
|
+
self.notifyListeners("enginePhase", data: ["requestId": requestId, "phase": phase])
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
fileprivate func notifyListeners(_ event: String, data: [String: Any]) {
|
|
173
|
+
operationLock.lock(); let callbacks = Array(listeners.values); operationLock.unlock()
|
|
174
|
+
for callback in callbacks { callback(event, data) }
|
|
175
|
+
}
|
|
176
|
+
private let storageQueue = DispatchQueue(label: "com.bendyline.gezel.mobile.storage")
|
|
177
|
+
private let inferenceQueue = DispatchQueue(label: "com.bendyline.gezel.mobile.inference", qos: .userInitiated)
|
|
178
|
+
private let operationLock = NSLock()
|
|
179
|
+
private var activeId: String?
|
|
180
|
+
private var activeNativeId: UInt64 = 0
|
|
181
|
+
private var nextNativeId: UInt64 = 0
|
|
182
|
+
private var cancelled = false
|
|
183
|
+
private var cancelWaiters: [NativeCall] = []
|
|
184
|
+
private var modelMutation = false
|
|
185
|
+
private var backgroundObserver: NSObjectProtocol?
|
|
186
|
+
private var foregroundObserver: NSObjectProtocol?
|
|
187
|
+
private var backgrounded = false
|
|
188
|
+
private var memoryObserver: NSObjectProtocol?
|
|
189
|
+
private var thermalObserver: NSObjectProtocol?
|
|
190
|
+
private var lastMemoryWarning: TimeInterval = -.infinity
|
|
191
|
+
private var appleTask: Task<Void, Never>?
|
|
192
|
+
/// Native tool calls parked until the app's tool loop completes them.
|
|
193
|
+
private var pendingToolCalls: [String: CheckedContinuation<NativeToolReply, Error>] = [:]
|
|
194
|
+
/// Awake-time start of the model's current stretch of generation. Time the
|
|
195
|
+
/// app spends executing a tool is not the model's and is not charged.
|
|
196
|
+
private var modelSegmentStart: TimeInterval = 0
|
|
197
|
+
private var activeFailure: MobileInferenceError?
|
|
198
|
+
private var releasing = false
|
|
199
|
+
private var releaseScheduled = false
|
|
200
|
+
private var releaseWaiters: [NativeCall] = []
|
|
201
|
+
private var downloads: ModelDownloads?
|
|
202
|
+
private var store: MobileModelStore?
|
|
203
|
+
private var storeError: Error?
|
|
204
|
+
private let engine = gezel_llama_create()
|
|
205
|
+
// Written and read under sizingLock: the model listing runs on the storage
|
|
206
|
+
// queue and fits windows against what the inference queue holds.
|
|
207
|
+
private var loadedModelId: String?
|
|
208
|
+
private var loadedPath: String?
|
|
209
|
+
private var loadedContextSize = 0
|
|
210
|
+
/// Serializes the bridge's dry-run sizing with model loads, and guards the
|
|
211
|
+
/// loaded-model fields and `allocations`.
|
|
212
|
+
private let sizingLock = NSLock()
|
|
213
|
+
/// Bytes per model file and window from the bridge's dry run, or -1 when
|
|
214
|
+
/// llama.cpp cannot load the file at that window. Imported files never
|
|
215
|
+
/// change in place, so an entry cannot go stale.
|
|
216
|
+
private var allocations: [String: Int64] = [:]
|
|
217
|
+
private var fittedWindows: [String: Int] = [:]
|
|
218
|
+
/// The bridge's own buffers beside llama.cpp's: the hybrid/windowed state
|
|
219
|
+
/// checkpoint, token vectors and the reply text.
|
|
220
|
+
private static let bridgeBufferBytes: Int64 = 128 * 1024 * 1024
|
|
221
|
+
/// Windows offered above the 4K floor, largest first. Phones and small
|
|
222
|
+
/// desktops aim for 8K-16K; the floor is what the phone prompt was sized for.
|
|
223
|
+
private static let contextLadder = [16384, 8192]
|
|
224
|
+
private static let floorContext = 4096
|
|
225
|
+
/// Room a larger window must leave in the process allowance, so the chosen
|
|
226
|
+
/// window is not the one that barely fits.
|
|
227
|
+
private static let ladderSpareBytes: Int64 = 256 * 1024 * 1024
|
|
228
|
+
/// A description needs the picture's tokens and a few hundred more.
|
|
229
|
+
private static let describeContext = 4096
|
|
230
|
+
/// The projector's encoder graph and image batch beside the weights;
|
|
231
|
+
/// desktop budgets the same flat amount (core model-fit.ts).
|
|
232
|
+
private static let visionComputeBytes: UInt64 = 384 * 1024 * 1024
|
|
233
|
+
/// Long edge of the pixels handed to the projector, which sizes them again
|
|
234
|
+
/// under its own token cap; more only costs the decode.
|
|
235
|
+
private static let describeMaxEdge = 768
|
|
236
|
+
|
|
237
|
+
private init(root: URL) {
|
|
238
|
+
do {
|
|
239
|
+
store = try MobileModelStore(root: root)
|
|
240
|
+
downloads = try ModelDownloads(store: store!)
|
|
241
|
+
} catch { storeError = error }
|
|
242
|
+
backgroundObserver = NotificationCenter.default.addObserver(forName: UIApplication.didEnterBackgroundNotification, object: nil, queue: .main) { [weak self] _ in
|
|
243
|
+
guard let self else { return }
|
|
244
|
+
self.operationLock.lock(); self.backgrounded = true; self.operationLock.unlock()
|
|
245
|
+
self.downloads?.suspend()
|
|
246
|
+
self.requestRelease(nil)
|
|
247
|
+
self.notifyListeners("appBackground", data: [:])
|
|
248
|
+
}
|
|
249
|
+
foregroundObserver = NotificationCenter.default.addObserver(forName: UIApplication.didBecomeActiveNotification, object: nil, queue: .main) { [weak self] _ in
|
|
250
|
+
guard let self else { return }
|
|
251
|
+
self.operationLock.lock(); self.backgrounded = false; self.operationLock.unlock()
|
|
252
|
+
}
|
|
253
|
+
memoryObserver = NotificationCenter.default.addObserver(forName: UIApplication.didReceiveMemoryWarningNotification, object: nil, queue: .main) { [weak self] _ in
|
|
254
|
+
guard let self else { return }
|
|
255
|
+
self.operationLock.lock()
|
|
256
|
+
self.lastMemoryWarning = ProcessInfo.processInfo.systemUptime
|
|
257
|
+
self.activeFailure = MobileInferenceError(code: "RESOURCE_LIMIT", message: "The device is low on memory. The current model has been released.")
|
|
258
|
+
self.operationLock.unlock()
|
|
259
|
+
self.requestRelease(nil)
|
|
260
|
+
}
|
|
261
|
+
// Only .critical stops a reply mid-generation. At .serious iOS asks apps
|
|
262
|
+
// to scale back and already throttles the clocks itself; aborting there
|
|
263
|
+
// ended every reply of a charging iPhone 14 Pro Max about 20 s in, so
|
|
264
|
+
// no task could finish (2026-09-30).
|
|
265
|
+
thermalObserver = NotificationCenter.default.addObserver(forName: ProcessInfo.thermalStateDidChangeNotification, object: nil, queue: .main) { [weak self] _ in
|
|
266
|
+
guard let self, ProcessInfo.processInfo.thermalState == .critical else { return }
|
|
267
|
+
self.operationLock.lock()
|
|
268
|
+
self.activeFailure = MobileInferenceError(code: "RESOURCE_LIMIT", message: "The device is too warm for local inference. Let it cool down before trying again.")
|
|
269
|
+
self.operationLock.unlock()
|
|
270
|
+
self.requestRelease(nil)
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
deinit {
|
|
276
|
+
downloads?.suspend()
|
|
277
|
+
if let backgroundObserver { NotificationCenter.default.removeObserver(backgroundObserver) }
|
|
278
|
+
if let foregroundObserver { NotificationCenter.default.removeObserver(foregroundObserver) }
|
|
279
|
+
if let memoryObserver { NotificationCenter.default.removeObserver(memoryObserver) }
|
|
280
|
+
if let thermalObserver { NotificationCenter.default.removeObserver(thermalObserver) }
|
|
281
|
+
// Inference closures retain this runtime until its blocking native call
|
|
282
|
+
// returns; no active operation can outlive this engine pointer.
|
|
283
|
+
gezel_llama_destroy(engine)
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
private func withStore(_ call: NativeCall, completion: (() -> Void)? = nil, _ action: @escaping (MobileModelStore) throws -> [String: Any]) {
|
|
287
|
+
storageQueue.async {
|
|
288
|
+
let result = Result {
|
|
289
|
+
guard let store = self.store else { throw self.storeError ?? MobileStoreError.invalidState }
|
|
290
|
+
return try action(store)
|
|
291
|
+
}
|
|
292
|
+
completion?()
|
|
293
|
+
DispatchQueue.main.async {
|
|
294
|
+
switch result {
|
|
295
|
+
case .success(let data): call.resolve(data)
|
|
296
|
+
case .failure(let error): call.reject(error.localizedDescription)
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
private func modelJSON(_ model: MobileModel) -> [String: Any] {
|
|
303
|
+
var result: [String:Any] = ["id": model.id, "name": model.name, "sizeBytes": model.sizeBytes]
|
|
304
|
+
if let source = model.source, let data = try? JSONEncoder().encode(source), let json = try? JSONSerialization.jsonObject(with: data) { result["source"] = json }
|
|
305
|
+
return result
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
public func listModels(_ call: NativeCall) {
|
|
309
|
+
withStore(call) { store in
|
|
310
|
+
let library = try store.listModels()
|
|
311
|
+
var models = library.models.map(self.modelJSON)
|
|
312
|
+
// The window the phone can hold for each model, which the product
|
|
313
|
+
// runtime uses unless the person chose one. Every model is sized,
|
|
314
|
+
// not only the selected one: a conversation keeps the model it
|
|
315
|
+
// started with, and a Gemma 4 E2B thread sized as if it were the
|
|
316
|
+
// selected Qwen 3.5 2B asked for a 16K window it could not hold
|
|
317
|
+
// (2026-09-30). Dry runs are cached per file and window.
|
|
318
|
+
for (index, model) in library.models.enumerated() {
|
|
319
|
+
if let located = try? store.modelURL(id: model.id),
|
|
320
|
+
let context = self.fitContext(id: model.id, path: located.1.path) {
|
|
321
|
+
models[index]["contextTokens"] = context
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
var result: [String: Any] = ["models": models]
|
|
325
|
+
if let selected = library.selectedModelId { result["selectedModelId"] = selected }
|
|
326
|
+
if let budget = self.memoryBudget() { result["memoryBudgetBytes"] = budget }
|
|
327
|
+
return result
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
/// What one model may use here: the process allowance plus what a loaded
|
|
332
|
+
/// model already holds. Settings hides catalog downloads that cannot fit
|
|
333
|
+
/// it, such as Gemma 4 E4B on a 6 GB iPhone. The simulator has no allowance.
|
|
334
|
+
private func memoryBudget() -> Int64? {
|
|
335
|
+
#if targetEnvironment(simulator)
|
|
336
|
+
return nil
|
|
337
|
+
#else
|
|
338
|
+
sizingLock.lock()
|
|
339
|
+
let heldPath = loadedPath, heldContext = loadedContextSize
|
|
340
|
+
sizingLock.unlock()
|
|
341
|
+
var available = Int64(os_proc_available_memory())
|
|
342
|
+
if let heldPath { available += max(0, allocation(path: heldPath, contextSize: heldContext)) }
|
|
343
|
+
return available > 0 ? available : nil
|
|
344
|
+
#endif
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
private func loadOptions(contextSize: Int) -> gezel_llama_load_options {
|
|
348
|
+
var options = gezel_llama_default_load_options()
|
|
349
|
+
options.context_tokens = UInt32(contextSize)
|
|
350
|
+
#if !targetEnvironment(simulator)
|
|
351
|
+
options.gpu_layers = -1
|
|
352
|
+
#endif
|
|
353
|
+
return options
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
/// Bytes a load at this window takes, from the bridge's dry run, or -1 when
|
|
357
|
+
/// llama.cpp cannot load the file at it. Unlike Android, iOS charges the
|
|
358
|
+
/// memory-mapped weights too: Metal wraps them, and whether the process
|
|
359
|
+
/// footprint then counts those pages has not been measured.
|
|
360
|
+
private func allocation(path: String, contextSize: Int) -> Int64 {
|
|
361
|
+
guard let engine else { return -1 }
|
|
362
|
+
let size = (try? FileManager.default.attributesOfItem(atPath: path)[.size] as? NSNumber)?.int64Value ?? 0
|
|
363
|
+
let key = "\(path)\n\(size)\n\(contextSize)"
|
|
364
|
+
sizingLock.lock(); defer { sizingLock.unlock() }
|
|
365
|
+
if let known = allocations[key] { return known }
|
|
366
|
+
var options = loadOptions(contextSize: contextSize)
|
|
367
|
+
var estimate = gezel_llama_memory_estimate()
|
|
368
|
+
estimate.struct_size = UInt32(MemoryLayout<gezel_llama_memory_estimate>.size)
|
|
369
|
+
estimate.abi_version = gezel_llama_abi_version()
|
|
370
|
+
var error = gezel_llama_error()
|
|
371
|
+
let status = path.withCString { gezel_llama_estimate_memory(engine, $0, &options, &estimate, &error) }
|
|
372
|
+
let bytes = status == 0
|
|
373
|
+
? Int64(estimate.model_bytes + estimate.context_bytes + estimate.compute_bytes) + Self.bridgeBufferBytes
|
|
374
|
+
: -1
|
|
375
|
+
allocations[key] = bytes
|
|
376
|
+
return bytes
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
/// Admission charge. A file the dry run cannot size keeps the old flat floor
|
|
380
|
+
/// (weights, 256 MiB, 64 KiB per token); its load then reports the real error.
|
|
381
|
+
private func requiredBytes(path: String, sizeBytes: Int64, contextSize: Int) -> UInt64 {
|
|
382
|
+
let bytes = allocation(path: path, contextSize: contextSize)
|
|
383
|
+
return bytes >= 0 ? UInt64(bytes) : UInt64(sizeBytes) + 256 * 1024 * 1024 + UInt64(contextSize) * 64 * 1024
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
/// 16K or 8K when the process allowance holds that window with room to
|
|
387
|
+
/// spare, else the 4K floor, which admission still checks. A loaded model
|
|
388
|
+
/// keeps its window, so a listing never makes the next turn reload it;
|
|
389
|
+
/// memory another loaded model holds counts as free, since loading this one
|
|
390
|
+
/// releases it.
|
|
391
|
+
/// Logs a model's window only when it changes; the listing is polled.
|
|
392
|
+
private func noteFittedWindow(id: String, context: Int, bytes: Int64, available: Int64) {
|
|
393
|
+
sizingLock.lock()
|
|
394
|
+
let changed = fittedWindows[id] != context
|
|
395
|
+
fittedWindows[id] = context
|
|
396
|
+
sizingLock.unlock()
|
|
397
|
+
if changed { NSLog("GezelRuntime window %ld for %@: needs %lld bytes, %lld available", context, id, bytes, available) }
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
private func fitContext(id: String, path: String) -> Int? {
|
|
401
|
+
guard engine != nil else { return nil }
|
|
402
|
+
sizingLock.lock()
|
|
403
|
+
let heldId = loadedModelId, heldPath = loadedPath, heldContext = loadedContextSize
|
|
404
|
+
sizingLock.unlock()
|
|
405
|
+
if heldId == id { return heldContext }
|
|
406
|
+
#if targetEnvironment(simulator)
|
|
407
|
+
// The simulator reports no allowance; checkResources keeps it small.
|
|
408
|
+
return nil
|
|
409
|
+
#else
|
|
410
|
+
var available = Int64(os_proc_available_memory())
|
|
411
|
+
if let heldPath { available += max(0, allocation(path: heldPath, contextSize: heldContext)) }
|
|
412
|
+
for context in Self.contextLadder {
|
|
413
|
+
let bytes = allocation(path: path, contextSize: context)
|
|
414
|
+
if bytes >= 0, bytes + Self.ladderSpareBytes <= available {
|
|
415
|
+
noteFittedWindow(id: id, context: context, bytes: bytes, available: available)
|
|
416
|
+
return context
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
noteFittedWindow(id: id, context: Self.floorContext, bytes: allocation(path: path, contextSize: Self.floorContext), available: available)
|
|
420
|
+
return Self.floorContext
|
|
421
|
+
#endif
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
private func downloadSource(_ call: NativeCall) throws -> MobileModelSource {
|
|
425
|
+
guard let raw = call.getObject("source") else { throw ModelDownloadError("A verified model source is required") }
|
|
426
|
+
return try JSONDecoder().decode(MobileModelSource.self, from: JSONSerialization.data(withJSONObject: raw))
|
|
427
|
+
}
|
|
428
|
+
private func reserveDownloadAdmission() -> Bool {
|
|
429
|
+
operationLock.lock(); defer { operationLock.unlock() }
|
|
430
|
+
guard activeId == nil, !modelMutation, !releasing, !backgrounded, downloads != nil else { return false }
|
|
431
|
+
modelMutation = true; return true
|
|
432
|
+
}
|
|
433
|
+
public func resolveModelSource(_ call: NativeCall) {
|
|
434
|
+
guard reserveDownloadAdmission() else { call.reject("Open the app and finish the current operation before checking a model", "BUSY"); return }
|
|
435
|
+
do {
|
|
436
|
+
let source = try downloadSource(call)
|
|
437
|
+
operationLock.lock()
|
|
438
|
+
guard !backgrounded else { operationLock.unlock(); throw ModelDownloadError("Open the app to check a model source") }
|
|
439
|
+
downloads!.resolve(source) { result in
|
|
440
|
+
DispatchQueue.main.async {
|
|
441
|
+
self.releaseModelMutation()
|
|
442
|
+
do { call.resolve(["source": try JSONSerialization.jsonObject(with: JSONEncoder().encode(result.get()))]) }
|
|
443
|
+
catch { call.reject(error.localizedDescription) }
|
|
444
|
+
}
|
|
445
|
+
}
|
|
446
|
+
operationLock.unlock()
|
|
447
|
+
} catch { releaseModelMutation(); call.reject(error.localizedDescription) }
|
|
448
|
+
}
|
|
449
|
+
public func cancelModelSourceResolution(_ call: NativeCall) { downloads?.cancelSourceResolution(); call.resolve() }
|
|
450
|
+
public func listModelDownloads(_ call: NativeCall) {
|
|
451
|
+
withStore(call) { _ in guard let downloads = self.downloads else { throw ModelDownloadError("Model storage is unavailable") }; return ["downloads": try downloads.list().map { try $0.json() }] }
|
|
452
|
+
}
|
|
453
|
+
public func startModelDownload(_ call: NativeCall) {
|
|
454
|
+
guard reserveDownloadAdmission() else { call.reject("Finish the current operation before downloading a model", "BUSY"); return }
|
|
455
|
+
withStore(call, completion: { self.releaseModelMutation() }) { _ in
|
|
456
|
+
// The lifecycle observers take operationLock on the main thread.
|
|
457
|
+
// Holding it across this call would stall them: starting or
|
|
458
|
+
// resuming waits on the downloads queue, and that queue may be
|
|
459
|
+
// part way through hashing a multi-gigabyte file. Read the flag
|
|
460
|
+
// under the lock and let go before the slow part.
|
|
461
|
+
self.operationLock.lock()
|
|
462
|
+
let backgrounded = self.backgrounded
|
|
463
|
+
self.operationLock.unlock()
|
|
464
|
+
guard !backgrounded, let name = call.getString("name") else { throw ModelDownloadError("Open the app and choose a model to download") }
|
|
465
|
+
return ["download": try self.downloads!.start(source: self.downloadSource(call), name: name).json()]
|
|
466
|
+
}
|
|
467
|
+
}
|
|
468
|
+
public func resumeModelDownload(_ call: NativeCall) {
|
|
469
|
+
guard reserveDownloadAdmission() else { call.reject("Finish the current operation before resuming a model", "BUSY"); return }
|
|
470
|
+
withStore(call, completion: { self.releaseModelMutation() }) { _ in
|
|
471
|
+
// The lifecycle observers take operationLock on the main thread.
|
|
472
|
+
// Holding it across this call would stall them: starting or
|
|
473
|
+
// resuming waits on the downloads queue, and that queue may be
|
|
474
|
+
// part way through hashing a multi-gigabyte file. Read the flag
|
|
475
|
+
// under the lock and let go before the slow part.
|
|
476
|
+
self.operationLock.lock()
|
|
477
|
+
let backgrounded = self.backgrounded
|
|
478
|
+
self.operationLock.unlock()
|
|
479
|
+
guard !backgrounded, let id = call.getString("id") else { throw ModelDownloadError("Open the app and choose a download to resume") }
|
|
480
|
+
return ["download": try self.downloads!.resume(id: id).json()]
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
public func cancelModelDownload(_ call: NativeCall) {
|
|
484
|
+
guard let id = call.getString("id"), let downloads else { call.reject("Download ID is required"); return }
|
|
485
|
+
downloads.cancel(id: id) { DispatchQueue.main.async { call.resolve() } }
|
|
486
|
+
}
|
|
487
|
+
public func removeModelDownload(_ call: NativeCall) {
|
|
488
|
+
guard let id = call.getString("id"), let downloads else { call.reject("Download ID is required"); return }
|
|
489
|
+
downloads.remove(id: id) { result in DispatchQueue.main.async {
|
|
490
|
+
do { try result.get(); call.resolve() } catch { call.reject(error.localizedDescription) }
|
|
491
|
+
}}
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
public func reserveModelMutation() -> Bool {
|
|
495
|
+
operationLock.lock(); defer { operationLock.unlock() }
|
|
496
|
+
guard activeId == nil, !modelMutation, !releasing, !backgrounded else { return false }
|
|
497
|
+
modelMutation = true
|
|
498
|
+
return true
|
|
499
|
+
}
|
|
500
|
+
|
|
501
|
+
public func releaseModelMutation() {
|
|
502
|
+
operationLock.lock(); modelMutation = false; operationLock.unlock()
|
|
503
|
+
scheduleReleaseIfIdle()
|
|
504
|
+
}
|
|
505
|
+
|
|
506
|
+
public func selectModel(_ call: NativeCall) {
|
|
507
|
+
guard let id = call.getString("id") else { call.reject("Model ID is required"); return }
|
|
508
|
+
guard reserveModelMutation() else { call.reject("Wait for the current conversation or import to finish", "BUSY"); return }
|
|
509
|
+
withStore(call, completion: { self.releaseModelMutation() }) { store in
|
|
510
|
+
return ["model": self.modelJSON(try store.selectModel(id: id))]
|
|
511
|
+
}
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
fileprivate func isCancelled(_ id: String) -> Bool {
|
|
515
|
+
operationLock.lock(); defer { operationLock.unlock() }
|
|
516
|
+
return activeId != id || cancelled
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
private func cancelActive(_ id: String?) {
|
|
520
|
+
operationLock.lock()
|
|
521
|
+
var task: Task<Void, Never>?
|
|
522
|
+
var parked: [CheckedContinuation<NativeToolReply, Error>] = []
|
|
523
|
+
if let activeId, id == nil || id == activeId {
|
|
524
|
+
cancelled = true
|
|
525
|
+
gezel_llama_cancel(engine, activeNativeId)
|
|
526
|
+
task = appleTask
|
|
527
|
+
parked = Array(pendingToolCalls.values); pendingToolCalls.removeAll()
|
|
528
|
+
}
|
|
529
|
+
operationLock.unlock()
|
|
530
|
+
parked.forEach { $0.resume(throwing: CancellationError()) }
|
|
531
|
+
task?.cancel()
|
|
532
|
+
}
|
|
533
|
+
|
|
534
|
+
public func cancel(_ call: NativeCall) {
|
|
535
|
+
guard let id = call.getString("requestId") else { call.reject("Request ID is required"); return }
|
|
536
|
+
operationLock.lock()
|
|
537
|
+
guard activeId == id else { operationLock.unlock(); call.resolve(); return }
|
|
538
|
+
cancelled = true
|
|
539
|
+
cancelWaiters.append(call)
|
|
540
|
+
gezel_llama_cancel(engine, activeNativeId)
|
|
541
|
+
let task = appleTask
|
|
542
|
+
let parked = Array(pendingToolCalls.values); pendingToolCalls.removeAll()
|
|
543
|
+
operationLock.unlock()
|
|
544
|
+
parked.forEach { $0.resume(throwing: CancellationError()) }
|
|
545
|
+
task?.cancel()
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
private func beginModelSegment() {
|
|
549
|
+
operationLock.lock(); defer { operationLock.unlock() }
|
|
550
|
+
modelSegmentStart = ProcessInfo.processInfo.systemUptime
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
/// Model time in the current stretch; zero while the app executes a tool.
|
|
554
|
+
private func modelSegmentSeconds() -> TimeInterval {
|
|
555
|
+
operationLock.lock(); defer { operationLock.unlock() }
|
|
556
|
+
return pendingToolCalls.isEmpty ? ProcessInfo.processInfo.systemUptime - modelSegmentStart : 0
|
|
557
|
+
}
|
|
558
|
+
|
|
559
|
+
/// Completes a native tool call with the app tool loop's result.
|
|
560
|
+
public func completeToolCall(_ call: NativeCall) {
|
|
561
|
+
guard let requestId = call.getString("requestId"), let callId = call.getString("callId") else {
|
|
562
|
+
call.reject("A request and tool call ID are required", "INVALID_ARGUMENT"); return
|
|
563
|
+
}
|
|
564
|
+
let result: Result<NativeToolReply, Error>
|
|
565
|
+
if let error = call.getString("error") {
|
|
566
|
+
result = .failure(MobileInferenceError(code: "TOOL_FAILED", message: String(error.prefix(2_000))))
|
|
567
|
+
} else if let output = call.getString("output"), output.utf8.count <= 262_144, !output.contains("\0") {
|
|
568
|
+
result = .success(NativeToolReply(output: output, endTurn: call.getBool("endTurn") ?? false))
|
|
569
|
+
} else {
|
|
570
|
+
call.reject("Invalid tool result", "INVALID_ARGUMENT"); return
|
|
571
|
+
}
|
|
572
|
+
operationLock.lock()
|
|
573
|
+
let continuation = activeId == requestId ? pendingToolCalls.removeValue(forKey: callId) : nil
|
|
574
|
+
if continuation != nil { modelSegmentStart = ProcessInfo.processInfo.systemUptime }
|
|
575
|
+
operationLock.unlock()
|
|
576
|
+
guard let continuation else { call.reject("This tool call is no longer running", "NOT_RUNNING"); return }
|
|
577
|
+
continuation.resume(with: result)
|
|
578
|
+
call.resolve()
|
|
579
|
+
}
|
|
580
|
+
|
|
581
|
+
/// Parks one native tool call until the app completes it, announcing it to JavaScript.
|
|
582
|
+
private func awaitToolCall(requestId: String, name: String, arguments: String) async throws -> NativeToolReply {
|
|
583
|
+
let callId = UUID().uuidString
|
|
584
|
+
return try await withTaskCancellationHandler {
|
|
585
|
+
try await withCheckedThrowingContinuation { (continuation: CheckedContinuation<NativeToolReply, Error>) in
|
|
586
|
+
operationLock.lock()
|
|
587
|
+
guard activeId == requestId, !cancelled else {
|
|
588
|
+
operationLock.unlock(); continuation.resume(throwing: CancellationError()); return
|
|
589
|
+
}
|
|
590
|
+
pendingToolCalls[callId] = continuation
|
|
591
|
+
operationLock.unlock()
|
|
592
|
+
DispatchQueue.main.async {
|
|
593
|
+
self.notifyListeners("toolCall", data: ["requestId": requestId, "callId": callId, "name": name, "arguments": arguments])
|
|
594
|
+
}
|
|
595
|
+
}
|
|
596
|
+
} onCancel: {
|
|
597
|
+
operationLock.lock(); let continuation = pendingToolCalls.removeValue(forKey: callId); operationLock.unlock()
|
|
598
|
+
continuation?.resume(throwing: CancellationError())
|
|
599
|
+
}
|
|
600
|
+
}
|
|
601
|
+
|
|
602
|
+
private func nextOperation(_ requestId: String) -> UInt64? {
|
|
603
|
+
operationLock.lock(); defer { operationLock.unlock() }
|
|
604
|
+
guard activeId == requestId, !cancelled else { return nil }
|
|
605
|
+
nextNativeId = nextNativeId >= UInt64(Int64.max) ? 1 : nextNativeId + 1
|
|
606
|
+
activeNativeId = nextNativeId
|
|
607
|
+
return activeNativeId
|
|
608
|
+
}
|
|
609
|
+
|
|
610
|
+
public func generate(_ call: NativeCall) {
|
|
611
|
+
guard let requestId = call.getString("requestId"), !requestId.isEmpty, requestId.utf8.count <= 128,
|
|
612
|
+
let messages = call.getArray("messages", [String: Any].self), !messages.isEmpty, messages.count <= 128 else {
|
|
613
|
+
call.reject("A request ID and conversation are required"); return
|
|
614
|
+
}
|
|
615
|
+
let providerId = call.getString("providerId") ?? "llama-cpp"
|
|
616
|
+
guard ["llama-cpp", "apple-foundation-models"].contains(providerId) else {
|
|
617
|
+
call.reject("This provider is not available on iOS", "UNAVAILABLE"); return
|
|
618
|
+
}
|
|
619
|
+
if providerId == "apple-foundation-models", let modelId = call.getString("modelId"), modelId != providerId {
|
|
620
|
+
call.reject("The requested model is not available from Apple on-device AI", "MODEL_UNAVAILABLE"); return
|
|
621
|
+
}
|
|
622
|
+
if providerId == "apple-foundation-models", let reason = AppleFoundationProvider.availability().reason {
|
|
623
|
+
call.reject(reason, "UNAVAILABLE"); return
|
|
624
|
+
}
|
|
625
|
+
var turns: [MobileChatTurn] = []
|
|
626
|
+
var inputBytes = 0
|
|
627
|
+
for message in messages {
|
|
628
|
+
guard let role = message["role"] as? String, ["system", "user", "assistant"].contains(role),
|
|
629
|
+
let content = message["content"] as? String, !content.contains("\0") else {
|
|
630
|
+
call.reject("Invalid conversation message"); return
|
|
631
|
+
}
|
|
632
|
+
inputBytes += content.utf8.count
|
|
633
|
+
turns.append(MobileChatTurn(role: role, content: content))
|
|
634
|
+
}
|
|
635
|
+
let modelId = call.getString("modelId")
|
|
636
|
+
if providerId == "llama-cpp", modelId?.isEmpty != false {
|
|
637
|
+
call.reject("Choose a model for this conversation"); return
|
|
638
|
+
}
|
|
639
|
+
guard ["maxTokens", "contextSize"].allSatisfy({ !call.contains($0) || call.getInt($0) != nil }) else {
|
|
640
|
+
call.reject("Token budgets must be integers", "INVALID_REQUEST"); return
|
|
641
|
+
}
|
|
642
|
+
var tools: [[String: Any]] = []
|
|
643
|
+
if call.contains("tools") {
|
|
644
|
+
guard providerId == "apple-foundation-models", let specs = call.getArray("tools", [String: Any].self), (1...64).contains(specs.count) else {
|
|
645
|
+
call.reject("Native tool calling is only available from Apple on-device AI", "UNSUPPORTED"); return
|
|
646
|
+
}
|
|
647
|
+
tools = specs
|
|
648
|
+
}
|
|
649
|
+
guard let sampling = LlamaSampling(call.getObject("sampling")) else {
|
|
650
|
+
call.reject("Sampling settings are outside the supported range", "INVALID_REQUEST"); return
|
|
651
|
+
}
|
|
652
|
+
let maxTokens = call.getInt("maxTokens") ?? 1024
|
|
653
|
+
let outputLimit = providerId == "llama-cpp" ? 4096 : AppleFoundationProvider.maximumOutputTokens
|
|
654
|
+
let contextSize = call.getInt("contextSize") ?? 4096
|
|
655
|
+
guard inputBytes <= 1_000_000, (1...outputLimit).contains(maxTokens), (512...(providerId == "llama-cpp" ? 16384 : AppleFoundationProvider.availability().contextTokens)).contains(contextSize), maxTokens + 128 < contextSize, turns.last?.role == "user" else {
|
|
656
|
+
call.reject("Conversation or token budget is outside the supported range"); return
|
|
657
|
+
}
|
|
658
|
+
operationLock.lock()
|
|
659
|
+
guard !backgrounded else { operationLock.unlock(); call.reject("Reopen the app to start a conversation", "BACKGROUND"); return }
|
|
660
|
+
guard activeId == nil, !modelMutation, !releasing, !backgrounded else { operationLock.unlock(); call.reject("Another conversation is running", "BUSY"); return }
|
|
661
|
+
activeId = requestId; cancelled = false; activeFailure = nil
|
|
662
|
+
if providerId == "apple-foundation-models" {
|
|
663
|
+
appleTask = Task {
|
|
664
|
+
await self.runAppleGeneration(call, requestId: requestId, turns: turns, maxTokens: maxTokens, contextSize: contextSize, tools: tools)
|
|
665
|
+
}
|
|
666
|
+
operationLock.unlock()
|
|
667
|
+
} else {
|
|
668
|
+
operationLock.unlock()
|
|
669
|
+
inferenceQueue.async {
|
|
670
|
+
self.runGeneration(call, requestId: requestId, modelId: modelId!, turns: turns, maxTokens: maxTokens, contextSize: contextSize, sampling: sampling)
|
|
671
|
+
}
|
|
672
|
+
}
|
|
673
|
+
}
|
|
674
|
+
|
|
675
|
+
/**
|
|
676
|
+
* An OpenAI-shaped chat request for an imported llama.cpp model, served by
|
|
677
|
+
* llama.cpp's own chat layer exactly as desktop's llama-server serves it:
|
|
678
|
+
* the model's template renders the tools and its parser returns structured
|
|
679
|
+
* tool calls. Every object llama-server would stream reaches JavaScript as a
|
|
680
|
+
* `chatChunk` event (batched, in order), including its error body.
|
|
681
|
+
*/
|
|
682
|
+
public func chat(_ call: NativeCall) {
|
|
683
|
+
guard let requestId = call.getString("requestId"), !requestId.isEmpty, requestId.utf8.count <= 128,
|
|
684
|
+
let requestJson = call.getString("requestJson") else {
|
|
685
|
+
call.reject("A request ID and chat request are required"); return
|
|
686
|
+
}
|
|
687
|
+
guard let modelId = call.getString("modelId"), !modelId.isEmpty else {
|
|
688
|
+
call.reject("Choose a model for this conversation"); return
|
|
689
|
+
}
|
|
690
|
+
guard !call.contains("contextSize") || call.getInt("contextSize") != nil else {
|
|
691
|
+
call.reject("Token budgets must be integers", "INVALID_REQUEST"); return
|
|
692
|
+
}
|
|
693
|
+
let contextSize = call.getInt("contextSize") ?? 4096
|
|
694
|
+
guard (512...16384).contains(contextSize) else {
|
|
695
|
+
call.reject("Context size is outside the supported range"); return
|
|
696
|
+
}
|
|
697
|
+
// The JSON text crosses unchanged. Re-encoding a dictionary reorders its
|
|
698
|
+
// keys from one request to the next, and with them the tool definitions
|
|
699
|
+
// at the top of the prompt, so no cached prefix ever matched.
|
|
700
|
+
func object(_ text: String) -> Data? {
|
|
701
|
+
let data = Data(text.utf8)
|
|
702
|
+
return (try? JSONSerialization.jsonObject(with: data)) is [String: Any] ? data : nil
|
|
703
|
+
}
|
|
704
|
+
guard requestJson.utf8.count <= 1024 * 1024, let body = object(requestJson) else {
|
|
705
|
+
call.reject("Chat request is too large or not a JSON object"); return
|
|
706
|
+
}
|
|
707
|
+
guard let config = call.getString("chatConfigJson").map(object) ?? Data("{}".utf8) else {
|
|
708
|
+
call.reject("Invalid chat settings"); return
|
|
709
|
+
}
|
|
710
|
+
guard engine != nil else { call.reject("The native inference engine could not initialize.", "UNAVAILABLE"); return }
|
|
711
|
+
operationLock.lock()
|
|
712
|
+
guard !backgrounded else { operationLock.unlock(); call.reject("Reopen the app to start a conversation", "BACKGROUND"); return }
|
|
713
|
+
guard activeId == nil, !modelMutation, !releasing else { operationLock.unlock(); call.reject("Another conversation is running", "BUSY"); return }
|
|
714
|
+
activeId = requestId; cancelled = false; activeFailure = nil
|
|
715
|
+
operationLock.unlock()
|
|
716
|
+
inferenceQueue.async {
|
|
717
|
+
self.runChat(call, requestId: requestId, modelId: modelId, contextSize: contextSize, body: body, config: config)
|
|
718
|
+
}
|
|
719
|
+
}
|
|
720
|
+
|
|
721
|
+
/// The `configure_chat` settings the loaded model holds; inference queue only.
|
|
722
|
+
private var appliedChatConfig: Data?
|
|
723
|
+
|
|
724
|
+
private func runChat(_ call: NativeCall, requestId: String, modelId: String, contextSize: Int, body: Data, config: Data) {
|
|
725
|
+
let batch = NativeChatBatch(plugin: self, requestId: requestId)
|
|
726
|
+
let cancellation = DispatchSource.makeTimerSource(queue: .global(qos: .userInitiated))
|
|
727
|
+
cancellation.schedule(deadline: .now() + .milliseconds(50), repeating: .milliseconds(50))
|
|
728
|
+
var ticks = 0
|
|
729
|
+
var lastProgress: (phase: UInt32, value: UInt32) = (UInt32.max, UInt32.max)
|
|
730
|
+
cancellation.setEventHandler { [weak self] in
|
|
731
|
+
guard let self else { return }
|
|
732
|
+
if self.isCancelled(requestId) {
|
|
733
|
+
self.cancelActive(requestId)
|
|
734
|
+
return
|
|
735
|
+
}
|
|
736
|
+
ticks += 1
|
|
737
|
+
if ticks % 5 == 0 { self.reportProgress(requestId, last: &lastProgress) }
|
|
738
|
+
}
|
|
739
|
+
cancellation.resume()
|
|
740
|
+
func nativeFailure(_ status: Int32, _ error: inout gezel_llama_error) -> NSError {
|
|
741
|
+
NSError(domain: "GezelLlama", code: Int(status), userInfo: [NSLocalizedDescriptionKey: errorText(&error)])
|
|
742
|
+
}
|
|
743
|
+
func perform() throws -> [String: Any] {
|
|
744
|
+
guard let engine, let store else { throw storeError ?? MobileStoreError.unknownModel }
|
|
745
|
+
let (model, url) = try store.modelURL(id: modelId)
|
|
746
|
+
var nativeError = gezel_llama_error()
|
|
747
|
+
if loadedModelId != model.id || loadedContextSize != contextSize {
|
|
748
|
+
try unloadLlama()
|
|
749
|
+
appliedChatConfig = nil
|
|
750
|
+
guard awaitCooling(requestId) else { return ["status": "cancelled"] }
|
|
751
|
+
try checkResources(additionalBytes: requiredBytes(path: url.path, sizeBytes: model.sizeBytes, contextSize: contextSize))
|
|
752
|
+
guard let operation = nextOperation(requestId) else { return ["status": "cancelled"] }
|
|
753
|
+
var options = loadOptions(contextSize: contextSize)
|
|
754
|
+
options.request_id = operation
|
|
755
|
+
notifyPhase(requestId, "loading_model")
|
|
756
|
+
sizingLock.lock()
|
|
757
|
+
let status = url.path.withCString { gezel_llama_load(engine, $0, &options, &nativeError) }
|
|
758
|
+
if status == 0 { loadedModelId = model.id; loadedPath = url.path; loadedContextSize = contextSize }
|
|
759
|
+
sizingLock.unlock()
|
|
760
|
+
guard status == 0 else {
|
|
761
|
+
if isCancelled(requestId) { return ["status": "cancelled"] }
|
|
762
|
+
throw nativeFailure(status, &nativeError)
|
|
763
|
+
}
|
|
764
|
+
}
|
|
765
|
+
if config != appliedChatConfig {
|
|
766
|
+
let status = config.withUnsafeBytes { raw in
|
|
767
|
+
gezel_llama_configure_chat(engine, raw.bindMemory(to: CChar.self).baseAddress, raw.count, &nativeError)
|
|
768
|
+
}
|
|
769
|
+
guard status == 0 else { throw nativeFailure(status, &nativeError) }
|
|
770
|
+
appliedChatConfig = config
|
|
771
|
+
}
|
|
772
|
+
guard awaitCooling(requestId) else { return ["status": "cancelled"] }
|
|
773
|
+
try checkResources(additionalBytes: 64 * 1024 * 1024)
|
|
774
|
+
guard let operation = nextOperation(requestId) else { return ["status": "cancelled"] }
|
|
775
|
+
var options = gezel_llama_default_chat_options()
|
|
776
|
+
options.request_id = operation
|
|
777
|
+
options.timeout_ms = 600_000
|
|
778
|
+
var result = gezel_llama_result()
|
|
779
|
+
notifyPhase(requestId, "prefill")
|
|
780
|
+
let status = body.withUnsafeBytes { raw in
|
|
781
|
+
gezel_llama_chat(engine, raw.bindMemory(to: CChar.self).baseAddress, raw.count, &options, receiveLlamaJSON,
|
|
782
|
+
Unmanaged.passUnretained(batch).toOpaque(), &result, &nativeError)
|
|
783
|
+
}
|
|
784
|
+
// Invalid options, busy and not-loaded end before a request starts,
|
|
785
|
+
// so no error body reached the batch; every other failure sent one.
|
|
786
|
+
if (1...3).contains(status) { throw nativeFailure(status, &nativeError) }
|
|
787
|
+
if status == 0 { return ["status": "ok"] }
|
|
788
|
+
if isCancelled(requestId) || status == 7 { return ["status": "cancelled"] }
|
|
789
|
+
return ["status": status == 8 ? "timeout" : "error"]
|
|
790
|
+
}
|
|
791
|
+
var terminal: Result<[String: Any], Error>
|
|
792
|
+
do { terminal = .success(try perform()) }
|
|
793
|
+
catch {
|
|
794
|
+
terminal = isCancelled(requestId) ? .success(["status": "cancelled"]) : .failure(error)
|
|
795
|
+
}
|
|
796
|
+
cancellation.cancel()
|
|
797
|
+
batch.flush()
|
|
798
|
+
finishGeneration(call, terminal: terminal)
|
|
799
|
+
}
|
|
800
|
+
|
|
801
|
+
/// Describes one photo with a small vision model: an installed llama.cpp
|
|
802
|
+
/// model and its projector, downloaded beside it as a second library entry.
|
|
803
|
+
/// The phone's describer for a device whose OS has none, such as an iPhone
|
|
804
|
+
/// without Apple Intelligence. The photo arrives as base64 and is decoded,
|
|
805
|
+
/// turned upright, and sized here.
|
|
806
|
+
public func describeImage(_ call: NativeCall) {
|
|
807
|
+
guard let requestId = call.getString("requestId"), !requestId.isEmpty, requestId.utf8.count <= 128,
|
|
808
|
+
let modelId = call.getString("modelId"), !modelId.isEmpty,
|
|
809
|
+
let projectorId = call.getString("projectorId"), !projectorId.isEmpty, projectorId != modelId,
|
|
810
|
+
let encoded = call.getString("image"), !encoded.isEmpty, encoded.utf8.count <= 22_400_000,
|
|
811
|
+
let image = Data(base64Encoded: encoded),
|
|
812
|
+
let user = call.getString("user"), !user.isEmpty, user.utf8.count <= 16 * 1024, !user.contains("\0") else {
|
|
813
|
+
call.reject("A vision model, its projector, a photo and an instruction are required", "INVALID_REQUEST"); return
|
|
814
|
+
}
|
|
815
|
+
let system = call.getString("system")
|
|
816
|
+
guard (system?.utf8.count ?? 0) <= 16 * 1024, system?.contains("\0") != true,
|
|
817
|
+
!call.contains("maxTokens") || call.getInt("maxTokens") != nil else {
|
|
818
|
+
call.reject("Invalid description settings", "INVALID_REQUEST"); return
|
|
819
|
+
}
|
|
820
|
+
let maxTokens = call.getInt("maxTokens") ?? 400
|
|
821
|
+
guard (1...1024).contains(maxTokens) else { call.reject("Invalid description settings", "INVALID_REQUEST"); return }
|
|
822
|
+
guard engine != nil else { call.reject("The native inference engine could not initialize.", "UNAVAILABLE"); return }
|
|
823
|
+
operationLock.lock()
|
|
824
|
+
guard !backgrounded else { operationLock.unlock(); call.reject("Reopen the app to read photos", "BACKGROUND"); return }
|
|
825
|
+
guard activeId == nil, !modelMutation, !releasing else { operationLock.unlock(); call.reject("Another conversation is running", "BUSY"); return }
|
|
826
|
+
activeId = requestId; cancelled = false; activeFailure = nil
|
|
827
|
+
operationLock.unlock()
|
|
828
|
+
inferenceQueue.async {
|
|
829
|
+
self.runDescribe(call, requestId: requestId, modelId: modelId, projectorId: projectorId, image: image,
|
|
830
|
+
system: system, user: user, maxTokens: maxTokens)
|
|
831
|
+
}
|
|
832
|
+
}
|
|
833
|
+
|
|
834
|
+
private func runDescribe(_ call: NativeCall, requestId: String, modelId: String, projectorId: String, image: Data,
|
|
835
|
+
system: String?, user: String, maxTokens: Int) {
|
|
836
|
+
let collector = NativeTextCollector(plugin: self, requestId: requestId)
|
|
837
|
+
let cancellation = DispatchSource.makeTimerSource(queue: .global(qos: .userInitiated))
|
|
838
|
+
cancellation.schedule(deadline: .now() + .milliseconds(50), repeating: .milliseconds(50))
|
|
839
|
+
cancellation.setEventHandler { [weak self] in
|
|
840
|
+
guard let self, self.isCancelled(requestId) else { return }
|
|
841
|
+
self.cancelActive(requestId)
|
|
842
|
+
}
|
|
843
|
+
cancellation.resume()
|
|
844
|
+
func nativeFailure(_ status: Int32, _ error: inout gezel_llama_error) -> NSError {
|
|
845
|
+
NSError(domain: "GezelLlama", code: Int(status), userInfo: [NSLocalizedDescriptionKey: errorText(&error)])
|
|
846
|
+
}
|
|
847
|
+
func perform() throws -> [String: Any] {
|
|
848
|
+
guard let engine, let store else { throw storeError ?? MobileStoreError.unknownModel }
|
|
849
|
+
let (model, url) = try store.modelURL(id: modelId)
|
|
850
|
+
let (projector, projectorURL) = try store.modelURL(id: projectorId)
|
|
851
|
+
let pixels = try Self.rgbPixels(image, maxEdge: Self.describeMaxEdge)
|
|
852
|
+
var nativeError = gezel_llama_error()
|
|
853
|
+
let visionBytes = UInt64(projector.sizeBytes) + Self.visionComputeBytes
|
|
854
|
+
if loadedModelId != model.id {
|
|
855
|
+
try unloadLlama()
|
|
856
|
+
appliedChatConfig = nil
|
|
857
|
+
guard awaitCooling(requestId) else { return ["status": "cancelled"] }
|
|
858
|
+
try checkResources(additionalBytes: requiredBytes(path: url.path, sizeBytes: model.sizeBytes, contextSize: Self.describeContext) + visionBytes)
|
|
859
|
+
guard let operation = nextOperation(requestId) else { return ["status": "cancelled"] }
|
|
860
|
+
var options = loadOptions(contextSize: Self.describeContext)
|
|
861
|
+
options.request_id = operation
|
|
862
|
+
notifyPhase(requestId, "loading_model")
|
|
863
|
+
sizingLock.lock()
|
|
864
|
+
let status = url.path.withCString { gezel_llama_load(engine, $0, &options, &nativeError) }
|
|
865
|
+
if status == 0 { loadedModelId = model.id; loadedPath = url.path; loadedContextSize = Self.describeContext }
|
|
866
|
+
sizingLock.unlock()
|
|
867
|
+
guard status == 0 else {
|
|
868
|
+
if isCancelled(requestId) { return ["status": "cancelled"] }
|
|
869
|
+
throw nativeFailure(status, &nativeError)
|
|
870
|
+
}
|
|
871
|
+
} else {
|
|
872
|
+
guard awaitCooling(requestId) else { return ["status": "cancelled"] }
|
|
873
|
+
try checkResources(additionalBytes: visionBytes)
|
|
874
|
+
}
|
|
875
|
+
guard let operation = nextOperation(requestId) else { return ["status": "cancelled"] }
|
|
876
|
+
var options = gezel_llama_default_image_options()
|
|
877
|
+
options.request_id = operation
|
|
878
|
+
options.timeout_ms = 180_000
|
|
879
|
+
options.max_tokens = UInt32(maxTokens)
|
|
880
|
+
var result = gezel_llama_result()
|
|
881
|
+
notifyPhase(requestId, "prefill")
|
|
882
|
+
let status = projectorURL.path.withCString { projectorPath in
|
|
883
|
+
user.withCString { instruction in
|
|
884
|
+
pixels.data.withUnsafeBytes { raw in
|
|
885
|
+
let rgb = raw.bindMemory(to: UInt8.self).baseAddress
|
|
886
|
+
let context = Unmanaged.passUnretained(collector).toOpaque()
|
|
887
|
+
if let system {
|
|
888
|
+
return system.withCString { systemPrompt in
|
|
889
|
+
gezel_llama_describe_image(engine, projectorPath, rgb, UInt32(pixels.width), UInt32(pixels.height),
|
|
890
|
+
systemPrompt, instruction, &options, receiveLlamaText, context, &result, &nativeError)
|
|
891
|
+
}
|
|
892
|
+
}
|
|
893
|
+
return gezel_llama_describe_image(engine, projectorPath, rgb, UInt32(pixels.width), UInt32(pixels.height),
|
|
894
|
+
nil, instruction, &options, receiveLlamaText, context, &result, &nativeError)
|
|
895
|
+
}
|
|
896
|
+
}
|
|
897
|
+
}
|
|
898
|
+
if status == 0 {
|
|
899
|
+
return ["status": "ok", "description": collector.text, "promptTokens": Int(result.prompt_tokens),
|
|
900
|
+
"generatedTokens": Int(result.generated_tokens), "truncated": result.finish_reason == 2]
|
|
901
|
+
}
|
|
902
|
+
if isCancelled(requestId) || status == 7 { return ["status": "cancelled"] }
|
|
903
|
+
throw nativeFailure(status, &nativeError)
|
|
904
|
+
}
|
|
905
|
+
var terminal: Result<[String: Any], Error>
|
|
906
|
+
do { terminal = .success(try perform()) }
|
|
907
|
+
catch {
|
|
908
|
+
terminal = isCancelled(requestId) ? .success(["status": "cancelled"]) : .failure(error)
|
|
909
|
+
}
|
|
910
|
+
cancellation.cancel()
|
|
911
|
+
finishGeneration(call, terminal: terminal)
|
|
912
|
+
}
|
|
913
|
+
|
|
914
|
+
/// Upright RGB pixels, at most `maxEdge` on the long side. The thumbnail
|
|
915
|
+
/// path applies EXIF orientation while decoding and reads HEIC.
|
|
916
|
+
private static func rgbPixels(_ data: Data, maxEdge: Int) throws -> (data: Data, width: Int, height: Int) {
|
|
917
|
+
let options: [CFString: Any] = [
|
|
918
|
+
kCGImageSourceCreateThumbnailFromImageAlways: true,
|
|
919
|
+
kCGImageSourceCreateThumbnailWithTransform: true,
|
|
920
|
+
kCGImageSourceThumbnailMaxPixelSize: maxEdge,
|
|
921
|
+
kCGImageSourceShouldCacheImmediately: true
|
|
922
|
+
]
|
|
923
|
+
guard let source = CGImageSourceCreateWithData(data as CFData, nil),
|
|
924
|
+
let image = CGImageSourceCreateThumbnailAtIndex(source, 0, options as CFDictionary) else {
|
|
925
|
+
throw MobileInferenceError(code: "INVALID_REQUEST", message: "This photo could not be read")
|
|
926
|
+
}
|
|
927
|
+
let width = image.width, height = image.height
|
|
928
|
+
var rgba = Data(count: width * height * 4)
|
|
929
|
+
let drawn = rgba.withUnsafeMutableBytes { raw -> Bool in
|
|
930
|
+
guard let context = CGContext(data: raw.baseAddress, width: width, height: height, bitsPerComponent: 8,
|
|
931
|
+
bytesPerRow: width * 4, space: CGColorSpaceCreateDeviceRGB(),
|
|
932
|
+
bitmapInfo: CGImageAlphaInfo.noneSkipLast.rawValue) else { return false }
|
|
933
|
+
context.draw(image, in: CGRect(x: 0, y: 0, width: width, height: height))
|
|
934
|
+
return true
|
|
935
|
+
}
|
|
936
|
+
guard drawn else { throw MobileInferenceError(code: "INVALID_REQUEST", message: "This photo could not be read") }
|
|
937
|
+
var rgb = Data(count: width * height * 3)
|
|
938
|
+
rgb.withUnsafeMutableBytes { output in
|
|
939
|
+
rgba.withUnsafeBytes { input in
|
|
940
|
+
let from = input.bindMemory(to: UInt8.self), to = output.bindMemory(to: UInt8.self)
|
|
941
|
+
for pixel in 0..<(width * height) {
|
|
942
|
+
to[pixel * 3] = from[pixel * 4]
|
|
943
|
+
to[pixel * 3 + 1] = from[pixel * 4 + 1]
|
|
944
|
+
to[pixel * 3 + 2] = from[pixel * 4 + 2]
|
|
945
|
+
}
|
|
946
|
+
}
|
|
947
|
+
}
|
|
948
|
+
return (rgb, width, height)
|
|
949
|
+
}
|
|
950
|
+
|
|
951
|
+
private func errorText(_ error: inout gezel_llama_error) -> String {
|
|
952
|
+
withUnsafePointer(to: &error.message) { pointer in
|
|
953
|
+
String(cString: UnsafeRawPointer(pointer).assumingMemoryBound(to: CChar.self))
|
|
954
|
+
}
|
|
955
|
+
}
|
|
956
|
+
|
|
957
|
+
private func resourceReason() -> String? {
|
|
958
|
+
operationLock.lock()
|
|
959
|
+
let hidden = backgrounded
|
|
960
|
+
let recentlyWarned = ProcessInfo.processInfo.systemUptime - lastMemoryWarning < 10
|
|
961
|
+
operationLock.unlock()
|
|
962
|
+
if hidden { return "Reopen the app to use on-device AI." }
|
|
963
|
+
if recentlyWarned { return "The device is low on memory. Wait before loading a model again." }
|
|
964
|
+
if ProcessInfo.processInfo.thermalState == .critical { return "The device is too warm for local inference. Let it cool down first." }
|
|
965
|
+
return nil
|
|
966
|
+
}
|
|
967
|
+
|
|
968
|
+
private static func gigabytes(_ bytes: UInt64) -> String {
|
|
969
|
+
String(format: "%.1f GB", Double(bytes) / 1_073_741_824)
|
|
970
|
+
}
|
|
971
|
+
|
|
972
|
+
/// How long a request waits for a critically hot device to cool before it is refused.
|
|
973
|
+
private static let coolingWaitSeconds: TimeInterval = 600
|
|
974
|
+
/// How long a request pauses at .serious before it proceeds anyway.
|
|
975
|
+
private static let seriousPauseSeconds: TimeInterval = 15
|
|
976
|
+
|
|
977
|
+
/// A critically hot device waits the heat out instead of failing the turn:
|
|
978
|
+
/// the same work, delayed, is what the person asked for. Bounded, so a
|
|
979
|
+
/// device that never cools still gets the refusal from `checkResources`.
|
|
980
|
+
/// At .serious, where iOS asks apps to scale back, each request only pauses
|
|
981
|
+
/// briefly so the device sheds some heat between steps, then proceeds.
|
|
982
|
+
/// False when the request was cancelled meanwhile.
|
|
983
|
+
private func awaitCooling(_ requestId: String) -> Bool {
|
|
984
|
+
let started = ProcessInfo.processInfo.systemUptime
|
|
985
|
+
var announced = false
|
|
986
|
+
while true {
|
|
987
|
+
let waited = ProcessInfo.processInfo.systemUptime - started
|
|
988
|
+
switch ProcessInfo.processInfo.thermalState {
|
|
989
|
+
case .critical: if waited >= Self.coolingWaitSeconds { return !isCancelled(requestId) }
|
|
990
|
+
case .serious: if waited >= Self.seriousPauseSeconds { return !isCancelled(requestId) }
|
|
991
|
+
default: return !isCancelled(requestId)
|
|
992
|
+
}
|
|
993
|
+
if isCancelled(requestId) { return false }
|
|
994
|
+
if !announced { notifyPhase(requestId, "cooling"); announced = true }
|
|
995
|
+
Thread.sleep(forTimeInterval: 2)
|
|
996
|
+
}
|
|
997
|
+
}
|
|
998
|
+
|
|
999
|
+
private func checkResources(additionalBytes: UInt64) throws {
|
|
1000
|
+
if let reason = resourceReason() { throw MobileInferenceError(code: "RESOURCE_LIMIT", message: reason) }
|
|
1001
|
+
#if targetEnvironment(simulator)
|
|
1002
|
+
// Simulator processes have no iOS dirty-memory allowance (the OS
|
|
1003
|
+
// query returns zero). Keep this developer path deliberately small;
|
|
1004
|
+
// it provides no evidence about a physical phone's memory admission.
|
|
1005
|
+
guard additionalBytes <= 1024 * 1024 * 1024 else {
|
|
1006
|
+
throw MobileInferenceError(code: "RESOURCE_LIMIT", message: "The simulator supports only small test models. Use a physical device to assess model memory requirements.")
|
|
1007
|
+
}
|
|
1008
|
+
#else
|
|
1009
|
+
let available = UInt64(os_proc_available_memory())
|
|
1010
|
+
if available < additionalBytes {
|
|
1011
|
+
NSLog("GezelRuntime admission refused: needs %llu bytes, %llu available", additionalBytes, available)
|
|
1012
|
+
throw MobileInferenceError(code: "RESOURCE_LIMIT", message: "There is not enough available memory for this model: it needs about \(Self.gigabytes(additionalBytes)), and \(Self.gigabytes(available)) is free. Choose a smaller model or close other apps.")
|
|
1013
|
+
}
|
|
1014
|
+
#endif
|
|
1015
|
+
}
|
|
1016
|
+
|
|
1017
|
+
public func providers(_ call: NativeCall) {
|
|
1018
|
+
storageQueue.async {
|
|
1019
|
+
let restriction = self.resourceReason()
|
|
1020
|
+
var llamaReason = restriction
|
|
1021
|
+
if llamaReason == nil {
|
|
1022
|
+
if self.engine == nil { llamaReason = "The native inference engine could not initialize." }
|
|
1023
|
+
else {
|
|
1024
|
+
do {
|
|
1025
|
+
guard let store = self.store else { throw self.storeError ?? MobileStoreError.unknownModel }
|
|
1026
|
+
guard !(try store.listModels().models.isEmpty) else { throw MobileStoreError.unknownModel }
|
|
1027
|
+
} catch { llamaReason = error.localizedDescription }
|
|
1028
|
+
}
|
|
1029
|
+
}
|
|
1030
|
+
let apple = AppleFoundationProvider.availability()
|
|
1031
|
+
func descriptor(_ id: String, _ name: String, _ reason: String?, _ context: Int, _ output: Int) -> [String: Any] {
|
|
1032
|
+
var value: [String: Any] = [
|
|
1033
|
+
"id": id, "name": name, "locality": "on-device",
|
|
1034
|
+
"availability": reason == nil ? "available" : "unavailable",
|
|
1035
|
+
"contextTokens": context, "maxOutputTokens": output,
|
|
1036
|
+
// structuredChat: llama.cpp's own chat layer, as desktop's llama-server runs it.
|
|
1037
|
+
"capabilities": ["text": true, "tools": id == "apple-foundation-models", "structuredOutput": false, "images": false, "foregroundOnly": true, "structuredChat": id == "llama-cpp"]
|
|
1038
|
+
]
|
|
1039
|
+
if let reason { value["reason"] = reason }
|
|
1040
|
+
return value
|
|
1041
|
+
}
|
|
1042
|
+
let result = [
|
|
1043
|
+
descriptor("llama-cpp", "Imported GGUF model", llamaReason, 16384, 4096),
|
|
1044
|
+
descriptor("apple-foundation-models", "Apple on-device AI", restriction ?? apple.reason, apple.contextTokens, AppleFoundationProvider.maximumOutputTokens)
|
|
1045
|
+
]
|
|
1046
|
+
DispatchQueue.main.async { call.resolve(["providers": result]) }
|
|
1047
|
+
}
|
|
1048
|
+
}
|
|
1049
|
+
|
|
1050
|
+
public func prepareProvider(_ call: NativeCall) {
|
|
1051
|
+
if call.getString("providerId") == "apple-foundation-models" {
|
|
1052
|
+
call.reject("iOS manages Apple on-device model downloads. Enable Apple Intelligence in Settings and wait for preparation to finish.", "OS_MANAGED")
|
|
1053
|
+
} else {
|
|
1054
|
+
call.reject("This provider cannot be prepared on iOS. Import and select a GGUF to use an imported model.", "UNAVAILABLE")
|
|
1055
|
+
}
|
|
1056
|
+
}
|
|
1057
|
+
|
|
1058
|
+
/// Nothing on iOS prepares a provider — `prepareProvider` always refuses,
|
|
1059
|
+
/// because the OS owns Apple's model downloads — so there is never a
|
|
1060
|
+
/// preparation in flight to stop. The method exists because the shared
|
|
1061
|
+
/// host contract declares it, and a host call that simply fails on one
|
|
1062
|
+
/// platform is worse than one that truthfully does nothing.
|
|
1063
|
+
public func cancelProviderPreparation(_ call: NativeCall) {
|
|
1064
|
+
call.resolve()
|
|
1065
|
+
}
|
|
1066
|
+
|
|
1067
|
+
/// Called only on inferenceQueue, with generation admission already held.
|
|
1068
|
+
private func unloadLlama() throws {
|
|
1069
|
+
if let engine {
|
|
1070
|
+
var error = gezel_llama_error()
|
|
1071
|
+
let status = gezel_llama_unload(engine, &error)
|
|
1072
|
+
guard status == 0 else { throw MobileInferenceError(code: "BUSY", message: errorText(&error)) }
|
|
1073
|
+
}
|
|
1074
|
+
sizingLock.lock()
|
|
1075
|
+
loadedModelId = nil; loadedPath = nil; loadedContextSize = 0
|
|
1076
|
+
sizingLock.unlock()
|
|
1077
|
+
}
|
|
1078
|
+
|
|
1079
|
+
public func releaseModel(_ call: NativeCall) { requestRelease(call) }
|
|
1080
|
+
|
|
1081
|
+
private func requestRelease(_ call: NativeCall?) {
|
|
1082
|
+
operationLock.lock()
|
|
1083
|
+
releasing = true
|
|
1084
|
+
if let call { releaseWaiters.append(call) }
|
|
1085
|
+
operationLock.unlock()
|
|
1086
|
+
cancelActive(nil)
|
|
1087
|
+
scheduleReleaseIfIdle()
|
|
1088
|
+
}
|
|
1089
|
+
|
|
1090
|
+
private func scheduleReleaseIfIdle() {
|
|
1091
|
+
operationLock.lock()
|
|
1092
|
+
guard releasing, activeId == nil, !modelMutation, !releaseScheduled else { operationLock.unlock(); return }
|
|
1093
|
+
releaseScheduled = true
|
|
1094
|
+
operationLock.unlock()
|
|
1095
|
+
inferenceQueue.async {
|
|
1096
|
+
let result = Result { try self.unloadLlama() }
|
|
1097
|
+
self.operationLock.lock()
|
|
1098
|
+
self.releasing = false; self.releaseScheduled = false
|
|
1099
|
+
let waiters = self.releaseWaiters
|
|
1100
|
+
self.releaseWaiters.removeAll()
|
|
1101
|
+
self.operationLock.unlock()
|
|
1102
|
+
DispatchQueue.main.async {
|
|
1103
|
+
for call in waiters {
|
|
1104
|
+
switch result {
|
|
1105
|
+
case .success: call.resolve()
|
|
1106
|
+
case .failure(let error): call.reject(error.localizedDescription)
|
|
1107
|
+
}
|
|
1108
|
+
}
|
|
1109
|
+
}
|
|
1110
|
+
}
|
|
1111
|
+
}
|
|
1112
|
+
|
|
1113
|
+
public func removeModel(_ call: NativeCall) {
|
|
1114
|
+
guard let id = call.getString("id") else { call.reject("Model ID is required"); return }
|
|
1115
|
+
guard reserveModelMutation() else { call.reject("Wait for the current conversation or import to finish", "BUSY"); return }
|
|
1116
|
+
inferenceQueue.async {
|
|
1117
|
+
do { try self.unloadLlama() }
|
|
1118
|
+
catch {
|
|
1119
|
+
self.releaseModelMutation()
|
|
1120
|
+
DispatchQueue.main.async { call.reject(error.localizedDescription) }
|
|
1121
|
+
return
|
|
1122
|
+
}
|
|
1123
|
+
self.withStore(call, completion: { self.releaseModelMutation() }) { store in
|
|
1124
|
+
try store.removeModel(id: id)
|
|
1125
|
+
return [:]
|
|
1126
|
+
}
|
|
1127
|
+
}
|
|
1128
|
+
}
|
|
1129
|
+
|
|
1130
|
+
private func failActive(_ id: String, error: MobileInferenceError) {
|
|
1131
|
+
operationLock.lock()
|
|
1132
|
+
guard activeId == id else { operationLock.unlock(); return }
|
|
1133
|
+
activeFailure = error
|
|
1134
|
+
operationLock.unlock()
|
|
1135
|
+
cancelActive(id)
|
|
1136
|
+
}
|
|
1137
|
+
|
|
1138
|
+
private func finishGeneration(_ call: NativeCall, terminal: Result<[String: Any], Error>) {
|
|
1139
|
+
operationLock.lock()
|
|
1140
|
+
let result: Result<[String: Any], Error> = activeFailure.map { .failure($0) } ?? terminal
|
|
1141
|
+
activeId = nil; activeNativeId = 0; appleTask = nil; activeFailure = nil
|
|
1142
|
+
let parked = Array(pendingToolCalls.values); pendingToolCalls.removeAll()
|
|
1143
|
+
let waiting = cancelWaiters
|
|
1144
|
+
cancelWaiters.removeAll()
|
|
1145
|
+
operationLock.unlock()
|
|
1146
|
+
parked.forEach { $0.resume(throwing: CancellationError()) }
|
|
1147
|
+
scheduleReleaseIfIdle()
|
|
1148
|
+
DispatchQueue.main.async {
|
|
1149
|
+
switch result {
|
|
1150
|
+
case .success(let data): call.resolve(data)
|
|
1151
|
+
case .failure(let error): call.reject(error.localizedDescription, (error as? MobileInferenceError)?.code)
|
|
1152
|
+
}
|
|
1153
|
+
waiting.forEach { $0.resolve() }
|
|
1154
|
+
}
|
|
1155
|
+
}
|
|
1156
|
+
|
|
1157
|
+
private func runAppleGeneration(_ call: NativeCall, requestId: String, turns: [MobileChatTurn], maxTokens: Int, contextSize: Int, tools: [[String: Any]]) async {
|
|
1158
|
+
var text = ""
|
|
1159
|
+
beginModelSegment()
|
|
1160
|
+
// A minute of the model's own time between tool results. Polling, rather
|
|
1161
|
+
// than one armed timer, lets a completed tool call restart the minute.
|
|
1162
|
+
let timeout = Task {
|
|
1163
|
+
while !Task.isCancelled {
|
|
1164
|
+
do { try await Task.sleep(nanoseconds: 1_000_000_000) } catch { return }
|
|
1165
|
+
if self.modelSegmentSeconds() > 60 {
|
|
1166
|
+
self.failActive(requestId, error: MobileInferenceError(code: "TIMEOUT", message: "Apple on-device AI exceeded the one-minute response limit."))
|
|
1167
|
+
return
|
|
1168
|
+
}
|
|
1169
|
+
}
|
|
1170
|
+
}
|
|
1171
|
+
defer { timeout.cancel() }
|
|
1172
|
+
let terminal: Result<[String: Any], Error>
|
|
1173
|
+
do {
|
|
1174
|
+
try await withCheckedThrowingContinuation { (continuation: CheckedContinuation<Void, Error>) in
|
|
1175
|
+
inferenceQueue.async { continuation.resume(with: Result { try self.unloadLlama() }) }
|
|
1176
|
+
}
|
|
1177
|
+
try checkResources(additionalBytes: 256 * 1024 * 1024)
|
|
1178
|
+
if isCancelled(requestId) { throw CancellationError() }
|
|
1179
|
+
guard #available(iOS 26.0, *) else { throw MobileInferenceError(code: "UNAVAILABLE", message: "Apple on-device AI requires iOS 26 or later.") }
|
|
1180
|
+
notifyPhase(requestId, "prefill")
|
|
1181
|
+
var generating = false
|
|
1182
|
+
let reason = try await AppleFoundationProvider.generate(
|
|
1183
|
+
turns: turns, maxTokens: maxTokens, contextSize: contextSize, tools: tools,
|
|
1184
|
+
invoke: { [weak self] name, arguments in
|
|
1185
|
+
guard let self else { throw CancellationError() }
|
|
1186
|
+
return try await self.awaitToolCall(requestId: requestId, name: name, arguments: arguments)
|
|
1187
|
+
}
|
|
1188
|
+
) { delta in
|
|
1189
|
+
if self.isCancelled(requestId) { throw CancellationError() }
|
|
1190
|
+
text.append(delta)
|
|
1191
|
+
if !generating {
|
|
1192
|
+
generating = true
|
|
1193
|
+
self.notifyPhase(requestId, "generating")
|
|
1194
|
+
}
|
|
1195
|
+
DispatchQueue.main.async {
|
|
1196
|
+
self.notifyListeners("chatDelta", data: ["requestId": requestId, "delta": delta])
|
|
1197
|
+
}
|
|
1198
|
+
}
|
|
1199
|
+
terminal = .success(["text": text, "stopReason": reason])
|
|
1200
|
+
} catch {
|
|
1201
|
+
terminal = isCancelled(requestId) || error is CancellationError
|
|
1202
|
+
? .success(["text": text, "stopReason": "cancelled"])
|
|
1203
|
+
: .failure(error)
|
|
1204
|
+
}
|
|
1205
|
+
finishGeneration(call, terminal: terminal)
|
|
1206
|
+
}
|
|
1207
|
+
|
|
1208
|
+
private func runGeneration(_ call: NativeCall, requestId: String, modelId: String, turns: [MobileChatTurn], maxTokens: Int, contextSize: Int, sampling: LlamaSampling) {
|
|
1209
|
+
let stream = NativeChatStream(plugin: self, requestId: requestId)
|
|
1210
|
+
// Reassert cancellation while a blocking call runs, including the tiny
|
|
1211
|
+
// interval between assigning its ID and entering the native function.
|
|
1212
|
+
let cancellation = DispatchSource.makeTimerSource(queue: .global(qos: .userInitiated))
|
|
1213
|
+
cancellation.schedule(deadline: .now() + .milliseconds(50), repeating: .milliseconds(50))
|
|
1214
|
+
var ticks = 0
|
|
1215
|
+
var lastProgress: (phase: UInt32, value: UInt32) = (UInt32.max, UInt32.max)
|
|
1216
|
+
cancellation.setEventHandler { [weak self] in
|
|
1217
|
+
guard let self else { return }
|
|
1218
|
+
if self.isCancelled(requestId) {
|
|
1219
|
+
self.cancelActive(requestId)
|
|
1220
|
+
return
|
|
1221
|
+
}
|
|
1222
|
+
// Every fifth tick (~250 ms) is plenty for a status pill.
|
|
1223
|
+
ticks += 1
|
|
1224
|
+
if ticks % 5 == 0 { self.reportProgress(requestId, last: &lastProgress) }
|
|
1225
|
+
}
|
|
1226
|
+
cancellation.resume()
|
|
1227
|
+
var terminal: Result<[String: Any], Error>
|
|
1228
|
+
func perform() throws -> [String: Any] {
|
|
1229
|
+
guard let engine, let store else { throw storeError ?? MobileStoreError.unknownModel }
|
|
1230
|
+
let (model, url) = try store.modelURL(id: modelId)
|
|
1231
|
+
var nativeError = gezel_llama_error()
|
|
1232
|
+
if loadedModelId != model.id || loadedContextSize != contextSize {
|
|
1233
|
+
try unloadLlama()
|
|
1234
|
+
// Native allocation and memory warnings still enforce the
|
|
1235
|
+
// model's actual working set.
|
|
1236
|
+
try checkResources(additionalBytes: requiredBytes(path: url.path, sizeBytes: model.sizeBytes, contextSize: contextSize))
|
|
1237
|
+
guard let operation = nextOperation(requestId) else {
|
|
1238
|
+
return ["text": "", "stopReason": "cancelled"]
|
|
1239
|
+
}
|
|
1240
|
+
var options = loadOptions(contextSize: contextSize)
|
|
1241
|
+
options.request_id = operation
|
|
1242
|
+
notifyPhase(requestId, "loading_model")
|
|
1243
|
+
sizingLock.lock()
|
|
1244
|
+
let status = url.path.withCString { gezel_llama_load(engine, $0, &options, &nativeError) }
|
|
1245
|
+
if status == 0 { loadedModelId = model.id; loadedPath = url.path; loadedContextSize = contextSize }
|
|
1246
|
+
sizingLock.unlock()
|
|
1247
|
+
guard status == 0 else {
|
|
1248
|
+
if isCancelled(requestId) { return ["text": "", "stopReason": "cancelled"] }
|
|
1249
|
+
throw NSError(domain: "GezelLlama", code: Int(status), userInfo: [NSLocalizedDescriptionKey: errorText(&nativeError)])
|
|
1250
|
+
}
|
|
1251
|
+
}
|
|
1252
|
+
try checkResources(additionalBytes: 64 * 1024 * 1024)
|
|
1253
|
+
guard let operation = nextOperation(requestId) else {
|
|
1254
|
+
return ["text": "", "stopReason": "cancelled"]
|
|
1255
|
+
}
|
|
1256
|
+
var options = gezel_llama_default_generation_options()
|
|
1257
|
+
options.request_id = operation
|
|
1258
|
+
options.max_tokens = UInt32(maxTokens)
|
|
1259
|
+
sampling.apply(to: &options)
|
|
1260
|
+
// The library's default deadline is a flat minute covering prompt
|
|
1261
|
+
// processing as well as decoding, which a long reply on a phone
|
|
1262
|
+
// passes routinely. Scale it with the reply actually asked for, and
|
|
1263
|
+
// keep a ceiling so a wedged decode still ends.
|
|
1264
|
+
options.timeout_ms = UInt32(min(600_000, 30_000 + maxTokens * 250))
|
|
1265
|
+
let strings = turns.flatMap { [strdup($0.role), strdup($0.content)] }
|
|
1266
|
+
defer { strings.forEach { free($0) } }
|
|
1267
|
+
let nativeTurns = turns.indices.map { index in
|
|
1268
|
+
gezel_llama_message(role: UnsafePointer(strings[index * 2]), content: UnsafePointer(strings[index * 2 + 1]))
|
|
1269
|
+
}
|
|
1270
|
+
var result = gezel_llama_result()
|
|
1271
|
+
notifyPhase(requestId, "prefill")
|
|
1272
|
+
let status = nativeTurns.withUnsafeBufferPointer { buffer in
|
|
1273
|
+
gezel_llama_generate(engine, buffer.baseAddress, buffer.count, &options, receiveLlamaChunk,
|
|
1274
|
+
Unmanaged.passUnretained(stream).toOpaque(), &result, &nativeError)
|
|
1275
|
+
}
|
|
1276
|
+
let stopped = isCancelled(requestId) || result.finish_reason == 3
|
|
1277
|
+
guard status == 0 || stopped else {
|
|
1278
|
+
throw NSError(domain: "GezelLlama", code: Int(status), userInfo: [NSLocalizedDescriptionKey: errorText(&nativeError)])
|
|
1279
|
+
}
|
|
1280
|
+
// 2 is the token ceiling and 4 the deadline: both mean the reply was
|
|
1281
|
+
// cut short rather than finished, which is what "length" tells the
|
|
1282
|
+
// product. Treating a timeout as "stop" would present a truncated
|
|
1283
|
+
// answer as a complete one.
|
|
1284
|
+
let truncated = result.finish_reason == 2 || result.finish_reason == 4
|
|
1285
|
+
let reason = stopped ? "cancelled" : (truncated ? "length" : "stop")
|
|
1286
|
+
return ["text": stream.text, "stopReason": reason]
|
|
1287
|
+
}
|
|
1288
|
+
do { terminal = .success(try perform()) }
|
|
1289
|
+
catch {
|
|
1290
|
+
terminal = isCancelled(requestId)
|
|
1291
|
+
? .success(["text": stream.text, "stopReason": "cancelled"])
|
|
1292
|
+
: .failure(error)
|
|
1293
|
+
}
|
|
1294
|
+
cancellation.cancel()
|
|
1295
|
+
finishGeneration(call, terminal: terminal)
|
|
1296
|
+
}
|
|
1297
|
+
public func importModel(_ call: NativeCall, from url: URL) {
|
|
1298
|
+
guard reserveModelMutation() else { call.reject("Finish the current operation before importing", "BUSY"); return }
|
|
1299
|
+
withStore(call, completion: { self.releaseModelMutation() }) { store in
|
|
1300
|
+
let access = url.startAccessingSecurityScopedResource()
|
|
1301
|
+
defer { if access { url.stopAccessingSecurityScopedResource() } }
|
|
1302
|
+
var result: Result<MobileModel, Error>?
|
|
1303
|
+
var coordinationError: NSError?
|
|
1304
|
+
NSFileCoordinator().coordinate(readingItemAt: url, options: [], error: &coordinationError) { coordinated in
|
|
1305
|
+
result = Result { try store.importModel(from: coordinated) }
|
|
1306
|
+
}
|
|
1307
|
+
if let coordinationError { throw coordinationError }
|
|
1308
|
+
guard let result else { throw MobileStoreError.invalidModel }
|
|
1309
|
+
return ["model": self.modelJSON(try result.get())]
|
|
1310
|
+
}
|
|
1311
|
+
}
|
|
1312
|
+
}
|
|
1313
|
+
|
|
1314
|
+
/// A model's catalog sampling, resolved by the product runtime the same way the
|
|
1315
|
+
/// desktop resolves it. Absent fields keep the bridge's defaults: greedy
|
|
1316
|
+
/// decoding, top-k 40 / top-p 0.95 when sampling, no repetition penalty.
|
|
1317
|
+
struct LlamaSampling {
|
|
1318
|
+
var temperature: Float = 0
|
|
1319
|
+
var topK: UInt32 = 40
|
|
1320
|
+
var topP: Float = 0.95
|
|
1321
|
+
var minP: Float = 0
|
|
1322
|
+
var repeatPenalty: Float = 1
|
|
1323
|
+
var repeatLastN: UInt32 = 64
|
|
1324
|
+
// An unset seed varies per reply, so a retry is not the same text again.
|
|
1325
|
+
var seed = UInt32.random(in: 0...UInt32(Int32.max))
|
|
1326
|
+
|
|
1327
|
+
init?(_ value: [String: Any]?) {
|
|
1328
|
+
guard let value else { return }
|
|
1329
|
+
func number(_ key: String) -> Double? { (value[key] as? NSNumber)?.doubleValue }
|
|
1330
|
+
if let v = number("temperature") { temperature = Float(v) }
|
|
1331
|
+
if let v = number("topK") { guard v >= 0, v <= 1000, v == v.rounded() else { return nil }; topK = UInt32(v) }
|
|
1332
|
+
if let v = number("topP") { topP = Float(v) }
|
|
1333
|
+
if let v = number("minP") { minP = Float(v) }
|
|
1334
|
+
if let v = number("repetitionPenalty") { repeatPenalty = Float(v) }
|
|
1335
|
+
if let v = number("repetitionContext") { guard v >= 0, v <= 4096, v == v.rounded() else { return nil }; repeatLastN = UInt32(v) }
|
|
1336
|
+
if let v = number("seed"), v == v.rounded() { seed = UInt32(truncatingIfNeeded: Int64(v)) & UInt32(Int32.max) }
|
|
1337
|
+
guard (0...2).contains(temperature), topP > 0, topP <= 1, minP >= 0, minP < 1, (1...2).contains(repeatPenalty) else { return nil }
|
|
1338
|
+
}
|
|
1339
|
+
|
|
1340
|
+
func apply(to options: inout gezel_llama_generation_options) {
|
|
1341
|
+
options.temperature = temperature
|
|
1342
|
+
options.top_k = topK
|
|
1343
|
+
options.top_p = topP
|
|
1344
|
+
options.min_p = minP
|
|
1345
|
+
options.repeat_penalty = repeatPenalty
|
|
1346
|
+
options.repeat_last_n = repeatLastN
|
|
1347
|
+
options.seed = seed
|
|
1348
|
+
}
|
|
1349
|
+
}
|