@bendyline/gezel-capacitor 0.0.0-stage → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (121) hide show
  1. package/LICENSE +21 -0
  2. package/Package.swift +26 -0
  3. package/README.md +230 -2
  4. package/android/build.gradle +30 -0
  5. package/android/src/main/AndroidManifest.xml +1 -0
  6. package/android/src/main/java/com/bendyline/gezel/capacitor/GezelKnowledgePlugin.java +153 -0
  7. package/android/src/main/java/com/bendyline/gezel/capacitor/GezelOfflineSpeechPlugin.java +216 -0
  8. package/android/src/main/java/com/bendyline/gezel/capacitor/GezelRuntimePlugin.java +84 -0
  9. package/android/src/main/java/com/bendyline/gezel/capacitor/KnowledgeFiles.java +101 -0
  10. package/android/src/main/java/com/bendyline/gezel/capacitor/SpeechModels.java +230 -0
  11. package/android/src/main/java/com/bendyline/gezel/speech/SpeechRuntime.java +38 -0
  12. package/dist/index.d.ts +561 -0
  13. package/dist/index.js +7512 -0
  14. package/dist/knowledge-worker.js +211 -0
  15. package/dist/licenses/npm/manifest.json +76 -0
  16. package/dist/licenses/npm/texts/1583809bf1155510d2961f94aa9bca6dc4b44b6f59fd773de33bd6c20860d781.txt +29 -0
  17. package/dist/licenses/npm/texts/1cd39aa4a566eba49eb206b969ef423295e79b50503df567bbdb2e8877d78892.txt +8 -0
  18. package/dist/licenses/npm/texts/3f1189b28e3866e0d979968d466b78f813f76827cfdca1fbb124cc0a5c8841f8.txt +21 -0
  19. package/dist/licenses/npm/texts/5b37224c080cdcc97c871ada971c224e9926370fe74f11b539aa1cf9f3b1aca1.txt +21 -0
  20. package/dist/licenses/npm/texts/c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4.txt +201 -0
  21. package/dist/sqlite3.wasm +0 -0
  22. package/embedding-manifest.json +210 -0
  23. package/ios/Sources/GezelCapacitor/GezelKnowledgePlugin.swift +200 -0
  24. package/ios/Sources/GezelCapacitor/GezelOfflineSpeechPlugin.swift +484 -0
  25. package/ios/Sources/GezelCapacitor/GezelRuntimePlugin.swift +102 -0
  26. package/kokoro-lexicon/LICENSE +202 -0
  27. package/kokoro-lexicon/lexicon-gb-en.txt.gz +0 -0
  28. package/kokoro-lexicon/lexicon-us-en.txt.gz +0 -0
  29. package/native/android/LICENSE-gezel.txt +21 -0
  30. package/native/android/PUBLIC-DISTRIBUTION.md +97 -0
  31. package/native/android/THIRD_PARTY_LICENSES/LICENSE-arm-optimized-routines-MIT-OR-Apache-2.0.txt +249 -0
  32. package/native/android/THIRD_PARTY_LICENSES/LICENSE-cpp-httplib-MIT.txt +22 -0
  33. package/native/android/THIRD_PARTY_LICENSES/LICENSE-fp16-MIT.txt +11 -0
  34. package/native/android/THIRD_PARTY_LICENSES/LICENSE-ggml-MIT.txt +21 -0
  35. package/native/android/THIRD_PARTY_LICENSES/LICENSE-llama-cpp-llamafile-sgemm-MIT.txt +21 -0
  36. package/native/android/THIRD_PARTY_LICENSES/LICENSE-llama-cpp-openvino-APACHE.txt +201 -0
  37. package/native/android/THIRD_PARTY_LICENSES/LICENSE-llama-cpp-pillow-MIT-CMU.txt +30 -0
  38. package/native/android/THIRD_PARTY_LICENSES/LICENSE-llama-cpp-rotate-bits-MIT.txt +21 -0
  39. package/native/android/THIRD_PARTY_LICENSES/LICENSE-llama-cpp-unicode-data-Unicode-3.0.txt +39 -0
  40. package/native/android/THIRD_PARTY_LICENSES/LICENSE-nlohmann-json-MIT.txt +21 -0
  41. package/native/android/THIRD_PARTY_LICENSES/LICENSE-nvidia-cub-BSD-3-Clause.txt +24 -0
  42. package/native/android/THIRD_PARTY_LICENSES/LICENSE-vulkan-headers-Apache-2.0-OR-MIT.txt +33 -0
  43. package/native/android/THIRD_PARTY_LICENSES/LICENSE-vulkan-loader-APACHE.txt +201 -0
  44. package/native/android/THIRD_PARTY_LICENSES/LICENSE-vulkan-loader-cjson-MIT.txt +32 -0
  45. package/native/android/THIRD_PARTY_LICENSES/LICENSE-yarn-MIT.txt +21 -0
  46. package/native/android/THIRD_PARTY_LICENSES/manifest.json +21 -0
  47. package/native/android/engine-manifest.json +88 -0
  48. package/native/android/maven/com/bendyline/gezel/gezel-llama/0.1.57/gezel-llama-0.1.57.aar +0 -0
  49. package/native/android/maven/com/bendyline/gezel/gezel-llama/0.1.57/gezel-llama-0.1.57.aar.sha256 +1 -0
  50. package/native/android/maven/com/bendyline/gezel/gezel-llama/0.1.57/gezel-llama-0.1.57.pom +8 -0
  51. package/native/android/maven/com/bendyline/gezel/gezel-llama/0.1.57/gezel-llama-0.1.57.pom.sha256 +1 -0
  52. package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57-sources.jar +0 -0
  53. package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57-sources.jar.md5 +1 -0
  54. package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57-sources.jar.sha1 +1 -0
  55. package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57-sources.jar.sha256 +1 -0
  56. package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57-sources.jar.sha512 +1 -0
  57. package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.aar +0 -0
  58. package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.aar.md5 +1 -0
  59. package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.aar.sha1 +1 -0
  60. package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.aar.sha256 +1 -0
  61. package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.aar.sha512 +1 -0
  62. package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.module +118 -0
  63. package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.module.md5 +1 -0
  64. package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.module.sha1 +1 -0
  65. package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.module.sha256 +1 -0
  66. package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.module.sha512 +1 -0
  67. package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.pom +51 -0
  68. package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.pom.md5 +1 -0
  69. package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.pom.sha1 +1 -0
  70. package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.pom.sha256 +1 -0
  71. package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.pom.sha512 +1 -0
  72. package/native/android/maven/com/bendyline/gezel/gezel-runtime/maven-metadata.xml +13 -0
  73. package/native/android/maven/com/bendyline/gezel/gezel-runtime/maven-metadata.xml.md5 +1 -0
  74. package/native/android/maven/com/bendyline/gezel/gezel-runtime/maven-metadata.xml.sha1 +1 -0
  75. package/native/android/maven/com/bendyline/gezel/gezel-runtime/maven-metadata.xml.sha256 +1 -0
  76. package/native/android/maven/com/bendyline/gezel/gezel-runtime/maven-metadata.xml.sha512 +1 -0
  77. package/native/android/sdk-manifest.json +98 -0
  78. package/native/ios/GezelLlama.xcframework/Info.plist +47 -0
  79. package/native/ios/GezelLlama.xcframework/ios-arm64/Headers/gezel_llama.h +290 -0
  80. package/native/ios/GezelLlama.xcframework/ios-arm64/Headers/module.modulemap +9 -0
  81. package/native/ios/GezelLlama.xcframework/ios-arm64/libGezelLlama.a +0 -0
  82. package/native/ios/GezelLlama.xcframework/ios-arm64-simulator/Headers/gezel_llama.h +290 -0
  83. package/native/ios/GezelLlama.xcframework/ios-arm64-simulator/Headers/module.modulemap +9 -0
  84. package/native/ios/GezelLlama.xcframework/ios-arm64-simulator/libGezelLlama.a +0 -0
  85. package/native/ios/LICENSE-gezel.txt +21 -0
  86. package/native/ios/LICENSE-ggml.txt +21 -0
  87. package/native/ios/LICENSE-llama-cpp.txt +21 -0
  88. package/native/ios/PUBLIC-DISTRIBUTION.md +97 -0
  89. package/native/ios/Package.swift +17 -0
  90. package/native/ios/Sources/GezelModelStorage/MobileModelStore.swift +286 -0
  91. package/native/ios/Sources/GezelModelStorage/ModelDownloadHTTP.swift +55 -0
  92. package/native/ios/Sources/GezelModelStorage/ModelDownloadSource.swift +46 -0
  93. package/native/ios/Sources/GezelModelStorage/ModelDownloads.swift +289 -0
  94. package/native/ios/Sources/GezelModelStorage/PrivacyInfo.xcprivacy +32 -0
  95. package/native/ios/Sources/GezelRuntime/AppleFoundationProvider.swift +292 -0
  96. package/native/ios/Sources/GezelRuntime/AppleNativeTools.swift +152 -0
  97. package/native/ios/Sources/GezelRuntime/GezelNativeRuntime.swift +1349 -0
  98. package/native/ios/Sources/GezelRuntime/NativeCall.swift +35 -0
  99. package/native/ios/Sources/GezelRuntime/PrivacyInfo.xcprivacy +31 -0
  100. package/native/ios/THIRD_PARTY_LICENSES/LICENSE-arm-optimized-routines-MIT-OR-Apache-2.0.txt +249 -0
  101. package/native/ios/THIRD_PARTY_LICENSES/LICENSE-cpp-httplib-MIT.txt +22 -0
  102. package/native/ios/THIRD_PARTY_LICENSES/LICENSE-fp16-MIT.txt +11 -0
  103. package/native/ios/THIRD_PARTY_LICENSES/LICENSE-ggml-MIT.txt +21 -0
  104. package/native/ios/THIRD_PARTY_LICENSES/LICENSE-llama-cpp-llamafile-sgemm-MIT.txt +21 -0
  105. package/native/ios/THIRD_PARTY_LICENSES/LICENSE-llama-cpp-openvino-APACHE.txt +201 -0
  106. package/native/ios/THIRD_PARTY_LICENSES/LICENSE-llama-cpp-pillow-MIT-CMU.txt +30 -0
  107. package/native/ios/THIRD_PARTY_LICENSES/LICENSE-llama-cpp-rotate-bits-MIT.txt +21 -0
  108. package/native/ios/THIRD_PARTY_LICENSES/LICENSE-llama-cpp-unicode-data-Unicode-3.0.txt +39 -0
  109. package/native/ios/THIRD_PARTY_LICENSES/LICENSE-nlohmann-json-MIT.txt +21 -0
  110. package/native/ios/THIRD_PARTY_LICENSES/LICENSE-nvidia-cub-BSD-3-Clause.txt +24 -0
  111. package/native/ios/THIRD_PARTY_LICENSES/LICENSE-vulkan-headers-Apache-2.0-OR-MIT.txt +33 -0
  112. package/native/ios/THIRD_PARTY_LICENSES/LICENSE-vulkan-loader-APACHE.txt +201 -0
  113. package/native/ios/THIRD_PARTY_LICENSES/LICENSE-vulkan-loader-cjson-MIT.txt +32 -0
  114. package/native/ios/THIRD_PARTY_LICENSES/LICENSE-yarn-MIT.txt +21 -0
  115. package/native/ios/THIRD_PARTY_LICENSES/manifest.json +21 -0
  116. package/native/ios/engine-manifest.json +102 -0
  117. package/native/ios/sdk-manifest.json +89 -0
  118. package/package.json +94 -3
  119. package/scripts/embedding-package.d.mts +23 -0
  120. package/scripts/embedding-package.mjs +185 -0
  121. package/scripts/stage-native.mjs +168 -0
@@ -0,0 +1,1349 @@
1
+ import Foundation
2
+ import ImageIO
3
+ import UIKit
4
+ import GezelLlama
5
+ import os
6
+
7
+ private final class NativeChatStream {
8
+ weak var plugin: GezelNativeRuntime?
9
+ let requestId: String
10
+ var text = ""
11
+ /** Set on the first chunk, which is when decoding began. */
12
+ var generating = false
13
+ init(plugin: GezelNativeRuntime, requestId: String) {
14
+ self.plugin = plugin
15
+ self.requestId = requestId
16
+ }
17
+ }
18
+
19
+ private func receiveLlamaChunk(_ bytes: UnsafePointer<CChar>?, _ length: Int, _ context: UnsafeMutableRawPointer?) -> Int32 {
20
+ guard let bytes, let context else { return 1 }
21
+ let stream = Unmanaged<NativeChatStream>.fromOpaque(context).takeUnretainedValue()
22
+ guard let plugin = stream.plugin, !plugin.isCancelled(stream.requestId) else { return 1 }
23
+ let delta = String(decoding: UnsafeRawBufferPointer(start: bytes, count: length), as: UTF8.self)
24
+ stream.text.append(delta)
25
+ let first = !stream.generating
26
+ stream.generating = true
27
+ DispatchQueue.main.async {
28
+ if first { plugin.notifyListeners("enginePhase", data: ["requestId": stream.requestId, "phase": "generating"]) }
29
+ plugin.notifyListeners("chatDelta", data: ["requestId": stream.requestId, "delta": delta])
30
+ }
31
+ return 0
32
+ }
33
+
34
+ /**
35
+ * Chat chunks travel to JavaScript in small batches: one bridge event per token
36
+ * costs more than the token, and the order must hold. Touched only on the
37
+ * inference thread; flushed about every 24 ms while tokens stream, and once
38
+ * more when the request ends.
39
+ */
40
+ private final class NativeChatBatch {
41
+ weak var plugin: GezelNativeRuntime?
42
+ let requestId: String
43
+ private var pending: [String] = []
44
+ private var flushedAt = DispatchTime.now().uptimeNanoseconds
45
+ var generating = false
46
+ init(plugin: GezelNativeRuntime, requestId: String) {
47
+ self.plugin = plugin
48
+ self.requestId = requestId
49
+ }
50
+ func add(_ chunk: String) {
51
+ pending.append(chunk)
52
+ if DispatchTime.now().uptimeNanoseconds - flushedAt >= 24_000_000 { flush() }
53
+ }
54
+ func flush() {
55
+ flushedAt = DispatchTime.now().uptimeNanoseconds
56
+ guard !pending.isEmpty, let plugin else { return }
57
+ let chunks = pending, requestId = requestId
58
+ pending.removeAll()
59
+ DispatchQueue.main.async { plugin.notifyListeners("chatChunk", data: ["requestId": requestId, "chunks": chunks]) }
60
+ }
61
+ }
62
+
63
+ /// The vision describer's text, collected without streaming it to JavaScript.
64
+ private final class NativeTextCollector {
65
+ weak var plugin: GezelNativeRuntime?
66
+ let requestId: String
67
+ var text = ""
68
+ init(plugin: GezelNativeRuntime, requestId: String) {
69
+ self.plugin = plugin
70
+ self.requestId = requestId
71
+ }
72
+ }
73
+
74
+ private func receiveLlamaText(_ bytes: UnsafePointer<CChar>?, _ length: Int, _ context: UnsafeMutableRawPointer?) -> Int32 {
75
+ guard let bytes, let context else { return 1 }
76
+ let collector = Unmanaged<NativeTextCollector>.fromOpaque(context).takeUnretainedValue()
77
+ guard let plugin = collector.plugin, !plugin.isCancelled(collector.requestId) else { return 1 }
78
+ collector.text.append(String(decoding: UnsafeRawBufferPointer(start: bytes, count: length), as: UTF8.self))
79
+ return 0
80
+ }
81
+
82
+ private func receiveLlamaJSON(_ bytes: UnsafePointer<CChar>?, _ length: Int, _ context: UnsafeMutableRawPointer?) -> Int32 {
83
+ guard let bytes, let context else { return 1 }
84
+ let batch = Unmanaged<NativeChatBatch>.fromOpaque(context).takeUnretainedValue()
85
+ guard let plugin = batch.plugin, !plugin.isCancelled(batch.requestId) else { return 1 }
86
+ if !batch.generating {
87
+ batch.generating = true
88
+ plugin.notifyPhase(batch.requestId, "generating")
89
+ }
90
+ batch.add(String(decoding: UnsafeRawBufferPointer(start: bytes, count: length), as: UTF8.self))
91
+ return plugin.isCancelled(batch.requestId) ? 1 : 0
92
+ }
93
+
94
+ #if canImport(GezelModelStorage)
95
+ import GezelModelStorage
96
+ #endif
97
+
98
+ /// One process-owned provider/model/lifecycle host shared by native and Capacitor clients.
99
+ public final class GezelNativeRuntime: @unchecked Sendable {
100
+ private static let sharedLock = NSLock()
101
+ private static var sharedRuntime: GezelNativeRuntime?
102
+ private static var sharedRoot: URL?
103
+ public static func shared(root: URL) throws -> GezelNativeRuntime {
104
+ sharedLock.lock(); defer { sharedLock.unlock() }
105
+ let canonical = try MobileModelStore.validatedRoot(root)
106
+ if let existing = sharedRuntime {
107
+ guard sharedRoot == canonical else { throw MobileInferenceError(code: "ALREADY_CONFIGURED", message: "The process runtime already owns another model root") }
108
+ return existing
109
+ }
110
+ let runtime = GezelNativeRuntime(root: canonical)
111
+ sharedRoot = canonical; sharedRuntime = runtime
112
+ return runtime
113
+ }
114
+ public static func shared() throws -> GezelNativeRuntime {
115
+ sharedLock.lock(); let existing = sharedRuntime; sharedLock.unlock()
116
+ if let existing { return existing }
117
+ let support = try FileManager.default.url(for: .applicationSupportDirectory, in: .userDomainMask, appropriateFor: nil, create: true)
118
+ return try shared(root: support.appendingPathComponent("Gezel", isDirectory: true))
119
+ }
120
+ private var listeners: [UUID: (String, [String: Any]) -> Void] = [:]
121
+ public func listen(_ callback: @escaping (String, [String: Any]) -> Void) -> UUID {
122
+ operationLock.lock(); defer { operationLock.unlock() }
123
+ let id = UUID(); listeners[id] = callback; return id
124
+ }
125
+ public func removeListener(_ id: UUID) {
126
+ operationLock.lock(); defer { operationLock.unlock() }; listeners.removeValue(forKey: id)
127
+ }
128
+ /**
129
+ * Model-loading and prompt-processing progress for the status pill, polled
130
+ * from the cancellation timer. Reports only changes.
131
+ */
132
+ fileprivate func reportProgress(_ requestId: String, last: inout (phase: UInt32, value: UInt32)) {
133
+ guard let engine else { return }
134
+ var progress = gezel_llama_progress()
135
+ progress.struct_size = UInt32(MemoryLayout<gezel_llama_progress>.size)
136
+ progress.abi_version = gezel_llama_abi_version()
137
+ guard gezel_llama_get_progress(engine, &progress) == 0 else { return }
138
+ let value: UInt32
139
+ switch progress.phase {
140
+ case 1: value = UInt32(progress.load_fraction * 1000)
141
+ case 2: value = progress.processed_tokens
142
+ default: return
143
+ }
144
+ guard progress.phase != last.phase || value != last.value else { return }
145
+ last = (progress.phase, value)
146
+ var data: [String: Any] = ["requestId": requestId]
147
+ if progress.phase == 1 {
148
+ data["phase"] = "loading_model"
149
+ data["progress"] = min(1.0, Double(progress.load_fraction))
150
+ } else {
151
+ data["phase"] = "prefill"
152
+ data["promptTokens"] = Int(progress.prompt_tokens)
153
+ data["processedTokens"] = Int(progress.processed_tokens)
154
+ data["reusedTokens"] = Int(progress.reused_tokens)
155
+ if progress.prompt_tokens > 0 {
156
+ data["progress"] = min(1.0, Double(progress.processed_tokens) / Double(progress.prompt_tokens))
157
+ }
158
+ }
159
+ DispatchQueue.main.async {
160
+ guard !self.isCancelled(requestId) else { return }
161
+ self.notifyListeners("enginePhase", data: data)
162
+ }
163
+ }
164
+
165
+ /** Engine phase for the status pill, ordered with `chatDelta` on the main queue. */
166
+ fileprivate func notifyPhase(_ requestId: String, _ phase: String) {
167
+ DispatchQueue.main.async {
168
+ self.notifyListeners("enginePhase", data: ["requestId": requestId, "phase": phase])
169
+ }
170
+ }
171
+
172
+ fileprivate func notifyListeners(_ event: String, data: [String: Any]) {
173
+ operationLock.lock(); let callbacks = Array(listeners.values); operationLock.unlock()
174
+ for callback in callbacks { callback(event, data) }
175
+ }
176
+ private let storageQueue = DispatchQueue(label: "com.bendyline.gezel.mobile.storage")
177
+ private let inferenceQueue = DispatchQueue(label: "com.bendyline.gezel.mobile.inference", qos: .userInitiated)
178
+ private let operationLock = NSLock()
179
+ private var activeId: String?
180
+ private var activeNativeId: UInt64 = 0
181
+ private var nextNativeId: UInt64 = 0
182
+ private var cancelled = false
183
+ private var cancelWaiters: [NativeCall] = []
184
+ private var modelMutation = false
185
+ private var backgroundObserver: NSObjectProtocol?
186
+ private var foregroundObserver: NSObjectProtocol?
187
+ private var backgrounded = false
188
+ private var memoryObserver: NSObjectProtocol?
189
+ private var thermalObserver: NSObjectProtocol?
190
+ private var lastMemoryWarning: TimeInterval = -.infinity
191
+ private var appleTask: Task<Void, Never>?
192
+ /// Native tool calls parked until the app's tool loop completes them.
193
+ private var pendingToolCalls: [String: CheckedContinuation<NativeToolReply, Error>] = [:]
194
+ /// Awake-time start of the model's current stretch of generation. Time the
195
+ /// app spends executing a tool is not the model's and is not charged.
196
+ private var modelSegmentStart: TimeInterval = 0
197
+ private var activeFailure: MobileInferenceError?
198
+ private var releasing = false
199
+ private var releaseScheduled = false
200
+ private var releaseWaiters: [NativeCall] = []
201
+ private var downloads: ModelDownloads?
202
+ private var store: MobileModelStore?
203
+ private var storeError: Error?
204
+ private let engine = gezel_llama_create()
205
+ // Written and read under sizingLock: the model listing runs on the storage
206
+ // queue and fits windows against what the inference queue holds.
207
+ private var loadedModelId: String?
208
+ private var loadedPath: String?
209
+ private var loadedContextSize = 0
210
+ /// Serializes the bridge's dry-run sizing with model loads, and guards the
211
+ /// loaded-model fields and `allocations`.
212
+ private let sizingLock = NSLock()
213
+ /// Bytes per model file and window from the bridge's dry run, or -1 when
214
+ /// llama.cpp cannot load the file at that window. Imported files never
215
+ /// change in place, so an entry cannot go stale.
216
+ private var allocations: [String: Int64] = [:]
217
+ private var fittedWindows: [String: Int] = [:]
218
+ /// The bridge's own buffers beside llama.cpp's: the hybrid/windowed state
219
+ /// checkpoint, token vectors and the reply text.
220
+ private static let bridgeBufferBytes: Int64 = 128 * 1024 * 1024
221
+ /// Windows offered above the 4K floor, largest first. Phones and small
222
+ /// desktops aim for 8K-16K; the floor is what the phone prompt was sized for.
223
+ private static let contextLadder = [16384, 8192]
224
+ private static let floorContext = 4096
225
+ /// Room a larger window must leave in the process allowance, so the chosen
226
+ /// window is not the one that barely fits.
227
+ private static let ladderSpareBytes: Int64 = 256 * 1024 * 1024
228
+ /// A description needs the picture's tokens and a few hundred more.
229
+ private static let describeContext = 4096
230
+ /// The projector's encoder graph and image batch beside the weights;
231
+ /// desktop budgets the same flat amount (core model-fit.ts).
232
+ private static let visionComputeBytes: UInt64 = 384 * 1024 * 1024
233
+ /// Long edge of the pixels handed to the projector, which sizes them again
234
+ /// under its own token cap; more only costs the decode.
235
+ private static let describeMaxEdge = 768
236
+
237
+ private init(root: URL) {
238
+ do {
239
+ store = try MobileModelStore(root: root)
240
+ downloads = try ModelDownloads(store: store!)
241
+ } catch { storeError = error }
242
+ backgroundObserver = NotificationCenter.default.addObserver(forName: UIApplication.didEnterBackgroundNotification, object: nil, queue: .main) { [weak self] _ in
243
+ guard let self else { return }
244
+ self.operationLock.lock(); self.backgrounded = true; self.operationLock.unlock()
245
+ self.downloads?.suspend()
246
+ self.requestRelease(nil)
247
+ self.notifyListeners("appBackground", data: [:])
248
+ }
249
+ foregroundObserver = NotificationCenter.default.addObserver(forName: UIApplication.didBecomeActiveNotification, object: nil, queue: .main) { [weak self] _ in
250
+ guard let self else { return }
251
+ self.operationLock.lock(); self.backgrounded = false; self.operationLock.unlock()
252
+ }
253
+ memoryObserver = NotificationCenter.default.addObserver(forName: UIApplication.didReceiveMemoryWarningNotification, object: nil, queue: .main) { [weak self] _ in
254
+ guard let self else { return }
255
+ self.operationLock.lock()
256
+ self.lastMemoryWarning = ProcessInfo.processInfo.systemUptime
257
+ self.activeFailure = MobileInferenceError(code: "RESOURCE_LIMIT", message: "The device is low on memory. The current model has been released.")
258
+ self.operationLock.unlock()
259
+ self.requestRelease(nil)
260
+ }
261
+ // Only .critical stops a reply mid-generation. At .serious iOS asks apps
262
+ // to scale back and already throttles the clocks itself; aborting there
263
+ // ended every reply of a charging iPhone 14 Pro Max about 20 s in, so
264
+ // no task could finish (2026-09-30).
265
+ thermalObserver = NotificationCenter.default.addObserver(forName: ProcessInfo.thermalStateDidChangeNotification, object: nil, queue: .main) { [weak self] _ in
266
+ guard let self, ProcessInfo.processInfo.thermalState == .critical else { return }
267
+ self.operationLock.lock()
268
+ self.activeFailure = MobileInferenceError(code: "RESOURCE_LIMIT", message: "The device is too warm for local inference. Let it cool down before trying again.")
269
+ self.operationLock.unlock()
270
+ self.requestRelease(nil)
271
+ }
272
+
273
+ }
274
+
275
+ deinit {
276
+ downloads?.suspend()
277
+ if let backgroundObserver { NotificationCenter.default.removeObserver(backgroundObserver) }
278
+ if let foregroundObserver { NotificationCenter.default.removeObserver(foregroundObserver) }
279
+ if let memoryObserver { NotificationCenter.default.removeObserver(memoryObserver) }
280
+ if let thermalObserver { NotificationCenter.default.removeObserver(thermalObserver) }
281
+ // Inference closures retain this runtime until its blocking native call
282
+ // returns; no active operation can outlive this engine pointer.
283
+ gezel_llama_destroy(engine)
284
+ }
285
+
286
+ private func withStore(_ call: NativeCall, completion: (() -> Void)? = nil, _ action: @escaping (MobileModelStore) throws -> [String: Any]) {
287
+ storageQueue.async {
288
+ let result = Result {
289
+ guard let store = self.store else { throw self.storeError ?? MobileStoreError.invalidState }
290
+ return try action(store)
291
+ }
292
+ completion?()
293
+ DispatchQueue.main.async {
294
+ switch result {
295
+ case .success(let data): call.resolve(data)
296
+ case .failure(let error): call.reject(error.localizedDescription)
297
+ }
298
+ }
299
+ }
300
+ }
301
+
302
+ private func modelJSON(_ model: MobileModel) -> [String: Any] {
303
+ var result: [String:Any] = ["id": model.id, "name": model.name, "sizeBytes": model.sizeBytes]
304
+ if let source = model.source, let data = try? JSONEncoder().encode(source), let json = try? JSONSerialization.jsonObject(with: data) { result["source"] = json }
305
+ return result
306
+ }
307
+
308
+ public func listModels(_ call: NativeCall) {
309
+ withStore(call) { store in
310
+ let library = try store.listModels()
311
+ var models = library.models.map(self.modelJSON)
312
+ // The window the phone can hold for each model, which the product
313
+ // runtime uses unless the person chose one. Every model is sized,
314
+ // not only the selected one: a conversation keeps the model it
315
+ // started with, and a Gemma 4 E2B thread sized as if it were the
316
+ // selected Qwen 3.5 2B asked for a 16K window it could not hold
317
+ // (2026-09-30). Dry runs are cached per file and window.
318
+ for (index, model) in library.models.enumerated() {
319
+ if let located = try? store.modelURL(id: model.id),
320
+ let context = self.fitContext(id: model.id, path: located.1.path) {
321
+ models[index]["contextTokens"] = context
322
+ }
323
+ }
324
+ var result: [String: Any] = ["models": models]
325
+ if let selected = library.selectedModelId { result["selectedModelId"] = selected }
326
+ if let budget = self.memoryBudget() { result["memoryBudgetBytes"] = budget }
327
+ return result
328
+ }
329
+ }
330
+
331
+ /// What one model may use here: the process allowance plus what a loaded
332
+ /// model already holds. Settings hides catalog downloads that cannot fit
333
+ /// it, such as Gemma 4 E4B on a 6 GB iPhone. The simulator has no allowance.
334
+ private func memoryBudget() -> Int64? {
335
+ #if targetEnvironment(simulator)
336
+ return nil
337
+ #else
338
+ sizingLock.lock()
339
+ let heldPath = loadedPath, heldContext = loadedContextSize
340
+ sizingLock.unlock()
341
+ var available = Int64(os_proc_available_memory())
342
+ if let heldPath { available += max(0, allocation(path: heldPath, contextSize: heldContext)) }
343
+ return available > 0 ? available : nil
344
+ #endif
345
+ }
346
+
347
+ private func loadOptions(contextSize: Int) -> gezel_llama_load_options {
348
+ var options = gezel_llama_default_load_options()
349
+ options.context_tokens = UInt32(contextSize)
350
+ #if !targetEnvironment(simulator)
351
+ options.gpu_layers = -1
352
+ #endif
353
+ return options
354
+ }
355
+
356
+ /// Bytes a load at this window takes, from the bridge's dry run, or -1 when
357
+ /// llama.cpp cannot load the file at it. Unlike Android, iOS charges the
358
+ /// memory-mapped weights too: Metal wraps them, and whether the process
359
+ /// footprint then counts those pages has not been measured.
360
+ private func allocation(path: String, contextSize: Int) -> Int64 {
361
+ guard let engine else { return -1 }
362
+ let size = (try? FileManager.default.attributesOfItem(atPath: path)[.size] as? NSNumber)?.int64Value ?? 0
363
+ let key = "\(path)\n\(size)\n\(contextSize)"
364
+ sizingLock.lock(); defer { sizingLock.unlock() }
365
+ if let known = allocations[key] { return known }
366
+ var options = loadOptions(contextSize: contextSize)
367
+ var estimate = gezel_llama_memory_estimate()
368
+ estimate.struct_size = UInt32(MemoryLayout<gezel_llama_memory_estimate>.size)
369
+ estimate.abi_version = gezel_llama_abi_version()
370
+ var error = gezel_llama_error()
371
+ let status = path.withCString { gezel_llama_estimate_memory(engine, $0, &options, &estimate, &error) }
372
+ let bytes = status == 0
373
+ ? Int64(estimate.model_bytes + estimate.context_bytes + estimate.compute_bytes) + Self.bridgeBufferBytes
374
+ : -1
375
+ allocations[key] = bytes
376
+ return bytes
377
+ }
378
+
379
+ /// Admission charge. A file the dry run cannot size keeps the old flat floor
380
+ /// (weights, 256 MiB, 64 KiB per token); its load then reports the real error.
381
+ private func requiredBytes(path: String, sizeBytes: Int64, contextSize: Int) -> UInt64 {
382
+ let bytes = allocation(path: path, contextSize: contextSize)
383
+ return bytes >= 0 ? UInt64(bytes) : UInt64(sizeBytes) + 256 * 1024 * 1024 + UInt64(contextSize) * 64 * 1024
384
+ }
385
+
386
+ /// 16K or 8K when the process allowance holds that window with room to
387
+ /// spare, else the 4K floor, which admission still checks. A loaded model
388
+ /// keeps its window, so a listing never makes the next turn reload it;
389
+ /// memory another loaded model holds counts as free, since loading this one
390
+ /// releases it.
391
+ /// Logs a model's window only when it changes; the listing is polled.
392
+ private func noteFittedWindow(id: String, context: Int, bytes: Int64, available: Int64) {
393
+ sizingLock.lock()
394
+ let changed = fittedWindows[id] != context
395
+ fittedWindows[id] = context
396
+ sizingLock.unlock()
397
+ if changed { NSLog("GezelRuntime window %ld for %@: needs %lld bytes, %lld available", context, id, bytes, available) }
398
+ }
399
+
400
+ private func fitContext(id: String, path: String) -> Int? {
401
+ guard engine != nil else { return nil }
402
+ sizingLock.lock()
403
+ let heldId = loadedModelId, heldPath = loadedPath, heldContext = loadedContextSize
404
+ sizingLock.unlock()
405
+ if heldId == id { return heldContext }
406
+ #if targetEnvironment(simulator)
407
+ // The simulator reports no allowance; checkResources keeps it small.
408
+ return nil
409
+ #else
410
+ var available = Int64(os_proc_available_memory())
411
+ if let heldPath { available += max(0, allocation(path: heldPath, contextSize: heldContext)) }
412
+ for context in Self.contextLadder {
413
+ let bytes = allocation(path: path, contextSize: context)
414
+ if bytes >= 0, bytes + Self.ladderSpareBytes <= available {
415
+ noteFittedWindow(id: id, context: context, bytes: bytes, available: available)
416
+ return context
417
+ }
418
+ }
419
+ noteFittedWindow(id: id, context: Self.floorContext, bytes: allocation(path: path, contextSize: Self.floorContext), available: available)
420
+ return Self.floorContext
421
+ #endif
422
+ }
423
+
424
+ private func downloadSource(_ call: NativeCall) throws -> MobileModelSource {
425
+ guard let raw = call.getObject("source") else { throw ModelDownloadError("A verified model source is required") }
426
+ return try JSONDecoder().decode(MobileModelSource.self, from: JSONSerialization.data(withJSONObject: raw))
427
+ }
428
+ private func reserveDownloadAdmission() -> Bool {
429
+ operationLock.lock(); defer { operationLock.unlock() }
430
+ guard activeId == nil, !modelMutation, !releasing, !backgrounded, downloads != nil else { return false }
431
+ modelMutation = true; return true
432
+ }
433
+ public func resolveModelSource(_ call: NativeCall) {
434
+ guard reserveDownloadAdmission() else { call.reject("Open the app and finish the current operation before checking a model", "BUSY"); return }
435
+ do {
436
+ let source = try downloadSource(call)
437
+ operationLock.lock()
438
+ guard !backgrounded else { operationLock.unlock(); throw ModelDownloadError("Open the app to check a model source") }
439
+ downloads!.resolve(source) { result in
440
+ DispatchQueue.main.async {
441
+ self.releaseModelMutation()
442
+ do { call.resolve(["source": try JSONSerialization.jsonObject(with: JSONEncoder().encode(result.get()))]) }
443
+ catch { call.reject(error.localizedDescription) }
444
+ }
445
+ }
446
+ operationLock.unlock()
447
+ } catch { releaseModelMutation(); call.reject(error.localizedDescription) }
448
+ }
449
+ public func cancelModelSourceResolution(_ call: NativeCall) { downloads?.cancelSourceResolution(); call.resolve() }
450
+ public func listModelDownloads(_ call: NativeCall) {
451
+ withStore(call) { _ in guard let downloads = self.downloads else { throw ModelDownloadError("Model storage is unavailable") }; return ["downloads": try downloads.list().map { try $0.json() }] }
452
+ }
453
+ public func startModelDownload(_ call: NativeCall) {
454
+ guard reserveDownloadAdmission() else { call.reject("Finish the current operation before downloading a model", "BUSY"); return }
455
+ withStore(call, completion: { self.releaseModelMutation() }) { _ in
456
+ // The lifecycle observers take operationLock on the main thread.
457
+ // Holding it across this call would stall them: starting or
458
+ // resuming waits on the downloads queue, and that queue may be
459
+ // part way through hashing a multi-gigabyte file. Read the flag
460
+ // under the lock and let go before the slow part.
461
+ self.operationLock.lock()
462
+ let backgrounded = self.backgrounded
463
+ self.operationLock.unlock()
464
+ guard !backgrounded, let name = call.getString("name") else { throw ModelDownloadError("Open the app and choose a model to download") }
465
+ return ["download": try self.downloads!.start(source: self.downloadSource(call), name: name).json()]
466
+ }
467
+ }
468
+ public func resumeModelDownload(_ call: NativeCall) {
469
+ guard reserveDownloadAdmission() else { call.reject("Finish the current operation before resuming a model", "BUSY"); return }
470
+ withStore(call, completion: { self.releaseModelMutation() }) { _ in
471
+ // The lifecycle observers take operationLock on the main thread.
472
+ // Holding it across this call would stall them: starting or
473
+ // resuming waits on the downloads queue, and that queue may be
474
+ // part way through hashing a multi-gigabyte file. Read the flag
475
+ // under the lock and let go before the slow part.
476
+ self.operationLock.lock()
477
+ let backgrounded = self.backgrounded
478
+ self.operationLock.unlock()
479
+ guard !backgrounded, let id = call.getString("id") else { throw ModelDownloadError("Open the app and choose a download to resume") }
480
+ return ["download": try self.downloads!.resume(id: id).json()]
481
+ }
482
+ }
483
+ public func cancelModelDownload(_ call: NativeCall) {
484
+ guard let id = call.getString("id"), let downloads else { call.reject("Download ID is required"); return }
485
+ downloads.cancel(id: id) { DispatchQueue.main.async { call.resolve() } }
486
+ }
487
+ public func removeModelDownload(_ call: NativeCall) {
488
+ guard let id = call.getString("id"), let downloads else { call.reject("Download ID is required"); return }
489
+ downloads.remove(id: id) { result in DispatchQueue.main.async {
490
+ do { try result.get(); call.resolve() } catch { call.reject(error.localizedDescription) }
491
+ }}
492
+ }
493
+
494
+ public func reserveModelMutation() -> Bool {
495
+ operationLock.lock(); defer { operationLock.unlock() }
496
+ guard activeId == nil, !modelMutation, !releasing, !backgrounded else { return false }
497
+ modelMutation = true
498
+ return true
499
+ }
500
+
501
+ public func releaseModelMutation() {
502
+ operationLock.lock(); modelMutation = false; operationLock.unlock()
503
+ scheduleReleaseIfIdle()
504
+ }
505
+
506
+ public func selectModel(_ call: NativeCall) {
507
+ guard let id = call.getString("id") else { call.reject("Model ID is required"); return }
508
+ guard reserveModelMutation() else { call.reject("Wait for the current conversation or import to finish", "BUSY"); return }
509
+ withStore(call, completion: { self.releaseModelMutation() }) { store in
510
+ return ["model": self.modelJSON(try store.selectModel(id: id))]
511
+ }
512
+ }
513
+
514
+ fileprivate func isCancelled(_ id: String) -> Bool {
515
+ operationLock.lock(); defer { operationLock.unlock() }
516
+ return activeId != id || cancelled
517
+ }
518
+
519
+ private func cancelActive(_ id: String?) {
520
+ operationLock.lock()
521
+ var task: Task<Void, Never>?
522
+ var parked: [CheckedContinuation<NativeToolReply, Error>] = []
523
+ if let activeId, id == nil || id == activeId {
524
+ cancelled = true
525
+ gezel_llama_cancel(engine, activeNativeId)
526
+ task = appleTask
527
+ parked = Array(pendingToolCalls.values); pendingToolCalls.removeAll()
528
+ }
529
+ operationLock.unlock()
530
+ parked.forEach { $0.resume(throwing: CancellationError()) }
531
+ task?.cancel()
532
+ }
533
+
534
+ public func cancel(_ call: NativeCall) {
535
+ guard let id = call.getString("requestId") else { call.reject("Request ID is required"); return }
536
+ operationLock.lock()
537
+ guard activeId == id else { operationLock.unlock(); call.resolve(); return }
538
+ cancelled = true
539
+ cancelWaiters.append(call)
540
+ gezel_llama_cancel(engine, activeNativeId)
541
+ let task = appleTask
542
+ let parked = Array(pendingToolCalls.values); pendingToolCalls.removeAll()
543
+ operationLock.unlock()
544
+ parked.forEach { $0.resume(throwing: CancellationError()) }
545
+ task?.cancel()
546
+ }
547
+
548
+ private func beginModelSegment() {
549
+ operationLock.lock(); defer { operationLock.unlock() }
550
+ modelSegmentStart = ProcessInfo.processInfo.systemUptime
551
+ }
552
+
553
+ /// Model time in the current stretch; zero while the app executes a tool.
554
+ private func modelSegmentSeconds() -> TimeInterval {
555
+ operationLock.lock(); defer { operationLock.unlock() }
556
+ return pendingToolCalls.isEmpty ? ProcessInfo.processInfo.systemUptime - modelSegmentStart : 0
557
+ }
558
+
559
+ /// Completes a native tool call with the app tool loop's result.
560
+ public func completeToolCall(_ call: NativeCall) {
561
+ guard let requestId = call.getString("requestId"), let callId = call.getString("callId") else {
562
+ call.reject("A request and tool call ID are required", "INVALID_ARGUMENT"); return
563
+ }
564
+ let result: Result<NativeToolReply, Error>
565
+ if let error = call.getString("error") {
566
+ result = .failure(MobileInferenceError(code: "TOOL_FAILED", message: String(error.prefix(2_000))))
567
+ } else if let output = call.getString("output"), output.utf8.count <= 262_144, !output.contains("\0") {
568
+ result = .success(NativeToolReply(output: output, endTurn: call.getBool("endTurn") ?? false))
569
+ } else {
570
+ call.reject("Invalid tool result", "INVALID_ARGUMENT"); return
571
+ }
572
+ operationLock.lock()
573
+ let continuation = activeId == requestId ? pendingToolCalls.removeValue(forKey: callId) : nil
574
+ if continuation != nil { modelSegmentStart = ProcessInfo.processInfo.systemUptime }
575
+ operationLock.unlock()
576
+ guard let continuation else { call.reject("This tool call is no longer running", "NOT_RUNNING"); return }
577
+ continuation.resume(with: result)
578
+ call.resolve()
579
+ }
580
+
581
+ /// Parks one native tool call until the app completes it, announcing it to JavaScript.
582
+ private func awaitToolCall(requestId: String, name: String, arguments: String) async throws -> NativeToolReply {
583
+ let callId = UUID().uuidString
584
+ return try await withTaskCancellationHandler {
585
+ try await withCheckedThrowingContinuation { (continuation: CheckedContinuation<NativeToolReply, Error>) in
586
+ operationLock.lock()
587
+ guard activeId == requestId, !cancelled else {
588
+ operationLock.unlock(); continuation.resume(throwing: CancellationError()); return
589
+ }
590
+ pendingToolCalls[callId] = continuation
591
+ operationLock.unlock()
592
+ DispatchQueue.main.async {
593
+ self.notifyListeners("toolCall", data: ["requestId": requestId, "callId": callId, "name": name, "arguments": arguments])
594
+ }
595
+ }
596
+ } onCancel: {
597
+ operationLock.lock(); let continuation = pendingToolCalls.removeValue(forKey: callId); operationLock.unlock()
598
+ continuation?.resume(throwing: CancellationError())
599
+ }
600
+ }
601
+
602
+ private func nextOperation(_ requestId: String) -> UInt64? {
603
+ operationLock.lock(); defer { operationLock.unlock() }
604
+ guard activeId == requestId, !cancelled else { return nil }
605
+ nextNativeId = nextNativeId >= UInt64(Int64.max) ? 1 : nextNativeId + 1
606
+ activeNativeId = nextNativeId
607
+ return activeNativeId
608
+ }
609
+
610
+ public func generate(_ call: NativeCall) {
611
+ guard let requestId = call.getString("requestId"), !requestId.isEmpty, requestId.utf8.count <= 128,
612
+ let messages = call.getArray("messages", [String: Any].self), !messages.isEmpty, messages.count <= 128 else {
613
+ call.reject("A request ID and conversation are required"); return
614
+ }
615
+ let providerId = call.getString("providerId") ?? "llama-cpp"
616
+ guard ["llama-cpp", "apple-foundation-models"].contains(providerId) else {
617
+ call.reject("This provider is not available on iOS", "UNAVAILABLE"); return
618
+ }
619
+ if providerId == "apple-foundation-models", let modelId = call.getString("modelId"), modelId != providerId {
620
+ call.reject("The requested model is not available from Apple on-device AI", "MODEL_UNAVAILABLE"); return
621
+ }
622
+ if providerId == "apple-foundation-models", let reason = AppleFoundationProvider.availability().reason {
623
+ call.reject(reason, "UNAVAILABLE"); return
624
+ }
625
+ var turns: [MobileChatTurn] = []
626
+ var inputBytes = 0
627
+ for message in messages {
628
+ guard let role = message["role"] as? String, ["system", "user", "assistant"].contains(role),
629
+ let content = message["content"] as? String, !content.contains("\0") else {
630
+ call.reject("Invalid conversation message"); return
631
+ }
632
+ inputBytes += content.utf8.count
633
+ turns.append(MobileChatTurn(role: role, content: content))
634
+ }
635
+ let modelId = call.getString("modelId")
636
+ if providerId == "llama-cpp", modelId?.isEmpty != false {
637
+ call.reject("Choose a model for this conversation"); return
638
+ }
639
+ guard ["maxTokens", "contextSize"].allSatisfy({ !call.contains($0) || call.getInt($0) != nil }) else {
640
+ call.reject("Token budgets must be integers", "INVALID_REQUEST"); return
641
+ }
642
+ var tools: [[String: Any]] = []
643
+ if call.contains("tools") {
644
+ guard providerId == "apple-foundation-models", let specs = call.getArray("tools", [String: Any].self), (1...64).contains(specs.count) else {
645
+ call.reject("Native tool calling is only available from Apple on-device AI", "UNSUPPORTED"); return
646
+ }
647
+ tools = specs
648
+ }
649
+ guard let sampling = LlamaSampling(call.getObject("sampling")) else {
650
+ call.reject("Sampling settings are outside the supported range", "INVALID_REQUEST"); return
651
+ }
652
+ let maxTokens = call.getInt("maxTokens") ?? 1024
653
+ let outputLimit = providerId == "llama-cpp" ? 4096 : AppleFoundationProvider.maximumOutputTokens
654
+ let contextSize = call.getInt("contextSize") ?? 4096
655
+ guard inputBytes <= 1_000_000, (1...outputLimit).contains(maxTokens), (512...(providerId == "llama-cpp" ? 16384 : AppleFoundationProvider.availability().contextTokens)).contains(contextSize), maxTokens + 128 < contextSize, turns.last?.role == "user" else {
656
+ call.reject("Conversation or token budget is outside the supported range"); return
657
+ }
658
+ operationLock.lock()
659
+ guard !backgrounded else { operationLock.unlock(); call.reject("Reopen the app to start a conversation", "BACKGROUND"); return }
660
+ guard activeId == nil, !modelMutation, !releasing, !backgrounded else { operationLock.unlock(); call.reject("Another conversation is running", "BUSY"); return }
661
+ activeId = requestId; cancelled = false; activeFailure = nil
662
+ if providerId == "apple-foundation-models" {
663
+ appleTask = Task {
664
+ await self.runAppleGeneration(call, requestId: requestId, turns: turns, maxTokens: maxTokens, contextSize: contextSize, tools: tools)
665
+ }
666
+ operationLock.unlock()
667
+ } else {
668
+ operationLock.unlock()
669
+ inferenceQueue.async {
670
+ self.runGeneration(call, requestId: requestId, modelId: modelId!, turns: turns, maxTokens: maxTokens, contextSize: contextSize, sampling: sampling)
671
+ }
672
+ }
673
+ }
674
+
675
+ /**
676
+ * An OpenAI-shaped chat request for an imported llama.cpp model, served by
677
+ * llama.cpp's own chat layer exactly as desktop's llama-server serves it:
678
+ * the model's template renders the tools and its parser returns structured
679
+ * tool calls. Every object llama-server would stream reaches JavaScript as a
680
+ * `chatChunk` event (batched, in order), including its error body.
681
+ */
682
+ public func chat(_ call: NativeCall) {
683
+ guard let requestId = call.getString("requestId"), !requestId.isEmpty, requestId.utf8.count <= 128,
684
+ let requestJson = call.getString("requestJson") else {
685
+ call.reject("A request ID and chat request are required"); return
686
+ }
687
+ guard let modelId = call.getString("modelId"), !modelId.isEmpty else {
688
+ call.reject("Choose a model for this conversation"); return
689
+ }
690
+ guard !call.contains("contextSize") || call.getInt("contextSize") != nil else {
691
+ call.reject("Token budgets must be integers", "INVALID_REQUEST"); return
692
+ }
693
+ let contextSize = call.getInt("contextSize") ?? 4096
694
+ guard (512...16384).contains(contextSize) else {
695
+ call.reject("Context size is outside the supported range"); return
696
+ }
697
+ // The JSON text crosses unchanged. Re-encoding a dictionary reorders its
698
+ // keys from one request to the next, and with them the tool definitions
699
+ // at the top of the prompt, so no cached prefix ever matched.
700
+ func object(_ text: String) -> Data? {
701
+ let data = Data(text.utf8)
702
+ return (try? JSONSerialization.jsonObject(with: data)) is [String: Any] ? data : nil
703
+ }
704
+ guard requestJson.utf8.count <= 1024 * 1024, let body = object(requestJson) else {
705
+ call.reject("Chat request is too large or not a JSON object"); return
706
+ }
707
+ guard let config = call.getString("chatConfigJson").map(object) ?? Data("{}".utf8) else {
708
+ call.reject("Invalid chat settings"); return
709
+ }
710
+ guard engine != nil else { call.reject("The native inference engine could not initialize.", "UNAVAILABLE"); return }
711
+ operationLock.lock()
712
+ guard !backgrounded else { operationLock.unlock(); call.reject("Reopen the app to start a conversation", "BACKGROUND"); return }
713
+ guard activeId == nil, !modelMutation, !releasing else { operationLock.unlock(); call.reject("Another conversation is running", "BUSY"); return }
714
+ activeId = requestId; cancelled = false; activeFailure = nil
715
+ operationLock.unlock()
716
+ inferenceQueue.async {
717
+ self.runChat(call, requestId: requestId, modelId: modelId, contextSize: contextSize, body: body, config: config)
718
+ }
719
+ }
720
+
721
+ /// The `configure_chat` settings the loaded model holds; inference queue only.
722
+ private var appliedChatConfig: Data?
723
+
724
+ private func runChat(_ call: NativeCall, requestId: String, modelId: String, contextSize: Int, body: Data, config: Data) {
725
+ let batch = NativeChatBatch(plugin: self, requestId: requestId)
726
+ let cancellation = DispatchSource.makeTimerSource(queue: .global(qos: .userInitiated))
727
+ cancellation.schedule(deadline: .now() + .milliseconds(50), repeating: .milliseconds(50))
728
+ var ticks = 0
729
+ var lastProgress: (phase: UInt32, value: UInt32) = (UInt32.max, UInt32.max)
730
+ cancellation.setEventHandler { [weak self] in
731
+ guard let self else { return }
732
+ if self.isCancelled(requestId) {
733
+ self.cancelActive(requestId)
734
+ return
735
+ }
736
+ ticks += 1
737
+ if ticks % 5 == 0 { self.reportProgress(requestId, last: &lastProgress) }
738
+ }
739
+ cancellation.resume()
740
+ func nativeFailure(_ status: Int32, _ error: inout gezel_llama_error) -> NSError {
741
+ NSError(domain: "GezelLlama", code: Int(status), userInfo: [NSLocalizedDescriptionKey: errorText(&error)])
742
+ }
743
+ func perform() throws -> [String: Any] {
744
+ guard let engine, let store else { throw storeError ?? MobileStoreError.unknownModel }
745
+ let (model, url) = try store.modelURL(id: modelId)
746
+ var nativeError = gezel_llama_error()
747
+ if loadedModelId != model.id || loadedContextSize != contextSize {
748
+ try unloadLlama()
749
+ appliedChatConfig = nil
750
+ guard awaitCooling(requestId) else { return ["status": "cancelled"] }
751
+ try checkResources(additionalBytes: requiredBytes(path: url.path, sizeBytes: model.sizeBytes, contextSize: contextSize))
752
+ guard let operation = nextOperation(requestId) else { return ["status": "cancelled"] }
753
+ var options = loadOptions(contextSize: contextSize)
754
+ options.request_id = operation
755
+ notifyPhase(requestId, "loading_model")
756
+ sizingLock.lock()
757
+ let status = url.path.withCString { gezel_llama_load(engine, $0, &options, &nativeError) }
758
+ if status == 0 { loadedModelId = model.id; loadedPath = url.path; loadedContextSize = contextSize }
759
+ sizingLock.unlock()
760
+ guard status == 0 else {
761
+ if isCancelled(requestId) { return ["status": "cancelled"] }
762
+ throw nativeFailure(status, &nativeError)
763
+ }
764
+ }
765
+ if config != appliedChatConfig {
766
+ let status = config.withUnsafeBytes { raw in
767
+ gezel_llama_configure_chat(engine, raw.bindMemory(to: CChar.self).baseAddress, raw.count, &nativeError)
768
+ }
769
+ guard status == 0 else { throw nativeFailure(status, &nativeError) }
770
+ appliedChatConfig = config
771
+ }
772
+ guard awaitCooling(requestId) else { return ["status": "cancelled"] }
773
+ try checkResources(additionalBytes: 64 * 1024 * 1024)
774
+ guard let operation = nextOperation(requestId) else { return ["status": "cancelled"] }
775
+ var options = gezel_llama_default_chat_options()
776
+ options.request_id = operation
777
+ options.timeout_ms = 600_000
778
+ var result = gezel_llama_result()
779
+ notifyPhase(requestId, "prefill")
780
+ let status = body.withUnsafeBytes { raw in
781
+ gezel_llama_chat(engine, raw.bindMemory(to: CChar.self).baseAddress, raw.count, &options, receiveLlamaJSON,
782
+ Unmanaged.passUnretained(batch).toOpaque(), &result, &nativeError)
783
+ }
784
+ // Invalid options, busy and not-loaded end before a request starts,
785
+ // so no error body reached the batch; every other failure sent one.
786
+ if (1...3).contains(status) { throw nativeFailure(status, &nativeError) }
787
+ if status == 0 { return ["status": "ok"] }
788
+ if isCancelled(requestId) || status == 7 { return ["status": "cancelled"] }
789
+ return ["status": status == 8 ? "timeout" : "error"]
790
+ }
791
+ var terminal: Result<[String: Any], Error>
792
+ do { terminal = .success(try perform()) }
793
+ catch {
794
+ terminal = isCancelled(requestId) ? .success(["status": "cancelled"]) : .failure(error)
795
+ }
796
+ cancellation.cancel()
797
+ batch.flush()
798
+ finishGeneration(call, terminal: terminal)
799
+ }
800
+
801
+ /// Describes one photo with a small vision model: an installed llama.cpp
802
+ /// model and its projector, downloaded beside it as a second library entry.
803
+ /// The phone's describer for a device whose OS has none, such as an iPhone
804
+ /// without Apple Intelligence. The photo arrives as base64 and is decoded,
805
+ /// turned upright, and sized here.
806
+ public func describeImage(_ call: NativeCall) {
807
+ guard let requestId = call.getString("requestId"), !requestId.isEmpty, requestId.utf8.count <= 128,
808
+ let modelId = call.getString("modelId"), !modelId.isEmpty,
809
+ let projectorId = call.getString("projectorId"), !projectorId.isEmpty, projectorId != modelId,
810
+ let encoded = call.getString("image"), !encoded.isEmpty, encoded.utf8.count <= 22_400_000,
811
+ let image = Data(base64Encoded: encoded),
812
+ let user = call.getString("user"), !user.isEmpty, user.utf8.count <= 16 * 1024, !user.contains("\0") else {
813
+ call.reject("A vision model, its projector, a photo and an instruction are required", "INVALID_REQUEST"); return
814
+ }
815
+ let system = call.getString("system")
816
+ guard (system?.utf8.count ?? 0) <= 16 * 1024, system?.contains("\0") != true,
817
+ !call.contains("maxTokens") || call.getInt("maxTokens") != nil else {
818
+ call.reject("Invalid description settings", "INVALID_REQUEST"); return
819
+ }
820
+ let maxTokens = call.getInt("maxTokens") ?? 400
821
+ guard (1...1024).contains(maxTokens) else { call.reject("Invalid description settings", "INVALID_REQUEST"); return }
822
+ guard engine != nil else { call.reject("The native inference engine could not initialize.", "UNAVAILABLE"); return }
823
+ operationLock.lock()
824
+ guard !backgrounded else { operationLock.unlock(); call.reject("Reopen the app to read photos", "BACKGROUND"); return }
825
+ guard activeId == nil, !modelMutation, !releasing else { operationLock.unlock(); call.reject("Another conversation is running", "BUSY"); return }
826
+ activeId = requestId; cancelled = false; activeFailure = nil
827
+ operationLock.unlock()
828
+ inferenceQueue.async {
829
+ self.runDescribe(call, requestId: requestId, modelId: modelId, projectorId: projectorId, image: image,
830
+ system: system, user: user, maxTokens: maxTokens)
831
+ }
832
+ }
833
+
834
+ private func runDescribe(_ call: NativeCall, requestId: String, modelId: String, projectorId: String, image: Data,
835
+ system: String?, user: String, maxTokens: Int) {
836
+ let collector = NativeTextCollector(plugin: self, requestId: requestId)
837
+ let cancellation = DispatchSource.makeTimerSource(queue: .global(qos: .userInitiated))
838
+ cancellation.schedule(deadline: .now() + .milliseconds(50), repeating: .milliseconds(50))
839
+ cancellation.setEventHandler { [weak self] in
840
+ guard let self, self.isCancelled(requestId) else { return }
841
+ self.cancelActive(requestId)
842
+ }
843
+ cancellation.resume()
844
+ func nativeFailure(_ status: Int32, _ error: inout gezel_llama_error) -> NSError {
845
+ NSError(domain: "GezelLlama", code: Int(status), userInfo: [NSLocalizedDescriptionKey: errorText(&error)])
846
+ }
847
+ func perform() throws -> [String: Any] {
848
+ guard let engine, let store else { throw storeError ?? MobileStoreError.unknownModel }
849
+ let (model, url) = try store.modelURL(id: modelId)
850
+ let (projector, projectorURL) = try store.modelURL(id: projectorId)
851
+ let pixels = try Self.rgbPixels(image, maxEdge: Self.describeMaxEdge)
852
+ var nativeError = gezel_llama_error()
853
+ let visionBytes = UInt64(projector.sizeBytes) + Self.visionComputeBytes
854
+ if loadedModelId != model.id {
855
+ try unloadLlama()
856
+ appliedChatConfig = nil
857
+ guard awaitCooling(requestId) else { return ["status": "cancelled"] }
858
+ try checkResources(additionalBytes: requiredBytes(path: url.path, sizeBytes: model.sizeBytes, contextSize: Self.describeContext) + visionBytes)
859
+ guard let operation = nextOperation(requestId) else { return ["status": "cancelled"] }
860
+ var options = loadOptions(contextSize: Self.describeContext)
861
+ options.request_id = operation
862
+ notifyPhase(requestId, "loading_model")
863
+ sizingLock.lock()
864
+ let status = url.path.withCString { gezel_llama_load(engine, $0, &options, &nativeError) }
865
+ if status == 0 { loadedModelId = model.id; loadedPath = url.path; loadedContextSize = Self.describeContext }
866
+ sizingLock.unlock()
867
+ guard status == 0 else {
868
+ if isCancelled(requestId) { return ["status": "cancelled"] }
869
+ throw nativeFailure(status, &nativeError)
870
+ }
871
+ } else {
872
+ guard awaitCooling(requestId) else { return ["status": "cancelled"] }
873
+ try checkResources(additionalBytes: visionBytes)
874
+ }
875
+ guard let operation = nextOperation(requestId) else { return ["status": "cancelled"] }
876
+ var options = gezel_llama_default_image_options()
877
+ options.request_id = operation
878
+ options.timeout_ms = 180_000
879
+ options.max_tokens = UInt32(maxTokens)
880
+ var result = gezel_llama_result()
881
+ notifyPhase(requestId, "prefill")
882
+ let status = projectorURL.path.withCString { projectorPath in
883
+ user.withCString { instruction in
884
+ pixels.data.withUnsafeBytes { raw in
885
+ let rgb = raw.bindMemory(to: UInt8.self).baseAddress
886
+ let context = Unmanaged.passUnretained(collector).toOpaque()
887
+ if let system {
888
+ return system.withCString { systemPrompt in
889
+ gezel_llama_describe_image(engine, projectorPath, rgb, UInt32(pixels.width), UInt32(pixels.height),
890
+ systemPrompt, instruction, &options, receiveLlamaText, context, &result, &nativeError)
891
+ }
892
+ }
893
+ return gezel_llama_describe_image(engine, projectorPath, rgb, UInt32(pixels.width), UInt32(pixels.height),
894
+ nil, instruction, &options, receiveLlamaText, context, &result, &nativeError)
895
+ }
896
+ }
897
+ }
898
+ if status == 0 {
899
+ return ["status": "ok", "description": collector.text, "promptTokens": Int(result.prompt_tokens),
900
+ "generatedTokens": Int(result.generated_tokens), "truncated": result.finish_reason == 2]
901
+ }
902
+ if isCancelled(requestId) || status == 7 { return ["status": "cancelled"] }
903
+ throw nativeFailure(status, &nativeError)
904
+ }
905
+ var terminal: Result<[String: Any], Error>
906
+ do { terminal = .success(try perform()) }
907
+ catch {
908
+ terminal = isCancelled(requestId) ? .success(["status": "cancelled"]) : .failure(error)
909
+ }
910
+ cancellation.cancel()
911
+ finishGeneration(call, terminal: terminal)
912
+ }
913
+
914
+ /// Upright RGB pixels, at most `maxEdge` on the long side. The thumbnail
915
+ /// path applies EXIF orientation while decoding and reads HEIC.
916
+ private static func rgbPixels(_ data: Data, maxEdge: Int) throws -> (data: Data, width: Int, height: Int) {
917
+ let options: [CFString: Any] = [
918
+ kCGImageSourceCreateThumbnailFromImageAlways: true,
919
+ kCGImageSourceCreateThumbnailWithTransform: true,
920
+ kCGImageSourceThumbnailMaxPixelSize: maxEdge,
921
+ kCGImageSourceShouldCacheImmediately: true
922
+ ]
923
+ guard let source = CGImageSourceCreateWithData(data as CFData, nil),
924
+ let image = CGImageSourceCreateThumbnailAtIndex(source, 0, options as CFDictionary) else {
925
+ throw MobileInferenceError(code: "INVALID_REQUEST", message: "This photo could not be read")
926
+ }
927
+ let width = image.width, height = image.height
928
+ var rgba = Data(count: width * height * 4)
929
+ let drawn = rgba.withUnsafeMutableBytes { raw -> Bool in
930
+ guard let context = CGContext(data: raw.baseAddress, width: width, height: height, bitsPerComponent: 8,
931
+ bytesPerRow: width * 4, space: CGColorSpaceCreateDeviceRGB(),
932
+ bitmapInfo: CGImageAlphaInfo.noneSkipLast.rawValue) else { return false }
933
+ context.draw(image, in: CGRect(x: 0, y: 0, width: width, height: height))
934
+ return true
935
+ }
936
+ guard drawn else { throw MobileInferenceError(code: "INVALID_REQUEST", message: "This photo could not be read") }
937
+ var rgb = Data(count: width * height * 3)
938
+ rgb.withUnsafeMutableBytes { output in
939
+ rgba.withUnsafeBytes { input in
940
+ let from = input.bindMemory(to: UInt8.self), to = output.bindMemory(to: UInt8.self)
941
+ for pixel in 0..<(width * height) {
942
+ to[pixel * 3] = from[pixel * 4]
943
+ to[pixel * 3 + 1] = from[pixel * 4 + 1]
944
+ to[pixel * 3 + 2] = from[pixel * 4 + 2]
945
+ }
946
+ }
947
+ }
948
+ return (rgb, width, height)
949
+ }
950
+
951
+ private func errorText(_ error: inout gezel_llama_error) -> String {
952
+ withUnsafePointer(to: &error.message) { pointer in
953
+ String(cString: UnsafeRawPointer(pointer).assumingMemoryBound(to: CChar.self))
954
+ }
955
+ }
956
+
957
+ private func resourceReason() -> String? {
958
+ operationLock.lock()
959
+ let hidden = backgrounded
960
+ let recentlyWarned = ProcessInfo.processInfo.systemUptime - lastMemoryWarning < 10
961
+ operationLock.unlock()
962
+ if hidden { return "Reopen the app to use on-device AI." }
963
+ if recentlyWarned { return "The device is low on memory. Wait before loading a model again." }
964
+ if ProcessInfo.processInfo.thermalState == .critical { return "The device is too warm for local inference. Let it cool down first." }
965
+ return nil
966
+ }
967
+
968
+ private static func gigabytes(_ bytes: UInt64) -> String {
969
+ String(format: "%.1f GB", Double(bytes) / 1_073_741_824)
970
+ }
971
+
972
+ /// How long a request waits for a critically hot device to cool before it is refused.
973
+ private static let coolingWaitSeconds: TimeInterval = 600
974
+ /// How long a request pauses at .serious before it proceeds anyway.
975
+ private static let seriousPauseSeconds: TimeInterval = 15
976
+
977
+ /// A critically hot device waits the heat out instead of failing the turn:
978
+ /// the same work, delayed, is what the person asked for. Bounded, so a
979
+ /// device that never cools still gets the refusal from `checkResources`.
980
+ /// At .serious, where iOS asks apps to scale back, each request only pauses
981
+ /// briefly so the device sheds some heat between steps, then proceeds.
982
+ /// False when the request was cancelled meanwhile.
983
+ private func awaitCooling(_ requestId: String) -> Bool {
984
+ let started = ProcessInfo.processInfo.systemUptime
985
+ var announced = false
986
+ while true {
987
+ let waited = ProcessInfo.processInfo.systemUptime - started
988
+ switch ProcessInfo.processInfo.thermalState {
989
+ case .critical: if waited >= Self.coolingWaitSeconds { return !isCancelled(requestId) }
990
+ case .serious: if waited >= Self.seriousPauseSeconds { return !isCancelled(requestId) }
991
+ default: return !isCancelled(requestId)
992
+ }
993
+ if isCancelled(requestId) { return false }
994
+ if !announced { notifyPhase(requestId, "cooling"); announced = true }
995
+ Thread.sleep(forTimeInterval: 2)
996
+ }
997
+ }
998
+
999
+ private func checkResources(additionalBytes: UInt64) throws {
1000
+ if let reason = resourceReason() { throw MobileInferenceError(code: "RESOURCE_LIMIT", message: reason) }
1001
+ #if targetEnvironment(simulator)
1002
+ // Simulator processes have no iOS dirty-memory allowance (the OS
1003
+ // query returns zero). Keep this developer path deliberately small;
1004
+ // it provides no evidence about a physical phone's memory admission.
1005
+ guard additionalBytes <= 1024 * 1024 * 1024 else {
1006
+ throw MobileInferenceError(code: "RESOURCE_LIMIT", message: "The simulator supports only small test models. Use a physical device to assess model memory requirements.")
1007
+ }
1008
+ #else
1009
+ let available = UInt64(os_proc_available_memory())
1010
+ if available < additionalBytes {
1011
+ NSLog("GezelRuntime admission refused: needs %llu bytes, %llu available", additionalBytes, available)
1012
+ throw MobileInferenceError(code: "RESOURCE_LIMIT", message: "There is not enough available memory for this model: it needs about \(Self.gigabytes(additionalBytes)), and \(Self.gigabytes(available)) is free. Choose a smaller model or close other apps.")
1013
+ }
1014
+ #endif
1015
+ }
1016
+
1017
+ public func providers(_ call: NativeCall) {
1018
+ storageQueue.async {
1019
+ let restriction = self.resourceReason()
1020
+ var llamaReason = restriction
1021
+ if llamaReason == nil {
1022
+ if self.engine == nil { llamaReason = "The native inference engine could not initialize." }
1023
+ else {
1024
+ do {
1025
+ guard let store = self.store else { throw self.storeError ?? MobileStoreError.unknownModel }
1026
+ guard !(try store.listModels().models.isEmpty) else { throw MobileStoreError.unknownModel }
1027
+ } catch { llamaReason = error.localizedDescription }
1028
+ }
1029
+ }
1030
+ let apple = AppleFoundationProvider.availability()
1031
+ func descriptor(_ id: String, _ name: String, _ reason: String?, _ context: Int, _ output: Int) -> [String: Any] {
1032
+ var value: [String: Any] = [
1033
+ "id": id, "name": name, "locality": "on-device",
1034
+ "availability": reason == nil ? "available" : "unavailable",
1035
+ "contextTokens": context, "maxOutputTokens": output,
1036
+ // structuredChat: llama.cpp's own chat layer, as desktop's llama-server runs it.
1037
+ "capabilities": ["text": true, "tools": id == "apple-foundation-models", "structuredOutput": false, "images": false, "foregroundOnly": true, "structuredChat": id == "llama-cpp"]
1038
+ ]
1039
+ if let reason { value["reason"] = reason }
1040
+ return value
1041
+ }
1042
+ let result = [
1043
+ descriptor("llama-cpp", "Imported GGUF model", llamaReason, 16384, 4096),
1044
+ descriptor("apple-foundation-models", "Apple on-device AI", restriction ?? apple.reason, apple.contextTokens, AppleFoundationProvider.maximumOutputTokens)
1045
+ ]
1046
+ DispatchQueue.main.async { call.resolve(["providers": result]) }
1047
+ }
1048
+ }
1049
+
1050
+ public func prepareProvider(_ call: NativeCall) {
1051
+ if call.getString("providerId") == "apple-foundation-models" {
1052
+ call.reject("iOS manages Apple on-device model downloads. Enable Apple Intelligence in Settings and wait for preparation to finish.", "OS_MANAGED")
1053
+ } else {
1054
+ call.reject("This provider cannot be prepared on iOS. Import and select a GGUF to use an imported model.", "UNAVAILABLE")
1055
+ }
1056
+ }
1057
+
1058
+ /// Nothing on iOS prepares a provider — `prepareProvider` always refuses,
1059
+ /// because the OS owns Apple's model downloads — so there is never a
1060
+ /// preparation in flight to stop. The method exists because the shared
1061
+ /// host contract declares it, and a host call that simply fails on one
1062
+ /// platform is worse than one that truthfully does nothing.
1063
+ public func cancelProviderPreparation(_ call: NativeCall) {
1064
+ call.resolve()
1065
+ }
1066
+
1067
+ /// Called only on inferenceQueue, with generation admission already held.
1068
+ private func unloadLlama() throws {
1069
+ if let engine {
1070
+ var error = gezel_llama_error()
1071
+ let status = gezel_llama_unload(engine, &error)
1072
+ guard status == 0 else { throw MobileInferenceError(code: "BUSY", message: errorText(&error)) }
1073
+ }
1074
+ sizingLock.lock()
1075
+ loadedModelId = nil; loadedPath = nil; loadedContextSize = 0
1076
+ sizingLock.unlock()
1077
+ }
1078
+
1079
+ public func releaseModel(_ call: NativeCall) { requestRelease(call) }
1080
+
1081
+ private func requestRelease(_ call: NativeCall?) {
1082
+ operationLock.lock()
1083
+ releasing = true
1084
+ if let call { releaseWaiters.append(call) }
1085
+ operationLock.unlock()
1086
+ cancelActive(nil)
1087
+ scheduleReleaseIfIdle()
1088
+ }
1089
+
1090
+ private func scheduleReleaseIfIdle() {
1091
+ operationLock.lock()
1092
+ guard releasing, activeId == nil, !modelMutation, !releaseScheduled else { operationLock.unlock(); return }
1093
+ releaseScheduled = true
1094
+ operationLock.unlock()
1095
+ inferenceQueue.async {
1096
+ let result = Result { try self.unloadLlama() }
1097
+ self.operationLock.lock()
1098
+ self.releasing = false; self.releaseScheduled = false
1099
+ let waiters = self.releaseWaiters
1100
+ self.releaseWaiters.removeAll()
1101
+ self.operationLock.unlock()
1102
+ DispatchQueue.main.async {
1103
+ for call in waiters {
1104
+ switch result {
1105
+ case .success: call.resolve()
1106
+ case .failure(let error): call.reject(error.localizedDescription)
1107
+ }
1108
+ }
1109
+ }
1110
+ }
1111
+ }
1112
+
1113
+ public func removeModel(_ call: NativeCall) {
1114
+ guard let id = call.getString("id") else { call.reject("Model ID is required"); return }
1115
+ guard reserveModelMutation() else { call.reject("Wait for the current conversation or import to finish", "BUSY"); return }
1116
+ inferenceQueue.async {
1117
+ do { try self.unloadLlama() }
1118
+ catch {
1119
+ self.releaseModelMutation()
1120
+ DispatchQueue.main.async { call.reject(error.localizedDescription) }
1121
+ return
1122
+ }
1123
+ self.withStore(call, completion: { self.releaseModelMutation() }) { store in
1124
+ try store.removeModel(id: id)
1125
+ return [:]
1126
+ }
1127
+ }
1128
+ }
1129
+
1130
+ private func failActive(_ id: String, error: MobileInferenceError) {
1131
+ operationLock.lock()
1132
+ guard activeId == id else { operationLock.unlock(); return }
1133
+ activeFailure = error
1134
+ operationLock.unlock()
1135
+ cancelActive(id)
1136
+ }
1137
+
1138
+ private func finishGeneration(_ call: NativeCall, terminal: Result<[String: Any], Error>) {
1139
+ operationLock.lock()
1140
+ let result: Result<[String: Any], Error> = activeFailure.map { .failure($0) } ?? terminal
1141
+ activeId = nil; activeNativeId = 0; appleTask = nil; activeFailure = nil
1142
+ let parked = Array(pendingToolCalls.values); pendingToolCalls.removeAll()
1143
+ let waiting = cancelWaiters
1144
+ cancelWaiters.removeAll()
1145
+ operationLock.unlock()
1146
+ parked.forEach { $0.resume(throwing: CancellationError()) }
1147
+ scheduleReleaseIfIdle()
1148
+ DispatchQueue.main.async {
1149
+ switch result {
1150
+ case .success(let data): call.resolve(data)
1151
+ case .failure(let error): call.reject(error.localizedDescription, (error as? MobileInferenceError)?.code)
1152
+ }
1153
+ waiting.forEach { $0.resolve() }
1154
+ }
1155
+ }
1156
+
1157
+ private func runAppleGeneration(_ call: NativeCall, requestId: String, turns: [MobileChatTurn], maxTokens: Int, contextSize: Int, tools: [[String: Any]]) async {
1158
+ var text = ""
1159
+ beginModelSegment()
1160
+ // A minute of the model's own time between tool results. Polling, rather
1161
+ // than one armed timer, lets a completed tool call restart the minute.
1162
+ let timeout = Task {
1163
+ while !Task.isCancelled {
1164
+ do { try await Task.sleep(nanoseconds: 1_000_000_000) } catch { return }
1165
+ if self.modelSegmentSeconds() > 60 {
1166
+ self.failActive(requestId, error: MobileInferenceError(code: "TIMEOUT", message: "Apple on-device AI exceeded the one-minute response limit."))
1167
+ return
1168
+ }
1169
+ }
1170
+ }
1171
+ defer { timeout.cancel() }
1172
+ let terminal: Result<[String: Any], Error>
1173
+ do {
1174
+ try await withCheckedThrowingContinuation { (continuation: CheckedContinuation<Void, Error>) in
1175
+ inferenceQueue.async { continuation.resume(with: Result { try self.unloadLlama() }) }
1176
+ }
1177
+ try checkResources(additionalBytes: 256 * 1024 * 1024)
1178
+ if isCancelled(requestId) { throw CancellationError() }
1179
+ guard #available(iOS 26.0, *) else { throw MobileInferenceError(code: "UNAVAILABLE", message: "Apple on-device AI requires iOS 26 or later.") }
1180
+ notifyPhase(requestId, "prefill")
1181
+ var generating = false
1182
+ let reason = try await AppleFoundationProvider.generate(
1183
+ turns: turns, maxTokens: maxTokens, contextSize: contextSize, tools: tools,
1184
+ invoke: { [weak self] name, arguments in
1185
+ guard let self else { throw CancellationError() }
1186
+ return try await self.awaitToolCall(requestId: requestId, name: name, arguments: arguments)
1187
+ }
1188
+ ) { delta in
1189
+ if self.isCancelled(requestId) { throw CancellationError() }
1190
+ text.append(delta)
1191
+ if !generating {
1192
+ generating = true
1193
+ self.notifyPhase(requestId, "generating")
1194
+ }
1195
+ DispatchQueue.main.async {
1196
+ self.notifyListeners("chatDelta", data: ["requestId": requestId, "delta": delta])
1197
+ }
1198
+ }
1199
+ terminal = .success(["text": text, "stopReason": reason])
1200
+ } catch {
1201
+ terminal = isCancelled(requestId) || error is CancellationError
1202
+ ? .success(["text": text, "stopReason": "cancelled"])
1203
+ : .failure(error)
1204
+ }
1205
+ finishGeneration(call, terminal: terminal)
1206
+ }
1207
+
1208
+ private func runGeneration(_ call: NativeCall, requestId: String, modelId: String, turns: [MobileChatTurn], maxTokens: Int, contextSize: Int, sampling: LlamaSampling) {
1209
+ let stream = NativeChatStream(plugin: self, requestId: requestId)
1210
+ // Reassert cancellation while a blocking call runs, including the tiny
1211
+ // interval between assigning its ID and entering the native function.
1212
+ let cancellation = DispatchSource.makeTimerSource(queue: .global(qos: .userInitiated))
1213
+ cancellation.schedule(deadline: .now() + .milliseconds(50), repeating: .milliseconds(50))
1214
+ var ticks = 0
1215
+ var lastProgress: (phase: UInt32, value: UInt32) = (UInt32.max, UInt32.max)
1216
+ cancellation.setEventHandler { [weak self] in
1217
+ guard let self else { return }
1218
+ if self.isCancelled(requestId) {
1219
+ self.cancelActive(requestId)
1220
+ return
1221
+ }
1222
+ // Every fifth tick (~250 ms) is plenty for a status pill.
1223
+ ticks += 1
1224
+ if ticks % 5 == 0 { self.reportProgress(requestId, last: &lastProgress) }
1225
+ }
1226
+ cancellation.resume()
1227
+ var terminal: Result<[String: Any], Error>
1228
+ func perform() throws -> [String: Any] {
1229
+ guard let engine, let store else { throw storeError ?? MobileStoreError.unknownModel }
1230
+ let (model, url) = try store.modelURL(id: modelId)
1231
+ var nativeError = gezel_llama_error()
1232
+ if loadedModelId != model.id || loadedContextSize != contextSize {
1233
+ try unloadLlama()
1234
+ // Native allocation and memory warnings still enforce the
1235
+ // model's actual working set.
1236
+ try checkResources(additionalBytes: requiredBytes(path: url.path, sizeBytes: model.sizeBytes, contextSize: contextSize))
1237
+ guard let operation = nextOperation(requestId) else {
1238
+ return ["text": "", "stopReason": "cancelled"]
1239
+ }
1240
+ var options = loadOptions(contextSize: contextSize)
1241
+ options.request_id = operation
1242
+ notifyPhase(requestId, "loading_model")
1243
+ sizingLock.lock()
1244
+ let status = url.path.withCString { gezel_llama_load(engine, $0, &options, &nativeError) }
1245
+ if status == 0 { loadedModelId = model.id; loadedPath = url.path; loadedContextSize = contextSize }
1246
+ sizingLock.unlock()
1247
+ guard status == 0 else {
1248
+ if isCancelled(requestId) { return ["text": "", "stopReason": "cancelled"] }
1249
+ throw NSError(domain: "GezelLlama", code: Int(status), userInfo: [NSLocalizedDescriptionKey: errorText(&nativeError)])
1250
+ }
1251
+ }
1252
+ try checkResources(additionalBytes: 64 * 1024 * 1024)
1253
+ guard let operation = nextOperation(requestId) else {
1254
+ return ["text": "", "stopReason": "cancelled"]
1255
+ }
1256
+ var options = gezel_llama_default_generation_options()
1257
+ options.request_id = operation
1258
+ options.max_tokens = UInt32(maxTokens)
1259
+ sampling.apply(to: &options)
1260
+ // The library's default deadline is a flat minute covering prompt
1261
+ // processing as well as decoding, which a long reply on a phone
1262
+ // passes routinely. Scale it with the reply actually asked for, and
1263
+ // keep a ceiling so a wedged decode still ends.
1264
+ options.timeout_ms = UInt32(min(600_000, 30_000 + maxTokens * 250))
1265
+ let strings = turns.flatMap { [strdup($0.role), strdup($0.content)] }
1266
+ defer { strings.forEach { free($0) } }
1267
+ let nativeTurns = turns.indices.map { index in
1268
+ gezel_llama_message(role: UnsafePointer(strings[index * 2]), content: UnsafePointer(strings[index * 2 + 1]))
1269
+ }
1270
+ var result = gezel_llama_result()
1271
+ notifyPhase(requestId, "prefill")
1272
+ let status = nativeTurns.withUnsafeBufferPointer { buffer in
1273
+ gezel_llama_generate(engine, buffer.baseAddress, buffer.count, &options, receiveLlamaChunk,
1274
+ Unmanaged.passUnretained(stream).toOpaque(), &result, &nativeError)
1275
+ }
1276
+ let stopped = isCancelled(requestId) || result.finish_reason == 3
1277
+ guard status == 0 || stopped else {
1278
+ throw NSError(domain: "GezelLlama", code: Int(status), userInfo: [NSLocalizedDescriptionKey: errorText(&nativeError)])
1279
+ }
1280
+ // 2 is the token ceiling and 4 the deadline: both mean the reply was
1281
+ // cut short rather than finished, which is what "length" tells the
1282
+ // product. Treating a timeout as "stop" would present a truncated
1283
+ // answer as a complete one.
1284
+ let truncated = result.finish_reason == 2 || result.finish_reason == 4
1285
+ let reason = stopped ? "cancelled" : (truncated ? "length" : "stop")
1286
+ return ["text": stream.text, "stopReason": reason]
1287
+ }
1288
+ do { terminal = .success(try perform()) }
1289
+ catch {
1290
+ terminal = isCancelled(requestId)
1291
+ ? .success(["text": stream.text, "stopReason": "cancelled"])
1292
+ : .failure(error)
1293
+ }
1294
+ cancellation.cancel()
1295
+ finishGeneration(call, terminal: terminal)
1296
+ }
1297
+ public func importModel(_ call: NativeCall, from url: URL) {
1298
+ guard reserveModelMutation() else { call.reject("Finish the current operation before importing", "BUSY"); return }
1299
+ withStore(call, completion: { self.releaseModelMutation() }) { store in
1300
+ let access = url.startAccessingSecurityScopedResource()
1301
+ defer { if access { url.stopAccessingSecurityScopedResource() } }
1302
+ var result: Result<MobileModel, Error>?
1303
+ var coordinationError: NSError?
1304
+ NSFileCoordinator().coordinate(readingItemAt: url, options: [], error: &coordinationError) { coordinated in
1305
+ result = Result { try store.importModel(from: coordinated) }
1306
+ }
1307
+ if let coordinationError { throw coordinationError }
1308
+ guard let result else { throw MobileStoreError.invalidModel }
1309
+ return ["model": self.modelJSON(try result.get())]
1310
+ }
1311
+ }
1312
+ }
1313
+
1314
+ /// A model's catalog sampling, resolved by the product runtime the same way the
1315
+ /// desktop resolves it. Absent fields keep the bridge's defaults: greedy
1316
+ /// decoding, top-k 40 / top-p 0.95 when sampling, no repetition penalty.
1317
+ struct LlamaSampling {
1318
+ var temperature: Float = 0
1319
+ var topK: UInt32 = 40
1320
+ var topP: Float = 0.95
1321
+ var minP: Float = 0
1322
+ var repeatPenalty: Float = 1
1323
+ var repeatLastN: UInt32 = 64
1324
+ // An unset seed varies per reply, so a retry is not the same text again.
1325
+ var seed = UInt32.random(in: 0...UInt32(Int32.max))
1326
+
1327
+ init?(_ value: [String: Any]?) {
1328
+ guard let value else { return }
1329
+ func number(_ key: String) -> Double? { (value[key] as? NSNumber)?.doubleValue }
1330
+ if let v = number("temperature") { temperature = Float(v) }
1331
+ if let v = number("topK") { guard v >= 0, v <= 1000, v == v.rounded() else { return nil }; topK = UInt32(v) }
1332
+ if let v = number("topP") { topP = Float(v) }
1333
+ if let v = number("minP") { minP = Float(v) }
1334
+ if let v = number("repetitionPenalty") { repeatPenalty = Float(v) }
1335
+ if let v = number("repetitionContext") { guard v >= 0, v <= 4096, v == v.rounded() else { return nil }; repeatLastN = UInt32(v) }
1336
+ if let v = number("seed"), v == v.rounded() { seed = UInt32(truncatingIfNeeded: Int64(v)) & UInt32(Int32.max) }
1337
+ guard (0...2).contains(temperature), topP > 0, topP <= 1, minP >= 0, minP < 1, (1...2).contains(repeatPenalty) else { return nil }
1338
+ }
1339
+
1340
+ func apply(to options: inout gezel_llama_generation_options) {
1341
+ options.temperature = temperature
1342
+ options.top_k = topK
1343
+ options.top_p = topP
1344
+ options.min_p = minP
1345
+ options.repeat_penalty = repeatPenalty
1346
+ options.repeat_last_n = repeatLastN
1347
+ options.seed = seed
1348
+ }
1349
+ }