@bendyline/gezel-capacitor 0.0.0-stage → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (121) hide show
  1. package/LICENSE +21 -0
  2. package/Package.swift +26 -0
  3. package/README.md +230 -2
  4. package/android/build.gradle +30 -0
  5. package/android/src/main/AndroidManifest.xml +1 -0
  6. package/android/src/main/java/com/bendyline/gezel/capacitor/GezelKnowledgePlugin.java +153 -0
  7. package/android/src/main/java/com/bendyline/gezel/capacitor/GezelOfflineSpeechPlugin.java +216 -0
  8. package/android/src/main/java/com/bendyline/gezel/capacitor/GezelRuntimePlugin.java +84 -0
  9. package/android/src/main/java/com/bendyline/gezel/capacitor/KnowledgeFiles.java +101 -0
  10. package/android/src/main/java/com/bendyline/gezel/capacitor/SpeechModels.java +230 -0
  11. package/android/src/main/java/com/bendyline/gezel/speech/SpeechRuntime.java +38 -0
  12. package/dist/index.d.ts +561 -0
  13. package/dist/index.js +7512 -0
  14. package/dist/knowledge-worker.js +211 -0
  15. package/dist/licenses/npm/manifest.json +76 -0
  16. package/dist/licenses/npm/texts/1583809bf1155510d2961f94aa9bca6dc4b44b6f59fd773de33bd6c20860d781.txt +29 -0
  17. package/dist/licenses/npm/texts/1cd39aa4a566eba49eb206b969ef423295e79b50503df567bbdb2e8877d78892.txt +8 -0
  18. package/dist/licenses/npm/texts/3f1189b28e3866e0d979968d466b78f813f76827cfdca1fbb124cc0a5c8841f8.txt +21 -0
  19. package/dist/licenses/npm/texts/5b37224c080cdcc97c871ada971c224e9926370fe74f11b539aa1cf9f3b1aca1.txt +21 -0
  20. package/dist/licenses/npm/texts/c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4.txt +201 -0
  21. package/dist/sqlite3.wasm +0 -0
  22. package/embedding-manifest.json +210 -0
  23. package/ios/Sources/GezelCapacitor/GezelKnowledgePlugin.swift +200 -0
  24. package/ios/Sources/GezelCapacitor/GezelOfflineSpeechPlugin.swift +484 -0
  25. package/ios/Sources/GezelCapacitor/GezelRuntimePlugin.swift +102 -0
  26. package/kokoro-lexicon/LICENSE +202 -0
  27. package/kokoro-lexicon/lexicon-gb-en.txt.gz +0 -0
  28. package/kokoro-lexicon/lexicon-us-en.txt.gz +0 -0
  29. package/native/android/LICENSE-gezel.txt +21 -0
  30. package/native/android/PUBLIC-DISTRIBUTION.md +97 -0
  31. package/native/android/THIRD_PARTY_LICENSES/LICENSE-arm-optimized-routines-MIT-OR-Apache-2.0.txt +249 -0
  32. package/native/android/THIRD_PARTY_LICENSES/LICENSE-cpp-httplib-MIT.txt +22 -0
  33. package/native/android/THIRD_PARTY_LICENSES/LICENSE-fp16-MIT.txt +11 -0
  34. package/native/android/THIRD_PARTY_LICENSES/LICENSE-ggml-MIT.txt +21 -0
  35. package/native/android/THIRD_PARTY_LICENSES/LICENSE-llama-cpp-llamafile-sgemm-MIT.txt +21 -0
  36. package/native/android/THIRD_PARTY_LICENSES/LICENSE-llama-cpp-openvino-APACHE.txt +201 -0
  37. package/native/android/THIRD_PARTY_LICENSES/LICENSE-llama-cpp-pillow-MIT-CMU.txt +30 -0
  38. package/native/android/THIRD_PARTY_LICENSES/LICENSE-llama-cpp-rotate-bits-MIT.txt +21 -0
  39. package/native/android/THIRD_PARTY_LICENSES/LICENSE-llama-cpp-unicode-data-Unicode-3.0.txt +39 -0
  40. package/native/android/THIRD_PARTY_LICENSES/LICENSE-nlohmann-json-MIT.txt +21 -0
  41. package/native/android/THIRD_PARTY_LICENSES/LICENSE-nvidia-cub-BSD-3-Clause.txt +24 -0
  42. package/native/android/THIRD_PARTY_LICENSES/LICENSE-vulkan-headers-Apache-2.0-OR-MIT.txt +33 -0
  43. package/native/android/THIRD_PARTY_LICENSES/LICENSE-vulkan-loader-APACHE.txt +201 -0
  44. package/native/android/THIRD_PARTY_LICENSES/LICENSE-vulkan-loader-cjson-MIT.txt +32 -0
  45. package/native/android/THIRD_PARTY_LICENSES/LICENSE-yarn-MIT.txt +21 -0
  46. package/native/android/THIRD_PARTY_LICENSES/manifest.json +21 -0
  47. package/native/android/engine-manifest.json +88 -0
  48. package/native/android/maven/com/bendyline/gezel/gezel-llama/0.1.57/gezel-llama-0.1.57.aar +0 -0
  49. package/native/android/maven/com/bendyline/gezel/gezel-llama/0.1.57/gezel-llama-0.1.57.aar.sha256 +1 -0
  50. package/native/android/maven/com/bendyline/gezel/gezel-llama/0.1.57/gezel-llama-0.1.57.pom +8 -0
  51. package/native/android/maven/com/bendyline/gezel/gezel-llama/0.1.57/gezel-llama-0.1.57.pom.sha256 +1 -0
  52. package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57-sources.jar +0 -0
  53. package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57-sources.jar.md5 +1 -0
  54. package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57-sources.jar.sha1 +1 -0
  55. package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57-sources.jar.sha256 +1 -0
  56. package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57-sources.jar.sha512 +1 -0
  57. package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.aar +0 -0
  58. package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.aar.md5 +1 -0
  59. package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.aar.sha1 +1 -0
  60. package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.aar.sha256 +1 -0
  61. package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.aar.sha512 +1 -0
  62. package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.module +118 -0
  63. package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.module.md5 +1 -0
  64. package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.module.sha1 +1 -0
  65. package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.module.sha256 +1 -0
  66. package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.module.sha512 +1 -0
  67. package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.pom +51 -0
  68. package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.pom.md5 +1 -0
  69. package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.pom.sha1 +1 -0
  70. package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.pom.sha256 +1 -0
  71. package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.pom.sha512 +1 -0
  72. package/native/android/maven/com/bendyline/gezel/gezel-runtime/maven-metadata.xml +13 -0
  73. package/native/android/maven/com/bendyline/gezel/gezel-runtime/maven-metadata.xml.md5 +1 -0
  74. package/native/android/maven/com/bendyline/gezel/gezel-runtime/maven-metadata.xml.sha1 +1 -0
  75. package/native/android/maven/com/bendyline/gezel/gezel-runtime/maven-metadata.xml.sha256 +1 -0
  76. package/native/android/maven/com/bendyline/gezel/gezel-runtime/maven-metadata.xml.sha512 +1 -0
  77. package/native/android/sdk-manifest.json +98 -0
  78. package/native/ios/GezelLlama.xcframework/Info.plist +47 -0
  79. package/native/ios/GezelLlama.xcframework/ios-arm64/Headers/gezel_llama.h +290 -0
  80. package/native/ios/GezelLlama.xcframework/ios-arm64/Headers/module.modulemap +9 -0
  81. package/native/ios/GezelLlama.xcframework/ios-arm64/libGezelLlama.a +0 -0
  82. package/native/ios/GezelLlama.xcframework/ios-arm64-simulator/Headers/gezel_llama.h +290 -0
  83. package/native/ios/GezelLlama.xcframework/ios-arm64-simulator/Headers/module.modulemap +9 -0
  84. package/native/ios/GezelLlama.xcframework/ios-arm64-simulator/libGezelLlama.a +0 -0
  85. package/native/ios/LICENSE-gezel.txt +21 -0
  86. package/native/ios/LICENSE-ggml.txt +21 -0
  87. package/native/ios/LICENSE-llama-cpp.txt +21 -0
  88. package/native/ios/PUBLIC-DISTRIBUTION.md +97 -0
  89. package/native/ios/Package.swift +17 -0
  90. package/native/ios/Sources/GezelModelStorage/MobileModelStore.swift +286 -0
  91. package/native/ios/Sources/GezelModelStorage/ModelDownloadHTTP.swift +55 -0
  92. package/native/ios/Sources/GezelModelStorage/ModelDownloadSource.swift +46 -0
  93. package/native/ios/Sources/GezelModelStorage/ModelDownloads.swift +289 -0
  94. package/native/ios/Sources/GezelModelStorage/PrivacyInfo.xcprivacy +32 -0
  95. package/native/ios/Sources/GezelRuntime/AppleFoundationProvider.swift +292 -0
  96. package/native/ios/Sources/GezelRuntime/AppleNativeTools.swift +152 -0
  97. package/native/ios/Sources/GezelRuntime/GezelNativeRuntime.swift +1349 -0
  98. package/native/ios/Sources/GezelRuntime/NativeCall.swift +35 -0
  99. package/native/ios/Sources/GezelRuntime/PrivacyInfo.xcprivacy +31 -0
  100. package/native/ios/THIRD_PARTY_LICENSES/LICENSE-arm-optimized-routines-MIT-OR-Apache-2.0.txt +249 -0
  101. package/native/ios/THIRD_PARTY_LICENSES/LICENSE-cpp-httplib-MIT.txt +22 -0
  102. package/native/ios/THIRD_PARTY_LICENSES/LICENSE-fp16-MIT.txt +11 -0
  103. package/native/ios/THIRD_PARTY_LICENSES/LICENSE-ggml-MIT.txt +21 -0
  104. package/native/ios/THIRD_PARTY_LICENSES/LICENSE-llama-cpp-llamafile-sgemm-MIT.txt +21 -0
  105. package/native/ios/THIRD_PARTY_LICENSES/LICENSE-llama-cpp-openvino-APACHE.txt +201 -0
  106. package/native/ios/THIRD_PARTY_LICENSES/LICENSE-llama-cpp-pillow-MIT-CMU.txt +30 -0
  107. package/native/ios/THIRD_PARTY_LICENSES/LICENSE-llama-cpp-rotate-bits-MIT.txt +21 -0
  108. package/native/ios/THIRD_PARTY_LICENSES/LICENSE-llama-cpp-unicode-data-Unicode-3.0.txt +39 -0
  109. package/native/ios/THIRD_PARTY_LICENSES/LICENSE-nlohmann-json-MIT.txt +21 -0
  110. package/native/ios/THIRD_PARTY_LICENSES/LICENSE-nvidia-cub-BSD-3-Clause.txt +24 -0
  111. package/native/ios/THIRD_PARTY_LICENSES/LICENSE-vulkan-headers-Apache-2.0-OR-MIT.txt +33 -0
  112. package/native/ios/THIRD_PARTY_LICENSES/LICENSE-vulkan-loader-APACHE.txt +201 -0
  113. package/native/ios/THIRD_PARTY_LICENSES/LICENSE-vulkan-loader-cjson-MIT.txt +32 -0
  114. package/native/ios/THIRD_PARTY_LICENSES/LICENSE-yarn-MIT.txt +21 -0
  115. package/native/ios/THIRD_PARTY_LICENSES/manifest.json +21 -0
  116. package/native/ios/engine-manifest.json +102 -0
  117. package/native/ios/sdk-manifest.json +89 -0
  118. package/package.json +94 -3
  119. package/scripts/embedding-package.d.mts +23 -0
  120. package/scripts/embedding-package.mjs +185 -0
  121. package/scripts/stage-native.mjs +168 -0
@@ -0,0 +1,484 @@
1
+ import Capacitor
2
+ import CryptoKit
3
+ import Foundation
4
+ import UIKit
5
+ import os
6
+ #if canImport(GezelSpeech)
7
+ import GezelSpeech
8
+ #endif
9
+
10
+ /// Offline speech: Whisper dictation and Kokoro narration on the optional
11
+ /// GezelSpeech framework, with models the host downloads on request. One
12
+ /// request runs at a time; models stay loaded between requests and are
13
+ /// released after a quiet spell or when the app leaves the foreground.
14
+ @objc(GezelOfflineSpeechPlugin)
15
+ public final class GezelOfflineSpeechPlugin: CAPPlugin, CAPBridgedPlugin {
16
+ public let identifier = "GezelOfflineSpeechPlugin"
17
+ public let jsName = "GezelOfflineSpeech"
18
+ public let pluginMethods: [CAPPluginMethod] = [
19
+ CAPPluginMethod(name: "status", returnType: CAPPluginReturnPromise),
20
+ CAPPluginMethod(name: "install", returnType: CAPPluginReturnPromise),
21
+ CAPPluginMethod(name: "cancelInstall", returnType: CAPPluginReturnPromise),
22
+ CAPPluginMethod(name: "remove", returnType: CAPPluginReturnPromise),
23
+ CAPPluginMethod(name: "transcribe", returnType: CAPPluginReturnPromise),
24
+ CAPPluginMethod(name: "synthesize", returnType: CAPPluginReturnPromise),
25
+ CAPPluginMethod(name: "cancel", returnType: CAPPluginReturnPromise),
26
+ CAPPluginMethod(name: "release", returnType: CAPPluginReturnPromise)
27
+ ]
28
+ private static let idleRelease: TimeInterval = 120
29
+ private let worker = DispatchQueue(label: "com.bendyline.gezel.speech", qos: .userInitiated)
30
+ private let downloads = DispatchQueue(label: "com.bendyline.gezel.speech.downloads", qos: .utility)
31
+ private let lock = NSLock()
32
+ private let models = SpeechModelStore()
33
+ private var engine: OpaquePointer?
34
+ private var activeId: String?
35
+ private var downloading: String?
36
+ private var stopped = false, backgrounded = false, downloadPaused = false
37
+ private var waiters: [CAPPluginCall] = []
38
+ private var idle: DispatchWorkItem?
39
+ private var observers: [NSObjectProtocol] = []
40
+
41
+ /// Null when the engine can run, else why not; the knowledge plugin's query models share it.
42
+ static var unavailableReason: String? {
43
+ #if canImport(GezelSpeech)
44
+ return gezel_speech_abi() == 3 ? nil : "This app's speech engine does not match its plugin."
45
+ #else
46
+ return "This app was built without the offline speech engine."
47
+ #endif
48
+ }
49
+
50
+ public override func load() {
51
+ let center = NotificationCenter.default
52
+ observers.append(center.addObserver(forName: UIApplication.didEnterBackgroundNotification, object: nil, queue: nil) { [weak self] _ in
53
+ self?.enterBackground()
54
+ })
55
+ observers.append(center.addObserver(forName: UIApplication.willEnterForegroundNotification, object: nil, queue: nil) { [weak self] _ in
56
+ guard let self else { return }
57
+ self.locked { self.backgrounded = false }
58
+ })
59
+ }
60
+ deinit {
61
+ observers.forEach(NotificationCenter.default.removeObserver)
62
+ models.cancel()
63
+ #if canImport(GezelSpeech)
64
+ if let engine { gezel_speech_destroy(engine) }
65
+ #endif
66
+ }
67
+ private func locked<T>(_ body: () -> T) -> T { lock.lock(); defer { lock.unlock() }; return body() }
68
+
69
+ @objc public func status(_ call: CAPPluginCall) {
70
+ do {
71
+ var result: [String: Any] = ["available": Self.unavailableReason == nil, "models": try models.list()]
72
+ if let reason = Self.unavailableReason { result["reason"] = reason }
73
+ if let downloading = locked({ downloading }) { result["downloading"] = downloading }
74
+ call.resolve(result)
75
+ } catch { call.reject("Speech models could not be read", "failed") }
76
+ }
77
+
78
+ @objc public func install(_ call: CAPPluginCall) {
79
+ guard let id = call.getString("id"), SpeechModelStore.valid(id: id), let files = call.getArray("files") as? [[String: Any]] else {
80
+ call.reject("Invalid speech model", "invalid-input"); return
81
+ }
82
+ let refusal: (String, String)? = locked {
83
+ if downloading != nil { return ("Another speech model is downloading", "busy") }
84
+ if backgrounded { return ("Speech downloads run in the foreground", "paused") }
85
+ downloading = id; downloadPaused = false
86
+ return nil
87
+ }
88
+ if let (message, code) = refusal { call.reject(message, code); return }
89
+ downloads.async { [self] in
90
+ defer { locked { downloading = nil } }
91
+ do {
92
+ try models.install(id, files) { received, total in
93
+ self.notifyListeners("speechDownload", data: ["id": id, "receivedBytes": received, "totalBytes": total])
94
+ }
95
+ call.resolve(["id": id])
96
+ } catch SpeechModelStore.Failure.cancelled {
97
+ let paused = locked { downloadPaused }
98
+ call.reject(paused ? "Download paused when the app left the foreground" : "Download cancelled", paused ? "paused" : "cancelled")
99
+ } catch let SpeechModelStore.Failure.failed(code, message) {
100
+ call.reject(message, code)
101
+ } catch {
102
+ call.reject(error.localizedDescription, "interrupted")
103
+ }
104
+ }
105
+ }
106
+
107
+ @objc public func cancelInstall(_ call: CAPPluginCall) { models.cancel(); call.resolve() }
108
+
109
+ @objc public func remove(_ call: CAPPluginCall) {
110
+ let id = call.getString("id") ?? ""
111
+ let refusal: String? = locked {
112
+ if id == downloading { return "Cancel the download before removing this model" }
113
+ return activeId == nil ? nil : "Speech is busy"
114
+ }
115
+ if let refusal { call.reject(refusal, "busy"); return }
116
+ worker.async { [self] in
117
+ releaseModels()
118
+ do { try models.remove(id); call.resolve() }
119
+ catch let SpeechModelStore.Failure.failed(code, message) { call.reject(message, code) }
120
+ catch { call.reject("The speech model could not be removed", "failed") }
121
+ }
122
+ }
123
+
124
+ @objc public func transcribe(_ call: CAPPluginCall) { run(call, synthesis: false) }
125
+ @objc public func synthesize(_ call: CAPPluginCall) { run(call, synthesis: true) }
126
+
127
+ private func run(_ call: CAPPluginCall, synthesis: Bool) {
128
+ guard call.getString("requestId").flatMap(UUID.init(uuidString:)) != nil else {
129
+ call.reject("A speech request identity is required", "invalid-input"); return
130
+ }
131
+ if let reason = Self.unavailableReason { call.reject(reason, "unavailable"); return }
132
+ let refusal: (String, String)? = locked {
133
+ if activeId != nil { return ("Speech is busy", "busy") }
134
+ if backgrounded { return ("Speech runs while the app is in the foreground", "cancelled") }
135
+ activeId = call.getString("requestId"); stopped = false
136
+ idle?.cancel()
137
+ return nil
138
+ }
139
+ if let (message, code) = refusal { call.reject(message, code); return }
140
+ worker.async { [self] in
141
+ let started = Date()
142
+ var result: [String: Any]?
143
+ var failure: (String, String)?
144
+ do {
145
+ result = try perform(call, synthesis: synthesis)
146
+ result?["durationMs"] = Int(Date().timeIntervalSince(started) * 1000)
147
+ } catch SpeechModelStore.Failure.cancelled {
148
+ failure = ("Speech stopped", "cancelled")
149
+ } catch let SpeechModelStore.Failure.failed(code, message) {
150
+ failure = (message, code)
151
+ } catch {
152
+ failure = (error.localizedDescription, "failed")
153
+ }
154
+ lock.lock()
155
+ if stopped { failure = ("Speech stopped", "cancelled") }
156
+ activeId = nil
157
+ let pending = waiters
158
+ waiters.removeAll()
159
+ let delay = backgrounded ? 0 : Self.idleRelease
160
+ lock.unlock()
161
+ if let failure { call.reject(failure.0, failure.1) } else { call.resolve(result ?? [:]) }
162
+ pending.forEach { $0.resolve() }
163
+ scheduleRelease(after: delay)
164
+ }
165
+ }
166
+
167
+ private func perform(_ call: CAPPluginCall, synthesis: Bool) throws -> [String: Any] {
168
+ #if canImport(GezelSpeech)
169
+ let model = try models.file(call.getString("model"), call.getString("file"))
170
+ try checkResources()
171
+ let handle = try begin()
172
+ if synthesis {
173
+ let voices = try models.file(call.getString("model"), call.getString("voices"))
174
+ guard let encoded = call.getArray("tokens") as? [Int], (3...511).contains(encoded.count),
175
+ encoded.allSatisfy({ (0...177).contains($0) }) else {
176
+ throw SpeechModelStore.Failure.failed("invalid-input", "Speech phonemes are missing or invalid")
177
+ }
178
+ let tokens = encoded.map(Int32.init)
179
+ let speed = Float(min(2, max(0.5, call.getDouble("speed") ?? 1)))
180
+ var samples: UnsafeMutablePointer<Float>?
181
+ var durations: UnsafeMutablePointer<Float>?
182
+ var sampleCount = 0, durationCount = 0
183
+ let status = tokens.withUnsafeBufferPointer {
184
+ gezel_speech_kokoro(handle, model.path, voices.path, Int32(call.getInt("voice") ?? 0), $0.baseAddress, $0.count,
185
+ speed, &samples, &sampleCount, &durations, &durationCount)
186
+ }
187
+ defer { gezel_speech_free(samples); gezel_speech_free(durations) }
188
+ try check(status, handle)
189
+ var pcm = Data(count: sampleCount * 2)
190
+ pcm.withUnsafeMutableBytes { raw in
191
+ let out = raw.bindMemory(to: Int16.self)
192
+ for index in 0..<sampleCount {
193
+ let value = samples![index].isFinite ? max(-1, min(1, samples![index])) : 0
194
+ out[index] = Int16(value * (value < 0 ? 32768 : 32767)).littleEndian
195
+ }
196
+ }
197
+ let timing = durationCount > 0 ? Array(UnsafeBufferPointer(start: durations, count: durationCount)).map(Double.init) : []
198
+ return ["pcm": pcm.base64EncodedString(), "sampleRate": 24_000, "durations": timing]
199
+ }
200
+ guard let encoded = call.getString("audio"), encoded.count <= 5_120_000, let pcm = Data(base64Encoded: encoded),
201
+ !pcm.isEmpty, pcm.count % 2 == 0 else {
202
+ throw SpeechModelStore.Failure.failed("invalid-input", "Use up to two minutes of speech")
203
+ }
204
+ var json: UnsafeMutablePointer<CChar>?
205
+ let status = pcm.withUnsafeBytes { raw in
206
+ gezel_speech_transcribe_json(handle, model.path, raw.bindMemory(to: UInt8.self).baseAddress, pcm.count,
207
+ call.getString("language"), call.getString("prompt"),
208
+ call.getBool("words") == true ? 1 : 0, &json)
209
+ }
210
+ defer { gezel_speech_free(json) }
211
+ try check(status, handle)
212
+ guard let json, let value = try JSONSerialization.jsonObject(with: Data(String(cString: json).utf8)) as? [String: Any] else {
213
+ throw SpeechModelStore.Failure.failed("failed", "Whisper returned an unreadable transcript")
214
+ }
215
+ return value
216
+ #else
217
+ throw SpeechModelStore.Failure.failed("unavailable", Self.unavailableReason ?? "")
218
+ #endif
219
+ }
220
+
221
+ #if canImport(GezelSpeech)
222
+ /// The long-lived engine, cleared of any earlier cancel; serialized with stop().
223
+ private func begin() throws -> OpaquePointer {
224
+ lock.lock(); defer { lock.unlock() }
225
+ if stopped || backgrounded { throw SpeechModelStore.Failure.cancelled }
226
+ if engine == nil { engine = gezel_speech_create() }
227
+ guard let engine else { throw SpeechModelStore.Failure.failed("resources", "Not enough memory for speech") }
228
+ gezel_speech_begin(engine)
229
+ return engine
230
+ }
231
+ private func check(_ status: Int32, _ handle: OpaquePointer) throws {
232
+ if status == 2 { throw SpeechModelStore.Failure.cancelled }
233
+ if status != 0 { throw SpeechModelStore.Failure.failed("failed", String(cString: gezel_speech_error(handle))) }
234
+ }
235
+ #endif
236
+
237
+ private func checkResources() throws {
238
+ if ProcessInfo.processInfo.thermalState == .critical || ProcessInfo.processInfo.thermalState == .serious {
239
+ throw SpeechModelStore.Failure.failed("resources", "Let this device cool before running speech")
240
+ }
241
+ // Zero means no limit is known (the simulator); only a real budget can refuse.
242
+ let headroom = os_proc_available_memory()
243
+ if headroom > 0 && headroom < 512 * 1024 * 1024 {
244
+ throw SpeechModelStore.Failure.failed("resources", "Not enough memory for speech. Close other apps and try again.")
245
+ }
246
+ }
247
+
248
+ private func releaseModels() {
249
+ #if canImport(GezelSpeech)
250
+ locked { if activeId == nil, let engine { gezel_speech_release(engine) } }
251
+ #endif
252
+ }
253
+ private func scheduleRelease(after delay: TimeInterval) {
254
+ let item = DispatchWorkItem { [weak self] in self?.worker.async { self?.releaseModels() } }
255
+ locked { idle?.cancel(); idle = item }
256
+ DispatchQueue.global(qos: .utility).asyncAfter(deadline: .now() + delay, execute: item)
257
+ }
258
+ private func stop() {
259
+ stopped = true
260
+ #if canImport(GezelSpeech)
261
+ if let engine { gezel_speech_cancel(engine) }
262
+ #endif
263
+ }
264
+
265
+ @objc public func cancel(_ call: CAPPluginCall) {
266
+ let waiting: Bool = locked {
267
+ guard let activeId, activeId == call.getString("requestId") else { return false }
268
+ waiters.append(call); stop()
269
+ return true
270
+ }
271
+ if !waiting { call.resolve() }
272
+ }
273
+
274
+ @objc public func release(_ call: CAPPluginCall) { scheduleRelease(after: 0); call.resolve() }
275
+
276
+ private func enterBackground() {
277
+ locked {
278
+ backgrounded = true
279
+ if activeId != nil { stop() }
280
+ if downloading != nil { downloadPaused = true }
281
+ }
282
+ models.cancel()
283
+ scheduleRelease(after: 0)
284
+ }
285
+ }
286
+
287
+ /// Speech model files in app-private storage excluded from backup: one folder
288
+ /// per model, declared files only, each verified against its SHA-256 before
289
+ /// the model counts as installed. JavaScript names models and files; it never
290
+ /// supplies paths.
291
+ final class SpeechModelStore: NSObject, URLSessionDataDelegate, @unchecked Sendable {
292
+ enum Failure: Error { case cancelled, failed(String, String) }
293
+ private static let marker = "installed.json"
294
+ /// The store's folder, in Application Support and excluded from backup.
295
+ let root: URL
296
+ private let maxFile: Int64
297
+ private let lock = NSLock()
298
+ private var cancelled = false
299
+ private var task: URLSessionDataTask?
300
+ private lazy var session = URLSession(configuration: .ephemeral, delegate: self, delegateQueue: nil)
301
+ // Transfer state for the one file in flight.
302
+ private var output: FileHandle?
303
+ private var expected: Int64 = 0, received: Int64 = 0, lastReport = Date.distantPast
304
+ private var transferError: Failure?
305
+ private var finished = DispatchSemaphore(value: 0)
306
+ private var onBytes: ((Int64) -> Void)?
307
+
308
+ init(folder: String = "gezel-speech/models", maxFile: Int64 = 1 << 30) {
309
+ var url = FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask)[0]
310
+ .appendingPathComponent(folder, isDirectory: true)
311
+ try? FileManager.default.createDirectory(at: url, withIntermediateDirectories: true)
312
+ var values = URLResourceValues()
313
+ values.isExcludedFromBackup = true
314
+ try? url.setResourceValues(values)
315
+ root = url
316
+ self.maxFile = maxFile
317
+ }
318
+
319
+ static func valid(id: String) -> Bool { id.range(of: "^[a-z0-9][a-z0-9._-]{0,63}$", options: .regularExpression) != nil }
320
+ static func valid(name: String) -> Bool {
321
+ let parts = name.split(separator: "/", omittingEmptySubsequences: false)
322
+ return name.count <= 256 && parts.count <= 3 && parts.allSatisfy {
323
+ String($0).range(of: "^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$", options: .regularExpression) != nil && !$0.hasSuffix(".part")
324
+ }
325
+ }
326
+ private func folder(_ id: String?) throws -> URL {
327
+ guard let id, Self.valid(id: id) else { throw Failure.failed("invalid-input", "Invalid speech model") }
328
+ return root.appendingPathComponent(id, isDirectory: true)
329
+ }
330
+ private func inside(_ folder: URL, _ name: String) throws -> URL {
331
+ guard Self.valid(name: name) else { throw Failure.failed("invalid-input", "Invalid speech model file") }
332
+ let file = folder.appendingPathComponent(name).standardizedFileURL
333
+ guard file.path.hasPrefix(folder.standardizedFileURL.path + "/") else { throw Failure.failed("invalid-input", "Invalid speech model file") }
334
+ return file
335
+ }
336
+ private func installed(_ folder: URL) -> [String: String]? {
337
+ guard let data = try? Data(contentsOf: folder.appendingPathComponent(Self.marker)), data.count <= 64 * 1024,
338
+ let value = try? JSONSerialization.jsonObject(with: data) as? [String: Any],
339
+ let files = value["files"] as? [String: String], files.keys.allSatisfy(Self.valid(name:)) else { return nil }
340
+ return files
341
+ }
342
+ func file(_ id: String?, _ name: String?) throws -> URL {
343
+ let folder = try folder(id)
344
+ guard let files = installed(folder) else { throw Failure.failed("model-missing", "Download this speech model first") }
345
+ guard let name, files[name] != nil else { throw Failure.failed("invalid-input", "This file is not part of the installed model") }
346
+ let file = try inside(folder, name)
347
+ guard FileManager.default.fileExists(atPath: file.path) else { throw Failure.failed("model-missing", "The speech model is incomplete; download it again") }
348
+ return file
349
+ }
350
+ func list() throws -> [[String: Any]] {
351
+ let folders = (try? FileManager.default.contentsOfDirectory(at: root, includingPropertiesForKeys: nil)) ?? []
352
+ return folders.compactMap { folder in
353
+ guard Self.valid(id: folder.lastPathComponent), let files = installed(folder) else { return nil }
354
+ return ["id": folder.lastPathComponent, "files": files.keys.sorted()]
355
+ }
356
+ }
357
+ func remove(_ id: String) throws {
358
+ let folder = try folder(id)
359
+ if FileManager.default.fileExists(atPath: folder.path) { try FileManager.default.removeItem(at: folder) }
360
+ }
361
+ func cancel() {
362
+ lock.lock(); cancelled = true; let current = task; lock.unlock()
363
+ current?.cancel()
364
+ }
365
+
366
+ /// Download and verify every file, resuming any partial file from where it
367
+ /// stopped. A file that fails verification is deleted so the next attempt
368
+ /// starts clean. Blocks the calling queue; cancel() stops it.
369
+ func install(_ id: String, _ specs: [[String: Any]], progress: @escaping (Int64, Int64) -> Void) throws {
370
+ lock.lock(); cancelled = false; lock.unlock()
371
+ let folder = try folder(id)
372
+ guard (1...64).contains(specs.count) else { throw Failure.failed("invalid-input", "Invalid speech model files") }
373
+ var files: [(name: String, url: URL, sha: String, size: Int64)] = []
374
+ for spec in specs {
375
+ guard let name = spec["name"] as? String, Self.valid(name: name), let sha = spec["sha256"] as? String,
376
+ sha.range(of: "^[0-9a-f]{64}$", options: .regularExpression) != nil,
377
+ let link = spec["url"] as? String, link.hasPrefix("https://huggingface.co/"), let url = URL(string: link),
378
+ let size = (spec["size"] as? NSNumber)?.int64Value, (1...maxFile).contains(size),
379
+ !files.contains(where: { $0.name == name }) else { throw Failure.failed("invalid-input", "Invalid speech model files") }
380
+ files.append((name, url, sha, size))
381
+ }
382
+ let total = files.reduce(0) { $0 + $1.size }
383
+ try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true)
384
+ try? FileManager.default.removeItem(at: folder.appendingPathComponent(Self.marker))
385
+ let free = (try? root.resourceValues(forKeys: [.volumeAvailableCapacityForImportantUsageKey]))?.volumeAvailableCapacityForImportantUsage ?? 0
386
+ if free < total + 64 * 1024 * 1024 { throw Failure.failed("storage", "Not enough storage for this speech model") }
387
+ var done: Int64 = 0
388
+ for file in files {
389
+ let target = try inside(folder, file.name)
390
+ let partial = target.appendingPathExtension("part")
391
+ try FileManager.default.createDirectory(at: target.deletingLastPathComponent(), withIntermediateDirectories: true)
392
+ let ready = (try? target.resourceValues(forKeys: [.fileSizeKey]).fileSize).map { Int64($0) == file.size } == true
393
+ && (try? Self.digest(target)) == file.sha
394
+ if !ready {
395
+ try? FileManager.default.removeItem(at: target)
396
+ let before = done
397
+ try transfer(file.url, to: partial, size: file.size) { progress(before + $0, total) }
398
+ guard try Self.digest(partial) == file.sha else {
399
+ try? FileManager.default.removeItem(at: partial)
400
+ throw Failure.failed("verification", "The downloaded speech model was damaged, so it was not installed")
401
+ }
402
+ try FileManager.default.moveItem(at: partial, to: target)
403
+ }
404
+ done += file.size
405
+ progress(done, total)
406
+ }
407
+ let marker = try JSONSerialization.data(withJSONObject: ["files": Dictionary(uniqueKeysWithValues: files.map { ($0.name, $0.sha) })])
408
+ try marker.write(to: folder.appendingPathComponent(Self.marker), options: .atomic)
409
+ }
410
+
411
+ private func transfer(_ url: URL, to partial: URL, size: Int64, report: @escaping (Int64) -> Void) throws {
412
+ var have = (try? partial.resourceValues(forKeys: [.fileSizeKey]).fileSize).map(Int64.init) ?? 0
413
+ if have > size { try? FileManager.default.removeItem(at: partial); have = 0 }
414
+ if have == size { return }
415
+ var request = URLRequest(url: url, timeoutInterval: 60)
416
+ request.setValue("identity", forHTTPHeaderField: "Accept-Encoding")
417
+ if have > 0 { request.setValue("bytes=\(have)-", forHTTPHeaderField: "Range") }
418
+ if !FileManager.default.fileExists(atPath: partial.path) { FileManager.default.createFile(atPath: partial.path, contents: nil) }
419
+ lock.lock()
420
+ if cancelled { lock.unlock(); throw Failure.cancelled }
421
+ output = try FileHandle(forWritingTo: partial)
422
+ expected = size; received = have; transferError = nil; onBytes = report; finished = DispatchSemaphore(value: 0)
423
+ let task = session.dataTask(with: request)
424
+ self.task = task
425
+ let signal = finished
426
+ lock.unlock()
427
+ task.resume()
428
+ signal.wait()
429
+ lock.lock()
430
+ defer { lock.unlock() }
431
+ try? output?.synchronize()
432
+ try? output?.close()
433
+ output = nil; self.task = nil; onBytes = nil
434
+ if cancelled { throw Failure.cancelled }
435
+ if let transferError { throw transferError }
436
+ if received != size { throw Failure.failed("interrupted", "The download ended early") }
437
+ }
438
+
439
+ func urlSession(_ session: URLSession, dataTask: URLSessionDataTask, didReceive response: URLResponse,
440
+ completionHandler: @escaping (URLSession.ResponseDisposition) -> Void) {
441
+ lock.lock(); defer { lock.unlock() }
442
+ let status = (response as? HTTPURLResponse)?.statusCode ?? 0
443
+ if status == 206 && received > 0 {
444
+ _ = try? output?.seekToEnd()
445
+ completionHandler(.allow)
446
+ } else if status == 200 {
447
+ // A full response restarts the file.
448
+ try? output?.truncate(atOffset: 0)
449
+ received = 0
450
+ completionHandler(.allow)
451
+ } else {
452
+ transferError = .failed(status == 408 || status == 429 || status >= 500 || status == 416 ? "interrupted" : "failed",
453
+ "Speech model download failed (HTTP \(status))")
454
+ completionHandler(.cancel)
455
+ }
456
+ }
457
+ func urlSession(_ session: URLSession, dataTask: URLSessionDataTask, didReceive data: Data) {
458
+ lock.lock()
459
+ received += Int64(data.count)
460
+ if received > expected { transferError = .failed("verification", "The speech model is larger than expected"); lock.unlock(); dataTask.cancel(); return }
461
+ do { try output?.write(contentsOf: data) } catch {
462
+ transferError = .failed("storage", "Cannot save the speech model"); lock.unlock(); dataTask.cancel(); return
463
+ }
464
+ let now = Date(), value = received, report = now.timeIntervalSince(lastReport) > 0.25 ? onBytes : nil
465
+ if report != nil { lastReport = now }
466
+ lock.unlock()
467
+ report?(value)
468
+ }
469
+ func urlSession(_ session: URLSession, task: URLSessionTask, didCompleteWithError error: Error?) {
470
+ lock.lock()
471
+ if let error, transferError == nil, !cancelled { transferError = .failed("interrupted", error.localizedDescription) }
472
+ let signal = finished
473
+ lock.unlock()
474
+ signal.signal()
475
+ }
476
+
477
+ static func digest(_ url: URL) throws -> String {
478
+ let handle = try FileHandle(forReadingFrom: url)
479
+ defer { try? handle.close() }
480
+ var hash = SHA256()
481
+ while let chunk = try handle.read(upToCount: 1 << 20), !chunk.isEmpty { hash.update(data: chunk) }
482
+ return hash.finalize().map { String(format: "%02x", $0) }.joined()
483
+ }
484
+ }
@@ -0,0 +1,102 @@
1
+ import Capacitor
2
+ import Foundation
3
+ import UIKit
4
+ import UniformTypeIdentifiers
5
+ import WebKit
6
+ #if canImport(GezelRuntime)
7
+ import GezelRuntime
8
+ #endif
9
+
10
+ @objc(GezelRuntimePlugin)
11
+ public final class GezelRuntimePlugin: CAPPlugin, CAPBridgedPlugin, UIDocumentPickerDelegate {
12
+ public let identifier = "GezelRuntimePlugin"
13
+ public let jsName = "GezelRuntime"
14
+ public let pluginMethods: [CAPPluginMethod] = [
15
+ CAPPluginMethod(name: "listModels", returnType: CAPPluginReturnPromise),
16
+ CAPPluginMethod(name: "resolveModelSource", returnType: CAPPluginReturnPromise),
17
+ CAPPluginMethod(name: "cancelModelSourceResolution", returnType: CAPPluginReturnPromise),
18
+ CAPPluginMethod(name: "listModelDownloads", returnType: CAPPluginReturnPromise),
19
+ CAPPluginMethod(name: "startModelDownload", returnType: CAPPluginReturnPromise),
20
+ CAPPluginMethod(name: "resumeModelDownload", returnType: CAPPluginReturnPromise),
21
+ CAPPluginMethod(name: "cancelModelDownload", returnType: CAPPluginReturnPromise),
22
+ CAPPluginMethod(name: "removeModelDownload", returnType: CAPPluginReturnPromise),
23
+ CAPPluginMethod(name: "providers", returnType: CAPPluginReturnPromise),
24
+ CAPPluginMethod(name: "prepareProvider", returnType: CAPPluginReturnPromise),
25
+ CAPPluginMethod(name: "cancelProviderPreparation", returnType: CAPPluginReturnPromise),
26
+ CAPPluginMethod(name: "selectModel", returnType: CAPPluginReturnPromise),
27
+ CAPPluginMethod(name: "removeModel", returnType: CAPPluginReturnPromise),
28
+ CAPPluginMethod(name: "generate", returnType: CAPPluginReturnPromise),
29
+ CAPPluginMethod(name: "chat", returnType: CAPPluginReturnPromise),
30
+ CAPPluginMethod(name: "describeImage", returnType: CAPPluginReturnPromise),
31
+ CAPPluginMethod(name: "cancel", returnType: CAPPluginReturnPromise),
32
+ CAPPluginMethod(name: "completeToolCall", returnType: CAPPluginReturnPromise),
33
+ CAPPluginMethod(name: "releaseModel", returnType: CAPPluginReturnPromise),
34
+ CAPPluginMethod(name: "importModel", returnType: CAPPluginReturnPromise)
35
+ ]
36
+ private var runtime: GezelNativeRuntime?
37
+ private var listener: UUID?
38
+ private var pickerCall: CAPPluginCall?
39
+ private let initializationLock = NSLock()
40
+ // Registration must not initialize engines or model storage in opt-in hosts.
41
+ private func resolveRuntime() throws -> GezelNativeRuntime {
42
+ initializationLock.lock()
43
+ defer { initializationLock.unlock() }
44
+ if let runtime { return runtime }
45
+ let created = try GezelNativeRuntime.shared()
46
+ listener = created.listen { [weak self] event, data in self?.notifyListeners(event, data: data) }
47
+ runtime = created
48
+ return created
49
+ }
50
+ deinit {
51
+ if let listener { runtime?.removeListener(listener) }
52
+ if pickerCall != nil { runtime?.releaseModelMutation() }
53
+ }
54
+ public static func adapt(_ call: CAPPluginCall) -> NativeCall {
55
+ NativeCall(call.options as? [String: Any] ?? [:], resolve: { call.resolve($0) }, reject: { call.reject($0, $1) })
56
+ }
57
+ private func withRuntime(_ call: CAPPluginCall, _ action: (GezelNativeRuntime) -> Void) {
58
+ do { action(try resolveRuntime()) }
59
+ catch { call.reject(error.localizedDescription, "UNAVAILABLE") }
60
+ }
61
+ @objc public func listModels(_ call: CAPPluginCall) { withRuntime(call) { $0.listModels(Self.adapt(call)) } }
62
+ @objc public func resolveModelSource(_ call: CAPPluginCall) { withRuntime(call) { $0.resolveModelSource(Self.adapt(call)) } }
63
+ @objc public func cancelModelSourceResolution(_ call: CAPPluginCall) { withRuntime(call) { $0.cancelModelSourceResolution(Self.adapt(call)) } }
64
+ @objc public func listModelDownloads(_ call: CAPPluginCall) { withRuntime(call) { $0.listModelDownloads(Self.adapt(call)) } }
65
+ @objc public func startModelDownload(_ call: CAPPluginCall) { withRuntime(call) { $0.startModelDownload(Self.adapt(call)) } }
66
+ @objc public func resumeModelDownload(_ call: CAPPluginCall) { withRuntime(call) { $0.resumeModelDownload(Self.adapt(call)) } }
67
+ @objc public func cancelModelDownload(_ call: CAPPluginCall) { withRuntime(call) { $0.cancelModelDownload(Self.adapt(call)) } }
68
+ @objc public func removeModelDownload(_ call: CAPPluginCall) { withRuntime(call) { $0.removeModelDownload(Self.adapt(call)) } }
69
+ @objc public func providers(_ call: CAPPluginCall) { withRuntime(call) { $0.providers(Self.adapt(call)) } }
70
+ @objc public func prepareProvider(_ call: CAPPluginCall) { withRuntime(call) { $0.prepareProvider(Self.adapt(call)) } }
71
+ @objc public func cancelProviderPreparation(_ call: CAPPluginCall) { withRuntime(call) { $0.cancelProviderPreparation(Self.adapt(call)) } }
72
+ @objc public func selectModel(_ call: CAPPluginCall) { withRuntime(call) { $0.selectModel(Self.adapt(call)) } }
73
+ @objc public func removeModel(_ call: CAPPluginCall) { withRuntime(call) { $0.removeModel(Self.adapt(call)) } }
74
+ @objc public func generate(_ call: CAPPluginCall) { withRuntime(call) { $0.generate(Self.adapt(call)) } }
75
+ @objc public func chat(_ call: CAPPluginCall) { withRuntime(call) { $0.chat(Self.adapt(call)) } }
76
+ @objc public func describeImage(_ call: CAPPluginCall) { withRuntime(call) { $0.describeImage(Self.adapt(call)) } }
77
+ @objc public func cancel(_ call: CAPPluginCall) { withRuntime(call) { $0.cancel(Self.adapt(call)) } }
78
+ @objc public func completeToolCall(_ call: CAPPluginCall) { withRuntime(call) { $0.completeToolCall(Self.adapt(call)) } }
79
+ @objc public func releaseModel(_ call: CAPPluginCall) { withRuntime(call) { $0.releaseModel(Self.adapt(call)) } }
80
+ @objc public func importModel(_ call: CAPPluginCall) {
81
+ withRuntime(call) { runtime in
82
+ DispatchQueue.main.async {
83
+ guard runtime.reserveModelMutation() else { call.reject("Finish the current operation before importing", "BUSY"); return }
84
+ guard let controller = self.bridge?.viewController, controller.presentedViewController == nil else { runtime.releaseModelMutation(); call.reject("Document picker unavailable"); return }
85
+ self.pickerCall = call
86
+ let picker = UIDocumentPickerViewController(forOpeningContentTypes: [.data], asCopy: false)
87
+ picker.allowsMultipleSelection = false; picker.delegate = self
88
+ controller.present(picker, animated: true)
89
+ }
90
+ }
91
+ }
92
+ public func documentPickerWasCancelled(_ controller: UIDocumentPickerViewController) {
93
+ guard let call = pickerCall else { return }; pickerCall = nil
94
+ runtime?.releaseModelMutation(); call.resolve(["model": NSNull()])
95
+ }
96
+ public func documentPicker(_ controller: UIDocumentPickerViewController, didPickDocumentsAt urls: [URL]) {
97
+ guard let call = pickerCall else { return }; pickerCall = nil
98
+ runtime?.releaseModelMutation()
99
+ guard let url = urls.first else { call.resolve(["model": NSNull()]); return }
100
+ withRuntime(call) { $0.importModel(Self.adapt(call), from: url) }
101
+ }
102
+ }