@bendyline/gezel-capacitor 0.0.0-stage → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/Package.swift +26 -0
- package/README.md +230 -2
- package/android/build.gradle +30 -0
- package/android/src/main/AndroidManifest.xml +1 -0
- package/android/src/main/java/com/bendyline/gezel/capacitor/GezelKnowledgePlugin.java +153 -0
- package/android/src/main/java/com/bendyline/gezel/capacitor/GezelOfflineSpeechPlugin.java +216 -0
- package/android/src/main/java/com/bendyline/gezel/capacitor/GezelRuntimePlugin.java +84 -0
- package/android/src/main/java/com/bendyline/gezel/capacitor/KnowledgeFiles.java +101 -0
- package/android/src/main/java/com/bendyline/gezel/capacitor/SpeechModels.java +230 -0
- package/android/src/main/java/com/bendyline/gezel/speech/SpeechRuntime.java +38 -0
- package/dist/index.d.ts +561 -0
- package/dist/index.js +7512 -0
- package/dist/knowledge-worker.js +211 -0
- package/dist/licenses/npm/manifest.json +76 -0
- package/dist/licenses/npm/texts/1583809bf1155510d2961f94aa9bca6dc4b44b6f59fd773de33bd6c20860d781.txt +29 -0
- package/dist/licenses/npm/texts/1cd39aa4a566eba49eb206b969ef423295e79b50503df567bbdb2e8877d78892.txt +8 -0
- package/dist/licenses/npm/texts/3f1189b28e3866e0d979968d466b78f813f76827cfdca1fbb124cc0a5c8841f8.txt +21 -0
- package/dist/licenses/npm/texts/5b37224c080cdcc97c871ada971c224e9926370fe74f11b539aa1cf9f3b1aca1.txt +21 -0
- package/dist/licenses/npm/texts/c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4.txt +201 -0
- package/dist/sqlite3.wasm +0 -0
- package/embedding-manifest.json +210 -0
- package/ios/Sources/GezelCapacitor/GezelKnowledgePlugin.swift +200 -0
- package/ios/Sources/GezelCapacitor/GezelOfflineSpeechPlugin.swift +484 -0
- package/ios/Sources/GezelCapacitor/GezelRuntimePlugin.swift +102 -0
- package/kokoro-lexicon/LICENSE +202 -0
- package/kokoro-lexicon/lexicon-gb-en.txt.gz +0 -0
- package/kokoro-lexicon/lexicon-us-en.txt.gz +0 -0
- package/native/android/LICENSE-gezel.txt +21 -0
- package/native/android/PUBLIC-DISTRIBUTION.md +97 -0
- package/native/android/THIRD_PARTY_LICENSES/LICENSE-arm-optimized-routines-MIT-OR-Apache-2.0.txt +249 -0
- package/native/android/THIRD_PARTY_LICENSES/LICENSE-cpp-httplib-MIT.txt +22 -0
- package/native/android/THIRD_PARTY_LICENSES/LICENSE-fp16-MIT.txt +11 -0
- package/native/android/THIRD_PARTY_LICENSES/LICENSE-ggml-MIT.txt +21 -0
- package/native/android/THIRD_PARTY_LICENSES/LICENSE-llama-cpp-llamafile-sgemm-MIT.txt +21 -0
- package/native/android/THIRD_PARTY_LICENSES/LICENSE-llama-cpp-openvino-APACHE.txt +201 -0
- package/native/android/THIRD_PARTY_LICENSES/LICENSE-llama-cpp-pillow-MIT-CMU.txt +30 -0
- package/native/android/THIRD_PARTY_LICENSES/LICENSE-llama-cpp-rotate-bits-MIT.txt +21 -0
- package/native/android/THIRD_PARTY_LICENSES/LICENSE-llama-cpp-unicode-data-Unicode-3.0.txt +39 -0
- package/native/android/THIRD_PARTY_LICENSES/LICENSE-nlohmann-json-MIT.txt +21 -0
- package/native/android/THIRD_PARTY_LICENSES/LICENSE-nvidia-cub-BSD-3-Clause.txt +24 -0
- package/native/android/THIRD_PARTY_LICENSES/LICENSE-vulkan-headers-Apache-2.0-OR-MIT.txt +33 -0
- package/native/android/THIRD_PARTY_LICENSES/LICENSE-vulkan-loader-APACHE.txt +201 -0
- package/native/android/THIRD_PARTY_LICENSES/LICENSE-vulkan-loader-cjson-MIT.txt +32 -0
- package/native/android/THIRD_PARTY_LICENSES/LICENSE-yarn-MIT.txt +21 -0
- package/native/android/THIRD_PARTY_LICENSES/manifest.json +21 -0
- package/native/android/engine-manifest.json +88 -0
- package/native/android/maven/com/bendyline/gezel/gezel-llama/0.1.57/gezel-llama-0.1.57.aar +0 -0
- package/native/android/maven/com/bendyline/gezel/gezel-llama/0.1.57/gezel-llama-0.1.57.aar.sha256 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-llama/0.1.57/gezel-llama-0.1.57.pom +8 -0
- package/native/android/maven/com/bendyline/gezel/gezel-llama/0.1.57/gezel-llama-0.1.57.pom.sha256 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57-sources.jar +0 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57-sources.jar.md5 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57-sources.jar.sha1 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57-sources.jar.sha256 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57-sources.jar.sha512 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.aar +0 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.aar.md5 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.aar.sha1 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.aar.sha256 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.aar.sha512 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.module +118 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.module.md5 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.module.sha1 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.module.sha256 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.module.sha512 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.pom +51 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.pom.md5 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.pom.sha1 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.pom.sha256 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.pom.sha512 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/maven-metadata.xml +13 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/maven-metadata.xml.md5 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/maven-metadata.xml.sha1 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/maven-metadata.xml.sha256 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/maven-metadata.xml.sha512 +1 -0
- package/native/android/sdk-manifest.json +98 -0
- package/native/ios/GezelLlama.xcframework/Info.plist +47 -0
- package/native/ios/GezelLlama.xcframework/ios-arm64/Headers/gezel_llama.h +290 -0
- package/native/ios/GezelLlama.xcframework/ios-arm64/Headers/module.modulemap +9 -0
- package/native/ios/GezelLlama.xcframework/ios-arm64/libGezelLlama.a +0 -0
- package/native/ios/GezelLlama.xcframework/ios-arm64-simulator/Headers/gezel_llama.h +290 -0
- package/native/ios/GezelLlama.xcframework/ios-arm64-simulator/Headers/module.modulemap +9 -0
- package/native/ios/GezelLlama.xcframework/ios-arm64-simulator/libGezelLlama.a +0 -0
- package/native/ios/LICENSE-gezel.txt +21 -0
- package/native/ios/LICENSE-ggml.txt +21 -0
- package/native/ios/LICENSE-llama-cpp.txt +21 -0
- package/native/ios/PUBLIC-DISTRIBUTION.md +97 -0
- package/native/ios/Package.swift +17 -0
- package/native/ios/Sources/GezelModelStorage/MobileModelStore.swift +286 -0
- package/native/ios/Sources/GezelModelStorage/ModelDownloadHTTP.swift +55 -0
- package/native/ios/Sources/GezelModelStorage/ModelDownloadSource.swift +46 -0
- package/native/ios/Sources/GezelModelStorage/ModelDownloads.swift +289 -0
- package/native/ios/Sources/GezelModelStorage/PrivacyInfo.xcprivacy +32 -0
- package/native/ios/Sources/GezelRuntime/AppleFoundationProvider.swift +292 -0
- package/native/ios/Sources/GezelRuntime/AppleNativeTools.swift +152 -0
- package/native/ios/Sources/GezelRuntime/GezelNativeRuntime.swift +1349 -0
- package/native/ios/Sources/GezelRuntime/NativeCall.swift +35 -0
- package/native/ios/Sources/GezelRuntime/PrivacyInfo.xcprivacy +31 -0
- package/native/ios/THIRD_PARTY_LICENSES/LICENSE-arm-optimized-routines-MIT-OR-Apache-2.0.txt +249 -0
- package/native/ios/THIRD_PARTY_LICENSES/LICENSE-cpp-httplib-MIT.txt +22 -0
- package/native/ios/THIRD_PARTY_LICENSES/LICENSE-fp16-MIT.txt +11 -0
- package/native/ios/THIRD_PARTY_LICENSES/LICENSE-ggml-MIT.txt +21 -0
- package/native/ios/THIRD_PARTY_LICENSES/LICENSE-llama-cpp-llamafile-sgemm-MIT.txt +21 -0
- package/native/ios/THIRD_PARTY_LICENSES/LICENSE-llama-cpp-openvino-APACHE.txt +201 -0
- package/native/ios/THIRD_PARTY_LICENSES/LICENSE-llama-cpp-pillow-MIT-CMU.txt +30 -0
- package/native/ios/THIRD_PARTY_LICENSES/LICENSE-llama-cpp-rotate-bits-MIT.txt +21 -0
- package/native/ios/THIRD_PARTY_LICENSES/LICENSE-llama-cpp-unicode-data-Unicode-3.0.txt +39 -0
- package/native/ios/THIRD_PARTY_LICENSES/LICENSE-nlohmann-json-MIT.txt +21 -0
- package/native/ios/THIRD_PARTY_LICENSES/LICENSE-nvidia-cub-BSD-3-Clause.txt +24 -0
- package/native/ios/THIRD_PARTY_LICENSES/LICENSE-vulkan-headers-Apache-2.0-OR-MIT.txt +33 -0
- package/native/ios/THIRD_PARTY_LICENSES/LICENSE-vulkan-loader-APACHE.txt +201 -0
- package/native/ios/THIRD_PARTY_LICENSES/LICENSE-vulkan-loader-cjson-MIT.txt +32 -0
- package/native/ios/THIRD_PARTY_LICENSES/LICENSE-yarn-MIT.txt +21 -0
- package/native/ios/THIRD_PARTY_LICENSES/manifest.json +21 -0
- package/native/ios/engine-manifest.json +102 -0
- package/native/ios/sdk-manifest.json +89 -0
- package/package.json +94 -3
- package/scripts/embedding-package.d.mts +23 -0
- package/scripts/embedding-package.mjs +185 -0
- package/scripts/stage-native.mjs +168 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Bendyline
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/Package.swift
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
// swift-tools-version: 6.0
|
|
2
|
+
import Foundation
|
|
3
|
+
import PackageDescription
|
|
4
|
+
|
|
5
|
+
// Offline speech is optional: without a staged runtime, GezelSpeech reports itself unavailable.
|
|
6
|
+
let speech = FileManager.default.fileExists(
|
|
7
|
+
atPath: Context.packageDirectory + "/native/speech-ios/GezelSpeech.xcframework/Info.plist")
|
|
8
|
+
|
|
9
|
+
let package = Package(
|
|
10
|
+
name: "BendylineGezelCapacitor",
|
|
11
|
+
platforms: [.iOS("16.4")],
|
|
12
|
+
products: [.library(name: "BendylineGezelCapacitor", targets: ["GezelCapacitor"])],
|
|
13
|
+
dependencies: [
|
|
14
|
+
.package(name: "GezelRuntime", path: "native/ios"),
|
|
15
|
+
// Matches the @capacitor/core peer range. An app pins this package exactly to
|
|
16
|
+
// its own Capacitor, so an exact version here would refuse every other one.
|
|
17
|
+
.package(url: "https://github.com/ionic-team/capacitor-swift-pm.git", from: "8.5.2")
|
|
18
|
+
],
|
|
19
|
+
targets: [.target(name: "GezelCapacitor", dependencies: [
|
|
20
|
+
.product(name: "GezelRuntime", package: "GezelRuntime"),
|
|
21
|
+
.product(name: "Capacitor", package: "capacitor-swift-pm"),
|
|
22
|
+
.product(name: "Cordova", package: "capacitor-swift-pm")
|
|
23
|
+
] + (speech ? ["GezelSpeech"] : []), path: "ios/Sources/GezelCapacitor")]
|
|
24
|
+
+ (speech ? [.binaryTarget(name: "GezelSpeech", path: "native/speech-ios/GezelSpeech.xcframework")] : []),
|
|
25
|
+
swiftLanguageModes: [.v5]
|
|
26
|
+
)
|
package/README.md
CHANGED
|
@@ -1,3 +1,231 @@
|
|
|
1
|
-
#
|
|
1
|
+
# Gezel for Capacitor
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
`@bendyline/gezel-capacitor` connects the existing Gezel App SDK to an iOS or
|
|
4
|
+
Android app's own on-device runtime. No daemon, HTTP server, account, product
|
|
5
|
+
workspace, or speech pack is required. The package contains prebuilt llama.cpp
|
|
6
|
+
binaries; installing apps do not build the engine.
|
|
7
|
+
|
|
8
|
+
```ts
|
|
9
|
+
import { connect, GezelRuntime } from '@bendyline/gezel-capacitor';
|
|
10
|
+
|
|
11
|
+
// Show a system file picker. The model is copied into this app's private storage.
|
|
12
|
+
const { model } = await GezelRuntime.importModel();
|
|
13
|
+
if (model) {
|
|
14
|
+
const app = connect(); // GezelApp<'portable'>, using the existing SDK client
|
|
15
|
+
try {
|
|
16
|
+
const stream = await app.chat({
|
|
17
|
+
model: `llama-cpp:${model.id}`,
|
|
18
|
+
messages: [{ role: 'user', content: 'Summarize these notes: …' }],
|
|
19
|
+
stream: true,
|
|
20
|
+
max_tokens: 512,
|
|
21
|
+
});
|
|
22
|
+
for await (const chunk of stream) {
|
|
23
|
+
console.log(chunk.choices[0]?.delta.content ?? '');
|
|
24
|
+
}
|
|
25
|
+
} finally {
|
|
26
|
+
await app.close();
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
`app.models()` lists imported models and system providers with availability,
|
|
32
|
+
capabilities and qualified model IDs. Use the ID of the model the person chose.
|
|
33
|
+
Apple Foundation Models and Android ML Kit use the same `chat` method. An
|
|
34
|
+
unavailable provider is an error; there is no implicit model or cloud fallback.
|
|
35
|
+
|
|
36
|
+
`app.ensureModel({ model })` checks readiness. It never downloads implicitly.
|
|
37
|
+
`GezelRuntime.prepareProvider({ providerId: 'android-mlkit' })` explicitly asks
|
|
38
|
+
ML Kit to prepare its model; iOS manages Apple model preparation in Settings.
|
|
39
|
+
The native API also exposes pinned-source resolution, resumable GGUF downloads,
|
|
40
|
+
selection, removal, and `releaseModel()`. It is separate from ordinary inference
|
|
41
|
+
so an availability probe cannot start a large download.
|
|
42
|
+
|
|
43
|
+
## System models in a model picker
|
|
44
|
+
|
|
45
|
+
The system provider for the current platform is included in `app.models()` even
|
|
46
|
+
when it is unavailable. Keep it visible with its `unavailable_reason` instead of
|
|
47
|
+
filtering it out; only enable selection when `availability === 'available'`.
|
|
48
|
+
The qualified IDs are `apple-foundation-models:apple-foundation-models` for Apple
|
|
49
|
+
Foundation Models and `android-mlkit:android-mlkit` for Gemini Nano through ML Kit.
|
|
50
|
+
Use the returned IDs rather than constructing a choice for a different platform.
|
|
51
|
+
Both run through the same `app.chat({ model: selectedId, ... })` call as a GGUF.
|
|
52
|
+
|
|
53
|
+
Apple readiness depends on supported hardware, OS version, Apple Intelligence
|
|
54
|
+
settings, and system preparation. The app cannot download Apple's model itself.
|
|
55
|
+
ML Kit can report `download-required` or `downloading`; offer an explicit action
|
|
56
|
+
that awaits `GezelRuntime.prepareProvider({ providerId: 'android-mlkit' })`, then
|
|
57
|
+
refresh `app.models()`. Listing and `ensureModel` never authorize preparation.
|
|
58
|
+
Refresh after returning from system Settings. If a saved choice becomes
|
|
59
|
+
unavailable, preserve it and show the reason; do not silently pick another model.
|
|
60
|
+
|
|
61
|
+
## Contract and lifecycle
|
|
62
|
+
|
|
63
|
+
- Text messages (`system`, `user`, `assistant`), streaming, explicit model ID,
|
|
64
|
+
and `max_tokens` are supported. Tools, images, JSON constraints, sampling
|
|
65
|
+
options and other unsupported controls reject with `unsupported_capability`.
|
|
66
|
+
- Without `max_tokens`, a reply may use up to half the context window (at most
|
|
67
|
+
4096 tokens on llama.cpp, 1024 on the system models). On llama.cpp that is a
|
|
68
|
+
ceiling within whatever room the prompt leaves, not a reservation.
|
|
69
|
+
- Replies preserve `stop`, `length`, and `cancelled`. Usage is absent when the
|
|
70
|
+
provider cannot measure it. Desktop's existing `GezelApp` default types remain
|
|
71
|
+
unchanged; portable clients use `GezelApp<'portable'>`.
|
|
72
|
+
- Pass `{ signal }` as the second `chat` argument to cancel. Breaking out of a
|
|
73
|
+
stream or closing its client also cancels and waits for native release.
|
|
74
|
+
- `enginePhase` events report `loading_model`, `prefill`, and `generating` for a
|
|
75
|
+
`requestId`, with load progress and prompt/processed/reused/output token
|
|
76
|
+
counts where the engine measures them. They are advisory: older native builds
|
|
77
|
+
and platform providers send fewer, so never wait on one.
|
|
78
|
+
- Clients share one process runtime and admission gate. Closing an idle client
|
|
79
|
+
does not stop another client's generation. `releaseModel()` is explicitly
|
|
80
|
+
process-wide and may cancel active work. A closed client cannot be reused.
|
|
81
|
+
- Backgrounding cancels inference and pauses downloads; foregrounding does not
|
|
82
|
+
automatically restart either. Memory/thermal admission remains native.
|
|
83
|
+
- Model files belong to the host app's sandbox. Sharing this package between
|
|
84
|
+
Docblocks and Qualla does **not** share their live model files or permissions.
|
|
85
|
+
- The embedding application owns WebView navigation and content trust, as with
|
|
86
|
+
other Capacitor native plugins. Keep untrusted pages out of the privileged
|
|
87
|
+
app bridge; this plugin does not override the application's navigation policy.
|
|
88
|
+
|
|
89
|
+
`connectRuntime(plugin)` accepts an injected implementation for alternate native
|
|
90
|
+
bridges and contract tests. `connect()` requires a native Capacitor platform;
|
|
91
|
+
there is no browser runtime stub pretending to run a local model.
|
|
92
|
+
|
|
93
|
+
## Offline speech
|
|
94
|
+
|
|
95
|
+
`createMobileSpeech()` adds Whisper dictation and Kokoro narration on the device,
|
|
96
|
+
through the `GezelOfflineSpeech` plugin and the optional gezel-speech runtime
|
|
97
|
+
(whisper.cpp and ONNX Runtime, built by
|
|
98
|
+
[`native/mobile/speech/build.py`](../../native/mobile/speech/build.py)). Without
|
|
99
|
+
a staged runtime the package still builds, and `status()` reports speech
|
|
100
|
+
unavailable.
|
|
101
|
+
|
|
102
|
+
```ts
|
|
103
|
+
import { createMobileSpeech } from '@bendyline/gezel-capacitor';
|
|
104
|
+
|
|
105
|
+
// The dictionaries ship in this package under kokoro-lexicon/; serve them as
|
|
106
|
+
// web assets and hand back the inflated text.
|
|
107
|
+
const speech = createMobileSpeech({ lexicon: (language) => fetchLexiconText(language) });
|
|
108
|
+
await speech.install('whisper-base.en', { onProgress: console.log }); // explicit, resumable
|
|
109
|
+
const transcript = await speech.transcribe({ wav, model: 'whisper-base.en', words: true });
|
|
110
|
+
await speech.synthesize({ text, voice: 'af_heart', onChunk: play });
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
- **Models.** `MOBILE_SPEECH_MODELS` downloads the same pinned Whisper files and
|
|
114
|
+
Kokoro voices as `@bendyline/gezel/speech-models` (a test holds them in step),
|
|
115
|
+
and Kokoro's int8 export of the same timestamped model (92 MB instead of
|
|
116
|
+
326 MB), so word timings survive. Native code stores only declared files in
|
|
117
|
+
app-private, no-backup storage, verifies each SHA-256, and resumes partial files
|
|
118
|
+
with `Range`. JavaScript names models and files; it never supplies paths.
|
|
119
|
+
`install()` retries interrupted transfers while they keep saving bytes.
|
|
120
|
+
- **Narration.** Phonemes are planned in JavaScript by `@bendyline/gezel/kokoro`
|
|
121
|
+
from the dictionaries in `kokoro-lexicon/`, so no eSpeak NG (GPL-3) ships.
|
|
122
|
+
Each sentence comes back as 24 kHz PCM with word timings from the model's
|
|
123
|
+
durations.
|
|
124
|
+
- **Lifecycle.** One request runs at a time and `AbortSignal` cancels it natively.
|
|
125
|
+
Models stay loaded between requests, are released after two idle minutes, and
|
|
126
|
+
are released, with any request stopped and any download paused, when the app
|
|
127
|
+
leaves the foreground.
|
|
128
|
+
- **Staging.** `build.py ios|android --package VERSION` lays out a verified
|
|
129
|
+
payload; `node scripts/stage-native.mjs speech-ios|speech-android <payload>`
|
|
130
|
+
stages it under `native/speech-<platform>`. Android adds the `gezel-speech` AAR
|
|
131
|
+
(it loads `libc++_shared.so` from `gezel-llama` of the same NDK); iOS adds the
|
|
132
|
+
`GezelSpeech` binary target. The JNI binding is compiled into the library, so
|
|
133
|
+
apps build no native code. `GEZEL_SPEECH_ABI` pairs the plugins with the
|
|
134
|
+
library.
|
|
135
|
+
|
|
136
|
+
## Knowledge catalogs
|
|
137
|
+
|
|
138
|
+
`createMobileKnowledge()` is the App SDK's knowledge client (`state`, `update`,
|
|
139
|
+
`retrieve`) for Gilde knowledge catalogs on the device, with the daemon routes'
|
|
140
|
+
semantics: give it to an embedding connection as `knowledge`, and
|
|
141
|
+
`knowledgeContext()` grounds requests exactly as it does on a desktop.
|
|
142
|
+
|
|
143
|
+
```ts
|
|
144
|
+
import { connectEmbeddingRuntime, createMobileKnowledge } from '@bendyline/gezel-capacitor';
|
|
145
|
+
|
|
146
|
+
// Serve dist/knowledge-worker.js and dist/sqlite3.wasm as web assets.
|
|
147
|
+
const knowledge = createMobileKnowledge({
|
|
148
|
+
workerUrl: '/assets/knowledge-worker.js',
|
|
149
|
+
wasmUrl: '/assets/sqlite3.wasm',
|
|
150
|
+
readDisabled: () => JSON.parse(localStorage.getItem('knowledge-off') ?? '[]'),
|
|
151
|
+
writeDisabled: (ids) => localStorage.setItem('knowledge-off', JSON.stringify(ids)),
|
|
152
|
+
});
|
|
153
|
+
const connection = { ...(await connectEmbeddingRuntime()), knowledge };
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
- **Catalogs.** `mobileKnowledgeCatalogs` is a snapshot of Gilde's catalog
|
|
157
|
+
items (`scripts/refresh-knowledge-catalog.ts`), each pinned to an immutable
|
|
158
|
+
Hugging Face revision, SHA-256 and size. The `GezelKnowledge` plugin downloads
|
|
159
|
+
one `.gezk` archive per catalog into app-private, no-backup storage, verifies
|
|
160
|
+
it, and resumes with `Range` like speech models; an update replaces the
|
|
161
|
+
version it supersedes only once it is verified.
|
|
162
|
+
- **Reading in place.** Every archive entry is stored uncompressed, so nothing is
|
|
163
|
+
extracted. A Web Worker runs the daemon's catalog host over Gezel's own
|
|
164
|
+
catalog reader on sqlite-wasm, whose read-only VFS fetches 64 KB blocks with
|
|
165
|
+
synchronous range requests: from `/_gezel_knowledge/` on Android (the app's
|
|
166
|
+
`WebViewClient` must call `KnowledgeFiles.intercept` first) and from
|
|
167
|
+
Capacitor's file URL on iOS.
|
|
168
|
+
- **Search.** The daemon's `searchUnifiedCatalogs`, with its vector floors.
|
|
169
|
+
Each catalog's embedding profile downloads its pinned query model with the
|
|
170
|
+
archive. Questions are tokenized by the profile's own tokenizer and embedded
|
|
171
|
+
by the speech runtime's ONNX Runtime (`gezel_speech_embed`), so query vectors
|
|
172
|
+
match the desktop's. Without the relevance model, `retrieve` admits only what
|
|
173
|
+
Gezel's bar for unjudged passages admits, which is vector evidence; until the
|
|
174
|
+
query model is installed a catalog is searched by keyword and contributes no
|
|
175
|
+
passages. `rerank: 'required'` and `prepare-reranker` are refused.
|
|
176
|
+
- **Licenses.** The worker and the tokenizer compile third-party code into
|
|
177
|
+
`dist/`; the build stages their license texts in `dist/licenses/npm/`.
|
|
178
|
+
|
|
179
|
+
## Local development and distribution
|
|
180
|
+
|
|
181
|
+
This is a local preview; registry publication is not required. The producer
|
|
182
|
+
stages the runtime once using [the native host guide](../../native/runtime/README.md),
|
|
183
|
+
then builds and packs this package. Install matching local tarballs for `gezk`,
|
|
184
|
+
core, client, App SDK and Capacitor in the consumer app, and run `npx cap sync`.
|
|
185
|
+
|
|
186
|
+
```sh
|
|
187
|
+
# In the Gezel checkout; dependency installation follows the repository's lease policy.
|
|
188
|
+
node packages/capacitor/scripts/stage-native.mjs ios /path/to/staged-ios-runtime
|
|
189
|
+
node packages/capacitor/scripts/stage-native.mjs android /path/to/staged-android-runtime
|
|
190
|
+
# Build core and app-sdk first, then this package. Pack with pnpm so workspace:*
|
|
191
|
+
# references become ordinary package versions.
|
|
192
|
+
pnpm --filter @bendyline/gezel-capacitor build
|
|
193
|
+
pnpm --filter @bendyline/gezel-capacitor pack --pack-destination /path/to/local-packages
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
Pack the other four workspace packages into that same folder with `pnpm pack`.
|
|
197
|
+
For pnpm 11 local previews, put overrides for the five `@bendyline/*` packages
|
|
198
|
+
in the **consumer's** `pnpm-workspace.yaml`, so transitive dependencies resolve to
|
|
199
|
+
the matching tarballs rather than looking up unreleased versions in a registry:
|
|
200
|
+
|
|
201
|
+
```yaml
|
|
202
|
+
overrides:
|
|
203
|
+
'@bendyline/gezel-capacitor': file:/path/to/local-packages/bendyline-gezel-capacitor-0.1.0.tgz
|
|
204
|
+
'@bendyline/gezel-app-sdk': file:/path/to/local-packages/bendyline-gezel-app-sdk-1.0.10.tgz
|
|
205
|
+
'@bendyline/gezel-client': file:/path/to/local-packages/bendyline-gezel-client-1.1.2.tgz
|
|
206
|
+
'@bendyline/gezel': file:/path/to/local-packages/bendyline-gezel-1.1.2.tgz
|
|
207
|
+
'@bendyline/gezk': file:/path/to/local-packages/bendyline-gezk-1.0.2.tgz
|
|
208
|
+
```
|
|
209
|
+
|
|
210
|
+
Use the filenames actually produced by your checkout. Then add the Capacitor
|
|
211
|
+
package and `@capacitor/core` to that app's dependencies normally. Registry
|
|
212
|
+
releases will remove the need for these local overrides. No `node_modules`
|
|
213
|
+
folder or running model service needs to be shared between the applications.
|
|
214
|
+
|
|
215
|
+
The `prepack` check verifies both native payloads and requires matching versions.
|
|
216
|
+
The tarball includes a self-contained Swift package and folder Maven repository.
|
|
217
|
+
There are no consumer references to `native/mobile/.build` or sibling checkouts.
|
|
218
|
+
Use Capacitor 8.5.2, Android API 28+ (compile SDK 36), and iOS 16.4+ with a current
|
|
219
|
+
Xcode toolchain supporting the Foundation Models adapter. System AI availability
|
|
220
|
+
depends on OS, hardware, configuration and model readiness. The current engine
|
|
221
|
+
preview includes arm64 Android, arm64 iOS devices and arm64 iOS simulators.
|
|
222
|
+
|
|
223
|
+
Native generation on physical devices, store distribution, and third-party
|
|
224
|
+
application adoption remain qualification steps; package compilation alone is
|
|
225
|
+
not an inference-quality certification.
|
|
226
|
+
|
|
227
|
+
## Application embedding wrapper
|
|
228
|
+
|
|
229
|
+
For opt-in lifecycle, model readiness/preparation, cancellation, and packaging, see
|
|
230
|
+
[the embedding guide](https://github.com/bendyline/gezel/blob/main/docs/embedding-sdk.md).
|
|
231
|
+
The existing low-level connection APIs remain available.
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
apply plugin: 'com.android.library'
|
|
2
|
+
|
|
3
|
+
def gezelNative = file('../native/android')
|
|
4
|
+
if (!new File(gezelNative, 'sdk-manifest.json').isFile()) {
|
|
5
|
+
throw new GradleException('Stage the Gezel Capacitor native package before syncing Android. See @bendyline/gezel-capacitor/README.md.')
|
|
6
|
+
}
|
|
7
|
+
def gezelVersion = new groovy.json.JsonSlurper().parse(new File(gezelNative, 'sdk-manifest.json')).packageVersion
|
|
8
|
+
// Offline speech is optional: without a staged runtime, GezelSpeech reports itself unavailable.
|
|
9
|
+
def gezelSpeech = file('../native/speech-android')
|
|
10
|
+
def speechManifest = new File(gezelSpeech, 'sdk-manifest.json')
|
|
11
|
+
def speechVersion = speechManifest.isFile() ? new groovy.json.JsonSlurper().parse(speechManifest).packageVersion : null
|
|
12
|
+
// Transitive Maven artifacts must also be visible to the consuming app project.
|
|
13
|
+
rootProject.allprojects { repositories {
|
|
14
|
+
google()
|
|
15
|
+
mavenCentral()
|
|
16
|
+
maven { url = uri(new File(gezelNative, 'maven')) }
|
|
17
|
+
if (speechVersion) maven { url = uri(new File(gezelSpeech, 'maven')) }
|
|
18
|
+
} }
|
|
19
|
+
android {
|
|
20
|
+
namespace 'com.bendyline.gezel.capacitor'
|
|
21
|
+
compileSdk rootProject.ext.has('compileSdkVersion') ? rootProject.ext.compileSdkVersion : 36
|
|
22
|
+
defaultConfig { minSdk 28 }
|
|
23
|
+
compileOptions { sourceCompatibility JavaVersion.VERSION_17; targetCompatibility JavaVersion.VERSION_17 }
|
|
24
|
+
}
|
|
25
|
+
dependencies {
|
|
26
|
+
implementation project(':capacitor-android')
|
|
27
|
+
implementation 'androidx.activity:activity:1.11.0'
|
|
28
|
+
api "com.bendyline.gezel:gezel-runtime:${gezelVersion}"
|
|
29
|
+
if (speechVersion) implementation "com.bendyline.gezel:gezel-speech:${speechVersion}"
|
|
30
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
<manifest xmlns:android="http://schemas.android.com/apk/res/android" />
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
package com.bendyline.gezel.capacitor;
|
|
2
|
+
|
|
3
|
+
import com.bendyline.gezel.speech.SpeechRuntime;
|
|
4
|
+
import com.getcapacitor.*;
|
|
5
|
+
import com.getcapacitor.annotation.CapacitorPlugin;
|
|
6
|
+
import java.io.File;
|
|
7
|
+
import java.util.concurrent.CancellationException;
|
|
8
|
+
import java.util.concurrent.ExecutorService;
|
|
9
|
+
import java.util.concurrent.Executors;
|
|
10
|
+
import org.json.JSONArray;
|
|
11
|
+
import org.json.JSONObject;
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Knowledge catalog archives on the device. Each catalog is one verified
|
|
15
|
+
* `.gezk` download that the WebView's reader opens in place, in byte ranges
|
|
16
|
+
* served by {@link KnowledgeFiles}; nothing is extracted. JavaScript chooses
|
|
17
|
+
* catalogs from its pinned snapshot; this plugin only stores what it is given.
|
|
18
|
+
* Each catalog's query model downloads into the same store and runs on the
|
|
19
|
+
* speech runtime's ONNX Runtime ({@link #embed}).
|
|
20
|
+
*/
|
|
21
|
+
@CapacitorPlugin(name = "GezelKnowledge")
|
|
22
|
+
public final class GezelKnowledgePlugin extends Plugin {
|
|
23
|
+
/** A sanity bound; the store refuses any download the device has no room for. */
|
|
24
|
+
private static final long MAX_ARCHIVE = 32L << 30;
|
|
25
|
+
private final ExecutorService downloads = Executors.newSingleThreadExecutor();
|
|
26
|
+
private final ExecutorService encoder = Executors.newSingleThreadExecutor();
|
|
27
|
+
private SpeechModels store;
|
|
28
|
+
private long engine;
|
|
29
|
+
private String downloading;
|
|
30
|
+
private boolean paused;
|
|
31
|
+
|
|
32
|
+
@Override public void load() { store = new SpeechModels(KnowledgeFiles.root(getContext()), MAX_ARCHIVE); }
|
|
33
|
+
|
|
34
|
+
@PluginMethod public void info(PluginCall call) {
|
|
35
|
+
try {
|
|
36
|
+
JSObject result = new JSObject().put("baseUrl", "https://localhost" + KnowledgeFiles.PREFIX.substring(0, KnowledgeFiles.PREFIX.length() - 1)).put("catalogs", store.list());
|
|
37
|
+
synchronized (this) { if (downloading != null) result.put("downloading", downloading); }
|
|
38
|
+
call.resolve(result);
|
|
39
|
+
} catch (Exception error) { call.reject("Knowledge catalogs could not be read", "failed"); }
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
@PluginMethod public void install(PluginCall call) {
|
|
43
|
+
String id;
|
|
44
|
+
try { id = SpeechModels.id(call.getString("id")); }
|
|
45
|
+
catch (Exception invalid) { call.reject("Invalid knowledge catalog", "invalid-input"); return; }
|
|
46
|
+
JSONArray files;
|
|
47
|
+
try {
|
|
48
|
+
files = new JSONArray().put(new JSONObject().put("name", call.getString("version") + "/catalog.gezk")
|
|
49
|
+
.put("url", call.getString("url")).put("sha256", call.getString("sha256")).put("size", call.getData().getLong("size")));
|
|
50
|
+
} catch (Exception invalid) { call.reject("Invalid knowledge catalog", "invalid-input"); return; }
|
|
51
|
+
synchronized (this) {
|
|
52
|
+
if (downloading != null) { call.reject("Another catalog is downloading", "busy"); return; }
|
|
53
|
+
downloading = id; paused = false;
|
|
54
|
+
}
|
|
55
|
+
downloads.execute(() -> {
|
|
56
|
+
try {
|
|
57
|
+
store.install(id, files, (received, total) -> notifyListeners("knowledgeDownload",
|
|
58
|
+
new JSObject().put("id", id).put("receivedBytes", received).put("totalBytes", total)));
|
|
59
|
+
// One version per catalog: a verified update replaces the archive it supersedes.
|
|
60
|
+
// Pruning only now keeps an interrupted download's partial file for resuming.
|
|
61
|
+
java.io.File[] versions = new java.io.File(store.root(), id).listFiles(java.io.File::isDirectory);
|
|
62
|
+
if (versions != null) for (java.io.File version : versions)
|
|
63
|
+
if (!version.getName().equals(call.getString("version"))) delete(version);
|
|
64
|
+
call.resolve(new JSObject().put("id", id));
|
|
65
|
+
} catch (CancellationException cancelled) {
|
|
66
|
+
boolean wasPaused; synchronized (this) { wasPaused = paused; }
|
|
67
|
+
call.reject(wasPaused ? "Download paused when the app left the foreground" : "Download cancelled", wasPaused ? "paused" : "cancelled");
|
|
68
|
+
} catch (SpeechModels.Failure failure) { call.reject(failure.getMessage(), failure.code); }
|
|
69
|
+
catch (Exception error) { call.reject(error.getMessage() == null ? "The catalog could not be saved" : error.getMessage(), "failed"); }
|
|
70
|
+
finally { synchronized (this) { downloading = null; } }
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
private static void delete(java.io.File file) {
|
|
75
|
+
java.io.File[] children = file.listFiles();
|
|
76
|
+
if (children != null) for (java.io.File child : children) delete(child);
|
|
77
|
+
//noinspection ResultOfMethodCallIgnored
|
|
78
|
+
file.delete();
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/** A catalog profile's query model, verified file by file like a catalog. */
|
|
82
|
+
@PluginMethod public void installModel(PluginCall call) {
|
|
83
|
+
String id;
|
|
84
|
+
try { id = SpeechModels.id(call.getString("id")); }
|
|
85
|
+
catch (Exception invalid) { call.reject("Invalid query model", "invalid-input"); return; }
|
|
86
|
+
JSONArray files = call.getArray("files");
|
|
87
|
+
synchronized (this) {
|
|
88
|
+
if (downloading != null) { call.reject("Another download is running", "busy"); return; }
|
|
89
|
+
downloading = id; paused = false;
|
|
90
|
+
}
|
|
91
|
+
downloads.execute(() -> {
|
|
92
|
+
try {
|
|
93
|
+
store.install(id, files, (received, total) -> notifyListeners("knowledgeDownload",
|
|
94
|
+
new JSObject().put("id", id).put("receivedBytes", received).put("totalBytes", total)));
|
|
95
|
+
call.resolve(new JSObject().put("id", id));
|
|
96
|
+
} catch (CancellationException cancelled) {
|
|
97
|
+
boolean wasPaused; synchronized (this) { wasPaused = paused; }
|
|
98
|
+
call.reject(wasPaused ? "Download paused when the app left the foreground" : "Download cancelled", wasPaused ? "paused" : "cancelled");
|
|
99
|
+
} catch (SpeechModels.Failure failure) { call.reject(failure.getMessage(), failure.code); }
|
|
100
|
+
catch (Exception error) { call.reject(error.getMessage() == null ? "The query model could not be saved" : error.getMessage(), "failed"); }
|
|
101
|
+
finally { synchronized (this) { downloading = null; } }
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/** A query's unit vector from the installed model; token ids come from the profile's tokenizer. */
|
|
106
|
+
@PluginMethod public void embed(PluginCall call) {
|
|
107
|
+
if (!SpeechRuntime.available()) { call.reject(SpeechRuntime.unavailableReason(), "unavailable"); return; }
|
|
108
|
+
encoder.execute(() -> {
|
|
109
|
+
try {
|
|
110
|
+
File model = store.file(call.getString("model"), call.getString("file"));
|
|
111
|
+
JSONArray encoded = call.getArray("ids");
|
|
112
|
+
if (encoded == null || encoded.length() == 0 || encoded.length() > 8192) throw new SpeechModels.Failure("invalid-input", "Invalid text for the query model");
|
|
113
|
+
long[] ids = new long[encoded.length()];
|
|
114
|
+
for (int i = 0; i < ids.length; i++) ids[i] = encoded.getLong(i);
|
|
115
|
+
long handle;
|
|
116
|
+
synchronized (this) {
|
|
117
|
+
if (engine == 0) engine = SpeechRuntime.create();
|
|
118
|
+
if (engine == 0) throw new OutOfMemoryError();
|
|
119
|
+
SpeechRuntime.begin(engine);
|
|
120
|
+
handle = engine;
|
|
121
|
+
}
|
|
122
|
+
call.resolve(new JSObject().put("vector", new JSONArray(SpeechRuntime.embed(handle, model.getPath(), ids))));
|
|
123
|
+
} catch (SpeechModels.Failure failure) { call.reject(failure.getMessage(), failure.code); }
|
|
124
|
+
catch (OutOfMemoryError error) { call.reject("Not enough memory for knowledge search", "resources"); }
|
|
125
|
+
catch (Exception | LinkageError error) { call.reject(error.getMessage() == null ? "The query model failed" : error.getMessage(), "failed"); }
|
|
126
|
+
});
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
@PluginMethod public void cancelInstall(PluginCall call) { store.cancel(); call.resolve(); }
|
|
130
|
+
|
|
131
|
+
@PluginMethod public void remove(PluginCall call) {
|
|
132
|
+
String id = call.getString("id");
|
|
133
|
+
synchronized (this) { if (id != null && id.equals(downloading)) { call.reject("Cancel the download before removing this catalog", "busy"); return; } }
|
|
134
|
+
downloads.execute(() -> {
|
|
135
|
+
try { store.remove(id); call.resolve(); }
|
|
136
|
+
catch (SpeechModels.Failure failure) { call.reject(failure.getMessage(), failure.code); }
|
|
137
|
+
catch (Exception error) { call.reject("The catalog could not be removed", "failed"); }
|
|
138
|
+
});
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
@Override protected void handleOnStop() {
|
|
142
|
+
synchronized (this) { if (downloading != null) paused = true; }
|
|
143
|
+
store.cancel();
|
|
144
|
+
// The query model holds hundreds of megabytes; load it again on the next search.
|
|
145
|
+
encoder.execute(() -> { synchronized (this) { if (engine != 0) SpeechRuntime.release(engine); } });
|
|
146
|
+
}
|
|
147
|
+
@Override protected void handleOnDestroy() {
|
|
148
|
+
store.cancel();
|
|
149
|
+
encoder.execute(() -> { synchronized (this) { if (engine != 0) SpeechRuntime.destroy(engine); engine = 0; } });
|
|
150
|
+
downloads.shutdown();
|
|
151
|
+
encoder.shutdown();
|
|
152
|
+
}
|
|
153
|
+
}
|