@bendyline/gezel-capacitor 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/Package.swift +26 -0
- package/README.md +230 -2
- package/android/build.gradle +30 -0
- package/android/src/main/AndroidManifest.xml +1 -0
- package/android/src/main/java/com/bendyline/gezel/capacitor/GezelKnowledgePlugin.java +153 -0
- package/android/src/main/java/com/bendyline/gezel/capacitor/GezelOfflineSpeechPlugin.java +216 -0
- package/android/src/main/java/com/bendyline/gezel/capacitor/GezelRuntimePlugin.java +84 -0
- package/android/src/main/java/com/bendyline/gezel/capacitor/KnowledgeFiles.java +101 -0
- package/android/src/main/java/com/bendyline/gezel/capacitor/SpeechModels.java +230 -0
- package/android/src/main/java/com/bendyline/gezel/speech/SpeechRuntime.java +38 -0
- package/dist/index.d.ts +561 -0
- package/dist/index.js +7512 -0
- package/dist/knowledge-worker.js +211 -0
- package/dist/licenses/npm/manifest.json +76 -0
- package/dist/licenses/npm/texts/1583809bf1155510d2961f94aa9bca6dc4b44b6f59fd773de33bd6c20860d781.txt +29 -0
- package/dist/licenses/npm/texts/1cd39aa4a566eba49eb206b969ef423295e79b50503df567bbdb2e8877d78892.txt +8 -0
- package/dist/licenses/npm/texts/3f1189b28e3866e0d979968d466b78f813f76827cfdca1fbb124cc0a5c8841f8.txt +21 -0
- package/dist/licenses/npm/texts/5b37224c080cdcc97c871ada971c224e9926370fe74f11b539aa1cf9f3b1aca1.txt +21 -0
- package/dist/licenses/npm/texts/c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4.txt +201 -0
- package/dist/sqlite3.wasm +0 -0
- package/embedding-manifest.json +210 -0
- package/ios/Sources/GezelCapacitor/GezelKnowledgePlugin.swift +200 -0
- package/ios/Sources/GezelCapacitor/GezelOfflineSpeechPlugin.swift +484 -0
- package/ios/Sources/GezelCapacitor/GezelRuntimePlugin.swift +102 -0
- package/kokoro-lexicon/LICENSE +202 -0
- package/kokoro-lexicon/lexicon-gb-en.txt.gz +0 -0
- package/kokoro-lexicon/lexicon-us-en.txt.gz +0 -0
- package/native/android/LICENSE-gezel.txt +21 -0
- package/native/android/PUBLIC-DISTRIBUTION.md +97 -0
- package/native/android/THIRD_PARTY_LICENSES/LICENSE-arm-optimized-routines-MIT-OR-Apache-2.0.txt +249 -0
- package/native/android/THIRD_PARTY_LICENSES/LICENSE-cpp-httplib-MIT.txt +22 -0
- package/native/android/THIRD_PARTY_LICENSES/LICENSE-fp16-MIT.txt +11 -0
- package/native/android/THIRD_PARTY_LICENSES/LICENSE-ggml-MIT.txt +21 -0
- package/native/android/THIRD_PARTY_LICENSES/LICENSE-llama-cpp-llamafile-sgemm-MIT.txt +21 -0
- package/native/android/THIRD_PARTY_LICENSES/LICENSE-llama-cpp-openvino-APACHE.txt +201 -0
- package/native/android/THIRD_PARTY_LICENSES/LICENSE-llama-cpp-pillow-MIT-CMU.txt +30 -0
- package/native/android/THIRD_PARTY_LICENSES/LICENSE-llama-cpp-rotate-bits-MIT.txt +21 -0
- package/native/android/THIRD_PARTY_LICENSES/LICENSE-llama-cpp-unicode-data-Unicode-3.0.txt +39 -0
- package/native/android/THIRD_PARTY_LICENSES/LICENSE-nlohmann-json-MIT.txt +21 -0
- package/native/android/THIRD_PARTY_LICENSES/LICENSE-nvidia-cub-BSD-3-Clause.txt +24 -0
- package/native/android/THIRD_PARTY_LICENSES/LICENSE-vulkan-headers-Apache-2.0-OR-MIT.txt +33 -0
- package/native/android/THIRD_PARTY_LICENSES/LICENSE-vulkan-loader-APACHE.txt +201 -0
- package/native/android/THIRD_PARTY_LICENSES/LICENSE-vulkan-loader-cjson-MIT.txt +32 -0
- package/native/android/THIRD_PARTY_LICENSES/LICENSE-yarn-MIT.txt +21 -0
- package/native/android/THIRD_PARTY_LICENSES/manifest.json +21 -0
- package/native/android/engine-manifest.json +88 -0
- package/native/android/maven/com/bendyline/gezel/gezel-llama/0.1.57/gezel-llama-0.1.57.aar +0 -0
- package/native/android/maven/com/bendyline/gezel/gezel-llama/0.1.57/gezel-llama-0.1.57.aar.sha256 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-llama/0.1.57/gezel-llama-0.1.57.pom +8 -0
- package/native/android/maven/com/bendyline/gezel/gezel-llama/0.1.57/gezel-llama-0.1.57.pom.sha256 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57-sources.jar +0 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57-sources.jar.md5 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57-sources.jar.sha1 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57-sources.jar.sha256 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57-sources.jar.sha512 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.aar +0 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.aar.md5 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.aar.sha1 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.aar.sha256 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.aar.sha512 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.module +118 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.module.md5 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.module.sha1 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.module.sha256 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.module.sha512 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.pom +51 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.pom.md5 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.pom.sha1 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.pom.sha256 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/0.1.57/gezel-runtime-0.1.57.pom.sha512 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/maven-metadata.xml +13 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/maven-metadata.xml.md5 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/maven-metadata.xml.sha1 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/maven-metadata.xml.sha256 +1 -0
- package/native/android/maven/com/bendyline/gezel/gezel-runtime/maven-metadata.xml.sha512 +1 -0
- package/native/android/sdk-manifest.json +98 -0
- package/native/ios/GezelLlama.xcframework/Info.plist +47 -0
- package/native/ios/GezelLlama.xcframework/ios-arm64/Headers/gezel_llama.h +290 -0
- package/native/ios/GezelLlama.xcframework/ios-arm64/Headers/module.modulemap +9 -0
- package/native/ios/GezelLlama.xcframework/ios-arm64/libGezelLlama.a +0 -0
- package/native/ios/GezelLlama.xcframework/ios-arm64-simulator/Headers/gezel_llama.h +290 -0
- package/native/ios/GezelLlama.xcframework/ios-arm64-simulator/Headers/module.modulemap +9 -0
- package/native/ios/GezelLlama.xcframework/ios-arm64-simulator/libGezelLlama.a +0 -0
- package/native/ios/LICENSE-gezel.txt +21 -0
- package/native/ios/LICENSE-ggml.txt +21 -0
- package/native/ios/LICENSE-llama-cpp.txt +21 -0
- package/native/ios/PUBLIC-DISTRIBUTION.md +97 -0
- package/native/ios/Package.swift +17 -0
- package/native/ios/Sources/GezelModelStorage/MobileModelStore.swift +286 -0
- package/native/ios/Sources/GezelModelStorage/ModelDownloadHTTP.swift +55 -0
- package/native/ios/Sources/GezelModelStorage/ModelDownloadSource.swift +46 -0
- package/native/ios/Sources/GezelModelStorage/ModelDownloads.swift +289 -0
- package/native/ios/Sources/GezelModelStorage/PrivacyInfo.xcprivacy +32 -0
- package/native/ios/Sources/GezelRuntime/AppleFoundationProvider.swift +292 -0
- package/native/ios/Sources/GezelRuntime/AppleNativeTools.swift +152 -0
- package/native/ios/Sources/GezelRuntime/GezelNativeRuntime.swift +1349 -0
- package/native/ios/Sources/GezelRuntime/NativeCall.swift +35 -0
- package/native/ios/Sources/GezelRuntime/PrivacyInfo.xcprivacy +31 -0
- package/native/ios/THIRD_PARTY_LICENSES/LICENSE-arm-optimized-routines-MIT-OR-Apache-2.0.txt +249 -0
- package/native/ios/THIRD_PARTY_LICENSES/LICENSE-cpp-httplib-MIT.txt +22 -0
- package/native/ios/THIRD_PARTY_LICENSES/LICENSE-fp16-MIT.txt +11 -0
- package/native/ios/THIRD_PARTY_LICENSES/LICENSE-ggml-MIT.txt +21 -0
- package/native/ios/THIRD_PARTY_LICENSES/LICENSE-llama-cpp-llamafile-sgemm-MIT.txt +21 -0
- package/native/ios/THIRD_PARTY_LICENSES/LICENSE-llama-cpp-openvino-APACHE.txt +201 -0
- package/native/ios/THIRD_PARTY_LICENSES/LICENSE-llama-cpp-pillow-MIT-CMU.txt +30 -0
- package/native/ios/THIRD_PARTY_LICENSES/LICENSE-llama-cpp-rotate-bits-MIT.txt +21 -0
- package/native/ios/THIRD_PARTY_LICENSES/LICENSE-llama-cpp-unicode-data-Unicode-3.0.txt +39 -0
- package/native/ios/THIRD_PARTY_LICENSES/LICENSE-nlohmann-json-MIT.txt +21 -0
- package/native/ios/THIRD_PARTY_LICENSES/LICENSE-nvidia-cub-BSD-3-Clause.txt +24 -0
- package/native/ios/THIRD_PARTY_LICENSES/LICENSE-vulkan-headers-Apache-2.0-OR-MIT.txt +33 -0
- package/native/ios/THIRD_PARTY_LICENSES/LICENSE-vulkan-loader-APACHE.txt +201 -0
- package/native/ios/THIRD_PARTY_LICENSES/LICENSE-vulkan-loader-cjson-MIT.txt +32 -0
- package/native/ios/THIRD_PARTY_LICENSES/LICENSE-yarn-MIT.txt +21 -0
- package/native/ios/THIRD_PARTY_LICENSES/manifest.json +21 -0
- package/native/ios/engine-manifest.json +102 -0
- package/native/ios/sdk-manifest.json +89 -0
- package/package.json +94 -3
- package/scripts/embedding-package.d.mts +23 -0
- package/scripts/embedding-package.mjs +185 -0
- package/scripts/stage-native.mjs +168 -0
|
@@ -0,0 +1,290 @@
|
|
|
1
|
+
#ifndef GEZEL_LLAMA_H
|
|
2
|
+
#define GEZEL_LLAMA_H
|
|
3
|
+
|
|
4
|
+
#include <stddef.h>
|
|
5
|
+
#include <stdint.h>
|
|
6
|
+
|
|
7
|
+
#ifdef __cplusplus
|
|
8
|
+
extern "C" {
|
|
9
|
+
#endif
|
|
10
|
+
|
|
11
|
+
#define GEZEL_LLAMA_ABI_VERSION 1u
|
|
12
|
+
|
|
13
|
+
typedef struct gezel_llama_engine gezel_llama_engine;
|
|
14
|
+
|
|
15
|
+
typedef enum gezel_llama_status {
|
|
16
|
+
GEZEL_LLAMA_OK = 0,
|
|
17
|
+
GEZEL_LLAMA_INVALID_ARGUMENT = 1,
|
|
18
|
+
GEZEL_LLAMA_BUSY = 2,
|
|
19
|
+
GEZEL_LLAMA_NOT_LOADED = 3,
|
|
20
|
+
GEZEL_LLAMA_LOAD_FAILED = 4,
|
|
21
|
+
GEZEL_LLAMA_UNSUPPORTED = 5,
|
|
22
|
+
GEZEL_LLAMA_CONTEXT_LIMIT = 6,
|
|
23
|
+
GEZEL_LLAMA_CANCELLED = 7,
|
|
24
|
+
GEZEL_LLAMA_TIMEOUT = 8,
|
|
25
|
+
GEZEL_LLAMA_INFERENCE_FAILED = 9,
|
|
26
|
+
GEZEL_LLAMA_RESOURCE_LIMIT = 10
|
|
27
|
+
} gezel_llama_status;
|
|
28
|
+
|
|
29
|
+
typedef enum gezel_llama_finish_reason {
|
|
30
|
+
GEZEL_LLAMA_FINISH_NONE = 0,
|
|
31
|
+
GEZEL_LLAMA_FINISH_STOP = 1,
|
|
32
|
+
GEZEL_LLAMA_FINISH_LENGTH = 2,
|
|
33
|
+
GEZEL_LLAMA_FINISH_CANCELLED = 3,
|
|
34
|
+
GEZEL_LLAMA_FINISH_TIMEOUT = 4,
|
|
35
|
+
GEZEL_LLAMA_FINISH_ERROR = 5,
|
|
36
|
+
/** Chat only: the reply ended with structured tool calls. */
|
|
37
|
+
GEZEL_LLAMA_FINISH_TOOL_CALLS = 6
|
|
38
|
+
} gezel_llama_finish_reason;
|
|
39
|
+
|
|
40
|
+
typedef struct gezel_llama_error {
|
|
41
|
+
int32_t code;
|
|
42
|
+
char message[256];
|
|
43
|
+
} gezel_llama_error;
|
|
44
|
+
|
|
45
|
+
typedef struct gezel_llama_load_options {
|
|
46
|
+
uint32_t struct_size;
|
|
47
|
+
uint32_t abi_version;
|
|
48
|
+
uint64_t request_id;
|
|
49
|
+
uint32_t context_tokens;
|
|
50
|
+
uint32_t batch_tokens;
|
|
51
|
+
uint32_t threads;
|
|
52
|
+
int32_t gpu_layers;
|
|
53
|
+
uint64_t max_model_bytes;
|
|
54
|
+
uint32_t timeout_ms;
|
|
55
|
+
} gezel_llama_load_options;
|
|
56
|
+
|
|
57
|
+
typedef struct gezel_llama_message {
|
|
58
|
+
/** NUL-terminated UTF-8. Allowed roles: system, user, assistant. */
|
|
59
|
+
const char * role;
|
|
60
|
+
const char * content;
|
|
61
|
+
} gezel_llama_message;
|
|
62
|
+
|
|
63
|
+
typedef struct gezel_llama_generation_options {
|
|
64
|
+
uint32_t struct_size;
|
|
65
|
+
uint32_t abi_version;
|
|
66
|
+
uint64_t request_id;
|
|
67
|
+
/** Ceiling on the reply, 1-4096. The reply gets whatever context the prompt
|
|
68
|
+
* leaves, up to this; GEZEL_LLAMA_CONTEXT_LIMIT only when the prompt leaves
|
|
69
|
+
* less than min(max_tokens, 256). */
|
|
70
|
+
uint32_t max_tokens;
|
|
71
|
+
/** Covers prompt processing and decoding; at most 600000. */
|
|
72
|
+
uint32_t timeout_ms;
|
|
73
|
+
uint32_t max_output_bytes;
|
|
74
|
+
/** Zero uses greedy decoding. Positive values use top-k/top-p/min-p/temperature. */
|
|
75
|
+
float temperature;
|
|
76
|
+
uint32_t seed;
|
|
77
|
+
/** Keep only the k likeliest tokens when sampling; 0 disables. */
|
|
78
|
+
uint32_t top_k;
|
|
79
|
+
/** Nucleus mass kept when sampling; 1 disables. */
|
|
80
|
+
float top_p;
|
|
81
|
+
/** Drop tokens below this fraction of the likeliest; 0 disables. */
|
|
82
|
+
float min_p;
|
|
83
|
+
/** Penalty on tokens repeated within `repeat_last_n`; 1 disables. Greedy too. */
|
|
84
|
+
float repeat_penalty;
|
|
85
|
+
uint32_t repeat_last_n;
|
|
86
|
+
} gezel_llama_generation_options;
|
|
87
|
+
|
|
88
|
+
typedef struct gezel_llama_result {
|
|
89
|
+
int32_t status;
|
|
90
|
+
int32_t finish_reason;
|
|
91
|
+
uint32_t prompt_tokens;
|
|
92
|
+
uint32_t generated_tokens;
|
|
93
|
+
uint32_t output_bytes;
|
|
94
|
+
} gezel_llama_result;
|
|
95
|
+
|
|
96
|
+
/** Called synchronously on the generation thread; bytes expire when it returns.
|
|
97
|
+
* Every chunk is valid UTF-8; concatenate chunks without adding separators.
|
|
98
|
+
* Return zero to continue, nonzero to cancel. Must not throw across this ABI.
|
|
99
|
+
* Reentrant load/generate/unload calls return BUSY. cancel is always allowed.
|
|
100
|
+
*/
|
|
101
|
+
typedef int32_t (*gezel_llama_chunk_callback)(const char * bytes, size_t length, void * user_data);
|
|
102
|
+
|
|
103
|
+
uint32_t gezel_llama_abi_version(void);
|
|
104
|
+
gezel_llama_load_options gezel_llama_default_load_options(void);
|
|
105
|
+
gezel_llama_generation_options gezel_llama_default_generation_options(void);
|
|
106
|
+
gezel_llama_engine * gezel_llama_create(void);
|
|
107
|
+
|
|
108
|
+
/** Requires exclusive lifetime ownership: wait for load/generate and any callers
|
|
109
|
+
* of cancel to finish first. No function may use the pointer after destroy.
|
|
110
|
+
*/
|
|
111
|
+
void gezel_llama_destroy(gezel_llama_engine * engine);
|
|
112
|
+
|
|
113
|
+
/** Blocking; call off the UI thread. Options are copied before inference starts.
|
|
114
|
+
* request_id must be in 1..INT64_MAX and unique for this engine's operation lifetime.
|
|
115
|
+
* Once admitted, a failed load leaves no model loaded. Validation/BUSY errors
|
|
116
|
+
* preserve the existing model. Input path must be a local regular file.
|
|
117
|
+
* The wrapper never downloads files, opens sockets, or executes scripts.
|
|
118
|
+
*/
|
|
119
|
+
int32_t gezel_llama_load(gezel_llama_engine * engine, const char * model_path,
|
|
120
|
+
const gezel_llama_load_options * options, gezel_llama_error * error);
|
|
121
|
+
int32_t gezel_llama_unload(gezel_llama_engine * engine, gezel_llama_error * error);
|
|
122
|
+
|
|
123
|
+
/** Bytes a load with the same options would allocate, as llama.cpp accounts
|
|
124
|
+
* them for this device's backends (repacked weight copies included). */
|
|
125
|
+
typedef struct gezel_llama_memory_estimate {
|
|
126
|
+
uint32_t struct_size;
|
|
127
|
+
uint32_t abi_version;
|
|
128
|
+
/** Weight buffers. */
|
|
129
|
+
uint64_t model_bytes;
|
|
130
|
+
/** The part of model_bytes a load serves straight from the memory-mapped
|
|
131
|
+
* file: clean pages the OS can reclaim and read again (mostly embedding
|
|
132
|
+
* tables, read a few rows at a time). The rest is allocated. */
|
|
133
|
+
uint64_t mapped_model_bytes;
|
|
134
|
+
/** KV cache and recurrent state at the requested context. */
|
|
135
|
+
uint64_t context_bytes;
|
|
136
|
+
/** Scratch buffers for one batch. */
|
|
137
|
+
uint64_t compute_bytes;
|
|
138
|
+
} gezel_llama_memory_estimate;
|
|
139
|
+
|
|
140
|
+
/** Blocking; reads GGUF metadata only and allocates no tensor data. It shares no
|
|
141
|
+
* state with a loaded model, so it may run beside one; hosts serialize it with
|
|
142
|
+
* loads anyway. Validates options and the file exactly as gezel_llama_load does. */
|
|
143
|
+
int32_t gezel_llama_estimate_memory(gezel_llama_engine * engine, const char * model_path,
|
|
144
|
+
const gezel_llama_load_options * options, gezel_llama_memory_estimate * estimate,
|
|
145
|
+
gezel_llama_error * error);
|
|
146
|
+
|
|
147
|
+
/** Blocking bounded text chat. Caller-owned strings/options stay valid until it
|
|
148
|
+
* returns. Every call starts a fresh KV context from the supplied full transcript.
|
|
149
|
+
* The last message must be user. The model must have a supported built-in chat
|
|
150
|
+
* template; missing/custom-only templates fail explicitly. No tool-call support.
|
|
151
|
+
* result receives partial token/byte counts even on cancellation or failure.
|
|
152
|
+
*/
|
|
153
|
+
int32_t gezel_llama_generate(gezel_llama_engine * engine,
|
|
154
|
+
const gezel_llama_message * messages, size_t message_count,
|
|
155
|
+
const gezel_llama_generation_options * options,
|
|
156
|
+
gezel_llama_chunk_callback on_chunk, void * user_data,
|
|
157
|
+
gezel_llama_result * result, gezel_llama_error * error);
|
|
158
|
+
|
|
159
|
+
/** What the engine's current operation is doing, for a host's status display. */
|
|
160
|
+
typedef enum gezel_llama_phase {
|
|
161
|
+
GEZEL_LLAMA_PHASE_IDLE = 0,
|
|
162
|
+
GEZEL_LLAMA_PHASE_LOADING = 1,
|
|
163
|
+
GEZEL_LLAMA_PHASE_PROMPT = 2,
|
|
164
|
+
GEZEL_LLAMA_PHASE_GENERATING = 3
|
|
165
|
+
} gezel_llama_phase;
|
|
166
|
+
|
|
167
|
+
typedef struct gezel_llama_progress {
|
|
168
|
+
uint32_t struct_size;
|
|
169
|
+
uint32_t abi_version;
|
|
170
|
+
/** A gezel_llama_phase value. */
|
|
171
|
+
uint32_t phase;
|
|
172
|
+
/** Model loading fraction in [0, 1]; meaningful while loading. */
|
|
173
|
+
float load_fraction;
|
|
174
|
+
/** Tokens in the formatted prompt, and how many of them are in memory so
|
|
175
|
+
* far — reused from the previous request or processed by this one. */
|
|
176
|
+
uint32_t prompt_tokens;
|
|
177
|
+
uint32_t processed_tokens;
|
|
178
|
+
uint32_t reused_tokens;
|
|
179
|
+
uint32_t generated_tokens;
|
|
180
|
+
} gezel_llama_progress;
|
|
181
|
+
|
|
182
|
+
/** Thread-safe snapshot of the current operation's counters; hosts poll it from
|
|
183
|
+
* their cancellation timer while load or generate runs. Never blocks. Requires
|
|
184
|
+
* struct_size and abi_version; returns INVALID_ARGUMENT otherwise. Added after
|
|
185
|
+
* ABI 1 shipped; hosts that must run against older libraries check the symbol. */
|
|
186
|
+
int32_t gezel_llama_get_progress(gezel_llama_engine * engine, gezel_llama_progress * progress);
|
|
187
|
+
|
|
188
|
+
/*
|
|
189
|
+
* OpenAI-shaped chat, the way desktop's llama-server serves it with --jinja.
|
|
190
|
+
*
|
|
191
|
+
* The request is the `/v1/chat/completions` body desktop sends llama-server:
|
|
192
|
+
* messages (assistant `tool_calls`, `tool` results), tools, tool_choice,
|
|
193
|
+
* sampling fields, max_tokens, stop, chat_template_kwargs, reasoning budget.
|
|
194
|
+
* llama.cpp's own chat layer renders it through the model's Jinja template,
|
|
195
|
+
* constrains tool calls with the template's lazy grammar, and parses the
|
|
196
|
+
* model's native call syntax back into structured tool calls. Unknown keys are
|
|
197
|
+
* ignored, as llama-server ignores them.
|
|
198
|
+
*
|
|
199
|
+
* on_event receives each object llama-server would send as an SSE `data:`
|
|
200
|
+
* payload, in order: `chat.completion.chunk` deltas (content,
|
|
201
|
+
* reasoning_content, tool_calls), then the finish chunk (`finish_reason`
|
|
202
|
+
* stop, length or tool_calls) and a usage chunk with timings. A request that
|
|
203
|
+
* fails instead delivers one `{"error":{code,message,type,...}}` object shaped
|
|
204
|
+
* like llama-server's error body; the call returns a failure status either way.
|
|
205
|
+
* A cancelled or timed-out request ends without a finish chunk, like a dropped
|
|
206
|
+
* connection. Returning non-zero from on_event cancels the request.
|
|
207
|
+
*/
|
|
208
|
+
typedef int32_t (*gezel_llama_json_callback)(const char * json, size_t length, void * user_data);
|
|
209
|
+
|
|
210
|
+
typedef struct gezel_llama_chat_options {
|
|
211
|
+
uint32_t struct_size;
|
|
212
|
+
uint32_t abi_version;
|
|
213
|
+
uint64_t request_id;
|
|
214
|
+
/** Covers prompt processing and decoding; at most 600000. */
|
|
215
|
+
uint32_t timeout_ms;
|
|
216
|
+
/** Ceiling on the text generated for one reply. */
|
|
217
|
+
uint32_t max_output_bytes;
|
|
218
|
+
/** GEZEL_LLAMA_CHAT_RENDER_ONLY: deliver one object describing the rendered
|
|
219
|
+
* prompt, grammar and triggers instead of generating (tests, diagnostics). */
|
|
220
|
+
uint32_t flags;
|
|
221
|
+
} gezel_llama_chat_options;
|
|
222
|
+
|
|
223
|
+
#define GEZEL_LLAMA_CHAT_RENDER_ONLY 1u
|
|
224
|
+
|
|
225
|
+
gezel_llama_chat_options gezel_llama_default_chat_options(void);
|
|
226
|
+
|
|
227
|
+
/** Settings llama-server takes from its launch flags, as JSON:
|
|
228
|
+
* {chat_template?, reasoning_format?, reasoning_budget?, reasoning_budget_message?,
|
|
229
|
+
* enable_thinking?, prefill_assistant?, chat_template_kwargs?}. Applies to the
|
|
230
|
+
* next chat request; a loaded model keeps them until the next call. */
|
|
231
|
+
int32_t gezel_llama_configure_chat(gezel_llama_engine * engine, const char * config_json, size_t length,
|
|
232
|
+
gezel_llama_error * error);
|
|
233
|
+
|
|
234
|
+
/** One chat completion. Blocks until it ends; the engine must be loaded. */
|
|
235
|
+
int32_t gezel_llama_chat(gezel_llama_engine * engine, const char * request_json, size_t length,
|
|
236
|
+
const gezel_llama_chat_options * options, gezel_llama_json_callback on_event, void * user_data,
|
|
237
|
+
gezel_llama_result * result, gezel_llama_error * error);
|
|
238
|
+
|
|
239
|
+
/*
|
|
240
|
+
* Describe one picture with the loaded model's vision projector.
|
|
241
|
+
*
|
|
242
|
+
* Added after ABI 1 shipped; hosts that must run against older libraries
|
|
243
|
+
* check the symbol. The engine must hold the text model the projector was
|
|
244
|
+
* made for (gezel_llama_load); the projector (an mmproj GGUF) is opened on
|
|
245
|
+
* first use and kept with that model until it unloads or a different
|
|
246
|
+
* projector path is given. A call starts from empty memory and leaves it
|
|
247
|
+
* empty, so the next chat request reads its prompt from the start.
|
|
248
|
+
*/
|
|
249
|
+
typedef struct gezel_llama_image_options {
|
|
250
|
+
uint32_t struct_size;
|
|
251
|
+
uint32_t abi_version;
|
|
252
|
+
uint64_t request_id;
|
|
253
|
+
/** Covers projector loading, image encoding, and decoding; at most 600000. */
|
|
254
|
+
uint32_t timeout_ms;
|
|
255
|
+
/** Ceiling on the description, 1-1024 tokens. */
|
|
256
|
+
uint32_t max_tokens;
|
|
257
|
+
uint32_t max_output_bytes;
|
|
258
|
+
/** Zero uses greedy decoding. */
|
|
259
|
+
float temperature;
|
|
260
|
+
uint32_t seed;
|
|
261
|
+
/** Cap on the picture's tokens for dynamic-resolution projectors; 0 keeps the projector's own. */
|
|
262
|
+
uint32_t image_max_tokens;
|
|
263
|
+
/** Projector file size limit. */
|
|
264
|
+
uint64_t max_projector_bytes;
|
|
265
|
+
} gezel_llama_image_options;
|
|
266
|
+
|
|
267
|
+
gezel_llama_image_options gezel_llama_default_image_options(void);
|
|
268
|
+
|
|
269
|
+
/** Blocking. rgb holds width*height*3 bytes, row-major, no padding; both sides
|
|
270
|
+
* 1-4096. system_prompt may be NULL; user_prompt is the instruction that
|
|
271
|
+
* follows the picture. on_chunk receives the description as it is generated,
|
|
272
|
+
* under the same rules as gezel_llama_generate's callback. */
|
|
273
|
+
int32_t gezel_llama_describe_image(gezel_llama_engine * engine, const char * projector_path,
|
|
274
|
+
const uint8_t * rgb, uint32_t width, uint32_t height,
|
|
275
|
+
const char * system_prompt, const char * user_prompt,
|
|
276
|
+
const gezel_llama_image_options * options,
|
|
277
|
+
gezel_llama_chunk_callback on_chunk, void * user_data,
|
|
278
|
+
gezel_llama_result * result, gezel_llama_error * error);
|
|
279
|
+
|
|
280
|
+
/** Thread-safe cooperative cancellation of the matching CURRENT request only.
|
|
281
|
+
* A late cancellation with an older request_id cannot cancel a later request.
|
|
282
|
+
* Native loading/Metal kernels can delay observation until their next safe point.
|
|
283
|
+
* Cancelling before an operation starts has no effect; schedule on the host queue.
|
|
284
|
+
*/
|
|
285
|
+
void gezel_llama_cancel(gezel_llama_engine * engine, uint64_t request_id);
|
|
286
|
+
|
|
287
|
+
#ifdef __cplusplus
|
|
288
|
+
}
|
|
289
|
+
#endif
|
|
290
|
+
#endif
|
|
Binary file
|
|
@@ -0,0 +1,290 @@
|
|
|
1
|
+
#ifndef GEZEL_LLAMA_H
|
|
2
|
+
#define GEZEL_LLAMA_H
|
|
3
|
+
|
|
4
|
+
#include <stddef.h>
|
|
5
|
+
#include <stdint.h>
|
|
6
|
+
|
|
7
|
+
#ifdef __cplusplus
|
|
8
|
+
extern "C" {
|
|
9
|
+
#endif
|
|
10
|
+
|
|
11
|
+
#define GEZEL_LLAMA_ABI_VERSION 1u
|
|
12
|
+
|
|
13
|
+
typedef struct gezel_llama_engine gezel_llama_engine;
|
|
14
|
+
|
|
15
|
+
typedef enum gezel_llama_status {
|
|
16
|
+
GEZEL_LLAMA_OK = 0,
|
|
17
|
+
GEZEL_LLAMA_INVALID_ARGUMENT = 1,
|
|
18
|
+
GEZEL_LLAMA_BUSY = 2,
|
|
19
|
+
GEZEL_LLAMA_NOT_LOADED = 3,
|
|
20
|
+
GEZEL_LLAMA_LOAD_FAILED = 4,
|
|
21
|
+
GEZEL_LLAMA_UNSUPPORTED = 5,
|
|
22
|
+
GEZEL_LLAMA_CONTEXT_LIMIT = 6,
|
|
23
|
+
GEZEL_LLAMA_CANCELLED = 7,
|
|
24
|
+
GEZEL_LLAMA_TIMEOUT = 8,
|
|
25
|
+
GEZEL_LLAMA_INFERENCE_FAILED = 9,
|
|
26
|
+
GEZEL_LLAMA_RESOURCE_LIMIT = 10
|
|
27
|
+
} gezel_llama_status;
|
|
28
|
+
|
|
29
|
+
typedef enum gezel_llama_finish_reason {
|
|
30
|
+
GEZEL_LLAMA_FINISH_NONE = 0,
|
|
31
|
+
GEZEL_LLAMA_FINISH_STOP = 1,
|
|
32
|
+
GEZEL_LLAMA_FINISH_LENGTH = 2,
|
|
33
|
+
GEZEL_LLAMA_FINISH_CANCELLED = 3,
|
|
34
|
+
GEZEL_LLAMA_FINISH_TIMEOUT = 4,
|
|
35
|
+
GEZEL_LLAMA_FINISH_ERROR = 5,
|
|
36
|
+
/** Chat only: the reply ended with structured tool calls. */
|
|
37
|
+
GEZEL_LLAMA_FINISH_TOOL_CALLS = 6
|
|
38
|
+
} gezel_llama_finish_reason;
|
|
39
|
+
|
|
40
|
+
typedef struct gezel_llama_error {
|
|
41
|
+
int32_t code;
|
|
42
|
+
char message[256];
|
|
43
|
+
} gezel_llama_error;
|
|
44
|
+
|
|
45
|
+
typedef struct gezel_llama_load_options {
|
|
46
|
+
uint32_t struct_size;
|
|
47
|
+
uint32_t abi_version;
|
|
48
|
+
uint64_t request_id;
|
|
49
|
+
uint32_t context_tokens;
|
|
50
|
+
uint32_t batch_tokens;
|
|
51
|
+
uint32_t threads;
|
|
52
|
+
int32_t gpu_layers;
|
|
53
|
+
uint64_t max_model_bytes;
|
|
54
|
+
uint32_t timeout_ms;
|
|
55
|
+
} gezel_llama_load_options;
|
|
56
|
+
|
|
57
|
+
typedef struct gezel_llama_message {
|
|
58
|
+
/** NUL-terminated UTF-8. Allowed roles: system, user, assistant. */
|
|
59
|
+
const char * role;
|
|
60
|
+
const char * content;
|
|
61
|
+
} gezel_llama_message;
|
|
62
|
+
|
|
63
|
+
typedef struct gezel_llama_generation_options {
|
|
64
|
+
uint32_t struct_size;
|
|
65
|
+
uint32_t abi_version;
|
|
66
|
+
uint64_t request_id;
|
|
67
|
+
/** Ceiling on the reply, 1-4096. The reply gets whatever context the prompt
|
|
68
|
+
* leaves, up to this; GEZEL_LLAMA_CONTEXT_LIMIT only when the prompt leaves
|
|
69
|
+
* less than min(max_tokens, 256). */
|
|
70
|
+
uint32_t max_tokens;
|
|
71
|
+
/** Covers prompt processing and decoding; at most 600000. */
|
|
72
|
+
uint32_t timeout_ms;
|
|
73
|
+
uint32_t max_output_bytes;
|
|
74
|
+
/** Zero uses greedy decoding. Positive values use top-k/top-p/min-p/temperature. */
|
|
75
|
+
float temperature;
|
|
76
|
+
uint32_t seed;
|
|
77
|
+
/** Keep only the k likeliest tokens when sampling; 0 disables. */
|
|
78
|
+
uint32_t top_k;
|
|
79
|
+
/** Nucleus mass kept when sampling; 1 disables. */
|
|
80
|
+
float top_p;
|
|
81
|
+
/** Drop tokens below this fraction of the likeliest; 0 disables. */
|
|
82
|
+
float min_p;
|
|
83
|
+
/** Penalty on tokens repeated within `repeat_last_n`; 1 disables. Greedy too. */
|
|
84
|
+
float repeat_penalty;
|
|
85
|
+
uint32_t repeat_last_n;
|
|
86
|
+
} gezel_llama_generation_options;
|
|
87
|
+
|
|
88
|
+
typedef struct gezel_llama_result {
|
|
89
|
+
int32_t status;
|
|
90
|
+
int32_t finish_reason;
|
|
91
|
+
uint32_t prompt_tokens;
|
|
92
|
+
uint32_t generated_tokens;
|
|
93
|
+
uint32_t output_bytes;
|
|
94
|
+
} gezel_llama_result;
|
|
95
|
+
|
|
96
|
+
/** Called synchronously on the generation thread; bytes expire when it returns.
|
|
97
|
+
* Every chunk is valid UTF-8; concatenate chunks without adding separators.
|
|
98
|
+
* Return zero to continue, nonzero to cancel. Must not throw across this ABI.
|
|
99
|
+
* Reentrant load/generate/unload calls return BUSY. cancel is always allowed.
|
|
100
|
+
*/
|
|
101
|
+
typedef int32_t (*gezel_llama_chunk_callback)(const char * bytes, size_t length, void * user_data);
|
|
102
|
+
|
|
103
|
+
uint32_t gezel_llama_abi_version(void);
|
|
104
|
+
gezel_llama_load_options gezel_llama_default_load_options(void);
|
|
105
|
+
gezel_llama_generation_options gezel_llama_default_generation_options(void);
|
|
106
|
+
gezel_llama_engine * gezel_llama_create(void);
|
|
107
|
+
|
|
108
|
+
/** Requires exclusive lifetime ownership: wait for load/generate and any callers
|
|
109
|
+
* of cancel to finish first. No function may use the pointer after destroy.
|
|
110
|
+
*/
|
|
111
|
+
void gezel_llama_destroy(gezel_llama_engine * engine);
|
|
112
|
+
|
|
113
|
+
/** Blocking; call off the UI thread. Options are copied before inference starts.
|
|
114
|
+
* request_id must be in 1..INT64_MAX and unique for this engine's operation lifetime.
|
|
115
|
+
* Once admitted, a failed load leaves no model loaded. Validation/BUSY errors
|
|
116
|
+
* preserve the existing model. Input path must be a local regular file.
|
|
117
|
+
* The wrapper never downloads files, opens sockets, or executes scripts.
|
|
118
|
+
*/
|
|
119
|
+
int32_t gezel_llama_load(gezel_llama_engine * engine, const char * model_path,
|
|
120
|
+
const gezel_llama_load_options * options, gezel_llama_error * error);
|
|
121
|
+
int32_t gezel_llama_unload(gezel_llama_engine * engine, gezel_llama_error * error);
|
|
122
|
+
|
|
123
|
+
/** Bytes a load with the same options would allocate, as llama.cpp accounts
|
|
124
|
+
* them for this device's backends (repacked weight copies included). */
|
|
125
|
+
typedef struct gezel_llama_memory_estimate {
|
|
126
|
+
uint32_t struct_size;
|
|
127
|
+
uint32_t abi_version;
|
|
128
|
+
/** Weight buffers. */
|
|
129
|
+
uint64_t model_bytes;
|
|
130
|
+
/** The part of model_bytes a load serves straight from the memory-mapped
|
|
131
|
+
* file: clean pages the OS can reclaim and read again (mostly embedding
|
|
132
|
+
* tables, read a few rows at a time). The rest is allocated. */
|
|
133
|
+
uint64_t mapped_model_bytes;
|
|
134
|
+
/** KV cache and recurrent state at the requested context. */
|
|
135
|
+
uint64_t context_bytes;
|
|
136
|
+
/** Scratch buffers for one batch. */
|
|
137
|
+
uint64_t compute_bytes;
|
|
138
|
+
} gezel_llama_memory_estimate;
|
|
139
|
+
|
|
140
|
+
/** Blocking; reads GGUF metadata only and allocates no tensor data. It shares no
|
|
141
|
+
* state with a loaded model, so it may run beside one; hosts serialize it with
|
|
142
|
+
* loads anyway. Validates options and the file exactly as gezel_llama_load does. */
|
|
143
|
+
int32_t gezel_llama_estimate_memory(gezel_llama_engine * engine, const char * model_path,
|
|
144
|
+
const gezel_llama_load_options * options, gezel_llama_memory_estimate * estimate,
|
|
145
|
+
gezel_llama_error * error);
|
|
146
|
+
|
|
147
|
+
/** Blocking bounded text chat. Caller-owned strings/options stay valid until it
|
|
148
|
+
* returns. Every call starts a fresh KV context from the supplied full transcript.
|
|
149
|
+
* The last message must be user. The model must have a supported built-in chat
|
|
150
|
+
* template; missing/custom-only templates fail explicitly. No tool-call support.
|
|
151
|
+
* result receives partial token/byte counts even on cancellation or failure.
|
|
152
|
+
*/
|
|
153
|
+
int32_t gezel_llama_generate(gezel_llama_engine * engine,
|
|
154
|
+
const gezel_llama_message * messages, size_t message_count,
|
|
155
|
+
const gezel_llama_generation_options * options,
|
|
156
|
+
gezel_llama_chunk_callback on_chunk, void * user_data,
|
|
157
|
+
gezel_llama_result * result, gezel_llama_error * error);
|
|
158
|
+
|
|
159
|
+
/** What the engine's current operation is doing, for a host's status display. */
|
|
160
|
+
typedef enum gezel_llama_phase {
|
|
161
|
+
GEZEL_LLAMA_PHASE_IDLE = 0,
|
|
162
|
+
GEZEL_LLAMA_PHASE_LOADING = 1,
|
|
163
|
+
GEZEL_LLAMA_PHASE_PROMPT = 2,
|
|
164
|
+
GEZEL_LLAMA_PHASE_GENERATING = 3
|
|
165
|
+
} gezel_llama_phase;
|
|
166
|
+
|
|
167
|
+
typedef struct gezel_llama_progress {
|
|
168
|
+
uint32_t struct_size;
|
|
169
|
+
uint32_t abi_version;
|
|
170
|
+
/** A gezel_llama_phase value. */
|
|
171
|
+
uint32_t phase;
|
|
172
|
+
/** Model loading fraction in [0, 1]; meaningful while loading. */
|
|
173
|
+
float load_fraction;
|
|
174
|
+
/** Tokens in the formatted prompt, and how many of them are in memory so
|
|
175
|
+
* far — reused from the previous request or processed by this one. */
|
|
176
|
+
uint32_t prompt_tokens;
|
|
177
|
+
uint32_t processed_tokens;
|
|
178
|
+
uint32_t reused_tokens;
|
|
179
|
+
uint32_t generated_tokens;
|
|
180
|
+
} gezel_llama_progress;
|
|
181
|
+
|
|
182
|
+
/** Thread-safe snapshot of the current operation's counters; hosts poll it from
|
|
183
|
+
* their cancellation timer while load or generate runs. Never blocks. Requires
|
|
184
|
+
* struct_size and abi_version; returns INVALID_ARGUMENT otherwise. Added after
|
|
185
|
+
* ABI 1 shipped; hosts that must run against older libraries check the symbol. */
|
|
186
|
+
int32_t gezel_llama_get_progress(gezel_llama_engine * engine, gezel_llama_progress * progress);
|
|
187
|
+
|
|
188
|
+
/*
|
|
189
|
+
* OpenAI-shaped chat, the way desktop's llama-server serves it with --jinja.
|
|
190
|
+
*
|
|
191
|
+
* The request is the `/v1/chat/completions` body desktop sends llama-server:
|
|
192
|
+
* messages (assistant `tool_calls`, `tool` results), tools, tool_choice,
|
|
193
|
+
* sampling fields, max_tokens, stop, chat_template_kwargs, reasoning budget.
|
|
194
|
+
* llama.cpp's own chat layer renders it through the model's Jinja template,
|
|
195
|
+
* constrains tool calls with the template's lazy grammar, and parses the
|
|
196
|
+
* model's native call syntax back into structured tool calls. Unknown keys are
|
|
197
|
+
* ignored, as llama-server ignores them.
|
|
198
|
+
*
|
|
199
|
+
* on_event receives each object llama-server would send as an SSE `data:`
|
|
200
|
+
* payload, in order: `chat.completion.chunk` deltas (content,
|
|
201
|
+
* reasoning_content, tool_calls), then the finish chunk (`finish_reason`
|
|
202
|
+
* stop, length or tool_calls) and a usage chunk with timings. A request that
|
|
203
|
+
* fails instead delivers one `{"error":{code,message,type,...}}` object shaped
|
|
204
|
+
* like llama-server's error body; the call returns a failure status either way.
|
|
205
|
+
* A cancelled or timed-out request ends without a finish chunk, like a dropped
|
|
206
|
+
* connection. Returning non-zero from on_event cancels the request.
|
|
207
|
+
*/
|
|
208
|
+
typedef int32_t (*gezel_llama_json_callback)(const char * json, size_t length, void * user_data);
|
|
209
|
+
|
|
210
|
+
typedef struct gezel_llama_chat_options {
|
|
211
|
+
uint32_t struct_size;
|
|
212
|
+
uint32_t abi_version;
|
|
213
|
+
uint64_t request_id;
|
|
214
|
+
/** Covers prompt processing and decoding; at most 600000. */
|
|
215
|
+
uint32_t timeout_ms;
|
|
216
|
+
/** Ceiling on the text generated for one reply. */
|
|
217
|
+
uint32_t max_output_bytes;
|
|
218
|
+
/** GEZEL_LLAMA_CHAT_RENDER_ONLY: deliver one object describing the rendered
|
|
219
|
+
* prompt, grammar and triggers instead of generating (tests, diagnostics). */
|
|
220
|
+
uint32_t flags;
|
|
221
|
+
} gezel_llama_chat_options;
|
|
222
|
+
|
|
223
|
+
#define GEZEL_LLAMA_CHAT_RENDER_ONLY 1u
|
|
224
|
+
|
|
225
|
+
gezel_llama_chat_options gezel_llama_default_chat_options(void);
|
|
226
|
+
|
|
227
|
+
/** Settings llama-server takes from its launch flags, as JSON:
|
|
228
|
+
* {chat_template?, reasoning_format?, reasoning_budget?, reasoning_budget_message?,
|
|
229
|
+
* enable_thinking?, prefill_assistant?, chat_template_kwargs?}. Applies to the
|
|
230
|
+
* next chat request; a loaded model keeps them until the next call. */
|
|
231
|
+
int32_t gezel_llama_configure_chat(gezel_llama_engine * engine, const char * config_json, size_t length,
|
|
232
|
+
gezel_llama_error * error);
|
|
233
|
+
|
|
234
|
+
/** One chat completion. Blocks until it ends; the engine must be loaded. */
|
|
235
|
+
int32_t gezel_llama_chat(gezel_llama_engine * engine, const char * request_json, size_t length,
|
|
236
|
+
const gezel_llama_chat_options * options, gezel_llama_json_callback on_event, void * user_data,
|
|
237
|
+
gezel_llama_result * result, gezel_llama_error * error);
|
|
238
|
+
|
|
239
|
+
/*
|
|
240
|
+
* Describe one picture with the loaded model's vision projector.
|
|
241
|
+
*
|
|
242
|
+
* Added after ABI 1 shipped; hosts that must run against older libraries
|
|
243
|
+
* check the symbol. The engine must hold the text model the projector was
|
|
244
|
+
* made for (gezel_llama_load); the projector (an mmproj GGUF) is opened on
|
|
245
|
+
* first use and kept with that model until it unloads or a different
|
|
246
|
+
* projector path is given. A call starts from empty memory and leaves it
|
|
247
|
+
* empty, so the next chat request reads its prompt from the start.
|
|
248
|
+
*/
|
|
249
|
+
typedef struct gezel_llama_image_options {
|
|
250
|
+
uint32_t struct_size;
|
|
251
|
+
uint32_t abi_version;
|
|
252
|
+
uint64_t request_id;
|
|
253
|
+
/** Covers projector loading, image encoding, and decoding; at most 600000. */
|
|
254
|
+
uint32_t timeout_ms;
|
|
255
|
+
/** Ceiling on the description, 1-1024 tokens. */
|
|
256
|
+
uint32_t max_tokens;
|
|
257
|
+
uint32_t max_output_bytes;
|
|
258
|
+
/** Zero uses greedy decoding. */
|
|
259
|
+
float temperature;
|
|
260
|
+
uint32_t seed;
|
|
261
|
+
/** Cap on the picture's tokens for dynamic-resolution projectors; 0 keeps the projector's own. */
|
|
262
|
+
uint32_t image_max_tokens;
|
|
263
|
+
/** Projector file size limit. */
|
|
264
|
+
uint64_t max_projector_bytes;
|
|
265
|
+
} gezel_llama_image_options;
|
|
266
|
+
|
|
267
|
+
gezel_llama_image_options gezel_llama_default_image_options(void);
|
|
268
|
+
|
|
269
|
+
/** Blocking. rgb holds width*height*3 bytes, row-major, no padding; both sides
|
|
270
|
+
* 1-4096. system_prompt may be NULL; user_prompt is the instruction that
|
|
271
|
+
* follows the picture. on_chunk receives the description as it is generated,
|
|
272
|
+
* under the same rules as gezel_llama_generate's callback. */
|
|
273
|
+
int32_t gezel_llama_describe_image(gezel_llama_engine * engine, const char * projector_path,
|
|
274
|
+
const uint8_t * rgb, uint32_t width, uint32_t height,
|
|
275
|
+
const char * system_prompt, const char * user_prompt,
|
|
276
|
+
const gezel_llama_image_options * options,
|
|
277
|
+
gezel_llama_chunk_callback on_chunk, void * user_data,
|
|
278
|
+
gezel_llama_result * result, gezel_llama_error * error);
|
|
279
|
+
|
|
280
|
+
/** Thread-safe cooperative cancellation of the matching CURRENT request only.
|
|
281
|
+
* A late cancellation with an older request_id cannot cancel a later request.
|
|
282
|
+
* Native loading/Metal kernels can delay observation until their next safe point.
|
|
283
|
+
* Cancelling before an operation starts has no effect; schedule on the host queue.
|
|
284
|
+
*/
|
|
285
|
+
void gezel_llama_cancel(gezel_llama_engine * engine, uint64_t request_id);
|
|
286
|
+
|
|
287
|
+
#ifdef __cplusplus
|
|
288
|
+
}
|
|
289
|
+
#endif
|
|
290
|
+
#endif
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Bendyline
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2023-2026 The ggml authors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2023-2026 The ggml authors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|