@bendyline/gezel-inference 0.1.0 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +90 -8
- package/dist/cache/adapter.d.ts +7 -0
- package/dist/cache/controller.d.ts +12 -0
- package/dist/cache/controller.js +24 -0
- package/dist/catalog/chat-models/bt/btl4-35b-q4/manifest.json +1 -0
- package/dist/catalog/chat-models/gl/glm-5.2-754b-q2/manifest.json +1 -0
- package/dist/catalog/chat-models/index.json +1 -1
- package/dist/catalog/chat-models/lf/lfm2.5-24b-a2b-q4/manifest.json +122 -0
- package/dist/catalog/chat-models/lf/lfm2.5-24b-a2b-q4/versions/1.0.0/manifest.json +15 -0
- package/dist/catalog/chat-models/mi/minicpm-v4.7-35b-a3b-q4/manifest.json +185 -0
- package/dist/catalog/chat-models/mi/minicpm-v4.7-35b-a3b-q4/versions/1.0.0/manifest.json +94 -0
- package/dist/catalog/chat-models/mi/minicpm5-1b-q4/manifest.json +1 -0
- package/dist/catalog/chat-models/mi/minicpm5-2b-q4/manifest.json +1 -0
- package/dist/catalog/chat-models/qw/qwen3.5-0.8b-q4/manifest.json +1 -0
- package/dist/catalog/chat-models/raw-index.json +1 -1
- package/dist/catalog/provenance.json +1 -1
- package/dist/engines/electron-native-reuse.d.ts +9 -0
- package/dist/engines/electron-native-reuse.js +146 -45
- package/dist/engines/native-manifest.d.ts +8 -0
- package/dist/engines/native-manifest.js +30 -0
- package/dist/http/openai-compat/apple-model.d.ts +2 -0
- package/dist/http/openai-compat/apple-model.js +21 -1
- package/dist/http/openai-compat/chat-target.d.ts +12 -0
- package/dist/http/openai-compat/chat-target.js +27 -1
- package/dist/http/routes/engines.js +9 -3
- package/dist/http/routes/v1-chat.js +21 -8
- package/dist/http/routes/v1-models.d.ts +6 -0
- package/dist/http/routes/v1-models.js +133 -28
- package/dist/http/routes/v1-responses.js +9 -2
- package/dist/in-flight-requests.d.ts +19 -0
- package/dist/in-flight-requests.js +34 -0
- package/dist/inference-service.js +3 -3
- package/dist/providers/llama-cpp/build-provider.js +33 -8
- package/dist/providers/llama-cpp/cache-adapter.d.ts +12 -0
- package/dist/providers/llama-cpp/cache-adapter.js +34 -0
- package/dist/providers/llama-cpp/gguf-metadata-async.js +7 -0
- package/dist/providers/mlx/provider.js +11 -7
- package/dist/providers/mlx/python/batch_processors.py +26 -0
- package/dist/providers/mlx/python/gezel_mlx_server.py +88 -20
- package/dist/providers/mlx/python/response_format.py +374 -0
- package/dist/providers/mlx/python/spec_decode.py +5 -0
- package/dist/providers/mlx/python/think_budget.py +35 -1
- package/dist/providers/mlx/python/tool_grammar.py +62 -3
- package/dist/providers/mlx/reasoning-stream.d.ts +20 -2
- package/dist/providers/mlx/reasoning-stream.js +77 -16
- package/dist/providers/mlx/runtime-diagnostics.d.ts +16 -0
- package/dist/providers/mlx/runtime-diagnostics.js +29 -2
- package/dist/providers/native/capacity-broker.d.ts +35 -0
- package/dist/providers/native/capacity-broker.js +69 -4
- package/dist/providers/native/local-engine-runtime.js +7 -2
- package/dist/remotes/identity.d.ts +1 -0
- package/dist/remotes/identity.js +6 -0
- package/dist/remotes/pinned-fetch.d.ts +2 -0
- package/dist/remotes/pinned-fetch.js +30 -1
- package/dist/remotes/types.d.ts +14 -0
- package/dist/runtime-lock.js +1 -1
- package/dist/system/memory.js +11 -1
- package/dist/utils/service-worker-entry.d.ts +4 -0
- package/dist/utils/service-worker-entry.js +4 -0
- package/package.json +15 -11
package/dist/remotes/types.d.ts
CHANGED
|
@@ -1,3 +1,11 @@
|
|
|
1
|
+
export interface RemoteKnownModel {
|
|
2
|
+
id: string;
|
|
3
|
+
name: string;
|
|
4
|
+
contextWindow?: number;
|
|
5
|
+
supportsTools?: boolean;
|
|
6
|
+
supportsReasoning?: boolean;
|
|
7
|
+
parameterSize?: string;
|
|
8
|
+
}
|
|
1
9
|
export interface PairedRemote {
|
|
2
10
|
/** Stable id A assigns to this server = its full identity fingerprint. */
|
|
3
11
|
remoteId: string;
|
|
@@ -29,6 +37,12 @@ export interface PairedRemote {
|
|
|
29
37
|
pairedAt: number;
|
|
30
38
|
/** Last time A successfully reached this server (in-memory unless flushed). */
|
|
31
39
|
lastSeenAt?: number;
|
|
40
|
+
/**
|
|
41
|
+
* The server's chat models the last time A listed them, already namespaced
|
|
42
|
+
* for A. Kept so a picker can show them, marked unavailable, while the
|
|
43
|
+
* server is asleep or A is off its network.
|
|
44
|
+
*/
|
|
45
|
+
knownModels?: RemoteKnownModel[];
|
|
32
46
|
/**
|
|
33
47
|
* Runtime-managed connections are never persisted and cannot be removed by
|
|
34
48
|
* the ordinary pairing UI. Today the only managed peer is this computer's
|
package/dist/runtime-lock.js
CHANGED
|
@@ -9,7 +9,7 @@ export class SingleInstanceError extends Error {
|
|
|
9
9
|
lockPath;
|
|
10
10
|
constructor(holderPid, lockPath) {
|
|
11
11
|
super(holderPid > 0
|
|
12
|
-
? `Another gezel daemon is already running for this home (pid ${holderPid}). Stop it first with \`gezel stop
|
|
12
|
+
? `Another gezel daemon is already running for this home (pid ${holderPid}). Stop it first with \`gezel stop\`, or use a different GEZEL_HOME.`
|
|
13
13
|
: `Could not acquire the single-instance lock at ${lockPath}.`);
|
|
14
14
|
this.holderPid = holderPid;
|
|
15
15
|
this.lockPath = lockPath;
|
package/dist/system/memory.js
CHANGED
|
@@ -487,12 +487,22 @@ function isKnownUnifiedNvidiaDevice(name) {
|
|
|
487
487
|
const normalized = name.toLowerCase();
|
|
488
488
|
return /\bgb10\b/.test(normalized) || normalized.includes('dgx spark');
|
|
489
489
|
}
|
|
490
|
+
/**
|
|
491
|
+
* A hybrid-graphics laptop powers its discrete GPU down when idle, and the
|
|
492
|
+
* first `nvidia-smi` after that waits for it to wake: 2.25–2.36 s on an RTX
|
|
493
|
+
* 5070 Ti laptop (2026-10-15), against ~0.1 s warm. The old 2 s limit read
|
|
494
|
+
* that as "no NVIDIA GPU", so the machine was sized as CPU-only for the next
|
|
495
|
+
* minute and offered a 2.4 GB model with a 12 GB card idle beside it. Startup
|
|
496
|
+
* never awaits this probe, so the margin only costs a caller that asks for the
|
|
497
|
+
* profile while the driver is truly hung.
|
|
498
|
+
*/
|
|
499
|
+
const NVIDIA_MEMORY_PROBE_TIMEOUT_MS = 10_000;
|
|
490
500
|
async function probeNvidiaMemory() {
|
|
491
501
|
try {
|
|
492
502
|
const { stdout } = await exec('nvidia-smi', ['--query-gpu=name,memory.total', '--format=csv,noheader,nounits'],
|
|
493
503
|
// Keep this short-lived console probe invisible without detaching it
|
|
494
504
|
// from the daemon that owns and awaits it.
|
|
495
|
-
{ timeout:
|
|
505
|
+
{ timeout: NVIDIA_MEMORY_PROBE_TIMEOUT_MS, ...windowsHeadlessSpawnOptions() });
|
|
496
506
|
return parseNvidiaMemoryProbe(stdout);
|
|
497
507
|
}
|
|
498
508
|
catch {
|
|
@@ -16,6 +16,10 @@ export declare const SERVICE_WORKER_ENTRIES: {
|
|
|
16
16
|
readonly 'static-index': {
|
|
17
17
|
readonly built: "index-store/static-index-worker.js";
|
|
18
18
|
};
|
|
19
|
+
readonly 'dashboard-composite': {
|
|
20
|
+
readonly built: "ambient/dashboard-composite-worker.js";
|
|
21
|
+
readonly source: "ambient/dashboard-composite-worker.ts";
|
|
22
|
+
};
|
|
19
23
|
readonly 'document-convert': {
|
|
20
24
|
readonly built: "index-store/convert-worker.js";
|
|
21
25
|
readonly source: "index-store/convert-worker.ts";
|
|
@@ -13,6 +13,10 @@ export const SERVICE_WORKER_ENTRIES = {
|
|
|
13
13
|
embeddings: { built: 'memory/embed-worker.js' },
|
|
14
14
|
'image-embeddings': { built: 'memory/image-embed-worker.js' },
|
|
15
15
|
'static-index': { built: 'index-store/static-index-worker.js' },
|
|
16
|
+
'dashboard-composite': {
|
|
17
|
+
built: 'ambient/dashboard-composite-worker.js',
|
|
18
|
+
source: 'ambient/dashboard-composite-worker.ts',
|
|
19
|
+
},
|
|
16
20
|
'document-convert': {
|
|
17
21
|
built: 'index-store/convert-worker.js',
|
|
18
22
|
source: 'index-store/convert-worker.ts',
|
package/package.json
CHANGED
|
@@ -1,16 +1,20 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bendyline/gezel-inference",
|
|
3
|
-
"version": "0.1
|
|
4
|
-
"description": "
|
|
3
|
+
"version": "1.0.1",
|
|
4
|
+
"description": "Gezel's on-device inference runtime: native engines, local model management, embeddings, knowledge search, and OpenAI-compatible routes.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"keywords": [
|
|
7
7
|
"gezel",
|
|
8
8
|
"ai",
|
|
9
|
-
"agents",
|
|
10
9
|
"local-first",
|
|
11
|
-
"
|
|
12
|
-
"
|
|
13
|
-
"
|
|
10
|
+
"llm",
|
|
11
|
+
"inference",
|
|
12
|
+
"on-device",
|
|
13
|
+
"local-ai",
|
|
14
|
+
"llama.cpp",
|
|
15
|
+
"mlx",
|
|
16
|
+
"embeddings",
|
|
17
|
+
"openai-compatible"
|
|
14
18
|
],
|
|
15
19
|
"homepage": "https://github.com/bendyline/gezel",
|
|
16
20
|
"bugs": {
|
|
@@ -69,16 +73,16 @@
|
|
|
69
73
|
"!dist/**/__pycache__"
|
|
70
74
|
],
|
|
71
75
|
"dependencies": {
|
|
76
|
+
"@bendyline/gezel": "1.2.6",
|
|
77
|
+
"@bendyline/gezel-catalog": "1.2.6",
|
|
78
|
+
"@bendyline/gezel-client": "1.2.6",
|
|
79
|
+
"@bendyline/gezel-knowledge": "1.2.6",
|
|
72
80
|
"hono": "4.13.9",
|
|
73
81
|
"tar": "7.5.21",
|
|
74
82
|
"undici": "8.11.2",
|
|
75
83
|
"yauzl": "^3.4.0",
|
|
76
84
|
"yazl": "^3.3.1",
|
|
77
|
-
"zod": "^4.4.3"
|
|
78
|
-
"@bendyline/gezel-client": "1.2.4",
|
|
79
|
-
"@bendyline/gezel-catalog": "1.2.4",
|
|
80
|
-
"@bendyline/gezel": "1.2.4",
|
|
81
|
-
"@bendyline/gezel-knowledge": "1.2.4"
|
|
85
|
+
"zod": "^4.4.3"
|
|
82
86
|
},
|
|
83
87
|
"peerDependencies": {
|
|
84
88
|
"@huggingface/transformers": "^4.3.1"
|