@bendyline/gezel-inference 0.1.0 → 1.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/README.md +90 -8
  2. package/dist/cache/adapter.d.ts +7 -0
  3. package/dist/cache/controller.d.ts +12 -0
  4. package/dist/cache/controller.js +24 -0
  5. package/dist/catalog/chat-models/bt/btl4-35b-q4/manifest.json +1 -0
  6. package/dist/catalog/chat-models/gl/glm-5.2-754b-q2/manifest.json +1 -0
  7. package/dist/catalog/chat-models/index.json +1 -1
  8. package/dist/catalog/chat-models/lf/lfm2.5-24b-a2b-q4/manifest.json +122 -0
  9. package/dist/catalog/chat-models/lf/lfm2.5-24b-a2b-q4/versions/1.0.0/manifest.json +15 -0
  10. package/dist/catalog/chat-models/mi/minicpm-v4.7-35b-a3b-q4/manifest.json +185 -0
  11. package/dist/catalog/chat-models/mi/minicpm-v4.7-35b-a3b-q4/versions/1.0.0/manifest.json +94 -0
  12. package/dist/catalog/chat-models/mi/minicpm5-1b-q4/manifest.json +1 -0
  13. package/dist/catalog/chat-models/mi/minicpm5-2b-q4/manifest.json +1 -0
  14. package/dist/catalog/chat-models/qw/qwen3.5-0.8b-q4/manifest.json +1 -0
  15. package/dist/catalog/chat-models/raw-index.json +1 -1
  16. package/dist/catalog/provenance.json +1 -1
  17. package/dist/engines/electron-native-reuse.d.ts +9 -0
  18. package/dist/engines/electron-native-reuse.js +146 -45
  19. package/dist/engines/native-manifest.d.ts +8 -0
  20. package/dist/engines/native-manifest.js +30 -0
  21. package/dist/http/openai-compat/apple-model.d.ts +2 -0
  22. package/dist/http/openai-compat/apple-model.js +21 -1
  23. package/dist/http/openai-compat/chat-target.d.ts +12 -0
  24. package/dist/http/openai-compat/chat-target.js +27 -1
  25. package/dist/http/routes/engines.js +9 -3
  26. package/dist/http/routes/v1-chat.js +21 -8
  27. package/dist/http/routes/v1-models.d.ts +6 -0
  28. package/dist/http/routes/v1-models.js +133 -28
  29. package/dist/http/routes/v1-responses.js +9 -2
  30. package/dist/in-flight-requests.d.ts +19 -0
  31. package/dist/in-flight-requests.js +34 -0
  32. package/dist/inference-service.js +3 -3
  33. package/dist/providers/llama-cpp/build-provider.js +33 -8
  34. package/dist/providers/llama-cpp/cache-adapter.d.ts +12 -0
  35. package/dist/providers/llama-cpp/cache-adapter.js +34 -0
  36. package/dist/providers/llama-cpp/gguf-metadata-async.js +7 -0
  37. package/dist/providers/mlx/provider.js +11 -7
  38. package/dist/providers/mlx/python/batch_processors.py +26 -0
  39. package/dist/providers/mlx/python/gezel_mlx_server.py +88 -20
  40. package/dist/providers/mlx/python/response_format.py +374 -0
  41. package/dist/providers/mlx/python/spec_decode.py +5 -0
  42. package/dist/providers/mlx/python/think_budget.py +35 -1
  43. package/dist/providers/mlx/python/tool_grammar.py +62 -3
  44. package/dist/providers/mlx/reasoning-stream.d.ts +20 -2
  45. package/dist/providers/mlx/reasoning-stream.js +77 -16
  46. package/dist/providers/mlx/runtime-diagnostics.d.ts +16 -0
  47. package/dist/providers/mlx/runtime-diagnostics.js +29 -2
  48. package/dist/providers/native/capacity-broker.d.ts +35 -0
  49. package/dist/providers/native/capacity-broker.js +69 -4
  50. package/dist/providers/native/local-engine-runtime.js +7 -2
  51. package/dist/remotes/identity.d.ts +1 -0
  52. package/dist/remotes/identity.js +6 -0
  53. package/dist/remotes/pinned-fetch.d.ts +2 -0
  54. package/dist/remotes/pinned-fetch.js +30 -1
  55. package/dist/remotes/types.d.ts +14 -0
  56. package/dist/runtime-lock.js +1 -1
  57. package/dist/system/memory.js +11 -1
  58. package/dist/utils/service-worker-entry.d.ts +4 -0
  59. package/dist/utils/service-worker-entry.js +4 -0
  60. package/package.json +15 -11
@@ -1,3 +1,11 @@
1
+ export interface RemoteKnownModel {
2
+ id: string;
3
+ name: string;
4
+ contextWindow?: number;
5
+ supportsTools?: boolean;
6
+ supportsReasoning?: boolean;
7
+ parameterSize?: string;
8
+ }
1
9
  export interface PairedRemote {
2
10
  /** Stable id A assigns to this server = its full identity fingerprint. */
3
11
  remoteId: string;
@@ -29,6 +37,12 @@ export interface PairedRemote {
29
37
  pairedAt: number;
30
38
  /** Last time A successfully reached this server (in-memory unless flushed). */
31
39
  lastSeenAt?: number;
40
+ /**
41
+ * The server's chat models the last time A listed them, already namespaced
42
+ * for A. Kept so a picker can show them, marked unavailable, while the
43
+ * server is asleep or A is off its network.
44
+ */
45
+ knownModels?: RemoteKnownModel[];
32
46
  /**
33
47
  * Runtime-managed connections are never persisted and cannot be removed by
34
48
  * the ordinary pairing UI. Today the only managed peer is this computer's
@@ -9,7 +9,7 @@ export class SingleInstanceError extends Error {
9
9
  lockPath;
10
10
  constructor(holderPid, lockPath) {
11
11
  super(holderPid > 0
12
- ? `Another gezel daemon is already running for this home (pid ${holderPid}). Stop it first with \`gezel stop --daemon\` (plain \`gezel stop\` only halts AI work), or use a different GEZEL_HOME.`
12
+ ? `Another gezel daemon is already running for this home (pid ${holderPid}). Stop it first with \`gezel stop\`, or use a different GEZEL_HOME.`
13
13
  : `Could not acquire the single-instance lock at ${lockPath}.`);
14
14
  this.holderPid = holderPid;
15
15
  this.lockPath = lockPath;
@@ -487,12 +487,22 @@ function isKnownUnifiedNvidiaDevice(name) {
487
487
  const normalized = name.toLowerCase();
488
488
  return /\bgb10\b/.test(normalized) || normalized.includes('dgx spark');
489
489
  }
490
+ /**
491
+ * A hybrid-graphics laptop powers its discrete GPU down when idle, and the
492
+ * first `nvidia-smi` after that waits for it to wake: 2.25–2.36 s on an RTX
493
+ * 5070 Ti laptop (2026-10-15), against ~0.1 s warm. The old 2 s limit read
494
+ * that as "no NVIDIA GPU", so the machine was sized as CPU-only for the next
495
+ * minute and offered a 2.4 GB model with a 12 GB card idle beside it. Startup
496
+ * never awaits this probe, so the margin only costs a caller that asks for the
497
+ * profile while the driver is truly hung.
498
+ */
499
+ const NVIDIA_MEMORY_PROBE_TIMEOUT_MS = 10_000;
490
500
  async function probeNvidiaMemory() {
491
501
  try {
492
502
  const { stdout } = await exec('nvidia-smi', ['--query-gpu=name,memory.total', '--format=csv,noheader,nounits'],
493
503
  // Keep this short-lived console probe invisible without detaching it
494
504
  // from the daemon that owns and awaits it.
495
- { timeout: 2000, ...windowsHeadlessSpawnOptions() });
505
+ { timeout: NVIDIA_MEMORY_PROBE_TIMEOUT_MS, ...windowsHeadlessSpawnOptions() });
496
506
  return parseNvidiaMemoryProbe(stdout);
497
507
  }
498
508
  catch {
@@ -16,6 +16,10 @@ export declare const SERVICE_WORKER_ENTRIES: {
16
16
  readonly 'static-index': {
17
17
  readonly built: "index-store/static-index-worker.js";
18
18
  };
19
+ readonly 'dashboard-composite': {
20
+ readonly built: "ambient/dashboard-composite-worker.js";
21
+ readonly source: "ambient/dashboard-composite-worker.ts";
22
+ };
19
23
  readonly 'document-convert': {
20
24
  readonly built: "index-store/convert-worker.js";
21
25
  readonly source: "index-store/convert-worker.ts";
@@ -13,6 +13,10 @@ export const SERVICE_WORKER_ENTRIES = {
13
13
  embeddings: { built: 'memory/embed-worker.js' },
14
14
  'image-embeddings': { built: 'memory/image-embed-worker.js' },
15
15
  'static-index': { built: 'index-store/static-index-worker.js' },
16
+ 'dashboard-composite': {
17
+ built: 'ambient/dashboard-composite-worker.js',
18
+ source: 'ambient/dashboard-composite-worker.ts',
19
+ },
16
20
  'document-convert': {
17
21
  built: 'index-store/convert-worker.js',
18
22
  source: 'index-store/convert-worker.ts',
package/package.json CHANGED
@@ -1,16 +1,20 @@
1
1
  {
2
2
  "name": "@bendyline/gezel-inference",
3
- "version": "0.1.0",
4
- "description": "Shared on-device inference, model management and knowledge runtime for Gezel hosts.",
3
+ "version": "1.0.1",
4
+ "description": "Gezel's on-device inference runtime: native engines, local model management, embeddings, knowledge search, and OpenAI-compatible routes.",
5
5
  "license": "MIT",
6
6
  "keywords": [
7
7
  "gezel",
8
8
  "ai",
9
- "agents",
10
9
  "local-first",
11
- "daemon",
12
- "server",
13
- "llm"
10
+ "llm",
11
+ "inference",
12
+ "on-device",
13
+ "local-ai",
14
+ "llama.cpp",
15
+ "mlx",
16
+ "embeddings",
17
+ "openai-compatible"
14
18
  ],
15
19
  "homepage": "https://github.com/bendyline/gezel",
16
20
  "bugs": {
@@ -69,16 +73,16 @@
69
73
  "!dist/**/__pycache__"
70
74
  ],
71
75
  "dependencies": {
76
+ "@bendyline/gezel": "1.2.6",
77
+ "@bendyline/gezel-catalog": "1.2.6",
78
+ "@bendyline/gezel-client": "1.2.6",
79
+ "@bendyline/gezel-knowledge": "1.2.6",
72
80
  "hono": "4.13.9",
73
81
  "tar": "7.5.21",
74
82
  "undici": "8.11.2",
75
83
  "yauzl": "^3.4.0",
76
84
  "yazl": "^3.3.1",
77
- "zod": "^4.4.3",
78
- "@bendyline/gezel-client": "1.2.4",
79
- "@bendyline/gezel-catalog": "1.2.4",
80
- "@bendyline/gezel": "1.2.4",
81
- "@bendyline/gezel-knowledge": "1.2.4"
85
+ "zod": "^4.4.3"
82
86
  },
83
87
  "peerDependencies": {
84
88
  "@huggingface/transformers": "^4.3.1"