@bendyline/gezel-inference 1.0.7 → 1.0.9

package/dist/providers/mlx/python/tool_args_json.py

@@ -22,13 +22,17 @@ chunk by chunk. Everything else (ints, floats, bools, objects, arrays) is
 small and needs the whole value for `_convert_param_value`-style coercion, so
 those are buffered per-parameter and emitted when their `</parameter>` lands.
 
-Semantics are copied from mlx_vlm 0.6.6 `tool_parsers/qwen3_coder.py` and must
-stay identical; `test_tool_args_json.py` asserts that by differential-testing
-every case against the reference parser. In particular:
+Semantics are copied from mlx_vlm 0.7.12 `tools/parsers/qwen3_coder.py` and
+must stay identical; `tool_args_json_test.py` asserts that by differential-
+testing every case against the reference parser. In particular:
 
   * exactly ONE leading and ONE trailing "\\n" is stripped from a value
-  * a literal "null" (any case) becomes JSON null
+  * a literal "null" (any case) becomes JSON null, except in a declared
+    string parameter, which keeps the text unless the schema says
+    `"nullable": true`
   * a parameter absent from the schema is treated as a string
+  * a declared parameter without a type decodes JSON objects and arrays and
+    keeps anything else as text
   * unknown/odd types fall through to `ast.literal_eval`, as upstream does
 
 Chunk boundaries are the hazard throughout: `</parameter>`, `</function>` and
@@ -68,29 +72,59 @@ def arguments_config(func_name: str, tools: Optional[Any]) -> Dict[str, Any]:
     return {}
 
 
+def _declared_string(param: Any) -> bool:
+    return (
+        isinstance(param, dict)
+        and isinstance(param.get("type"), str)
+        and param["type"].strip().lower() in _STRING_TYPES
+    )
+
+
 def is_string_param(param_name: str, param_config: Dict[str, Any]) -> bool:
     """Can this parameter stream, or must its value be buffered whole?
 
     Unknown parameters count as strings — upstream returns the raw value when
-    the schema has no entry, and raw-string is the streamable case.
+    the schema has no entry, and raw-string is the streamable case. A declared
+    parameter without a type is buffered: its value may decode to an object.
     """
-    param = param_config.get(param_name)
-    if not param:
-        return True
-    declared = param.get("type")
-    if declared is None:
+    if param_name not in param_config:
         return True
-    return str(declared).strip().lower() in _STRING_TYPES
+    param = param_config[param_name]
+    if not isinstance(param, dict) or "type" not in param:
+        return False
+    return str(param["type"]).strip().lower() in _STRING_TYPES
+
+
+def null_text_is_none(param_name: str, param_config: Dict[str, Any]) -> bool:
+    """Whether a complete value of "null" is JSON null for this parameter.
+
+    A declared string keeps the text unless the schema marks it nullable;
+    every other parameter, unknown ones included, reads it as null.
+    """
+    param = param_config.get(param_name)
+    if _declared_string(param):
+        return param.get("nullable") is True
+    return True
 
 
 def convert_param_value(value: str, param_name: str, param_config: Dict[str, Any]) -> Any:
     """Coerce a complete value, mirroring `_convert_param_value`."""
+    param = param_config.get(param_name)
+    if _declared_string(param):
+        if value.lower() == "null" and param.get("nullable") is True:
+            return None
+        return value
     if value.lower() == "null":
         return None
-    param = param_config.get(param_name)
-    if not param:
+    if param_name not in param_config:
         return value
-    param_type = str(param["type"]).strip().lower() if "type" in param else "string"
+    if not isinstance(param, dict) or "type" not in param:
+        try:
+            decoded = json.loads(value)
+        except json.JSONDecodeError:
+            return value
+        return decoded if isinstance(decoded, (dict, list)) else value
+    param_type = str(param["type"]).strip().lower()
     if param_type in _STRING_TYPES:
         return value
     if param_type.startswith(("int", "uint", "long", "short", "unsigned")):
@@ -194,10 +228,10 @@ class ToolArgsJsonEncoder:
                 self._leading_stripped = False
                 self._opened_quote = False
                 # NOTE: the opening quote is deliberately NOT emitted yet.
-                # `_convert_param_value` maps a whole value of "null" to JSON
-                # null BEFORE consulting the schema, so even a string-typed
-                # parameter can turn out to be null — and by then a quote
-                # would already be on the wire. Hold the first few bytes.
+                # A whole value of "null" is JSON null for an unknown or
+                # nullable string parameter (`null_text_is_none`) — and by
+                # then a quote would already be on the wire. Hold the first
+                # few bytes.
                 continue
 
             end = self._buf.find(_PARAM_CLOSE)
@@ -228,8 +262,8 @@ class ToolArgsJsonEncoder:
             #   * strip exactly one leading newline
             #   * strip exactly one trailing newline (so the final newline is
             #     held until we know whether anything follows it)
-            #   * a complete value of "null" is JSON null, not a string, so
-            #     the opening quote waits until the value is longer than
+            #   * a complete value of "null" can be JSON null, not a string,
+            #     so the opening quote waits until the value is longer than
             #     "null" or the parameter closes
             if not self._leading_stripped:
                 if chunk.startswith("\n"):
@@ -241,7 +275,11 @@ class ToolArgsJsonEncoder:
                 self._value_buf += chunk
                 if end < 0 and len(self._value_buf) <= 4:
                     break
-                if end >= 0 and self._value_buf.lower() == "null":
+                if (
+                    end >= 0
+                    and self._value_buf.lower() == "null"
+                    and null_text_is_none(self._value_name, self.param_config)
+                ):
                     out.append(f"{json.dumps(self._value_name)}:null")
                     self._in_value = False
                     continue

package/dist/providers/mlx/runtime-diagnostics.d.ts

@@ -38,6 +38,12 @@ export declare function buildPreFirstByteAbortMessage(lastPrefill: {
 } | null): string;
 /** Build the user-facing message for an engine stream that died mid-turn. */
 export declare function buildMidStreamDropMessage(charsReceived: number, plannedStop: boolean, engineStillRunning?: boolean): string;
+/**
+ * Prompt characters one attached image stands for. The sidecar bounds images
+ * to 1024px, so their patch tokens are bounded too, and base64 bytes are never
+ * read as language tokens.
+ */
+export declare const MLX_IMAGE_RESERVED_CHARS = 8192;
 /** Estimate request tokens from message bodies and serialized tool schemas. */
 export declare function estimatePromptTokens(messages: unknown, tools: unknown): number;
 export declare const PRE_FIRST_BYTE_BASE_MS = 300000;

package/dist/providers/mlx/runtime-diagnostics.js

@@ -146,11 +146,28 @@ export function buildMidStreamDropMessage(charsReceived, plannedStop, engineStil
     }
     return `[Mac AI] the on-device engine dropped the connection mid-turn (${got}). This usually means the mlx server crashed, ran out of memory, or was restarted while the turn was streaming — this turn's work was lost. Retry the turn; if it keeps happening, restart the engine in Settings → On-device.`;
 }
+/**
+ * Prompt characters one attached image stands for. The sidecar bounds images
+ * to 1024px, so their patch tokens are bounded too, and base64 bytes are never
+ * read as language tokens.
+ */
+export const MLX_IMAGE_RESERVED_CHARS = 8192;
 /** Estimate request tokens from message bodies and serialized tool schemas. */
 export function estimatePromptTokens(messages, tools) {
     let chars = 0;
     try {
-        chars += JSON.stringify(messages)?.length ?? 0;
+        // Counted as base64, one generated picture read as ~1.1M tokens: an
+        // image gezel's six renders and reads put the estimate past 6M while the
+        // engine prefilled 14K (qwen3.8-27b, 2026-11-04).
+        let images = 0;
+        const text = JSON.stringify(messages, (key, value) => {
+            if (key === 'images' && Array.isArray(value)) {
+                images += value.length;
+                return undefined;
+            }
+            return value;
+        });
+        chars += (text?.length ?? 0) + images * MLX_IMAGE_RESERVED_CHARS;
     }
     catch {
         /* circular or unstringifiable */

package/dist/providers/mlx/tool-call-protocol.d.ts

@@ -39,9 +39,10 @@ export declare function applyCallableToolGrammar(body: Record<string, unknown>,
  *
  * This deliberately checks presence only. A present-but-wrongly-typed value
  * remains the MCP validator's job; the MLX Hermes grammar promises only that
- * every declared required key is emitted before the call can close. Seeing a
- * missing key is therefore evidence that the live constrained-decoding path
- * broke its own contract, not an ordinary model validation miss.
+ * every declared required key is emitted before a constrained call can close.
+ * A missing key alone does not establish a decoder fault: calls salvaged from
+ * ordinary prose were not subject to that grammar. The caller must preserve
+ * their origin when deciding whether to disable speculative decoding.
  */
 export declare function missingTopLevelRequiredToolArgs(schema: Record<string, unknown> | undefined, args: Record<string, unknown>): string[];
 export declare function hermesRequiredArgGrammarRequested(body: Record<string, unknown>): boolean;

package/dist/providers/mlx/tool-call-protocol.js

@@ -59,9 +59,10 @@ export function applyCallableToolGrammar(body, bridges) {
  *
  * This deliberately checks presence only. A present-but-wrongly-typed value
  * remains the MCP validator's job; the MLX Hermes grammar promises only that
- * every declared required key is emitted before the call can close. Seeing a
- * missing key is therefore evidence that the live constrained-decoding path
- * broke its own contract, not an ordinary model validation miss.
+ * every declared required key is emitted before a constrained call can close.
+ * A missing key alone does not establish a decoder fault: calls salvaged from
+ * ordinary prose were not subject to that grammar. The caller must preserve
+ * their origin when deciding whether to disable speculative decoding.
  */
 export function missingTopLevelRequiredToolArgs(schema, args) {
     const required = schema?.required;

package/dist/providers/mlx/venv.d.ts

@@ -78,9 +78,27 @@
  * instead of consuming the Metal working set. Do not treat the library pin
  * alone as proof that every community conversion is correct or fits.
  *
+ * 0.7.12 (2026-11-05): the project moved to Neywa-Labs/mlx-vlm; same PyPI
+ * name. It resolves mlx 0.32.4 (mlx-audio pins it exactly) and transformers
+ * 5.22. Every private `speculative.mtp` helper the assisted round loop calls
+ * is unchanged for Qwen 3.5; Gemma 4 targets now rebuild the drafter's shared
+ * KV from committed caches, which `shared_kv_after_round` follows. Its
+ * qwen3_coder parser keeps "null" as text in a string parameter, which
+ * `tool_args_json.py` now mirrors. Verified on an M5 Max against 0.7.1
+ * through the real sidecar: gemma4-e4b and qwen3.8-27b + MTP drafter render
+ * identical prompt token counts and identical tool-call, JSON and vision
+ * output; 0.7.1-written disk caches load and extend (Gemma 4 rotating,
+ * Qwen 3.5 hybrid). Greedy MTP matches greedy no-spec on code at 0-30k
+ * tokens; on prose both versions flip bf16 one-step ties (top-2 gap 0.125
+ * nats), which two non-speculative decode paths also disagree on, so prose
+ * is not an exactness probe. Clean A/B/B/A decode on qwen3.8-27b-q4: MTP on
+ * the vlm tower at parity to 10k and +19% at 50k; vlm tower without MTP 2.0x
+ * at 50k (7.3 -> 14.8 tok/s), now level with the mlx_lm tower; mlx_lm tower
+ * 0.94x at 0-10k, within run-to-run noise.
+ *
  * Keep upgrades explicit and re-sweep the MLX catalog whenever this pin
  * moves. */
-export declare const MLX_DEFAULT_PACKAGE_SPEC = "mlx-vlm==0.7.1";
+export declare const MLX_DEFAULT_PACKAGE_SPEC = "mlx-vlm==0.7.12";
 /** Venv name passed to `UvRuntime.ensureVenv` (and its dir basename). */
 export declare const MLX_VENV_NAME = "mlx";
 /**

package/dist/providers/mlx/venv.js

@@ -78,9 +78,27 @@
  * instead of consuming the Metal working set. Do not treat the library pin
  * alone as proof that every community conversion is correct or fits.
  *
+ * 0.7.12 (2026-11-05): the project moved to Neywa-Labs/mlx-vlm; same PyPI
+ * name. It resolves mlx 0.32.4 (mlx-audio pins it exactly) and transformers
+ * 5.22. Every private `speculative.mtp` helper the assisted round loop calls
+ * is unchanged for Qwen 3.5; Gemma 4 targets now rebuild the drafter's shared
+ * KV from committed caches, which `shared_kv_after_round` follows. Its
+ * qwen3_coder parser keeps "null" as text in a string parameter, which
+ * `tool_args_json.py` now mirrors. Verified on an M5 Max against 0.7.1
+ * through the real sidecar: gemma4-e4b and qwen3.8-27b + MTP drafter render
+ * identical prompt token counts and identical tool-call, JSON and vision
+ * output; 0.7.1-written disk caches load and extend (Gemma 4 rotating,
+ * Qwen 3.5 hybrid). Greedy MTP matches greedy no-spec on code at 0-30k
+ * tokens; on prose both versions flip bf16 one-step ties (top-2 gap 0.125
+ * nats), which two non-speculative decode paths also disagree on, so prose
+ * is not an exactness probe. Clean A/B/B/A decode on qwen3.8-27b-q4: MTP on
+ * the vlm tower at parity to 10k and +19% at 50k; vlm tower without MTP 2.0x
+ * at 50k (7.3 -> 14.8 tok/s), now level with the mlx_lm tower; mlx_lm tower
+ * 0.94x at 0-10k, within run-to-run noise.
+ *
  * Keep upgrades explicit and re-sweep the MLX catalog whenever this pin
  * moves. */
-export const MLX_DEFAULT_PACKAGE_SPEC = 'mlx-vlm==0.7.1';
+export const MLX_DEFAULT_PACKAGE_SPEC = 'mlx-vlm==0.7.12';
 /** Venv name passed to `UvRuntime.ensureVenv` (and its dir basename). */
 export const MLX_VENV_NAME = 'mlx';
 /**
@@ -100,7 +118,8 @@ export function mlxVenvPackages(packageSpec) {
     // a dependency (0.6.6 did, which is the only reason the old list
     // worked). Dropping this pin kills the batch engine at import time.
     // 0.31.3 is also the combination used for the 0.7.1 Qwen4-Exp batch
-    // compatibility check above; keep it independent of mlx-vlm's deps.
+    // compatibility check and the 0.7.12 verification above; keep it
+    // independent of mlx-vlm's deps.
     return [
         packageSpec ?? MLX_DEFAULT_PACKAGE_SPEC,
         'mlx-lm==0.31.3',

package/dist/providers/native/device-capacity-ledger.d.ts

@@ -1,5 +1,6 @@
 import type { NativeCapacityCommand, NativeCapacityReply } from '@bendyline/gezel';
-import { readProcessIdentities } from './process-identity.js';
+import { ProcessIdentityCache, processAlive, type readProcessIdentities } from './process-identity.js';
+export { processAlive };
 export interface DeviceCapacitySample {
     budgetBytes: number;
     gpuBudgetBytes: number;
@@ -14,7 +15,6 @@ export interface DeviceCapacitySample {
     availableRamBytes?: number;
     serializeLoads: boolean;
 }
-export declare function processAlive(pid: number): boolean;
 /**
  * SQLite's process lock makes check-and-reserve atomic across GEZEL_HOME values.
  * Reservations survive a broker restart. A dead parent does not free the memory
@@ -22,11 +22,13 @@ export declare function processAlive(pid: number): boolean;
  */
 export declare class DeviceCapacityLedger {
     private readonly options;
+    private readonly identityCache;
     constructor(options: {
         directory: string;
         sample: () => Promise<DeviceCapacitySample>;
         alive?: (pid: number) => boolean;
         identities?: typeof readProcessIdentities;
+        identityCache?: ProcessIdentityCache;
     });
     execute(command: NativeCapacityCommand): Promise<NativeCapacityReply>;
     private transact;

package/dist/providers/native/device-capacity-ledger.js

@@ -2,7 +2,8 @@ import { chmod, mkdir } from 'node:fs/promises';
 import { createRequire } from 'node:module';
 import { join } from 'node:path';
 import { CapacityDeniedError } from './capacity-broker.js';
-import { readProcessIdentities } from './process-identity.js';
+import { ProcessIdentityCache, processAlive, sharedProcessIdentities, } from './process-identity.js';
+export { processAlive };
 // Keep the newer builtin behind a runtime require. The current tsup/esbuild
 // target rewrites a static `node:sqlite` import to the nonexistent npm package
 // `sqlite`, making the daemon bundle fail before it can write runtime state.
@@ -11,15 +12,6 @@ const { DatabaseSync } = nodeRequire('node:sqlite');
 function gb(bytes) {
     return `${(bytes / 1024 ** 3).toFixed(1)} GB`;
 }
-export function processAlive(pid) {
-    try {
-        process.kill(pid, 0);
-        return true;
-    }
-    catch (error) {
-        return error.code !== 'ESRCH';
-    }
-}
 /**
  * SQLite's process lock makes check-and-reserve atomic across GEZEL_HOME values.
  * Reservations survive a broker restart. A dead parent does not free the memory
@@ -27,8 +19,18 @@ export function processAlive(pid) {
  */
 export class DeviceCapacityLedger {
     options;
+    identityCache;
     constructor(options) {
         this.options = options;
+        // An injected reader still goes through a cache, so tests see the reads
+        // production makes. An injected `alive` alone checks liveness only.
+        this.identityCache =
+            options.identityCache ??
+                (options.identities
+                    ? new ProcessIdentityCache({ read: options.identities, alive: options.alive })
+                    : options.alive
+                        ? undefined
+                        : sharedProcessIdentities);
     }
     async execute(command) {
         const sample = await this.options.sample();
@@ -43,18 +45,31 @@ export class DeviceCapacityLedger {
                 .prepare('SELECT claim FROM claims')
                 .all()
                 .map((row) => JSON.parse(String(row.claim)));
-            const pids = snapshot.flatMap((claim) => [
-                claim.ownerPid,
-                ...(claim.childPid === undefined ? [] : [claim.childPid]),
-            ]);
-            if (command.action === 'acquire')
-                pids.push(command.ownerPid);
+            const record = [];
+            if (command.action === 'acquire' && !snapshot.some((claim) => claim.id === command.id))
+                record.push(command.ownerPid);
             if (command.action === 'bind')
-                pids.push(command.childPid);
+                record.push(command.childPid);
+            // Stamps only decide which reservations are stale, and a stale one costs
+            // someone only while a request waits for memory. With no request
+            // waiting, liveness alone prunes, so the status check that runs before
+            // every engine request reads no stamps.
+            const confirm = new Map();
+            const stamp = (pid, identity) => {
+                if (pid !== undefined && identity !== undefined)
+                    confirm.set(pid, [...(confirm.get(pid) ?? []), identity]);
+            };
+            if (command.action === 'acquire' || snapshot.some((claim) => claim.phase === 'waiting')) {
+                for (const claim of snapshot) {
+                    stamp(claim.childPid, claim.childIdentity);
+                    // An owner's stamp is only consulted while its claim is queued.
+                    if (claim.phase === 'waiting')
+                        stamp(claim.ownerPid, claim.ownerIdentity);
+                }
+            }
             // OS probes may spawn a helper. Never hold SQLite's writer lock while awaiting one.
-            const probe = this.options.identities ?? (this.options.alive ? undefined : readProcessIdentities);
-            const identities = command.action !== 'release' && probe
-                ? await probe(pids).catch(() => new Map())
+            const identities = command.action !== 'release' && this.identityCache
+                ? await this.identityCache.lookup(record, confirm).catch(() => new Map())
                 : new Map();
             db.exec('BEGIN IMMEDIATE');
             try {

package/dist/providers/native/device-capacity.d.ts

@@ -77,6 +77,10 @@ export declare function staticCapacityCeilings(budget: Pick<CapacityBudget, 'bud
     budgetBytes: number;
     gpuBudgetBytes: number;
 };
+/** The RAM ceiling the device ledger admits one launch under, for sizing a launch to fit it. */
+export declare function deviceModelBudgetBytes(config: {
+    localEngineMemoryGb?: number | null | undefined;
+}): Promise<number>;
 export declare function sampleDeviceCapacity(home?: string): Promise<DeviceCapacitySample>;
 export declare function localDeviceCapacity(home: string): DeviceCapacityLedger;
 /** Conservative fallback for media engines that do not expose a launch planner. */

package/dist/providers/native/device-capacity.js

@@ -155,6 +155,11 @@ export function staticCapacityCeilings(budget, ramBytes, configuredBytes, platfo
             : budget.fastBytes),
     };
 }
+/** The RAM ceiling the device ledger admits one launch under, for sizing a launch to fit it. */
+export async function deviceModelBudgetBytes(config) {
+    const ram = totalMemoryBytes();
+    return staticCapacityCeilings(await measuredCapacityBudget(), ram, configuredEngineMemoryBytes(config)).budgetBytes;
+}
 export async function sampleDeviceCapacity(home) {
     const budget = await measuredCapacityBudget();
     const ram = totalMemoryBytes();

package/dist/providers/native/local-engine-runtime.d.ts

@@ -37,7 +37,7 @@ export declare class LocalEngineRuntime {
     /** Product chat may prefill its own sessions; the engine runtime never resolves them. */
     protected prefillCacheSession(_sessionId: string): Promise<boolean>;
     /** Cached provider instances keyed by provider name. */
-    protected readonly providers: Map<"copilot" | "openai" | "anthropic" | "anthropic-cli" | "codex-cli" | "ollama" | "llama-cpp" | "mlx" | "ds4" | "apple-foundation-models" | "android-mlkit" | "remote", LLMProvider>;
+    protected readonly providers: Map<"copilot" | "openai" | "anthropic" | "anthropic-cli" | "codex-cli" | "ollama" | "llama-cpp" | "mlx" | "ds4" | "remote" | "apple-foundation-models" | "android-mlkit", LLMProvider>;
     protected readonly catalog: CatalogService;
     protected readonly llamaCppModels?: import('../llama-cpp/index.js').LlamaCppModelManager;
     protected readonly ds4Models?: import('../llama-cpp/index.js').LlamaCppModelManager;
@@ -384,6 +384,21 @@ export declare class LocalEngineRuntime {
          * the only guard against a window the engine cannot actually serve.
          */
         contextCeilingTokens?: number;
+        /** ds4 only: whether the launch would stream routed experts from SSD. */
+        ssdStreaming?: boolean;
+        /** ds4 only: `plannedResidentBytes` comes from the entry's measurement for this mode. */
+        memoryMeasured?: boolean;
+        /**
+         * ds4 only, streamed launches with a measured footprint: the expert cache
+         * the launcher would choose, so the model list's fit verdict comes from the
+         * same plan instead of a copy of its headroom rule.
+         */
+        expertCache?: {
+            cacheBytes: number;
+            requestedBytes: number;
+            clamped: boolean;
+            safe: boolean;
+        };
     }>;
     /**
      * Cancel a pending provider-queue entry by (provider, id). Resolves