@bendyline/gezel-inference 1.0.6 → 1.0.7

package/dist/system/host-memory.js

@@ -0,0 +1,89 @@
+import { readFileSync } from 'node:fs';
+import { availableParallelism, cpus, freemem, totalmem } from 'node:os';
+function readTrimmed(path) {
+    try {
+        return readFileSync(path, 'utf8').trim();
+    }
+    catch {
+        return undefined;
+    }
+}
+const nodeProbe = {
+    totalmem,
+    freemem,
+    constrainedMemory: () => typeof process.constrainedMemory === 'function' ? process.constrainedMemory() : undefined,
+    availableMemory: () => typeof process.availableMemory === 'function' ? process.availableMemory() : undefined,
+    availableParallelism,
+    hostCpuCount: () => cpus().length,
+    readFile: (path) => (process.platform === 'linux' ? readTrimmed(path) : undefined),
+};
+/**
+ * The cgroup memory limit when it is below the host's RAM, else null.
+ * Node reads cgroup v1 and v2; "no limit" comes back as 0 or as a value at
+ * or above physical RAM, depending on the libuv release.
+ */
+export function containerMemoryLimitBytes(probe = nodeProbe) {
+    const host = probe.totalmem();
+    const limit = probe.constrainedMemory();
+    if (typeof limit === 'number' && Number.isFinite(limit) && limit > 0 && limit < host) {
+        return limit;
+    }
+    return null;
+}
+/** RAM this process may use: host RAM, or the container's limit when lower. */
+export function totalMemoryBytes(probe = nodeProbe) {
+    return containerMemoryLimitBytes(probe) ?? probe.totalmem();
+}
+/** Free RAM, never more than the container's limit leaves. */
+export function freeMemoryBytes(probe = nodeProbe) {
+    return capToContainer(probe.freemem(), probe);
+}
+/**
+ * Caps a host-wide free-memory reading (`MemAvailable`, `vm_stat`) to what
+ * the container still has. Unchanged outside a container.
+ */
+export function capToContainer(hostFreeBytes, probe = nodeProbe) {
+    if (containerMemoryLimitBytes(probe) === null)
+        return hostFreeBytes;
+    const available = probe.availableMemory();
+    if (typeof available !== 'number' || !Number.isFinite(available) || available < 0) {
+        return hostFreeBytes;
+    }
+    return Math.min(hostFreeBytes, available);
+}
+/**
+ * CPUs this process may keep busy: the scheduler's CPU set, further capped
+ * by a cgroup v2 `cpu.max` quota (or v1 `cfs_quota_us`) rounded up.
+ */
+export function availableCpuCount(probe = nodeProbe) {
+    const cpus = Math.max(1, probe.availableParallelism());
+    const quota = cgroupCpuQuota(probe);
+    return quota === null ? cpus : Math.max(1, Math.min(cpus, Math.ceil(quota)));
+}
+export function cgroupCpuQuota(probe = nodeProbe) {
+    const v2 = probe.readFile('/sys/fs/cgroup/cpu.max');
+    if (v2) {
+        const [max, period] = v2.split(/\s+/);
+        const quota = Number(max);
+        const per = Number(period);
+        if (max !== 'max' && quota > 0 && per > 0)
+            return quota / per;
+        return null;
+    }
+    const v1Quota = Number(probe.readFile('/sys/fs/cgroup/cpu/cpu.cfs_quota_us'));
+    const v1Period = Number(probe.readFile('/sys/fs/cgroup/cpu/cpu.cfs_period_us'));
+    if (v1Quota > 0 && v1Period > 0)
+        return v1Quota / v1Period;
+    return null;
+}
+/**
+ * The CPUs a container (a `--cpus` quota or a `--cpuset-cpus` set) leaves this
+ * process when that is fewer than the host has, else null. Engines that size
+ * their own thread pool from the host's cores, as llama.cpp does, oversubscribe
+ * such a limit badly: 32 threads under a 4-CPU quota spend most of each period
+ * throttled. Measured in a 4-CPU container: a 0.8B model decoded at 3.7 t/s.
+ */
+export function containerCpuLimit(probe = nodeProbe) {
+    const allowed = availableCpuCount(probe);
+    return allowed < probe.hostCpuCount() ? allowed : null;
+}

package/dist/system/memory.js

@@ -1,11 +1,12 @@
 import { execFile } from 'node:child_process';
-import { freemem, platform as osPlatform, totalmem } from 'node:os';
+import { platform as osPlatform } from 'node:os';
 import { promisify } from 'node:util';
 import { readDeviceSystemFacts, windowsHeadlessSpawnOptions, } from '@bendyline/gezel/native';
 import { detectLlamaGpuVram, } from '../providers/llama-cpp/devices.js';
 import { autoDetectBudgetBytes, computeCapacityBudget, setDetectedGpuVramBytes, } from '../providers/native/capacity-budget.js';
 import { readAmdgpuAdapters } from './amdgpu-adapters.js';
 import { gpuCarveOutBytes, gpuVendorFromPciId, linuxGpuAddressableBytes, linuxGpuCarveOutBytes, matchGpuAdapter, } from './gpu-carve-out.js';
+import { freeMemoryBytes, totalMemoryBytes } from './host-memory.js';
 const exec = promisify(execFile);
 let cachedMemoryProfile = null;
 let pendingMemoryProfile = null;
@@ -25,7 +26,7 @@ let pendingMemoryProfile = null;
  */
 export async function detectMemoryProfile() {
     const plat = osPlatform();
-    const totalRam = totalmem();
+    const totalRam = totalMemoryBytes();
     if (plat === 'darwin') {
         setDetectedGpuVramBytes(null);
         return {
@@ -311,7 +312,7 @@ export function sampleMachineMemoryUsage(opts) {
     const residentModels = opts.residentModels ?? [];
     const sampledAt = opts.sampledAt ?? new Date().toISOString();
     if (mainMemory) {
-        const freeRamBytes = clamp(finiteNonNegative(opts.darwinSystemMemory?.freeBytes ?? opts.freeRamBytes ?? freemem()), 0, totalRamBytes);
+        const freeRamBytes = clamp(finiteNonNegative(opts.darwinSystemMemory?.freeBytes ?? opts.freeRamBytes ?? freeMemoryBytes()), 0, totalRamBytes);
         const cachedBytes = opts.darwinSystemMemory
             ? clamp(finiteNonNegative(opts.darwinSystemMemory.cachedBytes), 0, totalRamBytes - freeRamBytes)
             : null;
@@ -512,7 +513,7 @@ function clamp(value, min, max) {
  * held back only ~3 GiB for the OS, so a 95 GB chat model and a 10 GB image
  * model were co-admitted and the image engine died with a CUDA OOM at init.
  */
-export function parseNvidiaMemoryProbe(stdout, systemRamBytes = totalmem()) {
+export function parseNvidiaMemoryProbe(stdout, systemRamBytes = totalMemoryBytes()) {
     const devices = [];
     for (const rawLine of stdout.split(/\r?\n/)) {
         // GPU names can theoretically contain commas. Split on the final field,

package/dist/transformers-cache.js

@@ -19,6 +19,7 @@ import { createReadStream } from 'node:fs';
 import { mkdir, open, readFile, readdir, rename, rm, stat, writeFile } from 'node:fs/promises';
 import { basename, dirname, join, resolve } from 'node:path';
 import { createLogger } from '@bendyline/gezel';
+import { disableOnnxRuntimeTelemetry } from '@bendyline/gezel/native';
 import { isPathInside } from './fs/safe-paths.js';
 const log = createLogger('service');
 /**
@@ -330,6 +331,7 @@ export function isMissingModule(err, specifier) {
     return new RegExp(String.raw `Cannot find (?:package|module) '${escaped}'`).test(err.message);
 }
 async function defaultLoadEnv() {
+    disableOnnxRuntimeTelemetry();
     const mod = (await import('@huggingface/transformers'));
     return mod.env;
 }

package/dist/utils/download-common.d.ts

@@ -14,7 +14,7 @@ export declare const downloadLog: import("@bendyline/gezel").Logger;
  * transparent about who we are is the right posture for a third-party client
  * fetching public weights on the user's behalf.
  */
-export declare const GEZEL_DOWNLOAD_UA = "gezel/0.0.0 (+https://github.com/bendyline/gezel)";
+export declare const GEZEL_DOWNLOAD_UA: string;
 /**
  * Headers for an HF control-plane request. An optional operator token raises
  * public-model rate limits, but is never forwarded to Xet/CAS/CDN hosts.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bendyline/gezel-inference",
-  "version": "1.0.6",
+  "version": "1.0.7",
   "description": "Gezel's on-device inference runtime: native engines, local model management, embeddings, knowledge search, and OpenAI-compatible routes.",
   "license": "MIT",
   "keywords": [
@@ -73,16 +73,16 @@
     "!dist/**/__pycache__"
   ],
   "dependencies": {
-    "@bendyline/gezel": "1.2.11",
-    "@bendyline/gezel-catalog": "1.2.11",
-    "@bendyline/gezel-client": "1.2.11",
-    "@bendyline/gezel-knowledge": "1.2.9",
-    "hono": "4.13.9",
+    "@bendyline/gezel": "1.2.12",
+    "@bendyline/gezel-catalog": "1.2.12",
+    "@bendyline/gezel-client": "1.2.12",
+    "@bendyline/gezel-knowledge": "1.2.10",
+    "hono": "4.13.14",
     "tar": "7.5.21",
-    "undici": "8.11.2",
+    "undici": "8.12.3",
     "yauzl": "^3.4.0",
     "yazl": "^3.3.1",
-    "zod": "^4.4.3"
+    "zod": "4.6.7"
   },
   "peerDependencies": {
     "@huggingface/transformers": "^4.3.1"