inference-server 1.0.0-beta.19

package/dist/lib/util.js

@@ -0,0 +1,61 @@
+import { inspect } from 'node:util';
+import fs from 'node:fs';
+import path from 'node:path';
+export function elapsedMillis(since) {
+    const now = process.hrtime.bigint();
+    return Number(now - BigInt(since)) / 1e6;
+}
+export function omitEmptyValues(dict) {
+    return Object.fromEntries(Object.entries(dict).filter(([_, v]) => {
+        return v !== null && v !== undefined;
+    }));
+}
+export function touchFileSync(filePath) {
+    fs.closeSync(fs.openSync(filePath, 'w'));
+}
+function isSubpath(parent, child) {
+    // Normalize paths to resolve .. and . segments
+    const normalizedParent = path.normalize(parent);
+    const normalizedChild = path.normalize(child);
+    // Get relative path from parent to child
+    const relativePath = path.relative(normalizedParent, normalizedChild);
+    // Check if relative path:
+    // 1. Doesn't start with .. (which would mean going up directories)
+    // 2. Isn't an absolute path (which would start with / or C:\ etc)
+    return !relativePath.startsWith('..') && !path.isAbsolute(relativePath);
+}
+export function mergeAbortSignals(signals) {
+    const controller = new AbortController();
+    const onAbort = () => {
+        controller.abort();
+    };
+    for (const signal of signals) {
+        if (signal) {
+            signal.addEventListener('abort', onAbort);
+        }
+    }
+    return controller.signal;
+}
+export function getRandomNumber(min, max) {
+    min = Math.ceil(min);
+    max = Math.floor(max);
+    return Math.floor(Math.random() * (max - min)) + min;
+}
+export function printActiveHandles() {
+    //@ts-ignore
+    const handles = process._getActiveHandles();
+    //@ts-ignore
+    const requests = process._getActiveRequests();
+    console.log('Active Handles:', inspect(handles, { depth: 1 }));
+    console.log('Active Requests:', inspect(requests, { depth: 1 }));
+}
+export function formatBytesPerSecond(speed) {
+    const units = ['B/s', 'KB/s', 'MB/s', 'GB/s', 'TB/s'];
+    let unitIndex = 0;
+    while (speed >= 1024 && unitIndex < units.length - 1) {
+        speed /= 1024;
+        unitIndex++;
+    }
+    return `${speed.toFixed(2)} ${units[unitIndex]}`;
+}
+//# sourceMappingURL=util.js.map

package/dist/lib/util.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"util.js","sourceRoot":"","sources":["../../src/lib/util.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AACnC,OAAO,EAAE,MAAM,SAAS,CAAA;AACxB,OAAO,IAAI,MAAM,WAAW,CAAA;AAE5B,MAAM,UAAU,aAAa,CAAC,KAAa;IAC1C,MAAM,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,CAAA;IACnC,OAAO,MAAM,CAAC,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,CAAA;AACzC,CAAC;AAED,MAAM,UAAU,eAAe,CAAgC,IAAO;IACrE,OAAO,MAAM,CAAC,WAAW,CACxB,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE;QACtC,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,SAAS,CAAA;IACrC,CAAC,CAAC,CACG,CAAA;AACP,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,QAAgB;IAC7C,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,CAAA;AACzC,CAAC;AAED,SAAS,SAAS,CAAC,MAAc,EAAE,KAAa;IAC/C,+CAA+C;IAC/C,MAAM,gBAAgB,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAA;IAC/C,MAAM,eAAe,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;IAE7C,yCAAyC;IACzC,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,CAAC,gBAAgB,EAAE,eAAe,CAAC,CAAA;IAErE,0BAA0B;IAC1B,mEAAmE;IACnE,kEAAkE;IAClE,OAAO,CAAC,YAAY,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,CAAA;AACxE,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,OAAuC;IACxE,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAA;IACxC,MAAM,OAAO,GAAG,GAAG,EAAE;QACpB,UAAU,CAAC,KAAK,EAAE,CAAA;IACnB,CAAC,CAAA;IACD,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC9B,IAAI,MAAM,EAAE,CAAC;YACZ,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;QAC1C,CAAC;IACF,CAAC;IACD,OAAO,UAAU,CAAC,MAAM,CAAA;AACzB,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,GAAW,EAAE,GAAW;IACvD,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IACpB,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;IACrB,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,GAAG,GAAG,CAAA;AACrD,CAAC;AAED,MAAM,UAAU,kBAAkB;IACjC,YAAY;IACZ,MAAM,OAAO,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAA;IAC3C,YAAY;IACZ,MAAM,QAAQ,GAAG,OAAO,CAAC,kBAAkB,EAAE,CAAA;IAE7C,OAAO,CAAC,GAAG,CAAC,iBAAiB,EAAE,OAAO,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC,CAAA;IAC9D,OAAO,CAAC,GAAG,CAAC,kBAAkB,EAAE,OAAO,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC,CAAA;AACjE,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,KAAa;IACjD,MAAM,KAAK,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAA;IACrD,IAAI,SAAS,GAAG,CAAC,CAAA;IAEjB,OAAO,KAAK,IAAI,IAAI,IAAI,SAAS,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtD,KAAK,IAAI,IAAI,CAAA;QACb,SAAS,EAAE,CAAA;IACZ,CAAC;IAED,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,SAAS,CAAC,EAAE,CAAA;AACjD,CAAC"}

package/dist/lib/validateModelFile.d.ts

@@ -0,0 +1,9 @@
+interface ValidatableModelConfig {
+    url?: string;
+    location?: string;
+    modelsCachePath: string;
+    sha256?: string;
+    md5?: string;
+}
+export declare function validateModelFile(config: ValidatableModelConfig): Promise<string | undefined>;
+export {};

package/dist/lib/validateModelFile.js

@@ -0,0 +1,62 @@
+import fs from 'node:fs';
+import { calculateFileChecksum } from '../lib/calculateFileChecksum.js';
+import { resolveModelFileLocation } from '../lib/resolveModelFileLocation.js';
+export async function validateModelFile(config) {
+    const fileLocation = resolveModelFileLocation({
+        url: config.url,
+        filePath: config.location,
+        modelsCachePath: config.modelsCachePath,
+    });
+    if (!fs.existsSync(fileLocation)) {
+        return `Model file missing at ${fileLocation}`;
+    }
+    const hasChecksum = config.sha256 || config.md5;
+    let validatedChecksum = false;
+    const validateChecksum = async () => {
+        if (config.md5) {
+            const fileHash = await calculateFileChecksum(fileLocation, 'md5');
+            if (fileHash === config.md5) {
+                validatedChecksum = true;
+                return true;
+            }
+        }
+        else if (config.sha256) {
+            const fileHash = await calculateFileChecksum(fileLocation, 'sha256');
+            if (fileHash === config.sha256) {
+                validatedChecksum = true;
+                return true;
+            }
+        }
+        return false;
+    };
+    const ipullFile = fileLocation + '.ipull';
+    if (fs.existsSync(ipullFile)) {
+        // if we have a valid file at the download destination, we can remove the ipull file
+        if (hasChecksum) {
+            const isValid = await validateChecksum();
+            if (isValid) {
+                fs.unlinkSync(ipullFile);
+                validatedChecksum = true;
+            }
+        }
+        if (!validatedChecksum) {
+            return `Model file with incomplete download`;
+        }
+    }
+    if (!validatedChecksum && hasChecksum) {
+        if (config.sha256) {
+            const fileHash = await calculateFileChecksum(fileLocation, 'sha256');
+            if (fileHash !== config.sha256) {
+                return `File sha256 checksum mismatch: expected ${config.sha256} got ${fileHash} at ${fileLocation}`;
+            }
+        }
+        else if (config.md5) {
+            const fileHash = await calculateFileChecksum(fileLocation, 'md5');
+            if (fileHash !== config.md5) {
+                return `File md5 checksum mismatch: expected ${config.md5} got ${fileHash} at ${fileLocation}`;
+            }
+        }
+    }
+    return undefined;
+}
+//# sourceMappingURL=validateModelFile.js.map

package/dist/lib/validateModelFile.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"validateModelFile.js","sourceRoot":"","sources":["../../src/lib/validateModelFile.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,MAAM,SAAS,CAAA;AACxB,OAAO,EAAE,qBAAqB,EAAE,MAAM,uCAAuC,CAAA;AAC7E,OAAO,EAAE,wBAAwB,EAAE,MAAM,0CAA0C,CAAA;AAUnF,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,MAA8B;IACrE,MAAM,YAAY,GAAG,wBAAwB,CAAC;QAC7C,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,eAAe,EAAE,MAAM,CAAC,eAAe;KACvC,CAAC,CAAA;IACF,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QAClC,OAAO,yBAAyB,YAAY,EAAE,CAAA;IAC/C,CAAC;IACD,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,GAAG,CAAA;IAC/C,IAAI,iBAAiB,GAAG,KAAK,CAAA;IAC7B,MAAM,gBAAgB,GAAG,KAAK,IAAI,EAAE;QACnC,IAAI,MAAM,CAAC,GAAG,EAAE,CAAC;YAChB,MAAM,QAAQ,GAAG,MAAM,qBAAqB,CAAC,YAAY,EAAE,KAAK,CAAC,CAAA;YACjE,IAAI,QAAQ,KAAK,MAAM,CAAC,GAAG,EAAE,CAAC;gBAC7B,iBAAiB,GAAG,IAAI,CAAA;gBACxB,OAAO,IAAI,CAAA;YACZ,CAAC;QACF,CAAC;aAAM,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YAC1B,MAAM,QAAQ,GAAG,MAAM,qBAAqB,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAA;YACpE,IAAI,QAAQ,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC;gBAChC,iBAAiB,GAAG,IAAI,CAAA;gBACxB,OAAO,IAAI,CAAA;YACZ,CAAC;QACF,CAAC;QACD,OAAO,KAAK,CAAA;IACb,CAAC,CAAA;IACD,MAAM,SAAS,GAAG,YAAY,GAAG,QAAQ,CAAA;IACzC,IAAI,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC9B,oFAAoF;QACpF,IAAI,WAAW,EAAE,CAAC;YACjB,MAAM,OAAO,GAAG,MAAM,gBAAgB,EAAE,CAAA;YACxC,IAAI,OAAO,EAAE,CAAC;gBACb,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,CAAA;gBACxB,iBAAiB,GAAG,IAAI,CAAA;YACzB,CAAC;QACF,CAAC;QACD,IAAI,CAAC,iBAAiB,EAAE,CAAC;YACxB,OAAO,qCAAqC,CAAA;QAC7C,CAAC;IACF,CAAC;IAED,IAAI,CAAC,iBAAiB,IAAI,WAAW,EAAE,CAAC;QACvC,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YACnB,MAAM,QAAQ,GAAG,MAAM,qBAAqB,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAA;YACpE,IAAI,QAAQ,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC;gBAChC,OAAO,2CAA2C,MAAM,CAAC,MAAM,QAAQ,QAAQ,OAAO,YAAY,EAAE,CAAA;YACrG,CAAC;QACF,CAAC;aAAM,IAAI,MAAM,CAAC,GAAG,EAAE,CAAC;YACvB,MAAM,QAAQ,GAAG,MAAM,qBAAqB,CAAC,YAAY,EAAE,KAAK,CAAC,CAAA;YACjE,IAAI,QAAQ,KAAK,MAAM,CAAC,GAAG,EAAE,CAAC;gBAC7B,OAAO,wCAAwC,MAAM,CAAC,GAAG,QAAQ,QAAQ,OAAO,YAAY,EAAE,CAAA;YAC/F,CAAC;QACF,CAAC;IACF,CAAC;IACD,OAAO,SAAS,CAAA;AACjB,CAAC"}

package/dist/lib/validateModelOptions.d.ts

@@ -0,0 +1,3 @@
+import { BuiltInModelOptions } from '../types/index.js';
+export declare function validateModelId(id: string): void;
+export declare function validateModelOptions(id: string, modelOptions: BuiltInModelOptions): void;

package/dist/lib/validateModelOptions.js

@@ -0,0 +1,23 @@
+import { builtInEngineNames } from '../engines/index.js';
+const modelIdPattern = /^[a-zA-Z0-9_\-\.]+$/;
+export function validateModelId(id) {
+    if (!modelIdPattern.test(id)) {
+        throw new Error(`Model "${id}" has invalid name; requires ${modelIdPattern}`);
+    }
+}
+export function validateModelOptions(id, modelOptions) {
+    validateModelId(id);
+    if (!modelOptions.engine) {
+        throw new Error(`Model "${id}" must have an engine`);
+    }
+    const isSourceMissing = !('file' in modelOptions && modelOptions.file) &&
+        !modelOptions.url &&
+        !modelOptions.location;
+    if (builtInEngineNames.includes(modelOptions.engine) && isSourceMissing) {
+        throw new Error(`Model "${id}" must have either file or url`);
+    }
+    if (!modelOptions.task) {
+        throw new Error(`Model "${id}" must have a task`);
+    }
+}
+//# sourceMappingURL=validateModelOptions.js.map

package/dist/lib/validateModelOptions.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"validateModelOptions.js","sourceRoot":"","sources":["../../src/lib/validateModelOptions.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAA;AAE9D,MAAM,cAAc,GAAG,qBAAqB,CAAA;AAC5C,MAAM,UAAU,eAAe,CAAC,EAAU;IACzC,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CACd,UAAU,EAAE,gCAAgC,cAAc,EAAE,CAC5D,CAAA;IACF,CAAC;AACF,CAAC;AAED,MAAM,UAAU,oBAAoB,CACnC,EAAU,EACV,YAAiC;IAEjC,eAAe,CAAC,EAAE,CAAC,CAAA;IACnB,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,UAAU,EAAE,uBAAuB,CAAC,CAAA;IACrD,CAAC;IACD,MAAM,eAAe,GACpB,CAAC,CAAC,MAAM,IAAI,YAAY,IAAI,YAAY,CAAC,IAAI,CAAC;QAC9C,CAAC,YAAY,CAAC,GAAG;QACjB,CAAC,YAAY,CAAC,QAAQ,CAAA;IACvB,IAAI,kBAAkB,CAAC,QAAQ,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,eAAe,EAAE,CAAC;QACzE,MAAM,IAAI,KAAK,CAAC,UAAU,EAAE,gCAAgC,CAAC,CAAA;IAC9D,CAAC;IACD,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,UAAU,EAAE,oBAAoB,CAAC,CAAA;IAClD,CAAC;AACF,CAAC"}

package/dist/pool.d.ts

@@ -0,0 +1,61 @@
+import PQueue from 'p-queue';
+import EventEmitter3 from 'eventemitter3';
+import { ModelInstance } from './instance.js';
+import { ModelConfig, IncomingRequest, ModelEngine } from './types/index.js';
+import { Logger, LogLevel } from './lib/logger.js';
+export interface ModelInstanceHandle {
+    instance: ModelInstance;
+    release: () => Promise<void>;
+}
+type PrepareModelInstanceCallback = (instance: ModelInstance, signal?: AbortSignal) => Promise<void>;
+interface ModelPoolConfig {
+    concurrency: number;
+    models: Record<string, ModelConfig>;
+}
+export interface ModelPoolOptions {
+    concurrency?: number;
+    models: Record<string, ModelConfig>;
+    log?: Logger | LogLevel;
+}
+type ModelPoolEvent = 'ready' | 'spawn' | 'release';
+export declare class ModelPool extends EventEmitter3<ModelPoolEvent> {
+    queue: PQueue;
+    config: ModelPoolConfig;
+    instances: Record<string, ModelInstance>;
+    private engines?;
+    private cleanupInterval?;
+    private log;
+    private requestSequence;
+    private pendingRequests;
+    private shutdownController;
+    private gpuLock;
+    private prepareInstance?;
+    constructor(options: ModelPoolOptions, prepareInstance?: PrepareModelInstanceCallback);
+    init(engines: Record<string, ModelEngine>): Promise<void>;
+    ensureModelInstances(model: ModelConfig): Promise<ModelInstance<unknown>>[];
+    dispose(): Promise<void>;
+    private disposeOutdatedInstances;
+    getStatus(): {
+        processing: number;
+        pending: number;
+        instances: {
+            [k: string]: {
+                model: string;
+                status: "error" | "preparing" | "idle" | "busy" | "loading";
+                engine: (string & {}) | import("./engines").BuiltInEngineName;
+                device: string;
+                contextState: string | undefined;
+                lastUsed: string;
+            };
+        };
+    };
+    canSpawnInstance(modelId: string): boolean;
+    private disposeInstance;
+    private spawnInstance;
+    private acquireGpuInstance;
+    private acquireIdleInstance;
+    private acquireInstance;
+    private createRequestSequence;
+    requestInstance(incomingRequest: IncomingRequest, signal?: AbortSignal): Promise<ModelInstanceHandle>;
+}
+export {};