inference-server 1.0.0-beta.19

package/dist/server.js

@@ -0,0 +1,221 @@
+import path from 'node:path';
+import { builtInEngineNames } from './engines/index.js';
+import { ModelPool } from './pool.js';
+import { ModelStore } from './store.js';
+import { createSublogger, LogLevels } from './lib/logger.js';
+import { resolveModelFileLocation } from './lib/resolveModelFileLocation.js';
+import { validateModelOptions } from './lib/validateModelOptions.js';
+import { getCacheDirPath } from './lib/getCacheDirPath.js';
+export function startModelServer(options) {
+    const server = new ModelServer(options);
+    server.start();
+    return server;
+}
+export class ModelServer {
+    pool;
+    store;
+    engines = {};
+    log;
+    constructor(options) {
+        this.log = createSublogger(options.log);
+        let modelsCachePath = getCacheDirPath('models');
+        if (options.cachePath) {
+            modelsCachePath = path.join(options.cachePath, 'models');
+        }
+        const modelsWithDefaults = {};
+        const usedEngines = [];
+        for (const modelId in options.models) {
+            const modelOptions = options.models[modelId];
+            const isBuiltIn = builtInEngineNames.includes(modelOptions.engine);
+            if (isBuiltIn) {
+                const builtInModelOptions = modelOptions;
+                // can validate and resolve location of model files if a built-in engine is used
+                validateModelOptions(modelId, builtInModelOptions);
+                modelsWithDefaults[modelId] = {
+                    id: modelId,
+                    minInstances: 0,
+                    maxInstances: 1,
+                    modelsCachePath,
+                    location: resolveModelFileLocation({
+                        url: builtInModelOptions.url,
+                        filePath: builtInModelOptions.location,
+                        modelsCachePath,
+                    }),
+                    ...builtInModelOptions,
+                };
+            }
+            else {
+                const customEngineOptions = modelOptions;
+                modelsWithDefaults[modelId] = {
+                    id: modelId,
+                    minInstances: 0,
+                    maxInstances: 1,
+                    modelsCachePath,
+                    ...customEngineOptions,
+                };
+            }
+            usedEngines.push({
+                model: modelId,
+                engine: modelOptions.engine,
+            });
+        }
+        const customEngines = Object.keys(options.engines ?? {});
+        for (const ref of usedEngines) {
+            const isBuiltIn = builtInEngineNames.includes(ref.engine);
+            const isCustom = customEngines.includes(ref.engine);
+            if (!isBuiltIn && !isCustom) {
+                throw new Error(`Engine "${ref.engine}" used by model "${ref.model}" does not exist`);
+            }
+            if (isCustom) {
+                this.engines[ref.engine] = options.engines[ref.engine];
+            }
+        }
+        this.store = new ModelStore({
+            log: this.log,
+            // TODO expose this? or remove it?
+            // prepareConcurrency: 2,
+            models: modelsWithDefaults,
+            modelsCachePath,
+        });
+        this.pool = new ModelPool({
+            log: this.log,
+            concurrency: options.concurrency ?? 1,
+            models: modelsWithDefaults,
+        }, this.prepareInstance.bind(this));
+    }
+    modelExists(modelId) {
+        return !!this.pool.config.models[modelId];
+    }
+    async start() {
+        const engineStartPromises = [];
+        // call startEngine on custom engines
+        for (const [key, methods] of Object.entries(this.engines)) {
+            if (methods.start) {
+                engineStartPromises.push(methods.start(this));
+            }
+        }
+        // import built-in engines
+        for (const key of builtInEngineNames) {
+            // skip unused engines
+            const modelUsingEngine = Object.keys(this.store.models).find((modelId) => this.store.models[modelId].engine === key);
+            if (!modelUsingEngine) {
+                continue;
+            }
+            engineStartPromises.push(new Promise(async (resolve, reject) => {
+                try {
+                    const engine = await import(`./engines/${key}/engine.js`);
+                    this.engines[key] = engine;
+                    resolve({
+                        key,
+                        engine,
+                    });
+                }
+                catch (err) {
+                    reject(err);
+                }
+            }));
+        }
+        await Promise.all(engineStartPromises);
+        await Promise.all([this.store.init(this.engines), this.pool.init(this.engines)]);
+    }
+    async stop() {
+        this.log(LogLevels.info, 'Stopping model server');
+        this.pool.queue.clear();
+        this.store.dispose();
+        // need to make sure all tasks are canceled, waiting for idle can make stop hang
+        // await this.pool.queue.onIdle() // would wait until all completions are done
+        try {
+            await this.pool.dispose(); // might cause abort errors when there are still running tasks
+        }
+        catch (err) {
+            this.log(LogLevels.error, 'Error while stopping model server', err);
+        }
+        this.log(LogLevels.debug, 'Model server stopped');
+    }
+    async requestInstance(request, signal) {
+        return this.pool.requestInstance(request, signal);
+    }
+    // gets called by the pool right before a new instance is created
+    async prepareInstance(instance, signal) {
+        const model = instance.config;
+        const modelStoreStatus = this.store.models[model.id].status;
+        if (modelStoreStatus === 'unloaded') {
+            await this.store.prepareModel(model.id, signal);
+        }
+        if (modelStoreStatus === 'preparing') {
+            const modelReady = new Promise((resolve, reject) => {
+                const onCompleted = async (storeModel) => {
+                    if (storeModel.id === model.id) {
+                        this.store.prepareQueue.off('completed', onCompleted);
+                        if (storeModel.status === 'ready') {
+                            resolve();
+                        }
+                        else {
+                            reject();
+                        }
+                    }
+                };
+                this.store.prepareQueue.on('completed', onCompleted);
+            });
+            await modelReady;
+        }
+    }
+    async processChatCompletionTask(args, options) {
+        const lock = await this.requestInstance(args);
+        const task = lock.instance.processChatCompletionTask(args, options);
+        const result = await task.result;
+        await lock.release();
+        return result;
+    }
+    async processTextCompletionTask(args, options) {
+        const lock = await this.requestInstance(args);
+        const task = lock.instance.processTextCompletionTask(args, options);
+        const result = await task.result;
+        await lock.release();
+        return result;
+    }
+    async processEmbeddingTask(args, options) {
+        const lock = await this.requestInstance(args);
+        const task = lock.instance.processEmbeddingTask(args, options);
+        const result = await task.result;
+        await lock.release();
+        return result;
+    }
+    async processImageToTextTask(args, options) {
+        const lock = await this.requestInstance(args);
+        const task = lock.instance.processImageToTextTask(args, options);
+        const result = await task.result;
+        await lock.release();
+        return result;
+    }
+    async processSpeechToTextTask(args, options) {
+        const lock = await this.requestInstance(args);
+        const task = lock.instance.processSpeechToTextTask(args, options);
+        const result = await task.result;
+        await lock.release();
+        return result;
+    }
+    async processTextToImageTask(args, options) {
+        const lock = await this.requestInstance(args);
+        const task = lock.instance.processTextToImageTask(args, options);
+        const result = await task.result;
+        await lock.release();
+        return result;
+    }
+    async processImageToImageTask(args, options) {
+        const lock = await this.requestInstance(args);
+        const task = lock.instance.processImageToImageTask(args, options);
+        const result = await task.result;
+        await lock.release();
+        return result;
+    }
+    getStatus() {
+        const poolStatus = this.pool.getStatus();
+        const storeStatus = this.store.getStatus();
+        return {
+            pool: poolStatus,
+            store: storeStatus,
+        };
+    }
+}
+//# sourceMappingURL=server.js.map

package/dist/server.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"server.js","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAA;AAC9D,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AAE5C,OAAO,EAAE,UAAU,EAAe,MAAM,mBAAmB,CAAA;AAmB3D,OAAO,EAAoB,eAAe,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAA;AACrF,OAAO,EAAE,wBAAwB,EAAE,MAAM,0CAA0C,CAAA;AACnF,OAAO,EAAE,oBAAoB,EAAE,MAAM,sCAAsC,CAAA;AAC3E,OAAO,EAAE,eAAe,EAAE,MAAM,iCAAiC,CAAA;AAUjE,MAAM,UAAU,gBAAgB,CAAC,OAA2B;IAC3D,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,OAAO,CAAC,CAAA;IACvC,MAAM,CAAC,KAAK,EAAE,CAAA;IACd,OAAO,MAAM,CAAA;AACd,CAAC;AAED,MAAM,OAAO,WAAW;IACvB,IAAI,CAAW;IACf,KAAK,CAAY;IACjB,OAAO,GAAgC,EAAE,CAAA;IACzC,GAAG,CAAQ;IAEX,YAAY,OAA2B;QACtC,IAAI,CAAC,GAAG,GAAG,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QACvC,IAAI,eAAe,GAAG,eAAe,CAAC,QAAQ,CAAC,CAAA;QAC/C,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YACvB,eAAe,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAA;QACzD,CAAC;QAED,MAAM,kBAAkB,GAAoC,EAAE,CAAA;QAC9D,MAAM,WAAW,GAA6C,EAAE,CAAA;QAChE,KAAK,MAAM,OAAO,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACtC,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;YAC5C,MAAM,SAAS,GAAG,kBAAkB,CAAC,QAAQ,CAAC,YAAY,CAAC,MAAM,CAAC,CAAA;YAClE,IAAI,SAAS,EAAE,CAAC;gBACf,MAAM,mBAAmB,GAAG,YAAmC,CAAA;gBAC/D,gFAAgF;gBAChF,oBAAoB,CAAC,OAAO,EAAE,mBAAmB,CAAC,CAAA;gBAClD,kBAAkB,CAAC,OAAO,CAAC,GAAG;oBAC7B,EAAE,EAAE,OAAO;oBACX,YAAY,EAAE,CAAC;oBACf,YAAY,EAAE,CAAC;oBACf,eAAe;oBACf,QAAQ,EAAE,wBAAwB,CAAC;wBAClC,GAAG,EAAE,mBAAmB,CAAC,GAAG;wBAC5B,QAAQ,EAAE,mBAAmB,CAAC,QAAQ;wBACtC,eAAe;qBACf,CAAC;oBACF,GAAG,mBAAmB;iBACtB,CAAA;YACF,CAAC;iBAAM,CAAC;gBACP,MAAM,mBAAmB,GAAG,YAAwC,CAAA;gBACpE,kBAAkB,CAAC,OAAO,CAAC,GAAG;oBAC7B,EAAE,EAAE,OAAO;oBACX,YAAY,EAAE,CAAC;oBACf,YAAY,EAAE,CAAC;oBACf,eAAe;oBACf,GAAG,mBAAmB;iBACtB,CAAA;YACF,CAAC;YACD,WAAW,CAAC,IAAI,CAAC;gBAChB,KAAK,EAAE,OAAO;gBACd,MAAM,EAAE,YAAY,CAAC,MAAM;aAC3B,CAAC,CAAA;QACH,CAAC;QAED,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC,CAAA;QACxD,KAAK,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;YAC/B,MAAM,SAAS,GAAG,kBAAkB,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;YACzD,MAAM,QAAQ,GAAG,aAAa,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;YACnD,IAAI,CAAC,SAAS,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAC7B,MAAM,IAAI,KAAK,CAAC,WAAW,GAAG,CAAC,MAAM,oBAAoB,GAAG,CAAC,KAAK,kBAAkB,CAAC,CAAA;YACtF,CAAC;YACD,IAAI,QAAQ,EAAE,CAAC;gBACd,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,OAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;YACxD,CAAC;QACF,CAAC;QAED,IAAI,CAAC,KAAK,GAAG,IAAI,UAAU,CAAC;YAC3B,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,kCAAkC;YAClC,yBAAyB;YACzB,MAAM,EAAE,kBAAkB;YAC1B,eAAe;SACf,CAAC,CAAA;QACF,IAAI,CAAC,IAAI,GAAG,IAAI,SAAS,CACxB;YACC,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,CAAC;YACrC,MAAM,EAAE,kBAAkB;SAC1B,EACD,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAC/B,CAAA;IACF,CAAC;IAED,WAAW,CAAC,OAAe;QAC1B,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;IAC1C,CAAC;IAED,KAAK,CAAC,KAAK;QACV,MAAM,mBAAmB,GAAG,EAAE,CAAA;QAC9B,qCAAqC;QACrC,KAAK,MAAM,CAAC,GAAG,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAC3D,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;gBACnB,mBAAmB,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAA;YAC9C,CAAC;QACF,CAAC;QACD,0BAA0B;QAC1B,KAAK,MAAM,GAAG,IAAI,kBAAkB,EAAE,CAAC;YACtC,sBAAsB;YACtB,MAAM,gBAAgB,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAC3D,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,GAAG,CACtD,CAAA;YACD,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACvB,SAAQ;YACT,CAAC;YACD,mBAAmB,CAAC,IAAI,CACvB,IAAI,OAAO,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE;gBACrC,IAAI,CAAC;oBACJ,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,aAAa,GAAG,YAAY,CAAC,CAAA;oBACzD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,MAAM,CAAA;oBAC1B,OAAO,CAAC;wBACP,GAAG;wBACH,MAAM;qBACN,CAAC,CAAA;gBACH,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACd,MAAM,CAAC,GAAG,CAAC,CAAA;gBACZ,CAAC;YACF,CAAC,CAAC,CACF,CAAA;QACF,CAAC;QACD,MAAM,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAA;QACtC,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;IACjF,CAAC;IAED,KAAK,CAAC,IAAI;QACT,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE,uBAAuB,CAAC,CAAA;QACjD,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAA;QACvB,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAA;QACpB,gFAAgF;QAChF,8EAA8E;QAC9E,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAA,CAAC,8DAA8D;QACzF,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,EAAE,mCAAmC,EAAE,GAAG,CAAC,CAAA;QACpE,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,EAAE,sBAAsB,CAAC,CAAA;IAClD,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,OAAwB,EAAE,MAAoB;QACnE,OAAO,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;IAClD,CAAC;IAED,iEAAiE;IACzD,KAAK,CAAC,eAAe,CAAC,QAAuB,EAAE,MAAoB;QAC1E,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAA;QAC7B,MAAM,gBAAgB,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,MAAM,CAAA;QAC3D,IAAI,gBAAgB,KAAK,UAAU,EAAE,CAAC;YACrC,MAAM,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAA;QAChD,CAAC;QACD,IAAI,gBAAgB,KAAK,WAAW,EAAE,CAAC;YACtC,MAAM,UAAU,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;gBACxD,MAAM,WAAW,GAAG,KAAK,EAAE,UAAuB,EAAE,EAAE;oBACrD,IAAI,UAAU,CAAC,EAAE,KAAK,KAAK,CAAC,EAAE,EAAE,CAAC;wBAChC,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,GAAG,CAAC,WAAW,EAAE,WAAW,CAAC,CAAA;wBACrD,IAAI,UAAU,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;4BACnC,OAAO,EAAE,CAAA;wBACV,CAAC;6BAAM,CAAC;4BACP,MAAM,EAAE,CAAA;wBACT,CAAC;oBACF,CAAC;gBACF,CAAC,CAAA;gBACD,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC,WAAW,EAAE,WAAW,CAAC,CAAA;YACrD,CAAC,CAAC,CAAA;YACF,MAAM,UAAU,CAAA;QACjB,CAAC;IACF,CAAC;IAED,KAAK,CAAC,yBAAyB,CAAC,IAA2B,EAAE,OAAqC;QACjG,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,CAAA;QAC7C,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,yBAAyB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;QACnE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAA;QAChC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAA;QACpB,OAAO,MAAM,CAAA;IACd,CAAC;IAED,KAAK,CAAC,yBAAyB,CAAC,IAA2B,EAAE,OAAqC;QACjG,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,CAAA;QAC7C,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,yBAAyB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;QACnE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAA;QAChC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAA;QACpB,OAAO,MAAM,CAAA;IACd,CAAC;IAED,KAAK,CAAC,oBAAoB,CAAC,IAAsB,EAAE,OAA2B;QAC7E,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,CAAA;QAC7C,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,oBAAoB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;QAC9D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAA;QAChC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAA;QACpB,OAAO,MAAM,CAAA;IACd,CAAC;IAED,KAAK,CAAC,sBAAsB,CAAC,IAAwB,EAAE,OAA2B;QACjF,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,CAAA;QAC7C,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,sBAAsB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;QAChE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAA;QAChC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAA;QACpB,OAAO,MAAM,CAAA;IACd,CAAC;IAED,KAAK,CAAC,uBAAuB,CAAC,IAAyB,EAAE,OAAuC;QAC/F,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,CAAA;QAC7C,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,uBAAuB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;QACjE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAA;QAChC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAA;QACpB,OAAO,MAAM,CAAA;IACd,CAAC;IAED,KAAK,CAAC,sBAAsB,CAAC,IAAwB,EAAE,OAA2B;QACjF,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,CAAA;QAC7C,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,sBAAsB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;QAChE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAA;QAChC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAA;QACpB,OAAO,MAAM,CAAA;IACd,CAAC;IAED,KAAK,CAAC,uBAAuB,CAAC,IAAyB,EAAE,OAA2B;QACnF,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,CAAA;QAC7C,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,uBAAuB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;QACjE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAA;QAChC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAA;QACpB,OAAO,MAAM,CAAA;IACd,CAAC;IAED,SAAS;QACR,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAA;QACxC,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAA;QAC1C,OAAO;YACN,IAAI,EAAE,UAAU;YAChB,KAAK,EAAE,WAAW;SAClB,CAAA;IACF,CAAC;CACD"}

package/dist/standalone.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/standalone.js

@@ -0,0 +1,306 @@
+import { format as formatURL } from 'node:url';
+import { ModelHTTPServer } from './http.js';
+// Currently only used for debugging. Do not use.
+const serverOptions = {
+    listen: {
+        port: 3000,
+    },
+    log: 'debug',
+    concurrency: 2,
+    engines: {
+    // 'chat-with-vision': new ChatWithVisionEngine({
+    // 	imageToTextModel: 'florence2',
+    // 	chatModel: 'llama3-8b',
+    // }),
+    // 'voice-function-calling': new VoiceFunctionCallEngine({
+    // 	speechToTextModel: 'whisper-base',
+    // 	chatModel: 'functionary',
+    // }),
+    },
+    models: {
+    // 'sciphi-triplex': {
+    // 	url: 'https://huggingface.co/SciPhi/Triplex/blob/main/quantized_model-Q4_K_M.gguf',
+    // 	sha256: '6f8f6f1fca005640a1282dd0bd12512dedf22957d0c2135ba5e71583d33754fc',
+    // 	engine: 'node-llama-cpp',
+    // 	task: 'text-completion',
+    // },
+    // 'lite-mistral': {
+    // 	url: 'https://huggingface.co/bartowski/Lite-Mistral-150M-v2-Instruct-GGUF/resolve/main/Lite-Mistral-150M-v2-Instruct-Q8_0.gguf',
+    // 	sha256: 'b369c9b1ac20b66b2f94117d5cdc71d029a47a33948cefef9fe104615dcddfbd',
+    // 	engine: 'node-llama-cpp',
+    // 	task: 'text-completion',
+    // 	prepare: 'async',
+    // },
+    // 'gemma-9b': {
+    // 	url: 'https://huggingface.co/bartowski/gemma-2-9b-it-GGUF/blob/main/gemma-2-9b-it-Q4_K_M.gguf',
+    // 	sha256:
+    // 		'13b2a7b4115bbd0900162edcebe476da1ba1fc24e718e8b40d32f6e300f56dfe',
+    // 	engine: 'node-llama-cpp',
+    // 	task: 'text-completion',
+    // 	prepare: 'async',
+    // },
+    // 'llama3.1-8b': {
+    // 	url: 'https://huggingface.co/lmstudio-community/Meta-Llama-3.1-8B-Instruct-GGUF/blob/main/Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf',
+    // 	sha256:
+    // 		'2a4ca64e02e7126436cfdb066dd7311f2486eb487191910d3d000fde13826a4d',
+    // 	engine: 'node-llama-cpp',
+    // 	task: 'text-completion',
+    // 	prepare: 'async',
+    // },
+    // 'dolphin-nemo-12b': {
+    // 	url: 'https://huggingface.co/cognitivecomputations/dolphin-2.9.3-mistral-nemo-12b-gguf/blob/main/dolphin-2.9.3-mistral-nemo-Q4_K_M.gguf',
+    // 	sha256: '09f9114e06d88b791e322586cf28a844d2d0a3876d04d6deffe2dfb26616dd83',
+    // 	engine: 'node-llama-cpp',
+    // 	task: 'text-completion',
+    // 	prepare: 'async',
+    // },
+    // 'phi3-mini-4k': {
+    // 	task: 'text-completion',
+    // 	url: 'https://gpt4all.io/models/gguf/Phi-3-mini-4k-instruct.Q4_0.gguf',
+    // 	engine: 'gpt4all',
+    // 	maxInstances: 2,
+    // 	prepare: 'async',
+    // },
+    // 'mxbai-embed-large-v1': {
+    // 	url: 'https://huggingface.co/mixedbread-ai/mxbai-embed-large-v1',
+    // 	engine: 'transformers-js',
+    // 	task: 'embedding',
+    // 	prepare: 'blocking',
+    // 	device: {
+    // 		gpu: false,
+    // 	},
+    // },
+    // 'jina-clip-v1': {
+    // 	url: 'https://huggingface.co/jinaai/jina-clip-v1',
+    // 	engine: 'transformers-js',
+    // 	task: 'embedding',
+    // 	textModel: {
+    // 		modelClass: CLIPTextModelWithProjection,
+    // 	},
+    // 	visionModel: {
+    // 		processor: {
+    // 			url: 'https://huggingface.co/Xenova/clip-vit-base-patch32',
+    // 			// url: 'https://huggingface.co/Xenova/vit-base-patch16-224-in21k',
+    // 		},
+    // 		modelClass: CLIPVisionModelWithProjection,
+    // 	},
+    // 	prepare: 'blocking',
+    // 	device: {
+    // 		gpu: false,
+    // 	},
+    // },
+    // 'florence2-large': {
+    // 	url: 'https://huggingface.co/onnx-community/Florence-2-large-ft',
+    // 	engine: 'transformers-js',
+    // 	task: 'image-to-text',
+    // 	prepare: 'blocking',
+    // 	visionModel: {
+    // 		modelClass: Florence2ForConditionalGeneration,
+    // 		dtype: {
+    // 			embed_tokens: 'fp16',
+    // 			vision_encoder: 'fp32',
+    // 			encoder_model: 'fp16',
+    // 			decoder_model_merged: 'q4',
+    // 		},
+    // 	},
+    // 	device: {
+    // 		gpu: false,
+    // 	},
+    // },
+    // 'whisper-base': {
+    // 	url: 'https://huggingface.co/onnx-community/whisper-base',
+    // 	engine: 'transformers-js',
+    // 	task: 'speech-to-text',
+    // 	prepare: 'async',
+    // 	minInstances: 1,
+    // 	speechModel: {
+    // 		modelClass: WhisperForConditionalGeneration,
+    // 		dtype: {
+    // 			encoder_model: 'fp32', // 'fp16' works too
+    // 			decoder_model_merged: 'q4', // or 'fp32' ('fp16' is broken)
+    // 		},
+    // 	},
+    // 	device: {
+    // 		gpu: false,
+    // 	},
+    // },
+    // 'mistral-nemo-12b': {
+    // 	'url': 'https://huggingface.co/mradermacher/Mistral-Nemo-Instruct-2407-GGUF/blob/main/Mistral-Nemo-Instruct-2407.Q4_K_M.gguf',
+    // 	'sha256': '1ac4b6cdf0eeb1e2145f0097c6fd0a75df541e143f226a8ff25c8ae0e8dfff6f',
+    // 	'engine': 'node-llama-cpp',
+    // 	'task': 'text-completion',
+    // 	'prepare': 'async',
+    // },
+    // 'phi-3.5-mini': {
+    // 	url: 'https://huggingface.co/bartowski/Phi-3.5-mini-instruct-GGUF/blob/main/Phi-3.5-mini-instruct-Q4_K_M.gguf',
+    // 	sha256:
+    // 		'e4165e3a71af97f1b4820da61079826d8752a2088e313af0c7d346796c38eff5',
+    // 	engine: 'node-llama-cpp',
+    // 	task: 'text-completion',
+    // 	prepare: 'async',
+    // 	minInstances: 1,
+    // 	device: {
+    // 		gpu: 'vulkan',
+    // 	},
+    // },
+    // 'falcon-mamba-7b': {
+    // 	url: 'https://huggingface.co/mradermacher/falcon-mamba-7b-instruct-GGUF/blob/main/falcon-mamba-7b-instruct.Q4_K_M.gguf',
+    // 	sha256: 'f3357486034d89dd91fcefdb91bb1dfadfe0fd2969349a8a404e59d2bd3ad1b8',
+    // 	engine: 'node-llama-cpp',
+    // 	task: 'text-completion',
+    // 	prepare: 'async',
+    // },
+    // 'florence2-large': {
+    // 	url: 'https://huggingface.co/onnx-community/Florence-2-large-ft',
+    // 	engine: 'transformers-js',
+    // 	task: 'image-to-text',
+    // 	minInstances: 1,
+    // 	visionModel: {
+    // 		modelClass: Florence2ForConditionalGeneration,
+    // 		dtype: {
+    // 			embed_tokens: 'fp16',
+    // 			vision_encoder: 'fp32',
+    // 			encoder_model: 'fp16',
+    // 			decoder_model_merged: 'q4',
+    // 		},
+    // 	},
+    // 	device: {
+    // 		gpu: false,
+    // 	},
+    // },
+    // 'mxbai-embed-large-v1': {
+    // 	url: 'https://huggingface.co/mixedbread-ai/mxbai-embed-large-v1',
+    // 	engine: 'transformers-js',
+    // 	task: 'embedding',
+    // 	prepare: 'blocking',
+    // 	device: {
+    // 		gpu: false,
+    // 	},
+    // },
+    // 'functionary-3.2-small': {
+    // 	url: 'https://huggingface.co/meetkai/functionary-small-v3.2-GGUF/blob/main/functionary-small-v3.2.Q4_0.gguf',
+    // 	sha256: 'c0afdbbffa498a8490dea3401e34034ac0f2c6e337646513a7dbc04fcef1c3a4',
+    // 	engine: 'node-llama-cpp',
+    // 	task: 'text-completion',
+    // 	prepare: 'blocking',
+    // },
+    // 'flux-schnell': {
+    // 	url: 'https://huggingface.co/leejet/FLUX.1-schnell-gguf/resolve/main/flux1-schnell-q4_0.gguf',
+    // 	task: 'text-to-image',
+    // 	sha256: '4f30741d2bfc786c92934ce925fcb0a43df3441e76504b797c3d5d5f0878fa6f',
+    // 	engine: 'stable-diffusion-cpp',
+    // 	prepare: 'blocking',
+    // 	diffusionModel: true,
+    // 	samplingMethod: 'euler_a',
+    // 	vae: {
+    // 		url: 'https://huggingface.co/second-state/FLUX.1-schnell-GGUF/blob/main/ae.safetensors',
+    // 	},
+    // 	clipL: {
+    // 		url: 'https://huggingface.co/second-state/FLUX.1-schnell-GGUF/blob/main/clip_l.safetensors',
+    // 	},
+    // 	t5xxl: {
+    // 		// url: 'https://huggingface.co/second-state/FLUX.1-schnell-GGUF/blob/main/t5xxl_fp16.safetensors',
+    // 		url: 'https://huggingface.co/second-state/FLUX.1-schnell-GGUF/blob/main/t5xxl-Q8_0.gguf',
+    // 	},
+    // },
+    // 'sd-3.5-turbo': {
+    // 	url: 'https://huggingface.co/stduhpf/SD3.5-Large-Turbo-GGUF-mixed-sdcpp/blob/main/legacy/sd3.5_large_turbo-q4_0.gguf',
+    // 	sha256: '52495d9c4356065a1378a93c9556a9eb465e10014ba9ce364512674267405bb2',
+    // 	engine: 'stable-diffusion-cpp',
+    // 	task: 'text-to-image',
+    // 	prepare: 'blocking',
+    // 	samplingMethod: 'euler',
+    // 	clipG: {
+    // 		url: 'https://huggingface.co/Comfy-Org/stable-diffusion-3.5-fp8/blob/main/text_encoders/clip_g.safetensors',
+    // 		sha256: 'ec310df2af79c318e24d20511b601a591ca8cd4f1fce1d8dff822a356bcdb1f4',
+    // 	},
+    // 	clipL: {
+    // 		url: 'https://huggingface.co/Comfy-Org/stable-diffusion-3.5-fp8/blob/main/text_encoders/clip_l.safetensors',
+    // 		sha256: '660c6f5b1abae9dc498ac2d21e1347d2abdb0cf6c0c0c8576cd796491d9a6cdd',
+    // 	},
+    // 	t5xxl: {
+    // 		url: 'https://huggingface.co/Comfy-Org/stable-diffusion-3.5-fp8/blob/main/text_encoders/t5xxl_fp8_e4m3fn.safetensors',
+    // 		sha256: '7d330da4816157540d6bb7838bf63a0f02f573fc48ca4d8de34bb0cbfd514f09',
+    // 	},
+    // },
+    // 'trocr-printed': {
+    // 	url: 'https://huggingface.co/Xenova/trocr-small-printed',
+    // 	engine: 'transformers-js',
+    // 	task: 'image-to-text',
+    // 	prepare: 'blocking',
+    // 	minInstances: 1,
+    // 	// textModel: {
+    // 	//   modelClass: TrOCRPreTrainedModel,
+    // 	// 	processorClass: DeiTFeatureExtractor,
+    // 	// },
+    // 	device: {
+    // 		gpu: false,
+    // 	},
+    // },
+    // 'sdxl-turbo': {
+    // 	url: 'https://huggingface.co/stabilityai/sdxl-turbo/blob/main/sd_xl_turbo_1.0_fp16.safetensors',
+    // 	sha256:
+    // 		'e869ac7d6942cb327d68d5ed83a40447aadf20e0c3358d98b2cc9e270db0da26',
+    // 	engine: 'stable-diffusion-cpp',
+    // 	task: 'image-to-image',
+    // 	prepare: 'blocking',
+    // 	samplingMethod: 'euler',
+    // 	vae: {
+    // 		url: 'https://huggingface.co/madebyollin/sdxl-vae-fp16-fix/blob/main/sdxl.vae.safetensors',
+    // 		sha256:
+    // 			'235745af8d86bf4a4c1b5b4f529868b37019a10f7c0b2e79ad0abca3a22bc6e1',
+    // 	},
+    // },
+    // 'flux-light': {
+    // 	url: 'https://huggingface.co/city96/flux.1-lite-8B-alpha-gguf/blob/main/flux.1-lite-8B-alpha-Q8_0.gguf',
+    // 	sha256:
+    // 		'efc598d62123f2fdfd682948f533fee081f7fb1295b14d002ac1e66cae5f01a5',
+    // 	engine: 'stable-diffusion-cpp',
+    // 	task: 'image-to-image',
+    // 	prepare: 'blocking',
+    // },
+    // 'sd-3-medium': {
+    // 	url: 'https://huggingface.co/second-state/stable-diffusion-3-medium-GGUF/blob/main/sd3-medium-Q8_0.gguf',
+    // 	sha256: '7e34dfeb71f8cdbc8338677b63a444897cf4c5692ab4c1d98f04cbba6751885a',
+    // 	engine: 'stable-diffusion-cpp',
+    // 	task: 'text-to-image',
+    // 	prepare: 'async',
+    // },
+    // 'sd-1.5': {
+    // 	url: 'https://huggingface.co/second-state/stable-diffusion-v1-5-GGUF/blob/main/stable-diffusion-v1-5-pruned-emaonly-f32.gguf',
+    // 	sha256: '52c7ca39d8d48d6f44fa4ff2c44569f3c924d92311108cb38492958350d48ff8',
+    // 	engine: 'stable-diffusion-cpp',
+    // 	task: 'text-to-image',
+    // 	prepare: 'async',
+    // },
+    // 'llama-3.2-3b': {
+    // 	url: 'https://huggingface.co/bartowski/Llama-3.2-3B-Instruct-GGUF/blob/main/Llama-3.2-3B-Instruct-Q6_K_L.gguf',
+    // 	sha256:
+    // 		'c542b14ec07b8b3cb8d777e1a68ee5aabb964167719466d4c685c29fcfd04900',
+    // 	engine: 'node-llama-cpp',
+    // 	task: 'text-completion',
+    // 	prepare: 'blocking',
+    // },
+    },
+};
+async function main() {
+    const server = new ModelHTTPServer(serverOptions);
+    await server.start();
+    const { address, port } = server.httpServer.address();
+    const hostname = address === '' || address === '::' ? 'localhost' : address;
+    const url = formatURL({
+        protocol: 'http',
+        hostname,
+        port,
+        pathname: '/',
+    });
+    console.log(`Server listening at ${url}`);
+}
+main().catch((err) => {
+    console.error(err);
+    process.exit(1);
+});
+process.on('unhandledRejection', (err) => {
+    console.error('Unhandled rejection:', err);
+});
+//# sourceMappingURL=standalone.js.map

package/dist/standalone.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"standalone.js","sourceRoot":"","sources":["../src/standalone.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,IAAI,SAAS,EAAE,MAAM,UAAU,CAAA;AAC9C,OAAO,EAAE,eAAe,EAA0B,MAAM,kBAAkB,CAAA;AAY1E,iDAAiD;AACjD,MAAM,aAAa,GAA2B;IAC7C,MAAM,EAAE;QACP,IAAI,EAAE,IAAI;KACV;IACD,GAAG,EAAE,OAAO;IACZ,WAAW,EAAE,CAAC;IACd,OAAO,EAAE;IACR,iDAAiD;IACjD,kCAAkC;IAClC,2BAA2B;IAC3B,MAAM;IACN,0DAA0D;IAC1D,sCAAsC;IACtC,6BAA6B;IAC7B,MAAM;KACN;IACD,MAAM,EAAE;IACP,sBAAsB;IACtB,uFAAuF;IACvF,+EAA+E;IAC/E,6BAA6B;IAC7B,4BAA4B;IAC5B,KAAK;IACL,oBAAoB;IACpB,oIAAoI;IACpI,+EAA+E;IAC/E,6BAA6B;IAC7B,4BAA4B;IAC5B,qBAAqB;IACrB,KAAK;IACL,gBAAgB;IAChB,mGAAmG;IACnG,WAAW;IACX,wEAAwE;IACxE,6BAA6B;IAC7B,4BAA4B;IAC5B,qBAAqB;IACrB,KAAK;IACL,mBAAmB;IACnB,sIAAsI;IACtI,WAAW;IACX,wEAAwE;IACxE,6BAA6B;IAC7B,4BAA4B;IAC5B,qBAAqB;IACrB,KAAK;IACL,wBAAwB;IACxB,6IAA6I;IAC7I,+EAA+E;IAC/E,6BAA6B;IAC7B,4BAA4B;IAC5B,qBAAqB;IACrB,KAAK;IACL,oBAAoB;IACpB,4BAA4B;IAC5B,2EAA2E;IAC3E,sBAAsB;IACtB,oBAAoB;IACpB,qBAAqB;IACrB,KAAK;IACL,4BAA4B;IAC5B,qEAAqE;IACrE,8BAA8B;IAC9B,sBAAsB;IACtB,wBAAwB;IACxB,aAAa;IACb,gBAAgB;IAChB,MAAM;IACN,KAAK;IACL,oBAAoB;IACpB,sDAAsD;IACtD,8BAA8B;IAC9B,sBAAsB;IACtB,gBAAgB;IAChB,6CAA6C;IAC7C,MAAM;IACN,kBAAkB;IAClB,iBAAiB;IACjB,iEAAiE;IACjE,yEAAyE;IACzE,OAAO;IACP,+CAA+C;IAC/C,MAAM;IACN,wBAAwB;IACxB,aAAa;IACb,gBAAgB;IAChB,MAAM;IACN,KAAK;IACL,uBAAuB;IACvB,qEAAqE;IACrE,8BAA8B;IAC9B,0BAA0B;IAC1B,wBAAwB;IACxB,kBAAkB;IAClB,mDAAmD;IACnD,aAAa;IACb,2BAA2B;IAC3B,6BAA6B;IAC7B,4BAA4B;IAC5B,iCAAiC;IACjC,OAAO;IACP,MAAM;IACN,aAAa;IACb,gBAAgB;IAChB,MAAM;IACN,KAAK;IACL,oBAAoB;IACpB,8DAA8D;IAC9D,8BAA8B;IAC9B,2BAA2B;IAC3B,qBAAqB;IACrB,oBAAoB;IACpB,kBAAkB;IAClB,iDAAiD;IACjD,aAAa;IACb,gDAAgD;IAChD,iEAAiE;IACjE,OAAO;IACP,MAAM;IACN,aAAa;IACb,gBAAgB;IAChB,MAAM;IACN,KAAK;IACL,wBAAwB;IACxB,kIAAkI;IAClI,iFAAiF;IACjF,+BAA+B;IAC/B,8BAA8B;IAC9B,uBAAuB;IACvB,KAAK;IACL,oBAAoB;IACpB,mHAAmH;IACnH,WAAW;IACX,wEAAwE;IACxE,6BAA6B;IAC7B,4BAA4B;IAC5B,qBAAqB;IACrB,oBAAoB;IACpB,aAAa;IACb,mBAAmB;IACnB,MAAM;IACN,KAAK;IACL,uBAAuB;IACvB,4HAA4H;IAC5H,+EAA+E;IAC/E,6BAA6B;IAC7B,4BAA4B;IAC5B,qBAAqB;IACrB,KAAK;IACL,uBAAuB;IACvB,qEAAqE;IACrE,8BAA8B;IAC9B,0BAA0B;IAC1B,oBAAoB;IACpB,kBAAkB;IAClB,mDAAmD;IACnD,aAAa;IACb,2BAA2B;IAC3B,6BAA6B;IAC7B,4BAA4B;IAC5B,iCAAiC;IACjC,OAAO;IACP,MAAM;IACN,aAAa;IACb,gBAAgB;IAChB,MAAM;IACN,KAAK;IACL,4BAA4B;IAC5B,qEAAqE;IACrE,8BAA8B;IAC9B,sBAAsB;IACtB,wBAAwB;IACxB,aAAa;IACb,gBAAgB;IAChB,MAAM;IACN,KAAK;IACL,6BAA6B;IAC7B,iHAAiH;IACjH,+EAA+E;IAC/E,6BAA6B;IAC7B,4BAA4B;IAC5B,wBAAwB;IACxB,KAAK;IACL,oBAAoB;IACpB,kGAAkG;IAClG,0BAA0B;IAC1B,+EAA+E;IAC/E,mCAAmC;IACnC,wBAAwB;IACxB,yBAAyB;IACzB,8BAA8B;IAC9B,UAAU;IACV,6FAA6F;IAC7F,MAAM;IACN,YAAY;IACZ,iGAAiG;IACjG,MAAM;IACN,YAAY;IACZ,wGAAwG;IACxG,8FAA8F;IAC9F,MAAM;IACN,KAAK;IACL,oBAAoB;IACpB,0HAA0H;IAC1H,+EAA+E;IAC/E,mCAAmC;IACnC,0BAA0B;IAC1B,wBAAwB;IACxB,4BAA4B;IAC5B,YAAY;IACZ,iHAAiH;IACjH,gFAAgF;IAChF,MAAM;IACN,YAAY;IACZ,iHAAiH;IACjH,gFAAgF;IAChF,MAAM;IACN,YAAY;IACZ,2HAA2H;IAC3H,gFAAgF;IAChF,MAAM;IACN,KAAK;IACL,qBAAqB;IACrB,6DAA6D;IAC7D,8BAA8B;IAC9B,0BAA0B;IAC1B,wBAAwB;IACxB,oBAAoB;IACpB,mBAAmB;IACnB,0CAA0C;IAC1C,6CAA6C;IAC7C,SAAS;IACT,aAAa;IACb,gBAAgB;IAChB,MAAM;IACN,KAAK;IACL,kBAAkB;IAClB,oGAAoG;IACpG,WAAW;IACX,wEAAwE;IACxE,mCAAmC;IACnC,2BAA2B;IAC3B,wBAAwB;IACxB,4BAA4B;IAC5B,UAAU;IACV,gGAAgG;IAChG,YAAY;IACZ,yEAAyE;IACzE,MAAM;IACN,KAAK;IACL,kBAAkB;IAClB,4GAA4G;IAC5G,WAAW;IACX,wEAAwE;IACxE,mCAAmC;IACnC,2BAA2B;IAC3B,wBAAwB;IACxB,KAAK;IACL,mBAAmB;IACnB,6GAA6G;IAC7G,+EAA+E;IAC/E,mCAAmC;IACnC,0BAA0B;IAC1B,qBAAqB;IACrB,KAAK;IACL,cAAc;IACd,kIAAkI;IAClI,+EAA+E;IAC/E,mCAAmC;IACnC,0BAA0B;IAC1B,qBAAqB;IACrB,KAAK;IACL,oBAAoB;IACpB,mHAAmH;IACnH,WAAW;IACX,wEAAwE;IACxE,6BAA6B;IAC7B,4BAA4B;IAC5B,wBAAwB;IACxB,KAAK;KACL;CACD,CAAA;AAED,KAAK,UAAU,IAAI;IAClB,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC,aAAa,CAAC,CAAA;IACjD,MAAM,MAAM,CAAC,KAAK,EAAE,CAAA;IACpB,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,EAAiB,CAAA;IACpE,MAAM,QAAQ,GAAG,OAAO,KAAK,EAAE,IAAI,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,OAAO,CAAA;IAC3E,MAAM,GAAG,GAAG,SAAS,CAAC;QACrB,QAAQ,EAAE,MAAM;QAChB,QAAQ;QACR,IAAI;QACJ,QAAQ,EAAE,GAAG;KACb,CAAC,CAAA;IACF,OAAO,CAAC,GAAG,CAAC,uBAAuB,GAAG,EAAE,CAAC,CAAA;AAC1C,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAU,EAAE,EAAE;IAC3B,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;IAClB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;AAChB,CAAC,CAAC,CAAA;AAEF,OAAO,CAAC,EAAE,CAAC,oBAAoB,EAAE,CAAC,GAAG,EAAE,EAAE;IACxC,OAAO,CAAC,KAAK,CAAC,sBAAsB,EAAE,GAAG,CAAC,CAAA;AAC3C,CAAC,CAAC,CAAA"}

package/dist/store.d.ts

@@ -0,0 +1,60 @@
+import PQueue from 'p-queue';
+import { FileDownloadProgress, ModelConfig, ModelEngine } from './types/index.js';
+import { Logger, LogLevel } from './lib/logger.js';
+export interface StoredModel extends ModelConfig {
+    meta?: unknown;
+    downloads?: Map<string, DownloadTracker>;
+    status: 'unloaded' | 'preparing' | 'ready' | 'error';
+}
+export interface ModelStoreOptions {
+    modelsCachePath: string;
+    models: Record<string, ModelConfig>;
+    prepareConcurrency?: number;
+    log?: Logger | LogLevel;
+}
+export declare class ModelStore {
+    prepareQueue: PQueue;
+    models: Record<string, StoredModel>;
+    engines?: Record<string, ModelEngine>;
+    private prepareController;
+    private modelsCachePath;
+    private log;
+    constructor(options: ModelStoreOptions);
+    init(engines: Record<string, ModelEngine>): Promise<void>;
+    dispose(): void;
+    private onDownloadProgress;
+    prepareModel(modelId: string, signal?: AbortSignal): Promise<void>;
+    getStatus(): {
+        [k: string]: {
+            engine: (string & {}) | import("./engines").BuiltInEngineName;
+            device: {
+                gpu?: boolean | "auto" | (string & {});
+            } | undefined;
+            minInstances: number;
+            maxInstances: number;
+            status: "error" | "unloaded" | "preparing" | "ready";
+            downloads: any;
+        };
+    };
+}
+type ProgressState = {
+    loadedBytes: number;
+    totalBytes: number;
+    timestamp: number;
+};
+type DownloadStatus = {
+    percent: number;
+    speed: number;
+    etaSeconds: number;
+    loadedBytes: number;
+    totalBytes: number;
+};
+declare class DownloadTracker {
+    progressBuffer: ProgressState[];
+    private timeWindow;
+    constructor(timeWindow?: number);
+    pushProgress({ loadedBytes, totalBytes }: FileDownloadProgress): void;
+    private cleanup;
+    getStatus(): DownloadStatus | null;
+}
+export {};