inference-server 1.0.0-beta.19

package/dist/engines/transformers-js/validateModelFiles.js

@@ -0,0 +1,133 @@
+import fs from 'node:fs';
+import { AutoModel, AutoProcessor, AutoTokenizer } from '@huggingface/transformers';
+import { resolveModelFileLocation } from '../../lib/resolveModelFileLocation.js';
+import { parseHuggingfaceModelIdAndBranch, remoteFileExists } from './util.js';
+async function validateModel(modelOpts, config, modelPath) {
+    const modelClass = modelOpts.modelClass ?? AutoModel;
+    const device = config.device?.gpu ? 'gpu' : 'cpu';
+    try {
+        const model = await modelClass.from_pretrained(modelPath, {
+            local_files_only: true,
+            device: device,
+            dtype: modelOpts.dtype || 'fp32',
+        });
+        await model.dispose();
+    }
+    catch (error) {
+        return `Failed to load model (${error})`;
+    }
+    return undefined;
+}
+async function validateTokenizer(modelOpts, config, modelPath) {
+    const tokenizerClass = modelOpts.tokenizerClass ?? AutoTokenizer;
+    try {
+        await tokenizerClass.from_pretrained(modelPath, {
+            local_files_only: true,
+        });
+    }
+    catch (error) {
+        return `Failed to load tokenizer (${error})`;
+    }
+    return undefined;
+}
+async function validateProcessor(modelOpts, config, modelPath) {
+    const processorClass = modelOpts.processorClass ?? AutoProcessor;
+    try {
+        if (modelOpts.processor) {
+            const processorPath = resolveModelFileLocation({
+                url: modelOpts.processor.url,
+                filePath: modelOpts.processor.file,
+                modelsCachePath: config.modelsCachePath,
+            });
+            await processorClass.from_pretrained(processorPath, {
+                local_files_only: true,
+            });
+        }
+        else {
+            if (modelOpts.processorClass) {
+                await processorClass.from_pretrained(modelPath, {
+                    local_files_only: true,
+                });
+            }
+            else if (config.url) {
+                const { branch } = parseHuggingfaceModelIdAndBranch(config.url);
+                const [hasProcessor, hasPreprocessor] = await Promise.all([
+                    remoteFileExists(`${config.url}/blob/${branch}/processor_config.json`),
+                    remoteFileExists(`${config.url}/blob/${branch}/preprocessor_config.json`),
+                ]);
+                if (hasProcessor || hasPreprocessor) {
+                    await processorClass.from_pretrained(modelPath, {
+                        local_files_only: true,
+                    });
+                }
+            }
+        }
+    }
+    catch (error) {
+        return `Failed to load processor (${error})`;
+    }
+    return undefined;
+}
+export async function validateModelFiles(config) {
+    if (!fs.existsSync(config.location)) {
+        return {
+            message: `model directory does not exist: ${config.location}`,
+        };
+    }
+    let modelPath = config.location;
+    if (!modelPath.endsWith('/')) {
+        modelPath += '/';
+    }
+    const validateModelComponents = async (modelOpts) => {
+        const componentValidationPromises = [
+            validateModel(modelOpts, config, modelPath),
+            validateTokenizer(modelOpts, config, modelPath),
+            validateProcessor(modelOpts, config, modelPath),
+        ];
+        // if (modelOpts.processor) {
+        // 	componentValidationPromises.push(validateProcessor(modelOpts, config, modelPath))
+        // }
+        const [model, tokenizer, processor] = await Promise.all(componentValidationPromises);
+        const result = {};
+        if (model)
+            result.model = model;
+        if (tokenizer)
+            result.tokenizer = tokenizer;
+        if (processor)
+            result.processor = processor;
+        return result;
+    };
+    const modelValidationPromises = {};
+    const noModelConfigured = !config.textModel && !config.visionModel && !config.speechModel;
+    if (config.textModel || noModelConfigured) {
+        modelValidationPromises.textModel = validateModelComponents(config.textModel || {});
+    }
+    if (config.visionModel) {
+        modelValidationPromises.visionModel = validateModelComponents(config.visionModel);
+    }
+    if (config.speechModel) {
+        modelValidationPromises.speechModel = validateModelComponents(config.speechModel);
+    }
+    await Promise.all(Object.values(modelValidationPromises));
+    const validationErrors = {};
+    const textModelErrors = await modelValidationPromises.textModel;
+    if (textModelErrors && Object.keys(textModelErrors).length) {
+        validationErrors.textModel = textModelErrors;
+    }
+    const visionModelErrors = await modelValidationPromises.visionModel;
+    if (visionModelErrors && Object.keys(visionModelErrors).length) {
+        validationErrors.visionModel = visionModelErrors;
+    }
+    const speechModelErrors = await modelValidationPromises.speechModel;
+    if (speechModelErrors && Object.keys(speechModelErrors).length) {
+        validationErrors.speechModel = speechModelErrors;
+    }
+    if (Object.keys(validationErrors).length > 0) {
+        return {
+            message: 'Failed to validate model components',
+            errors: validationErrors,
+        };
+    }
+    return undefined;
+}
+//# sourceMappingURL=validateModelFiles.js.map

package/dist/engines/transformers-js/validateModelFiles.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"validateModelFiles.js","sourceRoot":"","sources":["../../../src/engines/transformers-js/validateModelFiles.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAA;AACxB,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAA;AAEnF,OAAO,EAAE,wBAAwB,EAAE,MAAM,0CAA0C,CAAA;AAEnF,OAAO,EAAE,gCAAgC,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAA;AAE9E,KAAK,UAAU,aAAa,CAC3B,SAA8B,EAC9B,MAAiC,EACjC,SAAiB;IAEjB,MAAM,UAAU,GAAG,SAAS,CAAC,UAAU,IAAI,SAAS,CAAA;IACpD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAA;IACjD,IAAI,CAAC;QACJ,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,eAAe,CAAC,SAAS,EAAE;YACzD,gBAAgB,EAAE,IAAI;YACtB,MAAM,EAAE,MAAM;YACd,KAAK,EAAE,SAAS,CAAC,KAAK,IAAI,MAAM;SAChC,CAAC,CAAA;QACF,MAAM,KAAK,CAAC,OAAO,EAAE,CAAA;IACtB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,OAAO,yBAAyB,KAAK,GAAG,CAAA;IACzC,CAAC;IACD,OAAO,SAAS,CAAA;AACjB,CAAC;AAED,KAAK,UAAU,iBAAiB,CAC/B,SAA8B,EAC9B,MAAiC,EACjC,SAAiB;IAEjB,MAAM,cAAc,GAAG,SAAS,CAAC,cAAc,IAAI,aAAa,CAAA;IAChE,IAAI,CAAC;QACJ,MAAM,cAAc,CAAC,eAAe,CAAC,SAAS,EAAE;YAC/C,gBAAgB,EAAE,IAAI;SACtB,CAAC,CAAA;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,OAAO,6BAA6B,KAAK,GAAG,CAAA;IAC7C,CAAC;IACD,OAAO,SAAS,CAAA;AACjB,CAAC;AAED,KAAK,UAAU,iBAAiB,CAC/B,SAA8B,EAC9B,MAAiC,EACjC,SAAiB;IAEjB,MAAM,cAAc,GAAG,SAAS,CAAC,cAAc,IAAI,aAAa,CAAA;IAChE,IAAI,CAAC;QACJ,IAAI,SAAS,CAAC,SAAS,EAAE,CAAC;YACzB,MAAM,aAAa,GAAG,wBAAwB,CAAC;gBAC9C,GAAG,EAAE,SAAS,CAAC,SAAS,CAAC,GAAG;gBAC5B,QAAQ,EAAE,SAAS,CAAC,SAAS,CAAC,IAAI;gBAClC,eAAe,EAAE,MAAM,CAAC,eAAe;aACvC,CAAC,CAAA;YACF,MAAM,cAAc,CAAC,eAAe,CAAC,aAAa,EAAE;gBACnD,gBAAgB,EAAE,IAAI;aACtB,CAAC,CAAA;QACH,CAAC;aAAM,CAAC;YACP,IAAI,SAAS,CAAC,cAAc,EAAE,CAAC;gBAC9B,MAAM,cAAc,CAAC,eAAe,CAAC,SAAS,EAAE;oBAC/C,gBAAgB,EAAE,IAAI;iBACtB,CAAC,CAAA;YACH,CAAC;iBAAM,IAAI,MAAM,CAAC,GAAG,EAAE,CAAC;gBACvB,MAAM,EAAE,MAAM,EAAE,GAAG,gCAAgC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;gBAC/D,MAAM,CAAC,YAAY,EAAE,eAAe,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;oBACzD,gBAAgB,CAAC,GAAG,MAAM,CAAC,GAAG,SAAS,MAAM,wBAAwB,CAAC;oBACtE,gBAAgB,CAAC,GAAG,MAAM,CAAC,GAAG,SAAS,MAAM,2BAA2B,CAAC;iBACzE,CAAC,CAAA;gBACF,IAAI,YAAY,IAAI,eAAe,EAAE,CAAC;oBACrC,MAAM,cAAc,CAAC,eAAe,CAAC,SAAS,EAAE;wBAC/C,gBAAgB,EAAE,IAAI;qBACtB,CAAC,CAAA;gBACH,CAAC;YACF,CAAC;QACF,CAAC;IACF,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,OAAO,6BAA6B,KAAK,GAAG,CAAA;IAC7C,CAAC;IACD,OAAO,SAAS,CAAA;AACjB,CAAC;AAmBD,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACvC,MAAiC;IAEjC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QACrC,OAAO;YACN,OAAO,EAAE,mCAAmC,MAAM,CAAC,QAAQ,EAAE;SAC7D,CAAA;IACF,CAAC;IAED,IAAI,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAA;IAC/B,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC9B,SAAS,IAAI,GAAG,CAAA;IACjB,CAAC;IAED,MAAM,uBAAuB,GAAG,KAAK,EAAE,SAA8B,EAAE,EAAE;QACxE,MAAM,2BAA2B,GAAG;YACnC,aAAa,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,CAAC;YAC3C,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,CAAC;YAC/C,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,CAAC;SAC/C,CAAA;QAGD,6BAA6B;QAC7B,qFAAqF;QACrF,IAAI;QACJ,MAAM,CAAC,KAAK,EAAE,SAAS,EAAE,SAAS,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAA;QACpF,MAAM,MAAM,GAA8B,EAAE,CAAA;QAC5C,IAAI,KAAK;YAAE,MAAM,CAAC,KAAK,GAAG,KAAK,CAAA;QAC/B,IAAI,SAAS;YAAE,MAAM,CAAC,SAAS,GAAG,SAAS,CAAA;QAC3C,IAAI,SAAS;YAAE,MAAM,CAAC,SAAS,GAAG,SAAS,CAAA;QAC3C,OAAO,MAAM,CAAA;IACd,CAAC,CAAA;IAED,MAAM,uBAAuB,GAAQ,EAAE,CAAA;IACvC,MAAM,iBAAiB,GAAG,CAAC,MAAM,CAAC,SAAS,IAAI,CAAC,MAAM,CAAC,WAAW,IAAI,CAAC,MAAM,CAAC,WAAW,CAAA;IACzF,IAAI,MAAM,CAAC,SAAS,IAAI,iBAAiB,EAAE,CAAC;QAC3C,uBAAuB,CAAC,SAAS,GAAG,uBAAuB,CAAC,MAAM,CAAC,SAAS,IAAI,EAAE,CAAC,CAAA;IACpF,CAAC;IACD,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;QACxB,uBAAuB,CAAC,WAAW,GAAG,uBAAuB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA;IAClF,CAAC;IACD,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;QACxB,uBAAuB,CAAC,WAAW,GAAG,uBAAuB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA;IAClF,CAAC;IAED,MAAM,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,uBAAuB,CAAC,CAAC,CAAA;IACzD,MAAM,gBAAgB,GAA0B,EAAE,CAAA;IAClD,MAAM,eAAe,GAAG,MAAM,uBAAuB,CAAC,SAAS,CAAA;IAC/D,IAAI,eAAe,IAAI,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,MAAM,EAAE,CAAC;QAC5D,gBAAgB,CAAC,SAAS,GAAG,eAAe,CAAA;IAC7C,CAAC;IACD,MAAM,iBAAiB,GAAG,MAAM,uBAAuB,CAAC,WAAW,CAAA;IACnE,IAAI,iBAAiB,IAAI,MAAM,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,MAAM,EAAE,CAAC;QAChE,gBAAgB,CAAC,WAAW,GAAG,iBAAiB,CAAA;IACjD,CAAC;IACD,MAAM,iBAAiB,GAAG,MAAM,uBAAuB,CAAC,WAAW,CAAA;IACnE,IAAI,iBAAiB,IAAI,MAAM,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,MAAM,EAAE,CAAC;QAChE,gBAAgB,CAAC,WAAW,GAAG,iBAAiB,CAAA;IACjD,CAAC;IAED,IAAI,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9C,OAAO;YACN,OAAO,EAAE,qCAAqC;YAC9C,MAAM,EAAE,gBAAgB;SACxB,CAAA;IACF,CAAC;IACD,OAAO,SAAS,CAAA;AACjB,CAAC"}

package/dist/experiments/ChatWithVision.d.ts

@@ -0,0 +1,11 @@
+import { EngineChatCompletionArgs, ModelEngine } from '../types/index.js';
+import { CustomEngine } from '../engines/index.js';
+export declare class ChatWithVisionEngine extends CustomEngine implements ModelEngine {
+    imageToTextModel: string;
+    chatModel: string;
+    constructor({ imageToTextModel, chatModel }: {
+        imageToTextModel: string;
+        chatModel: string;
+    });
+    processChatCompletionTask(args: EngineChatCompletionArgs): Promise<import("../types/index.js").EngineChatCompletionResult>;
+}

package/dist/experiments/ChatWithVision.js

@@ -0,0 +1,91 @@
+import { CustomEngine } from '../engines/index.js';
+// an experimental engine that replaces images with their descriptions before passing them to a chat model
+export class ChatWithVisionEngine extends CustomEngine {
+    imageToTextModel;
+    chatModel;
+    constructor({ imageToTextModel, chatModel }) {
+        super();
+        this.imageToTextModel = imageToTextModel;
+        this.chatModel = chatModel;
+    }
+    async processChatCompletionTask(args) {
+        const imageTextPromises = [];
+        const imageToTextModel = await this.pool.requestInstance({
+            model: this.imageToTextModel,
+        });
+        const messagesWithImageDescriptions = [...args.request.messages];
+        for (let m = 0; m < messagesWithImageDescriptions.length; m++) {
+            const message = messagesWithImageDescriptions[m];
+            if (!Array.isArray(message.content)) {
+                continue;
+            }
+            for (let p = 0; p < message.content.length; p++) {
+                const contentPart = message.content[p];
+                if (contentPart.type !== 'image') {
+                    continue;
+                }
+                imageTextPromises.push(new Promise(async (resolve, reject) => {
+                    // Florence2 prompt doc
+                    // "task_prompts_without_inputs": {
+                    // 	"<OCR>": "What is the text in the image?",
+                    // 	"<OCR_WITH_REGION>": "What is the text in the image, with regions?",
+                    // 	"<CAPTION>": "What does the image describe?",
+                    // 	"<DETAILED_CAPTION>": "Describe in detail what is shown in the image.",
+                    // 	"<MORE_DETAILED_CAPTION>": "Describe with a paragraph what is shown in the image.",
+                    // 	"<OD>": "Locate the objects with category name in the image.",
+                    // 	"<DENSE_REGION_CAPTION>": "Locate the objects in the image, with their descriptions.",
+                    // 	"<REGION_PROPOSAL>": "Locate the region proposals in the image."
+                    // },
+                    // "task_prompts_with_input": {
+                    // 	"<CAPTION_TO_PHRASE_GROUNDING>": "Locate the phrases in the caption: {input}",
+                    // 	"<REFERRING_EXPRESSION_SEGMENTATION>": "Locate {input} in the image with mask",
+                    // 	"<REGION_TO_SEGMENTATION>": "What is the polygon mask of region {input}",
+                    // 	"<OPEN_VOCABULARY_DETECTION>": "Locate {input} in the image.",
+                    // 	"<REGION_TO_CATEGORY>": "What is the region {input}?",
+                    // 	"<REGION_TO_DESCRIPTION>": "What does the region {input} describe?",
+                    // 	"<REGION_TO_OCR>": "What text is in the region {input}?"
+                    // }
+                    // const imageData = await fetch(contentPart.image.url).then((res) => res.arrayBuffer())
+                    const task = imageToTextModel.instance.processImageToTextTask({
+                        model: this.imageToTextModel,
+                        // url: contentPart.url,
+                        image: contentPart.image,
+                        prompt: 'What does the image describe?',
+                    });
+                    const result = await task.result;
+                    resolve({
+                        text: result.text,
+                        messageIndex: m,
+                        contentPartIndex: p,
+                    });
+                }));
+            }
+        }
+        const imageTextResults = await Promise.all(imageTextPromises);
+        imageToTextModel.release();
+        console.debug('Image text results', imageTextResults);
+        for (const imageTextResult of imageTextResults) {
+            const { text, messageIndex, contentPartIndex } = imageTextResult;
+            const message = messagesWithImageDescriptions[messageIndex];
+            // if ('type' in message.content[contentPartIndex]) {
+            // message.content[contentPartIndex].type = 'text'
+            // @ts-ignore
+            message.content[contentPartIndex] = {
+                type: 'text',
+                text: `User uploaded image: ${text}`,
+            };
+        }
+        const chatRequest = { ...args.request, messages: messagesWithImageDescriptions, model: this.chatModel };
+        const chatModel = await this.pool.requestInstance(chatRequest);
+        const task = chatModel.instance.processChatCompletionTask(chatRequest, {
+            onChunk: (chunk) => {
+                if (args.onChunk)
+                    args.onChunk(chunk);
+            },
+        });
+        const result = await task.result;
+        chatModel.release();
+        return result;
+    }
+}
+//# sourceMappingURL=ChatWithVision.js.map

package/dist/experiments/ChatWithVision.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ChatWithVision.js","sourceRoot":"","sources":["../../src/experiments/ChatWithVision.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAA;AAExD,0GAA0G;AAE1G,MAAM,OAAO,oBAAqB,SAAQ,YAAY;IACrD,gBAAgB,CAAQ;IACxB,SAAS,CAAQ;IAEjB,YAAY,EAAE,gBAAgB,EAAE,SAAS,EAAmD;QAC3F,KAAK,EAAE,CAAA;QACP,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAA;QACxC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAA;IAC3B,CAAC;IAED,KAAK,CAAC,yBAAyB,CAC9B,IAA8B;QAG9B,MAAM,iBAAiB,GAAU,EAAE,CAAA;QACnC,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC;YACxD,KAAK,EAAE,IAAI,CAAC,gBAAgB;SAC5B,CAAC,CAAA;QAEF,MAAM,6BAA6B,GAAG,CAAC,GAAI,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAA;QAEjE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,6BAA6B,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC/D,MAAM,OAAO,GAAG,6BAA6B,CAAC,CAAC,CAAC,CAAA;YAChD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;gBACrC,SAAQ;YACT,CAAC;YACD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACjD,MAAM,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;gBACtC,IAAI,WAAW,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;oBAClC,SAAQ;gBACT,CAAC;gBACD,iBAAiB,CAAC,IAAI,CAAC,IAAI,OAAO,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE;oBAC5D,uBAAuB;oBACvB,mCAAmC;oBACnC,8CAA8C;oBAC9C,wEAAwE;oBACxE,iDAAiD;oBACjD,2EAA2E;oBAC3E,uFAAuF;oBACvF,kEAAkE;oBAClE,0FAA0F;oBAC1F,oEAAoE;oBACpE,KAAK;oBACL,+BAA+B;oBAC/B,kFAAkF;oBAClF,mFAAmF;oBACnF,6EAA6E;oBAC7E,kEAAkE;oBAClE,0DAA0D;oBAC1D,wEAAwE;oBACxE,4DAA4D;oBAC5D,IAAI;oBACJ,wFAAwF;oBACxF,MAAM,IAAI,GAAG,gBAAgB,CAAC,QAAQ,CAAC,sBAAsB,CAAC;wBAC7D,KAAK,EAAE,IAAI,CAAC,gBAAgB;wBAC5B,wBAAwB;wBACxB,KAAK,EAAE,WAAW,CAAC,KAAK;wBACxB,MAAM,EAAE,+BAA+B;qBACvC,CAAC,CAAA;oBACF,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAA;oBAChC,OAAO,CAAC;wBACP,IAAI,EAAE,MAAM,CAAC,IAAI;wBACjB,YAAY,EAAE,CAAC;wBACf,gBAAgB,EAAE,CAAC;qBACnB,CAAC,CAAA;gBACH,CAAC,CAAC,CAAC,CAAA;YACJ,CAAC;QACF,CAAC;QAED,MAAM,gBAAgB,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAA;QAC7D,gBAAgB,CAAC,OAAO,EAAE,CAAA;QAC1B,OAAO,CAAC,KAAK,CAAC,oBAAoB,EAAE,gBAAgB,CAAC,CAAA;QAErD,KAAK,MAAM,eAAe,IAAI,gBAAgB,EAAE,CAAC;YAChD,MAAM,EAAE,IAAI,EAAE,YAAY,EAAE,gBAAgB,EAAE,GAAG,eAAe,CAAA;YAChE,MAAM,OAAO,GAAG,6BAA6B,CAAC,YAAY,CAAC,CAAA;YAC3D,qDAAqD;YACrD,kDAAkD;YAClD,aAAa;YACb,OAAO,CAAC,OAAO,CAAC,gBAAgB,CAAC,GAAG;gBACnC,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,wBAAwB,IAAI,EAAE;aACpC,CAAA;QACF,CAAC;QAED,MAAM,WAAW,GAAG,EAAE,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,6BAA6B,EAAE,KAAK,EAAE,IAAI,CAAC,SAAS,EAAE,CAAA;QACvG,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,WAAW,CAAC,CAAA;QAC9D,MAAM,IAAI,GAAG,SAAS,CAAC,QAAQ,CAAC,yBAAyB,CAAC,WAAW,EAAE;YACtE,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;gBAClB,IAAI,IAAI,CAAC,OAAO;oBAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;YACtC,CAAC;SACD,CAAC,CAAA;QACF,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAA;QAChC,SAAS,CAAC,OAAO,EAAE,CAAA;QACnB,OAAO,MAAM,CAAA;IACd,CAAC;CACD"}

package/dist/experiments/StableDiffPromptGenerator.js

@@ -0,0 +1,4 @@
+"use strict";
+// https://huggingface.co/Gustavosta/MagicPrompt-Stable-Diffusion
+// https://huggingface.co/QuantFactory/MagicPrompt-Stable-Diffusion-GGUF
+//# sourceMappingURL=StableDiffPromptGenerator.js.map

package/dist/experiments/StableDiffPromptGenerator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"StableDiffPromptGenerator.js","sourceRoot":"","sources":["../../src/experiments/StableDiffPromptGenerator.ts"],"names":[],"mappings":";AAAA,iEAAiE;AACjE,wEAAwE"}

package/dist/experiments/VoiceFunctionCall.d.ts

@@ -0,0 +1,18 @@
+import { EngineSpeechToTextArgs, ModelEngine, ToolDefinition } from '../types/index.js';
+import { CustomEngine } from '../engines/index.js';
+type EngineArgs = {
+    speechToTextModel: string;
+    chatModel: string;
+    tools: Record<string, ToolDefinition>;
+};
+export declare class VoiceFunctionCallEngine extends CustomEngine implements ModelEngine {
+    speechToTextModel: string;
+    chatModel: string;
+    tools: Record<string, ToolDefinition>;
+    constructor({ speechToTextModel, chatModel, tools }: EngineArgs);
+    createTranscription(args: EngineSpeechToTextArgs): Promise<string>;
+    processSpeechToTextTask(args: EngineSpeechToTextArgs): Promise<{
+        text: string;
+    }>;
+}
+export {};

package/dist/experiments/VoiceFunctionCall.js

@@ -0,0 +1,51 @@
+import { CustomEngine } from '../engines/index.js';
+// an experimental engine that forwards a transcription to a (function calling) chat model
+export class VoiceFunctionCallEngine extends CustomEngine {
+    speechToTextModel;
+    chatModel;
+    tools;
+    constructor({ speechToTextModel, chatModel, tools }) {
+        super();
+        this.speechToTextModel = speechToTextModel;
+        this.chatModel = chatModel;
+        this.tools = tools;
+    }
+    async createTranscription(args) {
+        const speechToTextModel = await this.pool.requestInstance({
+            model: this.speechToTextModel,
+        });
+        const transcriptionTask = speechToTextModel.instance.processSpeechToTextTask({
+            ...args.request,
+            model: this.speechToTextModel,
+        });
+        const transcription = await transcriptionTask.result;
+        speechToTextModel.release();
+        return transcription.text;
+    }
+    async processSpeechToTextTask(args) {
+        const [transcription, chatModel] = await Promise.all([
+            this.createTranscription(args),
+            this.pool.requestInstance({
+                model: this.chatModel,
+            }),
+        ]);
+        const chatTask = chatModel.instance.processChatCompletionTask({
+            model: this.chatModel,
+            tools: this.tools,
+            messages: [
+                {
+                    role: 'user',
+                    content: transcription,
+                },
+            ],
+        }, {
+            onChunk: args.onChunk,
+        });
+        const chatResponse = await chatTask.result;
+        chatModel.release();
+        return {
+            text: chatResponse.message.content,
+        };
+    }
+}
+//# sourceMappingURL=VoiceFunctionCall.js.map

package/dist/experiments/VoiceFunctionCall.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"VoiceFunctionCall.js","sourceRoot":"","sources":["../../src/experiments/VoiceFunctionCall.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAA;AAQxD,0FAA0F;AAE1F,MAAM,OAAO,uBACZ,SAAQ,YAAY;IAGpB,iBAAiB,CAAQ;IACzB,SAAS,CAAQ;IACjB,KAAK,CAAgC;IAErC,YAAY,EAAE,iBAAiB,EAAE,SAAS,EAAE,KAAK,EAAc;QAC9D,KAAK,EAAE,CAAA;QACP,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAA;QAC1C,IAAI,CAAC,SAAS,GAAG,SAAS,CAAA;QAC1B,IAAI,CAAC,KAAK,GAAG,KAAK,CAAA;IACnB,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,IAA4B;QACrD,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC;YACzD,KAAK,EAAE,IAAI,CAAC,iBAAiB;SAC7B,CAAC,CAAA;QACF,MAAM,iBAAiB,GAAG,iBAAiB,CAAC,QAAQ,CAAC,uBAAuB,CAC3E;YACC,GAAG,IAAI,CAAC,OAAO;YACf,KAAK,EAAE,IAAI,CAAC,iBAAiB;SAC7B,CACD,CAAA;QACD,MAAM,aAAa,GAAG,MAAM,iBAAiB,CAAC,MAAM,CAAA;QACpD,iBAAiB,CAAC,OAAO,EAAE,CAAA;QAC3B,OAAO,aAAa,CAAC,IAAI,CAAA;IAC1B,CAAC;IAED,KAAK,CAAC,uBAAuB,CAAC,IAA4B;QACzD,MAAM,CAAC,aAAa,EAAE,SAAS,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YACpD,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC;YAC9B,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC;gBACzB,KAAK,EAAE,IAAI,CAAC,SAAS;aACrB,CAAC;SACF,CAAC,CAAA;QACF,MAAM,QAAQ,GAAG,SAAS,CAAC,QAAQ,CAAC,yBAAyB,CAAC;YAC7D,KAAK,EAAE,IAAI,CAAC,SAAS;YACrB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,QAAQ,EAAE;gBACT;oBACC,IAAI,EAAE,MAAM;oBACZ,OAAO,EAAE,aAAa;iBACtB;aACD;SACD,EAAE;YACF,OAAO,EAAE,IAAI,CAAC,OAAO;SACrB,CAAC,CAAA;QACF,MAAM,YAAY,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAA;QAC1C,SAAS,CAAC,OAAO,EAAE,CAAA;QACnB,OAAO;YACN,IAAI,EAAE,YAAY,CAAC,OAAO,CAAC,OAAO;SAClC,CAAA;IACF,CAAC;CACD"}

package/dist/http.d.ts

@@ -0,0 +1,19 @@
+import http from 'node:http';
+import { ListenOptions } from 'node:net';
+import { LogLevel } from './lib/logger.js';
+import { ModelServer, ModelServerOptions } from './server.js';
+export declare function createOpenAIMiddleware(modelServer: ModelServer): import("express-serve-static-core").Router;
+export declare function createExpressMiddleware(modelServer: ModelServer): import("express-serve-static-core").Router;
+export interface ModelHTTPServerOptions extends ModelServerOptions {
+    listen?: ListenOptions;
+    logLevel?: LogLevel;
+}
+export declare class ModelHTTPServer {
+    httpServer: http.Server;
+    modelServer: ModelServer;
+    listenOptions: ListenOptions;
+    constructor(options: ModelHTTPServerOptions);
+    start(): Promise<void>;
+    stop(): Promise<void>;
+}
+export declare function startHTTPServer(options: ModelHTTPServerOptions): Promise<ModelHTTPServer>;

package/dist/http.js

@@ -0,0 +1,54 @@
+import http from 'node:http';
+import express from 'express';
+import cors from 'cors';
+import { createOpenAIRequestHandlers } from './api/openai/index.js';
+import { createAPIMiddleware } from './api/v1/index.js';
+import { ModelServer } from './server.js';
+export function createOpenAIMiddleware(modelServer) {
+    const router = express.Router();
+    const requestHandlers = createOpenAIRequestHandlers(modelServer);
+    router.get('/v1/models', requestHandlers.models);
+    router.post('/v1/completions', requestHandlers.completions);
+    router.post('/v1/chat/completions', requestHandlers.chatCompletions);
+    router.post('/v1/embeddings', requestHandlers.embeddings);
+    return router;
+}
+export function createExpressMiddleware(modelServer) {
+    const router = express.Router();
+    router.get('/', (req, res) => {
+        res.json(modelServer.getStatus());
+    });
+    router.use('/openai', createOpenAIMiddleware(modelServer));
+    router.use('/llama', createAPIMiddleware(modelServer));
+    return router;
+}
+export class ModelHTTPServer {
+    httpServer;
+    modelServer;
+    listenOptions;
+    constructor(options) {
+        const { listen, ...modelServerOpts } = options;
+        this.modelServer = new ModelServer(modelServerOpts);
+        this.listenOptions = listen ?? { port: 3000 };
+        const app = express();
+        app.use(cors(), express.json({ limit: '50mb' }), createExpressMiddleware(this.modelServer));
+        app.set('json spaces', 2);
+        this.httpServer = http.createServer(app);
+    }
+    async start() {
+        await this.modelServer.start();
+        await new Promise((resolve) => {
+            this.httpServer.listen(this.listenOptions, resolve);
+        });
+    }
+    async stop() {
+        this.httpServer.close();
+        await this.modelServer.stop();
+    }
+}
+export async function startHTTPServer(options) {
+    const server = new ModelHTTPServer(options);
+    await server.start();
+    return server;
+}
+//# sourceMappingURL=http.js.map

package/dist/http.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"http.js","sourceRoot":"","sources":["../src/http.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAA;AAE5B,OAAO,OAAO,MAAM,SAAS,CAAA;AAC7B,OAAO,IAAI,MAAM,MAAM,CAAA;AACvB,OAAO,EAAE,2BAA2B,EAAE,MAAM,8BAA8B,CAAA;AAC1E,OAAO,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAA;AAE9D,OAAO,EAAE,WAAW,EAAwC,MAAM,oBAAoB,CAAA;AAEtF,MAAM,UAAU,sBAAsB,CAAC,WAAwB;IAC9D,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAA;IAC/B,MAAM,eAAe,GAAG,2BAA2B,CAAC,WAAW,CAAC,CAAA;IAChE,MAAM,CAAC,GAAG,CAAC,YAAY,EAAE,eAAe,CAAC,MAAM,CAAC,CAAA;IAChD,MAAM,CAAC,IAAI,CAAC,iBAAiB,EAAE,eAAe,CAAC,WAAW,CAAC,CAAA;IAC3D,MAAM,CAAC,IAAI,CAAC,sBAAsB,EAAE,eAAe,CAAC,eAAe,CAAC,CAAA;IACpE,MAAM,CAAC,IAAI,CAAC,gBAAgB,EAAE,eAAe,CAAC,UAAU,CAAC,CAAA;IACzD,OAAO,MAAM,CAAA;AACd,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,WAAwB;IAC/D,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAA;IAC/B,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;QAC5B,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,CAAC,CAAA;IAClC,CAAC,CAAC,CAAA;IACF,MAAM,CAAC,GAAG,CAAC,SAAS,EAAE,sBAAsB,CAAC,WAAW,CAAC,CAAC,CAAA;IAC1D,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,mBAAmB,CAAC,WAAW,CAAC,CAAC,CAAA;IACtD,OAAO,MAAM,CAAA;AACd,CAAC;AAOD,MAAM,OAAO,eAAe;IAC3B,UAAU,CAAa;IACvB,WAAW,CAAa;IACxB,aAAa,CAAe;IAE5B,YAAY,OAA+B;QAC1C,MAAM,EAAE,MAAM,EAAE,GAAG,eAAe,EAAE,GAAG,OAAO,CAAA;QAC9C,IAAI,CAAC,WAAW,GAAG,IAAI,WAAW,CAAC,eAAe,CAAC,CAAA;QACnD,IAAI,CAAC,aAAa,GAAG,MAAM,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAA;QAC7C,MAAM,GAAG,GAAG,OAAO,EAAE,CAAA;QACrB,GAAG,CAAC,GAAG,CACN,IAAI,EAAE,EACN,OAAO,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,EAC/B,uBAAuB,CAAC,IAAI,CAAC,WAAW,CAAC,CACzC,CAAA;QAED,GAAG,CAAC,GAAG,CAAC,aAAa,EAAE,CAAC,CAAC,CAAA;QACzB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAA;IACzC,CAAC;IAED,KAAK,CAAC,KAAK;QACV,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAA;QAC9B,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YACnC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,EAAE,OAAO,CAAC,CAAA;QACpD,CAAC,CAAC,CAAA;IACH,CAAC;IAED,KAAK,CAAC,IAAI;QACT,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAA;QACvB,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,CAAA;IAC9B,CAAC;CACD;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,OAA+B;IACpE,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC,OAAO,CAAC,CAAA;IAC3C,MAAM,MAAM,CAAC,KAAK,EAAE,CAAA;IACpB,OAAO,MAAM,CAAA;AACd,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,7 @@
+export * from './api/openai/index.js';
+export * from './types/index.js';
+export * from './pool.js';
+export * from './instance.js';
+export * from './store.js';
+export * from './server.js';
+export * from './http.js';

package/dist/index.js

@@ -0,0 +1,8 @@
+export * from './api/openai/index.js';
+export * from './types/index.js';
+export * from './pool.js';
+export * from './instance.js';
+export * from './store.js';
+export * from './server.js';
+export * from './http.js';
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,uBAAuB,CAAA;AACrC,cAAc,kBAAkB,CAAA;AAChC,cAAc,WAAW,CAAA;AACzB,cAAc,eAAe,CAAA;AAC7B,cAAc,YAAY,CAAA;AAC1B,cAAc,aAAa,CAAA;AAC3B,cAAc,WAAW,CAAA"}

package/dist/instance.d.ts

@@ -0,0 +1,88 @@
+import { TextCompletionRequest, ChatCompletionRequest, ModelEngine, ModelConfig, ModelInstanceRequest, CompletionProcessingOptions, EmbeddingRequest, ImageToTextRequest, ProcessingOptions, SpeechToTextRequest, SpeechToTextProcessingOptions, EngineChatCompletionResult, EngineTextCompletionResult, TextToImageRequest, ImageToImageRequest } from './types/index.js';
+import { Logger } from './lib/logger.js';
+type ModelInstanceStatus = 'idle' | 'busy' | 'error' | 'loading' | 'preparing';
+interface ModelInstanceOptions extends ModelConfig {
+    log?: Logger;
+    gpu: boolean;
+}
+export declare class ModelInstance<TEngineState = unknown> {
+    id: string;
+    status: ModelInstanceStatus;
+    modelId: string;
+    config: ModelConfig;
+    fingerprint: string;
+    createdAt: Date;
+    lastUsed: number;
+    gpu: boolean;
+    ttl: number;
+    log: Logger;
+    private engine;
+    private contextIdentity?;
+    private needsContextReset;
+    private engineInstance?;
+    private currentRequest?;
+    private shutdownController;
+    constructor(engine: ModelEngine, { log, gpu, ...options }: ModelInstanceOptions);
+    private generateInstanceId;
+    private generateTaskId;
+    load(signal?: AbortSignal): Promise<void>;
+    dispose(): Promise<void>;
+    lock(request: ModelInstanceRequest): void;
+    unlock(): void;
+    resetContext(): void;
+    getContextStateIdentity(): string | undefined;
+    hasContextState(): boolean;
+    matchesContextState(request: ModelInstanceRequest): boolean;
+    matchesRequirements(request: ModelInstanceRequest): boolean;
+    private createTaskController;
+    processChatCompletionTask(request: ChatCompletionRequest, options?: CompletionProcessingOptions): {
+        id: string;
+        model: string;
+        createdAt: Date;
+        result: Promise<EngineChatCompletionResult>;
+        cancel: () => void;
+    };
+    processTextCompletionTask(request: TextCompletionRequest, options?: CompletionProcessingOptions): {
+        id: string;
+        model: string;
+        createdAt: Date;
+        cancel: () => void;
+        result: Promise<EngineTextCompletionResult>;
+    };
+    processEmbeddingTask(request: EmbeddingRequest, options?: ProcessingOptions): {
+        id: string;
+        model: string;
+        createdAt: Date;
+        cancel: () => void;
+        result: Promise<import("./types/index.js").EngineEmbeddingResult>;
+    };
+    processImageToTextTask(request: ImageToTextRequest, options?: ProcessingOptions): {
+        id: string;
+        model: string;
+        createdAt: Date;
+        cancel: () => void;
+        result: Promise<import("./types/index.js").EngineImageToTextResult>;
+    };
+    processImageToImageTask(request: ImageToImageRequest, options?: ProcessingOptions): {
+        id: string;
+        model: string;
+        createdAt: Date;
+        cancel: () => void;
+        result: Promise<import("./types/index.js").EngineImageToImageResult>;
+    };
+    processSpeechToTextTask(request: SpeechToTextRequest, options?: SpeechToTextProcessingOptions): {
+        id: string;
+        model: string;
+        createdAt: Date;
+        cancel: () => void;
+        result: Promise<import("./types/index.js").EngineSpeechToTextResult>;
+    };
+    processTextToImageTask(request: TextToImageRequest, options?: ProcessingOptions): {
+        id: string;
+        model: string;
+        createdAt: Date;
+        cancel: () => void;
+        result: Promise<import("./types/index.js").EngineTextToImageResult>;
+    };
+}
+export {};