inference-server 1.0.0-beta.19

package/dist/api/openai/handlers/completions.js

@@ -0,0 +1,169 @@
+import { parseJSONRequestBody } from '../../../api/parseJSONRequestBody.js';
+import { omitEmptyValues } from '../../../lib/util.js';
+import { finishReasonMap } from '../enums.js';
+// v1/completions
+// https://platform.openai.com/docs/api-reference/completions/create
+export function createCompletionHandler(modelServer) {
+    return async (req, res) => {
+        let args;
+        try {
+            const body = await parseJSONRequestBody(req);
+            args = body;
+        }
+        catch (e) {
+            console.error(e);
+            res.writeHead(400, { 'Content-Type': 'application/json' });
+            res.end(JSON.stringify({ error: 'Invalid request' }));
+            return;
+        }
+        // TODO ajv schema validation?
+        if (!args.model || !args.prompt) {
+            res.writeHead(400, { 'Content-Type': 'application/json' });
+            res.end(JSON.stringify({ error: 'Invalid request' }));
+            return;
+        }
+        if (!modelServer.modelExists(args.model)) {
+            res.writeHead(400, { 'Content-Type': 'application/json' });
+            res.end(JSON.stringify({ error: 'Invalid model' }));
+            return;
+        }
+        const controller = new AbortController();
+        req.on('close', () => {
+            console.debug('Client closed connection');
+            controller.abort();
+        });
+        req.on('end', () => {
+            console.debug('Client ended connection');
+            controller.abort();
+        });
+        try {
+            if (args.stream) {
+                res.writeHead(200, {
+                    'Content-Type': 'text/event-stream',
+                    'Cache-Control': 'no-cache',
+                    Connection: 'keep-alive',
+                });
+                res.flushHeaders();
+            }
+            let prompt = args.prompt;
+            if (typeof prompt !== 'string') {
+                throw new Error('Prompt must be a string');
+            }
+            let stop = args.stop ? args.stop : undefined;
+            if (typeof stop === 'string') {
+                stop = [stop];
+            }
+            const completionReq = omitEmptyValues({
+                model: args.model,
+                prompt: args.prompt,
+                temperature: args.temperature ? args.temperature : undefined,
+                stream: args.stream ? Boolean(args.stream) : false,
+                maxTokens: args.max_tokens ? args.max_tokens : undefined,
+                seed: args.seed ? args.seed : undefined,
+                stop,
+                frequencyPenalty: args.frequency_penalty
+                    ? args.frequency_penalty
+                    : undefined,
+                presencePenalty: args.presence_penalty
+                    ? args.presence_penalty
+                    : undefined,
+                tokenBias: args.logit_bias ? args.logit_bias : undefined,
+                topP: args.top_p ? args.top_p : undefined,
+                // additional non-spec params
+                repeatPenaltyNum: args.repeat_penalty_num
+                    ? args.repeat_penalty_num
+                    : undefined,
+                minP: args.min_p ? args.min_p : undefined,
+                topK: args.top_k ? args.top_k : undefined,
+            });
+            const { instance, release } = await modelServer.requestInstance(completionReq, controller.signal);
+            const task = instance.processTextCompletionTask(completionReq, {
+                signal: controller.signal,
+                onChunk: (chunk) => {
+                    if (args.stream) {
+                        const chunkData = {
+                            id: task.id,
+                            model: task.model,
+                            object: 'text_completion',
+                            created: Math.floor(task.createdAt.getTime() / 1000),
+                            choices: [
+                                {
+                                    index: 0,
+                                    text: chunk.text,
+                                    logprobs: null,
+                                    // @ts-ignore official api returns null here in the same case
+                                    finish_reason: null,
+                                },
+                            ],
+                        };
+                        res.write(`data: ${JSON.stringify(chunkData)}\n\n`);
+                    }
+                },
+            });
+            const result = await task.result;
+            release();
+            if (args.stream) {
+                if (args.stream_options?.include_usage) {
+                    const finalChunk = {
+                        id: task.id,
+                        model: task.model,
+                        object: 'text_completion',
+                        created: Math.floor(task.createdAt.getTime() / 1000),
+                        choices: [
+                            {
+                                index: 0,
+                                text: '',
+                                logprobs: null,
+                                // @ts-ignore
+                                finish_reason: result.finishReason
+                                    ? finishReasonMap[result.finishReason]
+                                    : 'stop',
+                            },
+                        ],
+                    };
+                    res.write(`data: ${JSON.stringify(finalChunk)}\n\n`);
+                }
+                res.write('data: [DONE]');
+                res.end();
+            }
+            else {
+                const response = {
+                    id: task.id,
+                    model: task.model,
+                    object: 'text_completion',
+                    created: Math.floor(task.createdAt.getTime() / 1000),
+                    system_fingerprint: instance.fingerprint,
+                    choices: [
+                        {
+                            index: 0,
+                            text: result.text,
+                            logprobs: null,
+                            // @ts-ignore
+                            finish_reason: result.finishReason
+                                ? finishReasonMap[result.finishReason]
+                                : 'stop',
+                        },
+                    ],
+                    usage: {
+                        prompt_tokens: result.promptTokens,
+                        completion_tokens: result.completionTokens,
+                        total_tokens: result.contextTokens,
+                    },
+                };
+                res.writeHead(200, { 'Content-Type': 'application/json' });
+                res.end(JSON.stringify(response, null, 2));
+            }
+        }
+        catch (err) {
+            console.error(err);
+            if (args.stream) {
+                res.write('data: [ERROR]');
+            }
+            else {
+                res.writeHead(500, { 'Content-Type': 'application/json' });
+                res.end(JSON.stringify({ error: 'Internal server error' }));
+            }
+        }
+    };
+}
+//# sourceMappingURL=completions.js.map

package/dist/api/openai/handlers/completions.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"completions.js","sourceRoot":"","sources":["../../../../src/api/openai/handlers/completions.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,oBAAoB,EAAE,MAAM,sCAAsC,CAAA;AAC3E,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAA;AACtD,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAc7C,iBAAiB;AACjB,oEAAoE;AACpE,MAAM,UAAU,uBAAuB,CAAC,WAAwB;IAC/D,OAAO,KAAK,EAAE,GAAoB,EAAE,GAAmB,EAAE,EAAE;QAC1D,IAAI,IAA4B,CAAA;QAEhC,IAAI,CAAC;YACJ,MAAM,IAAI,GAAG,MAAM,oBAAoB,CAAC,GAAG,CAAC,CAAA;YAC5C,IAAI,GAAG,IAAI,CAAA;QACZ,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACZ,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;YAChB,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE,CAAC,CAAA;YAC1D,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,iBAAiB,EAAE,CAAC,CAAC,CAAA;YACrD,OAAM;QACP,CAAC;QAED,8BAA8B;QAC9B,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE,CAAC,CAAA;YAC1D,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,iBAAiB,EAAE,CAAC,CAAC,CAAA;YACrD,OAAM;QACP,CAAC;QACD,IAAI,CAAC,WAAW,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YAC1C,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE,CAAC,CAAA;YAC1D,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,eAAe,EAAE,CAAC,CAAC,CAAA;YACnD,OAAM;QACP,CAAC;QAED,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAA;QACxC,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YACpB,OAAO,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAA;YACzC,UAAU,CAAC,KAAK,EAAE,CAAA;QACnB,CAAC,CAAC,CAAA;QACF,GAAG,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;YAClB,OAAO,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAA;YACxC,UAAU,CAAC,KAAK,EAAE,CAAA;QACnB,CAAC,CAAC,CAAA;QAEF,IAAI,CAAC;YACJ,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBACjB,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE;oBAClB,cAAc,EAAE,mBAAmB;oBACnC,eAAe,EAAE,UAAU;oBAC3B,UAAU,EAAE,YAAY;iBACxB,CAAC,CAAA;gBACF,GAAG,CAAC,YAAY,EAAE,CAAA;YACnB,CAAC;YAED,IAAI,MAAM,GAAG,IAAI,CAAC,MAAM,CAAA;YAExB,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;gBAChC,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAA;YAC3C,CAAC;YAED,IAAI,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAA;YAC5C,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;gBAC9B,IAAI,GAAG,CAAC,IAAI,CAAC,CAAA;YACd,CAAC;YAED,MAAM,aAAa,GAAG,eAAe,CAAwB;gBAC5D,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,MAAM,EAAE,IAAI,CAAC,MAAgB;gBAC7B,WAAW,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS;gBAC5D,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK;gBAClD,SAAS,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;gBACxD,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;gBACvC,IAAI;gBACJ,gBAAgB,EAAE,IAAI,CAAC,iBAAiB;oBACvC,CAAC,CAAC,IAAI,CAAC,iBAAiB;oBACxB,CAAC,CAAC,SAAS;gBACZ,eAAe,EAAE,IAAI,CAAC,gBAAgB;oBACrC,CAAC,CAAC,IAAI,CAAC,gBAAgB;oBACvB,CAAC,CAAC,SAAS;gBACZ,SAAS,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;gBACxD,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;gBACzC,6BAA6B;gBAC7B,gBAAgB,EAAE,IAAI,CAAC,kBAAkB;oBACxC,CAAC,CAAC,IAAI,CAAC,kBAAkB;oBACzB,CAAC,CAAC,SAAS;gBACZ,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;gBACzC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;aACzC,CAAC,CAAA;YAEF,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,MAAM,WAAW,CAAC,eAAe,CAC9D,aAAa,EACb,UAAU,CAAC,MAAM,CACjB,CAAA;YACD,MAAM,IAAI,GAAG,QAAQ,CAAC,yBAAyB,CAAC,aAAa,EAAE;gBAC9D,MAAM,EAAE,UAAU,CAAC,MAAM;gBACzB,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;oBAClB,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;wBACjB,MAAM,SAAS,GAA0B;4BACxC,EAAE,EAAE,IAAI,CAAC,EAAE;4BACX,KAAK,EAAE,IAAI,CAAC,KAAK;4BACjB,MAAM,EAAE,iBAAiB;4BACzB,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC;4BACpD,OAAO,EAAE;gCACR;oCACC,KAAK,EAAE,CAAC;oCACR,IAAI,EAAE,KAAK,CAAC,IAAI;oCAChB,QAAQ,EAAE,IAAI;oCACd,6DAA6D;oCAC7D,aAAa,EAAE,IAAI;iCACnB;6BACD;yBACD,CAAA;wBACD,GAAG,CAAC,KAAK,CAAC,SAAS,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,MAAM,CAAC,CAAA;oBACpD,CAAC;gBACF,CAAC;aACD,CAAC,CAAA;YACF,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAA;YAChC,OAAO,EAAE,CAAA;YAET,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBACjB,IAAI,IAAI,CAAC,cAAc,EAAE,aAAa,EAAE,CAAC;oBACxC,MAAM,UAAU,GAA0B;wBACzC,EAAE,EAAE,IAAI,CAAC,EAAE;wBACX,KAAK,EAAE,IAAI,CAAC,KAAK;wBACjB,MAAM,EAAE,iBAAiB;wBACzB,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC;wBACpD,OAAO,EAAE;4BACR;gCACC,KAAK,EAAE,CAAC;gCACR,IAAI,EAAE,EAAE;gCACR,QAAQ,EAAE,IAAI;gCACd,aAAa;gCACb,aAAa,EAAE,MAAM,CAAC,YAAY;oCACjC,CAAC,CAAC,eAAe,CAAC,MAAM,CAAC,YAAY,CAAC;oCACtC,CAAC,CAAC,MAAM;6BACT;yBACD;qBACD,CAAA;oBACD,GAAG,CAAC,KAAK,CACR,SAAS,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,MAAM,CACzC,CAAA;gBACF,CAAC;gBACD,GAAG,CAAC,KAAK,CAAC,cAAc,CAAC,CAAA;gBACzB,GAAG,CAAC,GAAG,EAAE,CAAA;YACV,CAAC;iBAAM,CAAC;gBACP,MAAM,QAAQ,GAAkC;oBAC/C,EAAE,EAAE,IAAI,CAAC,EAAE;oBACX,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,MAAM,EAAE,iBAAiB;oBACzB,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC;oBACpD,kBAAkB,EAAE,QAAQ,CAAC,WAAW;oBACxC,OAAO,EAAE;wBACR;4BACC,KAAK,EAAE,CAAC;4BACR,IAAI,EAAE,MAAM,CAAC,IAAI;4BACjB,QAAQ,EAAE,IAAI;4BACd,aAAa;4BACb,aAAa,EAAE,MAAM,CAAC,YAAY;gCACjC,CAAC,CAAC,eAAe,CAAC,MAAM,CAAC,YAAY,CAAC;gCACtC,CAAC,CAAC,MAAM;yBACT;qBACD;oBACD,KAAK,EAAE;wBACN,aAAa,EAAE,MAAM,CAAC,YAAY;wBAClC,iBAAiB,EAAE,MAAM,CAAC,gBAAgB;wBAC1C,YAAY,EAAE,MAAM,CAAC,aAAa;qBAClC;iBACD,CAAA;gBACD,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE,CAAC,CAAA;gBAC1D,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAA;YAC3C,CAAC;QACF,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;YAClB,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBACjB,GAAG,CAAC,KAAK,CAAC,eAAe,CAAC,CAAA;YAC3B,CAAC;iBAAM,CAAC;gBACP,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE,CAAC,CAAA;gBAC1D,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,uBAAuB,EAAE,CAAC,CAAC,CAAA;YAC5D,CAAC;QACF,CAAC;IACF,CAAC,CAAA;AACF,CAAC"}

package/dist/api/openai/handlers/embeddings.d.ts

@@ -0,0 +1,3 @@
+import { IncomingMessage, ServerResponse } from 'node:http';
+import { ModelServer } from '../../../server.js';
+export declare function createEmbeddingsHandler(modelServer: ModelServer): (req: IncomingMessage, res: ServerResponse) => Promise<void>;

package/dist/api/openai/handlers/embeddings.js

@@ -0,0 +1,74 @@
+import { parseJSONRequestBody } from '../../../api/parseJSONRequestBody.js';
+import { omitEmptyValues } from '../../../lib/util.js';
+// v1/embeddings
+// https://platform.openai.com/docs/api-reference/embeddings
+export function createEmbeddingsHandler(modelServer) {
+    return async (req, res) => {
+        let args;
+        try {
+            const body = await parseJSONRequestBody(req);
+            args = body;
+        }
+        catch (e) {
+            console.error(e);
+            res.writeHead(400, { 'Content-Type': 'application/json' });
+            res.end(JSON.stringify({ error: 'Invalid request' }));
+            return;
+        }
+        // TODO ajv schema validation?
+        if (!args.model || !args.input) {
+            res.writeHead(400, { 'Content-Type': 'application/json' });
+            res.end(JSON.stringify({ error: 'Invalid request' }));
+            return;
+        }
+        if (!modelServer.modelExists(args.model)) {
+            res.writeHead(400, { 'Content-Type': 'application/json' });
+            res.end(JSON.stringify({ error: 'Invalid model' }));
+            return;
+        }
+        const controller = new AbortController();
+        req.on('close', () => {
+            console.debug('Client closed connection');
+            controller.abort();
+        });
+        req.on('end', () => {
+            console.debug('Client ended connection');
+            controller.abort();
+        });
+        try {
+            let input = args.input;
+            if (typeof input !== 'string') {
+                throw new Error('Input must be a string');
+            }
+            const embeddingsReq = omitEmptyValues({
+                model: args.model,
+                input: args.input,
+            });
+            const { instance, release } = await modelServer.requestInstance(embeddingsReq, controller.signal);
+            const task = instance.processEmbeddingTask(embeddingsReq);
+            const result = await task.result;
+            release();
+            const response = {
+                model: instance.modelId,
+                object: 'list',
+                data: result.embeddings.map((embedding, index) => ({
+                    embedding: Array.from(embedding),
+                    index,
+                    object: 'embedding',
+                })),
+                usage: {
+                    prompt_tokens: result.inputTokens,
+                    total_tokens: result.inputTokens,
+                },
+            };
+            res.writeHead(200, { 'Content-Type': 'application/json' });
+            res.end(JSON.stringify(response, null, 2));
+        }
+        catch (err) {
+            console.error(err);
+            res.writeHead(500, { 'Content-Type': 'application/json' });
+            res.end(JSON.stringify({ error: 'Internal server error' }));
+        }
+    };
+}
+//# sourceMappingURL=embeddings.js.map

package/dist/api/openai/handlers/embeddings.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"embeddings.js","sourceRoot":"","sources":["../../../../src/api/openai/handlers/embeddings.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,oBAAoB,EAAE,MAAM,sCAAsC,CAAA;AAC3E,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAA;AAKtD,gBAAgB;AAChB,4DAA4D;AAC5D,MAAM,UAAU,uBAAuB,CAAC,WAAwB;IAC/D,OAAO,KAAK,EAAE,GAAoB,EAAE,GAAmB,EAAE,EAAE;QAC1D,IAAI,IAA4B,CAAA;QAEhC,IAAI,CAAC;YACJ,MAAM,IAAI,GAAG,MAAM,oBAAoB,CAAC,GAAG,CAAC,CAAA;YAC5C,IAAI,GAAG,IAAI,CAAA;QACZ,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACZ,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;YAChB,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE,CAAC,CAAA;YAC1D,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,iBAAiB,EAAE,CAAC,CAAC,CAAA;YACrD,OAAM;QACP,CAAC;QAED,8BAA8B;QAC9B,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YAChC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE,CAAC,CAAA;YAC1D,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,iBAAiB,EAAE,CAAC,CAAC,CAAA;YACrD,OAAM;QACP,CAAC;QACD,IAAI,CAAC,WAAW,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YAC1C,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE,CAAC,CAAA;YAC1D,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,eAAe,EAAE,CAAC,CAAC,CAAA;YACnD,OAAM;QACP,CAAC;QAED,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAA;QACxC,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YACpB,OAAO,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAA;YACzC,UAAU,CAAC,KAAK,EAAE,CAAA;QACnB,CAAC,CAAC,CAAA;QACF,GAAG,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;YAClB,OAAO,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAA;YACxC,UAAU,CAAC,KAAK,EAAE,CAAA;QACnB,CAAC,CAAC,CAAA;QAEF,IAAI,CAAC;YAGJ,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,CAAA;YAEtB,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;gBAC/B,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAA;YAC1C,CAAC;YAED,MAAM,aAAa,GAAG,eAAe,CAAmB;gBACvD,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,KAAK,EAAE,IAAI,CAAC,KAAe;aAC3B,CAAC,CAAA;YAEF,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,MAAM,WAAW,CAAC,eAAe,CAC9D,aAAa,EACb,UAAU,CAAC,MAAM,CACjB,CAAA;YACD,MAAM,IAAI,GAAG,QAAQ,CAAC,oBAAoB,CAAC,aAAa,CAAC,CAAA;YACzD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAA;YAChC,OAAO,EAAE,CAAA;YAET,MAAM,QAAQ,GAAmC;gBAChD,KAAK,EAAE,QAAQ,CAAC,OAAO;gBACvB,MAAM,EAAE,MAAM;gBACd,IAAI,EAAE,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;oBAClD,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC;oBAChC,KAAK;oBACL,MAAM,EAAE,WAAW;iBACnB,CAAC,CAAC;gBACH,KAAK,EAAE;oBACN,aAAa,EAAE,MAAM,CAAC,WAAW;oBACjC,YAAY,EAAE,MAAM,CAAC,WAAW;iBAChC;aACD,CAAA;YACD,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE,CAAC,CAAA;YAC1D,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAA;QAE3C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;YAClB,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE,CAAC,CAAA;YAC1D,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,uBAAuB,EAAE,CAAC,CAAC,CAAA;QAC5D,CAAC;IACF,CAAC,CAAA;AACF,CAAC"}

package/dist/api/openai/handlers/images.js

@@ -0,0 +1,4 @@
+"use strict";
+// v1/images endpoints
+// https://platform.openai.com/docs/api-reference/images/create
+//# sourceMappingURL=images.js.map

package/dist/api/openai/handlers/images.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"images.js","sourceRoot":"","sources":["../../../../src/api/openai/handlers/images.ts"],"names":[],"mappings":";AACA,sBAAsB;AACtB,+DAA+D"}

package/dist/api/openai/handlers/models.d.ts

@@ -0,0 +1,3 @@
+import type { IncomingMessage, ServerResponse } from 'node:http';
+import type { ModelServer } from '../../../server';
+export declare function createModelsHandler(modelServer: ModelServer): (req: IncomingMessage, res: ServerResponse) => Promise<void>;

package/dist/api/openai/handlers/models.js

@@ -0,0 +1,23 @@
+// https://platform.openai.com/docs/api-reference/models/list
+export function createModelsHandler(modelServer) {
+    return async (req, res) => {
+        const models = modelServer.store.getStatus();
+        const data = Object.entries(models).map(([id, info]) => {
+            // const lastModDate = new Date(info.source.lastModified)
+            // const created = Math.floor(lastModDate.getTime() / 1000)
+            // const dirPath = path.dirname(info.source.file);
+            // const lastDir = path.basename(dirPath);
+            // const baseName = path.basename(info.source.file);
+            const owned_by = info.engine; // + ':' + path.join(lastDir, baseName);
+            return {
+                object: 'model',
+                id,
+                created: 0,
+                owned_by,
+            };
+        });
+        res.writeHead(200, { 'Content-Type': 'application/json' });
+        res.end(JSON.stringify({ object: 'list', data }, null, 2));
+    };
+}
+//# sourceMappingURL=models.js.map

package/dist/api/openai/handlers/models.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"models.js","sourceRoot":"","sources":["../../../../src/api/openai/handlers/models.ts"],"names":[],"mappings":"AAKA,6DAA6D;AAC7D,MAAM,UAAU,mBAAmB,CAAC,WAAwB;IAC3D,OAAO,KAAK,EAAE,GAAoB,EAAE,GAAmB,EAAE,EAAE;QAE1D,MAAM,MAAM,GAAG,WAAW,CAAC,KAAK,CAAC,SAAS,EAAE,CAAA;QAC5C,MAAM,IAAI,GAAmB,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,CACtD,CAAC,CAAC,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE;YACd,yDAAyD;YACzD,2DAA2D;YAE3D,kDAAkD;YAClD,0CAA0C;YAC1C,oDAAoD;YACpD,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAA,CAAA,wCAAwC;YAEpE,OAAO;gBACN,MAAM,EAAE,OAAO;gBACf,EAAE;gBACF,OAAO,EAAE,CAAC;gBACV,QAAQ;aACR,CAAA;QACF,CAAC,CACD,CAAA;QAED,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE,CAAC,CAAA;QAC1D,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAA;IAC3D,CAAC,CAAA;AACF,CAAC"}

package/dist/api/openai/handlers/transcription.js

@@ -0,0 +1,4 @@
+"use strict";
+// TODO
+// https://platform.openai.com/docs/api-reference/audio/createTranscription
+//# sourceMappingURL=transcription.js.map

package/dist/api/openai/handlers/transcription.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"transcription.js","sourceRoot":"","sources":["../../../../src/api/openai/handlers/transcription.ts"],"names":[],"mappings":";AAAA,OAAO;AACP,2EAA2E"}

package/dist/api/openai/index.d.ts

@@ -0,0 +1,7 @@
+import type { ModelServer } from '../../server.js';
+export declare function createOpenAIRequestHandlers(modelServer: ModelServer): {
+    chatCompletions: (req: import("http").IncomingMessage, res: import("http").ServerResponse) => Promise<void>;
+    completions: (req: import("http").IncomingMessage, res: import("http").ServerResponse) => Promise<void>;
+    models: (req: import("http").IncomingMessage, res: import("http").ServerResponse) => Promise<void>;
+    embeddings: (req: import("http").IncomingMessage, res: import("http").ServerResponse) => Promise<void>;
+};

package/dist/api/openai/index.js

@@ -0,0 +1,14 @@
+import { createChatCompletionHandler } from './handlers/chat.js';
+import { createCompletionHandler } from './handlers/completions.js';
+import { createModelsHandler } from './handlers/models.js';
+import { createEmbeddingsHandler } from './handlers/embeddings.js';
+// See OpenAI API specs at https://github.com/openai/openai-openapi/blob/master/openapi.yaml
+export function createOpenAIRequestHandlers(modelServer) {
+    return {
+        chatCompletions: createChatCompletionHandler(modelServer),
+        completions: createCompletionHandler(modelServer),
+        models: createModelsHandler(modelServer),
+        embeddings: createEmbeddingsHandler(modelServer),
+    };
+}
+//# sourceMappingURL=index.js.map

package/dist/api/openai/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/api/openai/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,2BAA2B,EAAE,MAAM,oBAAoB,CAAA;AAChE,OAAO,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAA;AACnE,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAA;AAC1D,OAAO,EAAE,uBAAuB,EAAE,MAAM,0BAA0B,CAAA;AAGlE,4FAA4F;AAC5F,MAAM,UAAU,2BAA2B,CAAC,WAAwB;IACnE,OAAO;QACN,eAAe,EAAE,2BAA2B,CAAC,WAAW,CAAC;QACzD,WAAW,EAAE,uBAAuB,CAAC,WAAW,CAAC;QACjD,MAAM,EAAE,mBAAmB,CAAC,WAAW,CAAC;QACxC,UAAU,EAAE,uBAAuB,CAAC,WAAW,CAAC;KAChD,CAAA;AACF,CAAC"}

package/dist/api/parseJSONRequestBody.d.ts

@@ -0,0 +1,2 @@
+import { IncomingMessage } from 'node:http';
+export declare function parseJSONRequestBody(req: IncomingMessage | Request): Promise<any>;

package/dist/api/parseJSONRequestBody.js

@@ -0,0 +1,24 @@
+export function parseJSONRequestBody(req) {
+    return new Promise((resolve, reject) => {
+        // if request is from express theres no need to parse anything
+        if ('body' in req) {
+            resolve(req.body);
+            return;
+        }
+        // for native http server
+        let body = '';
+        req.on('data', (chunk) => {
+            body += chunk.toString();
+        });
+        req.on('end', () => {
+            try {
+                const data = JSON.parse(body);
+                resolve(data);
+            }
+            catch (error) {
+                reject(error);
+            }
+        });
+    });
+}
+//# sourceMappingURL=parseJSONRequestBody.js.map

package/dist/api/parseJSONRequestBody.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"parseJSONRequestBody.js","sourceRoot":"","sources":["../../src/api/parseJSONRequestBody.ts"],"names":[],"mappings":"AAEA,MAAM,UAAU,oBAAoB,CAAC,GAA8B;IAClE,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAEtC,8DAA8D;QAC9D,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;YACnB,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;YACjB,OAAM;QACP,CAAC;QAED,yBAAyB;QACzB,IAAI,IAAI,GAAG,EAAE,CAAA;QACb,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;YACxB,IAAI,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAA;QACzB,CAAC,CAAC,CAAA;QACF,GAAG,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;YAClB,IAAI,CAAC;gBACJ,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;gBAC7B,OAAO,CAAC,IAAI,CAAC,CAAA;YACd,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAChB,MAAM,CAAC,KAAK,CAAC,CAAA;YACd,CAAC;QACF,CAAC,CAAC,CAAA;IACH,CAAC,CAAC,CAAA;AACH,CAAC"}

package/dist/api/v1/index.d.ts

@@ -0,0 +1,2 @@
+import { ModelServer } from '../../server.js';
+export declare function createAPIMiddleware(server: ModelServer): import("express-serve-static-core").Router;

package/dist/api/v1/index.js

@@ -0,0 +1,29 @@
+import express from 'express';
+export function createAPIMiddleware(server) {
+    const router = express.Router();
+    router.use((req, res, next) => {
+        console.debug('API call', {
+            method: req.method,
+            url: req.url,
+            body: req.body,
+        });
+        next();
+    });
+    return router;
+    // return async (req: IncomingMessage, res: ServerResponse) => {
+    // 	let args: any
+    // 	try {
+    // 		const body = await parseJSONRequestBody(req)
+    // 		args = body
+    // 	} catch (e) {
+    // 		console.error(e)
+    // 		res.writeHead(400, { 'Content-Type': 'application/json' })
+    // 		res.end(JSON.stringify({ error: 'Invalid request' }))
+    // 		return
+    // 	}
+    // 	console.debug('Handler', JSON.stringify(args, null, 2))
+    // 	res.writeHead(200, { 'Content-Type': 'application/json' })
+    // 	res.end(JSON.stringify({ message: 'Hello' }))
+    // }
+}
+//# sourceMappingURL=index.js.map

package/dist/api/v1/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/api/v1/index.ts"],"names":[],"mappings":"AACA,OAAO,OAAO,MAAM,SAAS,CAAA;AAI7B,MAAM,UAAU,mBAAmB,CAAC,MAAmB;IACtD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAA;IAE/B,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE;QAC7B,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE;YACzB,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,GAAG,EAAE,GAAG,CAAC,GAAG;YACZ,IAAI,EAAE,GAAG,CAAC,IAAI;SACd,CAAC,CAAA;QACF,IAAI,EAAE,CAAA;IACP,CAAC,CAAC,CAAA;IACF,OAAO,MAAM,CAAA;IACb,gEAAgE;IAEhE,iBAAiB;IAEjB,SAAS;IACT,iDAAiD;IACjD,gBAAgB;IAChB,iBAAiB;IACjB,qBAAqB;IACrB,+DAA+D;IAC/D,0DAA0D;IAC1D,WAAW;IACX,KAAK;IAEL,2DAA2D;IAC3D,8DAA8D;IAC9D,iDAAiD;IAEjD,IAAI;AACL,CAAC"}

package/dist/cli.d.ts

@@ -0,0 +1 @@
+#!/usr/bin/env node

package/dist/cli.js

@@ -0,0 +1,10 @@
+#!/usr/bin/env node
+"use strict";
+process.stdout.write("Nothing here, yet.\n");
+/* API Ideas:
+- `infsrv ls` - list stored models
+- `infsrv rm <name>` - remove a stored model
+- `infsrv show <name>` - show details of a stored model
+- `infsrv prepare <config>` - download everything required for given config
+*/ 
+//# sourceMappingURL=cli.js.map

package/dist/cli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";;AACA,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;AAE7C;;;;;EAKE"}

package/dist/engines/gpt4all/engine.d.ts

@@ -0,0 +1,34 @@
+import { InferenceModel, EmbeddingModel } from 'gpt4all';
+import { EngineTextCompletionArgs, EngineChatCompletionArgs, EngineChatCompletionResult, EngineTextCompletionResult, EngineContext, EngineEmbeddingArgs, EngineEmbeddingResult, FileDownloadProgress, ModelConfig, TextCompletionParams, ChatMessage } from '../../types/index.js';
+export type GPT4AllInstance = InferenceModel | EmbeddingModel;
+export interface GPT4AllModelMeta {
+    url: string;
+    md5sum: string;
+    filename: string;
+    promptTemplate: string;
+    systemPrompt: string;
+    filesize: number;
+    ramrequired: number;
+}
+export interface GPT4AllModelConfig extends ModelConfig {
+    location: string;
+    md5?: string;
+    url?: string;
+    contextSize?: number;
+    batchSize?: number;
+    task: 'text-completion' | 'embedding';
+    initialMessages?: ChatMessage[];
+    completionDefaults?: TextCompletionParams;
+    device?: {
+        gpu?: boolean | 'auto' | (string & {});
+        gpuLayers?: number;
+        cpuThreads?: number;
+    };
+}
+export declare const autoGpu = true;
+export declare function prepareModel({ config, log }: EngineContext<GPT4AllModelConfig>, onProgress?: (progress: FileDownloadProgress) => void, signal?: AbortSignal): Promise<GPT4AllModelMeta | undefined>;
+export declare function createInstance({ config, log }: EngineContext<GPT4AllModelConfig>, signal?: AbortSignal): Promise<InferenceModel | EmbeddingModel>;
+export declare function disposeInstance(instance: GPT4AllInstance): Promise<void>;
+export declare function processTextCompletionTask({ request, config, onChunk }: EngineTextCompletionArgs<GPT4AllModelConfig>, instance: GPT4AllInstance, signal?: AbortSignal): Promise<EngineTextCompletionResult>;
+export declare function processChatCompletionTask({ request, config, resetContext, log, onChunk }: EngineChatCompletionArgs<GPT4AllModelConfig>, instance: GPT4AllInstance, signal?: AbortSignal): Promise<EngineChatCompletionResult>;
+export declare function processEmbeddingTask({ request, config }: EngineEmbeddingArgs, instance: GPT4AllInstance, signal?: AbortSignal): Promise<EngineEmbeddingResult>;