inference-server 1.0.0-beta.19

package/package.json

@@ -0,0 +1,113 @@
+{
+	"name": "inference-server",
+	"version": "1.0.0-beta.19",
+	"description": "Libraries and server to build AI applications. Adapters to various native bindings allowing local inference. Integrate it with your application, or use as a microservice.",
+	"main": "dist/index.js",
+	"source": "src/index.ts",
+	"types": "dist/index.d.ts",
+	"type": "module",
+	"license": "MIT",
+	"bin": {
+		"lllms": "./dist/cli.js"
+	},
+	"repository": "github:iimez/inference-server",
+	"bugs": {
+		"url": "https://github.com/iimez/inference-server/issues"
+	},
+	"scripts": {
+		"upgrade": "npx npm-check-updates -i",
+		"reinstall": "rimraf node_modules && npm install",
+		"clean": "rimraf dist",
+		"download-test-models": "node scripts/download-test-models.js",
+		"prebuild": "npm run clean",
+		"build": "tsc -p tsconfig.release.json && tsc-alias -p tsconfig.release.json",
+		"test": "vitest --run",
+		"test:pool": "vitest tests/pool.test.ts",
+		"test:openai": "vitest tests/openai.test.ts",
+		"test:gpt4all": "vitest tests/engines/gpt4all.test.ts",
+		"test:llama": "vitest tests/engines/node-llama-cpp.test.ts",
+		"test:transformers": "vitest tests/engines/transformers.test.ts",
+		"test:sd": "vitest tests/engines/stable-diffusion.test.ts",
+		"test:experiments": "vitest tests/engines/experiments.test.ts",
+		"test:server": "vitest tests/server.test.ts",
+		"prewatch": "npm run clean",
+		"watch": "tsc -w -p tsconfig.release.json",
+		"start": "cross-env NODE_ENV=production node dist/standalone.js"
+	},
+	"keywords": [
+		"local ai",
+		"inference server",
+		"model pool",
+		"gpt4all",
+		"node-llama-cpp",
+		"transformers.js",
+		"llama.cpp",
+		"chatbot",
+		"bot",
+		"llm",
+		"ai",
+		"nlp",
+		"openai api"
+	],
+	"engines": {
+		"node": ">=18.16.0"
+	},
+	"imports": {
+		"#package/*": "./dist/*"
+	},
+	"peerDependencies": {
+		"@huggingface/transformers": ">=3.0.0",
+		"gpt4all": ">=4.0.0",
+		"node-llama-cpp": ">=3.0.0",
+		"@lmagder/node-stable-diffusion-cpp": ">=0.1.6"
+	},
+	"peerDependenciesMeta": {
+		"node-llama-cpp": {
+			"optional": true
+		},
+		"gpt4all": {
+			"optional": true
+		},
+		"@huggingface/transformers": {
+			"optional": true
+		},
+		"@lmagder/node-stable-diffusion-cpp": {
+			"optional": true
+		}
+	},
+	"dependencies": {
+		"@alexanderolsen/libsamplerate-js": "^2.1.2",
+		"@huggingface/gguf": "^0.1.12",
+		"ajv": "^8.17.1",
+		"audio-decode": "^2.2.2",
+		"chalk": "^5.3.0",
+		"cors": "^2.8.5",
+		"express": "^4.21.1",
+		"ipull": "^3.9.0",
+		"nanoid": "^5.0.8",
+		"onnxruntime-node": "^1.19.2",
+		"p-queue": "^8.0.1",
+		"pretty-bytes": "^6.1.1",
+		"pretty-ms": "^9.1.0",
+		"proper-lockfile": "^4.1.2",
+		"sharp": "^0.33.5"
+	},
+	"devDependencies": {
+		"@huggingface/transformers": "^3.0.1",
+		"@lmagder/node-stable-diffusion-cpp": "^0.1.5",
+		"@types/cors": "^2.8.17",
+		"@types/express": "^5.0.0",
+		"@types/node": "^22.8.2",
+		"@types/proper-lockfile": "^4.1.4",
+		"@types/supertest": "^6.0.2",
+		"cross-env": "^7.0.3",
+		"gpt4all": "^4.0.0",
+		"node-llama-cpp": "^3.1.1",
+		"openai": "^4.68.4",
+		"supertest": "^7.0.0",
+		"tsc-alias": "^1.8.10",
+		"typescript": "^5.6.3",
+		"vite-tsconfig-paths": "^5.0.1",
+		"vitest": "^2.1.4"
+	}
+}

package/src/api/openai/enums.ts

@@ -0,0 +1,20 @@
+import { CompletionFinishReason, ChatMessage } from '#package/types/index.js'
+import OpenAI from 'openai'
+
+export const finishReasonMap: Record<CompletionFinishReason, OpenAI.ChatCompletion.Choice['finish_reason']> = {
+	maxTokens: 'length',
+	toolCalls: 'tool_calls',
+	eogToken: 'stop',
+	stopTrigger: 'stop',
+	timeout: 'stop',
+	cancel: 'stop',
+	abort: 'stop',
+} as const
+
+export const messageRoleMap: Record<OpenAI.ChatCompletionMessageParam['role'], ChatMessage['role']> = {
+	user: 'user',
+	system: 'system',
+	assistant: 'assistant',
+	tool: 'tool',
+	function: 'tool',
+}

package/src/api/openai/handlers/chat.ts

@@ -0,0 +1,408 @@
+import type { IncomingMessage, ServerResponse } from 'node:http'
+import type { OpenAI } from 'openai'
+import { ChatCompletionMessageParam } from 'openai/resources/chat/completions.js'
+import type { ModelServer } from '#package/server.js'
+import {
+	ChatCompletionRequest,
+	ToolDefinition,
+	ChatMessage,
+	MessageContentPart,
+	Image,
+} from '#package/types/index.js'
+import { parseJSONRequestBody } from '#package/api/parseJSONRequestBody.js'
+import { omitEmptyValues } from '#package/lib/util.js'
+import { loadImageFromUrl } from '#package/lib/loadImage.js'
+import { finishReasonMap, messageRoleMap } from '../enums.js'
+
+interface OpenAIChatCompletionParams
+	extends Omit<OpenAI.ChatCompletionCreateParamsStreaming, 'stream'> {
+	stream?: boolean
+	top_k?: number
+	min_p?: number
+	repeat_penalty_num?: number
+}
+
+interface OpenAIChatCompletionChunk extends OpenAI.ChatCompletionChunk {
+	usage?: OpenAI.CompletionUsage
+}
+
+async function prepareIncomingMessages(
+	messages: ChatCompletionMessageParam[]
+): Promise<ChatMessage[]> {
+	const downloadPromises: Record<string, Promise<Image>> = {}
+	const resultMessages: ChatMessage[] = []
+	
+	for (const message of messages) {
+		const role = messageRoleMap[message.role]
+		const resultMessage: any = {
+			role,
+			content: [],
+		}
+		if (role === 'tool' && 'tool_call_id' in message) {
+			resultMessage.callId = message.tool_call_id
+		}
+
+		if (typeof message.content === 'string') {
+			resultMessage.content.push({
+				type: 'text',
+				text: message.content,
+			})
+		} else if (Array.isArray(message.content)) {
+			for (const part of message.content) {
+				
+				if (typeof part === 'string') {
+					resultMessage.content.push({
+						type: 'text',
+						text: part,
+					})
+				} else if (part.type === 'text') {
+					resultMessage.content.push({
+						type: 'text',
+						text: part.text,
+					})
+				} else if (part.type === 'image_url') {
+					if (!downloadPromises[part.image_url.url]) {
+						downloadPromises[part.image_url.url] = loadImageFromUrl(part.image_url.url)
+					}
+					const content: Partial<MessageContentPart> = {
+						type: 'image',
+					}
+					resultMessage.content.push(content)
+					downloadPromises[part.image_url.url].then((image) => {
+						content.image = image
+					})
+				} else if (part.type === 'input_audio') {
+					resultMessage.content.push({
+						type: 'audio',
+						audio: part.input_audio,
+					})
+				} else if (part.type === 'refusal') {
+					resultMessage.content.push({
+						type: 'text',
+						text: part.refusal,
+					})
+				}
+			}
+		} else {
+			throw new Error('Invalid message content')
+		}
+		
+		resultMessages.push(resultMessage)
+	}
+	
+	await Promise.all(Object.values(downloadPromises))
+	
+	return resultMessages
+	
+}
+
+function createResponseMessageContent(
+	content: string | MessageContentPart[]
+): OpenAI.ChatCompletionMessage['content'] {
+	if (!content) {
+		return null
+	}
+	if (typeof content === 'string') {
+		return content
+	}
+	if (!Array.isArray(content)) {
+		throw new Error('Invalid response message content')
+	}
+
+	let text = ''
+	for (const part of content) {
+		if (part.type === 'text') {
+			text += part.text
+		}
+		// assistant may only respond with text in openai chat completions
+	}
+	return text
+}
+
+// v1/chat/completions
+// https://platform.openai.com/docs/api-reference/chat/create
+export function createChatCompletionHandler(modelServer: ModelServer) {
+	return async (req: IncomingMessage, res: ServerResponse) => {
+		let args: OpenAIChatCompletionParams
+
+		try {
+			const body = await parseJSONRequestBody(req)
+			args = body
+		} catch (e) {
+			console.error(e)
+			res.writeHead(400, { 'Content-Type': 'application/json' })
+			res.end(JSON.stringify({ error: 'Invalid request' }))
+			return
+		}
+
+		// TODO ajv schema validation?
+		if (!args.model || !args.messages) {
+			res.writeHead(400, { 'Content-Type': 'application/json' })
+			res.end(JSON.stringify({ error: 'Invalid request (need at least model and messages)' }))
+			return
+		}
+
+		if (!modelServer.modelExists(args.model)) {
+			res.writeHead(400, { 'Content-Type': 'application/json' })
+			res.end(JSON.stringify({ error: 'Model does not exist' }))
+			return
+		}
+
+		const controller = new AbortController()
+		req.on('close', () => {
+			console.debug('Client closed connection')
+			controller.abort()
+		})
+		req.on('end', () => {
+			console.debug('Client ended connection')
+			controller.abort()
+		})
+		req.on('aborted', () => {
+			console.debug('Client aborted connection')
+			controller.abort()
+		})
+		req.on('error', () => {
+			console.debug('Client error')
+			controller.abort()
+		})
+
+		try {
+			let ssePing: NodeJS.Timeout | undefined
+			if (args.stream) {
+				res.writeHead(200, {
+					'Content-Type': 'text/event-stream',
+					'Cache-Control': 'no-cache',
+					Connection: 'keep-alive',
+				})
+				res.flushHeaders()
+				ssePing = setInterval(() => {
+					res.write(':ping\n\n')
+				}, 30000)
+			}
+
+			let stop = args.stop ? args.stop : undefined
+			if (typeof stop === 'string') {
+				stop = [stop]
+			}
+
+			let completionGrammar: 'json' | undefined
+			if (args.response_format) {
+				if (args.response_format.type === 'json_object') {
+					completionGrammar = 'json'
+				}
+			}
+
+			let completionTools:
+				| Record<string, ToolDefinition>
+				| undefined
+
+			if (args.tools) {
+				const functionTools = args.tools
+					.filter((tool) => tool.type === 'function')
+					.map((tool) => {
+						return {
+							name: tool.function.name,
+							description: tool.function.description,
+							parameters: tool.function.parameters,
+						}
+					})
+				if (functionTools.length) {
+					if (!completionTools) {
+						completionTools = {}
+					}
+					for (const tool of functionTools) {
+						completionTools[tool.name] = {
+							description: tool.description,
+							parameters: tool.parameters,
+						} as ToolDefinition
+					}
+				}
+			}
+			
+			const messages = await prepareIncomingMessages(args.messages)
+			const completionReq = omitEmptyValues<ChatCompletionRequest>({
+				model: args.model,
+				messages,
+				temperature: args.temperature ? args.temperature : undefined,
+				stream: args.stream ? Boolean(args.stream) : false,
+				maxTokens: args.max_tokens ? args.max_tokens : undefined,
+				seed: args.seed ? args.seed : undefined,
+				stop,
+				frequencyPenalty: args.frequency_penalty
+					? args.frequency_penalty
+					: undefined,
+				presencePenalty: args.presence_penalty
+					? args.presence_penalty
+					: undefined,
+				topP: args.top_p ? args.top_p : undefined,
+				tokenBias: args.logit_bias ? args.logit_bias : undefined,
+				grammar: completionGrammar,
+				tools: completionTools,
+				// additional non-spec params
+				repeatPenaltyNum: args.repeat_penalty_num
+					? args.repeat_penalty_num
+					: undefined,
+				minP: args.min_p ? args.min_p : undefined,
+				topK: args.top_k ? args.top_k : undefined,
+			})
+			const { instance, release } = await modelServer.requestInstance(
+				completionReq,
+				controller.signal,
+			)
+			
+			if (ssePing) {
+				clearInterval(ssePing)
+			}
+			const task = instance.processChatCompletionTask(completionReq, {
+				signal: controller.signal,
+				onChunk: (chunk) => {
+					if (args.stream) {
+						const chunkData: OpenAIChatCompletionChunk = {
+							id: task.id,
+							object: 'chat.completion.chunk',
+							model: task.model,
+							created: Math.floor(task.createdAt.getTime() / 1000),
+							choices: [
+								{
+									index: 0,
+									delta: {
+										role: 'assistant',
+										content: chunk.text,
+									},
+									logprobs: null,
+									finish_reason: null,
+								},
+							],
+						}
+						res.write(`data: ${JSON.stringify(chunkData)}\n\n`)
+					}
+				},
+			})
+
+			const result = await task.result
+
+			release()
+
+			if (args.stream) {
+				if (result.finishReason === 'toolCalls') {
+					// currently not possible to stream function calls
+					// imitating a stream here by sending two chunks. makes it work with the openai client
+					const streamedToolCallChunk: OpenAIChatCompletionChunk = {
+						id: task.id,
+						object: 'chat.completion.chunk',
+						model: task.model,
+						created: Math.floor(task.createdAt.getTime() / 1000),
+						choices: [
+							{
+								index: 0,
+								delta: {
+									role: 'assistant',
+									content: null,
+								},
+								logprobs: null,
+								finish_reason: result.finishReason
+									? finishReasonMap[result.finishReason]
+									: 'stop',
+							},
+						],
+					}
+
+					const toolCalls: OpenAI.ChatCompletionChunk.Choice.Delta.ToolCall[] =
+						result.message.toolCalls!.map((call, index) => {
+							return {
+								index,
+								id: call.id,
+								type: 'function',
+								function: {
+									name: call.name,
+									arguments: JSON.stringify(call.parameters),
+								},
+							}
+						})
+					streamedToolCallChunk.choices[0].delta.tool_calls = toolCalls
+					res.write(`data: ${JSON.stringify(streamedToolCallChunk)}\n\n`)
+				}
+				if (args.stream_options?.include_usage) {
+					const finalChunk: OpenAIChatCompletionChunk = {
+						id: task.id,
+						object: 'chat.completion.chunk',
+						model: task.model,
+						created: Math.floor(task.createdAt.getTime() / 1000),
+						system_fingerprint: instance.fingerprint,
+						choices: [
+							{
+								index: 0,
+								delta: {},
+								logprobs: null,
+								finish_reason: result.finishReason
+									? finishReasonMap[result.finishReason]
+									: 'stop',
+							},
+						],
+						usage: {
+							prompt_tokens: result.promptTokens,
+							completion_tokens: result.completionTokens,
+							total_tokens: result.contextTokens,
+						},
+					}
+					res.write(`data: ${JSON.stringify(finalChunk)}\n\n`)
+				}
+				res.write('data: [DONE]')
+				res.end()
+			} else {
+				const response: OpenAI.ChatCompletion = {
+					id: task.id,
+					model: task.model,
+					object: 'chat.completion',
+					created: Math.floor(task.createdAt.getTime() / 1000),
+					system_fingerprint: instance.fingerprint,
+					choices: [
+						{
+							index: 0,
+							message: {
+								role: 'assistant',
+								content: createResponseMessageContent(result.message.content),
+								refusal: null,
+							},
+							logprobs: null,
+							finish_reason: result.finishReason
+								? finishReasonMap[result.finishReason]
+								: 'stop',
+						},
+					],
+					usage: {
+						prompt_tokens: result.promptTokens,
+						completion_tokens: result.completionTokens,
+						total_tokens: result.contextTokens,
+					},
+				}
+				if (
+					'toolCalls' in result.message &&
+					result.message.toolCalls?.length
+				) {
+					response.choices[0].message.tool_calls =
+						result.message.toolCalls.map((call) => {
+							return {
+								id: call.id,
+								type: 'function',
+								function: {
+									name: call.name,
+									arguments: JSON.stringify(call.parameters),
+								},
+							}
+						})
+				}
+				res.writeHead(200, { 'Content-Type': 'application/json' })
+				res.end(JSON.stringify(response, null, 2))
+			}
+		} catch (e) {
+			console.error(e)
+			if (args.stream) {
+				res.write('data: [ERROR]')
+			} else {
+				res.writeHead(500, { 'Content-Type': 'application/json' })
+				res.end(JSON.stringify({ error: 'Internal server error' }))
+			}
+		}
+	}
+}