inference-server 1.0.0-beta.19

package/src/api/openai/handlers/completions.ts

@@ -0,0 +1,196 @@
+import type { IncomingMessage, ServerResponse } from 'node:http'
+import type { OpenAI } from 'openai'
+import type { ModelServer } from '#package/server.js'
+import { TextCompletionRequest } from '#package/types/index.js'
+import { parseJSONRequestBody } from '#package/api/parseJSONRequestBody.js'
+import { omitEmptyValues } from '#package/lib/util.js'
+import { finishReasonMap } from '../enums.js'
+
+interface OpenAICompletionParams
+	extends Omit<OpenAI.CompletionCreateParamsStreaming, 'stream'> {
+	stream?: boolean
+	top_k?: number
+	min_p?: number
+	repeat_penalty_num?: number
+}
+
+interface OpenAICompletionChunk extends OpenAI.Completions.Completion {
+	usage?: OpenAI.CompletionUsage
+}
+
+// v1/completions
+// https://platform.openai.com/docs/api-reference/completions/create
+export function createCompletionHandler(modelServer: ModelServer) {
+	return async (req: IncomingMessage, res: ServerResponse) => {
+		let args: OpenAICompletionParams
+
+		try {
+			const body = await parseJSONRequestBody(req)
+			args = body
+		} catch (e) {
+			console.error(e)
+			res.writeHead(400, { 'Content-Type': 'application/json' })
+			res.end(JSON.stringify({ error: 'Invalid request' }))
+			return
+		}
+		
+		// TODO ajv schema validation?
+		if (!args.model || !args.prompt) {
+			res.writeHead(400, { 'Content-Type': 'application/json' })
+			res.end(JSON.stringify({ error: 'Invalid request' }))
+			return
+		}
+		if (!modelServer.modelExists(args.model)) {
+			res.writeHead(400, { 'Content-Type': 'application/json' })
+			res.end(JSON.stringify({ error: 'Invalid model' }))
+			return
+		}
+
+		const controller = new AbortController()
+		req.on('close', () => {
+			console.debug('Client closed connection')
+			controller.abort()
+		})
+		req.on('end', () => {
+			console.debug('Client ended connection')
+			controller.abort()
+		})
+
+		try {
+			if (args.stream) {
+				res.writeHead(200, {
+					'Content-Type': 'text/event-stream',
+					'Cache-Control': 'no-cache',
+					Connection: 'keep-alive',
+				})
+				res.flushHeaders()
+			}
+
+			let prompt = args.prompt
+
+			if (typeof prompt !== 'string') {
+				throw new Error('Prompt must be a string')
+			}
+
+			let stop = args.stop ? args.stop : undefined
+			if (typeof stop === 'string') {
+				stop = [stop]
+			}
+
+			const completionReq = omitEmptyValues<TextCompletionRequest>({
+				model: args.model,
+				prompt: args.prompt as string,
+				temperature: args.temperature ? args.temperature : undefined,
+				stream: args.stream ? Boolean(args.stream) : false,
+				maxTokens: args.max_tokens ? args.max_tokens : undefined,
+				seed: args.seed ? args.seed : undefined,
+				stop,
+				frequencyPenalty: args.frequency_penalty
+					? args.frequency_penalty
+					: undefined,
+				presencePenalty: args.presence_penalty
+					? args.presence_penalty
+					: undefined,
+				tokenBias: args.logit_bias ? args.logit_bias : undefined,
+				topP: args.top_p ? args.top_p : undefined,
+				// additional non-spec params
+				repeatPenaltyNum: args.repeat_penalty_num
+					? args.repeat_penalty_num
+					: undefined,
+				minP: args.min_p ? args.min_p : undefined,
+				topK: args.top_k ? args.top_k : undefined,
+			})
+
+			const { instance, release } = await modelServer.requestInstance(
+				completionReq,
+				controller.signal,
+			)
+			const task = instance.processTextCompletionTask(completionReq, {
+				signal: controller.signal,
+				onChunk: (chunk) => {
+					if (args.stream) {
+						const chunkData: OpenAICompletionChunk = {
+							id: task.id,
+							model: task.model,
+							object: 'text_completion',
+							created: Math.floor(task.createdAt.getTime() / 1000),
+							choices: [
+								{
+									index: 0,
+									text: chunk.text,
+									logprobs: null,
+									// @ts-ignore official api returns null here in the same case
+									finish_reason: null,
+								},
+							],
+						}
+						res.write(`data: ${JSON.stringify(chunkData)}\n\n`)
+					}
+				},
+			})
+			const result = await task.result
+			release()
+
+			if (args.stream) {
+				if (args.stream_options?.include_usage) {
+					const finalChunk: OpenAICompletionChunk = {
+						id: task.id,
+						model: task.model,
+						object: 'text_completion',
+						created: Math.floor(task.createdAt.getTime() / 1000),
+						choices: [
+							{
+								index: 0,
+								text: '',
+								logprobs: null,
+								// @ts-ignore
+								finish_reason: result.finishReason
+									? finishReasonMap[result.finishReason]
+									: 'stop',
+							},
+						],
+					}
+					res.write(
+						`data: ${JSON.stringify(finalChunk)}\n\n`,
+					)
+				}
+				res.write('data: [DONE]')
+				res.end()
+			} else {
+				const response: OpenAI.Completions.Completion = {
+					id: task.id,
+					model: task.model,
+					object: 'text_completion',
+					created: Math.floor(task.createdAt.getTime() / 1000),
+					system_fingerprint: instance.fingerprint,
+					choices: [
+						{
+							index: 0,
+							text: result.text,
+							logprobs: null,
+							// @ts-ignore
+							finish_reason: result.finishReason
+								? finishReasonMap[result.finishReason]
+								: 'stop',
+						},
+					],
+					usage: {
+						prompt_tokens: result.promptTokens,
+						completion_tokens: result.completionTokens,
+						total_tokens: result.contextTokens,
+					},
+				}
+				res.writeHead(200, { 'Content-Type': 'application/json' })
+				res.end(JSON.stringify(response, null, 2))
+			}
+		} catch (err) {
+			console.error(err)
+			if (args.stream) {
+				res.write('data: [ERROR]')
+			} else {
+				res.writeHead(500, { 'Content-Type': 'application/json' })
+				res.end(JSON.stringify({ error: 'Internal server error' }))
+			}
+		}
+	}
+}

package/src/api/openai/handlers/embeddings.ts

@@ -0,0 +1,92 @@
+import { IncomingMessage, ServerResponse } from 'node:http'
+import type { OpenAI } from 'openai'
+import { EmbeddingRequest } from '#package/types/index.js'
+import { parseJSONRequestBody } from '#package/api/parseJSONRequestBody.js'
+import { omitEmptyValues } from '#package/lib/util.js'
+import { ModelServer } from '#package/server.js'
+
+type OpenAIEmbeddingsParams = OpenAI.EmbeddingCreateParams
+
+// v1/embeddings
+// https://platform.openai.com/docs/api-reference/embeddings
+export function createEmbeddingsHandler(modelServer: ModelServer) {
+	return async (req: IncomingMessage, res: ServerResponse) => {
+		let args: OpenAIEmbeddingsParams
+
+		try {
+			const body = await parseJSONRequestBody(req)
+			args = body
+		} catch (e) {
+			console.error(e)
+			res.writeHead(400, { 'Content-Type': 'application/json' })
+			res.end(JSON.stringify({ error: 'Invalid request' }))
+			return
+		}
+		
+		// TODO ajv schema validation?
+		if (!args.model || !args.input) {
+			res.writeHead(400, { 'Content-Type': 'application/json' })
+			res.end(JSON.stringify({ error: 'Invalid request' }))
+			return
+		}
+		if (!modelServer.modelExists(args.model)) {
+			res.writeHead(400, { 'Content-Type': 'application/json' })
+			res.end(JSON.stringify({ error: 'Invalid model' }))
+			return
+		}
+
+		const controller = new AbortController()
+		req.on('close', () => {
+			console.debug('Client closed connection')
+			controller.abort()
+		})
+		req.on('end', () => {
+			console.debug('Client ended connection')
+			controller.abort()
+		})
+
+		try {
+
+
+			let input = args.input
+
+			if (typeof input !== 'string') {
+				throw new Error('Input must be a string')
+			}
+
+			const embeddingsReq = omitEmptyValues<EmbeddingRequest>({
+				model: args.model,
+				input: args.input as string,
+			})
+
+			const { instance, release } = await modelServer.requestInstance(
+				embeddingsReq,
+				controller.signal,
+			)
+			const task = instance.processEmbeddingTask(embeddingsReq)
+			const result = await task.result
+			release()
+
+			const response: OpenAI.CreateEmbeddingResponse = {
+				model: instance.modelId,
+				object: 'list',
+				data: result.embeddings.map((embedding, index) => ({
+					embedding: Array.from(embedding),
+					index,
+					object: 'embedding',
+				})),
+				usage: {
+					prompt_tokens: result.inputTokens,
+					total_tokens: result.inputTokens,
+				},
+			}
+			res.writeHead(200, { 'Content-Type': 'application/json' })
+			res.end(JSON.stringify(response, null, 2))
+
+		} catch (err) {
+			console.error(err)
+			res.writeHead(500, { 'Content-Type': 'application/json' })
+			res.end(JSON.stringify({ error: 'Internal server error' }))
+		}
+	}
+}

package/src/api/openai/handlers/images.ts

@@ -0,0 +1,3 @@
+
+// v1/images endpoints
+// https://platform.openai.com/docs/api-reference/images/create

package/src/api/openai/handlers/models.ts

@@ -0,0 +1,33 @@
+import type { IncomingMessage, ServerResponse } from 'node:http'
+import path from 'node:path'
+import type { OpenAI } from 'openai'
+import type { ModelServer } from '#package/server'
+
+// https://platform.openai.com/docs/api-reference/models/list
+export function createModelsHandler(modelServer: ModelServer) {
+	return async (req: IncomingMessage, res: ServerResponse) => {
+		
+		const models = modelServer.store.getStatus()
+		const data: OpenAI.Model[] = Object.entries(models).map(
+			([id, info]) => {
+				// const lastModDate = new Date(info.source.lastModified)
+				// const created = Math.floor(lastModDate.getTime() / 1000)
+				
+				// const dirPath = path.dirname(info.source.file);
+				// const lastDir = path.basename(dirPath);
+				// const baseName = path.basename(info.source.file);
+				const owned_by = info.engine// + ':' + path.join(lastDir, baseName);
+
+				return {
+					object: 'model',
+					id,
+					created: 0,
+					owned_by,
+				}
+			},
+		)
+
+		res.writeHead(200, { 'Content-Type': 'application/json' })
+		res.end(JSON.stringify({ object: 'list', data }, null, 2))
+	}
+}

package/src/api/openai/handlers/transcription.ts

@@ -0,0 +1,2 @@
+// TODO
+// https://platform.openai.com/docs/api-reference/audio/createTranscription

package/src/api/openai/index.ts

@@ -0,0 +1,16 @@
+import type { ModelServer } from '#package/server.js'
+import { createChatCompletionHandler } from './handlers/chat.js'
+import { createCompletionHandler } from './handlers/completions.js'
+import { createModelsHandler } from './handlers/models.js'
+import { createEmbeddingsHandler } from './handlers/embeddings.js'
+
+
+// See OpenAI API specs at https://github.com/openai/openai-openapi/blob/master/openapi.yaml
+export function createOpenAIRequestHandlers(modelServer: ModelServer) {
+	return {
+		chatCompletions: createChatCompletionHandler(modelServer),
+		completions: createCompletionHandler(modelServer),
+		models: createModelsHandler(modelServer),
+		embeddings: createEmbeddingsHandler(modelServer),
+	}
+}

package/src/api/parseJSONRequestBody.ts

@@ -0,0 +1,26 @@
+import { IncomingMessage } from 'node:http'
+
+export function parseJSONRequestBody(req: IncomingMessage | Request): Promise<any> {
+	return new Promise((resolve, reject) => {
+
+		// if request is from express theres no need to parse anything
+		if ('body' in req) {
+			resolve(req.body)
+			return
+		}
+		
+		// for native http server
+		let body = ''
+		req.on('data', (chunk) => {
+			body += chunk.toString()
+		})
+		req.on('end', () => {
+			try {
+				const data = JSON.parse(body)
+				resolve(data)
+			} catch (error) {
+				reject(error)
+			}
+		})
+	})
+}

package/src/api/v1/DRAFT.md

@@ -0,0 +1,16 @@
+See [discussion](https://github.com/iimez/inference-server/discussions/8) for more details.
+
+### Task API
+
+- `POST /tasks/text-completion`
+- `POST /tasks/chat-completion` should continue to be stateless
+- `GET /tasks/{task_id}`
+- `DELETE /tasks/{task_id}`
+
+### Thread API
+
+- `POST /threads`
+- `POST /threads/{thread_id}` mutate state without generating anything
+- `POST /threads/{thread_id}/generate` to generate a new assistant message
+- `GET /threads/{thread_id}`
+- `DELETE /threads/{thread_id}`

package/src/api/v1/index.ts

@@ -0,0 +1,37 @@
+import { IncomingMessage, ServerResponse } from 'node:http'
+import express from 'express'
+import { ModelPool } from '#package/pool.js'
+import { ModelServer } from '#package/server.js'
+
+export function createAPIMiddleware(server: ModelServer) {
+	const router = express.Router()
+	
+	router.use((req, res, next) => {
+		console.debug('API call', {
+			method: req.method,
+			url: req.url,
+			body: req.body,
+		})
+		next()
+	})
+	return router
+	// return async (req: IncomingMessage, res: ServerResponse) => {
+		
+	// 	let args: any
+		
+	// 	try {
+	// 		const body = await parseJSONRequestBody(req)
+	// 		args = body
+	// 	} catch (e) {
+	// 		console.error(e)
+	// 		res.writeHead(400, { 'Content-Type': 'application/json' })
+	// 		res.end(JSON.stringify({ error: 'Invalid request' }))
+	// 		return
+	// 	}
+		
+	// 	console.debug('Handler', JSON.stringify(args, null, 2))
+	// 	res.writeHead(200, { 'Content-Type': 'application/json' })
+	// 	res.end(JSON.stringify({ message: 'Hello' }))
+		
+	// }
+}

package/src/cli.ts

@@ -0,0 +1,9 @@
+#!/usr/bin/env node
+process.stdout.write("Nothing here, yet.\n");
+
+/* API Ideas:
+- `infsrv ls` - list stored models
+- `infsrv rm <name>` - remove a stored model
+- `infsrv show <name>` - show details of a stored model
+- `infsrv prepare <config>` - download everything required for given config
+*/