inference-server 1.0.0-beta.19

package/src/engines/transformers-js/util.ts

@@ -0,0 +1,40 @@
+export async function remoteFileExists(url: string): Promise<boolean> {
+	try {
+		const response = await fetch(url, { method: 'HEAD' })
+		return response.ok
+	} catch (error) {
+		console.error('Error checking remote file:', error)
+		return false
+	}
+}
+
+interface HuggingfaceModelInfo {
+	modelId: string
+	branch: string
+}
+
+export function parseHuggingfaceModelIdAndBranch(url: string): HuggingfaceModelInfo {
+	// url to the hub model, like https://huggingface.co/jinaai/jina-clip-v1
+	const parsedUrl = new URL(url)
+	const urlSegments = parsedUrl.pathname.split('/')
+	const repoOrg = urlSegments[1]
+	const repoName = urlSegments[2]
+	const branch = urlSegments[4] || 'main'
+	return {
+		modelId: `${repoOrg}/${repoName}`,
+		branch,
+	}
+	// if (filePath) { // path to the cached model, like /path/to/huggingface/jinaai/jina-clip-v1-main
+	// 	const filePathSegments = filePath.split('/')
+	// 	const modelDir = filePathSegments[filePathSegments.length - 1]
+	// 	const branch = modelDir.split('-').pop() || 'main'
+	// 	const repoName = modelDir.replace(new RegExp(`-${branch}$`), '')
+	// 	const repoOrg = filePathSegments[filePathSegments.length - 2]
+	// 	const modelId = `${repoOrg}/${repoName}`
+	// 	return {
+	// 		modelId,
+	// 		branch,
+	// 	}
+	// }
+	// throw new Error('Either url or filePath must be provided')
+}

package/src/engines/transformers-js/validateModelFiles.ts

@@ -0,0 +1,168 @@
+import fs from 'node:fs'
+import { AutoModel, AutoProcessor, AutoTokenizer } from '@huggingface/transformers'
+import { TransformersJsModel } from '#package/types/index.js'
+import { resolveModelFileLocation } from '#package/lib/resolveModelFileLocation.js'
+import { TransformersJsModelConfig } from './engine.js'
+import { parseHuggingfaceModelIdAndBranch, remoteFileExists } from './util.js'
+
+async function validateModel(
+	modelOpts: TransformersJsModel,
+	config: TransformersJsModelConfig,
+	modelPath: string,
+): Promise<string | undefined> {
+	const modelClass = modelOpts.modelClass ?? AutoModel
+	const device = config.device?.gpu ? 'gpu' : 'cpu'
+	try {
+		const model = await modelClass.from_pretrained(modelPath, {
+			local_files_only: true,
+			device: device,
+			dtype: modelOpts.dtype || 'fp32',
+		})
+		await model.dispose()
+	} catch (error) {
+		return `Failed to load model (${error})`
+	}
+	return undefined
+}
+
+async function validateTokenizer(
+	modelOpts: TransformersJsModel,
+	config: TransformersJsModelConfig,
+	modelPath: string,
+): Promise<string | undefined> {
+	const tokenizerClass = modelOpts.tokenizerClass ?? AutoTokenizer
+	try {
+		await tokenizerClass.from_pretrained(modelPath, {
+			local_files_only: true,
+		})
+	} catch (error) {
+		return `Failed to load tokenizer (${error})`
+	}
+	return undefined
+}
+
+async function validateProcessor(
+	modelOpts: TransformersJsModel,
+	config: TransformersJsModelConfig,
+	modelPath: string,
+): Promise<string | undefined> {
+	const processorClass = modelOpts.processorClass ?? AutoProcessor
+	try {
+		if (modelOpts.processor) {
+			const processorPath = resolveModelFileLocation({
+				url: modelOpts.processor.url,
+				filePath: modelOpts.processor.file,
+				modelsCachePath: config.modelsCachePath,
+			})
+			await processorClass.from_pretrained(processorPath, {
+				local_files_only: true,
+			})
+		} else {
+			if (modelOpts.processorClass) {
+				await processorClass.from_pretrained(modelPath, {
+					local_files_only: true,
+				})
+			} else if (config.url) {
+				const { branch } = parseHuggingfaceModelIdAndBranch(config.url)
+				const [hasProcessor, hasPreprocessor] = await Promise.all([
+					remoteFileExists(`${config.url}/blob/${branch}/processor_config.json`),
+					remoteFileExists(`${config.url}/blob/${branch}/preprocessor_config.json`),
+				])
+				if (hasProcessor || hasPreprocessor) {
+					await processorClass.from_pretrained(modelPath, {
+						local_files_only: true,
+					})
+				}
+			}
+		}
+	} catch (error) {
+		return `Failed to load processor (${error})`
+	}
+	return undefined
+}
+
+interface ComponentValidationErrors {
+	model?: string
+	tokenizer?: string
+	processor?: string
+}
+
+interface ModelValidationErrors {
+	textModel?: ComponentValidationErrors
+	visionModel?: ComponentValidationErrors
+	speechModel?: ComponentValidationErrors
+}
+
+export interface ModelValidationResult {
+	message: string
+	errors?: ModelValidationErrors
+}
+
+export async function validateModelFiles(
+	config: TransformersJsModelConfig,
+): Promise<ModelValidationResult | undefined> {
+	if (!fs.existsSync(config.location)) {
+		return {
+			message: `model directory does not exist: ${config.location}`,
+		}
+	}
+
+	let modelPath = config.location
+	if (!modelPath.endsWith('/')) {
+		modelPath += '/'
+	}
+
+	const validateModelComponents = async (modelOpts: TransformersJsModel) => {
+		const componentValidationPromises = [
+			validateModel(modelOpts, config, modelPath),
+			validateTokenizer(modelOpts, config, modelPath),
+			validateProcessor(modelOpts, config, modelPath),
+		]
+		
+
+		// if (modelOpts.processor) {
+		// 	componentValidationPromises.push(validateProcessor(modelOpts, config, modelPath))
+		// }
+		const [model, tokenizer, processor] = await Promise.all(componentValidationPromises)
+		const result: ComponentValidationErrors = {}
+		if (model) result.model = model
+		if (tokenizer) result.tokenizer = tokenizer
+		if (processor) result.processor = processor
+		return result
+	}
+
+	const modelValidationPromises: any = {}
+	const noModelConfigured = !config.textModel && !config.visionModel && !config.speechModel
+	if (config.textModel || noModelConfigured) {
+		modelValidationPromises.textModel = validateModelComponents(config.textModel || {})
+	}
+	if (config.visionModel) {
+		modelValidationPromises.visionModel = validateModelComponents(config.visionModel)
+	}
+	if (config.speechModel) {
+		modelValidationPromises.speechModel = validateModelComponents(config.speechModel)
+	}
+
+	await Promise.all(Object.values(modelValidationPromises))
+	const validationErrors: ModelValidationErrors = {}
+	const textModelErrors = await modelValidationPromises.textModel
+	if (textModelErrors && Object.keys(textModelErrors).length) {
+		validationErrors.textModel = textModelErrors
+	}
+	const visionModelErrors = await modelValidationPromises.visionModel
+	if (visionModelErrors && Object.keys(visionModelErrors).length) {
+		validationErrors.visionModel = visionModelErrors
+	}
+	const speechModelErrors = await modelValidationPromises.speechModel
+	if (speechModelErrors && Object.keys(speechModelErrors).length) {
+		validationErrors.speechModel = speechModelErrors
+	}
+
+	if (Object.keys(validationErrors).length > 0) {
+		return {
+			message: 'Failed to validate model components',
+			errors: validationErrors,
+		}
+	}
+	return undefined
+}

package/src/experiments/ChatWithVision.ts

@@ -0,0 +1,103 @@
+
+import { EngineChatCompletionArgs, ImageToTextRequest, ModelEngine } from '#package/types/index.js'
+import { CustomEngine } from '#package/engines/index.js'
+
+// an experimental engine that replaces images with their descriptions before passing them to a chat model
+
+export class ChatWithVisionEngine extends CustomEngine implements ModelEngine {
+	imageToTextModel: string
+	chatModel: string
+	
+	constructor({ imageToTextModel, chatModel }: { imageToTextModel: string, chatModel: string }) {
+		super()
+		this.imageToTextModel = imageToTextModel
+		this.chatModel = chatModel
+	}
+
+	async processChatCompletionTask (
+		args: EngineChatCompletionArgs,
+	) {
+
+		const imageTextPromises: any[] = []
+		const imageToTextModel = await this.pool.requestInstance({
+			model: this.imageToTextModel,
+		})
+		
+		const messagesWithImageDescriptions = [... args.request.messages]
+
+		for (let m = 0; m < messagesWithImageDescriptions.length; m++) {
+			const message = messagesWithImageDescriptions[m]
+			if (!Array.isArray(message.content)) {
+				continue
+			}
+			for (let p = 0; p < message.content.length; p++) {
+				const contentPart = message.content[p]
+				if (contentPart.type !== 'image') {
+					continue
+				}
+				imageTextPromises.push(new Promise(async (resolve, reject) => {
+					// Florence2 prompt doc
+					// "task_prompts_without_inputs": {
+					// 	"<OCR>": "What is the text in the image?",
+					// 	"<OCR_WITH_REGION>": "What is the text in the image, with regions?",
+					// 	"<CAPTION>": "What does the image describe?",
+					// 	"<DETAILED_CAPTION>": "Describe in detail what is shown in the image.",
+					// 	"<MORE_DETAILED_CAPTION>": "Describe with a paragraph what is shown in the image.",
+					// 	"<OD>": "Locate the objects with category name in the image.",
+					// 	"<DENSE_REGION_CAPTION>": "Locate the objects in the image, with their descriptions.",
+					// 	"<REGION_PROPOSAL>": "Locate the region proposals in the image."
+					// },
+					// "task_prompts_with_input": {
+					// 	"<CAPTION_TO_PHRASE_GROUNDING>": "Locate the phrases in the caption: {input}",
+					// 	"<REFERRING_EXPRESSION_SEGMENTATION>": "Locate {input} in the image with mask",
+					// 	"<REGION_TO_SEGMENTATION>": "What is the polygon mask of region {input}",
+					// 	"<OPEN_VOCABULARY_DETECTION>": "Locate {input} in the image.",
+					// 	"<REGION_TO_CATEGORY>": "What is the region {input}?",
+					// 	"<REGION_TO_DESCRIPTION>": "What does the region {input} describe?",
+					// 	"<REGION_TO_OCR>": "What text is in the region {input}?"
+					// }
+					// const imageData = await fetch(contentPart.image.url).then((res) => res.arrayBuffer())
+					const task = imageToTextModel.instance.processImageToTextTask({
+						model: this.imageToTextModel,
+						// url: contentPart.url,
+						image: contentPart.image,
+						prompt: 'What does the image describe?',
+					})
+					const result = await task.result
+					resolve({
+						text: result.text,
+						messageIndex: m,
+						contentPartIndex: p,
+					})
+				}))
+			}
+		}
+
+		const imageTextResults = await Promise.all(imageTextPromises)
+		imageToTextModel.release()
+		console.debug('Image text results', imageTextResults)
+		
+		for (const imageTextResult of imageTextResults) {
+			const { text, messageIndex, contentPartIndex } = imageTextResult
+			const message = messagesWithImageDescriptions[messageIndex]
+			// if ('type' in message.content[contentPartIndex]) {
+			// message.content[contentPartIndex].type = 'text'
+			// @ts-ignore
+			message.content[contentPartIndex] = {
+				type: 'text',
+				text: `User uploaded image: ${text}`,
+			}
+		}
+
+		const chatRequest = { ...args.request, messages: messagesWithImageDescriptions, model: this.chatModel }
+		const chatModel = await this.pool.requestInstance(chatRequest)
+		const task = chatModel.instance.processChatCompletionTask(chatRequest, {
+			onChunk: (chunk) => {
+				if (args.onChunk) args.onChunk(chunk)
+			},
+		})
+		const result = await task.result
+		chatModel.release()
+		return result
+	}
+}

package/src/experiments/StableDiffPromptGenerator.ts

@@ -0,0 +1,2 @@
+// https://huggingface.co/Gustavosta/MagicPrompt-Stable-Diffusion
+// https://huggingface.co/QuantFactory/MagicPrompt-Stable-Diffusion-GGUF

package/src/experiments/VoiceFunctionCall.ts

@@ -0,0 +1,71 @@
+import {
+	EngineSpeechToTextArgs,
+	ModelEngine,
+	ToolDefinition,
+} from '#package/types/index.js'
+import { CustomEngine } from '#package/engines/index.js'
+
+type EngineArgs = {
+	speechToTextModel: string
+	chatModel: string
+	tools: Record<string, ToolDefinition>
+}
+
+// an experimental engine that forwards a transcription to a (function calling) chat model
+
+export class VoiceFunctionCallEngine
+	extends CustomEngine
+	implements ModelEngine
+{
+	speechToTextModel: string
+	chatModel: string
+	tools: Record<string, ToolDefinition>
+
+	constructor({ speechToTextModel, chatModel, tools }: EngineArgs) {
+		super()
+		this.speechToTextModel = speechToTextModel
+		this.chatModel = chatModel
+		this.tools = tools
+	}
+	
+	async createTranscription(args: EngineSpeechToTextArgs) {
+		const speechToTextModel = await this.pool.requestInstance({
+			model: this.speechToTextModel,
+		})
+		const transcriptionTask = speechToTextModel.instance.processSpeechToTextTask(
+			{
+				...args.request,
+				model: this.speechToTextModel,
+			},
+		)
+		const transcription = await transcriptionTask.result
+		speechToTextModel.release()
+		return transcription.text
+	}
+
+	async processSpeechToTextTask(args: EngineSpeechToTextArgs) {
+		const [transcription, chatModel] = await Promise.all([
+			this.createTranscription(args),
+			this.pool.requestInstance({
+				model: this.chatModel,
+			}),
+		])
+		const chatTask = chatModel.instance.processChatCompletionTask({
+			model: this.chatModel,
+			tools: this.tools,
+			messages: [
+				{
+					role: 'user',
+					content: transcription,
+				},
+			],
+		}, {
+			onChunk: args.onChunk,
+		})
+		const chatResponse = await chatTask.result
+		chatModel.release()
+		return {
+			text: chatResponse.message.content,
+		}
+	}
+}

package/src/http.ts

@@ -0,0 +1,72 @@
+import http from 'node:http'
+import { ListenOptions } from 'node:net'
+import express from 'express'
+import cors from 'cors'
+import { createOpenAIRequestHandlers } from '#package/api/openai/index.js'
+import { createAPIMiddleware } from '#package/api/v1/index.js'
+import { LogLevel } from '#package/lib/logger.js'
+import { ModelServer, ModelServerOptions, startModelServer } from '#package/server.js'
+
+export function createOpenAIMiddleware(modelServer: ModelServer) {
+	const router = express.Router()
+	const requestHandlers = createOpenAIRequestHandlers(modelServer)
+	router.get('/v1/models', requestHandlers.models)
+	router.post('/v1/completions', requestHandlers.completions)
+	router.post('/v1/chat/completions', requestHandlers.chatCompletions)
+	router.post('/v1/embeddings', requestHandlers.embeddings)
+	return router
+}
+
+export function createExpressMiddleware(modelServer: ModelServer) {
+	const router = express.Router()
+	router.get('/', (req, res) => {
+		res.json(modelServer.getStatus())
+	})
+	router.use('/openai', createOpenAIMiddleware(modelServer))
+	router.use('/llama', createAPIMiddleware(modelServer))
+	return router
+}
+
+export interface ModelHTTPServerOptions extends ModelServerOptions {
+	listen?: ListenOptions
+	logLevel?: LogLevel
+}
+
+export class ModelHTTPServer {
+	httpServer: http.Server
+	modelServer: ModelServer
+	listenOptions: ListenOptions
+	
+	constructor(options: ModelHTTPServerOptions) {
+		const { listen, ...modelServerOpts } = options
+		this.modelServer = new ModelServer(modelServerOpts)
+		this.listenOptions = listen ?? { port: 3000 }
+		const app = express()
+		app.use(
+			cors(),
+			express.json({ limit: '50mb' }),
+			createExpressMiddleware(this.modelServer),
+		)
+	
+		app.set('json spaces', 2)
+		this.httpServer = http.createServer(app)
+	}
+	
+	async start() {
+		await this.modelServer.start()
+		await new Promise<void>((resolve) => {
+			this.httpServer.listen(this.listenOptions, resolve)
+		})
+	}
+	
+	async stop() {
+		this.httpServer.close()
+		await this.modelServer.stop()
+	}
+}
+
+export async function startHTTPServer(options: ModelHTTPServerOptions) {
+	const server = new ModelHTTPServer(options)
+	await server.start()
+	return server
+}

package/src/index.ts

@@ -0,0 +1,7 @@
+export * from './api/openai/index.js'
+export * from './types/index.js'
+export * from './pool.js'
+export * from './instance.js'
+export * from './store.js'
+export * from './server.js'
+export * from './http.js'