inference-server 1.0.0-beta.19

package/src/lib/acquireFileLock.ts

@@ -0,0 +1,38 @@
+import fs from 'node:fs'
+import path from 'node:path'
+import lockfile from 'proper-lockfile'
+import { touchFileSync } from '#package/lib/util.js'
+
+/**
+ * Acquire a lock on a file or directory, creating the resource if it does not exist.
+ * @param fileOrDirPath Path to the file or directory to lock.
+ * @param signal Abort signal to release the lock.
+ * @returns A function that releases the lock.
+ */
+export function acquireFileLock(fileOrDirPath: string, signal?: AbortSignal): Promise<() => void> {
+	const pathInfo = path.parse(fileOrDirPath)
+	const looksLikeFile = !!pathInfo.ext
+	const doesExist = fs.existsSync(fileOrDirPath)
+	if (looksLikeFile && !doesExist) {
+		touchFileSync(fileOrDirPath)
+	}
+	if (!looksLikeFile && !doesExist) {
+		fs.mkdirSync(fileOrDirPath, { recursive: true })
+	}
+	const isLocked = lockfile.checkSync(fileOrDirPath)
+	const lockExists = fs.existsSync(`${fileOrDirPath}.lock`)
+	if (!isLocked && lockExists) {
+		fs.unlinkSync(`${fileOrDirPath}.lock`)
+	}
+	return new Promise((resolve, reject) => {
+		lockfile.lock(fileOrDirPath, { retries: { forever: true } })
+		.then((release) => {
+			signal?.addEventListener('abort', release)
+			resolve(() => {
+				release()
+				signal?.removeEventListener('abort', release)
+			})
+		})
+		.catch(reject)
+	})
+}

package/src/lib/calculateContextIdentity.ts

@@ -0,0 +1,53 @@
+import crypto from 'node:crypto'
+import { ChatMessage } from '#package/types/index.js'
+import { flattenMessageTextContent } from './flattenMessageTextContent.js'
+
+export interface CalculateContextIdentityOptions {
+	messages?: ChatMessage[]
+	text?: string
+	dropLastMessage?: boolean
+}
+
+export function calculateContextIdentity({
+	messages,
+	text,
+	dropLastMessage = false,
+}: CalculateContextIdentityOptions): string {
+
+	if (text) {
+		return text
+	}
+	
+	if (messages?.length) {
+		const filteredMessages = messages.filter((message, i) => {
+			// remove all but the leading system message
+			if (message.role === 'system' && i !== 0) {
+				return false
+			}
+			if (message.role === 'tool') {
+				return false
+			}
+			const textContent = flattenMessageTextContent(message.content)
+			return !!textContent
+		})
+		if (dropLastMessage && filteredMessages.length > 1) {
+			if (filteredMessages[filteredMessages.length - 1].role !== 'user') {
+				console.warn('Dropping last message that is not a user message. This should not happen.')
+			}
+			filteredMessages.pop()
+		}
+		// we dont wanna json stringify because this would make message key order significant
+		const serializedMessages = filteredMessages
+			.map((message) => {
+				return message.role + ': ' + flattenMessageTextContent(message.content)
+			})
+			.join('\n')
+			const contextIdentity = crypto
+			.createHash('sha1')
+			.update(serializedMessages)
+			.digest('hex')
+		return contextIdentity
+	}
+	
+	return ''
+}

package/src/lib/calculateFileChecksum.ts

@@ -0,0 +1,18 @@
+import fs from 'node:fs'
+import crypto from 'node:crypto'
+
+export function calculateFileChecksum(filePath: string, hashType = 'sha256'): Promise<string> {
+	return new Promise((resolve, reject) => {
+		const fileStream = fs.createReadStream(filePath)
+		const result = crypto.createHash(hashType)
+		fileStream.on('error', reject)
+		fileStream.on('data', (chunk) => {
+				result.update(chunk)
+		})
+		fileStream.on('end', () => {
+			resolve(
+				result.digest('hex')
+			)
+		})
+	})
+}

package/src/lib/copyDirectory.ts

@@ -0,0 +1,29 @@
+import { promises as fs } from 'node:fs'
+import path from 'node:path'
+
+/**
+ * Recursively copy a directory and its contents
+ * @param src - Source directory
+ * @param dest - Destination directory
+ */
+export async function copyDirectory(src: string, dest: string): Promise<void> {
+	// Ensure the destination directory exists, create it if it doesn't
+	await fs.mkdir(dest, { recursive: true })
+
+	// Read the contents of the source directory
+	const entries = await fs.readdir(src, { withFileTypes: true })
+
+	// Iterate through all the entries (files and directories)
+	for (const entry of entries) {
+		const srcPath = path.join(src, entry.name)
+		const destPath = path.join(dest, entry.name)
+
+		if (entry.isDirectory()) {
+			// Recursively copy directories
+			await copyDirectory(srcPath, destPath)
+		} else {
+			// Copy files
+			await fs.copyFile(srcPath, destPath)
+		}
+	}
+}

package/src/lib/decodeAudio.ts

@@ -0,0 +1,39 @@
+import decode from 'audio-decode'
+import libSampleRate from '@alexanderolsen/libsamplerate-js'
+
+interface ResampleOptions {
+	inputSampleRate?: number
+	outputSampleRate: number
+	nChannels?: number
+}
+
+async function resampleAudioBuffer(input: Float32Array, opts: ResampleOptions) {
+	const nChannels = opts.nChannels ?? 2
+	const inputSampleRate = opts.inputSampleRate ?? 44100
+
+	const resampler = await libSampleRate.create(nChannels, inputSampleRate, opts.outputSampleRate, {
+		// http://www.mega-nerd.com/SRC/api_full.html http://www.mega-nerd.com/SRC/api_simple.html
+		converterType: libSampleRate.ConverterType.SRC_SINC_BEST_QUALITY, // default SRC_SINC_FASTEST. see API for more
+	})
+	const resampledData = resampler.simple(input)
+	resampler.destroy()
+	return resampledData
+}
+
+
+export async function decodeAudio(fileBuffer: ArrayBuffer | Uint8Array, sampleRate: number = 44100) {
+	const decodedAudio = await decode(fileBuffer)
+
+	let audio = decodedAudio.getChannelData(0)
+	
+	if (decodedAudio.sampleRate !== sampleRate) {
+		audio = await resampleAudioBuffer(audio, {
+			inputSampleRate: decodedAudio.sampleRate,
+			outputSampleRate: sampleRate,
+			nChannels: 1,
+		})
+	}
+	
+	return audio
+
+}

package/src/lib/downloadModelFile.ts

@@ -0,0 +1,70 @@
+import { downloadFile as createFileDownload } from 'ipull'
+import fs from 'node:fs'
+import path from 'node:path'
+import { FileDownloadProgress } from '#package/types/index.js'
+import { resolveModelFileLocation } from '#package/lib/resolveModelFileLocation.js'
+
+interface DownloadArgs {
+	url: string
+	filePath?: string
+	modelsCachePath: string
+	onProgress?: (progress: FileDownloadProgress) => void
+	signal?: AbortSignal
+}
+
+export async function downloadModelFile({ url, filePath, modelsCachePath, onProgress, signal }: DownloadArgs) {
+	let downloadUrl = url
+	const parsedUrl = new URL(url)
+	if (parsedUrl.hostname === 'huggingface.co') {
+		// TODO support auth headers
+		// https://ido-pluto.github.io/ipull/#md:custom-headers
+		const pathnameParts = parsedUrl.pathname.split('/')
+		if (pathnameParts.length > 3 && pathnameParts[3] === 'blob') {
+			const newUrl = new URL(url)
+			pathnameParts[3] = 'resolve'
+			newUrl.pathname = pathnameParts.join('/')
+			if (newUrl.searchParams.get('download') !== 'true') {
+				newUrl.searchParams.set('download', 'true')
+			}
+			downloadUrl = newUrl.href
+		}
+	}
+
+	const destinationFile = resolveModelFileLocation({
+		url: downloadUrl,
+		filePath,
+		modelsCachePath,
+	})
+
+	fs.mkdirSync(path.dirname(destinationFile), { recursive: true })
+	// TODO split gguf file support, could regex filename and download the rest
+	// see https://ido-pluto.github.io/ipull/#md:download-file-from-parts
+	const controller = await createFileDownload({
+		url: downloadUrl,
+		savePath: destinationFile,
+		skipExisting: true,
+	})
+
+	let partialBytes = 0
+	if (fs.existsSync(destinationFile)) {
+		partialBytes = fs.statSync(destinationFile).size
+	}
+	const progressInterval = setInterval(() => {
+		if (onProgress) {
+			onProgress({
+				file: destinationFile,
+				loadedBytes: controller.status.transferredBytes + partialBytes,
+				totalBytes: controller.status.totalBytes,
+			})
+		}
+	}, 3000)
+	if (signal) {
+		signal.addEventListener('abort', () => {
+			controller.close()
+		})
+	}
+	await controller.download()
+	if (progressInterval) {
+		clearInterval(progressInterval)
+	}
+}

package/src/lib/flattenMessageTextContent.ts

@@ -0,0 +1,19 @@
+import {
+	ChatMessage,
+	MessageContentPart,
+	MessageTextContentPart,
+} from '#package/types/index.js'
+
+function isTextContentPart(
+	part: MessageContentPart,
+): part is MessageTextContentPart {
+	return part.type === 'text'
+}
+
+export function flattenMessageTextContent(content: ChatMessage['content']): string {
+	if (typeof content === 'string') {
+		return content
+	}
+	const parts = content.filter(isTextContentPart)
+	return parts.map((part) => part.text).join('\n')
+}

package/src/lib/getCacheDirPath.ts

@@ -0,0 +1,34 @@
+import os from 'node:os'
+import path from 'node:path'
+
+/**
+ * Determines the appropriate cache directory for the current platform.
+ *
+ * Follows platform-specific conventions:
+ * - Windows: %LOCALAPPDATA%\Cache\inference-server
+ * - macOS: ~/Library/Caches/inference-server
+ * - Linux: $XDG_CACHE_HOME/inference-server or ~/.cache/inference-server
+ *
+ * @param {string} subDir - The name of the cache subdirectory
+ * @returns {string} The absolute path to the cache directory
+ */
+export function getCacheDirPath(subDir: string = ''): string {
+	const platform = process.platform
+	let basePath: string
+
+	switch (platform) {
+		case 'win32':
+			// Windows: Use %LOCALAPPDATA%\Cache
+			basePath = process.env.LOCALAPPDATA || path.join(os.homedir(), 'AppData', 'Local')
+			return path.join(basePath, 'Cache', 'inference-server', subDir)
+
+		case 'darwin':
+			// macOS: Use ~/Library/Caches
+			return path.join(os.homedir(), 'Library', 'Caches', 'inference-server', subDir)
+
+		default:
+			// Linux/Unix: Use $XDG_CACHE_HOME or ~/.cache
+			basePath = process.env.XDG_CACHE_HOME || path.join(os.homedir(), '.cache')
+			return path.join(basePath, 'inference-server', subDir)
+	}
+}

package/src/lib/loadImage.ts

@@ -0,0 +1,46 @@
+import { Image } from '#package/types/index.js'
+import sharp, { ResizeOptions } from 'sharp'
+
+interface ImageTransformationOptions {
+	resize?: {
+		width: number
+		height: number
+		fit?: ResizeOptions['fit']
+	}
+}
+
+export async function loadImageFromUrl(
+	url: string,
+	opts: ImageTransformationOptions = {},
+): Promise<Image> {
+	const imageBuffer = await fetch(url).then((res) => res.arrayBuffer())
+	const buffer = await Buffer.from(imageBuffer)
+	const sharpHandle = sharp(buffer).rotate()
+	if (opts.resize) {
+		sharpHandle.resize(opts.resize)
+	}
+	const { info } = await sharpHandle.toBuffer({ resolveWithObject: true })
+	return {
+		handle: sharpHandle,
+		height: opts.resize?.height ?? info.height,
+		width: opts.resize?.width ?? info.width,
+		channels: info.channels,
+	}
+}
+
+export async function loadImageFromFile(
+	filePath: string,
+	opts: ImageTransformationOptions = {},
+): Promise<Image> {
+	let sharpHandle = sharp(filePath).rotate()
+	if (opts.resize) {
+		sharpHandle.resize(opts.resize)
+	}
+	const { info } = await sharpHandle.toBuffer({ resolveWithObject: true })
+	return {
+		handle: sharpHandle,
+		height: info.height,
+		width: info.width,
+		channels: info.channels,
+	}
+}

package/src/lib/logger.ts

@@ -0,0 +1,112 @@
+import chalk from 'chalk'
+
+export const LogLevels = {
+	error: 'error',
+	warn: 'warn',
+	info: 'info',
+	debug: 'debug',
+	verbose: 'verbose',
+} as const
+
+export type LogLevel = keyof typeof LogLevels
+export type Logger = (level: LogLevel, message: string, meta?: any) => void
+
+export function withLogMeta(logger: Logger, meta: object) {
+	return (level: LogLevel, message: string, extraMeta: object = {}) => {
+		logger(level, message, { ...meta, ...extraMeta })
+	}
+}
+
+export function createSublogger(
+	minLevelOrLogger: LogLevel | Logger = LogLevels.warn,
+) {
+	if (minLevelOrLogger) {
+		return typeof minLevelOrLogger === 'string'
+			? createLogger(minLevelOrLogger)
+			: minLevelOrLogger
+	} else {
+		return createLogger(LogLevels.warn)
+	}
+}
+
+export function createLogger(minLevel: LogLevel) {
+	const levels = Object.keys(LogLevels).reverse()
+	const minLevelIndex = levels.indexOf(minLevel)
+
+	return function log(level: LogLevel, message: string, meta?: any) {
+		const levelIndex = levels.indexOf(level)
+		if (levelIndex >= minLevelIndex) {
+			const formattedMessage = formatMessage(level, message, meta)
+			switch (level) {
+				case LogLevels.error:
+					console.error(formattedMessage, meta?.error)
+					break
+				case LogLevels.warn:
+					console.warn(formattedMessage)
+					break
+				case LogLevels.info:
+					console.info(formattedMessage)
+					break
+				case LogLevels.debug:
+				case LogLevels.verbose:
+					console.debug(formattedMessage)
+					break
+			}
+		}
+	}
+}
+
+function formatMessage(level: LogLevel, message: string, meta: any = {}) {
+	const timestamp = new Date().toISOString().replace('T', ' ').substring(0, 22)
+	let messageStr = `[${timestamp}]`
+
+	switch (level) {
+		case LogLevels.error:
+			messageStr += chalk.red(`[erro]`)
+			break
+		case LogLevels.warn:
+			messageStr += chalk.yellow(`[warn]`)
+			break
+		case LogLevels.info:
+			messageStr += chalk.white(`[info]`)
+			break
+		case LogLevels.debug:
+			messageStr += chalk.gray(`[debg]`)
+			break
+		case LogLevels.verbose:
+			messageStr += chalk.gray(`[verb]`)
+			break
+	}
+
+	if (meta?.sequence) {
+		messageStr += ' ' + chalk.cyan(meta.sequence)
+	}
+
+	if (meta?.instance) {
+		messageStr += ' ' + chalk.magenta(meta.instance)
+	} else if (meta?.model) {
+		messageStr += ' ' + chalk.magenta(meta.model)
+	}
+
+	if (meta?.task) {
+		messageStr += ' ' + chalk.green(meta.task)
+	}
+
+	messageStr += ' ' + message
+
+	if (meta) {
+		const { instance, sequence, model, task, elapsed, error, ...otherData } =
+			meta
+		if (elapsed) {
+			if (elapsed < 1000) {
+				messageStr += ' ' + chalk.magenta(`+${elapsed.toFixed(2)}ms`)
+			} else {
+				messageStr += ' ' + chalk.magenta(`+${(elapsed / 1000).toFixed(2)}s`)
+			}
+		}
+		if (Object.keys(otherData).length > 0) {
+			messageStr += ' ' + JSON.stringify(otherData, null, 2)
+		}
+	}
+	return messageStr
+}

package/src/lib/math.ts

@@ -0,0 +1,31 @@
+/**
+ * Calculates the dot product of two vectors.
+ * @param vecA - The first vector.
+ * @param vecB - The second vector.
+ * @returns The dot product of vecA and vecB.
+ */
+function dotProduct(vecA: number[], vecB: number[]): number {
+	return vecA.reduce((sum, value, index) => sum + value * vecB[index], 0)
+}
+
+/**
+ * Calculates the magnitude of a vector.
+ * @param vec - The vector.
+ * @returns The magnitude of the vector.
+ */
+function magnitude(vec: number[]): number {
+	return Math.sqrt(vec.reduce((sum, value) => sum + value * value, 0))
+}
+
+/**
+ * Calculates the cosine similarity between two vectors.
+ * @param vecA - The first vector.
+ * @param vecB - The second vector.
+ * @returns The cosine similarity between vecA and vecB.
+ */
+export function cosineSimilarity(vecA: number[], vecB: number[]): number {
+	const dotProd = dotProduct(vecA, vecB)
+	const magnitudeA = magnitude(vecA)
+	const magnitudeB = magnitude(vecB)
+	return dotProd / (magnitudeA * magnitudeB)
+}

package/src/lib/resolveModelFileLocation.ts

@@ -0,0 +1,49 @@
+import path from 'node:path'
+
+interface ResolveModelFileLocationArgs {
+	url?: string
+	filePath?: string
+	modelsCachePath: string
+}
+
+/**
+ * Resolve a model file/url to an absolute path to either a file or directory.
+ * @param url - Optional URL to the model file. Location will be derived from it.
+ * @param filePath - Optional relative (to modelsCachePath) or absolute file path that short-circuits resolution.
+ * @param modelsCachePath - The path to the models cache directory.
+ * @returns The abs file path on the local filesystem.
+ * @throws If the model location could not be resolved.
+ */
+export function resolveModelFileLocation({ url, filePath, modelsCachePath }: ResolveModelFileLocationArgs) {
+
+	if (filePath) {
+		// immediately return if an absolute path is provided
+		if (path.isAbsolute(filePath)) {
+			return filePath
+		} else {
+			return path.join(modelsCachePath, filePath)
+		}
+	}
+
+	if (url) {
+		const parsedUrl = new URL(url)
+		let destinationPath = filePath
+		// support branches for huggingface URLs
+		if (parsedUrl.hostname === 'huggingface.co' && !destinationPath) {
+			const pathnameSegments = parsedUrl.pathname.split('/')
+			const repoOrg = pathnameSegments[1]
+			const repoName = pathnameSegments[2]
+			const branch = pathnameSegments[4] || 'main'
+			const trailingPath = pathnameSegments.slice(5).join('/')
+			destinationPath = path.join(modelsCachePath, parsedUrl.hostname, repoOrg, `${repoName}-${branch}`, trailingPath)
+		}
+		// otherwise, use the hostname and last path segment
+		if (!destinationPath) {
+			const fileName = parsedUrl.pathname.split('/').pop()
+			destinationPath = path.join(modelsCachePath, parsedUrl.hostname, fileName || '')
+		}
+		return destinationPath
+	}
+
+	throw new Error('Failed to resolve model location')
+}

package/src/lib/util.ts

@@ -0,0 +1,75 @@
+import { inspect } from 'node:util'
+import fs from 'node:fs'
+import path from 'node:path'
+
+export function elapsedMillis(since: bigint): number {
+	const now = process.hrtime.bigint()
+	return Number(now - BigInt(since)) / 1e6
+}
+
+export function omitEmptyValues<T extends Record<string, any>>(dict: T): T {
+	return Object.fromEntries(
+		Object.entries(dict).filter(([_, v]) => {
+			return v !== null && v !== undefined
+		}),
+	) as T
+}
+
+export function touchFileSync(filePath: string) {
+	fs.closeSync(fs.openSync(filePath, 'w'))
+}
+
+function isSubpath(parent: string, child: string): boolean {
+	// Normalize paths to resolve .. and . segments
+	const normalizedParent = path.normalize(parent)
+	const normalizedChild = path.normalize(child)
+
+	// Get relative path from parent to child
+	const relativePath = path.relative(normalizedParent, normalizedChild)
+
+	// Check if relative path:
+	// 1. Doesn't start with .. (which would mean going up directories)
+	// 2. Isn't an absolute path (which would start with / or C:\ etc)
+	return !relativePath.startsWith('..') && !path.isAbsolute(relativePath)
+}
+
+export function mergeAbortSignals(signals: Array<AbortSignal | undefined>): AbortSignal {
+	const controller = new AbortController()
+	const onAbort = () => {
+		controller.abort()
+	}
+	for (const signal of signals) {
+		if (signal) {
+			signal.addEventListener('abort', onAbort)
+		}
+	}
+	return controller.signal
+}
+
+export function getRandomNumber(min: number, max: number) {
+	min = Math.ceil(min)
+	max = Math.floor(max)
+	return Math.floor(Math.random() * (max - min)) + min
+}
+
+export function printActiveHandles() {
+	//@ts-ignore
+	const handles = process._getActiveHandles()
+	//@ts-ignore
+	const requests = process._getActiveRequests()
+
+	console.log('Active Handles:', inspect(handles, { depth: 1 }))
+	console.log('Active Requests:', inspect(requests, { depth: 1 }))
+}
+
+export function formatBytesPerSecond(speed: number) {
+	const units = ['B/s', 'KB/s', 'MB/s', 'GB/s', 'TB/s']
+	let unitIndex = 0
+
+	while (speed >= 1024 && unitIndex < units.length - 1) {
+		speed /= 1024
+		unitIndex++
+	}
+
+	return `${speed.toFixed(2)} ${units[unitIndex]}`
+}

package/src/lib/validateModelFile.ts

@@ -0,0 +1,71 @@
+
+
+import fs from 'node:fs'
+import { calculateFileChecksum } from '#package/lib/calculateFileChecksum.js'
+import { resolveModelFileLocation } from '#package/lib/resolveModelFileLocation.js'
+
+interface ValidatableModelConfig {
+	url?: string
+	location?: string
+	modelsCachePath: string
+	sha256?: string
+	md5?: string
+}
+
+export async function validateModelFile(config: ValidatableModelConfig): Promise<string | undefined> {
+	const fileLocation = resolveModelFileLocation({
+		url: config.url,
+		filePath: config.location,
+		modelsCachePath: config.modelsCachePath,
+	})
+	if (!fs.existsSync(fileLocation)) {
+		return `Model file missing at ${fileLocation}`
+	}
+	const hasChecksum = config.sha256 || config.md5
+	let validatedChecksum = false
+	const validateChecksum = async () => {
+		if (config.md5) {
+			const fileHash = await calculateFileChecksum(fileLocation, 'md5')
+			if (fileHash === config.md5) {
+				validatedChecksum = true
+				return true
+			}
+		} else if (config.sha256) {
+			const fileHash = await calculateFileChecksum(fileLocation, 'sha256')
+			if (fileHash === config.sha256) {
+				validatedChecksum = true
+				return true
+			}
+		}
+		return false
+	}
+	const ipullFile = fileLocation + '.ipull'
+	if (fs.existsSync(ipullFile)) {
+		// if we have a valid file at the download destination, we can remove the ipull file
+		if (hasChecksum) {
+			const isValid = await validateChecksum()
+			if (isValid) {
+				fs.unlinkSync(ipullFile)
+				validatedChecksum = true
+			}
+		}
+		if (!validatedChecksum) {
+			return `Model file with incomplete download`
+		}
+	}
+
+	if (!validatedChecksum && hasChecksum) {
+		if (config.sha256) {
+			const fileHash = await calculateFileChecksum(fileLocation, 'sha256')
+			if (fileHash !== config.sha256) {
+				return `File sha256 checksum mismatch: expected ${config.sha256} got ${fileHash} at ${fileLocation}`
+			}
+		} else if (config.md5) {
+			const fileHash = await calculateFileChecksum(fileLocation, 'md5')
+			if (fileHash !== config.md5) {
+				return `File md5 checksum mismatch: expected ${config.md5} got ${fileHash} at ${fileLocation}`
+			}
+		}
+	}
+	return undefined
+}