@infersec/conduit 1.6.3 → 1.7.0

package/dist/cli.js

@@ -6,7 +6,7 @@ const __dirname = __pathDirname(__filename);
 
 import { parseArgs } from 'node:util';
 import 'node:crypto';
-import { a as asError, s as startInferenceAgent } from './start-CQayYNYZ.js';
+import { a as asError, s as startInferenceAgent } from './start-BF-CnHXg.js';
 import 'argon2';
 import 'node:child_process';
 import 'node:stream';

package/dist/index.js

@@ -5,7 +5,7 @@ const __filename = __fileURLToPath(import.meta.url);
 const __dirname = __pathDirname(__filename);
 
 import 'node:crypto';
-import { s as startInferenceAgent, a as asError } from './start-CQayYNYZ.js';
+import { s as startInferenceAgent, a as asError } from './start-BF-CnHXg.js';
 import 'argon2';
 import 'node:child_process';
 import 'node:stream';

package/dist/modelManagement/ModelManager.d.ts

@@ -2,6 +2,7 @@ import { LLMEngine, LLMModel } from "@infersec/definitions";
 import { Logger } from "@infersec/logger";
 import EventEmitter from "eventemitter3";
 import { RequestInit, Response } from "undici";
+import { type ModelDownloadProgressUpdate } from "./download.js";
 interface ModelManagerEvents {
     engineError: (error: Error) => void;
     engineReady: () => void;
@@ -25,7 +26,9 @@ export declare class ModelManager extends EventEmitter<ModelManagerEvents> {
         root: string;
     });
     fetchOpenAI(path: string, opts?: RequestInit): Promise<Response>;
-    prepare(): Promise<void>;
+    prepare({ onDownloadProgress }?: {
+        onDownloadProgress?: (update: ModelDownloadProgressUpdate) => void;
+    }): Promise<void>;
     start(): Promise<void>;
 }
 export {};

package/dist/modelManagement/download.d.ts

@@ -1,8 +1,20 @@
 import { LLMModel } from "@infersec/definitions";
-export declare function downloadModelViaHuggingFace({ format, huggingFaceToken, modelSlug: rawModelSlug, progressFilePath, targetDirectory }: {
+export type ModelDownloadProgressUpdate = {
+    file: {
+        bytes: number;
+        name: string;
+        total: number;
+    };
+    total: {
+        completed: number;
+        total: number;
+    };
+};
+export declare function downloadModelViaHuggingFace({ format, huggingFaceToken, modelSlug: rawModelSlug, onProgress, progressFilePath, targetDirectory }: {
     format: LLMModel["format"];
     huggingFaceToken: string | null | undefined;
     modelSlug: string;
+    onProgress?: (update: ModelDownloadProgressUpdate) => void;
     progressFilePath: string;
     targetDirectory: string;
 }): Promise<void>;

package/dist/sse/handler.d.ts

@@ -1,9 +1,11 @@
 import { type APIResponse, type ServerToClientAPIRequest } from "@infersec/definitions";
 import { Logger } from "@infersec/logger";
 import { Configuration } from "../configuration.js";
-export declare function handleSSERequests({ apiURL, configuration, logger, onRequest }: {
+export declare function handleSSERequests({ apiURL, configuration, logger, onRequest, onRequestEnd, onRequestStart }: {
     apiURL: string;
     configuration: Configuration;
     logger: Logger;
     onRequest: (request: ServerToClientAPIRequest) => Promise<APIResponse>;
+    onRequestEnd?: (request: ServerToClientAPIRequest) => Promise<void> | void;
+    onRequestStart?: (request: ServerToClientAPIRequest) => Promise<void> | void;
 }): Promise<void>;

package/dist/{start-CQayYNYZ.js → start-BF-CnHXg.js}

@@ -96476,7 +96476,7 @@ const ModelDownloadProgressSchema = object({
 const DOWNLOAD_PROGRESS_TIMEOUT = 60000;
 const DOWNLOAD_RETRY_ATTEMPTS_FULL = 3;
 const DOWNLOAD_RETRY_ATTEMPTS_RANGE = 10;
-async function downloadModelViaHuggingFace({ format, huggingFaceToken, modelSlug: rawModelSlug, progressFilePath, targetDirectory }) {
+async function downloadModelViaHuggingFace({ format, huggingFaceToken, modelSlug: rawModelSlug, onProgress, progressFilePath, targetDirectory }) {
     // Sanitise model ID
     const [modelSlugWithRevision, variant = null] = rawModelSlug.split(":");
     const { modelSlug, revision } = parseModelRevision(modelSlugWithRevision);
@@ -96494,6 +96494,7 @@ async function downloadModelViaHuggingFace({ format, huggingFaceToken, modelSlug
     }
     console.log("\n\n\nFORMAT", { format, variant });
     const accessToken = huggingFaceToken ?? undefined;
+    const allFiles = [];
     // Get all files in model
     for await (const file of listFiles({
         accessToken,
@@ -96510,7 +96511,31 @@ async function downloadModelViaHuggingFace({ format, huggingFaceToken, modelSlug
             console.log("Skipping due variant not being needed:", file.path, file.size);
             continue;
         }
-        if (progress.completedFiles.includes(file.path)) {
+        allFiles.push({
+            path: file.path,
+            size: file.size
+        });
+    }
+    const completedSet = new Set(progress.completedFiles);
+    const totalFiles = allFiles.length;
+    let completedFiles = allFiles.filter(file => completedSet.has(file.path)).length;
+    const reportProgress = (update) => {
+        if (!onProgress)
+            return;
+        onProgress({
+            file: {
+                bytes: update.bytes,
+                name: update.name,
+                total: update.total
+            },
+            total: {
+                completed: completedFiles,
+                total: totalFiles
+            }
+        });
+    };
+    for (const file of allFiles) {
+        if (completedSet.has(file.path)) {
             console.log("Skipping due to already having completed file:", file.path, file.size);
             continue;
         }
@@ -96522,14 +96547,23 @@ async function downloadModelViaHuggingFace({ format, huggingFaceToken, modelSlug
             revision
         });
         const { partialPath } = getDownloadPaths({ filePath: file.path, targetDirectory });
+        const fileTotalSize = rangeInfo.totalSize ?? file.size;
+        reportProgress({ bytes: 0, name: file.path, total: fileTotalSize });
         const shouldAttemptRange = rangeInfo.supportsRanges || existsSync(partialPath);
         if (shouldAttemptRange) {
             try {
                 await downloadFileWithRange({
                     accessToken,
                     filePath: file.path,
-                    fileSize: rangeInfo.totalSize ?? file.size,
+                    fileSize: fileTotalSize,
                     modelSlug,
+                    onProgress: progressUpdate => {
+                        reportProgress({
+                            bytes: progressUpdate.bytes,
+                            name: file.path,
+                            total: progressUpdate.total
+                        });
+                    },
                     revision,
                     targetDirectory
                 });
@@ -96540,8 +96574,15 @@ async function downloadModelViaHuggingFace({ format, huggingFaceToken, modelSlug
                     await downloadFileFull({
                         accessToken,
                         filePath: file.path,
-                        fileSize: file.size,
+                        fileSize: fileTotalSize,
                         modelSlug,
+                        onProgress: progressUpdate => {
+                            reportProgress({
+                                bytes: progressUpdate.bytes,
+                                name: file.path,
+                                total: progressUpdate.total
+                            });
+                        },
                         targetDirectory
                     });
                 }
@@ -96557,13 +96598,22 @@ async function downloadModelViaHuggingFace({ format, huggingFaceToken, modelSlug
             await downloadFileFull({
                 accessToken,
                 filePath: file.path,
-                fileSize: file.size,
+                fileSize: fileTotalSize,
                 modelSlug,
+                onProgress: progressUpdate => {
+                    reportProgress({
+                        bytes: progressUpdate.bytes,
+                        name: file.path,
+                        total: progressUpdate.total
+                    });
+                },
                 targetDirectory
             });
         }
         // Update progress
         progress.completedFiles.push(file.path);
+        completedFiles += 1;
+        reportProgress({ bytes: fileTotalSize, name: file.path, total: fileTotalSize });
         await writeFile(progressFilePath, JSON.stringify(progress, undefined, 4));
     }
 }
@@ -96749,7 +96799,7 @@ async function getRangeInfo({ accessToken, filePath, fileSize, modelSlug, revisi
         };
     }
 }
-async function downloadFileFull({ accessToken, filePath, fileSize, modelSlug, targetDirectory }) {
+async function downloadFileFull({ accessToken, filePath, fileSize, modelSlug, onProgress, targetDirectory }) {
     const { finalPath, partialPath } = getDownloadPaths({ filePath, targetDirectory });
     let lastError = null;
     for (let attempt = 1; attempt <= DOWNLOAD_RETRY_ATTEMPTS_FULL; attempt++) {
@@ -96781,6 +96831,7 @@ async function downloadFileFull({ accessToken, filePath, fileSize, modelSlug, ta
             };
             input.on("data", resetProgressTimeout);
             resetProgressTimeout();
+            onProgress?.({ bytes: 0, total: fileSize });
             meter.progress.on("progress", (totalBytes) => {
                 lastProgressBytes = totalBytes;
                 const percentComplete = ((totalBytes / fileSize) * 100).toFixed(1);
@@ -96788,6 +96839,7 @@ async function downloadFileFull({ accessToken, filePath, fileSize, modelSlug, ta
                     console.log(` => ${percentComplete}% (${totalBytes} / ${fileSize})`);
                 }
                 lastPercentage = percentComplete;
+                onProgress?.({ bytes: totalBytes, total: fileSize });
             });
             try {
                 await pipeline(input, meter, output);
@@ -96809,7 +96861,7 @@ async function downloadFileFull({ accessToken, filePath, fileSize, modelSlug, ta
     const errorMessage = `Failed downloading ${filePath} (${fileSize} bytes) after ${DOWNLOAD_RETRY_ATTEMPTS_FULL} attempts. Last error: ${lastError?.message ?? "Unknown error"}`;
     throw new Error(errorMessage);
 }
-async function downloadFileWithRange({ accessToken, filePath, fileSize, modelSlug, revision, targetDirectory }) {
+async function downloadFileWithRange({ accessToken, filePath, fileSize, modelSlug, onProgress, revision, targetDirectory }) {
     const { finalPath, partialPath } = getDownloadPaths({ filePath, targetDirectory });
     let lastError = null;
     for (let attempt = 1; attempt <= DOWNLOAD_RETRY_ATTEMPTS_RANGE; attempt++) {
@@ -96826,6 +96878,7 @@ async function downloadFileWithRange({ accessToken, filePath, fileSize, modelSlu
                 return;
             }
         }
+        onProgress?.({ bytes: startOffset, total: fileSize });
         try {
             const resumeLabel = startOffset > 0 ? `resuming at ${startOffset}` : "starting";
             console.log("Downloading:", filePath, fileSize, `(attempt ${attempt}, ${resumeLabel})`);
@@ -96891,6 +96944,7 @@ async function downloadFileWithRange({ accessToken, filePath, fileSize, modelSlu
                     console.log(` => ${percentComplete}% (${lastProgressBytes} / ${fileSize})`);
                 }
                 lastPercentage = percentComplete;
+                onProgress?.({ bytes: lastProgressBytes, total: fileSize });
             });
             try {
                 await pipeline(input, meter, output);
@@ -104848,15 +104902,16 @@ async function findQuantizedModelTarget({ model, path }) {
     if (modelFiles.length <= 0) {
         throw new Error(`No models found for format: ${model.format}`);
     }
+    modelFiles.sort((left, right) => left.localeCompare(right, undefined, { sensitivity: "base" }));
     if (!variant) {
         // Just return the first
         return modelFiles[0];
     }
-    const target = modelFiles.find(fileName => fileName.toLowerCase().includes(variant.toLowerCase()));
-    if (!target) {
+    const matches = modelFiles.filter(fileName => fileName.toLowerCase().includes(variant.toLowerCase()));
+    if (matches.length === 0) {
         throw new Error(`No model found for format and variant: ${model.format} / ${variant}`);
     }
-    return target;
+    return matches[0];
 }
 async function startLlamacpp({ targetDirectory }) {
     const target = await findQuantizedModelTarget({ model: this.model, path: targetDirectory });
@@ -104952,7 +105007,7 @@ class ModelManager extends EventEmitter {
             }
         }
     }
-    async prepare() {
+    async prepare({ onDownloadProgress } = {}) {
         this.logger.info("Preparing LLM engine", {
             agentEngineType: this.engine,
             modelID: this.model.id
@@ -104967,6 +105022,7 @@ class ModelManager extends EventEmitter {
                     format: this.model.format,
                     huggingFaceToken: this.model.source.modelSecret,
                     modelSlug: this.model.source.slug,
+                    onProgress: onDownloadProgress,
                     progressFilePath: join(this.modelsDirectory, `${this.uniqueName}.progress.json`),
                     targetDirectory: join(this.modelsDirectory, this.uniqueName)
                 });
@@ -105034,7 +105090,7 @@ class ModelManager extends EventEmitter {
     }
 }
 
-async function handleSSERequests({ apiURL, configuration, logger, onRequest }) {
+async function handleSSERequests({ apiURL, configuration, logger, onRequest, onRequestEnd, onRequestStart }) {
     const streamURL = `${apiURL}/conduit/api/v1/source/${configuration.inferenceSourceID}/requests/stream`;
     await connectSSE(streamURL, {
         headers: {
@@ -105054,6 +105110,8 @@ async function handleSSERequests({ apiURL, configuration, logger, onRequest }) {
                 apiURL,
                 configuration,
                 logger,
+                onRequestEnd,
+                onRequestStart,
                 onRequest,
                 request: payload
             }).catch(error => {
@@ -105065,8 +105123,9 @@ async function handleSSERequests({ apiURL, configuration, logger, onRequest }) {
         }
     });
 }
-async function handleRequest({ apiURL, configuration, logger, onRequest, request }) {
+async function handleRequest({ apiURL, configuration, logger, onRequest, onRequestEnd, onRequestStart, request }) {
     try {
+        await onRequestStart?.(request);
         const response = await onRequest(request);
         await streamResponse({
             apiURL,
@@ -105102,6 +105161,9 @@ async function handleRequest({ apiURL, configuration, logger, onRequest, request
             requestID: request.requestID
         });
     }
+    finally {
+        await onRequestEnd?.(request);
+    }
 }
 async function streamResponse({ apiURL, configuration, logger, requestID, response }) {
     let sequence = 0;
@@ -114715,6 +114777,9 @@ async function createApplication({ abortController, apiClient, configuration, lo
             state: "initialising"
         }
     });
+    const modelFileName = getConduitModelFileName(conduitConfiguration);
+    const modelName = getConduitModelName(conduitConfiguration);
+    const idleReason = "Awaiting requests";
     const startup = Date.now();
     // Initialise model manager
     const modelManager = new ModelManager({
@@ -114743,20 +114808,44 @@ async function createApplication({ abortController, apiClient, configuration, lo
     });
     modelManager.on("engineReady", () => {
         conduitStateManager.setState({
-            modelName: getConduitModelName(conduitConfiguration),
-            state: "online"
+            reason: idleReason,
+            state: "idle"
         });
     });
     conduitStateManager.setState({
-        modelFileName: getConduitModelFileName(conduitConfiguration),
-        modelName: getConduitModelName(conduitConfiguration),
+        modelFileName,
+        modelName,
         state: "downloadingModelFiles",
         totalProgress: {
             file: 0,
             total: 0
         }
     });
-    await modelManager.prepare();
+    let lastDownloadKey = "";
+    const reportDownloadProgress = (update) => {
+        const filePercent = update.file.total > 0 ? Math.floor((update.file.bytes / update.file.total) * 100) : 0;
+        const downloadKey = `${update.total.completed}/${update.total.total}:${update.file.name}:${filePercent}`;
+        if (downloadKey === lastDownloadKey) {
+            return;
+        }
+        lastDownloadKey = downloadKey;
+        conduitStateManager.setState({
+            fileProgress: {
+                bytes: Math.max(0, Math.floor(update.file.bytes)),
+                total: Math.max(0, Math.floor(update.file.total))
+            },
+            modelFileName: update.file.name,
+            modelName,
+            state: "downloadingModelFiles",
+            totalProgress: {
+                file: Math.max(0, Math.floor(update.total.completed)),
+                total: Math.max(0, Math.floor(update.total.total))
+            }
+        });
+    };
+    await modelManager.prepare({
+        onDownloadProgress: reportDownloadProgress
+    });
     conduitStateManager.setState({
         state: "bootingEngine"
     });
@@ -114836,15 +114925,40 @@ async function createApplication({ abortController, apiClient, configuration, lo
             });
         });
     }, CONDUIT_STATE_INTERVAL_MS);
+    let activeRequests = 0;
+    const setIdleState = () => {
+        conduitStateManager.setState({
+            reason: idleReason,
+            state: "idle"
+        });
+    };
+    const setOnlineState = () => {
+        conduitStateManager.setState({
+            modelName,
+            state: "online"
+        });
+    };
     handleSSERequests({
         apiURL: configuration.apiURL,
         configuration,
         logger,
+        onRequestEnd: () => {
+            activeRequests = Math.max(0, activeRequests - 1);
+            if (activeRequests === 0) {
+                setIdleState();
+            }
+        },
         onRequest: async (request) => {
             return proxyRequest({
                 configuration,
                 request
             });
+        },
+        onRequestStart: () => {
+            activeRequests += 1;
+            if (activeRequests === 1) {
+                setOnlineState();
+            }
         }
     }).catch(error => {
         logger.error("SSE handler failed", {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@infersec/conduit",
   "description": "End user conduit agent for connecting local LLMs to the cloud.",
-  "version": "1.6.3",
+  "version": "1.7.0",
   "bin": {
     "infersec-conduit": "./dist/cli.js"
   },