@infersec/conduit 1.27.0 → 1.28.0

package/dist/apiClient/index.d.ts

@@ -1,6 +1,7 @@
 import { InferenceAgentConfiguration, InferenceAgentLLMMetricsPayload, InferenceAgentMachineReportPayload, ULID, type ConduitState } from "@infersec/definitions";
 import { Logger } from "@infersec/logger";
 export interface APIClient {
+    cycleInferenceSourceEngine: () => Promise<void>;
     getConduitConfiguration: () => Promise<InferenceAgentConfiguration>;
     reportConduitState: (state: ConduitState) => Promise<void>;
     reportMachineMetadata: (payload: InferenceAgentMachineReportPayload) => Promise<void>;

package/dist/cli.js

@@ -6,7 +6,7 @@ const __dirname = __pathDirname(__filename);
 
 import { parseArgs } from 'node:util';
 import 'node:crypto';
-import { a as asError, s as startInferenceAgent } from './start-Ca9mSCJR.js';
+import { a as asError, s as startInferenceAgent } from './start-DtAR3bT0.js';
 import 'argon2';
 import 'node:child_process';
 import 'node:stream';

package/dist/index.js

@@ -5,7 +5,7 @@ const __filename = __fileURLToPath(import.meta.url);
 const __dirname = __pathDirname(__filename);
 
 import 'node:crypto';
-import { s as startInferenceAgent, a as asError } from './start-Ca9mSCJR.js';
+import { s as startInferenceAgent, a as asError } from './start-DtAR3bT0.js';
 import 'argon2';
 import 'node:child_process';
 import 'node:stream';

package/dist/{start-Ca9mSCJR.js → start-DtAR3bT0.js}

@@ -14932,6 +14932,22 @@ const API_SERVICE_CONDUIT_API_REFERENCE = {
             }
         }
     },
+    "/conduit/api/v1/source/:sourceID/engine/cycle": {
+        POST: {
+            auth: {
+                type: "api-key"
+            },
+            parameters: {
+                sourceID: ULIDSchema
+            },
+            response: {
+                schema: object({
+                    acknowledged: literal(true)
+                }),
+                type: "rest"
+            }
+        }
+    },
     "/conduit/api/v1/source/:sourceID/requests/:requestID/chunk": {
         POST: {
             auth: {
@@ -15237,6 +15253,19 @@ const API_CLIENT_CONDUIT_GENERAL_REFERENCE = {
                 type: "rest"
             }
         }
+    },
+    "/conduit/engine/cycle": {
+        POST: {
+            auth: {
+                type: "none"
+            },
+            response: {
+                schema: object({
+                    acknowledged: literal(true)
+                }),
+                type: "rest"
+            }
+        }
     }
 };
 const API_CLIENT_CONDUIT_OPENAI_REFERENCE = {
@@ -98213,6 +98242,18 @@ function createAPIClient({ apiKey, apiURL, inferenceSourceID, logger }) {
             });
             return result.configuration;
         },
+        cycleInferenceSourceEngine: async () => {
+            await fetchByReference({
+                baseURL: apiURL,
+                fetch: fetchWithAPIKey,
+                method: "POST",
+                parameters: {
+                    sourceID: inferenceSourceID
+                },
+                reference: API_SERVICE_CONDUIT_API_REFERENCE,
+                route: "/conduit/api/v1/source/:sourceID/engine/cycle"
+            });
+        },
         reportMachineMetadata: async (payload) => {
             await fetchByReference({
                 baseURL: apiURL,
@@ -118921,7 +118962,7 @@ async function proxyOpenAIStreamingRoute({ body, configuration, logger, modelID,
 async function createApplication({ abortController, apiClient, configuration, logger }) {
     // Fetch configuration
     logger.info("Fetching conduit configuration");
-    const conduitConfiguration = await apiClient.getConduitConfiguration();
+    let conduitConfiguration = await apiClient.getConduitConfiguration();
     logger.info("Received configuration", {
         modelID: conduitConfiguration.targetModel.id
     });
@@ -118942,11 +118983,11 @@ async function createApplication({ abortController, apiClient, configuration, lo
             state: "initialising"
         }
     });
-    const modelFileName = getConduitModelFileName(conduitConfiguration);
-    const modelName = getConduitModelName(conduitConfiguration);
+    let modelFileName = getConduitModelFileName(conduitConfiguration);
+    let modelName = getConduitModelName(conduitConfiguration);
     const startup = Date.now();
     // Initialise model manager
-    const modelManager = new ModelManager({
+    let modelManager = new ModelManager({
         contextLength: conduitConfiguration.contextLength ?? null,
         engine: configuration.agentEngineType,
         logger,
@@ -118986,6 +119027,30 @@ async function createApplication({ abortController, apiClient, configuration, lo
         });
         conduitStateReportManager.reportStateChange();
     };
+    const attachLifecycleListeners = () => {
+        modelManager.on("engineError", err => {
+            logger.error("LLM engine error", {
+                error: err
+            });
+            stopRequestedByControl = false;
+            setErrorState({ error: err.message });
+        });
+        modelManager.on("engineTerminated", () => {
+            if (stopRequestedByControl) {
+                stopRequestedByControl = false;
+                setIdleState({ reason: "Remote shutdown requested" });
+                return;
+            }
+            conduitStateManager.setState({
+                state: "offline"
+            });
+            conduitStateReportManager.reportStateChange();
+        });
+        modelManager.on("engineReady", () => {
+            setOnlineState();
+        });
+    };
+    attachLifecycleListeners();
     let lastDownloadKey = "";
     const reportDownloadProgress = (update) => {
         const filePercent = update.file.total > 0 ? Math.floor((update.file.bytes / update.file.total) * 100) : 0;
@@ -119169,6 +119234,74 @@ async function createApplication({ abortController, apiClient, configuration, lo
                         };
                     }
                 }
+            },
+            "/conduit/engine/cycle": {
+                POST: async ({ body }) => {
+                    const sourceState = conduitStateManager.getState().state;
+                    if (sourceState !== "bootingEngine" &&
+                        sourceState !== "online" &&
+                        sourceState !== "idle") {
+                        return {
+                            status: 409,
+                            statusText: "Engine can only be cycled while booting, online, or idle"
+                        };
+                    }
+                    if (sourceState !== "idle" && !modelManager.canStop) {
+                        return {
+                            status: 409,
+                            statusText: `Engine cannot be cycled from current state: ${modelManager.state}`
+                        };
+                    }
+                    try {
+                        logger.info("Received remote engine cycle request");
+                        const sourceState = conduitStateManager.getState().state;
+                        if (sourceState !== "idle") {
+                            await stopEngine({
+                                reason: "Remote cycle requested"
+                            });
+                        }
+                        logger.info("Fetching new configuration");
+                        const newConduitConfiguration = await apiClient.getConduitConfiguration();
+                        logger.info("Received new configuration", {
+                            modelID: newConduitConfiguration.targetModel.id
+                        });
+                        logger.info("Updating configuration and model manager");
+                        conduitConfiguration = newConduitConfiguration;
+                        modelFileName = getConduitModelFileName(conduitConfiguration);
+                        modelName = getConduitModelName(conduitConfiguration);
+                        modelManager = new ModelManager({
+                            contextLength: conduitConfiguration.contextLength ?? null,
+                            engine: configuration.agentEngineType,
+                            logger,
+                            model: conduitConfiguration.targetModel,
+                            parallelism: conduitConfiguration.parallelism ?? null,
+                            root: configuration.rootDirectory
+                        });
+                        attachLifecycleListeners();
+                        if (sourceState === "idle") {
+                            logger.info("Restarting engine from idle");
+                            await startEngine();
+                        }
+                        else {
+                            logger.info("Restarting engine");
+                            await startEngine();
+                        }
+                        return {
+                            body: {
+                                acknowledged: true
+                            },
+                            status: 202
+                        };
+                    }
+                    catch (error) {
+                        const parsedError = asError(error);
+                        setErrorState({ error: parsedError.message });
+                        return {
+                            status: 500,
+                            statusText: parsedError.message
+                        };
+                    }
+                }
             }
         },
         logger,

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@infersec/conduit",
   "description": "End user conduit agent for connecting local LLMs to the cloud.",
-  "version": "1.27.0",
+  "version": "1.28.0",
   "bin": {
     "infersec-conduit": "./dist/cli.js"
   },