@infersec/conduit 1.23.0 → 1.24.0

package/dist/cli.js

@@ -6,7 +6,7 @@ const __dirname = __pathDirname(__filename);
 
 import { parseArgs } from 'node:util';
 import 'node:crypto';
-import { a as asError, s as startInferenceAgent } from './start-MoDI8K51.js';
+import { a as asError, s as startInferenceAgent } from './start-CpPE5_K5.js';
 import 'argon2';
 import 'node:child_process';
 import 'node:stream';

package/dist/index.js

@@ -5,7 +5,7 @@ const __filename = __fileURLToPath(import.meta.url);
 const __dirname = __pathDirname(__filename);
 
 import 'node:crypto';
-import { s as startInferenceAgent, a as asError } from './start-MoDI8K51.js';
+import { s as startInferenceAgent, a as asError } from './start-CpPE5_K5.js';
 import 'argon2';
 import 'node:child_process';
 import 'node:stream';

package/dist/{start-MoDI8K51.js → start-CpPE5_K5.js}

@@ -199,6 +199,22 @@ function ulid$2(seedTime, prng) {
     return encodeTime(seed, TIME_LEN) + encodeRandom(RANDOM_LEN, currentPRNG);
 }
 
+/**
+ * Calculates the effective context length per slot, accounting for
+ * parallelism when using llama.cpp. For llama.cpp, the total context
+ * window is divided across parallel slots; for other engines, the
+ * full context length is used.
+ */
+function getEffectiveContextLength({ contextLength, engine, parallelism }) {
+    if (contextLength === null || contextLength <= 0) {
+        return null;
+    }
+    if (engine === "llama.cpp" && parallelism !== null && parallelism > 0) {
+        return contextLength / parallelism;
+    }
+    return contextLength;
+}
+
 function asError(error) {
     if (error instanceof Error) {
         return error;
@@ -15103,7 +15119,11 @@ const ModelSchema = object({
     id: string$1(),
     object: literal("model"),
     created: number$1(),
-    owned_by: string$1()
+    owned_by: string$1(),
+    limit: object({
+        context: number$1().nullable()
+    })
+        .optional()
 });
 const ModelsPageSchema = object({
     object: literal("list"),
@@ -117962,15 +117982,15 @@ function monitorEngineResponseStream({ agentEngineType, body, contextLength, eng
                 const parsed = JSON.parse(payload);
                 if (parsed.usage) {
                     const usageChunk = parsed.usage;
+                    const effectiveContext = getEffectiveContextLength({
+                        contextLength,
+                        engine,
+                        parallelism
+                    });
                     if (usageChunk.context_usage === undefined &&
                         usageChunk.prompt_tokens !== undefined &&
-                        contextLength !== null &&
-                        contextLength > 0) {
-                        let totalContextSize = contextLength;
-                        if (engine === "llama.cpp" && parallelism !== null && parallelism > 0) {
-                            totalContextSize = contextLength / parallelism;
-                        }
-                        usageChunk.context_usage = usageChunk.prompt_tokens / totalContextSize;
+                        effectiveContext !== null) {
+                        usageChunk.context_usage = usageChunk.prompt_tokens / effectiveContext;
                         modifiedLines.push("data: " + JSON.stringify(parsed));
                         continue;
                     }
@@ -118002,11 +118022,15 @@ function monitorEngineResponseStream({ agentEngineType, body, contextLength, eng
                     const promptTokens = parsed.usage?.prompt_tokens ?? null;
                     const totalTokens = parsed.usage?.total_tokens ?? null;
                     let contextUsage = parsed.usage?.context_usage ?? null;
+                    const effectiveContextForUsage = getEffectiveContextLength({
+                        contextLength,
+                        engine,
+                        parallelism
+                    });
                     if (contextUsage === null &&
                         promptTokens !== null &&
-                        contextLength !== null &&
-                        contextLength > 0) {
-                        contextUsage = promptTokens / contextLength;
+                        effectiveContextForUsage !== null) {
+                        contextUsage = promptTokens / effectiveContextForUsage;
                     }
                     usage = {
                         completionTokens,
@@ -118440,6 +118464,11 @@ async function createApplication({ abortController, apiClient, configuration, lo
             },
             "/v1/models": {
                 GET: async () => {
+                    const effectiveContextLength = getEffectiveContextLength({
+                        contextLength: modelManager.contextLength,
+                        engine: configuration.agentEngineType,
+                        parallelism: modelManager.parallelism
+                    });
                     return {
                         body: {
                             object: "list",
@@ -118448,7 +118477,10 @@ async function createApplication({ abortController, apiClient, configuration, lo
                                     id: conduitConfiguration.targetModel.id,
                                     object: "model",
                                     created: startup / 1000,
-                                    owned_by: "infersec"
+                                    owned_by: "infersec",
+                                    limit: {
+                                        context: effectiveContextLength
+                                    }
                                 }
                             ]
                         },

package/dist/utils/__tests__/engineMetrics.test.d.ts

@@ -0,0 +1 @@
+export {};

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@infersec/conduit",
   "description": "End user conduit agent for connecting local LLMs to the cloud.",
-  "version": "1.23.0",
+  "version": "1.24.0",
   "bin": {
     "infersec-conduit": "./dist/cli.js"
   },
@@ -23,10 +23,11 @@
     "format": "prettier --write .",
     "prepublishOnly": "npm run build",
     "start": "npm run build && node ./dist/index.js",
-    "test": "npm run test:types && npm run test:lint && npm run test:format",
+    "test": "npm run test:types && npm run test:lint && npm run test:format && npm run test:unit",
     "test:format": "prettier --check .",
     "test:lint": "eslint source/**/*.ts",
-    "test:types": "tsc -p tsconfig.json --noEmit"
+    "test:types": "tsc -p tsconfig.json --noEmit",
+    "test:unit": "vitest run"
   },
   "prettier": "@infersec/prettier",
   "publishConfig": {
@@ -46,8 +47,10 @@
     "@rollup/plugin-typescript": "^12.1.4",
     "@types/express": "^4.17.23",
     "@types/supertest": "^6.0.3",
+    "@vitest/coverage-v8": "^3.0.5",
     "rollup": "^4.46.2",
-    "tslib": "^2.8.1"
+    "tslib": "^2.8.1",
+    "vitest": "^3.0.5"
   },
   "dependencies": {
     "@huggingface/hub": "^2.5.2",