@infersec/conduit 1.4.1

package/README.md

@@ -0,0 +1,29 @@
+# @infersec/conduit
+
+Inference agent for connecting local LLMs to the Infersec cloud.
+
+## Usage
+
+```bash
+npx @infersec/conduit --source <source-id> --key <api-key>
+```
+
+Optional flags:
+
+```bash
+--api-url <url>   API base URL
+--engine <type>   Engine type
+--port <number>   Port to listen on
+--root <path>     Root directory
+```
+
+## Environment variables
+
+- AGENT_ENGINE_TYPE
+- API_KEY
+- API_URL
+- INFERENCE_SOURCE_ID
+- PORT
+- ROOT_DIRECTORY
+
+CLI flags override environment variables when both are provided.

package/dist/apiClient/index.d.ts

@@ -0,0 +1,12 @@
+import { InferenceAgentConfiguration, InferenceAgentHeartbeatPayload, InferenceAgentLLMMetricsPayload, InferenceAgentMachineReportPayload, ULID } from "@infersec/definitions";
+export interface APIClient {
+    getAgentConfiguration: () => Promise<InferenceAgentConfiguration>;
+    reportHeartbeat: (payload: InferenceAgentHeartbeatPayload) => Promise<void>;
+    reportMachineMetadata: (payload: InferenceAgentMachineReportPayload) => Promise<void>;
+    reportPromptMetrics: (payload: InferenceAgentLLMMetricsPayload) => Promise<void>;
+}
+export declare function createAPIClient({ apiURL, inferenceSourceID }: {
+    apiKey: string;
+    apiURL: string;
+    inferenceSourceID: ULID;
+}): APIClient;

package/dist/application.d.ts

@@ -0,0 +1,10 @@
+import { Application } from "express";
+import { Logger } from "@infersec/logger";
+import { APIClient } from "./apiClient/index.js";
+import { Configuration } from "./configuration.js";
+export declare function createApplication({ abortController, apiClient, configuration, logger }: {
+    abortController: AbortController;
+    apiClient: APIClient;
+    configuration: Configuration;
+    logger: Logger;
+}): Promise<Application>;

package/dist/cli.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/cli.js

@@ -0,0 +1,134 @@
+#!/usr/bin/env node
+import { fileURLToPath as __fileURLToPath } from 'url';
+import { dirname } from 'path';
+const __filename = __fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+
+import { parseArgs } from 'node:util';
+import 'node:crypto';
+import { a as asError, s as startInferenceAgent } from './start-CijRnu19.js';
+import 'argon2';
+import 'node:child_process';
+import 'node:stream';
+import 'os';
+import 'fs';
+import 'util';
+import 'assert';
+import 'events';
+import 'stream';
+import 'node:assert';
+import 'node:net';
+import 'node:http';
+import 'node:querystring';
+import 'node:events';
+import 'node:diagnostics_channel';
+import 'node:tls';
+import 'node:buffer';
+import 'node:zlib';
+import 'node:perf_hooks';
+import 'node:util/types';
+import 'node:worker_threads';
+import 'node:url';
+import 'node:async_hooks';
+import 'node:console';
+import 'node:fs/promises';
+import 'node:path';
+import 'node:dns';
+import 'node:sqlite';
+import 'path';
+import 'tty';
+import 'net';
+import 'zlib';
+import 'buffer';
+import 'string_decoder';
+import 'querystring';
+import 'url';
+import 'http';
+import 'crypto';
+import 'node:fs';
+import 'node:string_decoder';
+import 'node:stream/promises';
+import 'fs/promises';
+import 'stream/promises';
+import 'node:os';
+import 'node:process';
+import 'node:tty';
+import 'child_process';
+import 'node:timers/promises';
+import 'node:v8';
+import 'systeminformation';
+
+const USAGE = `Usage: npx @infersec/conduit --source <id> --key <key> [options]
+
+Options:
+  --api-url <url>     API base URL (or API_URL)
+  --engine <type>     Engine type (or AGENT_ENGINE_TYPE)
+  --key <value>       API key (or API_KEY)
+  --port <number>     Port to listen on (or PORT)
+  --root <path>       Root directory (or ROOT_DIRECTORY)
+  --source <id>       Inference source ID (or INFERENCE_SOURCE_ID)
+  -h, --help          Show this help message
+`;
+async function run() {
+    const { values } = parseArgs({
+        allowPositionals: false,
+        options: {
+            "api-url": {
+                type: "string"
+            },
+            engine: {
+                type: "string"
+            },
+            help: {
+                short: "h",
+                type: "boolean"
+            },
+            key: {
+                type: "string"
+            },
+            port: {
+                type: "string"
+            },
+            root: {
+                type: "string"
+            },
+            source: {
+                type: "string"
+            }
+        }
+    });
+    if (values.help) {
+        console.log(USAGE);
+        return;
+    }
+    const configurationOverrides = {};
+    if (values.engine) {
+        configurationOverrides.agentEngineType = values.engine;
+    }
+    if (values["api-url"]) {
+        configurationOverrides.apiURL = values["api-url"];
+    }
+    if (values.key) {
+        configurationOverrides.apiKey = values.key;
+    }
+    if (values.source) {
+        configurationOverrides.inferenceSourceID = values.source;
+    }
+    if (values.root) {
+        configurationOverrides.rootDirectory = values.root;
+    }
+    if (values.port) {
+        const port = Number.parseInt(values.port, 10);
+        if (Number.isNaN(port)) {
+            throw new Error(`Invalid port: ${values.port}`);
+        }
+        configurationOverrides.port = port;
+    }
+    await startInferenceAgent({
+        configurationOverrides
+    });
+}
+run().catch(err => {
+    console.error(asError(err));
+    process.exit(1);
+});

package/dist/configuration.d.ts

@@ -0,0 +1,20 @@
+import { LLMEngine, ULID } from "@infersec/definitions";
+export interface Configuration {
+    agentEngineType: LLMEngine;
+    apiKey: string;
+    apiURL: string;
+    inferenceSourceID: ULID;
+    port: number;
+    rootDirectory: string;
+}
+export interface ConfigurationOverrides {
+    agentEngineType?: string;
+    apiKey?: string;
+    apiURL?: string;
+    inferenceSourceID?: ULID;
+    port?: number;
+    rootDirectory?: string;
+}
+export declare function getConfiguration({ overrides }?: {
+    overrides?: ConfigurationOverrides;
+}): Configuration;

package/dist/index.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/index.js

@@ -0,0 +1,64 @@
+#!/usr/bin/env node
+import { fileURLToPath as __fileURLToPath } from 'url';
+import { dirname } from 'path';
+const __filename = __fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+
+import 'node:crypto';
+import { s as startInferenceAgent, a as asError } from './start-CijRnu19.js';
+import 'argon2';
+import 'node:child_process';
+import 'node:stream';
+import 'os';
+import 'fs';
+import 'util';
+import 'assert';
+import 'events';
+import 'stream';
+import 'node:assert';
+import 'node:net';
+import 'node:http';
+import 'node:querystring';
+import 'node:events';
+import 'node:diagnostics_channel';
+import 'node:util';
+import 'node:tls';
+import 'node:buffer';
+import 'node:zlib';
+import 'node:perf_hooks';
+import 'node:util/types';
+import 'node:worker_threads';
+import 'node:url';
+import 'node:async_hooks';
+import 'node:console';
+import 'node:fs/promises';
+import 'node:path';
+import 'node:dns';
+import 'node:sqlite';
+import 'path';
+import 'tty';
+import 'net';
+import 'zlib';
+import 'buffer';
+import 'string_decoder';
+import 'querystring';
+import 'url';
+import 'http';
+import 'crypto';
+import 'node:fs';
+import 'node:string_decoder';
+import 'node:stream/promises';
+import 'fs/promises';
+import 'stream/promises';
+import 'node:os';
+import 'node:process';
+import 'node:tty';
+import 'child_process';
+import 'node:timers/promises';
+import 'node:v8';
+import 'systeminformation';
+
+startInferenceAgent({}).catch(err => {
+    console.error(asError(err));
+    process.exit(1);
+});

package/dist/modelManagement/ModelManager.d.ts

@@ -0,0 +1,31 @@
+import { LLMEngine, LLMModel } from "@infersec/definitions";
+import { Logger } from "@infersec/logger";
+import EventEmitter from "eventemitter3";
+import { RequestInit, Response } from "undici";
+interface ModelManagerEvents {
+    engineError: (error: Error) => void;
+    engineReady: () => void;
+    engineTerminated: () => void;
+}
+export declare class ModelManager extends EventEmitter<ModelManagerEvents> {
+    readonly engine: LLMEngine;
+    readonly model: LLMModel;
+    readonly parallelism: number | null;
+    private uniqueName;
+    readonly contextLength: number | null;
+    protected readonly logger: Logger;
+    private engineProcess;
+    protected readonly modelsDirectory: string;
+    constructor({ contextLength, engine, logger, model, parallelism, root }: {
+        contextLength?: number | null;
+        engine: LLMEngine;
+        logger: Logger;
+        model: LLMModel;
+        parallelism?: number | null;
+        root: string;
+    });
+    fetchOpenAI(path: string, opts?: RequestInit): Promise<Response>;
+    prepare(): Promise<void>;
+    start(): Promise<void>;
+}
+export {};

package/dist/modelManagement/download.d.ts

@@ -0,0 +1,8 @@
+import { LLMModel } from "@infersec/definitions";
+export declare function downloadModelViaHuggingFace({ format, huggingFaceToken, modelSlug: rawModelSlug, progressFilePath, targetDirectory }: {
+    format: LLMModel["format"];
+    huggingFaceToken: string | null | undefined;
+    modelSlug: string;
+    progressFilePath: string;
+    targetDirectory: string;
+}): Promise<void>;

package/dist/modelManagement/healthCheck.d.ts

@@ -0,0 +1,2 @@
+import { LLMEngine } from "@infersec/definitions";
+export declare function checkEngineHealth(engine: LLMEngine): Promise<boolean>;

package/dist/modelManagement/llamacpp.d.ts

@@ -0,0 +1,6 @@
+import { ProcessManager } from "@infersec/utils";
+import { ModelManager } from "./ModelManager.js";
+export declare function startLlamacpp(this: ModelManager, { targetDirectory }: {
+    targetDirectory: string;
+}): Promise<ProcessManager>;
+export declare function checkLlamacppHealth(): Promise<boolean>;

package/dist/modelManagement/ollama.d.ts

@@ -0,0 +1,4 @@
+import { ProcessManager } from "@infersec/utils";
+import type { ModelManager } from "./ModelManager.js";
+export declare function loadCurrentModel(this: ModelManager): Promise<void>;
+export declare function startOllama(this: ModelManager): Promise<ProcessManager>;

package/dist/modelManagement/schemas.d.ts

@@ -0,0 +1,5 @@
+import { z } from "zod";
+export declare const ModelDownloadProgressSchema: z.ZodObject<{
+    completedFiles: z.ZodArray<z.ZodString>;
+}, z.core.$strip>;
+export type ModelDownloadProgress = z.infer<typeof ModelDownloadProgressSchema>;

package/dist/modelManagement/vllm.d.ts

@@ -0,0 +1,5 @@
+import { ProcessManager } from "@infersec/utils";
+import type { ModelManager } from "./ModelManager.js";
+export declare function startVLLM(this: ModelManager, { targetDirectory }: {
+    targetDirectory: string;
+}): Promise<ProcessManager>;

package/dist/sse/handler.d.ts

@@ -0,0 +1,9 @@
+import { type APIResponse, type ServerToClientAPIRequest } from "@infersec/definitions";
+import { Logger } from "@infersec/logger";
+import { Configuration } from "../configuration.js";
+export declare function handleSSERequests({ apiURL, configuration, logger, onRequest }: {
+    apiURL: string;
+    configuration: Configuration;
+    logger: Logger;
+    onRequest: (request: ServerToClientAPIRequest) => Promise<APIResponse>;
+}): Promise<void>;

package/dist/sse/requestProxy.d.ts

@@ -0,0 +1,9 @@
+import type { APIResponse, ServerToClientAPIRequest } from "@infersec/definitions";
+import type { Configuration } from "../configuration.js";
+/**
+ * Proxy server requests to the local inference HTTP server.
+ */
+export declare function proxyRequest({ configuration, request }: {
+    configuration: Configuration;
+    request: ServerToClientAPIRequest;
+}): Promise<APIResponse>;