@infersec/conduit 1.58.1 → 1.60.0

package/dist/ConduitConnection.d.ts

@@ -0,0 +1,24 @@
+import type { LLMEngine, ULID } from "@infersec/definitions";
+import type { Logger } from "@infersec/logger";
+export interface ConduitConnectionOptions {
+    apiKey: string;
+    apiURL: string;
+    engine: LLMEngine;
+    enginePort: number;
+    logger: Logger;
+    port: number;
+    rootDirectory?: string;
+    sourceID: ULID;
+}
+export declare class ConduitConnection {
+    private abortController;
+    private configuration;
+    private logger;
+    private server;
+    private shutdownFn;
+    readonly port: number;
+    readonly enginePort: number;
+    constructor(options: ConduitConnectionOptions);
+    start(): Promise<void>;
+    stop(): Promise<void>;
+}

package/dist/benchmark/evaluation/bfclEvaluator.d.ts

@@ -0,0 +1,25 @@
+import type { BFCLAnswer, BFCLProblem } from "../types.js";
+export type { BFCLAnswer };
+export declare function loadBFCLProblems(): Promise<{
+    problems: BFCLProblem[];
+    answers: Map<string, BFCLAnswer>;
+}>;
+export declare function evaluateBFCL(options: {
+    apiKey: string;
+    answers: Map<string, BFCLAnswer>;
+    completionEndpoint: string;
+    concurrency: number;
+    endpointID: string;
+    onProgress?: (completed: number, total: number) => void;
+    problems: BFCLProblem[];
+}): Promise<{
+    accuracy: number;
+    completionTokens: number;
+    failedRequests: number;
+    latencies: number[];
+    promptTokens: number;
+    timeToFirstTokenMs: number[];
+    total: number;
+    totalLatencyMs: number;
+    totalRequests: number;
+}>;

package/dist/benchmark/evaluation/codeEvaluator.d.ts

@@ -0,0 +1,20 @@
+import type { BenchmarkStats, HumanEvalProblem } from "../types.js";
+interface ProblemResult {
+    completionTokens: number;
+    latencyMs: number;
+    passed: boolean;
+    promptTokens: number;
+    success: boolean;
+    timeToFirstTokenMs: number;
+}
+export declare function evaluateCodeCompletion(options: {
+    apiKey: string;
+    completionEndpoint: string;
+    concurrency: number;
+    endpointID: string;
+    onProgress?: (completed: number, total: number) => void;
+    problems: HumanEvalProblem[];
+}): Promise<BenchmarkStats>;
+export declare function buildStats(results: ProblemResult[], problemCount: number): BenchmarkStats;
+export declare function stripMarkdownFences(code: string): string;
+export {};

package/dist/benchmark/evaluation/codingEvaluator.d.ts

@@ -0,0 +1,3 @@
+import type { HumanEvalProblem } from "../types.js";
+export { evaluateCodeCompletion } from "./codeEvaluator.js";
+export declare function loadHumanEvalPlusProblems(): Promise<HumanEvalProblem[]>;

package/dist/benchmark/evaluation/completion.d.ts

@@ -0,0 +1,18 @@
+export declare function stripThinkTags(text: string): string;
+export interface CompletionResponse {
+    completionTokens: number;
+    content: string;
+    latencyMs: number;
+    promptTokens: number;
+    timeToFirstTokenMs: number;
+}
+export declare function getCompletion(options: {
+    apiKey: string;
+    endpoint: string;
+    endpointID: string;
+    maxTokens: number;
+    prompt: string;
+    stripThink?: boolean;
+    systemPrompt?: string;
+    temperature?: number;
+}): Promise<CompletionResponse>;

package/dist/benchmark/evaluation/mbppEvaluator.d.ts

@@ -0,0 +1,10 @@
+import type { BenchmarkStats, MBPPPlusProblem } from "../types.js";
+export declare function loadMBPPPlusProblems(): Promise<MBPPPlusProblem[]>;
+export declare function evaluateMBPPPlus(options: {
+    apiKey: string;
+    completionEndpoint: string;
+    concurrency: number;
+    endpointID: string;
+    onProgress?: (completed: number, total: number) => void;
+    problems: MBPPPlusProblem[];
+}): Promise<BenchmarkStats>;

package/dist/benchmark/output/writer.d.ts

@@ -0,0 +1,15 @@
+import type { HardwareConfig, ModelMetadata, RunResult } from "../types.js";
+export declare function slugToFilename(slug: string, quantization: string | null): string;
+export declare function writeOutput(options: {
+    hardware: HardwareConfig;
+    metadata?: ModelMetadata;
+    modelMetadata: {
+        format: string;
+        parameterCount: string;
+        quantization: string | null;
+    };
+    outputDir: string;
+    recommended?: string[] | false;
+    run: RunResult;
+    slug: string;
+}): Promise<void>;

package/dist/benchmark/provision.d.ts

@@ -0,0 +1,17 @@
+interface ProvisionResult {
+    endpointID: string;
+    modelID: string;
+    sourceIDs: string[];
+}
+export declare function provisionResources(options: {
+    accountId: string;
+    apiUrl: string;
+    apiKey: string;
+    contextLength: number;
+    engine: "llama.cpp" | "vllm";
+    format: string;
+    parallelism: number;
+    quantization: string | null;
+    slug: string;
+}): Promise<ProvisionResult>;
+export {};

package/dist/benchmark/runner.d.ts

@@ -0,0 +1,7 @@
+export declare function runBenchmark(options: {
+    apiUrl: string;
+    apiKey: string;
+    accountId: string;
+    configPath: string;
+    outputDir?: string;
+}): Promise<void>;

package/dist/benchmark/sampling/gpuMemory.d.ts

@@ -0,0 +1,2 @@
+export declare function readVRAMUsedMB(): Promise<number>;
+export declare function readVRAMTotalMB(): Promise<number>;

package/dist/benchmark/sampling/memorySampler.d.ts

@@ -0,0 +1,7 @@
+import type { MemorySampleResult } from "../types.js";
+export declare function createMemorySampler(options?: {
+    intervalMs?: number;
+}): {
+    start(): Promise<void>;
+    stop(): MemorySampleResult;
+};

package/dist/benchmark/sampling/processMemory.d.ts

@@ -0,0 +1,2 @@
+export declare function readRAMTotalMB(): Promise<number>;
+export declare function readRAMUsedMB(): Promise<number>;

package/dist/benchmark/teardown.d.ts

@@ -0,0 +1,7 @@
+export declare function teardownResources(options: {
+    apiKey: string;
+    apiUrl: string;
+    endpointID: string;
+    modelID: string;
+    sourceIDs: string[];
+}): Promise<void>;

package/dist/benchmark/types.d.ts

@@ -0,0 +1,167 @@
+export interface BenchmarkConfig {
+    hardware: HardwareConfig;
+    outputDir: string;
+    tests: TestEntry[];
+}
+export interface HardwareConfig {
+    device: string;
+    os: string;
+    ramTotalGB: number;
+    vramGB: number;
+}
+export interface TestEntry {
+    benchmarks?: {
+        bfcl?: boolean;
+        humanevalplus?: boolean;
+        mbppplus?: boolean;
+    };
+    concurrency: number;
+    contextLength: number;
+    engine: "llama.cpp" | "vllm";
+    format: string;
+    generate: boolean;
+    parallelism: number;
+    parameterCount: string;
+    quantization: string | null;
+    recommended?: string[] | false;
+    slug: string;
+    source?: ModelSource;
+    vllmImage?: string;
+}
+export interface ModelSource {
+    type: "huggingface";
+    url: string;
+}
+export interface RunResult {
+    concurrency: number;
+    contextLength: number;
+    date: string;
+    engine: string;
+    hardware: HardwareConfig;
+    maxTokens: number;
+    memory: MemoryResult;
+    parallelism: number;
+    performance: PerformanceResult;
+    quality: QualityResult;
+}
+export interface PerformanceResult {
+    avgLatencyMs: number;
+    completionTokens: number;
+    failedRequests: number;
+    p50LatencyMs: number;
+    p95LatencyMs: number;
+    p99LatencyMs: number;
+    promptTokens: number;
+    successRate: number;
+    timeToFirstTokenMs: number;
+    tokensPerSecond: number;
+    totalRequests: number;
+}
+export interface MemoryResult {
+    baselineRamUsedMB: number;
+    baselineVramUsedMB: number;
+    modelRamMB: number;
+    modelVramMB: number;
+    peakRamUsedMB: number;
+    peakVramUsedMB: number;
+    ramTotalMB: number;
+    vramTotalMB: number;
+}
+export interface QualityResult {
+    bfcl?: {
+        accuracy: number;
+        problems: number;
+    };
+    humanevalplus?: {
+        passed: number;
+        passRate: number;
+        problems: number;
+    };
+    mbppplus?: {
+        passed: number;
+        passRate: number;
+        problems: number;
+    };
+}
+export interface ModelMetadata {
+    author: string;
+    baseModel?: string;
+    contextLength?: number;
+    description?: string;
+    license?: string;
+    modelType?: string;
+    source: ModelSource;
+}
+export interface ModelOutput {
+    metadata?: ModelMetadata;
+    model: {
+        format: string;
+        parameterCount: string;
+        quantization: string | null;
+    };
+    recommended?: string[] | false;
+    runs: RunResult[];
+    slug: string;
+}
+export interface MemorySample {
+    ramTotalMB: number;
+    ramUsedMB: number;
+    timestamp: number;
+    vramTotalMB: number;
+    vramUsedMB: number;
+}
+export interface MemorySampleResult {
+    baseline: MemorySample;
+    peak: MemorySample;
+    samples: MemorySample[];
+}
+export interface BenchmarkStats {
+    avgLatencyMs: number;
+    completionTokens: number;
+    failedRequests: number;
+    latencies: number[];
+    passed: number;
+    p50LatencyMs: number;
+    p95LatencyMs: number;
+    p99LatencyMs: number;
+    problems: number;
+    promptTokens: number;
+    successRate: number;
+    timeToFirstTokenMs: number[];
+    totalLatencyMs: number;
+    totalRequests: number;
+}
+export interface HumanEvalProblem {
+    entryPoint: string;
+    id: string;
+    prompt: string;
+    test: string;
+}
+export interface MBPPPlusProblem {
+    assertions: string;
+    entryPoint: string;
+    id: string;
+    prompt: string;
+    solution: string;
+}
+export interface BFCLFunction {
+    description: string;
+    name: string;
+    parameters: {
+        properties: Record<string, Record<string, unknown>>;
+        required?: string[];
+        type: string;
+    };
+}
+export interface BFCLProblem {
+    function: BFCLFunction[];
+    id: string;
+    question: Array<Array<{
+        content: string;
+        role: string;
+    }>>;
+}
+export interface BFCLAnswer {
+    ground_truth: Array<Record<string, Record<string, unknown>>>;
+    id: string;
+}

package/dist/benchmark/waitForReady.d.ts

@@ -0,0 +1,5 @@
+export declare function waitForAllReady(options: {
+    apiKey: string;
+    apiUrl: string;
+    sourceIDs: string[];
+}): Promise<void>;