@use-solace/openllm 1.0.0

package/README.md

@@ -0,0 +1,185 @@
+# @use-solace/openllm
+
+OpenLLM model registry and API client with full TypeScript support for interacting with the OpenLLM inference engine.
+
+## Installation
+
+```bash
+npm install @use-solace/openllm@latest      # npm
+bun add @use-solace/openllm@latest          # bun (recommended)
+pnpm add @use-solace/openllm@latest         # pnpm
+deno add @use-solace/openllm@latest         # deno
+```
+
+## Usage
+
+### Model Registry
+
+Define your model registry:
+
+```ts
+import { ModelRegistry } from "@use-solace/openllm";
+
+export const models = ModelRegistry({
+  entries: {
+    "llama3.1-70b": {
+      inference: "ollama",
+      id: "llama3.1:70b",
+      context: 8192,
+      quant: "Q4_K_M",
+      capabilities: ["chat"],
+      latency: "slow",
+    },
+    "llama3.1-8b": {
+      inference: "llama",
+      id: "llama3.1:8b",
+      context: 8192,
+      quant: "Q4_K_M",
+      capabilities: ["chat", "completion"],
+      latency: "fast",
+    },
+  },
+});
+
+// Usage examples
+models.list();                   // returns all models
+models.get("llama3.1:70b");      // returns data for llama3.1:70b
+models.find({ capability: "chat", latency: "fast" });
+models.findOne({ capability: "chat" });
+models.has("llama3.1:70b");      // check if model exists
+models.add("new-model", {        // add a new model
+  inference: "ollama",
+  id: "new-model",
+  context: 4096,
+  capabilities: ["chat"],
+});
+models.remove("new-model");      // remove a model
+```
+
+### API Client
+
+Directly interact with the OpenLLM engine:
+
+```ts
+import { createOpenLLMClient } from "@use-solace/openllm";
+
+const client = createOpenLLMClient({ engine: 8080 });
+
+// Health check
+const health = await client.health();
+
+// List models
+const { models } = await client.listModels();
+
+// Register a model
+await client.registerModel({
+  id: "llama3.1:8b",
+  name: "llama3.1-8b",
+  inference: "ollama",
+  context: 8192,
+  quant: "Q4_K_M",
+  capabilities: ["chat"],
+  latency: "fast",
+});
+
+// Load a model
+await client.loadModel({ model_id: "llama3.1:8b" });
+
+// Run inference
+const result = await client.inference({
+  model_id: "llama3.1:8b",
+  prompt: "Hello, how are you?",
+  max_tokens: 512,
+  temperature: 0.7,
+});
+
+// Stream inference
+await client.inferenceStream(
+  {
+    model_id: "llama3.1:8b",
+    prompt: "Tell me a story",
+    max_tokens: 1024,
+  },
+  {
+    onToken: (token) => console.log(token.token),
+    onComplete: (response) => console.log("Done:", response),
+    onError: (error) => console.error("Error:", error),
+  },
+);
+
+// Unload a model
+await client.unloadModel("llama3.1:8b");
+```
+
+### Elysia Plugin
+
+```ts
+import { Elysia } from "elysia";
+import { openllm } from "@use-solace/openllm/elysia";
+import { models } from "./registry.ts";
+
+const app = new Elysia().use(
+  openllm({
+    prefix: "ollm",         // routes will be under /ollm/* instead of /openllm/*
+    modelrouter: true,       // enable model router
+    registry: models,        // pass the registry instance
+    engine: 4292,            // openllm-server port
+  }),
+);
+
+app.listen(3000);
+
+console.log("Server running on localhost:3000");
+```
+
+Available endpoints:
+- `GET /openllm/health` - Health check
+- `GET /openllm/models` - List registered models
+- `POST /openllm/models/register` - Register a new model
+- `POST /openllm/models/load` - Load a model
+- `POST /openllm/models/unload/:modelId` - Unload a model
+- `POST /openllm/inference` - Run inference
+- `POST /openllm/router/chat` - Chat with automatic model routing (if `modelrouter: true`)
+
+## Types
+
+The package provides full TypeScript type safety for all API interactions:
+
+```ts
+import type {
+  InferenceBackend,
+  ModelCapability,
+  LatencyProfile,
+  ModelRegistryEntry,
+  InferenceRequest,
+  InferenceResponse,
+  StreamToken,
+  // ... and more
+} from "@use-solace/openllm";
+```
+
+## Error Handling
+
+The package provides typed error classes:
+
+```ts
+import {
+  OpenLLMError,
+  ModelNotFoundError,
+  ModelNotLoadedError,
+  InferenceError,
+} from "@use-solace/openllm";
+
+try {
+  await client.inference({ model_id: "unknown", prompt: "test" });
+} catch (error) {
+  if (error instanceof ModelNotFoundError) {
+    console.error("Model not found:", error.message);
+    console.error("Status code:", error.statusCode);
+  }
+}
+```
+
+## License
+
+MIT

package/dist/client.d.ts

@@ -0,0 +1,21 @@
+import type { HealthResponse, InferenceRequest, InferenceResponse, LoadModelRequest, LoadModelResponse, ModelListResponse, OpenLLMConfig, RegisterModelRequest, RegisterModelResponse, StreamOptions, UnloadModelResponse } from "./types.js";
+export declare class OpenLLMClient {
+    private baseUrl;
+    private timeout;
+    constructor(config?: OpenLLMConfig);
+    private request;
+    private createError;
+    health(): Promise<HealthResponse>;
+    listModels(): Promise<ModelListResponse>;
+    registerModel(data: RegisterModelRequest): Promise<RegisterModelResponse>;
+    loadModel(data: LoadModelRequest): Promise<LoadModelResponse>;
+    unloadModel(modelId: string): Promise<UnloadModelResponse>;
+    inference(data: InferenceRequest): Promise<InferenceResponse>;
+    inferenceStream(data: InferenceRequest, options: StreamOptions): Promise<void>;
+    setBaseUrl(baseUrl: string): void;
+    getBaseUrl(): string;
+    setTimeout(timeout: number): void;
+    getTimeout(): number;
+}
+export declare function createOpenLLMClient(config?: OpenLLMConfig): OpenLLMClient;
+//# sourceMappingURL=client.d.ts.map

package/dist/client.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,cAAc,EACd,gBAAgB,EAChB,iBAAiB,EACjB,gBAAgB,EAChB,iBAAiB,EACjB,iBAAiB,EAGjB,aAAa,EAEb,oBAAoB,EACpB,qBAAqB,EACrB,aAAa,EAEb,mBAAmB,EACpB,MAAM,YAAY,CAAC;AAEpB,qBAAa,aAAa;IACxB,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,OAAO,CAAS;gBAEZ,MAAM,GAAE,aAAkB;YAKxB,OAAO;IAqCrB,OAAO,CAAC,WAAW;IA6Bb,MAAM,IAAI,OAAO,CAAC,cAAc,CAAC;IAIjC,UAAU,IAAI,OAAO,CAAC,iBAAiB,CAAC;IAIxC,aAAa,CACjB,IAAI,EAAE,oBAAoB,GACzB,OAAO,CAAC,qBAAqB,CAAC;IAO3B,SAAS,CAAC,IAAI,EAAE,gBAAgB,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAO7D,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,mBAAmB,CAAC;IAS1D,SAAS,CAAC,IAAI,EAAE,gBAAgB,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAO7D,eAAe,CACnB,IAAI,EAAE,gBAAgB,EACtB,OAAO,EAAE,aAAa,GACrB,OAAO,CAAC,IAAI,CAAC;IAwFhB,UAAU,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI;IAIjC,UAAU,IAAI,MAAM;IAIpB,UAAU,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI;IAIjC,UAAU,IAAI,MAAM;CAGrB;AAED,wBAAgB,mBAAmB,CAAC,MAAM,CAAC,EAAE,aAAa,GAAG,aAAa,CAEzE"}

package/dist/client.js

@@ -0,0 +1,189 @@
+export class OpenLLMClient {
+    baseUrl;
+    timeout;
+    constructor(config = {}) {
+        this.baseUrl = `http://localhost:${config.engine ?? 8080}`;
+        this.timeout = config.timeout ?? 30000;
+    }
+    async request(endpoint, options = {}) {
+        const controller = new AbortController();
+        const timeoutId = setTimeout(() => controller.abort(), this.timeout);
+        try {
+            const response = await fetch(`${this.baseUrl}${endpoint}`, {
+                ...options,
+                signal: controller.signal,
+                headers: {
+                    "Content-Type": "application/json",
+                    ...options.headers,
+                },
+            });
+            clearTimeout(timeoutId);
+            if (!response.ok) {
+                const errorText = await response.text();
+                throw this.createError(response.status, errorText);
+            }
+            return (await response.json());
+        }
+        catch (error) {
+            clearTimeout(timeoutId);
+            if (error instanceof Error) {
+                if (error.name === "AbortError") {
+                    throw new Error(`Request timeout after ${this.timeout}ms`);
+                }
+                throw error;
+            }
+            throw error;
+        }
+    }
+    createError(status, message) {
+        if (status === 404) {
+            const match = message.match(/Model '([^']+)'/);
+            if (match) {
+                const error = new Error(message);
+                error.name = "ModelNotFoundError";
+                error.code = "MODEL_NOT_FOUND";
+                error.statusCode = status;
+                return error;
+            }
+        }
+        if (status === 412) {
+            const match = message.match(/Model '([^']+)'/);
+            if (match) {
+                const error = new Error(message);
+                error.name = "ModelNotLoadedError";
+                error.code = "MODEL_NOT_LOADED";
+                error.statusCode = status;
+                return error;
+            }
+        }
+        const error = new Error(message);
+        error.name = "OpenLLMError";
+        error.code = "API_ERROR";
+        error.statusCode = status;
+        return error;
+    }
+    async health() {
+        return this.request("/health");
+    }
+    async listModels() {
+        return this.request("/v1/models");
+    }
+    async registerModel(data) {
+        return this.request("/v1/models/register", {
+            method: "POST",
+            body: JSON.stringify(data),
+        });
+    }
+    async loadModel(data) {
+        return this.request("/v1/models/load", {
+            method: "POST",
+            body: JSON.stringify(data),
+        });
+    }
+    async unloadModel(modelId) {
+        return this.request(`/v1/models/unload/${modelId}`, {
+            method: "POST",
+        });
+    }
+    async inference(data) {
+        return this.request("/v1/inference", {
+            method: "POST",
+            body: JSON.stringify(data),
+        });
+    }
+    async inferenceStream(data, options) {
+        const controller = new AbortController();
+        const timeoutId = setTimeout(() => controller.abort(), this.timeout);
+        try {
+            const response = await fetch(`${this.baseUrl}/v1/inference/stream`, {
+                method: "POST",
+                headers: {
+                    "Content-Type": "application/json",
+                },
+                body: JSON.stringify(data),
+                signal: controller.signal,
+            });
+            clearTimeout(timeoutId);
+            if (!response.ok) {
+                const errorText = await response.text();
+                const error = this.createError(response.status, errorText);
+                options.onError?.(error);
+                throw error;
+            }
+            const reader = response.body?.getReader();
+            if (!reader) {
+                const error = new Error("No response body");
+                options.onError?.(error);
+                throw error;
+            }
+            const decoder = new TextDecoder();
+            let buffer = "";
+            let accumulatedText = "";
+            let tokenCount = 0;
+            const modelId = data.model_id;
+            while (true) {
+                const { done, value } = await reader.read();
+                if (done)
+                    break;
+                buffer += decoder.decode(value, { stream: true });
+                const lines = buffer.split("\n");
+                buffer = lines.pop() ?? "";
+                for (const line of lines) {
+                    if (line.trim().startsWith("event: token")) {
+                        continue;
+                    }
+                    if (line.trim().startsWith("data: ")) {
+                        const data = line.trim().slice(6);
+                        if (data) {
+                            try {
+                                const token = JSON.parse(data);
+                                accumulatedText += token.token;
+                                tokenCount++;
+                                options.onToken(token);
+                                if (token.complete) {
+                                    options.onComplete?.({
+                                        model_id: modelId,
+                                        text: accumulatedText,
+                                        tokens_generated: tokenCount,
+                                        finish_reason: "stop",
+                                    });
+                                }
+                            }
+                            catch (e) {
+                                console.error("Failed to parse SSE data:", data, e);
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        catch (error) {
+            clearTimeout(timeoutId);
+            if (error instanceof Error) {
+                if (error.name === "AbortError") {
+                    const timeoutError = new Error(`Stream timeout after ${this.timeout}ms`);
+                    options.onError?.(timeoutError);
+                    throw timeoutError;
+                }
+                options.onError?.(error);
+                throw error;
+            }
+        }
+    }
+    setBaseUrl(baseUrl) {
+        this.baseUrl = baseUrl;
+    }
+    getBaseUrl() {
+        return this.baseUrl;
+    }
+    setTimeout(timeout) {
+        this.timeout = timeout;
+    }
+    getTimeout() {
+        return this.timeout;
+    }
+}
+export function createOpenLLMClient(config) {
+    return new OpenLLMClient(config);
+}
+//# sourceMappingURL=client.js.map

package/dist/client.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"client.js","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAkBA,MAAM,OAAO,aAAa;IAChB,OAAO,CAAS;IAChB,OAAO,CAAS;IAExB,YAAY,SAAwB,EAAE;QACpC,IAAI,CAAC,OAAO,GAAG,oBAAoB,MAAM,CAAC,MAAM,IAAI,IAAI,EAAE,CAAC;QAC3D,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,KAAK,CAAC;IACzC,CAAC;IAEO,KAAK,CAAC,OAAO,CACnB,QAAgB,EAChB,UAAuB,EAAE;QAEzB,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAErE,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,OAAO,GAAG,QAAQ,EAAE,EAAE;gBACzD,GAAG,OAAO;gBACV,MAAM,EAAE,UAAU,CAAC,MAAM;gBACzB,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;oBAClC,GAAG,OAAO,CAAC,OAAO;iBACnB;aACF,CAAC,CAAC;YAEH,YAAY,CAAC,SAAS,CAAC,CAAC;YAExB,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACxC,MAAM,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;YACrD,CAAC;YAED,OAAO,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAM,CAAC;QACtC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,YAAY,CAAC,SAAS,CAAC,CAAC;YACxB,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;gBAC3B,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;oBAChC,MAAM,IAAI,KAAK,CAAC,yBAAyB,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC;gBAC7D,CAAC;gBACD,MAAM,KAAK,CAAC;YACd,CAAC;YACD,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAEO,WAAW,CAAC,MAAc,EAAE,OAAe;QACjD,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;YACnB,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;YAC/C,IAAI,KAAK,EAAE,CAAC;gBACV,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,OAAO,CAAuB,CAAC;gBACvD,KAAK,CAAC,IAAI,GAAG,oBAAoB,CAAC;gBAClC,KAAK,CAAC,IAAI,GAAG,iBAAiB,CAAC;gBAC/B,KAAK,CAAC,UAAU,GAAG,MAAM,CAAC;gBAC1B,OAAO,KAAK,CAAC;YACf,CAAC;QACH,CAAC;QACD,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;YACnB,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;YAC/C,IAAI,KAAK,EAAE,CAAC;gBACV,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,OAAO,CAAwB,CAAC;gBACxD,KAAK,CAAC,IAAI,GAAG,qBAAqB,CAAC;gBACnC,KAAK,CAAC,IAAI,GAAG,kBAAkB,CAAC;gBAChC,KAAK,CAAC,UAAU,GAAG,MAAM,CAAC;gBAC1B,OAAO,KAAK,CAAC;YACf,CAAC;QACH,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,OAAO,CAAiB,CAAC;QACjD,KAAK,CAAC,IAAI,GAAG,cAAc,CAAC;QAC5B,KAAK,CAAC,IAAI,GAAG,WAAW,CAAC;QACzB,KAAK,CAAC,UAAU,GAAG,MAAM,CAAC;QAC1B,OAAO,KAAK,CAAC;IACf,CAAC;IAED,KAAK,CAAC,MAAM;QACV,OAAO,IAAI,CAAC,OAAO,CAAiB,SAAS,CAAC,CAAC;IACjD,CAAC;IAED,KAAK,CAAC,UAAU;QACd,OAAO,IAAI,CAAC,OAAO,CAAoB,YAAY,CAAC,CAAC;IACvD,CAAC;IAED,KAAK,CAAC,aAAa,CACjB,IAA0B;QAE1B,OAAO,IAAI,CAAC,OAAO,CAAwB,qBAAqB,EAAE;YAChE,MAAM,EAAE,MAAM;YACd,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;SAC3B,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,IAAsB;QACpC,OAAO,IAAI,CAAC,OAAO,CAAoB,iBAAiB,EAAE;YACxD,MAAM,EAAE,MAAM;YACd,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;SAC3B,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,OAAe;QAC/B,OAAO,IAAI,CAAC,OAAO,CACjB,qBAAqB,OAAO,EAAE,EAC9B;YACE,MAAM,EAAE,MAAM;SACf,CACF,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,IAAsB;QACpC,OAAO,IAAI,CAAC,OAAO,CAAoB,eAAe,EAAE;YACtD,MAAM,EAAE,MAAM;YACd,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;SAC3B,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,eAAe,CACnB,IAAsB,EACtB,OAAsB;QAEtB,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAErE,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,OAAO,sBAAsB,EAAE;gBAClE,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;iBACnC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;gBAC1B,MAAM,EAAE,UAAU,CAAC,MAAM;aAC1B,CAAC,CAAC;YAEH,YAAY,CAAC,SAAS,CAAC,CAAC;YAExB,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACxC,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;gBAC3D,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC;gBACzB,MAAM,KAAK,CAAC;YACd,CAAC;YAED,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,EAAE,SAAS,EAAE,CAAC;YAC1C,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;gBAC5C,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC;gBACzB,MAAM,KAAK,CAAC;YACd,CAAC;YAED,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;YAClC,IAAI,MAAM,GAAG,EAAE,CAAC;YAChB,IAAI,eAAe,GAAG,EAAE,CAAC;YACzB,IAAI,UAAU,GAAG,CAAC,CAAC;YACnB,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC;YAE9B,OAAO,IAAI,EAAE,CAAC;gBACZ,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;gBAC5C,IAAI,IAAI;oBAAE,MAAM;gBAEhB,MAAM,IAAI,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;gBAClD,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACjC,MAAM,GAAG,KAAK,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC;gBAE3B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;oBACzB,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,cAAc,CAAC,EAAE,CAAC;wBAC3C,SAAS;oBACX,CAAC;oBACD,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;wBACrC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;wBAClC,IAAI,IAAI,EAAE,CAAC;4BACT,IAAI,CAAC;gCACH,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAgB,CAAC;gCAC9C,eAAe,IAAI,KAAK,CAAC,KAAK,CAAC;gCAC/B,UAAU,EAAE,CAAC;gCACb,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;gCAEvB,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;oCACnB,OAAO,CAAC,UAAU,EAAE,CAAC;wCACnB,QAAQ,EAAE,OAAO;wCACjB,IAAI,EAAE,eAAe;wCACrB,gBAAgB,EAAE,UAAU;wCAC5B,aAAa,EAAE,MAAM;qCACtB,CAAC,CAAC;gCACL,CAAC;4BACH,CAAC;4BAAC,OAAO,CAAC,EAAE,CAAC;gCACX,OAAO,CAAC,KAAK,CAAC,2BAA2B,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;4BACtD,CAAC;wBACH,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,YAAY,CAAC,SAAS,CAAC,CAAC;YACxB,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;gBAC3B,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;oBAChC,MAAM,YAAY,GAAG,IAAI,KAAK,CAC5B,wBAAwB,IAAI,CAAC,OAAO,IAAI,CACzC,CAAC;oBACF,OAAO,CAAC,OAAO,EAAE,CAAC,YAAY,CAAC,CAAC;oBAChC,MAAM,YAAY,CAAC;gBACrB,CAAC;gBACD,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC;gBACzB,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;IACH,CAAC;IAED,UAAU,CAAC,OAAe;QACxB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IACzB,CAAC;IAED,UAAU;QACR,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED,UAAU,CAAC,OAAe;QACxB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IACzB,CAAC;IAED,UAAU;QACR,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;CACF;AAED,MAAM,UAAU,mBAAmB,CAAC,MAAsB;IACxD,OAAO,IAAI,aAAa,CAAC,MAAM,CAAC,CAAC;AACnC,CAAC"}

package/dist/elysia.d.ts

@@ -0,0 +1,31 @@
+import type { Elysia } from "elysia";
+import type { APIConfig } from "./types.js";
+export declare function openllm(config?: APIConfig): (app: Elysia) => Elysia<"", {
+    decorator: {};
+    store: {};
+    derive: {};
+    resolve: {};
+}, {
+    typebox: {};
+    error: {};
+}, {
+    schema: {};
+    standaloneSchema: {};
+    macro: {};
+    macroFn: {};
+    parser: {};
+    response: {};
+}, {}, {
+    derive: {};
+    resolve: {};
+    schema: {};
+    standaloneSchema: {};
+    response: {};
+}, {
+    derive: {};
+    resolve: {};
+    schema: {};
+    standaloneSchema: {};
+    response: {};
+}>;
+//# sourceMappingURL=elysia.d.ts.map

package/dist/elysia.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"elysia.d.ts","sourceRoot":"","sources":["../src/elysia.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AACrC,OAAO,KAAK,EACV,SAAS,EAMV,MAAM,YAAY,CAAC;AAGpB,wBAAgB,OAAO,CAAC,MAAM,GAAE,SAAc,SACvB,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsF5B"}

package/dist/elysia.js

@@ -0,0 +1,66 @@
+import { createOpenLLMClient } from "./client.js";
+export function openllm(config = {}) {
+    const plugin = (app) => {
+        const enginePort = config.engine ?? 8080;
+        const prefix = config.prefix ?? "/openllm";
+        const enableRouter = config.modelrouter ?? false;
+        const registry = config.registry;
+        const client = createOpenLLMClient({ engine: enginePort });
+        app.get(`${prefix}/health`, async () => {
+            return await client.health();
+        });
+        app.get(`${prefix}/models`, async () => {
+            const result = await client.listModels();
+            return result.models;
+        });
+        app.post(`${prefix}/models/register`, async ({ body }) => {
+            const req = body;
+            return await client.registerModel(req);
+        });
+        app.post(`${prefix}/models/load`, async ({ body }) => {
+            const req = body;
+            return await client.loadModel(req);
+        });
+        app.post(`${prefix}/models/unload/:modelId`, async ({ params }) => {
+            const modelId = params.modelId;
+            return await client.unloadModel(modelId);
+        });
+        app.post(`${prefix}/inference`, async ({ body }) => {
+            const req = body;
+            return await client.inference(req);
+        });
+        if (enableRouter && registry) {
+            app.post(`${prefix}/router/chat`, async ({ body }) => {
+                const request = body;
+                const options = request.options ?? {};
+                let model;
+                if (options.model) {
+                    model = registry.get(options.model);
+                    if (!model) {
+                        throw new Error(`Model '${options.model}' not found in registry`);
+                    }
+                }
+                else {
+                    model = registry.findOne({
+                        capability: "chat",
+                        latency: options.latency,
+                        inference: options.inference,
+                        minContext: options.minContext,
+                    });
+                    if (!model) {
+                        throw new Error("No suitable model found for the given constraints");
+                    }
+                }
+                return await client.inference({
+                    model_id: model.id,
+                    prompt: request.prompt,
+                    max_tokens: options.max_tokens,
+                    temperature: options.temperature,
+                });
+            });
+        }
+        return app;
+    };
+    return plugin;
+}
+//# sourceMappingURL=elysia.js.map

package/dist/elysia.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"elysia.js","sourceRoot":"","sources":["../src/elysia.ts"],"names":[],"mappings":"AASA,OAAO,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAElD,MAAM,UAAU,OAAO,CAAC,SAAoB,EAAE;IAC5C,MAAM,MAAM,GAAG,CAAC,GAAW,EAAE,EAAE;QAC7B,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,IAAI,IAAI,CAAC;QACzC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,UAAU,CAAC;QAC3C,MAAM,YAAY,GAAG,MAAM,CAAC,WAAW,IAAI,KAAK,CAAC;QACjD,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAyC,CAAC;QAElE,MAAM,MAAM,GAAG,mBAAmB,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;QAE3D,GAAG,CAAC,GAAG,CAAC,GAAG,MAAM,SAAS,EAAE,KAAK,IAAI,EAAE;YACrC,OAAO,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC;QAC/B,CAAC,CAAC,CAAC;QAEH,GAAG,CAAC,GAAG,CAAC,GAAG,MAAM,SAAS,EAAE,KAAK,IAAI,EAAE;YACrC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,UAAU,EAAE,CAAC;YACzC,OAAO,MAAM,CAAC,MAAM,CAAC;QACvB,CAAC,CAAC,CAAC;QAEH,GAAG,CAAC,IAAI,CAAC,GAAG,MAAM,kBAAkB,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE;YACvD,MAAM,GAAG,GAAG,IAA4B,CAAC;YACzC,OAAO,MAAM,MAAM,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;QACzC,CAAC,CAAC,CAAC;QAEH,GAAG,CAAC,IAAI,CAAC,GAAG,MAAM,cAAc,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE;YACnD,MAAM,GAAG,GAAG,IAA4B,CAAC;YACzC,OAAO,MAAM,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QACrC,CAAC,CAAC,CAAC;QAEH,GAAG,CAAC,IAAI,CAAC,GAAG,MAAM,yBAAyB,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE;YAChE,MAAM,OAAO,GAAG,MAAM,CAAC,OAAiB,CAAC;YACzC,OAAO,MAAM,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;QAC3C,CAAC,CAAC,CAAC;QAEH,GAAG,CAAC,IAAI,CAAC,GAAG,MAAM,YAAY,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE;YACjD,MAAM,GAAG,GAAG,IAAwB,CAAC;YACrC,OAAO,MAAM,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QACrC,CAAC,CAAC,CAAC;QAEH,IAAI,YAAY,IAAI,QAAQ,EAAE,CAAC;YAC7B,GAAG,CAAC,IAAI,CAAC,GAAG,MAAM,cAAc,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE;gBACnD,MAAM,OAAO,GAAG,IAUf,CAAC;gBACF,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;gBAEtC,IAAI,KAAqC,CAAC;gBAC1C,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;oBAClB,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;oBACpC,IAAI,CAAC,KAAK,EAAE,CAAC;wBACX,MAAM,IAAI,KAAK,CAAC,UAAU,OAAO,CAAC,KAAK,yBAAyB,CAAC,CAAC;oBACpE,CAAC;gBACH,CAAC;qBAAM,CAAC;oBACN,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC;wBACvB,UAAU,EAAE,MAAM;wBAClB,OAAO,EAAE,OAAO,CAAC,OAAc;wBAC/B,SAAS,EAAE,OAAO,CAAC,SAA6B;wBAChD,UAAU,EAAE,OAAO,CAAC,UAAU;qBAC/B,CAAC,CAAC;oBAEH,IAAI,CAAC,KAAK,EAAE,CAAC;wBACX,MAAM,IAAI,KAAK,CACb,mDAAmD,CACpD,CAAC;oBACJ,CAAC;gBACH,CAAC;gBAED,OAAO,MAAM,MAAM,CAAC,SAAS,CAAC;oBAC5B,QAAQ,EAAE,KAAK,CAAC,EAAE;oBAClB,MAAM,EAAE,OAAO,CAAC,MAAM;oBACtB,UAAU,EAAE,OAAO,CAAC,UAAU;oBAC9B,WAAW,EAAE,OAAO,CAAC,WAAW;iBACjC,CAAC,CAAC;YACL,CAAC,CAAC,CAAC;QACL,CAAC;QAED,OAAO,GAAG,CAAC;IACb,CAAC,CAAC;IAEF,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,11 @@
+export type { InferenceBackend, ModelCapability, LatencyProfile, ModelRegistryEntry, RegistryEntryInput, ModelRegistryConfig, HealthResponse, ModelListResponse, RegisterModelResponse, LoadModelRequest, LoadModelResponse, UnloadModelResponse, InferenceRequest, InferenceResponse, StreamToken, StreamCallback, StreamCompleteCallback, StreamErrorCallback, StreamOptions, OpenLLMConfig, FindModelOptions, APIConfig, OpenLLMError, ModelNotFoundError, ModelNotLoadedError, InferenceError, ModelRegistryInstance, ModelRegistryImpl, } from "./types.js";
+export { ModelRegistry, } from "./registry.js";
+export type { ModelRegistry as ModelRegistryType } from "./registry.js";
+export { OpenLLMClient, createOpenLLMClient, } from "./client.js";
+export { openllm } from "./elysia.js";
+export declare const openllmAPI: {
+    start: (config?: import("./types.js").APIConfig) => {
+        start: (apiPort: number) => void;
+    };
+};
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,YAAY,EACV,gBAAgB,EAChB,eAAe,EACf,cAAc,EACd,kBAAkB,EAClB,kBAAkB,EAClB,mBAAmB,EACnB,cAAc,EACd,iBAAiB,EACjB,qBAAqB,EACrB,gBAAgB,EAChB,iBAAiB,EACjB,mBAAmB,EACnB,gBAAgB,EAChB,iBAAiB,EACjB,WAAW,EACX,cAAc,EACd,sBAAsB,EACtB,mBAAmB,EACnB,aAAa,EACb,aAAa,EACb,gBAAgB,EAChB,SAAS,EACT,YAAY,EACZ,kBAAkB,EAClB,mBAAmB,EACnB,cAAc,EACd,qBAAqB,EACrB,iBAAiB,GAClB,MAAM,YAAY,CAAC;AAEpB,OAAO,EACL,aAAa,GACd,MAAM,eAAe,CAAC;AAEvB,YAAY,EAAE,aAAa,IAAI,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAExE,OAAO,EACL,aAAa,EACb,mBAAmB,GACpB,MAAM,aAAa,CAAC;AAErB,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAEtC,eAAO,MAAM,UAAU;qBACL,OAAO,YAAY,EAAE,SAAS;yBAGzB,MAAM;;CAM5B,CAAC"}

package/dist/index.js

@@ -0,0 +1,15 @@
+export { ModelRegistry, } from "./registry.js";
+export { OpenLLMClient, createOpenLLMClient, } from "./client.js";
+export { openllm } from "./elysia.js";
+export const openllmAPI = {
+    start: (config = {}) => {
+        const port = config.engine ?? 4292;
+        return {
+            start: (apiPort) => {
+                console.log(`Starting OpenLLM API on port ${apiPort}, connected to engine on port ${port}`);
+                console.log("Note: This is a client library. To start the actual server, use the Elysia plugin.");
+            },
+        };
+    },
+};
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AA+BA,OAAO,EACL,aAAa,GACd,MAAM,eAAe,CAAC;AAIvB,OAAO,EACL,aAAa,EACb,mBAAmB,GACpB,MAAM,aAAa,CAAC;AAErB,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAEtC,MAAM,CAAC,MAAM,UAAU,GAAG;IACxB,KAAK,EAAE,CAAC,SAAyC,EAAE,EAAE,EAAE;QACrD,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,IAAI,IAAI,CAAC;QACnC,OAAO;YACL,KAAK,EAAE,CAAC,OAAe,EAAE,EAAE;gBACzB,OAAO,CAAC,GAAG,CAAC,gCAAgC,OAAO,iCAAiC,IAAI,EAAE,CAAC,CAAC;gBAC5F,OAAO,CAAC,GAAG,CAAC,oFAAoF,CAAC,CAAC;YACpG,CAAC;SACF,CAAC;IACJ,CAAC;CACF,CAAC"}

package/dist/registry.d.ts

@@ -0,0 +1,20 @@
+import type { FindModelOptions, ModelRegistryConfig, ModelRegistryEntry, RegistryEntryInput } from "./types.js";
+export declare class ModelRegistryImpl {
+    private entries;
+    constructor(config: ModelRegistryConfig);
+    private registerEntry;
+    list(): ModelRegistryEntry[];
+    get(id: string): ModelRegistryEntry | undefined;
+    find(options?: FindModelOptions): ModelRegistryEntry[];
+    findOne(options?: FindModelOptions): ModelRegistryEntry | undefined;
+    has(id: string): boolean;
+    count(): number;
+    add(id: string, entry: RegistryEntryInput): ModelRegistryEntry;
+    update(id: string, updates: Partial<RegistryEntryInput>): ModelRegistryEntry;
+    remove(id: string): boolean;
+    clear(): void;
+    toObject(): Record<string, ModelRegistryEntry>;
+    fromObject(obj: Record<string, ModelRegistryEntry>): void;
+}
+export declare function ModelRegistry(config: ModelRegistryConfig): ModelRegistryImpl;
+//# sourceMappingURL=registry.d.ts.map

package/dist/registry.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"registry.d.ts","sourceRoot":"","sources":["../src/registry.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,gBAAgB,EAChB,mBAAmB,EACnB,kBAAkB,EAClB,kBAAkB,EACnB,MAAM,YAAY,CAAC;AAEpB,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,OAAO,CAA8C;gBAEjD,MAAM,EAAE,mBAAmB;IAOvC,OAAO,CAAC,aAAa;IAiBrB,IAAI,IAAI,kBAAkB,EAAE;IAI5B,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,kBAAkB,GAAG,SAAS;IAI/C,IAAI,CAAC,OAAO,GAAE,gBAAqB,GAAG,kBAAkB,EAAE;IAsB1D,OAAO,CAAC,OAAO,GAAE,gBAAqB,GAAG,kBAAkB,GAAG,SAAS;IAIvE,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO;IAIxB,KAAK,IAAI,MAAM;IAIf,GAAG,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,kBAAkB,GAAG,kBAAkB;IAO9D,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,kBAAkB,CAAC,GAAG,kBAAkB;IAe5E,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO;IAI3B,KAAK,IAAI,IAAI;IAIb,QAAQ,IAAI,MAAM,CAAC,MAAM,EAAE,kBAAkB,CAAC;IAI9C,UAAU,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,kBAAkB,CAAC,GAAG,IAAI;CAM1D;AAED,wBAAgB,aAAa,CAC3B,MAAM,EAAE,mBAAmB,GAC1B,iBAAiB,CAEnB"}