@use-solace/openllm 1.0.0

package/dist/registry.js

@@ -0,0 +1,99 @@
+export class ModelRegistryImpl {
+    entries = new Map();
+    constructor(config) {
+        const entries = config.entries ?? {};
+        for (const [id, entry] of Object.entries(entries)) {
+            this.registerEntry(id, entry);
+        }
+    }
+    registerEntry(id, entry) {
+        const model = {
+            id,
+            name: entry.id,
+            inference: entry.inference,
+            context: entry.context,
+            quant: entry.quant,
+            capabilities: entry.capabilities,
+            latency: entry.latency,
+            size_bytes: 4_000_000_000,
+            loaded: false,
+            loaded_at: undefined,
+        };
+        this.entries.set(id, model);
+        return model;
+    }
+    list() {
+        return Array.from(this.entries.values());
+    }
+    get(id) {
+        return this.entries.get(id);
+    }
+    find(options = {}) {
+        const results = this.list().filter((model) => {
+            if (options.capability && !model.capabilities.includes(options.capability)) {
+                return false;
+            }
+            if (options.latency && model.latency !== options.latency) {
+                return false;
+            }
+            if (options.inference && model.inference !== options.inference) {
+                return false;
+            }
+            if (options.minContext && model.context < options.minContext) {
+                return false;
+            }
+            if (options.loaded !== undefined && model.loaded !== options.loaded) {
+                return false;
+            }
+            return true;
+        });
+        return results;
+    }
+    findOne(options = {}) {
+        return this.find(options)[0];
+    }
+    has(id) {
+        return this.entries.has(id);
+    }
+    count() {
+        return this.entries.size;
+    }
+    add(id, entry) {
+        if (this.entries.has(id)) {
+            throw new Error(`Model with id '${id}' already exists`);
+        }
+        return this.registerEntry(id, entry);
+    }
+    update(id, updates) {
+        const existing = this.entries.get(id);
+        if (!existing) {
+            throw new Error(`Model with id '${id}' not found`);
+        }
+        const updated = {
+            ...existing,
+            ...updates,
+            id,
+        };
+        this.entries.set(id, updated);
+        return updated;
+    }
+    remove(id) {
+        return this.entries.delete(id);
+    }
+    clear() {
+        this.entries.clear();
+    }
+    toObject() {
+        return Object.fromEntries(this.entries);
+    }
+    fromObject(obj) {
+        this.entries.clear();
+        for (const [id, entry] of Object.entries(obj)) {
+            this.entries.set(id, entry);
+        }
+    }
+}
+export function ModelRegistry(config) {
+    return new ModelRegistryImpl(config);
+}
+//# sourceMappingURL=registry.js.map

package/dist/registry.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"registry.js","sourceRoot":"","sources":["../src/registry.ts"],"names":[],"mappings":"AAOA,MAAM,OAAO,iBAAiB;IACpB,OAAO,GAAoC,IAAI,GAAG,EAAE,CAAC;IAE7D,YAAY,MAA2B;QACrC,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC;QACrC,KAAK,MAAM,CAAC,EAAE,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;YAClD,IAAI,CAAC,aAAa,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;IAEO,aAAa,CAAC,EAAU,EAAE,KAAyB;QACzD,MAAM,KAAK,GAAuB;YAChC,EAAE;YACF,IAAI,EAAE,KAAK,CAAC,EAAE;YACd,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,YAAY,EAAE,KAAK,CAAC,YAAY;YAChC,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,UAAU,EAAE,aAAa;YACzB,MAAM,EAAE,KAAK;YACb,SAAS,EAAE,SAAS;SACrB,CAAC;QACF,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;QAC5B,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI;QACF,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAC3C,CAAC;IAED,GAAG,CAAC,EAAU;QACZ,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAC9B,CAAC;IAED,IAAI,CAAC,UAA4B,EAAE;QACjC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE;YAC3C,IAAI,OAAO,CAAC,UAAU,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,QAAQ,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;gBAC3E,OAAO,KAAK,CAAC;YACf,CAAC;YACD,IAAI,OAAO,CAAC,OAAO,IAAI,KAAK,CAAC,OAAO,KAAK,OAAO,CAAC,OAAO,EAAE,CAAC;gBACzD,OAAO,KAAK,CAAC;YACf,CAAC;YACD,IAAI,OAAO,CAAC,SAAS,IAAI,KAAK,CAAC,SAAS,KAAK,OAAO,CAAC,SAAS,EAAE,CAAC;gBAC/D,OAAO,KAAK,CAAC;YACf,CAAC;YACD,IAAI,OAAO,CAAC,UAAU,IAAI,KAAK,CAAC,OAAO,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC;gBAC7D,OAAO,KAAK,CAAC;YACf,CAAC;YACD,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,IAAI,KAAK,CAAC,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC;gBACpE,OAAO,KAAK,CAAC;YACf,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,CAAC;QACH,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,OAAO,CAAC,UAA4B,EAAE;QACpC,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/B,CAAC;IAED,GAAG,CAAC,EAAU;QACZ,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAC9B,CAAC;IAED,KAAK;QACH,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;IAC3B,CAAC;IAED,GAAG,CAAC,EAAU,EAAE,KAAyB;QACvC,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CAAC,kBAAkB,EAAE,kBAAkB,CAAC,CAAC;QAC1D,CAAC;QACD,OAAO,IAAI,CAAC,aAAa,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;IACvC,CAAC;IAED,MAAM,CAAC,EAAU,EAAE,OAAoC;QACrD,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACtC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CAAC,kBAAkB,EAAE,aAAa,CAAC,CAAC;QACrD,CAAC;QAED,MAAM,OAAO,GAAuB;YAClC,GAAG,QAAQ;YACX,GAAG,OAAO;YACV,EAAE;SACH,CAAC;QACF,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;QAC9B,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,MAAM,CAAC,EAAU;QACf,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACjC,CAAC;IAED,KAAK;QACH,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;IAED,QAAQ;QACN,OAAO,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC1C,CAAC;IAED,UAAU,CAAC,GAAuC;QAChD,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACrB,KAAK,MAAM,CAAC,EAAE,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;YAC9C,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC;CACF;AAED,MAAM,UAAU,aAAa,CAC3B,MAA2B;IAE3B,OAAO,IAAI,iBAAiB,CAAC,MAAM,CAAC,CAAC;AACvC,CAAC"}

package/dist/types.d.ts

@@ -0,0 +1,125 @@
+export type InferenceBackend = "ollama" | "llama" | "huggingface" | "openai";
+export type ModelCapability = "chat" | "vision" | "embedding" | "completion";
+export type LatencyProfile = "extreme" | "fast" | "slow";
+export interface ModelRegistryEntry {
+    id: string;
+    name: string;
+    inference: InferenceBackend;
+    context: number;
+    quant?: string;
+    capabilities: ModelCapability[];
+    latency?: LatencyProfile;
+    size_bytes: number;
+    loaded: boolean;
+    loaded_at?: string;
+}
+export interface RegistryEntryInput {
+    id: string;
+    inference: InferenceBackend;
+    context: number;
+    quant?: string;
+    capabilities: ModelCapability[];
+    latency?: LatencyProfile;
+}
+export interface ModelRegistryConfig {
+    entries: Record<string, RegistryEntryInput>;
+}
+export interface HealthResponse {
+    status: string;
+    timestamp: string;
+    models_loaded: number;
+}
+export interface ModelListResponse {
+    models: ModelRegistryEntry[];
+}
+export interface RegisterModelRequest {
+    id: string;
+    name: string;
+    inference: InferenceBackend;
+    context: number;
+    quant?: string;
+    capabilities: ModelCapability[];
+    latency?: LatencyProfile;
+    size_bytes?: number;
+}
+export interface RegisterModelResponse {
+    success: boolean;
+    model: ModelRegistryEntry;
+    message: string;
+}
+export interface LoadModelRequest {
+    model_id: string;
+}
+export interface LoadModelResponse {
+    success: boolean;
+    model_id: string;
+    message: string;
+}
+export interface UnloadModelResponse {
+    success: boolean;
+    model_id: string;
+    message: string;
+}
+export interface InferenceRequest {
+    model_id: string;
+    prompt: string;
+    max_tokens?: number;
+    temperature?: number;
+}
+export interface InferenceResponse {
+    model_id: string;
+    text: string;
+    tokens_generated: number;
+    finish_reason: string;
+}
+export interface StreamToken {
+    token: string;
+    token_id: number;
+    complete: boolean;
+}
+export type StreamCallback = (token: StreamToken) => void;
+export type StreamCompleteCallback = (response: InferenceResponse) => void;
+export type StreamErrorCallback = (error: Error) => void;
+export interface StreamOptions {
+    onToken: StreamCallback;
+    onComplete?: StreamCompleteCallback;
+    onError?: StreamErrorCallback;
+}
+export interface OpenLLMConfig {
+    engine?: string | number;
+    timeout?: number;
+}
+export interface FindModelOptions {
+    capability?: ModelCapability;
+    latency?: LatencyProfile;
+    inference?: InferenceBackend;
+    minContext?: number;
+    loaded?: boolean;
+}
+export interface APIConfig {
+    modelrouter?: boolean;
+    registry?: unknown | string;
+    engine?: string | number;
+    prefix?: string;
+}
+export declare class OpenLLMError extends Error {
+    code?: string | undefined;
+    statusCode?: number | undefined;
+    constructor(message: string, code?: string | undefined, statusCode?: number | undefined);
+}
+export declare class ModelNotFoundError extends OpenLLMError {
+    constructor(modelId: string);
+}
+export declare class ModelNotLoadedError extends OpenLLMError {
+    constructor(modelId: string);
+}
+export declare class InferenceError extends OpenLLMError {
+    constructor(message: string);
+}
+export interface ModelRegistryInstance {
+    list(): ReturnType<typeof import("./registry").ModelRegistryImpl.prototype.list>;
+    get(id: string): ReturnType<typeof import("./registry").ModelRegistryImpl.prototype.get>;
+    findOne(options: FindModelOptions): ReturnType<typeof import("./registry").ModelRegistryImpl.prototype.findOne>;
+}
+export type ModelRegistryImpl = ModelRegistryInstance;
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,gBAAgB,GAAG,QAAQ,GAAG,OAAO,GAAG,aAAa,GAAG,QAAQ,CAAC;AAE7E,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG,QAAQ,GAAG,WAAW,GAAG,YAAY,CAAC;AAE7E,MAAM,MAAM,cAAc,GAAG,SAAS,GAAG,MAAM,GAAG,MAAM,CAAC;AAEzD,MAAM,WAAW,kBAAkB;IACjC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,gBAAgB,CAAC;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,eAAe,EAAE,CAAC;IAChC,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,OAAO,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,kBAAkB;IACjC,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,gBAAgB,CAAC;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,eAAe,EAAE,CAAC;IAChC,OAAO,CAAC,EAAE,cAAc,CAAC;CAC1B;AAED,MAAM,WAAW,mBAAmB;IAClC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC;CAC7C;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,iBAAiB;IAChC,MAAM,EAAE,kBAAkB,EAAE,CAAC;CAC9B;AAED,MAAM,WAAW,oBAAoB;IACnC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,gBAAgB,CAAC;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,eAAe,EAAE,CAAC;IAChC,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,qBAAqB;IACpC,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,kBAAkB,CAAC;IAC1B,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,OAAO,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,mBAAmB;IAClC,OAAO,EAAE,OAAO,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,WAAW;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,MAAM,cAAc,GAAG,CAAC,KAAK,EAAE,WAAW,KAAK,IAAI,CAAC;AAC1D,MAAM,MAAM,sBAAsB,GAAG,CAAC,QAAQ,EAAE,iBAAiB,KAAK,IAAI,CAAC;AAC3E,MAAM,MAAM,mBAAmB,GAAG,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC;AAEzD,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,cAAc,CAAC;IACxB,UAAU,CAAC,EAAE,sBAAsB,CAAC;IACpC,OAAO,CAAC,EAAE,mBAAmB,CAAC;CAC/B;AAED,MAAM,WAAW,aAAa;IAC5B,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IACzB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,gBAAgB;IAC/B,UAAU,CAAC,EAAE,eAAe,CAAC;IAC7B,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB,SAAS,CAAC,EAAE,gBAAgB,CAAC;IAC7B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,SAAS;IACxB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,QAAQ,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;IAC5B,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IACzB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,qBAAa,YAAa,SAAQ,KAAK;IAG5B,IAAI,CAAC,EAAE,MAAM;IACb,UAAU,CAAC,EAAE,MAAM;gBAF1B,OAAO,EAAE,MAAM,EACR,IAAI,CAAC,EAAE,MAAM,YAAA,EACb,UAAU,CAAC,EAAE,MAAM,YAAA;CAK7B;AAED,qBAAa,kBAAmB,SAAQ,YAAY;gBACtC,OAAO,EAAE,MAAM;CAI5B;AAED,qBAAa,mBAAoB,SAAQ,YAAY;gBACvC,OAAO,EAAE,MAAM;CAI5B;AAED,qBAAa,cAAe,SAAQ,YAAY;gBAClC,OAAO,EAAE,MAAM;CAI5B;AAED,MAAM,WAAW,qBAAqB;IACpC,IAAI,IAAI,UAAU,CAAC,cAAc,YAAY,EAAE,iBAAiB,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IACjF,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,UAAU,CAAC,cAAc,YAAY,EAAE,iBAAiB,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IACzF,OAAO,CAAC,OAAO,EAAE,gBAAgB,GAAG,UAAU,CAAC,cAAc,YAAY,EAAE,iBAAiB,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;CACjH;AAED,MAAM,MAAM,iBAAiB,GAAG,qBAAqB,CAAC"}

package/dist/types.js

@@ -0,0 +1,29 @@
+export class OpenLLMError extends Error {
+    code;
+    statusCode;
+    constructor(message, code, statusCode) {
+        super(message);
+        this.code = code;
+        this.statusCode = statusCode;
+        this.name = "OpenLLMError";
+    }
+}
+export class ModelNotFoundError extends OpenLLMError {
+    constructor(modelId) {
+        super(`Model '${modelId}' not found`, "MODEL_NOT_FOUND", 404);
+        this.name = "ModelNotFoundError";
+    }
+}
+export class ModelNotLoadedError extends OpenLLMError {
+    constructor(modelId) {
+        super(`Model '${modelId}' is not loaded`, "MODEL_NOT_LOADED", 412);
+        this.name = "ModelNotLoadedError";
+    }
+}
+export class InferenceError extends OpenLLMError {
+    constructor(message) {
+        super(message, "INFERENCE_ERROR", 502);
+        this.name = "InferenceError";
+    }
+}
+//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AA6HA,MAAM,OAAO,YAAa,SAAQ,KAAK;IAG5B;IACA;IAHT,YACE,OAAe,EACR,IAAa,EACb,UAAmB;QAE1B,KAAK,CAAC,OAAO,CAAC,CAAC;QAHR,SAAI,GAAJ,IAAI,CAAS;QACb,eAAU,GAAV,UAAU,CAAS;QAG1B,IAAI,CAAC,IAAI,GAAG,cAAc,CAAC;IAC7B,CAAC;CACF;AAED,MAAM,OAAO,kBAAmB,SAAQ,YAAY;IAClD,YAAY,OAAe;QACzB,KAAK,CAAC,UAAU,OAAO,aAAa,EAAE,iBAAiB,EAAE,GAAG,CAAC,CAAC;QAC9D,IAAI,CAAC,IAAI,GAAG,oBAAoB,CAAC;IACnC,CAAC;CACF;AAED,MAAM,OAAO,mBAAoB,SAAQ,YAAY;IACnD,YAAY,OAAe;QACzB,KAAK,CAAC,UAAU,OAAO,iBAAiB,EAAE,kBAAkB,EAAE,GAAG,CAAC,CAAC;QACnE,IAAI,CAAC,IAAI,GAAG,qBAAqB,CAAC;IACpC,CAAC;CACF;AAED,MAAM,OAAO,cAAe,SAAQ,YAAY;IAC9C,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,EAAE,iBAAiB,EAAE,GAAG,CAAC,CAAC;QACvC,IAAI,CAAC,IAAI,GAAG,gBAAgB,CAAC;IAC/B,CAAC;CACF"}

package/example.ts

@@ -0,0 +1,62 @@
+import { createOpenLLMClient } from "@use-solace/openllm";
+
+async function main() {
+  const client = createOpenLLMClient({ engine: 8080 });
+
+  try {
+    console.log("1. Registering mistral model...");
+    const registerResponse = await client.registerModel({
+      id: "mistral",
+      name: "Mistral 7B",
+      inference: "ollama",
+      context: 8192,
+      quant: "Q4_K_M",
+      capabilities: ["chat", "completion"],
+      latency: "fast",
+    });
+    console.log("   Registered:", registerResponse.model.name);
+
+    console.log("\n2. Loading mistral model...");
+    const loadResponse = await client.loadModel({ model_id: "mistral" });
+    console.log("   Load response:", loadResponse.message);
+
+    console.log("\n3. Running streamed inference...");
+    console.log("   Prompt: 'What is the meaning of life?'");
+    console.log("\n   Response:");
+
+    let fullResponse = "";
+
+    await client.inferenceStream(
+      {
+        model_id: "mistral",
+        prompt: "What is the meaning of life?",
+        max_tokens: 512,
+        temperature: 0.7,
+      },
+      {
+        onToken: (token) => {
+          process.stdout.write(token.token);
+          fullResponse += token.token;
+        },
+        onComplete: (response) => {
+          console.log("\n\n   Stream completed!");
+          console.log("   Total tokens:", response.tokens_generated);
+        },
+        onError: (error) => {
+          console.error("\n   Stream error:", error.message);
+        },
+      },
+    );
+
+    console.log("\n\n4. Unloading model...");
+    const unloadResponse = await client.unloadModel("mistral");
+    console.log("   Unload response:", unloadResponse.message);
+
+  } catch (error) {
+    if (error instanceof Error) {
+      console.error("Error:", error.message);
+    }
+  }
+}
+
+main();

package/package.json

@@ -0,0 +1,46 @@
+{
+  "name": "@use-solace/openllm",
+  "version": "1.0.0",
+  "description": "OpenLLM model registry and API client with full TypeScript support",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "scripts": {
+    "build": "tsc",
+    "dev": "tsc --watch",
+    "prepublishOnly": "npm run build"
+  },
+  "keywords": [
+    "openllm",
+    "ollama",
+    "llama.cpp",
+    "huggingface",
+    "inference",
+    "model-registry",
+    "elysia"
+  ],
+  "author": "Solace Contributors",
+  "license": "MIT",
+  "peerDependencies": {
+    "elysia": "^1.0.0"
+  },
+  "peerDependenciesMeta": {
+    "elysia": {
+      "optional": true
+    }
+  },
+  "dependencies": {
+    "@elysiajs/bearer": "^1.0.0"
+  },
+  "devDependencies": {
+    "@types/node": "^20.0.0",
+    "typescript": "^5.0.0",
+    "elysia": "^1.0.0"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  }
+}

package/src/client.ts

@@ -0,0 +1,244 @@
+import type {
+  HealthResponse,
+  InferenceRequest,
+  InferenceResponse,
+  LoadModelRequest,
+  LoadModelResponse,
+  ModelListResponse,
+  ModelNotFoundError,
+  ModelNotLoadedError,
+  OpenLLMConfig,
+  OpenLLMError,
+  RegisterModelRequest,
+  RegisterModelResponse,
+  StreamOptions,
+  StreamToken,
+  UnloadModelResponse,
+} from "./types.js";
+
+export class OpenLLMClient {
+  private baseUrl: string;
+  private timeout: number;
+
+  constructor(config: OpenLLMConfig = {}) {
+    this.baseUrl = `http://localhost:${config.engine ?? 8080}`;
+    this.timeout = config.timeout ?? 30000;
+  }
+
+  private async request<T>(
+    endpoint: string,
+    options: RequestInit = {},
+  ): Promise<T> {
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), this.timeout);
+
+    try {
+      const response = await fetch(`${this.baseUrl}${endpoint}`, {
+        ...options,
+        signal: controller.signal,
+        headers: {
+          "Content-Type": "application/json",
+          ...options.headers,
+        },
+      });
+
+      clearTimeout(timeoutId);
+
+      if (!response.ok) {
+        const errorText = await response.text();
+        throw this.createError(response.status, errorText);
+      }
+
+      return (await response.json()) as T;
+    } catch (error) {
+      clearTimeout(timeoutId);
+      if (error instanceof Error) {
+        if (error.name === "AbortError") {
+          throw new Error(`Request timeout after ${this.timeout}ms`);
+        }
+        throw error;
+      }
+      throw error;
+    }
+  }
+
+  private createError(status: number, message: string): OpenLLMError {
+    if (status === 404) {
+      const match = message.match(/Model '([^']+)'/);
+      if (match) {
+        const error = new Error(message) as ModelNotFoundError;
+        error.name = "ModelNotFoundError";
+        error.code = "MODEL_NOT_FOUND";
+        error.statusCode = status;
+        return error;
+      }
+    }
+    if (status === 412) {
+      const match = message.match(/Model '([^']+)'/);
+      if (match) {
+        const error = new Error(message) as ModelNotLoadedError;
+        error.name = "ModelNotLoadedError";
+        error.code = "MODEL_NOT_LOADED";
+        error.statusCode = status;
+        return error;
+      }
+    }
+
+    const error = new Error(message) as OpenLLMError;
+    error.name = "OpenLLMError";
+    error.code = "API_ERROR";
+    error.statusCode = status;
+    return error;
+  }
+
+  async health(): Promise<HealthResponse> {
+    return this.request<HealthResponse>("/health");
+  }
+
+  async listModels(): Promise<ModelListResponse> {
+    return this.request<ModelListResponse>("/v1/models");
+  }
+
+  async registerModel(
+    data: RegisterModelRequest,
+  ): Promise<RegisterModelResponse> {
+    return this.request<RegisterModelResponse>("/v1/models/register", {
+      method: "POST",
+      body: JSON.stringify(data),
+    });
+  }
+
+  async loadModel(data: LoadModelRequest): Promise<LoadModelResponse> {
+    return this.request<LoadModelResponse>("/v1/models/load", {
+      method: "POST",
+      body: JSON.stringify(data),
+    });
+  }
+
+  async unloadModel(modelId: string): Promise<UnloadModelResponse> {
+    return this.request<UnloadModelResponse>(
+      `/v1/models/unload/${modelId}`,
+      {
+        method: "POST",
+      },
+    );
+  }
+
+  async inference(data: InferenceRequest): Promise<InferenceResponse> {
+    return this.request<InferenceResponse>("/v1/inference", {
+      method: "POST",
+      body: JSON.stringify(data),
+    });
+  }
+
+  async inferenceStream(
+    data: InferenceRequest,
+    options: StreamOptions,
+  ): Promise<void> {
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), this.timeout);
+
+    try {
+      const response = await fetch(`${this.baseUrl}/v1/inference/stream`, {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+        },
+        body: JSON.stringify(data),
+        signal: controller.signal,
+      });
+
+      clearTimeout(timeoutId);
+
+      if (!response.ok) {
+        const errorText = await response.text();
+        const error = this.createError(response.status, errorText);
+        options.onError?.(error);
+        throw error;
+      }
+
+      const reader = response.body?.getReader();
+      if (!reader) {
+        const error = new Error("No response body");
+        options.onError?.(error);
+        throw error;
+      }
+
+      const decoder = new TextDecoder();
+      let buffer = "";
+      let accumulatedText = "";
+      let tokenCount = 0;
+      const modelId = data.model_id;
+
+      while (true) {
+        const { done, value } = await reader.read();
+        if (done) break;
+
+        buffer += decoder.decode(value, { stream: true });
+        const lines = buffer.split("\n");
+        buffer = lines.pop() ?? "";
+
+        for (const line of lines) {
+          if (line.trim().startsWith("event: token")) {
+            continue;
+          }
+          if (line.trim().startsWith("data: ")) {
+            const data = line.trim().slice(6);
+            if (data) {
+              try {
+                const token = JSON.parse(data) as StreamToken;
+                accumulatedText += token.token;
+                tokenCount++;
+                options.onToken(token);
+
+                if (token.complete) {
+                  options.onComplete?.({
+                    model_id: modelId,
+                    text: accumulatedText,
+                    tokens_generated: tokenCount,
+                    finish_reason: "stop",
+                  });
+                }
+              } catch (e) {
+                console.error("Failed to parse SSE data:", data, e);
+              }
+            }
+          }
+        }
+      }
+    } catch (error) {
+      clearTimeout(timeoutId);
+      if (error instanceof Error) {
+        if (error.name === "AbortError") {
+          const timeoutError = new Error(
+            `Stream timeout after ${this.timeout}ms`,
+          );
+          options.onError?.(timeoutError);
+          throw timeoutError;
+        }
+        options.onError?.(error);
+        throw error;
+      }
+    }
+  }
+
+  setBaseUrl(baseUrl: string): void {
+    this.baseUrl = baseUrl;
+  }
+
+  getBaseUrl(): string {
+    return this.baseUrl;
+  }
+
+  setTimeout(timeout: number): void {
+    this.timeout = timeout;
+  }
+
+  getTimeout(): number {
+    return this.timeout;
+  }
+}
+
+export function createOpenLLMClient(config?: OpenLLMConfig): OpenLLMClient {
+  return new OpenLLMClient(config);
+}