@infernetprotocol/engine 0.1.22

package/src/backends/vllm.js

@@ -0,0 +1,244 @@
+/**
+ * vLLM backend — talks to a local vLLM server over HTTP using the
+ * OpenAI-compatible chat completions endpoint.
+ *
+ * Why this exists alongside the Ollama backend: vLLM is the high-throughput
+ * serving engine of choice for serious GPU operators (PagedAttention,
+ * tensor + pipeline parallelism via Ray, request batching). Ollama wins on
+ * "just works on any GPU + CPU + Apple Silicon"; vLLM wins on throughput
+ * and on serving big models that don't fit on one GPU. Operators pick
+ * whichever fits their hardware — we speak both.
+ *
+ * Discovery: `isVllmReachable()` does a fast probe of `/v1/models`. The
+ * factory in `index.js` calls it during auto-selection so a node with
+ * vLLM running picks `vllm` automatically; nodes without it fall through
+ * to `ollama` (or `stub`).
+ *
+ * Streaming: vLLM uses OpenAI-style Server-Sent Events — each line is
+ * `data: {...}` JSON, terminated by `data: [DONE]`. We translate that
+ * into our v1 protocol events.
+ *
+ * Cancellation: each generation owns an AbortController; `cancel()`
+ * aborts the in-flight fetch, which surfaces a `done` event with
+ * `reason: "cancel"`.
+ */
+
+import { randomUUID } from "node:crypto";
+import { AsyncQueue } from "../async-queue.js";
+import { MSG, PROTOCOL_VERSION } from "../protocol.js";
+
+const DEFAULT_HOST = "http://localhost:8000";
+const PROBE_TIMEOUT_MS = 500;
+
+export const VLLM_DEFAULT_HOST = DEFAULT_HOST;
+
+export async function isVllmReachable(host = DEFAULT_HOST, timeoutMs = PROBE_TIMEOUT_MS) {
+    try {
+        const ctrl = new AbortController();
+        const t = setTimeout(() => ctrl.abort(), timeoutMs);
+        const res = await fetch(new URL("/v1/models", host), { signal: ctrl.signal });
+        clearTimeout(t);
+        return res.ok;
+    } catch {
+        return false;
+    }
+}
+
+export async function createVllmBackend({
+    host = process.env.VLLM_HOST ?? DEFAULT_HOST,
+    apiKey = process.env.VLLM_API_KEY ?? null,
+    defaultModel = process.env.VLLM_MODEL ?? process.env.INFERNET_ENGINE_MODEL ?? null,
+    skipProbe = false,
+    fetchImpl = globalThis.fetch
+} = {}) {
+    if (!skipProbe) {
+        const ok = await isVllmReachable(host);
+        if (!ok) {
+            throw new Error(
+                `vLLM not reachable at ${host} — start with 'vllm serve <model>' or set VLLM_HOST`
+            );
+        }
+    }
+
+    return {
+        kind: "vllm",
+        host,
+        hasApiKey: Boolean(apiKey),
+        generate({ messages, id, model = null, max_tokens, temperature } = {}) {
+            const genId = id ?? randomUUID();
+            const stream = new AsyncQueue();
+            const ctrl = new AbortController();
+            const resolvedModel = model ?? defaultModel;
+
+            (async () => {
+                if (!resolvedModel) {
+                    stream.push({
+                        v: PROTOCOL_VERSION,
+                        type: MSG.ERROR,
+                        id: genId,
+                        message:
+                            "vllm backend: no model — set VLLM_MODEL or pass model in the job"
+                    });
+                    stream.end();
+                    return;
+                }
+
+                const body = {
+                    model: resolvedModel,
+                    messages: Array.isArray(messages) ? messages : [],
+                    stream: true
+                };
+                if (typeof temperature === "number") body.temperature = temperature;
+                if (typeof max_tokens === "number") body.max_tokens = max_tokens;
+
+                const headers = { "content-type": "application/json" };
+                if (apiKey) headers["authorization"] = `Bearer ${apiKey}`;
+
+                let res;
+                try {
+                    res = await fetchImpl(new URL("/v1/chat/completions", host), {
+                        method: "POST",
+                        headers,
+                        body: JSON.stringify(body),
+                        signal: ctrl.signal
+                    });
+                } catch (err) {
+                    pushTerminal(stream, genId, ctrl, err, "");
+                    return;
+                }
+
+                if (!res.ok) {
+                    let detail = "";
+                    try {
+                        detail = await res.text();
+                    } catch {
+                        // best-effort
+                    }
+                    stream.push({
+                        v: PROTOCOL_VERSION,
+                        type: MSG.ERROR,
+                        id: genId,
+                        message: `vllm HTTP ${res.status}: ${detail.slice(0, 200)}`
+                    });
+                    stream.end();
+                    return;
+                }
+
+                let metaSent = false;
+                let acc = "";
+                let buf = "";
+                let finishReason = null;
+                const decoder = new TextDecoder();
+
+                try {
+                    for await (const chunk of res.body) {
+                        buf += decoder.decode(chunk, { stream: true });
+                        let nl;
+                        while ((nl = buf.indexOf("\n")) !== -1) {
+                            const rawLine = buf.slice(0, nl);
+                            buf = buf.slice(nl + 1);
+                            const line = rawLine.trim();
+                            if (!line || !line.startsWith("data:")) continue;
+                            const payload = line.slice("data:".length).trim();
+                            if (payload === "[DONE]") {
+                                stream.push({
+                                    v: PROTOCOL_VERSION,
+                                    type: MSG.DONE,
+                                    id: genId,
+                                    reason: finishReason ?? "stop",
+                                    text: acc,
+                                    finished_at: new Date().toISOString()
+                                });
+                                stream.end();
+                                return;
+                            }
+                            let parsed;
+                            try {
+                                parsed = JSON.parse(payload);
+                            } catch {
+                                continue;
+                            }
+
+                            if (!metaSent) {
+                                stream.push({
+                                    v: PROTOCOL_VERSION,
+                                    type: MSG.META,
+                                    id: genId,
+                                    model: parsed.model ?? resolvedModel,
+                                    started_at: new Date().toISOString()
+                                });
+                                metaSent = true;
+                            }
+
+                            const choice = Array.isArray(parsed.choices) ? parsed.choices[0] : null;
+                            const text = choice?.delta?.content ?? "";
+                            if (text) {
+                                acc += text;
+                                stream.push({
+                                    v: PROTOCOL_VERSION,
+                                    type: MSG.TOKEN,
+                                    id: genId,
+                                    text
+                                });
+                            }
+                            if (choice?.finish_reason) {
+                                finishReason = choice.finish_reason;
+                            }
+                        }
+                    }
+                    // body ended without [DONE] — surface as stop with whatever we have.
+                    stream.push({
+                        v: PROTOCOL_VERSION,
+                        type: MSG.DONE,
+                        id: genId,
+                        reason: finishReason ?? "stop",
+                        text: acc,
+                        finished_at: new Date().toISOString()
+                    });
+                    stream.end();
+                } catch (err) {
+                    pushTerminal(stream, genId, ctrl, err, acc);
+                }
+            })().catch((err) => {
+                stream.push({
+                    v: PROTOCOL_VERSION,
+                    type: MSG.ERROR,
+                    id: genId,
+                    message: err?.message ?? String(err)
+                });
+                stream.end();
+            });
+
+            return {
+                id: genId,
+                stream,
+                cancel: () => ctrl.abort()
+            };
+        },
+        async shutdown() {
+            // The vLLM server is owned by the operator, not by us.
+            // Nothing to clean up.
+        }
+    };
+}
+
+function pushTerminal(stream, genId, ctrl, err, accSoFar) {
+    if (ctrl.signal.aborted) {
+        stream.push({
+            v: PROTOCOL_VERSION,
+            type: MSG.DONE,
+            id: genId,
+            reason: "cancel",
+            text: accSoFar,
+            finished_at: new Date().toISOString()
+        });
+    } else {
+        stream.push({
+            v: PROTOCOL_VERSION,
+            type: MSG.ERROR,
+            id: genId,
+            message: err?.message ?? String(err)
+        });
+    }
+    stream.end();
+}

package/src/engine-process.js

@@ -0,0 +1,167 @@
+/**
+ * Long-lived child-process wrapper around an engine binary.
+ *
+ * Spawns the binary once, keeps it warm so model load cost isn't paid per
+ * request, and demultiplexes streamed responses by generation id. Each call
+ * to `generate()` returns an async-iterable stream of protocol events that
+ * terminates on the first `done` or `error` for that id.
+ *
+ * The wrapper is binary-agnostic — it talks to anything that speaks the
+ * NDJSON protocol in `./protocol.js`. The Mojo backend uses it; tests use
+ * it with a fake Node-based engine.
+ */
+
+import { spawn } from "node:child_process";
+import { randomUUID } from "node:crypto";
+import { AsyncQueue } from "./async-queue.js";
+import { encode, MSG, NdjsonSplitter } from "./protocol.js";
+
+export class EngineProcess {
+    /**
+     * @param {{ binary: string, args?: string[], env?: Record<string,string>, model?: string|null, cwd?: string }} opts
+     */
+    constructor({ binary, args = [], env = {}, model = null, cwd } = {}) {
+        if (!binary) throw new Error("EngineProcess: binary is required");
+        this.binary = binary;
+        this.args = args;
+        this.env = env;
+        this.model = model;
+        this.cwd = cwd;
+        this.child = null;
+        this.streams = new Map();
+        this.splitter = new NdjsonSplitter();
+        this.closed = false;
+        this._readyResolve = null;
+        this._readyReject = null;
+    }
+
+    async start() {
+        this.child = spawn(this.binary, this.args, {
+            stdio: ["pipe", "pipe", "inherit"],
+            env: { ...process.env, ...this.env },
+            cwd: this.cwd
+        });
+        this.child.stdout.setEncoding("utf8");
+        this.child.stdout.on("data", (chunk) => this._onStdout(chunk));
+        this.child.on("exit", (code, signal) => this._onExit(code, signal));
+        this.child.on("error", (err) => this._onChildError(err));
+
+        const ready = new Promise((resolve, reject) => {
+            this._readyResolve = resolve;
+            this._readyReject = reject;
+        });
+
+        if (this.model) {
+            this._send({ type: MSG.LOAD, model: this.model });
+        }
+
+        return ready;
+    }
+
+    _send(msg) {
+        if (!this.child || this.closed) throw new Error("engine not running");
+        this.child.stdin.write(encode(msg));
+    }
+
+    _onStdout(chunk) {
+        for (const msg of this.splitter.push(chunk)) {
+            if (msg.type === MSG.READY) {
+                if (this._readyResolve) {
+                    this._readyResolve(msg);
+                    this._readyResolve = null;
+                    this._readyReject = null;
+                }
+                continue;
+            }
+            if (msg.type === MSG.LOG) {
+                const level = msg.level ?? "info";
+                process.stderr.write(`[engine ${level}] ${msg.message ?? ""}\n`);
+                continue;
+            }
+            const id = msg.id;
+            if (id && this.streams.has(id)) {
+                const q = this.streams.get(id);
+                q.push(msg);
+                if (msg.type === MSG.DONE || msg.type === MSG.ERROR) {
+                    q.end();
+                    this.streams.delete(id);
+                }
+                continue;
+            }
+            // Engine-level error before ready.
+            if (msg.type === MSG.ERROR && this._readyReject) {
+                this._readyReject(new Error(msg.message ?? "engine error during start"));
+                this._readyResolve = null;
+                this._readyReject = null;
+            }
+        }
+    }
+
+    _onExit(code, signal) {
+        if (this.closed) return;
+        this.closed = true;
+        const err = new Error(`engine exited (code=${code ?? "null"}, signal=${signal ?? "null"})`);
+        for (const q of this.streams.values()) q.fail(err);
+        this.streams.clear();
+        if (this._readyReject) {
+            this._readyReject(err);
+            this._readyResolve = null;
+            this._readyReject = null;
+        }
+    }
+
+    _onChildError(err) {
+        if (this._readyReject) {
+            this._readyReject(err);
+            this._readyResolve = null;
+            this._readyReject = null;
+        }
+    }
+
+    /**
+     * Start a generation. Returns `{ id, stream, cancel }` where `stream`
+     * is an async-iterable yielding protocol events (`meta`, `token`,
+     * `done`, `error`) until the generation terminates.
+     */
+    generate(req = {}) {
+        if (this.closed) throw new Error("engine closed");
+        const id = req.id ?? randomUUID();
+        const q = new AsyncQueue();
+        this.streams.set(id, q);
+        this._send({ type: MSG.GENERATE, ...req, id });
+        return {
+            id,
+            stream: q,
+            cancel: () => {
+                try {
+                    this._send({ type: MSG.CANCEL, id });
+                } catch {
+                    // engine already gone — caller will see done/error via stream
+                }
+            }
+        };
+    }
+
+    async shutdown({ timeoutMs = 5000 } = {}) {
+        if (this.closed || !this.child) return;
+        try {
+            this._send({ type: MSG.SHUTDOWN });
+        } catch {
+            // best-effort
+        }
+        await new Promise((resolve) => {
+            const t = setTimeout(() => {
+                try {
+                    this.child.kill("SIGTERM");
+                } catch {
+                    // already dead
+                }
+                resolve();
+            }, timeoutMs);
+            this.child.once("exit", () => {
+                clearTimeout(t);
+                resolve();
+            });
+        });
+    }
+}

package/src/index.js

@@ -0,0 +1,87 @@
+/**
+ * @infernetprotocol/engine — pluggable inference engine.
+ *
+ *   const engine = await createEngine();          // auto-selects
+ *   const { stream } = engine.generate({ messages, model });
+ *   for await (const ev of stream) {
+ *     // ev.type ∈ "meta" | "token" | "done" | "error"
+ *   }
+ *   await engine.shutdown();
+ *
+ * Backends:
+ *   - "max"     — Modular MAX serve (highest throughput, OpenAI-compatible).
+ *                 MODULAR_MAX_HOST or default :8000.
+ *   - "sglang"  — SGLang (RadixAttention, speculative decoding). SGLANG_HOST
+ *                 or default :30000.
+ *   - "vllm"    — vLLM (PagedAttention, tensor parallelism via Ray). VLLM_HOST
+ *                 or default :8000.
+ *   - "llamacpp" — llama.cpp / llama-swap. Lightweight, no Python. :8080.
+ *   - "ollama"  — Ollama daemon. Easiest setup; CUDA/ROCm/Metal/CPU.
+ *   - "mojo"    — Mojo + MAX binary. Experimental.
+ *   - "stub"    — in-process canned tokens. Fallback when nothing else runs.
+ *
+ * Auto-selection precedence (only if `opts.backend` is not set):
+ *   1. INFERNET_ENGINE_BACKEND env var — explicit override
+ *   2. INFERNET_ENGINE_BIN set         — operator's Mojo binary
+ *   3. MODULAR_MAX_HOST set + reachable — MAX (best throughput)
+ *   4. SGLang reachable on SGLANG_HOST  — :30000
+ *   5. vLLM reachable on VLLM_HOST      — :8000
+ *   6. llama.cpp reachable              — :8080
+ *   7. Ollama reachable                 — easy default
+ *   8. "stub"
+ */
+
+import { createMojoBackend } from "./backends/mojo.js";
+import { createOllamaBackend, isOllamaReachable } from "./backends/ollama.js";
+import { createVllmBackend, isVllmReachable } from "./backends/vllm.js";
+import { createSglangBackend, isSglangReachable } from "./backends/sglang.js";
+import { createMaxBackend, isMaxReachable } from "./backends/max.js";
+import { createLlamacppBackend, isLlamacppReachable } from "./backends/llamacpp.js";
+import { createStubBackend } from "./backends/stub.js";
+
+export * from "./protocol.js";
+export { AsyncQueue } from "./async-queue.js";
+export { EngineProcess } from "./engine-process.js";
+export { resolveBinary } from "./resolve-binary.js";
+export { isOllamaReachable } from "./backends/ollama.js";
+export { isVllmReachable } from "./backends/vllm.js";
+export { isSglangReachable } from "./backends/sglang.js";
+export { isMaxReachable } from "./backends/max.js";
+export { isLlamacppReachable } from "./backends/llamacpp.js";
+
+export async function createEngine(opts = {}) {
+    const backend = opts.backend ?? (await autoSelectBackend());
+    switch (backend) {
+        case "max":
+            return createMaxBackend(opts);
+        case "sglang":
+            return createSglangBackend(opts);
+        case "vllm":
+            return createVllmBackend(opts);
+        case "llamacpp":
+            return createLlamacppBackend(opts);
+        case "ollama":
+            return createOllamaBackend(opts);
+        case "mojo":
+            return createMojoBackend(opts);
+        case "stub":
+            return createStubBackend(opts);
+        default:
+            throw new Error(`unknown engine backend: ${backend}`);
+    }
+}
+
+async function autoSelectBackend() {
+    const explicit = process.env.INFERNET_ENGINE_BACKEND;
+    if (explicit) return explicit;
+    if (process.env.INFERNET_ENGINE_BIN) return "mojo";
+    // Precedence: MAX > SGLang > vLLM (throughput order per benchmarks) >
+    // llama.cpp (lightweight, no Python) > Ollama (widest hardware support).
+    // MAX shares port 8000 with vLLM — distinguish via MODULAR_MAX_HOST.
+    if (process.env.MODULAR_MAX_HOST && await isMaxReachable(process.env.MODULAR_MAX_HOST)) return "max";
+    if (await isSglangReachable(process.env.SGLANG_HOST)) return "sglang";
+    if (await isVllmReachable(process.env.VLLM_HOST)) return "vllm";
+    if (await isLlamacppReachable(process.env.LLAMACPP_HOST)) return "llamacpp";
+    if (await isOllamaReachable(process.env.OLLAMA_HOST)) return "ollama";
+    return "stub";
+}

package/src/protocol.js

@@ -0,0 +1,88 @@
+/**
+ * Engine IPC protocol — v1.
+ *
+ * Source of truth for messages exchanged between the JS sidecar wrapper
+ * (EngineProcess) and an engine binary on stdio. Format is NDJSON: one
+ * JSON object per line, each terminated by `\n`.
+ *
+ * Every message carries `v` (protocol version) and `type`. Generation-scoped
+ * messages also carry `id` so multiple in-flight generations can be
+ * demultiplexed on a single stdio pair.
+ *
+ * Outbound (JS → engine):
+ *   { v:1, type:"load",     model }
+ *   { v:1, type:"generate", id, messages, model?, max_tokens?, temperature?, ... }
+ *   { v:1, type:"cancel",   id }
+ *   { v:1, type:"shutdown" }
+ *
+ * Inbound (engine → JS):
+ *   { v:1, type:"ready",    model? }                              // engine-level
+ *   { v:1, type:"meta",     id, model?, started_at }              // generation start
+ *   { v:1, type:"token",    id, text }                            // streamed token
+ *   { v:1, type:"done",     id, reason:"stop"|"length"|"cancel", text }
+ *   { v:1, type:"error",    id?, message }                        // id absent → engine-level
+ *   { v:1, type:"log",      level:"info"|"warn"|"error", message }
+ *
+ * Bumping PROTOCOL_VERSION is a breaking change. Mirror any change in
+ * engine/mojo/src/main.mojo.
+ */
+
+export const PROTOCOL_VERSION = 1;
+
+export const MSG = Object.freeze({
+    // outbound
+    LOAD: "load",
+    GENERATE: "generate",
+    CANCEL: "cancel",
+    SHUTDOWN: "shutdown",
+    // inbound
+    READY: "ready",
+    META: "meta",
+    TOKEN: "token",
+    DONE: "done",
+    ERROR: "error",
+    LOG: "log"
+});
+
+export function encode(msg) {
+    return JSON.stringify({ v: PROTOCOL_VERSION, ...msg }) + "\n";
+}
+
+export function decode(line) {
+    const trimmed = typeof line === "string" ? line.trim() : "";
+    if (!trimmed) return null;
+    let parsed;
+    try {
+        parsed = JSON.parse(trimmed);
+    } catch (err) {
+        return { type: MSG.ERROR, message: `bad ndjson: ${err.message}`, raw: trimmed };
+    }
+    if (parsed.v !== PROTOCOL_VERSION) {
+        return {
+            type: MSG.ERROR,
+            message: `protocol version mismatch: expected ${PROTOCOL_VERSION}, got ${parsed.v}`
+        };
+    }
+    return parsed;
+}
+
+/**
+ * Buffered NDJSON splitter. Stdio chunks don't respect line boundaries, so
+ * we accumulate and yield one message per `\n` boundary.
+ */
+export class NdjsonSplitter {
+    constructor() {
+        this.buf = "";
+    }
+
+    *push(chunk) {
+        this.buf += chunk;
+        let idx;
+        while ((idx = this.buf.indexOf("\n")) !== -1) {
+            const line = this.buf.slice(0, idx);
+            this.buf = this.buf.slice(idx + 1);
+            const msg = decode(line);
+            if (msg) yield msg;
+        }
+    }
+}

package/src/resolve-binary.js

@@ -0,0 +1,21 @@
+/**
+ * Resolve the path to an engine binary.
+ *
+ * Lookup order:
+ *   1. Explicit env var (e.g. INFERNET_ENGINE_BIN) pointing at an existing file.
+ *   2. Bare command name — defer to PATH lookup at spawn time.
+ *
+ * A bundled-prebuild step (per-platform tarballs in `vendor/`) is intentional
+ * future work: the contract is just "return something spawn() can run".
+ */
+
+import { existsSync } from "node:fs";
+
+export function resolveBinary({ envVar, name } = {}) {
+    if (envVar) {
+        const fromEnv = process.env[envVar];
+        if (fromEnv && existsSync(fromEnv)) return fromEnv;
+    }
+    if (name) return name;
+    return null;
+}