@infernetprotocol/engine 0.1.22

package/LICENSE

@@ -0,0 +1,15 @@
+ISC License
+
+Copyright (c) 2025 Infernet Protocol
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted, provided that the above
+copyright notice and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
+OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+PERFORMANCE OF THIS SOFTWARE.

package/README.md

@@ -0,0 +1,88 @@
+# @infernetprotocol/engine
+
+Pluggable inference engine for the Infernet daemon. One async-iterator API
+across backends; the consumer (chat executor, future job runners) doesn't
+care which is loaded.
+
+## Backends
+
+| Backend  | Where it runs                       | When it's selected                                          |
+| -------- | ----------------------------------- | ----------------------------------------------------------- |
+| `ollama` | Local Ollama daemon (HTTP)          | **Default** when Ollama is reachable on `OLLAMA_HOST`       |
+| `mojo`   | External Mojo+MAX binary            | `INFERNET_ENGINE_BIN` set, or `backend:"mojo"` (experimental) |
+| `stub`   | In-process canned tokens            | Fallback when neither is available — daemon still works     |
+
+**Ollama is the recommended default.** It already solves model download
++ cache, multi-vendor GPU (CUDA / ROCm / Metal), per-platform installers,
+and a stable streaming chat API. Operators install it once
+(<https://ollama.com>), `ollama pull qwen2.5:7b`, and they're serving.
+
+The `mojo` backend is kept for the eventual case where a custom Mojo+MAX
+binary is worth squeezing extra perf out of — see
+[`engine/mojo/`](../../engine/mojo) for the scaffold.
+
+## Usage
+
+```js
+import { createEngine } from "@infernetprotocol/engine";
+
+const engine = await createEngine();   // auto-selects backend
+const { id, stream, cancel } = engine.generate({
+    messages: [{ role: "user", content: "hi" }],
+    model: "qwen2.5:7b"
+});
+
+for await (const ev of stream) {
+    if (ev.type === "token") process.stdout.write(ev.text);
+    if (ev.type === "done")  console.log("\nfinished:", ev.reason);
+    if (ev.type === "error") throw new Error(ev.message);
+}
+
+await engine.shutdown();
+```
+
+## Auto-selection precedence
+
+1. `INFERNET_ENGINE_BACKEND` env var (explicit override: `ollama`, `mojo`, `stub`)
+2. `INFERNET_ENGINE_BIN` set → `mojo`
+3. Ollama reachable on `OLLAMA_HOST` (default `http://localhost:11434`) → `ollama`
+4. → `stub`
+
+## Operator setup (typical)
+
+```bash
+# 1. Install Ollama (one line per platform — see ollama.com).
+curl -fsSL https://ollama.com/install.sh | sh
+
+# 2. Pull whichever model your node will serve.
+ollama pull qwen2.5:7b      # 4.4 GB — fits a 6+ GB GPU
+ollama pull qwen2.5:0.5b    # 400 MB — useful for CPU smoke tests
+
+# 3. Tell the daemon which model this node serves.
+INFERNET_ENGINE_MODEL=qwen2.5:7b infernet start
+```
+
+Pick whichever model fits your VRAM. Qwen 2.5 family is a reasonable
+default; the engine is model-agnostic.
+
+## Wire protocol — `mojo` backend
+
+The Mojo backend communicates with its binary over NDJSON v1 stdio.
+Source of truth: [`src/protocol.js`](./src/protocol.js); mirrored on the
+Mojo side in [`engine/mojo/src/main.mojo`](../../engine/mojo/src/main.mojo).
+Bumping `PROTOCOL_VERSION` is a breaking change — both sides must move
+together.
+
+The Ollama backend talks HTTP to the operator's Ollama daemon, so the v1
+protocol is purely an internal-IPC concern there.
+
+## Testing
+
+`pnpm vitest run tests/engine.test.js` — 16 tests covering:
+- Protocol codec round-trip + version mismatch handling
+- NDJSON splitter across chunk boundaries
+- `EngineProcess` against `test/fake-engine.js` (Node script speaking the
+  protocol — no Mojo toolchain required)
+- Ollama backend against an in-process fake HTTP server (no real Ollama
+  required) — streaming, cancellation, HTTP errors, missing model
+- `createEngine` auto-selection

package/package.json

@@ -0,0 +1,26 @@
+{
+  "name": "@infernetprotocol/engine",
+  "version": "0.1.22",
+  "description": "Inference engine abstraction for Infernet — pluggable backends (Mojo+MAX, in-process stub) over a stable async-iterator API.",
+  "license": "MIT",
+  "type": "module",
+  "main": "./src/index.js",
+  "exports": {
+    ".": "./src/index.js",
+    "./protocol": "./src/protocol.js",
+    "./engine-process": "./src/engine-process.js"
+  },
+  "files": [
+    "src/**/*.js",
+    "README.md"
+  ],
+  "engines": {
+    "node": ">=18"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/profullstack/infernet-protocol.git",
+    "directory": "packages/engine"
+  },
+  "homepage": "https://github.com/profullstack/infernet-protocol#readme"
+}

package/src/async-queue.js

@@ -0,0 +1,67 @@
+/**
+ * Single-producer / single-consumer async queue used to surface engine
+ * events as an async-iterable to backend callers.
+ *
+ * Shared by all backends (stub, ollama, mojo via EngineProcess) so the
+ * iteration semantics are identical regardless of whether events arrive
+ * from a child process, an HTTP stream, or in-process timers.
+ *
+ *   const q = new AsyncQueue();
+ *   q.push(value); q.end(); q.fail(err);
+ *   for await (const v of q) { ... }
+ */
+
+export class AsyncQueue {
+    constructor() {
+        this.values = [];
+        this.waiters = [];
+        this.closed = false;
+        this.error = null;
+    }
+
+    push(v) {
+        if (this.closed) return;
+        const w = this.waiters.shift();
+        if (w) w({ value: v, done: false });
+        else this.values.push(v);
+    }
+
+    fail(err) {
+        if (this.closed) return;
+        this.error = err;
+        this.closed = true;
+        while (this.waiters.length) {
+            const w = this.waiters.shift();
+            w({ value: undefined, done: true, _err: err });
+        }
+    }
+
+    end() {
+        if (this.closed) return;
+        this.closed = true;
+        while (this.waiters.length) {
+            const w = this.waiters.shift();
+            w({ value: undefined, done: true });
+        }
+    }
+
+    next() {
+        if (this.values.length) {
+            return Promise.resolve({ value: this.values.shift(), done: false });
+        }
+        if (this.closed) {
+            if (this.error) return Promise.reject(this.error);
+            return Promise.resolve({ value: undefined, done: true });
+        }
+        return new Promise((resolve, reject) => {
+            this.waiters.push((res) => {
+                if (res._err) reject(res._err);
+                else resolve(res);
+            });
+        });
+    }
+
+    [Symbol.asyncIterator]() {
+        return this;
+    }
+}

package/src/backends/llamacpp.js

@@ -0,0 +1,234 @@
+/**
+ * llama.cpp / llama-swap backend.
+ *
+ * llama.cpp's `llama-server` exposes OpenAI-compatible chat-completions
+ * (default port 8080). llama-swap is a thin proxy that routes per-model
+ * to underlying llama.cpp instances and hot-swaps weights without
+ * keeping every model resident — same wire protocol either way.
+ *
+ * Why this exists alongside Ollama and vLLM:
+ *   - lighter than Ollama for Apple Silicon / ARM / pure-CPU operators
+ *   - no Python dependency like vLLM (single static C++ binary)
+ *   - supports gguf quantizations Ollama doesn't always ship
+ *   - llama-swap's model-swap behavior is great for nodes serving
+ *     many small models on commodity hardware
+ *
+ * The adapter speaks OpenAI's /v1/chat/completions SSE — same logic
+ * as vllm.js, different defaults.
+ */
+
+import { randomUUID } from "node:crypto";
+import { AsyncQueue } from "../async-queue.js";
+import { MSG, PROTOCOL_VERSION } from "../protocol.js";
+
+const DEFAULT_HOST = "http://localhost:8080";
+const PROBE_TIMEOUT_MS = 500;
+
+export const LLAMACPP_DEFAULT_HOST = DEFAULT_HOST;
+
+export async function isLlamacppReachable(host = DEFAULT_HOST, timeoutMs = PROBE_TIMEOUT_MS) {
+    try {
+        const ctrl = new AbortController();
+        const t = setTimeout(() => ctrl.abort(), timeoutMs);
+        // /health is llama.cpp; /v1/models works for both llama-server and
+        // llama-swap. Try /v1/models first (broadest compatibility).
+        const res = await fetch(new URL("/v1/models", host), { signal: ctrl.signal });
+        clearTimeout(t);
+        return res.ok;
+    } catch {
+        return false;
+    }
+}
+
+export async function createLlamacppBackend({
+    host = process.env.LLAMACPP_HOST ?? DEFAULT_HOST,
+    apiKey = process.env.LLAMACPP_API_KEY ?? null,
+    defaultModel = process.env.LLAMACPP_MODEL ?? process.env.INFERNET_ENGINE_MODEL ?? null,
+    skipProbe = false,
+    fetchImpl = globalThis.fetch
+} = {}) {
+    if (!skipProbe) {
+        const ok = await isLlamacppReachable(host);
+        if (!ok) {
+            throw new Error(
+                `llama.cpp not reachable at ${host} — start with 'llama-server' or 'llama-swap', or set LLAMACPP_HOST`
+            );
+        }
+    }
+
+    return {
+        kind: "llamacpp",
+        host,
+        hasApiKey: Boolean(apiKey),
+        generate({ messages, id, model = null, max_tokens, temperature } = {}) {
+            const genId = id ?? randomUUID();
+            const stream = new AsyncQueue();
+            const ctrl = new AbortController();
+            const resolvedModel = model ?? defaultModel;
+
+            (async () => {
+                if (!resolvedModel) {
+                    stream.push({
+                        v: PROTOCOL_VERSION,
+                        type: MSG.ERROR,
+                        id: genId,
+                        message:
+                            "llamacpp backend: no model — set LLAMACPP_MODEL or pass model in the job"
+                    });
+                    stream.end();
+                    return;
+                }
+
+                const body = {
+                    model: resolvedModel,
+                    messages: Array.isArray(messages) ? messages : [],
+                    stream: true
+                };
+                if (typeof temperature === "number") body.temperature = temperature;
+                if (typeof max_tokens === "number") body.max_tokens = max_tokens;
+
+                const headers = { "content-type": "application/json" };
+                if (apiKey) headers["authorization"] = `Bearer ${apiKey}`;
+
+                let res;
+                try {
+                    res = await fetchImpl(new URL("/v1/chat/completions", host), {
+                        method: "POST",
+                        headers,
+                        body: JSON.stringify(body),
+                        signal: ctrl.signal
+                    });
+                } catch (err) {
+                    pushTerminal(stream, genId, ctrl, err, "");
+                    return;
+                }
+
+                if (!res.ok) {
+                    let detail = "";
+                    try { detail = await res.text(); } catch { /* best-effort */ }
+                    stream.push({
+                        v: PROTOCOL_VERSION,
+                        type: MSG.ERROR,
+                        id: genId,
+                        message: `llamacpp HTTP ${res.status}: ${detail.slice(0, 200)}`
+                    });
+                    stream.end();
+                    return;
+                }
+
+                let metaSent = false;
+                let acc = "";
+                let buf = "";
+                let finishReason = null;
+                const decoder = new TextDecoder();
+
+                try {
+                    for await (const chunk of res.body) {
+                        buf += decoder.decode(chunk, { stream: true });
+                        let nl;
+                        while ((nl = buf.indexOf("\n")) !== -1) {
+                            const rawLine = buf.slice(0, nl);
+                            buf = buf.slice(nl + 1);
+                            const line = rawLine.trim();
+                            if (!line || !line.startsWith("data:")) continue;
+                            const payload = line.slice("data:".length).trim();
+                            if (payload === "[DONE]") {
+                                stream.push({
+                                    v: PROTOCOL_VERSION,
+                                    type: MSG.DONE,
+                                    id: genId,
+                                    reason: finishReason ?? "stop",
+                                    text: acc,
+                                    finished_at: new Date().toISOString()
+                                });
+                                stream.end();
+                                return;
+                            }
+                            let parsed;
+                            try {
+                                parsed = JSON.parse(payload);
+                            } catch {
+                                continue;
+                            }
+
+                            if (!metaSent) {
+                                stream.push({
+                                    v: PROTOCOL_VERSION,
+                                    type: MSG.META,
+                                    id: genId,
+                                    model: parsed.model ?? resolvedModel,
+                                    started_at: new Date().toISOString()
+                                });
+                                metaSent = true;
+                            }
+
+                            const choice = Array.isArray(parsed.choices) ? parsed.choices[0] : null;
+                            const text = choice?.delta?.content ?? "";
+                            if (text) {
+                                acc += text;
+                                stream.push({
+                                    v: PROTOCOL_VERSION,
+                                    type: MSG.TOKEN,
+                                    id: genId,
+                                    text
+                                });
+                            }
+                            if (choice?.finish_reason) {
+                                finishReason = choice.finish_reason;
+                            }
+                        }
+                    }
+                    stream.push({
+                        v: PROTOCOL_VERSION,
+                        type: MSG.DONE,
+                        id: genId,
+                        reason: finishReason ?? "stop",
+                        text: acc,
+                        finished_at: new Date().toISOString()
+                    });
+                    stream.end();
+                } catch (err) {
+                    pushTerminal(stream, genId, ctrl, err, acc);
+                }
+            })().catch((err) => {
+                stream.push({
+                    v: PROTOCOL_VERSION,
+                    type: MSG.ERROR,
+                    id: genId,
+                    message: err?.message ?? String(err)
+                });
+                stream.end();
+            });
+
+            return {
+                id: genId,
+                stream,
+                cancel: () => ctrl.abort()
+            };
+        },
+        async shutdown() {
+            // The llama.cpp / llama-swap server is owned by the operator.
+        }
+    };
+}
+
+function pushTerminal(stream, genId, ctrl, err, accSoFar) {
+    if (ctrl.signal.aborted) {
+        stream.push({
+            v: PROTOCOL_VERSION,
+            type: MSG.DONE,
+            id: genId,
+            reason: "cancel",
+            text: accSoFar,
+            finished_at: new Date().toISOString()
+        });
+    } else {
+        stream.push({
+            v: PROTOCOL_VERSION,
+            type: MSG.ERROR,
+            id: genId,
+            message: err?.message ?? String(err)
+        });
+    }
+    stream.end();
+}

package/src/backends/max.js

@@ -0,0 +1,196 @@
+/**
+ * Modular MAX backend — talks to a local MAX serve instance over the
+ * OpenAI-compatible chat completions endpoint.
+ *
+ * MAX (Modular Accelerated Xecution) is Modular's high-performance inference
+ * engine. It benchmarks slightly ahead of vLLM on throughput (PagedAttention
+ * equivalent, speculative decoding on by default). NVIDIA + CPU targets.
+ *
+ * Start: max serve --model-path <huggingface-id> [--port 8000]
+ *
+ * Wire protocol is OpenAI SSE — identical to vLLM. Default port is 8000
+ * (same as vLLM), so MODULAR_MAX_HOST must be set explicitly if both run
+ * on the same box (unusual). Auto-selection probes MAX_HOST first in the
+ * precedence chain.
+ */
+
+import { randomUUID } from "node:crypto";
+import { AsyncQueue } from "../async-queue.js";
+import { MSG, PROTOCOL_VERSION } from "../protocol.js";
+
+const DEFAULT_HOST = "http://localhost:8000";
+const PROBE_TIMEOUT_MS = 500;
+
+export const MAX_DEFAULT_HOST = DEFAULT_HOST;
+
+export async function isMaxReachable(host = DEFAULT_HOST, timeoutMs = PROBE_TIMEOUT_MS) {
+    try {
+        const ctrl = new AbortController();
+        const t = setTimeout(() => ctrl.abort(), timeoutMs);
+        // MAX exposes /v1/models just like vLLM. We distinguish at selection
+        // time via MODULAR_MAX_HOST — if the env var is set, we assume MAX.
+        const res = await fetch(new URL("/v1/models", host), { signal: ctrl.signal });
+        clearTimeout(t);
+        return res.ok;
+    } catch {
+        return false;
+    }
+}
+
+export async function createMaxBackend({
+    host = process.env.MODULAR_MAX_HOST ?? DEFAULT_HOST,
+    apiKey = process.env.MAX_API_KEY ?? null,
+    defaultModel = process.env.MAX_MODEL ?? process.env.INFERNET_ENGINE_MODEL ?? null,
+    skipProbe = false,
+    fetchImpl = globalThis.fetch
+} = {}) {
+    if (!skipProbe) {
+        const ok = await isMaxReachable(host);
+        if (!ok) {
+            throw new Error(
+                `MAX not reachable at ${host} — start with 'max serve --model-path <model>' or set MODULAR_MAX_HOST`
+            );
+        }
+    }
+
+    return {
+        kind: "max",
+        host,
+        hasApiKey: Boolean(apiKey),
+        generate({ messages, id, model = null, max_tokens, temperature } = {}) {
+            const genId = id ?? randomUUID();
+            const stream = new AsyncQueue();
+            const ctrl = new AbortController();
+            const resolvedModel = model ?? defaultModel;
+
+            (async () => {
+                if (!resolvedModel) {
+                    stream.push({
+                        v: PROTOCOL_VERSION,
+                        type: MSG.ERROR,
+                        id: genId,
+                        message: "max backend: no model — set MAX_MODEL or pass model in the job"
+                    });
+                    stream.end();
+                    return;
+                }
+
+                const body = {
+                    model: resolvedModel,
+                    messages: Array.isArray(messages) ? messages : [],
+                    stream: true
+                };
+                if (typeof temperature === "number") body.temperature = temperature;
+                if (typeof max_tokens === "number") body.max_tokens = max_tokens;
+
+                const headers = { "content-type": "application/json" };
+                if (apiKey) headers["authorization"] = `Bearer ${apiKey}`;
+
+                let res;
+                try {
+                    res = await fetchImpl(new URL("/v1/chat/completions", host), {
+                        method: "POST",
+                        headers,
+                        body: JSON.stringify(body),
+                        signal: ctrl.signal
+                    });
+                } catch (err) {
+                    pushTerminal(stream, genId, ctrl, err, "");
+                    return;
+                }
+
+                if (!res.ok) {
+                    let detail = "";
+                    try { detail = await res.text(); } catch { /* best-effort */ }
+                    stream.push({
+                        v: PROTOCOL_VERSION,
+                        type: MSG.ERROR,
+                        id: genId,
+                        message: `max HTTP ${res.status}: ${detail.slice(0, 200)}`
+                    });
+                    stream.end();
+                    return;
+                }
+
+                let metaSent = false;
+                let acc = "";
+                let buf = "";
+                let finishReason = null;
+                const decoder = new TextDecoder();
+
+                try {
+                    for await (const chunk of res.body) {
+                        buf += decoder.decode(chunk, { stream: true });
+                        let nl;
+                        while ((nl = buf.indexOf("\n")) !== -1) {
+                            const rawLine = buf.slice(0, nl);
+                            buf = buf.slice(nl + 1);
+                            const line = rawLine.trim();
+                            if (!line || !line.startsWith("data:")) continue;
+                            const payload = line.slice("data:".length).trim();
+                            if (payload === "[DONE]") {
+                                stream.push({
+                                    v: PROTOCOL_VERSION,
+                                    type: MSG.DONE,
+                                    id: genId,
+                                    reason: finishReason ?? "stop",
+                                    text: acc,
+                                    finished_at: new Date().toISOString()
+                                });
+                                stream.end();
+                                return;
+                            }
+                            let parsed;
+                            try { parsed = JSON.parse(payload); } catch { continue; }
+
+                            if (!metaSent) {
+                                stream.push({
+                                    v: PROTOCOL_VERSION,
+                                    type: MSG.META,
+                                    id: genId,
+                                    model: parsed.model ?? resolvedModel,
+                                    started_at: new Date().toISOString()
+                                });
+                                metaSent = true;
+                            }
+
+                            const choice = Array.isArray(parsed.choices) ? parsed.choices[0] : null;
+                            const text = choice?.delta?.content ?? "";
+                            if (text) {
+                                acc += text;
+                                stream.push({ v: PROTOCOL_VERSION, type: MSG.TOKEN, id: genId, text });
+                            }
+                            if (choice?.finish_reason) finishReason = choice.finish_reason;
+                        }
+                    }
+                    stream.push({
+                        v: PROTOCOL_VERSION,
+                        type: MSG.DONE,
+                        id: genId,
+                        reason: finishReason ?? "stop",
+                        text: acc,
+                        finished_at: new Date().toISOString()
+                    });
+                    stream.end();
+                } catch (err) {
+                    pushTerminal(stream, genId, ctrl, err, acc);
+                }
+            })().catch((err) => {
+                stream.push({ v: PROTOCOL_VERSION, type: MSG.ERROR, id: genId, message: err?.message ?? String(err) });
+                stream.end();
+            });
+
+            return { id: genId, stream, cancel: () => ctrl.abort() };
+        },
+        async shutdown() {}
+    };
+}
+
+function pushTerminal(stream, genId, ctrl, err, accSoFar) {
+    if (ctrl.signal.aborted) {
+        stream.push({ v: PROTOCOL_VERSION, type: MSG.DONE, id: genId, reason: "cancel", text: accSoFar, finished_at: new Date().toISOString() });
+    } else {
+        stream.push({ v: PROTOCOL_VERSION, type: MSG.ERROR, id: genId, message: err?.message ?? String(err) });
+    }
+    stream.end();
+}

package/src/backends/mojo.js

@@ -0,0 +1,33 @@
+/**
+ * Mojo backend — drives an external Mojo+MAX engine binary over the v1
+ * NDJSON protocol. The binary is built from `engine/mojo/` (sibling to
+ * `supabase/`); see that directory's README for build instructions.
+ */
+
+import { EngineProcess } from "../engine-process.js";
+import { resolveBinary } from "../resolve-binary.js";
+
+export async function createMojoBackend({
+    binary,
+    model = null,
+    args = [],
+    env = {}
+} = {}) {
+    const bin =
+        binary ?? resolveBinary({ envVar: "INFERNET_ENGINE_BIN", name: "infernet-engine" });
+    if (!bin) {
+        throw new Error(
+            "no Mojo engine binary found — set INFERNET_ENGINE_BIN or place `infernet-engine` on PATH"
+        );
+    }
+
+    const proc = new EngineProcess({ binary: bin, args, env, model });
+    await proc.start();
+
+    return {
+        kind: "mojo",
+        binary: bin,
+        generate: (req) => proc.generate(req),
+        shutdown: (opts) => proc.shutdown(opts)
+    };
+}