@infernetprotocol/rpc-adapter 0.1.54

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/LICENSE ADDED
@@ -0,0 +1,15 @@
1
+ ISC License
2
+
3
+ Copyright (c) 2025 Infernet Protocol
4
+
5
+ Permission to use, copy, modify, and/or distribute this software for any
6
+ purpose with or without fee is hereby granted, provided that the above
7
+ copyright notice and this permission notice appear in all copies.
8
+
9
+ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
10
+ REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
11
+ AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
12
+ INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
13
+ LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
14
+ OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
15
+ PERFORMANCE OF THIS SOFTWARE.
package/package.json ADDED
@@ -0,0 +1,28 @@
1
+ {
2
+ "name": "@infernetprotocol/rpc-adapter",
3
+ "version": "0.1.54",
4
+ "description": "llama.cpp RPC adapter — spawn llama-server with --rpc peer list, parse layer assignments, stream chat tokens (IPIP-0033).",
5
+ "license": "MIT",
6
+ "type": "module",
7
+ "main": "./src/index.js",
8
+ "exports": {
9
+ ".": "./src/index.js",
10
+ "./constants": "./src/constants.js",
11
+ "./parser": "./src/stderr-parser.js",
12
+ "./spawn": "./src/spawn.js",
13
+ "./stream": "./src/stream.js"
14
+ },
15
+ "files": [
16
+ "src/**/*.js",
17
+ "README.md"
18
+ ],
19
+ "engines": {
20
+ "node": ">=18"
21
+ },
22
+ "repository": {
23
+ "type": "git",
24
+ "url": "git+https://github.com/InfernetProtocol/infernet-protocol.git",
25
+ "directory": "packages/rpc-adapter"
26
+ },
27
+ "homepage": "https://github.com/InfernetProtocol/infernet-protocol#readme"
28
+ }
@@ -0,0 +1,17 @@
1
+ /**
2
+ * IPIP-0033 §3 — control-plane policy knobs. The control plane refuses
3
+ * a `distributed: true` request when fewer than MIN_RPC_PEERS slices
4
+ * are available. MAX_RPC_PEERS caps the --rpc list length the primary
5
+ * is handed.
6
+ *
7
+ * Both are starting values. The IPIP §Open-questions calls out that
8
+ * MAX_RPC_PEERS=8 is a guess pending real measurement.
9
+ */
10
+ export const MIN_RPC_PEERS = 2;
11
+ export const MAX_RPC_PEERS = 8;
12
+
13
+ /** §4 — drop a peer the daemon hasn't seen handshake for in this long. */
14
+ export const RPC_PEER_FRESHNESS_MS = 5 * 60 * 1000;
15
+
16
+ /** Backend identity surfaced on every routing event (§5). */
17
+ export const ENGINE_ID = 'llama.cpp';
package/src/index.js ADDED
@@ -0,0 +1,12 @@
1
+ /**
2
+ * @infernetprotocol/rpc-adapter — llama.cpp RPC adapter.
3
+ *
4
+ * IPIP-0033 Phase 2: spawn `llama-server` with an --rpc peer list,
5
+ * parse its stderr for layer assignments, and stream tokens back to
6
+ * the daemon's HTTP layer in the same shape every other Infernet
7
+ * adapter uses.
8
+ */
9
+ export { MIN_RPC_PEERS, MAX_RPC_PEERS, RPC_PEER_FRESHNESS_MS, ENGINE_ID } from './constants.js';
10
+ export { parseLlamaStderrLine, aggregateLayerAssignments } from './stderr-parser.js';
11
+ export { spawnLlamaServer } from './spawn.js';
12
+ export { streamChatCompletion } from './stream.js';
package/src/spawn.js ADDED
@@ -0,0 +1,196 @@
1
+ /**
2
+ * Spawn `llama-server` with an --rpc peer list (IPIP-0033 §4).
3
+ *
4
+ * Returns a handle the caller can:
5
+ * - `await ready` to wait until the HTTP server is listening
6
+ * - iterate `events()` for parsed stderr (layer assignments,
7
+ * peer-failed signals, etc.)
8
+ * - `kill()` to terminate the child cleanly
9
+ *
10
+ * The shape mirrors `nim-adapter` in spirit — a thin wrapper, no
11
+ * hidden state, easy to mock by injecting `spawnFn` in tests.
12
+ */
13
+ import { EventEmitter } from 'node:events';
14
+ import { parseLlamaStderrLine } from './stderr-parser.js';
15
+ import { MAX_RPC_PEERS } from './constants.js';
16
+
17
+ const READY_TIMEOUT_MS = 60_000;
18
+
19
+ /**
20
+ * @param {Object} args
21
+ * @param {string} args.modelPath Absolute path to a GGUF.
22
+ * @param {Array<{host:string, port:number}>} args.rpcPeers
23
+ * Up to MAX_RPC_PEERS slice servers to connect to. Capped silently
24
+ * when over the limit — caller should check length beforehand.
25
+ * @param {string} [args.host='127.0.0.1'] Bind interface for the HTTP
26
+ * server. Daemons typically use 127.0.0.1 and proxy through the
27
+ * existing daemon HTTP listener.
28
+ * @param {number} [args.port=0] 0 = let the OS choose.
29
+ * @param {number} [args.gpuLayers] --gpu-layers value. When
30
+ * omitted, llama-server picks; when set, the primary keeps that
31
+ * many on local GPU and farms the rest to RPC peers.
32
+ * @param {string} [args.binary='llama-server']
33
+ * @param {string[]} [args.extraArgs] Pass-through CLI args.
34
+ * @param {(cmd:string,args:string[],opts?:object)=>any} [args.spawnFn]
35
+ * Test seam — defaults to node:child_process.spawn.
36
+ * @returns {Promise<RpcServerHandle>}
37
+ */
38
+ export async function spawnLlamaServer(args) {
39
+ const {
40
+ modelPath,
41
+ rpcPeers,
42
+ host = '127.0.0.1',
43
+ port = 0,
44
+ gpuLayers,
45
+ binary = 'llama-server',
46
+ extraArgs = [],
47
+ spawnFn
48
+ } = args ?? {};
49
+
50
+ if (typeof modelPath !== 'string' || !modelPath) {
51
+ throw new Error('spawnLlamaServer: modelPath is required');
52
+ }
53
+ if (!Array.isArray(rpcPeers)) {
54
+ throw new Error('spawnLlamaServer: rpcPeers must be an array');
55
+ }
56
+
57
+ const cappedPeers = rpcPeers.slice(0, MAX_RPC_PEERS);
58
+ const rpcArg = cappedPeers
59
+ .filter((p) => p && typeof p.host === 'string' && Number.isFinite(p.port))
60
+ .map((p) => `${p.host}:${p.port}`)
61
+ .join(',');
62
+
63
+ const childArgs = [
64
+ '--model', modelPath,
65
+ '--host', host,
66
+ '--port', String(port)
67
+ ];
68
+ if (rpcArg) childArgs.push('--rpc', rpcArg);
69
+ if (Number.isFinite(gpuLayers)) childArgs.push('--gpu-layers', String(gpuLayers));
70
+ childArgs.push(...extraArgs);
71
+
72
+ const spawn = spawnFn ?? (await import('node:child_process')).spawn;
73
+ const child = spawn(binary, childArgs, { stdio: ['ignore', 'pipe', 'pipe'] });
74
+
75
+ return new RpcServerHandle({ child, requestedPort: port });
76
+ }
77
+
78
+ class RpcServerHandle extends EventEmitter {
79
+ constructor({ child, requestedPort }) {
80
+ super();
81
+ this._child = child;
82
+ this._buffered = '';
83
+ this._eventsQueue = [];
84
+ this._waiters = [];
85
+ this._closed = false;
86
+ this._serverReady = false;
87
+ this._serverInfo = null;
88
+ this._exited = null;
89
+
90
+ this._readyResolve = null;
91
+ this._readyReject = null;
92
+ this.ready = new Promise((resolve, reject) => {
93
+ this._readyResolve = resolve;
94
+ this._readyReject = reject;
95
+ });
96
+ // Attach a no-op catch so a caller that never awaits `ready`
97
+ // (e.g. tests that just want to inspect spawn args) doesn't
98
+ // leak an unhandled rejection when the child exits or kills.
99
+ // Real callers that DO await ready still see the rejection.
100
+ this.ready.catch(() => { /* swallow */ });
101
+
102
+ const readyTimeout = setTimeout(() => {
103
+ if (!this._serverReady) {
104
+ this._readyReject?.(new Error(`llama-server did not become ready within ${READY_TIMEOUT_MS}ms`));
105
+ }
106
+ }, READY_TIMEOUT_MS);
107
+ if (typeof readyTimeout.unref === 'function') readyTimeout.unref();
108
+
109
+ const onStderr = (chunk) => {
110
+ this._buffered += typeof chunk === 'string' ? chunk : chunk.toString('utf8');
111
+ let nl;
112
+ while ((nl = this._buffered.indexOf('\n')) >= 0) {
113
+ const line = this._buffered.slice(0, nl);
114
+ this._buffered = this._buffered.slice(nl + 1);
115
+ this._handleLine(line);
116
+ }
117
+ };
118
+ child.stderr?.on?.('data', onStderr);
119
+ child.stdout?.on?.('data', onStderr); // some llama.cpp builds log to stdout
120
+
121
+ child.on?.('exit', (code, signal) => {
122
+ this._exited = { code, signal };
123
+ this._closed = true;
124
+ clearTimeout(readyTimeout);
125
+ if (!this._serverReady) {
126
+ this._readyReject?.(new Error(`llama-server exited before ready (code=${code} signal=${signal})`));
127
+ }
128
+ this._drainQueue();
129
+ this._waiters.forEach((w) => w({ done: true, value: undefined }));
130
+ this._waiters.length = 0;
131
+ this.emit('exit', this._exited);
132
+ });
133
+ }
134
+
135
+ /**
136
+ * Async iterable of parsed stderr events — { type, ... } objects
137
+ * matching parseLlamaStderrLine's shape, plus a synthetic
138
+ * 'log' event with `text` for unparseable lines so the daemon can
139
+ * relay them as IPIP-0033 §UX log frames.
140
+ */
141
+ events() {
142
+ const self = this;
143
+ return {
144
+ [Symbol.asyncIterator]() {
145
+ return {
146
+ next() {
147
+ if (self._eventsQueue.length > 0) {
148
+ return Promise.resolve({ value: self._eventsQueue.shift(), done: false });
149
+ }
150
+ if (self._closed) {
151
+ return Promise.resolve({ value: undefined, done: true });
152
+ }
153
+ return new Promise((resolve) => self._waiters.push(resolve));
154
+ }
155
+ };
156
+ }
157
+ };
158
+ }
159
+
160
+ get serverInfo() { return this._serverInfo; }
161
+ get exited() { return this._exited; }
162
+
163
+ kill(signal = 'SIGTERM') {
164
+ if (this._closed || !this._child) return;
165
+ try { this._child.kill(signal); } catch { /* ignore */ }
166
+ }
167
+
168
+ _handleLine(rawLine) {
169
+ const text = rawLine.replace(/\r$/, '');
170
+ const parsed = parseLlamaStderrLine(text);
171
+ const ev = parsed ?? { type: 'log', text };
172
+
173
+ if (ev.type === 'server_ready' && !this._serverReady) {
174
+ this._serverReady = true;
175
+ this._serverInfo = { host: ev.host, port: ev.port };
176
+ this._readyResolve?.({ host: ev.host, port: ev.port });
177
+ }
178
+
179
+ this._enqueue(ev);
180
+ }
181
+
182
+ _enqueue(ev) {
183
+ const waiter = this._waiters.shift();
184
+ if (waiter) {
185
+ waiter({ value: ev, done: false });
186
+ } else {
187
+ this._eventsQueue.push(ev);
188
+ }
189
+ this.emit(ev.type, ev);
190
+ }
191
+
192
+ _drainQueue() {
193
+ // After the child exits, any remaining queued events are still
194
+ // delivered before the iterator reports done.
195
+ }
196
+ }
@@ -0,0 +1,222 @@
1
+ /**
2
+ * llama.cpp stderr parser — extracts the structured signals we need
3
+ * to fill IPIP-0033 §5's `routing` event. llama-server logs in plain
4
+ * text and the format drifts between releases, so this parser is
5
+ * intentionally pattern-matching, not strict.
6
+ *
7
+ * Returns `null` for lines we don't understand — the caller (the
8
+ * daemon) MAY still relay them as `log` SSE frames so operators can
9
+ * see the raw output.
10
+ *
11
+ * Recognized event types:
12
+ *
13
+ * { type: 'layer_assigned', layer, peer }
14
+ * `peer` is { kind: 'rpc', host, port } for an RPC slice
15
+ * and { kind: 'local', device } for a local device.
16
+ *
17
+ * { type: 'peer_connected', host, port }
18
+ * { type: 'peer_failed', host, port, reason }
19
+ * { type: 'server_ready', host, port }
20
+ * { type: 'load_progress', percent }
21
+ */
22
+
23
+ const RPC_TARGET_RE = /\bRPC\[\s*([^:\]\s]+)\s*:\s*(\d+)\s*\]/;
24
+ const LOCAL_TARGET_RE = /(?:assigned to|offloaded to|on)\s+(?:device\s+)?(CUDA\d+|CPU|Metal|Vulkan\d*|ROCm\d*)/i;
25
+ const LAYER_INDEX_RE = /\blayer\s*[#:]?\s*(\d+)\b/i;
26
+
27
+ const PEER_CONNECTED_RES = [
28
+ /\bconnected to (?:rpc[- ]?server\s+)?([^:\s]+):(\d+)/i,
29
+ /\brpc[- ]?server\s+([^:\s]+):(\d+)\s+is up/i
30
+ ];
31
+
32
+ const PEER_FAILED_RES = [
33
+ /\b(?:failed to connect to|rpc backend connect failed for)\s+([^:\s]+):(\d+)(?:\s*[:\-]\s*(.*))?/i,
34
+ /\brpc[- ]?(?:server\s+)?([^:\s]+):(\d+)\s+(?:disconnected|dropped|lost)\s*[:\-]?\s*(.*)?/i
35
+ ];
36
+
37
+ const SERVER_READY_RES = [
38
+ // Modern llama-server form
39
+ /\bserver is listening on https?:\/\/([^:/\s]+):(\d+)/i,
40
+ // Older form
41
+ /\bHTTP server listening at https?:\/\/([^:/\s]+):(\d+)/i,
42
+ /\blistening on \[?([0-9a-fA-F:.]+)\]?:(\d+)/i
43
+ ];
44
+
45
+ const LOAD_PROGRESS_RE = /\bload(?:ing)?[: ]\s*(\d+(?:\.\d+)?)\s*%/i;
46
+
47
+ /**
48
+ * Parse a single stderr line into a structured event. Returns null
49
+ * when the line doesn't match any known signal.
50
+ *
51
+ * @param {string} line
52
+ * @returns {object | null}
53
+ */
54
+ export function parseLlamaStderrLine(line) {
55
+ if (typeof line !== 'string') return null;
56
+ const trimmed = line.trim();
57
+ if (!trimmed) return null;
58
+
59
+ // Layer assignment is the load-bearing signal (used by IPIP-0033
60
+ // §5 routing events). Check for it first so a noisier subsequent
61
+ // pattern doesn't preempt it.
62
+ const layerMatch = trimmed.match(LAYER_INDEX_RE);
63
+ if (layerMatch) {
64
+ const layer = Number.parseInt(layerMatch[1], 10);
65
+ if (Number.isFinite(layer)) {
66
+ const rpc = trimmed.match(RPC_TARGET_RE);
67
+ if (rpc) {
68
+ return {
69
+ type: 'layer_assigned',
70
+ layer,
71
+ peer: { kind: 'rpc', host: rpc[1], port: Number.parseInt(rpc[2], 10) }
72
+ };
73
+ }
74
+ const local = trimmed.match(LOCAL_TARGET_RE);
75
+ if (local) {
76
+ return {
77
+ type: 'layer_assigned',
78
+ layer,
79
+ peer: { kind: 'local', device: local[1] }
80
+ };
81
+ }
82
+ }
83
+ }
84
+
85
+ for (const re of PEER_FAILED_RES) {
86
+ const m = trimmed.match(re);
87
+ if (m) {
88
+ return {
89
+ type: 'peer_failed',
90
+ host: m[1],
91
+ port: Number.parseInt(m[2], 10),
92
+ reason: (m[3] ?? '').trim() || null
93
+ };
94
+ }
95
+ }
96
+
97
+ for (const re of PEER_CONNECTED_RES) {
98
+ const m = trimmed.match(re);
99
+ if (m) {
100
+ return {
101
+ type: 'peer_connected',
102
+ host: m[1],
103
+ port: Number.parseInt(m[2], 10)
104
+ };
105
+ }
106
+ }
107
+
108
+ for (const re of SERVER_READY_RES) {
109
+ const m = trimmed.match(re);
110
+ if (m) {
111
+ return {
112
+ type: 'server_ready',
113
+ host: m[1],
114
+ port: Number.parseInt(m[2], 10)
115
+ };
116
+ }
117
+ }
118
+
119
+ const prog = trimmed.match(LOAD_PROGRESS_RE);
120
+ if (prog) {
121
+ return { type: 'load_progress', percent: Number.parseFloat(prog[1]) };
122
+ }
123
+
124
+ return null;
125
+ }
126
+
127
+ /**
128
+ * Roll up a stream of stderr events into per-peer layer ranges +
129
+ * lifecycle status. Drops `local` device assignments — the routing
130
+ * event is the remote-RPC peer breakdown that the chat UI surfaces.
131
+ * Gaps inside a single peer's range are tolerated (the IPIP routing
132
+ * shape carries a single { start, end }).
133
+ *
134
+ * Status comes from the most recent lifecycle event for that
135
+ * (host, port):
136
+ *
137
+ * peer_connected → 'ok'
138
+ * layer_assigned → 'ok' (implicitly — assignment means the slice is in use)
139
+ * peer_failed → 'dropped' (with `reason` carried through)
140
+ *
141
+ * A peer that flaps (failed → connected → failed) ends in whichever
142
+ * state the most recent event suggests.
143
+ *
144
+ * Returns an array shape compatible with IPIP-0033 §5:
145
+ *
146
+ * [{ host, port, layers: { start, end }, count,
147
+ * status: 'ok' | 'dropped', reason? }, ...]
148
+ *
149
+ * Sorted by `layers.start` ascending.
150
+ */
151
+ export function aggregateLayerAssignments(events) {
152
+ const byPeer = new Map(); // "host:port" → { host, port, start, end, count, status, reason }
153
+
154
+ for (const ev of events ?? []) {
155
+ let key;
156
+ let host;
157
+ let port;
158
+ if (ev?.type === 'layer_assigned' && ev.peer?.kind === 'rpc') {
159
+ host = ev.peer.host;
160
+ port = ev.peer.port;
161
+ } else if (ev?.type === 'peer_failed' || ev?.type === 'peer_connected') {
162
+ host = ev.host;
163
+ port = ev.port;
164
+ } else {
165
+ continue;
166
+ }
167
+ key = `${host}:${port}`;
168
+
169
+ const cur = byPeer.get(key);
170
+ const base = cur ?? {
171
+ host,
172
+ port,
173
+ start: null,
174
+ end: null,
175
+ count: 0,
176
+ status: 'ok',
177
+ reason: null
178
+ };
179
+
180
+ if (ev.type === 'layer_assigned') {
181
+ base.start = base.start === null ? ev.layer : Math.min(base.start, ev.layer);
182
+ base.end = base.end === null ? ev.layer : Math.max(base.end, ev.layer);
183
+ base.count += 1;
184
+ // A layer-assigned event always implies the peer is alive
185
+ // RIGHT NOW; a stale 'dropped' status from earlier is
186
+ // overwritten because llama.cpp wouldn't be assigning
187
+ // layers to a peer it can't reach.
188
+ base.status = 'ok';
189
+ base.reason = null;
190
+ } else if (ev.type === 'peer_failed') {
191
+ base.status = 'dropped';
192
+ if (typeof ev.reason === 'string' && ev.reason.length > 0) {
193
+ base.reason = ev.reason;
194
+ }
195
+ } else if (ev.type === 'peer_connected') {
196
+ base.status = 'ok';
197
+ base.reason = null;
198
+ }
199
+
200
+ byPeer.set(key, base);
201
+ }
202
+
203
+ return [...byPeer.values()]
204
+ // Drop peers we only saw a status event for and never had a
205
+ // layer assigned to — they aren't routing-event peers.
206
+ .filter((p) => p.start !== null)
207
+ .sort((a, b) => a.start - b.start)
208
+ .map((p) => ({
209
+ host: p.host,
210
+ port: p.port,
211
+ layers: { start: p.start, end: p.end },
212
+ count: p.count,
213
+ status: p.status,
214
+ ...(p.reason ? { reason: p.reason } : {})
215
+ }));
216
+ }
217
+
218
+ export const __test__ = {
219
+ RPC_TARGET_RE,
220
+ LOCAL_TARGET_RE,
221
+ LAYER_INDEX_RE
222
+ };
package/src/stream.js ADDED
@@ -0,0 +1,111 @@
1
+ /**
2
+ * Streaming chat helper for llama-server's HTTP API. Mirrors the
3
+ * `streamChatCompletion` shape from `@infernetprotocol/nim-adapter` so
4
+ * the chat-stream proxy can swap one for the other without reshaping
5
+ * events.
6
+ *
7
+ * llama-server speaks two relevant endpoints:
8
+ * - POST /completion (its native streaming format)
9
+ * - POST /v1/chat/completions (OpenAI-compatible)
10
+ *
11
+ * We use the OpenAI-compatible one because it's stable across
12
+ * versions and the rest of Infernet already speaks that shape.
13
+ */
14
+
15
+ /**
16
+ * Yields events:
17
+ * { type: 'meta', data: { engine: 'llama.cpp', model, started_at } }
18
+ * { type: 'token', data: { text } }
19
+ * { type: 'done', data: { text, finished_at } }
20
+ * { type: 'error', data: { message } }
21
+ *
22
+ * @param {{
23
+ * baseUrl: string, // e.g. http://127.0.0.1:8080
24
+ * model: string,
25
+ * messages: Array<{role:string, content:string}>,
26
+ * maxTokens?: number,
27
+ * temperature?: number,
28
+ * signal?: AbortSignal
29
+ * }} opts
30
+ */
31
+ export async function* streamChatCompletion(opts = {}) {
32
+ const { baseUrl, model, messages, maxTokens = 512, temperature = 0.7, signal } = opts;
33
+ if (!baseUrl) throw new Error('rpc-adapter: baseUrl is required');
34
+ if (!Array.isArray(messages) || messages.length === 0) {
35
+ throw new Error('rpc-adapter: messages[] is required');
36
+ }
37
+
38
+ const url = `${baseUrl.replace(/\/+$/, '')}/v1/chat/completions`;
39
+ const startedAt = new Date().toISOString();
40
+ yield { type: 'meta', data: { engine: 'llama.cpp', model, started_at: startedAt } };
41
+
42
+ let res;
43
+ try {
44
+ res = await fetch(url, {
45
+ method: 'POST',
46
+ headers: { 'content-type': 'application/json' },
47
+ body: JSON.stringify({
48
+ model,
49
+ messages,
50
+ stream: true,
51
+ max_tokens: maxTokens,
52
+ temperature
53
+ }),
54
+ signal
55
+ });
56
+ } catch (err) {
57
+ yield { type: 'error', data: { message: `llama-server fetch failed: ${err?.message ?? err}` } };
58
+ return;
59
+ }
60
+
61
+ if (!res.ok || !res.body) {
62
+ yield { type: 'error', data: { message: `llama-server HTTP ${res.status}` } };
63
+ return;
64
+ }
65
+
66
+ const reader = res.body.getReader();
67
+ const decoder = new TextDecoder();
68
+ let buffer = '';
69
+ let fullText = '';
70
+
71
+ try {
72
+ while (true) {
73
+ const { value, done } = await reader.read();
74
+ if (done) break;
75
+ buffer += decoder.decode(value, { stream: true });
76
+ let nl;
77
+ while ((nl = buffer.indexOf('\n')) >= 0) {
78
+ const line = buffer.slice(0, nl).trim();
79
+ buffer = buffer.slice(nl + 1);
80
+ if (!line.startsWith('data:')) continue;
81
+ const payload = line.slice(5).trim();
82
+ if (payload === '[DONE]') {
83
+ yield {
84
+ type: 'done',
85
+ data: { text: fullText, finished_at: new Date().toISOString() }
86
+ };
87
+ return;
88
+ }
89
+ let chunk;
90
+ try { chunk = JSON.parse(payload); } catch { continue; }
91
+ const delta = chunk?.choices?.[0]?.delta?.content;
92
+ if (typeof delta === 'string' && delta.length > 0) {
93
+ fullText += delta;
94
+ yield { type: 'token', data: { text: delta } };
95
+ }
96
+ const finish = chunk?.choices?.[0]?.finish_reason;
97
+ if (finish && finish !== null) {
98
+ yield {
99
+ type: 'done',
100
+ data: { text: fullText, finished_at: new Date().toISOString(), finish_reason: finish }
101
+ };
102
+ return;
103
+ }
104
+ }
105
+ }
106
+ // Stream ended without [DONE] — emit done so callers don't hang.
107
+ yield { type: 'done', data: { text: fullText, finished_at: new Date().toISOString(), incomplete: true } };
108
+ } catch (err) {
109
+ yield { type: 'error', data: { message: err?.message ?? String(err) } };
110
+ }
111
+ }