agentic-server 0.24.0 → 0.25.0

package/README.md

@@ -46,6 +46,10 @@ app.listen(3001, () => {
 | GET | `/v1/providers` | List configured providers |
 | GET | `/healthz` | Health check |
 
+## Streaming
+
+`POST /v1/chat/completions` with `stream: true` is relayed as a `text/event-stream`: the gateway forwards the provider's frames verbatim, so a client receives tokens as they are produced. It adds `stream_options: { include_usage: true }` unless the caller set `stream_options` itself, and meters the usage the final frame carries. Streaming is available for OpenAI-compatible providers; a provider whose stream would need translation (`ollama`, `anthropic`) is rejected `501` rather than answered with a non-streaming body.
+
 ## Identity & tenancy
 
 Requests carry tenant identity via headers: `X-Database-Id` (**required** — requests without it are rejected `400`), `X-Entity-Id`, and `X-Actor-Id`. Routing to a specific provider can be forced with `X-LLM-Provider`.

package/esm/router.js

@@ -6,11 +6,13 @@
  *   - Model-based routing: "anthropic/claude-3.5-sonnet" → anthropic provider
  *   - Header-based routing: X-LLM-Provider: ollama → ollama provider
  *   - Fire-and-forget inference metering via an injected InferenceSink
+ *   - Streaming chat completions relayed as server-sent events
  *   - /v1/usage reporting endpoint for external usage submission
  */
 import { Logger } from '@pgpmjs/logger';
 import { Router } from 'express';
 import { buildProviderHeaders, resolveProvider, resolveUpstreamUrl } from './providers';
+import { relayEventStream, withUsageStreamOptions } from './streaming';
 import { transformChatRequest, transformChatResponse, transformEmbedRequest, transformEmbedResponse } from './transforms';
 const log = new Logger('agentic-server');
 export const createRouter = (options) => {
@@ -36,14 +38,39 @@ export const createRouter = (options) => {
             provider: provider.type,
             model: req.body?.model
         });
+        // Only the OpenAI-compatible wire carries stream frames the gateway can
+        // relay; translating Ollama's or Anthropic's stream formats is separate
+        // work, and answering a stream request with a whole JSON body would break
+        // the client silently.
+        const streaming = req.body?.stream === true;
+        if (streaming && provider.type !== 'openai') {
+            res.status(501).json({
+                error: {
+                    message: `streaming is not supported for provider type '${provider.type}'`
+                }
+            });
+            return;
+        }
         try {
             const upstreamUrl = resolveUpstreamUrl(provider, '/v1/chat/completions');
-            const body = transformChatRequest(provider, req.body || {});
+            const transformed = transformChatRequest(provider, req.body || {});
+            const body = streaming ? withUsageStreamOptions(transformed) : transformed;
             const headers = buildProviderHeaders(provider);
+            const abort = new AbortController();
+            // A client that hangs up mid-stream stops the upstream read; the request
+            // stream's own 'close' fires as soon as the body is consumed, so the
+            // response is what reports the disconnect.
+            if (streaming) {
+                res.on('close', () => {
+                    if (!res.writableEnded)
+                        abort.abort();
+                });
+            }
             const upstream = await fetch(upstreamUrl, {
                 method: 'POST',
                 headers,
-                body: JSON.stringify(body)
+                body: JSON.stringify(body),
+                ...(streaming ? { signal: abort.signal } : {})
             });
             const latencyMs = Number(process.hrtime.bigint() - startTime) / 1e6;
             if (!upstream.ok) {
@@ -67,6 +94,34 @@ export const createRouter = (options) => {
                 });
                 return;
             }
+            if (streaming) {
+                const streamUsage = await relayEventStream(upstream, res);
+                const streamLatencyMs = Number(process.hrtime.bigint() - startTime) / 1e6;
+                log.info('inference complete', {
+                    databaseId,
+                    provider: provider.type,
+                    model: req.body?.model,
+                    promptTokens: streamUsage?.prompt_tokens,
+                    completionTokens: streamUsage?.completion_tokens,
+                    streamed: true
+                });
+                if (sink) {
+                    sink.logInference({
+                        databaseId, entityId, actorId,
+                        model: String(req.body?.model || body.model || ''),
+                        provider: provider.type,
+                        service: 'chat',
+                        operation: 'chat/completions',
+                        inputTokens: streamUsage?.prompt_tokens || 0,
+                        outputTokens: streamUsage?.completion_tokens || 0,
+                        totalTokens: streamUsage?.total_tokens || 0,
+                        latencyMs: streamLatencyMs,
+                        status: 'ok',
+                        ...(streamUsage ? { rawUsage: streamUsage } : {})
+                    });
+                }
+                return;
+            }
             const data = await upstream.json();
             const { body: responseBody, usage } = transformChatResponse(data, provider);
             log.info('inference complete', {
@@ -109,6 +164,12 @@ export const createRouter = (options) => {
                     errorType: err.message
                 });
             }
+            // A stream that failed mid-relay has already sent its status and frames;
+            // the client sees the truncated stream rather than a JSON error.
+            if (res.headersSent) {
+                res.end();
+                return;
+            }
             res.status(502).json({
                 error: { message: 'Failed to reach LLM provider', details: err.message }
             });

package/esm/streaming.js

@@ -0,0 +1,85 @@
+/**
+ * Server-sent-event passthrough for streaming chat completions.
+ *
+ * The gateway relays an OpenAI-compatible `text/event-stream` body verbatim so
+ * clients receive tokens as the provider emits them, while scanning the frames
+ * for the terminal usage object that metering needs.
+ */
+/** Accumulates SSE frames, exposing the last usage object the stream carried. */
+export class UsageScanner {
+    buffer = '';
+    usage;
+    /** Feed one decoded chunk of the event stream. */
+    push(chunk) {
+        this.buffer += chunk;
+        const frames = this.buffer.split('\n');
+        this.buffer = frames.pop() ?? '';
+        for (const frame of frames) {
+            const line = frame.trim();
+            if (!line.startsWith('data:'))
+                continue;
+            const payload = line.slice('data:'.length).trim();
+            if (payload === '' || payload === '[DONE]')
+                continue;
+            let parsed;
+            try {
+                parsed = JSON.parse(payload);
+            }
+            catch {
+                // A provider frame the gateway cannot parse carries no usage it could
+                // report; relaying is the router's job and happens regardless.
+                continue;
+            }
+            const usage = parsed.usage;
+            if (!usage)
+                continue;
+            this.usage = {
+                prompt_tokens: usage.prompt_tokens ?? 0,
+                completion_tokens: usage.completion_tokens ?? 0,
+                total_tokens: usage.total_tokens ?? (usage.prompt_tokens ?? 0) + (usage.completion_tokens ?? 0)
+            };
+        }
+    }
+    /** The last usage object seen, or undefined when the provider sent none. */
+    result() {
+        return this.usage;
+    }
+}
+/**
+ * Relay an upstream event-stream response to the client byte for byte and
+ * answer the usage its final frame carried, or undefined when it carried none.
+ *
+ * @param upstream - the provider response whose body is being relayed.
+ * @param res - the client response the frames are written to.
+ * @returns the usage the stream reported, if any.
+ */
+export async function relayEventStream(upstream, res) {
+    res.setHeader('Content-Type', upstream.headers.get('content-type') ?? 'text/event-stream');
+    res.setHeader('Cache-Control', 'no-cache, no-transform');
+    res.setHeader('Connection', 'keep-alive');
+    res.flushHeaders?.();
+    const scanner = new UsageScanner();
+    if (upstream.body) {
+        const decoder = new TextDecoder();
+        const reader = upstream.body.getReader();
+        for (;;) {
+            const { done, value } = await reader.read();
+            if (done)
+                break;
+            const text = decoder.decode(value, { stream: true });
+            scanner.push(text);
+            res.write(text);
+        }
+    }
+    res.end();
+    return scanner.result();
+}
+/**
+ * Ask an OpenAI-compatible provider to include usage in its final frame.
+ * Callers that already set `stream_options` keep their own choice.
+ */
+export function withUsageStreamOptions(body) {
+    if (body.stream_options !== undefined)
+        return body;
+    return { ...body, stream_options: { include_usage: true } };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agentic-server",
-  "version": "0.24.0",
+  "version": "0.25.0",
   "description": "Standalone OpenAI-compatible LLM gateway — multi-provider proxy with pluggable inference metering",
   "main": "index.js",
   "module": "esm/index.js",
@@ -51,5 +51,5 @@
     "metering",
     "constructive"
   ],
-  "gitHead": "b5b6b9f05276b7e9cbd3d1f4acd31dd315c353ea"
+  "gitHead": "e35db12526094b515596ced2dd27a1ec8edd99d5"
 }

package/router.d.ts

@@ -6,6 +6,7 @@
  *   - Model-based routing: "anthropic/claude-3.5-sonnet" → anthropic provider
  *   - Header-based routing: X-LLM-Provider: ollama → ollama provider
  *   - Fire-and-forget inference metering via an injected InferenceSink
+ *   - Streaming chat completions relayed as server-sent events
  *   - /v1/usage reporting endpoint for external usage submission
  */
 import { Router } from 'express';

package/router.js

@@ -7,6 +7,7 @@
  *   - Model-based routing: "anthropic/claude-3.5-sonnet" → anthropic provider
  *   - Header-based routing: X-LLM-Provider: ollama → ollama provider
  *   - Fire-and-forget inference metering via an injected InferenceSink
+ *   - Streaming chat completions relayed as server-sent events
  *   - /v1/usage reporting endpoint for external usage submission
  */
 Object.defineProperty(exports, "__esModule", { value: true });
@@ -14,6 +15,7 @@ exports.createRouter = void 0;
 const logger_1 = require("@pgpmjs/logger");
 const express_1 = require("express");
 const providers_1 = require("./providers");
+const streaming_1 = require("./streaming");
 const transforms_1 = require("./transforms");
 const log = new logger_1.Logger('agentic-server');
 const createRouter = (options) => {
@@ -39,14 +41,39 @@ const createRouter = (options) => {
             provider: provider.type,
             model: req.body?.model
         });
+        // Only the OpenAI-compatible wire carries stream frames the gateway can
+        // relay; translating Ollama's or Anthropic's stream formats is separate
+        // work, and answering a stream request with a whole JSON body would break
+        // the client silently.
+        const streaming = req.body?.stream === true;
+        if (streaming && provider.type !== 'openai') {
+            res.status(501).json({
+                error: {
+                    message: `streaming is not supported for provider type '${provider.type}'`
+                }
+            });
+            return;
+        }
         try {
             const upstreamUrl = (0, providers_1.resolveUpstreamUrl)(provider, '/v1/chat/completions');
-            const body = (0, transforms_1.transformChatRequest)(provider, req.body || {});
+            const transformed = (0, transforms_1.transformChatRequest)(provider, req.body || {});
+            const body = streaming ? (0, streaming_1.withUsageStreamOptions)(transformed) : transformed;
             const headers = (0, providers_1.buildProviderHeaders)(provider);
+            const abort = new AbortController();
+            // A client that hangs up mid-stream stops the upstream read; the request
+            // stream's own 'close' fires as soon as the body is consumed, so the
+            // response is what reports the disconnect.
+            if (streaming) {
+                res.on('close', () => {
+                    if (!res.writableEnded)
+                        abort.abort();
+                });
+            }
             const upstream = await fetch(upstreamUrl, {
                 method: 'POST',
                 headers,
-                body: JSON.stringify(body)
+                body: JSON.stringify(body),
+                ...(streaming ? { signal: abort.signal } : {})
             });
             const latencyMs = Number(process.hrtime.bigint() - startTime) / 1e6;
             if (!upstream.ok) {
@@ -70,6 +97,34 @@ const createRouter = (options) => {
                 });
                 return;
             }
+            if (streaming) {
+                const streamUsage = await (0, streaming_1.relayEventStream)(upstream, res);
+                const streamLatencyMs = Number(process.hrtime.bigint() - startTime) / 1e6;
+                log.info('inference complete', {
+                    databaseId,
+                    provider: provider.type,
+                    model: req.body?.model,
+                    promptTokens: streamUsage?.prompt_tokens,
+                    completionTokens: streamUsage?.completion_tokens,
+                    streamed: true
+                });
+                if (sink) {
+                    sink.logInference({
+                        databaseId, entityId, actorId,
+                        model: String(req.body?.model || body.model || ''),
+                        provider: provider.type,
+                        service: 'chat',
+                        operation: 'chat/completions',
+                        inputTokens: streamUsage?.prompt_tokens || 0,
+                        outputTokens: streamUsage?.completion_tokens || 0,
+                        totalTokens: streamUsage?.total_tokens || 0,
+                        latencyMs: streamLatencyMs,
+                        status: 'ok',
+                        ...(streamUsage ? { rawUsage: streamUsage } : {})
+                    });
+                }
+                return;
+            }
             const data = await upstream.json();
             const { body: responseBody, usage } = (0, transforms_1.transformChatResponse)(data, provider);
             log.info('inference complete', {
@@ -112,6 +167,12 @@ const createRouter = (options) => {
                     errorType: err.message
                 });
             }
+            // A stream that failed mid-relay has already sent its status and frames;
+            // the client sees the truncated stream rather than a JSON error.
+            if (res.headersSent) {
+                res.end();
+                return;
+            }
             res.status(502).json({
                 error: { message: 'Failed to reach LLM provider', details: err.message }
             });

package/streaming.d.ts

@@ -0,0 +1,41 @@
+/**
+ * Server-sent-event passthrough for streaming chat completions.
+ *
+ * The gateway relays an OpenAI-compatible `text/event-stream` body verbatim so
+ * clients receive tokens as the provider emits them, while scanning the frames
+ * for the terminal usage object that metering needs.
+ */
+import type { UsageResult } from './types';
+/** Accumulates SSE frames, exposing the last usage object the stream carried. */
+export declare class UsageScanner {
+    private buffer;
+    private usage;
+    /** Feed one decoded chunk of the event stream. */
+    push(chunk: string): void;
+    /** The last usage object seen, or undefined when the provider sent none. */
+    result(): UsageResult | undefined;
+}
+/**
+ * Relay an upstream event-stream response to the client byte for byte and
+ * answer the usage its final frame carried, or undefined when it carried none.
+ *
+ * @param upstream - the provider response whose body is being relayed.
+ * @param res - the client response the frames are written to.
+ * @returns the usage the stream reported, if any.
+ */
+export declare function relayEventStream(upstream: {
+    body: ReadableStream<Uint8Array> | null;
+    headers: {
+        get(name: string): string | null;
+    };
+}, res: {
+    setHeader(name: string, value: string): void;
+    flushHeaders?(): void;
+    write(chunk: string): void;
+    end(): void;
+}): Promise<UsageResult | undefined>;
+/**
+ * Ask an OpenAI-compatible provider to include usage in its final frame.
+ * Callers that already set `stream_options` keep their own choice.
+ */
+export declare function withUsageStreamOptions(body: Record<string, unknown>): Record<string, unknown>;

package/streaming.js

@@ -0,0 +1,91 @@
+"use strict";
+/**
+ * Server-sent-event passthrough for streaming chat completions.
+ *
+ * The gateway relays an OpenAI-compatible `text/event-stream` body verbatim so
+ * clients receive tokens as the provider emits them, while scanning the frames
+ * for the terminal usage object that metering needs.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.UsageScanner = void 0;
+exports.relayEventStream = relayEventStream;
+exports.withUsageStreamOptions = withUsageStreamOptions;
+/** Accumulates SSE frames, exposing the last usage object the stream carried. */
+class UsageScanner {
+    buffer = '';
+    usage;
+    /** Feed one decoded chunk of the event stream. */
+    push(chunk) {
+        this.buffer += chunk;
+        const frames = this.buffer.split('\n');
+        this.buffer = frames.pop() ?? '';
+        for (const frame of frames) {
+            const line = frame.trim();
+            if (!line.startsWith('data:'))
+                continue;
+            const payload = line.slice('data:'.length).trim();
+            if (payload === '' || payload === '[DONE]')
+                continue;
+            let parsed;
+            try {
+                parsed = JSON.parse(payload);
+            }
+            catch {
+                // A provider frame the gateway cannot parse carries no usage it could
+                // report; relaying is the router's job and happens regardless.
+                continue;
+            }
+            const usage = parsed.usage;
+            if (!usage)
+                continue;
+            this.usage = {
+                prompt_tokens: usage.prompt_tokens ?? 0,
+                completion_tokens: usage.completion_tokens ?? 0,
+                total_tokens: usage.total_tokens ?? (usage.prompt_tokens ?? 0) + (usage.completion_tokens ?? 0)
+            };
+        }
+    }
+    /** The last usage object seen, or undefined when the provider sent none. */
+    result() {
+        return this.usage;
+    }
+}
+exports.UsageScanner = UsageScanner;
+/**
+ * Relay an upstream event-stream response to the client byte for byte and
+ * answer the usage its final frame carried, or undefined when it carried none.
+ *
+ * @param upstream - the provider response whose body is being relayed.
+ * @param res - the client response the frames are written to.
+ * @returns the usage the stream reported, if any.
+ */
+async function relayEventStream(upstream, res) {
+    res.setHeader('Content-Type', upstream.headers.get('content-type') ?? 'text/event-stream');
+    res.setHeader('Cache-Control', 'no-cache, no-transform');
+    res.setHeader('Connection', 'keep-alive');
+    res.flushHeaders?.();
+    const scanner = new UsageScanner();
+    if (upstream.body) {
+        const decoder = new TextDecoder();
+        const reader = upstream.body.getReader();
+        for (;;) {
+            const { done, value } = await reader.read();
+            if (done)
+                break;
+            const text = decoder.decode(value, { stream: true });
+            scanner.push(text);
+            res.write(text);
+        }
+    }
+    res.end();
+    return scanner.result();
+}
+/**
+ * Ask an OpenAI-compatible provider to include usage in its final frame.
+ * Callers that already set `stream_options` keep their own choice.
+ */
+function withUsageStreamOptions(body) {
+    if (body.stream_options !== undefined)
+        return body;
+    return { ...body, stream_options: { include_usage: true } };
+}