jeopi-agent-core 16.2.13

package/src/proxy.ts

@@ -0,0 +1,376 @@
+/**
+ * Proxy stream function for apps that route LLM calls through a server.
+ * The server manages auth and proxies requests to LLM providers.
+ */
+import {
+	type AssistantMessage,
+	type AssistantMessageEvent,
+	type Context,
+	EventStream,
+	type FetchImpl,
+	type Model,
+	type SimpleStreamOptions,
+	type StopReason,
+	type ToolCall,
+} from "jeopi-ai";
+import {
+	clearStreamingPartialJson,
+	kStreamingPartialJson,
+	type StreamingPartialJsonCarrier,
+	setStreamingPartialJson,
+} from "jeopi-ai/utils/block-symbols";
+import { calculateCost } from "jeopi-catalog/models";
+import { parseStreamingJson, readSseJson } from "jeopi-utils";
+
+// Event stream adapter for proxy SSE events
+export class ProxyMessageEventStream extends EventStream<AssistantMessageEvent, AssistantMessage> {
+	constructor() {
+		super(
+			event => event.type === "done" || event.type === "error",
+			event => {
+				if (event.type === "done") return event.message;
+				if (event.type === "error") return event.error;
+				throw new Error("Unexpected event type");
+			},
+		);
+	}
+}
+
+/**
+ * Proxy event types - server sends these with partial field stripped to reduce bandwidth.
+ */
+export type ProxyAssistantMessageEvent =
+	| { type: "start" }
+	| { type: "text_start"; contentIndex: number }
+	| { type: "text_delta"; contentIndex: number; delta: string }
+	| { type: "text_end"; contentIndex: number; contentSignature?: string }
+	| { type: "thinking_start"; contentIndex: number }
+	| { type: "thinking_delta"; contentIndex: number; delta: string }
+	| { type: "thinking_end"; contentIndex: number; contentSignature?: string }
+	| { type: "toolcall_start"; contentIndex: number; id: string; toolName: string }
+	| { type: "toolcall_delta"; contentIndex: number; delta: string }
+	| { type: "toolcall_end"; contentIndex: number }
+	| {
+			type: "done";
+			reason: Extract<StopReason, "stop" | "length" | "toolUse">;
+			usage: AssistantMessage["usage"];
+	  }
+	| {
+			type: "error";
+			reason: Extract<StopReason, "aborted" | "error">;
+			errorMessage?: string;
+			usage: AssistantMessage["usage"];
+	  };
+
+export interface ProxyStreamOptions extends SimpleStreamOptions {
+	/** Auth token for the proxy server */
+	authToken: string;
+	/** Proxy server URL (e.g., "https://genai.example.com") */
+	proxyUrl: string;
+	/** Optional fetch implementation; defaults to global fetch. */
+	fetch?: FetchImpl;
+}
+
+/**
+ * Stream function that proxies through a server instead of calling LLM providers directly.
+ * The server strips the partial field from delta events to reduce bandwidth.
+ * We reconstruct the partial message client-side.
+ *
+ * Use this as the `streamFn` option when creating an Agent that needs to go through a proxy.
+ *
+ * @example
+ * ```typescript
+ * const agent = new Agent({
+ *   streamFn: (model, context, options) =>
+ *     streamProxy(model, context, {
+ *       ...options,
+ *       authToken: await getAuthToken(),
+ *       proxyUrl: "https://genai.example.com",
+ *     }),
+ * });
+ * ```
+ */
+export function streamProxy(model: Model, context: Context, options: ProxyStreamOptions): ProxyMessageEventStream {
+	const stream = new ProxyMessageEventStream();
+
+	(async () => {
+		// Initialize the partial message that we'll build up from events
+		const partial: AssistantMessage = {
+			role: "assistant",
+			stopReason: "stop",
+			content: [],
+			api: model.api,
+			provider: model.provider,
+			model: model.id,
+			usage: {
+				input: 0,
+				output: 0,
+				cacheRead: 0,
+				cacheWrite: 0,
+				totalTokens: 0,
+				cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
+			},
+			timestamp: Date.now(),
+		};
+
+		let response: Response | null = null;
+		const abortHandler = () => {
+			const body = response?.body;
+			if (body) {
+				body.cancel("Request aborted by user").catch(() => {});
+			}
+		};
+		if (options.signal) {
+			options.signal.addEventListener("abort", abortHandler, { once: true });
+		}
+
+		try {
+			response = await (options.fetch ?? fetch)(`${options.proxyUrl}/api/stream`, {
+				method: "POST",
+				headers: {
+					Authorization: `Bearer ${options.authToken}`,
+					"Content-Type": "application/json",
+				},
+				body: JSON.stringify({
+					model,
+					context,
+					options: {
+						temperature: options.temperature,
+						topP: options.topP,
+						topK: options.topK,
+						minP: options.minP,
+						presencePenalty: options.presencePenalty,
+						repetitionPenalty: options.repetitionPenalty,
+						maxTokens: options.maxTokens,
+						reasoning: options.reasoning,
+					},
+				}),
+				signal: options.signal,
+			});
+
+			if (!response.ok) {
+				let errorMessage = `Proxy error: ${response.status} ${response.statusText}`;
+				try {
+					const errorData = (await response.json()) as { error?: string };
+					if (errorData.error) {
+						errorMessage = `Proxy error: ${errorData.error}`;
+					}
+				} catch {
+					// Couldn't parse error response
+				}
+				throw new Error(errorMessage);
+			}
+
+			let sawTerminalEvent = false;
+			const partialJsonByIndex = new Map<number, string>();
+			for await (const event of readSseJson<ProxyAssistantMessageEvent>(
+				response.body as ReadableStream<Uint8Array>,
+				options.signal,
+			)) {
+				const parsedEvent = processProxyEvent(model, event, partial, partialJsonByIndex);
+				if (parsedEvent) {
+					if (parsedEvent.type === "done" || parsedEvent.type === "error") {
+						sawTerminalEvent = true;
+					}
+					stream.push(parsedEvent);
+				}
+			}
+
+			if (!sawTerminalEvent) {
+				if (options.signal?.aborted) {
+					const reason = options.signal.reason;
+					throw reason instanceof Error ? reason : new Error(String(reason ?? "Request aborted"));
+				}
+				throw new Error("Proxy stream ended without a terminal event (done or error)");
+			}
+
+			stream.end();
+		} catch (error) {
+			const errorMessage = error instanceof Error ? error.message : String(error);
+			const reason = options.signal?.aborted ? "aborted" : "error";
+			partial.stopReason = reason;
+			partial.errorMessage = errorMessage;
+			scrubPartialJson(partial);
+			stream.push({
+				type: "error",
+				reason,
+				error: partial,
+			});
+			stream.end();
+		} finally {
+			if (options.signal) {
+				options.signal.removeEventListener("abort", abortHandler);
+			}
+		}
+	})();
+
+	return stream;
+}
+
+/**
+ * Clear the `partialJson` streaming symbol from any tool-call content blocks
+ * that still carry it (e.g. when the stream ended without a `toolcall_end`), so
+ * the finalized `AssistantMessage` no longer reads as still-streaming.
+ */
+function scrubPartialJson(partial: AssistantMessage): void {
+	for (const block of partial.content) {
+		if (block?.type === "toolCall") clearStreamingPartialJson(block);
+	}
+}
+
+/**
+ * Process a proxy event and update the partial message.
+ *
+ * Streaming `partialJson` for in-progress tool calls is accumulated in a
+ * side-channel map keyed by `contentIndex` and also written onto the content
+ * object as a symbol-keyed field so downstream renderers can read it
+ * during streaming. The field is cleared at `toolcall_end` and scrubbed from any
+ * remaining blocks at `done`/`error` so the finalized `AssistantMessage` never
+ * reads as still-streaming.
+ */
+function processProxyEvent(
+	model: Model,
+	proxyEvent: ProxyAssistantMessageEvent,
+	partial: AssistantMessage,
+	partialJsonByIndex: Map<number, string>,
+): AssistantMessageEvent | undefined {
+	switch (proxyEvent.type) {
+		case "start":
+			partial.content.length = 0;
+			partial.usage = {
+				input: 0,
+				output: 0,
+				cacheRead: 0,
+				cacheWrite: 0,
+				totalTokens: 0,
+				cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
+			};
+			partial.errorMessage = undefined;
+			partial.errorId = undefined;
+			partial.duration = undefined;
+			(partial as { stopReason?: string }).stopReason = undefined;
+			return { type: "start", partial };
+
+		case "text_start":
+			partial.content[proxyEvent.contentIndex] = { type: "text", text: "" };
+			return { type: "text_start", contentIndex: proxyEvent.contentIndex, partial };
+
+		case "text_delta": {
+			const content = partial.content[proxyEvent.contentIndex];
+			if (content?.type === "text") {
+				content.text += proxyEvent.delta;
+				return {
+					type: "text_delta",
+					contentIndex: proxyEvent.contentIndex,
+					delta: proxyEvent.delta,
+					partial,
+				};
+			}
+			throw new Error("Received text_delta for non-text content");
+		}
+
+		case "text_end": {
+			const content = partial.content[proxyEvent.contentIndex];
+			if (content?.type === "text") {
+				content.textSignature = proxyEvent.contentSignature;
+				return {
+					type: "text_end",
+					contentIndex: proxyEvent.contentIndex,
+					content: content.text,
+					partial,
+				};
+			}
+			throw new Error("Received text_end for non-text content");
+		}
+
+		case "thinking_start":
+			partial.content[proxyEvent.contentIndex] = { type: "thinking", thinking: "" };
+			return { type: "thinking_start", contentIndex: proxyEvent.contentIndex, partial };
+
+		case "thinking_delta": {
+			const content = partial.content[proxyEvent.contentIndex];
+			if (content?.type === "thinking") {
+				content.thinking += proxyEvent.delta;
+				return {
+					type: "thinking_delta",
+					contentIndex: proxyEvent.contentIndex,
+					delta: proxyEvent.delta,
+					partial,
+				};
+			}
+			throw new Error("Received thinking_delta for non-thinking content");
+		}
+
+		case "thinking_end": {
+			const content = partial.content[proxyEvent.contentIndex];
+			if (content?.type === "thinking") {
+				content.thinkingSignature = proxyEvent.contentSignature;
+				return {
+					type: "thinking_end",
+					contentIndex: proxyEvent.contentIndex,
+					content: content.thinking,
+					partial,
+				};
+			}
+			throw new Error("Received thinking_end for non-thinking content");
+		}
+
+		case "toolcall_start":
+			partial.content[proxyEvent.contentIndex] = {
+				type: "toolCall",
+				id: proxyEvent.id,
+				name: proxyEvent.toolName,
+				arguments: {},
+				[kStreamingPartialJson]: "",
+			} as ToolCall & StreamingPartialJsonCarrier;
+			partialJsonByIndex.set(proxyEvent.contentIndex, "");
+			return { type: "toolcall_start", contentIndex: proxyEvent.contentIndex, partial };
+		case "toolcall_delta": {
+			const content = partial.content[proxyEvent.contentIndex];
+			if (content?.type === "toolCall") {
+				const acc = (partialJsonByIndex.get(proxyEvent.contentIndex) ?? "") + proxyEvent.delta;
+				partialJsonByIndex.set(proxyEvent.contentIndex, acc);
+				content.arguments = parseStreamingJson(acc) || {};
+				setStreamingPartialJson(content, acc);
+				partial.content[proxyEvent.contentIndex] = { ...content }; // Trigger reactivity
+				return {
+					type: "toolcall_delta",
+					contentIndex: proxyEvent.contentIndex,
+					delta: proxyEvent.delta,
+					partial,
+				};
+			}
+			throw new Error("Received toolcall_delta for non-toolCall content");
+		}
+
+		case "toolcall_end": {
+			const content = partial.content[proxyEvent.contentIndex];
+			if (content?.type === "toolCall") {
+				partialJsonByIndex.delete(proxyEvent.contentIndex);
+				clearStreamingPartialJson(content);
+				return {
+					type: "toolcall_end",
+					contentIndex: proxyEvent.contentIndex,
+					toolCall: content,
+					partial,
+				};
+			}
+			return undefined;
+		}
+
+		case "done":
+			partial.stopReason = proxyEvent.reason;
+			partial.usage = proxyEvent.usage;
+			calculateCost(model, partial.usage);
+			scrubPartialJson(partial);
+			return { type: "done", reason: proxyEvent.reason, message: partial };
+
+		case "error":
+			partial.stopReason = proxyEvent.reason;
+			partial.errorMessage = proxyEvent.errorMessage;
+			partial.usage = proxyEvent.usage;
+			calculateCost(model, partial.usage);
+			scrubPartialJson(partial);
+			return { type: "error", reason: proxyEvent.reason, error: partial };
+	}
+}

package/src/replay-policy.ts

@@ -0,0 +1,13 @@
+import type { AssistantMessage, Message } from "jeopi-ai";
+
+/** Detects API-level provider refusals that are terminal errors, not dialogue to replay. */
+export function isProviderRefusalMessage(message: AssistantMessage): boolean {
+	if (message.stopReason !== "error") return false;
+	const stopType = message.stopDetails?.type;
+	return stopType === "refusal" || stopType === "sensitive";
+}
+
+/** Removes API-level provider refusals from live provider replay while preserving other messages. */
+export function filterProviderReplayMessages(messages: readonly Message[]): Message[] {
+	return messages.filter(message => message.role !== "assistant" || !isProviderRefusalMessage(message));
+}