@tjamescouch/gro 1.3.2

package/src/drivers/streaming-openai.ts

@@ -0,0 +1,245 @@
+/**
+ * Streaming OpenAI-compatible chat driver.
+ * Works with OpenAI, Anthropic (via proxy), LM Studio, Ollama, etc.
+ */
+import { Logger } from "../logger.js";
+import { asError } from "../errors.js";
+import { rateLimiter } from "../utils/rate-limiter.js";
+import { timedFetch } from "../utils/timed-fetch.js";
+import { MAX_RETRIES, isRetryable, retryDelay, sleep } from "../utils/retry.js";
+import type { ChatDriver, ChatMessage, ChatOutput, ChatToolCall } from "./types.js";
+
+export interface OpenAiDriverConfig {
+  baseUrl: string;
+  model: string;
+  apiKey?: string;
+  timeoutMs?: number;
+}
+
+function yieldToLoop(): Promise<void> {
+  return new Promise<void>((resolve) =>
+    typeof (globalThis as any).setImmediate === "function"
+      ? (globalThis as any).setImmediate(resolve)
+      : setTimeout(resolve, 0)
+  );
+}
+
+class YieldBudget {
+  private bytesSince = 0;
+  private last = Date.now();
+  constructor(
+    private readonly byteBudget = 1024,
+    private readonly msBudget = 8
+  ) {}
+  async maybe(extraBytes = 0): Promise<void> {
+    this.bytesSince += extraBytes;
+    const now = Date.now();
+    if (this.bytesSince >= this.byteBudget || (now - this.last) >= this.msBudget) {
+      this.bytesSince = 0;
+      this.last = now;
+      await yieldToLoop();
+    }
+  }
+}
+
+export function makeStreamingOpenAiDriver(cfg: OpenAiDriverConfig): ChatDriver {
+  const base = cfg.baseUrl.replace(/\/+$/, "");
+  const endpoint = `${base}/v1/chat/completions`;
+  const defaultTimeout = cfg.timeoutMs ?? 2 * 60 * 60 * 1000;
+
+  async function chat(messages: ChatMessage[], opts?: any): Promise<ChatOutput> {
+    await rateLimiter.limit("llm-ask", 1);
+    Logger.debug("streaming messages out", messages);
+
+    const controller = new AbortController();
+    const userSignal: AbortSignal | undefined = opts?.signal;
+    const linkAbort = () => controller.abort();
+    if (userSignal) {
+      if (userSignal.aborted) controller.abort();
+      else userSignal.addEventListener("abort", linkAbort, { once: true });
+    }
+    const timer = setTimeout(() => controller.abort(), defaultTimeout);
+
+    const model = opts?.model ?? cfg.model;
+    const tools = Array.isArray(opts?.tools) && opts.tools.length ? opts.tools : undefined;
+    const onToken: ((t: string) => void) | undefined = opts?.onToken;
+    const onReasoningToken: ((t: string) => void) | undefined = opts?.onReasoningToken;
+    const onToolCallDelta: ((t: ChatToolCall) => void) | undefined = opts?.onToolCallDelta;
+
+    const headers: Record<string, string> = { "Content-Type": "application/json" };
+    if (cfg.apiKey) headers["Authorization"] = `Bearer ${cfg.apiKey}`;
+
+    const payload: any = { model, messages, stream: true };
+    if (tools) {
+      payload.tools = tools;
+      payload.tool_choice = "auto";
+    }
+
+    try {
+      let res!: Response;
+      for (let attempt = 0; ; attempt++) {
+        res = await timedFetch(endpoint, {
+          method: "POST",
+          headers,
+          body: JSON.stringify(payload),
+          signal: controller.signal,
+          where: "driver:openai:stream",
+          timeoutMs: defaultTimeout,
+        });
+
+        if (res.ok) break;
+
+        if (isRetryable(res.status) && attempt < MAX_RETRIES) {
+          const delay = retryDelay(attempt);
+          Logger.warn(`OpenAI ${res.status}, retry ${attempt + 1}/${MAX_RETRIES} in ${Math.round(delay)}ms`);
+          await sleep(delay);
+          continue;
+        }
+
+        const text = await res.text().catch(() => "");
+        throw new Error(`OpenAI chat (stream) failed (${res.status}): ${text}`);
+      }
+
+      const ct = (res.headers.get("content-type") || "").toLowerCase();
+      if (!ct.includes("text/event-stream")) {
+        const data = await res.json().catch(() => ({}));
+        const choice = data?.choices?.[0];
+        const msg = choice?.message || {};
+        const content = typeof msg?.content === "string" ? msg.content : "";
+        const toolCalls: ChatToolCall[] = Array.isArray(msg?.tool_calls) ? msg.tool_calls : [];
+        if (content && onToken) onToken(content);
+        return { text: content, reasoning: msg?.reasoning || undefined, toolCalls };
+      }
+
+      // SSE streaming
+      const decoder = new TextDecoder("utf-8");
+      const yb = new YieldBudget(1024, 8);
+      let buf = "";
+      let fullText = "";
+      let fullReasoning = "";
+      const toolByIndex = new Map<number, ChatToolCall>();
+
+      const pumpEvent = async (rawEvent: string) => {
+        const dataLines = rawEvent
+          .split("\n")
+          .map((l) => l.trim())
+          .filter((l) => l.startsWith("data:"))
+          .map((l) => l.replace(/^data:\s?/, ""));
+
+        if (!dataLines.length) return;
+        const joined = dataLines.join("\n").trim();
+        if (!joined || joined === "[DONE]") return;
+
+        let payload: any;
+        try { payload = JSON.parse(joined); } catch { return; }
+
+        const delta = payload?.choices?.[0]?.delta;
+        if (!delta) return;
+
+        if (typeof delta.content === "string" && delta.content.length) {
+          fullText += delta.content;
+          if (onToken) {
+            let s = delta.content;
+            while (s.length) {
+              const piece = s.slice(0, 512);
+              s = s.slice(512);
+              try { onToken(piece); } catch {}
+              await yb.maybe(piece.length);
+            }
+          } else {
+            await yb.maybe(delta.content.length);
+          }
+        }
+
+        if (typeof delta.reasoning === "string" && delta.reasoning.length) {
+          fullReasoning += delta.reasoning;
+          if (onReasoningToken) {
+            let s = delta.reasoning;
+            while (s.length) {
+              const piece = s.slice(0, 512);
+              s = s.slice(512);
+              try { onReasoningToken(piece); } catch {}
+              await yb.maybe(piece.length);
+            }
+          } else {
+            await yb.maybe(delta.reasoning.length);
+          }
+        }
+
+        if (Array.isArray(delta.tool_calls)) {
+          for (const item of delta.tool_calls) {
+            const idx: number = typeof item?.index === "number" ? item.index : 0;
+            const prev = toolByIndex.get(idx) ?? {
+              id: "", type: "function", function: { name: "", arguments: "" }
+            };
+            if (typeof item.id === "string" && item.id) prev.id = item.id;
+            if (typeof item.type === "string" && item.type) (prev as any).type = item.type;
+            const f = item?.function ?? {};
+            if (typeof f.name === "string" && f.name) prev.function.name += f.name;
+            if (typeof f.arguments === "string" && f.arguments) prev.function.arguments += f.arguments;
+            toolByIndex.set(idx, prev);
+            if (onToolCallDelta) {
+              try { onToolCallDelta(prev); } catch {}
+            }
+            await yb.maybe(64);
+          }
+        }
+      };
+
+      const body: any = res.body;
+
+      if (body && typeof body.getReader === "function") {
+        const reader = body.getReader();
+        try {
+          while (true) {
+            const { value, done } = await reader.read();
+            if (done) break;
+            buf += decoder.decode(value, { stream: true });
+            await yb.maybe((value as Uint8Array)?.byteLength ?? 0);
+            let sepIdx: number;
+            while ((sepIdx = buf.indexOf("\n\n")) !== -1) {
+              const rawEvent = buf.slice(0, sepIdx).trim();
+              buf = buf.slice(sepIdx + 2);
+              if (rawEvent) await pumpEvent(rawEvent);
+            }
+          }
+          if (buf.trim()) await pumpEvent(buf.trim());
+        } finally {
+          reader.cancel().catch(() => {});
+        }
+      } else if (body && typeof body[Symbol.asyncIterator] === "function") {
+        for await (const chunk of body as AsyncIterable<Uint8Array>) {
+          buf += decoder.decode(chunk, { stream: true });
+          await yb.maybe(chunk.byteLength);
+          let sepIdx: number;
+          while ((sepIdx = buf.indexOf("\n\n")) !== -1) {
+            const rawEvent = buf.slice(0, sepIdx).trim();
+            buf = buf.slice(sepIdx + 2);
+            if (rawEvent) await pumpEvent(rawEvent);
+          }
+        }
+        if (buf.trim()) await pumpEvent(buf.trim());
+      } else {
+        const txt = await res.text();
+        for (const part of txt.split("\n\n").map((s) => s.trim())) {
+          if (part) await pumpEvent(part);
+        }
+      }
+
+      const toolCalls: ChatToolCall[] = Array.from(toolByIndex.entries())
+        .sort((a, b) => a[0] - b[0])
+        .map(([, v]) => v);
+
+      return { text: fullText, reasoning: fullReasoning || undefined, toolCalls };
+    } catch (e: unknown) {
+      const wrapped = asError(e);
+      if (wrapped.name === "AbortError") Logger.debug("timeout(stream)", { ms: defaultTimeout });
+      throw wrapped;
+    } finally {
+      clearTimeout(timer);
+      if (userSignal) userSignal.removeEventListener("abort", linkAbort);
+    }
+  }
+
+  return { chat };
+}

package/src/drivers/types.ts

@@ -0,0 +1,33 @@
+export interface ChatMessage {
+  role: string;
+  from: string;
+  content: string;
+  reasoning?: string;
+  /** When role==="tool", the id of the tool call being answered. */
+  tool_call_id?: string;
+  /** Optional: tool/function name for clarity. */
+  name?: string;
+}
+
+export interface ChatToolCall {
+  id: string;
+  type: "function" | "custom";
+  function: { name: string; arguments: string };
+  raw?: string;
+}
+
+export interface ChatOutput {
+  text: string;
+  toolCalls: ChatToolCall[];
+  reasoning?: string;
+}
+
+export interface ChatDriver {
+  chat(messages: ChatMessage[], opts?: {
+    model?: string;
+    tools?: any[];
+    onToken?: (s: string) => void;
+    onReasoningToken?: (s: string) => void;
+    onToolCallDelta?: (s: ChatToolCall) => void;
+  }): Promise<ChatOutput>;
+}

package/src/errors.ts

@@ -0,0 +1,97 @@
+/**
+ * Structured error types for gro.
+ *
+ * All error boundaries should wrap errors using GroError instead of
+ * stringifying with e.message. This preserves stack traces, enables
+ * retry logic, and provides consistent logging fields.
+ */
+
+export type GroErrorKind =
+  | "provider_error"
+  | "tool_error"
+  | "config_error"
+  | "mcp_error"
+  | "timeout_error"
+  | "session_error";
+
+export interface GroError extends Error {
+  kind: GroErrorKind;
+  provider?: string;
+  model?: string;
+  request_id?: string;
+  retryable: boolean;
+  latency_ms?: number;
+  cause?: unknown;
+}
+
+/**
+ * Create a GroError with structured fields.
+ */
+export function groError(
+  kind: GroErrorKind,
+  message: string,
+  opts: {
+    provider?: string;
+    model?: string;
+    request_id?: string;
+    retryable?: boolean;
+    latency_ms?: number;
+    cause?: unknown;
+  } = {},
+): GroError {
+  const err = new Error(message) as GroError;
+  err.kind = kind;
+  err.retryable = opts.retryable ?? false;
+  if (opts.provider) err.provider = opts.provider;
+  if (opts.model) err.model = opts.model;
+  if (opts.request_id) err.request_id = opts.request_id;
+  if (opts.latency_ms !== undefined) err.latency_ms = opts.latency_ms;
+  if (opts.cause !== undefined) err.cause = opts.cause;
+  return err;
+}
+
+/**
+ * Normalize an unknown thrown value into an Error.
+ * Handles strings, objects, nulls — the full JS throw spectrum.
+ */
+export function asError(e: unknown): Error {
+  if (e instanceof Error) return e;
+  if (typeof e === "string") return new Error(e.slice(0, 1024));
+  if (e === null || e === undefined) return new Error("Unknown error");
+  try {
+    const s = String(e);
+    return new Error(s.length > 1024 ? s.slice(0, 1024) + "..." : s);
+  } catch {
+    return new Error("Unknown error (unstringifiable)");
+  }
+}
+
+/**
+ * Check if an error is a GroError with structured fields.
+ */
+export function isGroError(e: unknown): e is GroError {
+  return e instanceof Error && "kind" in e && "retryable" in e;
+}
+
+/**
+ * Format a GroError for structured logging.
+ * Returns a plain object suitable for JSON.stringify or structured loggers.
+ */
+export function errorLogFields(e: GroError): Record<string, unknown> {
+  const fields: Record<string, unknown> = {
+    kind: e.kind,
+    message: e.message,
+    retryable: e.retryable,
+  };
+  if (e.provider) fields.provider = e.provider;
+  if (e.model) fields.model = e.model;
+  if (e.request_id) fields.request_id = e.request_id;
+  if (e.latency_ms !== undefined) fields.latency_ms = e.latency_ms;
+  if (e.cause) {
+    const cause = asError(e.cause);
+    fields.cause_message = cause.message;
+    if (cause.stack) fields.cause_stack = cause.stack;
+  }
+  if (e.stack) fields.stack = e.stack;
+  return fields;
+}

package/src/logger.ts

@@ -0,0 +1,28 @@
+export const C = {
+  reset: "\x1b[0m",
+  bold: (s: string) => `\x1b[1m${s}\x1b[0m`,
+  red: (s: string) => `\x1b[31m${s}\x1b[0m`,
+  green: (s: string) => `\x1b[32m${s}\x1b[0m`,
+  yellow: (s: string) => `\x1b[33m${s}\x1b[0m`,
+  blue: (s: string) => `\x1b[34m${s}\x1b[0m`,
+  magenta: (s: string) => `\x1b[35m${s}\x1b[0m`,
+  cyan: (s: string) => `\x1b[36m${s}\x1b[0m`,
+  gray: (s: string) => `\x1b[90m${s}\x1b[0m`,
+};
+
+function writeRaw(s: string) {
+  const g: any = globalThis as any;
+  if (g?.Bun?.stdout?.write) { g.Bun.stdout.write(s); return; }
+  g?.process?.stdout?.write?.(s);
+}
+
+export class Logger {
+  static info(...a: any[]) { console.log(...a); }
+  static warn(...a: any[]) { console.warn(...a); }
+  static error(...a: any[]) { console.error(...a); }
+  static debug(...a: any[]) {
+    if ((process.env.GRO_LOG_LEVEL ?? "").toUpperCase() === "DEBUG") console.log(...a);
+  }
+  static streamInfo(s: string) { writeRaw(s); }
+  static endStreamLine(suffix = "") { writeRaw(suffix + "\n"); }
+}