@ryanfw/prompt-orchestration-pipeline 1.2.2 → 1.2.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ryanfw/prompt-orchestration-pipeline",
-  "version": "1.2.2",
+  "version": "1.2.4",
   "description": "A Prompt-orchestration pipeline (POP) is a framework for building, running, and experimenting with complex chains of LLM tasks.",
   "type": "module",
   "main": "src/ui/server/index.ts",

package/src/core/config.ts

@@ -94,7 +94,7 @@ export const defaultConfig = {
   taskRunner: {
     maxRefinementAttempts: 3,
     stageTimeout: 300000,
-    llmRequestTimeout: 120000,
+    llmRequestTimeout: 3600000,
   },
   llm: {
     defaultProvider: "openai",

package/src/core/task-runner.ts

@@ -245,6 +245,16 @@ export function deriveModelKeyAndTokens(metric: Record<string, unknown>): TokenU
   return [modelKey, inputTokens, outputTokens, cost];
 }
 
+// ─── Safe clone ─────────────────────────────────────────────────────────────
+
+function safeClone<T>(value: T): T {
+  try {
+    return structuredClone(value);
+  } catch {
+    return JSON.parse(JSON.stringify(value));
+  }
+}
+
 // ─── Error normalization ──────────────────────────────────────────────────────
 
 /**
@@ -630,15 +640,18 @@ export async function runPipeline(
         // Best-effort: swallow status write errors
       }
 
-      // Clone data, flags, output via structuredClone into StageContext
+      // Clone data, flags, output into StageContext.
+      // structuredClone throws on non-cloneable values (functions, streams,
+      // class instances with internal slots). Fall back to JSON round-trip
+      // which silently strips those fields.
       const stageContext: StageContext = {
         io,
         llm,
         meta: context.meta,
-        data: structuredClone(context.data),
-        flags: structuredClone(context.flags),
+        data: safeClone(context.data),
+        flags: safeClone(context.flags),
         currentStage: stageName,
-        output: structuredClone(lastStageOutput),
+        output: safeClone(lastStageOutput),
         previousStage,
         validators: context.validators,
       };

package/src/llm/index.ts

@@ -92,43 +92,43 @@ function inferJsonFormat(options: ChatOptions): ChatOptions {
 async function callAdapter(
   options: ChatOptions,
 ): Promise<AdapterResponse> {
-  const { provider, messages, model, temperature, maxTokens, responseFormat, topP, stop, maxRetries } = options;
+  const { provider, messages, model, temperature, maxTokens, responseFormat, topP, stop, maxRetries, requestTimeoutMs } = options;
 
   switch (provider) {
     case "alibaba":
       return alibabaChat({
-        messages, model, temperature, maxTokens, responseFormat, topP, stop, maxRetries,
+        messages, model, temperature, maxTokens, responseFormat, topP, stop, maxRetries, requestTimeoutMs,
         frequencyPenalty: options.frequencyPenalty,
         presencePenalty: options.presencePenalty,
       });
     case "anthropic":
-      return anthropicChat({ messages, model, temperature, maxTokens, responseFormat, topP, stop, maxRetries });
+      return anthropicChat({ messages, model, temperature, maxTokens, responseFormat, topP, stop, maxRetries, requestTimeoutMs });
     case "openai":
       return openaiChat({
-        messages, model, temperature, maxTokens, responseFormat, topP, stop, maxRetries,
+        messages, model, temperature, maxTokens, responseFormat, topP, stop, maxRetries, requestTimeoutMs,
         seed: undefined,
         frequencyPenalty: options.frequencyPenalty,
         presencePenalty: options.presencePenalty,
       });
     case "gemini":
       return geminiChat({
-        messages, model, temperature, maxTokens, responseFormat, topP, stop, maxRetries,
+        messages, model, temperature, maxTokens, responseFormat, topP, stop, maxRetries, requestTimeoutMs,
         frequencyPenalty: options.frequencyPenalty,
         presencePenalty: options.presencePenalty,
       });
     case "deepseek":
       return deepseekChat({
-        messages, model, temperature, maxTokens, responseFormat, topP, stop, maxRetries,
+        messages, model, temperature, maxTokens, responseFormat, topP, stop, maxRetries, requestTimeoutMs,
         frequencyPenalty: options.frequencyPenalty,
         presencePenalty: options.presencePenalty,
       });
     case "moonshot":
-      return moonshotChat({ messages, model, maxTokens, responseFormat, maxRetries });
+      return moonshotChat({ messages, model, maxTokens, responseFormat, maxRetries, requestTimeoutMs });
     case "zai":
     case "zhipu":
-      return zaiChat({ messages, model, temperature, maxTokens, responseFormat, topP, stop, maxRetries });
+      return zaiChat({ messages, model, temperature, maxTokens, responseFormat, topP, stop, maxRetries, requestTimeoutMs });
     case "claudecode":
-      return claudeCodeChat({ messages, model, maxTokens, responseFormat, maxRetries });
+      return claudeCodeChat({ messages, model, maxTokens, responseFormat, maxRetries, requestTimeoutMs });
     case "mock": {
       if (!mockProvider) {
         throw new Error("No mock provider registered. Call registerMockProvider() first.");
@@ -180,7 +180,11 @@ async function writeDebugLog(options: ChatOptions, response: ChatResponse): Prom
 
 export async function chat(options: ChatOptions): Promise<ChatResponse> {
   ensureMessagesPresent(options.messages, options.provider);
-  const opts = inferJsonFormat(options);
+  const configTimeout = getConfig().taskRunner.llmRequestTimeout;
+  const opts = inferJsonFormat({
+    ...options,
+    requestTimeoutMs: options.requestTimeoutMs ?? configTimeout,
+  });
   const id = `llm-${++requestCounter}-${Date.now()}`;
   const model = opts.model ?? "";
   const startTime = Date.now();

package/src/providers/__tests__/alibaba.test.ts

@@ -4,25 +4,66 @@ import { ProviderJsonParseError } from "../types.ts";
 import type { AlibabaOptions } from "../types.ts";
 import type { Mock } from "vitest";
 
-function makeAlibabaResponse(
-  content: string,
-  promptTokens = 10,
-  completionTokens = 20,
-) {
-  return {
-    choices: [{ message: { content } }],
-    usage: {
-      prompt_tokens: promptTokens,
-      completion_tokens: completionTokens,
-      total_tokens: promptTokens + completionTokens,
+/**
+ * Creates a mock ReadableStream that yields SSE-formatted data.
+ */
+function makeSSEStream(events: string[]): ReadableStream<Uint8Array> {
+  const encoder = new TextEncoder();
+  const chunks = events.map((e) => encoder.encode(e));
+  let index = 0;
+
+  return new ReadableStream({
+    pull(controller) {
+      if (index < chunks.length) {
+        controller.enqueue(chunks[index]!);
+        index++;
+      } else {
+        controller.close();
+      }
     },
-  };
+  });
+}
+
+/**
+ * Builds SSE events for an OpenAI-compatible streaming response.
+ */
+function makeOpenAiSseEvents(
+  textChunks: string[],
+  usage?: { prompt_tokens: number; completion_tokens: number; total_tokens: number },
+): string[] {
+  const events: string[] = [];
+  for (const chunk of textChunks) {
+    events.push(
+      `data: ${JSON.stringify({ choices: [{ delta: { content: chunk } }] })}\n\n`,
+    );
+  }
+  if (usage) {
+    events.push(
+      `data: ${JSON.stringify({ choices: [{ delta: {}, finish_reason: "stop" }], usage })}\n\n`,
+    );
+  } else {
+    events.push(
+      `data: ${JSON.stringify({ choices: [{ delta: {}, finish_reason: "stop" }] })}\n\n`,
+    );
+  }
+  events.push("data: [DONE]\n\n");
+  return events;
 }
 
-function mockFetchResponse(body: unknown, status = 200) {
+function mockStreamingResponse(events: string[], status = 200) {
   return {
     ok: status >= 200 && status < 300,
     status,
+    body: makeSSEStream(events),
+    json: vi.fn(),
+    text: vi.fn(),
+  } as unknown as Response;
+}
+
+function mockErrorResponse(body: unknown, status: number) {
+  return {
+    ok: false,
+    status,
     json: vi.fn().mockResolvedValue(body),
     text: vi.fn().mockResolvedValue(JSON.stringify(body)),
   } as unknown as Response;
@@ -56,11 +97,11 @@ describe("alibabaChat", () => {
 
   it("returns parsed JSON content with usage on success", async () => {
     const jsonPayload = { result: "success", count: 42 };
-    fetchMock.mockResolvedValue(
-      mockFetchResponse(
-        makeAlibabaResponse(JSON.stringify(jsonPayload), 15, 25),
-      ),
+    const events = makeOpenAiSseEvents(
+      [JSON.stringify(jsonPayload)],
+      { prompt_tokens: 15, completion_tokens: 25, total_tokens: 40 },
     );
+    fetchMock.mockResolvedValue(mockStreamingResponse(events));
 
     const result = await alibabaChat(baseOptions);
 
@@ -70,14 +111,25 @@ describe("alibabaChat", () => {
       completion_tokens: 25,
       total_tokens: 40,
     });
-    expect(result.raw).toBeDefined();
+  });
+
+  it("sends stream: true and stream_options in request body", async () => {
+    const events = makeOpenAiSseEvents([JSON.stringify({ ok: true })]);
+    fetchMock.mockResolvedValue(mockStreamingResponse(events));
+
+    await alibabaChat(baseOptions);
+
+    const body = JSON.parse(
+      (fetchMock.mock.calls[0] as [string, RequestInit])[1].body as string,
+    );
+    expect(body.stream).toBe(true);
+    expect(body.stream_options).toEqual({ include_usage: true });
   });
 
   it("throws ProviderJsonParseError on invalid JSON when responseFormat is json_object", async () => {
     const nonJsonText = "This is plain text, not JSON at all.";
-    fetchMock.mockResolvedValue(
-      mockFetchResponse(makeAlibabaResponse(nonJsonText)),
-    );
+    const events = makeOpenAiSseEvents([nonJsonText]);
+    fetchMock.mockResolvedValue(mockStreamingResponse(events));
 
     try {
       await alibabaChat(baseOptions);
@@ -93,18 +145,15 @@ describe("alibabaChat", () => {
 
   it("retries on HTTP 500 with exponential backoff", async () => {
     const jsonPayload = { retried: true };
+    const events = makeOpenAiSseEvents([JSON.stringify(jsonPayload)]);
     fetchMock
       .mockResolvedValueOnce(
-        mockFetchResponse({ error: { message: "Server error" } }, 500),
+        mockErrorResponse({ error: { message: "Server error" } }, 500),
       )
       .mockResolvedValueOnce(
-        mockFetchResponse({ error: { message: "Server error" } }, 500),
+        mockErrorResponse({ error: { message: "Server error" } }, 500),
       )
-      .mockResolvedValueOnce(
-        mockFetchResponse(
-          makeAlibabaResponse(JSON.stringify(jsonPayload)),
-        ),
-      );
+      .mockResolvedValueOnce(mockStreamingResponse(events));
 
     const result = await alibabaChat({ ...baseOptions, maxRetries: 3 });
 
@@ -114,7 +163,7 @@ describe("alibabaChat", () => {
 
   it("does NOT retry on HTTP 401", async () => {
     fetchMock.mockResolvedValue(
-      mockFetchResponse({ error: { message: "Unauthorized" } }, 401),
+      mockErrorResponse({ error: { message: "Unauthorized" } }, 401),
     );
 
     await expect(
@@ -126,10 +175,8 @@ describe("alibabaChat", () => {
 
   it("uses ALIBABA_BASE_URL env var when set", async () => {
     process.env["ALIBABA_BASE_URL"] = "https://custom.api.example.com";
-    const jsonPayload = { ok: true };
-    fetchMock.mockResolvedValue(
-      mockFetchResponse(makeAlibabaResponse(JSON.stringify(jsonPayload))),
-    );
+    const events = makeOpenAiSseEvents([JSON.stringify({ ok: true })]);
+    fetchMock.mockResolvedValue(mockStreamingResponse(events));
 
     await alibabaChat(baseOptions);
 
@@ -140,10 +187,8 @@ describe("alibabaChat", () => {
   });
 
   it("passes frequencyPenalty and presencePenalty in request body", async () => {
-    const jsonPayload = { ok: true };
-    fetchMock.mockResolvedValue(
-      mockFetchResponse(makeAlibabaResponse(JSON.stringify(jsonPayload))),
-    );
+    const events = makeOpenAiSseEvents([JSON.stringify({ ok: true })]);
+    fetchMock.mockResolvedValue(mockStreamingResponse(events));
 
     await alibabaChat({
       ...baseOptions,
@@ -158,10 +203,9 @@ describe("alibabaChat", () => {
     expect(body.presence_penalty).toBe(0.2);
   });
 
-  it("passes an AbortSignal to fetch", async () => {
-    fetchMock.mockResolvedValue(
-      mockFetchResponse(makeAlibabaResponse(JSON.stringify({ ok: true }))),
-    );
+  it("passes an AbortSignal to fetch (IdleTimeoutController)", async () => {
+    const events = makeOpenAiSseEvents([JSON.stringify({ ok: true })]);
+    fetchMock.mockResolvedValue(mockStreamingResponse(events));
 
     await alibabaChat(baseOptions);
 
@@ -169,22 +213,9 @@ describe("alibabaChat", () => {
     expect(init.signal).toBeInstanceOf(AbortSignal);
   });
 
-  it("uses custom requestTimeoutMs for the abort signal", async () => {
-    const timeoutSpy = vi.spyOn(AbortSignal, "timeout");
-    fetchMock.mockResolvedValue(
-      mockFetchResponse(makeAlibabaResponse(JSON.stringify({ ok: true }))),
-    );
-
-    await alibabaChat({ ...baseOptions, requestTimeoutMs: 5000 });
-
-    expect(timeoutSpy).toHaveBeenCalledWith(5000);
-    timeoutSpy.mockRestore();
-  });
-
   it("sends enable_thinking true by default", async () => {
-    fetchMock.mockResolvedValue(
-      mockFetchResponse(makeAlibabaResponse(JSON.stringify({ ok: true }))),
-    );
+    const events = makeOpenAiSseEvents([JSON.stringify({ ok: true })]);
+    fetchMock.mockResolvedValue(mockStreamingResponse(events));
 
     await alibabaChat(baseOptions);
 
@@ -195,9 +226,8 @@ describe("alibabaChat", () => {
   });
 
   it("sends enable_thinking false when thinking is disabled", async () => {
-    fetchMock.mockResolvedValue(
-      mockFetchResponse(makeAlibabaResponse(JSON.stringify({ ok: true }))),
-    );
+    const events = makeOpenAiSseEvents([JSON.stringify({ ok: true })]);
+    fetchMock.mockResolvedValue(mockStreamingResponse(events));
 
     await alibabaChat({ ...baseOptions, thinking: "disabled" });
 
@@ -206,4 +236,67 @@ describe("alibabaChat", () => {
     );
     expect(body.enable_thinking).toBe(false);
   });
+
+  describe("streaming accumulation", () => {
+    it("accumulates text across multiple SSE chunks", async () => {
+      const events = [
+        'data: {"choices":[{"delta":{"content":"{\\"he"}}]}\n\n',
+        'data: {"choices":[{"delta":{"content":"llo\\":\\"world\\"}"}}]}\n\n',
+        'data: {"choices":[{"delta":{},"finish_reason":"stop"}]}\n\n',
+        "data: [DONE]\n\n",
+      ];
+
+      fetchMock.mockResolvedValue(mockStreamingResponse(events));
+
+      const result = await alibabaChat(baseOptions);
+      expect(result.content).toEqual({ hello: "world" });
+    });
+
+    it("captures usage from the final streaming chunk", async () => {
+      const events = makeOpenAiSseEvents(
+        [JSON.stringify({ ok: true })],
+        { prompt_tokens: 50, completion_tokens: 30, total_tokens: 80 },
+      );
+
+      fetchMock.mockResolvedValue(mockStreamingResponse(events));
+
+      const result = await alibabaChat(baseOptions);
+      expect(result.usage).toEqual({
+        prompt_tokens: 50,
+        completion_tokens: 30,
+        total_tokens: 80,
+      });
+    });
+
+    it("defaults usage to zeros when stream provides no usage", async () => {
+      const events = makeOpenAiSseEvents([JSON.stringify({ ok: true })]);
+
+      fetchMock.mockResolvedValue(mockStreamingResponse(events));
+
+      const result = await alibabaChat(baseOptions);
+      expect(result.usage).toEqual({
+        prompt_tokens: 0,
+        completion_tokens: 0,
+        total_tokens: 0,
+      });
+    });
+
+    it("retries on timeout then succeeds on second attempt", async () => {
+      const events = makeOpenAiSseEvents([JSON.stringify({ ok: true })]);
+
+      fetchMock
+        .mockRejectedValueOnce(
+          new DOMException("signal timed out", "TimeoutError"),
+        )
+        .mockResolvedValueOnce(mockStreamingResponse(events));
+
+      const result = await alibabaChat({
+        ...baseOptions,
+        maxRetries: 1,
+      });
+
+      expect(fetchMock).toHaveBeenCalledTimes(2);
+      expect(result.content).toEqual({ ok: true });
+    });
+  });
 });