@startsimpli/llm 0.4.20 → 0.4.22

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@startsimpli/llm",
-  "version": "0.4.20",
+  "version": "0.4.22",
   "description": "Generic LLM service infrastructure — provider abstraction, retry logic, JSON extraction",
   "main": "./src/index.ts",
   "types": "./src/index.ts",

package/src/__tests__/openai-provider.test.ts

@@ -144,6 +144,21 @@ describe('adapting to what a model refuses', () => {
     expect(sent).toHaveLength(3);
   });
 
+  it('parallel first calls all succeed: a lesson learned by one request applies to the others in flight', async () => {
+    // Regression: crochet's assembly pipeline fires its connection calls in
+    // parallel. All of them sent 'none' before any answer; the first 400 taught
+    // the provider 'low', and the others saw that lesson as already learned and
+    // threw instead of retrying (gpt-6-astra: 12 of 12 connections failed).
+    const p = provider('gpt-6.1-sol');
+    const effortErr = { code: 'unsupported_value', param: 'reasoning_effort', message: "Unsupported value: 'reasoning_effort' does not support 'none' with this model. Supported values are: 'low', 'medium', 'high', and 'xhigh'." };
+    reject(effortErr);
+    reject(effortErr);
+    reject(effortErr);
+    const results = await Promise.all([1, 2, 3].map(() => p.generate('u', 's', { reasoningEffort: 'none' })));
+    expect(results.every((r) => r.content === '{"ok":true}')).toBe(true);
+    expect(sent.slice(3).every((b) => b.reasoning_effort === 'low')).toBe(true);
+  });
+
   it('adapts on the tool-calling path too', async () => {
     const p = provider('gpt-6-sol');
     reject({ code: 'unsupported_value', param: 'temperature', message: "Unsupported value: 'temperature' does not support 0.2 with this model. Only the default (1) value is supported." });
@@ -197,3 +212,56 @@ describe('response provenance', () => {
     expect(res.model).toBe('server-reported-model');
   });
 });
+
+/** A fetch Response whose body streams OpenAI SSE chunks. */
+function sse(chunks: object[], split = 7) {
+  const text = chunks.map((c) => `data: ${JSON.stringify(c)}\n\n`).join('') + 'data: [DONE]\n\n';
+  // Deliver in small arbitrary slices so the parser must buffer partial lines.
+  const parts: Uint8Array[] = [];
+  const enc = new TextEncoder();
+  for (let i = 0; i < text.length; i += split) parts.push(enc.encode(text.slice(i, i + split)));
+  const body = new ReadableStream<Uint8Array>({ start(c) { parts.forEach((p) => c.enqueue(p)); c.close(); } });
+  return { ok: true, status: 200, body } as unknown as Response;
+}
+
+describe('generateStream', () => {
+  const delta = (content: string) => ({ model: 'gpt-6-astra', choices: [{ index: 0, delta: { content }, finish_reason: null }] });
+
+  it('calls onDelta with each piece of content in order and returns the whole response', async () => {
+    fetchMock.mockImplementationOnce(async (_u: unknown, init: RequestInit) => {
+      sent.push(JSON.parse(String(init.body)) as Sent);
+      return sse([
+        delta('{"pie'), delta('ces":[{"name":"He'), delta('ad"}]}'),
+        { model: 'gpt-6-astra', choices: [{ index: 0, delta: {}, finish_reason: 'stop' }] },
+        { model: 'gpt-6-astra', choices: [], usage: { prompt_tokens: 9, completion_tokens: 5, total_tokens: 14 } },
+      ]);
+    });
+    const seen: string[] = [];
+    const res = await provider('gpt-6-astra').generateStream!('u', 's', (d) => seen.push(d));
+    expect(seen).toEqual(['{"pie', 'ces":[{"name":"He', 'ad"}]}']);
+    expect(res.content).toBe('{"pieces":[{"name":"Head"}]}');
+    expect(res.model).toBe('gpt-6-astra');
+    expect(res.provider).toBe('openai');
+    expect(res.finishReason).toBe('stop');
+    expect(res.usage).toEqual({ promptTokens: 9, completionTokens: 5, totalTokens: 14 });
+    const body = sent[0] as Sent & { stream?: boolean; stream_options?: unknown };
+    expect(body.stream).toBe(true);
+    expect(body.stream_options).toEqual({ include_usage: true });
+  });
+
+  it('adapts to a refused parameter before the stream starts', async () => {
+    reject({ code: 'unsupported_value', param: 'temperature', message: "Unsupported value: 'temperature' does not support 0.2 with this model. Only the default (1) value is supported." });
+    fetchMock.mockImplementationOnce(async (_u: unknown, init: RequestInit) => {
+      sent.push(JSON.parse(String(init.body)) as Sent);
+      return sse([delta('{"ok":true}')]);
+    });
+    const res = await provider('gpt-6-sol').generateStream!('u', 's', () => {});
+    expect(res.content).toBe('{"ok":true}');
+    expect(sent[1]!.temperature).toBeUndefined();
+  });
+
+  it('throws on an empty stream, like generate() does on empty content', async () => {
+    fetchMock.mockImplementationOnce(async () => sse([{ model: 'm', choices: [{ index: 0, delta: {}, finish_reason: 'length' }] }]));
+    await expect(provider('gpt-6-astra').generateStream!('u', 's', () => {})).rejects.toThrow(/Empty response/);
+  });
+});

package/src/providers/interface.ts

@@ -35,6 +35,19 @@ export interface ILLMProvider {
     options?: GenerateOptions
   ): Promise<LLMResponse>;
 
+  /**
+   * Optional STREAMING generate: same contract as generate(), and calls
+   * `onDelta` with each chunk of content as the model writes it, so a caller
+   * can show progress during a long generation. Callers should fall back to
+   * generate() when a provider doesn't implement it.
+   */
+  generateStream?(
+    userPrompt: string,
+    systemPrompt: string,
+    onDelta: (text: string) => void,
+    options?: GenerateOptions
+  ): Promise<LLMResponse>;
+
   /**
    * Optional NATIVE (structured) tool-calling. Providers backed by a real tools
    * API (OpenAI functions, Anthropic tool blocks) implement this so the agent

package/src/providers/openai.ts

@@ -268,26 +268,133 @@ export class OpenAIProvider implements ILLMProvider {
     sent: { effort?: ReasoningEffort; maxTokens: number },
     signal: AbortSignal,
   ): Promise<T> {
+    const response = await this.postAdaptivelyRaw(model, build, sent, signal);
+    return (await response.json()) as T;
+  }
+
+  /** postAdaptively, returning the successful Response unread (for streaming). */
+  private async postAdaptivelyRaw(
+    model: string,
+    build: (q: ModelQuirks) => object,
+    sent: { effort?: ReasoningEffort; maxTokens: number },
+    signal: AbortSignal,
+  ): Promise<Response> {
     const q = this.quirksFor(model);
     for (let attempt = 0; ; attempt++) {
+      const body = JSON.stringify(build(q));
       const response = await fetch(`${this.config.baseUrl}/chat/completions`, {
         method: 'POST',
         headers: {
           'Content-Type': 'application/json',
           Authorization: `Bearer ${this.config.apiKey}`,
         },
-        body: JSON.stringify(build(q)),
+        body,
         signal,
       });
-      if (response.ok) return (await response.json()) as T;
+      if (response.ok) return response;
       const errorData = (await response.json().catch(() => ({
         error: { message: 'Unknown error' },
       }))) as OpenAIErrorResponse;
-      if (response.status === 400 && attempt < MAX_ADAPTIVE_RETRIES && learnFromError(q, errorData.error, sent)) continue;
+      if (response.status === 400 && attempt < MAX_ADAPTIVE_RETRIES) {
+        const learned = learnFromError(q, errorData.error, sent);
+        // Calls to a new model often go out in parallel: by the time this one
+        // fails, another may already have learned the lesson, so retry whenever
+        // the request would now be built differently from the one just sent.
+        if (learned || JSON.stringify(build(q)) !== body) continue;
+      }
       throw this.handleApiError(response.status, errorData);
     }
   }
 
+  async generateStream(
+    userPrompt: string,
+    systemPrompt: string,
+    onDelta: (text: string) => void,
+    options?: GenerateOptions
+  ): Promise<LLMResponse> {
+    if (!this.isAvailable()) {
+      throw new LLMProviderError('OpenAI API key not configured', 'AUTHENTICATION_ERROR', false, undefined, this.name);
+    }
+    const model = options?.model ?? this.config.defaultModel;
+    const temperature = options?.temperature ?? this.config.defaultTemperature;
+    const maxTokens = options?.maxTokens ?? this.config.defaultMaxTokens;
+    const effort = options?.reasoningEffort ?? this.config.defaultReasoningEffort;
+    const timeout = options?.timeout ?? this.config.timeoutMs;
+    const base = {
+      model,
+      messages: [
+        { role: 'system' as const, content: systemPrompt },
+        { role: 'user' as const, content: userPrompt },
+      ],
+      response_format: { type: 'json_object' as const },
+      stream: true,
+      stream_options: { include_usage: true },
+    };
+
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), timeout);
+    try {
+      const response = await this.postAdaptivelyRaw(model, (q) => ({
+        ...base,
+        ...samplingParams(q, temperature, maxTokens, effort),
+      }), { effort, maxTokens }, controller.signal);
+
+      let content = '';
+      let reportedModel = model;
+      let finishReason: string | undefined;
+      let responseId: string | undefined;
+      const usage = { promptTokens: 0, completionTokens: 0, totalTokens: 0 };
+      const reader = response.body!.getReader();
+      const decoder = new TextDecoder();
+      let buffer = '';
+      for (;;) {
+        const { done, value } = await reader.read();
+        if (done) break;
+        buffer += decoder.decode(value, { stream: true });
+        let nl: number;
+        while ((nl = buffer.indexOf('\n')) >= 0) {
+          const line = buffer.slice(0, nl).trim();
+          buffer = buffer.slice(nl + 1);
+          if (!line.startsWith('data:')) continue;
+          const payload = line.slice(5).trim();
+          if (payload === '[DONE]') continue;
+          const chunk = JSON.parse(payload) as {
+            id?: string;
+            model?: string;
+            choices?: Array<{ delta?: { content?: string | null }; finish_reason?: string | null }>;
+            usage?: { prompt_tokens: number; completion_tokens: number; total_tokens: number };
+          };
+          if (chunk.id) responseId = chunk.id;
+          if (chunk.model) reportedModel = chunk.model;
+          const choice = chunk.choices?.[0];
+          if (choice?.finish_reason) finishReason = choice.finish_reason;
+          const delta = choice?.delta?.content;
+          if (delta) {
+            content += delta;
+            onDelta(delta);
+          }
+          if (chunk.usage) {
+            usage.promptTokens = chunk.usage.prompt_tokens;
+            usage.completionTokens = chunk.usage.completion_tokens;
+            usage.totalTokens = chunk.usage.total_tokens;
+          }
+        }
+      }
+      clearTimeout(timeoutId);
+      if (!content) {
+        throw new LLMProviderError('Empty response from OpenAI', 'UNKNOWN_ERROR', true, undefined, this.name);
+      }
+      return { content, usage, model: reportedModel, provider: this.name, responseId, finishReason };
+    } catch (error) {
+      clearTimeout(timeoutId);
+      if (error instanceof LLMProviderError) throw error;
+      if (error instanceof Error && error.name === 'AbortError') {
+        throw new LLMProviderError(`Request timed out after ${timeout}ms`, 'TIMEOUT', true, undefined, this.name);
+      }
+      throw new LLMProviderError(error instanceof Error ? error.message : 'Unknown error', 'UNKNOWN_ERROR', true, undefined, this.name);
+    }
+  }
+
   isAvailable(): boolean {
     return Boolean(this.config.apiKey);
   }