@librechat/agents 3.3.13 → 3.4.0

package/src/llm/stream/smoother.test.ts

@@ -0,0 +1,519 @@
+import {
+  smoothStream,
+  resolveStreamDelay,
+  STREAM_ABORT_MESSAGE,
+  STREAM_CHUNK_MIN_SIZE,
+  DEFAULT_STREAM_DELAY,
+  PRODUCER_CLOSE_GRACE_MS,
+  MAX_SMOOTH_ITEM_SEGMENT_CHARS,
+  STREAM_BOUNDARY_LOOKAHEAD_CHARS,
+  findStreamChunkBoundary,
+  computeAdaptivePieceSize,
+  SMOOTH_TARGET_LATENCY_MS,
+  MAX_STREAM_QUEUE_TEXT_CHARS,
+} from './smoother';
+import type { SmoothItem, SmoothPiece } from './smoother';
+
+type Emitted = SmoothPiece & { tag: string; at: number };
+
+function makeItem(
+  tag: string,
+  text: string,
+  smooth: boolean,
+  atomic?: boolean
+): SmoothItem<Emitted> {
+  return {
+    text,
+    smooth,
+    atomic,
+    emit: (piece) => ({ ...piece, tag, at: Date.now() }),
+  };
+}
+
+async function* arraySource<T>(
+  items: SmoothItem<T>[],
+  onYield?: (index: number) => void
+): AsyncGenerator<SmoothItem<T>> {
+  for (let i = 0; i < items.length; i++) {
+    onYield?.(i);
+    yield items[i];
+  }
+}
+
+async function collect(
+  stream: AsyncGenerator<Emitted>
+): Promise<Emitted[]> {
+  const out: Emitted[] = [];
+  for await (const piece of stream) {
+    out.push(piece);
+  }
+  return out;
+}
+
+const sleep = (ms: number): Promise<void> =>
+  new Promise((resolve) => setTimeout(resolve, ms));
+
+describe('resolveStreamDelay', () => {
+  it('defaults to DEFAULT_STREAM_DELAY when unset', () => {
+    expect(resolveStreamDelay(undefined)).toBe(DEFAULT_STREAM_DELAY);
+  });
+
+  it('passes explicit values through and clamps negatives to 0', () => {
+    expect(resolveStreamDelay(10)).toBe(10);
+    expect(resolveStreamDelay(0)).toBe(0);
+    expect(resolveStreamDelay(-5)).toBe(0);
+  });
+
+  it('normalizes non-finite values to the default', () => {
+    expect(resolveStreamDelay(Number.NaN)).toBe(DEFAULT_STREAM_DELAY);
+    expect(resolveStreamDelay(Number.POSITIVE_INFINITY)).toBe(
+      DEFAULT_STREAM_DELAY
+    );
+  });
+});
+
+describe('computeAdaptivePieceSize', () => {
+  it('floors at STREAM_CHUNK_MIN_SIZE for small backlogs', () => {
+    expect(computeAdaptivePieceSize(0, 25)).toBe(STREAM_CHUNK_MIN_SIZE);
+    expect(computeAdaptivePieceSize(1, 25)).toBe(STREAM_CHUNK_MIN_SIZE);
+    expect(computeAdaptivePieceSize(39, 25)).toBe(STREAM_CHUNK_MIN_SIZE);
+  });
+
+  it('scales proportionally with backlog', () => {
+    expect(computeAdaptivePieceSize(250, 25)).toBe(25);
+    expect(computeAdaptivePieceSize(2500, 25)).toBe(250);
+    expect(computeAdaptivePieceSize(1000, 10)).toBe(40);
+  });
+
+  it('returns the full backlog when the tick is non-positive', () => {
+    expect(computeAdaptivePieceSize(500, 0)).toBe(500);
+  });
+
+  it('holds steady-state backlog near arrival_rate x target latency', () => {
+    const tick = 25;
+    const arrivalPerTick = 100;
+    let backlog = 0;
+    const observed: number[] = [];
+    for (let i = 0; i < 200; i++) {
+      backlog += arrivalPerTick;
+      backlog -= Math.min(backlog, computeAdaptivePieceSize(backlog, tick));
+      observed.push(backlog);
+    }
+    const expected = (arrivalPerTick * SMOOTH_TARGET_LATENCY_MS) / tick;
+    const tail = observed.slice(-20);
+    for (const value of tail) {
+      expect(value).toBeGreaterThan(expected * 0.8);
+      expect(value).toBeLessThan(expected * 1.2);
+    }
+  });
+
+  it('drains a stopped stream geometrically', () => {
+    const tick = 25;
+    const perTickTicks = Math.ceil(SMOOTH_TARGET_LATENCY_MS / tick);
+    let backlog = 1000;
+    let ticks = 0;
+    const snapshots: number[] = [];
+    while (backlog > 0 && ticks < 100) {
+      backlog -= Math.min(backlog, computeAdaptivePieceSize(backlog, tick));
+      ticks++;
+      snapshots.push(backlog);
+    }
+    expect(backlog).toBe(0);
+    expect(snapshots[perTickTicks - 1]).toBeLessThan(1000 * 0.4);
+    expect(ticks).toBeLessThan(50);
+  });
+});
+
+describe('findStreamChunkBoundary', () => {
+  it('extends to the next boundary character inclusive', () => {
+    expect(findStreamChunkBoundary('hello world', 4)).toBe(6);
+  });
+
+  it('returns full length when no boundary follows within the lookahead', () => {
+    expect(findStreamChunkBoundary('hello', 4)).toBe(5);
+    expect(findStreamChunkBoundary('hi', 4)).toBe(2);
+  });
+
+  it('hard-cuts boundary-free runs at the lookahead limit', () => {
+    const unbroken = 'a'.repeat(10000);
+    expect(findStreamChunkBoundary(unbroken, 4096)).toBe(
+      4096 + STREAM_BOUNDARY_LOOKAHEAD_CHARS
+    );
+  });
+});
+
+describe('smoothStream', () => {
+  it('splits a large smooth item into word-boundary pieces at cadence', async () => {
+    const text =
+      'The quick brown fox jumps over the lazy dog and keeps on running through the field. ';
+    const pieces = await collect(
+      smoothStream({
+        source: arraySource([makeItem('a', text, true)]),
+        delayMs: 5,
+      })
+    );
+
+    expect(pieces.length).toBeGreaterThan(1);
+    expect(pieces.map((p) => p.text).join('')).toBe(text);
+    expect(pieces[0].isFirst).toBe(true);
+    expect(pieces[pieces.length - 1].isLast).toBe(true);
+    for (const piece of pieces.slice(1, -1)) {
+      expect(piece.isFirst).toBe(false);
+      expect(piece.isLast).toBe(false);
+    }
+  });
+
+  it('preserves strict FIFO order across smooth and passthrough items', async () => {
+    const items = [
+      makeItem('text1', 'hello world again and again ', true),
+      makeItem('tool', '', false),
+      makeItem('text2', 'more visible text here now ', true),
+      makeItem('usage', '', false),
+    ];
+    const pieces = await collect(
+      smoothStream({ source: arraySource(items), delayMs: 3 })
+    );
+
+    const tagRuns = pieces.map((p) => p.tag);
+    const collapsed = tagRuns.filter((tag, i) => tagRuns[i - 1] !== tag);
+    expect(collapsed).toEqual(['text1', 'tool', 'text2', 'usage']);
+  });
+
+  it('emits the first visible piece without delay (TTFT)', async () => {
+    const start = Date.now();
+    const stream = smoothStream({
+      source: arraySource([makeItem('a', 'hello world and more text ', true)]),
+      delayMs: 50,
+    });
+    const first = await stream.next();
+    expect(first.done).toBe(false);
+    expect(Date.now() - start).toBeLessThan(40);
+    await stream.return(undefined);
+  });
+
+  it('never delays passthrough items even between paced text', async () => {
+    const items = [
+      makeItem('text1', 'some visible words streaming along nicely ', true),
+      makeItem('seal', '', false),
+    ];
+    const pieces = await collect(
+      smoothStream({ source: arraySource(items), delayMs: 20 })
+    );
+    const sealPiece = pieces.find((p) => p.tag === 'seal');
+    const lastText = pieces.filter((p) => p.tag === 'text1').pop();
+    expect(sealPiece).toBeDefined();
+    expect(lastText).toBeDefined();
+    expect((sealPiece as Emitted).at - (lastText as Emitted).at).toBeLessThan(
+      15
+    );
+  });
+
+  it('paces atomic items whole without splitting', async () => {
+    const text = 'this text carries logprobs and must never be split apart ';
+    const pieces = await collect(
+      smoothStream({
+        source: arraySource([
+          makeItem('lead', 'lead text goes first here ', true),
+          makeItem('logprobs', text, true, true),
+        ]),
+        delayMs: 3,
+      })
+    );
+    const atomicPieces = pieces.filter((p) => p.tag === 'logprobs');
+    expect(atomicPieces).toHaveLength(1);
+    expect(atomicPieces[0].text).toBe(text);
+    expect(atomicPieces[0].isFirst).toBe(true);
+    expect(atomicPieces[0].isLast).toBe(true);
+  });
+
+  it('marks exactly one isFirst piece per item for usage dedup', async () => {
+    const pieces = await collect(
+      smoothStream({
+        source: arraySource([
+          makeItem('a', 'first item with several words to split ', true),
+          makeItem('b', 'second item also has words to split ', true),
+        ]),
+        delayMs: 2,
+      })
+    );
+    for (const tag of ['a', 'b']) {
+      const tagged = pieces.filter((p) => p.tag === tag);
+      expect(tagged.filter((p) => p.isFirst)).toHaveLength(1);
+      expect(tagged[0].isFirst).toBe(true);
+    }
+  });
+
+  it('passes everything through unsplit and undelayed when delayMs is 0', async () => {
+    const text = 'a very long text that would normally be split into pieces ';
+    const start = Date.now();
+    const pieces = await collect(
+      smoothStream({
+        source: arraySource([
+          makeItem('a', text, true),
+          makeItem('b', '', false),
+        ]),
+        delayMs: 0,
+      })
+    );
+    expect(pieces).toHaveLength(2);
+    expect(pieces[0].text).toBe(text);
+    expect(pieces[0].isFirst).toBe(true);
+    expect(pieces[0].isLast).toBe(true);
+    expect(Date.now() - start).toBeLessThan(50);
+  });
+
+  it('coalesces token-sized items so backlog drains within the target window', async () => {
+    const tick = 10;
+    const items = Array.from({ length: 400 }, (_, i) =>
+      makeItem(`t${i}`, i % 5 === 4 ? 'x ' : 'x', true)
+    );
+    const expected = items.map((i) => i.text).join('');
+
+    const start = Date.now();
+    const pieces = await collect(
+      smoothStream({ source: arraySource(items), delayMs: tick })
+    );
+    const elapsed = Date.now() - start;
+
+    expect(pieces.map((p) => p.text).join('')).toBe(expected);
+    expect(elapsed).toBeLessThan(2000);
+    const tags = pieces.map((p) => p.tag);
+    expect(tags).toEqual([...tags].sort((a, b) => Number(a.slice(1)) - Number(b.slice(1))));
+  });
+
+  it('rethrows a producer failure even when the rejection value is nullish', async () => {
+    async function* failingSource(): AsyncGenerator<SmoothItem<Emitted>> {
+      yield makeItem('a', 'some text ', true);
+      throw undefined;
+    }
+
+    await expect(
+      collect(smoothStream({ source: failingSource(), delayMs: 1 }))
+    ).rejects.toThrow('Stream producer failed.');
+  });
+
+  it('stays fully lazy when smoothing is disabled', async () => {
+    const yielded: number[] = [];
+    const items = [
+      makeItem('a', 'first ', true),
+      makeItem('b', 'second ', true),
+      makeItem('c', 'third ', true),
+    ];
+    const stream = smoothStream({
+      source: arraySource(items, (i) => yielded.push(i)),
+      delayMs: 0,
+    });
+
+    await stream.next();
+    expect(yielded).toEqual([0]);
+    await stream.return(undefined);
+    expect(yielded).toEqual([0]);
+  });
+
+  it('segments an oversized item so pieces reassemble with exactly one isFirst/isLast', async () => {
+    const bigText = 'lorem ipsum dolor sit amet '.repeat(400);
+    const pieces = await collect(
+      smoothStream({
+        source: arraySource([makeItem('big', bigText, true)]),
+        delayMs: 1,
+      })
+    );
+
+    expect(pieces.map((p) => p.text).join('')).toBe(bigText);
+    expect(pieces.filter((p) => p.isFirst)).toHaveLength(1);
+    expect(pieces.filter((p) => p.isLast)).toHaveLength(1);
+    expect(pieces[0].isFirst).toBe(true);
+    expect(pieces[pieces.length - 1].isLast).toBe(true);
+  });
+
+  it('bounds pieces of boundary-free runs at the segment cap plus lookahead', async () => {
+    const unbroken = 'x'.repeat(20000);
+    const pieces = await collect(
+      smoothStream({
+        source: arraySource([makeItem('blob', unbroken, true)]),
+        delayMs: 1,
+      })
+    );
+
+    expect(pieces.map((p) => p.text).join('')).toBe(unbroken);
+    const maxPiece = Math.max(...pieces.map((p) => p.text.length));
+    expect(maxPiece).toBeLessThanOrEqual(
+      MAX_SMOOTH_ITEM_SEGMENT_CHARS + STREAM_BOUNDARY_LOOKAHEAD_CHARS
+    );
+  });
+
+  it('skips empty-text smooth items entirely', async () => {
+    const pieces = await collect(
+      smoothStream({
+        source: arraySource([
+          makeItem('empty', '', true),
+          makeItem('real', 'actual text ', true),
+        ]),
+        delayMs: 3,
+      })
+    );
+    expect(pieces.every((p) => p.tag === 'real')).toBe(true);
+    expect(pieces.length).toBeGreaterThan(0);
+  });
+
+  it('wakes a consumer parked on an empty queue when the signal aborts', async () => {
+    const controller = new AbortController();
+    const abortUpstream = jest.fn();
+    async function* stalledSource(): AsyncGenerator<SmoothItem<Emitted>> {
+      yield makeItem('a', 'first text ', true);
+      await new Promise<void>(() => undefined);
+    }
+
+    const stream = smoothStream({
+      source: stalledSource(),
+      delayMs: 5,
+      signal: controller.signal,
+      abortUpstream,
+    });
+
+    let next = await stream.next();
+    while (next.done !== true && (next.value as Emitted).isLast !== true) {
+      next = await stream.next();
+    }
+
+    const start = Date.now();
+    const parked = stream.next();
+    setTimeout(() => controller.abort(), 20);
+
+    await expect(parked).rejects.toThrow(STREAM_ABORT_MESSAGE);
+    expect(Date.now() - start).toBeLessThan(PRODUCER_CLOSE_GRACE_MS + 1500);
+    expect(abortUpstream).toHaveBeenCalled();
+  });
+
+  it('completes an early break promptly even when the source is stalled', async () => {
+    async function* stalledSource(): AsyncGenerator<SmoothItem<Emitted>> {
+      yield makeItem('a', 'first text ', true);
+      await new Promise<void>(() => undefined);
+    }
+
+    const stream = smoothStream({ source: stalledSource(), delayMs: 5 });
+    await stream.next();
+
+    const start = Date.now();
+    const closed = (async (): Promise<boolean> => {
+      await stream.return(undefined);
+      return true;
+    })();
+    const result = await Promise.race([
+      closed,
+      sleep(PRODUCER_CLOSE_GRACE_MS + 2000).then(() => false),
+    ]);
+    expect(result).toBe(true);
+    expect(Date.now() - start).toBeLessThan(PRODUCER_CLOSE_GRACE_MS + 1500);
+  });
+
+  it('throws the canonical abort message when aborted during the pacing sleep', async () => {
+    const controller = new AbortController();
+    const abortUpstream = jest.fn();
+    const stream = smoothStream({
+      source: arraySource([
+        makeItem('a', 'plenty of text to keep the consumer pacing along ', true),
+      ]),
+      delayMs: 50,
+      signal: controller.signal,
+      abortUpstream,
+    });
+
+    const first = await stream.next();
+    expect(first.done).toBe(false);
+    setTimeout(() => controller.abort(), 10);
+
+    await expect(async () => {
+      await collect(stream as AsyncGenerator<Emitted>);
+    }).rejects.toThrow(STREAM_ABORT_MESSAGE);
+    expect(abortUpstream).toHaveBeenCalled();
+  });
+
+  it('parks the producer at the text capacity and resumes during consumer pacing', async () => {
+    const bigText = 'word '.repeat(
+      Math.ceil(MAX_STREAM_QUEUE_TEXT_CHARS / 5) + 200
+    );
+    const yielded: number[] = [];
+    const items = [
+      makeItem('a', bigText, true),
+      makeItem('b', bigText, true),
+      makeItem('c', 'tail text ', true),
+    ];
+
+    const stream = smoothStream({
+      source: arraySource(items, (i) => yielded.push(i)),
+      delayMs: 1,
+    });
+
+    const first = await stream.next();
+    expect(first.done).toBe(false);
+    /** Segmented admission parks the producer mid-way through the first
+     * oversized item — later items are not read ahead past the text cap. */
+    expect(yielded).toContain(0);
+    expect(yielded).not.toContain(2);
+
+    const rest = await collect(stream as AsyncGenerator<Emitted>);
+    expect(yielded).toContain(1);
+    expect(yielded).toContain(2);
+    const all = [first.value as Emitted, ...rest];
+    expect(all.map((p) => p.text).join('')).toBe(bigText + bigText + 'tail text ');
+  });
+
+  it('closes a parked producer promptly after an early consumer break', async () => {
+    const bigText = 'word '.repeat(MAX_STREAM_QUEUE_TEXT_CHARS);
+    let sourceFinallyRan = false;
+    async function* source(): AsyncGenerator<SmoothItem<Emitted>> {
+      try {
+        yield makeItem('a', bigText, true);
+        yield makeItem('b', bigText, true);
+        yield makeItem('c', bigText, true);
+      } finally {
+        sourceFinallyRan = true;
+      }
+    }
+
+    const abortUpstream = jest.fn();
+    const stream = smoothStream({
+      source: source(),
+      delayMs: 25,
+      abortUpstream,
+    });
+
+    await stream.next();
+    await stream.next();
+
+    const closed = (async (): Promise<boolean> => {
+      await stream.return(undefined);
+      return true;
+    })();
+    const result = await Promise.race([
+      closed,
+      sleep(1000).then(() => false),
+    ]);
+    expect(result).toBe(true);
+    expect(abortUpstream).toHaveBeenCalled();
+    expect(sourceFinallyRan).toBe(true);
+  });
+
+  it('keeps render lag bounded near the target latency under a fast big-chunk producer', async () => {
+    const tick = 5;
+    const chunk = 'lorem ipsum dolor sit amet consectetur adipiscing elit '.repeat(2);
+    let producerDoneAt = 0;
+    async function* fastSource(): AsyncGenerator<SmoothItem<Emitted>> {
+      for (let i = 0; i < 12; i++) {
+        yield makeItem(`c${i}`, chunk, true);
+        await sleep(10);
+      }
+      producerDoneAt = Date.now();
+    }
+
+    const pieces = await collect(
+      smoothStream({ source: fastSource(), delayMs: tick })
+    );
+    const renderDoneAt = pieces[pieces.length - 1].at;
+    const lag = renderDoneAt - producerDoneAt;
+
+    expect(pieces.map((p) => p.text).join('')).toBe(chunk.repeat(12));
+    expect(lag).toBeLessThan(SMOOTH_TARGET_LATENCY_MS * 2.5);
+  });
+});