@librechat/agents 3.1.71-dev.0 → 3.1.71

package/src/tools/__tests__/annotateMessagesForLLM.test.ts

@@ -0,0 +1,479 @@
+import { AIMessage, HumanMessage, ToolMessage } from '@langchain/core/messages';
+import { describe, it, expect } from '@jest/globals';
+import {
+  annotateMessagesForLLM,
+  ToolOutputReferenceRegistry,
+  TOOL_OUTPUT_REF_KEY,
+  TOOL_OUTPUT_UNRESOLVED_KEY,
+} from '../toolOutputReferences';
+
+function makeToolMessage(fields: {
+  content: ToolMessage['content'];
+  name?: string;
+  tool_call_id?: string;
+  status?: 'success' | 'error';
+  artifact?: unknown;
+  additional_kwargs?: Record<string, unknown>;
+}): ToolMessage {
+  return new ToolMessage({
+    name: fields.name ?? 'echo',
+    tool_call_id: fields.tool_call_id ?? 'tc1',
+    status: fields.status ?? 'success',
+    artifact: fields.artifact,
+    additional_kwargs: fields.additional_kwargs,
+    content: fields.content,
+  });
+}
+
+describe('annotateMessagesForLLM', () => {
+  it('returns the input array reference when registry is undefined', () => {
+    const messages = [
+      new HumanMessage('hi'),
+      makeToolMessage({
+        content: 'data',
+        additional_kwargs: { _refKey: 'tool0turn0' },
+      }),
+    ];
+    const out = annotateMessagesForLLM(messages, undefined, 'r1');
+    expect(out).toBe(messages);
+  });
+
+  it('does not iterate any message when the feature is disabled (registry undefined)', () => {
+    /**
+     * Hard guarantee: when the host hasn't enabled
+     * `RunConfig.toolOutputReferences`, calling
+     * `annotateMessagesForLLM` must short-circuit at O(1) without
+     * touching a single ToolMessage. We assert by spying on
+     * `_getType` — the first per-message call inside the loop — and
+     * confirming it was never invoked.
+     */
+    const messages = [
+      makeToolMessage({ content: 'a' }),
+      makeToolMessage({ content: 'b' }),
+      makeToolMessage({
+        content: 'c',
+        additional_kwargs: { _refKey: 'tool0turn0' },
+      }),
+    ];
+    const spies = messages.map((m) => jest.spyOn(m, '_getType'));
+    const out = annotateMessagesForLLM(messages, undefined, undefined);
+    expect(out).toBe(messages);
+    for (const spy of spies) {
+      expect(spy).not.toHaveBeenCalled();
+      spy.mockRestore();
+    }
+  });
+
+  it('returns the input array reference when no ToolMessage carries metadata', () => {
+    const registry = new ToolOutputReferenceRegistry();
+    registry.set('r1', 'tool0turn0', 'stored');
+    const messages = [
+      new HumanMessage('hi'),
+      makeToolMessage({ content: 'data' }),
+      new AIMessage('answer'),
+    ];
+    const out = annotateMessagesForLLM(messages, registry, 'r1');
+    expect(out).toBe(messages);
+  });
+
+  it('annotates string content when _refKey is live in the registry', () => {
+    const registry = new ToolOutputReferenceRegistry();
+    registry.set('r1', 'tool0turn0', 'stored-raw');
+    const tm = makeToolMessage({
+      content: 'output',
+      additional_kwargs: { _refKey: 'tool0turn0' },
+    });
+    const out = annotateMessagesForLLM([tm], registry, 'r1');
+    expect(out[0].content).toBe('[ref: tool0turn0]\noutput');
+    expect(tm.content).toBe('output');
+    expect(out).not.toBe([tm]);
+    expect(out[0]).not.toBe(tm);
+  });
+
+  it('leaves content untouched but strips framework metadata when _refKey is stale', () => {
+    /**
+     * Stale `_refKey` (not in registry) doesn't trigger annotation,
+     * but the message still gets projected so framework keys are
+     * removed from `additional_kwargs` before the bytes leave for the
+     * provider. This protects against custom or future serializers
+     * that transmit `additional_kwargs`.
+     */
+    const registry = new ToolOutputReferenceRegistry();
+    const tm = makeToolMessage({
+      content: 'output',
+      additional_kwargs: { _refKey: 'tool0turn0', userField: 'preserved' },
+    });
+    const out = annotateMessagesForLLM([tm], registry, 'r1');
+    expect(out[0].content).toBe('output');
+    expect(out[0]).not.toBe(tm);
+    const projectedKwargs = (out[0] as ToolMessage).additional_kwargs;
+    expect(projectedKwargs._refKey).toBeUndefined();
+    expect(projectedKwargs.userField).toBe('preserved');
+    expect(tm.additional_kwargs._refKey).toBe('tool0turn0');
+  });
+
+  it('always applies _unresolvedRefs even when there is no registry entry', () => {
+    const registry = new ToolOutputReferenceRegistry();
+    const tm = makeToolMessage({
+      content: 'output',
+      additional_kwargs: { _unresolvedRefs: ['tool9turn9'] },
+    });
+    const out = annotateMessagesForLLM([tm], registry, 'r1');
+    expect(out[0].content).toBe('output\n[unresolved refs: tool9turn9]');
+  });
+
+  it('injects _ref into JSON-object string content', () => {
+    const registry = new ToolOutputReferenceRegistry();
+    registry.set('r1', 'tool0turn0', '{"a":1}');
+    const tm = makeToolMessage({
+      content: '{"a":1,"b":"x"}',
+      additional_kwargs: { _refKey: 'tool0turn0' },
+    });
+    const out = annotateMessagesForLLM([tm], registry, 'r1');
+    const parsed = JSON.parse(out[0].content as string);
+    expect(parsed[TOOL_OUTPUT_REF_KEY]).toBe('tool0turn0');
+    expect(parsed.a).toBe(1);
+    expect(parsed.b).toBe('x');
+  });
+
+  it('injects both _ref and _unresolved_refs into JSON-object content', () => {
+    /**
+     * Combined path: when a ToolMessage carries both a live `_refKey`
+     * and unresolved-ref hints, JSON-object content should receive
+     * both `_ref` and `_unresolved_refs` fields rather than falling
+     * back to the prefix/trailer form. Exercises
+     * `annotateToolOutputWithReference`'s collision-detection logic
+     * through the projection entry point.
+     */
+    const registry = new ToolOutputReferenceRegistry();
+    registry.set('r1', 'tool0turn0', '{"a":1}');
+    const tm = makeToolMessage({
+      content: '{"a":1,"b":"x"}',
+      additional_kwargs: {
+        _refKey: 'tool0turn0',
+        _unresolvedRefs: ['tool9turn9'],
+      },
+    });
+    const out = annotateMessagesForLLM([tm], registry, 'r1');
+    const parsed = JSON.parse(out[0].content as string);
+    expect(parsed[TOOL_OUTPUT_REF_KEY]).toBe('tool0turn0');
+    expect(parsed[TOOL_OUTPUT_UNRESOLVED_KEY]).toEqual(['tool9turn9']);
+    expect(parsed.a).toBe(1);
+    expect(parsed.b).toBe('x');
+  });
+
+  it('uses [ref: …] prefix for non-JSON string content', () => {
+    const registry = new ToolOutputReferenceRegistry();
+    registry.set('r1', 'tool0turn0', 'raw');
+    const tm = makeToolMessage({
+      content: 'plain output',
+      additional_kwargs: { _refKey: 'tool0turn0' },
+    });
+    const out = annotateMessagesForLLM([tm], registry, 'r1');
+    expect(out[0].content).toBe('[ref: tool0turn0]\nplain output');
+  });
+
+  it('prepends an unresolved-refs warning text block to multi-part content', () => {
+    const registry = new ToolOutputReferenceRegistry();
+    const tm = makeToolMessage({
+      content: [
+        { type: 'text', text: 'data' },
+        { type: 'image_url', image_url: { url: 'data:...' } },
+      ] as unknown as ToolMessage['content'],
+      additional_kwargs: { _unresolvedRefs: ['tool9turn9'] },
+    });
+    const out = annotateMessagesForLLM([tm], registry, 'r1');
+    const blocks = out[0].content as Array<{ type: string; text?: string }>;
+    expect(blocks).toHaveLength(3);
+    expect(blocks[0].type).toBe('text');
+    expect(blocks[0].text).toBe('[unresolved refs: tool9turn9]');
+    expect(blocks[1].type).toBe('text');
+    expect(blocks[1].text).toBe('data');
+    expect(blocks[2].type).toBe('image_url');
+  });
+
+  it('preserves artifact on the projected ToolMessage', () => {
+    /**
+     * Hosts attach `artifact` to ToolMessages via the
+     * `content_and_artifact` response format (e.g. code execution
+     * sessions, MCP tools that return structured side-data). The
+     * projection must round-trip the artifact untouched so downstream
+     * consumers (audit logs, code-session tracking) keep working.
+     */
+    const registry = new ToolOutputReferenceRegistry();
+    registry.set('r1', 'tool0turn0', 'raw');
+    const artifact = {
+      session_id: 'abc',
+      files: [{ id: 'f1', name: 'a.txt' }],
+    };
+    const tm = makeToolMessage({
+      content: 'output',
+      artifact,
+      additional_kwargs: { _refKey: 'tool0turn0' },
+    });
+    const out = annotateMessagesForLLM([tm], registry, 'r1');
+    const projected = out[0] as ToolMessage;
+    expect(projected.artifact).toBe(artifact);
+    expect(projected.content).toBe('[ref: tool0turn0]\noutput');
+  });
+
+  it('does not mutate the original ToolMessage instance or its content', () => {
+    const registry = new ToolOutputReferenceRegistry();
+    registry.set('r1', 'tool0turn0', 'raw');
+    const tm = makeToolMessage({
+      content: 'output',
+      additional_kwargs: { _refKey: 'tool0turn0' },
+    });
+    const originalContent = tm.content;
+    const originalKwargs = { ...tm.additional_kwargs };
+    annotateMessagesForLLM([tm], registry, 'r1');
+    expect(tm.content).toBe(originalContent);
+    expect(tm.additional_kwargs).toEqual(originalKwargs);
+  });
+
+  it('strips framework ref metadata from the projected additional_kwargs but preserves other fields', () => {
+    /**
+     * Defensive: even though LangChain's standard provider serializers
+     * do not transmit `additional_kwargs`, a custom adapter or future
+     * LangChain change could. Strip our three framework-owned keys on
+     * the projection so the metadata never reaches the wire under any
+     * serializer behavior. Non-framework fields stay put.
+     */
+    const registry = new ToolOutputReferenceRegistry();
+    registry.set('r1', 'tool0turn0', 'raw');
+    const tm = makeToolMessage({
+      content: 'output',
+      additional_kwargs: {
+        _refKey: 'tool0turn0',
+        _refScope: 'r1',
+        _unresolvedRefs: ['tool9turn9'],
+        someOtherField: 'preserved',
+      },
+    });
+    const out = annotateMessagesForLLM([tm], registry, 'r1');
+    const projected = out[0] as ToolMessage;
+    expect(projected.additional_kwargs._refKey).toBeUndefined();
+    expect(projected.additional_kwargs._refScope).toBeUndefined();
+    expect(projected.additional_kwargs._unresolvedRefs).toBeUndefined();
+    expect(projected.additional_kwargs.someOtherField).toBe('preserved');
+    expect(tm.additional_kwargs._refKey).toBe('tool0turn0');
+    expect(tm.additional_kwargs._refScope).toBe('r1');
+  });
+
+  it('leaves additional_kwargs empty when stripping removed every framework key', () => {
+    /**
+     * `stripFrameworkRefMetadata` returns `undefined` when no non-
+     * framework keys remain, and the LangChain `ToolMessage`
+     * constructor normalizes that to `{}` — so the projected message
+     * exposes an empty object, not the original kwargs.
+     */
+    const registry = new ToolOutputReferenceRegistry();
+    registry.set('r1', 'tool0turn0', 'raw');
+    const tm = makeToolMessage({
+      content: 'output',
+      additional_kwargs: {
+        _refKey: 'tool0turn0',
+        _refScope: 'r1',
+      },
+    });
+    const out = annotateMessagesForLLM([tm], registry, 'r1');
+    const projected = out[0] as ToolMessage;
+    expect(projected.additional_kwargs).toEqual({});
+  });
+
+  it('passes through non-ToolMessages unchanged in the projected array', () => {
+    const registry = new ToolOutputReferenceRegistry();
+    registry.set('r1', 'tool0turn0', 'raw');
+    const human = new HumanMessage('hi');
+    const ai = new AIMessage('answer');
+    const tm = makeToolMessage({
+      content: 'output',
+      additional_kwargs: { _refKey: 'tool0turn0' },
+    });
+    const out = annotateMessagesForLLM([human, ai, tm], registry, 'r1');
+    expect(out[0]).toBe(human);
+    expect(out[1]).toBe(ai);
+    expect(out[2]).not.toBe(tm);
+  });
+
+  it('projects (to strip metadata) but does not annotate when only stale _refKey is present', () => {
+    const registry = new ToolOutputReferenceRegistry();
+    registry.set('r1', 'tool1turn0', 'somethingelse');
+    const tm = makeToolMessage({
+      content: 'output',
+      additional_kwargs: { _refKey: 'tool0turn0' },
+    });
+    const messages = [tm];
+    const out = annotateMessagesForLLM(messages, registry, 'r1');
+    expect(out).not.toBe(messages);
+    expect(out[0].content).toBe('output');
+    expect((out[0] as ToolMessage).additional_kwargs._refKey).toBeUndefined();
+  });
+
+  it('returns the input array reference-equal when no ToolMessage carries any framework metadata', () => {
+    const registry = new ToolOutputReferenceRegistry();
+    registry.set('r1', 'tool0turn0', 'stored');
+    const messages = [
+      new HumanMessage('hi'),
+      makeToolMessage({
+        content: 'output',
+        additional_kwargs: { unrelated: 'value' },
+      }),
+      new AIMessage('answer'),
+    ];
+    const out = annotateMessagesForLLM(messages, registry, 'r1');
+    expect(out).toBe(messages);
+  });
+
+  it('annotates only the live ref when both ref and unresolved are present', () => {
+    const registry = new ToolOutputReferenceRegistry();
+    registry.set('r1', 'tool0turn0', 'raw');
+    const tm = makeToolMessage({
+      content: 'output',
+      additional_kwargs: {
+        _refKey: 'tool0turn0',
+        _unresolvedRefs: ['tool9turn9'],
+      },
+    });
+    const out = annotateMessagesForLLM([tm], registry, 'r1');
+    expect(out[0].content).toBe(
+      '[ref: tool0turn0]\noutput\n[unresolved refs: tool9turn9]'
+    );
+  });
+
+  it('uses _refScope for the registry lookup when present, ignoring runId', () => {
+    /**
+     * Anonymous ToolNode batches register under a synthetic scope
+     * (`\0anon-<n>`) that `config.configurable.run_id` cannot recover.
+     * The transform must follow the message-stamped `_refScope`
+     * instead of the config-derived runId.
+     */
+    const registry = new ToolOutputReferenceRegistry();
+    const anonScope = '\0anon-3';
+    registry.set(anonScope, 'tool0turn0', 'raw');
+
+    const tm = makeToolMessage({
+      content: 'output',
+      additional_kwargs: {
+        _refKey: 'tool0turn0',
+        _refScope: anonScope,
+      },
+    });
+    const out = annotateMessagesForLLM([tm], registry, undefined);
+    expect(out[0].content).toBe('[ref: tool0turn0]\noutput');
+  });
+
+  it('falls back to runId when _refScope is absent (legacy / pre-scope messages)', () => {
+    const registry = new ToolOutputReferenceRegistry();
+    registry.set('r1', 'tool0turn0', 'raw');
+    const tm = makeToolMessage({
+      content: 'output',
+      additional_kwargs: { _refKey: 'tool0turn0' },
+    });
+    const out = annotateMessagesForLLM([tm], registry, 'r1');
+    expect(out[0].content).toBe('[ref: tool0turn0]\noutput');
+  });
+
+  it('coerces a non-array _unresolvedRefs to empty without throwing', () => {
+    /**
+     * Defensive against malformed hydrated messages:
+     * `additional_kwargs._unresolvedRefs` is untyped at the LangChain
+     * layer, so a persisted message could carry a string/object/null
+     * by mistake. The transform must not crash the run on
+     * `.length` / `.join` — coerce to an empty list, strip the
+     * malformed key, and proceed.
+     */
+    const registry = new ToolOutputReferenceRegistry();
+    const tm = makeToolMessage({
+      content: 'output',
+      additional_kwargs: {
+        _unresolvedRefs: 'tool9turn9' as unknown as string[],
+      },
+    });
+    const out = annotateMessagesForLLM([tm], registry, 'r1');
+    expect(out[0].content).toBe('output');
+    expect(
+      (out[0] as ToolMessage).additional_kwargs._unresolvedRefs
+    ).toBeUndefined();
+  });
+
+  it('filters non-string entries out of _unresolvedRefs', () => {
+    const registry = new ToolOutputReferenceRegistry();
+    const tm = makeToolMessage({
+      content: 'output',
+      additional_kwargs: {
+        _unresolvedRefs: [
+          'tool9turn9',
+          42,
+          null,
+          { not: 'a string' },
+          'tool8turn8',
+        ] as unknown as string[],
+      },
+    });
+    const out = annotateMessagesForLLM([tm], registry, 'r1');
+    expect(out[0].content).toBe(
+      'output\n[unresolved refs: tool9turn9, tool8turn8]'
+    );
+  });
+
+  it('ignores a non-string _refKey rather than poisoning the registry lookup', () => {
+    const registry = new ToolOutputReferenceRegistry();
+    registry.set('r1', 'tool0turn0', 'raw');
+    const tm = makeToolMessage({
+      content: 'output',
+      additional_kwargs: {
+        _refKey: { malformed: true } as unknown as string,
+      },
+    });
+    const out = annotateMessagesForLLM([tm], registry, 'r1');
+    expect(out[0].content).toBe('output');
+    expect((out[0] as ToolMessage).additional_kwargs._refKey).toBeUndefined();
+  });
+
+  it('skips a ToolMessage whose additional_kwargs is a primitive without throwing', () => {
+    /**
+     * The `in` operator throws `TypeError` on primitives, so without
+     * a runtime object guard, a hydrated ToolMessage carrying e.g.
+     * `additional_kwargs: 'not-an-object'` (from a buggy serializer)
+     * would crash `attemptInvoke` before the provider call. Verify
+     * we skip that message and process subsequent live-ref messages
+     * normally.
+     */
+    const registry = new ToolOutputReferenceRegistry();
+    registry.set('r1', 'tool0turn0', 'raw');
+
+    const malformed = new ToolMessage({
+      name: 'echo',
+      tool_call_id: 'mal',
+      status: 'success',
+      content: 'malformed-output',
+    });
+    /* Force a primitive past LangChain's typed setter via a cast. */
+    (malformed as unknown as { additional_kwargs: unknown }).additional_kwargs =
+      'not-an-object' as unknown;
+
+    const live = makeToolMessage({
+      content: 'live-output',
+      additional_kwargs: { _refKey: 'tool0turn0' },
+    });
+
+    const out = annotateMessagesForLLM([malformed, live], registry, 'r1');
+    /* Malformed message passes through unchanged; live ref still annotates. */
+    expect(out[0]).toBe(malformed);
+    expect(out[1].content).toBe('[ref: tool0turn0]\nlive-output');
+  });
+
+  it('treats stale _refKey but live unresolved as unresolved-only', () => {
+    const registry = new ToolOutputReferenceRegistry();
+    const tm = makeToolMessage({
+      content: 'output',
+      additional_kwargs: {
+        _refKey: 'tool0turn0',
+        _unresolvedRefs: ['tool9turn9'],
+      },
+    });
+    const out = annotateMessagesForLLM([tm], registry, 'r1');
+    expect(out[0].content).toBe('output\n[unresolved refs: tool9turn9]');
+  });
+});