@librechat/agents 3.4.0 → 3.4.2

package/src/specs/graph-subagent.live.test.ts

@@ -0,0 +1,242 @@
+import { config as dotenvConfig } from 'dotenv';
+
+dotenvConfig(
+  process.env.LIVE_ENV_PATH == null
+    ? undefined
+    : { path: process.env.LIVE_ENV_PATH }
+);
+dotenvConfig();
+
+import { describe, expect, it, jest } from '@jest/globals';
+import type { RunnableConfig } from '@langchain/core/runnables';
+import type * as t from '@/types';
+import { Constants, GraphEvents, Providers } from '@/common';
+import { StandardGraph } from '@/graphs/Graph';
+import { Run } from '@/run';
+
+jest.setTimeout(180_000);
+
+const liveEnabled = process.env.RUN_GRAPH_SUBAGENT_LIVE_TESTS === '1';
+const requestedProvider = process.env.GRAPH_SUBAGENT_LIVE_PROVIDER;
+const describeIfLive = liveEnabled ? describe : describe.skip;
+
+type LiveProviderCase = {
+  label: string;
+  provider: Providers;
+  model: string;
+  apiKey?: string;
+};
+
+const LIVE_PROVIDER_CASES: readonly LiveProviderCase[] = [
+  {
+    label: 'Anthropic Haiku',
+    provider: Providers.ANTHROPIC,
+    model:
+      process.env.ANTHROPIC_GRAPH_SUBAGENT_LIVE_MODEL ?? 'claude-haiku-4-5',
+    apiKey: process.env.ANTHROPIC_API_KEY,
+  },
+  {
+    label: 'OpenAI GPT-4.1 mini',
+    provider: Providers.OPENAI,
+    model: process.env.OPENAI_GRAPH_SUBAGENT_LIVE_MODEL ?? 'gpt-4.1-mini',
+    apiKey: process.env.OPENAI_API_KEY,
+  },
+];
+
+const explicitlyRequestedCase =
+  requestedProvider == null || requestedProvider === ''
+    ? undefined
+    : LIVE_PROVIDER_CASES.find(
+      ({ provider }) =>
+        provider.toLowerCase() === requestedProvider.toLowerCase()
+    );
+
+if (
+  liveEnabled &&
+  requestedProvider != null &&
+  requestedProvider !== '' &&
+  explicitlyRequestedCase == null
+) {
+  throw new Error(
+    `Unknown GRAPH_SUBAGENT_LIVE_PROVIDER "${requestedProvider}".`
+  );
+}
+if (
+  liveEnabled &&
+  explicitlyRequestedCase != null &&
+  (explicitlyRequestedCase.apiKey == null ||
+    explicitlyRequestedCase.apiKey === '')
+) {
+  throw new Error(
+    `Missing credentials for GRAPH_SUBAGENT_LIVE_PROVIDER "${requestedProvider}".`
+  );
+}
+if (
+  liveEnabled &&
+  explicitlyRequestedCase == null &&
+  !LIVE_PROVIDER_CASES.some(({ apiKey }) => apiKey != null && apiKey !== '')
+) {
+  throw new Error(
+    'No graph-subagent live-provider credentials are configured.'
+  );
+}
+
+function createLiveAgent(
+  testCase: LiveProviderCase,
+  agentId: string,
+  marker: string
+): t.AgentInputs {
+  return {
+    agentId,
+    provider: testCase.provider,
+    clientOptions: {
+      modelName: testCase.model,
+      apiKey: testCase.apiKey,
+      temperature: 0,
+      maxTokens: 64,
+      streaming: true,
+      streamUsage: true,
+    },
+    instructions: `You are the ${agentId} validation stage. When asked to proceed, reply with exactly ${marker} and no other text.`,
+    maxContextTokens: 8_000,
+  };
+}
+
+function getGraphSubagentTool(run: Run<t.IState>): t.GenericTool {
+  const tools = (run.Graph as StandardGraph).agentContexts.get('parent')
+    ?.graphTools as t.GenericTool[] | undefined;
+  const subagentTool = tools?.find(
+    (tool) => 'name' in tool && tool.name === Constants.SUBAGENT
+  );
+  if (subagentTool == null) {
+    throw new Error('Expected graph subagent tool');
+  }
+  return subagentTool;
+}
+
+describeIfLive('Graph subagent live providers', () => {
+  for (const testCase of LIVE_PROVIDER_CASES) {
+    const providerSelected =
+      requestedProvider == null ||
+      requestedProvider === '' ||
+      requestedProvider.toLowerCase() === testCase.provider.toLowerCase();
+    const itIfAvailable =
+      providerSelected && testCase.apiKey != null && testCase.apiKey !== ''
+        ? it
+        : it.skip;
+
+    itIfAvailable(
+      `${testCase.label} runs a bounded team and returns only its result member`,
+      async () => {
+        const usageEvents: t.SubagentUsageEvent[] = [];
+        const updateEvents: t.SubagentUpdateEvent[] = [];
+        const entry = createLiveAgent(testCase, 'entry', 'ENTRY_MARKER');
+        const worker = createLiveAgent(testCase, 'worker', 'WORKER_MARKER');
+        const resultAgent = createLiveAgent(testCase, 'result', 'FINAL_MARKER');
+        const parent: t.AgentInputs = {
+          ...createLiveAgent(testCase, 'parent', 'PARENT_MARKER'),
+          maxSubagentDepth: 1,
+          subagentConfigs: [
+            {
+              kind: 'graph',
+              type: 'live-team',
+              name: 'Live Team',
+              description: 'Runs a three-member live-provider chain.',
+              maxTurns: 2,
+              agents: [entry, worker, resultAgent],
+              edges: [
+                {
+                  from: 'entry',
+                  to: 'worker',
+                  edgeType: 'direct',
+                  prompt: 'Proceed with the worker validation stage.',
+                },
+                {
+                  from: 'worker',
+                  to: 'result',
+                  edgeType: 'direct',
+                  prompt: 'Proceed with the result validation stage.',
+                },
+              ],
+              entryAgentId: 'entry',
+              resultAgentId: 'result',
+            },
+          ],
+        };
+        const rootRunId = `graph-subagent-live-${testCase.provider}-${Date.now()}`;
+        const run = await Run.create<t.IState>({
+          runId: rootRunId,
+          graphConfig: { type: 'standard', agents: [parent] },
+          returnContent: true,
+          skipCleanup: true,
+          subagentUsageSink: (event) => {
+            usageEvents.push(event);
+          },
+          customHandlers: {
+            [GraphEvents.ON_SUBAGENT_UPDATE]: {
+              handle: (_event, data): void => {
+                updateEvents.push(data as t.SubagentUpdateEvent);
+              },
+            },
+          },
+        });
+        const invokeConfig: RunnableConfig = {
+          configurable: {
+            thread_id: `graph-subagent-live-${testCase.provider}`,
+          },
+        };
+
+        const output = await getGraphSubagentTool(run).invoke(
+          {
+            description: 'Begin the entry validation stage.',
+            subagent_type: 'live-team',
+          },
+          invokeConfig
+        );
+        const outputText = String(output);
+
+        expect(outputText).toContain('FINAL_MARKER');
+        expect(outputText).not.toContain('ENTRY_MARKER');
+        expect(outputText).not.toContain('WORKER_MARKER');
+        expect(usageEvents.map((event) => event.memberAgentId).sort()).toEqual([
+          'entry',
+          'result',
+          'worker',
+        ]);
+        for (const event of usageEvents) {
+          expect(event.runId).toBe(rootRunId);
+          expect(event.parentRunId).toBe(rootRunId);
+          expect(event.depth).toBe(1);
+          expect(event.ancestry?.map((entry) => entry.subagentType)).toEqual([
+            'live-team',
+          ]);
+          expect(event.subagentKind).toBe('graph');
+          expect(event.provider).toBe(testCase.provider);
+          expect(event.model).toBeTruthy();
+          expect(event.usage.total_tokens).toBeGreaterThan(0);
+        }
+        expect(
+          updateEvents.filter((event) => event.phase === 'start')
+        ).toHaveLength(1);
+        expect(
+          updateEvents.filter((event) => event.phase === 'stop')
+        ).toHaveLength(1);
+        for (const event of updateEvents) {
+          expect(event.runId).toBe(rootRunId);
+          expect(event.parentRunId).toBe(rootRunId);
+          expect(event.depth).toBe(1);
+          expect(event.ancestry?.map((entry) => entry.subagentType)).toEqual([
+            'live-team',
+          ]);
+        }
+        expect(
+          new Set(
+            updateEvents
+              .map((event) => event.memberAgentId)
+              .filter((agentId): agentId is string => agentId != null)
+          )
+        ).toEqual(new Set(['entry', 'worker', 'result']));
+      }
+    );
+  }
+});

package/src/specs/graph-subagent.test.ts

@@ -0,0 +1,322 @@
+import { z } from 'zod';
+import { tool } from '@langchain/core/tools';
+import { AIMessageChunk } from '@langchain/core/messages';
+import { ChatGenerationChunk } from '@langchain/core/outputs';
+import { FakeListChatModel } from '@langchain/core/utils/testing';
+import type { RunnableConfig } from '@langchain/core/runnables';
+import type { GraphSubagentConfig } from '@/types';
+import type * as t from '@/types';
+import { createFakeStreamingLLM } from '@/llm/fake';
+import { Constants, Providers } from '@/common';
+import { StandardGraph } from '@/graphs/Graph';
+import { Run } from '@/run';
+
+const invokeConfig: RunnableConfig = {
+  configurable: { thread_id: 'graph-subagent-test' },
+};
+
+const usage = {
+  input_tokens: 5,
+  output_tokens: 3,
+  total_tokens: 8,
+};
+
+class UsageFakeChatModel extends FakeListChatModel {
+  constructor() {
+    super({ responses: ['member response'] });
+  }
+
+  override async *_streamResponseChunks(
+    ...args: Parameters<FakeListChatModel['_streamResponseChunks']>
+  ): ReturnType<FakeListChatModel['_streamResponseChunks']> {
+    yield* super._streamResponseChunks(...args);
+    yield new ChatGenerationChunk({
+      text: '',
+      message: new AIMessageChunk({
+        content: '',
+        usage_metadata: { ...usage },
+      }),
+    });
+  }
+}
+
+class LoopingFakeChatModel extends FakeListChatModel {
+  private invocationCount = 0;
+
+  constructor() {
+    super({ responses: ['unused'] });
+  }
+
+  override async *_streamResponseChunks(
+    ..._args: Parameters<FakeListChatModel['_streamResponseChunks']>
+  ): ReturnType<FakeListChatModel['_streamResponseChunks']> {
+    this.invocationCount++;
+    if (this.invocationCount <= 2) {
+      yield new ChatGenerationChunk({
+        text: '',
+        message: new AIMessageChunk({
+          content: '',
+          tool_call_chunks: [
+            {
+              name: 'loop_tool',
+              args: '{}',
+              id: `loop-${this.invocationCount}`,
+              index: 0,
+              type: 'tool_call_chunk',
+            },
+          ],
+        }),
+      });
+      return;
+    }
+    yield new ChatGenerationChunk({
+      text: 'escaped member turn limit',
+      message: new AIMessageChunk('escaped member turn limit'),
+    });
+  }
+}
+
+const makeAgent = (agentId: string): t.AgentInputs => ({
+  agentId,
+  provider: Providers.OPENAI,
+  clientOptions: { modelName: `${agentId}-model`, apiKey: 'test-key' },
+  instructions: `You are ${agentId}.`,
+});
+
+const makeGraphConfig = (): GraphSubagentConfig => ({
+  kind: 'graph',
+  type: 'research-team',
+  name: 'Research Team',
+  description: 'Coordinates parallel research and synthesis.',
+  entryAgentId: 'coordinator',
+  resultAgentId: 'synthesizer',
+  agents: [
+    makeAgent('coordinator'),
+    makeAgent('left'),
+    makeAgent('right'),
+    makeAgent('synthesizer'),
+  ],
+  edges: [
+    {
+      from: 'coordinator',
+      to: ['left', 'right'],
+      edgeType: 'direct',
+    },
+    {
+      from: ['left', 'right'],
+      to: 'synthesizer',
+      edgeType: 'direct',
+    },
+  ],
+});
+
+const createRun = async (
+  graphConfig: GraphSubagentConfig,
+  overrides: Partial<t.RunConfig> = {}
+): Promise<Run<t.IState>> =>
+  Run.create<t.IState>({
+    runId: `graph-subagent-${Date.now()}-${Math.random()}`,
+    graphConfig: {
+      type: 'standard',
+      agents: [
+        {
+          ...makeAgent('parent'),
+          maxSubagentDepth: 2,
+          subagentConfigs: [graphConfig],
+        },
+      ],
+    },
+    returnContent: true,
+    skipCleanup: true,
+    ...overrides,
+  });
+
+const getGraphSubagentTool = (run: Run<t.IState>): t.GenericTool => {
+  const tools = (run.Graph as StandardGraph).agentContexts.get('parent')
+    ?.graphTools as t.GenericTool[] | undefined;
+  const tool = tools?.find(
+    (candidate) => 'name' in candidate && candidate.name === Constants.SUBAGENT
+  );
+  if (tool == null) {
+    throw new Error('Expected graph subagent tool');
+  }
+  return tool;
+};
+
+describe('Graph subagent integration', () => {
+  it('runs a convergent direct DAG and returns only the designated result', async () => {
+    const graphConfig = { ...makeGraphConfig(), maxTurns: 1 };
+    const run = await createRun(graphConfig);
+    (run.Graph as StandardGraph).setSubagentModelOverride(
+      createFakeStreamingLLM({
+        responses: [
+          'coordinator plan',
+          'left research',
+          'right research',
+          'synthesized answer',
+        ],
+      })
+    );
+
+    const result = await getGraphSubagentTool(run).invoke(
+      {
+        description: 'Research and synthesize the question.',
+        subagent_type: 'research-team',
+      },
+      invokeConfig
+    );
+
+    expect(result).toBe('synthesized answer');
+  });
+
+  it('does not fall back to worker text when the result member is textless', async () => {
+    const graphConfig: GraphSubagentConfig = {
+      ...makeGraphConfig(),
+      agents: [makeAgent('worker'), makeAgent('result')],
+      edges: [{ from: 'worker', to: 'result', edgeType: 'direct' }],
+      entryAgentId: 'worker',
+      resultAgentId: 'result',
+    };
+    const run = await createRun(graphConfig);
+    (run.Graph as StandardGraph).setSubagentModelOverride(
+      createFakeStreamingLLM({ responses: ['worker text', ''] })
+    );
+
+    const result = await getGraphSubagentTool(run).invoke(
+      {
+        description: 'Complete the two-step task.',
+        subagent_type: 'research-team',
+      },
+      invokeConfig
+    );
+
+    expect(result).toBe('Task completed');
+  });
+
+  it('keeps each member turn budget independent from the outer graph budget', async () => {
+    const loopTool = tool(async (): Promise<string> => 'continue', {
+      name: 'loop_tool',
+      description: 'Continue the member loop.',
+      schema: z.object({}),
+    });
+    const graphConfig: GraphSubagentConfig = {
+      ...makeGraphConfig(),
+      maxTurns: 1,
+      agents: [
+        { ...makeAgent('entry'), graphTools: [loopTool] },
+        makeAgent('result'),
+      ],
+      edges: [{ from: 'entry', to: 'result', edgeType: 'direct' }],
+      entryAgentId: 'entry',
+      resultAgentId: 'result',
+    };
+    const run = await createRun(graphConfig);
+    (run.Graph as StandardGraph).setSubagentModelOverride(
+      new LoopingFakeChatModel()
+    );
+
+    const output = await getGraphSubagentTool(run).invoke(
+      {
+        description: 'Try to exceed the member turn budget.',
+        subagent_type: 'research-team',
+      },
+      invokeConfig
+    );
+
+    expect(output).toMatch(/Subagent error: Recursion limit of 3 reached/);
+  });
+
+  it('fails closed when human-in-the-loop is enabled', async () => {
+    const run = await createRun(makeGraphConfig(), {
+      humanInTheLoop: { enabled: true },
+    });
+
+    const result = await getGraphSubagentTool(run).invoke(
+      {
+        description: 'Do not start this graph.',
+        subagent_type: 'research-team',
+      },
+      invokeConfig
+    );
+
+    expect(result).toBe(
+      'Error: Human-in-the-loop execution is not yet supported for graph subagents.'
+    );
+  });
+
+  it('attributes usage to each graph member', async () => {
+    const graphConfig: GraphSubagentConfig = {
+      ...makeGraphConfig(),
+      agents: [makeAgent('entry'), makeAgent('worker'), makeAgent('result')],
+      edges: [
+        { from: 'entry', to: 'worker', edgeType: 'direct' },
+        { from: 'worker', to: 'result', edgeType: 'direct' },
+      ],
+      entryAgentId: 'entry',
+      resultAgentId: 'result',
+    };
+    const usageEvents: t.SubagentUsageEvent[] = [];
+    const run = await createRun(graphConfig, {
+      subagentUsageSink: (event) => {
+        usageEvents.push(event);
+      },
+    });
+    (run.Graph as StandardGraph).setSubagentModelOverride(
+      new UsageFakeChatModel()
+    );
+
+    await getGraphSubagentTool(run).invoke(
+      {
+        description: 'Complete the measured graph.',
+        subagent_type: 'research-team',
+      },
+      invokeConfig
+    );
+
+    expect(usageEvents).toHaveLength(3);
+    expect(
+      usageEvents.map((event) => ({
+        memberAgentId: event.memberAgentId,
+        model: event.model,
+        provider: event.provider,
+        subagentKind: event.subagentKind,
+      }))
+    ).toEqual([
+      {
+        memberAgentId: 'entry',
+        model: 'entry-model',
+        provider: Providers.OPENAI,
+        subagentKind: 'graph',
+      },
+      {
+        memberAgentId: 'worker',
+        model: 'worker-model',
+        provider: Providers.OPENAI,
+        subagentKind: 'graph',
+      },
+      {
+        memberAgentId: 'result',
+        model: 'result-model',
+        provider: Providers.OPENAI,
+        subagentKind: 'graph',
+      },
+    ]);
+    expect(
+      new Set(usageEvents.map((event) => event.subagentAgentId)).size
+    ).toBe(1);
+    const rootRunId = (run.Graph as StandardGraph).runId;
+    for (const event of usageEvents) {
+      expect(event).toMatchObject({
+        runId: rootRunId,
+        parentRunId: rootRunId,
+        depth: 1,
+      });
+      expect(event.ancestry).toHaveLength(1);
+      expect(event.ancestry?.[0]).toMatchObject({
+        subagentType: 'research-team',
+        subagentKind: 'graph',
+        parentRunId: rootRunId,
+      });
+    }
+  });
+});

package/src/specs/langfuse-routing.integration.test.ts

@@ -319,6 +319,50 @@ function createAgent(tenantId: string): t.AgentInputs {
   };
 }
 
+function createGraphSubagentParent(tenantId: string): t.AgentInputs {
+  const makeMember = (agentId: string): t.AgentInputs => ({
+    agentId,
+    name: `${agentId} ${tenantId}`,
+    provider: Providers.OPENAI,
+    clientOptions: { modelName: 'gpt-4o-mini', apiKey: 'test-key' },
+    instructions: `Complete the ${agentId} stage.`,
+    maxContextTokens: 8000,
+  });
+  return {
+    ...makeMember('parent'),
+    maxSubagentDepth: 1,
+    subagentConfigs: [
+      {
+        kind: 'graph',
+        type: 'research-team',
+        name: 'Research Team',
+        description: 'Runs a bounded research chain.',
+        agents: [
+          makeMember('entry'),
+          makeMember('worker'),
+          makeMember('result'),
+        ],
+        edges: [
+          {
+            from: 'entry',
+            to: 'worker',
+            edgeType: 'direct',
+            prompt: 'Proceed to worker.',
+          },
+          {
+            from: 'worker',
+            to: 'result',
+            edgeType: 'direct',
+            prompt: 'Proceed to result.',
+          },
+        ],
+        entryAgentId: 'entry',
+        resultAgentId: 'result',
+      },
+    ],
+  };
+}
+
 async function runTenantFlow(tenantId: string): Promise<void> {
   const runId = `routing-${tenantId}`;
   const run = await Run.create<t.IState>({
@@ -382,6 +426,49 @@ async function runTenantFlow(tenantId: string): Promise<void> {
   });
 }
 
+async function runGraphSubagentTenantFlow(tenantId: string): Promise<void> {
+  const runId = `routing-graph-${tenantId}`;
+  const run = await Run.create<t.IState>({
+    runId,
+    graphConfig: {
+      type: 'standard',
+      agents: [createGraphSubagentParent(tenantId)],
+    },
+    langfuse: tenantLangfuse(tenantId),
+    returnContent: true,
+    skipCleanup: true,
+  });
+  run.Graph?.overrideTestModel(
+    [
+      `Delegating graph work for ${tenantId}.`,
+      `Graph work complete for ${tenantId}.`,
+    ],
+    1,
+    [
+      {
+        id: `call_graph_subagent_${tenantId}`,
+        name: Constants.SUBAGENT,
+        args: {
+          description: `Run the research chain for ${tenantId}.`,
+          subagent_type: 'research-team',
+        },
+        type: 'tool_call',
+      },
+    ]
+  );
+
+  await run.processStream(
+    { messages: [new HumanMessage(`Run graph work for ${tenantId}`)] },
+    {
+      ...callerConfig,
+      configurable: {
+        thread_id: `graph-thread-${tenantId}`,
+        user_id: `graph-user-${tenantId}`,
+      },
+    }
+  );
+}
+
 const compactingTokenCounter: t.TokenCounter = (message) => {
   if (message._getType() === 'system') {
     return 1;
@@ -454,7 +541,7 @@ describe('Langfuse per-run routing integration', () => {
       .spyOn(providers, 'getChatModelClass')
       .mockImplementation(((provider: Providers) => {
         if (provider === Providers.OPENAI) {
-          return class extends FakeListChatModel {
+          return class RoutingProviderFakeChatModel extends FakeListChatModel {
             // eslint-disable-next-line @typescript-eslint/no-explicit-any
             constructor(_options: any) {
               super({ responses: ['provider response'] });
@@ -531,6 +618,38 @@ describe('Langfuse per-run routing integration', () => {
     }
   });
 
+  it('keeps graph-subagent member spans in the owning tenant trace', async () => {
+    await Promise.all([
+      runGraphSubagentTenantFlow('graph-tenant-a'),
+      runGraphSubagentTenantFlow('graph-tenant-b'),
+    ]);
+
+    for (const tenantId of ['graph-tenant-a', 'graph-tenant-b']) {
+      const otherTenantId =
+        tenantId === 'graph-tenant-a' ? 'graph-tenant-b' : 'graph-tenant-a';
+      const starts = startsForTenant(tenantId);
+      const traceId = traceIdFromSeed(`routing-graph-${tenantId}`);
+
+      expectTenantCredentials(starts, tenantId);
+      expectOnlyTraceIds(starts, [traceId]);
+      expectNamedSpansUseTraceId({
+        starts,
+        traceId,
+        names: [
+          `LibreChat Agent: parent ${tenantId}`,
+          'FakeChatModel',
+          'subagent',
+        ],
+      });
+      expect(
+        starts.filter(
+          (record) => record.name === 'RoutingProviderFakeChatModel'
+        )
+      ).toHaveLength(3);
+      expectNoCrossTenantTrace({ tenantId, otherTenantId, traceId });
+    }
+  });
+
   it('routes parallel summarization spans to each run config', async () => {
     await Promise.all([
       runTenantSummarizationFlow('tenant-a'),