@librechat/agents 3.4.5 → 3.4.6

package/src/specs/activity-label-observability.live.test.ts

@@ -0,0 +1,262 @@
+/**
+ * Live provider + Langfuse export verification for activity-label traces.
+ *
+ * Run with:
+ * RUN_ACTIVITY_LABEL_LANGFUSE_LIVE=1 OPENAI_API_KEY=... \
+ * LANGFUSE_PUBLIC_KEY=... LANGFUSE_SECRET_KEY=... LANGFUSE_BASE_URL=... \
+ * LANGFUSE_FORCE_FLUSH_ON_DISPOSE=true \
+ * npm test -- activity-label-observability.live.test.ts --runInBand
+ */
+import { config as dotenvConfig } from 'dotenv';
+dotenvConfig();
+
+import { HumanMessage } from '@langchain/core/messages';
+import { describe, expect, it, jest } from '@jest/globals';
+import { Providers } from '@/common';
+import { Run } from '@/run';
+
+type LangfuseMetadata = {
+  sourceRunId?: string;
+  responseId?: string;
+  activityIndex?: string | number;
+  phaseIndex?: string | number;
+  activityCount?: string | number;
+};
+
+type LangfuseObservation = {
+  id: string;
+  parentObservationId?: string | null;
+  type: string;
+  name: string;
+  model?: string | null;
+  usage?: { input?: number; output?: number; total?: number };
+};
+
+type LangfuseTrace = {
+  id: string;
+  name: string;
+  tags?: string[];
+  metadata?: LangfuseMetadata;
+  observations?: LangfuseObservation[];
+};
+
+type LangfuseTraceList = {
+  data: LangfuseTrace[];
+};
+
+const shouldRunLive =
+  process.env.RUN_ACTIVITY_LABEL_LANGFUSE_LIVE === '1' &&
+  (process.env.OPENAI_API_KEY ?? '') !== '' &&
+  (process.env.LANGFUSE_PUBLIC_KEY ?? '') !== '' &&
+  (process.env.LANGFUSE_SECRET_KEY ?? '') !== '' &&
+  (process.env.LANGFUSE_BASE_URL ?? '') !== '';
+
+const describeIfLive = shouldRunLive ? describe : describe.skip;
+
+function langfuseHeaders(): HeadersInit {
+  const credentials = Buffer.from(
+    `${process.env.LANGFUSE_PUBLIC_KEY}:${process.env.LANGFUSE_SECRET_KEY}`
+  ).toString('base64');
+  return { Authorization: `Basic ${credentials}` };
+}
+
+async function requestLangfuse(path: string): Promise<Response> {
+  const baseUrl = process.env.LANGFUSE_BASE_URL?.replace(/\/$/, '');
+  return fetch(`${baseUrl}${path}`, {
+    headers: langfuseHeaders(),
+  });
+}
+
+async function getJson<T>(path: string): Promise<T> {
+  const response = await requestLangfuse(path);
+  if (!response.ok) {
+    throw new Error(`Langfuse request failed: ${response.status}`);
+  }
+  return (await response.json()) as T;
+}
+
+async function getJsonIfPresent<T>(path: string): Promise<T | undefined> {
+  const response = await requestLangfuse(path);
+  if (response.status === 404) {
+    return undefined;
+  }
+  if (!response.ok) {
+    throw new Error(`Langfuse request failed: ${response.status}`);
+  }
+  return (await response.json()) as T;
+}
+
+async function findExportedTraces(
+  sourceRunId: string,
+  fromTimestamp: string
+): Promise<{ label: LangfuseTrace; phase: LangfuseTrace }> {
+  const query = new URLSearchParams({
+    limit: '100',
+    fromTimestamp,
+  });
+  for (let attempt = 0; attempt < 20; attempt += 1) {
+    const list = await getJson<LangfuseTraceList>(
+      `/api/public/traces?${query.toString()}`
+    );
+    const candidates = list.data.filter(
+      (trace) => trace.metadata?.sourceRunId === sourceRunId
+    );
+    const labelSummary = candidates.find(
+      (trace) => trace.tags?.includes('activity-label') === true
+    );
+    const phaseSummary = candidates.find(
+      (trace) => trace.tags?.includes('activity-phase') === true
+    );
+    if (labelSummary != null && phaseSummary != null) {
+      const [label, phase] = await Promise.all([
+        getJsonIfPresent<LangfuseTrace>(
+          `/api/public/traces/${labelSummary.id}`
+        ),
+        getJsonIfPresent<LangfuseTrace>(
+          `/api/public/traces/${phaseSummary.id}`
+        ),
+      ]);
+      if (label != null && phase != null) {
+        const labelReady =
+          label.observations?.some(
+            (observation) => observation.type === 'GENERATION'
+          ) === true;
+        const phaseRootReady =
+          phase.observations?.some(
+            (observation) =>
+              observation.parentObservationId == null &&
+              observation.type === 'CHAIN'
+          ) === true;
+        const phaseGenerationReady =
+          phase.observations?.some(
+            (observation) => observation.type === 'GENERATION'
+          ) === true;
+        if (labelReady && phaseRootReady && phaseGenerationReady) {
+          return { label, phase };
+        }
+      }
+    }
+    await new Promise((resolve) => setTimeout(resolve, 3000));
+  }
+  throw new Error(`Timed out waiting for activity traces from ${sourceRunId}`);
+}
+
+describeIfLive('activity label Langfuse export (live)', () => {
+  jest.setTimeout(120_000);
+
+  it('exports stable, correlated traces with correct observation types', async () => {
+    const startedAt = new Date(Date.now() - 5000).toISOString();
+    const sourceRunId = `activity-label-live-${Date.now()}`;
+    const sessionId = `${sourceRunId}-session`;
+    const model = process.env.ACTIVITY_LABEL_LIVE_MODEL ?? 'gpt-4.1-mini';
+    const run = await Run.create({
+      runId: sourceRunId,
+      graphConfig: {
+        type: 'standard',
+        agents: [
+          {
+            agentId: 'activity-label-live-agent',
+            name: 'Volatile Agent Display Name',
+            provider: Providers.OPENAI,
+            clientOptions: {
+              apiKey: process.env.OPENAI_API_KEY,
+              model,
+            },
+            tools: [],
+          },
+        ],
+      },
+    });
+    if (run.Graph != null) {
+      run.Graph.messages = [
+        new HumanMessage('Verify activity-label Langfuse trace semantics'),
+      ];
+    }
+    const chainOptions = {
+      configurable: {
+        thread_id: sessionId,
+        user_id: 'activity-label-live-user',
+        requestBody: { parentMessageId: 'activity-label-live-parent' },
+      },
+    };
+
+    await run.generateActivityLabel({
+      provider: Providers.OPENAI,
+      agentId: 'activity-label-live-agent',
+      clientOptions: {
+        apiKey: process.env.OPENAI_API_KEY,
+        model,
+      },
+      entries: [
+        {
+          toolName: 'inspect_runtime',
+          toolInput: { target: 'trace-shape' },
+          toolOutput: { status: 'verified' },
+          status: 'success',
+        },
+      ],
+      chainOptions,
+    });
+    await run.generateActivityPhaseLabel({
+      provider: Providers.OPENAI,
+      clientOptions: {
+        apiKey: process.env.OPENAI_API_KEY,
+        model,
+      },
+      activities: [
+        { label: 'Inspected the activity-label trace shape' },
+        { label: 'Verified stable correlation metadata' },
+      ],
+      sourceRunId,
+      responseId: sourceRunId,
+      phaseIndex: 0,
+      chainOptions,
+    });
+
+    const { label, phase } = await findExportedTraces(sourceRunId, startedAt);
+    expect(label).toMatchObject({
+      name: 'LibreChat Activity Label',
+      metadata: {
+        sourceRunId,
+        responseId: sourceRunId,
+        activityIndex: 0,
+      },
+    });
+    expect(phase).toMatchObject({
+      name: 'LibreChat Activity Phase',
+      metadata: {
+        sourceRunId,
+        responseId: sourceRunId,
+        phaseIndex: 0,
+        activityCount: 2,
+      },
+    });
+
+    const labelGeneration = label.observations?.find(
+      (observation) => observation.type === 'GENERATION'
+    );
+    expect(labelGeneration).toMatchObject({
+      parentObservationId: null,
+      name: 'llm',
+      model: expect.stringContaining(model),
+    });
+    expect(labelGeneration?.usage?.total).toBeGreaterThan(0);
+
+    const phaseRoot = phase.observations?.find(
+      (observation) => observation.parentObservationId == null
+    );
+    expect(phaseRoot).toMatchObject({
+      type: 'CHAIN',
+      name: 'summarize-activity-phase',
+    });
+    const phaseGeneration = phase.observations?.find(
+      (observation) => observation.type === 'GENERATION'
+    );
+    expect(phaseGeneration).toMatchObject({
+      parentObservationId: phaseRoot?.id,
+      name: 'llm',
+      model: expect.stringContaining(model),
+    });
+    expect(phaseGeneration?.usage?.total).toBeGreaterThan(0);
+  });
+});

package/src/specs/activity-label-observability.test.ts

@@ -0,0 +1,167 @@
+import { CallbackHandler } from '@langfuse/langchain';
+import { propagateAttributes } from '@langfuse/tracing';
+import { AIMessage, HumanMessage } from '@langchain/core/messages';
+import { Providers } from '@/common';
+import { Run } from '@/run';
+
+const invoke = jest.fn();
+
+jest.mock('@/llm/init', () => ({
+  initializeModel: jest.fn(() => ({ invoke })),
+}));
+
+jest.mock('@langfuse/langchain', () => ({
+  CallbackHandler: jest.fn().mockImplementation(() => ({})),
+}));
+
+jest.mock('@langfuse/tracing', () => ({
+  ...jest.requireActual('@langfuse/tracing'),
+  propagateAttributes: jest.fn((_params, action: () => unknown) => action()),
+}));
+
+const MockedCallbackHandler = CallbackHandler as jest.MockedClass<
+  typeof CallbackHandler
+>;
+const MockedPropagateAttributes = propagateAttributes as jest.MockedFunction<
+  typeof propagateAttributes
+>;
+
+async function createRun(): Promise<Run<never>> {
+  const run = await Run.create({
+    runId: 'response-1',
+    graphConfig: {
+      type: 'standard',
+      agents: [
+        {
+          agentId: 'agent-1',
+          name: 'Changing Model Name',
+          provider: Providers.OPENAI,
+          clientOptions: { model: 'gpt-4.1-mini' },
+          tools: [],
+        },
+      ],
+    },
+  });
+  if (run.Graph != null) {
+    run.Graph.messages = [new HumanMessage('Why did the run fail?')];
+  }
+  return run;
+}
+
+describe('activity label observability', () => {
+  const originalEnv = process.env;
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+    invoke.mockResolvedValue(new AIMessage('Resolved the failing run'));
+    process.env = {
+      ...originalEnv,
+      LANGFUSE_SECRET_KEY: 'sk-test',
+      LANGFUSE_PUBLIC_KEY: 'pk-test',
+      LANGFUSE_BASE_URL: 'https://langfuse.test',
+    };
+  });
+
+  afterEach(() => {
+    process.env = originalEnv;
+  });
+
+  it('uses a stable trace name and source-run metadata for batch labels', async () => {
+    const run = await createRun();
+
+    await run.generateActivityLabel({
+      provider: Providers.OPENAI,
+      entries: [
+        {
+          toolName: 'inspect_status',
+          toolInput: { id: 'one' },
+          toolOutput: { fixed: true },
+          status: 'success',
+        },
+      ],
+      chainOptions: {
+        configurable: {
+          thread_id: 'thread-1',
+          user_id: 'user-1',
+          requestBody: { parentMessageId: 'parent-1' },
+        },
+      },
+    });
+
+    expect(MockedCallbackHandler).toHaveBeenCalledTimes(1);
+    expect(MockedCallbackHandler.mock.calls[0][0]).toMatchObject({
+      traceMetadata: {
+        messageId: 'activity-label-response-1',
+        parentMessageId: 'parent-1',
+        agentId: 'agent-1',
+        agentName: 'Changing Model Name',
+        sourceRunId: 'response-1',
+        responseId: 'response-1',
+        activityIndex: '0',
+      },
+      tags: ['librechat', 'activity-label'],
+    });
+    expect(MockedPropagateAttributes.mock.calls[0][0]).toMatchObject({
+      traceName: 'LibreChat Activity Label',
+      metadata: {
+        sourceRunId: 'response-1',
+        responseId: 'response-1',
+        activityIndex: '0',
+      },
+    });
+    expect(invoke.mock.calls[0][1]).toMatchObject({
+      runName: 'LibreChat Activity Label',
+      tags: ['librechat', 'activity-label'],
+      metadata: {
+        sourceRunId: 'response-1',
+        responseId: 'response-1',
+        activityIndex: 0,
+        parentMessageId: 'parent-1',
+        agentId: 'agent-1',
+        agentName: 'Changing Model Name',
+      },
+    });
+  });
+
+  it('uses a stable trace name for phase summaries', async () => {
+    const run = await createRun();
+
+    await run.generateActivityPhaseLabel({
+      provider: Providers.OPENAI,
+      activities: [
+        { label: 'Inspected session refresh behavior' },
+        { label: 'Fixed refresh token validation' },
+      ],
+      sourceRunId: 'response-1',
+      responseId: 'response-1',
+      phaseIndex: 0,
+      chainOptions: {
+        configurable: {
+          thread_id: 'thread-1',
+          user_id: 'user-1',
+          requestBody: { parentMessageId: 'parent-1' },
+        },
+      },
+    });
+
+    expect(MockedCallbackHandler).toHaveBeenCalledTimes(1);
+    expect(MockedCallbackHandler.mock.calls[0][0]).toMatchObject({
+      tags: ['librechat', 'activity-phase', 'agent-run-summary', 'agent'],
+      traceMetadata: {
+        sourceRunId: 'response-1',
+        responseId: 'response-1',
+        phaseIndex: '0',
+        activityCount: '2',
+      },
+    });
+    expect(MockedPropagateAttributes.mock.calls[0][0]).toMatchObject({
+      traceName: 'LibreChat Activity Phase',
+      metadata: {
+        sourceRunId: 'response-1',
+        responseId: 'response-1',
+        phaseIndex: '0',
+        activityCount: '2',
+      },
+    });
+  });
+});

package/src/specs/langfuse-trace-shaping.test.ts

@@ -486,6 +486,38 @@ describe('shapeLangfuseSpan', () => {
     expect(span.attributes[TRACE_OUTPUT]).toBe('Conversation title');
   });
 
+  it('marks activity-phase roots as chains even when tagged as agent work', () => {
+    const span = createSpan('summarize-activity-phase', {
+      [TRACE_TAGS]: JSON.stringify([
+        'librechat',
+        'activity-phase',
+        'agent-run-summary',
+        'agent',
+      ]),
+      [INPUT]: 'What changed?',
+      [OUTPUT]: 'Reconciled the implementation and verified the fix',
+    });
+
+    shapeLangfuseSpan(span);
+
+    expect(span.attributes[OBSERVATION_TYPE]).toBe('chain');
+    expect(span.attributes[TRACE_INPUT]).toBe('What changed?');
+    expect(span.attributes[TRACE_OUTPUT]).toBe(
+      'Reconciled the implementation and verified the fix'
+    );
+  });
+
+  it('does not treat a user-supplied activity-phase tag as an operation type', () => {
+    const span = createSpan('LibreChat Agent', {
+      [TRACE_TAGS]: JSON.stringify(['librechat', 'agent', 'activity-phase']),
+      [INPUT]: 'Run an ordinary agent',
+    });
+
+    shapeLangfuseSpan(span);
+
+    expect(span.attributes[OBSERVATION_TYPE]).toBe('agent');
+  });
+
   it('does not classify untagged root spans as agents', () => {
     const span = createSpan('Custom root', { [INPUT]: 'input' });